From 68d956dac226bdc8625cd772ca9c5a7149c56b4a Mon Sep 17 00:00:00 2001 From: Steve Gerbino Date: Fri, 9 Oct 2026 00:58:37 +0200 Subject: [PATCH 1/8] fix: hold the scheduler lock while select snapshots its descriptors run_task is entered without the scheduler lock when the caller has just handed queued work to a peer thread, and the snapshot then walked registered_descs_ while deregister_descriptor() erased from it under that lock. Take the lock for the snapshot when the caller released it. --- .../boost/corosio/native/detail/select/select_scheduler.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/boost/corosio/native/detail/select/select_scheduler.hpp b/include/boost/corosio/native/detail/select/select_scheduler.hpp index 12a4f073e..89b76b810 100644 --- a/include/boost/corosio/native/detail/select/select_scheduler.hpp +++ b/include/boost/corosio/native/detail/select/select_scheduler.hpp @@ -349,6 +349,11 @@ select_scheduler::run_task(lock_type& lock, context_type& ctx, long timeout_us) fd_entry snapshot[FD_SETSIZE]; int snapshot_count = 0; + // The caller may have released the lock to hand queued work to a + // peer, and deregister_descriptor() erases from this map under it. + if (!lock.owns_lock()) + lock.lock(); + for (auto& [fd, desc] : registered_descs_) { if (snapshot_count < FD_SETSIZE) From ca7a675ee16b00d8407d337a75ff9e04e3a2c11f Mon Sep 17 00:00:00 2001 From: Steve Gerbino Date: Wed, 7 Oct 2026 23:37:14 +0200 Subject: [PATCH 2/8] feat: recycle I/O objects and operations through pools Opening and closing sockets, acceptors, timers, files, resolvers and signal sets allocated and freed an implementation each time, and accepting, random-access file I/O and launching a tcp_server connection allocated per operation. Implementations now come from per-type object pools and are held through counted object_ref handles, so an operation in flight keeps its object alive. Accept nodes, random-access file operations and connection launches are recycled, multishot accept hands its nodes to the kernel, and the IOCP wait reactor reuses its drain buffers, so steady-state I/O no longer allocates. Tests poison recycled implementations, churn them, and check that steady-state operations do not allocate. The fault harness chains its shadows into the sanitizer interceptors. --- include/boost/corosio/detail/intrusive.hpp | 38 +- include/boost/corosio/detail/object_pool.hpp | 319 ++++ include/boost/corosio/detail/object_ref.hpp | 152 ++ include/boost/corosio/detail/platform.hpp | 14 + include/boost/corosio/detail/timer.hpp | 31 +- .../boost/corosio/detail/timer_service.hpp | 108 +- include/boost/corosio/io/io_object.hpp | 59 +- include/boost/corosio/ip_address.hpp | 6 +- .../boost/corosio/native/detail/coro_op.hpp | 3 +- .../native/detail/coro_op_complete.hpp | 10 +- .../native/detail/epoll/epoll_scheduler.hpp | 42 +- .../native/detail/iocp/win_file_service.hpp | 66 +- .../detail/iocp/win_local_stream_acceptor.hpp | 91 +- .../win_local_stream_acceptor_service.hpp | 239 +-- .../detail/iocp/win_local_stream_service.hpp | 399 ++--- .../detail/iocp/win_local_stream_socket.hpp | 130 +- .../detail/iocp/win_object_handle_service.hpp | 99 +- .../detail/iocp/win_overlapped_handle.hpp | 177 +- .../detail/iocp/win_random_access_file.hpp | 29 +- .../iocp/win_random_access_file_service.hpp | 63 +- .../iocp/win_random_access_handle_service.hpp | 70 +- .../native/detail/iocp/win_resolver.hpp | 40 +- .../detail/iocp/win_resolver_service.hpp | 127 +- .../corosio/native/detail/iocp/win_signal.hpp | 21 + .../native/detail/iocp/win_signals.hpp | 99 +- .../native/detail/iocp/win_stream_file.hpp | 29 +- .../detail/iocp/win_stream_handle_service.hpp | 70 +- .../native/detail/iocp/win_tcp_acceptor.hpp | 101 +- .../detail/iocp/win_tcp_acceptor_service.hpp | 1570 ++--------------- .../native/detail/iocp/win_tcp_service.hpp | 1090 +++++++++++- .../native/detail/iocp/win_tcp_socket.hpp | 127 +- .../native/detail/iocp/win_udp_service.hpp | 395 ++--- .../native/detail/iocp/win_udp_socket.hpp | 149 +- .../native/detail/iocp/win_wait_reactor.hpp | 11 +- .../native/detail/kqueue/kqueue_scheduler.hpp | 55 +- .../native/detail/native_socket_base.hpp | 11 +- .../detail/posix/posix_random_access_file.hpp | 102 +- .../posix_random_access_file_service.hpp | 98 +- .../native/detail/posix/posix_resolver.hpp | 40 +- .../detail/posix/posix_resolver_service.hpp | 78 +- .../native/detail/posix/posix_signal.hpp | 25 + .../detail/posix/posix_signal_service.hpp | 81 +- .../native/detail/posix/posix_stream_file.hpp | 38 +- .../posix/posix_stream_file_service.hpp | 68 +- .../detail/reactor/reactor_acceptor.hpp | 85 +- .../reactor/reactor_acceptor_service.hpp | 29 +- .../native/detail/reactor/reactor_backend.hpp | 39 +- .../detail/reactor/reactor_basic_socket.hpp | 51 +- .../reactor/reactor_datagram_socket.hpp | 62 +- .../detail/reactor/reactor_descriptor.hpp | 63 +- .../reactor/reactor_descriptor_service.hpp | 36 +- .../reactor/reactor_descriptor_state.hpp | 46 +- .../native/detail/reactor/reactor_io_core.hpp | 219 ++- .../native/detail/reactor/reactor_op.hpp | 6 +- .../native/detail/reactor/reactor_op_base.hpp | 9 +- .../detail/reactor/reactor_scheduler.hpp | 107 ++ .../detail/reactor/reactor_service_state.hpp | 19 +- .../detail/reactor/reactor_socket_finals.hpp | 27 + .../detail/reactor/reactor_socket_service.hpp | 51 +- .../detail/reactor/reactor_stream_socket.hpp | 61 +- .../native/detail/select/select_scheduler.hpp | 20 +- .../native/detail/speculative_state.hpp | 14 + .../detail/uring/uring_acceptor_ops.hpp | 119 +- .../native/detail/uring/uring_descriptor.hpp | 41 +- .../detail/uring/uring_descriptor_service.hpp | 6 + .../native/detail/uring/uring_dgram_ops.hpp | 48 +- .../native/detail/uring/uring_file_ops.hpp | 8 +- .../detail/uring/uring_file_service_base.hpp | 79 +- .../detail/uring/uring_multishot_acceptor.hpp | 898 +++++----- .../corosio/native/detail/uring/uring_op.hpp | 20 +- .../detail/uring/uring_random_access_file.hpp | 170 +- .../native/detail/uring/uring_scheduler.hpp | 352 +--- .../native/detail/uring/uring_socket_ops.hpp | 228 +-- .../uring/uring_socket_service_base.hpp | 84 +- .../native/detail/uring/uring_stream_file.hpp | 42 +- .../native/detail/uring/uring_types.hpp | 489 +++-- include/boost/corosio/tcp_server.hpp | 32 +- src/corosio/src/tcp_socket.cpp | 18 +- src/corosio/src/timer.cpp | 27 +- test/unit/Jamfile | 9 +- test/unit/alloc_counter.hpp | 85 + test/unit/continuation_post.cpp | 179 +- test/unit/fault/fault_posix.cpp | 79 + test/unit/fault/fault_uring.cpp | 45 + test/unit/fault/iocp_faults.cpp | 6 +- test/unit/fault/uring_faults.cpp | 124 +- test/unit/mt_reactor.cpp | 214 +++ .../unit/native/uring/descriptor_continue.cpp | 49 +- test/unit/native/uring/multishot_acceptor.cpp | 217 ++- test/unit/object_pool.cpp | 288 +++ test/unit/object_ref.cpp | 96 + test/unit/object_reuse.cpp | 859 +++++++++ test/unit/resolver.cpp | 4 +- test/unit/signal_set.cpp | 27 + test/unit/tcp_acceptor.cpp | 109 +- test/unit/zero_alloc.cpp | 529 ++++++ 96 files changed, 8255 insertions(+), 4839 deletions(-) create mode 100644 include/boost/corosio/detail/object_pool.hpp create mode 100644 include/boost/corosio/detail/object_ref.hpp create mode 100644 test/unit/alloc_counter.hpp create mode 100644 test/unit/object_pool.cpp create mode 100644 test/unit/object_ref.cpp create mode 100644 test/unit/object_reuse.cpp create mode 100644 test/unit/zero_alloc.cpp diff --git a/include/boost/corosio/detail/intrusive.hpp b/include/boost/corosio/detail/intrusive.hpp index f5267634f..4223a80e8 100644 --- a/include/boost/corosio/detail/intrusive.hpp +++ b/include/boost/corosio/detail/intrusive.hpp @@ -69,6 +69,18 @@ class intrusive_list return head_; } + void push_front(T* w) noexcept + { + auto* n = static_cast(w); + n->prev_ = nullptr; + n->next_ = head_; + if (head_) + static_cast(head_)->prev_ = w; + else + tail_ = w; + head_ = w; + } + void push_back(T* w) noexcept { auto* n = static_cast(w); @@ -118,12 +130,33 @@ class intrusive_list return w; } - void remove(T* w) noexcept + /** Unlink @p w if it is a member of this list. + + @pre @p w is either a genuine member of `*this` list, or is + fully detached (not linked in *any* `intrusive_list`, + including a different instance over the same `T`). This is + not a membership test: `T::node` holds one shared `next_`/ + `prev_` pair regardless of which list currently owns it, so a + node actually linked in a different list looks "not detached" + here and would be spliced out using *this* list's `head_`/ + `tail_` for the boundary cases — corrupting both lists. A + caller that cannot rule that out needs its own bookkeeping + (e.g. a flag for which list currently owns the node) before + calling this. + + @return True if @p w was linked in and has now been removed; + false if it was already detached (e.g. a reentrant call from + a self-referential keepalive dropping during the same + teardown that already popped it elsewhere). Callers that gate + a one-time action (recycling, deleting) on actually having + removed something rely on this to make that action idempotent. + */ + bool remove(T* w) noexcept { auto* n = static_cast(w); // Already detached — nothing to do. if (!n->next_ && !n->prev_ && head_ != w && tail_ != w) - return; + return false; if (n->prev_) static_cast(n->prev_)->next_ = n->next_; else @@ -134,6 +167,7 @@ class intrusive_list tail_ = n->prev_; n->next_ = nullptr; n->prev_ = nullptr; + return true; } /// Invoke @p f for each element in the list. diff --git a/include/boost/corosio/detail/object_pool.hpp b/include/boost/corosio/detail/object_pool.hpp new file mode 100644 index 000000000..e334a1181 --- /dev/null +++ b/include/boost/corosio/detail/object_pool.hpp @@ -0,0 +1,319 @@ +// +// Copyright (c) 2025 Steve Gerbino (steve@gerbino.co) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// Official repository: https://github.com/cppalliance/corosio +// + +#ifndef BOOST_COROSIO_DETAIL_OBJECT_POOL_HPP +#define BOOST_COROSIO_DETAIL_OBJECT_POOL_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace boost::corosio::detail { + +/** Per-service recycling pool for I/O implementations. + + Replaces per-impl `shared_ptr` ownership: a service news an impl + once, then `recycle()` (the impl's `retire()` action) parks + it on the free list and `acquire()` hands it back out, so steady-state + construct/destroy allocates nothing. Unbounded; memory is released + only on pool destruction. Debug poisoning, where an impl does it, + happens in that impl's `retire()`, not here (the pool cannot know + which bytes are resettable). + + A service tears down its impls by calling `shutdown()`, which + enters shutting-down mode and visits every live impl in one + critical section; subsequent zero-ref crossings delete instead of + recycling. + + `adopt()` and `remove()` are protected ownership-transfer + primitives for sanctioned extensions — a derived pool exposing + them to a caller that tracks some of its live impls through its + own structure instead of this pool's `live_` list (the timer + service, via its expiry heap) — rather than part of the public + surface every ordinary service uses. + + @par Thread Safety + Distinct objects: Safe. Shared objects: Safe; all operations lock + an internal mutex. +*/ +template +class object_pool +{ + std::mutex mutex_; + intrusive_list live_; + intrusive_list free_; + bool shutting_down_ = false; + + /** Marks `refs_` on an impl currently parked in `free_`. + + `intrusive_list::remove()` cannot distinguish "linked in + `free_`" from "linked in `live_`" by inspecting the node + alone — both lists share one `next_`/`prev_` pair per impl — + so `recycle()` cannot safely call `live_.remove()` on an impl + it does not already know is live. Stamping this sentinel into + `refs_` the moment an impl joins `free_`, and checking it + before touching either list, is what lets `recycle()` and + `remove()` tell the two cases apart in every build mode, not + just under asserts. `acquire()` always overwrites it with 1 + before the impl is usable again. + */ + static constexpr std::size_t pooled_sentinel = pooled_refs; + + /// Check whether @p impl is parked in `free_` (`refs_` reads the + /// sentinel). Callers must check this before any list operation — + /// see the class-level comment on `pooled_sentinel`. + static bool is_pooled(Impl const* impl) noexcept + { + return impl->refs_.load(std::memory_order_relaxed) == + pooled_sentinel; + } + +protected: + /** Track a fresh or reset impl as live. + + @pre @p impl is not parked in `free_` (`refs_` must not read + `pooled_sentinel`). The same list-corruption hazard + `recycle()`'s class-level comment documents applies here: a + free-listed impl pushed onto `live_` directly, bypassing + `acquire()`, would end up linked in both lists through one shared + `next_`/`prev_` pair. + + @pre The caller is not racing `shutdown()`. Every caller adopts + while constructing an object (`acquire()`, or a derived pool's + cache handing a parked impl back out), and construction is + expected to stop before the service's own `shutdown()` runs; + this is checked only in debug builds because the failure mode + is a silently orphaned impl (no close callback), not memory + corruption. + */ + void adopt(Impl* impl) + { + std::lock_guard lock(mutex_); + if (is_pooled(impl)) + { + BOOST_COROSIO_ASSERT(false); + return; + } + BOOST_COROSIO_ASSERT(!shutting_down_); + live_.push_back(impl); + } + + /** Unlink a live impl without recycling (ownership transfer). + + Used by a caller that is taking an impl out of this pool's + bookkeeping entirely (a thread-local cache slot, a direct + delete during shutdown) rather than parking it on `free_`. + + @pre @p impl is not parked in `free_` (`refs_` must not read + `pooled_sentinel`). The same list-corruption hazard + `recycle()`'s class-level comment documents applies here: a + free-listed impl would be spliced out using `live_`'s + boundary pointers instead of `free_`'s. + + @return True if @p impl was linked in `live_` and has now + been removed; false if it was already detached. + */ + bool remove(Impl* impl) noexcept + { + std::lock_guard lock(mutex_); + if (is_pooled(impl)) + { + BOOST_COROSIO_ASSERT(false); + return false; + } + return live_.remove(impl); + } + +public: + /** Destroy the pool, freeing every impl still parked or live. + + A live impl is deleted whatever its count. Keepalives it holds + on itself (an embedded op's `object_ref`) are released by its + own destructor; the count is pinned first so none of those + releases reaches zero and calls `retire()` on an object that + is already being destroyed. + + @pre `shutdown()` has already run, or no impls are live. + */ + ~object_pool() + { + BOOST_COROSIO_ASSERT(shutting_down_ || live_.empty()); + for (auto* list : {&free_, &live_}) + { + while (auto* impl = list->pop_front()) + { + impl->refs_.store(pooled_sentinel, std::memory_order_relaxed); + delete impl; + } + } + } + + object_pool() = default; + object_pool(object_pool const&) = delete; + object_pool& operator=(object_pool const&) = delete; + + /** Pop-or-create an impl with the acquisition protocol applied. + + Recycled impls are `reuse()`-reset and their reference count + restored from the pooled sentinel; fresh impls are constructed + in place from `args` on the cold path. Either way the impl is + adopted as live. This is the sequence every service's + `construct()` used to hand-roll. Making the pool itself the + only place that calls `new Impl` gives the whole library one + audit point for allocation policy. + + @pre Not called concurrently with, or after, `shutdown()` on + this pool. Every caller reaches this through a service's + `construct()`, and service shutdown order guarantees no + `construct()` outlives that service's own `shutdown()`; the + window is only asserted in debug builds (see `adopt()`) + because the failure mode is an orphaned impl, not corruption. + + @param args Constructor arguments for `Impl` (cold path only). + + @return The acquired impl, `refs_ == 1`, tracked live. + */ + template + Impl* acquire(Args&&... args) + { + Impl* impl; + { + std::lock_guard lock(mutex_); + BOOST_COROSIO_ASSERT(!shutting_down_); + impl = free_.pop_front(); + if (impl) + { + impl->refs_.store(1, std::memory_order_relaxed); + live_.push_back(impl); + } + } + // reuse() runs unlocked: acquire()'s precondition excludes a + // concurrent shutdown() visiting the impl mid-reset. + if (impl) + { + impl->reuse(); + return impl; + } + impl = new Impl(std::forward(args)...); + adopt(impl); + return impl; + } + + /** Move a zero-ref impl from live to the free list. + + Called from `retire()`, possibly on a scheduler thread. + Deletes instead when shutting down. The impl must not be + touched after this returns. + + Two accounting bugs are possible here, neither from this + pool's own bookkeeping but from a caller's refcounting (e.g. + a duplicate `retire()` for one zero-crossing). Both are + made safe in every build mode, not just under asserts, because + neither can be told apart from the correct case by a + `BOOST_COROSIO_ASSERT` alone that still runs the list + operations around it: + + - `impl` is already parked in `free_` — the list-corruption + hazard the class-level `pooled_sentinel` comment documents. + The `is_pooled()` check runs first and returns before either + list is touched. + - `impl` is fully detached from `live_` and is not the + sentinel (not shutting down): `live_.remove()` correctly + returns false, but then the impl is reachable through + neither `acquire()` nor `shutdown()`'s visit while still + allocated — stranded. Deleted instead of left to leak. + + While shutting down, an impl that is no longer in `live_` is + left alone rather than deleted: something else already took + it out of the pool's bookkeeping (a derived pool's owner, or a + duplicate retirement of an impl already deleted), and deleting + it again here could be a double free. + */ + void recycle(Impl* impl) + { + bool do_delete = false; + { + std::lock_guard lock(mutex_); + if (is_pooled(impl)) + { + BOOST_COROSIO_ASSERT(false); + return; + } + + bool const was_live = live_.remove(impl); + if (shutting_down_) + { + do_delete = was_live; + } + else if (was_live) + { + impl->refs_.store( + pooled_sentinel, std::memory_order_relaxed); + // LIFO: the most recently retired impl is the + // cache-warmest candidate for the next acquire(). + free_.push_front(impl); + } + else + { + BOOST_COROSIO_ASSERT(false); + do_delete = true; + } + } + if (do_delete) + delete impl; + } + + /** Enter shutdown mode and visit every live impl. + + Sets shutting-down (subsequent zero-crossings delete instead + of recycling) and runs `f` on each live impl, all under one + critical section. Each visited impl is pinned with a reference + for the call and released after the critical section ends, so + `f` may hold on to it; an impl whose count already reached zero + is retiring and is skipped rather than revived. + + @note The callback runs under the pool's non-recursive mutex; + it must not call recycle(), adopt(), or remove() on this pool. + */ + template + void shutdown(F&& f) + { + std::vector pinned; + { + std::lock_guard lock(mutex_); + shutting_down_ = true; + // Sized before any pin, so a failure here leaves none held. + std::size_t n = 0; + live_.for_each([&](Impl*) { ++n; }); + pinned.reserve(n); + live_.for_each( + [&](Impl* impl) + { + if (try_acquire(impl)) + { + pinned.push_back(impl); + f(impl); + } + }); + } + for (auto* impl : pinned) + release(impl); + } +}; + +} // namespace boost::corosio::detail + +#endif diff --git a/include/boost/corosio/detail/object_ref.hpp b/include/boost/corosio/detail/object_ref.hpp new file mode 100644 index 000000000..eaa56972c --- /dev/null +++ b/include/boost/corosio/detail/object_ref.hpp @@ -0,0 +1,152 @@ +// +// Copyright (c) 2025 Vinnie Falco (vinnie.falco@gmail.com) +// Copyright (c) 2026 Steve Gerbino +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// Official repository: https://github.com/cppalliance/corosio +// + +#ifndef BOOST_COROSIO_DETAIL_OBJECT_REF_HPP +#define BOOST_COROSIO_DETAIL_OBJECT_REF_HPP + +#include +#include +#include +#include +#include +#include + +namespace boost::corosio::detail { + +/// Add one reference to `impl`. `impl` must be non-null. +inline void +acquire(io_object::implementation* impl) noexcept +{ + impl->refs_.fetch_add(1, std::memory_order_relaxed); +} + +/** The count an `object_pool` stamps on an impl parked for reuse. + + It holds no reference; see `object_pool`. +*/ +inline constexpr std::size_t pooled_refs = + (std::numeric_limits::max)(); + +/** Add one reference to `impl` unless it is retiring or parked. + + A count at zero means `retire()` is running or about to, and a + count of `pooled_refs` means the impl waits in its pool for a new + owner; neither may be revived. + + @return True if a reference was added. +*/ +inline bool +try_acquire(io_object::implementation* impl) noexcept +{ + auto n = impl->refs_.load(std::memory_order_relaxed); + while (n != 0 && n != pooled_refs) + { + if (impl->refs_.compare_exchange_weak( + n, n + 1, std::memory_order_relaxed)) + return true; + } + return false; +} + +/** Drop one reference; runs `retire()` at zero. `impl` must be + non-null. + + Uses release on the decrement and an acquire fence before the + zero-action so writes made while holding a reference are visible + to the recycler. TSan cannot model a standalone fence, so under + it the decrement itself is acq_rel instead -- strictly stronger, + and TSan's happens-before tracking models the exchange directly. +*/ +inline void +release(io_object::implementation* impl) noexcept +{ +#if BOOST_COROSIO_TSAN + if (impl->refs_.fetch_sub(1, std::memory_order_acq_rel) == 1) + impl->retire(); +#else + if (impl->refs_.fetch_sub(1, std::memory_order_release) == 1) + { + std::atomic_thread_fence(std::memory_order_acquire); + impl->retire(); + } +#endif +} + +/** Owning intrusive reference to an I/O implementation. + + Replaces `std::shared_ptr` op keepalives: copy acquires, + move steals, destruction releases. Kept deliberately minimal — + one pointer, half the size of a shared_ptr, and no control block. +*/ +class object_ref +{ + io_object::implementation* p_ = nullptr; + +public: + /// Destroy the object_ref, releasing the held reference, if any. + ~object_ref() { reset(); } + + /// Construct an object_ref holding no reference. + object_ref() noexcept = default; + + /// Construct an object_ref that acquires a new reference to `p`. + explicit object_ref(io_object::implementation* p) noexcept : p_(p) + { + if (p_) + acquire(p_); + } + + /// Construct a copy, acquiring another reference to `other`'s target. + object_ref(object_ref const& other) noexcept : p_(other.p_) + { + if (p_) + acquire(p_); + } + + /// Construct by transferring the reference out of `other`. + object_ref(object_ref&& other) noexcept + : p_(std::exchange(other.p_, nullptr)) + { + } + + /// Release the held reference, if any, and acquire `other`'s target. + object_ref& operator=(object_ref const& other) noexcept + { + object_ref(other).swap(*this); + return *this; + } + + /// Release the held reference, if any, and take ownership of `other`'s. + object_ref& operator=(object_ref&& other) noexcept + { + object_ref(std::move(other)).swap(*this); + return *this; + } + + /// Release the held reference, if any. + void reset() noexcept + { + if (p_) + release(std::exchange(p_, nullptr)); + } + + /// Return the held implementation pointer. + io_object::implementation* get() const noexcept { return p_; } + + /// Return true if a reference is held. + explicit operator bool() const noexcept { return p_ != nullptr; } + + /// Exchange the implementation pointers held by `*this` and `other`. + void swap(object_ref& other) noexcept { std::swap(p_, other.p_); } +}; + +} // namespace boost::corosio::detail + +#endif diff --git a/include/boost/corosio/detail/platform.hpp b/include/boost/corosio/detail/platform.hpp index 7afd49331..58795e9c7 100644 --- a/include/boost/corosio/detail/platform.hpp +++ b/include/boost/corosio/detail/platform.hpp @@ -86,4 +86,18 @@ #endif // BOOST_COROSIO_MRDOCS +// ThreadSanitizer cannot model a standalone std::atomic_thread_fence; +// code that needs a different (stronger) ordering under it checks this. +#if defined(__SANITIZE_THREAD__) +#define BOOST_COROSIO_TSAN 1 +#elif defined(__has_feature) +#if __has_feature(thread_sanitizer) +#define BOOST_COROSIO_TSAN 1 +#endif +#endif + +#ifndef BOOST_COROSIO_TSAN +#define BOOST_COROSIO_TSAN 0 +#endif + #endif // BOOST_COROSIO_DETAIL_PLATFORM_HPP diff --git a/include/boost/corosio/detail/timer.hpp b/include/boost/corosio/detail/timer.hpp index 332943fac..0f10f8363 100644 --- a/include/boost/corosio/detail/timer.hpp +++ b/include/boost/corosio/detail/timer.hpp @@ -80,7 +80,9 @@ class BOOST_COROSIO_DECL timer : public io_object so `wait` is a plain member function rather than a virtual dispatch point. */ - struct implementation : io_object::implementation + struct implementation + : io_object::implementation + , intrusive_list::node { /// Sentinel value indicating the timer is not in the heap. static constexpr std::size_t npos = @@ -119,12 +121,33 @@ class BOOST_COROSIO_DECL timer : public io_object /// The waiter published on this timer, or `nullptr`. waiter_node* waiter_ = nullptr; - /// Free list linkage, reused when this impl is recycled. - implementation* next_free_ = nullptr; - /// Construct bound to the given timer service. explicit implementation(timer_service& svc) noexcept : svc_(&svc) {} + /** Reset recycled state to fresh-impl values. + + Both recycling tiers in `timer_service::construct()` — the + thread-local single-slot cache and the service's + `object_pool` fallback — call this, so the reset exists in + one place rather than duplicated per tier. + */ + void reuse() noexcept + { + expiry_ = {}; + heap_index_.store(npos, std::memory_order_relaxed); + might_have_pending_waits_.store( + false, std::memory_order_relaxed); + BOOST_COROSIO_ASSERT(waiter_ == nullptr); + } + + /** Recycle into the owning service's pool at zero references. + + Defined in timer.cpp: calling `svc_->pool_` needs + `timer_service`'s complete type, which this header only + forward-declares. + */ + void retire() noexcept override; + /** Check whether the timer is expired and absent from the heap. The single definition of the already-expired fast-path diff --git a/include/boost/corosio/detail/timer_service.hpp b/include/boost/corosio/detail/timer_service.hpp index 36b3dc3da..14613fc56 100644 --- a/include/boost/corosio/detail/timer_service.hpp +++ b/include/boost/corosio/detail/timer_service.hpp @@ -14,6 +14,8 @@ #include #include #include +#include +#include #include #include #include @@ -53,16 +55,19 @@ struct scheduler; list still threads waiters through their intrusive hooks when collecting several timers' waiters past the lock. - timer_service owns a min-heap of active timers and a free list - of recycled impls. The heap is ordered by expiry time; the - scheduler queries nearest_expiry() to set the epoll/timerfd - timeout. + timer_service owns a min-heap of active timers and recycles + retired impls through a per-service object_pool, fronted by a + thread-local single-slot cache. The heap is ordered by expiry + time; the scheduler queries nearest_expiry() to set the + epoll/timerfd timeout. Optimization Strategy --------------------- 1. Deferred heap insertion — expires_after() stores the expiry but does not insert into the heap. Insertion happens in wait(). - 2. Thread-local impl cache — single-slot per-thread cache. + 2. Thread-local impl cache — single-slot per-thread cache, in + front of the object_pool fallback; keeps a thread's hot impl + out of the shared free list. 3. Frame-resident waiter_node with embedded completion_op — eliminates heap allocation per wait/fire/cancel. 4. Cached nearest expiry — atomic avoids mutex in nearest_expiry(). @@ -79,12 +84,30 @@ struct scheduler; inline void timer_service_invalidate_cache() noexcept; +/** The timer service's recycling pool. + + Adds no behavior of its own — it exists only to re-expose + `adopt()` and `remove()` as public, so the timer service's + thread-local cache can transfer a cached impl's `live_` tracking + on both sides of the handoff: `construct()` re-adopts a + TL-cached impl as live, `shutdown()` detaches one before its + direct delete. +*/ +class timer_object_pool : public object_pool +{ +public: + using object_pool::adopt; + using object_pool::remove; +}; + // timer_service class body — member function definitions are // out-of-class (after implementation and waiter_node are complete) class BOOST_COROSIO_DECL timer_service final : public capy::execution_context::service , public io_object::io_service { + friend struct timer::implementation; + public: using clock_type = std::chrono::steady_clock; using time_point = clock_type::time_point; @@ -128,7 +151,7 @@ class BOOST_COROSIO_DECL timer_service final BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // std:: members, dll-interface mutable std::mutex mutex_; std::vector heap_; - timer::implementation* free_list_ = nullptr; + timer_object_pool pool_; callback on_earliest_changed_; bool shutting_down_ = false; // Avoids mutex in nearest_expiry() and empty() @@ -213,9 +236,10 @@ class BOOST_COROSIO_DECL timer_service final inline void swap_heap(std::size_t i1, std::size_t i2); }; -// Thread-local cache avoids hot-path mutex acquisitions: -// single-slot impl cache, validated by comparing svc_. Cleared by -// timer_service_invalidate_cache() during shutdown. +// Thread-local single-slot impl cache, validated by comparing svc_. +// Keeps a hot impl out of the shared free list; both sides still +// take the pool lock to move it in and out of live tracking. +// Cleared by timer_service_invalidate_cache() during shutdown. inline thread_local_ptr tl_cached_impl; @@ -282,6 +306,9 @@ timer_service::shutdown() { timer_service_invalidate_cache(); shutting_down_ = true; + // Flag only; the real drain walks the expiry heap below, not + // object_pool's live_ list, so the visit callback is a no-op. + pool_.shutdown([](timer::implementation*) {}); // Snapshot impls and detach them from the heap so that // coroutine-owned timer destructors (triggered by h.destroy() @@ -316,16 +343,16 @@ timer_service::shutdown() if (h) h.destroy(); } + // Unlink from the pool's live_ list before the direct delete + // below, or ~object_pool()'s unconditional sweep would delete + // this impl a second time. + [[maybe_unused]] bool const removed = pool_.remove(impl); + BOOST_COROSIO_ASSERT(removed); delete impl; } - // Delete free-listed impls - while (free_list_) - { - auto* next = free_list_->next_free_; - delete free_list_; - free_list_ = next; - } + // Anything still parked in pool_ (free-listed or live with no + // pending wait at shutdown) is freed by object_pool's own destructor. } inline io_object::implementation* @@ -335,36 +362,16 @@ timer_service::construct() if (impl) { impl->svc_ = this; - // Reset expiry_ too: a recycled impl must behave like a fresh - // one, whose default expiry reads as already elapsed - impl->expiry_ = {}; - impl->heap_index_.store( - (std::numeric_limits::max)(), - std::memory_order_relaxed); - impl->might_have_pending_waits_.store(false, std::memory_order_relaxed); - BOOST_COROSIO_ASSERT(impl->waiter_ == nullptr); + impl->reuse(); + impl->refs_.store(1, std::memory_order_relaxed); + // The TL push that parked impl here also removed it from + // live_ (see destroy_impl) — re-adopt it now that it is live + // again. + pool_.adopt(impl); return impl; } - std::lock_guard lock(mutex_); - if (free_list_) - { - impl = free_list_; - free_list_ = impl->next_free_; - impl->next_free_ = nullptr; - impl->svc_ = this; - impl->expiry_ = {}; - impl->heap_index_.store( - (std::numeric_limits::max)(), - std::memory_order_relaxed); - impl->might_have_pending_waits_.store(false, std::memory_order_relaxed); - BOOST_COROSIO_ASSERT(impl->waiter_ == nullptr); - } - else - { - impl = new timer::implementation(*this); - } - return impl; + return pool_.acquire(*this); } inline void @@ -402,11 +409,20 @@ timer_service::destroy_impl(timer::implementation& impl) } if (try_push_tl_cache(&impl)) + { + // The TL slot takes sole ownership: it is deleted directly by + // tl_cache_owner / the stale-service path, never through this + // pool, so it must leave live_ now or ~object_pool() would see + // it as abandoned (and, at the wrong moment, double-delete it + // after a later construct() re-adopts it). + [[maybe_unused]] bool const removed = pool_.remove(&impl); + BOOST_COROSIO_ASSERT(removed); return; + } - std::lock_guard lock(mutex_); - impl.next_free_ = free_list_; - free_list_ = &impl; + // No op keepalives on a timer impl: this is the only reference, + // so release() always recycles immediately via retire(). + release(&impl); } inline void diff --git a/include/boost/corosio/io/io_object.hpp b/include/boost/corosio/io/io_object.hpp index 49a251b26..d90f52a08 100644 --- a/include/boost/corosio/io/io_object.hpp +++ b/include/boost/corosio/io/io_object.hpp @@ -15,6 +15,8 @@ #include #include +#include +#include #include namespace boost::corosio { @@ -49,11 +51,35 @@ class BOOST_COROSIO_DECL io_object class handle; /** Derived types dispatch platform-specific I/O operations through it. + + Reference-counted: the owning service holds one reference + (`refs_` starts at 1); in-flight operations hold additional + references through `detail::object_ref`. When the count reaches + zero, `retire()` runs — services recycle the impl into + their free-list there. */ struct implementation { /// Destroy the implementation; called only through @ref io_service. virtual ~implementation() = default; + + /** Retire the implementation once the reference count reaches zero. + + Every concrete implementation's override follows the + same shape: recycle into the owning service's pool member + as the final statement. The service befriends the + implementation so it can reach that private member + directly. Any required last-rites cleanup — releasing + state that is only safe to tear down once idle — runs as + ordinary statements immediately before the recycle. + Whichever action runs — recycle or delete — must be the + final statement: nothing touches the implementation + after. + */ + virtual void retire() noexcept = 0; + + /// In-flight + service references; starts at the service's 1. + std::atomic refs_{1}; }; /** Constructs, closes, and destroys platform implementations on @@ -65,10 +91,28 @@ class BOOST_COROSIO_DECL io_object /// Destroy the service; the execution context outlives it. virtual ~io_service() = default; - /// Construct a new implementation instance. + /** Construct a new implementation instance. + + May return a recycled implementation taken from the + service's free-list (populated by `implementation::retire`) + instead of allocating new storage. + */ virtual implementation* construct() = 0; - /// Destroy the implementation, closing kernel resources and freeing memory. + /** Close kernel resources and release the service's reference. + + Called whenever a handle relinquishes its current + implementation, not only on destruction. Handle + destruction, move-assignment onto a handle that already + owns one (the replaced implementation is destroyed, the + incoming one is not), and `handle::reset()` all invoke + this. It closes the underlying descriptor and drops the + service's own reference. In-flight operations may still + hold references of their own, so the implementation is + not necessarily recycled or freed here. That happens in + `implementation::retire` whenever the reference count + actually reaches zero. + */ virtual void destroy(implementation* impl) = 0; /// Close the I/O object, releasing kernel resources without deallocating. @@ -156,6 +200,14 @@ class BOOST_COROSIO_DECL io_object /** Replace the implementation, destroying the old one. + @pre The handle is already bound to a service (constructed + via the service-taking constructor, not the default + one) — the old implementation is destroyed through + that service. + @pre @p p, if non-null, was constructed by this handle's + own service. The service that destroys it later must + be the one that knows how to close it. + @param p The new implementation to own. May be nullptr. */ void reset(implementation* p) noexcept @@ -190,7 +242,8 @@ class BOOST_COROSIO_DECL io_object /** Create a handle bound to a service found in the context. - @tparam Service The service type whose key_type is used for lookup. + @tparam Service The service type whose `key_type` is used for + lookup. @param ctx The execution context to search for the service. @return A handle owning a freshly constructed implementation. diff --git a/include/boost/corosio/ip_address.hpp b/include/boost/corosio/ip_address.hpp index 98a2215e7..59c9fcc88 100644 --- a/include/boost/corosio/ip_address.hpp +++ b/include/boost/corosio/ip_address.hpp @@ -31,9 +31,9 @@ namespace boost::corosio { This class holds either an IPv4 or an IPv6 address. Code that works with both families carries one value instead of branching between @ref - ipv4_address and @ref ipv6_address. Family-generic queries such as @ref - is_loopback dispatch to the held address, and @ref to_v4 / @ref to_v6 - recover the family-specific form. + ipv4_address and @ref ipv6_address. Family-generic queries such as + @ref is_loopback dispatch to the held address, and @ref to_v4 / + @ref to_v6 recover the family-specific form. A v4-mapped IPv6 address (`::ffff:a.b.c.d`) is an IPv6-family value: it does not compare equal to the IPv4 address it maps. diff --git a/include/boost/corosio/native/detail/coro_op.hpp b/include/boost/corosio/native/detail/coro_op.hpp index 460bb6c87..bac26b0bc 100644 --- a/include/boost/corosio/native/detail/coro_op.hpp +++ b/include/boost/corosio/native/detail/coro_op.hpp @@ -12,6 +12,7 @@ #include #include +#include #include #include @@ -93,7 +94,7 @@ struct coro_op : scheduler_op /// Keeps the owning impl alive while the op is in flight (the kernel /// owns user buffers until completion). Dropped in the handler's resume /// tail (see coro_op_complete.hpp). - std::shared_ptr impl_ptr; + detail::object_ref object_ref_; /// Default-construct for virtual-dispatch backends (the reactors, which /// override operator()/destroy() and leave func_ null). diff --git a/include/boost/corosio/native/detail/coro_op_complete.hpp b/include/boost/corosio/native/detail/coro_op_complete.hpp index 24293de24..298aff82b 100644 --- a/include/boost/corosio/native/detail/coro_op_complete.hpp +++ b/include/boost/corosio/native/detail/coro_op_complete.hpp @@ -25,7 +25,7 @@ 1. disarm the stop_callback, 2. on the shutdown-drain path (owner == nullptr) just break the - impl_ptr keepalive cycle and return without resuming, + object_ref_ keepalive cycle and return without resuming, 3. otherwise resume the coroutine on its executor, dropping the keepalive only after the continuation has been handed off. @@ -113,7 +113,7 @@ decode_io_result( @param self The completing op. @return True if this was a shutdown drain — the caller must `return` immediately without decoding or resuming. On that path the - impl_ptr keepalive is dropped here (which may destroy the impl, + object_ref_ keepalive is dropped here (which may destroy the impl, and with it the op storage). */ inline bool @@ -122,7 +122,7 @@ coro_drain_if_shutdown(void* owner, coro_op* self) noexcept self->stop_cb.reset(); if (owner == nullptr) { - auto suicide = std::move(self->impl_ptr); + auto suicide = std::move(self->object_ref_); return true; } return false; @@ -130,7 +130,7 @@ coro_drain_if_shutdown(void* owner, coro_op* self) noexcept /** Resume tail shared by every proactor handler. - Resumes the op's coroutine on its executor and then drops the impl_ptr + Resumes the op's coroutine on its executor and then drops the object_ref_ keepalive. The keepalive is moved into a local that is released *after* `resume()` returns, matching the existing io_uring ordering: the impl (and therefore this op's storage) may be destroyed as the local goes out of @@ -146,7 +146,7 @@ coro_resume(coro_op* self) noexcept // Clear the keepalive before publishing the continuation: a strand // drained on another thread can reuse this op via reset() the instant // it runs, so this write must be ordered before the publish, not after. - auto suicide = std::move(self->impl_ptr); + auto suicide = std::move(self->object_ref_); auto next = dispatch_coro(self->ex, self->cont); next.resume(); // suicide drops here; may destroy impl + self. diff --git a/include/boost/corosio/native/detail/epoll/epoll_scheduler.hpp b/include/boost/corosio/native/detail/epoll/epoll_scheduler.hpp index 1fdfda1f0..b180357c4 100644 --- a/include/boost/corosio/native/detail/epoll/epoll_scheduler.hpp +++ b/include/boost/corosio/native/detail/epoll/epoll_scheduler.hpp @@ -260,27 +260,37 @@ epoll_scheduler::register_descriptor( ev.events = EPOLLIN | EPOLLOUT | EPOLLET | EPOLLERR | EPOLLHUP; ev.data.ptr = desc; - bool unpollable = false; + // Initialized before the add: another run thread's reactor can + // dispatch an event for the descriptor as soon as it is added. + // A completion queued by an earlier registration of the same state + // may still be running, so the fields it reads change under its + // mutex, and the mutex mode is only written when it differs. + if (desc->mutex.enabled() != reactor_io_locking_) + desc->mutex.set_enabled(reactor_io_locking_); + { + conditionally_enabled_mutex::scoped_lock lock(desc->mutex); + desc->registered_events = ev.events; + desc->unpollable = false; + desc->fd = fd; + desc->scheduler_ = this; + desc->ready_events_.store(0, std::memory_order_release); + // object_ref_ is not touched: a queued invocation from an + // earlier registration still owns the reference in it. + desc->read_ready = false; + desc->write_ready = false; + } + if (::epoll_ctl(epoll_fd_, EPOLL_CTL_ADD, fd, &ev) < 0) { + int const err = errno; + conditionally_enabled_mutex::scoped_lock lock(desc->mutex); + desc->registered_events = 0; // EPERM: a file type epoll cannot watch. Its I/O does not // block, so adopt it unwatched, as asio does. - if (errno != EPERM) - return make_err(errno); - unpollable = true; + if (err != EPERM) + return make_err(err); + desc->unpollable = true; } - - desc->registered_events = unpollable ? 0 : ev.events; - desc->unpollable = unpollable; - desc->fd = fd; - desc->scheduler_ = this; - desc->mutex.set_enabled(reactor_io_locking_); - desc->ready_events_.store(0, std::memory_order_relaxed); - - conditionally_enabled_mutex::scoped_lock lock(desc->mutex); - desc->impl_ref_.reset(); - desc->read_ready = false; - desc->write_ready = false; return {}; } diff --git a/include/boost/corosio/native/detail/iocp/win_file_service.hpp b/include/boost/corosio/native/detail/iocp/win_file_service.hpp index 8dfdaa314..704c180be 100644 --- a/include/boost/corosio/native/detail/iocp/win_file_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_file_service.hpp @@ -19,6 +19,8 @@ #include #include #include +#include +#include #include #include #include @@ -42,6 +44,8 @@ namespace boost::corosio::detail { */ class BOOST_COROSIO_DECL win_file_service final : public file_service { + friend class win_stream_file; + public: using key_type = win_file_service; @@ -91,7 +95,7 @@ class BOOST_COROSIO_DECL win_file_service final : public file_service win_scheduler& sched_; BOOST_COROSIO_MSVC_WARNING_PUSH BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // detail:: members, dll-interface - win_handle_registry reg_; + object_pool pool_; BOOST_COROSIO_MSVC_WARNING_POP void* iocp_; nt_flush_fn nt_flush_buffers_file_ex_; @@ -185,20 +189,22 @@ win_stream_file_internal::seek( // win_stream_file wrapper // --------------------------------------------------------------------------- -inline win_stream_file::win_stream_file( - std::shared_ptr internal) noexcept - : internal_(std::move(internal)) +inline win_stream_file::win_stream_file(win_file_service& svc) noexcept + : svc_(svc) + , internal_(svc.sched_, *this, svc) { } inline void -win_stream_file::close_internal() noexcept +win_stream_file::retire() noexcept { - if (internal_) - { - internal_->close_handle(); - internal_.reset(); - } + svc_.pool_.recycle(this); +} + +inline void +win_stream_file::reuse() noexcept +{ + BOOST_COROSIO_ASSERT(!internal_.is_open()); } inline std::coroutine_handle<> @@ -210,7 +216,7 @@ win_stream_file::read_some( std::error_code* ec, std::size_t* bytes) { - return internal_->read_some(h, d, buf, token, ec, bytes); + return internal_.read_some(h, d, buf, token, ec, bytes); } inline std::coroutine_handle<> @@ -222,68 +228,68 @@ win_stream_file::write_some( std::error_code* ec, std::size_t* bytes) { - return internal_->write_some(h, d, buf, token, ec, bytes); + return internal_.write_some(h, d, buf, token, ec, bytes); } inline native_handle_type win_stream_file::native_handle() const noexcept { - return reinterpret_cast(internal_->native_handle()); + return reinterpret_cast(internal_.native_handle()); } inline void win_stream_file::cancel() noexcept { - internal_->cancel(); + internal_.cancel(); } inline std::uint64_t win_stream_file::size() const { - return internal_->size(); + return internal_.size(); } inline std::error_code win_stream_file::resize(std::uint64_t new_size) noexcept { - return internal_->resize(new_size); + return internal_.resize(new_size); } inline std::error_code win_stream_file::sync_data() noexcept { - return internal_->sync_data(); + return internal_.sync_data(); } inline std::error_code win_stream_file::sync_all() noexcept { - return internal_->sync_all(); + return internal_.sync_all(); } inline native_handle_type win_stream_file::release() { - return internal_->release(); + return internal_.release(); } inline std::error_code win_stream_file::assign(native_handle_type handle) noexcept { - return internal_->assign(handle, handle_kind::stream_file); + return internal_.assign(handle, handle_kind::stream_file); } inline capy::io_result win_stream_file::seek( std::int64_t offset, file_base::seek_basis origin) noexcept { - return internal_->seek(offset, origin); + return internal_.seek(offset, origin); } inline win_stream_file_internal* -win_stream_file::get_internal() const noexcept +win_stream_file::get_internal() noexcept { - return internal_.get(); + return &internal_; } // --------------------------------------------------------------------------- @@ -308,16 +314,17 @@ inline win_file_service::~win_file_service() = default; inline io_object::implementation* win_file_service::construct() { - auto internal = std::make_shared( - sched_, reg_.states(), *this); - return reg_.add(new win_stream_file(std::move(internal))); + return pool_.acquire(*this); } inline void win_file_service::destroy(io_object::implementation* p) { - if (p) - reg_.destroy(static_cast(*p)); + if (!p) + return; + auto* f = static_cast(p); + f->get_internal()->close_handle(); + release(f); } inline void @@ -329,7 +336,8 @@ win_file_service::close(io_object::handle& h) inline void win_file_service::shutdown() { - reg_.shutdown(); + pool_.shutdown( + [](win_stream_file* f) { f->get_internal()->close_handle(); }); } inline std::error_code diff --git a/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor.hpp b/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor.hpp index 89fd2c0e8..c455cdca1 100644 --- a/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor.hpp +++ b/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor.hpp @@ -17,13 +17,13 @@ #include #include #include +#include #include #include #include #include #include -#include #include #include @@ -33,7 +33,7 @@ namespace boost::corosio::detail { class win_local_stream_acceptor_service; class win_local_stream_service; class win_local_stream_socket; -class win_local_stream_acceptor_internal; +class win_local_stream_acceptor; /** Accept operation state for local stream sockets. @@ -43,7 +43,7 @@ struct local_stream_accept_op : overlapped_op { SOCKET accepted_socket = INVALID_SOCKET; win_local_stream_socket* peer_wrapper = nullptr; - std::shared_ptr acceptor_ptr; + win_local_stream_acceptor& acceptor; SOCKET listen_socket = INVALID_SOCKET; io_object::implementation** impl_out = nullptr; // 2 * (sizeof(un_sa_t) + 16) = 2 * (110 + 16) = 252 @@ -56,13 +56,13 @@ struct local_stream_accept_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - local_stream_accept_op() noexcept; + explicit local_stream_accept_op(win_local_stream_acceptor& acceptor_) noexcept; }; /** Readiness-wait operation state for a local stream acceptor. */ struct local_stream_acceptor_wait_op : overlapped_op { - std::shared_ptr acceptor_ptr; + win_local_stream_acceptor& acceptor; SOCKET listen_socket = INVALID_SOCKET; static void do_complete( @@ -72,66 +72,48 @@ struct local_stream_acceptor_wait_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - local_stream_acceptor_wait_op() noexcept; + explicit local_stream_acceptor_wait_op( + win_local_stream_acceptor& acceptor_) noexcept; }; -/* Internal acceptor state for IOCP local stream I/O. */ -class win_local_stream_acceptor_internal - : public intrusive_list::node - , public std::enable_shared_from_this +/** Acceptor implementation for IOCP local stream I/O. + + Collapses the historical internal-state/wrapper split into one + pooled `io_object::implementation`. +*/ +class win_local_stream_acceptor final + : public local_stream_acceptor::implementation + , public intrusive_list::node { friend class win_local_stream_service; - friend class win_local_stream_acceptor; - -public: - explicit win_local_stream_acceptor_internal( - win_local_stream_service& svc) noexcept; - ~win_local_stream_acceptor_internal(); - - win_local_stream_service& socket_service() noexcept; - - std::coroutine_handle<> accept( - std::coroutine_handle<>, - capy::executor_ref, - std::stop_token, - std::error_code*, - io_object::implementation**); - - std::coroutine_handle<> wait( - std::coroutine_handle<>, - capy::executor_ref, - wait_type, - std::stop_token, - std::error_code*); - - SOCKET native_handle() const noexcept; - corosio::local_endpoint local_endpoint() const noexcept; - bool is_open() const noexcept; - void cancel() noexcept; - void close_socket() noexcept; - void set_local_endpoint(corosio::local_endpoint ep) noexcept; + friend struct local_stream_accept_op; + friend struct local_stream_acceptor_wait_op; + win_local_stream_service& svc_; local_stream_accept_op acc_; local_stream_acceptor_wait_op wt_; - -private: - win_local_stream_service& svc_; SOCKET socket_ = INVALID_SOCKET; corosio::local_endpoint local_endpoint_; -}; - -/* Acceptor implementation wrapper for IOCP local stream I/O. */ -class win_local_stream_acceptor final - : public local_stream_acceptor::implementation - , public intrusive_list::node -{ - std::shared_ptr internal_; public: - explicit win_local_stream_acceptor( - std::shared_ptr internal) noexcept; + explicit win_local_stream_acceptor(win_local_stream_service& svc) noexcept; - void close_internal() noexcept; + /// Recycle into the owning service's pool. Defined out-of-line + /// after `win_local_stream_service` is complete (needs + /// `svc_.acceptor_pool_`). + void retire() noexcept override; + + /** Reset recycled state for reuse. + + Each op's `OVERLAPPED` fields are re-seeded by + `overlapped_op::reset()` at its next submit, so they are not + zeroed here. + + @pre refs_ == 0, socket closed, no op in flight. + */ + void reuse() noexcept; + + win_local_stream_service& socket_service() noexcept; std::coroutine_handle<> accept( std::coroutine_handle<> h, @@ -169,7 +151,8 @@ class win_local_stream_acceptor final get_option(int level, int optname, void* data, std::size_t* size) const noexcept override; - win_local_stream_acceptor_internal* get_internal() const noexcept; + void set_local_endpoint(corosio::local_endpoint ep) noexcept; + void close_socket() noexcept; }; } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor_service.hpp b/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor_service.hpp index 39aafb6d3..4a6d9e985 100644 --- a/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor_service.hpp @@ -43,8 +43,7 @@ class BOOST_COROSIO_DECL win_local_stream_acceptor_service final : public local_stream_acceptor_service { public: - explicit win_local_stream_acceptor_service( - capy::execution_context& ctx); + explicit win_local_stream_acceptor_service(capy::execution_context& ctx); io_object::implementation* construct() override; @@ -79,8 +78,10 @@ class BOOST_COROSIO_DECL win_local_stream_acceptor_service final // local_stream_accept_op // ============================================================ -inline local_stream_accept_op::local_stream_accept_op() noexcept +inline local_stream_accept_op::local_stream_accept_op( + win_local_stream_acceptor& acceptor_) noexcept : overlapped_op(&do_complete) + , acceptor(acceptor_) { cancel_func_ = &do_cancel_impl; } @@ -95,8 +96,10 @@ local_stream_accept_op::do_cancel_impl(overlapped_op* base) noexcept } } -inline local_stream_acceptor_wait_op::local_stream_acceptor_wait_op() noexcept +inline local_stream_acceptor_wait_op::local_stream_acceptor_wait_op( + win_local_stream_acceptor& acceptor_) noexcept : overlapped_op(&do_complete) + , acceptor(acceptor_) { cancel_func_ = &do_cancel_impl; } @@ -110,10 +113,7 @@ local_stream_acceptor_wait_op::do_cancel_impl(overlapped_op* base) noexcept { ::CancelIoEx(reinterpret_cast(op->listen_socket), op); } - if (op->acceptor_ptr) - { - op->acceptor_ptr->socket_service().scheduler().cancel_wait(op); - } + op->acceptor.socket_service().scheduler().cancel_wait(op); } // accept_op completion handler @@ -137,12 +137,12 @@ local_stream_accept_op::do_complete( if (op->peer_wrapper) { - op->peer_wrapper->close_internal(); + op->acceptor.socket_service().destroy(op->peer_wrapper); op->peer_wrapper = nullptr; } op->cleanup_only(); - op->acceptor_ptr.reset(); + op->object_ref_.reset(); return; } @@ -167,7 +167,7 @@ local_stream_accept_op::do_complete( op->accepted_socket, SOL_SOCKET, SO_UPDATE_ACCEPT_CONTEXT, reinterpret_cast(&op->listen_socket), sizeof(SOCKET)); - op->peer_wrapper->get_internal()->set_socket(op->accepted_socket); + op->peer_wrapper->set_socket(op->accepted_socket); sockaddr_storage local_storage{}; int local_len = sizeof(local_storage); @@ -186,11 +186,14 @@ local_stream_accept_op::do_complete( remote_ep = from_sockaddr_local( remote_storage, static_cast(remote_len)); - op->peer_wrapper->get_internal()->set_endpoints(local_ep, remote_ep); + op->peer_wrapper->set_endpoints(local_ep, remote_ep); op->accepted_socket = INVALID_SOCKET; if (op->impl_out) *op->impl_out = op->peer_wrapper; + // Handed off to the caller: don't let a failed early return on + // the next accept() see this (now user-owned) pointer again. + op->peer_wrapper = nullptr; } else { @@ -202,7 +205,7 @@ local_stream_accept_op::do_complete( if (op->peer_wrapper) { - op->acceptor_ptr->socket_service().destroy(op->peer_wrapper); + op->acceptor.socket_service().destroy(op->peer_wrapper); op->peer_wrapper = nullptr; } @@ -212,7 +215,7 @@ local_stream_accept_op::do_complete( op->cont.h = op->h; auto saved_ex = op->ex; - auto prevent_premature_destruction = std::move(op->acceptor_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); dispatch_coro(saved_ex, op->cont).resume(); } @@ -229,62 +232,70 @@ local_stream_acceptor_wait_op::do_complete( if (!owner) { op->cleanup_only(); - op->acceptor_ptr.reset(); + op->object_ref_.reset(); return; } - auto prevent_premature_destruction = std::move(op->acceptor_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } // ============================================================ -// win_local_stream_acceptor_internal +// win_local_stream_acceptor // ============================================================ -inline win_local_stream_acceptor_internal::win_local_stream_acceptor_internal( +inline win_local_stream_acceptor::win_local_stream_acceptor( win_local_stream_service& svc) noexcept : svc_(svc) + , acc_(*this) + , wt_(*this) { } -inline win_local_stream_acceptor_internal::~win_local_stream_acceptor_internal() +inline void +win_local_stream_acceptor::reuse() noexcept { - svc_.unregister_acceptor_impl(*this); + BOOST_COROSIO_ASSERT(socket_ == INVALID_SOCKET); + BOOST_COROSIO_ASSERT(local_endpoint_ == corosio::local_endpoint{}); + BOOST_COROSIO_ASSERT(!acc_.stop_cb); + BOOST_COROSIO_ASSERT(!wt_.stop_cb); + BOOST_COROSIO_ASSERT(acc_.peer_wrapper == nullptr); + BOOST_COROSIO_ASSERT(acc_.accepted_socket == INVALID_SOCKET); } inline win_local_stream_service& -win_local_stream_acceptor_internal::socket_service() noexcept +win_local_stream_acceptor::socket_service() noexcept { return svc_; } -inline SOCKET -win_local_stream_acceptor_internal::native_handle() const noexcept +inline native_handle_type +win_local_stream_acceptor::native_handle() const noexcept { - return socket_; + return static_cast(socket_); } inline corosio::local_endpoint -win_local_stream_acceptor_internal::local_endpoint() const noexcept +win_local_stream_acceptor::local_endpoint() const noexcept { return local_endpoint_; } inline bool -win_local_stream_acceptor_internal::is_open() const noexcept +win_local_stream_acceptor::is_open() const noexcept { return socket_ != INVALID_SOCKET; } inline void -win_local_stream_acceptor_internal::set_local_endpoint( +win_local_stream_acceptor::set_local_endpoint( corosio::local_endpoint ep) noexcept { local_endpoint_ = ep; } inline void -win_local_stream_acceptor_internal::cancel() noexcept +win_local_stream_acceptor::cancel() noexcept { if (socket_ != INVALID_SOCKET) ::CancelIoEx(reinterpret_cast(socket_), nullptr); @@ -294,14 +305,14 @@ win_local_stream_acceptor_internal::cancel() noexcept } inline std::coroutine_handle<> -win_local_stream_acceptor_internal::wait( +win_local_stream_acceptor::wait( std::coroutine_handle<> h, capy::executor_ref d, wait_type w, std::stop_token token, std::error_code* ec) { - wt_.acceptor_ptr = shared_from_this(); + wt_.object_ref_ = detail::object_ref(this); wt_.listen_socket = socket_; auto& op = wt_; @@ -337,7 +348,7 @@ win_local_stream_acceptor_internal::wait( } inline void -win_local_stream_acceptor_internal::close_socket() noexcept +win_local_stream_acceptor::close_socket() noexcept { // Flag the accept op cancelled before closing so a closesocket-delivered // ERROR_NETNAME_DELETED is short-circuited to canceled rather than mapped @@ -356,26 +367,33 @@ win_local_stream_acceptor_internal::close_socket() noexcept } inline std::coroutine_handle<> -win_local_stream_acceptor_internal::accept( +win_local_stream_acceptor::accept( std::coroutine_handle<> h, capy::executor_ref d, std::stop_token token, std::error_code* ec, io_object::implementation** impl_out) { - acc_.acceptor_ptr = shared_from_this(); + acc_.object_ref_ = detail::object_ref(this); auto& op = acc_; op.reset(); - op.h = h; - op.ex = d; - op.ec_out = ec; - op.impl_out = impl_out; + // reset() doesn't touch these -- a value left over from a prior + // accept() (e.g. the socket already handed to the user via + // *impl_out) must not survive into a completion this call fails + // before reassigning; do_complete's failure branch would destroy() + // whatever stale pointer it finds here. + op.peer_wrapper = nullptr; + op.accepted_socket = INVALID_SOCKET; + op.h = h; + op.ex = d; + op.ec_out = ec; + op.impl_out = impl_out; op.start(token); svc_.work_started(); - // Create wrapper for the peer socket + // Create a (possibly recycled) peer socket auto* peer_ptr = svc_.construct(); if (!peer_ptr) { @@ -453,87 +471,16 @@ win_local_stream_acceptor_internal::accept( return std::noop_coroutine(); } -// ============================================================ -// win_local_stream_acceptor (wrapper) -// ============================================================ - -inline win_local_stream_acceptor::win_local_stream_acceptor( - std::shared_ptr internal) noexcept - : internal_(std::move(internal)) -{ -} - -inline void -win_local_stream_acceptor::close_internal() noexcept -{ - if (internal_) - { - internal_->close_socket(); - internal_.reset(); - } -} - -inline std::coroutine_handle<> -win_local_stream_acceptor::accept( - std::coroutine_handle<> h, - capy::executor_ref d, - std::stop_token token, - std::error_code* ec, - io_object::implementation** impl_out) -{ - return internal_->accept(h, d, token, ec, impl_out); -} - -inline std::coroutine_handle<> -win_local_stream_acceptor::wait( - std::coroutine_handle<> h, - capy::executor_ref d, - wait_type w, - std::stop_token token, - std::error_code* ec) -{ - return internal_->wait(h, d, w, token, ec); -} - -inline corosio::local_endpoint -win_local_stream_acceptor::local_endpoint() const noexcept -{ - return internal_->local_endpoint(); -} - -inline bool -win_local_stream_acceptor::is_open() const noexcept -{ - return internal_ && internal_->is_open(); -} - -inline void -win_local_stream_acceptor::cancel() noexcept -{ - if (internal_) - internal_->cancel(); -} - -inline native_handle_type -win_local_stream_acceptor::native_handle() const noexcept -{ - if (!internal_) - return static_cast(INVALID_SOCKET); - return static_cast(internal_->native_handle()); -} - inline native_handle_type win_local_stream_acceptor::release_socket() noexcept { - if (!internal_) - return static_cast(INVALID_SOCKET); - SOCKET s = internal_->socket_; + SOCKET s = socket_; if (s != INVALID_SOCKET) { - internal_->cancel(); + cancel(); dissociate_from_iocp(s); - internal_->socket_ = INVALID_SOCKET; - internal_->local_endpoint_ = corosio::local_endpoint{}; + socket_ = INVALID_SOCKET; + local_endpoint_ = corosio::local_endpoint{}; } return static_cast(s); } @@ -542,11 +489,11 @@ inline std::error_code win_local_stream_acceptor::set_option( int level, int optname, void const* data, std::size_t size) noexcept { - if (!internal_ || !internal_->is_open()) + if (socket_ == INVALID_SOCKET) return make_err(WSAENOTSOCK); if (::setsockopt( - internal_->native_handle(), level, optname, - reinterpret_cast(data), static_cast(size)) != 0) + socket_, level, optname, reinterpret_cast(data), + static_cast(size)) != 0) return make_err(WSAGetLastError()); return {}; } @@ -555,21 +502,20 @@ inline std::error_code win_local_stream_acceptor::get_option( int level, int optname, void* data, std::size_t* size) const noexcept { - if (!internal_ || !internal_->is_open()) + if (socket_ == INVALID_SOCKET) return make_err(WSAENOTSOCK); int len = static_cast(*size); if (::getsockopt( - internal_->native_handle(), level, optname, - reinterpret_cast(data), &len) != 0) + socket_, level, optname, reinterpret_cast(data), &len) != 0) return make_err(WSAGetLastError()); *size = static_cast(len); return {}; } -inline win_local_stream_acceptor_internal* -win_local_stream_acceptor::get_internal() const noexcept +inline void +win_local_stream_acceptor::retire() noexcept { - return internal_.get(); + svc_.acceptor_pool_.recycle(this); } // ============================================================ @@ -585,38 +531,23 @@ inline win_local_stream_acceptor_service::win_local_stream_acceptor_service( inline io_object::implementation* win_local_stream_acceptor_service::construct() { - auto internal = std::make_shared(svc_); - - // Allocate wrapper before mutating lists so a throw from - // new doesn't leave a dangling pointer in acceptor_list_. - auto* raw = internal.get(); - auto* wrapper = new win_local_stream_acceptor(std::move(internal)); - - { - std::lock_guard lock(svc_.mutex_); - svc_.acceptor_list_.push_back(raw); - svc_.acceptor_wrapper_list_.push_back(wrapper); - } - - return wrapper; + return svc_.acquire_acceptor_impl(); } inline void win_local_stream_acceptor_service::destroy(io_object::implementation* p) { - if (p) - { - auto& wrapper = static_cast(*p); - wrapper.close_internal(); - svc_.destroy_acceptor_impl(wrapper); - } + if (!p) + return; + auto* a = static_cast(p); + a->close_socket(); + release(a); } inline void win_local_stream_acceptor_service::close(io_object::handle& h) { - auto& wrapper = static_cast(*h.get()); - wrapper.get_internal()->close_socket(); + static_cast(h.get())->close_socket(); } inline std::error_code @@ -626,36 +557,32 @@ win_local_stream_acceptor_service::open_acceptor_socket( int type, int protocol) { - auto* internal = - static_cast(impl).get_internal(); - return svc_.open_acceptor_socket(*internal, family, type, protocol); + auto& acc = static_cast(impl); + return svc_.open_acceptor_socket(acc, family, type, protocol); } inline std::error_code win_local_stream_acceptor_service::assign_socket( local_stream_acceptor::implementation& impl, native_handle_type fd) { - auto* internal = - static_cast(impl).get_internal(); - return svc_.assign_acceptor_socket(*internal, fd); + auto& acc = static_cast(impl); + return svc_.assign_acceptor_socket(acc, fd); } inline std::error_code win_local_stream_acceptor_service::bind_acceptor( local_stream_acceptor::implementation& impl, corosio::local_endpoint ep) { - auto* internal = - static_cast(impl).get_internal(); - return svc_.bind_acceptor(*internal, ep); + auto& acc = static_cast(impl); + return svc_.bind_acceptor(acc, ep); } inline std::error_code win_local_stream_acceptor_service::listen_acceptor( local_stream_acceptor::implementation& impl, int backlog) { - auto* internal = - static_cast(impl).get_internal(); - return svc_.listen_acceptor(*internal, backlog); + auto& acc = static_cast(impl); + return svc_.listen_acceptor(acc, backlog); } inline void diff --git a/include/boost/corosio/native/detail/iocp/win_local_stream_service.hpp b/include/boost/corosio/native/detail/iocp/win_local_stream_service.hpp index 861c615ec..30dd708ce 100644 --- a/include/boost/corosio/native/detail/iocp/win_local_stream_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_local_stream_service.hpp @@ -15,6 +15,7 @@ #if BOOST_COROSIO_HAS_IOCP #include +#include #include #include @@ -23,7 +24,6 @@ #include #include #include -#include #include #include @@ -34,10 +34,6 @@ namespace boost::corosio::detail { -class win_local_stream_acceptor; -class win_local_stream_acceptor_internal; -class win_local_stream_acceptor_service; - /* IOCP local stream socket service. Inherits from local_stream_service to enable runtime polymorphism @@ -48,6 +44,9 @@ class BOOST_COROSIO_DECL win_local_stream_service final : private win_wsa_init , public local_stream_service { + friend class win_local_stream_socket; + friend class win_local_stream_acceptor; + public: io_object::implementation* construct() override; @@ -57,8 +56,6 @@ class BOOST_COROSIO_DECL win_local_stream_service final explicit win_local_stream_service(capy::execution_context& ctx); - ~win_local_stream_service(); - win_local_stream_service(win_local_stream_service const&) = delete; win_local_stream_service& operator=(win_local_stream_service const&) = delete; @@ -75,34 +72,22 @@ class BOOST_COROSIO_DECL win_local_stream_service final local_stream_socket::implementation& impl, native_handle_type fd) override; - void destroy_impl(win_local_stream_socket& impl); - - void unregister_impl(win_local_stream_socket_internal& impl); - std::error_code open_socket_internal( - win_local_stream_socket_internal& impl, - int family, - int type, - int protocol); + win_local_stream_socket& impl, int family, int type, int protocol); - void destroy_acceptor_impl(win_local_stream_acceptor& impl); - - void unregister_acceptor_impl(win_local_stream_acceptor_internal& impl); + win_local_stream_acceptor* acquire_acceptor_impl(); std::error_code open_acceptor_socket( - win_local_stream_acceptor_internal& impl, - int family, - int type, - int protocol); + win_local_stream_acceptor& impl, int family, int type, int protocol); std::error_code assign_acceptor_socket( - win_local_stream_acceptor_internal& impl, native_handle_type fd); + win_local_stream_acceptor& impl, native_handle_type fd); - std::error_code bind_acceptor( - win_local_stream_acceptor_internal& impl, corosio::local_endpoint ep); + std::error_code + bind_acceptor(win_local_stream_acceptor& impl, corosio::local_endpoint ep); std::error_code - listen_acceptor(win_local_stream_acceptor_internal& impl, int backlog); + listen_acceptor(win_local_stream_acceptor& impl, int backlog); void* native_handle() const noexcept; LPFN_CONNECTEX connect_ex() const noexcept; @@ -121,19 +106,14 @@ class BOOST_COROSIO_DECL win_local_stream_service final } private: - friend class win_local_stream_acceptor_service; - win_tcp_service& tcp_svc_; win_scheduler& sched_; + void* iocp_; BOOST_COROSIO_MSVC_WARNING_PUSH BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // detail:: members, dll-interface - win_mutex mutex_; - intrusive_list socket_list_; - intrusive_list acceptor_list_; - intrusive_list socket_wrapper_list_; - intrusive_list acceptor_wrapper_list_; + object_pool pool_; + object_pool acceptor_pool_; BOOST_COROSIO_MSVC_WARNING_POP - void* iocp_; }; // ============================================================ @@ -141,7 +121,7 @@ class BOOST_COROSIO_DECL win_local_stream_service final // ============================================================ inline local_stream_connect_op::local_stream_connect_op( - win_local_stream_socket_internal& internal_) noexcept + win_local_stream_socket& internal_) noexcept : overlapped_op(&do_complete) , internal(internal_) { @@ -149,7 +129,7 @@ inline local_stream_connect_op::local_stream_connect_op( } inline local_stream_read_op::local_stream_read_op( - win_local_stream_socket_internal& internal_) noexcept + win_local_stream_socket& internal_) noexcept : overlapped_op(&do_complete) , internal(internal_) { @@ -157,7 +137,7 @@ inline local_stream_read_op::local_stream_read_op( } inline local_stream_write_op::local_stream_write_op( - win_local_stream_socket_internal& internal_) noexcept + win_local_stream_socket& internal_) noexcept : overlapped_op(&do_complete) , internal(internal_) { @@ -165,7 +145,7 @@ inline local_stream_write_op::local_stream_write_op( } inline local_stream_wait_op::local_stream_wait_op( - win_local_stream_socket_internal& internal_) noexcept + win_local_stream_socket& internal_) noexcept : overlapped_op(&do_complete) , internal(internal_) { @@ -181,10 +161,9 @@ local_stream_connect_op::do_cancel_impl(overlapped_op* base) noexcept { auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) + if (op->internal.socket_ != INVALID_SOCKET) { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); } } @@ -193,10 +172,9 @@ local_stream_read_op::do_cancel_impl(overlapped_op* base) noexcept { auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) + if (op->internal.socket_ != INVALID_SOCKET) { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); } } @@ -205,10 +183,9 @@ local_stream_write_op::do_cancel_impl(overlapped_op* base) noexcept { auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) + if (op->internal.socket_ != INVALID_SOCKET) { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); } } @@ -217,10 +194,9 @@ local_stream_wait_op::do_cancel_impl(overlapped_op* base) noexcept { auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) + if (op->internal.socket_ != INVALID_SOCKET) { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); } op->internal.svc_.scheduler().cancel_wait(op); } @@ -241,31 +217,31 @@ local_stream_connect_op::do_complete( if (!owner) { op->cleanup_only(); - op->internal_ptr.reset(); + op->object_ref_.reset(); return; } bool success = (op->dwError == 0 && !op->cancelled.load(std::memory_order_acquire)); - if (success && op->internal.is_open()) + if (success && op->internal.socket_ != INVALID_SOCKET) { // Required after ConnectEx ::setsockopt( - op->internal.native_handle(), SOL_SOCKET, SO_UPDATE_CONNECT_CONTEXT, + op->internal.socket_, SOL_SOCKET, SO_UPDATE_CONNECT_CONTEXT, nullptr, 0); corosio::local_endpoint local_ep; sockaddr_storage local_storage{}; int local_len = sizeof(local_storage); if (::getsockname( - op->internal.native_handle(), + op->internal.socket_, reinterpret_cast(&local_storage), &local_len) == 0) local_ep = from_sockaddr_local( local_storage, static_cast(local_len)); op->internal.set_endpoints(local_ep, op->target_endpoint); } - auto prevent_premature_destruction = std::move(op->internal_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } @@ -285,11 +261,11 @@ local_stream_read_op::do_complete( if (!owner) { op->cleanup_only(); - op->internal_ptr.reset(); + op->object_ref_.reset(); return; } - auto prevent_premature_destruction = std::move(op->internal_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } @@ -309,11 +285,11 @@ local_stream_write_op::do_complete( if (!owner) { op->cleanup_only(); - op->internal_ptr.reset(); + op->object_ref_.reset(); return; } - auto prevent_premature_destruction = std::move(op->internal_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } @@ -333,7 +309,7 @@ local_stream_wait_op::do_complete( if (!owner) { op->cleanup_only(); - op->internal_ptr.reset(); + op->object_ref_.reset(); return; } @@ -342,15 +318,15 @@ local_stream_wait_op::do_complete( if (op->w == wait_type::read) op->dwError = normalize_read_wait_error(op->dwError); - auto prevent_premature_destruction = std::move(op->internal_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } // ============================================================ -// win_local_stream_socket_internal +// win_local_stream_socket // ============================================================ -inline win_local_stream_socket_internal::win_local_stream_socket_internal( +inline win_local_stream_socket::win_local_stream_socket( win_local_stream_service& svc) noexcept : svc_(svc) , conn_(*this) @@ -360,43 +336,28 @@ inline win_local_stream_socket_internal::win_local_stream_socket_internal( { } -inline win_local_stream_socket_internal::~win_local_stream_socket_internal() -{ - svc_.unregister_impl(*this); -} - -inline SOCKET -win_local_stream_socket_internal::native_handle() const noexcept -{ - return socket_; -} - -inline corosio::local_endpoint -win_local_stream_socket_internal::local_endpoint() const noexcept -{ - return local_endpoint_; -} - -inline corosio::local_endpoint -win_local_stream_socket_internal::remote_endpoint() const noexcept -{ - return remote_endpoint_; -} - -inline bool -win_local_stream_socket_internal::is_open() const noexcept +inline void +win_local_stream_socket::reuse() noexcept { - return socket_ != INVALID_SOCKET; + BOOST_COROSIO_ASSERT(socket_ == INVALID_SOCKET); + // A connect completing on another thread while the socket closed + // can still have written these. + local_endpoint_ = corosio::local_endpoint{}; + remote_endpoint_ = corosio::local_endpoint{}; + BOOST_COROSIO_ASSERT(!conn_.stop_cb); + BOOST_COROSIO_ASSERT(!rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wr_.stop_cb); + BOOST_COROSIO_ASSERT(!wt_.stop_cb); } inline void -win_local_stream_socket_internal::set_socket(SOCKET s) noexcept +win_local_stream_socket::set_socket(SOCKET s) noexcept { socket_ = s; } inline void -win_local_stream_socket_internal::set_endpoints( +win_local_stream_socket::set_endpoints( corosio::local_endpoint local, corosio::local_endpoint remote) noexcept { local_endpoint_ = local; @@ -404,14 +365,14 @@ win_local_stream_socket_internal::set_endpoints( } inline std::coroutine_handle<> -win_local_stream_socket_internal::connect( +win_local_stream_socket::connect( std::coroutine_handle<> h, capy::executor_ref d, corosio::local_endpoint ep, std::stop_token token, std::error_code* ec) { - conn_.internal_ptr = shared_from_this(); + conn_.object_ref_ = detail::object_ref(this); auto& op = conn_; op.reset(); @@ -474,7 +435,7 @@ win_local_stream_socket_internal::connect( } inline std::coroutine_handle<> -win_local_stream_socket_internal::read_some( +win_local_stream_socket::read_some( std::coroutine_handle<> h, capy::executor_ref d, buffer_param param, @@ -482,7 +443,7 @@ win_local_stream_socket_internal::read_some( std::error_code* ec, std::size_t* bytes_out) { - rd_.internal_ptr = shared_from_this(); + rd_.object_ref_ = detail::object_ref(this); auto& op = rd_; op.reset(); @@ -545,7 +506,7 @@ win_local_stream_socket_internal::read_some( } inline std::coroutine_handle<> -win_local_stream_socket_internal::write_some( +win_local_stream_socket::write_some( std::coroutine_handle<> h, capy::executor_ref d, buffer_param param, @@ -553,7 +514,7 @@ win_local_stream_socket_internal::write_some( std::error_code* ec, std::size_t* bytes_out) { - wr_.internal_ptr = shared_from_this(); + wr_.object_ref_ = detail::object_ref(this); auto& op = wr_; op.reset(); @@ -612,14 +573,14 @@ win_local_stream_socket_internal::write_some( } inline std::coroutine_handle<> -win_local_stream_socket_internal::wait( +win_local_stream_socket::wait( std::coroutine_handle<> h, capy::executor_ref d, wait_type w, std::stop_token token, std::error_code* ec) { - wt_.internal_ptr = shared_from_this(); + wt_.object_ref_ = detail::object_ref(this); auto& op = wt_; op.reset(); @@ -678,7 +639,7 @@ win_local_stream_socket_internal::wait( } inline void -win_local_stream_socket_internal::cancel() noexcept +win_local_stream_socket::cancel() noexcept { if (socket_ != INVALID_SOCKET) { @@ -693,7 +654,7 @@ win_local_stream_socket_internal::cancel() noexcept } inline void -win_local_stream_socket_internal::close_socket() noexcept +win_local_stream_socket::close_socket() noexcept { // Flag every op cancelled before closing so a closesocket-delivered // ERROR_NETNAME_DELETED is short-circuited to canceled rather than mapped @@ -715,72 +676,6 @@ win_local_stream_socket_internal::close_socket() noexcept remote_endpoint_ = corosio::local_endpoint{}; } -// ============================================================ -// win_local_stream_socket (wrapper) -// ============================================================ - -inline win_local_stream_socket::win_local_stream_socket( - std::shared_ptr internal) noexcept - : internal_(std::move(internal)) -{ -} - -inline void -win_local_stream_socket::close_internal() noexcept -{ - if (internal_) - { - internal_->close_socket(); - internal_.reset(); - } -} - -inline std::coroutine_handle<> -win_local_stream_socket::connect( - std::coroutine_handle<> h, - capy::executor_ref d, - corosio::local_endpoint ep, - std::stop_token token, - std::error_code* ec) -{ - return internal_->connect(h, d, ep, token, ec); -} - -inline std::coroutine_handle<> -win_local_stream_socket::read_some( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->read_some(h, d, buf, token, ec, bytes); -} - -inline std::coroutine_handle<> -win_local_stream_socket::write_some( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->write_some(h, d, buf, token, ec, bytes); -} - -inline std::coroutine_handle<> -win_local_stream_socket::wait( - std::coroutine_handle<> h, - capy::executor_ref d, - wait_type w, - std::stop_token token, - std::error_code* ec) -{ - return internal_->wait(h, d, w, token, ec); -} - inline std::error_code win_local_stream_socket::shutdown( local_stream_socket::shutdown_type what) noexcept @@ -800,7 +695,7 @@ win_local_stream_socket::shutdown( default: return make_err(WSAEINVAL); } - if (::shutdown(internal_->native_handle(), how) != 0) + if (::shutdown(socket_, how) != 0) return make_err(WSAGetLastError()); return {}; } @@ -808,23 +703,23 @@ win_local_stream_socket::shutdown( inline native_handle_type win_local_stream_socket::native_handle() const noexcept { - return static_cast(internal_->native_handle()); + return static_cast(socket_); } inline native_handle_type win_local_stream_socket::release_socket() noexcept { - SOCKET s = internal_->socket_; + SOCKET s = socket_; if (s != INVALID_SOCKET) { - internal_->cancel(); + cancel(); // Sever the port association so the descriptor can be // adopted again; best-effort, the caller keeps a working // socket either way. dissociate_from_iocp(s); - internal_->socket_ = INVALID_SOCKET; - internal_->local_endpoint_ = corosio::local_endpoint{}; - internal_->remote_endpoint_ = corosio::local_endpoint{}; + socket_ = INVALID_SOCKET; + local_endpoint_ = corosio::local_endpoint{}; + remote_endpoint_ = corosio::local_endpoint{}; } return static_cast(s); } @@ -834,8 +729,8 @@ win_local_stream_socket::set_option( int level, int optname, void const* data, std::size_t size) noexcept { if (::setsockopt( - internal_->native_handle(), level, optname, - reinterpret_cast(data), static_cast(size)) != 0) + socket_, level, optname, reinterpret_cast(data), + static_cast(size)) != 0) return make_err(WSAGetLastError()); return {}; } @@ -846,8 +741,7 @@ win_local_stream_socket::get_option( { int len = static_cast(*size); if (::getsockopt( - internal_->native_handle(), level, optname, - reinterpret_cast(data), &len) != 0) + socket_, level, optname, reinterpret_cast(data), &len) != 0) return make_err(WSAGetLastError()); *size = static_cast(len); return {}; @@ -856,25 +750,19 @@ win_local_stream_socket::get_option( inline corosio::local_endpoint win_local_stream_socket::local_endpoint() const noexcept { - return internal_->local_endpoint(); + return local_endpoint_; } inline corosio::local_endpoint win_local_stream_socket::remote_endpoint() const noexcept { - return internal_->remote_endpoint(); + return remote_endpoint_; } inline void -win_local_stream_socket::cancel() noexcept +win_local_stream_socket::retire() noexcept { - internal_->cancel(); -} - -inline win_local_stream_socket_internal* -win_local_stream_socket::get_internal() const noexcept -{ - return internal_.get(); + svc_.pool_.recycle(this); } // ============================================================ @@ -889,89 +777,36 @@ inline win_local_stream_service::win_local_stream_service( { } -inline win_local_stream_service::~win_local_stream_service() -{ - for (auto* w = socket_wrapper_list_.pop_front(); w != nullptr; - w = socket_wrapper_list_.pop_front()) - delete w; - - for (auto* w = acceptor_wrapper_list_.pop_front(); w != nullptr; - w = acceptor_wrapper_list_.pop_front()) - delete w; -} - inline void win_local_stream_service::shutdown() { - std::lock_guard lock(mutex_); + pool_.shutdown( + [](win_local_stream_socket* impl) { impl->close_socket(); }); - for (auto* impl = socket_list_.pop_front(); impl != nullptr; - impl = socket_list_.pop_front()) - { - impl->close_socket(); - } - - for (auto* impl = acceptor_list_.pop_front(); impl != nullptr; - impl = acceptor_list_.pop_front()) - { - impl->close_socket(); - } + acceptor_pool_.shutdown( + [](win_local_stream_acceptor* impl) { impl->close_socket(); }); } inline io_object::implementation* win_local_stream_service::construct() { - auto internal = std::make_shared(*this); - - { - std::lock_guard lock(mutex_); - socket_list_.push_back(internal.get()); - } - - auto* wrapper = new win_local_stream_socket(std::move(internal)); - - { - std::lock_guard lock(mutex_); - socket_wrapper_list_.push_back(wrapper); - } - - return wrapper; + return pool_.acquire(*this); } inline void win_local_stream_service::destroy(io_object::implementation* p) { - if (p) - { - auto& wrapper = static_cast(*p); - wrapper.close_internal(); - destroy_impl(wrapper); - } + if (!p) + return; + auto* s = static_cast(p); + s->close_socket(); + release(s); } inline void win_local_stream_service::close(io_object::handle& h) { - auto& wrapper = static_cast(*h.get()); - wrapper.get_internal()->close_socket(); -} - -inline void -win_local_stream_service::destroy_impl(win_local_stream_socket& impl) -{ - { - std::lock_guard lock(mutex_); - socket_wrapper_list_.remove(&impl); - } - delete &impl; -} - -inline void -win_local_stream_service::unregister_impl( - win_local_stream_socket_internal& impl) -{ - std::lock_guard lock(mutex_); - socket_list_.remove(&impl); + static_cast(h.get())->close_socket(); } inline std::error_code @@ -981,20 +816,18 @@ win_local_stream_service::open_socket( int type, int protocol) { - auto& wrapper = static_cast(impl); - return open_socket_internal( - *wrapper.get_internal(), family, type, protocol); + auto& sock = static_cast(impl); + return open_socket_internal(sock, family, type, protocol); } inline std::error_code win_local_stream_service::assign_socket( local_stream_socket::implementation& impl, native_handle_type fd) { - auto& wrapper = static_cast(impl); - auto& internal = *wrapper.get_internal(); + auto& sock = static_cast(impl); - SOCKET sock = static_cast(fd); - if (sock == INVALID_SOCKET) + SOCKET s = static_cast(fd); + if (s == INVALID_SOCKET) return make_err(WSAENOTSOCK); // SO_PROTOCOL_INFOW works on an unbound socket, unlike getsockname @@ -1003,7 +836,7 @@ win_local_stream_service::assign_socket( WSAPROTOCOL_INFOW proto_info{}; int proto_len = sizeof(proto_info); if (::getsockopt( - sock, SOL_SOCKET, SO_PROTOCOL_INFOW, + s, SOL_SOCKET, SO_PROTOCOL_INFOW, reinterpret_cast(&proto_info), &proto_len) != 0) return make_err(::WSAGetLastError()); if (proto_info.iAddressFamily != AF_UNIX) @@ -1012,31 +845,30 @@ win_local_stream_service::assign_socket( return make_err(WSAEPROTOTYPE); HANDLE result = ::CreateIoCompletionPort( - reinterpret_cast(sock), static_cast(iocp_), key_io, 0); + reinterpret_cast(s), static_cast(iocp_), key_io, 0); if (result == nullptr) return make_err(::GetLastError()); - internal.socket_ = sock; + sock.socket_ = s; sockaddr_storage local{}; int local_len = sizeof(local); corosio::local_endpoint lep{}, rep{}; - if (::getsockname(sock, reinterpret_cast(&local), &local_len) == - 0) + if (::getsockname(s, reinterpret_cast(&local), &local_len) == 0) lep = from_sockaddr_local(local, static_cast(local_len)); sockaddr_storage remote{}; int remote_len = sizeof(remote); - if (::getpeername( - sock, reinterpret_cast(&remote), &remote_len) == 0) + if (::getpeername(s, reinterpret_cast(&remote), &remote_len) == + 0) rep = from_sockaddr_local(remote, static_cast(remote_len)); - internal.local_endpoint_ = lep; - internal.remote_endpoint_ = rep; + sock.local_endpoint_ = lep; + sock.remote_endpoint_ = rep; return {}; } inline std::error_code win_local_stream_service::open_socket_internal( - win_local_stream_socket_internal& impl, int family, int type, int protocol) + win_local_stream_socket& impl, int family, int type, int protocol) { impl.close_socket(); @@ -1111,30 +943,15 @@ win_local_stream_service::work_finished() noexcept sched_.work_finished(); } -inline void -win_local_stream_service::destroy_acceptor_impl(win_local_stream_acceptor& impl) +inline win_local_stream_acceptor* +win_local_stream_service::acquire_acceptor_impl() { - { - std::lock_guard lock(mutex_); - acceptor_wrapper_list_.remove(&impl); - } - delete &impl; -} - -inline void -win_local_stream_service::unregister_acceptor_impl( - win_local_stream_acceptor_internal& impl) -{ - std::lock_guard lock(mutex_); - acceptor_list_.remove(&impl); + return acceptor_pool_.acquire(*this); } inline std::error_code win_local_stream_service::open_acceptor_socket( - win_local_stream_acceptor_internal& impl, - int family, - int type, - int protocol) + win_local_stream_acceptor& impl, int family, int type, int protocol) { impl.close_socket(); @@ -1160,7 +977,7 @@ win_local_stream_service::open_acceptor_socket( inline std::error_code win_local_stream_service::assign_acceptor_socket( - win_local_stream_acceptor_internal& impl, native_handle_type fd) + win_local_stream_acceptor& impl, native_handle_type fd) { SOCKET sock = static_cast(fd); if (sock == INVALID_SOCKET) @@ -1199,7 +1016,7 @@ win_local_stream_service::assign_acceptor_socket( inline std::error_code win_local_stream_service::bind_acceptor( - win_local_stream_acceptor_internal& impl, corosio::local_endpoint ep) + win_local_stream_acceptor& impl, corosio::local_endpoint ep) { // Reject abstract sockets on Windows if (ep.is_abstract()) @@ -1220,7 +1037,7 @@ win_local_stream_service::bind_acceptor( inline std::error_code win_local_stream_service::listen_acceptor( - win_local_stream_acceptor_internal& impl, int backlog) + win_local_stream_acceptor& impl, int backlog) { SOCKET sock = impl.socket_; diff --git a/include/boost/corosio/native/detail/iocp/win_local_stream_socket.hpp b/include/boost/corosio/native/detail/iocp/win_local_stream_socket.hpp index 40c0b3887..20cca18d1 100644 --- a/include/boost/corosio/native/detail/iocp/win_local_stream_socket.hpp +++ b/include/boost/corosio/native/detail/iocp/win_local_stream_socket.hpp @@ -15,6 +15,7 @@ #if BOOST_COROSIO_HAS_IOCP #include +#include #include #include #include @@ -23,20 +24,18 @@ #include #include -#include #include namespace boost::corosio::detail { class win_local_stream_service; -class win_local_stream_socket_internal; +class win_local_stream_socket; /** Connect operation state for local stream sockets. */ struct local_stream_connect_op : overlapped_op { - win_local_stream_socket_internal& internal; - std::shared_ptr internal_ptr; + win_local_stream_socket& internal; corosio::local_endpoint target_endpoint; static void do_complete( @@ -47,7 +46,7 @@ struct local_stream_connect_op : overlapped_op static void do_cancel_impl(overlapped_op* op) noexcept; explicit local_stream_connect_op( - win_local_stream_socket_internal& internal_) noexcept; + win_local_stream_socket& internal_) noexcept; }; /** Read operation state for local stream sockets. */ @@ -57,8 +56,7 @@ struct local_stream_read_op : overlapped_op WSABUF wsabufs[max_buffers]; DWORD wsabuf_count = 0; DWORD flags = 0; - win_local_stream_socket_internal& internal; - std::shared_ptr internal_ptr; + win_local_stream_socket& internal; static void do_complete( void* owner, @@ -67,8 +65,7 @@ struct local_stream_read_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit local_stream_read_op( - win_local_stream_socket_internal& internal_) noexcept; + explicit local_stream_read_op(win_local_stream_socket& internal_) noexcept; }; /** Write operation state for local stream sockets. */ @@ -77,8 +74,7 @@ struct local_stream_write_op : overlapped_op static constexpr std::size_t max_buffers = 16; WSABUF wsabufs[max_buffers]; DWORD wsabuf_count = 0; - win_local_stream_socket_internal& internal; - std::shared_ptr internal_ptr; + win_local_stream_socket& internal; static void do_complete( void* owner, @@ -87,8 +83,7 @@ struct local_stream_write_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit local_stream_write_op( - win_local_stream_socket_internal& internal_) noexcept; + explicit local_stream_write_op(win_local_stream_socket& internal_) noexcept; }; /** Readiness-wait operation state for local stream sockets. */ @@ -97,8 +92,7 @@ struct local_stream_wait_op : overlapped_op WSABUF wsabuf{}; DWORD flags = 0; wait_type w = wait_type::read; - win_local_stream_socket_internal& internal; - std::shared_ptr internal_ptr; + win_local_stream_socket& internal; static void do_complete( void* owner, @@ -107,23 +101,23 @@ struct local_stream_wait_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit local_stream_wait_op( - win_local_stream_socket_internal& internal_) noexcept; + explicit local_stream_wait_op(win_local_stream_socket& internal_) noexcept; }; -/* Internal socket state for IOCP local stream I/O. +/** Socket implementation for IOCP local stream I/O. - Holds the native SOCKET handle, cached endpoints, and - pending operation slots. Derives from enable_shared_from_this - so in-flight operations can extend its lifetime past the - wrapper's destruction. + Collapses the historical internal-state/wrapper split into one + pooled `io_object::implementation`: it both owns the native SOCKET + handle and pending operations, and directly implements + `local_stream_socket::implementation`. `win_local_stream_service` + recycles instances through its pool instead of freeing them on + every close. */ -class win_local_stream_socket_internal - : public intrusive_list::node - , public std::enable_shared_from_this +class win_local_stream_socket final + : public local_stream_socket::implementation + , public intrusive_list::node { friend class win_local_stream_service; - friend class win_local_stream_socket; friend struct local_stream_read_op; friend struct local_stream_write_op; friend struct local_stream_connect_op; @@ -135,74 +129,26 @@ class win_local_stream_socket_internal local_stream_write_op wr_; local_stream_wait_op wt_; SOCKET socket_ = INVALID_SOCKET; - -public: - explicit win_local_stream_socket_internal( - win_local_stream_service& svc) noexcept; - ~win_local_stream_socket_internal(); - - std::coroutine_handle<> connect( - std::coroutine_handle<>, - capy::executor_ref, - corosio::local_endpoint, - std::stop_token, - std::error_code*); - - std::coroutine_handle<> read_some( - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - std::stop_token, - std::error_code*, - std::size_t*); - - std::coroutine_handle<> write_some( - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - std::stop_token, - std::error_code*, - std::size_t*); - - std::coroutine_handle<> wait( - std::coroutine_handle<>, - capy::executor_ref, - wait_type, - std::stop_token, - std::error_code*); - - SOCKET native_handle() const noexcept; - corosio::local_endpoint local_endpoint() const noexcept; - corosio::local_endpoint remote_endpoint() const noexcept; - bool is_open() const noexcept; - void cancel() noexcept; - void close_socket() noexcept; - void set_socket(SOCKET s) noexcept; - void set_endpoints( - corosio::local_endpoint local, corosio::local_endpoint remote) noexcept; - -private: corosio::local_endpoint local_endpoint_; corosio::local_endpoint remote_endpoint_; -}; -/* Socket implementation wrapper for IOCP local stream I/O. +public: + explicit win_local_stream_socket(win_local_stream_service& svc) noexcept; - Public-facing implementation holding a shared_ptr to the - internal state. Inherits from local_stream_socket::implementation - to satisfy the virtual interface. -*/ -class win_local_stream_socket final - : public local_stream_socket::implementation - , public intrusive_list::node -{ - std::shared_ptr internal_; + /// Recycle into the owning service's pool. Defined out-of-line + /// after `win_local_stream_service` is complete (needs + /// `svc_.pool_`). + void retire() noexcept override; -public: - explicit win_local_stream_socket( - std::shared_ptr internal) noexcept; + /** Reset recycled state for reuse. - void close_internal() noexcept; + Each op's `OVERLAPPED` fields are re-seeded by + `overlapped_op::reset()` at its next submit, so they are not + zeroed here. + + @pre refs_ == 0, socket closed, no op in flight. + */ + void reuse() noexcept; std::coroutine_handle<> connect( std::coroutine_handle<> h, @@ -260,7 +206,13 @@ class win_local_stream_socket final corosio::local_endpoint remote_endpoint() const noexcept override; void cancel() noexcept override; - win_local_stream_socket_internal* get_internal() const noexcept; + /// Used by the acceptor's `local_stream_accept_op` to finish + /// setting up the peer. + void set_socket(SOCKET s) noexcept; + void set_endpoints( + corosio::local_endpoint local, corosio::local_endpoint remote) noexcept; + + void close_socket() noexcept; }; } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/iocp/win_object_handle_service.hpp b/include/boost/corosio/native/detail/iocp/win_object_handle_service.hpp index a7bfca60a..8300f13ad 100644 --- a/include/boost/corosio/native/detail/iocp/win_object_handle_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_object_handle_service.hpp @@ -16,6 +16,8 @@ #include #include +#include +#include #include #include #include @@ -23,7 +25,6 @@ #include #include -#include #include /* win_object_handle on the Windows thread pool. @@ -51,19 +52,20 @@ namespace boost::corosio::detail { +/** Object handle state, embedded in the implementation named by `io`. + + The thread-pool wait survives recycling and is closed with the + implementation's storage. +*/ class win_object_handle_state - : public intrusive_list::node - , public std::enable_shared_from_this { public: win_object_handle_state( - win_scheduler& sched, - win_state_list& list) noexcept + win_scheduler& sched, io_object::implementation& io) noexcept : sched_(sched) - , list_(list) + , io_(io) { op_.self = this; - list_.add(*this); } ~win_object_handle_state() @@ -74,7 +76,6 @@ class win_object_handle_state ::WaitForThreadpoolWaitCallbacks(tp_wait_, FALSE); ::CloseThreadpoolWait(tp_wait_); } - list_.remove(*this); } win_object_handle_state(win_object_handle_state const&) = delete; @@ -110,10 +111,10 @@ class win_object_handle_state std::uint64_t const armed = (gen << 2) | phase_armed; op_.reset(); - op_.keep_alive = shared_from_this(); + op_.object_ref_ = detail::object_ref(&io_); // The op may complete and the object be destroyed on another // thread before wait() returns; this pin outlives both. - auto const pin = op_.keep_alive; + auto const pin = op_.object_ref_; op_.user_cont = &cont; op_.h = cont.h; op_.ex = ex; @@ -180,6 +181,13 @@ class win_object_handle_state return reinterpret_cast(h); } + /// Return true if no wait is armed or being delivered. + bool is_idle() const noexcept + { + return (state_.load(std::memory_order_acquire) & phase_mask) == + phase_idle; + } + std::error_code assign(native_handle_type nh) noexcept { HANDLE h = reinterpret_cast(nh); @@ -198,7 +206,6 @@ class win_object_handle_state struct wait_op : overlapped_op { win_object_handle_state* self = nullptr; - std::shared_ptr keep_alive; capy::continuation* user_cont = nullptr; wait_op() noexcept : overlapped_op(&do_complete) @@ -219,7 +226,7 @@ class win_object_handle_state { auto* op = static_cast(base); auto* self = op->self; - auto prevent_premature_destruction = std::move(op->keep_alive); + auto prevent_premature_destruction = std::move(op->object_ref_); if (!owner) { @@ -299,7 +306,7 @@ class win_object_handle_state } win_scheduler& sched_; - win_state_list& list_; + io_object::implementation& io_; HANDLE handle_ = INVALID_HANDLE_VALUE; PTP_WAIT tp_wait_ = nullptr; std::atomic state_{phase_idle}; @@ -308,32 +315,41 @@ class win_object_handle_state wait_op op_; }; +class win_object_handle_service; + /** IOCP implementation of @ref win_object_handle. */ class win_object_handle_impl final : public win_object_handle::implementation , public intrusive_list::node { - std::shared_ptr internal_; + win_object_handle_service& svc_; + win_object_handle_state internal_; public: - explicit win_object_handle_impl( - std::shared_ptr internal) noexcept - : internal_(std::move(internal)) + win_object_handle_impl( + win_object_handle_service& svc, win_scheduler& sched) noexcept + : svc_(svc) + , internal_(sched, *this) { } - void close_internal() noexcept + /// Recycle into the owning service's pool. Defined out-of-line + /// after the service is complete. + void retire() noexcept override; + + /** Assert the closed state a recycled handle starts from. + + @pre refs_ == 0, handle closed, no wait in flight. + */ + void reuse() noexcept { - if (internal_) - { - internal_->close_handle(); - internal_.reset(); - } + BOOST_COROSIO_ASSERT(!internal_.is_open()); + BOOST_COROSIO_ASSERT(internal_.is_idle()); } - win_object_handle_state* get_internal() const noexcept + win_object_handle_state* get_internal() noexcept { - return internal_.get(); + return &internal_; } std::coroutine_handle<> wait( @@ -342,22 +358,22 @@ class win_object_handle_impl final std::stop_token token, std::error_code* ec) override { - return internal_->wait(cont, ex, std::move(token), ec); + return internal_.wait(cont, ex, std::move(token), ec); } native_handle_type native_handle() const noexcept override { - return reinterpret_cast(internal_->native_handle()); + return reinterpret_cast(internal_.native_handle()); } native_handle_type release_handle() noexcept override { - return internal_->release(); + return internal_.release(); } void cancel() noexcept override { - internal_->cancel(); + internal_.cancel(); } }; @@ -365,6 +381,8 @@ class win_object_handle_impl final class BOOST_COROSIO_DECL win_object_handle_service final : public object_handle_service { + friend class win_object_handle_impl; + public: explicit win_object_handle_service(capy::execution_context& ctx) : sched_(ctx.use_service()) @@ -373,15 +391,16 @@ class BOOST_COROSIO_DECL win_object_handle_service final io_object::implementation* construct() override { - auto internal = - std::make_shared(sched_, reg_.states()); - return reg_.add(new win_object_handle_impl(std::move(internal))); + return pool_.acquire(*this, sched_); } void destroy(io_object::implementation* p) override { - if (p) - reg_.destroy(static_cast(*p)); + if (!p) + return; + auto* impl = static_cast(p); + impl->get_internal()->close_handle(); + release(impl); } void close(io_object::handle& h) override @@ -393,7 +412,9 @@ class BOOST_COROSIO_DECL win_object_handle_service final void shutdown() override { - reg_.shutdown(); + pool_.shutdown([](win_object_handle_impl* impl) { + impl->get_internal()->close_handle(); + }); } std::error_code assign_object_handle( @@ -412,10 +433,16 @@ class BOOST_COROSIO_DECL win_object_handle_service final win_scheduler& sched_; BOOST_COROSIO_MSVC_WARNING_PUSH BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // detail:: members, dll-interface - win_handle_registry reg_; + object_pool pool_; BOOST_COROSIO_MSVC_WARNING_POP }; +inline void +win_object_handle_impl::retire() noexcept +{ + svc_.pool_.recycle(this); +} + } // namespace boost::corosio::detail #endif // BOOST_COROSIO_HAS_IOCP diff --git a/include/boost/corosio/native/detail/iocp/win_overlapped_handle.hpp b/include/boost/corosio/native/detail/iocp/win_overlapped_handle.hpp index a41001314..0ea3aa0ad 100644 --- a/include/boost/corosio/native/detail/iocp/win_overlapped_handle.hpp +++ b/include/boost/corosio/native/detail/iocp/win_overlapped_handle.hpp @@ -19,6 +19,8 @@ #include #include #include +#include +#include #include #include #include @@ -34,24 +36,23 @@ #include #include #include -#include #include #include #include #include -#include /* The overlapped-handle core shared by stream_file, random_access_file, win_stream_handle and win_random_access_handle. Two op models sit on one handle base. The slot model embeds one read and one write op (one of each in flight) and either tracks a file - position or submits at offset 0. The concurrent model allocates an op - per call so any number of positional ops can be in flight. + position or submits at offset 0. The concurrent model takes an op + per call from a per-handle free list, so any number of positional + ops can be in flight and steady state allocates nothing. - Each front keeps the shared-pointer "state + thin wrapper" layering: - ops hold the state alive until their last packet drains, which can - outlive the wrapper. + Each front embeds its handle core in the pooled implementation the + service recycles. Ops hold a reference on that implementation until + their last packet drains, which can outlive the I/O object. */ namespace boost::corosio::detail { @@ -76,95 +77,6 @@ normalize_handle_error(DWORD err, bool is_read) noexcept return err; } -/// A service's list of live states, walked by `shutdown()`. -template -class win_state_list -{ -public: - void add(State& s) noexcept - { - std::lock_guard lock(mutex_); - list_.push_back(&s); - } - - void remove(State& s) noexcept - { - std::lock_guard lock(mutex_); - list_.remove(&s); - } - - void close_all() noexcept - { - // A state whose last owner is mid-destruction cannot be pinned; - // it is blocked in remove() on this mutex and closes itself. - // Pins are released outside the lock, since dropping the last - // one runs a destructor that calls remove(). - std::vector> pins; - { - std::lock_guard lock(mutex_); - for (auto* s = list_.pop_front(); s != nullptr; - s = list_.pop_front()) - if (auto pin = s->weak_from_this().lock()) - pins.push_back(std::move(pin)); - } - for (auto& p : pins) - p->close_handle(); - } - -private: - win_mutex mutex_; - intrusive_list list_; -}; - -/// A service's states and the wrappers it handed out. -template -class win_handle_registry -{ -public: - win_handle_registry() = default; - win_handle_registry(win_handle_registry const&) = delete; - win_handle_registry& operator=(win_handle_registry const&) = delete; - - ~win_handle_registry() - { - for (auto* w = wrappers_.pop_front(); w != nullptr; - w = wrappers_.pop_front()) - delete w; - } - - win_state_list& states() noexcept - { - return states_; - } - - Wrapper* add(Wrapper* w) - { - std::lock_guard lock(mutex_); - wrappers_.push_back(w); - return w; - } - - void destroy(Wrapper& w) noexcept - { - w.close_internal(); - { - std::lock_guard lock(mutex_); - wrappers_.remove(&w); - } - delete &w; - } - - void shutdown() noexcept - { - states_.close_all(); - } - -private: - win_state_list states_; - win_mutex mutex_; - intrusive_list wrappers_; -}; - class win_handle_base; /// An overlapped read or write on an adopted handle. @@ -173,29 +85,24 @@ struct handle_io_op : overlapped_op void* buf = nullptr; DWORD buf_len = 0; win_handle_base* owner = nullptr; - std::shared_ptr keep_alive; explicit handle_io_op(func_type f) noexcept; static void do_cancel_impl(overlapped_op* base) noexcept; }; -/** State shared by every overlapped handle front. */ +/** State shared by every overlapped handle front. + + Embedded in the implementation named by `io`, whose reference the + in-flight ops hold. +*/ class win_handle_base - : public intrusive_list::node - , public std::enable_shared_from_this { public: win_handle_base( - win_scheduler& sched, win_state_list& list) noexcept + win_scheduler& sched, io_object::implementation& io) noexcept : sched_(sched) - , list_(list) - { - list_.add(*this); - } - - ~win_handle_base() + , io_(io) { - list_.remove(*this); } win_handle_base(win_handle_base const&) = delete; @@ -268,7 +175,7 @@ class win_handle_base /** Issue ReadFile/WriteFile for an op the caller has set up. The caller has filled `h`, `ex`, `ec_out`, `bytes_out`, - `is_read`, `owner` and `keep_alive`, called `start(token)` and + `is_read`, `owner` and `object_ref_`, called `start(token)` and `work_started()`. */ void start_io( @@ -325,7 +232,7 @@ class win_handle_base } win_scheduler& sched_; - win_state_list& list_; + io_object::implementation& io_; HANDLE handle_ = INVALID_HANDLE_VALUE; }; @@ -349,9 +256,9 @@ class win_slot_handle : public win_handle_base public: win_slot_handle( win_scheduler& sched, - win_state_list& list, + io_object::implementation& io, bool track_offset) noexcept - : win_handle_base(sched, list) + : win_handle_base(sched, io) , track_offset_(track_offset) { } @@ -438,7 +345,7 @@ class win_slot_handle : public win_handle_base if (!owner) { op->cleanup_only(); - op->keep_alive.reset(); + op->object_ref_.reset(); return; } @@ -453,7 +360,7 @@ class win_slot_handle : public win_handle_base op->bytes_transferred, std::memory_order_acq_rel); op->dwError = normalize_handle_error(op->dwError, op->is_read); - auto prevent_premature_destruction = std::move(op->keep_alive); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } }; @@ -468,7 +375,7 @@ class win_slot_handle : public win_handle_base std::error_code* ec, std::size_t* bytes_out) { - op.keep_alive = shared_from_this(); + op.object_ref_ = detail::object_ref(&io_); op.reset(); op.owner = this; op.is_read = is_read; @@ -501,6 +408,14 @@ class win_concurrent_handle : public win_handle_base public: using win_handle_base::win_handle_base; + // The recycled ops belong to the implementation's storage, which + // survives recycling; only its destruction frees them. + ~win_concurrent_handle() + { + while (auto* op = free_ops_.pop_front()) + delete op; + } + std::coroutine_handle<> read_some_at( std::uint64_t offset, capy::continuation& cont, @@ -574,17 +489,15 @@ class win_concurrent_handle : public win_handle_base { auto* op = static_cast(base); auto* self = static_cast(op->owner); - auto keep = std::move(op->keep_alive); + // Dropped after the op is back on the free list, so a final + // release never runs under ops_mutex_. + auto keep = std::move(op->object_ref_); op->stop_cb.reset(); - { - std::lock_guard lock(self->ops_mutex_); - self->outstanding_ops_.remove(op); - } if (!owner) { op->h = {}; - delete op; + self->recycle_op(op); return; } @@ -600,11 +513,20 @@ class win_concurrent_handle : public win_handle_base capy::continuation* c = op->user_cont; c->h = op->h; capy::executor_ref ex = op->ex; - delete op; + // Once recycled, a concurrent initiation may refill the op. + self->recycle_op(op); dispatch_coro(ex, *c).resume(); } }; + /// Move a finished op from `outstanding_ops_` to `free_ops_`. + void recycle_op(concurrent_op* op) noexcept + { + std::lock_guard lock(ops_mutex_); + outstanding_ops_.remove(op); + free_ops_.push_front(op); + } + void request_cancel_all() noexcept { std::lock_guard lock(ops_mutex_); @@ -622,8 +544,14 @@ class win_concurrent_handle : public win_handle_base std::error_code* ec, std::size_t* bytes_out) { - auto* op = new concurrent_op(); - op->keep_alive = shared_from_this(); + concurrent_op* op; + { + std::lock_guard lock(ops_mutex_); + op = free_ops_.pop_front(); + } + if (!op) + op = new concurrent_op(); + op->object_ref_ = detail::object_ref(&io_); op->reset(); op->owner = this; op->is_read = is_read; @@ -645,6 +573,7 @@ class win_concurrent_handle : public win_handle_base win_mutex ops_mutex_; intrusive_list outstanding_ops_; + intrusive_list free_ops_; }; } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/iocp/win_random_access_file.hpp b/include/boost/corosio/native/detail/iocp/win_random_access_file.hpp index 79000d891..96324fa7a 100644 --- a/include/boost/corosio/native/detail/iocp/win_random_access_file.hpp +++ b/include/boost/corosio/native/detail/iocp/win_random_access_file.hpp @@ -24,7 +24,6 @@ #include #include -#include namespace boost::corosio::detail { @@ -38,9 +37,9 @@ class win_random_access_file_internal : public win_concurrent_handle public: win_random_access_file_internal( win_scheduler& sched, - win_state_list& list, + io_object::implementation& io, win_random_access_file_service& svc) noexcept - : win_concurrent_handle(sched, list) + : win_concurrent_handle(sched, io) , svc_(svc) { } @@ -54,18 +53,32 @@ class win_random_access_file_internal : public win_concurrent_handle win_random_access_file_service& svc_; }; -/** Random-access file implementation wrapper for IOCP-based I/O. */ +/** Random-access file implementation for IOCP-based I/O. + + Embeds its handle state and delegates every virtual call to it. + `win_random_access_file_service` recycles instances through its + pool instead of freeing them on every close. +*/ class win_random_access_file final : public random_access_file::implementation , public intrusive_list::node { - std::shared_ptr internal_; + win_random_access_file_service& svc_; + win_random_access_file_internal internal_; public: explicit win_random_access_file( - std::shared_ptr internal) noexcept; + win_random_access_file_service& svc) noexcept; + + /// Recycle into the owning service's pool. Defined out-of-line + /// after `win_random_access_file_service` is complete. + void retire() noexcept override; + + /** Assert the closed state a recycled file starts from. - void close_internal() noexcept; + @pre refs_ == 0, handle closed, no op in flight. + */ + void reuse() noexcept; std::coroutine_handle<> read_some_at( std::uint64_t offset, @@ -94,7 +107,7 @@ class win_random_access_file final native_handle_type release() override; std::error_code assign(native_handle_type handle) noexcept override; - win_random_access_file_internal* get_internal() const noexcept; + win_random_access_file_internal* get_internal() noexcept; }; } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/iocp/win_random_access_file_service.hpp b/include/boost/corosio/native/detail/iocp/win_random_access_file_service.hpp index 2d975d21c..a339371d2 100644 --- a/include/boost/corosio/native/detail/iocp/win_random_access_file_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_random_access_file_service.hpp @@ -19,6 +19,8 @@ #include #include #include +#include +#include #include #include #include @@ -44,6 +46,8 @@ namespace boost::corosio::detail { class BOOST_COROSIO_DECL win_random_access_file_service final : public random_access_file_service { + friend class win_random_access_file; + public: using key_type = win_random_access_file_service; @@ -95,7 +99,7 @@ class BOOST_COROSIO_DECL win_random_access_file_service final win_scheduler& sched_; BOOST_COROSIO_MSVC_WARNING_PUSH BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // detail:: members, dll-interface - win_handle_registry reg_; + object_pool pool_; BOOST_COROSIO_MSVC_WARNING_POP void* iocp_; nt_flush_fn nt_flush_buffers_file_ex_; @@ -158,19 +162,22 @@ win_random_access_file_internal::sync_all() noexcept // --------------------------------------------------------------------------- inline win_random_access_file::win_random_access_file( - std::shared_ptr internal) noexcept - : internal_(std::move(internal)) + win_random_access_file_service& svc) noexcept + : svc_(svc) + , internal_(svc.sched_, *this, svc) { } inline void -win_random_access_file::close_internal() noexcept +win_random_access_file::retire() noexcept { - if (internal_) - { - internal_->close_handle(); - internal_.reset(); - } + svc_.pool_.recycle(this); +} + +inline void +win_random_access_file::reuse() noexcept +{ + BOOST_COROSIO_ASSERT(!internal_.is_open()); } inline std::coroutine_handle<> @@ -183,7 +190,7 @@ win_random_access_file::read_some_at( std::error_code* ec, std::size_t* bytes) { - return internal_->read_some_at(offset, cont, d, buf, token, ec, bytes); + return internal_.read_some_at(offset, cont, d, buf, token, ec, bytes); } inline std::coroutine_handle<> @@ -196,61 +203,61 @@ win_random_access_file::write_some_at( std::error_code* ec, std::size_t* bytes) { - return internal_->write_some_at(offset, cont, d, buf, token, ec, bytes); + return internal_.write_some_at(offset, cont, d, buf, token, ec, bytes); } inline native_handle_type win_random_access_file::native_handle() const noexcept { - return reinterpret_cast(internal_->native_handle()); + return reinterpret_cast(internal_.native_handle()); } inline void win_random_access_file::cancel() noexcept { - internal_->cancel(); + internal_.cancel(); } inline std::uint64_t win_random_access_file::size() const { - return internal_->size(); + return internal_.size(); } inline std::error_code win_random_access_file::resize(std::uint64_t new_size) noexcept { - return internal_->resize(new_size); + return internal_.resize(new_size); } inline std::error_code win_random_access_file::sync_data() noexcept { - return internal_->sync_data(); + return internal_.sync_data(); } inline std::error_code win_random_access_file::sync_all() noexcept { - return internal_->sync_all(); + return internal_.sync_all(); } inline native_handle_type win_random_access_file::release() { - return internal_->release(); + return internal_.release(); } inline std::error_code win_random_access_file::assign(native_handle_type handle) noexcept { - return internal_->assign(handle, handle_kind::random_access_file); + return internal_.assign(handle, handle_kind::random_access_file); } inline win_random_access_file_internal* -win_random_access_file::get_internal() const noexcept +win_random_access_file::get_internal() noexcept { - return internal_.get(); + return &internal_; } // --------------------------------------------------------------------------- @@ -277,16 +284,17 @@ inline win_random_access_file_service::~win_random_access_file_service() = inline io_object::implementation* win_random_access_file_service::construct() { - auto internal = std::make_shared( - sched_, reg_.states(), *this); - return reg_.add(new win_random_access_file(std::move(internal))); + return pool_.acquire(*this); } inline void win_random_access_file_service::destroy(io_object::implementation* p) { - if (p) - reg_.destroy(static_cast(*p)); + if (!p) + return; + auto* f = static_cast(p); + f->get_internal()->close_handle(); + release(f); } inline void @@ -300,7 +308,8 @@ win_random_access_file_service::close(io_object::handle& h) inline void win_random_access_file_service::shutdown() { - reg_.shutdown(); + pool_.shutdown( + [](win_random_access_file* f) { f->get_internal()->close_handle(); }); } inline std::error_code diff --git a/include/boost/corosio/native/detail/iocp/win_random_access_handle_service.hpp b/include/boost/corosio/native/detail/iocp/win_random_access_handle_service.hpp index 43b4f5082..72f61e139 100644 --- a/include/boost/corosio/native/detail/iocp/win_random_access_handle_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_random_access_handle_service.hpp @@ -15,39 +15,47 @@ #if BOOST_COROSIO_HAS_IOCP #include +#include +#include #include #include -#include - namespace boost::corosio::detail { +class win_random_access_handle_service; + /** IOCP implementation of @ref win_random_access_handle: any number of positional operations. */ class win_random_access_handle_impl final : public win_random_access_handle::implementation , public intrusive_list::node { - std::shared_ptr internal_; + win_random_access_handle_service& svc_; + win_concurrent_handle internal_; public: - explicit win_random_access_handle_impl( - std::shared_ptr internal) noexcept - : internal_(std::move(internal)) + win_random_access_handle_impl( + win_random_access_handle_service& svc, win_scheduler& sched) noexcept + : svc_(svc) + , internal_(sched, *this) { } - void close_internal() noexcept + /// Recycle into the owning service's pool. Defined out-of-line + /// after the service is complete. + void retire() noexcept override; + + /** Assert the closed state a recycled handle starts from. + + @pre refs_ == 0, handle closed, no op in flight. + */ + void reuse() noexcept { - if (internal_) - { - internal_->close_handle(); - internal_.reset(); - } + BOOST_COROSIO_ASSERT(!internal_.is_open()); } - win_concurrent_handle* get_internal() const noexcept + win_concurrent_handle* get_internal() noexcept { - return internal_.get(); + return &internal_; } std::coroutine_handle<> read_some_at( @@ -59,7 +67,7 @@ class win_random_access_handle_impl final std::error_code* ec, std::size_t* bytes) override { - return internal_->read_some_at( + return internal_.read_some_at( offset, cont, ex, buf, std::move(token), ec, bytes); } @@ -72,23 +80,23 @@ class win_random_access_handle_impl final std::error_code* ec, std::size_t* bytes) override { - return internal_->write_some_at( + return internal_.write_some_at( offset, cont, ex, buf, std::move(token), ec, bytes); } native_handle_type native_handle() const noexcept override { - return reinterpret_cast(internal_->native_handle()); + return reinterpret_cast(internal_.native_handle()); } native_handle_type release_handle() override { - return internal_->release(); + return internal_.release(); } void cancel() noexcept override { - internal_->cancel(); + internal_.cancel(); } }; @@ -96,6 +104,8 @@ class win_random_access_handle_impl final class BOOST_COROSIO_DECL win_random_access_handle_service final : public random_access_handle_service { + friend class win_random_access_handle_impl; + public: explicit win_random_access_handle_service(capy::execution_context& ctx) : sched_(ctx.use_service()) @@ -104,15 +114,16 @@ class BOOST_COROSIO_DECL win_random_access_handle_service final io_object::implementation* construct() override { - auto internal = std::make_shared( - sched_, reg_.states()); - return reg_.add(new win_random_access_handle_impl(std::move(internal))); + return pool_.acquire(*this, sched_); } void destroy(io_object::implementation* p) override { - if (p) - reg_.destroy(static_cast(*p)); + if (!p) + return; + auto* impl = static_cast(p); + impl->get_internal()->close_handle(); + release(impl); } void close(io_object::handle& h) override @@ -124,7 +135,8 @@ class BOOST_COROSIO_DECL win_random_access_handle_service final void shutdown() override { - reg_.shutdown(); + pool_.shutdown( + [](win_random_access_handle_impl* impl) { impl->get_internal()->close_handle(); }); } std::error_code assign_random_access_handle( @@ -140,10 +152,16 @@ class BOOST_COROSIO_DECL win_random_access_handle_service final win_scheduler& sched_; BOOST_COROSIO_MSVC_WARNING_PUSH BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // detail:: members, dll-interface - win_handle_registry reg_; + object_pool pool_; BOOST_COROSIO_MSVC_WARNING_POP }; +inline void +win_random_access_handle_impl::retire() noexcept +{ + svc_.pool_.recycle(this); +} + } // namespace boost::corosio::detail #endif // BOOST_COROSIO_HAS_IOCP diff --git a/include/boost/corosio/native/detail/iocp/win_resolver.hpp b/include/boost/corosio/native/detail/iocp/win_resolver.hpp index 873be70df..071451182 100644 --- a/include/boost/corosio/native/detail/iocp/win_resolver.hpp +++ b/include/boost/corosio/native/detail/iocp/win_resolver.hpp @@ -23,6 +23,7 @@ #endif #include +#include #include #include #include @@ -75,11 +76,11 @@ extern "C" Class Hierarchy --------------- - win_resolver_service (execution_context::service) - - Owns all win_resolver instances via shared_ptr + - Owns all win_resolver instances through a recycling object_pool - Coordinates with win_scheduler for work tracking - win_resolver (one per resolver object) - Contains embedded resolve_op and reverse_resolve_op - - Inherits from enable_shared_from_this for thread safety + - Kept alive during I/O by each op's intrusive object_ref_ - resolve_op (overlapped_op subclass) - OVERLAPPED base enables IOCP integration - Static completion() callback invoked by Windows @@ -88,9 +89,10 @@ extern "C" Shutdown -------- - The resolver service cancels all resolvers and clears the impl map. - The thread pool service shuts down separately via execution_context - service ordering, joining all worker threads. + The resolver service cancels all resolvers; each then stays alive + until its own in-flight completion drains, same as every other + pooled impl. The thread pool service shuts down separately via + execution_context service ordering, joining all worker threads. Cancellation ------------ @@ -239,7 +241,6 @@ struct reverse_resolve_op : overlapped_op */ class win_resolver final : public resolver::implementation - , public std::enable_shared_from_this , public intrusive_list::node { friend class win_resolver_service; @@ -254,11 +255,27 @@ class win_resolver final win_resolver* resolver_ = nullptr; /// Prevent impl destruction while work is in flight. - std::shared_ptr ref_; + detail::object_ref ref_; }; explicit win_resolver(win_resolver_service& svc) noexcept; + /// Recycle into the owning service's pool. Defined out-of-line + /// after `win_resolver_service` is complete (needs `svc_.pool_`). + void retire() noexcept override; + + /** Reset recycled state for reuse. + + Does not clear `host`/`service`/`stored_host`/`stored_service` + itself -- each op's own `reset()` (called at the top of the + next `resolve()`/`reverse_resolve()`) already does that, + preserving allocated capacity. That same `reset()` re-seeds + the op's `OVERLAPPED` fields, so they are not zeroed here. + + @pre refs_ == 0, no wait in flight. + */ + void reuse() noexcept; + std::coroutine_handle<> resolve( std::coroutine_handle<>, capy::executor_ref, @@ -298,6 +315,15 @@ class win_resolver final win_resolver_service& svc_; }; +inline void +win_resolver::reuse() noexcept +{ + BOOST_COROSIO_ASSERT(!op_.stop_cb); + BOOST_COROSIO_ASSERT(!reverse_op_.stop_cb); + BOOST_COROSIO_ASSERT(op_.results == nullptr); + BOOST_COROSIO_ASSERT(op_.cancel_handle == nullptr); +} + } // namespace boost::corosio::detail #endif // BOOST_COROSIO_HAS_IOCP diff --git a/include/boost/corosio/native/detail/iocp/win_resolver_service.hpp b/include/boost/corosio/native/detail/iocp/win_resolver_service.hpp index fffa05d13..e0b9c3787 100644 --- a/include/boost/corosio/native/detail/iocp/win_resolver_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_resolver_service.hpp @@ -17,10 +17,13 @@ #if BOOST_COROSIO_HAS_IOCP #include +#include #include #include -#include +#include +#include +#include namespace boost::corosio::detail { @@ -46,6 +49,8 @@ class BOOST_COROSIO_DECL win_resolver_service final , public capy::execution_context::service , public io_object::io_service { + friend class win_resolver; + public: using key_type = win_resolver_service; @@ -55,7 +60,7 @@ class BOOST_COROSIO_DECL win_resolver_service final { auto& impl = static_cast(*p); impl.cancel(); - destroy_impl(impl); + release(&impl); } /** Construct the resolver service. @@ -73,9 +78,6 @@ class BOOST_COROSIO_DECL win_resolver_service final /** Shut down the service. */ void shutdown() override; - /** Destroy a resolver implementation. */ - void destroy_impl(win_resolver& impl); - /** Post an operation for completion. */ void post(overlapped_op* op); @@ -104,12 +106,16 @@ class BOOST_COROSIO_DECL win_resolver_service final } private: + friend struct resolve_op; + scheduler& sched_; thread_pool_ref pool_; - win_mutex mutex_; - intrusive_list resolver_list_; - std::unordered_map> - resolver_ptrs_; + object_pool impls_; + + // GetAddrInfoExW callbacks not yet returned. Each holds a keepalive + // on its impl, so shutdown() waits for them before the pool can be + // destroyed under them. + std::atomic callbacks_pending_{0}; }; BOOST_COROSIO_MSVC_WARNING_POP @@ -229,27 +235,32 @@ resolve_op::completion(DWORD dwError, DWORD /*bytes*/, OVERLAPPED* ov) { auto* op = static_cast(ov); - // The post below can be drained and the win_resolver freed before - // this returns, and Windows owns the OVERLAPPED embedded in it until - // then. - auto keepalive = op->impl->shared_from_this(); - - op->dwError = dwError; - // Posting may free the impl svc_ is read through. auto& svc = op->impl->svc_; - // The handle dies with this callback's entry; a racing cancel() must - // not reach one Windows has already reclaimed. { - std::lock_guard lock(op->impl->cancel_mutex_); - op->cancel_handle = nullptr; + // The post below can be drained and the win_resolver recycled + // or freed before this returns, and Windows owns the OVERLAPPED + // embedded in it until then. + detail::object_ref keepalive(op->impl); + + op->dwError = dwError; + + // The handle dies with this callback's entry; a racing cancel() + // must not reach one Windows has already reclaimed. + { + std::lock_guard lock(op->impl->cancel_mutex_); + op->cancel_handle = nullptr; + } + + // The initiation credit rides with the op and is released where + // it is consumed, as on POSIX; releasing it here would let the + // count reach zero with the op still queued. + svc.post(op); } - // The initiation credit rides with the op and is released where it is - // consumed, as on POSIX; releasing it here would let the count reach - // zero with the op still queued. - svc.post(op); + // Last touch: once this reaches zero shutdown() may let the pool go. + svc.callbacks_pending_.fetch_sub(1, std::memory_order_release); } inline resolve_op::resolve_op() noexcept : overlapped_op(&do_complete) {} @@ -277,7 +288,7 @@ resolve_op::do_complete( } // Dropping the keepalive may destroy the implementation this op // is embedded in, so nothing may touch it afterwards. - op->impl_ptr.reset(); + op->object_ref_.reset(); svc.work_finished(); return; } @@ -309,7 +320,7 @@ resolve_op::do_complete( op->cont.h = op->h; // Hold the keepalive across the dispatch: it may be the last // reference to the implementation this op is embedded in. - auto prevent_destroy = std::move(op->impl_ptr); + auto prevent_destroy = std::move(op->object_ref_); svc.work_finished(); dispatch_coro(op->ex, op->cont).resume(); } @@ -338,7 +349,7 @@ reverse_resolve_op::do_complete( op->stop_cb.reset(); // Dropping the keepalive may destroy the implementation this // op is embedded in, so nothing may touch it afterwards. - op->impl_ptr.reset(); + op->object_ref_.reset(); svc.work_finished(); return; } @@ -365,7 +376,7 @@ reverse_resolve_op::do_complete( op->cont.h = op->h; // Hold the keepalive across the dispatch: it may be the last // reference to the implementation this op is embedded in. - auto prevent_destroy = std::move(op->impl_ptr); + auto prevent_destroy = std::move(op->object_ref_); svc.work_finished(); dispatch_coro(op->ex, op->cont).resume(); } @@ -414,7 +425,7 @@ win_resolver::resolve( // its completion waits in the scheduler queue: the op is embedded in // this win_resolver, which teardown may otherwise free before the // queued completion drains. Mirrors the reverse path's keepalive. - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); // Under the lock: the handle written here and the callback that // retires it must not interleave. @@ -423,6 +434,7 @@ win_resolver::resolve( std::lock_guard lock(cancel_mutex_); op.cancel_handle = nullptr; + svc.callbacks_pending_.fetch_add(1, std::memory_order_relaxed); result = ::GetAddrInfoExW( op.host_w.empty() ? nullptr : op.host_w.c_str(), op.service_w.empty() ? nullptr : op.service_w.c_str(), NS_DNS, @@ -431,7 +443,10 @@ win_resolver::resolve( // No callback runs on synchronous completion. if (result != WSA_IO_PENDING) + { op.cancel_handle = nullptr; + svc.callbacks_pending_.fetch_sub(1, std::memory_order_relaxed); + } } if (result != WSA_IO_PENDING) @@ -480,7 +495,7 @@ win_resolver::reverse_resolve( // Prevent impl destruction while work is in flight reverse_pool_op_.resolver_ = this; - reverse_pool_op_.ref_ = this->shared_from_this(); + reverse_pool_op_.ref_ = detail::object_ref(this); reverse_pool_op_.func_ = &win_resolver::do_reverse_resolve_work; if (auto pec = svc_.pool().post(&reverse_pool_op_)) { @@ -570,7 +585,7 @@ win_resolver::do_reverse_resolve_work(pool_work_item* w) noexcept // Hand the keepalive to the op: the completion waits in the // scheduler's queue, and the implementation embedding it must // outlive that wait. Nothing may touch *self after the post. - self->reverse_op_.impl_ptr = std::move(pw->ref_); + self->reverse_op_.object_ref_ = std::move(pw->ref_); // Post only; the initiation credit is released where the op is // consumed. @@ -591,42 +606,44 @@ inline win_resolver_service::~win_resolver_service() {} inline void win_resolver_service::shutdown() { - std::lock_guard lock(mutex_); - - // Cancel all resolvers (sets cancelled flag checked by pool threads) - for (auto* impl = resolver_list_.pop_front(); impl != nullptr; - impl = resolver_list_.pop_front()) + // Snapshot live impls under an acquired reference, then cancel + // without the pool lock held. shutdown() sets shutting-down and + // takes the snapshot in one critical section, so each cancel's own + // release (if it drops the last ref) deletes rather than recycles. + // In-flight resolves hold their own object_ref_, so each resolver + // stays alive while its completion (GetAddrInfoExW callback or pool + // thread) drains. The thread pool service shuts down separately via + // execution_context service ordering. + std::vector live; + impls_.shutdown( + [&](win_resolver* r) + { + acquire(r); + live.push_back(r); + }); + for (auto* r : live) { - impl->cancel(); + r->cancel(); + release(r); } - // Clear the map which releases shared_ptrs. - // The thread pool service shuts down separately via - // execution_context service ordering. - resolver_ptrs_.clear(); + // Every pending lookup was cancelled above or is already inside its + // callback, so this wait is short. Without it a callback's keepalive + // could outlive impls_ and release into a destroyed pool. + while (callbacks_pending_.load(std::memory_order_acquire) != 0) + std::this_thread::yield(); } inline io_object::implementation* win_resolver_service::construct() { - auto ptr = std::make_shared(*this); - auto* impl = ptr.get(); - - { - std::lock_guard lock(mutex_); - resolver_list_.push_back(impl); - resolver_ptrs_[impl] = std::move(ptr); - } - - return impl; + return impls_.acquire(*this); } inline void -win_resolver_service::destroy_impl(win_resolver& impl) +win_resolver::retire() noexcept { - std::lock_guard lock(mutex_); - resolver_list_.remove(&impl); - resolver_ptrs_.erase(&impl); + svc_.impls_.recycle(this); } inline void diff --git a/include/boost/corosio/native/detail/iocp/win_signal.hpp b/include/boost/corosio/native/detail/iocp/win_signal.hpp index 00e4a2979..1e921ece4 100644 --- a/include/boost/corosio/native/detail/iocp/win_signal.hpp +++ b/include/boost/corosio/native/detail/iocp/win_signal.hpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -125,6 +126,16 @@ class win_signal final public: explicit win_signal(win_signals& svc) noexcept; + /// Recycle into the owning service's pool. Defined out-of-line + /// after `win_signals` is complete (needs `svc_.pool_`). + void retire() noexcept override; + + /** Reset recycled state for reuse. + + @pre refs_ == 0, no registrations, no wait in flight. + */ + void reuse() noexcept; + std::coroutine_handle<> wait( std::coroutine_handle<>, capy::executor_ref, @@ -144,6 +155,16 @@ class win_signal final } }; +inline void +win_signal::reuse() noexcept +{ + BOOST_COROSIO_ASSERT(signals_ == nullptr); + BOOST_COROSIO_ASSERT(!waiting_); + BOOST_COROSIO_ASSERT(!stop_cb_); + cancelled_ = false; + token_cancelled_ = false; +} + } // namespace boost::corosio::detail #endif // BOOST_COROSIO_HAS_IOCP diff --git a/include/boost/corosio/native/detail/iocp/win_signals.hpp b/include/boost/corosio/native/detail/iocp/win_signals.hpp index 2c3df431f..f344d5d3d 100644 --- a/include/boost/corosio/native/detail/iocp/win_signals.hpp +++ b/include/boost/corosio/native/detail/iocp/win_signals.hpp @@ -19,6 +19,8 @@ #include #include +#include +#include #include #include #include @@ -28,6 +30,7 @@ #include #include #include +#include #include @@ -75,7 +78,7 @@ 2. win_signals (one per execution_context) - Maintains registrations_[] table indexed by signal number - Each slot is a doubly-linked list of all signal_registrations for that signal - - Also maintains impl_list_ of all win_signal objects it owns + - Also owns a recycling pool of all win_signal objects it owns 3. win_signal (one per signal_set) - Owns a singly-linked list (sorted by signal number) of signal_registrations @@ -154,11 +157,21 @@ class BOOST_COROSIO_DECL win_signals final : public capy::execution_context::service , public io_object::io_service { + friend class win_signal; + public: using key_type = win_signals; io_object::implementation* construct() override; - void destroy(io_object::implementation*) override; + + void destroy(io_object::implementation* p) override + { + auto& impl = static_cast(*p); + impl.clear(); + impl.disarm_stop(); + impl.cancel(); + release(&impl); + } /** Construct the signal service. @@ -174,13 +187,13 @@ class BOOST_COROSIO_DECL win_signals final /** Shut down the service. - Destroys every implementation the service still owns and gives - each of their registrations back to the process-global table. + Gives every registration its live implementations still hold + back to the process-global table. The implementations are + freed when their handles release them, or when the service is + destroyed if a handle outlives it. */ void shutdown() override; - /** Destroy a signal implementation. */ - void destroy_impl(win_signal& impl); /** Add a signal to a signal set. @@ -265,7 +278,7 @@ class BOOST_COROSIO_DECL win_signals final BOOST_COROSIO_MSVC_WARNING_PUSH BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // detail:: members, dll-interface win_mutex mutex_; - intrusive_list impl_list_; + object_pool pool_; BOOST_COROSIO_MSVC_WARNING_POP // Per-signal registration table for this service @@ -439,21 +452,31 @@ inline win_signals::~win_signals() inline void win_signals::shutdown() { - // Collected under the locks below and deleted after they are released: - // ~win_signal destroys an armed stop_cb_, and ~stop_callback blocks - // until a concurrently running token_canceller returns -- which takes - // mutex_. Deleting while still holding mutex_ would self-deadlock the - // same way disarm_stop() would if called inside the locked loop. - intrusive_list doomed; + // Each visited impl is pinned with acquire() so a handle + // destroy() racing this walk cannot free it mid-walk. The + // matching release() runs after the locks drop: if that handle's + // destroy() raced in, this release() is the last one, and + // ~win_signal destroys an armed stop_cb_ whose ~stop_callback + // waits on a token_canceller that takes mutex_. + std::vector doomed; { signal_detail::signal_state* state = signal_detail::get_signal_state(); std::lock_guard state_lock(state->mutex); std::lock_guard lock(mutex_); - for (auto* impl = impl_list_.pop_front(); impl != nullptr; - impl = impl_list_.pop_front()) + pool_.shutdown( + [&](win_signal* impl) + { + acquire(impl); + doomed.push_back(impl); + }); + for (auto* impl : doomed) { + // A parked wait is abandoned like every other op at + // teardown; a later destroy() must not resume it mid-drain. + impl->waiting_ = false; + while (auto* reg = impl->signals_) { int const signal_number = reg->signal_number; @@ -472,57 +495,31 @@ win_signals::shutdown() impl->signals_ = reg->next_in_set; delete reg; } - doomed.push_back(impl); } - // Every live registration hung off an implementation in impl_list_, - // so the whole table goes stale at once and can be dropped wholesale - // rather than node by node. It has to be dropped: deliver_signal() - // walks this service until the destructor unlinks it from the global - // list. + // Every live registration hung off an implementation this pool + // owns, so the whole table goes stale at once and can be dropped + // wholesale rather than node by node. It has to be dropped: + // deliver_signal() walks this service until the destructor + // unlinks it from the global list. for (int i = 0; i < max_signal_number; ++i) registrations_[i] = nullptr; } - for (auto* impl = doomed.pop_front(); impl != nullptr; - impl = doomed.pop_front()) - { - delete impl; - } + for (auto* impl : doomed) + release(impl); } inline io_object::implementation* win_signals::construct() { - auto* impl = new win_signal(*this); - - { - std::lock_guard lock(mutex_); - impl_list_.push_back(impl); - } - - return impl; -} - -inline void -win_signals::destroy(io_object::implementation* p) -{ - auto& impl = static_cast(*p); - impl.clear(); - impl.disarm_stop(); - impl.cancel(); - destroy_impl(impl); + return pool_.acquire(*this); } inline void -win_signals::destroy_impl(win_signal& impl) +win_signal::retire() noexcept { - { - std::lock_guard lock(mutex_); - impl_list_.remove(&impl); - } - - delete &impl; + svc_.pool_.recycle(this); } inline std::error_code diff --git a/include/boost/corosio/native/detail/iocp/win_stream_file.hpp b/include/boost/corosio/native/detail/iocp/win_stream_file.hpp index ffce11337..dd898e920 100644 --- a/include/boost/corosio/native/detail/iocp/win_stream_file.hpp +++ b/include/boost/corosio/native/detail/iocp/win_stream_file.hpp @@ -38,9 +38,9 @@ class win_stream_file_internal : public win_slot_handle public: win_stream_file_internal( win_scheduler& sched, - win_state_list& list, + io_object::implementation& io, win_file_service& svc) noexcept - : win_slot_handle(sched, list, /*track_offset=*/true) + : win_slot_handle(sched, io, /*track_offset=*/true) , svc_(svc) { } @@ -56,22 +56,31 @@ class win_stream_file_internal : public win_slot_handle win_file_service& svc_; }; -/** Stream file implementation wrapper for IOCP-based I/O. +/** Stream file implementation for IOCP-based I/O. - Public-facing implementation that holds a shared_ptr to - the internal state. Delegates all virtual calls. + Embeds its handle state and delegates every virtual call to it. + `win_file_service` recycles instances through its pool instead of + freeing them on every close. */ class win_stream_file final : public stream_file::implementation , public intrusive_list::node { - std::shared_ptr internal_; + win_file_service& svc_; + win_stream_file_internal internal_; public: - explicit win_stream_file( - std::shared_ptr internal) noexcept; + explicit win_stream_file(win_file_service& svc) noexcept; + + /// Recycle into the owning service's pool. Defined out-of-line + /// after `win_file_service` is complete (needs `svc_.pool_`). + void retire() noexcept override; + + /** Assert the closed state a recycled file starts from. - void close_internal() noexcept; + @pre refs_ == 0, handle closed, no op in flight. + */ + void reuse() noexcept; std::coroutine_handle<> read_some( std::coroutine_handle<> h, @@ -100,7 +109,7 @@ class win_stream_file final capy::io_result seek(std::int64_t offset, file_base::seek_basis origin) noexcept override; - win_stream_file_internal* get_internal() const noexcept; + win_stream_file_internal* get_internal() noexcept; }; } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/iocp/win_stream_handle_service.hpp b/include/boost/corosio/native/detail/iocp/win_stream_handle_service.hpp index ff793e55b..bcaa9c279 100644 --- a/include/boost/corosio/native/detail/iocp/win_stream_handle_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_stream_handle_service.hpp @@ -15,39 +15,47 @@ #if BOOST_COROSIO_HAS_IOCP #include +#include +#include #include #include -#include - namespace boost::corosio::detail { +class win_stream_handle_service; + /** IOCP implementation of @ref win_stream_handle: the slot model at offset 0. */ class win_stream_handle_impl final : public win_stream_handle::implementation , public intrusive_list::node { - std::shared_ptr internal_; + win_stream_handle_service& svc_; + win_slot_handle internal_; public: - explicit win_stream_handle_impl( - std::shared_ptr internal) noexcept - : internal_(std::move(internal)) + win_stream_handle_impl( + win_stream_handle_service& svc, win_scheduler& sched) noexcept + : svc_(svc) + , internal_(sched, *this, /*track_offset=*/false) { } - void close_internal() noexcept + /// Recycle into the owning service's pool. Defined out-of-line + /// after the service is complete. + void retire() noexcept override; + + /** Assert the closed state a recycled handle starts from. + + @pre refs_ == 0, handle closed, no op in flight. + */ + void reuse() noexcept { - if (internal_) - { - internal_->close_handle(); - internal_.reset(); - } + BOOST_COROSIO_ASSERT(!internal_.is_open()); } - win_slot_handle* get_internal() const noexcept + win_slot_handle* get_internal() noexcept { - return internal_.get(); + return &internal_; } std::coroutine_handle<> read_some( @@ -58,7 +66,7 @@ class win_stream_handle_impl final std::error_code* ec, std::size_t* bytes) override { - return internal_->read_some(h, ex, buf, std::move(token), ec, bytes); + return internal_.read_some(h, ex, buf, std::move(token), ec, bytes); } std::coroutine_handle<> write_some( @@ -69,22 +77,22 @@ class win_stream_handle_impl final std::error_code* ec, std::size_t* bytes) override { - return internal_->write_some(h, ex, buf, std::move(token), ec, bytes); + return internal_.write_some(h, ex, buf, std::move(token), ec, bytes); } native_handle_type native_handle() const noexcept override { - return reinterpret_cast(internal_->native_handle()); + return reinterpret_cast(internal_.native_handle()); } native_handle_type release_handle() override { - return internal_->release(); + return internal_.release(); } void cancel() noexcept override { - internal_->cancel(); + internal_.cancel(); } }; @@ -92,6 +100,8 @@ class win_stream_handle_impl final class BOOST_COROSIO_DECL win_stream_handle_service final : public stream_handle_service { + friend class win_stream_handle_impl; + public: explicit win_stream_handle_service(capy::execution_context& ctx) : sched_(ctx.use_service()) @@ -100,15 +110,16 @@ class BOOST_COROSIO_DECL win_stream_handle_service final io_object::implementation* construct() override { - auto internal = std::make_shared( - sched_, reg_.states(), /*track_offset=*/false); - return reg_.add(new win_stream_handle_impl(std::move(internal))); + return pool_.acquire(*this, sched_); } void destroy(io_object::implementation* p) override { - if (p) - reg_.destroy(static_cast(*p)); + if (!p) + return; + auto* impl = static_cast(p); + impl->get_internal()->close_handle(); + release(impl); } void close(io_object::handle& h) override @@ -120,7 +131,8 @@ class BOOST_COROSIO_DECL win_stream_handle_service final void shutdown() override { - reg_.shutdown(); + pool_.shutdown( + [](win_stream_handle_impl* impl) { impl->get_internal()->close_handle(); }); } std::error_code assign_stream_handle( @@ -136,10 +148,16 @@ class BOOST_COROSIO_DECL win_stream_handle_service final win_scheduler& sched_; BOOST_COROSIO_MSVC_WARNING_PUSH BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // detail:: members, dll-interface - win_handle_registry reg_; + object_pool pool_; BOOST_COROSIO_MSVC_WARNING_POP }; +inline void +win_stream_handle_impl::retire() noexcept +{ + svc_.pool_.recycle(this); +} + } // namespace boost::corosio::detail #endif // BOOST_COROSIO_HAS_IOCP diff --git a/include/boost/corosio/native/detail/iocp/win_tcp_acceptor.hpp b/include/boost/corosio/native/detail/iocp/win_tcp_acceptor.hpp index 722ca55b0..420317910 100644 --- a/include/boost/corosio/native/detail/iocp/win_tcp_acceptor.hpp +++ b/include/boost/corosio/native/detail/iocp/win_tcp_acceptor.hpp @@ -19,12 +19,12 @@ #include #include #include +#include #include #include #include #include -#include #include #include @@ -34,14 +34,14 @@ namespace boost::corosio::detail { class win_tcp_acceptor_service; class win_tcp_service; class win_tcp_socket; -class win_tcp_acceptor_internal; +class win_tcp_acceptor; /** Accept operation state. */ struct accept_op : overlapped_op { SOCKET accepted_socket = INVALID_SOCKET; win_tcp_socket* peer_wrapper = nullptr; - std::shared_ptr acceptor_ptr; + win_tcp_acceptor& acceptor; SOCKET listen_socket = INVALID_SOCKET; io_object::implementation** impl_out = nullptr; char addr_buf[2 * (sizeof(sockaddr_in6) + 16)]; @@ -53,13 +53,13 @@ struct accept_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - accept_op() noexcept; + explicit accept_op(win_tcp_acceptor& acceptor_) noexcept; }; /** Readiness-wait operation state for an acceptor. */ struct acceptor_wait_op : overlapped_op { - std::shared_ptr acceptor_ptr; + win_tcp_acceptor& acceptor; SOCKET listen_socket = INVALID_SOCKET; static void do_complete( @@ -69,79 +69,57 @@ struct acceptor_wait_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - acceptor_wait_op() noexcept; + explicit acceptor_wait_op(win_tcp_acceptor& acceptor_) noexcept; }; -/** Internal acceptor state for IOCP-based I/O. +/** Acceptor implementation for IOCP-based I/O. - This class contains the actual state for a listening socket, including - the native socket handle and pending accept operation. + Collapses the historical internal-state/wrapper split into one + pooled `io_object::implementation`: it both owns the listening + socket and pending operations, and directly implements + `tcp_acceptor::implementation`. `win_tcp_service` recycles + instances through its acceptor pool instead of freeing them on + every close. @note Internal implementation detail. Users interact with acceptor class. */ -class win_tcp_acceptor_internal - : public intrusive_list::node - , public std::enable_shared_from_this +class win_tcp_acceptor final + : public tcp_acceptor::implementation + , public intrusive_list::node { friend class win_tcp_service; - friend class win_tcp_acceptor; - -public: - explicit win_tcp_acceptor_internal(win_tcp_service& svc) noexcept; - ~win_tcp_acceptor_internal(); - - /// Return the owning socket service. - win_tcp_service& socket_service() noexcept; - - std::coroutine_handle<> accept( - std::coroutine_handle<>, - capy::executor_ref, - std::stop_token, - std::error_code*, - io_object::implementation**); - - std::coroutine_handle<> wait( - std::coroutine_handle<>, - capy::executor_ref, - wait_type, - std::stop_token, - std::error_code*); - - SOCKET native_handle() const noexcept; - endpoint local_endpoint() const noexcept; - bool is_open() const noexcept; - void cancel() noexcept; - void close_socket() noexcept; - void set_local_endpoint(endpoint ep) noexcept; + friend struct accept_op; + friend struct acceptor_wait_op; + win_tcp_service& svc_; accept_op acc_; acceptor_wait_op wt_; - -private: - win_tcp_service& svc_; SOCKET socket_ = INVALID_SOCKET; int family_ = AF_UNSPEC; endpoint local_endpoint_; -}; -/** Acceptor implementation wrapper for IOCP-based I/O. +public: + explicit win_tcp_acceptor(win_tcp_service& svc) noexcept; - This class is the public-facing implementation that holds a shared_ptr - to the internal state. The shared_ptr is hidden from the public interface. + /// Recycle into the owning service's pool. Defined out-of-line + /// after `win_tcp_service` is complete (needs `svc_.acceptor_pool_`). + void retire() noexcept override; - @note Internal implementation detail. Users interact with acceptor class. -*/ -class win_tcp_acceptor final - : public tcp_acceptor::implementation - , public intrusive_list::node -{ - std::shared_ptr internal_; + /** Reset recycled state for reuse. -public: - explicit win_tcp_acceptor( - std::shared_ptr internal) noexcept; + `close_socket()` already drives the socket handle, family, + and cached endpoint to their closed values, and each op's + completion drops its `object_ref_` before `refs_` can reach + zero, so this only asserts. Each op's `OVERLAPPED` fields are + re-seeded by `overlapped_op::reset()` at its next submit, so + they are not zeroed here. - void close_internal() noexcept; + @pre refs_ == 0, socket closed, no op in flight. + */ + void reuse() noexcept; + + /// Return the owning socket service. + win_tcp_service& socket_service() noexcept; std::coroutine_handle<> accept( std::coroutine_handle<> h, @@ -167,7 +145,7 @@ class win_tcp_acceptor final { // getsockname fails with WSAEINVAL on an unbound socket, so // the family recorded at socket creation is authoritative - return to_family(internal_ ? internal_->family_ : AF_UNSPEC); + return to_family(family_); } native_handle_type release_socket() noexcept override; @@ -180,7 +158,8 @@ class win_tcp_acceptor final get_option(int level, int optname, void* data, std::size_t* size) const noexcept override; - win_tcp_acceptor_internal* get_internal() const noexcept; + void set_local_endpoint(endpoint ep) noexcept; + void close_socket() noexcept; }; } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/iocp/win_tcp_acceptor_service.hpp b/include/boost/corosio/native/detail/iocp/win_tcp_acceptor_service.hpp index 42e4f6d0e..8994c5320 100644 --- a/include/boost/corosio/native/detail/iocp/win_tcp_acceptor_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_tcp_acceptor_service.hpp @@ -38,7 +38,8 @@ namespace boost::corosio::detail { /** IOCP acceptor service wrapping win_tcp_service for acceptor lifecycle. Provides io_service + acceptor_service interface for tcp_acceptor - on Windows. Delegates to win_tcp_service for actual socket operations. + on Windows. Delegates to win_tcp_service for actual socket operations + and for the acceptor pool itself. */ class BOOST_COROSIO_DECL win_tcp_acceptor_service final : public capy::execution_context::service @@ -79,1382 +80,248 @@ class BOOST_COROSIO_DECL win_tcp_acceptor_service final // Operation constructors -inline connect_op::connect_op(win_tcp_socket_internal& internal_) noexcept +inline accept_op::accept_op(win_tcp_acceptor& acceptor_) noexcept : overlapped_op(&do_complete) - , internal(internal_) + , acceptor(acceptor_) { cancel_func_ = &do_cancel_impl; } -inline read_op::read_op(win_tcp_socket_internal& internal_) noexcept +inline acceptor_wait_op::acceptor_wait_op(win_tcp_acceptor& acceptor_) noexcept : overlapped_op(&do_complete) - , internal(internal_) + , acceptor(acceptor_) { cancel_func_ = &do_cancel_impl; } -inline write_op::write_op(win_tcp_socket_internal& internal_) noexcept - : overlapped_op(&do_complete) - , internal(internal_) -{ - cancel_func_ = &do_cancel_impl; -} - -inline wait_op::wait_op(win_tcp_socket_internal& internal_) noexcept - : overlapped_op(&do_complete) - , internal(internal_) -{ - cancel_func_ = &do_cancel_impl; -} - -inline accept_op::accept_op() noexcept : overlapped_op(&do_complete) -{ - cancel_func_ = &do_cancel_impl; -} - -inline acceptor_wait_op::acceptor_wait_op() noexcept - : overlapped_op(&do_complete) -{ - cancel_func_ = &do_cancel_impl; -} - -// Cancellation functions - -inline void -connect_op::do_cancel_impl(overlapped_op* base) noexcept -{ - auto* op = static_cast(base); - if (op->internal.is_open()) - { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); - } -} - -inline void -read_op::do_cancel_impl(overlapped_op* base) noexcept -{ - auto* op = static_cast(base); - op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) - { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); - } -} - -inline void -write_op::do_cancel_impl(overlapped_op* base) noexcept -{ - auto* op = static_cast(base); - op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) - { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); - } -} - -inline void -wait_op::do_cancel_impl(overlapped_op* base) noexcept -{ - auto* op = static_cast(base); - op->cancelled.store(true, std::memory_order_release); - // Best-effort cancel of any pending zero-byte WSARecv issued for - // wait_type::read. ERROR_NOT_FOUND when nothing overlapped is - // pending is harmless. - if (op->internal.is_open()) - { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); - } - // wait_type::error parks the op in the auxiliary select reactor; - // wake it so the reactor can post a cancelled completion. A cancel - // for an op the reactor never registered finds nothing and returns. - op->internal.svc_.scheduler().cancel_wait(op); -} - -inline void -accept_op::do_cancel_impl(overlapped_op* base) noexcept -{ - auto* op = static_cast(base); - if (op->listen_socket != INVALID_SOCKET) - { - ::CancelIoEx(reinterpret_cast(op->listen_socket), op); - } -} - -inline void -acceptor_wait_op::do_cancel_impl(overlapped_op* base) noexcept -{ - auto* op = static_cast(base); - op->cancelled.store(true, std::memory_order_release); - if (op->listen_socket != INVALID_SOCKET) - { - ::CancelIoEx(reinterpret_cast(op->listen_socket), op); - } - if (op->acceptor_ptr) - { - op->acceptor_ptr->socket_service().scheduler().cancel_wait(op); - } -} - -// accept_op completion handler - -inline void -accept_op::do_complete( - void* owner, - scheduler_op* base, - std::uint32_t /*bytes*/, - std::uint32_t /*error*/) -{ - auto* op = static_cast(base); - - if (!owner) - { - if (op->accepted_socket != INVALID_SOCKET) - { - ::closesocket(op->accepted_socket); - op->accepted_socket = INVALID_SOCKET; - } - - if (op->peer_wrapper) - { - op->peer_wrapper->close_internal(); - op->peer_wrapper = nullptr; - } - - op->cleanup_only(); - op->acceptor_ptr.reset(); - return; - } - - op->stop_cb.reset(); - - bool success = - (op->dwError == 0 && !op->cancelled.load(std::memory_order_acquire)); - - if (op->ec_out) - { - if (op->cancelled.load(std::memory_order_acquire)) - *op->ec_out = capy::error::canceled; - else if (op->dwError != 0) - *op->ec_out = iocp_make_err(op->dwError, /*accept_path=*/true); - else - *op->ec_out = {}; - } - - if (success && op->accepted_socket != INVALID_SOCKET && op->peer_wrapper) - { - ::setsockopt( - op->accepted_socket, SOL_SOCKET, SO_UPDATE_ACCEPT_CONTEXT, - reinterpret_cast(&op->listen_socket), sizeof(SOCKET)); - - op->peer_wrapper->get_internal()->set_socket(op->accepted_socket); - - sockaddr_storage local_storage{}; - int local_len = sizeof(local_storage); - sockaddr_storage remote_storage{}; - int remote_len = sizeof(remote_storage); - - endpoint local_ep, remote_ep; - if (::getsockname( - op->accepted_socket, - reinterpret_cast(&local_storage), &local_len) == 0) - local_ep = from_sockaddr(local_storage); - if (::getpeername( - op->accepted_socket, - reinterpret_cast(&remote_storage), &remote_len) == 0) - remote_ep = from_sockaddr(remote_storage); - - op->peer_wrapper->get_internal()->set_endpoints(local_ep, remote_ep); - op->accepted_socket = INVALID_SOCKET; - - if (op->impl_out) - *op->impl_out = op->peer_wrapper; - } - else - { - if (op->accepted_socket != INVALID_SOCKET) - { - ::closesocket(op->accepted_socket); - op->accepted_socket = INVALID_SOCKET; - } - - if (op->peer_wrapper) - { - op->acceptor_ptr->socket_service().destroy(op->peer_wrapper); - op->peer_wrapper = nullptr; - } - - if (op->impl_out) - *op->impl_out = nullptr; - } - - op->cont.h = op->h; - auto saved_ex = op->ex; - auto prevent_premature_destruction = std::move(op->acceptor_ptr); - - dispatch_coro(saved_ex, op->cont).resume(); -} - -// acceptor_wait_op completion handler - -inline void -acceptor_wait_op::do_complete( - void* owner, - scheduler_op* base, - std::uint32_t /*bytes*/, - std::uint32_t /*error*/) -{ - auto* op = static_cast(base); - - if (!owner) - { - op->cleanup_only(); - op->acceptor_ptr.reset(); - return; - } - - auto prevent_premature_destruction = std::move(op->acceptor_ptr); - op->invoke_handler(); -} - -// connect_op completion handler - -inline void -connect_op::do_complete( - void* owner, - scheduler_op* base, - std::uint32_t /*bytes*/, - std::uint32_t /*error*/) -{ - auto* op = static_cast(base); - - if (!owner) - { - op->cleanup_only(); - op->internal_ptr.reset(); - return; - } - - bool success = - (op->dwError == 0 && !op->cancelled.load(std::memory_order_acquire)); - if (success && op->internal.is_open()) - { - // Required after ConnectEx to enable shutdown(), getsockname(), etc. - ::setsockopt( - op->internal.native_handle(), SOL_SOCKET, SO_UPDATE_CONNECT_CONTEXT, - nullptr, 0); - - endpoint local_ep; - sockaddr_storage local_storage{}; - int local_len = sizeof(local_storage); - if (::getsockname( - op->internal.native_handle(), - reinterpret_cast(&local_storage), &local_len) == 0) - local_ep = from_sockaddr(local_storage); - op->internal.set_endpoints(local_ep, op->target_endpoint); - } - - auto prevent_premature_destruction = std::move(op->internal_ptr); - op->invoke_handler(); -} - -// read_op completion handler - -inline void -read_op::do_complete( - void* owner, - scheduler_op* base, - std::uint32_t /*bytes*/, - std::uint32_t /*error*/) -{ - auto* op = static_cast(base); - - if (!owner) - { - op->cleanup_only(); - op->internal_ptr.reset(); - return; - } - - auto prevent_premature_destruction = std::move(op->internal_ptr); - op->invoke_handler(); -} - -// write_op completion handler - -inline void -write_op::do_complete( - void* owner, - scheduler_op* base, - std::uint32_t /*bytes*/, - std::uint32_t /*error*/) -{ - auto* op = static_cast(base); - - if (!owner) - { - op->cleanup_only(); - op->internal_ptr.reset(); - return; - } - - auto prevent_premature_destruction = std::move(op->internal_ptr); - op->invoke_handler(); -} - -// wait_op completion handler - -inline void -wait_op::do_complete( - void* owner, - scheduler_op* base, - std::uint32_t /*bytes*/, - std::uint32_t /*error*/) -{ - auto* op = static_cast(base); - - if (!owner) - { - op->cleanup_only(); - op->internal_ptr.reset(); - return; - } - - // Only the zero-byte WSARecv read wait; write and error waits come - // from the poll reactor through this op too. - if (op->w == wait_type::read) - op->dwError = normalize_read_wait_error(op->dwError); - - auto prevent_premature_destruction = std::move(op->internal_ptr); - op->invoke_handler(); -} - -// win_tcp_socket_internal - -inline win_tcp_socket_internal::win_tcp_socket_internal( - win_tcp_service& svc) noexcept - : svc_(svc) - , conn_(*this) - , rd_(*this) - , wr_(*this) - , wt_(*this) -{ -} - -inline win_tcp_socket_internal::~win_tcp_socket_internal() -{ - svc_.unregister_impl(*this); -} - -inline SOCKET -win_tcp_socket_internal::native_handle() const noexcept -{ - return socket_; -} - -inline endpoint -win_tcp_socket_internal::local_endpoint() const noexcept -{ - return local_endpoint_; -} - -inline endpoint -win_tcp_socket_internal::remote_endpoint() const noexcept -{ - return remote_endpoint_; -} - -inline bool -win_tcp_socket_internal::is_open() const noexcept -{ - return socket_ != INVALID_SOCKET; -} - -inline void -win_tcp_socket_internal::set_socket(SOCKET s) noexcept -{ - socket_ = s; -} - -inline void -win_tcp_socket_internal::set_endpoints(endpoint local, endpoint remote) noexcept -{ - local_endpoint_ = local; - remote_endpoint_ = remote; -} - -inline std::coroutine_handle<> -win_tcp_socket_internal::connect( - std::coroutine_handle<> h, - capy::executor_ref d, - endpoint ep, - std::stop_token token, - std::error_code* ec) -{ - // Keep internal alive during I/O - conn_.internal_ptr = shared_from_this(); - - auto& op = conn_; - op.reset(); - op.h = h; - op.ex = d; - op.ec_out = ec; - op.target_endpoint = ep; - op.start(token); - - svc_.work_started(); - - // ConnectEx requires the socket to be bound. Skip if already bound - // (e.g. the caller used tcp_socket::bind() before connect). - if (local_endpoint_ == endpoint{}) - { - sockaddr_storage bind_storage{}; - socklen_t bind_len; - if (family_ == AF_INET6) - { - sockaddr_in6 sa6{}; - sa6.sin6_family = AF_INET6; - sa6.sin6_port = 0; - sa6.sin6_addr = in6addr_any; - std::memcpy(&bind_storage, &sa6, sizeof(sa6)); - bind_len = sizeof(sa6); - } - else - { - sockaddr_in sa4{}; - sa4.sin_family = AF_INET; - sa4.sin_addr.s_addr = INADDR_ANY; - sa4.sin_port = 0; - std::memcpy(&bind_storage, &sa4, sizeof(sa4)); - bind_len = sizeof(sa4); - } - - if (::bind( - socket_, reinterpret_cast(&bind_storage), - bind_len) == SOCKET_ERROR) - { - svc_.on_completion(&op, ::WSAGetLastError(), 0); - return std::noop_coroutine(); - } - } - - auto connect_ex = svc_.connect_ex(); - if (!connect_ex) - { - svc_.on_completion(&op, WSAEOPNOTSUPP, 0); - return std::noop_coroutine(); - } - - sockaddr_storage storage{}; - socklen_t addrlen = detail::to_sockaddr(ep, family_, storage); - - BOOL result = connect_ex( - socket_, reinterpret_cast(&storage), - static_cast(addrlen), nullptr, 0, nullptr, &op); - - if (!result) - { - DWORD err = ::WSAGetLastError(); - if (err != ERROR_IO_PENDING) - { - svc_.on_completion(&op, err, 0); - return std::noop_coroutine(); - } - } - - svc_.on_pending(&op); - return std::noop_coroutine(); -} - -inline std::coroutine_handle<> -win_tcp_socket_internal::read_some( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param param, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes_out) -{ - // Keep internal alive during I/O - rd_.internal_ptr = shared_from_this(); - - auto& op = rd_; - op.reset(); - op.is_read = true; - op.h = h; - op.ex = d; - op.ec_out = ec; - op.bytes_out = bytes_out; - op.start(token); - - svc_.work_started(); - - // Closed-object contract: complete with bad_file_descriptor - // without touching the kernel. - if (socket_ == INVALID_SOCKET) - { - svc_.on_completion(&op, WSAEBADF, 0); - return std::noop_coroutine(); - } - - // Prepare buffers - capy::mutable_buffer bufs[read_op::max_buffers]; - op.wsabuf_count = - static_cast(param.copy_to(bufs, read_op::max_buffers)); - - // Handle empty buffer: complete with 0 bytes - if (op.wsabuf_count == 0) - { - op.empty_buffer = true; - svc_.on_completion(&op, 0, 0); - return std::noop_coroutine(); - } - - for (DWORD i = 0; i < op.wsabuf_count; ++i) - { - op.wsabufs[i].buf = static_cast(bufs[i].data()); - op.wsabufs[i].len = static_cast( - (std::min)(bufs[i].size(), std::size_t(0x7fffffff))); - } - - op.flags = 0; - - int result = ::WSARecv( - socket_, op.wsabufs, op.wsabuf_count, nullptr, &op.flags, &op, nullptr); - - if (result == SOCKET_ERROR) - { - DWORD err = ::WSAGetLastError(); - if (err != WSA_IO_PENDING) - { - svc_.on_completion(&op, err, 0); - return std::noop_coroutine(); - } - } - - svc_.on_pending(&op); - - // Re-check cancellation after I/O is pending - if (op.cancelled.load(std::memory_order_acquire)) - ::CancelIoEx(reinterpret_cast(socket_), &op); - - return std::noop_coroutine(); -} - -inline std::coroutine_handle<> -win_tcp_socket_internal::write_some( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param param, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes_out) -{ - // Keep internal alive during I/O - wr_.internal_ptr = shared_from_this(); - - auto& op = wr_; - op.reset(); - op.h = h; - op.ex = d; - op.ec_out = ec; - op.bytes_out = bytes_out; - op.start(token); - - svc_.work_started(); - - // Closed-object contract: complete with bad_file_descriptor - // without touching the kernel. - if (socket_ == INVALID_SOCKET) - { - svc_.on_completion(&op, WSAEBADF, 0); - return std::noop_coroutine(); - } - - // Prepare buffers - capy::mutable_buffer bufs[write_op::max_buffers]; - op.wsabuf_count = - static_cast(param.copy_to(bufs, write_op::max_buffers)); - - // Handle empty buffer: complete immediately with 0 bytes - if (op.wsabuf_count == 0) - { - svc_.on_completion(&op, 0, 0); - return std::noop_coroutine(); - } - - for (DWORD i = 0; i < op.wsabuf_count; ++i) - { - op.wsabufs[i].buf = static_cast(bufs[i].data()); - op.wsabufs[i].len = static_cast( - (std::min)(bufs[i].size(), std::size_t(0x7fffffff))); - } - - int result = ::WSASend( - socket_, op.wsabufs, op.wsabuf_count, nullptr, 0, &op, nullptr); - - if (result == SOCKET_ERROR) - { - DWORD err = ::WSAGetLastError(); - if (err != WSA_IO_PENDING) - { - svc_.on_completion(&op, err, 0); - return std::noop_coroutine(); - } - } - - svc_.on_pending(&op); - - // Re-check cancellation after I/O is pending - if (op.cancelled.load(std::memory_order_acquire)) - ::CancelIoEx(reinterpret_cast(socket_), &op); - - return std::noop_coroutine(); -} - -inline std::coroutine_handle<> -win_tcp_socket_internal::wait( - std::coroutine_handle<> h, - capy::executor_ref d, - wait_type w, - std::stop_token token, - std::error_code* ec) -{ - wt_.internal_ptr = shared_from_this(); - - auto& op = wt_; - op.reset(); - op.h = h; - op.ex = d; - op.ec_out = ec; - op.bytes_out = nullptr; - op.empty_buffer = true; // skip EOF translation in invoke_handler - op.w = w; - op.start(token); - - svc_.work_started(); - - // Closed-object contract: complete with bad_file_descriptor - // without touching the kernel or the wait reactor. - if (socket_ == INVALID_SOCKET) - { - svc_.on_completion(&op, WSAEBADF, 0); - return std::noop_coroutine(); - } - - if (w == wait_type::read) - { - // Zero-byte WSARecv: kernel signals completion when data is - // available without consuming any bytes from the stream. This - // is the documented Winsock pattern for "is the socket - // readable" notifications. - op.wsabuf = WSABUF{0, nullptr}; - op.flags = 0; - - int result = - ::WSARecv(socket_, &op.wsabuf, 1, nullptr, &op.flags, &op, nullptr); - - if (result == SOCKET_ERROR) - { - DWORD err = ::WSAGetLastError(); - if (err != WSA_IO_PENDING) - { - svc_.on_completion(&op, err, 0); - return std::noop_coroutine(); - } - } - - svc_.on_pending(&op); - - // Re-check cancellation after I/O is pending. - if (op.cancelled.load(std::memory_order_acquire)) - ::CancelIoEx(reinterpret_cast(socket_), &op); - - return std::noop_coroutine(); - } - - // wait_type::write and wait_type::error: route through the - // auxiliary poll reactor. There is no overlapped primitive for - // "the send buffer has room" that does not also transfer bytes, - // and a write wait must report real writability. - svc_.scheduler().wait_reactor().register_wait(socket_, w, &op); - return std::noop_coroutine(); -} - -inline void -win_tcp_socket_internal::cancel() noexcept -{ - if (socket_ != INVALID_SOCKET) - { - ::CancelIoEx(reinterpret_cast(socket_), nullptr); - } - - conn_.request_cancel(); - rd_.request_cancel(); - wr_.request_cancel(); - wt_.request_cancel(); - // CancelIoEx covers overlapped I/O on the socket but cannot reach - // a wait op parked in the auxiliary reactor (no overlapped is - // outstanding). Route through the reactor explicitly. - svc_.scheduler().cancel_wait(&wt_); -} - -inline void -win_tcp_socket_internal::close_socket() noexcept -{ - // Flag every op cancelled before tearing down the handle. closesocket() - // can complete a pending overlapped op with ERROR_NETNAME_DELETED (not the - // MSDN-documented ERROR_OPERATION_ABORTED), which iocp_make_err now maps to - // connection_reset -- so a locally-closed op must be short-circuited to - // canceled via the flag rather than relying on the raw error, exactly as - // cancel() does. The store happens-before the completion is decoded. - conn_.request_cancel(); - rd_.request_cancel(); - wr_.request_cancel(); - // wt_ is parked in the aux reactor (no overlapped outstanding), so it also - // needs an explicit reactor deregister before the SOCKET handle is closed; - // otherwise the reactor would keep polling a dangling fd (and on a Winsock - // SOCKET-id reuse the wrong fd could be polled briefly). - wt_.request_cancel(); - svc_.scheduler().cancel_wait(&wt_); - - if (socket_ != INVALID_SOCKET) - { - ::CancelIoEx(reinterpret_cast(socket_), nullptr); - ::closesocket(socket_); - socket_ = INVALID_SOCKET; - } - - family_ = AF_UNSPEC; - - // Clear cached endpoints - local_endpoint_ = endpoint{}; - remote_endpoint_ = endpoint{}; -} - -// win_tcp_socket - -inline win_tcp_socket::win_tcp_socket( - std::shared_ptr internal) noexcept - : internal_(std::move(internal)) -{ -} - -inline void -win_tcp_socket::close_internal() noexcept -{ - if (internal_) - { - internal_->close_socket(); - internal_.reset(); - } -} - -inline std::coroutine_handle<> -win_tcp_socket::connect( - std::coroutine_handle<> h, - capy::executor_ref d, - endpoint ep, - std::stop_token token, - std::error_code* ec) -{ - return internal_->connect(h, d, ep, token, ec); -} - -inline std::coroutine_handle<> -win_tcp_socket::read_some( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->read_some(h, d, buf, token, ec, bytes); -} - -inline std::coroutine_handle<> -win_tcp_socket::write_some( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->write_some(h, d, buf, token, ec, bytes); -} - -inline std::coroutine_handle<> -win_tcp_socket::wait( - std::coroutine_handle<> h, - capy::executor_ref d, - wait_type w, - std::stop_token token, - std::error_code* ec) -{ - return internal_->wait(h, d, w, token, ec); -} - -inline std::error_code -win_tcp_socket::shutdown(tcp_socket::shutdown_type what) noexcept -{ - int how; - switch (what) - { - case tcp_socket::shutdown_receive: - how = SD_RECEIVE; - break; - case tcp_socket::shutdown_send: - how = SD_SEND; - break; - case tcp_socket::shutdown_both: - how = SD_BOTH; - break; - default: - return make_err(WSAEINVAL); - } - if (::shutdown(internal_->native_handle(), how) != 0) - return make_err(WSAGetLastError()); - return {}; -} - -inline native_handle_type -win_tcp_socket::native_handle() const noexcept -{ - return static_cast(internal_->native_handle()); -} - -inline native_handle_type -win_tcp_socket::release_socket() noexcept -{ - SOCKET s = internal_->socket_; - if (s != INVALID_SOCKET) - { - internal_->cancel(); - // Sever the port association so the descriptor can be - // adopted again; best-effort, the caller keeps a working - // socket either way. - dissociate_from_iocp(s); - internal_->socket_ = INVALID_SOCKET; - internal_->family_ = AF_UNSPEC; - internal_->local_endpoint_ = endpoint{}; - internal_->remote_endpoint_ = endpoint{}; - } - return static_cast(s); -} - -inline std::error_code -win_tcp_socket::set_option( - int level, int optname, void const* data, std::size_t size) noexcept -{ - if (::setsockopt( - internal_->native_handle(), level, optname, - reinterpret_cast(data), static_cast(size)) != 0) - return make_err(WSAGetLastError()); - return {}; -} - -inline std::error_code -win_tcp_socket::get_option( - int level, int optname, void* data, std::size_t* size) const noexcept -{ - int len = static_cast(*size); - if (::getsockopt( - internal_->native_handle(), level, optname, - reinterpret_cast(data), &len) != 0) - return make_err(WSAGetLastError()); - *size = static_cast(len); - return {}; -} - -inline endpoint -win_tcp_socket::local_endpoint() const noexcept -{ - return internal_->local_endpoint(); -} - -inline endpoint -win_tcp_socket::remote_endpoint() const noexcept -{ - return internal_->remote_endpoint(); -} - -inline void -win_tcp_socket::cancel() noexcept -{ - internal_->cancel(); -} - -inline win_tcp_socket_internal* -win_tcp_socket::get_internal() const noexcept -{ - return internal_.get(); -} - -// win_tcp_service - -inline win_tcp_service::win_tcp_service(capy::execution_context& ctx) - : sched_(ctx.use_service()) - , iocp_(sched_.native_handle()) -{ - load_extension_functions(); -} - -inline win_tcp_service::~win_tcp_service() -{ - // Delete wrappers that survived shutdown. This runs after - // win_scheduler is destroyed (reverse creation order), so - // all coroutine frames and their tcp_socket members are gone. - for (auto* w = socket_wrapper_list_.pop_front(); w != nullptr; - w = socket_wrapper_list_.pop_front()) - delete w; - - for (auto* w = acceptor_wrapper_list_.pop_front(); w != nullptr; - w = acceptor_wrapper_list_.pop_front()) - delete w; -} - -inline void -win_tcp_service::shutdown() -{ - std::lock_guard lock(mutex_); - - // Close all sockets to force pending I/O to complete via IOCP. - // Wrappers are NOT deleted here - coroutine frames destroyed - // during scheduler shutdown may still hold tcp_socket objects - // that reference them. Wrapper deletion is deferred to ~win_tcp_service - // after the scheduler has drained all outstanding operations. - for (auto* impl = socket_list_.pop_front(); impl != nullptr; - impl = socket_list_.pop_front()) - { - impl->close_socket(); - } - - for (auto* impl = acceptor_list_.pop_front(); impl != nullptr; - impl = acceptor_list_.pop_front()) - { - impl->close_socket(); - } -} - -inline io_object::implementation* -win_tcp_service::construct() -{ - auto internal = std::make_shared(*this); - - { - std::lock_guard lock(mutex_); - socket_list_.push_back(internal.get()); - } - - auto* wrapper = new win_tcp_socket(std::move(internal)); - - { - std::lock_guard lock(mutex_); - socket_wrapper_list_.push_back(wrapper); - } - - return wrapper; -} - -inline void -win_tcp_service::destroy(io_object::implementation* p) -{ - if (p) - { - auto& wrapper = static_cast(*p); - wrapper.close_internal(); - destroy_impl(wrapper); - } -} - -inline void -win_tcp_service::close(io_object::handle& h) -{ - auto& wrapper = static_cast(*h.get()); - wrapper.get_internal()->close_socket(); -} +// Cancellation functions inline void -win_tcp_service::destroy_impl(win_tcp_socket& impl) +accept_op::do_cancel_impl(overlapped_op* base) noexcept { + auto* op = static_cast(base); + if (op->listen_socket != INVALID_SOCKET) { - std::lock_guard lock(mutex_); - socket_wrapper_list_.remove(&impl); + ::CancelIoEx(reinterpret_cast(op->listen_socket), op); } - delete &impl; } inline void -win_tcp_service::unregister_impl(win_tcp_socket_internal& impl) -{ - std::lock_guard lock(mutex_); - socket_list_.remove(&impl); -} - -inline std::error_code -win_tcp_service::open_socket( - win_tcp_socket_internal& impl, int family, int type, int protocol) +acceptor_wait_op::do_cancel_impl(overlapped_op* base) noexcept { - impl.close_socket(); - - SOCKET sock = - ::WSASocketW(family, type, protocol, nullptr, 0, WSA_FLAG_OVERLAPPED); - - if (sock == INVALID_SOCKET) - return make_err(::WSAGetLastError()); - - if (family == AF_INET6) - { - DWORD one = 1; - ::setsockopt( - sock, IPPROTO_IPV6, IPV6_V6ONLY, reinterpret_cast(&one), - sizeof(one)); - } - - HANDLE result = ::CreateIoCompletionPort( - reinterpret_cast(sock), static_cast(iocp_), key_io, 0); - - if (result == nullptr) + auto* op = static_cast(base); + op->cancelled.store(true, std::memory_order_release); + if (op->listen_socket != INVALID_SOCKET) { - DWORD dwError = ::GetLastError(); - ::closesocket(sock); - return make_err(dwError); + ::CancelIoEx(reinterpret_cast(op->listen_socket), op); } - - impl.socket_ = sock; - impl.family_ = family; - return {}; -} - -inline std::error_code -win_tcp_service::assign_socket( - win_tcp_socket_internal& impl, native_handle_type fd) -{ - SOCKET sock = static_cast(fd); - if (sock == INVALID_SOCKET) - return make_err(WSAENOTSOCK); - - // SO_PROTOCOL_INFOW works on an unbound socket, unlike getsockname - // (WSAEINVAL until bind/connect names it) -- an adopted socket may - // have reached connected state without an explicit bind. - WSAPROTOCOL_INFOW proto_info{}; - int proto_len = sizeof(proto_info); - if (::getsockopt( - sock, SOL_SOCKET, SO_PROTOCOL_INFOW, - reinterpret_cast(&proto_info), &proto_len) != 0) - return make_err(::WSAGetLastError()); - if (proto_info.iAddressFamily != AF_INET && - proto_info.iAddressFamily != AF_INET6) - return make_err(WSAEAFNOSUPPORT); - if (proto_info.iSocketType != SOCK_STREAM) - return make_err(WSAEPROTOTYPE); - - HANDLE result = ::CreateIoCompletionPort( - reinterpret_cast(sock), static_cast(iocp_), key_io, 0); - if (result == nullptr) - return make_err(::GetLastError()); - - impl.socket_ = sock; - impl.family_ = proto_info.iAddressFamily; - - endpoint local_ep, remote_ep; - sockaddr_storage local_storage{}; - int local_len = sizeof(local_storage); - if (::getsockname( - sock, reinterpret_cast(&local_storage), &local_len) == 0) - local_ep = detail::from_sockaddr(local_storage); - sockaddr_storage remote_storage{}; - int remote_len = sizeof(remote_storage); - if (::getpeername( - sock, reinterpret_cast(&remote_storage), &remote_len) == - 0) - remote_ep = detail::from_sockaddr(remote_storage); - impl.set_endpoints(local_ep, remote_ep); - - return {}; -} - -inline std::error_code -win_tcp_service::bind_socket(win_tcp_socket_internal& impl, endpoint ep) -{ - SOCKET sock = impl.socket_; - - sockaddr_storage storage{}; - socklen_t addrlen = detail::to_sockaddr(ep, storage); - if (::bind( - sock, reinterpret_cast(&storage), - static_cast(addrlen)) == SOCKET_ERROR) - return make_err(::WSAGetLastError()); - - // Cache local endpoint (resolves ephemeral port) - sockaddr_storage local_storage{}; - int local_len = sizeof(local_storage); - if (::getsockname( - sock, reinterpret_cast(&local_storage), &local_len) == 0) - impl.local_endpoint_ = detail::from_sockaddr(local_storage); - - return {}; -} - -inline void* -win_tcp_service::native_handle() const noexcept -{ - return iocp_; -} - -inline void -win_tcp_service::post(overlapped_op* op) -{ - sched_.post(op); -} - -inline void -win_tcp_service::on_pending(overlapped_op* op) noexcept -{ - sched_.on_pending(op); + op->acceptor.socket_service().scheduler().cancel_wait(op); } -inline void -win_tcp_service::on_completion( - overlapped_op* op, DWORD error, DWORD bytes) noexcept -{ - sched_.on_completion(op, error, bytes); -} +// accept_op completion handler inline void -win_tcp_service::work_started() noexcept +accept_op::do_complete( + void* owner, + scheduler_op* base, + std::uint32_t /*bytes*/, + std::uint32_t /*error*/) { - sched_.work_started(); -} + auto* op = static_cast(base); -inline void -win_tcp_service::work_finished() noexcept -{ - sched_.work_finished(); -} + if (!owner) + { + if (op->accepted_socket != INVALID_SOCKET) + { + ::closesocket(op->accepted_socket); + op->accepted_socket = INVALID_SOCKET; + } -inline void -win_tcp_service::load_extension_functions() -{ - SOCKET sock = ::WSASocketW( - AF_INET, SOCK_STREAM, IPPROTO_TCP, nullptr, 0, WSA_FLAG_OVERLAPPED); + if (op->peer_wrapper) + { + op->acceptor.socket_service().destroy(op->peer_wrapper); + op->peer_wrapper = nullptr; + } - if (sock == INVALID_SOCKET) + op->cleanup_only(); + op->object_ref_.reset(); return; + } - DWORD bytes = 0; - - GUID connect_ex_guid = WSAID_CONNECTEX; - ::WSAIoctl( - sock, SIO_GET_EXTENSION_FUNCTION_POINTER, &connect_ex_guid, - sizeof(connect_ex_guid), &connect_ex_, sizeof(connect_ex_), &bytes, - nullptr, nullptr); - - GUID accept_ex_guid = WSAID_ACCEPTEX; - ::WSAIoctl( - sock, SIO_GET_EXTENSION_FUNCTION_POINTER, &accept_ex_guid, - sizeof(accept_ex_guid), &accept_ex_, sizeof(accept_ex_), &bytes, - nullptr, nullptr); + op->stop_cb.reset(); - ::closesocket(sock); -} + bool success = + (op->dwError == 0 && !op->cancelled.load(std::memory_order_acquire)); -inline void -win_tcp_service::destroy_acceptor_impl(win_tcp_acceptor& impl) -{ + if (op->ec_out) { - std::lock_guard lock(mutex_); - acceptor_wrapper_list_.remove(&impl); + if (op->cancelled.load(std::memory_order_acquire)) + *op->ec_out = capy::error::canceled; + else if (op->dwError != 0) + *op->ec_out = iocp_make_err(op->dwError, /*accept_path=*/true); + else + *op->ec_out = {}; } - delete &impl; -} - -inline void -win_tcp_service::unregister_acceptor_impl(win_tcp_acceptor_internal& impl) -{ - std::lock_guard lock(mutex_); - acceptor_list_.remove(&impl); -} - -inline std::error_code -win_tcp_service::open_acceptor_socket( - win_tcp_acceptor_internal& impl, int family, int type, int protocol) -{ - impl.close_socket(); - - SOCKET sock = - ::WSASocketW(family, type, protocol, nullptr, 0, WSA_FLAG_OVERLAPPED); - - if (sock == INVALID_SOCKET) - return make_err(::WSAGetLastError()); - if (family == AF_INET6) + if (success && op->accepted_socket != INVALID_SOCKET && op->peer_wrapper) { - DWORD val = 0; // dual-stack default ::setsockopt( - sock, IPPROTO_IPV6, IPV6_V6ONLY, reinterpret_cast(&val), - sizeof(val)); - } - - HANDLE result = ::CreateIoCompletionPort( - reinterpret_cast(sock), static_cast(iocp_), key_io, 0); + op->accepted_socket, SOL_SOCKET, SO_UPDATE_ACCEPT_CONTEXT, + reinterpret_cast(&op->listen_socket), sizeof(SOCKET)); - if (result == nullptr) - { - DWORD dwError = ::GetLastError(); - ::closesocket(sock); - return make_err(dwError); - } + op->peer_wrapper->set_socket(op->accepted_socket); - impl.socket_ = sock; - impl.family_ = family; - return {}; -} + sockaddr_storage local_storage{}; + int local_len = sizeof(local_storage); + sockaddr_storage remote_storage{}; + int remote_len = sizeof(remote_storage); -inline std::error_code -win_tcp_service::assign_acceptor_socket( - win_tcp_acceptor_internal& impl, native_handle_type fd) -{ - SOCKET sock = static_cast(fd); - if (sock == INVALID_SOCKET) - return make_err(WSAENOTSOCK); - - // SO_PROTOCOL_INFOW works on an unbound socket, unlike getsockname - // (WSAEINVAL until bind names it). - WSAPROTOCOL_INFOW proto_info{}; - int proto_len = sizeof(proto_info); - if (::getsockopt( - sock, SOL_SOCKET, SO_PROTOCOL_INFOW, - reinterpret_cast(&proto_info), &proto_len) != 0) - return make_err(::WSAGetLastError()); - if (proto_info.iAddressFamily != AF_INET && - proto_info.iAddressFamily != AF_INET6) - return make_err(WSAEAFNOSUPPORT); - if (proto_info.iSocketType != SOCK_STREAM) - return make_err(WSAEPROTOTYPE); + endpoint local_ep, remote_ep; + if (::getsockname( + op->accepted_socket, + reinterpret_cast(&local_storage), &local_len) == 0) + local_ep = from_sockaddr(local_storage); + if (::getpeername( + op->accepted_socket, + reinterpret_cast(&remote_storage), &remote_len) == 0) + remote_ep = from_sockaddr(remote_storage); - HANDLE result = ::CreateIoCompletionPort( - reinterpret_cast(sock), static_cast(iocp_), key_io, 0); - if (result == nullptr) - return make_err(::GetLastError()); + op->peer_wrapper->set_endpoints(local_ep, remote_ep); + op->accepted_socket = INVALID_SOCKET; - impl.socket_ = sock; - impl.family_ = proto_info.iAddressFamily; + if (op->impl_out) + *op->impl_out = op->peer_wrapper; + // Handed off to the caller: don't let a failed early return on + // the next accept() see this (now user-owned) pointer again. + op->peer_wrapper = nullptr; + } + else + { + if (op->accepted_socket != INVALID_SOCKET) + { + ::closesocket(op->accepted_socket); + op->accepted_socket = INVALID_SOCKET; + } - // AcceptEx sizes its address buffers from this cache, so an - // unseeded endpoint breaks accepts on an adopted v6 listener. - sockaddr_storage local_storage{}; - int local_len = sizeof(local_storage); - if (::getsockname( - sock, reinterpret_cast(&local_storage), &local_len) == 0) - impl.set_local_endpoint(detail::from_sockaddr(local_storage)); + if (op->peer_wrapper) + { + op->acceptor.socket_service().destroy(op->peer_wrapper); + op->peer_wrapper = nullptr; + } - return {}; -} + if (op->impl_out) + *op->impl_out = nullptr; + } -inline std::error_code -win_tcp_service::bind_acceptor(win_tcp_acceptor_internal& impl, endpoint ep) -{ - SOCKET sock = impl.socket_; - - sockaddr_storage storage{}; - socklen_t addrlen = detail::to_sockaddr(ep, storage); - if (::bind( - sock, reinterpret_cast(&storage), - static_cast(addrlen)) == SOCKET_ERROR) - return make_err(::WSAGetLastError()); - - // Cache local endpoint (resolves ephemeral port) - sockaddr_storage local_storage{}; - int local_len = sizeof(local_storage); - if (::getsockname( - sock, reinterpret_cast(&local_storage), &local_len) == 0) - impl.set_local_endpoint(detail::from_sockaddr(local_storage)); + op->cont.h = op->h; + auto saved_ex = op->ex; + auto prevent_premature_destruction = std::move(op->object_ref_); - return {}; + dispatch_coro(saved_ex, op->cont).resume(); } -inline std::error_code -win_tcp_service::listen_acceptor(win_tcp_acceptor_internal& impl, int backlog) +// acceptor_wait_op completion handler + +inline void +acceptor_wait_op::do_complete( + void* owner, + scheduler_op* base, + std::uint32_t /*bytes*/, + std::uint32_t /*error*/) { - SOCKET sock = impl.socket_; + auto* op = static_cast(base); - if (::listen(sock, backlog) == SOCKET_ERROR) - return make_err(::WSAGetLastError()); + if (!owner) + { + op->cleanup_only(); + op->object_ref_.reset(); + return; + } - return {}; + auto prevent_premature_destruction = std::move(op->object_ref_); + op->invoke_handler(); } -// win_tcp_acceptor_internal +// win_tcp_acceptor -inline win_tcp_acceptor_internal::win_tcp_acceptor_internal( - win_tcp_service& svc) noexcept +inline win_tcp_acceptor::win_tcp_acceptor(win_tcp_service& svc) noexcept : svc_(svc) + , acc_(*this) + , wt_(*this) { } -inline win_tcp_acceptor_internal::~win_tcp_acceptor_internal() +inline void +win_tcp_acceptor::reuse() noexcept { - svc_.unregister_acceptor_impl(*this); + BOOST_COROSIO_ASSERT(socket_ == INVALID_SOCKET); + BOOST_COROSIO_ASSERT(family_ == AF_UNSPEC); + BOOST_COROSIO_ASSERT(local_endpoint_ == endpoint{}); + BOOST_COROSIO_ASSERT(!acc_.stop_cb); + BOOST_COROSIO_ASSERT(!wt_.stop_cb); + BOOST_COROSIO_ASSERT(acc_.peer_wrapper == nullptr); + BOOST_COROSIO_ASSERT(acc_.accepted_socket == INVALID_SOCKET); } inline win_tcp_service& -win_tcp_acceptor_internal::socket_service() noexcept +win_tcp_acceptor::socket_service() noexcept { return svc_; } -inline SOCKET -win_tcp_acceptor_internal::native_handle() const noexcept +inline native_handle_type +win_tcp_acceptor::native_handle() const noexcept { - return socket_; + return static_cast(socket_); } inline endpoint -win_tcp_acceptor_internal::local_endpoint() const noexcept +win_tcp_acceptor::local_endpoint() const noexcept { return local_endpoint_; } inline bool -win_tcp_acceptor_internal::is_open() const noexcept +win_tcp_acceptor::is_open() const noexcept { return socket_ != INVALID_SOCKET; } inline void -win_tcp_acceptor_internal::set_local_endpoint(endpoint ep) noexcept +win_tcp_acceptor::set_local_endpoint(endpoint ep) noexcept { local_endpoint_ = ep; } inline std::coroutine_handle<> -win_tcp_acceptor_internal::accept( +win_tcp_acceptor::accept( std::coroutine_handle<> h, capy::executor_ref d, std::stop_token token, std::error_code* ec, io_object::implementation** impl_out) { - // Keep acceptor internal alive during I/O - acc_.acceptor_ptr = shared_from_this(); + // Keep this acceptor alive during I/O + acc_.object_ref_ = detail::object_ref(this); auto& op = acc_; op.reset(); - op.h = h; - op.ex = d; - op.ec_out = ec; - op.impl_out = impl_out; + // reset() doesn't touch these -- a value left over from a prior + // accept() (e.g. the socket already handed to the user via + // *impl_out) must not survive into a completion this call fails + // before reassigning; do_complete's failure branch would destroy() + // whatever stale pointer it finds here. + op.peer_wrapper = nullptr; + op.accepted_socket = INVALID_SOCKET; + op.h = h; + op.ex = d; + op.ec_out = ec; + op.impl_out = impl_out; op.start(token); svc_.work_started(); - // Create wrapper for the peer socket (service owns it) + // Create a (possibly recycled) peer socket; the service owns it. auto& peer_wrapper = static_cast(*svc_.construct()); // Derive AF from the listening socket's cached local endpoint @@ -1535,14 +402,14 @@ win_tcp_acceptor_internal::accept( } inline std::coroutine_handle<> -win_tcp_acceptor_internal::wait( +win_tcp_acceptor::wait( std::coroutine_handle<> h, capy::executor_ref d, wait_type w, std::stop_token token, std::error_code* ec) { - wt_.acceptor_ptr = shared_from_this(); + wt_.object_ref_ = detail::object_ref(this); wt_.listen_socket = socket_; auto& op = wt_; @@ -1578,7 +445,7 @@ win_tcp_acceptor_internal::wait( } inline void -win_tcp_acceptor_internal::cancel() noexcept +win_tcp_acceptor::cancel() noexcept { if (socket_ != INVALID_SOCKET) { @@ -1591,7 +458,7 @@ win_tcp_acceptor_internal::cancel() noexcept } inline void -win_tcp_acceptor_internal::close_socket() noexcept +win_tcp_acceptor::close_socket() noexcept { // Flag the accept op cancelled before closing so a closesocket-delivered // ERROR_NETNAME_DELETED is short-circuited to canceled rather than mapped @@ -1614,85 +481,17 @@ win_tcp_acceptor_internal::close_socket() noexcept local_endpoint_ = endpoint{}; } -// win_tcp_acceptor - -inline win_tcp_acceptor::win_tcp_acceptor( - std::shared_ptr internal) noexcept - : internal_(std::move(internal)) -{ -} - -inline void -win_tcp_acceptor::close_internal() noexcept -{ - if (internal_) - { - internal_->close_socket(); - internal_.reset(); - } -} - -inline std::coroutine_handle<> -win_tcp_acceptor::accept( - std::coroutine_handle<> h, - capy::executor_ref d, - std::stop_token token, - std::error_code* ec, - io_object::implementation** impl_out) -{ - return internal_->accept(h, d, token, ec, impl_out); -} - -inline std::coroutine_handle<> -win_tcp_acceptor::wait( - std::coroutine_handle<> h, - capy::executor_ref d, - wait_type w, - std::stop_token token, - std::error_code* ec) -{ - return internal_->wait(h, d, w, token, ec); -} - -inline endpoint -win_tcp_acceptor::local_endpoint() const noexcept -{ - return internal_->local_endpoint(); -} - -inline bool -win_tcp_acceptor::is_open() const noexcept -{ - return internal_ && internal_->is_open(); -} - -inline void -win_tcp_acceptor::cancel() noexcept -{ - internal_->cancel(); -} - -inline native_handle_type -win_tcp_acceptor::native_handle() const noexcept -{ - if (!internal_) - return static_cast(INVALID_SOCKET); - return static_cast(internal_->native_handle()); -} - inline native_handle_type win_tcp_acceptor::release_socket() noexcept { - if (!internal_) - return static_cast(INVALID_SOCKET); - SOCKET s = internal_->socket_; + SOCKET s = socket_; if (s != INVALID_SOCKET) { - internal_->cancel(); + cancel(); dissociate_from_iocp(s); - internal_->socket_ = INVALID_SOCKET; - internal_->family_ = AF_UNSPEC; - internal_->local_endpoint_ = endpoint{}; + socket_ = INVALID_SOCKET; + family_ = AF_UNSPEC; + local_endpoint_ = endpoint{}; } return static_cast(s); } @@ -1702,8 +501,8 @@ win_tcp_acceptor::set_option( int level, int optname, void const* data, std::size_t size) noexcept { if (::setsockopt( - internal_->native_handle(), level, optname, - reinterpret_cast(data), static_cast(size)) != 0) + socket_, level, optname, reinterpret_cast(data), + static_cast(size)) != 0) return make_err(WSAGetLastError()); return {}; } @@ -1714,17 +513,16 @@ win_tcp_acceptor::get_option( { int len = static_cast(*size); if (::getsockopt( - internal_->native_handle(), level, optname, - reinterpret_cast(data), &len) != 0) + socket_, level, optname, reinterpret_cast(data), &len) != 0) return make_err(WSAGetLastError()); *size = static_cast(len); return {}; } -inline win_tcp_acceptor_internal* -win_tcp_acceptor::get_internal() const noexcept +inline void +win_tcp_acceptor::retire() noexcept { - return internal_.get(); + svc_.acceptor_pool_.recycle(this); } // win_tcp_acceptor_service @@ -1738,77 +536,61 @@ inline win_tcp_acceptor_service::win_tcp_acceptor_service( inline io_object::implementation* win_tcp_acceptor_service::construct() { - auto internal = std::make_shared(svc_); - - { - std::lock_guard lock(svc_.mutex_); - svc_.acceptor_list_.push_back(internal.get()); - } - - auto* wrapper = new win_tcp_acceptor(std::move(internal)); - - { - std::lock_guard lock(svc_.mutex_); - svc_.acceptor_wrapper_list_.push_back(wrapper); - } - - return wrapper; + return svc_.acquire_acceptor_impl(); } inline void win_tcp_acceptor_service::destroy(io_object::implementation* p) { - if (p) - { - auto& wrapper = static_cast(*p); - wrapper.close_internal(); - svc_.destroy_acceptor_impl(wrapper); - } + if (!p) + return; + auto* a = static_cast(p); + a->close_socket(); + release(a); } inline void win_tcp_acceptor_service::close(io_object::handle& h) { - auto& wrapper = static_cast(*h.get()); - wrapper.get_internal()->close_socket(); + static_cast(h.get())->close_socket(); } inline std::error_code win_tcp_acceptor_service::open_acceptor_socket( tcp_acceptor::implementation& impl, int family, int type, int protocol) { - auto& wrapper = static_cast(impl); - return svc_.open_acceptor_socket( - *wrapper.get_internal(), family, type, protocol); + auto& acc = static_cast(impl); + return svc_.open_acceptor_socket(acc, family, type, protocol); } inline std::error_code win_tcp_acceptor_service::assign_socket( tcp_acceptor::implementation& impl, native_handle_type fd) { - auto& wrapper = static_cast(impl); - return svc_.assign_acceptor_socket(*wrapper.get_internal(), fd); + auto& acc = static_cast(impl); + return svc_.assign_acceptor_socket(acc, fd); } inline std::error_code win_tcp_acceptor_service::bind_acceptor( tcp_acceptor::implementation& impl, endpoint ep) { - auto& wrapper = static_cast(impl); - return svc_.bind_acceptor(*wrapper.get_internal(), ep); + auto& acc = static_cast(impl); + return svc_.bind_acceptor(acc, ep); } inline std::error_code win_tcp_acceptor_service::listen_acceptor( tcp_acceptor::implementation& impl, int backlog) { - auto& wrapper = static_cast(impl); - return svc_.listen_acceptor(*wrapper.get_internal(), backlog); + auto& acc = static_cast(impl); + return svc_.listen_acceptor(acc, backlog); } inline void win_tcp_acceptor_service::shutdown() { + // Socket/acceptor shutdown is handled by win_tcp_service::shutdown() } } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/iocp/win_tcp_service.hpp b/include/boost/corosio/native/detail/iocp/win_tcp_service.hpp index 2203b2fa0..e0b1c8b55 100644 --- a/include/boost/corosio/native/detail/iocp/win_tcp_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_tcp_service.hpp @@ -16,13 +16,18 @@ #if BOOST_COROSIO_HAS_IOCP #include +#include +#include #include #include -#include +#include #include #include +#include +#include #include +#include #include @@ -30,18 +35,18 @@ namespace boost::corosio::detail { class win_scheduler; class win_tcp_acceptor; -class win_tcp_acceptor_internal; class win_tcp_acceptor_service; /** Windows IOCP socket management service. - This service owns all socket implementations and coordinates their - lifecycle with the IOCP. It provides: + This service owns all socket and acceptor implementations and + coordinates their lifecycle with the IOCP. It provides: - - Socket implementation allocation and deallocation + - Socket/acceptor implementation recycling through `object_pool` - IOCP handle association for sockets - Function pointer loading for ConnectEx/AcceptEx - - Graceful shutdown - destroys all implementations when io_context stops + - Graceful shutdown - closes every live implementation when the + io_context stops; pool destruction frees whatever is left @par Thread Safety All public member functions are thread-safe. @@ -53,6 +58,9 @@ class BOOST_COROSIO_DECL win_tcp_service final , public capy::execution_context::service , public io_object::io_service { + friend class win_tcp_socket; + friend class win_tcp_acceptor; + public: using key_type = win_tcp_service; @@ -71,32 +79,19 @@ class BOOST_COROSIO_DECL win_tcp_service final */ explicit win_tcp_service(capy::execution_context& ctx); - /** Destroy the socket service. */ - ~win_tcp_service(); - win_tcp_service(win_tcp_service const&) = delete; win_tcp_service& operator=(win_tcp_service const&) = delete; /** Shut down the service. */ void shutdown() override; - /** Destroy a socket implementation wrapper. - Removes from tracking list and deletes. - */ - void destroy_impl(win_tcp_socket& impl); - - /** Unregister a socket implementation from the service list. - Called by the internal impl destructor. - */ - void unregister_impl(win_tcp_socket_internal& impl); - /** Create and register a socket with the IOCP. - @param impl The socket implementation internal to initialize. + @param impl The socket implementation to initialize. @return Error code, or success. */ std::error_code open_socket( - win_tcp_socket_internal& impl, int family, int type, int protocol); + win_tcp_socket& impl, int family, int type, int protocol); /** Adopt an existing socket handle into an implementation. @@ -105,30 +100,24 @@ class BOOST_COROSIO_DECL win_tcp_service final impl takes ownership and will close the handle; on failure the caller retains ownership. - @param impl The socket implementation internal to assign to. + @param impl The socket implementation to assign to. @param fd The native socket handle to adopt. @return Error code, or success. */ - std::error_code - assign_socket(win_tcp_socket_internal& impl, native_handle_type fd); + std::error_code assign_socket(win_tcp_socket& impl, native_handle_type fd); /** Bind a stream socket to a local endpoint. - @param impl The socket implementation internal to bind. + @param impl The socket implementation to bind. @param ep The local endpoint to bind to. @return Error code, or success. */ - std::error_code bind_socket(win_tcp_socket_internal& impl, endpoint ep); - - /** Destroy an acceptor implementation wrapper. - Removes from tracking list and deletes. - */ - void destroy_acceptor_impl(win_tcp_acceptor& impl); + std::error_code bind_socket(win_tcp_socket& impl, endpoint ep); - /** Unregister an acceptor implementation from the service list. - Called by the internal impl destructor. + /** Pop a recycled acceptor or news one, with the service reference + held. Used by `win_tcp_acceptor_service::construct()`. */ - void unregister_acceptor_impl(win_tcp_acceptor_internal& impl); + win_tcp_acceptor* acquire_acceptor_impl(); /** Create an acceptor socket without binding or listening. @@ -136,14 +125,14 @@ class BOOST_COROSIO_DECL win_tcp_service final For IPv6, dual-stack is enabled by default. Does not set SO_REUSEADDR. - @param impl The acceptor implementation internal to initialize. + @param impl The acceptor implementation to initialize. @param family Address family (e.g. `AF_INET`, `AF_INET6`). @param type Socket type (e.g. `SOCK_STREAM`). @param protocol Protocol number (e.g. `IPPROTO_TCP`). @return Error code, or success. */ std::error_code open_acceptor_socket( - win_tcp_acceptor_internal& impl, int family, int type, int protocol); + win_tcp_acceptor& impl, int family, int type, int protocol); /** Adopt an existing listening socket into an acceptor. @@ -151,30 +140,29 @@ class BOOST_COROSIO_DECL win_tcp_service final then releases the socket the acceptor already held. Listen state is not verified. - @param impl The acceptor implementation internal. + @param impl The acceptor implementation. @param fd The native socket to adopt. Ownership transfers only on success. @return Error code, or success. */ - std::error_code assign_acceptor_socket( - win_tcp_acceptor_internal& impl, native_handle_type fd); + std::error_code + assign_acceptor_socket(win_tcp_acceptor& impl, native_handle_type fd); /** Bind an open acceptor to a local endpoint. - @param impl The acceptor implementation internal. + @param impl The acceptor implementation. @param ep The local endpoint to bind to. @return Error code, or success. */ - std::error_code bind_acceptor(win_tcp_acceptor_internal& impl, endpoint ep); + std::error_code bind_acceptor(win_tcp_acceptor& impl, endpoint ep); /** Start listening for incoming connections. - @param impl The acceptor implementation internal. + @param impl The acceptor implementation. @param backlog The listen backlog. @return Error code, or success. */ - std::error_code - listen_acceptor(win_tcp_acceptor_internal& impl, int backlog); + std::error_code listen_acceptor(win_tcp_acceptor& impl, int backlog); /** Return the IOCP handle. */ void* native_handle() const noexcept; @@ -213,24 +201,1018 @@ class BOOST_COROSIO_DECL win_tcp_service final } private: - friend class win_tcp_acceptor_service; - void load_extension_functions(); win_scheduler& sched_; - BOOST_COROSIO_MSVC_WARNING_PUSH - BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // detail:: members, dll-interface - win_mutex mutex_; - intrusive_list socket_list_; - intrusive_list acceptor_list_; - intrusive_list socket_wrapper_list_; - intrusive_list acceptor_wrapper_list_; - BOOST_COROSIO_MSVC_WARNING_POP void* iocp_; LPFN_CONNECTEX connect_ex_ = nullptr; LPFN_ACCEPTEX accept_ex_ = nullptr; + BOOST_COROSIO_MSVC_WARNING_PUSH + BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // detail:: members, dll-interface + object_pool pool_; + object_pool acceptor_pool_; + BOOST_COROSIO_MSVC_WARNING_POP }; +// Operation constructors + +inline connect_op::connect_op(win_tcp_socket& internal_) noexcept + : overlapped_op(&do_complete) + , internal(internal_) +{ + cancel_func_ = &do_cancel_impl; +} + +inline read_op::read_op(win_tcp_socket& internal_) noexcept + : overlapped_op(&do_complete) + , internal(internal_) +{ + cancel_func_ = &do_cancel_impl; +} + +inline write_op::write_op(win_tcp_socket& internal_) noexcept + : overlapped_op(&do_complete) + , internal(internal_) +{ + cancel_func_ = &do_cancel_impl; +} + +inline wait_op::wait_op(win_tcp_socket& internal_) noexcept + : overlapped_op(&do_complete) + , internal(internal_) +{ + cancel_func_ = &do_cancel_impl; +} + +// Cancellation functions + +inline void +connect_op::do_cancel_impl(overlapped_op* base) noexcept +{ + auto* op = static_cast(base); + if (op->internal.socket_ != INVALID_SOCKET) + { + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); + } +} + +inline void +read_op::do_cancel_impl(overlapped_op* base) noexcept +{ + auto* op = static_cast(base); + op->cancelled.store(true, std::memory_order_release); + if (op->internal.socket_ != INVALID_SOCKET) + { + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); + } +} + +inline void +write_op::do_cancel_impl(overlapped_op* base) noexcept +{ + auto* op = static_cast(base); + op->cancelled.store(true, std::memory_order_release); + if (op->internal.socket_ != INVALID_SOCKET) + { + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); + } +} + +inline void +wait_op::do_cancel_impl(overlapped_op* base) noexcept +{ + auto* op = static_cast(base); + op->cancelled.store(true, std::memory_order_release); + // Best-effort cancel of any pending zero-byte WSARecv issued for + // wait_type::read. ERROR_NOT_FOUND when nothing overlapped is + // pending is harmless. + if (op->internal.socket_ != INVALID_SOCKET) + { + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); + } + // wait_type::error parks the op in the auxiliary select reactor; + // wake it so the reactor can post a cancelled completion. A cancel + // for an op the reactor never registered finds nothing and returns. + op->internal.svc_.scheduler().cancel_wait(op); +} + +// connect_op completion handler + +inline void +connect_op::do_complete( + void* owner, + scheduler_op* base, + std::uint32_t /*bytes*/, + std::uint32_t /*error*/) +{ + auto* op = static_cast(base); + + if (!owner) + { + op->cleanup_only(); + op->object_ref_.reset(); + return; + } + + bool success = + (op->dwError == 0 && !op->cancelled.load(std::memory_order_acquire)); + if (success && op->internal.socket_ != INVALID_SOCKET) + { + // Required after ConnectEx to enable shutdown(), getsockname(), etc. + ::setsockopt( + op->internal.socket_, SOL_SOCKET, SO_UPDATE_CONNECT_CONTEXT, + nullptr, 0); + + endpoint local_ep; + sockaddr_storage local_storage{}; + int local_len = sizeof(local_storage); + if (::getsockname( + op->internal.socket_, + reinterpret_cast(&local_storage), &local_len) == 0) + local_ep = from_sockaddr(local_storage); + op->internal.set_endpoints(local_ep, op->target_endpoint); + } + + auto prevent_premature_destruction = std::move(op->object_ref_); + op->invoke_handler(); +} + +// read_op completion handler + +inline void +read_op::do_complete( + void* owner, + scheduler_op* base, + std::uint32_t /*bytes*/, + std::uint32_t /*error*/) +{ + auto* op = static_cast(base); + + if (!owner) + { + op->cleanup_only(); + op->object_ref_.reset(); + return; + } + + auto prevent_premature_destruction = std::move(op->object_ref_); + op->invoke_handler(); +} + +// write_op completion handler + +inline void +write_op::do_complete( + void* owner, + scheduler_op* base, + std::uint32_t /*bytes*/, + std::uint32_t /*error*/) +{ + auto* op = static_cast(base); + + if (!owner) + { + op->cleanup_only(); + op->object_ref_.reset(); + return; + } + + auto prevent_premature_destruction = std::move(op->object_ref_); + op->invoke_handler(); +} + +// wait_op completion handler + +inline void +wait_op::do_complete( + void* owner, + scheduler_op* base, + std::uint32_t /*bytes*/, + std::uint32_t /*error*/) +{ + auto* op = static_cast(base); + + if (!owner) + { + op->cleanup_only(); + op->object_ref_.reset(); + return; + } + + // Only the zero-byte WSARecv read wait; write and error waits come + // from the poll reactor through this op too. + if (op->w == wait_type::read) + op->dwError = normalize_read_wait_error(op->dwError); + + auto prevent_premature_destruction = std::move(op->object_ref_); + op->invoke_handler(); +} + +// win_tcp_socket + +inline win_tcp_socket::win_tcp_socket(win_tcp_service& svc) noexcept + : svc_(svc) + , conn_(*this) + , rd_(*this) + , wr_(*this) + , wt_(*this) +{ +} + +inline void +win_tcp_socket::retire() noexcept +{ + svc_.pool_.recycle(this); +} + +inline void +win_tcp_socket::reuse() noexcept +{ + BOOST_COROSIO_ASSERT(socket_ == INVALID_SOCKET); + BOOST_COROSIO_ASSERT(family_ == AF_UNSPEC); + // A connect completing on another thread while the socket closed + // can still have written these. + local_endpoint_ = endpoint{}; + remote_endpoint_ = endpoint{}; + BOOST_COROSIO_ASSERT(!conn_.stop_cb); + BOOST_COROSIO_ASSERT(!rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wr_.stop_cb); + BOOST_COROSIO_ASSERT(!wt_.stop_cb); +} + +inline void +win_tcp_socket::set_socket(SOCKET s) noexcept +{ + socket_ = s; +} + +inline void +win_tcp_socket::set_endpoints(endpoint local, endpoint remote) noexcept +{ + local_endpoint_ = local; + remote_endpoint_ = remote; +} + +inline std::coroutine_handle<> +win_tcp_socket::connect( + std::coroutine_handle<> h, + capy::executor_ref d, + endpoint ep, + std::stop_token token, + std::error_code* ec) +{ + // Keep this socket alive during I/O + conn_.object_ref_ = detail::object_ref(this); + + auto& op = conn_; + op.reset(); + op.h = h; + op.ex = d; + op.ec_out = ec; + op.target_endpoint = ep; + op.start(token); + + svc_.work_started(); + + // ConnectEx requires the socket to be bound. Skip if already bound + // (e.g. the caller used tcp_socket::bind() before connect). + if (local_endpoint_ == endpoint{}) + { + sockaddr_storage bind_storage{}; + socklen_t bind_len; + if (family_ == AF_INET6) + { + sockaddr_in6 sa6{}; + sa6.sin6_family = AF_INET6; + sa6.sin6_port = 0; + sa6.sin6_addr = in6addr_any; + std::memcpy(&bind_storage, &sa6, sizeof(sa6)); + bind_len = sizeof(sa6); + } + else + { + sockaddr_in sa4{}; + sa4.sin_family = AF_INET; + sa4.sin_addr.s_addr = INADDR_ANY; + sa4.sin_port = 0; + std::memcpy(&bind_storage, &sa4, sizeof(sa4)); + bind_len = sizeof(sa4); + } + + if (::bind( + socket_, reinterpret_cast(&bind_storage), + bind_len) == SOCKET_ERROR) + { + svc_.on_completion(&op, ::WSAGetLastError(), 0); + return std::noop_coroutine(); + } + } + + auto connect_ex = svc_.connect_ex(); + if (!connect_ex) + { + svc_.on_completion(&op, WSAEOPNOTSUPP, 0); + return std::noop_coroutine(); + } + + sockaddr_storage storage{}; + socklen_t addrlen = detail::to_sockaddr(ep, family_, storage); + + BOOL result = connect_ex( + socket_, reinterpret_cast(&storage), + static_cast(addrlen), nullptr, 0, nullptr, &op); + + if (!result) + { + DWORD err = ::WSAGetLastError(); + if (err != ERROR_IO_PENDING) + { + svc_.on_completion(&op, err, 0); + return std::noop_coroutine(); + } + } + + svc_.on_pending(&op); + return std::noop_coroutine(); +} + +inline std::coroutine_handle<> +win_tcp_socket::read_some( + std::coroutine_handle<> h, + capy::executor_ref d, + buffer_param param, + std::stop_token token, + std::error_code* ec, + std::size_t* bytes_out) +{ + // Keep this socket alive during I/O + rd_.object_ref_ = detail::object_ref(this); + + auto& op = rd_; + op.reset(); + op.is_read = true; + op.h = h; + op.ex = d; + op.ec_out = ec; + op.bytes_out = bytes_out; + op.start(token); + + svc_.work_started(); + + // Closed-object contract: complete with bad_file_descriptor + // without touching the kernel. + if (socket_ == INVALID_SOCKET) + { + svc_.on_completion(&op, WSAEBADF, 0); + return std::noop_coroutine(); + } + + // Prepare buffers + capy::mutable_buffer bufs[read_op::max_buffers]; + op.wsabuf_count = + static_cast(param.copy_to(bufs, read_op::max_buffers)); + + // Handle empty buffer: complete with 0 bytes + if (op.wsabuf_count == 0) + { + op.empty_buffer = true; + svc_.on_completion(&op, 0, 0); + return std::noop_coroutine(); + } + + for (DWORD i = 0; i < op.wsabuf_count; ++i) + { + op.wsabufs[i].buf = static_cast(bufs[i].data()); + op.wsabufs[i].len = static_cast( + (std::min)(bufs[i].size(), std::size_t(0x7fffffff))); + } + + op.flags = 0; + + int result = ::WSARecv( + socket_, op.wsabufs, op.wsabuf_count, nullptr, &op.flags, &op, nullptr); + + if (result == SOCKET_ERROR) + { + DWORD err = ::WSAGetLastError(); + if (err != WSA_IO_PENDING) + { + svc_.on_completion(&op, err, 0); + return std::noop_coroutine(); + } + } + + svc_.on_pending(&op); + + // Re-check cancellation after I/O is pending + if (op.cancelled.load(std::memory_order_acquire)) + ::CancelIoEx(reinterpret_cast(socket_), &op); + + return std::noop_coroutine(); +} + +inline std::coroutine_handle<> +win_tcp_socket::write_some( + std::coroutine_handle<> h, + capy::executor_ref d, + buffer_param param, + std::stop_token token, + std::error_code* ec, + std::size_t* bytes_out) +{ + // Keep this socket alive during I/O + wr_.object_ref_ = detail::object_ref(this); + + auto& op = wr_; + op.reset(); + op.h = h; + op.ex = d; + op.ec_out = ec; + op.bytes_out = bytes_out; + op.start(token); + + svc_.work_started(); + + // Closed-object contract: complete with bad_file_descriptor + // without touching the kernel. + if (socket_ == INVALID_SOCKET) + { + svc_.on_completion(&op, WSAEBADF, 0); + return std::noop_coroutine(); + } + + // Prepare buffers + capy::mutable_buffer bufs[write_op::max_buffers]; + op.wsabuf_count = + static_cast(param.copy_to(bufs, write_op::max_buffers)); + + // Handle empty buffer: complete immediately with 0 bytes + if (op.wsabuf_count == 0) + { + svc_.on_completion(&op, 0, 0); + return std::noop_coroutine(); + } + + for (DWORD i = 0; i < op.wsabuf_count; ++i) + { + op.wsabufs[i].buf = static_cast(bufs[i].data()); + op.wsabufs[i].len = static_cast( + (std::min)(bufs[i].size(), std::size_t(0x7fffffff))); + } + + int result = ::WSASend( + socket_, op.wsabufs, op.wsabuf_count, nullptr, 0, &op, nullptr); + + if (result == SOCKET_ERROR) + { + DWORD err = ::WSAGetLastError(); + if (err != WSA_IO_PENDING) + { + svc_.on_completion(&op, err, 0); + return std::noop_coroutine(); + } + } + + svc_.on_pending(&op); + + // Re-check cancellation after I/O is pending + if (op.cancelled.load(std::memory_order_acquire)) + ::CancelIoEx(reinterpret_cast(socket_), &op); + + return std::noop_coroutine(); +} + +inline std::coroutine_handle<> +win_tcp_socket::wait( + std::coroutine_handle<> h, + capy::executor_ref d, + wait_type w, + std::stop_token token, + std::error_code* ec) +{ + wt_.object_ref_ = detail::object_ref(this); + + auto& op = wt_; + op.reset(); + op.h = h; + op.ex = d; + op.ec_out = ec; + op.bytes_out = nullptr; + op.empty_buffer = true; // skip EOF translation in invoke_handler + op.w = w; + op.start(token); + + svc_.work_started(); + + // Closed-object contract: complete with bad_file_descriptor + // without touching the kernel or the wait reactor. + if (socket_ == INVALID_SOCKET) + { + svc_.on_completion(&op, WSAEBADF, 0); + return std::noop_coroutine(); + } + + if (w == wait_type::read) + { + // Zero-byte WSARecv: kernel signals completion when data is + // available without consuming any bytes from the stream. This + // is the documented Winsock pattern for "is the socket + // readable" notifications. + op.wsabuf = WSABUF{0, nullptr}; + op.flags = 0; + + int result = + ::WSARecv(socket_, &op.wsabuf, 1, nullptr, &op.flags, &op, nullptr); + + if (result == SOCKET_ERROR) + { + DWORD err = ::WSAGetLastError(); + if (err != WSA_IO_PENDING) + { + svc_.on_completion(&op, err, 0); + return std::noop_coroutine(); + } + } + + svc_.on_pending(&op); + + // Re-check cancellation after I/O is pending. + if (op.cancelled.load(std::memory_order_acquire)) + ::CancelIoEx(reinterpret_cast(socket_), &op); + + return std::noop_coroutine(); + } + + // wait_type::write and wait_type::error: route through the + // auxiliary poll reactor. There is no overlapped primitive for + // "the send buffer has room" that does not also transfer bytes, + // and a write wait must report real writability. + svc_.scheduler().wait_reactor().register_wait(socket_, w, &op); + return std::noop_coroutine(); +} + +inline void +win_tcp_socket::cancel() noexcept +{ + if (socket_ != INVALID_SOCKET) + { + ::CancelIoEx(reinterpret_cast(socket_), nullptr); + } + + conn_.request_cancel(); + rd_.request_cancel(); + wr_.request_cancel(); + wt_.request_cancel(); + // CancelIoEx covers overlapped I/O on the socket but cannot reach + // a wait op parked in the auxiliary reactor (no overlapped is + // outstanding). Route through the reactor explicitly. + svc_.scheduler().cancel_wait(&wt_); +} + +inline void +win_tcp_socket::close_socket() noexcept +{ + // Flag every op cancelled before tearing down the handle. closesocket() + // can complete a pending overlapped op with ERROR_NETNAME_DELETED (not the + // MSDN-documented ERROR_OPERATION_ABORTED), which iocp_make_err now maps to + // connection_reset -- so a locally-closed op must be short-circuited to + // canceled via the flag rather than relying on the raw error, exactly as + // cancel() does. The store happens-before the completion is decoded. + conn_.request_cancel(); + rd_.request_cancel(); + wr_.request_cancel(); + // wt_ is parked in the aux reactor (no overlapped outstanding), so it also + // needs an explicit reactor deregister before the SOCKET handle is closed; + // otherwise the reactor would keep polling a dangling fd (and on a Winsock + // SOCKET-id reuse the wrong fd could be polled briefly). + wt_.request_cancel(); + svc_.scheduler().cancel_wait(&wt_); + + if (socket_ != INVALID_SOCKET) + { + ::CancelIoEx(reinterpret_cast(socket_), nullptr); + ::closesocket(socket_); + socket_ = INVALID_SOCKET; + } + + family_ = AF_UNSPEC; + + // Clear cached endpoints + local_endpoint_ = endpoint{}; + remote_endpoint_ = endpoint{}; +} + +inline std::error_code +win_tcp_socket::shutdown(tcp_socket::shutdown_type what) noexcept +{ + int how; + switch (what) + { + case tcp_socket::shutdown_receive: + how = SD_RECEIVE; + break; + case tcp_socket::shutdown_send: + how = SD_SEND; + break; + case tcp_socket::shutdown_both: + how = SD_BOTH; + break; + default: + return make_err(WSAEINVAL); + } + if (::shutdown(socket_, how) != 0) + return make_err(WSAGetLastError()); + return {}; +} + +inline native_handle_type +win_tcp_socket::native_handle() const noexcept +{ + return static_cast(socket_); +} + +inline native_handle_type +win_tcp_socket::release_socket() noexcept +{ + SOCKET s = socket_; + if (s != INVALID_SOCKET) + { + cancel(); + // Sever the port association so the descriptor can be + // adopted again; best-effort, the caller keeps a working + // socket either way. + dissociate_from_iocp(s); + socket_ = INVALID_SOCKET; + family_ = AF_UNSPEC; + local_endpoint_ = endpoint{}; + remote_endpoint_ = endpoint{}; + } + return static_cast(s); +} + +inline std::error_code +win_tcp_socket::set_option( + int level, int optname, void const* data, std::size_t size) noexcept +{ + if (::setsockopt( + socket_, level, optname, reinterpret_cast(data), + static_cast(size)) != 0) + return make_err(WSAGetLastError()); + return {}; +} + +inline std::error_code +win_tcp_socket::get_option( + int level, int optname, void* data, std::size_t* size) const noexcept +{ + int len = static_cast(*size); + if (::getsockopt( + socket_, level, optname, reinterpret_cast(data), &len) != 0) + return make_err(WSAGetLastError()); + *size = static_cast(len); + return {}; +} + +inline endpoint +win_tcp_socket::local_endpoint() const noexcept +{ + return local_endpoint_; +} + +inline endpoint +win_tcp_socket::remote_endpoint() const noexcept +{ + return remote_endpoint_; +} + +// win_tcp_service + +inline win_tcp_service::win_tcp_service(capy::execution_context& ctx) + : sched_(ctx.use_service()) + , iocp_(sched_.native_handle()) +{ + load_extension_functions(); +} + +inline void +win_tcp_service::shutdown() +{ + // Close all sockets/acceptors to force pending I/O to complete via + // IOCP. Not deleted here: in-flight ops hold their own object_ref, so + // each impl stays alive until its cancel completion drains; the + // pools' own destructors free whatever is still live once this + // service itself is torn down (after the scheduler, which drains + // every queued completion first). + pool_.shutdown([](win_tcp_socket* impl) { impl->close_socket(); }); + + acceptor_pool_.shutdown( + [](win_tcp_acceptor* impl) { impl->close_socket(); }); +} + +inline io_object::implementation* +win_tcp_service::construct() +{ + return pool_.acquire(*this); +} + +inline void +win_tcp_service::destroy(io_object::implementation* p) +{ + if (!p) + return; + auto* s = static_cast(p); + s->close_socket(); + release(s); +} + +inline void +win_tcp_service::close(io_object::handle& h) +{ + static_cast(h.get())->close_socket(); +} + +inline std::error_code +win_tcp_service::open_socket( + win_tcp_socket& impl, int family, int type, int protocol) +{ + impl.close_socket(); + + SOCKET sock = + ::WSASocketW(family, type, protocol, nullptr, 0, WSA_FLAG_OVERLAPPED); + + if (sock == INVALID_SOCKET) + return make_err(::WSAGetLastError()); + + if (family == AF_INET6) + { + DWORD one = 1; + ::setsockopt( + sock, IPPROTO_IPV6, IPV6_V6ONLY, reinterpret_cast(&one), + sizeof(one)); + } + + HANDLE result = ::CreateIoCompletionPort( + reinterpret_cast(sock), static_cast(iocp_), key_io, 0); + + if (result == nullptr) + { + DWORD dwError = ::GetLastError(); + ::closesocket(sock); + return make_err(dwError); + } + + impl.socket_ = sock; + impl.family_ = family; + return {}; +} + +inline std::error_code +win_tcp_service::assign_socket(win_tcp_socket& impl, native_handle_type fd) +{ + SOCKET sock = static_cast(fd); + if (sock == INVALID_SOCKET) + return make_err(WSAENOTSOCK); + + // SO_PROTOCOL_INFOW works on an unbound socket, unlike getsockname + // (WSAEINVAL until bind/connect names it) -- an adopted socket may + // have reached connected state without an explicit bind. + WSAPROTOCOL_INFOW proto_info{}; + int proto_len = sizeof(proto_info); + if (::getsockopt( + sock, SOL_SOCKET, SO_PROTOCOL_INFOW, + reinterpret_cast(&proto_info), &proto_len) != 0) + return make_err(::WSAGetLastError()); + if (proto_info.iAddressFamily != AF_INET && + proto_info.iAddressFamily != AF_INET6) + return make_err(WSAEAFNOSUPPORT); + if (proto_info.iSocketType != SOCK_STREAM) + return make_err(WSAEPROTOTYPE); + + HANDLE result = ::CreateIoCompletionPort( + reinterpret_cast(sock), static_cast(iocp_), key_io, 0); + if (result == nullptr) + return make_err(::GetLastError()); + + impl.socket_ = sock; + impl.family_ = proto_info.iAddressFamily; + + endpoint local_ep, remote_ep; + sockaddr_storage local_storage{}; + int local_len = sizeof(local_storage); + if (::getsockname( + sock, reinterpret_cast(&local_storage), &local_len) == 0) + local_ep = detail::from_sockaddr(local_storage); + sockaddr_storage remote_storage{}; + int remote_len = sizeof(remote_storage); + if (::getpeername( + sock, reinterpret_cast(&remote_storage), &remote_len) == + 0) + remote_ep = detail::from_sockaddr(remote_storage); + impl.set_endpoints(local_ep, remote_ep); + + return {}; +} + +inline std::error_code +win_tcp_service::bind_socket(win_tcp_socket& impl, endpoint ep) +{ + SOCKET sock = impl.socket_; + + sockaddr_storage storage{}; + socklen_t addrlen = detail::to_sockaddr(ep, storage); + if (::bind( + sock, reinterpret_cast(&storage), + static_cast(addrlen)) == SOCKET_ERROR) + return make_err(::WSAGetLastError()); + + // Cache local endpoint (resolves ephemeral port) + sockaddr_storage local_storage{}; + int local_len = sizeof(local_storage); + if (::getsockname( + sock, reinterpret_cast(&local_storage), &local_len) == 0) + impl.local_endpoint_ = detail::from_sockaddr(local_storage); + + return {}; +} + +inline void* +win_tcp_service::native_handle() const noexcept +{ + return iocp_; +} + +inline void +win_tcp_service::post(overlapped_op* op) +{ + sched_.post(op); +} + +inline void +win_tcp_service::on_pending(overlapped_op* op) noexcept +{ + sched_.on_pending(op); +} + +inline void +win_tcp_service::on_completion( + overlapped_op* op, DWORD error, DWORD bytes) noexcept +{ + sched_.on_completion(op, error, bytes); +} + +inline void +win_tcp_service::work_started() noexcept +{ + sched_.work_started(); +} + +inline void +win_tcp_service::work_finished() noexcept +{ + sched_.work_finished(); +} + +inline void +win_tcp_service::load_extension_functions() +{ + SOCKET sock = ::WSASocketW( + AF_INET, SOCK_STREAM, IPPROTO_TCP, nullptr, 0, WSA_FLAG_OVERLAPPED); + + if (sock == INVALID_SOCKET) + return; + + DWORD bytes = 0; + + GUID connect_ex_guid = WSAID_CONNECTEX; + ::WSAIoctl( + sock, SIO_GET_EXTENSION_FUNCTION_POINTER, &connect_ex_guid, + sizeof(connect_ex_guid), &connect_ex_, sizeof(connect_ex_), &bytes, + nullptr, nullptr); + + GUID accept_ex_guid = WSAID_ACCEPTEX; + ::WSAIoctl( + sock, SIO_GET_EXTENSION_FUNCTION_POINTER, &accept_ex_guid, + sizeof(accept_ex_guid), &accept_ex_, sizeof(accept_ex_), &bytes, + nullptr, nullptr); + + ::closesocket(sock); +} + +inline win_tcp_acceptor* +win_tcp_service::acquire_acceptor_impl() +{ + return acceptor_pool_.acquire(*this); +} + +inline std::error_code +win_tcp_service::open_acceptor_socket( + win_tcp_acceptor& impl, int family, int type, int protocol) +{ + impl.close_socket(); + + SOCKET sock = + ::WSASocketW(family, type, protocol, nullptr, 0, WSA_FLAG_OVERLAPPED); + + if (sock == INVALID_SOCKET) + return make_err(::WSAGetLastError()); + + if (family == AF_INET6) + { + DWORD val = 0; // dual-stack default + ::setsockopt( + sock, IPPROTO_IPV6, IPV6_V6ONLY, reinterpret_cast(&val), + sizeof(val)); + } + + HANDLE result = ::CreateIoCompletionPort( + reinterpret_cast(sock), static_cast(iocp_), key_io, 0); + + if (result == nullptr) + { + DWORD dwError = ::GetLastError(); + ::closesocket(sock); + return make_err(dwError); + } + + impl.socket_ = sock; + impl.family_ = family; + return {}; +} + +inline std::error_code +win_tcp_service::assign_acceptor_socket( + win_tcp_acceptor& impl, native_handle_type fd) +{ + SOCKET sock = static_cast(fd); + if (sock == INVALID_SOCKET) + return make_err(WSAENOTSOCK); + + // SO_PROTOCOL_INFOW works on an unbound socket, unlike getsockname + // (WSAEINVAL until bind names it). + WSAPROTOCOL_INFOW proto_info{}; + int proto_len = sizeof(proto_info); + if (::getsockopt( + sock, SOL_SOCKET, SO_PROTOCOL_INFOW, + reinterpret_cast(&proto_info), &proto_len) != 0) + return make_err(::WSAGetLastError()); + if (proto_info.iAddressFamily != AF_INET && + proto_info.iAddressFamily != AF_INET6) + return make_err(WSAEAFNOSUPPORT); + if (proto_info.iSocketType != SOCK_STREAM) + return make_err(WSAEPROTOTYPE); + + HANDLE result = ::CreateIoCompletionPort( + reinterpret_cast(sock), static_cast(iocp_), key_io, 0); + if (result == nullptr) + return make_err(::GetLastError()); + + impl.socket_ = sock; + impl.family_ = proto_info.iAddressFamily; + + // AcceptEx sizes its address buffers from this cache, so an + // unseeded endpoint breaks accepts on an adopted v6 listener. + sockaddr_storage local_storage{}; + int local_len = sizeof(local_storage); + if (::getsockname( + sock, reinterpret_cast(&local_storage), &local_len) == 0) + impl.set_local_endpoint(detail::from_sockaddr(local_storage)); + + return {}; +} + +inline std::error_code +win_tcp_service::bind_acceptor(win_tcp_acceptor& impl, endpoint ep) +{ + SOCKET sock = impl.socket_; + + sockaddr_storage storage{}; + socklen_t addrlen = detail::to_sockaddr(ep, storage); + if (::bind( + sock, reinterpret_cast(&storage), + static_cast(addrlen)) == SOCKET_ERROR) + return make_err(::WSAGetLastError()); + + // Cache local endpoint (resolves ephemeral port) + sockaddr_storage local_storage{}; + int local_len = sizeof(local_storage); + if (::getsockname( + sock, reinterpret_cast(&local_storage), &local_len) == 0) + impl.set_local_endpoint(detail::from_sockaddr(local_storage)); + + return {}; +} + +inline std::error_code +win_tcp_service::listen_acceptor(win_tcp_acceptor& impl, int backlog) +{ + SOCKET sock = impl.socket_; + + if (::listen(sock, backlog) == SOCKET_ERROR) + return make_err(::WSAGetLastError()); + + return {}; +} + } // namespace boost::corosio::detail #endif // BOOST_COROSIO_HAS_IOCP diff --git a/include/boost/corosio/native/detail/iocp/win_tcp_socket.hpp b/include/boost/corosio/native/detail/iocp/win_tcp_socket.hpp index 5bfb23035..d28a05167 100644 --- a/include/boost/corosio/native/detail/iocp/win_tcp_socket.hpp +++ b/include/boost/corosio/native/detail/iocp/win_tcp_socket.hpp @@ -18,6 +18,7 @@ #if BOOST_COROSIO_HAS_IOCP #include +#include #include #include #include @@ -26,20 +27,18 @@ #include #include -#include #include namespace boost::corosio::detail { class win_tcp_service; -class win_tcp_socket_internal; +class win_tcp_socket; /** Connect operation state. */ struct connect_op : overlapped_op { - win_tcp_socket_internal& internal; - std::shared_ptr internal_ptr; + win_tcp_socket& internal; endpoint target_endpoint; static void do_complete( @@ -49,7 +48,7 @@ struct connect_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit connect_op(win_tcp_socket_internal& internal_) noexcept; + explicit connect_op(win_tcp_socket& internal_) noexcept; }; /** Read operation state with buffer descriptors. */ @@ -59,8 +58,7 @@ struct read_op : overlapped_op WSABUF wsabufs[max_buffers]; DWORD wsabuf_count = 0; DWORD flags = 0; - win_tcp_socket_internal& internal; - std::shared_ptr internal_ptr; + win_tcp_socket& internal; static void do_complete( void* owner, @@ -69,7 +67,7 @@ struct read_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit read_op(win_tcp_socket_internal& internal_) noexcept; + explicit read_op(win_tcp_socket& internal_) noexcept; }; /** Write operation state with buffer descriptors. */ @@ -78,8 +76,7 @@ struct write_op : overlapped_op static constexpr std::size_t max_buffers = 16; WSABUF wsabufs[max_buffers]; DWORD wsabuf_count = 0; - win_tcp_socket_internal& internal; - std::shared_ptr internal_ptr; + win_tcp_socket& internal; static void do_complete( void* owner, @@ -88,7 +85,7 @@ struct write_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit write_op(win_tcp_socket_internal& internal_) noexcept; + explicit write_op(win_tcp_socket& internal_) noexcept; }; /** Readiness-wait operation state. @@ -105,8 +102,7 @@ struct wait_op : overlapped_op WSABUF wsabuf{}; DWORD flags = 0; wait_type w = wait_type::read; - win_tcp_socket_internal& internal; - std::shared_ptr internal_ptr; + win_tcp_socket& internal; static void do_complete( void* owner, @@ -115,23 +111,26 @@ struct wait_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit wait_op(win_tcp_socket_internal& internal_) noexcept; + explicit wait_op(win_tcp_socket& internal_) noexcept; }; -/** Internal socket state for IOCP-based I/O. +/** Socket implementation for IOCP-based I/O. - This class contains the actual state for a single socket, including - the native socket handle and pending operations. It derives from - enable_shared_from_this so operations can extend its lifetime. + Collapses the historical internal-state/wrapper split into one + pooled `io_object::implementation`: it both owns the native socket + handle and pending operations, and directly implements + `tcp_socket::implementation`. `win_tcp_service` recycles instances + through its pool instead of freeing them on every close, so the + keepalive each embedded op holds is the intrusive `object_ref` from + `coro_op`, not a `shared_ptr`. @note Internal implementation detail. Users interact with socket class. */ -class win_tcp_socket_internal - : public intrusive_list::node - , public std::enable_shared_from_this +class win_tcp_socket final + : public tcp_socket::implementation + , public intrusive_list::node { friend class win_tcp_service; - friend class win_tcp_socket; friend struct read_op; friend struct write_op; friend struct connect_op; @@ -144,73 +143,29 @@ class win_tcp_socket_internal wait_op wt_; SOCKET socket_ = INVALID_SOCKET; int family_ = AF_UNSPEC; - -public: - explicit win_tcp_socket_internal(win_tcp_service& svc) noexcept; - ~win_tcp_socket_internal(); - - std::coroutine_handle<> connect( - std::coroutine_handle<>, - capy::executor_ref, - endpoint, - std::stop_token, - std::error_code*); - - std::coroutine_handle<> read_some( - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - std::stop_token, - std::error_code*, - std::size_t*); - - std::coroutine_handle<> write_some( - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - std::stop_token, - std::error_code*, - std::size_t*); - - std::coroutine_handle<> wait( - std::coroutine_handle<>, - capy::executor_ref, - wait_type, - std::stop_token, - std::error_code*); - - SOCKET native_handle() const noexcept; - endpoint local_endpoint() const noexcept; - endpoint remote_endpoint() const noexcept; - bool is_open() const noexcept; - void cancel() noexcept; - void close_socket() noexcept; - void set_socket(SOCKET s) noexcept; - void set_endpoints(endpoint local, endpoint remote) noexcept; - -private: endpoint local_endpoint_; endpoint remote_endpoint_; -}; -/** Socket implementation wrapper for IOCP-based I/O. +public: + explicit win_tcp_socket(win_tcp_service& svc) noexcept; - This class is the public-facing implementation that holds a shared_ptr - to the internal state. The shared_ptr is hidden from the public interface. + /// Recycle into the owning service's pool. Defined out-of-line + /// after `win_tcp_service` is complete (needs `svc_.pool_`). + void retire() noexcept override; - @note Internal implementation detail. Users interact with socket class. -*/ -class win_tcp_socket final - : public tcp_socket::implementation - , public intrusive_list::node -{ - std::shared_ptr internal_; + /** Reset recycled state for reuse. -public: - explicit win_tcp_socket( - std::shared_ptr internal) noexcept; + `close_socket()` already drives the socket handle and family + to their closed values, and each op's completion drops its + `object_ref_` before `refs_` can reach zero. The endpoints are + reset here: a connect completing on another thread can write + them after the close. Each op's `OVERLAPPED` fields are + re-seeded by `overlapped_op::reset()` at its next submit, so + they are not zeroed here. - void close_internal() noexcept; + @pre refs_ == 0, socket closed, no op in flight. + */ + void reuse() noexcept; std::coroutine_handle<> connect( std::coroutine_handle<> h, @@ -250,7 +205,7 @@ class win_tcp_socket final { // getsockname fails with WSAEINVAL on an unbound socket, so // the family recorded at socket creation is authoritative - return to_family(internal_ ? internal_->family_ : AF_UNSPEC); + return to_family(family_); } native_handle_type release_socket() noexcept override; @@ -268,7 +223,11 @@ class win_tcp_socket final endpoint remote_endpoint() const noexcept override; void cancel() noexcept override; - win_tcp_socket_internal* get_internal() const noexcept; + /// Used by the acceptor's `accept_op` to finish setting up the peer. + void set_socket(SOCKET s) noexcept; + void set_endpoints(endpoint local, endpoint remote) noexcept; + + void close_socket() noexcept; }; } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/iocp/win_udp_service.hpp b/include/boost/corosio/native/detail/iocp/win_udp_service.hpp index 7c45a6510..4b9960495 100644 --- a/include/boost/corosio/native/detail/iocp/win_udp_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_udp_service.hpp @@ -16,13 +16,13 @@ #if BOOST_COROSIO_HAS_IOCP #include +#include #include #include #include #include #include -#include #include #include @@ -45,6 +45,8 @@ class BOOST_COROSIO_DECL win_udp_service final : private win_wsa_init , public udp_service { + friend class win_udp_socket; + public: io_object::implementation* construct() override; @@ -54,8 +56,6 @@ class BOOST_COROSIO_DECL win_udp_service final explicit win_udp_service(capy::execution_context& ctx); - ~win_udp_service(); - win_udp_service(win_udp_service const&) = delete; win_udp_service& operator=(win_udp_service const&) = delete; @@ -71,12 +71,8 @@ class BOOST_COROSIO_DECL win_udp_service final std::error_code bind_datagram(udp_socket::implementation& impl, endpoint ep) override; - void destroy_impl(win_udp_socket& impl); - - void unregister_impl(win_udp_socket_internal& impl); - std::error_code open_socket( - win_udp_socket_internal& impl, int family, int type, int protocol); + win_udp_socket& impl, int family, int type, int protocol); void post(overlapped_op* op); void on_pending(overlapped_op* op) noexcept; @@ -92,54 +88,51 @@ class BOOST_COROSIO_DECL win_udp_service final private: win_scheduler& sched_; + void* iocp_; BOOST_COROSIO_MSVC_WARNING_PUSH BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // detail:: members, dll-interface - win_mutex mutex_; - intrusive_list socket_list_; - intrusive_list wrapper_list_; + object_pool pool_; BOOST_COROSIO_MSVC_WARNING_POP - void* iocp_; }; // Operation constructors -inline send_to_op::send_to_op(win_udp_socket_internal& internal_) noexcept +inline send_to_op::send_to_op(win_udp_socket& internal_) noexcept : overlapped_op(&do_complete) , internal(internal_) { cancel_func_ = &do_cancel_impl; } -inline recv_from_op::recv_from_op(win_udp_socket_internal& internal_) noexcept +inline recv_from_op::recv_from_op(win_udp_socket& internal_) noexcept : overlapped_op(&do_complete) , internal(internal_) { cancel_func_ = &do_cancel_impl; } -inline udp_connect_op::udp_connect_op( - win_udp_socket_internal& internal_) noexcept +inline udp_connect_op::udp_connect_op(win_udp_socket& internal_) noexcept : overlapped_op(&do_complete) , internal(internal_) { cancel_func_ = &do_cancel_impl; } -inline udp_send_op::udp_send_op(win_udp_socket_internal& internal_) noexcept +inline udp_send_op::udp_send_op(win_udp_socket& internal_) noexcept : overlapped_op(&do_complete) , internal(internal_) { cancel_func_ = &do_cancel_impl; } -inline udp_recv_op::udp_recv_op(win_udp_socket_internal& internal_) noexcept +inline udp_recv_op::udp_recv_op(win_udp_socket& internal_) noexcept : overlapped_op(&do_complete) , internal(internal_) { cancel_func_ = &do_cancel_impl; } -inline udp_wait_op::udp_wait_op(win_udp_socket_internal& internal_) noexcept +inline udp_wait_op::udp_wait_op(win_udp_socket& internal_) noexcept : overlapped_op(&do_complete) , internal(internal_) { @@ -153,10 +146,9 @@ send_to_op::do_cancel_impl(overlapped_op* base) noexcept { auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) + if (op->internal.socket_ != INVALID_SOCKET) { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); } } @@ -165,10 +157,9 @@ recv_from_op::do_cancel_impl(overlapped_op* base) noexcept { auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) + if (op->internal.socket_ != INVALID_SOCKET) { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); } } @@ -186,11 +177,11 @@ send_to_op::do_complete( if (!owner) { op->cleanup_only(); - op->internal_ptr.reset(); + op->object_ref_.reset(); return; } - auto prevent_premature_destruction = std::move(op->internal_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } @@ -208,7 +199,7 @@ recv_from_op::do_complete( if (!owner) { op->cleanup_only(); - op->internal_ptr.reset(); + op->object_ref_.reset(); return; } @@ -220,7 +211,7 @@ recv_from_op::do_complete( *op->source_out = from_sockaddr(op->source_storage); } - auto prevent_premature_destruction = std::move(op->internal_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } @@ -238,10 +229,9 @@ udp_send_op::do_cancel_impl(overlapped_op* base) noexcept { auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) + if (op->internal.socket_ != INVALID_SOCKET) { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); } } @@ -250,10 +240,9 @@ udp_recv_op::do_cancel_impl(overlapped_op* base) noexcept { auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) + if (op->internal.socket_ != INVALID_SOCKET) { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); } } @@ -262,10 +251,9 @@ udp_wait_op::do_cancel_impl(overlapped_op* base) noexcept { auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) + if (op->internal.socket_ != INVALID_SOCKET) { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); } op->internal.svc_.scheduler().cancel_wait(op); } @@ -284,7 +272,7 @@ udp_connect_op::do_complete( if (!owner) { op->cleanup_only(); - op->internal_ptr.reset(); + op->object_ref_.reset(); return; } @@ -302,7 +290,7 @@ udp_connect_op::do_complete( op->internal.remote_endpoint_ = op->target_endpoint; } - auto prevent_premature_destruction = std::move(op->internal_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } @@ -318,11 +306,11 @@ udp_send_op::do_complete( if (!owner) { op->cleanup_only(); - op->internal_ptr.reset(); + op->object_ref_.reset(); return; } - auto prevent_premature_destruction = std::move(op->internal_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } @@ -338,11 +326,11 @@ udp_recv_op::do_complete( if (!owner) { op->cleanup_only(); - op->internal_ptr.reset(); + op->object_ref_.reset(); return; } - auto prevent_premature_destruction = std::move(op->internal_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } @@ -358,18 +346,17 @@ udp_wait_op::do_complete( if (!owner) { op->cleanup_only(); - op->internal_ptr.reset(); + op->object_ref_.reset(); return; } - auto prevent_premature_destruction = std::move(op->internal_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } -// win_udp_socket_internal +// win_udp_socket -inline win_udp_socket_internal::win_udp_socket_internal( - win_udp_service& svc) noexcept +inline win_udp_socket::win_udp_socket(win_udp_service& svc) noexcept : svc_(svc) , wr_(*this) , rd_(*this) @@ -380,37 +367,43 @@ inline win_udp_socket_internal::win_udp_socket_internal( { } -inline win_udp_socket_internal::~win_udp_socket_internal() +inline void +win_udp_socket::reuse() noexcept { - svc_.unregister_impl(*this); + BOOST_COROSIO_ASSERT(socket_ == INVALID_SOCKET); + BOOST_COROSIO_ASSERT(family_ == AF_UNSPEC); + // A connect completing on another thread while the socket closed + // can still have written these. + local_endpoint_ = endpoint{}; + remote_endpoint_ = endpoint{}; + BOOST_COROSIO_ASSERT(!wr_.stop_cb); + BOOST_COROSIO_ASSERT(!rd_.stop_cb); + BOOST_COROSIO_ASSERT(!conn_.stop_cb); + BOOST_COROSIO_ASSERT(!send_wr_.stop_cb); + BOOST_COROSIO_ASSERT(!recv_rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wt_.stop_cb); } -inline SOCKET -win_udp_socket_internal::native_handle() const noexcept +inline native_handle_type +win_udp_socket::native_handle() const noexcept { - return socket_; + return static_cast(socket_); } inline endpoint -win_udp_socket_internal::local_endpoint() const noexcept +win_udp_socket::local_endpoint() const noexcept { return local_endpoint_; } inline endpoint -win_udp_socket_internal::remote_endpoint() const noexcept +win_udp_socket::remote_endpoint() const noexcept { return remote_endpoint_; } -inline bool -win_udp_socket_internal::is_open() const noexcept -{ - return socket_ != INVALID_SOCKET; -} - inline std::coroutine_handle<> -win_udp_socket_internal::send_to( +win_udp_socket::send_to( std::coroutine_handle<> h, capy::executor_ref d, buffer_param param, @@ -420,8 +413,8 @@ win_udp_socket_internal::send_to( std::error_code* ec, std::size_t* bytes_out) { - // Keep internal alive during I/O - wr_.internal_ptr = shared_from_this(); + // Keep this socket alive during I/O + wr_.object_ref_ = detail::object_ref(this); auto& op = wr_; op.reset(); @@ -473,7 +466,7 @@ win_udp_socket_internal::send_to( } inline std::coroutine_handle<> -win_udp_socket_internal::recv_from( +win_udp_socket::recv_from( std::coroutine_handle<> h, capy::executor_ref d, buffer_param param, @@ -483,8 +476,8 @@ win_udp_socket_internal::recv_from( std::error_code* ec, std::size_t* bytes_out) { - // Keep internal alive during I/O - rd_.internal_ptr = shared_from_this(); + // Keep this socket alive during I/O + rd_.object_ref_ = detail::object_ref(this); auto& op = rd_; op.reset(); @@ -546,14 +539,14 @@ win_udp_socket_internal::recv_from( // UDP connect is synchronous on Windows inline std::coroutine_handle<> -win_udp_socket_internal::connect( +win_udp_socket::connect( std::coroutine_handle<> h, capy::executor_ref d, endpoint ep, std::stop_token token, std::error_code* ec) { - conn_.internal_ptr = shared_from_this(); + conn_.object_ref_ = detail::object_ref(this); auto& op = conn_; op.reset(); @@ -580,7 +573,7 @@ win_udp_socket_internal::connect( } inline std::coroutine_handle<> -win_udp_socket_internal::send( +win_udp_socket::send( std::coroutine_handle<> h, capy::executor_ref d, buffer_param param, @@ -589,7 +582,7 @@ win_udp_socket_internal::send( std::error_code* ec, std::size_t* bytes_out) { - send_wr_.internal_ptr = shared_from_this(); + send_wr_.object_ref_ = detail::object_ref(this); auto& op = send_wr_; op.reset(); @@ -634,7 +627,7 @@ win_udp_socket_internal::send( } inline std::coroutine_handle<> -win_udp_socket_internal::recv( +win_udp_socket::recv( std::coroutine_handle<> h, capy::executor_ref d, buffer_param param, @@ -643,7 +636,7 @@ win_udp_socket_internal::recv( std::error_code* ec, std::size_t* bytes_out) { - recv_rd_.internal_ptr = shared_from_this(); + recv_rd_.object_ref_ = detail::object_ref(this); auto& op = recv_rd_; op.reset(); @@ -696,14 +689,14 @@ win_udp_socket_internal::recv( } inline std::coroutine_handle<> -win_udp_socket_internal::wait( +win_udp_socket::wait( std::coroutine_handle<> h, capy::executor_ref d, wait_type w, std::stop_token token, std::error_code* ec) { - wt_.internal_ptr = shared_from_this(); + wt_.object_ref_ = detail::object_ref(this); auto& op = wt_; op.reset(); @@ -734,7 +727,7 @@ win_udp_socket_internal::wait( } inline void -win_udp_socket_internal::cancel() noexcept +win_udp_socket::cancel() noexcept { if (socket_ != INVALID_SOCKET) { @@ -751,7 +744,7 @@ win_udp_socket_internal::cancel() noexcept } inline void -win_udp_socket_internal::close_socket() noexcept +win_udp_socket::close_socket() noexcept { // Flag every op cancelled before closing so a closesocket-delivered // ERROR_NETNAME_DELETED is short-circuited to canceled rather than mapped @@ -777,106 +770,6 @@ win_udp_socket_internal::close_socket() noexcept remote_endpoint_ = endpoint{}; } -// win_udp_socket - -inline win_udp_socket::win_udp_socket( - std::shared_ptr internal) noexcept - : internal_(std::move(internal)) -{ -} - -inline void -win_udp_socket::close_internal() noexcept -{ - if (internal_) - { - internal_->close_socket(); - internal_.reset(); - } -} - -inline std::coroutine_handle<> -win_udp_socket::send_to( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - endpoint dest, - int flags, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->send_to(h, d, buf, dest, flags, token, ec, bytes); -} - -inline std::coroutine_handle<> -win_udp_socket::recv_from( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - endpoint* source, - int flags, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->recv_from(h, d, buf, source, flags, token, ec, bytes); -} - -inline std::coroutine_handle<> -win_udp_socket::connect( - std::coroutine_handle<> h, - capy::executor_ref d, - endpoint ep, - std::stop_token token, - std::error_code* ec) -{ - return internal_->connect(h, d, ep, token, ec); -} - -inline std::coroutine_handle<> -win_udp_socket::send( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - int flags, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->send(h, d, buf, flags, token, ec, bytes); -} - -inline std::coroutine_handle<> -win_udp_socket::recv( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - int flags, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->recv(h, d, buf, flags, token, ec, bytes); -} - -inline std::coroutine_handle<> -win_udp_socket::wait( - std::coroutine_handle<> h, - capy::executor_ref d, - wait_type w, - std::stop_token token, - std::error_code* ec) -{ - return internal_->wait(h, d, w, token, ec); -} - -inline native_handle_type -win_udp_socket::native_handle() const noexcept -{ - return static_cast(internal_->native_handle()); -} - inline std::error_code win_udp_socket::shutdown(udp_socket::shutdown_type what) noexcept { @@ -895,7 +788,7 @@ win_udp_socket::shutdown(udp_socket::shutdown_type what) noexcept default: return make_err(WSAEINVAL); } - if (::shutdown(internal_->native_handle(), how) != 0) + if (::shutdown(socket_, how) != 0) return make_err(WSAGetLastError()); return {}; } @@ -903,15 +796,15 @@ win_udp_socket::shutdown(udp_socket::shutdown_type what) noexcept inline native_handle_type win_udp_socket::release_socket() noexcept { - SOCKET s = internal_->socket_; + SOCKET s = socket_; if (s != INVALID_SOCKET) { - internal_->cancel(); + cancel(); dissociate_from_iocp(s); - internal_->socket_ = INVALID_SOCKET; - internal_->family_ = AF_UNSPEC; - internal_->local_endpoint_ = endpoint{}; - internal_->remote_endpoint_ = endpoint{}; + socket_ = INVALID_SOCKET; + family_ = AF_UNSPEC; + local_endpoint_ = endpoint{}; + remote_endpoint_ = endpoint{}; } return static_cast(s); } @@ -921,8 +814,8 @@ win_udp_socket::set_option( int level, int optname, void const* data, std::size_t size) noexcept { if (::setsockopt( - internal_->native_handle(), level, optname, - reinterpret_cast(data), static_cast(size)) != 0) + socket_, level, optname, reinterpret_cast(data), + static_cast(size)) != 0) return make_err(WSAGetLastError()); return {}; } @@ -933,35 +826,16 @@ win_udp_socket::get_option( { int len = static_cast(*size); if (::getsockopt( - internal_->native_handle(), level, optname, - reinterpret_cast(data), &len) != 0) + socket_, level, optname, reinterpret_cast(data), &len) != 0) return make_err(WSAGetLastError()); *size = static_cast(len); return {}; } -inline endpoint -win_udp_socket::local_endpoint() const noexcept -{ - return internal_->local_endpoint(); -} - -inline endpoint -win_udp_socket::remote_endpoint() const noexcept -{ - return internal_->remote_endpoint(); -} - inline void -win_udp_socket::cancel() noexcept +win_udp_socket::retire() noexcept { - internal_->cancel(); -} - -inline win_udp_socket_internal* -win_udp_socket::get_internal() const noexcept -{ - return internal_.get(); + svc_.pool_.recycle(this); } // win_udp_service @@ -972,83 +846,37 @@ inline win_udp_service::win_udp_service(capy::execution_context& ctx) { } -inline win_udp_service::~win_udp_service() -{ - for (auto* w = wrapper_list_.pop_front(); w != nullptr; - w = wrapper_list_.pop_front()) - delete w; -} - inline void win_udp_service::shutdown() { - std::lock_guard lock(mutex_); - - for (auto* impl = socket_list_.pop_front(); impl != nullptr; - impl = socket_list_.pop_front()) - { - impl->close_socket(); - } + pool_.shutdown([](win_udp_socket* impl) { impl->close_socket(); }); } inline io_object::implementation* win_udp_service::construct() { - auto internal = std::make_shared(*this); - - { - std::lock_guard lock(mutex_); - socket_list_.push_back(internal.get()); - } - - auto* wrapper = new win_udp_socket(std::move(internal)); - - { - std::lock_guard lock(mutex_); - wrapper_list_.push_back(wrapper); - } - - return wrapper; + return pool_.acquire(*this); } inline void win_udp_service::destroy(io_object::implementation* p) { - if (p) - { - auto& wrapper = static_cast(*p); - wrapper.close_internal(); - destroy_impl(wrapper); - } + if (!p) + return; + auto* s = static_cast(p); + s->close_socket(); + release(s); } inline void win_udp_service::close(io_object::handle& h) { - auto& wrapper = static_cast(*h.get()); - wrapper.get_internal()->close_socket(); -} - -inline void -win_udp_service::destroy_impl(win_udp_socket& impl) -{ - { - std::lock_guard lock(mutex_); - wrapper_list_.remove(&impl); - } - delete &impl; -} - -inline void -win_udp_service::unregister_impl(win_udp_socket_internal& impl) -{ - std::lock_guard lock(mutex_); - socket_list_.remove(&impl); + static_cast(h.get())->close_socket(); } inline std::error_code win_udp_service::open_socket( - win_udp_socket_internal& impl, int family, int type, int protocol) + win_udp_socket& impl, int family, int type, int protocol) { impl.close_socket(); @@ -1085,19 +913,18 @@ inline std::error_code win_udp_service::open_datagram_socket( udp_socket::implementation& impl, int family, int type, int protocol) { - auto& wrapper = static_cast(impl); - return open_socket(*wrapper.get_internal(), family, type, protocol); + auto& sock = static_cast(impl); + return open_socket(sock, family, type, protocol); } inline std::error_code win_udp_service::assign_socket( udp_socket::implementation& impl, native_handle_type fd) { - auto& wrapper = static_cast(impl); - auto* internal = wrapper.get_internal(); + auto& sock = static_cast(impl); - SOCKET sock = static_cast(fd); - if (sock == INVALID_SOCKET) + SOCKET s = static_cast(fd); + if (s == INVALID_SOCKET) return make_err(WSAENOTSOCK); // SO_PROTOCOL_INFOW works on an unbound socket, unlike getsockname @@ -1105,7 +932,7 @@ win_udp_service::assign_socket( WSAPROTOCOL_INFOW proto_info{}; int proto_len = sizeof(proto_info); if (::getsockopt( - sock, SOL_SOCKET, SO_PROTOCOL_INFOW, + s, SOL_SOCKET, SO_PROTOCOL_INFOW, reinterpret_cast(&proto_info), &proto_len) != 0) return make_err(::WSAGetLastError()); if (proto_info.iAddressFamily != AF_INET && @@ -1115,27 +942,26 @@ win_udp_service::assign_socket( return make_err(WSAEPROTOTYPE); HANDLE result = ::CreateIoCompletionPort( - reinterpret_cast(sock), static_cast(iocp_), key_io, 0); + reinterpret_cast(s), static_cast(iocp_), key_io, 0); if (result == nullptr) return make_err(::GetLastError()); - internal->socket_ = sock; - internal->family_ = proto_info.iAddressFamily; + sock.socket_ = s; + sock.family_ = proto_info.iAddressFamily; endpoint local_ep, remote_ep; sockaddr_storage local_storage{}; int local_len = sizeof(local_storage); if (::getsockname( - sock, reinterpret_cast(&local_storage), &local_len) == 0) + s, reinterpret_cast(&local_storage), &local_len) == 0) local_ep = detail::from_sockaddr(local_storage); sockaddr_storage remote_storage{}; int remote_len = sizeof(remote_storage); if (::getpeername( - sock, reinterpret_cast(&remote_storage), &remote_len) == - 0) + s, reinterpret_cast(&remote_storage), &remote_len) == 0) remote_ep = detail::from_sockaddr(remote_storage); - internal->local_endpoint_ = local_ep; - internal->remote_endpoint_ = remote_ep; + sock.local_endpoint_ = local_ep; + sock.remote_endpoint_ = remote_ep; return {}; } @@ -1143,14 +969,13 @@ win_udp_service::assign_socket( inline std::error_code win_udp_service::bind_datagram(udp_socket::implementation& impl, endpoint ep) { - auto& wrapper = static_cast(impl); - auto* internal = wrapper.get_internal(); - SOCKET sock = internal->socket_; + auto& sock = static_cast(impl); + SOCKET s = sock.socket_; sockaddr_storage storage{}; socklen_t addrlen = detail::to_sockaddr(ep, storage); if (::bind( - sock, reinterpret_cast(&storage), + s, reinterpret_cast(&storage), static_cast(addrlen)) == SOCKET_ERROR) return make_err(::WSAGetLastError()); @@ -1158,8 +983,8 @@ win_udp_service::bind_datagram(udp_socket::implementation& impl, endpoint ep) sockaddr_storage local_storage{}; int local_len = sizeof(local_storage); if (::getsockname( - sock, reinterpret_cast(&local_storage), &local_len) == 0) - internal->local_endpoint_ = detail::from_sockaddr(local_storage); + s, reinterpret_cast(&local_storage), &local_len) == 0) + sock.local_endpoint_ = detail::from_sockaddr(local_storage); return {}; } diff --git a/include/boost/corosio/native/detail/iocp/win_udp_socket.hpp b/include/boost/corosio/native/detail/iocp/win_udp_socket.hpp index 9679265d6..e489d58ab 100644 --- a/include/boost/corosio/native/detail/iocp/win_udp_socket.hpp +++ b/include/boost/corosio/native/detail/iocp/win_udp_socket.hpp @@ -16,6 +16,7 @@ #if BOOST_COROSIO_HAS_IOCP #include +#include #include #include #include @@ -24,12 +25,11 @@ #include #include -#include namespace boost::corosio::detail { class win_udp_service; -class win_udp_socket_internal; +class win_udp_socket; /** Send-to operation state with buffer and destination address. */ struct send_to_op : overlapped_op @@ -39,8 +39,7 @@ struct send_to_op : overlapped_op DWORD wsabuf_count = 0; sockaddr_storage dest_storage{}; int dest_len = 0; - win_udp_socket_internal& internal; - std::shared_ptr internal_ptr; + win_udp_socket& internal; static void do_complete( void* owner, @@ -49,7 +48,7 @@ struct send_to_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit send_to_op(win_udp_socket_internal& internal_) noexcept; + explicit send_to_op(win_udp_socket& internal_) noexcept; }; /** Recv-from operation state with buffer and source address capture. */ @@ -62,8 +61,7 @@ struct recv_from_op : overlapped_op sockaddr_storage source_storage{}; INT source_len = sizeof(sockaddr_storage); endpoint* source_out = nullptr; - win_udp_socket_internal& internal; - std::shared_ptr internal_ptr; + win_udp_socket& internal; static void do_complete( void* owner, @@ -72,15 +70,14 @@ struct recv_from_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit recv_from_op(win_udp_socket_internal& internal_) noexcept; + explicit recv_from_op(win_udp_socket& internal_) noexcept; }; /** Connect operation for connected-mode UDP on IOCP. */ struct udp_connect_op : overlapped_op { endpoint target_endpoint; - win_udp_socket_internal& internal; - std::shared_ptr internal_ptr; + win_udp_socket& internal; static void do_complete( void* owner, @@ -89,7 +86,7 @@ struct udp_connect_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit udp_connect_op(win_udp_socket_internal& internal_) noexcept; + explicit udp_connect_op(win_udp_socket& internal_) noexcept; }; /** Connected send operation (no destination address). */ @@ -98,8 +95,7 @@ struct udp_send_op : overlapped_op static constexpr std::size_t max_buffers = 16; WSABUF wsabufs[max_buffers]; DWORD wsabuf_count = 0; - win_udp_socket_internal& internal; - std::shared_ptr internal_ptr; + win_udp_socket& internal; static void do_complete( void* owner, @@ -108,7 +104,7 @@ struct udp_send_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit udp_send_op(win_udp_socket_internal& internal_) noexcept; + explicit udp_send_op(win_udp_socket& internal_) noexcept; }; /** Connected recv operation (no source address). */ @@ -118,8 +114,7 @@ struct udp_recv_op : overlapped_op WSABUF wsabufs[max_buffers]; DWORD wsabuf_count = 0; DWORD flags = 0; - win_udp_socket_internal& internal; - std::shared_ptr internal_ptr; + win_udp_socket& internal; static void do_complete( void* owner, @@ -128,14 +123,13 @@ struct udp_recv_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit udp_recv_op(win_udp_socket_internal& internal_) noexcept; + explicit udp_recv_op(win_udp_socket& internal_) noexcept; }; /** Readiness-wait operation (datagram socket). */ struct udp_wait_op : overlapped_op { - win_udp_socket_internal& internal; - std::shared_ptr internal_ptr; + win_udp_socket& internal; static void do_complete( void* owner, @@ -144,24 +138,24 @@ struct udp_wait_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit udp_wait_op(win_udp_socket_internal& internal_) noexcept; + explicit udp_wait_op(win_udp_socket& internal_) noexcept; }; -/** Internal datagram socket state for IOCP-based I/O. +/** Datagram socket implementation for IOCP-based I/O. - This class contains the actual state for a single UDP socket, - including the native socket handle and pending operations. It - derives from enable_shared_from_this so operations can extend - its lifetime. + Collapses the historical internal-state/wrapper split into one + pooled `io_object::implementation`: it both owns the native socket + handle and pending operations, and directly implements + `udp_socket::implementation`. `win_udp_service` recycles instances + through its pool instead of freeing them on every close. @note Internal implementation detail. */ -class win_udp_socket_internal - : public intrusive_list::node - , public std::enable_shared_from_this +class win_udp_socket final + : public udp_socket::implementation + , public intrusive_list::node { friend class win_udp_service; - friend class win_udp_socket; friend struct send_to_op; friend struct recv_from_op; friend struct udp_connect_op; @@ -178,94 +172,25 @@ class win_udp_socket_internal udp_wait_op wt_; SOCKET socket_ = INVALID_SOCKET; int family_ = AF_UNSPEC; - -public: - explicit win_udp_socket_internal(win_udp_service& svc) noexcept; - ~win_udp_socket_internal(); - - std::coroutine_handle<> send_to( - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - endpoint, - int flags, - std::stop_token, - std::error_code*, - std::size_t*); - - std::coroutine_handle<> recv_from( - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - endpoint*, - int flags, - std::stop_token, - std::error_code*, - std::size_t*); - - std::coroutine_handle<> connect( - std::coroutine_handle<>, - capy::executor_ref, - endpoint, - std::stop_token, - std::error_code*); - - std::coroutine_handle<> send( - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - int flags, - std::stop_token, - std::error_code*, - std::size_t*); - - std::coroutine_handle<> recv( - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - int flags, - std::stop_token, - std::error_code*, - std::size_t*); - - std::coroutine_handle<> wait( - std::coroutine_handle<>, - capy::executor_ref, - wait_type, - std::stop_token, - std::error_code*); - - SOCKET native_handle() const noexcept; - endpoint local_endpoint() const noexcept; - endpoint remote_endpoint() const noexcept; - bool is_open() const noexcept; - void cancel() noexcept; - void close_socket() noexcept; - -private: endpoint local_endpoint_; endpoint remote_endpoint_; -}; -/** Datagram socket implementation wrapper for IOCP-based I/O. +public: + explicit win_udp_socket(win_udp_service& svc) noexcept; - This class is the public-facing implementation that holds a - shared_ptr to the internal state. The shared_ptr is hidden - from the public interface. + /// Recycle into the owning service's pool. Defined out-of-line + /// after `win_udp_service` is complete (needs `svc_.pool_`). + void retire() noexcept override; - @note Internal implementation detail. -*/ -class win_udp_socket final - : public udp_socket::implementation - , public intrusive_list::node -{ - std::shared_ptr internal_; + /** Reset recycled state for reuse. -public: - explicit win_udp_socket( - std::shared_ptr internal) noexcept; + Each op's `OVERLAPPED` fields are re-seeded by + `overlapped_op::reset()` at its next submit, so they are not + zeroed here. - void close_internal() noexcept; + @pre refs_ == 0, socket closed, no op in flight. + */ + void reuse() noexcept; std::coroutine_handle<> send_to( std::coroutine_handle<> h, @@ -325,7 +250,7 @@ class win_udp_socket final { // getsockname fails with WSAEINVAL on an unbound socket, so // the family recorded at socket creation is authoritative - return to_family(internal_ ? internal_->family_ : AF_UNSPEC); + return to_family(family_); } std::error_code shutdown(udp_socket::shutdown_type what) noexcept override; @@ -345,7 +270,7 @@ class win_udp_socket final endpoint remote_endpoint() const noexcept override; void cancel() noexcept override; - win_udp_socket_internal* get_internal() const noexcept; + void close_socket() noexcept; }; } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/iocp/win_wait_reactor.hpp b/include/boost/corosio/native/detail/iocp/win_wait_reactor.hpp index 171c37c70..bbd67fc81 100644 --- a/include/boost/corosio/native/detail/iocp/win_wait_reactor.hpp +++ b/include/boost/corosio/native/detail/iocp/win_wait_reactor.hpp @@ -476,6 +476,13 @@ win_wait_reactor::run() { std::vector pollfds; + // Hoisted like pollfds: the swap under the lock trades buffers + // with pending_register_/pending_cancel_, so the capacities + // ping-pong between these locals and the members and no wake + // allocates once both sides are warm. + std::vector to_add; + std::vector to_cancel; + // What the ops still parked here are told on the way out, and what // every later register_wait is told too. Ending on a poll the // provider refused is not a cancellation, and an op that reports @@ -485,8 +492,8 @@ win_wait_reactor::run() while (!stop_.load(std::memory_order_acquire)) { // Drain pending register/cancel under the lock. - std::vector to_add; - std::vector to_cancel; + to_add.clear(); + to_cancel.clear(); { std::lock_guard lock(mutex_); to_add.swap(pending_register_); diff --git a/include/boost/corosio/native/detail/kqueue/kqueue_scheduler.hpp b/include/boost/corosio/native/detail/kqueue/kqueue_scheduler.hpp index dee7257ee..a612ef85a 100644 --- a/include/boost/corosio/native/detail/kqueue/kqueue_scheduler.hpp +++ b/include/boost/corosio/native/detail/kqueue/kqueue_scheduler.hpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include @@ -324,23 +325,36 @@ inline std::error_code kqueue_scheduler::register_descriptor( int fd, reactor_descriptor_state* desc) const { - int const err = kqueue_add_filter(kq_fd_, fd, EVFILT_READ, desc); - // EINVAL/ENODEV: a device with no kqfilter. As on epoll's EPERM, - // adopt it unwatched. - bool const unpollable = err == EINVAL || err == ENODEV; - if (err != 0 && !unpollable) - return make_err(err); - desc->registered_events = unpollable ? 0 : reactor_event_read; - desc->unpollable = unpollable; - desc->fd = fd; - desc->scheduler_ = this; - desc->mutex.set_enabled(reactor_io_locking_); - desc->ready_events_.store(0, std::memory_order_relaxed); - - conditionally_enabled_mutex::scoped_lock lock(desc->mutex); - desc->impl_ref_.reset(); - desc->read_ready = false; - desc->write_ready = false; + // Initialized before the add: another run thread's reactor can + // dispatch an event for the descriptor as soon as it is added. + // A completion queued by an earlier registration of the same state + // may still be running, so the fields it reads change under its + // mutex, and the mutex mode is only written when it differs. + if (desc->mutex.enabled() != reactor_io_locking_) + desc->mutex.set_enabled(reactor_io_locking_); + { + conditionally_enabled_mutex::scoped_lock lock(desc->mutex); + desc->registered_events = reactor_event_read; + desc->unpollable = false; + desc->fd = fd; + desc->scheduler_ = this; + desc->ready_events_.store(0, std::memory_order_release); + // object_ref_ is not touched: a queued invocation from an + // earlier registration still owns the reference in it. + desc->read_ready = false; + desc->write_ready = false; + } + + if (int const err = kqueue_add_filter(kq_fd_, fd, EVFILT_READ, desc)) + { + conditionally_enabled_mutex::scoped_lock lock(desc->mutex); + desc->registered_events = 0; + // EINVAL/ENODEV: a device with no kqfilter. As on epoll's + // EPERM, adopt it unwatched. + if (err != EINVAL && err != ENODEV) + return make_err(err); + desc->unpollable = true; + } return {}; } @@ -489,6 +503,13 @@ kqueue_scheduler::run_task(lock_type& lock, context_type& ctx, long timeout_us) ready |= reactor_event_error; } +#if BOOST_COROSIO_TSAN + // kevent hands the descriptor over in the kernel, where the + // sanitizer sees no ordering, and it checks a read-modify-write + // before applying its acquire. This load gives it the acquire + // that pairs with register_descriptor's release store first. + std::ignore = desc->ready_events_.load(std::memory_order_acquire); +#endif desc->add_ready_events(ready); bool expected = false; diff --git a/include/boost/corosio/native/detail/native_socket_base.hpp b/include/boost/corosio/native/detail/native_socket_base.hpp index c067cd1ac..7fd6709e6 100644 --- a/include/boost/corosio/native/detail/native_socket_base.hpp +++ b/include/boost/corosio/native/detail/native_socket_base.hpp @@ -27,9 +27,10 @@ Holds the part of a socket impl that does not care whether the backend is readiness-based (epoll/kqueue/select, which park ops on a descriptor_state) or completion-based (io_uring, which submits SQEs): - the file descriptor, the cached local endpoint, the impl lifecycle - bases (enable_shared_from_this + the service's intrusive tracking node), - and the synchronous accessors (native_handle / options / bind). + the file descriptor, the cached local endpoint, and the synchronous + accessors (native_handle / options / bind). The impl lifecycle (the + intrusive refcount + the owning service's recycling pool) is carried + by `io_object::implementation`, not by this base. reactor_basic_socket derives from this and adds the readiness machinery (descriptor_state, register_op, the cancel/close that deregister from @@ -39,8 +40,7 @@ readiness-agnostic socket surface, while the on-EAGAIN action (park vs submit-SQE) and the op model stay backend-specific. - @tparam Derived The concrete socket type (CRTP, for shared_from_this - and the intrusive node). + @tparam Derived The concrete socket type (CRTP). @tparam ImplBase The public vtable base (tcp_socket::implementation, udp_socket::implementation, ...). @tparam Endpoint The endpoint type (endpoint or local_endpoint). @@ -51,7 +51,6 @@ namespace boost::corosio::detail { template class native_socket_base : public ImplBase - , public std::enable_shared_from_this { protected: // CRTP base: not publicly constructible. The check's preferred fix diff --git a/include/boost/corosio/native/detail/posix/posix_random_access_file.hpp b/include/boost/corosio/native/detail/posix/posix_random_access_file.hpp index 5a00a2c0b..d7da706c7 100644 --- a/include/boost/corosio/native/detail/posix/posix_random_access_file.hpp +++ b/include/boost/corosio/native/detail/posix/posix_random_access_file.hpp @@ -38,7 +38,6 @@ #include #include #include -#include #include #include #include @@ -54,12 +53,13 @@ POSIX Random-Access File Implementation ======================================== - Each async read/write heap-allocates an raf_op that serves - as both the thread-pool work item and the scheduler completion - op. This allows unlimited concurrent operations on the same - file object, matching Asio's per-op allocation model. + Each async read/write acquires an raf_op that serves as both the + thread-pool work item and the scheduler completion op. This + allows unlimited concurrent operations on the same file object. - The raf_op self-deletes on completion or shutdown. + Ops recycle through the file's free_ops_ list on completion or + shutdown, so steady-state reads and writes allocate nothing; the + file destructor frees the recycled storage. */ namespace boost::corosio::detail { @@ -70,7 +70,6 @@ class posix_random_access_file_service; /** Random-access file implementation for POSIX backends. */ class posix_random_access_file final : public random_access_file::implementation - , public std::enable_shared_from_this , public intrusive_list::node { friend class posix_random_access_file_service; @@ -78,13 +77,15 @@ class posix_random_access_file final public: static constexpr std::size_t max_buffers = 16; - /** Per-operation state, heap-allocated for each async call. + /** Per-operation state, acquired from the file's free list. Inherits from `coro_op` (for scheduler completion plus the shared coroutine, cancellation and keepalive machinery) and - `pool_work_item` (for thread-pool dispatch). Linked into the - file's outstanding_ops_ list for cancellation tracking. `coro_op` - leads the base list so a `scheduler_op*` round-trips. + `pool_work_item` (for thread-pool dispatch). The intrusive hook + links it into `outstanding_ops_` while in flight and `free_ops_` + once recycled — membership is strictly sequential, so one hook + serves both under `ops_mutex_`. `coro_op` leads the base list so + a `scheduler_op*` round-trips. */ struct raf_op final : coro_op @@ -101,7 +102,7 @@ class posix_random_access_file final int errn = 0; std::size_t bytes_transferred = 0; - // Raw back-pointer for the typed work; `impl_ptr` is the keepalive. + // Raw back-pointer for the typed work; `object_ref_` is the keepalive. posix_random_access_file* file_ = nullptr; // The awaitable's, not the embedded `cont`: this op is freed @@ -118,6 +119,34 @@ class posix_random_access_file final explicit posix_random_access_file( posix_random_access_file_service& svc) noexcept; + /// Recycle into the owning service's pool. Defined out-of-line + /// after posix_random_access_file_service for its complete type. + void retire() noexcept override; + + /** Reset for recycling. + + `close_file()` already drove fd_ to its closed value before + the refcount reached zero. Every `raf_op` holds its own + `object_ref` while in flight, so the refcount cannot reach + zero while one is outstanding — asserting `outstanding_ops_` + is empty is therefore a precondition check, not a defensive + one. `free_ops_` deliberately survives recycling: the storage + belongs to this impl and only the destructor frees it. + + @pre refs_ == 0, fd closed, no op in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(fd_ == -1); + BOOST_COROSIO_ASSERT(outstanding_ops_.empty()); + } + + ~posix_random_access_file() override + { + while (auto* op = free_ops_.pop_front()) + delete op; + } + // -- random_access_file::implementation -- std::coroutine_handle<> read_some_at( @@ -163,10 +192,32 @@ class posix_random_access_file final void close_file() noexcept; private: + /** Pop a recycled op, or allocate on the cold path. + + Per-use fields are filled by the caller; the fields `prepare`- + style reuse must not inherit from the previous run (`errn`, + `bytes_transferred`) are reset here. `start()` resets the + cancellation machinery. + */ + raf_op* acquire_op() + { + raf_op* op; + { + std::lock_guard lock(ops_mutex_); + op = free_ops_.pop_front(); + } + if (!op) + op = new raf_op(); + op->errn = 0; + op->bytes_transferred = 0; + return op; + } + posix_random_access_file_service& svc_; int fd_ = -1; std::mutex ops_mutex_; intrusive_list outstanding_ops_; + intrusive_list free_ops_; }; // --------------------------------------------------------------------------- @@ -307,18 +358,21 @@ posix_random_access_file::raf_op::operator()() errn != 0 ? make_err(errn) : std::error_code{}, is_read, bytes_transferred, /*empty_buffer=*/false); + // Copy out everything needed after recycling: once this op is on + // free_ops_ a concurrent initiation may pop and refill it. The + // keepalive drops after the push so a final release never runs + // under ops_mutex_. + auto keep = std::move(object_ref_); + auto* c = awaiting; + auto exec = ex; { std::lock_guard lock(file_->ops_mutex_); file_->outstanding_ops_.remove(this); + file_->free_ops_.push_front(this); } - - impl_ptr.reset(); - - auto* c = awaiting; - auto local_ex = ex; - local_ex.on_work_finished(); - delete this; - dispatch_coro(local_ex, *c).resume(); + keep.reset(); + exec.on_work_finished(); + dispatch_coro(exec, *c).resume(); } // -- raf_op shutdown cleanup -- @@ -327,13 +381,15 @@ inline void posix_random_access_file::raf_op::destroy() { stop_cb.reset(); + auto keep = std::move(object_ref_); + auto exec = ex; { std::lock_guard lock(file_->ops_mutex_); file_->outstanding_ops_.remove(this); + file_->free_ops_.push_front(this); } - impl_ptr.reset(); - ex.on_work_finished(); - delete this; + keep.reset(); + exec.on_work_finished(); } } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/posix/posix_random_access_file_service.hpp b/include/boost/corosio/native/detail/posix/posix_random_access_file_service.hpp index 6a3e91664..317d291ed 100644 --- a/include/boost/corosio/native/detail/posix/posix_random_access_file_service.hpp +++ b/include/boost/corosio/native/detail/posix/posix_random_access_file_service.hpp @@ -17,11 +17,12 @@ #include #include #include +#include +#include #include #include -#include -#include +#include namespace boost::corosio::detail { @@ -29,6 +30,8 @@ namespace boost::corosio::detail { class BOOST_COROSIO_DECL posix_random_access_file_service final : public random_access_file_service { + friend class posix_random_access_file; + public: explicit posix_random_access_file_service(capy::execution_context& ctx) : sched_(&get_scheduler(ctx)) @@ -45,16 +48,7 @@ class BOOST_COROSIO_DECL posix_random_access_file_service final io_object::implementation* construct() override { - auto ptr = std::make_shared(*this); - auto* impl = ptr.get(); - - { - std::lock_guard lock(mutex_); - file_list_.push_back(impl); - file_ptrs_[impl] = std::move(ptr); - } - - return impl; + return object_pool_.acquire(*this); } void destroy(io_object::implementation* p) override @@ -62,7 +56,7 @@ class BOOST_COROSIO_DECL posix_random_access_file_service final auto& impl = static_cast(*p); impl.cancel(); impl.close_file(); - destroy_impl(impl); + release(&impl); } void close(io_object::handle& h) override @@ -90,21 +84,24 @@ class BOOST_COROSIO_DECL posix_random_access_file_service final void shutdown() override { - std::lock_guard lock(mutex_); - for (auto* impl = file_list_.pop_front(); impl != nullptr; - impl = file_list_.pop_front()) + // See uring_socket_service_base::shutdown(): snapshot under an + // acquired reference, cancel+close without the pool lock held; + // shutdown() sets shutting-down and takes the snapshot in one + // critical section, so a close that drops the last ref deletes + // rather than recycles. + std::vector live; + object_pool_.shutdown( + [&](posix_random_access_file* f) + { + acquire(f); + live.push_back(f); + }); + for (auto* f : live) { - impl->cancel(); - impl->close_file(); + f->cancel(); + f->close_file(); + release(f); } - file_ptrs_.clear(); - } - - void destroy_impl(posix_random_access_file& impl) - { - std::lock_guard lock(mutex_); - file_list_.remove(&impl); - file_ptrs_.erase(&impl); } void post(scheduler_op* op) @@ -143,12 +140,7 @@ class BOOST_COROSIO_DECL posix_random_access_file_service final private: scheduler* sched_; thread_pool_ref pool_; - std::mutex mutex_; - intrusive_list file_list_; - std::unordered_map< - posix_random_access_file*, - std::shared_ptr> - file_ptrs_; + object_pool object_pool_; }; // --------------------------------------------------------------------------- @@ -183,7 +175,7 @@ posix_random_access_file::read_some_at( return cont.h; } - auto* op = new raf_op(); + auto* op = acquire_op(); op->is_read = true; op->offset = offset; op->fd = fd_; @@ -195,13 +187,13 @@ posix_random_access_file::read_some_at( op->iovecs[i].iov_len = bufs[i].size(); } - op->h = cont.h; - op->awaiting = &cont; - op->ex = ex; - op->ec_out = ec; - op->bytes_out = bytes_out; - op->file_ = this; - op->impl_ptr = this->shared_from_this(); + op->h = cont.h; + op->awaiting = &cont; + op->ex = ex; + op->ec_out = ec; + op->bytes_out = bytes_out; + op->file_ = this; + op->object_ref_ = detail::object_ref(this); op->start(token); op->ex.on_work_started(); @@ -219,7 +211,7 @@ posix_random_access_file::read_some_at( // like the closed-descriptor and zero-length exits above rather // than through a completion the scheduler has to carry back. // destroy() is the discard the op never reaching the queue - // needs: it unlinks, unwinds the work count and frees. + // needs: it unlinks, unwinds the work count and recycles. op->destroy(); *ec = pec; *bytes_out = 0; @@ -256,7 +248,7 @@ posix_random_access_file::write_some_at( return cont.h; } - auto* op = new raf_op(); + auto* op = acquire_op(); op->is_read = false; op->offset = offset; op->fd = fd_; @@ -268,13 +260,13 @@ posix_random_access_file::write_some_at( op->iovecs[i].iov_len = bufs[i].size(); } - op->h = cont.h; - op->awaiting = &cont; - op->ex = ex; - op->ec_out = ec; - op->bytes_out = bytes_out; - op->file_ = this; - op->impl_ptr = this->shared_from_this(); + op->h = cont.h; + op->awaiting = &cont; + op->ex = ex; + op->ec_out = ec; + op->bytes_out = bytes_out; + op->file_ = this; + op->object_ref_ = detail::object_ref(this); op->start(token); op->ex.on_work_started(); @@ -292,7 +284,7 @@ posix_random_access_file::write_some_at( // like the closed-descriptor and zero-length exits above rather // than through a completion the scheduler has to carry back. // destroy() is the discard the op never reaching the queue - // needs: it unlinks, unwinds the work count and frees. + // needs: it unlinks, unwinds the work count and recycles. op->destroy(); *ec = pec; *bytes_out = 0; @@ -360,6 +352,12 @@ posix_random_access_file::raf_op::do_work(pool_work_item* w) noexcept self->svc_.post(static_cast(op)); } +inline void +posix_random_access_file::retire() noexcept +{ + svc_.object_pool_.recycle(this); +} + } // namespace boost::corosio::detail #endif // BOOST_COROSIO_POSIX diff --git a/include/boost/corosio/native/detail/posix/posix_resolver.hpp b/include/boost/corosio/native/detail/posix/posix_resolver.hpp index a72010b79..0498b022d 100644 --- a/include/boost/corosio/native/detail/posix/posix_resolver.hpp +++ b/include/boost/corosio/native/detail/posix/posix_resolver.hpp @@ -57,11 +57,12 @@ Class Hierarchy --------------- - posix_resolver_service (execution_context service, one per context) - - Owns all posix_resolver instances via shared_ptr + - Owns all posix_resolver instances via a recycling object_pool - Stores scheduler* for posting completions - posix_resolver (one per resolver object) - Contains embedded resolve_op and reverse_resolve_op for reuse - - Uses shared_from_this to prevent premature destruction + - Each op holds an object_ref keepalive while its pool work is + in flight, preventing premature destruction - resolve_op (forward resolution state) - Uses getaddrinfo() to resolve host/service to endpoints - reverse_resolve_op (reverse resolution state) @@ -86,9 +87,12 @@ Shutdown -------- - The resolver service cancels all resolvers and clears the impl map. - The thread pool service shuts down separately via execution_context - service ordering, joining all worker threads. + The resolver service cancels all resolvers and releases the + service's own reference on each; the recycling pool frees (rather + than recycles) once each resolver's in-flight pool work also + releases its reference. The thread pool service shuts down + separately via execution_context service ordering, joining all + worker threads. */ namespace boost::corosio::detail { @@ -154,7 +158,6 @@ class posix_resolver_service; */ class posix_resolver final : public resolver::implementation - , public std::enable_shared_from_this , public intrusive_list::node { friend class posix_resolver_service; @@ -213,11 +216,34 @@ class posix_resolver final posix_resolver* resolver_ = nullptr; /// Prevent impl destruction while work is in flight. - std::shared_ptr ref_; + detail::object_ref ref_; }; explicit posix_resolver(posix_resolver_service& svc) noexcept; + /// Recycle into the owning service's pool. Defined out-of-line + /// after posix_resolver_service for its complete type. + void retire() noexcept override; + + /** Reset for recycling. + + Both ops' own `reset()` already clears `host`/`service`/ + `stored_host`/`stored_service` via `.clear()` (preserving their + allocated capacity — the whole point of recycling), run at the + top of the next `resolve()`/`reverse_resolve()`. refs_ cannot + reach zero while a resolve is in flight (the pool op's + `object_ref` holds a reference for the duration), so asserting + `stop_cb` disengaged on both ops is a precondition check, not + a defensive one. + + @pre refs_ == 0, no resolve in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(!op_.stop_cb); + BOOST_COROSIO_ASSERT(!reverse_op_.stop_cb); + } + std::coroutine_handle<> resolve( std::coroutine_handle<>, capy::executor_ref, diff --git a/include/boost/corosio/native/detail/posix/posix_resolver_service.hpp b/include/boost/corosio/native/detail/posix/posix_resolver_service.hpp index 83b90230a..7bcc216d2 100644 --- a/include/boost/corosio/native/detail/posix/posix_resolver_service.hpp +++ b/include/boost/corosio/native/detail/posix/posix_resolver_service.hpp @@ -17,9 +17,11 @@ #include #include +#include +#include #include -#include +#include namespace boost::corosio::detail { @@ -32,6 +34,8 @@ class BOOST_COROSIO_DECL posix_resolver_service final : public capy::execution_context::service , public io_object::io_service { + friend class posix_resolver; + public: using key_type = posix_resolver_service; @@ -52,11 +56,10 @@ class BOOST_COROSIO_DECL posix_resolver_service final { auto& impl = static_cast(*p); impl.cancel(); - destroy_impl(impl); + release(&impl); } void shutdown() override; - void destroy_impl(posix_resolver& impl); void post(scheduler_op* op); @@ -89,10 +92,7 @@ class BOOST_COROSIO_DECL posix_resolver_service final private: scheduler* sched_; thread_pool_ref pool_; - std::mutex mutex_; - intrusive_list resolver_list_; - std::unordered_map> - resolver_ptrs_; + object_pool object_pool_; }; // --------------------------------------------------------------------------- @@ -271,7 +271,7 @@ posix_resolver::resolve_op::operator()() // Hold the keepalive across the dispatch: it may be the last // reference to the implementation this op is embedded in. - auto prevent_destroy = std::move(impl_ptr); + auto prevent_destroy = std::move(object_ref_); ex.on_work_finished(); cont.h = h; dispatch_coro(ex, cont).resume(); @@ -283,7 +283,7 @@ posix_resolver::resolve_op::destroy() stop_cb.reset(); auto local_ex = ex; // May destroy the implementation, and with it this op. - impl_ptr.reset(); + object_ref_.reset(); local_ex.on_work_finished(); } @@ -328,7 +328,7 @@ posix_resolver::reverse_resolve_op::operator()() // Hold the keepalive across the dispatch: it may be the last // reference to the implementation this op is embedded in. - auto prevent_destroy = std::move(impl_ptr); + auto prevent_destroy = std::move(object_ref_); ex.on_work_finished(); cont.h = h; dispatch_coro(ex, cont).resume(); @@ -340,7 +340,7 @@ posix_resolver::reverse_resolve_op::destroy() stop_cb.reset(); auto local_ex = ex; // May destroy the implementation, and with it this op. - impl_ptr.reset(); + object_ref_.reset(); local_ex.on_work_finished(); } @@ -380,7 +380,7 @@ posix_resolver::resolve( // Prevent impl destruction while work is in flight resolve_pool_op_.resolver_ = this; - resolve_pool_op_.ref_ = this->shared_from_this(); + resolve_pool_op_.ref_ = detail::object_ref(this); resolve_pool_op_.func_ = &posix_resolver::do_resolve_work; if (auto pec = svc_.pool().post(&resolve_pool_op_)) { @@ -430,7 +430,7 @@ posix_resolver::reverse_resolve( // Prevent impl destruction while work is in flight reverse_pool_op_.resolver_ = this; - reverse_pool_op_.ref_ = this->shared_from_this(); + reverse_pool_op_.ref_ = detail::object_ref(this); reverse_pool_op_.func_ = &posix_resolver::do_reverse_resolve_work; if (auto pec = svc_.pool().post(&reverse_pool_op_)) { @@ -492,7 +492,7 @@ posix_resolver::do_resolve_work(pool_work_item* w) noexcept // Hand the keepalive to the op: the completion waits in the // scheduler's queue, and the implementation embedding it must // outlive that wait. Nothing may touch *self after the post. - self->op_.impl_ptr = std::move(pw->ref_); + self->op_.object_ref_ = std::move(pw->ref_); self->svc_.post(&self->op_); } @@ -543,7 +543,7 @@ posix_resolver::do_reverse_resolve_work(pool_work_item* w) noexcept // Hand the keepalive to the op: the completion waits in the // scheduler's queue, and the implementation embedding it must // outlive that wait. Nothing may touch *self after the post. - self->reverse_op_.impl_ptr = std::move(pw->ref_); + self->reverse_op_.object_ref_ = std::move(pw->ref_); self->svc_.post(&self->reverse_op_); } @@ -552,48 +552,42 @@ posix_resolver::do_reverse_resolve_work(pool_work_item* w) noexcept inline void posix_resolver_service::shutdown() { - std::lock_guard lock(mutex_); - - // Cancel all resolvers (sets cancelled flag checked by pool threads) - for (auto* impl = resolver_list_.pop_front(); impl != nullptr; - impl = resolver_list_.pop_front()) + // See uring_socket_service_base::shutdown(): snapshot under an + // acquired reference, cancel without the pool lock held; shutdown() + // sets shutting-down and takes the snapshot in one critical + // section, so a cancel that drops the last ref deletes rather than + // recycles. The thread pool service shuts down separately via + // execution_context ordering. + std::vector live; + object_pool_.shutdown( + [&](posix_resolver* r) + { + acquire(r); + live.push_back(r); + }); + for (auto* r : live) { - impl->cancel(); + r->cancel(); + release(r); } - - // Clear the map which releases shared_ptrs. - // The thread pool service shuts down separately via - // execution_context service ordering. - resolver_ptrs_.clear(); } inline io_object::implementation* posix_resolver_service::construct() { - auto ptr = std::make_shared(*this); - auto* impl = ptr.get(); - - { - std::lock_guard lock(mutex_); - resolver_list_.push_back(impl); - resolver_ptrs_[impl] = std::move(ptr); - } - - return impl; + return object_pool_.acquire(*this); } inline void -posix_resolver_service::destroy_impl(posix_resolver& impl) +posix_resolver_service::post(scheduler_op* op) { - std::lock_guard lock(mutex_); - resolver_list_.remove(&impl); - resolver_ptrs_.erase(&impl); + sched_->post(op); } inline void -posix_resolver_service::post(scheduler_op* op) +posix_resolver::retire() noexcept { - sched_->post(op); + svc_.object_pool_.recycle(this); } } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/posix/posix_signal.hpp b/include/boost/corosio/native/detail/posix/posix_signal.hpp index eef8a9cce..7ba5fc133 100644 --- a/include/boost/corosio/native/detail/posix/posix_signal.hpp +++ b/include/boost/corosio/native/detail/posix/posix_signal.hpp @@ -123,6 +123,31 @@ class posix_signal final public: explicit posix_signal(posix_signal_service& svc) noexcept; + /// Recycle into the owning service's pool. Defined out-of-line + /// after posix_signal_service for its complete type. + void retire() noexcept override; + + /** Reset for recycling. + + `destroy()` already calls `clear()` (empties the registration + list) and `disarm_stop()` before the refcount reaches zero, so + those are asserted rather than re-cleared. `destroy()` also + calls `cancel()`, which deliberately latches `cancelled_` to + abort any wait still unwinding — that latch is this object's + to clear for its next, unrelated use, not something reset() + can skip by asserting it already false. + + @pre refs_ == 0, registration list empty, no wait in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(signals_ == nullptr); + BOOST_COROSIO_ASSERT(!waiting_); + BOOST_COROSIO_ASSERT(!stop_cb_); + cancelled_ = false; + token_cancelled_ = false; + } + std::coroutine_handle<> wait( std::coroutine_handle<>, capy::executor_ref, diff --git a/include/boost/corosio/native/detail/posix/posix_signal_service.hpp b/include/boost/corosio/native/detail/posix/posix_signal_service.hpp index 420b24df5..a9502d91b 100644 --- a/include/boost/corosio/native/detail/posix/posix_signal_service.hpp +++ b/include/boost/corosio/native/detail/posix/posix_signal_service.hpp @@ -18,6 +18,8 @@ #include #include +#include +#include #include #include #include @@ -25,6 +27,7 @@ #include #include +#include #include #include @@ -65,7 +68,7 @@ 2. posix_signal_service (one per execution_context) - Maintains registrations_[] table indexed by signal number - Each slot is a doubly-linked list of signal_registrations for that signal - - Also maintains impl_list_ of all posix_signal objects it owns + - Also owns a recycling pool of all posix_signal objects it owns 3. posix_signal (one per signal_set) - Owns a singly-linked list (sorted by signal number) of signal_registrations @@ -154,6 +157,8 @@ class BOOST_COROSIO_DECL posix_signal_service final : public capy::execution_context::service , public io_object::io_service { + friend class posix_signal; + public: using key_type = posix_signal_service; @@ -171,18 +176,18 @@ class BOOST_COROSIO_DECL posix_signal_service final [[maybe_unused]] auto n = impl.clear(); impl.disarm_stop(); impl.cancel(); - destroy_impl(impl); + release(&impl); } /** Shut down the service. - Destroys every implementation the service still owns and gives - each of their registrations back to the process-global table. + Gives every registration its live implementations still hold + back to the process-global table. The implementations are + freed when their handles release them, or when the service is + destroyed if a handle outlives it. */ void shutdown() override; - void destroy_impl(posix_signal& impl); - std::error_code add_signal( posix_signal& impl, int signal_number, signal_set::flags_t flags); @@ -233,7 +238,7 @@ class BOOST_COROSIO_DECL posix_signal_service final std::mutex reader_mutex_; bool reader_registered_ = false; - intrusive_list impl_list_; + object_pool pool_; // Per-signal registration table signal_registration* registrations_[max_signal_number]; @@ -526,12 +531,13 @@ inline posix_signal_service::~posix_signal_service() inline void posix_signal_service::shutdown() { - // Collected under the locks below and deleted after they are released: - // ~posix_signal destroys an armed stop_cb_, and ~stop_callback blocks - // until a concurrently running token_canceller returns -- which takes - // mutex_. Deleting while still holding mutex_ would self-deadlock the - // same way disarm_stop() would if called inside the locked loop. - intrusive_list doomed; + // Each visited impl is pinned with acquire() so a handle + // destroy() racing this walk cannot free it mid-walk. The + // matching release() runs after the locks drop: if that handle's + // destroy() raced in, this release() is the last one, and + // ~posix_signal destroys an armed stop_cb_ whose ~stop_callback + // waits on a token_canceller that takes mutex_. + std::vector doomed; { posix_signal_detail::signal_state* state = @@ -539,9 +545,18 @@ posix_signal_service::shutdown() std::lock_guard state_lock(state->mutex); std::lock_guard lock(mutex_); - for (auto* impl = impl_list_.pop_front(); impl != nullptr; - impl = impl_list_.pop_front()) + pool_.shutdown( + [&](posix_signal* impl) + { + acquire(impl); + doomed.push_back(impl); + }); + for (auto* impl : doomed) { + // A parked wait is abandoned like every other op at + // teardown; a later destroy() must not resume it mid-drain. + impl->waiting_ = false; + while (auto* reg = impl->signals_) { int const signal_number = reg->signal_number; @@ -568,47 +583,31 @@ posix_signal_service::shutdown() impl->signals_ = reg->next_in_set; delete reg; } - doomed.push_back(impl); } - // Every live registration hung off an implementation in impl_list_, - // so the whole table goes stale at once and can be dropped wholesale - // rather than node by node. It has to be dropped: deliver_signal() - // walks this service until the destructor unlinks it from the global - // list. + // Every live registration hung off an implementation this pool + // owns, so the whole table goes stale at once and can be dropped + // wholesale rather than node by node. It has to be dropped: + // deliver_signal() walks this service until the destructor + // unlinks it from the global list. for (int i = 0; i < max_signal_number; ++i) registrations_[i] = nullptr; } - for (auto* impl = doomed.pop_front(); impl != nullptr; - impl = doomed.pop_front()) - { - delete impl; - } + for (auto* impl : doomed) + release(impl); } inline io_object::implementation* posix_signal_service::construct() { - auto* impl = new posix_signal(*this); - - { - std::lock_guard lock(mutex_); - impl_list_.push_back(impl); - } - - return impl; + return pool_.acquire(*this); } inline void -posix_signal_service::destroy_impl(posix_signal& impl) +posix_signal::retire() noexcept { - { - std::lock_guard lock(mutex_); - impl_list_.remove(&impl); - } - - delete &impl; + svc_.pool_.recycle(this); } inline std::error_code diff --git a/include/boost/corosio/native/detail/posix/posix_stream_file.hpp b/include/boost/corosio/native/detail/posix/posix_stream_file.hpp index 9697c49d9..f59483311 100644 --- a/include/boost/corosio/native/detail/posix/posix_stream_file.hpp +++ b/include/boost/corosio/native/detail/posix/posix_stream_file.hpp @@ -59,7 +59,8 @@ to a shared thread pool, with completion posted back to the scheduler. This follows the same pattern as posix_resolver: pool_work_item for - dispatch, scheduler_op for completion, shared_from_this for lifetime. + dispatch, scheduler_op for completion, an object_ref keepalive on + each op for lifetime while its pool work is in flight. Completion Flow --------------- @@ -87,7 +88,6 @@ class posix_stream_file_service; */ class posix_stream_file final : public stream_file::implementation - , public std::enable_shared_from_this , public intrusive_list::node { friend class posix_stream_file_service; @@ -126,7 +126,7 @@ class posix_stream_file final is_read = false; cancelled.store(false, std::memory_order_relaxed); stop_cb.reset(); - impl_ptr.reset(); + object_ref_.reset(); ec_out = nullptr; bytes_out = nullptr; } @@ -139,11 +139,31 @@ class posix_stream_file final struct pool_op : pool_work_item { posix_stream_file* file_ = nullptr; - std::shared_ptr ref_; + detail::object_ref ref_; }; explicit posix_stream_file(posix_stream_file_service& svc) noexcept; + /// Recycle into the owning service's pool. Defined out-of-line + /// after posix_stream_file_service for its complete type. + void retire() noexcept override; + + /** Reset op slots for recycling. + + `close_file()` already drove fd_ to its closed value before + the refcount reached zero; each op's own `reset()` runs again + before its next use, so only the `stop_cb` invariant is worth + asserting here. + + @pre refs_ == 0, fd closed, no op in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(fd_ == -1); + BOOST_COROSIO_ASSERT(!read_op_.stop_cb); + BOOST_COROSIO_ASSERT(!write_op_.stop_cb); + } + // -- io_stream::implementation -- std::coroutine_handle<> read_some( @@ -417,10 +437,10 @@ posix_stream_file::file_op::operator()() errn != 0 ? make_err(errn) : std::error_code{}, is_read, bytes_transferred, /*empty_buffer=*/false); - // Move impl_ptr to a local so members remain valid through - // dispatch — impl_ptr may be the last shared_ptr keeping - // the parent posix_stream_file (which embeds this file_op) alive. - auto prevent_destroy = std::move(impl_ptr); + // Move object_ref_ to a local so members remain valid through + // dispatch — this may be the last reference keeping the parent + // posix_stream_file (which embeds this file_op) alive. + auto prevent_destroy = std::move(object_ref_); ex.on_work_finished(); cont.h = h; dispatch_coro(ex, cont).resume(); @@ -431,7 +451,7 @@ posix_stream_file::file_op::destroy() { stop_cb.reset(); auto local_ex = ex; - impl_ptr.reset(); + object_ref_.reset(); local_ex.on_work_finished(); } diff --git a/include/boost/corosio/native/detail/posix/posix_stream_file_service.hpp b/include/boost/corosio/native/detail/posix/posix_stream_file_service.hpp index adbd43599..75fd6ff34 100644 --- a/include/boost/corosio/native/detail/posix/posix_stream_file_service.hpp +++ b/include/boost/corosio/native/detail/posix/posix_stream_file_service.hpp @@ -17,10 +17,11 @@ #include #include #include +#include +#include #include -#include -#include +#include namespace boost::corosio::detail { @@ -31,6 +32,8 @@ namespace boost::corosio::detail { */ class BOOST_COROSIO_DECL posix_stream_file_service final : public file_service { + friend class posix_stream_file; + public: explicit posix_stream_file_service(capy::execution_context& ctx) : sched_(&get_scheduler(ctx)) @@ -46,16 +49,7 @@ class BOOST_COROSIO_DECL posix_stream_file_service final : public file_service io_object::implementation* construct() override { - auto ptr = std::make_shared(*this); - auto* impl = ptr.get(); - - { - std::lock_guard lock(mutex_); - file_list_.push_back(impl); - file_ptrs_[impl] = std::move(ptr); - } - - return impl; + return object_pool_.acquire(*this); } void destroy(io_object::implementation* p) override @@ -63,7 +57,7 @@ class BOOST_COROSIO_DECL posix_stream_file_service final : public file_service auto& impl = static_cast(*p); impl.cancel(); impl.close_file(); - destroy_impl(impl); + release(&impl); } void close(io_object::handle& h) override @@ -90,21 +84,24 @@ class BOOST_COROSIO_DECL posix_stream_file_service final : public file_service void shutdown() override { - std::lock_guard lock(mutex_); - for (auto* impl = file_list_.pop_front(); impl != nullptr; - impl = file_list_.pop_front()) + // See uring_socket_service_base::shutdown(): snapshot under an + // acquired reference, cancel+close without the pool lock held; + // shutdown() sets shutting-down and takes the snapshot in one + // critical section, so a close that drops the last ref deletes + // rather than recycles. + std::vector live; + object_pool_.shutdown( + [&](posix_stream_file* f) + { + acquire(f); + live.push_back(f); + }); + for (auto* f : live) { - impl->cancel(); - impl->close_file(); + f->cancel(); + f->close_file(); + release(f); } - file_ptrs_.clear(); - } - - void destroy_impl(posix_stream_file& impl) - { - std::lock_guard lock(mutex_); - file_list_.remove(&impl); - file_ptrs_.erase(&impl); } void post(scheduler_op* op) @@ -143,10 +140,7 @@ class BOOST_COROSIO_DECL posix_stream_file_service final : public file_service private: scheduler* sched_; thread_pool_ref pool_; - std::mutex mutex_; - intrusive_list file_list_; - std::unordered_map> - file_ptrs_; + object_pool object_pool_; }; // --------------------------------------------------------------------------- @@ -205,7 +199,7 @@ posix_stream_file::read_some( op.ex.on_work_started(); read_pool_op_.file_ = this; - read_pool_op_.ref_ = this->shared_from_this(); + read_pool_op_.ref_ = detail::object_ref(this); read_pool_op_.func_ = &posix_stream_file::do_read_work; if (auto pec = svc_.pool().post(&read_pool_op_)) { @@ -260,7 +254,7 @@ posix_stream_file::do_read_work(pool_work_item* w) noexcept } } - op.impl_ptr = std::move(pw->ref_); + op.object_ref_ = std::move(pw->ref_); self->svc_.post(&op); } @@ -316,7 +310,7 @@ posix_stream_file::write_some( op.ex.on_work_started(); write_pool_op_.file_ = this; - write_pool_op_.ref_ = this->shared_from_this(); + write_pool_op_.ref_ = detail::object_ref(this); write_pool_op_.func_ = &posix_stream_file::do_write_work; if (auto pec = svc_.pool().post(&write_pool_op_)) { @@ -371,10 +365,16 @@ posix_stream_file::do_write_work(pool_work_item* w) noexcept } } - op.impl_ptr = std::move(pw->ref_); + op.object_ref_ = std::move(pw->ref_); self->svc_.post(&op); } +inline void +posix_stream_file::retire() noexcept +{ + svc_.object_pool_.recycle(this); +} + } // namespace boost::corosio::detail #endif // BOOST_COROSIO_POSIX diff --git a/include/boost/corosio/native/detail/reactor/reactor_acceptor.hpp b/include/boost/corosio/native/detail/reactor/reactor_acceptor.hpp index 58cd73e7c..29dcc72d8 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_acceptor.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_acceptor.hpp @@ -14,11 +14,14 @@ #include #include #include +#include +#include #include #include #include #include +#include #include #include #include @@ -59,7 +62,6 @@ template< class Endpoint = endpoint> class reactor_acceptor : public ImplBase - , public std::enable_shared_from_this , public reactor_io_core , public intrusive_list::node { @@ -252,7 +254,6 @@ class reactor_acceptor void do_close_socket() noexcept { this->abandon_all(); - this->unregister_fd(fd_); if (fd_ >= 0) { ::close(fd_); @@ -266,12 +267,52 @@ class reactor_acceptor { this->abandon_all(); native_handle_type released = fd_; - this->unregister_fd(fd_); + fd_ = -1; + local_endpoint_ = Endpoint{}; + return released; + } + + /** Reset descriptor state for recycling. + + Actively re-initializes every field it owns rather than + trusting `close_socket()`'s prior writes to still be there — + see reactor_basic_socket::reuse() for the same rationale. + `object_ref_` and `is_enqueued_` are asserted instead: `poison()` + never touches them, so they must already hold their + zero-action-complete state. + + @pre refs_ == 0, fd closed and deregistered, no op in flight. + */ + void reuse() noexcept + { fd_ = -1; local_endpoint_ = Endpoint{}; - return released; + this->reset_desc_state(); + BOOST_COROSIO_ASSERT(!acc_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_wr_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_er_.stop_cb); } +#if !defined(NDEBUG) + /** Poison the fields `reuse()` re-initializes, before the pool + parks this impl on the free list. See + reactor_basic_socket::poison() for the full rationale and the + list of fields deliberately left untouched. + + @pre refs_ == 0, fd closed and deregistered, no op in flight. + */ + void poison() noexcept + { + auto smash = [](auto& field) { + std::memset(static_cast(&field), 0xDB, sizeof(field)); + }; + smash(fd_); + smash(local_endpoint_); + this->poison_desc_state(); + } +#endif + /** Bind the acceptor socket to an endpoint. Caches the resolved local endpoint (including ephemeral @@ -434,7 +475,7 @@ reactor_acceptor< op.ec_out = ec; op.fd = this->fd_; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(ENOTSUP, 0); svc_.post(&op); return std::noop_coroutine(); @@ -465,7 +506,7 @@ reactor_acceptor< op.ec_out = ec; op.fd = this->fd_; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); // A listener's readiness can predate the wait: an adopted or // shared descriptor has history the reactor never saw, and an @@ -481,25 +522,35 @@ reactor_acceptor< svc_.work_started(); - std::lock_guard lock(desc_state_.mutex); - if (op.cancelled.load(std::memory_order_acquire)) + // Posted after the descriptor lock is released: the scheduler + // takes its own lock before descriptor locks. + bool post_now = false; { - svc_.post(&op); - svc_.work_finished(); + std::lock_guard lock(desc_state_.mutex); + if (op.cancelled.load(std::memory_order_acquire)) + { + post_now = true; + } + else if (WaitOp::probe(this->fd_, event, perr)) + { + // Close the probe-to-park window: an edge that landed after + // the first probe was consumed, so re-check under the mutex + // the dispatch path holds. + op.complete(perr, 0); + post_now = true; + } + else + { + *desc_slot_ptr = &op; + } } - else if (WaitOp::probe(this->fd_, event, perr)) + if (post_now) { - // Close the probe-to-park window: an edge that landed after - // the first probe was consumed, so re-check under the mutex - // the dispatch path holds. - op.complete(perr, 0); svc_.post(&op); svc_.work_finished(); } else { - *desc_slot_ptr = &op; - // Select watches an fd only while an op is parked; see // register_op. if constexpr (Service::needs_park_notification) diff --git a/include/boost/corosio/native/detail/reactor/reactor_acceptor_service.hpp b/include/boost/corosio/native/detail/reactor/reactor_acceptor_service.hpp index 5063161e0..7bd11e1c4 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_acceptor_service.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_acceptor_service.hpp @@ -11,12 +11,12 @@ #define BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_ACCEPTOR_SERVICE_HPP #include +#include #include #include #include #include -#include namespace boost::corosio::detail { @@ -45,6 +45,13 @@ template< class reactor_acceptor_service : public ServiceBase { friend Derived; + + // The intermediate CRTP template below (not Impl itself) is what + // actually reaches into state_->pool_ from retire() -- see + // reactor_socket_finals.hpp. + template + friend class reactor_acceptor_impl; + using state_type = reactor_service_state; protected: @@ -62,31 +69,23 @@ class reactor_acceptor_service : public ServiceBase void shutdown() override { - std::lock_guard lock(state_->mutex_); + state_->pool_.shutdown([](Impl* impl) { impl->close_socket(); }); - while (auto* impl = state_->impl_list_.pop_front()) - impl->close_socket(); + // See reactor_socket_service::shutdown(): queued ops hold their + // own reference; the scheduler's drain releases them after us, + // and shutting-down mode deletes rather than recycles at zero. } io_object::implementation* construct() override { - auto impl = std::make_shared(static_cast(*this)); - auto* raw = impl.get(); - - std::lock_guard lock(state_->mutex_); - state_->impl_ptrs_.emplace(raw, std::move(impl)); - state_->impl_list_.push_back(raw); - - return raw; + return state_->pool_.acquire(static_cast(*this)); } void destroy(io_object::implementation* impl) override { auto* typed = static_cast(impl); typed->close_socket(); - std::lock_guard lock(state_->mutex_); - state_->impl_list_.remove(typed); - state_->impl_ptrs_.erase(typed); + release(typed); } void close(io_object::handle& h) override diff --git a/include/boost/corosio/native/detail/reactor/reactor_backend.hpp b/include/boost/corosio/native/detail/reactor/reactor_backend.hpp index a60287576..0bcaee5d0 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_backend.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_backend.hpp @@ -120,6 +120,8 @@ reactor_acceptor_impl< if (impl_out) *impl_out = nullptr; } + // Completed without parking: nothing left for a stop to cancel. + op.stop_cb.reset(); op.cont.h = h; return dispatch_coro(ex, op.cont); } @@ -128,36 +130,43 @@ reactor_acceptor_impl< op.peer_storage = peer_storage; op.peer_addrlen = peer_addrlen; op.complete(0, 0); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->svc_.post(&op); return std::noop_coroutine(); } if (errno == EAGAIN || errno == EWOULDBLOCK) { - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->svc_.work_started(); - std::lock_guard lock(this->desc_state_.mutex); - bool io_done = false; - if (this->desc_state_.read_ready) + // Posted after the descriptor lock is released: the scheduler + // takes its own lock before descriptor locks. + bool post_now = false; { - this->desc_state_.read_ready = false; - op.perform_io(); - io_done = (op.errn != EAGAIN && op.errn != EWOULDBLOCK); - if (!io_done) - op.errn = 0; - } + std::lock_guard lock(this->desc_state_.mutex); + bool io_done = false; + if (this->desc_state_.read_ready) + { + this->desc_state_.read_ready = false; + op.perform_io(); + io_done = (op.errn != EAGAIN && op.errn != EWOULDBLOCK); + if (!io_done) + op.errn = 0; + } - if (io_done || op.cancelled.load(std::memory_order_acquire)) + if (io_done || op.cancelled.load(std::memory_order_acquire)) + post_now = true; + else + this->desc_state_.read_op = &op; + } + if (post_now) { this->svc_.post(&op); this->svc_.work_finished(); } else { - this->desc_state_.read_op = &op; - // Select watches reads only for parked ops; see register_op. if constexpr (Traits::needs_park_notification) this->svc_.scheduler().notify_reactor(); @@ -166,7 +175,7 @@ reactor_acceptor_impl< } op.complete(errno, 0); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->svc_.post(&op); return std::noop_coroutine(); } diff --git a/include/boost/corosio/native/detail/reactor/reactor_basic_socket.hpp b/include/boost/corosio/native/detail/reactor/reactor_basic_socket.hpp index 0b72fbe8a..d8df5e7f2 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_basic_socket.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_basic_socket.hpp @@ -11,6 +11,7 @@ #ifndef BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_BASIC_SOCKET_HPP #define BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_BASIC_SOCKET_HPP +#include #include #include #include @@ -19,6 +20,7 @@ #include #include +#include #include #include #include @@ -130,7 +132,6 @@ class reactor_basic_socket void do_close_socket() noexcept { this->abandon_all(); - this->unregister_fd(fd_); if (fd_ >= 0) { ::close(fd_); @@ -148,11 +149,55 @@ class reactor_basic_socket { this->abandon_all(); native_handle_type released = fd_; - this->unregister_fd(fd_); + fd_ = -1; + local_endpoint_ = Endpoint{}; + return released; + } + + /** Reset descriptor state for recycling. + + Called by the owning service's `construct()` when popping this + impl back off the free list. Actively re-initializes every + field it owns rather than trusting `close_socket()`'s prior + writes to still be there — `poison()` below depends on that, + and a future field added to either side without the other + would otherwise go unnoticed until this one re-sets it. + `object_ref_` and `is_enqueued_` are asserted instead: `poison()` + never touches them, so they must already hold their + zero-action-complete state. + + @pre refs_ == 0, fd closed and deregistered, no op in flight. + */ + void reuse() noexcept + { fd_ = -1; local_endpoint_ = Endpoint{}; - return released; + this->reset_desc_state(); + } + +#if !defined(NDEBUG) + /** Poison the fields `reuse()` re-initializes, before the pool + parks this impl on the free list. + + A field `reuse()` forgets to reset then reads back as this + unmistakable byte pattern instead of silently matching + whatever `close_socket()` happened to leave, turning a missed + reset into an immediate, loud failure rather than a latent + stale-state bug. See `reactor_io_core::poison_desc_state()` + for what is left alone. + + @pre refs_ == 0, fd closed and deregistered, no op in flight. + */ + void poison() noexcept + { + auto smash = [](auto& field) { + std::memset(static_cast(&field), 0xDB, sizeof(field)); + }; + smash(fd_); + smash(local_endpoint_); + this->poison_desc_state(); } +#endif }; } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/reactor/reactor_datagram_socket.hpp b/include/boost/corosio/native/detail/reactor/reactor_datagram_socket.hpp index ef997ef52..73ccc88e7 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_datagram_socket.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_datagram_socket.hpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -21,6 +22,7 @@ #include #include +#include #include #include @@ -327,6 +329,40 @@ class reactor_datagram_socket return fd; } + /** Reset op slots and the cached peer for recycling. + + See reactor_stream_socket::reuse() — same rationale: op state + is overwritten by the next do_* call, so only `remote_endpoint_` + needs an active reset here; every other slot only asserts + `stop_cb` disengaged. + + @pre refs_ == 0, fd closed and deregistered, no op in flight. + */ + void reuse() noexcept + { + base_type::reuse(); + remote_endpoint_ = Endpoint{}; + BOOST_COROSIO_ASSERT(!conn_.stop_cb); + BOOST_COROSIO_ASSERT(!wr_.stop_cb); + BOOST_COROSIO_ASSERT(!rd_.stop_cb); + BOOST_COROSIO_ASSERT(!send_wr_.stop_cb); + BOOST_COROSIO_ASSERT(!recv_rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_wr_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_er_.stop_cb); + } + +#if !defined(NDEBUG) + /// Extend base_type::poison() to the cached peer endpoint. + void poison() noexcept + { + base_type::poison(); + std::memset( + static_cast(&remote_endpoint_), 0xDB, + sizeof(remote_endpoint_)); + } +#endif + /** Shut down part or all of the full-duplex connection. Not an override — concrete backends forward here. @@ -499,7 +535,7 @@ reactor_datagram_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(err, bytes); this->svc_.post(&op); return std::noop_coroutine(); @@ -511,7 +547,7 @@ reactor_datagram_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->register_op( op, this->desc_state_.write_op, this->desc_state_.write_ready, true); @@ -569,7 +605,7 @@ reactor_datagram_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(0, 0); this->svc_.post(&op); return std::noop_coroutine(); @@ -621,7 +657,7 @@ reactor_datagram_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(err, bytes); this->svc_.post(&op); return std::noop_coroutine(); @@ -633,7 +669,7 @@ reactor_datagram_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->register_op( op, this->desc_state_.read_op, this->desc_state_.read_ready); @@ -710,7 +746,7 @@ reactor_datagram_socket< op.fd = this->fd_; op.target_endpoint = ep; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(err, 0); this->svc_.post(&op); return std::noop_coroutine(); @@ -802,7 +838,7 @@ reactor_datagram_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(err, bytes); this->svc_.post(&op); return std::noop_coroutine(); @@ -814,7 +850,7 @@ reactor_datagram_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->register_op( op, this->desc_state_.write_op, this->desc_state_.write_ready, true); @@ -870,7 +906,7 @@ reactor_datagram_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(0, 0); this->svc_.post(&op); return std::noop_coroutine(); @@ -914,7 +950,7 @@ reactor_datagram_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(err, bytes); this->svc_.post(&op); return std::noop_coroutine(); @@ -926,7 +962,7 @@ reactor_datagram_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->register_op( op, this->desc_state_.read_op, this->desc_state_.read_ready); @@ -1012,7 +1048,7 @@ reactor_datagram_socket< op.ec_out = ec; op.fd = this->fd_; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(perr, 0); this->svc_.post(&op); return std::noop_coroutine(); @@ -1025,7 +1061,7 @@ reactor_datagram_socket< op.ec_out = ec; op.fd = this->fd_; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); // Force register_op's ready path so the wait op re-probes under // the descriptor mutex before parking. An edge consumed between diff --git a/include/boost/corosio/native/detail/reactor/reactor_descriptor.hpp b/include/boost/corosio/native/detail/reactor/reactor_descriptor.hpp index 92e477c51..7623d79ee 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_descriptor.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_descriptor.hpp @@ -19,6 +19,8 @@ #include #include #include +#include +#include #include #include #include @@ -29,6 +31,7 @@ #include #include +#include #include #include #include @@ -195,7 +198,6 @@ reactor_descriptor_wait_op::operator()() template class reactor_descriptor : public posix_stream_descriptor::implementation - , public std::enable_shared_from_this , public reactor_io_core , public intrusive_list::node { @@ -265,8 +267,42 @@ class reactor_descriptor this->cancel_all(); } + /// Recycle into the owning service's pool at zero references. + void retire() noexcept override + { +#if !defined(NDEBUG) + poison(); +#endif + svc_.state_->pool_.recycle(static_cast(this)); + } + // --- Service-facing (non-virtual) --- + /** Reset descriptor state for recycling. + + Re-initializes every field it owns rather than trusting + `close_descriptor()`'s prior writes; see + reactor_basic_socket::reuse(). + + @pre refs_ == 0, fd closed and deregistered, no op in flight. + */ + void reuse() noexcept + { + fd_ = -1; + nonblocking_.store(false, std::memory_order_relaxed); + this->reset_desc_state(); + } + +#if !defined(NDEBUG) + /// Poison the fields `reuse()` re-initializes; see + /// reactor_basic_socket::poison(). + void poison() noexcept + { + std::memset(static_cast(&fd_), 0xDB, sizeof(fd_)); + this->poison_desc_state(); + } +#endif + /** Adopt the fd, initialize descriptor state, and register it. @param fd The descriptor to adopt. @@ -383,7 +419,6 @@ void reactor_descriptor::quiesce() noexcept { this->abandon_all(); - this->unregister_fd(fd_); // The next adopted fd starts from an unknown flag state. nonblocking_.store(false, std::memory_order_relaxed); } @@ -463,7 +498,7 @@ reactor_descriptor::do_read_some( if (fd_ < 0) { op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(EBADF, 0); svc_.post(&op); return std::noop_coroutine(); @@ -477,7 +512,7 @@ reactor_descriptor::do_read_some( { op.empty_buffer_read = true; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(0, 0); svc_.post(&op); return std::noop_coroutine(); @@ -488,7 +523,7 @@ reactor_descriptor::do_read_some( if (int const nerr = arm_nonblocking()) { op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(nerr, 0); svc_.post(&op); return std::noop_coroutine(); @@ -538,7 +573,7 @@ reactor_descriptor::do_read_some( return dispatch_coro(ex, op.cont); } op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(err, bytes); svc_.post(&op); return std::noop_coroutine(); @@ -547,7 +582,7 @@ reactor_descriptor::do_read_some( // EAGAIN — register with reactor op.fd = fd_; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->register_op(op, desc_state_.read_op, desc_state_.read_ready); return std::noop_coroutine(); @@ -573,7 +608,7 @@ reactor_descriptor::do_write_some( if (fd_ < 0) { op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(EBADF, 0); svc_.post(&op); return std::noop_coroutine(); @@ -586,7 +621,7 @@ reactor_descriptor::do_write_some( if (op.iovec_count == 0 || (op.iovec_count == 1 && bufs[0].size() == 0)) { op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(0, 0); svc_.post(&op); return std::noop_coroutine(); @@ -595,7 +630,7 @@ reactor_descriptor::do_write_some( if (int const nerr = arm_nonblocking()) { op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(nerr, 0); svc_.post(&op); return std::noop_coroutine(); @@ -632,7 +667,7 @@ reactor_descriptor::do_write_some( return dispatch_coro(ex, op.cont); } op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(err, bytes); svc_.post(&op); return std::noop_coroutine(); @@ -641,7 +676,7 @@ reactor_descriptor::do_write_some( // EAGAIN — register with reactor op.fd = fd_; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->register_op(op, desc_state_.write_op, desc_state_.write_ready, true); return std::noop_coroutine(); @@ -702,7 +737,7 @@ reactor_descriptor::do_wait( op.ec_out = ec; op.fd = fd_; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(perr, 0); svc_.post(&op); return std::noop_coroutine(); @@ -715,7 +750,7 @@ reactor_descriptor::do_wait( op.ec_out = ec; op.fd = fd_; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); // Force register_op's ready path so the wait op re-probes under the // descriptor mutex before parking. A stale write_ready latched at diff --git a/include/boost/corosio/native/detail/reactor/reactor_descriptor_service.hpp b/include/boost/corosio/native/detail/reactor/reactor_descriptor_service.hpp index df5be69ac..437eedb38 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_descriptor_service.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_descriptor_service.hpp @@ -34,9 +34,9 @@ PARALLEL COPY: construct, destroy, close and shutdown here mirror the same four members of reactor_socket_service.hpp, which this cannot reuse because it calls close_socket() by name. A fix to the service - lifecycle -- the construct/destroy bookkeeping under state_->mutex_, - or shutdown's deliberate retention of impl_ptrs_ so impls outlive the - scheduler's drain -- belongs in both files. + lifecycle -- the pool's acquire/recycle, or shutdown's reliance on + queued ops holding their own reference through the scheduler's + drain -- belongs in both files. */ namespace boost::corosio::detail { @@ -55,6 +55,10 @@ class reactor_descriptor_service : public descriptor_service friend Derived; + // The CRTP base reaches into state_->pool_ from retire(). + template + friend class reactor_descriptor; + protected: // NOLINTNEXTLINE(bugprone-crtp-constructor-accessibility) explicit reactor_descriptor_service(capy::execution_context& ctx) @@ -77,36 +81,24 @@ class reactor_descriptor_service : public descriptor_service void shutdown() override { - std::lock_guard lock(state_->mutex_); - - while (auto* impl = state_->impl_list_.pop_front()) - impl->close_descriptor(); + state_->pool_.shutdown( + [](DescFinal* impl) { impl->close_descriptor(); }); - // Don't clear impl_ptrs_ here: the scheduler shuts down after us - // and drains completed_ops_, so every impl must outlive that. + // See reactor_socket_service::shutdown(): queued ops hold their + // own reference; the scheduler's drain releases them after us, + // and shutting-down mode deletes rather than recycles at zero. } io_object::implementation* construct() override { - auto impl = std::make_shared(static_cast(*this)); - auto* raw = impl.get(); - - { - std::lock_guard lock(state_->mutex_); - state_->impl_ptrs_.emplace(raw, std::move(impl)); - state_->impl_list_.push_back(raw); - } - - return raw; + return state_->pool_.acquire(static_cast(*this)); } void destroy(io_object::implementation* impl) override { auto* typed = static_cast(impl); typed->close_descriptor(); - std::lock_guard lock(state_->mutex_); - state_->impl_list_.remove(typed); - state_->impl_ptrs_.erase(typed); + release(typed); } void close(io_object::handle& h) override diff --git a/include/boost/corosio/native/detail/reactor/reactor_descriptor_state.hpp b/include/boost/corosio/native/detail/reactor/reactor_descriptor_state.hpp index a41f07ede..d6620c7ba 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_descriptor_state.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_descriptor_state.hpp @@ -43,7 +43,9 @@ namespace boost::corosio::detail { The mutex protects operation pointers and ready flags. ready_events_ and is_enqueued_ are atomic for lock-free reactor access. */ -struct reactor_descriptor_state : scheduler_op +struct reactor_descriptor_state + : scheduler_op + , reactor_retirable { /// Protects operation pointers and ready/cancel flags. /// Becomes a no-op in single-threaded mode. @@ -93,15 +95,18 @@ struct reactor_descriptor_state : scheduler_op reactor_scheduler const* scheduler_ = nullptr; /// Prevents impl destruction while queued in the scheduler. - std::shared_ptr impl_ref_; + detail::object_ref object_ref_; /// Add ready events atomically. /// Release pairs with the consumer's acquire exchange on - /// ready_events_ so the consumer sees all flags. On x86 (TSO) - /// this compiles to the same LOCK OR as relaxed. + /// ready_events_ so the consumer sees all flags. Acquire pairs with + /// registration's release store, so a reactor pass on another + /// thread sees the state registration initialized before handing + /// it to the kernel. On x86 (TSO) this compiles to the same LOCK OR + /// as relaxed. void add_ready_events(std::uint32_t ev) noexcept { - ready_events_.fetch_or(ev, std::memory_order_release); + ready_events_.fetch_or(ev, std::memory_order_acq_rel); } /// Invoke deferred I/O and dispatch completions. @@ -110,12 +115,23 @@ struct reactor_descriptor_state : scheduler_op invoke_deferred_io(); } + /// Move @p keep into this state if it is queued without one. + void hand_off(object_ref& keep) noexcept override + { + // Locked even to read the flag: a popped invocation clears it at + // the start of its critical section and still uses this state + // until it unlocks, so an unlocked false does not mean idle. + conditionally_enabled_mutex::scoped_lock lock(mutex); + if (is_enqueued_.load(std::memory_order_acquire) && !object_ref_) + object_ref_ = std::move(keep); + } + /// Destroy without invoking. - /// Called during scheduler::shutdown() drain. Clear impl_ref_ to break + /// Called during scheduler::shutdown() drain. Clear object_ref_ to break /// the self-referential cycle set by close_socket(). void destroy() override { - impl_ref_.reset(); + object_ref_.reset(); } /** Perform deferred I/O and queue completions. @@ -130,18 +146,22 @@ struct reactor_descriptor_state : scheduler_op inline void reactor_descriptor_state::invoke_deferred_io() { - std::shared_ptr prevent_impl_destruction; + detail::object_ref prevent_impl_destruction; ready_queue local_ops; + // Read under the lock: once it drops, a retired owner may be + // recycled, so nothing below touches this state. + reactor_scheduler const* sched = nullptr; { conditionally_enabled_mutex::scoped_lock lock(mutex); + sched = scheduler_; - // Must clear is_enqueued_ and move impl_ref_ under the same + // Must clear is_enqueued_ and move object_ref_ under the same // lock that processes I/O. close_socket() checks is_enqueued_ // under this mutex — without atomicity between the flag store // and the ref move, close_socket() could see is_enqueued_==false, - // skip setting impl_ref_, and destroy the impl under us. - prevent_impl_destruction = std::move(impl_ref_); + // skip setting object_ref_, and destroy the impl under us. + prevent_impl_destruction = std::move(object_ref_); is_enqueued_.store(false, std::memory_order_release); std::uint32_t ev = ready_events_.exchange(0, std::memory_order_acquire); @@ -339,12 +359,12 @@ reactor_descriptor_state::invoke_deferred_io() scheduler_op* first = ready_as_op(local_ops.pop()); if (first) { - scheduler_->post_deferred_completions(local_ops); + sched->post_deferred_completions(local_ops); (*first)(); } else { - scheduler_->compensating_work_started(); + sched->compensating_work_started(); } } diff --git a/include/boost/corosio/native/detail/reactor/reactor_io_core.hpp b/include/boost/corosio/native/detail/reactor/reactor_io_core.hpp index 91e3fdac7..d273b58a3 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_io_core.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_io_core.hpp @@ -11,10 +11,11 @@ #define BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_IO_CORE_HPP #include +#include #include #include -#include +#include #include #include #include @@ -31,8 +32,8 @@ objects' own verbs stay per type. Derived supplies its op slots through for_each_op, - for_each_desc_entry and op_to_desc_slot, and must derive from - std::enable_shared_from_this. + for_each_desc_entry and op_to_desc_slot, and is an + io_object::implementation whose callers hold a reference on it. */ namespace boost::corosio::detail { @@ -69,10 +70,6 @@ class reactor_io_core template void cancel_single_op(Op& op) noexcept { - auto self = self_ptr()->weak_from_this().lock(); - if (!self) - return; - op.request_cancel(); reactor_op_base** desc_op_ptr = self_ptr()->op_to_desc_slot(op); @@ -92,7 +89,7 @@ class reactor_io_core } if (claimed) { - op.impl_ptr = self; + op.object_ref_ = detail::object_ref(self_ptr()); svc_.post(&op); svc_.work_finished(); } @@ -106,14 +103,17 @@ class reactor_io_core */ std::error_code register_fd(int fd) noexcept { - desc_state_.fd = fd; { + // A completion from an earlier registration may still read + // these under the mutex. std::lock_guard lock(desc_state_.mutex); + desc_state_.fd = fd; self_ptr()->for_each_desc_entry( [](auto&, reactor_op_base*& slot) { slot = nullptr; }); } if (auto ec = svc_.scheduler().register_descriptor(fd, &desc_state_)) { + std::lock_guard lock(desc_state_.mutex); desc_state_.fd = -1; desc_state_.registered_events = 0; return ec; @@ -134,61 +134,23 @@ class reactor_io_core bool is_write_direction = false) noexcept { svc_.work_started(); - - std::lock_guard lock(desc_state_.mutex); - bool io_done = false; - if (ready_flag) + if (park_op(op, desc_slot, ready_flag, is_write_direction)) { - ready_flag = false; - op.perform_io(); - io_done = (op.errn != EAGAIN && op.errn != EWOULDBLOCK); - if (!io_done) - op.errn = 0; - } - - if (io_done || op.cancelled.load(std::memory_order_acquire)) - { - svc_.post(&op); - svc_.work_finished(); + // Select rebuilds its fd_sets from parked ops only, so + // parking must wake it. Compiled away for epoll and kqueue. + if constexpr (Service::needs_park_notification) + svc_.scheduler().notify_reactor(); return; } - - if (desc_state_.unpollable) - { - // Nothing will ever report readiness for this fd. - op.complete(EOPNOTSUPP, 0); - svc_.post(&op); - svc_.work_finished(); - return; - } - - if (is_write_direction) - { - if (auto ec = svc_.scheduler().ensure_write_registered( - desc_state_.fd, &desc_state_)) - { - op.complete(ec.value(), 0); - svc_.post(&op); - svc_.work_finished(); - return; - } - } - - desc_slot = &op; - - // Select rebuilds its fd_sets from parked ops only, so parking - // must wake it. Compiled away for epoll and kqueue. - if constexpr (Service::needs_park_notification) - svc_.scheduler().notify_reactor(); + // Posted after the descriptor lock is released: the scheduler + // takes its own lock before descriptor locks. + svc_.post(&op); + svc_.work_finished(); } /// Cancel every pending operation. void cancel_all() noexcept { - auto self = self_ptr()->weak_from_this().lock(); - if (!self) - return; - self_ptr()->for_each_op([](auto& op) { op.request_cancel(); }); reactor_op_base* claimed[max_claimed]; @@ -204,22 +166,35 @@ class reactor_io_core } }); } - post_claimed(claimed, count, self); + post_claimed(claimed, count); } - /** Cancel every operation and claim every parked one for teardown. + /** Cancel every operation, drop the reactor registration, and + claim every parked op for teardown. - Also clears the cached edge flags and, if the state is queued - in the scheduler, pins the object alive until it is drained. + Deregistration hands a reference to the scheduler, which keeps + the object alive while a reactor pass may still deliver an + event for the descriptor. Afterwards the descriptor state reads + as closed, so the caller may close or release the fd. */ void abandon_all() noexcept { - auto self = self_ptr()->weak_from_this().lock(); - if (!self) - return; - self_ptr()->for_each_op([](auto& op) { op.request_cancel(); }); + int fd = -1; + bool registered = false; + { + std::lock_guard lock(desc_state_.mutex); + fd = desc_state_.fd; + registered = desc_state_.registered_events != 0; + } + if (fd >= 0 && registered) + { + svc_.scheduler().deregister_descriptor(fd); + svc_.scheduler().retire_descriptor( + desc_state_, detail::object_ref(self_ptr())); + } + reactor_op_base* claimed[max_claimed]; int count = 0; { @@ -234,25 +209,67 @@ class reactor_io_core }); desc_state_.read_ready = false; desc_state_.write_ready = false; - - // Must be set under the same lock that invoke_deferred_io - // clears is_enqueued_ under, or the object could be destroyed - // while the scheduler still holds the queued descriptor_state. - if (desc_state_.is_enqueued_.load(std::memory_order_acquire)) - desc_state_.impl_ref_ = self; + // Cleared before the fd is closed: a queued completion reads + // it under this mutex, and the number can be reused once + // closed. + desc_state_.fd = -1; + desc_state_.registered_events = 0; + desc_state_.unpollable = false; } - post_claimed(claimed, count, self); + post_claimed(claimed, count); } - /// Drop the reactor registration of @a fd and reset the state. - void unregister_fd(int fd) noexcept + /** Reset the descriptor state a recycled object starts from. + + Every field is reset rather than asserted so a debug build may + poison them on retirement. References and the queued flag are + asserted: nothing may hold one once the count reached zero. + */ + void reset_desc_state() noexcept { - if (fd >= 0 && desc_state_.registered_events != 0) - svc_.scheduler().deregister_descriptor(fd); desc_state_.fd = -1; desc_state_.registered_events = 0; desc_state_.unpollable = false; + desc_state_.read_op = nullptr; + desc_state_.write_op = nullptr; + desc_state_.connect_op = nullptr; + desc_state_.wait_read_op = nullptr; + desc_state_.wait_write_op = nullptr; + desc_state_.wait_error_op = nullptr; + desc_state_.read_ready = false; + desc_state_.write_ready = false; + BOOST_COROSIO_ASSERT(!desc_state_.object_ref_); + BOOST_COROSIO_ASSERT(!desc_state_.retired_ref_); + BOOST_COROSIO_ASSERT(!desc_state_.dropped_ref_); + BOOST_COROSIO_ASSERT( + !desc_state_.is_enqueued_.load(std::memory_order_relaxed)); + } + +#if !defined(NDEBUG) + /** Poison the descriptor fields `reset_desc_state()` re-initializes. + + Leaves the mutex, the atomics, `scheduler_`, the references and + the retirement links alone: the scheduler and `recycle()` may + still read them. + */ + void poison_desc_state() noexcept + { + auto smash = [](auto& field) { + std::memset(static_cast(&field), 0xDB, sizeof(field)); + }; + smash(desc_state_.fd); + smash(desc_state_.registered_events); + smash(desc_state_.unpollable); + smash(desc_state_.read_op); + smash(desc_state_.write_op); + smash(desc_state_.connect_op); + smash(desc_state_.wait_read_op); + smash(desc_state_.wait_write_op); + smash(desc_state_.wait_error_op); + smash(desc_state_.read_ready); + smash(desc_state_.write_ready); } +#endif private: // A claim empties its slot, so no more ops are claimed than @@ -260,14 +277,56 @@ class reactor_io_core // wait_write, wait_error), however many ops share them. static constexpr int max_claimed = 6; - void post_claimed( - reactor_op_base** claimed, - int count, - std::shared_ptr const& self) noexcept + /** Park @a op unless it can complete now. + + @return True if parked; false if the caller must post it. + */ + template + bool park_op( + Op& op, + reactor_op_base*& desc_slot, + bool& ready_flag, + bool is_write_direction) noexcept + { + std::lock_guard lock(desc_state_.mutex); + if (ready_flag) + { + ready_flag = false; + op.perform_io(); + if (op.errn != EAGAIN && op.errn != EWOULDBLOCK) + return false; + op.errn = 0; + } + + if (op.cancelled.load(std::memory_order_acquire)) + return false; + + if (desc_state_.unpollable) + { + // Nothing will ever report readiness for this fd. + op.complete(EOPNOTSUPP, 0); + return false; + } + + if (is_write_direction) + { + if (auto ec = svc_.scheduler().ensure_write_registered( + desc_state_.fd, &desc_state_)) + { + op.complete(ec.value(), 0); + return false; + } + } + + desc_slot = &op; + return true; + } + + void post_claimed(reactor_op_base** claimed, int count) noexcept { for (int i = 0; i < count; ++i) { - claimed[i]->impl_ptr = self; + claimed[i]->object_ref_ = detail::object_ref(self_ptr()); svc_.post(claimed[i]); svc_.work_finished(); } diff --git a/include/boost/corosio/native/detail/reactor/reactor_op.hpp b/include/boost/corosio/native/detail/reactor/reactor_op.hpp index 986e590f0..1f6ab425d 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_op.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_op.hpp @@ -38,7 +38,7 @@ namespace boost::corosio::detail { impl pointers. Fields shared across all backends (errn, bytes_transferred, - cancelled, impl_ptr, perform_io, complete) live in + cancelled, object_ref_, perform_io, complete) live in reactor_op_base so the scheduler and descriptor_state can access them without template instantiation. @@ -49,7 +49,7 @@ template struct reactor_op : reactor_op_base { // The op envelope — coroutine handle h, cont, executor ex, ec_out, - // bytes_out, cancelled, stop_cb (+ its canceller), impl_ptr — lives in + // bytes_out, cancelled, stop_cb (+ its canceller), object_ref_ — lives in // coro_op (via reactor_op_base) and is shared with io_uring/IOCP. // reactor_op adds only the reactor-specific routing state below. @@ -71,7 +71,7 @@ struct reactor_op : reactor_op_base errn = 0; bytes_transferred = 0; cancelled.store(false, std::memory_order_relaxed); - impl_ptr.reset(); + object_ref_.reset(); socket_impl_ = nullptr; acceptor_impl_ = nullptr; } diff --git a/include/boost/corosio/native/detail/reactor/reactor_op_base.hpp b/include/boost/corosio/native/detail/reactor/reactor_op_base.hpp index 0f91b9eaf..516a75108 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_op_base.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_op_base.hpp @@ -21,7 +21,7 @@ namespace boost::corosio::detail { Adds the reactor-specific result model (the recorded errno + bytes and the perform_io() readiness re-run) on top of coro_op, the shared op envelope (coroutine handle, executor, output pointers, stop_token wiring, - cancelled flag, impl_ptr keepalive) common to every backend. + cancelled flag, object_ref_ keepalive) common to every backend. Kept as a non-template layer so reactor_descriptor_state and the scheduler hot paths can touch op state without knowing the concrete @@ -37,7 +37,7 @@ struct reactor_op_base : coro_op /// Bytes transferred on success. std::size_t bytes_transferred = 0; - // cancelled, impl_ptr, and request_cancel() are inherited from coro_op. + // cancelled, object_ref_, and request_cancel() are inherited from coro_op. /// Record the result of an I/O attempt. void complete(int err, std::size_t bytes) noexcept @@ -49,10 +49,11 @@ struct reactor_op_base : coro_op /// Perform the I/O syscall (implemented by concrete op types). virtual void perform_io() noexcept = 0; - /// Destroy without invoking — drop the keepalive (impl_ptr from coro_op). + /// Destroy without invoking — drop the keepalive held in + /// coro_op's `object_ref_`. void destroy() override { - impl_ptr.reset(); + object_ref_.reset(); } }; diff --git a/include/boost/corosio/native/detail/reactor/reactor_scheduler.hpp b/include/boost/corosio/native/detail/reactor/reactor_scheduler.hpp index 6b77c5193..b43da9733 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_scheduler.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_scheduler.hpp @@ -11,6 +11,7 @@ #define BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_SCHEDULER_HPP #include +#include #include #include @@ -36,6 +37,37 @@ namespace boost::corosio::detail { class reactor_scheduler; class timer_service; +/** A deregistered descriptor whose owner must outlive the reactor pass + that may still hold it. + + An event the reactor has already read from the kernel can name a + descriptor that another thread closes before the reactor queues it. + The closing thread hands over a reference instead of dropping it, so + the owner is neither freed nor recycled until that pass ends. +*/ +struct reactor_retirable +{ + // All four fields are guarded by the scheduler's mutex. + + /// Next in the scheduler's retired list. + reactor_retirable* retired_next_ = nullptr; + + /// Owner reference held until the pass ends. + object_ref retired_ref_; + + /// Next in a settling thread's list of references to drop. + reactor_retirable* dropped_next_ = nullptr; + + /// Owner reference waiting to be dropped outside the mutex. + object_ref dropped_ref_; + + /// Move @p keep into a queued completion, if one is pending. + virtual void hand_off(object_ref& keep) noexcept = 0; + +protected: + ~reactor_retirable() = default; +}; + /** Per-thread state for a reactor scheduler. Each thread running a scheduler's event loop has one of these @@ -120,6 +152,18 @@ class reactor_scheduler /// Post a scheduler operation for deferred execution. void post(scheduler_op* h) const override; + /** Release an owner reference once no reactor pass can hold `d`. + + Called after a descriptor is deregistered. If a reactor pass is + running, `keep` is parked on `d` until that pass ends. Then, or + at once if no pass is running, it moves to `d`'s queued + invocation if `d` is queued, and is dropped otherwise. + + @param d The deregistered descriptor. + @param keep A reference to the descriptor's owner. + */ + void retire_descriptor(reactor_retirable& d, object_ref keep) const; + /// Post a continuation for deferred execution. void post(capy::continuation&) const override; @@ -263,6 +307,14 @@ class reactor_scheduler mutable std::atomic task_running_{false}; mutable bool task_interrupted_ = false; + // Descriptors retired during the running reactor pass (guarded by + // mutex_). + mutable reactor_retirable* retired_ = nullptr; + + /// Settle every descriptor retired before the current pass ended. + /// @pre `lock` holds `mutex_`; it is held again on return. + void settle_retired(lock_type& lock) const; + // Runtime-configurable reactor tuning parameters. // Defaults match the library's built-in values. unsigned max_events_per_poll_ = 128; @@ -670,10 +722,64 @@ reactor_scheduler::post_deferred_completions(ready_queue& ops) const wake_one_thread_and_unlock(lock); } +inline void +reactor_scheduler::retire_descriptor( + reactor_retirable& d, object_ref keep) const +{ + lock_type lock(mutex_); + if (!task_running_.load(std::memory_order_relaxed)) + { + // No pass can hold an event, but an earlier one may have + // queued d. + d.hand_off(keep); + return; + } + // Already parked for this pass: the held reference suffices. + if (d.retired_ref_) + return; + d.retired_ref_ = std::move(keep); + d.retired_next_ = retired_; + retired_ = &d; +} + +inline void +reactor_scheduler::settle_retired(lock_type& lock) const +{ + // Handed off under the lock, since a later pass may retire the same + // descriptor again as soon as it is released. A reference that is + // not handed off may be the owner's last, and its release recycles + // through the pool mutex, so it is dropped with the lock released. + reactor_retirable* dropped = nullptr; + while (auto* d = retired_) + { + retired_ = d->retired_next_; + d->retired_next_ = nullptr; + auto keep = std::move(d->retired_ref_); + d->hand_off(keep); + // A drop already pending keeps the owner alive, so this one + // cannot be the last and is released here. + if (!keep || d->dropped_ref_) + continue; + d->dropped_ref_ = std::move(keep); + d->dropped_next_ = dropped; + dropped = d; + } + while (auto* d = dropped) + { + dropped = d->dropped_next_; + d->dropped_next_ = nullptr; + auto keep = std::move(d->dropped_ref_); + lock.unlock(); + keep.reset(); + lock.lock(); + } +} + inline void reactor_scheduler::shutdown_drain() { lock_type lock(mutex_); + settle_retired(lock); while (auto e = completed_ops_.pop()) { @@ -854,6 +960,7 @@ reactor_scheduler::do_one(lock_type& lock, long timeout_us, context_type& ctx) task_running_.store(false, std::memory_order_relaxed); completed_ops_.push(&task_op_); + settle_retired(lock); if (timeout_us > 0) return 0; continue; diff --git a/include/boost/corosio/native/detail/reactor/reactor_service_state.hpp b/include/boost/corosio/native/detail/reactor/reactor_service_state.hpp index 636ca388d..e04965e01 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_service_state.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_service_state.hpp @@ -10,18 +10,15 @@ #ifndef BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_SERVICE_STATE_HPP #define BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_SERVICE_STATE_HPP +#include #include -#include -#include -#include - namespace boost::corosio::detail { /** Shared service state for reactor backends. - Holds the scheduler reference, service mutex, and per-impl - ownership tracking. Used by both socket and acceptor services. + Holds the scheduler reference and the impl recycling pool. Used + by both socket and acceptor services. @tparam Scheduler The backend's scheduler type. @tparam Impl The backend's socket or acceptor impl type. @@ -35,14 +32,8 @@ struct reactor_service_state /// Reference to the owning scheduler. Scheduler& sched_; - /// Protects `impl_list_` and `impl_ptrs_`. - std::mutex mutex_; - - /// All live impl objects for shutdown traversal. - intrusive_list impl_list_; - - /// Shared ownership of each impl, keyed by raw pointer. - std::unordered_map> impl_ptrs_; + /// Owns every impl: live for shutdown traversal, free for reuse. + object_pool pool_; }; } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/reactor/reactor_socket_finals.hpp b/include/boost/corosio/native/detail/reactor/reactor_socket_finals.hpp index aa405ff5c..5d37ce30a 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_socket_finals.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_socket_finals.hpp @@ -87,6 +87,15 @@ class reactor_stream_socket_impl ~reactor_stream_socket_impl() override = default; + /// Recycle into the owning service's pool at zero references. + void retire() noexcept override + { +#if !defined(NDEBUG) + this->poison(); +#endif + this->svc_.state_->pool_.recycle(static_cast(this)); + } + std::error_code set_option( int level, int optname, @@ -151,6 +160,15 @@ class reactor_dgram_socket_impl using impl_base_type = ImplBase; ~reactor_dgram_socket_impl() override = default; + + /// Recycle into the owning service's pool at zero references. + void retire() noexcept override + { +#if !defined(NDEBUG) + this->poison(); +#endif + this->svc_.state_->pool_.recycle(static_cast(this)); + } }; // ============================================================ @@ -197,6 +215,15 @@ class reactor_acceptor_impl ~reactor_acceptor_impl() override = default; + /// Recycle into the owning service's pool at zero references. + void retire() noexcept override + { +#if !defined(NDEBUG) + this->poison(); +#endif + this->svc_.state_->pool_.recycle(static_cast(this)); + } + std::coroutine_handle<> accept( std::coroutine_handle<>, capy::executor_ref, diff --git a/include/boost/corosio/native/detail/reactor/reactor_socket_service.hpp b/include/boost/corosio/native/detail/reactor/reactor_socket_service.hpp index 292254075..fa0790b21 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_socket_service.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_socket_service.hpp @@ -11,12 +11,12 @@ #define BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_SOCKET_SERVICE_HPP #include +#include #include #include #include #include -#include namespace boost::corosio::detail { @@ -36,6 +36,15 @@ template class reactor_socket_service : public ServiceBase { friend Derived; + + // The intermediate CRTP templates below (not Impl itself) are what + // actually reach into state_->pool_ from retire() -- see + // reactor_socket_finals.hpp. + template + friend class reactor_stream_socket_impl; + template + friend class reactor_dgram_socket_impl; + using state_type = reactor_service_state; protected: @@ -52,33 +61,23 @@ class reactor_socket_service : public ServiceBase void shutdown() override { - std::lock_guard lock(state_->mutex_); - - while (auto* impl = state_->impl_list_.pop_front()) - { - static_cast(this)->pre_shutdown(impl); - impl->close_socket(); - } - - // Don't clear impl_ptrs_ here. The scheduler shuts down after us - // and drains completed_ops_, calling destroy() on each queued op. - // Letting ~state_ release the ptrs (during service destruction, - // after scheduler shutdown) keeps every impl alive until all ops - // have been drained. + state_->pool_.shutdown( + [this](Impl* impl) + { + static_cast(this)->pre_shutdown(impl); + impl->close_socket(); + }); + + // Queued ops still hold references (their own object_ref copy); + // the scheduler shuts down after us and drains completed_ops_, + // each op's destroy() releasing its reference. Shutting-down + // mode makes that final release() delete instead of recycle, + // so every impl stays alive until its ops are gone, then frees. } io_object::implementation* construct() override { - auto impl = std::make_shared(static_cast(*this)); - auto* raw = impl.get(); - - { - std::lock_guard lock(state_->mutex_); - state_->impl_ptrs_.emplace(raw, std::move(impl)); - state_->impl_list_.push_back(raw); - } - - return raw; + return state_->pool_.acquire(static_cast(*this)); } void destroy(io_object::implementation* impl) override @@ -86,9 +85,7 @@ class reactor_socket_service : public ServiceBase auto* typed = static_cast(impl); static_cast(this)->pre_destroy(typed); typed->close_socket(); - std::lock_guard lock(state_->mutex_); - state_->impl_list_.remove(typed); - state_->impl_ptrs_.erase(typed); + release(typed); } void close(io_object::handle& h) override diff --git a/include/boost/corosio/native/detail/reactor/reactor_stream_socket.hpp b/include/boost/corosio/native/detail/reactor/reactor_stream_socket.hpp index 0a61a689f..0165550a4 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_stream_socket.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_stream_socket.hpp @@ -14,12 +14,14 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include @@ -284,6 +286,41 @@ class reactor_stream_socket return fd; } + /** Reset op slots and the cached peer for recycling. + + Each op's own `reset()` runs again at its next `do_*` call + before any field is read, so only `remote_endpoint_` needs an + active reset here; `stop_cb` is checked instead of reset — a + still-armed callback would mean a stop_token outlived the op's + completion, which `reset()` always disengages, so asserting it + is what actually catches a missed completion rather than + papering over it. + + @pre refs_ == 0, fd closed and deregistered, no op in flight. + */ + void reuse() noexcept + { + base_type::reuse(); + remote_endpoint_ = Endpoint{}; + BOOST_COROSIO_ASSERT(!conn_.stop_cb); + BOOST_COROSIO_ASSERT(!rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wr_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_wr_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_er_.stop_cb); + } + +#if !defined(NDEBUG) + /// Extend base_type::poison() to the cached peer endpoint. + void poison() noexcept + { + base_type::poison(); + std::memset( + static_cast(&remote_endpoint_), 0xDB, + sizeof(remote_endpoint_)); + } +#endif + private: // CRTP callbacks for reactor_io_core cancel/close @@ -391,7 +428,7 @@ reactor_stream_socket< op.fd = this->fd_; op.target_endpoint = ep; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(err, 0); this->svc_.post(&op); return std::noop_coroutine(); @@ -405,7 +442,7 @@ reactor_stream_socket< op.fd = this->fd_; op.target_endpoint = ep; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->register_op( op, this->desc_state_.connect_op, this->desc_state_.write_ready, true); @@ -453,7 +490,7 @@ reactor_stream_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(EBADF, 0); this->svc_.post(&op); return std::noop_coroutine(); @@ -470,7 +507,7 @@ reactor_stream_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(0, 0); this->svc_.post(&op); return std::noop_coroutine(); @@ -524,7 +561,7 @@ reactor_stream_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(err, bytes); this->svc_.post(&op); return std::noop_coroutine(); @@ -537,7 +574,7 @@ reactor_stream_socket< op.bytes_out = bytes_out; op.fd = this->fd_; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->register_op( op, this->desc_state_.read_op, this->desc_state_.read_ready); @@ -585,7 +622,7 @@ reactor_stream_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(EBADF, 0); this->svc_.post(&op); return std::noop_coroutine(); @@ -602,7 +639,7 @@ reactor_stream_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(0, 0); this->svc_.post(&op); return std::noop_coroutine(); @@ -645,7 +682,7 @@ reactor_stream_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(err, bytes); this->svc_.post(&op); return std::noop_coroutine(); @@ -658,7 +695,7 @@ reactor_stream_socket< op.bytes_out = bytes_out; op.fd = this->fd_; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->register_op( op, this->desc_state_.write_op, this->desc_state_.write_ready, true); @@ -739,7 +776,7 @@ reactor_stream_socket< op.ec_out = ec; op.fd = this->fd_; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(perr, 0); this->svc_.post(&op); return std::noop_coroutine(); @@ -752,7 +789,7 @@ reactor_stream_socket< op.ec_out = ec; op.fd = this->fd_; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); // Force register_op's ready path so the wait op re-probes under // the descriptor mutex before parking. An edge consumed between diff --git a/include/boost/corosio/native/detail/select/select_scheduler.hpp b/include/boost/corosio/native/detail/select/select_scheduler.hpp index 89b76b810..3b93abb17 100644 --- a/include/boost/corosio/native/detail/select/select_scheduler.hpp +++ b/include/boost/corosio/native/detail/select/select_scheduler.hpp @@ -224,16 +224,20 @@ select_scheduler::register_descriptor( if (fd < 0 || fd >= FD_SETSIZE) return make_err(EMFILE); - desc->registered_events = reactor_event_read | reactor_event_write; - desc->unpollable = false; // the state is reused across adoptions - desc->fd = fd; - desc->scheduler_ = this; - desc->mutex.set_enabled(reactor_io_locking_); - desc->ready_events_.store(0, std::memory_order_relaxed); - + // A completion queued by an earlier registration of the same state + // may still be running, so the fields it reads change under its + // mutex, and the mutex mode is only written when it differs. + if (desc->mutex.enabled() != reactor_io_locking_) + desc->mutex.set_enabled(reactor_io_locking_); { conditionally_enabled_mutex::scoped_lock lock(desc->mutex); - desc->impl_ref_.reset(); + desc->registered_events = reactor_event_read | reactor_event_write; + desc->unpollable = false; // the state is reused across adoptions + desc->fd = fd; + desc->scheduler_ = this; + desc->ready_events_.store(0, std::memory_order_release); + // object_ref_ is not touched: a queued invocation from an + // earlier registration still owns the reference in it. desc->read_ready = false; desc->write_ready = false; } diff --git a/include/boost/corosio/native/detail/speculative_state.hpp b/include/boost/corosio/native/detail/speculative_state.hpp index c29b8253b..9dbdd48bb 100644 --- a/include/boost/corosio/native/detail/speculative_state.hpp +++ b/include/boost/corosio/native/detail/speculative_state.hpp @@ -109,6 +109,20 @@ class speculative_state { try_write_.store(true, std::memory_order_relaxed); } + + /** Restore the default, fully-speculative state. + + The learned hint is scoped to one logical connection; a recycled + socket starts a new one and should not inherit the prior + connection's streak or perma-off latch. + */ + void reset() noexcept + { + try_read_.store(true, std::memory_order_relaxed); + try_write_.store(true, std::memory_order_relaxed); + read_eagain_streak_.store(0, std::memory_order_relaxed); + perma_off_read_.store(false, std::memory_order_relaxed); + } }; } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/uring/uring_acceptor_ops.hpp b/include/boost/corosio/native/detail/uring/uring_acceptor_ops.hpp index 3473c2096..253c758de 100644 --- a/include/boost/corosio/native/detail/uring/uring_acceptor_ops.hpp +++ b/include/boost/corosio/native/detail/uring/uring_acceptor_ops.hpp @@ -18,9 +18,11 @@ #include #include +#include #include #include #include +#include #include #include @@ -29,19 +31,22 @@ namespace boost::corosio::detail { -/** Multishot accept op — one submitted per acceptor lifetime. +/** Multishot accept op: one kernel arming of an acceptor's listener. - The kernel produces a CQE for each accepted connection. Each CQE - carries the new fd in `res` (>= 0) or a negative errno on failure. - The `IORING_CQE_F_MORE` flag is set on every CQE except the last, - indicating whether the multishot armament is still active. + The kernel produces a CQE for each accepted connection, carrying + the new fd in `res` (>= 0) or a negative errno. Every CQE but the + last sets `IORING_CQE_F_MORE`; the last ends the arming. - `do_cqe` does NOT push self into `local` — the owning acceptor's - `on_cqe` handler decides whether to dispatch immediately (waiter - present) or park the fd (no waiter). The multishot op persists - across CQEs; only `acceptor_impl` owns its lifetime. + While armed, the op is owned by the kernel: it holds a reference on + its acceptor (`object_ref_`) and the acceptor only remembers it by + address. Deliveries are routed inline by `on_cqe` under the ring + mutex. The terminal CQE queues the op for dispatch, and its handler + (`on_end`, off every lock) either re-arms it or returns it to the + acceptor's free list and drops the reference. */ -struct uring_multi_accept_op : uring_op +struct uring_multi_accept_op + : uring_op + , intrusive_list::node { /// Filled by the kernel for each accept. Address of this struct /// is registered with the SQE; kernel writes peer address here. @@ -49,19 +54,34 @@ struct uring_multi_accept_op : uring_op socklen_t peer_len = sizeof(peer_storage); int listen_fd = -1; - /// Owning acceptor; raw because the op IS owned by the acceptor. + /// The acceptor that armed this op. void* acceptor_impl = nullptr; - /** Callback into the acceptor for each accept CQE. + /** Route one accept CQE; runs under the ring mutex. - @param acceptor The owning acceptor_impl pointer. + @param acceptor The acceptor that armed @p op. + @param op The arming the CQE belongs to. @param new_fd Accepted fd on success, -1 on error. @param err errno value on failure, 0 on success. - @param more True unless this is the terminating CQE - (e.g. kernel dropped multishot on -ENOMEM). + @param more True unless this CQE ends the arming. */ - void (*on_cqe)(void* acceptor, int new_fd, int err, bool more) noexcept = - nullptr; + void (*on_cqe)( + void* acceptor, + uring_multi_accept_op* op, + int new_fd, + int err, + bool more) noexcept = nullptr; + + /** Handle the end of an arming; runs from dispatch, off every lock. + + @param acceptor The acceptor that armed @p op. + @param op The ended arming. + @param live False when the scheduler is shutting down. + */ + void (*on_end)( + void* acceptor, + uring_multi_accept_op* op, + bool live) noexcept = nullptr; uring_multi_accept_op() noexcept : uring_op(&do_handler, &do_cqe, &do_prep) { @@ -76,48 +96,31 @@ struct uring_multi_accept_op : uring_op SOCK_NONBLOCK | SOCK_CLOEXEC); } - /** Dispose of a connection the kernel accepted for a retired - arming. - - The acceptor that armed this op has moved to another - descriptor, so no waiter will ever take delivery. The fd is - already installed in the process table — dropping the CQE - without closing it leaks it for the life of the process. - */ static void - do_retired_cqe(uring_op* /*base*/, int res, unsigned /*flags*/) noexcept - { - if (res >= 0) // LCOV_EXCL_LINE adopt-over-armed race leak guard - ::close(res); // LCOV_EXCL_LINE adopt-over-armed race leak guard - } - - static void do_cqe( - uring_op* base, - int res, - unsigned flags, - ready_queue& /*local*/) noexcept + do_cqe(uring_op* base, int res, unsigned flags, ready_queue& local) noexcept { auto* self = static_cast(base); bool more = (flags & IORING_CQE_F_MORE) != 0; int err = (res < 0) ? -res : 0; int new_fd = (res >= 0) ? res : -1; - if (self->on_cqe) - self->on_cqe(self->acceptor_impl, new_fd, err, more); - // Intentionally NOT pushed into local: the acceptor decides - // whether to surface the fd via a waiter or park it. + self->on_cqe(self->acceptor_impl, self, new_fd, err, more); + if (!more) + { + // Balance the work_finished() do_one runs after dispatching. + self->sched_->work_started(); + local.push(self); + } } - // LCOV_EXCL_START: never invoked; the multishot op is owned by - // the acceptor and never queued for handler dispatch. Provided so - // the vtable is complete. static void do_handler( - void* /*owner*/, - scheduler_op* /*base*/, + void* owner, + scheduler_op* base, std::uint32_t /*bytes*/, std::uint32_t /*error*/) noexcept { + auto* self = static_cast(base); + self->on_end(self->acceptor_impl, self, owner != nullptr); } - // LCOV_EXCL_STOP }; /** Synthesized accept op — manufactured by the acceptor for parked fds. @@ -154,8 +157,23 @@ struct uring_accept_op : uring_op sockaddr_storage const& peer, socklen_t peer_len) noexcept = nullptr; + /** Disposal hook, run exactly once when the op is consumed. + + Heap-allocated ops delete themselves (the default); the + multishot acceptor's pooled nodes override this to return to + their acceptor's free list instead. Factoring disposal out of + `do_handler` is what lets one completion routine serve both + ownership models. + */ + void (*dispose)(uring_accept_op*) noexcept = &dispose_delete; + uring_accept_op() noexcept : uring_op(&do_handler, &do_cqe) {} + static void dispose_delete(uring_accept_op* op) noexcept + { + delete op; + } + // LCOV_EXCL_START: never receives a CQE; present for vtable // completeness. static void do_cqe(uring_op*, int, unsigned, ready_queue&) noexcept {} @@ -172,7 +190,10 @@ struct uring_accept_op : uring_op if (owner == nullptr) { - delete self; + // A delivered connection that nobody will adopt. + if (self->accepted_fd >= 0) + ::close(self->accepted_fd); + self->dispose(self); return; } @@ -186,7 +207,7 @@ struct uring_accept_op : uring_op : make_err(self->err); self->cont.h = self->h; auto next = dispatch_coro(self->ex, self->cont); - delete self; + self->dispose(self); next.resume(); return; } @@ -207,7 +228,7 @@ struct uring_accept_op : uring_op self->cont.h = self->h; auto next = dispatch_coro(self->ex, self->cont); - delete self; + self->dispose(self); next.resume(); } }; diff --git a/include/boost/corosio/native/detail/uring/uring_descriptor.hpp b/include/boost/corosio/native/detail/uring/uring_descriptor.hpp index 0b5c2eff2..b6c7addbb 100644 --- a/include/boost/corosio/native/detail/uring/uring_descriptor.hpp +++ b/include/boost/corosio/native/detail/uring/uring_descriptor.hpp @@ -218,12 +218,16 @@ fd_is_pollable(int fd) noexcept return pollable; } +class uring_descriptor_service; + class BOOST_COROSIO_DECL uring_descriptor final : public posix_stream_descriptor::implementation - , public std::enable_shared_from_this , public intrusive_list::node { - uring_scheduler* sched_ = nullptr; + friend class uring_descriptor_service; + + uring_descriptor_service* svc_ = nullptr; + uring_scheduler* sched_ = nullptr; int fd_ = -1; bool pollable_ = true; @@ -239,7 +243,10 @@ class BOOST_COROSIO_DECL uring_descriptor final uring_wait_op wait_er_; public: - explicit uring_descriptor(uring_scheduler& sched) noexcept : sched_(&sched) + uring_descriptor( + uring_descriptor_service& svc, uring_scheduler& sched) noexcept + : svc_(&svc) + , sched_(&sched) { } @@ -248,6 +255,28 @@ class BOOST_COROSIO_DECL uring_descriptor final close_descriptor(); } + /// Recycle into the owning service's pool. Defined out-of-line + /// after uring_descriptor_service for its complete type. + void retire() noexcept override; + + /** Assert the closed state a recycled descriptor starts from. + + `close_descriptor()` already drove fd_ to its closed value + before the refcount reached zero; each op's own `prepare()` + overwrites its state before the next use. + + @pre refs_ == 0, fd closed, no op in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(fd_ == -1); + BOOST_COROSIO_ASSERT(!rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wr_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_wr_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_er_.stop_cb); + } + // -- io_stream::implementation -- std::coroutine_handle<> read_some( @@ -260,7 +289,7 @@ class BOOST_COROSIO_DECL uring_descriptor final { rd_.prepare( h, ex, ec, bytes, fd_, /*file_offset=*/-1, sched_, - shared_from_this(), buffers, token); + detail::object_ref(this), buffers, token); arm_slot(rd_); sched_->work_started(); @@ -293,7 +322,7 @@ class BOOST_COROSIO_DECL uring_descriptor final { wr_.prepare( h, ex, ec, bytes, fd_, /*file_offset=*/-1, sched_, - shared_from_this(), buffers, token); + detail::object_ref(this), buffers, token); arm_slot(wr_); sched_->work_started(); @@ -345,7 +374,7 @@ class BOOST_COROSIO_DECL uring_descriptor final } op->prepare( - h, ex, ec, fd_, sched_, shared_from_this(), poll_flags, token); + h, ex, ec, fd_, sched_, detail::object_ref(this), poll_flags, token); sched_->work_started(); if (fd_ < 0) diff --git a/include/boost/corosio/native/detail/uring/uring_descriptor_service.hpp b/include/boost/corosio/native/detail/uring/uring_descriptor_service.hpp index 7f0739390..6b6a92f70 100644 --- a/include/boost/corosio/native/detail/uring/uring_descriptor_service.hpp +++ b/include/boost/corosio/native/detail/uring/uring_descriptor_service.hpp @@ -71,6 +71,12 @@ class BOOST_COROSIO_DECL uring_descriptor_service final } }; +inline void +uring_descriptor::retire() noexcept +{ + svc_->pool_.recycle(this); +} + } // namespace boost::corosio::detail #endif // BOOST_COROSIO_HAS_URING diff --git a/include/boost/corosio/native/detail/uring/uring_dgram_ops.hpp b/include/boost/corosio/native/detail/uring/uring_dgram_ops.hpp index b4338e068..f4cdeaebd 100644 --- a/include/boost/corosio/native/detail/uring/uring_dgram_ops.hpp +++ b/include/boost/corosio/native/detail/uring/uring_dgram_ops.hpp @@ -69,7 +69,7 @@ struct uring_dgram_send_op : uring_op std::size_t* bytes, int file_descriptor, uring_scheduler* scheduler, - std::shared_ptr impl, + detail::object_ref impl, detail::speculative_state* spec, buffer_param buffers, socklen_t dest_addr_len, @@ -77,17 +77,17 @@ struct uring_dgram_send_op : uring_op int flags, std::stop_token const& token) noexcept { - h = handle; - ex = executor; - ec_out = ec; - bytes_out = bytes; - fd = file_descriptor; - sched_ = scheduler; - impl_ptr = std::move(impl); - spec_state = spec; - res = 0; - cqe_flags = 0; - msg_flags = flags; + h = handle; + ex = executor; + ec_out = ec; + bytes_out = bytes; + fd = file_descriptor; + sched_ = scheduler; + object_ref_ = std::move(impl); + spec_state = spec; + res = 0; + cqe_flags = 0; + msg_flags = flags; iovec_count = copy_to_iovec(buffers, iovecs); @@ -207,7 +207,7 @@ struct uring_dgram_recv_op : uring_op std::size_t* bytes, int file_descriptor, uring_scheduler* scheduler, - std::shared_ptr impl, + detail::object_ref impl, detail::speculative_state* spec, buffer_param buffers, void* source_ctx, @@ -215,17 +215,17 @@ struct uring_dgram_recv_op : uring_op int flags, std::stop_token const& token) noexcept { - h = handle; - ex = executor; - ec_out = ec; - bytes_out = bytes; - fd = file_descriptor; - sched_ = scheduler; - impl_ptr = std::move(impl); - spec_state = spec; - res = 0; - cqe_flags = 0; - msg_flags = flags; + h = handle; + ex = executor; + ec_out = ec; + bytes_out = bytes; + fd = file_descriptor; + sched_ = scheduler; + object_ref_ = std::move(impl); + spec_state = spec; + res = 0; + cqe_flags = 0; + msg_flags = flags; iovec_count = copy_to_iovec(buffers, iovecs); diff --git a/include/boost/corosio/native/detail/uring/uring_file_ops.hpp b/include/boost/corosio/native/detail/uring/uring_file_ops.hpp index 467d50966..159d16192 100644 --- a/include/boost/corosio/native/detail/uring/uring_file_ops.hpp +++ b/include/boost/corosio/native/detail/uring/uring_file_ops.hpp @@ -71,7 +71,7 @@ struct uring_file_read_op_base : uring_op int file_descriptor, std::int64_t file_offset, uring_scheduler* scheduler, - std::shared_ptr impl, + detail::object_ref impl, buffer_param buffers, std::stop_token const& token) noexcept { @@ -82,7 +82,7 @@ struct uring_file_read_op_base : uring_op fd = file_descriptor; offset = file_offset; sched_ = scheduler; - impl_ptr = std::move(impl); + object_ref_ = std::move(impl); res = 0; cqe_flags = 0; iovec_count = copy_to_iovec(buffers, iovecs); @@ -219,7 +219,7 @@ struct uring_file_write_op_base : uring_op int file_descriptor, std::int64_t file_offset, uring_scheduler* scheduler, - std::shared_ptr impl, + detail::object_ref impl, buffer_param buffers, std::stop_token const& token) noexcept { @@ -230,7 +230,7 @@ struct uring_file_write_op_base : uring_op fd = file_descriptor; offset = file_offset; sched_ = scheduler; - impl_ptr = std::move(impl); + object_ref_ = std::move(impl); res = 0; cqe_flags = 0; iovec_count = copy_to_iovec(buffers, iovecs); diff --git a/include/boost/corosio/native/detail/uring/uring_file_service_base.hpp b/include/boost/corosio/native/detail/uring/uring_file_service_base.hpp index 06ff20706..829824958 100644 --- a/include/boost/corosio/native/detail/uring/uring_file_service_base.hpp +++ b/include/boost/corosio/native/detail/uring/uring_file_service_base.hpp @@ -14,35 +14,35 @@ #if BOOST_COROSIO_HAS_URING -#include +#include +#include #include #include -#include -#include -#include +#include /* Shared lifecycle plumbing for io_uring file services. uring_stream_file_service and uring_random_access_file_service were byte-for-byte identical apart from the impl type and open_file's parameter - type: both make_shared the file impl from the scheduler, track it in an - intrusive list + raw->shared_ptr map, and close every file on shutdown. - This base factors that out; the concrete services add only open_file. + type: both pop-or-new the file impl from a per-service recycling pool + and close every file on shutdown. This base factors that out; the + concrete services add only open_file. This is a separate base from uring_socket_service_base because file - services differ from socket services in three ways that match the reactor - socket service instead: they track via an intrusive list + map (sockets: - map only), they CLOSE files on shutdown (sockets: cancel only), and the - impl ctor takes just the scheduler (sockets: service + scheduler). See - tasks/proactor-dedup-decisions.md (#14). - - Requirements on File: derive from enable_shared_from_this and - intrusive_list::node, a `File(uring_scheduler&)` constructor, and - a `void close_file() noexcept` method (cancel in-flight ops + close fd). - uring_descriptor_service reuses it too, as asio's file service reuses - its descriptor service. + services differ from socket services in two ways that match the reactor + socket service instead: they CLOSE files on shutdown (sockets: cancel + only), and the impl ctor takes just the scheduler (sockets: service + + scheduler). See tasks/proactor-dedup-decisions.md (#14). + + Requirements on File: derive from intrusive_list::node, a + `File(Derived&, uring_scheduler&)` constructor, a + `void close_file() noexcept` method (cancel in-flight ops + close fd), + a `void reuse() noexcept` method, and a `retire()` override that + recycles through the owning service's private `pool_` member + (reachable because `File` is a friend). uring_descriptor_service + reuses it too, as asio's file service reuses its descriptor service. @tparam Derived The concrete service (CRTP; unused today but kept for symmetry / future hooks). @@ -57,6 +57,7 @@ template class uring_file_service_base : public ServiceBase { friend Derived; + friend File; // Private CRTP ctor: only `Derived` (the concrete service, a friend) // constructs the base — prevents inheriting with the wrong Derived @@ -71,14 +72,7 @@ class uring_file_service_base : public ServiceBase io_object::implementation* construct() override { - auto ptr = std::make_shared(*sched_); - auto* impl = ptr.get(); - { - std::lock_guard lock(mutex_); - file_list_.push_back(impl); - file_ptrs_[impl] = std::move(ptr); - } - return impl; + return pool_.acquire(static_cast(*this), *sched_); } void destroy(io_object::implementation* p) override @@ -86,9 +80,7 @@ class uring_file_service_base : public ServiceBase // close_file() already does cancel_and_flush(fd_) before ::close. auto& impl = static_cast(*p); impl.close_file(); - std::lock_guard lock(mutex_); - file_list_.remove(&impl); - file_ptrs_.erase(&impl); + release(&impl); } void close(io_object::handle& h) override @@ -99,14 +91,23 @@ class uring_file_service_base : public ServiceBase void shutdown() override { - // file_ptrs_ is deliberately not cleared: the scheduler shuts - // down after this service and drains its completed ops, so - // every impl must outlive that drain. The map frees them when - // the service is destroyed. - std::lock_guard lock(mutex_); - for (auto* impl = file_list_.pop_front(); impl != nullptr; - impl = file_list_.pop_front()) - impl->close_file(); + // See uring_socket_service_base::shutdown(): snapshot under an + // acquired reference, close without the pool lock held; + // shutdown() sets shutting-down and takes the snapshot in one + // critical section, so a close that drops the last ref deletes + // rather than recycles. + std::vector live; + pool_.shutdown( + [&](File* f) + { + acquire(f); + live.push_back(f); + }); + for (auto* f : live) + { + f->close_file(); + release(f); + } } /// Return the scheduler used by files created by this service. @@ -117,9 +118,7 @@ class uring_file_service_base : public ServiceBase protected: uring_scheduler* sched_; - std::mutex mutex_; - intrusive_list file_list_; - std::unordered_map> file_ptrs_; + object_pool pool_; private: uring_file_service_base(uring_file_service_base const&) = delete; diff --git a/include/boost/corosio/native/detail/uring/uring_multishot_acceptor.hpp b/include/boost/corosio/native/detail/uring/uring_multishot_acceptor.hpp index 4ded4e8e9..46e4c888e 100644 --- a/include/boost/corosio/native/detail/uring/uring_multishot_acceptor.hpp +++ b/include/boost/corosio/native/detail/uring/uring_multishot_acceptor.hpp @@ -11,6 +11,7 @@ #define BOOST_COROSIO_NATIVE_DETAIL_URING_URING_MULTISHOT_ACCEPTOR_HPP #include +#include #include #if BOOST_COROSIO_HAS_URING @@ -30,9 +31,7 @@ #include #include #include -#include #include -#include #include #include @@ -62,10 +61,15 @@ fd_is_listening(int fd) noexcept return accepting != 0; } -template +template< + class Derived, + class ImplBase, + class Endpoint, + class PeerService, + class AcceptorService> class uring_multishot_acceptor_base : public ImplBase - , public std::enable_shared_from_this + , public intrusive_list::node { protected: struct ready_fd_node : intrusive_list::node @@ -75,22 +79,34 @@ class uring_multishot_acceptor_base socklen_t peer_len = 0; }; - struct waiter_node; - - struct waiter_canceller - { - waiter_node* w; - void operator()() const noexcept; - }; - - struct waiter_node : intrusive_list::node + /** Pooled accept node: parked waiter and posted completion in one. + + A parked accept used to be a `waiter_node` whose fields were + copied into a freshly allocated `uring_accept_op` at every + completion boundary. The two had strictly sequential lifetimes + (a waiter *becomes* a completion), so this node is both at + once — the intrusive hook parks it in `waiters_`/`free_nodes_` + and the inherited op posts it to the scheduler. Nodes recycle + through the owning acceptor's `free_nodes_` via `dispose`, so + steady-state accepts allocate nothing. + + The inherited `cancelled` flag is the claim token: delivery, + arming-failure, and cancellation each claim the node with one + `exchange`, and the loser leaves completion to the winner. + Because the flag also decides how `do_handler` reports the + completion, every claiming path stores the final status after + `stop_cb.reset()` (which synchronizes with any in-flight + canceller) and before posting. + + While in flight — parked or posted — the node holds an + `object_ref` on its acceptor (the inherited `object_ref_` + slot), so the impl cannot retire out from under `dispose`. + */ + struct accept_node + : uring_accept_op + , intrusive_list::node { - std::coroutine_handle<> h; - capy::executor_ref ex; - std::error_code* ec_out = nullptr; - io_object::implementation** impl_out = nullptr; - Derived* owner = nullptr; - std::atomic cancelled{false}; + Derived* owner = nullptr; /// True once linked into `waiters_` (guarded by `mutex_`). /// The stop callback is armed before the node is queued, so /// cancel_waiter must not unlink a node it never queued. @@ -98,22 +114,58 @@ class uring_multishot_acceptor_base /// A readiness wait rather than an accept: completion /// observes a pending connection without consuming it. bool peek = false; - std::optional> stop_cb; + + accept_node() noexcept + { + this->dispose = &recycle_thunk; + } + + /// Claim the node for cancellation (the stop token fired); + /// losing the exchange means a delivery already owns it. + void on_cancel() noexcept override + { + if (this->cancelled.exchange(true, std::memory_order_acq_rel)) + return; + owner->cancel_waiter(this); + } + + static void recycle_thunk(uring_accept_op* op) noexcept + { + auto* n = static_cast(op); + n->owner->release_node(n); + } }; int fd_ = -1; uring_scheduler* sched_; PeerService* peer_service_; + /// Owning acceptor service; exposes the recycling pool to + /// retire(). Not used for anything else — the service already + /// drives shutdown/destroy from its own side. + AcceptorService* acceptor_svc_; Endpoint local_endpoint_{}; mutable std::mutex mutex_; intrusive_list ready_fds_; - intrusive_list waiters_; + intrusive_list waiters_; + /// Recycled accept nodes and ready-fd nodes (guarded by `mutex_`). + /// Both survive impl recycling — the storage belongs to this impl + /// and only the destructor frees it — so steady-state accepts + /// allocate nothing. + intrusive_list free_nodes_; + intrusive_list free_ready_; /// Single parked readiness wait (guarded by `mutex_`). Multishot /// accepting drains the kernel queue instantly, so a listener's /// readiness lives in `ready_fds_`, not in `poll()`; the wait is /// completed by the next delivery instead of a kernel poll. - waiter_node* read_wait_ = nullptr; - std::unique_ptr multi_op_; + accept_node* read_wait_ = nullptr; + /// Ended armings, kept for the next one (guarded by `mutex_`). + /// Survives impl recycling like `free_nodes_`; entries hold no + /// reference. + intrusive_list free_ops_; + /// The arming whose deliveries belong to this acceptor (guarded by + /// `mutex_`). Identity only: the kernel owns the op and its + /// reference. Never set while `closing_` is. + uring_multi_accept_op* armed_ = nullptr; bool closing_ = false; /// Non-zero once an arming failed to reach the kernel (guarded by /// `mutex_`). Nothing will ever deliver a connection through an @@ -121,14 +173,6 @@ class uring_multishot_acceptor_base /// parking on a delivery that cannot come. Cleared by the next /// arming that does reach the kernel. int arm_err_ = 0; - /// Bumped whenever an arming is retired. A re-arm posted for an - /// earlier generation must not resubmit: `multi_op_` now names a - /// different op, and resubmitting a live one would alias a single - /// `user_data` across two kernel armings. The re-arm's check is - /// not atomic with its submit — a retirement landing between them - /// (concurrent `assign()` on another thread) can still double-arm; - /// closing that needs a generation-aware submit. - std::atomic arm_generation_{0}; private: // CRTP ctor private + Derived friended so the base cannot be @@ -136,49 +180,154 @@ class uring_multishot_acceptor_base // (clang-tidy bugprone-crtp-constructor-accessibility). friend Derived; uring_multishot_acceptor_base( - uring_scheduler& sched, PeerService& peer_svc) noexcept + AcceptorService& acceptor_svc, + uring_scheduler& sched, + PeerService& peer_svc) noexcept : sched_(&sched) , peer_service_(&peer_svc) + , acceptor_svc_(&acceptor_svc) { } -public: - ~uring_multishot_acceptor_base() override +protected: + /** Pop a recycled accept node, or allocate on the cold path. + + Resets every per-use field and arms the `object_ref` keepalive + on this acceptor; the caller fills the request fields and + either parks or posts the node. OOM on the cold path + terminates, matching every other noexcept initiation path. + */ + accept_node* acquire_node() noexcept { + accept_node* n; { std::lock_guard lk(mutex_); - closing_ = true; + n = free_nodes_.pop_front(); } - if (fd_ >= 0) + if (!n) { - sched_->submit_cancel_by_fd(fd_); - ::close(fd_); - fd_ = -1; + // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept initiation path: OOM => std::terminate is the intended behavior + n = new accept_node(); + } + n->owner = static_cast(this); + n->queued = false; + n->peek = false; + n->err = 0; + n->accepted_fd = -1; + n->peer_len = 0; + n->ec_out = nullptr; + n->impl_out = nullptr; + n->peer_endpoint_out = nullptr; + n->peer_service = nullptr; + n->adopt_fn = nullptr; + n->cancelled.store(false, std::memory_order_relaxed); + n->object_ref_ = detail::object_ref(this); + return n; + } + + /** Return a consumed node to the free list; `dispose` target. + + The keepalive is moved out first and dropped after the lock, + so a final release (which may reenter this impl's retire()) + never runs under `mutex_`. + */ + void release_node(accept_node* n) noexcept + { + auto keep = std::move(n->object_ref_); + { + std::lock_guard lk(mutex_); + free_nodes_.push_front(n); } + } + + /// Pop a recycled ready-fd node, or allocate on the cold path. + /// @pre `mutex_` is held. + ready_fd_node* acquire_ready_locked() noexcept + { + if (auto* r = free_ready_.pop_front()) + { + r->fd = -1; + r->peer_len = 0; + return r; + } + // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — CQE handler: noexcept, OOM => std::terminate is the intended behavior + return new ready_fd_node{}; + } - // Drain parked accepted-connection fds unconditionally. These are - // distinct from the listener fd and can be present even when the - // service close() path already closed and cleared fd_ — that path - // does not touch ready_fds_, so the drain must run here. +public: + /** Close and free every fd parked in `ready_fds_`. + + A multishot CQE can deliver a connection before any `accept()` + is outstanding to claim it; that connection's fd (and tracking + node) sits in `ready_fds_` until a later `accept()` consumes it + or the acceptor goes away. Idempotent: safe to call from both + `retire()` (recycling) and the destructor (pool-force-sweep + backstop) — the second call always sees an empty list. + */ + void drain_ready_fds() noexcept + { intrusive_list drained; { std::lock_guard lk(mutex_); while (auto* r = ready_fds_.pop_front()) drained.push_back(r); } - while (auto* r = drained.pop_front()) + discard_ready(drained); + } + + /// Close each parked connection in @p stale and return its node + /// to `free_ready_`. @pre `mutex_` is not held. + void discard_ready(intrusive_list& stale) noexcept + { + if (stale.empty()) + return; + stale.for_each([](ready_fd_node* r) { ::close(r->fd); }); + std::lock_guard lk(mutex_); + while (auto* r = stale.pop_front()) + free_ready_.push_front(r); + } + + /** Recycle into the owning service's pool at zero references. + + No arming can be live here: an armed op holds a reference. + */ + void retire() noexcept override + { + drain_ready_fds(); + acceptor_svc_->pool_.recycle(static_cast(this)); + } + + ~uring_multishot_acceptor_base() override + { { - ::close(r->fd); - delete r; + std::lock_guard lk(mutex_); + closing_ = true; } - - // Break the multi_op_ → impl_ptr (shared_ptr) cycle and - // drain pending CQEs so unique_ptr can free safely. - if (multi_op_) + if (fd_ >= 0) { - multi_op_->impl_ptr.reset(); - sched_->drain_cqes_for(multi_op_.get()); + sched_->submit_cancel_by_fd(fd_); + ::close(fd_); + fd_ = -1; } + + // Backstop: retire() already drains this on the normal + // recycle path. This call only does real work when this impl + // is reached by the pool's unconditional force-sweep at + // shutdown without ever going through retire() first. + drain_ready_fds(); + + // The recycled-node storage belongs to this impl and is only + // freed here; in-flight nodes hold an object_ref on this impl, + // so none can still be parked or posted once the count reached + // zero (or the pool's force-sweep ran after scheduler drain). + // An arming the kernel never returned is not in `free_ops_` + // and is deliberately leaked: the ring may still write to it. + while (auto* n = free_nodes_.pop_front()) + delete n; + while (auto* r = free_ready_.pop_front()) + delete r; + while (auto* op = free_ops_.pop_front()) + delete op; } Endpoint local_endpoint() const noexcept override @@ -203,18 +352,12 @@ class uring_multishot_acceptor_base native_handle_type release_socket() noexcept override { - // Mirror the service close() path: cancel the multishot SQE and - // break the multi_op_ -> impl_ptr (shared_ptr) cycle that - // start_multishot established. Without this, the cycle keeps the - // acceptor and its multi_op_ alive after the caller takes the fd, - // which LeakSanitizer reports on process exit. Caller still owns - // the returned fd, so we do NOT ::close it here. + // Mirror the service close() path, except that the caller now + // owns the fd. if (fd_ >= 0) { - sched_->cancel_and_flush(fd_); drain_waiters_only(); - if (multi_op_) - multi_op_->impl_ptr.reset(); + sched_->cancel_and_flush(fd_); } int fd = fd_; fd_ = -1; @@ -229,16 +372,45 @@ class uring_multishot_acceptor_base sched_->submit_cancel_by_fd(fd_); } - /// Drain queued waiters with operation_aborted but do NOT submit - /// any kernel cancel for the fd. Used by service close() paths - /// that have already submitted (or are about to submit) the - /// cancel-by-fd themselves via `cancel_and_flush`. + /** Reset state for recycling. + + `close()`/`release_socket()` already drove fd_, local_endpoint_, + and the waiter/ready-fd queues to empty before the refcount + reached zero, so those are asserted rather than re-cleared. + `closing_` and `arm_err_` are the two fields that are NOT reset + by that close path (closing_ is only ever cleared by a re-listen + on a still-live acceptor, not by close) and must be explicitly + reset here or the next session would see this impl as still + shutting down. + + @pre refs_ == 0, fd closed, no op or waiter in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(fd_ == -1); + BOOST_COROSIO_ASSERT(local_endpoint_ == Endpoint{}); + BOOST_COROSIO_ASSERT(ready_fds_.empty()); + BOOST_COROSIO_ASSERT(waiters_.empty()); + BOOST_COROSIO_ASSERT(read_wait_ == nullptr); + BOOST_COROSIO_ASSERT(armed_ == nullptr); + closing_ = false; + arm_err_ = 0; + } + + /** Disown the arming and drain queued waiters with + operation_aborted, without submitting a kernel cancel. + + Runs before the caller's cancel-by-fd: an arming admitted after + this sees `closing_` and is refused, and one admitted before it + is already in the submission queue ahead of the cancel. + */ void drain_waiters_only() noexcept { - intrusive_list drained; + intrusive_list drained; { std::lock_guard lk(mutex_); closing_ = true; + armed_ = nullptr; // Drain under the lock — the kernel cancel may not produce // a !more CQE before the fd is closed, so we can't rely on // on_accept_cqe_impl to surface operation_aborted. @@ -253,16 +425,11 @@ class uring_multishot_acceptor_base while (auto* w = drained.pop_front()) { + // reset() synchronizes with an in-flight canceller; after + // it the stored status is ours to set. w->stop_cb.reset(); - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept destructor path: OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); - op->h = w->h; - op->ex = w->ex; - op->ec_out = w->ec_out; - op->impl_out = w->impl_out; - op->cancelled.store(true, std::memory_order_release); - delete w; - sched_->post(op); + w->cancelled.store(true, std::memory_order_release); + sched_->post(w); sched_->work_finished(); } } @@ -280,6 +447,12 @@ class uring_multishot_acceptor_base std::stop_token const& token, std::error_code* ec) noexcept { + auto* w = acquire_node(); + w->h = h; + w->ex = ex; + w->ec_out = ec; + w->peek = true; + bool ready = false; bool aborted = false; { @@ -295,31 +468,17 @@ class uring_multishot_acceptor_base } if (ready || aborted) { - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept initiation path: OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); - op->h = h; - op->ex = ex; - op->ec_out = ec; if (aborted) - op->cancelled.store(true, std::memory_order_release); - sched_->post(op); + w->cancelled.store(true, std::memory_order_release); + sched_->post(w); return; } - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept initiation path: OOM => std::terminate is the intended behavior - auto* w = new waiter_node{}; - w->h = h; - w->ex = ex; - w->ec_out = ec; - w->owner = static_cast(this); - w->peek = true; - // Same protocol as accept parking: arm the callback before // the node is visible and outside `mutex_` (a pre-stopped // token invokes the canceller synchronously, and the // canceller takes `mutex_`). - if (token.stop_possible()) - w->stop_cb.emplace(token, waiter_canceller{w}); + w->start(token); bool was_cancelled = false; int arm_err = 0; @@ -348,16 +507,9 @@ class uring_multishot_acceptor_base } w->stop_cb.reset(); - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept initiation path: OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); - op->h = w->h; - op->ex = w->ex; - op->ec_out = w->ec_out; - op->err = arm_err; - if (was_cancelled) - op->cancelled.store(true, std::memory_order_release); - delete w; - sched_->post(op); + w->err = arm_err; + w->cancelled.store(was_cancelled, std::memory_order_release); + sched_->post(w); } std::error_code set_option( @@ -389,55 +541,14 @@ class uring_multishot_acceptor_base return {}; } - /** Retire the multishot op before the acceptor changes descriptor. - - `cancel_and_flush` and `submit_cancel_by_fd` only submit: the - terminating CQE for the previous arming is still queued when - the caller returns. Left alone, a subsequent `start_multishot` - would alias one `user_data` across two kernel ops, and the - stale `!more` CQE would observe a cleared `closing_` and take - the re-arm branch. - - Ownership therefore moves to the scheduler rather than being - drained here. Draining is a teardown-only tool — it consumes - CQEs without dispatching them, so on a live context it would - swallow unrelated ops' completions and park their coroutines - forever. Handing the op over keeps the normal run loop in - charge of every CQE, and the op stays allocated (so its - `user_data` stays reserved) until the kernel is done with it. - - Safe with no op armed, and safe after `release_socket` left - `fd_` cleared with the op still in flight. - */ - void retire_multishot() noexcept - { - // Every field below is read by the leader mid-dispatch, so all - // of it is published inside retire_op's ring_mutex_ critical - // section — including moving multi_op_ out, since - // on_accept_cqe_impl dereferences it for peer_storage. Taking - // the acceptor mutex_ too keeps the generation bump ordered - // against the re-arm path's check. Lock order is - // ring_mutex_ -> mutex_, the same order the dispatch path - // acquires them in. - sched_->retire_op( - multi_op_, [this](uring_multi_accept_op& op) noexcept { - std::lock_guard lk(mutex_); - op.impl_ptr.reset(); - op.acceptor_impl = nullptr; - op.on_cqe = nullptr; - op.retire_func = &uring_multi_accept_op::do_retired_cqe; - arm_generation_.fetch_add(1, std::memory_order_acq_rel); - }); - } - /** Take over an already-listening descriptor. Clears the shutdown latch a previous release left behind and discards connections parked from the replaced descriptor: those belong to the socket the caller is handing away. - @pre `retire_multishot` has run, so no arming from a previous - descriptor is still in flight. + @pre No arming is live (`drain_waiters_only` has run if the + acceptor had a descriptor). @param fd The adopted descriptor. */ @@ -446,114 +557,68 @@ class uring_multishot_acceptor_base intrusive_list stale; { std::lock_guard lk(mutex_); + BOOST_COROSIO_ASSERT(armed_ == nullptr); fd_ = fd; closing_ = false; while (auto* r = ready_fds_.pop_front()) stale.push_back(r); } - while (auto* r = stale.pop_front()) - { - ::close(r->fd); - delete r; - } + discard_ready(stale); } - /** Ready an acceptor whose own descriptor is about to be armed. - - The open/bind/listen path reaches `start_multishot` without - going through `adopt_listening_fd`, and a released acceptor may - be opened and listened on again. Both of the things that path - would otherwise inherit belong to the descriptor the caller - already took away: the shutdown latch, which would have every - connection the kernel hands back closed on arrival, and the - arming still owed a terminal CQE, which a fresh submission - would alias by `user_data`. - */ /** Ready the acceptor for a listen-time arming. Returns `false` when a live arming already covers the - descriptor: a re-listen only changes the backlog, and retiring - a live arming here would leave it un-cancelled in the kernel — - two armings on one listener, with the retired one's deliveries - closed on arrival. - - An arming that failed to reach the kernel is not one of those. - It covers nothing, so a re-listen is the caller's way back and - has to be allowed through. + descriptor: a re-listen only changes the backlog. Otherwise + clears the shutdown latch a released descriptor left behind and + discards the connections it delivered, which belong to the + socket the caller took away. */ bool prepare_listen_arm() noexcept { - bool submitted = true; - { - std::lock_guard lk(mutex_); - if (multi_op_ && !closing_ && arm_err_ == 0) - return false; - // The op behind a failed arming was never handed to the - // ring, so no CQE is owed for it and it is still ours. - submitted = (arm_err_ == 0); - } - // Retiring is for an op the kernel still holds: it parks the op - // in the scheduler until a terminal CQE releases it. One that - // was never submitted would wait there for a completion that - // cannot come, so it is reused in place instead. - if (submitted) - retire_multishot(); intrusive_list stale; { std::lock_guard lk(mutex_); + if (armed_) + return false; closing_ = false; - // Deliveries queued by a released descriptor belong to - // the socket the caller took away, not to this listener. while (auto* r = ready_fds_.pop_front()) stale.push_back(r); } - while (auto* r = stale.pop_front()) - { - ::close(r->fd); - delete r; - } + discard_ready(stale); return true; } - void start_multishot() - { - if (!multi_op_) - { - multi_op_ = std::make_unique(); - multi_op_->listen_fd = fd_; - multi_op_->acceptor_impl = this; - multi_op_->on_cqe = &uring_multishot_acceptor_base::on_accept_cqe; - multi_op_->impl_ptr = this->shared_from_this(); - } - else - { - // Reuse the existing op (re-arm path). Reset peer scratch - // so the kernel writes into a clean slot. listen_fd and - // impl_ptr are re-seeded so the op can never carry state - // from an arming that has since been torn down. `res` is - // one of those: an arming that failed to submit left - // -EAGAIN there, and the reader of `res` cannot tell a - // result the kernel wrote from one it did not. - multi_op_->peer_storage = sockaddr_storage{}; - multi_op_->peer_len = sizeof(sockaddr_storage); - multi_op_->res = 0; - multi_op_->listen_fd = fd_; - multi_op_->impl_ptr = this->shared_from_this(); - } + /** Arm a multishot accept on the current descriptor. - auto* op = multi_op_.get(); - // Deliberately no work_started(): the multishot SQE is a persistent - // internal mechanism. User-visible work is tracked per-accept call. - // The try_ spelling is what says so: it keeps a failed submission - // off the scheduler's completion queue, which spends a - // work_finished() on everything it dispatches. - if (uring_try_submit_op(*sched_, op)) + The op is handed to the kernel with a reference on this + acceptor; the caller must hold one. Refused when closing or + when an arming is already live. + */ + void start_multishot() noexcept + { + uring_multi_accept_op* op; { std::lock_guard lk(mutex_); - arm_err_ = 0; - return; + op = free_ops_.pop_front(); } - fail_arm(EAGAIN); + if (!op) + { + // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept initiation path: OOM => std::terminate is the intended behavior + op = new uring_multi_accept_op(); + op->sched_ = sched_; + op->acceptor_impl = this; + op->on_cqe = &uring_multishot_acceptor_base::on_accept_cqe; + op->on_end = &uring_multishot_acceptor_base::on_arming_end; + } + op->object_ref_ = detail::object_ref(this); + submit_arming(op, [this, op]() noexcept { + if (closing_ || armed_) + return false; + armed_ = op; + op->listen_fd = fd_; + return true; + }); } /** Report an arming that never reached the kernel. @@ -569,7 +634,7 @@ class uring_multishot_acceptor_base */ void fail_arm(int err) noexcept { - intrusive_list claimed; + intrusive_list claimed; { std::lock_guard lk(mutex_); arm_err_ = err; @@ -577,7 +642,7 @@ class uring_multishot_acceptor_base // canceller already claimed belongs to it: cancel_waiter // is waiting on this mutex to unlink the node itself, so // it has to still be in the list when it gets in. - intrusive_list keep; + intrusive_list keep; while (auto* w = waiters_.pop_front()) { if (!w->cancelled.exchange(true, std::memory_order_acq_rel)) @@ -598,16 +663,13 @@ class uring_multishot_acceptor_base while (auto* w = claimed.pop_front()) { + // The exchange above was a claim, not the completion + // status: after reset() quiesces the canceller, restore + // the flag so the handler reports `err`, not canceled. w->stop_cb.reset(); - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept arming path: OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); - op->h = w->h; - op->ex = w->ex; - op->ec_out = w->ec_out; - op->impl_out = w->impl_out; - op->err = err; - delete w; - sched_->post(op); + w->err = err; + w->cancelled.store(false, std::memory_order_release); + sched_->post(w); sched_->work_finished(); // balance the waiter's work_started } } @@ -622,6 +684,12 @@ class uring_multishot_acceptor_base std::error_code* ec, io_object::implementation** impl_out) { + auto* w = acquire_node(); + w->h = h; + w->ex = ex; + w->ec_out = ec; + w->impl_out = impl_out; + sockaddr_storage peer_storage{}; socklen_t peer_len = sizeof(peer_storage); int accepted_fd = ::accept4( @@ -629,78 +697,55 @@ class uring_multishot_acceptor_base SOCK_NONBLOCK | SOCK_CLOEXEC); if (accepted_fd >= 0) { - auto* op = new uring_accept_op(); - op->h = h; - op->ex = ex; - op->ec_out = ec; - op->impl_out = impl_out; - op->peer_service = peer_service_; - op->adopt_fn = &Derived::adopt_thunk; - op->accepted_fd = accepted_fd; - op->peer_storage = peer_storage; - op->peer_len = peer_len; - sched_->post(op); + w->peer_service = peer_service_; + w->adopt_fn = &Derived::adopt_thunk; + w->accepted_fd = accepted_fd; + w->peer_storage = peer_storage; + w->peer_len = peer_len; + sched_->post(w); return; } // accept4 returned <0 — only EAGAIN/EWOULDBLOCK should fall // through to the parked/waiter path. Other errors (EBADF, etc.) // surface through the existing scheduler-completion path so the - // user sees them via the op's ec_out. Build an op with `err` - // set so do_handler delivers make_err(err). + // user sees them via the node's ec_out: `err` set means + // do_handler delivers make_err(err). if (errno != EAGAIN && errno != EWOULDBLOCK) { - int saved_errno = errno; - auto* op = new uring_accept_op(); - op->h = h; - op->ex = ex; - op->ec_out = ec; - op->impl_out = impl_out; - op->err = saved_errno; - sched_->post(op); + w->err = errno; + sched_->post(w); return; } - uring_accept_op* ready_op = nullptr; + bool have_ready = false; { std::lock_guard lk(mutex_); if (auto* r = ready_fds_.pop_front()) { - ready_op = new uring_accept_op(); - ready_op->h = h; - ready_op->ex = ex; - ready_op->ec_out = ec; - ready_op->impl_out = impl_out; - ready_op->peer_service = peer_service_; - ready_op->adopt_fn = &Derived::adopt_thunk; - ready_op->accepted_fd = r->fd; - ready_op->peer_storage = r->peer; - ready_op->peer_len = r->peer_len; - delete r; + w->peer_service = peer_service_; + w->adopt_fn = &Derived::adopt_thunk; + w->accepted_fd = r->fd; + w->peer_storage = r->peer; + w->peer_len = r->peer_len; + free_ready_.push_front(r); + have_ready = true; } } - if (ready_op) + if (have_ready) { // Post outside the lock — acceptor mutex_ must never be // held while dispatch_mutex_ is acquired by sched_->post(). - sched_->post(ready_op); + sched_->post(w); return; } - auto* w = new waiter_node{}; - w->h = h; - w->ex = ex; - w->ec_out = ec; - w->impl_out = impl_out; - w->owner = static_cast(this); - // Arm the stop callback before the node is visible in // `waiters_` and outside `mutex_`: an already-stopped token // invokes the canceller synchronously from emplace, and // cancel_waiter takes `mutex_` (self-deadlock if held). // Arming pre-queue also keeps the CQE handler from claiming - // and deleting a node whose callback is not yet constructed. - if (token.stop_possible()) - w->stop_cb.emplace(token, waiter_canceller{w}); + // a node whose callback is not yet constructed. + w->start(token); bool was_cancelled = false; { @@ -715,29 +760,18 @@ class uring_multishot_acceptor_base { // A connection arrived while the callback was armed; // prefer it over parking the waiter behind it. - ready_op = new uring_accept_op(); - ready_op->h = h; - ready_op->ex = ex; - ready_op->ec_out = ec; - ready_op->impl_out = impl_out; - ready_op->peer_service = peer_service_; - ready_op->adopt_fn = &Derived::adopt_thunk; - ready_op->accepted_fd = r->fd; - ready_op->peer_storage = r->peer; - ready_op->peer_len = r->peer_len; - delete r; + w->peer_service = peer_service_; + w->adopt_fn = &Derived::adopt_thunk; + w->accepted_fd = r->fd; + w->peer_storage = r->peer; + w->peer_len = r->peer_len; + free_ready_.push_front(r); } else if (arm_err_ != 0) { // No arming reached the kernel, so no CQE will deliver // a connection: parking here would park for good. - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept accept path: OOM => std::terminate is the intended behavior - ready_op = new uring_accept_op(); - ready_op->h = h; - ready_op->ex = ex; - ready_op->ec_out = ec; - ready_op->impl_out = impl_out; - ready_op->err = arm_err_; + w->err = arm_err_; } else { @@ -748,34 +782,20 @@ class uring_multishot_acceptor_base } } - if (was_cancelled) - { - auto* op = new uring_accept_op(); - op->h = w->h; - op->ex = w->ex; - op->ec_out = w->ec_out; - op->impl_out = w->impl_out; - op->cancelled.store(true, std::memory_order_release); - w->stop_cb.reset(); - delete w; - sched_->post(op); - return; - } - w->stop_cb.reset(); - delete w; - sched_->post(ready_op); + w->cancelled.store(was_cancelled, std::memory_order_release); + sched_->post(w); } - void cancel_waiter(waiter_node* w) noexcept + void cancel_waiter(accept_node* w) noexcept { { std::lock_guard lk(mutex_); if (closing_) - return; // on_accept_cqe_impl will drain with closing_ set + return; // drain_waiters_only will complete with closing_ set if (!w->queued) return; // not queued yet; the parking path observes - // `cancelled` and completes the op + // `cancelled` and completes the node if (w->peek) { if (read_wait_ != w) @@ -787,38 +807,117 @@ class uring_multishot_acceptor_base waiters_.remove(w); } } - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — stop-token callback: noexcept, OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); - op->h = w->h; - op->ex = w->ex; - op->ec_out = w->ec_out; - op->impl_out = w->impl_out; - op->cancelled.store(true, std::memory_order_release); - delete w; - // post() increments outstanding_work_; balances the work_started() - // from accept() when the waiter was queued. - sched_->post(op); + // `cancelled` is already true (the claim that routed here); + // that is also the completion status, so post as-is. The node's + // stop_cb is still engaged — do_handler resets it, and running + // that reset from the handler thread while this callback + // returns is the stop_callback destructor's documented + // wait-or-self case. + sched_->post(w); sched_->work_finished(); // balance the work_started() from accept() } private: + /** Submit @p op as the live arming, or recycle it. + + @param admit Runs under `mutex_`, itself under the ring mutex; + claims `armed_` for @p op and returns true, or returns + false to refuse. + */ + template + void submit_arming(uring_multi_accept_op* op, Admit admit) noexcept + { + op->peer_storage = sockaddr_storage{}; + op->peer_len = sizeof(sockaddr_storage); + op->res = 0; + // Deliberately no work_started(): the arming is an internal + // mechanism; user-visible work is tracked per accept. + // Once submitted, a re-arm's only reference to this acceptor + // is the op's, which the kernel may already have ended, so + // the error is cleared while admission still holds the lock. + auto r = uring_try_submit_op_if(*sched_, op, [&]() noexcept { + std::lock_guard lk(mutex_); + if (!admit()) + return false; + arm_err_ = 0; + return true; + }); + if (r == uring_guarded_submit::submitted) + return; + if (r == uring_guarded_submit::sq_full) + { + { + std::lock_guard lk(mutex_); + if (armed_ == op) + armed_ = nullptr; + } + fail_arm(EAGAIN); + } + recycle_arming(op); + } + + /// Return an ended arming to `free_ops_`, then drop its reference, + /// which may retire this impl, after the lock. + void recycle_arming(uring_multi_accept_op* op) noexcept + { + auto keep = std::move(op->object_ref_); + std::lock_guard lk(mutex_); + free_ops_.push_front(op); + } + + static void on_accept_cqe( + void* self_ptr, + uring_multi_accept_op* op, + int new_fd, + int err, + bool more) noexcept + { + static_cast(self_ptr)->on_accept_cqe_impl( + op, new_fd, err, more); + } + static void - on_accept_cqe(void* self_ptr, int new_fd, int err, bool more) noexcept + on_arming_end(void* self_ptr, uring_multi_accept_op* op, bool live) noexcept { - static_cast(self_ptr)->on_accept_cqe_impl(new_fd, err, more); + auto* self = static_cast(self_ptr); + if (live) + { + bool rearm; + { + std::lock_guard lk(self->mutex_); + rearm = self->armed_ == op && !self->closing_; + } + if (rearm) + { + // The kernel ended a live arming on its own; keep + // accepting with the same op and reference. + self->submit_arming(op, [self, op]() noexcept { + return self->armed_ == op && !self->closing_; + }); + return; + } + } + { + std::lock_guard lk(self->mutex_); + if (self->armed_ == op) + self->armed_ = nullptr; + } + self->recycle_arming(op); } protected: - void on_accept_cqe_impl(int new_fd, int err, bool more) noexcept + void on_accept_cqe_impl( + uring_multi_accept_op* op, int new_fd, int err, bool more) noexcept { - bool was_closing = false; - waiter_node* matched = nullptr; - waiter_node* claimed_peek = nullptr; - intrusive_list closing_waiters; + accept_node* matched = nullptr; + accept_node* claimed_peek = nullptr; + intrusive_list closing_waiters; { std::lock_guard lk(mutex_); - was_closing = closing_; - if (!was_closing && new_fd >= 0 && read_wait_ && + // A disowned arming's deliveries belong to a descriptor + // this acceptor no longer has. + bool const stale = (op != armed_); + if (!stale && new_fd >= 0 && read_wait_ && !read_wait_->cancelled.exchange( true, std::memory_order_acq_rel)) { @@ -828,11 +927,11 @@ class uring_multishot_acceptor_base claimed_peek = read_wait_; read_wait_ = nullptr; } - if (was_closing) + if (stale) { if (new_fd >= 0) ::close(new_fd); - if (!more) + if (!more && closing_) { // Collect waiters to drain after the lock is released. while (auto* w = waiters_.pop_front()) @@ -855,139 +954,64 @@ class uring_multishot_acceptor_base } else if (new_fd >= 0) { - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — CQE handler: noexcept, OOM => std::terminate is the intended behavior - auto* node = new ready_fd_node{}; + auto* node = acquire_ready_locked(); node->fd = new_fd; - node->peer = multi_op_->peer_storage; - node->peer_len = multi_op_->peer_len; + node->peer = op->peer_storage; + node->peer_len = op->peer_len; ready_fds_.push_back(node); } } else if (new_fd >= 0) { - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — CQE handler: noexcept, OOM => std::terminate is the intended behavior - auto* node = new ready_fd_node{}; + auto* node = acquire_ready_locked(); node->fd = new_fd; - node->peer = multi_op_->peer_storage; - node->peer_len = multi_op_->peer_len; + node->peer = op->peer_storage; + node->peer_len = op->peer_len; ready_fds_.push_back(node); } } + // Each claim's exchange set `cancelled`; after reset() has + // quiesced any in-flight canceller, restore the flag to the + // real completion status before posting. if (claimed_peek) { claimed_peek->stop_cb.reset(); - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — CQE handler: noexcept, OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); - op->h = claimed_peek->h; - op->ex = claimed_peek->ex; - op->ec_out = claimed_peek->ec_out; - delete claimed_peek; - sched_->post(op); + claimed_peek->cancelled.store(false, std::memory_order_release); + sched_->post(claimed_peek); sched_->work_finished(); // balance the parking work_started } if (matched) { matched->stop_cb.reset(); - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — CQE handler: noexcept, OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); - op->h = matched->h; - op->ex = matched->ex; - op->ec_out = matched->ec_out; - op->impl_out = matched->impl_out; - op->peer_service = peer_service_; - op->adopt_fn = &Derived::adopt_thunk; + matched->peer_service = peer_service_; + matched->adopt_fn = &Derived::adopt_thunk; if (err) { - op->err = err; + matched->err = err; } else if (new_fd >= 0) { - op->accepted_fd = new_fd; - op->peer_storage = multi_op_->peer_storage; - op->peer_len = multi_op_->peer_len; + matched->accepted_fd = new_fd; + matched->peer_storage = op->peer_storage; + matched->peer_len = op->peer_len; } - delete matched; - sched_->post(op); + matched->cancelled.store(false, std::memory_order_release); + sched_->post(matched); sched_->work_finished(); // balance waiter's work_started } while (auto* w = closing_waiters.pop_front()) { w->stop_cb.reset(); - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — CQE handler shutdown path: noexcept, OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); - op->h = w->h; - op->ex = w->ex; - op->ec_out = w->ec_out; - op->impl_out = w->impl_out; - op->cancelled.store(true, std::memory_order_release); - delete w; - sched_->post(op); + w->cancelled.store(true, std::memory_order_release); + sched_->post(w); sched_->work_finished(); // balance waiter's work_started } - - if (!more && !was_closing) - { - // Re-arm: kernel terminated multishot non-fatally. - struct rearm_op final : scheduler_op - { - std::shared_ptr self_; - std::uint64_t generation_; - rearm_op( - std::shared_ptr s, - std::uint64_t generation) noexcept - : self_(std::move(s)) - , generation_(generation) - { - } - - void operator()() override - { - auto self = std::move(self_); - auto generation = generation_; - delete this; - { - std::lock_guard lk(self->mutex_); - if (self->closing_) - return; - // The arming this was posted for may have been - // retired by assign() in the meantime. multi_op_ - // then names a different, already-armed op, and - // resubmitting it would alias one user_data - // across two kernel armings — a use-after-free - // once the first terminal CQE frees the op. - if (self->arm_generation_.load( - std::memory_order_acquire) != generation) - return; - } - self->start_multishot(); - } - - void destroy() override - { - delete this; - } - }; - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — CQE handler re-arm: noexcept, OOM => std::terminate is the intended behavior - sched_->post(new rearm_op( - this->shared_from_this(), - arm_generation_.load(std::memory_order_acquire))); - } } }; -template -inline void -uring_multishot_acceptor_base:: - waiter_canceller::operator()() const noexcept -{ - if (w->cancelled.exchange(true, std::memory_order_acq_rel)) - return; - w->owner->cancel_waiter(w); -} - } // namespace boost::corosio::detail #endif // BOOST_COROSIO_HAS_URING diff --git a/include/boost/corosio/native/detail/uring/uring_op.hpp b/include/boost/corosio/native/detail/uring/uring_op.hpp index 2128731de..82e3bd940 100644 --- a/include/boost/corosio/native/detail/uring/uring_op.hpp +++ b/include/boost/corosio/native/detail/uring/uring_op.hpp @@ -33,7 +33,7 @@ namespace boost::corosio::detail { `coro_op` supplies the fields common to both proactor backends (coroutine handle, executor, output pointers, stop_token wiring, - impl_ptr keepalive). This type adds the CQE result (`res`/`cqe_flags`), + object_ref_ keepalive). This type adds the CQE result (`res`/`cqe_flags`), the ring-cancel visibility flag (`sqe_set`), and the two function pointers the run loop uses to prep an SQE and dispatch a CQE without template instantiation. @@ -53,13 +53,6 @@ struct uring_op : coro_op /// data-driven (no template instantiation, no allocation). using prep_func_type = void (*)(uring_op*, ::io_uring_sqe*) noexcept; - /// Retired-CQE dispatcher type. Called in place of `cqe_func` once - /// the op is retired, so the op type can release whatever `res` - /// owns (an accepted descriptor, a registered buffer) that no - /// handler will now take delivery of. - using retire_func_type = - void (*)(uring_op*, int res, unsigned flags) noexcept; - explicit uring_op( func_type post_func, cqe_func_type cqe_fn, @@ -85,17 +78,6 @@ struct uring_op : coro_op /// Scheduler reference for submitting cancel SQEs on stop_token. uring_scheduler* sched_ = nullptr; - /// Set when the op's owner went away while the kernel still held - /// its user_data (see `uring_scheduler::retire_op`). A retired - /// op belongs to the scheduler: the run loop routes its CQEs to - /// `retire_func` instead of `cqe_func` and frees the op on the - /// terminal CQE. - bool retired = false; - - /// Disposal hook used while `retired` is set. May be null when the - /// op's result owns nothing. - retire_func_type retire_func = nullptr; - /// Bridge virtual dispatch to func-pointer dispatch. Lets the run /// loop dispatch any scheduler_op via `(*op)()` — both reactor-style /// services posted into the queue and proactor-style io_uring ops. diff --git a/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp b/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp index 1c3a8f5a4..4e72e3ae2 100644 --- a/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp +++ b/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp @@ -16,6 +16,7 @@ #if BOOST_COROSIO_HAS_URING #include +#include #include #include #include @@ -28,7 +29,6 @@ #include #include #include -#include #include #include #include @@ -57,28 +57,136 @@ class uring_random_access_file_service; */ class BOOST_COROSIO_DECL uring_random_access_file final : public random_access_file::implementation - , public std::enable_shared_from_this , public intrusive_list::node { friend class uring_random_access_file_service; - int fd_ = -1; - uring_scheduler* sched_ = nullptr; + int fd_ = -1; + uring_scheduler* sched_ = nullptr; + uring_random_access_file_service* svc_ = nullptr; // Random-access files legitimately support concurrent ops at // different offsets on the same fd (e.g. parallel reads in // testConcurrentReads). Embedding a single slot would smash - // state across calls; ops are heap-allocated per submission. + // state across calls; ops recycle through per-file free lists + // so concurrency is unbounded but steady state allocates + // nothing. The storage belongs to this impl (it survives + // recycling) and only the destructor frees it. + + /** Pooled random-access op: completion recycles instead of + deleting. + + Reuses the heap variant's submission/finish machinery but + swaps the completion handler (the func-pointer convention + makes that one protected assignment) for one that returns the + op to its file's free list. The inherited `object_ref_` + keepalive is dropped only after the push, so a final release + never runs under `ops_mutex_`. + */ + template + struct pooled_op final + : Base + , intrusive_list>::node + { + uring_random_access_file* file_ = nullptr; + + pooled_op() noexcept + { + this->func_ = &do_pooled_handler; + } + + static void do_pooled_handler( + void* owner, + scheduler_op* base, + std::uint32_t /*bytes*/, + std::uint32_t /*error*/) noexcept + { + auto* self = static_cast(base); + self->stop_cb.reset(); + auto keep = std::move(self->object_ref_); + if (owner == nullptr) + { + self->file_->release_op(self); + return; + } + Base::finish(self); + auto* c = self->awaiting; + auto ex = self->ex; + self->file_->release_op(self); + dispatch_coro(ex, *c).resume(); + } + }; + + using pooled_read_op = pooled_op; + using pooled_write_op = pooled_op; + + std::mutex ops_mutex_; + intrusive_list free_reads_; + intrusive_list free_writes_; + + /// Pop a recycled op of the given kind, or allocate on the cold + /// path; `prepare()` resets every per-use field. + template + Op* acquire_op(intrusive_list& free_list) + { + Op* op; + { + std::lock_guard lock(ops_mutex_); + op = free_list.pop_front(); + } + if (!op) + op = new Op(); + op->file_ = this; + return op; + } + + void release_op(pooled_read_op* op) noexcept + { + std::lock_guard lock(ops_mutex_); + free_reads_.push_front(op); + } + + void release_op(pooled_write_op* op) noexcept + { + std::lock_guard lock(ops_mutex_); + free_writes_.push_front(op); + } public: - explicit uring_random_access_file(uring_scheduler& sched) noexcept + explicit uring_random_access_file( + uring_random_access_file_service& svc, + uring_scheduler& sched) noexcept : sched_(&sched) + , svc_(&svc) { } ~uring_random_access_file() override { close_file(); + while (auto* op = free_reads_.pop_front()) + delete op; + while (auto* op = free_writes_.pop_front()) + delete op; + } + + /// Recycle into the owning service's pool. Defined out-of-line + /// after uring_random_access_file_service for its complete type. + void retire() noexcept override; + + /** Reset for recycling. + + `close_file()` already drove fd_ to its closed value before the + refcount reached zero; every op holds its own `object_ref` + while in flight, so none can be outstanding here. The free + lists deliberately survive recycling — the storage belongs to + this impl and only the destructor frees it. + + @pre refs_ == 0, fd closed, no op in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(fd_ == -1); } // -- random_access_file::implementation -- @@ -239,32 +347,32 @@ uring_random_access_file::read_some_at( std::error_code* ec, std::size_t* bytes) { - auto op_guard = std::make_unique(); - op_guard->prepare( + auto* op = acquire_op(free_reads_); + op->prepare( cont.h, ex, ec, bytes, fd_, static_cast(user_offset), - sched_, shared_from_this(), buffers, token); - op_guard->awaiting = &cont; + sched_, detail::object_ref(this), buffers, token); + op->awaiting = &cont; sched_->work_started(); // Closed-object contract outranks the zero-length no-op. if (fd_ < 0) { - op_guard->empty_buffer = false; - op_guard->res = -EBADF; + op->empty_buffer = false; + op->res = -EBADF; uring_scheduler::lock_type lock(sched_->dispatch_mutex()); - sched_->push_completed_locked(op_guard.release()); + sched_->push_completed_locked(op); return std::noop_coroutine(); } - if (op_guard->empty_buffer || - op_guard->cancelled.load(std::memory_order_acquire)) + if (op->empty_buffer || + op->cancelled.load(std::memory_order_acquire)) { uring_scheduler::lock_type lock(sched_->dispatch_mutex()); - sched_->push_completed_locked(op_guard.release()); + sched_->push_completed_locked(op); return std::noop_coroutine(); } - uring_submit_op(*sched_, op_guard.release()); + uring_submit_op(*sched_, op); return std::noop_coroutine(); } @@ -278,32 +386,32 @@ uring_random_access_file::write_some_at( std::error_code* ec, std::size_t* bytes) { - auto op_guard = std::make_unique(); - op_guard->prepare( + auto* op = acquire_op(free_writes_); + op->prepare( cont.h, ex, ec, bytes, fd_, static_cast(user_offset), - sched_, shared_from_this(), buffers, token); - op_guard->awaiting = &cont; + sched_, detail::object_ref(this), buffers, token); + op->awaiting = &cont; sched_->work_started(); // Closed-object contract outranks the zero-length no-op. if (fd_ < 0) { - op_guard->empty_buffer = false; - op_guard->res = -EBADF; + op->empty_buffer = false; + op->res = -EBADF; uring_scheduler::lock_type lock(sched_->dispatch_mutex()); - sched_->push_completed_locked(op_guard.release()); + sched_->push_completed_locked(op); return std::noop_coroutine(); } - if (op_guard->empty_buffer || - op_guard->cancelled.load(std::memory_order_acquire)) + if (op->empty_buffer || + op->cancelled.load(std::memory_order_acquire)) { uring_scheduler::lock_type lock(sched_->dispatch_mutex()); - sched_->push_completed_locked(op_guard.release()); + sched_->push_completed_locked(op); return std::noop_coroutine(); } - uring_submit_op(*sched_, op_guard.release()); + uring_submit_op(*sched_, op); return std::noop_coroutine(); } @@ -345,6 +453,12 @@ class BOOST_COROSIO_DECL uring_random_access_file_service final } }; +inline void +uring_random_access_file::retire() noexcept +{ + svc_->pool_.recycle(this); +} + } // namespace boost::corosio::detail #endif // BOOST_COROSIO_HAS_URING diff --git a/include/boost/corosio/native/detail/uring/uring_scheduler.hpp b/include/boost/corosio/native/detail/uring/uring_scheduler.hpp index 87c67b408..cb1c22315 100644 --- a/include/boost/corosio/native/detail/uring/uring_scheduler.hpp +++ b/include/boost/corosio/native/detail/uring/uring_scheduler.hpp @@ -223,8 +223,7 @@ class BOOST_COROSIO_DECL uring_scheduler final Test-only helper: `uring_inflight_` is an internal accounting counter (it gates the `do_one` ring pump), with no bearing on the public API. It is exposed solely so tests can assert the counter - stays balanced across op submission and teardown — in particular - that `drain_cqes_for` does not leak counts. Not thread-safe with + stays balanced across op submission and teardown. Not thread-safe with respect to a concurrently running scheduler; call it from a quiesced context. @@ -302,91 +301,6 @@ class BOOST_COROSIO_DECL uring_scheduler final */ void cancel_and_flush(int fd) noexcept; - /** Take ownership of an op the kernel may still complete. - - For member-owned ops (e.g. a multishot accept arming) whose - owner goes away on a *non-terminal* path — the owning acceptor - adopting a different descriptor, say. Draining the op's CQEs - there is not an option: `drain_cqes_for` consumes without - dispatching, which is only tolerable while the whole context - is being torn down. - - The op stays allocated, and its `user_data` therefore stays - reserved, until the kernel delivers its terminal CQE. That is - what keeps a freshly submitted op from aliasing the retired - one. `process_completions` routes a retired op's CQEs to its - `retire_func` and frees the op on the terminal CQE; anything - still outstanding is released when the scheduler is destroyed, - after `io_uring_queue_exit`. - - @par Invariant - Retirement state — the `retired` flag, and every owner - back-pointer @p prepare clears — is written and read only - under `ring_mutex_`. `process_completions` runs the whole CQE - dispatch under that mutex, so taking it here is the only thing - that orders this publication against a leader mid-dispatch. - Everything is therefore done inside one critical section: - @p prepare runs, the flag is set, and the op moves into the - retired list before the mutex is released. - - @par Thread Safety - Safe to call from any thread. Takes `ring_mutex_` then - `retired_mutex_`, the same order `release_retired_op` uses. - Callers must hold neither. - - @pre After @p prepare runs, @p slot must hold no reference back - to its owner (`impl_ptr` cleared, back-pointers nulled). - `release_retired_op` deletes the op from inside the CQE - loop with `ring_mutex_` held, so its destructor must not - re-enter the scheduler or touch the owner. - - @param slot The owner's pointer to the op. Emptied on return; - ignored if already empty. - @param prepare Invoked as `prepare(*slot)` under `ring_mutex_` - to clear back-pointers and install a `retire_func`. Must - not take `ring_mutex_` or post work. - */ - template - void retire_op(std::unique_ptr& slot, PrepareFn prepare) noexcept - { - // Interrupt first so a leader parked in the kernel drops - // ring_mutex_ promptly, as cancel_and_flush does. - interrupt_reactor(); - lock_type lock(ring_mutex_); - if (!slot) - return; - prepare(*slot); - slot->retired = true; - std::lock_guard retired_lock(retired_mutex_); - retired_ops_.push_back(std::move(slot)); - } - - /** Drain pending CQEs for a specific op's `user_data`. - - Submits an ASYNC_CANCEL by user_data to short-circuit any - in-flight op holding `target`, then iterates the CQ ring and - consumes every CQE matching `target` so its memory can be - freed safely. Used by member-owned ops (e.g. - `uring_multi_accept_op`) whose destructor cannot tolerate - outstanding CQEs. - - @warning Teardown paths only. Every CQE this walks is consumed - and none are dispatched, so an unrelated op completing inside - the drain window loses its handler and its coroutine never - resumes. That is only tolerable when the owning object is - being destroyed. To retire a still-live op on a non-terminal - path, use @ref retire_op instead. - - @par Thread Safety - Safe to call from any thread. Internally takes `ring_mutex_` - to serialise against the run-loop leader; calls - `interrupt_reactor()` first so the leader returns from its - kernel wait promptly. - - @param target The op pointer used as user_data on the SQE. - */ - void drain_cqes_for(uring_op* target) noexcept; - /** Queue an already-counted op while the caller holds dispatch_mutex_. Does NOT increment `outstanding_work_`. Use for synchronous @@ -492,20 +406,6 @@ class BOOST_COROSIO_DECL uring_scheduler final int cancel_sentinel_ = 0; - // Ops adopted by retire_op, kept alive so the kernel never sees - // their user_data reused. Declared before ring_ is exited only in - // the sense that the destructor body runs io_uring_queue_exit - // first; the vector is then destroyed with the rest of the members, - // by which point the kernel can no longer reference these ops. - // A plain std::mutex (not the conditionally-enabled mutex_type): - // retirement happens on user threads regardless of the threading - // configuration. Leaf lock — nothing else is taken under it. - mutable std::mutex retired_mutex_; - std::vector> retired_ops_; - - /// Free a retired op once its terminal CQE has been consumed. - void release_retired_op(uring_op* op) noexcept; - // Signal self-pipe integration. The read end is watched via a multishot // POLL SQE tagged with &signal_pipe_sentinel_ (distinct from nullptr = // wakeup eventfd and &cancel_sentinel_). On its CQE we re-arm the poll if @@ -560,12 +460,6 @@ class BOOST_COROSIO_DECL uring_scheduler final /// Single embedded `submit_sqes_op` instance, owned by the scheduler. mutable submit_sqes_op submit_op_; - // drain_cqes_for tuning. The bound exists to avoid stalling a - // destructor if the kernel never returns a cancel completion (best- - // effort drain); 8 rounds * 1ms == 8ms worst case. - static constexpr int drain_cqes_max_rounds = 8; - static constexpr unsigned long drain_cqes_kick_ns = 1'000'000; - // ring_inited_ goes true once the ring exists. The init is deferred // from the constructor so configure_threading() and configure_sqpoll() // can take effect before io_uring_queue_init_params chooses flags. @@ -733,19 +627,57 @@ uring_scheduler::shutdown() { stopped_.store(true, std::memory_order_release); - // Drain posted ops, calling destroy() on each so embedded handles - // (coroutine frames, error_code outputs) get torn down rather - // than leaked. Mirrors reactor_scheduler::shutdown_drain. - // - // Service shutdown order (driven by capy::execution_context): - // each socket/acceptor service::shutdown() submits a cancel SQE - // for every live impl. The CQEs that result either land in - // completed_ops_ (drained here as op->destroy()) or stay in the - // kernel ring; ~scheduler's io_uring_queue_exit cleans the - // latter up at process teardown. Self-referential impl_ptr - // cycles (e.g. multishot acceptor's multi_op_->impl_ptr) are - // broken explicitly inside each service before the scheduler - // shutdown runs. + // Cancel every request still in the kernel and wait for each to + // come back: the services free their impls next, and an op still + // armed would write into freed memory. One cancel-any covers ops + // whose own service-side cancel could not be queued. The kernel + // answers every cancel, so once it is submitted the wait ends; a + // ring that will not take it leaves nothing that could end a wait. + // A signal or a full completion queue only delays either step. + if (ring_inited_) + { + lock_type ring_lock(ring_mutex_); + io_uring_sqe* sqe = ::io_uring_get_sqe(&ring_); + if (!sqe) + { + ::io_uring_submit(&ring_); + sqe = ::io_uring_get_sqe(&ring_); + } + if (sqe) + { + ::io_uring_prep_cancel64(sqe, 0, IORING_ASYNC_CANCEL_ANY); + ::io_uring_sqe_set_data(sqe, &cancel_sentinel_); + inflight_inc(); + } + int submitted = -EINVAL; + if (sqe) + { + do + { + submitted = ::io_uring_submit(&ring_); + if (submitted == -EBUSY) + process_completions(); + } + while (submitted == -EINTR || submitted == -EAGAIN || + submitted == -EBUSY); + } + while (submitted >= 0 && + uring_inflight_.load(std::memory_order_acquire) > 0) + { + ::io_uring_cqe* cqe = nullptr; + int const rc = ::io_uring_wait_cqe_timeout(&ring_, &cqe, nullptr); + if (rc == -EINTR || rc == -EAGAIN || rc == -ETIME) + continue; + if (rc != 0) + break; + process_completions(); + } + } + + // Drain posted ops, including the completions reaped above, + // calling destroy() on each so embedded handles (coroutine frames, + // error_code outputs) get torn down rather than leaked. Mirrors + // reactor_scheduler::shutdown_drain. lock_type lock(dispatch_mutex_); while (auto e = completed_ops_.pop()) { @@ -1426,31 +1358,12 @@ uring_scheduler::process_completions() else { auto* iop = static_cast(ud); - if (iop->retired) - { - // The owner handed this op to retire_op and is no - // longer listening. Never dispatch cqe_func — the - // handler's output pointers and coroutine are gone. - // retire_func disposes of whatever the result owns - // (an accepted descriptor, say), since only the op - // type knows what `res` means. - if (iop->retire_func) - (*iop->retire_func)(iop, cqe->res, cqe->flags); - if ((cqe->flags & IORING_CQE_F_MORE) == 0) - { - ++inflight_dec; - release_retired_op(iop); - } - } - else - { - (*iop->cqe_func)(iop, cqe->res, cqe->flags, local_ops); - // Decrement inflight on the terminal CQE only — multishot - // ops (acceptor) hold the SQE alive across F_MORE CQEs and - // free it only when F_MORE is cleared. - if ((cqe->flags & IORING_CQE_F_MORE) == 0) - ++inflight_dec; - } + (*iop->cqe_func)(iop, cqe->res, cqe->flags, local_ops); + // Decrement inflight on the terminal CQE only — multishot + // ops (acceptor) hold the SQE alive across F_MORE CQEs and + // free it only when F_MORE is cleared. + if ((cqe->flags & IORING_CQE_F_MORE) == 0) + ++inflight_dec; } ++consumed; } @@ -1577,155 +1490,6 @@ uring_scheduler::cancel_and_flush(int fd) noexcept io_uring_submit(&ring_); } -inline void -uring_scheduler::release_retired_op(uring_op* op) noexcept -{ - // Called from the CQE loop with ring_mutex_ held; takes - // retired_mutex_ under it, matching retire_op's order. Deleting - // here is safe only because retire_op requires the op to hold no - // reference back to its owner, so ~op cannot re-enter the - // scheduler or touch a destroyed acceptor. - std::unique_ptr owned; - { - std::lock_guard lock(retired_mutex_); - for (auto it = retired_ops_.begin(); it != retired_ops_.end(); ++it) - { - if (it->get() == op) - { - owned = std::move(*it); - retired_ops_.erase(it); - break; - } - } - } -} - -inline void -uring_scheduler::drain_cqes_for(uring_op* target) noexcept -{ - lazy_init_ring(); - // Submit a cancel by user_data so the kernel returns CQEs for - // the target promptly, then iterate the CQ ring and consume - // every CQE that matches `target`. ring_mutex_ serializes against - // the leader's kernel wait and any concurrent cancel path; the - // interrupt_reactor() ensures the leader returns promptly so we - // can take the mutex. - interrupt_reactor(); - { - lock_type lock(ring_mutex_); - if (auto* sqe = io_uring_get_sqe(&ring_)) - { - io_uring_prep_cancel(sqe, target, 0); - io_uring_sqe_set_data(sqe, &cancel_sentinel_); - inflight_inc(); - } - io_uring_submit(&ring_); - } - - // Loop a few rounds: cancel SQE submission, then drain CQEs. - // Bounded loop avoids stalls if the kernel never returns a - // cancel completion — best-effort. - for (int rounds = 0; rounds < drain_cqes_max_rounds; ++rounds) - { - lock_type lock(ring_mutex_); - - unsigned head; - ::io_uring_cqe* cqe; - unsigned consumed = 0; - bool saw_target = false; - std::int64_t inflight_dec = 0; - - io_uring_for_each_cqe(&ring_, head, cqe) - { - // Mirror process_completions' uring_inflight_ accounting. - // That counter gates the do_one ring pump, so every CQE we - // advance past here must adjust it exactly as the normal - // drain would — otherwise it drifts upward (each teardown - // leaks the counts of the CQEs it swallows), defeating the - // idle-skip optimisation for the lifetime of the io_context. - // We do NOT dispatch real ops — the target is being - // destructed and siblings may already be freed — but we still - // account for and house-keep each CQE we consume. - void* ud = io_uring_cqe_get_data(cqe); - if (ud == nullptr) - { - // Wakeup eventfd CQE — our own interrupt_reactor() above - // very likely produced one. Drain the byte and re-arm if - // the multishot terminated, exactly as process_completions - // does. Never incremented, so never decremented. - drain_wakeup_eventfd(); - if ((cqe->flags & IORING_CQE_F_MORE) == 0) - std::ignore = prep_multishot_poll(wakeup_eventfd_, nullptr); - } - else if (ud == &signal_pipe_sentinel_) - { - // Signal self-pipe readiness. Re-arm if the multishot - // terminated; the still-readable pipe re-fires on the next - // kernel enter so process_completions delivers the signal — - // we deliberately do NOT enqueue signal_drain_op_ from this - // teardown path. Not counted by uring_inflight_ (the poll - // was armed via prep_multishot_poll, which never increments), - // so it must NOT be decremented. - if ((cqe->flags & IORING_CQE_F_MORE) == 0) - std::ignore = prep_multishot_poll( - signal_pipe_read_fd_, &signal_pipe_sentinel_); - } - else if (ud == &cancel_sentinel_) - { - // ASYNC_CANCEL CQE (one-shot, no F_MORE), including the - // cancel SQE we submitted just above. Decrement inflight. - ++inflight_dec; - } - else if (ud == target) - { - saw_target = true; - // Don't dispatch — caller is destructing target; just - // consume so the CQE doesn't dangle. Decrement inflight on - // the terminal CQE only: the target is a multishot op - // whose intermediate CQEs carry F_MORE. - if ((cqe->flags & IORING_CQE_F_MORE) == 0) - ++inflight_dec; - } - else - { - // Some other op's CQE. Intentionally NOT dispatched: it - // may belong to an op freed by a sibling teardown (other - // acceptors / sockets), and dispatching would UAF. We - // still account for its terminal CQE so inflight stays - // balanced — the submit that produced it incremented the - // counter. The io_context's destructor sequence runs - // services' shutdowns before ~scheduler, so any still-live - // ops drain through their own paths first. - if ((cqe->flags & IORING_CQE_F_MORE) == 0) - ++inflight_dec; - } - ++consumed; - } - if (consumed) - { - io_uring_cq_advance(&ring_, consumed); - if (inflight_dec) - uring_inflight_.fetch_sub( - inflight_dec, std::memory_order_acq_rel); - if (saw_target) - break; - continue; - } - - // Nothing in the CQ — kick the kernel briefly. Hold - // ring_mutex_ across the wait so we don't race with the - // run-loop leader. - __kernel_timespec ts{0, static_cast(drain_cqes_kick_ns)}; - ::io_uring_cqe* one = nullptr; - int rc = - ::io_uring_submit_and_wait_timeout(&ring_, &one, 1, &ts, nullptr); - if (rc < 0 && rc != -ETIME && rc != -EINTR) - break; - if (rc == -ETIME) - break; - } -} - } // namespace boost::corosio::detail #endif // BOOST_COROSIO_HAS_URING diff --git a/include/boost/corosio/native/detail/uring/uring_socket_ops.hpp b/include/boost/corosio/native/detail/uring/uring_socket_ops.hpp index af5cc9ced..01c0a770b 100644 --- a/include/boost/corosio/native/detail/uring/uring_socket_ops.hpp +++ b/include/boost/corosio/native/detail/uring/uring_socket_ops.hpp @@ -129,7 +129,7 @@ struct uring_read_op : uring_op std::size_t* bytes, int file_descriptor, uring_scheduler* scheduler, - std::shared_ptr impl, + detail::object_ref impl, detail::speculative_state* spec, buffer_param buffers, std::stop_token const& token) noexcept @@ -140,7 +140,7 @@ struct uring_read_op : uring_op bytes_out = bytes; fd = file_descriptor; sched_ = scheduler; - impl_ptr = std::move(impl); + object_ref_ = std::move(impl); spec_state = spec; res = 0; cqe_flags = 0; @@ -227,7 +227,7 @@ struct uring_write_op : uring_op std::size_t* bytes, int file_descriptor, uring_scheduler* scheduler, - std::shared_ptr impl, + detail::object_ref impl, detail::speculative_state* spec, buffer_param buffers, std::stop_token const& token) noexcept @@ -238,7 +238,7 @@ struct uring_write_op : uring_op bytes_out = bytes; fd = file_descriptor; sched_ = scheduler; - impl_ptr = std::move(impl); + object_ref_ = std::move(impl); spec_state = spec; res = 0; cqe_flags = 0; @@ -336,7 +336,7 @@ struct uring_connect_op : uring_op std::error_code* ec, int file_descriptor, uring_scheduler* scheduler, - std::shared_ptr impl, + detail::object_ref impl, endpoint target, endpoint* remote_out, endpoint* local_out, @@ -348,7 +348,7 @@ struct uring_connect_op : uring_op bytes_out = nullptr; fd = file_descriptor; sched_ = scheduler; - impl_ptr = std::move(impl); + object_ref_ = std::move(impl); res = 0; cqe_flags = 0; target_endpoint = target; @@ -390,8 +390,9 @@ struct uring_connect_op : uring_op uring_set_result(self, false, false); - // Write endpoints only on success. - if (self->res >= 0) + // Write endpoints only on success; a close cancels the op. + if (self->res >= 0 && + !self->cancelled.load(std::memory_order_acquire)) { if (self->remote_endpoint_out) *self->remote_endpoint_out = self->target_endpoint; @@ -410,27 +411,60 @@ struct uring_connect_op : uring_op } }; -/** Submit an `uring_op`, reporting an SQ that stayed full. +/** Prepare @p sqe for @p op and count it in flight. - The body behind @ref uring_submit_op and - @ref uring_try_submit_op; `counted` selects which of the two - answers an exhausted SQ ring gets. + @pre The caller holds the ring mutex. + + @return True when this submitter must post the scheduler's + `submit_sqes_op` to flush the batch. +*/ +inline bool +uring_link_sqe( + uring_scheduler& sched, uring_op* op, ::io_uring_sqe* sqe) noexcept +{ + op->prep_func(op, sqe); + ::io_uring_sqe_set_data(sqe, op); + // Count this op against the in-flight gate in do_one: it + // expects exactly one F_MORE-less CQE per submitted SQE + // (multishot ops decrement only on the terminal CQE). + sched.inflight_inc(); + // Release pairs with the acquire in uring_op::request_cancel: + // a stop_token firing after we release the mutex will see + // sqe_set==true and submit a cancel-by-user_data SQE. + op->sqe_set.store(true, std::memory_order_release); + // First submitter in a batch wins the CAS and will post + // submit_sqes_op; others piggyback on the same flush. + return !sched.submit_op_posted_exchange(true); +} + +/** Submit an `uring_op` a `work_started()` already paid for. + + Acquires the ring mutex, prepares the SQE, and (under the same + mutex) CAS-sets `submit_op_posted_`. The first submitter of a + batch wins the CAS and posts the scheduler's `submit_sqes_op`, + which later flushes all queued SQEs in a single + `io_uring_submit_and_get_events` call and drains any ready CQEs. + Subsequent submitters in the same batch piggyback — their SQEs + sit in the user-space SQ ring until that op dispatches. + + On SQ-ring exhaustion (after one flush retry), completes the op + with `EAGAIN` and queues it so its handler dispatches on the next + `do_one` cycle, exactly as if the kernel had returned that error. @pre `op->prep_func != nullptr`. + @pre A `work_started()` backs this op, so the `work_finished()` the + scheduler spends on everything it dispatches is owed. @par Exception Safety Nothrow. @param sched The scheduler owning the ring. @param op The operation to submit. - @param counted True when a `work_started()` backs this op, so its - completion may ride the scheduler's queue. - @return `false` when the SQ stayed full and the op was left for the - caller to report; `true` otherwise. + @see uring_try_submit_op_if */ -inline bool -uring_do_submit_op(uring_scheduler& sched, uring_op* op, bool counted) noexcept +inline void +uring_submit_op(uring_scheduler& sched, uring_op* op) noexcept { sched.lazy_init_ring(); @@ -458,34 +492,12 @@ uring_do_submit_op(uring_scheduler& sched, uring_op* op, bool counted) noexcept // work_started() pays for the work_finished() do_one spends // on it. (CAS path is not entered here.) op->res = -EAGAIN; - if (!counted) - { - // Nothing counted this op, so queueing it would spend a - // work_finished() the context never owed and drive - // outstanding_work_ below what is really outstanding. - // The owner reports the failure instead. - return false; - } typename uring_scheduler::lock_type lock(sched.dispatch_mutex()); sched.push_completed_locked(op); - return true; + return; } - op->prep_func(op, sqe); - ::io_uring_sqe_set_data(sqe, op); - // Count this op against the in-flight gate in do_one: it - // expects exactly one F_MORE-less CQE per submitted SQE - // (multishot ops decrement only on the terminal CQE). - sched.inflight_inc(); - // Release pairs with the acquire in uring_op::request_cancel: - // a stop_token firing after we release the mutex will see - // sqe_set==true and submit a cancel-by-user_data SQE. - op->sqe_set.store(true, std::memory_order_release); - - // First submitter in a batch wins the CAS and will post - // submit_sqes_op; others piggyback on the same flush. - if (!sched.submit_op_posted_exchange(true)) - need_post = true; + need_post = uring_link_sqe(sched, op, sqe); } if (need_post) @@ -493,72 +505,69 @@ uring_do_submit_op(uring_scheduler& sched, uring_op* op, bool counted) noexcept // Flush is deferred to submit_sqes_op; post() owns the wake. sched.post(&sched.submit_op_ref()); } - return true; } -/** Submit an `uring_op` a `work_started()` already paid for. +/// Outcome of @ref uring_try_submit_op_if. +enum class uring_guarded_submit +{ + /// The SQE was prepared and will be flushed with the batch. + submitted, + /// `admit` declined; nothing was prepared. + refused, + /// `admit` accepted but the SQ stayed full after one flush. + sq_full +}; - Acquires the ring mutex, prepares the SQE, and (under the same - mutex) CAS-sets `submit_op_posted_`. The first submitter of a - batch wins the CAS and posts the scheduler's `submit_sqes_op`, - which later flushes all queued SQEs in a single - `io_uring_submit_and_get_events` call and drains any ready CQEs. - Subsequent submitters in the same batch piggyback — their SQEs - sit in the user-space SQ ring until that op dispatches. +/** Submit an uncounted op if @p admit allows it. - On SQ-ring exhaustion (after one flush retry), completes the op - with `EAGAIN` and queues it so its handler dispatches on the next - `do_one` cycle, exactly as if the kernel had returned that error. - That is why this spelling reports nothing: the failure reaches the - caller as the operation's own `EAGAIN` completion. - - @pre `op->prep_func != nullptr`. - @pre A `work_started()` backs this op, so the `work_finished()` the - scheduler spends on everything it dispatches is owed. + `admit` runs under the ring mutex before an SQE is taken. A cancel + flushed under that mutex (`cancel_and_flush`) therefore either + runs before the check, which then sees the owner's closing state, + or follows this op's SQE in the submission queue and cancels it. - @par Exception Safety - Nothrow. + @pre `op->prep_func != nullptr`. No `work_started()` backs this + submission, so a full SQ is reported to the caller rather than + queued as the op's completion. @param sched The scheduler owning the ring. @param op The operation to submit. + @param admit Called as `admit()` under the ring mutex; returns + whether to submit. May take a lock ordered after the ring + mutex. On `sq_full` it has already returned true, so the caller + undoes whatever it recorded. - @see uring_try_submit_op + @return What happened to the op. */ -inline void -uring_submit_op(uring_scheduler& sched, uring_op* op) noexcept +template +[[nodiscard]] uring_guarded_submit +uring_try_submit_op_if( + uring_scheduler& sched, uring_op* op, Admit admit) noexcept { - // The result is true by construction: a counted op's SQ-full path - // queues the op and answers through its own completion. - uring_do_submit_op(sched, op, true); -} - -/** Submit an `uring_op` nothing counted, reporting a full SQ. - - The scheduler spends a `work_finished()` on everything it - dispatches out of `completed_ops_`, so an op no `work_started()` - backs cannot be completed through that queue: doing so drives - `outstanding_work_` below what is really outstanding. This spelling - hands an SQ that stayed full back to the owner instead, which - reports it through its own channel — the multishot accept arm - latches it and answers the accepts it can no longer serve. - - @pre `op->prep_func != nullptr`. - - @par Exception Safety - Nothrow. + sched.lazy_init_ring(); - @param sched The scheduler owning the ring. - @param op The operation to submit. + bool need_post = false; + { + typename uring_scheduler::lock_type ring_lock(sched.ring_mutex()); + if (!admit()) + return uring_guarded_submit::refused; - @return True when the SQE was prepared; false when the SQ stayed - full after one flush and the caller owns the failure. + ::io_uring_sqe* sqe = ::io_uring_get_sqe(sched.ring()); + if (!sqe) + { + ::io_uring_submit(sched.ring()); + sqe = ::io_uring_get_sqe(sched.ring()); + } + if (!sqe) + { + op->res = -EAGAIN; + return uring_guarded_submit::sq_full; + } + need_post = uring_link_sqe(sched, op, sqe); + } - @see uring_submit_op -*/ -[[nodiscard]] inline bool -uring_try_submit_op(uring_scheduler& sched, uring_op* op) noexcept -{ - return uring_do_submit_op(sched, op, false); + if (need_post) + sched.post(&sched.submit_op_ref()); + return uring_guarded_submit::submitted; } /** Readiness wait via `IORING_OP_POLL_ADD`. @@ -586,20 +595,20 @@ struct uring_wait_op : uring_op std::error_code* ec, int file_descriptor, uring_scheduler* scheduler, - std::shared_ptr impl, + detail::object_ref impl, int flags, std::stop_token const& token) noexcept { - h = handle; - ex = executor; - ec_out = ec; - bytes_out = nullptr; - fd = file_descriptor; - sched_ = scheduler; - impl_ptr = std::move(impl); - poll_flags = flags; - res = 0; - cqe_flags = 0; + h = handle; + ex = executor; + ec_out = ec; + bytes_out = nullptr; + fd = file_descriptor; + sched_ = scheduler; + object_ref_ = std::move(impl); + poll_flags = flags; + res = 0; + cqe_flags = 0; start(token); } @@ -711,7 +720,7 @@ struct uring_local_connect_op : uring_op std::error_code* ec, int file_descriptor, uring_scheduler* scheduler, - std::shared_ptr impl, + detail::object_ref impl, corosio::local_endpoint target, corosio::local_endpoint* remote_out, corosio::local_endpoint* local_out, @@ -723,7 +732,7 @@ struct uring_local_connect_op : uring_op bytes_out = nullptr; fd = file_descriptor; sched_ = scheduler; - impl_ptr = std::move(impl); + object_ref_ = std::move(impl); res = 0; cqe_flags = 0; target_endpoint = target; @@ -764,8 +773,9 @@ struct uring_local_connect_op : uring_op uring_set_result(self, false, false); - // Write endpoints only on success. - if (self->res >= 0) + // Write endpoints only on success; a close cancels the op. + if (self->res >= 0 && + !self->cancelled.load(std::memory_order_acquire)) { if (self->remote_endpoint_out) *self->remote_endpoint_out = self->target_endpoint; diff --git a/include/boost/corosio/native/detail/uring/uring_socket_service_base.hpp b/include/boost/corosio/native/detail/uring/uring_socket_service_base.hpp index bf510575e..be8d30417 100644 --- a/include/boost/corosio/native/detail/uring/uring_socket_service_base.hpp +++ b/include/boost/corosio/native/detail/uring/uring_socket_service_base.hpp @@ -14,13 +14,12 @@ #if BOOST_COROSIO_HAS_URING +#include +#include #include #include #include -#include -#include -#include #include /* @@ -28,23 +27,23 @@ construct / destroy / shutdown / close / scheduler() are identical across uring_tcp_service, uring_udp_service, uring_local_stream_service, - and uring_local_datagram_service — they all make_shared the impl, track - it in a raw->shared_ptr map, cancel on shutdown, and close eagerly. This + and uring_local_datagram_service — they all pop-or-new the impl from a + per-service recycling pool, cancel on shutdown, and close eagerly. This base factors that out; the concrete services add only the protocol- specific open/bind/adopt. This is io_uring's own service base rather than a reuse of - reactor_socket_service: io_uring tracks impls in a map (the reactor uses - an intrusive list + map), cancels (not closes) on shutdown, and constructs - impls with a (service&, scheduler&) ctor. Reusing the reactor template - would force io_uring sockets to adopt the intrusive node, a close-on- - shutdown behavior change, and an Impl(Derived&) ctor — high churn and a - teardown behavior change for marginal extra sharing. See + reactor_socket_service: io_uring cancels (not closes) on shutdown and + constructs impls with a (service&, scheduler&) ctor, vs. the reactor's + close-on-shutdown and Impl(Derived&) ctor. Reusing the reactor template + would force a teardown behavior change for marginal extra sharing. See tasks/proactor-dedup-decisions.md (#13). Requirements on Socket: a `(Derived& service, uring_scheduler& sched)` - constructor and a `void close_socket() noexcept` method (cancel in-flight - ops + close fd + reset cached endpoints). + constructor, a `void close_socket() noexcept` method (cancel in-flight + ops + close fd + reset cached endpoints), a `void reuse() noexcept` + method, and derivation from `intrusive_list::node` (the pool's + live/free linkage). @tparam Derived The concrete service (CRTP, passed to the Socket ctor). @tparam ServiceBase The abstract service vtable base (tcp_service, ...). @@ -57,6 +56,7 @@ template class uring_socket_service_base : public ServiceBase { friend Derived; + friend Socket; // Private CRTP ctor: only `Derived` (the concrete service, a friend) // constructs the base — prevents inheriting with the wrong Derived @@ -71,41 +71,42 @@ class uring_socket_service_base : public ServiceBase void shutdown() override { - // Snapshot live impls, then cancel without the lock held to avoid - // inversion if cancel() ever re-enters the service. Impls stay owned - // by impls_ until ~service (after the scheduler drains its queue), - // keeping every impl alive while its cancel CQEs are processed. - std::vector> live; + // Snapshot live impls under an acquired reference, then cancel + // without the pool lock held to avoid inversion if cancel() ever + // re-enters the service. shutdown() sets shutting-down and + // takes the snapshot under one critical section, so each + // cancel's own release (if it drops the last ref) deletes + // rather than recycles. In-flight ops hold their own + // references, so every impl stays alive while its cancel CQEs + // are processed. + std::vector live; + pool_.shutdown( + [&](Socket* s) + { + acquire(s); + live.push_back(s); + }); + for (auto* s : live) { - std::lock_guard lk(mutex_); - live.reserve(impls_.size()); - for (auto& [_, p] : impls_) - live.push_back(p); + s->cancel(); + release(s); } - for (auto& p : live) - p->cancel(); } io_object::implementation* construct() override { - auto p = - std::make_shared(static_cast(*this), *sched_); - auto* raw = p.get(); - std::lock_guard lk(mutex_); - impls_.emplace(raw, std::move(p)); - return raw; + return acquire_impl(); } void destroy(io_object::implementation* p) override { if (!p) return; - std::lock_guard lk(mutex_); - impls_.erase(static_cast(p)); + release(static_cast(p)); } // Close the fd eagerly when the public close() is called, before - // destroy() drops the shared_ptr and the destructor runs. + // destroy() drops the service's reference and recycling runs. void close(io_object::handle& h) override { if (auto* sock = static_cast(h.get())) @@ -119,19 +120,18 @@ class uring_socket_service_base : public ServiceBase } protected: - /// Register an externally-built impl (used by adopt_fd on stream - /// services after accept(2)). Returns the raw pointer. - Socket* register_impl(std::shared_ptr p) + /** Pop a recycled impl or news one, with the service reference held. + + Used by both `construct()` and the `adopt_fd` accepted-connection + path (which additionally calls `assign_fd()` on the result). + */ + Socket* acquire_impl() { - auto* raw = p.get(); - std::lock_guard lk(mutex_); - impls_.emplace(raw, std::move(p)); - return raw; + return pool_.acquire(static_cast(*this), *sched_); } uring_scheduler* sched_; - std::mutex mutex_; - std::unordered_map> impls_; + object_pool pool_; private: uring_socket_service_base(uring_socket_service_base const&) = delete; diff --git a/include/boost/corosio/native/detail/uring/uring_stream_file.hpp b/include/boost/corosio/native/detail/uring/uring_stream_file.hpp index 978cee54d..c0cf1bbd1 100644 --- a/include/boost/corosio/native/detail/uring/uring_stream_file.hpp +++ b/include/boost/corosio/native/detail/uring/uring_stream_file.hpp @@ -16,6 +16,7 @@ #if BOOST_COROSIO_HAS_URING #include +#include #include #include #include @@ -65,13 +66,13 @@ class uring_stream_file_service; */ class BOOST_COROSIO_DECL uring_stream_file final : public stream_file::implementation - , public std::enable_shared_from_this , public intrusive_list::node { friend class uring_stream_file_service; - int fd_ = -1; - uring_scheduler* sched_ = nullptr; + int fd_ = -1; + uring_scheduler* sched_ = nullptr; + uring_stream_file_service* svc_ = nullptr; // Per-fd op slots — embedded to eliminate per-call heap allocation. // Single-pending invariant per slot. @@ -79,7 +80,10 @@ class BOOST_COROSIO_DECL uring_stream_file final uring_file_write_op wr_; public: - explicit uring_stream_file(uring_scheduler& sched) noexcept : sched_(&sched) + explicit uring_stream_file( + uring_stream_file_service& svc, uring_scheduler& sched) noexcept + : sched_(&sched) + , svc_(&svc) { } @@ -88,6 +92,26 @@ class BOOST_COROSIO_DECL uring_stream_file final close_file(); } + /// Recycle into the owning service's pool. Defined out-of-line + /// after uring_stream_file_service for its complete type. + void retire() noexcept override; + + /** Reset op slots for recycling. + + `close_file()` already drove fd_ to its closed value before the + refcount reached zero; each op's own `prepare()` overwrites its + state before the next use, so only the `stop_cb` invariant is + worth asserting here. + + @pre refs_ == 0, fd closed, no op in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(fd_ == -1); + BOOST_COROSIO_ASSERT(!rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wr_.stop_cb); + } + // -- io_stream::implementation -- std::coroutine_handle<> read_some( @@ -263,7 +287,7 @@ uring_stream_file::read_some( std::size_t* bytes) { rd_.prepare( - h, ex, ec, bytes, fd_, /*file_offset=*/-1, sched_, shared_from_this(), + h, ex, ec, bytes, fd_, /*file_offset=*/-1, sched_, detail::object_ref(this), buffers, token); sched_->work_started(); @@ -298,7 +322,7 @@ uring_stream_file::write_some( std::size_t* bytes) { wr_.prepare( - h, ex, ec, bytes, fd_, /*file_offset=*/-1, sched_, shared_from_this(), + h, ex, ec, bytes, fd_, /*file_offset=*/-1, sched_, detail::object_ref(this), buffers, token); sched_->work_started(); @@ -358,6 +382,12 @@ class BOOST_COROSIO_DECL uring_stream_file_service final } }; +inline void +uring_stream_file::retire() noexcept +{ + svc_->pool_.recycle(this); +} + } // namespace boost::corosio::detail #endif // BOOST_COROSIO_HAS_URING diff --git a/include/boost/corosio/native/detail/uring/uring_types.hpp b/include/boost/corosio/native/detail/uring/uring_types.hpp index f17dc50f6..a70513a02 100644 --- a/include/boost/corosio/native/detail/uring/uring_types.hpp +++ b/include/boost/corosio/native/detail/uring/uring_types.hpp @@ -15,6 +15,7 @@ #if BOOST_COROSIO_HAS_URING +#include #include #include #include @@ -69,8 +70,8 @@ class uring_local_datagram_service; read, write, and connect operations are submitted to the kernel via `uring_submit_op` and complete through the ring's CQE path. - The object is always owned by a `shared_ptr` managed by the service. - In-flight ops hold an additional `shared_ptr` copy (`impl_ptr`) so + The service holds one intrusive reference for as long as the impl + is live; in-flight ops hold an additional `object_ref_` keepalive so the kernel's user-data pointer remains valid until the CQE arrives. @par Thread Safety @@ -83,12 +84,13 @@ class BOOST_COROSIO_DECL uring_tcp_socket final uring_tcp_socket, tcp_socket::implementation, endpoint> + , public intrusive_list::node { friend uring_tcp_service; int family_ = AF_UNSPEC; // cached at open_socket uring_scheduler* sched_ = nullptr; - [[maybe_unused]] uring_tcp_service* svc_ = nullptr; + uring_tcp_service* svc_ = nullptr; // fd_ and local_endpoint_ are provided by native_socket_base (the // readiness/completion-agnostic socket base shared with the reactor @@ -207,7 +209,7 @@ class BOOST_COROSIO_DECL uring_tcp_socket final return dispatch_coro(ex, rd_.cont); } rd_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, token); if (stop_now) rd_.cancelled.store(true, std::memory_order_release); @@ -222,7 +224,7 @@ class BOOST_COROSIO_DECL uring_tcp_socket final } rd_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, buffers, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, token); sched_->work_started(); if (rd_.cancelled.load(std::memory_order_acquire)) @@ -286,7 +288,7 @@ class BOOST_COROSIO_DECL uring_tcp_socket final return dispatch_coro(ex, wr_.cont); } wr_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, token); if (stop_now) wr_.cancelled.store(true, std::memory_order_release); @@ -301,7 +303,7 @@ class BOOST_COROSIO_DECL uring_tcp_socket final } wr_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, buffers, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, token); sched_->work_started(); if (wr_.cancelled.load(std::memory_order_acquire)) @@ -337,7 +339,7 @@ class BOOST_COROSIO_DECL uring_tcp_socket final } conn_.addrlen = to_sockaddr(ep, family_, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), ep, + h, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, &local_endpoint_, token); conn_.cancelled.store(true, std::memory_order_release); sched_->work_started(); @@ -352,7 +354,7 @@ class BOOST_COROSIO_DECL uring_tcp_socket final // a subsequent IORING_OP_CONNECT would see EALREADY — avoid. conn_.addrlen = to_sockaddr(ep, family_, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), ep, &remote_endpoint_, + h, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, &local_endpoint_, token); sched_->work_started(); if (conn_.cancelled.load(std::memory_order_acquire)) @@ -386,7 +388,7 @@ class BOOST_COROSIO_DECL uring_tcp_socket final break; } wait_op_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), poll_flags, token); + h, ex, ec, fd_, sched_, detail::object_ref(this), poll_flags, token); sched_->work_started(); if (wait_op_.cancelled.load(std::memory_order_acquire)) { @@ -436,6 +438,9 @@ class BOOST_COROSIO_DECL uring_tcp_socket final /// resolves before the fd number can be recycled. void close_socket() noexcept { + // A connect completing after this must not write endpoints into + // a closed impl that may be recycled. + conn_.request_cancel(); if (fd_ >= 0) { sched_->cancel_and_flush(fd_); @@ -448,6 +453,50 @@ class BOOST_COROSIO_DECL uring_tcp_socket final endpoint_state::unresolved, std::memory_order_release); } + /** Recycle into the owning service's pool at zero references. + + Defined out-of-line after `uring_tcp_service` — its body needs + the service's complete type to reach the private `pool_` + member (reachable because this impl is a friend), and this + non-template member would otherwise require it right here, + before the service class exists in the file. + */ + void retire() noexcept override; + + /** Reset op slots, cached endpoints, and the speculation hint for + recycling. + + `close_socket()` already closed fd_ before the refcount reached + zero; each op's own `prepare()` overwrites its iovec/addr scratch + before the next use. What is left: the endpoints and their + resolution state (a connect completing on another thread can + write them after close), `family_` + (cached by `open_socket()`, but never touched by the accepted- + connection `adopt_fd()` path, so a socket recycled through + `adopt_fd()` would otherwise keep the prior session's family), + the speculation hint (a per-connection performance heuristic + that must not leak into the next logical connection), and the + `stop_cb` invariant on every op slot. + + @pre refs_ == 0, fd closed, no op in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(fd_ == -1); + // A connect completing on another thread while the socket closed + // can still have written these. + local_endpoint_ = endpoint{}; + remote_endpoint_ = endpoint{}; + local_endpoint_state_.store( + endpoint_state::unresolved, std::memory_order_relaxed); + BOOST_COROSIO_ASSERT(!rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wr_.stop_cb); + BOOST_COROSIO_ASSERT(!conn_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_op_.stop_cb); + family_ = AF_UNSPEC; + spec_.reset(); + } + endpoint local_endpoint() const noexcept override { // Lazy resolution: only fire the getsockname syscall when @@ -484,9 +533,10 @@ class BOOST_COROSIO_DECL uring_tcp_socket final Satisfies the `tcp_service` interface so the generic `tcp_socket` front-end can call `open_socket` and `bind_socket` transparently. - Socket impls are reference-counted inside the service map; raw - pointers returned from `construct()` remain valid until `destroy()` - or `shutdown()` is called. + Socket impls live in the service's `object_pool`. `construct()` + hands out a pooled impl with one reference already held; `destroy()` + releases it, and `retire()` recycles it onto the free list + instead of freeing it. @par Thread Safety All public member functions are thread-safe. @@ -643,7 +693,7 @@ class BOOST_COROSIO_DECL uring_tcp_service final */ uring_tcp_socket* adopt_fd(int fd, endpoint const& peer) { - auto p = std::make_shared(*this, *sched_); + auto* p = this->acquire_impl(); p->fd_ = fd; p->remote_endpoint_ = peer; // Mark the local endpoint as authoritative-but-unresolved. @@ -654,10 +704,16 @@ class BOOST_COROSIO_DECL uring_tcp_service final uring_tcp_socket::endpoint_state::lazy_pending, std::memory_order_release); - return this->register_impl(std::move(p)); + return p; } }; +inline void +uring_tcp_socket::retire() noexcept +{ + svc_->pool_.recycle(this); +} + /** TCP acceptor implementation for io_uring. Inherits the multishot machinery (parked-fd queue, waiter queue, @@ -672,7 +728,8 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor final uring_tcp_acceptor, tcp_acceptor::implementation, endpoint, - uring_tcp_service> + uring_tcp_service, + uring_tcp_acceptor_service> { friend uring_tcp_acceptor_service; @@ -680,7 +737,8 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor final uring_tcp_acceptor, tcp_acceptor::implementation, endpoint, - uring_tcp_service>; + uring_tcp_service, + uring_tcp_acceptor_service>; // Readiness-wait slot. The multishot accept op delivers accepted // fds, but `wait()` reports raw poll readiness on the listening fd @@ -689,13 +747,23 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor final public: explicit uring_tcp_acceptor( - uring_tcp_acceptor_service&, + uring_tcp_acceptor_service& svc, uring_scheduler& sched, uring_tcp_service& peer_svc) noexcept - : base_type(sched, peer_svc) + : base_type(svc, sched, peer_svc) { } + /** Extend the base reset with this acceptor's own `wait_op_` slot. + + @pre refs_ == 0, no wait in flight (see base_type::reuse()). + */ + void reuse() noexcept + { + base_type::reuse(); + BOOST_COROSIO_ASSERT(!wait_op_.stop_cb); + } + std::coroutine_handle<> accept( std::coroutine_handle<> h, capy::executor_ref ex, @@ -718,8 +786,7 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor final // instead of parking a waiter no accept machinery will signal. if (this->fd_ < 0) { - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept-adjacent initiation path: OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); + auto* op = this->acquire_node(); op->h = h; op->ex = ex; op->ec_out = ec; @@ -739,8 +806,7 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor final // fail uniformly instead of never completing. if (w == wait_type::write) { - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept-adjacent initiation path: OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); + auto* op = this->acquire_node(); op->h = h; op->ex = ex; op->ec_out = ec; @@ -751,7 +817,7 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor final // Errors are not consumed by the accept machinery, so the // error wait still polls the descriptor. wait_op_.prepare( - h, ex, ec, this->fd_, this->sched_, this->shared_from_this(), + h, ex, ec, this->fd_, this->sched_, detail::object_ref(this), POLLPRI | POLLERR | POLLHUP, token); this->sched_->work_started(); if (wait_op_.cancelled.load(std::memory_order_acquire)) @@ -782,9 +848,10 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor final `tcp_acceptor` front-end can call `open_acceptor_socket`, `bind_acceptor`, and `listen_acceptor` transparently. - Acceptor impls are reference-counted inside the service map; raw - pointers returned from `construct()` remain valid until `destroy()` - or `shutdown()` is called. + Acceptor impls live in the service's `object_pool`. `construct()` + hands out a pooled impl with one reference already held; `destroy()` + releases it, and `retire()` recycles it onto the free list + instead of freeing it. @par Thread Safety All public member functions are thread-safe. @@ -792,6 +859,9 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor final class BOOST_COROSIO_DECL uring_tcp_acceptor_service final : public tcp_acceptor_service { + template + friend class uring_multishot_acceptor_base; + public: /// Identifies this service for `execution_context` lookup. using key_type = tcp_acceptor_service; @@ -809,60 +879,52 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor_service final void shutdown() override { - std::vector> live; + // See uring_socket_service_base::shutdown(): snapshot under an + // acquired reference, cancel without the pool lock held; + // shutdown() sets shutting-down and takes the snapshot in one + // critical section, so a cancel that drops the last ref deletes + // rather than recycles. + std::vector live; + pool_.shutdown( + [&](uring_tcp_acceptor* a) + { + acquire(a); + live.push_back(a); + }); + for (auto* a : live) { - std::lock_guard lk(mutex_); - live.reserve(impls_.size()); - for (auto& [_, p] : impls_) - live.push_back(p); + a->cancel(); + release(a); } - // Cancel without the lock held to avoid inversion if cancel() - // re-enters the service. - for (auto& p : live) - p->cancel(); } io_object::implementation* construct() override { - auto p = - std::make_shared(*this, *sched_, *peer_svc_); - auto* raw = p.get(); - std::lock_guard lk(mutex_); - impls_.emplace(raw, std::move(p)); - return raw; + return acquire_impl(); } void destroy(io_object::implementation* p) override { if (!p) return; - std::lock_guard lk(mutex_); - impls_.erase(static_cast(p)); + release(static_cast(p)); } // Close the fd eagerly when tcp_acceptor::close() is called, before - // destroy() drops the shared_ptr and the destructor runs. + // destroy() releases the service's reference and retire() may + // recycle this impl. void close(io_object::handle& h) override { auto* acc = static_cast(h.get()); if (acc && acc->fd_ >= 0) { - // Flush the cancel SQE before closing the fd so the kernel - // resolves the file from the fd number while it is still - // valid. drain_waiters_only avoids submitting cancel-by-fd - // a second time (cancel_and_flush already did it). - sched_->cancel_and_flush(acc->fd_); + // Disown the arming before the cancel, then flush the + // cancel while the fd still names the listener. acc->drain_waiters_only(); + sched_->cancel_and_flush(acc->fd_); ::close(acc->fd_); - acc->fd_ = -1; - - // Break the multi_op_ -> impl_ptr (shared_ptr) cycle - // start_multishot established. The acceptor destructor's - // drain_cqes_for(multi_op_.get()) is the safety net; here - // we just drop the cycle so the impl can be released when - // the user's last shared_ptr does. - if (acc->multi_op_) - acc->multi_op_->impl_ptr.reset(); + acc->fd_ = -1; + acc->local_endpoint_ = endpoint{}; } } @@ -919,10 +981,6 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor_service final if (auto ec = validate_socket_fd(nfd, SOCK_STREAM, true)) return ec; - // Unconditional: release_socket() also leaves the op in flight, - // and it clears fd_ before returning. - acc.retire_multishot(); - acc.adopt_listening_fd(nfd); acc.local_endpoint_ = endpoint{}; @@ -987,11 +1045,15 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor_service final } private: + /// Pop a recycled impl or news one, with the service reference held. + uring_tcp_acceptor* acquire_impl() + { + return pool_.acquire(*this, *sched_, *peer_svc_); + } + uring_scheduler* sched_; uring_tcp_service* peer_svc_; - std::mutex mutex_; - std::unordered_map> - impls_; + object_pool pool_; }; /** Unix domain stream socket implementation for io_uring. @@ -1001,8 +1063,8 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor_service final kernel via `uring_submit_op` and complete through the ring's CQE path. - The object is always owned by a `shared_ptr` managed by the service. - In-flight ops hold an additional `shared_ptr` copy (`impl_ptr`) so + The service holds one intrusive reference for as long as the impl + is live; in-flight ops hold an additional `object_ref_` keepalive so the kernel's user-data pointer remains valid until the CQE arrives. @par Thread Safety @@ -1015,11 +1077,12 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final uring_local_stream_socket, local_stream_socket::implementation, corosio::local_endpoint> + , public intrusive_list::node { friend uring_local_stream_service; - uring_scheduler* sched_ = nullptr; - [[maybe_unused]] uring_local_stream_service* svc_ = nullptr; + uring_scheduler* sched_ = nullptr; + uring_local_stream_service* svc_ = nullptr; // fd_ and local_endpoint_ live in native_socket_base, which also // provides native_handle/is_open/set_option/get_option/local_endpoint. @@ -1114,7 +1177,7 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final return dispatch_coro(ex, rd_.cont); } rd_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, token); if (stop_now) rd_.cancelled.store(true, std::memory_order_release); @@ -1129,7 +1192,7 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final } rd_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, buffers, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, token); sched_->work_started(); if (rd_.cancelled.load(std::memory_order_acquire)) @@ -1193,7 +1256,7 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final return dispatch_coro(ex, wr_.cont); } wr_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, token); if (stop_now) wr_.cancelled.store(true, std::memory_order_release); @@ -1208,7 +1271,7 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final } wr_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, buffers, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, token); sched_->work_started(); if (wr_.cancelled.load(std::memory_order_acquire)) @@ -1244,7 +1307,7 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final } conn_.addrlen = to_sockaddr(ep, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), ep, + h, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, &local_endpoint_, token); conn_.cancelled.store(true, std::memory_order_release); sched_->work_started(); @@ -1259,7 +1322,7 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final // a subsequent IORING_OP_CONNECT would see EALREADY — avoid. conn_.addrlen = to_sockaddr(ep, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), ep, &remote_endpoint_, + h, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, &local_endpoint_, token); sched_->work_started(); if (conn_.cancelled.load(std::memory_order_acquire)) @@ -1293,7 +1356,7 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final break; } wait_op_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), poll_flags, token); + h, ex, ec, fd_, sched_, detail::object_ref(this), poll_flags, token); sched_->work_started(); if (wait_op_.cancelled.load(std::memory_order_acquire)) { @@ -1340,6 +1403,9 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final /// Called by the service on close()/teardown. void close_socket() noexcept { + // A connect completing after this must not write endpoints into + // a closed impl that may be recycled. + conn_.request_cancel(); if (fd_ >= 0) { sched_->cancel_and_flush(fd_); @@ -1350,6 +1416,29 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final remote_endpoint_ = corosio::local_endpoint{}; } + /// Recycle into the owning service's pool. See + /// uring_tcp_socket::retire() for why this is out-of-line. + void retire() noexcept override; + + /** Reset op slots, cached endpoints, and the speculation hint for + recycling. See uring_tcp_socket::reuse() for the rationale. + + @pre refs_ == 0, fd closed, no op in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(fd_ == -1); + // A connect completing on another thread while the socket closed + // can still have written these. + local_endpoint_ = corosio::local_endpoint{}; + remote_endpoint_ = corosio::local_endpoint{}; + BOOST_COROSIO_ASSERT(!rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wr_.stop_cb); + BOOST_COROSIO_ASSERT(!conn_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_op_.stop_cb); + spec_.reset(); + } + corosio::local_endpoint remote_endpoint() const noexcept override { return remote_endpoint_; @@ -1363,9 +1452,10 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final generic `local_stream_socket` front-end can call `open_socket` and `assign_socket` transparently. - Socket impls are reference-counted inside the service map; raw - pointers returned from `construct()` remain valid until `destroy()` - or `shutdown()` is called. + Socket impls live in the service's `object_pool`. `construct()` + hands out a pooled impl with one reference already held; `destroy()` + releases it, and `retire()` recycles it onto the free list + instead of freeing it. @par Thread Safety All public member functions are thread-safe. @@ -1483,7 +1573,7 @@ class BOOST_COROSIO_DECL uring_local_stream_service final uring_local_stream_socket* adopt_fd(int fd, corosio::local_endpoint const& peer) { - auto p = std::make_shared(*this, *sched_); + auto* p = this->acquire_impl(); p->fd_ = fd; p->remote_endpoint_ = peer; @@ -1492,10 +1582,16 @@ class BOOST_COROSIO_DECL uring_local_stream_service final if (::getsockname(fd, reinterpret_cast(&local), &len) == 0) p->local_endpoint_ = sockaddr_to_local_endpoint(local, len); - return this->register_impl(std::move(p)); + return p; } }; +inline void +uring_local_stream_socket::retire() noexcept +{ + svc_->pool_.recycle(this); +} + /** Local-stream (Unix domain) acceptor for io_uring. Inherits all multishot machinery (parked-fd queue, waiter queue, @@ -1509,7 +1605,8 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor final uring_local_stream_acceptor, local_stream_acceptor::implementation, corosio::local_endpoint, - uring_local_stream_service> + uring_local_stream_service, + uring_local_stream_acceptor_service> { friend uring_local_stream_acceptor_service; @@ -1517,20 +1614,31 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor final uring_local_stream_acceptor, local_stream_acceptor::implementation, corosio::local_endpoint, - uring_local_stream_service>; + uring_local_stream_service, + uring_local_stream_acceptor_service>; // Readiness-wait slot. See uring_tcp_acceptor::wait_op_. uring_wait_op wait_op_; public: explicit uring_local_stream_acceptor( - uring_local_stream_acceptor_service&, + uring_local_stream_acceptor_service& svc, uring_scheduler& sched, uring_local_stream_service& peer_svc) noexcept - : base_type(sched, peer_svc) + : base_type(svc, sched, peer_svc) { } + /** Extend the base reset with this acceptor's own `wait_op_` slot. + + @pre refs_ == 0, no wait in flight (see base_type::reuse()). + */ + void reuse() noexcept + { + base_type::reuse(); + BOOST_COROSIO_ASSERT(!wait_op_.stop_cb); + } + std::coroutine_handle<> accept( std::coroutine_handle<> h, capy::executor_ref ex, @@ -1553,8 +1661,7 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor final // instead of parking a waiter no accept machinery will signal. if (this->fd_ < 0) { - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept-adjacent initiation path: OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); + auto* op = this->acquire_node(); op->h = h; op->ex = ex; op->ec_out = ec; @@ -1574,8 +1681,7 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor final // fail uniformly instead of never completing. if (w == wait_type::write) { - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept-adjacent initiation path: OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); + auto* op = this->acquire_node(); op->h = h; op->ex = ex; op->ec_out = ec; @@ -1586,7 +1692,7 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor final // Errors are not consumed by the accept machinery, so the // error wait still polls the descriptor. wait_op_.prepare( - h, ex, ec, this->fd_, this->sched_, this->shared_from_this(), + h, ex, ec, this->fd_, this->sched_, detail::object_ref(this), POLLPRI | POLLERR | POLLHUP, token); this->sched_->work_started(); if (wait_op_.cancelled.load(std::memory_order_acquire)) @@ -1618,9 +1724,10 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor final `open_acceptor_socket`, `bind_acceptor`, and `listen_acceptor` transparently. - Acceptor impls are reference-counted inside the service map; raw - pointers returned from `construct()` remain valid until `destroy()` - or `shutdown()` is called. + Acceptor impls live in the service's `object_pool`. `construct()` + hands out a pooled impl with one reference already held; `destroy()` + releases it, and `retire()` recycles it onto the free list + instead of freeing it. @par Thread Safety All public member functions are thread-safe. @@ -1628,6 +1735,9 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor final class BOOST_COROSIO_DECL uring_local_stream_acceptor_service final : public local_stream_acceptor_service { + template + friend class uring_multishot_acceptor_base; + public: /// Identifies this service for `execution_context` lookup. using key_type = local_stream_acceptor_service; @@ -1645,56 +1755,47 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor_service final void shutdown() override { - std::vector> live; + // See uring_tcp_acceptor_service::shutdown() for the ordering + // rationale. + std::vector live; + pool_.shutdown( + [&](uring_local_stream_acceptor* a) + { + acquire(a); + live.push_back(a); + }); + for (auto* a : live) { - std::lock_guard lk(mutex_); - live.reserve(impls_.size()); - for (auto& [_, p] : impls_) - live.push_back(p); + a->cancel(); + release(a); } - // Cancel without the lock held to avoid inversion if cancel() - // re-enters the service. - for (auto& p : live) - p->cancel(); } io_object::implementation* construct() override { - auto p = std::make_shared( - *this, *sched_, *peer_svc_); - auto* raw = p.get(); - std::lock_guard lk(mutex_); - impls_.emplace(raw, std::move(p)); - return raw; + return acquire_impl(); } void destroy(io_object::implementation* p) override { if (!p) return; - std::lock_guard lk(mutex_); - impls_.erase(static_cast(p)); + release(static_cast(p)); } // Close the fd eagerly when local_stream_acceptor::close() is called, - // before destroy() drops the shared_ptr and the destructor runs. + // before destroy() drops the service's reference and recycling runs. void close(io_object::handle& h) override { auto* acc = static_cast(h.get()); if (acc && acc->fd_ >= 0) { - // cancel_and_flush submits cancel-by-fd; drain_waiters_only - // drains queued waiters without re-submitting it. - sched_->cancel_and_flush(acc->fd_); + // See uring_tcp_acceptor_service::close. acc->drain_waiters_only(); + sched_->cancel_and_flush(acc->fd_); ::close(acc->fd_); - acc->fd_ = -1; - - // Break the multi_op_ -> impl_ptr (shared_ptr) cycle - // start_multishot established. See the symmetric comment - // in uring_tcp_acceptor_service::close. - if (acc->multi_op_) - acc->multi_op_->impl_ptr.reset(); + acc->fd_ = -1; + acc->local_endpoint_ = corosio::local_endpoint{}; } } @@ -1745,10 +1846,6 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor_service final if (auto ec = validate_socket_fd(nfd, SOCK_STREAM, false)) return ec; - // Unconditional: release_socket() also leaves the op in flight, - // and it clears fd_ before returning. - acc.retire_multishot(); - acc.adopt_listening_fd(nfd); acc.local_endpoint_ = corosio::local_endpoint{}; @@ -1814,13 +1911,15 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor_service final } private: + /// Pop a recycled impl or news one, with the service reference held. + uring_local_stream_acceptor* acquire_impl() + { + return pool_.acquire(*this, *sched_, *peer_svc_); + } + uring_scheduler* sched_; uring_local_stream_service* peer_svc_; - std::mutex mutex_; - std::unordered_map< - uring_local_stream_acceptor*, - std::shared_ptr> - impls_; + object_pool pool_; }; /** UDP socket implementation for io_uring. @@ -1830,8 +1929,8 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor_service final to the kernel via `uring_submit_op` and complete through the ring's CQE path. - The object is always owned by a `shared_ptr` managed by the service. - In-flight ops hold an additional `shared_ptr` copy (`impl_ptr`) so + The service holds one intrusive reference for as long as the impl + is live; in-flight ops hold an additional `object_ref_` keepalive so the kernel's user-data pointer remains valid until the CQE arrives. @par Thread Safety @@ -1844,12 +1943,13 @@ class BOOST_COROSIO_DECL uring_udp_socket final uring_udp_socket, udp_socket::implementation, corosio::endpoint> + , public intrusive_list::node { friend uring_udp_service; int family_ = AF_UNSPEC; // cached at open_socket uring_scheduler* sched_ = nullptr; - [[maybe_unused]] uring_udp_service* svc_ = nullptr; + uring_udp_service* svc_ = nullptr; // fd_ and local_endpoint_ live in native_socket_base, which also // provides native_handle/is_open/set_option/get_option/local_endpoint. @@ -1964,7 +2064,7 @@ class BOOST_COROSIO_DECL uring_udp_socket final } conn_.addrlen = to_sockaddr(ep, family_, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), ep, + h, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, &local_endpoint_, token); conn_.cancelled.store(true, std::memory_order_release); sched_->work_started(); @@ -1979,7 +2079,7 @@ class BOOST_COROSIO_DECL uring_udp_socket final // a prior speculative ::connect would leave EINPROGRESS → EALREADY. conn_.addrlen = to_sockaddr(ep, family_, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), ep, &remote_endpoint_, + h, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, &local_endpoint_, token); sched_->work_started(); if (conn_.cancelled.load(std::memory_order_acquire)) @@ -2013,7 +2113,7 @@ class BOOST_COROSIO_DECL uring_udp_socket final break; } wait_op_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), poll_flags, token); + h, ex, ec, fd_, sched_, detail::object_ref(this), poll_flags, token); sched_->work_started(); if (wait_op_.cancelled.load(std::memory_order_acquire)) { @@ -2059,6 +2159,9 @@ class BOOST_COROSIO_DECL uring_udp_socket final /// Called by the service on close()/teardown. void close_socket() noexcept { + // A connect completing after this must not write endpoints into + // a closed impl that may be recycled. + conn_.request_cancel(); if (fd_ >= 0) { sched_->cancel_and_flush(fd_); @@ -2069,6 +2172,35 @@ class BOOST_COROSIO_DECL uring_udp_socket final remote_endpoint_ = endpoint{}; } + /// Recycle into the owning service's pool. See + /// uring_tcp_socket::retire() for why this is out-of-line. + void retire() noexcept override; + + /** Reset op slots, cached endpoints, the speculation hint, and + `family_` for recycling. `family_` is set by `open_socket()`/ + `assign_socket()`, not by this reset, so resetting it to + `AF_UNSPEC` here is defensive — it guards any future entry + path that constructs a usable socket without going through + either of those, the same hazard `uring_tcp_socket::reuse()` + documents for its `adopt_fd()` path. + + @pre refs_ == 0, fd closed, no op in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(fd_ == -1); + // A connect completing on another thread while the socket closed + // can still have written these. + local_endpoint_ = endpoint{}; + remote_endpoint_ = endpoint{}; + BOOST_COROSIO_ASSERT(!conn_.stop_cb); + BOOST_COROSIO_ASSERT(!send_.stop_cb); + BOOST_COROSIO_ASSERT(!recv_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_op_.stop_cb); + family_ = AF_UNSPEC; + spec_.reset(); + } + endpoint remote_endpoint() const noexcept override { return remote_endpoint_; @@ -2136,7 +2268,7 @@ class BOOST_COROSIO_DECL uring_udp_socket final return dispatch_coro(ex, send_.cont); } send_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, dest_len, dest_storage, to_native_msg_flags(flags), token); if (stop_now) @@ -2152,7 +2284,7 @@ class BOOST_COROSIO_DECL uring_udp_socket final } send_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, buffers, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, dest_len, dest_storage, to_native_msg_flags(flags), token); sched_->work_started(); if (send_.cancelled.load(std::memory_order_acquire)) @@ -2230,7 +2362,7 @@ class BOOST_COROSIO_DECL uring_udp_socket final return dispatch_coro(ex, recv_.cont); } recv_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, source_out, want_source ? &write_ip_source : nullptr, to_native_msg_flags(flags), token); if (stop_now) @@ -2256,7 +2388,7 @@ class BOOST_COROSIO_DECL uring_udp_socket final } recv_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, buffers, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, source_out, want_source ? &write_ip_source : nullptr, to_native_msg_flags(flags), token); sched_->work_started(); @@ -2286,9 +2418,10 @@ class BOOST_COROSIO_DECL uring_udp_socket final front-end can call `open_datagram_socket` and `bind_datagram` transparently. - Socket impls are reference-counted inside the service map; raw - pointers returned from `construct()` remain valid until `destroy()` - or `shutdown()` is called. + Socket impls live in the service's `object_pool`. `construct()` + hands out a pooled impl with one reference already held; `destroy()` + releases it, and `retire()` recycles it onto the free list + instead of freeing it. @par Thread Safety All public member functions are thread-safe. @@ -2425,6 +2558,12 @@ class BOOST_COROSIO_DECL uring_udp_service final } }; +inline void +uring_udp_socket::retire() noexcept +{ + svc_->pool_.recycle(this); +} + /** Unix domain datagram socket implementation for io_uring. Implements `local_datagram_socket::implementation` using a proactor @@ -2432,8 +2571,8 @@ class BOOST_COROSIO_DECL uring_udp_service final submitted to the kernel via `uring_submit_op` and complete through the ring's CQE path. - The object is always owned by a `shared_ptr` managed by the service. - In-flight ops hold an additional `shared_ptr` copy (`impl_ptr`) so + The service holds one intrusive reference for as long as the impl + is live; in-flight ops hold an additional `object_ref_` keepalive so the kernel's user-data pointer remains valid until the CQE arrives. @par Thread Safety @@ -2446,11 +2585,12 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final uring_local_datagram_socket, local_datagram_socket::implementation, corosio::local_endpoint> + , public intrusive_list::node { friend uring_local_datagram_service; - uring_scheduler* sched_ = nullptr; - [[maybe_unused]] uring_local_datagram_service* svc_ = nullptr; + uring_scheduler* sched_ = nullptr; + uring_local_datagram_service* svc_ = nullptr; // fd_ and local_endpoint_ live in native_socket_base, which also // provides native_handle/is_open/set_option/get_option/local_endpoint. @@ -2565,7 +2705,7 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final } conn_.addrlen = to_sockaddr(ep, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), ep, + h, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, &local_endpoint_, token); conn_.cancelled.store(true, std::memory_order_release); sched_->work_started(); @@ -2580,7 +2720,7 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final // a prior speculative ::connect would leave EINPROGRESS → EALREADY. conn_.addrlen = to_sockaddr(ep, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), ep, &remote_endpoint_, + h, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, &local_endpoint_, token); sched_->work_started(); if (conn_.cancelled.load(std::memory_order_acquire)) @@ -2614,7 +2754,7 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final break; } wait_op_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), poll_flags, token); + h, ex, ec, fd_, sched_, detail::object_ref(this), poll_flags, token); sched_->work_started(); if (wait_op_.cancelled.load(std::memory_order_acquire)) { @@ -2661,6 +2801,9 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final /// Called by the service on close()/teardown. void close_socket() noexcept { + // A connect completing after this must not write endpoints into + // a closed impl that may be recycled. + conn_.request_cancel(); if (fd_ >= 0) { sched_->cancel_and_flush(fd_); @@ -2671,6 +2814,29 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final remote_endpoint_ = corosio::local_endpoint{}; } + /// Recycle into the owning service's pool. See + /// uring_tcp_socket::retire() for why this is out-of-line. + void retire() noexcept override; + + /** Reset op slots, cached endpoints, and the speculation hint for + recycling. See uring_tcp_socket::reuse() for the rationale. + + @pre refs_ == 0, fd closed, no op in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(fd_ == -1); + // A connect completing on another thread while the socket closed + // can still have written these. + local_endpoint_ = corosio::local_endpoint{}; + remote_endpoint_ = corosio::local_endpoint{}; + BOOST_COROSIO_ASSERT(!conn_.stop_cb); + BOOST_COROSIO_ASSERT(!send_.stop_cb); + BOOST_COROSIO_ASSERT(!recv_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_op_.stop_cb); + spec_.reset(); + } + corosio::local_endpoint remote_endpoint() const noexcept override { return remote_endpoint_; @@ -2757,7 +2923,7 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final return dispatch_coro(ex, send_.cont); } send_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, dest_len, dest_storage, to_native_msg_flags(flags), token); if (stop_now) @@ -2773,7 +2939,7 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final } send_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, buffers, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, dest_len, dest_storage, to_native_msg_flags(flags), token); sched_->work_started(); if (send_.cancelled.load(std::memory_order_acquire)) @@ -2852,7 +3018,7 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final return dispatch_coro(ex, recv_.cont); } recv_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, source_out, want_source ? &write_local_source : nullptr, to_native_msg_flags(flags), token); @@ -2879,7 +3045,7 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final } recv_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, buffers, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, source_out, want_source ? &write_local_source : nullptr, to_native_msg_flags(flags), token); sched_->work_started(); @@ -2909,9 +3075,10 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final generic `local_datagram_socket` front-end can call `open_socket` and `bind_socket` transparently. - Socket impls are reference-counted inside the service map; raw - pointers returned from `construct()` remain valid until `destroy()` - or `shutdown()` is called. + Socket impls live in the service's `object_pool`. `construct()` + hands out a pooled impl with one reference already held; `destroy()` + releases it, and `retire()` recycles it onto the free list + instead of freeing it. @par Thread Safety All public member functions are thread-safe. @@ -3041,6 +3208,12 @@ class BOOST_COROSIO_DECL uring_local_datagram_service final } }; +inline void +uring_local_datagram_socket::retire() noexcept +{ + svc_->pool_.recycle(this); +} + } // namespace boost::corosio::detail #endif // BOOST_COROSIO_HAS_URING diff --git a/include/boost/corosio/tcp_server.hpp b/include/boost/corosio/tcp_server.hpp index 494cde682..dfa7fea0d 100644 --- a/include/boost/corosio/tcp_server.hpp +++ b/include/boost/corosio/tcp_server.hpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -171,10 +172,16 @@ class BOOST_COROSIO_DECL tcp_server template struct launch_wrapper { - struct promise_type + // frame_alloc_mixin routes the frame through the thread-local + // recycling allocator, so a warmed per-connection launch does + // not hit the global allocator. + struct promise_type : capy::frame_alloc_mixin { Ex ex; // Executor stored directly in frame (outlives child tasks) capy::io_env env_; + /// Embedded post node: the start posts this instead of the + /// bare handle, which would heap-allocate a wrapper op. + capy::continuation cont_; // For regular coroutines: first arg is executor, second is stop token template @@ -526,15 +533,25 @@ class BOOST_COROSIO_DECL tcp_server } } guard{srv_, w}; - // Reset worker's stop source for this connection - w->stop_ = {}; - auto st = w->stop_.get_token(); + // A stop_source allocates shared state on construction; + // reuse the worker's across connections and replace it + // only once a stop has actually been delivered (the state + // is then latched for good). + if (w->stop_.stop_requested()) + w->stop_ = {}; + auto st = w->stop_.get_token(); auto wrapper = launch_coro{}(ex, st, srv_, std::move(task), w); - // Executor and stop token stored in promise via constructor - ex.post(std::exchange(wrapper.h, nullptr)); // Release before post + // Executor and stop token stored in promise via + // constructor. Post through the frame-embedded + // continuation — posting the bare handle would allocate a + // wrapper op. The frame stays suspended until the + // executor resumes it, so the node outlives the queue. + auto h = std::exchange(wrapper.h, nullptr); // Release before post + h.promise().cont_.h = h; + ex.post(h.promise().cont_); guard.w = nullptr; // Success - dismiss guard } }; @@ -686,6 +703,9 @@ class BOOST_COROSIO_DECL tcp_server the accept loop ends. - Requests stop on each active worker's stop token. - Workers observing their stop token should exit promptly. + - A worker reuses its stop token across connections until a stop + is requested. Work that keeps a connection's token past that + connection's end therefore also observes this stop. @par Postconditions The server accepts no new connections. Active workers continue diff --git a/src/corosio/src/tcp_socket.cpp b/src/corosio/src/tcp_socket.cpp index 0361b4f7b..b49865dea 100644 --- a/src/corosio/src/tcp_socket.cpp +++ b/src/corosio/src/tcp_socket.cpp @@ -61,10 +61,8 @@ tcp_socket::open_for_family(int family, int type, int protocol) noexcept { #if BOOST_COROSIO_HAS_IOCP auto& svc = static_cast(h_.service()); - auto& wrapper = static_cast(*h_.get()); - std::error_code ec = svc.open_socket( - *static_cast(wrapper).get_internal(), family, - type, protocol); + auto& sock = static_cast(*h_.get()); + std::error_code ec = svc.open_socket(sock, family, type, protocol); #else auto& svc = static_cast(h_.service()); std::error_code ec = svc.open_socket( @@ -81,9 +79,8 @@ tcp_socket::assign(native_handle_type fd) noexcept return make_error_code(error::already_open); #if BOOST_COROSIO_HAS_IOCP auto& svc = static_cast(h_.service()); - auto& wrapper = static_cast(*h_.get()); - std::error_code ec = svc.assign_socket( - *static_cast(wrapper).get_internal(), fd); + auto& sock = static_cast(*h_.get()); + std::error_code ec = svc.assign_socket(sock, fd); #else auto& svc = static_cast(h_.service()); std::error_code ec = svc.assign_socket( @@ -108,10 +105,9 @@ tcp_socket::bind(endpoint ep) noexcept if (!is_open()) return make_error_code(std::errc::bad_file_descriptor); #if BOOST_COROSIO_HAS_IOCP - auto& svc = static_cast(h_.service()); - auto& wrapper = static_cast(*h_.get()); - return svc.bind_socket( - *static_cast(wrapper).get_internal(), ep); + auto& svc = static_cast(h_.service()); + auto& sock = static_cast(*h_.get()); + return svc.bind_socket(sock, ep); #else auto& svc = static_cast(h_.service()); return svc.bind_socket( diff --git a/src/corosio/src/timer.cpp b/src/corosio/src/timer.cpp index d28e9389b..5612045f0 100644 --- a/src/corosio/src/timer.cpp +++ b/src/corosio/src/timer.cpp @@ -20,6 +20,15 @@ timer::timer(capy::execution_context& ctx) { } +// Not inline for the same reason as wait() below: retire() needs +// timer_service's complete type (to reach pool_), which timer.hpp only +// forward-declares. +void +timer::implementation::retire() noexcept +{ + svc_->pool_.recycle(this); +} + // Not inline: wait_awaitable::await_suspend (defined in timer.hpp) calls // this from translation units that may never include timer_service.hpp, // so this must be the one strong definition the linker can always find @@ -60,7 +69,23 @@ timer::implementation::publish(waiter_node& w) if (w.token_->stop_possible()) w.arm_stop_cb(); - svc_->insert_waiter(*this, &w); + // insert_waiter() grows the heap before publishing anything (see + // its own comment), so a throw out of it leaves w.impl_ null and + // the waiter never visible to the fire/cancel paths above -- but + // work_started() and arm_stop_cb() already ran and must be + // unwound here, or the escaping exception leaves a dangling + // stop-callback registration (UAF on a later cancel) and a work + // count that never balances. + try + { + svc_->insert_waiter(*this, &w); + } + catch (...) + { + w.reset_stop_cb(); + svc_->get_scheduler().work_finished(); + throw; + } return std::noop_coroutine(); } diff --git a/test/unit/Jamfile b/test/unit/Jamfile index 261697224..b5df005da 100644 --- a/test/unit/Jamfile +++ b/test/unit/Jamfile @@ -38,13 +38,20 @@ project boost/corosio/test/unit # applies the exclusion patterns to directory entries too and does not # descend into a match. Those targets need the hook translation unit # linked in, which only fault/Jamfile does. -for local f in [ glob-tree-ex . : *.cpp : openssl_stream.cpp wolfssl_stream.cpp cross_ssl_stream.cpp tls_stream.cpp tls_stream_stress.cpp iocp_shutdown.cpp iocp_error_map.cpp openssl_engine.cpp wolfssl_engine.cpp cross_engine.cpp fault ] +for local f in [ glob-tree-ex . : *.cpp : openssl_stream.cpp wolfssl_stream.cpp cross_ssl_stream.cpp tls_stream.cpp tls_stream_stress.cpp iocp_shutdown.cpp iocp_error_map.cpp openssl_engine.cpp wolfssl_engine.cpp cross_engine.cpp zero_alloc.cpp fault ] { run $(f) ; } build-project fault ; +# zero_alloc.cpp counts against alloc_counter.hpp's extern counters, whose +# one definition (plus the global operator new/delete override) lives in +# continuation_post.cpp. Unlike the CMake build, which links every test +# source into a single binary, each b2 `run` target is its own executable, +# so the extra translation unit must be named explicitly here. +run zero_alloc.cpp continuation_post.cpp ; + # IOCP-specific native tests (Windows host only; skip on other hosts to # avoid triggering a cross-compile attempt). if [ os.name ] = NT diff --git a/test/unit/alloc_counter.hpp b/test/unit/alloc_counter.hpp new file mode 100644 index 000000000..5b3f16ae7 --- /dev/null +++ b/test/unit/alloc_counter.hpp @@ -0,0 +1,85 @@ +// +// Copyright (c) 2026 Steve Gerbino +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// Official repository: https://github.com/cppalliance/corosio +// + +#ifndef BOOST_COROSIO_TEST_ALLOC_COUNTER_HPP +#define BOOST_COROSIO_TEST_ALLOC_COUNTER_HPP + +/* Shared global operator new/delete counting interposer. + + The replaceable global allocation functions are process-wide and + may be defined exactly once; continuation_post.cpp owns that one + definition (guarded against TSan, which replaces them itself). + Every other test that needs to assert an allocation count -- + zero_alloc.cpp included -- counts against these same extern + counters instead of layering on a second interposer, which would + be a link error. +*/ + +#include +#include +#include + +// TSan's runtime already replaces global operator new/delete, so a +// counting replacement cannot link under -fsanitize=thread (and the +// counts would reflect the sanitizer's allocator anyway). +#if defined(__SANITIZE_THREAD__) +#define COROSIO_TEST_HAS_TSAN 1 +#elif defined(__has_feature) +#if __has_feature(thread_sanitizer) +#define COROSIO_TEST_HAS_TSAN 1 +#endif +#endif + +namespace boost::corosio { + +#ifndef COROSIO_TEST_HAS_TSAN +/// True while allocations should be tallied into alloc_count. +extern std::atomic alloc_armed; +/// Running count of operator new calls while alloc_armed is set. +extern std::atomic alloc_count; + +/** Check that the counting interposer actually runs, loud-skipping if not. + + ASan and TSan are excluded at compile time, but Valgrind replaces + the global allocator at run time: the interposer never executes, + every count reads zero, and a gate would either fail spuriously + (a documented non-zero pin) or pass vacuously (a zero pin). The + probe tests the behavior rather than the tool, as the fault + harness's allocation probe does, so any runtime replacement is + caught; the skip names its reason on stderr, as the fault suites' + Valgrind skip does, so the leg passes with a visible cause. + + @param suite Name printed in the skip notice. + + @return `true` if allocations are observable and the gate may run. +*/ +inline bool +alloc_counter_is_live(char const* suite) noexcept +{ + // Called through a volatile pointer so the probe allocation + // cannot be elided. + void* (*volatile op_new)(std::size_t) = &::operator new; + alloc_count.store(0, std::memory_order_relaxed); + alloc_armed.store(true, std::memory_order_relaxed); + ::operator delete(op_new(1)); + alloc_armed.store(false, std::memory_order_relaxed); + if (alloc_count.load(std::memory_order_relaxed) != 0) + return true; + std::fprintf( + stderr, + "%s: the global allocator is replaced at run time (Valgrind?), " + "so allocation counts are unobservable; skipping this suite\n", + suite); + return false; +} +#endif + +} // namespace boost::corosio + +#endif diff --git a/test/unit/continuation_post.cpp b/test/unit/continuation_post.cpp index 10aff4586..017809d14 100644 --- a/test/unit/continuation_post.cpp +++ b/test/unit/continuation_post.cpp @@ -16,10 +16,12 @@ #include #include +#include #include #include #include +#include "alloc_counter.hpp" #include "context.hpp" #include "test_suite.hpp" @@ -53,32 +55,76 @@ bare_post_task(capy::continuation* cont, bool& ran) ran = true; } -// TSan's runtime already replaces global operator new/delete, so the -// counting replacements below cannot link under -fsanitize=thread (and -// the counts would reflect the sanitizer's allocator anyway). -#if defined(__SANITIZE_THREAD__) -#define COROSIO_TEST_HAS_TSAN 1 -#elif defined(__has_feature) -#if __has_feature(thread_sanitizer) -#define COROSIO_TEST_HAS_TSAN 1 -#endif -#endif +} // namespace +// Definitions for alloc_counter.hpp's extern declarations. Not in the +// anonymous namespace above: zero_alloc.cpp needs these by extern +// reference, and an unnamed namespace's members are unreachable from +// another translation unit even though formally external-linkage. #ifndef COROSIO_TEST_HAS_TSAN std::atomic alloc_armed{false}; std::atomic alloc_count{0}; -#endif + +namespace { + +inline void +count_alloc() noexcept +{ + if (alloc_armed.load(std::memory_order_relaxed)) + alloc_count.fetch_add(1, std::memory_order_relaxed); +} + +/* Over-aligned new/delete, implemented without a platform-specific + aligned allocator (aligned_alloc/posix_memalign/_aligned_malloc all + differ in availability or requirements): stash the raw malloc() + pointer just before the returned block so the matching delete can + recover it, the same trick a custom allocator uses. +*/ +void* +aligned_alloc_impl(std::size_t n, std::size_t align) +{ + if (align < alignof(void*)) + align = alignof(void*); + std::size_t total = (n ? n : 1) + align + sizeof(void*); + void* raw = std::malloc(total); + if (!raw) + return nullptr; + auto base = reinterpret_cast(raw) + sizeof(void*); + auto aligned = (base + align - 1) & ~(align - 1); + *reinterpret_cast(aligned - sizeof(void*)) = raw; + return reinterpret_cast(aligned); +} + +void +aligned_free_impl(void* p) noexcept +{ + if (!p) + return; + void* raw = *reinterpret_cast( + reinterpret_cast(p) - sizeof(void*)); + std::free(raw); +} } // namespace +#endif } // namespace boost::corosio #ifndef COROSIO_TEST_HAS_TSAN +// Plain forms void* operator new(std::size_t n) { - if (boost::corosio::alloc_armed.load(std::memory_order_relaxed)) - boost::corosio::alloc_count.fetch_add(1, std::memory_order_relaxed); + boost::corosio::count_alloc(); + if (void* p = std::malloc(n ? n : 1)) + return p; + throw std::bad_alloc{}; +} + +void* +operator new[](std::size_t n) +{ + boost::corosio::count_alloc(); if (void* p = std::malloc(n ? n : 1)) return p; throw std::bad_alloc{}; @@ -94,6 +140,111 @@ operator delete(void* p, std::size_t) noexcept { std::free(p); } +void +operator delete[](void* p) noexcept +{ + std::free(p); +} +void +operator delete[](void* p, std::size_t) noexcept +{ + std::free(p); +} + +// nothrow forms +void* +operator new(std::size_t n, std::nothrow_t const&) noexcept +{ + boost::corosio::count_alloc(); + return std::malloc(n ? n : 1); +} +void* +operator new[](std::size_t n, std::nothrow_t const&) noexcept +{ + boost::corosio::count_alloc(); + return std::malloc(n ? n : 1); +} +void +operator delete(void* p, std::nothrow_t const&) noexcept +{ + std::free(p); +} +void +operator delete[](void* p, std::nothrow_t const&) noexcept +{ + std::free(p); +} + +// Over-aligned forms (std::align_val_t) -- an aligned allocation on a +// measured path would otherwise bypass this interposer silently. +void* +operator new(std::size_t n, std::align_val_t align) +{ + boost::corosio::count_alloc(); + if (void* p = boost::corosio::aligned_alloc_impl( + n, static_cast(align))) + return p; + throw std::bad_alloc{}; +} +void* +operator new[](std::size_t n, std::align_val_t align) +{ + boost::corosio::count_alloc(); + if (void* p = boost::corosio::aligned_alloc_impl( + n, static_cast(align))) + return p; + throw std::bad_alloc{}; +} +void +operator delete(void* p, std::align_val_t) noexcept +{ + boost::corosio::aligned_free_impl(p); +} +void +operator delete(void* p, std::size_t, std::align_val_t) noexcept +{ + boost::corosio::aligned_free_impl(p); +} +void +operator delete[](void* p, std::align_val_t) noexcept +{ + boost::corosio::aligned_free_impl(p); +} +void +operator delete[](void* p, std::size_t, std::align_val_t) noexcept +{ + boost::corosio::aligned_free_impl(p); +} + +// Over-aligned nothrow forms +void* +operator new(std::size_t n, std::align_val_t align, std::nothrow_t const&) + noexcept +{ + boost::corosio::count_alloc(); + return boost::corosio::aligned_alloc_impl( + n, static_cast(align)); +} +void* +operator new[](std::size_t n, std::align_val_t align, std::nothrow_t const&) + noexcept +{ + boost::corosio::count_alloc(); + return boost::corosio::aligned_alloc_impl( + n, static_cast(align)); +} +void +operator delete( + void* p, std::align_val_t, std::nothrow_t const&) noexcept +{ + boost::corosio::aligned_free_impl(p); +} +void +operator delete[]( + void* p, std::align_val_t, std::nothrow_t const&) noexcept +{ + boost::corosio::aligned_free_impl(p); +} #endif namespace boost::corosio { @@ -150,6 +301,8 @@ struct continuation_zero_alloc_test void run() { + if (!alloc_counter_is_live("boost.corosio.continuation_post.zero_alloc")) + return; testPostIsZeroAlloc(); } }; diff --git a/test/unit/fault/fault_posix.cpp b/test/unit/fault/fault_posix.cpp index c4b193203..7b6d7dca9 100644 --- a/test/unit/fault/fault_posix.cpp +++ b/test/unit/fault/fault_posix.cpp @@ -61,6 +61,78 @@ #define O_TMPFILE 0 #endif +// Sanitizer runtimes intercept part of the census for their own +// bookkeeping (ASan's thread registry, TSan's happens-before graph). +// When these shadows are linked into the executable they can bind +// ahead of the runtime's interceptors (observed with clang >= 23's +// static ASan runtime: `ASAN_OPTIONS=verbosity=1` reports "failed to +// intercept 'pthread_create'", and the first pthread_join aborts with +// "Joining already joined thread" because the thread was created +// behind the runtime's back). Each sanitizer exports its interceptor +// as `__interceptor_` precisely so an interposer like this one +// can cooperate: chaining into it keeps the bookkeeping intact and +// still reaches libc. The references are weak, so a name the runtime +// does not intercept resolves to null and falls back to the plain +// RTLD_NEXT lookup; in unsanitized builds every reference is null. +#if defined(__linux__) +#if defined(__SANITIZE_ADDRESS__) || defined(__SANITIZE_THREAD__) +#define COROSIO_FAULT_SANITIZER 1 +#elif defined(__has_feature) +#if __has_feature(address_sanitizer) || __has_feature(thread_sanitizer) || \ + __has_feature(memory_sanitizer) +#define COROSIO_FAULT_SANITIZER 1 +#endif +#endif +#endif +#if !defined(COROSIO_FAULT_SANITIZER) +#define COROSIO_FAULT_SANITIZER 0 +#endif + +#if COROSIO_FAULT_SANITIZER + +// The libc half of the census (the SSL and liburing entries have no +// sanitizer interceptors). A name listed here that the runtime does +// not actually intercept costs nothing: its weak reference is null. +#define COROSIO_FAULT_INTERCEPTED_SYMBOLS(X) \ + X(socket) X(socketpair) X(bind) X(listen) X(accept) X(connect) \ + X(getsockname) X(getpeername) X(getsockopt) X(setsockopt) \ + X(shutdown) X(close) X(read) X(write) X(readv) X(writev) \ + X(pread64) X(preadv) \ + X(pwritev) X(recv) X(send) X(recvmsg) X(sendmsg) X(poll) X(pipe) \ + X(fcntl) X(ioctl) X(open) X(fstat) X(ftruncate) X(fsync) \ + X(unlink) X(sigaction) X(pthread_create) X(getaddrinfo) \ + X(freeaddrinfo) X(getnameinfo) X(gethostname) X(fdatasync) \ + X(posix_fadvise) X(select) X(accept4) X(lseek) X(epoll_create1) \ + X(epoll_ctl) X(epoll_wait) X(eventfd) X(timerfd_create) \ + X(timerfd_settime) X(__read_chk) X(__recv_chk) \ + X(__recvfrom_chk) X(__poll_chk) X(__pread64_chk) X(__open_2) \ + X(__gethostname_chk) + +// NOLINTNEXTLINE(bugprone-macro-parentheses) — token-paste operand cannot be parenthesized +#define COROSIO_FAULT_DECLARE_INTERCEPTOR(name) \ + extern "C" void __interceptor_##name() __attribute__((weak)); +COROSIO_FAULT_INTERCEPTED_SYMBOLS(COROSIO_FAULT_DECLARE_INTERCEPTOR) +#undef COROSIO_FAULT_DECLARE_INTERCEPTOR + +namespace { + +struct sanitizer_interceptor +{ + char const* name; + void (*fn)(); +}; + +sanitizer_interceptor const sanitizer_interceptors[] = { +// NOLINTNEXTLINE(bugprone-macro-parentheses) — token-paste operand cannot be parenthesized +#define COROSIO_FAULT_INTERCEPTOR_ENTRY(name) {#name, &__interceptor_##name}, + COROSIO_FAULT_INTERCEPTED_SYMBOLS(COROSIO_FAULT_INTERCEPTOR_ENTRY) +#undef COROSIO_FAULT_INTERCEPTOR_ENTRY +}; + +} // namespace + +#endif // COROSIO_FAULT_SANITIZER + namespace boost::corosio::test::fault { thread_local cqe_slot tls_cqe; @@ -76,6 +148,13 @@ publish_error(int err) noexcept void* real_symbol(char const* name) noexcept { +#if COROSIO_FAULT_SANITIZER + // Chain into the sanitizer's own interceptor when it has one for + // this name — see the block comment above the table. + for (auto const& e : sanitizer_interceptors) + if (e.fn != nullptr && std::strcmp(e.name, name) == 0) + return reinterpret_cast(e.fn); +#endif void* p = ::dlsym(RTLD_NEXT, name); if (!p) { diff --git a/test/unit/fault/fault_uring.cpp b/test/unit/fault/fault_uring.cpp index 95e0aa694..de692579c 100644 --- a/test/unit/fault/fault_uring.cpp +++ b/test/unit/fault/fault_uring.cpp @@ -12,7 +12,9 @@ #include #include #include +#include #include +#include // liburing 2.6 and later declare their API noexcept in C++; older // headers leave it unspecified, and a shadow that adds a specification @@ -128,10 +130,47 @@ scan_pending_sqes(io_uring* ring) noexcept } } +// A multishot CQE rewritten as terminal leaves the kernel arming live, +// which a real termination never does. corosio may free the op once +// it sees the fake terminal, so the arming's real terminal CQE, which +// would name that op, is turned into a wakeup CQE (null user_data, +// more to come) that corosio skips. Deliveries before it pass through, +// since a re-arm reuses the same user_data. Outlives the rewriting +// scope, as the arming can. +struct orphan_state +{ + std::uint64_t user_data = 0; + bool closes_fds = false; +}; + +orphan_state orphan; + +void +swallow_orphan_terminal(io_uring* ring) noexcept +{ + if (orphan.user_data == 0) + return; + unsigned head; + io_uring_cqe* cqe; + io_uring_for_each_cqe(ring, head, cqe) + { + if (cqe->user_data != orphan.user_data || + (cqe->flags & IORING_CQE_F_MORE) != 0) + continue; + if (orphan.closes_fds && cqe->res >= 0) + ::close(cqe->res); + cqe->user_data = 0; + cqe->flags |= IORING_CQE_F_MORE; + orphan = {}; + return; + } +} + // Overwrite `res` on the visible CQE carrying the recorded user_data. void rewrite_visible_cqes(io_uring* ring) noexcept { + swallow_orphan_terminal(ring); auto& c = tls_cqe; if (!c.armed || !c.have_user_data) return; @@ -141,6 +180,12 @@ rewrite_visible_cqes(io_uring* ring) noexcept { if (cqe->user_data == c.user_data) { + if ((c.flags_clear & IORING_CQE_F_MORE) && + (cqe->flags & IORING_CQE_F_MORE)) + { + orphan.user_data = c.user_data; + orphan.closes_fds = c.opcode == IORING_OP_ACCEPT; + } cqe->res = c.res; cqe->flags &= ~c.flags_clear; c.fired = true; diff --git a/test/unit/fault/iocp_faults.cpp b/test/unit/fault/iocp_faults.cpp index c22aa774d..6daedac88 100644 --- a/test/unit/fault/iocp_faults.cpp +++ b/test/unit/fault/iocp_faults.cpp @@ -1034,7 +1034,7 @@ struct iocp_faults through close_socket, which asks the reactor to drop any wait op the descriptor might have had, and cancel() asks again. Both asks reach wake_self whether or not anything is parked - (win_tcp_socket_internal::cancel and ::close_socket). + (win_tcp_socket::cancel and ::close_socket). */ void testWakeSendFails() { @@ -1470,7 +1470,7 @@ struct iocp_faults Only the write and error waits reach the reactor at all: a wait for readability is a zero-byte WSARecv - (win_tcp_socket_internal::wait). The peer is reset before + (win_tcp_socket::wait). The peer is reset before the wait is registered, so the reactor's first poll already has the condition in hand and nothing here depends on a round landing between two coroutines. @@ -1630,7 +1630,7 @@ struct iocp_faults the entry before it closes the handle, but the ask is read on the reactor's next pass and the handle is gone before then, so a poll carrying the dead descriptor is reachable in ordinary - use (win_tcp_socket_internal::close_socket says so). + use (win_tcp_socket::close_socket says so). Closed behind the library's back rather than through a fault_scope: the reactor answers the refusal by asking about diff --git a/test/unit/fault/uring_faults.cpp b/test/unit/fault/uring_faults.cpp index 5d242dec5..972d76bdd 100644 --- a/test/unit/fault/uring_faults.cpp +++ b/test/unit/fault/uring_faults.cpp @@ -275,10 +275,62 @@ struct uring_faults } } - // The acceptor's close path drains its multishot CQEs; a failing - // submit_and_wait_timeout breaks that loop silently, so the only - // observable is that the fault was reached. - void testAcceptorDrainSubmitFails() + // Parks a receive in the kernel, so the context's teardown has a + // request to cancel and wait for. + static void park_receive(io_context& ioc, udp_socket& s, char* buf) + { + BOOST_TEST(!s.open(family::v4)); + BOOST_TEST(!s.bind(endpoint(ipv4_address::loopback(), 0))); + capy::run_async(ioc.get_executor())( + [](udp_socket& s, char* b) -> capy::task<> { + endpoint from; + std::ignore = + co_await s.recv_from(capy::mutable_buffer(b, 8), from); + }(s, buf)); + ioc.poll(); + } + + // Teardown waits for the kernel to finish every request it holds; + // a signal interrupting that wait must not end it early, or the + // services free implementations the kernel can still write to. + void testShutdownWaitRetriesAfterEintr() + { + char buf[8]; + std::optional first; + std::optional again; + { + io_context ioc(uring); + udp_socket s(ioc); + park_receive(ioc, s, buf); + first.emplace(sys::io_uring_wait_cqe_timeout, EINTR); + again.emplace(sys::io_uring_wait_cqe_timeout, EINTR, 2); + } + BOOST_TEST(first->fired()); + BOOST_TEST(again->fired()); // the interrupted wait was retried + } + + // Same for the cancel that teardown submits before waiting. + void testShutdownCancelRetriesAfterEintr() + { + char buf[8]; + std::optional waits; + std::optional submit; + { + io_context ioc(uring); + udp_socket s(ioc); + park_receive(ioc, s, buf); + waits.emplace(sys::io_uring_wait_cqe_timeout, EINTR, 1000); + submit.emplace(sys::io_uring_submit, EINTR); + } + BOOST_TEST(submit->fired()); + BOOST_TEST_GE(waits->count(), 1u); + } + + // Closing and destroying a listening acceptor on a live context + // leaves the arming's terminal completion to the run loop. Waiting + // on the ring for it instead would swallow other ops' completions; + // the kernel wait is the observable and must not be reached. + void testAcceptorCloseDoesNotWaitOnRing() { io_context ioc(uring); fault_scope f(sys::io_uring_submit_and_wait_timeout, EBADF); @@ -296,7 +348,7 @@ struct uring_faults ioc.poll(); BOOST_TEST(aec == capy::error::canceled); } - BOOST_TEST(f.fired()); + BOOST_TEST(!f.fired()); } // A ring clamped to one SQE spends it on the wakeup poll, and the @@ -879,6 +931,63 @@ struct uring_faults BOOST_TEST(!peer.is_open()); } + // A multishot accept the kernel terminates on its own (F_MORE + // cleared, no fatal error) is re-armed, so later connections are + // still accepted. + void testMultishotRearmKeepsAccepting() + { + io_context ioc(uring); + auto ex = ioc.get_executor(); + + tcp_acceptor acc(ioc); + BOOST_TEST(!acc.open(family::v4)); + BOOST_TEST(!acc.bind(endpoint(ipv4_address::loopback(), 0))); + cqe_fault_scope q(-1, IORING_OP_ACCEPT, -EAGAIN, IORING_CQE_F_MORE); + BOOST_TEST(!acc.listen()); + + sockaddr_in sa{}; + sa.sin_family = AF_INET; + sa.sin_port = htons(acc.local_endpoint().port()); + sa.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + auto connect_raw = [&] { + int fd = ::socket(AF_INET, SOCK_STREAM, 0); + BOOST_TEST_EQ( + ::connect(fd, reinterpret_cast(&sa), sizeof(sa)), 0); + return fd; + }; + + // The first connection provokes the rewritten terminal CQE. + int c1 = connect_raw(); + for (int i = 0; i < 10000 && !q.fired(); ++i) + { + capy::run_async(ex)([]() -> capy::task<> { co_return; }()); + ioc.restart(); + ioc.poll(); + } + BOOST_TEST(q.fired()); + + int c2 = connect_raw(); + tcp_socket peer(ioc); + bool accepted = false; + capy::run_async(ex)( + [](tcp_acceptor& a, tcp_socket& p, bool& done) -> capy::task<> { + auto [ec] = co_await a.accept(p); + done = !ec; + }(acc, peer, accepted)); + ioc.restart(); + std::ignore = ioc.run_for(std::chrono::seconds(2)); + if (!accepted) + { + acc.close(); + ioc.restart(); + ioc.run(); + } + BOOST_TEST(accepted); + + ::close(c1); + ::close(c2); + } + // A multishot accept the kernel terminates (F_MORE cleared, no // fatal error) is re-armed; a re-arm that cannot get an SQE must // drain the parked waiters with the error rather than strand them. @@ -1029,6 +1138,7 @@ struct uring_faults testRingInitSqExhaustion(); testCancelSqFullBestEffort(); testMultishotArmFailure(); + testMultishotRearmKeepsAccepting(); testMultishotRearmFailureWithWaiter(); testWakeupPollRearm(); testSignalPipePollRearm(); @@ -1039,7 +1149,9 @@ struct uring_faults testAcceptorAssignProbeFails(); testCancelSqFull(); testWaitFails(); - testAcceptorDrainSubmitFails(); + testAcceptorCloseDoesNotWaitOnRing(); + testShutdownWaitRetriesAfterEintr(); + testShutdownCancelRetriesAfterEintr(); testAcceptorArmSqFull(); testSqFull(); testConnectCqeRewrite(); diff --git a/test/unit/mt_reactor.cpp b/test/unit/mt_reactor.cpp index 1929666fd..d7d1696c8 100644 --- a/test/unit/mt_reactor.cpp +++ b/test/unit/mt_reactor.cpp @@ -26,12 +26,15 @@ #include #include +#include +#include #include #include #include #include #include +#include #include #include @@ -40,6 +43,7 @@ #include #include #include +#include #include #include @@ -222,11 +226,221 @@ struct mt_reactor_test ioc.run(); } + // A reactor pass can hold an event for a descriptor that another run + // thread closes before the pass queues it. The closed socket's impl + // must outlive that pass, not be recycled under it. Each iteration + // makes a descriptor readable and destroys its socket while a second + // run thread polls. + void testCloseRacesReactorPass() + { + int lfd = ::socket(AF_INET, SOCK_STREAM, 0); + BOOST_TEST(lfd >= 0); + int one = 1; + ::setsockopt(lfd, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)); + sockaddr_in addr{}; + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + socklen_t len = sizeof(addr); + BOOST_TEST( + ::bind(lfd, reinterpret_cast(&addr), sizeof(addr)) == + 0); + BOOST_TEST( + ::getsockname(lfd, reinterpret_cast(&addr), &len) == 0); + BOOST_TEST(::listen(lfd, 64) == 0); + + io_context ioc(Backend, 2); + int completed = 0; + capy::run_async(ioc.get_executor())( + [](io_context& c, int listener, sockaddr_in a, + int& done) -> capy::task<> { + for (int i = 0; i < 2000; ++i) + { + int peer = ::socket(AF_INET, SOCK_STREAM, 0); + if (::connect( + peer, reinterpret_cast(&a), + sizeof(a)) != 0) + { + ::close(peer); + co_return; + } + int fd = ::accept(listener, nullptr, nullptr); + { + tcp_socket s(c); + std::ignore = s.assign(fd); + char const b = 'x'; + std::ignore = ::send(peer, &b, 1, 0); + } + ::close(peer); + ++done; + } + }(ioc, lfd, addr, completed)); + + std::thread second([&] { ioc.run(); }); + ioc.run(); + second.join(); + ::close(lfd); + BOOST_TEST_EQ(completed, 2000); + } + + // An op started from a thread that is not running the context posts + // its completion through the scheduler lock. It must not do so while + // holding the descriptor lock, which the reactor takes after the + // scheduler lock when settling a descriptor closed during its pass. + // A stop racing the start makes it complete without parking, taking + // that posting path while the reactor settles the previous + // incarnation's close. + void testForeignStartRacesSettle() + { + int lfd = ::socket(AF_INET, SOCK_STREAM, 0); + BOOST_TEST(lfd >= 0); + sockaddr_in addr{}; + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + socklen_t len = sizeof(addr); + BOOST_TEST( + ::bind(lfd, reinterpret_cast(&addr), sizeof(addr)) == + 0); + BOOST_TEST( + ::getsockname(lfd, reinterpret_cast(&addr), &len) == 0); + BOOST_TEST(::listen(lfd, 64) == 0); + auto make_pair = [&](int& mine, int& peer) { + peer = ::socket(AF_INET, SOCK_STREAM, 0); + std::ignore = ::connect( + peer, reinterpret_cast(&addr), sizeof(addr)); + mine = ::accept(lfd, nullptr, nullptr); + }; + + io_context ioc(Backend, 2); + capy::thread_pool pool(1); + + // A parked wait keeps run() polling until the end. + int idle_fd = -1; + int idle_peer = -1; + make_pair(idle_fd, idle_peer); + tcp_socket idle(ioc); + BOOST_TEST(!idle.assign(idle_fd)); + capy::run_async(ioc.get_executor())( + [](tcp_socket& i) -> capy::task<> { + std::ignore = co_await i.wait(wait_type::read); + }(idle)); + std::thread runner([&] { ioc.run(); }); + + tcp_socket s(ioc); + int peer = -1; + std::atomic finished{0}; + for (int i = 0; i < 500; ++i) + { + if (peer >= 0) + ::close(peer); + s.close(); + int fd = -1; + make_pair(fd, peer); + BOOST_TEST(!s.assign(fd)); + std::stop_source stop; + capy::run_async(pool.get_executor(), stop.get_token())( + [](tcp_socket& x, std::atomic& n) -> capy::task<> { + std::ignore = co_await x.wait(wait_type::read); + ++n; + }(s, finished)); + stop.request_stop(); + while (finished.load() == i) + std::this_thread::yield(); + } + + char const b = 'x'; + std::ignore = ::send(idle_peer, &b, 1, 0); + runner.join(); + pool.join(); + ::close(peer); + ::close(idle_peer); + ::close(lfd); + BOOST_TEST_EQ(finished.load(), 500); + } + + // An invocation popped from the queue clears the descriptor's + // queued flag inside its critical section and keeps using the state + // until it unlocks. Retiring the descriptor meanwhile must not take + // that cleared flag to mean nothing still runs on it. + void testRetireWaitsForRunningInvocation() + { + struct watched_impl : io_object::implementation + { + std::atomic* inside; + std::atomic* retired; + std::atomic* retired_inside; + + void retire() noexcept override + { + if (inside->load()) + retired_inside->store(true); + retired->store(true); + } + }; + + // Stays inside the invocation's critical section until the + // descriptor is retired, or long enough to show it was not. + struct holding_op final : detail::reactor_op_base + { + std::atomic* inside; + std::atomic* retired; + + void perform_io() noexcept override + { + inside->store(true); + // Bounds the wait, not a timing assertion. + auto const until = std::chrono::steady_clock::now() + + std::chrono::milliseconds(200); + while (!retired->load() && + std::chrono::steady_clock::now() < until) + std::this_thread::yield(); + inside->store(false); + errn = EAGAIN; + } + void operator()() override {} + }; + + io_context ioc(Backend); + auto& sched = + static_cast(detail::get_scheduler(ioc)); + + std::atomic inside{false}; + std::atomic retired{false}; + std::atomic retired_inside{false}; + watched_impl impl; + impl.inside = &inside; + impl.retired = &retired; + impl.retired_inside = &retired_inside; + impl.refs_ = 0; + detail::object_ref keep(&impl); + + detail::reactor_descriptor_state desc; + holding_op op; + op.inside = &inside; + op.retired = &retired; + desc.scheduler_ = &sched; + desc.read_op = &op; + desc.ready_events_.store(detail::reactor_event_read); + desc.is_enqueued_.store(true); + + std::thread popped([&] { desc.invoke_deferred_io(); }); + while (!inside.load()) + std::this_thread::yield(); + sched.retire_descriptor(desc, std::move(keep)); + popped.join(); + + BOOST_TEST(retired.load()); + BOOST_TEST(!retired_inside.load()); + desc.read_op = nullptr; + } + void run() { + testRetireWaitsForRunningInvocation(); testEventCompletesBatchedOps(); testForeignPostWakesParkedFollower(); testAcceptorWaitWakesBlockedReactor(); + testCloseRacesReactorPass(); + testForeignStartRacesSettle(); } }; diff --git a/test/unit/native/uring/descriptor_continue.cpp b/test/unit/native/uring/descriptor_continue.cpp index fc58b9046..092b7f175 100644 --- a/test/unit/native/uring/descriptor_continue.cpp +++ b/test/unit/native/uring/descriptor_continue.cpp @@ -27,6 +27,7 @@ #include #include +#include #include #include @@ -43,13 +44,33 @@ namespace boost::corosio { -// Exposes the io_uring scheduler, the way multishot_acceptor.cpp does. +// Exposes the io_uring scheduler, the way multishot_acceptor.cpp does, +// and hands out descriptor impls the way a handle obtains them. struct uring_descriptor_test_context : native_io_context { + struct descriptor_deleter + { + detail::uring_descriptor_service* svc; + void operator()(detail::uring_descriptor* d) const noexcept + { + svc->destroy(d); + } + }; + using descriptor_ptr = + std::unique_ptr; + detail::uring_scheduler& scheduler() noexcept { return *static_cast(sched_); } + + descriptor_ptr make_descriptor() + { + auto& svc = use_service(); + return descriptor_ptr( + static_cast(svc.construct()), + descriptor_deleter{&svc}); + } }; // A real coroutine handle for the completing op to resume. The body runs @@ -113,7 +134,7 @@ struct uring_descriptor_continue_test // dispatch. `res` still holds the revents mask, which the // completion decode would read as a byte count. uring_descriptor_test_context ctx; - auto d = std::make_shared(ctx.scheduler()); + auto d = ctx.make_descriptor(); detail::uring_descriptor_read_op op; arm(*d, op); @@ -139,7 +160,7 @@ struct uring_descriptor_continue_test // cancel-by-fd SQE finds nothing and the op's own `cancelled` // flag is never set. The epoch is the only thing that sees it. uring_descriptor_test_context ctx; - auto d = std::make_shared(ctx.scheduler()); + auto d = ctx.make_descriptor(); detail::uring_descriptor_write_op op; arm(*d, op); @@ -160,7 +181,7 @@ struct uring_descriptor_continue_test // would otherwise arm a fresh poll nothing can cancel, and // run() would never return. uring_descriptor_test_context ctx; - auto d = std::make_shared(ctx.scheduler()); + auto d = ctx.make_descriptor(); detail::uring_descriptor_read_op op; arm(*d, op); @@ -183,7 +204,7 @@ struct uring_descriptor_continue_test BOOST_TEST_EQ(::pipe(fds), 0); uring_descriptor_test_context ctx; - auto d = std::make_shared(ctx.scheduler()); + auto d = ctx.make_descriptor(); d->set_descriptor(fds[0]); detail::uring_descriptor_read_op op; @@ -212,7 +233,7 @@ struct uring_descriptor_continue_test void testTerminalResultsAreLeftAlone() { uring_descriptor_test_context ctx; - auto d = std::make_shared(ctx.scheduler()); + auto d = ctx.make_descriptor(); // A poll that failed or was cancelled is already the answer. detail::uring_descriptor_read_op poll_failed; @@ -247,7 +268,7 @@ struct uring_descriptor_continue_test uring_descriptor_test_context ctx; auto ex = ctx.get_executor(); - auto d = std::make_shared(ctx.scheduler()); + auto d = ctx.make_descriptor(); d->set_descriptor(fds[0]); bool resumed = false; @@ -261,7 +282,7 @@ struct uring_descriptor_continue_test detail::uring_descriptor_read_op op; op.prepare( coro.h, ex, &ec, &bytes, fds[0], /*file_offset=*/-1, - &ctx.scheduler(), d, mb, std::stop_token{}); + &ctx.scheduler(), detail::object_ref(d.get()), mb, std::stop_token{}); arm(*d, op); // Data is already waiting, so the poll this arms fires at once. @@ -289,7 +310,7 @@ struct uring_descriptor_continue_test // cancel that bumped the epoch before the prep must turn the // SQE into a NOP, never a transfer on the descriptor. uring_descriptor_test_context ctx; - auto d = std::make_shared(ctx.scheduler()); + auto d = ctx.make_descriptor(); detail::uring_descriptor_read_op op; arm(*d, op); @@ -311,7 +332,7 @@ struct uring_descriptor_continue_test BOOST_TEST_EQ(::pipe(fds), 0); uring_descriptor_test_context ctx; - auto d = std::make_shared(ctx.scheduler()); + auto d = ctx.make_descriptor(); d->set_descriptor(fds[0]); detail::uring_descriptor_write_op op; @@ -331,7 +352,7 @@ struct uring_descriptor_continue_test // A stop landing after the transfer's CQE abandons nothing: the // transfer is done, and its byte count is the answer. uring_descriptor_test_context ctx; - auto d = std::make_shared(ctx.scheduler()); + auto d = ctx.make_descriptor(); detail::uring_descriptor_read_op op; arm(*d, op); @@ -349,7 +370,7 @@ struct uring_descriptor_continue_test // kernel and came back with the full-SQ path's -EAGAIN. A cancel // that landed meanwhile is owed canceled, not try-again. uring_descriptor_test_context ctx; - auto d = std::make_shared(ctx.scheduler()); + auto d = ctx.make_descriptor(); detail::uring_descriptor_read_op op; arm(*d, op); @@ -367,7 +388,7 @@ struct uring_descriptor_continue_test // is interrupted by the cancel and comes back as -EINTR. The // caller asked for the cancel, so it is owed canceled. uring_descriptor_test_context ctx; - auto d = std::make_shared(ctx.scheduler()); + auto d = ctx.make_descriptor(); detail::uring_descriptor_read_op op; arm(*d, op); @@ -390,7 +411,7 @@ struct uring_descriptor_continue_test // A file with no poll support is ready to every poll, so a // re-armed poll would retry the transfer forever. uring_descriptor_test_context ctx; - auto d = std::make_shared(ctx.scheduler()); + auto d = ctx.make_descriptor(); d->set_descriptor(-1, /*pollable=*/false); detail::uring_descriptor_read_op op; diff --git a/test/unit/native/uring/multishot_acceptor.cpp b/test/unit/native/uring/multishot_acceptor.cpp index d15a34d42..cff40e774 100644 --- a/test/unit/native/uring/multishot_acceptor.cpp +++ b/test/unit/native/uring/multishot_acceptor.cpp @@ -15,6 +15,7 @@ #if BOOST_COROSIO_HAS_URING #include +#include #include #include #include @@ -23,7 +24,16 @@ #include #include +#include #include +#include +#include +#include + +#include +#include +#include +#include // The coroutine tasks below take their state through by-value parameters // (references to test-scope objects), never by lambda capture: a captured @@ -42,6 +52,58 @@ struct uring_test_context : native_io_context } }; +inline capy::task<> +noop_task() +{ + co_return; +} + +// Reap completions nobody awaits, such as a cancelled arming's +// terminal CQE: poll() only pumps the ring while work is outstanding. +inline void +reap(native_io_context& ioc) +{ + for (int i = 0; i < 8; ++i) + { + capy::run_async(ioc.get_executor())(noop_task()); + ioc.restart(); + ioc.poll(); + } + ioc.restart(); +} + +// A blocking loopback connect; completes from the listener's backlog. +inline int +raw_connect(std::uint16_t port) +{ + int fd = ::socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0); + sockaddr_in sa{}; + sa.sin_family = AF_INET; + sa.sin_port = htons(port); + sa.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + if (::connect(fd, reinterpret_cast(&sa), sizeof(sa)) != 0) + { + ::close(fd); + return -1; + } + return fd; +} + +inline std::uint16_t +peer_port(tcp_socket const& s) +{ + return s.remote_endpoint().port(); +} + +inline std::uint16_t +local_port(int fd) +{ + sockaddr_in sa{}; + socklen_t len = sizeof(sa); + ::getsockname(fd, reinterpret_cast(&sa), &len); + return ntohs(sa.sin_port); +} + struct multishot_acceptor_test { void testContextConstructs() @@ -50,14 +112,12 @@ struct multishot_acceptor_test BOOST_TEST(!ioc.stopped()); } - // Regression: drain_cqes_for (run from the multishot acceptor's - // destructor) used to advance CQEs without decrementing - // uring_inflight_, leaking the counts of the multishot accept SQE - // and the cancel SQEs it submits. The counter gates the do_one ring - // pump, so the leak accumulated across every acceptor teardown for the - // lifetime of the io_context. After the fix the counter returns to - // zero once teardown settles. - void testDrainCqesBalancesInflight() + // Destroying an acceptor must leave uring_inflight_ balanced: the + // multishot accept SQE and the cancel SQEs close submits are each + // counted once and must each be uncounted by their CQEs. The + // counter gates the do_one ring pump, so a leak would accumulate + // across every acceptor teardown for the lifetime of the io_context. + void testCloseReapsArming() { uring_test_context ctx; @@ -77,14 +137,19 @@ struct multishot_acceptor_test ctx.poll(); BOOST_TEST(ctx.inflight() >= 1); } - // acc destroyed: the impl destructor runs cancel-by-fd and - // drain_cqes_for(multi_op_). Drain the CQEs those produce that the - // destructor's bounded loop did not itself consume. + // acc destroyed: close cancelled the arming, whose terminal CQE + // the run loop reaps. poll() only pumps the ring while work is + // outstanding. + auto ex = ctx.get_executor(); for (int i = 0; i < 64 && ctx.inflight() != 0; ++i) + { + capy::run_async(ex)(noop_task()); + ctx.restart(); ctx.poll(); + } // Every submitted SQE (the multishot accept plus the teardown - // cancels) is now accounted for. Before the fix this stayed > 0. + // cancels) is now accounted for. BOOST_TEST_EQ(ctx.inflight(), 0); } @@ -200,13 +265,139 @@ struct multishot_acceptor_test c2.close(); } + // Accept one connection through `acc` and report the peer port. + static std::uint16_t + accept_one(native_io_context& ioc, tcp_acceptor& acc) + { + tcp_socket peer(ioc); + bool accepted = false; + capy::run_async(ioc.get_executor())( + [](tcp_acceptor& a, tcp_socket& p, bool& done) -> capy::task<> { + auto [ec] = co_await a.accept(p); + done = !ec; + }(acc, peer, accepted)); + for (int i = 0; i < 10000 && !accepted; ++i) + ioc.poll(); + ioc.restart(); + BOOST_TEST(accepted); + return accepted ? peer_port(peer) : 0; + } + + // A released listener keeps its connections: once the acceptor + // listens again, it accepts only from its new descriptor, even + // while the released one still has connections queued. + void testReleaseThenRelisten() + { + native_io_context ioc; + + tcp_acceptor acc(ioc); + BOOST_TEST(!acc.open()); + BOOST_TEST(!acc.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!acc.listen()); + int released = static_cast(acc.release()); + BOOST_TEST_GE(released, 0); + + BOOST_TEST(!acc.open()); + BOOST_TEST(!acc.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!acc.listen()); + + int stale = raw_connect(local_port(released)); + int fresh = raw_connect(acc.local_endpoint().port()); + BOOST_TEST_GE(stale, 0); + BOOST_TEST_GE(fresh, 0); + + BOOST_TEST_EQ(accept_one(ioc, acc), local_port(fresh)); + + // The released descriptor still holds its own connection. + int got = ::accept4(released, nullptr, nullptr, SOCK_NONBLOCK); + BOOST_TEST_GE(got, 0); + + ::close(got); + ::close(stale); + ::close(fresh); + ::close(released); + acc.close(); + reap(ioc); + } + + // Assigning right after a close, while the cancelled arming's + // terminal completion is still unreaped, moves accepting to the new + // descriptor; a re-listen on it does not arm a second time. + void testAssignReplacesArming() + { + native_io_context ioc; + + tcp_acceptor acc(ioc); + BOOST_TEST(!acc.open()); + BOOST_TEST(!acc.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!acc.listen()); + + int nfd = + ::socket(AF_INET, SOCK_STREAM | SOCK_NONBLOCK | SOCK_CLOEXEC, 0); + sockaddr_in sa{}; + sa.sin_family = AF_INET; + sa.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + BOOST_TEST_EQ( + ::bind(nfd, reinterpret_cast(&sa), sizeof(sa)), 0); + BOOST_TEST_EQ(::listen(nfd, 16), 0); + acc.close(); + BOOST_TEST(!acc.assign(nfd)); + BOOST_TEST(!acc.listen()); + + for (int i = 0; i < 3; ++i) + { + int c = raw_connect(acc.local_endpoint().port()); + BOOST_TEST_GE(c, 0); + BOOST_TEST_EQ(accept_one(ioc, acc), local_port(c)); + ::close(c); + } + + acc.close(); + reap(ioc); + } + + // Close on one thread races deliveries and terminal completions + // being dispatched on another. + void testCloseRacesDispatch() + { + native_io_context ioc; + auto ex = ioc.get_executor(); + { + tcp_acceptor warm(ioc); + } + + std::stop_source keep; + capy::run_async(ex, keep.get_token())([]() -> capy::task<> { + std::ignore = co_await delay(std::chrono::seconds(60)); + }()); + std::thread runner([&] { ioc.run(); }); + + for (int i = 0; i < 200; ++i) + { + tcp_acceptor acc(ioc); + BOOST_TEST(!acc.open()); + BOOST_TEST(!acc.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!acc.listen()); + int c = raw_connect(acc.local_endpoint().port()); + if (i % 2) + acc.close(); + ::close(c); + } + + keep.request_stop(); + runner.join(); + } + void run() { testContextConstructs(); - testDrainCqesBalancesInflight(); + testCloseReapsArming(); testAcceptBufferedConnection(); testAcceptParkedThenDelivered(); testDestroyWithBufferedConnections(); + testReleaseThenRelisten(); + testAssignReplacesArming(); + testCloseRacesDispatch(); } }; diff --git a/test/unit/object_pool.cpp b/test/unit/object_pool.cpp new file mode 100644 index 000000000..11b173e29 --- /dev/null +++ b/test/unit/object_pool.cpp @@ -0,0 +1,288 @@ +#include +#include +#include +#include "test_suite.hpp" + +namespace boost::corosio::detail { + +namespace { + +struct pool_impl final + : io_object::implementation + , intrusive_list::node +{ + static inline int instances = 0; + + object_pool* pool = nullptr; + int reused_count = 0; + + pool_impl() { ++instances; } + ~pool_impl() override { --instances; } + + void reuse() noexcept { ++reused_count; } + void retire() noexcept override { pool->recycle(this); } +}; + +// Exercises the sanctioned-extension mechanism documented on +// object_pool: a derived pool re-exposing adopt()/remove() as public, +// the same way timer_object_pool does for the timer service. +struct test_pool : object_pool +{ + using object_pool::adopt; + using object_pool::remove; +}; + +// An impl holding an embedded object_ref to itself never reaches +// zero through ordinary release() calls; only ~object_pool()'s sweep +// can free it, and that sweep's `delete` releases the embedded ref. +struct self_ref_impl final + : io_object::implementation + , intrusive_list::node +{ + static inline int instances = 0; + + object_pool* pool = nullptr; + object_ref self_ref; + + self_ref_impl() { ++instances; } + ~self_ref_impl() override { --instances; } + + void reuse() noexcept { } + void retire() noexcept override { pool->recycle(this); } +}; + +// The same self-reference held by a base class while only the final +// class overrides retire(), the layout reactor impls use: a release +// reaching zero during `delete` would call retire() through a base +// whose own override is pure. +struct self_ref_base + : io_object::implementation + , intrusive_list::node +{ + object_ref self_ref; +}; + +struct self_ref_derived final : self_ref_base +{ + static inline int instances = 0; + + object_pool* pool = nullptr; + + self_ref_derived() { ++instances; } + ~self_ref_derived() override { --instances; } + + void reuse() noexcept { } + void retire() noexcept override { pool->recycle(this); } +}; + +} // namespace + +struct object_pool_test +{ + void run() + { + // try_acquire refuses an impl parked on the free list: its count + // holds the pool's sentinel, not a live reference + { + object_pool pool; + auto* a = pool.acquire(); + a->pool = &pool; + release(a); // parked in free_ + BOOST_TEST(!try_acquire(a)); + auto* b = pool.acquire(); // still recyclable intact + BOOST_TEST_EQ(b, a); + BOOST_TEST_EQ(b->refs_.load(), 1u); + release(b); + } + // acquire news on a cold pool and hands the same impl back, + // reuse()-reset, after it is recycled + { + object_pool pool; + auto* a = pool.acquire(); + a->pool = &pool; + BOOST_TEST_EQ(a->reused_count, 0); + release(a); // service ref -> recycle + BOOST_TEST_EQ(pool_impl::instances, 1); + auto* b = pool.acquire(); // warm: recycled, not newed + BOOST_TEST_EQ(b, a); + BOOST_TEST_EQ(b->reused_count, 1); + BOOST_TEST_EQ(pool_impl::instances, 1); + release(b); // back to free; dtor frees + } + BOOST_TEST_EQ(pool_impl::instances, 0); + // keepalive defers recycling until the last ref drops + { + object_pool pool; + auto* a = pool.acquire(); + a->pool = &pool; + object_ref op_ref(a); // simulated in-flight op + release(a); // service ref gone... + auto* b = pool.acquire(); // ...a not recycled: news + b->pool = &pool; + BOOST_TEST_NE(b, a); + BOOST_TEST_EQ(pool_impl::instances, 2); + op_ref.reset(); // op drains -> a recycles + auto* c = pool.acquire(); // warm again + BOOST_TEST_EQ(c, a); + release(b); + release(c); + } + BOOST_TEST_EQ(pool_impl::instances, 0); + // shutdown mode deletes instead of pooling + { + object_pool pool; + auto* a = pool.acquire(); + a->pool = &pool; + pool.shutdown([](pool_impl*) {}); + release(a); // deleted, not pooled + BOOST_TEST_EQ(pool_impl::instances, 0); + } + // shutdown() skips an impl that already reached zero: its + // retire() is about to recycle (and, once shutting down, + // delete) it, so the callback must never take a reference. + { + object_pool pool; + auto* live = pool.acquire(); + auto* retiring = pool.acquire(); + live->pool = &pool; + retiring->pool = &pool; + retiring->refs_.store(0); // its recycle() has not run yet + int visited = 0; + pool.shutdown( + [&](pool_impl* p) + { + BOOST_TEST_EQ(p, live); + ++visited; + }); + BOOST_TEST_EQ(visited, 1); + pool.recycle(retiring); // shutting down: deleted + BOOST_TEST_EQ(pool_impl::instances, 1); + release(live); + } + BOOST_TEST_EQ(pool_impl::instances, 0); + // shutdown() pins each visited impl for the callback and drops + // the pin afterwards; the last reference released by that drop + // deletes, since the pool is shutting down. + { + object_pool pool; + auto* a = pool.acquire(); + a->pool = &pool; + pool.shutdown( + [&](pool_impl* p) + { + BOOST_TEST_EQ(p->refs_.load(), 2u); + release(p); // the handle's reference goes meanwhile + }); + BOOST_TEST_EQ(pool_impl::instances, 0); + } + // remove() transfers ownership out of the pool entirely: the + // impl leaves live_ and the caller now owns it outright (a + // direct delete here must not be seen again by the pool's own + // destructor -- the scope exit below proves no double free). + { + test_pool pool; + auto* a = pool.acquire(); + a->pool = &pool; + BOOST_TEST(pool.remove(a)); // unlinked from live_ + delete a; // caller-owned now + // pool goes out of scope here with empty live_/free_: if + // remove() had failed to unlink `a`, ~object_pool() would + // delete it a second time. + } + BOOST_TEST_EQ(pool_impl::instances, 0); + // remove() is idempotent: a second call on an already-detached + // impl returns false instead of corrupting the list (mirrors + // intrusive_list::remove()'s own already-detached contract). + { + test_pool pool; + auto* a = pool.acquire(); + a->pool = &pool; + BOOST_TEST(pool.remove(a)); + BOOST_TEST(!pool.remove(a)); // already detached + delete a; + } + // Sentinel guard: remove() must refuse to touch a free_-parked + // impl (the same hazard recycle() guards against in reverse). + // The guard's debug-build path asserts and aborts before + // returning, so only its release-build-observable contract -- + // refused, false, free list left intact -- can be exercised + // here; it is skipped entirely when assertions are compiled in + // (the normal state of this test binary). +#ifdef NDEBUG + { + test_pool pool; + auto* a = pool.acquire(); + a->pool = &pool; + release(a); // parks a on free_ (refs_ == pooled_sentinel) + BOOST_TEST(!pool.remove(a)); // refused + auto* b = pool.acquire(); // free_ untouched: warm hit + BOOST_TEST_EQ(b, a); + release(b); // back to free_; pool dtor frees it + } +#endif + BOOST_TEST_EQ(pool_impl::instances, 0); + // recycle()'s own guards, release-build contract, mirroring + // the remove() block above: both skip entirely under + // assertions (the normal state of this test binary) because + // the guarded branch asserts and aborts before returning. +#ifdef NDEBUG + { + // Sentinel guard: a duplicate recycle() on an impl + // already parked in free_ (e.g. a double retire() for + // one zero-crossing) must refuse before touching either + // list, not corrupt free_ by removing it through live_'s + // boundary pointers. + test_pool pool; + auto* a = pool.acquire(); + a->pool = &pool; + release(a); // 1 -> 0: recycle() parks a on free_ + BOOST_TEST_EQ(pool_impl::instances, 1); + pool.recycle(a); // duplicate retire while parked: refused + auto* b = pool.acquire(); // free_ untouched: still a warm hit + BOOST_TEST_EQ(b, a); + release(b); + } + BOOST_TEST_EQ(pool_impl::instances, 0); + { + // Accounting-bug branch: an impl detached from live_ + // (not parked, not shutting down) reaching recycle() is + // deleted outright instead of being stranded (reachable + // through neither acquire() nor shutdown()'s visit while + // still allocated). + test_pool pool; + auto* a = pool.acquire(); + a->pool = &pool; + BOOST_TEST(pool.remove(a)); // detach from live_ without recycling + pool.recycle(a); // not live, not pooled: deleted + BOOST_TEST_EQ(pool_impl::instances, 0); + } +#endif + // A self-referential impl is freed exactly once by the + // destructor's sweep; releasing its embedded ref during that + // `delete` must not retire it again. + { + object_pool pool; + auto* a = pool.acquire(); + a->pool = &pool; + a->self_ref = object_ref(a); // refs_: 1 -> 2 + release(a); // refs_: 2 -> 1; self-ref keeps it alive + BOOST_TEST_EQ(self_ref_impl::instances, 1); + pool.shutdown([](self_ref_impl*) {}); + } + BOOST_TEST_EQ(self_ref_impl::instances, 0); + // Same, with the self-reference held by a base class. + { + object_pool pool; + auto* a = pool.acquire(); + a->pool = &pool; + a->self_ref = object_ref(a); + release(a); + pool.shutdown([](self_ref_derived*) {}); + } + BOOST_TEST_EQ(self_ref_derived::instances, 0); + } +}; + +TEST_SUITE(object_pool_test, "boost.corosio.object_pool"); + +} // namespace boost::corosio::detail diff --git a/test/unit/object_ref.cpp b/test/unit/object_ref.cpp new file mode 100644 index 000000000..f99d7069e --- /dev/null +++ b/test/unit/object_ref.cpp @@ -0,0 +1,96 @@ +#include +#include +#include "test_suite.hpp" + +namespace boost::corosio::detail { + +namespace { + +struct counted_impl final : io_object::implementation +{ + int* last_ref_hits; + explicit counted_impl(int* hits) : last_ref_hits(hits) {} + void retire() noexcept override { ++*last_ref_hits; } +}; + +} // namespace + +struct object_ref_test +{ + void run() + { + // acquire/release pairs; zero-action fires exactly once + { + int hits = 0; + counted_impl impl(&hits); // refs_ starts at 1 + release(&impl); // 1 -> 0 + BOOST_TEST_EQ(hits, 1); + } + // object_ref RAII: ctor acquires, dtor releases + { + int hits = 0; + counted_impl impl(&hits); + { + object_ref r(&impl); // 1 -> 2 + BOOST_TEST(r); + object_ref r2 = r; // copy: 2 -> 3 + object_ref r3 = std::move(r2); // move: still 3 + BOOST_TEST(!r2); + BOOST_TEST_EQ(r3.get(), &impl); + } // back to 1 + BOOST_TEST_EQ(hits, 0); + release(&impl); + BOOST_TEST_EQ(hits, 1); + } + // reset releases early + { + int hits = 0; + counted_impl impl(&hits); + object_ref r(&impl); + release(&impl); // service ref gone, r holds it + BOOST_TEST_EQ(hits, 0); + r.reset(); + BOOST_TEST_EQ(hits, 1); + } + // copy-assignment: operator=(object_ref const&) is never + // otherwise executed anywhere in the tree. Verify it releases + // the old target and acquires the new one, and is safe under + // self-assignment. + { + int hits_a = 0; + int hits_b = 0; + counted_impl a(&hits_a); + counted_impl b(&hits_b); + + object_ref ra(&a); // a.refs_: 1 -> 2 + object_ref rb(&b); // b.refs_: 1 -> 2 + + rb = ra; // rb releases b, acquires a + BOOST_TEST_EQ(rb.get(), &a); + BOOST_TEST_EQ( + a.refs_.load(), 3u); // service + ra + rb + release(&b); // b's last (service) ref + BOOST_TEST_EQ(hits_b, 1); // retired: rb no longer holds it + + // Deliberate self-assignment: must not release before + // acquiring. Laundered through a named alias rather than + // `rb = rb;` -- clang's -Wself-assign-overloaded flags the + // literal spelling even when it is intentional. + object_ref& rb_alias = rb; + rb = rb_alias; + BOOST_TEST_EQ(rb.get(), &a); + BOOST_TEST_EQ(a.refs_.load(), 3u); + BOOST_TEST_EQ(hits_a, 0); // no spurious retire + + ra.reset(); // a.refs_: 3 -> 2 + release(&a); // service ref: 2 -> 1 + BOOST_TEST_EQ(hits_a, 0); + rb.reset(); // a.refs_: 1 -> 0 -> retire + BOOST_TEST_EQ(hits_a, 1); + } + } +}; + +TEST_SUITE(object_ref_test, "boost.corosio.object_ref"); + +} // namespace boost::corosio::detail diff --git a/test/unit/object_reuse.cpp b/test/unit/object_reuse.cpp new file mode 100644 index 000000000..204078678 --- /dev/null +++ b/test/unit/object_reuse.cpp @@ -0,0 +1,859 @@ +// +// Copyright (c) 2026 Steve Gerbino +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// Official repository: https://github.com/cppalliance/corosio +// + +// Keepalive-extension regression for the object_pool conversion: dropping +// a socket while an op is still parked must not use-after-free the +// impl, and a socket constructed afterward on the same context must +// get a fully working impl, whether the pool recycled the dropped one +// or allocated a new one -- the caller cannot tell which, and must not +// need to. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if BOOST_COROSIO_POSIX +#include +#include +#include +#endif +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "context.hpp" +#include "temp_path.hpp" +#include "test_suite.hpp" + +namespace boost::corosio { + +template +struct object_reuse_test +{ + // Open a connected pair, park a read on one side with no writer + // on the other, then drop that socket while the read is still + // parked. close()/destroy() cancel the read and release the + // service's own reference; the read's own reference (its object_ref + // keepalive) keeps the impl alive until the cancellation actually + // drains, at which point the impl is recycled -- not freed. A + // sanitizer build is what actually validates no UAF happened; this + // test's own assertions are the behavioral half: the abandoned read + // completes, and the context keeps working afterward. + void testAbandonedReadThenReuse() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + // Declared at function scope, not inside the block below: the + // abandoned read's cancellation resumes the coroutine after the + // block has exited, and it writes `resumed`. + char buf[16]; + bool resumed = false; + + { + auto [s1, s2] = test::make_socket_pair(ioc); + std::ignore = s2; // keeps the peer fd open; never written to + + capy::run_async(ex)( + [](tcp_socket& s, char* b, std::size_t n, + bool& done) -> capy::task<> { + std::ignore = + co_await s.read_some(capy::mutable_buffer(b, n)); + done = true; + }(s1, buf, sizeof(buf), resumed)); + + // Let the read actually register as parked before s1 (and + // its parked read) go out of scope at the end of this block. + std::ignore = ioc.run_one(); + } + + // Drain the cancellation the destructor produced. + ioc.run(); + ioc.restart(); + BOOST_TEST(resumed); + + // A fresh socket on the same context must get a working impl. + tcp_socket probe(ioc); + BOOST_TEST(!probe.open()); + BOOST_TEST(probe.is_open()); + + auto [p1, p2] = test::make_socket_pair(ioc); + char const out = 'x'; + char in = 0; + bool wrote = false, read_ok = false; + capy::run_async(ex)( + [](tcp_socket& s, char const* b, bool& done) -> capy::task<> { + auto [ec, n] = co_await s.write_some(capy::const_buffer(b, 1)); + done = !ec && n == 1; + }(p1, &out, wrote)); + capy::run_async(ex)( + [](tcp_socket& s, char* b, bool& done) -> capy::task<> { + auto [ec, n] = co_await s.read_some(capy::mutable_buffer(b, 1)); + done = !ec && n == 1; + }(p2, &in, read_ok)); + ioc.restart(); + ioc.run(); + + BOOST_TEST(wrote); + BOOST_TEST(read_ok); + BOOST_TEST_EQ(in, out); + } + + // io_uring's multishot accept can deliver a connection before any + // accept() is outstanding to claim it; that connection's fd (and + // tracking node) parks in the acceptor impl's ready_fds_ list. + // Reaching zero references recycles the impl rather than freeing + // it, so the parked fd must be drained on that path too: destroy + // an acceptor holding one buffered connection, then construct a + // fresh acceptor on the same context, which pops the recycled impl + // and asserts `ready_fds_.empty()` in reuse(). On epoll/select + // there is no ready_fds_/multishot concept, so this is a plain + // acceptor recycle-and-reuse check there. + // A socket destroyed while its connect is in flight must not have the + // connect's completion write endpoints into it: the impl is recycled + // and the next socket would report the old peer. + void testConnectThenDestroyRecycles() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + tcp_acceptor acc(ioc); + BOOST_TEST(!acc.open()); + acc.set_option(socket_option::reuse_address(true)); + BOOST_TEST(!acc.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!acc.listen()); + endpoint ep(ipv4_address::loopback(), acc.local_endpoint().port()); + + for (int i = 0; i < 8; ++i) + { + bool started = false; + { + tcp_socket s(ioc); + BOOST_TEST(!s.open()); + capy::run_async(ex)( + [](tcp_socket& sock, endpoint e, + bool& began) -> capy::task<> { + began = true; + std::ignore = co_await sock.connect(e); + }(s, ep, started)); + ioc.restart(); + while (!started && ioc.poll_one() != 0) + { + } + } + ioc.restart(); + ioc.run(); + + tcp_socket fresh(ioc); + BOOST_TEST(!fresh.open()); + BOOST_TEST(fresh.remote_endpoint() == endpoint{}); + } + } + + void testAcceptorRecycleDrainsBufferedConnection() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + { + tcp_acceptor acc(ioc); + BOOST_TEST(!acc.open()); + acc.set_option(socket_option::reuse_address(true)); + BOOST_TEST(!acc.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!acc.listen()); + auto port = acc.local_endpoint().port(); + + tcp_socket client(ioc); + bool connected = false; + capy::run_async(ex)( + [](tcp_socket& s, endpoint ep, bool& done) -> capy::task<> { + auto [ec] = co_await s.connect(ep); + done = !ec; + }(client, endpoint(ipv4_address::loopback(), port), + connected)); + + // No accept() is outstanding: on uring the connection's + // multishot CQE buffers in ready_fds_ instead of completing + // an accept. run() (not poll()) drives the connector fully + // to completion, which is also where the server's accept + // CQE lands -- same idiom as + // multishot_acceptor_test::testDestroyWithBufferedConnections. + ioc.run(); + BOOST_TEST(connected); + + client.close(); + + // acc drops here holding a buffered connection in + // ready_fds_ (uring) or nothing to hold (epoll/select). + } + + // On uring the impl recycles only once the cancelled arming's + // terminal CQE is reaped, and poll() pumps the ring only while + // work is outstanding. + for (int i = 0; i < 8; ++i) + { + capy::run_async(ex)([]() -> capy::task<> { co_return; }()); + ioc.restart(); + ioc.poll(); + } + ioc.restart(); + + // A fresh acceptor on the same context pops the just-recycled + // impl, whose reuse() asserts `ready_fds_.empty()` (uring only). + tcp_acceptor acc2(ioc); + BOOST_TEST(!acc2.open()); + acc2.set_option(socket_option::reuse_address(true)); + BOOST_TEST(!acc2.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!acc2.listen()); + auto port2 = acc2.local_endpoint().port(); + + // Full exercise: accept a real connection through the recycled + // (or freshly allocated) impl to prove it still works. + tcp_socket server2(ioc); + tcp_socket client2(ioc); + bool accept_done = false, connect_done = false; + capy::run_async(ex)( + [](tcp_acceptor& a, tcp_socket& s, bool& done) -> capy::task<> { + auto [ec] = co_await a.accept(s); + done = !ec; + }(acc2, server2, accept_done)); + capy::run_async(ex)( + [](tcp_socket& s, endpoint ep, bool& done) -> capy::task<> { + auto [ec] = co_await s.connect(ep); + done = !ec; + }(client2, endpoint(ipv4_address::loopback(), port2), + connect_done)); + ioc.run(); + + BOOST_TEST(accept_done); + BOOST_TEST(connect_done); + BOOST_TEST(server2.is_open()); + } + + void run() + { + testAbandonedReadThenReuse(); + testConnectThenDestroyRecycles(); + testAcceptorRecycleDrainsBufferedConnection(); + } +}; + +// Churns every pooled object kind through 32 construct/destroy cycles, +// then runs one representative op per kind so the exercise below runs +// entirely on recycled (not freshly allocated) impls. Registered on +// the reactor backends plus io_uring; IOCP has no churn coverage yet. +template +struct churn_then_exercise_test +{ + static constexpr int churn_n = 32; + + void churnSockets(io_context& ioc) + { + for (int i = 0; i < churn_n; ++i) + { + tcp_socket s(ioc); + } + } + + void churnAcceptors(io_context& ioc) + { + for (int i = 0; i < churn_n; ++i) + { + tcp_acceptor a(ioc); + } + } + + void churnUdp(io_context& ioc) + { + for (int i = 0; i < churn_n; ++i) + { + udp_socket u(ioc); + } + } + + void churnResolvers(io_context& ioc) + { + for (int i = 0; i < churn_n; ++i) + { + resolver r(ioc); + } + } + + void churnSignalSets(io_context& ioc) + { +#if BOOST_COROSIO_POSIX + for (int i = 0; i < churn_n; ++i) + { + signal_set s(ioc, SIGUSR1); + } +#else + // SIGUSR1 doesn't exist on Windows; this template is never + // instantiated there (COROSIO_NON_IOCP_BACKEND_TESTS registers + // nothing on Windows), but non-dependent names inside it must + // still resolve at definition time regardless of instantiation. + std::ignore = ioc; +#endif + } + + void churnFiles(io_context& ioc) + { + test::temp_file tmp("object_reuse_churn_"); + for (int i = 0; i < churn_n; ++i) + { + stream_file f(ioc); + std::ignore = f.open( + tmp.path, + file_base::read_write | file_base::create | + file_base::truncate); + } + } + + void churnTimers(io_context& ioc, capy::executor_ref ex) + { + for (int i = 0; i < churn_n; ++i) + { + bool done = false; + capy::run_async(ex)([](bool& d) -> capy::task<> { + auto [ec] = co_await delay(std::chrono::microseconds(1)); + d = !ec; + }(done)); + ioc.run(); + ioc.restart(); + BOOST_TEST(done); + } + } + + // socket + acceptor: accept, connect, read, write, then cancel a + // fresh read via stop_token, run on the impls + // churnSockets()/churnAcceptors() just recycled. + void exerciseSocketAndAcceptor(io_context& ioc, capy::executor_ref ex) + { + tcp_acceptor acc(ioc); + BOOST_TEST(!acc.open()); + acc.set_option(socket_option::reuse_address(true)); + BOOST_TEST(!acc.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!acc.listen()); + auto port = acc.local_endpoint().port(); + + tcp_socket server(ioc); + tcp_socket client(ioc); + bool accept_done = false, connect_done = false; + + capy::run_async(ex)( + [](tcp_acceptor& a, tcp_socket& s, bool& done) -> capy::task<> { + auto [ec] = co_await a.accept(s); + done = !ec; + }(acc, server, accept_done)); + capy::run_async(ex)( + [](tcp_socket& s, endpoint ep, bool& done) -> capy::task<> { + auto [ec] = co_await s.connect(ep); + done = !ec; + }(client, endpoint(ipv4_address::loopback(), port), + connect_done)); + ioc.run(); + ioc.restart(); + BOOST_TEST(accept_done); + BOOST_TEST(connect_done); + + char const out = 'q'; + char in = 0; + bool wrote = false, read_ok = false; + capy::run_async(ex)( + [](tcp_socket& s, char const* b, bool& done) -> capy::task<> { + auto [ec, n] = co_await s.write_some(capy::const_buffer(b, 1)); + done = !ec && n == 1; + }(client, &out, wrote)); + capy::run_async(ex)( + [](tcp_socket& s, char* b, bool& done) -> capy::task<> { + auto [ec, n] = co_await s.read_some(capy::mutable_buffer(b, 1)); + done = !ec && n == 1; + }(server, &in, read_ok)); + ioc.run(); + ioc.restart(); + BOOST_TEST(wrote); + BOOST_TEST(read_ok); + BOOST_TEST_EQ(in, out); + + // Cancel-via-stop_token: park a second read with no writer, + // then cancel it through the token instead of close()/ + // destroy() -- a different code path through the same + // recycled impl's op machinery. + std::stop_source src; + bool canceled = false; + capy::run_async(ex, src.get_token())( + [](tcp_socket& s, bool& done) -> capy::task<> { + char buf[1]; + auto [ec, n] = + co_await s.read_some(capy::mutable_buffer(buf, 1)); + std::ignore = n; + done = (ec == capy::cond::canceled); + }(client, canceled)); + ioc.run_one(); + src.request_stop(); + ioc.run(); + ioc.restart(); + BOOST_TEST(canceled); + + client.close(); + server.close(); + acc.close(); + } + + // udp: send_to/recv_from on the recycled churnUdp() impls. + void exerciseUdp(io_context& ioc, capy::executor_ref ex) + { + udp_socket receiver(ioc); + BOOST_TEST(!receiver.open()); + BOOST_TEST(!receiver.bind(endpoint(ipv4_address::loopback(), 0))); + auto rport = receiver.local_endpoint().port(); + + udp_socket sender(ioc); + BOOST_TEST(!sender.open()); + BOOST_TEST(!sender.bind(endpoint(ipv4_address::loopback(), 0))); + + char const out = 'u'; + char in = 0; + bool sent = false, received = false; + endpoint dest(ipv4_address::loopback(), rport); + + capy::run_async(ex)( + [](udp_socket& s, char const* b, endpoint ep, + bool& done) -> capy::task<> { + auto [ec, n] = + co_await s.send_to(capy::const_buffer(b, 1), ep); + done = !ec && n == 1; + }(sender, &out, dest, sent)); + endpoint source; + capy::run_async(ex)( + [](udp_socket& s, char* b, endpoint& src, + bool& done) -> capy::task<> { + auto [ec, n] = + co_await s.recv_from(capy::mutable_buffer(b, 1), src); + done = !ec && n == 1; + }(receiver, &in, source, received)); + ioc.run(); + ioc.restart(); + + BOOST_TEST(sent); + BOOST_TEST(received); + BOOST_TEST_EQ(in, out); + } + + // resolver: resolve localhost on a recycled churnResolvers() impl. + void exerciseResolver(io_context& ioc, capy::executor_ref ex) + { + resolver r(ioc); + bool done = false; + std::error_code resolve_ec; + + capy::run_async(ex)( + [](resolver& r_ref, bool& done_out, + std::error_code& ec_out) -> capy::task<> { + auto [ec, results] = co_await r_ref.resolve("localhost", "80"); + std::ignore = results; + ec_out = ec; + done_out = true; + }(r, done, resolve_ec)); + ioc.run(); + ioc.restart(); + + BOOST_TEST(done); + BOOST_TEST(!resolve_ec); + } + + void run() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + churnSockets(ioc); + churnAcceptors(ioc); + churnUdp(ioc); + churnResolvers(ioc); + churnSignalSets(ioc); + churnFiles(ioc); + churnTimers(ioc, ex); + + exerciseSocketAndAcceptor(ioc, ex); + exerciseUdp(ioc, ex); + exerciseResolver(ioc, ex); + + // signal_set: wait() completed by raise() on a recycled + // churnSignalSets() impl. +#if BOOST_COROSIO_POSIX + { + signal_set sig(ioc, SIGUSR1); + bool done = false; + capy::run_async(ex)( + [](signal_set& s, bool& done_out) -> capy::task<> { + [[maybe_unused]] auto [ec, signum] = co_await s.wait(); + done_out = !ec; + }(sig, done)); + ioc.run_one(); + std::raise(SIGUSR1); + ioc.run(); + ioc.restart(); + BOOST_TEST(done); + } +#endif + + // files: write_some on a recycled churnFiles() impl. + { + test::temp_file tmp("object_reuse_exercise_"); + stream_file f(ioc); + BOOST_TEST(!f.open( + tmp.path, + file_base::read_write | file_base::create | + file_base::truncate)); + bool done = false; + char const data = 'f'; + capy::run_async(ex)( + [](stream_file& file, char const* b, + bool& done_out) -> capy::task<> { + auto [ec, n] = + co_await file.write_some(capy::const_buffer(b, 1)); + done_out = !ec && n == 1; + }(f, &data, done)); + ioc.run(); + ioc.restart(); + BOOST_TEST(done); + } + + // timer: one more delay() wait on a recycled churnTimers() impl. + { + bool done = false; + capy::run_async(ex)([](bool& d) -> capy::task<> { + auto [ec] = co_await delay(std::chrono::microseconds(1)); + d = !ec; + }(done)); + ioc.run(); + ioc.restart(); + BOOST_TEST(done); + } + } +}; + +COROSIO_BACKEND_TESTS(object_reuse_test, "boost.corosio.object_reuse") +COROSIO_NON_IOCP_BACKEND_TESTS( + churn_then_exercise_test, "boost.corosio.object_reuse.churn") + +// AF_UNIX churn lane. local_stream_socket/local_stream_acceptor/ +// local_datagram_socket instantiate the exact same reactor_stream_ +// socket/reactor_acceptor/reactor_datagram_socket templates patched +// above, just parameterized on local_endpoint instead of endpoint -- +// local_endpoint_ is precisely the field poison() caught a missing +// reset for, and nothing else loops these types the way +// churn_then_exercise_test loops the IP-socket kinds. Windows +// loopback rules don't apply: this never binds an IP endpoint, only +// AF_UNIX paths, which are POSIX-only on epoll/select and also +// supported on Windows 10 1803+ -- irrelevant here since +// COROSIO_NON_IOCP_BACKEND_TESTS registers nothing on Windows anyway. +template +struct local_socket_churn_then_exercise_test +{ + static constexpr int churn_n = 32; + + void churnStreamSockets(io_context& ioc) + { + for (int i = 0; i < churn_n; ++i) + { + local_stream_socket s(ioc); + } + } + + void churnAcceptors(io_context& ioc) + { + for (int i = 0; i < churn_n; ++i) + { + local_stream_acceptor a(ioc); + } + } + + void churnDatagramSockets(io_context& ioc) + { +#if BOOST_COROSIO_POSIX + for (int i = 0; i < churn_n; ++i) + { + local_datagram_socket u(ioc); + } +#else + // local_datagram_socket is POSIX-only (local_datagram_socket.hpp). + // This template is never instantiated on Windows + // (COROSIO_NON_IOCP_BACKEND_TESTS registers nothing there), but + // non-dependent names inside it must still resolve at definition + // time regardless of instantiation. + std::ignore = ioc; +#endif + } + + // local_stream_socket + local_stream_acceptor: accept, connect, + // echo one byte each way, then close -- the same shape as + // churn_then_exercise_test's TCP exercise, run on the AF_UNIX + // impls churnStreamSockets()/churnAcceptors() just recycled. + void exerciseStreamAndAcceptor(io_context& ioc, capy::executor_ref ex) + { + test::temp_socket_dir dir; + local_endpoint ep(dir.path()); + + local_stream_acceptor acc(ioc, ep); + BOOST_TEST(acc.is_open()); + + local_stream_socket server(ioc); + local_stream_socket client(ioc); + bool accept_done = false, connect_done = false; + + capy::run_async(ex)( + [](local_stream_acceptor& a, local_stream_socket& s, + bool& done) -> capy::task<> { + auto [ec] = co_await a.accept(s); + done = !ec; + }(acc, server, accept_done)); + capy::run_async(ex)( + [](local_stream_socket& s, local_endpoint ep_, + bool& done) -> capy::task<> { + auto [ec] = co_await s.connect(ep_); + done = !ec; + }(client, ep, connect_done)); + ioc.run(); + ioc.restart(); + BOOST_TEST(accept_done); + BOOST_TEST(connect_done); + + char const out = 'l'; + char in = 0; + bool wrote = false, read_ok = false; + capy::run_async(ex)( + [](local_stream_socket& s, char const* b, + bool& done) -> capy::task<> { + auto [ec, n] = co_await s.write_some(capy::const_buffer(b, 1)); + done = !ec && n == 1; + }(client, &out, wrote)); + capy::run_async(ex)( + [](local_stream_socket& s, char* b, bool& done) -> capy::task<> { + auto [ec, n] = + co_await s.read_some(capy::mutable_buffer(b, 1)); + done = !ec && n == 1; + }(server, &in, read_ok)); + ioc.run(); + ioc.restart(); + BOOST_TEST(wrote); + BOOST_TEST(read_ok); + BOOST_TEST_EQ(in, out); + + client.close(); + server.close(); + acc.close(); + } + + // local_datagram_socket: connect_pair() + connected-mode send()/ + // recv(), on the recycled impls churnDatagramSockets() just + // warmed. + void exerciseDatagram(io_context& ioc, capy::executor_ref ex) + { +#if BOOST_COROSIO_POSIX + local_datagram_socket a(ioc); + local_datagram_socket b(ioc); + BOOST_TEST(!connect_pair(a, b)); + + char const out = 'd'; + char in = 0; + bool sent = false, received = false; + + capy::run_async(ex)( + [](local_datagram_socket& s, char const* buf, + bool& done) -> capy::task<> { + auto [ec, n] = co_await s.send(capy::const_buffer(buf, 1)); + done = !ec && n == 1; + }(a, &out, sent)); + capy::run_async(ex)( + [](local_datagram_socket& s, char* buf, + bool& done) -> capy::task<> { + auto [ec, n] = co_await s.recv(capy::mutable_buffer(buf, 1)); + done = !ec && n == 1; + }(b, &in, received)); + ioc.run(); + ioc.restart(); + + BOOST_TEST(sent); + BOOST_TEST(received); + BOOST_TEST_EQ(in, out); + + a.close(); + b.close(); +#else + std::ignore = ioc; + std::ignore = ex; +#endif + } + + void run() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + churnStreamSockets(ioc); + churnAcceptors(ioc); + churnDatagramSockets(ioc); + + exerciseStreamAndAcceptor(ioc, ex); + exerciseDatagram(ioc, ex); + } +}; + +COROSIO_NON_IOCP_BACKEND_TESTS( + local_socket_churn_then_exercise_test, + "boost.corosio.object_reuse.churn_local") + +#if BOOST_COROSIO_POSIX +// Descriptor lane: each cycle abandons a parked read on a recycled +// descriptor impl, so a reset reuse() forgets surfaces as the poison +// pattern (debug) or a stale op on the next cycle's impl. +template +struct descriptor_churn_then_exercise_test +{ + static constexpr int cycles = 16; + + void run() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + for (int i = 0; i < cycles; ++i) + { + int fds[2]; + BOOST_TEST_EQ(::pipe(fds), 0); + std::error_code rec; + bool done = false; + char buf[4]; + { + posix_stream_descriptor d(ioc); + BOOST_TEST(!d.assign(fds[0])); + capy::run_async(ex)( + [](posix_stream_descriptor& d, + char* b, + std::error_code& ec, + bool& done) -> capy::task<> { + auto [e, n] = + co_await d.read_some(capy::mutable_buffer(b, 4)); + std::ignore = n; + ec = e; + done = true; + }(d, buf, rec, done)); + ioc.poll(); + d.cancel(); + ioc.run(); + ioc.restart(); + } + BOOST_TEST(done); + BOOST_TEST(rec == capy::cond::canceled); + ::close(fds[1]); + } + + // Closing a recycled impl that was never assigned tears down + // whatever reuse() left in its descriptor state. + { + posix_stream_descriptor unused(ioc); + unused.cancel(); + } + ioc.poll(); + ioc.restart(); + + int fds[2]; + BOOST_TEST_EQ(::pipe(fds), 0); + posix_stream_descriptor d(ioc); + BOOST_TEST(!d.assign(fds[0])); + BOOST_TEST_EQ(::write(fds[1], "abc", 3), 3); + std::size_t got = 0; + char buf[4]; + capy::run_async(ex)( + [](posix_stream_descriptor& d, + char* b, + std::size_t& got) -> capy::task<> { + auto [e, n] = co_await d.read_some(capy::mutable_buffer(b, 4)); + if (!e) + got = n; + }(d, buf, got)); + ioc.run(); + BOOST_TEST_EQ(got, 3u); + ::close(fds[1]); + } +}; + +COROSIO_NON_IOCP_BACKEND_TESTS( + descriptor_churn_then_exercise_test, + "boost.corosio.object_reuse.churn_descriptor") + +// A close hands the reference covering a still-queued reactor +// invocation to that invocation. Re-adopting a descriptor the reactor +// cannot watch, then closing it again, must not drop that reference: +// the invocation would run on an impl already recycled to a new owner. +template +struct descriptor_requeue_test +{ + void run() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + int fds[2]; + BOOST_TEST_EQ(::pipe(fds), 0); + auto d = std::make_unique(ioc); + BOOST_TEST(!d->assign(fds[0])); + BOOST_TEST_EQ(::write(fds[1], "x", 1), 1); + + // Posted ahead of the first reactor pass, so it runs after the + // pass has queued the descriptor's invocation and before that + // invocation does. + bool ran = false; + capy::run_async(ex)( + [](std::unique_ptr& d, + io_context& ioc, + bool& ran) -> capy::task<> { + d->close(); + int nul = ::open("/dev/null", O_RDONLY | O_CLOEXEC); + std::ignore = d->assign(nul); + d.reset(); + // Pops the recycled impl if the invocation lost its + // reference; its reuse() asserts nothing is queued. + posix_stream_descriptor fresh(ioc); + ran = true; + co_return; + }(d, ioc, ran)); + ioc.run(); + BOOST_TEST(ran); + ::close(fds[1]); + } +}; + +COROSIO_NON_IOCP_BACKEND_TESTS( + descriptor_requeue_test, "boost.corosio.object_reuse.descriptor_requeue") +#endif + +} // namespace boost::corosio diff --git a/test/unit/resolver.cpp b/test/unit/resolver.cpp index 85ea2d2db..c5476c8c2 100644 --- a/test/unit/resolver.cpp +++ b/test/unit/resolver.cpp @@ -1297,7 +1297,7 @@ struct resolver_test // Forward resolution on IOCP dispatches through GetAddrInfoExW and // posts its completion (resolve_op, embedded in the win_resolver) to // the scheduler queue. Unlike the reverse path and both POSIX paths, - // the forward op takes no shared_from_this()/impl_ptr keepalive, so + // the forward op takes no object_ref_ keepalive, so // destroying the resolver frees the win_resolver the queued // resolve_op lives in before teardown drains that op -- a // use-after-free that ASan catches. localhost resolves from the hosts @@ -1310,7 +1310,7 @@ struct resolver_test // only under ASan; without a sanitizer the freed read is silent. // // Contrast the reverse path, which holds the keepalive on - // reverse_op_.impl_ptr across the queued completion + // reverse_op_.object_ref_ across the queued completion // (win_resolver_service.hpp do_reverse_resolve_work / do_complete), // and so survives teardown intact. void testDestroyWithForwardResolveQueued() diff --git a/test/unit/signal_set.cpp b/test/unit/signal_set.cpp index 487bfcfb6..32bdd1441 100644 --- a/test/unit/signal_set.cpp +++ b/test/unit/signal_set.cpp @@ -680,6 +680,32 @@ struct signal_set_test BOOST_TEST_PASS(); } + // A wait still parked when its context is torn down is abandoned, + // not resumed, even when a frame destroyed during the teardown owns + // the set and its destructor cancels the wait. Only IOCP can fail + // this: it resumes a cancelled wait inline, where POSIX posts it to a + // scheduler that has already drained. + void testTeardownDoesNotResumeParkedWait() + { + bool resumed = false; + { + io_context ioc(Backend); + auto owner = [](io_context& c, bool& r) -> capy::task<> { + signal_set s(c, SIGINT); + capy::run_async(c.get_executor())( + [](signal_set& set, bool& out) -> capy::task<> { + std::ignore = co_await set.wait(); + out = true; + }(s, r)); + std::ignore = co_await corosio::delay(std::chrono::hours(1)); + }; + capy::run_async(ioc.get_executor())(owner(ioc, resumed)); + std::ignore = ioc.poll(); // owner parks on the delay, waiter on s + BOOST_TEST(!resumed); + } + BOOST_TEST(!resumed); + } + // The process signal table outlives every io_context, so a set still // registered when its context shuts down -- which an abandoned frame // is the only way to arrange -- has to hand the registration back @@ -1473,6 +1499,7 @@ struct signal_set_test #if !COROSIO_TEST_HAS_ASAN // Abandon parked coroutine frames by design; see context.hpp. testShutdownReleasesRegistration(); + testTeardownDoesNotResumeParkedWait(); testShutdownKeepsOtherContextRegistered(); #endif } diff --git a/test/unit/tcp_acceptor.cpp b/test/unit/tcp_acceptor.cpp index d993c9f95..43b323392 100644 --- a/test/unit/tcp_acceptor.cpp +++ b/test/unit/tcp_acceptor.cpp @@ -146,6 +146,20 @@ native_connect_loopback(native_handle_type h, std::uint16_t port, bool v6) #endif } +// Write one byte without the io_context, so the peer's completion is +// already waiting when the next run starts. Unused when the tests that +// abandon parked frames are compiled out. +[[maybe_unused]] bool +native_send_byte(native_handle_type h) +{ + char const b = 'x'; +#if BOOST_COROSIO_HAS_IOCP + return ::send(static_cast(h), &b, 1, 0) == 1; +#else + return ::send(static_cast(h), &b, 1, 0) == 1; +#endif +} + // Take ownership of a released listener the way a caller would: clear // the non-blocking flag the library set, then accept. native_handle_type @@ -266,8 +280,8 @@ struct tcp_acceptor_test void testCancelAccept() { // Tests that cancel() properly cancels a pending accept operation. - // This exercises the acceptor_ptr shared_ptr that keeps the - // acceptor impl alive until IOCP delivers the cancellation. + // The accept op's keepalive must hold the acceptor impl until + // IOCP delivers the cancellation. io_context ioc(Backend); tcp_acceptor acc(ioc); BOOST_TEST(!acc.open()); @@ -455,13 +469,100 @@ struct tcp_acceptor_test BOOST_TEST_PASS(); } + void testDestroyListeningAcceptorKeepsOtherOps() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + auto sockets = test::make_socket_pair(ioc); + auto& server = sockets.first; + auto& client = sockets.second; + + bool read_done = false; + char buf[8]; + capy::run_async(ex)( + [](tcp_socket& s, char* b, bool& done) -> capy::task<> { + auto [ec, n] = + co_await s.read_some(capy::mutable_buffer(b, 8)); + done = !ec && n == 1; + }(server, buf, read_done)); + ioc.restart(); + std::ignore = ioc.poll(); + BOOST_TEST(!read_done); + + { + tcp_acceptor acc(ioc); + BOOST_TEST(!acc.open()); + BOOST_TEST(!acc.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!acc.listen()); + std::ignore = ioc.poll(); + + // The read's completion is now pending when the acceptor + // goes away; tearing it down must not consume it. + BOOST_TEST(native_send_byte(client.native_handle())); + } + + ioc.restart(); + while (!read_done && ioc.run_one_for(std::chrono::seconds(5)) != 0) + { + } + BOOST_TEST(read_done); + } + + // An accept that completes inline under a stoppable token must not + // leave its stop callback registered on an impl the pool recycles. + void testInlineAcceptUnderStopTokenRecycles() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + std::stop_source src; + { + tcp_acceptor acc(ioc); + BOOST_TEST(!acc.open()); + acc.set_option(socket_option::reuse_address(true)); + BOOST_TEST(!acc.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!acc.listen()); + endpoint ep(ipv4_address::loopback(), acc.local_endpoint().port()); + + // Both connections are pending before the accepts start, so + // the second accept completes without parking. + tcp_socket c1(ioc); + tcp_socket c2(ioc); + for (auto* c : {&c1, &c2}) + capy::run_async(ex)( + [](tcp_socket& s, endpoint e) -> capy::task<> { + std::ignore = co_await s.connect(e); + }(*c, ep)); + ioc.run(); + ioc.restart(); + + tcp_socket s1(ioc); + tcp_socket s2(ioc); + bool accepted = false; + capy::run_async(ex, src.get_token())( + [](tcp_acceptor& a, tcp_socket& x, tcp_socket& y, + bool& done) -> capy::task<> { + auto [ec1] = co_await a.accept(x); + auto [ec2] = co_await a.accept(y); + done = !ec1 && !ec2; + }(acc, s1, s2, accepted)); + ioc.run(); + ioc.restart(); + BOOST_TEST(accepted); + } + + // Pops the recycled impl; reuse() requires no armed stop callback. + tcp_acceptor acc2(ioc); + BOOST_TEST(!acc2.is_open()); + } + void testCloseWhilePendingAccept() { // Tests that close() properly handles a pending accept operation. // This is the key test for the cancel/destruction race condition: // when close() is called, CancelIoEx is invoked, the tcp_socket is closed, // but the impl must stay alive until IOCP delivers the cancellation. - // The acceptor_ptr shared_ptr in accept_op ensures this. + // The accept op's keepalive ensures this. io_context ioc(Backend); tcp_acceptor acc(ioc); BOOST_TEST(!acc.open()); @@ -1866,6 +1967,8 @@ struct tcp_acceptor_test // Abandon parked coroutine frames by design; see context.hpp. testDestroyWithParkedAccept(); testDestroyWithParkedRead(); + testInlineAcceptUnderStopTokenRecycles(); + testDestroyListeningAcceptorKeepsOtherOps(); testDestroyWithQueuedAccept(); testDestroyWithQueuedAcceptorWait(); #endif diff --git a/test/unit/zero_alloc.cpp b/test/unit/zero_alloc.cpp new file mode 100644 index 000000000..3fdce588b --- /dev/null +++ b/test/unit/zero_alloc.cpp @@ -0,0 +1,529 @@ +// +// Copyright (c) 2026 Steve Gerbino +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// Official repository: https://github.com/cppalliance/corosio +// + +// Acceptance gate for the object_pool conversion: steady-state +// construct()/destroy() and the accept/connect/read/write dispatch +// path must not call the global allocator once the per-service free +// list has been warmed. Counts against alloc_counter.hpp's shared +// interposer -- see that header for why this TU does not define its +// own operator new/delete. +// +// Excluded under ASan: ASan instruments its own replacement of the +// global allocation functions, and a second counting layer on top of +// that does not reliably observe the same calls. Excluded under TSan +// for the same reason alloc_counter.hpp's counters don't exist there +// (its runtime replaces operator new/delete itself). +#include "context.hpp" // COROSIO_TEST_HAS_ASAN +#include "alloc_counter.hpp" // COROSIO_TEST_HAS_TSAN, alloc_armed/alloc_count + +#if !COROSIO_TEST_HAS_ASAN && !defined(COROSIO_TEST_HAS_TSAN) + +#include +#include +#include +#include +#include +#include +#include +#include +#if BOOST_COROSIO_POSIX +#include +#include +#endif + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "temp_path.hpp" +#include "test_suite.hpp" + +namespace boost::corosio { + +namespace { + +// Cycles warmed with counting off, before the assertion window opens. +constexpr int warmup_cycles = 8; + +// Cycles measured with counting on. +constexpr int measured_cycles = 8; + +} // namespace + +template +struct zero_alloc_test +{ + // One accept -> echo one buffer -> close round trip. `server` is + // constructed and destroyed by the caller on every cycle -- that + // construct()/destroy() pair is the thing under test, so it must + // not live across cycles the way `acc` and `client` do. + void runCycle( + io_context& ioc, + capy::executor_ref ex, + tcp_acceptor& acc, + tcp_socket& client, + endpoint ep) + { + tcp_socket server(ioc); + + bool accepted = false, read_ok = false; + bool connected = false, write_ok = false; + char in = 0; + char const out = 'x'; + + // Server side: accept into the reused socket, then echo the + // one byte the client sends back to it. + capy::run_async(ex)( + [](tcp_acceptor& a, tcp_socket& s, char* b, bool& got_accept, + bool& got_read) -> capy::task<> { + auto [ec1] = co_await a.accept(s); + got_accept = !ec1; + auto [ec2, n2] = co_await s.read_some( + capy::mutable_buffer(b, 1)); + got_read = !ec2 && n2 == 1; + }(acc, server, &in, accepted, read_ok)); + + // Client side: connect, then send the one byte. A failed connect + // leaves nothing for the accept to receive. + capy::run_async(ex)( + [](tcp_socket& s, tcp_acceptor& a, endpoint ep_, char const* b, + bool& got_connect, bool& got_write) -> capy::task<> { + auto [ec1] = co_await s.connect(ep_); + got_connect = !ec1; + if (ec1) + { + a.cancel(); + co_return; + } + auto [ec2, n2] = co_await s.write_some( + capy::const_buffer(b, 1)); + got_write = !ec2 && n2 == 1; + }(client, acc, ep, &out, connected, write_ok)); + + ioc.run(); + ioc.restart(); + + BOOST_TEST(accepted); + BOOST_TEST(connected); + BOOST_TEST(write_ok); + BOOST_TEST(read_ok); + BOOST_TEST_EQ(in, out); + + client.close(); + + // server destructs here -- recycles its impl for the next + // cycle's construct() instead of freeing it. + } + + void testAcceptEchoCloseIsZeroAlloc() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + tcp_acceptor acc(ioc); + BOOST_TEST(!acc.open()); + acc.set_option(socket_option::reuse_address(true)); + BOOST_TEST(!acc.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!acc.listen()); + auto port = acc.local_endpoint().port(); + endpoint ep(ipv4_address::loopback(), port); + + // client is pre-created and outlives every cycle: only the + // accept side's socket construct()/destroy() is under the + // gate here. + tcp_socket client(ioc); + + for (int i = 0; i < warmup_cycles; ++i) + runCycle(ioc, ex, acc, client, ep); + + alloc_count.store(0, std::memory_order_relaxed); + alloc_armed.store(true, std::memory_order_relaxed); + for (int i = 0; i < measured_cycles; ++i) + runCycle(ioc, ex, acc, client, ep); + alloc_armed.store(false, std::memory_order_relaxed); + + // epoll/kqueue: the reactor dispatch path (accept, connect, + // read, write, close) and the recycled server-socket impl are + // both allocation-free once warmed. + // + // select: registered_descs_ (select_scheduler.hpp) is a + // std::map keyed by fd; register_descriptor()/ + // deregister_descriptor() insert/erase a node for each of the + // 2 fds this cycle creates (the accepted server fd, the + // reconnected client fd) every cycle, regardless of impl + // recycling; 2 allocations/cycle pins that known cost. + // + // uring: the multishot acceptor's accept nodes and ready-fd + // nodes recycle through per-acceptor free lists + // (uring_multishot_acceptor.hpp), so warmed accept churn is + // allocation-free like epoll. + // + // Either way, pinning the exact number (0, or the documented + // non-zero count) means a regression or an improvement both + // trip this assert instead of silently drifting. +#if BOOST_COROSIO_HAS_SELECT + if constexpr ( + std::is_same_v, select_t>) + BOOST_TEST_EQ( + alloc_count.load(std::memory_order_relaxed), + static_cast(2 * measured_cycles)); + else +#endif + BOOST_TEST_EQ(alloc_count.load(std::memory_order_relaxed), 0LL); + } + + // One acceptor reopened and re-listened each cycle. On uring each + // listen arms a multishot accept and each close cancels it; the + // ended armings recycle through the acceptor once their terminal + // completions are reaped. + void listenCycle(io_context& ioc, capy::executor_ref ex, tcp_acceptor& acc) + { + BOOST_TEST(!acc.open()); + BOOST_TEST(!acc.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!acc.listen()); + acc.close(); + for (int i = 0; i < 4; ++i) + { + capy::run_async(ex)([]() -> capy::task<> { co_return; }()); + ioc.restart(); + ioc.poll(); + } + } + + void testListenChurnIsZeroAlloc() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + tcp_acceptor acc(ioc); + + for (int i = 0; i < warmup_cycles; ++i) + listenCycle(ioc, ex, acc); + + alloc_count.store(0, std::memory_order_relaxed); + alloc_armed.store(true, std::memory_order_relaxed); + for (int i = 0; i < measured_cycles; ++i) + listenCycle(ioc, ex, acc); + alloc_armed.store(false, std::memory_order_relaxed); + + // select pays its known map node for the one fd each cycle + // registers; see testAcceptEchoCloseIsZeroAlloc. +#if BOOST_COROSIO_HAS_SELECT + if constexpr ( + std::is_same_v, select_t>) + BOOST_TEST_EQ( + alloc_count.load(std::memory_order_relaxed), + static_cast(measured_cycles)); + else +#endif + BOOST_TEST_EQ(alloc_count.load(std::memory_order_relaxed), 0LL); + } + + // Reuse identity at the tcp_socket level: once the pool has one + // spare impl, a construct()/destroy()/construct() sequence pops + // and recycles that same impl both times instead of allocating. +#if BOOST_COROSIO_POSIX + // A descriptor adopted, read through once, and destroyed: the + // recycled impl and its embedded ops make the cycle allocation-free. + void descriptorCycle(io_context& ioc, capy::executor_ref ex, int (&fds)[2]) + { + posix_stream_descriptor d(ioc); + BOOST_TEST(!d.assign(::dup(fds[0]))); + BOOST_TEST_EQ(::write(fds[1], "x", 1), 1); + bool got = false; + char b = 0; + capy::run_async(ex)( + [](posix_stream_descriptor& d, char* b, bool& got) -> capy::task<> { + auto [ec, n] = co_await d.read_some(capy::mutable_buffer(b, 1)); + got = !ec && n == 1; + }(d, &b, got)); + ioc.run(); + ioc.restart(); + BOOST_TEST(got); + } + + void testDescriptorChurnIsZeroAlloc() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + int fds[2]; + BOOST_TEST_EQ(::pipe(fds), 0); + + for (int i = 0; i < warmup_cycles; ++i) + descriptorCycle(ioc, ex, fds); + + alloc_count.store(0, std::memory_order_relaxed); + alloc_armed.store(true, std::memory_order_relaxed); + for (int i = 0; i < measured_cycles; ++i) + descriptorCycle(ioc, ex, fds); + alloc_armed.store(false, std::memory_order_relaxed); + + // select pays its known map node for the one fd each cycle + // registers; see testAcceptEchoCloseIsZeroAlloc. +#if BOOST_COROSIO_HAS_SELECT + if constexpr ( + std::is_same_v, select_t>) + BOOST_TEST_EQ( + alloc_count.load(std::memory_order_relaxed), + static_cast(measured_cycles)); + else +#endif + BOOST_TEST_EQ(alloc_count.load(std::memory_order_relaxed), 0LL); + ::close(fds[0]); + ::close(fds[1]); + } +#endif + + void testReuseIdentityIsZeroAlloc() + { + io_context ioc(Backend); + + { + tcp_socket warm(ioc); // first-ever construct: pool is cold + } // destroy() recycles it + + alloc_count.store(0, std::memory_order_relaxed); + alloc_armed.store(true, std::memory_order_relaxed); + { + tcp_socket s1(ioc); // pop from free_ + } // destroy() -> recycle + { + tcp_socket s2(ioc); // pop the same recycled impl again + } + alloc_armed.store(false, std::memory_order_relaxed); + + BOOST_TEST_EQ(alloc_count.load(std::memory_order_relaxed), 0LL); + } + + // Timer churn: the thread-local single-slot tier plus capy's + // frame-embedded waiter should make a delay() await literally + // zero-alloc on every backend, including select, whose socket + // path has a known per-fd cost. The delay must be non-zero: a + // zero delay completes inline without touching the timer service. + void testDelayChurnIsZeroAlloc() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + auto one_delay = [&] { + bool done = false; + capy::run_async(ex)([](bool& d) -> capy::task<> { + auto [ec] = co_await delay(std::chrono::milliseconds(1)); + d = !ec; + }(done)); + ioc.run(); + ioc.restart(); + BOOST_TEST(done); + }; + + // A wait that finds its timer already expired never touches + // the expiry heap, so a delay alone may never grow it during + // warmup. One cancelled long wait grows it deterministically. + { + std::stop_source stop; + capy::run_async(ex, stop.get_token())([]() -> capy::task<> { + std::ignore = co_await delay(std::chrono::hours(1)); + }()); + std::ignore = ioc.poll(); + stop.request_stop(); + ioc.run(); + ioc.restart(); + } + + // Warms both the timer_service pool and capy's coroutine + // frame pool -- the latter's own first-use allocation would + // otherwise show up as noise in the counted window below. + for (int i = 0; i < warmup_cycles; ++i) + one_delay(); + + alloc_count.store(0, std::memory_order_relaxed); + alloc_armed.store(true, std::memory_order_relaxed); + for (int i = 0; i < measured_cycles; ++i) + one_delay(); + alloc_armed.store(false, std::memory_order_relaxed); + + BOOST_TEST_EQ(alloc_count.load(std::memory_order_relaxed), 0LL); + } + + // Random-access file churn: each read_some_at/write_some_at used + // to heap-allocate its op; both now recycle through the file's + // free lists, so a warmed write+read round trip is zero-alloc on + // every backend (posix thread-pool path and native uring path). + void testRafChurnIsZeroAlloc() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + test::temp_file tmp("zero_alloc_raf_", "warm"); + random_access_file f(ioc); + BOOST_TEST(!f.open(tmp.path, file_base::read_write)); + + char buf[4] = {}; + auto one_rw = [&] { + bool wrote = false, read = false; + capy::run_async(ex)( + [](random_access_file& file, char* b, bool& w, + bool& r) -> capy::task<> { + auto [ec1, n1] = co_await file.write_some_at( + 0, capy::const_buffer("abcd", 4)); + w = !ec1 && n1 == 4; + auto [ec2, n2] = co_await file.read_some_at( + 0, capy::mutable_buffer(b, 4)); + r = !ec2 && n2 == 4; + }(f, buf, wrote, read)); + ioc.run(); + ioc.restart(); + BOOST_TEST(wrote); + BOOST_TEST(read); + }; + + for (int i = 0; i < warmup_cycles; ++i) + one_rw(); + + alloc_count.store(0, std::memory_order_relaxed); + alloc_armed.store(true, std::memory_order_relaxed); + for (int i = 0; i < measured_cycles; ++i) + one_rw(); + alloc_armed.store(false, std::memory_order_relaxed); + + BOOST_TEST_EQ(alloc_count.load(std::memory_order_relaxed), 0LL); + } + + // tcp_server connection churn: launching a worker used to cost a + // fresh stop_source state, a globally allocated launch_wrapper + // frame, and a heap wrapper op for the bare-handle post. All + // three now recycle (reused stop state, frame_alloc_mixin, + // frame-embedded continuation), so a warmed connect/echo/close + // round trip through the server allocates nothing. Select keeps + // its documented 2/cycle registered_descs_ cost. + void testServerChurnIsZeroAlloc() + { + struct echo_worker : tcp_server::worker_base + { + io_context& ctx; + tcp_socket sock; + explicit echo_worker(io_context& c) : ctx(c), sock(c) {} + tcp_socket& socket() override + { + return sock; + } + void run(tcp_server::launcher launch) override + { + launch( + ctx.get_executor(), + [](tcp_socket* s) -> capy::task<> + { + char b[8]; + auto [ec, n] = co_await s->read_some( + capy::mutable_buffer(b, sizeof(b))); + if (!ec) + std::ignore = co_await s->write_some( + capy::const_buffer(b, n)); + s->close(); + }(&sock)); + } + }; + + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + tcp_server srv(ioc, ex); + { + std::vector> v; + for (int i = 0; i < 2; ++i) + v.push_back(std::make_unique(ioc)); + srv.set_workers(std::move(v)); + } + BOOST_TEST(!srv.bind(endpoint(ipv4_address::loopback(), 0))); + auto port = srv.local_endpoint().port(); + endpoint ep(ipv4_address::loopback(), port); + srv.start(); + + auto one_conn = [&] { + bool echoed = false; + capy::run_async(ex)( + [](io_context& c, endpoint target, bool& ok) -> capy::task<> + { + tcp_socket s(c); + auto [ec1] = co_await s.connect(target); + if (ec1) + co_return; + char const out = 'z'; + auto [ec2, n2] = + co_await s.write_some(capy::const_buffer(&out, 1)); + if (ec2 || n2 != 1) + co_return; + char in = 0; + auto [ec3, n3] = + co_await s.read_some(capy::mutable_buffer(&in, 1)); + ok = !ec3 && n3 == 1 && in == out; + }(ioc, ep, echoed)); + // Pump one event at a time: the server's pending accept + // keeps the context busy forever, so a plain run() (or a + // full run_for window) would never return promptly. + while (!echoed && ioc.run_one_for(std::chrono::seconds(5)) != 0) + { + } + ioc.restart(); + BOOST_TEST(echoed); + }; + + for (int i = 0; i < warmup_cycles; ++i) + one_conn(); + + alloc_count.store(0, std::memory_order_relaxed); + alloc_armed.store(true, std::memory_order_relaxed); + for (int i = 0; i < measured_cycles; ++i) + one_conn(); + alloc_armed.store(false, std::memory_order_relaxed); + +#if BOOST_COROSIO_HAS_SELECT + if constexpr ( + std::is_same_v, select_t>) + BOOST_TEST_EQ( + alloc_count.load(std::memory_order_relaxed), + static_cast(2 * measured_cycles)); + else +#endif + BOOST_TEST_EQ(alloc_count.load(std::memory_order_relaxed), 0LL); + + srv.stop(); + ioc.run(); + } + + void run() + { + if (!alloc_counter_is_live("boost.corosio.zero_alloc")) + return; + testAcceptEchoCloseIsZeroAlloc(); + testReuseIdentityIsZeroAlloc(); +#if BOOST_COROSIO_POSIX + testDescriptorChurnIsZeroAlloc(); +#endif + testListenChurnIsZeroAlloc(); + testDelayChurnIsZeroAlloc(); + testRafChurnIsZeroAlloc(); + testServerChurnIsZeroAlloc(); + } +}; + +COROSIO_TEST_EPOLL_(zero_alloc_test, "boost.corosio.zero_alloc") +COROSIO_TEST_KQUEUE_(zero_alloc_test, "boost.corosio.zero_alloc") +COROSIO_TEST_SELECT_(zero_alloc_test, "boost.corosio.zero_alloc") +COROSIO_TEST_URING_(zero_alloc_test, "boost.corosio.zero_alloc") + +} // namespace boost::corosio + +#endif // !COROSIO_TEST_HAS_ASAN && !COROSIO_TEST_HAS_TSAN From e367040c4242159221a3a78a23fc39dd84aed440 Mon Sep 17 00:00:00 2001 From: Steve Gerbino Date: Fri, 9 Oct 2026 18:50:32 +0200 Subject: [PATCH 3/8] fix: close lifetime and cancellation races in operations and streams An awaiter is resumed through the continuation its awaitable holds, on its own context rather than the one that completed it. A file keeps its descriptor number allocated while a pool worker uses it, and adopting a descriptor is refused in the unsafe locking tier. Timers shut down before the scheduler drains, a drain's writes are ordered before a lock-free cancel trusts them, a stop request on another thread can race timer teardown, and timeout()'s detached timer coroutine is allocated outside the awaiting chain. Signals are delivered outside the signal pipe's lock. A TLS stream can be destroyed or moved with an operation in flight, and a wolfSSL shutdown finishes when its close_notify arrives behind another record. An accepted peer whose awaiter never resumes is closed, including one a drained reactor accept already took. A socket or mocket pair whose connect is refused fails. --- .../boost/corosio/detail/dispatch_coro.hpp | 14 +- include/boost/corosio/detail/object_ref.hpp | 18 + include/boost/corosio/detail/op_base.hpp | 26 +- include/boost/corosio/detail/scheduler.hpp | 36 ++ .../corosio/detail/timeout_awaitable.hpp | 13 +- include/boost/corosio/detail/timeout_coro.hpp | 7 +- include/boost/corosio/detail/timer.hpp | 7 +- .../boost/corosio/detail/timer_service.hpp | 92 +++- include/boost/corosio/io/io_object.hpp | 19 + include/boost/corosio/io/io_read_stream.hpp | 11 +- include/boost/corosio/io/io_signal_set.hpp | 11 +- include/boost/corosio/io/io_stream.hpp | 26 +- include/boost/corosio/io/io_write_stream.hpp | 11 +- include/boost/corosio/io_context.hpp | 5 +- .../boost/corosio/local_datagram_socket.hpp | 56 +- .../boost/corosio/local_stream_acceptor.hpp | 51 +- include/boost/corosio/local_stream_socket.hpp | 20 +- .../boost/corosio/native/detail/coro_op.hpp | 7 +- .../native/detail/coro_op_complete.hpp | 3 +- .../native/detail/iocp/win_file_service.hpp | 9 +- .../detail/iocp/win_local_stream_acceptor.hpp | 5 +- .../win_local_stream_acceptor_service.hpp | 12 +- .../detail/iocp/win_local_stream_service.hpp | 16 +- .../detail/iocp/win_local_stream_socket.hpp | 9 +- .../detail/iocp/win_object_handle_service.hpp | 7 +- .../detail/iocp/win_overlapped_handle.hpp | 24 +- .../native/detail/iocp/win_overlapped_op.hpp | 7 +- .../iocp/win_random_access_file_service.hpp | 1 + .../native/detail/iocp/win_resolver.hpp | 5 +- .../detail/iocp/win_resolver_service.hpp | 17 +- .../native/detail/iocp/win_scheduler.hpp | 2 + .../corosio/native/detail/iocp/win_signal.hpp | 9 +- .../native/detail/iocp/win_signals.hpp | 44 +- .../native/detail/iocp/win_stream_file.hpp | 5 +- .../detail/iocp/win_stream_handle_service.hpp | 8 +- .../native/detail/iocp/win_tcp_acceptor.hpp | 5 +- .../detail/iocp/win_tcp_acceptor_service.hpp | 12 +- .../native/detail/iocp/win_tcp_service.hpp | 17 +- .../native/detail/iocp/win_tcp_socket.hpp | 9 +- .../native/detail/iocp/win_udp_service.hpp | 24 +- .../native/detail/iocp/win_udp_socket.hpp | 13 +- .../native/detail/iocp/win_wait_reactor.hpp | 2 +- .../native/detail/posix/posix_fd_gate.hpp | 137 +++++ .../detail/posix/posix_random_access_file.hpp | 33 +- .../posix_random_access_file_service.hpp | 49 +- .../native/detail/posix/posix_resolver.hpp | 5 +- .../detail/posix/posix_resolver_service.hpp | 26 +- .../native/detail/posix/posix_signal.hpp | 9 +- .../detail/posix/posix_signal_service.hpp | 62 ++- .../native/detail/posix/posix_stream_file.hpp | 38 +- .../posix/posix_stream_file_service.hpp | 157 ++++-- .../detail/reactor/reactor_acceptor.hpp | 12 +- .../native/detail/reactor/reactor_backend.hpp | 7 +- .../reactor/reactor_datagram_socket.hpp | 93 ++-- .../detail/reactor/reactor_descriptor.hpp | 42 +- .../native/detail/reactor/reactor_op.hpp | 12 + .../detail/reactor/reactor_scheduler.hpp | 20 +- .../detail/reactor/reactor_signal_pipe.hpp | 48 +- .../detail/reactor/reactor_socket_finals.hpp | 2 +- .../detail/reactor/reactor_stream_socket.hpp | 69 ++- .../native/detail/select/select_scheduler.hpp | 6 + .../detail/uring/uring_acceptor_ops.hpp | 6 +- .../native/detail/uring/uring_descriptor.hpp | 13 +- .../native/detail/uring/uring_dgram_ops.hpp | 8 +- .../native/detail/uring/uring_file_ops.hpp | 20 +- .../detail/uring/uring_multishot_acceptor.hpp | 8 +- .../detail/uring/uring_random_access_file.hpp | 8 +- .../native/detail/uring/uring_scheduler.hpp | 6 + .../native/detail/uring/uring_socket_ops.hpp | 20 +- .../native/detail/uring/uring_stream_file.hpp | 16 +- .../native/detail/uring/uring_types.hpp | 239 ++++---- .../native/native_local_datagram_socket.hpp | 25 +- .../native/native_local_stream_acceptor.hpp | 12 +- .../native/native_local_stream_socket.hpp | 17 +- .../native/native_posix_stream_descriptor.hpp | 13 +- .../native/native_random_access_file.hpp | 12 +- .../boost/corosio/native/native_resolver.hpp | 8 +- .../corosio/native/native_signal_set.hpp | 4 +- .../corosio/native/native_stream_file.hpp | 8 +- .../corosio/native/native_tcp_acceptor.hpp | 12 +- .../corosio/native/native_tcp_socket.hpp | 17 +- .../corosio/native/native_udp_socket.hpp | 25 +- .../native/native_win_object_handle.hpp | 7 +- .../native_win_random_access_handle.hpp | 12 +- .../native/native_win_stream_handle.hpp | 8 +- include/boost/corosio/openssl_stream.hpp | 59 +- .../boost/corosio/posix_stream_descriptor.hpp | 11 +- include/boost/corosio/random_access_file.hpp | 24 +- include/boost/corosio/resolver.hpp | 24 +- include/boost/corosio/stream_file.hpp | 12 +- include/boost/corosio/tcp_acceptor.hpp | 51 +- include/boost/corosio/tcp_socket.hpp | 20 +- include/boost/corosio/test/mocket.hpp | 25 +- include/boost/corosio/test/socket_pair.hpp | 25 +- include/boost/corosio/tls_stream.hpp | 68 +++ include/boost/corosio/udp_socket.hpp | 56 +- include/boost/corosio/win_object_handle.hpp | 8 +- .../corosio/win_random_access_handle.hpp | 12 +- include/boost/corosio/wolfssl_stream.hpp | 59 +- src/corosio/src/local_stream_acceptor.cpp | 14 + src/corosio/src/tcp_acceptor.cpp | 25 + src/corosio/src/tls/detail/engine_driver.hpp | 293 +++++++++- src/openssl/src/openssl_stream.cpp | 67 ++- src/wolfssl/src/detail/engine.cpp | 38 ++ src/wolfssl/src/wolfssl_stream.cpp | 67 ++- test/unit/delay.cpp | 172 ++++++ test/unit/fault/fault.hpp | 27 + test/unit/fault/fault_posix.cpp | 116 +++- test/unit/fault/posix_faults.cpp | 113 ++++ test/unit/local_stream_socket.cpp | 59 ++ .../unit/native/uring/descriptor_continue.cpp | 8 +- test/unit/object_pool.cpp | 1 + test/unit/object_ref.cpp | 16 + test/unit/object_reuse.cpp | 67 +++ test/unit/openssl_stream.cpp | 16 + test/unit/random_access_file.cpp | 98 ++++ test/unit/signal_set.cpp | 201 +++++++ test/unit/socket_stress.cpp | 14 + test/unit/stream_file.cpp | 24 + test/unit/tcp_acceptor.cpp | 170 ++++++ test/unit/tcp_socket.cpp | 43 ++ test/unit/timeout.cpp | 48 ++ test/unit/tls_stream_tests.hpp | 516 ++++++++++++++++++ test/unit/udp_socket.cpp | 48 ++ test/unit/wolfssl_stream.cpp | 16 + 125 files changed, 3758 insertions(+), 1019 deletions(-) create mode 100644 include/boost/corosio/native/detail/posix/posix_fd_gate.hpp diff --git a/include/boost/corosio/detail/dispatch_coro.hpp b/include/boost/corosio/detail/dispatch_coro.hpp index c4d9f05dc..1ef1b9576 100644 --- a/include/boost/corosio/detail/dispatch_coro.hpp +++ b/include/boost/corosio/detail/dispatch_coro.hpp @@ -21,9 +21,11 @@ namespace boost::corosio::detail { /** Returns a handle for symmetric transfer on I/O completion. - If the executor is io_context::executor_type, returns `c.h` - directly (fast path). Otherwise dispatches through the - executor, which returns `c.h` or `noop_coroutine()`. + Returns `c.h` when the executor's context is running on the + calling thread, so the awaiter resumes by symmetric transfer. + Otherwise the executor queues `c` and `noop_coroutine()` is + returned. An `io_context` executor is dispatched without type + erasure. Callers in coroutine machinery should return the result for symmetric transfer. Callers at the scheduler pump @@ -38,8 +40,10 @@ namespace boost::corosio::detail { inline std::coroutine_handle<> dispatch_coro(capy::executor_ref ex, capy::continuation& c) { - if (ex.target() != nullptr) - return c.h; + // A completion from another context must not resume the awaiter + // on this thread, so the fast path still asks the executor. + if (auto const* io_ex = ex.target()) + return io_ex->dispatch(c); return ex.dispatch(c); } diff --git a/include/boost/corosio/detail/object_ref.hpp b/include/boost/corosio/detail/object_ref.hpp index eaa56972c..6d5dca557 100644 --- a/include/boost/corosio/detail/object_ref.hpp +++ b/include/boost/corosio/detail/object_ref.hpp @@ -103,6 +103,24 @@ class object_ref acquire(p_); } + /** Create a reference to `p` unless its count already reached zero. + + The way to take a reference from a raw pointer: a context that + holds none may find the count at zero, and must not revive it. + + @param p The implementation to reference. + + @return A reference to `p`, or an empty `object_ref` if `p` is + retiring. + */ + static object_ref try_from(io_object::implementation* p) noexcept + { + object_ref r; + if (try_acquire(p)) + r.p_ = p; + return r; + } + /// Construct a copy, acquiring another reference to `other`'s target. object_ref(object_ref const& other) noexcept : p_(other.p_) { diff --git a/include/boost/corosio/detail/op_base.hpp b/include/boost/corosio/detail/op_base.hpp index cf854fb48..796614d9b 100644 --- a/include/boost/corosio/detail/op_base.hpp +++ b/include/boost/corosio/detail/op_base.hpp @@ -11,6 +11,7 @@ #define BOOST_COROSIO_DETAIL_OP_BASE_HPP #include +#include #include #include #include @@ -27,7 +28,7 @@ namespace boost::corosio::detail { Derived classes must provide: std::coroutine_handle<> dispatch( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex) const; which forwards to the backend implementation method, passing @@ -43,6 +44,9 @@ class bytes_op_base std::stop_token token_; mutable std::error_code ec_; mutable std::size_t bytes_ = 0; + // Lives in the awaiting frame until resumption, so a completion + // posted through an executor never shares the object's state. + mutable capy::continuation cont_; bool await_ready() const noexcept { @@ -69,7 +73,9 @@ class bytes_op_base ec_ = capy::error::canceled; return h; } - return static_cast(this)->dispatch(h, env->executor); + cont_.h = h; + return static_cast(this)->dispatch( + cont_, env->executor); } }; @@ -79,7 +85,7 @@ class bytes_op_base Derived classes must provide: std::coroutine_handle<> dispatch( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex) const; which forwards to the backend implementation method, passing @@ -95,6 +101,8 @@ class value_op_base std::stop_token token_; mutable std::error_code ec_; mutable Value value_{}; + // See bytes_op_base::cont_. + mutable capy::continuation cont_; bool await_ready() const noexcept { @@ -119,7 +127,9 @@ class value_op_base ec_ = capy::error::canceled; return h; } - return static_cast(this)->dispatch(h, env->executor); + cont_.h = h; + return static_cast(this)->dispatch( + cont_, env->executor); } }; @@ -128,7 +138,7 @@ class value_op_base Derived classes must provide: std::coroutine_handle<> dispatch( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex) const; which forwards to the backend implementation method, passing @@ -143,6 +153,8 @@ class void_op_base public: std::stop_token token_; mutable std::error_code ec_; + // See bytes_op_base::cont_. + mutable capy::continuation cont_; bool await_ready() const noexcept { @@ -167,7 +179,9 @@ class void_op_base ec_ = capy::error::canceled; return h; } - return static_cast(this)->dispatch(h, env->executor); + cont_.h = h; + return static_cast(this)->dispatch( + cont_, env->executor); } }; diff --git a/include/boost/corosio/detail/scheduler.hpp b/include/boost/corosio/detail/scheduler.hpp index 5e946d6d6..08c76725d 100644 --- a/include/boost/corosio/detail/scheduler.hpp +++ b/include/boost/corosio/detail/scheduler.hpp @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -139,6 +140,41 @@ struct BOOST_COROSIO_DECL scheduler @throws std::logic_error If the context has no backend installed. */ +/** The innermost scheduler running on this thread, or null. + + Each backend's run guard sets it and restores the previous value, + so a completion can ask whether its awaiter's context is running + here with one compare. Nested runs of other schedulers are still + answered by the scheduler's own frame stack. +*/ +inline thread_local_ptr running_scheduler; + +/** RAII guard marking a scheduler as innermost on this thread. + + Restores the previous innermost scheduler on destruction. +*/ +class running_scheduler_guard +{ + scheduler const* prev_; + +public: + /// Construct the guard, making @p s the innermost scheduler. + explicit running_scheduler_guard(scheduler const* s) noexcept + : prev_(running_scheduler.get()) + { + running_scheduler.set(s); + } + + /// Destroy the guard, restoring the previous innermost scheduler. + ~running_scheduler_guard() + { + running_scheduler.set(prev_); + } + + running_scheduler_guard(running_scheduler_guard const&) = delete; + running_scheduler_guard& operator=(running_scheduler_guard const&) = delete; +}; + inline scheduler& get_scheduler(capy::execution_context& ctx) { diff --git a/include/boost/corosio/detail/timeout_awaitable.hpp b/include/boost/corosio/detail/timeout_awaitable.hpp index c46973131..c6008ff49 100644 --- a/include/boost/corosio/detail/timeout_awaitable.hpp +++ b/include/boost/corosio/detail/timeout_awaitable.hpp @@ -16,7 +16,9 @@ #include #include #include +#include #include +#include #include #include @@ -186,12 +188,19 @@ struct timeout_awaitable else timer_->expires_after(dur_); + // The detached coroutine can outlive this chain and its frame + // allocator, so it allocates from the process-wide recycling + // resource instead; the chain's allocator is put back for the + // inner operation. + auto* const rmr = capy::get_recycling_memory_resource(); + auto* const prev = capy::get_current_frame_allocator(); + capy::set_current_frame_allocator(rmr); // Launch fire-and-forget timeout (starts suspended) auto timeout = make_timeout(*timer_, stop_src_); - timeout.h_.promise().set_env_owned( - ex, stop_src_.get_token(), env->frame_allocator); + timeout.h_.promise().set_env_owned(ex, stop_src_.get_token(), rmr); // Runs synchronously until timer.wait() suspends timeout.h_.resume(); + capy::set_current_frame_allocator(prev); // timeout goes out of scope; destructor is a no-op, // the coroutine self-destroys via suspend_never diff --git a/include/boost/corosio/detail/timeout_coro.hpp b/include/boost/corosio/detail/timeout_coro.hpp index e7db54514..1578d62fc 100644 --- a/include/boost/corosio/detail/timeout_coro.hpp +++ b/include/boost/corosio/detail/timeout_coro.hpp @@ -36,10 +36,9 @@ the MSVC symmetric-transfer workaround) to inject io_env into IoAwaitable co_await expressions. - The detached frame allocates from the awaiting chain's frame - allocator and can outlive that chain, extending the allocator's - required lifetime until the io_context drains. This is safe with - the default recycling resource. */ + The detached frame can outlive the awaiting chain, so it never + allocates from that chain's frame allocator: timeout_awaitable + creates it under the process-wide recycling resource. */ namespace boost::corosio::detail { diff --git a/include/boost/corosio/detail/timer.hpp b/include/boost/corosio/detail/timer.hpp index 0f10f8363..0fa95b331 100644 --- a/include/boost/corosio/detail/timer.hpp +++ b/include/boost/corosio/detail/timer.hpp @@ -101,9 +101,10 @@ class BOOST_COROSIO_DECL timer : public io_object // unlocked fast-out that reads them is either re-validated under // the mutex or safe under a stale value in both directions, and // any locked writer / locked reader pair is already ordered by - // the mutex. All accesses therefore use memory_order_relaxed, - // which keeps the lock-free fast paths fence-free while making - // the concurrent reads well-defined. + // the mutex, so heap_index_ is relaxed. A false flag is the + // exception: a canceller that trusts it skips the lock and may + // recycle the impl, so drains store it with release and the + // fast-out loads it with acquire. /// Index in the timer service's min-heap, or `npos`. std::atomic heap_index_{npos}; diff --git a/include/boost/corosio/detail/timer_service.hpp b/include/boost/corosio/detail/timer_service.hpp index 14613fc56..7fbaaff5a 100644 --- a/include/boost/corosio/detail/timer_service.hpp +++ b/include/boost/corosio/detail/timer_service.hpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -154,6 +155,10 @@ class BOOST_COROSIO_DECL timer_service final timer_object_pool pool_; callback on_earliest_changed_; bool shutting_down_ = false; + + // Cancellers that claimed a waiter and have yet to post it; + // shutdown waits them out so the scheduler's drain finds the post. + std::size_t cancels_posting_ = 0; // guarded by mutex_ // Avoids mutex in nearest_expiry() and empty() mutable std::atomic cached_nearest_ns_{ (std::numeric_limits::max)()}; @@ -313,36 +318,66 @@ timer_service::shutdown() // Snapshot impls and detach them from the heap so that // coroutine-owned timer destructors (triggered by h.destroy() // below) cannot re-enter remove_timer_impl() and mutate the - // vector during iteration. + // vector during iteration. A stop requested on another thread + // reaches cancel_waiter() under the same lock: whichever side + // claims a waiter first owns it, and a waiter its canceller + // claimed is posted, so the scheduler's drain destroys its frame. std::vector impls; - impls.reserve(heap_.size()); - for (auto& entry : heap_) + std::vector waiters; { - entry.timer_->heap_index_.store( - (std::numeric_limits::max)(), - std::memory_order_relaxed); - impls.push_back(entry.timer_); + std::lock_guard lock(mutex_); + impls.reserve(heap_.size()); + waiters.reserve(heap_.size()); + for (auto& entry : heap_) + { + entry.timer_->heap_index_.store( + (std::numeric_limits::max)(), + std::memory_order_relaxed); + impls.push_back(entry.timer_); + if (auto* w = std::exchange(entry.timer_->waiter_, nullptr)) + { + w->impl_ = nullptr; + waiters.push_back(w); + } + } + heap_.clear(); + cached_nearest_ns_.store( + (std::numeric_limits::max)(), + std::memory_order_release); + } + + // A stop on another thread may have claimed its waiter before the + // pass above and still be posting it; the scheduler drains next, + // and must find that post. + for (;;) + { + { + std::lock_guard lock(mutex_); + if (cancels_posting_ == 0) + break; + } + std::this_thread::yield(); } - heap_.clear(); - cached_nearest_ns_.store( - (std::numeric_limits::max)(), std::memory_order_release); // Cancel waiting timers. Each waiter called work_started() // in implementation::wait(). On IOCP the scheduler shutdown // loop exits when outstanding_work_ reaches zero, so we must // call work_finished() here to balance it. On other backends - // this is harmless. + // this is harmless. Resetting the stop callback outside the lock + // waits out a canceller already running, which finds the waiter + // claimed. + for (auto* w : waiters) + { + w->reset_stop_cb(); + auto h = std::exchange(w->h_, {}); + sched_->work_finished(); + // Destroying the frame also ends the node's storage + if (h) + h.destroy(); + } + for (auto* impl : impls) { - if (auto* w = std::exchange(impl->waiter_, nullptr)) - { - w->reset_stop_cb(); - auto h = std::exchange(w->h_, {}); - sched_->work_finished(); - // Destroying the frame also ends the node's storage - if (h) - h.destroy(); - } // Unlink from the pool's live_ list before the direct delete // below, or ~object_pool()'s unconditional sweep would delete // this impl a second time. @@ -463,7 +498,7 @@ timer_service::insert_waiter(timer::implementation& impl, waiter_node* w) impl.waiter_ = nullptr; remove_timer_impl(impl); impl.might_have_pending_waits_.store( - false, std::memory_order_relaxed); + false, std::memory_order_release); refresh_cached_nearest(); lost_cancel = true; notify = false; // insertion undone; nearest unchanged @@ -481,7 +516,10 @@ timer_service::insert_waiter(timer::implementation& impl, waiter_node* w) inline void timer_service::cancel_timer(timer::implementation& impl) { - if (!impl.might_have_pending_waits_.load(std::memory_order_relaxed)) + // Acquire pairs with the drains' release: a caller that skips the + // lock may recycle the impl next, so the drain's writes to it must + // happen before. + if (!impl.might_have_pending_waits_.load(std::memory_order_acquire)) return; // No unlocked already-done fast-out here: it would need the @@ -501,7 +539,7 @@ timer_service::cancel_timer(timer::implementation& impl) canceled->impl_ = nullptr; // Store false as the final touch of the impl under the lock so // a pre-lock false-flag check trusts it unqualified. - impl.might_have_pending_waits_.store(false, std::memory_order_relaxed); + impl.might_have_pending_waits_.store(false, std::memory_order_release); refresh_cached_nearest(); } @@ -525,12 +563,16 @@ timer_service::cancel_waiter(waiter_node* w) w->impl_ = nullptr; impl->waiter_ = nullptr; remove_timer_impl(*impl); - impl->might_have_pending_waits_.store(false, std::memory_order_relaxed); + impl->might_have_pending_waits_.store(false, std::memory_order_release); refresh_cached_nearest(); + ++cancels_posting_; } w->ec_ = make_error_code(capy::error::canceled); sched_->post(&w->op_); + + std::lock_guard lock(mutex_); + --cancels_posting_; } inline std::size_t @@ -553,7 +595,7 @@ timer_service::process_expired() expired.push_back(w); } t->might_have_pending_waits_.store( - false, std::memory_order_relaxed); + false, std::memory_order_release); } refresh_cached_nearest(); diff --git a/include/boost/corosio/io/io_object.hpp b/include/boost/corosio/io/io_object.hpp index d90f52a08..f18cefcad 100644 --- a/include/boost/corosio/io/io_object.hpp +++ b/include/boost/corosio/io/io_object.hpp @@ -142,6 +142,25 @@ class BOOST_COROSIO_DECL io_object /// Construct an empty handle. handle() = default; + /** Construct a handle that takes over an existing implementation. + + @pre @p p, if non-null, was constructed by @p svc and is + owned by no other handle. + + @param ctx The context @p svc belongs to. + @param svc The service that constructed @p p. + @param p The implementation to own. + */ + handle( + capy::execution_context& ctx, + io_service& svc, + implementation* p) noexcept + : ctx_(&ctx) + , svc_(&svc) + , impl_(p) + { + } + /// Construct a handle bound to a context and service. handle(capy::execution_context& ctx, io_service& svc) : ctx_(&ctx) diff --git a/include/boost/corosio/io/io_read_stream.hpp b/include/boost/corosio/io/io_read_stream.hpp index c77e8238d..838646b90 100644 --- a/include/boost/corosio/io/io_read_stream.hpp +++ b/include/boost/corosio/io/io_read_stream.hpp @@ -18,6 +18,8 @@ #include #include +#include + #include #include #include @@ -65,16 +67,17 @@ class BOOST_COROSIO_DECL io_read_stream : virtual public io_object MutableBufferSequence buffers_; std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return ios_.do_read_some( - h, ex, buffers_, this->token_, &this->ec_, &this->bytes_); + cont, ex, buffers_, this->token_, &this->ec_, &this->bytes_); } }; /** Dispatch a read through the concrete implementation. - @param h Coroutine handle to resume on completion. + @param cont Continuation to resume on completion; it lives in + the awaiting frame until then. @param ex Executor for dispatching the completion. @param buffers Target buffer sequence. @param token Stop token for cancellation. @@ -84,7 +87,7 @@ class BOOST_COROSIO_DECL io_read_stream : virtual public io_object @return Coroutine handle to resume immediately. */ virtual std::coroutine_handle<> do_read_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buffers, std::stop_token token, diff --git a/include/boost/corosio/io/io_signal_set.hpp b/include/boost/corosio/io/io_signal_set.hpp index 85e72e3c2..293db87cd 100644 --- a/include/boost/corosio/io/io_signal_set.hpp +++ b/include/boost/corosio/io/io_signal_set.hpp @@ -18,6 +18,8 @@ #include #include +#include + #include #include #include @@ -49,9 +51,9 @@ class BOOST_COROSIO_DECL io_signal_set : public io_object explicit wait_awaitable(io_signal_set& s) noexcept : s_(s) {} std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - return s_.get().wait(h, ex, token_, &ec_, &value_); + return s_.get().wait(cont, ex, token_, &ec_, &value_); } }; @@ -65,7 +67,8 @@ class BOOST_COROSIO_DECL io_signal_set : public io_object { /** Initiate an asynchronous wait for a signal. - @param h Coroutine handle to resume on completion. + @param cont Continuation to resume on completion; it lives in + the awaiting frame until then. @param ex Executor for dispatching the completion. @param token Stop token for cancellation. @param ec Output error code. @@ -74,7 +77,7 @@ class BOOST_COROSIO_DECL io_signal_set : public io_object @return Coroutine handle to resume immediately. */ virtual std::coroutine_handle<> wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, std::stop_token token, std::error_code* ec, diff --git a/include/boost/corosio/io/io_stream.hpp b/include/boost/corosio/io/io_stream.hpp index e2eea98f0..7fdc4bba5 100644 --- a/include/boost/corosio/io/io_stream.hpp +++ b/include/boost/corosio/io/io_stream.hpp @@ -18,6 +18,8 @@ #include #include +#include + #include #include #include @@ -72,7 +74,8 @@ class BOOST_COROSIO_DECL io_stream { /** Initiate platform read operation. - @param h Coroutine handle to resume on completion. + @param cont Continuation to resume on completion; it lives in + the awaiting frame until then. @param ex Executor for dispatching the completion. @param buffers Target buffer sequence. @param token Stop token for cancellation. @@ -82,7 +85,7 @@ class BOOST_COROSIO_DECL io_stream @return Coroutine handle to resume immediately. */ virtual std::coroutine_handle<> read_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buffers, std::stop_token token, @@ -91,7 +94,8 @@ class BOOST_COROSIO_DECL io_stream /** Initiate platform write operation. - @param h Coroutine handle to resume on completion. + @param cont Continuation to resume on completion; it lives in + the awaiting frame until then. @param ex Executor for dispatching the completion. @param buffers Source buffer sequence. @param token Stop token for cancellation. @@ -101,7 +105,7 @@ class BOOST_COROSIO_DECL io_stream @return Coroutine handle to resume immediately. */ virtual std::coroutine_handle<> write_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buffers, std::stop_token token, @@ -118,7 +122,8 @@ class BOOST_COROSIO_DECL io_stream /** Dispatch read through implementation vtable. - @param h Coroutine handle to resume on completion. + @param cont Continuation to resume on completion; it lives in + the awaiting frame until then. @param ex Executor for dispatching the completion. @param buffers Target buffer sequence. @param token Stop token for cancellation. @@ -128,19 +133,20 @@ class BOOST_COROSIO_DECL io_stream @return Coroutine handle to resume immediately. */ std::coroutine_handle<> do_read_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buffers, std::stop_token token, std::error_code* ec, std::size_t* bytes) override { - return get().read_some(h, ex, buffers, std::move(token), ec, bytes); + return get().read_some(cont, ex, buffers, std::move(token), ec, bytes); } /** Dispatch write through implementation vtable. - @param h Coroutine handle to resume on completion. + @param cont Continuation to resume on completion; it lives in + the awaiting frame until then. @param ex Executor for dispatching the completion. @param buffers Source buffer sequence. @param token Stop token for cancellation. @@ -150,14 +156,14 @@ class BOOST_COROSIO_DECL io_stream @return Coroutine handle to resume immediately. */ std::coroutine_handle<> do_write_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buffers, std::stop_token token, std::error_code* ec, std::size_t* bytes) override { - return get().write_some(h, ex, buffers, std::move(token), ec, bytes); + return get().write_some(cont, ex, buffers, std::move(token), ec, bytes); } private: diff --git a/include/boost/corosio/io/io_write_stream.hpp b/include/boost/corosio/io/io_write_stream.hpp index 33dcdc159..c9b0163f7 100644 --- a/include/boost/corosio/io/io_write_stream.hpp +++ b/include/boost/corosio/io/io_write_stream.hpp @@ -18,6 +18,8 @@ #include #include +#include + #include #include #include @@ -64,16 +66,17 @@ class BOOST_COROSIO_DECL io_write_stream : virtual public io_object ConstBufferSequence buffers_; std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return ios_.do_write_some( - h, ex, buffers_, this->token_, &this->ec_, &this->bytes_); + cont, ex, buffers_, this->token_, &this->ec_, &this->bytes_); } }; /** Dispatch a write through the concrete implementation. - @param h Coroutine handle to resume on completion. + @param cont Continuation to resume on completion; it lives in + the awaiting frame until then. @param ex Executor for dispatching the completion. @param buffers Source buffer sequence. @param token Stop token for cancellation. @@ -83,7 +86,7 @@ class BOOST_COROSIO_DECL io_write_stream : virtual public io_object @return Coroutine handle to resume immediately. */ virtual std::coroutine_handle<> do_write_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buffers, std::stop_token token, diff --git a/include/boost/corosio/io_context.hpp b/include/boost/corosio/io_context.hpp index 00b762bef..e105ad5b0 100644 --- a/include/boost/corosio/io_context.hpp +++ b/include/boost/corosio/io_context.hpp @@ -620,7 +620,10 @@ class io_context::executor_type */ bool running_in_this_thread() const noexcept { - return ctx_->sched_->running_in_this_thread(); + // The innermost scheduler answers the common case without a + // virtual call or a frame walk. + return detail::running_scheduler.get() == ctx_->sched_ || + ctx_->sched_->running_in_this_thread(); } /** Informs the executor that work is beginning. diff --git a/include/boost/corosio/local_datagram_socket.hpp b/include/boost/corosio/local_datagram_socket.hpp index 40de6ae76..f97370e9e 100644 --- a/include/boost/corosio/local_datagram_socket.hpp +++ b/include/boost/corosio/local_datagram_socket.hpp @@ -35,6 +35,8 @@ #include #include +#include + #include #include #include @@ -99,7 +101,8 @@ class BOOST_COROSIO_DECL local_datagram_socket : public io_object { /** Initiate an asynchronous `send_to` operation. - @param h Coroutine handle to resume on completion. + @param cont Continuation to resume on completion; it lives in + the awaiting frame until then. @param ex Executor for dispatching the completion. @param buf The buffer data to send. @param dest The destination endpoint. @@ -111,7 +114,7 @@ class BOOST_COROSIO_DECL local_datagram_socket : public io_object @return Coroutine handle to resume immediately. */ virtual std::coroutine_handle<> send_to( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buf, corosio::local_endpoint dest, @@ -122,7 +125,8 @@ class BOOST_COROSIO_DECL local_datagram_socket : public io_object /** Initiate an asynchronous `recv_from` operation. - @param h Coroutine handle to resume on completion. + @param cont Continuation to resume on completion; it lives in + the awaiting frame until then. @param ex Executor for dispatching the completion. @param buf The buffer to receive into. @param source Output endpoint for the sender's address. @@ -134,7 +138,7 @@ class BOOST_COROSIO_DECL local_datagram_socket : public io_object @return Coroutine handle to resume immediately. */ virtual std::coroutine_handle<> recv_from( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buf, corosio::local_endpoint* source, @@ -145,7 +149,8 @@ class BOOST_COROSIO_DECL local_datagram_socket : public io_object /** Initiate an asynchronous connect to set the default peer. - @param h Coroutine handle to resume on completion. + @param cont Continuation to resume on completion; it lives in + the awaiting frame until then. @param ex Executor for dispatching the completion. @param ep The remote endpoint to connect to. @param token Stop token for cancellation. @@ -154,7 +159,7 @@ class BOOST_COROSIO_DECL local_datagram_socket : public io_object @return Coroutine handle to resume immediately. */ virtual std::coroutine_handle<> connect( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, corosio::local_endpoint ep, std::stop_token token, @@ -162,7 +167,8 @@ class BOOST_COROSIO_DECL local_datagram_socket : public io_object /** Initiate an asynchronous connected send operation. - @param h Coroutine handle to resume on completion. + @param cont Continuation to resume on completion; it lives in + the awaiting frame until then. @param ex Executor for dispatching the completion. @param buf The buffer data to send. @param flags Message flags (e.g. `message_flags::do_not_route`). @@ -173,7 +179,7 @@ class BOOST_COROSIO_DECL local_datagram_socket : public io_object @return Coroutine handle to resume immediately. */ virtual std::coroutine_handle<> send( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buf, int flags, @@ -183,7 +189,8 @@ class BOOST_COROSIO_DECL local_datagram_socket : public io_object /** Initiate an asynchronous connected `recv` operation. - @param h Coroutine handle to resume on completion. + @param cont Continuation to resume on completion; it lives in + the awaiting frame until then. @param ex Executor for dispatching the completion. @param buf The buffer to receive into. @param flags Message flags (e.g. `message_flags::peek`). @@ -194,7 +201,7 @@ class BOOST_COROSIO_DECL local_datagram_socket : public io_object @return Coroutine handle to resume immediately. */ virtual std::coroutine_handle<> recv( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buf, int flags, @@ -208,7 +215,8 @@ class BOOST_COROSIO_DECL local_datagram_socket : public io_object specified direction, or an error condition is reported. No bytes are transferred. - @param h Coroutine handle to resume on completion. + @param cont Continuation to resume on completion; it lives in + the awaiting frame until then. @param ex Executor for dispatching the completion. @param w The direction to wait on. @param token Stop token for cancellation. @@ -217,7 +225,7 @@ class BOOST_COROSIO_DECL local_datagram_socket : public io_object @return Coroutine handle to resume immediately. */ virtual std::coroutine_handle<> wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, wait_type w, std::stop_token token, @@ -334,10 +342,10 @@ class BOOST_COROSIO_DECL local_datagram_socket : public io_object int flags_; std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return s_.get().send_to( - h, ex, buf_, dest_, flags_, token_, &ec_, &bytes_); + cont, ex, buf_, dest_, flags_, token_, &ec_, &bytes_); } }; @@ -371,10 +379,10 @@ class BOOST_COROSIO_DECL local_datagram_socket : public io_object int flags_; std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return s_.get().recv_from( - h, ex, buf_, &source_, flags_, token_, &ec_, &bytes_); + cont, ex, buf_, &source_, flags_, token_, &ec_, &bytes_); } }; @@ -401,9 +409,9 @@ class BOOST_COROSIO_DECL local_datagram_socket : public io_object corosio::local_endpoint endpoint_; std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - return s_.get().connect(h, ex, endpoint_, token_, &ec_); + return s_.get().connect(cont, ex, endpoint_, token_, &ec_); } }; @@ -425,9 +433,9 @@ class BOOST_COROSIO_DECL local_datagram_socket : public io_object wait_type w_; std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - return s_.get().wait(h, ex, w_, token_, &ec_); + return s_.get().wait(cont, ex, w_, token_, &ec_); } }; @@ -456,9 +464,9 @@ class BOOST_COROSIO_DECL local_datagram_socket : public io_object int flags_; std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - return s_.get().send(h, ex, buf_, flags_, token_, &ec_, &bytes_); + return s_.get().send(cont, ex, buf_, flags_, token_, &ec_, &bytes_); } }; @@ -487,9 +495,9 @@ class BOOST_COROSIO_DECL local_datagram_socket : public io_object int flags_; std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - return s_.get().recv(h, ex, buf_, flags_, token_, &ec_, &bytes_); + return s_.get().recv(cont, ex, buf_, flags_, token_, &ec_, &bytes_); } }; diff --git a/include/boost/corosio/local_stream_acceptor.hpp b/include/boost/corosio/local_stream_acceptor.hpp index 0192e4d96..9467a9a69 100644 --- a/include/boost/corosio/local_stream_acceptor.hpp +++ b/include/boost/corosio/local_stream_acceptor.hpp @@ -26,9 +26,12 @@ #include #include +#include #include #include +#include + #include #include #include @@ -86,9 +89,9 @@ class BOOST_COROSIO_DECL local_stream_acceptor : public io_object wait_type w_; std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - return acc_.get().wait(h, ex, w_, token_, &ec_); + return acc_.get().wait(cont, ex, w_, token_, &ec_); } }; @@ -101,19 +104,28 @@ class BOOST_COROSIO_DECL local_stream_acceptor : public io_object local_stream_acceptor& acc_; mutable io_object::implementation* peer_impl_ = nullptr; + explicit move_accept_awaitable(local_stream_acceptor& acc) noexcept : acc_(acc) { } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return acc_.get().accept( - h, ex, this->token_, &this->ec_, &peer_impl_); + cont, ex, this->token_, &this->ec_, &peer_impl_); } public: + // A peer accepted for an awaiter that never resumed is closed + // here, or nothing would own it. + ~move_accept_awaitable() + { + if (peer_impl_) + discard_peer(acc_, peer_impl_); + } + [[nodiscard]] capy::io_result await_resume() const noexcept { @@ -121,7 +133,7 @@ class BOOST_COROSIO_DECL local_stream_acceptor : public io_object return {this->ec_, local_stream_socket()}; local_stream_socket peer(acc_.ctx_); - reset_peer_impl(peer, peer_impl_); + reset_peer_impl(peer, std::exchange(peer_impl_, nullptr)); return {this->ec_, std::move(peer)}; } }; @@ -136,6 +148,7 @@ class BOOST_COROSIO_DECL local_stream_acceptor : public io_object local_stream_socket& peer_; mutable io_object::implementation* peer_impl_ = nullptr; + accept_awaitable( local_stream_acceptor& acc, local_stream_socket& peer) noexcept : acc_(acc) @@ -144,17 +157,25 @@ class BOOST_COROSIO_DECL local_stream_acceptor : public io_object } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return acc_.get().accept( - h, ex, this->token_, &this->ec_, &peer_impl_); + cont, ex, this->token_, &this->ec_, &peer_impl_); } public: + // A peer accepted for an awaiter that never resumed is closed + // here, or nothing would own it. + ~accept_awaitable() + { + if (peer_impl_) + discard_peer(acc_, peer_impl_); + } + [[nodiscard]] capy::io_result<> await_resume() const noexcept { if (!this->ec_ && peer_impl_) - peer_.h_.reset(peer_impl_); + peer_.h_.reset(std::exchange(peer_impl_, nullptr)); return {this->ec_}; } }; @@ -543,7 +564,8 @@ class BOOST_COROSIO_DECL local_stream_acceptor : public io_object success, stores a pointer to the new socket implementation in @p *impl_out. - @param h Coroutine handle to resume. + @param cont Continuation to resume on completion; it lives in + the awaiting frame until then. @param ex Executor for dispatching the completion. @param token Stop token for cancellation. @param ec Output error code. @@ -551,7 +573,7 @@ class BOOST_COROSIO_DECL local_stream_acceptor : public io_object @return Coroutine handle to resume immediately. */ virtual std::coroutine_handle<> accept( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, std::stop_token token, std::error_code* ec, @@ -562,7 +584,8 @@ class BOOST_COROSIO_DECL local_stream_acceptor : public io_object Completes when the listen socket becomes ready for the specified direction. No connection is consumed. - @param h Coroutine handle to resume on completion. + @param cont Continuation to resume on completion; it lives in + the awaiting frame until then. @param ex Executor for dispatching the completion. @param w The direction to wait on. @param token Stop token for cancellation. @@ -571,7 +594,7 @@ class BOOST_COROSIO_DECL local_stream_acceptor : public io_object @return Coroutine handle to resume immediately. */ virtual std::coroutine_handle<> wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, wait_type w, std::stop_token token, @@ -681,6 +704,10 @@ class BOOST_COROSIO_DECL local_stream_acceptor : public io_object { return *static_cast(h_.get()); } + + /// Close and release a non-null accepted peer no socket took over. + static void discard_peer( + local_stream_acceptor& acc, io_object::implementation* impl) noexcept; }; } // namespace boost::corosio diff --git a/include/boost/corosio/local_stream_socket.hpp b/include/boost/corosio/local_stream_socket.hpp index 1379fdd4b..cd9455dc7 100644 --- a/include/boost/corosio/local_stream_socket.hpp +++ b/include/boost/corosio/local_stream_socket.hpp @@ -31,6 +31,8 @@ #include #include +#include + #include #include #include @@ -83,7 +85,8 @@ class BOOST_COROSIO_DECL local_stream_socket : public io_stream { /** Initiate an asynchronous connect to the given endpoint. - @param h Coroutine handle to resume on completion. + @param cont Continuation to resume on completion; it lives in + the awaiting frame until then. @param ex Executor for dispatching the completion. @param ep The local endpoint (path) to connect to. @param token Stop token for cancellation. @@ -92,7 +95,7 @@ class BOOST_COROSIO_DECL local_stream_socket : public io_stream @return Coroutine handle to resume immediately. */ virtual std::coroutine_handle<> connect( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, corosio::local_endpoint ep, std::stop_token token, @@ -104,7 +107,8 @@ class BOOST_COROSIO_DECL local_stream_socket : public io_stream specified direction, or an error condition is reported. No bytes are transferred. - @param h Coroutine handle to resume on completion. + @param cont Continuation to resume on completion; it lives in + the awaiting frame until then. @param ex Executor for dispatching the completion. @param w The direction to wait on. @param token Stop token for cancellation. @@ -113,7 +117,7 @@ class BOOST_COROSIO_DECL local_stream_socket : public io_stream @return Coroutine handle to resume immediately. */ virtual std::coroutine_handle<> wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, wait_type w, std::stop_token token, @@ -210,9 +214,9 @@ class BOOST_COROSIO_DECL local_stream_socket : public io_stream corosio::local_endpoint endpoint_; std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - return s_.get().connect(h, ex, endpoint_, token_, &ec_); + return s_.get().connect(cont, ex, endpoint_, token_, &ec_); } }; @@ -234,9 +238,9 @@ class BOOST_COROSIO_DECL local_stream_socket : public io_stream wait_type w_; std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - return s_.get().wait(h, ex, w_, token_, &ec_); + return s_.get().wait(cont, ex, w_, token_, &ec_); } }; diff --git a/include/boost/corosio/native/detail/coro_op.hpp b/include/boost/corosio/native/detail/coro_op.hpp index bac26b0bc..9743519d4 100644 --- a/include/boost/corosio/native/detail/coro_op.hpp +++ b/include/boost/corosio/native/detail/coro_op.hpp @@ -77,8 +77,11 @@ struct coro_op : scheduler_op } }; - std::coroutine_handle<> h; - capy::continuation cont; + /// The awaiting coroutine's continuation. It lives in the awaitable, + /// which stays in the awaiting frame until resumption, so a + /// completion posted through an executor never shares this op's + /// storage, which is reused by the next operation. + capy::continuation* cont = nullptr; capy::executor_ref ex; std::error_code* ec_out = nullptr; std::size_t* bytes_out = nullptr; diff --git a/include/boost/corosio/native/detail/coro_op_complete.hpp b/include/boost/corosio/native/detail/coro_op_complete.hpp index 298aff82b..aa35da246 100644 --- a/include/boost/corosio/native/detail/coro_op_complete.hpp +++ b/include/boost/corosio/native/detail/coro_op_complete.hpp @@ -142,12 +142,11 @@ coro_drain_if_shutdown(void* owner, coro_op* self) noexcept inline void coro_resume(coro_op* self) noexcept { - self->cont.h = self->h; // Clear the keepalive before publishing the continuation: a strand // drained on another thread can reuse this op via reset() the instant // it runs, so this write must be ordered before the publish, not after. auto suicide = std::move(self->object_ref_); - auto next = dispatch_coro(self->ex, self->cont); + auto next = dispatch_coro(self->ex, *self->cont); next.resume(); // suicide drops here; may destroy impl + self. } diff --git a/include/boost/corosio/native/detail/iocp/win_file_service.hpp b/include/boost/corosio/native/detail/iocp/win_file_service.hpp index 704c180be..b973f1c12 100644 --- a/include/boost/corosio/native/detail/iocp/win_file_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_file_service.hpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -209,26 +210,26 @@ win_stream_file::reuse() noexcept inline std::coroutine_handle<> win_stream_file::read_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, buffer_param buf, std::stop_token token, std::error_code* ec, std::size_t* bytes) { - return internal_.read_some(h, d, buf, token, ec, bytes); + return internal_.read_some(cont, d, buf, token, ec, bytes); } inline std::coroutine_handle<> win_stream_file::write_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, buffer_param buf, std::stop_token token, std::error_code* ec, std::size_t* bytes) { - return internal_.write_some(h, d, buf, token, ec, bytes); + return internal_.write_some(cont, d, buf, token, ec, bytes); } inline native_handle_type diff --git a/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor.hpp b/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor.hpp index c455cdca1..4f11a3018 100644 --- a/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor.hpp +++ b/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor.hpp @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -116,14 +117,14 @@ class win_local_stream_acceptor final win_local_stream_service& socket_service() noexcept; std::coroutine_handle<> accept( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, std::stop_token token, std::error_code* ec, io_object::implementation** impl_out) override; std::coroutine_handle<> wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, wait_type w, std::stop_token token, diff --git a/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor_service.hpp b/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor_service.hpp index 4a6d9e985..0ad455286 100644 --- a/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor_service.hpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -213,11 +214,10 @@ local_stream_accept_op::do_complete( *op->impl_out = nullptr; } - op->cont.h = op->h; auto saved_ex = op->ex; auto prevent_premature_destruction = std::move(op->object_ref_); - dispatch_coro(saved_ex, op->cont).resume(); + dispatch_coro(saved_ex, *op->cont).resume(); } inline void @@ -306,7 +306,7 @@ win_local_stream_acceptor::cancel() noexcept inline std::coroutine_handle<> win_local_stream_acceptor::wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, wait_type w, std::stop_token token, @@ -317,7 +317,7 @@ win_local_stream_acceptor::wait( auto& op = wt_; op.reset(); - op.h = h; + op.cont = &cont; op.ex = d; op.ec_out = ec; op.bytes_out = nullptr; @@ -368,7 +368,7 @@ win_local_stream_acceptor::close_socket() noexcept inline std::coroutine_handle<> win_local_stream_acceptor::accept( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, std::stop_token token, std::error_code* ec, @@ -385,7 +385,7 @@ win_local_stream_acceptor::accept( // whatever stale pointer it finds here. op.peer_wrapper = nullptr; op.accepted_socket = INVALID_SOCKET; - op.h = h; + op.cont = &cont; op.ex = d; op.ec_out = ec; op.impl_out = impl_out; diff --git a/include/boost/corosio/native/detail/iocp/win_local_stream_service.hpp b/include/boost/corosio/native/detail/iocp/win_local_stream_service.hpp index 30dd708ce..190accad7 100644 --- a/include/boost/corosio/native/detail/iocp/win_local_stream_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_local_stream_service.hpp @@ -366,7 +366,7 @@ win_local_stream_socket::set_endpoints( inline std::coroutine_handle<> win_local_stream_socket::connect( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, corosio::local_endpoint ep, std::stop_token token, @@ -376,7 +376,7 @@ win_local_stream_socket::connect( auto& op = conn_; op.reset(); - op.h = h; + op.cont = &cont; op.ex = d; op.ec_out = ec; op.target_endpoint = ep; @@ -436,7 +436,7 @@ win_local_stream_socket::connect( inline std::coroutine_handle<> win_local_stream_socket::read_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, buffer_param param, std::stop_token token, @@ -448,7 +448,7 @@ win_local_stream_socket::read_some( auto& op = rd_; op.reset(); op.is_read = true; - op.h = h; + op.cont = &cont; op.ex = d; op.ec_out = ec; op.bytes_out = bytes_out; @@ -507,7 +507,7 @@ win_local_stream_socket::read_some( inline std::coroutine_handle<> win_local_stream_socket::write_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, buffer_param param, std::stop_token token, @@ -518,7 +518,7 @@ win_local_stream_socket::write_some( auto& op = wr_; op.reset(); - op.h = h; + op.cont = &cont; op.ex = d; op.ec_out = ec; op.bytes_out = bytes_out; @@ -574,7 +574,7 @@ win_local_stream_socket::write_some( inline std::coroutine_handle<> win_local_stream_socket::wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, wait_type w, std::stop_token token, @@ -584,7 +584,7 @@ win_local_stream_socket::wait( auto& op = wt_; op.reset(); - op.h = h; + op.cont = &cont; op.ex = d; op.ec_out = ec; op.bytes_out = nullptr; diff --git a/include/boost/corosio/native/detail/iocp/win_local_stream_socket.hpp b/include/boost/corosio/native/detail/iocp/win_local_stream_socket.hpp index 20cca18d1..80bb06aa3 100644 --- a/include/boost/corosio/native/detail/iocp/win_local_stream_socket.hpp +++ b/include/boost/corosio/native/detail/iocp/win_local_stream_socket.hpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -151,14 +152,14 @@ class win_local_stream_socket final void reuse() noexcept; std::coroutine_handle<> connect( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, corosio::local_endpoint ep, std::stop_token token, std::error_code* ec) override; std::coroutine_handle<> read_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, buffer_param buf, std::stop_token token, @@ -166,7 +167,7 @@ class win_local_stream_socket final std::size_t* bytes) override; std::coroutine_handle<> write_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, buffer_param buf, std::stop_token token, @@ -174,7 +175,7 @@ class win_local_stream_socket final std::size_t* bytes) override; std::coroutine_handle<> wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, wait_type w, std::stop_token token, diff --git a/include/boost/corosio/native/detail/iocp/win_object_handle_service.hpp b/include/boost/corosio/native/detail/iocp/win_object_handle_service.hpp index 8300f13ad..dff370952 100644 --- a/include/boost/corosio/native/detail/iocp/win_object_handle_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_object_handle_service.hpp @@ -115,8 +115,7 @@ class win_object_handle_state // The op may complete and the object be destroyed on another // thread before wait() returns; this pin outlives both. auto const pin = op_.object_ref_; - op_.user_cont = &cont; - op_.h = cont.h; + op_.cont = &cont; op_.ex = ex; op_.ec_out = ec; op_.bytes_out = nullptr; @@ -206,7 +205,6 @@ class win_object_handle_state struct wait_op : overlapped_op { win_object_handle_state* self = nullptr; - capy::continuation* user_cont = nullptr; wait_op() noexcept : overlapped_op(&do_complete) { @@ -249,8 +247,7 @@ class win_object_handle_state iocp_make_err(op->dwError, /*accept_path=*/false); } - capy::continuation* cont = op->user_cont; - cont->h = op->h; + capy::continuation* cont = op->cont; capy::executor_ref ex = op->ex; // Last touch of op_: from here a new wait() may reset it. diff --git a/include/boost/corosio/native/detail/iocp/win_overlapped_handle.hpp b/include/boost/corosio/native/detail/iocp/win_overlapped_handle.hpp index 0ea3aa0ad..6666c594e 100644 --- a/include/boost/corosio/native/detail/iocp/win_overlapped_handle.hpp +++ b/include/boost/corosio/native/detail/iocp/win_overlapped_handle.hpp @@ -264,25 +264,27 @@ class win_slot_handle : public win_handle_base } std::coroutine_handle<> read_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param param, std::stop_token token, std::error_code* ec, std::size_t* bytes_out) { - return start(rd_, true, h, ex, param, std::move(token), ec, bytes_out); + return start( + rd_, true, cont, ex, param, std::move(token), ec, bytes_out); } std::coroutine_handle<> write_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param param, std::stop_token token, std::error_code* ec, std::size_t* bytes_out) { - return start(wr_, false, h, ex, param, std::move(token), ec, bytes_out); + return start( + wr_, false, cont, ex, param, std::move(token), ec, bytes_out); } void cancel() noexcept @@ -368,7 +370,7 @@ class win_slot_handle : public win_handle_base std::coroutine_handle<> start( slot_op& op, bool is_read, - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param param, std::stop_token token, @@ -379,7 +381,7 @@ class win_slot_handle : public win_handle_base op.reset(); op.owner = this; op.is_read = is_read; - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.bytes_out = bytes_out; @@ -476,8 +478,6 @@ class win_concurrent_handle : public win_handle_base : handle_io_op , intrusive_list::node { - capy::continuation* user_cont = nullptr; - concurrent_op() noexcept : handle_io_op(&do_complete) {} // The continuation lives in the awaitable, so the op can go first. @@ -496,7 +496,7 @@ class win_concurrent_handle : public win_handle_base if (!owner) { - op->h = {}; + op->cont = nullptr; self->recycle_op(op); return; } @@ -510,8 +510,7 @@ class win_concurrent_handle : public win_handle_base op->is_read, static_cast(op->bytes_transferred), op->empty_buffer); - capy::continuation* c = op->user_cont; - c->h = op->h; + capy::continuation* c = op->cont; capy::executor_ref ex = op->ex; // Once recycled, a concurrent initiation may refill the op. self->recycle_op(op); @@ -555,8 +554,7 @@ class win_concurrent_handle : public win_handle_base op->reset(); op->owner = this; op->is_read = is_read; - op->user_cont = &cont; - op->h = cont.h; + op->cont = &cont; op->ex = ex; op->ec_out = ec; op->bytes_out = bytes_out; diff --git a/include/boost/corosio/native/detail/iocp/win_overlapped_op.hpp b/include/boost/corosio/native/detail/iocp/win_overlapped_op.hpp index 538a198d9..12470259e 100644 --- a/include/boost/corosio/native/detail/iocp/win_overlapped_op.hpp +++ b/include/boost/corosio/native/detail/iocp/win_overlapped_op.hpp @@ -236,15 +236,14 @@ struct overlapped_op : std::error_code{}, is_read, static_cast(bytes_transferred), empty_buffer); - cont.h = h; - dispatch_coro(ex, cont).resume(); + dispatch_coro(ex, *cont).resume(); } - /** Disarm cancellation and abandon the coroutine handle. */ + /** Disarm cancellation and abandon the continuation. */ void cleanup_only() { stop_cb.reset(); - h = {}; + cont = nullptr; } }; diff --git a/include/boost/corosio/native/detail/iocp/win_random_access_file_service.hpp b/include/boost/corosio/native/detail/iocp/win_random_access_file_service.hpp index a339371d2..249c32aec 100644 --- a/include/boost/corosio/native/detail/iocp/win_random_access_file_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_random_access_file_service.hpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include diff --git a/include/boost/corosio/native/detail/iocp/win_resolver.hpp b/include/boost/corosio/native/detail/iocp/win_resolver.hpp index 071451182..312ab3a0a 100644 --- a/include/boost/corosio/native/detail/iocp/win_resolver.hpp +++ b/include/boost/corosio/native/detail/iocp/win_resolver.hpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -277,7 +278,7 @@ class win_resolver final void reuse() noexcept; std::coroutine_handle<> resolve( - std::coroutine_handle<>, + capy::continuation&, capy::executor_ref, std::string_view host, std::string_view service, @@ -287,7 +288,7 @@ class win_resolver final std::vector*) override; std::coroutine_handle<> reverse_resolve( - std::coroutine_handle<>, + capy::continuation&, capy::executor_ref, endpoint const& ep, reverse_flags flags, diff --git a/include/boost/corosio/native/detail/iocp/win_resolver_service.hpp b/include/boost/corosio/native/detail/iocp/win_resolver_service.hpp index e0b9c3787..b0b14fe78 100644 --- a/include/boost/corosio/native/detail/iocp/win_resolver_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_resolver_service.hpp @@ -317,12 +317,11 @@ resolve_op::do_complete( op->results = nullptr; } - op->cont.h = op->h; // Hold the keepalive across the dispatch: it may be the last // reference to the implementation this op is embedded in. auto prevent_destroy = std::move(op->object_ref_); svc.work_finished(); - dispatch_coro(op->ex, op->cont).resume(); + dispatch_coro(op->ex, *op->cont).resume(); } // reverse_resolve_op @@ -373,12 +372,11 @@ reverse_resolve_op::do_complete( std::move(op->stored_host), std::move(op->stored_service)}; } - op->cont.h = op->h; // Hold the keepalive across the dispatch: it may be the last // reference to the implementation this op is embedded in. auto prevent_destroy = std::move(op->object_ref_); svc.work_finished(); - dispatch_coro(op->ex, op->cont).resume(); + dispatch_coro(op->ex, *op->cont).resume(); } // win_resolver @@ -390,7 +388,7 @@ inline win_resolver::win_resolver(win_resolver_service& svc) noexcept inline std::coroutine_handle<> win_resolver::resolve( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, std::string_view host, std::string_view service, @@ -401,7 +399,7 @@ win_resolver::resolve( { auto& op = op_; op.reset(); - op.h = h; + op.cont = &cont; op.ex = d; op.ec_out = ec; op.out = out; @@ -471,7 +469,7 @@ win_resolver::resolve( inline std::coroutine_handle<> win_resolver::reverse_resolve( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, endpoint const& ep, reverse_flags flags, @@ -481,7 +479,7 @@ win_resolver::reverse_resolve( { auto& op = reverse_op_; op.reset(); - op.h = h; + op.cont = &cont; op.ex = d; op.ec_out = ec; op.result_out = result_out; @@ -507,8 +505,7 @@ win_resolver::reverse_resolve( op.stop_cb.reset(); svc_.work_finished(); *ec = pec; - op.cont.h = h; - return dispatch_coro(d, op.cont); + return dispatch_coro(d, cont); } // The work the pool took completes on its own thread and is always // posted to the scheduler queue, never inline. diff --git a/include/boost/corosio/native/detail/iocp/win_scheduler.hpp b/include/boost/corosio/native/detail/iocp/win_scheduler.hpp index c7ed01c0e..25baff946 100644 --- a/include/boost/corosio/native/detail/iocp/win_scheduler.hpp +++ b/include/boost/corosio/native/detail/iocp/win_scheduler.hpp @@ -193,9 +193,11 @@ inline thread_local_ptr context_stack; struct thread_context_guard { scheduler_context frame_; + running_scheduler_guard running_; explicit thread_context_guard(win_scheduler const* ctx) noexcept : frame_{ctx, context_stack.get()} + , running_(ctx) { context_stack.set(&frame_); } diff --git a/include/boost/corosio/native/detail/iocp/win_signal.hpp b/include/boost/corosio/native/detail/iocp/win_signal.hpp index 1e921ece4..8d30aaaaf 100644 --- a/include/boost/corosio/native/detail/iocp/win_signal.hpp +++ b/include/boost/corosio/native/detail/iocp/win_signal.hpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -46,9 +47,11 @@ enum /** Signal wait operation state. */ struct signal_op : scheduler_op { - std::coroutine_handle<> h; - capy::continuation cont; + /// The awaitable's continuation; see coro_op::cont. + capy::continuation* cont = nullptr; capy::executor_ref d; + /// Holds the owning set while this op sits in the scheduler queue. + detail::object_ref keep; std::error_code* ec_out = nullptr; int* signal_out = nullptr; int signal_number = 0; @@ -137,7 +140,7 @@ class win_signal final void reuse() noexcept; std::coroutine_handle<> wait( - std::coroutine_handle<>, + capy::continuation&, capy::executor_ref, std::stop_token, std::error_code*, diff --git a/include/boost/corosio/native/detail/iocp/win_signals.hpp b/include/boost/corosio/native/detail/iocp/win_signals.hpp index f344d5d3d..8c0472720 100644 --- a/include/boost/corosio/native/detail/iocp/win_signals.hpp +++ b/include/boost/corosio/native/detail/iocp/win_signals.hpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -333,8 +334,10 @@ signal_op::do_complete( std::uint32_t /*error*/) { auto* op = static_cast(base); + // Embedded in win_signal: copied out below before the queued + // reference can drop, and nothing touches op afterwards. + auto keep = std::move(op->keep); - // Destroy path - no-op: signal_op is embedded in win_signal if (!owner) return; @@ -345,9 +348,10 @@ signal_op::do_complete( auto* service = op->svc; op->svc = nullptr; + auto* c = op->cont; + auto ex = op->d; - op->cont.h = op->h; - dispatch_coro(op->d, op->cont).resume(); + dispatch_coro(ex, *c).resume(); if (service) service->work_finished(); @@ -361,13 +365,13 @@ inline win_signal::win_signal(win_signals& svc) noexcept : svc_(svc) {} inline std::coroutine_handle<> win_signal::wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, std::stop_token token, std::error_code* ec, int* signal_out) { - pending_op_.h = h; + pending_op_.cont = &cont; pending_op_.d = d; pending_op_.ec_out = ec; pending_op_.signal_out = signal_out; @@ -388,8 +392,7 @@ win_signal::wait( *ec = make_error_code(capy::error::canceled); if (signal_out) *signal_out = 0; - pending_op_.cont.h = h; - dispatch_coro(d, pending_op_.cont).resume(); + dispatch_coro(d, *pending_op_.cont).resume(); // resumed inline for an io_context executor, not posted to the // scheduler queue. return std::noop_coroutine(); @@ -693,8 +696,7 @@ win_signals::cancel_wait(win_signal& impl) *op->ec_out = make_error_code(capy::error::canceled); if (op->signal_out) *op->signal_out = 0; - op->cont.h = op->h; - dispatch_coro(op->d, op->cont).resume(); + dispatch_coro(op->d, *op->cont).resume(); sched_.work_finished(); } } @@ -724,8 +726,7 @@ win_signals::cancel_wait_token(win_signal& impl) noexcept *op->ec_out = make_error_code(capy::error::canceled); if (op->signal_out) *op->signal_out = 0; - op->cont.h = op->h; - dispatch_coro(op->d, op->cont).resume(); + dispatch_coro(op->d, *op->cont).resume(); sched_.work_finished(); } } @@ -753,7 +754,6 @@ win_signals::start_wait(win_signal& impl, signal_op* op) *op->ec_out = make_error_code(capy::error::canceled); if (op->signal_out) *op->signal_out = 0; - op->cont.h = op->h; } else if (impl.token_cancelled_) { @@ -767,7 +767,6 @@ win_signals::start_wait(win_signal& impl, signal_op* op) *op->ec_out = make_error_code(capy::error::canceled); if (op->signal_out) *op->signal_out = 0; - op->cont.h = op->h; } else { @@ -780,6 +779,8 @@ win_signals::start_wait(win_signal& impl, signal_op* op) --reg->undelivered; op->signal_number = reg->signal_number; op->svc = nullptr; // No extra work_finished needed + // The caller's wait() holds the handle's reference. + op->keep = detail::object_ref(&impl); // Post for immediate completion - post() handles work tracking post(op); return; @@ -799,7 +800,7 @@ win_signals::start_wait(win_signal& impl, signal_op* op) // Dispatch outside the lock to avoid deadlock if the resumed // coroutine re-enters cancel()/add()/remove() if (was_cancelled) - dispatch_coro(op->d, op->cont).resume(); + dispatch_coro(op->d, *op->cont).resume(); } inline void @@ -829,10 +830,17 @@ win_signals::deliver_signal(int signal_number) if (impl->waiting_) { - // Complete the pending wait - impl->waiting_ = false; - impl->pending_op_.signal_number = signal_number; - service->post(&impl->pending_op_); + // Reached through the registry, not a held reference. + // A set whose count reached zero is being destroyed, + // and that destruction completes its wait itself. + auto keep = detail::object_ref::try_from(impl); + if (keep) + { + impl->waiting_ = false; + impl->pending_op_.signal_number = signal_number; + impl->pending_op_.keep = std::move(keep); + service->post(&impl->pending_op_); + } } else { diff --git a/include/boost/corosio/native/detail/iocp/win_stream_file.hpp b/include/boost/corosio/native/detail/iocp/win_stream_file.hpp index dd898e920..31db3ef6f 100644 --- a/include/boost/corosio/native/detail/iocp/win_stream_file.hpp +++ b/include/boost/corosio/native/detail/iocp/win_stream_file.hpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -83,7 +84,7 @@ class win_stream_file final void reuse() noexcept; std::coroutine_handle<> read_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, buffer_param buf, std::stop_token token, @@ -91,7 +92,7 @@ class win_stream_file final std::size_t* bytes) override; std::coroutine_handle<> write_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, buffer_param buf, std::stop_token token, diff --git a/include/boost/corosio/native/detail/iocp/win_stream_handle_service.hpp b/include/boost/corosio/native/detail/iocp/win_stream_handle_service.hpp index bcaa9c279..b6afb9b95 100644 --- a/include/boost/corosio/native/detail/iocp/win_stream_handle_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_stream_handle_service.hpp @@ -59,25 +59,25 @@ class win_stream_handle_impl final } std::coroutine_handle<> read_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buf, std::stop_token token, std::error_code* ec, std::size_t* bytes) override { - return internal_.read_some(h, ex, buf, std::move(token), ec, bytes); + return internal_.read_some(cont, ex, buf, std::move(token), ec, bytes); } std::coroutine_handle<> write_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buf, std::stop_token token, std::error_code* ec, std::size_t* bytes) override { - return internal_.write_some(h, ex, buf, std::move(token), ec, bytes); + return internal_.write_some(cont, ex, buf, std::move(token), ec, bytes); } native_handle_type native_handle() const noexcept override diff --git a/include/boost/corosio/native/detail/iocp/win_tcp_acceptor.hpp b/include/boost/corosio/native/detail/iocp/win_tcp_acceptor.hpp index 420317910..46e70da69 100644 --- a/include/boost/corosio/native/detail/iocp/win_tcp_acceptor.hpp +++ b/include/boost/corosio/native/detail/iocp/win_tcp_acceptor.hpp @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -122,14 +123,14 @@ class win_tcp_acceptor final win_tcp_service& socket_service() noexcept; std::coroutine_handle<> accept( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, std::stop_token token, std::error_code* ec, io_object::implementation** impl_out) override; std::coroutine_handle<> wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, wait_type w, std::stop_token token, diff --git a/include/boost/corosio/native/detail/iocp/win_tcp_acceptor_service.hpp b/include/boost/corosio/native/detail/iocp/win_tcp_acceptor_service.hpp index 8994c5320..d8482dc0b 100644 --- a/include/boost/corosio/native/detail/iocp/win_tcp_acceptor_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_tcp_acceptor_service.hpp @@ -18,6 +18,7 @@ #include #include +#include #include #include @@ -213,11 +214,10 @@ accept_op::do_complete( *op->impl_out = nullptr; } - op->cont.h = op->h; auto saved_ex = op->ex; auto prevent_premature_destruction = std::move(op->object_ref_); - dispatch_coro(saved_ex, op->cont).resume(); + dispatch_coro(saved_ex, *op->cont).resume(); } // acceptor_wait_op completion handler @@ -295,7 +295,7 @@ win_tcp_acceptor::set_local_endpoint(endpoint ep) noexcept inline std::coroutine_handle<> win_tcp_acceptor::accept( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, std::stop_token token, std::error_code* ec, @@ -313,7 +313,7 @@ win_tcp_acceptor::accept( // whatever stale pointer it finds here. op.peer_wrapper = nullptr; op.accepted_socket = INVALID_SOCKET; - op.h = h; + op.cont = &cont; op.ex = d; op.ec_out = ec; op.impl_out = impl_out; @@ -403,7 +403,7 @@ win_tcp_acceptor::accept( inline std::coroutine_handle<> win_tcp_acceptor::wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, wait_type w, std::stop_token token, @@ -414,7 +414,7 @@ win_tcp_acceptor::wait( auto& op = wt_; op.reset(); - op.h = h; + op.cont = &cont; op.ex = d; op.ec_out = ec; op.bytes_out = nullptr; diff --git a/include/boost/corosio/native/detail/iocp/win_tcp_service.hpp b/include/boost/corosio/native/detail/iocp/win_tcp_service.hpp index e0b1c8b55..65a2b5823 100644 --- a/include/boost/corosio/native/detail/iocp/win_tcp_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_tcp_service.hpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -455,7 +456,7 @@ win_tcp_socket::set_endpoints(endpoint local, endpoint remote) noexcept inline std::coroutine_handle<> win_tcp_socket::connect( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, endpoint ep, std::stop_token token, @@ -466,7 +467,7 @@ win_tcp_socket::connect( auto& op = conn_; op.reset(); - op.h = h; + op.cont = &cont; op.ex = d; op.ec_out = ec; op.target_endpoint = ep; @@ -538,7 +539,7 @@ win_tcp_socket::connect( inline std::coroutine_handle<> win_tcp_socket::read_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, buffer_param param, std::stop_token token, @@ -551,7 +552,7 @@ win_tcp_socket::read_some( auto& op = rd_; op.reset(); op.is_read = true; - op.h = h; + op.cont = &cont; op.ex = d; op.ec_out = ec; op.bytes_out = bytes_out; @@ -613,7 +614,7 @@ win_tcp_socket::read_some( inline std::coroutine_handle<> win_tcp_socket::write_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, buffer_param param, std::stop_token token, @@ -625,7 +626,7 @@ win_tcp_socket::write_some( auto& op = wr_; op.reset(); - op.h = h; + op.cont = &cont; op.ex = d; op.ec_out = ec; op.bytes_out = bytes_out; @@ -684,7 +685,7 @@ win_tcp_socket::write_some( inline std::coroutine_handle<> win_tcp_socket::wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, wait_type w, std::stop_token token, @@ -694,7 +695,7 @@ win_tcp_socket::wait( auto& op = wt_; op.reset(); - op.h = h; + op.cont = &cont; op.ex = d; op.ec_out = ec; op.bytes_out = nullptr; diff --git a/include/boost/corosio/native/detail/iocp/win_tcp_socket.hpp b/include/boost/corosio/native/detail/iocp/win_tcp_socket.hpp index d28a05167..a97787da0 100644 --- a/include/boost/corosio/native/detail/iocp/win_tcp_socket.hpp +++ b/include/boost/corosio/native/detail/iocp/win_tcp_socket.hpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -168,14 +169,14 @@ class win_tcp_socket final void reuse() noexcept; std::coroutine_handle<> connect( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, endpoint ep, std::stop_token token, std::error_code* ec) override; std::coroutine_handle<> read_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, buffer_param buf, std::stop_token token, @@ -183,7 +184,7 @@ class win_tcp_socket final std::size_t* bytes) override; std::coroutine_handle<> write_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, buffer_param buf, std::stop_token token, @@ -191,7 +192,7 @@ class win_tcp_socket final std::size_t* bytes) override; std::coroutine_handle<> wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, wait_type w, std::stop_token token, diff --git a/include/boost/corosio/native/detail/iocp/win_udp_service.hpp b/include/boost/corosio/native/detail/iocp/win_udp_service.hpp index 4b9960495..794a68103 100644 --- a/include/boost/corosio/native/detail/iocp/win_udp_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_udp_service.hpp @@ -404,7 +404,7 @@ win_udp_socket::remote_endpoint() const noexcept inline std::coroutine_handle<> win_udp_socket::send_to( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, buffer_param param, endpoint dest, @@ -418,7 +418,7 @@ win_udp_socket::send_to( auto& op = wr_; op.reset(); - op.h = h; + op.cont = &cont; op.ex = d; op.ec_out = ec; op.bytes_out = bytes_out; @@ -467,7 +467,7 @@ win_udp_socket::send_to( inline std::coroutine_handle<> win_udp_socket::recv_from( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, buffer_param param, endpoint* source, @@ -481,7 +481,7 @@ win_udp_socket::recv_from( auto& op = rd_; op.reset(); - op.h = h; + op.cont = &cont; op.ex = d; op.ec_out = ec; op.bytes_out = bytes_out; @@ -540,7 +540,7 @@ win_udp_socket::recv_from( // UDP connect is synchronous on Windows inline std::coroutine_handle<> win_udp_socket::connect( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, endpoint ep, std::stop_token token, @@ -550,7 +550,7 @@ win_udp_socket::connect( auto& op = conn_; op.reset(); - op.h = h; + op.cont = &cont; op.ex = d; op.ec_out = ec; op.target_endpoint = ep; @@ -574,7 +574,7 @@ win_udp_socket::connect( inline std::coroutine_handle<> win_udp_socket::send( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, buffer_param param, int flags, @@ -586,7 +586,7 @@ win_udp_socket::send( auto& op = send_wr_; op.reset(); - op.h = h; + op.cont = &cont; op.ex = d; op.ec_out = ec; op.bytes_out = bytes_out; @@ -628,7 +628,7 @@ win_udp_socket::send( inline std::coroutine_handle<> win_udp_socket::recv( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, buffer_param param, int flags, @@ -640,7 +640,7 @@ win_udp_socket::recv( auto& op = recv_rd_; op.reset(); - op.h = h; + op.cont = &cont; op.ex = d; op.ec_out = ec; op.bytes_out = bytes_out; @@ -690,7 +690,7 @@ win_udp_socket::recv( inline std::coroutine_handle<> win_udp_socket::wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, wait_type w, std::stop_token token, @@ -700,7 +700,7 @@ win_udp_socket::wait( auto& op = wt_; op.reset(); - op.h = h; + op.cont = &cont; op.ex = d; op.ec_out = ec; op.bytes_out = nullptr; diff --git a/include/boost/corosio/native/detail/iocp/win_udp_socket.hpp b/include/boost/corosio/native/detail/iocp/win_udp_socket.hpp index e489d58ab..49bbf3d15 100644 --- a/include/boost/corosio/native/detail/iocp/win_udp_socket.hpp +++ b/include/boost/corosio/native/detail/iocp/win_udp_socket.hpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -193,7 +194,7 @@ class win_udp_socket final void reuse() noexcept; std::coroutine_handle<> send_to( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, buffer_param buf, endpoint dest, @@ -203,7 +204,7 @@ class win_udp_socket final std::size_t* bytes) override; std::coroutine_handle<> recv_from( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, buffer_param buf, endpoint* source, @@ -213,14 +214,14 @@ class win_udp_socket final std::size_t* bytes) override; std::coroutine_handle<> connect( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, endpoint ep, std::stop_token token, std::error_code* ec) override; std::coroutine_handle<> send( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, buffer_param buf, int flags, @@ -229,7 +230,7 @@ class win_udp_socket final std::size_t* bytes) override; std::coroutine_handle<> recv( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, buffer_param buf, int flags, @@ -238,7 +239,7 @@ class win_udp_socket final std::size_t* bytes) override; std::coroutine_handle<> wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, wait_type w, std::stop_token token, diff --git a/include/boost/corosio/native/detail/iocp/win_wait_reactor.hpp b/include/boost/corosio/native/detail/iocp/win_wait_reactor.hpp index bbd67fc81..6f759f9a7 100644 --- a/include/boost/corosio/native/detail/iocp/win_wait_reactor.hpp +++ b/include/boost/corosio/native/detail/iocp/win_wait_reactor.hpp @@ -55,7 +55,7 @@ namespace boost::corosio::detail { across backends. Per-op lifecycle: - 1. Caller sets up an overlapped_op (h, ex, ec_out, cancelled flag). + 1. Caller sets up an overlapped_op (cont, ex, ec_out, cancelled flag). 2. Caller calls register_wait(fd, w, op) and returns std::noop_coroutine. The op is parked in the reactor's table. 3. Reactor thread polls. When the fd is ready, the op is removed diff --git a/include/boost/corosio/native/detail/posix/posix_fd_gate.hpp b/include/boost/corosio/native/detail/posix/posix_fd_gate.hpp new file mode 100644 index 000000000..c508435b9 --- /dev/null +++ b/include/boost/corosio/native/detail/posix/posix_fd_gate.hpp @@ -0,0 +1,137 @@ +// +// Copyright (c) 2026 Steve Gerbino +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// Official repository: https://github.com/cppalliance/corosio +// + +#ifndef BOOST_COROSIO_NATIVE_DETAIL_POSIX_POSIX_FD_GATE_HPP +#define BOOST_COROSIO_NATIVE_DETAIL_POSIX_POSIX_FD_GATE_HPP + +#include + +#if BOOST_COROSIO_POSIX + +#include +#include + +#include + +namespace boost::corosio::detail { + +/** Keep a descriptor number allocated while pool workers use it. + + A worker copies the number when its operation is queued and calls + the kernel later, on another thread. Once the object closes the + descriptor, the number can be handed to any other open in the + process, and a worker still on its way to the kernel would then + transfer through an unrelated file. Workers enter the gate before + their system call and leave it after; closing while one is inside + defers the `::close` to the last one out, so the number stays + allocated to the file the worker meant. + + @par Thread Safety + Distinct objects: Safe. + Shared objects: Safe. +*/ +class posix_fd_gate +{ + std::mutex mutex_; + int fd_ = -1; + int users_ = 0; + std::vector deferred_; + +public: + /// Destroy the gate, closing any descriptor still deferred. + ~posix_fd_gate() + { + for (int fd : deferred_) + ::close(fd); + } + + posix_fd_gate() = default; + posix_fd_gate(posix_fd_gate const&) = delete; + posix_fd_gate& operator=(posix_fd_gate const&) = delete; + + /// Admit workers for @p fd, the descriptor the object now holds. + void open(int fd) noexcept + { + std::lock_guard lock(mutex_); + fd_ = fd; + } + + /** Enter the gate for a system call on @p fd. + + @return False if the object no longer holds @p fd; the caller + must not use the number. + */ + bool enter(int fd) noexcept + { + std::lock_guard lock(mutex_); + if (fd < 0 || fd != fd_) + return false; + ++users_; + return true; + } + + /// Leave the gate after a successful @ref enter. + void leave() noexcept + { + std::vector close_now; + { + std::lock_guard lock(mutex_); + if (--users_ == 0) + close_now.swap(deferred_); + } + for (int fd : close_now) + ::close(fd); + } + + /// Close the held descriptor once no worker is using it. + void close() noexcept + { + int fd = -1; + { + std::lock_guard lock(mutex_); + fd = fd_; + fd_ = -1; + if (fd >= 0 && users_ > 0) + { + deferred_.push_back(fd); + fd = -1; + } + } + if (fd >= 0) + ::close(fd); + } + + /** Stop admitting workers for the held descriptor, keeping it open. + + @return False, leaving the descriptor admitted, if a worker is + using it; the caller cannot take ownership of a number a + worker may still pass to the kernel. + */ + bool release() noexcept + { + std::lock_guard lock(mutex_); + if (users_ > 0) + return false; + fd_ = -1; + return true; + } + + /// Return true if no descriptor is held and no worker is inside. + bool idle() noexcept + { + std::lock_guard lock(mutex_); + return fd_ == -1 && users_ == 0; + } +}; + +} // namespace boost::corosio::detail + +#endif // BOOST_COROSIO_POSIX + +#endif // BOOST_COROSIO_NATIVE_DETAIL_POSIX_POSIX_FD_GATE_HPP diff --git a/include/boost/corosio/native/detail/posix/posix_random_access_file.hpp b/include/boost/corosio/native/detail/posix/posix_random_access_file.hpp index d7da706c7..792345ac4 100644 --- a/include/boost/corosio/native/detail/posix/posix_random_access_file.hpp +++ b/include/boost/corosio/native/detail/posix/posix_random_access_file.hpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -105,10 +106,6 @@ class posix_random_access_file final // Raw back-pointer for the typed work; `object_ref_` is the keepalive. posix_random_access_file* file_ = nullptr; - // The awaitable's, not the embedded `cont`: this op is freed - // before the coroutine resumes. - capy::continuation* awaiting = nullptr; - void operator()() override; void destroy() override; @@ -138,6 +135,7 @@ class posix_random_access_file final void reuse() noexcept { BOOST_COROSIO_ASSERT(fd_ == -1); + BOOST_COROSIO_ASSERT(gate_.idle()); BOOST_COROSIO_ASSERT(outstanding_ops_.empty()); } @@ -215,6 +213,8 @@ class posix_random_access_file final posix_random_access_file_service& svc_; int fd_ = -1; + /// Keeps fd_'s number allocated while a pool worker uses it. + posix_fd_gate gate_; std::mutex ops_mutex_; intrusive_list outstanding_ops_; intrusive_list free_ops_; @@ -261,6 +261,7 @@ posix_random_access_file::open_file( return make_err(errno); fd_ = fd; + gate_.open(fd); #ifdef POSIX_FADV_RANDOM ::posix_fadvise(fd_, 0, 0, POSIX_FADV_RANDOM); @@ -272,11 +273,8 @@ posix_random_access_file::open_file( inline void posix_random_access_file::close_file() noexcept { - if (fd_ >= 0) - { - ::close(fd_); - fd_ = -1; - } + gate_.close(); + fd_ = -1; } inline std::uint64_t @@ -325,22 +323,15 @@ posix_random_access_file::release() // A queued op has already copied the fd number; it must not run // after the caller closes it and the number is recycled. cancel(); + if (!gate_.release()) + throw_system_error( + std::make_error_code(std::errc::device_or_resource_busy), + "random_access_file::release"); int fd = fd_; fd_ = -1; return fd; } -inline std::error_code -posix_random_access_file::assign(native_handle_type handle) noexcept -{ - // The public assign() guarantees the object is closed. - if (auto ec = validate_file_fd(handle)) - return ec; - - fd_ = handle; - return {}; -} - // read_some_at, write_some_at are defined in // posix_random_access_file_service.hpp after the service. @@ -363,7 +354,7 @@ posix_random_access_file::raf_op::operator()() // keepalive drops after the push so a final release never runs // under ops_mutex_. auto keep = std::move(object_ref_); - auto* c = awaiting; + auto* c = cont; auto exec = ex; { std::lock_guard lock(file_->ops_mutex_); diff --git a/include/boost/corosio/native/detail/posix/posix_random_access_file_service.hpp b/include/boost/corosio/native/detail/posix/posix_random_access_file_service.hpp index 317d291ed..9fc0f0f5a 100644 --- a/include/boost/corosio/native/detail/posix/posix_random_access_file_service.hpp +++ b/include/boost/corosio/native/detail/posix/posix_random_access_file_service.hpp @@ -74,14 +74,22 @@ class BOOST_COROSIO_DECL posix_random_access_file_service final std::filesystem::path const& path, file_base::flags mode) override { - // Unavailable in the unsafe tier: the file thread pool completes - // cross-thread, which the lockless scheduler cannot accept. - if (sched_->scheduler_locking_disabled()) + if (pool_unavailable()) return std::make_error_code(std::errc::operation_not_supported); return static_cast(impl).open_file( path, mode); } + /** Check whether file I/O is unavailable on this context. + + The file thread pool completes cross-thread, which the + lockless scheduler of the unsafe tier cannot accept. + */ + bool pool_unavailable() const noexcept + { + return sched_->scheduler_locking_disabled(); + } + void shutdown() override { // See uring_socket_service_base::shutdown(): snapshot under an @@ -187,8 +195,7 @@ posix_random_access_file::read_some_at( op->iovecs[i].iov_len = bufs[i].size(); } - op->h = cont.h; - op->awaiting = &cont; + op->cont = &cont; op->ex = ex; op->ec_out = ec; op->bytes_out = bytes_out; @@ -260,8 +267,7 @@ posix_random_access_file::write_some_at( op->iovecs[i].iov_len = bufs[i].size(); } - op->h = cont.h; - op->awaiting = &cont; + op->cont = &cont; op->ex = ex; op->ec_out = ec; op->bytes_out = bytes_out; @@ -313,6 +319,20 @@ posix_random_access_file::raf_op::do_work(pool_work_item* w) noexcept op->errn = EOVERFLOW; op->bytes_transferred = 0; } + else if (!self->gate_.enter(op->fd)) + { + // Closed or replaced since this was queued. + op->errn = ECANCELED; + op->bytes_transferred = 0; + } + else if (op->cancelled.load(std::memory_order_acquire)) + { + // Closed and reopened onto the same number since the check + // above: the gate admits the number, but not this op. + self->gate_.leave(); + op->errn = ECANCELED; + op->bytes_transferred = 0; + } else { ssize_t n; @@ -347,11 +367,26 @@ posix_random_access_file::raf_op::do_work(pool_work_item* w) noexcept op->errn = errno; op->bytes_transferred = 0; } + self->gate_.leave(); } self->svc_.post(static_cast(op)); } +inline std::error_code +posix_random_access_file::assign(native_handle_type handle) noexcept +{ + // The public assign() guarantees the object is closed. + if (svc_.pool_unavailable()) + return std::make_error_code(std::errc::operation_not_supported); + if (auto ec = validate_file_fd(handle)) + return ec; + + fd_ = handle; + gate_.open(handle); + return {}; +} + inline void posix_random_access_file::retire() noexcept { diff --git a/include/boost/corosio/native/detail/posix/posix_resolver.hpp b/include/boost/corosio/native/detail/posix/posix_resolver.hpp index 0498b022d..2cfc35275 100644 --- a/include/boost/corosio/native/detail/posix/posix_resolver.hpp +++ b/include/boost/corosio/native/detail/posix/posix_resolver.hpp @@ -16,6 +16,7 @@ #include #include +#include #include #include @@ -245,7 +246,7 @@ class posix_resolver final } std::coroutine_handle<> resolve( - std::coroutine_handle<>, + capy::continuation&, capy::executor_ref, std::string_view host, std::string_view service, @@ -255,7 +256,7 @@ class posix_resolver final std::vector*) override; std::coroutine_handle<> reverse_resolve( - std::coroutine_handle<>, + capy::continuation&, capy::executor_ref, endpoint const& ep, reverse_flags flags, diff --git a/include/boost/corosio/native/detail/posix/posix_resolver_service.hpp b/include/boost/corosio/native/detail/posix/posix_resolver_service.hpp index 7bcc216d2..3ba36da51 100644 --- a/include/boost/corosio/native/detail/posix/posix_resolver_service.hpp +++ b/include/boost/corosio/native/detail/posix/posix_resolver_service.hpp @@ -273,8 +273,7 @@ posix_resolver::resolve_op::operator()() // reference to the implementation this op is embedded in. auto prevent_destroy = std::move(object_ref_); ex.on_work_finished(); - cont.h = h; - dispatch_coro(ex, cont).resume(); + dispatch_coro(ex, *cont).resume(); } inline void @@ -330,8 +329,7 @@ posix_resolver::reverse_resolve_op::operator()() // reference to the implementation this op is embedded in. auto prevent_destroy = std::move(object_ref_); ex.on_work_finished(); - cont.h = h; - dispatch_coro(ex, cont).resume(); + dispatch_coro(ex, *cont).resume(); } inline void @@ -348,7 +346,7 @@ posix_resolver::reverse_resolve_op::destroy() inline std::coroutine_handle<> posix_resolver::resolve( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, std::string_view host, std::string_view service, @@ -360,13 +358,12 @@ posix_resolver::resolve( if (svc_.resolver_unavailable()) { *ec = std::make_error_code(std::errc::operation_not_supported); - op_.cont.h = h; - return dispatch_coro(ex, op_.cont); + return dispatch_coro(ex, cont); } auto& op = op_; op.reset(); - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.out = out; @@ -392,15 +389,14 @@ posix_resolver::resolve( op.stop_cb.reset(); op.ex.on_work_finished(); *ec = pec; - op.cont.h = h; - return dispatch_coro(ex, op.cont); + return dispatch_coro(ex, cont); } return std::noop_coroutine(); } inline std::coroutine_handle<> posix_resolver::reverse_resolve( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, endpoint const& ep, reverse_flags flags, @@ -411,13 +407,12 @@ posix_resolver::reverse_resolve( if (svc_.resolver_unavailable()) { *ec = std::make_error_code(std::errc::operation_not_supported); - reverse_op_.cont.h = h; - return dispatch_coro(ex, reverse_op_.cont); + return dispatch_coro(ex, cont); } auto& op = reverse_op_; op.reset(); - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.result_out = result_out; @@ -442,8 +437,7 @@ posix_resolver::reverse_resolve( op.stop_cb.reset(); op.ex.on_work_finished(); *ec = pec; - op.cont.h = h; - return dispatch_coro(ex, op.cont); + return dispatch_coro(ex, cont); } return std::noop_coroutine(); } diff --git a/include/boost/corosio/native/detail/posix/posix_signal.hpp b/include/boost/corosio/native/detail/posix/posix_signal.hpp index 7ba5fc133..7e3aae078 100644 --- a/include/boost/corosio/native/detail/posix/posix_signal.hpp +++ b/include/boost/corosio/native/detail/posix/posix_signal.hpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -45,9 +46,11 @@ enum struct signal_op : scheduler_op { - std::coroutine_handle<> h; - capy::continuation cont; + /// The awaitable's continuation; see coro_op::cont. + capy::continuation* cont = nullptr; capy::executor_ref d; + /// Holds the owning set while this op sits in the scheduler queue. + detail::object_ref keep; std::error_code* ec_out = nullptr; int* signal_out = nullptr; int signal_number = 0; @@ -149,7 +152,7 @@ class posix_signal final } std::coroutine_handle<> wait( - std::coroutine_handle<>, + capy::continuation&, capy::executor_ref, std::stop_token, std::error_code*, diff --git a/include/boost/corosio/native/detail/posix/posix_signal_service.hpp b/include/boost/corosio/native/detail/posix/posix_signal_service.hpp index a9502d91b..a3197f5e9 100644 --- a/include/boost/corosio/native/detail/posix/posix_signal_service.hpp +++ b/include/boost/corosio/native/detail/posix/posix_signal_service.hpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -414,12 +415,15 @@ signal_op::operator()() if (signal_out) *signal_out = signal_number; - // Capture svc before resuming (coro may destroy us) + // Copied out before the reference drops: this op is embedded in + // the set, which the release may recycle. auto* service = svc; svc = nullptr; + auto* c = cont; + auto ex = d; + auto k = std::move(keep); - cont.h = h; - d.post(cont); + ex.post(*c); // Balance the work_started() from start_wait if (service) @@ -429,7 +433,9 @@ signal_op::operator()() inline void signal_op::destroy() { - // No-op: signal_op is embedded in posix_signal + // Embedded in posix_signal: dropping the queued reference is the + // only cleanup, and nothing touches this op afterwards. + auto k = std::move(keep); } // posix_signal implementation @@ -441,13 +447,13 @@ inline posix_signal::posix_signal(posix_signal_service& svc) noexcept inline std::coroutine_handle<> posix_signal::wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref d, std::stop_token token, std::error_code* ec, int* signal_out) { - pending_op_.h = h; + pending_op_.cont = &cont; pending_op_.d = d; pending_op_.ec_out = ec; pending_op_.signal_out = signal_out; @@ -467,8 +473,7 @@ posix_signal::wait( *ec = make_error_code(capy::error::canceled); if (signal_out) *signal_out = 0; - pending_op_.cont.h = h; - d.post(pending_op_.cont); + d.post(*pending_op_.cont); // completion is always posted to scheduler queue, never inline. return std::noop_coroutine(); } @@ -626,13 +631,12 @@ posix_signal_service::add_signal( posix_signal_detail::get_signal_state(); // Ensure the global self-pipe exists and this service's scheduler is - // watching its read end, BEFORE taking the registration locks. The - // reactor drain path locks the descriptor mutex and then the signal-state - // and service mutexes; register_signal_reader locks the descriptor mutex - // (via register_descriptor), so it must run holding neither of those or - // the lock order would invert (a real deadlock, caught by TSan). call_once - // makes the once-per-service registration safe when two signal_sets on - // this context race add() from different threads. + // watching its read end, BEFORE taking the registration locks: + // register_signal_reader takes the descriptor mutex (via + // register_descriptor), which no path may take while holding the + // signal-state or service mutexes. call_once makes the once-per-service + // registration safe when two signal_sets on this context race add() + // from different threads. { std::lock_guard state_lock(state->mutex); if (auto ec = posix_signal_detail::open_signal_pipe(state)) @@ -844,8 +848,7 @@ posix_signal_service::cancel_wait(posix_signal& impl) *op->ec_out = make_error_code(capy::error::canceled); if (op->signal_out) *op->signal_out = 0; - op->cont.h = op->h; - op->d.post(op->cont); + op->d.post(*op->cont); sched_->work_finished(); } } @@ -876,8 +879,7 @@ posix_signal_service::cancel_wait_token(posix_signal& impl) noexcept *op->ec_out = make_error_code(capy::error::canceled); if (op->signal_out) *op->signal_out = 0; - op->cont.h = op->h; - op->d.post(op->cont); + op->d.post(*op->cont); sched_->work_finished(); } } @@ -902,8 +904,7 @@ posix_signal_service::start_wait(posix_signal& impl, signal_op* op) *op->ec_out = make_error_code(capy::error::canceled); if (op->signal_out) *op->signal_out = 0; - op->cont.h = op->h; - op->d.post(op->cont); + op->d.post(*op->cont); return; } @@ -916,8 +917,7 @@ posix_signal_service::start_wait(posix_signal& impl, signal_op* op) *op->ec_out = make_error_code(capy::error::canceled); if (op->signal_out) *op->signal_out = 0; - op->cont.h = op->h; - op->d.post(op->cont); + op->d.post(*op->cont); return; } @@ -931,6 +931,8 @@ posix_signal_service::start_wait(posix_signal& impl, signal_op* op) op->signal_number = reg->signal_number; // svc=nullptr: no work_finished needed since we never called work_started op->svc = nullptr; + // The caller's wait() holds the handle's reference. + op->keep = detail::object_ref(&impl); sched_->post(op); return; } @@ -967,9 +969,17 @@ posix_signal_service::deliver_signal(int signal_number) if (impl->waiting_) { - impl->waiting_ = false; - impl->pending_op_.signal_number = signal_number; - service->post(&impl->pending_op_); + // Reached through the registry, not a held reference. + // A set whose count reached zero is being destroyed, + // and that destruction completes its wait itself. + auto keep = detail::object_ref::try_from(impl); + if (keep) + { + impl->waiting_ = false; + impl->pending_op_.signal_number = signal_number; + impl->pending_op_.keep = std::move(keep); + service->post(&impl->pending_op_); + } } else { diff --git a/include/boost/corosio/native/detail/posix/posix_stream_file.hpp b/include/boost/corosio/native/detail/posix/posix_stream_file.hpp index f59483311..c5b3f4253 100644 --- a/include/boost/corosio/native/detail/posix/posix_stream_file.hpp +++ b/include/boost/corosio/native/detail/posix/posix_stream_file.hpp @@ -27,7 +27,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -160,6 +162,7 @@ class posix_stream_file final void reuse() noexcept { BOOST_COROSIO_ASSERT(fd_ == -1); + BOOST_COROSIO_ASSERT(gate_.idle()); BOOST_COROSIO_ASSERT(!read_op_.stop_cb); BOOST_COROSIO_ASSERT(!write_op_.stop_cb); } @@ -167,7 +170,7 @@ class posix_stream_file final // -- io_stream::implementation -- std::coroutine_handle<> read_some( - std::coroutine_handle<>, + capy::continuation&, capy::executor_ref, buffer_param, std::stop_token, @@ -175,7 +178,7 @@ class posix_stream_file final std::size_t*) override; std::coroutine_handle<> write_some( - std::coroutine_handle<>, + capy::continuation&, capy::executor_ref, buffer_param, std::stop_token, @@ -222,6 +225,8 @@ class posix_stream_file final // only if this still matches its op's snapshot; the mutex makes the // check and the advance one step against a concurrent assign(). std::mutex offset_mutex_; + /// Keeps fd_'s number allocated while a pool worker uses it. + posix_fd_gate gate_; std::uint64_t generation_ = 0; file_op read_op_; @@ -280,6 +285,7 @@ posix_stream_file::open_file( fd_ = fd; offset_ = 0; + gate_.open(fd); // Append mode: position at end-of-file (preadv/pwritev use // explicit offsets, so O_APPEND alone is not sufficient). @@ -289,7 +295,7 @@ posix_stream_file::open_file( if (file_fstat(fd, &st) < 0) { int err = errno; - ::close(fd); + gate_.close(); fd_ = -1; return make_err(err); } @@ -314,11 +320,8 @@ inline void posix_stream_file::close_file() noexcept { bump_generation(); - if (fd_ >= 0) - { - ::close(fd_); - fd_ = -1; - } + gate_.close(); + fd_ = -1; } inline std::uint64_t @@ -367,6 +370,10 @@ posix_stream_file::release() // A queued op has already copied the fd number; it must not run // after the caller closes it and the number is recycled. cancel(); + if (!gate_.release()) + throw_system_error( + std::make_error_code(std::errc::device_or_resource_busy), + "stream_file::release"); bump_generation(); int fd = fd_; fd_ = -1; @@ -374,18 +381,6 @@ posix_stream_file::release() return fd; } -inline std::error_code -posix_stream_file::assign(native_handle_type handle) noexcept -{ - // The public assign() guarantees the object is closed. - if (auto ec = validate_file_fd(handle)) - return ec; - - fd_ = handle; - offset_ = 0; - return {}; -} - inline capy::io_result posix_stream_file::seek( std::int64_t offset, file_base::seek_basis origin) noexcept @@ -442,8 +437,7 @@ posix_stream_file::file_op::operator()() // posix_stream_file (which embeds this file_op) alive. auto prevent_destroy = std::move(object_ref_); ex.on_work_finished(); - cont.h = h; - dispatch_coro(ex, cont).resume(); + dispatch_coro(ex, *cont).resume(); } inline void diff --git a/include/boost/corosio/native/detail/posix/posix_stream_file_service.hpp b/include/boost/corosio/native/detail/posix/posix_stream_file_service.hpp index 75fd6ff34..03b038546 100644 --- a/include/boost/corosio/native/detail/posix/posix_stream_file_service.hpp +++ b/include/boost/corosio/native/detail/posix/posix_stream_file_service.hpp @@ -75,13 +75,21 @@ class BOOST_COROSIO_DECL posix_stream_file_service final : public file_service std::filesystem::path const& path, file_base::flags mode) override { - // Unavailable in the unsafe tier: the file thread pool completes - // cross-thread, which the lockless scheduler cannot accept. - if (sched_->scheduler_locking_disabled()) + if (pool_unavailable()) return std::make_error_code(std::errc::operation_not_supported); return static_cast(impl).open_file(path, mode); } + /** Check whether file I/O is unavailable on this context. + + The file thread pool completes cross-thread, which the + lockless scheduler of the unsafe tier cannot accept. + */ + bool pool_unavailable() const noexcept + { + return sched_->scheduler_locking_disabled(); + } + void shutdown() override { // See uring_socket_service_base::shutdown(): snapshot under an @@ -149,7 +157,7 @@ class BOOST_COROSIO_DECL posix_stream_file_service final : public file_service inline std::coroutine_handle<> posix_stream_file::read_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param param, std::stop_token token, @@ -165,8 +173,7 @@ posix_stream_file::read_some( { *ec = make_error_code(std::errc::bad_file_descriptor); *bytes_out = 0; - op.cont.h = h; - return dispatch_coro(ex, op.cont); + return dispatch_coro(ex, cont); } capy::mutable_buffer bufs[max_buffers]; @@ -176,8 +183,7 @@ posix_stream_file::read_some( { *ec = {}; *bytes_out = 0; - op.cont.h = h; - return dispatch_coro(ex, op.cont); + return dispatch_coro(ex, cont); } for (int i = 0; i < op.iovec_count; ++i) @@ -186,7 +192,7 @@ posix_stream_file::read_some( op.iovecs[i].iov_len = bufs[i].size(); } - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.bytes_out = bytes_out; @@ -212,8 +218,7 @@ posix_stream_file::read_some( op.ex.on_work_finished(); *ec = pec; *bytes_out = 0; - op.cont.h = h; - return dispatch_coro(ex, op.cont); + return dispatch_coro(ex, cont); } return std::noop_coroutine(); } @@ -227,30 +232,45 @@ posix_stream_file::do_read_work(pool_work_item* w) noexcept if (!op.cancelled.load(std::memory_order_acquire)) { - ssize_t n; - do + if (!self->gate_.enter(op.fd)) { - n = file_preadv( - op.fd, op.iovecs, op.iovec_count, - static_cast(op.offset)); + // Closed or replaced since this was queued. + op.request_cancel(); } - while (n < 0 && errno == EINTR); - - if (n >= 0) + else if (op.cancelled.load(std::memory_order_acquire)) { - op.errn = 0; - op.bytes_transferred = static_cast(n); - - // The file may have been replaced while this ran; its - // position belongs to whatever it now holds. - std::lock_guard lock(self->offset_mutex_); - if (self->generation_ == op.generation) - self->offset_ += static_cast(n); + // Closed and reopened onto the same number since the check + // above: the gate admits the number, but not this op. + self->gate_.leave(); } else { - op.errn = errno; - op.bytes_transferred = 0; + ssize_t n; + do + { + n = file_preadv( + op.fd, op.iovecs, op.iovec_count, + static_cast(op.offset)); + } + while (n < 0 && errno == EINTR); + + if (n >= 0) + { + op.errn = 0; + op.bytes_transferred = static_cast(n); + + // The file may have been replaced while this ran; its + // position belongs to whatever it now holds. + std::lock_guard lock(self->offset_mutex_); + if (self->generation_ == op.generation) + self->offset_ += static_cast(n); + } + else + { + op.errn = errno; + op.bytes_transferred = 0; + } + self->gate_.leave(); } } @@ -260,7 +280,7 @@ posix_stream_file::do_read_work(pool_work_item* w) noexcept inline std::coroutine_handle<> posix_stream_file::write_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param param, std::stop_token token, @@ -276,8 +296,7 @@ posix_stream_file::write_some( { *ec = make_error_code(std::errc::bad_file_descriptor); *bytes_out = 0; - op.cont.h = h; - return dispatch_coro(ex, op.cont); + return dispatch_coro(ex, cont); } capy::mutable_buffer bufs[max_buffers]; @@ -287,8 +306,7 @@ posix_stream_file::write_some( { *ec = {}; *bytes_out = 0; - op.cont.h = h; - return dispatch_coro(ex, op.cont); + return dispatch_coro(ex, cont); } for (int i = 0; i < op.iovec_count; ++i) @@ -297,7 +315,7 @@ posix_stream_file::write_some( op.iovecs[i].iov_len = bufs[i].size(); } - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.bytes_out = bytes_out; @@ -323,8 +341,7 @@ posix_stream_file::write_some( op.ex.on_work_finished(); *ec = pec; *bytes_out = 0; - op.cont.h = h; - return dispatch_coro(ex, op.cont); + return dispatch_coro(ex, cont); } return std::noop_coroutine(); } @@ -338,30 +355,45 @@ posix_stream_file::do_write_work(pool_work_item* w) noexcept if (!op.cancelled.load(std::memory_order_acquire)) { - ssize_t n; - do + if (!self->gate_.enter(op.fd)) { - n = file_pwritev( - op.fd, op.iovecs, op.iovec_count, - static_cast(op.offset)); + // Closed or replaced since this was queued. + op.request_cancel(); } - while (n < 0 && errno == EINTR); - - if (n >= 0) + else if (op.cancelled.load(std::memory_order_acquire)) { - op.errn = 0; - op.bytes_transferred = static_cast(n); - - // The file may have been replaced while this ran; its - // position belongs to whatever it now holds. - std::lock_guard lock(self->offset_mutex_); - if (self->generation_ == op.generation) - self->offset_ += static_cast(n); + // Closed and reopened onto the same number since the check + // above: the gate admits the number, but not this op. + self->gate_.leave(); } else { - op.errn = errno; - op.bytes_transferred = 0; + ssize_t n; + do + { + n = file_pwritev( + op.fd, op.iovecs, op.iovec_count, + static_cast(op.offset)); + } + while (n < 0 && errno == EINTR); + + if (n >= 0) + { + op.errn = 0; + op.bytes_transferred = static_cast(n); + + // The file may have been replaced while this ran; its + // position belongs to whatever it now holds. + std::lock_guard lock(self->offset_mutex_); + if (self->generation_ == op.generation) + self->offset_ += static_cast(n); + } + else + { + op.errn = errno; + op.bytes_transferred = 0; + } + self->gate_.leave(); } } @@ -369,6 +401,21 @@ posix_stream_file::do_write_work(pool_work_item* w) noexcept self->svc_.post(&op); } +inline std::error_code +posix_stream_file::assign(native_handle_type handle) noexcept +{ + // The public assign() guarantees the object is closed. + if (svc_.pool_unavailable()) + return std::make_error_code(std::errc::operation_not_supported); + if (auto ec = validate_file_fd(handle)) + return ec; + + fd_ = handle; + offset_ = 0; + gate_.open(handle); + return {}; +} + inline void posix_stream_file::retire() noexcept { diff --git a/include/boost/corosio/native/detail/reactor/reactor_acceptor.hpp b/include/boost/corosio/native/detail/reactor/reactor_acceptor.hpp index 29dcc72d8..6b6c51037 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_acceptor.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_acceptor.hpp @@ -212,13 +212,13 @@ class reactor_acceptor } std::coroutine_handle<> wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, wait_type w, std::stop_token token, std::error_code* ec) override { - return do_wait(h, ex, w, token, ec); + return do_wait(cont, ex, w, token, ec); } /** Wait for readiness on the listen socket. @@ -233,7 +233,7 @@ class reactor_acceptor listening socket. */ std::coroutine_handle<> do_wait( - std::coroutine_handle<>, + capy::continuation&, capy::executor_ref, wait_type, std::stop_token const&, @@ -456,7 +456,7 @@ reactor_acceptor< ImplBase, Endpoint>:: do_wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, wait_type w, std::stop_token const& token, @@ -470,7 +470,7 @@ reactor_acceptor< auto& op = wait_wr_; op.reset(); op.wait_event = reactor_event_write; - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.fd = this->fd_; @@ -501,7 +501,7 @@ reactor_acceptor< auto& op = *op_ptr; op.reset(); op.wait_event = event; - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.fd = this->fd_; diff --git a/include/boost/corosio/native/detail/reactor/reactor_backend.hpp b/include/boost/corosio/native/detail/reactor/reactor_backend.hpp index 0bcaee5d0..51e16e271 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_backend.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_backend.hpp @@ -47,7 +47,7 @@ reactor_acceptor_impl< AccImplBase, Endpoint>:: accept( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, std::stop_token token, std::error_code* ec, @@ -55,7 +55,7 @@ reactor_acceptor_impl< { auto& op = this->acc_; op.reset(); - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.impl_out = impl_out; @@ -122,8 +122,7 @@ reactor_acceptor_impl< } // Completed without parking: nothing left for a stop to cancel. op.stop_cb.reset(); - op.cont.h = h; - return dispatch_coro(ex, op.cont); + return dispatch_coro(ex, cont); } op.accepted_fd = accepted; diff --git a/include/boost/corosio/native/detail/reactor/reactor_datagram_socket.hpp b/include/boost/corosio/native/detail/reactor/reactor_datagram_socket.hpp index 73ccc88e7..a7ee251df 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_datagram_socket.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_datagram_socket.hpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -132,7 +133,7 @@ class reactor_datagram_socket // --- Virtual method overrides (satisfy ImplBase pure virtuals) --- std::coroutine_handle<> send_to( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buf, Endpoint dest, @@ -141,11 +142,11 @@ class reactor_datagram_socket std::error_code* ec, std::size_t* bytes_out) override { - return do_send_to(h, ex, buf, dest, flags, token, ec, bytes_out); + return do_send_to(cont, ex, buf, dest, flags, token, ec, bytes_out); } std::coroutine_handle<> recv_from( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buf, Endpoint* source, @@ -154,21 +155,21 @@ class reactor_datagram_socket std::error_code* ec, std::size_t* bytes_out) override { - return do_recv_from(h, ex, buf, source, flags, token, ec, bytes_out); + return do_recv_from(cont, ex, buf, source, flags, token, ec, bytes_out); } std::coroutine_handle<> connect( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, Endpoint ep, std::stop_token token, std::error_code* ec) override { - return do_connect(h, ex, ep, token, ec); + return do_connect(cont, ex, ep, token, ec); } std::coroutine_handle<> send( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buf, int flags, @@ -176,11 +177,11 @@ class reactor_datagram_socket std::error_code* ec, std::size_t* bytes_out) override { - return do_send(h, ex, buf, flags, token, ec, bytes_out); + return do_send(cont, ex, buf, flags, token, ec, bytes_out); } std::coroutine_handle<> recv( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buf, int flags, @@ -188,7 +189,7 @@ class reactor_datagram_socket std::error_code* ec, std::size_t* bytes_out) override { - return do_recv(h, ex, buf, flags, token, ec, bytes_out); + return do_recv(cont, ex, buf, flags, token, ec, bytes_out); } void cancel() noexcept override @@ -197,13 +198,13 @@ class reactor_datagram_socket } std::coroutine_handle<> wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, wait_type w, std::stop_token token, std::error_code* ec) override { - return do_wait(h, ex, w, token, ec); + return do_wait(cont, ex, w, token, ec); } // --- End virtual overrides --- @@ -228,7 +229,7 @@ class reactor_datagram_socket On EAGAIN, registers with the reactor. */ std::coroutine_handle<> do_send_to( - std::coroutine_handle<>, + capy::continuation&, capy::executor_ref, buffer_param, Endpoint const&, @@ -244,7 +245,7 @@ class reactor_datagram_socket On EAGAIN, registers with the reactor. */ std::coroutine_handle<> do_recv_from( - std::coroutine_handle<>, + capy::continuation&, capy::executor_ref, buffer_param, Endpoint*, @@ -261,7 +262,7 @@ class reactor_datagram_socket records the peer address in the kernel. */ std::coroutine_handle<> do_connect( - std::coroutine_handle<>, + capy::continuation&, capy::executor_ref, Endpoint const&, std::stop_token const&, @@ -273,7 +274,7 @@ class reactor_datagram_socket with msg_name=nullptr. */ std::coroutine_handle<> do_send( - std::coroutine_handle<>, + capy::continuation&, capy::executor_ref, buffer_param, int flags, @@ -287,7 +288,7 @@ class reactor_datagram_socket with msg_name=nullptr. */ std::coroutine_handle<> do_recv( - std::coroutine_handle<>, + capy::continuation&, capy::executor_ref, buffer_param, int flags, @@ -305,7 +306,7 @@ class reactor_datagram_socket completes only while a non-blocking write can make progress. */ std::coroutine_handle<> do_wait( - std::coroutine_handle<>, + capy::continuation&, capy::executor_ref, wait_type, std::stop_token const&, @@ -471,7 +472,7 @@ reactor_datagram_socket< ImplBase, Endpoint>:: do_send_to( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param param, Endpoint const& dest, @@ -527,10 +528,9 @@ reactor_datagram_socket< { *ec = err ? make_err(err) : std::error_code{}; *bytes_out = bytes; - op.cont.h = h; - return dispatch_coro(ex, op.cont); + return dispatch_coro(ex, cont); } - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.bytes_out = bytes_out; @@ -542,7 +542,7 @@ reactor_datagram_socket< } // EAGAIN — register with reactor - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.bytes_out = bytes_out; @@ -582,7 +582,7 @@ reactor_datagram_socket< ImplBase, Endpoint>:: do_recv_from( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param param, Endpoint* source, @@ -600,7 +600,7 @@ reactor_datagram_socket< if (op.iovec_count == 0 || (op.iovec_count == 1 && bufs[0].size() == 0)) { - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.bytes_out = bytes_out; @@ -649,10 +649,9 @@ reactor_datagram_socket< if (source && !err && n >= 0) *source = from_sockaddr_as( op.source_storage, op.source_addrlen, Endpoint{}); - op.cont.h = h; - return dispatch_coro(ex, op.cont); + return dispatch_coro(ex, cont); } - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.bytes_out = bytes_out; @@ -664,7 +663,7 @@ reactor_datagram_socket< } // EAGAIN — register with reactor - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.bytes_out = bytes_out; @@ -704,7 +703,7 @@ reactor_datagram_socket< ImplBase, Endpoint>:: do_connect( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, Endpoint const& ep, std::stop_token const& token, @@ -736,11 +735,10 @@ reactor_datagram_socket< if (this->svc_.scheduler().try_consume_inline_budget()) { *ec = err ? make_err(err) : std::error_code{}; - op.cont.h = h; - return dispatch_coro(ex, op.cont); + return dispatch_coro(ex, cont); } op.reset(); - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.fd = this->fd_; @@ -780,7 +778,7 @@ reactor_datagram_socket< ImplBase, Endpoint>:: do_send( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param param, int flags, @@ -830,10 +828,9 @@ reactor_datagram_socket< { *ec = err ? make_err(err) : std::error_code{}; *bytes_out = bytes; - op.cont.h = h; - return dispatch_coro(ex, op.cont); + return dispatch_coro(ex, cont); } - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.bytes_out = bytes_out; @@ -845,7 +842,7 @@ reactor_datagram_socket< } // EAGAIN — register with reactor - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.bytes_out = bytes_out; @@ -885,7 +882,7 @@ reactor_datagram_socket< ImplBase, Endpoint>:: do_recv( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param param, int flags, @@ -901,7 +898,7 @@ reactor_datagram_socket< if (op.iovec_count == 0 || (op.iovec_count == 1 && bufs[0].size() == 0)) { - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.bytes_out = bytes_out; @@ -942,10 +939,9 @@ reactor_datagram_socket< { *ec = err ? make_err(err) : std::error_code{}; *bytes_out = bytes; - op.cont.h = h; - return dispatch_coro(ex, op.cont); + return dispatch_coro(ex, cont); } - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.bytes_out = bytes_out; @@ -957,7 +953,7 @@ reactor_datagram_socket< } // EAGAIN — register with reactor - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.bytes_out = bytes_out; @@ -997,7 +993,7 @@ reactor_datagram_socket< ImplBase, Endpoint>:: do_wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, wait_type w, std::stop_token const& token, @@ -1038,12 +1034,11 @@ reactor_datagram_socket< if (this->svc_.scheduler().try_consume_inline_budget()) { *ec = perr ? make_err(perr) : std::error_code{}; - op.cont.h = h; - return dispatch_coro(ex, op.cont); + return dispatch_coro(ex, cont); } op.reset(); op.wait_event = event; - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.fd = this->fd_; @@ -1056,7 +1051,7 @@ reactor_datagram_socket< op.reset(); op.wait_event = event; - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.fd = this->fd_; diff --git a/include/boost/corosio/native/detail/reactor/reactor_descriptor.hpp b/include/boost/corosio/native/detail/reactor/reactor_descriptor.hpp index 7623d79ee..96f48df6c 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_descriptor.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_descriptor.hpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -224,35 +225,35 @@ class reactor_descriptor // --- Virtual method overrides --- std::coroutine_handle<> read_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param param, std::stop_token token, std::error_code* ec, std::size_t* bytes_out) override { - return do_read_some(h, ex, param, token, ec, bytes_out); + return do_read_some(cont, ex, param, token, ec, bytes_out); } std::coroutine_handle<> write_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param param, std::stop_token token, std::error_code* ec, std::size_t* bytes_out) override { - return do_write_some(h, ex, param, token, ec, bytes_out); + return do_write_some(cont, ex, param, token, ec, bytes_out); } std::coroutine_handle<> wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, wait_type w, std::stop_token token, std::error_code* ec) override { - return do_wait(h, ex, w, token, ec); + return do_wait(cont, ex, w, token, ec); } native_handle_type native_handle() const noexcept override @@ -337,7 +338,7 @@ class reactor_descriptor } std::coroutine_handle<> do_read_some( - std::coroutine_handle<>, + capy::continuation&, capy::executor_ref, buffer_param, std::stop_token const&, @@ -345,7 +346,7 @@ class reactor_descriptor std::size_t*); std::coroutine_handle<> do_write_some( - std::coroutine_handle<>, + capy::continuation&, capy::executor_ref, buffer_param, std::stop_token const&, @@ -353,7 +354,7 @@ class reactor_descriptor std::size_t*); std::coroutine_handle<> do_wait( - std::coroutine_handle<>, + capy::continuation&, capy::executor_ref, wait_type, std::stop_token const&, @@ -479,7 +480,7 @@ reactor_descriptor::op_to_desc_slot( template std::coroutine_handle<> reactor_descriptor::do_read_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param param, std::stop_token const& token, @@ -488,7 +489,7 @@ reactor_descriptor::do_read_some( { auto& op = rd_; op.reset(); - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.bytes_out = bytes_out; @@ -569,8 +570,7 @@ reactor_descriptor::do_read_some( else *ec = {}; *bytes_out = bytes; - op.cont.h = h; - return dispatch_coro(ex, op.cont); + return dispatch_coro(ex, cont); } op.start(token, static_cast(this)); op.object_ref_ = detail::object_ref(this); @@ -591,7 +591,7 @@ reactor_descriptor::do_read_some( template std::coroutine_handle<> reactor_descriptor::do_write_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param param, std::stop_token const& token, @@ -600,7 +600,7 @@ reactor_descriptor::do_write_some( { auto& op = wr_; op.reset(); - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.bytes_out = bytes_out; @@ -663,8 +663,7 @@ reactor_descriptor::do_write_some( { *ec = err ? make_err(err) : std::error_code{}; *bytes_out = bytes; - op.cont.h = h; - return dispatch_coro(ex, op.cont); + return dispatch_coro(ex, cont); } op.start(token, static_cast(this)); op.object_ref_ = detail::object_ref(this); @@ -685,7 +684,7 @@ reactor_descriptor::do_write_some( template std::coroutine_handle<> reactor_descriptor::do_wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, wait_type w, std::stop_token const& token, @@ -727,12 +726,11 @@ reactor_descriptor::do_wait( if (svc_.scheduler().try_consume_inline_budget()) { *ec = perr ? make_err(perr) : std::error_code{}; - op.cont.h = h; - return dispatch_coro(ex, op.cont); + return dispatch_coro(ex, cont); } op.reset(); op.wait_event = event; - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.fd = fd_; @@ -745,7 +743,7 @@ reactor_descriptor::do_wait( op.reset(); op.wait_event = event; - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.fd = fd_; diff --git a/include/boost/corosio/native/detail/reactor/reactor_op.hpp b/include/boost/corosio/native/detail/reactor/reactor_op.hpp index 1f6ab425d..3eecc534f 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_op.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_op.hpp @@ -27,6 +27,7 @@ #include #include #include +#include namespace boost::corosio::detail { @@ -393,6 +394,17 @@ struct reactor_accept_op : Base peer_addrlen = 0; } + /// Destroy without invoking, closing a connection already taken. + void destroy() override + { + if (accepted_fd >= 0) + { + ::close(accepted_fd); + accepted_fd = -1; + } + Base::destroy(); + } + void perform_io() noexcept override { int new_fd = diff --git a/include/boost/corosio/native/detail/reactor/reactor_scheduler.hpp b/include/boost/corosio/native/detail/reactor/reactor_scheduler.hpp index b43da9733..83aec7192 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_scheduler.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_scheduler.hpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -35,7 +36,6 @@ namespace boost::corosio::detail { // Forward declarations class reactor_scheduler; -class timer_service; /** A deregistered descriptor whose owner must outlive the reactor pass that may still hold it. @@ -219,6 +219,13 @@ class reactor_scheduler no handler will be executed. Must be called from a scheduler thread. */ + /** Note that an op was parked outside the usual registration path. + + A reactor that builds its poll set from parked ops must rebuild + it; one the kernel tracks needs nothing. + */ + virtual void notify_reparked() const {} + void compensating_work_started() const noexcept; /** Post completed operations for deferred invocation. @@ -385,10 +392,14 @@ struct reactor_thread_context_guard /// The context frame managed by this guard. reactor_scheduler_context frame_; + /// Marks the scheduler innermost for the executor's fast check. + running_scheduler_guard running_; + /// Construct the guard, pushing a frame for @a sched. explicit reactor_thread_context_guard( reactor_scheduler const* sched) noexcept : frame_(sched, reactor_context_stack.get()) + , running_(sched) { reactor_context_stack.set(&frame_); } @@ -778,6 +789,13 @@ reactor_scheduler::settle_retired(lock_type& lock) const inline void reactor_scheduler::shutdown_drain() { + // This scheduler's constructor registered the timer service, so + // the context shuts it down later; its waiters' frames must be + // destroyed before the drain, which then destroys whatever their + // destructors post. The context's own call later finds no waiter. + if (timer_svc_) + timer_svc_->shutdown(); + lock_type lock(mutex_); settle_retired(lock); diff --git a/include/boost/corosio/native/detail/reactor/reactor_signal_pipe.hpp b/include/boost/corosio/native/detail/reactor/reactor_signal_pipe.hpp index dccfb29af..debce7cd1 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_signal_pipe.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_signal_pipe.hpp @@ -29,29 +29,52 @@ one of these for its lifetime and, in register_signal_reader(), parks the drain op as the descriptor's read_op then calls register_descriptor(). - The drain op never completes: on each read-readiness edge, invoke_deferred_io - calls perform_io(), which drains the pipe and delivers the pending signals, - then reports EAGAIN so the op stays parked and re-fires on the next edge — - exactly the path a socket read op takes when the kernel has no more data. - Because deliver_signal() runs here, in normal dispatch context (not in the - signal handler and not under the reactor poll lock), its mutex locking is - safe. + On each read-readiness edge the drain op completes without touching the + pipe, and invoke_deferred_io runs it once the descriptor lock is + released. Only then does it drain the pipe and deliver the signals: + delivery posts into every context watching the pipe, and a post made + under this descriptor's lock would order it before another context's + scheduler lock, the reverse of that scheduler's own order. The op then + parks itself again, draining once more if an edge arrived meanwhile. */ namespace boost::corosio::detail { struct reactor_signal_pipe_reader { - // Parked read op: drains the self-pipe and re-arms via EAGAIN. struct drain_op final : reactor_op_base { + reactor_signal_pipe_reader* reader = nullptr; + void perform_io() noexcept override { - posix_signal_detail::drain_signal_pipe(); - // Stay parked: EAGAIN tells invoke_deferred_io to keep this op - // installed as read_op and re-run it on the next readiness edge. - errn = EAGAIN; + errn = 0; } + + void operator()() override + { + auto& desc = reader->desc; + for (;;) + { + posix_signal_detail::drain_signal_pipe(); + conditionally_enabled_mutex::scoped_lock lock(desc.mutex); + // An edge seen while unparked set read_ready instead. + if (!desc.read_ready) + { + desc.read_op = this; + break; + } + desc.read_ready = false; + } + // A select() started while this op was out ran without the + // pipe in its read set. + desc.scheduler_->notify_reparked(); + // Nothing counted this op as work; balance the decrement + // that follows the inline run of a completed op. + desc.scheduler_->compensating_work_started(); + } + + void destroy() override {} }; reactor_descriptor_state desc; @@ -61,6 +84,7 @@ struct reactor_signal_pipe_reader // scheduler::register_descriptor(read_fd, ...). reactor_descriptor_state* arm() noexcept { + op.reader = this; desc.read_op = &op; return &desc; } diff --git a/include/boost/corosio/native/detail/reactor/reactor_socket_finals.hpp b/include/boost/corosio/native/detail/reactor/reactor_socket_finals.hpp index 5d37ce30a..f9e926cc2 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_socket_finals.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_socket_finals.hpp @@ -225,7 +225,7 @@ class reactor_acceptor_impl } std::coroutine_handle<> accept( - std::coroutine_handle<>, + capy::continuation&, capy::executor_ref, std::stop_token, std::error_code*, diff --git a/include/boost/corosio/native/detail/reactor/reactor_stream_socket.hpp b/include/boost/corosio/native/detail/reactor/reactor_stream_socket.hpp index 0165550a4..f621ec5c6 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_stream_socket.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_stream_socket.hpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -118,45 +119,45 @@ class reactor_stream_socket // --- Virtual method overrides (satisfy ImplBase pure virtuals) --- std::coroutine_handle<> connect( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, Endpoint ep, std::stop_token token, std::error_code* ec) override { - return do_connect(h, ex, ep, token, ec); + return do_connect(cont, ex, ep, token, ec); } std::coroutine_handle<> read_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param param, std::stop_token token, std::error_code* ec, std::size_t* bytes_out) override { - return do_read_some(h, ex, param, token, ec, bytes_out); + return do_read_some(cont, ex, param, token, ec, bytes_out); } std::coroutine_handle<> write_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param param, std::stop_token token, std::error_code* ec, std::size_t* bytes_out) override { - return do_write_some(h, ex, param, token, ec, bytes_out); + return do_write_some(cont, ex, param, token, ec, bytes_out); } std::coroutine_handle<> wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, wait_type w, std::stop_token token, std::error_code* ec) override { - return do_wait(h, ex, w, token, ec); + return do_wait(cont, ex, w, token, ec); } std::error_code shutdown(corosio::shutdown_type what) noexcept override @@ -217,7 +218,7 @@ class reactor_stream_socket On EINPROGRESS, registers with the reactor. */ std::coroutine_handle<> do_connect( - std::coroutine_handle<>, + capy::continuation&, capy::executor_ref, Endpoint const&, std::stop_token const&, @@ -230,7 +231,7 @@ class reactor_stream_socket On EAGAIN, registers with the reactor. */ std::coroutine_handle<> do_read_some( - std::coroutine_handle<>, + capy::continuation&, capy::executor_ref, buffer_param, std::stop_token const&, @@ -244,7 +245,7 @@ class reactor_stream_socket posts through queue. On EAGAIN, registers with the reactor. */ std::coroutine_handle<> do_write_some( - std::coroutine_handle<>, + capy::continuation&, capy::executor_ref, buffer_param, std::stop_token const&, @@ -261,7 +262,7 @@ class reactor_stream_socket completes only while a non-blocking write can make progress. */ std::coroutine_handle<> do_wait( - std::coroutine_handle<>, + capy::continuation&, capy::executor_ref, wait_type, std::stop_token const&, @@ -387,7 +388,7 @@ reactor_stream_socket< ImplBase, Endpoint>:: do_connect( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, Endpoint const& ep, std::stop_token const& token, @@ -418,11 +419,10 @@ reactor_stream_socket< if (this->svc_.scheduler().try_consume_inline_budget()) { *ec = err ? make_err(err) : std::error_code{}; - op.cont.h = h; - return dispatch_coro(ex, op.cont); + return dispatch_coro(ex, cont); } op.reset(); - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.fd = this->fd_; @@ -436,7 +436,7 @@ reactor_stream_socket< // EINPROGRESS — register with reactor op.reset(); - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.fd = this->fd_; @@ -471,7 +471,7 @@ reactor_stream_socket< ImplBase, Endpoint>:: do_read_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param param, std::stop_token const& token, @@ -485,7 +485,7 @@ reactor_stream_socket< // touching the kernel or the unregistered descriptor state. if (this->fd_ < 0) { - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.bytes_out = bytes_out; @@ -502,7 +502,7 @@ reactor_stream_socket< if (op.iovec_count == 0 || (op.iovec_count == 1 && bufs[0].size() == 0)) { op.empty_buffer_read = true; - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.bytes_out = bytes_out; @@ -553,10 +553,9 @@ reactor_stream_socket< else *ec = {}; *bytes_out = bytes; - op.cont.h = h; - return dispatch_coro(ex, op.cont); + return dispatch_coro(ex, cont); } - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.bytes_out = bytes_out; @@ -568,7 +567,7 @@ reactor_stream_socket< } // EAGAIN — register with reactor - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.bytes_out = bytes_out; @@ -603,7 +602,7 @@ reactor_stream_socket< ImplBase, Endpoint>:: do_write_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param param, std::stop_token const& token, @@ -617,7 +616,7 @@ reactor_stream_socket< // touching the kernel or the unregistered descriptor state. if (this->fd_ < 0) { - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.bytes_out = bytes_out; @@ -634,7 +633,7 @@ reactor_stream_socket< if (op.iovec_count == 0 || (op.iovec_count == 1 && bufs[0].size() == 0)) { - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.bytes_out = bytes_out; @@ -674,10 +673,9 @@ reactor_stream_socket< { *ec = err ? make_err(err) : std::error_code{}; *bytes_out = bytes; - op.cont.h = h; - return dispatch_coro(ex, op.cont); + return dispatch_coro(ex, cont); } - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.bytes_out = bytes_out; @@ -689,7 +687,7 @@ reactor_stream_socket< } // EAGAIN — register with reactor - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.bytes_out = bytes_out; @@ -724,7 +722,7 @@ reactor_stream_socket< ImplBase, Endpoint>:: do_wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, wait_type w, std::stop_token const& token, @@ -766,12 +764,11 @@ reactor_stream_socket< if (this->svc_.scheduler().try_consume_inline_budget()) { *ec = perr ? make_err(perr) : std::error_code{}; - op.cont.h = h; - return dispatch_coro(ex, op.cont); + return dispatch_coro(ex, cont); } op.reset(); op.wait_event = event; - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.fd = this->fd_; @@ -784,7 +781,7 @@ reactor_stream_socket< op.reset(); op.wait_event = event; - op.h = h; + op.cont = &cont; op.ex = ex; op.ec_out = ec; op.fd = this->fd_; diff --git a/include/boost/corosio/native/detail/select/select_scheduler.hpp b/include/boost/corosio/native/detail/select/select_scheduler.hpp index 3b93abb17..b16207fb2 100644 --- a/include/boost/corosio/native/detail/select/select_scheduler.hpp +++ b/include/boost/corosio/native/detail/select/select_scheduler.hpp @@ -143,6 +143,12 @@ class BOOST_COROSIO_DECL select_scheduler final : public reactor_scheduler private: void run_task(lock_type& lock, context_type& ctx, long timeout_us) override; void interrupt_reactor() const override; + + /// Rebuild the poll set a blocked select() was given. + void notify_reparked() const override + { + interrupt_reactor(); + } long calculate_timeout(long requested_timeout_us) const; // Watches the global signal self-pipe's read end (armed lazily by diff --git a/include/boost/corosio/native/detail/uring/uring_acceptor_ops.hpp b/include/boost/corosio/native/detail/uring/uring_acceptor_ops.hpp index 253c758de..0bd88ec12 100644 --- a/include/boost/corosio/native/detail/uring/uring_acceptor_ops.hpp +++ b/include/boost/corosio/native/detail/uring/uring_acceptor_ops.hpp @@ -205,8 +205,7 @@ struct uring_accept_op : uring_op *self->ec_out = was_cancelled ? std::error_code(capy::error::canceled) : make_err(self->err); - self->cont.h = self->h; - auto next = dispatch_coro(self->ex, self->cont); + auto next = dispatch_coro(self->ex, *self->cont); self->dispose(self); next.resume(); return; @@ -226,8 +225,7 @@ struct uring_accept_op : uring_op if (self->ec_out) *self->ec_out = {}; - self->cont.h = self->h; - auto next = dispatch_coro(self->ex, self->cont); + auto next = dispatch_coro(self->ex, *self->cont); self->dispose(self); next.resume(); } diff --git a/include/boost/corosio/native/detail/uring/uring_descriptor.hpp b/include/boost/corosio/native/detail/uring/uring_descriptor.hpp index b6c7addbb..16a144720 100644 --- a/include/boost/corosio/native/detail/uring/uring_descriptor.hpp +++ b/include/boost/corosio/native/detail/uring/uring_descriptor.hpp @@ -280,7 +280,7 @@ class BOOST_COROSIO_DECL uring_descriptor final // -- io_stream::implementation -- std::coroutine_handle<> read_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buffers, std::stop_token token, @@ -288,7 +288,7 @@ class BOOST_COROSIO_DECL uring_descriptor final std::size_t* bytes) override { rd_.prepare( - h, ex, ec, bytes, fd_, /*file_offset=*/-1, sched_, + cont, ex, ec, bytes, fd_, /*file_offset=*/-1, sched_, detail::object_ref(this), buffers, token); arm_slot(rd_); sched_->work_started(); @@ -313,7 +313,7 @@ class BOOST_COROSIO_DECL uring_descriptor final } std::coroutine_handle<> write_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buffers, std::stop_token token, @@ -321,7 +321,7 @@ class BOOST_COROSIO_DECL uring_descriptor final std::size_t* bytes) override { wr_.prepare( - h, ex, ec, bytes, fd_, /*file_offset=*/-1, sched_, + cont, ex, ec, bytes, fd_, /*file_offset=*/-1, sched_, detail::object_ref(this), buffers, token); arm_slot(wr_); sched_->work_started(); @@ -347,7 +347,7 @@ class BOOST_COROSIO_DECL uring_descriptor final // -- posix_stream_descriptor::implementation -- std::coroutine_handle<> wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, wait_type w, std::stop_token token, @@ -374,7 +374,8 @@ class BOOST_COROSIO_DECL uring_descriptor final } op->prepare( - h, ex, ec, fd_, sched_, detail::object_ref(this), poll_flags, token); + cont, ex, ec, fd_, sched_, detail::object_ref(this), poll_flags, + token); sched_->work_started(); if (fd_ < 0) diff --git a/include/boost/corosio/native/detail/uring/uring_dgram_ops.hpp b/include/boost/corosio/native/detail/uring/uring_dgram_ops.hpp index f4cdeaebd..1736a62ba 100644 --- a/include/boost/corosio/native/detail/uring/uring_dgram_ops.hpp +++ b/include/boost/corosio/native/detail/uring/uring_dgram_ops.hpp @@ -63,7 +63,7 @@ struct uring_dgram_send_op : uring_op `dest_addr_storage` with the destination address. */ void prepare( - std::coroutine_handle<> handle, + capy::continuation& awaiting, capy::executor_ref executor, std::error_code* ec, std::size_t* bytes, @@ -77,7 +77,7 @@ struct uring_dgram_send_op : uring_op int flags, std::stop_token const& token) noexcept { - h = handle; + cont = &awaiting; ex = executor; ec_out = ec; bytes_out = bytes; @@ -201,7 +201,7 @@ struct uring_dgram_recv_op : uring_op otherwise block forever. */ void prepare( - std::coroutine_handle<> handle, + capy::continuation& awaiting, capy::executor_ref executor, std::error_code* ec, std::size_t* bytes, @@ -215,7 +215,7 @@ struct uring_dgram_recv_op : uring_op int flags, std::stop_token const& token) noexcept { - h = handle; + cont = &awaiting; ex = executor; ec_out = ec; bytes_out = bytes; diff --git a/include/boost/corosio/native/detail/uring/uring_file_ops.hpp b/include/boost/corosio/native/detail/uring/uring_file_ops.hpp index 159d16192..25fcc944a 100644 --- a/include/boost/corosio/native/detail/uring/uring_file_ops.hpp +++ b/include/boost/corosio/native/detail/uring/uring_file_ops.hpp @@ -64,7 +64,7 @@ struct uring_file_read_op_base : uring_op offset for random-access files. */ void prepare( - std::coroutine_handle<> handle, + capy::continuation& awaiting, capy::executor_ref executor, std::error_code* ec, std::size_t* bytes, @@ -75,7 +75,7 @@ struct uring_file_read_op_base : uring_op buffer_param buffers, std::stop_token const& token) noexcept { - h = handle; + cont = &awaiting; ex = executor; ec_out = ec; bytes_out = bytes; @@ -150,10 +150,6 @@ struct uring_file_read_op : uring_file_read_op_base /// at different offsets on the same fd can be in flight concurrently. struct uring_random_access_read_op : uring_file_read_op_base { - /// The awaitable's continuation, not the embedded `cont`: this op is - /// freed before the coroutine resumes, possibly on another thread. - capy::continuation* awaiting = nullptr; - uring_random_access_read_op() noexcept : uring_file_read_op_base(&do_handler) { @@ -175,7 +171,7 @@ struct uring_random_access_read_op : uring_file_read_op_base } finish(self); - auto* c = self->awaiting; + auto* c = self->cont; auto ex = self->ex; delete self; dispatch_coro(ex, *c).resume(); @@ -212,7 +208,7 @@ struct uring_file_write_op_base : uring_op See uring_file_read_op_base::prepare for the offset convention. */ void prepare( - std::coroutine_handle<> handle, + capy::continuation& awaiting, capy::executor_ref executor, std::error_code* ec, std::size_t* bytes, @@ -223,7 +219,7 @@ struct uring_file_write_op_base : uring_op buffer_param buffers, std::stop_token const& token) noexcept { - h = handle; + cont = &awaiting; ex = executor; ec_out = ec; bytes_out = bytes; @@ -294,10 +290,6 @@ struct uring_file_write_op : uring_file_write_op_base /// Heap-allocated file write op for random_access_file. struct uring_random_access_write_op : uring_file_write_op_base { - /// The awaitable's continuation, not the embedded `cont`: this op is - /// freed before the coroutine resumes, possibly on another thread. - capy::continuation* awaiting = nullptr; - uring_random_access_write_op() noexcept : uring_file_write_op_base(&do_handler) { @@ -319,7 +311,7 @@ struct uring_random_access_write_op : uring_file_write_op_base } finish(self); - auto* c = self->awaiting; + auto* c = self->cont; auto ex = self->ex; delete self; dispatch_coro(ex, *c).resume(); diff --git a/include/boost/corosio/native/detail/uring/uring_multishot_acceptor.hpp b/include/boost/corosio/native/detail/uring/uring_multishot_acceptor.hpp index 46e4c888e..a1cc1d004 100644 --- a/include/boost/corosio/native/detail/uring/uring_multishot_acceptor.hpp +++ b/include/boost/corosio/native/detail/uring/uring_multishot_acceptor.hpp @@ -442,13 +442,13 @@ class uring_multishot_acceptor_base readiness is the impl's ready queue plus future deliveries. */ void park_read_wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, std::stop_token const& token, std::error_code* ec) noexcept { auto* w = acquire_node(); - w->h = h; + w->cont = &cont; w->ex = ex; w->ec_out = ec; w->peek = true; @@ -678,14 +678,14 @@ class uring_multishot_acceptor_base /// Either case ends with the calling coroutine suspending; the /// caller returns `std::noop_coroutine()` unconditionally. void dispatch_or_queue( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, std::stop_token const& token, std::error_code* ec, io_object::implementation** impl_out) { auto* w = acquire_node(); - w->h = h; + w->cont = &cont; w->ex = ex; w->ec_out = ec; w->impl_out = impl_out; diff --git a/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp b/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp index 4e72e3ae2..12f70a95d 100644 --- a/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp +++ b/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp @@ -110,7 +110,7 @@ class BOOST_COROSIO_DECL uring_random_access_file final return; } Base::finish(self); - auto* c = self->awaiting; + auto* c = self->cont; auto ex = self->ex; self->file_->release_op(self); dispatch_coro(ex, *c).resume(); @@ -349,9 +349,8 @@ uring_random_access_file::read_some_at( { auto* op = acquire_op(free_reads_); op->prepare( - cont.h, ex, ec, bytes, fd_, static_cast(user_offset), + cont, ex, ec, bytes, fd_, static_cast(user_offset), sched_, detail::object_ref(this), buffers, token); - op->awaiting = &cont; sched_->work_started(); // Closed-object contract outranks the zero-length no-op. @@ -388,9 +387,8 @@ uring_random_access_file::write_some_at( { auto* op = acquire_op(free_writes_); op->prepare( - cont.h, ex, ec, bytes, fd_, static_cast(user_offset), + cont, ex, ec, bytes, fd_, static_cast(user_offset), sched_, detail::object_ref(this), buffers, token); - op->awaiting = &cont; sched_->work_started(); // Closed-object contract outranks the zero-length no-op. diff --git a/include/boost/corosio/native/detail/uring/uring_scheduler.hpp b/include/boost/corosio/native/detail/uring/uring_scheduler.hpp index cb1c22315..ab79328c6 100644 --- a/include/boost/corosio/native/detail/uring/uring_scheduler.hpp +++ b/include/boost/corosio/native/detail/uring/uring_scheduler.hpp @@ -627,6 +627,10 @@ uring_scheduler::shutdown() { stopped_.store(true, std::memory_order_release); + // See reactor_scheduler::shutdown_drain(). + if (timer_svc_) + timer_svc_->shutdown(); + // Cancel every request still in the kernel and wait for each to // come back: the services free their impls next, and an op still // armed would write into freed memory. One cancel-any covers ops @@ -931,11 +935,13 @@ inline constexpr int uring_inline_budget_max = 16; struct uring_run_guard { uring_scheduler_frame frame_; + running_scheduler_guard running_; explicit uring_run_guard(uring_scheduler const* self) noexcept : frame_{ self, tl_running_scheduler_frame_, uring_inline_budget_initial, uring_inline_budget_max} + , running_(self) { tl_running_scheduler_frame_ = &frame_; } diff --git a/include/boost/corosio/native/detail/uring/uring_socket_ops.hpp b/include/boost/corosio/native/detail/uring/uring_socket_ops.hpp index 01c0a770b..42916b42c 100644 --- a/include/boost/corosio/native/detail/uring/uring_socket_ops.hpp +++ b/include/boost/corosio/native/detail/uring/uring_socket_ops.hpp @@ -123,7 +123,7 @@ struct uring_read_op : uring_op @pre This slot has no in-flight op (its prior op completed). */ void prepare( - std::coroutine_handle<> handle, + capy::continuation& awaiting, capy::executor_ref executor, std::error_code* ec, std::size_t* bytes, @@ -134,7 +134,7 @@ struct uring_read_op : uring_op buffer_param buffers, std::stop_token const& token) noexcept { - h = handle; + cont = &awaiting; ex = executor; ec_out = ec; bytes_out = bytes; @@ -221,7 +221,7 @@ struct uring_write_op : uring_op /** Reset and initialize for a new submission. See uring_read_op::prepare. */ void prepare( - std::coroutine_handle<> handle, + capy::continuation& awaiting, capy::executor_ref executor, std::error_code* ec, std::size_t* bytes, @@ -232,7 +232,7 @@ struct uring_write_op : uring_op buffer_param buffers, std::stop_token const& token) noexcept { - h = handle; + cont = &awaiting; ex = executor; ec_out = ec; bytes_out = bytes; @@ -331,7 +331,7 @@ struct uring_connect_op : uring_op caller, not the op. */ void prepare( - std::coroutine_handle<> handle, + capy::continuation& awaiting, capy::executor_ref executor, std::error_code* ec, int file_descriptor, @@ -342,7 +342,7 @@ struct uring_connect_op : uring_op endpoint* local_out, std::stop_token const& token) noexcept { - h = handle; + cont = &awaiting; ex = executor; ec_out = ec; bytes_out = nullptr; @@ -590,7 +590,7 @@ struct uring_wait_op : uring_op /** Reset and initialize for a new submission. */ void prepare( - std::coroutine_handle<> handle, + capy::continuation& awaiting, capy::executor_ref executor, std::error_code* ec, int file_descriptor, @@ -599,7 +599,7 @@ struct uring_wait_op : uring_op int flags, std::stop_token const& token) noexcept { - h = handle; + cont = &awaiting; ex = executor; ec_out = ec; bytes_out = nullptr; @@ -715,7 +715,7 @@ struct uring_local_connect_op : uring_op Caller pre-fills `addr` and `addrlen` (see uring_connect_op::prepare). */ void prepare( - std::coroutine_handle<> handle, + capy::continuation& awaiting, capy::executor_ref executor, std::error_code* ec, int file_descriptor, @@ -726,7 +726,7 @@ struct uring_local_connect_op : uring_op corosio::local_endpoint* local_out, std::stop_token const& token) noexcept { - h = handle; + cont = &awaiting; ex = executor; ec_out = ec; bytes_out = nullptr; diff --git a/include/boost/corosio/native/detail/uring/uring_stream_file.hpp b/include/boost/corosio/native/detail/uring/uring_stream_file.hpp index c0cf1bbd1..52794a6f6 100644 --- a/include/boost/corosio/native/detail/uring/uring_stream_file.hpp +++ b/include/boost/corosio/native/detail/uring/uring_stream_file.hpp @@ -115,7 +115,7 @@ class BOOST_COROSIO_DECL uring_stream_file final // -- io_stream::implementation -- std::coroutine_handle<> read_some( - std::coroutine_handle<>, + capy::continuation&, capy::executor_ref, buffer_param, std::stop_token, @@ -123,7 +123,7 @@ class BOOST_COROSIO_DECL uring_stream_file final std::size_t*) override; std::coroutine_handle<> write_some( - std::coroutine_handle<>, + capy::continuation&, capy::executor_ref, buffer_param, std::stop_token, @@ -279,7 +279,7 @@ class BOOST_COROSIO_DECL uring_stream_file final inline std::coroutine_handle<> uring_stream_file::read_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buffers, std::stop_token token, @@ -287,8 +287,8 @@ uring_stream_file::read_some( std::size_t* bytes) { rd_.prepare( - h, ex, ec, bytes, fd_, /*file_offset=*/-1, sched_, detail::object_ref(this), - buffers, token); + cont, ex, ec, bytes, fd_, /*file_offset=*/-1, sched_, + detail::object_ref(this), buffers, token); sched_->work_started(); // Closed-object contract outranks the zero-length no-op. @@ -314,7 +314,7 @@ uring_stream_file::read_some( inline std::coroutine_handle<> uring_stream_file::write_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buffers, std::stop_token token, @@ -322,8 +322,8 @@ uring_stream_file::write_some( std::size_t* bytes) { wr_.prepare( - h, ex, ec, bytes, fd_, /*file_offset=*/-1, sched_, detail::object_ref(this), - buffers, token); + cont, ex, ec, bytes, fd_, /*file_offset=*/-1, sched_, + detail::object_ref(this), buffers, token); sched_->work_started(); // Closed-object contract outranks the zero-length no-op. diff --git a/include/boost/corosio/native/detail/uring/uring_types.hpp b/include/boost/corosio/native/detail/uring/uring_types.hpp index a70513a02..6d50a062f 100644 --- a/include/boost/corosio/native/detail/uring/uring_types.hpp +++ b/include/boost/corosio/native/detail/uring/uring_types.hpp @@ -156,7 +156,7 @@ class BOOST_COROSIO_DECL uring_tcp_socket final // ---------------------------------------------------------------- std::coroutine_handle<> read_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buffers, std::stop_token token, @@ -205,12 +205,11 @@ class BOOST_COROSIO_DECL uring_tcp_socket final err ? make_err(err) : std::error_code{}, /*is_read=*/true, n < 0 ? 0u : static_cast(n), empty_buf); - rd_.cont.h = h; - return dispatch_coro(ex, rd_.cont); + return dispatch_coro(ex, cont); } rd_.prepare( - h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, - buffers, token); + cont, ex, ec, bytes, fd_, sched_, detail::object_ref(this), + &spec_, buffers, token); if (stop_now) rd_.cancelled.store(true, std::memory_order_release); else @@ -224,8 +223,8 @@ class BOOST_COROSIO_DECL uring_tcp_socket final } rd_.prepare( - h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, - token); + cont, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, + buffers, token); sched_->work_started(); if (rd_.cancelled.load(std::memory_order_acquire)) { @@ -238,7 +237,7 @@ class BOOST_COROSIO_DECL uring_tcp_socket final } std::coroutine_handle<> write_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buffers, std::stop_token token, @@ -284,12 +283,11 @@ class BOOST_COROSIO_DECL uring_tcp_socket final err ? make_err(err) : std::error_code{}, /*is_read=*/false, n < 0 ? 0u : static_cast(n), /*empty_buffer=*/false); - wr_.cont.h = h; - return dispatch_coro(ex, wr_.cont); + return dispatch_coro(ex, cont); } wr_.prepare( - h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, - buffers, token); + cont, ex, ec, bytes, fd_, sched_, detail::object_ref(this), + &spec_, buffers, token); if (stop_now) wr_.cancelled.store(true, std::memory_order_release); else @@ -303,8 +301,8 @@ class BOOST_COROSIO_DECL uring_tcp_socket final } wr_.prepare( - h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, - token); + cont, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, + buffers, token); sched_->work_started(); if (wr_.cancelled.load(std::memory_order_acquire)) { @@ -321,7 +319,7 @@ class BOOST_COROSIO_DECL uring_tcp_socket final // ---------------------------------------------------------------- std::coroutine_handle<> connect( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, endpoint ep, std::stop_token token, @@ -334,12 +332,11 @@ class BOOST_COROSIO_DECL uring_tcp_socket final { if (ec) *ec = capy::error::canceled; - conn_.cont.h = h; - return dispatch_coro(ex, conn_.cont); + return dispatch_coro(ex, cont); } conn_.addrlen = to_sockaddr(ep, family_, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, detail::object_ref(this), ep, + cont, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, &local_endpoint_, token); conn_.cancelled.store(true, std::memory_order_release); sched_->work_started(); @@ -354,8 +351,8 @@ class BOOST_COROSIO_DECL uring_tcp_socket final // a subsequent IORING_OP_CONNECT would see EALREADY — avoid. conn_.addrlen = to_sockaddr(ep, family_, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, - &local_endpoint_, token); + cont, ex, ec, fd_, sched_, detail::object_ref(this), ep, + &remote_endpoint_, &local_endpoint_, token); sched_->work_started(); if (conn_.cancelled.load(std::memory_order_acquire)) { @@ -368,7 +365,7 @@ class BOOST_COROSIO_DECL uring_tcp_socket final } std::coroutine_handle<> wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, wait_type w, std::stop_token token, @@ -388,7 +385,8 @@ class BOOST_COROSIO_DECL uring_tcp_socket final break; } wait_op_.prepare( - h, ex, ec, fd_, sched_, detail::object_ref(this), poll_flags, token); + cont, ex, ec, fd_, sched_, detail::object_ref(this), poll_flags, + token); sched_->work_started(); if (wait_op_.cancelled.load(std::memory_order_acquire)) { @@ -765,18 +763,18 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor final } std::coroutine_handle<> accept( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, std::stop_token token, std::error_code* ec, io_object::implementation** impl_out) override { - base_type::dispatch_or_queue(h, ex, token, ec, impl_out); + base_type::dispatch_or_queue(cont, ex, token, ec, impl_out); return std::noop_coroutine(); } std::coroutine_handle<> wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, wait_type w, std::stop_token token, @@ -787,7 +785,7 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor final if (this->fd_ < 0) { auto* op = this->acquire_node(); - op->h = h; + op->cont = &cont; op->ex = ex; op->ec_out = ec; op->err = EBADF; @@ -799,7 +797,7 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor final // readable; read waits complete from the delivery queue. if (w == wait_type::read) { - this->park_read_wait(h, ex, token, ec); + this->park_read_wait(cont, ex, token, ec); return std::noop_coroutine(); } // Writability carries no meaning for a listening socket; @@ -807,7 +805,7 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor final if (w == wait_type::write) { auto* op = this->acquire_node(); - op->h = h; + op->cont = &cont; op->ex = ex; op->ec_out = ec; op->err = ENOTSUP; @@ -817,7 +815,7 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor final // Errors are not consumed by the accept machinery, so the // error wait still polls the descriptor. wait_op_.prepare( - h, ex, ec, this->fd_, this->sched_, detail::object_ref(this), + cont, ex, ec, this->fd_, this->sched_, detail::object_ref(this), POLLPRI | POLLERR | POLLHUP, token); this->sched_->work_started(); if (wait_op_.cancelled.load(std::memory_order_acquire)) @@ -1124,7 +1122,7 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final // ---------------------------------------------------------------- std::coroutine_handle<> read_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buffers, std::stop_token token, @@ -1173,12 +1171,11 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final err ? make_err(err) : std::error_code{}, /*is_read=*/true, n < 0 ? 0u : static_cast(n), empty_buf); - rd_.cont.h = h; - return dispatch_coro(ex, rd_.cont); + return dispatch_coro(ex, cont); } rd_.prepare( - h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, - buffers, token); + cont, ex, ec, bytes, fd_, sched_, detail::object_ref(this), + &spec_, buffers, token); if (stop_now) rd_.cancelled.store(true, std::memory_order_release); else @@ -1192,8 +1189,8 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final } rd_.prepare( - h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, - token); + cont, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, + buffers, token); sched_->work_started(); if (rd_.cancelled.load(std::memory_order_acquire)) { @@ -1206,7 +1203,7 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final } std::coroutine_handle<> write_some( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buffers, std::stop_token token, @@ -1252,12 +1249,11 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final err ? make_err(err) : std::error_code{}, /*is_read=*/false, n < 0 ? 0u : static_cast(n), /*empty_buffer=*/false); - wr_.cont.h = h; - return dispatch_coro(ex, wr_.cont); + return dispatch_coro(ex, cont); } wr_.prepare( - h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, - buffers, token); + cont, ex, ec, bytes, fd_, sched_, detail::object_ref(this), + &spec_, buffers, token); if (stop_now) wr_.cancelled.store(true, std::memory_order_release); else @@ -1271,8 +1267,8 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final } wr_.prepare( - h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, - token); + cont, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, + buffers, token); sched_->work_started(); if (wr_.cancelled.load(std::memory_order_acquire)) { @@ -1289,7 +1285,7 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final // ---------------------------------------------------------------- std::coroutine_handle<> connect( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, corosio::local_endpoint ep, std::stop_token token, @@ -1302,12 +1298,11 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final { if (ec) *ec = capy::error::canceled; - conn_.cont.h = h; - return dispatch_coro(ex, conn_.cont); + return dispatch_coro(ex, cont); } conn_.addrlen = to_sockaddr(ep, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, detail::object_ref(this), ep, + cont, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, &local_endpoint_, token); conn_.cancelled.store(true, std::memory_order_release); sched_->work_started(); @@ -1322,8 +1317,8 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final // a subsequent IORING_OP_CONNECT would see EALREADY — avoid. conn_.addrlen = to_sockaddr(ep, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, - &local_endpoint_, token); + cont, ex, ec, fd_, sched_, detail::object_ref(this), ep, + &remote_endpoint_, &local_endpoint_, token); sched_->work_started(); if (conn_.cancelled.load(std::memory_order_acquire)) { @@ -1336,7 +1331,7 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final } std::coroutine_handle<> wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, wait_type w, std::stop_token token, @@ -1356,7 +1351,8 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final break; } wait_op_.prepare( - h, ex, ec, fd_, sched_, detail::object_ref(this), poll_flags, token); + cont, ex, ec, fd_, sched_, detail::object_ref(this), poll_flags, + token); sched_->work_started(); if (wait_op_.cancelled.load(std::memory_order_acquire)) { @@ -1640,18 +1636,18 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor final } std::coroutine_handle<> accept( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, std::stop_token token, std::error_code* ec, io_object::implementation** impl_out) override { - base_type::dispatch_or_queue(h, ex, token, ec, impl_out); + base_type::dispatch_or_queue(cont, ex, token, ec, impl_out); return std::noop_coroutine(); } std::coroutine_handle<> wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, wait_type w, std::stop_token token, @@ -1662,7 +1658,7 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor final if (this->fd_ < 0) { auto* op = this->acquire_node(); - op->h = h; + op->cont = &cont; op->ex = ex; op->ec_out = ec; op->err = EBADF; @@ -1674,7 +1670,7 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor final // readable; read waits complete from the delivery queue. if (w == wait_type::read) { - this->park_read_wait(h, ex, token, ec); + this->park_read_wait(cont, ex, token, ec); return std::noop_coroutine(); } // Writability carries no meaning for a listening socket; @@ -1682,7 +1678,7 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor final if (w == wait_type::write) { auto* op = this->acquire_node(); - op->h = h; + op->cont = &cont; op->ex = ex; op->ec_out = ec; op->err = ENOTSUP; @@ -1692,7 +1688,7 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor final // Errors are not consumed by the accept machinery, so the // error wait still polls the descriptor. wait_op_.prepare( - h, ex, ec, this->fd_, this->sched_, detail::object_ref(this), + cont, ex, ec, this->fd_, this->sched_, detail::object_ref(this), POLLPRI | POLLERR | POLLHUP, token); this->sched_->work_started(); if (wait_op_.cancelled.load(std::memory_order_acquire)) @@ -1991,7 +1987,7 @@ class BOOST_COROSIO_DECL uring_udp_socket final // ---------------------------------------------------------------- std::coroutine_handle<> send_to( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buf, endpoint dest, @@ -2002,11 +1998,12 @@ class BOOST_COROSIO_DECL uring_udp_socket final { sockaddr_storage addr{}; socklen_t len = endpoint_to_sockaddr(dest, addr); - return submit_send(h, ex, buf, len, addr, flags, token, ec, bytes_out); + return submit_send( + cont, ex, buf, len, addr, flags, token, ec, bytes_out); } std::coroutine_handle<> recv_from( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buf, endpoint* source, @@ -2016,11 +2013,12 @@ class BOOST_COROSIO_DECL uring_udp_socket final std::size_t* bytes_out) override { return submit_recv( - h, ex, buf, source != nullptr, source, flags, token, ec, bytes_out); + cont, ex, buf, source != nullptr, source, flags, token, ec, + bytes_out); } std::coroutine_handle<> send( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buf, int flags, @@ -2029,11 +2027,12 @@ class BOOST_COROSIO_DECL uring_udp_socket final std::size_t* bytes_out) override { sockaddr_storage empty{}; - return submit_send(h, ex, buf, 0, empty, flags, token, ec, bytes_out); + return submit_send( + cont, ex, buf, 0, empty, flags, token, ec, bytes_out); } std::coroutine_handle<> recv( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buf, int flags, @@ -2042,11 +2041,11 @@ class BOOST_COROSIO_DECL uring_udp_socket final std::size_t* bytes_out) override { return submit_recv( - h, ex, buf, false, nullptr, flags, token, ec, bytes_out); + cont, ex, buf, false, nullptr, flags, token, ec, bytes_out); } std::coroutine_handle<> connect( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, endpoint ep, std::stop_token token, @@ -2059,12 +2058,11 @@ class BOOST_COROSIO_DECL uring_udp_socket final { if (ec) *ec = capy::error::canceled; - conn_.cont.h = h; - return dispatch_coro(ex, conn_.cont); + return dispatch_coro(ex, cont); } conn_.addrlen = to_sockaddr(ep, family_, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, detail::object_ref(this), ep, + cont, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, &local_endpoint_, token); conn_.cancelled.store(true, std::memory_order_release); sched_->work_started(); @@ -2079,8 +2077,8 @@ class BOOST_COROSIO_DECL uring_udp_socket final // a prior speculative ::connect would leave EINPROGRESS → EALREADY. conn_.addrlen = to_sockaddr(ep, family_, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, - &local_endpoint_, token); + cont, ex, ec, fd_, sched_, detail::object_ref(this), ep, + &remote_endpoint_, &local_endpoint_, token); sched_->work_started(); if (conn_.cancelled.load(std::memory_order_acquire)) { @@ -2093,7 +2091,7 @@ class BOOST_COROSIO_DECL uring_udp_socket final } std::coroutine_handle<> wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, wait_type w, std::stop_token token, @@ -2113,7 +2111,8 @@ class BOOST_COROSIO_DECL uring_udp_socket final break; } wait_op_.prepare( - h, ex, ec, fd_, sched_, detail::object_ref(this), poll_flags, token); + cont, ex, ec, fd_, sched_, detail::object_ref(this), poll_flags, + token); sched_->work_started(); if (wait_op_.cancelled.load(std::memory_order_acquire)) { @@ -2208,7 +2207,7 @@ class BOOST_COROSIO_DECL uring_udp_socket final private: std::coroutine_handle<> submit_send( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buffers, socklen_t dest_len, @@ -2264,13 +2263,12 @@ class BOOST_COROSIO_DECL uring_udp_socket final err ? make_err(err) : std::error_code{}, /*is_read=*/false, n < 0 ? 0u : static_cast(n), /*empty_buffer=*/false); - send_.cont.h = h; - return dispatch_coro(ex, send_.cont); + return dispatch_coro(ex, cont); } send_.prepare( - h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, - buffers, dest_len, dest_storage, to_native_msg_flags(flags), - token); + cont, ex, ec, bytes, fd_, sched_, detail::object_ref(this), + &spec_, buffers, dest_len, dest_storage, + to_native_msg_flags(flags), token); if (stop_now) send_.cancelled.store(true, std::memory_order_release); else @@ -2284,8 +2282,8 @@ class BOOST_COROSIO_DECL uring_udp_socket final } send_.prepare( - h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, - dest_len, dest_storage, to_native_msg_flags(flags), token); + cont, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, + buffers, dest_len, dest_storage, to_native_msg_flags(flags), token); sched_->work_started(); if (send_.cancelled.load(std::memory_order_acquire)) { @@ -2298,7 +2296,7 @@ class BOOST_COROSIO_DECL uring_udp_socket final } std::coroutine_handle<> submit_recv( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buffers, bool want_source, @@ -2358,12 +2356,12 @@ class BOOST_COROSIO_DECL uring_udp_socket final /*empty_buffer=*/false); if (n >= 0 && want_source && source_out && !empty_buf) *source_out = sockaddr_to_endpoint(src_storage); - recv_.cont.h = h; - return dispatch_coro(ex, recv_.cont); + return dispatch_coro(ex, cont); } recv_.prepare( - h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, - buffers, source_out, want_source ? &write_ip_source : nullptr, + cont, ex, ec, bytes, fd_, sched_, detail::object_ref(this), + &spec_, buffers, source_out, + want_source ? &write_ip_source : nullptr, to_native_msg_flags(flags), token); if (stop_now) recv_.cancelled.store(true, std::memory_order_release); @@ -2388,8 +2386,8 @@ class BOOST_COROSIO_DECL uring_udp_socket final } recv_.prepare( - h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, - source_out, want_source ? &write_ip_source : nullptr, + cont, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, + buffers, source_out, want_source ? &write_ip_source : nullptr, to_native_msg_flags(flags), token); sched_->work_started(); if (recv_.iovec_count == 0 || @@ -2632,7 +2630,7 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final // ---------------------------------------------------------------- std::coroutine_handle<> send_to( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buf, corosio::local_endpoint dest, @@ -2643,11 +2641,12 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final { sockaddr_storage addr{}; socklen_t len = endpoint_to_sockaddr(dest, addr); - return submit_send(h, ex, buf, len, addr, flags, token, ec, bytes_out); + return submit_send( + cont, ex, buf, len, addr, flags, token, ec, bytes_out); } std::coroutine_handle<> recv_from( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buf, corosio::local_endpoint* source, @@ -2657,11 +2656,12 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final std::size_t* bytes_out) override { return submit_recv( - h, ex, buf, source != nullptr, source, flags, token, ec, bytes_out); + cont, ex, buf, source != nullptr, source, flags, token, ec, + bytes_out); } std::coroutine_handle<> send( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buf, int flags, @@ -2670,11 +2670,12 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final std::size_t* bytes_out) override { sockaddr_storage empty{}; - return submit_send(h, ex, buf, 0, empty, flags, token, ec, bytes_out); + return submit_send( + cont, ex, buf, 0, empty, flags, token, ec, bytes_out); } std::coroutine_handle<> recv( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buf, int flags, @@ -2683,11 +2684,11 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final std::size_t* bytes_out) override { return submit_recv( - h, ex, buf, false, nullptr, flags, token, ec, bytes_out); + cont, ex, buf, false, nullptr, flags, token, ec, bytes_out); } std::coroutine_handle<> connect( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, corosio::local_endpoint ep, std::stop_token token, @@ -2700,12 +2701,11 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final { if (ec) *ec = capy::error::canceled; - conn_.cont.h = h; - return dispatch_coro(ex, conn_.cont); + return dispatch_coro(ex, cont); } conn_.addrlen = to_sockaddr(ep, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, detail::object_ref(this), ep, + cont, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, &local_endpoint_, token); conn_.cancelled.store(true, std::memory_order_release); sched_->work_started(); @@ -2720,8 +2720,8 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final // a prior speculative ::connect would leave EINPROGRESS → EALREADY. conn_.addrlen = to_sockaddr(ep, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, - &local_endpoint_, token); + cont, ex, ec, fd_, sched_, detail::object_ref(this), ep, + &remote_endpoint_, &local_endpoint_, token); sched_->work_started(); if (conn_.cancelled.load(std::memory_order_acquire)) { @@ -2734,7 +2734,7 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final } std::coroutine_handle<> wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, wait_type w, std::stop_token token, @@ -2754,7 +2754,8 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final break; } wait_op_.prepare( - h, ex, ec, fd_, sched_, detail::object_ref(this), poll_flags, token); + cont, ex, ec, fd_, sched_, detail::object_ref(this), poll_flags, + token); sched_->work_started(); if (wait_op_.cancelled.load(std::memory_order_acquire)) { @@ -2863,7 +2864,7 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final private: std::coroutine_handle<> submit_send( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buffers, socklen_t dest_len, @@ -2919,13 +2920,12 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final err ? make_err(err) : std::error_code{}, /*is_read=*/false, n < 0 ? 0u : static_cast(n), /*empty_buffer=*/false); - send_.cont.h = h; - return dispatch_coro(ex, send_.cont); + return dispatch_coro(ex, cont); } send_.prepare( - h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, - buffers, dest_len, dest_storage, to_native_msg_flags(flags), - token); + cont, ex, ec, bytes, fd_, sched_, detail::object_ref(this), + &spec_, buffers, dest_len, dest_storage, + to_native_msg_flags(flags), token); if (stop_now) send_.cancelled.store(true, std::memory_order_release); else @@ -2939,8 +2939,8 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final } send_.prepare( - h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, - dest_len, dest_storage, to_native_msg_flags(flags), token); + cont, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, + buffers, dest_len, dest_storage, to_native_msg_flags(flags), token); sched_->work_started(); if (send_.cancelled.load(std::memory_order_acquire)) { @@ -2953,7 +2953,7 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final } std::coroutine_handle<> submit_recv( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buffers, bool want_source, @@ -3014,12 +3014,11 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final if (n >= 0 && want_source && source_out && !empty_buf) *source_out = sockaddr_to_local_endpoint(src_storage, src_namelen); - recv_.cont.h = h; - return dispatch_coro(ex, recv_.cont); + return dispatch_coro(ex, cont); } recv_.prepare( - h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, - buffers, source_out, + cont, ex, ec, bytes, fd_, sched_, detail::object_ref(this), + &spec_, buffers, source_out, want_source ? &write_local_source : nullptr, to_native_msg_flags(flags), token); if (stop_now) @@ -3045,8 +3044,8 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final } recv_.prepare( - h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, - source_out, want_source ? &write_local_source : nullptr, + cont, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, + buffers, source_out, want_source ? &write_local_source : nullptr, to_native_msg_flags(flags), token); sched_->work_started(); if (recv_.iovec_count == 0 || diff --git a/include/boost/corosio/native/native_local_datagram_socket.hpp b/include/boost/corosio/native/native_local_datagram_socket.hpp index 319a0413c..a62d456d8 100644 --- a/include/boost/corosio/native/native_local_datagram_socket.hpp +++ b/include/boost/corosio/native/native_local_datagram_socket.hpp @@ -101,10 +101,10 @@ class native_local_datagram_socket : public local_datagram_socket } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return self_.get_impl().send_to( - h, ex, buffers_, dest_, flags_, this->token_, &this->ec_, + cont, ex, buffers_, dest_, flags_, this->token_, &this->ec_, &this->bytes_); } }; @@ -132,10 +132,10 @@ class native_local_datagram_socket : public local_datagram_socket } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return self_.get_impl().recv_from( - h, ex, buffers_, &source_, flags_, this->token_, &this->ec_, + cont, ex, buffers_, &source_, flags_, this->token_, &this->ec_, &this->bytes_); } }; @@ -153,9 +153,10 @@ class native_local_datagram_socket : public local_datagram_socket } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - return self_.get_impl().wait(h, ex, w_, this->token_, &this->ec_); + return self_.get_impl().wait( + cont, ex, w_, this->token_, &this->ec_); } }; @@ -174,10 +175,10 @@ class native_local_datagram_socket : public local_datagram_socket } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return self_.get_impl().connect( - h, ex, endpoint_, this->token_, &this->ec_); + cont, ex, endpoint_, this->token_, &this->ec_); } }; @@ -200,10 +201,10 @@ class native_local_datagram_socket : public local_datagram_socket } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return self_.get_impl().send( - h, ex, buffers_, flags_, this->token_, &this->ec_, + cont, ex, buffers_, flags_, this->token_, &this->ec_, &this->bytes_); } }; @@ -227,10 +228,10 @@ class native_local_datagram_socket : public local_datagram_socket } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return self_.get_impl().recv( - h, ex, buffers_, flags_, this->token_, &this->ec_, + cont, ex, buffers_, flags_, this->token_, &this->ec_, &this->bytes_); } }; diff --git a/include/boost/corosio/native/native_local_stream_acceptor.hpp b/include/boost/corosio/native/native_local_stream_acceptor.hpp index 7e5402f5e..f6213ba2e 100644 --- a/include/boost/corosio/native/native_local_stream_acceptor.hpp +++ b/include/boost/corosio/native/native_local_stream_acceptor.hpp @@ -89,9 +89,9 @@ class native_local_stream_acceptor : public local_stream_acceptor } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - return acc_.get_impl().wait(h, ex, w_, this->token_, &this->ec_); + return acc_.get_impl().wait(cont, ex, w_, this->token_, &this->ec_); } }; @@ -118,10 +118,10 @@ class native_local_stream_acceptor : public local_stream_acceptor } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return acc_.get_impl().accept( - h, ex, this->token_, &this->ec_, &peer_impl_); + cont, ex, this->token_, &this->ec_, &peer_impl_); } }; @@ -151,10 +151,10 @@ class native_local_stream_acceptor : public local_stream_acceptor } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return acc_.get_impl().accept( - h, ex, this->token_, &this->ec_, &peer_impl_); + cont, ex, this->token_, &this->ec_, &peer_impl_); } }; diff --git a/include/boost/corosio/native/native_local_stream_socket.hpp b/include/boost/corosio/native/native_local_stream_socket.hpp index 6fef772f2..4c4ffa4a6 100644 --- a/include/boost/corosio/native/native_local_stream_socket.hpp +++ b/include/boost/corosio/native/native_local_stream_socket.hpp @@ -93,10 +93,10 @@ class native_local_stream_socket : public local_stream_socket } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return self_.get_impl().read_some( - h, ex, buffers_, this->token_, &this->ec_, &this->bytes_); + cont, ex, buffers_, this->token_, &this->ec_, &this->bytes_); } }; @@ -116,10 +116,10 @@ class native_local_stream_socket : public local_stream_socket } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return self_.get_impl().write_some( - h, ex, buffers_, this->token_, &this->ec_, &this->bytes_); + cont, ex, buffers_, this->token_, &this->ec_, &this->bytes_); } }; @@ -136,9 +136,10 @@ class native_local_stream_socket : public local_stream_socket } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - return self_.get_impl().wait(h, ex, w_, this->token_, &this->ec_); + return self_.get_impl().wait( + cont, ex, w_, this->token_, &this->ec_); } }; @@ -157,10 +158,10 @@ class native_local_stream_socket : public local_stream_socket } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return self_.get_impl().connect( - h, ex, endpoint_, this->token_, &this->ec_); + cont, ex, endpoint_, this->token_, &this->ec_); } }; diff --git a/include/boost/corosio/native/native_posix_stream_descriptor.hpp b/include/boost/corosio/native/native_posix_stream_descriptor.hpp index 8d76fcabe..32cabff8a 100644 --- a/include/boost/corosio/native/native_posix_stream_descriptor.hpp +++ b/include/boost/corosio/native/native_posix_stream_descriptor.hpp @@ -92,10 +92,10 @@ class native_posix_stream_descriptor : public posix_stream_descriptor } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return self_.get_impl().read_some( - h, ex, buffers_, this->token_, &this->ec_, &this->bytes_); + cont, ex, buffers_, this->token_, &this->ec_, &this->bytes_); } }; @@ -115,10 +115,10 @@ class native_posix_stream_descriptor : public posix_stream_descriptor } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return self_.get_impl().write_some( - h, ex, buffers_, this->token_, &this->ec_, &this->bytes_); + cont, ex, buffers_, this->token_, &this->ec_, &this->bytes_); } }; @@ -135,9 +135,10 @@ class native_posix_stream_descriptor : public posix_stream_descriptor } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - return self_.get_impl().wait(h, ex, w_, this->token_, &this->ec_); + return self_.get_impl().wait( + cont, ex, w_, this->token_, &this->ec_); } }; diff --git a/include/boost/corosio/native/native_random_access_file.hpp b/include/boost/corosio/native/native_random_access_file.hpp index 24256824e..8ab17d2df 100644 --- a/include/boost/corosio/native/native_random_access_file.hpp +++ b/include/boost/corosio/native/native_random_access_file.hpp @@ -84,7 +84,6 @@ class native_random_access_file : public random_access_file native_random_access_file& self_; std::uint64_t offset_; MutableBufferSequence buffers_; - mutable capy::continuation cont_; native_read_at_awaitable( native_random_access_file& self, @@ -97,11 +96,10 @@ class native_random_access_file : public random_access_file } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - cont_.h = h; return self_.get_impl().read_some_at( - offset_, cont_, ex, buffers_, this->token_, &this->ec_, + offset_, cont, ex, buffers_, this->token_, &this->ec_, &this->bytes_); } }; @@ -113,7 +111,6 @@ class native_random_access_file : public random_access_file native_random_access_file& self_; std::uint64_t offset_; ConstBufferSequence buffers_; - mutable capy::continuation cont_; native_write_at_awaitable( native_random_access_file& self, @@ -126,11 +123,10 @@ class native_random_access_file : public random_access_file } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - cont_.h = h; return self_.get_impl().write_some_at( - offset_, cont_, ex, buffers_, this->token_, &this->ec_, + offset_, cont, ex, buffers_, this->token_, &this->ec_, &this->bytes_); } }; diff --git a/include/boost/corosio/native/native_resolver.hpp b/include/boost/corosio/native/native_resolver.hpp index d517466eb..59a76a567 100644 --- a/include/boost/corosio/native/native_resolver.hpp +++ b/include/boost/corosio/native/native_resolver.hpp @@ -80,10 +80,10 @@ class native_resolver : public resolver } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return self_.get_impl().resolve( - h, ex, host_, service_, flags_, this->token_, &this->ec_, + cont, ex, host_, service_, flags_, this->token_, &this->ec_, &this->value_); } }; @@ -106,10 +106,10 @@ class native_resolver : public resolver } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return self_.get_impl().reverse_resolve( - h, ex, ep_, flags_, this->token_, &this->ec_, &this->value_); + cont, ex, ep_, flags_, this->token_, &this->ec_, &this->value_); } }; diff --git a/include/boost/corosio/native/native_signal_set.hpp b/include/boost/corosio/native/native_signal_set.hpp index df0f391ce..f6b956ff3 100644 --- a/include/boost/corosio/native/native_signal_set.hpp +++ b/include/boost/corosio/native/native_signal_set.hpp @@ -69,10 +69,10 @@ class native_signal_set : public signal_set } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return self_.get_impl().wait( - h, ex, this->token_, &this->ec_, &this->value_); + cont, ex, this->token_, &this->ec_, &this->value_); } }; diff --git a/include/boost/corosio/native/native_stream_file.hpp b/include/boost/corosio/native/native_stream_file.hpp index 496372b54..868276e42 100644 --- a/include/boost/corosio/native/native_stream_file.hpp +++ b/include/boost/corosio/native/native_stream_file.hpp @@ -92,10 +92,10 @@ class native_stream_file : public stream_file } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return self_.get_impl().read_some( - h, ex, buffers_, this->token_, &this->ec_, &this->bytes_); + cont, ex, buffers_, this->token_, &this->ec_, &this->bytes_); } }; @@ -114,10 +114,10 @@ class native_stream_file : public stream_file } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return self_.get_impl().write_some( - h, ex, buffers_, this->token_, &this->ec_, &this->bytes_); + cont, ex, buffers_, this->token_, &this->ec_, &this->bytes_); } }; diff --git a/include/boost/corosio/native/native_tcp_acceptor.hpp b/include/boost/corosio/native/native_tcp_acceptor.hpp index be754e036..bb9e06334 100644 --- a/include/boost/corosio/native/native_tcp_acceptor.hpp +++ b/include/boost/corosio/native/native_tcp_acceptor.hpp @@ -82,9 +82,9 @@ class native_tcp_acceptor : public tcp_acceptor } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - return acc_.get_impl().wait(h, ex, w_, this->token_, &this->ec_); + return acc_.get_impl().wait(cont, ex, w_, this->token_, &this->ec_); } }; @@ -110,10 +110,10 @@ class native_tcp_acceptor : public tcp_acceptor } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return acc_.get_impl().accept( - h, ex, this->token_, &this->ec_, &peer_impl_); + cont, ex, this->token_, &this->ec_, &peer_impl_); } }; @@ -138,10 +138,10 @@ class native_tcp_acceptor : public tcp_acceptor } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return acc_.get_impl().accept( - h, ex, this->token_, &this->ec_, &peer_impl_); + cont, ex, this->token_, &this->ec_, &peer_impl_); } }; diff --git a/include/boost/corosio/native/native_tcp_socket.hpp b/include/boost/corosio/native/native_tcp_socket.hpp index 955f4fec1..9dce5d4f3 100644 --- a/include/boost/corosio/native/native_tcp_socket.hpp +++ b/include/boost/corosio/native/native_tcp_socket.hpp @@ -92,10 +92,10 @@ class native_tcp_socket : public tcp_socket } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return self_.get_impl().read_some( - h, ex, buffers_, this->token_, &this->ec_, &this->bytes_); + cont, ex, buffers_, this->token_, &this->ec_, &this->bytes_); } }; @@ -114,10 +114,10 @@ class native_tcp_socket : public tcp_socket } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return self_.get_impl().write_some( - h, ex, buffers_, this->token_, &this->ec_, &this->bytes_); + cont, ex, buffers_, this->token_, &this->ec_, &this->bytes_); } }; @@ -133,9 +133,10 @@ class native_tcp_socket : public tcp_socket } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - return self_.get_impl().wait(h, ex, w_, this->token_, &this->ec_); + return self_.get_impl().wait( + cont, ex, w_, this->token_, &this->ec_); } }; @@ -152,10 +153,10 @@ class native_tcp_socket : public tcp_socket } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return self_.get_impl().connect( - h, ex, endpoint_, this->token_, &this->ec_); + cont, ex, endpoint_, this->token_, &this->ec_); } }; diff --git a/include/boost/corosio/native/native_udp_socket.hpp b/include/boost/corosio/native/native_udp_socket.hpp index ad8a88f84..0cf16248e 100644 --- a/include/boost/corosio/native/native_udp_socket.hpp +++ b/include/boost/corosio/native/native_udp_socket.hpp @@ -99,10 +99,10 @@ class native_udp_socket : public udp_socket } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return self_.get_impl().send_to( - h, ex, buffers_, dest_, flags_, this->token_, &this->ec_, + cont, ex, buffers_, dest_, flags_, this->token_, &this->ec_, &this->bytes_); } }; @@ -130,10 +130,10 @@ class native_udp_socket : public udp_socket } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return self_.get_impl().recv_from( - h, ex, buffers_, &source_, flags_, this->token_, &this->ec_, + cont, ex, buffers_, &source_, flags_, this->token_, &this->ec_, &this->bytes_); } }; @@ -150,9 +150,10 @@ class native_udp_socket : public udp_socket } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - return self_.get_impl().wait(h, ex, w_, this->token_, &this->ec_); + return self_.get_impl().wait( + cont, ex, w_, this->token_, &this->ec_); } }; @@ -169,10 +170,10 @@ class native_udp_socket : public udp_socket } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return self_.get_impl().connect( - h, ex, endpoint_, this->token_, &this->ec_); + cont, ex, endpoint_, this->token_, &this->ec_); } }; @@ -195,10 +196,10 @@ class native_udp_socket : public udp_socket } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return self_.get_impl().send( - h, ex, buffers_, flags_, this->token_, &this->ec_, + cont, ex, buffers_, flags_, this->token_, &this->ec_, &this->bytes_); } }; @@ -222,10 +223,10 @@ class native_udp_socket : public udp_socket } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return self_.get_impl().recv( - h, ex, buffers_, flags_, this->token_, &this->ec_, + cont, ex, buffers_, flags_, this->token_, &this->ec_, &this->bytes_); } }; diff --git a/include/boost/corosio/native/native_win_object_handle.hpp b/include/boost/corosio/native/native_win_object_handle.hpp index 2d69510a6..2003d6b72 100644 --- a/include/boost/corosio/native/native_win_object_handle.hpp +++ b/include/boost/corosio/native/native_win_object_handle.hpp @@ -59,7 +59,6 @@ class native_win_object_handle : public win_object_handle struct native_wait_awaitable : detail::void_op_base { native_win_object_handle& self_; - mutable capy::continuation cont_; explicit native_wait_awaitable(native_win_object_handle& self) noexcept : self_(self) @@ -67,11 +66,9 @@ class native_win_object_handle : public win_object_handle } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - cont_.h = h; - return self_.get_impl().wait( - cont_, ex, this->token_, &this->ec_); + return self_.get_impl().wait(cont, ex, this->token_, &this->ec_); } }; diff --git a/include/boost/corosio/native/native_win_random_access_handle.hpp b/include/boost/corosio/native/native_win_random_access_handle.hpp index add6565c0..dc0bcca75 100644 --- a/include/boost/corosio/native/native_win_random_access_handle.hpp +++ b/include/boost/corosio/native/native_win_random_access_handle.hpp @@ -68,7 +68,6 @@ class native_win_random_access_handle : public win_random_access_handle native_win_random_access_handle& self_; std::uint64_t offset_; MutableBufferSequence buffers_; - mutable capy::continuation cont_; native_read_at_awaitable( native_win_random_access_handle& self, @@ -81,11 +80,10 @@ class native_win_random_access_handle : public win_random_access_handle } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - cont_.h = h; return self_.get_impl().read_some_at( - offset_, cont_, ex, buffers_, this->token_, &this->ec_, + offset_, cont, ex, buffers_, this->token_, &this->ec_, &this->bytes_); } }; @@ -97,7 +95,6 @@ class native_win_random_access_handle : public win_random_access_handle native_win_random_access_handle& self_; std::uint64_t offset_; ConstBufferSequence buffers_; - mutable capy::continuation cont_; native_write_at_awaitable( native_win_random_access_handle& self, @@ -110,11 +107,10 @@ class native_win_random_access_handle : public win_random_access_handle } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - cont_.h = h; return self_.get_impl().write_some_at( - offset_, cont_, ex, buffers_, this->token_, &this->ec_, + offset_, cont, ex, buffers_, this->token_, &this->ec_, &this->bytes_); } }; diff --git a/include/boost/corosio/native/native_win_stream_handle.hpp b/include/boost/corosio/native/native_win_stream_handle.hpp index 206fa3723..c15fcf844 100644 --- a/include/boost/corosio/native/native_win_stream_handle.hpp +++ b/include/boost/corosio/native/native_win_stream_handle.hpp @@ -73,10 +73,10 @@ class native_win_stream_handle : public win_stream_handle } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return self_.get_impl().read_some( - h, ex, buffers_, this->token_, &this->ec_, &this->bytes_); + cont, ex, buffers_, this->token_, &this->ec_, &this->bytes_); } }; @@ -96,10 +96,10 @@ class native_win_stream_handle : public win_stream_handle } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return self_.get_impl().write_some( - h, ex, buffers_, this->token_, &this->ec_, &this->bytes_); + cont, ex, buffers_, this->token_, &this->ec_, &this->bytes_); } }; diff --git a/include/boost/corosio/openssl_stream.hpp b/include/boost/corosio/openssl_stream.hpp index 8d39b7be3..ebd56bd62 100644 --- a/include/boost/corosio/openssl_stream.hpp +++ b/include/boost/corosio/openssl_stream.hpp @@ -21,7 +21,9 @@ #include #include +#include #include +#include namespace boost::corosio { @@ -62,10 +64,6 @@ namespace boost::corosio { class BOOST_COROSIO_DECL openssl_stream final : public tls_stream { struct implementation; - BOOST_COROSIO_MSVC_WARNING_PUSH - BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // capy::any_stream, dll-interface - capy::any_stream stream_; // must be first - impl_ holds reference - BOOST_COROSIO_MSVC_WARNING_POP implementation* impl_; public: @@ -83,8 +81,7 @@ class BOOST_COROSIO_DECL openssl_stream final : public tls_stream template requires(!std::same_as, openssl_stream>) openssl_stream(S stream, tls_context const& ctx) - : stream_(std::move(stream)) - , impl_(make_implementation(stream_, ctx)) + : impl_(adopt(std::make_unique(std::move(stream)), ctx)) { } @@ -100,20 +97,28 @@ class BOOST_COROSIO_DECL openssl_stream final : public tls_stream */ template openssl_stream(S* stream, tls_context const& ctx) - : stream_(stream) - , impl_(make_implementation(stream_, ctx)) + : impl_(make_implementation(capy::any_stream(stream), {}, ctx)) { } /** Destroy the OpenSSL stream. - Releases the underlying OpenSSL resources. If constructed - in owning mode, also destroys the underlying stream. + Operations still in flight complete with + `capy::error::canceled` without touching this object, and the + TLS state they need is released when the last of them + finishes. In owning mode the underlying stream is destroyed + when the last of those operations finishes. An underlying + stream with a `cancel()` member is told to cancel them now. In reference + mode, and for an owned stream without `cancel()`, an operation + parked on the underlying stream finishes when that stream's + operation does. */ ~openssl_stream() override; /** Move construct from another OpenSSL stream. + Operations in flight on @p other continue on this stream. + @param other The source stream. After the move, @p other may only be destroyed or assigned to. */ @@ -121,6 +126,9 @@ class BOOST_COROSIO_DECL openssl_stream final : public tls_stream /** Move assign from another OpenSSL stream. + Operations in flight on this stream complete as if it were + destroyed; those on @p other continue on this stream. + @param other The source stream. After the move, @p other may only be destroyed or assigned to. @@ -164,6 +172,9 @@ class BOOST_COROSIO_DECL openssl_stream final : public tls_stream */ [[nodiscard]] capy::io_task<> shutdown() override; + /// Cancel operations pending on the owned underlying stream. + void cancel() noexcept override; + /** Reset TLS session state for reuse. Clears internal buffers and session data so the stream @@ -188,16 +199,10 @@ class BOOST_COROSIO_DECL openssl_stream final : public tls_stream void set_hostname(std::string_view hostname) override; /// Return the underlying stream. - capy::any_stream& next_layer() noexcept override - { - return stream_; - } + capy::any_stream& next_layer() noexcept override; /// Return the underlying stream. - capy::any_stream const& next_layer() const noexcept override - { - return stream_; - } + capy::any_stream const& next_layer() const noexcept override; /// Return the TLS backend name ("openssl"). std::string_view name() const noexcept override; @@ -217,8 +222,24 @@ class BOOST_COROSIO_DECL openssl_stream final : public tls_stream override; private: + static implementation* make_implementation( + capy::any_stream stream, + detail::tls_owned_transport owned, + tls_context const& ctx); + + // The transport is boxed apart from the any_stream that refers to + // it, so it outlives any operation still parked on it after the + // stream is destroyed. + template static implementation* - make_implementation(capy::any_stream& stream, tls_context const& ctx); + adopt(std::unique_ptr owned, tls_context const& ctx) + { + auto* impl = make_implementation( + capy::any_stream(owned.get()), + detail::make_owned_transport(owned.get()), ctx); + std::ignore = owned.release(); + return impl; + } }; /** Return the error category for raw OpenSSL errors. diff --git a/include/boost/corosio/posix_stream_descriptor.hpp b/include/boost/corosio/posix_stream_descriptor.hpp index 81f459339..6234167ff 100644 --- a/include/boost/corosio/posix_stream_descriptor.hpp +++ b/include/boost/corosio/posix_stream_descriptor.hpp @@ -26,6 +26,8 @@ #include #include +#include + #include #include #include @@ -148,7 +150,8 @@ class BOOST_COROSIO_DECL posix_stream_descriptor : public io_stream given direction, or an error condition is reported. No bytes are transferred and no descriptor flag is changed. - @param h Coroutine handle to resume on completion. + @param cont Continuation to resume on completion; it lives in + the awaiting frame until then. @param ex Executor for dispatching the completion. @param w The direction to wait on. @param token Stop token for cancellation. @@ -156,7 +159,7 @@ class BOOST_COROSIO_DECL posix_stream_descriptor : public io_stream @return Coroutine handle to resume immediately. */ virtual std::coroutine_handle<> wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, wait_type w, std::stop_token token, @@ -201,9 +204,9 @@ class BOOST_COROSIO_DECL posix_stream_descriptor : public io_stream wait_type w_; std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - return d_.get().wait(h, ex, w_, token_, &ec_); + return d_.get().wait(cont, ex, w_, token_, &ec_); } }; diff --git a/include/boost/corosio/random_access_file.hpp b/include/boost/corosio/random_access_file.hpp index fea1c8ee8..2c7d8ada9 100644 --- a/include/boost/corosio/random_access_file.hpp +++ b/include/boost/corosio/random_access_file.hpp @@ -169,7 +169,6 @@ class BOOST_COROSIO_DECL random_access_file : public io_object random_access_file& f_; std::uint64_t offset_; MutableBufferSequence buffers_; - mutable capy::continuation cont_; read_some_at_awaitable( random_access_file& f, @@ -185,14 +184,13 @@ class BOOST_COROSIO_DECL random_access_file : public io_object } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { // The continuation lives in the awaiting frame, which stays // put until resumption -- unlike the per-call op, which is // freed before the coroutine runs. - cont_.h = h; return f_.get().read_some_at( - offset_, cont_, ex, buffers_, this->token_, &this->ec_, + offset_, cont, ex, buffers_, this->token_, &this->ec_, &this->bytes_); } }; @@ -210,7 +208,6 @@ class BOOST_COROSIO_DECL random_access_file : public io_object random_access_file& f_; std::uint64_t offset_; ConstBufferSequence buffers_; - mutable capy::continuation cont_; write_some_at_awaitable( random_access_file& f, @@ -226,11 +223,10 @@ class BOOST_COROSIO_DECL random_access_file : public io_object } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - cont_.h = h; return f_.get().write_some_at( - offset_, cont_, ex, buffers_, this->token_, &this->ec_, + offset_, cont, ex, buffers_, this->token_, &this->ec_, &this->bytes_); } }; @@ -411,17 +407,19 @@ class BOOST_COROSIO_DECL random_access_file : public io_object `release()` cancels pending operations first. On Windows, the object keeps the handle and this throws if one is still in flight. It does the same if Windows refuses to detach the - handle from the execution context's completion port. Call + handle from the execution context's completion port. Where + file I/O runs on a thread pool, it throws likewise while a + worker thread is already performing an operation. Call `release()` again once the cancelled operations have completed. Detaching requires Windows 8.1 or later. @return The native file descriptor or handle. @throws std::system_error `errc::bad_file_descriptor` if the - file is not open. On Windows, - `errc::device_or_resource_busy` if an operation is still in - flight, or `errc::operation_not_supported` if the handle - cannot be detached. + file is not open, or `errc::device_or_resource_busy` if an + operation is still using the handle, as described above. On + Windows, it also throws `errc::operation_not_supported` if + the handle cannot be detached. */ native_handle_type release(); diff --git a/include/boost/corosio/resolver.hpp b/include/boost/corosio/resolver.hpp index 3bcac6874..be1af9d32 100644 --- a/include/boost/corosio/resolver.hpp +++ b/include/boost/corosio/resolver.hpp @@ -26,6 +26,8 @@ #include #include +#include + #include #include #include @@ -217,10 +219,10 @@ class BOOST_COROSIO_DECL resolver : public io_object resolve_flags flags_; std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return r_.get().resolve( - h, ex, host_, service_, flags_, token_, &ec_, &value_); + cont, ex, host_, service_, flags_, token_, &ec_, &value_); } }; @@ -245,12 +247,12 @@ class BOOST_COROSIO_DECL resolver : public io_object resolve_flags flags_; std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { // An empty service reaches the system resolver as null, // which is the host-only query return r_.get().resolve( - h, ex, host_, {}, flags_, token_, &ec_, &value_); + cont, ex, host_, {}, flags_, token_, &ec_, &value_); } public: @@ -304,10 +306,10 @@ class BOOST_COROSIO_DECL resolver : public io_object reverse_flags flags_; std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return r_.get().reverse_resolve( - h, ex, ep_, flags_, token_, &ec_, &value_); + cont, ex, ep_, flags_, token_, &ec_, &value_); } }; @@ -519,7 +521,8 @@ class BOOST_COROSIO_DECL resolver : public io_object { /** Initiate an asynchronous forward DNS resolution. - @param h Coroutine handle to resume on completion. + @param cont Continuation to resume on completion; it lives in + the awaiting frame until then. @param ex Executor for dispatching the completion. @param host The host name or address literal to resolve. @param service The service name or port number. @@ -531,7 +534,7 @@ class BOOST_COROSIO_DECL resolver : public io_object @return Coroutine handle to resume immediately. */ virtual std::coroutine_handle<> resolve( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, std::string_view host, std::string_view service, @@ -542,7 +545,8 @@ class BOOST_COROSIO_DECL resolver : public io_object /** Initiate an asynchronous reverse DNS resolution. - @param h Coroutine handle to resume on completion. + @param cont Continuation to resume on completion; it lives in + the awaiting frame until then. @param ex Executor for dispatching the completion. @param ep The endpoint to resolve. @param flags Flags controlling the lookup. @@ -553,7 +557,7 @@ class BOOST_COROSIO_DECL resolver : public io_object @return Coroutine handle to resume immediately. */ virtual std::coroutine_handle<> reverse_resolve( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, endpoint const& ep, reverse_flags flags, diff --git a/include/boost/corosio/stream_file.hpp b/include/boost/corosio/stream_file.hpp index 9778c911f..23faddce5 100644 --- a/include/boost/corosio/stream_file.hpp +++ b/include/boost/corosio/stream_file.hpp @@ -279,17 +279,19 @@ class BOOST_COROSIO_DECL stream_file : public io_stream `release()` cancels pending operations first. On Windows, the object keeps the handle and this throws if one is still in flight. It does the same if Windows refuses to detach the - handle from the execution context's completion port. Call + handle from the execution context's completion port. Where + file I/O runs on a thread pool, it throws likewise while a + worker thread is already performing an operation. Call `release()` again once the cancelled operations have completed. Detaching requires Windows 8.1 or later. @return The native file descriptor or handle. @throws std::system_error `errc::bad_file_descriptor` if the - file is not open. On Windows, - `errc::device_or_resource_busy` if an operation is still in - flight, or `errc::operation_not_supported` if the handle - cannot be detached. + file is not open, or `errc::device_or_resource_busy` if an + operation is still using the handle, as described above. On + Windows, it also throws `errc::operation_not_supported` if + the handle cannot be detached. */ native_handle_type release(); diff --git a/include/boost/corosio/tcp_acceptor.hpp b/include/boost/corosio/tcp_acceptor.hpp index b7624124f..c66e6082b 100644 --- a/include/boost/corosio/tcp_acceptor.hpp +++ b/include/boost/corosio/tcp_acceptor.hpp @@ -31,10 +31,13 @@ #include #include +#include + #include #include #include #include +#include namespace boost::corosio { @@ -81,9 +84,9 @@ class BOOST_COROSIO_DECL tcp_acceptor : public io_object wait_type w_; std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - return acc_.get().wait(h, ex, w_, token_, &ec_); + return acc_.get().wait(cont, ex, w_, token_, &ec_); } }; @@ -97,6 +100,7 @@ class BOOST_COROSIO_DECL tcp_acceptor : public io_object tcp_socket& peer_; mutable io_object::implementation* peer_impl_ = nullptr; + accept_awaitable(tcp_acceptor& acc, tcp_socket& peer) noexcept : acc_(acc) , peer_(peer) @@ -104,17 +108,25 @@ class BOOST_COROSIO_DECL tcp_acceptor : public io_object } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return acc_.get().accept( - h, ex, this->token_, &this->ec_, &peer_impl_); + cont, ex, this->token_, &this->ec_, &peer_impl_); } public: + // A peer accepted for an awaiter that never resumed is closed + // here, or nothing would own it. + ~accept_awaitable() + { + if (peer_impl_) + discard_peer(acc_, peer_impl_); + } + [[nodiscard]] capy::io_result<> await_resume() const noexcept { if (!this->ec_ && peer_impl_) - peer_.h_.reset(peer_impl_); + peer_.h_.reset(std::exchange(peer_impl_, nullptr)); return {this->ec_}; } }; @@ -128,18 +140,27 @@ class BOOST_COROSIO_DECL tcp_acceptor : public io_object tcp_acceptor& acc_; mutable io_object::implementation* peer_impl_ = nullptr; + explicit accept_value_awaitable(tcp_acceptor& acc) noexcept : acc_(acc) { } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return acc_.get().accept( - h, ex, this->token_, &this->ec_, &peer_impl_); + cont, ex, this->token_, &this->ec_, &peer_impl_); } public: + // A peer accepted for an awaiter that never resumed is closed + // here, or nothing would own it. + ~accept_value_awaitable() + { + if (peer_impl_) + discard_peer(acc_, peer_impl_); + } + [[nodiscard]] capy::io_result await_resume() noexcept { // The peer is built only on success: error paths must not @@ -148,7 +169,7 @@ class BOOST_COROSIO_DECL tcp_acceptor : public io_object return {this->ec_, tcp_socket()}; tcp_socket peer(acc_.context()); - peer.h_.reset(peer_impl_); + peer.h_.reset(std::exchange(peer_impl_, nullptr)); return {this->ec_, std::move(peer)}; } }; @@ -608,7 +629,8 @@ class BOOST_COROSIO_DECL tcp_acceptor : public io_object { /** Initiate an asynchronous accept operation. - @param h Coroutine handle to resume on completion. + @param cont Continuation to resume on completion; it lives in + the awaiting frame until then. @param ex Executor for dispatching the completion. @param token Stop token for cancellation. @param ec Output error code. @@ -617,7 +639,7 @@ class BOOST_COROSIO_DECL tcp_acceptor : public io_object @return Coroutine handle to resume immediately. */ virtual std::coroutine_handle<> accept( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, std::stop_token token, std::error_code* ec, @@ -630,7 +652,8 @@ class BOOST_COROSIO_DECL tcp_acceptor : public io_object for an incoming connection), or an error condition is reported. No connection is consumed. - @param h Coroutine handle to resume on completion. + @param cont Continuation to resume on completion; it lives in + the awaiting frame until then. @param ex Executor for dispatching the completion. @param w The direction to wait on. @param token Stop token for cancellation. @@ -639,7 +662,7 @@ class BOOST_COROSIO_DECL tcp_acceptor : public io_object @return Coroutine handle to resume immediately. */ virtual std::coroutine_handle<> wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, wait_type w, std::stop_token token, @@ -737,6 +760,10 @@ class BOOST_COROSIO_DECL tcp_acceptor : public io_object { return *static_cast(h_.get()); } + + /// Close and release a non-null accepted peer no socket took over. + static void + discard_peer(tcp_acceptor& acc, io_object::implementation* impl) noexcept; }; } // namespace boost::corosio diff --git a/include/boost/corosio/tcp_socket.hpp b/include/boost/corosio/tcp_socket.hpp index c3d315e5c..dc081a25e 100644 --- a/include/boost/corosio/tcp_socket.hpp +++ b/include/boost/corosio/tcp_socket.hpp @@ -33,6 +33,8 @@ #include #include +#include + #include #include #include @@ -83,7 +85,8 @@ class BOOST_COROSIO_DECL tcp_socket : public io_stream { /** Initiate an asynchronous connect to the given endpoint. - @param h Coroutine handle to resume on completion. + @param cont Continuation to resume on completion; it lives in + the awaiting frame until then. @param ex Executor for dispatching the completion. @param ep The remote endpoint to connect to. @param token Stop token for cancellation. @@ -92,7 +95,7 @@ class BOOST_COROSIO_DECL tcp_socket : public io_stream @return Coroutine handle to resume immediately. */ virtual std::coroutine_handle<> connect( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, endpoint ep, std::stop_token token, @@ -104,7 +107,8 @@ class BOOST_COROSIO_DECL tcp_socket : public io_stream specified direction, or an error condition is reported. No bytes are transferred. - @param h Coroutine handle to resume on completion. + @param cont Continuation to resume on completion; it lives in + the awaiting frame until then. @param ex Executor for dispatching the completion. @param w The direction to wait on. @param token Stop token for cancellation. @@ -113,7 +117,7 @@ class BOOST_COROSIO_DECL tcp_socket : public io_stream @return Coroutine handle to resume immediately. */ virtual std::coroutine_handle<> wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, wait_type w, std::stop_token token, @@ -208,9 +212,9 @@ class BOOST_COROSIO_DECL tcp_socket : public io_stream endpoint endpoint_; std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - return s_.get().connect(h, ex, endpoint_, token_, &ec_); + return s_.get().connect(cont, ex, endpoint_, token_, &ec_); } }; @@ -228,9 +232,9 @@ class BOOST_COROSIO_DECL tcp_socket : public io_stream wait_type w_; std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - return s_.get().wait(h, ex, w_, token_, &ec_); + return s_.get().wait(cont, ex, w_, token_, &ec_); } }; diff --git a/include/boost/corosio/test/mocket.hpp b/include/boost/corosio/test/mocket.hpp index 577dc410c..61ead5578 100644 --- a/include/boost/corosio/test/mocket.hpp +++ b/include/boost/corosio/test/mocket.hpp @@ -620,27 +620,21 @@ make_mocket_pair( done_out = true; }(acc, accepted_socket, accept_ec, accept_done)); + // A failed connect leaves nothing for the accept to receive. capy::run_async(ex)( - [](Socket& s, endpoint ep, std::error_code& ec_out, + [](Socket& s, Acceptor& a, endpoint ep, std::error_code& ec_out, bool& done_out) -> capy::task<> { auto [ec] = co_await s.connect(ep); ec_out = ec; done_out = true; - }(peer, endpoint(ipv4_address::loopback(), port), connect_ec, + if (ec) + a.cancel(); + }(peer, acc, endpoint(ipv4_address::loopback(), port), connect_ec, connect_done)); ctx.run(); ctx.restart(); - if (!accept_done || accept_ec) - { - std::fprintf( - stderr, "make_mocket_pair: accept failed (done=%d, ec=%s)\n", - accept_done, accept_ec.message().c_str()); - acc.close(); - throw std::runtime_error("mocket accept failed"); - } - if (!connect_done || connect_ec) { std::fprintf( @@ -651,6 +645,15 @@ make_mocket_pair( throw std::runtime_error("mocket connect failed"); } + if (!accept_done || accept_ec) + { + std::fprintf( + stderr, "make_mocket_pair: accept failed (done=%d, ec=%s)\n", + accept_done, accept_ec.message().c_str()); + acc.close(); + throw std::runtime_error("mocket accept failed"); + } + m.socket() = std::move(accepted_socket); acc.close(); diff --git a/include/boost/corosio/test/socket_pair.hpp b/include/boost/corosio/test/socket_pair.hpp index 8e0855d99..54eb4a58c 100644 --- a/include/boost/corosio/test/socket_pair.hpp +++ b/include/boost/corosio/test/socket_pair.hpp @@ -81,27 +81,21 @@ make_socket_pair(io_context& ctx) done_out = true; }(acc, s1, accept_ec, accept_done)); + // A failed connect leaves nothing for the accept to receive. capy::run_async(ex)( - [](Socket& s, endpoint ep, std::error_code& ec_out, + [](Socket& s, Acceptor& a, endpoint ep, std::error_code& ec_out, bool& done_out) -> capy::task<> { auto [ec] = co_await s.connect(ep); ec_out = ec; done_out = true; - }(s2, endpoint(ipv4_address::loopback(), port), connect_ec, + if (ec) + a.cancel(); + }(s2, acc, endpoint(ipv4_address::loopback(), port), connect_ec, connect_done)); ctx.run(); ctx.restart(); - if (!accept_done || accept_ec) - { - std::fprintf( - stderr, "socket_pair: accept failed (done=%d, ec=%s)\n", - accept_done, accept_ec.message().c_str()); - acc.close(); - throw std::runtime_error("socket_pair accept failed"); - } - if (!connect_done || connect_ec) { std::fprintf( @@ -112,6 +106,15 @@ make_socket_pair(io_context& ctx) throw std::runtime_error("socket_pair connect failed"); } + if (!accept_done || accept_ec) + { + std::fprintf( + stderr, "socket_pair: accept failed (done=%d, ec=%s)\n", + accept_done, accept_ec.message().c_str()); + acc.close(); + throw std::runtime_error("socket_pair accept failed"); + } + acc.close(); if constexpr (Linger) diff --git a/include/boost/corosio/tls_stream.hpp b/include/boost/corosio/tls_stream.hpp index ee1df6a60..c6f32fbb8 100644 --- a/include/boost/corosio/tls_stream.hpp +++ b/include/boost/corosio/tls_stream.hpp @@ -23,6 +23,57 @@ namespace boost::corosio { +namespace detail { + +/** A transport a TLS stream owns, held apart from its `any_stream`. + + Destroying the stream cancels the transport's pending operations + while the TLS state they complete into stays alive; the transport + itself outlives every operation that could be parked on it. +*/ +struct tls_owned_transport +{ + /// The owned transport, or null in reference mode. + void* p = nullptr; + + /// Destroy the transport `p` points to. + void (*destroy)(void*) noexcept = nullptr; + + /// Cancel the transport's pending operations, or null if it can't. + void (*cancel)(void*) noexcept = nullptr; +}; + +/// Destroy an owned transport of type `S`. +template +void +destroy_owned_transport(void* p) noexcept +{ + delete static_cast(p); +} + +/// Cancel an owned transport of type `S`'s pending operations. +template +void +cancel_owned_transport(void* p) noexcept +{ + static_cast(p)->cancel(); +} + +/// Make the owned-transport record for a transport of type `S`. +template +tls_owned_transport +make_owned_transport(S* p) noexcept +{ + tls_owned_transport t{p, &destroy_owned_transport, nullptr}; + if constexpr (requires(S& s) { + { s.cancel() } noexcept; + }) + t.cancel = &cancel_owned_transport; + return t; +} + +} // namespace detail + /** TLS handshake role. Specifies whether to perform the TLS handshake as a client or server. @@ -167,6 +218,23 @@ class BOOST_COROSIO_DECL tls_stream */ [[nodiscard]] virtual capy::io_task<> shutdown() = 0; + /** Cancel operations pending on the owned underlying stream. + + Operations parked on the underlying stream complete with + `capy::error::canceled`. A stream that owns another TLS stream + reaches the innermost transport through this, so destroying + the outer stream cancels a read parked through both. + + In reference mode, and when the owned stream has no `cancel()` + member, this does nothing; cancel the underlying stream + directly instead. + + @par Thread Safety + Not thread safe with respect to destroying or moving this + stream. + */ + virtual void cancel() noexcept = 0; + /** Reset TLS session state for reuse. Releases TLS session state including session keys and peer diff --git a/include/boost/corosio/udp_socket.hpp b/include/boost/corosio/udp_socket.hpp index ae9013e2b..083bca48a 100644 --- a/include/boost/corosio/udp_socket.hpp +++ b/include/boost/corosio/udp_socket.hpp @@ -33,6 +33,8 @@ #include #include +#include + #include #include #include @@ -83,7 +85,8 @@ class BOOST_COROSIO_DECL udp_socket : public io_object { /** Initiate an asynchronous `send_to` operation. - @param h Coroutine handle to resume on completion. + @param cont Continuation to resume on completion; it lives in + the awaiting frame until then. @param ex Executor for dispatching the completion. @param buf The buffer data to send. @param dest The destination endpoint. @@ -97,7 +100,7 @@ class BOOST_COROSIO_DECL udp_socket : public io_object @return Coroutine handle to resume immediately. */ virtual std::coroutine_handle<> send_to( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buf, endpoint dest, @@ -108,7 +111,8 @@ class BOOST_COROSIO_DECL udp_socket : public io_object /** Initiate an asynchronous `recv_from` operation. - @param h Coroutine handle to resume on completion. + @param cont Continuation to resume on completion; it lives in + the awaiting frame until then. @param ex Executor for dispatching the completion. @param buf The buffer to receive into. @param source Output endpoint for the sender's address. @@ -122,7 +126,7 @@ class BOOST_COROSIO_DECL udp_socket : public io_object @return Coroutine handle to resume immediately. */ virtual std::coroutine_handle<> recv_from( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buf, endpoint* source, @@ -203,7 +207,8 @@ class BOOST_COROSIO_DECL udp_socket : public io_object /** Initiate an asynchronous connect to set the default peer. - @param h Coroutine handle to resume on completion. + @param cont Continuation to resume on completion; it lives in + the awaiting frame until then. @param ex Executor for dispatching the completion. @param ep The remote endpoint to connect to. @param token Stop token for cancellation. @@ -212,7 +217,7 @@ class BOOST_COROSIO_DECL udp_socket : public io_object @return Coroutine handle to resume immediately. */ virtual std::coroutine_handle<> connect( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, endpoint ep, std::stop_token token, @@ -220,7 +225,8 @@ class BOOST_COROSIO_DECL udp_socket : public io_object /** Initiate an asynchronous connected send operation. - @param h Coroutine handle to resume on completion. + @param cont Continuation to resume on completion; it lives in + the awaiting frame until then. @param ex Executor for dispatching the completion. @param buf The buffer data to send. @param flags Portable @ref message_flags bits (for example @@ -233,7 +239,7 @@ class BOOST_COROSIO_DECL udp_socket : public io_object @return Coroutine handle to resume immediately. */ virtual std::coroutine_handle<> send( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buf, int flags, @@ -243,7 +249,8 @@ class BOOST_COROSIO_DECL udp_socket : public io_object /** Initiate an asynchronous connected `recv` operation. - @param h Coroutine handle to resume on completion. + @param cont Continuation to resume on completion; it lives in + the awaiting frame until then. @param ex Executor for dispatching the completion. @param buf The buffer to receive into. @param flags Portable @ref message_flags bits (for example @@ -256,7 +263,7 @@ class BOOST_COROSIO_DECL udp_socket : public io_object @return Coroutine handle to resume immediately. */ virtual std::coroutine_handle<> recv( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, buffer_param buf, int flags, @@ -270,7 +277,8 @@ class BOOST_COROSIO_DECL udp_socket : public io_object specified direction, or an error condition is reported. No bytes are transferred. - @param h Coroutine handle to resume on completion. + @param cont Continuation to resume on completion; it lives in + the awaiting frame until then. @param ex Executor for dispatching the completion. @param w The direction to wait on. @param token Stop token for cancellation. @@ -279,7 +287,7 @@ class BOOST_COROSIO_DECL udp_socket : public io_object @return Coroutine handle to resume immediately. */ virtual std::coroutine_handle<> wait( - std::coroutine_handle<> h, + capy::continuation& cont, capy::executor_ref ex, wait_type w, std::stop_token token, @@ -316,10 +324,10 @@ class BOOST_COROSIO_DECL udp_socket : public io_object int flags_; std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return s_.get().send_to( - h, ex, buf_, dest_, flags_, token_, &ec_, &bytes_); + cont, ex, buf_, dest_, flags_, token_, &ec_, &bytes_); } }; @@ -353,10 +361,10 @@ class BOOST_COROSIO_DECL udp_socket : public io_object int flags_; std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { return s_.get().recv_from( - h, ex, buf_, &source_, flags_, token_, &ec_, &bytes_); + cont, ex, buf_, &source_, flags_, token_, &ec_, &bytes_); } }; @@ -378,9 +386,9 @@ class BOOST_COROSIO_DECL udp_socket : public io_object endpoint endpoint_; std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - return s_.get().connect(h, ex, endpoint_, token_, &ec_); + return s_.get().connect(cont, ex, endpoint_, token_, &ec_); } }; @@ -398,9 +406,9 @@ class BOOST_COROSIO_DECL udp_socket : public io_object wait_type w_; std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - return s_.get().wait(h, ex, w_, token_, &ec_); + return s_.get().wait(cont, ex, w_, token_, &ec_); } }; @@ -424,9 +432,9 @@ class BOOST_COROSIO_DECL udp_socket : public io_object int flags_; std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - return s_.get().send(h, ex, buf_, flags_, token_, &ec_, &bytes_); + return s_.get().send(cont, ex, buf_, flags_, token_, &ec_, &bytes_); } }; @@ -450,9 +458,9 @@ class BOOST_COROSIO_DECL udp_socket : public io_object int flags_; std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - return s_.get().recv(h, ex, buf_, flags_, token_, &ec_, &bytes_); + return s_.get().recv(cont, ex, buf_, flags_, token_, &ec_, &bytes_); } }; diff --git a/include/boost/corosio/win_object_handle.hpp b/include/boost/corosio/win_object_handle.hpp index dc5c8c547..ab04202bd 100644 --- a/include/boost/corosio/win_object_handle.hpp +++ b/include/boost/corosio/win_object_handle.hpp @@ -128,15 +128,11 @@ class BOOST_COROSIO_DECL win_object_handle : public io_object friend detail::void_op_base; win_object_handle& o_; - // Lives in the awaiting frame until resumption, so a completion - // posted through an executor never shares the object's state. - mutable capy::continuation cont_; std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - cont_.h = h; - return o_.get().wait(cont_, ex, token_, &ec_); + return o_.get().wait(cont, ex, token_, &ec_); } }; diff --git a/include/boost/corosio/win_random_access_handle.hpp b/include/boost/corosio/win_random_access_handle.hpp index de0432a0c..7135cc19f 100644 --- a/include/boost/corosio/win_random_access_handle.hpp +++ b/include/boost/corosio/win_random_access_handle.hpp @@ -175,7 +175,6 @@ class BOOST_COROSIO_DECL win_random_access_handle : public io_object win_random_access_handle& f_; std::uint64_t offset_; MutableBufferSequence buffers_; - mutable capy::continuation cont_; read_some_at_awaitable( win_random_access_handle& f, @@ -191,11 +190,10 @@ class BOOST_COROSIO_DECL win_random_access_handle : public io_object } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - cont_.h = h; return f_.get().read_some_at( - offset_, cont_, ex, buffers_, this->token_, &this->ec_, + offset_, cont, ex, buffers_, this->token_, &this->ec_, &this->bytes_); } }; @@ -213,7 +211,6 @@ class BOOST_COROSIO_DECL win_random_access_handle : public io_object win_random_access_handle& f_; std::uint64_t offset_; ConstBufferSequence buffers_; - mutable capy::continuation cont_; write_some_at_awaitable( win_random_access_handle& f, @@ -229,11 +226,10 @@ class BOOST_COROSIO_DECL win_random_access_handle : public io_object } std::coroutine_handle<> - dispatch(std::coroutine_handle<> h, capy::executor_ref ex) const + dispatch(capy::continuation& cont, capy::executor_ref ex) const { - cont_.h = h; return f_.get().write_some_at( - offset_, cont_, ex, buffers_, this->token_, &this->ec_, + offset_, cont, ex, buffers_, this->token_, &this->ec_, &this->bytes_); } }; diff --git a/include/boost/corosio/wolfssl_stream.hpp b/include/boost/corosio/wolfssl_stream.hpp index f0196ff4d..f633e5792 100644 --- a/include/boost/corosio/wolfssl_stream.hpp +++ b/include/boost/corosio/wolfssl_stream.hpp @@ -21,7 +21,9 @@ #include #include +#include #include +#include namespace boost::corosio { @@ -62,10 +64,6 @@ namespace boost::corosio { class BOOST_COROSIO_DECL wolfssl_stream final : public tls_stream { struct implementation; - BOOST_COROSIO_MSVC_WARNING_PUSH - BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // capy::any_stream, dll-interface - capy::any_stream stream_; // must be first - impl_ holds reference - BOOST_COROSIO_MSVC_WARNING_POP implementation* impl_; public: @@ -82,8 +80,7 @@ class BOOST_COROSIO_DECL wolfssl_stream final : public tls_stream template requires(!std::same_as, wolfssl_stream>) wolfssl_stream(S stream, tls_context const& ctx) - : stream_(std::move(stream)) - , impl_(make_implementation(stream_, ctx)) + : impl_(adopt(std::make_unique(std::move(stream)), ctx)) { } @@ -99,20 +96,28 @@ class BOOST_COROSIO_DECL wolfssl_stream final : public tls_stream */ template wolfssl_stream(S* stream, tls_context const& ctx) - : stream_(stream) - , impl_(make_implementation(stream_, ctx)) + : impl_(make_implementation(capy::any_stream(stream), {}, ctx)) { } /** Destroy the WolfSSL stream. - Releases the underlying WolfSSL resources. If constructed - in owning mode, also destroys the underlying stream. + Operations still in flight complete with + `capy::error::canceled` without touching this object, and the + TLS state they need is released when the last of them + finishes. In owning mode the underlying stream is destroyed + when the last of those operations finishes. An underlying + stream with a `cancel()` member is told to cancel them now. In reference + mode, and for an owned stream without `cancel()`, an operation + parked on the underlying stream finishes when that stream's + operation does. */ ~wolfssl_stream() override; /** Move construct from another WolfSSL stream. + Operations in flight on @p other continue on this stream. + @param other The source stream. After the move, @p other may only be destroyed or assigned to. */ @@ -120,6 +125,9 @@ class BOOST_COROSIO_DECL wolfssl_stream final : public tls_stream /** Move assign from another WolfSSL stream. + Operations in flight on this stream complete as if it were + destroyed; those on @p other continue on this stream. + @param other The source stream. After the move, @p other may only be destroyed or assigned to. @@ -163,6 +171,9 @@ class BOOST_COROSIO_DECL wolfssl_stream final : public tls_stream */ [[nodiscard]] capy::io_task<> shutdown() override; + /// Cancel operations pending on the owned underlying stream. + void cancel() noexcept override; + /** Reset TLS session state for reuse. Clears internal buffers and session data so the stream @@ -183,16 +194,10 @@ class BOOST_COROSIO_DECL wolfssl_stream final : public tls_stream void set_hostname(std::string_view hostname) override; /// Return the underlying stream. - capy::any_stream& next_layer() noexcept override - { - return stream_; - } + capy::any_stream& next_layer() noexcept override; /// Return the underlying stream. - capy::any_stream const& next_layer() const noexcept override - { - return stream_; - } + capy::any_stream const& next_layer() const noexcept override; /// Return the TLS backend name ("wolfssl"). std::string_view name() const noexcept override; @@ -212,8 +217,24 @@ class BOOST_COROSIO_DECL wolfssl_stream final : public tls_stream override; private: + static implementation* make_implementation( + capy::any_stream stream, + detail::tls_owned_transport owned, + tls_context const& ctx); + + // The transport is boxed apart from the any_stream that refers to + // it, so it outlives any operation still parked on it after the + // stream is destroyed. + template static implementation* - make_implementation(capy::any_stream& stream, tls_context const& ctx); + adopt(std::unique_ptr owned, tls_context const& ctx) + { + auto* impl = make_implementation( + capy::any_stream(owned.get()), + detail::make_owned_transport(owned.get()), ctx); + std::ignore = owned.release(); + return impl; + } }; /** Return the error category for raw WolfSSL errors. diff --git a/src/corosio/src/local_stream_acceptor.cpp b/src/corosio/src/local_stream_acceptor.cpp index 8de8132e6..6b1346a13 100644 --- a/src/corosio/src/local_stream_acceptor.cpp +++ b/src/corosio/src/local_stream_acceptor.cpp @@ -175,4 +175,18 @@ local_stream_acceptor::local_endpoint() const noexcept return get().local_endpoint(); } +void +local_stream_acceptor::discard_peer( + local_stream_acceptor& acc, io_object::implementation* impl) noexcept +{ + // Its own handle closes and releases the peer, as the socket the + // awaiter would have received does. + handle h( + acc.ctx_, + detail::use_backend_service< + detail::local_stream_service_of, + detail::local_stream_service>(acc.ctx_), + impl); +} + } // namespace boost::corosio diff --git a/src/corosio/src/tcp_acceptor.cpp b/src/corosio/src/tcp_acceptor.cpp index 1fe236ac4..9dfa38169 100644 --- a/src/corosio/src/tcp_acceptor.cpp +++ b/src/corosio/src/tcp_acceptor.cpp @@ -19,6 +19,7 @@ #include #else #include +#include #endif #include @@ -27,6 +28,16 @@ namespace boost::corosio { +namespace { + +#if BOOST_COROSIO_HAS_IOCP +using tcp_peer_service = detail::win_tcp_service; +#else +using tcp_peer_service = detail::tcp_service; +#endif + +} // namespace + #if BOOST_COROSIO_HAS_IOCP namespace { @@ -203,4 +214,18 @@ tcp_acceptor::local_endpoint() const noexcept return get().local_endpoint(); } +void +tcp_acceptor::discard_peer( + tcp_acceptor& acc, io_object::implementation* impl) noexcept +{ + // Its own handle closes and releases the peer, as the socket the + // awaiter would have received does. + auto& ctx = acc.context(); + handle h( + ctx, + detail::use_backend_service( + ctx), + impl); +} + } // namespace boost::corosio diff --git a/src/corosio/src/tls/detail/engine_driver.hpp b/src/corosio/src/tls/detail/engine_driver.hpp index 5267b0eab..56f2fb1f5 100644 --- a/src/corosio/src/tls/detail/engine_driver.hpp +++ b/src/corosio/src/tls/detail/engine_driver.hpp @@ -23,6 +23,7 @@ #include #include +#include #include #include #include @@ -157,7 +158,23 @@ class engine_driver // Large enough to hold the largest possible TLS record. static constexpr std::size_t buffer_size_ = std::size_t{17} * 1024; - capy::any_stream* s_; + // Operations hold a reference across their suspensions, so the + // driver outlives the stream that created it until they finish. + std::atomic refs_{1}; + + // Set when the stream is destroyed: no transport call may follow. + std::atomic orphaned_{false}; + + // Transport calls in flight. Once orphaned, the owned transport is + // destroyed when this reaches zero: right away for a stream with no + // call parked, so it never outlives its context waiting on an + // operation nobody awaits. + std::atomic calls_{0}; + std::atomic transport_gone_{false}; + + // The transport in owning mode; s_ refers to it without owning. + tls_owned_transport owned_; + capy::any_stream s_; tls_context ctx_; Engine eng_; @@ -209,6 +226,9 @@ class engine_driver // order even when both directions produce output. while (eng_.pending_output() > 0 || out_len_ > 0) { + if (orphaned()) + co_return make_error_code(capy::error::canceled); + // Take the abandoned tail (from an earlier failed write in // this loop or a prior flush) into a local and append fresh // engine ciphertext behind it, preserving record order. @@ -229,8 +249,34 @@ class engine_driver co_return make_error_code( std::errc:: no_buffer_space); // LCOV_EXCL_LINE unreachable: transport returned 0 with no error unreachable: pending bytes confirmed - auto [ec, wn] = co_await capy::write( - *s_, capy::const_buffer(out_buf_.data(), n)); + // capy::write's loop, with the stream's destruction checked + // between partial writes: once orphaned, the transport is + // no longer this driver's to write to. + std::error_code ec; + std::size_t wn = 0; + while (wn < n) + { + transport_call call(*this); + if (!call) + { + ec = make_error_code(capy::error::canceled); + break; + } + auto [wec, k] = co_await s_.write_some( + capy::const_buffer(out_buf_.data() + wn, n - wn)); + call.end(); + wn += k; + if (wn < n && wec) + { + ec = wec; + break; + } + if (wn < n && orphaned()) + { + ec = make_error_code(capy::error::canceled); + break; + } + } if (ec) { // wn bytes already reached the peer; keep only the unsent @@ -274,6 +320,9 @@ class engine_driver if (read_gen_ != gen) co_return std::error_code{}; + if (orphaned()) + co_return make_error_code(capy::error::canceled); + // Read the transport straight into the engine's input staging: // input_area() hands back the contiguous writable run, so no // staging buffer or deposit copy sits between the socket and the @@ -285,7 +334,16 @@ class engine_driver if (cap == 0) co_return std::error_code{}; - auto [ec, n] = co_await s_->read_some(capy::mutable_buffer(dst, cap)); + transport_call call(*this); + if (!call) + co_return make_error_code(capy::error::canceled); + auto [ec, n] = co_await s_.read_some(capy::mutable_buffer(dst, cap)); + call.end(); + + // The stream went away while the read was parked; its result + // belongs to no one. + if (orphaned()) + co_return make_error_code(capy::error::canceled); // ReadStream permits n>0 alongside ec (IOCP forwards // bytes_transferred on failed completions; a canceled read can @@ -310,6 +368,66 @@ class engine_driver no_buffer_space); // LCOV_EXCL_LINE unreachable: staging cannot stay empty } + void destroy_transport() noexcept + { + if (transport_gone_.exchange(true, std::memory_order_acq_rel)) + return; + auto const owned = std::exchange(owned_, tls_owned_transport{}); + if (owned.destroy) + owned.destroy(owned.p); + } + + // Return false once orphaned: the transport may already be gone. + // The orphan check follows the count's increment, pairing with + // orphan()'s store-then-load, so one side always sees the other. + bool begin_transport_call() noexcept + { + calls_.fetch_add(1, std::memory_order_seq_cst); + if (!orphaned_.load(std::memory_order_seq_cst)) + return true; + end_transport_call(); + return false; + } + + void end_transport_call() noexcept + { + if (calls_.fetch_sub(1, std::memory_order_seq_cst) == 1 && + orphaned_.load(std::memory_order_seq_cst)) + destroy_transport(); + } + + // A transport call that also ends when its frame is unwound while + // parked, so the count never pins an orphaned transport. + class transport_call + { + engine_driver* d_; + + public: + ~transport_call() + { + end(); + } + + explicit transport_call(engine_driver& d) noexcept + : d_(d.begin_transport_call() ? &d : nullptr) + { + } + + transport_call(transport_call const&) = delete; + transport_call& operator=(transport_call const&) = delete; + + explicit operator bool() const noexcept + { + return d_ != nullptr; + } + + void end() noexcept + { + if (d_) + std::exchange(d_, nullptr)->end_transport_call(); + } + }; + // A prior read/write already reported its full transfer as success; // the trailing flush error it deferred surfaces on the next call // instead. Each entry point takes it exactly once. @@ -323,18 +441,90 @@ class engine_driver public: /** Construct a driver over a transport stream. - @param s The transport; the caller keeps it alive and repoints - it on move via `rebind_stream`. + The driver starts with one reference, the creating stream's. + + @param s The transport. + @param owned The transport `s` refers to, when the stream owns + it; the driver destroys it on @ref orphan. @param ctx The TLS context handed to the engine's handshake preparation. */ - engine_driver(capy::any_stream& s, tls_context ctx) - : s_(&s) + engine_driver( + capy::any_stream s, tls_owned_transport owned, tls_context ctx) + : owned_(owned) + , s_(std::move(s)) , ctx_(std::move(ctx)) { out_buf_.resize(buffer_size_); } + ~engine_driver() + { + destroy_transport(); + } + + engine_driver(engine_driver const&) = delete; + engine_driver& operator=(engine_driver const&) = delete; + + /// Add a reference for an operation in flight. + void add_ref() noexcept + { + refs_.fetch_add(1, std::memory_order_relaxed); + } + + /// Drop a reference; return true if it was the last. + bool release_ref() noexcept + { + return refs_.fetch_sub(1, std::memory_order_acq_rel) == 1; + } + + /** Detach the driver from its destroyed stream. + + Operations still in flight finish as canceled without touching + the transport again. An owned transport is destroyed now when + no transport call is parked on it; otherwise it is told to + cancel, if it can, and destroyed when the last parked call + returns. + */ + void orphan() noexcept + { + // Counted as a call itself, so a parked call returning meanwhile + // cannot destroy the transport under the cancel. + calls_.fetch_add(1, std::memory_order_seq_cst); + orphaned_.store(true, std::memory_order_seq_cst); + if (owned_.cancel) + owned_.cancel(owned_.p); + end_transport_call(); + } + + /// Cancel the owned transport's pending operations, if it can. + void cancel_transport() noexcept + { + if (!begin_transport_call()) + return; + if (owned_.cancel) + owned_.cancel(owned_.p); + end_transport_call(); + } + + /// Return true once the stream has been destroyed. + bool orphaned() const noexcept + { + return orphaned_.load(std::memory_order_acquire); + } + + /// Return the transport. + capy::any_stream& stream() noexcept + { + return s_; + } + + /// Return the transport. + capy::any_stream const& stream() const noexcept + { + return s_; + } + /// Return the engine for backend-specific setup. Engine& engine() noexcept { @@ -347,12 +537,6 @@ class engine_driver return ctx_; } - /// Point the driver at the transport's post-move location. - void rebind_stream(capy::any_stream& s) noexcept - { - s_ = &s; - } - /// Set the hostname applied to the next client handshake. void set_hostname(std::string_view hostname) { @@ -694,6 +878,87 @@ class engine_driver } }; +/** A reference to a TLS stream's driver held by an operation. + + Taken when the operation is created, so a stream destroyed or + moved before the operation is awaited or finishes leaves its + driver alive until the operation lets go. + + @tparam Impl The stream's implementation, derived from + @ref engine_driver. +*/ +template +class driver_ref +{ + Impl* p_; + +public: + /// Construct, adding a reference to @p p. + explicit driver_ref(Impl* p) noexcept : p_(p) + { + p_->add_ref(); + } + + /// Construct by taking @p other's reference. + driver_ref(driver_ref&& other) noexcept + : p_(std::exchange(other.p_, nullptr)) + { + } + + driver_ref(driver_ref const&) = delete; + driver_ref& operator=(driver_ref const&) = delete; + driver_ref& operator=(driver_ref&&) = delete; + + /// Destroy, deleting the driver if this was its last reference. + ~driver_ref() + { + if (p_ && p_->release_ref()) + delete p_; + } + + /// Return the driver. + Impl* operator->() const noexcept + { + return p_; + } +}; + +/// Read through a driver an operation keeps alive. +template +capy::io_task +driver_read_some( + driver_ref d, + capy::detail::mutable_buffer_array buffers) +{ + co_return co_await d->do_read_some(buffers); +} + +/// Write through a driver an operation keeps alive. +template +capy::io_task +driver_write_some( + driver_ref d, + capy::detail::const_buffer_array buffers) +{ + co_return co_await d->do_write_some(buffers); +} + +/// Handshake through a driver an operation keeps alive. +template +capy::io_task<> +driver_handshake(driver_ref d, tls_role role) +{ + co_return co_await d->do_handshake(role); +} + +/// Shut down through a driver an operation keeps alive. +template +capy::io_task<> +driver_shutdown(driver_ref d) +{ + co_return co_await d->do_shutdown(); +} + } // namespace detail } // namespace boost::corosio diff --git a/src/openssl/src/openssl_stream.cpp b/src/openssl/src/openssl_stream.cpp index 1dca999be..e317778f9 100644 --- a/src/openssl/src/openssl_stream.cpp +++ b/src/openssl/src/openssl_stream.cpp @@ -32,27 +32,40 @@ struct openssl_stream::implementation openssl_stream::implementation* openssl_stream::make_implementation( - capy::any_stream& stream, tls_context const& ctx) + capy::any_stream stream, + detail::tls_owned_transport owned, + tls_context const& ctx) { // Session creation is deferred to handshake time (the engine's // prepare hook builds it lazily), so a session setup failure // surfaces through the handshake completion instead of leaving a // null implementation behind. - return new implementation(stream, ctx); + return new implementation(std::move(stream), owned, ctx); } +namespace { + +template +void +release_driver(Impl* impl) noexcept +{ + if (!impl) + return; + impl->orphan(); + if (impl->release_ref()) + delete impl; +} + +} // namespace + openssl_stream::~openssl_stream() { - delete impl_; + release_driver(impl_); } openssl_stream::openssl_stream(openssl_stream&& other) noexcept - : stream_(std::move(other.stream_)) - , impl_(other.impl_) + : impl_(std::exchange(other.impl_, nullptr)) { - other.impl_ = nullptr; - if (impl_) - impl_->rebind_stream(stream_); } openssl_stream& @@ -60,40 +73,58 @@ openssl_stream::operator=(openssl_stream&& other) noexcept { if (this != &other) { - delete impl_; - stream_ = std::move(other.stream_); - impl_ = other.impl_; - other.impl_ = nullptr; - if (impl_) - impl_->rebind_stream(stream_); + release_driver(impl_); + impl_ = std::exchange(other.impl_, nullptr); } return *this; } +capy::any_stream& +openssl_stream::next_layer() noexcept +{ + return impl_->stream(); +} + +capy::any_stream const& +openssl_stream::next_layer() const noexcept +{ + return impl_->stream(); +} + capy::io_task openssl_stream::do_read_some( capy::detail::mutable_buffer_array buffers) { - co_return co_await impl_->do_read_some(buffers); + return detail::driver_read_some( + detail::driver_ref(impl_), buffers); } capy::io_task openssl_stream::do_write_some( capy::detail::const_buffer_array buffers) { - co_return co_await impl_->do_write_some(buffers); + return detail::driver_write_some( + detail::driver_ref(impl_), buffers); } capy::io_task<> openssl_stream::handshake(tls_role role) { - co_return co_await impl_->do_handshake(role); + return detail::driver_handshake( + detail::driver_ref(impl_), role); } capy::io_task<> openssl_stream::shutdown() { - co_return co_await impl_->do_shutdown(); + return detail::driver_shutdown(detail::driver_ref(impl_)); +} + +void +openssl_stream::cancel() noexcept +{ + if (impl_) + impl_->cancel_transport(); } void diff --git a/src/wolfssl/src/detail/engine.cpp b/src/wolfssl/src/detail/engine.cpp index 4b78f328b..ead492f7a 100644 --- a/src/wolfssl/src/detail/engine.cpp +++ b/src/wolfssl/src/detail/engine.cpp @@ -925,6 +925,7 @@ engine::perform(engine_op op, void* data, std::size_t len) ret = wolfSSL_write(ssl_, data, static_cast(len)); break; case engine_op::shutdown: + { // Both close_notifies already exchanged: the bidirectional close // is complete. Re-calling wolfSSL_shutdown on some builds // re-initiates the close (clearing RECEIVED_SHUTDOWN and re-queuing @@ -944,9 +945,46 @@ engine::perform(engine_op op, void* data, std::size_t len) : engine_want::done, {}, 0}; + + bool const resumed = + (wolfSSL_get_shutdown(ssl_) & WOLFSSL_SENT_SHUTDOWN) != 0; ret = wolfSSL_shutdown(ssl_); + // wolfSSL_shutdown returns after each non-alert record (a session + // ticket, application data), as SHUTDOWN_NOT_DONE or WANT_READ, + // even when the peer's close_notify is already staged behind it. + // Keep feeding it what is staged; parking on the transport would + // wait for bytes that have already arrived. + while (resumed && + (ret == WOLFSSL_SHUTDOWN_NOT_DONE || + (ret != WOLFSSL_SUCCESS && + wolfSSL_get_error(ssl_, ret) == WOLFSSL_ERROR_WANT_READ))) + { + // wolfSSL will not look past application data nobody has + // read. Like OpenSSL, drop it and fail the shutdown rather + // than hand it to a later read. + if (wolfSSL_pending(ssl_) > 0) + { + char sink[256]; + while (wolfSSL_pending(ssl_) > 0 && + wolfSSL_read(ssl_, sink, sizeof(sink)) > 0) + { + } + return { + pending_output() > 0 ? engine_want::output_then_done + : engine_want::done, + std::error_code(APP_DATA_READY, wolfssl_category()), 0}; + } + + auto const staged = in_len_ - in_pos_; + if (staged == 0) + break; + ret = wolfSSL_shutdown(ssl_); + if (in_len_ - in_pos_ == staged && wolfSSL_pending(ssl_) == 0) + break; + } break; } + } // Some wolfSSL builds clear the shutdown bitmask on a read that // follows a completed close, so latch the peer's close_notify the diff --git a/src/wolfssl/src/wolfssl_stream.cpp b/src/wolfssl/src/wolfssl_stream.cpp index 6cf25263e..a84659e9e 100644 --- a/src/wolfssl/src/wolfssl_stream.cpp +++ b/src/wolfssl/src/wolfssl_stream.cpp @@ -32,26 +32,39 @@ struct wolfssl_stream::implementation wolfssl_stream::implementation* wolfssl_stream::make_implementation( - capy::any_stream& stream, tls_context const& ctx) + capy::any_stream stream, + detail::tls_owned_transport owned, + tls_context const& ctx) { // Session creation is deferred to handshake time when the role is // known (the engine's prepare hook builds it from the role's // cached native context). - return new implementation(stream, ctx); + return new implementation(std::move(stream), owned, ctx); } +namespace { + +template +void +release_driver(Impl* impl) noexcept +{ + if (!impl) + return; + impl->orphan(); + if (impl->release_ref()) + delete impl; +} + +} // namespace + wolfssl_stream::~wolfssl_stream() { - delete impl_; + release_driver(impl_); } wolfssl_stream::wolfssl_stream(wolfssl_stream&& other) noexcept - : stream_(std::move(other.stream_)) - , impl_(other.impl_) + : impl_(std::exchange(other.impl_, nullptr)) { - other.impl_ = nullptr; - if (impl_) - impl_->rebind_stream(stream_); } wolfssl_stream& @@ -59,40 +72,58 @@ wolfssl_stream::operator=(wolfssl_stream&& other) noexcept { if (this != &other) { - delete impl_; - stream_ = std::move(other.stream_); - impl_ = other.impl_; - other.impl_ = nullptr; - if (impl_) - impl_->rebind_stream(stream_); + release_driver(impl_); + impl_ = std::exchange(other.impl_, nullptr); } return *this; } +capy::any_stream& +wolfssl_stream::next_layer() noexcept +{ + return impl_->stream(); +} + +capy::any_stream const& +wolfssl_stream::next_layer() const noexcept +{ + return impl_->stream(); +} + capy::io_task wolfssl_stream::do_read_some( capy::detail::mutable_buffer_array buffers) { - co_return co_await impl_->do_read_some(buffers); + return detail::driver_read_some( + detail::driver_ref(impl_), buffers); } capy::io_task wolfssl_stream::do_write_some( capy::detail::const_buffer_array buffers) { - co_return co_await impl_->do_write_some(buffers); + return detail::driver_write_some( + detail::driver_ref(impl_), buffers); } capy::io_task<> wolfssl_stream::handshake(tls_role role) { - co_return co_await impl_->do_handshake(role); + return detail::driver_handshake( + detail::driver_ref(impl_), role); } capy::io_task<> wolfssl_stream::shutdown() { - co_return co_await impl_->do_shutdown(); + return detail::driver_shutdown(detail::driver_ref(impl_)); +} + +void +wolfssl_stream::cancel() noexcept +{ + if (impl_) + impl_->cancel_transport(); } void diff --git a/test/unit/delay.cpp b/test/unit/delay.cpp index 657e04a56..4a38309f1 100644 --- a/test/unit/delay.cpp +++ b/test/unit/delay.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -24,6 +25,7 @@ #include #include #include +#include #include #include @@ -110,6 +112,34 @@ static_assert(std::same_as< decltype(delay(test_clock::time_point{})), clock_delay_awaitable>); +struct count_on_exit +{ + int& c_; + ~count_on_exit() + { + ++c_; + } +}; + +// Namespace scope: GCC 11 miscompiles a co_await in a static member +// coroutine of a class template. +static capy::task> +parked_child(int& down) +{ + count_on_exit g{down}; + co_return co_await delay(std::chrono::hours(1)); +} + +static capy::task> +queued_child(io_context& ioc, int& down) +{ + count_on_exit g{down}; + detail::timer t(ioc); + t.expires_after(std::chrono::nanoseconds(0)); + auto [ec] = co_await t.wait(); + co_return capy::io_result<>{ec}; +} + template struct delay_test { @@ -637,6 +667,60 @@ struct delay_test BOOST_TEST_PASS(); } + // Teardown destroys a frame a parent task is awaiting; the whole + // chain must go, each frame once. + void testTeardownNestedTaskOnTimer() + { + int child = 0, parent = 0; + { + io_context ioc(Backend); + capy::run_async(ioc.get_executor())( + [](int& child, int& parent) -> capy::task<> { + count_on_exit g{parent}; + std::ignore = co_await parked_child(child); + }(child, parent)); + ioc.poll(); + } + BOOST_TEST_EQ(child, 1); + BOOST_TEST_EQ(parent, 1); + } + + // The same with timeout() owning the awaited task. + void testTeardownTimeoutOfTask() + { + int child = 0, parent = 0; + { + io_context ioc(Backend); + capy::run_async(ioc.get_executor())( + [](int& child, int& parent) -> capy::task<> { + count_on_exit g{parent}; + std::ignore = co_await timeout( + parked_child(child), std::chrono::hours(2)); + }(child, parent)); + ioc.poll(); + } + BOOST_TEST_EQ(child, 1); + BOOST_TEST_EQ(parent, 1); + } + + // The same when the child's completion is already queued, so the + // scheduler's drain destroys it rather than the timer service. + void testTeardownQueuedNestedContinuation() + { + int child = 0, parent = 0; + { + io_context ioc(Backend); + capy::run_async(ioc.get_executor())( + [](io_context& ioc, int& child, int& parent) -> capy::task<> { + count_on_exit g{parent}; + std::ignore = co_await queued_child(ioc, child); + }(ioc, child, parent)); + std::ignore = ioc.poll_one(); + } + BOOST_TEST_EQ(child, 1); + BOOST_TEST_EQ(parent, 1); + } + void testShutdownReentrantThreeFrames() { // Three suspended delay()/timeout() frames each own a timer that @@ -956,9 +1040,94 @@ struct delay_test } } + // Context teardown destroys a frame parked on a timer. Work its + // destructors post must still reach the scheduler's drain, which + // destroys it, rather than be queued after the drain and lost. + struct count_on_destroy + { + int* n; + explicit count_on_destroy(int* p) noexcept : n(p) {} + count_on_destroy(count_on_destroy&& o) noexcept + : n(std::exchange(o.n, nullptr)) + { + } + ~count_on_destroy() + { + if (n) + ++*n; + } + }; + + static capy::task<> never_started(count_on_destroy) + { + co_return; + } + + struct post_on_destroy + { + io_context& ioc; + int* destroyed; + ~post_on_destroy() + { + capy::run_async(ioc.get_executor())( + never_started(count_on_destroy(destroyed))); + } + }; + + void testTeardownDrainsWorkPostedByTimerFrames() + { + int destroyed = 0; + { + io_context ioc(Backend); + capy::run_async(ioc.get_executor())( + [](io_context& ioc, int* d) -> capy::task<> { + post_on_destroy guard{ioc, d}; + std::ignore = + co_await corosio::delay(std::chrono::hours(1)); + }(ioc, &destroyed)); + ioc.poll(); // parks on the timer + } + BOOST_TEST_EQ(destroyed, 1); + } + + // A stop requested on another thread can reach a parked delay while + // the context tears its timers down; the two must not both claim + // the waiter. + void testStopRacingTeardown() + { + for (int round = 0; round < 200; ++round) + { + std::vector stops(8); + std::atomic go{false}; + std::thread stopper; + { + io_context ioc(Backend); + for (auto& src : stops) + capy::run_async(ioc.get_executor(), src.get_token())( + []() -> capy::task<> { + std::ignore = + co_await corosio::delay(std::chrono::hours(1)); + }()); + std::ignore = ioc.poll(); // parks every delay + stopper = std::thread([&] { + while (!go.load(std::memory_order_acquire)) + { + } + for (auto& src : stops) + src.request_stop(); + }); + go.store(true, std::memory_order_release); + } + stopper.join(); + } + BOOST_TEST_PASS(); + } + void run() { testDurationCompletes(); + testTeardownDrainsWorkPostedByTimerFrames(); + testStopRacingTeardown(); testTimePointCompletes(); testZeroDurationCompletesImmediately(); testPastTimePointCompletesImmediately(); @@ -982,6 +1151,9 @@ struct delay_test testMultiTimerExpiryOrder(); testAbruptStopWithPendingDelays(); testShutdownReentrantThreeFrames(); + testTeardownNestedTaskOnTimer(); + testTeardownTimeoutOfTask(); + testTeardownQueuedNestedContinuation(); testInitiationStopRace(); testClockDeadlineCompletes(); testClockPastDeadlineCompletesImmediately(); diff --git a/test/unit/fault/fault.hpp b/test/unit/fault/fault.hpp index de39b37c8..27b063add 100644 --- a/test/unit/fault/fault.hpp +++ b/test/unit/fault/fault.hpp @@ -429,6 +429,33 @@ class preadv_hold preadv_hold& operator=(preadv_hold const&) = delete; }; +/** Hold the first `pwritev` on any thread, before the real call. + + For a test that must act while a pool worker has committed to a + transfer but not yet reached the kernel. The held call proceeds + once the hold is released or destroyed. + + @par Preconditions + No other `pwritev_hold` is alive. POSIX only. +*/ +class pwritev_hold +{ +public: + pwritev_hold(); + + /// Release the held call, if any. + ~pwritev_hold(); + + /// Block until a `pwritev` is being held. + void wait_held(); + + /// Let the held call proceed. + void release(); + + pwritev_hold(pwritev_hold const&) = delete; + pwritev_hold& operator=(pwritev_hold const&) = delete; +}; + } // namespace boost::corosio::test::fault #endif diff --git a/test/unit/fault/fault_posix.cpp b/test/unit/fault/fault_posix.cpp index 7b6d7dca9..51bb2b14e 100644 --- a/test/unit/fault/fault_posix.cpp +++ b/test/unit/fault/fault_posix.cpp @@ -220,65 +220,127 @@ cqe_fault_scope::fired() const noexcept namespace { -// Static, not owned by the hold: a held call can still be leaving it +// Static, not owned by a hold: a held call can still be leaving it // when the hold is destroyed. -struct preadv_hold_state +struct hold_state { std::mutex m; std::condition_variable cv; bool armed = false; bool held = false; bool released = false; -} g_preadv_hold; +}; -} // namespace +hold_state g_preadv_hold; +hold_state g_pwritev_hold; -preadv_hold::preadv_hold() +void +arm_hold(hold_state& h, char const* name) { - std::lock_guard lock(g_preadv_hold.m); - if (g_preadv_hold.armed) + std::lock_guard lock(h.m); + if (h.armed) { - std::fputs("preadv_hold: a hold is already alive\n", stderr); + std::fprintf(stderr, "%s: a hold is already alive\n", name); std::abort(); } - g_preadv_hold.armed = true; - g_preadv_hold.held = false; - g_preadv_hold.released = false; + h.armed = true; + h.held = false; + h.released = false; +} + +void +disarm_hold(hold_state& h) +{ + std::lock_guard lock(h.m); + h.armed = false; + h.released = true; + h.cv.notify_all(); +} + +void +wait_hold(hold_state& h) +{ + std::unique_lock lock(h.m); + h.cv.wait(lock, [&] { return h.held; }); +} + +void +release_hold(hold_state& h) +{ + std::lock_guard lock(h.m); + h.released = true; + h.cv.notify_all(); +} + +void +hold_if_armed(hold_state& h) noexcept +{ + std::unique_lock lock(h.m); + if (!h.armed || h.held) + return; + h.held = true; + h.cv.notify_all(); + h.cv.wait(lock, [&] { return h.released; }); +} + +} // namespace + +preadv_hold::preadv_hold() +{ + arm_hold(g_preadv_hold, "preadv_hold"); } preadv_hold::~preadv_hold() { - std::lock_guard lock(g_preadv_hold.m); - g_preadv_hold.armed = false; - g_preadv_hold.released = true; - g_preadv_hold.cv.notify_all(); + disarm_hold(g_preadv_hold); } void preadv_hold::wait_held() { - std::unique_lock lock(g_preadv_hold.m); - g_preadv_hold.cv.wait(lock, [] { return g_preadv_hold.held; }); + wait_hold(g_preadv_hold); } void preadv_hold::release() { - std::lock_guard lock(g_preadv_hold.m); - g_preadv_hold.released = true; - g_preadv_hold.cv.notify_all(); + release_hold(g_preadv_hold); +} + +pwritev_hold::pwritev_hold() +{ + arm_hold(g_pwritev_hold, "pwritev_hold"); +} + +pwritev_hold::~pwritev_hold() +{ + disarm_hold(g_pwritev_hold); +} + +void +pwritev_hold::wait_held() +{ + wait_hold(g_pwritev_hold); +} + +void +pwritev_hold::release() +{ + release_hold(g_pwritev_hold); } // Called by the preadv shadow once the real call has returned. void hold_preadv_if_armed() noexcept { - std::unique_lock lock(g_preadv_hold.m); - if (!g_preadv_hold.armed || g_preadv_hold.held) - return; - g_preadv_hold.held = true; - g_preadv_hold.cv.notify_all(); - g_preadv_hold.cv.wait(lock, [] { return g_preadv_hold.released; }); + hold_if_armed(g_preadv_hold); +} + +// Called by the pwritev shadow before the real call. +void +hold_pwritev_if_armed() noexcept +{ + hold_if_armed(g_pwritev_hold); } } // namespace boost::corosio::test::fault @@ -914,6 +976,7 @@ extern "C" ssize_t pwritev(int fd, iovec const* v, int n, off_t o) { COROSIO_FAULT_REAL(pwritev, ssize_t (*)(int, iovec const*, int, off_t)); + hold_pwritev_if_armed(); if (should_fail(sys::pwritev)) return -1; std::size_t c; @@ -955,6 +1018,7 @@ extern "C" ssize_t pwritev64(int fd, iovec const* v, int n, off64_t o) { COROSIO_FAULT_REAL(pwritev64, ssize_t (*)(int, iovec const*, int, off64_t)); + hold_pwritev_if_armed(); if (should_fail(sys::pwritev)) return -1; std::size_t c; diff --git a/test/unit/fault/posix_faults.cpp b/test/unit/fault/posix_faults.cpp index 19461a39d..3fe53d5b5 100644 --- a/test/unit/fault/posix_faults.cpp +++ b/test/unit/fault/posix_faults.cpp @@ -28,9 +28,12 @@ #include #include +#include #include #include #include +#include +#include #include #include @@ -85,6 +88,56 @@ struct posix_common_faults BOOST_TEST(!s.is_open()); } + // A connect refused outright (an exhausted ephemeral range) must + // fail the pair rather than leave it waiting on an accept that can + // no longer arrive. io_uring connects through the ring, out of the + // hook's reach. + void testSocketPairConnectFails() + { +#if BOOST_COROSIO_HAS_URING + if constexpr (std::is_same_v< + std::remove_cvref_t, uring_t>) + return; +#endif + io_context ioc(Backend); + fault_scope f(sys::connect, EADDRNOTAVAIL); + bool threw = false; + try + { + std::ignore = + test::make_socket_pair(ioc); + } + catch (std::runtime_error const&) + { + threw = true; + } + BOOST_TEST(f.fired()); + BOOST_TEST(threw); + } + + // Same for the mocket pair, built the same way over a socket pair. + void testMocketPairConnectFails() + { +#if BOOST_COROSIO_HAS_URING + if constexpr (std::is_same_v< + std::remove_cvref_t, uring_t>) + return; +#endif + io_context ioc(Backend); + fault_scope f(sys::connect, EADDRNOTAVAIL); + bool threw = false; + try + { + std::ignore = test::make_mocket_pair(ioc); + } + catch (std::runtime_error const&) + { + threw = true; + } + BOOST_TEST(f.fired()); + BOOST_TEST(threw); + } + void testBindFails() { io_context ioc(Backend); @@ -673,12 +726,71 @@ struct posix_common_faults ::unlink(p2.c_str()); } + // A pool worker that has committed to a write must not write + // through the descriptor number after close() and an unrelated + // open() have given that number to another file. + template + void closeDuringPoolWrite(char const* tag) + { + if constexpr (ring_files) + return; + if (!hook_is_live(sys::pwritev)) + { + skip_dead_hook("pwritev"); + return; + } + auto p1 = temp_path((std::string(tag) + "_a").c_str()); + auto p2 = temp_path((std::string(tag) + "_b").c_str()); + std::ofstream(p1).flush(); + std::ofstream(p2).flush(); + + io_context ioc(Backend); + File f(ioc); + BOOST_TEST(!f.open(p1, file_base::write_only)); + + pwritev_hold hold; + std::string const payload(4096, 'a'); + int other = -1; + auto writer = [&]() -> capy::task<> { + if constexpr (std::is_same_v) + std::ignore = co_await f.write_some( + capy::const_buffer(payload.data(), payload.size())); + else + std::ignore = co_await f.write_some_at( + 0, capy::const_buffer(payload.data(), payload.size())); + }; + auto swapper = [&]() -> capy::task<> { + hold.wait_held(); + f.close(); + other = ::open(p2.c_str(), O_WRONLY | O_CLOEXEC); + BOOST_TEST(other >= 0); + hold.release(); + co_return; + }; + capy::run_async(ioc.get_executor())(writer()); + capy::run_async(ioc.get_executor())(swapper()); + ioc.run(); + + ::close(other); + BOOST_TEST_EQ(std::filesystem::file_size(p2), 0u); + ::unlink(p1.c_str()); + ::unlink(p2.c_str()); + } + + void testCloseDuringPoolWriteSparesReusedDescriptor() + { + closeDuringPoolWrite("sf_reuse"); + closeDuringPoolWrite("raf_reuse"); + } + void run() { if (skip_under_valgrind()) return; testSocketOpenFails(); testBindFails(); + testSocketPairConnectFails(); + testMocketPairConnectFails(); testSetOptionFails(); testGetOptionFails(); testAssignValidateFails(); @@ -693,6 +805,7 @@ struct posix_common_faults testStreamFileSyncOps(); testStreamFileIoFails(); testAssignDuringFinishedReadKeepsNewOffset(); + testCloseDuringPoolWriteSparesReusedDescriptor(); testRandomAccessFileFails(); testResolverFails(); testSignalTeardownWalk(); diff --git a/test/unit/local_stream_socket.cpp b/test/unit/local_stream_socket.cpp index a79490fba..3bfcae4de 100644 --- a/test/unit/local_stream_socket.cpp +++ b/test/unit/local_stream_socket.cpp @@ -32,6 +32,7 @@ #if BOOST_COROSIO_POSIX #include +#include #include #include #include @@ -127,6 +128,60 @@ struct local_stream_socket_test acc.close(); } +#if BOOST_COROSIO_POSIX + // An accept that completed but whose awaiter is destroyed before it + // resumes must not strand the accepted connection: the awaiter's + // context queues the completion, and its teardown destroys the + // frame without resuming it. + void testAcceptedPeerClosedWhenAwaiterDestroyed(bool by_value) + { + io_context a(Backend); + test::temp_socket_dir tmp; + local_stream_acceptor acc(a, local_endpoint(tmp.path())); + local_stream_socket peer(a); + + int client = ::socket(AF_UNIX, SOCK_STREAM, 0); + BOOST_TEST(client >= 0); + { + io_context b(Backend); + if (by_value) + capy::run_async(b.get_executor())( + [](local_stream_acceptor& acc) -> capy::task<> { + std::ignore = co_await acc.accept(); + }(acc)); + else + capy::run_async(b.get_executor())( + [](local_stream_acceptor& acc, + local_stream_socket& p) -> capy::task<> { + std::ignore = co_await acc.accept(p); + }(acc, peer)); + std::ignore = b.poll(); // parks the accept on a's acceptor + + sockaddr_un addr{}; + addr.sun_family = AF_UNIX; + std::strncpy( + addr.sun_path, std::string(tmp.path()).c_str(), + sizeof(addr.sun_path) - 1); + BOOST_TEST( + ::connect( + client, reinterpret_cast(&addr), + sizeof(addr)) == 0); + a.run(); // completes the accept into b's queue + } + + // With the accepted socket closed, the client reads end of file; + // a stranded one never becomes readable. Some stacks deliver the + // close asynchronously, so readability is awaited, bounded only + // to fail rather than hang. + pollfd pfd{client, POLLIN, 0}; + BOOST_TEST_EQ(::poll(&pfd, 1, 5000), 1); + ::fcntl(client, F_SETFL, ::fcntl(client, F_GETFL) | O_NONBLOCK); + char c; + BOOST_TEST_EQ(::recv(client, &c, 1, 0), 0); + ::close(client); + } +#endif + void testConnectAccept() { io_context ioc(Backend); @@ -1913,6 +1968,10 @@ struct local_stream_socket_test void run() { +#if BOOST_COROSIO_POSIX + testAcceptedPeerClosedWhenAwaiterDestroyed(false); + testAcceptedPeerClosedWhenAwaiterDestroyed(true); +#endif testAssignOnOpenIsAlreadyOpen(); testAcceptorAssignOnOpenIsAlreadyOpen(); testAcceptorAssignSelfAndWrongType(); diff --git a/test/unit/native/uring/descriptor_continue.cpp b/test/unit/native/uring/descriptor_continue.cpp index 092b7f175..4c4d35504 100644 --- a/test/unit/native/uring/descriptor_continue.cpp +++ b/test/unit/native/uring/descriptor_continue.cpp @@ -31,6 +31,7 @@ #include #include +#include #include #include @@ -280,9 +281,12 @@ struct uring_descriptor_continue_test std::size_t bytes = 0; detail::uring_descriptor_read_op op; + // The continuation an awaitable would own, in the awaiting frame. + capy::continuation cont; + cont.h = coro.h; op.prepare( - coro.h, ex, &ec, &bytes, fds[0], /*file_offset=*/-1, - &ctx.scheduler(), detail::object_ref(d.get()), mb, std::stop_token{}); + cont, ex, &ec, &bytes, fds[0], /*file_offset=*/-1, &ctx.scheduler(), + detail::object_ref(d.get()), mb, std::stop_token{}); arm(*d, op); // Data is already waiting, so the poll this arms fires at once. diff --git a/test/unit/object_pool.cpp b/test/unit/object_pool.cpp index 11b173e29..8bf93ca36 100644 --- a/test/unit/object_pool.cpp +++ b/test/unit/object_pool.cpp @@ -89,6 +89,7 @@ struct object_pool_test a->pool = &pool; release(a); // parked in free_ BOOST_TEST(!try_acquire(a)); + BOOST_TEST(!object_ref::try_from(a)); auto* b = pool.acquire(); // still recyclable intact BOOST_TEST_EQ(b, a); BOOST_TEST_EQ(b->refs_.load(), 1u); diff --git a/test/unit/object_ref.cpp b/test/unit/object_ref.cpp index f99d7069e..fa5ae0e3c 100644 --- a/test/unit/object_ref.cpp +++ b/test/unit/object_ref.cpp @@ -42,6 +42,22 @@ struct object_ref_test release(&impl); BOOST_TEST_EQ(hits, 1); } + // try_from adds a reference to a live impl and refuses a + // retiring one + { + int hits = 0; + counted_impl impl(&hits); + { + auto r = object_ref::try_from(&impl); // 1 -> 2 + BOOST_TEST_EQ(r.get(), &impl); + release(&impl); // 2 -> 1 + BOOST_TEST_EQ(hits, 0); + } // 1 -> 0 + BOOST_TEST_EQ(hits, 1); + auto dead = object_ref::try_from(&impl); + BOOST_TEST(!dead); + BOOST_TEST_EQ(hits, 1); + } // reset releases early { int hits = 0; diff --git a/test/unit/object_reuse.cpp b/test/unit/object_reuse.cpp index 204078678..8b86c6c8a 100644 --- a/test/unit/object_reuse.cpp +++ b/test/unit/object_reuse.cpp @@ -38,12 +38,14 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -257,11 +259,76 @@ struct object_reuse_test BOOST_TEST(server2.is_open()); } + // A completion resumed through a strand is queued there by + // reference. Destroying the socket recycles its impl while that + // entry waits, and a new socket may reuse the impl and complete an + // operation of its own before the strand reaches the entry. The + // original reader must still be resumed, once, with its own result. + void testStrandCompletionOutlivesRecycle() + { + io_context ioc(Backend); + capy::strand st(ioc.get_executor()); + auto [s1, s2] = + test::make_socket_pair(ioc); + auto sock = std::make_unique(std::move(s1)); + auto [q1, q2] = + test::make_socket_pair(ioc); + ioc.restart(); + + int resumed = 0; + std::error_code rec; + char buf[1]; + capy::run_async(st)( + [](tcp_socket& s, char* b, int& n, + std::error_code& ec) -> capy::task<> { + auto [e, k] = co_await s.read_some(capy::mutable_buffer(b, 1)); + std::ignore = k; + ec = e; + ++n; + }(*sock, buf, resumed, rec)); + + std::optional fresh; + bool reused_done = false; + char pbuf[1]; + auto reuse = [](io_context& ioc, std::optional& f, + native_handle_type h, char* pb, + bool& done) -> capy::task<> { + f.emplace(ioc); + BOOST_TEST(!f->assign(h)); + auto [e, k] = co_await f->read_some(capy::mutable_buffer(pb, 1)); + BOOST_TEST(!e); + BOOST_TEST_EQ(k, 1u); + done = true; + }; + + // Destroying the socket queues the reader's completion ahead of + // the strand's turn, so the strand holds the reuse first and the + // reader behind it. The byte written first lets the reuse's read + // complete without waiting. + capy::run_async(ioc.get_executor())( + [](capy::strand st, + std::unique_ptr& sock, tcp_socket& peer, + capy::task<> reuse) -> capy::task<> { + char c = 'x'; + auto [e, k] = co_await peer.write_some(capy::const_buffer(&c, 1)); + BOOST_TEST(!e); + BOOST_TEST_EQ(k, 1u); + sock.reset(); + capy::run_async(st)(std::move(reuse)); + }(st, sock, q2, reuse(ioc, fresh, q1.release(), pbuf, reused_done))); + + ioc.run(); + BOOST_TEST_EQ(resumed, 1); + BOOST_TEST(rec == capy::cond::canceled); + BOOST_TEST(reused_done); + } + void run() { testAbandonedReadThenReuse(); testConnectThenDestroyRecycles(); testAcceptorRecycleDrainsBufferedConnection(); + testStrandCompletionOutlivesRecycle(); } }; diff --git a/test/unit/openssl_stream.cpp b/test/unit/openssl_stream.cpp index 7dbbfe900..e4e17fded 100644 --- a/test/unit/openssl_stream.cpp +++ b/test/unit/openssl_stream.cpp @@ -11,6 +11,9 @@ // Test that header file is self-contained. #include +#ifdef BOOST_COROSIO_HAS_WOLFSSL +#include +#endif #include @@ -464,6 +467,11 @@ struct openssl_stream_test void run() { test::testIoBeforeHandshake(make_stream); + test::testDestroyWithPendingRead(make_stream, false); + test::testDestroyWithPendingRead(make_stream, true); + test::testMoveWithPendingRead(make_stream); + test::testDestroyWithPartialWriteInFlight(make_stream); + test::testDestroyTypeErasedOwnerWithPendingRead(make_stream); test::testHandshakeFuse(make_stream); test::testReadWriteFuse(make_stream); test::testShutdownFuse(make_stream); @@ -485,6 +493,14 @@ struct openssl_stream_test test::testRecordBoundaryTransfer(make_stream); test::testShutdownOverRead(make_stream); test::testShutdownSimultaneousClose(make_stream); + test::testShutdownWithUnreadRecordAhead(make_stream); + test::testUnawaitedReadOutlivesContext(make_stream); + test::testUnawaitedReadOutlivesUnwoundOwner(make_stream); + test::testCancelPendingRead(make_stream, true); + test::testCancelPendingRead(make_stream, false); +#ifdef BOOST_COROSIO_HAS_WOLFSSL + test::testDestroyNestedOwnerWithPendingRead(); +#endif test::testPartialReadWithError(make_stream); test::testCancelParkedReader(make_stream); test::testFullDuplexMtStrand(make_stream); diff --git a/test/unit/random_access_file.cpp b/test/unit/random_access_file.cpp index 18c56f87e..3c9ba090a 100644 --- a/test/unit/random_access_file.cpp +++ b/test/unit/random_access_file.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include "context.hpp" @@ -33,16 +34,19 @@ #include #include +#include #include #include #include #include #include #include +#include #include #include #include #include +#include #include #include "temp_path.hpp" @@ -992,8 +996,101 @@ struct random_access_file_test #endif } + // A read awaited from a non-io_context executor takes the deferring + // branch. The queued continuation must survive the next read reusing + // the recycled op. + void testDeferredExecutorCompletion() + { + std::string data = "ABCDEFGHIJ"; + temp_file tmp("raf_deferred_", data); + io_context ioc(Backend); + capy::thread_pool pool(1); + random_access_file f(ioc); + BOOST_TEST(!f.open(tmp.path, file_base::read_only)); + + std::latch started(1); + std::latch hold(1); + std::atomic c1_done{false}; + std::atomic c2_done{false}; + char b1[2] = {}; + char b2[2] = {}; + + auto reader = [](random_access_file& f, std::uint64_t at, char* b, + std::atomic& done) -> capy::task<> { + auto [ec, n] = + co_await f.read_some_at(at, capy::mutable_buffer(b, 2)); + done = !ec && n == 2; + }; + + // Awaited on the pool, so the completion takes the deferring path. + capy::run_async(pool.get_executor())(reader(f, 0, b1, c1_done)); + + // Parked behind the first read on the single worker: once this + // runs, that read has been issued and its coroutine suspended. + capy::run_async(pool.get_executor())( + [](std::latch& s, std::latch& h) -> capy::task<> { + s.count_down(); + h.wait(); + co_return; + }(started, hold)); + started.wait(); + + // The completion queues the first read's resume into the parked + // pool and recycles the op. + ioc.run(); + ioc.restart(); + + // Reuses the op while the pool still holds the first continuation. + capy::run_async(ioc.get_executor())(reader(f, 4, b2, c2_done)); + ioc.run(); + ioc.restart(); + BOOST_TEST(c2_done.load()); + + hold.count_down(); + std::latch drained(1); + capy::run_async(pool.get_executor())( + [](std::latch& d) -> capy::task<> { + d.count_down(); + co_return; + }(drained)); + drained.wait(); + + BOOST_TEST(c1_done.load()); + BOOST_TEST(std::memcmp(b1, "AB", 2) == 0); + BOOST_TEST(std::memcmp(b2, "EF", 2) == 0); + pool.join(); + } + +#if BOOST_COROSIO_POSIX + // Adopting a descriptor is refused in the unsafe tier for the + // same reason opening is, and the caller keeps the descriptor. + void testAssignSingleThreadedNotSupported() + { +#if BOOST_COROSIO_HAS_URING + if constexpr ( + std::is_same_v, uring_t>) + return; +#endif + temp_file tmp("raf_assign_st_", "data"); + io_context_options opts; + opts.locking = locking_mode::unsafe; + io_context ioc(Backend, opts, 1); + random_access_file f(ioc); + + int const fd = ::open(tmp.path.c_str(), O_RDONLY | O_CLOEXEC); + BOOST_TEST(fd >= 0); + BOOST_TEST(f.assign(fd) == std::errc::operation_not_supported); + BOOST_TEST(!f.is_open()); + BOOST_TEST(::fcntl(fd, F_GETFD) >= 0); + ::close(fd); + } +#endif + void run() { +#if BOOST_COROSIO_POSIX + testAssignSingleThreadedNotSupported(); +#endif testAssignOnOpenIsAlreadyOpen(); testResumesOnAwaitingExecutor(); testConstruction(); @@ -1007,6 +1104,7 @@ struct random_access_file_test testResize(); testReadSomeAt(); + testDeferredExecutorCompletion(); testReadSomeAtBeginning(); testReadSomeAtEOF(); diff --git a/test/unit/signal_set.cpp b/test/unit/signal_set.cpp index 32bdd1441..173d49132 100644 --- a/test/unit/signal_set.cpp +++ b/test/unit/signal_set.cpp @@ -15,12 +15,17 @@ #include #include +#include #include #include #include +#include +#include #include #include +#include +#include #include #include @@ -471,6 +476,50 @@ struct signal_set_test BOOST_TEST(!second_ec); } + // With two threads running the context, a signal raised while one + // is blocked in the reactor must still be delivered after the other + // re-arms the signal pipe. + void testDeliveryWithTwoRunThreads() + { + io_context ioc(Backend, 2); + signal_set s(ioc, SIGINT); + constexpr int n = 200; + std::atomic got{0}; + + capy::run_async(ioc.get_executor())( + [](signal_set& s, std::atomic& got) -> capy::task<> { + for (int i = 0; i < n; ++i) + { + auto [ec, signum] = co_await s.wait(); + std::ignore = signum; + if (ec) + co_return; + ++got; + } + }(s, got)); + + std::thread t1([&] { ioc.run(); }); + std::thread t2([&] { ioc.run(); }); + int delivered = 0; + for (int i = 0; i < n; ++i) + { + std::raise(SIGINT); + // Bounds a hang, not a timing assertion. + auto const until = + std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (got.load() <= i && std::chrono::steady_clock::now() < until) + std::this_thread::yield(); + if (got.load() <= i) + break; + delivered = i + 1; + } + BOOST_TEST_EQ(delivered, n); + s.cancel(); + ioc.stop(); + t1.join(); + t2.join(); + } + void testWaitAgainAfterStopCancel() { // Review Focus 4: the token path must not set the sticky @@ -1406,10 +1455,74 @@ struct signal_set_test #endif // BOOST_COROSIO_POSIX + // A wait that completes from an already-queued signal posts its + // completion. Destroying the set before that completion runs must + // neither lose it nor let it reach a set created afterwards. + void testDestroyWithQueuedSignalCompletion() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + // Keeps the context from running out of work between polls. + std::stop_source keep; + capy::run_async(ex, keep.get_token())([]() -> capy::task<> { + std::ignore = co_await corosio::delay(std::chrono::hours(1)); + }()); + + int resumed = 0; + { + signal_set s(ioc, SIGINT); + std::raise(SIGINT); + ioc.restart(); + std::ignore = ioc.poll(); // record the signal on the set + + capy::run_async(ex)( + [](signal_set& set, int& count) -> capy::task<> { + std::ignore = co_await set.wait(); + ++count; + }(s, resumed)); + // Start the coroutine only: its wait finds the recorded + // signal and queues the completion. + ioc.restart(); + BOOST_TEST_EQ(ioc.poll_one(), 1u); + BOOST_TEST_EQ(resumed, 0); // completion queued, not yet run + } + + // A new set whose wait starts before the queued completion runs + // must not receive it. The wait is started on this thread, ahead + // of that completion in the queue. + int resumed2 = 0; + signal_set s2(ioc, SIGTERM); + auto waiter = [](signal_set& set, int& count) -> capy::task<> { + std::ignore = co_await set.wait(); + ++count; + }; + capy::io_env env{ex, std::stop_token{}, nullptr}; + std::optional> parked; + parked.emplace(waiter(s2, resumed2)); + parked->await_suspend(std::noop_coroutine(), &env).resume(); + + ioc.restart(); + std::ignore = ioc.poll(); + BOOST_TEST_EQ(resumed, 1); + BOOST_TEST_EQ(resumed2, 0); + s2.cancel(); + + // Bounds a hang, not a timing assertion. + keep.request_stop(); + ioc.restart(); + while (ioc.run_one_for(std::chrono::seconds(5)) != 0) + { + } + BOOST_TEST_EQ(resumed2, 1); + } + void run() { // Construction and move semantics + testDeliveryWithTwoRunThreads(); testConstruction(); + testDestroyWithQueuedSignalCompletion(); testConstructWithOneSignal(); testConstructWithTwoSignals(); testConstructWithThreeSignals(); @@ -1507,4 +1620,92 @@ struct signal_set_test COROSIO_BACKEND_TESTS(signal_set_test, "boost.corosio.signal_set") +#if BOOST_COROSIO_POSIX && BOOST_COROSIO_HAS_SELECT +namespace { + +// Namespace scope: GCC 11 miscompiles a co_await in a static member +// coroutine of a class template. +capy::task<> +count_signals(signal_set& s, std::atomic& n) +{ + for (;;) + { + auto [ec, sig] = co_await s.wait(); + std::ignore = sig; + if (ec) + co_return; + ++n; + } +} + +} // namespace + +// Two contexts watch the process's signal pipe. Whichever drains a +// signal delivers it to every context's sets, so a drain running under +// one context's pipe lock posts into the other context's scheduler. +// Neither drain may hold a lock the other context's reactor takes +// first, or the two run threads of a select context and a drain on +// another context deadlock. +template +struct signal_set_cross_context_test +{ + static constexpr int rounds = 400; + + void run() + { + io_context a(Backend); + io_context b(select); + signal_set sa(a, SIGINT); + signal_set sb(b, SIGINT); + std::atomic na{0}, nb{0}; + // A stop token, unlike cancel(), also ends a wait the counter + // starts after the request, so run() returns once both finish. + std::stop_source quit; + capy::run_async( + a.get_executor(), quit.get_token())(count_signals(sa, na)); + capy::run_async( + b.get_executor(), quit.get_token())(count_signals(sb, nb)); + + std::thread ra([&] { a.run(); }); + std::thread rb1([&] { b.run(); }); + std::thread rb2([&] { b.run(); }); + + // Turns a deadlock into a failure rather than a hung test. + std::atomic finished{false}; + std::thread watchdog([&] { + for (int i = 0; i < 300 && !finished.load(); ++i) + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + if (!finished.load()) + { + std::fputs( + "signal_set_cross_context: deadlocked\n", stderr); + std::abort(); + } + }); + + for (int i = 0; i < rounds; ++i) + { + std::raise(SIGINT); + // Lets each round's deliveries race the next drain. + while (na.load() <= i / 2 && nb.load() <= i / 2) + std::this_thread::yield(); + } + while (na.load() < rounds / 2 || nb.load() < rounds / 2) + std::this_thread::yield(); + + quit.request_stop(); + ra.join(); + rb1.join(); + rb2.join(); + finished = true; + watchdog.join(); + BOOST_TEST_GE(na.load(), rounds / 2); + BOOST_TEST_GE(nb.load(), rounds / 2); + } +}; + +COROSIO_NON_IOCP_BACKEND_TESTS( + signal_set_cross_context_test, "boost.corosio.signal_set.cross_context") +#endif + } // namespace boost::corosio diff --git a/test/unit/socket_stress.cpp b/test/unit/socket_stress.cpp index 687187c20..d182f113f 100644 --- a/test/unit/socket_stress.cpp +++ b/test/unit/socket_stress.cpp @@ -33,6 +33,7 @@ #include #include +#include #include #include #include @@ -659,6 +660,19 @@ struct accept_stress_test continue; } ++connections; + // Reset rather than FIN: the client then never reaches + // TIME_WAIT, which at this rate would drain the ephemeral + // range for every test running alongside. Resetting the + // client instead can kill the connection before it is + // accepted, which some stacks drop from the queue. + try + { + peer.set_option(socket_option::linger(true, 0)); + } + catch (std::system_error const&) + { + // Already reset by the peer: nothing left to linger. + } peer.close(); } }; diff --git a/test/unit/stream_file.cpp b/test/unit/stream_file.cpp index 2ec1cc271..3f1ba8000 100644 --- a/test/unit/stream_file.cpp +++ b/test/unit/stream_file.cpp @@ -513,6 +513,29 @@ struct stream_file_test BOOST_TEST(ec == std::errc::operation_not_supported); } + // Adopting a descriptor is refused in the unsafe tier for the + // same reason opening is, and the caller keeps the descriptor. + void testAssignSingleThreadedNotSupported() + { +#if BOOST_COROSIO_HAS_URING + if constexpr ( + std::is_same_v, uring_t>) + return; +#endif + temp_file tmp("sf_assign_st_", "data"); + io_context_options opts; + opts.locking = locking_mode::unsafe; + io_context ioc(Backend, opts, 1); + stream_file f(ioc); + + int const fd = ::open(tmp.path.c_str(), O_RDONLY | O_CLOEXEC); + BOOST_TEST(fd >= 0); + BOOST_TEST(f.assign(fd) == std::errc::operation_not_supported); + BOOST_TEST(!f.is_open()); + BOOST_TEST(::fcntl(fd, F_GETFD) >= 0); + ::close(fd); + } + void testOpenUnsafeIoStillSupported() { #if BOOST_COROSIO_HAS_URING @@ -1466,6 +1489,7 @@ struct stream_file_test testWriteEmptyBuffer(); #if BOOST_COROSIO_POSIX testOpenSingleThreadedNotSupported(); + testAssignSingleThreadedNotSupported(); testOpenUnsafeIoStillSupported(); #endif testTruncate(); diff --git a/test/unit/tcp_acceptor.cpp b/test/unit/tcp_acceptor.cpp index 43b323392..ddd0ade03 100644 --- a/test/unit/tcp_acceptor.cpp +++ b/test/unit/tcp_acceptor.cpp @@ -21,10 +21,14 @@ #include #include #include +#include #include +#include #include #include +#include +#include #include #include #include @@ -36,6 +40,7 @@ // netinet/in.h and unistd.h support the raw-socket backlog setup in // testAcceptPendingConnection; fcntl.h supports the adoption tests. #include +#include #include #include #include @@ -1951,6 +1956,165 @@ struct tcp_acceptor_test close_native_socket(second); } +#if BOOST_COROSIO_POSIX + // An accept that took a connection, but was queued to its context + // rather than run, must close that connection when the context is + // torn down without running it. + void testQueuedAcceptClosesPeerOnTeardown() + { + auto client = make_native_socket(AF_INET, SOCK_STREAM); + BOOST_TEST(client != invalid_native_socket); + // The drained accept abandons its awaiting frame, as every + // drained I/O operation does; the arena reclaims it. + std::pmr::monotonic_buffer_resource arena; + { + io_context a(Backend); + tcp_acceptor acc(a); + BOOST_TEST(!acc.open()); + acc.set_option(socket_option::reuse_address(true)); + BOOST_TEST(!acc.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!acc.listen()); + BOOST_TEST(native_connect_loopback( + client, acc.local_endpoint().port(), false)); + + io_context b(Backend); + capy::run_async(b.get_executor(), &arena)( + [](tcp_acceptor& acc) -> capy::task<> { + std::ignore = co_await acc.accept(); + }(acc)); + std::ignore = b.poll(); // the accept completes into a's queue + } + + pollfd pfd{client, POLLIN, 0}; + BOOST_TEST_EQ(::poll(&pfd, 1, 5000), 1); + ::fcntl(client, F_SETFL, ::fcntl(client, F_GETFL) | O_NONBLOCK); + char c; + BOOST_TEST_EQ(::recv(client, &c, 1, 0), 0); + close_native_socket(client); + } +#endif + +#if BOOST_COROSIO_POSIX + // An accept that completed but whose awaiter is destroyed before it + // resumes must not strand the accepted connection: the awaiter's + // context queues the completion, and its teardown destroys the + // frame without resuming it. + void testAcceptedPeerClosedWhenAwaiterDestroyed(bool by_value) + { + io_context a(Backend); + tcp_acceptor acc(a); + BOOST_TEST(!acc.open()); + acc.set_option(socket_option::reuse_address(true)); + BOOST_TEST(!acc.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!acc.listen()); + auto const port = acc.local_endpoint().port(); + + tcp_socket peer(a); + auto client = make_native_socket(AF_INET, SOCK_STREAM); + BOOST_TEST(client != invalid_native_socket); + { + io_context b(Backend); + if (by_value) + capy::run_async(b.get_executor())( + [](tcp_acceptor& acc) -> capy::task<> { + std::ignore = co_await acc.accept(); + }(acc)); + else + capy::run_async(b.get_executor())( + [](tcp_acceptor& acc, tcp_socket& p) -> capy::task<> { + std::ignore = co_await acc.accept(p); + }(acc, peer)); + std::ignore = b.poll(); // parks the accept on a's acceptor + + BOOST_TEST(native_connect_loopback(client, port, false)); + a.run(); // completes the accept into b's queue + } + + // With the accepted socket closed, the client reads end of file; + // a stranded one never becomes readable. Some stacks deliver the + // close asynchronously, so readability is awaited, bounded only + // to fail rather than hang. + pollfd pfd{client, POLLIN, 0}; + BOOST_TEST_EQ(::poll(&pfd, 1, 5000), 1); + ::fcntl(client, F_SETFL, ::fcntl(client, F_GETFL) | O_NONBLOCK); + char c; + BOOST_TEST_EQ(::recv(client, &c, 1, 0), 0); + close_native_socket(client); + } +#endif + + // An accept awaited from a non-io_context executor takes the deferring + // branch. The queued continuation must survive the next accept reusing + // the op. + void testDeferredExecutorCompletion() + { + io_context ioc(Backend); + capy::thread_pool pool(1); + + tcp_acceptor acc(ioc); + BOOST_TEST(!acc.open()); + acc.set_option(socket_option::reuse_address(true)); + BOOST_TEST(!acc.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!acc.listen()); + endpoint ep(ipv4_address::loopback(), acc.local_endpoint().port()); + + std::latch started(1); + std::latch hold(1); + std::atomic c1_done{false}; + std::atomic c2_done{false}; + tcp_socket s1(ioc); + tcp_socket s2(ioc); + + auto accepter = [](tcp_acceptor& a, tcp_socket& s, + std::atomic& done) -> capy::task<> { + auto [ec] = co_await a.accept(s); + done = !ec; + }; + auto connector = [](tcp_socket& c, endpoint e) -> capy::task<> { + std::ignore = co_await c.connect(e); + }; + + // Awaited on the pool, so the completion takes the deferring path. + capy::run_async(pool.get_executor())(accepter(acc, s1, c1_done)); + + // Parked behind the first accept on the single worker: once this + // runs, that accept has parked and suspended. + capy::run_async(pool.get_executor())( + [](std::latch& s, std::latch& h) -> capy::task<> { + s.count_down(); + h.wait(); + co_return; + }(started, hold)); + started.wait(); + + // The completion queues the first accept's resume into the + // parked pool and consumes the op. + tcp_socket client1(ioc); + capy::run_async(ioc.get_executor())(connector(client1, ep)); + ioc.run(); + ioc.restart(); + + // Reuses the op while the pool still holds the first continuation. + tcp_socket client2(ioc); + capy::run_async(ioc.get_executor())(accepter(acc, s2, c2_done)); + capy::run_async(ioc.get_executor())(connector(client2, ep)); + ioc.run(); + ioc.restart(); + BOOST_TEST(c2_done.load()); + + hold.count_down(); + std::latch drained(1); + capy::run_async(pool.get_executor())( + [](std::latch& d) -> capy::task<> { + d.count_down(); + co_return; + }(drained)); + drained.wait(); + + BOOST_TEST(c1_done.load()); + pool.join(); + } + void run() { testAssignOnOpenIsAlreadyOpen(); @@ -1962,6 +2126,12 @@ struct tcp_acceptor_test // Cancellation testCancelAccept(); + testDeferredExecutorCompletion(); +#if BOOST_COROSIO_POSIX + testQueuedAcceptClosesPeerOnTeardown(); + testAcceptedPeerClosedWhenAwaiterDestroyed(false); + testAcceptedPeerClosedWhenAwaiterDestroyed(true); +#endif testCloseWhilePendingAccept(); #if !COROSIO_TEST_HAS_ASAN // Abandon parked coroutine frames by design; see context.hpp. diff --git a/test/unit/tcp_socket.cpp b/test/unit/tcp_socket.cpp index 4e6b5d04d..cc4a5a14b 100644 --- a/test/unit/tcp_socket.cpp +++ b/test/unit/tcp_socket.cpp @@ -2011,6 +2011,7 @@ struct tcp_socket_test testAssignRejections(); testAssignFailureKeepsSocket(); testAssignOverOpenKeepsPending(); + testCompletionResumesOnAwaitersContext(); testRelease(); testReleaseClosedThrows(); testAssignV6(); @@ -2545,6 +2546,48 @@ struct tcp_socket_test close_native_socket(nfd); } + // A completion is delivered by the socket's context, but the awaiter + // belongs to its own executor: a coroutine running on another + // context must resume there, not inline on the completing thread. + void testCompletionResumesOnAwaitersContext() + { + io_context a(Backend); + io_context b(Backend); + auto [s1, s2] = + test::make_socket_pair(a); + a.restart(); + + bool resumed = false; + bool on_b = false; + char buf[1]; + auto reader = [](tcp_socket& s, char* b, io_context& ctx, + bool& resumed, bool& on_b) -> capy::task<> { + auto [ec, n] = co_await s.read_some(capy::mutable_buffer(b, 1)); + BOOST_TEST(!ec); + BOOST_TEST_EQ(n, 1u); + resumed = true; + on_b = ctx.get_executor().running_in_this_thread(); + }; + capy::run_async(b.get_executor())(reader(s1, buf, b, resumed, on_b)); + std::ignore = b.poll(); // parks the read on a's socket + BOOST_TEST(!resumed); + + char const c = 'x'; + capy::run_async(a.get_executor())([](tcp_socket& s, char const* c) + -> capy::task<> { + auto [ec, n] = co_await s.write_some(capy::const_buffer(c, 1)); + BOOST_TEST(!ec); + BOOST_TEST_EQ(n, 1u); + }(s2, &c)); + a.run(); + BOOST_TEST(!resumed); + + b.restart(); + std::ignore = b.poll(); + BOOST_TEST(resumed); + BOOST_TEST(on_b); + } + // release() hands ownership to the caller only after pending // operations have been cancelled, and the descriptor is still // connected to the peer. diff --git a/test/unit/timeout.cpp b/test/unit/timeout.cpp index c46d248a3..7eb46448a 100644 --- a/test/unit/timeout.cpp +++ b/test/unit/timeout.cpp @@ -28,7 +28,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -77,6 +79,29 @@ stop_only_payload() co_return capy::io_result{ec, 99}; } +// Counts the blocks it hands out that are still live. +struct counting_resource : std::pmr::memory_resource +{ + int live = 0; + + void* do_allocate(std::size_t n, std::size_t align) override + { + ++live; + return std::pmr::new_delete_resource()->allocate(n, align); + } + + void do_deallocate(void* p, std::size_t n, std::size_t align) override + { + --live; + std::pmr::new_delete_resource()->deallocate(p, n, align); + } + + bool do_is_equal(std::pmr::memory_resource const& o) const noexcept override + { + return this == &o; + } +}; + template struct timeout_test { @@ -743,8 +768,31 @@ struct timeout_test } } + // The timer coroutine behind timeout() can outlive the chain that + // awaited it, so it must not be allocated from that chain's + // allocator, which the chain's owner may free once it completes. + void testDetachedTimerOutlivesChainAllocator() + { + io_context ioc(Backend); + counting_resource mr; + bool done = false; + capy::run_async(ioc.get_executor(), &mr, [&]() { + done = true; + })([]() -> capy::task<> { + std::ignore = co_await timeout( + delay(std::chrono::milliseconds(1)), std::chrono::seconds(10)); + }()); + while (!done && ioc.run_one() != 0) + { + } + BOOST_TEST(done); + BOOST_TEST_EQ(mr.live, 0); + ioc.run(); + } + void run() { + testDetachedTimerOutlivesChainAllocator(); testInnerWins(); testDeadlineWins(); testTimePointDeadlineWins(); diff --git a/test/unit/tls_stream_tests.hpp b/test/unit/tls_stream_tests.hpp index c4ecc8c3f..11f12490c 100644 --- a/test/unit/tls_stream_tests.hpp +++ b/test/unit/tls_stream_tests.hpp @@ -19,7 +19,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -27,7 +29,10 @@ #include #include +#include #include +#include +#include #include #include #include @@ -1599,6 +1604,420 @@ testVerifyCallbackOnSuccess(StreamFactory make_stream) } } +/** Handshake a client and server stream over a mocket pair. + + @return True if both sides completed without error. +*/ +template +bool +handshake_pair(io_context& ioc, Client& client, Server& server) +{ + bool ok = true; + auto first = [](Client& c, bool& ok) -> capy::task<> { + auto [ec] = co_await c.handshake(tls_role::client); + ok = ok && !ec; + }; + auto second = [](Server& s, bool& ok) -> capy::task<> { + auto [ec] = co_await s.handshake(tls_role::server); + ok = ok && !ec; + }; + capy::run_async(ioc.get_executor())(first(client, ok)); + capy::run_async(ioc.get_executor())(second(server, ok)); + ioc.run(); + ioc.restart(); + return ok; +} + +/** Destroy a stream while a read on it is parked in the transport. + + The read must still complete exactly once, as cancelled, and + nothing may touch the destroyed stream's state afterwards. With + @p owning the stream owns its socket; otherwise it refers to one + that stays open. +*/ +template +void +testDestroyWithPendingRead(StreamFactory make_stream, bool owning) +{ + io_context ioc; + auto [m1, m2] = corosio::test::make_mocket_pair(ioc); + auto client_ctx = make_client_context(); + auto server_ctx = make_server_context(); + using stream_t = decltype(make_stream(m1, client_ctx)); + + auto client = owning + ? std::make_unique(std::move(m1), client_ctx) + : std::make_unique(make_stream(m1, client_ctx)); + auto server = make_stream(m2, server_ctx); + BOOST_TEST(handshake_pair(ioc, *client, server)); + + int resumed = 0; + std::error_code rec; + char buf[16]; + auto reader = [](stream_t& s, char* b, std::error_code& ec, + int& n) -> capy::task<> { + auto [e, k] = co_await s.read_some(capy::mutable_buffer(b, 16)); + std::ignore = k; + ec = e; + ++n; + }; + capy::run_async(ioc.get_executor())(reader(*client, buf, rec, resumed)); + std::ignore = ioc.poll(); + BOOST_TEST_EQ(resumed, 0); + + client.reset(); + + // Data arriving for the destroyed stream must not reach its state. + auto writer = [](stream_t& s) -> capy::task<> { + std::ignore = co_await s.write_some(capy::const_buffer("hello", 5)); + }; + capy::run_async(ioc.get_executor())(writer(server)); + ioc.run(); + BOOST_TEST_EQ(resumed, 1); + BOOST_TEST(rec == capy::cond::canceled); +} + +/** Destroy an owning stream with an operation created but never + awaited, and let that operation outlive the context. The owned + transport must go with the stream, not with the operation: by then + its context and service are gone. +*/ +template +void +testUnawaitedReadOutlivesContext(StreamFactory make_stream) +{ + auto client_ctx = make_client_context(); + char buf[16]; + std::optional> op; + { + io_context ioc; + auto [m1, m2] = corosio::test::make_mocket_pair(ioc); + using stream_t = decltype(make_stream(m1, client_ctx)); + auto client = std::make_unique(std::move(m1), client_ctx); + op.emplace(client->read_some(capy::mutable_buffer(buf, sizeof(buf)))); + client.reset(); + } + op.reset(); + BOOST_TEST_PASS(); +} + +/** Unwind a coroutine that owns a stream while its read is parked in + the transport, with another operation on the stream created but + never awaited and kept past the context. The owned transport must + go with the stream, not with that operation. +*/ +template +void +testUnawaitedReadOutlivesUnwoundOwner(StreamFactory make_stream) +{ + auto client_ctx = make_client_context(); + auto server_ctx = make_server_context(); + char buf[16]; + char spare[16]; + std::optional> op; + { + io_context ioc; + auto [m1, m2] = corosio::test::make_mocket_pair(ioc); + using stream_t = decltype(make_stream(m1, client_ctx)); + auto client = std::make_unique(std::move(m1), client_ctx); + auto server = make_stream(m2, server_ctx); + BOOST_TEST(handshake_pair(ioc, *client, server)); + ioc.restart(); + + auto owner = [](std::unique_ptr s, + std::optional>& o, + char* b, char* sp) -> capy::task<> { + o.emplace(s->read_some(capy::mutable_buffer(sp, 16))); + std::ignore = co_await s->read_some(capy::mutable_buffer(b, 16)); + }; + capy::run_async(ioc.get_executor())( + owner(std::move(client), op, buf, spare)); + std::ignore = ioc.poll(); + } + op.reset(); + BOOST_TEST_PASS(); +} + +/** Destroy a stream that owns another TLS stream while a read is + parked through both. The cancellation must reach the innermost + transport, so the read completes without help from the peer. +*/ +template +void +testDestroyNestedOwnerWithPendingRead() +{ + io_context ioc; + auto [m1, m2] = corosio::test::make_mocket_pair(ioc); + auto client_ctx = make_client_context(); + auto server_ctx = make_server_context(); + + Inner inner_client(std::move(m1), client_ctx); + Inner inner_server(&m2, server_ctx); + BOOST_TEST(handshake_pair(ioc, inner_client, inner_server)); + ioc.restart(); + + auto client = std::make_unique(std::move(inner_client), client_ctx); + Outer server(&inner_server, server_ctx); + BOOST_TEST(handshake_pair(ioc, *client, server)); + ioc.restart(); + + int resumed = 0; + std::error_code rec; + bool failsafe_hit = false; + std::stop_source failsafe_stop; + char buf[16]; + auto reader = [&]() -> capy::task<> { + auto [e, k] = + co_await client->read_some(capy::mutable_buffer(buf, sizeof(buf))); + std::ignore = k; + rec = e; + ++resumed; + failsafe_stop.request_stop(); + }; + auto failsafe_task = [&]() -> capy::task<> { + auto [ec] = co_await corosio::delay(failsafe_timeout); + if (!ec) + { + failsafe_hit = true; + if (m2.is_open()) + m2.close(); + } + }; + capy::run_async(ioc.get_executor())(reader()); + std::ignore = ioc.poll(); + BOOST_TEST_EQ(resumed, 0); + + client.reset(); + capy::run_async( + ioc.get_executor(), failsafe_stop.get_token())(failsafe_task()); + ioc.run(); + + BOOST_TEST(!failsafe_hit); + BOOST_TEST_EQ(resumed, 1); + BOOST_TEST(rec == capy::cond::canceled); +} + +/** cancel() ends a read parked on an owned transport and leaves the + stream usable; on a referenced transport it does nothing. +*/ +template +void +testCancelPendingRead(StreamFactory make_stream, bool owning) +{ + io_context ioc; + auto [m1, m2] = corosio::test::make_mocket_pair(ioc); + auto client_ctx = make_client_context(); + auto server_ctx = make_server_context(); + using stream_t = decltype(make_stream(m1, client_ctx)); + + auto client = owning + ? std::make_unique(std::move(m1), client_ctx) + : std::make_unique(make_stream(m1, client_ctx)); + auto server = make_stream(m2, server_ctx); + BOOST_TEST(handshake_pair(ioc, *client, server)); + ioc.restart(); + + std::error_code rec; + std::size_t rn = 0; + char buf[16]; + auto reader = [&]() -> capy::task<> { + auto [e, k] = + co_await client->read_some(capy::mutable_buffer(buf, sizeof(buf))); + rec = e; + rn = k; + }; + auto writer = [&]() -> capy::task<> { + std::ignore = co_await server.write_some(capy::const_buffer("hi", 2)); + }; + + capy::run_async(ioc.get_executor())(reader()); + std::ignore = ioc.poll(); + client->cancel(); + if (owning) + { + std::ignore = ioc.poll(); + BOOST_TEST(rec == capy::cond::canceled); + ioc.restart(); + + // The session survives: the next read gets the peer's data. + rec = {}; + capy::run_async(ioc.get_executor())(reader()); + } + capy::run_async(ioc.get_executor())(writer()); + ioc.run(); + BOOST_TEST(!rec); + BOOST_TEST_EQ(rn, 2u); +} + +/// An awaitable that resumes its caller through the executor's queue. +struct posted_resume +{ + capy::continuation c; + + bool await_ready() const noexcept + { + return false; + } + + void await_suspend(std::coroutine_handle<> h, capy::io_env const* env) + { + c.h = h; + env->executor.post(c); + } + + void await_resume() noexcept {} +}; + +/** A transport whose writes, once armed, report one byte through a + completion queued on the executor. +*/ +struct partial_write_transport +{ + corosio::test::mocket m; + bool const* armed; + int* armed_writes; + + template + capy::io_task read_some(Buffers buffers) + { + co_return co_await m.read_some(buffers); + } + + template + capy::io_task write_some(Buffers buffers) + { + if (*armed) + { + ++*armed_writes; + co_await posted_resume{}; + co_return {std::error_code{}, 1}; + } + co_return co_await m.write_some(buffers); + } +}; + +/** Destroy a stream while a partial transport write is still to report. + + The write loop must not go back to the transport the stream owned + once the stream is gone. +*/ +template +void +testDestroyWithPartialWriteInFlight(StreamFactory make_stream) +{ + io_context ioc; + auto [m1, m2] = corosio::test::make_mocket_pair(ioc); + auto client_ctx = make_client_context(); + auto server_ctx = make_server_context(); + using stream_t = decltype(make_stream(m1, client_ctx)); + + bool armed = false; + int armed_writes = 0; + auto client = std::make_unique( + partial_write_transport{std::move(m1), &armed, &armed_writes}, + client_ctx); + auto server = make_stream(m2, server_ctx); + BOOST_TEST(handshake_pair(ioc, *client, server)); + + armed = true; + auto writer = [](stream_t& s) -> capy::task<> { + std::ignore = co_await s.write_some(capy::const_buffer("hello", 5)); + }; + auto destroyer = [](std::unique_ptr& c) -> capy::task<> { + c.reset(); + co_return; + }; + capy::run_async(ioc.get_executor())(writer(*client)); + capy::run_async(ioc.get_executor())(destroyer(client)); + ioc.run(); + BOOST_TEST_EQ(armed_writes, 1); +} + +/** Destroy an owning stream whose transport keeps its own operation + state, while a read on it is parked. +*/ +template +void +testDestroyTypeErasedOwnerWithPendingRead(StreamFactory make_stream) +{ + io_context ioc; + auto [m1, m2] = corosio::test::make_mocket_pair(ioc); + auto client_ctx = make_client_context(); + auto server_ctx = make_server_context(); + using stream_t = decltype(make_stream(m1, client_ctx)); + + auto client = std::make_unique( + capy::any_stream(std::move(m1)), client_ctx); + auto server = make_stream(m2, server_ctx); + BOOST_TEST(handshake_pair(ioc, *client, server)); + + int resumed = 0; + std::error_code rec; + char buf[16]; + auto reader = [](stream_t& s, char* b, std::error_code& ec, + int& n) -> capy::task<> { + auto [e, k] = co_await s.read_some(capy::mutable_buffer(b, 16)); + std::ignore = k; + ec = e; + ++n; + }; + capy::run_async(ioc.get_executor())(reader(*client, buf, rec, resumed)); + std::ignore = ioc.poll(); + + client.reset(); + + auto writer = [](stream_t& s) -> capy::task<> { + std::ignore = co_await s.write_some(capy::const_buffer("hello", 5)); + }; + capy::run_async(ioc.get_executor())(writer(server)); + ioc.run(); + BOOST_TEST_EQ(resumed, 1); + BOOST_TEST(rec == capy::cond::canceled); +} + +/** Move a stream while a read on it is parked in the transport. + + The read completes through the moved-to stream, and the moved-from + stream may be destroyed first. +*/ +template +void +testMoveWithPendingRead(StreamFactory make_stream) +{ + io_context ioc; + auto [m1, m2] = corosio::test::make_mocket_pair(ioc); + auto client_ctx = make_client_context(); + auto server_ctx = make_server_context(); + using stream_t = decltype(make_stream(m1, client_ctx)); + + auto client = std::make_unique(make_stream(m1, client_ctx)); + auto server = make_stream(m2, server_ctx); + BOOST_TEST(handshake_pair(ioc, *client, server)); + + std::error_code rec; + std::size_t got = 0; + char buf[16]; + auto reader = [](stream_t& s, char* b, std::error_code& ec, + std::size_t& n) -> capy::task<> { + auto [e, k] = co_await s.read_some(capy::mutable_buffer(b, 16)); + ec = e; + n = k; + }; + capy::run_async(ioc.get_executor())(reader(*client, buf, rec, got)); + std::ignore = ioc.poll(); + + stream_t moved(std::move(*client)); + client.reset(); + + auto writer = [](stream_t& s) -> capy::task<> { + std::ignore = co_await s.write_some(capy::const_buffer("hello", 5)); + }; + capy::run_async(ioc.get_executor())(writer(server)); + ioc.run(); + BOOST_TEST(!rec); + BOOST_TEST_EQ(got, 5u); +} + /** Test move construction and move assignment of a live stream. The moved-into stream must keep working after a completed @@ -2794,6 +3213,103 @@ testShutdownSimultaneousClose(StreamFactory make_stream) m2.close(); } +/** Shutdown with an unread record queued ahead of the peer's + close_notify: the server writes and then shuts down before the + client reads anything, so the client's shutdown receives both + records in one transport read. The shutdown must complete instead of + parking on a transport read for a close_notify that has already + arrived; the unread application data is discarded and reported as + an error on both engines. */ +template +void +testShutdownWithUnreadRecordAhead(StreamFactory make_stream) +{ + io_context ioc; + auto [m1, m2] = corosio::test::make_mocket_pair(ioc); + + auto client_ctx = make_client_context(); + auto server_ctx = make_server_context(); + + auto client = make_stream(m1, client_ctx); + auto server = make_stream(m2, server_ctx); + + { + auto hs_client = [&]() -> capy::task<> { + auto [ec] = co_await client.handshake(tls_role::client); + BOOST_TEST(!ec); + }; + auto hs_server = [&]() -> capy::task<> { + auto [ec] = co_await server.handshake(tls_role::server); + BOOST_TEST(!ec); + }; + capy::run_async(ioc.get_executor())(hs_client()); + capy::run_async(ioc.get_executor())(hs_server()); + ioc.run(); + ioc.restart(); + } + + bool client_sd_done = false; + std::error_code client_sd_ec; + bool server_sd_ok = false; + bool failsafe_hit = false; + int remaining = 2; + std::stop_source failsafe_stop; + auto finished = [&] { + if (--remaining == 0) + failsafe_stop.request_stop(); + }; + + // Parks in its shutdown once the data record and its close_notify + // are both on the wire. + auto server_task = [&]() -> capy::task<> { + char const one = 'x'; + auto [wec, wn] = + co_await server.write_some(capy::const_buffer(&one, 1)); + BOOST_TEST(!wec); + BOOST_TEST_EQ(wn, 1u); + auto [ec] = co_await server.shutdown(); + server_sd_ok = !ec; + finished(); + }; + capy::run_async(ioc.get_executor())(server_task()); + while (ioc.poll() > 0) + { + } + + auto client_task = [&]() -> capy::task<> { + auto [ec] = co_await client.shutdown(); + client_sd_ec = ec; + client_sd_done = true; + finished(); + }; + auto failsafe_task = [&]() -> capy::task<> { + auto [ec] = co_await corosio::delay(failsafe_timeout); + if (!ec) + { + failsafe_hit = true; + if (m1.is_open()) + m1.close(); + if (m2.is_open()) + m2.close(); + } + }; + + capy::run_async(ioc.get_executor())(client_task()); + capy::run_async( + ioc.get_executor(), failsafe_stop.get_token())(failsafe_task()); + ioc.run(); + + BOOST_TEST(!failsafe_hit); + BOOST_TEST(client_sd_done); + BOOST_TEST(!!client_sd_ec); + BOOST_TEST(server_sd_ok); + + if (m1.is_open()) + m1.close(); + if (m2.is_open()) + m2.close(); +} + /** Stream wrapper that injects an error alongside a real transport read, exercising the ReadStream contract's legal partial-transfer- with-error case (IOCP's failed completions carry bytes_transferred; diff --git a/test/unit/udp_socket.cpp b/test/unit/udp_socket.cpp index c8eaf3e51..76f985b3e 100644 --- a/test/unit/udp_socket.cpp +++ b/test/unit/udp_socket.cpp @@ -1909,6 +1909,53 @@ struct udp_socket_test BOOST_TEST(!wec); } + // A datagram larger than the receive buffer completes its receive + // exactly once, whatever the platform reports about the truncation, + // and leaves the next receive to the next datagram. + void testReceiveIntoSmallBufferCompletesOnce() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + udp_socket s1(ioc), s2(ioc); + BOOST_TEST(!s1.open(family::v4)); + BOOST_TEST(!s2.open(family::v4)); + BOOST_TEST(!s1.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!s2.bind(endpoint(ipv4_address::loopback(), 0))); + + int first_done = 0; + std::size_t second_n = 0; + std::error_code second_ec; + char second_buf[8] = {}; + auto receiver = [&]() -> capy::task<> { + char small[8]; + endpoint from; + auto [e1, n1] = co_await s2.recv_from( + capy::mutable_buffer(small, sizeof(small)), from); + std::ignore = e1; + std::ignore = n1; + ++first_done; + auto [e2, n2] = co_await s2.recv_from( + capy::mutable_buffer(second_buf, sizeof(second_buf)), from); + second_ec = e2; + second_n = n2; + }; + auto sender = [&]() -> capy::task<> { + std::vector big(64, 'x'); + std::ignore = co_await s1.send_to( + capy::const_buffer(big.data(), big.size()), + s2.local_endpoint()); + std::ignore = co_await s1.send_to( + capy::const_buffer("ok", 2), s2.local_endpoint()); + }; + capy::run_async(ex)(receiver()); + capy::run_async(ex)(sender()); + ioc.run(); + BOOST_TEST_EQ(first_done, 1); + BOOST_TEST(!second_ec); + BOOST_TEST_EQ(second_n, 2u); + BOOST_TEST(std::memcmp(second_buf, "ok", 2) == 0); + } + void testOversizedSendReportsError() { io_context ioc(Backend); @@ -1988,6 +2035,7 @@ struct udp_socket_test testConnectedShutdownSendSucceeds(); testWaitWriteReady(); testOversizedSendReportsError(); + testReceiveIntoSmallBufferCompletesOnce(); testAssignConnectedFdCachesRemote(); testConstruction(); diff --git a/test/unit/wolfssl_stream.cpp b/test/unit/wolfssl_stream.cpp index 1084be3d3..9c93c8f1c 100644 --- a/test/unit/wolfssl_stream.cpp +++ b/test/unit/wolfssl_stream.cpp @@ -16,6 +16,9 @@ // Test that header file is self-contained. #include +#ifdef BOOST_COROSIO_HAS_OPENSSL +#include +#endif #include "tls_stream_tests.hpp" @@ -154,6 +157,11 @@ struct wolfssl_stream_test void run() { test::testIoBeforeHandshake(make_stream); + test::testDestroyWithPendingRead(make_stream, false); + test::testDestroyWithPendingRead(make_stream, true); + test::testMoveWithPendingRead(make_stream); + test::testDestroyWithPartialWriteInFlight(make_stream); + test::testDestroyTypeErasedOwnerWithPendingRead(make_stream); test::testHandshakeFuse(make_stream); test::testReadWriteFuse(make_stream); test::testShutdownFuse(make_stream); @@ -177,6 +185,14 @@ struct wolfssl_stream_test test::testRecordBoundaryTransfer(make_stream); test::testShutdownOverRead(make_stream); test::testShutdownSimultaneousClose(make_stream); + test::testShutdownWithUnreadRecordAhead(make_stream); + test::testUnawaitedReadOutlivesContext(make_stream); + test::testUnawaitedReadOutlivesUnwoundOwner(make_stream); + test::testCancelPendingRead(make_stream, true); + test::testCancelPendingRead(make_stream, false); +#ifdef BOOST_COROSIO_HAS_OPENSSL + test::testDestroyNestedOwnerWithPendingRead(); +#endif test::testPartialReadWithError(make_stream); test::testCancelParkedReader(make_stream); test::testFullDuplexMtStrand(make_stream); From 646962783570d047b9c302652b1ee2d2d497d936 Mon Sep 17 00:00:00 2001 From: Steve Gerbino Date: Fri, 9 Oct 2026 19:36:19 +0200 Subject: [PATCH 4/8] fix: correct io_uring cancellation, wakeups and run_for A pending connect is cancelled when its socket is released, and an acceptor keeps listening after cancel(). Cancel requests target only the object's own operations and are never dropped, an operation whose stop lands before its SQE is linked is cancelled, and a wait cancelled by close skips the SO_ERROR probe. A post from another thread under unsafe_io wakes the run thread, and run_for is bounded by its duration when a later timer is armed. --- .../native/detail/uring/uring_descriptor.hpp | 27 +- .../detail/uring/uring_file_service_base.hpp | 2 +- .../detail/uring/uring_multishot_acceptor.hpp | 93 ++++- .../detail/uring/uring_random_access_file.hpp | 53 ++- .../native/detail/uring/uring_scheduler.hpp | 331 ++++++++++++---- .../native/detail/uring/uring_socket_ops.hpp | 35 +- .../native/detail/uring/uring_stream_file.hpp | 11 +- .../native/detail/uring/uring_types.hpp | 102 +++-- test/unit/fault/fault_uring.cpp | 13 +- test/unit/fault/uring_faults.cpp | 359 +++++++++++++++++- test/unit/io_context.cpp | 22 ++ test/unit/local_stream_socket.cpp | 57 +++ .../unit/native/uring/descriptor_continue.cpp | 5 +- test/unit/native/uring/mt_scheduler.cpp | 191 ++++++++++ test/unit/posix_stream_descriptor.cpp | 54 +++ test/unit/tcp_acceptor.cpp | 59 +++ test/unit/teardown_inflight.cpp | 2 +- 17 files changed, 1215 insertions(+), 201 deletions(-) diff --git a/include/boost/corosio/native/detail/uring/uring_descriptor.hpp b/include/boost/corosio/native/detail/uring/uring_descriptor.hpp index 16a144720..850a4f11b 100644 --- a/include/boost/corosio/native/detail/uring/uring_descriptor.hpp +++ b/include/boost/corosio/native/detail/uring/uring_descriptor.hpp @@ -64,7 +64,7 @@ The gap between a CQE and its dispatch is the whole difficulty of that shape, and one epoch counter closes it. Nothing of a transfer - waiting in that gap is in the ring, so cancel-by-fd cannot reach + waiting in that gap is in the ring, so a kernel cancel cannot reach it. cancel() and every descriptor change bump epoch_ before they take ring_mutex_, and do_prep, which runs under ring_mutex_, compares the op's snapshot against it: an op that no longer @@ -411,13 +411,13 @@ class BOOST_COROSIO_DECL uring_descriptor final native_handle_type release_descriptor() noexcept override { - // Bump before the flush, as close_descriptor does. Flush the - // cancel while the fd is still open so the kernel resolves it - // before the caller can close and recycle the number. Do NOT + // Bump before the flush, as close_descriptor does. Do NOT // close -- the caller takes ownership. epoch_.fetch_add(1, std::memory_order_release); + cancel_waits(); if (fd_ >= 0) - sched_->cancel_and_flush(fd_); + sched_->flush_after_cancel( + fd_, {&rd_, &wr_, &wait_rd_, &wait_wr_, &wait_er_}); native_handle_type released = fd_; fd_ = -1; return released; @@ -425,12 +425,12 @@ class BOOST_COROSIO_DECL uring_descriptor final void cancel() noexcept override { - // Bump before the SQE: cancel-by-fd reaches only what the ring + // Bump before the SQE: the cancel reaches only what the ring // currently holds, and an op waiting for its handler to run // holds nothing there. The epoch is what that op consults. epoch_.fetch_add(1, std::memory_order_release); if (fd_ >= 0) - sched_->submit_cancel_by_fd(fd_); + sched_->cancel_ops({&rd_, &wr_, &wait_rd_, &wait_wr_, &wait_er_}); } /// Epoch bumped by every @ref cancel and every descriptor change. @@ -475,15 +475,26 @@ class BOOST_COROSIO_DECL uring_descriptor final // Bump before the flush: an op prepping concurrently must see // the change, or its SQE could land on a recycled fd number. epoch_.fetch_add(1, std::memory_order_release); + cancel_waits(); // Both kernel entries below can run a queued pipe write as task // work; with the reader already gone that raises SIGPIPE. scoped_sigpipe_block no_sigpipe; - sched_->cancel_and_flush(fd_); + sched_->flush_after_cancel( + fd_, {&rd_, &wr_, &wait_rd_, &wait_wr_, &wait_er_}); ::close(fd_); fd_ = -1; } private: + // A wait completing after a close or release must not probe the + // old number for SO_ERROR; see uring_wait_op. + void cancel_waits() noexcept + { + wait_rd_.request_cancel(); + wait_wr_.request_cancel(); + wait_er_.request_cancel(); + } + /** Bind a transfer slot to this descriptor for a fresh submission. The epoch snapshot taken here is what every later prep compares diff --git a/include/boost/corosio/native/detail/uring/uring_file_service_base.hpp b/include/boost/corosio/native/detail/uring/uring_file_service_base.hpp index 829824958..4d99cc984 100644 --- a/include/boost/corosio/native/detail/uring/uring_file_service_base.hpp +++ b/include/boost/corosio/native/detail/uring/uring_file_service_base.hpp @@ -77,7 +77,7 @@ class uring_file_service_base : public ServiceBase void destroy(io_object::implementation* p) override { - // close_file() already does cancel_and_flush(fd_) before ::close. + // close_file() closes the fd only once its cancel reached the kernel. auto& impl = static_cast(*p); impl.close_file(); release(&impl); diff --git a/include/boost/corosio/native/detail/uring/uring_multishot_acceptor.hpp b/include/boost/corosio/native/detail/uring/uring_multishot_acceptor.hpp index a1cc1d004..4143ca277 100644 --- a/include/boost/corosio/native/detail/uring/uring_multishot_acceptor.hpp +++ b/include/boost/corosio/native/detail/uring/uring_multishot_acceptor.hpp @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -299,13 +300,18 @@ class uring_multishot_acceptor_base ~uring_multishot_acceptor_base() override { + uring_multi_accept_op* arming; { std::lock_guard lk(mutex_); closing_ = true; + arming = std::exchange(armed_, nullptr); } if (fd_ >= 0) { - sched_->submit_cancel_by_fd(fd_); + sched_->flush_after_cancel(fd_, [&]() noexcept { + if (arming) + sched_->cancel_op_locked(arming); + }); ::close(fd_); fd_ = -1; } @@ -355,21 +361,77 @@ class uring_multishot_acceptor_base // Mirror the service close() path, except that the caller now // owns the fd. if (fd_ >= 0) - { - drain_waiters_only(); - sched_->cancel_and_flush(fd_); - } + flush_listener(); int fd = fd_; fd_ = -1; local_endpoint_ = Endpoint{}; return fd; } + /// Abort every operation and stop accepting, for service teardown. + void abort_all() noexcept + { + auto* arming = drain_waiters_only(); + static_cast(this)->cancel_wait_op(); + if (arming) + sched_->cancel_ops({arming}); + } + + /** Stop accepting and flush every SQE naming the listener. + + Disowns the arming and cancels it with the readiness wait, so + the caller may then close or hand off the descriptor. + + @pre The descriptor is open. + */ + void flush_listener() noexcept + { + auto* arming = drain_waiters_only(); + static_cast(this)->cancel_wait_op(); + sched_->flush_after_cancel(fd_, [&]() noexcept { + if (arming) + sched_->cancel_op_locked(arming); + }); + } + + /** Cancel the pending accepts and waits. + + The arming is left alone: the acceptor keeps listening, and a + connection that arrives meanwhile waits for the next accept. + */ void cancel() noexcept override { - drain_waiters_only(); - if (fd_ >= 0) - sched_->submit_cancel_by_fd(fd_); + intrusive_list claimed; + { + std::lock_guard lk(mutex_); + // A waiter the stop callback already claimed belongs to it; + // see fail_arm(). + intrusive_list keep; + while (auto* w = waiters_.pop_front()) + { + if (!w->cancelled.exchange(true, std::memory_order_acq_rel)) + claimed.push_back(w); + else + keep.push_back(w); + } + while (auto* w = keep.pop_front()) + waiters_.push_back(w); + if (read_wait_ && + !read_wait_->cancelled.exchange( + true, std::memory_order_acq_rel)) + { + claimed.push_back(read_wait_); + read_wait_ = nullptr; + } + } + while (auto* w = claimed.pop_front()) + { + // The claim is also the completion status. + w->stop_cb.reset(); + sched_->post(w); + sched_->work_finished(); + } + static_cast(this)->cancel_wait_op(); } /** Reset state for recycling. @@ -400,17 +462,21 @@ class uring_multishot_acceptor_base /** Disown the arming and drain queued waiters with operation_aborted, without submitting a kernel cancel. - Runs before the caller's cancel-by-fd: an arming admitted after - this sees `closing_` and is refused, and one admitted before it - is already in the submission queue ahead of the cancel. + Runs before the caller cancels the arming it returns: an arming + admitted after this sees `closing_` and is refused, and the one + admitted before it is already in the submission queue ahead of + the cancel. + + @return The disowned arming, or null if none was live. */ - void drain_waiters_only() noexcept + uring_multi_accept_op* drain_waiters_only() noexcept { intrusive_list drained; + uring_multi_accept_op* arming; { std::lock_guard lk(mutex_); closing_ = true; - armed_ = nullptr; + arming = std::exchange(armed_, nullptr); // Drain under the lock — the kernel cancel may not produce // a !more CQE before the fd is closed, so we can't rely on // on_accept_cqe_impl to surface operation_aborted. @@ -432,6 +498,7 @@ class uring_multishot_acceptor_base sched_->post(w); sched_->work_finished(); } + return arming; } /** Park a readiness wait, or complete it if a connection is diff --git a/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp b/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp index 12f70a95d..f52e21c57 100644 --- a/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp +++ b/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp @@ -123,35 +123,47 @@ class BOOST_COROSIO_DECL uring_random_access_file final std::mutex ops_mutex_; intrusive_list free_reads_; intrusive_list free_writes_; + // In flight, so a cancel can name each one. + intrusive_list live_reads_; + intrusive_list live_writes_; /// Pop a recycled op of the given kind, or allocate on the cold - /// path; `prepare()` resets every per-use field. + /// path, and list it live; `prepare()` resets every per-use field. template - Op* acquire_op(intrusive_list& free_list) + Op* acquire_op(intrusive_list& free_list, intrusive_list& live) { - Op* op; - { - std::lock_guard lock(ops_mutex_); - op = free_list.pop_front(); - } + std::lock_guard lock(ops_mutex_); + Op* op = free_list.pop_front(); if (!op) op = new Op(); op->file_ = this; + live.push_back(op); return op; } void release_op(pooled_read_op* op) noexcept { std::lock_guard lock(ops_mutex_); + live_reads_.remove(op); free_reads_.push_front(op); } void release_op(pooled_write_op* op) noexcept { std::lock_guard lock(ops_mutex_); + live_writes_.remove(op); free_writes_.push_front(op); } + /// Cancel every live op. @pre `ops_mutex_` and the ring mutex are held. + void cancel_live_locked() noexcept + { + live_reads_.for_each( + [this](pooled_read_op* op) { sched_->cancel_op_locked(op); }); + live_writes_.for_each( + [this](pooled_write_op* op) { sched_->cancel_op_locked(op); }); + } + public: explicit uring_random_access_file( uring_random_access_file_service& svc, @@ -216,8 +228,10 @@ class BOOST_COROSIO_DECL uring_random_access_file final void cancel() noexcept override { - if (fd_ >= 0) - sched_->submit_cancel_by_fd(fd_); + if (fd_ < 0) + return; + std::lock_guard lock(ops_mutex_); + sched_->cancel_ops([this]() noexcept { cancel_live_locked(); }); } std::uint64_t size() const override @@ -258,11 +272,10 @@ class BOOST_COROSIO_DECL uring_random_access_file final native_handle_type release() override { - // Flush the cancel while the fd is still open, so the kernel - // resolves it before the caller can close and recycle the - // number. + // Flush while the fd is still open, so no queued SQE can name + // a number the caller closes and recycles. if (fd_ >= 0) - sched_->cancel_and_flush(fd_); + flush_live(); int fd = fd_; fd_ = -1; return fd; @@ -321,6 +334,14 @@ class BOOST_COROSIO_DECL uring_random_access_file final return {}; } + /// Cancel the live ops and flush every SQE naming `fd_`. + void flush_live() noexcept + { + std::lock_guard lock(ops_mutex_); + sched_->flush_after_cancel( + fd_, [this]() noexcept { cancel_live_locked(); }); + } + /// Cancel any in-flight ops and close the fd. Idempotent. void close_file() noexcept { @@ -330,7 +351,7 @@ class BOOST_COROSIO_DECL uring_random_access_file final // either kernel entry below; with the reader already gone // that raises SIGPIPE. scoped_sigpipe_block no_sigpipe; - sched_->cancel_and_flush(fd_); + flush_live(); ::close(fd_); fd_ = -1; } @@ -347,7 +368,7 @@ uring_random_access_file::read_some_at( std::error_code* ec, std::size_t* bytes) { - auto* op = acquire_op(free_reads_); + auto* op = acquire_op(free_reads_, live_reads_); op->prepare( cont, ex, ec, bytes, fd_, static_cast(user_offset), sched_, detail::object_ref(this), buffers, token); @@ -385,7 +406,7 @@ uring_random_access_file::write_some_at( std::error_code* ec, std::size_t* bytes) { - auto* op = acquire_op(free_writes_); + auto* op = acquire_op(free_writes_, live_writes_); op->prepare( cont, ex, ec, bytes, fd_, static_cast(user_offset), sched_, detail::object_ref(this), buffers, token); diff --git a/include/boost/corosio/native/detail/uring/uring_scheduler.hpp b/include/boost/corosio/native/detail/uring/uring_scheduler.hpp index ab79328c6..926536723 100644 --- a/include/boost/corosio/native/detail/uring/uring_scheduler.hpp +++ b/include/boost/corosio/native/detail/uring/uring_scheduler.hpp @@ -40,10 +40,13 @@ #include #include #include +#include #include +#include #include #include +#include #include #include #include @@ -275,31 +278,76 @@ class BOOST_COROSIO_DECL uring_scheduler final */ void submit_cancel_by_user_data(uring_op* target) noexcept; - /** Submit `IORING_OP_ASYNC_CANCEL` with `IORING_ASYNC_CANCEL_FD` - to cancel every in-flight op on the given fd in one SQE. + /** Cancel @p op by its user_data if the kernel may hold it. - Best-effort: if the SQ is full after one flush attempt the - function returns without cancelling. + A cancel that cannot be queued yet is deferred, and queued + ahead of the next operation's SQE. - @param fd The file descriptor whose in-flight ops should be - cancelled. + @pre `ring_mutex()` is held. + + @param op The operation to cancel. + */ + void cancel_op_locked(uring_op* op) noexcept; + + /** Cancel each of @p ops the kernel may hold. + + Only these operations are cancelled, never another object's + operation on the same file. + + @param ops The operations to cancel. + */ + void cancel_ops(std::initializer_list ops) noexcept; + + /** Cancel an object's operations. + + @param cancel Called under the ring mutex; cancels the object's + operations with @ref cancel_op_locked. + */ + template + void cancel_ops(Cancel&& cancel) noexcept + { + lazy_init_ring(); + interrupt_reactor(); + lock_type lock(ring_mutex_); + cancel(); + } + + /** Cancel an object's operations and flush every SQE naming `fd`. + + Once this returns no queued SQE refers to `fd` by number, so + the caller may close it, or hand it to an owner who may. If the + flush fails, this object's SQEs still queued are turned into + no-ops and their operations complete as cancelled. + + @param fd The object's descriptor. + @param cancel Called under the ring mutex; cancels the object's + operations with @ref cancel_op_locked. */ - void submit_cancel_by_fd(int fd) noexcept; + template + void flush_after_cancel(int fd, Cancel&& cancel) noexcept; - /** Submit `IORING_OP_ASYNC_CANCEL` for `fd` and immediately flush - the submission ring to the kernel. + /// @copydoc flush_after_cancel + void + flush_after_cancel(int fd, std::initializer_list ops) noexcept + { + flush_after_cancel(fd, [&]() noexcept { + for (auto* op : ops) + cancel_op_locked(op); + }); + } + + /** Return a submission queue entry for a new operation. - Must be called while `fd` is still open so the kernel can - resolve the file from the fd number before it is closed and - potentially recycled. + Deferred cancels are queued first: they name operations by + address, and an operation armed later at the same address must + not be what they reach. Flushes once if the queue is full. - Best-effort: if the SQ is full the function still flushes any - earlier pending SQEs to the kernel. + @pre `ring_mutex()` is held. - @param fd The file descriptor whose in-flight ops should be - cancelled. + @return The entry, or null if the queue stays full or a + deferred cancel still cannot be queued. */ - void cancel_and_flush(int fd) noexcept; + ::io_uring_sqe* acquire_sqe() noexcept; /** Queue an already-counted op while the caller holds dispatch_mutex_. @@ -406,6 +454,31 @@ class BOOST_COROSIO_DECL uring_scheduler final int cancel_sentinel_ = 0; + std::vector deferred_cancels_; // guarded by ring_mutex_ + std::atomic has_deferred_cancels_{false}; + + /// io_uring_submit, retried while a signal or a full completion + /// queue delays it. @pre ring_mutex_ is held. + int submit_retrying() noexcept; + + // Submit as submit_retrying() does, and under SQPOLL also wait for + // the kernel thread to take every queued entry: an SQE names its + // file by number until it is issued, so the number must stay open + // until then. + int submit_issued() noexcept; + + /// Queue a cancel of @p op by user_data; return false if the SQ + /// stays full. @pre ring_mutex_ is held. + bool queue_cancel_locked(uring_op* op) noexcept; + + /// Queue the deferred cancels; return false if one still cannot + /// be. @pre ring_mutex_ is held. + bool queue_deferred_cancels() noexcept; + + /// Turn every SQE on @p fd the kernel has not taken into a no-op, + /// completing its operation as cancelled. @pre ring_mutex_ is held. + void neutralize_pending_locked(int fd) noexcept; + // Signal self-pipe integration. The read end is watched via a multishot // POLL SQE tagged with &signal_pipe_sentinel_ (distinct from nullptr = // wakeup eventfd and &cancel_sentinel_). On its CQE we re-arm the poll if @@ -535,7 +608,7 @@ uring_scheduler::lazy_init_ring_unlocked() const // io_uring_submit_and_wait_timeout always sets // IORING_ENTER_GETEVENTS when wait_nr > 0, regardless of // ts. Our run loop's only kernel-wait call passes wait_nr=1. - // Submit-only paths (cancel_and_flush, etc.) leave their + // Submit-only paths (flush_after_cancel, etc.) leave their // CQEs queued until the leader's next GETEVENTS-bearing // wait — benign. // @@ -572,7 +645,17 @@ uring_scheduler::lazy_init_ring_unlocked() const } } + // Keeps a batch going past an entry the kernel rejects, so a cancel + // queued behind it is still issued. Kernels that predate the flag + // refuse it, and the ring is set up without. + io_uring_params const base = params; + params.flags |= IORING_SETUP_SUBMIT_ALL; int rc = ::io_uring_queue_init_params(256, &ring_, ¶ms); + if (rc == -EINVAL) + { + params = base; + rc = ::io_uring_queue_init_params(256, &ring_, ¶ms); + } if (rc < 0) detail::throw_system_error(make_err(-rc), "io_uring_queue_init_params"); @@ -644,7 +727,7 @@ uring_scheduler::shutdown() io_uring_sqe* sqe = ::io_uring_get_sqe(&ring_); if (!sqe) { - ::io_uring_submit(&ring_); + submit_issued(); sqe = ::io_uring_get_sqe(&ring_); } if (sqe) @@ -653,18 +736,7 @@ uring_scheduler::shutdown() ::io_uring_sqe_set_data(sqe, &cancel_sentinel_); inflight_inc(); } - int submitted = -EINVAL; - if (sqe) - { - do - { - submitted = ::io_uring_submit(&ring_); - if (submitted == -EBUSY) - process_completions(); - } - while (submitted == -EINTR || submitted == -EAGAIN || - submitted == -EBUSY); - } + int const submitted = sqe ? submit_retrying() : -EINVAL; while (submitted >= 0 && uring_inflight_.load(std::memory_order_acquire) > 0) { @@ -676,6 +748,9 @@ uring_scheduler::shutdown() break; process_completions(); } + + deferred_cancels_.clear(); + has_deferred_cancels_.store(false, std::memory_order_release); } // Drain posted ops, including the completions reaped above, @@ -755,12 +830,16 @@ uring_scheduler::interrupt_reactor() const noexcept if (!ring_inited_) return; - // Lockless tiers (reactor-I/O locking off): cross-thread post() is - // forbidden, so interrupt_reactor is only ever reached from the leader - // thread's own coroutines — it is not in kernel wait, nothing to wake. - // Under the safe tier (including concurrency_hint == 1) cross-thread - // post() is allowed, so the eventfd write below must always fire. - if (!reactor_io_locking_) + // Only the unsafe tier forbids cross-thread post(); there the caller + // is the run thread itself, which is not in a kernel wait. Under + // unsafe_io a pool thread (the resolver) may post, and the eventfd + // write leaves the single ring submitter untouched. + if (scheduler_locking_disabled_) + return; + + // Under unsafe_io a single thread runs the ring; when that thread is + // the caller it is not in a kernel wait. + if (!reactor_io_locking_ && running_scheduler.get() == this) return; // Multi-thread: write the eventfd unconditionally. CAS-coalescing @@ -1117,6 +1196,8 @@ uring_scheduler::do_one(long timeout_us) if (ring_inited_) { lock_type ring_lock(ring_mutex_); + if (has_deferred_cancels_.load(std::memory_order_acquire)) + queue_deferred_cancels(); if (uring_inflight_.load(std::memory_order_acquire) != 0 || ::io_uring_sq_ready(&ring_) != 0 || ::io_uring_cq_ready(&ring_) != 0) @@ -1220,6 +1301,8 @@ uring_scheduler::do_one(long timeout_us) .count(); if (delta_ns < 0) delta_ns = 0; + if (timeout_us > 0 && timeout_us < delta_ns / 1000) + delta_ns = static_cast(timeout_us) * 1000; ts.tv_sec = delta_ns / 1'000'000'000; ts.tv_nsec = delta_ns % 1'000'000'000; ts_ptr = &ts; @@ -1259,6 +1342,8 @@ uring_scheduler::do_one(long timeout_us) // Phase 1 — submit any pending SQEs to the kernel. { lock_type ring_lock(ring_mutex_); + if (has_deferred_cancels_.load(std::memory_order_acquire)) + queue_deferred_cancels(); ::io_uring_submit(&ring_); } @@ -1364,6 +1449,9 @@ uring_scheduler::process_completions() else { auto* iop = static_cast(ud); + // Nothing is left for a cancel to reach. + if ((cqe->flags & IORING_CQE_F_MORE) == 0) + iop->sqe_set.store(false, std::memory_order_release); (*iop->cqe_func)(iop, cqe->res, cqe->flags, local_ops); // Decrement inflight on the terminal CQE only — multishot // ops (acceptor) hold the SQE alive across F_MORE CQEs and @@ -1417,58 +1505,55 @@ uring_scheduler::submit_cancel_by_user_data(uring_op* target) noexcept lazy_init_ring(); // Wake the leader (if any) so its submit_and_wait_timeout returns // and releases ring_mutex_; otherwise we'd block here until the - // next CQE arrives organically. Cancellation is best-effort if - // the SQ stays full after one flush — the op completes on its - // own and reports cancelled via the in-flight `cancelled` flag. + // next CQE arrives organically. interrupt_reactor(); lock_type lock(ring_mutex_); + cancel_op_locked(target); +} + +inline bool +uring_scheduler::queue_cancel_locked(uring_op* target) noexcept +{ io_uring_sqe* sqe = io_uring_get_sqe(&ring_); if (!sqe) { - io_uring_submit(&ring_); + // A flush alone frees no space under SQPOLL until the kernel + // thread consumes it. + submit_issued(); sqe = io_uring_get_sqe(&ring_); } if (!sqe) - return; + return false; io_uring_prep_cancel(sqe, target, 0); io_uring_sqe_set_data(sqe, &cancel_sentinel_); inflight_inc(); + return true; } inline void -uring_scheduler::submit_cancel_by_fd(int fd) noexcept +uring_scheduler::cancel_op_locked(uring_op* op) noexcept { - lazy_init_ring(); - interrupt_reactor(); - lock_type lock(ring_mutex_); - io_uring_sqe* sqe = io_uring_get_sqe(&ring_); - if (!sqe) - { - io_uring_submit(&ring_); - sqe = io_uring_get_sqe(&ring_); - } - if (!sqe) + if (!op->sqe_set.load(std::memory_order_acquire)) return; - - io_uring_prep_cancel_fd(sqe, fd, IORING_ASYNC_CANCEL_ALL); - io_uring_sqe_set_data(sqe, &cancel_sentinel_); - inflight_inc(); + if (queue_cancel_locked(op)) + return; + deferred_cancels_.push_back(op); + has_deferred_cancels_.store(true, std::memory_order_release); } inline void -uring_op::on_cancel() noexcept +uring_scheduler::cancel_ops(std::initializer_list ops) noexcept { - request_cancel(); // coro_op: records the cancellation (sets the flag) - // Skip the cancel SQE if we never linked an SQE to this op — the - // bypass path in the caller will see cancelled=true and complete - // synchronously without a kernel round-trip. - if (sched_ && sqe_set.load(std::memory_order_acquire)) - sched_->submit_cancel_by_user_data(this); + cancel_ops([&]() noexcept { + for (auto* op : ops) + cancel_op_locked(op); + }); } -inline void -uring_scheduler::cancel_and_flush(int fd) noexcept +template +void +uring_scheduler::flush_after_cancel(int fd, Cancel&& cancel) noexcept { // The flush can execute a queued write on `fd` inline; when the // fd is a pipe whose reader has already closed — service @@ -1479,21 +1564,115 @@ uring_scheduler::cancel_and_flush(int fd) noexcept lazy_init_ring(); interrupt_reactor(); lock_type lock(ring_mutex_); - io_uring_sqe* sqe = io_uring_get_sqe(&ring_); + cancel(); + if (submit_issued() < 0) + neutralize_pending_locked(fd); +} + +inline void +uring_op::on_cancel() noexcept +{ + // Paired with uring_link_sqe: with no SQE linked yet, the + // initiator either sees the flag and completes without the kernel, + // or links and queues the cancel itself. + cancelled.store(true, std::memory_order_seq_cst); + if (sched_ && sqe_set.load(std::memory_order_seq_cst)) + sched_->submit_cancel_by_user_data(this); +} + +inline int +uring_scheduler::submit_retrying() noexcept +{ + int r; + do + { + r = ::io_uring_submit(&ring_); + if (r == -EBUSY) + process_completions(); + } + while (r == -EINTR || r == -EAGAIN || r == -EBUSY); + return r; +} + +inline int +uring_scheduler::submit_issued() noexcept +{ + bool const sqpoll = (ring_.flags & IORING_SETUP_SQPOLL) != 0; + int r = submit_retrying(); + // The kernel can take fewer entries than were flushed, and under + // SQPOLL takes them later on its own thread. + while (r >= 0 && ::io_uring_sq_ready(&ring_) > 0) + { + if (sqpoll) + std::this_thread::yield(); + else if (r == 0) + return -EBUSY; + // Under SQPOLL this also wakes the kernel thread if it went idle. + r = submit_retrying(); + } + return r; +} + +inline bool +uring_scheduler::queue_deferred_cancels() noexcept +{ + std::size_t kept = 0; + for (auto* op : deferred_cancels_) + if (!queue_cancel_locked(op)) + deferred_cancels_[kept++] = op; + deferred_cancels_.resize(kept); + has_deferred_cancels_.store(kept != 0, std::memory_order_release); + return kept == 0; +} + +inline ::io_uring_sqe* +uring_scheduler::acquire_sqe() noexcept +{ + if (has_deferred_cancels_.load(std::memory_order_relaxed) && + !queue_deferred_cancels()) + return nullptr; + ::io_uring_sqe* sqe = ::io_uring_get_sqe(&ring_); if (!sqe) { - io_uring_submit(&ring_); - sqe = io_uring_get_sqe(&ring_); + ::io_uring_submit(&ring_); + sqe = ::io_uring_get_sqe(&ring_); } - if (sqe) + return sqe; +} + +inline void +uring_scheduler::neutralize_pending_locked(int fd) noexcept +{ + // Without SQPOLL the kernel reads the SQ only inside + // io_uring_enter, which runs under ring_mutex_, so everything from + // its head on is still ours. The SQPOLL thread reads published + // entries on its own; only the unpublished ones are safe to touch. + auto& sq = ring_.sq; + bool const sqpoll = (ring_.flags & IORING_SETUP_SQPOLL) != 0; + unsigned const from = + sqpoll ? sq.sqe_head : __atomic_load_n(sq.khead, __ATOMIC_ACQUIRE); + + ready_queue local_ops; + for (unsigned i = from; i != sq.sqe_tail; ++i) + { + auto* sqe = &sq.sqes[i & sq.ring_mask]; + // NOLINTNEXTLINE(performance-no-int-to-ptr) — user_data holds the address io_uring_sqe_set_data stored + auto* ud = reinterpret_cast(sqe->user_data); + if (sqe->fd != fd || ud == nullptr || ud == &cancel_sentinel_ || + ud == &signal_pipe_sentinel_) + continue; + auto* op = static_cast(ud); + ::io_uring_prep_nop(sqe); + ::io_uring_sqe_set_data(sqe, &cancel_sentinel_); + op->sqe_set.store(false, std::memory_order_release); + (*op->cqe_func)(op, -ECANCELED, 0, local_ops); + } + if (!local_ops.empty()) { - io_uring_prep_cancel_fd(sqe, fd, IORING_ASYNC_CANCEL_ALL); - io_uring_sqe_set_data(sqe, &cancel_sentinel_); - inflight_inc(); + lock_type lock(dispatch_mutex_); + completed_ops_.splice(local_ops); + cond_.notify_one(); } - // Flush while fd is still open so the kernel resolves the file - // from the fd number before the caller closes and recycles it. - io_uring_submit(&ring_); } } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/uring/uring_socket_ops.hpp b/include/boost/corosio/native/detail/uring/uring_socket_ops.hpp index 42916b42c..a566eed29 100644 --- a/include/boost/corosio/native/detail/uring/uring_socket_ops.hpp +++ b/include/boost/corosio/native/detail/uring/uring_socket_ops.hpp @@ -428,10 +428,13 @@ uring_link_sqe( // expects exactly one F_MORE-less CQE per submitted SQE // (multishot ops decrement only on the terminal CQE). sched.inflight_inc(); - // Release pairs with the acquire in uring_op::request_cancel: - // a stop_token firing after we release the mutex will see - // sqe_set==true and submit a cancel-by-user_data SQE. - op->sqe_set.store(true, std::memory_order_release); + // Paired with uring_op::on_cancel: of a stop and this link, the + // later one sees the other and queues the cancel. A stop that came + // after the initiator's own check would otherwise find no SQE to + // cancel and leave the op armed. + op->sqe_set.store(true, std::memory_order_seq_cst); + if (op->cancelled.load(std::memory_order_seq_cst)) + sched.cancel_op_locked(op); // First submitter in a batch wins the CAS and will post // submit_sqes_op; others piggyback on the same flush. return !sched.submit_op_posted_exchange(true); @@ -472,14 +475,7 @@ uring_submit_op(uring_scheduler& sched, uring_op* op) noexcept { typename uring_scheduler::lock_type ring_lock(sched.ring_mutex()); - ::io_uring_sqe* sqe = ::io_uring_get_sqe(sched.ring()); - if (!sqe) - { - // SQ ring full — flush to kernel and retry once. - ::io_uring_submit(sched.ring()); - sqe = ::io_uring_get_sqe(sched.ring()); - } - + ::io_uring_sqe* sqe = sched.acquire_sqe(); if (!sqe) { // SQ stayed full after one flush — synchronous failure path. @@ -521,7 +517,7 @@ enum class uring_guarded_submit /** Submit an uncounted op if @p admit allows it. `admit` runs under the ring mutex before an SQE is taken. A cancel - flushed under that mutex (`cancel_and_flush`) therefore either + flushed under that mutex (`flush_after_cancel`) therefore either runs before the check, which then sees the owner's closing state, or follows this op's SQE in the submission queue and cancels it. @@ -551,12 +547,7 @@ uring_try_submit_op_if( if (!admit()) return uring_guarded_submit::refused; - ::io_uring_sqe* sqe = ::io_uring_get_sqe(sched.ring()); - if (!sqe) - { - ::io_uring_submit(sched.ring()); - sqe = ::io_uring_get_sqe(sched.ring()); - } + ::io_uring_sqe* sqe = sched.acquire_sqe(); if (!sqe) { op->res = -EAGAIN; @@ -662,8 +653,12 @@ struct uring_wait_op : uring_op { ec = make_err(EBADF); } - else if (!readiness_wait && (self->res & (POLLERR | POLLHUP))) + else if ( + !readiness_wait && (self->res & (POLLERR | POLLHUP)) && + !self->cancelled.load(std::memory_order_acquire)) { + // Not after a close: the number may name another socket now, + // and the probe would read and clear its error. int so_err = 0; socklen_t len = sizeof(so_err); if (::getsockopt(self->fd, SOL_SOCKET, SO_ERROR, &so_err, &len) < 0) diff --git a/include/boost/corosio/native/detail/uring/uring_stream_file.hpp b/include/boost/corosio/native/detail/uring/uring_stream_file.hpp index 52794a6f6..ea516a388 100644 --- a/include/boost/corosio/native/detail/uring/uring_stream_file.hpp +++ b/include/boost/corosio/native/detail/uring/uring_stream_file.hpp @@ -140,7 +140,7 @@ class BOOST_COROSIO_DECL uring_stream_file final void cancel() noexcept override { if (fd_ >= 0) - sched_->submit_cancel_by_fd(fd_); + sched_->cancel_ops({&rd_, &wr_}); } std::uint64_t size() const override @@ -181,11 +181,10 @@ class BOOST_COROSIO_DECL uring_stream_file final native_handle_type release() override { - // Flush the cancel while the fd is still open, so the kernel - // resolves it before the caller can close and recycle the - // number. + // Flush while the fd is still open, so no queued SQE can name + // a number the caller closes and recycles. if (fd_ >= 0) - sched_->cancel_and_flush(fd_); + sched_->flush_after_cancel(fd_, {&rd_, &wr_}); int fd = fd_; fd_ = -1; return fd; @@ -270,7 +269,7 @@ class BOOST_COROSIO_DECL uring_stream_file final // either kernel entry below; with the reader already gone // that raises SIGPIPE. scoped_sigpipe_block no_sigpipe; - sched_->cancel_and_flush(fd_); + sched_->flush_after_cancel(fd_, {&rd_, &wr_}); ::close(fd_); fd_ = -1; } diff --git a/include/boost/corosio/native/detail/uring/uring_types.hpp b/include/boost/corosio/native/detail/uring/uring_types.hpp index 6d50a062f..c5045879f 100644 --- a/include/boost/corosio/native/detail/uring/uring_types.hpp +++ b/include/boost/corosio/native/detail/uring/uring_types.hpp @@ -413,8 +413,12 @@ class BOOST_COROSIO_DECL uring_tcp_socket final // Flush while the fd is still open so the kernel resolves // pending SQEs before the caller can close and recycle the // number (same reasoning as close_socket). + // A connect completing after this belongs to the descriptor + // the caller takes; see close_socket(). + conn_.request_cancel(); + wait_op_.request_cancel(); if (fd_ >= 0) - sched_->cancel_and_flush(fd_); + sched_->flush_after_cancel(fd_, {&rd_, &wr_, &conn_, &wait_op_}); int fd = fd_; fd_ = -1; local_endpoint_ = endpoint{}; @@ -427,21 +431,21 @@ class BOOST_COROSIO_DECL uring_tcp_socket final void cancel() noexcept override { if (fd_ >= 0) - sched_->submit_cancel_by_fd(fd_); + sched_->cancel_ops({&rd_, &wr_, &conn_, &wait_op_}); } /// Cancel in-flight ops, close the fd, and reset cached endpoints. - /// Called by the service on close()/teardown. cancel_and_flush submits - /// the cancel SQE while the fd is still open so IORING_ASYNC_CANCEL_FD - /// resolves before the fd number can be recycled. + /// Called by the service on close()/teardown. The flush runs while + /// the fd is still open, so no queued SQE can name a recycled number. void close_socket() noexcept { // A connect completing after this must not write endpoints into // a closed impl that may be recycled. conn_.request_cancel(); + wait_op_.request_cancel(); if (fd_ >= 0) { - sched_->cancel_and_flush(fd_); + sched_->flush_after_cancel(fd_, {&rd_, &wr_, &conn_, &wait_op_}); ::close(fd_); fd_ = -1; } @@ -590,10 +594,7 @@ class BOOST_COROSIO_DECL uring_tcp_service final // LCOV_EXCL_START: dead — open() guards is_open(), so open_socket // never runs against an already-open fd (assign_socket handles that). if (sock.fd_ >= 0) - { - sched_->submit_cancel_by_fd(sock.fd_); - ::close(sock.fd_); - } + sock.close_socket(); // LCOV_EXCL_STOP sock.fd_ = fd; sock.family_ = family; @@ -762,6 +763,12 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor final BOOST_COROSIO_ASSERT(!wait_op_.stop_cb); } + /// Cancel this acceptor's readiness wait, if one is in flight. + void cancel_wait_op() noexcept + { + wait_op_.on_cancel(); + } + std::coroutine_handle<> accept( capy::continuation& cont, capy::executor_ref ex, @@ -891,7 +898,7 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor_service final }); for (auto* a : live) { - a->cancel(); + a->abort_all(); release(a); } } @@ -916,10 +923,8 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor_service final auto* acc = static_cast(h.get()); if (acc && acc->fd_ >= 0) { - // Disown the arming before the cancel, then flush the - // cancel while the fd still names the listener. - acc->drain_waiters_only(); - sched_->cancel_and_flush(acc->fd_); + // Close only once no queued SQE can name the listener. + acc->flush_listener(); ::close(acc->fd_); acc->fd_ = -1; acc->local_endpoint_ = endpoint{}; @@ -949,7 +954,7 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor_service final // never runs against an already-open fd (assign_socket handles that). if (acc.fd_ >= 0) { - sched_->submit_cancel_by_fd(acc.fd_); + acc.flush_listener(); ::close(acc.fd_); } // LCOV_EXCL_STOP @@ -1380,8 +1385,12 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final // Flush while the fd is still open so the kernel resolves // pending SQEs before the caller can close and recycle the // number (same reasoning as close_socket). + // A connect completing after this belongs to the descriptor + // the caller takes; see close_socket(). + conn_.request_cancel(); + wait_op_.request_cancel(); if (fd_ >= 0) - sched_->cancel_and_flush(fd_); + sched_->flush_after_cancel(fd_, {&rd_, &wr_, &conn_, &wait_op_}); int fd = fd_; fd_ = -1; local_endpoint_ = corosio::local_endpoint{}; @@ -1392,7 +1401,7 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final void cancel() noexcept override { if (fd_ >= 0) - sched_->submit_cancel_by_fd(fd_); + sched_->cancel_ops({&rd_, &wr_, &conn_, &wait_op_}); } /// Cancel in-flight ops, close the fd, and reset cached endpoints. @@ -1402,9 +1411,10 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final // A connect completing after this must not write endpoints into // a closed impl that may be recycled. conn_.request_cancel(); + wait_op_.request_cancel(); if (fd_ >= 0) { - sched_->cancel_and_flush(fd_); + sched_->flush_after_cancel(fd_, {&rd_, &wr_, &conn_, &wait_op_}); ::close(fd_); fd_ = -1; } @@ -1509,10 +1519,7 @@ class BOOST_COROSIO_DECL uring_local_stream_service final // LCOV_EXCL_START: dead — open() guards is_open(), so open_socket // never runs against an already-open fd (assign_socket handles that). if (sock.fd_ >= 0) - { - sched_->submit_cancel_by_fd(sock.fd_); - ::close(sock.fd_); - } + sock.close_socket(); // LCOV_EXCL_STOP sock.fd_ = fd; return {}; @@ -1635,6 +1642,12 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor final BOOST_COROSIO_ASSERT(!wait_op_.stop_cb); } + /// Cancel this acceptor's readiness wait, if one is in flight. + void cancel_wait_op() noexcept + { + wait_op_.on_cancel(); + } + std::coroutine_handle<> accept( capy::continuation& cont, capy::executor_ref ex, @@ -1762,7 +1775,7 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor_service final }); for (auto* a : live) { - a->cancel(); + a->abort_all(); release(a); } } @@ -1787,8 +1800,7 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor_service final if (acc && acc->fd_ >= 0) { // See uring_tcp_acceptor_service::close. - acc->drain_waiters_only(); - sched_->cancel_and_flush(acc->fd_); + acc->flush_listener(); ::close(acc->fd_); acc->fd_ = -1; acc->local_endpoint_ = corosio::local_endpoint{}; @@ -1818,7 +1830,7 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor_service final // never runs against an already-open fd (assign_socket handles that). if (acc.fd_ >= 0) { - sched_->submit_cancel_by_fd(acc.fd_); + acc.flush_listener(); ::close(acc.fd_); } // LCOV_EXCL_STOP @@ -2139,8 +2151,13 @@ class BOOST_COROSIO_DECL uring_udp_socket final // Flush while the fd is still open so the kernel resolves // pending SQEs before the caller can close and recycle the // number (same reasoning as close_socket). + // A connect completing after this belongs to the descriptor + // the caller takes; see close_socket(). + conn_.request_cancel(); + wait_op_.request_cancel(); if (fd_ >= 0) - sched_->cancel_and_flush(fd_); + sched_->flush_after_cancel( + fd_, {&conn_, &send_, &recv_, &wait_op_}); int fd = fd_; fd_ = -1; local_endpoint_ = endpoint{}; @@ -2151,7 +2168,7 @@ class BOOST_COROSIO_DECL uring_udp_socket final void cancel() noexcept override { if (fd_ >= 0) - sched_->submit_cancel_by_fd(fd_); + sched_->cancel_ops({&conn_, &send_, &recv_, &wait_op_}); } /// Cancel in-flight ops, close the fd, and reset cached endpoints. @@ -2161,9 +2178,11 @@ class BOOST_COROSIO_DECL uring_udp_socket final // A connect completing after this must not write endpoints into // a closed impl that may be recycled. conn_.request_cancel(); + wait_op_.request_cancel(); if (fd_ >= 0) { - sched_->cancel_and_flush(fd_); + sched_->flush_after_cancel( + fd_, {&conn_, &send_, &recv_, &wait_op_}); ::close(fd_); fd_ = -1; } @@ -2475,10 +2494,7 @@ class BOOST_COROSIO_DECL uring_udp_service final // LCOV_EXCL_START: dead — open() guards is_open(), so open_socket // never runs against an already-open fd (assign_socket handles that). if (sock.fd_ >= 0) - { - sched_->submit_cancel_by_fd(sock.fd_); - ::close(sock.fd_); - } + sock.close_socket(); // LCOV_EXCL_STOP sock.fd_ = fd; sock.family_ = family; @@ -2783,8 +2799,13 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final // Flush while the fd is still open so the kernel resolves // pending SQEs before the caller can close and recycle the // number (same reasoning as close_socket). + // A connect completing after this belongs to the descriptor + // the caller takes; see close_socket(). + conn_.request_cancel(); + wait_op_.request_cancel(); if (fd_ >= 0) - sched_->cancel_and_flush(fd_); + sched_->flush_after_cancel( + fd_, {&conn_, &send_, &recv_, &wait_op_}); int fd = fd_; fd_ = -1; local_endpoint_ = corosio::local_endpoint{}; @@ -2795,7 +2816,7 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final void cancel() noexcept override { if (fd_ >= 0) - sched_->submit_cancel_by_fd(fd_); + sched_->cancel_ops({&conn_, &send_, &recv_, &wait_op_}); } /// Cancel in-flight ops, close the fd, and reset cached endpoints. @@ -2805,9 +2826,11 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final // A connect completing after this must not write endpoints into // a closed impl that may be recycled. conn_.request_cancel(); + wait_op_.request_cancel(); if (fd_ >= 0) { - sched_->cancel_and_flush(fd_); + sched_->flush_after_cancel( + fd_, {&conn_, &send_, &recv_, &wait_op_}); ::close(fd_); fd_ = -1; } @@ -3135,10 +3158,7 @@ class BOOST_COROSIO_DECL uring_local_datagram_service final // LCOV_EXCL_START: dead — open() guards is_open(), so open_socket // never runs against an already-open fd (assign_socket handles that). if (sock.fd_ >= 0) - { - sched_->submit_cancel_by_fd(sock.fd_); - ::close(sock.fd_); - } + sock.close_socket(); // LCOV_EXCL_STOP sock.fd_ = fd; return {}; diff --git a/test/unit/fault/fault_uring.cpp b/test/unit/fault/fault_uring.cpp index de692579c..c2789a64f 100644 --- a/test/unit/fault/fault_uring.cpp +++ b/test/unit/fault/fault_uring.cpp @@ -90,7 +90,18 @@ apply_sq_fill(io_uring* ring) noexcept { st.ring = ring; st.saved_tail = ring->sq.sqe_tail; - ring->sq.sqe_tail += io_uring_sq_space_left(ring); + // The claimed entries hold whatever earlier requests left + // there; blank them so nothing reading the queue mistakes + // them for queued requests. + unsigned const space = io_uring_sq_space_left(ring); + for (unsigned i = 0; i != space; ++i) + { + auto& sqe = + ring->sq.sqes[(st.saved_tail + i) & ring->sq.ring_mask]; + io_uring_prep_nop(&sqe); + sqe.fd = -1; + } + ring->sq.sqe_tail += space; st.filled = true; s->fired = true; } diff --git a/test/unit/fault/uring_faults.cpp b/test/unit/fault/uring_faults.cpp index 972d76bdd..133ce5dca 100644 --- a/test/unit/fault/uring_faults.cpp +++ b/test/unit/fault/uring_faults.cpp @@ -32,6 +32,8 @@ #include #include + +#include #include #include #include @@ -202,34 +204,31 @@ struct uring_faults acc.close(); } - /* Cancellation is best-effort when the submission queue is full. + /* A full submission queue does not hold a close. A ring clamped to one SQE spends it on the wakeup poll, and the flush that would free it is a no-op while the arm is alive, so every cancel below finds the queue still full after its own - retry and gives up. Nothing observable changes — the descriptor - is closed either way — which is why the arm's own report is the - only witness that the give-up was taken. + retry. Cancels name operations rather than the descriptor, so a + close still closes at once. */ void testCancelSqFull() { std::optional f; f.emplace(sys::uring_sqe_full, 0); io_context ioc(uring); + int before = 0; { - // close() submits the cancel while the descriptor is still - // open, so the kernel resolves it before the number can be - // recycled. tcp_socket s(ioc); BOOST_TEST(!s.open(family::v4)); - int const before = open_fds(); + before = open_fds(); s.close(); BOOST_TEST(!s.is_open()); BOOST_TEST_EQ(open_fds(), before - 1); } { - // cancel() reaches the by-descriptor cancel without going - // through a close. + // cancel() reaches the cancel path without going through a + // close. tcp_socket s(ioc); BOOST_TEST(!s.open(family::v4)); s.cancel(); @@ -243,6 +242,12 @@ struct uring_faults } BOOST_TEST(f->fired()); f.reset(); + + // Nothing was left waiting for the queue to drain. + capy::run_async(ioc.get_executor())( + []() -> capy::task<> { co_return; }()); + ioc.poll(); + BOOST_TEST_EQ(open_fds(), before - 1); } void testWaitFails() @@ -326,6 +331,291 @@ struct uring_faults BOOST_TEST_GE(waits->count(), 1u); } + // A connect whose success arrives after release() belongs to the + // descriptor the caller took away: it must not leave endpoints on + // the socket, which may already hold a new descriptor. The kernel's answer is rewritten to success + // so it lands after the release whatever the cancel did. + void testReleaseDuringConnect() + { + io_context ioc(uring); + auto ex = ioc.get_executor(); + tcp_acceptor acc(ioc); + BOOST_TEST(!acc.open(family::v4)); + BOOST_TEST(!acc.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!acc.listen()); + + tcp_socket s(ioc); + BOOST_TEST(!s.open(family::v4)); + cqe_fault_scope q(-1, IORING_OP_CONNECT, 0); + bool started = false; + bool done = false; + capy::run_async(ex)( + [](tcp_socket& s, endpoint ep, bool& started, + bool& done) -> capy::task<> { + started = true; + std::ignore = co_await s.connect(ep); + done = true; + }(s, acc.local_endpoint(), started, done)); + while (!started && ioc.poll_one() != 0) + { + } + BOOST_TEST(started); + + int released = static_cast(s.release()); + // The reopened socket must not inherit the old connect's + // endpoints when its completion runs. + BOOST_TEST(!s.open(family::v4)); + ioc.restart(); + while (!done && ioc.run_one_for(std::chrono::seconds(5)) != 0) + { + } + BOOST_TEST(done); + BOOST_TEST(q.fired()); + BOOST_TEST(s.remote_endpoint() == endpoint{}); + BOOST_TEST(s.local_endpoint() == endpoint{}); + ::close(released); + } + + // An error wait whose completion runs after the socket was closed + // and reopened must not probe SO_ERROR through the old descriptor + // number, which may now name another socket. The cancelled poll's + // answer is rewritten to an error so it reaches that probe. + void testErrorWaitAfterCloseSkipsProbe() + { + io_context ioc(uring); + auto ex = ioc.get_executor(); + tcp_socket s(ioc); + BOOST_TEST(!s.open(family::v4)); + + cqe_fault_scope q(-1, IORING_OP_POLL_ADD, POLLERR); + bool started = false; + bool done = false; + std::error_code wec; + capy::run_async(ex)( + [](tcp_socket& s, bool& started, bool& done, + std::error_code& ec) -> capy::task<> { + started = true; + auto [e] = co_await s.wait(wait_type::error); + ec = e; + done = true; + }(s, started, done, wec)); + while (!started && ioc.poll_one() != 0) + { + } + BOOST_TEST(started); + + s.close(); + BOOST_TEST(!s.open(family::v4)); + ioc.restart(); + while (!done && ioc.run_one_for(std::chrono::seconds(5)) != 0) + { + } + BOOST_TEST(done); + BOOST_TEST(q.fired()); + BOOST_TEST(wec == capy::cond::canceled); + } + + // A close whose cancel cannot be queued must not strand the + // request: the descriptor stays open until the cancel reaches the + // kernel, and the receive completes canceled. + void testCloseWithFullQueueStillCancels() + { + io_context ioc(uring); + char buf[8]; + udp_socket s(ioc); + BOOST_TEST(!s.open(family::v4)); + BOOST_TEST(!s.bind(endpoint(ipv4_address::loopback(), 0))); + bool done = false; + std::error_code rec; + capy::run_async(ioc.get_executor())( + [](udp_socket& s, char* b, bool& done, + std::error_code& ec) -> capy::task<> { + endpoint from; + auto [e, n] = + co_await s.recv_from(capy::mutable_buffer(b, 8), from); + std::ignore = n; + ec = e; + done = true; + }(s, buf, done, rec)); + bool fired = false; + auto closer = [&]() -> capy::task<> { + // A run-loop flush applies the fill; see + // testCancelSqFullBestEffort. + fault_scope f(sys::uring_sq_fill, 0); + std::ignore = co_await corosio::delay(std::chrono::milliseconds(1)); + s.close(); + fired = f.fired(); + }; + capy::run_async(ioc.get_executor())(closer()); + while (!done && ioc.run_one_for(std::chrono::seconds(5)) != 0) + { + } + BOOST_TEST(fired); + BOOST_TEST(done); + BOOST_TEST(rec == capy::cond::canceled); + } + + // release() hands the descriptor to a caller who may close it at + // once. A cancel that could not reach the kernel first must still + // end the parked receive, and must not later cancel whatever + // reuses the number. + void testReleaseWithFullQueueStillCancels() + { + io_context ioc(uring); + char buf[8]; + udp_socket s(ioc); + BOOST_TEST(!s.open(family::v4)); + BOOST_TEST(!s.bind(endpoint(ipv4_address::loopback(), 0))); + bool done = false; + std::error_code rec; + capy::run_async(ioc.get_executor())( + [](udp_socket& s, char* b, bool& done, + std::error_code& ec) -> capy::task<> { + endpoint from; + auto [e, n] = + co_await s.recv_from(capy::mutable_buffer(b, 8), from); + std::ignore = n; + ec = e; + done = true; + }(s, buf, done, rec)); + bool fired = false; + auto releaser = [&]() -> capy::task<> { + // A run-loop flush applies the fill; see + // testCancelSqFullBestEffort. + fault_scope f(sys::uring_sq_fill, 0); + std::ignore = co_await corosio::delay(std::chrono::milliseconds(1)); + ::close(static_cast(s.release())); + fired = f.fired(); + }; + capy::run_async(ioc.get_executor())(releaser()); + while (!done && ioc.run_one_for(std::chrono::seconds(5)) != 0) + { + } + BOOST_TEST(fired); + BOOST_TEST(done); + BOOST_TEST(rec == capy::cond::canceled); + } + + // A release that could not queue its cancel still hands back the + // descriptor itself and cancels the receive later. That cancel names + // the receive, so another object adopting the descriptor keeps its + // own receive. + void testDeferredCancelSparesReadoptedDescriptor() + { + io_context ioc(uring); + char buf[8]; + udp_socket s(ioc); + BOOST_TEST(!s.open(family::v4)); + BOOST_TEST(!s.bind(endpoint(ipv4_address::loopback(), 0))); + int const orig = static_cast(s.native_handle()); + bool done1 = false; + std::error_code rec1; + capy::run_async(ioc.get_executor())( + [](udp_socket& s, char* b, bool& done, + std::error_code& ec) -> capy::task<> { + endpoint from; + auto [e, n] = + co_await s.recv_from(capy::mutable_buffer(b, 8), from); + std::ignore = n; + ec = e; + done = true; + }(s, buf, done1, rec1)); + + udp_socket s2(ioc); + char buf2[8]; + bool done2 = false; + std::error_code rec2; + std::size_t n2 = 0; + int released = -1; + bool fired = false; + auto releaser = [&]() -> capy::task<> { + { + // A run-loop flush applies the fill; see + // testCancelSqFullBestEffort. + fault_scope f(sys::uring_sq_fill, 0); + std::ignore = + co_await corosio::delay(std::chrono::milliseconds(1)); + released = static_cast(s.release()); + fired = f.fired(); + } + BOOST_TEST(!s2.assign(released)); + endpoint from; + auto [e, n] = + co_await s2.recv_from(capy::mutable_buffer(buf2, 8), from); + rec2 = e; + n2 = n; + done2 = true; + }; + capy::run_async(ioc.get_executor())(releaser()); + while (!done1 && ioc.run_one_for(std::chrono::seconds(5)) != 0) + { + } + BOOST_TEST(fired); + BOOST_TEST_EQ(released, orig); + BOOST_TEST(done1); + BOOST_TEST(rec1 == capy::cond::canceled); + + // The deferred cancel has run; the adopter's receive must still + // be waiting for data. + udp_socket sender(ioc); + BOOST_TEST(!sender.open(family::v4)); + auto to = s2.local_endpoint(); + capy::run_async(ioc.get_executor())( + [](udp_socket& u, endpoint ep) -> capy::task<> { + std::ignore = + co_await u.send_to(capy::const_buffer("hi", 2), ep); + }(sender, to)); + while (!done2 && ioc.run_one_for(std::chrono::seconds(5)) != 0) + { + } + BOOST_TEST(done2); + BOOST_TEST(!rec2); + BOOST_TEST_EQ(n2, 2u); + s2.close(); + ioc.restart(); + ioc.run(); + } + + // A close whose flush fails must not leave an SQE that still names + // the descriptor by number: the queued receive completes canceled + // before the number can be reused. + void testCloseWithFailedFlushCancelsQueuedOp() + { + io_context ioc(uring); + char buf[8]; + udp_socket s(ioc); + BOOST_TEST(!s.open(family::v4)); + BOOST_TEST(!s.bind(endpoint(ipv4_address::loopback(), 0))); + bool done = false; + std::error_code rec; + bool fired = false; + // The receive links its SQE and posts the flush behind the + // closer, which therefore closes with the SQE still queued. + capy::run_async(ioc.get_executor())( + [](udp_socket& s, char* b, bool& done, + std::error_code& ec) -> capy::task<> { + endpoint from; + auto [e, n] = + co_await s.recv_from(capy::mutable_buffer(b, 8), from); + std::ignore = n; + ec = e; + done = true; + }(s, buf, done, rec)); + capy::run_async(ioc.get_executor())( + [](udp_socket& s, bool& fired) -> capy::task<> { + fault_scope f(sys::io_uring_submit, ENOMEM); + s.close(); + fired = f.fired(); + co_return; + }(s, fired)); + while (!done && ioc.run_one_for(std::chrono::seconds(5)) != 0) + { + } + BOOST_TEST(fired); + BOOST_TEST(done); + BOOST_TEST(rec == capy::cond::canceled); + } + // Closing and destroying a listening acceptor on a live context // leaves the arming's terminal completion to the run loop. Waiting // on the ring for it instead would swallow other ops' completions; @@ -369,7 +659,8 @@ struct uring_faults return; } make_native_adoptable(sv[1]); - fault_scope f(sys::uring_sqe_full, 0); + std::optional f; + f.emplace(sys::uring_sqe_full, 0); fault_scope g(sys::io_uring_submit_and_get_events, EBADF); io_context ioc(uring); local_stream_socket b(ioc); @@ -397,7 +688,9 @@ struct uring_faults }; capy::run_async(ioc.get_executor())(read_body()); ioc.run(); - BOOST_TEST(f.fired()); + BOOST_TEST(f->fired()); + // A real flush would take shutdown's own cancel. + f.reset(); BOOST_TEST(g.fired()); BOOST_TEST(rec == std::errc::resource_unavailable_try_again); BOOST_TEST_EQ(n1, 0u); @@ -417,7 +710,8 @@ struct uring_faults return; } make_native_adoptable(sv[1]); - fault_scope f(sys::uring_sqe_full, 0); + std::optional f; + f.emplace(sys::uring_sqe_full, 0); fault_scope g(sys::io_uring_submit_and_get_events, EBADF); io_context ioc(uring); local_stream_socket b(ioc); @@ -436,7 +730,9 @@ struct uring_faults }; capy::run_async(ioc.get_executor())(write_body()); ioc.run(); - BOOST_TEST(f.fired()); + BOOST_TEST(f->fired()); + // A real flush would take shutdown's own cancel. + f.reset(); BOOST_TEST(g.fired()); BOOST_TEST(wec == std::errc::resource_unavailable_try_again); BOOST_TEST_EQ(wn, 0u); @@ -454,7 +750,8 @@ struct uring_faults return; } make_native_adoptable(sv[1]); - fault_scope f(sys::uring_sqe_full, 0); + std::optional f; + f.emplace(sys::uring_sqe_full, 0); fault_scope g(sys::io_uring_submit_and_get_events, EBADF); io_context ioc(uring); local_stream_socket b(ioc); @@ -466,7 +763,9 @@ struct uring_faults }; capy::run_async(ioc.get_executor())(wait_body()); ioc.run(); - BOOST_TEST(f.fired()); + BOOST_TEST(f->fired()); + // A real flush would take shutdown's own cancel. + f.reset(); BOOST_TEST(g.fired()); BOOST_TEST(pec == std::errc::resource_unavailable_try_again); BOOST_TEST(b.is_open()); @@ -1133,6 +1432,27 @@ struct uring_faults }); } + // Under unsafe_io one thread runs the ring, and when it arms a timer + // itself it is not in a kernel wait: waking it costs a syscall per + // arm for nothing. + void testUnsafeIoRunThreadArmsWithoutWake() + { + io_context_options opts; + opts.locking = locking_mode::unsafe_io; + io_context ioc(uring, opts, 1); + constexpr int arms = 200; + capy::run_async(ioc.get_executor())([]() -> capy::task<> { + for (int i = 0; i < arms; ++i) + std::ignore = + co_await corosio::delay(std::chrono::microseconds(1)); + }()); + // Counts this thread's writes; the call it would fail is never + // reached. + fault_scope writes(sys::write, EIO, 1u << 30); + ioc.run(); + BOOST_TEST_LT(writes.count(), unsigned(arms / 10)); + } + void run() { testRingInitSqExhaustion(); @@ -1152,6 +1472,13 @@ struct uring_faults testAcceptorCloseDoesNotWaitOnRing(); testShutdownWaitRetriesAfterEintr(); testShutdownCancelRetriesAfterEintr(); + testReleaseDuringConnect(); + testErrorWaitAfterCloseSkipsProbe(); + testCloseWithFullQueueStillCancels(); + testReleaseWithFullQueueStillCancels(); + testDeferredCancelSparesReadoptedDescriptor(); + testCloseWithFailedFlushCancelsQueuedOp(); + testUnsafeIoRunThreadArmsWithoutWake(); testAcceptorArmSqFull(); testSqFull(); testConnectCqeRewrite(); diff --git a/test/unit/io_context.cpp b/test/unit/io_context.cpp index f0a8a73e8..2d5207896 100644 --- a/test/unit/io_context.cpp +++ b/test/unit/io_context.cpp @@ -9,6 +9,7 @@ // Test that header file is self-contained. #include +#include #include #include @@ -19,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -687,6 +689,25 @@ struct io_context_test ex.on_work_finished(); } + // A timer armed beyond the bound must not stretch run_for past it. + // A genuine hang surfaces as a harness timeout. + void testRunForWithLaterTimer() + { + io_context ioc(Backend); + std::stop_source stop; + capy::run_async(ioc.get_executor(), stop.get_token())( + []() -> capy::task<> { + std::ignore = co_await delay(std::chrono::hours(1)); + }()); + + std::ignore = ioc.run_for(std::chrono::milliseconds(50)); + BOOST_TEST(ioc.run_one_for(std::chrono::milliseconds(50)) == 0); + + stop.request_stop(); + ioc.restart(); + ioc.run(); + } + void testExecutorRunningInThisThread() { io_context ioc(Backend); @@ -1003,6 +1024,7 @@ struct io_context_test testRunFor(); testRunForWithOutstandingWork(); testRunOneForWithOutstandingWork(); + testRunForWithLaterTimer(); testExecutorRunningInThisThread(); testMultithreaded(); testHintOneIsThreadSafe(); diff --git a/test/unit/local_stream_socket.cpp b/test/unit/local_stream_socket.cpp index 3bfcae4de..33173e5fa 100644 --- a/test/unit/local_stream_socket.cpp +++ b/test/unit/local_stream_socket.cpp @@ -529,6 +529,62 @@ struct local_stream_socket_test closed.shutdown(shutdown_send) == std::errc::bad_file_descriptor); } +#if BOOST_COROSIO_POSIX + // Two sockets on one open file: cancelling, closing or releasing + // one must leave the other's pending read alone. + void doSharedFileSpared(int how) + { + io_context ioc(Backend); + int fds[2]; + BOOST_TEST(::socketpair(AF_UNIX, SOCK_STREAM, 0, fds) == 0); + // Sockets are adopted as given; the duplicate shares the flag. + ::fcntl(fds[0], F_SETFL, ::fcntl(fds[0], F_GETFL) | O_NONBLOCK); + local_stream_socket a(ioc), b(ioc); + BOOST_TEST(!a.assign(static_cast(fds[0]))); + BOOST_TEST(!b.assign(static_cast(::dup(fds[0])))); + + std::error_code ec; + std::size_t n = 0; + bool done = false; + char buf[8]{}; + capy::run_async(ioc.get_executor())( + [](local_stream_socket& b, char* p, std::error_code& ec, + std::size_t& n, bool& done) -> capy::task<> { + auto [rec, rn] = + co_await b.read_some(capy::mutable_buffer(p, 8)); + ec = rec; + n = rn; + done = true; + }(b, buf, ec, n, done)); + ioc.poll(); + + native_handle_type released = -1; + if (how == 0) + a.cancel(); + else if (how == 1) + a.close(); + else + released = a.release(); + BOOST_TEST_EQ(::write(fds[1], "x", 1), 1); + ioc.restart(); + ioc.run(); + + BOOST_TEST(done); + BOOST_TEST(!ec); + BOOST_TEST_EQ(n, 1u); + if (released >= 0) + ::close(released); + ::close(fds[1]); + } + + void testSharedFileSpared() + { + doSharedFileSpared(0); + doSharedFileSpared(1); + doSharedFileSpared(2); + } +#endif + #if BOOST_COROSIO_POSIX // assign() over an open socket is refused and leaves its pending // operations undisturbed. @@ -1969,6 +2025,7 @@ struct local_stream_socket_test void run() { #if BOOST_COROSIO_POSIX + testSharedFileSpared(); testAcceptedPeerClosedWhenAwaiterDestroyed(false); testAcceptedPeerClosedWhenAwaiterDestroyed(true); #endif diff --git a/test/unit/native/uring/descriptor_continue.cpp b/test/unit/native/uring/descriptor_continue.cpp index 4c4d35504..5dda6dd19 100644 --- a/test/unit/native/uring/descriptor_continue.cpp +++ b/test/unit/native/uring/descriptor_continue.cpp @@ -158,8 +158,9 @@ struct uring_descriptor_continue_test void testCancelInTheGapStopsTheRearm() { // cancel() lands while the op is out of the ring, so its - // cancel-by-fd SQE finds nothing and the op's own `cancelled` - // flag is never set. The epoch is the only thing that sees it. + // cancel has nothing there to reach and the op's own + // `cancelled` flag is never set. The epoch is the only thing + // that sees it. uring_descriptor_test_context ctx; auto d = ctx.make_descriptor(); diff --git a/test/unit/native/uring/mt_scheduler.cpp b/test/unit/native/uring/mt_scheduler.cpp index 0e1f2bfb5..761864a0b 100644 --- a/test/unit/native/uring/mt_scheduler.cpp +++ b/test/unit/native/uring/mt_scheduler.cpp @@ -18,22 +18,81 @@ #if BOOST_COROSIO_HAS_URING +#include #include +#include +#include #include #include #include #include +#include #include #include #include #include +#include +#include +#include +#include #include +#include +#include +#include +#include + namespace boost::corosio { +// A poll on a pipe that never becomes readable, so only a cancel can +// complete it. +struct cancel_probe_op : detail::uring_op +{ + int fd = -1; + std::atomic done{false}; + int result = 0; + + cancel_probe_op() noexcept : uring_op(&on_done, &on_cqe, &prep) {} + + static void prep(detail::uring_op* base, ::io_uring_sqe* sqe) noexcept + { + ::io_uring_prep_poll_add( + sqe, static_cast(base)->fd, POLLIN); + } + + static void on_cqe( + detail::uring_op* base, + int res, + unsigned /*flags*/, + detail::ready_queue& local) noexcept + { + base->res = res; + local.push(base); + } + + static void on_done( + void* /*owner*/, + detail::scheduler_op* base, + std::uint32_t /*bytes*/, + std::uint32_t /*error*/) noexcept + { + auto* self = static_cast(base); + self->result = self->res; + self->done = true; + } +}; + +struct uring_probe_context : native_io_context +{ + detail::uring_scheduler& scheduler() noexcept + { + return *static_cast(sched_); + } +}; + struct uring_mt_scheduler_test { void testForeignPostWakesParkedFollower() @@ -96,9 +155,141 @@ struct uring_mt_scheduler_test BOOST_TEST(resumed); } + // unsafe_io keeps the scheduler lock, so a foreign thread may post; + // the run thread blocked in the kernel must wake for it rather than + // sleep until the pending hour-long timer expires. + void testForeignPostWakesUnsafeIoRun() + { + io_context_options opts; + opts.locking = locking_mode::unsafe_io; + native_io_context ioc(opts, 1); + auto ex = ioc.get_executor(); + + std::stop_source keep; + capy::run_async(ex, keep.get_token())([]() -> capy::task<> { + std::ignore = co_await delay(std::chrono::hours(1)); + }()); + + std::atomic finished{false}; + std::thread watchdog([&] { + for (int i = 0; i < 300 && !finished.load(); ++i) + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + if (!finished.load()) + { + std::fputs("unsafe_io: foreign post never woke run()\n", stderr); + std::abort(); + } + }); + std::thread poster([&] { + // Lets run() block in the kernel first; a post that lands + // earlier only lets the test pass without exercising the wake. + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + capy::run_async(ex)([](std::stop_source& k) -> capy::task<> { + k.request_stop(); + co_return; + }(keep)); + }); + + ioc.run(); + finished = true; + poster.join(); + watchdog.join(); + } + + // With a kernel thread polling the submission queue, submitting a + // cancel only hands it over. Closing the descriptor before that + // thread resolves the number would leave the read armed for good. + void testSqpollCloseCancelsParkedRead() + { + io_context_options opts; + opts.enable_sqpoll = true; + opts.sq_thread_idle_ms = 1000; + native_io_context ioc(opts, 1); + int stranded = 0; + for (int i = 0; i < 50; ++i) + { + int sv[2]; + BOOST_TEST( + ::socketpair( + AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK | SOCK_CLOEXEC, 0, + sv) == 0); + local_stream_socket s(ioc); + BOOST_TEST(!s.assign(sv[0])); + bool done = false; + std::error_code rec; + char buf[8]; + capy::run_async(ioc.get_executor())( + [](local_stream_socket& s, char* b, bool& done, + std::error_code& ec) -> capy::task<> { + auto [e, n] = + co_await s.read_some(capy::mutable_buffer(b, 8)); + std::ignore = n; + ec = e; + done = true; + }(s, buf, done, rec)); + ioc.restart(); + std::ignore = ioc.poll(); + // Lets the kernel thread take the read, so it is armed. + std::this_thread::sleep_for(std::chrono::microseconds(200)); + s.close(); + ioc.restart(); + // Bounded only to fail rather than hang. + for (int k = 0; k < 50 && !done; ++k) + std::ignore = ioc.run_one_for(std::chrono::milliseconds(20)); + if (!done || rec != capy::cond::canceled) + ++stranded; + ::close(sv[1]); + ioc.restart(); + while (!done && ioc.run_one_for(std::chrono::seconds(1)) != 0) + { + } + } + BOOST_TEST_EQ(stranded, 0); + } + + // A stop that lands after an initiator found the op uncancelled, + // but before its SQE is linked, finds nothing to cancel yet; the + // link must cancel the op itself. + void testStopBeforeLinkCancelsOp() + { + uring_probe_context ctx; + auto& sched = ctx.scheduler(); + int p[2]; + BOOST_TEST_EQ(::pipe2(p, O_NONBLOCK | O_CLOEXEC), 0); + + cancel_probe_op op; + op.fd = p[0]; + op.sched_ = &sched; + std::stop_source src; + op.start(src.get_token()); + sched.work_started(); + + src.request_stop(); + detail::uring_submit_op(sched, &op); + + // Bounds a hang, not a timing assertion. + while (!op.done && ctx.run_one_for(std::chrono::seconds(5)) != 0) + { + } + BOOST_TEST(op.done); + BOOST_TEST_EQ(op.result, -ECANCELED); + + if (!op.done) + { + std::ignore = ::write(p[1], "x", 1); + ctx.restart(); + ctx.run(); + } + ::close(p[0]); + ::close(p[1]); + } + void run() { + testStopBeforeLinkCancelsOp(); testForeignPostWakesParkedFollower(); + testForeignPostWakesUnsafeIoRun(); + testSqpollCloseCancelsParkedRead(); } }; diff --git a/test/unit/posix_stream_descriptor.cpp b/test/unit/posix_stream_descriptor.cpp index 99a305fe7..5659b4b92 100644 --- a/test/unit/posix_stream_descriptor.cpp +++ b/test/unit/posix_stream_descriptor.cpp @@ -437,6 +437,59 @@ struct posix_stream_descriptor_test ::close(fds[1]); } + // Two descriptors on one open file: cancelling, closing or + // releasing one must leave the other's pending read alone. + void doSharedFileSpared(int how) + { + io_context ioc(Backend); + int fds[2]; + make_pipe(fds); + posix_stream_descriptor a(ioc); + posix_stream_descriptor b(ioc); + BOOST_TEST(!a.assign(fds[0])); + BOOST_TEST(!b.assign(::dup(fds[0]))); + + std::error_code ec; + std::size_t n = 0; + bool done = false; + char buf[8]{}; + capy::run_async(ioc.get_executor())( + [](posix_stream_descriptor& b, char* p, std::error_code& ec, + std::size_t& n, bool& done) -> capy::task<> { + auto [rec, rn] = + co_await b.read_some(capy::mutable_buffer(p, 8)); + ec = rec; + n = rn; + done = true; + }(b, buf, ec, n, done)); + ioc.poll(); + + native_handle_type released = -1; + if (how == 0) + a.cancel(); + else if (how == 1) + a.close(); + else + released = a.release(); + BOOST_TEST_EQ(::write(fds[1], "x", 1), 1); + ioc.restart(); + ioc.run(); + + BOOST_TEST(done); + BOOST_TEST(!ec); + BOOST_TEST_EQ(n, 1u); + if (released >= 0) + ::close(released); + ::close(fds[1]); + } + + void testSharedFileSpared() + { + doSharedFileSpared(0); + doSharedFileSpared(1); + doSharedFileSpared(2); + } + void testEofOnClosedWriteEnd() { io_context ioc(Backend); @@ -1407,6 +1460,7 @@ struct posix_stream_descriptor_test void run() { + testSharedFileSpared(); testConstruction(); testAssignRejectsRegularFile(); testAssignRejectsBadFd(); diff --git a/test/unit/tcp_acceptor.cpp b/test/unit/tcp_acceptor.cpp index ddd0ade03..d1193ef2d 100644 --- a/test/unit/tcp_acceptor.cpp +++ b/test/unit/tcp_acceptor.cpp @@ -282,6 +282,64 @@ struct tcp_acceptor_test // Cancellation Tests + // cancel() aborts the pending accept but leaves the acceptor + // listening: a later accept must still take the next connection. + void testAcceptAfterCancel() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + tcp_acceptor acc(ioc); + BOOST_TEST(!acc.open()); + acc.set_option(socket_option::reuse_address(true)); + BOOST_TEST(!acc.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!acc.listen()); + endpoint ep(ipv4_address::loopback(), acc.local_endpoint().port()); + + tcp_socket s1(ioc); + std::error_code ec1; + bool done1 = false; + capy::run_async(ex)( + [](tcp_acceptor& a, tcp_socket& s, std::error_code& ec, + bool& done) -> capy::task<> { + auto [e] = co_await a.accept(s); + ec = e; + done = true; + }(acc, s1, ec1, done1)); + std::ignore = ioc.poll(); + acc.cancel(); + ioc.restart(); + std::ignore = ioc.run(); + BOOST_TEST(done1); + BOOST_TEST(ec1 == capy::cond::canceled); + + tcp_socket s2(ioc); + tcp_socket client(ioc); + std::error_code ec2; + bool done2 = false; + capy::run_async(ex)( + [](tcp_acceptor& a, tcp_socket& s, std::error_code& ec, + bool& done) -> capy::task<> { + auto [e] = co_await a.accept(s); + ec = e; + done = true; + }(acc, s2, ec2, done2)); + capy::run_async(ex)([](tcp_socket& c, endpoint e) -> capy::task<> { + std::ignore = co_await c.connect(e); + }(client, ep)); + + // Bounds a hang, not a timing assertion. + ioc.restart(); + while (!done2 && ioc.run_one_for(std::chrono::seconds(5)) != 0) + { + } + BOOST_TEST(done2); + BOOST_TEST(!ec2); + acc.close(); + ioc.restart(); + std::ignore = ioc.run(); + } + void testCancelAccept() { // Tests that cancel() properly cancels a pending accept operation. @@ -2126,6 +2184,7 @@ struct tcp_acceptor_test // Cancellation testCancelAccept(); + testAcceptAfterCancel(); testDeferredExecutorCompletion(); #if BOOST_COROSIO_POSIX testQueuedAcceptClosesPeerOnTeardown(); diff --git a/test/unit/teardown_inflight.cpp b/test/unit/teardown_inflight.cpp index f5fd40ee2..efb5d9378 100644 --- a/test/unit/teardown_inflight.cpp +++ b/test/unit/teardown_inflight.cpp @@ -316,7 +316,7 @@ struct uring_teardown_test // scoped_sigpipe_block kills the process. After two run_one() // calls what is in the ring is a WRITEV parked in the kernel's // internal poll, and the SIGPIPE cover for its flush comes from - // cancel_and_flush's own scoped_sigpipe_block, so deleting + // flush_after_cancel's own scoped_sigpipe_block, so deleting // uring_descriptor::close_descriptor()'s guard leaves this // passing. What this test pins is the drain coverage named // above. From 1be667f6d977ae7a6894f70499b57bf5c6e36e9a Mon Sep 17 00:00:00 2001 From: Steve Gerbino Date: Fri, 9 Oct 2026 20:45:06 +0200 Subject: [PATCH 5/8] fix: correct IOCP cancellation, accept, adoption and wait races Socket cancellation is re-checked before an operation can complete, and connects and lookups stopped before they are issued are cancelled. An object handle's wait is reserved atomically and its stop callback engaged before it is published. An accept's peer is constructed before the accept is armed, the local acceptor's close is defined in every local socket translation unit, and sockets in skip-on-success mode are refused for adoption. The wait reactor skips the SO_ERROR probe for a closed socket, ends a wait whose stop lands while it registers, and no longer has cancels queued before it runs. Shutdown sweeps the port past the stop signal, and signal-wait completions are posted instead of resumed inline. --- .../boost/corosio/local_stream_acceptor.hpp | 6 + include/boost/corosio/local_stream_socket.hpp | 6 + .../win_local_stream_acceptor_service.hpp | 21 +- .../detail/iocp/win_local_stream_service.hpp | 32 ++- .../detail/iocp/win_object_handle_service.hpp | 32 ++- .../native/detail/iocp/win_resolver.hpp | 6 + .../detail/iocp/win_resolver_service.hpp | 31 ++- .../native/detail/iocp/win_scheduler.hpp | 12 +- .../native/detail/iocp/win_signals.hpp | 16 +- .../detail/iocp/win_tcp_acceptor_service.hpp | 17 +- .../native/detail/iocp/win_tcp_service.hpp | 34 ++- .../native/detail/iocp/win_udp_service.hpp | 30 ++- .../detail/iocp/win_validate_handle.hpp | 32 ++- .../native/detail/iocp/win_wait_reactor.hpp | 56 +++-- .../native/native_local_stream_socket.hpp | 2 +- include/boost/corosio/tcp_acceptor.hpp | 6 + include/boost/corosio/tcp_socket.hpp | 6 + include/boost/corosio/udp_socket.hpp | 6 + test/unit/fault/iocp_faults.cpp | 69 ++++-- test/unit/io_context.cpp | 20 ++ test/unit/iocp_paths.cpp | 199 ++++++++++++++++++ test/unit/signal_set.cpp | 24 +++ test/unit/win_object_handle.cpp | 21 ++ 23 files changed, 563 insertions(+), 121 deletions(-) diff --git a/include/boost/corosio/local_stream_acceptor.hpp b/include/boost/corosio/local_stream_acceptor.hpp index 9467a9a69..c014c9cc5 100644 --- a/include/boost/corosio/local_stream_acceptor.hpp +++ b/include/boost/corosio/local_stream_acceptor.hpp @@ -437,6 +437,12 @@ class BOOST_COROSIO_DECL local_stream_acceptor : public io_object pending operations without closing the descriptor. The caller takes ownership of the returned handle. + @note With the IOCP backend, a socket released while operations + on it are in flight stays bound to this context's completion + port for good. Windows refuses to unbind a handle with I/O + outstanding, so no context, this one included, can assign it + afterwards. Its cancelled operations still complete normally. + @return The native handle. @throws std::system_error `errc::bad_file_descriptor` if the diff --git a/include/boost/corosio/local_stream_socket.hpp b/include/boost/corosio/local_stream_socket.hpp index cd9455dc7..44bf2d3ce 100644 --- a/include/boost/corosio/local_stream_socket.hpp +++ b/include/boost/corosio/local_stream_socket.hpp @@ -422,6 +422,12 @@ class BOOST_COROSIO_DECL local_stream_socket : public io_stream operations without closing the descriptor. The caller takes ownership of the returned handle. + @note With the IOCP backend, a socket released while operations + on it are in flight stays bound to this context's completion + port for good. Windows refuses to unbind a handle with I/O + outstanding, so no context, this one included, can assign it + afterwards. Its cancelled operations still complete normally. + @return The native handle. @throws std::system_error `errc::bad_file_descriptor` if the diff --git a/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor_service.hpp b/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor_service.hpp index 0ad455286..27259f4c4 100644 --- a/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor_service.hpp @@ -374,6 +374,11 @@ win_local_stream_acceptor::accept( std::error_code* ec, io_object::implementation** impl_out) { + // Before anything is armed, so a throw leaves no reference, stop + // callback or work count behind. + auto& peer_wrapper = + static_cast(*svc_.construct()); + acc_.object_ref_ = detail::object_ref(this); auto& op = acc_; @@ -393,15 +398,6 @@ win_local_stream_acceptor::accept( svc_.work_started(); - // Create a (possibly recycled) peer socket - auto* peer_ptr = svc_.construct(); - if (!peer_ptr) - { - svc_.on_completion(&op, ERROR_OUTOFMEMORY, 0); - return std::noop_coroutine(); - } - auto& peer_wrapper = static_cast(*peer_ptr); - // Always AF_UNIX for local sockets SOCKET accepted = ::WSASocketW(AF_UNIX, SOCK_STREAM, 0, nullptr, 0, WSA_FLAG_OVERLAPPED); @@ -462,12 +458,13 @@ win_local_stream_acceptor::accept( } } - svc_.on_pending(&op); - - // Re-check cancellation after I/O is pending + // Must precede on_pending: once it runs another thread may + // complete the op and recycle this impl. if (op.cancelled.load(std::memory_order_acquire)) ::CancelIoEx(reinterpret_cast(socket_), &op); + svc_.on_pending(&op); + return std::noop_coroutine(); } diff --git a/include/boost/corosio/native/detail/iocp/win_local_stream_service.hpp b/include/boost/corosio/native/detail/iocp/win_local_stream_service.hpp index 190accad7..dabdc4b06 100644 --- a/include/boost/corosio/native/detail/iocp/win_local_stream_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_local_stream_service.hpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -425,12 +426,13 @@ win_local_stream_socket::connect( } } - svc_.on_pending(&op); - - // Re-check cancellation after I/O is pending + // Must precede on_pending: once it runs another thread may + // complete the op and recycle this impl. if (op.cancelled.load(std::memory_order_acquire)) ::CancelIoEx(reinterpret_cast(socket_), &op); + svc_.on_pending(&op); + return std::noop_coroutine(); } @@ -497,11 +499,13 @@ win_local_stream_socket::read_some( } } - svc_.on_pending(&op); - + // Must precede on_pending: once it runs another thread may + // complete the op and recycle this impl. if (op.cancelled.load(std::memory_order_acquire)) ::CancelIoEx(reinterpret_cast(socket_), &op); + svc_.on_pending(&op); + return std::noop_coroutine(); } @@ -564,11 +568,13 @@ win_local_stream_socket::write_some( } } - svc_.on_pending(&op); - + // Must precede on_pending: once it runs another thread may + // complete the op and recycle this impl. if (op.cancelled.load(std::memory_order_acquire)) ::CancelIoEx(reinterpret_cast(socket_), &op); + svc_.on_pending(&op); + return std::noop_coroutine(); } @@ -622,11 +628,13 @@ win_local_stream_socket::wait( } } - svc_.on_pending(&op); - + // Must precede on_pending: once it runs another thread may + // complete the op and recycle this impl. if (op.cancelled.load(std::memory_order_acquire)) ::CancelIoEx(reinterpret_cast(socket_), &op); + svc_.on_pending(&op); + return std::noop_coroutine(); } @@ -844,6 +852,9 @@ win_local_stream_service::assign_socket( if (proto_info.iSocketType != SOCK_STREAM) return make_err(WSAEPROTOTYPE); + if (has_skip_on_success(reinterpret_cast(s))) + return std::make_error_code(std::errc::operation_not_supported); + HANDLE result = ::CreateIoCompletionPort( reinterpret_cast(s), static_cast(iocp_), key_io, 0); if (result == nullptr) @@ -996,6 +1007,9 @@ win_local_stream_service::assign_acceptor_socket( if (proto_info.iSocketType != SOCK_STREAM) return make_err(WSAEPROTOTYPE); + if (has_skip_on_success(reinterpret_cast(sock))) + return std::make_error_code(std::errc::operation_not_supported); + HANDLE result = ::CreateIoCompletionPort( reinterpret_cast(sock), static_cast(iocp_), key_io, 0); if (result == nullptr) diff --git a/include/boost/corosio/native/detail/iocp/win_object_handle_service.hpp b/include/boost/corosio/native/detail/iocp/win_object_handle_service.hpp index dff370952..788877d8b 100644 --- a/include/boost/corosio/native/detail/iocp/win_object_handle_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_object_handle_service.hpp @@ -97,8 +97,14 @@ class win_object_handle_state std::stop_token token, std::error_code* ec) { - std::uint64_t const cur = state_.load(std::memory_order_acquire); - if ((cur & phase_mask) != phase_idle) + // Reserved as claimed under the current generation until armed: + // a concurrent wait() sees it busy, nothing else claims it, and + // a rundown in the meantime records a generation this wait never + // uses. + std::uint64_t cur = state_.load(std::memory_order_acquire); + if ((cur & phase_mask) != phase_idle || + !state_.compare_exchange_strong( + cur, cur | phase_claimed, std::memory_order_acq_rel)) { // Complete immediately: the returned handle is resumed by // symmetric transfer on the caller's own executor. @@ -140,19 +146,25 @@ class win_object_handle_state } } - state_.store(armed, std::memory_order_release); - // May run the rundown synchronously if the token is already - // stopped; that claims the op before it is ever armed. + // The stop callback is engaged before the wait is published, so + // a completion can never find it mid-construction. A stop that + // runs before publication finds nothing to claim; the check + // below claims for it. op_.start(token); + state_.store(armed, std::memory_order_release); { - // A rundown that disarmed this generation first (from a - // stop request on another thread) has claimed, or will - // claim, the op; arming now would let a later signal be - // consumed with nobody left to report it. + // A rundown that disarmed this generation first has claimed, + // or will claim, the op; arming now would let a later signal + // be consumed with nobody left to report it. std::lock_guard lock(arm_mutex_); if (state_.load(std::memory_order_acquire) == armed && disarmed_gen_ != gen) - ::SetThreadpoolWait(tp_wait_, handle_, nullptr); + { + if (!op_.cancelled.load(std::memory_order_acquire)) + ::SetThreadpoolWait(tp_wait_, handle_, nullptr); + else if (claim(armed)) + sched_.on_completion(&op_, ERROR_OPERATION_ABORTED, 0); + } } return std::noop_coroutine(); } diff --git a/include/boost/corosio/native/detail/iocp/win_resolver.hpp b/include/boost/corosio/native/detail/iocp/win_resolver.hpp index 312ab3a0a..f4ad5a93e 100644 --- a/include/boost/corosio/native/detail/iocp/win_resolver.hpp +++ b/include/boost/corosio/native/detail/iocp/win_resolver.hpp @@ -170,6 +170,9 @@ struct resolve_op : overlapped_op /** Completion callback for GetAddrInfoExW. */ static void CALLBACK completion(DWORD dwError, DWORD bytes, OVERLAPPED* ov); + /// Cancel the in-flight lookup on a stop request. + static void do_cancel_impl(overlapped_op* base) noexcept; + static void do_complete( void* owner, scheduler_op* base, @@ -308,6 +311,9 @@ class win_resolver final static void do_reverse_resolve_work(pool_work_item*) noexcept; private: + /// Cancel the in-flight forward lookup, if any. + void cancel_lookup() noexcept; + // The cancel handle dies when the completion callback is entered, so // cancel() and that callback must not both claim it. Held across the // claim only: GetAddrInfoExCancel can wait on the callback. diff --git a/include/boost/corosio/native/detail/iocp/win_resolver_service.hpp b/include/boost/corosio/native/detail/iocp/win_resolver_service.hpp index b0b14fe78..aba3a07a5 100644 --- a/include/boost/corosio/native/detail/iocp/win_resolver_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_resolver_service.hpp @@ -263,7 +263,16 @@ resolve_op::completion(DWORD dwError, DWORD /*bytes*/, OVERLAPPED* ov) svc.callbacks_pending_.fetch_sub(1, std::memory_order_release); } -inline resolve_op::resolve_op() noexcept : overlapped_op(&do_complete) {} +inline resolve_op::resolve_op() noexcept : overlapped_op(&do_complete) +{ + cancel_func_ = &do_cancel_impl; +} + +inline void +resolve_op::do_cancel_impl(overlapped_op* base) noexcept +{ + static_cast(base)->impl->cancel_lookup(); +} inline void resolve_op::do_complete( @@ -428,6 +437,7 @@ win_resolver::resolve( // Under the lock: the handle written here and the callback that // retires it must not interleave. int result; + HANDLE late_cancel = nullptr; { std::lock_guard lock(cancel_mutex_); op.cancel_handle = nullptr; @@ -445,6 +455,20 @@ win_resolver::resolve( op.cancel_handle = nullptr; svc.callbacks_pending_.fetch_sub(1, std::memory_order_relaxed); } + // A stop before the lookup was issued found no handle to cancel. + // Claimed under the lock: once it is released the callback may + // complete the op and recycle this impl. + else if (op.cancelled.load(std::memory_order_acquire)) + { + late_cancel = op.cancel_handle; + op.cancel_handle = nullptr; + } + } + + if (late_cancel) + { + ::GetAddrInfoExCancel(&late_cancel); + return std::noop_coroutine(); } if (result != WSA_IO_PENDING) @@ -517,7 +541,12 @@ win_resolver::cancel() noexcept { op_.request_cancel(); reverse_op_.request_cancel(); + cancel_lookup(); +} +inline void +win_resolver::cancel_lookup() noexcept +{ // Whoever claims the handle owns it: the callback retires it on // entry, and a claim consumes it, so neither a racing callback nor a // second cancel() reaches one Windows has reclaimed. diff --git a/include/boost/corosio/native/detail/iocp/win_scheduler.hpp b/include/boost/corosio/native/detail/iocp/win_scheduler.hpp index 25baff946..875462dba 100644 --- a/include/boost/corosio/native/detail/iocp/win_scheduler.hpp +++ b/include/boost/corosio/native/detail/iocp/win_scheduler.hpp @@ -839,9 +839,17 @@ win_scheduler::shutdown() DWORD bytes; ULONG_PTR key; LPOVERLAPPED overlapped; - ::GetQueuedCompletionStatus(iocp_, &bytes, &key, &overlapped, 0); + BOOL const dequeued = + ::GetQueuedCompletionStatus(iocp_, &bytes, &key, &overlapped, 0); if (!overlapped) - break; + { + // A signal such as stop()'s carries no operation, and + // packets posted after it still need sweeping; only an + // empty port ends the sweep. + if (!dequeued) + break; + continue; + } if (key == key_posted) { reinterpret_cast(overlapped)->destroy(); diff --git a/include/boost/corosio/native/detail/iocp/win_signals.hpp b/include/boost/corosio/native/detail/iocp/win_signals.hpp index 8c0472720..9c1a2a993 100644 --- a/include/boost/corosio/native/detail/iocp/win_signals.hpp +++ b/include/boost/corosio/native/detail/iocp/win_signals.hpp @@ -392,9 +392,9 @@ win_signal::wait( *ec = make_error_code(capy::error::canceled); if (signal_out) *signal_out = 0; - dispatch_coro(d, *pending_op_.cont).resume(); - // resumed inline for an io_context executor, not posted to the - // scheduler queue. + // Posted, never resumed inline: this is the caller's own + // suspend, and a loop of such waits would nest frames. + d.post(*pending_op_.cont); return std::noop_coroutine(); } @@ -696,7 +696,7 @@ win_signals::cancel_wait(win_signal& impl) *op->ec_out = make_error_code(capy::error::canceled); if (op->signal_out) *op->signal_out = 0; - dispatch_coro(op->d, *op->cont).resume(); + op->d.post(*op->cont); sched_.work_finished(); } } @@ -726,7 +726,7 @@ win_signals::cancel_wait_token(win_signal& impl) noexcept *op->ec_out = make_error_code(capy::error::canceled); if (op->signal_out) *op->signal_out = 0; - dispatch_coro(op->d, *op->cont).resume(); + op->d.post(*op->cont); sched_.work_finished(); } } @@ -797,10 +797,10 @@ win_signals::start_wait(win_signal& impl, signal_op* op) } } - // Dispatch outside the lock to avoid deadlock if the resumed - // coroutine re-enters cancel()/add()/remove() + // Posted outside the lock: this runs inside the caller's own + // suspend, so resuming it here would nest frames. if (was_cancelled) - dispatch_coro(op->d, *op->cont).resume(); + op->d.post(*op->cont); } inline void diff --git a/include/boost/corosio/native/detail/iocp/win_tcp_acceptor_service.hpp b/include/boost/corosio/native/detail/iocp/win_tcp_acceptor_service.hpp index d8482dc0b..c937107cf 100644 --- a/include/boost/corosio/native/detail/iocp/win_tcp_acceptor_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_tcp_acceptor_service.hpp @@ -301,6 +301,10 @@ win_tcp_acceptor::accept( std::error_code* ec, io_object::implementation** impl_out) { + // Before anything is armed, so a throw leaves no reference, stop + // callback or work count behind. + auto& peer_wrapper = static_cast(*svc_.construct()); + // Keep this acceptor alive during I/O acc_.object_ref_ = detail::object_ref(this); @@ -321,9 +325,6 @@ win_tcp_acceptor::accept( svc_.work_started(); - // Create a (possibly recycled) peer socket; the service owns it. - auto& peer_wrapper = static_cast(*svc_.construct()); - // Derive AF from the listening socket's cached local endpoint int af = native_family(local_endpoint_.address().family()); @@ -389,15 +390,13 @@ win_tcp_acceptor::accept( } } - svc_.on_pending(&op); - - // If the stop_token was already cancelled when start() was called, - // the CancelIoEx in the canceller fired before AcceptEx was - // submitted and had no effect. Re-issue now that the I/O is - // pending so the completion posts with ERROR_OPERATION_ABORTED. + // Must precede on_pending: once it runs another thread may + // complete the op and recycle this impl. if (op.cancelled.load(std::memory_order_acquire)) ::CancelIoEx(reinterpret_cast(socket_), &op); + svc_.on_pending(&op); + return std::noop_coroutine(); } diff --git a/include/boost/corosio/native/detail/iocp/win_tcp_service.hpp b/include/boost/corosio/native/detail/iocp/win_tcp_service.hpp index 65a2b5823..7ab73c1aa 100644 --- a/include/boost/corosio/native/detail/iocp/win_tcp_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_tcp_service.hpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -533,6 +534,12 @@ win_tcp_socket::connect( } } + // A stop before ConnectEx was issued found nothing to cancel. Must + // precede on_pending: once it runs another thread may complete the + // op and recycle this impl. + if (op.cancelled.load(std::memory_order_acquire)) + ::CancelIoEx(reinterpret_cast(socket_), &op); + svc_.on_pending(&op); return std::noop_coroutine(); } @@ -603,12 +610,13 @@ win_tcp_socket::read_some( } } - svc_.on_pending(&op); - - // Re-check cancellation after I/O is pending + // Must precede on_pending: once it runs another thread may + // complete the op and recycle this impl. if (op.cancelled.load(std::memory_order_acquire)) ::CancelIoEx(reinterpret_cast(socket_), &op); + svc_.on_pending(&op); + return std::noop_coroutine(); } @@ -674,12 +682,13 @@ win_tcp_socket::write_some( } } - svc_.on_pending(&op); - - // Re-check cancellation after I/O is pending + // Must precede on_pending: once it runs another thread may + // complete the op and recycle this impl. if (op.cancelled.load(std::memory_order_acquire)) ::CancelIoEx(reinterpret_cast(socket_), &op); + svc_.on_pending(&op); + return std::noop_coroutine(); } @@ -735,12 +744,13 @@ win_tcp_socket::wait( } } - svc_.on_pending(&op); - - // Re-check cancellation after I/O is pending. + // Must precede on_pending: once it runs another thread may + // complete the op and recycle this impl. if (op.cancelled.load(std::memory_order_acquire)) ::CancelIoEx(reinterpret_cast(socket_), &op); + svc_.on_pending(&op); + return std::noop_coroutine(); } @@ -989,6 +999,9 @@ win_tcp_service::assign_socket(win_tcp_socket& impl, native_handle_type fd) if (proto_info.iSocketType != SOCK_STREAM) return make_err(WSAEPROTOTYPE); + if (has_skip_on_success(reinterpret_cast(sock))) + return std::make_error_code(std::errc::operation_not_supported); + HANDLE result = ::CreateIoCompletionPort( reinterpret_cast(sock), static_cast(iocp_), key_io, 0); if (result == nullptr) @@ -1162,6 +1175,9 @@ win_tcp_service::assign_acceptor_socket( if (proto_info.iSocketType != SOCK_STREAM) return make_err(WSAEPROTOTYPE); + if (has_skip_on_success(reinterpret_cast(sock))) + return std::make_error_code(std::errc::operation_not_supported); + HANDLE result = ::CreateIoCompletionPort( reinterpret_cast(sock), static_cast(iocp_), key_io, 0); if (result == nullptr) diff --git a/include/boost/corosio/native/detail/iocp/win_udp_service.hpp b/include/boost/corosio/native/detail/iocp/win_udp_service.hpp index 794a68103..43997f69b 100644 --- a/include/boost/corosio/native/detail/iocp/win_udp_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_udp_service.hpp @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -456,12 +457,13 @@ win_udp_socket::send_to( } } - svc_.on_pending(&op); - - // Re-check cancellation after I/O is pending + // Must precede on_pending: once it runs another thread may + // complete the op and recycle this impl. if (op.cancelled.load(std::memory_order_acquire)) ::CancelIoEx(reinterpret_cast(socket_), &op); + svc_.on_pending(&op); + return std::noop_coroutine(); } @@ -528,12 +530,13 @@ win_udp_socket::recv_from( } } - svc_.on_pending(&op); - - // Re-check cancellation after I/O is pending + // Must precede on_pending: once it runs another thread may + // complete the op and recycle this impl. if (op.cancelled.load(std::memory_order_acquire)) ::CancelIoEx(reinterpret_cast(socket_), &op); + svc_.on_pending(&op); + return std::noop_coroutine(); } @@ -618,11 +621,13 @@ win_udp_socket::send( } } - svc_.on_pending(&op); - + // Must precede on_pending: once it runs another thread may + // complete the op and recycle this impl. if (op.cancelled.load(std::memory_order_acquire)) ::CancelIoEx(reinterpret_cast(socket_), &op); + svc_.on_pending(&op); + return std::noop_coroutine(); } @@ -680,11 +685,13 @@ win_udp_socket::recv( } } - svc_.on_pending(&op); - + // Must precede on_pending: once it runs another thread may + // complete the op and recycle this impl. if (op.cancelled.load(std::memory_order_acquire)) ::CancelIoEx(reinterpret_cast(socket_), &op); + svc_.on_pending(&op); + return std::noop_coroutine(); } @@ -941,6 +948,9 @@ win_udp_service::assign_socket( if (proto_info.iSocketType != SOCK_DGRAM) return make_err(WSAEPROTOTYPE); + if (has_skip_on_success(reinterpret_cast(s))) + return std::make_error_code(std::errc::operation_not_supported); + HANDLE result = ::CreateIoCompletionPort( reinterpret_cast(s), static_cast(iocp_), key_io, 0); if (result == nullptr) diff --git a/include/boost/corosio/native/detail/iocp/win_validate_handle.hpp b/include/boost/corosio/native/detail/iocp/win_validate_handle.hpp index cbe2ce697..8b806be3b 100644 --- a/include/boost/corosio/native/detail/iocp/win_validate_handle.hpp +++ b/include/boost/corosio/native/detail/iocp/win_validate_handle.hpp @@ -116,6 +116,30 @@ query_object() noexcept } // namespace win_nt +/** Check whether a handle is in skip-completion-port-on-success mode. + + Such a handle queues no packet for an operation that succeeds + synchronously, so an operation issued through a completion port + would never complete. The mode cannot be cleared. + + @param h The handle to inspect. Never read, written or waited on. + + @return `true` if the mode is set. A failed query means it never + was. +*/ +inline bool +has_skip_on_success(HANDLE h) noexcept +{ + auto const query = win_nt::query_information_file(); + if (!query) + return false; + win_nt::io_status_block iosb{}; + ULONG notify = 0; + return query(h, &iosb, ¬ify, sizeof(notify), + win_nt::file_io_completion_notification_information) >= 0 && + (notify & win_nt::file_skip_completion_port_on_success); +} + /** Validate a handle for adoption by an overlapped type. @param h The handle to inspect. Never read, written or waited on. @@ -161,13 +185,7 @@ validate_overlapped_handle(HANDLE h, handle_kind kind) noexcept win_nt::file_synchronous_io_nonalert)) return not_supported; - // A handle already in skip-on-success mode queues no packet for a - // synchronous success, so the op would never complete. The mode - // cannot be cleared. A failed query means the mode was never set. - ULONG notify = 0; - if (query(h, &iosb, ¬ify, sizeof(notify), - win_nt::file_io_completion_notification_information) >= 0 && - (notify & win_nt::file_skip_completion_port_on_success)) + if (has_skip_on_success(h)) return not_supported; // A failed query (volumes, some devices) means "not a directory". diff --git a/include/boost/corosio/native/detail/iocp/win_wait_reactor.hpp b/include/boost/corosio/native/detail/iocp/win_wait_reactor.hpp index 6f759f9a7..63ca24b2a 100644 --- a/include/boost/corosio/native/detail/iocp/win_wait_reactor.hpp +++ b/include/boost/corosio/native/detail/iocp/win_wait_reactor.hpp @@ -344,22 +344,12 @@ win_wait_reactor::wake_self() noexcept inline void win_wait_reactor::register_wait(SOCKET fd, wait_type w, overlapped_op* op) { - // If the op was already cancelled (e.g. pre-cancelled stop_token - // fired synchronously before this call), complete immediately - // instead of registering. Otherwise the reactor would park the - // op forever because the earlier cancel_wait() found nothing to - // cancel in registered_. - if (op->cancelled.load(std::memory_order_acquire)) - { - sched_.on_completion(op, 0, 0); - return; - } - if (DWORD const err = queue_register(entry{fd, w, op}); err != 0) { // Nothing would ever drain a parked op, and the refusal knows - // why: the abort a stop drains with, the error the polling - // thread died of, or the system declining a thread. + // why: a cancel that came first, the abort a stop drains with, + // the error the polling thread died of, or the system declining + // a thread. sched_.on_completion(op, err, 0); return; } @@ -388,6 +378,13 @@ win_wait_reactor::queue_register(entry const& e) if (dead_err_) return dead_err_; + // A cancel that arrived before this register found nothing parked + // and was dropped. Every canceller flags the op before it takes + // this lock to ask, so either the flag is seen here or the ask is + // queued after the register and finds it. + if (e.op->cancelled.load(std::memory_order_acquire)) + return ERROR_OPERATION_ABORTED; + // A polling thread costs a thread per context, and a context that // never waits never pays for one; the first wait is what starts it. // A system that will not give one refuses this wait alone: nothing @@ -422,6 +419,10 @@ win_wait_reactor::cancel_wait(overlapped_op* op) // arrived after them was refused at its caller. if (stop_.load(std::memory_order_acquire) || dead_err_) return; + // No thread yet means nothing was ever parked; every socket's + // close asks, and the queue would only grow. + if (!thread_.joinable()) + return; pending_cancel_.push_back(op); } wake_self(); @@ -594,16 +595,25 @@ win_wait_reactor::run() // wait, because reading it resets it. if (e.w == wait_type::error && (pfd.revents & err_bits)) { - int so_err = 0; - int sz = sizeof(so_err); - if (::getsockopt( - e.fd, SOL_SOCKET, SO_ERROR, - reinterpret_cast(&so_err), &sz) == 0 && - so_err != 0) - err = static_cast(so_err); - else - // The contract is to report a non-zero error_code. - err = WSAECONNABORTED; + // close() flags the op and queues its cancel under this + // lock before closesocket, so the probe either finishes + // first or sees the flag. A flagged op's number may + // already name another socket, whose error a probe + // would steal; the handler reports the cancel instead. + std::lock_guard lock(mutex_); + if (!e.op->cancelled.load(std::memory_order_acquire)) + { + int so_err = 0; + int sz = sizeof(so_err); + if (::getsockopt( + e.fd, SOL_SOCKET, SO_ERROR, + reinterpret_cast(&so_err), &sz) == 0 && + so_err != 0) + err = static_cast(so_err); + else + // The contract is to report a non-zero error_code. + err = WSAECONNABORTED; + } } sched_.on_completion(e.op, err, 0); diff --git a/include/boost/corosio/native/native_local_stream_socket.hpp b/include/boost/corosio/native/native_local_stream_socket.hpp index 4c4ffa4a6..b2449618e 100644 --- a/include/boost/corosio/native/native_local_stream_socket.hpp +++ b/include/boost/corosio/native/native_local_stream_socket.hpp @@ -33,7 +33,7 @@ #endif #if BOOST_COROSIO_HAS_IOCP -#include +#include #endif #endif // !BOOST_COROSIO_MRDOCS diff --git a/include/boost/corosio/tcp_acceptor.hpp b/include/boost/corosio/tcp_acceptor.hpp index c66e6082b..fd18b2cd1 100644 --- a/include/boost/corosio/tcp_acceptor.hpp +++ b/include/boost/corosio/tcp_acceptor.hpp @@ -533,6 +533,12 @@ class BOOST_COROSIO_DECL tcp_acceptor : public io_object operations without closing the descriptor. The caller takes ownership of the returned handle. + @note With the IOCP backend, a socket released while operations + on it are in flight stays bound to this context's completion + port for good. Windows refuses to unbind a handle with I/O + outstanding, so no context, this one included, can assign it + afterwards. Its cancelled operations still complete normally. + @return The native handle. @throws std::system_error `errc::bad_file_descriptor` if the diff --git a/include/boost/corosio/tcp_socket.hpp b/include/boost/corosio/tcp_socket.hpp index dc081a25e..954daab3d 100644 --- a/include/boost/corosio/tcp_socket.hpp +++ b/include/boost/corosio/tcp_socket.hpp @@ -488,6 +488,12 @@ class BOOST_COROSIO_DECL tcp_socket : public io_stream operations without closing the descriptor. The caller takes ownership of the returned handle. + @note With the IOCP backend, a socket released while operations + on it are in flight stays bound to this context's completion + port for good. Windows refuses to unbind a handle with I/O + outstanding, so no context, this one included, can assign it + afterwards. Its cancelled operations still complete normally. + @return The native handle. @throws std::system_error `errc::bad_file_descriptor` if the diff --git a/include/boost/corosio/udp_socket.hpp b/include/boost/corosio/udp_socket.hpp index 083bca48a..105e575b3 100644 --- a/include/boost/corosio/udp_socket.hpp +++ b/include/boost/corosio/udp_socket.hpp @@ -625,6 +625,12 @@ class BOOST_COROSIO_DECL udp_socket : public io_object operations without closing the descriptor. The caller takes ownership of the returned handle. + @note With the IOCP backend, a socket released while operations + on it are in flight stays bound to this context's completion + port for good. Windows refuses to unbind a handle with I/O + outstanding, so no context, this one included, can assign it + afterwards. Its cancelled operations still complete normally. + @return The native handle. @throws std::system_error `errc::bad_file_descriptor` if the diff --git a/test/unit/fault/iocp_faults.cpp b/test/unit/fault/iocp_faults.cpp index 6daedac88..d54ae7104 100644 --- a/test/unit/fault/iocp_faults.cpp +++ b/test/unit/fault/iocp_faults.cpp @@ -1027,14 +1027,12 @@ struct iocp_faults coalesces into that phantom, and the poll they were meant to end is indefinite. - This context never registers a wait, so no polling thread is - ever started and nothing drains the flag -- a latch here would - be permanent, which is what makes the second send the - observable. Both sends come from the socket: open() goes - through close_socket, which asks the reactor to drop any wait op - the descriptor might have had, and cancel() asks again. Both - asks reach wake_self whether or not anything is parked - (win_tcp_socket::cancel and ::close_socket). + A parked wait starts the polling thread, so the cancels below + have a thread to wake. A cancel only sends once the byte already + in the channel is drained, so they repeat until two sends have + failed; a latch would coalesce every later wake into the phantom, + and the final cancel, sent with nothing failing, would never + reach the thread. */ void testWakeSendFails() { @@ -1045,18 +1043,49 @@ struct iocp_faults } { io_context ioc(iocp); - // Thread-local, not any_thread: two process-wide scopes - // cannot be alive at once, and every call below is on this - // thread. Armed before open(), whose own ask is the first - // send and the one that has to leave the flag clear. - fault_scope first(sys::send, WSAENOBUFS, 1); - fault_scope second(sys::send, WSAENOBUFS, 2); - tcp_socket s(ioc); - BOOST_TEST(!s.open(family::v4)); - s.cancel(); - BOOST_TEST(first.fired()); - BOOST_TEST(second.fired()); - s.close(); + auto pair = make_socket_pair(ioc); + auto& s1 = pair.first; + auto& s2 = pair.second; + std::error_code parked_ec; + bool parked_done = false; + bool expired = false; + bool fired = false; + auto parked = [&]() -> capy::task<> { + auto [ec] = co_await s1.wait(wait_type::error); + parked_ec = ec; + parked_done = true; + }; + auto breaker = [&]() -> capy::task<> { + { + // Thread-local, not any_thread: every send below is + // on this thread. The pauses bound the retries, not + // a timing assertion. + fault_scope first(sys::send, WSAENOBUFS, 1); + fault_scope second(sys::send, WSAENOBUFS, 2); + for (int i = 0; i < 100 && !second.fired(); ++i) + { + s1.cancel(); + std::ignore = co_await corosio::delay( + std::chrono::milliseconds(5)); + } + fired = first.fired() && second.fired(); + } + s1.cancel(); + for (int i = 0; i < 100 && !parked_done; ++i) + std::ignore = + co_await corosio::delay(std::chrono::milliseconds(5)); + ioc.stop(); + }; + capy::run_async(ioc.get_executor())(parked()); + capy::run_async(ioc.get_executor())(breaker()); + capy::run_async(ioc.get_executor())(stop_guard(ioc, expired)); + ioc.run(); + BOOST_TEST(fired); + BOOST_TEST(!expired); + BOOST_TEST(parked_done); + BOOST_TEST(parked_ec == capy::error::canceled); + s1.close(); + s2.close(); } // And a context whose wakes all failed still answers a wait // rather than leaving it outstanding. Nothing raises the error diff --git a/test/unit/io_context.cpp b/test/unit/io_context.cpp index 2d5207896..c3bb6f557 100644 --- a/test/unit/io_context.cpp +++ b/test/unit/io_context.cpp @@ -1059,6 +1059,25 @@ struct io_context_shutdown_test BOOST_TEST_EQ(destroyed, 3); } + // A stopped context still owns what was posted after the stop: its + // shutdown destroys those frames even though the stop's own wakeup + // is queued ahead of them. + void testShutdownAfterStopDestroysPostedFrames() + { + int destroyed = 0; + + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + ioc.stop(); + post_coro(ex, make_destroy_coro(destroyed)); + post_coro(ex, make_destroy_coro(destroyed)); + } + + BOOST_TEST_EQ(destroyed, 2); + } + void testConstructionWithBackendAndOptions() { // Exercises the templated io_context(Backend, options, hint) @@ -1076,6 +1095,7 @@ struct io_context_shutdown_test void run() { testShutdownDestroysPostedCoroutineFrames(); + testShutdownAfterStopDestroysPostedFrames(); testConstructionWithBackendAndOptions(); } }; diff --git a/test/unit/iocp_paths.cpp b/test/unit/iocp_paths.cpp index 0d9d5a0fd..9af20eb81 100644 --- a/test/unit/iocp_paths.cpp +++ b/test/unit/iocp_paths.cpp @@ -41,11 +41,15 @@ #include #include +#include +#include #include #include +#include #include #include #include +#include #include #include "context.hpp" @@ -236,6 +240,92 @@ struct iocp_paths_test BOOST_TEST(lacc.assign(lacc.native_handle()) == error::already_open); } + // A socket in skip-on-success mode queues no packet for a + // synchronous success, so every adopting type must refuse it and + // leave it with the caller. + void testAssignRejectsSkipOnSuccessSockets() + { + io_context ioc(iocp); + + auto skipping = [](int af, int type, int proto) { + SOCKET s = ::WSASocketW( + af, type, proto, nullptr, 0, WSA_FLAG_OVERLAPPED); + BOOST_TEST(s != INVALID_SOCKET); + BOOST_TEST(::SetFileCompletionNotificationModes( + reinterpret_cast(s), + FILE_SKIP_COMPLETION_PORT_ON_SUCCESS)); + return s; + }; + auto refused = [](auto& obj, SOCKET s) { + BOOST_TEST( + obj.assign(static_cast(s)) == + std::errc::operation_not_supported); + BOOST_TEST(!obj.is_open()); + BOOST_TEST(::closesocket(s) == 0); + }; + + tcp_socket t(ioc); + refused(t, skipping(AF_INET, SOCK_STREAM, IPPROTO_TCP)); + tcp_acceptor ta(ioc); + refused(ta, skipping(AF_INET, SOCK_STREAM, IPPROTO_TCP)); + udp_socket u(ioc); + refused(u, skipping(AF_INET, SOCK_DGRAM, IPPROTO_UDP)); + local_stream_socket l(ioc); + refused(l, skipping(AF_UNIX, SOCK_STREAM, 0)); + local_stream_acceptor la(ioc); + refused(la, skipping(AF_UNIX, SOCK_STREAM, 0)); + } + + // A stop that lands while a wait is being registered with the wait + // reactor must still end it: the reactor may already have dropped + // the cancel as one for an op it never parked. + void testStopRacingWaitRegistration() + { + io_context ioc(iocp, 2); + udp_socket u(ioc); + BOOST_TEST(!u.open(family::v4)); + BOOST_TEST(!u.bind(endpoint(ipv4_address::loopback(), 0))); + + std::atomic quit{false}; + std::vector runners; + for (int i = 0; i < 2; ++i) + runners.emplace_back([&] { + while (!quit.load()) + std::ignore = ioc.run_one_for(std::chrono::milliseconds(5)); + }); + + int lost = 0; + for (int i = 0; i < 500; ++i) + { + std::stop_source ss; + std::atomic done{false}; + capy::run_async(ioc.get_executor(), ss.get_token())( + [](udp_socket& u, std::atomic& d) -> capy::task<> { + std::ignore = co_await u.wait(wait_type::error); + d = true; + }(u, done)); + ss.request_stop(); + // Bounded only to fail rather than hang. + for (int k = 0; k < 400 && !done.load(); ++k) + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + if (!done.load()) + { + ++lost; + // The parked wait still refers to `done`: end it while + // this iteration's locals are alive. + u.close(); + for (int k = 0; k < 400 && !done.load(); ++k) + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + break; + } + } + BOOST_TEST_EQ(lost, 0); + u.close(); + quit = true; + for (auto& t : runners) + t.join(); + } + void testShutdownReceiveVariants() { io_context ioc(iocp); @@ -374,6 +464,112 @@ struct iocp_paths_test BOOST_TEST(!ec); } + // release() cancels each op and then tries to unbind the socket from + // the port. Every cancelled op must still report through the port, + // or the op pins its object and run() never returns. Windows keeps a + // socket with I/O outstanding bound, so such a socket stays on this + // context's port. + void testReleaseWithKernelOpsInFlight() + { + io_context ioc(iocp); + auto ex = ioc.get_executor(); + auto [r1, r2] = test::make_socket_pair(ioc); + test::temp_socket_dir tmp; + auto [l1, l2] = make_local_pair(ioc, tmp); + + tcp_acceptor acc(ioc); + BOOST_TEST(!acc.open(family::v4)); + BOOST_TEST(!acc.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!acc.listen()); + tcp_socket peer(ioc); + + test::temp_socket_dir ltmp; + local_stream_acceptor lacc(ioc); + BOOST_TEST(!lacc.open()); + BOOST_TEST(!lacc.bind(local_endpoint(ltmp.path()))); + BOOST_TEST(!lacc.listen()); + local_stream_socket lpeer(ioc); + + // Loopback connects to a closed port retry for seconds on + // Windows, so the connect stays in flight. + std::uint16_t closed_port = 0; + { + tcp_acceptor probe(ioc); + BOOST_TEST(!probe.open(family::v4)); + BOOST_TEST(!probe.bind(endpoint(ipv4_address::loopback(), 0))); + closed_port = probe.local_endpoint().port(); + } + tcp_socket conn(ioc); + BOOST_TEST(!conn.open(family::v4)); + + udp_socket u(ioc); + BOOST_TEST(!u.open(family::v4)); + BOOST_TEST(!u.bind(endpoint(ipv4_address::loopback(), 0))); + + std::error_code ec[6]; + int done = 0; + char rbuf[8], lbuf[8], ubuf[8]; + endpoint from; + auto read_r = [&]() -> capy::task<> { + auto [e, n] = + co_await r1.read_some(capy::mutable_buffer(rbuf, sizeof(rbuf))); + std::ignore = n; + ec[0] = e; + ++done; + }; + auto accept_t = [&]() -> capy::task<> { + auto [e] = co_await acc.accept(peer); + ec[1] = e; + ++done; + }; + auto connect_t = [&]() -> capy::task<> { + auto [e] = co_await conn.connect( + endpoint(ipv4_address::loopback(), closed_port)); + ec[2] = e; + ++done; + }; + auto recv_u = [&]() -> capy::task<> { + auto [e, n] = co_await u.recv_from( + capy::mutable_buffer(ubuf, sizeof(ubuf)), from); + std::ignore = n; + ec[3] = e; + ++done; + }; + auto read_l = [&]() -> capy::task<> { + auto [e, n] = + co_await l1.read_some(capy::mutable_buffer(lbuf, sizeof(lbuf))); + std::ignore = n; + ec[4] = e; + ++done; + }; + auto accept_l = [&]() -> capy::task<> { + auto [e] = co_await lacc.accept(lpeer); + ec[5] = e; + ++done; + }; + capy::run_async(ex)(read_r()); + capy::run_async(ex)(accept_t()); + capy::run_async(ex)(connect_t()); + capy::run_async(ex)(recv_u()); + capy::run_async(ex)(read_l()); + capy::run_async(ex)(accept_l()); + std::ignore = ioc.poll(); + ioc.restart(); + BOOST_TEST_EQ(done, 0); + + native_handle_type released[] = { + r1.release(), acc.release(), conn.release(), + u.release(), l1.release(), lacc.release()}; + + std::ignore = ioc.run_for(std::chrono::seconds(30)); + BOOST_TEST_EQ(done, 6); + for (auto const& e : ec) + BOOST_TEST(e == capy::cond::canceled); + + for (auto h : released) + ::closesocket(static_cast(h)); + } + #if !COROSIO_TEST_HAS_ASAN // These abandon parked coroutine frames by design; see context.hpp. @@ -511,12 +707,15 @@ struct iocp_paths_test testStopCancelsLocalStreamOps(); testStopCancelsAcceptorWaits(); testAssignValidation(); + testAssignRejectsSkipOnSuccessSockets(); + testStopRacingWaitRegistration(); testShutdownReceiveVariants(); testZeroLengthUdpReceive(); testResolverEmptyInputs(); testReleasedAcceptorAccessors(); testTruncateWithoutCreate(); testStaleWaitCancelsFromDestroyedObjects(); + testReleaseWithKernelOpsInFlight(); #if !COROSIO_TEST_HAS_ASAN testDestroyWithParkedSocketOps(); testDestroyWithParkedLocalOps(); diff --git a/test/unit/signal_set.cpp b/test/unit/signal_set.cpp index 173d49132..23700ca34 100644 --- a/test/unit/signal_set.cpp +++ b/test/unit/signal_set.cpp @@ -476,6 +476,29 @@ struct signal_set_test BOOST_TEST(!second_ec); } + // A cancel() with no wait pending makes the next wait complete at + // once inside the backend. Those completions must not nest. + void testRepeatedCancelledWaitsDoNotNest() + { + io_context ioc(Backend); + signal_set s(ioc, SIGINT); + constexpr int n = 100000; + int canceled = 0; + capy::run_async(ioc.get_executor())( + [](signal_set& s, int& canceled) -> capy::task<> { + for (int i = 0; i < n; ++i) + { + s.cancel(); + auto [ec, signum] = co_await s.wait(); + std::ignore = signum; + if (ec == capy::cond::canceled) + ++canceled; + } + }(s, canceled)); + ioc.run(); + BOOST_TEST_EQ(canceled, n); + } + // With two threads running the context, a signal raised while one // is blocked in the reactor must still be delivered after the other // re-arms the signal pipe. @@ -1521,6 +1544,7 @@ struct signal_set_test { // Construction and move semantics testDeliveryWithTwoRunThreads(); + testRepeatedCancelledWaitsDoNotNest(); testConstruction(); testDestroyWithQueuedSignalCompletion(); testConstructWithOneSignal(); diff --git a/test/unit/win_object_handle.cpp b/test/unit/win_object_handle.cpp index 629074e76..bbb4f5fd5 100644 --- a/test/unit/win_object_handle.cpp +++ b/test/unit/win_object_handle.cpp @@ -253,6 +253,26 @@ struct win_object_handle_test BOOST_TEST(ec == capy::cond::canceled); } + // The stop lands before the wait is published, so wait() itself + // must claim the op for it, consuming no signal. + void testStopTokenStoppedBeforeWait() + { + io_context ioc(Backend); + win_object_handle o(ioc); + HANDLE ev = adopt_event(o, false, true); + std::stop_source ss; + ss.request_stop(); + std::error_code ec; + auto waiter = [&]() -> capy::task<> { + auto [e] = co_await o.wait(); + ec = e; + }; + capy::run_async(ioc.get_executor(), ss.get_token())(waiter()); + ioc.run(); + BOOST_TEST(ec == capy::cond::canceled); + BOOST_TEST(::WaitForSingleObject(ev, 0) == WAIT_OBJECT_0); + } + // Busy-waits about @p n iterations; volatile keeps the loop. static void spin(int n) noexcept { @@ -585,6 +605,7 @@ struct win_object_handle_test testSecondWaitIsInProgress(); testCancel(); testStopToken(); + testStopTokenStoppedBeforeWait(); testSignalBeforeCancelReportsSuccess(); testSignalRacingCancelIsNeverLost(); testCloseWithPendingWait(); From 35ae04d1a328f345e16e54ff5063fc983e96a2b0 Mon Sep 17 00:00:00 2001 From: Steve Gerbino Date: Fri, 9 Oct 2026 20:02:58 +0200 Subject: [PATCH 6/8] fix: let tcp_server be moved, destroyed and stopped while it runs A tcp_server can be moved or destroyed while it runs, a connection launched after stop() is stopped, and set_workers is refused while a worker is in use. A worker stays out of the pool until its run() returns, and a connection is allocated outside the chain that launches it. The documentation says what an exception from a worker's run() does and that the server binds only while no session is active. --- include/boost/corosio/tcp_server.hpp | 472 +++++++++++++++-------- src/corosio/src/tcp_server.cpp | 185 ++++++--- test/unit/tcp_server.cpp | 548 +++++++++++++++++++++++++++ 3 files changed, 990 insertions(+), 215 deletions(-) diff --git a/include/boost/corosio/tcp_server.hpp b/include/boost/corosio/tcp_server.hpp index dfa7fea0d..10219dcfe 100644 --- a/include/boost/corosio/tcp_server.hpp +++ b/include/boost/corosio/tcp_server.hpp @@ -25,11 +25,16 @@ #include #include #include +#include #include +#include #include #include +#include +#include #include +#include #include namespace boost::corosio { @@ -101,73 +106,189 @@ class BOOST_COROSIO_DECL tcp_server struct waiter { waiter* next; - std::coroutine_handle<> h; capy::continuation cont; worker_base* w; }; struct impl; - static impl* make_impl(capy::execution_context& ctx); - - impl* impl_; - capy::any_executor ex_; - waiter* waiters_ = nullptr; - worker_base* idle_head_ = nullptr; // Forward list: available workers - worker_base* active_head_ = - nullptr; // Doubly linked: workers handling connections - worker_base* active_tail_ = nullptr; // Tail for O(1) push_back - std::size_t active_accepts_ = 0; // Number of active do_accept coroutines - std::shared_ptr storage_; // Owns the worker container (type-erased) - bool running_ = false; - - // Idle list (forward/singly linked) - push front, pop front - void idle_push(worker_base* w) noexcept - { - w->next_ = idle_head_; - idle_head_ = w; - } + /** Everything the server's coroutines reach. - worker_base* idle_pop() noexcept + Accept loops, launchers and the coroutines that start + connections each hold a reference, so moving or destroying the + server never strands them. The lists are guarded by `mutex`, + because workers return from whichever thread their coroutine + finished on. + */ + struct state { - auto* w = idle_head_; - if (w) - idle_head_ = w->next_; - return w; - } + std::atomic refs{1}; + impl* impl_; + capy::any_executor ex; + std::mutex mutex; + waiter* waiters = nullptr; + worker_base* idle_head = nullptr; // forward list + worker_base* active_head = nullptr; // doubly linked + worker_base* active_tail = nullptr; + std::size_t active_accepts = 0; // guarded by the join mutex + std::size_t lent = 0; // borrowed workers, guarded by mutex + std::shared_ptr storage; // owns the worker container + bool running = false; + // Set by stop(); accept loops waiting for a worker give up. + bool stopping = false; // guarded by mutex + + state(impl* i, capy::any_executor const& e) noexcept : impl_(i), ex(e) + { + } - bool idle_empty() const noexcept - { - return idle_head_ == nullptr; - } + // The list operations below require `mutex`. + + void idle_push(worker_base* w) noexcept + { + w->next_ = idle_head; + idle_head = w; + } + + worker_base* idle_pop() noexcept + { + auto* w = idle_head; + if (w) + { + idle_head = w->next_; + ++lent; + } + return w; + } + + void active_push(worker_base* w) noexcept + { + w->next_ = nullptr; + w->prev_ = active_tail; + if (active_tail) + active_tail->next_ = w; + else + active_head = w; + active_tail = w; + } + + void active_remove(worker_base* w) noexcept + { + // Skip if not in active list (e.g., after failed accept) + if (w != active_head && w->prev_ == nullptr) + return; + if (w->prev_) + w->prev_->next_ = w->next_; + else + active_head = w->next_; + if (w->next_) + w->next_->prev_ = w->prev_; + else + active_tail = w->prev_; + w->prev_ = nullptr; // Mark as not in active list + } + + /// Return @p w to the pool, handing it to a waiting accept loop + /// if there is one. + void push(worker_base& w) noexcept + { + waiter* wake = nullptr; + { + std::lock_guard lock(mutex); + active_remove(&w); + if (waiters) + { + wake = waiters; + waiters = wake->next; + wake->w = &w; + } + else + { + idle_push(&w); + --lent; + } + } + if (wake) + ex.post(wake->cont); + } + + /// Drop one hold on @p w, returning it to the pool with the last. + void drop_hold(worker_base& w) noexcept + { + if (w.holds_.fetch_sub(1, std::memory_order_acq_rel) == 1) + push(w); + } + + /// Mark the pool as stopping and release every accept loop + /// waiting for a worker, with none. + void stop_waiters() noexcept + { + waiter* list = nullptr; + { + std::lock_guard lock(mutex); + stopping = true; + list = std::exchange(waiters, nullptr); + } + while (list) + { + auto* next = list->next; + ex.post(list->cont); + list = next; + } + } + }; - // Active list (doubly linked) - push back, remove anywhere - void active_push(worker_base* w) noexcept + /// Add a reference to @p s. + static void add_ref(state* s) noexcept { - w->next_ = nullptr; - w->prev_ = active_tail_; - if (active_tail_) - active_tail_->next_ = w; - else - active_head_ = w; - active_tail_ = w; + s->refs.fetch_add(1, std::memory_order_relaxed); } - void active_remove(worker_base* w) noexcept + /// Drop a reference to @p s, freeing it with the last one. + static void release(state* s) noexcept; + + /// An owned reference to the server's state. + class state_ref { - // Skip if not in active list (e.g., after failed accept) - if (w != active_head_ && w->prev_ == nullptr) - return; - if (w->prev_) - w->prev_->next_ = w->next_; - else - active_head_ = w->next_; - if (w->next_) - w->next_->prev_ = w->prev_; - else - active_tail_ = w->prev_; - w->prev_ = nullptr; // Mark as not in active list - } + state* p_ = nullptr; + + public: + explicit state_ref(state* p) noexcept : p_(p) + { + add_ref(p_); + } + + state_ref(state_ref const& o) noexcept : p_(o.p_) + { + if (p_) + add_ref(p_); + } + + state_ref(state_ref&& o) noexcept : p_(std::exchange(o.p_, nullptr)) {} + + state_ref& operator=(state_ref const&) = delete; + state_ref& operator=(state_ref&&) = delete; + + ~state_ref() + { + if (p_) + release(p_); + } + + state* operator->() const noexcept + { + return p_; + } + + state& operator*() const noexcept + { + return *p_; + } + }; + + static state* + make_state(capy::execution_context& ctx, capy::any_executor const& ex); + + state* st_; template struct launch_wrapper @@ -286,28 +407,24 @@ class BOOST_COROSIO_DECL tcp_server launch_wrapper operator()( Executor, std::stop_token, - tcp_server* self, + state_ref st, capy::task t, worker_base* wp) { // Executor and stop token stored in promise via constructor co_await std::move(t); - co_await self->push(*wp); // worker goes back to idle list + co_await push_awaitable{*st, *wp}; // worker back to the pool } }; class push_awaitable { - tcp_server& self_; + state& st_; worker_base& w_; capy::continuation cont_; public: - push_awaitable(tcp_server& self, worker_base& w) noexcept - : self_(self) - , w_(w) - { - } + push_awaitable(state& st, worker_base& w) noexcept : st_(st), w_(w) {} bool await_ready() const noexcept { @@ -319,92 +436,59 @@ class BOOST_COROSIO_DECL tcp_server { // Symmetric transfer to server's executor cont_.h = h; - return self_.ex_.dispatch(cont_); + return st_.ex.dispatch(cont_); } void await_resume() noexcept { - // Running on server executor - safe to modify lists - // Remove from active (if present), then wake waiter or add to idle - self_.active_remove(&w_); - if (self_.waiters_) - { - auto* wait = self_.waiters_; - self_.waiters_ = wait->next; - wait->w = &w_; - wait->cont.h = wait->h; - self_.ex_.post(wait->cont); - } - else - { - self_.idle_push(&w_); - } + st_.drop_hold(w_); } }; class pop_awaitable { - tcp_server& self_; + state& st_; waiter wait_; public: - pop_awaitable(tcp_server& self) noexcept : self_(self), wait_{} {} + pop_awaitable(state& st) noexcept : st_(st), wait_{} {} bool await_ready() const noexcept { - return !self_.idle_empty(); + return false; } bool await_suspend(std::coroutine_handle<> h, capy::io_env const*) noexcept { - // Running on server executor (do_accept runs there) - wait_.h = h; - wait_.w = nullptr; - wait_.next = self_.waiters_; - self_.waiters_ = &wait_; + std::lock_guard lock(st_.mutex); + if (st_.stopping) + { + wait_.w = nullptr; + return false; + } + if (auto* w = st_.idle_pop()) + { + wait_.w = w; + return false; + } + wait_.cont.h = h; + wait_.w = nullptr; + wait_.next = st_.waiters; + st_.waiters = &wait_; return true; } - worker_base& await_resume() noexcept + /// Return the worker, or null once the server is stopping. + worker_base* await_resume() noexcept { - // Running on server executor - if (wait_.w) - return *wait_.w; // Woken by push_awaitable - return *self_.idle_pop(); + // Set by await_suspend, or by the push that woke us. + return wait_.w; } }; - push_awaitable push(worker_base& w) - { - return push_awaitable{*this, w}; - } - - // Synchronous version for destructor/guard paths - // Must be called from server executor context - void push_sync(worker_base& w) noexcept - { - active_remove(&w); - if (waiters_) - { - auto* wait = waiters_; - waiters_ = wait->next; - wait->w = &w; - wait->cont.h = wait->h; - ex_.post(wait->cont); - } - else - { - idle_push(&w); - } - } - - pop_awaitable pop() - { - return pop_awaitable{*this}; - } - - capy::task do_accept(tcp_acceptor& acc); + static capy::task do_accept(state_ref st, tcp_acceptor& acc); + static capy::task<> do_stop(state_ref st); public: /** Handles one accepted connection using a socket the derived class owns. @@ -424,6 +508,9 @@ class BOOST_COROSIO_DECL tcp_server std::stop_source stop_; // ~16 bytes worker_base* next_ = nullptr; // 8 bytes - used by idle and active lists worker_base* prev_ = nullptr; // 8 bytes - used only by active list + // One for the accept loop until run() returns, one for the + // launcher and its connection; the last one returns the worker. + std::atomic holds_{0}; friend class tcp_server; @@ -440,6 +527,15 @@ class BOOST_COROSIO_DECL tcp_server connection. The implementation must invoke the launcher exactly once to start the handling coroutine. + @par Exception Safety + An exception escaping this function ends the process. A + worker that cannot start the connection should handle the + failure here; destroying the launcher without invoking it + returns the worker to the pool. + + The worker is not given another connection before this + returns, even if the connection it started has finished. + @param launch Handle to start the connection coroutine. */ virtual void run(launcher launch) = 0; @@ -463,13 +559,14 @@ class BOOST_COROSIO_DECL tcp_server */ class BOOST_COROSIO_DECL launcher { - tcp_server* srv_; + state* st_; worker_base* w_; friend class tcp_server; - launcher(tcp_server& srv, worker_base& w) noexcept : srv_(&srv), w_(&w) + launcher(state& st, worker_base& w) noexcept : st_(&st), w_(&w) { + add_ref(st_); } public: @@ -477,7 +574,9 @@ class BOOST_COROSIO_DECL tcp_server ~launcher() { if (w_) - srv_->push_sync(*w_); + st_->drop_hold(*w_); + if (st_) + release(st_); } /** Move construct, transferring the borrowed worker. @@ -486,7 +585,7 @@ class BOOST_COROSIO_DECL tcp_server holding none, so only one of the two returns it. */ launcher(launcher&& o) noexcept - : srv_(o.srv_) + : st_(std::exchange(o.st_, nullptr)) , w_(std::exchange(o.w_, nullptr)) { } @@ -503,6 +602,14 @@ class BOOST_COROSIO_DECL tcp_server the coroutine completes, the worker is automatically returned to the idle pool. + @pre @p task was created on a chain whose frame allocator + outlives the connection. The task's frame is allocated + where it is created, and it lives until the connection + ends. Inside @ref worker_base::run that allocator is the + accept loop's, the frame allocator of the server's + execution context. By default that is the process-wide + recycling resource, so a task created there meets this. + @tparam Executor Executor type satisfying capy::Executor. @param ex The executor to run the coroutine on. @@ -518,31 +625,62 @@ class BOOST_COROSIO_DECL tcp_server auto* w = std::exchange(w_, nullptr); - // Worker is being dispatched - add to active list - srv_->active_push(w); + // A stop_source allocates shared state on construction; + // reuse the worker's across connections and replace it + // only once a stop has actually been delivered (the state + // is then latched for good). Replaced before the worker is + // listed active, under the lock stop() copies it under, so + // that copy never races the replacement or misses it. + std::stop_token st; + bool stopped_before = false; + { + std::lock_guard lock(st_->mutex); + if (w->stop_.stop_requested()) + w->stop_ = {}; + st = w->stop_.get_token(); + st_->active_push(w); + stopped_before = st_->stopping; + } + // A stop that came before this launch never saw the worker + // listed. The token is not handed out yet, so no callback + // runs here. + if (stopped_before) + w->stop_.request_stop(); // Return worker to pool if coroutine setup throws struct guard_t { - tcp_server* srv; + state* st; worker_base* w; ~guard_t() { if (w) - srv->push_sync(*w); + st->drop_hold(*w); } - } guard{srv_, w}; - - // A stop_source allocates shared state on construction; - // reuse the worker's across connections and replace it - // only once a stop has actually been delivered (the state - // is then latched for good). - if (w->stop_.stop_requested()) - w->stop_ = {}; - auto st = w->stop_.get_token(); + } guard{st_, w}; - auto wrapper = - launch_coro{}(ex, st, srv_, std::move(task), w); + // The connection can outlive whatever chain invoked this + // launcher, so its frame comes from the process-wide + // recycling resource rather than that chain's allocator. + struct allocator_scope + { + std::pmr::memory_resource* prev = + capy::get_current_frame_allocator(); + allocator_scope() noexcept + { + capy::set_current_frame_allocator( + capy::get_recycling_memory_resource()); + } + ~allocator_scope() + { + capy::set_current_frame_allocator(prev); + } + }; + launch_wrapper wrapper = [&] { + allocator_scope scope; + return launch_coro{}( + ex, st, state_ref(st_), std::move(task), w); + }(); // Executor and stop token stored in promise via // constructor. Post through the frame-embedded @@ -568,13 +706,22 @@ class BOOST_COROSIO_DECL tcp_server @par !example tcp_server */ template - tcp_server(Ctx& ctx, Ex ex) : impl_(make_impl(ctx)) - , ex_(std::move(ex)) + tcp_server(Ctx& ctx, Ex ex) + : st_(make_state(ctx, capy::any_executor(std::move(ex)))) { } public: - /// Destroy the server, stopping all accept loops. + /** Destroy the server, stopping it. + + A running server is stopped and its listeners closed, so its + accept loops end. The loops and any active workers finish on + the server's executor without reaching the destroyed object; + the workers stay alive until they do. + + @par Thread Safety + Not thread safe. + */ ~tcp_server(); /// Copy construction is disabled; the server owns its worker storage. @@ -604,6 +751,9 @@ class BOOST_COROSIO_DECL tcp_server Multiple endpoints can be bound by calling this method multiple times before @ref start. + @pre No session is active: the server was never started, or + it was stopped and joined. + @param ep The local endpoint to bind to. @return An error code indicating success, or the reason binding @@ -617,6 +767,16 @@ class BOOST_COROSIO_DECL tcp_server previous workers are released and the idle/active lists are cleared before populating with new workers. + @pre No worker is waiting on or serving a connection: the + server was never started, or it was stopped and joined + and every connection has finished. + + @par Exception Safety + The server keeps its current workers if this throws. + + @throws std::logic_error If a worker is waiting on or serving + a connection. + @tparam Range Forward range of pointer-like objects to worker_base. @param workers Range of workers to manage. Each element must @@ -632,19 +792,24 @@ class BOOST_COROSIO_DECL tcp_server worker_base*> void set_workers(Range&& workers) { - // Clear existing state - storage_.reset(); - idle_head_ = nullptr; - active_head_ = nullptr; - active_tail_ = nullptr; + // The previous workers are released after the lock. + std::shared_ptr previous; + std::lock_guard lock(st_->mutex); + if (st_->lent != 0) + detail::throw_logic_error( + "tcp_server::set_workers: a worker is still in use"); - // Take ownership and populate idle list using StorageType = std::decay_t; auto* p = new StorageType(std::forward(workers)); - storage_ = std::shared_ptr( + std::shared_ptr storage( p, [](void* ptr) { delete static_cast(ptr); }); - for (auto&& elem : *static_cast(p)) - idle_push(std::to_address(elem)); + + st_->idle_head = nullptr; + st_->active_head = nullptr; + st_->active_tail = nullptr; + previous = std::exchange(st_->storage, std::move(storage)); + for (auto&& elem : *p) + st_->idle_push(std::to_address(elem)); } /** Start accepting connections. @@ -715,7 +880,8 @@ class BOOST_COROSIO_DECL tcp_server After calling `stop()`: 1. Let `ioc.run()` return (drains pending completions). 2. Call @ref join to wait for accept loops to finish. - 3. Only then is it safe to restart or destroy the server. + 3. Only then may the server be restarted. It may be destroyed at + any time; see @ref ~tcp_server. @par Thread Safety Not thread safe. @@ -754,7 +920,7 @@ class BOOST_COROSIO_DECL tcp_server void join(); private: - capy::task<> do_stop(); + void discard() noexcept; }; #ifdef _MSC_VER diff --git a/src/corosio/src/tcp_server.cpp b/src/corosio/src/tcp_server.cpp index bdbd91dd4..313d17d99 100644 --- a/src/corosio/src/tcp_server.cpp +++ b/src/corosio/src/tcp_server.cpp @@ -26,53 +26,85 @@ struct tcp_server::impl capy::execution_context& ctx; std::vector ports; std::stop_source stop; + // Guarded by join_mutex. Each listener is closed exactly once after + // the server is gone: by its loop's completion if the loop was still + // running when the server went, otherwise by the departing server. + std::vector loop_running; + bool discarded = false; explicit impl(capy::execution_context& c) noexcept : ctx(c) {} }; -tcp_server::impl* -tcp_server::make_impl(capy::execution_context& ctx) +tcp_server::state* +tcp_server::make_state( + capy::execution_context& ctx, capy::any_executor const& ex) { - return new impl(ctx); + auto* i = new impl(ctx); + try + { + return new state(i, ex); + } + catch (...) + { + delete i; + throw; + } +} + +void +tcp_server::release(state* s) noexcept +{ + if (s->refs.fetch_sub(1, std::memory_order_acq_rel) == 1) + { + delete s->impl_; + delete s; + } +} + +void +tcp_server::discard() noexcept +{ + if (!st_) + return; + auto& im = *st_->impl_; + std::vector idle; + { + std::lock_guard lock(im.join_mutex); + im.discarded = true; + for (std::size_t i = 0; i < im.ports.size(); ++i) + if (i >= im.loop_running.size() || !im.loop_running[i]) + idle.push_back(i); + } + for (auto i : idle) + im.ports[i].close(); + stop(); + release(std::exchange(st_, nullptr)); } tcp_server::~tcp_server() { - delete impl_; + discard(); } tcp_server::tcp_server(tcp_server&& o) noexcept - : impl_(std::exchange(o.impl_, nullptr)) - , ex_(o.ex_) - , waiters_(std::exchange(o.waiters_, nullptr)) - , idle_head_(std::exchange(o.idle_head_, nullptr)) - , active_head_(std::exchange(o.active_head_, nullptr)) - , active_tail_(std::exchange(o.active_tail_, nullptr)) - , active_accepts_(std::exchange(o.active_accepts_, 0)) - , storage_(std::move(o.storage_)) - , running_(std::exchange(o.running_, false)) + : st_(std::exchange(o.st_, nullptr)) { } tcp_server& tcp_server::operator=(tcp_server&& o) noexcept { - delete impl_; - impl_ = std::exchange(o.impl_, nullptr); - ex_ = o.ex_; - waiters_ = std::exchange(o.waiters_, nullptr); - idle_head_ = std::exchange(o.idle_head_, nullptr); - active_head_ = std::exchange(o.active_head_, nullptr); - active_tail_ = std::exchange(o.active_tail_, nullptr); - active_accepts_ = std::exchange(o.active_accepts_, 0); - storage_ = std::move(o.storage_); - running_ = std::exchange(o.running_, false); + if (this != &o) + { + discard(); + st_ = std::exchange(o.st_, nullptr); + } return *this; } // Accept loop: wait for idle worker, accept connection, dispatch capy::task -tcp_server::do_accept(tcp_acceptor& acc) +tcp_server::do_accept(state_ref st, tcp_acceptor& acc) { // Analyzer can't trace value through coroutine await_transform // NOLINTNEXTLINE(clang-analyzer-core.uninitialized.UndefReturn) @@ -80,14 +112,19 @@ tcp_server::do_accept(tcp_acceptor& acc) while (!env->stop_token.stop_requested()) { // Wait for an idle worker before blocking on accept - auto& w = co_await pop(); - auto [ec] = co_await acc.accept(w.socket()); + auto* w = co_await pop_awaitable{*st}; + if (!w) + break; + w->holds_.store(1, std::memory_order_relaxed); + auto [ec] = co_await acc.accept(w->socket()); if (ec) { - co_await push(w); + co_await push_awaitable{*st, *w}; continue; } - w.run(launcher{*this, w}); + w->holds_.fetch_add(1, std::memory_order_relaxed); + w->run(launcher{*st, *w}); + st->drop_hold(*w); } } @@ -96,7 +133,7 @@ tcp_server::bind(endpoint ep) { try { - impl_->ports.emplace_back(impl_->ctx, ep); + st_->impl_->ports.emplace_back(st_->impl_->ctx, ep); return {}; } catch (std::system_error const& e) @@ -108,68 +145,92 @@ tcp_server::bind(endpoint ep) endpoint tcp_server::local_endpoint(std::size_t index) const noexcept { - if (index >= impl_->ports.size()) + if (!st_ || index >= st_->impl_->ports.size()) return endpoint{}; - return impl_->ports[index].local_endpoint(); + return st_->impl_->ports[index].local_endpoint(); } void tcp_server::start() { // Idempotent - only start if not already running - if (running_) + if (st_->running) return; - // Previous session must be fully stopped before restart - if (active_accepts_ != 0) - detail::throw_logic_error( - "tcp_server::start: previous session not joined"); - - running_ = true; - - impl_->stop = {}; // Fresh stop source - auto st = impl_->stop.get_token(); - - active_accepts_ = impl_->ports.size(); + auto& im = *st_->impl_; + { + // Previous session must be fully stopped before restart + std::lock_guard lock(im.join_mutex); + if (st_->active_accepts != 0) + detail::throw_logic_error( + "tcp_server::start: previous session not joined"); + st_->active_accepts = im.ports.size(); + im.loop_running.assign(im.ports.size(), true); + } + { + std::lock_guard lock(st_->mutex); + st_->stopping = false; + } - // Launch with completion handler that decrements counter - for (auto& t : impl_->ports) - capy::run_async(ex_, st, [this]() { - std::lock_guard lock(impl_->join_mutex); - if (--active_accepts_ == 0) - impl_->join_cv.notify_all(); - })(do_accept(t)); + st_->running = true; + im.stop = {}; // Fresh stop source + auto token = im.stop.get_token(); + + // The completion runs once the loop's frame is done with its + // listener, so a close from here cannot race the loop. + for (std::size_t i = 0; i < im.ports.size(); ++i) + capy::run_async(st_->ex, token, [st = state_ref(st_), i]() { + auto& im = *st->impl_; + bool close = false; + { + std::lock_guard lock(im.join_mutex); + im.loop_running[i] = false; + close = im.discarded; + if (--st->active_accepts == 0) + im.join_cv.notify_all(); + } + if (close) + im.ports[i].close(); + })(do_accept(state_ref(st_), im.ports[i])); } void tcp_server::stop() { // Idempotent - only stop if running - if (!running_) + if (!st_ || !st_->running) return; - running_ = false; + st_->running = false; - // Stop accept loops - impl_->stop.request_stop(); + // Stop accept loops, including those waiting for a worker + st_->impl_->stop.request_stop(); + st_->stop_waiters(); // Launch cancellation coroutine on server executor - capy::run_async(ex_, std::stop_token{})(do_stop()); + capy::run_async(st_->ex, std::stop_token{})(do_stop(state_ref(st_))); } void tcp_server::join() { - std::unique_lock lock(impl_->join_mutex); - impl_->join_cv.wait(lock, [this] { return active_accepts_ == 0; }); + auto& im = *st_->impl_; + std::unique_lock lock(im.join_mutex); + im.join_cv.wait(lock, [this] { return st_->active_accepts == 0; }); } capy::task<> -tcp_server::do_stop() +tcp_server::do_stop(state_ref st) { - // Running on server executor - safe to iterate active list - // Just cancel, don't modify list - workers return themselves when done - for (auto* w = active_head_; w; w = w->next_) - w->stop_.request_stop(); + // Requested outside the lock: a stop callback may return its worker, + // which takes the lock. + std::vector active; + { + std::lock_guard lock(st->mutex); + for (auto* w = st->active_head; w; w = w->next_) + active.push_back(w->stop_); + } + for (auto& s : active) + s.request_stop(); co_return; } diff --git a/test/unit/tcp_server.cpp b/test/unit/tcp_server.cpp index 729cfa21c..83d85f17c 100644 --- a/test/unit/tcp_server.cpp +++ b/test/unit/tcp_server.cpp @@ -11,15 +11,24 @@ #include #include +#include #include #include +#include #include #include #include #include +#include +#include +#include #include +#include #include +#include +#include +#include #include "context.hpp" #include "test_suite.hpp" @@ -74,6 +83,29 @@ class test_server : public tcp_server } }; +// Counts the blocks it hands out that are still live. +struct counting_resource : std::pmr::memory_resource +{ + int live = 0; + + void* do_allocate(std::size_t n, std::size_t align) override + { + ++live; + return std::pmr::new_delete_resource()->allocate(n, align); + } + + void do_deallocate(void* p, std::size_t n, std::size_t align) override + { + --live; + std::pmr::new_delete_resource()->deallocate(p, n, align); + } + + bool do_is_equal(std::pmr::memory_resource const& o) const noexcept override + { + return this == &o; + } +}; + template struct tcp_server_test { @@ -929,6 +961,512 @@ struct tcp_server_test BOOST_TEST_EQ(run_count.load(), 2); } + // Sends one byte through a fresh connection and waits for the echo. + static bool echo_once(io_context& ioc, endpoint ep) + { + tcp_socket c(ioc); + bool ok = false; + capy::run_async(ioc.get_executor())( + [](tcp_socket& c, endpoint ep, bool& ok) -> capy::task<> { + auto [cec] = co_await c.connect(ep); + if (cec) + co_return; + char out = 'x'; + auto [wec, wn] = + co_await c.write_some(capy::const_buffer(&out, 1)); + if (wec) + co_return; + char in = 0; + auto [rec, rn] = + co_await c.read_some(capy::mutable_buffer(&in, 1)); + ok = !rec && rn == 1 && in == 'x'; + }(c, ep, ok)); + ioc.restart(); + // Bounds a hang, not a timing assertion. + while (!ok && ioc.run_one_for(std::chrono::seconds(5)) != 0) + { + } + return ok; + } + + // Destroying a running server stops it: its accept loops and the + // workers they launched finish without touching the destroyed + // object, and its listeners close. + void testDestroyWhileRunning() + { + io_context ioc(Backend); + endpoint ep; + { + test_server srv(ioc); + BOOST_TEST(!srv.bind(endpoint(ipv4_address::loopback(), 0))); + ep = srv.local_endpoint(); + srv.start(); + BOOST_TEST(echo_once(ioc, ep)); + } + ioc.restart(); + ioc.run(); // returns: nothing of the server is left running + + tcp_socket c(ioc); + std::error_code cec; + capy::run_async(ioc.get_executor())( + [](tcp_socket& c, endpoint ep, std::error_code& ec) + -> capy::task<> { + auto [e] = co_await c.connect(ep); + ec = e; + }(c, ep, cec)); + ioc.restart(); + ioc.run(); + BOOST_TEST(!!cec); // the listener is closed + } + + // A running server keeps serving after it is moved. + void testMoveWhileRunning() + { + io_context ioc(Backend); + test_server src(ioc); + BOOST_TEST(!src.bind(endpoint(ipv4_address::loopback(), 0))); + auto ep = src.local_endpoint(); + src.start(); + BOOST_TEST(echo_once(ioc, ep)); + + { + test_server dst(std::move(src)); + BOOST_TEST(echo_once(ioc, ep)); + BOOST_TEST(echo_once(ioc, ep)); + } + ioc.restart(); + ioc.run(); // the destroyed server's loops finish + } + + // Workers return from several run threads at once. + void testConcurrentWorkersOnMultithreadedContext() + { + io_context ioc(Backend); + auto srv = std::make_unique(ioc); + BOOST_TEST(!srv->bind(endpoint(ipv4_address::loopback(), 0))); + auto ep = srv->local_endpoint(); + srv->start(); + + std::vector runners; + for (int i = 0; i < 4; ++i) + runners.emplace_back([&] { ioc.run(); }); + + std::atomic echoed{0}; + std::vector clients; + for (int i = 0; i < 4; ++i) + clients.emplace_back([&] { + io_context cctx(Backend); + for (int k = 0; k < 25; ++k) + if (echo_once(cctx, ep)) + ++echoed; + }); + for (auto& t : clients) + t.join(); + BOOST_TEST_EQ(echoed.load(), 100); + + ioc.stop(); + for (auto& t : runners) + t.join(); + srv.reset(); + ioc.restart(); + ioc.run(); // the destroyed server's loops finish + } + + // Connects and closes at once, so connections keep arriving. The + // close resets: a graceful one would park each connection's port + // in TIME_WAIT, and at this rate that drains the ephemeral range + // for every test running alongside. + static void connect_once(io_context& ioc, endpoint ep) + { + tcp_socket c(ioc); + BOOST_TEST(!c.open()); + c.set_option(socket_option::linger(true, 0)); + capy::run_async(ioc.get_executor())( + [](tcp_socket& c, endpoint ep) -> capy::task<> { + std::ignore = co_await c.connect(ep); + }(c, ep)); + ioc.restart(); + ioc.run(); + } + + static std::error_code connect_result(io_context& ioc, endpoint ep) + { + tcp_socket c(ioc); + std::error_code ec; + capy::run_async(ioc.get_executor())( + [](tcp_socket& c, endpoint ep, std::error_code& ec) + -> capy::task<> { + auto [e] = co_await c.connect(ep); + ec = e; + }(c, ep, ec)); + ioc.restart(); + ioc.run(); + return ec; + } + + // A worker that keeps its launcher instead of starting it, so the + // test decides when, or whether, the connection runs. + class holding_worker : public tcp_server::worker_base + { + corosio::tcp_socket sock_; + std::optional* slot_; + + public: + holding_worker( + io_context& ctx, std::optional* s) + : sock_(ctx) + , slot_(s) + { + } + + corosio::tcp_socket& socket() override + { + return sock_; + } + + void run(tcp_server::launcher launch) override + { + slot_->emplace(std::move(launch)); + } + }; + + class holding_server : public tcp_server + { + public: + holding_server( + io_context& ctx, std::optional* s) + : tcp_server(ctx, ctx.get_executor()) + { + std::vector> v; + v.push_back(std::make_unique(ctx, s)); + set_workers(std::move(v)); + } + }; + + // An accept loop parked for a worker, because the only worker is + // still busy, must not keep the listener open once the server is + // destroyed. + void testDestroyWithBusyWorkerClosesListener() + { + io_context ioc(Backend); + std::optional held; + + auto srv = std::make_unique(ioc, &held); + BOOST_TEST(!srv->bind(endpoint(ipv4_address::loopback(), 0))); + auto ep = srv->local_endpoint(); + srv->start(); + + io_context cctx; + BOOST_TEST(!connect_result(cctx, ep)); + // Bounds a hang, not a timing assertion. + while (!held && ioc.run_one_for(std::chrono::seconds(5)) != 0) + { + } + BOOST_TEST(held.has_value()); + while (ioc.poll() != 0) + { + } + + srv.reset(); + while (ioc.poll() != 0) + { + } + + BOOST_TEST(!!connect_result(cctx, ep)); + + held.reset(); + ioc.restart(); + ioc.run(); + } + + // A connection dispatched before stop() but launched after it + // must still see the stop. + void testStopReachesLaunchAfterStop() + { + io_context ioc(Backend); + std::optional held; + holding_server srv(ioc, &held); + BOOST_TEST(!srv.bind(endpoint(ipv4_address::loopback(), 0))); + auto ep = srv.local_endpoint(); + srv.start(); + + io_context cctx; + BOOST_TEST(!connect_result(cctx, ep)); + // Bounds a hang, not a timing assertion. + while (!held && ioc.run_one_for(std::chrono::seconds(5)) != 0) + { + } + BOOST_TEST(held.has_value()); + + srv.stop(); + while (ioc.poll() != 0) + { + } + + bool done = false; + bool canceled = false; + (*held)( + ioc.get_executor(), + [](bool& d, bool& c) -> capy::task<> { + auto [ec] = co_await corosio::delay(std::chrono::hours(1)); + c = ec == capy::cond::canceled; + d = true; + }(done, canceled)); + held.reset(); + ioc.restart(); + while (!done && ioc.run_one_for(std::chrono::seconds(5)) != 0) + { + } + BOOST_TEST(done); + BOOST_TEST(canceled); + ioc.run(); + srv.join(); + } + + // The connection a launcher starts outlives the chain that invoked + // it, so it must not be allocated from that chain's allocator. + void testLateLaunchOutlivesInvokingChain() + { + io_context ioc(Backend); + std::optional held; + holding_server srv(ioc, &held); + BOOST_TEST(!srv.bind(endpoint(ipv4_address::loopback(), 0))); + auto ep = srv.local_endpoint(); + srv.start(); + + io_context cctx; + BOOST_TEST(!connect_result(cctx, ep)); + // Bounds a hang, not a timing assertion. + while (!held && ioc.run_one_for(std::chrono::seconds(5)) != 0) + { + } + BOOST_TEST(held.has_value()); + + counting_resource mr; + bool done = false; + auto session = []() -> capy::task<> { co_return; }(); + capy::run_async(ioc.get_executor(), &mr, [&]() { done = true; })( + [](tcp_server::launcher& l, capy::task<> t, + io_context::executor_type ex) -> capy::task<> { + l(ex, std::move(t)); + co_return; + }(*held, std::move(session), ioc.get_executor())); + while (!done && ioc.run_one() != 0) + { + } + BOOST_TEST(done); + BOOST_TEST_EQ(mr.live, 0); + + held.reset(); + srv.stop(); + ioc.run(); + srv.join(); + } + + struct overlap_counts + { + std::atomic in_run{0}; + std::atomic max_in_run{0}; + std::atomic runs{0}; + }; + + // Starts its connection on another context, then lingers in run() + // until a second run() begins or a bound passes. + class lingering_worker : public tcp_server::worker_base + { + corosio::tcp_socket sock_; + io_context& session_ctx_; + overlap_counts& counts_; + + public: + lingering_worker( + io_context& ctx, io_context& session_ctx, overlap_counts& counts) + : sock_(ctx) + , session_ctx_(session_ctx) + , counts_(counts) + { + } + + corosio::tcp_socket& socket() override + { + return sock_; + } + + void run(tcp_server::launcher launch) override + { + int const n = ++counts_.in_run; + int seen = counts_.max_in_run.load(); + while (n > seen && + !counts_.max_in_run.compare_exchange_weak(seen, n)) + { + } + ++counts_.runs; + launch( + session_ctx_.get_executor(), + [](corosio::tcp_socket* s) -> capy::task<> { + s->close(); + co_return; + }(&sock_)); + // Bounds the wait, not a timing assertion. + auto const until = std::chrono::steady_clock::now() + + std::chrono::milliseconds(500); + while (counts_.runs.load() < 2 && + std::chrono::steady_clock::now() < until) + std::this_thread::yield(); + --counts_.in_run; + } + }; + + // A worker is not handed to another connection while its run() is + // still executing, even once the connection it launched is done. + void testWorkerNotReusedDuringRun() + { + io_context ioc(Backend); + io_context session_ctx(Backend); + overlap_counts counts; + + class server : public tcp_server + { + public: + server(io_context& ctx, io_context& sctx, overlap_counts& c) + : tcp_server(ctx, ctx.get_executor()) + { + std::vector> v; + v.push_back(std::make_unique(ctx, sctx, c)); + set_workers(std::move(v)); + } + }; + + server srv(ioc, session_ctx, counts); + BOOST_TEST(!srv.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!srv.bind(endpoint(ipv4_address::loopback(), 0))); + io_context cctx; + BOOST_TEST(!connect_result(cctx, srv.local_endpoint(0))); + BOOST_TEST(!connect_result(cctx, srv.local_endpoint(1))); + + auto sex = session_ctx.get_executor(); + sex.on_work_started(); + std::thread session_thread([&] { session_ctx.run(); }); + + srv.start(); + std::thread t1([&] { ioc.run(); }); + std::thread t2([&] { ioc.run(); }); + + // Bounds a hang, not a timing assertion. + auto const until = + std::chrono::steady_clock::now() + std::chrono::seconds(10); + while ((counts.runs.load() < 2 || counts.in_run.load() != 0) && + std::chrono::steady_clock::now() < until) + std::this_thread::yield(); + + srv.stop(); + t1.join(); + t2.join(); + srv.join(); + sex.on_work_finished(); + session_thread.join(); + + BOOST_TEST_EQ(counts.runs.load(), 2); + BOOST_TEST_EQ(counts.max_in_run.load(), 1); + } + + // An accept loop holds a worker while it waits for a connection, + // so replacing the workers then would free one in use. + void testSetWorkersWhileAcceptingThrows() + { + io_context ioc(Backend); + test_server srv(ioc); + BOOST_TEST(!srv.bind(endpoint(ipv4_address::loopback(), 0))); + srv.start(); + while (ioc.poll() != 0) + { + } + + BOOST_TEST_THROWS( + srv.set_workers(make_test_workers(ioc, 1)), std::logic_error); + + srv.stop(); + ioc.run(); + srv.join(); + srv.set_workers(make_test_workers(ioc, 1)); + } + + // A launcher kept past join still borrows its worker. + void testSetWorkersWithBorrowedWorkerThrows() + { + io_context ioc(Backend); + std::optional held; + holding_server srv(ioc, &held); + BOOST_TEST(!srv.bind(endpoint(ipv4_address::loopback(), 0))); + auto ep = srv.local_endpoint(); + srv.start(); + + io_context cctx; + BOOST_TEST(!connect_result(cctx, ep)); + // Bounds a hang, not a timing assertion. + while (!held && ioc.run_one_for(std::chrono::seconds(5)) != 0) + { + } + BOOST_TEST(held.has_value()); + + srv.stop(); + ioc.run(); + srv.join(); + BOOST_TEST_THROWS( + srv.set_workers(make_test_workers(ioc, 1)), std::logic_error); + + held.reset(); + srv.set_workers(make_test_workers(ioc, 1)); + } + + // Destroying the server from a thread that is not running its + // executor must not race the accept loops still running there. + void testDestroyWhileRunnersAccept() + { + for (int round = 0; round < 20; ++round) + { + io_context ioc(Backend); + auto srv = std::make_unique(ioc); + BOOST_TEST(!srv->bind(endpoint(ipv4_address::loopback(), 0))); + auto ep = srv->local_endpoint(); + srv->start(); + + std::vector runners; + for (int i = 0; i < 4; ++i) + runners.emplace_back([&] { ioc.run(); }); + + std::atomic quit{false}; + std::atomic connects{0}; + std::vector clients; + // The default backend for the clients: io_uring maps its + // rings with raw syscalls the thread sanitizer cannot see, + // so each round's client threads would appear to race the + // last round's over reused addresses. + for (int i = 0; i < 3; ++i) + clients.emplace_back([&] { + io_context cctx; + while (!quit.load()) + { + connect_once(cctx, ep); + ++connects; + } + }); + while (connects.load() < 20) + std::this_thread::yield(); + + srv.reset(); + quit = true; + for (auto& t : clients) + t.join(); + ioc.stop(); + for (auto& t : runners) + t.join(); + ioc.restart(); + ioc.run(); // the destroyed server's loops finish + } + BOOST_TEST_PASS(); + } + void run() { testLauncherDropWakesWaitingAccept(); @@ -951,6 +1489,16 @@ struct tcp_server_test testLauncherDoubleInvokeThrows(); testLauncherDtorReturnsWorker(); testMultipleActiveConnections(); + testDestroyWhileRunning(); + testMoveWhileRunning(); + testConcurrentWorkersOnMultithreadedContext(); + testDestroyWhileRunnersAccept(); + testDestroyWithBusyWorkerClosesListener(); + testStopReachesLaunchAfterStop(); + testLateLaunchOutlivesInvokingChain(); + testWorkerNotReusedDuringRun(); + testSetWorkersWhileAcceptingThrows(); + testSetWorkersWithBorrowedWorkerThrows(); } }; From 00b7bbd178b7aaa4e12326cbbf0ce9c1b2ff8af6 Mon Sep 17 00:00:00 2001 From: Steve Gerbino Date: Tue, 13 Oct 2026 17:54:58 +0200 Subject: [PATCH 7/8] feat: tear down suspended coroutines through their chain's owner Destroying an io_context left coroutines suspended in its operations, timers, signal waits, blocking-pool work and queues suspended forever, with their frames and everything they held leaked. Each holder now gives a continuation it will never resume to capy::abandon, which unwinds the coroutine's chain from its owner on the owner's executor: reactor, io_uring and IOCP operations, timers, signal waits, resolver and file work on the blocking pool, posted work, and tcp_server connections and workers. Schedulers finish with themselves before work posted to them can run, wait for cancellations still posting before they drain, destroy work posted after the drain, and stop under their lock. A lockless io_uring context can be torn down from any thread. Select is woken before the lock an operation parked under is released, and posting a ready reactor operation is its initiator's last touch. Blocking-pool operations count as work on the context that owns them, and a run call restores the thread's frame allocator. A completed operation whose executor throws terminates. tcp_server holds work on a connection's context until it ends, returns workers when that context is torn down, gives back a launch whose post throws, and transfers from its launch coroutine as a task does. Socket and mocket pairs finish their setup before returning or throwing. Teardown tests run under AddressSanitizer, and the documentation states what io_context teardown destroys and the order between contexts. --- .../ROOT/pages/3.tutorials/3f.reconnect.adoc | 6 +- .../ROOT/pages/4.guide/4c.io-context.adoc | 23 +- .../ROOT/pages/4.guide/4k.tcp-server.adoc | 14 +- include/boost/corosio/backend.hpp | 2 +- .../boost/corosio/detail/dispatch_coro.hpp | 6 +- include/boost/corosio/detail/op_base.hpp | 3 + include/boost/corosio/detail/platform.hpp | 2 - include/boost/corosio/detail/scheduler.hpp | 44 +- include/boost/corosio/detail/thread_pool.hpp | 53 +- .../corosio/detail/timeout_awaitable.hpp | 17 +- include/boost/corosio/detail/timer.hpp | 13 +- .../boost/corosio/detail/timer_service.hpp | 26 +- include/boost/corosio/io_context.hpp | 67 +- .../boost/corosio/native/detail/coro_op.hpp | 26 +- .../native/detail/coro_op_complete.hpp | 9 +- .../win_local_stream_acceptor_service.hpp | 6 +- .../detail/iocp/win_local_stream_service.hpp | 12 +- .../detail/iocp/win_object_handle_service.hpp | 6 +- .../detail/iocp/win_overlapped_handle.hpp | 7 +- .../native/detail/iocp/win_overlapped_op.hpp | 7 - .../native/detail/iocp/win_resolver.hpp | 9 +- .../detail/iocp/win_resolver_service.hpp | 27 +- .../native/detail/iocp/win_scheduler.hpp | 133 ++-- .../native/detail/iocp/win_signals.hpp | 35 +- .../detail/iocp/win_tcp_acceptor_service.hpp | 6 +- .../native/detail/iocp/win_tcp_service.hpp | 12 +- .../native/detail/iocp/win_udp_service.hpp | 18 +- .../detail/posix/posix_random_access_file.hpp | 61 +- .../posix_random_access_file_service.hpp | 29 +- .../native/detail/posix/posix_resolver.hpp | 14 +- .../detail/posix/posix_resolver_service.hpp | 70 +- .../native/detail/posix/posix_signal.hpp | 2 +- .../detail/posix/posix_signal_service.hpp | 41 +- .../native/detail/posix/posix_stream_file.hpp | 23 +- .../posix/posix_stream_file_service.hpp | 48 +- .../detail/reactor/reactor_acceptor.hpp | 16 +- .../reactor/reactor_acceptor_service.hpp | 5 - .../native/detail/reactor/reactor_backend.hpp | 19 +- .../reactor/reactor_descriptor_service.hpp | 5 - .../native/detail/reactor/reactor_io_core.hpp | 25 +- .../native/detail/reactor/reactor_op.hpp | 1 - .../native/detail/reactor/reactor_op_base.hpp | 6 +- .../detail/reactor/reactor_scheduler.hpp | 65 +- .../detail/reactor/reactor_socket_service.hpp | 5 - .../detail/uring/uring_acceptor_ops.hpp | 4 + .../native/detail/uring/uring_file_ops.hpp | 10 + .../detail/uring/uring_multishot_acceptor.hpp | 2 + .../detail/uring/uring_random_access_file.hpp | 6 + .../native/detail/uring/uring_scheduler.hpp | 176 +++-- .../native/detail/uring/uring_socket_ops.hpp | 22 +- .../corosio/native/native_io_context.hpp | 5 + include/boost/corosio/tcp_server.hpp | 171 +++-- include/boost/corosio/test/mocket.hpp | 24 +- include/boost/corosio/test/socket_pair.hpp | 24 +- src/corosio/src/tcp_server.cpp | 6 +- src/corosio/src/timer.cpp | 15 +- test/unit/context.hpp | 8 +- test/unit/delay.cpp | 22 + test/unit/fault/posix_faults.cpp | 2 +- test/unit/io_context.cpp | 75 ++ test/unit/iocp_paths.cpp | 6 - test/unit/local_datagram_socket.cpp | 3 - test/unit/local_stream_socket.cpp | 5 +- test/unit/native/native_io.cpp | 35 + test/unit/pool_teardown.hpp | 43 +- test/unit/random_access_file.cpp | 40 +- test/unit/reactor_paths.cpp | 3 - test/unit/reactor_teardown_inflight.cpp | 4 +- test/unit/resolver.cpp | 52 +- test/unit/signal_set.cpp | 48 +- test/unit/stream_file.cpp | 40 +- test/unit/tcp_acceptor.cpp | 80 +- test/unit/tcp_server.cpp | 699 +++++++++++++++++- test/unit/tcp_socket.cpp | 96 ++- test/unit/teardown_destroy.cpp | 515 +++++++++++++ test/unit/teardown_inflight.cpp | 8 +- test/unit/test/mocket.cpp | 26 + test/unit/test/socket_pair.cpp | 25 + test/unit/thread_pool.cpp | 92 ++- test/unit/udp_socket.cpp | 3 - 80 files changed, 2766 insertions(+), 653 deletions(-) create mode 100644 test/unit/teardown_destroy.cpp diff --git a/doc/modules/ROOT/pages/3.tutorials/3f.reconnect.adoc b/doc/modules/ROOT/pages/3.tutorials/3f.reconnect.adoc index c6ca20a14..e14854b71 100644 --- a/doc/modules/ROOT/pages/3.tutorials/3f.reconnect.adoc +++ b/doc/modules/ROOT/pages/3.tutorials/3f.reconnect.adoc @@ -125,8 +125,8 @@ Contrast with calling `stop()` directly: 1. `run()` exits immediately. 2. The coroutine remains suspended: it never sees an error. -3. `~io_context` calls `h.destroy()` on the coroutine frame, bypassing its - error-handling logic. +3. `~io_context` abandons the coroutine: its frames are unwound from the top, + bypassing its error-handling logic. Both paths are safe (no leaks or crashes), but only the stop token path executes the coroutine's own cleanup code. @@ -144,7 +144,7 @@ executes the coroutine's own cleanup code. | Pause and resume the event loop | `~io_context` -| No: frames destroyed via `h.destroy()` +| No: frames destroyed without resuming | Final cleanup (after `stop()` or natural exit) |=== diff --git a/doc/modules/ROOT/pages/4.guide/4c.io-context.adoc b/doc/modules/ROOT/pages/4.guide/4c.io-context.adoc index 71652b2e7..f7d28cd34 100644 --- a/doc/modules/ROOT/pages/4.guide/4c.io-context.adoc +++ b/doc/modules/ROOT/pages/4.guide/4c.io-context.adoc @@ -239,9 +239,26 @@ threads that ran the loop, and only then destroy the context. include::example$snippets/4c_io_context.cpp[tag=teardown,indent=0] ---- -WARNING: `stop()` abandons pending work rather than draining it. Destroying the -context after `stop()` while other threads may still post to it is undefined -behavior — join those threads first. +WARNING: `stop()` leaves pending work suspended rather than draining it. +Destroying the context after `stop()` while other threads may still post to it +is undefined behavior — join those threads first. + +Destroying the cpp:io_context[] abandons every coroutine still suspended in its +operations or queued on it. Each chain is unwound from the top, without +resuming, on the executor of the coroutine that started it. A chain started on +this context is unwound while the context is destroyed. A chain started on +another execution context, for example with `capy::run_async` on that context's +executor, is unwound there, when that context runs or is destroyed. + +So work must not cross contexts at teardown. A coroutine started on one context +may wait on an operation of another, or its frames may own I/O objects of +another. No order of destruction is then safe, because its unwind runs after +one of the two contexts is gone. Stop such work, and run both contexts until it +ends, before destroying either. + +Work posted after the services shut down, for example from a service's +destructor, is destroyed after every other service. Its destructors must not +use them. == Inheritance from execution_context diff --git a/doc/modules/ROOT/pages/4.guide/4k.tcp-server.adoc b/doc/modules/ROOT/pages/4.guide/4k.tcp-server.adoc index a3871025b..397e4ed9f 100644 --- a/doc/modules/ROOT/pages/4.guide/4k.tcp-server.adoc +++ b/doc/modules/ROOT/pages/4.guide/4k.tcp-server.adoc @@ -97,7 +97,8 @@ include::example$snippets/4k_tcp_server.cpp[tag=launcher_run,indent=0] The launcher: -1. Starts your coroutine on the specified executor +1. Starts your coroutine on the specified executor, counting it as work there + until it ends 2. Tracks the worker as in-use 3. Returns the worker to the pool when the coroutine completes @@ -119,6 +120,17 @@ I/O context's executor: include::example$snippets/4k_tcp_server.cpp[tag=launch_call,indent=0] ---- +A coroutine started on another context's executor keeps that context's `run()` +going until the connection ends. Such a connection crosses contexts. Its +worker's socket, and the server state it returns the worker to, belong to the +server's context, so that context must outlive the connection. The +connection's own context may be destroyed first, but only while the connection +is not waiting on an operation of the server's context. A read on the worker's +socket is such an operation. Destroying that context then abandons the +connection and returns its worker. +Otherwise, stop the server and run both contexts until their `run()` returns, +then destroy them. + == Binding and Starting === bind() diff --git a/include/boost/corosio/backend.hpp b/include/boost/corosio/backend.hpp index 94b8ffd57..1b7e50254 100644 --- a/include/boost/corosio/backend.hpp +++ b/include/boost/corosio/backend.hpp @@ -21,7 +21,7 @@ class execution_context; namespace boost::corosio { namespace detail { -struct scheduler; +class scheduler; } // namespace detail #if BOOST_COROSIO_HAS_EPOLL diff --git a/include/boost/corosio/detail/dispatch_coro.hpp b/include/boost/corosio/detail/dispatch_coro.hpp index 1ef1b9576..a03afab8f 100644 --- a/include/boost/corosio/detail/dispatch_coro.hpp +++ b/include/boost/corosio/detail/dispatch_coro.hpp @@ -31,6 +31,8 @@ namespace boost::corosio::detail { for symmetric transfer. Callers at the scheduler pump level should call `.resume()` on the result. + @note A `dispatch` that throws calls `std::terminate`. + @param ex The executor to dispatch through. @param c The continuation to dispatch. Must remain at a stable address until dequeued by the executor. @@ -38,8 +40,10 @@ namespace boost::corosio::detail { @return A handle for symmetric transfer or `std::noop_coroutine()`. */ inline std::coroutine_handle<> -dispatch_coro(capy::executor_ref ex, capy::continuation& c) +dispatch_coro(capy::executor_ref ex, capy::continuation& c) noexcept { + // An exception from dispatch would leave the completed operation + // with nobody to resume or abandon its coroutine. // A completion from another context must not resume the awaiter // on this thread, so the fast path still asks the executor. if (auto const* io_ex = ex.target()) diff --git a/include/boost/corosio/detail/op_base.hpp b/include/boost/corosio/detail/op_base.hpp index 796614d9b..25660c32a 100644 --- a/include/boost/corosio/detail/op_base.hpp +++ b/include/boost/corosio/detail/op_base.hpp @@ -74,6 +74,7 @@ class bytes_op_base return h; } cont_.h = h; + cont_.owner = env->owner; return static_cast(this)->dispatch( cont_, env->executor); } @@ -128,6 +129,7 @@ class value_op_base return h; } cont_.h = h; + cont_.owner = env->owner; return static_cast(this)->dispatch( cont_, env->executor); } @@ -180,6 +182,7 @@ class void_op_base return h; } cont_.h = h; + cont_.owner = env->owner; return static_cast(this)->dispatch( cont_, env->executor); } diff --git a/include/boost/corosio/detail/platform.hpp b/include/boost/corosio/detail/platform.hpp index 58795e9c7..674122ccd 100644 --- a/include/boost/corosio/detail/platform.hpp +++ b/include/boost/corosio/detail/platform.hpp @@ -59,8 +59,6 @@ #endif // io_uring - Linux 6.0+ proactor (requires liburing 2.5+ at build time). -// Single-threaded mode additionally requires Linux 6.1+ for -// IORING_SETUP_DEFER_TASKRUN; multi-threaded mode runs on 6.0. #if defined(__linux__) && BOOST_COROSIO_HAVE_LIBURING #define BOOST_COROSIO_HAS_URING 1 #else diff --git a/include/boost/corosio/detail/scheduler.hpp b/include/boost/corosio/detail/scheduler.hpp index 08c76725d..2c28a006b 100644 --- a/include/boost/corosio/detail/scheduler.hpp +++ b/include/boost/corosio/detail/scheduler.hpp @@ -19,9 +19,11 @@ #include #include +#include #include #include #include +#include namespace boost::corosio::detail { @@ -39,9 +41,10 @@ class scheduler_op; @see io_context */ -struct BOOST_COROSIO_DECL scheduler - : capy::execution_context::service +class BOOST_COROSIO_DECL scheduler + : public capy::execution_context::service { +public: using key_type = scheduler; ~scheduler() override = default; @@ -134,6 +137,43 @@ struct BOOST_COROSIO_DECL scheduler /// Apply @ref threading_config. virtual void configure_threading(threading_config) noexcept = 0; + + /** Note a canceller that claimed an operation and has yet to post it. + + Called no later than the claim becomes visible: under the lock + that makes it, or before a lock-free claim. A service's + shutdown then either finds the operation or finds it counted + here. + */ + void cancel_posting_begin() noexcept + { + cancels_posting_.fetch_add(1, std::memory_order_relaxed); + } + + /// End a @ref cancel_posting_begin once its post is made. + void cancel_posting_end() noexcept + { + cancels_posting_.fetch_sub(1, std::memory_order_release); + } + +protected: + /** Wait out the cancellers counted by @ref cancel_posting_begin. + + Called before the shutdown drain, so it finds their posts; the + services that own those operations shut down earlier and missed + them. + */ + void wait_cancels_posting() const noexcept + { + while (cancels_posting_.load(std::memory_order_acquire) != 0) + std::this_thread::yield(); + } + +private: + BOOST_COROSIO_MSVC_WARNING_PUSH + BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) + std::atomic cancels_posting_{0}; + BOOST_COROSIO_MSVC_WARNING_POP }; /** Return the scheduler registered with the context. diff --git a/include/boost/corosio/detail/thread_pool.hpp b/include/boost/corosio/detail/thread_pool.hpp index 4d11635ca..ba623f93f 100644 --- a/include/boost/corosio/detail/thread_pool.hpp +++ b/include/boost/corosio/detail/thread_pool.hpp @@ -58,6 +58,15 @@ struct pool_work_item : intrusive_queue::node /// Completion handler invoked by the worker thread. func_type func_ = nullptr; + + /** Teardown for an item the pool shuts down before running. + + An operation that owns a suspended coroutine sets this, so the + coroutine is abandoned at shutdown instead of the pool making + the blocking call nobody is left to wait for. When null, the + item is run by `func_` instead. + */ + func_type destroy_ = nullptr; }; /** Shared thread pool for dispatching blocking operations. @@ -78,13 +87,15 @@ struct pool_work_item : intrusive_queue::node @par Shutdown Sets a shutdown flag, notifies all threads, and joins them. In-flight blocking calls complete naturally before the thread - exits. + exits. Items still queued are not run: each is handed to its + `destroy_`, or run by `func_` on the shutting-down thread when it + has none. @note Create this service after the scheduler its work items post completions to. Services shut down newest first, so a pool created earlier joins its workers only after the scheduler has drained its completion queue, and the completion the last worker posts is then - neither run nor destroyed. + destroyed only when the scheduler is deleted. @note The type is symbol-visible because services are keyed by type identity: with RTTI, hidden behind a shared library boundary, a @@ -160,7 +171,7 @@ class BOOST_COROSIO_SYMBOL_VISIBLE thread_pool final Safe. Racing first posts start the workers once. @param w The work item to execute. Must remain valid until - its `func_` has been called. + its `func_` or `destroy_` has been called. @return An empty code if the item was enqueued; `capy::error::canceled` if the pool has already shut @@ -185,8 +196,9 @@ class BOOST_COROSIO_SYMBOL_VISIBLE thread_pool final /** Shut down the thread pool. - Signals all threads to exit after draining any - remaining queued work, then joins them. + Signals all threads to exit once their current item is done, + joins them, then disposes of every item still queued without + running its blocking work where the item says how. */ void shutdown() override; }; @@ -208,13 +220,11 @@ thread_pool::worker_loop(unsigned index) cv_.wait( lock, [this] { return shutdown_ || !work_queue_.empty(); }); + // What is still queued is left to shutdown(), which does not + // make blocking calls whose results nobody will read. + if (shutdown_) + return; w = work_queue_.pop(); - if (!w) - { - if (shutdown_) - return; - continue; - } } w->func_(w); } @@ -294,20 +304,29 @@ thread_pool::shutdown() } threads_.clear(); + intrusive_queue leftover; { std::lock_guard lock(mutex_); - while (work_queue_.pop()) - ; + leftover.splice(work_queue_); + } + // Unlocked: an abandoned coroutine's unwind can reach post() + // through the objects its frame owned. + while (auto* w = leftover.pop()) + { + if (w->destroy_) + w->destroy_(w); + else + w->func_(w); } } /** A reference to the context's shared thread pool, bound on first use. Services that hand blocking work to the pool hold one of these - instead of a reference bound at construction. They are constructed - from the scheduler's constructor, where the pool they created would - be older than the scheduler and would join too late; binding on - first use puts the pool after it instead. + instead of a reference bound at construction. A service made before + the pool, as one made by the scheduler's constructor would be, would + otherwise create the pool itself, older than the scheduler and so + joined too late; binding on first use puts the pool after it. The owning `io_context` creates the pool service during construction, so by the time any operation can run the binding only diff --git a/include/boost/corosio/detail/timeout_awaitable.hpp b/include/boost/corosio/detail/timeout_awaitable.hpp index c6008ff49..a3dcb72e1 100644 --- a/include/boost/corosio/detail/timeout_awaitable.hpp +++ b/include/boost/corosio/detail/timeout_awaitable.hpp @@ -90,15 +90,17 @@ struct timeout_awaitable using time_point = std::chrono::steady_clock::time_point; using stop_cb_type = std::stop_callback; + // Declared before inner_: a task destroyed while lent reads the + // environment it was given. + std::stop_source stop_src_; + std::stop_token parent_token_; + capy::io_env inner_env_; A inner_; std::optional timer_; time_point deadline_; std::chrono::nanoseconds dur_{}; - bool has_deadline_ = true; - std::stop_source stop_src_; - std::stop_token parent_token_; - capy::io_env inner_env_; alignas(stop_cb_type) unsigned char cb_buf_[sizeof(stop_cb_type)]; + bool has_deadline_ = true; bool cb_active_ = false; /// Construct without a timer, deadline given as an absolute time. @@ -124,12 +126,12 @@ struct timeout_awaitable // Only moved before await_suspend, when cb_active_ is false timeout_awaitable(timeout_awaitable&& o) noexcept( std::is_nothrow_move_constructible_v) - : inner_(std::move(o.inner_)) + : stop_src_(std::move(o.stop_src_)) + , inner_(std::move(o.inner_)) , timer_(std::move(o.timer_)) , deadline_(o.deadline_) , dur_(o.dur_) , has_deadline_(o.has_deadline_) - , stop_src_(std::move(o.stop_src_)) { } @@ -210,7 +212,8 @@ struct timeout_awaitable // Start the inner op with our interposed stop_token inner_env_ = { - env->executor, stop_src_.get_token(), env->frame_allocator}; + env->executor, stop_src_.get_token(), env->frame_allocator, + env->owner}; return inner_.await_suspend(h, &inner_env_); } diff --git a/include/boost/corosio/detail/timer.hpp b/include/boost/corosio/detail/timer.hpp index 0fa95b331..36d11c0e1 100644 --- a/include/boost/corosio/detail/timer.hpp +++ b/include/boost/corosio/detail/timer.hpp @@ -380,7 +380,7 @@ class BOOST_COROSIO_DECL timer : public io_object awaitable on the suspended coroutine's frame — never allocated. Once published by `implementation::wait()` the node may be completed from any thread; every completion path finishes - touching the node before resuming or destroying the coroutine, + touching the node before resuming or abandoning the coroutine, because either act may end the node's storage. The node owns no resources: the stop token is borrowed from the @@ -407,7 +407,7 @@ struct BOOST_COROSIO_SYMBOL_VISIBLE waiter_node completion_op() noexcept : scheduler_op(&do_complete) {} - BOOST_COROSIO_DECL void operator()() override; + BOOST_COROSIO_DECL void operator()() noexcept override; BOOST_COROSIO_DECL void destroy() override; }; @@ -427,9 +427,6 @@ struct BOOST_COROSIO_SYMBOL_VISIBLE waiter_node /// The timer service that completes this waiter. timer_service* svc_ = nullptr; - /// The suspended coroutine, destroyed by the shutdown drains. - std::coroutine_handle<> h_; - /// The continuation posted to resume the coroutine. capy::continuation cont_; @@ -490,9 +487,9 @@ struct BOOST_COROSIO_SYMBOL_VISIBLE waiter_node */ void bind(std::coroutine_handle<> h, capy::io_env const& env) noexcept { - h_ = h; - cont_.h = h; - d_ = env.executor; + cont_.h = h; + cont_.owner = env.owner; + d_ = env.executor; token_ = &env.stop_token; } diff --git a/include/boost/corosio/detail/timer_service.hpp b/include/boost/corosio/detail/timer_service.hpp index 7fbaaff5a..846769ce7 100644 --- a/include/boost/corosio/detail/timer_service.hpp +++ b/include/boost/corosio/detail/timer_service.hpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -36,7 +37,7 @@ namespace boost::corosio::detail { -struct scheduler; +class scheduler; /* Timer Service @@ -316,12 +317,12 @@ timer_service::shutdown() pool_.shutdown([](timer::implementation*) {}); // Snapshot impls and detach them from the heap so that - // coroutine-owned timer destructors (triggered by h.destroy() - // below) cannot re-enter remove_timer_impl() and mutate the - // vector during iteration. A stop requested on another thread + // coroutine-owned timer destructors (run by the unwinds the + // abandons below start) cannot re-enter remove_timer_impl() and + // mutate the vector during iteration. A stop requested on another thread // reaches cancel_waiter() under the same lock: whichever side // claims a waiter first owns it, and a waiter its canceller - // claimed is posted, so the scheduler's drain destroys its frame. + // claimed is posted, so the scheduler's drain abandons it. std::vector impls; std::vector waiters; { @@ -360,20 +361,19 @@ timer_service::shutdown() } // Cancel waiting timers. Each waiter called work_started() - // in implementation::wait(). On IOCP the scheduler shutdown - // loop exits when outstanding_work_ reaches zero, so we must - // call work_finished() here to balance it. On other backends - // this is harmless. Resetting the stop callback outside the lock + // in implementation::wait(); a waiter given up here never + // completes, so its work is finished here to keep the count + // balanced. Resetting the stop callback outside the lock // waits out a canceller already running, which finds the waiter // claimed. for (auto* w : waiters) { w->reset_stop_cb(); - auto h = std::exchange(w->h_, {}); + auto c = std::exchange(w->cont_, {}); sched_->work_finished(); - // Destroying the frame also ends the node's storage - if (h) - h.destroy(); + // Unwinding the frame also ends the node's storage + if (c.h) + capy::abandon(c); } for (auto* impl : impls) diff --git a/include/boost/corosio/io_context.hpp b/include/boost/corosio/io_context.hpp index e105ad5b0..d21822df5 100644 --- a/include/boost/corosio/io_context.hpp +++ b/include/boost/corosio/io_context.hpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -56,7 +57,8 @@ enum class locking_mode @par Restrictions - Only one thread may call `run()` (or any run variant). - Posting work from another thread is undefined behavior. - - DNS resolution returns `operation_not_supported`. + - DNS resolution returns `operation_not_supported`, except on + IOCP, whose resolver does not use the thread pool. - POSIX file I/O returns `operation_not_supported`. - `win_object_handle::assign()` returns `operation_not_supported`. - Signal sets should not be shared across contexts. */ @@ -212,7 +214,9 @@ effective_concurrency_hint( outstanding work remains. Finally join the threads that ran the loop. Only then destroy the context. Work started with `capy::run` / `capy::run_async` is work-tracked, so a normal - `run()` completion already waits for it. + `run()` completion already waits for it. Destroying the context + abandons every coroutine still suspended in it (see + @ref ~io_context). @par Exception Safety A context that constructs is usable. The infrastructure its backend @@ -221,6 +225,10 @@ effective_concurrency_hint( throws from the constructor rather than from the first operation. The failed construction leaves nothing open. + An operation that completes resumes its coroutine through the + executor the coroutine awaited it on. If that executor's `post` or + `dispatch` throws, `std::terminate` is called. + @par Thread Safety Distinct objects: Safe.@n Shared objects: Safe, unless the context was constructed with a @@ -253,6 +261,25 @@ class BOOST_COROSIO_DECL io_context : public capy::execution_context protected: detail::scheduler* sched_; + // Resuming a chain installs its frame allocator on the running + // thread; a run call puts back what it found, so a coroutine created + // afterwards never reaches a resource that died with those frames. + struct frame_allocator_restore + { + std::pmr::memory_resource* saved = + capy::get_current_frame_allocator(); + + frame_allocator_restore() = default; + frame_allocator_restore(frame_allocator_restore const&) = delete; + frame_allocator_restore& + operator=(frame_allocator_restore const&) = delete; + + ~frame_allocator_restore() + { + capy::set_current_frame_allocator(saved); + } + }; + public: /** Dispatches and posts work to this context; see the executor_type definition below. */ @@ -361,7 +388,29 @@ class BOOST_COROSIO_DECL io_context : public capy::execution_context apply_options_post_(opts, eff); } - /// Destroy the context; stops the loop and destroys every service. + /** Destroy the context, its services and the coroutines it holds. + + The destructor stops the loop and destroys every service. Each + coroutine suspended in an operation of this context, or posted to + it and not yet resumed, is abandoned. Its chain is unwound from + the top, on the executor of the chain's owner (see + `capy::abandon`). The precondition below makes that executor this + context's, so the unwind runs during this destruction. + + @pre No thread is executing a run variant. + + @pre No work crosses into another execution context. A chain + held here must be owned on this context. No frame of a chain + owned elsewhere may own I/O objects of this context, or wait + on a primitive that a chain held here owns. Such a chain's + unwind would run on the other context after this one is gone, + whatever the order of destruction. Stop such work and run both + contexts until it ends before destroying either. + + @note Work posted after the services have shut down, for example + from a service's destructor, is abandoned while the services + are being destroyed. Its destructors must not use them. + */ ~io_context(); /// Copy construction is disabled; the context owns its services. @@ -421,6 +470,7 @@ class BOOST_COROSIO_DECL io_context : public capy::execution_context */ std::size_t run() { + frame_allocator_restore restore; return sched_->run(); } @@ -437,6 +487,7 @@ class BOOST_COROSIO_DECL io_context : public capy::execution_context */ std::size_t run_one() { + frame_allocator_restore restore; return sched_->run_one(); } @@ -519,6 +570,7 @@ class BOOST_COROSIO_DECL io_context : public capy::execution_context std::size_t run_one_until(std::chrono::time_point const& abs_time) { + frame_allocator_restore restore; typename Clock::time_point now = Clock::now(); for (;;) { @@ -557,6 +609,7 @@ class BOOST_COROSIO_DECL io_context : public capy::execution_context */ std::size_t poll() { + frame_allocator_restore restore; return sched_->poll(); } @@ -573,6 +626,7 @@ class BOOST_COROSIO_DECL io_context : public capy::execution_context */ std::size_t poll_one() { + frame_allocator_restore restore; return sched_->poll_one(); } }; @@ -689,11 +743,18 @@ class io_context::executor_type via the `post(capy::continuation&)` overload to avoid the allocation. + A handle still queued when the context is torn down is + destroyed alone, since it carries no owner to unwind its chain + from. Post a coroutine that another frame awaits through the + `post(capy::continuation&)` overload, with its owner set. + @param h The coroutine handle to post. @pre The associated context must outlive this call. Posting concurrently with, or after, the context's destruction is undefined behavior. + + @pre Nothing awaits `h`. */ void post(std::coroutine_handle<> h) const { diff --git a/include/boost/corosio/native/detail/coro_op.hpp b/include/boost/corosio/native/detail/coro_op.hpp index 9743519d4..853c3ed28 100644 --- a/include/boost/corosio/native/detail/coro_op.hpp +++ b/include/boost/corosio/native/detail/coro_op.hpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -23,6 +24,7 @@ #include #include #include +#include /* Shared, non-template op envelope for every native backend — the readiness @@ -95,8 +97,8 @@ struct coro_op : scheduler_op std::optional> stop_cb; /// Keeps the owning impl alive while the op is in flight (the kernel - /// owns user buffers until completion). Dropped in the handler's resume - /// tail (see coro_op_complete.hpp). + /// owns user buffers until completion). Released when the op resumes or + /// abandons its coroutine (see coro_op_complete.hpp). detail::object_ref object_ref_; /// Default-construct for virtual-dispatch backends (the reactors, which @@ -121,6 +123,26 @@ struct coro_op : scheduler_op stop_cb.emplace(token, canceller{this}); } + /** Give up the awaiting coroutine without resuming it. + + Used when the operation is drained at teardown: the operation + holds the suspended coroutine, so it abandons it, which unwinds + its chain from the chain's owner. The keepalive is held across + the call because the unwind may run at once and destroy the + object whose impl stores this op; nothing here touches the op + afterwards. + */ + void destroy_awaiting() noexcept + { + stop_cb.reset(); + auto keep = std::move(object_ref_); + if (auto* c = std::exchange(cont, nullptr)) + { + if (c->h) + capy::abandon(*c); + } + } + /// Mark this op cancellation-requested. Shared by every backend. void request_cancel() noexcept { diff --git a/include/boost/corosio/native/detail/coro_op_complete.hpp b/include/boost/corosio/native/detail/coro_op_complete.hpp index aa35da246..37cff08fa 100644 --- a/include/boost/corosio/native/detail/coro_op_complete.hpp +++ b/include/boost/corosio/native/detail/coro_op_complete.hpp @@ -113,18 +113,19 @@ decode_io_result( @param self The completing op. @return True if this was a shutdown drain — the caller must `return` immediately without decoding or resuming. On that path the - object_ref_ keepalive is dropped here (which may destroy the impl, - and with it the op storage). + awaiting coroutine is abandoned (see `coro_op::destroy_awaiting`) + and the object_ref_ keepalive dropped, which may destroy the impl + and with it the op storage. */ inline bool coro_drain_if_shutdown(void* owner, coro_op* self) noexcept { - self->stop_cb.reset(); if (owner == nullptr) { - auto suicide = std::move(self->object_ref_); + self->destroy_awaiting(); return true; } + self->stop_cb.reset(); return false; } diff --git a/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor_service.hpp b/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor_service.hpp index 27259f4c4..a93432735 100644 --- a/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor_service.hpp @@ -142,8 +142,7 @@ local_stream_accept_op::do_complete( op->peer_wrapper = nullptr; } - op->cleanup_only(); - op->object_ref_.reset(); + op->destroy_awaiting(); return; } @@ -231,8 +230,7 @@ local_stream_acceptor_wait_op::do_complete( if (!owner) { - op->cleanup_only(); - op->object_ref_.reset(); + op->destroy_awaiting(); return; } diff --git a/include/boost/corosio/native/detail/iocp/win_local_stream_service.hpp b/include/boost/corosio/native/detail/iocp/win_local_stream_service.hpp index dabdc4b06..6eadc9ecd 100644 --- a/include/boost/corosio/native/detail/iocp/win_local_stream_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_local_stream_service.hpp @@ -217,8 +217,7 @@ local_stream_connect_op::do_complete( if (!owner) { - op->cleanup_only(); - op->object_ref_.reset(); + op->destroy_awaiting(); return; } @@ -261,8 +260,7 @@ local_stream_read_op::do_complete( if (!owner) { - op->cleanup_only(); - op->object_ref_.reset(); + op->destroy_awaiting(); return; } @@ -285,8 +283,7 @@ local_stream_write_op::do_complete( if (!owner) { - op->cleanup_only(); - op->object_ref_.reset(); + op->destroy_awaiting(); return; } @@ -309,8 +306,7 @@ local_stream_wait_op::do_complete( if (!owner) { - op->cleanup_only(); - op->object_ref_.reset(); + op->destroy_awaiting(); return; } diff --git a/include/boost/corosio/native/detail/iocp/win_object_handle_service.hpp b/include/boost/corosio/native/detail/iocp/win_object_handle_service.hpp index 788877d8b..bdd0f3df5 100644 --- a/include/boost/corosio/native/detail/iocp/win_object_handle_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_object_handle_service.hpp @@ -241,7 +241,7 @@ class win_object_handle_state if (!owner) { self->set_idle(); - op->cleanup_only(); + op->destroy_awaiting(); return; } @@ -306,12 +306,16 @@ class win_object_handle_state } ::WaitForThreadpoolWaitCallbacks(tp_wait_, FALSE); + // Counted from before the claim: a claimed op is invisible to the + // scheduler's shutdown until on_completion queues it. + sched_.cancel_posting_begin(); std::uint64_t cur = state_.load(std::memory_order_acquire); if ((cur & phase_mask) == phase_armed && claim(cur)) { op_.request_cancel(); sched_.on_completion(&op_, ERROR_OPERATION_ABORTED, 0); } + sched_.cancel_posting_end(); } win_scheduler& sched_; diff --git a/include/boost/corosio/native/detail/iocp/win_overlapped_handle.hpp b/include/boost/corosio/native/detail/iocp/win_overlapped_handle.hpp index 6666c594e..ea454c625 100644 --- a/include/boost/corosio/native/detail/iocp/win_overlapped_handle.hpp +++ b/include/boost/corosio/native/detail/iocp/win_overlapped_handle.hpp @@ -346,8 +346,7 @@ class win_slot_handle : public win_handle_base op->in_flight.store(false, std::memory_order_release); if (!owner) { - op->cleanup_only(); - op->object_ref_.reset(); + op->destroy_awaiting(); return; } @@ -496,8 +495,10 @@ class win_concurrent_handle : public win_handle_base if (!owner) { - op->cont = nullptr; + auto* c = std::exchange(op->cont, nullptr); self->recycle_op(op); + if (c && c->h) + capy::abandon(*c); return; } diff --git a/include/boost/corosio/native/detail/iocp/win_overlapped_op.hpp b/include/boost/corosio/native/detail/iocp/win_overlapped_op.hpp index 12470259e..1629c92b4 100644 --- a/include/boost/corosio/native/detail/iocp/win_overlapped_op.hpp +++ b/include/boost/corosio/native/detail/iocp/win_overlapped_op.hpp @@ -238,13 +238,6 @@ struct overlapped_op dispatch_coro(ex, *cont).resume(); } - - /** Disarm cancellation and abandon the continuation. */ - void cleanup_only() - { - stop_cb.reset(); - cont = nullptr; - } }; /** Cast OVERLAPPED* to overlapped_op*. diff --git a/include/boost/corosio/native/detail/iocp/win_resolver.hpp b/include/boost/corosio/native/detail/iocp/win_resolver.hpp index f4ad5a93e..7e8d04041 100644 --- a/include/boost/corosio/native/detail/iocp/win_resolver.hpp +++ b/include/boost/corosio/native/detail/iocp/win_resolver.hpp @@ -121,7 +121,7 @@ extern "C" 2. GetAddrInfoExW() is called with our completion callback 3. If it returns WSA_IO_PENDING, completion comes later via callback 4. If it returns immediately (0 or error), we post completion manually - 5. completion() callback stores error, calls work_finished(), posts op + 5. completion() callback stores error, posts op 6. op_() resumes the coroutine with results or error Reverse Resolution (GetNameInfoW) @@ -142,8 +142,8 @@ extern "C" Work Tracking ------------- work_started() is called before async operations to keep io_context alive. - work_finished() is called when the operation completes (in callback for - forward resolution, in worker thread for reverse resolution). + work_finished() is called when the posted operation completes or is + drained at shutdown. */ namespace boost::corosio::detail { @@ -310,6 +310,9 @@ class win_resolver final /// Execute blocking `GetNameInfoW()` on a pool thread. static void do_reverse_resolve_work(pool_work_item*) noexcept; + /// Destroy a reverse resolution the pool shut down before running. + static void destroy_reverse_resolve_work(pool_work_item*) noexcept; + private: /// Cancel the in-flight forward lookup, if any. void cancel_lookup() noexcept; diff --git a/include/boost/corosio/native/detail/iocp/win_resolver_service.hpp b/include/boost/corosio/native/detail/iocp/win_resolver_service.hpp index aba3a07a5..b1556c3db 100644 --- a/include/boost/corosio/native/detail/iocp/win_resolver_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_resolver_service.hpp @@ -288,16 +288,15 @@ resolve_op::do_complete( if (!owner) { - // Destroy path - op->stop_cb.reset(); if (op->results) { ::FreeAddrInfoExW(op->results); op->results = nullptr; } - // Dropping the keepalive may destroy the implementation this op - // is embedded in, so nothing may touch it afterwards. - op->object_ref_.reset(); + // The abandon may unwind the frame at once, destroying the + // implementation this op is embedded in, so nothing may touch + // it afterwards. + op->destroy_awaiting(); svc.work_finished(); return; } @@ -354,10 +353,10 @@ reverse_resolve_op::do_complete( if (!owner) { - op->stop_cb.reset(); - // Dropping the keepalive may destroy the implementation this - // op is embedded in, so nothing may touch it afterwards. - op->object_ref_.reset(); + // The abandon may unwind the frame at once, destroying the + // implementation this op is embedded in, so nothing may touch + // it afterwards. + op->destroy_awaiting(); svc.work_finished(); return; } @@ -519,6 +518,7 @@ win_resolver::reverse_resolve( reverse_pool_op_.resolver_ = this; reverse_pool_op_.ref_ = detail::object_ref(this); reverse_pool_op_.func_ = &win_resolver::do_reverse_resolve_work; + reverse_pool_op_.destroy_ = &win_resolver::destroy_reverse_resolve_work; if (auto pec = svc_.pool().post(&reverse_pool_op_)) { // The pool is shutting down, or the system refused it a thread. @@ -618,6 +618,15 @@ win_resolver::do_reverse_resolve_work(pool_work_item* w) noexcept self->svc_.post(&self->reverse_op_); } +inline void +win_resolver::destroy_reverse_resolve_work(pool_work_item* w) noexcept +{ + auto* pw = static_cast(w); + auto* self = pw->resolver_; + self->reverse_op_.object_ref_ = std::move(pw->ref_); + self->reverse_op_.destroy(); +} + // win_resolver_service inline win_resolver_service::win_resolver_service( diff --git a/include/boost/corosio/native/detail/iocp/win_scheduler.hpp b/include/boost/corosio/native/detail/iocp/win_scheduler.hpp index 875462dba..e002de352 100644 --- a/include/boost/corosio/native/detail/iocp/win_scheduler.hpp +++ b/include/boost/corosio/native/detail/iocp/win_scheduler.hpp @@ -24,6 +24,7 @@ #include #include +#include #include #include @@ -112,6 +113,14 @@ class BOOST_COROSIO_DECL win_scheduler final void update_timeout(); private: + /// Destroy every packet and deferred entry still queued, without + /// running them; stops once the port is empty. + void sweep_port() noexcept; + + // Posts an allocated copy, which survives the caller and keeps the + // owner for teardown. + void post_copy(capy::continuation const& c) const; + static void on_timer_changed(void* ctx); void post_deferred_completions(op_queue& ops); std::size_t do_one(unsigned long timeout_ms); @@ -124,7 +133,7 @@ class BOOST_COROSIO_DECL win_scheduler final // overlapped-op memory: kernel completions owed after a pending // submission, plus successful stored-result posts. Shutdown reaps // until this is zero before the services free op storage. The - // run-loop counter cannot serve that role: frames abandoned at + // run-loop counter cannot serve that role: frames destroyed at // teardown never return their work-guard credits. mutable long pending_io_ = 0; mutable long stopped_; @@ -218,54 +227,39 @@ struct thread_context_guard inline void win_scheduler::post(std::coroutine_handle<> h) const +{ + post_copy(capy::continuation{h}); +} + +inline void +win_scheduler::post_copy(capy::continuation const& c) const { struct post_handler final : scheduler_op { - std::coroutine_handle<> h_; + capy::continuation c_; static void do_complete( void* owner, scheduler_op* base, std::uint32_t, std::uint32_t) { auto* self = static_cast(base); + auto c = self->c_; + delete self; if (!owner) { - // Shutdown path: destroy the coroutine frame synchronously. - // - // Bounded destruction invariant: the chain triggered by - // coro.destroy() is at most two levels deep: - // 1. task frame destroyed → ~io_awaitable_promise_base() - // destroys stored continuation (if != noop_coroutine) - // 2. continuation (trampoline) destroyed → final_suspend - // returns suspend_never, no further continuation - // - // If a future refactor adds deeper continuation chains, - // this would reintroduce re-entrant stack overflow risk. -#ifndef NDEBUG - static thread_local int destroy_depth = 0; - ++destroy_depth; - BOOST_COROSIO_ASSERT(destroy_depth <= 2); -#endif - auto coro = self->h_; - delete self; - coro.destroy(); -#ifndef NDEBUG - --destroy_depth; -#endif + capy::abandon(c); return; } - auto coro = self->h_; - delete self; - coro.resume(); + c.h.resume(); } - explicit post_handler(std::coroutine_handle<> coro) + explicit post_handler(capy::continuation const& c) : scheduler_op(&do_complete) - , h_(coro) + , c_(c) { } }; - auto* ph = new post_handler(h); + auto* ph = new post_handler(c); ::InterlockedIncrement(&outstanding_work_); if (!::PostQueuedCompletionStatus( @@ -300,10 +294,10 @@ win_scheduler::post(capy::continuation& c) const iocp_, 0, key_continuation, reinterpret_cast(&c))) { // completed_ops_ is an op_queue and cannot carry a raw continuation, - // so on the rare PQCS failure fall back to the allocating handle - // path. Drop the increment first; post(c.h) does its own accounting. + // so on the rare PQCS failure fall back to an allocated copy, which + // keeps the owner. Drop the increment first; post_copy counts its own. ::InterlockedDecrement(&outstanding_work_); - post(c.h); + post_copy(c); } } @@ -343,26 +337,24 @@ win_scheduler::on_pending(overlapped_op* op) const // skip (uncounted) and the re-post is the dispatching packet. A // failed re-post falls back to the deferred queue, which holds the // op memory itself — no packet, no count. + // + // Counted before the packet can be consumed: once it can, a context + // whose thread runs it may finish and be destroyed, so this call + // touches nothing of the scheduler afterwards. + ::InterlockedIncrement(&pending_io_); long expected = 0; if (op->ready_.compare_exchange_strong( expected, 1, std::memory_order_acq_rel, std::memory_order_acquire)) - { - ::InterlockedIncrement(&pending_io_); - } - else - { - if (::PostQueuedCompletionStatus( - iocp_, 0, key_result_stored, static_cast(op))) - { - ::InterlockedIncrement(&pending_io_); - } - else - { - std::lock_guard lock(dispatch_mutex_); - completed_ops_.push(op); - ::InterlockedExchange(&dispatch_required_, 1); - } - } + return; + if (::PostQueuedCompletionStatus( + iocp_, 0, key_result_stored, static_cast(op))) + return; + std::lock_guard lock(dispatch_mutex_); + completed_ops_.push(op); + ::InterlockedExchange(&dispatch_required_, 1); + // Uncounted only once queued, so a shutdown that sees no packet + // left finds the op in the queue. + ::InterlockedDecrement(&pending_io_); } inline void @@ -375,17 +367,17 @@ win_scheduler::on_completion(overlapped_op* op, DWORD error, DWORD bytes) const op->bytes_transferred = bytes; op->ready_.store(1, std::memory_order_release); + // Counted before the packet can be consumed, as in on_pending. + ::InterlockedIncrement(&pending_io_); if (::PostQueuedCompletionStatus( iocp_, 0, key_result_stored, static_cast(op))) - { - ::InterlockedIncrement(&pending_io_); - } - else - { - std::lock_guard lock(dispatch_mutex_); - completed_ops_.push(op); - ::InterlockedExchange(&dispatch_required_, 1); - } + return; + std::lock_guard lock(dispatch_mutex_); + completed_ops_.push(op); + ::InterlockedExchange(&dispatch_required_, 1); + // Uncounted only once queued, so a shutdown that sees no packet + // left finds the op in the queue. + ::InterlockedDecrement(&pending_io_); } inline void @@ -765,8 +757,9 @@ win_scheduler::shutdown() timers_->stop(); // Drain timer heap before the work-counting loop. The timer_service - // was registered after this scheduler (nested make_service from our - // constructor), so execution_context::shutdown() calls us first. + // is created from our constructor, so it finishes, and is linked, + // first; execution_context::shutdown() runs newest first, so it + // calls us first. // Asio avoids this by owning timer queues directly inside the // scheduler; we bridge the gap by shutting down the timer service // early. The subsequent call from execution_context is a no-op. @@ -778,6 +771,8 @@ win_scheduler::shutdown() // posts them as cancelled and the pending count can reach zero. wait_reactor_->stop(); + wait_cancels_posting(); + // Reap every packet still owed to the port before the services // free the op memory those packets reference. Work-guard credits, // posted handlers, and queued continuations have no bearing here. @@ -808,10 +803,10 @@ win_scheduler::shutdown() } else if (key == key_continuation) { - // Drain without resuming: destroy the parked frame. + // Drain without resuming: abandon the parked coroutine. auto* c = reinterpret_cast(overlapped); if (c->h) - c->h.destroy(); + capy::abandon(*c); } else { @@ -826,6 +821,12 @@ win_scheduler::shutdown() // without a pending_io_ count (posted directly against the // handle). Destroy them so service teardown does not free state // they still reference. + sweep_port(); +} + +inline void +win_scheduler::sweep_port() noexcept +{ for (;;) { op_queue ops; @@ -858,7 +859,7 @@ win_scheduler::shutdown() { auto* c = reinterpret_cast(overlapped); if (c->h) - c->h.destroy(); + capy::abandon(*c); } else { @@ -873,7 +874,11 @@ inline win_scheduler::~win_scheduler() wait_reactor_.reset(); if (iocp_ != nullptr) + { + // Service destructors run after the drain and may still post. + sweep_port(); ::CloseHandle(iocp_); + } } inline win_wait_reactor& diff --git a/include/boost/corosio/native/detail/iocp/win_signals.hpp b/include/boost/corosio/native/detail/iocp/win_signals.hpp index 9c1a2a993..7de3680e2 100644 --- a/include/boost/corosio/native/detail/iocp/win_signals.hpp +++ b/include/boost/corosio/native/detail/iocp/win_signals.hpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -29,8 +30,10 @@ #include #include +#include #include #include +#include #include #include @@ -225,7 +228,7 @@ class BOOST_COROSIO_DECL win_signals final @param impl The signal implementation to cancel. */ - void cancel_wait(win_signal& impl); + void cancel_wait(win_signal& impl) noexcept; /** Cancel an in-flight wait on behalf of a stop token. @@ -339,7 +342,14 @@ signal_op::do_complete( auto keep = std::move(op->keep); if (!owner) + { + // The held reference outlives the destroy: the coroutine may + // own the set this op is embedded in. + auto* dc = std::exchange(op->cont, nullptr); + if (dc && dc->h) + capy::abandon(*dc); return; + } if (op->ec_out) *op->ec_out = {}; @@ -462,6 +472,7 @@ win_signals::shutdown() // ~win_signal destroys an armed stop_cb_ whose ~stop_callback // waits on a token_canceller that takes mutex_. std::vector doomed; + std::vector parked; { signal_detail::signal_state* state = signal_detail::get_signal_state(); @@ -476,9 +487,16 @@ win_signals::shutdown() }); for (auto* impl : doomed) { - // A parked wait is abandoned like every other op at - // teardown; a later destroy() must not resume it mid-drain. - impl->waiting_ = false; + // The parked wait's coroutine is abandoned once the locks + // drop: its unwind can destroy the set, which takes them. + if (std::exchange(impl->waiting_, false)) + { + if (auto* c = std::exchange(impl->pending_op_.cont, nullptr)) + { + if (c->h) + parked.push_back(*c); + } + } while (auto* reg = impl->signals_) { @@ -509,6 +527,9 @@ win_signals::shutdown() registrations_[i] = nullptr; } + for (auto& c : parked) + capy::abandon(c); + for (auto* impl : doomed) release(impl); } @@ -674,7 +695,7 @@ win_signals::clear_signals(win_signal& impl) } inline void -win_signals::cancel_wait(win_signal& impl) +win_signals::cancel_wait(win_signal& impl) noexcept { bool was_waiting = false; signal_op* op = nullptr; @@ -687,6 +708,7 @@ win_signals::cancel_wait(win_signal& impl) was_waiting = true; impl.waiting_ = false; op = &impl.pending_op_; + sched_.cancel_posting_begin(); } } @@ -698,6 +720,7 @@ win_signals::cancel_wait(win_signal& impl) *op->signal_out = 0; op->d.post(*op->cont); sched_.work_finished(); + sched_.cancel_posting_end(); } } @@ -717,6 +740,7 @@ win_signals::cancel_wait_token(win_signal& impl) noexcept was_waiting = true; impl.waiting_ = false; op = &impl.pending_op_; + sched_.cancel_posting_begin(); } } @@ -728,6 +752,7 @@ win_signals::cancel_wait_token(win_signal& impl) noexcept *op->signal_out = 0; op->d.post(*op->cont); sched_.work_finished(); + sched_.cancel_posting_end(); } } diff --git a/include/boost/corosio/native/detail/iocp/win_tcp_acceptor_service.hpp b/include/boost/corosio/native/detail/iocp/win_tcp_acceptor_service.hpp index c937107cf..bdc001e84 100644 --- a/include/boost/corosio/native/detail/iocp/win_tcp_acceptor_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_tcp_acceptor_service.hpp @@ -144,8 +144,7 @@ accept_op::do_complete( op->peer_wrapper = nullptr; } - op->cleanup_only(); - op->object_ref_.reset(); + op->destroy_awaiting(); return; } @@ -233,8 +232,7 @@ acceptor_wait_op::do_complete( if (!owner) { - op->cleanup_only(); - op->object_ref_.reset(); + op->destroy_awaiting(); return; } diff --git a/include/boost/corosio/native/detail/iocp/win_tcp_service.hpp b/include/boost/corosio/native/detail/iocp/win_tcp_service.hpp index 7ab73c1aa..fb84613a2 100644 --- a/include/boost/corosio/native/detail/iocp/win_tcp_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_tcp_service.hpp @@ -311,8 +311,7 @@ connect_op::do_complete( if (!owner) { - op->cleanup_only(); - op->object_ref_.reset(); + op->destroy_awaiting(); return; } @@ -352,8 +351,7 @@ read_op::do_complete( if (!owner) { - op->cleanup_only(); - op->object_ref_.reset(); + op->destroy_awaiting(); return; } @@ -374,8 +372,7 @@ write_op::do_complete( if (!owner) { - op->cleanup_only(); - op->object_ref_.reset(); + op->destroy_awaiting(); return; } @@ -396,8 +393,7 @@ wait_op::do_complete( if (!owner) { - op->cleanup_only(); - op->object_ref_.reset(); + op->destroy_awaiting(); return; } diff --git a/include/boost/corosio/native/detail/iocp/win_udp_service.hpp b/include/boost/corosio/native/detail/iocp/win_udp_service.hpp index 43997f69b..43c5a687e 100644 --- a/include/boost/corosio/native/detail/iocp/win_udp_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_udp_service.hpp @@ -177,8 +177,7 @@ send_to_op::do_complete( if (!owner) { - op->cleanup_only(); - op->object_ref_.reset(); + op->destroy_awaiting(); return; } @@ -199,8 +198,7 @@ recv_from_op::do_complete( if (!owner) { - op->cleanup_only(); - op->object_ref_.reset(); + op->destroy_awaiting(); return; } @@ -272,8 +270,7 @@ udp_connect_op::do_complete( if (!owner) { - op->cleanup_only(); - op->object_ref_.reset(); + op->destroy_awaiting(); return; } @@ -306,8 +303,7 @@ udp_send_op::do_complete( if (!owner) { - op->cleanup_only(); - op->object_ref_.reset(); + op->destroy_awaiting(); return; } @@ -326,8 +322,7 @@ udp_recv_op::do_complete( if (!owner) { - op->cleanup_only(); - op->object_ref_.reset(); + op->destroy_awaiting(); return; } @@ -346,8 +341,7 @@ udp_wait_op::do_complete( if (!owner) { - op->cleanup_only(); - op->object_ref_.reset(); + op->destroy_awaiting(); return; } diff --git a/include/boost/corosio/native/detail/posix/posix_random_access_file.hpp b/include/boost/corosio/native/detail/posix/posix_random_access_file.hpp index 792345ac4..23721f381 100644 --- a/include/boost/corosio/native/detail/posix/posix_random_access_file.hpp +++ b/include/boost/corosio/native/detail/posix/posix_random_access_file.hpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -43,6 +44,7 @@ #include #include #include +#include #include #include @@ -65,7 +67,7 @@ namespace boost::corosio::detail { -struct scheduler; +class scheduler; class posix_random_access_file_service; /** Random-access file implementation for POSIX backends. */ @@ -109,10 +111,24 @@ class posix_random_access_file final void operator()() override; void destroy() override; + /// Recycle an op the pool refused, leaving its coroutine to + /// the initiator that answers it. + void discard() noexcept; + /// Thread-pool work function: executes preadv/pwritev. static void do_work(pool_work_item*) noexcept; + + /// Destroy an op the pool shut down before running. + static void destroy_work(pool_work_item*) noexcept; + + /// Return this op to its file's free list. + void recycle() noexcept; }; + /// Release the work an op holds on its service's context. + static void + release_work(posix_random_access_file_service& svc) noexcept; + explicit posix_random_access_file( posix_random_access_file_service& svc) noexcept; @@ -356,31 +372,58 @@ posix_random_access_file::raf_op::operator()() auto keep = std::move(object_ref_); auto* c = cont; auto exec = ex; + auto& svc = file_->svc_; { std::lock_guard lock(file_->ops_mutex_); file_->outstanding_ops_.remove(this); file_->free_ops_.push_front(this); } keep.reset(); - exec.on_work_finished(); + release_work(svc); dispatch_coro(exec, *c).resume(); } // -- raf_op shutdown cleanup -- +inline void +posix_random_access_file::raf_op::recycle() noexcept +{ + std::lock_guard lock(file_->ops_mutex_); + file_->outstanding_ops_.remove(this); + file_->free_ops_.push_front(this); +} + inline void posix_random_access_file::raf_op::destroy() { stop_cb.reset(); auto keep = std::move(object_ref_); - auto exec = ex; - { - std::lock_guard lock(file_->ops_mutex_); - file_->outstanding_ops_.remove(this); - file_->free_ops_.push_front(this); - } + auto& svc = file_->svc_; + auto* c = std::exchange(cont, nullptr); + recycle(); + // The frame may own the file, so it goes while the keepalive + // still holds the implementation the op was recycled into. + if (c && c->h) + capy::abandon(*c); + keep.reset(); + release_work(svc); +} + +inline void +posix_random_access_file::raf_op::discard() noexcept +{ + stop_cb.reset(); + auto keep = std::move(object_ref_); + auto& svc = file_->svc_; + recycle(); keep.reset(); - exec.on_work_finished(); + release_work(svc); +} + +inline void +posix_random_access_file::raf_op::destroy_work(pool_work_item* w) noexcept +{ + static_cast(w)->destroy(); } } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/posix/posix_random_access_file_service.hpp b/include/boost/corosio/native/detail/posix/posix_random_access_file_service.hpp index 9fc0f0f5a..c1575ea03 100644 --- a/include/boost/corosio/native/detail/posix/posix_random_access_file_service.hpp +++ b/include/boost/corosio/native/detail/posix/posix_random_access_file_service.hpp @@ -203,23 +203,24 @@ posix_random_access_file::read_some_at( op->object_ref_ = detail::object_ref(this); op->start(token); - op->ex.on_work_started(); + svc_.work_started(); { std::lock_guard lock(ops_mutex_); outstanding_ops_.push_back(op); } - static_cast(op)->func_ = &raf_op::do_work; + static_cast(op)->func_ = &raf_op::do_work; + static_cast(op)->destroy_ = &raf_op::destroy_work; if (auto pec = svc_.pool().post(static_cast(op))) { // The pool is shutting down, or the system refused it a thread. // Nothing of this read went cross-thread, so it answers here // like the closed-descriptor and zero-length exits above rather // than through a completion the scheduler has to carry back. - // destroy() is the discard the op never reaching the queue - // needs: it unlinks, unwinds the work count and recycles. - op->destroy(); + // The op never reached the queue: it unlinks, unwinds the + // work count and recycles, and the coroutine resumes below. + op->discard(); *ec = pec; *bytes_out = 0; return cont.h; @@ -275,23 +276,24 @@ posix_random_access_file::write_some_at( op->object_ref_ = detail::object_ref(this); op->start(token); - op->ex.on_work_started(); + svc_.work_started(); { std::lock_guard lock(ops_mutex_); outstanding_ops_.push_back(op); } - static_cast(op)->func_ = &raf_op::do_work; + static_cast(op)->func_ = &raf_op::do_work; + static_cast(op)->destroy_ = &raf_op::destroy_work; if (auto pec = svc_.pool().post(static_cast(op))) { // The pool is shutting down, or the system refused it a thread. // Nothing of this write went cross-thread, so it answers here // like the closed-descriptor and zero-length exits above rather // than through a completion the scheduler has to carry back. - // destroy() is the discard the op never reaching the queue - // needs: it unlinks, unwinds the work count and recycles. - op->destroy(); + // The op never reached the queue: it unlinks, unwinds the + // work count and recycles, and the coroutine resumes below. + op->discard(); *ec = pec; *bytes_out = 0; return cont.h; @@ -393,6 +395,13 @@ posix_random_access_file::retire() noexcept svc_.object_pool_.recycle(this); } +inline void +posix_random_access_file::release_work( + posix_random_access_file_service& svc) noexcept +{ + svc.work_finished(); +} + } // namespace boost::corosio::detail #endif // BOOST_COROSIO_POSIX diff --git a/include/boost/corosio/native/detail/posix/posix_resolver.hpp b/include/boost/corosio/native/detail/posix/posix_resolver.hpp index 2cfc35275..857af7a81 100644 --- a/include/boost/corosio/native/detail/posix/posix_resolver.hpp +++ b/include/boost/corosio/native/detail/posix/posix_resolver.hpp @@ -98,7 +98,7 @@ namespace boost::corosio::detail { -struct scheduler; +class scheduler; namespace posix_resolver_detail { @@ -171,6 +171,9 @@ class posix_resolver final /// Where the endpoints are handed back. std::vector* out = nullptr; + /// The service whose context counts this op as work. + posix_resolver_service* svc = nullptr; + // Input parameters (owned copies for thread safety) std::string host; std::string service; @@ -194,6 +197,9 @@ class posix_resolver final /// Where the name is handed back. endpoint_name* result_out = nullptr; + /// The service whose context counts this op as work. + posix_resolver_service* svc = nullptr; + // Input parameters endpoint ep; reverse_flags flags = reverse_flags::none; @@ -281,6 +287,12 @@ class posix_resolver final /// Execute blocking `getnameinfo()` on a pool thread. static void do_reverse_resolve_work(pool_work_item*) noexcept; + /// Destroy a forward resolution the pool shut down before running. + static void destroy_resolve_work(pool_work_item*) noexcept; + + /// Destroy a reverse resolution the pool shut down before running. + static void destroy_reverse_resolve_work(pool_work_item*) noexcept; + private: posix_resolver_service& svc_; }; diff --git a/include/boost/corosio/native/detail/posix/posix_resolver_service.hpp b/include/boost/corosio/native/detail/posix/posix_resolver_service.hpp index 3ba36da51..b658439bb 100644 --- a/include/boost/corosio/native/detail/posix/posix_resolver_service.hpp +++ b/include/boost/corosio/native/detail/posix/posix_resolver_service.hpp @@ -89,6 +89,18 @@ class BOOST_COROSIO_DECL posix_resolver_service final return sched_->scheduler_locking_disabled(); } + /// Count an operation as work on this service's context. + void work_started() noexcept + { + sched_->work_started(); + } + + /// Release a count taken by @ref work_started. + void work_finished() noexcept + { + sched_->work_finished(); + } + private: scheduler* sched_; thread_pool_ref pool_; @@ -272,18 +284,18 @@ posix_resolver::resolve_op::operator()() // Hold the keepalive across the dispatch: it may be the last // reference to the implementation this op is embedded in. auto prevent_destroy = std::move(object_ref_); - ex.on_work_finished(); + svc->work_finished(); dispatch_coro(ex, *cont).resume(); } inline void posix_resolver::resolve_op::destroy() { - stop_cb.reset(); - auto local_ex = ex; - // May destroy the implementation, and with it this op. - object_ref_.reset(); - local_ex.on_work_finished(); + // Copied first: the abandon may unwind the frame at once, destroying + // the resolver, and with it this op. + auto* s = svc; + destroy_awaiting(); + s->work_finished(); } // posix_resolver::reverse_resolve_op implementation @@ -328,18 +340,18 @@ posix_resolver::reverse_resolve_op::operator()() // Hold the keepalive across the dispatch: it may be the last // reference to the implementation this op is embedded in. auto prevent_destroy = std::move(object_ref_); - ex.on_work_finished(); + svc->work_finished(); dispatch_coro(ex, *cont).resume(); } inline void posix_resolver::reverse_resolve_op::destroy() { - stop_cb.reset(); - auto local_ex = ex; - // May destroy the implementation, and with it this op. - object_ref_.reset(); - local_ex.on_work_finished(); + // Copied first: the abandon may unwind the frame at once, destroying + // the resolver, and with it this op. + auto* s = svc; + destroy_awaiting(); + s->work_finished(); } // posix_resolver implementation @@ -372,13 +384,15 @@ posix_resolver::resolve( op.flags = flags; op.start(token); - // Keep io_context alive while resolution is pending - op.ex.on_work_started(); + // Keep this context alive while resolution is pending + op.svc = &svc_; + svc_.work_started(); // Prevent impl destruction while work is in flight resolve_pool_op_.resolver_ = this; resolve_pool_op_.ref_ = detail::object_ref(this); resolve_pool_op_.func_ = &posix_resolver::do_resolve_work; + resolve_pool_op_.destroy_ = &posix_resolver::destroy_resolve_work; if (auto pec = svc_.pool().post(&resolve_pool_op_)) { // The pool is shutting down, or the system refused it a thread. @@ -387,7 +401,7 @@ posix_resolver::resolve( // completion the scheduler has to carry back. resolve_pool_op_.ref_.reset(); op.stop_cb.reset(); - op.ex.on_work_finished(); + svc_.work_finished(); *ec = pec; return dispatch_coro(ex, cont); } @@ -420,13 +434,15 @@ posix_resolver::reverse_resolve( op.flags = flags; op.start(token); - // Keep io_context alive while resolution is pending - op.ex.on_work_started(); + // Keep this context alive while resolution is pending + op.svc = &svc_; + svc_.work_started(); // Prevent impl destruction while work is in flight reverse_pool_op_.resolver_ = this; reverse_pool_op_.ref_ = detail::object_ref(this); reverse_pool_op_.func_ = &posix_resolver::do_reverse_resolve_work; + reverse_pool_op_.destroy_ = &posix_resolver::destroy_reverse_resolve_work; if (auto pec = svc_.pool().post(&reverse_pool_op_)) { // The pool is shutting down, or the system refused it a thread. @@ -435,7 +451,7 @@ posix_resolver::reverse_resolve( // completion the scheduler has to carry back. reverse_pool_op_.ref_.reset(); op.stop_cb.reset(); - op.ex.on_work_finished(); + svc_.work_finished(); *ec = pec; return dispatch_coro(ex, cont); } @@ -541,6 +557,24 @@ posix_resolver::do_reverse_resolve_work(pool_work_item* w) noexcept self->svc_.post(&self->reverse_op_); } +inline void +posix_resolver::destroy_resolve_work(pool_work_item* w) noexcept +{ + auto* pw = static_cast(w); + auto* self = pw->resolver_; + self->op_.object_ref_ = std::move(pw->ref_); + self->op_.destroy(); +} + +inline void +posix_resolver::destroy_reverse_resolve_work(pool_work_item* w) noexcept +{ + auto* pw = static_cast(w); + auto* self = pw->resolver_; + self->reverse_op_.object_ref_ = std::move(pw->ref_); + self->reverse_op_.destroy(); +} + // posix_resolver_service implementation inline void diff --git a/include/boost/corosio/native/detail/posix/posix_signal.hpp b/include/boost/corosio/native/detail/posix/posix_signal.hpp index 7e3aae078..219c09466 100644 --- a/include/boost/corosio/native/detail/posix/posix_signal.hpp +++ b/include/boost/corosio/native/detail/posix/posix_signal.hpp @@ -56,7 +56,7 @@ struct signal_op : scheduler_op int signal_number = 0; posix_signal_service* svc = nullptr; // For work_finished callback - void operator()() override; + void operator()() noexcept override; void destroy() override; }; diff --git a/include/boost/corosio/native/detail/posix/posix_signal_service.hpp b/include/boost/corosio/native/detail/posix/posix_signal_service.hpp index a3197f5e9..7e91eed6b 100644 --- a/include/boost/corosio/native/detail/posix/posix_signal_service.hpp +++ b/include/boost/corosio/native/detail/posix/posix_signal_service.hpp @@ -21,13 +21,16 @@ #include #include #include +#include #include #include #include #include +#include #include #include +#include #include #include @@ -196,7 +199,7 @@ class BOOST_COROSIO_DECL posix_signal_service final std::error_code clear_signals(posix_signal& impl); - void cancel_wait(posix_signal& impl); + void cancel_wait(posix_signal& impl) noexcept; void start_wait(posix_signal& impl, signal_op* op); /** Cancel an in-flight wait on behalf of a stop token. @@ -408,7 +411,7 @@ drain_signal_pipe() // signal_op implementation inline void -signal_op::operator()() +signal_op::operator()() noexcept { if (ec_out) *ec_out = {}; @@ -423,6 +426,8 @@ signal_op::operator()() auto ex = d; auto k = std::move(keep); + // A throw terminates: the wait is no longer parked, so nothing else + // could resume or abandon the coroutine. ex.post(*c); // Balance the work_started() from start_wait @@ -433,9 +438,12 @@ signal_op::operator()() inline void signal_op::destroy() { - // Embedded in posix_signal: dropping the queued reference is the - // only cleanup, and nothing touches this op afterwards. - auto k = std::move(keep); + // Embedded in posix_signal: the queued reference keeps it alive + // while the awaiting coroutine, which may own the set, is destroyed. + auto* c = std::exchange(cont, nullptr); + auto k = std::move(keep); + if (c && c->h) + capy::abandon(*c); } // posix_signal implementation @@ -543,6 +551,7 @@ posix_signal_service::shutdown() // ~posix_signal destroys an armed stop_cb_ whose ~stop_callback // waits on a token_canceller that takes mutex_. std::vector doomed; + std::vector parked; { posix_signal_detail::signal_state* state = @@ -558,9 +567,16 @@ posix_signal_service::shutdown() }); for (auto* impl : doomed) { - // A parked wait is abandoned like every other op at - // teardown; a later destroy() must not resume it mid-drain. - impl->waiting_ = false; + // The parked wait's coroutine is destroyed once the locks + // drop: destroying it can destroy the set, which takes them. + if (std::exchange(impl->waiting_, false)) + { + if (auto* c = std::exchange(impl->pending_op_.cont, nullptr)) + { + if (c->h) + parked.push_back(*c); + } + } while (auto* reg = impl->signals_) { @@ -599,6 +615,9 @@ posix_signal_service::shutdown() registrations_[i] = nullptr; } + for (auto& c : parked) + capy::abandon(c); + for (auto* impl : doomed) release(impl); } @@ -826,7 +845,7 @@ posix_signal_service::clear_signals(posix_signal& impl) } inline void -posix_signal_service::cancel_wait(posix_signal& impl) +posix_signal_service::cancel_wait(posix_signal& impl) noexcept { bool was_waiting = false; signal_op* op = nullptr; @@ -839,6 +858,7 @@ posix_signal_service::cancel_wait(posix_signal& impl) was_waiting = true; impl.waiting_ = false; op = &impl.pending_op_; + sched_->cancel_posting_begin(); } } @@ -850,6 +870,7 @@ posix_signal_service::cancel_wait(posix_signal& impl) *op->signal_out = 0; op->d.post(*op->cont); sched_->work_finished(); + sched_->cancel_posting_end(); } } @@ -870,6 +891,7 @@ posix_signal_service::cancel_wait_token(posix_signal& impl) noexcept was_waiting = true; impl.waiting_ = false; op = &impl.pending_op_; + sched_->cancel_posting_begin(); } } @@ -881,6 +903,7 @@ posix_signal_service::cancel_wait_token(posix_signal& impl) noexcept *op->signal_out = 0; op->d.post(*op->cont); sched_->work_finished(); + sched_->cancel_posting_end(); } } diff --git a/include/boost/corosio/native/detail/posix/posix_stream_file.hpp b/include/boost/corosio/native/detail/posix/posix_stream_file.hpp index c5b3f4253..7f8e1c60b 100644 --- a/include/boost/corosio/native/detail/posix/posix_stream_file.hpp +++ b/include/boost/corosio/native/detail/posix/posix_stream_file.hpp @@ -80,7 +80,7 @@ namespace boost::corosio::detail { -struct scheduler; +class scheduler; class posix_stream_file_service; /** Stream file implementation for POSIX backends. @@ -118,6 +118,10 @@ class posix_stream_file final int errn = 0; std::size_t bytes_transferred = 0; + // The op is work on this service's context, which outlives the + // file the op is embedded in. + posix_stream_file_service* svc = nullptr; + file_op() = default; void reset() noexcept @@ -137,6 +141,10 @@ class posix_stream_file final void destroy() override; }; + /// Release the work an op holds on its service's context. + static void + release_work(posix_stream_file_service& svc) noexcept; + /** Pool work item for thread pool dispatch. */ struct pool_op : pool_work_item { @@ -238,6 +246,8 @@ class posix_stream_file final static void do_read_work(pool_work_item*) noexcept; static void do_write_work(pool_work_item*) noexcept; + static void destroy_read_work(pool_work_item*) noexcept; + static void destroy_write_work(pool_work_item*) noexcept; }; // --------------------------------------------------------------------------- @@ -436,17 +446,18 @@ posix_stream_file::file_op::operator()() // dispatch — this may be the last reference keeping the parent // posix_stream_file (which embeds this file_op) alive. auto prevent_destroy = std::move(object_ref_); - ex.on_work_finished(); + release_work(*svc); dispatch_coro(ex, *cont).resume(); } inline void posix_stream_file::file_op::destroy() { - stop_cb.reset(); - auto local_ex = ex; - object_ref_.reset(); - local_ex.on_work_finished(); + // Copied first: the abandon may unwind the frame at once, destroying + // the file, and with it this op. + auto* s = svc; + destroy_awaiting(); + release_work(*s); } } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/posix/posix_stream_file_service.hpp b/include/boost/corosio/native/detail/posix/posix_stream_file_service.hpp index 03b038546..e043440a2 100644 --- a/include/boost/corosio/native/detail/posix/posix_stream_file_service.hpp +++ b/include/boost/corosio/native/detail/posix/posix_stream_file_service.hpp @@ -201,12 +201,14 @@ posix_stream_file::read_some( op.fd = fd_; op.offset = offset_; op.generation = generation_; + op.svc = &svc_; - op.ex.on_work_started(); + svc_.work_started(); - read_pool_op_.file_ = this; - read_pool_op_.ref_ = detail::object_ref(this); - read_pool_op_.func_ = &posix_stream_file::do_read_work; + read_pool_op_.file_ = this; + read_pool_op_.ref_ = detail::object_ref(this); + read_pool_op_.func_ = &posix_stream_file::do_read_work; + read_pool_op_.destroy_ = &posix_stream_file::destroy_read_work; if (auto pec = svc_.pool().post(&read_pool_op_)) { // The pool is shutting down, or the system refused it a thread. @@ -215,7 +217,7 @@ posix_stream_file::read_some( // than through a completion the scheduler has to carry back. read_pool_op_.ref_.reset(); op.stop_cb.reset(); - op.ex.on_work_finished(); + svc_.work_finished(); *ec = pec; *bytes_out = 0; return dispatch_coro(ex, cont); @@ -324,12 +326,14 @@ posix_stream_file::write_some( op.fd = fd_; op.offset = offset_; op.generation = generation_; + op.svc = &svc_; - op.ex.on_work_started(); + svc_.work_started(); - write_pool_op_.file_ = this; - write_pool_op_.ref_ = detail::object_ref(this); - write_pool_op_.func_ = &posix_stream_file::do_write_work; + write_pool_op_.file_ = this; + write_pool_op_.ref_ = detail::object_ref(this); + write_pool_op_.func_ = &posix_stream_file::do_write_work; + write_pool_op_.destroy_ = &posix_stream_file::destroy_write_work; if (auto pec = svc_.pool().post(&write_pool_op_)) { // The pool is shutting down, or the system refused it a thread. @@ -338,7 +342,7 @@ posix_stream_file::write_some( // than through a completion the scheduler has to carry back. write_pool_op_.ref_.reset(); op.stop_cb.reset(); - op.ex.on_work_finished(); + svc_.work_finished(); *ec = pec; *bytes_out = 0; return dispatch_coro(ex, cont); @@ -401,6 +405,24 @@ posix_stream_file::do_write_work(pool_work_item* w) noexcept self->svc_.post(&op); } +inline void +posix_stream_file::destroy_read_work(pool_work_item* w) noexcept +{ + auto* pw = static_cast(w); + auto* self = pw->file_; + self->read_op_.object_ref_ = std::move(pw->ref_); + self->read_op_.destroy(); +} + +inline void +posix_stream_file::destroy_write_work(pool_work_item* w) noexcept +{ + auto* pw = static_cast(w); + auto* self = pw->file_; + self->write_op_.object_ref_ = std::move(pw->ref_); + self->write_op_.destroy(); +} + inline std::error_code posix_stream_file::assign(native_handle_type handle) noexcept { @@ -422,6 +444,12 @@ posix_stream_file::retire() noexcept svc_.object_pool_.recycle(this); } +inline void +posix_stream_file::release_work(posix_stream_file_service& svc) noexcept +{ + svc.work_finished(); +} + } // namespace boost::corosio::detail #endif // BOOST_COROSIO_POSIX diff --git a/include/boost/corosio/native/detail/reactor/reactor_acceptor.hpp b/include/boost/corosio/native/detail/reactor/reactor_acceptor.hpp index 6b6c51037..a37b88536 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_acceptor.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_acceptor.hpp @@ -542,20 +542,14 @@ reactor_acceptor< else { *desc_slot_ptr = &op; + // Select watches an fd only while an op is parked; see + // park_op. + if constexpr (Service::needs_park_notification) + svc_.scheduler().notify_reactor(); } } if (post_now) - { - svc_.post(&op); - svc_.work_finished(); - } - else - { - // Select watches an fd only while an op is parked; see - // register_op. - if constexpr (Service::needs_park_notification) - svc_.scheduler().notify_reactor(); - } + svc_.scheduler().post_counted(&op); return std::noop_coroutine(); } diff --git a/include/boost/corosio/native/detail/reactor/reactor_acceptor_service.hpp b/include/boost/corosio/native/detail/reactor/reactor_acceptor_service.hpp index 7bd11e1c4..63d152b40 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_acceptor_service.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_acceptor_service.hpp @@ -108,11 +108,6 @@ class reactor_acceptor_service : public ServiceBase state_->sched_.work_started(); } - void work_finished() noexcept - { - state_->sched_.work_finished(); - } - StreamService* stream_service() const noexcept { return stream_svc_; diff --git a/include/boost/corosio/native/detail/reactor/reactor_backend.hpp b/include/boost/corosio/native/detail/reactor/reactor_backend.hpp index 51e16e271..f14eb238d 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_backend.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_backend.hpp @@ -155,21 +155,20 @@ reactor_acceptor_impl< } if (io_done || op.cancelled.load(std::memory_order_acquire)) + { post_now = true; + } else + { this->desc_state_.read_op = &op; + // Select watches reads only for parked ops; see + // park_op. + if constexpr (Traits::needs_park_notification) + this->svc_.scheduler().notify_reactor(); + } } if (post_now) - { - this->svc_.post(&op); - this->svc_.work_finished(); - } - else - { - // Select watches reads only for parked ops; see register_op. - if constexpr (Traits::needs_park_notification) - this->svc_.scheduler().notify_reactor(); - } + this->svc_.scheduler().post_counted(&op); return std::noop_coroutine(); } diff --git a/include/boost/corosio/native/detail/reactor/reactor_descriptor_service.hpp b/include/boost/corosio/native/detail/reactor/reactor_descriptor_service.hpp index 437eedb38..a053eb48d 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_descriptor_service.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_descriptor_service.hpp @@ -121,11 +121,6 @@ class reactor_descriptor_service : public descriptor_service state_->sched_.work_started(); } - void work_finished() noexcept - { - state_->sched_.work_finished(); - } - protected: std::unique_ptr state_; diff --git a/include/boost/corosio/native/detail/reactor/reactor_io_core.hpp b/include/boost/corosio/native/detail/reactor/reactor_io_core.hpp index d273b58a3..20ffb5355 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_io_core.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_io_core.hpp @@ -80,7 +80,10 @@ class reactor_io_core { std::lock_guard lock(desc_state_.mutex); if (*desc_op_ptr == &op) + { claimed = std::exchange(*desc_op_ptr, nullptr); + svc_.scheduler().cancel_posting_begin(); + } // Not in the slot: request_cancel() above already set // op.cancelled, which register_op consults before parking // and the completion decode consults on delivery. Latching @@ -90,8 +93,8 @@ class reactor_io_core if (claimed) { op.object_ref_ = detail::object_ref(self_ptr()); - svc_.post(&op); - svc_.work_finished(); + svc_.scheduler().post_counted(&op); + svc_.scheduler().cancel_posting_end(); } } @@ -135,17 +138,10 @@ class reactor_io_core { svc_.work_started(); if (park_op(op, desc_slot, ready_flag, is_write_direction)) - { - // Select rebuilds its fd_sets from parked ops only, so - // parking must wake it. Compiled away for epoll and kqueue. - if constexpr (Service::needs_park_notification) - svc_.scheduler().notify_reactor(); return; - } // Posted after the descriptor lock is released: the scheduler // takes its own lock before descriptor locks. - svc_.post(&op); - svc_.work_finished(); + svc_.scheduler().post_counted(&op); } /// Cancel every pending operation. @@ -319,6 +315,12 @@ class reactor_io_core } desc_slot = &op; + // Select rebuilds its fd_sets from parked ops only, so parking + // must wake it, under the lock: once released, the op can + // complete and its context be destroyed. Compiled away for + // epoll and kqueue. + if constexpr (Service::needs_park_notification) + svc_.scheduler().notify_reactor(); return true; } @@ -327,8 +329,7 @@ class reactor_io_core for (int i = 0; i < count; ++i) { claimed[i]->object_ref_ = detail::object_ref(self_ptr()); - svc_.post(claimed[i]); - svc_.work_finished(); + svc_.scheduler().post_counted(claimed[i]); } } }; diff --git a/include/boost/corosio/native/detail/reactor/reactor_op.hpp b/include/boost/corosio/native/detail/reactor/reactor_op.hpp index 3eecc534f..c03430739 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_op.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_op.hpp @@ -96,7 +96,6 @@ struct reactor_op : reactor_op_base /// Destroy without invoking. void destroy() override { - stop_cb.reset(); reactor_op_base::destroy(); } diff --git a/include/boost/corosio/native/detail/reactor/reactor_op_base.hpp b/include/boost/corosio/native/detail/reactor/reactor_op_base.hpp index 516a75108..6078f24a0 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_op_base.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_op_base.hpp @@ -49,11 +49,11 @@ struct reactor_op_base : coro_op /// Perform the I/O syscall (implemented by concrete op types). virtual void perform_io() noexcept = 0; - /// Destroy without invoking — drop the keepalive held in - /// coro_op's `object_ref_`. + /// Destroy without invoking: the drained op abandons the coroutine + /// it holds. void destroy() override { - object_ref_.reset(); + destroy_awaiting(); } }; diff --git a/include/boost/corosio/native/detail/reactor/reactor_scheduler.hpp b/include/boost/corosio/native/detail/reactor/reactor_scheduler.hpp index 83aec7192..148b28c02 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_scheduler.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_scheduler.hpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -228,6 +229,19 @@ class reactor_scheduler void compensating_work_started() const noexcept; + /** Post an operation whose work is already counted. + + The posting thread touches nothing of the scheduler after this + returns, unlike a @ref post followed by @ref work_finished: once + posted, the operation can complete and let its context be + destroyed. + + @pre work_started() was called for @p op. + + @param op The operation to post. + */ + void post_counted(scheduler_op* op) const; + /** Post completed operations for deferred invocation. If called from a thread running this scheduler, operations @@ -288,6 +302,9 @@ class reactor_scheduler reactor_scheduler() = default; + /// Destroy, destroying whatever was posted after the drain. + ~reactor_scheduler() override; + /** Drain completed_ops during shutdown. Pops all operations from the global queue and destroys them, @@ -297,6 +314,9 @@ class reactor_scheduler */ void shutdown_drain(); + /// Destroy every queued operation without running it. + void destroy_queued(lock_type& lock) noexcept; + /// RAII guard that re-inserts the task sentinel after `run_task`. struct task_cleanup { @@ -517,6 +537,8 @@ reactor_scheduler::post(std::coroutine_handle<> h) const void destroy() override { + // A bare handle carries no owner to unwind from: it is + // destroyed directly. auto saved = h_; delete this; saved.destroy(); @@ -716,6 +738,20 @@ reactor_scheduler::compensating_work_started() const noexcept ++ctx->private_outstanding_work; } +inline void +reactor_scheduler::post_counted(scheduler_op* op) const +{ + if (auto* ctx = reactor_find_context(this)) + { + ctx->private_queue.push(op); + return; + } + + lock_type lock(mutex_); + completed_ops_.push(op); + wake_one_thread_and_unlock(lock); +} + inline void reactor_scheduler::post_deferred_completions(ready_queue& ops) const { @@ -796,16 +832,31 @@ reactor_scheduler::shutdown_drain() if (timer_svc_) timer_svc_->shutdown(); + wait_cancels_posting(); + lock_type lock(mutex_); settle_retired(lock); + destroy_queued(lock); + signal_all(lock); +} + +inline reactor_scheduler::~reactor_scheduler() +{ + // Service destructors run after the drain and may still post. + lock_type lock(mutex_); + destroy_queued(lock); +} +inline void +reactor_scheduler::destroy_queued(lock_type& lock) noexcept +{ while (auto e = completed_ops_.pop()) { if (ready_is_continuation(e)) { lock.unlock(); - if (auto h = ready_as_cont(e)->h) - h.destroy(); + if (auto* c = ready_as_cont(e); c->h) + capy::abandon(*c); lock.lock(); } else @@ -818,8 +869,6 @@ reactor_scheduler::shutdown_drain() lock.lock(); } } - - signal_all(lock); } inline void @@ -835,8 +884,11 @@ reactor_scheduler::maybe_unlock_and_signal_one(lock_type& lock) const state_ |= signaled_bit; if (state_ > signaled_bit) { - lock.unlock(); + // Signalled before unlocking: a post from another context's + // thread may be the last touch of this scheduler before the + // posted work runs, finishes and lets it be destroyed. cond_.notify_one(); + lock.unlock(); return true; } return false; @@ -890,8 +942,9 @@ reactor_scheduler::wake_one_thread_and_unlock(lock_type& lock) const if (task_running_.load(std::memory_order_relaxed) && !task_interrupted_) { task_interrupted_ = true; - lock.unlock(); + // Under the lock, for the same reason as the signal above. interrupt_reactor(); + lock.unlock(); } else { diff --git a/include/boost/corosio/native/detail/reactor/reactor_socket_service.hpp b/include/boost/corosio/native/detail/reactor/reactor_socket_service.hpp index fa0790b21..9532790dd 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_socket_service.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_socket_service.hpp @@ -108,11 +108,6 @@ class reactor_socket_service : public ServiceBase state_->sched_.work_started(); } - void work_finished() noexcept - { - state_->sched_.work_finished(); - } - protected: // Override in derived to add pre-close logic. No backend currently needs // it; the hooks exist so a trait can run fd-level teardown before close. diff --git a/include/boost/corosio/native/detail/uring/uring_acceptor_ops.hpp b/include/boost/corosio/native/detail/uring/uring_acceptor_ops.hpp index 0bd88ec12..36b63239c 100644 --- a/include/boost/corosio/native/detail/uring/uring_acceptor_ops.hpp +++ b/include/boost/corosio/native/detail/uring/uring_acceptor_ops.hpp @@ -193,6 +193,10 @@ struct uring_accept_op : uring_op // A delivered connection that nobody will adopt. if (self->accepted_fd >= 0) ::close(self->accepted_fd); + // Outlives dispose(), which may hand the node back to an + // acceptor that the unwound frame would otherwise release. + auto keep = std::move(self->object_ref_); + self->destroy_awaiting(); self->dispose(self); return; } diff --git a/include/boost/corosio/native/detail/uring/uring_file_ops.hpp b/include/boost/corosio/native/detail/uring/uring_file_ops.hpp index 25fcc944a..3add8bb97 100644 --- a/include/boost/corosio/native/detail/uring/uring_file_ops.hpp +++ b/include/boost/corosio/native/detail/uring/uring_file_ops.hpp @@ -19,8 +19,10 @@ #include #include #include +#include #include +#include #include namespace boost::corosio::detail { @@ -166,7 +168,11 @@ struct uring_random_access_read_op : uring_file_read_op_base if (owner == nullptr) { + auto keep = std::move(self->object_ref_); + auto* c = std::exchange(self->cont, nullptr); delete self; + if (c && c->h) + capy::abandon(*c); return; } @@ -306,7 +312,11 @@ struct uring_random_access_write_op : uring_file_write_op_base if (owner == nullptr) { + auto keep = std::move(self->object_ref_); + auto* c = std::exchange(self->cont, nullptr); delete self; + if (c && c->h) + capy::abandon(*c); return; } diff --git a/include/boost/corosio/native/detail/uring/uring_multishot_acceptor.hpp b/include/boost/corosio/native/detail/uring/uring_multishot_acceptor.hpp index 4143ca277..24db98d7b 100644 --- a/include/boost/corosio/native/detail/uring/uring_multishot_acceptor.hpp +++ b/include/boost/corosio/native/detail/uring/uring_multishot_acceptor.hpp @@ -873,6 +873,7 @@ class uring_multishot_acceptor_base { waiters_.remove(w); } + sched_->cancel_posting_begin(); } // `cancelled` is already true (the claim that routed here); // that is also the completion status, so post as-is. The node's @@ -882,6 +883,7 @@ class uring_multishot_acceptor_base // wait-or-self case. sched_->post(w); sched_->work_finished(); // balance the work_started() from accept() + sched_->cancel_posting_end(); } private: diff --git a/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp b/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp index f52e21c57..fb9b8e5ae 100644 --- a/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp +++ b/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -32,6 +33,7 @@ #include #include #include +#include #include #include @@ -106,7 +108,11 @@ class BOOST_COROSIO_DECL uring_random_access_file final auto keep = std::move(self->object_ref_); if (owner == nullptr) { + auto* c = std::exchange(self->cont, nullptr); self->file_->release_op(self); + // The frame may own the file; the keepalive outlives it. + if (c && c->h) + capy::abandon(*c); return; } Base::finish(self); diff --git a/include/boost/corosio/native/detail/uring/uring_scheduler.hpp b/include/boost/corosio/native/detail/uring/uring_scheduler.hpp index 926536723..9788b8af9 100644 --- a/include/boost/corosio/native/detail/uring/uring_scheduler.hpp +++ b/include/boost/corosio/native/detail/uring/uring_scheduler.hpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include @@ -213,9 +214,8 @@ class BOOST_COROSIO_DECL uring_scheduler final } /// Increment the io_uring in-flight counter. Callers prep an SQE - /// whose CQE will require IORING_ENTER_GETEVENTS to surface under - /// DEFER_TASKRUN. Excluded: the wakeup-eventfd multishot SQE, whose - /// progress doesn't depend on userspace getevents. + /// whose CQE must be reaped. Excluded: the wakeup-eventfd poll, + /// which stays armed for the ring's lifetime. void inflight_inc() const noexcept { uring_inflight_.fetch_add(1, std::memory_order_release); @@ -265,6 +265,14 @@ class BOOST_COROSIO_DECL uring_scheduler final /// been signalled and not yet acked. void interrupt_reactor() const noexcept; + /** Wake a thread to run newly queued work. + + Called under `dispatch_mutex_`: the caller may be on another + context's thread, and once the lock is released the work can + run, finish and let this scheduler be destroyed. + */ + void wake_one_locked() const noexcept; + /** Submit `IORING_OP_ASYNC_CANCEL` targeting an in-flight op by its user_data pointer. @@ -307,9 +315,13 @@ class BOOST_COROSIO_DECL uring_scheduler final void cancel_ops(Cancel&& cancel) noexcept { lazy_init_ring(); + { + lock_type lock(ring_mutex_); + cancel(); + } + // After queuing: a leader woken first could re-enter its wait + // before the cancel SQE exists, leaving it unsubmitted. interrupt_reactor(); - lock_type lock(ring_mutex_); - cancel(); } /** Cancel an object's operations and flush every SQE naming `fd`. @@ -360,13 +372,16 @@ class BOOST_COROSIO_DECL uring_scheduler final void push_completed_locked(scheduler_op* op) const noexcept { completed_ops_.push(op); + // A run thread finds the op on its next pass; any other caller + // must wake one, or it waits for the nearest timer. + if (!running_in_this_thread()) + wake_one_locked(); } void configure_threading(threading_config cfg) noexcept override { scheduler_locking_disabled_ = !cfg.scheduler_locking; - // reactor_io_locking off also drives SINGLE_ISSUER/DEFER_TASKRUN and - // the eventfd wake elision (see lazy_init_ring_unlocked and + // reactor_io_locking off also drives the eventfd wake elision (see // interrupt_reactor). one_thread is unused: the leader-follower wake // model is not gated on it. reactor_io_locking_ = cfg.reactor_io_locking; @@ -381,10 +396,6 @@ class BOOST_COROSIO_DECL uring_scheduler final cached and read when it is. No-op if `enable` is false (the default). - @note When combined with single-threaded mode, - IORING_SETUP_DEFER_TASKRUN is suppressed — the kernel - rejects that combination. SINGLE_ISSUER still applies. - @param enable Set IORING_SETUP_SQPOLL on ring init. @param idle_ms sq_thread_idle in milliseconds; 0 = kernel default (1ms). @@ -405,6 +416,9 @@ class BOOST_COROSIO_DECL uring_scheduler final } private: + /// Destroy every queued operation without running it. + void destroy_queued(lock_type& lock) noexcept; + // ring_ + wakeup_eventfd_ are mutable so lazy_init_ring() (called // from const contexts like post()) can populate them on first use. mutable struct ::io_uring ring_{}; @@ -434,10 +448,8 @@ class BOOST_COROSIO_DECL uring_scheduler final // would cause false sharing and unnecessary cache-line ping-pong. // Hold each on its own line. alignas(64) mutable std::atomic outstanding_work_{0}; - // Count of io_uring SQEs in flight whose completion requires user- - // space to enter the kernel via IORING_ENTER_GETEVENTS for task - // work to progress under IORING_SETUP_DEFER_TASKRUN. Excludes the - // wakeup-eventfd multishot poll (registered in lazy_init_ring), and + // Count of io_uring SQEs in flight whose completion must be reaped. + // Excludes the wakeup-eventfd poll (registered in lazy_init_ring), and // is updated by uring_submit_op and by process_completions on // each non-F_MORE, non-eventfd CQE. Used by do_one to skip the // ring pump when there is no io_uring work pending. @@ -571,6 +583,11 @@ inline uring_scheduler::uring_scheduler( inline uring_scheduler::~uring_scheduler() { + // Service destructors run after the drain and may still post. + { + lock_type lock(dispatch_mutex_); + destroy_queued(lock); + } if (ring_inited_) { // Ring teardown can still run a doomed pipe write as task @@ -591,50 +608,17 @@ uring_scheduler::lazy_init_ring() const inline void uring_scheduler::lazy_init_ring_unlocked() const { + // No flags bind the ring to one thread, even in the lockless tiers, + // so a context may be made, run and destroyed on different threads, + // and its teardown can always drain the ring. io_uring_params params{}; - // The unsafe_io and unsafe tiers guarantee a single ring submitter. - if (!reactor_io_locking_) - { - // SINGLE_ISSUER promises the kernel one submitter thread, - // letting it skip internal SQ locking. DEFER_TASKRUN tells - // it to batch task_work delivery at io_uring_enter(GETEVENTS) - // boundaries instead of interrupting the run thread via - // TWA_SIGNAL — eliminates cache pollution from mid-flight - // task_work and gives a meaningful single-threaded - // throughput uplift. - // - // Plan 3 disabled DEFER_TASKRUN defensively over a misread - // of the GETEVENTS contract. Plan 4a re-enabled it: liburing's - // io_uring_submit_and_wait_timeout always sets - // IORING_ENTER_GETEVENTS when wait_nr > 0, regardless of - // ts. Our run loop's only kernel-wait call passes wait_nr=1. - // Submit-only paths (flush_after_cancel, etc.) leave their - // CQEs queued until the leader's next GETEVENTS-bearing - // wait — benign. - // - // Multi-thread mode never sets these flags: SINGLE_ISSUER - // would be unsafe with multiple submitter threads. - // - // DEFER_TASKRUN is suppressed when SQPOLL is also enabled - // — the kernel rejects that combination with -EINVAL. The - // SQPOLL polling thread already delivers completions - // without TWA_SIGNAL interruption, so DEFER_TASKRUN's - // benefit is moot in that mode. - params.flags = IORING_SETUP_SINGLE_ISSUER; - if (!enable_sqpoll_) - params.flags |= IORING_SETUP_DEFER_TASKRUN; - } if (enable_sqpoll_) { // SQPOLL forks a kernel thread that busy-polls the SQ ring; - // submission becomes a userspace-only memory store. Combines - // with SINGLE_ISSUER (the kernel accepts that pair) but NOT - // with DEFER_TASKRUN (kernel returns -EINVAL); the - // reactor-I/O-lockless branch above suppresses DEFER_TASKRUN - // when SQPOLL is also set. Idle timeout 0 means kernel - // default (1ms); we only forward when explicitly set so - // the kernel default is preserved. + // submission becomes a userspace-only memory store. Idle + // timeout 0 means kernel default (1ms); we only forward when + // explicitly set so the kernel default is preserved. params.flags |= IORING_SETUP_SQPOLL; if (sq_thread_idle_ms_ != 0) params.sq_thread_idle = sq_thread_idle_ms_; @@ -753,18 +737,27 @@ uring_scheduler::shutdown() has_deferred_cancels_.store(false, std::memory_order_release); } + wait_cancels_posting(); + // Drain posted ops, including the completions reaped above, // calling destroy() on each so embedded handles (coroutine frames, // error_code outputs) get torn down rather than leaked. Mirrors // reactor_scheduler::shutdown_drain. lock_type lock(dispatch_mutex_); + destroy_queued(lock); + cond_.notify_all(); +} + +inline void +uring_scheduler::destroy_queued(lock_type& lock) noexcept +{ while (auto e = completed_ops_.pop()) { if (ready_is_continuation(e)) { lock.unlock(); - if (auto h = ready_as_cont(e)->h) - h.destroy(); + if (auto* c = ready_as_cont(e); c->h) + capy::abandon(*c); lock.lock(); } else @@ -774,17 +767,17 @@ uring_scheduler::shutdown() lock.lock(); } } - cond_.notify_all(); } inline void uring_scheduler::stop() { + // All under the lock: releasing the last work on another thread + // gets here, and a run thread that sees stopped_ may return and + // let this scheduler be destroyed. + lock_type lock(dispatch_mutex_); stopped_.store(true, std::memory_order_release); - { - lock_type lock(dispatch_mutex_); - cond_.notify_all(); - } + cond_.notify_all(); // Force-wake unconditionally — bypass interrupt_reactor's CAS // coalescing. A dropped wake here leaves the leader blocked // forever in submit_and_wait_timeout (no further CQE will @@ -929,6 +922,8 @@ uring_scheduler::post(std::coroutine_handle<> h) const void destroy() override { + // A bare handle carries no owner to unwind from: it is + // destroyed directly. auto saved = h_; delete this; if (saved) @@ -939,16 +934,11 @@ uring_scheduler::post(std::coroutine_handle<> h) const auto* op = new post_handler(h); lazy_init_ring(); outstanding_work_.fetch_add(1, std::memory_order_relaxed); - bool wake_leader; { lock_type lock(dispatch_mutex_); completed_ops_.push(op); - wake_leader = task_running_; - if (!wake_leader) - cond_.notify_one(); + wake_one_locked(); } - if (wake_leader) - interrupt_reactor(); } inline void @@ -956,16 +946,11 @@ uring_scheduler::post(scheduler_op* op) const { lazy_init_ring(); outstanding_work_.fetch_add(1, std::memory_order_relaxed); - bool wake_leader; { lock_type lock(dispatch_mutex_); completed_ops_.push(op); - wake_leader = task_running_; - if (!wake_leader) - cond_.notify_one(); + wake_one_locked(); } - if (wake_leader) - interrupt_reactor(); } inline void @@ -973,15 +958,19 @@ uring_scheduler::post(capy::continuation& c) const { lazy_init_ring(); outstanding_work_.fetch_add(1, std::memory_order_relaxed); - bool wake_leader; { lock_type lock(dispatch_mutex_); completed_ops_.push(c); - wake_leader = task_running_; - if (!wake_leader) - cond_.notify_one(); + wake_one_locked(); } - if (wake_leader) +} + +inline void +uring_scheduler::wake_one_locked() const noexcept +{ + if (!task_running_) + cond_.notify_one(); + else interrupt_reactor(); } @@ -1170,22 +1159,28 @@ uring_scheduler::do_one(long timeout_us) // threads either dispatch ready ops from completed_ops_ or wait // on cond_ until the leader returns from the kernel. if (stopped_.load(std::memory_order_acquire)) + { + // stop() sets the flag under the lock and still writes the + // eventfd; returning before it unlocks lets the caller destroy + // this scheduler under it. + lock_type lock(dispatch_mutex_); return 0; + } // submit_sqes_op only pumps the ring once per SQE batch. If the user // keeps a non-empty completed_ops_ (e.g. timer with 0ns expiry as a // yield primitive), the leader-phase kernel pass below never runs // and CQEs accumulate in the ring forever — sub_request's read CQE - // never gets drained and the bench spins. submit_and_get_events - // (not plain submit) is required because IORING_SETUP_DEFER_TASKRUN - // gates task work on IORING_ENTER_GETEVENTS. + // never gets drained and the bench spins. submit_and_get_events, + // not plain submit, so completions the kernel holds pending reach + // the CQ ring before it is drained. // // Gate the kernel pump on there being io_uring-specific work. The // check is performed under ring_mutex_ so a concurrent cross-thread // submitter cannot prep an SQE that we then race past — both this // path and uring_submit_op acquire ring_mutex_ before touching // the ring. When all three sources are empty (no io_uring ops in - // flight needing DEFER_TASKRUN GETEVENTS, no userspace-pending + // flight, no userspace-pending // SQEs, no kernel-ready CQEs) a kernel entry would have no work — // saves ~8 pp of cycles on the no-I/O microbenchmark // (io_context:single_threaded). We deliberately do NOT include @@ -1348,8 +1343,7 @@ uring_scheduler::do_one(long timeout_us) } // Phase 2 — wait for at least one CQE without holding the - // mutex. Multi-thread `io_uring_enter` is permitted without - // SINGLE_ISSUER. wait_cqe_timeout only peeks the CQ ring; + // mutex. wait_cqe_timeout only peeks the CQ ring; // head advancement happens under the mutex in // process_completions below. ::io_uring_cqe* cqe = nullptr; @@ -1431,8 +1425,8 @@ uring_scheduler::process_completions() // Signal self-pipe readiness. Re-arm the multishot poll if it // terminated (F_MORE cleared), then enqueue signal_drain_op_ to // drain + deliver in dispatch context. Not counted in - // uring_inflight_ (like the wakeup eventfd poll): its progress - // does not gate DEFER_TASKRUN GETEVENTS. + // uring_inflight_ (like the wakeup eventfd poll): it stays + // armed for the ring's lifetime. if ((cqe->flags & IORING_CQE_F_MORE) == 0) std::ignore = prep_multishot_poll( signal_pipe_read_fd_, &signal_pipe_sentinel_); @@ -1503,12 +1497,12 @@ inline void uring_scheduler::submit_cancel_by_user_data(uring_op* target) noexcept { lazy_init_ring(); - // Wake the leader (if any) so its submit_and_wait_timeout returns - // and releases ring_mutex_; otherwise we'd block here until the - // next CQE arrives organically. + { + lock_type lock(ring_mutex_); + cancel_op_locked(target); + } + // After queuing, so the woken leader's submit carries the cancel. interrupt_reactor(); - lock_type lock(ring_mutex_); - cancel_op_locked(target); } inline bool diff --git a/include/boost/corosio/native/detail/uring/uring_socket_ops.hpp b/include/boost/corosio/native/detail/uring/uring_socket_ops.hpp index a566eed29..e4e1070ab 100644 --- a/include/boost/corosio/native/detail/uring/uring_socket_ops.hpp +++ b/include/boost/corosio/native/detail/uring/uring_socket_ops.hpp @@ -471,7 +471,6 @@ uring_submit_op(uring_scheduler& sched, uring_op* op) noexcept { sched.lazy_init_ring(); - bool need_post = false; { typename uring_scheduler::lock_type ring_lock(sched.ring_mutex()); @@ -493,13 +492,12 @@ uring_submit_op(uring_scheduler& sched, uring_op* op) noexcept return; } - need_post = uring_link_sqe(sched, op, sqe); - } - - if (need_post) - { - // Flush is deferred to submit_sqes_op; post() owns the wake. - sched.post(&sched.submit_op_ref()); + // Posted under the ring lock: once it is released, the scheduler + // can complete this op and, if it belongs to another context, + // finish and be destroyed. Flush is deferred to submit_sqes_op; + // post() owns the wake. + if (uring_link_sqe(sched, op, sqe)) + sched.post(&sched.submit_op_ref()); } } @@ -541,7 +539,6 @@ uring_try_submit_op_if( { sched.lazy_init_ring(); - bool need_post = false; { typename uring_scheduler::lock_type ring_lock(sched.ring_mutex()); if (!admit()) @@ -553,11 +550,10 @@ uring_try_submit_op_if( op->res = -EAGAIN; return uring_guarded_submit::sq_full; } - need_post = uring_link_sqe(sched, op, sqe); + // Posted under the ring lock, as in uring_submit_op. + if (uring_link_sqe(sched, op, sqe)) + sched.post(&sched.submit_op_ref()); } - - if (need_post) - sched.post(&sched.submit_op_ref()); return uring_guarded_submit::submitted; } diff --git a/include/boost/corosio/native/native_io_context.hpp b/include/boost/corosio/native/native_io_context.hpp index 46fc06ced..7c5fae7e0 100644 --- a/include/boost/corosio/native/native_io_context.hpp +++ b/include/boost/corosio/native/native_io_context.hpp @@ -132,6 +132,7 @@ class native_io_context : public io_context */ std::size_t run() { + frame_allocator_restore restore; return sched().run(); } @@ -141,6 +142,7 @@ class native_io_context : public io_context */ std::size_t run_one() { + frame_allocator_restore restore; return sched().run_one(); } @@ -195,6 +197,7 @@ class native_io_context : public io_context std::size_t run_one_until(std::chrono::time_point const& abs_time) { + frame_allocator_restore restore; typename Clock::time_point now = Clock::now(); for (;;) { @@ -226,6 +229,7 @@ class native_io_context : public io_context */ std::size_t poll() { + frame_allocator_restore restore; return sched().poll(); } @@ -235,6 +239,7 @@ class native_io_context : public io_context */ std::size_t poll_one() { + frame_allocator_restore restore; return sched().poll_one(); } }; diff --git a/include/boost/corosio/tcp_server.hpp b/include/boost/corosio/tcp_server.hpp index 10219dcfe..63ad220f3 100644 --- a/include/boost/corosio/tcp_server.hpp +++ b/include/boost/corosio/tcp_server.hpp @@ -22,11 +22,15 @@ #include #include #include +#include +#include +#include #include #include #include #include #include +#include #include #include @@ -299,6 +303,9 @@ class BOOST_COROSIO_DECL tcp_server struct promise_type : capy::frame_alloc_mixin { Ex ex; // Executor stored directly in frame (outlives child tasks) + // Owns the connection's chain: abandoning it resumes, or has + // the executor destroy, this frame to unwind the task. + capy::chain_owner owner_; capy::io_env env_; /// Embedded post node: the start posts this instead of the /// bare handle, which would heap-allocate a wrapper op. @@ -311,7 +318,7 @@ class BOOST_COROSIO_DECL tcp_server : ex(std::move(e)) , env_{ capy::executor_ref(ex), std::move(s), - capy::get_current_frame_allocator()} + capy::get_current_frame_allocator(), &owner_} { } @@ -323,7 +330,7 @@ class BOOST_COROSIO_DECL tcp_server : ex(std::move(e)) , env_{ capy::executor_ref(ex), std::move(s), - capy::get_current_frame_allocator()} + capy::get_current_frame_allocator(), &owner_} { } @@ -368,9 +375,13 @@ class BOOST_COROSIO_DECL tcp_server return aw.await_resume(); } + // Where the returned handle would be stored in a frame + // another thread may already have resumed and freed, + // it is resumed here instead, as a task does. auto await_suspend(std::coroutine_handle h) { - return aw.await_suspend(h, env); + return capy::detail::symmetric_transfer( + capy::detail::call_await_suspend(&aw, h, env)); } }; return adapter{std::forward(a), &env_}; @@ -386,6 +397,8 @@ class BOOST_COROSIO_DECL tcp_server ~launch_wrapper() { + // An unstarted root owns no chain yet: destroying it is + // the whole teardown. if (h) h.destroy(); } @@ -400,31 +413,88 @@ class BOOST_COROSIO_DECL tcp_server launch_wrapper& operator=(launch_wrapper&&) = delete; }; + /// One hold on a worker, dropped when this is destroyed. + class worker_hold + { + state_ref st_; + worker_base* w_; + + public: + worker_hold(state& st, worker_base& w) noexcept : st_(&st), w_(&w) {} + + worker_hold(worker_hold&& o) noexcept + : st_(std::move(o.st_)) + , w_(std::exchange(o.w_, nullptr)) + { + } + + worker_hold(worker_hold const&) = delete; + worker_hold& operator=(worker_hold const&) = delete; + worker_hold& operator=(worker_hold&&) = delete; + + ~worker_hold() + { + if (w_) + st_->drop_hold(*w_); + } + + state& server() const noexcept + { + return *st_; + } + }; + + // The holds travel in the frame so that they are also released + // when the frame is destroyed before it starts or finishes. One + // parameter, not three: compilers disagree on the order parameter + // copies are destroyed, but members always go in reverse, so the + // task, whose parameters may still use the worker, goes first. + template + struct session + { + worker_hold hold; + // Keeps the session's context running until the connection ends. + capy::work_guard work; + capy::task t; + }; + // Named functor to avoid incomplete lambda type in coroutine promise template struct launch_coro { - launch_wrapper operator()( - Executor, - std::stop_token, - state_ref st, - capy::task t, - worker_base* wp) + launch_wrapper + operator()(Executor, std::stop_token, session in) { - // Executor and stop token stored in promise via constructor - co_await std::move(t); - co_await push_awaitable{*st, *wp}; // worker back to the pool + // Executor and stop token stored in promise via constructor. + // A local, so a frame destroyed unfinished destroys the task + // while the promise and its environment still exist. + auto s = std::move(in); + // Resumed with the task unfinished: the chain was abandoned, + // and leaving destroys the task, which unwinds it. + if (!co_await capy::detail::abandonable>{s.t}) + co_return; + // The connection's work ends with its task: the hop below is + // the server's, and its context may no longer be running. + s.work.reset(); + // The hold returns the worker from the server's executor. + co_await hop_awaitable{s.hold.server(), true}; } }; - class push_awaitable + /// Resume on the server's executor. + class hop_awaitable { state& st_; - worker_base& w_; + bool root_; capy::continuation cont_; public: - push_awaitable(state& st, worker_base& w) noexcept : st_(st), w_(w) {} + // `root` is true when the awaiting coroutine is its chain's root. + explicit hop_awaitable(state& st, bool root = false) noexcept + : st_(st) + , root_(root) + { + } bool await_ready() const noexcept { @@ -432,17 +502,19 @@ class BOOST_COROSIO_DECL tcp_server } std::coroutine_handle<> - await_suspend(std::coroutine_handle<> h, capy::io_env const*) noexcept + await_suspend( + std::coroutine_handle<> h, capy::io_env const* env) noexcept { - // Symmetric transfer to server's executor cont_.h = h; + // A root has no owner: the launcher's would unwind it on the + // session's executor, which may already be gone. Given up + // here, it is destroyed in place. + if (!root_) + cont_.owner = env->owner; return st_.ex.dispatch(cont_); } - void await_resume() noexcept - { - st_.drop_hold(w_); - } + void await_resume() noexcept {} }; class pop_awaitable @@ -459,7 +531,7 @@ class BOOST_COROSIO_DECL tcp_server } bool - await_suspend(std::coroutine_handle<> h, capy::io_env const*) noexcept + await_suspend(std::coroutine_handle<> h, capy::io_env const* env) noexcept { std::lock_guard lock(st_.mutex); if (st_.stopping) @@ -472,8 +544,9 @@ class BOOST_COROSIO_DECL tcp_server wait_.w = w; return false; } - wait_.cont.h = h; - wait_.w = nullptr; + wait_.cont.h = h; + wait_.cont.owner = env->owner; + wait_.w = nullptr; wait_.next = st_.waiters; st_.waiters = &wait_; return true; @@ -553,7 +626,9 @@ class BOOST_COROSIO_DECL tcp_server The launcher must be invoked exactly once via `operator()`. If destroyed without invoking, the worker is returned to the - idle pool automatically. + idle pool automatically. So is the worker of a coroutine + abandoned before it completes, as when the execution context + holding it is destroyed. @see worker_base::run */ @@ -598,9 +673,24 @@ class BOOST_COROSIO_DECL tcp_server /** Start the connection-handling coroutine. - Starts the given coroutine on the specified executor. When - the coroutine completes, the worker is automatically returned - to the idle pool. + Starts the given coroutine on the specified executor, which + counts it as work until it ends. That execution context's + `run()` therefore does not return while a connection is + active. When the coroutine completes, or is abandoned before + completing, the worker is automatically returned to the idle + pool. + + @pre When @p ex belongs to another execution context, the + server's context is not destroyed until the coroutine has + finished. The coroutine holds the worker, whose socket + belongs to the server's context. The coroutine's own + context may be destroyed first, but only while the + coroutine is not suspended in an operation of the server's + context. An operation on the worker's socket is one such + operation. Destroying + that context then abandons the coroutine and returns its + worker. Otherwise stop the server and run both contexts + until their `run()` returns first. @pre @p task was created on a chain whose frame allocator outlives the connection. The task's frame is allocated @@ -647,17 +737,7 @@ class BOOST_COROSIO_DECL tcp_server if (stopped_before) w->stop_.request_stop(); - // Return worker to pool if coroutine setup throws - struct guard_t - { - state* st; - worker_base* w; - ~guard_t() - { - if (w) - st->drop_hold(*w); - } - } guard{st_, w}; + worker_hold hold(*st_, *w); // The connection can outlive whatever chain invoked this // launcher, so its frame comes from the process-wide @@ -679,7 +759,10 @@ class BOOST_COROSIO_DECL tcp_server launch_wrapper wrapper = [&] { allocator_scope scope; return launch_coro{}( - ex, st, state_ref(st_), std::move(task), w); + ex, st, + session{ + std::move(hold), capy::work_guard(ex), + std::move(task)}); }(); // Executor and stop token stored in promise via @@ -687,10 +770,14 @@ class BOOST_COROSIO_DECL tcp_server // continuation — posting the bare handle would allocate a // wrapper op. The frame stays suspended until the // executor resumes it, so the node outlives the queue. - auto h = std::exchange(wrapper.h, nullptr); // Release before post + auto h = wrapper.h; + h.promise().owner_.bind(h, h.promise().ex); h.promise().cont_.h = h; + // Released only once posted: a post that throws leaves the + // frame to the wrapper, which destroys it and with it the + // hold, the work, and the task. ex.post(h.promise().cont_); - guard.w = nullptr; // Success - dismiss guard + wrapper.h = nullptr; } }; diff --git a/include/boost/corosio/test/mocket.hpp b/include/boost/corosio/test/mocket.hpp index 61ead5578..284ab8fd2 100644 --- a/include/boost/corosio/test/mocket.hpp +++ b/include/boost/corosio/test/mocket.hpp @@ -557,6 +557,9 @@ class basic_mocket::write_some_awaitable supports provide/expect buffers for test instrumentation. The socket is the "peer" end with no test instrumentation. + Runs `ctx` until the connection is made, restarting it first if + it was stopped, and leaves it restarted. + Optional `max_read_size` and `max_write_size` parameters limit the number of bytes transferred per I/O operation on the mocket, simulating chunked network delivery for testing purposes. @@ -632,24 +635,29 @@ make_mocket_pair( }(peer, acc, endpoint(ipv4_address::loopback(), port), connect_ec, connect_done)); - ctx.run(); - ctx.restart(); + // Both coroutines write to locals here, so they must finish before + // this returns or throws, even on a context something stopped. + while (!accept_done || !connect_done) + { + ctx.run(); + ctx.restart(); + } - if (!connect_done || connect_ec) + if (connect_ec) { std::fprintf( - stderr, "make_mocket_pair: connect failed (done=%d, ec=%s)\n", - connect_done, connect_ec.message().c_str()); + stderr, "make_mocket_pair: connect failed: %s\n", + connect_ec.message().c_str()); acc.close(); accepted_socket.close(); throw std::runtime_error("mocket connect failed"); } - if (!accept_done || accept_ec) + if (accept_ec) { std::fprintf( - stderr, "make_mocket_pair: accept failed (done=%d, ec=%s)\n", - accept_done, accept_ec.message().c_str()); + stderr, "make_mocket_pair: accept failed: %s\n", + accept_ec.message().c_str()); acc.close(); throw std::runtime_error("mocket accept failed"); } diff --git a/include/boost/corosio/test/socket_pair.hpp b/include/boost/corosio/test/socket_pair.hpp index 54eb4a58c..869ab1403 100644 --- a/include/boost/corosio/test/socket_pair.hpp +++ b/include/boost/corosio/test/socket_pair.hpp @@ -30,6 +30,9 @@ namespace boost::corosio::test { Creates two sockets connected via loopback TCP sockets. Data written to one socket can be read from the other. + Runs `ctx` until the connection is made, restarting it first if + it was stopped, and leaves it restarted. + @tparam Socket The socket type (default `tcp_socket`). @tparam Acceptor The acceptor type (default `tcp_acceptor`). @tparam Linger Whether to enable `SO_LINGER` with a zero timeout @@ -93,24 +96,29 @@ make_socket_pair(io_context& ctx) }(s2, acc, endpoint(ipv4_address::loopback(), port), connect_ec, connect_done)); - ctx.run(); - ctx.restart(); + // Both coroutines write to locals here, so they must finish before + // this returns or throws, even on a context something stopped. + while (!accept_done || !connect_done) + { + ctx.run(); + ctx.restart(); + } - if (!connect_done || connect_ec) + if (connect_ec) { std::fprintf( - stderr, "socket_pair: connect failed (done=%d, ec=%s)\n", - connect_done, connect_ec.message().c_str()); + stderr, "socket_pair: connect failed: %s\n", + connect_ec.message().c_str()); acc.close(); s1.close(); throw std::runtime_error("socket_pair connect failed"); } - if (!accept_done || accept_ec) + if (accept_ec) { std::fprintf( - stderr, "socket_pair: accept failed (done=%d, ec=%s)\n", - accept_done, accept_ec.message().c_str()); + stderr, "socket_pair: accept failed: %s\n", + accept_ec.message().c_str()); acc.close(); throw std::runtime_error("socket_pair accept failed"); } diff --git a/src/corosio/src/tcp_server.cpp b/src/corosio/src/tcp_server.cpp index 313d17d99..2188fdfef 100644 --- a/src/corosio/src/tcp_server.cpp +++ b/src/corosio/src/tcp_server.cpp @@ -115,11 +115,15 @@ tcp_server::do_accept(state_ref st, tcp_acceptor& acc) auto* w = co_await pop_awaitable{*st}; if (!w) break; + // Not dropped if this frame is destroyed while accepting: that + // happens only when the listener's context is torn down, which + // the server cannot outlive. w->holds_.store(1, std::memory_order_relaxed); auto [ec] = co_await acc.accept(w->socket()); if (ec) { - co_await push_awaitable{*st, *w}; + co_await hop_awaitable{*st}; + st->drop_hold(*w); continue; } w->holds_.fetch_add(1, std::memory_order_relaxed); diff --git a/src/corosio/src/timer.cpp b/src/corosio/src/timer.cpp index 5612045f0..acdbbcb50 100644 --- a/src/corosio/src/timer.cpp +++ b/src/corosio/src/timer.cpp @@ -10,6 +10,7 @@ #include #include +#include namespace boost::corosio::detail { @@ -146,7 +147,7 @@ waiter_node::completion_op::do_complete( } void -waiter_node::completion_op::operator()() +waiter_node::completion_op::operator()() noexcept { // The node lives in the resuming coroutine's frame: posting the // continuation is the last access, since the frame (and node) @@ -161,6 +162,8 @@ waiter_node::completion_op::operator()() w->reset_stop_cb(); auto d = w->d_; auto& sched = w->svc_->get_scheduler(); + // A throw terminates: the waiter is already off the heap, so nothing + // else could resume or abandon the coroutine. d.post(w->cont_); sched.work_finished(); } @@ -177,15 +180,15 @@ waiter_node::completion_op::destroy() // This override also prevents scheduler_op::destroy() from calling // do_complete(nullptr, ...). See also: timer_service::shutdown() // which drains waiters still in the timer heap (the other path). - // Destroying the frame also ends the node's storage, so it is - // the last access. + // Unwinding the frame also ends the node's storage, so the + // abandon is the last access. auto* w = waiter_; w->reset_stop_cb(); - auto h = std::exchange(w->h_, {}); + auto c = std::exchange(w->cont_, {}); auto& sched = w->svc_->get_scheduler(); sched.work_finished(); - if (h) - h.destroy(); + if (c.h) + capy::abandon(c); } } // namespace boost::corosio::detail diff --git a/test/unit/context.hpp b/test/unit/context.hpp index 8bc2f6ee9..d4185dece 100644 --- a/test/unit/context.hpp +++ b/test/unit/context.hpp @@ -101,11 +101,9 @@ COROSIO_TEST_SELECT_(impl, name) \ COROSIO_TEST_URING_(impl, name) -// Tests that destroy the io_context with ops still parked abandon the -// suspended coroutine frames: op destroy() must not resume or destroy -// them (resuming runs user code during teardown; destroying recurses -// through the promise destructor). LeakSanitizer rightly reports those -// frames, so such tests are skipped under ASAN. +// Defined when building with AddressSanitizer. Teardown tests run under +// it: drained operations destroy their coroutine frames, so LeakSanitizer +// checks that teardown leaks nothing. #if defined(__SANITIZE_ADDRESS__) #define COROSIO_TEST_HAS_ASAN 1 #elif defined(__has_feature) diff --git a/test/unit/delay.cpp b/test/unit/delay.cpp index 4a38309f1..e6b6c734c 100644 --- a/test/unit/delay.cpp +++ b/test/unit/delay.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -685,6 +686,26 @@ struct delay_test BOOST_TEST_EQ(parent, 1); } + // A nested chain parked on a timer, started with an allocator held by + // value, is torn down with each frame freed before the trampoline + // holding that allocator. + void testTeardownNestedChainWithValueAllocator() + { + int child = 0, parent = 0; + { + io_context ioc(Backend); + capy::run_async(ioc.get_executor(), std::allocator{})( + [](int& child, int& parent) -> capy::task<> { + count_on_exit g{parent}; + std::ignore = co_await parked_child(child); + }(child, parent)); + std::ignore = ioc.poll(); + BOOST_TEST_EQ(child, 0); + } + BOOST_TEST_EQ(child, 1); + BOOST_TEST_EQ(parent, 1); + } + // The same with timeout() owning the awaited task. void testTeardownTimeoutOfTask() { @@ -1154,6 +1175,7 @@ struct delay_test testTeardownNestedTaskOnTimer(); testTeardownTimeoutOfTask(); testTeardownQueuedNestedContinuation(); + testTeardownNestedChainWithValueAllocator(); testInitiationStopRace(); testClockDeadlineCompletes(); testClockPastDeadlineCompletesImmediately(); diff --git a/test/unit/fault/posix_faults.cpp b/test/unit/fault/posix_faults.cpp index 3fe53d5b5..85612c683 100644 --- a/test/unit/fault/posix_faults.cpp +++ b/test/unit/fault/posix_faults.cpp @@ -648,7 +648,7 @@ struct posix_common_faults // The signal service's shutdown walks the implementations it still // owns, giving back each registration hanging off them. A set held - // in an abandoned coroutine frame is the only way to reach that + // in a coroutine frame parked in wait() is the only way to reach that // walk; the frame dies with the child rather than being reported // against the suite. void testSignalTeardownWalk() diff --git a/test/unit/io_context.cpp b/test/unit/io_context.cpp index c3bb6f557..47b16dfbf 100644 --- a/test/unit/io_context.cpp +++ b/test/unit/io_context.cpp @@ -10,8 +10,11 @@ // Test that header file is self-contained. #include #include +#include +#include #include +#include #include #include #include @@ -19,12 +22,15 @@ #include #include #include +#include +#include #include #include #include #include #include #include +#include #include "context.hpp" #include "test_suite.hpp" @@ -427,6 +433,46 @@ struct io_context_test BOOST_TEST(counter == 1); } + // A lockless context destroyed on another thread than the one that + // made and ran it still abandons the coroutines suspended in it. + void testUnsafeIoDestroyedOnAnotherThread() + { + struct counted + { + int* down_; + + ~counted() + { + if (down_) + ++*down_; + } + + explicit counted(int* d) noexcept : down_(d) {} + + counted(counted&& o) noexcept + : down_(std::exchange(o.down_, nullptr)) + { + } + }; + + int down = 0; + io_context_options opts; + opts.locking = locking_mode::unsafe_io; + auto ioc = std::make_unique(Backend, opts, 1); + { + auto [a, b] = test::make_socket_pair(*ioc); + capy::run_async(ioc->get_executor())( + [](tcp_socket s, tcp_socket, counted) -> capy::task<> { + char c; + std::ignore = + co_await s.read_some(capy::mutable_buffer(&c, 1)); + }(std::move(a), std::move(b), counted(&down))); + } + std::ignore = ioc->poll(); // the read parks + std::thread([&] { ioc.reset(); }).join(); + BOOST_TEST_EQ(down, 1); + } + void testGetExecutor() { io_context ioc(Backend); @@ -1000,16 +1046,45 @@ struct io_context_test } #endif + // A run call leaves the thread's frame allocator as it found it, so a + // coroutine created afterwards never uses a resource that died with + // the frames it ran. + template + void checkFrameAllocatorRestored(Drive drive) + { + io_context ioc(Backend); + auto* before = capy::get_current_frame_allocator(); + auto t = []() -> capy::task<> { co_return; }; + capy::run_async(ioc.get_executor(), std::allocator{})(t()); + drive(ioc); + BOOST_TEST(capy::get_current_frame_allocator() == before); + } + + void testRunRestoresFrameAllocator() + { + checkFrameAllocatorRestored([](io_context& c) { std::ignore = c.run(); }); + checkFrameAllocatorRestored( + [](io_context& c) { std::ignore = c.run_one(); }); + checkFrameAllocatorRestored([](io_context& c) { std::ignore = c.poll(); }); + checkFrameAllocatorRestored( + [](io_context& c) { std::ignore = c.poll_one(); }); + checkFrameAllocatorRestored([](io_context& c) { + std::ignore = c.run_for(std::chrono::seconds(5)); + }); + } + void run() { #if BOOST_COROSIO_POSIX testZeroThreadPoolSizeThrows(); #endif testConstruction(); + testRunRestoresFrameAllocator(); testConstructionWithOptions(); testConstructionWithThreadPoolSize(); testConstructionSingleThreaded(); testConstructionUnsafeIo(); + testUnsafeIoDestroyedOnAnotherThread(); testEffectiveConcurrencyHint(); testGetExecutor(); testRun(); diff --git a/test/unit/iocp_paths.cpp b/test/unit/iocp_paths.cpp index 9af20eb81..a1b450269 100644 --- a/test/unit/iocp_paths.cpp +++ b/test/unit/iocp_paths.cpp @@ -570,9 +570,6 @@ struct iocp_paths_test ::closesocket(static_cast(h)); } -#if !COROSIO_TEST_HAS_ASAN - // These abandon parked coroutine frames by design; see context.hpp. - void testDestroyWithParkedSocketOps() { int resumed = 0; @@ -700,7 +697,6 @@ struct iocp_paths_test } BOOST_TEST_EQ(resumed, 0); } -#endif // !COROSIO_TEST_HAS_ASAN void run() { @@ -716,11 +712,9 @@ struct iocp_paths_test testTruncateWithoutCreate(); testStaleWaitCancelsFromDestroyedObjects(); testReleaseWithKernelOpsInFlight(); -#if !COROSIO_TEST_HAS_ASAN testDestroyWithParkedSocketOps(); testDestroyWithParkedLocalOps(); testDestroyWithParkedTcpStreamOps(); -#endif } }; diff --git a/test/unit/local_datagram_socket.cpp b/test/unit/local_datagram_socket.cpp index f800f9f44..a9d960c7e 100644 --- a/test/unit/local_datagram_socket.cpp +++ b/test/unit/local_datagram_socket.cpp @@ -1026,10 +1026,7 @@ struct local_datagram_socket_test testDatagramBoundary(); testRecvPeek(); testRecvFromPeek(); -#if !COROSIO_TEST_HAS_ASAN - // Abandon parked coroutine frames by design; see context.hpp. testDestroyWithQueuedReceive(); -#endif #ifdef __linux__ testAbstractSocket(); #endif diff --git a/test/unit/local_stream_socket.cpp b/test/unit/local_stream_socket.cpp index 33173e5fa..b72923992 100644 --- a/test/unit/local_stream_socket.cpp +++ b/test/unit/local_stream_socket.cpp @@ -1333,7 +1333,7 @@ struct local_stream_socket_test } // Destroy the io_context with a read still parked on a connected - // pair; the socket service's shutdown must drain the abandoned + // pair; the socket service's shutdown must drain the parked // operation without resuming it. void testDestroyWithParkedRead() { @@ -2077,15 +2077,12 @@ struct local_stream_socket_test testAcceptPendingConnection(); #endif testAcceptWithoutListen(); -#if !COROSIO_TEST_HAS_ASAN - // Abandon parked coroutine frames by design; see context.hpp. testDestroyWithParkedAccept(); testDestroyWithParkedRead(); testDestroyWithQueuedRead(); testDestroyWithQueuedConnect(); testDestroyWithQueuedWait(); testDestroyWithQueuedWrite(); -#endif testAcceptorOnClosedNoOp(); testAcceptorBindClosedThrows(); testAcceptorListenClosedThrows(); diff --git a/test/unit/native/native_io.cpp b/test/unit/native/native_io.cpp index 592c0ffe2..0fc6ee696 100644 --- a/test/unit/native/native_io.cpp +++ b/test/unit/native/native_io.cpp @@ -14,9 +14,15 @@ #include #include +#include #include #include +#include +#include +#include +#include + #include "context.hpp" #include "test_suite.hpp" @@ -82,8 +88,37 @@ struct native_io_test BOOST_TEST(bytes == 5u); } + // A run call leaves the thread's frame allocator as it found it, so + // a coroutine created afterwards never uses a resource that died + // with the frames it ran. + template + void checkFrameAllocatorRestored(Drive drive) + { + native_io_context ctx; + auto* before = capy::get_current_frame_allocator(); + auto t = []() -> capy::task<> { co_return; }; + capy::run_async(ctx.get_executor(), std::allocator{})(t()); + drive(ctx); + BOOST_TEST(capy::get_current_frame_allocator() == before); + } + + void testRunRestoresFrameAllocator() + { + using ctx_t = native_io_context; + checkFrameAllocatorRestored([](ctx_t& c) { std::ignore = c.run(); }); + checkFrameAllocatorRestored( + [](ctx_t& c) { std::ignore = c.run_one(); }); + checkFrameAllocatorRestored([](ctx_t& c) { std::ignore = c.poll(); }); + checkFrameAllocatorRestored( + [](ctx_t& c) { std::ignore = c.poll_one(); }); + checkFrameAllocatorRestored([](ctx_t& c) { + std::ignore = c.run_for(std::chrono::seconds(5)); + }); + } + void run() { + testRunRestoresFrameAllocator(); testNativeIO(); } }; diff --git a/test/unit/pool_teardown.hpp b/test/unit/pool_teardown.hpp index ae11a0cf7..60e19fb4c 100644 --- a/test/unit/pool_teardown.hpp +++ b/test/unit/pool_teardown.hpp @@ -15,7 +15,9 @@ #include #include +#include #include +#include namespace boost::corosio::test { @@ -68,8 +70,9 @@ class pool_blocker : public detail::pool_work_item Create it last so the newest-first shutdown walk runs it before any library service: the blocked worker is let go at the very top of - teardown, and the operation queued behind it therefore completes - while the context is being destroyed rather than before. + teardown, and the operation queued behind it is therefore still + pending, on the pool or as a queued completion, while the context + is being destroyed. */ class pool_release_gate final : public capy::execution_context::service { @@ -115,6 +118,42 @@ park_pool_worker(io_context& ioc, pool_blocker& blocker) return !pool->post(&blocker); } +/** Shut down a context's pool while a blocker still holds its worker. + + Everything queued behind the blocker is then still queued when the + pool stops taking work, so the pool disposes of it at shutdown + instead of running it. The context itself stays alive. + + @param ioc The context whose pool is shut down. + @param blocker The item occupying the pool's only worker. +*/ +inline void +shutdown_pool_behind(io_context& ioc, pool_blocker& blocker) +{ + struct probe : detail::pool_work_item + { + static void ignore(detail::pool_work_item*) noexcept {} + + probe() noexcept + { + func_ = &probe::ignore; + destroy_ = &probe::ignore; + } + }; + + auto& pool = ioc.use_service(); + std::thread stopper([&pool] { pool.shutdown(); }); + + // A refused post is the only sign that shutdown has begun; the + // probes accepted before then queue harmlessly behind the rest. + std::list probes; + while (!pool.post(&probes.emplace_back())) + std::this_thread::yield(); + + blocker.release(); + stopper.join(); +} + } // namespace boost::corosio::test #endif // BOOST_COROSIO_TEST_POOL_TEARDOWN_HPP diff --git a/test/unit/random_access_file.cpp b/test/unit/random_access_file.cpp index 3c9ba090a..5b5197c70 100644 --- a/test/unit/random_access_file.cpp +++ b/test/unit/random_access_file.cpp @@ -693,9 +693,9 @@ struct random_access_file_test // the completion the worker posts on its way out is neither run nor // destroyed and the operation's keepalive leaks. // - // The task is started by hand and owned by the test, so the frame - // the library abandons at teardown is destroyed here rather than - // leaked: what LeakSanitizer sees left over is the defect alone. + // Teardown abandons the task wherever the work then sits, on the + // pool queue or the scheduler's, and the task is unwound without + // resuming. void testDestroyWithPoolWorkQueued() { #if BOOST_COROSIO_HAS_URING @@ -708,9 +708,6 @@ struct random_access_file_test auto const path = tmp.path; bool resumed = false; test::pool_blocker blocker; - std::optional ex; - std::optional env; - std::optional> parked; { io_context ioc(Backend); BOOST_TEST(test::park_pool_worker(ioc, blocker)); @@ -724,10 +721,8 @@ struct random_access_file_test resumed = true; }; - ex.emplace(ioc.get_executor()); - env.emplace(capy::io_env{*ex, std::stop_token{}, nullptr}); - parked.emplace(reader()); - parked->await_suspend(std::noop_coroutine(), &*env).resume(); + capy::run_async(ioc.get_executor())(reader()); + std::ignore = ioc.poll_one(); } BOOST_TEST(!resumed); } @@ -771,17 +766,11 @@ struct random_access_file_test result_bytes = n; resumed = true; }; - - std::optional ex; - std::optional env; - std::optional> parked; - ex.emplace(ioc.get_executor()); - env.emplace(capy::io_env{*ex, std::stop_token{}, nullptr}); - parked.emplace(reader()); - // Synchronously runs the coroutine up to its first suspension, + capy::run_async(ioc.get_executor())(reader()); + // Runs the coroutine up to its first suspension, // which posts the read to the pool -- queued behind the parked // worker, not yet executed. - parked->await_suspend(std::noop_coroutine(), &*env).resume(); + std::ignore = ioc.poll_one(); int fd2 = ::open(tmp2.path.c_str(), O_RDONLY); BOOST_TEST(fd2 >= 0); @@ -831,15 +820,9 @@ struct random_access_file_test result_ec = ec; resumed = true; }; - - std::optional ex; - std::optional env; - std::optional> parked; - ex.emplace(ioc.get_executor()); - env.emplace(capy::io_env{*ex, std::stop_token{}, nullptr}); - parked.emplace(writer()); + capy::run_async(ioc.get_executor())(writer()); // Queues the write behind the parked worker. - parked->await_suspend(std::noop_coroutine(), &*env).resume(); + std::ignore = ioc.poll_one(); // Recycle the released number onto the other file. int raw = static_cast(f.release()); @@ -1162,10 +1145,7 @@ struct random_access_file_test testReleaseCancelsQueuedWrite(); #endif -#if !COROSIO_TEST_HAS_ASAN - // Abandon parked coroutine frames by design; see context.hpp. testDestroyWithLiveFile(); -#endif } // Operations on closed file diff --git a/test/unit/reactor_paths.cpp b/test/unit/reactor_paths.cpp index deca606fb..83dc57a05 100644 --- a/test/unit/reactor_paths.cpp +++ b/test/unit/reactor_paths.cpp @@ -1866,10 +1866,7 @@ struct reactor_paths_test testLocalDgramShutdownReceive(); testStopTokenLocalStreamRead(); testLocalStreamWriteEAGAIN(); -#if !COROSIO_TEST_HAS_ASAN - // Abandons parked coroutine frames by design; see context.hpp. testShutdownWithParkedOps(); -#endif #endif } }; diff --git a/test/unit/reactor_teardown_inflight.cpp b/test/unit/reactor_teardown_inflight.cpp index 1632938b8..6db4fc5f3 100644 --- a/test/unit/reactor_teardown_inflight.cpp +++ b/test/unit/reactor_teardown_inflight.cpp @@ -156,7 +156,7 @@ struct reactor_teardown_test io_context ioc(Backend); // Made outside the coroutines (it drains the context); each // socket moves into a frame so its fd stays open when the - // frame is abandoned. + // frame is destroyed. auto [w1, w2] = test::make_socket_pair(ioc); auto [r1, r2] = @@ -223,11 +223,9 @@ struct reactor_teardown_test void run() { -#if !COROSIO_TEST_HAS_ASAN testDestroyWithParkedDescriptorOps(); testDestroyWithParkedSocketOps(); testDestroyDrainsBothEndsOfOnePipe(); -#endif } }; diff --git a/test/unit/resolver.cpp b/test/unit/resolver.cpp index c5476c8c2..e433fa368 100644 --- a/test/unit/resolver.cpp +++ b/test/unit/resolver.cpp @@ -1226,9 +1226,9 @@ struct resolver_test // drains, or the completion the worker posts on its way out is // neither run nor destroyed and the operation's keepalive leaks. // - // The task is started by hand and owned by the test, so the frame - // the library abandons at teardown is destroyed here rather than - // leaked: what LeakSanitizer sees left over is the defect alone. + // Teardown abandons the task wherever the work then sits, on the + // pool queue or the scheduler's, and the task is unwound without + // resuming. // // Forward resolution reaches the pool on POSIX only; IOCP resolves // through GetAddrInfoExW and the completion port. @@ -1236,9 +1236,6 @@ struct resolver_test { bool resumed = false; test::pool_blocker blocker; - std::optional ex; - std::optional env; - std::optional> parked; { io_context ioc; BOOST_TEST(test::park_pool_worker(ioc, blocker)); @@ -1253,10 +1250,8 @@ struct resolver_test resumed = true; }; - ex.emplace(ioc.get_executor()); - env.emplace(capy::io_env{*ex, std::stop_token{}, nullptr}); - parked.emplace(query()); - parked->await_suspend(std::noop_coroutine(), &*env).resume(); + capy::run_async(ioc.get_executor())(query()); + std::ignore = ioc.poll_one(); } BOOST_TEST(!resumed); } @@ -1268,9 +1263,6 @@ struct resolver_test { bool resumed = false; test::pool_blocker blocker; - std::optional ex; - std::optional env; - std::optional> parked; { io_context ioc; BOOST_TEST(test::park_pool_worker(ioc, blocker)); @@ -1285,10 +1277,8 @@ struct resolver_test resumed = true; }; - ex.emplace(ioc.get_executor()); - env.emplace(capy::io_env{*ex, std::stop_token{}, nullptr}); - parked.emplace(query()); - parked->await_suspend(std::noop_coroutine(), &*env).resume(); + capy::run_async(ioc.get_executor())(query()); + std::ignore = ioc.poll_one(); } BOOST_TEST(!resumed); } @@ -1304,10 +1294,9 @@ struct resolver_test // file synchronously, so GetAddrInfoExW posts the op inline and it is // reliably queued when the resolver is freed. // - // The task is started by hand and owned by the test, so the frame the - // library abandons at teardown is destroyed here rather than leaked: - // what a sanitizer reports is the defect alone. The bug is observable - // only under ASan; without a sanitizer the freed read is silent. + // Teardown abandons the task, which is unwound without resuming. + // The bug is observable only under ASan; without a + // sanitizer the freed read is silent. // // Contrast the reverse path, which holds the keepalive on // reverse_op_.object_ref_ across the queued completion @@ -1316,9 +1305,6 @@ struct resolver_test void testDestroyWithForwardResolveQueued() { bool resumed = false; - std::optional ex; - std::optional env; - std::optional> parked; { io_context ioc; resolver r(ioc); @@ -1327,10 +1313,8 @@ struct resolver_test resumed = true; }; - ex.emplace(ioc.get_executor()); - env.emplace(capy::io_env{*ex, std::stop_token{}, nullptr}); - parked.emplace(query()); - parked->await_suspend(std::noop_coroutine(), &*env).resume(); + capy::run_async(ioc.get_executor())(query()); + std::ignore = ioc.poll_one(); } BOOST_TEST(!resumed); } @@ -1346,9 +1330,6 @@ struct resolver_test { for (int i = 0; i < 4; ++i) { - std::optional ex; - std::optional env; - std::optional> parked; std::string host = "corosio-no-such-host-" + std::to_string(i) + "-" + @@ -1363,10 +1344,8 @@ struct resolver_test std::ignore = co_await r.resolve(host, "80"); }; - ex.emplace(ioc.get_executor()); - env.emplace(capy::io_env{*ex, std::stop_token{}, nullptr}); - parked.emplace(query()); - parked->await_suspend(std::noop_coroutine(), &*env).resume(); + capy::run_async(ioc.get_executor())(query()); + std::ignore = ioc.poll_one(); } } BOOST_TEST_PASS(); @@ -1487,10 +1466,7 @@ struct resolver_test testDestroyWithForwardResolvePending(); #endif -#if !COROSIO_TEST_HAS_ASAN - // Abandon parked coroutine frames by design; see context.hpp. testDestroyWithLiveResolver(); -#endif } }; diff --git a/test/unit/signal_set.cpp b/test/unit/signal_set.cpp index 23700ca34..0079c13f9 100644 --- a/test/unit/signal_set.cpp +++ b/test/unit/signal_set.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include "context.hpp" #include "test_suite.hpp" @@ -633,6 +634,39 @@ struct signal_set_test BOOST_TEST(got == SIGINT); } + // A stop requested on another thread can claim a parked wait while + // the context tears its signal sets down; the drain must still find + // the wait the stop posts. + void testStopRacingTeardown() + { + for (int round = 0; round < 200; ++round) + { + std::vector stops(8); + std::atomic go{false}; + std::thread stopper; + { + io_context ioc(Backend); + for (auto& src : stops) + capy::run_async(ioc.get_executor(), src.get_token())( + [](io_context& ioc) -> capy::task<> { + signal_set s(ioc, SIGINT); + std::ignore = co_await s.wait(); + }(ioc)); + std::ignore = ioc.poll(); // parks every wait + stopper = std::thread([&] { + while (!go.load(std::memory_order_acquire)) + { + } + for (auto& src : stops) + src.request_stop(); + }); + go.store(true, std::memory_order_release); + } + stopper.join(); + } + BOOST_TEST_PASS(); + } + void testDestroyWithArmedStopToken() { // Review Focus 3: destroying the set while a callback is armed @@ -752,7 +786,7 @@ struct signal_set_test BOOST_TEST_PASS(); } - // A wait still parked when its context is torn down is abandoned, + // A wait still parked when its context is torn down is destroyed, // not resumed, even when a frame destroyed during the teardown owns // the set and its destructor cancels the wait. Only IOCP can fail // this: it resumes a cancelled wait inline, where POSIX posts it to a @@ -779,10 +813,10 @@ struct signal_set_test } // The process signal table outlives every io_context, so a set still - // registered when its context shuts down -- which an abandoned frame - // is the only way to arrange -- has to hand the registration back - // there: a stale entry keeps the signal installed with the old flags - // and refuses the next add() of it. + // registered when its context shuts down -- which a frame parked in + // wait() is the only way to arrange -- has to hand the registration + // back there: a stale entry keeps the signal installed with the old + // flags and refuses the next add() of it. void testShutdownReleasesRegistration() { #if BOOST_COROSIO_POSIX @@ -1583,6 +1617,7 @@ struct signal_set_test testNormalDeliveryWithLiveToken(); testDestroyWithArmedStopToken(); testConcurrentStopRequestRace(); + testStopRacingTeardown(); testShutdownWithPendingSignalSet(); // Multiple signal set tests @@ -1633,12 +1668,9 @@ struct signal_set_test testFlagsNotSupportedOnWindows(); #endif -#if !COROSIO_TEST_HAS_ASAN - // Abandon parked coroutine frames by design; see context.hpp. testShutdownReleasesRegistration(); testTeardownDoesNotResumeParkedWait(); testShutdownKeepsOtherContextRegistered(); -#endif } }; diff --git a/test/unit/stream_file.cpp b/test/unit/stream_file.cpp index 3f1ba8000..8f960ca7a 100644 --- a/test/unit/stream_file.cpp +++ b/test/unit/stream_file.cpp @@ -1191,9 +1191,9 @@ struct stream_file_test // the completion the worker posts on its way out is neither run nor // destroyed and the operation's keepalive leaks. // - // The task is started by hand and owned by the test, so the frame - // the library abandons at teardown is destroyed here rather than - // leaked: what LeakSanitizer sees left over is the defect alone. + // Teardown abandons the task wherever the work then sits, on the + // pool queue or the scheduler's, and the task is unwound without + // resuming. void testDestroyWithPoolWorkQueued() { #if BOOST_COROSIO_HAS_URING @@ -1206,9 +1206,6 @@ struct stream_file_test auto const path = tmp.path; bool resumed = false; test::pool_blocker blocker; - std::optional ex; - std::optional env; - std::optional> parked; { io_context ioc(Backend); BOOST_TEST(test::park_pool_worker(ioc, blocker)); @@ -1222,10 +1219,8 @@ struct stream_file_test resumed = true; }; - ex.emplace(ioc.get_executor()); - env.emplace(capy::io_env{*ex, std::stop_token{}, nullptr}); - parked.emplace(reader()); - parked->await_suspend(std::noop_coroutine(), &*env).resume(); + capy::run_async(ioc.get_executor())(reader()); + std::ignore = ioc.poll_one(); } BOOST_TEST(!resumed); } @@ -1269,17 +1264,11 @@ struct stream_file_test result_bytes = n; resumed = true; }; - - std::optional ex; - std::optional env; - std::optional> parked; - ex.emplace(ioc.get_executor()); - env.emplace(capy::io_env{*ex, std::stop_token{}, nullptr}); - parked.emplace(reader()); - // Synchronously runs the coroutine up to its first suspension, + capy::run_async(ioc.get_executor())(reader()); + // Runs the coroutine up to its first suspension, // which posts the read to the pool -- queued behind the parked // worker, not yet executed. - parked->await_suspend(std::noop_coroutine(), &*env).resume(); + std::ignore = ioc.poll_one(); int fd2 = ::open(tmp2.path.c_str(), O_RDONLY); BOOST_TEST(fd2 >= 0); @@ -1329,15 +1318,9 @@ struct stream_file_test result_ec = ec; resumed = true; }; - - std::optional ex; - std::optional env; - std::optional> parked; - ex.emplace(ioc.get_executor()); - env.emplace(capy::io_env{*ex, std::stop_token{}, nullptr}); - parked.emplace(writer()); + capy::run_async(ioc.get_executor())(writer()); // Queues the write behind the parked worker. - parked->await_suspend(std::noop_coroutine(), &*env).resume(); + std::ignore = ioc.poll_one(); // Recycle the released number onto the other file. int raw = static_cast(f.release()); @@ -1522,10 +1505,7 @@ struct stream_file_test #endif testReassignDuringInFlightReadKeepsNewOffset(); -#if !COROSIO_TEST_HAS_ASAN - // Abandon parked coroutine frames by design; see context.hpp. testDestroyWithLiveFile(); -#endif } // Cancellation diff --git a/test/unit/tcp_acceptor.cpp b/test/unit/tcp_acceptor.cpp index d1193ef2d..2119ac6d4 100644 --- a/test/unit/tcp_acceptor.cpp +++ b/test/unit/tcp_acceptor.cpp @@ -32,7 +32,9 @@ #include #include #include +#include #include +#include #ifndef _WIN32 // For the SO_REUSEPORT guard around testReusePort. The corosio public @@ -152,9 +154,8 @@ native_connect_loopback(native_handle_type h, std::uint16_t port, bool v6) } // Write one byte without the io_context, so the peer's completion is -// already waiting when the next run starts. Unused when the tests that -// abandon parked frames are compiled out. -[[maybe_unused]] bool +// already waiting when the next run starts. +bool native_send_byte(native_handle_type h) { char const b = 'x'; @@ -487,7 +488,7 @@ struct tcp_acceptor_test } // Destroy the io_context with a read still parked on a connected - // socket; service shutdown must drain the abandoned operation + // socket; service shutdown must drain the parked operation // without resuming it. void testDestroyWithParkedRead() { @@ -2022,11 +2023,13 @@ struct tcp_acceptor_test { auto client = make_native_socket(AF_INET, SOCK_STREAM); BOOST_TEST(client != invalid_native_socket); - // The drained accept abandons its awaiting frame, as every - // drained I/O operation does; the arena reclaims it. - std::pmr::monotonic_buffer_resource arena; + bool resumed = false; { - io_context a(Backend); + // No inline completion: the accept that takes the + // connection is queued instead of resuming its awaiter. + io_context_options opts; + opts.inline_budget_max = 0; + io_context a(Backend, opts); tcp_acceptor acc(a); BOOST_TEST(!acc.open()); acc.set_option(socket_option::reuse_address(true)); @@ -2034,14 +2037,21 @@ struct tcp_acceptor_test BOOST_TEST(!acc.listen()); BOOST_TEST(native_connect_loopback( client, acc.local_endpoint().port(), false)); - - io_context b(Backend); - capy::run_async(b.get_executor(), &arena)( - [](tcp_acceptor& acc) -> capy::task<> { + // A connect can return before the connection reaches the + // listen queue, so the accept would park instead of taking it. + // Bounds a hang, not a timing assertion. + pollfd lpfd{ + static_cast(acc.native_handle()), POLLIN, 0}; + BOOST_TEST_EQ(::poll(&lpfd, 1, 5000), 1); + + capy::run_async(a.get_executor())( + [](tcp_acceptor& acc, bool& r) -> capy::task<> { std::ignore = co_await acc.accept(); - }(acc)); - std::ignore = b.poll(); // the accept completes into a's queue + r = true; + }(acc, resumed)); + BOOST_TEST_EQ(a.poll_one(), 1u); // starts the accept } + BOOST_TEST(!resumed); pollfd pfd{client, POLLIN, 0}; BOOST_TEST_EQ(::poll(&pfd, 1, 5000), 1); @@ -2173,6 +2183,44 @@ struct tcp_acceptor_test pool.join(); } + // A stop requested on another thread can claim a parked accept + // while the context tears its acceptors down; the drain must still + // find the accept the stop posts. + void testAcceptStopRacingTeardown() + { + for (int round = 0; round < 50; ++round) + { + std::vector stops(8); + std::atomic go{false}; + std::thread stopper; + { + io_context ioc(Backend); + for (auto& src : stops) + capy::run_async(ioc.get_executor(), src.get_token())( + [](io_context& ioc) -> capy::task<> { + tcp_acceptor acc(ioc); + std::ignore = acc.open(); + std::ignore = acc.bind( + endpoint(ipv4_address::loopback(), 0)); + std::ignore = acc.listen(); + tcp_socket peer(ioc); + std::ignore = co_await acc.accept(peer); + }(ioc)); + std::ignore = ioc.poll(); // parks every accept + stopper = std::thread([&] { + while (!go.load(std::memory_order_acquire)) + { + } + for (auto& src : stops) + src.request_stop(); + }); + go.store(true, std::memory_order_release); + } + stopper.join(); + } + BOOST_TEST_PASS(); + } + void run() { testAssignOnOpenIsAlreadyOpen(); @@ -2185,6 +2233,7 @@ struct tcp_acceptor_test // Cancellation testCancelAccept(); testAcceptAfterCancel(); + testAcceptStopRacingTeardown(); testDeferredExecutorCompletion(); #if BOOST_COROSIO_POSIX testQueuedAcceptClosesPeerOnTeardown(); @@ -2192,15 +2241,12 @@ struct tcp_acceptor_test testAcceptedPeerClosedWhenAwaiterDestroyed(true); #endif testCloseWhilePendingAccept(); -#if !COROSIO_TEST_HAS_ASAN - // Abandon parked coroutine frames by design; see context.hpp. testDestroyWithParkedAccept(); testDestroyWithParkedRead(); testInlineAcceptUnderStopTokenRecycles(); testDestroyListeningAcceptorKeepsOtherOps(); testDestroyWithQueuedAccept(); testDestroyWithQueuedAcceptorWait(); -#endif // IPv6 testListenV6(); diff --git a/test/unit/tcp_server.cpp b/test/unit/tcp_server.cpp index 83d85f17c..97fef836f 100644 --- a/test/unit/tcp_server.cpp +++ b/test/unit/tcp_server.cpp @@ -14,13 +14,17 @@ #include #include #include +#include #include #include +#include #include #include #include #include +#include +#include #include #include #include @@ -807,7 +811,7 @@ struct tcp_server_test { // With one worker handling two sequential connections, the // second accept loop iteration must wait for the worker to - // return (exercises pop_awaitable suspend / push_awaitable wake). + // return (exercises pop_awaitable suspend / worker return wake). io_context ioc(Backend); class one_worker_server : public tcp_server @@ -1317,12 +1321,370 @@ struct tcp_server_test } }; + // Reads one byte from its socket on another context. + class reading_worker : public tcp_server::worker_base + { + corosio::tcp_socket sock_; + io_context& session_ctx_; + bool& launched_; + + public: + reading_worker(io_context& ctx, io_context& session_ctx, bool& l) + : sock_(ctx) + , session_ctx_(session_ctx) + , launched_(l) + { + } + + corosio::tcp_socket& socket() override + { + return sock_; + } + + void run(tcp_server::launcher launch) override + { + launched_ = true; + launch( + session_ctx_.get_executor(), + [](corosio::tcp_socket& s) -> capy::task<> { + char c; + std::ignore = + co_await s.read_some(capy::mutable_buffer(&c, 1)); + }(sock_)); + } + }; + + // A connection is work on the context it runs on, so that context + // does not run out of work while the connection waits elsewhere. + void testSessionHoldsWorkOnItsContext() + { + io_context session_ctx(Backend); + io_context ioc(Backend); + bool launched = false; + + class server : public tcp_server + { + public: + server(io_context& ctx, io_context& sctx, bool& l) + : tcp_server(ctx, ctx.get_executor()) + { + std::vector> v; + v.push_back(std::make_unique(ctx, sctx, l)); + set_workers(std::move(v)); + } + }; + + server srv(ioc, session_ctx, launched); + BOOST_TEST(!srv.bind(endpoint(ipv4_address::loopback(), 0))); + srv.start(); + tcp_socket client(ioc); + capy::run_async(ioc.get_executor())( + [](tcp_socket& c, endpoint ep) -> capy::task<> { + std::ignore = co_await c.connect(ep); + }(client, srv.local_endpoint())); + // Bounds a hang, not a timing assertion. + while (!launched && ioc.run_one_for(std::chrono::seconds(5)) != 0) + { + } + BOOST_TEST(launched); + + std::ignore = session_ctx.poll(); + BOOST_TEST(!session_ctx.stopped()); + + // The connection ends, and with it the work. + client.close(); + while (!session_ctx.stopped() && + ioc.run_one_for(std::chrono::seconds(5)) != 0) + { + std::ignore = session_ctx.poll(); + } + BOOST_TEST(session_ctx.stopped()); + + srv.stop(); + std::ignore = ioc.run(); + srv.join(); + } + + // A finished connection's hop back to the server's executor, left + // queued when that context is destroyed, is torn down there, never + // sent to the session's context, which may already be gone. + void testQueuedHopTornDownOnServerContext() + { + auto session_ctx = std::make_unique(Backend); + auto ioc = std::make_unique(Backend); + bool launched = false; + + class server : public tcp_server + { + public: + server(io_context& ctx, io_context& sctx, bool& l) + : tcp_server(ctx, ctx.get_executor()) + { + std::vector> v; + v.push_back(std::make_unique(ctx, sctx, l)); + set_workers(std::move(v)); + } + }; + + { + server srv(*ioc, *session_ctx, launched); + BOOST_TEST(!srv.bind(endpoint(ipv4_address::loopback(), 0))); + srv.start(); + tcp_socket client(*ioc); + capy::run_async(ioc->get_executor())( + [](tcp_socket& c, endpoint ep) -> capy::task<> { + std::ignore = co_await c.connect(ep); + }(client, srv.local_endpoint())); + // Bounds a hang, not a timing assertion. + while (!launched && ioc->run_one_for(std::chrono::seconds(5)) != 0) + { + } + BOOST_TEST(launched); + std::ignore = session_ctx->poll(); + + client.close(); + while (!session_ctx->stopped() && + ioc->run_one_for(std::chrono::seconds(5)) != 0) + { + std::ignore = session_ctx->poll(); + } + BOOST_TEST(session_ctx->stopped()); + srv.stop(); + } + session_ctx.reset(); + ioc.reset(); + } + + // Holds what is posted or dispatched to it until the test runs it + // or gives it up, as a context's queue does. + struct held_executor + { + io_context* ctx_; + std::deque* q_; + + bool operator==(held_executor const& o) const noexcept + { + return q_ == o.q_; + } + + io_context& context() const noexcept + { + return *ctx_; + } + + void on_work_started() const noexcept {} + void on_work_finished() const noexcept {} + + std::coroutine_handle<> dispatch(capy::continuation& c) const + { + q_->push_back(c); + return std::noop_coroutine(); + } + + void post(capy::continuation& c) const + { + q_->push_back(c); + } + }; + + // Counts work; refuses every post. + struct refusing_executor + { + io_context* ctx_; + int* work_; + + bool operator==(refusing_executor const& o) const noexcept + { + return work_ == o.work_; + } + + io_context& context() const noexcept + { + return *ctx_; + } + + void on_work_started() const noexcept + { + ++*work_; + } + + void on_work_finished() const noexcept + { + --*work_; + } + + std::coroutine_handle<> dispatch(capy::continuation&) const + { + throw std::runtime_error("dispatch"); + } + + void post(capy::continuation&) const + { + throw std::runtime_error("post"); + } + }; + + struct destroy_count + { + int* n; + + explicit destroy_count(int& c) noexcept + : n(&c) + { + } + + destroy_count(destroy_count&& o) noexcept + : n(std::exchange(o.n, nullptr)) + { + } + + ~destroy_count() + { + if (n) + ++*n; + } + }; + + // A launch whose post throws hands the exception to the worker and + // gives back what it took: the task, its work, and the worker. + void testLaunchPostThrows() + { + io_context ioc(Backend); + int work = 0, runs = 0, caught = 0, down = 0; + + class refused_worker : public tcp_server::worker_base + { + corosio::tcp_socket sock_; + io_context& ctx_; + int& work_; + int& runs_; + int& caught_; + int& down_; + + public: + refused_worker( + io_context& ctx, int& work, int& runs, int& caught, int& down) + : sock_(ctx) + , ctx_(ctx) + , work_(work) + , runs_(runs) + , caught_(caught) + , down_(down) + { + } + + corosio::tcp_socket& socket() override + { + return sock_; + } + + void run(tcp_server::launcher launch) override + { + ++runs_; + sock_.close(); + try + { + launch( + refusing_executor{&ctx_, &work_}, + [](destroy_count) -> capy::task<> { co_return; }( + destroy_count(down_))); + } + catch (std::runtime_error const&) + { + ++caught_; + } + } + }; + + class server : public tcp_server + { + public: + server(io_context& ctx, int& w, int& r, int& c, int& d) + : tcp_server(ctx, ctx.get_executor()) + { + std::vector> v; + v.push_back(std::make_unique(ctx, w, r, c, d)); + set_workers(std::move(v)); + } + }; + + server srv(ioc, work, runs, caught, down); + BOOST_TEST(!srv.bind(endpoint(ipv4_address::loopback(), 0))); + srv.start(); + tcp_socket c1(ioc), c2(ioc); + auto connect = [](tcp_socket& c, endpoint ep) -> capy::task<> { + std::ignore = co_await c.connect(ep); + }; + capy::run_async(ioc.get_executor())(connect(c1, srv.local_endpoint())); + capy::run_async(ioc.get_executor())(connect(c2, srv.local_endpoint())); + // Bounds a hang, not a timing assertion. + while (runs < 2 && ioc.run_one_for(std::chrono::seconds(5)) != 0) + { + } + BOOST_TEST_EQ(runs, 2); + BOOST_TEST_EQ(caught, 2); + BOOST_TEST_EQ(down, 2); + BOOST_TEST_EQ(work, 0); + srv.stop(); + std::ignore = ioc.run(); + srv.join(); + } + + // An accept loop's hop back to the server's executor, given up while + // queued there, unwinds the loop from the coroutine awaiting it + // rather than destroying the loop's frame under that coroutine. + void testQueuedAcceptLoopHopGivenUp() + { + io_context ioc(Backend); + std::deque q; + bool launched = false; + + class server : public tcp_server + { + public: + server(io_context& ctx, held_executor ex, bool& l) + : tcp_server(ctx, ex) + { + std::vector> v; + v.push_back(std::make_unique(ctx, ctx, l)); + set_workers(std::move(v)); + } + }; + + auto run_queued = [&q](std::size_t n) + { + while (n-- != 0 && !q.empty()) + { + auto c = q.front(); + q.pop_front(); + c.h.resume(); + } + }; + + { + server srv(ioc, held_executor{&ioc, &q}, launched); + BOOST_TEST(!srv.bind(endpoint(ipv4_address::loopback(), 0))); + srv.start(); + run_queued(q.size()); // the loop parks its accept + srv.stop(); + std::ignore = ioc.poll(); // the canceled accept is queued + run_queued(q.size()); // the loop hops back, and stays queued + while (!q.empty()) + { + auto c = q.front(); + q.pop_front(); + capy::abandon(c); + } + } + BOOST_TEST(!launched); + } + // A worker is not handed to another connection while its run() is // still executing, even once the connection it launched is done. void testWorkerNotReusedDuringRun() { - io_context ioc(Backend); io_context session_ctx(Backend); + io_context ioc(Backend); overlap_counts counts; class server : public tcp_server @@ -1370,6 +1732,331 @@ struct tcp_server_test BOOST_TEST_EQ(counts.max_in_run.load(), 1); } + struct destroy_count_t + { + int* n; + explicit destroy_count_t(int& c) : n(&c) {} + destroy_count_t(destroy_count_t&& o) noexcept + : n(std::exchange(o.n, nullptr)) + { + } + ~destroy_count_t() + { + if (n) + ++*n; + } + }; + + // Connects a socket the caller keeps open. + static void connect_kept(io_context& ioc, tcp_socket& c, endpoint ep) + { + std::error_code ec; + capy::run_async(ioc.get_executor())( + [](tcp_socket& c, endpoint ep, + std::error_code& ec) -> capy::task<> { + auto [e] = co_await c.connect(ep); + ec = e; + }(c, ep, ec)); + ioc.restart(); + ioc.run(); + BOOST_TEST(!ec); + } + + // Its first connection reads forever on a socket pair of another + // context, which owns nothing else of the server; later + // connections are declined. + class session_worker : public tcp_server::worker_base + { + corosio::tcp_socket sock_; + io_context& session_ctx_; + int& runs_; + int& down_; + + public: + session_worker(io_context& ctx, io_context& sctx, int& runs, int& down) + : sock_(ctx) + , session_ctx_(sctx) + , runs_(runs) + , down_(down) + { + } + + corosio::tcp_socket& socket() override + { + return sock_; + } + + void run(tcp_server::launcher launch) override + { + sock_.close(); + if (runs_++ != 0) + return; + auto [a, b] = test::make_socket_pair(session_ctx_); + launch( + session_ctx_.get_executor(), + [](tcp_socket s, tcp_socket, destroy_count_t) -> capy::task<> { + char c; + std::ignore = + co_await s.read_some(capy::mutable_buffer(&c, 1)); + }(std::move(a), std::move(b), destroy_count_t(down_))); + } + }; + + class session_server : public tcp_server + { + public: + session_server(io_context& ctx, io_context& sctx, int& runs, int& down) + : tcp_server(ctx, ctx.get_executor()) + { + std::vector> v; + v.push_back( + std::make_unique(ctx, sctx, runs, down)); + set_workers(std::move(v)); + } + }; + + // Tearing down the context a connection runs on returns its worker, + // so an accept loop waiting for one gets it. + void testSessionTeardownWakesWaitingAccept() + { + int runs = 0, down = 0; + io_context ioc(Backend); + std::optional session_ctx(std::in_place, Backend); + session_server srv(ioc, *session_ctx, runs, down); + BOOST_TEST(!srv.bind(endpoint(ipv4_address::loopback(), 0))); + auto ep = srv.local_endpoint(); + srv.start(); + + io_context cctx; + tcp_socket c1(cctx), c2(cctx); + connect_kept(cctx, c1, ep); + // Bounds a hang, not a timing assertion. + while (runs < 1 && ioc.run_one_for(std::chrono::seconds(5)) != 0) + { + } + BOOST_TEST_EQ(runs, 1); + std::ignore = session_ctx->poll(); + connect_kept(cctx, c2, ep); + while (ioc.poll() != 0) + { + } + BOOST_TEST_EQ(runs, 1); // the accept loop waits for the worker + + session_ctx.reset(); + BOOST_TEST_EQ(down, 1); + + ioc.restart(); + while (runs < 2 && ioc.run_one_for(std::chrono::seconds(5)) != 0) + { + } + BOOST_TEST_EQ(runs, 2); + + srv.stop(); + ioc.restart(); + ioc.run(); + srv.join(); + } + + // A connection whose context is torn down, before it starts or + // while it reads, returns its worker, and the server outlives + // that context. + void testSessionTeardownReturnsWorker() + { + for (bool started : {false, true}) + { + int runs = 0, down = 0; + io_context ioc(Backend); + std::optional session_ctx(std::in_place, Backend); + { + session_server srv(ioc, *session_ctx, runs, down); + BOOST_TEST(!srv.bind(endpoint(ipv4_address::loopback(), 0))); + auto ep = srv.local_endpoint(); + srv.start(); + + io_context cctx; + tcp_socket c(cctx); + connect_kept(cctx, c, ep); + // Bounds a hang, not a timing assertion. + while (runs < 1 && + ioc.run_one_for(std::chrono::seconds(5)) != 0) + { + } + BOOST_TEST_EQ(runs, 1); + if (started) + std::ignore = session_ctx->poll(); + BOOST_TEST_EQ(down, 0); + + session_ctx.reset(); + BOOST_TEST_EQ(down, 1); + + srv.stop(); + ioc.restart(); + ioc.run(); + srv.join(); + // No worker is still lent. + BOOST_TEST_NO_THROW(srv.set_workers(make_test_workers(ioc, 1))); + } + BOOST_TEST_EQ(down, 1); + } + } + + // Destroying a connection that never started destroys its task + // before returning the worker, so nothing the task's parameters + // release can race the worker's next connection. + void testUnstartedLaunchKeepsWorkerUntilTaskDestroyed() + { + io_context ioc(Backend); + std::optional session_ctx(std::in_place, Backend); + std::optional held; + holding_server srv(ioc, &held); + BOOST_TEST(!srv.bind(endpoint(ipv4_address::loopback(), 0))); + auto ep = srv.local_endpoint(); + srv.start(); + + io_context cctx; + BOOST_TEST(!connect_result(cctx, ep)); + // Bounds a hang, not a timing assertion. + while (!held && ioc.run_one_for(std::chrono::seconds(5)) != 0) + { + } + BOOST_TEST(held.has_value()); + + struct lent_probe + { + holding_server* srv; + io_context* ioc; + bool* checked; + lent_probe(holding_server& s, io_context& c, bool& b) + : srv(&s) + , ioc(&c) + , checked(&b) + { + } + lent_probe(lent_probe&& o) noexcept + : srv(std::exchange(o.srv, nullptr)) + , ioc(o.ioc) + , checked(o.checked) + { + } + ~lent_probe() + { + if (!srv) + return; + *checked = true; + BOOST_TEST_THROWS( + srv->set_workers(make_test_workers(*ioc, 1)), + std::logic_error); + } + }; + + bool checked = false; + (*held)(session_ctx->get_executor(), [](lent_probe) -> capy::task<> { + co_return; + }(lent_probe(srv, ioc, checked))); + held.reset(); + // With no accept loop left to take it, a returned worker goes + // back to the pool, where set_workers would release it. + srv.stop(); + ioc.restart(); + ioc.run(); + srv.join(); + + session_ctx.reset(); + BOOST_TEST(checked); + } + + // A server owned by a coroutine the context's teardown destroys, + // with its accept loop waiting for the only worker, which is busy + // in a read: every frame goes with the context. + void testTeardownWithAcceptWaitingForWorker() + { + int run_down = 0, owner_down = 0; + io_context cctx; + tcp_socket c1(cctx), c2(cctx); + { + io_context ioc(Backend); + endpoint ep; + int runs = 0; + + class reading_worker : public tcp_server::worker_base + { + corosio::tcp_socket sock_; + io_context& ctx_; + int& runs_; + int& down_; + + public: + reading_worker(io_context& ctx, int& runs, int& down) + : sock_(ctx) + , ctx_(ctx) + , runs_(runs) + , down_(down) + { + } + + corosio::tcp_socket& socket() override + { + return sock_; + } + + void run(tcp_server::launcher launch) override + { + ++runs_; + launch( + ctx_.get_executor(), + [](tcp_socket& s, destroy_count_t) -> capy::task<> { + char c; + std::ignore = co_await s.read_some( + capy::mutable_buffer(&c, 1)); + }(sock_, destroy_count_t(down_))); + } + }; + + class reading_server : public tcp_server + { + public: + reading_server(io_context& ctx, int& runs, int& down) + : tcp_server(ctx, ctx.get_executor()) + { + std::vector> v; + v.push_back( + std::make_unique(ctx, runs, down)); + set_workers(std::move(v)); + } + }; + + capy::run_async(ioc.get_executor())( + [](io_context& ioc, endpoint& ep, int& runs, int& run_down, + destroy_count_t) -> capy::task<> { + reading_server srv(ioc, runs, run_down); + BOOST_TEST( + !srv.bind(endpoint(ipv4_address::loopback(), 0))); + ep = srv.local_endpoint(); + srv.start(); + std::ignore = + co_await corosio::delay(std::chrono::hours(1)); + }(ioc, ep, runs, run_down, destroy_count_t(owner_down))); + std::ignore = ioc.poll(); + BOOST_TEST_NE(ep.port(), 0); + + connect_kept(cctx, c1, ep); + // Bounds a hang, not a timing assertion. + while (runs < 1 && ioc.run_one_for(std::chrono::seconds(5)) != 0) + { + } + BOOST_TEST_EQ(runs, 1); + connect_kept(cctx, c2, ep); + while (ioc.poll() != 0) + { + } + BOOST_TEST_EQ(runs, 1); + BOOST_TEST_EQ(run_down, 0); + BOOST_TEST_EQ(owner_down, 0); + } + BOOST_TEST_EQ(run_down, 1); + BOOST_TEST_EQ(owner_down, 1); + } + // An accept loop holds a worker while it waits for a connection, // so replacing the workers then would free one in use. void testSetWorkersWhileAcceptingThrows() @@ -1496,7 +2183,15 @@ struct tcp_server_test testDestroyWithBusyWorkerClosesListener(); testStopReachesLaunchAfterStop(); testLateLaunchOutlivesInvokingChain(); + testQueuedHopTornDownOnServerContext(); + testQueuedAcceptLoopHopGivenUp(); + testLaunchPostThrows(); testWorkerNotReusedDuringRun(); + testSessionHoldsWorkOnItsContext(); + testSessionTeardownWakesWaitingAccept(); + testSessionTeardownReturnsWorker(); + testTeardownWithAcceptWaitingForWorker(); + testUnstartedLaunchKeepsWorkerUntilTaskDestroyed(); testSetWorkersWhileAcceptingThrows(); testSetWorkersWithBorrowedWorkerThrows(); } diff --git a/test/unit/tcp_socket.cpp b/test/unit/tcp_socket.cpp index cc4a5a14b..f15cac72e 100644 --- a/test/unit/tcp_socket.cpp +++ b/test/unit/tcp_socket.cpp @@ -30,15 +30,20 @@ #include #include +#include #include #include #include #include +#include #include #include #include +#include #include #include +#include +#include #if BOOST_COROSIO_POSIX #include // getpid() @@ -2012,16 +2017,15 @@ struct tcp_socket_test testAssignFailureKeepsSocket(); testAssignOverOpenKeepsPending(); testCompletionResumesOnAwaitersContext(); + testCrossContextCompletionOutlivesItsTarget(); + testReadStopRacingTeardown(); testRelease(); testReleaseClosedThrows(); testAssignV6(); -#if !COROSIO_TEST_HAS_ASAN - // Abandon parked coroutine frames by design; see context.hpp. testDestroyWithQueuedWrite(); testDestroyWithQueuedConnect(); testDestroyWithQueuedWait(); -#endif } void testConnectV6() @@ -2588,6 +2592,92 @@ struct tcp_socket_test BOOST_TEST(on_b); } + // A stop requested on another thread can claim a parked read while + // the context tears its sockets down; the drain must still find the + // read the stop posts. + void testReadStopRacingTeardown() + { + for (int round = 0; round < 25; ++round) + { + std::vector stops(8); + std::atomic go{false}; + std::thread stopper; + { + io_context ioc(Backend); + // Paired first: each pairing runs the context, which a + // parked read would keep from returning. + std::vector> pairs; + for (std::size_t i = 0; i < stops.size(); ++i) + { + pairs.push_back(test::make_socket_pair< + tcp_socket, tcp_acceptor, false>(ioc)); + ioc.restart(); + } + for (std::size_t i = 0; i < stops.size(); ++i) + capy::run_async(ioc.get_executor(), stops[i].get_token())( + [](tcp_socket s, tcp_socket) -> capy::task<> { + char c; + std::ignore = co_await s.read_some( + capy::mutable_buffer(&c, 1)); + }(std::move(pairs[i].first), + std::move(pairs[i].second))); + std::ignore = ioc.poll(); // parks every read + stopper = std::thread([&] { + while (!go.load(std::memory_order_acquire)) + { + } + for (auto& src : stops) + src.request_stop(); + }); + go.store(true, std::memory_order_release); + } + stopper.join(); + } + BOOST_TEST_PASS(); + } + + // An operation of a long-lived context completes into a short-lived + // one, which finishes the chain and is destroyed at once: the + // completing thread must be done with the short-lived context's + // scheduler before that can happen. + void testCrossContextCompletionOutlivesItsTarget() + { + io_context a(Backend); + std::vector> pairs; + for (int i = 0; i < 100; ++i) + { + pairs.push_back( + test::make_socket_pair(a)); + a.restart(); + } + auto aex = a.get_executor(); + aex.on_work_started(); + std::thread ta([&] { a.run(); }); + + char const c = 'x'; + for (auto& [s1, s2] : pairs) + { + auto b = std::make_unique(Backend, 2); + char buf[1]; + capy::run_async(b->get_executor())( + [](tcp_socket& s, char* p) -> capy::task<> { + std::ignore = + co_await s.read_some(capy::mutable_buffer(p, 1)); + }(s1, buf)); + std::thread b2([&] { b->run(); }); + capy::run_async(aex)( + [](tcp_socket& s, char const* p) -> capy::task<> { + std::ignore = + co_await s.write_some(capy::const_buffer(p, 1)); + }(s2, &c)); + b->run(); + b2.join(); + b.reset(); + } + aex.on_work_finished(); + ta.join(); + } + // release() hands ownership to the caller only after pending // operations have been cancelled, and the descriptor is still // connected to the peer. diff --git a/test/unit/teardown_destroy.cpp b/test/unit/teardown_destroy.cpp new file mode 100644 index 000000000..a063ed4e7 --- /dev/null +++ b/test/unit/teardown_destroy.cpp @@ -0,0 +1,515 @@ +// +// Copyright (c) 2026 Steve Gerbino +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// Official repository: https://github.com/cppalliance/corosio +// + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "context.hpp" +#include "pool_teardown.hpp" +#include "temp_path.hpp" +#include "test_suite.hpp" + +namespace boost::corosio { + +// Tearing down a context destroys every frame suspended in one of its +// operations, and with it the chain awaiting that frame, exactly once. +template +struct teardown_destroy_test +{ + // Frames from the recycling allocator are never really freed, which + // hides a use-after-free from ASan; these go through operator new. + static auto spawn(io_context& ioc) + { + return capy::run_async( + ioc.get_executor(), std::allocator{}); + } + + struct destroy_count_t + { + int* n; + explicit destroy_count_t(int& c) : n(&c) {} + destroy_count_t(destroy_count_t&& o) noexcept + : n(std::exchange(o.n, nullptr)) {} + ~destroy_count_t() + { + if (n) + ++*n; + } + }; + + void testParkedReadDestroysChain() + { + int child_down = 0, parent_down = 0; + { + io_context ioc(Backend); + auto [a, b] = test::make_socket_pair(ioc); + auto child = [](tcp_socket& s, destroy_count_t) -> capy::task<> { + char c; + std::ignore = co_await s.read_some(capy::mutable_buffer(&c, 1)); + }; + auto parent = [&](tcp_socket& s, destroy_count_t) -> capy::task<> { + co_await child(s, destroy_count_t(child_down)); + }; + spawn(ioc)( + parent(a, destroy_count_t(parent_down))); + std::ignore = ioc.poll(); + BOOST_TEST_EQ(child_down, 0); + BOOST_TEST_EQ(parent_down, 0); + } + BOOST_TEST_EQ(child_down, 1); + BOOST_TEST_EQ(parent_down, 1); + } + + // The frame owns the socket whose impl stores the parked op, so + // destroying the frame releases that impl while the op is still + // inside its own teardown. + void testParkedReadOwningSocketDestroysChain() + { + int down = 0; + { + io_context ioc(Backend); + auto [a, b] = test::make_socket_pair(ioc); + auto owner = [](tcp_socket s, destroy_count_t) -> capy::task<> { + char c; + std::ignore = co_await s.read_some(capy::mutable_buffer(&c, 1)); + }; + spawn(ioc)( + owner(std::move(a), destroy_count_t(down))); + std::ignore = ioc.poll(); + BOOST_TEST_EQ(down, 0); + } + BOOST_TEST_EQ(down, 1); + } + + void testParkedAcceptDestroysChain() + { + int parent_down = 0; + { + io_context ioc(Backend); + tcp_acceptor acc(ioc); + BOOST_TEST(!acc.open()); + acc.set_option(socket_option::reuse_address(true)); + BOOST_TEST(!acc.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!acc.listen()); + auto parent = [](tcp_acceptor& a, destroy_count_t) -> capy::task<> { + std::ignore = co_await a.accept(); + }; + spawn(ioc)( + parent(acc, destroy_count_t(parent_down))); + std::ignore = ioc.poll(); + BOOST_TEST_EQ(parent_down, 0); + } + BOOST_TEST_EQ(parent_down, 1); + } + + void testParkedSignalWaitDestroysChain() + { + int parent_down = 0; + { + io_context ioc(Backend); + signal_set sigs(ioc, SIGINT); + auto parent = [](signal_set& s, destroy_count_t) -> capy::task<> { + std::ignore = co_await s.wait(); + }; + spawn(ioc)( + parent(sigs, destroy_count_t(parent_down))); + std::ignore = ioc.poll(); + BOOST_TEST_EQ(parent_down, 0); + } + BOOST_TEST_EQ(parent_down, 1); + } + + // The frame owns the set whose impl stores the parked wait. + void testParkedSignalWaitOwningSetDestroysChain() + { + int down = 0; + { + io_context ioc(Backend); + auto owner = [](signal_set s, destroy_count_t) -> capy::task<> { + std::ignore = co_await s.wait(); + }; + spawn(ioc)( + owner(signal_set(ioc, SIGINT), destroy_count_t(down))); + std::ignore = ioc.poll(); + BOOST_TEST_EQ(down, 0); + } + BOOST_TEST_EQ(down, 1); + } + +#if BOOST_COROSIO_POSIX + // The read waits behind a parked worker, so at teardown it is still + // queued on the blocking pool or its completion is still queued on + // the scheduler; either way its frame is destroyed. + void testQueuedFileReadDestroysChain() + { + int parent_down = 0; + test::temp_file tmp("teardown_destroy_", "hello world"); + test::pool_blocker blocker; + { + io_context ioc(Backend); + BOOST_TEST(test::park_pool_worker(ioc, blocker)); + random_access_file f(ioc); + BOOST_TEST(!f.open(tmp.path, file_base::read_only)); + auto parent = [](random_access_file& rf, + destroy_count_t) -> capy::task<> { + char c; + std::ignore = + co_await rf.read_some_at(0, capy::mutable_buffer(&c, 1)); + }; + spawn(ioc)( + parent(f, destroy_count_t(parent_down))); + std::ignore = ioc.poll(); + } + BOOST_TEST_EQ(parent_down, 1); + } +#endif + + // Each operation still queued on the pool when it shuts down is + // abandoned there, before the context goes, and its blocking call + // never runs. + template + void checkPoolShutdownDestroys(Start start) + { + int parent_down = 0; + test::pool_blocker blocker; + io_context ioc(Backend); + BOOST_TEST(test::park_pool_worker(ioc, blocker)); + start(ioc, destroy_count_t(parent_down)); + std::ignore = ioc.poll(); + BOOST_TEST_EQ(parent_down, 0); + test::shutdown_pool_behind(ioc, blocker); + // The pool abandons the chain; its unwind runs on the context. + ioc.restart(); + std::ignore = ioc.poll(); + BOOST_TEST_EQ(parent_down, 1); + } + + void testPoolShutdownDestroysQueuedOps() + { + // IOCP reads files and resolves names without the pool; only + // its reverse resolve uses it. +#if BOOST_COROSIO_POSIX + test::temp_file tmp("teardown_destroy_", "hello world"); + +#if BOOST_COROSIO_HAS_URING + constexpr bool on_ring = + std::is_same_v, uring_t>; +#else + constexpr bool on_ring = false; +#endif + // io_uring reads files through the ring, not the pool. + if constexpr (!on_ring) + { + checkPoolShutdownDestroys([&](io_context& ioc, destroy_count_t d) { + auto op = [](random_access_file f, + destroy_count_t) -> capy::task<> { + char c; + std::ignore = + co_await f.read_some_at(0, capy::mutable_buffer(&c, 1)); + }; + random_access_file f(ioc); + BOOST_TEST(!f.open(tmp.path, file_base::read_only)); + spawn(ioc)( + op(std::move(f), std::move(d))); + }); + checkPoolShutdownDestroys([&](io_context& ioc, destroy_count_t d) { + auto op = [](stream_file f, destroy_count_t) -> capy::task<> { + char c; + std::ignore = + co_await f.read_some(capy::mutable_buffer(&c, 1)); + }; + stream_file f(ioc); + BOOST_TEST(!f.open(tmp.path, file_base::read_only)); + spawn(ioc)( + op(std::move(f), std::move(d))); + }); + } + checkPoolShutdownDestroys([](io_context& ioc, destroy_count_t d) { + auto op = [](resolver r, destroy_count_t) -> capy::task<> { + std::ignore = co_await r.resolve( + "127.0.0.1", "80", + resolve_flags::numeric_host | + resolve_flags::numeric_service); + }; + spawn(ioc)( + op(resolver(ioc), std::move(d))); + }); +#endif + checkPoolShutdownDestroys([](io_context& ioc, destroy_count_t d) { + auto op = [](resolver r, destroy_count_t) -> capy::task<> { + std::ignore = co_await r.resolve( + endpoint(ipv4_address::loopback(), 80), + reverse_flags::numeric_host | + reverse_flags::numeric_service); + }; + spawn(ioc)( + op(resolver(ioc), std::move(d))); + }); + } + +#if BOOST_COROSIO_POSIX + // A blocking-pool operation is work on the context that owns it, + // whichever context the awaiting coroutine runs on. + void testPoolOpIsWorkOnItsOwnContext() + { + test::pool_blocker blocker; + io_context a(Backend); + io_context b(Backend); + BOOST_TEST(test::park_pool_worker(a, blocker)); + resolver r(a); + bool done = false; + auto op = [](resolver& rs, bool& d) -> capy::task<> { + std::ignore = co_await rs.resolve( + "127.0.0.1", "80", + resolve_flags::numeric_host | + resolve_flags::numeric_service); + d = true; + }; + spawn(b)(op(r, done)); + std::ignore = b.poll(); + std::ignore = a.poll(); + BOOST_TEST(!a.stopped()); + if (a.stopped()) + a.restart(); + blocker.release(); + std::ignore = a.run(); + std::ignore = b.run(); + BOOST_TEST(done); + } +#endif + +#if BOOST_COROSIO_POSIX + // A session runs on another context, as the launcher allows, and + // awaits a blocking-pool operation of the server's context. + struct pool_session_worker : tcp_server::worker_base + { + tcp_socket sock_; + io_context& session_ctx_; + bool use_file_; + random_access_file* file_; + resolver* resolver_; + int& down_; + bool& launched_; + + pool_session_worker( + io_context& server_ctx, + io_context& session_ctx, + bool use_file, + random_access_file* file, + resolver* r, + int& down, + bool& launched) + : sock_(server_ctx) + , session_ctx_(session_ctx) + , use_file_(use_file) + , file_(file) + , resolver_(r) + , down_(down) + , launched_(launched) + { + } + + tcp_socket& socket() override + { + return sock_; + } + + void run(tcp_server::launcher launch) override + { + launched_ = true; + if (use_file_) + launch( + session_ctx_.get_executor(), + [](random_access_file f, destroy_count_t) -> capy::task<> { + char c; + std::ignore = co_await f.read_some_at( + 0, capy::mutable_buffer(&c, 1)); + }(std::move(*file_), destroy_count_t(down_))); + else + launch( + session_ctx_.get_executor(), + [](resolver r, destroy_count_t) -> capy::task<> { + std::ignore = co_await r.resolve( + "127.0.0.1", "80", + resolve_flags::numeric_host | + resolve_flags::numeric_service); + }(std::move(*resolver_), destroy_count_t(down_))); + } + }; + + struct pool_session_server : tcp_server + { + pool_session_server( + io_context& server_ctx, + io_context& session_ctx, + bool use_file, + random_access_file* file, + resolver* r, + int& down, + bool& launched) + : tcp_server(server_ctx, server_ctx.get_executor()) + { + std::vector> v; + v.push_back(std::make_unique( + server_ctx, session_ctx, use_file, file, r, down, launched)); + set_workers(std::move(v)); + } + }; + + // Neither context may be destroyed while such a session waits: the + // unwind of a context's teardown runs on the session's context and + // would outlive the server's. Stopping the server cancels the + // operation, and running both contexts ends the session first. + void checkCrossContextSessionDrains(bool use_file) + { + int down = 0; + bool launched = false; + test::temp_file tmp("teardown_destroy_", "hello world"); + test::pool_blocker blocker; + auto a = std::make_unique(Backend); + auto b = std::make_unique(Backend); + BOOST_TEST(test::park_pool_worker(*a, blocker)); + std::optional f(std::in_place, *a); + BOOST_TEST(!f->open(tmp.path, file_base::read_only)); + std::optional r(std::in_place, *a); + { + pool_session_server srv(*a, *b, use_file, &*f, &*r, down, launched); + BOOST_TEST(!srv.bind(endpoint(ipv4_address::loopback(), 0))); + auto ep = srv.local_endpoint(); + srv.start(); + tcp_socket c(*a); + spawn(*a)([](tcp_socket& s, endpoint e) -> capy::task<> { + std::ignore = co_await s.connect(e); + }(c, ep)); + while (!launched && a->run_one_for(std::chrono::seconds(5)) != 0) + { + } + BOOST_TEST(launched); + std::ignore = b->poll(); + std::ignore = a->poll(); + BOOST_TEST_EQ(down, 0); + + srv.stop(); + blocker.release(); + // Bounds a hang, not a timing assertion. + for (int i = 0; down == 0 && i < 500; ++i) + { + a->restart(); + b->restart(); + std::ignore = a->run_one_for(std::chrono::milliseconds(10)); + std::ignore = b->poll(); + } + BOOST_TEST_EQ(down, 1); + a->restart(); + std::ignore = a->poll(); + } + f.reset(); + r.reset(); + a.reset(); + b.reset(); + } + + void testCrossContextSessionDrains() + { +#if BOOST_COROSIO_HAS_URING + constexpr bool on_ring = + std::is_same_v, uring_t>; +#else + constexpr bool on_ring = false; +#endif + // io_uring reads files through the ring, not the pool. + if constexpr (!on_ring) + checkCrossContextSessionDrains(true); + checkCrossContextSessionDrains(false); + } +#endif + + // Posts from a service's destructor, after the teardown drain. + struct late_poster : capy::execution_context::service + { + io_context::executor_type ex; + capy::continuation* c; + + late_poster( + capy::execution_context&, + io_context::executor_type e, + capy::continuation& cc) + : ex(e) + , c(&cc) + { + } + + ~late_poster() + { + ex.post(*c); + } + + void shutdown() override {} + }; + + // Work posted after the drain is destroyed with the context, not + // left queued. + void testPostAfterDrainDestroyed() + { + int down = 0; + auto probe = [](destroy_count_t) -> capy::task<> { co_return; }; + auto t = probe(destroy_count_t(down)); + capy::continuation c{t.handle()}; + t.release(); + { + io_context ioc(Backend); + ioc.make_service(ioc.get_executor(), c); + } + BOOST_TEST_EQ(down, 1); + } + + void run() + { + testParkedReadDestroysChain(); + testParkedReadOwningSocketDestroysChain(); + testParkedAcceptDestroysChain(); + testParkedSignalWaitDestroysChain(); + testParkedSignalWaitOwningSetDestroysChain(); +#if BOOST_COROSIO_POSIX + testQueuedFileReadDestroysChain(); +#endif + testPoolShutdownDestroysQueuedOps(); +#if BOOST_COROSIO_POSIX + testPoolOpIsWorkOnItsOwnContext(); + testCrossContextSessionDrains(); +#endif + testPostAfterDrainDestroyed(); + } +}; + +COROSIO_BACKEND_TESTS(teardown_destroy_test, "boost.corosio.teardown_destroy") + +} // namespace boost::corosio diff --git a/test/unit/teardown_inflight.cpp b/test/unit/teardown_inflight.cpp index efb5d9378..5fcd88dd1 100644 --- a/test/unit/teardown_inflight.cpp +++ b/test/unit/teardown_inflight.cpp @@ -178,9 +178,6 @@ struct uring_teardown_test BOOST_TEST(!!wec); } -#if !COROSIO_TEST_HAS_ASAN - // These abandon parked coroutine frames by design; see context.hpp. - void testDestroyWithPendingSocketWrite() { bool resumed = false; @@ -188,7 +185,7 @@ struct uring_teardown_test io_context ioc(uring); // The pair is made outside the coroutine (it drains the // context internally); the socket moves into the frame so - // its fd stays open when the frame is abandoned. + // its fd stays open when the frame is destroyed. auto [s1, s2] = test::make_socket_pair(ioc); fill_fd(static_cast(s1.native_handle())); @@ -527,14 +524,12 @@ struct uring_teardown_test } BOOST_TEST(!resumed); } -#endif // !COROSIO_TEST_HAS_ASAN void run() { testAcceptorWaitAfterClose(); testAcceptorWaitStopRequested(); testAcceptorCloseWithParkedWait(); -#if !COROSIO_TEST_HAS_ASAN testDestroyWithPendingSocketWrite(); testDestroyWithPendingDatagramSend(); testDestroyWithPendingDescriptorOps(); @@ -545,7 +540,6 @@ struct uring_teardown_test testDestroyWithQueuedDescriptorOps(); testDestroyWithQueuedRandomAccessOps(); testDestroyWithParkedLocalAccept(); -#endif } }; diff --git a/test/unit/test/mocket.cpp b/test/unit/test/mocket.cpp index c289c63cb..0aa1a01a6 100644 --- a/test/unit/test/mocket.cpp +++ b/test/unit/test/mocket.cpp @@ -21,6 +21,8 @@ #include #include +#include + #include "context.hpp" #include "test_suite.hpp" @@ -222,6 +224,29 @@ struct mocket_test peer.close(); } + // The pair's setup runs on the context, so a context stopped + // beforehand must not leave it queued after the call returns. + void testStoppedContext() + { + io_context ioc; + capy::test::fuse f; + ioc.stop(); + + bool made = false; + try + { + auto [m, peer] = make_mocket_pair(ioc, f); + made = m.is_open() && peer.is_open(); + } + catch (std::runtime_error const&) + { + } + BOOST_TEST(made); + + ioc.restart(); + ioc.run(); + } + void run() { testProvideExpect(); @@ -230,6 +255,7 @@ struct mocket_test testPassthrough(); testExpectShorterThanWrite(); testPreStoppedTokenShortCircuits(); + testStoppedContext(); } }; diff --git a/test/unit/test/socket_pair.cpp b/test/unit/test/socket_pair.cpp index 4b00a7a06..a2c8f397f 100644 --- a/test/unit/test/socket_pair.cpp +++ b/test/unit/test/socket_pair.cpp @@ -19,6 +19,8 @@ #include #include +#include + #include "context.hpp" #include "test_suite.hpp" @@ -77,10 +79,33 @@ struct socket_pair_test s2.close(); } + // The pair's setup runs on the context, so a context stopped + // beforehand must not leave it queued after the call returns. + void testStoppedContext() + { + io_context ioc; + ioc.stop(); + + bool made = false; + try + { + auto [s1, s2] = make_socket_pair(ioc); + made = s1.is_open() && s2.is_open(); + } + catch (std::runtime_error const&) + { + } + BOOST_TEST(made); + + ioc.restart(); + ioc.run(); + } + void run() { testCreate(); testBidirectional(); + testStoppedContext(); } }; diff --git a/test/unit/thread_pool.cpp b/test/unit/thread_pool.cpp index 37113efec..ed4be1157 100644 --- a/test/unit/thread_pool.cpp +++ b/test/unit/thread_pool.cpp @@ -13,6 +13,7 @@ #include #include +#include #include #if defined(__linux__) @@ -89,7 +90,7 @@ struct thread_pool_test BOOST_TEST(!pool.post(&items[i])); } - // Shutdown should drain all queued tasks + // Items with no destroy function run, by a worker or by shutdown() pool.shutdown(); BOOST_TEST(counter.load() == n); @@ -211,12 +212,97 @@ struct thread_pool_test } } + // Shutdown takes no further items off the queue, so every one + // has to be running before it is asked. + while (arrived.load() < num_threads) + std::this_thread::yield(); pool.shutdown(); // All items completed — proves all 4 threads ran concurrently BOOST_TEST(done.load() == static_cast(num_threads)); } + // An item still queued at shutdown is destroyed, never run; one + // without a destroy function is run by shutdown() instead. + void testShutdownDestroysQueuedWork() + { + struct gate_work : detail::pool_work_item + { + std::atomic running{false}; + std::atomic released{false}; + + static void execute(detail::pool_work_item* p) noexcept + { + auto* self = static_cast(p); + self->running.store(true); + while (!self->released.load()) + std::this_thread::yield(); + } + }; + + struct counted_work : detail::pool_work_item + { + std::atomic ran{0}; + std::atomic destroyed{0}; + + static void execute(detail::pool_work_item* p) noexcept + { + static_cast(p)->ran.fetch_add(1); + } + + static void discard(detail::pool_work_item* p) noexcept + { + static_cast(p)->destroyed.fetch_add(1); + } + }; + + io_context ioc; + detail::thread_pool pool(ioc, 1); + + gate_work gate; + gate.func_ = &gate_work::execute; + counted_work queued; + queued.func_ = &counted_work::execute; + queued.destroy_ = &counted_work::discard; + test_work plain; + std::atomic plain_ran{0}; + plain.counter = &plain_ran; + plain.func_ = &test_work::execute; + + BOOST_TEST(!pool.post(&gate)); + while (!gate.running.load()) + std::this_thread::yield(); + BOOST_TEST(!pool.post(&queued)); + BOOST_TEST(!pool.post(&plain)); + + std::thread stopper([&] { pool.shutdown(); }); + + // A refused post is the only sign that shutdown has begun; the + // probes it accepts before then queue behind the others. + std::list probes; + for (;;) + { + auto& probe = probes.emplace_back(); + probe.func_ = &counted_work::execute; + probe.destroy_ = &counted_work::discard; + if (pool.post(&probe)) + break; + std::this_thread::yield(); + } + gate.released.store(true); + stopper.join(); + + BOOST_TEST_EQ(queued.ran.load(), 0); + BOOST_TEST_EQ(queued.destroyed.load(), 1); + BOOST_TEST_EQ(plain_ran.load(), 1); + probes.pop_back(); + for (auto& probe : probes) + { + BOOST_TEST_EQ(probe.ran.load(), 0); + BOOST_TEST_EQ(probe.destroyed.load(), 1); + } + } + void testLazyWorkers() { #if defined(__linux__) @@ -257,7 +343,8 @@ struct thread_pool_test // A pool created outside the context's service registry (so the // context's shutdown walk never reaches it) must join its worker and - // drain queued work from the destructor net when destroyed directly. + // dispose of queued work from the destructor net when destroyed + // directly. void testDestructorShutsDown() { io_context ioc; @@ -281,6 +368,7 @@ struct thread_pool_test testPostAfterShutdown(); testZeroThreads(); testMultipleThreads(); + testShutdownDestroysQueuedWork(); testLazyWorkers(); testDestructorShutsDown(); } diff --git a/test/unit/udp_socket.cpp b/test/unit/udp_socket.cpp index 76f985b3e..c0a3dc296 100644 --- a/test/unit/udp_socket.cpp +++ b/test/unit/udp_socket.cpp @@ -2092,10 +2092,7 @@ struct udp_socket_test testReleaseClosedThrows(); testAssignV6(); -#if !COROSIO_TEST_HAS_ASAN - // Abandon parked coroutine frames by design; see context.hpp. testDestroyWithQueuedDatagrams(); -#endif } }; From dbb0bed21a1ac524a681cf5c2f18b3bd4dcca50a Mon Sep 17 00:00:00 2001 From: Steve Gerbino Date: Thu, 15 Oct 2026 18:57:00 +0200 Subject: [PATCH 8/8] docs: add a lifetime and teardown page One page now says how long I/O objects, buffers and contexts must live, how to stop work, what destroying a context does to coroutines still suspended in it, why work must not cross contexts at teardown, and what happens when an executor throws as an operation completes. It ends with a checklist for code that suspends and resumes coroutines, covering the ways such code can break teardown. --- doc/design/lifetime-contract.md | 258 ++++++++++++++++++ doc/modules/ROOT/nav.adoc | 1 + .../ROOT/pages/4.guide/4t.lifetime.adoc | 133 +++++++++ 3 files changed, 392 insertions(+) create mode 100644 doc/design/lifetime-contract.md create mode 100644 doc/modules/ROOT/pages/4.guide/4t.lifetime.adoc diff --git a/doc/design/lifetime-contract.md b/doc/design/lifetime-contract.md new file mode 100644 index 000000000..3d04e2b4a --- /dev/null +++ b/doc/design/lifetime-contract.md @@ -0,0 +1,258 @@ +# Corosio Implementation Lifetime Contract + +When an I/O implementation may be touched, by whom, and when its +memory may be reused or freed. + +## 1. The Invariant + +Every I/O object (socket, acceptor, file, descriptor, Windows handle, +resolver, signal set, timer) is a handle to an *implementation* (`io_object::implementation`) owned +by its service's `object_pool`. The implementation carries an +intrusive count, `refs_`. + +> **Nothing touches an implementation after its count reaches zero, +> and nothing raises a count that has reached zero.** + +At zero, `retire()` runs and the pool recycles the implementation: +its memory is handed to the next `construct()`. A late access does +not crash; it corrupts an unrelated object. The invariant has no +benign violations. + +Debug builds on the reactor backends poison recycled state, so a field +`reuse()` fails to reset reads as garbage. Elsewhere `reuse()` asserts +the fields the contract guarantees. Release builds rely on the +contract alone. + +## 2. Who Holds a Reference + +Anything that can reach an implementation holds an `object_ref` for +exactly as long as it can reach it. The holders are: + +| Holder | Taken | Released | +|---|---|---| +| The handle | `construct()` (the count starts at 1) | `destroy()` | +| An in-flight operation | before the op is handed to the kernel, a thread or a queue | by its completion, after the last touch of the impl | +| A reactor registration | when the descriptor can produce events | when no reactor pass can still deliver one (§5.1) | +| A kernel-owned multishot op | when armed | on its terminal completion (§5.2) | +| A teardown pin | `object_pool::shutdown()` | after every visitor has run, outside the pool lock | +| A cross-thread work item (resolver, file) | before the hand-off | moved into the completion op, never dropped on the worker | +| An accept's peer | `construct()`: in the initiator before anything is armed (IOCP), or in the completion once the kernel returns the descriptor (reactor, io_uring) | passed to the caller's handle on success, `destroy()`ed on failure or drain | + +An op that is merely embedded in the implementation (`rd_`, `wr_`, +`conn_`) is part of it, not a separate lifetime. The *reference* it +carries while in flight is what matters. + +## 3. Acquiring a Reference + +A reference may be created in exactly two ways: + +1. **From a held reference.** Copying an `object_ref`, or + `object_ref(this)` inside a member reached through a handle or an + op that already holds one. The count is at least one and cannot + reach zero concurrently. +2. **From a raw pointer, with `try_acquire`.** A context that reached + the implementation through a raw pointer holds no reference, and + the count may already be zero. That covers a registry walk, a + kernel completion naming it, or an OS callback. Such a context + must use `try_acquire`, which refuses to revive a zero count, and + must give up that path when it fails. + +`object_ref(p)` from a raw-pointer context is the one way to break +the invariant's second half. Every revival bug found so far was this +pattern. + +## 4. Releasing a Reference + +- The last touch of an implementation precedes the release that may + retire it. Read what you need (scheduler pointer, continuation, + executor) into locals under the lock, then release. +- A release that may reach zero runs with no lock held that + `retire()` → `recycle()` could need. In practice that means no + lock at all: recycle takes the pool mutex, and pool shutdown takes + service and scheduler locks after it. +- A completion must not write implementation state that close resets + (endpoints, `fd`) once the implementation was closed underneath it. + Close, cancel and release flag every op they reach before touching + the descriptor, and a flagged completion writes nothing. That covers + calls serialized with the completion; a close concurrent with one is + outside the objects' thread-safety contract. + +## 5. Raw Pointers Outside the Process's Control + +Some pointers are held where an `object_ref` cannot live: in the +kernel, in an OS thread pool, in an executor's queue. Each needs an +explicit owner of the reference that stands in for it. + +### 5.1 Reactor registrations (epoll, kqueue, select) + +`epoll_event.data.ptr` and the select snapshot name a descriptor +state with no reference. A reactor pass that has already read an +event can deliver it after another thread closes the socket. + +Rule: **the reactor holds a reference for as long as a pass can +deliver an event for the descriptor.** While the descriptor is +registered, the handle's reference serves: every path that releases a +handle's last reference deregisters first. Deregistration passes a +reference to the scheduler. If a pass is running, the scheduler parks +it until that pass ends. Then, or at once if no pass is running, it +moves to the queued invocation if the descriptor is queued, and is +released otherwise. + +Registration publishes the descriptor to the kernel only after its +state (fd, scheduler, mutex mode) is initialized. Fields a queued +invocation reads change only under the descriptor mutex. + +Sockets, acceptors and posix descriptors share one implementation of +this protocol (`reactor_io_core`), so the rule is stated once. + +### 5.2 io_uring operations + +`user_data` names an op. Ordinary ops carry their impl reference +until their completion handler runs. + +Rule for multishot ops: **a multishot op is owned by the kernel +while armed.** It holds the implementation reference. Its terminal +completion (no `IORING_CQE_F_MORE`) is dispatched like any other op, +and its handler either re-arms it or returns it to the owner's free +list and drops that reference. The owner remembers the live arming by +address only (`armed_`); deliveries from any other arming are stale +and closed on arrival. Close disowns the arming, then cancels it. An +arming is admitted under the ring mutex, re-checking that the owner is +not closing, so it either precedes the cancel in the submission queue +or is refused. + +### 5.3 IOCP and OS callbacks + +An `OVERLAPPED` inside an op is held by the kernel until its packet is +dequeued; the op's reference covers that. Callbacks on OS threads +(`GetAddrInfoExW`) take their own reference and are counted, so that +teardown can wait for them (§6). + +Once an initiator hands a pending op to the scheduler +(`on_pending`), another thread may complete it and recycle the +implementation, so every touch of the op or the implementation, +including the post-submit cancellation re-check, comes first. + +The object handle's thread-pool wait callback reaches its state by raw +pointer but only claims an armed wait, whose op holds the reference; +every rundown drains queued callbacks before claiming for +cancellation, so no callback outlives the reference that covers it. +A wait engages its stop callback before it is published as armed. + +### 5.4 Executor queues + +A continuation handed to an executor is held by reference until the +executor dequeues it. It must live in storage that outlives that +window. The awaiting coroutine's frame does; an op's embedded storage +does not, because the op is consumed when its completion is. + +So every awaitable owns its continuation and passes it to the +implementation; ops only point at it. The continuation carries the +owner of the awaiting coroutine's chain (`env->owner`), so a holder +that gives it up at teardown passes it to `capy::abandon`, which +unwinds the chain from its owner. An op embedded in an +implementation that is queued in the scheduler (rather than handed to +the kernel) still holds a reference, like any other in-flight op. + +### 5.5 Descriptor numbers + +A descriptor number outlives the object that closed it only as a +number, and the process may hand it to the next open. Anything that +names an implementation's descriptor by number after a close could +reach an unrelated file: + +- **Pool workers** (posix files) enter a gate around each system call; + close defers the `::close` to the last worker out. +- **io_uring** cancels an object's ops by their user data, never by + descriptor, so the cancel needs no number. A close or release first + flushes the submission queue so no queued SQE still names the + descriptor; if the flush fails, the queued SQEs that name the + descriptor become NOPs and their ops complete cancelled. A cancel that cannot be queued is + deferred by op and queued ahead of the next op's SQE, which may reuse + the address. +- **`SO_ERROR` probes** for error waits are skipped once the op is + flagged cancelled on io_uring and the IOCP wait reactor, since + reading the option clears it. The IOCP wait reactor probes under the + lock close takes before `closesocket`. The reactor backends probe + under the descriptor mutex, and close clears the descriptor's number + under that mutex before closing it, so a probe never reads a reused + number. + +## 6. Teardown + +`io_context` destruction runs every service's `shutdown()` newest +first, then deletes the services in the same order. Services created +after the scheduler come before it; the timer service, which the +scheduler's constructor creates, comes after it. The contract for that +sequence: + +0. **Timers first.** Every scheduler shuts the timer service down + before it drains, so work a timer's teardown posts is drained too. +1. **Cancel.** Each service cancels or closes every live + implementation. Its pool's `shutdown()` pins each one with + `try_acquire` and switches the pool to delete-on-zero. +2. **Quiesce.** Before any service is destroyed, every agent outside + the process's control must have stopped reaching implementations: + - io_uring: cancel everything in the kernel and wait until every + submitted op has completed; + - IOCP: drain every outstanding packet (`pending_io_`), stop the + wait reactor, and wait for counted OS callbacks; + - reactors: no pass is running, which holds because no thread is + inside `run()` when the context is destroyed, and every queued + descriptor has been drained; + - thread pools: joined. A queued item that owns a suspended + coroutine is not run; its `destroy_` abandons the coroutine. An + item without one runs its `func_` on the shutting-down thread. +3. **Drain.** A reactor scheduler first settles the registration + references parked by `retire_descriptor`. The scheduler then + destroys every queued op without resuming it, which releases the + references those ops hold. Destroying an op + abandons the coroutine it holds, and the owners those abandons post + land in the same queue. So every chain started on this context is + unwound during the drain, and the handles in its frames release + their references. +4. **Free.** `~object_pool` deletes whatever remains. After step 3 a + remaining reference can belong only to a handle the application + still holds, or to a frame of a chain started on another context. + The contract excludes both: I/O objects are destroyed before their + context, and work that crosses contexts is finished before either + is destroyed. The count is pinned before each delete, so references + an implementation holds on itself cannot re-enter `retire()`. +5. **Late work.** Work posted after the drain, for example from a + service's destructor, is destroyed when the scheduler is deleted, + after every service created after it. + +A step that cannot complete must not be skipped silently. For +example, a cancel that cannot be submitted is the one case where +io_uring does not wait, because nothing could end the wait. + +## 7. Recycling + +- `reuse()` restores every field the next owner could observe. On the + io_uring, file and IOCP backends, fields the contract guarantees are + already reset (descriptor closed, no op in flight, not queued) are + asserted rather than reset, so a contract violation fails in debug + instead of being masked. The reactor backends reset the descriptor + fields and assert only the reference count and the queued flag. +- State that a late completion can legitimately write (§4, last + bullet) is reset, not asserted. +- Free lists owned by an implementation (op pools, accept nodes) may + survive recycling. Their entries hold no reference. + +## 8. Conformance + +How the code meets each rule: + +| Rule | Status | +|---|---| +| §2 op references | Conforms on all backends, including a signal set's queued completion. | +| §3 raw-pointer acquisition | Conforms. `object_pool::shutdown` uses `try_acquire`; the uring re-arm keeps the reference its arming already held. | +| §4 release ordering | Conforms after the review fixes (`invoke_deferred_io`, posts after unlock). | +| §4 completion writes | Conforms for serialized close on all backends: close, cancel and release flag the connect op, and `reuse()` resets the endpoints. | +| §5.1 reactor | Conforms, sockets, acceptors and posix descriptors alike (`reactor_io_core::abandon_all`). Close deregisters, then hands its reference to `retire_descriptor`; whether the descriptor is queued is consulted only in `hand_off`. | +| §5.2 multishot | Conforms. The fault harness still fakes terminal completions, so it swallows the real one that follows. | +| §5.3 IOCP | Conforms: socket and acceptor initiators re-check cancellation before `on_pending`, as the handle initiators do. IOCP is checked only by Windows CI. | +| §5.5 descriptor numbers | Conforms: posix file gate, uring flush before close or release, cancelled waits skip the probe on uring and the IOCP wait reactor. | +| §5.4 executor queues | Conforms: every awaitable owns its continuation; no op embeds one. | +| §6 teardown | Conforms on all backends, timers first everywhere; implemented per backend with no shared statement. | +| §7 recycling | Mixed. Some `reuse()` methods assert fields others reset. | diff --git a/doc/modules/ROOT/nav.adoc b/doc/modules/ROOT/nav.adoc index 57869fad6..58d9e724d 100644 --- a/doc/modules/ROOT/nav.adoc +++ b/doc/modules/ROOT/nav.adoc @@ -25,6 +25,7 @@ ** xref:4.guide/4b.concurrent-programming.adoc[Concurrent Programming] ** xref:4.guide/4c.io-context.adoc[I/O Context] *** xref:4.guide/4c2.configuration.adoc[Configuration] +** xref:4.guide/4t.lifetime.adoc[Lifetime and Teardown] ** xref:4.guide/4d.sockets.adoc[Sockets] ** xref:4.guide/4e.tcp-acceptor.adoc[Acceptors] ** xref:4.guide/4f.endpoints.adoc[Endpoints] diff --git a/doc/modules/ROOT/pages/4.guide/4t.lifetime.adoc b/doc/modules/ROOT/pages/4.guide/4t.lifetime.adoc new file mode 100644 index 000000000..331a2455f --- /dev/null +++ b/doc/modules/ROOT/pages/4.guide/4t.lifetime.adoc @@ -0,0 +1,133 @@ +// +// Copyright (c) 2026 Steve Gerbino +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// Official repository: https://github.com/cppalliance/corosio +// + += Lifetime and Teardown +:page-mode: explanation + +Corosio's coroutines follow Capy's lifetime rules: + +* Take by value whatever a task reads after it may be stored. +* Pass a lambda coroutine what it needs as parameters, not captures. +* Never destroy the handle of a coroutine that another coroutine is awaiting. + +The _Lifetime and Ownership_ page of the Capy documentation covers those rules +and how a suspended coroutine is owned. + +This page covers what Corosio adds: how long I/O objects, buffers and contexts +must live, and how to stop work. It also says what destroying a context does to +the coroutines still suspended in it. + +== I/O Objects + +Closing or destroying an I/O object cancels its pending operations. Each +completes with `capy::error::canceled`, and its coroutine resumes and returns +through its own code. Test for it portably with `ec == capy::cond::canceled`. + +An I/O object belongs to the context it was created on. Destroy it before that +context: its destructor closes it through a service the context owns. Objects +held in a coroutine frame are destroyed with the frame, including when the +context's teardown unwinds the frame. Use an object only from coroutines +running on its context's executor; see +xref:4.guide/4b.concurrent-programming.adoc#_cross_executor_access[Cross-Executor Access]. + +== Buffers + +An operation reads or writes the memory its buffers describe until it +completes, and it completes after it is canceled, not when. The storage must +outlive the operation, which it does when the operation is awaited from the +frame that owns the storage. See +xref:4.guide/4n.buffers.adoc#_lifetime[Buffer Sequences]. + +== Stopping Work + +To end work, request a stop on the token it was started with. Pending +operations complete with `capy::error::canceled`, the coroutines return +through their own code, and `run()` returns once no work remains: + +[source,cpp] +---- +include::example$snippets/3f_reconnect.cpp[tag=stop_token_shutdown,indent=0] +---- + +cpp:io_context::stop[] does not do this. It stops the event loop and leaves +suspended coroutines where they are, so that cpp:io_context::restart[] can +continue them. See +xref:3.tutorials/3f.reconnect.adoc#_graceful_shutdown_with_stop_tokens[Graceful Shutdown with Stop Tokens]. + +== Destroying a Context + +Destroying a cpp:io_context[] abandons every coroutine still suspended in its +operations or queued on it. Nothing abandoned is resumed. Each chain is +destroyed from the top, on the executor of the coroutine that started it. Its +frames run their destructors: sockets close, guards release, memory is freed. +No code after a suspension point runs and no result is delivered. + +Teardown is a backstop for work that was not stopped, not a way to stop it. +The order that keeps every destructor safe is: + +. Request a stop on the work, and stop submitting new work. +. Let every `run()` return, and join the threads that called it. +. Destroy the I/O objects the application holds. +. Destroy the context. + +[source,cpp] +---- +include::example$snippets/4c_io_context.cpp[tag=teardown,indent=0] +---- + +== Work Across Contexts + +A chain is unwound on the executor of the coroutine that started it. Take a +chain started on one context that waits on an operation of another, or whose +frames own I/O objects of another. No order of destruction is safe for it: +whichever context goes first, the unwind runs after one of the two is gone. +Stop such work, and run both contexts until it ends, before destroying either. +See +xref:4.guide/4c.io-context.adoc#_lifetime_and_teardown[Lifetime and Teardown] +and, for connections a cpp:tcp_server[] runs on other contexts, +xref:4.guide/4k.tcp-server.adoc[TCP Server]. + +== When an Executor Throws + +An operation that completes resumes its coroutine through the executor the +coroutine awaited it on. If that executor's `post` or `dispatch` throws, +`std::terminate` is called: the operation has already finished, and nothing +else could resume the coroutine or unwind it. + +== Checklist for Operation Authors + +Code that suspends a coroutine and resumes it later, such as a custom +operation, awaitable, service or type-erased wrapper, takes part in teardown. +Check each of these: + +. *Carry the owner.* The `capy::continuation` handed to whatever resumes the + coroutine carries the chain's owner from the environment: + `continuation{h, nullptr, env->owner}`. Without it, abandoning the + continuation destroys one frame and leaves the frames above it suspended. +. *Give up, never destroy.* A holder that is shutting down, and so can no + longer resume a continuation, passes it to `capy::abandon`. It never calls + `destroy()` on the handle. +. *One owner of a continuation.* Each continuation is resumed, abandoned, or + still held by the caller after a `post` that threw. Never two of these, and + never none. +. *The environment outlives the work it is lent to.* An inner task or + awaitable may use an `io_env`, stop source or stop token held in another + member. Declare that member first, so it is destroyed after the work. +. *End an operation before what it references.* A wrapper that caches an inner + operation ends it before destroying or replacing the object the operation + uses, in its destructor and in move assignment. +. *Handle a stop that already happened.* When the stop token is already + stopped as the operation suspends, it completes at once with + `capy::error::canceled` rather than registering a callback that never fires. +. *Resume through the executor.* A stop callback, or a completion running on + another thread, posts the continuation to its executor instead of resuming + it inline. +. *Do not leave work referring to a returned frame.* A function that starts + coroutines referring to its locals does not return, or throw, until they + have finished.