From 7f3aadbf5b608cbbe96d754fbac70ce756e1eeb0 Mon Sep 17 00:00:00 2001 From: Steve Gerbino Date: Wed, 7 Oct 2026 23:37:14 +0200 Subject: [PATCH 01/12] feat: add object_ref and object_pool --- include/boost/corosio/detail/intrusive.hpp | 38 ++- include/boost/corosio/detail/object_pool.hpp | 303 +++++++++++++++++++ include/boost/corosio/detail/object_ref.hpp | 122 ++++++++ include/boost/corosio/detail/platform.hpp | 14 + include/boost/corosio/io/io_object.hpp | 49 ++- test/unit/object_pool.cpp | 216 +++++++++++++ test/unit/object_ref.cpp | 96 ++++++ 7 files changed, 834 insertions(+), 4 deletions(-) create mode 100644 include/boost/corosio/detail/object_pool.hpp create mode 100644 include/boost/corosio/detail/object_ref.hpp create mode 100644 test/unit/object_pool.cpp create mode 100644 test/unit/object_ref.cpp diff --git a/include/boost/corosio/detail/intrusive.hpp b/include/boost/corosio/detail/intrusive.hpp index f5267634f..4223a80e8 100644 --- a/include/boost/corosio/detail/intrusive.hpp +++ b/include/boost/corosio/detail/intrusive.hpp @@ -69,6 +69,18 @@ class intrusive_list return head_; } + void push_front(T* w) noexcept + { + auto* n = static_cast(w); + n->prev_ = nullptr; + n->next_ = head_; + if (head_) + static_cast(head_)->prev_ = w; + else + tail_ = w; + head_ = w; + } + void push_back(T* w) noexcept { auto* n = static_cast(w); @@ -118,12 +130,33 @@ class intrusive_list return w; } - void remove(T* w) noexcept + /** Unlink @p w if it is a member of this list. + + @pre @p w is either a genuine member of `*this` list, or is + fully detached (not linked in *any* `intrusive_list`, + including a different instance over the same `T`). This is + not a membership test: `T::node` holds one shared `next_`/ + `prev_` pair regardless of which list currently owns it, so a + node actually linked in a different list looks "not detached" + here and would be spliced out using *this* list's `head_`/ + `tail_` for the boundary cases — corrupting both lists. A + caller that cannot rule that out needs its own bookkeeping + (e.g. a flag for which list currently owns the node) before + calling this. + + @return True if @p w was linked in and has now been removed; + false if it was already detached (e.g. a reentrant call from + a self-referential keepalive dropping during the same + teardown that already popped it elsewhere). Callers that gate + a one-time action (recycling, deleting) on actually having + removed something rely on this to make that action idempotent. + */ + bool remove(T* w) noexcept { auto* n = static_cast(w); // Already detached — nothing to do. if (!n->next_ && !n->prev_ && head_ != w && tail_ != w) - return; + return false; if (n->prev_) static_cast(n->prev_)->next_ = n->next_; else @@ -134,6 +167,7 @@ class intrusive_list tail_ = n->prev_; n->next_ = nullptr; n->prev_ = nullptr; + return true; } /// Invoke @p f for each element in the list. diff --git a/include/boost/corosio/detail/object_pool.hpp b/include/boost/corosio/detail/object_pool.hpp new file mode 100644 index 000000000..6b95484ef --- /dev/null +++ b/include/boost/corosio/detail/object_pool.hpp @@ -0,0 +1,303 @@ +// +// Copyright (c) 2025 Steve Gerbino (steve@gerbino.co) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// Official repository: https://github.com/cppalliance/corosio +// + +#ifndef BOOST_COROSIO_DETAIL_OBJECT_POOL_HPP +#define BOOST_COROSIO_DETAIL_OBJECT_POOL_HPP + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace boost::corosio::detail { + +/** Per-service recycling pool for I/O implementations. + + Replaces per-impl `shared_ptr` ownership: a service news an impl + once, then `recycle()` (the impl's `retire()` action) parks + it on the free list and `acquire()` hands it back out, so steady-state + construct/destroy allocates nothing. Unbounded; memory is released + only on pool destruction. In debug builds, recycled impls are + poisoned by the service's `reuse()` discipline, not here (the pool + cannot know which bytes are resettable). + + A service tears down its impls by calling `shutdown()`, which + enters shutting-down mode and visits every live impl in one + critical section; subsequent zero-ref crossings delete instead of + recycling. + + `adopt()` and `remove()` are protected ownership-transfer + primitives for sanctioned extensions — a derived pool exposing + them to a caller that tracks some of its live impls through its + own structure instead of this pool's `live_` list (the timer + service, via its expiry heap) — rather than part of the public + surface every ordinary service uses. + + @par Thread Safety + Distinct objects: Safe. Shared objects: Safe; all operations lock + an internal mutex. +*/ +template +class object_pool +{ + std::mutex mutex_; + intrusive_list live_; + intrusive_list free_; + bool shutting_down_ = false; + + /** Marks `refs_` on an impl currently parked in `free_`. + + `intrusive_list::remove()` cannot distinguish "linked in + `free_`" from "linked in `live_`" by inspecting the node + alone — both lists share one `next_`/`prev_` pair per impl — + so `recycle()` cannot safely call `live_.remove()` on an impl + it does not already know is live. Stamping this sentinel into + `refs_` the moment an impl joins `free_`, and checking it + before touching either list, is what lets `recycle()` and + `remove()` tell the two cases apart in every build mode, not + just under asserts. `acquire()` always overwrites it with 1 + before the impl is usable again. + */ + static constexpr std::size_t pooled_sentinel = + (std::numeric_limits::max)(); + + /// Check whether @p impl is parked in `free_` (`refs_` reads the + /// sentinel). Callers must check this before any list operation — + /// see the class-level comment on `pooled_sentinel`. + static bool is_pooled(Impl const* impl) noexcept + { + return impl->refs_.load(std::memory_order_relaxed) == + pooled_sentinel; + } + +protected: + /** Track a fresh or reset impl as live. + + @pre @p impl is not parked in `free_` (`refs_` must not read + `pooled_sentinel`). The same list-corruption hazard + `recycle()`'s class-level comment documents applies here: a + free-listed impl pushed onto `live_` directly, bypassing + `acquire()`, would end up linked in both lists through one shared + `next_`/`prev_` pair. + + @pre The caller is not racing `shutdown()`. `acquire()` is + this method's only caller, and a service's `construct()` — + `acquire()`'s only caller — is expected to stop being called + before the service's own `shutdown()` runs; this is checked + only in debug builds because the failure mode is a silently + orphaned impl (no close callback), not memory corruption. + */ + void adopt(Impl* impl) + { + std::lock_guard lock(mutex_); + if (is_pooled(impl)) + { + BOOST_COROSIO_ASSERT(false); + return; + } + BOOST_COROSIO_ASSERT(!shutting_down_); + live_.push_back(impl); + } + + /** Unlink a live impl without recycling (ownership transfer). + + Used by a caller that is taking an impl out of this pool's + bookkeeping entirely (a thread-local cache slot, a direct + delete during shutdown) rather than parking it on `free_`. + + @pre @p impl is not parked in `free_` (`refs_` must not read + `pooled_sentinel`). The same list-corruption hazard + `recycle()`'s class-level comment documents applies here: a + free-listed impl would be spliced out using `live_`'s + boundary pointers instead of `free_`'s. + + @return True if @p impl was linked in `live_` and has now + been removed; false if it was already detached. + */ + bool remove(Impl* impl) noexcept + { + std::lock_guard lock(mutex_); + if (is_pooled(impl)) + { + BOOST_COROSIO_ASSERT(false); + return false; + } + return live_.remove(impl); + } + +public: + /** Destroy the pool, freeing every impl still parked or live. + + @pre `shutdown()` has already run, or no impls are live. + Without it, deleting a live impl here would let that impl's + own destructor reenter `recycle()` outside the shutting-down + branch that makes a reentrant retirement (a self-referential + op's `object_ref` releasing its last reference while this + destructor's own `delete` is still on the stack) a safe no-op + — it would instead look exactly like the accounting-bug case + `recycle()` deletes a second time. + */ + ~object_pool() + { + BOOST_COROSIO_ASSERT(shutting_down_ || live_.empty()); + for (auto* list : {&free_, &live_}) + while (auto* impl = list->pop_front()) + delete impl; + } + + object_pool() = default; + object_pool(object_pool const&) = delete; + object_pool& operator=(object_pool const&) = delete; + + /** Pop-or-create an impl with the acquisition protocol applied. + + Recycled impls are `reuse()`-reset and their reference count + restored from the pooled sentinel; fresh impls are constructed + in place from `args` on the cold path. Either way the impl is + adopted as live. This is the sequence every service's + `construct()` used to hand-roll. Making the pool itself the + only place that calls `new Impl` gives the whole library one + audit point for allocation policy. + + @pre Not called concurrently with, or after, `shutdown()` on + this pool. Every caller reaches this through a service's + `construct()`, and service shutdown order guarantees no + `construct()` outlives that service's own `shutdown()`; the + window is only asserted in debug builds (see `adopt()`) + because the failure mode is an orphaned impl, not corruption. + + @param args Constructor arguments for `Impl` (cold path only). + + @return The acquired impl, `refs_ == 1`, tracked live. + */ + template + Impl* acquire(Args&&... args) + { + Impl* impl; + { + std::lock_guard lock(mutex_); + BOOST_COROSIO_ASSERT(!shutting_down_); + impl = free_.pop_front(); + if (impl) + { + impl->refs_.store(1, std::memory_order_relaxed); + live_.push_back(impl); + } + } + // reuse() runs unlocked: acquire()'s precondition excludes a + // concurrent shutdown() visiting the impl mid-reset. + if (impl) + { + impl->reuse(); + return impl; + } + impl = new Impl(std::forward(args)...); + adopt(impl); + return impl; + } + + /** Move a zero-ref impl from live to the free list. + + Called from `retire()`, possibly on a scheduler thread. + Deletes instead when shutting down. The impl must not be + touched after this returns. + + Two accounting bugs are possible here, neither from this + pool's own bookkeeping but from a caller's refcounting (e.g. + a duplicate `retire()` for one zero-crossing). Both are + made safe in every build mode, not just under asserts, because + neither can be told apart from the correct case by a + `BOOST_COROSIO_ASSERT` alone that still runs the list + operations around it: + + - `impl` is already parked in `free_` — the list-corruption + hazard the class-level `pooled_sentinel` comment documents. + The `is_pooled()` check runs first and returns before either + list is touched. + - `impl` is fully detached from `live_` and is not the + sentinel (not shutting down): `live_.remove()` correctly + returns false, but then the impl is reachable through + neither `acquire()` nor `shutdown()`'s visit while still + allocated — stranded. Deleted instead of left to leak. + + The one legitimate "already detached" case — an impl with an + embedded op whose `object_ref` points back at itself reaching + zero *during* `delete impl`, e.g. `~object_pool()`'s own + unconditional sweep tearing down that op's `object_ref` member + before the outer `delete` returns — is distinguished from the + accounting-bug case by `shutting_down_`: that reentrant path + only exists because `shutdown()` — called with a no-op + callback by a caller like the timer service that tracks its + live impls through its own structure instead of this pool's + `live_` list, just to flip the flag — always precedes + `~object_pool()`'s sweep (service shutdown order), so a + not-live, not-pooled impl while `shutting_down_` is exactly + that reentrant no-op, and the (already false) `was_live` means + skip the delete — the outer frame's `delete` is still running. + */ + void recycle(Impl* impl) + { + bool do_delete = false; + { + std::lock_guard lock(mutex_); + if (is_pooled(impl)) + { + BOOST_COROSIO_ASSERT(false); + return; + } + + bool const was_live = live_.remove(impl); + if (shutting_down_) + { + do_delete = was_live; + } + else if (was_live) + { + impl->refs_.store( + pooled_sentinel, std::memory_order_relaxed); + // LIFO: the most recently retired impl is the + // cache-warmest candidate for the next acquire(). + free_.push_front(impl); + } + else + { + BOOST_COROSIO_ASSERT(false); + do_delete = true; + } + } + if (do_delete) + delete impl; + } + + /** Enter shutdown mode and visit every live impl. + + Sets shutting-down (subsequent zero-crossings delete instead + of recycling) and runs `f` on each live impl, all under one + critical section. + + @note The callback runs under the pool's non-recursive mutex; + it must not call acquire(), recycle(), adopt(), or remove() + on this pool. + */ + template + void shutdown(F&& f) + { + std::lock_guard lock(mutex_); + shutting_down_ = true; + live_.for_each(std::forward(f)); + } +}; + +} // namespace boost::corosio::detail + +#endif diff --git a/include/boost/corosio/detail/object_ref.hpp b/include/boost/corosio/detail/object_ref.hpp new file mode 100644 index 000000000..8f44c4e41 --- /dev/null +++ b/include/boost/corosio/detail/object_ref.hpp @@ -0,0 +1,122 @@ +// +// Copyright (c) 2025 Vinnie Falco (vinnie.falco@gmail.com) +// Copyright (c) 2026 Steve Gerbino +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// Official repository: https://github.com/cppalliance/corosio +// + +#ifndef BOOST_COROSIO_DETAIL_OBJECT_REF_HPP +#define BOOST_COROSIO_DETAIL_OBJECT_REF_HPP + +#include +#include +#include +#include + +namespace boost::corosio::detail { + +/// Add one reference to `impl`. `impl` must be non-null. +inline void +acquire(io_object::implementation* impl) noexcept +{ + impl->refs_.fetch_add(1, std::memory_order_relaxed); +} + +/** Drop one reference; runs `retire()` at zero. `impl` must be + non-null. + + Uses release on the decrement and an acquire fence before the + zero-action so writes made while holding a reference are visible + to the recycler. TSan cannot model a standalone fence, so under + it the decrement itself is acq_rel instead -- strictly stronger, + and TSan's happens-before tracking models the exchange directly. +*/ +inline void +release(io_object::implementation* impl) noexcept +{ +#if BOOST_COROSIO_TSAN + if (impl->refs_.fetch_sub(1, std::memory_order_acq_rel) == 1) + impl->retire(); +#else + if (impl->refs_.fetch_sub(1, std::memory_order_release) == 1) + { + std::atomic_thread_fence(std::memory_order_acquire); + impl->retire(); + } +#endif +} + +/** Owning intrusive reference to an I/O implementation. + + Replaces `std::shared_ptr` op keepalives: copy acquires, + move steals, destruction releases. Kept deliberately minimal — + two words smaller than a shared_ptr and no control block. +*/ +class object_ref +{ + io_object::implementation* p_ = nullptr; + +public: + /// Destroy the object_ref, releasing the held reference, if any. + ~object_ref() { reset(); } + + /// Construct an object_ref holding no reference. + object_ref() noexcept = default; + + /// Construct an object_ref that acquires a new reference to `p`. + explicit object_ref(io_object::implementation* p) noexcept : p_(p) + { + if (p_) + acquire(p_); + } + + /// Construct a copy, acquiring another reference to `other`'s target. + object_ref(object_ref const& other) noexcept : p_(other.p_) + { + if (p_) + acquire(p_); + } + + /// Construct by transferring the reference out of `other`. + object_ref(object_ref&& other) noexcept + : p_(std::exchange(other.p_, nullptr)) + { + } + + /// Release the held reference, if any, and acquire `other`'s target. + object_ref& operator=(object_ref const& other) noexcept + { + object_ref(other).swap(*this); + return *this; + } + + /// Release the held reference, if any, and take ownership of `other`'s. + object_ref& operator=(object_ref&& other) noexcept + { + object_ref(std::move(other)).swap(*this); + return *this; + } + + /// Release the held reference, if any. + void reset() noexcept + { + if (p_) + release(std::exchange(p_, nullptr)); + } + + /// Return the held implementation pointer. + io_object::implementation* get() const noexcept { return p_; } + + /// Return true if a reference is held. + explicit operator bool() const noexcept { return p_ != nullptr; } + + /// Exchange the implementation pointers held by `*this` and `other`. + void swap(object_ref& other) noexcept { std::swap(p_, other.p_); } +}; + +} // namespace boost::corosio::detail + +#endif diff --git a/include/boost/corosio/detail/platform.hpp b/include/boost/corosio/detail/platform.hpp index 7afd49331..58795e9c7 100644 --- a/include/boost/corosio/detail/platform.hpp +++ b/include/boost/corosio/detail/platform.hpp @@ -86,4 +86,18 @@ #endif // BOOST_COROSIO_MRDOCS +// ThreadSanitizer cannot model a standalone std::atomic_thread_fence; +// code that needs a different (stronger) ordering under it checks this. +#if defined(__SANITIZE_THREAD__) +#define BOOST_COROSIO_TSAN 1 +#elif defined(__has_feature) +#if __has_feature(thread_sanitizer) +#define BOOST_COROSIO_TSAN 1 +#endif +#endif + +#ifndef BOOST_COROSIO_TSAN +#define BOOST_COROSIO_TSAN 0 +#endif + #endif // BOOST_COROSIO_DETAIL_PLATFORM_HPP diff --git a/include/boost/corosio/io/io_object.hpp b/include/boost/corosio/io/io_object.hpp index 49a251b26..795d44dd4 100644 --- a/include/boost/corosio/io/io_object.hpp +++ b/include/boost/corosio/io/io_object.hpp @@ -15,6 +15,8 @@ #include #include +#include +#include #include namespace boost::corosio { @@ -49,11 +51,28 @@ class BOOST_COROSIO_DECL io_object class handle; /** Derived types dispatch platform-specific I/O operations through it. + + Reference-counted: the owning service holds one reference + (`refs_` starts at 1); in-flight operations hold additional + references through `detail::object_ref`. When the count reaches + zero, `retire()` runs — services recycle the impl into + their free-list there. */ struct implementation { /// Destroy the implementation; called only through @ref io_service. virtual ~implementation() = default; + + /** Called when the reference count reaches zero. + + @note Transitional default is a no-op while backends are + migrated; it becomes pure virtual at the end of the + migration. + */ + virtual void retire() noexcept {} + + /// In-flight + service references; starts at the service's 1. + std::atomic refs_{1}; }; /** Constructs, closes, and destroys platform implementations on @@ -65,10 +84,28 @@ class BOOST_COROSIO_DECL io_object /// Destroy the service; the execution context outlives it. virtual ~io_service() = default; - /// Construct a new implementation instance. + /** Construct a new implementation instance. + + May return a recycled implementation taken from the + service's free-list (populated by `implementation::retire`) + instead of allocating new storage. + */ virtual implementation* construct() = 0; - /// Destroy the implementation, closing kernel resources and freeing memory. + /** Close kernel resources and release the service's reference. + + Called whenever a handle relinquishes its current + implementation, not only on destruction: handle + destruction, move-assignment onto a handle that already + owns one (the replaced implementation is destroyed, the + incoming one is not), and `handle::reset()` all invoke + this. Closes the underlying descriptor and drops the + service's own reference; in-flight operations may still + hold references of their own, so the implementation is + not necessarily recycled or freed here — that happens in + `implementation::retire` whenever the refcount + actually reaches zero. + */ virtual void destroy(implementation* impl) = 0; /// Close the I/O object, releasing kernel resources without deallocating. @@ -156,6 +193,14 @@ class BOOST_COROSIO_DECL io_object /** Replace the implementation, destroying the old one. + @pre The handle is already bound to a service (constructed + via the service-taking constructor, not the default + one) — the old implementation is destroyed through + that service. + @pre @p p, if non-null, was constructed by this handle's + own service: the service that destroys it later must + be the one that knows how to close it. + @param p The new implementation to own. May be nullptr. */ void reset(implementation* p) noexcept diff --git a/test/unit/object_pool.cpp b/test/unit/object_pool.cpp new file mode 100644 index 000000000..83ff0a3d4 --- /dev/null +++ b/test/unit/object_pool.cpp @@ -0,0 +1,216 @@ +#include +#include +#include +#include "test_suite.hpp" + +namespace boost::corosio::detail { + +namespace { + +struct pool_impl final + : io_object::implementation + , intrusive_list::node +{ + static inline int instances = 0; + + object_pool* pool = nullptr; + int reused_count = 0; + + pool_impl() { ++instances; } + ~pool_impl() override { --instances; } + + void reuse() noexcept { ++reused_count; } + void retire() noexcept override { pool->recycle(this); } +}; + +// Exercises the sanctioned-extension mechanism documented on +// object_pool: a derived pool re-exposing adopt()/remove() as public, +// the same way timer_object_pool does for the timer service. +struct test_pool : object_pool +{ + using object_pool::adopt; + using object_pool::remove; +}; + +// Fixture for the shutting-down-gated reentrant delete branch in +// recycle(): an impl holding an embedded object_ref to itself never +// reaches zero through ordinary release() calls, so the only thing +// that can retire it is ~object_pool()'s own sweep destroying that +// embedded ref as part of `delete impl` -- a reentrant recycle() +// call made from inside the outer delete. +struct self_ref_impl final + : io_object::implementation + , intrusive_list::node +{ + static inline int instances = 0; + + object_pool* pool = nullptr; + object_ref self_ref; + + self_ref_impl() { ++instances; } + ~self_ref_impl() override { --instances; } + + void reuse() noexcept { } + void retire() noexcept override { pool->recycle(this); } +}; + +} // namespace + +struct object_pool_test +{ + void run() + { + // acquire news on a cold pool and hands the same impl back, + // reuse()-reset, after it is recycled + { + object_pool pool; + auto* a = pool.acquire(); + a->pool = &pool; + BOOST_TEST_EQ(a->reused_count, 0); + release(a); // service ref -> recycle + BOOST_TEST_EQ(pool_impl::instances, 1); + auto* b = pool.acquire(); // warm: recycled, not newed + BOOST_TEST_EQ(b, a); + BOOST_TEST_EQ(b->reused_count, 1); + BOOST_TEST_EQ(pool_impl::instances, 1); + release(b); // back to free; dtor frees + } + BOOST_TEST_EQ(pool_impl::instances, 0); + // keepalive defers recycling until the last ref drops + { + object_pool pool; + auto* a = pool.acquire(); + a->pool = &pool; + object_ref op_ref(a); // simulated in-flight op + release(a); // service ref gone... + auto* b = pool.acquire(); // ...a not recycled: news + b->pool = &pool; + BOOST_TEST_NE(b, a); + BOOST_TEST_EQ(pool_impl::instances, 2); + op_ref.reset(); // op drains -> a recycles + auto* c = pool.acquire(); // warm again + BOOST_TEST_EQ(c, a); + release(b); + release(c); + } + BOOST_TEST_EQ(pool_impl::instances, 0); + // shutdown mode deletes instead of pooling + { + object_pool pool; + auto* a = pool.acquire(); + a->pool = &pool; + pool.shutdown([](pool_impl*) {}); + release(a); // deleted, not pooled + BOOST_TEST_EQ(pool_impl::instances, 0); + } + // remove() transfers ownership out of the pool entirely: the + // impl leaves live_ and the caller now owns it outright (a + // direct delete here must not be seen again by the pool's own + // destructor -- the scope exit below proves no double free). + { + test_pool pool; + auto* a = pool.acquire(); + a->pool = &pool; + BOOST_TEST(pool.remove(a)); // unlinked from live_ + delete a; // caller-owned now + // pool goes out of scope here with empty live_/free_: if + // remove() had failed to unlink `a`, ~object_pool() would + // delete it a second time. + } + BOOST_TEST_EQ(pool_impl::instances, 0); + // remove() is idempotent: a second call on an already-detached + // impl returns false instead of corrupting the list (mirrors + // intrusive_list::remove()'s own already-detached contract). + { + test_pool pool; + auto* a = pool.acquire(); + a->pool = &pool; + BOOST_TEST(pool.remove(a)); + BOOST_TEST(!pool.remove(a)); // already detached + delete a; + } + // Sentinel guard: remove() must refuse to touch a free_-parked + // impl (the same hazard recycle() guards against in reverse). + // The guard's debug-build path asserts and aborts before + // returning, so only its release-build-observable contract -- + // refused, false, free list left intact -- can be exercised + // here; it is skipped entirely when assertions are compiled in + // (the normal state of this test binary). +#ifdef NDEBUG + { + test_pool pool; + auto* a = pool.acquire(); + a->pool = &pool; + release(a); // parks a on free_ (refs_ == pooled_sentinel) + BOOST_TEST(!pool.remove(a)); // refused + auto* b = pool.acquire(); // free_ untouched: warm hit + BOOST_TEST_EQ(b, a); + release(b); // back to free_; pool dtor frees it + } +#endif + BOOST_TEST_EQ(pool_impl::instances, 0); + // recycle()'s own guards, release-build contract, mirroring + // the remove() block above: both skip entirely under + // assertions (the normal state of this test binary) because + // the guarded branch asserts and aborts before returning. +#ifdef NDEBUG + { + // Sentinel guard: a duplicate recycle() on an impl + // already parked in free_ (e.g. a double retire() for + // one zero-crossing) must refuse before touching either + // list, not corrupt free_ by removing it through live_'s + // boundary pointers. + test_pool pool; + auto* a = pool.acquire(); + a->pool = &pool; + release(a); // 1 -> 0: recycle() parks a on free_ + BOOST_TEST_EQ(pool_impl::instances, 1); + pool.recycle(a); // duplicate retire while parked: refused + auto* b = pool.acquire(); // free_ untouched: still a warm hit + BOOST_TEST_EQ(b, a); + release(b); + } + BOOST_TEST_EQ(pool_impl::instances, 0); + { + // Accounting-bug branch: an impl detached from live_ + // (not parked, not shutting down) reaching recycle() is + // deleted outright instead of being stranded (reachable + // through neither acquire() nor shutdown()'s visit while + // still allocated). + test_pool pool; + auto* a = pool.acquire(); + a->pool = &pool; + BOOST_TEST(pool.remove(a)); // detach from live_ without recycling + pool.recycle(a); // not live, not pooled: deleted + BOOST_TEST_EQ(pool_impl::instances, 0); + } +#endif + // Shutting-down-gated reentrant delete: a self-referential + // impl's refcount never reaches zero through release() + // alone, so the only path that can retire it is + // ~object_pool()'s own sweep destroying the embedded + // object_ref as part of `delete impl`. shutdown() must run + // first (the destructor's own precondition), so the + // reentrant recycle() call lands in the shutting-down branch + // instead of trying to re-park a half-destroyed impl on + // free_ or being mistaken for the accounting-bug case. + { + object_pool pool; + auto* a = pool.acquire(); + a->pool = &pool; + a->self_ref = object_ref(a); // refs_: 1 -> 2 + release(a); // refs_: 2 -> 1; self-ref keeps it alive + BOOST_TEST_EQ(self_ref_impl::instances, 1); + pool.shutdown([](self_ref_impl*) {}); + // pool destructs here: the live_ sweep deletes `a`; + // its destructor tears down self_ref, releasing the + // last reference and reentering recycle() -- exactly + // one delete must happen, no crash, no double free. + } + BOOST_TEST_EQ(self_ref_impl::instances, 0); + } +}; + +TEST_SUITE(object_pool_test, "boost.corosio.object_pool"); + +} // namespace boost::corosio::detail diff --git a/test/unit/object_ref.cpp b/test/unit/object_ref.cpp new file mode 100644 index 000000000..f99d7069e --- /dev/null +++ b/test/unit/object_ref.cpp @@ -0,0 +1,96 @@ +#include +#include +#include "test_suite.hpp" + +namespace boost::corosio::detail { + +namespace { + +struct counted_impl final : io_object::implementation +{ + int* last_ref_hits; + explicit counted_impl(int* hits) : last_ref_hits(hits) {} + void retire() noexcept override { ++*last_ref_hits; } +}; + +} // namespace + +struct object_ref_test +{ + void run() + { + // acquire/release pairs; zero-action fires exactly once + { + int hits = 0; + counted_impl impl(&hits); // refs_ starts at 1 + release(&impl); // 1 -> 0 + BOOST_TEST_EQ(hits, 1); + } + // object_ref RAII: ctor acquires, dtor releases + { + int hits = 0; + counted_impl impl(&hits); + { + object_ref r(&impl); // 1 -> 2 + BOOST_TEST(r); + object_ref r2 = r; // copy: 2 -> 3 + object_ref r3 = std::move(r2); // move: still 3 + BOOST_TEST(!r2); + BOOST_TEST_EQ(r3.get(), &impl); + } // back to 1 + BOOST_TEST_EQ(hits, 0); + release(&impl); + BOOST_TEST_EQ(hits, 1); + } + // reset releases early + { + int hits = 0; + counted_impl impl(&hits); + object_ref r(&impl); + release(&impl); // service ref gone, r holds it + BOOST_TEST_EQ(hits, 0); + r.reset(); + BOOST_TEST_EQ(hits, 1); + } + // copy-assignment: operator=(object_ref const&) is never + // otherwise executed anywhere in the tree. Verify it releases + // the old target and acquires the new one, and is safe under + // self-assignment. + { + int hits_a = 0; + int hits_b = 0; + counted_impl a(&hits_a); + counted_impl b(&hits_b); + + object_ref ra(&a); // a.refs_: 1 -> 2 + object_ref rb(&b); // b.refs_: 1 -> 2 + + rb = ra; // rb releases b, acquires a + BOOST_TEST_EQ(rb.get(), &a); + BOOST_TEST_EQ( + a.refs_.load(), 3u); // service + ra + rb + release(&b); // b's last (service) ref + BOOST_TEST_EQ(hits_b, 1); // retired: rb no longer holds it + + // Deliberate self-assignment: must not release before + // acquiring. Laundered through a named alias rather than + // `rb = rb;` -- clang's -Wself-assign-overloaded flags the + // literal spelling even when it is intentional. + object_ref& rb_alias = rb; + rb = rb_alias; + BOOST_TEST_EQ(rb.get(), &a); + BOOST_TEST_EQ(a.refs_.load(), 3u); + BOOST_TEST_EQ(hits_a, 0); // no spurious retire + + ra.reset(); // a.refs_: 3 -> 2 + release(&a); // service ref: 2 -> 1 + BOOST_TEST_EQ(hits_a, 0); + rb.reset(); // a.refs_: 1 -> 0 -> retire + BOOST_TEST_EQ(hits_a, 1); + } + } +}; + +TEST_SUITE(object_ref_test, "boost.corosio.object_ref"); + +} // namespace boost::corosio::detail From c1546846c2f4671f73b87251f0b5e46550d1dcd4 Mon Sep 17 00:00:00 2001 From: Steve Gerbino Date: Wed, 7 Oct 2026 23:39:49 +0200 Subject: [PATCH 02/12] refactor: recycle io_object impls through object_pool --- .../boost/corosio/native/detail/coro_op.hpp | 3 +- .../native/detail/coro_op_complete.hpp | 10 +- .../native/detail/epoll/epoll_scheduler.hpp | 2 +- .../native/detail/kqueue/kqueue_scheduler.hpp | 2 +- .../native/detail/native_socket_base.hpp | 11 +- .../detail/posix/posix_random_access_file.hpp | 28 +- .../posix_random_access_file_service.hpp | 86 ++-- .../native/detail/posix/posix_resolver.hpp | 40 +- .../detail/posix/posix_resolver_service.hpp | 78 ++-- .../native/detail/posix/posix_signal.hpp | 25 + .../detail/posix/posix_signal_service.hpp | 83 ++-- .../native/detail/posix/posix_stream_file.hpp | 38 +- .../posix/posix_stream_file_service.hpp | 68 +-- .../detail/reactor/reactor_acceptor.hpp | 171 +++---- .../reactor/reactor_acceptor_service.hpp | 29 +- .../native/detail/reactor/reactor_backend.hpp | 6 +- .../detail/reactor/reactor_basic_socket.hpp | 168 +++---- .../reactor/reactor_datagram_socket.hpp | 51 ++- .../reactor/reactor_descriptor_state.hpp | 14 +- .../native/detail/reactor/reactor_op.hpp | 6 +- .../native/detail/reactor/reactor_op_base.hpp | 9 +- .../detail/reactor/reactor_service_state.hpp | 19 +- .../detail/reactor/reactor_socket_finals.hpp | 18 + .../detail/reactor/reactor_socket_service.hpp | 51 +-- .../detail/reactor/reactor_stream_socket.hpp | 49 +- .../native/detail/select/select_scheduler.hpp | 2 +- .../native/detail/speculative_state.hpp | 14 + .../native/detail/uring/uring_dgram_ops.hpp | 48 +- .../native/detail/uring/uring_file_ops.hpp | 8 +- .../detail/uring/uring_file_service_base.hpp | 71 +-- .../detail/uring/uring_multishot_acceptor.hpp | 177 +++++-- .../corosio/native/detail/uring/uring_op.hpp | 2 +- .../detail/uring/uring_random_access_file.hpp | 40 +- .../native/detail/uring/uring_scheduler.hpp | 6 +- .../native/detail/uring/uring_socket_ops.hpp | 38 +- .../uring/uring_socket_service_base.hpp | 84 ++-- .../native/detail/uring/uring_stream_file.hpp | 42 +- .../native/detail/uring/uring_types.hpp | 431 +++++++++++++----- test/unit/object_reuse.cpp | 209 +++++++++ test/unit/resolver.cpp | 4 +- 40 files changed, 1492 insertions(+), 749 deletions(-) create mode 100644 test/unit/object_reuse.cpp diff --git a/include/boost/corosio/native/detail/coro_op.hpp b/include/boost/corosio/native/detail/coro_op.hpp index 460bb6c87..bac26b0bc 100644 --- a/include/boost/corosio/native/detail/coro_op.hpp +++ b/include/boost/corosio/native/detail/coro_op.hpp @@ -12,6 +12,7 @@ #include #include +#include #include #include @@ -93,7 +94,7 @@ struct coro_op : scheduler_op /// Keeps the owning impl alive while the op is in flight (the kernel /// owns user buffers until completion). Dropped in the handler's resume /// tail (see coro_op_complete.hpp). - std::shared_ptr impl_ptr; + detail::object_ref object_ref_; /// Default-construct for virtual-dispatch backends (the reactors, which /// override operator()/destroy() and leave func_ null). diff --git a/include/boost/corosio/native/detail/coro_op_complete.hpp b/include/boost/corosio/native/detail/coro_op_complete.hpp index 24293de24..298aff82b 100644 --- a/include/boost/corosio/native/detail/coro_op_complete.hpp +++ b/include/boost/corosio/native/detail/coro_op_complete.hpp @@ -25,7 +25,7 @@ 1. disarm the stop_callback, 2. on the shutdown-drain path (owner == nullptr) just break the - impl_ptr keepalive cycle and return without resuming, + object_ref_ keepalive cycle and return without resuming, 3. otherwise resume the coroutine on its executor, dropping the keepalive only after the continuation has been handed off. @@ -113,7 +113,7 @@ decode_io_result( @param self The completing op. @return True if this was a shutdown drain — the caller must `return` immediately without decoding or resuming. On that path the - impl_ptr keepalive is dropped here (which may destroy the impl, + object_ref_ keepalive is dropped here (which may destroy the impl, and with it the op storage). */ inline bool @@ -122,7 +122,7 @@ coro_drain_if_shutdown(void* owner, coro_op* self) noexcept self->stop_cb.reset(); if (owner == nullptr) { - auto suicide = std::move(self->impl_ptr); + auto suicide = std::move(self->object_ref_); return true; } return false; @@ -130,7 +130,7 @@ coro_drain_if_shutdown(void* owner, coro_op* self) noexcept /** Resume tail shared by every proactor handler. - Resumes the op's coroutine on its executor and then drops the impl_ptr + Resumes the op's coroutine on its executor and then drops the object_ref_ keepalive. The keepalive is moved into a local that is released *after* `resume()` returns, matching the existing io_uring ordering: the impl (and therefore this op's storage) may be destroyed as the local goes out of @@ -146,7 +146,7 @@ coro_resume(coro_op* self) noexcept // Clear the keepalive before publishing the continuation: a strand // drained on another thread can reuse this op via reset() the instant // it runs, so this write must be ordered before the publish, not after. - auto suicide = std::move(self->impl_ptr); + auto suicide = std::move(self->object_ref_); auto next = dispatch_coro(self->ex, self->cont); next.resume(); // suicide drops here; may destroy impl + self. diff --git a/include/boost/corosio/native/detail/epoll/epoll_scheduler.hpp b/include/boost/corosio/native/detail/epoll/epoll_scheduler.hpp index 574c25665..953b110e7 100644 --- a/include/boost/corosio/native/detail/epoll/epoll_scheduler.hpp +++ b/include/boost/corosio/native/detail/epoll/epoll_scheduler.hpp @@ -263,7 +263,7 @@ epoll_scheduler::register_descriptor( desc->ready_events_.store(0, std::memory_order_relaxed); conditionally_enabled_mutex::scoped_lock lock(desc->mutex); - desc->impl_ref_.reset(); + desc->object_ref_.reset(); desc->read_ready = false; desc->write_ready = false; return {}; diff --git a/include/boost/corosio/native/detail/kqueue/kqueue_scheduler.hpp b/include/boost/corosio/native/detail/kqueue/kqueue_scheduler.hpp index 4471646d2..4560bf559 100644 --- a/include/boost/corosio/native/detail/kqueue/kqueue_scheduler.hpp +++ b/include/boost/corosio/native/detail/kqueue/kqueue_scheduler.hpp @@ -249,7 +249,7 @@ kqueue_scheduler::register_descriptor( desc->ready_events_.store(0, std::memory_order_relaxed); conditionally_enabled_mutex::scoped_lock lock(desc->mutex); - desc->impl_ref_.reset(); + desc->object_ref_.reset(); desc->read_ready = false; desc->write_ready = false; return {}; diff --git a/include/boost/corosio/native/detail/native_socket_base.hpp b/include/boost/corosio/native/detail/native_socket_base.hpp index c067cd1ac..7fd6709e6 100644 --- a/include/boost/corosio/native/detail/native_socket_base.hpp +++ b/include/boost/corosio/native/detail/native_socket_base.hpp @@ -27,9 +27,10 @@ Holds the part of a socket impl that does not care whether the backend is readiness-based (epoll/kqueue/select, which park ops on a descriptor_state) or completion-based (io_uring, which submits SQEs): - the file descriptor, the cached local endpoint, the impl lifecycle - bases (enable_shared_from_this + the service's intrusive tracking node), - and the synchronous accessors (native_handle / options / bind). + the file descriptor, the cached local endpoint, and the synchronous + accessors (native_handle / options / bind). The impl lifecycle (the + intrusive refcount + the owning service's recycling pool) is carried + by `io_object::implementation`, not by this base. reactor_basic_socket derives from this and adds the readiness machinery (descriptor_state, register_op, the cancel/close that deregister from @@ -39,8 +40,7 @@ readiness-agnostic socket surface, while the on-EAGAIN action (park vs submit-SQE) and the op model stay backend-specific. - @tparam Derived The concrete socket type (CRTP, for shared_from_this - and the intrusive node). + @tparam Derived The concrete socket type (CRTP). @tparam ImplBase The public vtable base (tcp_socket::implementation, udp_socket::implementation, ...). @tparam Endpoint The endpoint type (endpoint or local_endpoint). @@ -51,7 +51,6 @@ namespace boost::corosio::detail { template class native_socket_base : public ImplBase - , public std::enable_shared_from_this { protected: // CRTP base: not publicly constructible. The check's preferred fix diff --git a/include/boost/corosio/native/detail/posix/posix_random_access_file.hpp b/include/boost/corosio/native/detail/posix/posix_random_access_file.hpp index c43431f47..6c73f750f 100644 --- a/include/boost/corosio/native/detail/posix/posix_random_access_file.hpp +++ b/include/boost/corosio/native/detail/posix/posix_random_access_file.hpp @@ -67,7 +67,6 @@ class posix_random_access_file_service; /** Random-access file implementation for POSIX backends. */ class posix_random_access_file final : public random_access_file::implementation - , public std::enable_shared_from_this , public intrusive_list::node { friend class posix_random_access_file_service; @@ -95,7 +94,7 @@ class posix_random_access_file final int errn = 0; std::size_t bytes_transferred = 0; - // Raw back-pointer for the typed work; `impl_ptr` is the keepalive. + // Raw back-pointer for the typed work; `object_ref_` is the keepalive. posix_random_access_file* file_ = nullptr; void operator()() override; @@ -108,6 +107,27 @@ class posix_random_access_file final explicit posix_random_access_file( posix_random_access_file_service& svc) noexcept; + /// Recycle into the owning service's pool. Defined out-of-line + /// after posix_random_access_file_service for its complete type. + void retire() noexcept override; + + /** Reset for recycling. + + `close_file()` already drove fd_ to its closed value before + the refcount reached zero. Every `raf_op` is heap-allocated + per submission and holds its own `object_ref`, so the refcount + cannot reach zero while one is outstanding — asserting + `outstanding_ops_` is empty is therefore a precondition check, + not a defensive one. + + @pre refs_ == 0, fd closed, no op in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(fd_ == -1); + BOOST_COROSIO_ASSERT(outstanding_ops_.empty()); + } + // -- random_access_file::implementation -- std::coroutine_handle<> read_some_at( @@ -296,7 +316,7 @@ posix_random_access_file::raf_op::operator()() file_->outstanding_ops_.remove(this); } - impl_ptr.reset(); + object_ref_.reset(); auto coro = h; ex.on_work_finished(); @@ -314,7 +334,7 @@ posix_random_access_file::raf_op::destroy() std::lock_guard lock(file_->ops_mutex_); file_->outstanding_ops_.remove(this); } - impl_ptr.reset(); + object_ref_.reset(); ex.on_work_finished(); delete this; } diff --git a/include/boost/corosio/native/detail/posix/posix_random_access_file_service.hpp b/include/boost/corosio/native/detail/posix/posix_random_access_file_service.hpp index 4729fab1d..7497f02b3 100644 --- a/include/boost/corosio/native/detail/posix/posix_random_access_file_service.hpp +++ b/include/boost/corosio/native/detail/posix/posix_random_access_file_service.hpp @@ -17,11 +17,12 @@ #include #include #include +#include +#include #include #include -#include -#include +#include namespace boost::corosio::detail { @@ -29,6 +30,8 @@ namespace boost::corosio::detail { class BOOST_COROSIO_DECL posix_random_access_file_service final : public random_access_file_service { + friend class posix_random_access_file; + public: explicit posix_random_access_file_service(capy::execution_context& ctx) : sched_(&get_scheduler(ctx)) @@ -45,16 +48,7 @@ class BOOST_COROSIO_DECL posix_random_access_file_service final io_object::implementation* construct() override { - auto ptr = std::make_shared(*this); - auto* impl = ptr.get(); - - { - std::lock_guard lock(mutex_); - file_list_.push_back(impl); - file_ptrs_[impl] = std::move(ptr); - } - - return impl; + return object_pool_.acquire(*this); } void destroy(io_object::implementation* p) override @@ -62,7 +56,7 @@ class BOOST_COROSIO_DECL posix_random_access_file_service final auto& impl = static_cast(*p); impl.cancel(); impl.close_file(); - destroy_impl(impl); + release(&impl); } void close(io_object::handle& h) override @@ -90,21 +84,24 @@ class BOOST_COROSIO_DECL posix_random_access_file_service final void shutdown() override { - std::lock_guard lock(mutex_); - for (auto* impl = file_list_.pop_front(); impl != nullptr; - impl = file_list_.pop_front()) + // See uring_socket_service_base::shutdown(): snapshot under an + // acquired reference, cancel+close without the pool lock held; + // shutdown() sets shutting-down and takes the snapshot in one + // critical section, so a close that drops the last ref deletes + // rather than recycles. + std::vector live; + object_pool_.shutdown( + [&](posix_random_access_file* f) + { + acquire(f); + live.push_back(f); + }); + for (auto* f : live) { - impl->cancel(); - impl->close_file(); + f->cancel(); + f->close_file(); + release(f); } - file_ptrs_.clear(); - } - - void destroy_impl(posix_random_access_file& impl) - { - std::lock_guard lock(mutex_); - file_list_.remove(&impl); - file_ptrs_.erase(&impl); } void post(scheduler_op* op) @@ -143,12 +140,7 @@ class BOOST_COROSIO_DECL posix_random_access_file_service final private: scheduler* sched_; thread_pool_ref pool_; - std::mutex mutex_; - intrusive_list file_list_; - std::unordered_map< - posix_random_access_file*, - std::shared_ptr> - file_ptrs_; + object_pool object_pool_; }; // --------------------------------------------------------------------------- @@ -194,12 +186,12 @@ posix_random_access_file::read_some_at( op->iovecs[i].iov_len = bufs[i].size(); } - op->h = h; - op->ex = ex; - op->ec_out = ec; - op->bytes_out = bytes_out; - op->file_ = this; - op->impl_ptr = this->shared_from_this(); + op->h = h; + op->ex = ex; + op->ec_out = ec; + op->bytes_out = bytes_out; + op->file_ = this; + op->object_ref_ = detail::object_ref(this); op->start(token); op->ex.on_work_started(); @@ -265,12 +257,12 @@ posix_random_access_file::write_some_at( op->iovecs[i].iov_len = bufs[i].size(); } - op->h = h; - op->ex = ex; - op->ec_out = ec; - op->bytes_out = bytes_out; - op->file_ = this; - op->impl_ptr = this->shared_from_this(); + op->h = h; + op->ex = ex; + op->ec_out = ec; + op->bytes_out = bytes_out; + op->file_ = this; + op->object_ref_ = detail::object_ref(this); op->start(token); op->ex.on_work_started(); @@ -356,6 +348,12 @@ posix_random_access_file::raf_op::do_work(pool_work_item* w) noexcept self->svc_.post(static_cast(op)); } +inline void +posix_random_access_file::retire() noexcept +{ + svc_.object_pool_.recycle(this); +} + } // namespace boost::corosio::detail #endif // BOOST_COROSIO_POSIX diff --git a/include/boost/corosio/native/detail/posix/posix_resolver.hpp b/include/boost/corosio/native/detail/posix/posix_resolver.hpp index a72010b79..0498b022d 100644 --- a/include/boost/corosio/native/detail/posix/posix_resolver.hpp +++ b/include/boost/corosio/native/detail/posix/posix_resolver.hpp @@ -57,11 +57,12 @@ Class Hierarchy --------------- - posix_resolver_service (execution_context service, one per context) - - Owns all posix_resolver instances via shared_ptr + - Owns all posix_resolver instances via a recycling object_pool - Stores scheduler* for posting completions - posix_resolver (one per resolver object) - Contains embedded resolve_op and reverse_resolve_op for reuse - - Uses shared_from_this to prevent premature destruction + - Each op holds an object_ref keepalive while its pool work is + in flight, preventing premature destruction - resolve_op (forward resolution state) - Uses getaddrinfo() to resolve host/service to endpoints - reverse_resolve_op (reverse resolution state) @@ -86,9 +87,12 @@ Shutdown -------- - The resolver service cancels all resolvers and clears the impl map. - The thread pool service shuts down separately via execution_context - service ordering, joining all worker threads. + The resolver service cancels all resolvers and releases the + service's own reference on each; the recycling pool frees (rather + than recycles) once each resolver's in-flight pool work also + releases its reference. The thread pool service shuts down + separately via execution_context service ordering, joining all + worker threads. */ namespace boost::corosio::detail { @@ -154,7 +158,6 @@ class posix_resolver_service; */ class posix_resolver final : public resolver::implementation - , public std::enable_shared_from_this , public intrusive_list::node { friend class posix_resolver_service; @@ -213,11 +216,34 @@ class posix_resolver final posix_resolver* resolver_ = nullptr; /// Prevent impl destruction while work is in flight. - std::shared_ptr ref_; + detail::object_ref ref_; }; explicit posix_resolver(posix_resolver_service& svc) noexcept; + /// Recycle into the owning service's pool. Defined out-of-line + /// after posix_resolver_service for its complete type. + void retire() noexcept override; + + /** Reset for recycling. + + Both ops' own `reset()` already clears `host`/`service`/ + `stored_host`/`stored_service` via `.clear()` (preserving their + allocated capacity — the whole point of recycling), run at the + top of the next `resolve()`/`reverse_resolve()`. refs_ cannot + reach zero while a resolve is in flight (the pool op's + `object_ref` holds a reference for the duration), so asserting + `stop_cb` disengaged on both ops is a precondition check, not + a defensive one. + + @pre refs_ == 0, no resolve in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(!op_.stop_cb); + BOOST_COROSIO_ASSERT(!reverse_op_.stop_cb); + } + std::coroutine_handle<> resolve( std::coroutine_handle<>, capy::executor_ref, diff --git a/include/boost/corosio/native/detail/posix/posix_resolver_service.hpp b/include/boost/corosio/native/detail/posix/posix_resolver_service.hpp index 83b90230a..7bcc216d2 100644 --- a/include/boost/corosio/native/detail/posix/posix_resolver_service.hpp +++ b/include/boost/corosio/native/detail/posix/posix_resolver_service.hpp @@ -17,9 +17,11 @@ #include #include +#include +#include #include -#include +#include namespace boost::corosio::detail { @@ -32,6 +34,8 @@ class BOOST_COROSIO_DECL posix_resolver_service final : public capy::execution_context::service , public io_object::io_service { + friend class posix_resolver; + public: using key_type = posix_resolver_service; @@ -52,11 +56,10 @@ class BOOST_COROSIO_DECL posix_resolver_service final { auto& impl = static_cast(*p); impl.cancel(); - destroy_impl(impl); + release(&impl); } void shutdown() override; - void destroy_impl(posix_resolver& impl); void post(scheduler_op* op); @@ -89,10 +92,7 @@ class BOOST_COROSIO_DECL posix_resolver_service final private: scheduler* sched_; thread_pool_ref pool_; - std::mutex mutex_; - intrusive_list resolver_list_; - std::unordered_map> - resolver_ptrs_; + object_pool object_pool_; }; // --------------------------------------------------------------------------- @@ -271,7 +271,7 @@ posix_resolver::resolve_op::operator()() // Hold the keepalive across the dispatch: it may be the last // reference to the implementation this op is embedded in. - auto prevent_destroy = std::move(impl_ptr); + auto prevent_destroy = std::move(object_ref_); ex.on_work_finished(); cont.h = h; dispatch_coro(ex, cont).resume(); @@ -283,7 +283,7 @@ posix_resolver::resolve_op::destroy() stop_cb.reset(); auto local_ex = ex; // May destroy the implementation, and with it this op. - impl_ptr.reset(); + object_ref_.reset(); local_ex.on_work_finished(); } @@ -328,7 +328,7 @@ posix_resolver::reverse_resolve_op::operator()() // Hold the keepalive across the dispatch: it may be the last // reference to the implementation this op is embedded in. - auto prevent_destroy = std::move(impl_ptr); + auto prevent_destroy = std::move(object_ref_); ex.on_work_finished(); cont.h = h; dispatch_coro(ex, cont).resume(); @@ -340,7 +340,7 @@ posix_resolver::reverse_resolve_op::destroy() stop_cb.reset(); auto local_ex = ex; // May destroy the implementation, and with it this op. - impl_ptr.reset(); + object_ref_.reset(); local_ex.on_work_finished(); } @@ -380,7 +380,7 @@ posix_resolver::resolve( // Prevent impl destruction while work is in flight resolve_pool_op_.resolver_ = this; - resolve_pool_op_.ref_ = this->shared_from_this(); + resolve_pool_op_.ref_ = detail::object_ref(this); resolve_pool_op_.func_ = &posix_resolver::do_resolve_work; if (auto pec = svc_.pool().post(&resolve_pool_op_)) { @@ -430,7 +430,7 @@ posix_resolver::reverse_resolve( // Prevent impl destruction while work is in flight reverse_pool_op_.resolver_ = this; - reverse_pool_op_.ref_ = this->shared_from_this(); + reverse_pool_op_.ref_ = detail::object_ref(this); reverse_pool_op_.func_ = &posix_resolver::do_reverse_resolve_work; if (auto pec = svc_.pool().post(&reverse_pool_op_)) { @@ -492,7 +492,7 @@ posix_resolver::do_resolve_work(pool_work_item* w) noexcept // Hand the keepalive to the op: the completion waits in the // scheduler's queue, and the implementation embedding it must // outlive that wait. Nothing may touch *self after the post. - self->op_.impl_ptr = std::move(pw->ref_); + self->op_.object_ref_ = std::move(pw->ref_); self->svc_.post(&self->op_); } @@ -543,7 +543,7 @@ posix_resolver::do_reverse_resolve_work(pool_work_item* w) noexcept // Hand the keepalive to the op: the completion waits in the // scheduler's queue, and the implementation embedding it must // outlive that wait. Nothing may touch *self after the post. - self->reverse_op_.impl_ptr = std::move(pw->ref_); + self->reverse_op_.object_ref_ = std::move(pw->ref_); self->svc_.post(&self->reverse_op_); } @@ -552,48 +552,42 @@ posix_resolver::do_reverse_resolve_work(pool_work_item* w) noexcept inline void posix_resolver_service::shutdown() { - std::lock_guard lock(mutex_); - - // Cancel all resolvers (sets cancelled flag checked by pool threads) - for (auto* impl = resolver_list_.pop_front(); impl != nullptr; - impl = resolver_list_.pop_front()) + // See uring_socket_service_base::shutdown(): snapshot under an + // acquired reference, cancel without the pool lock held; shutdown() + // sets shutting-down and takes the snapshot in one critical + // section, so a cancel that drops the last ref deletes rather than + // recycles. The thread pool service shuts down separately via + // execution_context ordering. + std::vector live; + object_pool_.shutdown( + [&](posix_resolver* r) + { + acquire(r); + live.push_back(r); + }); + for (auto* r : live) { - impl->cancel(); + r->cancel(); + release(r); } - - // Clear the map which releases shared_ptrs. - // The thread pool service shuts down separately via - // execution_context service ordering. - resolver_ptrs_.clear(); } inline io_object::implementation* posix_resolver_service::construct() { - auto ptr = std::make_shared(*this); - auto* impl = ptr.get(); - - { - std::lock_guard lock(mutex_); - resolver_list_.push_back(impl); - resolver_ptrs_[impl] = std::move(ptr); - } - - return impl; + return object_pool_.acquire(*this); } inline void -posix_resolver_service::destroy_impl(posix_resolver& impl) +posix_resolver_service::post(scheduler_op* op) { - std::lock_guard lock(mutex_); - resolver_list_.remove(&impl); - resolver_ptrs_.erase(&impl); + sched_->post(op); } inline void -posix_resolver_service::post(scheduler_op* op) +posix_resolver::retire() noexcept { - sched_->post(op); + svc_.object_pool_.recycle(this); } } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/posix/posix_signal.hpp b/include/boost/corosio/native/detail/posix/posix_signal.hpp index eef8a9cce..7ba5fc133 100644 --- a/include/boost/corosio/native/detail/posix/posix_signal.hpp +++ b/include/boost/corosio/native/detail/posix/posix_signal.hpp @@ -123,6 +123,31 @@ class posix_signal final public: explicit posix_signal(posix_signal_service& svc) noexcept; + /// Recycle into the owning service's pool. Defined out-of-line + /// after posix_signal_service for its complete type. + void retire() noexcept override; + + /** Reset for recycling. + + `destroy()` already calls `clear()` (empties the registration + list) and `disarm_stop()` before the refcount reaches zero, so + those are asserted rather than re-cleared. `destroy()` also + calls `cancel()`, which deliberately latches `cancelled_` to + abort any wait still unwinding — that latch is this object's + to clear for its next, unrelated use, not something reset() + can skip by asserting it already false. + + @pre refs_ == 0, registration list empty, no wait in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(signals_ == nullptr); + BOOST_COROSIO_ASSERT(!waiting_); + BOOST_COROSIO_ASSERT(!stop_cb_); + cancelled_ = false; + token_cancelled_ = false; + } + std::coroutine_handle<> wait( std::coroutine_handle<>, capy::executor_ref, diff --git a/include/boost/corosio/native/detail/posix/posix_signal_service.hpp b/include/boost/corosio/native/detail/posix/posix_signal_service.hpp index 420b24df5..847db35e7 100644 --- a/include/boost/corosio/native/detail/posix/posix_signal_service.hpp +++ b/include/boost/corosio/native/detail/posix/posix_signal_service.hpp @@ -18,6 +18,8 @@ #include #include +#include +#include #include #include #include @@ -25,6 +27,7 @@ #include #include +#include #include #include @@ -65,7 +68,7 @@ 2. posix_signal_service (one per execution_context) - Maintains registrations_[] table indexed by signal number - Each slot is a doubly-linked list of signal_registrations for that signal - - Also maintains impl_list_ of all posix_signal objects it owns + - Also owns a recycling pool of all posix_signal objects it owns 3. posix_signal (one per signal_set) - Owns a singly-linked list (sorted by signal number) of signal_registrations @@ -154,6 +157,8 @@ class BOOST_COROSIO_DECL posix_signal_service final : public capy::execution_context::service , public io_object::io_service { + friend class posix_signal; + public: using key_type = posix_signal_service; @@ -171,7 +176,7 @@ class BOOST_COROSIO_DECL posix_signal_service final [[maybe_unused]] auto n = impl.clear(); impl.disarm_stop(); impl.cancel(); - destroy_impl(impl); + release(&impl); } /** Shut down the service. @@ -181,8 +186,6 @@ class BOOST_COROSIO_DECL posix_signal_service final */ void shutdown() override; - void destroy_impl(posix_signal& impl); - std::error_code add_signal( posix_signal& impl, int signal_number, signal_set::flags_t flags); @@ -233,7 +236,7 @@ class BOOST_COROSIO_DECL posix_signal_service final std::mutex reader_mutex_; bool reader_registered_ = false; - intrusive_list impl_list_; + object_pool pool_; // Per-signal registration table signal_registration* registrations_[max_signal_number]; @@ -526,12 +529,25 @@ inline posix_signal_service::~posix_signal_service() inline void posix_signal_service::shutdown() { - // Collected under the locks below and deleted after they are released: - // ~posix_signal destroys an armed stop_cb_, and ~stop_callback blocks - // until a concurrently running token_canceller returns -- which takes - // mutex_. Deleting while still holding mutex_ would self-deadlock the - // same way disarm_stop() would if called inside the locked loop. - intrusive_list doomed; + // Collected under the locks below and released after they are + // released: ~posix_signal destroys an armed stop_cb_, and + // ~stop_callback blocks until a concurrently running + // token_canceller returns -- which takes mutex_. Releasing while + // still holding mutex_ would self-deadlock the same way + // disarm_stop() would if called inside the locked loop. + // + // The acquire()+release() pair below nets to zero change on each + // impl's refs_ -- it exists only so the lock-held loop above can + // safely walk `doomed` without a concurrent recycle() tearing one + // down mid-walk. It does NOT drop these impls to zero: that still + // only happens when the handle's own destroy() runs, which this + // context-wide shutdown does not do (shutdown() cancels/clears + // registrations service-wide; it never calls destroy() per impl). + // Every impl here survives at refs_ == 1 (the service's own floor + // reference) until ~object_pool()'s unconditional sweep deletes + // whatever is still in `live_`, bypassing refs_ entirely -- not + // through this release() reaching zero. + std::vector doomed; { posix_signal_detail::signal_state* state = @@ -539,8 +555,13 @@ posix_signal_service::shutdown() std::lock_guard state_lock(state->mutex); std::lock_guard lock(mutex_); - for (auto* impl = impl_list_.pop_front(); impl != nullptr; - impl = impl_list_.pop_front()) + pool_.shutdown( + [&](posix_signal* impl) + { + acquire(impl); + doomed.push_back(impl); + }); + for (auto* impl : doomed) { while (auto* reg = impl->signals_) { @@ -568,47 +589,31 @@ posix_signal_service::shutdown() impl->signals_ = reg->next_in_set; delete reg; } - doomed.push_back(impl); } - // Every live registration hung off an implementation in impl_list_, - // so the whole table goes stale at once and can be dropped wholesale - // rather than node by node. It has to be dropped: deliver_signal() - // walks this service until the destructor unlinks it from the global - // list. + // Every live registration hung off an implementation this pool + // owns, so the whole table goes stale at once and can be dropped + // wholesale rather than node by node. It has to be dropped: + // deliver_signal() walks this service until the destructor + // unlinks it from the global list. for (int i = 0; i < max_signal_number; ++i) registrations_[i] = nullptr; } - for (auto* impl = doomed.pop_front(); impl != nullptr; - impl = doomed.pop_front()) - { - delete impl; - } + for (auto* impl : doomed) + release(impl); } inline io_object::implementation* posix_signal_service::construct() { - auto* impl = new posix_signal(*this); - - { - std::lock_guard lock(mutex_); - impl_list_.push_back(impl); - } - - return impl; + return pool_.acquire(*this); } inline void -posix_signal_service::destroy_impl(posix_signal& impl) +posix_signal::retire() noexcept { - { - std::lock_guard lock(mutex_); - impl_list_.remove(&impl); - } - - delete &impl; + svc_.pool_.recycle(this); } inline std::error_code diff --git a/include/boost/corosio/native/detail/posix/posix_stream_file.hpp b/include/boost/corosio/native/detail/posix/posix_stream_file.hpp index 0f0d6c092..fcc7e1d01 100644 --- a/include/boost/corosio/native/detail/posix/posix_stream_file.hpp +++ b/include/boost/corosio/native/detail/posix/posix_stream_file.hpp @@ -56,7 +56,8 @@ to a shared thread pool, with completion posted back to the scheduler. This follows the same pattern as posix_resolver: pool_work_item for - dispatch, scheduler_op for completion, shared_from_this for lifetime. + dispatch, scheduler_op for completion, an object_ref keepalive on + each op for lifetime while its pool work is in flight. Completion Flow --------------- @@ -84,7 +85,6 @@ class posix_stream_file_service; */ class posix_stream_file final : public stream_file::implementation - , public std::enable_shared_from_this , public intrusive_list::node { friend class posix_stream_file_service; @@ -117,7 +117,7 @@ class posix_stream_file final is_read = false; cancelled.store(false, std::memory_order_relaxed); stop_cb.reset(); - impl_ptr.reset(); + object_ref_.reset(); ec_out = nullptr; bytes_out = nullptr; } @@ -130,11 +130,31 @@ class posix_stream_file final struct pool_op : pool_work_item { posix_stream_file* file_ = nullptr; - std::shared_ptr ref_; + detail::object_ref ref_; }; explicit posix_stream_file(posix_stream_file_service& svc) noexcept; + /// Recycle into the owning service's pool. Defined out-of-line + /// after posix_stream_file_service for its complete type. + void retire() noexcept override; + + /** Reset op slots for recycling. + + `close_file()` already drove fd_ to its closed value before + the refcount reached zero; each op's own `reset()` runs again + before its next use, so only the `stop_cb` invariant is worth + asserting here. + + @pre refs_ == 0, fd closed, no op in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(fd_ == -1); + BOOST_COROSIO_ASSERT(!read_op_.stop_cb); + BOOST_COROSIO_ASSERT(!write_op_.stop_cb); + } + // -- io_stream::implementation -- std::coroutine_handle<> read_some( @@ -385,10 +405,10 @@ posix_stream_file::file_op::operator()() errn != 0 ? make_err(errn) : std::error_code{}, is_read, bytes_transferred, /*empty_buffer=*/false); - // Move impl_ptr to a local so members remain valid through - // dispatch — impl_ptr may be the last shared_ptr keeping - // the parent posix_stream_file (which embeds this file_op) alive. - auto prevent_destroy = std::move(impl_ptr); + // Move object_ref_ to a local so members remain valid through + // dispatch — this may be the last reference keeping the parent + // posix_stream_file (which embeds this file_op) alive. + auto prevent_destroy = std::move(object_ref_); ex.on_work_finished(); cont.h = h; dispatch_coro(ex, cont).resume(); @@ -399,7 +419,7 @@ posix_stream_file::file_op::destroy() { stop_cb.reset(); auto local_ex = ex; - impl_ptr.reset(); + object_ref_.reset(); local_ex.on_work_finished(); } diff --git a/include/boost/corosio/native/detail/posix/posix_stream_file_service.hpp b/include/boost/corosio/native/detail/posix/posix_stream_file_service.hpp index 268143474..89ac9ee3a 100644 --- a/include/boost/corosio/native/detail/posix/posix_stream_file_service.hpp +++ b/include/boost/corosio/native/detail/posix/posix_stream_file_service.hpp @@ -17,10 +17,11 @@ #include #include #include +#include +#include #include -#include -#include +#include namespace boost::corosio::detail { @@ -31,6 +32,8 @@ namespace boost::corosio::detail { */ class BOOST_COROSIO_DECL posix_stream_file_service final : public file_service { + friend class posix_stream_file; + public: explicit posix_stream_file_service(capy::execution_context& ctx) : sched_(&get_scheduler(ctx)) @@ -46,16 +49,7 @@ class BOOST_COROSIO_DECL posix_stream_file_service final : public file_service io_object::implementation* construct() override { - auto ptr = std::make_shared(*this); - auto* impl = ptr.get(); - - { - std::lock_guard lock(mutex_); - file_list_.push_back(impl); - file_ptrs_[impl] = std::move(ptr); - } - - return impl; + return object_pool_.acquire(*this); } void destroy(io_object::implementation* p) override @@ -63,7 +57,7 @@ class BOOST_COROSIO_DECL posix_stream_file_service final : public file_service auto& impl = static_cast(*p); impl.cancel(); impl.close_file(); - destroy_impl(impl); + release(&impl); } void close(io_object::handle& h) override @@ -90,21 +84,24 @@ class BOOST_COROSIO_DECL posix_stream_file_service final : public file_service void shutdown() override { - std::lock_guard lock(mutex_); - for (auto* impl = file_list_.pop_front(); impl != nullptr; - impl = file_list_.pop_front()) + // See uring_socket_service_base::shutdown(): snapshot under an + // acquired reference, cancel+close without the pool lock held; + // shutdown() sets shutting-down and takes the snapshot in one + // critical section, so a close that drops the last ref deletes + // rather than recycles. + std::vector live; + object_pool_.shutdown( + [&](posix_stream_file* f) + { + acquire(f); + live.push_back(f); + }); + for (auto* f : live) { - impl->cancel(); - impl->close_file(); + f->cancel(); + f->close_file(); + release(f); } - file_ptrs_.clear(); - } - - void destroy_impl(posix_stream_file& impl) - { - std::lock_guard lock(mutex_); - file_list_.remove(&impl); - file_ptrs_.erase(&impl); } void post(scheduler_op* op) @@ -143,10 +140,7 @@ class BOOST_COROSIO_DECL posix_stream_file_service final : public file_service private: scheduler* sched_; thread_pool_ref pool_; - std::mutex mutex_; - intrusive_list file_list_; - std::unordered_map> - file_ptrs_; + object_pool object_pool_; }; // --------------------------------------------------------------------------- @@ -201,7 +195,7 @@ posix_stream_file::read_some( op.ex.on_work_started(); read_pool_op_.file_ = this; - read_pool_op_.ref_ = this->shared_from_this(); + read_pool_op_.ref_ = detail::object_ref(this); read_pool_op_.func_ = &posix_stream_file::do_read_work; if (auto pec = svc_.pool().post(&read_pool_op_)) { @@ -251,7 +245,7 @@ posix_stream_file::do_read_work(pool_work_item* w) noexcept } } - op.impl_ptr = std::move(pw->ref_); + op.object_ref_ = std::move(pw->ref_); self->svc_.post(&op); } @@ -303,7 +297,7 @@ posix_stream_file::write_some( op.ex.on_work_started(); write_pool_op_.file_ = this; - write_pool_op_.ref_ = this->shared_from_this(); + write_pool_op_.ref_ = detail::object_ref(this); write_pool_op_.func_ = &posix_stream_file::do_write_work; if (auto pec = svc_.pool().post(&write_pool_op_)) { @@ -353,10 +347,16 @@ posix_stream_file::do_write_work(pool_work_item* w) noexcept } } - op.impl_ptr = std::move(pw->ref_); + op.object_ref_ = std::move(pw->ref_); self->svc_.post(&op); } +inline void +posix_stream_file::retire() noexcept +{ + svc_.object_pool_.recycle(this); +} + } // namespace boost::corosio::detail #endif // BOOST_COROSIO_POSIX diff --git a/include/boost/corosio/native/detail/reactor/reactor_acceptor.hpp b/include/boost/corosio/native/detail/reactor/reactor_acceptor.hpp index d89595b76..ed7d745b8 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_acceptor.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_acceptor.hpp @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -58,7 +59,6 @@ template< class Endpoint = endpoint> class reactor_acceptor : public ImplBase - , public std::enable_shared_from_this , public intrusive_list::node { friend Derived; @@ -258,6 +258,33 @@ class reactor_acceptor /** Release the acceptor without closing the fd. */ native_handle_type do_release_socket() noexcept; + /** Reset descriptor state for recycling. + + `close_socket()` already drove fd_ and every parked op pointer + to their closed state before the refcount reached zero, so + this only asserts those invariants — see + reactor_basic_socket::reuse() for the same rationale. + + @pre refs_ == 0, fd closed and deregistered, no op in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(fd_ == -1); + BOOST_COROSIO_ASSERT(desc_state_.fd == -1); + BOOST_COROSIO_ASSERT(desc_state_.read_op == nullptr); + BOOST_COROSIO_ASSERT(desc_state_.wait_read_op == nullptr); + BOOST_COROSIO_ASSERT(desc_state_.wait_write_op == nullptr); + BOOST_COROSIO_ASSERT(desc_state_.wait_error_op == nullptr); + BOOST_COROSIO_ASSERT(!desc_state_.object_ref_); + BOOST_COROSIO_ASSERT(!desc_state_.is_enqueued_.load( + std::memory_order_relaxed)); + BOOST_COROSIO_ASSERT(!acc_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_wr_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_er_.stop_cb); + BOOST_COROSIO_ASSERT(local_endpoint_ == Endpoint{}); + } + /** Bind the acceptor socket to an endpoint. Caches the resolved local endpoint (including ephemeral @@ -300,10 +327,6 @@ reactor_acceptor< ImplBase, Endpoint>::cancel_single_op(Op& op) noexcept { - auto self = this->weak_from_this().lock(); - if (!self) - return; - op.request_cancel(); reactor_op_base* claimed = nullptr; @@ -320,7 +343,7 @@ reactor_acceptor< } if (claimed) { - op.impl_ptr = self; + op.object_ref_ = detail::object_ref(this); svc_.post(&op); svc_.work_finished(); } @@ -372,44 +395,40 @@ reactor_acceptor< ImplBase, Endpoint>::do_close_socket() noexcept { - auto self = this->weak_from_this().lock(); - if (self) + acc_.request_cancel(); + wait_rd_.request_cancel(); + wait_wr_.request_cancel(); + wait_er_.request_cancel(); + + reactor_op_base* claimed_acc = nullptr; + reactor_op_base* claimed_wr = nullptr; + reactor_op_base* claimed_ww = nullptr; + reactor_op_base* claimed_we = nullptr; { - acc_.request_cancel(); - wait_rd_.request_cancel(); - wait_wr_.request_cancel(); - wait_er_.request_cancel(); - - reactor_op_base* claimed_acc = nullptr; - reactor_op_base* claimed_wr = nullptr; - reactor_op_base* claimed_ww = nullptr; - reactor_op_base* claimed_we = nullptr; + std::lock_guard lock(desc_state_.mutex); + claimed_acc = std::exchange(desc_state_.read_op, nullptr); + claimed_wr = std::exchange(desc_state_.wait_read_op, nullptr); + claimed_ww = std::exchange(desc_state_.wait_write_op, nullptr); + claimed_we = std::exchange(desc_state_.wait_error_op, nullptr); + desc_state_.read_ready = false; + desc_state_.write_ready = false; + + if (desc_state_.is_enqueued_.load(std::memory_order_acquire)) + desc_state_.object_ref_ = detail::object_ref(this); + } + + auto repost = [&](reactor_op_base* claimed, reactor_op_base& op) { + if (claimed) { - std::lock_guard lock(desc_state_.mutex); - claimed_acc = std::exchange(desc_state_.read_op, nullptr); - claimed_wr = std::exchange(desc_state_.wait_read_op, nullptr); - claimed_ww = std::exchange(desc_state_.wait_write_op, nullptr); - claimed_we = std::exchange(desc_state_.wait_error_op, nullptr); - desc_state_.read_ready = false; - desc_state_.write_ready = false; - - if (desc_state_.is_enqueued_.load(std::memory_order_acquire)) - desc_state_.impl_ref_ = self; + op.object_ref_ = detail::object_ref(this); + svc_.post(&op); + svc_.work_finished(); } - - auto repost = [&](reactor_op_base* claimed, reactor_op_base& op) { - if (claimed) - { - op.impl_ptr = self; - svc_.post(&op); - svc_.work_finished(); - } - }; - repost(claimed_acc, acc_); - repost(claimed_wr, wait_rd_); - repost(claimed_ww, wait_wr_); - repost(claimed_we, wait_er_); - } + }; + repost(claimed_acc, acc_); + repost(claimed_wr, wait_rd_); + repost(claimed_ww, wait_wr_); + repost(claimed_we, wait_er_); if (fd_ >= 0) { @@ -445,44 +464,40 @@ reactor_acceptor< ImplBase, Endpoint>::do_release_socket() noexcept { - auto self = this->weak_from_this().lock(); - if (self) + acc_.request_cancel(); + wait_rd_.request_cancel(); + wait_wr_.request_cancel(); + wait_er_.request_cancel(); + + reactor_op_base* claimed_acc = nullptr; + reactor_op_base* claimed_wr = nullptr; + reactor_op_base* claimed_ww = nullptr; + reactor_op_base* claimed_we = nullptr; { - acc_.request_cancel(); - wait_rd_.request_cancel(); - wait_wr_.request_cancel(); - wait_er_.request_cancel(); - - reactor_op_base* claimed_acc = nullptr; - reactor_op_base* claimed_wr = nullptr; - reactor_op_base* claimed_ww = nullptr; - reactor_op_base* claimed_we = nullptr; + std::lock_guard lock(desc_state_.mutex); + claimed_acc = std::exchange(desc_state_.read_op, nullptr); + claimed_wr = std::exchange(desc_state_.wait_read_op, nullptr); + claimed_ww = std::exchange(desc_state_.wait_write_op, nullptr); + claimed_we = std::exchange(desc_state_.wait_error_op, nullptr); + desc_state_.read_ready = false; + desc_state_.write_ready = false; + + if (desc_state_.is_enqueued_.load(std::memory_order_acquire)) + desc_state_.object_ref_ = detail::object_ref(this); + } + + auto repost = [&](reactor_op_base* claimed, reactor_op_base& op) { + if (claimed) { - std::lock_guard lock(desc_state_.mutex); - claimed_acc = std::exchange(desc_state_.read_op, nullptr); - claimed_wr = std::exchange(desc_state_.wait_read_op, nullptr); - claimed_ww = std::exchange(desc_state_.wait_write_op, nullptr); - claimed_we = std::exchange(desc_state_.wait_error_op, nullptr); - desc_state_.read_ready = false; - desc_state_.write_ready = false; - - if (desc_state_.is_enqueued_.load(std::memory_order_acquire)) - desc_state_.impl_ref_ = self; + op.object_ref_ = detail::object_ref(this); + svc_.post(&op); + svc_.work_finished(); } - - auto repost = [&](reactor_op_base* claimed, reactor_op_base& op) { - if (claimed) - { - op.impl_ptr = self; - svc_.post(&op); - svc_.work_finished(); - } - }; - repost(claimed_acc, acc_); - repost(claimed_wr, wait_rd_); - repost(claimed_ww, wait_wr_); - repost(claimed_we, wait_er_); - } + }; + repost(claimed_acc, acc_); + repost(claimed_wr, wait_rd_); + repost(claimed_ww, wait_wr_); + repost(claimed_we, wait_er_); native_handle_type released = fd_; @@ -606,7 +621,7 @@ reactor_acceptor< op.ec_out = ec; op.fd = this->fd_; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(ENOTSUP, 0); svc_.post(&op); return std::noop_coroutine(); @@ -637,7 +652,7 @@ reactor_acceptor< op.ec_out = ec; op.fd = this->fd_; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); // A listener's readiness can predate the wait: an adopted or // shared descriptor has history the reactor never saw, and an diff --git a/include/boost/corosio/native/detail/reactor/reactor_acceptor_service.hpp b/include/boost/corosio/native/detail/reactor/reactor_acceptor_service.hpp index 5063161e0..7bd11e1c4 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_acceptor_service.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_acceptor_service.hpp @@ -11,12 +11,12 @@ #define BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_ACCEPTOR_SERVICE_HPP #include +#include #include #include #include #include -#include namespace boost::corosio::detail { @@ -45,6 +45,13 @@ template< class reactor_acceptor_service : public ServiceBase { friend Derived; + + // The intermediate CRTP template below (not Impl itself) is what + // actually reaches into state_->pool_ from retire() -- see + // reactor_socket_finals.hpp. + template + friend class reactor_acceptor_impl; + using state_type = reactor_service_state; protected: @@ -62,31 +69,23 @@ class reactor_acceptor_service : public ServiceBase void shutdown() override { - std::lock_guard lock(state_->mutex_); + state_->pool_.shutdown([](Impl* impl) { impl->close_socket(); }); - while (auto* impl = state_->impl_list_.pop_front()) - impl->close_socket(); + // See reactor_socket_service::shutdown(): queued ops hold their + // own reference; the scheduler's drain releases them after us, + // and shutting-down mode deletes rather than recycles at zero. } io_object::implementation* construct() override { - auto impl = std::make_shared(static_cast(*this)); - auto* raw = impl.get(); - - std::lock_guard lock(state_->mutex_); - state_->impl_ptrs_.emplace(raw, std::move(impl)); - state_->impl_list_.push_back(raw); - - return raw; + return state_->pool_.acquire(static_cast(*this)); } void destroy(io_object::implementation* impl) override { auto* typed = static_cast(impl); typed->close_socket(); - std::lock_guard lock(state_->mutex_); - state_->impl_list_.remove(typed); - state_->impl_ptrs_.erase(typed); + release(typed); } void close(io_object::handle& h) override diff --git a/include/boost/corosio/native/detail/reactor/reactor_backend.hpp b/include/boost/corosio/native/detail/reactor/reactor_backend.hpp index bf7b6ed9c..59adf8a26 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_backend.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_backend.hpp @@ -128,14 +128,14 @@ reactor_acceptor_impl< op.peer_storage = peer_storage; op.peer_addrlen = peer_addrlen; op.complete(0, 0); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->svc_.post(&op); return std::noop_coroutine(); } if (errno == EAGAIN || errno == EWOULDBLOCK) { - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->svc_.work_started(); std::lock_guard lock(this->desc_state_.mutex); @@ -162,7 +162,7 @@ reactor_acceptor_impl< } op.complete(errno, 0); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->svc_.post(&op); return std::noop_coroutine(); } diff --git a/include/boost/corosio/native/detail/reactor/reactor_basic_socket.hpp b/include/boost/corosio/native/detail/reactor/reactor_basic_socket.hpp index b3165cb00..ba2b78279 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_basic_socket.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_basic_socket.hpp @@ -10,6 +10,7 @@ #ifndef BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_BASIC_SOCKET_HPP #define BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_BASIC_SOCKET_HPP +#include #include #include #include @@ -174,6 +175,33 @@ class reactor_basic_socket Returns the fd so the caller can take ownership. */ native_handle_type do_release_socket() noexcept; + + /** Reset descriptor state for recycling. + + Called by the owning service's `construct()` when popping this + impl back off the free list. `close_socket()` already drove fd_, + the descriptor's registration, and every parked op pointer to + their closed state before the refcount reached zero, so this + only asserts those invariants rather than re-clearing them — + a non-null parked op or a lingering `object_ref_` here would mean + a reference survived close, which is the real bug to catch. + + @pre refs_ == 0, fd closed and deregistered, no op in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(fd_ == -1); + BOOST_COROSIO_ASSERT(desc_state_.fd == -1); + BOOST_COROSIO_ASSERT(desc_state_.read_op == nullptr); + BOOST_COROSIO_ASSERT(desc_state_.write_op == nullptr); + BOOST_COROSIO_ASSERT(desc_state_.connect_op == nullptr); + BOOST_COROSIO_ASSERT(desc_state_.wait_read_op == nullptr); + BOOST_COROSIO_ASSERT(desc_state_.wait_write_op == nullptr); + BOOST_COROSIO_ASSERT(desc_state_.wait_error_op == nullptr); + BOOST_COROSIO_ASSERT(!desc_state_.object_ref_); + BOOST_COROSIO_ASSERT(!desc_state_.is_enqueued_.load( + std::memory_order_relaxed)); + } }; template< @@ -238,10 +266,6 @@ void reactor_basic_socket:: cancel_single_op(Op& op) noexcept { - auto self = this->weak_from_this().lock(); - if (!self) - return; - op.request_cancel(); auto* d = static_cast(this); @@ -262,7 +286,7 @@ reactor_basic_socket:: } if (claimed) { - op.impl_ptr = self; + op.object_ref_ = detail::object_ref(this); svc_.post(&op); svc_.work_finished(); } @@ -279,10 +303,6 @@ void reactor_basic_socket:: do_cancel() noexcept { - auto self = this->weak_from_this().lock(); - if (!self) - return; - auto* d = static_cast(this); d->for_each_op([](auto& op) { op.request_cancel(); }); @@ -311,7 +331,7 @@ reactor_basic_socket:: for (int i = 0; i < count; ++i) { - claimed[i].base->impl_ptr = self; + claimed[i].base->object_ref_ = detail::object_ref(this); svc_.post(claimed[i].base); svc_.work_finished(); } @@ -327,44 +347,40 @@ void reactor_basic_socket:: do_close_socket() noexcept { - auto self = this->weak_from_this().lock(); - if (self) - { - auto* d = static_cast(this); + auto* d = static_cast(this); - d->for_each_op([](auto& op) { op.request_cancel(); }); + d->for_each_op([](auto& op) { op.request_cancel(); }); - struct claimed_entry - { - reactor_op_base* base = nullptr; - }; - claimed_entry claimed[8]; - int count = 0; + struct claimed_entry + { + reactor_op_base* base = nullptr; + }; + claimed_entry claimed[8]; + int count = 0; - { - std::lock_guard lock(desc_state_.mutex); - d->for_each_desc_entry( - [&](auto& /*op*/, reactor_op_base*& desc_slot) { - auto* c = std::exchange(desc_slot, nullptr); - if (c) - { - claimed[count].base = c; - ++count; - } - }); - desc_state_.read_ready = false; - desc_state_.write_ready = false; - - if (desc_state_.is_enqueued_.load(std::memory_order_acquire)) - desc_state_.impl_ref_ = self; - } + { + std::lock_guard lock(desc_state_.mutex); + d->for_each_desc_entry( + [&](auto& /*op*/, reactor_op_base*& desc_slot) { + auto* c = std::exchange(desc_slot, nullptr); + if (c) + { + claimed[count].base = c; + ++count; + } + }); + desc_state_.read_ready = false; + desc_state_.write_ready = false; + + if (desc_state_.is_enqueued_.load(std::memory_order_acquire)) + desc_state_.object_ref_ = detail::object_ref(this); + } - for (int i = 0; i < count; ++i) - { - claimed[i].base->impl_ptr = self; - svc_.post(claimed[i].base); - svc_.work_finished(); - } + for (int i = 0; i < count; ++i) + { + claimed[i].base->object_ref_ = detail::object_ref(this); + svc_.post(claimed[i].base); + svc_.work_finished(); } if (fd_ >= 0) @@ -392,44 +408,40 @@ reactor_basic_socket:: do_release_socket() noexcept { // Cancel pending ops (same as do_close_socket) - auto self = this->weak_from_this().lock(); - if (self) - { - auto* d = static_cast(this); + auto* d = static_cast(this); - d->for_each_op([](auto& op) { op.request_cancel(); }); + d->for_each_op([](auto& op) { op.request_cancel(); }); - struct claimed_entry - { - reactor_op_base* base = nullptr; - }; - claimed_entry claimed[8]; - int count = 0; + struct claimed_entry + { + reactor_op_base* base = nullptr; + }; + claimed_entry claimed[8]; + int count = 0; - { - std::lock_guard lock(desc_state_.mutex); - d->for_each_desc_entry( - [&](auto& /*op*/, reactor_op_base*& desc_slot) { - auto* c = std::exchange(desc_slot, nullptr); - if (c) - { - claimed[count].base = c; - ++count; - } - }); - desc_state_.read_ready = false; - desc_state_.write_ready = false; - - if (desc_state_.is_enqueued_.load(std::memory_order_acquire)) - desc_state_.impl_ref_ = self; - } + { + std::lock_guard lock(desc_state_.mutex); + d->for_each_desc_entry( + [&](auto& /*op*/, reactor_op_base*& desc_slot) { + auto* c = std::exchange(desc_slot, nullptr); + if (c) + { + claimed[count].base = c; + ++count; + } + }); + desc_state_.read_ready = false; + desc_state_.write_ready = false; + + if (desc_state_.is_enqueued_.load(std::memory_order_acquire)) + desc_state_.object_ref_ = detail::object_ref(this); + } - for (int i = 0; i < count; ++i) - { - claimed[i].base->impl_ptr = self; - svc_.post(claimed[i].base); - svc_.work_finished(); - } + for (int i = 0; i < count; ++i) + { + claimed[i].base->object_ref_ = detail::object_ref(this); + svc_.post(claimed[i].base); + svc_.work_finished(); } native_handle_type released = fd_; diff --git a/include/boost/corosio/native/detail/reactor/reactor_datagram_socket.hpp b/include/boost/corosio/native/detail/reactor/reactor_datagram_socket.hpp index b88afa9e9..ed43e0ebe 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_datagram_socket.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_datagram_socket.hpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -325,6 +326,30 @@ class reactor_datagram_socket return fd; } + /** Reset op slots and the cached peer for recycling. + + See reactor_stream_socket::reuse() — same rationale: op state + is overwritten by the next do_* call, `remote_endpoint_` and + every parked op pointer are already cleared by + `do_close_socket()`, so this only asserts `stop_cb` disengaged + on each slot. + + @pre refs_ == 0, fd closed and deregistered, no op in flight. + */ + void reuse() noexcept + { + base_type::reuse(); + BOOST_COROSIO_ASSERT(!conn_.stop_cb); + BOOST_COROSIO_ASSERT(!wr_.stop_cb); + BOOST_COROSIO_ASSERT(!rd_.stop_cb); + BOOST_COROSIO_ASSERT(!send_wr_.stop_cb); + BOOST_COROSIO_ASSERT(!recv_rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_wr_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_er_.stop_cb); + BOOST_COROSIO_ASSERT(remote_endpoint_ == Endpoint{}); + } + /** Shut down part or all of the full-duplex connection. Not an override — concrete backends forward here. @@ -497,7 +522,7 @@ reactor_datagram_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(err, bytes); this->svc_.post(&op); return std::noop_coroutine(); @@ -509,7 +534,7 @@ reactor_datagram_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->register_op( op, this->desc_state_.write_op, this->desc_state_.write_ready, true); @@ -567,7 +592,7 @@ reactor_datagram_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(0, 0); this->svc_.post(&op); return std::noop_coroutine(); @@ -619,7 +644,7 @@ reactor_datagram_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(err, bytes); this->svc_.post(&op); return std::noop_coroutine(); @@ -631,7 +656,7 @@ reactor_datagram_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->register_op( op, this->desc_state_.read_op, this->desc_state_.read_ready); @@ -708,7 +733,7 @@ reactor_datagram_socket< op.fd = this->fd_; op.target_endpoint = ep; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(err, 0); this->svc_.post(&op); return std::noop_coroutine(); @@ -800,7 +825,7 @@ reactor_datagram_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(err, bytes); this->svc_.post(&op); return std::noop_coroutine(); @@ -812,7 +837,7 @@ reactor_datagram_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->register_op( op, this->desc_state_.write_op, this->desc_state_.write_ready, true); @@ -868,7 +893,7 @@ reactor_datagram_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(0, 0); this->svc_.post(&op); return std::noop_coroutine(); @@ -912,7 +937,7 @@ reactor_datagram_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(err, bytes); this->svc_.post(&op); return std::noop_coroutine(); @@ -924,7 +949,7 @@ reactor_datagram_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->register_op( op, this->desc_state_.read_op, this->desc_state_.read_ready); @@ -1010,7 +1035,7 @@ reactor_datagram_socket< op.ec_out = ec; op.fd = this->fd_; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(perr, 0); this->svc_.post(&op); return std::noop_coroutine(); @@ -1023,7 +1048,7 @@ reactor_datagram_socket< op.ec_out = ec; op.fd = this->fd_; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); // Force register_op's ready path so the wait op re-probes under // the descriptor mutex before parking. An edge consumed between diff --git a/include/boost/corosio/native/detail/reactor/reactor_descriptor_state.hpp b/include/boost/corosio/native/detail/reactor/reactor_descriptor_state.hpp index 4dda0eb76..8789a68ab 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_descriptor_state.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_descriptor_state.hpp @@ -88,7 +88,7 @@ struct reactor_descriptor_state : scheduler_op reactor_scheduler const* scheduler_ = nullptr; /// Prevents impl destruction while queued in the scheduler. - std::shared_ptr impl_ref_; + detail::object_ref object_ref_; /// Add ready events atomically. /// Release pairs with the consumer's acquire exchange on @@ -106,11 +106,11 @@ struct reactor_descriptor_state : scheduler_op } /// Destroy without invoking. - /// Called during scheduler::shutdown() drain. Clear impl_ref_ to break + /// Called during scheduler::shutdown() drain. Clear object_ref_ to break /// the self-referential cycle set by close_socket(). void destroy() override { - impl_ref_.reset(); + object_ref_.reset(); } /** Perform deferred I/O and queue completions. @@ -125,18 +125,18 @@ struct reactor_descriptor_state : scheduler_op inline void reactor_descriptor_state::invoke_deferred_io() { - std::shared_ptr prevent_impl_destruction; + detail::object_ref prevent_impl_destruction; ready_queue local_ops; { conditionally_enabled_mutex::scoped_lock lock(mutex); - // Must clear is_enqueued_ and move impl_ref_ under the same + // Must clear is_enqueued_ and move object_ref_ under the same // lock that processes I/O. close_socket() checks is_enqueued_ // under this mutex — without atomicity between the flag store // and the ref move, close_socket() could see is_enqueued_==false, - // skip setting impl_ref_, and destroy the impl under us. - prevent_impl_destruction = std::move(impl_ref_); + // skip setting object_ref_, and destroy the impl under us. + prevent_impl_destruction = std::move(object_ref_); is_enqueued_.store(false, std::memory_order_release); std::uint32_t ev = ready_events_.exchange(0, std::memory_order_acquire); diff --git a/include/boost/corosio/native/detail/reactor/reactor_op.hpp b/include/boost/corosio/native/detail/reactor/reactor_op.hpp index 986e590f0..1f6ab425d 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_op.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_op.hpp @@ -38,7 +38,7 @@ namespace boost::corosio::detail { impl pointers. Fields shared across all backends (errn, bytes_transferred, - cancelled, impl_ptr, perform_io, complete) live in + cancelled, object_ref_, perform_io, complete) live in reactor_op_base so the scheduler and descriptor_state can access them without template instantiation. @@ -49,7 +49,7 @@ template struct reactor_op : reactor_op_base { // The op envelope — coroutine handle h, cont, executor ex, ec_out, - // bytes_out, cancelled, stop_cb (+ its canceller), impl_ptr — lives in + // bytes_out, cancelled, stop_cb (+ its canceller), object_ref_ — lives in // coro_op (via reactor_op_base) and is shared with io_uring/IOCP. // reactor_op adds only the reactor-specific routing state below. @@ -71,7 +71,7 @@ struct reactor_op : reactor_op_base errn = 0; bytes_transferred = 0; cancelled.store(false, std::memory_order_relaxed); - impl_ptr.reset(); + object_ref_.reset(); socket_impl_ = nullptr; acceptor_impl_ = nullptr; } diff --git a/include/boost/corosio/native/detail/reactor/reactor_op_base.hpp b/include/boost/corosio/native/detail/reactor/reactor_op_base.hpp index 0f91b9eaf..516a75108 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_op_base.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_op_base.hpp @@ -21,7 +21,7 @@ namespace boost::corosio::detail { Adds the reactor-specific result model (the recorded errno + bytes and the perform_io() readiness re-run) on top of coro_op, the shared op envelope (coroutine handle, executor, output pointers, stop_token wiring, - cancelled flag, impl_ptr keepalive) common to every backend. + cancelled flag, object_ref_ keepalive) common to every backend. Kept as a non-template layer so reactor_descriptor_state and the scheduler hot paths can touch op state without knowing the concrete @@ -37,7 +37,7 @@ struct reactor_op_base : coro_op /// Bytes transferred on success. std::size_t bytes_transferred = 0; - // cancelled, impl_ptr, and request_cancel() are inherited from coro_op. + // cancelled, object_ref_, and request_cancel() are inherited from coro_op. /// Record the result of an I/O attempt. void complete(int err, std::size_t bytes) noexcept @@ -49,10 +49,11 @@ struct reactor_op_base : coro_op /// Perform the I/O syscall (implemented by concrete op types). virtual void perform_io() noexcept = 0; - /// Destroy without invoking — drop the keepalive (impl_ptr from coro_op). + /// Destroy without invoking — drop the keepalive held in + /// coro_op's `object_ref_`. void destroy() override { - impl_ptr.reset(); + object_ref_.reset(); } }; diff --git a/include/boost/corosio/native/detail/reactor/reactor_service_state.hpp b/include/boost/corosio/native/detail/reactor/reactor_service_state.hpp index 636ca388d..e04965e01 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_service_state.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_service_state.hpp @@ -10,18 +10,15 @@ #ifndef BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_SERVICE_STATE_HPP #define BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_SERVICE_STATE_HPP +#include #include -#include -#include -#include - namespace boost::corosio::detail { /** Shared service state for reactor backends. - Holds the scheduler reference, service mutex, and per-impl - ownership tracking. Used by both socket and acceptor services. + Holds the scheduler reference and the impl recycling pool. Used + by both socket and acceptor services. @tparam Scheduler The backend's scheduler type. @tparam Impl The backend's socket or acceptor impl type. @@ -35,14 +32,8 @@ struct reactor_service_state /// Reference to the owning scheduler. Scheduler& sched_; - /// Protects `impl_list_` and `impl_ptrs_`. - std::mutex mutex_; - - /// All live impl objects for shutdown traversal. - intrusive_list impl_list_; - - /// Shared ownership of each impl, keyed by raw pointer. - std::unordered_map> impl_ptrs_; + /// Owns every impl: live for shutdown traversal, free for reuse. + object_pool pool_; }; } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/reactor/reactor_socket_finals.hpp b/include/boost/corosio/native/detail/reactor/reactor_socket_finals.hpp index aa405ff5c..d110aeaca 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_socket_finals.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_socket_finals.hpp @@ -87,6 +87,12 @@ class reactor_stream_socket_impl ~reactor_stream_socket_impl() override = default; + /// Recycle into the owning service's pool at zero references. + void retire() noexcept override + { + this->svc_.state_->pool_.recycle(static_cast(this)); + } + std::error_code set_option( int level, int optname, @@ -151,6 +157,12 @@ class reactor_dgram_socket_impl using impl_base_type = ImplBase; ~reactor_dgram_socket_impl() override = default; + + /// Recycle into the owning service's pool at zero references. + void retire() noexcept override + { + this->svc_.state_->pool_.recycle(static_cast(this)); + } }; // ============================================================ @@ -197,6 +209,12 @@ class reactor_acceptor_impl ~reactor_acceptor_impl() override = default; + /// Recycle into the owning service's pool at zero references. + void retire() noexcept override + { + this->svc_.state_->pool_.recycle(static_cast(this)); + } + std::coroutine_handle<> accept( std::coroutine_handle<>, capy::executor_ref, diff --git a/include/boost/corosio/native/detail/reactor/reactor_socket_service.hpp b/include/boost/corosio/native/detail/reactor/reactor_socket_service.hpp index 292254075..fa0790b21 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_socket_service.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_socket_service.hpp @@ -11,12 +11,12 @@ #define BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_SOCKET_SERVICE_HPP #include +#include #include #include #include #include -#include namespace boost::corosio::detail { @@ -36,6 +36,15 @@ template class reactor_socket_service : public ServiceBase { friend Derived; + + // The intermediate CRTP templates below (not Impl itself) are what + // actually reach into state_->pool_ from retire() -- see + // reactor_socket_finals.hpp. + template + friend class reactor_stream_socket_impl; + template + friend class reactor_dgram_socket_impl; + using state_type = reactor_service_state; protected: @@ -52,33 +61,23 @@ class reactor_socket_service : public ServiceBase void shutdown() override { - std::lock_guard lock(state_->mutex_); - - while (auto* impl = state_->impl_list_.pop_front()) - { - static_cast(this)->pre_shutdown(impl); - impl->close_socket(); - } - - // Don't clear impl_ptrs_ here. The scheduler shuts down after us - // and drains completed_ops_, calling destroy() on each queued op. - // Letting ~state_ release the ptrs (during service destruction, - // after scheduler shutdown) keeps every impl alive until all ops - // have been drained. + state_->pool_.shutdown( + [this](Impl* impl) + { + static_cast(this)->pre_shutdown(impl); + impl->close_socket(); + }); + + // Queued ops still hold references (their own object_ref copy); + // the scheduler shuts down after us and drains completed_ops_, + // each op's destroy() releasing its reference. Shutting-down + // mode makes that final release() delete instead of recycle, + // so every impl stays alive until its ops are gone, then frees. } io_object::implementation* construct() override { - auto impl = std::make_shared(static_cast(*this)); - auto* raw = impl.get(); - - { - std::lock_guard lock(state_->mutex_); - state_->impl_ptrs_.emplace(raw, std::move(impl)); - state_->impl_list_.push_back(raw); - } - - return raw; + return state_->pool_.acquire(static_cast(*this)); } void destroy(io_object::implementation* impl) override @@ -86,9 +85,7 @@ class reactor_socket_service : public ServiceBase auto* typed = static_cast(impl); static_cast(this)->pre_destroy(typed); typed->close_socket(); - std::lock_guard lock(state_->mutex_); - state_->impl_list_.remove(typed); - state_->impl_ptrs_.erase(typed); + release(typed); } void close(io_object::handle& h) override diff --git a/include/boost/corosio/native/detail/reactor/reactor_stream_socket.hpp b/include/boost/corosio/native/detail/reactor/reactor_stream_socket.hpp index e010b2e34..b140436df 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_stream_socket.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_stream_socket.hpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -282,6 +283,30 @@ class reactor_stream_socket return fd; } + /** Reset op slots and the cached peer for recycling. + + `do_close_socket()` already cleared `remote_endpoint_` and every + parked op pointer; each op's own `reset()` runs again at its next + `do_*` call before any field is read. The one thing worth + checking here is `stop_cb`: a still-armed callback would mean a + stop_token outlived the op's completion, which `reset()` always + disengages, so asserting it is what actually catches a missed + completion rather than papering over it. + + @pre refs_ == 0, fd closed and deregistered, no op in flight. + */ + void reuse() noexcept + { + base_type::reuse(); + BOOST_COROSIO_ASSERT(!conn_.stop_cb); + BOOST_COROSIO_ASSERT(!rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wr_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_wr_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_er_.stop_cb); + BOOST_COROSIO_ASSERT(remote_endpoint_ == Endpoint{}); + } + private: // CRTP callbacks for reactor_basic_socket cancel/close @@ -389,7 +414,7 @@ reactor_stream_socket< op.fd = this->fd_; op.target_endpoint = ep; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(err, 0); this->svc_.post(&op); return std::noop_coroutine(); @@ -403,7 +428,7 @@ reactor_stream_socket< op.fd = this->fd_; op.target_endpoint = ep; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->register_op( op, this->desc_state_.connect_op, this->desc_state_.write_ready, true); @@ -451,7 +476,7 @@ reactor_stream_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(EBADF, 0); this->svc_.post(&op); return std::noop_coroutine(); @@ -468,7 +493,7 @@ reactor_stream_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(0, 0); this->svc_.post(&op); return std::noop_coroutine(); @@ -522,7 +547,7 @@ reactor_stream_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(err, bytes); this->svc_.post(&op); return std::noop_coroutine(); @@ -535,7 +560,7 @@ reactor_stream_socket< op.bytes_out = bytes_out; op.fd = this->fd_; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->register_op( op, this->desc_state_.read_op, this->desc_state_.read_ready); @@ -583,7 +608,7 @@ reactor_stream_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(EBADF, 0); this->svc_.post(&op); return std::noop_coroutine(); @@ -600,7 +625,7 @@ reactor_stream_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(0, 0); this->svc_.post(&op); return std::noop_coroutine(); @@ -643,7 +668,7 @@ reactor_stream_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(err, bytes); this->svc_.post(&op); return std::noop_coroutine(); @@ -656,7 +681,7 @@ reactor_stream_socket< op.bytes_out = bytes_out; op.fd = this->fd_; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->register_op( op, this->desc_state_.write_op, this->desc_state_.write_ready, true); @@ -737,7 +762,7 @@ reactor_stream_socket< op.ec_out = ec; op.fd = this->fd_; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(perr, 0); this->svc_.post(&op); return std::noop_coroutine(); @@ -750,7 +775,7 @@ reactor_stream_socket< op.ec_out = ec; op.fd = this->fd_; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); // Force register_op's ready path so the wait op re-probes under // the descriptor mutex before parking. An edge consumed between diff --git a/include/boost/corosio/native/detail/select/select_scheduler.hpp b/include/boost/corosio/native/detail/select/select_scheduler.hpp index d1a8731d3..83363d638 100644 --- a/include/boost/corosio/native/detail/select/select_scheduler.hpp +++ b/include/boost/corosio/native/detail/select/select_scheduler.hpp @@ -225,7 +225,7 @@ select_scheduler::register_descriptor( { conditionally_enabled_mutex::scoped_lock lock(desc->mutex); - desc->impl_ref_.reset(); + desc->object_ref_.reset(); desc->read_ready = false; desc->write_ready = false; } diff --git a/include/boost/corosio/native/detail/speculative_state.hpp b/include/boost/corosio/native/detail/speculative_state.hpp index c29b8253b..9dbdd48bb 100644 --- a/include/boost/corosio/native/detail/speculative_state.hpp +++ b/include/boost/corosio/native/detail/speculative_state.hpp @@ -109,6 +109,20 @@ class speculative_state { try_write_.store(true, std::memory_order_relaxed); } + + /** Restore the default, fully-speculative state. + + The learned hint is scoped to one logical connection; a recycled + socket starts a new one and should not inherit the prior + connection's streak or perma-off latch. + */ + void reset() noexcept + { + try_read_.store(true, std::memory_order_relaxed); + try_write_.store(true, std::memory_order_relaxed); + read_eagain_streak_.store(0, std::memory_order_relaxed); + perma_off_read_.store(false, std::memory_order_relaxed); + } }; } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/uring/uring_dgram_ops.hpp b/include/boost/corosio/native/detail/uring/uring_dgram_ops.hpp index b4338e068..f4cdeaebd 100644 --- a/include/boost/corosio/native/detail/uring/uring_dgram_ops.hpp +++ b/include/boost/corosio/native/detail/uring/uring_dgram_ops.hpp @@ -69,7 +69,7 @@ struct uring_dgram_send_op : uring_op std::size_t* bytes, int file_descriptor, uring_scheduler* scheduler, - std::shared_ptr impl, + detail::object_ref impl, detail::speculative_state* spec, buffer_param buffers, socklen_t dest_addr_len, @@ -77,17 +77,17 @@ struct uring_dgram_send_op : uring_op int flags, std::stop_token const& token) noexcept { - h = handle; - ex = executor; - ec_out = ec; - bytes_out = bytes; - fd = file_descriptor; - sched_ = scheduler; - impl_ptr = std::move(impl); - spec_state = spec; - res = 0; - cqe_flags = 0; - msg_flags = flags; + h = handle; + ex = executor; + ec_out = ec; + bytes_out = bytes; + fd = file_descriptor; + sched_ = scheduler; + object_ref_ = std::move(impl); + spec_state = spec; + res = 0; + cqe_flags = 0; + msg_flags = flags; iovec_count = copy_to_iovec(buffers, iovecs); @@ -207,7 +207,7 @@ struct uring_dgram_recv_op : uring_op std::size_t* bytes, int file_descriptor, uring_scheduler* scheduler, - std::shared_ptr impl, + detail::object_ref impl, detail::speculative_state* spec, buffer_param buffers, void* source_ctx, @@ -215,17 +215,17 @@ struct uring_dgram_recv_op : uring_op int flags, std::stop_token const& token) noexcept { - h = handle; - ex = executor; - ec_out = ec; - bytes_out = bytes; - fd = file_descriptor; - sched_ = scheduler; - impl_ptr = std::move(impl); - spec_state = spec; - res = 0; - cqe_flags = 0; - msg_flags = flags; + h = handle; + ex = executor; + ec_out = ec; + bytes_out = bytes; + fd = file_descriptor; + sched_ = scheduler; + object_ref_ = std::move(impl); + spec_state = spec; + res = 0; + cqe_flags = 0; + msg_flags = flags; iovec_count = copy_to_iovec(buffers, iovecs); diff --git a/include/boost/corosio/native/detail/uring/uring_file_ops.hpp b/include/boost/corosio/native/detail/uring/uring_file_ops.hpp index e151a3ad5..6dbf9a2dc 100644 --- a/include/boost/corosio/native/detail/uring/uring_file_ops.hpp +++ b/include/boost/corosio/native/detail/uring/uring_file_ops.hpp @@ -71,7 +71,7 @@ struct uring_file_read_op_base : uring_op int file_descriptor, std::int64_t file_offset, uring_scheduler* scheduler, - std::shared_ptr impl, + detail::object_ref impl, buffer_param buffers, std::stop_token const& token) noexcept { @@ -82,7 +82,7 @@ struct uring_file_read_op_base : uring_op fd = file_descriptor; offset = file_offset; sched_ = scheduler; - impl_ptr = std::move(impl); + object_ref_ = std::move(impl); res = 0; cqe_flags = 0; iovec_count = copy_to_iovec(buffers, iovecs); @@ -217,7 +217,7 @@ struct uring_file_write_op_base : uring_op int file_descriptor, std::int64_t file_offset, uring_scheduler* scheduler, - std::shared_ptr impl, + detail::object_ref impl, buffer_param buffers, std::stop_token const& token) noexcept { @@ -228,7 +228,7 @@ struct uring_file_write_op_base : uring_op fd = file_descriptor; offset = file_offset; sched_ = scheduler; - impl_ptr = std::move(impl); + object_ref_ = std::move(impl); res = 0; cqe_flags = 0; iovec_count = copy_to_iovec(buffers, iovecs); diff --git a/include/boost/corosio/native/detail/uring/uring_file_service_base.hpp b/include/boost/corosio/native/detail/uring/uring_file_service_base.hpp index 2d9629839..0352e1e4e 100644 --- a/include/boost/corosio/native/detail/uring/uring_file_service_base.hpp +++ b/include/boost/corosio/native/detail/uring/uring_file_service_base.hpp @@ -14,33 +14,34 @@ #if BOOST_COROSIO_HAS_URING -#include +#include +#include #include #include -#include -#include -#include +#include /* Shared lifecycle plumbing for io_uring file services. uring_stream_file_service and uring_random_access_file_service were byte-for-byte identical apart from the impl type and open_file's parameter - type: both make_shared the file impl from the scheduler, track it in an - intrusive list + raw->shared_ptr map, and close every file on shutdown. - This base factors that out; the concrete services add only open_file. + type: both pop-or-new the file impl from a per-service recycling pool + and close every file on shutdown. This base factors that out; the + concrete services add only open_file. This is a separate base from uring_socket_service_base because file - services differ from socket services in three ways that match the reactor - socket service instead: they track via an intrusive list + map (sockets: - map only), they CLOSE files on shutdown (sockets: cancel only), and the - impl ctor takes just the scheduler (sockets: service + scheduler). See - tasks/proactor-dedup-decisions.md (#14). - - Requirements on File: derive from enable_shared_from_this and - intrusive_list::node, a `File(uring_scheduler&)` constructor, and - a `void close_file() noexcept` method (cancel in-flight ops + close fd). + services differ from socket services in two ways that match the reactor + socket service instead: they CLOSE files on shutdown (sockets: cancel + only), and the impl ctor takes just the scheduler (sockets: service + + scheduler). See tasks/proactor-dedup-decisions.md (#14). + + Requirements on File: derive from intrusive_list::node, a + `File(Derived&, uring_scheduler&)` constructor, a + `void close_file() noexcept` method (cancel in-flight ops + close fd), + a `void reuse() noexcept` method, and a `retire()` override that + recycles through the owning service's private `pool_` member + (reachable because `File` is a friend). @tparam Derived The concrete service (CRTP; unused today but kept for symmetry / future hooks). @@ -55,6 +56,7 @@ template class uring_file_service_base : public ServiceBase { friend Derived; + friend File; // Private CRTP ctor: only `Derived` (the concrete service, a friend) // constructs the base — prevents inheriting with the wrong Derived @@ -69,14 +71,7 @@ class uring_file_service_base : public ServiceBase io_object::implementation* construct() override { - auto ptr = std::make_shared(*sched_); - auto* impl = ptr.get(); - { - std::lock_guard lock(mutex_); - file_list_.push_back(impl); - file_ptrs_[impl] = std::move(ptr); - } - return impl; + return pool_.acquire(static_cast(*this), *sched_); } void destroy(io_object::implementation* p) override @@ -84,9 +79,7 @@ class uring_file_service_base : public ServiceBase // close_file() already does cancel_and_flush(fd_) before ::close. auto& impl = static_cast(*p); impl.close_file(); - std::lock_guard lock(mutex_); - file_list_.remove(&impl); - file_ptrs_.erase(&impl); + release(&impl); } void close(io_object::handle& h) override @@ -97,13 +90,23 @@ class uring_file_service_base : public ServiceBase void shutdown() override { - std::lock_guard lock(mutex_); - for (auto* impl = file_list_.pop_front(); impl != nullptr; - impl = file_list_.pop_front()) + // See uring_socket_service_base::shutdown(): snapshot under an + // acquired reference, close without the pool lock held; + // shutdown() sets shutting-down and takes the snapshot in one + // critical section, so a close that drops the last ref deletes + // rather than recycles. + std::vector live; + pool_.shutdown( + [&](File* f) + { + acquire(f); + live.push_back(f); + }); + for (auto* f : live) { - impl->close_file(); + f->close_file(); + release(f); } - file_ptrs_.clear(); } /// Return the scheduler used by files created by this service. @@ -114,9 +117,7 @@ class uring_file_service_base : public ServiceBase protected: uring_scheduler* sched_; - std::mutex mutex_; - intrusive_list file_list_; - std::unordered_map> file_ptrs_; + object_pool pool_; private: uring_file_service_base(uring_file_service_base const&) = delete; diff --git a/include/boost/corosio/native/detail/uring/uring_multishot_acceptor.hpp b/include/boost/corosio/native/detail/uring/uring_multishot_acceptor.hpp index 4ded4e8e9..7da993a36 100644 --- a/include/boost/corosio/native/detail/uring/uring_multishot_acceptor.hpp +++ b/include/boost/corosio/native/detail/uring/uring_multishot_acceptor.hpp @@ -11,6 +11,7 @@ #define BOOST_COROSIO_NATIVE_DETAIL_URING_URING_MULTISHOT_ACCEPTOR_HPP #include +#include #include #if BOOST_COROSIO_HAS_URING @@ -62,10 +63,15 @@ fd_is_listening(int fd) noexcept return accepting != 0; } -template +template< + class Derived, + class ImplBase, + class Endpoint, + class PeerService, + class AcceptorService> class uring_multishot_acceptor_base : public ImplBase - , public std::enable_shared_from_this + , public intrusive_list::node { protected: struct ready_fd_node : intrusive_list::node @@ -104,6 +110,10 @@ class uring_multishot_acceptor_base int fd_ = -1; uring_scheduler* sched_; PeerService* peer_service_; + /// Owning acceptor service; exposes the recycling pool to + /// retire(). Not used for anything else — the service already + /// drives shutdown/destroy from its own side. + AcceptorService* acceptor_svc_; Endpoint local_endpoint_{}; mutable std::mutex mutex_; intrusive_list ready_fds_; @@ -136,13 +146,80 @@ class uring_multishot_acceptor_base // (clang-tidy bugprone-crtp-constructor-accessibility). friend Derived; uring_multishot_acceptor_base( - uring_scheduler& sched, PeerService& peer_svc) noexcept + AcceptorService& acceptor_svc, + uring_scheduler& sched, + PeerService& peer_svc) noexcept : sched_(&sched) , peer_service_(&peer_svc) + , acceptor_svc_(&acceptor_svc) { } public: + /** Close and free every fd parked in `ready_fds_`. + + A multishot CQE can deliver a connection before any `accept()` + is outstanding to claim it; that connection's fd (and tracking + node) sits in `ready_fds_` until a later `accept()` consumes it + or the acceptor goes away. Idempotent: safe to call from both + `retire()` (recycling) and the destructor (pool-force-sweep + backstop) — the second call always sees an empty list. + */ + void drain_ready_fds() noexcept + { + intrusive_list drained; + { + std::lock_guard lk(mutex_); + while (auto* r = ready_fds_.pop_front()) + drained.push_back(r); + } + while (auto* r = drained.pop_front()) + { + ::close(r->fd); + delete r; + } + } + + /** Recycle into the owning service's pool at zero references. + + Reaching zero used to mean the shared_ptr destructor was about + to run, which is where the ready_fds_ drain and the multi_op_ + cycle-break + CQE drain below used to live (the destructor + still runs them too, as a backstop for the pool's own + unconditional force-sweep at shutdown — both are idempotent). + Now reaching zero can instead mean this impl is about to go + back on the free list for reuse — its memory is never freed, + so the destructor never runs. Do the drain here, once, for + both outcomes: `close()` already reset `multi_op_->object_ref_` on + the normal path, so that reset is usually a no-op; it only + does real work when zero is reached some other way. Skipping + the ready_fds_ drain here specifically would leak every fd and + node still parked in it and trip `reuse()`'s `ready_fds_.empty()` + assert on the very next construct() that pops this impl back + out. + + `multi_op_` itself is freed here too, rather than kept for the + next session: `prepare_listen_arm()` treats a non-null + `multi_op_` with `closing_` false and `arm_err_` zero as + already armed and skips re-arming — true across a re-listen + on the same still-open acceptor, but not across reuse, where + the next `listen()` is a different logical acceptor on a + different fd. Leaving the old `multi_op_` in place would make + that listen() silently believe it was already armed and never + submit a multishot SQE for the new fd. + */ + void retire() noexcept override + { + drain_ready_fds(); + if (multi_op_) + { + multi_op_->object_ref_.reset(); + sched_->drain_cqes_for(multi_op_.get()); + multi_op_.reset(); + } + acceptor_svc_->pool_.recycle(static_cast(this)); + } + ~uring_multishot_acceptor_base() override { { @@ -156,27 +233,17 @@ class uring_multishot_acceptor_base fd_ = -1; } - // Drain parked accepted-connection fds unconditionally. These are - // distinct from the listener fd and can be present even when the - // service close() path already closed and cleared fd_ — that path - // does not touch ready_fds_, so the drain must run here. - intrusive_list drained; - { - std::lock_guard lk(mutex_); - while (auto* r = ready_fds_.pop_front()) - drained.push_back(r); - } - while (auto* r = drained.pop_front()) - { - ::close(r->fd); - delete r; - } + // Backstop: retire() already drains this on the normal + // recycle path. This call only does real work when this impl + // is reached by the pool's unconditional force-sweep at + // shutdown without ever going through retire() first. + drain_ready_fds(); - // Break the multi_op_ → impl_ptr (shared_ptr) cycle and + // Break the multi_op_ → object_ref_ (object_ref) cycle and // drain pending CQEs so unique_ptr can free safely. if (multi_op_) { - multi_op_->impl_ptr.reset(); + multi_op_->object_ref_.reset(); sched_->drain_cqes_for(multi_op_.get()); } } @@ -204,7 +271,7 @@ class uring_multishot_acceptor_base native_handle_type release_socket() noexcept override { // Mirror the service close() path: cancel the multishot SQE and - // break the multi_op_ -> impl_ptr (shared_ptr) cycle that + // break the multi_op_ -> object_ref_ (object_ref) cycle that // start_multishot established. Without this, the cycle keeps the // acceptor and its multi_op_ alive after the caller takes the fd, // which LeakSanitizer reports on process exit. Caller still owns @@ -214,7 +281,7 @@ class uring_multishot_acceptor_base sched_->cancel_and_flush(fd_); drain_waiters_only(); if (multi_op_) - multi_op_->impl_ptr.reset(); + multi_op_->object_ref_.reset(); } int fd = fd_; fd_ = -1; @@ -229,6 +296,35 @@ class uring_multishot_acceptor_base sched_->submit_cancel_by_fd(fd_); } + /** Reset state for recycling. + + `close()`/`release_socket()` already drove fd_, local_endpoint_, + and the waiter/ready-fd queues to empty before the refcount + reached zero, so those are asserted rather than re-cleared. + `closing_` and `arm_err_` are the two fields that are NOT reset + by that close path (closing_ is only ever cleared by a re-listen + on a still-live acceptor, not by close) and must be explicitly + reset here or the next session would see this impl as still + shutting down. `multi_op_` is freed by `retire()` rather + than kept, so a fresh one is allocated the next time this impl + starts a multishot arming — see its comment for why keeping it + across reuse would leave the new session un-armed. + + @pre refs_ == 0, fd closed, no op or waiter in flight, + multi_op_ already freed. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(fd_ == -1); + BOOST_COROSIO_ASSERT(local_endpoint_ == Endpoint{}); + BOOST_COROSIO_ASSERT(ready_fds_.empty()); + BOOST_COROSIO_ASSERT(waiters_.empty()); + BOOST_COROSIO_ASSERT(read_wait_ == nullptr); + BOOST_COROSIO_ASSERT(!multi_op_); + closing_ = false; + arm_err_ = 0; + } + /// Drain queued waiters with operation_aborted but do NOT submit /// any kernel cancel for the fd. Used by service close() paths /// that have already submitted (or are about to submit) the @@ -422,7 +518,7 @@ class uring_multishot_acceptor_base sched_->retire_op( multi_op_, [this](uring_multi_accept_op& op) noexcept { std::lock_guard lk(mutex_); - op.impl_ptr.reset(); + op.object_ref_.reset(); op.acceptor_impl = nullptr; op.on_cqe = nullptr; op.retire_func = &uring_multi_accept_op::do_retired_cqe; @@ -523,13 +619,13 @@ class uring_multishot_acceptor_base multi_op_->listen_fd = fd_; multi_op_->acceptor_impl = this; multi_op_->on_cqe = &uring_multishot_acceptor_base::on_accept_cqe; - multi_op_->impl_ptr = this->shared_from_this(); + multi_op_->object_ref_ = detail::object_ref(this); } else { // Reuse the existing op (re-arm path). Reset peer scratch // so the kernel writes into a clean slot. listen_fd and - // impl_ptr are re-seeded so the op can never carry state + // object_ref_ are re-seeded so the op can never carry state // from an arming that has since been torn down. `res` is // one of those: an arming that failed to submit left // -EAGAIN there, and the reader of `res` cannot tell a @@ -538,7 +634,7 @@ class uring_multishot_acceptor_base multi_op_->peer_len = sizeof(sockaddr_storage); multi_op_->res = 0; multi_op_->listen_fd = fd_; - multi_op_->impl_ptr = this->shared_from_this(); + multi_op_->object_ref_ = detail::object_ref(this); } auto* op = multi_op_.get(); @@ -933,19 +1029,23 @@ class uring_multishot_acceptor_base // Re-arm: kernel terminated multishot non-fatally. struct rearm_op final : scheduler_op { - std::shared_ptr self_; + detail::object_ref self_ref_; + Derived* self_; std::uint64_t generation_; rearm_op( - std::shared_ptr s, + detail::object_ref ref, + Derived* self, std::uint64_t generation) noexcept - : self_(std::move(s)) + : self_ref_(std::move(ref)) + , self_(self) , generation_(generation) { } void operator()() override { - auto self = std::move(self_); + auto self_ref = std::move(self_ref_); + auto* self = self_; auto generation = generation_; delete this; { @@ -972,15 +1072,26 @@ class uring_multishot_acceptor_base }; // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — CQE handler re-arm: noexcept, OOM => std::terminate is the intended behavior sched_->post(new rearm_op( - this->shared_from_this(), + detail::object_ref(this), + static_cast(this), arm_generation_.load(std::memory_order_acquire))); } } }; -template +template< + class Derived, + class ImplBase, + class Endpoint, + class PeerService, + class AcceptorService> inline void -uring_multishot_acceptor_base:: +uring_multishot_acceptor_base< + Derived, + ImplBase, + Endpoint, + PeerService, + AcceptorService>:: waiter_canceller::operator()() const noexcept { if (w->cancelled.exchange(true, std::memory_order_acq_rel)) diff --git a/include/boost/corosio/native/detail/uring/uring_op.hpp b/include/boost/corosio/native/detail/uring/uring_op.hpp index 2128731de..027867797 100644 --- a/include/boost/corosio/native/detail/uring/uring_op.hpp +++ b/include/boost/corosio/native/detail/uring/uring_op.hpp @@ -33,7 +33,7 @@ namespace boost::corosio::detail { `coro_op` supplies the fields common to both proactor backends (coroutine handle, executor, output pointers, stop_token wiring, - impl_ptr keepalive). This type adds the CQE result (`res`/`cqe_flags`), + object_ref_ keepalive). This type adds the CQE result (`res`/`cqe_flags`), the ring-cancel visibility flag (`sqe_set`), and the two function pointers the run loop uses to prep an SQE and dispatch a CQE without template instantiation. diff --git a/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp b/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp index 773b049bf..4493cd719 100644 --- a/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp +++ b/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp @@ -15,6 +15,7 @@ #if BOOST_COROSIO_HAS_URING #include +#include #include #include #include @@ -54,13 +55,13 @@ class uring_random_access_file_service; */ class BOOST_COROSIO_DECL uring_random_access_file final : public random_access_file::implementation - , public std::enable_shared_from_this , public intrusive_list::node { friend class uring_random_access_file_service; - int fd_ = -1; - uring_scheduler* sched_ = nullptr; + int fd_ = -1; + uring_scheduler* sched_ = nullptr; + uring_random_access_file_service* svc_ = nullptr; // Random-access files legitimately support concurrent ops at // different offsets on the same fd (e.g. parallel reads in @@ -68,8 +69,11 @@ class BOOST_COROSIO_DECL uring_random_access_file final // state across calls; ops are heap-allocated per submission. public: - explicit uring_random_access_file(uring_scheduler& sched) noexcept + explicit uring_random_access_file( + uring_random_access_file_service& svc, + uring_scheduler& sched) noexcept : sched_(&sched) + , svc_(&svc) { } @@ -78,6 +82,24 @@ class BOOST_COROSIO_DECL uring_random_access_file final close_file(); } + /// Recycle into the owning service's pool. Defined out-of-line + /// after uring_random_access_file_service for its complete type. + void retire() noexcept override; + + /** Reset for recycling. + + `close_file()` already drove fd_ to its closed value before the + refcount reached zero; every op is heap-allocated per submission + and owns its own `stop_cb`/`object_ref` lifetime, so there is no + embedded op state to assert here. + + @pre refs_ == 0, fd closed, no op in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(fd_ == -1); + } + // -- random_access_file::implementation -- std::coroutine_handle<> read_some_at( @@ -231,7 +253,7 @@ uring_random_access_file::read_some_at( auto op_guard = std::make_unique(); op_guard->prepare( h, ex, ec, bytes, fd_, static_cast(user_offset), sched_, - shared_from_this(), buffers, token); + detail::object_ref(this), buffers, token); sched_->work_started(); // Closed-object contract outranks the zero-length no-op. @@ -269,7 +291,7 @@ uring_random_access_file::write_some_at( auto op_guard = std::make_unique(); op_guard->prepare( h, ex, ec, bytes, fd_, static_cast(user_offset), sched_, - shared_from_this(), buffers, token); + detail::object_ref(this), buffers, token); sched_->work_started(); // Closed-object contract outranks the zero-length no-op. @@ -332,6 +354,12 @@ class BOOST_COROSIO_DECL uring_random_access_file_service final } }; +inline void +uring_random_access_file::retire() noexcept +{ + svc_->pool_.recycle(this); +} + } // namespace boost::corosio::detail #endif // BOOST_COROSIO_HAS_URING diff --git a/include/boost/corosio/native/detail/uring/uring_scheduler.hpp b/include/boost/corosio/native/detail/uring/uring_scheduler.hpp index 87c67b408..3cb753555 100644 --- a/include/boost/corosio/native/detail/uring/uring_scheduler.hpp +++ b/include/boost/corosio/native/detail/uring/uring_scheduler.hpp @@ -335,7 +335,7 @@ class BOOST_COROSIO_DECL uring_scheduler final Callers must hold neither. @pre After @p prepare runs, @p slot must hold no reference back - to its owner (`impl_ptr` cleared, back-pointers nulled). + to its owner (`object_ref_` cleared, back-pointers nulled). `release_retired_op` deletes the op from inside the CQE loop with `ring_mutex_` held, so its destructor must not re-enter the scheduler or touch the owner. @@ -742,8 +742,8 @@ uring_scheduler::shutdown() // for every live impl. The CQEs that result either land in // completed_ops_ (drained here as op->destroy()) or stay in the // kernel ring; ~scheduler's io_uring_queue_exit cleans the - // latter up at process teardown. Self-referential impl_ptr - // cycles (e.g. multishot acceptor's multi_op_->impl_ptr) are + // latter up at process teardown. Self-referential object_ref_ + // cycles (e.g. multishot acceptor's multi_op_->object_ref_) are // broken explicitly inside each service before the scheduler // shutdown runs. lock_type lock(dispatch_mutex_); diff --git a/include/boost/corosio/native/detail/uring/uring_socket_ops.hpp b/include/boost/corosio/native/detail/uring/uring_socket_ops.hpp index 0b564aa7d..bd850b93c 100644 --- a/include/boost/corosio/native/detail/uring/uring_socket_ops.hpp +++ b/include/boost/corosio/native/detail/uring/uring_socket_ops.hpp @@ -129,7 +129,7 @@ struct uring_read_op : uring_op std::size_t* bytes, int file_descriptor, uring_scheduler* scheduler, - std::shared_ptr impl, + detail::object_ref impl, detail::speculative_state* spec, buffer_param buffers, std::stop_token const& token) noexcept @@ -140,7 +140,7 @@ struct uring_read_op : uring_op bytes_out = bytes; fd = file_descriptor; sched_ = scheduler; - impl_ptr = std::move(impl); + object_ref_ = std::move(impl); spec_state = spec; res = 0; cqe_flags = 0; @@ -227,7 +227,7 @@ struct uring_write_op : uring_op std::size_t* bytes, int file_descriptor, uring_scheduler* scheduler, - std::shared_ptr impl, + detail::object_ref impl, detail::speculative_state* spec, buffer_param buffers, std::stop_token const& token) noexcept @@ -238,7 +238,7 @@ struct uring_write_op : uring_op bytes_out = bytes; fd = file_descriptor; sched_ = scheduler; - impl_ptr = std::move(impl); + object_ref_ = std::move(impl); spec_state = spec; res = 0; cqe_flags = 0; @@ -336,7 +336,7 @@ struct uring_connect_op : uring_op std::error_code* ec, int file_descriptor, uring_scheduler* scheduler, - std::shared_ptr impl, + detail::object_ref impl, endpoint target, endpoint* remote_out, endpoint* local_out, @@ -348,7 +348,7 @@ struct uring_connect_op : uring_op bytes_out = nullptr; fd = file_descriptor; sched_ = scheduler; - impl_ptr = std::move(impl); + object_ref_ = std::move(impl); res = 0; cqe_flags = 0; target_endpoint = target; @@ -586,20 +586,20 @@ struct uring_wait_op : uring_op std::error_code* ec, int file_descriptor, uring_scheduler* scheduler, - std::shared_ptr impl, + detail::object_ref impl, int flags, std::stop_token const& token) noexcept { - h = handle; - ex = executor; - ec_out = ec; - bytes_out = nullptr; - fd = file_descriptor; - sched_ = scheduler; - impl_ptr = std::move(impl); - poll_flags = flags; - res = 0; - cqe_flags = 0; + h = handle; + ex = executor; + ec_out = ec; + bytes_out = nullptr; + fd = file_descriptor; + sched_ = scheduler; + object_ref_ = std::move(impl); + poll_flags = flags; + res = 0; + cqe_flags = 0; start(token); } @@ -691,7 +691,7 @@ struct uring_local_connect_op : uring_op std::error_code* ec, int file_descriptor, uring_scheduler* scheduler, - std::shared_ptr impl, + detail::object_ref impl, corosio::local_endpoint target, corosio::local_endpoint* remote_out, corosio::local_endpoint* local_out, @@ -703,7 +703,7 @@ struct uring_local_connect_op : uring_op bytes_out = nullptr; fd = file_descriptor; sched_ = scheduler; - impl_ptr = std::move(impl); + object_ref_ = std::move(impl); res = 0; cqe_flags = 0; target_endpoint = target; diff --git a/include/boost/corosio/native/detail/uring/uring_socket_service_base.hpp b/include/boost/corosio/native/detail/uring/uring_socket_service_base.hpp index bf510575e..be8d30417 100644 --- a/include/boost/corosio/native/detail/uring/uring_socket_service_base.hpp +++ b/include/boost/corosio/native/detail/uring/uring_socket_service_base.hpp @@ -14,13 +14,12 @@ #if BOOST_COROSIO_HAS_URING +#include +#include #include #include #include -#include -#include -#include #include /* @@ -28,23 +27,23 @@ construct / destroy / shutdown / close / scheduler() are identical across uring_tcp_service, uring_udp_service, uring_local_stream_service, - and uring_local_datagram_service — they all make_shared the impl, track - it in a raw->shared_ptr map, cancel on shutdown, and close eagerly. This + and uring_local_datagram_service — they all pop-or-new the impl from a + per-service recycling pool, cancel on shutdown, and close eagerly. This base factors that out; the concrete services add only the protocol- specific open/bind/adopt. This is io_uring's own service base rather than a reuse of - reactor_socket_service: io_uring tracks impls in a map (the reactor uses - an intrusive list + map), cancels (not closes) on shutdown, and constructs - impls with a (service&, scheduler&) ctor. Reusing the reactor template - would force io_uring sockets to adopt the intrusive node, a close-on- - shutdown behavior change, and an Impl(Derived&) ctor — high churn and a - teardown behavior change for marginal extra sharing. See + reactor_socket_service: io_uring cancels (not closes) on shutdown and + constructs impls with a (service&, scheduler&) ctor, vs. the reactor's + close-on-shutdown and Impl(Derived&) ctor. Reusing the reactor template + would force a teardown behavior change for marginal extra sharing. See tasks/proactor-dedup-decisions.md (#13). Requirements on Socket: a `(Derived& service, uring_scheduler& sched)` - constructor and a `void close_socket() noexcept` method (cancel in-flight - ops + close fd + reset cached endpoints). + constructor, a `void close_socket() noexcept` method (cancel in-flight + ops + close fd + reset cached endpoints), a `void reuse() noexcept` + method, and derivation from `intrusive_list::node` (the pool's + live/free linkage). @tparam Derived The concrete service (CRTP, passed to the Socket ctor). @tparam ServiceBase The abstract service vtable base (tcp_service, ...). @@ -57,6 +56,7 @@ template class uring_socket_service_base : public ServiceBase { friend Derived; + friend Socket; // Private CRTP ctor: only `Derived` (the concrete service, a friend) // constructs the base — prevents inheriting with the wrong Derived @@ -71,41 +71,42 @@ class uring_socket_service_base : public ServiceBase void shutdown() override { - // Snapshot live impls, then cancel without the lock held to avoid - // inversion if cancel() ever re-enters the service. Impls stay owned - // by impls_ until ~service (after the scheduler drains its queue), - // keeping every impl alive while its cancel CQEs are processed. - std::vector> live; + // Snapshot live impls under an acquired reference, then cancel + // without the pool lock held to avoid inversion if cancel() ever + // re-enters the service. shutdown() sets shutting-down and + // takes the snapshot under one critical section, so each + // cancel's own release (if it drops the last ref) deletes + // rather than recycles. In-flight ops hold their own + // references, so every impl stays alive while its cancel CQEs + // are processed. + std::vector live; + pool_.shutdown( + [&](Socket* s) + { + acquire(s); + live.push_back(s); + }); + for (auto* s : live) { - std::lock_guard lk(mutex_); - live.reserve(impls_.size()); - for (auto& [_, p] : impls_) - live.push_back(p); + s->cancel(); + release(s); } - for (auto& p : live) - p->cancel(); } io_object::implementation* construct() override { - auto p = - std::make_shared(static_cast(*this), *sched_); - auto* raw = p.get(); - std::lock_guard lk(mutex_); - impls_.emplace(raw, std::move(p)); - return raw; + return acquire_impl(); } void destroy(io_object::implementation* p) override { if (!p) return; - std::lock_guard lk(mutex_); - impls_.erase(static_cast(p)); + release(static_cast(p)); } // Close the fd eagerly when the public close() is called, before - // destroy() drops the shared_ptr and the destructor runs. + // destroy() drops the service's reference and recycling runs. void close(io_object::handle& h) override { if (auto* sock = static_cast(h.get())) @@ -119,19 +120,18 @@ class uring_socket_service_base : public ServiceBase } protected: - /// Register an externally-built impl (used by adopt_fd on stream - /// services after accept(2)). Returns the raw pointer. - Socket* register_impl(std::shared_ptr p) + /** Pop a recycled impl or news one, with the service reference held. + + Used by both `construct()` and the `adopt_fd` accepted-connection + path (which additionally calls `assign_fd()` on the result). + */ + Socket* acquire_impl() { - auto* raw = p.get(); - std::lock_guard lk(mutex_); - impls_.emplace(raw, std::move(p)); - return raw; + return pool_.acquire(static_cast(*this), *sched_); } uring_scheduler* sched_; - std::mutex mutex_; - std::unordered_map> impls_; + object_pool pool_; private: uring_socket_service_base(uring_socket_service_base const&) = delete; diff --git a/include/boost/corosio/native/detail/uring/uring_stream_file.hpp b/include/boost/corosio/native/detail/uring/uring_stream_file.hpp index 05bb2ccf6..e893c6b62 100644 --- a/include/boost/corosio/native/detail/uring/uring_stream_file.hpp +++ b/include/boost/corosio/native/detail/uring/uring_stream_file.hpp @@ -15,6 +15,7 @@ #if BOOST_COROSIO_HAS_URING #include +#include #include #include #include @@ -62,13 +63,13 @@ class uring_stream_file_service; */ class BOOST_COROSIO_DECL uring_stream_file final : public stream_file::implementation - , public std::enable_shared_from_this , public intrusive_list::node { friend class uring_stream_file_service; - int fd_ = -1; - uring_scheduler* sched_ = nullptr; + int fd_ = -1; + uring_scheduler* sched_ = nullptr; + uring_stream_file_service* svc_ = nullptr; // Per-fd op slots — embedded to eliminate per-call heap allocation. // Single-pending invariant per slot. @@ -76,7 +77,10 @@ class BOOST_COROSIO_DECL uring_stream_file final uring_file_write_op wr_; public: - explicit uring_stream_file(uring_scheduler& sched) noexcept : sched_(&sched) + explicit uring_stream_file( + uring_stream_file_service& svc, uring_scheduler& sched) noexcept + : sched_(&sched) + , svc_(&svc) { } @@ -85,6 +89,26 @@ class BOOST_COROSIO_DECL uring_stream_file final close_file(); } + /// Recycle into the owning service's pool. Defined out-of-line + /// after uring_stream_file_service for its complete type. + void retire() noexcept override; + + /** Reset op slots for recycling. + + `close_file()` already drove fd_ to its closed value before the + refcount reached zero; each op's own `prepare()` overwrites its + state before the next use, so only the `stop_cb` invariant is + worth asserting here. + + @pre refs_ == 0, fd closed, no op in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(fd_ == -1); + BOOST_COROSIO_ASSERT(!rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wr_.stop_cb); + } + // -- io_stream::implementation -- std::coroutine_handle<> read_some( @@ -252,7 +276,7 @@ uring_stream_file::read_some( std::size_t* bytes) { rd_.prepare( - h, ex, ec, bytes, fd_, /*file_offset=*/-1, sched_, shared_from_this(), + h, ex, ec, bytes, fd_, /*file_offset=*/-1, sched_, detail::object_ref(this), buffers, token); sched_->work_started(); @@ -287,7 +311,7 @@ uring_stream_file::write_some( std::size_t* bytes) { wr_.prepare( - h, ex, ec, bytes, fd_, /*file_offset=*/-1, sched_, shared_from_this(), + h, ex, ec, bytes, fd_, /*file_offset=*/-1, sched_, detail::object_ref(this), buffers, token); sched_->work_started(); @@ -347,6 +371,12 @@ class BOOST_COROSIO_DECL uring_stream_file_service final } }; +inline void +uring_stream_file::retire() noexcept +{ + svc_->pool_.recycle(this); +} + } // namespace boost::corosio::detail #endif // BOOST_COROSIO_HAS_URING diff --git a/include/boost/corosio/native/detail/uring/uring_types.hpp b/include/boost/corosio/native/detail/uring/uring_types.hpp index f297004b9..4138dc5c6 100644 --- a/include/boost/corosio/native/detail/uring/uring_types.hpp +++ b/include/boost/corosio/native/detail/uring/uring_types.hpp @@ -15,6 +15,7 @@ #if BOOST_COROSIO_HAS_URING +#include #include #include #include @@ -69,8 +70,8 @@ class uring_local_datagram_service; read, write, and connect operations are submitted to the kernel via `uring_submit_op` and complete through the ring's CQE path. - The object is always owned by a `shared_ptr` managed by the service. - In-flight ops hold an additional `shared_ptr` copy (`impl_ptr`) so + The service holds one intrusive reference for as long as the impl + is live; in-flight ops hold an additional `object_ref_` keepalive so the kernel's user-data pointer remains valid until the CQE arrives. @par Thread Safety @@ -83,12 +84,13 @@ class BOOST_COROSIO_DECL uring_tcp_socket final uring_tcp_socket, tcp_socket::implementation, endpoint> + , public intrusive_list::node { friend uring_tcp_service; int family_ = AF_UNSPEC; // cached at open_socket uring_scheduler* sched_ = nullptr; - [[maybe_unused]] uring_tcp_service* svc_ = nullptr; + uring_tcp_service* svc_ = nullptr; // fd_ and local_endpoint_ are provided by native_socket_base (the // readiness/completion-agnostic socket base shared with the reactor @@ -207,7 +209,7 @@ class BOOST_COROSIO_DECL uring_tcp_socket final return dispatch_coro(ex, rd_.cont); } rd_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, token); if (stop_now) rd_.cancelled.store(true, std::memory_order_release); @@ -222,7 +224,7 @@ class BOOST_COROSIO_DECL uring_tcp_socket final } rd_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, buffers, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, token); sched_->work_started(); if (rd_.cancelled.load(std::memory_order_acquire)) @@ -286,7 +288,7 @@ class BOOST_COROSIO_DECL uring_tcp_socket final return dispatch_coro(ex, wr_.cont); } wr_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, token); if (stop_now) wr_.cancelled.store(true, std::memory_order_release); @@ -301,7 +303,7 @@ class BOOST_COROSIO_DECL uring_tcp_socket final } wr_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, buffers, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, token); sched_->work_started(); if (wr_.cancelled.load(std::memory_order_acquire)) @@ -337,7 +339,7 @@ class BOOST_COROSIO_DECL uring_tcp_socket final } conn_.addrlen = to_sockaddr(ep, family_, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), ep, + h, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, &local_endpoint_, token); conn_.cancelled.store(true, std::memory_order_release); sched_->work_started(); @@ -352,7 +354,7 @@ class BOOST_COROSIO_DECL uring_tcp_socket final // a subsequent IORING_OP_CONNECT would see EALREADY — avoid. conn_.addrlen = to_sockaddr(ep, family_, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), ep, &remote_endpoint_, + h, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, &local_endpoint_, token); sched_->work_started(); if (conn_.cancelled.load(std::memory_order_acquire)) @@ -386,7 +388,7 @@ class BOOST_COROSIO_DECL uring_tcp_socket final break; } wait_op_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), poll_flags, token); + h, ex, ec, fd_, sched_, detail::object_ref(this), poll_flags, token); sched_->work_started(); if (wait_op_.cancelled.load(std::memory_order_acquire)) { @@ -448,6 +450,45 @@ class BOOST_COROSIO_DECL uring_tcp_socket final endpoint_state::unresolved, std::memory_order_release); } + /** Recycle into the owning service's pool at zero references. + + Defined out-of-line after `uring_tcp_service` — its body needs + the service's complete type to reach the private `pool_` + member (reachable because this impl is a friend), and this + non-template member would otherwise require it right here, + before the service class exists in the file. + */ + void retire() noexcept override; + + /** Reset op slots, cached endpoints, and the speculation hint for + recycling. + + `close_socket()` already drove fd_, the endpoints, and the + endpoint-resolution state to their closed values before the + refcount reached zero; each op's own `prepare()` overwrites its + iovec/addr scratch before the next use. What is left: `family_` + (cached by `open_socket()`, but never touched by the accepted- + connection `adopt_fd()` path, so a socket recycled through + `adopt_fd()` would otherwise keep the prior session's family), + the speculation hint (a per-connection performance heuristic + that must not leak into the next logical connection), and the + `stop_cb` invariant on every op slot. + + @pre refs_ == 0, fd closed, no op in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(fd_ == -1); + BOOST_COROSIO_ASSERT(local_endpoint_ == endpoint{}); + BOOST_COROSIO_ASSERT(remote_endpoint_ == endpoint{}); + BOOST_COROSIO_ASSERT(!rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wr_.stop_cb); + BOOST_COROSIO_ASSERT(!conn_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_op_.stop_cb); + family_ = AF_UNSPEC; + spec_.reset(); + } + endpoint local_endpoint() const noexcept override { // Lazy resolution: only fire the getsockname syscall when @@ -484,9 +525,10 @@ class BOOST_COROSIO_DECL uring_tcp_socket final Satisfies the `tcp_service` interface so the generic `tcp_socket` front-end can call `open_socket` and `bind_socket` transparently. - Socket impls are reference-counted inside the service map; raw - pointers returned from `construct()` remain valid until `destroy()` - or `shutdown()` is called. + Socket impls live in the service's `object_pool`. `construct()` + hands out a pooled impl with one reference already held; `destroy()` + releases it, and `retire()` recycles it onto the free list + instead of freeing it. @par Thread Safety All public member functions are thread-safe. @@ -649,7 +691,7 @@ class BOOST_COROSIO_DECL uring_tcp_service final */ uring_tcp_socket* adopt_fd(int fd, endpoint const& peer) { - auto p = std::make_shared(*this, *sched_); + auto* p = this->acquire_impl(); p->fd_ = fd; p->remote_endpoint_ = peer; // Mark the local endpoint as authoritative-but-unresolved. @@ -660,10 +702,16 @@ class BOOST_COROSIO_DECL uring_tcp_service final uring_tcp_socket::endpoint_state::lazy_pending, std::memory_order_release); - return this->register_impl(std::move(p)); + return p; } }; +inline void +uring_tcp_socket::retire() noexcept +{ + svc_->pool_.recycle(this); +} + /** TCP acceptor implementation for io_uring. Inherits the multishot machinery (parked-fd queue, waiter queue, @@ -678,7 +726,8 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor final uring_tcp_acceptor, tcp_acceptor::implementation, endpoint, - uring_tcp_service> + uring_tcp_service, + uring_tcp_acceptor_service> { friend uring_tcp_acceptor_service; @@ -686,7 +735,8 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor final uring_tcp_acceptor, tcp_acceptor::implementation, endpoint, - uring_tcp_service>; + uring_tcp_service, + uring_tcp_acceptor_service>; // Readiness-wait slot. The multishot accept op delivers accepted // fds, but `wait()` reports raw poll readiness on the listening fd @@ -695,11 +745,21 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor final public: explicit uring_tcp_acceptor( - uring_tcp_acceptor_service&, + uring_tcp_acceptor_service& svc, uring_scheduler& sched, uring_tcp_service& peer_svc) noexcept - : base_type(sched, peer_svc) + : base_type(svc, sched, peer_svc) + { + } + + /** Extend the base reset with this acceptor's own `wait_op_` slot. + + @pre refs_ == 0, no wait in flight (see base_type::reuse()). + */ + void reuse() noexcept { + base_type::reuse(); + BOOST_COROSIO_ASSERT(!wait_op_.stop_cb); } std::coroutine_handle<> accept( @@ -757,7 +817,7 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor final // Errors are not consumed by the accept machinery, so the // error wait still polls the descriptor. wait_op_.prepare( - h, ex, ec, this->fd_, this->sched_, this->shared_from_this(), + h, ex, ec, this->fd_, this->sched_, detail::object_ref(this), POLLPRI | POLLERR | POLLHUP, token); this->sched_->work_started(); if (wait_op_.cancelled.load(std::memory_order_acquire)) @@ -788,9 +848,10 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor final `tcp_acceptor` front-end can call `open_acceptor_socket`, `bind_acceptor`, and `listen_acceptor` transparently. - Acceptor impls are reference-counted inside the service map; raw - pointers returned from `construct()` remain valid until `destroy()` - or `shutdown()` is called. + Acceptor impls live in the service's `object_pool`. `construct()` + hands out a pooled impl with one reference already held; `destroy()` + releases it, and `retire()` recycles it onto the free list + instead of freeing it. @par Thread Safety All public member functions are thread-safe. @@ -798,6 +859,9 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor final class BOOST_COROSIO_DECL uring_tcp_acceptor_service final : public tcp_acceptor_service { + template + friend class uring_multishot_acceptor_base; + public: /// Identifies this service for `execution_context` lookup. using key_type = tcp_acceptor_service; @@ -815,39 +879,40 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor_service final void shutdown() override { - std::vector> live; + // See uring_socket_service_base::shutdown(): snapshot under an + // acquired reference, cancel without the pool lock held; + // shutdown() sets shutting-down and takes the snapshot in one + // critical section, so a cancel that drops the last ref deletes + // rather than recycles. + std::vector live; + pool_.shutdown( + [&](uring_tcp_acceptor* a) + { + acquire(a); + live.push_back(a); + }); + for (auto* a : live) { - std::lock_guard lk(mutex_); - live.reserve(impls_.size()); - for (auto& [_, p] : impls_) - live.push_back(p); + a->cancel(); + release(a); } - // Cancel without the lock held to avoid inversion if cancel() - // re-enters the service. - for (auto& p : live) - p->cancel(); } io_object::implementation* construct() override { - auto p = - std::make_shared(*this, *sched_, *peer_svc_); - auto* raw = p.get(); - std::lock_guard lk(mutex_); - impls_.emplace(raw, std::move(p)); - return raw; + return acquire_impl(); } void destroy(io_object::implementation* p) override { if (!p) return; - std::lock_guard lk(mutex_); - impls_.erase(static_cast(p)); + release(static_cast(p)); } // Close the fd eagerly when tcp_acceptor::close() is called, before - // destroy() drops the shared_ptr and the destructor runs. + // destroy() releases the service's reference and retire() may + // recycle this impl. void close(io_object::handle& h) override { auto* acc = static_cast(h.get()); @@ -860,15 +925,16 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor_service final sched_->cancel_and_flush(acc->fd_); acc->drain_waiters_only(); ::close(acc->fd_); - acc->fd_ = -1; + acc->fd_ = -1; + acc->local_endpoint_ = endpoint{}; - // Break the multi_op_ -> impl_ptr (shared_ptr) cycle + // Break the multi_op_ -> object_ref_ (object_ref) cycle // start_multishot established. The acceptor destructor's // drain_cqes_for(multi_op_.get()) is the safety net; here - // we just drop the cycle so the impl can be released when - // the user's last shared_ptr does. + // we just drop the cycle so the impl can be recycled once + // its last object_ref_ releases. if (acc->multi_op_) - acc->multi_op_->impl_ptr.reset(); + acc->multi_op_->object_ref_.reset(); } } @@ -1002,11 +1068,15 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor_service final } private: + /// Pop a recycled impl or news one, with the service reference held. + uring_tcp_acceptor* acquire_impl() + { + return pool_.acquire(*this, *sched_, *peer_svc_); + } + uring_scheduler* sched_; uring_tcp_service* peer_svc_; - std::mutex mutex_; - std::unordered_map> - impls_; + object_pool pool_; }; /** Unix domain stream socket implementation for io_uring. @@ -1016,8 +1086,8 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor_service final kernel via `uring_submit_op` and complete through the ring's CQE path. - The object is always owned by a `shared_ptr` managed by the service. - In-flight ops hold an additional `shared_ptr` copy (`impl_ptr`) so + The service holds one intrusive reference for as long as the impl + is live; in-flight ops hold an additional `object_ref_` keepalive so the kernel's user-data pointer remains valid until the CQE arrives. @par Thread Safety @@ -1030,11 +1100,12 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final uring_local_stream_socket, local_stream_socket::implementation, corosio::local_endpoint> + , public intrusive_list::node { friend uring_local_stream_service; - uring_scheduler* sched_ = nullptr; - [[maybe_unused]] uring_local_stream_service* svc_ = nullptr; + uring_scheduler* sched_ = nullptr; + uring_local_stream_service* svc_ = nullptr; // fd_ and local_endpoint_ live in native_socket_base, which also // provides native_handle/is_open/set_option/get_option/local_endpoint. @@ -1129,7 +1200,7 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final return dispatch_coro(ex, rd_.cont); } rd_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, token); if (stop_now) rd_.cancelled.store(true, std::memory_order_release); @@ -1144,7 +1215,7 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final } rd_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, buffers, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, token); sched_->work_started(); if (rd_.cancelled.load(std::memory_order_acquire)) @@ -1208,7 +1279,7 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final return dispatch_coro(ex, wr_.cont); } wr_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, token); if (stop_now) wr_.cancelled.store(true, std::memory_order_release); @@ -1223,7 +1294,7 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final } wr_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, buffers, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, token); sched_->work_started(); if (wr_.cancelled.load(std::memory_order_acquire)) @@ -1259,7 +1330,7 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final } conn_.addrlen = to_sockaddr(ep, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), ep, + h, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, &local_endpoint_, token); conn_.cancelled.store(true, std::memory_order_release); sched_->work_started(); @@ -1274,7 +1345,7 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final // a subsequent IORING_OP_CONNECT would see EALREADY — avoid. conn_.addrlen = to_sockaddr(ep, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), ep, &remote_endpoint_, + h, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, &local_endpoint_, token); sched_->work_started(); if (conn_.cancelled.load(std::memory_order_acquire)) @@ -1308,7 +1379,7 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final break; } wait_op_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), poll_flags, token); + h, ex, ec, fd_, sched_, detail::object_ref(this), poll_flags, token); sched_->work_started(); if (wait_op_.cancelled.load(std::memory_order_acquire)) { @@ -1365,6 +1436,27 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final remote_endpoint_ = corosio::local_endpoint{}; } + /// Recycle into the owning service's pool. See + /// uring_tcp_socket::retire() for why this is out-of-line. + void retire() noexcept override; + + /** Reset op slots, cached endpoints, and the speculation hint for + recycling. See uring_tcp_socket::reuse() for the rationale. + + @pre refs_ == 0, fd closed, no op in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(fd_ == -1); + BOOST_COROSIO_ASSERT(local_endpoint_ == corosio::local_endpoint{}); + BOOST_COROSIO_ASSERT(remote_endpoint_ == corosio::local_endpoint{}); + BOOST_COROSIO_ASSERT(!rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wr_.stop_cb); + BOOST_COROSIO_ASSERT(!conn_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_op_.stop_cb); + spec_.reset(); + } + corosio::local_endpoint remote_endpoint() const noexcept override { return remote_endpoint_; @@ -1378,9 +1470,10 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final generic `local_stream_socket` front-end can call `open_socket` and `assign_socket` transparently. - Socket impls are reference-counted inside the service map; raw - pointers returned from `construct()` remain valid until `destroy()` - or `shutdown()` is called. + Socket impls live in the service's `object_pool`. `construct()` + hands out a pooled impl with one reference already held; `destroy()` + releases it, and `retire()` recycles it onto the free list + instead of freeing it. @par Thread Safety All public member functions are thread-safe. @@ -1504,7 +1597,7 @@ class BOOST_COROSIO_DECL uring_local_stream_service final uring_local_stream_socket* adopt_fd(int fd, corosio::local_endpoint const& peer) { - auto p = std::make_shared(*this, *sched_); + auto* p = this->acquire_impl(); p->fd_ = fd; p->remote_endpoint_ = peer; @@ -1513,10 +1606,16 @@ class BOOST_COROSIO_DECL uring_local_stream_service final if (::getsockname(fd, reinterpret_cast(&local), &len) == 0) p->local_endpoint_ = sockaddr_to_local_endpoint(local, len); - return this->register_impl(std::move(p)); + return p; } }; +inline void +uring_local_stream_socket::retire() noexcept +{ + svc_->pool_.recycle(this); +} + /** Local-stream (Unix domain) acceptor for io_uring. Inherits all multishot machinery (parked-fd queue, waiter queue, @@ -1530,7 +1629,8 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor final uring_local_stream_acceptor, local_stream_acceptor::implementation, corosio::local_endpoint, - uring_local_stream_service> + uring_local_stream_service, + uring_local_stream_acceptor_service> { friend uring_local_stream_acceptor_service; @@ -1538,18 +1638,29 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor final uring_local_stream_acceptor, local_stream_acceptor::implementation, corosio::local_endpoint, - uring_local_stream_service>; + uring_local_stream_service, + uring_local_stream_acceptor_service>; // Readiness-wait slot. See uring_tcp_acceptor::wait_op_. uring_wait_op wait_op_; public: explicit uring_local_stream_acceptor( - uring_local_stream_acceptor_service&, + uring_local_stream_acceptor_service& svc, uring_scheduler& sched, uring_local_stream_service& peer_svc) noexcept - : base_type(sched, peer_svc) + : base_type(svc, sched, peer_svc) + { + } + + /** Extend the base reset with this acceptor's own `wait_op_` slot. + + @pre refs_ == 0, no wait in flight (see base_type::reuse()). + */ + void reuse() noexcept { + base_type::reuse(); + BOOST_COROSIO_ASSERT(!wait_op_.stop_cb); } std::coroutine_handle<> accept( @@ -1607,7 +1718,7 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor final // Errors are not consumed by the accept machinery, so the // error wait still polls the descriptor. wait_op_.prepare( - h, ex, ec, this->fd_, this->sched_, this->shared_from_this(), + h, ex, ec, this->fd_, this->sched_, detail::object_ref(this), POLLPRI | POLLERR | POLLHUP, token); this->sched_->work_started(); if (wait_op_.cancelled.load(std::memory_order_acquire)) @@ -1639,9 +1750,10 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor final `open_acceptor_socket`, `bind_acceptor`, and `listen_acceptor` transparently. - Acceptor impls are reference-counted inside the service map; raw - pointers returned from `construct()` remain valid until `destroy()` - or `shutdown()` is called. + Acceptor impls live in the service's `object_pool`. `construct()` + hands out a pooled impl with one reference already held; `destroy()` + releases it, and `retire()` recycles it onto the free list + instead of freeing it. @par Thread Safety All public member functions are thread-safe. @@ -1649,6 +1761,9 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor final class BOOST_COROSIO_DECL uring_local_stream_acceptor_service final : public local_stream_acceptor_service { + template + friend class uring_multishot_acceptor_base; + public: /// Identifies this service for `execution_context` lookup. using key_type = local_stream_acceptor_service; @@ -1666,39 +1781,36 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor_service final void shutdown() override { - std::vector> live; + // See uring_tcp_acceptor_service::shutdown() for the ordering + // rationale. + std::vector live; + pool_.shutdown( + [&](uring_local_stream_acceptor* a) + { + acquire(a); + live.push_back(a); + }); + for (auto* a : live) { - std::lock_guard lk(mutex_); - live.reserve(impls_.size()); - for (auto& [_, p] : impls_) - live.push_back(p); + a->cancel(); + release(a); } - // Cancel without the lock held to avoid inversion if cancel() - // re-enters the service. - for (auto& p : live) - p->cancel(); } io_object::implementation* construct() override { - auto p = std::make_shared( - *this, *sched_, *peer_svc_); - auto* raw = p.get(); - std::lock_guard lk(mutex_); - impls_.emplace(raw, std::move(p)); - return raw; + return acquire_impl(); } void destroy(io_object::implementation* p) override { if (!p) return; - std::lock_guard lk(mutex_); - impls_.erase(static_cast(p)); + release(static_cast(p)); } // Close the fd eagerly when local_stream_acceptor::close() is called, - // before destroy() drops the shared_ptr and the destructor runs. + // before destroy() drops the service's reference and recycling runs. void close(io_object::handle& h) override { auto* acc = static_cast(h.get()); @@ -1709,13 +1821,14 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor_service final sched_->cancel_and_flush(acc->fd_); acc->drain_waiters_only(); ::close(acc->fd_); - acc->fd_ = -1; + acc->fd_ = -1; + acc->local_endpoint_ = corosio::local_endpoint{}; - // Break the multi_op_ -> impl_ptr (shared_ptr) cycle + // Break the multi_op_ -> object_ref_ (object_ref) cycle // start_multishot established. See the symmetric comment // in uring_tcp_acceptor_service::close. if (acc->multi_op_) - acc->multi_op_->impl_ptr.reset(); + acc->multi_op_->object_ref_.reset(); } } @@ -1844,13 +1957,15 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor_service final } private: + /// Pop a recycled impl or news one, with the service reference held. + uring_local_stream_acceptor* acquire_impl() + { + return pool_.acquire(*this, *sched_, *peer_svc_); + } + uring_scheduler* sched_; uring_local_stream_service* peer_svc_; - std::mutex mutex_; - std::unordered_map< - uring_local_stream_acceptor*, - std::shared_ptr> - impls_; + object_pool pool_; }; /** UDP socket implementation for io_uring. @@ -1860,8 +1975,8 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor_service final to the kernel via `uring_submit_op` and complete through the ring's CQE path. - The object is always owned by a `shared_ptr` managed by the service. - In-flight ops hold an additional `shared_ptr` copy (`impl_ptr`) so + The service holds one intrusive reference for as long as the impl + is live; in-flight ops hold an additional `object_ref_` keepalive so the kernel's user-data pointer remains valid until the CQE arrives. @par Thread Safety @@ -1874,12 +1989,13 @@ class BOOST_COROSIO_DECL uring_udp_socket final uring_udp_socket, udp_socket::implementation, corosio::endpoint> + , public intrusive_list::node { friend uring_udp_service; int family_ = AF_UNSPEC; // cached at open_socket uring_scheduler* sched_ = nullptr; - [[maybe_unused]] uring_udp_service* svc_ = nullptr; + uring_udp_service* svc_ = nullptr; // fd_ and local_endpoint_ live in native_socket_base, which also // provides native_handle/is_open/set_option/get_option/local_endpoint. @@ -1994,7 +2110,7 @@ class BOOST_COROSIO_DECL uring_udp_socket final } conn_.addrlen = to_sockaddr(ep, family_, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), ep, + h, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, &local_endpoint_, token); conn_.cancelled.store(true, std::memory_order_release); sched_->work_started(); @@ -2009,7 +2125,7 @@ class BOOST_COROSIO_DECL uring_udp_socket final // a prior speculative ::connect would leave EINPROGRESS → EALREADY. conn_.addrlen = to_sockaddr(ep, family_, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), ep, &remote_endpoint_, + h, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, &local_endpoint_, token); sched_->work_started(); if (conn_.cancelled.load(std::memory_order_acquire)) @@ -2043,7 +2159,7 @@ class BOOST_COROSIO_DECL uring_udp_socket final break; } wait_op_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), poll_flags, token); + h, ex, ec, fd_, sched_, detail::object_ref(this), poll_flags, token); sched_->work_started(); if (wait_op_.cancelled.load(std::memory_order_acquire)) { @@ -2099,6 +2215,33 @@ class BOOST_COROSIO_DECL uring_udp_socket final remote_endpoint_ = endpoint{}; } + /// Recycle into the owning service's pool. See + /// uring_tcp_socket::retire() for why this is out-of-line. + void retire() noexcept override; + + /** Reset op slots, cached endpoints, the speculation hint, and + `family_` for recycling. `family_` is set by `open_socket()`/ + `assign_socket()`, not by this reset, so resetting it to + `AF_UNSPEC` here is defensive — it guards any future entry + path that constructs a usable socket without going through + either of those, the same hazard `uring_tcp_socket::reuse()` + documents for its `adopt_fd()` path. + + @pre refs_ == 0, fd closed, no op in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(fd_ == -1); + BOOST_COROSIO_ASSERT(local_endpoint_ == endpoint{}); + BOOST_COROSIO_ASSERT(remote_endpoint_ == endpoint{}); + BOOST_COROSIO_ASSERT(!conn_.stop_cb); + BOOST_COROSIO_ASSERT(!send_.stop_cb); + BOOST_COROSIO_ASSERT(!recv_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_op_.stop_cb); + family_ = AF_UNSPEC; + spec_.reset(); + } + endpoint remote_endpoint() const noexcept override { return remote_endpoint_; @@ -2166,7 +2309,7 @@ class BOOST_COROSIO_DECL uring_udp_socket final return dispatch_coro(ex, send_.cont); } send_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, dest_len, dest_storage, to_native_msg_flags(flags), token); if (stop_now) @@ -2182,7 +2325,7 @@ class BOOST_COROSIO_DECL uring_udp_socket final } send_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, buffers, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, dest_len, dest_storage, to_native_msg_flags(flags), token); sched_->work_started(); if (send_.cancelled.load(std::memory_order_acquire)) @@ -2260,7 +2403,7 @@ class BOOST_COROSIO_DECL uring_udp_socket final return dispatch_coro(ex, recv_.cont); } recv_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, source_out, want_source ? &write_ip_source : nullptr, to_native_msg_flags(flags), token); if (stop_now) @@ -2286,7 +2429,7 @@ class BOOST_COROSIO_DECL uring_udp_socket final } recv_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, buffers, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, source_out, want_source ? &write_ip_source : nullptr, to_native_msg_flags(flags), token); sched_->work_started(); @@ -2316,9 +2459,10 @@ class BOOST_COROSIO_DECL uring_udp_socket final front-end can call `open_datagram_socket` and `bind_datagram` transparently. - Socket impls are reference-counted inside the service map; raw - pointers returned from `construct()` remain valid until `destroy()` - or `shutdown()` is called. + Socket impls live in the service's `object_pool`. `construct()` + hands out a pooled impl with one reference already held; `destroy()` + releases it, and `retire()` recycles it onto the free list + instead of freeing it. @par Thread Safety All public member functions are thread-safe. @@ -2461,6 +2605,12 @@ class BOOST_COROSIO_DECL uring_udp_service final } }; +inline void +uring_udp_socket::retire() noexcept +{ + svc_->pool_.recycle(this); +} + /** Unix domain datagram socket implementation for io_uring. Implements `local_datagram_socket::implementation` using a proactor @@ -2468,8 +2618,8 @@ class BOOST_COROSIO_DECL uring_udp_service final submitted to the kernel via `uring_submit_op` and complete through the ring's CQE path. - The object is always owned by a `shared_ptr` managed by the service. - In-flight ops hold an additional `shared_ptr` copy (`impl_ptr`) so + The service holds one intrusive reference for as long as the impl + is live; in-flight ops hold an additional `object_ref_` keepalive so the kernel's user-data pointer remains valid until the CQE arrives. @par Thread Safety @@ -2482,11 +2632,12 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final uring_local_datagram_socket, local_datagram_socket::implementation, corosio::local_endpoint> + , public intrusive_list::node { friend uring_local_datagram_service; - uring_scheduler* sched_ = nullptr; - [[maybe_unused]] uring_local_datagram_service* svc_ = nullptr; + uring_scheduler* sched_ = nullptr; + uring_local_datagram_service* svc_ = nullptr; // fd_ and local_endpoint_ live in native_socket_base, which also // provides native_handle/is_open/set_option/get_option/local_endpoint. @@ -2601,7 +2752,7 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final } conn_.addrlen = to_sockaddr(ep, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), ep, + h, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, &local_endpoint_, token); conn_.cancelled.store(true, std::memory_order_release); sched_->work_started(); @@ -2616,7 +2767,7 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final // a prior speculative ::connect would leave EINPROGRESS → EALREADY. conn_.addrlen = to_sockaddr(ep, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), ep, &remote_endpoint_, + h, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, &local_endpoint_, token); sched_->work_started(); if (conn_.cancelled.load(std::memory_order_acquire)) @@ -2650,7 +2801,7 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final break; } wait_op_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), poll_flags, token); + h, ex, ec, fd_, sched_, detail::object_ref(this), poll_flags, token); sched_->work_started(); if (wait_op_.cancelled.load(std::memory_order_acquire)) { @@ -2707,6 +2858,27 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final remote_endpoint_ = corosio::local_endpoint{}; } + /// Recycle into the owning service's pool. See + /// uring_tcp_socket::retire() for why this is out-of-line. + void retire() noexcept override; + + /** Reset op slots, cached endpoints, and the speculation hint for + recycling. See uring_tcp_socket::reuse() for the rationale. + + @pre refs_ == 0, fd closed, no op in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(fd_ == -1); + BOOST_COROSIO_ASSERT(local_endpoint_ == corosio::local_endpoint{}); + BOOST_COROSIO_ASSERT(remote_endpoint_ == corosio::local_endpoint{}); + BOOST_COROSIO_ASSERT(!conn_.stop_cb); + BOOST_COROSIO_ASSERT(!send_.stop_cb); + BOOST_COROSIO_ASSERT(!recv_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_op_.stop_cb); + spec_.reset(); + } + corosio::local_endpoint remote_endpoint() const noexcept override { return remote_endpoint_; @@ -2793,7 +2965,7 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final return dispatch_coro(ex, send_.cont); } send_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, dest_len, dest_storage, to_native_msg_flags(flags), token); if (stop_now) @@ -2809,7 +2981,7 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final } send_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, buffers, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, dest_len, dest_storage, to_native_msg_flags(flags), token); sched_->work_started(); if (send_.cancelled.load(std::memory_order_acquire)) @@ -2888,7 +3060,7 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final return dispatch_coro(ex, recv_.cont); } recv_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, source_out, want_source ? &write_local_source : nullptr, to_native_msg_flags(flags), token); @@ -2915,7 +3087,7 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final } recv_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, buffers, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, source_out, want_source ? &write_local_source : nullptr, to_native_msg_flags(flags), token); sched_->work_started(); @@ -2945,9 +3117,10 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final generic `local_datagram_socket` front-end can call `open_socket` and `bind_socket` transparently. - Socket impls are reference-counted inside the service map; raw - pointers returned from `construct()` remain valid until `destroy()` - or `shutdown()` is called. + Socket impls live in the service's `object_pool`. `construct()` + hands out a pooled impl with one reference already held; `destroy()` + releases it, and `retire()` recycles it onto the free list + instead of freeing it. @par Thread Safety All public member functions are thread-safe. @@ -3083,6 +3256,12 @@ class BOOST_COROSIO_DECL uring_local_datagram_service final } }; +inline void +uring_local_datagram_socket::retire() noexcept +{ + svc_->pool_.recycle(this); +} + } // namespace boost::corosio::detail #endif // BOOST_COROSIO_HAS_URING diff --git a/test/unit/object_reuse.cpp b/test/unit/object_reuse.cpp new file mode 100644 index 000000000..dbdefefe9 --- /dev/null +++ b/test/unit/object_reuse.cpp @@ -0,0 +1,209 @@ +// +// Copyright (c) 2026 Steve Gerbino +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// Official repository: https://github.com/cppalliance/corosio +// + +// Keepalive-extension regression for the object_pool conversion: dropping +// a socket while an op is still parked must not use-after-free the +// impl, and a socket constructed afterward on the same context must +// get a fully working impl, whether the pool recycled the dropped one +// or allocated a new one -- the caller cannot tell which, and must not +// need to. + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +#include "context.hpp" +#include "test_suite.hpp" + +namespace boost::corosio { + +template +struct object_reuse_test +{ + // Open a connected pair, park a read on one side with no writer + // on the other, then drop that socket while the read is still + // parked. close()/destroy() cancel the read and release the + // service's own reference; the read's own reference (its object_ref + // keepalive) keeps the impl alive until the cancellation actually + // drains, at which point the impl is recycled -- not freed. A + // sanitizer build is what actually validates no UAF happened; this + // test's own assertion is the behavioral half: the context keeps + // working afterward. + void testAbandonedReadThenReuse() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + // Declared at function scope, not inside the block below: if + // the abandoned read's cancellation is ever actually delivered + // (reactor backends always drain it; io_uring may instead + // abandon it in the kernel ring at shutdown), the coroutine + // resumes into this lambda and writes `resumed` — which must + // still be live stack memory whenever that happens, including + // after the block below has already exited. + char buf[16]; + bool resumed = false; + + { + auto [s1, s2] = test::make_socket_pair(ioc); + std::ignore = s2; // keeps the peer fd open; never written to + + capy::run_async(ex)( + [](tcp_socket& s, char* b, std::size_t n, + bool& done) -> capy::task<> { + std::ignore = + co_await s.read_some(capy::mutable_buffer(b, n)); + done = true; + }(s1, buf, sizeof(buf), resumed)); + + // Let the read actually register as parked before s1 (and + // its parked read) go out of scope at the end of this block. + std::ignore = ioc.run_one(); + } + + // Drain whatever the destructor's cancellation produced. This + // must not crash, hang, or corrupt the pool regardless of + // whether the cancellation's completion ever arrives (io_uring + // may abandon it in the kernel ring at io_context shutdown; + // the reactor backends always drain it here). + ioc.run(); + ioc.restart(); + + // A fresh socket on the same context must get a working impl. + tcp_socket probe(ioc); + BOOST_TEST(!probe.open()); + BOOST_TEST(probe.is_open()); + + auto [p1, p2] = test::make_socket_pair(ioc); + char const out = 'x'; + char in = 0; + bool wrote = false, read_ok = false; + capy::run_async(ex)( + [](tcp_socket& s, char const* b, bool& done) -> capy::task<> { + auto [ec, n] = co_await s.write_some(capy::const_buffer(b, 1)); + done = !ec && n == 1; + }(p1, &out, wrote)); + capy::run_async(ex)( + [](tcp_socket& s, char* b, bool& done) -> capy::task<> { + auto [ec, n] = co_await s.read_some(capy::mutable_buffer(b, 1)); + done = !ec && n == 1; + }(p2, &in, read_ok)); + ioc.restart(); + ioc.run(); + + BOOST_TEST(wrote); + BOOST_TEST(read_ok); + BOOST_TEST_EQ(in, out); + } + + // io_uring's multishot accept can deliver a connection before any + // accept() is outstanding to claim it; that connection's fd (and + // tracking node) parks in the acceptor impl's ready_fds_ list. If + // the acceptor is destroyed with a parked fd still there, draining + // it was originally destructor-only -- which never runs on the + // recycle path (reaching zero references usually means "back on + // the free list", not "actually freed"). Reproduces deterministically: + // destroy an acceptor holding one buffered connection, then + // construct a fresh acceptor on the same context -- that pops the + // just-recycled impl and, before the fix, immediately failed + // reuse()'s `ready_fds_.empty()` assert. On epoll/select there is + // no ready_fds_/multishot concept, so this is a plain acceptor + // recycle-and-reuse sanity check there; the bug this guards against + // is uring-only. + void testAcceptorRecycleDrainsBufferedConnection() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + { + tcp_acceptor acc(ioc); + BOOST_TEST(!acc.open()); + acc.set_option(socket_option::reuse_address(true)); + BOOST_TEST(!acc.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!acc.listen()); + auto port = acc.local_endpoint().port(); + + tcp_socket client(ioc); + bool connected = false; + capy::run_async(ex)( + [](tcp_socket& s, endpoint ep, bool& done) -> capy::task<> { + auto [ec] = co_await s.connect(ep); + done = !ec; + }(client, endpoint(ipv4_address::loopback(), port), + connected)); + + // No accept() is outstanding: on uring the connection's + // multishot CQE buffers in ready_fds_ instead of completing + // an accept. run() (not poll()) drives the connector fully + // to completion, which is also where the server's accept + // CQE lands -- same idiom as + // multishot_acceptor_test::testDestroyWithBufferedConnections. + ioc.run(); + BOOST_TEST(connected); + + client.close(); + + // acc drops here holding a buffered connection in + // ready_fds_ (uring) or nothing to hold (epoll/select). + } + + ioc.restart(); + + // A fresh acceptor on the same context pops the just-recycled + // impl. Before the fix this aborted on reuse()'s + // `ready_fds_.empty()` assert (uring only). + tcp_acceptor acc2(ioc); + BOOST_TEST(!acc2.open()); + acc2.set_option(socket_option::reuse_address(true)); + BOOST_TEST(!acc2.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!acc2.listen()); + auto port2 = acc2.local_endpoint().port(); + + // Full exercise: accept a real connection through the recycled + // (or freshly allocated) impl to prove it still works. + tcp_socket server2(ioc); + tcp_socket client2(ioc); + bool accept_done = false, connect_done = false; + capy::run_async(ex)( + [](tcp_acceptor& a, tcp_socket& s, bool& done) -> capy::task<> { + auto [ec] = co_await a.accept(s); + done = !ec; + }(acc2, server2, accept_done)); + capy::run_async(ex)( + [](tcp_socket& s, endpoint ep, bool& done) -> capy::task<> { + auto [ec] = co_await s.connect(ep); + done = !ec; + }(client2, endpoint(ipv4_address::loopback(), port2), + connect_done)); + ioc.run(); + + BOOST_TEST(accept_done); + BOOST_TEST(connect_done); + BOOST_TEST(server2.is_open()); + } + + void run() + { + testAbandonedReadThenReuse(); + testAcceptorRecycleDrainsBufferedConnection(); + } +}; + +COROSIO_BACKEND_TESTS(object_reuse_test, "boost.corosio.object_reuse") + +} // namespace boost::corosio diff --git a/test/unit/resolver.cpp b/test/unit/resolver.cpp index 85ea2d2db..c5476c8c2 100644 --- a/test/unit/resolver.cpp +++ b/test/unit/resolver.cpp @@ -1297,7 +1297,7 @@ struct resolver_test // Forward resolution on IOCP dispatches through GetAddrInfoExW and // posts its completion (resolve_op, embedded in the win_resolver) to // the scheduler queue. Unlike the reverse path and both POSIX paths, - // the forward op takes no shared_from_this()/impl_ptr keepalive, so + // the forward op takes no object_ref_ keepalive, so // destroying the resolver frees the win_resolver the queued // resolve_op lives in before teardown drains that op -- a // use-after-free that ASan catches. localhost resolves from the hosts @@ -1310,7 +1310,7 @@ struct resolver_test // only under ASan; without a sanitizer the freed read is silent. // // Contrast the reverse path, which holds the keepalive on - // reverse_op_.impl_ptr across the queued completion + // reverse_op_.object_ref_ across the queued completion // (win_resolver_service.hpp do_reverse_resolve_work / do_complete), // and so survives teardown intact. void testDestroyWithForwardResolveQueued() From b8225102870004aa144397f6651f118beb832333 Mon Sep 17 00:00:00 2001 From: Steve Gerbino Date: Wed, 7 Oct 2026 23:40:19 +0200 Subject: [PATCH 03/12] refactor: collapse IOCP wrappers and recycle impls through object_pool --- .../native/detail/iocp/win_file_service.hpp | 262 +-- .../detail/iocp/win_local_stream_acceptor.hpp | 91 +- .../win_local_stream_acceptor_service.hpp | 239 +-- .../detail/iocp/win_local_stream_service.hpp | 401 ++--- .../detail/iocp/win_local_stream_socket.hpp | 130 +- .../detail/iocp/win_random_access_file.hpp | 85 +- .../iocp/win_random_access_file_service.hpp | 240 +-- .../native/detail/iocp/win_resolver.hpp | 40 +- .../detail/iocp/win_resolver_service.hpp | 85 +- .../corosio/native/detail/iocp/win_signal.hpp | 21 + .../native/detail/iocp/win_signals.hpp | 97 +- .../native/detail/iocp/win_stream_file.hpp | 93 +- .../native/detail/iocp/win_tcp_acceptor.hpp | 101 +- .../detail/iocp/win_tcp_acceptor_service.hpp | 1576 ++--------------- .../native/detail/iocp/win_tcp_service.hpp | 1094 +++++++++++- .../native/detail/iocp/win_tcp_socket.hpp | 126 +- .../native/detail/iocp/win_udp_service.hpp | 397 ++--- .../native/detail/iocp/win_udp_socket.hpp | 149 +- src/corosio/src/tcp_socket.cpp | 18 +- 19 files changed, 2026 insertions(+), 3219 deletions(-) diff --git a/include/boost/corosio/native/detail/iocp/win_file_service.hpp b/include/boost/corosio/native/detail/iocp/win_file_service.hpp index 42dd5860a..73c937504 100644 --- a/include/boost/corosio/native/detail/iocp/win_file_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_file_service.hpp @@ -18,8 +18,7 @@ #include #include #include -#include -#include +#include #include #include #include @@ -42,11 +41,12 @@ namespace boost::corosio::detail { */ class BOOST_COROSIO_DECL win_file_service final : public file_service { + friend class win_stream_file; + public: using key_type = win_file_service; explicit win_file_service(capy::execution_context& ctx); - ~win_file_service(); win_file_service(win_file_service const&) = delete; win_file_service& operator=(win_file_service const&) = delete; @@ -61,9 +61,6 @@ class BOOST_COROSIO_DECL win_file_service final : public file_service std::filesystem::path const& path, file_base::flags mode) override; - void destroy_impl(win_stream_file& impl); - void unregister_impl(win_stream_file_internal& impl); - void post(overlapped_op* op); void on_pending(overlapped_op* op) noexcept; void on_completion(overlapped_op* op, DWORD error, DWORD bytes) noexcept; @@ -100,14 +97,12 @@ class BOOST_COROSIO_DECL win_file_service final : public file_service LONG(NTAPI*)(HANDLE, ULONG, void*, ULONG, io_status_block*); win_scheduler& sched_; + void* iocp_; + nt_flush_fn nt_flush_buffers_file_ex_; BOOST_COROSIO_MSVC_WARNING_PUSH BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // detail:: members, dll-interface - win_mutex mutex_; - intrusive_list file_list_; - intrusive_list wrapper_list_; + object_pool pool_; BOOST_COROSIO_MSVC_WARNING_POP - void* iocp_; - nt_flush_fn nt_flush_buffers_file_ex_; }; /** Get or create the stream file service for the given context. */ @@ -121,14 +116,14 @@ get_stream_file_service(capy::execution_context& ctx, win_scheduler&) // Operation constructors // --------------------------------------------------------------------------- -inline file_read_op::file_read_op(win_stream_file_internal& f) noexcept +inline file_read_op::file_read_op(win_stream_file& f) noexcept : overlapped_op(&do_complete) , file_(f) { cancel_func_ = &do_cancel_impl; } -inline file_write_op::file_write_op(win_stream_file_internal& f) noexcept +inline file_write_op::file_write_op(win_stream_file& f) noexcept : overlapped_op(&do_complete) , file_(f) { @@ -145,7 +140,7 @@ file_read_op::do_cancel_impl(overlapped_op* base) noexcept auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); if (op->file_.is_open()) - ::CancelIoEx(op->file_.native_handle(), op); + ::CancelIoEx(op->file_.handle_, op); } inline void @@ -154,7 +149,7 @@ file_write_op::do_cancel_impl(overlapped_op* base) noexcept auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); if (op->file_.is_open()) - ::CancelIoEx(op->file_.native_handle(), op); + ::CancelIoEx(op->file_.handle_, op); } // --------------------------------------------------------------------------- @@ -173,7 +168,7 @@ file_read_op::do_complete( if (!owner) { op->cleanup_only(); - op->file_ptr.reset(); + op->object_ref_.reset(); return; } @@ -181,7 +176,7 @@ file_read_op::do_complete( if (op->dwError == 0 && op->bytes_transferred > 0) op->file_.offset_ += op->bytes_transferred; - auto prevent_premature_destruction = std::move(op->file_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } @@ -197,7 +192,7 @@ file_write_op::do_complete( if (!owner) { op->cleanup_only(); - op->file_ptr.reset(); + op->object_ref_.reset(); return; } @@ -205,41 +200,44 @@ file_write_op::do_complete( if (op->dwError == 0 && op->bytes_transferred > 0) op->file_.offset_ += op->bytes_transferred; - auto prevent_premature_destruction = std::move(op->file_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } // --------------------------------------------------------------------------- -// win_stream_file_internal +// win_stream_file // --------------------------------------------------------------------------- -inline win_stream_file_internal::win_stream_file_internal( - win_file_service& svc) noexcept +inline win_stream_file::win_stream_file(win_file_service& svc) noexcept : svc_(svc) , rd_(*this) , wr_(*this) { } -inline win_stream_file_internal::~win_stream_file_internal() +inline void +win_stream_file::reuse() noexcept { - svc_.unregister_impl(*this); + BOOST_COROSIO_ASSERT(handle_ == INVALID_HANDLE_VALUE); + BOOST_COROSIO_ASSERT(offset_ == 0); + BOOST_COROSIO_ASSERT(!rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wr_.stop_cb); } -inline HANDLE -win_stream_file_internal::native_handle() const noexcept +inline native_handle_type +win_stream_file::native_handle() const noexcept { - return handle_; + return reinterpret_cast(handle_); } inline bool -win_stream_file_internal::is_open() const noexcept +win_stream_file::is_open() const noexcept { return handle_ != INVALID_HANDLE_VALUE; } inline void -win_stream_file_internal::cancel() noexcept +win_stream_file::cancel() noexcept { if (handle_ != INVALID_HANDLE_VALUE) ::CancelIoEx(handle_, nullptr); @@ -249,7 +247,7 @@ win_stream_file_internal::cancel() noexcept } inline void -win_stream_file_internal::close_handle() noexcept +win_stream_file::close_handle() noexcept { if (handle_ != INVALID_HANDLE_VALUE) { @@ -261,7 +259,7 @@ win_stream_file_internal::close_handle() noexcept } inline std::uint64_t -win_stream_file_internal::size() const +win_stream_file::size() const { LARGE_INTEGER li; if (!::GetFileSizeEx(handle_, &li)) @@ -270,7 +268,7 @@ win_stream_file_internal::size() const } inline std::error_code -win_stream_file_internal::resize(std::uint64_t new_size) noexcept +win_stream_file::resize(std::uint64_t new_size) noexcept { LARGE_INTEGER li; li.QuadPart = static_cast(new_size); @@ -282,7 +280,7 @@ win_stream_file_internal::resize(std::uint64_t new_size) noexcept } inline std::error_code -win_stream_file_internal::sync_data() noexcept +win_stream_file::sync_data() noexcept { // Attempt data-only flush; fall back to full flush if (svc_.try_flush_data(handle_)) @@ -293,7 +291,7 @@ win_stream_file_internal::sync_data() noexcept } inline std::error_code -win_stream_file_internal::sync_all() noexcept +win_stream_file::sync_all() noexcept { if (!::FlushFileBuffers(handle_)) return make_err(::GetLastError()); @@ -301,7 +299,7 @@ win_stream_file_internal::sync_all() noexcept } inline native_handle_type -win_stream_file_internal::release() +win_stream_file::release() { HANDLE h = handle_; handle_ = INVALID_HANDLE_VALUE; @@ -310,7 +308,7 @@ win_stream_file_internal::release() } inline std::error_code -win_stream_file_internal::assign(native_handle_type handle) noexcept +win_stream_file::assign(native_handle_type handle) noexcept { close_handle(); HANDLE h = reinterpret_cast(handle); @@ -326,8 +324,7 @@ win_stream_file_internal::assign(native_handle_type handle) noexcept } inline capy::io_result -win_stream_file_internal::seek( - std::int64_t offset, file_base::seek_basis origin) noexcept +win_stream_file::seek(std::int64_t offset, file_base::seek_basis origin) noexcept { // We manage offset_ ourselves (same as POSIX impl). std::int64_t new_pos; @@ -356,7 +353,7 @@ win_stream_file_internal::seek( } inline std::coroutine_handle<> -win_stream_file_internal::read_some( +win_stream_file::read_some( std::coroutine_handle<> h, capy::executor_ref ex, buffer_param param, @@ -366,8 +363,8 @@ win_stream_file_internal::read_some( { static constexpr std::size_t max_buffers = 16; - // Keep internal alive during I/O - rd_.file_ptr = shared_from_this(); + // Keep this file alive during I/O + rd_.object_ref_ = detail::object_ref(this); auto& op = rd_; op.reset(); @@ -428,7 +425,7 @@ win_stream_file_internal::read_some( } inline std::coroutine_handle<> -win_stream_file_internal::write_some( +win_stream_file::write_some( std::coroutine_handle<> h, capy::executor_ref ex, buffer_param param, @@ -438,8 +435,8 @@ win_stream_file_internal::write_some( { static constexpr std::size_t max_buffers = 16; - // Keep internal alive during I/O - wr_.file_ptr = shared_from_this(); + // Keep this file alive during I/O + wr_.object_ref_ = detail::object_ref(this); auto& op = wr_; op.reset(); @@ -497,109 +494,10 @@ win_stream_file_internal::write_some( return std::noop_coroutine(); } -// --------------------------------------------------------------------------- -// win_stream_file wrapper -// --------------------------------------------------------------------------- - -inline win_stream_file::win_stream_file( - std::shared_ptr internal) noexcept - : internal_(std::move(internal)) -{ -} - -inline void -win_stream_file::close_internal() noexcept -{ - if (internal_) - { - internal_->close_handle(); - internal_.reset(); - } -} - -inline std::coroutine_handle<> -win_stream_file::read_some( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->read_some(h, d, buf, token, ec, bytes); -} - -inline std::coroutine_handle<> -win_stream_file::write_some( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->write_some(h, d, buf, token, ec, bytes); -} - -inline native_handle_type -win_stream_file::native_handle() const noexcept -{ - return reinterpret_cast(internal_->native_handle()); -} - inline void -win_stream_file::cancel() noexcept -{ - internal_->cancel(); -} - -inline std::uint64_t -win_stream_file::size() const -{ - return internal_->size(); -} - -inline std::error_code -win_stream_file::resize(std::uint64_t new_size) noexcept -{ - return internal_->resize(new_size); -} - -inline std::error_code -win_stream_file::sync_data() noexcept -{ - return internal_->sync_data(); -} - -inline std::error_code -win_stream_file::sync_all() noexcept -{ - return internal_->sync_all(); -} - -inline native_handle_type -win_stream_file::release() -{ - return internal_->release(); -} - -inline std::error_code -win_stream_file::assign(native_handle_type handle) noexcept -{ - return internal_->assign(handle); -} - -inline capy::io_result -win_stream_file::seek( - std::int64_t offset, file_base::seek_basis origin) noexcept -{ - return internal_->seek(offset, origin); -} - -inline win_stream_file_internal* -win_stream_file::get_internal() const noexcept +win_stream_file::retire() noexcept { - return internal_.get(); + svc_.pool_.recycle(this); } // --------------------------------------------------------------------------- @@ -619,61 +517,32 @@ inline win_file_service::win_file_service(capy::execution_context& ctx) } } -inline win_file_service::~win_file_service() -{ - for (auto* w = wrapper_list_.pop_front(); w != nullptr; - w = wrapper_list_.pop_front()) - delete w; -} - inline io_object::implementation* win_file_service::construct() { - auto internal = std::make_shared(*this); - - { - std::lock_guard lock(mutex_); - file_list_.push_back(internal.get()); - } - - auto* wrapper = new win_stream_file(std::move(internal)); - - { - std::lock_guard lock(mutex_); - wrapper_list_.push_back(wrapper); - } - - return wrapper; + return pool_.acquire(*this); } inline void win_file_service::destroy(io_object::implementation* p) { - if (p) - { - auto& wrapper = static_cast(*p); - wrapper.close_internal(); - destroy_impl(wrapper); - } + if (!p) + return; + auto* f = static_cast(p); + f->close_handle(); + release(f); } inline void win_file_service::close(io_object::handle& h) { - auto& wrapper = static_cast(*h.get()); - wrapper.get_internal()->close_handle(); + static_cast(h.get())->close_handle(); } inline void win_file_service::shutdown() { - std::lock_guard lock(mutex_); - - for (auto* impl = file_list_.pop_front(); impl != nullptr; - impl = file_list_.pop_front()) - { - impl->close_handle(); - } + pool_.shutdown([](win_stream_file* impl) { impl->close_handle(); }); } inline std::error_code @@ -736,9 +605,9 @@ win_file_service::open_file( } } - auto& internal = *static_cast(impl).get_internal(); - internal.handle_ = h; - internal.offset_ = 0; + auto& file = static_cast(impl); + file.handle_ = h; + file.offset_ = 0; // Handle append: seek to end if (mode & file_base::append) @@ -746,34 +615,17 @@ win_file_service::open_file( LARGE_INTEGER sz; if (!::GetFileSizeEx(h, &sz)) { - DWORD err = ::GetLastError(); - internal.handle_ = INVALID_HANDLE_VALUE; + DWORD err = ::GetLastError(); + file.handle_ = INVALID_HANDLE_VALUE; ::CloseHandle(h); return make_err(err); } - internal.offset_ = static_cast(sz.QuadPart); + file.offset_ = static_cast(sz.QuadPart); } return {}; } -inline void -win_file_service::destroy_impl(win_stream_file& impl) -{ - { - std::lock_guard lock(mutex_); - wrapper_list_.remove(&impl); - } - delete &impl; -} - -inline void -win_file_service::unregister_impl(win_stream_file_internal& impl) -{ - std::lock_guard lock(mutex_); - file_list_.remove(&impl); -} - inline void win_file_service::post(overlapped_op* op) { diff --git a/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor.hpp b/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor.hpp index 89fd2c0e8..c455cdca1 100644 --- a/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor.hpp +++ b/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor.hpp @@ -17,13 +17,13 @@ #include #include #include +#include #include #include #include #include #include -#include #include #include @@ -33,7 +33,7 @@ namespace boost::corosio::detail { class win_local_stream_acceptor_service; class win_local_stream_service; class win_local_stream_socket; -class win_local_stream_acceptor_internal; +class win_local_stream_acceptor; /** Accept operation state for local stream sockets. @@ -43,7 +43,7 @@ struct local_stream_accept_op : overlapped_op { SOCKET accepted_socket = INVALID_SOCKET; win_local_stream_socket* peer_wrapper = nullptr; - std::shared_ptr acceptor_ptr; + win_local_stream_acceptor& acceptor; SOCKET listen_socket = INVALID_SOCKET; io_object::implementation** impl_out = nullptr; // 2 * (sizeof(un_sa_t) + 16) = 2 * (110 + 16) = 252 @@ -56,13 +56,13 @@ struct local_stream_accept_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - local_stream_accept_op() noexcept; + explicit local_stream_accept_op(win_local_stream_acceptor& acceptor_) noexcept; }; /** Readiness-wait operation state for a local stream acceptor. */ struct local_stream_acceptor_wait_op : overlapped_op { - std::shared_ptr acceptor_ptr; + win_local_stream_acceptor& acceptor; SOCKET listen_socket = INVALID_SOCKET; static void do_complete( @@ -72,66 +72,48 @@ struct local_stream_acceptor_wait_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - local_stream_acceptor_wait_op() noexcept; + explicit local_stream_acceptor_wait_op( + win_local_stream_acceptor& acceptor_) noexcept; }; -/* Internal acceptor state for IOCP local stream I/O. */ -class win_local_stream_acceptor_internal - : public intrusive_list::node - , public std::enable_shared_from_this +/** Acceptor implementation for IOCP local stream I/O. + + Collapses the historical internal-state/wrapper split into one + pooled `io_object::implementation`. +*/ +class win_local_stream_acceptor final + : public local_stream_acceptor::implementation + , public intrusive_list::node { friend class win_local_stream_service; - friend class win_local_stream_acceptor; - -public: - explicit win_local_stream_acceptor_internal( - win_local_stream_service& svc) noexcept; - ~win_local_stream_acceptor_internal(); - - win_local_stream_service& socket_service() noexcept; - - std::coroutine_handle<> accept( - std::coroutine_handle<>, - capy::executor_ref, - std::stop_token, - std::error_code*, - io_object::implementation**); - - std::coroutine_handle<> wait( - std::coroutine_handle<>, - capy::executor_ref, - wait_type, - std::stop_token, - std::error_code*); - - SOCKET native_handle() const noexcept; - corosio::local_endpoint local_endpoint() const noexcept; - bool is_open() const noexcept; - void cancel() noexcept; - void close_socket() noexcept; - void set_local_endpoint(corosio::local_endpoint ep) noexcept; + friend struct local_stream_accept_op; + friend struct local_stream_acceptor_wait_op; + win_local_stream_service& svc_; local_stream_accept_op acc_; local_stream_acceptor_wait_op wt_; - -private: - win_local_stream_service& svc_; SOCKET socket_ = INVALID_SOCKET; corosio::local_endpoint local_endpoint_; -}; - -/* Acceptor implementation wrapper for IOCP local stream I/O. */ -class win_local_stream_acceptor final - : public local_stream_acceptor::implementation - , public intrusive_list::node -{ - std::shared_ptr internal_; public: - explicit win_local_stream_acceptor( - std::shared_ptr internal) noexcept; + explicit win_local_stream_acceptor(win_local_stream_service& svc) noexcept; - void close_internal() noexcept; + /// Recycle into the owning service's pool. Defined out-of-line + /// after `win_local_stream_service` is complete (needs + /// `svc_.acceptor_pool_`). + void retire() noexcept override; + + /** Reset recycled state for reuse. + + Each op's `OVERLAPPED` fields are re-seeded by + `overlapped_op::reset()` at its next submit, so they are not + zeroed here. + + @pre refs_ == 0, socket closed, no op in flight. + */ + void reuse() noexcept; + + win_local_stream_service& socket_service() noexcept; std::coroutine_handle<> accept( std::coroutine_handle<> h, @@ -169,7 +151,8 @@ class win_local_stream_acceptor final get_option(int level, int optname, void* data, std::size_t* size) const noexcept override; - win_local_stream_acceptor_internal* get_internal() const noexcept; + void set_local_endpoint(corosio::local_endpoint ep) noexcept; + void close_socket() noexcept; }; } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor_service.hpp b/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor_service.hpp index 39aafb6d3..d8cc3e723 100644 --- a/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor_service.hpp @@ -43,8 +43,7 @@ class BOOST_COROSIO_DECL win_local_stream_acceptor_service final : public local_stream_acceptor_service { public: - explicit win_local_stream_acceptor_service( - capy::execution_context& ctx); + explicit win_local_stream_acceptor_service(capy::execution_context& ctx); io_object::implementation* construct() override; @@ -79,8 +78,10 @@ class BOOST_COROSIO_DECL win_local_stream_acceptor_service final // local_stream_accept_op // ============================================================ -inline local_stream_accept_op::local_stream_accept_op() noexcept +inline local_stream_accept_op::local_stream_accept_op( + win_local_stream_acceptor& acceptor_) noexcept : overlapped_op(&do_complete) + , acceptor(acceptor_) { cancel_func_ = &do_cancel_impl; } @@ -95,8 +96,10 @@ local_stream_accept_op::do_cancel_impl(overlapped_op* base) noexcept } } -inline local_stream_acceptor_wait_op::local_stream_acceptor_wait_op() noexcept +inline local_stream_acceptor_wait_op::local_stream_acceptor_wait_op( + win_local_stream_acceptor& acceptor_) noexcept : overlapped_op(&do_complete) + , acceptor(acceptor_) { cancel_func_ = &do_cancel_impl; } @@ -110,10 +113,7 @@ local_stream_acceptor_wait_op::do_cancel_impl(overlapped_op* base) noexcept { ::CancelIoEx(reinterpret_cast(op->listen_socket), op); } - if (op->acceptor_ptr) - { - op->acceptor_ptr->socket_service().scheduler().cancel_wait(op); - } + op->acceptor.socket_service().scheduler().cancel_wait(op); } // accept_op completion handler @@ -137,12 +137,12 @@ local_stream_accept_op::do_complete( if (op->peer_wrapper) { - op->peer_wrapper->close_internal(); + op->peer_wrapper->close_socket(); op->peer_wrapper = nullptr; } op->cleanup_only(); - op->acceptor_ptr.reset(); + op->object_ref_.reset(); return; } @@ -167,7 +167,7 @@ local_stream_accept_op::do_complete( op->accepted_socket, SOL_SOCKET, SO_UPDATE_ACCEPT_CONTEXT, reinterpret_cast(&op->listen_socket), sizeof(SOCKET)); - op->peer_wrapper->get_internal()->set_socket(op->accepted_socket); + op->peer_wrapper->set_socket(op->accepted_socket); sockaddr_storage local_storage{}; int local_len = sizeof(local_storage); @@ -186,11 +186,14 @@ local_stream_accept_op::do_complete( remote_ep = from_sockaddr_local( remote_storage, static_cast(remote_len)); - op->peer_wrapper->get_internal()->set_endpoints(local_ep, remote_ep); + op->peer_wrapper->set_endpoints(local_ep, remote_ep); op->accepted_socket = INVALID_SOCKET; if (op->impl_out) *op->impl_out = op->peer_wrapper; + // Handed off to the caller: don't let a failed early return on + // the next accept() see this (now user-owned) pointer again. + op->peer_wrapper = nullptr; } else { @@ -202,7 +205,7 @@ local_stream_accept_op::do_complete( if (op->peer_wrapper) { - op->acceptor_ptr->socket_service().destroy(op->peer_wrapper); + op->acceptor.socket_service().destroy(op->peer_wrapper); op->peer_wrapper = nullptr; } @@ -212,7 +215,7 @@ local_stream_accept_op::do_complete( op->cont.h = op->h; auto saved_ex = op->ex; - auto prevent_premature_destruction = std::move(op->acceptor_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); dispatch_coro(saved_ex, op->cont).resume(); } @@ -229,62 +232,70 @@ local_stream_acceptor_wait_op::do_complete( if (!owner) { op->cleanup_only(); - op->acceptor_ptr.reset(); + op->object_ref_.reset(); return; } - auto prevent_premature_destruction = std::move(op->acceptor_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } // ============================================================ -// win_local_stream_acceptor_internal +// win_local_stream_acceptor // ============================================================ -inline win_local_stream_acceptor_internal::win_local_stream_acceptor_internal( +inline win_local_stream_acceptor::win_local_stream_acceptor( win_local_stream_service& svc) noexcept : svc_(svc) + , acc_(*this) + , wt_(*this) { } -inline win_local_stream_acceptor_internal::~win_local_stream_acceptor_internal() +inline void +win_local_stream_acceptor::reuse() noexcept { - svc_.unregister_acceptor_impl(*this); + BOOST_COROSIO_ASSERT(socket_ == INVALID_SOCKET); + BOOST_COROSIO_ASSERT(local_endpoint_ == corosio::local_endpoint{}); + BOOST_COROSIO_ASSERT(!acc_.stop_cb); + BOOST_COROSIO_ASSERT(!wt_.stop_cb); + BOOST_COROSIO_ASSERT(acc_.peer_wrapper == nullptr); + BOOST_COROSIO_ASSERT(acc_.accepted_socket == INVALID_SOCKET); } inline win_local_stream_service& -win_local_stream_acceptor_internal::socket_service() noexcept +win_local_stream_acceptor::socket_service() noexcept { return svc_; } -inline SOCKET -win_local_stream_acceptor_internal::native_handle() const noexcept +inline native_handle_type +win_local_stream_acceptor::native_handle() const noexcept { - return socket_; + return static_cast(socket_); } inline corosio::local_endpoint -win_local_stream_acceptor_internal::local_endpoint() const noexcept +win_local_stream_acceptor::local_endpoint() const noexcept { return local_endpoint_; } inline bool -win_local_stream_acceptor_internal::is_open() const noexcept +win_local_stream_acceptor::is_open() const noexcept { return socket_ != INVALID_SOCKET; } inline void -win_local_stream_acceptor_internal::set_local_endpoint( +win_local_stream_acceptor::set_local_endpoint( corosio::local_endpoint ep) noexcept { local_endpoint_ = ep; } inline void -win_local_stream_acceptor_internal::cancel() noexcept +win_local_stream_acceptor::cancel() noexcept { if (socket_ != INVALID_SOCKET) ::CancelIoEx(reinterpret_cast(socket_), nullptr); @@ -294,14 +305,14 @@ win_local_stream_acceptor_internal::cancel() noexcept } inline std::coroutine_handle<> -win_local_stream_acceptor_internal::wait( +win_local_stream_acceptor::wait( std::coroutine_handle<> h, capy::executor_ref d, wait_type w, std::stop_token token, std::error_code* ec) { - wt_.acceptor_ptr = shared_from_this(); + wt_.object_ref_ = detail::object_ref(this); wt_.listen_socket = socket_; auto& op = wt_; @@ -337,7 +348,7 @@ win_local_stream_acceptor_internal::wait( } inline void -win_local_stream_acceptor_internal::close_socket() noexcept +win_local_stream_acceptor::close_socket() noexcept { // Flag the accept op cancelled before closing so a closesocket-delivered // ERROR_NETNAME_DELETED is short-circuited to canceled rather than mapped @@ -356,26 +367,33 @@ win_local_stream_acceptor_internal::close_socket() noexcept } inline std::coroutine_handle<> -win_local_stream_acceptor_internal::accept( +win_local_stream_acceptor::accept( std::coroutine_handle<> h, capy::executor_ref d, std::stop_token token, std::error_code* ec, io_object::implementation** impl_out) { - acc_.acceptor_ptr = shared_from_this(); + acc_.object_ref_ = detail::object_ref(this); auto& op = acc_; op.reset(); - op.h = h; - op.ex = d; - op.ec_out = ec; - op.impl_out = impl_out; + // reset() doesn't touch these -- a value left over from a prior + // accept() (e.g. the socket already handed to the user via + // *impl_out) must not survive into a completion this call fails + // before reassigning; do_complete's failure branch would destroy() + // whatever stale pointer it finds here. + op.peer_wrapper = nullptr; + op.accepted_socket = INVALID_SOCKET; + op.h = h; + op.ex = d; + op.ec_out = ec; + op.impl_out = impl_out; op.start(token); svc_.work_started(); - // Create wrapper for the peer socket + // Create a (possibly recycled) peer socket auto* peer_ptr = svc_.construct(); if (!peer_ptr) { @@ -453,87 +471,16 @@ win_local_stream_acceptor_internal::accept( return std::noop_coroutine(); } -// ============================================================ -// win_local_stream_acceptor (wrapper) -// ============================================================ - -inline win_local_stream_acceptor::win_local_stream_acceptor( - std::shared_ptr internal) noexcept - : internal_(std::move(internal)) -{ -} - -inline void -win_local_stream_acceptor::close_internal() noexcept -{ - if (internal_) - { - internal_->close_socket(); - internal_.reset(); - } -} - -inline std::coroutine_handle<> -win_local_stream_acceptor::accept( - std::coroutine_handle<> h, - capy::executor_ref d, - std::stop_token token, - std::error_code* ec, - io_object::implementation** impl_out) -{ - return internal_->accept(h, d, token, ec, impl_out); -} - -inline std::coroutine_handle<> -win_local_stream_acceptor::wait( - std::coroutine_handle<> h, - capy::executor_ref d, - wait_type w, - std::stop_token token, - std::error_code* ec) -{ - return internal_->wait(h, d, w, token, ec); -} - -inline corosio::local_endpoint -win_local_stream_acceptor::local_endpoint() const noexcept -{ - return internal_->local_endpoint(); -} - -inline bool -win_local_stream_acceptor::is_open() const noexcept -{ - return internal_ && internal_->is_open(); -} - -inline void -win_local_stream_acceptor::cancel() noexcept -{ - if (internal_) - internal_->cancel(); -} - -inline native_handle_type -win_local_stream_acceptor::native_handle() const noexcept -{ - if (!internal_) - return static_cast(INVALID_SOCKET); - return static_cast(internal_->native_handle()); -} - inline native_handle_type win_local_stream_acceptor::release_socket() noexcept { - if (!internal_) - return static_cast(INVALID_SOCKET); - SOCKET s = internal_->socket_; + SOCKET s = socket_; if (s != INVALID_SOCKET) { - internal_->cancel(); + cancel(); dissociate_from_iocp(s); - internal_->socket_ = INVALID_SOCKET; - internal_->local_endpoint_ = corosio::local_endpoint{}; + socket_ = INVALID_SOCKET; + local_endpoint_ = corosio::local_endpoint{}; } return static_cast(s); } @@ -542,11 +489,11 @@ inline std::error_code win_local_stream_acceptor::set_option( int level, int optname, void const* data, std::size_t size) noexcept { - if (!internal_ || !internal_->is_open()) + if (socket_ == INVALID_SOCKET) return make_err(WSAENOTSOCK); if (::setsockopt( - internal_->native_handle(), level, optname, - reinterpret_cast(data), static_cast(size)) != 0) + socket_, level, optname, reinterpret_cast(data), + static_cast(size)) != 0) return make_err(WSAGetLastError()); return {}; } @@ -555,21 +502,20 @@ inline std::error_code win_local_stream_acceptor::get_option( int level, int optname, void* data, std::size_t* size) const noexcept { - if (!internal_ || !internal_->is_open()) + if (socket_ == INVALID_SOCKET) return make_err(WSAENOTSOCK); int len = static_cast(*size); if (::getsockopt( - internal_->native_handle(), level, optname, - reinterpret_cast(data), &len) != 0) + socket_, level, optname, reinterpret_cast(data), &len) != 0) return make_err(WSAGetLastError()); *size = static_cast(len); return {}; } -inline win_local_stream_acceptor_internal* -win_local_stream_acceptor::get_internal() const noexcept +inline void +win_local_stream_acceptor::retire() noexcept { - return internal_.get(); + svc_.acceptor_pool_.recycle(this); } // ============================================================ @@ -585,38 +531,23 @@ inline win_local_stream_acceptor_service::win_local_stream_acceptor_service( inline io_object::implementation* win_local_stream_acceptor_service::construct() { - auto internal = std::make_shared(svc_); - - // Allocate wrapper before mutating lists so a throw from - // new doesn't leave a dangling pointer in acceptor_list_. - auto* raw = internal.get(); - auto* wrapper = new win_local_stream_acceptor(std::move(internal)); - - { - std::lock_guard lock(svc_.mutex_); - svc_.acceptor_list_.push_back(raw); - svc_.acceptor_wrapper_list_.push_back(wrapper); - } - - return wrapper; + return svc_.acquire_acceptor_impl(); } inline void win_local_stream_acceptor_service::destroy(io_object::implementation* p) { - if (p) - { - auto& wrapper = static_cast(*p); - wrapper.close_internal(); - svc_.destroy_acceptor_impl(wrapper); - } + if (!p) + return; + auto* a = static_cast(p); + a->close_socket(); + release(a); } inline void win_local_stream_acceptor_service::close(io_object::handle& h) { - auto& wrapper = static_cast(*h.get()); - wrapper.get_internal()->close_socket(); + static_cast(h.get())->close_socket(); } inline std::error_code @@ -626,36 +557,32 @@ win_local_stream_acceptor_service::open_acceptor_socket( int type, int protocol) { - auto* internal = - static_cast(impl).get_internal(); - return svc_.open_acceptor_socket(*internal, family, type, protocol); + auto& acc = static_cast(impl); + return svc_.open_acceptor_socket(acc, family, type, protocol); } inline std::error_code win_local_stream_acceptor_service::assign_socket( local_stream_acceptor::implementation& impl, native_handle_type fd) { - auto* internal = - static_cast(impl).get_internal(); - return svc_.assign_acceptor_socket(*internal, fd); + auto& acc = static_cast(impl); + return svc_.assign_acceptor_socket(acc, fd); } inline std::error_code win_local_stream_acceptor_service::bind_acceptor( local_stream_acceptor::implementation& impl, corosio::local_endpoint ep) { - auto* internal = - static_cast(impl).get_internal(); - return svc_.bind_acceptor(*internal, ep); + auto& acc = static_cast(impl); + return svc_.bind_acceptor(acc, ep); } inline std::error_code win_local_stream_acceptor_service::listen_acceptor( local_stream_acceptor::implementation& impl, int backlog) { - auto* internal = - static_cast(impl).get_internal(); - return svc_.listen_acceptor(*internal, backlog); + auto& acc = static_cast(impl); + return svc_.listen_acceptor(acc, backlog); } inline void diff --git a/include/boost/corosio/native/detail/iocp/win_local_stream_service.hpp b/include/boost/corosio/native/detail/iocp/win_local_stream_service.hpp index 1125931d4..e6cac7f25 100644 --- a/include/boost/corosio/native/detail/iocp/win_local_stream_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_local_stream_service.hpp @@ -15,6 +15,7 @@ #if BOOST_COROSIO_HAS_IOCP #include +#include #include #include @@ -23,7 +24,6 @@ #include #include #include -#include #include #include @@ -34,10 +34,6 @@ namespace boost::corosio::detail { -class win_local_stream_acceptor; -class win_local_stream_acceptor_internal; -class win_local_stream_acceptor_service; - /* IOCP local stream socket service. Inherits from local_stream_service to enable runtime polymorphism @@ -48,6 +44,9 @@ class BOOST_COROSIO_DECL win_local_stream_service final : private win_wsa_init , public local_stream_service { + friend class win_local_stream_socket; + friend class win_local_stream_acceptor; + public: io_object::implementation* construct() override; @@ -57,8 +56,6 @@ class BOOST_COROSIO_DECL win_local_stream_service final explicit win_local_stream_service(capy::execution_context& ctx); - ~win_local_stream_service(); - win_local_stream_service(win_local_stream_service const&) = delete; win_local_stream_service& operator=(win_local_stream_service const&) = delete; @@ -75,34 +72,22 @@ class BOOST_COROSIO_DECL win_local_stream_service final local_stream_socket::implementation& impl, native_handle_type fd) override; - void destroy_impl(win_local_stream_socket& impl); - - void unregister_impl(win_local_stream_socket_internal& impl); - std::error_code open_socket_internal( - win_local_stream_socket_internal& impl, - int family, - int type, - int protocol); + win_local_stream_socket& impl, int family, int type, int protocol); - void destroy_acceptor_impl(win_local_stream_acceptor& impl); - - void unregister_acceptor_impl(win_local_stream_acceptor_internal& impl); + win_local_stream_acceptor* acquire_acceptor_impl(); std::error_code open_acceptor_socket( - win_local_stream_acceptor_internal& impl, - int family, - int type, - int protocol); + win_local_stream_acceptor& impl, int family, int type, int protocol); std::error_code assign_acceptor_socket( - win_local_stream_acceptor_internal& impl, native_handle_type fd); + win_local_stream_acceptor& impl, native_handle_type fd); - std::error_code bind_acceptor( - win_local_stream_acceptor_internal& impl, corosio::local_endpoint ep); + std::error_code + bind_acceptor(win_local_stream_acceptor& impl, corosio::local_endpoint ep); std::error_code - listen_acceptor(win_local_stream_acceptor_internal& impl, int backlog); + listen_acceptor(win_local_stream_acceptor& impl, int backlog); void* native_handle() const noexcept; LPFN_CONNECTEX connect_ex() const noexcept; @@ -121,19 +106,14 @@ class BOOST_COROSIO_DECL win_local_stream_service final } private: - friend class win_local_stream_acceptor_service; - win_tcp_service& tcp_svc_; win_scheduler& sched_; + void* iocp_; BOOST_COROSIO_MSVC_WARNING_PUSH BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // detail:: members, dll-interface - win_mutex mutex_; - intrusive_list socket_list_; - intrusive_list acceptor_list_; - intrusive_list socket_wrapper_list_; - intrusive_list acceptor_wrapper_list_; + object_pool pool_; + object_pool acceptor_pool_; BOOST_COROSIO_MSVC_WARNING_POP - void* iocp_; }; // ============================================================ @@ -141,7 +121,7 @@ class BOOST_COROSIO_DECL win_local_stream_service final // ============================================================ inline local_stream_connect_op::local_stream_connect_op( - win_local_stream_socket_internal& internal_) noexcept + win_local_stream_socket& internal_) noexcept : overlapped_op(&do_complete) , internal(internal_) { @@ -149,7 +129,7 @@ inline local_stream_connect_op::local_stream_connect_op( } inline local_stream_read_op::local_stream_read_op( - win_local_stream_socket_internal& internal_) noexcept + win_local_stream_socket& internal_) noexcept : overlapped_op(&do_complete) , internal(internal_) { @@ -157,7 +137,7 @@ inline local_stream_read_op::local_stream_read_op( } inline local_stream_write_op::local_stream_write_op( - win_local_stream_socket_internal& internal_) noexcept + win_local_stream_socket& internal_) noexcept : overlapped_op(&do_complete) , internal(internal_) { @@ -165,7 +145,7 @@ inline local_stream_write_op::local_stream_write_op( } inline local_stream_wait_op::local_stream_wait_op( - win_local_stream_socket_internal& internal_) noexcept + win_local_stream_socket& internal_) noexcept : overlapped_op(&do_complete) , internal(internal_) { @@ -181,10 +161,9 @@ local_stream_connect_op::do_cancel_impl(overlapped_op* base) noexcept { auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) + if (op->internal.socket_ != INVALID_SOCKET) { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); } } @@ -193,10 +172,9 @@ local_stream_read_op::do_cancel_impl(overlapped_op* base) noexcept { auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) + if (op->internal.socket_ != INVALID_SOCKET) { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); } } @@ -205,10 +183,9 @@ local_stream_write_op::do_cancel_impl(overlapped_op* base) noexcept { auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) + if (op->internal.socket_ != INVALID_SOCKET) { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); } } @@ -217,10 +194,9 @@ local_stream_wait_op::do_cancel_impl(overlapped_op* base) noexcept { auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) + if (op->internal.socket_ != INVALID_SOCKET) { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); } op->internal.svc_.scheduler().cancel_wait(op); } @@ -241,31 +217,31 @@ local_stream_connect_op::do_complete( if (!owner) { op->cleanup_only(); - op->internal_ptr.reset(); + op->object_ref_.reset(); return; } bool success = (op->dwError == 0 && !op->cancelled.load(std::memory_order_acquire)); - if (success && op->internal.is_open()) + if (success && op->internal.socket_ != INVALID_SOCKET) { // Required after ConnectEx ::setsockopt( - op->internal.native_handle(), SOL_SOCKET, SO_UPDATE_CONNECT_CONTEXT, + op->internal.socket_, SOL_SOCKET, SO_UPDATE_CONNECT_CONTEXT, nullptr, 0); corosio::local_endpoint local_ep; sockaddr_storage local_storage{}; int local_len = sizeof(local_storage); if (::getsockname( - op->internal.native_handle(), + op->internal.socket_, reinterpret_cast(&local_storage), &local_len) == 0) local_ep = from_sockaddr_local( local_storage, static_cast(local_len)); op->internal.set_endpoints(local_ep, op->target_endpoint); } - auto prevent_premature_destruction = std::move(op->internal_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } @@ -285,11 +261,11 @@ local_stream_read_op::do_complete( if (!owner) { op->cleanup_only(); - op->internal_ptr.reset(); + op->object_ref_.reset(); return; } - auto prevent_premature_destruction = std::move(op->internal_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } @@ -309,11 +285,11 @@ local_stream_write_op::do_complete( if (!owner) { op->cleanup_only(); - op->internal_ptr.reset(); + op->object_ref_.reset(); return; } - auto prevent_premature_destruction = std::move(op->internal_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } @@ -333,19 +309,19 @@ local_stream_wait_op::do_complete( if (!owner) { op->cleanup_only(); - op->internal_ptr.reset(); + op->object_ref_.reset(); return; } - auto prevent_premature_destruction = std::move(op->internal_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } // ============================================================ -// win_local_stream_socket_internal +// win_local_stream_socket // ============================================================ -inline win_local_stream_socket_internal::win_local_stream_socket_internal( +inline win_local_stream_socket::win_local_stream_socket( win_local_stream_service& svc) noexcept : svc_(svc) , conn_(*this) @@ -355,43 +331,26 @@ inline win_local_stream_socket_internal::win_local_stream_socket_internal( { } -inline win_local_stream_socket_internal::~win_local_stream_socket_internal() -{ - svc_.unregister_impl(*this); -} - -inline SOCKET -win_local_stream_socket_internal::native_handle() const noexcept -{ - return socket_; -} - -inline corosio::local_endpoint -win_local_stream_socket_internal::local_endpoint() const noexcept -{ - return local_endpoint_; -} - -inline corosio::local_endpoint -win_local_stream_socket_internal::remote_endpoint() const noexcept -{ - return remote_endpoint_; -} - -inline bool -win_local_stream_socket_internal::is_open() const noexcept +inline void +win_local_stream_socket::reuse() noexcept { - return socket_ != INVALID_SOCKET; + BOOST_COROSIO_ASSERT(socket_ == INVALID_SOCKET); + BOOST_COROSIO_ASSERT(local_endpoint_ == corosio::local_endpoint{}); + BOOST_COROSIO_ASSERT(remote_endpoint_ == corosio::local_endpoint{}); + BOOST_COROSIO_ASSERT(!conn_.stop_cb); + BOOST_COROSIO_ASSERT(!rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wr_.stop_cb); + BOOST_COROSIO_ASSERT(!wt_.stop_cb); } inline void -win_local_stream_socket_internal::set_socket(SOCKET s) noexcept +win_local_stream_socket::set_socket(SOCKET s) noexcept { socket_ = s; } inline void -win_local_stream_socket_internal::set_endpoints( +win_local_stream_socket::set_endpoints( corosio::local_endpoint local, corosio::local_endpoint remote) noexcept { local_endpoint_ = local; @@ -399,14 +358,14 @@ win_local_stream_socket_internal::set_endpoints( } inline std::coroutine_handle<> -win_local_stream_socket_internal::connect( +win_local_stream_socket::connect( std::coroutine_handle<> h, capy::executor_ref d, corosio::local_endpoint ep, std::stop_token token, std::error_code* ec) { - conn_.internal_ptr = shared_from_this(); + conn_.object_ref_ = detail::object_ref(this); auto& op = conn_; op.reset(); @@ -469,7 +428,7 @@ win_local_stream_socket_internal::connect( } inline std::coroutine_handle<> -win_local_stream_socket_internal::read_some( +win_local_stream_socket::read_some( std::coroutine_handle<> h, capy::executor_ref d, buffer_param param, @@ -477,7 +436,7 @@ win_local_stream_socket_internal::read_some( std::error_code* ec, std::size_t* bytes_out) { - rd_.internal_ptr = shared_from_this(); + rd_.object_ref_ = detail::object_ref(this); auto& op = rd_; op.reset(); @@ -540,7 +499,7 @@ win_local_stream_socket_internal::read_some( } inline std::coroutine_handle<> -win_local_stream_socket_internal::write_some( +win_local_stream_socket::write_some( std::coroutine_handle<> h, capy::executor_ref d, buffer_param param, @@ -548,7 +507,7 @@ win_local_stream_socket_internal::write_some( std::error_code* ec, std::size_t* bytes_out) { - wr_.internal_ptr = shared_from_this(); + wr_.object_ref_ = detail::object_ref(this); auto& op = wr_; op.reset(); @@ -607,14 +566,14 @@ win_local_stream_socket_internal::write_some( } inline std::coroutine_handle<> -win_local_stream_socket_internal::wait( +win_local_stream_socket::wait( std::coroutine_handle<> h, capy::executor_ref d, wait_type w, std::stop_token token, std::error_code* ec) { - wt_.internal_ptr = shared_from_this(); + wt_.object_ref_ = detail::object_ref(this); auto& op = wt_; op.reset(); @@ -672,7 +631,7 @@ win_local_stream_socket_internal::wait( } inline void -win_local_stream_socket_internal::cancel() noexcept +win_local_stream_socket::cancel() noexcept { if (socket_ != INVALID_SOCKET) { @@ -687,7 +646,7 @@ win_local_stream_socket_internal::cancel() noexcept } inline void -win_local_stream_socket_internal::close_socket() noexcept +win_local_stream_socket::close_socket() noexcept { // Flag every op cancelled before closing so a closesocket-delivered // ERROR_NETNAME_DELETED is short-circuited to canceled rather than mapped @@ -709,72 +668,6 @@ win_local_stream_socket_internal::close_socket() noexcept remote_endpoint_ = corosio::local_endpoint{}; } -// ============================================================ -// win_local_stream_socket (wrapper) -// ============================================================ - -inline win_local_stream_socket::win_local_stream_socket( - std::shared_ptr internal) noexcept - : internal_(std::move(internal)) -{ -} - -inline void -win_local_stream_socket::close_internal() noexcept -{ - if (internal_) - { - internal_->close_socket(); - internal_.reset(); - } -} - -inline std::coroutine_handle<> -win_local_stream_socket::connect( - std::coroutine_handle<> h, - capy::executor_ref d, - corosio::local_endpoint ep, - std::stop_token token, - std::error_code* ec) -{ - return internal_->connect(h, d, ep, token, ec); -} - -inline std::coroutine_handle<> -win_local_stream_socket::read_some( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->read_some(h, d, buf, token, ec, bytes); -} - -inline std::coroutine_handle<> -win_local_stream_socket::write_some( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->write_some(h, d, buf, token, ec, bytes); -} - -inline std::coroutine_handle<> -win_local_stream_socket::wait( - std::coroutine_handle<> h, - capy::executor_ref d, - wait_type w, - std::stop_token token, - std::error_code* ec) -{ - return internal_->wait(h, d, w, token, ec); -} - inline std::error_code win_local_stream_socket::shutdown( local_stream_socket::shutdown_type what) noexcept @@ -794,7 +687,7 @@ win_local_stream_socket::shutdown( default: return make_err(WSAEINVAL); } - if (::shutdown(internal_->native_handle(), how) != 0) + if (::shutdown(socket_, how) != 0) return make_err(WSAGetLastError()); return {}; } @@ -802,23 +695,23 @@ win_local_stream_socket::shutdown( inline native_handle_type win_local_stream_socket::native_handle() const noexcept { - return static_cast(internal_->native_handle()); + return static_cast(socket_); } inline native_handle_type win_local_stream_socket::release_socket() noexcept { - SOCKET s = internal_->socket_; + SOCKET s = socket_; if (s != INVALID_SOCKET) { - internal_->cancel(); + cancel(); // Sever the port association so the descriptor can be // adopted again; best-effort, the caller keeps a working // socket either way. dissociate_from_iocp(s); - internal_->socket_ = INVALID_SOCKET; - internal_->local_endpoint_ = corosio::local_endpoint{}; - internal_->remote_endpoint_ = corosio::local_endpoint{}; + socket_ = INVALID_SOCKET; + local_endpoint_ = corosio::local_endpoint{}; + remote_endpoint_ = corosio::local_endpoint{}; } return static_cast(s); } @@ -828,8 +721,8 @@ win_local_stream_socket::set_option( int level, int optname, void const* data, std::size_t size) noexcept { if (::setsockopt( - internal_->native_handle(), level, optname, - reinterpret_cast(data), static_cast(size)) != 0) + socket_, level, optname, reinterpret_cast(data), + static_cast(size)) != 0) return make_err(WSAGetLastError()); return {}; } @@ -840,8 +733,7 @@ win_local_stream_socket::get_option( { int len = static_cast(*size); if (::getsockopt( - internal_->native_handle(), level, optname, - reinterpret_cast(data), &len) != 0) + socket_, level, optname, reinterpret_cast(data), &len) != 0) return make_err(WSAGetLastError()); *size = static_cast(len); return {}; @@ -850,25 +742,19 @@ win_local_stream_socket::get_option( inline corosio::local_endpoint win_local_stream_socket::local_endpoint() const noexcept { - return internal_->local_endpoint(); + return local_endpoint_; } inline corosio::local_endpoint win_local_stream_socket::remote_endpoint() const noexcept { - return internal_->remote_endpoint(); + return remote_endpoint_; } inline void -win_local_stream_socket::cancel() noexcept -{ - internal_->cancel(); -} - -inline win_local_stream_socket_internal* -win_local_stream_socket::get_internal() const noexcept +win_local_stream_socket::retire() noexcept { - return internal_.get(); + svc_.pool_.recycle(this); } // ============================================================ @@ -883,89 +769,36 @@ inline win_local_stream_service::win_local_stream_service( { } -inline win_local_stream_service::~win_local_stream_service() -{ - for (auto* w = socket_wrapper_list_.pop_front(); w != nullptr; - w = socket_wrapper_list_.pop_front()) - delete w; - - for (auto* w = acceptor_wrapper_list_.pop_front(); w != nullptr; - w = acceptor_wrapper_list_.pop_front()) - delete w; -} - inline void win_local_stream_service::shutdown() { - std::lock_guard lock(mutex_); + pool_.shutdown( + [](win_local_stream_socket* impl) { impl->close_socket(); }); - for (auto* impl = socket_list_.pop_front(); impl != nullptr; - impl = socket_list_.pop_front()) - { - impl->close_socket(); - } - - for (auto* impl = acceptor_list_.pop_front(); impl != nullptr; - impl = acceptor_list_.pop_front()) - { - impl->close_socket(); - } + acceptor_pool_.shutdown( + [](win_local_stream_acceptor* impl) { impl->close_socket(); }); } inline io_object::implementation* win_local_stream_service::construct() { - auto internal = std::make_shared(*this); - - { - std::lock_guard lock(mutex_); - socket_list_.push_back(internal.get()); - } - - auto* wrapper = new win_local_stream_socket(std::move(internal)); - - { - std::lock_guard lock(mutex_); - socket_wrapper_list_.push_back(wrapper); - } - - return wrapper; + return pool_.acquire(*this); } inline void win_local_stream_service::destroy(io_object::implementation* p) { - if (p) - { - auto& wrapper = static_cast(*p); - wrapper.close_internal(); - destroy_impl(wrapper); - } + if (!p) + return; + auto* s = static_cast(p); + s->close_socket(); + release(s); } inline void win_local_stream_service::close(io_object::handle& h) { - auto& wrapper = static_cast(*h.get()); - wrapper.get_internal()->close_socket(); -} - -inline void -win_local_stream_service::destroy_impl(win_local_stream_socket& impl) -{ - { - std::lock_guard lock(mutex_); - socket_wrapper_list_.remove(&impl); - } - delete &impl; -} - -inline void -win_local_stream_service::unregister_impl( - win_local_stream_socket_internal& impl) -{ - std::lock_guard lock(mutex_); - socket_list_.remove(&impl); + static_cast(h.get())->close_socket(); } inline std::error_code @@ -975,22 +808,20 @@ win_local_stream_service::open_socket( int type, int protocol) { - auto& wrapper = static_cast(impl); - return open_socket_internal( - *wrapper.get_internal(), family, type, protocol); + auto& sock = static_cast(impl); + return open_socket_internal(sock, family, type, protocol); } inline std::error_code win_local_stream_service::assign_socket( local_stream_socket::implementation& impl, native_handle_type fd) { - auto& wrapper = static_cast(impl); - auto& internal = *wrapper.get_internal(); + auto& sock = static_cast(impl); - SOCKET sock = static_cast(fd); - if (sock == INVALID_SOCKET) + SOCKET s = static_cast(fd); + if (s == INVALID_SOCKET) return make_err(WSAENOTSOCK); - if (sock == internal.socket_) + if (s == sock.socket_) return std::make_error_code(std::errc::invalid_argument); // SO_PROTOCOL_INFOW works on an unbound socket, unlike getsockname @@ -999,7 +830,7 @@ win_local_stream_service::assign_socket( WSAPROTOCOL_INFOW proto_info{}; int proto_len = sizeof(proto_info); if (::getsockopt( - sock, SOL_SOCKET, SO_PROTOCOL_INFOW, + s, SOL_SOCKET, SO_PROTOCOL_INFOW, reinterpret_cast(&proto_info), &proto_len) != 0) return make_err(::WSAGetLastError()); if (proto_info.iAddressFamily != AF_UNIX) @@ -1011,32 +842,31 @@ win_local_stream_service::assign_socket( // shares descriptor state the way the reactor path does, so a // failed association must not cost the caller their old socket. HANDLE result = ::CreateIoCompletionPort( - reinterpret_cast(sock), static_cast(iocp_), key_io, 0); + reinterpret_cast(s), static_cast(iocp_), key_io, 0); if (result == nullptr) return make_err(::GetLastError()); - internal.close_socket(); - internal.socket_ = sock; + sock.close_socket(); + sock.socket_ = s; sockaddr_storage local{}; int local_len = sizeof(local); corosio::local_endpoint lep{}, rep{}; - if (::getsockname(sock, reinterpret_cast(&local), &local_len) == - 0) + if (::getsockname(s, reinterpret_cast(&local), &local_len) == 0) lep = from_sockaddr_local(local, static_cast(local_len)); sockaddr_storage remote{}; int remote_len = sizeof(remote); - if (::getpeername( - sock, reinterpret_cast(&remote), &remote_len) == 0) + if (::getpeername(s, reinterpret_cast(&remote), &remote_len) == + 0) rep = from_sockaddr_local(remote, static_cast(remote_len)); - internal.local_endpoint_ = lep; - internal.remote_endpoint_ = rep; + sock.local_endpoint_ = lep; + sock.remote_endpoint_ = rep; return {}; } inline std::error_code win_local_stream_service::open_socket_internal( - win_local_stream_socket_internal& impl, int family, int type, int protocol) + win_local_stream_socket& impl, int family, int type, int protocol) { impl.close_socket(); @@ -1111,30 +941,15 @@ win_local_stream_service::work_finished() noexcept sched_.work_finished(); } -inline void -win_local_stream_service::destroy_acceptor_impl(win_local_stream_acceptor& impl) +inline win_local_stream_acceptor* +win_local_stream_service::acquire_acceptor_impl() { - { - std::lock_guard lock(mutex_); - acceptor_wrapper_list_.remove(&impl); - } - delete &impl; -} - -inline void -win_local_stream_service::unregister_acceptor_impl( - win_local_stream_acceptor_internal& impl) -{ - std::lock_guard lock(mutex_); - acceptor_list_.remove(&impl); + return acceptor_pool_.acquire(*this); } inline std::error_code win_local_stream_service::open_acceptor_socket( - win_local_stream_acceptor_internal& impl, - int family, - int type, - int protocol) + win_local_stream_acceptor& impl, int family, int type, int protocol) { impl.close_socket(); @@ -1160,7 +975,7 @@ win_local_stream_service::open_acceptor_socket( inline std::error_code win_local_stream_service::assign_acceptor_socket( - win_local_stream_acceptor_internal& impl, native_handle_type fd) + win_local_stream_acceptor& impl, native_handle_type fd) { SOCKET sock = static_cast(fd); if (sock == INVALID_SOCKET) @@ -1205,7 +1020,7 @@ win_local_stream_service::assign_acceptor_socket( inline std::error_code win_local_stream_service::bind_acceptor( - win_local_stream_acceptor_internal& impl, corosio::local_endpoint ep) + win_local_stream_acceptor& impl, corosio::local_endpoint ep) { // Reject abstract sockets on Windows if (ep.is_abstract()) @@ -1226,7 +1041,7 @@ win_local_stream_service::bind_acceptor( inline std::error_code win_local_stream_service::listen_acceptor( - win_local_stream_acceptor_internal& impl, int backlog) + win_local_stream_acceptor& impl, int backlog) { SOCKET sock = impl.socket_; diff --git a/include/boost/corosio/native/detail/iocp/win_local_stream_socket.hpp b/include/boost/corosio/native/detail/iocp/win_local_stream_socket.hpp index f52bb983c..a5278195a 100644 --- a/include/boost/corosio/native/detail/iocp/win_local_stream_socket.hpp +++ b/include/boost/corosio/native/detail/iocp/win_local_stream_socket.hpp @@ -15,6 +15,7 @@ #if BOOST_COROSIO_HAS_IOCP #include +#include #include #include #include @@ -23,20 +24,18 @@ #include #include -#include #include namespace boost::corosio::detail { class win_local_stream_service; -class win_local_stream_socket_internal; +class win_local_stream_socket; /** Connect operation state for local stream sockets. */ struct local_stream_connect_op : overlapped_op { - win_local_stream_socket_internal& internal; - std::shared_ptr internal_ptr; + win_local_stream_socket& internal; corosio::local_endpoint target_endpoint; static void do_complete( @@ -47,7 +46,7 @@ struct local_stream_connect_op : overlapped_op static void do_cancel_impl(overlapped_op* op) noexcept; explicit local_stream_connect_op( - win_local_stream_socket_internal& internal_) noexcept; + win_local_stream_socket& internal_) noexcept; }; /** Read operation state for local stream sockets. */ @@ -57,8 +56,7 @@ struct local_stream_read_op : overlapped_op WSABUF wsabufs[max_buffers]; DWORD wsabuf_count = 0; DWORD flags = 0; - win_local_stream_socket_internal& internal; - std::shared_ptr internal_ptr; + win_local_stream_socket& internal; static void do_complete( void* owner, @@ -67,8 +65,7 @@ struct local_stream_read_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit local_stream_read_op( - win_local_stream_socket_internal& internal_) noexcept; + explicit local_stream_read_op(win_local_stream_socket& internal_) noexcept; }; /** Write operation state for local stream sockets. */ @@ -77,8 +74,7 @@ struct local_stream_write_op : overlapped_op static constexpr std::size_t max_buffers = 16; WSABUF wsabufs[max_buffers]; DWORD wsabuf_count = 0; - win_local_stream_socket_internal& internal; - std::shared_ptr internal_ptr; + win_local_stream_socket& internal; static void do_complete( void* owner, @@ -87,8 +83,7 @@ struct local_stream_write_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit local_stream_write_op( - win_local_stream_socket_internal& internal_) noexcept; + explicit local_stream_write_op(win_local_stream_socket& internal_) noexcept; }; /** Readiness-wait operation state for local stream sockets. */ @@ -96,8 +91,7 @@ struct local_stream_wait_op : overlapped_op { WSABUF wsabuf{}; DWORD flags = 0; - win_local_stream_socket_internal& internal; - std::shared_ptr internal_ptr; + win_local_stream_socket& internal; static void do_complete( void* owner, @@ -106,23 +100,23 @@ struct local_stream_wait_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit local_stream_wait_op( - win_local_stream_socket_internal& internal_) noexcept; + explicit local_stream_wait_op(win_local_stream_socket& internal_) noexcept; }; -/* Internal socket state for IOCP local stream I/O. +/** Socket implementation for IOCP local stream I/O. - Holds the native SOCKET handle, cached endpoints, and - pending operation slots. Derives from enable_shared_from_this - so in-flight operations can extend its lifetime past the - wrapper's destruction. + Collapses the historical internal-state/wrapper split into one + pooled `io_object::implementation`: it both owns the native SOCKET + handle and pending operations, and directly implements + `local_stream_socket::implementation`. `win_local_stream_service` + recycles instances through its pool instead of freeing them on + every close. */ -class win_local_stream_socket_internal - : public intrusive_list::node - , public std::enable_shared_from_this +class win_local_stream_socket final + : public local_stream_socket::implementation + , public intrusive_list::node { friend class win_local_stream_service; - friend class win_local_stream_socket; friend struct local_stream_read_op; friend struct local_stream_write_op; friend struct local_stream_connect_op; @@ -134,74 +128,26 @@ class win_local_stream_socket_internal local_stream_write_op wr_; local_stream_wait_op wt_; SOCKET socket_ = INVALID_SOCKET; - -public: - explicit win_local_stream_socket_internal( - win_local_stream_service& svc) noexcept; - ~win_local_stream_socket_internal(); - - std::coroutine_handle<> connect( - std::coroutine_handle<>, - capy::executor_ref, - corosio::local_endpoint, - std::stop_token, - std::error_code*); - - std::coroutine_handle<> read_some( - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - std::stop_token, - std::error_code*, - std::size_t*); - - std::coroutine_handle<> write_some( - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - std::stop_token, - std::error_code*, - std::size_t*); - - std::coroutine_handle<> wait( - std::coroutine_handle<>, - capy::executor_ref, - wait_type, - std::stop_token, - std::error_code*); - - SOCKET native_handle() const noexcept; - corosio::local_endpoint local_endpoint() const noexcept; - corosio::local_endpoint remote_endpoint() const noexcept; - bool is_open() const noexcept; - void cancel() noexcept; - void close_socket() noexcept; - void set_socket(SOCKET s) noexcept; - void set_endpoints( - corosio::local_endpoint local, corosio::local_endpoint remote) noexcept; - -private: corosio::local_endpoint local_endpoint_; corosio::local_endpoint remote_endpoint_; -}; -/* Socket implementation wrapper for IOCP local stream I/O. +public: + explicit win_local_stream_socket(win_local_stream_service& svc) noexcept; - Public-facing implementation holding a shared_ptr to the - internal state. Inherits from local_stream_socket::implementation - to satisfy the virtual interface. -*/ -class win_local_stream_socket final - : public local_stream_socket::implementation - , public intrusive_list::node -{ - std::shared_ptr internal_; + /// Recycle into the owning service's pool. Defined out-of-line + /// after `win_local_stream_service` is complete (needs + /// `svc_.pool_`). + void retire() noexcept override; -public: - explicit win_local_stream_socket( - std::shared_ptr internal) noexcept; + /** Reset recycled state for reuse. - void close_internal() noexcept; + Each op's `OVERLAPPED` fields are re-seeded by + `overlapped_op::reset()` at its next submit, so they are not + zeroed here. + + @pre refs_ == 0, socket closed, no op in flight. + */ + void reuse() noexcept; std::coroutine_handle<> connect( std::coroutine_handle<> h, @@ -259,7 +205,13 @@ class win_local_stream_socket final corosio::local_endpoint remote_endpoint() const noexcept override; void cancel() noexcept override; - win_local_stream_socket_internal* get_internal() const noexcept; + /// Used by the acceptor's `local_stream_accept_op` to finish + /// setting up the peer. + void set_socket(SOCKET s) noexcept; + void set_endpoints( + corosio::local_endpoint local, corosio::local_endpoint remote) noexcept; + + void close_socket() noexcept; }; } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/iocp/win_random_access_file.hpp b/include/boost/corosio/native/detail/iocp/win_random_access_file.hpp index 4e3c1c4d3..5dfb8421c 100644 --- a/include/boost/corosio/native/detail/iocp/win_random_access_file.hpp +++ b/include/boost/corosio/native/detail/iocp/win_random_access_file.hpp @@ -15,6 +15,7 @@ #if BOOST_COROSIO_HAS_IOCP #include +#include #include #include #include @@ -25,12 +26,11 @@ #include #include -#include namespace boost::corosio::detail { class win_random_access_file_service; -class win_random_access_file_internal; +class win_random_access_file; /** Per-operation state for concurrent random-access file IOCP I/O. @@ -42,10 +42,9 @@ struct raf_concurrent_op : overlapped_op , intrusive_list::node { - void* buf = nullptr; - DWORD buf_len = 0; - win_random_access_file_internal* file_ = nullptr; - std::shared_ptr file_ref; + void* buf = nullptr; + DWORD buf_len = 0; + win_random_access_file* file_ = nullptr; static void do_complete( void* owner, @@ -54,20 +53,21 @@ struct raf_concurrent_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit raf_concurrent_op(win_random_access_file_internal& f) noexcept; + explicit raf_concurrent_op(win_random_access_file& f) noexcept; }; -/** Internal random-access file state for IOCP-based I/O. +/** Random-access file implementation for IOCP-based I/O. - Each async operation heap-allocates a raf_concurrent_op, - allowing unlimited concurrent reads and writes. + Collapses the historical internal-state/wrapper split into one + pooled `io_object::implementation`. Each async operation + heap-allocates a `raf_concurrent_op`, allowing unlimited + concurrent reads and writes. */ -class win_random_access_file_internal - : public intrusive_list::node - , public std::enable_shared_from_this +class win_random_access_file final + : public random_access_file::implementation + , public intrusive_list::node { friend class win_random_access_file_service; - friend class win_random_access_file; friend struct raf_concurrent_op; win_random_access_file_service& svc_; @@ -76,53 +76,21 @@ class win_random_access_file_internal HANDLE handle_ = INVALID_HANDLE_VALUE; public: - explicit win_random_access_file_internal( + explicit win_random_access_file( win_random_access_file_service& svc) noexcept; - ~win_random_access_file_internal(); - - std::coroutine_handle<> read_some_at( - std::uint64_t offset, - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - std::stop_token, - std::error_code*, - std::size_t*); - - std::coroutine_handle<> write_some_at( - std::uint64_t offset, - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - std::stop_token, - std::error_code*, - std::size_t*); - - HANDLE native_handle() const noexcept; - bool is_open() const noexcept; - void cancel() noexcept; - void close_handle() noexcept; - std::uint64_t size() const; - std::error_code resize(std::uint64_t new_size) noexcept; - std::error_code sync_data() noexcept; - std::error_code sync_all() noexcept; - native_handle_type release(); - std::error_code assign(native_handle_type handle) noexcept; -}; + /// Recycle into the owning service's pool. Defined out-of-line + /// after `win_random_access_file_service` is complete (needs + /// `svc_.pool_`). + void retire() noexcept override; -/** Random-access file implementation wrapper for IOCP-based I/O. */ -class win_random_access_file final - : public random_access_file::implementation - , public intrusive_list::node -{ - std::shared_ptr internal_; + /** Reset recycled state for reuse. -public: - explicit win_random_access_file( - std::shared_ptr internal) noexcept; - - void close_internal() noexcept; + @pre refs_ == 0, handle closed, no op in flight (per-op + `raf_concurrent_op` holds its own `object_ref`, so `refs_` + cannot reach zero while one is outstanding). + */ + void reuse() noexcept; std::coroutine_handle<> read_some_at( std::uint64_t offset, @@ -143,6 +111,7 @@ class win_random_access_file final std::size_t* bytes) override; native_handle_type native_handle() const noexcept override; + bool is_open() const noexcept; void cancel() noexcept override; std::uint64_t size() const override; std::error_code resize(std::uint64_t new_size) noexcept override; @@ -151,7 +120,7 @@ class win_random_access_file final native_handle_type release() override; std::error_code assign(native_handle_type handle) noexcept override; - win_random_access_file_internal* get_internal() const noexcept; + void close_handle() noexcept; }; } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/iocp/win_random_access_file_service.hpp b/include/boost/corosio/native/detail/iocp/win_random_access_file_service.hpp index e62eddda4..fcba5fefa 100644 --- a/include/boost/corosio/native/detail/iocp/win_random_access_file_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_random_access_file_service.hpp @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include #include #include @@ -44,11 +44,12 @@ namespace boost::corosio::detail { class BOOST_COROSIO_DECL win_random_access_file_service final : public random_access_file_service { + friend class win_random_access_file; + public: using key_type = win_random_access_file_service; explicit win_random_access_file_service(capy::execution_context& ctx); - ~win_random_access_file_service(); win_random_access_file_service(win_random_access_file_service const&) = delete; @@ -65,9 +66,6 @@ class BOOST_COROSIO_DECL win_random_access_file_service final std::filesystem::path const& path, file_base::flags mode) override; - void destroy_impl(win_random_access_file& impl); - void unregister_impl(win_random_access_file_internal& impl); - void post(overlapped_op* op); void on_pending(overlapped_op* op) noexcept; void on_completion(overlapped_op* op, DWORD error, DWORD bytes) noexcept; @@ -104,14 +102,12 @@ class BOOST_COROSIO_DECL win_random_access_file_service final LONG(NTAPI*)(HANDLE, ULONG, void*, ULONG, io_status_block*); win_scheduler& sched_; + void* iocp_; + nt_flush_fn nt_flush_buffers_file_ex_; BOOST_COROSIO_MSVC_WARNING_PUSH BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // detail:: members, dll-interface - win_mutex mutex_; - intrusive_list file_list_; - intrusive_list wrapper_list_; + object_pool pool_; BOOST_COROSIO_MSVC_WARNING_POP - void* iocp_; - nt_flush_fn nt_flush_buffers_file_ex_; }; /** Get or create the random-access file service for the given context. */ @@ -126,7 +122,7 @@ get_random_access_file_service(capy::execution_context& ctx, win_scheduler&) // --------------------------------------------------------------------------- inline raf_concurrent_op::raf_concurrent_op( - win_random_access_file_internal& f) noexcept + win_random_access_file& f) noexcept : overlapped_op(&do_complete) , file_(&f) { @@ -139,7 +135,7 @@ raf_concurrent_op::do_cancel_impl(overlapped_op* base) noexcept auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); if (op->file_->is_open()) - ::CancelIoEx(op->file_->native_handle(), op); + ::CancelIoEx(op->file_->handle_, op); } inline void @@ -160,7 +156,7 @@ raf_concurrent_op::do_complete( std::lock_guard lock(op->file_->ops_mutex_); op->file_->outstanding_ops_.remove(op); } - op->file_ref.reset(); + op->object_ref_.reset(); delete op; return; } @@ -184,42 +180,46 @@ raf_concurrent_op::do_complete( op->file_->outstanding_ops_.remove(op); } - op->file_ref.reset(); - auto coro = op->h; + // Hold the keepalive across the resume, like every other IOCP op: + // it may be the last reference to the implementation the op was + // working on behalf of. + auto prevent_premature_destruction = std::move(op->object_ref_); delete op; coro.resume(); } // --------------------------------------------------------------------------- -// win_random_access_file_internal +// win_random_access_file // --------------------------------------------------------------------------- -inline win_random_access_file_internal::win_random_access_file_internal( +inline win_random_access_file::win_random_access_file( win_random_access_file_service& svc) noexcept : svc_(svc) { } -inline win_random_access_file_internal::~win_random_access_file_internal() +inline void +win_random_access_file::reuse() noexcept { - svc_.unregister_impl(*this); + BOOST_COROSIO_ASSERT(handle_ == INVALID_HANDLE_VALUE); + BOOST_COROSIO_ASSERT(outstanding_ops_.empty()); } -inline HANDLE -win_random_access_file_internal::native_handle() const noexcept +inline native_handle_type +win_random_access_file::native_handle() const noexcept { - return handle_; + return reinterpret_cast(handle_); } inline bool -win_random_access_file_internal::is_open() const noexcept +win_random_access_file::is_open() const noexcept { return handle_ != INVALID_HANDLE_VALUE; } inline void -win_random_access_file_internal::cancel() noexcept +win_random_access_file::cancel() noexcept { if (handle_ != INVALID_HANDLE_VALUE) ::CancelIoEx(handle_, nullptr); @@ -230,7 +230,7 @@ win_random_access_file_internal::cancel() noexcept } inline void -win_random_access_file_internal::close_handle() noexcept +win_random_access_file::close_handle() noexcept { if (handle_ != INVALID_HANDLE_VALUE) { @@ -241,7 +241,7 @@ win_random_access_file_internal::close_handle() noexcept } inline std::uint64_t -win_random_access_file_internal::size() const +win_random_access_file::size() const { LARGE_INTEGER li; if (!::GetFileSizeEx(handle_, &li)) @@ -251,7 +251,7 @@ win_random_access_file_internal::size() const } inline std::error_code -win_random_access_file_internal::resize(std::uint64_t new_size) noexcept +win_random_access_file::resize(std::uint64_t new_size) noexcept { LARGE_INTEGER li; li.QuadPart = static_cast(new_size); @@ -263,7 +263,7 @@ win_random_access_file_internal::resize(std::uint64_t new_size) noexcept } inline std::error_code -win_random_access_file_internal::sync_data() noexcept +win_random_access_file::sync_data() noexcept { // Attempt data-only flush; fall back to full flush if (svc_.try_flush_data(handle_)) @@ -274,7 +274,7 @@ win_random_access_file_internal::sync_data() noexcept } inline std::error_code -win_random_access_file_internal::sync_all() noexcept +win_random_access_file::sync_all() noexcept { if (!::FlushFileBuffers(handle_)) return make_err(::GetLastError()); @@ -282,7 +282,7 @@ win_random_access_file_internal::sync_all() noexcept } inline native_handle_type -win_random_access_file_internal::release() +win_random_access_file::release() { HANDLE h = handle_; handle_ = INVALID_HANDLE_VALUE; @@ -290,7 +290,7 @@ win_random_access_file_internal::release() } inline std::error_code -win_random_access_file_internal::assign(native_handle_type handle) noexcept +win_random_access_file::assign(native_handle_type handle) noexcept { close_handle(); HANDLE h = reinterpret_cast(handle); @@ -305,7 +305,7 @@ win_random_access_file_internal::assign(native_handle_type handle) noexcept } inline std::coroutine_handle<> -win_random_access_file_internal::read_some_at( +win_random_access_file::read_some_at( std::uint64_t offset, std::coroutine_handle<> h, capy::executor_ref ex, @@ -316,8 +316,8 @@ win_random_access_file_internal::read_some_at( { static constexpr std::size_t max_buffers = 16; - auto* op = new raf_concurrent_op(*this); - op->file_ref = shared_from_this(); + auto* op = new raf_concurrent_op(*this); + op->object_ref_ = detail::object_ref(this); op->reset(); op->is_read = true; @@ -387,7 +387,7 @@ win_random_access_file_internal::read_some_at( } inline std::coroutine_handle<> -win_random_access_file_internal::write_some_at( +win_random_access_file::write_some_at( std::uint64_t offset, std::coroutine_handle<> h, capy::executor_ref ex, @@ -398,8 +398,8 @@ win_random_access_file_internal::write_some_at( { static constexpr std::size_t max_buffers = 16; - auto* op = new raf_concurrent_op(*this); - op->file_ref = shared_from_this(); + auto* op = new raf_concurrent_op(*this); + op->object_ref_ = detail::object_ref(this); op->reset(); op->is_read = false; @@ -467,104 +467,10 @@ win_random_access_file_internal::write_some_at( return std::noop_coroutine(); } -// --------------------------------------------------------------------------- -// win_random_access_file wrapper -// --------------------------------------------------------------------------- - -inline win_random_access_file::win_random_access_file( - std::shared_ptr internal) noexcept - : internal_(std::move(internal)) -{ -} - inline void -win_random_access_file::close_internal() noexcept +win_random_access_file::retire() noexcept { - if (internal_) - { - internal_->close_handle(); - internal_.reset(); - } -} - -inline std::coroutine_handle<> -win_random_access_file::read_some_at( - std::uint64_t offset, - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->read_some_at(offset, h, d, buf, token, ec, bytes); -} - -inline std::coroutine_handle<> -win_random_access_file::write_some_at( - std::uint64_t offset, - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->write_some_at(offset, h, d, buf, token, ec, bytes); -} - -inline native_handle_type -win_random_access_file::native_handle() const noexcept -{ - return reinterpret_cast(internal_->native_handle()); -} - -inline void -win_random_access_file::cancel() noexcept -{ - internal_->cancel(); -} - -inline std::uint64_t -win_random_access_file::size() const -{ - return internal_->size(); -} - -inline std::error_code -win_random_access_file::resize(std::uint64_t new_size) noexcept -{ - return internal_->resize(new_size); -} - -inline std::error_code -win_random_access_file::sync_data() noexcept -{ - return internal_->sync_data(); -} - -inline std::error_code -win_random_access_file::sync_all() noexcept -{ - return internal_->sync_all(); -} - -inline native_handle_type -win_random_access_file::release() -{ - return internal_->release(); -} - -inline std::error_code -win_random_access_file::assign(native_handle_type handle) noexcept -{ - return internal_->assign(handle); -} - -inline win_random_access_file_internal* -win_random_access_file::get_internal() const noexcept -{ - return internal_.get(); + svc_.pool_.recycle(this); } // --------------------------------------------------------------------------- @@ -585,61 +491,33 @@ inline win_random_access_file_service::win_random_access_file_service( } } -inline win_random_access_file_service::~win_random_access_file_service() -{ - for (auto* w = wrapper_list_.pop_front(); w != nullptr; - w = wrapper_list_.pop_front()) - delete w; -} - inline io_object::implementation* win_random_access_file_service::construct() { - auto internal = std::make_shared(*this); - - { - std::lock_guard lock(mutex_); - file_list_.push_back(internal.get()); - } - - auto* wrapper = new win_random_access_file(std::move(internal)); - - { - std::lock_guard lock(mutex_); - wrapper_list_.push_back(wrapper); - } - - return wrapper; + return pool_.acquire(*this); } inline void win_random_access_file_service::destroy(io_object::implementation* p) { - if (p) - { - auto& wrapper = static_cast(*p); - wrapper.close_internal(); - destroy_impl(wrapper); - } + if (!p) + return; + auto* f = static_cast(p); + f->close_handle(); + release(f); } inline void win_random_access_file_service::close(io_object::handle& h) { - auto& wrapper = static_cast(*h.get()); - wrapper.get_internal()->close_handle(); + static_cast(h.get())->close_handle(); } inline void win_random_access_file_service::shutdown() { - std::lock_guard lock(mutex_); - - for (auto* impl = file_list_.pop_front(); impl != nullptr; - impl = file_list_.pop_front()) - { - impl->close_handle(); - } + pool_.shutdown( + [](win_random_access_file* impl) { impl->close_handle(); }); } inline std::error_code @@ -702,30 +580,12 @@ win_random_access_file_service::open_file( } } - auto& internal = *static_cast(impl).get_internal(); - internal.handle_ = h; + auto& file = static_cast(impl); + file.handle_ = h; return {}; } -inline void -win_random_access_file_service::destroy_impl(win_random_access_file& impl) -{ - { - std::lock_guard lock(mutex_); - wrapper_list_.remove(&impl); - } - delete &impl; -} - -inline void -win_random_access_file_service::unregister_impl( - win_random_access_file_internal& impl) -{ - std::lock_guard lock(mutex_); - file_list_.remove(&impl); -} - inline void win_random_access_file_service::post(overlapped_op* op) { diff --git a/include/boost/corosio/native/detail/iocp/win_resolver.hpp b/include/boost/corosio/native/detail/iocp/win_resolver.hpp index 873be70df..071451182 100644 --- a/include/boost/corosio/native/detail/iocp/win_resolver.hpp +++ b/include/boost/corosio/native/detail/iocp/win_resolver.hpp @@ -23,6 +23,7 @@ #endif #include +#include #include #include #include @@ -75,11 +76,11 @@ extern "C" Class Hierarchy --------------- - win_resolver_service (execution_context::service) - - Owns all win_resolver instances via shared_ptr + - Owns all win_resolver instances through a recycling object_pool - Coordinates with win_scheduler for work tracking - win_resolver (one per resolver object) - Contains embedded resolve_op and reverse_resolve_op - - Inherits from enable_shared_from_this for thread safety + - Kept alive during I/O by each op's intrusive object_ref_ - resolve_op (overlapped_op subclass) - OVERLAPPED base enables IOCP integration - Static completion() callback invoked by Windows @@ -88,9 +89,10 @@ extern "C" Shutdown -------- - The resolver service cancels all resolvers and clears the impl map. - The thread pool service shuts down separately via execution_context - service ordering, joining all worker threads. + The resolver service cancels all resolvers; each then stays alive + until its own in-flight completion drains, same as every other + pooled impl. The thread pool service shuts down separately via + execution_context service ordering, joining all worker threads. Cancellation ------------ @@ -239,7 +241,6 @@ struct reverse_resolve_op : overlapped_op */ class win_resolver final : public resolver::implementation - , public std::enable_shared_from_this , public intrusive_list::node { friend class win_resolver_service; @@ -254,11 +255,27 @@ class win_resolver final win_resolver* resolver_ = nullptr; /// Prevent impl destruction while work is in flight. - std::shared_ptr ref_; + detail::object_ref ref_; }; explicit win_resolver(win_resolver_service& svc) noexcept; + /// Recycle into the owning service's pool. Defined out-of-line + /// after `win_resolver_service` is complete (needs `svc_.pool_`). + void retire() noexcept override; + + /** Reset recycled state for reuse. + + Does not clear `host`/`service`/`stored_host`/`stored_service` + itself -- each op's own `reset()` (called at the top of the + next `resolve()`/`reverse_resolve()`) already does that, + preserving allocated capacity. That same `reset()` re-seeds + the op's `OVERLAPPED` fields, so they are not zeroed here. + + @pre refs_ == 0, no wait in flight. + */ + void reuse() noexcept; + std::coroutine_handle<> resolve( std::coroutine_handle<>, capy::executor_ref, @@ -298,6 +315,15 @@ class win_resolver final win_resolver_service& svc_; }; +inline void +win_resolver::reuse() noexcept +{ + BOOST_COROSIO_ASSERT(!op_.stop_cb); + BOOST_COROSIO_ASSERT(!reverse_op_.stop_cb); + BOOST_COROSIO_ASSERT(op_.results == nullptr); + BOOST_COROSIO_ASSERT(op_.cancel_handle == nullptr); +} + } // namespace boost::corosio::detail #endif // BOOST_COROSIO_HAS_IOCP diff --git a/include/boost/corosio/native/detail/iocp/win_resolver_service.hpp b/include/boost/corosio/native/detail/iocp/win_resolver_service.hpp index fffa05d13..d6e28622b 100644 --- a/include/boost/corosio/native/detail/iocp/win_resolver_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_resolver_service.hpp @@ -17,10 +17,11 @@ #if BOOST_COROSIO_HAS_IOCP #include +#include #include #include -#include +#include namespace boost::corosio::detail { @@ -46,6 +47,8 @@ class BOOST_COROSIO_DECL win_resolver_service final , public capy::execution_context::service , public io_object::io_service { + friend class win_resolver; + public: using key_type = win_resolver_service; @@ -55,7 +58,7 @@ class BOOST_COROSIO_DECL win_resolver_service final { auto& impl = static_cast(*p); impl.cancel(); - destroy_impl(impl); + release(&impl); } /** Construct the resolver service. @@ -73,9 +76,6 @@ class BOOST_COROSIO_DECL win_resolver_service final /** Shut down the service. */ void shutdown() override; - /** Destroy a resolver implementation. */ - void destroy_impl(win_resolver& impl); - /** Post an operation for completion. */ void post(overlapped_op* op); @@ -106,10 +106,7 @@ class BOOST_COROSIO_DECL win_resolver_service final private: scheduler& sched_; thread_pool_ref pool_; - win_mutex mutex_; - intrusive_list resolver_list_; - std::unordered_map> - resolver_ptrs_; + object_pool impls_; }; BOOST_COROSIO_MSVC_WARNING_POP @@ -229,10 +226,10 @@ resolve_op::completion(DWORD dwError, DWORD /*bytes*/, OVERLAPPED* ov) { auto* op = static_cast(ov); - // The post below can be drained and the win_resolver freed before - // this returns, and Windows owns the OVERLAPPED embedded in it until - // then. - auto keepalive = op->impl->shared_from_this(); + // The post below can be drained and the win_resolver recycled or + // freed before this returns, and Windows owns the OVERLAPPED + // embedded in it until then. + detail::object_ref keepalive(op->impl); op->dwError = dwError; @@ -277,7 +274,7 @@ resolve_op::do_complete( } // Dropping the keepalive may destroy the implementation this op // is embedded in, so nothing may touch it afterwards. - op->impl_ptr.reset(); + op->object_ref_.reset(); svc.work_finished(); return; } @@ -309,7 +306,7 @@ resolve_op::do_complete( op->cont.h = op->h; // Hold the keepalive across the dispatch: it may be the last // reference to the implementation this op is embedded in. - auto prevent_destroy = std::move(op->impl_ptr); + auto prevent_destroy = std::move(op->object_ref_); svc.work_finished(); dispatch_coro(op->ex, op->cont).resume(); } @@ -338,7 +335,7 @@ reverse_resolve_op::do_complete( op->stop_cb.reset(); // Dropping the keepalive may destroy the implementation this // op is embedded in, so nothing may touch it afterwards. - op->impl_ptr.reset(); + op->object_ref_.reset(); svc.work_finished(); return; } @@ -365,7 +362,7 @@ reverse_resolve_op::do_complete( op->cont.h = op->h; // Hold the keepalive across the dispatch: it may be the last // reference to the implementation this op is embedded in. - auto prevent_destroy = std::move(op->impl_ptr); + auto prevent_destroy = std::move(op->object_ref_); svc.work_finished(); dispatch_coro(op->ex, op->cont).resume(); } @@ -414,7 +411,7 @@ win_resolver::resolve( // its completion waits in the scheduler queue: the op is embedded in // this win_resolver, which teardown may otherwise free before the // queued completion drains. Mirrors the reverse path's keepalive. - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); // Under the lock: the handle written here and the callback that // retires it must not interleave. @@ -480,7 +477,7 @@ win_resolver::reverse_resolve( // Prevent impl destruction while work is in flight reverse_pool_op_.resolver_ = this; - reverse_pool_op_.ref_ = this->shared_from_this(); + reverse_pool_op_.ref_ = detail::object_ref(this); reverse_pool_op_.func_ = &win_resolver::do_reverse_resolve_work; if (auto pec = svc_.pool().post(&reverse_pool_op_)) { @@ -570,7 +567,7 @@ win_resolver::do_reverse_resolve_work(pool_work_item* w) noexcept // Hand the keepalive to the op: the completion waits in the // scheduler's queue, and the implementation embedding it must // outlive that wait. Nothing may touch *self after the post. - self->reverse_op_.impl_ptr = std::move(pw->ref_); + self->reverse_op_.object_ref_ = std::move(pw->ref_); // Post only; the initiation credit is released where the op is // consumed. @@ -591,42 +588,40 @@ inline win_resolver_service::~win_resolver_service() {} inline void win_resolver_service::shutdown() { - std::lock_guard lock(mutex_); - - // Cancel all resolvers (sets cancelled flag checked by pool threads) - for (auto* impl = resolver_list_.pop_front(); impl != nullptr; - impl = resolver_list_.pop_front()) + // Snapshot live impls under an acquired reference, then cancel + // without the pool lock held -- mirrors + // uring_socket_service_base::shutdown(). shutdown() sets + // shutting-down and takes the snapshot in one critical section, so + // each cancel's own release (if it drops the last ref) deletes + // rather than recycles. In-flight resolves hold their own + // object_ref_, so each resolver stays alive while its completion + // (GetAddrInfoExW callback or pool thread) drains. The thread pool + // service shuts down separately via execution_context service + // ordering. + std::vector live; + impls_.shutdown( + [&](win_resolver* r) + { + acquire(r); + live.push_back(r); + }); + for (auto* r : live) { - impl->cancel(); + r->cancel(); + release(r); } - - // Clear the map which releases shared_ptrs. - // The thread pool service shuts down separately via - // execution_context service ordering. - resolver_ptrs_.clear(); } inline io_object::implementation* win_resolver_service::construct() { - auto ptr = std::make_shared(*this); - auto* impl = ptr.get(); - - { - std::lock_guard lock(mutex_); - resolver_list_.push_back(impl); - resolver_ptrs_[impl] = std::move(ptr); - } - - return impl; + return impls_.acquire(*this); } inline void -win_resolver_service::destroy_impl(win_resolver& impl) +win_resolver::retire() noexcept { - std::lock_guard lock(mutex_); - resolver_list_.remove(&impl); - resolver_ptrs_.erase(&impl); + svc_.impls_.recycle(this); } inline void diff --git a/include/boost/corosio/native/detail/iocp/win_signal.hpp b/include/boost/corosio/native/detail/iocp/win_signal.hpp index 00e4a2979..1e921ece4 100644 --- a/include/boost/corosio/native/detail/iocp/win_signal.hpp +++ b/include/boost/corosio/native/detail/iocp/win_signal.hpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -125,6 +126,16 @@ class win_signal final public: explicit win_signal(win_signals& svc) noexcept; + /// Recycle into the owning service's pool. Defined out-of-line + /// after `win_signals` is complete (needs `svc_.pool_`). + void retire() noexcept override; + + /** Reset recycled state for reuse. + + @pre refs_ == 0, no registrations, no wait in flight. + */ + void reuse() noexcept; + std::coroutine_handle<> wait( std::coroutine_handle<>, capy::executor_ref, @@ -144,6 +155,16 @@ class win_signal final } }; +inline void +win_signal::reuse() noexcept +{ + BOOST_COROSIO_ASSERT(signals_ == nullptr); + BOOST_COROSIO_ASSERT(!waiting_); + BOOST_COROSIO_ASSERT(!stop_cb_); + cancelled_ = false; + token_cancelled_ = false; +} + } // namespace boost::corosio::detail #endif // BOOST_COROSIO_HAS_IOCP diff --git a/include/boost/corosio/native/detail/iocp/win_signals.hpp b/include/boost/corosio/native/detail/iocp/win_signals.hpp index 2c3df431f..6fb0c4a57 100644 --- a/include/boost/corosio/native/detail/iocp/win_signals.hpp +++ b/include/boost/corosio/native/detail/iocp/win_signals.hpp @@ -19,6 +19,8 @@ #include #include +#include +#include #include #include #include @@ -28,6 +30,7 @@ #include #include #include +#include #include @@ -75,7 +78,7 @@ 2. win_signals (one per execution_context) - Maintains registrations_[] table indexed by signal number - Each slot is a doubly-linked list of all signal_registrations for that signal - - Also maintains impl_list_ of all win_signal objects it owns + - Also owns a recycling pool of all win_signal objects it owns 3. win_signal (one per signal_set) - Owns a singly-linked list (sorted by signal number) of signal_registrations @@ -154,11 +157,21 @@ class BOOST_COROSIO_DECL win_signals final : public capy::execution_context::service , public io_object::io_service { + friend class win_signal; + public: using key_type = win_signals; io_object::implementation* construct() override; - void destroy(io_object::implementation*) override; + + void destroy(io_object::implementation* p) override + { + auto& impl = static_cast(*p); + impl.clear(); + impl.disarm_stop(); + impl.cancel(); + release(&impl); + } /** Construct the signal service. @@ -179,8 +192,6 @@ class BOOST_COROSIO_DECL win_signals final */ void shutdown() override; - /** Destroy a signal implementation. */ - void destroy_impl(win_signal& impl); /** Add a signal to a signal set. @@ -265,7 +276,7 @@ class BOOST_COROSIO_DECL win_signals final BOOST_COROSIO_MSVC_WARNING_PUSH BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // detail:: members, dll-interface win_mutex mutex_; - intrusive_list impl_list_; + object_pool pool_; BOOST_COROSIO_MSVC_WARNING_POP // Per-signal registration table for this service @@ -439,20 +450,34 @@ inline win_signals::~win_signals() inline void win_signals::shutdown() { - // Collected under the locks below and deleted after they are released: - // ~win_signal destroys an armed stop_cb_, and ~stop_callback blocks - // until a concurrently running token_canceller returns -- which takes - // mutex_. Deleting while still holding mutex_ would self-deadlock the - // same way disarm_stop() would if called inside the locked loop. - intrusive_list doomed; + // Collected under the locks below and released after they are + // released: ~win_signal destroys an armed stop_cb_, and + // ~stop_callback blocks until a concurrently running + // token_canceller returns -- which takes mutex_. Releasing while + // still holding mutex_ would self-deadlock the same way + // disarm_stop() would if called inside the locked loop. + // + // The acquire()+release() pair below nets to zero change on each + // impl's refs_ -- it exists only so the lock-held loop above can + // safely walk `doomed` without a concurrent recycle() tearing one + // down mid-walk. It does NOT drop these impls to zero: every impl + // here survives at refs_ == 1 (the service's own floor reference) + // until ~object_pool()'s unconditional sweep deletes whatever is + // still in `live_`, bypassing refs_ entirely. + std::vector doomed; { signal_detail::signal_state* state = signal_detail::get_signal_state(); std::lock_guard state_lock(state->mutex); std::lock_guard lock(mutex_); - for (auto* impl = impl_list_.pop_front(); impl != nullptr; - impl = impl_list_.pop_front()) + pool_.shutdown( + [&](win_signal* impl) + { + acquire(impl); + doomed.push_back(impl); + }); + for (auto* impl : doomed) { while (auto* reg = impl->signals_) { @@ -472,57 +497,31 @@ win_signals::shutdown() impl->signals_ = reg->next_in_set; delete reg; } - doomed.push_back(impl); } - // Every live registration hung off an implementation in impl_list_, - // so the whole table goes stale at once and can be dropped wholesale - // rather than node by node. It has to be dropped: deliver_signal() - // walks this service until the destructor unlinks it from the global - // list. + // Every live registration hung off an implementation this pool + // owns, so the whole table goes stale at once and can be dropped + // wholesale rather than node by node. It has to be dropped: + // deliver_signal() walks this service until the destructor + // unlinks it from the global list. for (int i = 0; i < max_signal_number; ++i) registrations_[i] = nullptr; } - for (auto* impl = doomed.pop_front(); impl != nullptr; - impl = doomed.pop_front()) - { - delete impl; - } + for (auto* impl : doomed) + release(impl); } inline io_object::implementation* win_signals::construct() { - auto* impl = new win_signal(*this); - - { - std::lock_guard lock(mutex_); - impl_list_.push_back(impl); - } - - return impl; -} - -inline void -win_signals::destroy(io_object::implementation* p) -{ - auto& impl = static_cast(*p); - impl.clear(); - impl.disarm_stop(); - impl.cancel(); - destroy_impl(impl); + return pool_.acquire(*this); } inline void -win_signals::destroy_impl(win_signal& impl) +win_signal::retire() noexcept { - { - std::lock_guard lock(mutex_); - impl_list_.remove(&impl); - } - - delete &impl; + svc_.pool_.recycle(this); } inline std::error_code diff --git a/include/boost/corosio/native/detail/iocp/win_stream_file.hpp b/include/boost/corosio/native/detail/iocp/win_stream_file.hpp index 3cb3a9bc8..a69b8e1ab 100644 --- a/include/boost/corosio/native/detail/iocp/win_stream_file.hpp +++ b/include/boost/corosio/native/detail/iocp/win_stream_file.hpp @@ -15,6 +15,7 @@ #if BOOST_COROSIO_HAS_IOCP #include +#include #include #include #include @@ -24,20 +25,18 @@ #include #include -#include namespace boost::corosio::detail { class win_file_service; -class win_stream_file_internal; +class win_stream_file; /** Read operation state for stream file IOCP I/O. */ struct file_read_op : overlapped_op { void* buf = nullptr; DWORD buf_len = 0; - win_stream_file_internal& file_; - std::shared_ptr file_ptr; + win_stream_file& file_; static void do_complete( void* owner, @@ -46,7 +45,7 @@ struct file_read_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit file_read_op(win_stream_file_internal& f) noexcept; + explicit file_read_op(win_stream_file& f) noexcept; }; /** Write operation state for stream file IOCP I/O. */ @@ -54,8 +53,7 @@ struct file_write_op : overlapped_op { void* buf = nullptr; DWORD buf_len = 0; - win_stream_file_internal& file_; - std::shared_ptr file_ptr; + win_stream_file& file_; static void do_complete( void* owner, @@ -64,22 +62,23 @@ struct file_write_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit file_write_op(win_stream_file_internal& f) noexcept; + explicit file_write_op(win_stream_file& f) noexcept; }; -/** Internal stream file state for IOCP-based I/O. +/** Stream file implementation for IOCP-based I/O. - Contains the actual state for a single file, including - the native handle, position tracking, and pending operations. - Derives from enable_shared_from_this so operations can extend - its lifetime. + Collapses the historical internal-state/wrapper split into one + pooled `io_object::implementation`: it both owns the native + handle and pending operations, and directly implements + `stream_file::implementation`. `win_file_service` recycles + instances through its pool instead of freeing them on every + close. */ -class win_stream_file_internal - : public intrusive_list::node - , public std::enable_shared_from_this +class win_stream_file final + : public stream_file::implementation + , public intrusive_list::node { friend class win_file_service; - friend class win_stream_file; friend struct file_read_op; friend struct file_write_op; @@ -90,56 +89,21 @@ class win_stream_file_internal std::uint64_t offset_ = 0; public: - explicit win_stream_file_internal(win_file_service& svc) noexcept; - ~win_stream_file_internal(); - - std::coroutine_handle<> read_some( - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - std::stop_token, - std::error_code*, - std::size_t*); - - std::coroutine_handle<> write_some( - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - std::stop_token, - std::error_code*, - std::size_t*); - - HANDLE native_handle() const noexcept; - bool is_open() const noexcept; - void cancel() noexcept; - void close_handle() noexcept; - - std::uint64_t size() const; - std::error_code resize(std::uint64_t new_size) noexcept; - std::error_code sync_data() noexcept; - std::error_code sync_all() noexcept; - native_handle_type release(); - std::error_code assign(native_handle_type handle) noexcept; - capy::io_result - seek(std::int64_t offset, file_base::seek_basis origin) noexcept; -}; + explicit win_stream_file(win_file_service& svc) noexcept; -/** Stream file implementation wrapper for IOCP-based I/O. + /// Recycle into the owning service's pool. Defined out-of-line + /// after `win_file_service` is complete (needs `svc_.pool_`). + void retire() noexcept override; - Public-facing implementation that holds a shared_ptr to - the internal state. Delegates all virtual calls. -*/ -class win_stream_file final - : public stream_file::implementation - , public intrusive_list::node -{ - std::shared_ptr internal_; + /** Reset recycled state for reuse. -public: - explicit win_stream_file( - std::shared_ptr internal) noexcept; + Each op's `OVERLAPPED` fields are re-seeded by + `overlapped_op::reset()` at its next submit, so they are not + zeroed here. - void close_internal() noexcept; + @pre refs_ == 0, handle closed, no op in flight. + */ + void reuse() noexcept; std::coroutine_handle<> read_some( std::coroutine_handle<> h, @@ -158,6 +122,7 @@ class win_stream_file final std::size_t* bytes) override; native_handle_type native_handle() const noexcept override; + bool is_open() const noexcept; void cancel() noexcept override; std::uint64_t size() const override; std::error_code resize(std::uint64_t new_size) noexcept override; @@ -168,7 +133,7 @@ class win_stream_file final capy::io_result seek(std::int64_t offset, file_base::seek_basis origin) noexcept override; - win_stream_file_internal* get_internal() const noexcept; + void close_handle() noexcept; }; } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/iocp/win_tcp_acceptor.hpp b/include/boost/corosio/native/detail/iocp/win_tcp_acceptor.hpp index 722ca55b0..756210911 100644 --- a/include/boost/corosio/native/detail/iocp/win_tcp_acceptor.hpp +++ b/include/boost/corosio/native/detail/iocp/win_tcp_acceptor.hpp @@ -19,12 +19,12 @@ #include #include #include +#include #include #include #include #include -#include #include #include @@ -34,14 +34,14 @@ namespace boost::corosio::detail { class win_tcp_acceptor_service; class win_tcp_service; class win_tcp_socket; -class win_tcp_acceptor_internal; +class win_tcp_acceptor; /** Accept operation state. */ struct accept_op : overlapped_op { SOCKET accepted_socket = INVALID_SOCKET; win_tcp_socket* peer_wrapper = nullptr; - std::shared_ptr acceptor_ptr; + win_tcp_acceptor& acceptor; SOCKET listen_socket = INVALID_SOCKET; io_object::implementation** impl_out = nullptr; char addr_buf[2 * (sizeof(sockaddr_in6) + 16)]; @@ -53,13 +53,13 @@ struct accept_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - accept_op() noexcept; + explicit accept_op(win_tcp_acceptor& acceptor_) noexcept; }; /** Readiness-wait operation state for an acceptor. */ struct acceptor_wait_op : overlapped_op { - std::shared_ptr acceptor_ptr; + win_tcp_acceptor& acceptor; SOCKET listen_socket = INVALID_SOCKET; static void do_complete( @@ -69,79 +69,57 @@ struct acceptor_wait_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - acceptor_wait_op() noexcept; + explicit acceptor_wait_op(win_tcp_acceptor& acceptor_) noexcept; }; -/** Internal acceptor state for IOCP-based I/O. +/** Acceptor implementation for IOCP-based I/O. - This class contains the actual state for a listening socket, including - the native socket handle and pending accept operation. + Collapses the historical internal-state/wrapper split into one + pooled `io_object::implementation`: it both owns the listening + socket and pending operations, and directly implements + `tcp_acceptor::implementation`. `win_tcp_service` recycles + instances through its acceptor pool instead of freeing them on + every close. @note Internal implementation detail. Users interact with acceptor class. */ -class win_tcp_acceptor_internal - : public intrusive_list::node - , public std::enable_shared_from_this +class win_tcp_acceptor final + : public tcp_acceptor::implementation + , public intrusive_list::node { friend class win_tcp_service; - friend class win_tcp_acceptor; - -public: - explicit win_tcp_acceptor_internal(win_tcp_service& svc) noexcept; - ~win_tcp_acceptor_internal(); - - /// Return the owning socket service. - win_tcp_service& socket_service() noexcept; - - std::coroutine_handle<> accept( - std::coroutine_handle<>, - capy::executor_ref, - std::stop_token, - std::error_code*, - io_object::implementation**); - - std::coroutine_handle<> wait( - std::coroutine_handle<>, - capy::executor_ref, - wait_type, - std::stop_token, - std::error_code*); - - SOCKET native_handle() const noexcept; - endpoint local_endpoint() const noexcept; - bool is_open() const noexcept; - void cancel() noexcept; - void close_socket() noexcept; - void set_local_endpoint(endpoint ep) noexcept; + friend struct accept_op; + friend struct acceptor_wait_op; + win_tcp_service& svc_; accept_op acc_; acceptor_wait_op wt_; - -private: - win_tcp_service& svc_; SOCKET socket_ = INVALID_SOCKET; int family_ = AF_UNSPEC; endpoint local_endpoint_; -}; -/** Acceptor implementation wrapper for IOCP-based I/O. +public: + explicit win_tcp_acceptor(win_tcp_service& svc) noexcept; - This class is the public-facing implementation that holds a shared_ptr - to the internal state. The shared_ptr is hidden from the public interface. + /// Recycle into the owning service's pool. Defined out-of-line + /// after `win_tcp_service` is complete (needs `svc_.acceptor_pool_`). + void retire() noexcept override; - @note Internal implementation detail. Users interact with acceptor class. -*/ -class win_tcp_acceptor final - : public tcp_acceptor::implementation - , public intrusive_list::node -{ - std::shared_ptr internal_; + /** Reset recycled state for reuse. -public: - explicit win_tcp_acceptor( - std::shared_ptr internal) noexcept; + `close_socket()` already drives the socket handle, family, + and cached endpoint to their closed values, and drops every + embedded op's `object_ref_` before `refs_` can reach zero, so + there is little left to reset here -- only to assert. Each + op's `OVERLAPPED` fields are re-seeded by `overlapped_op:: + reset()` at its next submit, so they are not zeroed here. - void close_internal() noexcept; + @pre refs_ == 0, socket closed, no op in flight. + */ + void reuse() noexcept; + + /// Return the owning socket service. + win_tcp_service& socket_service() noexcept; std::coroutine_handle<> accept( std::coroutine_handle<> h, @@ -167,7 +145,7 @@ class win_tcp_acceptor final { // getsockname fails with WSAEINVAL on an unbound socket, so // the family recorded at socket creation is authoritative - return to_family(internal_ ? internal_->family_ : AF_UNSPEC); + return to_family(family_); } native_handle_type release_socket() noexcept override; @@ -180,7 +158,8 @@ class win_tcp_acceptor final get_option(int level, int optname, void* data, std::size_t* size) const noexcept override; - win_tcp_acceptor_internal* get_internal() const noexcept; + void set_local_endpoint(endpoint ep) noexcept; + void close_socket() noexcept; }; } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/iocp/win_tcp_acceptor_service.hpp b/include/boost/corosio/native/detail/iocp/win_tcp_acceptor_service.hpp index 651de18ba..d6a2ae2e0 100644 --- a/include/boost/corosio/native/detail/iocp/win_tcp_acceptor_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_tcp_acceptor_service.hpp @@ -38,7 +38,8 @@ namespace boost::corosio::detail { /** IOCP acceptor service wrapping win_tcp_service for acceptor lifecycle. Provides io_service + acceptor_service interface for tcp_acceptor - on Windows. Delegates to win_tcp_service for actual socket operations. + on Windows. Delegates to win_tcp_service for actual socket operations + and for the acceptor pool itself. */ class BOOST_COROSIO_DECL win_tcp_acceptor_service final : public capy::execution_context::service @@ -79,1388 +80,248 @@ class BOOST_COROSIO_DECL win_tcp_acceptor_service final // Operation constructors -inline connect_op::connect_op(win_tcp_socket_internal& internal_) noexcept +inline accept_op::accept_op(win_tcp_acceptor& acceptor_) noexcept : overlapped_op(&do_complete) - , internal(internal_) + , acceptor(acceptor_) { cancel_func_ = &do_cancel_impl; } -inline read_op::read_op(win_tcp_socket_internal& internal_) noexcept +inline acceptor_wait_op::acceptor_wait_op(win_tcp_acceptor& acceptor_) noexcept : overlapped_op(&do_complete) - , internal(internal_) + , acceptor(acceptor_) { cancel_func_ = &do_cancel_impl; } -inline write_op::write_op(win_tcp_socket_internal& internal_) noexcept - : overlapped_op(&do_complete) - , internal(internal_) -{ - cancel_func_ = &do_cancel_impl; -} - -inline wait_op::wait_op(win_tcp_socket_internal& internal_) noexcept - : overlapped_op(&do_complete) - , internal(internal_) -{ - cancel_func_ = &do_cancel_impl; -} - -inline accept_op::accept_op() noexcept : overlapped_op(&do_complete) -{ - cancel_func_ = &do_cancel_impl; -} - -inline acceptor_wait_op::acceptor_wait_op() noexcept - : overlapped_op(&do_complete) -{ - cancel_func_ = &do_cancel_impl; -} - -// Cancellation functions - -inline void -connect_op::do_cancel_impl(overlapped_op* base) noexcept -{ - auto* op = static_cast(base); - if (op->internal.is_open()) - { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); - } -} - -inline void -read_op::do_cancel_impl(overlapped_op* base) noexcept -{ - auto* op = static_cast(base); - op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) - { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); - } -} - -inline void -write_op::do_cancel_impl(overlapped_op* base) noexcept -{ - auto* op = static_cast(base); - op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) - { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); - } -} - -inline void -wait_op::do_cancel_impl(overlapped_op* base) noexcept -{ - auto* op = static_cast(base); - op->cancelled.store(true, std::memory_order_release); - // Best-effort cancel of any pending zero-byte WSARecv issued for - // wait_type::read. ERROR_NOT_FOUND when nothing overlapped is - // pending is harmless. - if (op->internal.is_open()) - { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); - } - // wait_type::error parks the op in the auxiliary select reactor; - // wake it so the reactor can post a cancelled completion. A cancel - // for an op the reactor never registered finds nothing and returns. - op->internal.svc_.scheduler().cancel_wait(op); -} - -inline void -accept_op::do_cancel_impl(overlapped_op* base) noexcept -{ - auto* op = static_cast(base); - if (op->listen_socket != INVALID_SOCKET) - { - ::CancelIoEx(reinterpret_cast(op->listen_socket), op); - } -} - -inline void -acceptor_wait_op::do_cancel_impl(overlapped_op* base) noexcept -{ - auto* op = static_cast(base); - op->cancelled.store(true, std::memory_order_release); - if (op->listen_socket != INVALID_SOCKET) - { - ::CancelIoEx(reinterpret_cast(op->listen_socket), op); - } - if (op->acceptor_ptr) - { - op->acceptor_ptr->socket_service().scheduler().cancel_wait(op); - } -} - -// accept_op completion handler - -inline void -accept_op::do_complete( - void* owner, - scheduler_op* base, - std::uint32_t /*bytes*/, - std::uint32_t /*error*/) -{ - auto* op = static_cast(base); - - if (!owner) - { - if (op->accepted_socket != INVALID_SOCKET) - { - ::closesocket(op->accepted_socket); - op->accepted_socket = INVALID_SOCKET; - } - - if (op->peer_wrapper) - { - op->peer_wrapper->close_internal(); - op->peer_wrapper = nullptr; - } - - op->cleanup_only(); - op->acceptor_ptr.reset(); - return; - } - - op->stop_cb.reset(); - - bool success = - (op->dwError == 0 && !op->cancelled.load(std::memory_order_acquire)); - - if (op->ec_out) - { - if (op->cancelled.load(std::memory_order_acquire)) - *op->ec_out = capy::error::canceled; - else if (op->dwError != 0) - *op->ec_out = iocp_make_err(op->dwError, /*accept_path=*/true); - else - *op->ec_out = {}; - } - - if (success && op->accepted_socket != INVALID_SOCKET && op->peer_wrapper) - { - ::setsockopt( - op->accepted_socket, SOL_SOCKET, SO_UPDATE_ACCEPT_CONTEXT, - reinterpret_cast(&op->listen_socket), sizeof(SOCKET)); - - op->peer_wrapper->get_internal()->set_socket(op->accepted_socket); - - sockaddr_storage local_storage{}; - int local_len = sizeof(local_storage); - sockaddr_storage remote_storage{}; - int remote_len = sizeof(remote_storage); - - endpoint local_ep, remote_ep; - if (::getsockname( - op->accepted_socket, - reinterpret_cast(&local_storage), &local_len) == 0) - local_ep = from_sockaddr(local_storage); - if (::getpeername( - op->accepted_socket, - reinterpret_cast(&remote_storage), &remote_len) == 0) - remote_ep = from_sockaddr(remote_storage); - - op->peer_wrapper->get_internal()->set_endpoints(local_ep, remote_ep); - op->accepted_socket = INVALID_SOCKET; - - if (op->impl_out) - *op->impl_out = op->peer_wrapper; - } - else - { - if (op->accepted_socket != INVALID_SOCKET) - { - ::closesocket(op->accepted_socket); - op->accepted_socket = INVALID_SOCKET; - } - - if (op->peer_wrapper) - { - op->acceptor_ptr->socket_service().destroy(op->peer_wrapper); - op->peer_wrapper = nullptr; - } - - if (op->impl_out) - *op->impl_out = nullptr; - } - - op->cont.h = op->h; - auto saved_ex = op->ex; - auto prevent_premature_destruction = std::move(op->acceptor_ptr); - - dispatch_coro(saved_ex, op->cont).resume(); -} - -// acceptor_wait_op completion handler - -inline void -acceptor_wait_op::do_complete( - void* owner, - scheduler_op* base, - std::uint32_t /*bytes*/, - std::uint32_t /*error*/) -{ - auto* op = static_cast(base); - - if (!owner) - { - op->cleanup_only(); - op->acceptor_ptr.reset(); - return; - } - - auto prevent_premature_destruction = std::move(op->acceptor_ptr); - op->invoke_handler(); -} - -// connect_op completion handler - -inline void -connect_op::do_complete( - void* owner, - scheduler_op* base, - std::uint32_t /*bytes*/, - std::uint32_t /*error*/) -{ - auto* op = static_cast(base); - - if (!owner) - { - op->cleanup_only(); - op->internal_ptr.reset(); - return; - } - - bool success = - (op->dwError == 0 && !op->cancelled.load(std::memory_order_acquire)); - if (success && op->internal.is_open()) - { - // Required after ConnectEx to enable shutdown(), getsockname(), etc. - ::setsockopt( - op->internal.native_handle(), SOL_SOCKET, SO_UPDATE_CONNECT_CONTEXT, - nullptr, 0); - - endpoint local_ep; - sockaddr_storage local_storage{}; - int local_len = sizeof(local_storage); - if (::getsockname( - op->internal.native_handle(), - reinterpret_cast(&local_storage), &local_len) == 0) - local_ep = from_sockaddr(local_storage); - op->internal.set_endpoints(local_ep, op->target_endpoint); - } - - auto prevent_premature_destruction = std::move(op->internal_ptr); - op->invoke_handler(); -} - -// read_op completion handler - -inline void -read_op::do_complete( - void* owner, - scheduler_op* base, - std::uint32_t /*bytes*/, - std::uint32_t /*error*/) -{ - auto* op = static_cast(base); - - if (!owner) - { - op->cleanup_only(); - op->internal_ptr.reset(); - return; - } - - auto prevent_premature_destruction = std::move(op->internal_ptr); - op->invoke_handler(); -} - -// write_op completion handler - -inline void -write_op::do_complete( - void* owner, - scheduler_op* base, - std::uint32_t /*bytes*/, - std::uint32_t /*error*/) -{ - auto* op = static_cast(base); - - if (!owner) - { - op->cleanup_only(); - op->internal_ptr.reset(); - return; - } - - auto prevent_premature_destruction = std::move(op->internal_ptr); - op->invoke_handler(); -} - -// wait_op completion handler - -inline void -wait_op::do_complete( - void* owner, - scheduler_op* base, - std::uint32_t /*bytes*/, - std::uint32_t /*error*/) -{ - auto* op = static_cast(base); - - if (!owner) - { - op->cleanup_only(); - op->internal_ptr.reset(); - return; - } - - auto prevent_premature_destruction = std::move(op->internal_ptr); - op->invoke_handler(); -} - -// win_tcp_socket_internal - -inline win_tcp_socket_internal::win_tcp_socket_internal( - win_tcp_service& svc) noexcept - : svc_(svc) - , conn_(*this) - , rd_(*this) - , wr_(*this) - , wt_(*this) -{ -} - -inline win_tcp_socket_internal::~win_tcp_socket_internal() -{ - svc_.unregister_impl(*this); -} - -inline SOCKET -win_tcp_socket_internal::native_handle() const noexcept -{ - return socket_; -} - -inline endpoint -win_tcp_socket_internal::local_endpoint() const noexcept -{ - return local_endpoint_; -} - -inline endpoint -win_tcp_socket_internal::remote_endpoint() const noexcept -{ - return remote_endpoint_; -} - -inline bool -win_tcp_socket_internal::is_open() const noexcept -{ - return socket_ != INVALID_SOCKET; -} - -inline void -win_tcp_socket_internal::set_socket(SOCKET s) noexcept -{ - socket_ = s; -} - -inline void -win_tcp_socket_internal::set_endpoints(endpoint local, endpoint remote) noexcept -{ - local_endpoint_ = local; - remote_endpoint_ = remote; -} - -inline std::coroutine_handle<> -win_tcp_socket_internal::connect( - std::coroutine_handle<> h, - capy::executor_ref d, - endpoint ep, - std::stop_token token, - std::error_code* ec) -{ - // Keep internal alive during I/O - conn_.internal_ptr = shared_from_this(); - - auto& op = conn_; - op.reset(); - op.h = h; - op.ex = d; - op.ec_out = ec; - op.target_endpoint = ep; - op.start(token); - - svc_.work_started(); - - // ConnectEx requires the socket to be bound. Skip if already bound - // (e.g. the caller used tcp_socket::bind() before connect). - if (local_endpoint_ == endpoint{}) - { - sockaddr_storage bind_storage{}; - socklen_t bind_len; - if (family_ == AF_INET6) - { - sockaddr_in6 sa6{}; - sa6.sin6_family = AF_INET6; - sa6.sin6_port = 0; - sa6.sin6_addr = in6addr_any; - std::memcpy(&bind_storage, &sa6, sizeof(sa6)); - bind_len = sizeof(sa6); - } - else - { - sockaddr_in sa4{}; - sa4.sin_family = AF_INET; - sa4.sin_addr.s_addr = INADDR_ANY; - sa4.sin_port = 0; - std::memcpy(&bind_storage, &sa4, sizeof(sa4)); - bind_len = sizeof(sa4); - } - - if (::bind( - socket_, reinterpret_cast(&bind_storage), - bind_len) == SOCKET_ERROR) - { - svc_.on_completion(&op, ::WSAGetLastError(), 0); - return std::noop_coroutine(); - } - } - - auto connect_ex = svc_.connect_ex(); - if (!connect_ex) - { - svc_.on_completion(&op, WSAEOPNOTSUPP, 0); - return std::noop_coroutine(); - } - - sockaddr_storage storage{}; - socklen_t addrlen = detail::to_sockaddr(ep, family_, storage); - - BOOL result = connect_ex( - socket_, reinterpret_cast(&storage), - static_cast(addrlen), nullptr, 0, nullptr, &op); - - if (!result) - { - DWORD err = ::WSAGetLastError(); - if (err != ERROR_IO_PENDING) - { - svc_.on_completion(&op, err, 0); - return std::noop_coroutine(); - } - } - - svc_.on_pending(&op); - return std::noop_coroutine(); -} - -inline std::coroutine_handle<> -win_tcp_socket_internal::read_some( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param param, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes_out) -{ - // Keep internal alive during I/O - rd_.internal_ptr = shared_from_this(); - - auto& op = rd_; - op.reset(); - op.is_read = true; - op.h = h; - op.ex = d; - op.ec_out = ec; - op.bytes_out = bytes_out; - op.start(token); - - svc_.work_started(); - - // Closed-object contract: complete with bad_file_descriptor - // without touching the kernel. - if (socket_ == INVALID_SOCKET) - { - svc_.on_completion(&op, WSAEBADF, 0); - return std::noop_coroutine(); - } - - // Prepare buffers - capy::mutable_buffer bufs[read_op::max_buffers]; - op.wsabuf_count = - static_cast(param.copy_to(bufs, read_op::max_buffers)); - - // Handle empty buffer: complete with 0 bytes - if (op.wsabuf_count == 0) - { - op.empty_buffer = true; - svc_.on_completion(&op, 0, 0); - return std::noop_coroutine(); - } - - for (DWORD i = 0; i < op.wsabuf_count; ++i) - { - op.wsabufs[i].buf = static_cast(bufs[i].data()); - op.wsabufs[i].len = static_cast( - (std::min)(bufs[i].size(), std::size_t(0x7fffffff))); - } - - op.flags = 0; - - int result = ::WSARecv( - socket_, op.wsabufs, op.wsabuf_count, nullptr, &op.flags, &op, nullptr); - - if (result == SOCKET_ERROR) - { - DWORD err = ::WSAGetLastError(); - if (err != WSA_IO_PENDING) - { - svc_.on_completion(&op, err, 0); - return std::noop_coroutine(); - } - } - - svc_.on_pending(&op); - - // Re-check cancellation after I/O is pending - if (op.cancelled.load(std::memory_order_acquire)) - ::CancelIoEx(reinterpret_cast(socket_), &op); - - return std::noop_coroutine(); -} - -inline std::coroutine_handle<> -win_tcp_socket_internal::write_some( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param param, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes_out) -{ - // Keep internal alive during I/O - wr_.internal_ptr = shared_from_this(); - - auto& op = wr_; - op.reset(); - op.h = h; - op.ex = d; - op.ec_out = ec; - op.bytes_out = bytes_out; - op.start(token); - - svc_.work_started(); - - // Closed-object contract: complete with bad_file_descriptor - // without touching the kernel. - if (socket_ == INVALID_SOCKET) - { - svc_.on_completion(&op, WSAEBADF, 0); - return std::noop_coroutine(); - } - - // Prepare buffers - capy::mutable_buffer bufs[write_op::max_buffers]; - op.wsabuf_count = - static_cast(param.copy_to(bufs, write_op::max_buffers)); - - // Handle empty buffer: complete immediately with 0 bytes - if (op.wsabuf_count == 0) - { - svc_.on_completion(&op, 0, 0); - return std::noop_coroutine(); - } - - for (DWORD i = 0; i < op.wsabuf_count; ++i) - { - op.wsabufs[i].buf = static_cast(bufs[i].data()); - op.wsabufs[i].len = static_cast( - (std::min)(bufs[i].size(), std::size_t(0x7fffffff))); - } - - int result = ::WSASend( - socket_, op.wsabufs, op.wsabuf_count, nullptr, 0, &op, nullptr); - - if (result == SOCKET_ERROR) - { - DWORD err = ::WSAGetLastError(); - if (err != WSA_IO_PENDING) - { - svc_.on_completion(&op, err, 0); - return std::noop_coroutine(); - } - } - - svc_.on_pending(&op); - - // Re-check cancellation after I/O is pending - if (op.cancelled.load(std::memory_order_acquire)) - ::CancelIoEx(reinterpret_cast(socket_), &op); - - return std::noop_coroutine(); -} - -inline std::coroutine_handle<> -win_tcp_socket_internal::wait( - std::coroutine_handle<> h, - capy::executor_ref d, - wait_type w, - std::stop_token token, - std::error_code* ec) -{ - wt_.internal_ptr = shared_from_this(); - - auto& op = wt_; - op.reset(); - op.h = h; - op.ex = d; - op.ec_out = ec; - op.bytes_out = nullptr; - op.empty_buffer = true; // skip EOF translation in invoke_handler - op.start(token); - - svc_.work_started(); - - // Closed-object contract: complete with bad_file_descriptor - // without touching the kernel or the wait reactor. - if (socket_ == INVALID_SOCKET) - { - svc_.on_completion(&op, WSAEBADF, 0); - return std::noop_coroutine(); - } - - if (w == wait_type::read) - { - // Zero-byte WSARecv: kernel signals completion when data is - // available without consuming any bytes from the stream. This - // is the documented Winsock pattern for "is the socket - // readable" notifications. - op.wsabuf = WSABUF{0, nullptr}; - op.flags = 0; - - int result = - ::WSARecv(socket_, &op.wsabuf, 1, nullptr, &op.flags, &op, nullptr); - - if (result == SOCKET_ERROR) - { - DWORD err = ::WSAGetLastError(); - if (err != WSA_IO_PENDING) - { - svc_.on_completion(&op, err, 0); - return std::noop_coroutine(); - } - } - - svc_.on_pending(&op); - - // Re-check cancellation after I/O is pending. - if (op.cancelled.load(std::memory_order_acquire)) - ::CancelIoEx(reinterpret_cast(socket_), &op); - - return std::noop_coroutine(); - } - - // wait_type::write and wait_type::error: route through the - // auxiliary poll reactor. There is no overlapped primitive for - // "the send buffer has room" that does not also transfer bytes, - // and a write wait must report real writability. - svc_.scheduler().wait_reactor().register_wait(socket_, w, &op); - return std::noop_coroutine(); -} - -inline void -win_tcp_socket_internal::cancel() noexcept -{ - if (socket_ != INVALID_SOCKET) - { - ::CancelIoEx(reinterpret_cast(socket_), nullptr); - } - - conn_.request_cancel(); - rd_.request_cancel(); - wr_.request_cancel(); - wt_.request_cancel(); - // CancelIoEx covers overlapped I/O on the socket but cannot reach - // a wait op parked in the auxiliary reactor (no overlapped is - // outstanding). Route through the reactor explicitly. - svc_.scheduler().cancel_wait(&wt_); -} - -inline void -win_tcp_socket_internal::close_socket() noexcept -{ - // Flag every op cancelled before tearing down the handle. closesocket() - // can complete a pending overlapped op with ERROR_NETNAME_DELETED (not the - // MSDN-documented ERROR_OPERATION_ABORTED), which iocp_make_err now maps to - // connection_reset -- so a locally-closed op must be short-circuited to - // canceled via the flag rather than relying on the raw error, exactly as - // cancel() does. The store happens-before the completion is decoded. - conn_.request_cancel(); - rd_.request_cancel(); - wr_.request_cancel(); - // wt_ is parked in the aux reactor (no overlapped outstanding), so it also - // needs an explicit reactor deregister before the SOCKET handle is closed; - // otherwise the reactor would keep polling a dangling fd (and on a Winsock - // SOCKET-id reuse the wrong fd could be polled briefly). - wt_.request_cancel(); - svc_.scheduler().cancel_wait(&wt_); - - if (socket_ != INVALID_SOCKET) - { - ::CancelIoEx(reinterpret_cast(socket_), nullptr); - ::closesocket(socket_); - socket_ = INVALID_SOCKET; - } - - family_ = AF_UNSPEC; - - // Clear cached endpoints - local_endpoint_ = endpoint{}; - remote_endpoint_ = endpoint{}; -} - -// win_tcp_socket - -inline win_tcp_socket::win_tcp_socket( - std::shared_ptr internal) noexcept - : internal_(std::move(internal)) -{ -} - -inline void -win_tcp_socket::close_internal() noexcept -{ - if (internal_) - { - internal_->close_socket(); - internal_.reset(); - } -} - -inline std::coroutine_handle<> -win_tcp_socket::connect( - std::coroutine_handle<> h, - capy::executor_ref d, - endpoint ep, - std::stop_token token, - std::error_code* ec) -{ - return internal_->connect(h, d, ep, token, ec); -} - -inline std::coroutine_handle<> -win_tcp_socket::read_some( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->read_some(h, d, buf, token, ec, bytes); -} - -inline std::coroutine_handle<> -win_tcp_socket::write_some( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->write_some(h, d, buf, token, ec, bytes); -} - -inline std::coroutine_handle<> -win_tcp_socket::wait( - std::coroutine_handle<> h, - capy::executor_ref d, - wait_type w, - std::stop_token token, - std::error_code* ec) -{ - return internal_->wait(h, d, w, token, ec); -} - -inline std::error_code -win_tcp_socket::shutdown(tcp_socket::shutdown_type what) noexcept -{ - int how; - switch (what) - { - case tcp_socket::shutdown_receive: - how = SD_RECEIVE; - break; - case tcp_socket::shutdown_send: - how = SD_SEND; - break; - case tcp_socket::shutdown_both: - how = SD_BOTH; - break; - default: - return make_err(WSAEINVAL); - } - if (::shutdown(internal_->native_handle(), how) != 0) - return make_err(WSAGetLastError()); - return {}; -} - -inline native_handle_type -win_tcp_socket::native_handle() const noexcept -{ - return static_cast(internal_->native_handle()); -} - -inline native_handle_type -win_tcp_socket::release_socket() noexcept -{ - SOCKET s = internal_->socket_; - if (s != INVALID_SOCKET) - { - internal_->cancel(); - // Sever the port association so the descriptor can be - // adopted again; best-effort, the caller keeps a working - // socket either way. - dissociate_from_iocp(s); - internal_->socket_ = INVALID_SOCKET; - internal_->family_ = AF_UNSPEC; - internal_->local_endpoint_ = endpoint{}; - internal_->remote_endpoint_ = endpoint{}; - } - return static_cast(s); -} - -inline std::error_code -win_tcp_socket::set_option( - int level, int optname, void const* data, std::size_t size) noexcept -{ - if (::setsockopt( - internal_->native_handle(), level, optname, - reinterpret_cast(data), static_cast(size)) != 0) - return make_err(WSAGetLastError()); - return {}; -} - -inline std::error_code -win_tcp_socket::get_option( - int level, int optname, void* data, std::size_t* size) const noexcept -{ - int len = static_cast(*size); - if (::getsockopt( - internal_->native_handle(), level, optname, - reinterpret_cast(data), &len) != 0) - return make_err(WSAGetLastError()); - *size = static_cast(len); - return {}; -} - -inline endpoint -win_tcp_socket::local_endpoint() const noexcept -{ - return internal_->local_endpoint(); -} - -inline endpoint -win_tcp_socket::remote_endpoint() const noexcept -{ - return internal_->remote_endpoint(); -} - -inline void -win_tcp_socket::cancel() noexcept -{ - internal_->cancel(); -} - -inline win_tcp_socket_internal* -win_tcp_socket::get_internal() const noexcept -{ - return internal_.get(); -} - -// win_tcp_service - -inline win_tcp_service::win_tcp_service(capy::execution_context& ctx) - : sched_(ctx.use_service()) - , iocp_(sched_.native_handle()) -{ - load_extension_functions(); -} - -inline win_tcp_service::~win_tcp_service() -{ - // Delete wrappers that survived shutdown. This runs after - // win_scheduler is destroyed (reverse creation order), so - // all coroutine frames and their tcp_socket members are gone. - for (auto* w = socket_wrapper_list_.pop_front(); w != nullptr; - w = socket_wrapper_list_.pop_front()) - delete w; - - for (auto* w = acceptor_wrapper_list_.pop_front(); w != nullptr; - w = acceptor_wrapper_list_.pop_front()) - delete w; -} - -inline void -win_tcp_service::shutdown() -{ - std::lock_guard lock(mutex_); - - // Close all sockets to force pending I/O to complete via IOCP. - // Wrappers are NOT deleted here - coroutine frames destroyed - // during scheduler shutdown may still hold tcp_socket objects - // that reference them. Wrapper deletion is deferred to ~win_tcp_service - // after the scheduler has drained all outstanding operations. - for (auto* impl = socket_list_.pop_front(); impl != nullptr; - impl = socket_list_.pop_front()) - { - impl->close_socket(); - } - - for (auto* impl = acceptor_list_.pop_front(); impl != nullptr; - impl = acceptor_list_.pop_front()) - { - impl->close_socket(); - } -} - -inline io_object::implementation* -win_tcp_service::construct() -{ - auto internal = std::make_shared(*this); - - { - std::lock_guard lock(mutex_); - socket_list_.push_back(internal.get()); - } - - auto* wrapper = new win_tcp_socket(std::move(internal)); - - { - std::lock_guard lock(mutex_); - socket_wrapper_list_.push_back(wrapper); - } - - return wrapper; -} - -inline void -win_tcp_service::destroy(io_object::implementation* p) -{ - if (p) - { - auto& wrapper = static_cast(*p); - wrapper.close_internal(); - destroy_impl(wrapper); - } -} - -inline void -win_tcp_service::close(io_object::handle& h) -{ - auto& wrapper = static_cast(*h.get()); - wrapper.get_internal()->close_socket(); -} +// Cancellation functions inline void -win_tcp_service::destroy_impl(win_tcp_socket& impl) +accept_op::do_cancel_impl(overlapped_op* base) noexcept { + auto* op = static_cast(base); + if (op->listen_socket != INVALID_SOCKET) { - std::lock_guard lock(mutex_); - socket_wrapper_list_.remove(&impl); + ::CancelIoEx(reinterpret_cast(op->listen_socket), op); } - delete &impl; } inline void -win_tcp_service::unregister_impl(win_tcp_socket_internal& impl) -{ - std::lock_guard lock(mutex_); - socket_list_.remove(&impl); -} - -inline std::error_code -win_tcp_service::open_socket( - win_tcp_socket_internal& impl, int family, int type, int protocol) +acceptor_wait_op::do_cancel_impl(overlapped_op* base) noexcept { - impl.close_socket(); - - SOCKET sock = - ::WSASocketW(family, type, protocol, nullptr, 0, WSA_FLAG_OVERLAPPED); - - if (sock == INVALID_SOCKET) - return make_err(::WSAGetLastError()); - - if (family == AF_INET6) - { - DWORD one = 1; - ::setsockopt( - sock, IPPROTO_IPV6, IPV6_V6ONLY, reinterpret_cast(&one), - sizeof(one)); - } - - HANDLE result = ::CreateIoCompletionPort( - reinterpret_cast(sock), static_cast(iocp_), key_io, 0); - - if (result == nullptr) + auto* op = static_cast(base); + op->cancelled.store(true, std::memory_order_release); + if (op->listen_socket != INVALID_SOCKET) { - DWORD dwError = ::GetLastError(); - ::closesocket(sock); - return make_err(dwError); + ::CancelIoEx(reinterpret_cast(op->listen_socket), op); } - - impl.socket_ = sock; - impl.family_ = family; - return {}; -} - -inline std::error_code -win_tcp_service::assign_socket( - win_tcp_socket_internal& impl, native_handle_type fd) -{ - SOCKET sock = static_cast(fd); - if (sock == INVALID_SOCKET) - return make_err(WSAENOTSOCK); - if (sock == impl.socket_) - return std::make_error_code(std::errc::invalid_argument); - - // SO_PROTOCOL_INFOW works on an unbound socket, unlike getsockname - // (WSAEINVAL until bind/connect names it) -- an adopted socket may - // have reached connected state without an explicit bind. - WSAPROTOCOL_INFOW proto_info{}; - int proto_len = sizeof(proto_info); - if (::getsockopt( - sock, SOL_SOCKET, SO_PROTOCOL_INFOW, - reinterpret_cast(&proto_info), &proto_len) != 0) - return make_err(::WSAGetLastError()); - if (proto_info.iAddressFamily != AF_INET && - proto_info.iAddressFamily != AF_INET6) - return make_err(WSAEAFNOSUPPORT); - if (proto_info.iSocketType != SOCK_STREAM) - return make_err(WSAEPROTOTYPE); - - // Associate before releasing the held socket: on IOCP nothing - // shares descriptor state the way the reactor path does, so a - // failed association must not cost the caller their old socket. - HANDLE result = ::CreateIoCompletionPort( - reinterpret_cast(sock), static_cast(iocp_), key_io, 0); - if (result == nullptr) - return make_err(::GetLastError()); - - impl.close_socket(); - impl.socket_ = sock; - impl.family_ = proto_info.iAddressFamily; - - endpoint local_ep, remote_ep; - sockaddr_storage local_storage{}; - int local_len = sizeof(local_storage); - if (::getsockname( - sock, reinterpret_cast(&local_storage), &local_len) == 0) - local_ep = detail::from_sockaddr(local_storage); - sockaddr_storage remote_storage{}; - int remote_len = sizeof(remote_storage); - if (::getpeername( - sock, reinterpret_cast(&remote_storage), &remote_len) == - 0) - remote_ep = detail::from_sockaddr(remote_storage); - impl.set_endpoints(local_ep, remote_ep); - - return {}; -} - -inline std::error_code -win_tcp_service::bind_socket(win_tcp_socket_internal& impl, endpoint ep) -{ - SOCKET sock = impl.socket_; - - sockaddr_storage storage{}; - socklen_t addrlen = detail::to_sockaddr(ep, storage); - if (::bind( - sock, reinterpret_cast(&storage), - static_cast(addrlen)) == SOCKET_ERROR) - return make_err(::WSAGetLastError()); - - // Cache local endpoint (resolves ephemeral port) - sockaddr_storage local_storage{}; - int local_len = sizeof(local_storage); - if (::getsockname( - sock, reinterpret_cast(&local_storage), &local_len) == 0) - impl.local_endpoint_ = detail::from_sockaddr(local_storage); - - return {}; -} - -inline void* -win_tcp_service::native_handle() const noexcept -{ - return iocp_; -} - -inline void -win_tcp_service::post(overlapped_op* op) -{ - sched_.post(op); -} - -inline void -win_tcp_service::on_pending(overlapped_op* op) noexcept -{ - sched_.on_pending(op); + op->acceptor.socket_service().scheduler().cancel_wait(op); } -inline void -win_tcp_service::on_completion( - overlapped_op* op, DWORD error, DWORD bytes) noexcept -{ - sched_.on_completion(op, error, bytes); -} +// accept_op completion handler inline void -win_tcp_service::work_started() noexcept +accept_op::do_complete( + void* owner, + scheduler_op* base, + std::uint32_t /*bytes*/, + std::uint32_t /*error*/) { - sched_.work_started(); -} + auto* op = static_cast(base); -inline void -win_tcp_service::work_finished() noexcept -{ - sched_.work_finished(); -} + if (!owner) + { + if (op->accepted_socket != INVALID_SOCKET) + { + ::closesocket(op->accepted_socket); + op->accepted_socket = INVALID_SOCKET; + } -inline void -win_tcp_service::load_extension_functions() -{ - SOCKET sock = ::WSASocketW( - AF_INET, SOCK_STREAM, IPPROTO_TCP, nullptr, 0, WSA_FLAG_OVERLAPPED); + if (op->peer_wrapper) + { + op->peer_wrapper->close_socket(); + op->peer_wrapper = nullptr; + } - if (sock == INVALID_SOCKET) + op->cleanup_only(); + op->object_ref_.reset(); return; + } - DWORD bytes = 0; - - GUID connect_ex_guid = WSAID_CONNECTEX; - ::WSAIoctl( - sock, SIO_GET_EXTENSION_FUNCTION_POINTER, &connect_ex_guid, - sizeof(connect_ex_guid), &connect_ex_, sizeof(connect_ex_), &bytes, - nullptr, nullptr); - - GUID accept_ex_guid = WSAID_ACCEPTEX; - ::WSAIoctl( - sock, SIO_GET_EXTENSION_FUNCTION_POINTER, &accept_ex_guid, - sizeof(accept_ex_guid), &accept_ex_, sizeof(accept_ex_), &bytes, - nullptr, nullptr); + op->stop_cb.reset(); - ::closesocket(sock); -} + bool success = + (op->dwError == 0 && !op->cancelled.load(std::memory_order_acquire)); -inline void -win_tcp_service::destroy_acceptor_impl(win_tcp_acceptor& impl) -{ + if (op->ec_out) { - std::lock_guard lock(mutex_); - acceptor_wrapper_list_.remove(&impl); + if (op->cancelled.load(std::memory_order_acquire)) + *op->ec_out = capy::error::canceled; + else if (op->dwError != 0) + *op->ec_out = iocp_make_err(op->dwError, /*accept_path=*/true); + else + *op->ec_out = {}; } - delete &impl; -} - -inline void -win_tcp_service::unregister_acceptor_impl(win_tcp_acceptor_internal& impl) -{ - std::lock_guard lock(mutex_); - acceptor_list_.remove(&impl); -} -inline std::error_code -win_tcp_service::open_acceptor_socket( - win_tcp_acceptor_internal& impl, int family, int type, int protocol) -{ - impl.close_socket(); - - SOCKET sock = - ::WSASocketW(family, type, protocol, nullptr, 0, WSA_FLAG_OVERLAPPED); - - if (sock == INVALID_SOCKET) - return make_err(::WSAGetLastError()); - - if (family == AF_INET6) + if (success && op->accepted_socket != INVALID_SOCKET && op->peer_wrapper) { - DWORD val = 0; // dual-stack default ::setsockopt( - sock, IPPROTO_IPV6, IPV6_V6ONLY, reinterpret_cast(&val), - sizeof(val)); - } + op->accepted_socket, SOL_SOCKET, SO_UPDATE_ACCEPT_CONTEXT, + reinterpret_cast(&op->listen_socket), sizeof(SOCKET)); - HANDLE result = ::CreateIoCompletionPort( - reinterpret_cast(sock), static_cast(iocp_), key_io, 0); + op->peer_wrapper->set_socket(op->accepted_socket); - if (result == nullptr) - { - DWORD dwError = ::GetLastError(); - ::closesocket(sock); - return make_err(dwError); - } + sockaddr_storage local_storage{}; + int local_len = sizeof(local_storage); + sockaddr_storage remote_storage{}; + int remote_len = sizeof(remote_storage); - impl.socket_ = sock; - impl.family_ = family; - return {}; -} + endpoint local_ep, remote_ep; + if (::getsockname( + op->accepted_socket, + reinterpret_cast(&local_storage), &local_len) == 0) + local_ep = from_sockaddr(local_storage); + if (::getpeername( + op->accepted_socket, + reinterpret_cast(&remote_storage), &remote_len) == 0) + remote_ep = from_sockaddr(remote_storage); -inline std::error_code -win_tcp_service::assign_acceptor_socket( - win_tcp_acceptor_internal& impl, native_handle_type fd) -{ - SOCKET sock = static_cast(fd); - if (sock == INVALID_SOCKET) - return make_err(WSAENOTSOCK); - if (sock == impl.socket_) - return std::make_error_code(std::errc::invalid_argument); - - // SO_PROTOCOL_INFOW works on an unbound socket, unlike getsockname - // (WSAEINVAL until bind names it). - WSAPROTOCOL_INFOW proto_info{}; - int proto_len = sizeof(proto_info); - if (::getsockopt( - sock, SOL_SOCKET, SO_PROTOCOL_INFOW, - reinterpret_cast(&proto_info), &proto_len) != 0) - return make_err(::WSAGetLastError()); - if (proto_info.iAddressFamily != AF_INET && - proto_info.iAddressFamily != AF_INET6) - return make_err(WSAEAFNOSUPPORT); - if (proto_info.iSocketType != SOCK_STREAM) - return make_err(WSAEPROTOTYPE); - - // Associate before releasing the held socket: on IOCP nothing - // shares descriptor state the way the reactor path does, so a - // failed association must not cost the caller their old socket. - HANDLE result = ::CreateIoCompletionPort( - reinterpret_cast(sock), static_cast(iocp_), key_io, 0); - if (result == nullptr) - return make_err(::GetLastError()); + op->peer_wrapper->set_endpoints(local_ep, remote_ep); + op->accepted_socket = INVALID_SOCKET; - impl.close_socket(); - impl.socket_ = sock; - impl.family_ = proto_info.iAddressFamily; + if (op->impl_out) + *op->impl_out = op->peer_wrapper; + // Handed off to the caller: don't let a failed early return on + // the next accept() see this (now user-owned) pointer again. + op->peer_wrapper = nullptr; + } + else + { + if (op->accepted_socket != INVALID_SOCKET) + { + ::closesocket(op->accepted_socket); + op->accepted_socket = INVALID_SOCKET; + } - // AcceptEx sizes its address buffers from this cache, so an - // unseeded endpoint breaks accepts on an adopted v6 listener. - sockaddr_storage local_storage{}; - int local_len = sizeof(local_storage); - if (::getsockname( - sock, reinterpret_cast(&local_storage), &local_len) == 0) - impl.set_local_endpoint(detail::from_sockaddr(local_storage)); + if (op->peer_wrapper) + { + op->acceptor.socket_service().destroy(op->peer_wrapper); + op->peer_wrapper = nullptr; + } - return {}; -} + if (op->impl_out) + *op->impl_out = nullptr; + } -inline std::error_code -win_tcp_service::bind_acceptor(win_tcp_acceptor_internal& impl, endpoint ep) -{ - SOCKET sock = impl.socket_; - - sockaddr_storage storage{}; - socklen_t addrlen = detail::to_sockaddr(ep, storage); - if (::bind( - sock, reinterpret_cast(&storage), - static_cast(addrlen)) == SOCKET_ERROR) - return make_err(::WSAGetLastError()); - - // Cache local endpoint (resolves ephemeral port) - sockaddr_storage local_storage{}; - int local_len = sizeof(local_storage); - if (::getsockname( - sock, reinterpret_cast(&local_storage), &local_len) == 0) - impl.set_local_endpoint(detail::from_sockaddr(local_storage)); + op->cont.h = op->h; + auto saved_ex = op->ex; + auto prevent_premature_destruction = std::move(op->object_ref_); - return {}; + dispatch_coro(saved_ex, op->cont).resume(); } -inline std::error_code -win_tcp_service::listen_acceptor(win_tcp_acceptor_internal& impl, int backlog) +// acceptor_wait_op completion handler + +inline void +acceptor_wait_op::do_complete( + void* owner, + scheduler_op* base, + std::uint32_t /*bytes*/, + std::uint32_t /*error*/) { - SOCKET sock = impl.socket_; + auto* op = static_cast(base); - if (::listen(sock, backlog) == SOCKET_ERROR) - return make_err(::WSAGetLastError()); + if (!owner) + { + op->cleanup_only(); + op->object_ref_.reset(); + return; + } - return {}; + auto prevent_premature_destruction = std::move(op->object_ref_); + op->invoke_handler(); } -// win_tcp_acceptor_internal +// win_tcp_acceptor -inline win_tcp_acceptor_internal::win_tcp_acceptor_internal( - win_tcp_service& svc) noexcept +inline win_tcp_acceptor::win_tcp_acceptor(win_tcp_service& svc) noexcept : svc_(svc) + , acc_(*this) + , wt_(*this) { } -inline win_tcp_acceptor_internal::~win_tcp_acceptor_internal() +inline void +win_tcp_acceptor::reuse() noexcept { - svc_.unregister_acceptor_impl(*this); + BOOST_COROSIO_ASSERT(socket_ == INVALID_SOCKET); + BOOST_COROSIO_ASSERT(family_ == AF_UNSPEC); + BOOST_COROSIO_ASSERT(local_endpoint_ == endpoint{}); + BOOST_COROSIO_ASSERT(!acc_.stop_cb); + BOOST_COROSIO_ASSERT(!wt_.stop_cb); + BOOST_COROSIO_ASSERT(acc_.peer_wrapper == nullptr); + BOOST_COROSIO_ASSERT(acc_.accepted_socket == INVALID_SOCKET); } inline win_tcp_service& -win_tcp_acceptor_internal::socket_service() noexcept +win_tcp_acceptor::socket_service() noexcept { return svc_; } -inline SOCKET -win_tcp_acceptor_internal::native_handle() const noexcept +inline native_handle_type +win_tcp_acceptor::native_handle() const noexcept { - return socket_; + return static_cast(socket_); } inline endpoint -win_tcp_acceptor_internal::local_endpoint() const noexcept +win_tcp_acceptor::local_endpoint() const noexcept { return local_endpoint_; } inline bool -win_tcp_acceptor_internal::is_open() const noexcept +win_tcp_acceptor::is_open() const noexcept { return socket_ != INVALID_SOCKET; } inline void -win_tcp_acceptor_internal::set_local_endpoint(endpoint ep) noexcept +win_tcp_acceptor::set_local_endpoint(endpoint ep) noexcept { local_endpoint_ = ep; } inline std::coroutine_handle<> -win_tcp_acceptor_internal::accept( +win_tcp_acceptor::accept( std::coroutine_handle<> h, capy::executor_ref d, std::stop_token token, std::error_code* ec, io_object::implementation** impl_out) { - // Keep acceptor internal alive during I/O - acc_.acceptor_ptr = shared_from_this(); + // Keep this acceptor alive during I/O + acc_.object_ref_ = detail::object_ref(this); auto& op = acc_; op.reset(); - op.h = h; - op.ex = d; - op.ec_out = ec; - op.impl_out = impl_out; + // reset() doesn't touch these -- a value left over from a prior + // accept() (e.g. the socket already handed to the user via + // *impl_out) must not survive into a completion this call fails + // before reassigning; do_complete's failure branch would destroy() + // whatever stale pointer it finds here. + op.peer_wrapper = nullptr; + op.accepted_socket = INVALID_SOCKET; + op.h = h; + op.ex = d; + op.ec_out = ec; + op.impl_out = impl_out; op.start(token); svc_.work_started(); - // Create wrapper for the peer socket (service owns it) + // Create a (possibly recycled) peer socket; the service owns it. auto& peer_wrapper = static_cast(*svc_.construct()); // Derive AF from the listening socket's cached local endpoint @@ -1541,14 +402,14 @@ win_tcp_acceptor_internal::accept( } inline std::coroutine_handle<> -win_tcp_acceptor_internal::wait( +win_tcp_acceptor::wait( std::coroutine_handle<> h, capy::executor_ref d, wait_type w, std::stop_token token, std::error_code* ec) { - wt_.acceptor_ptr = shared_from_this(); + wt_.object_ref_ = detail::object_ref(this); wt_.listen_socket = socket_; auto& op = wt_; @@ -1584,7 +445,7 @@ win_tcp_acceptor_internal::wait( } inline void -win_tcp_acceptor_internal::cancel() noexcept +win_tcp_acceptor::cancel() noexcept { if (socket_ != INVALID_SOCKET) { @@ -1597,7 +458,7 @@ win_tcp_acceptor_internal::cancel() noexcept } inline void -win_tcp_acceptor_internal::close_socket() noexcept +win_tcp_acceptor::close_socket() noexcept { // Flag the accept op cancelled before closing so a closesocket-delivered // ERROR_NETNAME_DELETED is short-circuited to canceled rather than mapped @@ -1620,85 +481,17 @@ win_tcp_acceptor_internal::close_socket() noexcept local_endpoint_ = endpoint{}; } -// win_tcp_acceptor - -inline win_tcp_acceptor::win_tcp_acceptor( - std::shared_ptr internal) noexcept - : internal_(std::move(internal)) -{ -} - -inline void -win_tcp_acceptor::close_internal() noexcept -{ - if (internal_) - { - internal_->close_socket(); - internal_.reset(); - } -} - -inline std::coroutine_handle<> -win_tcp_acceptor::accept( - std::coroutine_handle<> h, - capy::executor_ref d, - std::stop_token token, - std::error_code* ec, - io_object::implementation** impl_out) -{ - return internal_->accept(h, d, token, ec, impl_out); -} - -inline std::coroutine_handle<> -win_tcp_acceptor::wait( - std::coroutine_handle<> h, - capy::executor_ref d, - wait_type w, - std::stop_token token, - std::error_code* ec) -{ - return internal_->wait(h, d, w, token, ec); -} - -inline endpoint -win_tcp_acceptor::local_endpoint() const noexcept -{ - return internal_->local_endpoint(); -} - -inline bool -win_tcp_acceptor::is_open() const noexcept -{ - return internal_ && internal_->is_open(); -} - -inline void -win_tcp_acceptor::cancel() noexcept -{ - internal_->cancel(); -} - -inline native_handle_type -win_tcp_acceptor::native_handle() const noexcept -{ - if (!internal_) - return static_cast(INVALID_SOCKET); - return static_cast(internal_->native_handle()); -} - inline native_handle_type win_tcp_acceptor::release_socket() noexcept { - if (!internal_) - return static_cast(INVALID_SOCKET); - SOCKET s = internal_->socket_; + SOCKET s = socket_; if (s != INVALID_SOCKET) { - internal_->cancel(); + cancel(); dissociate_from_iocp(s); - internal_->socket_ = INVALID_SOCKET; - internal_->family_ = AF_UNSPEC; - internal_->local_endpoint_ = endpoint{}; + socket_ = INVALID_SOCKET; + family_ = AF_UNSPEC; + local_endpoint_ = endpoint{}; } return static_cast(s); } @@ -1708,8 +501,8 @@ win_tcp_acceptor::set_option( int level, int optname, void const* data, std::size_t size) noexcept { if (::setsockopt( - internal_->native_handle(), level, optname, - reinterpret_cast(data), static_cast(size)) != 0) + socket_, level, optname, reinterpret_cast(data), + static_cast(size)) != 0) return make_err(WSAGetLastError()); return {}; } @@ -1720,17 +513,16 @@ win_tcp_acceptor::get_option( { int len = static_cast(*size); if (::getsockopt( - internal_->native_handle(), level, optname, - reinterpret_cast(data), &len) != 0) + socket_, level, optname, reinterpret_cast(data), &len) != 0) return make_err(WSAGetLastError()); *size = static_cast(len); return {}; } -inline win_tcp_acceptor_internal* -win_tcp_acceptor::get_internal() const noexcept +inline void +win_tcp_acceptor::retire() noexcept { - return internal_.get(); + svc_.acceptor_pool_.recycle(this); } // win_tcp_acceptor_service @@ -1744,77 +536,61 @@ inline win_tcp_acceptor_service::win_tcp_acceptor_service( inline io_object::implementation* win_tcp_acceptor_service::construct() { - auto internal = std::make_shared(svc_); - - { - std::lock_guard lock(svc_.mutex_); - svc_.acceptor_list_.push_back(internal.get()); - } - - auto* wrapper = new win_tcp_acceptor(std::move(internal)); - - { - std::lock_guard lock(svc_.mutex_); - svc_.acceptor_wrapper_list_.push_back(wrapper); - } - - return wrapper; + return svc_.acquire_acceptor_impl(); } inline void win_tcp_acceptor_service::destroy(io_object::implementation* p) { - if (p) - { - auto& wrapper = static_cast(*p); - wrapper.close_internal(); - svc_.destroy_acceptor_impl(wrapper); - } + if (!p) + return; + auto* a = static_cast(p); + a->close_socket(); + release(a); } inline void win_tcp_acceptor_service::close(io_object::handle& h) { - auto& wrapper = static_cast(*h.get()); - wrapper.get_internal()->close_socket(); + static_cast(h.get())->close_socket(); } inline std::error_code win_tcp_acceptor_service::open_acceptor_socket( tcp_acceptor::implementation& impl, int family, int type, int protocol) { - auto& wrapper = static_cast(impl); - return svc_.open_acceptor_socket( - *wrapper.get_internal(), family, type, protocol); + auto& acc = static_cast(impl); + return svc_.open_acceptor_socket(acc, family, type, protocol); } inline std::error_code win_tcp_acceptor_service::assign_socket( tcp_acceptor::implementation& impl, native_handle_type fd) { - auto& wrapper = static_cast(impl); - return svc_.assign_acceptor_socket(*wrapper.get_internal(), fd); + auto& acc = static_cast(impl); + return svc_.assign_acceptor_socket(acc, fd); } inline std::error_code win_tcp_acceptor_service::bind_acceptor( tcp_acceptor::implementation& impl, endpoint ep) { - auto& wrapper = static_cast(impl); - return svc_.bind_acceptor(*wrapper.get_internal(), ep); + auto& acc = static_cast(impl); + return svc_.bind_acceptor(acc, ep); } inline std::error_code win_tcp_acceptor_service::listen_acceptor( tcp_acceptor::implementation& impl, int backlog) { - auto& wrapper = static_cast(impl); - return svc_.listen_acceptor(*wrapper.get_internal(), backlog); + auto& acc = static_cast(impl); + return svc_.listen_acceptor(acc, backlog); } inline void win_tcp_acceptor_service::shutdown() { + // Socket/acceptor shutdown is handled by win_tcp_service::shutdown() } } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/iocp/win_tcp_service.hpp b/include/boost/corosio/native/detail/iocp/win_tcp_service.hpp index 2203b2fa0..aa57b196e 100644 --- a/include/boost/corosio/native/detail/iocp/win_tcp_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_tcp_service.hpp @@ -16,13 +16,18 @@ #if BOOST_COROSIO_HAS_IOCP #include +#include +#include #include #include -#include +#include #include #include +#include +#include #include +#include #include @@ -30,18 +35,18 @@ namespace boost::corosio::detail { class win_scheduler; class win_tcp_acceptor; -class win_tcp_acceptor_internal; class win_tcp_acceptor_service; /** Windows IOCP socket management service. - This service owns all socket implementations and coordinates their - lifecycle with the IOCP. It provides: + This service owns all socket and acceptor implementations and + coordinates their lifecycle with the IOCP. It provides: - - Socket implementation allocation and deallocation + - Socket/acceptor implementation recycling through `object_pool` - IOCP handle association for sockets - Function pointer loading for ConnectEx/AcceptEx - - Graceful shutdown - destroys all implementations when io_context stops + - Graceful shutdown - closes every live implementation when the + io_context stops; pool destruction frees whatever is left @par Thread Safety All public member functions are thread-safe. @@ -53,6 +58,9 @@ class BOOST_COROSIO_DECL win_tcp_service final , public capy::execution_context::service , public io_object::io_service { + friend class win_tcp_socket; + friend class win_tcp_acceptor; + public: using key_type = win_tcp_service; @@ -71,32 +79,19 @@ class BOOST_COROSIO_DECL win_tcp_service final */ explicit win_tcp_service(capy::execution_context& ctx); - /** Destroy the socket service. */ - ~win_tcp_service(); - win_tcp_service(win_tcp_service const&) = delete; win_tcp_service& operator=(win_tcp_service const&) = delete; /** Shut down the service. */ void shutdown() override; - /** Destroy a socket implementation wrapper. - Removes from tracking list and deletes. - */ - void destroy_impl(win_tcp_socket& impl); - - /** Unregister a socket implementation from the service list. - Called by the internal impl destructor. - */ - void unregister_impl(win_tcp_socket_internal& impl); - /** Create and register a socket with the IOCP. - @param impl The socket implementation internal to initialize. + @param impl The socket implementation to initialize. @return Error code, or success. */ std::error_code open_socket( - win_tcp_socket_internal& impl, int family, int type, int protocol); + win_tcp_socket& impl, int family, int type, int protocol); /** Adopt an existing socket handle into an implementation. @@ -105,30 +100,24 @@ class BOOST_COROSIO_DECL win_tcp_service final impl takes ownership and will close the handle; on failure the caller retains ownership. - @param impl The socket implementation internal to assign to. + @param impl The socket implementation to assign to. @param fd The native socket handle to adopt. @return Error code, or success. */ - std::error_code - assign_socket(win_tcp_socket_internal& impl, native_handle_type fd); + std::error_code assign_socket(win_tcp_socket& impl, native_handle_type fd); /** Bind a stream socket to a local endpoint. - @param impl The socket implementation internal to bind. + @param impl The socket implementation to bind. @param ep The local endpoint to bind to. @return Error code, or success. */ - std::error_code bind_socket(win_tcp_socket_internal& impl, endpoint ep); - - /** Destroy an acceptor implementation wrapper. - Removes from tracking list and deletes. - */ - void destroy_acceptor_impl(win_tcp_acceptor& impl); + std::error_code bind_socket(win_tcp_socket& impl, endpoint ep); - /** Unregister an acceptor implementation from the service list. - Called by the internal impl destructor. + /** Pop a recycled acceptor or news one, with the service reference + held. Used by `win_tcp_acceptor_service::construct()`. */ - void unregister_acceptor_impl(win_tcp_acceptor_internal& impl); + win_tcp_acceptor* acquire_acceptor_impl(); /** Create an acceptor socket without binding or listening. @@ -136,14 +125,14 @@ class BOOST_COROSIO_DECL win_tcp_service final For IPv6, dual-stack is enabled by default. Does not set SO_REUSEADDR. - @param impl The acceptor implementation internal to initialize. + @param impl The acceptor implementation to initialize. @param family Address family (e.g. `AF_INET`, `AF_INET6`). @param type Socket type (e.g. `SOCK_STREAM`). @param protocol Protocol number (e.g. `IPPROTO_TCP`). @return Error code, or success. */ std::error_code open_acceptor_socket( - win_tcp_acceptor_internal& impl, int family, int type, int protocol); + win_tcp_acceptor& impl, int family, int type, int protocol); /** Adopt an existing listening socket into an acceptor. @@ -151,30 +140,29 @@ class BOOST_COROSIO_DECL win_tcp_service final then releases the socket the acceptor already held. Listen state is not verified. - @param impl The acceptor implementation internal. + @param impl The acceptor implementation. @param fd The native socket to adopt. Ownership transfers only on success. @return Error code, or success. */ - std::error_code assign_acceptor_socket( - win_tcp_acceptor_internal& impl, native_handle_type fd); + std::error_code + assign_acceptor_socket(win_tcp_acceptor& impl, native_handle_type fd); /** Bind an open acceptor to a local endpoint. - @param impl The acceptor implementation internal. + @param impl The acceptor implementation. @param ep The local endpoint to bind to. @return Error code, or success. */ - std::error_code bind_acceptor(win_tcp_acceptor_internal& impl, endpoint ep); + std::error_code bind_acceptor(win_tcp_acceptor& impl, endpoint ep); /** Start listening for incoming connections. - @param impl The acceptor implementation internal. + @param impl The acceptor implementation. @param backlog The listen backlog. @return Error code, or success. */ - std::error_code - listen_acceptor(win_tcp_acceptor_internal& impl, int backlog); + std::error_code listen_acceptor(win_tcp_acceptor& impl, int backlog); /** Return the IOCP handle. */ void* native_handle() const noexcept; @@ -213,24 +201,1022 @@ class BOOST_COROSIO_DECL win_tcp_service final } private: - friend class win_tcp_acceptor_service; - void load_extension_functions(); win_scheduler& sched_; - BOOST_COROSIO_MSVC_WARNING_PUSH - BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // detail:: members, dll-interface - win_mutex mutex_; - intrusive_list socket_list_; - intrusive_list acceptor_list_; - intrusive_list socket_wrapper_list_; - intrusive_list acceptor_wrapper_list_; - BOOST_COROSIO_MSVC_WARNING_POP void* iocp_; LPFN_CONNECTEX connect_ex_ = nullptr; LPFN_ACCEPTEX accept_ex_ = nullptr; + BOOST_COROSIO_MSVC_WARNING_PUSH + BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // detail:: members, dll-interface + object_pool pool_; + object_pool acceptor_pool_; + BOOST_COROSIO_MSVC_WARNING_POP }; +// Operation constructors + +inline connect_op::connect_op(win_tcp_socket& internal_) noexcept + : overlapped_op(&do_complete) + , internal(internal_) +{ + cancel_func_ = &do_cancel_impl; +} + +inline read_op::read_op(win_tcp_socket& internal_) noexcept + : overlapped_op(&do_complete) + , internal(internal_) +{ + cancel_func_ = &do_cancel_impl; +} + +inline write_op::write_op(win_tcp_socket& internal_) noexcept + : overlapped_op(&do_complete) + , internal(internal_) +{ + cancel_func_ = &do_cancel_impl; +} + +inline wait_op::wait_op(win_tcp_socket& internal_) noexcept + : overlapped_op(&do_complete) + , internal(internal_) +{ + cancel_func_ = &do_cancel_impl; +} + +// Cancellation functions + +inline void +connect_op::do_cancel_impl(overlapped_op* base) noexcept +{ + auto* op = static_cast(base); + if (op->internal.socket_ != INVALID_SOCKET) + { + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); + } +} + +inline void +read_op::do_cancel_impl(overlapped_op* base) noexcept +{ + auto* op = static_cast(base); + op->cancelled.store(true, std::memory_order_release); + if (op->internal.socket_ != INVALID_SOCKET) + { + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); + } +} + +inline void +write_op::do_cancel_impl(overlapped_op* base) noexcept +{ + auto* op = static_cast(base); + op->cancelled.store(true, std::memory_order_release); + if (op->internal.socket_ != INVALID_SOCKET) + { + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); + } +} + +inline void +wait_op::do_cancel_impl(overlapped_op* base) noexcept +{ + auto* op = static_cast(base); + op->cancelled.store(true, std::memory_order_release); + // Best-effort cancel of any pending zero-byte WSARecv issued for + // wait_type::read. ERROR_NOT_FOUND when nothing overlapped is + // pending is harmless. + if (op->internal.socket_ != INVALID_SOCKET) + { + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); + } + // wait_type::error parks the op in the auxiliary select reactor; + // wake it so the reactor can post a cancelled completion. A cancel + // for an op the reactor never registered finds nothing and returns. + op->internal.svc_.scheduler().cancel_wait(op); +} + +// connect_op completion handler + +inline void +connect_op::do_complete( + void* owner, + scheduler_op* base, + std::uint32_t /*bytes*/, + std::uint32_t /*error*/) +{ + auto* op = static_cast(base); + + if (!owner) + { + op->cleanup_only(); + op->object_ref_.reset(); + return; + } + + bool success = + (op->dwError == 0 && !op->cancelled.load(std::memory_order_acquire)); + if (success && op->internal.socket_ != INVALID_SOCKET) + { + // Required after ConnectEx to enable shutdown(), getsockname(), etc. + ::setsockopt( + op->internal.socket_, SOL_SOCKET, SO_UPDATE_CONNECT_CONTEXT, + nullptr, 0); + + endpoint local_ep; + sockaddr_storage local_storage{}; + int local_len = sizeof(local_storage); + if (::getsockname( + op->internal.socket_, + reinterpret_cast(&local_storage), &local_len) == 0) + local_ep = from_sockaddr(local_storage); + op->internal.set_endpoints(local_ep, op->target_endpoint); + } + + auto prevent_premature_destruction = std::move(op->object_ref_); + op->invoke_handler(); +} + +// read_op completion handler + +inline void +read_op::do_complete( + void* owner, + scheduler_op* base, + std::uint32_t /*bytes*/, + std::uint32_t /*error*/) +{ + auto* op = static_cast(base); + + if (!owner) + { + op->cleanup_only(); + op->object_ref_.reset(); + return; + } + + auto prevent_premature_destruction = std::move(op->object_ref_); + op->invoke_handler(); +} + +// write_op completion handler + +inline void +write_op::do_complete( + void* owner, + scheduler_op* base, + std::uint32_t /*bytes*/, + std::uint32_t /*error*/) +{ + auto* op = static_cast(base); + + if (!owner) + { + op->cleanup_only(); + op->object_ref_.reset(); + return; + } + + auto prevent_premature_destruction = std::move(op->object_ref_); + op->invoke_handler(); +} + +// wait_op completion handler + +inline void +wait_op::do_complete( + void* owner, + scheduler_op* base, + std::uint32_t /*bytes*/, + std::uint32_t /*error*/) +{ + auto* op = static_cast(base); + + if (!owner) + { + op->cleanup_only(); + op->object_ref_.reset(); + return; + } + + auto prevent_premature_destruction = std::move(op->object_ref_); + op->invoke_handler(); +} + +// win_tcp_socket + +inline win_tcp_socket::win_tcp_socket(win_tcp_service& svc) noexcept + : svc_(svc) + , conn_(*this) + , rd_(*this) + , wr_(*this) + , wt_(*this) +{ +} + +inline void +win_tcp_socket::retire() noexcept +{ + svc_.pool_.recycle(this); +} + +inline void +win_tcp_socket::reuse() noexcept +{ + BOOST_COROSIO_ASSERT(socket_ == INVALID_SOCKET); + BOOST_COROSIO_ASSERT(family_ == AF_UNSPEC); + BOOST_COROSIO_ASSERT(local_endpoint_ == endpoint{}); + BOOST_COROSIO_ASSERT(remote_endpoint_ == endpoint{}); + BOOST_COROSIO_ASSERT(!conn_.stop_cb); + BOOST_COROSIO_ASSERT(!rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wr_.stop_cb); + BOOST_COROSIO_ASSERT(!wt_.stop_cb); +} + +inline void +win_tcp_socket::set_socket(SOCKET s) noexcept +{ + socket_ = s; +} + +inline void +win_tcp_socket::set_endpoints(endpoint local, endpoint remote) noexcept +{ + local_endpoint_ = local; + remote_endpoint_ = remote; +} + +inline std::coroutine_handle<> +win_tcp_socket::connect( + std::coroutine_handle<> h, + capy::executor_ref d, + endpoint ep, + std::stop_token token, + std::error_code* ec) +{ + // Keep this socket alive during I/O + conn_.object_ref_ = detail::object_ref(this); + + auto& op = conn_; + op.reset(); + op.h = h; + op.ex = d; + op.ec_out = ec; + op.target_endpoint = ep; + op.start(token); + + svc_.work_started(); + + // ConnectEx requires the socket to be bound. Skip if already bound + // (e.g. the caller used tcp_socket::bind() before connect). + if (local_endpoint_ == endpoint{}) + { + sockaddr_storage bind_storage{}; + socklen_t bind_len; + if (family_ == AF_INET6) + { + sockaddr_in6 sa6{}; + sa6.sin6_family = AF_INET6; + sa6.sin6_port = 0; + sa6.sin6_addr = in6addr_any; + std::memcpy(&bind_storage, &sa6, sizeof(sa6)); + bind_len = sizeof(sa6); + } + else + { + sockaddr_in sa4{}; + sa4.sin_family = AF_INET; + sa4.sin_addr.s_addr = INADDR_ANY; + sa4.sin_port = 0; + std::memcpy(&bind_storage, &sa4, sizeof(sa4)); + bind_len = sizeof(sa4); + } + + if (::bind( + socket_, reinterpret_cast(&bind_storage), + bind_len) == SOCKET_ERROR) + { + svc_.on_completion(&op, ::WSAGetLastError(), 0); + return std::noop_coroutine(); + } + } + + auto connect_ex = svc_.connect_ex(); + if (!connect_ex) + { + svc_.on_completion(&op, WSAEOPNOTSUPP, 0); + return std::noop_coroutine(); + } + + sockaddr_storage storage{}; + socklen_t addrlen = detail::to_sockaddr(ep, family_, storage); + + BOOL result = connect_ex( + socket_, reinterpret_cast(&storage), + static_cast(addrlen), nullptr, 0, nullptr, &op); + + if (!result) + { + DWORD err = ::WSAGetLastError(); + if (err != ERROR_IO_PENDING) + { + svc_.on_completion(&op, err, 0); + return std::noop_coroutine(); + } + } + + svc_.on_pending(&op); + return std::noop_coroutine(); +} + +inline std::coroutine_handle<> +win_tcp_socket::read_some( + std::coroutine_handle<> h, + capy::executor_ref d, + buffer_param param, + std::stop_token token, + std::error_code* ec, + std::size_t* bytes_out) +{ + // Keep this socket alive during I/O + rd_.object_ref_ = detail::object_ref(this); + + auto& op = rd_; + op.reset(); + op.is_read = true; + op.h = h; + op.ex = d; + op.ec_out = ec; + op.bytes_out = bytes_out; + op.start(token); + + svc_.work_started(); + + // Closed-object contract: complete with bad_file_descriptor + // without touching the kernel. + if (socket_ == INVALID_SOCKET) + { + svc_.on_completion(&op, WSAEBADF, 0); + return std::noop_coroutine(); + } + + // Prepare buffers + capy::mutable_buffer bufs[read_op::max_buffers]; + op.wsabuf_count = + static_cast(param.copy_to(bufs, read_op::max_buffers)); + + // Handle empty buffer: complete with 0 bytes + if (op.wsabuf_count == 0) + { + op.empty_buffer = true; + svc_.on_completion(&op, 0, 0); + return std::noop_coroutine(); + } + + for (DWORD i = 0; i < op.wsabuf_count; ++i) + { + op.wsabufs[i].buf = static_cast(bufs[i].data()); + op.wsabufs[i].len = static_cast( + (std::min)(bufs[i].size(), std::size_t(0x7fffffff))); + } + + op.flags = 0; + + int result = ::WSARecv( + socket_, op.wsabufs, op.wsabuf_count, nullptr, &op.flags, &op, nullptr); + + if (result == SOCKET_ERROR) + { + DWORD err = ::WSAGetLastError(); + if (err != WSA_IO_PENDING) + { + svc_.on_completion(&op, err, 0); + return std::noop_coroutine(); + } + } + + svc_.on_pending(&op); + + // Re-check cancellation after I/O is pending + if (op.cancelled.load(std::memory_order_acquire)) + ::CancelIoEx(reinterpret_cast(socket_), &op); + + return std::noop_coroutine(); +} + +inline std::coroutine_handle<> +win_tcp_socket::write_some( + std::coroutine_handle<> h, + capy::executor_ref d, + buffer_param param, + std::stop_token token, + std::error_code* ec, + std::size_t* bytes_out) +{ + // Keep this socket alive during I/O + wr_.object_ref_ = detail::object_ref(this); + + auto& op = wr_; + op.reset(); + op.h = h; + op.ex = d; + op.ec_out = ec; + op.bytes_out = bytes_out; + op.start(token); + + svc_.work_started(); + + // Closed-object contract: complete with bad_file_descriptor + // without touching the kernel. + if (socket_ == INVALID_SOCKET) + { + svc_.on_completion(&op, WSAEBADF, 0); + return std::noop_coroutine(); + } + + // Prepare buffers + capy::mutable_buffer bufs[write_op::max_buffers]; + op.wsabuf_count = + static_cast(param.copy_to(bufs, write_op::max_buffers)); + + // Handle empty buffer: complete immediately with 0 bytes + if (op.wsabuf_count == 0) + { + svc_.on_completion(&op, 0, 0); + return std::noop_coroutine(); + } + + for (DWORD i = 0; i < op.wsabuf_count; ++i) + { + op.wsabufs[i].buf = static_cast(bufs[i].data()); + op.wsabufs[i].len = static_cast( + (std::min)(bufs[i].size(), std::size_t(0x7fffffff))); + } + + int result = ::WSASend( + socket_, op.wsabufs, op.wsabuf_count, nullptr, 0, &op, nullptr); + + if (result == SOCKET_ERROR) + { + DWORD err = ::WSAGetLastError(); + if (err != WSA_IO_PENDING) + { + svc_.on_completion(&op, err, 0); + return std::noop_coroutine(); + } + } + + svc_.on_pending(&op); + + // Re-check cancellation after I/O is pending + if (op.cancelled.load(std::memory_order_acquire)) + ::CancelIoEx(reinterpret_cast(socket_), &op); + + return std::noop_coroutine(); +} + +inline std::coroutine_handle<> +win_tcp_socket::wait( + std::coroutine_handle<> h, + capy::executor_ref d, + wait_type w, + std::stop_token token, + std::error_code* ec) +{ + wt_.object_ref_ = detail::object_ref(this); + + auto& op = wt_; + op.reset(); + op.h = h; + op.ex = d; + op.ec_out = ec; + op.bytes_out = nullptr; + op.empty_buffer = true; // skip EOF translation in invoke_handler + op.start(token); + + svc_.work_started(); + + // Closed-object contract: complete with bad_file_descriptor + // without touching the kernel or the wait reactor. + if (socket_ == INVALID_SOCKET) + { + svc_.on_completion(&op, WSAEBADF, 0); + return std::noop_coroutine(); + } + + if (w == wait_type::read) + { + // Zero-byte WSARecv: kernel signals completion when data is + // available without consuming any bytes from the stream. This + // is the documented Winsock pattern for "is the socket + // readable" notifications. + op.wsabuf = WSABUF{0, nullptr}; + op.flags = 0; + + int result = + ::WSARecv(socket_, &op.wsabuf, 1, nullptr, &op.flags, &op, nullptr); + + if (result == SOCKET_ERROR) + { + DWORD err = ::WSAGetLastError(); + if (err != WSA_IO_PENDING) + { + svc_.on_completion(&op, err, 0); + return std::noop_coroutine(); + } + } + + svc_.on_pending(&op); + + // Re-check cancellation after I/O is pending. + if (op.cancelled.load(std::memory_order_acquire)) + ::CancelIoEx(reinterpret_cast(socket_), &op); + + return std::noop_coroutine(); + } + + // wait_type::write and wait_type::error: route through the + // auxiliary poll reactor. There is no overlapped primitive for + // "the send buffer has room" that does not also transfer bytes, + // and a write wait must report real writability. + svc_.scheduler().wait_reactor().register_wait(socket_, w, &op); + return std::noop_coroutine(); +} + +inline void +win_tcp_socket::cancel() noexcept +{ + if (socket_ != INVALID_SOCKET) + { + ::CancelIoEx(reinterpret_cast(socket_), nullptr); + } + + conn_.request_cancel(); + rd_.request_cancel(); + wr_.request_cancel(); + wt_.request_cancel(); + // CancelIoEx covers overlapped I/O on the socket but cannot reach + // a wait op parked in the auxiliary reactor (no overlapped is + // outstanding). Route through the reactor explicitly. + svc_.scheduler().cancel_wait(&wt_); +} + +inline void +win_tcp_socket::close_socket() noexcept +{ + // Flag every op cancelled before tearing down the handle. closesocket() + // can complete a pending overlapped op with ERROR_NETNAME_DELETED (not the + // MSDN-documented ERROR_OPERATION_ABORTED), which iocp_make_err now maps to + // connection_reset -- so a locally-closed op must be short-circuited to + // canceled via the flag rather than relying on the raw error, exactly as + // cancel() does. The store happens-before the completion is decoded. + conn_.request_cancel(); + rd_.request_cancel(); + wr_.request_cancel(); + // wt_ is parked in the aux reactor (no overlapped outstanding), so it also + // needs an explicit reactor deregister before the SOCKET handle is closed; + // otherwise the reactor would keep polling a dangling fd (and on a Winsock + // SOCKET-id reuse the wrong fd could be polled briefly). + wt_.request_cancel(); + svc_.scheduler().cancel_wait(&wt_); + + if (socket_ != INVALID_SOCKET) + { + ::CancelIoEx(reinterpret_cast(socket_), nullptr); + ::closesocket(socket_); + socket_ = INVALID_SOCKET; + } + + family_ = AF_UNSPEC; + + // Clear cached endpoints + local_endpoint_ = endpoint{}; + remote_endpoint_ = endpoint{}; +} + +inline std::error_code +win_tcp_socket::shutdown(tcp_socket::shutdown_type what) noexcept +{ + int how; + switch (what) + { + case tcp_socket::shutdown_receive: + how = SD_RECEIVE; + break; + case tcp_socket::shutdown_send: + how = SD_SEND; + break; + case tcp_socket::shutdown_both: + how = SD_BOTH; + break; + default: + return make_err(WSAEINVAL); + } + if (::shutdown(socket_, how) != 0) + return make_err(WSAGetLastError()); + return {}; +} + +inline native_handle_type +win_tcp_socket::native_handle() const noexcept +{ + return static_cast(socket_); +} + +inline native_handle_type +win_tcp_socket::release_socket() noexcept +{ + SOCKET s = socket_; + if (s != INVALID_SOCKET) + { + cancel(); + // Sever the port association so the descriptor can be + // adopted again; best-effort, the caller keeps a working + // socket either way. + dissociate_from_iocp(s); + socket_ = INVALID_SOCKET; + family_ = AF_UNSPEC; + local_endpoint_ = endpoint{}; + remote_endpoint_ = endpoint{}; + } + return static_cast(s); +} + +inline std::error_code +win_tcp_socket::set_option( + int level, int optname, void const* data, std::size_t size) noexcept +{ + if (::setsockopt( + socket_, level, optname, reinterpret_cast(data), + static_cast(size)) != 0) + return make_err(WSAGetLastError()); + return {}; +} + +inline std::error_code +win_tcp_socket::get_option( + int level, int optname, void* data, std::size_t* size) const noexcept +{ + int len = static_cast(*size); + if (::getsockopt( + socket_, level, optname, reinterpret_cast(data), &len) != 0) + return make_err(WSAGetLastError()); + *size = static_cast(len); + return {}; +} + +inline endpoint +win_tcp_socket::local_endpoint() const noexcept +{ + return local_endpoint_; +} + +inline endpoint +win_tcp_socket::remote_endpoint() const noexcept +{ + return remote_endpoint_; +} + +// win_tcp_service + +inline win_tcp_service::win_tcp_service(capy::execution_context& ctx) + : sched_(ctx.use_service()) + , iocp_(sched_.native_handle()) +{ + load_extension_functions(); +} + +inline void +win_tcp_service::shutdown() +{ + // Close all sockets/acceptors to force pending I/O to complete via + // IOCP. Not deleted here: in-flight ops hold their own object_ref, so + // each impl stays alive until its cancel completion drains; the + // pools' own destructors free whatever is still live once this + // service itself is torn down (after the scheduler, which drains + // every queued completion first). + pool_.shutdown([](win_tcp_socket* impl) { impl->close_socket(); }); + + acceptor_pool_.shutdown( + [](win_tcp_acceptor* impl) { impl->close_socket(); }); +} + +inline io_object::implementation* +win_tcp_service::construct() +{ + return pool_.acquire(*this); +} + +inline void +win_tcp_service::destroy(io_object::implementation* p) +{ + if (!p) + return; + auto* s = static_cast(p); + s->close_socket(); + release(s); +} + +inline void +win_tcp_service::close(io_object::handle& h) +{ + static_cast(h.get())->close_socket(); +} + +inline std::error_code +win_tcp_service::open_socket( + win_tcp_socket& impl, int family, int type, int protocol) +{ + impl.close_socket(); + + SOCKET sock = + ::WSASocketW(family, type, protocol, nullptr, 0, WSA_FLAG_OVERLAPPED); + + if (sock == INVALID_SOCKET) + return make_err(::WSAGetLastError()); + + if (family == AF_INET6) + { + DWORD one = 1; + ::setsockopt( + sock, IPPROTO_IPV6, IPV6_V6ONLY, reinterpret_cast(&one), + sizeof(one)); + } + + HANDLE result = ::CreateIoCompletionPort( + reinterpret_cast(sock), static_cast(iocp_), key_io, 0); + + if (result == nullptr) + { + DWORD dwError = ::GetLastError(); + ::closesocket(sock); + return make_err(dwError); + } + + impl.socket_ = sock; + impl.family_ = family; + return {}; +} + +inline std::error_code +win_tcp_service::assign_socket(win_tcp_socket& impl, native_handle_type fd) +{ + SOCKET sock = static_cast(fd); + if (sock == INVALID_SOCKET) + return make_err(WSAENOTSOCK); + if (sock == impl.socket_) + return std::make_error_code(std::errc::invalid_argument); + + // SO_PROTOCOL_INFOW works on an unbound socket, unlike getsockname + // (WSAEINVAL until bind/connect names it) -- an adopted socket may + // have reached connected state without an explicit bind. + WSAPROTOCOL_INFOW proto_info{}; + int proto_len = sizeof(proto_info); + if (::getsockopt( + sock, SOL_SOCKET, SO_PROTOCOL_INFOW, + reinterpret_cast(&proto_info), &proto_len) != 0) + return make_err(::WSAGetLastError()); + if (proto_info.iAddressFamily != AF_INET && + proto_info.iAddressFamily != AF_INET6) + return make_err(WSAEAFNOSUPPORT); + if (proto_info.iSocketType != SOCK_STREAM) + return make_err(WSAEPROTOTYPE); + + // Associate before releasing the held socket: on IOCP nothing + // shares descriptor state the way the reactor path does, so a + // failed association must not cost the caller their old socket. + HANDLE result = ::CreateIoCompletionPort( + reinterpret_cast(sock), static_cast(iocp_), key_io, 0); + if (result == nullptr) + return make_err(::GetLastError()); + + impl.close_socket(); + impl.socket_ = sock; + impl.family_ = proto_info.iAddressFamily; + + endpoint local_ep, remote_ep; + sockaddr_storage local_storage{}; + int local_len = sizeof(local_storage); + if (::getsockname( + sock, reinterpret_cast(&local_storage), &local_len) == 0) + local_ep = detail::from_sockaddr(local_storage); + sockaddr_storage remote_storage{}; + int remote_len = sizeof(remote_storage); + if (::getpeername( + sock, reinterpret_cast(&remote_storage), &remote_len) == + 0) + remote_ep = detail::from_sockaddr(remote_storage); + impl.set_endpoints(local_ep, remote_ep); + + return {}; +} + +inline std::error_code +win_tcp_service::bind_socket(win_tcp_socket& impl, endpoint ep) +{ + SOCKET sock = impl.socket_; + + sockaddr_storage storage{}; + socklen_t addrlen = detail::to_sockaddr(ep, storage); + if (::bind( + sock, reinterpret_cast(&storage), + static_cast(addrlen)) == SOCKET_ERROR) + return make_err(::WSAGetLastError()); + + // Cache local endpoint (resolves ephemeral port) + sockaddr_storage local_storage{}; + int local_len = sizeof(local_storage); + if (::getsockname( + sock, reinterpret_cast(&local_storage), &local_len) == 0) + impl.local_endpoint_ = detail::from_sockaddr(local_storage); + + return {}; +} + +inline void* +win_tcp_service::native_handle() const noexcept +{ + return iocp_; +} + +inline void +win_tcp_service::post(overlapped_op* op) +{ + sched_.post(op); +} + +inline void +win_tcp_service::on_pending(overlapped_op* op) noexcept +{ + sched_.on_pending(op); +} + +inline void +win_tcp_service::on_completion( + overlapped_op* op, DWORD error, DWORD bytes) noexcept +{ + sched_.on_completion(op, error, bytes); +} + +inline void +win_tcp_service::work_started() noexcept +{ + sched_.work_started(); +} + +inline void +win_tcp_service::work_finished() noexcept +{ + sched_.work_finished(); +} + +inline void +win_tcp_service::load_extension_functions() +{ + SOCKET sock = ::WSASocketW( + AF_INET, SOCK_STREAM, IPPROTO_TCP, nullptr, 0, WSA_FLAG_OVERLAPPED); + + if (sock == INVALID_SOCKET) + return; + + DWORD bytes = 0; + + GUID connect_ex_guid = WSAID_CONNECTEX; + ::WSAIoctl( + sock, SIO_GET_EXTENSION_FUNCTION_POINTER, &connect_ex_guid, + sizeof(connect_ex_guid), &connect_ex_, sizeof(connect_ex_), &bytes, + nullptr, nullptr); + + GUID accept_ex_guid = WSAID_ACCEPTEX; + ::WSAIoctl( + sock, SIO_GET_EXTENSION_FUNCTION_POINTER, &accept_ex_guid, + sizeof(accept_ex_guid), &accept_ex_, sizeof(accept_ex_), &bytes, + nullptr, nullptr); + + ::closesocket(sock); +} + +inline win_tcp_acceptor* +win_tcp_service::acquire_acceptor_impl() +{ + return acceptor_pool_.acquire(*this); +} + +inline std::error_code +win_tcp_service::open_acceptor_socket( + win_tcp_acceptor& impl, int family, int type, int protocol) +{ + impl.close_socket(); + + SOCKET sock = + ::WSASocketW(family, type, protocol, nullptr, 0, WSA_FLAG_OVERLAPPED); + + if (sock == INVALID_SOCKET) + return make_err(::WSAGetLastError()); + + if (family == AF_INET6) + { + DWORD val = 0; // dual-stack default + ::setsockopt( + sock, IPPROTO_IPV6, IPV6_V6ONLY, reinterpret_cast(&val), + sizeof(val)); + } + + HANDLE result = ::CreateIoCompletionPort( + reinterpret_cast(sock), static_cast(iocp_), key_io, 0); + + if (result == nullptr) + { + DWORD dwError = ::GetLastError(); + ::closesocket(sock); + return make_err(dwError); + } + + impl.socket_ = sock; + impl.family_ = family; + return {}; +} + +inline std::error_code +win_tcp_service::assign_acceptor_socket( + win_tcp_acceptor& impl, native_handle_type fd) +{ + SOCKET sock = static_cast(fd); + if (sock == INVALID_SOCKET) + return make_err(WSAENOTSOCK); + if (sock == impl.socket_) + return std::make_error_code(std::errc::invalid_argument); + + // SO_PROTOCOL_INFOW works on an unbound socket, unlike getsockname + // (WSAEINVAL until bind names it). + WSAPROTOCOL_INFOW proto_info{}; + int proto_len = sizeof(proto_info); + if (::getsockopt( + sock, SOL_SOCKET, SO_PROTOCOL_INFOW, + reinterpret_cast(&proto_info), &proto_len) != 0) + return make_err(::WSAGetLastError()); + if (proto_info.iAddressFamily != AF_INET && + proto_info.iAddressFamily != AF_INET6) + return make_err(WSAEAFNOSUPPORT); + if (proto_info.iSocketType != SOCK_STREAM) + return make_err(WSAEPROTOTYPE); + + // Associate before releasing the held socket: on IOCP nothing + // shares descriptor state the way the reactor path does, so a + // failed association must not cost the caller their old socket. + HANDLE result = ::CreateIoCompletionPort( + reinterpret_cast(sock), static_cast(iocp_), key_io, 0); + if (result == nullptr) + return make_err(::GetLastError()); + + impl.close_socket(); + impl.socket_ = sock; + impl.family_ = proto_info.iAddressFamily; + + // AcceptEx sizes its address buffers from this cache, so an + // unseeded endpoint breaks accepts on an adopted v6 listener. + sockaddr_storage local_storage{}; + int local_len = sizeof(local_storage); + if (::getsockname( + sock, reinterpret_cast(&local_storage), &local_len) == 0) + impl.set_local_endpoint(detail::from_sockaddr(local_storage)); + + return {}; +} + +inline std::error_code +win_tcp_service::bind_acceptor(win_tcp_acceptor& impl, endpoint ep) +{ + SOCKET sock = impl.socket_; + + sockaddr_storage storage{}; + socklen_t addrlen = detail::to_sockaddr(ep, storage); + if (::bind( + sock, reinterpret_cast(&storage), + static_cast(addrlen)) == SOCKET_ERROR) + return make_err(::WSAGetLastError()); + + // Cache local endpoint (resolves ephemeral port) + sockaddr_storage local_storage{}; + int local_len = sizeof(local_storage); + if (::getsockname( + sock, reinterpret_cast(&local_storage), &local_len) == 0) + impl.set_local_endpoint(detail::from_sockaddr(local_storage)); + + return {}; +} + +inline std::error_code +win_tcp_service::listen_acceptor(win_tcp_acceptor& impl, int backlog) +{ + SOCKET sock = impl.socket_; + + if (::listen(sock, backlog) == SOCKET_ERROR) + return make_err(::WSAGetLastError()); + + return {}; +} + } // namespace boost::corosio::detail #endif // BOOST_COROSIO_HAS_IOCP diff --git a/include/boost/corosio/native/detail/iocp/win_tcp_socket.hpp b/include/boost/corosio/native/detail/iocp/win_tcp_socket.hpp index 8b3f6ae67..2a8fd61f5 100644 --- a/include/boost/corosio/native/detail/iocp/win_tcp_socket.hpp +++ b/include/boost/corosio/native/detail/iocp/win_tcp_socket.hpp @@ -17,6 +17,7 @@ #if BOOST_COROSIO_HAS_IOCP #include +#include #include #include #include @@ -25,20 +26,18 @@ #include #include -#include #include namespace boost::corosio::detail { class win_tcp_service; -class win_tcp_socket_internal; +class win_tcp_socket; /** Connect operation state. */ struct connect_op : overlapped_op { - win_tcp_socket_internal& internal; - std::shared_ptr internal_ptr; + win_tcp_socket& internal; endpoint target_endpoint; static void do_complete( @@ -48,7 +47,7 @@ struct connect_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit connect_op(win_tcp_socket_internal& internal_) noexcept; + explicit connect_op(win_tcp_socket& internal_) noexcept; }; /** Read operation state with buffer descriptors. */ @@ -58,8 +57,7 @@ struct read_op : overlapped_op WSABUF wsabufs[max_buffers]; DWORD wsabuf_count = 0; DWORD flags = 0; - win_tcp_socket_internal& internal; - std::shared_ptr internal_ptr; + win_tcp_socket& internal; static void do_complete( void* owner, @@ -68,7 +66,7 @@ struct read_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit read_op(win_tcp_socket_internal& internal_) noexcept; + explicit read_op(win_tcp_socket& internal_) noexcept; }; /** Write operation state with buffer descriptors. */ @@ -77,8 +75,7 @@ struct write_op : overlapped_op static constexpr std::size_t max_buffers = 16; WSABUF wsabufs[max_buffers]; DWORD wsabuf_count = 0; - win_tcp_socket_internal& internal; - std::shared_ptr internal_ptr; + win_tcp_socket& internal; static void do_complete( void* owner, @@ -87,7 +84,7 @@ struct write_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit write_op(win_tcp_socket_internal& internal_) noexcept; + explicit write_op(win_tcp_socket& internal_) noexcept; }; /** Readiness-wait operation state. @@ -103,8 +100,7 @@ struct wait_op : overlapped_op { WSABUF wsabuf{}; DWORD flags = 0; - win_tcp_socket_internal& internal; - std::shared_ptr internal_ptr; + win_tcp_socket& internal; static void do_complete( void* owner, @@ -113,23 +109,26 @@ struct wait_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit wait_op(win_tcp_socket_internal& internal_) noexcept; + explicit wait_op(win_tcp_socket& internal_) noexcept; }; -/** Internal socket state for IOCP-based I/O. +/** Socket implementation for IOCP-based I/O. - This class contains the actual state for a single socket, including - the native socket handle and pending operations. It derives from - enable_shared_from_this so operations can extend its lifetime. + Collapses the historical internal-state/wrapper split into one + pooled `io_object::implementation`: it both owns the native socket + handle and pending operations, and directly implements + `tcp_socket::implementation`. `win_tcp_service` recycles instances + through its pool instead of freeing them on every close, so the + keepalive each embedded op holds is the intrusive `object_ref` from + `coro_op`, not a `shared_ptr`. @note Internal implementation detail. Users interact with socket class. */ -class win_tcp_socket_internal - : public intrusive_list::node - , public std::enable_shared_from_this +class win_tcp_socket final + : public tcp_socket::implementation + , public intrusive_list::node { friend class win_tcp_service; - friend class win_tcp_socket; friend struct read_op; friend struct write_op; friend struct connect_op; @@ -142,73 +141,28 @@ class win_tcp_socket_internal wait_op wt_; SOCKET socket_ = INVALID_SOCKET; int family_ = AF_UNSPEC; - -public: - explicit win_tcp_socket_internal(win_tcp_service& svc) noexcept; - ~win_tcp_socket_internal(); - - std::coroutine_handle<> connect( - std::coroutine_handle<>, - capy::executor_ref, - endpoint, - std::stop_token, - std::error_code*); - - std::coroutine_handle<> read_some( - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - std::stop_token, - std::error_code*, - std::size_t*); - - std::coroutine_handle<> write_some( - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - std::stop_token, - std::error_code*, - std::size_t*); - - std::coroutine_handle<> wait( - std::coroutine_handle<>, - capy::executor_ref, - wait_type, - std::stop_token, - std::error_code*); - - SOCKET native_handle() const noexcept; - endpoint local_endpoint() const noexcept; - endpoint remote_endpoint() const noexcept; - bool is_open() const noexcept; - void cancel() noexcept; - void close_socket() noexcept; - void set_socket(SOCKET s) noexcept; - void set_endpoints(endpoint local, endpoint remote) noexcept; - -private: endpoint local_endpoint_; endpoint remote_endpoint_; -}; -/** Socket implementation wrapper for IOCP-based I/O. +public: + explicit win_tcp_socket(win_tcp_service& svc) noexcept; - This class is the public-facing implementation that holds a shared_ptr - to the internal state. The shared_ptr is hidden from the public interface. + /// Recycle into the owning service's pool. Defined out-of-line + /// after `win_tcp_service` is complete (needs `svc_.pool_`). + void retire() noexcept override; - @note Internal implementation detail. Users interact with socket class. -*/ -class win_tcp_socket final - : public tcp_socket::implementation - , public intrusive_list::node -{ - std::shared_ptr internal_; + /** Reset recycled state for reuse. -public: - explicit win_tcp_socket( - std::shared_ptr internal) noexcept; + `close_socket()` already drives the socket handle, family, + and cached endpoints to their closed values, and drops every + embedded op's `object_ref_` before `refs_` can reach zero, so + there is little left to reset here -- only to assert. Each + op's `OVERLAPPED` fields are re-seeded by `overlapped_op:: + reset()` at its next submit, so they are not zeroed here. - void close_internal() noexcept; + @pre refs_ == 0, socket closed, no op in flight. + */ + void reuse() noexcept; std::coroutine_handle<> connect( std::coroutine_handle<> h, @@ -248,7 +202,7 @@ class win_tcp_socket final { // getsockname fails with WSAEINVAL on an unbound socket, so // the family recorded at socket creation is authoritative - return to_family(internal_ ? internal_->family_ : AF_UNSPEC); + return to_family(family_); } native_handle_type release_socket() noexcept override; @@ -266,7 +220,11 @@ class win_tcp_socket final endpoint remote_endpoint() const noexcept override; void cancel() noexcept override; - win_tcp_socket_internal* get_internal() const noexcept; + /// Used by the acceptor's `accept_op` to finish setting up the peer. + void set_socket(SOCKET s) noexcept; + void set_endpoints(endpoint local, endpoint remote) noexcept; + + void close_socket() noexcept; }; } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/iocp/win_udp_service.hpp b/include/boost/corosio/native/detail/iocp/win_udp_service.hpp index 280d0c5be..f89120bfc 100644 --- a/include/boost/corosio/native/detail/iocp/win_udp_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_udp_service.hpp @@ -15,13 +15,13 @@ #if BOOST_COROSIO_HAS_IOCP #include +#include #include #include #include #include #include -#include #include #include @@ -44,6 +44,8 @@ class BOOST_COROSIO_DECL win_udp_service final : private win_wsa_init , public udp_service { + friend class win_udp_socket; + public: io_object::implementation* construct() override; @@ -53,8 +55,6 @@ class BOOST_COROSIO_DECL win_udp_service final explicit win_udp_service(capy::execution_context& ctx); - ~win_udp_service(); - win_udp_service(win_udp_service const&) = delete; win_udp_service& operator=(win_udp_service const&) = delete; @@ -70,12 +70,8 @@ class BOOST_COROSIO_DECL win_udp_service final std::error_code bind_datagram(udp_socket::implementation& impl, endpoint ep) override; - void destroy_impl(win_udp_socket& impl); - - void unregister_impl(win_udp_socket_internal& impl); - std::error_code open_socket( - win_udp_socket_internal& impl, int family, int type, int protocol); + win_udp_socket& impl, int family, int type, int protocol); void post(overlapped_op* op); void on_pending(overlapped_op* op) noexcept; @@ -91,54 +87,51 @@ class BOOST_COROSIO_DECL win_udp_service final private: win_scheduler& sched_; + void* iocp_; BOOST_COROSIO_MSVC_WARNING_PUSH BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // detail:: members, dll-interface - win_mutex mutex_; - intrusive_list socket_list_; - intrusive_list wrapper_list_; + object_pool pool_; BOOST_COROSIO_MSVC_WARNING_POP - void* iocp_; }; // Operation constructors -inline send_to_op::send_to_op(win_udp_socket_internal& internal_) noexcept +inline send_to_op::send_to_op(win_udp_socket& internal_) noexcept : overlapped_op(&do_complete) , internal(internal_) { cancel_func_ = &do_cancel_impl; } -inline recv_from_op::recv_from_op(win_udp_socket_internal& internal_) noexcept +inline recv_from_op::recv_from_op(win_udp_socket& internal_) noexcept : overlapped_op(&do_complete) , internal(internal_) { cancel_func_ = &do_cancel_impl; } -inline udp_connect_op::udp_connect_op( - win_udp_socket_internal& internal_) noexcept +inline udp_connect_op::udp_connect_op(win_udp_socket& internal_) noexcept : overlapped_op(&do_complete) , internal(internal_) { cancel_func_ = &do_cancel_impl; } -inline udp_send_op::udp_send_op(win_udp_socket_internal& internal_) noexcept +inline udp_send_op::udp_send_op(win_udp_socket& internal_) noexcept : overlapped_op(&do_complete) , internal(internal_) { cancel_func_ = &do_cancel_impl; } -inline udp_recv_op::udp_recv_op(win_udp_socket_internal& internal_) noexcept +inline udp_recv_op::udp_recv_op(win_udp_socket& internal_) noexcept : overlapped_op(&do_complete) , internal(internal_) { cancel_func_ = &do_cancel_impl; } -inline udp_wait_op::udp_wait_op(win_udp_socket_internal& internal_) noexcept +inline udp_wait_op::udp_wait_op(win_udp_socket& internal_) noexcept : overlapped_op(&do_complete) , internal(internal_) { @@ -152,10 +145,9 @@ send_to_op::do_cancel_impl(overlapped_op* base) noexcept { auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) + if (op->internal.socket_ != INVALID_SOCKET) { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); } } @@ -164,10 +156,9 @@ recv_from_op::do_cancel_impl(overlapped_op* base) noexcept { auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) + if (op->internal.socket_ != INVALID_SOCKET) { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); } } @@ -185,11 +176,11 @@ send_to_op::do_complete( if (!owner) { op->cleanup_only(); - op->internal_ptr.reset(); + op->object_ref_.reset(); return; } - auto prevent_premature_destruction = std::move(op->internal_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } @@ -207,7 +198,7 @@ recv_from_op::do_complete( if (!owner) { op->cleanup_only(); - op->internal_ptr.reset(); + op->object_ref_.reset(); return; } @@ -219,7 +210,7 @@ recv_from_op::do_complete( *op->source_out = from_sockaddr(op->source_storage); } - auto prevent_premature_destruction = std::move(op->internal_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } @@ -237,10 +228,9 @@ udp_send_op::do_cancel_impl(overlapped_op* base) noexcept { auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) + if (op->internal.socket_ != INVALID_SOCKET) { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); } } @@ -249,10 +239,9 @@ udp_recv_op::do_cancel_impl(overlapped_op* base) noexcept { auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) + if (op->internal.socket_ != INVALID_SOCKET) { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); } } @@ -261,10 +250,9 @@ udp_wait_op::do_cancel_impl(overlapped_op* base) noexcept { auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) + if (op->internal.socket_ != INVALID_SOCKET) { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); } op->internal.svc_.scheduler().cancel_wait(op); } @@ -283,7 +271,7 @@ udp_connect_op::do_complete( if (!owner) { op->cleanup_only(); - op->internal_ptr.reset(); + op->object_ref_.reset(); return; } @@ -301,7 +289,7 @@ udp_connect_op::do_complete( op->internal.remote_endpoint_ = op->target_endpoint; } - auto prevent_premature_destruction = std::move(op->internal_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } @@ -317,11 +305,11 @@ udp_send_op::do_complete( if (!owner) { op->cleanup_only(); - op->internal_ptr.reset(); + op->object_ref_.reset(); return; } - auto prevent_premature_destruction = std::move(op->internal_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } @@ -337,11 +325,11 @@ udp_recv_op::do_complete( if (!owner) { op->cleanup_only(); - op->internal_ptr.reset(); + op->object_ref_.reset(); return; } - auto prevent_premature_destruction = std::move(op->internal_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } @@ -357,18 +345,17 @@ udp_wait_op::do_complete( if (!owner) { op->cleanup_only(); - op->internal_ptr.reset(); + op->object_ref_.reset(); return; } - auto prevent_premature_destruction = std::move(op->internal_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } -// win_udp_socket_internal +// win_udp_socket -inline win_udp_socket_internal::win_udp_socket_internal( - win_udp_service& svc) noexcept +inline win_udp_socket::win_udp_socket(win_udp_service& svc) noexcept : svc_(svc) , wr_(*this) , rd_(*this) @@ -379,37 +366,41 @@ inline win_udp_socket_internal::win_udp_socket_internal( { } -inline win_udp_socket_internal::~win_udp_socket_internal() +inline void +win_udp_socket::reuse() noexcept { - svc_.unregister_impl(*this); + BOOST_COROSIO_ASSERT(socket_ == INVALID_SOCKET); + BOOST_COROSIO_ASSERT(family_ == AF_UNSPEC); + BOOST_COROSIO_ASSERT(local_endpoint_ == endpoint{}); + BOOST_COROSIO_ASSERT(remote_endpoint_ == endpoint{}); + BOOST_COROSIO_ASSERT(!wr_.stop_cb); + BOOST_COROSIO_ASSERT(!rd_.stop_cb); + BOOST_COROSIO_ASSERT(!conn_.stop_cb); + BOOST_COROSIO_ASSERT(!send_wr_.stop_cb); + BOOST_COROSIO_ASSERT(!recv_rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wt_.stop_cb); } -inline SOCKET -win_udp_socket_internal::native_handle() const noexcept +inline native_handle_type +win_udp_socket::native_handle() const noexcept { - return socket_; + return static_cast(socket_); } inline endpoint -win_udp_socket_internal::local_endpoint() const noexcept +win_udp_socket::local_endpoint() const noexcept { return local_endpoint_; } inline endpoint -win_udp_socket_internal::remote_endpoint() const noexcept +win_udp_socket::remote_endpoint() const noexcept { return remote_endpoint_; } -inline bool -win_udp_socket_internal::is_open() const noexcept -{ - return socket_ != INVALID_SOCKET; -} - inline std::coroutine_handle<> -win_udp_socket_internal::send_to( +win_udp_socket::send_to( std::coroutine_handle<> h, capy::executor_ref d, buffer_param param, @@ -419,8 +410,8 @@ win_udp_socket_internal::send_to( std::error_code* ec, std::size_t* bytes_out) { - // Keep internal alive during I/O - wr_.internal_ptr = shared_from_this(); + // Keep this socket alive during I/O + wr_.object_ref_ = detail::object_ref(this); auto& op = wr_; op.reset(); @@ -472,7 +463,7 @@ win_udp_socket_internal::send_to( } inline std::coroutine_handle<> -win_udp_socket_internal::recv_from( +win_udp_socket::recv_from( std::coroutine_handle<> h, capy::executor_ref d, buffer_param param, @@ -482,8 +473,8 @@ win_udp_socket_internal::recv_from( std::error_code* ec, std::size_t* bytes_out) { - // Keep internal alive during I/O - rd_.internal_ptr = shared_from_this(); + // Keep this socket alive during I/O + rd_.object_ref_ = detail::object_ref(this); auto& op = rd_; op.reset(); @@ -545,14 +536,14 @@ win_udp_socket_internal::recv_from( // UDP connect is synchronous on Windows inline std::coroutine_handle<> -win_udp_socket_internal::connect( +win_udp_socket::connect( std::coroutine_handle<> h, capy::executor_ref d, endpoint ep, std::stop_token token, std::error_code* ec) { - conn_.internal_ptr = shared_from_this(); + conn_.object_ref_ = detail::object_ref(this); auto& op = conn_; op.reset(); @@ -579,7 +570,7 @@ win_udp_socket_internal::connect( } inline std::coroutine_handle<> -win_udp_socket_internal::send( +win_udp_socket::send( std::coroutine_handle<> h, capy::executor_ref d, buffer_param param, @@ -588,7 +579,7 @@ win_udp_socket_internal::send( std::error_code* ec, std::size_t* bytes_out) { - send_wr_.internal_ptr = shared_from_this(); + send_wr_.object_ref_ = detail::object_ref(this); auto& op = send_wr_; op.reset(); @@ -633,7 +624,7 @@ win_udp_socket_internal::send( } inline std::coroutine_handle<> -win_udp_socket_internal::recv( +win_udp_socket::recv( std::coroutine_handle<> h, capy::executor_ref d, buffer_param param, @@ -642,7 +633,7 @@ win_udp_socket_internal::recv( std::error_code* ec, std::size_t* bytes_out) { - recv_rd_.internal_ptr = shared_from_this(); + recv_rd_.object_ref_ = detail::object_ref(this); auto& op = recv_rd_; op.reset(); @@ -695,14 +686,14 @@ win_udp_socket_internal::recv( } inline std::coroutine_handle<> -win_udp_socket_internal::wait( +win_udp_socket::wait( std::coroutine_handle<> h, capy::executor_ref d, wait_type w, std::stop_token token, std::error_code* ec) { - wt_.internal_ptr = shared_from_this(); + wt_.object_ref_ = detail::object_ref(this); auto& op = wt_; op.reset(); @@ -733,7 +724,7 @@ win_udp_socket_internal::wait( } inline void -win_udp_socket_internal::cancel() noexcept +win_udp_socket::cancel() noexcept { if (socket_ != INVALID_SOCKET) { @@ -750,7 +741,7 @@ win_udp_socket_internal::cancel() noexcept } inline void -win_udp_socket_internal::close_socket() noexcept +win_udp_socket::close_socket() noexcept { // Flag every op cancelled before closing so a closesocket-delivered // ERROR_NETNAME_DELETED is short-circuited to canceled rather than mapped @@ -776,106 +767,6 @@ win_udp_socket_internal::close_socket() noexcept remote_endpoint_ = endpoint{}; } -// win_udp_socket - -inline win_udp_socket::win_udp_socket( - std::shared_ptr internal) noexcept - : internal_(std::move(internal)) -{ -} - -inline void -win_udp_socket::close_internal() noexcept -{ - if (internal_) - { - internal_->close_socket(); - internal_.reset(); - } -} - -inline std::coroutine_handle<> -win_udp_socket::send_to( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - endpoint dest, - int flags, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->send_to(h, d, buf, dest, flags, token, ec, bytes); -} - -inline std::coroutine_handle<> -win_udp_socket::recv_from( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - endpoint* source, - int flags, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->recv_from(h, d, buf, source, flags, token, ec, bytes); -} - -inline std::coroutine_handle<> -win_udp_socket::connect( - std::coroutine_handle<> h, - capy::executor_ref d, - endpoint ep, - std::stop_token token, - std::error_code* ec) -{ - return internal_->connect(h, d, ep, token, ec); -} - -inline std::coroutine_handle<> -win_udp_socket::send( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - int flags, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->send(h, d, buf, flags, token, ec, bytes); -} - -inline std::coroutine_handle<> -win_udp_socket::recv( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - int flags, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->recv(h, d, buf, flags, token, ec, bytes); -} - -inline std::coroutine_handle<> -win_udp_socket::wait( - std::coroutine_handle<> h, - capy::executor_ref d, - wait_type w, - std::stop_token token, - std::error_code* ec) -{ - return internal_->wait(h, d, w, token, ec); -} - -inline native_handle_type -win_udp_socket::native_handle() const noexcept -{ - return static_cast(internal_->native_handle()); -} - inline std::error_code win_udp_socket::shutdown(udp_socket::shutdown_type what) noexcept { @@ -894,7 +785,7 @@ win_udp_socket::shutdown(udp_socket::shutdown_type what) noexcept default: return make_err(WSAEINVAL); } - if (::shutdown(internal_->native_handle(), how) != 0) + if (::shutdown(socket_, how) != 0) return make_err(WSAGetLastError()); return {}; } @@ -902,15 +793,15 @@ win_udp_socket::shutdown(udp_socket::shutdown_type what) noexcept inline native_handle_type win_udp_socket::release_socket() noexcept { - SOCKET s = internal_->socket_; + SOCKET s = socket_; if (s != INVALID_SOCKET) { - internal_->cancel(); + cancel(); dissociate_from_iocp(s); - internal_->socket_ = INVALID_SOCKET; - internal_->family_ = AF_UNSPEC; - internal_->local_endpoint_ = endpoint{}; - internal_->remote_endpoint_ = endpoint{}; + socket_ = INVALID_SOCKET; + family_ = AF_UNSPEC; + local_endpoint_ = endpoint{}; + remote_endpoint_ = endpoint{}; } return static_cast(s); } @@ -920,8 +811,8 @@ win_udp_socket::set_option( int level, int optname, void const* data, std::size_t size) noexcept { if (::setsockopt( - internal_->native_handle(), level, optname, - reinterpret_cast(data), static_cast(size)) != 0) + socket_, level, optname, reinterpret_cast(data), + static_cast(size)) != 0) return make_err(WSAGetLastError()); return {}; } @@ -932,35 +823,16 @@ win_udp_socket::get_option( { int len = static_cast(*size); if (::getsockopt( - internal_->native_handle(), level, optname, - reinterpret_cast(data), &len) != 0) + socket_, level, optname, reinterpret_cast(data), &len) != 0) return make_err(WSAGetLastError()); *size = static_cast(len); return {}; } -inline endpoint -win_udp_socket::local_endpoint() const noexcept -{ - return internal_->local_endpoint(); -} - -inline endpoint -win_udp_socket::remote_endpoint() const noexcept -{ - return internal_->remote_endpoint(); -} - inline void -win_udp_socket::cancel() noexcept +win_udp_socket::retire() noexcept { - internal_->cancel(); -} - -inline win_udp_socket_internal* -win_udp_socket::get_internal() const noexcept -{ - return internal_.get(); + svc_.pool_.recycle(this); } // win_udp_service @@ -971,83 +843,37 @@ inline win_udp_service::win_udp_service(capy::execution_context& ctx) { } -inline win_udp_service::~win_udp_service() -{ - for (auto* w = wrapper_list_.pop_front(); w != nullptr; - w = wrapper_list_.pop_front()) - delete w; -} - inline void win_udp_service::shutdown() { - std::lock_guard lock(mutex_); - - for (auto* impl = socket_list_.pop_front(); impl != nullptr; - impl = socket_list_.pop_front()) - { - impl->close_socket(); - } + pool_.shutdown([](win_udp_socket* impl) { impl->close_socket(); }); } inline io_object::implementation* win_udp_service::construct() { - auto internal = std::make_shared(*this); - - { - std::lock_guard lock(mutex_); - socket_list_.push_back(internal.get()); - } - - auto* wrapper = new win_udp_socket(std::move(internal)); - - { - std::lock_guard lock(mutex_); - wrapper_list_.push_back(wrapper); - } - - return wrapper; + return pool_.acquire(*this); } inline void win_udp_service::destroy(io_object::implementation* p) { - if (p) - { - auto& wrapper = static_cast(*p); - wrapper.close_internal(); - destroy_impl(wrapper); - } + if (!p) + return; + auto* s = static_cast(p); + s->close_socket(); + release(s); } inline void win_udp_service::close(io_object::handle& h) { - auto& wrapper = static_cast(*h.get()); - wrapper.get_internal()->close_socket(); -} - -inline void -win_udp_service::destroy_impl(win_udp_socket& impl) -{ - { - std::lock_guard lock(mutex_); - wrapper_list_.remove(&impl); - } - delete &impl; -} - -inline void -win_udp_service::unregister_impl(win_udp_socket_internal& impl) -{ - std::lock_guard lock(mutex_); - socket_list_.remove(&impl); + static_cast(h.get())->close_socket(); } inline std::error_code win_udp_service::open_socket( - win_udp_socket_internal& impl, int family, int type, int protocol) + win_udp_socket& impl, int family, int type, int protocol) { impl.close_socket(); @@ -1084,21 +910,20 @@ inline std::error_code win_udp_service::open_datagram_socket( udp_socket::implementation& impl, int family, int type, int protocol) { - auto& wrapper = static_cast(impl); - return open_socket(*wrapper.get_internal(), family, type, protocol); + auto& sock = static_cast(impl); + return open_socket(sock, family, type, protocol); } inline std::error_code win_udp_service::assign_socket( udp_socket::implementation& impl, native_handle_type fd) { - auto& wrapper = static_cast(impl); - auto* internal = wrapper.get_internal(); + auto& sock = static_cast(impl); - SOCKET sock = static_cast(fd); - if (sock == INVALID_SOCKET) + SOCKET s = static_cast(fd); + if (s == INVALID_SOCKET) return make_err(WSAENOTSOCK); - if (sock == internal->socket_) + if (s == sock.socket_) return std::make_error_code(std::errc::invalid_argument); // SO_PROTOCOL_INFOW works on an unbound socket, unlike getsockname @@ -1106,7 +931,7 @@ win_udp_service::assign_socket( WSAPROTOCOL_INFOW proto_info{}; int proto_len = sizeof(proto_info); if (::getsockopt( - sock, SOL_SOCKET, SO_PROTOCOL_INFOW, + s, SOL_SOCKET, SO_PROTOCOL_INFOW, reinterpret_cast(&proto_info), &proto_len) != 0) return make_err(::WSAGetLastError()); if (proto_info.iAddressFamily != AF_INET && @@ -1119,28 +944,27 @@ win_udp_service::assign_socket( // shares descriptor state the way the reactor path does, so a // failed association must not cost the caller their old socket. HANDLE result = ::CreateIoCompletionPort( - reinterpret_cast(sock), static_cast(iocp_), key_io, 0); + reinterpret_cast(s), static_cast(iocp_), key_io, 0); if (result == nullptr) return make_err(::GetLastError()); - internal->close_socket(); - internal->socket_ = sock; - internal->family_ = proto_info.iAddressFamily; + sock.close_socket(); + sock.socket_ = s; + sock.family_ = proto_info.iAddressFamily; endpoint local_ep, remote_ep; sockaddr_storage local_storage{}; int local_len = sizeof(local_storage); if (::getsockname( - sock, reinterpret_cast(&local_storage), &local_len) == 0) + s, reinterpret_cast(&local_storage), &local_len) == 0) local_ep = detail::from_sockaddr(local_storage); sockaddr_storage remote_storage{}; int remote_len = sizeof(remote_storage); if (::getpeername( - sock, reinterpret_cast(&remote_storage), &remote_len) == - 0) + s, reinterpret_cast(&remote_storage), &remote_len) == 0) remote_ep = detail::from_sockaddr(remote_storage); - internal->local_endpoint_ = local_ep; - internal->remote_endpoint_ = remote_ep; + sock.local_endpoint_ = local_ep; + sock.remote_endpoint_ = remote_ep; return {}; } @@ -1148,14 +972,13 @@ win_udp_service::assign_socket( inline std::error_code win_udp_service::bind_datagram(udp_socket::implementation& impl, endpoint ep) { - auto& wrapper = static_cast(impl); - auto* internal = wrapper.get_internal(); - SOCKET sock = internal->socket_; + auto& sock = static_cast(impl); + SOCKET s = sock.socket_; sockaddr_storage storage{}; socklen_t addrlen = detail::to_sockaddr(ep, storage); if (::bind( - sock, reinterpret_cast(&storage), + s, reinterpret_cast(&storage), static_cast(addrlen)) == SOCKET_ERROR) return make_err(::WSAGetLastError()); @@ -1163,8 +986,8 @@ win_udp_service::bind_datagram(udp_socket::implementation& impl, endpoint ep) sockaddr_storage local_storage{}; int local_len = sizeof(local_storage); if (::getsockname( - sock, reinterpret_cast(&local_storage), &local_len) == 0) - internal->local_endpoint_ = detail::from_sockaddr(local_storage); + s, reinterpret_cast(&local_storage), &local_len) == 0) + sock.local_endpoint_ = detail::from_sockaddr(local_storage); return {}; } diff --git a/include/boost/corosio/native/detail/iocp/win_udp_socket.hpp b/include/boost/corosio/native/detail/iocp/win_udp_socket.hpp index 9679265d6..e489d58ab 100644 --- a/include/boost/corosio/native/detail/iocp/win_udp_socket.hpp +++ b/include/boost/corosio/native/detail/iocp/win_udp_socket.hpp @@ -16,6 +16,7 @@ #if BOOST_COROSIO_HAS_IOCP #include +#include #include #include #include @@ -24,12 +25,11 @@ #include #include -#include namespace boost::corosio::detail { class win_udp_service; -class win_udp_socket_internal; +class win_udp_socket; /** Send-to operation state with buffer and destination address. */ struct send_to_op : overlapped_op @@ -39,8 +39,7 @@ struct send_to_op : overlapped_op DWORD wsabuf_count = 0; sockaddr_storage dest_storage{}; int dest_len = 0; - win_udp_socket_internal& internal; - std::shared_ptr internal_ptr; + win_udp_socket& internal; static void do_complete( void* owner, @@ -49,7 +48,7 @@ struct send_to_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit send_to_op(win_udp_socket_internal& internal_) noexcept; + explicit send_to_op(win_udp_socket& internal_) noexcept; }; /** Recv-from operation state with buffer and source address capture. */ @@ -62,8 +61,7 @@ struct recv_from_op : overlapped_op sockaddr_storage source_storage{}; INT source_len = sizeof(sockaddr_storage); endpoint* source_out = nullptr; - win_udp_socket_internal& internal; - std::shared_ptr internal_ptr; + win_udp_socket& internal; static void do_complete( void* owner, @@ -72,15 +70,14 @@ struct recv_from_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit recv_from_op(win_udp_socket_internal& internal_) noexcept; + explicit recv_from_op(win_udp_socket& internal_) noexcept; }; /** Connect operation for connected-mode UDP on IOCP. */ struct udp_connect_op : overlapped_op { endpoint target_endpoint; - win_udp_socket_internal& internal; - std::shared_ptr internal_ptr; + win_udp_socket& internal; static void do_complete( void* owner, @@ -89,7 +86,7 @@ struct udp_connect_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit udp_connect_op(win_udp_socket_internal& internal_) noexcept; + explicit udp_connect_op(win_udp_socket& internal_) noexcept; }; /** Connected send operation (no destination address). */ @@ -98,8 +95,7 @@ struct udp_send_op : overlapped_op static constexpr std::size_t max_buffers = 16; WSABUF wsabufs[max_buffers]; DWORD wsabuf_count = 0; - win_udp_socket_internal& internal; - std::shared_ptr internal_ptr; + win_udp_socket& internal; static void do_complete( void* owner, @@ -108,7 +104,7 @@ struct udp_send_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit udp_send_op(win_udp_socket_internal& internal_) noexcept; + explicit udp_send_op(win_udp_socket& internal_) noexcept; }; /** Connected recv operation (no source address). */ @@ -118,8 +114,7 @@ struct udp_recv_op : overlapped_op WSABUF wsabufs[max_buffers]; DWORD wsabuf_count = 0; DWORD flags = 0; - win_udp_socket_internal& internal; - std::shared_ptr internal_ptr; + win_udp_socket& internal; static void do_complete( void* owner, @@ -128,14 +123,13 @@ struct udp_recv_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit udp_recv_op(win_udp_socket_internal& internal_) noexcept; + explicit udp_recv_op(win_udp_socket& internal_) noexcept; }; /** Readiness-wait operation (datagram socket). */ struct udp_wait_op : overlapped_op { - win_udp_socket_internal& internal; - std::shared_ptr internal_ptr; + win_udp_socket& internal; static void do_complete( void* owner, @@ -144,24 +138,24 @@ struct udp_wait_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit udp_wait_op(win_udp_socket_internal& internal_) noexcept; + explicit udp_wait_op(win_udp_socket& internal_) noexcept; }; -/** Internal datagram socket state for IOCP-based I/O. +/** Datagram socket implementation for IOCP-based I/O. - This class contains the actual state for a single UDP socket, - including the native socket handle and pending operations. It - derives from enable_shared_from_this so operations can extend - its lifetime. + Collapses the historical internal-state/wrapper split into one + pooled `io_object::implementation`: it both owns the native socket + handle and pending operations, and directly implements + `udp_socket::implementation`. `win_udp_service` recycles instances + through its pool instead of freeing them on every close. @note Internal implementation detail. */ -class win_udp_socket_internal - : public intrusive_list::node - , public std::enable_shared_from_this +class win_udp_socket final + : public udp_socket::implementation + , public intrusive_list::node { friend class win_udp_service; - friend class win_udp_socket; friend struct send_to_op; friend struct recv_from_op; friend struct udp_connect_op; @@ -178,94 +172,25 @@ class win_udp_socket_internal udp_wait_op wt_; SOCKET socket_ = INVALID_SOCKET; int family_ = AF_UNSPEC; - -public: - explicit win_udp_socket_internal(win_udp_service& svc) noexcept; - ~win_udp_socket_internal(); - - std::coroutine_handle<> send_to( - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - endpoint, - int flags, - std::stop_token, - std::error_code*, - std::size_t*); - - std::coroutine_handle<> recv_from( - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - endpoint*, - int flags, - std::stop_token, - std::error_code*, - std::size_t*); - - std::coroutine_handle<> connect( - std::coroutine_handle<>, - capy::executor_ref, - endpoint, - std::stop_token, - std::error_code*); - - std::coroutine_handle<> send( - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - int flags, - std::stop_token, - std::error_code*, - std::size_t*); - - std::coroutine_handle<> recv( - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - int flags, - std::stop_token, - std::error_code*, - std::size_t*); - - std::coroutine_handle<> wait( - std::coroutine_handle<>, - capy::executor_ref, - wait_type, - std::stop_token, - std::error_code*); - - SOCKET native_handle() const noexcept; - endpoint local_endpoint() const noexcept; - endpoint remote_endpoint() const noexcept; - bool is_open() const noexcept; - void cancel() noexcept; - void close_socket() noexcept; - -private: endpoint local_endpoint_; endpoint remote_endpoint_; -}; -/** Datagram socket implementation wrapper for IOCP-based I/O. +public: + explicit win_udp_socket(win_udp_service& svc) noexcept; - This class is the public-facing implementation that holds a - shared_ptr to the internal state. The shared_ptr is hidden - from the public interface. + /// Recycle into the owning service's pool. Defined out-of-line + /// after `win_udp_service` is complete (needs `svc_.pool_`). + void retire() noexcept override; - @note Internal implementation detail. -*/ -class win_udp_socket final - : public udp_socket::implementation - , public intrusive_list::node -{ - std::shared_ptr internal_; + /** Reset recycled state for reuse. -public: - explicit win_udp_socket( - std::shared_ptr internal) noexcept; + Each op's `OVERLAPPED` fields are re-seeded by + `overlapped_op::reset()` at its next submit, so they are not + zeroed here. - void close_internal() noexcept; + @pre refs_ == 0, socket closed, no op in flight. + */ + void reuse() noexcept; std::coroutine_handle<> send_to( std::coroutine_handle<> h, @@ -325,7 +250,7 @@ class win_udp_socket final { // getsockname fails with WSAEINVAL on an unbound socket, so // the family recorded at socket creation is authoritative - return to_family(internal_ ? internal_->family_ : AF_UNSPEC); + return to_family(family_); } std::error_code shutdown(udp_socket::shutdown_type what) noexcept override; @@ -345,7 +270,7 @@ class win_udp_socket final endpoint remote_endpoint() const noexcept override; void cancel() noexcept override; - win_udp_socket_internal* get_internal() const noexcept; + void close_socket() noexcept; }; } // namespace boost::corosio::detail diff --git a/src/corosio/src/tcp_socket.cpp b/src/corosio/src/tcp_socket.cpp index f01d9ae15..1c6b08666 100644 --- a/src/corosio/src/tcp_socket.cpp +++ b/src/corosio/src/tcp_socket.cpp @@ -59,10 +59,8 @@ tcp_socket::open_for_family(int family, int type, int protocol) noexcept { #if BOOST_COROSIO_HAS_IOCP auto& svc = static_cast(h_.service()); - auto& wrapper = static_cast(*h_.get()); - std::error_code ec = svc.open_socket( - *static_cast(wrapper).get_internal(), family, - type, protocol); + auto& sock = static_cast(*h_.get()); + std::error_code ec = svc.open_socket(sock, family, type, protocol); #else auto& svc = static_cast(h_.service()); std::error_code ec = svc.open_socket( @@ -77,9 +75,8 @@ tcp_socket::assign(native_handle_type fd) noexcept { #if BOOST_COROSIO_HAS_IOCP auto& svc = static_cast(h_.service()); - auto& wrapper = static_cast(*h_.get()); - std::error_code ec = svc.assign_socket( - *static_cast(wrapper).get_internal(), fd); + auto& sock = static_cast(*h_.get()); + std::error_code ec = svc.assign_socket(sock, fd); #else auto& svc = static_cast(h_.service()); std::error_code ec = svc.assign_socket( @@ -104,10 +101,9 @@ tcp_socket::bind(endpoint ep) noexcept if (!is_open()) return make_error_code(std::errc::bad_file_descriptor); #if BOOST_COROSIO_HAS_IOCP - auto& svc = static_cast(h_.service()); - auto& wrapper = static_cast(*h_.get()); - return svc.bind_socket( - *static_cast(wrapper).get_internal(), ep); + auto& svc = static_cast(h_.service()); + auto& sock = static_cast(*h_.get()); + return svc.bind_socket(sock, ep); #else auto& svc = static_cast(h_.service()); return svc.bind_socket( From 15ae22d21fa6f724061a64b5937237a9a3181cce Mon Sep 17 00:00:00 2001 From: Steve Gerbino Date: Wed, 7 Oct 2026 23:49:12 +0200 Subject: [PATCH 04/12] refactor: unify timer recycling onto object_pool --- include/boost/corosio/detail/timer.hpp | 31 +++++- .../boost/corosio/detail/timer_service.hpp | 100 ++++++++++-------- include/boost/corosio/io/io_object.hpp | 34 +++--- src/corosio/src/timer.cpp | 27 ++++- 4 files changed, 131 insertions(+), 61 deletions(-) diff --git a/include/boost/corosio/detail/timer.hpp b/include/boost/corosio/detail/timer.hpp index 332943fac..0f10f8363 100644 --- a/include/boost/corosio/detail/timer.hpp +++ b/include/boost/corosio/detail/timer.hpp @@ -80,7 +80,9 @@ class BOOST_COROSIO_DECL timer : public io_object so `wait` is a plain member function rather than a virtual dispatch point. */ - struct implementation : io_object::implementation + struct implementation + : io_object::implementation + , intrusive_list::node { /// Sentinel value indicating the timer is not in the heap. static constexpr std::size_t npos = @@ -119,12 +121,33 @@ class BOOST_COROSIO_DECL timer : public io_object /// The waiter published on this timer, or `nullptr`. waiter_node* waiter_ = nullptr; - /// Free list linkage, reused when this impl is recycled. - implementation* next_free_ = nullptr; - /// Construct bound to the given timer service. explicit implementation(timer_service& svc) noexcept : svc_(&svc) {} + /** Reset recycled state to fresh-impl values. + + Both recycling tiers in `timer_service::construct()` — the + thread-local single-slot cache and the service's + `object_pool` fallback — call this, so the reset exists in + one place rather than duplicated per tier. + */ + void reuse() noexcept + { + expiry_ = {}; + heap_index_.store(npos, std::memory_order_relaxed); + might_have_pending_waits_.store( + false, std::memory_order_relaxed); + BOOST_COROSIO_ASSERT(waiter_ == nullptr); + } + + /** Recycle into the owning service's pool at zero references. + + Defined in timer.cpp: calling `svc_->pool_` needs + `timer_service`'s complete type, which this header only + forward-declares. + */ + void retire() noexcept override; + /** Check whether the timer is expired and absent from the heap. The single definition of the already-expired fast-path diff --git a/include/boost/corosio/detail/timer_service.hpp b/include/boost/corosio/detail/timer_service.hpp index 36b3dc3da..21d322be4 100644 --- a/include/boost/corosio/detail/timer_service.hpp +++ b/include/boost/corosio/detail/timer_service.hpp @@ -14,6 +14,8 @@ #include #include #include +#include +#include #include #include #include @@ -53,16 +55,18 @@ struct scheduler; list still threads waiters through their intrusive hooks when collecting several timers' waiters past the lock. - timer_service owns a min-heap of active timers and a free list - of recycled impls. The heap is ordered by expiry time; the - scheduler queries nearest_expiry() to set the epoll/timerfd - timeout. + timer_service owns a min-heap of active timers and recycles + retired impls through a per-service object_pool, fronted by a + thread-local single-slot cache. The heap is ordered by expiry + time; the scheduler queries nearest_expiry() to set the + epoll/timerfd timeout. Optimization Strategy --------------------- 1. Deferred heap insertion — expires_after() stores the expiry but does not insert into the heap. Insertion happens in wait(). - 2. Thread-local impl cache — single-slot per-thread cache. + 2. Thread-local impl cache — single-slot per-thread cache, in + front of the object_pool fallback. 3. Frame-resident waiter_node with embedded completion_op — eliminates heap allocation per wait/fire/cancel. 4. Cached nearest expiry — atomic avoids mutex in nearest_expiry(). @@ -79,12 +83,30 @@ struct scheduler; inline void timer_service_invalidate_cache() noexcept; +/** The timer service's recycling pool. + + Adds no behavior of its own — it exists only to re-expose + `adopt()` and `remove()` as public, so the timer service's + thread-local cache can transfer a cached impl's `live_` tracking + on both sides of the handoff: `construct()` re-adopts a + TL-cached impl as live, `shutdown()` detaches one before its + direct delete. +*/ +class timer_object_pool : public object_pool +{ +public: + using object_pool::adopt; + using object_pool::remove; +}; + // timer_service class body — member function definitions are // out-of-class (after implementation and waiter_node are complete) class BOOST_COROSIO_DECL timer_service final : public capy::execution_context::service , public io_object::io_service { + friend struct timer::implementation; + public: using clock_type = std::chrono::steady_clock; using time_point = clock_type::time_point; @@ -128,7 +150,7 @@ class BOOST_COROSIO_DECL timer_service final BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // std:: members, dll-interface mutable std::mutex mutex_; std::vector heap_; - timer::implementation* free_list_ = nullptr; + timer_object_pool pool_; callback on_earliest_changed_; bool shutting_down_ = false; // Avoids mutex in nearest_expiry() and empty() @@ -282,6 +304,9 @@ timer_service::shutdown() { timer_service_invalidate_cache(); shutting_down_ = true; + // Flag only; the real drain walks the expiry heap below, not + // object_pool's live_ list, so the visit callback is a no-op. + pool_.shutdown([](timer::implementation*) {}); // Snapshot impls and detach them from the heap so that // coroutine-owned timer destructors (triggered by h.destroy() @@ -316,16 +341,16 @@ timer_service::shutdown() if (h) h.destroy(); } + // Unlink from the pool's live_ list before the direct delete + // below, or ~object_pool()'s unconditional sweep would delete + // this impl a second time. + [[maybe_unused]] bool const removed = pool_.remove(impl); + BOOST_COROSIO_ASSERT(removed); delete impl; } - // Delete free-listed impls - while (free_list_) - { - auto* next = free_list_->next_free_; - delete free_list_; - free_list_ = next; - } + // Anything still parked in pool_ (free-listed or live with no + // pending wait at shutdown) is freed by object_pool's own destructor. } inline io_object::implementation* @@ -335,36 +360,16 @@ timer_service::construct() if (impl) { impl->svc_ = this; - // Reset expiry_ too: a recycled impl must behave like a fresh - // one, whose default expiry reads as already elapsed - impl->expiry_ = {}; - impl->heap_index_.store( - (std::numeric_limits::max)(), - std::memory_order_relaxed); - impl->might_have_pending_waits_.store(false, std::memory_order_relaxed); - BOOST_COROSIO_ASSERT(impl->waiter_ == nullptr); + impl->reuse(); + impl->refs_.store(1, std::memory_order_relaxed); + // The TL push that parked impl here also removed it from + // live_ (see destroy_impl) — re-adopt it now that it is live + // again. + pool_.adopt(impl); return impl; } - std::lock_guard lock(mutex_); - if (free_list_) - { - impl = free_list_; - free_list_ = impl->next_free_; - impl->next_free_ = nullptr; - impl->svc_ = this; - impl->expiry_ = {}; - impl->heap_index_.store( - (std::numeric_limits::max)(), - std::memory_order_relaxed); - impl->might_have_pending_waits_.store(false, std::memory_order_relaxed); - BOOST_COROSIO_ASSERT(impl->waiter_ == nullptr); - } - else - { - impl = new timer::implementation(*this); - } - return impl; + return pool_.acquire(*this); } inline void @@ -402,11 +407,20 @@ timer_service::destroy_impl(timer::implementation& impl) } if (try_push_tl_cache(&impl)) + { + // The TL slot takes sole ownership: it is deleted directly by + // tl_cache_owner / the stale-service path, never through this + // pool, so it must leave live_ now or ~object_pool() would see + // it as abandoned (and, at the wrong moment, double-delete it + // after a later construct() re-adopts it). + [[maybe_unused]] bool const removed = pool_.remove(&impl); + BOOST_COROSIO_ASSERT(removed); return; + } - std::lock_guard lock(mutex_); - impl.next_free_ = free_list_; - free_list_ = &impl; + // No op keepalives on a timer impl: this is the only reference, + // so release() always recycles immediately via retire(). + release(&impl); } inline void diff --git a/include/boost/corosio/io/io_object.hpp b/include/boost/corosio/io/io_object.hpp index 795d44dd4..d90f52a08 100644 --- a/include/boost/corosio/io/io_object.hpp +++ b/include/boost/corosio/io/io_object.hpp @@ -63,13 +63,20 @@ class BOOST_COROSIO_DECL io_object /// Destroy the implementation; called only through @ref io_service. virtual ~implementation() = default; - /** Called when the reference count reaches zero. - - @note Transitional default is a no-op while backends are - migrated; it becomes pure virtual at the end of the - migration. + /** Retire the implementation once the reference count reaches zero. + + Every concrete implementation's override follows the + same shape: recycle into the owning service's pool member + as the final statement. The service befriends the + implementation so it can reach that private member + directly. Any required last-rites cleanup — releasing + state that is only safe to tear down once idle — runs as + ordinary statements immediately before the recycle. + Whichever action runs — recycle or delete — must be the + final statement: nothing touches the implementation + after. */ - virtual void retire() noexcept {} + virtual void retire() noexcept = 0; /// In-flight + service references; starts at the service's 1. std::atomic refs_{1}; @@ -95,15 +102,15 @@ class BOOST_COROSIO_DECL io_object /** Close kernel resources and release the service's reference. Called whenever a handle relinquishes its current - implementation, not only on destruction: handle + implementation, not only on destruction. Handle destruction, move-assignment onto a handle that already owns one (the replaced implementation is destroyed, the incoming one is not), and `handle::reset()` all invoke - this. Closes the underlying descriptor and drops the - service's own reference; in-flight operations may still + this. It closes the underlying descriptor and drops the + service's own reference. In-flight operations may still hold references of their own, so the implementation is - not necessarily recycled or freed here — that happens in - `implementation::retire` whenever the refcount + not necessarily recycled or freed here. That happens in + `implementation::retire` whenever the reference count actually reaches zero. */ virtual void destroy(implementation* impl) = 0; @@ -198,7 +205,7 @@ class BOOST_COROSIO_DECL io_object one) — the old implementation is destroyed through that service. @pre @p p, if non-null, was constructed by this handle's - own service: the service that destroys it later must + own service. The service that destroys it later must be the one that knows how to close it. @param p The new implementation to own. May be nullptr. @@ -235,7 +242,8 @@ class BOOST_COROSIO_DECL io_object /** Create a handle bound to a service found in the context. - @tparam Service The service type whose key_type is used for lookup. + @tparam Service The service type whose `key_type` is used for + lookup. @param ctx The execution context to search for the service. @return A handle owning a freshly constructed implementation. diff --git a/src/corosio/src/timer.cpp b/src/corosio/src/timer.cpp index d28e9389b..5612045f0 100644 --- a/src/corosio/src/timer.cpp +++ b/src/corosio/src/timer.cpp @@ -20,6 +20,15 @@ timer::timer(capy::execution_context& ctx) { } +// Not inline for the same reason as wait() below: retire() needs +// timer_service's complete type (to reach pool_), which timer.hpp only +// forward-declares. +void +timer::implementation::retire() noexcept +{ + svc_->pool_.recycle(this); +} + // Not inline: wait_awaitable::await_suspend (defined in timer.hpp) calls // this from translation units that may never include timer_service.hpp, // so this must be the one strong definition the linker can always find @@ -60,7 +69,23 @@ timer::implementation::publish(waiter_node& w) if (w.token_->stop_possible()) w.arm_stop_cb(); - svc_->insert_waiter(*this, &w); + // insert_waiter() grows the heap before publishing anything (see + // its own comment), so a throw out of it leaves w.impl_ null and + // the waiter never visible to the fire/cancel paths above -- but + // work_started() and arm_stop_cb() already ran and must be + // unwound here, or the escaping exception leaves a dangling + // stop-callback registration (UAF on a later cancel) and a work + // count that never balances. + try + { + svc_->insert_waiter(*this, &w); + } + catch (...) + { + w.reset_stop_cb(); + svc_->get_scheduler().work_finished(); + throw; + } return std::noop_coroutine(); } From 26879b901a0d7e659072ee0d130ab55feec42879 Mon Sep 17 00:00:00 2001 From: Steve Gerbino Date: Wed, 7 Oct 2026 23:52:28 +0200 Subject: [PATCH 05/12] test: add recycled-impl poison, churn exercise, and zero-allocation gate --- .../detail/reactor/reactor_acceptor.hpp | 54 +- .../detail/reactor/reactor_basic_socket.hpp | 71 ++- .../reactor/reactor_datagram_socket.hpp | 21 +- .../detail/reactor/reactor_socket_finals.hpp | 9 + .../detail/reactor/reactor_stream_socket.hpp | 28 +- test/unit/Jamfile | 9 +- test/unit/alloc_counter.hpp | 85 ++++ test/unit/continuation_post.cpp | 179 ++++++- test/unit/object_reuse.cpp | 479 ++++++++++++++++++ test/unit/zero_alloc.cpp | 244 +++++++++ 10 files changed, 1127 insertions(+), 52 deletions(-) create mode 100644 test/unit/alloc_counter.hpp create mode 100644 test/unit/zero_alloc.cpp diff --git a/include/boost/corosio/native/detail/reactor/reactor_acceptor.hpp b/include/boost/corosio/native/detail/reactor/reactor_acceptor.hpp index ed7d745b8..69664c56f 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_acceptor.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_acceptor.hpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -260,21 +261,27 @@ class reactor_acceptor /** Reset descriptor state for recycling. - `close_socket()` already drove fd_ and every parked op pointer - to their closed state before the refcount reached zero, so - this only asserts those invariants — see - reactor_basic_socket::reuse() for the same rationale. + Actively re-initializes every field it owns rather than + trusting `close_socket()`'s prior writes to still be there — + see reactor_basic_socket::reuse() for the same rationale. + `object_ref_` and `is_enqueued_` are asserted instead: `poison()` + never touches them, so they must already hold their + zero-action-complete state. @pre refs_ == 0, fd closed and deregistered, no op in flight. */ void reuse() noexcept { - BOOST_COROSIO_ASSERT(fd_ == -1); - BOOST_COROSIO_ASSERT(desc_state_.fd == -1); - BOOST_COROSIO_ASSERT(desc_state_.read_op == nullptr); - BOOST_COROSIO_ASSERT(desc_state_.wait_read_op == nullptr); - BOOST_COROSIO_ASSERT(desc_state_.wait_write_op == nullptr); - BOOST_COROSIO_ASSERT(desc_state_.wait_error_op == nullptr); + fd_ = -1; + local_endpoint_ = Endpoint{}; + desc_state_.fd = -1; + desc_state_.registered_events = 0; + desc_state_.read_op = nullptr; + desc_state_.wait_read_op = nullptr; + desc_state_.wait_write_op = nullptr; + desc_state_.wait_error_op = nullptr; + desc_state_.read_ready = false; + desc_state_.write_ready = false; BOOST_COROSIO_ASSERT(!desc_state_.object_ref_); BOOST_COROSIO_ASSERT(!desc_state_.is_enqueued_.load( std::memory_order_relaxed)); @@ -282,9 +289,34 @@ class reactor_acceptor BOOST_COROSIO_ASSERT(!wait_rd_.stop_cb); BOOST_COROSIO_ASSERT(!wait_wr_.stop_cb); BOOST_COROSIO_ASSERT(!wait_er_.stop_cb); - BOOST_COROSIO_ASSERT(local_endpoint_ == Endpoint{}); } +#if !defined(NDEBUG) + /** Poison the fields `reuse()` re-initializes, before the pool + parks this impl on the free list. See + reactor_basic_socket::poison() for the full rationale and the + list of fields deliberately left untouched. + + @pre refs_ == 0, fd closed and deregistered, no op in flight. + */ + void poison() noexcept + { + auto smash = [](auto& field) { + std::memset(static_cast(&field), 0xDB, sizeof(field)); + }; + smash(fd_); + smash(local_endpoint_); + smash(desc_state_.fd); + smash(desc_state_.registered_events); + smash(desc_state_.read_op); + smash(desc_state_.wait_read_op); + smash(desc_state_.wait_write_op); + smash(desc_state_.wait_error_op); + smash(desc_state_.read_ready); + smash(desc_state_.write_ready); + } +#endif + /** Bind the acceptor socket to an endpoint. Caches the resolved local endpoint (including ephemeral diff --git a/include/boost/corosio/native/detail/reactor/reactor_basic_socket.hpp b/include/boost/corosio/native/detail/reactor/reactor_basic_socket.hpp index ba2b78279..eaf90bf16 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_basic_socket.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_basic_socket.hpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -179,29 +180,71 @@ class reactor_basic_socket /** Reset descriptor state for recycling. Called by the owning service's `construct()` when popping this - impl back off the free list. `close_socket()` already drove fd_, - the descriptor's registration, and every parked op pointer to - their closed state before the refcount reached zero, so this - only asserts those invariants rather than re-clearing them — - a non-null parked op or a lingering `object_ref_` here would mean - a reference survived close, which is the real bug to catch. + impl back off the free list. Actively re-initializes every + field it owns rather than trusting `close_socket()`'s prior + writes to still be there — `poison()` below depends on that, + and a future field added to either side without the other + would otherwise go unnoticed until this one re-sets it. + `object_ref_` and `is_enqueued_` are asserted instead: `poison()` + never touches them, so they must already hold their + zero-action-complete state. @pre refs_ == 0, fd closed and deregistered, no op in flight. */ void reuse() noexcept { - BOOST_COROSIO_ASSERT(fd_ == -1); - BOOST_COROSIO_ASSERT(desc_state_.fd == -1); - BOOST_COROSIO_ASSERT(desc_state_.read_op == nullptr); - BOOST_COROSIO_ASSERT(desc_state_.write_op == nullptr); - BOOST_COROSIO_ASSERT(desc_state_.connect_op == nullptr); - BOOST_COROSIO_ASSERT(desc_state_.wait_read_op == nullptr); - BOOST_COROSIO_ASSERT(desc_state_.wait_write_op == nullptr); - BOOST_COROSIO_ASSERT(desc_state_.wait_error_op == nullptr); + fd_ = -1; + local_endpoint_ = Endpoint{}; + desc_state_.fd = -1; + desc_state_.registered_events = 0; + desc_state_.read_op = nullptr; + desc_state_.write_op = nullptr; + desc_state_.connect_op = nullptr; + desc_state_.wait_read_op = nullptr; + desc_state_.wait_write_op = nullptr; + desc_state_.wait_error_op = nullptr; + desc_state_.read_ready = false; + desc_state_.write_ready = false; BOOST_COROSIO_ASSERT(!desc_state_.object_ref_); BOOST_COROSIO_ASSERT(!desc_state_.is_enqueued_.load( std::memory_order_relaxed)); } + +#if !defined(NDEBUG) + /** Poison the fields `reuse()` re-initializes, before the pool + parks this impl on the free list. + + A field `reuse()` forgets to reset then reads back as this + unmistakable byte pattern instead of silently matching + whatever `close_socket()` happened to leave, turning a missed + reset into an immediate, loud failure rather than a latent + stale-state bug. Does not touch the mutex, `ready_events_` / + `is_enqueued_` (atomics), `scheduler_`, `object_ref_`, the + `intrusive_list` hook, or `refs_`: `recycle()` still reads + some of those after this call returns, and smashing the rest + would be undefined rather than diagnostic. + + @pre refs_ == 0, fd closed and deregistered, no op in flight. + */ + void poison() noexcept + { + auto smash = [](auto& field) { + std::memset(static_cast(&field), 0xDB, sizeof(field)); + }; + smash(fd_); + smash(local_endpoint_); + smash(desc_state_.fd); + smash(desc_state_.registered_events); + smash(desc_state_.read_op); + smash(desc_state_.write_op); + smash(desc_state_.connect_op); + smash(desc_state_.wait_read_op); + smash(desc_state_.wait_write_op); + smash(desc_state_.wait_error_op); + smash(desc_state_.read_ready); + smash(desc_state_.write_ready); + } +#endif }; template< diff --git a/include/boost/corosio/native/detail/reactor/reactor_datagram_socket.hpp b/include/boost/corosio/native/detail/reactor/reactor_datagram_socket.hpp index ed43e0ebe..e861e09b4 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_datagram_socket.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_datagram_socket.hpp @@ -21,6 +21,7 @@ #include #include +#include #include #include @@ -329,16 +330,16 @@ class reactor_datagram_socket /** Reset op slots and the cached peer for recycling. See reactor_stream_socket::reuse() — same rationale: op state - is overwritten by the next do_* call, `remote_endpoint_` and - every parked op pointer are already cleared by - `do_close_socket()`, so this only asserts `stop_cb` disengaged - on each slot. + is overwritten by the next do_* call, so only `remote_endpoint_` + needs an active reset here; every other slot only asserts + `stop_cb` disengaged. @pre refs_ == 0, fd closed and deregistered, no op in flight. */ void reuse() noexcept { base_type::reuse(); + remote_endpoint_ = Endpoint{}; BOOST_COROSIO_ASSERT(!conn_.stop_cb); BOOST_COROSIO_ASSERT(!wr_.stop_cb); BOOST_COROSIO_ASSERT(!rd_.stop_cb); @@ -347,9 +348,19 @@ class reactor_datagram_socket BOOST_COROSIO_ASSERT(!wait_rd_.stop_cb); BOOST_COROSIO_ASSERT(!wait_wr_.stop_cb); BOOST_COROSIO_ASSERT(!wait_er_.stop_cb); - BOOST_COROSIO_ASSERT(remote_endpoint_ == Endpoint{}); } +#if !defined(NDEBUG) + /// Extend base_type::poison() to the cached peer endpoint. + void poison() noexcept + { + base_type::poison(); + std::memset( + static_cast(&remote_endpoint_), 0xDB, + sizeof(remote_endpoint_)); + } +#endif + /** Shut down part or all of the full-duplex connection. Not an override — concrete backends forward here. diff --git a/include/boost/corosio/native/detail/reactor/reactor_socket_finals.hpp b/include/boost/corosio/native/detail/reactor/reactor_socket_finals.hpp index d110aeaca..5d37ce30a 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_socket_finals.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_socket_finals.hpp @@ -90,6 +90,9 @@ class reactor_stream_socket_impl /// Recycle into the owning service's pool at zero references. void retire() noexcept override { +#if !defined(NDEBUG) + this->poison(); +#endif this->svc_.state_->pool_.recycle(static_cast(this)); } @@ -161,6 +164,9 @@ class reactor_dgram_socket_impl /// Recycle into the owning service's pool at zero references. void retire() noexcept override { +#if !defined(NDEBUG) + this->poison(); +#endif this->svc_.state_->pool_.recycle(static_cast(this)); } }; @@ -212,6 +218,9 @@ class reactor_acceptor_impl /// Recycle into the owning service's pool at zero references. void retire() noexcept override { +#if !defined(NDEBUG) + this->poison(); +#endif this->svc_.state_->pool_.recycle(static_cast(this)); } diff --git a/include/boost/corosio/native/detail/reactor/reactor_stream_socket.hpp b/include/boost/corosio/native/detail/reactor/reactor_stream_socket.hpp index b140436df..e350c959f 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_stream_socket.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_stream_socket.hpp @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -285,28 +286,39 @@ class reactor_stream_socket /** Reset op slots and the cached peer for recycling. - `do_close_socket()` already cleared `remote_endpoint_` and every - parked op pointer; each op's own `reset()` runs again at its next - `do_*` call before any field is read. The one thing worth - checking here is `stop_cb`: a still-armed callback would mean a - stop_token outlived the op's completion, which `reset()` always - disengages, so asserting it is what actually catches a missed - completion rather than papering over it. + Each op's own `reset()` runs again at its next `do_*` call + before any field is read, so only `remote_endpoint_` needs an + active reset here; `stop_cb` is checked instead of reset — a + still-armed callback would mean a stop_token outlived the op's + completion, which `reset()` always disengages, so asserting it + is what actually catches a missed completion rather than + papering over it. @pre refs_ == 0, fd closed and deregistered, no op in flight. */ void reuse() noexcept { base_type::reuse(); + remote_endpoint_ = Endpoint{}; BOOST_COROSIO_ASSERT(!conn_.stop_cb); BOOST_COROSIO_ASSERT(!rd_.stop_cb); BOOST_COROSIO_ASSERT(!wr_.stop_cb); BOOST_COROSIO_ASSERT(!wait_rd_.stop_cb); BOOST_COROSIO_ASSERT(!wait_wr_.stop_cb); BOOST_COROSIO_ASSERT(!wait_er_.stop_cb); - BOOST_COROSIO_ASSERT(remote_endpoint_ == Endpoint{}); } +#if !defined(NDEBUG) + /// Extend base_type::poison() to the cached peer endpoint. + void poison() noexcept + { + base_type::poison(); + std::memset( + static_cast(&remote_endpoint_), 0xDB, + sizeof(remote_endpoint_)); + } +#endif + private: // CRTP callbacks for reactor_basic_socket cancel/close diff --git a/test/unit/Jamfile b/test/unit/Jamfile index 261697224..b5df005da 100644 --- a/test/unit/Jamfile +++ b/test/unit/Jamfile @@ -38,13 +38,20 @@ project boost/corosio/test/unit # applies the exclusion patterns to directory entries too and does not # descend into a match. Those targets need the hook translation unit # linked in, which only fault/Jamfile does. -for local f in [ glob-tree-ex . : *.cpp : openssl_stream.cpp wolfssl_stream.cpp cross_ssl_stream.cpp tls_stream.cpp tls_stream_stress.cpp iocp_shutdown.cpp iocp_error_map.cpp openssl_engine.cpp wolfssl_engine.cpp cross_engine.cpp fault ] +for local f in [ glob-tree-ex . : *.cpp : openssl_stream.cpp wolfssl_stream.cpp cross_ssl_stream.cpp tls_stream.cpp tls_stream_stress.cpp iocp_shutdown.cpp iocp_error_map.cpp openssl_engine.cpp wolfssl_engine.cpp cross_engine.cpp zero_alloc.cpp fault ] { run $(f) ; } build-project fault ; +# zero_alloc.cpp counts against alloc_counter.hpp's extern counters, whose +# one definition (plus the global operator new/delete override) lives in +# continuation_post.cpp. Unlike the CMake build, which links every test +# source into a single binary, each b2 `run` target is its own executable, +# so the extra translation unit must be named explicitly here. +run zero_alloc.cpp continuation_post.cpp ; + # IOCP-specific native tests (Windows host only; skip on other hosts to # avoid triggering a cross-compile attempt). if [ os.name ] = NT diff --git a/test/unit/alloc_counter.hpp b/test/unit/alloc_counter.hpp new file mode 100644 index 000000000..5b3f16ae7 --- /dev/null +++ b/test/unit/alloc_counter.hpp @@ -0,0 +1,85 @@ +// +// Copyright (c) 2026 Steve Gerbino +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// Official repository: https://github.com/cppalliance/corosio +// + +#ifndef BOOST_COROSIO_TEST_ALLOC_COUNTER_HPP +#define BOOST_COROSIO_TEST_ALLOC_COUNTER_HPP + +/* Shared global operator new/delete counting interposer. + + The replaceable global allocation functions are process-wide and + may be defined exactly once; continuation_post.cpp owns that one + definition (guarded against TSan, which replaces them itself). + Every other test that needs to assert an allocation count -- + zero_alloc.cpp included -- counts against these same extern + counters instead of layering on a second interposer, which would + be a link error. +*/ + +#include +#include +#include + +// TSan's runtime already replaces global operator new/delete, so a +// counting replacement cannot link under -fsanitize=thread (and the +// counts would reflect the sanitizer's allocator anyway). +#if defined(__SANITIZE_THREAD__) +#define COROSIO_TEST_HAS_TSAN 1 +#elif defined(__has_feature) +#if __has_feature(thread_sanitizer) +#define COROSIO_TEST_HAS_TSAN 1 +#endif +#endif + +namespace boost::corosio { + +#ifndef COROSIO_TEST_HAS_TSAN +/// True while allocations should be tallied into alloc_count. +extern std::atomic alloc_armed; +/// Running count of operator new calls while alloc_armed is set. +extern std::atomic alloc_count; + +/** Check that the counting interposer actually runs, loud-skipping if not. + + ASan and TSan are excluded at compile time, but Valgrind replaces + the global allocator at run time: the interposer never executes, + every count reads zero, and a gate would either fail spuriously + (a documented non-zero pin) or pass vacuously (a zero pin). The + probe tests the behavior rather than the tool, as the fault + harness's allocation probe does, so any runtime replacement is + caught; the skip names its reason on stderr, as the fault suites' + Valgrind skip does, so the leg passes with a visible cause. + + @param suite Name printed in the skip notice. + + @return `true` if allocations are observable and the gate may run. +*/ +inline bool +alloc_counter_is_live(char const* suite) noexcept +{ + // Called through a volatile pointer so the probe allocation + // cannot be elided. + void* (*volatile op_new)(std::size_t) = &::operator new; + alloc_count.store(0, std::memory_order_relaxed); + alloc_armed.store(true, std::memory_order_relaxed); + ::operator delete(op_new(1)); + alloc_armed.store(false, std::memory_order_relaxed); + if (alloc_count.load(std::memory_order_relaxed) != 0) + return true; + std::fprintf( + stderr, + "%s: the global allocator is replaced at run time (Valgrind?), " + "so allocation counts are unobservable; skipping this suite\n", + suite); + return false; +} +#endif + +} // namespace boost::corosio + +#endif diff --git a/test/unit/continuation_post.cpp b/test/unit/continuation_post.cpp index 10aff4586..017809d14 100644 --- a/test/unit/continuation_post.cpp +++ b/test/unit/continuation_post.cpp @@ -16,10 +16,12 @@ #include #include +#include #include #include #include +#include "alloc_counter.hpp" #include "context.hpp" #include "test_suite.hpp" @@ -53,32 +55,76 @@ bare_post_task(capy::continuation* cont, bool& ran) ran = true; } -// TSan's runtime already replaces global operator new/delete, so the -// counting replacements below cannot link under -fsanitize=thread (and -// the counts would reflect the sanitizer's allocator anyway). -#if defined(__SANITIZE_THREAD__) -#define COROSIO_TEST_HAS_TSAN 1 -#elif defined(__has_feature) -#if __has_feature(thread_sanitizer) -#define COROSIO_TEST_HAS_TSAN 1 -#endif -#endif +} // namespace +// Definitions for alloc_counter.hpp's extern declarations. Not in the +// anonymous namespace above: zero_alloc.cpp needs these by extern +// reference, and an unnamed namespace's members are unreachable from +// another translation unit even though formally external-linkage. #ifndef COROSIO_TEST_HAS_TSAN std::atomic alloc_armed{false}; std::atomic alloc_count{0}; -#endif + +namespace { + +inline void +count_alloc() noexcept +{ + if (alloc_armed.load(std::memory_order_relaxed)) + alloc_count.fetch_add(1, std::memory_order_relaxed); +} + +/* Over-aligned new/delete, implemented without a platform-specific + aligned allocator (aligned_alloc/posix_memalign/_aligned_malloc all + differ in availability or requirements): stash the raw malloc() + pointer just before the returned block so the matching delete can + recover it, the same trick a custom allocator uses. +*/ +void* +aligned_alloc_impl(std::size_t n, std::size_t align) +{ + if (align < alignof(void*)) + align = alignof(void*); + std::size_t total = (n ? n : 1) + align + sizeof(void*); + void* raw = std::malloc(total); + if (!raw) + return nullptr; + auto base = reinterpret_cast(raw) + sizeof(void*); + auto aligned = (base + align - 1) & ~(align - 1); + *reinterpret_cast(aligned - sizeof(void*)) = raw; + return reinterpret_cast(aligned); +} + +void +aligned_free_impl(void* p) noexcept +{ + if (!p) + return; + void* raw = *reinterpret_cast( + reinterpret_cast(p) - sizeof(void*)); + std::free(raw); +} } // namespace +#endif } // namespace boost::corosio #ifndef COROSIO_TEST_HAS_TSAN +// Plain forms void* operator new(std::size_t n) { - if (boost::corosio::alloc_armed.load(std::memory_order_relaxed)) - boost::corosio::alloc_count.fetch_add(1, std::memory_order_relaxed); + boost::corosio::count_alloc(); + if (void* p = std::malloc(n ? n : 1)) + return p; + throw std::bad_alloc{}; +} + +void* +operator new[](std::size_t n) +{ + boost::corosio::count_alloc(); if (void* p = std::malloc(n ? n : 1)) return p; throw std::bad_alloc{}; @@ -94,6 +140,111 @@ operator delete(void* p, std::size_t) noexcept { std::free(p); } +void +operator delete[](void* p) noexcept +{ + std::free(p); +} +void +operator delete[](void* p, std::size_t) noexcept +{ + std::free(p); +} + +// nothrow forms +void* +operator new(std::size_t n, std::nothrow_t const&) noexcept +{ + boost::corosio::count_alloc(); + return std::malloc(n ? n : 1); +} +void* +operator new[](std::size_t n, std::nothrow_t const&) noexcept +{ + boost::corosio::count_alloc(); + return std::malloc(n ? n : 1); +} +void +operator delete(void* p, std::nothrow_t const&) noexcept +{ + std::free(p); +} +void +operator delete[](void* p, std::nothrow_t const&) noexcept +{ + std::free(p); +} + +// Over-aligned forms (std::align_val_t) -- an aligned allocation on a +// measured path would otherwise bypass this interposer silently. +void* +operator new(std::size_t n, std::align_val_t align) +{ + boost::corosio::count_alloc(); + if (void* p = boost::corosio::aligned_alloc_impl( + n, static_cast(align))) + return p; + throw std::bad_alloc{}; +} +void* +operator new[](std::size_t n, std::align_val_t align) +{ + boost::corosio::count_alloc(); + if (void* p = boost::corosio::aligned_alloc_impl( + n, static_cast(align))) + return p; + throw std::bad_alloc{}; +} +void +operator delete(void* p, std::align_val_t) noexcept +{ + boost::corosio::aligned_free_impl(p); +} +void +operator delete(void* p, std::size_t, std::align_val_t) noexcept +{ + boost::corosio::aligned_free_impl(p); +} +void +operator delete[](void* p, std::align_val_t) noexcept +{ + boost::corosio::aligned_free_impl(p); +} +void +operator delete[](void* p, std::size_t, std::align_val_t) noexcept +{ + boost::corosio::aligned_free_impl(p); +} + +// Over-aligned nothrow forms +void* +operator new(std::size_t n, std::align_val_t align, std::nothrow_t const&) + noexcept +{ + boost::corosio::count_alloc(); + return boost::corosio::aligned_alloc_impl( + n, static_cast(align)); +} +void* +operator new[](std::size_t n, std::align_val_t align, std::nothrow_t const&) + noexcept +{ + boost::corosio::count_alloc(); + return boost::corosio::aligned_alloc_impl( + n, static_cast(align)); +} +void +operator delete( + void* p, std::align_val_t, std::nothrow_t const&) noexcept +{ + boost::corosio::aligned_free_impl(p); +} +void +operator delete[]( + void* p, std::align_val_t, std::nothrow_t const&) noexcept +{ + boost::corosio::aligned_free_impl(p); +} #endif namespace boost::corosio { @@ -150,6 +301,8 @@ struct continuation_zero_alloc_test void run() { + if (!alloc_counter_is_live("boost.corosio.continuation_post.zero_alloc")) + return; testPostIsZeroAlloc(); } }; diff --git a/test/unit/object_reuse.cpp b/test/unit/object_reuse.cpp index dbdefefe9..54d918a4a 100644 --- a/test/unit/object_reuse.cpp +++ b/test/unit/object_reuse.cpp @@ -17,17 +17,31 @@ #include #include #include +#include +#include +#include +#include +#include #include +#include +#include +#include +#include #include #include +#include #include #include +#include +#include #include +#include #include #include "context.hpp" +#include "temp_path.hpp" #include "test_suite.hpp" namespace boost::corosio { @@ -204,6 +218,471 @@ struct object_reuse_test } }; +// Churns every pooled object kind through 32 construct/destroy cycles, +// then runs one representative op per kind so the exercise below runs +// entirely on recycled (not freshly allocated) impls. Registered on +// the reactor backends plus io_uring -- not IOCP, which is covered by +// its own object_pool conversion separately. +template +struct churn_then_exercise_test +{ + static constexpr int churn_n = 32; + + void churnSockets(io_context& ioc) + { + for (int i = 0; i < churn_n; ++i) + { + tcp_socket s(ioc); + } + } + + void churnAcceptors(io_context& ioc) + { + for (int i = 0; i < churn_n; ++i) + { + tcp_acceptor a(ioc); + } + } + + void churnUdp(io_context& ioc) + { + for (int i = 0; i < churn_n; ++i) + { + udp_socket u(ioc); + } + } + + void churnResolvers(io_context& ioc) + { + for (int i = 0; i < churn_n; ++i) + { + resolver r(ioc); + } + } + + void churnSignalSets(io_context& ioc) + { +#if BOOST_COROSIO_POSIX + for (int i = 0; i < churn_n; ++i) + { + signal_set s(ioc, SIGUSR1); + } +#else + // SIGUSR1 doesn't exist on Windows; this template is never + // instantiated there (COROSIO_NON_IOCP_BACKEND_TESTS registers + // nothing on Windows), but non-dependent names inside it must + // still resolve at definition time regardless of instantiation. + (void)ioc; +#endif + } + + void churnFiles(io_context& ioc) + { + test::temp_file tmp("object_reuse_churn_"); + for (int i = 0; i < churn_n; ++i) + { + stream_file f(ioc); + std::ignore = f.open( + tmp.path, + file_base::read_write | file_base::create | + file_base::truncate); + } + } + + void churnTimers(io_context& ioc, capy::executor_ref ex) + { + for (int i = 0; i < churn_n; ++i) + { + bool done = false; + capy::run_async(ex)([](bool& d) -> capy::task<> { + auto [ec] = co_await delay(std::chrono::milliseconds(0)); + d = !ec; + }(done)); + ioc.run(); + ioc.restart(); + BOOST_TEST(done); + } + } + + // socket + acceptor: accept, connect, read, write, then cancel a + // fresh read via stop_token -- every op the brief names, run on + // the impls churnSockets()/churnAcceptors() just recycled. + void exerciseSocketAndAcceptor(io_context& ioc, capy::executor_ref ex) + { + tcp_acceptor acc(ioc); + BOOST_TEST(!acc.open()); + acc.set_option(socket_option::reuse_address(true)); + BOOST_TEST(!acc.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!acc.listen()); + auto port = acc.local_endpoint().port(); + + tcp_socket server(ioc); + tcp_socket client(ioc); + bool accept_done = false, connect_done = false; + + capy::run_async(ex)( + [](tcp_acceptor& a, tcp_socket& s, bool& done) -> capy::task<> { + auto [ec] = co_await a.accept(s); + done = !ec; + }(acc, server, accept_done)); + capy::run_async(ex)( + [](tcp_socket& s, endpoint ep, bool& done) -> capy::task<> { + auto [ec] = co_await s.connect(ep); + done = !ec; + }(client, endpoint(ipv4_address::loopback(), port), + connect_done)); + ioc.run(); + ioc.restart(); + BOOST_TEST(accept_done); + BOOST_TEST(connect_done); + + char const out = 'q'; + char in = 0; + bool wrote = false, read_ok = false; + capy::run_async(ex)( + [](tcp_socket& s, char const* b, bool& done) -> capy::task<> { + auto [ec, n] = co_await s.write_some(capy::const_buffer(b, 1)); + done = !ec && n == 1; + }(client, &out, wrote)); + capy::run_async(ex)( + [](tcp_socket& s, char* b, bool& done) -> capy::task<> { + auto [ec, n] = co_await s.read_some(capy::mutable_buffer(b, 1)); + done = !ec && n == 1; + }(server, &in, read_ok)); + ioc.run(); + ioc.restart(); + BOOST_TEST(wrote); + BOOST_TEST(read_ok); + BOOST_TEST_EQ(in, out); + + // Cancel-via-stop_token: park a second read with no writer, + // then cancel it through the token instead of close()/ + // destroy() -- a different code path through the same + // recycled impl's op machinery. + std::stop_source src; + bool canceled = false; + capy::run_async(ex, src.get_token())( + [](tcp_socket& s, bool& done) -> capy::task<> { + char buf[1]; + auto [ec, n] = + co_await s.read_some(capy::mutable_buffer(buf, 1)); + std::ignore = n; + done = (ec == capy::cond::canceled); + }(client, canceled)); + ioc.run_one(); + src.request_stop(); + ioc.run(); + ioc.restart(); + BOOST_TEST(canceled); + + client.close(); + server.close(); + acc.close(); + } + + // udp: send_to/recv_from on the recycled churnUdp() impls. + void exerciseUdp(io_context& ioc, capy::executor_ref ex) + { + udp_socket receiver(ioc); + BOOST_TEST(!receiver.open()); + BOOST_TEST(!receiver.bind(endpoint(ipv4_address::loopback(), 0))); + auto rport = receiver.local_endpoint().port(); + + udp_socket sender(ioc); + BOOST_TEST(!sender.open()); + BOOST_TEST(!sender.bind(endpoint(ipv4_address::loopback(), 0))); + + char const out = 'u'; + char in = 0; + bool sent = false, received = false; + endpoint dest(ipv4_address::loopback(), rport); + + capy::run_async(ex)( + [](udp_socket& s, char const* b, endpoint ep, + bool& done) -> capy::task<> { + auto [ec, n] = + co_await s.send_to(capy::const_buffer(b, 1), ep); + done = !ec && n == 1; + }(sender, &out, dest, sent)); + endpoint source; + capy::run_async(ex)( + [](udp_socket& s, char* b, endpoint& src, + bool& done) -> capy::task<> { + auto [ec, n] = + co_await s.recv_from(capy::mutable_buffer(b, 1), src); + done = !ec && n == 1; + }(receiver, &in, source, received)); + ioc.run(); + ioc.restart(); + + BOOST_TEST(sent); + BOOST_TEST(received); + BOOST_TEST_EQ(in, out); + } + + // resolver: resolve localhost on a recycled churnResolvers() impl. + void exerciseResolver(io_context& ioc, capy::executor_ref ex) + { + resolver r(ioc); + bool done = false; + std::error_code resolve_ec; + + capy::run_async(ex)( + [](resolver& r_ref, bool& done_out, + std::error_code& ec_out) -> capy::task<> { + auto [ec, results] = co_await r_ref.resolve("localhost", "80"); + std::ignore = results; + ec_out = ec; + done_out = true; + }(r, done, resolve_ec)); + ioc.run(); + ioc.restart(); + + BOOST_TEST(done); + BOOST_TEST(!resolve_ec); + } + + void run() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + churnSockets(ioc); + churnAcceptors(ioc); + churnUdp(ioc); + churnResolvers(ioc); + churnSignalSets(ioc); + churnFiles(ioc); + churnTimers(ioc, ex); + + exerciseSocketAndAcceptor(ioc, ex); + exerciseUdp(ioc, ex); + exerciseResolver(ioc, ex); + + // signal_set: wait() completed by raise() on a recycled + // churnSignalSets() impl. +#if BOOST_COROSIO_POSIX + { + signal_set sig(ioc, SIGUSR1); + bool done = false; + capy::run_async(ex)( + [](signal_set& s, bool& done_out) -> capy::task<> { + [[maybe_unused]] auto [ec, signum] = co_await s.wait(); + done_out = !ec; + }(sig, done)); + ioc.run_one(); + std::raise(SIGUSR1); + ioc.run(); + ioc.restart(); + BOOST_TEST(done); + } +#endif + + // files: write_some on a recycled churnFiles() impl. + { + test::temp_file tmp("object_reuse_exercise_"); + stream_file f(ioc); + BOOST_TEST(!f.open( + tmp.path, + file_base::read_write | file_base::create | + file_base::truncate)); + bool done = false; + char const data = 'f'; + capy::run_async(ex)( + [](stream_file& file, char const* b, + bool& done_out) -> capy::task<> { + auto [ec, n] = + co_await file.write_some(capy::const_buffer(b, 1)); + done_out = !ec && n == 1; + }(f, &data, done)); + ioc.run(); + ioc.restart(); + BOOST_TEST(done); + } + + // timer: one more delay() wait on a recycled churnTimers() impl. + { + bool done = false; + capy::run_async(ex)([](bool& d) -> capy::task<> { + auto [ec] = co_await delay(std::chrono::milliseconds(0)); + d = !ec; + }(done)); + ioc.run(); + ioc.restart(); + BOOST_TEST(done); + } + } +}; + COROSIO_BACKEND_TESTS(object_reuse_test, "boost.corosio.object_reuse") +COROSIO_NON_IOCP_BACKEND_TESTS( + churn_then_exercise_test, "boost.corosio.object_reuse.churn") + +// AF_UNIX churn lane. local_stream_socket/local_stream_acceptor/ +// local_datagram_socket instantiate the exact same reactor_stream_ +// socket/reactor_acceptor/reactor_datagram_socket templates patched +// above, just parameterized on local_endpoint instead of endpoint -- +// local_endpoint_ is precisely the field poison() caught a missing +// reset for, and nothing else loops these types the way +// churn_then_exercise_test loops the IP-socket kinds. Windows +// loopback rules don't apply: this never binds an IP endpoint, only +// AF_UNIX paths, which are POSIX-only on epoll/select and also +// supported on Windows 10 1803+ -- irrelevant here since +// COROSIO_NON_IOCP_BACKEND_TESTS registers nothing on Windows anyway. +template +struct local_socket_churn_then_exercise_test +{ + static constexpr int churn_n = 32; + + void churnStreamSockets(io_context& ioc) + { + for (int i = 0; i < churn_n; ++i) + { + local_stream_socket s(ioc); + } + } + + void churnAcceptors(io_context& ioc) + { + for (int i = 0; i < churn_n; ++i) + { + local_stream_acceptor a(ioc); + } + } + + void churnDatagramSockets(io_context& ioc) + { +#if BOOST_COROSIO_POSIX + for (int i = 0; i < churn_n; ++i) + { + local_datagram_socket u(ioc); + } +#else + // local_datagram_socket is POSIX-only (local_datagram_socket.hpp). + // This template is never instantiated on Windows + // (COROSIO_NON_IOCP_BACKEND_TESTS registers nothing there), but + // non-dependent names inside it must still resolve at definition + // time regardless of instantiation. + (void)ioc; +#endif + } + + // local_stream_socket + local_stream_acceptor: accept, connect, + // echo one byte each way, then close -- the same shape as + // churn_then_exercise_test's TCP exercise, run on the AF_UNIX + // impls churnStreamSockets()/churnAcceptors() just recycled. + void exerciseStreamAndAcceptor(io_context& ioc, capy::executor_ref ex) + { + test::temp_socket_dir dir; + local_endpoint ep(dir.path()); + + local_stream_acceptor acc(ioc, ep); + BOOST_TEST(acc.is_open()); + + local_stream_socket server(ioc); + local_stream_socket client(ioc); + bool accept_done = false, connect_done = false; + + capy::run_async(ex)( + [](local_stream_acceptor& a, local_stream_socket& s, + bool& done) -> capy::task<> { + auto [ec] = co_await a.accept(s); + done = !ec; + }(acc, server, accept_done)); + capy::run_async(ex)( + [](local_stream_socket& s, local_endpoint ep_, + bool& done) -> capy::task<> { + auto [ec] = co_await s.connect(ep_); + done = !ec; + }(client, ep, connect_done)); + ioc.run(); + ioc.restart(); + BOOST_TEST(accept_done); + BOOST_TEST(connect_done); + + char const out = 'l'; + char in = 0; + bool wrote = false, read_ok = false; + capy::run_async(ex)( + [](local_stream_socket& s, char const* b, + bool& done) -> capy::task<> { + auto [ec, n] = co_await s.write_some(capy::const_buffer(b, 1)); + done = !ec && n == 1; + }(client, &out, wrote)); + capy::run_async(ex)( + [](local_stream_socket& s, char* b, bool& done) -> capy::task<> { + auto [ec, n] = + co_await s.read_some(capy::mutable_buffer(b, 1)); + done = !ec && n == 1; + }(server, &in, read_ok)); + ioc.run(); + ioc.restart(); + BOOST_TEST(wrote); + BOOST_TEST(read_ok); + BOOST_TEST_EQ(in, out); + + client.close(); + server.close(); + acc.close(); + } + + // local_datagram_socket: connect_pair() + connected-mode send()/ + // recv(), on the recycled impls churnDatagramSockets() just + // warmed. + void exerciseDatagram(io_context& ioc, capy::executor_ref ex) + { +#if BOOST_COROSIO_POSIX + local_datagram_socket a(ioc); + local_datagram_socket b(ioc); + BOOST_TEST(!connect_pair(a, b)); + + char const out = 'd'; + char in = 0; + bool sent = false, received = false; + + capy::run_async(ex)( + [](local_datagram_socket& s, char const* buf, + bool& done) -> capy::task<> { + auto [ec, n] = co_await s.send(capy::const_buffer(buf, 1)); + done = !ec && n == 1; + }(a, &out, sent)); + capy::run_async(ex)( + [](local_datagram_socket& s, char* buf, + bool& done) -> capy::task<> { + auto [ec, n] = co_await s.recv(capy::mutable_buffer(buf, 1)); + done = !ec && n == 1; + }(b, &in, received)); + ioc.run(); + ioc.restart(); + + BOOST_TEST(sent); + BOOST_TEST(received); + BOOST_TEST_EQ(in, out); + + a.close(); + b.close(); +#else + (void)ioc; + (void)ex; +#endif + } + + void run() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + churnStreamSockets(ioc); + churnAcceptors(ioc); + churnDatagramSockets(ioc); + + exerciseStreamAndAcceptor(ioc, ex); + exerciseDatagram(ioc, ex); + } +}; + +COROSIO_NON_IOCP_BACKEND_TESTS( + local_socket_churn_then_exercise_test, + "boost.corosio.object_reuse.churn_local") } // namespace boost::corosio diff --git a/test/unit/zero_alloc.cpp b/test/unit/zero_alloc.cpp new file mode 100644 index 000000000..12e4a17cc --- /dev/null +++ b/test/unit/zero_alloc.cpp @@ -0,0 +1,244 @@ +// +// Copyright (c) 2026 Steve Gerbino +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// Official repository: https://github.com/cppalliance/corosio +// + +// Acceptance gate for the object_pool conversion: steady-state +// construct()/destroy() and the accept/connect/read/write dispatch +// path must not call the global allocator once the per-service free +// list has been warmed. Counts against alloc_counter.hpp's shared +// interposer -- see that header for why this TU does not define its +// own operator new/delete. +// +// Excluded under ASan: ASan instruments its own replacement of the +// global allocation functions, and a second counting layer on top of +// that does not reliably observe the same calls. Excluded under TSan +// for the same reason alloc_counter.hpp's counters don't exist there +// (its runtime replaces operator new/delete itself). +#include "context.hpp" // COROSIO_TEST_HAS_ASAN +#include "alloc_counter.hpp" // COROSIO_TEST_HAS_TSAN, alloc_armed/alloc_count + +#if !COROSIO_TEST_HAS_ASAN && !defined(COROSIO_TEST_HAS_TSAN) + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +#include "test_suite.hpp" + +namespace boost::corosio { + +namespace { + +// Cycles warmed with counting off, before the assertion window opens. +constexpr int warmup_cycles = 8; + +// Cycles measured with counting on. +constexpr int measured_cycles = 8; + +} // namespace + +template +struct zero_alloc_test +{ + // One accept -> echo one buffer -> close round trip. `server` is + // constructed and destroyed by the caller on every cycle -- that + // construct()/destroy() pair is the thing under test, so it must + // not live across cycles the way `acc` and `client` do. + void runCycle( + io_context& ioc, + capy::executor_ref ex, + tcp_acceptor& acc, + tcp_socket& client, + endpoint ep) + { + tcp_socket server(ioc); + + bool accepted = false, read_ok = false; + bool connected = false, write_ok = false; + char in = 0; + char const out = 'x'; + + // Server side: accept into the reused socket, then echo the + // one byte the client sends back to it. + capy::run_async(ex)( + [](tcp_acceptor& a, tcp_socket& s, char* b, bool& got_accept, + bool& got_read) -> capy::task<> { + auto [ec1] = co_await a.accept(s); + got_accept = !ec1; + auto [ec2, n2] = co_await s.read_some( + capy::mutable_buffer(b, 1)); + got_read = !ec2 && n2 == 1; + }(acc, server, &in, accepted, read_ok)); + + // Client side: connect, then send the one byte. + capy::run_async(ex)( + [](tcp_socket& s, endpoint ep_, char const* b, + bool& got_connect, bool& got_write) -> capy::task<> { + auto [ec1] = co_await s.connect(ep_); + got_connect = !ec1; + auto [ec2, n2] = co_await s.write_some( + capy::const_buffer(b, 1)); + got_write = !ec2 && n2 == 1; + }(client, ep, &out, connected, write_ok)); + + ioc.run(); + ioc.restart(); + + BOOST_TEST(accepted); + BOOST_TEST(connected); + BOOST_TEST(write_ok); + BOOST_TEST(read_ok); + BOOST_TEST_EQ(in, out); + + client.close(); + + // server destructs here -- recycles its impl for the next + // cycle's construct() instead of freeing it. + } + + void testAcceptEchoCloseIsZeroAlloc() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + tcp_acceptor acc(ioc); + BOOST_TEST(!acc.open()); + acc.set_option(socket_option::reuse_address(true)); + BOOST_TEST(!acc.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!acc.listen()); + auto port = acc.local_endpoint().port(); + endpoint ep(ipv4_address::loopback(), port); + + // client is pre-created and outlives every cycle: only the + // accept side's socket construct()/destroy() is under the + // gate here, matching the brief's accept(tcp_socket&) idiom. + tcp_socket client(ioc); + + for (int i = 0; i < warmup_cycles; ++i) + runCycle(ioc, ex, acc, client, ep); + + alloc_count.store(0, std::memory_order_relaxed); + alloc_armed.store(true, std::memory_order_relaxed); + for (int i = 0; i < measured_cycles; ++i) + runCycle(ioc, ex, acc, client, ep); + alloc_armed.store(false, std::memory_order_relaxed); + + // epoll: the reactor dispatch path (accept, connect, read, + // write, close) and the recycled server-socket impl are both + // allocation-free once warmed -- this is the task's primary + // claim. + // + // select: registered_descs_ (select_scheduler.hpp) is a + // std::map keyed by fd; register_descriptor()/ + // deregister_descriptor() insert/erase a node for each of the + // 2 fds this cycle creates (the accepted server fd, the + // reconnected client fd) every cycle, regardless of impl + // recycling. Documented out of scope for this task (design + // doc section 1); 2 allocations/cycle pins the known cost. + // + // uring: the multishot acceptor heap-allocates a fresh + // uring_accept_op plus a waiter_node per accepted connection + // (uring_multishot_acceptor.hpp) -- also out of scope here; + // 2 allocations/cycle pins that known cost. + // + // Either way, pinning the exact number (0, or the documented + // non-zero count) means a regression or an improvement both + // trip this assert instead of silently drifting. +#if BOOST_COROSIO_HAS_EPOLL + if constexpr (std::is_same_v, epoll_t>) + BOOST_TEST_EQ(alloc_count.load(std::memory_order_relaxed), 0LL); + else +#endif + BOOST_TEST_EQ( + alloc_count.load(std::memory_order_relaxed), + static_cast(2 * measured_cycles)); + } + + // Reuse identity at the tcp_socket level: once the pool has one + // spare impl, a construct()/destroy()/construct() sequence pops + // and recycles that same impl both times instead of allocating. + void testReuseIdentityIsZeroAlloc() + { + io_context ioc(Backend); + + { + tcp_socket warm(ioc); // first-ever construct: pool is cold + } // destroy() recycles it + + alloc_count.store(0, std::memory_order_relaxed); + alloc_armed.store(true, std::memory_order_relaxed); + { + tcp_socket s1(ioc); // pop from free_ + } // destroy() -> recycle + { + tcp_socket s2(ioc); // pop the same recycled impl again + } + alloc_armed.store(false, std::memory_order_relaxed); + + BOOST_TEST_EQ(alloc_count.load(std::memory_order_relaxed), 0LL); + } + + // Timer churn: the thread-local single-slot tier plus capy's + // frame-embedded waiter should make a delay() await literally + // zero-alloc on every backend, not just the two the socket path + // tolerates a known gap on. + void testDelayChurnIsZeroAlloc() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + auto one_delay = [&] { + bool done = false; + capy::run_async(ex)([](bool& d) -> capy::task<> { + auto [ec] = co_await delay(std::chrono::milliseconds(0)); + d = !ec; + }(done)); + ioc.run(); + ioc.restart(); + BOOST_TEST(done); + }; + + // Warms both the timer_service pool and capy's coroutine + // frame pool -- the latter's own first-use allocation would + // otherwise show up as noise in the counted window below. + for (int i = 0; i < warmup_cycles; ++i) + one_delay(); + + alloc_count.store(0, std::memory_order_relaxed); + alloc_armed.store(true, std::memory_order_relaxed); + for (int i = 0; i < measured_cycles; ++i) + one_delay(); + alloc_armed.store(false, std::memory_order_relaxed); + + BOOST_TEST_EQ(alloc_count.load(std::memory_order_relaxed), 0LL); + } + + void run() + { + testAcceptEchoCloseIsZeroAlloc(); + testReuseIdentityIsZeroAlloc(); + testDelayChurnIsZeroAlloc(); + } +}; + +COROSIO_TEST_EPOLL_(zero_alloc_test, "boost.corosio.zero_alloc") +COROSIO_TEST_SELECT_(zero_alloc_test, "boost.corosio.zero_alloc") +COROSIO_TEST_URING_(zero_alloc_test, "boost.corosio.zero_alloc") + +} // namespace boost::corosio + +#endif // !COROSIO_TEST_HAS_ASAN && !COROSIO_TEST_HAS_TSAN From db6a26beaf78780b1ae4f96b65a806c604b695fc Mon Sep 17 00:00:00 2001 From: Steve Gerbino Date: Thu, 8 Oct 2026 15:12:45 +0200 Subject: [PATCH 06/12] refactor: pool uring accept nodes for zero-alloc accept churn The multishot acceptor allocated a waiter_node when parking an accept and a fresh uring_accept_op at every completion boundary, copying the request fields across and deleting the waiter -- two heap round trips per accepted connection, plus a ready_fd_node whenever a delivery arrived before an accept was parked. A parked waiter and its completion have strictly sequential lifetimes, so fold them into one accept_node that is both an intrusive list node and the posted op. Nodes recycle through a per-acceptor free list via the op's new dispose hook, ready-fd nodes recycle the same way, and each in-flight node keeps the acceptor alive through the op envelope's object_ref slot. The inherited cancelled flag keeps its claim-token role; claiming paths now restore the real completion status after stop_cb.reset() quiesces any in-flight canceller. The uring zero-alloc gate pins accept churn at 0 allocations per cycle, matching epoll; only select retains its documented 2/cycle registered_descs_ cost. --- .../detail/uring/uring_acceptor_ops.hpp | 21 +- .../detail/uring/uring_multishot_acceptor.hpp | 446 +++++++++--------- .../native/detail/uring/uring_types.hpp | 12 +- test/unit/zero_alloc.cpp | 19 +- 4 files changed, 260 insertions(+), 238 deletions(-) diff --git a/include/boost/corosio/native/detail/uring/uring_acceptor_ops.hpp b/include/boost/corosio/native/detail/uring/uring_acceptor_ops.hpp index 3473c2096..88e0aeda8 100644 --- a/include/boost/corosio/native/detail/uring/uring_acceptor_ops.hpp +++ b/include/boost/corosio/native/detail/uring/uring_acceptor_ops.hpp @@ -154,8 +154,23 @@ struct uring_accept_op : uring_op sockaddr_storage const& peer, socklen_t peer_len) noexcept = nullptr; + /** Disposal hook, run exactly once when the op is consumed. + + Heap-allocated ops delete themselves (the default); the + multishot acceptor's pooled nodes override this to return to + their acceptor's free list instead. Factoring disposal out of + `do_handler` is what lets one completion routine serve both + ownership models. + */ + void (*dispose)(uring_accept_op*) noexcept = &dispose_delete; + uring_accept_op() noexcept : uring_op(&do_handler, &do_cqe) {} + static void dispose_delete(uring_accept_op* op) noexcept + { + delete op; + } + // LCOV_EXCL_START: never receives a CQE; present for vtable // completeness. static void do_cqe(uring_op*, int, unsigned, ready_queue&) noexcept {} @@ -172,7 +187,7 @@ struct uring_accept_op : uring_op if (owner == nullptr) { - delete self; + self->dispose(self); return; } @@ -186,7 +201,7 @@ struct uring_accept_op : uring_op : make_err(self->err); self->cont.h = self->h; auto next = dispatch_coro(self->ex, self->cont); - delete self; + self->dispose(self); next.resume(); return; } @@ -207,7 +222,7 @@ struct uring_accept_op : uring_op self->cont.h = self->h; auto next = dispatch_coro(self->ex, self->cont); - delete self; + self->dispose(self); next.resume(); } }; diff --git a/include/boost/corosio/native/detail/uring/uring_multishot_acceptor.hpp b/include/boost/corosio/native/detail/uring/uring_multishot_acceptor.hpp index 7da993a36..1e40c6f8f 100644 --- a/include/boost/corosio/native/detail/uring/uring_multishot_acceptor.hpp +++ b/include/boost/corosio/native/detail/uring/uring_multishot_acceptor.hpp @@ -33,7 +33,6 @@ #include #include #include -#include #include #include @@ -81,22 +80,34 @@ class uring_multishot_acceptor_base socklen_t peer_len = 0; }; - struct waiter_node; - - struct waiter_canceller - { - waiter_node* w; - void operator()() const noexcept; - }; - - struct waiter_node : intrusive_list::node + /** Pooled accept node: parked waiter and posted completion in one. + + A parked accept used to be a `waiter_node` whose fields were + copied into a freshly allocated `uring_accept_op` at every + completion boundary. The two had strictly sequential lifetimes + (a waiter *becomes* a completion), so this node is both at + once — the intrusive hook parks it in `waiters_`/`free_nodes_` + and the inherited op posts it to the scheduler. Nodes recycle + through the owning acceptor's `free_nodes_` via `dispose`, so + steady-state accepts allocate nothing. + + The inherited `cancelled` flag is the claim token: delivery, + arming-failure, and cancellation each claim the node with one + `exchange`, and the loser leaves completion to the winner. + Because the flag also decides how `do_handler` reports the + completion, every claiming path stores the final status after + `stop_cb.reset()` (which synchronizes with any in-flight + canceller) and before posting. + + While in flight — parked or posted — the node holds an + `object_ref` on its acceptor (the inherited `object_ref_` + slot), so the impl cannot retire out from under `dispose`. + */ + struct accept_node + : uring_accept_op + , intrusive_list::node { - std::coroutine_handle<> h; - capy::executor_ref ex; - std::error_code* ec_out = nullptr; - io_object::implementation** impl_out = nullptr; - Derived* owner = nullptr; - std::atomic cancelled{false}; + Derived* owner = nullptr; /// True once linked into `waiters_` (guarded by `mutex_`). /// The stop callback is armed before the node is queued, so /// cancel_waiter must not unlink a node it never queued. @@ -104,7 +115,26 @@ class uring_multishot_acceptor_base /// A readiness wait rather than an accept: completion /// observes a pending connection without consuming it. bool peek = false; - std::optional> stop_cb; + + accept_node() noexcept + { + this->dispose = &recycle_thunk; + } + + /// Claim the node for cancellation (the stop token fired); + /// losing the exchange means a delivery already owns it. + void on_cancel() noexcept override + { + if (this->cancelled.exchange(true, std::memory_order_acq_rel)) + return; + owner->cancel_waiter(this); + } + + static void recycle_thunk(uring_accept_op* op) noexcept + { + auto* n = static_cast(op); + n->owner->release_node(n); + } }; int fd_ = -1; @@ -117,12 +147,18 @@ class uring_multishot_acceptor_base Endpoint local_endpoint_{}; mutable std::mutex mutex_; intrusive_list ready_fds_; - intrusive_list waiters_; + intrusive_list waiters_; + /// Recycled accept nodes and ready-fd nodes (guarded by `mutex_`). + /// Both survive impl recycling — the storage belongs to this impl + /// and only the destructor frees it — so steady-state accepts + /// allocate nothing. + intrusive_list free_nodes_; + intrusive_list free_ready_; /// Single parked readiness wait (guarded by `mutex_`). Multishot /// accepting drains the kernel queue instantly, so a listener's /// readiness lives in `ready_fds_`, not in `poll()`; the wait is /// completed by the next delivery instead of a kernel poll. - waiter_node* read_wait_ = nullptr; + accept_node* read_wait_ = nullptr; std::unique_ptr multi_op_; bool closing_ = false; /// Non-zero once an arming failed to reach the kernel (guarded by @@ -155,6 +191,71 @@ class uring_multishot_acceptor_base { } +protected: + /** Pop a recycled accept node, or allocate on the cold path. + + Resets every per-use field and arms the `object_ref` keepalive + on this acceptor; the caller fills the request fields and + either parks or posts the node. OOM on the cold path + terminates, matching every other noexcept initiation path. + */ + accept_node* acquire_node() noexcept + { + accept_node* n; + { + std::lock_guard lk(mutex_); + n = free_nodes_.pop_front(); + } + if (!n) + { + // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept initiation path: OOM => std::terminate is the intended behavior + n = new accept_node(); + } + n->owner = static_cast(this); + n->queued = false; + n->peek = false; + n->err = 0; + n->accepted_fd = -1; + n->peer_len = 0; + n->ec_out = nullptr; + n->impl_out = nullptr; + n->peer_endpoint_out = nullptr; + n->peer_service = nullptr; + n->adopt_fn = nullptr; + n->cancelled.store(false, std::memory_order_relaxed); + n->object_ref_ = detail::object_ref(this); + return n; + } + + /** Return a consumed node to the free list; `dispose` target. + + The keepalive is moved out first and dropped after the lock, + so a final release (which may reenter this impl's retire()) + never runs under `mutex_`. + */ + void release_node(accept_node* n) noexcept + { + auto keep = std::move(n->object_ref_); + { + std::lock_guard lk(mutex_); + free_nodes_.push_front(n); + } + } + + /// Pop a recycled ready-fd node, or allocate on the cold path. + /// @pre `mutex_` is held. + ready_fd_node* acquire_ready_locked() noexcept + { + if (auto* r = free_ready_.pop_front()) + { + r->fd = -1; + r->peer_len = 0; + return r; + } + // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — CQE handler: noexcept, OOM => std::terminate is the intended behavior + return new ready_fd_node{}; + } + public: /** Close and free every fd parked in `ready_fds_`. @@ -173,11 +274,17 @@ class uring_multishot_acceptor_base while (auto* r = ready_fds_.pop_front()) drained.push_back(r); } + if (drained.empty()) + return; + intrusive_list recycled; while (auto* r = drained.pop_front()) { ::close(r->fd); - delete r; + recycled.push_back(r); } + std::lock_guard lk(mutex_); + while (auto* r = recycled.pop_front()) + free_ready_.push_front(r); } /** Recycle into the owning service's pool at zero references. @@ -246,6 +353,15 @@ class uring_multishot_acceptor_base multi_op_->object_ref_.reset(); sched_->drain_cqes_for(multi_op_.get()); } + + // The recycled-node storage belongs to this impl and is only + // freed here; in-flight nodes hold an object_ref on this impl, + // so none can still be parked or posted once the count reached + // zero (or the pool's force-sweep ran after scheduler drain). + while (auto* n = free_nodes_.pop_front()) + delete n; + while (auto* r = free_ready_.pop_front()) + delete r; } Endpoint local_endpoint() const noexcept override @@ -331,7 +447,7 @@ class uring_multishot_acceptor_base /// cancel-by-fd themselves via `cancel_and_flush`. void drain_waiters_only() noexcept { - intrusive_list drained; + intrusive_list drained; { std::lock_guard lk(mutex_); closing_ = true; @@ -349,16 +465,11 @@ class uring_multishot_acceptor_base while (auto* w = drained.pop_front()) { + // reset() synchronizes with an in-flight canceller; after + // it the stored status is ours to set. w->stop_cb.reset(); - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept destructor path: OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); - op->h = w->h; - op->ex = w->ex; - op->ec_out = w->ec_out; - op->impl_out = w->impl_out; - op->cancelled.store(true, std::memory_order_release); - delete w; - sched_->post(op); + w->cancelled.store(true, std::memory_order_release); + sched_->post(w); sched_->work_finished(); } } @@ -376,6 +487,12 @@ class uring_multishot_acceptor_base std::stop_token const& token, std::error_code* ec) noexcept { + auto* w = acquire_node(); + w->h = h; + w->ex = ex; + w->ec_out = ec; + w->peek = true; + bool ready = false; bool aborted = false; { @@ -391,31 +508,17 @@ class uring_multishot_acceptor_base } if (ready || aborted) { - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept initiation path: OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); - op->h = h; - op->ex = ex; - op->ec_out = ec; if (aborted) - op->cancelled.store(true, std::memory_order_release); - sched_->post(op); + w->cancelled.store(true, std::memory_order_release); + sched_->post(w); return; } - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept initiation path: OOM => std::terminate is the intended behavior - auto* w = new waiter_node{}; - w->h = h; - w->ex = ex; - w->ec_out = ec; - w->owner = static_cast(this); - w->peek = true; - // Same protocol as accept parking: arm the callback before // the node is visible and outside `mutex_` (a pre-stopped // token invokes the canceller synchronously, and the // canceller takes `mutex_`). - if (token.stop_possible()) - w->stop_cb.emplace(token, waiter_canceller{w}); + w->start(token); bool was_cancelled = false; int arm_err = 0; @@ -444,16 +547,9 @@ class uring_multishot_acceptor_base } w->stop_cb.reset(); - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept initiation path: OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); - op->h = w->h; - op->ex = w->ex; - op->ec_out = w->ec_out; - op->err = arm_err; - if (was_cancelled) - op->cancelled.store(true, std::memory_order_release); - delete w; - sched_->post(op); + w->err = arm_err; + w->cancelled.store(was_cancelled, std::memory_order_release); + sched_->post(w); } std::error_code set_option( @@ -665,7 +761,7 @@ class uring_multishot_acceptor_base */ void fail_arm(int err) noexcept { - intrusive_list claimed; + intrusive_list claimed; { std::lock_guard lk(mutex_); arm_err_ = err; @@ -673,7 +769,7 @@ class uring_multishot_acceptor_base // canceller already claimed belongs to it: cancel_waiter // is waiting on this mutex to unlink the node itself, so // it has to still be in the list when it gets in. - intrusive_list keep; + intrusive_list keep; while (auto* w = waiters_.pop_front()) { if (!w->cancelled.exchange(true, std::memory_order_acq_rel)) @@ -694,16 +790,13 @@ class uring_multishot_acceptor_base while (auto* w = claimed.pop_front()) { + // The exchange above was a claim, not the completion + // status: after reset() quiesces the canceller, restore + // the flag so the handler reports `err`, not canceled. w->stop_cb.reset(); - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept arming path: OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); - op->h = w->h; - op->ex = w->ex; - op->ec_out = w->ec_out; - op->impl_out = w->impl_out; - op->err = err; - delete w; - sched_->post(op); + w->err = err; + w->cancelled.store(false, std::memory_order_release); + sched_->post(w); sched_->work_finished(); // balance the waiter's work_started } } @@ -718,6 +811,12 @@ class uring_multishot_acceptor_base std::error_code* ec, io_object::implementation** impl_out) { + auto* w = acquire_node(); + w->h = h; + w->ex = ex; + w->ec_out = ec; + w->impl_out = impl_out; + sockaddr_storage peer_storage{}; socklen_t peer_len = sizeof(peer_storage); int accepted_fd = ::accept4( @@ -725,78 +824,55 @@ class uring_multishot_acceptor_base SOCK_NONBLOCK | SOCK_CLOEXEC); if (accepted_fd >= 0) { - auto* op = new uring_accept_op(); - op->h = h; - op->ex = ex; - op->ec_out = ec; - op->impl_out = impl_out; - op->peer_service = peer_service_; - op->adopt_fn = &Derived::adopt_thunk; - op->accepted_fd = accepted_fd; - op->peer_storage = peer_storage; - op->peer_len = peer_len; - sched_->post(op); + w->peer_service = peer_service_; + w->adopt_fn = &Derived::adopt_thunk; + w->accepted_fd = accepted_fd; + w->peer_storage = peer_storage; + w->peer_len = peer_len; + sched_->post(w); return; } // accept4 returned <0 — only EAGAIN/EWOULDBLOCK should fall // through to the parked/waiter path. Other errors (EBADF, etc.) // surface through the existing scheduler-completion path so the - // user sees them via the op's ec_out. Build an op with `err` - // set so do_handler delivers make_err(err). + // user sees them via the node's ec_out: `err` set means + // do_handler delivers make_err(err). if (errno != EAGAIN && errno != EWOULDBLOCK) { - int saved_errno = errno; - auto* op = new uring_accept_op(); - op->h = h; - op->ex = ex; - op->ec_out = ec; - op->impl_out = impl_out; - op->err = saved_errno; - sched_->post(op); + w->err = errno; + sched_->post(w); return; } - uring_accept_op* ready_op = nullptr; + bool have_ready = false; { std::lock_guard lk(mutex_); if (auto* r = ready_fds_.pop_front()) { - ready_op = new uring_accept_op(); - ready_op->h = h; - ready_op->ex = ex; - ready_op->ec_out = ec; - ready_op->impl_out = impl_out; - ready_op->peer_service = peer_service_; - ready_op->adopt_fn = &Derived::adopt_thunk; - ready_op->accepted_fd = r->fd; - ready_op->peer_storage = r->peer; - ready_op->peer_len = r->peer_len; - delete r; + w->peer_service = peer_service_; + w->adopt_fn = &Derived::adopt_thunk; + w->accepted_fd = r->fd; + w->peer_storage = r->peer; + w->peer_len = r->peer_len; + free_ready_.push_front(r); + have_ready = true; } } - if (ready_op) + if (have_ready) { // Post outside the lock — acceptor mutex_ must never be // held while dispatch_mutex_ is acquired by sched_->post(). - sched_->post(ready_op); + sched_->post(w); return; } - auto* w = new waiter_node{}; - w->h = h; - w->ex = ex; - w->ec_out = ec; - w->impl_out = impl_out; - w->owner = static_cast(this); - // Arm the stop callback before the node is visible in // `waiters_` and outside `mutex_`: an already-stopped token // invokes the canceller synchronously from emplace, and // cancel_waiter takes `mutex_` (self-deadlock if held). // Arming pre-queue also keeps the CQE handler from claiming - // and deleting a node whose callback is not yet constructed. - if (token.stop_possible()) - w->stop_cb.emplace(token, waiter_canceller{w}); + // a node whose callback is not yet constructed. + w->start(token); bool was_cancelled = false; { @@ -811,29 +887,18 @@ class uring_multishot_acceptor_base { // A connection arrived while the callback was armed; // prefer it over parking the waiter behind it. - ready_op = new uring_accept_op(); - ready_op->h = h; - ready_op->ex = ex; - ready_op->ec_out = ec; - ready_op->impl_out = impl_out; - ready_op->peer_service = peer_service_; - ready_op->adopt_fn = &Derived::adopt_thunk; - ready_op->accepted_fd = r->fd; - ready_op->peer_storage = r->peer; - ready_op->peer_len = r->peer_len; - delete r; + w->peer_service = peer_service_; + w->adopt_fn = &Derived::adopt_thunk; + w->accepted_fd = r->fd; + w->peer_storage = r->peer; + w->peer_len = r->peer_len; + free_ready_.push_front(r); } else if (arm_err_ != 0) { // No arming reached the kernel, so no CQE will deliver // a connection: parking here would park for good. - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept accept path: OOM => std::terminate is the intended behavior - ready_op = new uring_accept_op(); - ready_op->h = h; - ready_op->ex = ex; - ready_op->ec_out = ec; - ready_op->impl_out = impl_out; - ready_op->err = arm_err_; + w->err = arm_err_; } else { @@ -844,34 +909,20 @@ class uring_multishot_acceptor_base } } - if (was_cancelled) - { - auto* op = new uring_accept_op(); - op->h = w->h; - op->ex = w->ex; - op->ec_out = w->ec_out; - op->impl_out = w->impl_out; - op->cancelled.store(true, std::memory_order_release); - w->stop_cb.reset(); - delete w; - sched_->post(op); - return; - } - w->stop_cb.reset(); - delete w; - sched_->post(ready_op); + w->cancelled.store(was_cancelled, std::memory_order_release); + sched_->post(w); } - void cancel_waiter(waiter_node* w) noexcept + void cancel_waiter(accept_node* w) noexcept { { std::lock_guard lk(mutex_); if (closing_) - return; // on_accept_cqe_impl will drain with closing_ set + return; // drain_waiters_only will complete with closing_ set if (!w->queued) return; // not queued yet; the parking path observes - // `cancelled` and completes the op + // `cancelled` and completes the node if (w->peek) { if (read_wait_ != w) @@ -883,17 +934,13 @@ class uring_multishot_acceptor_base waiters_.remove(w); } } - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — stop-token callback: noexcept, OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); - op->h = w->h; - op->ex = w->ex; - op->ec_out = w->ec_out; - op->impl_out = w->impl_out; - op->cancelled.store(true, std::memory_order_release); - delete w; - // post() increments outstanding_work_; balances the work_started() - // from accept() when the waiter was queued. - sched_->post(op); + // `cancelled` is already true (the claim that routed here); + // that is also the completion status, so post as-is. The node's + // stop_cb is still engaged — do_handler resets it, and running + // that reset from the handler thread while this callback + // returns is the stop_callback destructor's documented + // wait-or-self case. + sched_->post(w); sched_->work_finished(); // balance the work_started() from accept() } @@ -908,9 +955,9 @@ class uring_multishot_acceptor_base void on_accept_cqe_impl(int new_fd, int err, bool more) noexcept { bool was_closing = false; - waiter_node* matched = nullptr; - waiter_node* claimed_peek = nullptr; - intrusive_list closing_waiters; + accept_node* matched = nullptr; + accept_node* claimed_peek = nullptr; + intrusive_list closing_waiters; { std::lock_guard lk(mutex_); was_closing = closing_; @@ -951,8 +998,7 @@ class uring_multishot_acceptor_base } else if (new_fd >= 0) { - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — CQE handler: noexcept, OOM => std::terminate is the intended behavior - auto* node = new ready_fd_node{}; + auto* node = acquire_ready_locked(); node->fd = new_fd; node->peer = multi_op_->peer_storage; node->peer_len = multi_op_->peer_len; @@ -961,8 +1007,7 @@ class uring_multishot_acceptor_base } else if (new_fd >= 0) { - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — CQE handler: noexcept, OOM => std::terminate is the intended behavior - auto* node = new ready_fd_node{}; + auto* node = acquire_ready_locked(); node->fd = new_fd; node->peer = multi_op_->peer_storage; node->peer_len = multi_op_->peer_len; @@ -970,57 +1015,42 @@ class uring_multishot_acceptor_base } } + // Each claim's exchange set `cancelled`; after reset() has + // quiesced any in-flight canceller, restore the flag to the + // real completion status before posting. if (claimed_peek) { claimed_peek->stop_cb.reset(); - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — CQE handler: noexcept, OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); - op->h = claimed_peek->h; - op->ex = claimed_peek->ex; - op->ec_out = claimed_peek->ec_out; - delete claimed_peek; - sched_->post(op); + claimed_peek->cancelled.store(false, std::memory_order_release); + sched_->post(claimed_peek); sched_->work_finished(); // balance the parking work_started } if (matched) { matched->stop_cb.reset(); - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — CQE handler: noexcept, OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); - op->h = matched->h; - op->ex = matched->ex; - op->ec_out = matched->ec_out; - op->impl_out = matched->impl_out; - op->peer_service = peer_service_; - op->adopt_fn = &Derived::adopt_thunk; + matched->peer_service = peer_service_; + matched->adopt_fn = &Derived::adopt_thunk; if (err) { - op->err = err; + matched->err = err; } else if (new_fd >= 0) { - op->accepted_fd = new_fd; - op->peer_storage = multi_op_->peer_storage; - op->peer_len = multi_op_->peer_len; + matched->accepted_fd = new_fd; + matched->peer_storage = multi_op_->peer_storage; + matched->peer_len = multi_op_->peer_len; } - delete matched; - sched_->post(op); + matched->cancelled.store(false, std::memory_order_release); + sched_->post(matched); sched_->work_finished(); // balance waiter's work_started } while (auto* w = closing_waiters.pop_front()) { w->stop_cb.reset(); - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — CQE handler shutdown path: noexcept, OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); - op->h = w->h; - op->ex = w->ex; - op->ec_out = w->ec_out; - op->impl_out = w->impl_out; - op->cancelled.store(true, std::memory_order_release); - delete w; - sched_->post(op); + w->cancelled.store(true, std::memory_order_release); + sched_->post(w); sched_->work_finished(); // balance waiter's work_started } @@ -1079,26 +1109,6 @@ class uring_multishot_acceptor_base } }; -template< - class Derived, - class ImplBase, - class Endpoint, - class PeerService, - class AcceptorService> -inline void -uring_multishot_acceptor_base< - Derived, - ImplBase, - Endpoint, - PeerService, - AcceptorService>:: - waiter_canceller::operator()() const noexcept -{ - if (w->cancelled.exchange(true, std::memory_order_acq_rel)) - return; - w->owner->cancel_waiter(w); -} - } // namespace boost::corosio::detail #endif // BOOST_COROSIO_HAS_URING diff --git a/include/boost/corosio/native/detail/uring/uring_types.hpp b/include/boost/corosio/native/detail/uring/uring_types.hpp index 4138dc5c6..a684326a2 100644 --- a/include/boost/corosio/native/detail/uring/uring_types.hpp +++ b/include/boost/corosio/native/detail/uring/uring_types.hpp @@ -784,8 +784,7 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor final // instead of parking a waiter no accept machinery will signal. if (this->fd_ < 0) { - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept-adjacent initiation path: OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); + auto* op = this->acquire_node(); op->h = h; op->ex = ex; op->ec_out = ec; @@ -805,8 +804,7 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor final // fail uniformly instead of never completing. if (w == wait_type::write) { - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept-adjacent initiation path: OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); + auto* op = this->acquire_node(); op->h = h; op->ex = ex; op->ec_out = ec; @@ -1685,8 +1683,7 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor final // instead of parking a waiter no accept machinery will signal. if (this->fd_ < 0) { - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept-adjacent initiation path: OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); + auto* op = this->acquire_node(); op->h = h; op->ex = ex; op->ec_out = ec; @@ -1706,8 +1703,7 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor final // fail uniformly instead of never completing. if (w == wait_type::write) { - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept-adjacent initiation path: OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); + auto* op = this->acquire_node(); op->h = h; op->ex = ex; op->ec_out = ec; diff --git a/test/unit/zero_alloc.cpp b/test/unit/zero_alloc.cpp index 12e4a17cc..2beeb3685 100644 --- a/test/unit/zero_alloc.cpp +++ b/test/unit/zero_alloc.cpp @@ -150,22 +150,23 @@ struct zero_alloc_test // recycling. Documented out of scope for this task (design // doc section 1); 2 allocations/cycle pins the known cost. // - // uring: the multishot acceptor heap-allocates a fresh - // uring_accept_op plus a waiter_node per accepted connection - // (uring_multishot_acceptor.hpp) -- also out of scope here; - // 2 allocations/cycle pins that known cost. + // uring: the multishot acceptor's accept nodes and ready-fd + // nodes recycle through per-acceptor free lists + // (uring_multishot_acceptor.hpp), so warmed accept churn is + // allocation-free like epoll. // // Either way, pinning the exact number (0, or the documented // non-zero count) means a regression or an improvement both // trip this assert instead of silently drifting. -#if BOOST_COROSIO_HAS_EPOLL - if constexpr (std::is_same_v, epoll_t>) - BOOST_TEST_EQ(alloc_count.load(std::memory_order_relaxed), 0LL); - else -#endif +#if BOOST_COROSIO_HAS_SELECT + if constexpr ( + std::is_same_v, select_t>) BOOST_TEST_EQ( alloc_count.load(std::memory_order_relaxed), static_cast(2 * measured_cycles)); + else +#endif + BOOST_TEST_EQ(alloc_count.load(std::memory_order_relaxed), 0LL); } // Reuse identity at the tcp_socket level: once the pool has one From caeeb98c001073138f3d3b239f82dec72a4ceb94 Mon Sep 17 00:00:00 2001 From: Steve Gerbino Date: Thu, 8 Oct 2026 15:20:02 +0200 Subject: [PATCH 07/12] refactor: recycle random-access file ops through per-file free lists Every read_some_at/write_some_at heap-allocated its op -- a deliberate choice so concurrent operations at distinct offsets can overlap, but one that cost ~500 bytes of malloc/free per call on the posix, io_uring, and IOCP backends alike. Keep the unlimited-concurrency model and recycle the ops instead: completion pushes the op onto a per-file free list (posix reuses the outstanding-ops hook and mutex; uring wraps the heap op types in a pooled subclass that swaps the completion handler; IOCP recycles raf_concurrent_op under its existing ops_mutex_). Each completion copies out what it needs before the push, since a concurrent initiation may refill the op immediately, and drops its impl keepalive only after leaving the list mutex. The storage belongs to the impl, survives impl recycling, and is freed by the impl destructor. The zero-alloc gate grows a raf churn test pinning a warmed write+read round trip at 0 allocations on every backend. --- .../detail/iocp/win_random_access_file.hpp | 44 +++++- .../iocp/win_random_access_file_service.hpp | 26 ++-- .../detail/posix/posix_random_access_file.hpp | 85 ++++++++---- .../posix_random_access_file_service.hpp | 8 +- .../detail/uring/uring_random_access_file.hpp | 130 ++++++++++++++---- test/unit/zero_alloc.cpp | 47 +++++++ 6 files changed, 268 insertions(+), 72 deletions(-) diff --git a/include/boost/corosio/native/detail/iocp/win_random_access_file.hpp b/include/boost/corosio/native/detail/iocp/win_random_access_file.hpp index 5dfb8421c..e60716576 100644 --- a/include/boost/corosio/native/detail/iocp/win_random_access_file.hpp +++ b/include/boost/corosio/native/detail/iocp/win_random_access_file.hpp @@ -25,6 +25,7 @@ #include #include +#include #include namespace boost::corosio::detail { @@ -34,9 +35,11 @@ class win_random_access_file; /** Per-operation state for concurrent random-access file IOCP I/O. - Heap-allocated for each async read/write, enabling unlimited - concurrent operations on the same file. Self-deletes on - completion or shutdown. + Acquired from the file's free list for each async read/write, + enabling unlimited concurrent operations on the same file. + Recycles into that free list on completion or shutdown, so + steady-state reads and writes allocate nothing; the file + destructor frees the recycled storage. */ struct raf_concurrent_op : overlapped_op @@ -59,9 +62,10 @@ struct raf_concurrent_op /** Random-access file implementation for IOCP-based I/O. Collapses the historical internal-state/wrapper split into one - pooled `io_object::implementation`. Each async operation - heap-allocates a `raf_concurrent_op`, allowing unlimited - concurrent reads and writes. + pooled `io_object::implementation`. Each async operation acquires + a `raf_concurrent_op` from the per-file free list, allowing + unlimited concurrent reads and writes without steady-state + allocation. */ class win_random_access_file final : public random_access_file::implementation @@ -73,8 +77,27 @@ class win_random_access_file final win_random_access_file_service& svc_; win_mutex ops_mutex_; intrusive_list outstanding_ops_; + /// Recycled ops (guarded by `ops_mutex_`). Survives impl + /// recycling — the storage belongs to this impl and only the + /// destructor frees it. + intrusive_list free_ops_; HANDLE handle_ = INVALID_HANDLE_VALUE; + /// Pop a recycled op (re-arming its keepalive), or allocate on + /// the cold path; the caller's `reset()` clears per-use state. + raf_concurrent_op* acquire_op() + { + raf_concurrent_op* op; + { + std::lock_guard lock(ops_mutex_); + op = free_ops_.pop_front(); + } + if (!op) + op = new raf_concurrent_op(*this); + op->object_ref_ = detail::object_ref(this); + return op; + } + public: explicit win_random_access_file( win_random_access_file_service& svc) noexcept; @@ -86,12 +109,21 @@ class win_random_access_file final /** Reset recycled state for reuse. + `free_ops_` deliberately survives recycling; only the + destructor frees it. + @pre refs_ == 0, handle closed, no op in flight (per-op `raf_concurrent_op` holds its own `object_ref`, so `refs_` cannot reach zero while one is outstanding). */ void reuse() noexcept; + ~win_random_access_file() override + { + while (auto* op = free_ops_.pop_front()) + delete op; + } + std::coroutine_handle<> read_some_at( std::uint64_t offset, std::coroutine_handle<> h, diff --git a/include/boost/corosio/native/detail/iocp/win_random_access_file_service.hpp b/include/boost/corosio/native/detail/iocp/win_random_access_file_service.hpp index fcba5fefa..4856e5301 100644 --- a/include/boost/corosio/native/detail/iocp/win_random_access_file_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_random_access_file_service.hpp @@ -152,12 +152,12 @@ raf_concurrent_op::do_complete( // Shutdown path: clean up without invoking op->stop_cb.reset(); op->h = {}; + auto keep = std::move(op->object_ref_); { std::lock_guard lock(op->file_->ops_mutex_); op->file_->outstanding_ops_.remove(op); + op->file_->free_ops_.push_front(op); } - op->object_ref_.reset(); - delete op; return; } @@ -175,17 +175,17 @@ raf_concurrent_op::do_complete( op->is_read, static_cast(op->bytes_transferred), op->empty_buffer); - { - std::lock_guard lock(op->file_->ops_mutex_); - op->file_->outstanding_ops_.remove(op); - } - auto coro = op->h; // Hold the keepalive across the resume, like every other IOCP op: // it may be the last reference to the implementation the op was - // working on behalf of. + // working on behalf of. Copy out everything needed first — once + // the op is on free_ops_ a concurrent initiation may refill it. auto prevent_premature_destruction = std::move(op->object_ref_); - delete op; + { + std::lock_guard lock(op->file_->ops_mutex_); + op->file_->outstanding_ops_.remove(op); + op->file_->free_ops_.push_front(op); + } coro.resume(); } @@ -316,9 +316,7 @@ win_random_access_file::read_some_at( { static constexpr std::size_t max_buffers = 16; - auto* op = new raf_concurrent_op(*this); - op->object_ref_ = detail::object_ref(this); - + auto* op = acquire_op(); op->reset(); op->is_read = true; op->h = h; @@ -398,9 +396,7 @@ win_random_access_file::write_some_at( { static constexpr std::size_t max_buffers = 16; - auto* op = new raf_concurrent_op(*this); - op->object_ref_ = detail::object_ref(this); - + auto* op = acquire_op(); op->reset(); op->is_read = false; op->h = h; diff --git a/include/boost/corosio/native/detail/posix/posix_random_access_file.hpp b/include/boost/corosio/native/detail/posix/posix_random_access_file.hpp index 6c73f750f..1679ec31e 100644 --- a/include/boost/corosio/native/detail/posix/posix_random_access_file.hpp +++ b/include/boost/corosio/native/detail/posix/posix_random_access_file.hpp @@ -35,7 +35,6 @@ #include #include #include -#include #include #include #include @@ -51,12 +50,13 @@ POSIX Random-Access File Implementation ======================================== - Each async read/write heap-allocates an raf_op that serves - as both the thread-pool work item and the scheduler completion - op. This allows unlimited concurrent operations on the same - file object, matching Asio's per-op allocation model. + Each async read/write acquires an raf_op that serves as both the + thread-pool work item and the scheduler completion op. This + allows unlimited concurrent operations on the same file object. - The raf_op self-deletes on completion or shutdown. + Ops recycle through the file's free_ops_ list on completion or + shutdown, so steady-state reads and writes allocate nothing; the + file destructor frees the recycled storage. */ namespace boost::corosio::detail { @@ -74,13 +74,15 @@ class posix_random_access_file final public: static constexpr std::size_t max_buffers = 16; - /** Per-operation state, heap-allocated for each async call. + /** Per-operation state, acquired from the file's free list. Inherits from `coro_op` (for scheduler completion plus the shared coroutine, cancellation and keepalive machinery) and - `pool_work_item` (for thread-pool dispatch). Linked into the - file's outstanding_ops_ list for cancellation tracking. `coro_op` - leads the base list so a `scheduler_op*` round-trips. + `pool_work_item` (for thread-pool dispatch). The intrusive hook + links it into `outstanding_ops_` while in flight and `free_ops_` + once recycled — membership is strictly sequential, so one hook + serves both under `ops_mutex_`. `coro_op` leads the base list so + a `scheduler_op*` round-trips. */ struct raf_op final : coro_op @@ -114,11 +116,12 @@ class posix_random_access_file final /** Reset for recycling. `close_file()` already drove fd_ to its closed value before - the refcount reached zero. Every `raf_op` is heap-allocated - per submission and holds its own `object_ref`, so the refcount - cannot reach zero while one is outstanding — asserting - `outstanding_ops_` is empty is therefore a precondition check, - not a defensive one. + the refcount reached zero. Every `raf_op` holds its own + `object_ref` while in flight, so the refcount cannot reach + zero while one is outstanding — asserting `outstanding_ops_` + is empty is therefore a precondition check, not a defensive + one. `free_ops_` deliberately survives recycling: the storage + belongs to this impl and only the destructor frees it. @pre refs_ == 0, fd closed, no op in flight. */ @@ -128,6 +131,12 @@ class posix_random_access_file final BOOST_COROSIO_ASSERT(outstanding_ops_.empty()); } + ~posix_random_access_file() override + { + while (auto* op = free_ops_.pop_front()) + delete op; + } + // -- random_access_file::implementation -- std::coroutine_handle<> read_some_at( @@ -173,10 +182,32 @@ class posix_random_access_file final void close_file() noexcept; private: + /** Pop a recycled op, or allocate on the cold path. + + Per-use fields are filled by the caller; the fields `prepare`- + style reuse must not inherit from the previous run (`errn`, + `bytes_transferred`) are reset here. `start()` resets the + cancellation machinery. + */ + raf_op* acquire_op() + { + raf_op* op; + { + std::lock_guard lock(ops_mutex_); + op = free_ops_.pop_front(); + } + if (!op) + op = new raf_op(); + op->errn = 0; + op->bytes_transferred = 0; + return op; + } + posix_random_access_file_service& svc_; int fd_ = -1; std::mutex ops_mutex_; intrusive_list outstanding_ops_; + intrusive_list free_ops_; }; // --------------------------------------------------------------------------- @@ -311,16 +342,20 @@ posix_random_access_file::raf_op::operator()() errn != 0 ? make_err(errn) : std::error_code{}, is_read, bytes_transferred, /*empty_buffer=*/false); + // Copy out everything needed after recycling: once this op is on + // free_ops_ a concurrent initiation may pop and refill it. The + // keepalive drops after the push so a final release never runs + // under ops_mutex_. + auto keep = std::move(object_ref_); + auto coro = h; + auto exec = ex; { std::lock_guard lock(file_->ops_mutex_); file_->outstanding_ops_.remove(this); + file_->free_ops_.push_front(this); } - - object_ref_.reset(); - - auto coro = h; - ex.on_work_finished(); - delete this; + keep.reset(); + exec.on_work_finished(); coro.resume(); } @@ -330,13 +365,15 @@ inline void posix_random_access_file::raf_op::destroy() { stop_cb.reset(); + auto keep = std::move(object_ref_); + auto exec = ex; { std::lock_guard lock(file_->ops_mutex_); file_->outstanding_ops_.remove(this); + file_->free_ops_.push_front(this); } - object_ref_.reset(); - ex.on_work_finished(); - delete this; + keep.reset(); + exec.on_work_finished(); } } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/posix/posix_random_access_file_service.hpp b/include/boost/corosio/native/detail/posix/posix_random_access_file_service.hpp index 7497f02b3..2c90926e4 100644 --- a/include/boost/corosio/native/detail/posix/posix_random_access_file_service.hpp +++ b/include/boost/corosio/native/detail/posix/posix_random_access_file_service.hpp @@ -175,7 +175,7 @@ posix_random_access_file::read_some_at( return h; } - auto* op = new raf_op(); + auto* op = acquire_op(); op->is_read = true; op->offset = offset; @@ -209,7 +209,7 @@ posix_random_access_file::read_some_at( // like the closed-descriptor and zero-length exits above rather // than through a completion the scheduler has to carry back. // destroy() is the discard the op never reaching the queue - // needs: it unlinks, unwinds the work count and frees. + // needs: it unlinks, unwinds the work count and recycles. op->destroy(); *ec = pec; *bytes_out = 0; @@ -246,7 +246,7 @@ posix_random_access_file::write_some_at( return h; } - auto* op = new raf_op(); + auto* op = acquire_op(); op->is_read = false; op->offset = offset; @@ -280,7 +280,7 @@ posix_random_access_file::write_some_at( // like the closed-descriptor and zero-length exits above rather // than through a completion the scheduler has to carry back. // destroy() is the discard the op never reaching the queue - // needs: it unlinks, unwinds the work count and frees. + // needs: it unlinks, unwinds the work count and recycles. op->destroy(); *ec = pec; *bytes_out = 0; diff --git a/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp b/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp index 4493cd719..3623abaeb 100644 --- a/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp +++ b/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp @@ -26,7 +26,6 @@ #include #include #include -#include #include #include #include @@ -66,7 +65,87 @@ class BOOST_COROSIO_DECL uring_random_access_file final // Random-access files legitimately support concurrent ops at // different offsets on the same fd (e.g. parallel reads in // testConcurrentReads). Embedding a single slot would smash - // state across calls; ops are heap-allocated per submission. + // state across calls; ops recycle through per-file free lists + // so concurrency is unbounded but steady state allocates + // nothing. The storage belongs to this impl (it survives + // recycling) and only the destructor frees it. + + /** Pooled random-access op: completion recycles instead of + deleting. + + Reuses the heap variant's submission/finish machinery but + swaps the completion handler (the func-pointer convention + makes that one protected assignment) for one that returns the + op to its file's free list. The inherited `object_ref_` + keepalive is dropped only after the push, so a final release + never runs under `ops_mutex_`. + */ + template + struct pooled_op final + : Base + , intrusive_list>::node + { + uring_random_access_file* file_ = nullptr; + + pooled_op() noexcept + { + this->func_ = &do_pooled_handler; + } + + static void do_pooled_handler( + void* owner, + scheduler_op* base, + std::uint32_t /*bytes*/, + std::uint32_t /*error*/) noexcept + { + auto* self = static_cast(base); + self->stop_cb.reset(); + auto keep = std::move(self->object_ref_); + if (owner == nullptr) + { + self->file_->release_op(self); + return; + } + auto next = Base::finish(self); + self->file_->release_op(self); + next.resume(); + } + }; + + using pooled_read_op = pooled_op; + using pooled_write_op = pooled_op; + + std::mutex ops_mutex_; + intrusive_list free_reads_; + intrusive_list free_writes_; + + /// Pop a recycled op of the given kind, or allocate on the cold + /// path; `prepare()` resets every per-use field. + template + Op* acquire_op(intrusive_list& free_list) + { + Op* op; + { + std::lock_guard lock(ops_mutex_); + op = free_list.pop_front(); + } + if (!op) + op = new Op(); + op->file_ = this; + return op; + } + + void release_op(pooled_read_op* op) noexcept + { + std::lock_guard lock(ops_mutex_); + free_reads_.push_front(op); + } + + void release_op(pooled_write_op* op) noexcept + { + std::lock_guard lock(ops_mutex_); + free_writes_.push_front(op); + } public: explicit uring_random_access_file( @@ -80,6 +159,10 @@ class BOOST_COROSIO_DECL uring_random_access_file final ~uring_random_access_file() override { close_file(); + while (auto* op = free_reads_.pop_front()) + delete op; + while (auto* op = free_writes_.pop_front()) + delete op; } /// Recycle into the owning service's pool. Defined out-of-line @@ -89,9 +172,10 @@ class BOOST_COROSIO_DECL uring_random_access_file final /** Reset for recycling. `close_file()` already drove fd_ to its closed value before the - refcount reached zero; every op is heap-allocated per submission - and owns its own `stop_cb`/`object_ref` lifetime, so there is no - embedded op state to assert here. + refcount reached zero; every op holds its own `object_ref` + while in flight, so none can be outstanding here. The free + lists deliberately survive recycling — the storage belongs to + this impl and only the destructor frees it. @pre refs_ == 0, fd closed, no op in flight. */ @@ -250,8 +334,8 @@ uring_random_access_file::read_some_at( std::error_code* ec, std::size_t* bytes) { - auto op_guard = std::make_unique(); - op_guard->prepare( + auto* op = acquire_op(free_reads_); + op->prepare( h, ex, ec, bytes, fd_, static_cast(user_offset), sched_, detail::object_ref(this), buffers, token); sched_->work_started(); @@ -259,22 +343,22 @@ uring_random_access_file::read_some_at( // Closed-object contract outranks the zero-length no-op. if (fd_ < 0) { - op_guard->empty_buffer = false; - op_guard->res = -EBADF; + op->empty_buffer = false; + op->res = -EBADF; uring_scheduler::lock_type lock(sched_->dispatch_mutex()); - sched_->push_completed_locked(op_guard.release()); + sched_->push_completed_locked(op); return std::noop_coroutine(); } - if (op_guard->empty_buffer || - op_guard->cancelled.load(std::memory_order_acquire)) + if (op->empty_buffer || + op->cancelled.load(std::memory_order_acquire)) { uring_scheduler::lock_type lock(sched_->dispatch_mutex()); - sched_->push_completed_locked(op_guard.release()); + sched_->push_completed_locked(op); return std::noop_coroutine(); } - uring_submit_op(*sched_, op_guard.release()); + uring_submit_op(*sched_, op); return std::noop_coroutine(); } @@ -288,8 +372,8 @@ uring_random_access_file::write_some_at( std::error_code* ec, std::size_t* bytes) { - auto op_guard = std::make_unique(); - op_guard->prepare( + auto* op = acquire_op(free_writes_); + op->prepare( h, ex, ec, bytes, fd_, static_cast(user_offset), sched_, detail::object_ref(this), buffers, token); sched_->work_started(); @@ -297,22 +381,22 @@ uring_random_access_file::write_some_at( // Closed-object contract outranks the zero-length no-op. if (fd_ < 0) { - op_guard->empty_buffer = false; - op_guard->res = -EBADF; + op->empty_buffer = false; + op->res = -EBADF; uring_scheduler::lock_type lock(sched_->dispatch_mutex()); - sched_->push_completed_locked(op_guard.release()); + sched_->push_completed_locked(op); return std::noop_coroutine(); } - if (op_guard->empty_buffer || - op_guard->cancelled.load(std::memory_order_acquire)) + if (op->empty_buffer || + op->cancelled.load(std::memory_order_acquire)) { uring_scheduler::lock_type lock(sched_->dispatch_mutex()); - sched_->push_completed_locked(op_guard.release()); + sched_->push_completed_locked(op); return std::noop_coroutine(); } - uring_submit_op(*sched_, op_guard.release()); + uring_submit_op(*sched_, op); return std::noop_coroutine(); } diff --git a/test/unit/zero_alloc.cpp b/test/unit/zero_alloc.cpp index 2beeb3685..e0653fad5 100644 --- a/test/unit/zero_alloc.cpp +++ b/test/unit/zero_alloc.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -37,6 +38,7 @@ #include #include +#include "temp_path.hpp" #include "test_suite.hpp" namespace boost::corosio { @@ -228,11 +230,56 @@ struct zero_alloc_test BOOST_TEST_EQ(alloc_count.load(std::memory_order_relaxed), 0LL); } + // Random-access file churn: each read_some_at/write_some_at used + // to heap-allocate its op; both now recycle through the file's + // free lists, so a warmed write+read round trip is zero-alloc on + // every backend (posix thread-pool path and native uring path). + void testRafChurnIsZeroAlloc() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + test::temp_file tmp("zero_alloc_raf_", "warm"); + random_access_file f(ioc); + BOOST_TEST(!f.open(tmp.path, file_base::read_write)); + + char buf[4] = {}; + auto one_rw = [&] { + bool wrote = false, read = false; + capy::run_async(ex)( + [](random_access_file& file, char* b, bool& w, + bool& r) -> capy::task<> { + auto [ec1, n1] = co_await file.write_some_at( + 0, capy::const_buffer("abcd", 4)); + w = !ec1 && n1 == 4; + auto [ec2, n2] = co_await file.read_some_at( + 0, capy::mutable_buffer(b, 4)); + r = !ec2 && n2 == 4; + }(f, buf, wrote, read)); + ioc.run(); + ioc.restart(); + BOOST_TEST(wrote); + BOOST_TEST(read); + }; + + for (int i = 0; i < warmup_cycles; ++i) + one_rw(); + + alloc_count.store(0, std::memory_order_relaxed); + alloc_armed.store(true, std::memory_order_relaxed); + for (int i = 0; i < measured_cycles; ++i) + one_rw(); + alloc_armed.store(false, std::memory_order_relaxed); + + BOOST_TEST_EQ(alloc_count.load(std::memory_order_relaxed), 0LL); + } + void run() { testAcceptEchoCloseIsZeroAlloc(); testReuseIdentityIsZeroAlloc(); testDelayChurnIsZeroAlloc(); + testRafChurnIsZeroAlloc(); } }; From a4dca0be1c3f0eccec1f54cbad3ed973115bb849 Mon Sep 17 00:00:00 2001 From: Steve Gerbino Date: Thu, 8 Oct 2026 15:28:16 +0200 Subject: [PATCH 08/12] refactor: make tcp_server connection launch allocation-free Launching a worker for an accepted connection paid three heap round trips: a fresh std::stop_source shared state, a launch_wrapper coroutine frame through the global allocator, and a wrapper op for posting the bare coroutine handle. Reuse the worker's stop_source across connections and replace it only once a stop has actually been delivered; route the launch_wrapper frame through capy's frame_alloc_mixin so the thread-local recycling allocator serves it; and post the launch through a frame-embedded capy::continuation, which the Executor concept already guarantees every executor accepts. The zero-alloc gate grows a server churn test pinning a warmed connect/echo/close round trip through tcp_server at 0 allocations (select keeps its documented 2/cycle registered_descs_ cost). --- include/boost/corosio/tcp_server.hpp | 29 +++++-- test/unit/zero_alloc.cpp | 109 +++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 6 deletions(-) diff --git a/include/boost/corosio/tcp_server.hpp b/include/boost/corosio/tcp_server.hpp index 494cde682..ac42314f8 100644 --- a/include/boost/corosio/tcp_server.hpp +++ b/include/boost/corosio/tcp_server.hpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -171,10 +172,16 @@ class BOOST_COROSIO_DECL tcp_server template struct launch_wrapper { - struct promise_type + // frame_alloc_mixin routes the frame through the thread-local + // recycling allocator, so a warmed per-connection launch does + // not hit the global allocator. + struct promise_type : capy::frame_alloc_mixin { Ex ex; // Executor stored directly in frame (outlives child tasks) capy::io_env env_; + /// Embedded post node: the start posts this instead of the + /// bare handle, which would heap-allocate a wrapper op. + capy::continuation cont_; // For regular coroutines: first arg is executor, second is stop token template @@ -526,15 +533,25 @@ class BOOST_COROSIO_DECL tcp_server } } guard{srv_, w}; - // Reset worker's stop source for this connection - w->stop_ = {}; - auto st = w->stop_.get_token(); + // A stop_source allocates shared state on construction; + // reuse the worker's across connections and replace it + // only once a stop has actually been delivered (the state + // is then latched for good). + if (w->stop_.stop_requested()) + w->stop_ = {}; + auto st = w->stop_.get_token(); auto wrapper = launch_coro{}(ex, st, srv_, std::move(task), w); - // Executor and stop token stored in promise via constructor - ex.post(std::exchange(wrapper.h, nullptr)); // Release before post + // Executor and stop token stored in promise via + // constructor. Post through the frame-embedded + // continuation — posting the bare handle would allocate a + // wrapper op. The frame stays suspended until the + // executor resumes it, so the node outlives the queue. + auto h = std::exchange(wrapper.h, nullptr); // Release before post + h.promise().cont_.h = h; + ex.post(h.promise().cont_); guard.w = nullptr; // Success - dismiss guard } }; diff --git a/test/unit/zero_alloc.cpp b/test/unit/zero_alloc.cpp index e0653fad5..1285b959d 100644 --- a/test/unit/zero_alloc.cpp +++ b/test/unit/zero_alloc.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -36,6 +37,9 @@ #include #include +#include +#include +#include #include #include "temp_path.hpp" @@ -274,12 +278,117 @@ struct zero_alloc_test BOOST_TEST_EQ(alloc_count.load(std::memory_order_relaxed), 0LL); } + // tcp_server connection churn: launching a worker used to cost a + // fresh stop_source state, a globally allocated launch_wrapper + // frame, and a heap wrapper op for the bare-handle post. All + // three now recycle (reused stop state, frame_alloc_mixin, + // frame-embedded continuation), so a warmed connect/echo/close + // round trip through the server allocates nothing. Select keeps + // its documented 2/cycle registered_descs_ cost. + void testServerChurnIsZeroAlloc() + { + struct echo_worker : tcp_server::worker_base + { + io_context& ctx; + tcp_socket sock; + explicit echo_worker(io_context& c) : ctx(c), sock(c) {} + tcp_socket& socket() override + { + return sock; + } + void run(tcp_server::launcher launch) override + { + launch( + ctx.get_executor(), + [](tcp_socket* s) -> capy::task<> + { + char b[8]; + auto [ec, n] = co_await s->read_some( + capy::mutable_buffer(b, sizeof(b))); + if (!ec) + std::ignore = co_await s->write_some( + capy::const_buffer(b, n)); + s->close(); + }(&sock)); + } + }; + + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + tcp_server srv(ioc, ex); + { + std::vector> v; + for (int i = 0; i < 2; ++i) + v.push_back(std::make_unique(ioc)); + srv.set_workers(std::move(v)); + } + BOOST_TEST(!srv.bind(endpoint(ipv4_address::loopback(), 0))); + auto port = srv.local_endpoint().port(); + endpoint ep(ipv4_address::loopback(), port); + srv.start(); + + auto one_conn = [&] { + bool echoed = false; + capy::run_async(ex)( + [](io_context& c, endpoint target, bool& ok) -> capy::task<> + { + tcp_socket s(c); + auto [ec1] = co_await s.connect(target); + if (ec1) + co_return; + char const out = 'z'; + auto [ec2, n2] = + co_await s.write_some(capy::const_buffer(&out, 1)); + if (ec2 || n2 != 1) + co_return; + char in = 0; + auto [ec3, n3] = + co_await s.read_some(capy::mutable_buffer(&in, 1)); + ok = !ec3 && n3 == 1 && in == out; + }(ioc, ep, echoed)); + // Pump one event at a time: the server's pending accept + // keeps the context busy forever, so a plain run() (or a + // full run_for window) would never return promptly. + while (!echoed && ioc.run_one_for(std::chrono::seconds(5)) != 0) + { + } + ioc.restart(); + BOOST_TEST(echoed); + }; + + for (int i = 0; i < warmup_cycles; ++i) + one_conn(); + + alloc_count.store(0, std::memory_order_relaxed); + alloc_armed.store(true, std::memory_order_relaxed); + for (int i = 0; i < measured_cycles; ++i) + one_conn(); + alloc_armed.store(false, std::memory_order_relaxed); + +#if BOOST_COROSIO_HAS_SELECT + if constexpr ( + std::is_same_v, select_t>) + BOOST_TEST_EQ( + alloc_count.load(std::memory_order_relaxed), + static_cast(2 * measured_cycles)); + else +#endif + BOOST_TEST_EQ(alloc_count.load(std::memory_order_relaxed), 0LL); + + srv.stop(); + ioc.run(); + } + void run() { + if (!alloc_counter_is_live("boost.corosio.zero_alloc")) + return; testAcceptEchoCloseIsZeroAlloc(); testReuseIdentityIsZeroAlloc(); testDelayChurnIsZeroAlloc(); testRafChurnIsZeroAlloc(); + testServerChurnIsZeroAlloc(); } }; From 4b39d5a37cc37c8c7ba3e756f34cebf6acf65e34 Mon Sep 17 00:00:00 2001 From: Steve Gerbino Date: Thu, 8 Oct 2026 15:56:38 +0200 Subject: [PATCH 09/12] fix(test): chain fault shadows into sanitizer interceptors Since clang 23's static ASan runtime, the fault harness's interposed symbols bind ahead of the sanitizer's interceptors (verbosity=1 reports "failed to intercept 'pthread_create'"), so threads created through the raw libc pthread_create bypass ASan's thread registry and the first join aborts with "Joining already joined thread" -- every fault test failed instantly under local ASan builds. Sanitizers export each interceptor as __interceptor_ precisely so an interposer can cooperate. Under a sanitizer build, resolve the census's libc half through those exports (weak references, so a name the runtime does not intercept falls back to the RTLD_NEXT lookup unchanged). Injection semantics are untouched: shadows still run first and armed faults short-circuit before the chained call; unsanitized builds compile the mechanism away entirely. --- test/unit/fault/fault_posix.cpp | 78 +++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/test/unit/fault/fault_posix.cpp b/test/unit/fault/fault_posix.cpp index 24938fd81..eb5fcd82a 100644 --- a/test/unit/fault/fault_posix.cpp +++ b/test/unit/fault/fault_posix.cpp @@ -57,6 +57,77 @@ #define O_TMPFILE 0 #endif +// Sanitizer runtimes intercept part of the census for their own +// bookkeeping (ASan's thread registry, TSan's happens-before graph). +// When these shadows are linked into the executable they can bind +// ahead of the runtime's interceptors (observed with clang >= 23's +// static ASan runtime: `ASAN_OPTIONS=verbosity=1` reports "failed to +// intercept 'pthread_create'", and the first pthread_join aborts with +// "Joining already joined thread" because the thread was created +// behind the runtime's back). Each sanitizer exports its interceptor +// as `__interceptor_` precisely so an interposer like this one +// can cooperate: chaining into it keeps the bookkeeping intact and +// still reaches libc. The references are weak, so a name the runtime +// does not intercept resolves to null and falls back to the plain +// RTLD_NEXT lookup; in unsanitized builds every reference is null. +#if defined(__linux__) +#if defined(__SANITIZE_ADDRESS__) || defined(__SANITIZE_THREAD__) +#define COROSIO_FAULT_SANITIZER 1 +#elif defined(__has_feature) +#if __has_feature(address_sanitizer) || __has_feature(thread_sanitizer) || \ + __has_feature(memory_sanitizer) +#define COROSIO_FAULT_SANITIZER 1 +#endif +#endif +#endif +#if !defined(COROSIO_FAULT_SANITIZER) +#define COROSIO_FAULT_SANITIZER 0 +#endif + +#if COROSIO_FAULT_SANITIZER + +// The libc half of the census (the SSL and liburing entries have no +// sanitizer interceptors). A name listed here that the runtime does +// not actually intercept costs nothing: its weak reference is null. +#define COROSIO_FAULT_INTERCEPTED_SYMBOLS(X) \ + X(socket) X(socketpair) X(bind) X(listen) X(accept) X(connect) \ + X(getsockname) X(getpeername) X(getsockopt) X(setsockopt) \ + X(shutdown) X(close) X(read) X(write) X(readv) X(preadv) \ + X(pwritev) X(recv) X(send) X(recvmsg) X(sendmsg) X(poll) X(pipe) \ + X(fcntl) X(ioctl) X(open) X(fstat) X(ftruncate) X(fsync) \ + X(unlink) X(sigaction) X(pthread_create) X(getaddrinfo) \ + X(freeaddrinfo) X(getnameinfo) X(gethostname) X(fdatasync) \ + X(posix_fadvise) X(select) X(accept4) X(lseek) X(epoll_create1) \ + X(epoll_ctl) X(epoll_wait) X(eventfd) X(timerfd_create) \ + X(timerfd_settime) X(__read_chk) X(__recv_chk) \ + X(__recvfrom_chk) X(__poll_chk) X(__pread64_chk) X(__open_2) \ + X(__gethostname_chk) + +// NOLINTNEXTLINE(bugprone-macro-parentheses) — token-paste operand cannot be parenthesized +#define COROSIO_FAULT_DECLARE_INTERCEPTOR(name) \ + extern "C" void __interceptor_##name() __attribute__((weak)); +COROSIO_FAULT_INTERCEPTED_SYMBOLS(COROSIO_FAULT_DECLARE_INTERCEPTOR) +#undef COROSIO_FAULT_DECLARE_INTERCEPTOR + +namespace { + +struct sanitizer_interceptor +{ + char const* name; + void (*fn)(); +}; + +sanitizer_interceptor const sanitizer_interceptors[] = { +// NOLINTNEXTLINE(bugprone-macro-parentheses) — token-paste operand cannot be parenthesized +#define COROSIO_FAULT_INTERCEPTOR_ENTRY(name) {#name, &__interceptor_##name}, + COROSIO_FAULT_INTERCEPTED_SYMBOLS(COROSIO_FAULT_INTERCEPTOR_ENTRY) +#undef COROSIO_FAULT_INTERCEPTOR_ENTRY +}; + +} // namespace + +#endif // COROSIO_FAULT_SANITIZER + namespace boost::corosio::test::fault { thread_local cqe_slot tls_cqe; @@ -72,6 +143,13 @@ publish_error(int err) noexcept void* real_symbol(char const* name) noexcept { +#if COROSIO_FAULT_SANITIZER + // Chain into the sanitizer's own interceptor when it has one for + // this name — see the block comment above the table. + for (auto const& e : sanitizer_interceptors) + if (e.fn != nullptr && std::strcmp(e.name, name) == 0) + return reinterpret_cast(e.fn); +#endif void* p = ::dlsym(RTLD_NEXT, name); if (!p) { From dc892234431b810c1f61f0c39a61fdc3b851b95a Mon Sep 17 00:00:00 2001 From: Steve Gerbino Date: Thu, 8 Oct 2026 16:30:25 +0200 Subject: [PATCH 10/12] fix: never queue completion-op storage on a deferring executor A completion whose awaiter runs on a non-io_context executor takes dispatch_coro's deferring branch, which handed the executor the op's EMBEDDED continuation -- held on its queue by reference until dequeue (a documented stable-address contract). Every completion tail then consumes its op: recycling it into a free list (pooled uring accept/raf nodes), reusing the embedded slot on the next operation (reactor and IOCP ops), or deleting it (heap ops). A concurrent operation could therefore rewrite or free a queued node, and the dequeued resume targeted a dead or rewritten handle. Two new tests reproduce it deterministically (latch-sequenced, no timing): park a single-worker pool behind a latch so the deferred completion queues into it, refill the op from the io_context side, then release the worker. They crashed on every backend -- uring (pooled nodes), epoll/select (embedded accept op via coro_resume), and IOCP in CI (embedded accept op) -- so the fix lives at the one chokepoint every path shares: dispatch_coro's deferring branch now lends the executor a continuation owned by a self-destroying trampoline frame (post_resume; frames served by capy's recycling pool) and no longer retains the caller's. The io_context fast path is unchanged, and callers keep the uniform consume-then-resume shape since nothing they own stays queued. run_async could not serve as the trampoline: its launch wrapper calls ex.on_work_finished() after the task completes, but the op's executor_ref points into the awaiting coroutine's environment, which the resume destroys -- completions must never touch the executor after resuming the handle, and the trampoline doesn't. --- .../boost/corosio/detail/dispatch_coro.hpp | 120 +++++++++++++++++- .../native/detail/coro_op_complete.hpp | 2 + .../detail/uring/uring_acceptor_ops.hpp | 17 ++- .../detail/uring/uring_random_access_file.hpp | 4 + test/unit/random_access_file.cpp | 95 ++++++++++++++ test/unit/tcp_acceptor.cpp | 100 +++++++++++++++ 6 files changed, 328 insertions(+), 10 deletions(-) diff --git a/include/boost/corosio/detail/dispatch_coro.hpp b/include/boost/corosio/detail/dispatch_coro.hpp index c4d9f05dc..f48246af7 100644 --- a/include/boost/corosio/detail/dispatch_coro.hpp +++ b/include/boost/corosio/detail/dispatch_coro.hpp @@ -14,33 +14,141 @@ #include #include #include +#include +#include #include #include +#include namespace boost::corosio::detail { +/** Trampoline frame that lends an executor a stable continuation. + + An executor's queue holds a dispatched continuation by reference + until dequeue (its stable-address contract) — storage a + completion about to recycle, reuse, or delete its op cannot + provide. The trampoline's frame owns the continuation for exactly + the queue-residency window; when the executor resumes it, it + resumes the real handle and self-destroys. + + run_async cannot serve here: its launch wrapper calls + `ex.on_work_finished()` after the task completes, but a + completion's `executor_ref` points into the awaiting coroutine's + environment, which the resume destroys — completions must never + touch the executor after resuming the handle. This trampoline + touches nothing after the resume (same accounting as the plain + `ex.dispatch` it replaces). +*/ +struct resume_trampoline +{ + struct promise_type : capy::frame_alloc_mixin + { + capy::continuation cont; + + resume_trampoline get_return_object() + { + return { + std::coroutine_handle::from_promise(*this)}; + } + std::suspend_always initial_suspend() noexcept + { + return {}; + } + std::suspend_never final_suspend() noexcept + { + return {}; + } + void return_void() noexcept {} + void unhandled_exception() + { + std::terminate(); + } + }; + + std::coroutine_handle h; +}; + +inline resume_trampoline +make_resume_trampoline(std::coroutine_handle<> target) +{ + // A nested resume, not a symmetric transfer that destroys this + // frame inside await_suspend: that idiom is legal but miscompiled + // by cl 19.44 and Apple Clang 16 (access violations on exactly + // this path). The cost is one stack frame per executor hop, the + // only path that reaches here. + target.resume(); + co_return; +} + +/** Dispatch a handle to an executor without lending it caller storage. + + Follows the executor's dispatch semantics: the resume may run + inline or be queued, at the executor's discretion. + + Allocates one trampoline frame (capy's recycling frame pool); + intended for the cold cross-executor completion path only. An + allocation failure propagates out of the (noexcept-adjacent) + completion path and terminates, matching the initiation paths' + OOM policy. The caller may dispose its op before calling. + + @param ex The executor the resume must run on. + @param h The coroutine to resume there. +*/ +inline void +dispatch_resume(capy::executor_ref ex, std::coroutine_handle<> h) +{ + auto tramp = make_resume_trampoline(h); + tramp.h.promise().cont.h = tramp.h; + ex.dispatch(tramp.h.promise().cont).resume(); +} + /** Returns a handle for symmetric transfer on I/O completion. If the executor is io_context::executor_type, returns `c.h` - directly (fast path). Otherwise dispatches through the - executor, which returns `c.h` or `noop_coroutine()`. + directly (fast path). Otherwise the handle is dispatched through + the executor via a trampoline frame and `noop_coroutine()` is + returned. Callers in coroutine machinery should return the result for symmetric transfer. Callers at the scheduler pump level should call `.resume()` on the result. + @p c is NOT retained past this call: completion ops embed their + continuation in storage that is recycled, reused, or freed the + moment the completion is consumed, so handing a deferring + executor that storage lets a concurrent operation rewrite a + queued node. The deferring branch therefore lends the executor a + frame-owned continuation instead — see `dispatch_resume` — at the + cost of one pooled frame on that cold path. + @param ex The executor to dispatch through. - @param c The continuation to dispatch. Must remain at a - stable address until dequeued by the executor. + @param c Carries the handle to resume; not retained. @return A handle for symmetric transfer or `std::noop_coroutine()`. */ +/// Check whether @p ex is the io_context executor, directly or +/// type-erased through `capy::any_executor` (run_async callers and +/// tcp_server erase it that way). The held TYPE is enough: +/// completions only run on the run thread, where +/// `io_context::executor_type::dispatch` returns `c.h` inline — +/// exactly what the fast path reproduces. +inline bool +is_io_context_executor(capy::executor_ref ex) noexcept +{ + if (ex.target() != nullptr) + return true; + if (auto* any = ex.target()) + return any->target_type() == typeid(io_context::executor_type); + return false; +} + inline std::coroutine_handle<> dispatch_coro(capy::executor_ref ex, capy::continuation& c) { - if (ex.target() != nullptr) + if (is_io_context_executor(ex)) return c.h; - return ex.dispatch(c); + dispatch_resume(ex, c.h); + return std::noop_coroutine(); } } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/coro_op_complete.hpp b/include/boost/corosio/native/detail/coro_op_complete.hpp index 298aff82b..7476c2a62 100644 --- a/include/boost/corosio/native/detail/coro_op_complete.hpp +++ b/include/boost/corosio/native/detail/coro_op_complete.hpp @@ -146,6 +146,8 @@ coro_resume(coro_op* self) noexcept // Clear the keepalive before publishing the continuation: a strand // drained on another thread can reuse this op via reset() the instant // it runs, so this write must be ordered before the publish, not after. + // dispatch_coro does not retain cont, so the op may be reused the + // moment the awaiter runs; nothing of self is touched after this. auto suicide = std::move(self->object_ref_); auto next = dispatch_coro(self->ex, self->cont); next.resume(); diff --git a/include/boost/corosio/native/detail/uring/uring_acceptor_ops.hpp b/include/boost/corosio/native/detail/uring/uring_acceptor_ops.hpp index 88e0aeda8..76cd42f0c 100644 --- a/include/boost/corosio/native/detail/uring/uring_acceptor_ops.hpp +++ b/include/boost/corosio/native/detail/uring/uring_acceptor_ops.hpp @@ -199,10 +199,7 @@ struct uring_accept_op : uring_op *self->ec_out = was_cancelled ? std::error_code(capy::error::canceled) : make_err(self->err); - self->cont.h = self->h; - auto next = dispatch_coro(self->ex, self->cont); - self->dispose(self); - next.resume(); + resume_and_dispose(self); return; } @@ -220,6 +217,18 @@ struct uring_accept_op : uring_op if (self->ec_out) *self->ec_out = {}; + resume_and_dispose(self); + } + + /** Dispose the op, then resume its awaiter. + + dispatch_coro does not retain the op's embedded continuation + (a deferring executor gets a frame-owned one), so disposing + before the resume is safe: a concurrent accept may refill the + recycled node immediately without touching anything queued. + */ + static void resume_and_dispose(uring_accept_op* self) + { self->cont.h = self->h; auto next = dispatch_coro(self->ex, self->cont); self->dispose(self); diff --git a/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp b/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp index 3623abaeb..06e1ea0e8 100644 --- a/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp +++ b/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp @@ -106,6 +106,10 @@ class BOOST_COROSIO_DECL uring_random_access_file final self->file_->release_op(self); return; } + // finish() routes through dispatch_coro, which does not + // retain the op's embedded continuation, so releasing + // before the resume is safe even for a deferring + // executor. auto next = Base::finish(self); self->file_->release_op(self); next.resume(); diff --git a/test/unit/random_access_file.cpp b/test/unit/random_access_file.cpp index 8186aa653..346e56eae 100644 --- a/test/unit/random_access_file.cpp +++ b/test/unit/random_access_file.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include "context.hpp" @@ -36,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -719,6 +721,94 @@ struct random_access_file_test BOOST_TEST(!resumed); } + // A completion whose awaiting coroutine runs on a non-io_context + // executor takes dispatch_coro's deferring branch: the executor + // queues the op's EMBEDDED continuation by reference and returns + // noop, after which the handler recycles the op. A second + // initiation can then pop the recycled op and its own inline + // completion overwrites the continuation the foreign queue still + // points into — the queued resume then targets the wrong (already + // finished) coroutine and the first awaiter is lost. + // + // Sequencing is causal, not timed: a single-worker pool runs C1 + // up to its submission, then parks on a latch (FIFO guarantees + // C1 suspended first); the io_context thread processes C1's + // completion (queued into the parked pool), refills the op via + // C2, and only then releases the worker. + void testDeferredExecutorCompletion() + { + io_context ioc(Backend); + capy::thread_pool pool(1); + + temp_file tmp("raf_deferred_", "abcdefgh"); + random_access_file f(ioc); + BOOST_TEST(!f.open(tmp.path, file_base::read_only)); + + std::latch started(1); + std::latch hold(1); + std::atomic c1_done{false}; + std::atomic c2_done{false}; + char b1[4] = {}; + char b2[4] = {}; + + // C1: initiates from the pool worker; completion must be + // dispatched back to the pool executor (the deferring path). + capy::run_async(pool.get_executor())( + [](random_access_file& fl, char* b, + std::atomic& done) -> capy::task<> + { + auto [ec, n] = + co_await fl.read_some_at(0, capy::mutable_buffer(b, 4)); + done = !ec && n == 4; + }(f, b1, c1_done)); + + // Parked behind C1 on the single worker: when this runs, C1 + // has already submitted and suspended. + capy::run_async(pool.get_executor())( + [](std::latch& s, std::latch& h) -> capy::task<> + { + s.count_down(); + h.wait(); + co_return; + }(started, hold)); + + started.wait(); + + // Process C1's completion: the handler queues C1's resume + // into the (parked) pool and recycles the op. + ioc.run(); + ioc.restart(); + + // Refill: C2 pops the recycled op; its inline completion + // reuses the op's continuation while the pool still holds it. + capy::run_async(ioc.get_executor())( + [](random_access_file& fl, char* b, + std::atomic& done) -> capy::task<> + { + auto [ec, n] = + co_await fl.read_some_at(0, capy::mutable_buffer(b, 4)); + done = !ec && n == 4; + }(f, b2, c2_done)); + ioc.run(); + ioc.restart(); + BOOST_TEST(c2_done.load()); + + // Release the worker; it now dequeues what it believes is + // C1's continuation. Drain with a sentinel queued after it. + hold.count_down(); + std::latch drained(1); + capy::run_async(pool.get_executor())( + [](std::latch& d) -> capy::task<> + { + d.count_down(); + co_return; + }(drained)); + drained.wait(); + + BOOST_TEST(c1_done.load()); + pool.join(); + } + void run() { testConstruction(); @@ -757,6 +847,11 @@ struct random_access_file_test testClosedFileErrors(); testClosedAtOpsComplete(); testStopRaceReportsTransfer(); + // The recycling hazard under test is uring's (pooled op + + // deferring dispatch); the property asserted — an awaiter on + // a foreign executor resumes exactly once — holds on every + // backend, so all of them run it. + testDeferredExecutorCompletion(); #if BOOST_COROSIO_POSIX testSyncOnPipeFails(); testHugeOffsetFails(); diff --git a/test/unit/tcp_acceptor.cpp b/test/unit/tcp_acceptor.cpp index 3efc2c261..ae7ab9f2c 100644 --- a/test/unit/tcp_acceptor.cpp +++ b/test/unit/tcp_acceptor.cpp @@ -19,9 +19,13 @@ #include #include #include +#include #include +#include #include +#include +#include #include #include #include @@ -1833,6 +1837,97 @@ struct tcp_acceptor_test BOOST_TEST(acceptOneThrough(ioc, acc, port, true)); } + // Accept-path twin of random_access_file's + // testDeferredExecutorCompletion: an accept awaited from a + // non-io_context executor takes dispatch_coro's deferring branch, + // where the queued continuation must survive the accept node's + // recycling. Sequencing is causal (latches + single-worker FIFO), + // not timed. + void testDeferredExecutorCompletion() + { + io_context ioc(Backend); + capy::thread_pool pool(1); + + tcp_acceptor acc(ioc); + BOOST_TEST(!acc.open()); + acc.set_option(socket_option::reuse_address(true)); + BOOST_TEST(!acc.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!acc.listen()); + endpoint ep(ipv4_address::loopback(), acc.local_endpoint().port()); + + std::latch started(1); + std::latch hold(1); + std::atomic c1_done{false}; + std::atomic c2_done{false}; + tcp_socket s1(ioc); + tcp_socket s2(ioc); + + // C1: accept awaited on the pool executor (deferring path). + capy::run_async(pool.get_executor())( + [](tcp_acceptor& a, tcp_socket& s, + std::atomic& done) -> capy::task<> + { + auto [ec] = co_await a.accept(s); + done = !ec; + }(acc, s1, c1_done)); + + // Parked behind C1 on the single worker: when this runs, C1 + // has already parked its accept and suspended. + capy::run_async(pool.get_executor())( + [](std::latch& s, std::latch& h) -> capy::task<> + { + s.count_down(); + h.wait(); + co_return; + }(started, hold)); + started.wait(); + + // Deliver C1's connection: the completion queues C1's resume + // into the parked pool and recycles the accept node. + tcp_socket client1(ioc); + capy::run_async(ioc.get_executor())( + [](tcp_socket& c, endpoint e) -> capy::task<> + { + std::ignore = co_await c.connect(e); + }(client1, ep)); + ioc.run(); + ioc.restart(); + + // Refill: C2 accepts on the io_context executor, popping the + // recycled node; its inline completion reuses the node's + // continuation while the pool still holds it. + tcp_socket client2(ioc); + capy::run_async(ioc.get_executor())( + [](tcp_acceptor& a, tcp_socket& s, + std::atomic& done) -> capy::task<> + { + auto [ec] = co_await a.accept(s); + done = !ec; + }(acc, s2, c2_done)); + capy::run_async(ioc.get_executor())( + [](tcp_socket& c, endpoint e) -> capy::task<> + { + std::ignore = co_await c.connect(e); + }(client2, ep)); + ioc.run(); + ioc.restart(); + BOOST_TEST(c2_done.load()); + + // Release the worker and drain with a sentinel. + hold.count_down(); + std::latch drained(1); + capy::run_async(pool.get_executor())( + [](std::latch& d) -> capy::task<> + { + d.count_down(); + co_return; + }(drained)); + drained.wait(); + + BOOST_TEST(c1_done.load()); + pool.join(); + } + void run() { testConstruction(); @@ -1841,6 +1936,11 @@ struct tcp_acceptor_test testMoveConstruct(); testMoveAssign(); + // The recycling hazard under test is uring's (pooled accept + // node + deferring dispatch); the exactly-once-resume + // property holds on every backend, so all of them run it. + testDeferredExecutorCompletion(); + // Cancellation testCancelAccept(); testCloseWhilePendingAccept(); From 55661f4cdc59b2d42be71207ba81dbcd0811c5ce Mon Sep 17 00:00:00 2001 From: Steve Gerbino Date: Thu, 8 Oct 2026 17:51:38 +0200 Subject: [PATCH 11/12] fix: hoist win_wait_reactor's drain buffers out of the wake loop Each reactor wake constructed to_add/to_cancel inside the loop, swapped away pending_register_/pending_cancel_'s capacity, and freed it at scope end -- two malloc/free per wake (pollfds beside them was already hoisted). Hoist both and clear per iteration: the capacities then ping-pong between the locals and the members through the under-lock swap, and a warmed reactor wake allocates nothing. --- .../corosio/native/detail/iocp/win_wait_reactor.hpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/include/boost/corosio/native/detail/iocp/win_wait_reactor.hpp b/include/boost/corosio/native/detail/iocp/win_wait_reactor.hpp index 78a250f49..408a6e81b 100644 --- a/include/boost/corosio/native/detail/iocp/win_wait_reactor.hpp +++ b/include/boost/corosio/native/detail/iocp/win_wait_reactor.hpp @@ -476,6 +476,13 @@ win_wait_reactor::run() { std::vector pollfds; + // Hoisted like pollfds: the swap under the lock trades buffers + // with pending_register_/pending_cancel_, so the capacities + // ping-pong between these locals and the members and no wake + // allocates once both sides are warm. + std::vector to_add; + std::vector to_cancel; + // What the ops still parked here are told on the way out, and what // every later register_wait is told too. Ending on a poll the // provider refused is not a cancellation, and an op that reports @@ -485,8 +492,8 @@ win_wait_reactor::run() while (!stop_.load(std::memory_order_acquire)) { // Drain pending register/cancel under the lock. - std::vector to_add; - std::vector to_cancel; + to_add.clear(); + to_cancel.clear(); { std::lock_guard lock(mutex_); to_add.swap(pending_register_); From 23880ff31a657cf6bf064c465c9921a22bcc37bb Mon Sep 17 00:00:00 2001 From: Steve Gerbino Date: Thu, 8 Oct 2026 19:01:58 +0200 Subject: [PATCH 12/12] docs: keep @ref and its target on one line in ip_address Vale's docstring lint reads the wrapped line's leading identifier as bare prose; rewrapping keeps the reference intact. --- include/boost/corosio/ip_address.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/boost/corosio/ip_address.hpp b/include/boost/corosio/ip_address.hpp index 98a2215e7..59c9fcc88 100644 --- a/include/boost/corosio/ip_address.hpp +++ b/include/boost/corosio/ip_address.hpp @@ -31,9 +31,9 @@ namespace boost::corosio { This class holds either an IPv4 or an IPv6 address. Code that works with both families carries one value instead of branching between @ref - ipv4_address and @ref ipv6_address. Family-generic queries such as @ref - is_loopback dispatch to the held address, and @ref to_v4 / @ref to_v6 - recover the family-specific form. + ipv4_address and @ref ipv6_address. Family-generic queries such as + @ref is_loopback dispatch to the held address, and @ref to_v4 / + @ref to_v6 recover the family-specific form. A v4-mapped IPv6 address (`::ffff:a.b.c.d`) is an IPv6-family value: it does not compare equal to the IPv4 address it maps.