diff --git a/include/boost/corosio/detail/dispatch_coro.hpp b/include/boost/corosio/detail/dispatch_coro.hpp index c4d9f05dc..f48246af7 100644 --- a/include/boost/corosio/detail/dispatch_coro.hpp +++ b/include/boost/corosio/detail/dispatch_coro.hpp @@ -14,33 +14,141 @@ #include #include #include +#include +#include #include #include +#include namespace boost::corosio::detail { +/** Trampoline frame that lends an executor a stable continuation. + + An executor's queue holds a dispatched continuation by reference + until dequeue (its stable-address contract) — storage a + completion about to recycle, reuse, or delete its op cannot + provide. The trampoline's frame owns the continuation for exactly + the queue-residency window; when the executor resumes it, it + resumes the real handle and self-destroys. + + run_async cannot serve here: its launch wrapper calls + `ex.on_work_finished()` after the task completes, but a + completion's `executor_ref` points into the awaiting coroutine's + environment, which the resume destroys — completions must never + touch the executor after resuming the handle. This trampoline + touches nothing after the resume (same accounting as the plain + `ex.dispatch` it replaces). +*/ +struct resume_trampoline +{ + struct promise_type : capy::frame_alloc_mixin + { + capy::continuation cont; + + resume_trampoline get_return_object() + { + return { + std::coroutine_handle::from_promise(*this)}; + } + std::suspend_always initial_suspend() noexcept + { + return {}; + } + std::suspend_never final_suspend() noexcept + { + return {}; + } + void return_void() noexcept {} + void unhandled_exception() + { + std::terminate(); + } + }; + + std::coroutine_handle h; +}; + +inline resume_trampoline +make_resume_trampoline(std::coroutine_handle<> target) +{ + // A nested resume, not a symmetric transfer that destroys this + // frame inside await_suspend: that idiom is legal but miscompiled + // by cl 19.44 and Apple Clang 16 (access violations on exactly + // this path). The cost is one stack frame per executor hop, the + // only path that reaches here. + target.resume(); + co_return; +} + +/** Dispatch a handle to an executor without lending it caller storage. + + Follows the executor's dispatch semantics: the resume may run + inline or be queued, at the executor's discretion. + + Allocates one trampoline frame (capy's recycling frame pool); + intended for the cold cross-executor completion path only. An + allocation failure propagates out of the (noexcept-adjacent) + completion path and terminates, matching the initiation paths' + OOM policy. The caller may dispose its op before calling. + + @param ex The executor the resume must run on. + @param h The coroutine to resume there. +*/ +inline void +dispatch_resume(capy::executor_ref ex, std::coroutine_handle<> h) +{ + auto tramp = make_resume_trampoline(h); + tramp.h.promise().cont.h = tramp.h; + ex.dispatch(tramp.h.promise().cont).resume(); +} + /** Returns a handle for symmetric transfer on I/O completion. If the executor is io_context::executor_type, returns `c.h` - directly (fast path). Otherwise dispatches through the - executor, which returns `c.h` or `noop_coroutine()`. + directly (fast path). Otherwise the handle is dispatched through + the executor via a trampoline frame and `noop_coroutine()` is + returned. Callers in coroutine machinery should return the result for symmetric transfer. Callers at the scheduler pump level should call `.resume()` on the result. + @p c is NOT retained past this call: completion ops embed their + continuation in storage that is recycled, reused, or freed the + moment the completion is consumed, so handing a deferring + executor that storage lets a concurrent operation rewrite a + queued node. The deferring branch therefore lends the executor a + frame-owned continuation instead — see `dispatch_resume` — at the + cost of one pooled frame on that cold path. + @param ex The executor to dispatch through. - @param c The continuation to dispatch. Must remain at a - stable address until dequeued by the executor. + @param c Carries the handle to resume; not retained. @return A handle for symmetric transfer or `std::noop_coroutine()`. */ +/// Check whether @p ex is the io_context executor, directly or +/// type-erased through `capy::any_executor` (run_async callers and +/// tcp_server erase it that way). The held TYPE is enough: +/// completions only run on the run thread, where +/// `io_context::executor_type::dispatch` returns `c.h` inline — +/// exactly what the fast path reproduces. +inline bool +is_io_context_executor(capy::executor_ref ex) noexcept +{ + if (ex.target() != nullptr) + return true; + if (auto* any = ex.target()) + return any->target_type() == typeid(io_context::executor_type); + return false; +} + inline std::coroutine_handle<> dispatch_coro(capy::executor_ref ex, capy::continuation& c) { - if (ex.target() != nullptr) + if (is_io_context_executor(ex)) return c.h; - return ex.dispatch(c); + dispatch_resume(ex, c.h); + return std::noop_coroutine(); } } // namespace boost::corosio::detail diff --git a/include/boost/corosio/detail/intrusive.hpp b/include/boost/corosio/detail/intrusive.hpp index f5267634f..4223a80e8 100644 --- a/include/boost/corosio/detail/intrusive.hpp +++ b/include/boost/corosio/detail/intrusive.hpp @@ -69,6 +69,18 @@ class intrusive_list return head_; } + void push_front(T* w) noexcept + { + auto* n = static_cast(w); + n->prev_ = nullptr; + n->next_ = head_; + if (head_) + static_cast(head_)->prev_ = w; + else + tail_ = w; + head_ = w; + } + void push_back(T* w) noexcept { auto* n = static_cast(w); @@ -118,12 +130,33 @@ class intrusive_list return w; } - void remove(T* w) noexcept + /** Unlink @p w if it is a member of this list. + + @pre @p w is either a genuine member of `*this` list, or is + fully detached (not linked in *any* `intrusive_list`, + including a different instance over the same `T`). This is + not a membership test: `T::node` holds one shared `next_`/ + `prev_` pair regardless of which list currently owns it, so a + node actually linked in a different list looks "not detached" + here and would be spliced out using *this* list's `head_`/ + `tail_` for the boundary cases — corrupting both lists. A + caller that cannot rule that out needs its own bookkeeping + (e.g. a flag for which list currently owns the node) before + calling this. + + @return True if @p w was linked in and has now been removed; + false if it was already detached (e.g. a reentrant call from + a self-referential keepalive dropping during the same + teardown that already popped it elsewhere). Callers that gate + a one-time action (recycling, deleting) on actually having + removed something rely on this to make that action idempotent. + */ + bool remove(T* w) noexcept { auto* n = static_cast(w); // Already detached — nothing to do. if (!n->next_ && !n->prev_ && head_ != w && tail_ != w) - return; + return false; if (n->prev_) static_cast(n->prev_)->next_ = n->next_; else @@ -134,6 +167,7 @@ class intrusive_list tail_ = n->prev_; n->next_ = nullptr; n->prev_ = nullptr; + return true; } /// Invoke @p f for each element in the list. diff --git a/include/boost/corosio/detail/object_pool.hpp b/include/boost/corosio/detail/object_pool.hpp new file mode 100644 index 000000000..6b95484ef --- /dev/null +++ b/include/boost/corosio/detail/object_pool.hpp @@ -0,0 +1,303 @@ +// +// Copyright (c) 2025 Steve Gerbino (steve@gerbino.co) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// Official repository: https://github.com/cppalliance/corosio +// + +#ifndef BOOST_COROSIO_DETAIL_OBJECT_POOL_HPP +#define BOOST_COROSIO_DETAIL_OBJECT_POOL_HPP + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace boost::corosio::detail { + +/** Per-service recycling pool for I/O implementations. + + Replaces per-impl `shared_ptr` ownership: a service news an impl + once, then `recycle()` (the impl's `retire()` action) parks + it on the free list and `acquire()` hands it back out, so steady-state + construct/destroy allocates nothing. Unbounded; memory is released + only on pool destruction. In debug builds, recycled impls are + poisoned by the service's `reuse()` discipline, not here (the pool + cannot know which bytes are resettable). + + A service tears down its impls by calling `shutdown()`, which + enters shutting-down mode and visits every live impl in one + critical section; subsequent zero-ref crossings delete instead of + recycling. + + `adopt()` and `remove()` are protected ownership-transfer + primitives for sanctioned extensions — a derived pool exposing + them to a caller that tracks some of its live impls through its + own structure instead of this pool's `live_` list (the timer + service, via its expiry heap) — rather than part of the public + surface every ordinary service uses. + + @par Thread Safety + Distinct objects: Safe. Shared objects: Safe; all operations lock + an internal mutex. +*/ +template +class object_pool +{ + std::mutex mutex_; + intrusive_list live_; + intrusive_list free_; + bool shutting_down_ = false; + + /** Marks `refs_` on an impl currently parked in `free_`. + + `intrusive_list::remove()` cannot distinguish "linked in + `free_`" from "linked in `live_`" by inspecting the node + alone — both lists share one `next_`/`prev_` pair per impl — + so `recycle()` cannot safely call `live_.remove()` on an impl + it does not already know is live. Stamping this sentinel into + `refs_` the moment an impl joins `free_`, and checking it + before touching either list, is what lets `recycle()` and + `remove()` tell the two cases apart in every build mode, not + just under asserts. `acquire()` always overwrites it with 1 + before the impl is usable again. + */ + static constexpr std::size_t pooled_sentinel = + (std::numeric_limits::max)(); + + /// Check whether @p impl is parked in `free_` (`refs_` reads the + /// sentinel). Callers must check this before any list operation — + /// see the class-level comment on `pooled_sentinel`. + static bool is_pooled(Impl const* impl) noexcept + { + return impl->refs_.load(std::memory_order_relaxed) == + pooled_sentinel; + } + +protected: + /** Track a fresh or reset impl as live. + + @pre @p impl is not parked in `free_` (`refs_` must not read + `pooled_sentinel`). The same list-corruption hazard + `recycle()`'s class-level comment documents applies here: a + free-listed impl pushed onto `live_` directly, bypassing + `acquire()`, would end up linked in both lists through one shared + `next_`/`prev_` pair. + + @pre The caller is not racing `shutdown()`. `acquire()` is + this method's only caller, and a service's `construct()` — + `acquire()`'s only caller — is expected to stop being called + before the service's own `shutdown()` runs; this is checked + only in debug builds because the failure mode is a silently + orphaned impl (no close callback), not memory corruption. + */ + void adopt(Impl* impl) + { + std::lock_guard lock(mutex_); + if (is_pooled(impl)) + { + BOOST_COROSIO_ASSERT(false); + return; + } + BOOST_COROSIO_ASSERT(!shutting_down_); + live_.push_back(impl); + } + + /** Unlink a live impl without recycling (ownership transfer). + + Used by a caller that is taking an impl out of this pool's + bookkeeping entirely (a thread-local cache slot, a direct + delete during shutdown) rather than parking it on `free_`. + + @pre @p impl is not parked in `free_` (`refs_` must not read + `pooled_sentinel`). The same list-corruption hazard + `recycle()`'s class-level comment documents applies here: a + free-listed impl would be spliced out using `live_`'s + boundary pointers instead of `free_`'s. + + @return True if @p impl was linked in `live_` and has now + been removed; false if it was already detached. + */ + bool remove(Impl* impl) noexcept + { + std::lock_guard lock(mutex_); + if (is_pooled(impl)) + { + BOOST_COROSIO_ASSERT(false); + return false; + } + return live_.remove(impl); + } + +public: + /** Destroy the pool, freeing every impl still parked or live. + + @pre `shutdown()` has already run, or no impls are live. + Without it, deleting a live impl here would let that impl's + own destructor reenter `recycle()` outside the shutting-down + branch that makes a reentrant retirement (a self-referential + op's `object_ref` releasing its last reference while this + destructor's own `delete` is still on the stack) a safe no-op + — it would instead look exactly like the accounting-bug case + `recycle()` deletes a second time. + */ + ~object_pool() + { + BOOST_COROSIO_ASSERT(shutting_down_ || live_.empty()); + for (auto* list : {&free_, &live_}) + while (auto* impl = list->pop_front()) + delete impl; + } + + object_pool() = default; + object_pool(object_pool const&) = delete; + object_pool& operator=(object_pool const&) = delete; + + /** Pop-or-create an impl with the acquisition protocol applied. + + Recycled impls are `reuse()`-reset and their reference count + restored from the pooled sentinel; fresh impls are constructed + in place from `args` on the cold path. Either way the impl is + adopted as live. This is the sequence every service's + `construct()` used to hand-roll. Making the pool itself the + only place that calls `new Impl` gives the whole library one + audit point for allocation policy. + + @pre Not called concurrently with, or after, `shutdown()` on + this pool. Every caller reaches this through a service's + `construct()`, and service shutdown order guarantees no + `construct()` outlives that service's own `shutdown()`; the + window is only asserted in debug builds (see `adopt()`) + because the failure mode is an orphaned impl, not corruption. + + @param args Constructor arguments for `Impl` (cold path only). + + @return The acquired impl, `refs_ == 1`, tracked live. + */ + template + Impl* acquire(Args&&... args) + { + Impl* impl; + { + std::lock_guard lock(mutex_); + BOOST_COROSIO_ASSERT(!shutting_down_); + impl = free_.pop_front(); + if (impl) + { + impl->refs_.store(1, std::memory_order_relaxed); + live_.push_back(impl); + } + } + // reuse() runs unlocked: acquire()'s precondition excludes a + // concurrent shutdown() visiting the impl mid-reset. + if (impl) + { + impl->reuse(); + return impl; + } + impl = new Impl(std::forward(args)...); + adopt(impl); + return impl; + } + + /** Move a zero-ref impl from live to the free list. + + Called from `retire()`, possibly on a scheduler thread. + Deletes instead when shutting down. The impl must not be + touched after this returns. + + Two accounting bugs are possible here, neither from this + pool's own bookkeeping but from a caller's refcounting (e.g. + a duplicate `retire()` for one zero-crossing). Both are + made safe in every build mode, not just under asserts, because + neither can be told apart from the correct case by a + `BOOST_COROSIO_ASSERT` alone that still runs the list + operations around it: + + - `impl` is already parked in `free_` — the list-corruption + hazard the class-level `pooled_sentinel` comment documents. + The `is_pooled()` check runs first and returns before either + list is touched. + - `impl` is fully detached from `live_` and is not the + sentinel (not shutting down): `live_.remove()` correctly + returns false, but then the impl is reachable through + neither `acquire()` nor `shutdown()`'s visit while still + allocated — stranded. Deleted instead of left to leak. + + The one legitimate "already detached" case — an impl with an + embedded op whose `object_ref` points back at itself reaching + zero *during* `delete impl`, e.g. `~object_pool()`'s own + unconditional sweep tearing down that op's `object_ref` member + before the outer `delete` returns — is distinguished from the + accounting-bug case by `shutting_down_`: that reentrant path + only exists because `shutdown()` — called with a no-op + callback by a caller like the timer service that tracks its + live impls through its own structure instead of this pool's + `live_` list, just to flip the flag — always precedes + `~object_pool()`'s sweep (service shutdown order), so a + not-live, not-pooled impl while `shutting_down_` is exactly + that reentrant no-op, and the (already false) `was_live` means + skip the delete — the outer frame's `delete` is still running. + */ + void recycle(Impl* impl) + { + bool do_delete = false; + { + std::lock_guard lock(mutex_); + if (is_pooled(impl)) + { + BOOST_COROSIO_ASSERT(false); + return; + } + + bool const was_live = live_.remove(impl); + if (shutting_down_) + { + do_delete = was_live; + } + else if (was_live) + { + impl->refs_.store( + pooled_sentinel, std::memory_order_relaxed); + // LIFO: the most recently retired impl is the + // cache-warmest candidate for the next acquire(). + free_.push_front(impl); + } + else + { + BOOST_COROSIO_ASSERT(false); + do_delete = true; + } + } + if (do_delete) + delete impl; + } + + /** Enter shutdown mode and visit every live impl. + + Sets shutting-down (subsequent zero-crossings delete instead + of recycling) and runs `f` on each live impl, all under one + critical section. + + @note The callback runs under the pool's non-recursive mutex; + it must not call acquire(), recycle(), adopt(), or remove() + on this pool. + */ + template + void shutdown(F&& f) + { + std::lock_guard lock(mutex_); + shutting_down_ = true; + live_.for_each(std::forward(f)); + } +}; + +} // namespace boost::corosio::detail + +#endif diff --git a/include/boost/corosio/detail/object_ref.hpp b/include/boost/corosio/detail/object_ref.hpp new file mode 100644 index 000000000..8f44c4e41 --- /dev/null +++ b/include/boost/corosio/detail/object_ref.hpp @@ -0,0 +1,122 @@ +// +// Copyright (c) 2025 Vinnie Falco (vinnie.falco@gmail.com) +// Copyright (c) 2026 Steve Gerbino +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// Official repository: https://github.com/cppalliance/corosio +// + +#ifndef BOOST_COROSIO_DETAIL_OBJECT_REF_HPP +#define BOOST_COROSIO_DETAIL_OBJECT_REF_HPP + +#include +#include +#include +#include + +namespace boost::corosio::detail { + +/// Add one reference to `impl`. `impl` must be non-null. +inline void +acquire(io_object::implementation* impl) noexcept +{ + impl->refs_.fetch_add(1, std::memory_order_relaxed); +} + +/** Drop one reference; runs `retire()` at zero. `impl` must be + non-null. + + Uses release on the decrement and an acquire fence before the + zero-action so writes made while holding a reference are visible + to the recycler. TSan cannot model a standalone fence, so under + it the decrement itself is acq_rel instead -- strictly stronger, + and TSan's happens-before tracking models the exchange directly. +*/ +inline void +release(io_object::implementation* impl) noexcept +{ +#if BOOST_COROSIO_TSAN + if (impl->refs_.fetch_sub(1, std::memory_order_acq_rel) == 1) + impl->retire(); +#else + if (impl->refs_.fetch_sub(1, std::memory_order_release) == 1) + { + std::atomic_thread_fence(std::memory_order_acquire); + impl->retire(); + } +#endif +} + +/** Owning intrusive reference to an I/O implementation. + + Replaces `std::shared_ptr` op keepalives: copy acquires, + move steals, destruction releases. Kept deliberately minimal — + two words smaller than a shared_ptr and no control block. +*/ +class object_ref +{ + io_object::implementation* p_ = nullptr; + +public: + /// Destroy the object_ref, releasing the held reference, if any. + ~object_ref() { reset(); } + + /// Construct an object_ref holding no reference. + object_ref() noexcept = default; + + /// Construct an object_ref that acquires a new reference to `p`. + explicit object_ref(io_object::implementation* p) noexcept : p_(p) + { + if (p_) + acquire(p_); + } + + /// Construct a copy, acquiring another reference to `other`'s target. + object_ref(object_ref const& other) noexcept : p_(other.p_) + { + if (p_) + acquire(p_); + } + + /// Construct by transferring the reference out of `other`. + object_ref(object_ref&& other) noexcept + : p_(std::exchange(other.p_, nullptr)) + { + } + + /// Release the held reference, if any, and acquire `other`'s target. + object_ref& operator=(object_ref const& other) noexcept + { + object_ref(other).swap(*this); + return *this; + } + + /// Release the held reference, if any, and take ownership of `other`'s. + object_ref& operator=(object_ref&& other) noexcept + { + object_ref(std::move(other)).swap(*this); + return *this; + } + + /// Release the held reference, if any. + void reset() noexcept + { + if (p_) + release(std::exchange(p_, nullptr)); + } + + /// Return the held implementation pointer. + io_object::implementation* get() const noexcept { return p_; } + + /// Return true if a reference is held. + explicit operator bool() const noexcept { return p_ != nullptr; } + + /// Exchange the implementation pointers held by `*this` and `other`. + void swap(object_ref& other) noexcept { std::swap(p_, other.p_); } +}; + +} // namespace boost::corosio::detail + +#endif diff --git a/include/boost/corosio/detail/platform.hpp b/include/boost/corosio/detail/platform.hpp index 7afd49331..58795e9c7 100644 --- a/include/boost/corosio/detail/platform.hpp +++ b/include/boost/corosio/detail/platform.hpp @@ -86,4 +86,18 @@ #endif // BOOST_COROSIO_MRDOCS +// ThreadSanitizer cannot model a standalone std::atomic_thread_fence; +// code that needs a different (stronger) ordering under it checks this. +#if defined(__SANITIZE_THREAD__) +#define BOOST_COROSIO_TSAN 1 +#elif defined(__has_feature) +#if __has_feature(thread_sanitizer) +#define BOOST_COROSIO_TSAN 1 +#endif +#endif + +#ifndef BOOST_COROSIO_TSAN +#define BOOST_COROSIO_TSAN 0 +#endif + #endif // BOOST_COROSIO_DETAIL_PLATFORM_HPP diff --git a/include/boost/corosio/detail/timer.hpp b/include/boost/corosio/detail/timer.hpp index 332943fac..0f10f8363 100644 --- a/include/boost/corosio/detail/timer.hpp +++ b/include/boost/corosio/detail/timer.hpp @@ -80,7 +80,9 @@ class BOOST_COROSIO_DECL timer : public io_object so `wait` is a plain member function rather than a virtual dispatch point. */ - struct implementation : io_object::implementation + struct implementation + : io_object::implementation + , intrusive_list::node { /// Sentinel value indicating the timer is not in the heap. static constexpr std::size_t npos = @@ -119,12 +121,33 @@ class BOOST_COROSIO_DECL timer : public io_object /// The waiter published on this timer, or `nullptr`. waiter_node* waiter_ = nullptr; - /// Free list linkage, reused when this impl is recycled. - implementation* next_free_ = nullptr; - /// Construct bound to the given timer service. explicit implementation(timer_service& svc) noexcept : svc_(&svc) {} + /** Reset recycled state to fresh-impl values. + + Both recycling tiers in `timer_service::construct()` — the + thread-local single-slot cache and the service's + `object_pool` fallback — call this, so the reset exists in + one place rather than duplicated per tier. + */ + void reuse() noexcept + { + expiry_ = {}; + heap_index_.store(npos, std::memory_order_relaxed); + might_have_pending_waits_.store( + false, std::memory_order_relaxed); + BOOST_COROSIO_ASSERT(waiter_ == nullptr); + } + + /** Recycle into the owning service's pool at zero references. + + Defined in timer.cpp: calling `svc_->pool_` needs + `timer_service`'s complete type, which this header only + forward-declares. + */ + void retire() noexcept override; + /** Check whether the timer is expired and absent from the heap. The single definition of the already-expired fast-path diff --git a/include/boost/corosio/detail/timer_service.hpp b/include/boost/corosio/detail/timer_service.hpp index 36b3dc3da..21d322be4 100644 --- a/include/boost/corosio/detail/timer_service.hpp +++ b/include/boost/corosio/detail/timer_service.hpp @@ -14,6 +14,8 @@ #include #include #include +#include +#include #include #include #include @@ -53,16 +55,18 @@ struct scheduler; list still threads waiters through their intrusive hooks when collecting several timers' waiters past the lock. - timer_service owns a min-heap of active timers and a free list - of recycled impls. The heap is ordered by expiry time; the - scheduler queries nearest_expiry() to set the epoll/timerfd - timeout. + timer_service owns a min-heap of active timers and recycles + retired impls through a per-service object_pool, fronted by a + thread-local single-slot cache. The heap is ordered by expiry + time; the scheduler queries nearest_expiry() to set the + epoll/timerfd timeout. Optimization Strategy --------------------- 1. Deferred heap insertion — expires_after() stores the expiry but does not insert into the heap. Insertion happens in wait(). - 2. Thread-local impl cache — single-slot per-thread cache. + 2. Thread-local impl cache — single-slot per-thread cache, in + front of the object_pool fallback. 3. Frame-resident waiter_node with embedded completion_op — eliminates heap allocation per wait/fire/cancel. 4. Cached nearest expiry — atomic avoids mutex in nearest_expiry(). @@ -79,12 +83,30 @@ struct scheduler; inline void timer_service_invalidate_cache() noexcept; +/** The timer service's recycling pool. + + Adds no behavior of its own — it exists only to re-expose + `adopt()` and `remove()` as public, so the timer service's + thread-local cache can transfer a cached impl's `live_` tracking + on both sides of the handoff: `construct()` re-adopts a + TL-cached impl as live, `shutdown()` detaches one before its + direct delete. +*/ +class timer_object_pool : public object_pool +{ +public: + using object_pool::adopt; + using object_pool::remove; +}; + // timer_service class body — member function definitions are // out-of-class (after implementation and waiter_node are complete) class BOOST_COROSIO_DECL timer_service final : public capy::execution_context::service , public io_object::io_service { + friend struct timer::implementation; + public: using clock_type = std::chrono::steady_clock; using time_point = clock_type::time_point; @@ -128,7 +150,7 @@ class BOOST_COROSIO_DECL timer_service final BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // std:: members, dll-interface mutable std::mutex mutex_; std::vector heap_; - timer::implementation* free_list_ = nullptr; + timer_object_pool pool_; callback on_earliest_changed_; bool shutting_down_ = false; // Avoids mutex in nearest_expiry() and empty() @@ -282,6 +304,9 @@ timer_service::shutdown() { timer_service_invalidate_cache(); shutting_down_ = true; + // Flag only; the real drain walks the expiry heap below, not + // object_pool's live_ list, so the visit callback is a no-op. + pool_.shutdown([](timer::implementation*) {}); // Snapshot impls and detach them from the heap so that // coroutine-owned timer destructors (triggered by h.destroy() @@ -316,16 +341,16 @@ timer_service::shutdown() if (h) h.destroy(); } + // Unlink from the pool's live_ list before the direct delete + // below, or ~object_pool()'s unconditional sweep would delete + // this impl a second time. + [[maybe_unused]] bool const removed = pool_.remove(impl); + BOOST_COROSIO_ASSERT(removed); delete impl; } - // Delete free-listed impls - while (free_list_) - { - auto* next = free_list_->next_free_; - delete free_list_; - free_list_ = next; - } + // Anything still parked in pool_ (free-listed or live with no + // pending wait at shutdown) is freed by object_pool's own destructor. } inline io_object::implementation* @@ -335,36 +360,16 @@ timer_service::construct() if (impl) { impl->svc_ = this; - // Reset expiry_ too: a recycled impl must behave like a fresh - // one, whose default expiry reads as already elapsed - impl->expiry_ = {}; - impl->heap_index_.store( - (std::numeric_limits::max)(), - std::memory_order_relaxed); - impl->might_have_pending_waits_.store(false, std::memory_order_relaxed); - BOOST_COROSIO_ASSERT(impl->waiter_ == nullptr); + impl->reuse(); + impl->refs_.store(1, std::memory_order_relaxed); + // The TL push that parked impl here also removed it from + // live_ (see destroy_impl) — re-adopt it now that it is live + // again. + pool_.adopt(impl); return impl; } - std::lock_guard lock(mutex_); - if (free_list_) - { - impl = free_list_; - free_list_ = impl->next_free_; - impl->next_free_ = nullptr; - impl->svc_ = this; - impl->expiry_ = {}; - impl->heap_index_.store( - (std::numeric_limits::max)(), - std::memory_order_relaxed); - impl->might_have_pending_waits_.store(false, std::memory_order_relaxed); - BOOST_COROSIO_ASSERT(impl->waiter_ == nullptr); - } - else - { - impl = new timer::implementation(*this); - } - return impl; + return pool_.acquire(*this); } inline void @@ -402,11 +407,20 @@ timer_service::destroy_impl(timer::implementation& impl) } if (try_push_tl_cache(&impl)) + { + // The TL slot takes sole ownership: it is deleted directly by + // tl_cache_owner / the stale-service path, never through this + // pool, so it must leave live_ now or ~object_pool() would see + // it as abandoned (and, at the wrong moment, double-delete it + // after a later construct() re-adopts it). + [[maybe_unused]] bool const removed = pool_.remove(&impl); + BOOST_COROSIO_ASSERT(removed); return; + } - std::lock_guard lock(mutex_); - impl.next_free_ = free_list_; - free_list_ = &impl; + // No op keepalives on a timer impl: this is the only reference, + // so release() always recycles immediately via retire(). + release(&impl); } inline void diff --git a/include/boost/corosio/io/io_object.hpp b/include/boost/corosio/io/io_object.hpp index 49a251b26..d90f52a08 100644 --- a/include/boost/corosio/io/io_object.hpp +++ b/include/boost/corosio/io/io_object.hpp @@ -15,6 +15,8 @@ #include #include +#include +#include #include namespace boost::corosio { @@ -49,11 +51,35 @@ class BOOST_COROSIO_DECL io_object class handle; /** Derived types dispatch platform-specific I/O operations through it. + + Reference-counted: the owning service holds one reference + (`refs_` starts at 1); in-flight operations hold additional + references through `detail::object_ref`. When the count reaches + zero, `retire()` runs — services recycle the impl into + their free-list there. */ struct implementation { /// Destroy the implementation; called only through @ref io_service. virtual ~implementation() = default; + + /** Retire the implementation once the reference count reaches zero. + + Every concrete implementation's override follows the + same shape: recycle into the owning service's pool member + as the final statement. The service befriends the + implementation so it can reach that private member + directly. Any required last-rites cleanup — releasing + state that is only safe to tear down once idle — runs as + ordinary statements immediately before the recycle. + Whichever action runs — recycle or delete — must be the + final statement: nothing touches the implementation + after. + */ + virtual void retire() noexcept = 0; + + /// In-flight + service references; starts at the service's 1. + std::atomic refs_{1}; }; /** Constructs, closes, and destroys platform implementations on @@ -65,10 +91,28 @@ class BOOST_COROSIO_DECL io_object /// Destroy the service; the execution context outlives it. virtual ~io_service() = default; - /// Construct a new implementation instance. + /** Construct a new implementation instance. + + May return a recycled implementation taken from the + service's free-list (populated by `implementation::retire`) + instead of allocating new storage. + */ virtual implementation* construct() = 0; - /// Destroy the implementation, closing kernel resources and freeing memory. + /** Close kernel resources and release the service's reference. + + Called whenever a handle relinquishes its current + implementation, not only on destruction. Handle + destruction, move-assignment onto a handle that already + owns one (the replaced implementation is destroyed, the + incoming one is not), and `handle::reset()` all invoke + this. It closes the underlying descriptor and drops the + service's own reference. In-flight operations may still + hold references of their own, so the implementation is + not necessarily recycled or freed here. That happens in + `implementation::retire` whenever the reference count + actually reaches zero. + */ virtual void destroy(implementation* impl) = 0; /// Close the I/O object, releasing kernel resources without deallocating. @@ -156,6 +200,14 @@ class BOOST_COROSIO_DECL io_object /** Replace the implementation, destroying the old one. + @pre The handle is already bound to a service (constructed + via the service-taking constructor, not the default + one) — the old implementation is destroyed through + that service. + @pre @p p, if non-null, was constructed by this handle's + own service. The service that destroys it later must + be the one that knows how to close it. + @param p The new implementation to own. May be nullptr. */ void reset(implementation* p) noexcept @@ -190,7 +242,8 @@ class BOOST_COROSIO_DECL io_object /** Create a handle bound to a service found in the context. - @tparam Service The service type whose key_type is used for lookup. + @tparam Service The service type whose `key_type` is used for + lookup. @param ctx The execution context to search for the service. @return A handle owning a freshly constructed implementation. diff --git a/include/boost/corosio/ip_address.hpp b/include/boost/corosio/ip_address.hpp index 98a2215e7..59c9fcc88 100644 --- a/include/boost/corosio/ip_address.hpp +++ b/include/boost/corosio/ip_address.hpp @@ -31,9 +31,9 @@ namespace boost::corosio { This class holds either an IPv4 or an IPv6 address. Code that works with both families carries one value instead of branching between @ref - ipv4_address and @ref ipv6_address. Family-generic queries such as @ref - is_loopback dispatch to the held address, and @ref to_v4 / @ref to_v6 - recover the family-specific form. + ipv4_address and @ref ipv6_address. Family-generic queries such as + @ref is_loopback dispatch to the held address, and @ref to_v4 / + @ref to_v6 recover the family-specific form. A v4-mapped IPv6 address (`::ffff:a.b.c.d`) is an IPv6-family value: it does not compare equal to the IPv4 address it maps. diff --git a/include/boost/corosio/native/detail/coro_op.hpp b/include/boost/corosio/native/detail/coro_op.hpp index 460bb6c87..bac26b0bc 100644 --- a/include/boost/corosio/native/detail/coro_op.hpp +++ b/include/boost/corosio/native/detail/coro_op.hpp @@ -12,6 +12,7 @@ #include #include +#include #include #include @@ -93,7 +94,7 @@ struct coro_op : scheduler_op /// Keeps the owning impl alive while the op is in flight (the kernel /// owns user buffers until completion). Dropped in the handler's resume /// tail (see coro_op_complete.hpp). - std::shared_ptr impl_ptr; + detail::object_ref object_ref_; /// Default-construct for virtual-dispatch backends (the reactors, which /// override operator()/destroy() and leave func_ null). diff --git a/include/boost/corosio/native/detail/coro_op_complete.hpp b/include/boost/corosio/native/detail/coro_op_complete.hpp index 24293de24..7476c2a62 100644 --- a/include/boost/corosio/native/detail/coro_op_complete.hpp +++ b/include/boost/corosio/native/detail/coro_op_complete.hpp @@ -25,7 +25,7 @@ 1. disarm the stop_callback, 2. on the shutdown-drain path (owner == nullptr) just break the - impl_ptr keepalive cycle and return without resuming, + object_ref_ keepalive cycle and return without resuming, 3. otherwise resume the coroutine on its executor, dropping the keepalive only after the continuation has been handed off. @@ -113,7 +113,7 @@ decode_io_result( @param self The completing op. @return True if this was a shutdown drain — the caller must `return` immediately without decoding or resuming. On that path the - impl_ptr keepalive is dropped here (which may destroy the impl, + object_ref_ keepalive is dropped here (which may destroy the impl, and with it the op storage). */ inline bool @@ -122,7 +122,7 @@ coro_drain_if_shutdown(void* owner, coro_op* self) noexcept self->stop_cb.reset(); if (owner == nullptr) { - auto suicide = std::move(self->impl_ptr); + auto suicide = std::move(self->object_ref_); return true; } return false; @@ -130,7 +130,7 @@ coro_drain_if_shutdown(void* owner, coro_op* self) noexcept /** Resume tail shared by every proactor handler. - Resumes the op's coroutine on its executor and then drops the impl_ptr + Resumes the op's coroutine on its executor and then drops the object_ref_ keepalive. The keepalive is moved into a local that is released *after* `resume()` returns, matching the existing io_uring ordering: the impl (and therefore this op's storage) may be destroyed as the local goes out of @@ -146,7 +146,9 @@ coro_resume(coro_op* self) noexcept // Clear the keepalive before publishing the continuation: a strand // drained on another thread can reuse this op via reset() the instant // it runs, so this write must be ordered before the publish, not after. - auto suicide = std::move(self->impl_ptr); + // dispatch_coro does not retain cont, so the op may be reused the + // moment the awaiter runs; nothing of self is touched after this. + auto suicide = std::move(self->object_ref_); auto next = dispatch_coro(self->ex, self->cont); next.resume(); // suicide drops here; may destroy impl + self. diff --git a/include/boost/corosio/native/detail/epoll/epoll_scheduler.hpp b/include/boost/corosio/native/detail/epoll/epoll_scheduler.hpp index 574c25665..953b110e7 100644 --- a/include/boost/corosio/native/detail/epoll/epoll_scheduler.hpp +++ b/include/boost/corosio/native/detail/epoll/epoll_scheduler.hpp @@ -263,7 +263,7 @@ epoll_scheduler::register_descriptor( desc->ready_events_.store(0, std::memory_order_relaxed); conditionally_enabled_mutex::scoped_lock lock(desc->mutex); - desc->impl_ref_.reset(); + desc->object_ref_.reset(); desc->read_ready = false; desc->write_ready = false; return {}; diff --git a/include/boost/corosio/native/detail/iocp/win_file_service.hpp b/include/boost/corosio/native/detail/iocp/win_file_service.hpp index 42dd5860a..73c937504 100644 --- a/include/boost/corosio/native/detail/iocp/win_file_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_file_service.hpp @@ -18,8 +18,7 @@ #include #include #include -#include -#include +#include #include #include #include @@ -42,11 +41,12 @@ namespace boost::corosio::detail { */ class BOOST_COROSIO_DECL win_file_service final : public file_service { + friend class win_stream_file; + public: using key_type = win_file_service; explicit win_file_service(capy::execution_context& ctx); - ~win_file_service(); win_file_service(win_file_service const&) = delete; win_file_service& operator=(win_file_service const&) = delete; @@ -61,9 +61,6 @@ class BOOST_COROSIO_DECL win_file_service final : public file_service std::filesystem::path const& path, file_base::flags mode) override; - void destroy_impl(win_stream_file& impl); - void unregister_impl(win_stream_file_internal& impl); - void post(overlapped_op* op); void on_pending(overlapped_op* op) noexcept; void on_completion(overlapped_op* op, DWORD error, DWORD bytes) noexcept; @@ -100,14 +97,12 @@ class BOOST_COROSIO_DECL win_file_service final : public file_service LONG(NTAPI*)(HANDLE, ULONG, void*, ULONG, io_status_block*); win_scheduler& sched_; + void* iocp_; + nt_flush_fn nt_flush_buffers_file_ex_; BOOST_COROSIO_MSVC_WARNING_PUSH BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // detail:: members, dll-interface - win_mutex mutex_; - intrusive_list file_list_; - intrusive_list wrapper_list_; + object_pool pool_; BOOST_COROSIO_MSVC_WARNING_POP - void* iocp_; - nt_flush_fn nt_flush_buffers_file_ex_; }; /** Get or create the stream file service for the given context. */ @@ -121,14 +116,14 @@ get_stream_file_service(capy::execution_context& ctx, win_scheduler&) // Operation constructors // --------------------------------------------------------------------------- -inline file_read_op::file_read_op(win_stream_file_internal& f) noexcept +inline file_read_op::file_read_op(win_stream_file& f) noexcept : overlapped_op(&do_complete) , file_(f) { cancel_func_ = &do_cancel_impl; } -inline file_write_op::file_write_op(win_stream_file_internal& f) noexcept +inline file_write_op::file_write_op(win_stream_file& f) noexcept : overlapped_op(&do_complete) , file_(f) { @@ -145,7 +140,7 @@ file_read_op::do_cancel_impl(overlapped_op* base) noexcept auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); if (op->file_.is_open()) - ::CancelIoEx(op->file_.native_handle(), op); + ::CancelIoEx(op->file_.handle_, op); } inline void @@ -154,7 +149,7 @@ file_write_op::do_cancel_impl(overlapped_op* base) noexcept auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); if (op->file_.is_open()) - ::CancelIoEx(op->file_.native_handle(), op); + ::CancelIoEx(op->file_.handle_, op); } // --------------------------------------------------------------------------- @@ -173,7 +168,7 @@ file_read_op::do_complete( if (!owner) { op->cleanup_only(); - op->file_ptr.reset(); + op->object_ref_.reset(); return; } @@ -181,7 +176,7 @@ file_read_op::do_complete( if (op->dwError == 0 && op->bytes_transferred > 0) op->file_.offset_ += op->bytes_transferred; - auto prevent_premature_destruction = std::move(op->file_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } @@ -197,7 +192,7 @@ file_write_op::do_complete( if (!owner) { op->cleanup_only(); - op->file_ptr.reset(); + op->object_ref_.reset(); return; } @@ -205,41 +200,44 @@ file_write_op::do_complete( if (op->dwError == 0 && op->bytes_transferred > 0) op->file_.offset_ += op->bytes_transferred; - auto prevent_premature_destruction = std::move(op->file_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } // --------------------------------------------------------------------------- -// win_stream_file_internal +// win_stream_file // --------------------------------------------------------------------------- -inline win_stream_file_internal::win_stream_file_internal( - win_file_service& svc) noexcept +inline win_stream_file::win_stream_file(win_file_service& svc) noexcept : svc_(svc) , rd_(*this) , wr_(*this) { } -inline win_stream_file_internal::~win_stream_file_internal() +inline void +win_stream_file::reuse() noexcept { - svc_.unregister_impl(*this); + BOOST_COROSIO_ASSERT(handle_ == INVALID_HANDLE_VALUE); + BOOST_COROSIO_ASSERT(offset_ == 0); + BOOST_COROSIO_ASSERT(!rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wr_.stop_cb); } -inline HANDLE -win_stream_file_internal::native_handle() const noexcept +inline native_handle_type +win_stream_file::native_handle() const noexcept { - return handle_; + return reinterpret_cast(handle_); } inline bool -win_stream_file_internal::is_open() const noexcept +win_stream_file::is_open() const noexcept { return handle_ != INVALID_HANDLE_VALUE; } inline void -win_stream_file_internal::cancel() noexcept +win_stream_file::cancel() noexcept { if (handle_ != INVALID_HANDLE_VALUE) ::CancelIoEx(handle_, nullptr); @@ -249,7 +247,7 @@ win_stream_file_internal::cancel() noexcept } inline void -win_stream_file_internal::close_handle() noexcept +win_stream_file::close_handle() noexcept { if (handle_ != INVALID_HANDLE_VALUE) { @@ -261,7 +259,7 @@ win_stream_file_internal::close_handle() noexcept } inline std::uint64_t -win_stream_file_internal::size() const +win_stream_file::size() const { LARGE_INTEGER li; if (!::GetFileSizeEx(handle_, &li)) @@ -270,7 +268,7 @@ win_stream_file_internal::size() const } inline std::error_code -win_stream_file_internal::resize(std::uint64_t new_size) noexcept +win_stream_file::resize(std::uint64_t new_size) noexcept { LARGE_INTEGER li; li.QuadPart = static_cast(new_size); @@ -282,7 +280,7 @@ win_stream_file_internal::resize(std::uint64_t new_size) noexcept } inline std::error_code -win_stream_file_internal::sync_data() noexcept +win_stream_file::sync_data() noexcept { // Attempt data-only flush; fall back to full flush if (svc_.try_flush_data(handle_)) @@ -293,7 +291,7 @@ win_stream_file_internal::sync_data() noexcept } inline std::error_code -win_stream_file_internal::sync_all() noexcept +win_stream_file::sync_all() noexcept { if (!::FlushFileBuffers(handle_)) return make_err(::GetLastError()); @@ -301,7 +299,7 @@ win_stream_file_internal::sync_all() noexcept } inline native_handle_type -win_stream_file_internal::release() +win_stream_file::release() { HANDLE h = handle_; handle_ = INVALID_HANDLE_VALUE; @@ -310,7 +308,7 @@ win_stream_file_internal::release() } inline std::error_code -win_stream_file_internal::assign(native_handle_type handle) noexcept +win_stream_file::assign(native_handle_type handle) noexcept { close_handle(); HANDLE h = reinterpret_cast(handle); @@ -326,8 +324,7 @@ win_stream_file_internal::assign(native_handle_type handle) noexcept } inline capy::io_result -win_stream_file_internal::seek( - std::int64_t offset, file_base::seek_basis origin) noexcept +win_stream_file::seek(std::int64_t offset, file_base::seek_basis origin) noexcept { // We manage offset_ ourselves (same as POSIX impl). std::int64_t new_pos; @@ -356,7 +353,7 @@ win_stream_file_internal::seek( } inline std::coroutine_handle<> -win_stream_file_internal::read_some( +win_stream_file::read_some( std::coroutine_handle<> h, capy::executor_ref ex, buffer_param param, @@ -366,8 +363,8 @@ win_stream_file_internal::read_some( { static constexpr std::size_t max_buffers = 16; - // Keep internal alive during I/O - rd_.file_ptr = shared_from_this(); + // Keep this file alive during I/O + rd_.object_ref_ = detail::object_ref(this); auto& op = rd_; op.reset(); @@ -428,7 +425,7 @@ win_stream_file_internal::read_some( } inline std::coroutine_handle<> -win_stream_file_internal::write_some( +win_stream_file::write_some( std::coroutine_handle<> h, capy::executor_ref ex, buffer_param param, @@ -438,8 +435,8 @@ win_stream_file_internal::write_some( { static constexpr std::size_t max_buffers = 16; - // Keep internal alive during I/O - wr_.file_ptr = shared_from_this(); + // Keep this file alive during I/O + wr_.object_ref_ = detail::object_ref(this); auto& op = wr_; op.reset(); @@ -497,109 +494,10 @@ win_stream_file_internal::write_some( return std::noop_coroutine(); } -// --------------------------------------------------------------------------- -// win_stream_file wrapper -// --------------------------------------------------------------------------- - -inline win_stream_file::win_stream_file( - std::shared_ptr internal) noexcept - : internal_(std::move(internal)) -{ -} - -inline void -win_stream_file::close_internal() noexcept -{ - if (internal_) - { - internal_->close_handle(); - internal_.reset(); - } -} - -inline std::coroutine_handle<> -win_stream_file::read_some( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->read_some(h, d, buf, token, ec, bytes); -} - -inline std::coroutine_handle<> -win_stream_file::write_some( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->write_some(h, d, buf, token, ec, bytes); -} - -inline native_handle_type -win_stream_file::native_handle() const noexcept -{ - return reinterpret_cast(internal_->native_handle()); -} - inline void -win_stream_file::cancel() noexcept -{ - internal_->cancel(); -} - -inline std::uint64_t -win_stream_file::size() const -{ - return internal_->size(); -} - -inline std::error_code -win_stream_file::resize(std::uint64_t new_size) noexcept -{ - return internal_->resize(new_size); -} - -inline std::error_code -win_stream_file::sync_data() noexcept -{ - return internal_->sync_data(); -} - -inline std::error_code -win_stream_file::sync_all() noexcept -{ - return internal_->sync_all(); -} - -inline native_handle_type -win_stream_file::release() -{ - return internal_->release(); -} - -inline std::error_code -win_stream_file::assign(native_handle_type handle) noexcept -{ - return internal_->assign(handle); -} - -inline capy::io_result -win_stream_file::seek( - std::int64_t offset, file_base::seek_basis origin) noexcept -{ - return internal_->seek(offset, origin); -} - -inline win_stream_file_internal* -win_stream_file::get_internal() const noexcept +win_stream_file::retire() noexcept { - return internal_.get(); + svc_.pool_.recycle(this); } // --------------------------------------------------------------------------- @@ -619,61 +517,32 @@ inline win_file_service::win_file_service(capy::execution_context& ctx) } } -inline win_file_service::~win_file_service() -{ - for (auto* w = wrapper_list_.pop_front(); w != nullptr; - w = wrapper_list_.pop_front()) - delete w; -} - inline io_object::implementation* win_file_service::construct() { - auto internal = std::make_shared(*this); - - { - std::lock_guard lock(mutex_); - file_list_.push_back(internal.get()); - } - - auto* wrapper = new win_stream_file(std::move(internal)); - - { - std::lock_guard lock(mutex_); - wrapper_list_.push_back(wrapper); - } - - return wrapper; + return pool_.acquire(*this); } inline void win_file_service::destroy(io_object::implementation* p) { - if (p) - { - auto& wrapper = static_cast(*p); - wrapper.close_internal(); - destroy_impl(wrapper); - } + if (!p) + return; + auto* f = static_cast(p); + f->close_handle(); + release(f); } inline void win_file_service::close(io_object::handle& h) { - auto& wrapper = static_cast(*h.get()); - wrapper.get_internal()->close_handle(); + static_cast(h.get())->close_handle(); } inline void win_file_service::shutdown() { - std::lock_guard lock(mutex_); - - for (auto* impl = file_list_.pop_front(); impl != nullptr; - impl = file_list_.pop_front()) - { - impl->close_handle(); - } + pool_.shutdown([](win_stream_file* impl) { impl->close_handle(); }); } inline std::error_code @@ -736,9 +605,9 @@ win_file_service::open_file( } } - auto& internal = *static_cast(impl).get_internal(); - internal.handle_ = h; - internal.offset_ = 0; + auto& file = static_cast(impl); + file.handle_ = h; + file.offset_ = 0; // Handle append: seek to end if (mode & file_base::append) @@ -746,34 +615,17 @@ win_file_service::open_file( LARGE_INTEGER sz; if (!::GetFileSizeEx(h, &sz)) { - DWORD err = ::GetLastError(); - internal.handle_ = INVALID_HANDLE_VALUE; + DWORD err = ::GetLastError(); + file.handle_ = INVALID_HANDLE_VALUE; ::CloseHandle(h); return make_err(err); } - internal.offset_ = static_cast(sz.QuadPart); + file.offset_ = static_cast(sz.QuadPart); } return {}; } -inline void -win_file_service::destroy_impl(win_stream_file& impl) -{ - { - std::lock_guard lock(mutex_); - wrapper_list_.remove(&impl); - } - delete &impl; -} - -inline void -win_file_service::unregister_impl(win_stream_file_internal& impl) -{ - std::lock_guard lock(mutex_); - file_list_.remove(&impl); -} - inline void win_file_service::post(overlapped_op* op) { diff --git a/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor.hpp b/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor.hpp index 89fd2c0e8..c455cdca1 100644 --- a/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor.hpp +++ b/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor.hpp @@ -17,13 +17,13 @@ #include #include #include +#include #include #include #include #include #include -#include #include #include @@ -33,7 +33,7 @@ namespace boost::corosio::detail { class win_local_stream_acceptor_service; class win_local_stream_service; class win_local_stream_socket; -class win_local_stream_acceptor_internal; +class win_local_stream_acceptor; /** Accept operation state for local stream sockets. @@ -43,7 +43,7 @@ struct local_stream_accept_op : overlapped_op { SOCKET accepted_socket = INVALID_SOCKET; win_local_stream_socket* peer_wrapper = nullptr; - std::shared_ptr acceptor_ptr; + win_local_stream_acceptor& acceptor; SOCKET listen_socket = INVALID_SOCKET; io_object::implementation** impl_out = nullptr; // 2 * (sizeof(un_sa_t) + 16) = 2 * (110 + 16) = 252 @@ -56,13 +56,13 @@ struct local_stream_accept_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - local_stream_accept_op() noexcept; + explicit local_stream_accept_op(win_local_stream_acceptor& acceptor_) noexcept; }; /** Readiness-wait operation state for a local stream acceptor. */ struct local_stream_acceptor_wait_op : overlapped_op { - std::shared_ptr acceptor_ptr; + win_local_stream_acceptor& acceptor; SOCKET listen_socket = INVALID_SOCKET; static void do_complete( @@ -72,66 +72,48 @@ struct local_stream_acceptor_wait_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - local_stream_acceptor_wait_op() noexcept; + explicit local_stream_acceptor_wait_op( + win_local_stream_acceptor& acceptor_) noexcept; }; -/* Internal acceptor state for IOCP local stream I/O. */ -class win_local_stream_acceptor_internal - : public intrusive_list::node - , public std::enable_shared_from_this +/** Acceptor implementation for IOCP local stream I/O. + + Collapses the historical internal-state/wrapper split into one + pooled `io_object::implementation`. +*/ +class win_local_stream_acceptor final + : public local_stream_acceptor::implementation + , public intrusive_list::node { friend class win_local_stream_service; - friend class win_local_stream_acceptor; - -public: - explicit win_local_stream_acceptor_internal( - win_local_stream_service& svc) noexcept; - ~win_local_stream_acceptor_internal(); - - win_local_stream_service& socket_service() noexcept; - - std::coroutine_handle<> accept( - std::coroutine_handle<>, - capy::executor_ref, - std::stop_token, - std::error_code*, - io_object::implementation**); - - std::coroutine_handle<> wait( - std::coroutine_handle<>, - capy::executor_ref, - wait_type, - std::stop_token, - std::error_code*); - - SOCKET native_handle() const noexcept; - corosio::local_endpoint local_endpoint() const noexcept; - bool is_open() const noexcept; - void cancel() noexcept; - void close_socket() noexcept; - void set_local_endpoint(corosio::local_endpoint ep) noexcept; + friend struct local_stream_accept_op; + friend struct local_stream_acceptor_wait_op; + win_local_stream_service& svc_; local_stream_accept_op acc_; local_stream_acceptor_wait_op wt_; - -private: - win_local_stream_service& svc_; SOCKET socket_ = INVALID_SOCKET; corosio::local_endpoint local_endpoint_; -}; - -/* Acceptor implementation wrapper for IOCP local stream I/O. */ -class win_local_stream_acceptor final - : public local_stream_acceptor::implementation - , public intrusive_list::node -{ - std::shared_ptr internal_; public: - explicit win_local_stream_acceptor( - std::shared_ptr internal) noexcept; + explicit win_local_stream_acceptor(win_local_stream_service& svc) noexcept; - void close_internal() noexcept; + /// Recycle into the owning service's pool. Defined out-of-line + /// after `win_local_stream_service` is complete (needs + /// `svc_.acceptor_pool_`). + void retire() noexcept override; + + /** Reset recycled state for reuse. + + Each op's `OVERLAPPED` fields are re-seeded by + `overlapped_op::reset()` at its next submit, so they are not + zeroed here. + + @pre refs_ == 0, socket closed, no op in flight. + */ + void reuse() noexcept; + + win_local_stream_service& socket_service() noexcept; std::coroutine_handle<> accept( std::coroutine_handle<> h, @@ -169,7 +151,8 @@ class win_local_stream_acceptor final get_option(int level, int optname, void* data, std::size_t* size) const noexcept override; - win_local_stream_acceptor_internal* get_internal() const noexcept; + void set_local_endpoint(corosio::local_endpoint ep) noexcept; + void close_socket() noexcept; }; } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor_service.hpp b/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor_service.hpp index 39aafb6d3..d8cc3e723 100644 --- a/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_local_stream_acceptor_service.hpp @@ -43,8 +43,7 @@ class BOOST_COROSIO_DECL win_local_stream_acceptor_service final : public local_stream_acceptor_service { public: - explicit win_local_stream_acceptor_service( - capy::execution_context& ctx); + explicit win_local_stream_acceptor_service(capy::execution_context& ctx); io_object::implementation* construct() override; @@ -79,8 +78,10 @@ class BOOST_COROSIO_DECL win_local_stream_acceptor_service final // local_stream_accept_op // ============================================================ -inline local_stream_accept_op::local_stream_accept_op() noexcept +inline local_stream_accept_op::local_stream_accept_op( + win_local_stream_acceptor& acceptor_) noexcept : overlapped_op(&do_complete) + , acceptor(acceptor_) { cancel_func_ = &do_cancel_impl; } @@ -95,8 +96,10 @@ local_stream_accept_op::do_cancel_impl(overlapped_op* base) noexcept } } -inline local_stream_acceptor_wait_op::local_stream_acceptor_wait_op() noexcept +inline local_stream_acceptor_wait_op::local_stream_acceptor_wait_op( + win_local_stream_acceptor& acceptor_) noexcept : overlapped_op(&do_complete) + , acceptor(acceptor_) { cancel_func_ = &do_cancel_impl; } @@ -110,10 +113,7 @@ local_stream_acceptor_wait_op::do_cancel_impl(overlapped_op* base) noexcept { ::CancelIoEx(reinterpret_cast(op->listen_socket), op); } - if (op->acceptor_ptr) - { - op->acceptor_ptr->socket_service().scheduler().cancel_wait(op); - } + op->acceptor.socket_service().scheduler().cancel_wait(op); } // accept_op completion handler @@ -137,12 +137,12 @@ local_stream_accept_op::do_complete( if (op->peer_wrapper) { - op->peer_wrapper->close_internal(); + op->peer_wrapper->close_socket(); op->peer_wrapper = nullptr; } op->cleanup_only(); - op->acceptor_ptr.reset(); + op->object_ref_.reset(); return; } @@ -167,7 +167,7 @@ local_stream_accept_op::do_complete( op->accepted_socket, SOL_SOCKET, SO_UPDATE_ACCEPT_CONTEXT, reinterpret_cast(&op->listen_socket), sizeof(SOCKET)); - op->peer_wrapper->get_internal()->set_socket(op->accepted_socket); + op->peer_wrapper->set_socket(op->accepted_socket); sockaddr_storage local_storage{}; int local_len = sizeof(local_storage); @@ -186,11 +186,14 @@ local_stream_accept_op::do_complete( remote_ep = from_sockaddr_local( remote_storage, static_cast(remote_len)); - op->peer_wrapper->get_internal()->set_endpoints(local_ep, remote_ep); + op->peer_wrapper->set_endpoints(local_ep, remote_ep); op->accepted_socket = INVALID_SOCKET; if (op->impl_out) *op->impl_out = op->peer_wrapper; + // Handed off to the caller: don't let a failed early return on + // the next accept() see this (now user-owned) pointer again. + op->peer_wrapper = nullptr; } else { @@ -202,7 +205,7 @@ local_stream_accept_op::do_complete( if (op->peer_wrapper) { - op->acceptor_ptr->socket_service().destroy(op->peer_wrapper); + op->acceptor.socket_service().destroy(op->peer_wrapper); op->peer_wrapper = nullptr; } @@ -212,7 +215,7 @@ local_stream_accept_op::do_complete( op->cont.h = op->h; auto saved_ex = op->ex; - auto prevent_premature_destruction = std::move(op->acceptor_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); dispatch_coro(saved_ex, op->cont).resume(); } @@ -229,62 +232,70 @@ local_stream_acceptor_wait_op::do_complete( if (!owner) { op->cleanup_only(); - op->acceptor_ptr.reset(); + op->object_ref_.reset(); return; } - auto prevent_premature_destruction = std::move(op->acceptor_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } // ============================================================ -// win_local_stream_acceptor_internal +// win_local_stream_acceptor // ============================================================ -inline win_local_stream_acceptor_internal::win_local_stream_acceptor_internal( +inline win_local_stream_acceptor::win_local_stream_acceptor( win_local_stream_service& svc) noexcept : svc_(svc) + , acc_(*this) + , wt_(*this) { } -inline win_local_stream_acceptor_internal::~win_local_stream_acceptor_internal() +inline void +win_local_stream_acceptor::reuse() noexcept { - svc_.unregister_acceptor_impl(*this); + BOOST_COROSIO_ASSERT(socket_ == INVALID_SOCKET); + BOOST_COROSIO_ASSERT(local_endpoint_ == corosio::local_endpoint{}); + BOOST_COROSIO_ASSERT(!acc_.stop_cb); + BOOST_COROSIO_ASSERT(!wt_.stop_cb); + BOOST_COROSIO_ASSERT(acc_.peer_wrapper == nullptr); + BOOST_COROSIO_ASSERT(acc_.accepted_socket == INVALID_SOCKET); } inline win_local_stream_service& -win_local_stream_acceptor_internal::socket_service() noexcept +win_local_stream_acceptor::socket_service() noexcept { return svc_; } -inline SOCKET -win_local_stream_acceptor_internal::native_handle() const noexcept +inline native_handle_type +win_local_stream_acceptor::native_handle() const noexcept { - return socket_; + return static_cast(socket_); } inline corosio::local_endpoint -win_local_stream_acceptor_internal::local_endpoint() const noexcept +win_local_stream_acceptor::local_endpoint() const noexcept { return local_endpoint_; } inline bool -win_local_stream_acceptor_internal::is_open() const noexcept +win_local_stream_acceptor::is_open() const noexcept { return socket_ != INVALID_SOCKET; } inline void -win_local_stream_acceptor_internal::set_local_endpoint( +win_local_stream_acceptor::set_local_endpoint( corosio::local_endpoint ep) noexcept { local_endpoint_ = ep; } inline void -win_local_stream_acceptor_internal::cancel() noexcept +win_local_stream_acceptor::cancel() noexcept { if (socket_ != INVALID_SOCKET) ::CancelIoEx(reinterpret_cast(socket_), nullptr); @@ -294,14 +305,14 @@ win_local_stream_acceptor_internal::cancel() noexcept } inline std::coroutine_handle<> -win_local_stream_acceptor_internal::wait( +win_local_stream_acceptor::wait( std::coroutine_handle<> h, capy::executor_ref d, wait_type w, std::stop_token token, std::error_code* ec) { - wt_.acceptor_ptr = shared_from_this(); + wt_.object_ref_ = detail::object_ref(this); wt_.listen_socket = socket_; auto& op = wt_; @@ -337,7 +348,7 @@ win_local_stream_acceptor_internal::wait( } inline void -win_local_stream_acceptor_internal::close_socket() noexcept +win_local_stream_acceptor::close_socket() noexcept { // Flag the accept op cancelled before closing so a closesocket-delivered // ERROR_NETNAME_DELETED is short-circuited to canceled rather than mapped @@ -356,26 +367,33 @@ win_local_stream_acceptor_internal::close_socket() noexcept } inline std::coroutine_handle<> -win_local_stream_acceptor_internal::accept( +win_local_stream_acceptor::accept( std::coroutine_handle<> h, capy::executor_ref d, std::stop_token token, std::error_code* ec, io_object::implementation** impl_out) { - acc_.acceptor_ptr = shared_from_this(); + acc_.object_ref_ = detail::object_ref(this); auto& op = acc_; op.reset(); - op.h = h; - op.ex = d; - op.ec_out = ec; - op.impl_out = impl_out; + // reset() doesn't touch these -- a value left over from a prior + // accept() (e.g. the socket already handed to the user via + // *impl_out) must not survive into a completion this call fails + // before reassigning; do_complete's failure branch would destroy() + // whatever stale pointer it finds here. + op.peer_wrapper = nullptr; + op.accepted_socket = INVALID_SOCKET; + op.h = h; + op.ex = d; + op.ec_out = ec; + op.impl_out = impl_out; op.start(token); svc_.work_started(); - // Create wrapper for the peer socket + // Create a (possibly recycled) peer socket auto* peer_ptr = svc_.construct(); if (!peer_ptr) { @@ -453,87 +471,16 @@ win_local_stream_acceptor_internal::accept( return std::noop_coroutine(); } -// ============================================================ -// win_local_stream_acceptor (wrapper) -// ============================================================ - -inline win_local_stream_acceptor::win_local_stream_acceptor( - std::shared_ptr internal) noexcept - : internal_(std::move(internal)) -{ -} - -inline void -win_local_stream_acceptor::close_internal() noexcept -{ - if (internal_) - { - internal_->close_socket(); - internal_.reset(); - } -} - -inline std::coroutine_handle<> -win_local_stream_acceptor::accept( - std::coroutine_handle<> h, - capy::executor_ref d, - std::stop_token token, - std::error_code* ec, - io_object::implementation** impl_out) -{ - return internal_->accept(h, d, token, ec, impl_out); -} - -inline std::coroutine_handle<> -win_local_stream_acceptor::wait( - std::coroutine_handle<> h, - capy::executor_ref d, - wait_type w, - std::stop_token token, - std::error_code* ec) -{ - return internal_->wait(h, d, w, token, ec); -} - -inline corosio::local_endpoint -win_local_stream_acceptor::local_endpoint() const noexcept -{ - return internal_->local_endpoint(); -} - -inline bool -win_local_stream_acceptor::is_open() const noexcept -{ - return internal_ && internal_->is_open(); -} - -inline void -win_local_stream_acceptor::cancel() noexcept -{ - if (internal_) - internal_->cancel(); -} - -inline native_handle_type -win_local_stream_acceptor::native_handle() const noexcept -{ - if (!internal_) - return static_cast(INVALID_SOCKET); - return static_cast(internal_->native_handle()); -} - inline native_handle_type win_local_stream_acceptor::release_socket() noexcept { - if (!internal_) - return static_cast(INVALID_SOCKET); - SOCKET s = internal_->socket_; + SOCKET s = socket_; if (s != INVALID_SOCKET) { - internal_->cancel(); + cancel(); dissociate_from_iocp(s); - internal_->socket_ = INVALID_SOCKET; - internal_->local_endpoint_ = corosio::local_endpoint{}; + socket_ = INVALID_SOCKET; + local_endpoint_ = corosio::local_endpoint{}; } return static_cast(s); } @@ -542,11 +489,11 @@ inline std::error_code win_local_stream_acceptor::set_option( int level, int optname, void const* data, std::size_t size) noexcept { - if (!internal_ || !internal_->is_open()) + if (socket_ == INVALID_SOCKET) return make_err(WSAENOTSOCK); if (::setsockopt( - internal_->native_handle(), level, optname, - reinterpret_cast(data), static_cast(size)) != 0) + socket_, level, optname, reinterpret_cast(data), + static_cast(size)) != 0) return make_err(WSAGetLastError()); return {}; } @@ -555,21 +502,20 @@ inline std::error_code win_local_stream_acceptor::get_option( int level, int optname, void* data, std::size_t* size) const noexcept { - if (!internal_ || !internal_->is_open()) + if (socket_ == INVALID_SOCKET) return make_err(WSAENOTSOCK); int len = static_cast(*size); if (::getsockopt( - internal_->native_handle(), level, optname, - reinterpret_cast(data), &len) != 0) + socket_, level, optname, reinterpret_cast(data), &len) != 0) return make_err(WSAGetLastError()); *size = static_cast(len); return {}; } -inline win_local_stream_acceptor_internal* -win_local_stream_acceptor::get_internal() const noexcept +inline void +win_local_stream_acceptor::retire() noexcept { - return internal_.get(); + svc_.acceptor_pool_.recycle(this); } // ============================================================ @@ -585,38 +531,23 @@ inline win_local_stream_acceptor_service::win_local_stream_acceptor_service( inline io_object::implementation* win_local_stream_acceptor_service::construct() { - auto internal = std::make_shared(svc_); - - // Allocate wrapper before mutating lists so a throw from - // new doesn't leave a dangling pointer in acceptor_list_. - auto* raw = internal.get(); - auto* wrapper = new win_local_stream_acceptor(std::move(internal)); - - { - std::lock_guard lock(svc_.mutex_); - svc_.acceptor_list_.push_back(raw); - svc_.acceptor_wrapper_list_.push_back(wrapper); - } - - return wrapper; + return svc_.acquire_acceptor_impl(); } inline void win_local_stream_acceptor_service::destroy(io_object::implementation* p) { - if (p) - { - auto& wrapper = static_cast(*p); - wrapper.close_internal(); - svc_.destroy_acceptor_impl(wrapper); - } + if (!p) + return; + auto* a = static_cast(p); + a->close_socket(); + release(a); } inline void win_local_stream_acceptor_service::close(io_object::handle& h) { - auto& wrapper = static_cast(*h.get()); - wrapper.get_internal()->close_socket(); + static_cast(h.get())->close_socket(); } inline std::error_code @@ -626,36 +557,32 @@ win_local_stream_acceptor_service::open_acceptor_socket( int type, int protocol) { - auto* internal = - static_cast(impl).get_internal(); - return svc_.open_acceptor_socket(*internal, family, type, protocol); + auto& acc = static_cast(impl); + return svc_.open_acceptor_socket(acc, family, type, protocol); } inline std::error_code win_local_stream_acceptor_service::assign_socket( local_stream_acceptor::implementation& impl, native_handle_type fd) { - auto* internal = - static_cast(impl).get_internal(); - return svc_.assign_acceptor_socket(*internal, fd); + auto& acc = static_cast(impl); + return svc_.assign_acceptor_socket(acc, fd); } inline std::error_code win_local_stream_acceptor_service::bind_acceptor( local_stream_acceptor::implementation& impl, corosio::local_endpoint ep) { - auto* internal = - static_cast(impl).get_internal(); - return svc_.bind_acceptor(*internal, ep); + auto& acc = static_cast(impl); + return svc_.bind_acceptor(acc, ep); } inline std::error_code win_local_stream_acceptor_service::listen_acceptor( local_stream_acceptor::implementation& impl, int backlog) { - auto* internal = - static_cast(impl).get_internal(); - return svc_.listen_acceptor(*internal, backlog); + auto& acc = static_cast(impl); + return svc_.listen_acceptor(acc, backlog); } inline void diff --git a/include/boost/corosio/native/detail/iocp/win_local_stream_service.hpp b/include/boost/corosio/native/detail/iocp/win_local_stream_service.hpp index 1125931d4..e6cac7f25 100644 --- a/include/boost/corosio/native/detail/iocp/win_local_stream_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_local_stream_service.hpp @@ -15,6 +15,7 @@ #if BOOST_COROSIO_HAS_IOCP #include +#include #include #include @@ -23,7 +24,6 @@ #include #include #include -#include #include #include @@ -34,10 +34,6 @@ namespace boost::corosio::detail { -class win_local_stream_acceptor; -class win_local_stream_acceptor_internal; -class win_local_stream_acceptor_service; - /* IOCP local stream socket service. Inherits from local_stream_service to enable runtime polymorphism @@ -48,6 +44,9 @@ class BOOST_COROSIO_DECL win_local_stream_service final : private win_wsa_init , public local_stream_service { + friend class win_local_stream_socket; + friend class win_local_stream_acceptor; + public: io_object::implementation* construct() override; @@ -57,8 +56,6 @@ class BOOST_COROSIO_DECL win_local_stream_service final explicit win_local_stream_service(capy::execution_context& ctx); - ~win_local_stream_service(); - win_local_stream_service(win_local_stream_service const&) = delete; win_local_stream_service& operator=(win_local_stream_service const&) = delete; @@ -75,34 +72,22 @@ class BOOST_COROSIO_DECL win_local_stream_service final local_stream_socket::implementation& impl, native_handle_type fd) override; - void destroy_impl(win_local_stream_socket& impl); - - void unregister_impl(win_local_stream_socket_internal& impl); - std::error_code open_socket_internal( - win_local_stream_socket_internal& impl, - int family, - int type, - int protocol); + win_local_stream_socket& impl, int family, int type, int protocol); - void destroy_acceptor_impl(win_local_stream_acceptor& impl); - - void unregister_acceptor_impl(win_local_stream_acceptor_internal& impl); + win_local_stream_acceptor* acquire_acceptor_impl(); std::error_code open_acceptor_socket( - win_local_stream_acceptor_internal& impl, - int family, - int type, - int protocol); + win_local_stream_acceptor& impl, int family, int type, int protocol); std::error_code assign_acceptor_socket( - win_local_stream_acceptor_internal& impl, native_handle_type fd); + win_local_stream_acceptor& impl, native_handle_type fd); - std::error_code bind_acceptor( - win_local_stream_acceptor_internal& impl, corosio::local_endpoint ep); + std::error_code + bind_acceptor(win_local_stream_acceptor& impl, corosio::local_endpoint ep); std::error_code - listen_acceptor(win_local_stream_acceptor_internal& impl, int backlog); + listen_acceptor(win_local_stream_acceptor& impl, int backlog); void* native_handle() const noexcept; LPFN_CONNECTEX connect_ex() const noexcept; @@ -121,19 +106,14 @@ class BOOST_COROSIO_DECL win_local_stream_service final } private: - friend class win_local_stream_acceptor_service; - win_tcp_service& tcp_svc_; win_scheduler& sched_; + void* iocp_; BOOST_COROSIO_MSVC_WARNING_PUSH BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // detail:: members, dll-interface - win_mutex mutex_; - intrusive_list socket_list_; - intrusive_list acceptor_list_; - intrusive_list socket_wrapper_list_; - intrusive_list acceptor_wrapper_list_; + object_pool pool_; + object_pool acceptor_pool_; BOOST_COROSIO_MSVC_WARNING_POP - void* iocp_; }; // ============================================================ @@ -141,7 +121,7 @@ class BOOST_COROSIO_DECL win_local_stream_service final // ============================================================ inline local_stream_connect_op::local_stream_connect_op( - win_local_stream_socket_internal& internal_) noexcept + win_local_stream_socket& internal_) noexcept : overlapped_op(&do_complete) , internal(internal_) { @@ -149,7 +129,7 @@ inline local_stream_connect_op::local_stream_connect_op( } inline local_stream_read_op::local_stream_read_op( - win_local_stream_socket_internal& internal_) noexcept + win_local_stream_socket& internal_) noexcept : overlapped_op(&do_complete) , internal(internal_) { @@ -157,7 +137,7 @@ inline local_stream_read_op::local_stream_read_op( } inline local_stream_write_op::local_stream_write_op( - win_local_stream_socket_internal& internal_) noexcept + win_local_stream_socket& internal_) noexcept : overlapped_op(&do_complete) , internal(internal_) { @@ -165,7 +145,7 @@ inline local_stream_write_op::local_stream_write_op( } inline local_stream_wait_op::local_stream_wait_op( - win_local_stream_socket_internal& internal_) noexcept + win_local_stream_socket& internal_) noexcept : overlapped_op(&do_complete) , internal(internal_) { @@ -181,10 +161,9 @@ local_stream_connect_op::do_cancel_impl(overlapped_op* base) noexcept { auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) + if (op->internal.socket_ != INVALID_SOCKET) { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); } } @@ -193,10 +172,9 @@ local_stream_read_op::do_cancel_impl(overlapped_op* base) noexcept { auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) + if (op->internal.socket_ != INVALID_SOCKET) { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); } } @@ -205,10 +183,9 @@ local_stream_write_op::do_cancel_impl(overlapped_op* base) noexcept { auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) + if (op->internal.socket_ != INVALID_SOCKET) { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); } } @@ -217,10 +194,9 @@ local_stream_wait_op::do_cancel_impl(overlapped_op* base) noexcept { auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) + if (op->internal.socket_ != INVALID_SOCKET) { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); } op->internal.svc_.scheduler().cancel_wait(op); } @@ -241,31 +217,31 @@ local_stream_connect_op::do_complete( if (!owner) { op->cleanup_only(); - op->internal_ptr.reset(); + op->object_ref_.reset(); return; } bool success = (op->dwError == 0 && !op->cancelled.load(std::memory_order_acquire)); - if (success && op->internal.is_open()) + if (success && op->internal.socket_ != INVALID_SOCKET) { // Required after ConnectEx ::setsockopt( - op->internal.native_handle(), SOL_SOCKET, SO_UPDATE_CONNECT_CONTEXT, + op->internal.socket_, SOL_SOCKET, SO_UPDATE_CONNECT_CONTEXT, nullptr, 0); corosio::local_endpoint local_ep; sockaddr_storage local_storage{}; int local_len = sizeof(local_storage); if (::getsockname( - op->internal.native_handle(), + op->internal.socket_, reinterpret_cast(&local_storage), &local_len) == 0) local_ep = from_sockaddr_local( local_storage, static_cast(local_len)); op->internal.set_endpoints(local_ep, op->target_endpoint); } - auto prevent_premature_destruction = std::move(op->internal_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } @@ -285,11 +261,11 @@ local_stream_read_op::do_complete( if (!owner) { op->cleanup_only(); - op->internal_ptr.reset(); + op->object_ref_.reset(); return; } - auto prevent_premature_destruction = std::move(op->internal_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } @@ -309,11 +285,11 @@ local_stream_write_op::do_complete( if (!owner) { op->cleanup_only(); - op->internal_ptr.reset(); + op->object_ref_.reset(); return; } - auto prevent_premature_destruction = std::move(op->internal_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } @@ -333,19 +309,19 @@ local_stream_wait_op::do_complete( if (!owner) { op->cleanup_only(); - op->internal_ptr.reset(); + op->object_ref_.reset(); return; } - auto prevent_premature_destruction = std::move(op->internal_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } // ============================================================ -// win_local_stream_socket_internal +// win_local_stream_socket // ============================================================ -inline win_local_stream_socket_internal::win_local_stream_socket_internal( +inline win_local_stream_socket::win_local_stream_socket( win_local_stream_service& svc) noexcept : svc_(svc) , conn_(*this) @@ -355,43 +331,26 @@ inline win_local_stream_socket_internal::win_local_stream_socket_internal( { } -inline win_local_stream_socket_internal::~win_local_stream_socket_internal() -{ - svc_.unregister_impl(*this); -} - -inline SOCKET -win_local_stream_socket_internal::native_handle() const noexcept -{ - return socket_; -} - -inline corosio::local_endpoint -win_local_stream_socket_internal::local_endpoint() const noexcept -{ - return local_endpoint_; -} - -inline corosio::local_endpoint -win_local_stream_socket_internal::remote_endpoint() const noexcept -{ - return remote_endpoint_; -} - -inline bool -win_local_stream_socket_internal::is_open() const noexcept +inline void +win_local_stream_socket::reuse() noexcept { - return socket_ != INVALID_SOCKET; + BOOST_COROSIO_ASSERT(socket_ == INVALID_SOCKET); + BOOST_COROSIO_ASSERT(local_endpoint_ == corosio::local_endpoint{}); + BOOST_COROSIO_ASSERT(remote_endpoint_ == corosio::local_endpoint{}); + BOOST_COROSIO_ASSERT(!conn_.stop_cb); + BOOST_COROSIO_ASSERT(!rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wr_.stop_cb); + BOOST_COROSIO_ASSERT(!wt_.stop_cb); } inline void -win_local_stream_socket_internal::set_socket(SOCKET s) noexcept +win_local_stream_socket::set_socket(SOCKET s) noexcept { socket_ = s; } inline void -win_local_stream_socket_internal::set_endpoints( +win_local_stream_socket::set_endpoints( corosio::local_endpoint local, corosio::local_endpoint remote) noexcept { local_endpoint_ = local; @@ -399,14 +358,14 @@ win_local_stream_socket_internal::set_endpoints( } inline std::coroutine_handle<> -win_local_stream_socket_internal::connect( +win_local_stream_socket::connect( std::coroutine_handle<> h, capy::executor_ref d, corosio::local_endpoint ep, std::stop_token token, std::error_code* ec) { - conn_.internal_ptr = shared_from_this(); + conn_.object_ref_ = detail::object_ref(this); auto& op = conn_; op.reset(); @@ -469,7 +428,7 @@ win_local_stream_socket_internal::connect( } inline std::coroutine_handle<> -win_local_stream_socket_internal::read_some( +win_local_stream_socket::read_some( std::coroutine_handle<> h, capy::executor_ref d, buffer_param param, @@ -477,7 +436,7 @@ win_local_stream_socket_internal::read_some( std::error_code* ec, std::size_t* bytes_out) { - rd_.internal_ptr = shared_from_this(); + rd_.object_ref_ = detail::object_ref(this); auto& op = rd_; op.reset(); @@ -540,7 +499,7 @@ win_local_stream_socket_internal::read_some( } inline std::coroutine_handle<> -win_local_stream_socket_internal::write_some( +win_local_stream_socket::write_some( std::coroutine_handle<> h, capy::executor_ref d, buffer_param param, @@ -548,7 +507,7 @@ win_local_stream_socket_internal::write_some( std::error_code* ec, std::size_t* bytes_out) { - wr_.internal_ptr = shared_from_this(); + wr_.object_ref_ = detail::object_ref(this); auto& op = wr_; op.reset(); @@ -607,14 +566,14 @@ win_local_stream_socket_internal::write_some( } inline std::coroutine_handle<> -win_local_stream_socket_internal::wait( +win_local_stream_socket::wait( std::coroutine_handle<> h, capy::executor_ref d, wait_type w, std::stop_token token, std::error_code* ec) { - wt_.internal_ptr = shared_from_this(); + wt_.object_ref_ = detail::object_ref(this); auto& op = wt_; op.reset(); @@ -672,7 +631,7 @@ win_local_stream_socket_internal::wait( } inline void -win_local_stream_socket_internal::cancel() noexcept +win_local_stream_socket::cancel() noexcept { if (socket_ != INVALID_SOCKET) { @@ -687,7 +646,7 @@ win_local_stream_socket_internal::cancel() noexcept } inline void -win_local_stream_socket_internal::close_socket() noexcept +win_local_stream_socket::close_socket() noexcept { // Flag every op cancelled before closing so a closesocket-delivered // ERROR_NETNAME_DELETED is short-circuited to canceled rather than mapped @@ -709,72 +668,6 @@ win_local_stream_socket_internal::close_socket() noexcept remote_endpoint_ = corosio::local_endpoint{}; } -// ============================================================ -// win_local_stream_socket (wrapper) -// ============================================================ - -inline win_local_stream_socket::win_local_stream_socket( - std::shared_ptr internal) noexcept - : internal_(std::move(internal)) -{ -} - -inline void -win_local_stream_socket::close_internal() noexcept -{ - if (internal_) - { - internal_->close_socket(); - internal_.reset(); - } -} - -inline std::coroutine_handle<> -win_local_stream_socket::connect( - std::coroutine_handle<> h, - capy::executor_ref d, - corosio::local_endpoint ep, - std::stop_token token, - std::error_code* ec) -{ - return internal_->connect(h, d, ep, token, ec); -} - -inline std::coroutine_handle<> -win_local_stream_socket::read_some( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->read_some(h, d, buf, token, ec, bytes); -} - -inline std::coroutine_handle<> -win_local_stream_socket::write_some( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->write_some(h, d, buf, token, ec, bytes); -} - -inline std::coroutine_handle<> -win_local_stream_socket::wait( - std::coroutine_handle<> h, - capy::executor_ref d, - wait_type w, - std::stop_token token, - std::error_code* ec) -{ - return internal_->wait(h, d, w, token, ec); -} - inline std::error_code win_local_stream_socket::shutdown( local_stream_socket::shutdown_type what) noexcept @@ -794,7 +687,7 @@ win_local_stream_socket::shutdown( default: return make_err(WSAEINVAL); } - if (::shutdown(internal_->native_handle(), how) != 0) + if (::shutdown(socket_, how) != 0) return make_err(WSAGetLastError()); return {}; } @@ -802,23 +695,23 @@ win_local_stream_socket::shutdown( inline native_handle_type win_local_stream_socket::native_handle() const noexcept { - return static_cast(internal_->native_handle()); + return static_cast(socket_); } inline native_handle_type win_local_stream_socket::release_socket() noexcept { - SOCKET s = internal_->socket_; + SOCKET s = socket_; if (s != INVALID_SOCKET) { - internal_->cancel(); + cancel(); // Sever the port association so the descriptor can be // adopted again; best-effort, the caller keeps a working // socket either way. dissociate_from_iocp(s); - internal_->socket_ = INVALID_SOCKET; - internal_->local_endpoint_ = corosio::local_endpoint{}; - internal_->remote_endpoint_ = corosio::local_endpoint{}; + socket_ = INVALID_SOCKET; + local_endpoint_ = corosio::local_endpoint{}; + remote_endpoint_ = corosio::local_endpoint{}; } return static_cast(s); } @@ -828,8 +721,8 @@ win_local_stream_socket::set_option( int level, int optname, void const* data, std::size_t size) noexcept { if (::setsockopt( - internal_->native_handle(), level, optname, - reinterpret_cast(data), static_cast(size)) != 0) + socket_, level, optname, reinterpret_cast(data), + static_cast(size)) != 0) return make_err(WSAGetLastError()); return {}; } @@ -840,8 +733,7 @@ win_local_stream_socket::get_option( { int len = static_cast(*size); if (::getsockopt( - internal_->native_handle(), level, optname, - reinterpret_cast(data), &len) != 0) + socket_, level, optname, reinterpret_cast(data), &len) != 0) return make_err(WSAGetLastError()); *size = static_cast(len); return {}; @@ -850,25 +742,19 @@ win_local_stream_socket::get_option( inline corosio::local_endpoint win_local_stream_socket::local_endpoint() const noexcept { - return internal_->local_endpoint(); + return local_endpoint_; } inline corosio::local_endpoint win_local_stream_socket::remote_endpoint() const noexcept { - return internal_->remote_endpoint(); + return remote_endpoint_; } inline void -win_local_stream_socket::cancel() noexcept -{ - internal_->cancel(); -} - -inline win_local_stream_socket_internal* -win_local_stream_socket::get_internal() const noexcept +win_local_stream_socket::retire() noexcept { - return internal_.get(); + svc_.pool_.recycle(this); } // ============================================================ @@ -883,89 +769,36 @@ inline win_local_stream_service::win_local_stream_service( { } -inline win_local_stream_service::~win_local_stream_service() -{ - for (auto* w = socket_wrapper_list_.pop_front(); w != nullptr; - w = socket_wrapper_list_.pop_front()) - delete w; - - for (auto* w = acceptor_wrapper_list_.pop_front(); w != nullptr; - w = acceptor_wrapper_list_.pop_front()) - delete w; -} - inline void win_local_stream_service::shutdown() { - std::lock_guard lock(mutex_); + pool_.shutdown( + [](win_local_stream_socket* impl) { impl->close_socket(); }); - for (auto* impl = socket_list_.pop_front(); impl != nullptr; - impl = socket_list_.pop_front()) - { - impl->close_socket(); - } - - for (auto* impl = acceptor_list_.pop_front(); impl != nullptr; - impl = acceptor_list_.pop_front()) - { - impl->close_socket(); - } + acceptor_pool_.shutdown( + [](win_local_stream_acceptor* impl) { impl->close_socket(); }); } inline io_object::implementation* win_local_stream_service::construct() { - auto internal = std::make_shared(*this); - - { - std::lock_guard lock(mutex_); - socket_list_.push_back(internal.get()); - } - - auto* wrapper = new win_local_stream_socket(std::move(internal)); - - { - std::lock_guard lock(mutex_); - socket_wrapper_list_.push_back(wrapper); - } - - return wrapper; + return pool_.acquire(*this); } inline void win_local_stream_service::destroy(io_object::implementation* p) { - if (p) - { - auto& wrapper = static_cast(*p); - wrapper.close_internal(); - destroy_impl(wrapper); - } + if (!p) + return; + auto* s = static_cast(p); + s->close_socket(); + release(s); } inline void win_local_stream_service::close(io_object::handle& h) { - auto& wrapper = static_cast(*h.get()); - wrapper.get_internal()->close_socket(); -} - -inline void -win_local_stream_service::destroy_impl(win_local_stream_socket& impl) -{ - { - std::lock_guard lock(mutex_); - socket_wrapper_list_.remove(&impl); - } - delete &impl; -} - -inline void -win_local_stream_service::unregister_impl( - win_local_stream_socket_internal& impl) -{ - std::lock_guard lock(mutex_); - socket_list_.remove(&impl); + static_cast(h.get())->close_socket(); } inline std::error_code @@ -975,22 +808,20 @@ win_local_stream_service::open_socket( int type, int protocol) { - auto& wrapper = static_cast(impl); - return open_socket_internal( - *wrapper.get_internal(), family, type, protocol); + auto& sock = static_cast(impl); + return open_socket_internal(sock, family, type, protocol); } inline std::error_code win_local_stream_service::assign_socket( local_stream_socket::implementation& impl, native_handle_type fd) { - auto& wrapper = static_cast(impl); - auto& internal = *wrapper.get_internal(); + auto& sock = static_cast(impl); - SOCKET sock = static_cast(fd); - if (sock == INVALID_SOCKET) + SOCKET s = static_cast(fd); + if (s == INVALID_SOCKET) return make_err(WSAENOTSOCK); - if (sock == internal.socket_) + if (s == sock.socket_) return std::make_error_code(std::errc::invalid_argument); // SO_PROTOCOL_INFOW works on an unbound socket, unlike getsockname @@ -999,7 +830,7 @@ win_local_stream_service::assign_socket( WSAPROTOCOL_INFOW proto_info{}; int proto_len = sizeof(proto_info); if (::getsockopt( - sock, SOL_SOCKET, SO_PROTOCOL_INFOW, + s, SOL_SOCKET, SO_PROTOCOL_INFOW, reinterpret_cast(&proto_info), &proto_len) != 0) return make_err(::WSAGetLastError()); if (proto_info.iAddressFamily != AF_UNIX) @@ -1011,32 +842,31 @@ win_local_stream_service::assign_socket( // shares descriptor state the way the reactor path does, so a // failed association must not cost the caller their old socket. HANDLE result = ::CreateIoCompletionPort( - reinterpret_cast(sock), static_cast(iocp_), key_io, 0); + reinterpret_cast(s), static_cast(iocp_), key_io, 0); if (result == nullptr) return make_err(::GetLastError()); - internal.close_socket(); - internal.socket_ = sock; + sock.close_socket(); + sock.socket_ = s; sockaddr_storage local{}; int local_len = sizeof(local); corosio::local_endpoint lep{}, rep{}; - if (::getsockname(sock, reinterpret_cast(&local), &local_len) == - 0) + if (::getsockname(s, reinterpret_cast(&local), &local_len) == 0) lep = from_sockaddr_local(local, static_cast(local_len)); sockaddr_storage remote{}; int remote_len = sizeof(remote); - if (::getpeername( - sock, reinterpret_cast(&remote), &remote_len) == 0) + if (::getpeername(s, reinterpret_cast(&remote), &remote_len) == + 0) rep = from_sockaddr_local(remote, static_cast(remote_len)); - internal.local_endpoint_ = lep; - internal.remote_endpoint_ = rep; + sock.local_endpoint_ = lep; + sock.remote_endpoint_ = rep; return {}; } inline std::error_code win_local_stream_service::open_socket_internal( - win_local_stream_socket_internal& impl, int family, int type, int protocol) + win_local_stream_socket& impl, int family, int type, int protocol) { impl.close_socket(); @@ -1111,30 +941,15 @@ win_local_stream_service::work_finished() noexcept sched_.work_finished(); } -inline void -win_local_stream_service::destroy_acceptor_impl(win_local_stream_acceptor& impl) +inline win_local_stream_acceptor* +win_local_stream_service::acquire_acceptor_impl() { - { - std::lock_guard lock(mutex_); - acceptor_wrapper_list_.remove(&impl); - } - delete &impl; -} - -inline void -win_local_stream_service::unregister_acceptor_impl( - win_local_stream_acceptor_internal& impl) -{ - std::lock_guard lock(mutex_); - acceptor_list_.remove(&impl); + return acceptor_pool_.acquire(*this); } inline std::error_code win_local_stream_service::open_acceptor_socket( - win_local_stream_acceptor_internal& impl, - int family, - int type, - int protocol) + win_local_stream_acceptor& impl, int family, int type, int protocol) { impl.close_socket(); @@ -1160,7 +975,7 @@ win_local_stream_service::open_acceptor_socket( inline std::error_code win_local_stream_service::assign_acceptor_socket( - win_local_stream_acceptor_internal& impl, native_handle_type fd) + win_local_stream_acceptor& impl, native_handle_type fd) { SOCKET sock = static_cast(fd); if (sock == INVALID_SOCKET) @@ -1205,7 +1020,7 @@ win_local_stream_service::assign_acceptor_socket( inline std::error_code win_local_stream_service::bind_acceptor( - win_local_stream_acceptor_internal& impl, corosio::local_endpoint ep) + win_local_stream_acceptor& impl, corosio::local_endpoint ep) { // Reject abstract sockets on Windows if (ep.is_abstract()) @@ -1226,7 +1041,7 @@ win_local_stream_service::bind_acceptor( inline std::error_code win_local_stream_service::listen_acceptor( - win_local_stream_acceptor_internal& impl, int backlog) + win_local_stream_acceptor& impl, int backlog) { SOCKET sock = impl.socket_; diff --git a/include/boost/corosio/native/detail/iocp/win_local_stream_socket.hpp b/include/boost/corosio/native/detail/iocp/win_local_stream_socket.hpp index f52bb983c..a5278195a 100644 --- a/include/boost/corosio/native/detail/iocp/win_local_stream_socket.hpp +++ b/include/boost/corosio/native/detail/iocp/win_local_stream_socket.hpp @@ -15,6 +15,7 @@ #if BOOST_COROSIO_HAS_IOCP #include +#include #include #include #include @@ -23,20 +24,18 @@ #include #include -#include #include namespace boost::corosio::detail { class win_local_stream_service; -class win_local_stream_socket_internal; +class win_local_stream_socket; /** Connect operation state for local stream sockets. */ struct local_stream_connect_op : overlapped_op { - win_local_stream_socket_internal& internal; - std::shared_ptr internal_ptr; + win_local_stream_socket& internal; corosio::local_endpoint target_endpoint; static void do_complete( @@ -47,7 +46,7 @@ struct local_stream_connect_op : overlapped_op static void do_cancel_impl(overlapped_op* op) noexcept; explicit local_stream_connect_op( - win_local_stream_socket_internal& internal_) noexcept; + win_local_stream_socket& internal_) noexcept; }; /** Read operation state for local stream sockets. */ @@ -57,8 +56,7 @@ struct local_stream_read_op : overlapped_op WSABUF wsabufs[max_buffers]; DWORD wsabuf_count = 0; DWORD flags = 0; - win_local_stream_socket_internal& internal; - std::shared_ptr internal_ptr; + win_local_stream_socket& internal; static void do_complete( void* owner, @@ -67,8 +65,7 @@ struct local_stream_read_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit local_stream_read_op( - win_local_stream_socket_internal& internal_) noexcept; + explicit local_stream_read_op(win_local_stream_socket& internal_) noexcept; }; /** Write operation state for local stream sockets. */ @@ -77,8 +74,7 @@ struct local_stream_write_op : overlapped_op static constexpr std::size_t max_buffers = 16; WSABUF wsabufs[max_buffers]; DWORD wsabuf_count = 0; - win_local_stream_socket_internal& internal; - std::shared_ptr internal_ptr; + win_local_stream_socket& internal; static void do_complete( void* owner, @@ -87,8 +83,7 @@ struct local_stream_write_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit local_stream_write_op( - win_local_stream_socket_internal& internal_) noexcept; + explicit local_stream_write_op(win_local_stream_socket& internal_) noexcept; }; /** Readiness-wait operation state for local stream sockets. */ @@ -96,8 +91,7 @@ struct local_stream_wait_op : overlapped_op { WSABUF wsabuf{}; DWORD flags = 0; - win_local_stream_socket_internal& internal; - std::shared_ptr internal_ptr; + win_local_stream_socket& internal; static void do_complete( void* owner, @@ -106,23 +100,23 @@ struct local_stream_wait_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit local_stream_wait_op( - win_local_stream_socket_internal& internal_) noexcept; + explicit local_stream_wait_op(win_local_stream_socket& internal_) noexcept; }; -/* Internal socket state for IOCP local stream I/O. +/** Socket implementation for IOCP local stream I/O. - Holds the native SOCKET handle, cached endpoints, and - pending operation slots. Derives from enable_shared_from_this - so in-flight operations can extend its lifetime past the - wrapper's destruction. + Collapses the historical internal-state/wrapper split into one + pooled `io_object::implementation`: it both owns the native SOCKET + handle and pending operations, and directly implements + `local_stream_socket::implementation`. `win_local_stream_service` + recycles instances through its pool instead of freeing them on + every close. */ -class win_local_stream_socket_internal - : public intrusive_list::node - , public std::enable_shared_from_this +class win_local_stream_socket final + : public local_stream_socket::implementation + , public intrusive_list::node { friend class win_local_stream_service; - friend class win_local_stream_socket; friend struct local_stream_read_op; friend struct local_stream_write_op; friend struct local_stream_connect_op; @@ -134,74 +128,26 @@ class win_local_stream_socket_internal local_stream_write_op wr_; local_stream_wait_op wt_; SOCKET socket_ = INVALID_SOCKET; - -public: - explicit win_local_stream_socket_internal( - win_local_stream_service& svc) noexcept; - ~win_local_stream_socket_internal(); - - std::coroutine_handle<> connect( - std::coroutine_handle<>, - capy::executor_ref, - corosio::local_endpoint, - std::stop_token, - std::error_code*); - - std::coroutine_handle<> read_some( - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - std::stop_token, - std::error_code*, - std::size_t*); - - std::coroutine_handle<> write_some( - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - std::stop_token, - std::error_code*, - std::size_t*); - - std::coroutine_handle<> wait( - std::coroutine_handle<>, - capy::executor_ref, - wait_type, - std::stop_token, - std::error_code*); - - SOCKET native_handle() const noexcept; - corosio::local_endpoint local_endpoint() const noexcept; - corosio::local_endpoint remote_endpoint() const noexcept; - bool is_open() const noexcept; - void cancel() noexcept; - void close_socket() noexcept; - void set_socket(SOCKET s) noexcept; - void set_endpoints( - corosio::local_endpoint local, corosio::local_endpoint remote) noexcept; - -private: corosio::local_endpoint local_endpoint_; corosio::local_endpoint remote_endpoint_; -}; -/* Socket implementation wrapper for IOCP local stream I/O. +public: + explicit win_local_stream_socket(win_local_stream_service& svc) noexcept; - Public-facing implementation holding a shared_ptr to the - internal state. Inherits from local_stream_socket::implementation - to satisfy the virtual interface. -*/ -class win_local_stream_socket final - : public local_stream_socket::implementation - , public intrusive_list::node -{ - std::shared_ptr internal_; + /// Recycle into the owning service's pool. Defined out-of-line + /// after `win_local_stream_service` is complete (needs + /// `svc_.pool_`). + void retire() noexcept override; -public: - explicit win_local_stream_socket( - std::shared_ptr internal) noexcept; + /** Reset recycled state for reuse. - void close_internal() noexcept; + Each op's `OVERLAPPED` fields are re-seeded by + `overlapped_op::reset()` at its next submit, so they are not + zeroed here. + + @pre refs_ == 0, socket closed, no op in flight. + */ + void reuse() noexcept; std::coroutine_handle<> connect( std::coroutine_handle<> h, @@ -259,7 +205,13 @@ class win_local_stream_socket final corosio::local_endpoint remote_endpoint() const noexcept override; void cancel() noexcept override; - win_local_stream_socket_internal* get_internal() const noexcept; + /// Used by the acceptor's `local_stream_accept_op` to finish + /// setting up the peer. + void set_socket(SOCKET s) noexcept; + void set_endpoints( + corosio::local_endpoint local, corosio::local_endpoint remote) noexcept; + + void close_socket() noexcept; }; } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/iocp/win_random_access_file.hpp b/include/boost/corosio/native/detail/iocp/win_random_access_file.hpp index 4e3c1c4d3..e60716576 100644 --- a/include/boost/corosio/native/detail/iocp/win_random_access_file.hpp +++ b/include/boost/corosio/native/detail/iocp/win_random_access_file.hpp @@ -15,6 +15,7 @@ #if BOOST_COROSIO_HAS_IOCP #include +#include #include #include #include @@ -24,28 +25,29 @@ #include #include +#include #include -#include namespace boost::corosio::detail { class win_random_access_file_service; -class win_random_access_file_internal; +class win_random_access_file; /** Per-operation state for concurrent random-access file IOCP I/O. - Heap-allocated for each async read/write, enabling unlimited - concurrent operations on the same file. Self-deletes on - completion or shutdown. + Acquired from the file's free list for each async read/write, + enabling unlimited concurrent operations on the same file. + Recycles into that free list on completion or shutdown, so + steady-state reads and writes allocate nothing; the file + destructor frees the recycled storage. */ struct raf_concurrent_op : overlapped_op , intrusive_list::node { - void* buf = nullptr; - DWORD buf_len = 0; - win_random_access_file_internal* file_ = nullptr; - std::shared_ptr file_ref; + void* buf = nullptr; + DWORD buf_len = 0; + win_random_access_file* file_ = nullptr; static void do_complete( void* owner, @@ -54,75 +56,73 @@ struct raf_concurrent_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit raf_concurrent_op(win_random_access_file_internal& f) noexcept; + explicit raf_concurrent_op(win_random_access_file& f) noexcept; }; -/** Internal random-access file state for IOCP-based I/O. +/** Random-access file implementation for IOCP-based I/O. - Each async operation heap-allocates a raf_concurrent_op, - allowing unlimited concurrent reads and writes. + Collapses the historical internal-state/wrapper split into one + pooled `io_object::implementation`. Each async operation acquires + a `raf_concurrent_op` from the per-file free list, allowing + unlimited concurrent reads and writes without steady-state + allocation. */ -class win_random_access_file_internal - : public intrusive_list::node - , public std::enable_shared_from_this +class win_random_access_file final + : public random_access_file::implementation + , public intrusive_list::node { friend class win_random_access_file_service; - friend class win_random_access_file; friend struct raf_concurrent_op; win_random_access_file_service& svc_; win_mutex ops_mutex_; intrusive_list outstanding_ops_; + /// Recycled ops (guarded by `ops_mutex_`). Survives impl + /// recycling — the storage belongs to this impl and only the + /// destructor frees it. + intrusive_list free_ops_; HANDLE handle_ = INVALID_HANDLE_VALUE; + /// Pop a recycled op (re-arming its keepalive), or allocate on + /// the cold path; the caller's `reset()` clears per-use state. + raf_concurrent_op* acquire_op() + { + raf_concurrent_op* op; + { + std::lock_guard lock(ops_mutex_); + op = free_ops_.pop_front(); + } + if (!op) + op = new raf_concurrent_op(*this); + op->object_ref_ = detail::object_ref(this); + return op; + } + public: - explicit win_random_access_file_internal( + explicit win_random_access_file( win_random_access_file_service& svc) noexcept; - ~win_random_access_file_internal(); - std::coroutine_handle<> read_some_at( - std::uint64_t offset, - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - std::stop_token, - std::error_code*, - std::size_t*); + /// Recycle into the owning service's pool. Defined out-of-line + /// after `win_random_access_file_service` is complete (needs + /// `svc_.pool_`). + void retire() noexcept override; - std::coroutine_handle<> write_some_at( - std::uint64_t offset, - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - std::stop_token, - std::error_code*, - std::size_t*); - - HANDLE native_handle() const noexcept; - bool is_open() const noexcept; - void cancel() noexcept; - void close_handle() noexcept; + /** Reset recycled state for reuse. - std::uint64_t size() const; - std::error_code resize(std::uint64_t new_size) noexcept; - std::error_code sync_data() noexcept; - std::error_code sync_all() noexcept; - native_handle_type release(); - std::error_code assign(native_handle_type handle) noexcept; -}; - -/** Random-access file implementation wrapper for IOCP-based I/O. */ -class win_random_access_file final - : public random_access_file::implementation - , public intrusive_list::node -{ - std::shared_ptr internal_; + `free_ops_` deliberately survives recycling; only the + destructor frees it. -public: - explicit win_random_access_file( - std::shared_ptr internal) noexcept; + @pre refs_ == 0, handle closed, no op in flight (per-op + `raf_concurrent_op` holds its own `object_ref`, so `refs_` + cannot reach zero while one is outstanding). + */ + void reuse() noexcept; - void close_internal() noexcept; + ~win_random_access_file() override + { + while (auto* op = free_ops_.pop_front()) + delete op; + } std::coroutine_handle<> read_some_at( std::uint64_t offset, @@ -143,6 +143,7 @@ class win_random_access_file final std::size_t* bytes) override; native_handle_type native_handle() const noexcept override; + bool is_open() const noexcept; void cancel() noexcept override; std::uint64_t size() const override; std::error_code resize(std::uint64_t new_size) noexcept override; @@ -151,7 +152,7 @@ class win_random_access_file final native_handle_type release() override; std::error_code assign(native_handle_type handle) noexcept override; - win_random_access_file_internal* get_internal() const noexcept; + void close_handle() noexcept; }; } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/iocp/win_random_access_file_service.hpp b/include/boost/corosio/native/detail/iocp/win_random_access_file_service.hpp index e62eddda4..4856e5301 100644 --- a/include/boost/corosio/native/detail/iocp/win_random_access_file_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_random_access_file_service.hpp @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include #include #include @@ -44,11 +44,12 @@ namespace boost::corosio::detail { class BOOST_COROSIO_DECL win_random_access_file_service final : public random_access_file_service { + friend class win_random_access_file; + public: using key_type = win_random_access_file_service; explicit win_random_access_file_service(capy::execution_context& ctx); - ~win_random_access_file_service(); win_random_access_file_service(win_random_access_file_service const&) = delete; @@ -65,9 +66,6 @@ class BOOST_COROSIO_DECL win_random_access_file_service final std::filesystem::path const& path, file_base::flags mode) override; - void destroy_impl(win_random_access_file& impl); - void unregister_impl(win_random_access_file_internal& impl); - void post(overlapped_op* op); void on_pending(overlapped_op* op) noexcept; void on_completion(overlapped_op* op, DWORD error, DWORD bytes) noexcept; @@ -104,14 +102,12 @@ class BOOST_COROSIO_DECL win_random_access_file_service final LONG(NTAPI*)(HANDLE, ULONG, void*, ULONG, io_status_block*); win_scheduler& sched_; + void* iocp_; + nt_flush_fn nt_flush_buffers_file_ex_; BOOST_COROSIO_MSVC_WARNING_PUSH BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // detail:: members, dll-interface - win_mutex mutex_; - intrusive_list file_list_; - intrusive_list wrapper_list_; + object_pool pool_; BOOST_COROSIO_MSVC_WARNING_POP - void* iocp_; - nt_flush_fn nt_flush_buffers_file_ex_; }; /** Get or create the random-access file service for the given context. */ @@ -126,7 +122,7 @@ get_random_access_file_service(capy::execution_context& ctx, win_scheduler&) // --------------------------------------------------------------------------- inline raf_concurrent_op::raf_concurrent_op( - win_random_access_file_internal& f) noexcept + win_random_access_file& f) noexcept : overlapped_op(&do_complete) , file_(&f) { @@ -139,7 +135,7 @@ raf_concurrent_op::do_cancel_impl(overlapped_op* base) noexcept auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); if (op->file_->is_open()) - ::CancelIoEx(op->file_->native_handle(), op); + ::CancelIoEx(op->file_->handle_, op); } inline void @@ -156,12 +152,12 @@ raf_concurrent_op::do_complete( // Shutdown path: clean up without invoking op->stop_cb.reset(); op->h = {}; + auto keep = std::move(op->object_ref_); { std::lock_guard lock(op->file_->ops_mutex_); op->file_->outstanding_ops_.remove(op); + op->file_->free_ops_.push_front(op); } - op->file_ref.reset(); - delete op; return; } @@ -179,47 +175,51 @@ raf_concurrent_op::do_complete( op->is_read, static_cast(op->bytes_transferred), op->empty_buffer); + auto coro = op->h; + // Hold the keepalive across the resume, like every other IOCP op: + // it may be the last reference to the implementation the op was + // working on behalf of. Copy out everything needed first — once + // the op is on free_ops_ a concurrent initiation may refill it. + auto prevent_premature_destruction = std::move(op->object_ref_); { std::lock_guard lock(op->file_->ops_mutex_); op->file_->outstanding_ops_.remove(op); + op->file_->free_ops_.push_front(op); } - - op->file_ref.reset(); - - auto coro = op->h; - delete op; coro.resume(); } // --------------------------------------------------------------------------- -// win_random_access_file_internal +// win_random_access_file // --------------------------------------------------------------------------- -inline win_random_access_file_internal::win_random_access_file_internal( +inline win_random_access_file::win_random_access_file( win_random_access_file_service& svc) noexcept : svc_(svc) { } -inline win_random_access_file_internal::~win_random_access_file_internal() +inline void +win_random_access_file::reuse() noexcept { - svc_.unregister_impl(*this); + BOOST_COROSIO_ASSERT(handle_ == INVALID_HANDLE_VALUE); + BOOST_COROSIO_ASSERT(outstanding_ops_.empty()); } -inline HANDLE -win_random_access_file_internal::native_handle() const noexcept +inline native_handle_type +win_random_access_file::native_handle() const noexcept { - return handle_; + return reinterpret_cast(handle_); } inline bool -win_random_access_file_internal::is_open() const noexcept +win_random_access_file::is_open() const noexcept { return handle_ != INVALID_HANDLE_VALUE; } inline void -win_random_access_file_internal::cancel() noexcept +win_random_access_file::cancel() noexcept { if (handle_ != INVALID_HANDLE_VALUE) ::CancelIoEx(handle_, nullptr); @@ -230,7 +230,7 @@ win_random_access_file_internal::cancel() noexcept } inline void -win_random_access_file_internal::close_handle() noexcept +win_random_access_file::close_handle() noexcept { if (handle_ != INVALID_HANDLE_VALUE) { @@ -241,7 +241,7 @@ win_random_access_file_internal::close_handle() noexcept } inline std::uint64_t -win_random_access_file_internal::size() const +win_random_access_file::size() const { LARGE_INTEGER li; if (!::GetFileSizeEx(handle_, &li)) @@ -251,7 +251,7 @@ win_random_access_file_internal::size() const } inline std::error_code -win_random_access_file_internal::resize(std::uint64_t new_size) noexcept +win_random_access_file::resize(std::uint64_t new_size) noexcept { LARGE_INTEGER li; li.QuadPart = static_cast(new_size); @@ -263,7 +263,7 @@ win_random_access_file_internal::resize(std::uint64_t new_size) noexcept } inline std::error_code -win_random_access_file_internal::sync_data() noexcept +win_random_access_file::sync_data() noexcept { // Attempt data-only flush; fall back to full flush if (svc_.try_flush_data(handle_)) @@ -274,7 +274,7 @@ win_random_access_file_internal::sync_data() noexcept } inline std::error_code -win_random_access_file_internal::sync_all() noexcept +win_random_access_file::sync_all() noexcept { if (!::FlushFileBuffers(handle_)) return make_err(::GetLastError()); @@ -282,7 +282,7 @@ win_random_access_file_internal::sync_all() noexcept } inline native_handle_type -win_random_access_file_internal::release() +win_random_access_file::release() { HANDLE h = handle_; handle_ = INVALID_HANDLE_VALUE; @@ -290,7 +290,7 @@ win_random_access_file_internal::release() } inline std::error_code -win_random_access_file_internal::assign(native_handle_type handle) noexcept +win_random_access_file::assign(native_handle_type handle) noexcept { close_handle(); HANDLE h = reinterpret_cast(handle); @@ -305,7 +305,7 @@ win_random_access_file_internal::assign(native_handle_type handle) noexcept } inline std::coroutine_handle<> -win_random_access_file_internal::read_some_at( +win_random_access_file::read_some_at( std::uint64_t offset, std::coroutine_handle<> h, capy::executor_ref ex, @@ -316,9 +316,7 @@ win_random_access_file_internal::read_some_at( { static constexpr std::size_t max_buffers = 16; - auto* op = new raf_concurrent_op(*this); - op->file_ref = shared_from_this(); - + auto* op = acquire_op(); op->reset(); op->is_read = true; op->h = h; @@ -387,7 +385,7 @@ win_random_access_file_internal::read_some_at( } inline std::coroutine_handle<> -win_random_access_file_internal::write_some_at( +win_random_access_file::write_some_at( std::uint64_t offset, std::coroutine_handle<> h, capy::executor_ref ex, @@ -398,9 +396,7 @@ win_random_access_file_internal::write_some_at( { static constexpr std::size_t max_buffers = 16; - auto* op = new raf_concurrent_op(*this); - op->file_ref = shared_from_this(); - + auto* op = acquire_op(); op->reset(); op->is_read = false; op->h = h; @@ -467,104 +463,10 @@ win_random_access_file_internal::write_some_at( return std::noop_coroutine(); } -// --------------------------------------------------------------------------- -// win_random_access_file wrapper -// --------------------------------------------------------------------------- - -inline win_random_access_file::win_random_access_file( - std::shared_ptr internal) noexcept - : internal_(std::move(internal)) -{ -} - -inline void -win_random_access_file::close_internal() noexcept -{ - if (internal_) - { - internal_->close_handle(); - internal_.reset(); - } -} - -inline std::coroutine_handle<> -win_random_access_file::read_some_at( - std::uint64_t offset, - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->read_some_at(offset, h, d, buf, token, ec, bytes); -} - -inline std::coroutine_handle<> -win_random_access_file::write_some_at( - std::uint64_t offset, - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->write_some_at(offset, h, d, buf, token, ec, bytes); -} - -inline native_handle_type -win_random_access_file::native_handle() const noexcept -{ - return reinterpret_cast(internal_->native_handle()); -} - inline void -win_random_access_file::cancel() noexcept -{ - internal_->cancel(); -} - -inline std::uint64_t -win_random_access_file::size() const -{ - return internal_->size(); -} - -inline std::error_code -win_random_access_file::resize(std::uint64_t new_size) noexcept -{ - return internal_->resize(new_size); -} - -inline std::error_code -win_random_access_file::sync_data() noexcept -{ - return internal_->sync_data(); -} - -inline std::error_code -win_random_access_file::sync_all() noexcept -{ - return internal_->sync_all(); -} - -inline native_handle_type -win_random_access_file::release() -{ - return internal_->release(); -} - -inline std::error_code -win_random_access_file::assign(native_handle_type handle) noexcept +win_random_access_file::retire() noexcept { - return internal_->assign(handle); -} - -inline win_random_access_file_internal* -win_random_access_file::get_internal() const noexcept -{ - return internal_.get(); + svc_.pool_.recycle(this); } // --------------------------------------------------------------------------- @@ -585,61 +487,33 @@ inline win_random_access_file_service::win_random_access_file_service( } } -inline win_random_access_file_service::~win_random_access_file_service() -{ - for (auto* w = wrapper_list_.pop_front(); w != nullptr; - w = wrapper_list_.pop_front()) - delete w; -} - inline io_object::implementation* win_random_access_file_service::construct() { - auto internal = std::make_shared(*this); - - { - std::lock_guard lock(mutex_); - file_list_.push_back(internal.get()); - } - - auto* wrapper = new win_random_access_file(std::move(internal)); - - { - std::lock_guard lock(mutex_); - wrapper_list_.push_back(wrapper); - } - - return wrapper; + return pool_.acquire(*this); } inline void win_random_access_file_service::destroy(io_object::implementation* p) { - if (p) - { - auto& wrapper = static_cast(*p); - wrapper.close_internal(); - destroy_impl(wrapper); - } + if (!p) + return; + auto* f = static_cast(p); + f->close_handle(); + release(f); } inline void win_random_access_file_service::close(io_object::handle& h) { - auto& wrapper = static_cast(*h.get()); - wrapper.get_internal()->close_handle(); + static_cast(h.get())->close_handle(); } inline void win_random_access_file_service::shutdown() { - std::lock_guard lock(mutex_); - - for (auto* impl = file_list_.pop_front(); impl != nullptr; - impl = file_list_.pop_front()) - { - impl->close_handle(); - } + pool_.shutdown( + [](win_random_access_file* impl) { impl->close_handle(); }); } inline std::error_code @@ -702,30 +576,12 @@ win_random_access_file_service::open_file( } } - auto& internal = *static_cast(impl).get_internal(); - internal.handle_ = h; + auto& file = static_cast(impl); + file.handle_ = h; return {}; } -inline void -win_random_access_file_service::destroy_impl(win_random_access_file& impl) -{ - { - std::lock_guard lock(mutex_); - wrapper_list_.remove(&impl); - } - delete &impl; -} - -inline void -win_random_access_file_service::unregister_impl( - win_random_access_file_internal& impl) -{ - std::lock_guard lock(mutex_); - file_list_.remove(&impl); -} - inline void win_random_access_file_service::post(overlapped_op* op) { diff --git a/include/boost/corosio/native/detail/iocp/win_resolver.hpp b/include/boost/corosio/native/detail/iocp/win_resolver.hpp index 873be70df..071451182 100644 --- a/include/boost/corosio/native/detail/iocp/win_resolver.hpp +++ b/include/boost/corosio/native/detail/iocp/win_resolver.hpp @@ -23,6 +23,7 @@ #endif #include +#include #include #include #include @@ -75,11 +76,11 @@ extern "C" Class Hierarchy --------------- - win_resolver_service (execution_context::service) - - Owns all win_resolver instances via shared_ptr + - Owns all win_resolver instances through a recycling object_pool - Coordinates with win_scheduler for work tracking - win_resolver (one per resolver object) - Contains embedded resolve_op and reverse_resolve_op - - Inherits from enable_shared_from_this for thread safety + - Kept alive during I/O by each op's intrusive object_ref_ - resolve_op (overlapped_op subclass) - OVERLAPPED base enables IOCP integration - Static completion() callback invoked by Windows @@ -88,9 +89,10 @@ extern "C" Shutdown -------- - The resolver service cancels all resolvers and clears the impl map. - The thread pool service shuts down separately via execution_context - service ordering, joining all worker threads. + The resolver service cancels all resolvers; each then stays alive + until its own in-flight completion drains, same as every other + pooled impl. The thread pool service shuts down separately via + execution_context service ordering, joining all worker threads. Cancellation ------------ @@ -239,7 +241,6 @@ struct reverse_resolve_op : overlapped_op */ class win_resolver final : public resolver::implementation - , public std::enable_shared_from_this , public intrusive_list::node { friend class win_resolver_service; @@ -254,11 +255,27 @@ class win_resolver final win_resolver* resolver_ = nullptr; /// Prevent impl destruction while work is in flight. - std::shared_ptr ref_; + detail::object_ref ref_; }; explicit win_resolver(win_resolver_service& svc) noexcept; + /// Recycle into the owning service's pool. Defined out-of-line + /// after `win_resolver_service` is complete (needs `svc_.pool_`). + void retire() noexcept override; + + /** Reset recycled state for reuse. + + Does not clear `host`/`service`/`stored_host`/`stored_service` + itself -- each op's own `reset()` (called at the top of the + next `resolve()`/`reverse_resolve()`) already does that, + preserving allocated capacity. That same `reset()` re-seeds + the op's `OVERLAPPED` fields, so they are not zeroed here. + + @pre refs_ == 0, no wait in flight. + */ + void reuse() noexcept; + std::coroutine_handle<> resolve( std::coroutine_handle<>, capy::executor_ref, @@ -298,6 +315,15 @@ class win_resolver final win_resolver_service& svc_; }; +inline void +win_resolver::reuse() noexcept +{ + BOOST_COROSIO_ASSERT(!op_.stop_cb); + BOOST_COROSIO_ASSERT(!reverse_op_.stop_cb); + BOOST_COROSIO_ASSERT(op_.results == nullptr); + BOOST_COROSIO_ASSERT(op_.cancel_handle == nullptr); +} + } // namespace boost::corosio::detail #endif // BOOST_COROSIO_HAS_IOCP diff --git a/include/boost/corosio/native/detail/iocp/win_resolver_service.hpp b/include/boost/corosio/native/detail/iocp/win_resolver_service.hpp index fffa05d13..d6e28622b 100644 --- a/include/boost/corosio/native/detail/iocp/win_resolver_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_resolver_service.hpp @@ -17,10 +17,11 @@ #if BOOST_COROSIO_HAS_IOCP #include +#include #include #include -#include +#include namespace boost::corosio::detail { @@ -46,6 +47,8 @@ class BOOST_COROSIO_DECL win_resolver_service final , public capy::execution_context::service , public io_object::io_service { + friend class win_resolver; + public: using key_type = win_resolver_service; @@ -55,7 +58,7 @@ class BOOST_COROSIO_DECL win_resolver_service final { auto& impl = static_cast(*p); impl.cancel(); - destroy_impl(impl); + release(&impl); } /** Construct the resolver service. @@ -73,9 +76,6 @@ class BOOST_COROSIO_DECL win_resolver_service final /** Shut down the service. */ void shutdown() override; - /** Destroy a resolver implementation. */ - void destroy_impl(win_resolver& impl); - /** Post an operation for completion. */ void post(overlapped_op* op); @@ -106,10 +106,7 @@ class BOOST_COROSIO_DECL win_resolver_service final private: scheduler& sched_; thread_pool_ref pool_; - win_mutex mutex_; - intrusive_list resolver_list_; - std::unordered_map> - resolver_ptrs_; + object_pool impls_; }; BOOST_COROSIO_MSVC_WARNING_POP @@ -229,10 +226,10 @@ resolve_op::completion(DWORD dwError, DWORD /*bytes*/, OVERLAPPED* ov) { auto* op = static_cast(ov); - // The post below can be drained and the win_resolver freed before - // this returns, and Windows owns the OVERLAPPED embedded in it until - // then. - auto keepalive = op->impl->shared_from_this(); + // The post below can be drained and the win_resolver recycled or + // freed before this returns, and Windows owns the OVERLAPPED + // embedded in it until then. + detail::object_ref keepalive(op->impl); op->dwError = dwError; @@ -277,7 +274,7 @@ resolve_op::do_complete( } // Dropping the keepalive may destroy the implementation this op // is embedded in, so nothing may touch it afterwards. - op->impl_ptr.reset(); + op->object_ref_.reset(); svc.work_finished(); return; } @@ -309,7 +306,7 @@ resolve_op::do_complete( op->cont.h = op->h; // Hold the keepalive across the dispatch: it may be the last // reference to the implementation this op is embedded in. - auto prevent_destroy = std::move(op->impl_ptr); + auto prevent_destroy = std::move(op->object_ref_); svc.work_finished(); dispatch_coro(op->ex, op->cont).resume(); } @@ -338,7 +335,7 @@ reverse_resolve_op::do_complete( op->stop_cb.reset(); // Dropping the keepalive may destroy the implementation this // op is embedded in, so nothing may touch it afterwards. - op->impl_ptr.reset(); + op->object_ref_.reset(); svc.work_finished(); return; } @@ -365,7 +362,7 @@ reverse_resolve_op::do_complete( op->cont.h = op->h; // Hold the keepalive across the dispatch: it may be the last // reference to the implementation this op is embedded in. - auto prevent_destroy = std::move(op->impl_ptr); + auto prevent_destroy = std::move(op->object_ref_); svc.work_finished(); dispatch_coro(op->ex, op->cont).resume(); } @@ -414,7 +411,7 @@ win_resolver::resolve( // its completion waits in the scheduler queue: the op is embedded in // this win_resolver, which teardown may otherwise free before the // queued completion drains. Mirrors the reverse path's keepalive. - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); // Under the lock: the handle written here and the callback that // retires it must not interleave. @@ -480,7 +477,7 @@ win_resolver::reverse_resolve( // Prevent impl destruction while work is in flight reverse_pool_op_.resolver_ = this; - reverse_pool_op_.ref_ = this->shared_from_this(); + reverse_pool_op_.ref_ = detail::object_ref(this); reverse_pool_op_.func_ = &win_resolver::do_reverse_resolve_work; if (auto pec = svc_.pool().post(&reverse_pool_op_)) { @@ -570,7 +567,7 @@ win_resolver::do_reverse_resolve_work(pool_work_item* w) noexcept // Hand the keepalive to the op: the completion waits in the // scheduler's queue, and the implementation embedding it must // outlive that wait. Nothing may touch *self after the post. - self->reverse_op_.impl_ptr = std::move(pw->ref_); + self->reverse_op_.object_ref_ = std::move(pw->ref_); // Post only; the initiation credit is released where the op is // consumed. @@ -591,42 +588,40 @@ inline win_resolver_service::~win_resolver_service() {} inline void win_resolver_service::shutdown() { - std::lock_guard lock(mutex_); - - // Cancel all resolvers (sets cancelled flag checked by pool threads) - for (auto* impl = resolver_list_.pop_front(); impl != nullptr; - impl = resolver_list_.pop_front()) + // Snapshot live impls under an acquired reference, then cancel + // without the pool lock held -- mirrors + // uring_socket_service_base::shutdown(). shutdown() sets + // shutting-down and takes the snapshot in one critical section, so + // each cancel's own release (if it drops the last ref) deletes + // rather than recycles. In-flight resolves hold their own + // object_ref_, so each resolver stays alive while its completion + // (GetAddrInfoExW callback or pool thread) drains. The thread pool + // service shuts down separately via execution_context service + // ordering. + std::vector live; + impls_.shutdown( + [&](win_resolver* r) + { + acquire(r); + live.push_back(r); + }); + for (auto* r : live) { - impl->cancel(); + r->cancel(); + release(r); } - - // Clear the map which releases shared_ptrs. - // The thread pool service shuts down separately via - // execution_context service ordering. - resolver_ptrs_.clear(); } inline io_object::implementation* win_resolver_service::construct() { - auto ptr = std::make_shared(*this); - auto* impl = ptr.get(); - - { - std::lock_guard lock(mutex_); - resolver_list_.push_back(impl); - resolver_ptrs_[impl] = std::move(ptr); - } - - return impl; + return impls_.acquire(*this); } inline void -win_resolver_service::destroy_impl(win_resolver& impl) +win_resolver::retire() noexcept { - std::lock_guard lock(mutex_); - resolver_list_.remove(&impl); - resolver_ptrs_.erase(&impl); + svc_.impls_.recycle(this); } inline void diff --git a/include/boost/corosio/native/detail/iocp/win_signal.hpp b/include/boost/corosio/native/detail/iocp/win_signal.hpp index 00e4a2979..1e921ece4 100644 --- a/include/boost/corosio/native/detail/iocp/win_signal.hpp +++ b/include/boost/corosio/native/detail/iocp/win_signal.hpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -125,6 +126,16 @@ class win_signal final public: explicit win_signal(win_signals& svc) noexcept; + /// Recycle into the owning service's pool. Defined out-of-line + /// after `win_signals` is complete (needs `svc_.pool_`). + void retire() noexcept override; + + /** Reset recycled state for reuse. + + @pre refs_ == 0, no registrations, no wait in flight. + */ + void reuse() noexcept; + std::coroutine_handle<> wait( std::coroutine_handle<>, capy::executor_ref, @@ -144,6 +155,16 @@ class win_signal final } }; +inline void +win_signal::reuse() noexcept +{ + BOOST_COROSIO_ASSERT(signals_ == nullptr); + BOOST_COROSIO_ASSERT(!waiting_); + BOOST_COROSIO_ASSERT(!stop_cb_); + cancelled_ = false; + token_cancelled_ = false; +} + } // namespace boost::corosio::detail #endif // BOOST_COROSIO_HAS_IOCP diff --git a/include/boost/corosio/native/detail/iocp/win_signals.hpp b/include/boost/corosio/native/detail/iocp/win_signals.hpp index 2c3df431f..6fb0c4a57 100644 --- a/include/boost/corosio/native/detail/iocp/win_signals.hpp +++ b/include/boost/corosio/native/detail/iocp/win_signals.hpp @@ -19,6 +19,8 @@ #include #include +#include +#include #include #include #include @@ -28,6 +30,7 @@ #include #include #include +#include #include @@ -75,7 +78,7 @@ 2. win_signals (one per execution_context) - Maintains registrations_[] table indexed by signal number - Each slot is a doubly-linked list of all signal_registrations for that signal - - Also maintains impl_list_ of all win_signal objects it owns + - Also owns a recycling pool of all win_signal objects it owns 3. win_signal (one per signal_set) - Owns a singly-linked list (sorted by signal number) of signal_registrations @@ -154,11 +157,21 @@ class BOOST_COROSIO_DECL win_signals final : public capy::execution_context::service , public io_object::io_service { + friend class win_signal; + public: using key_type = win_signals; io_object::implementation* construct() override; - void destroy(io_object::implementation*) override; + + void destroy(io_object::implementation* p) override + { + auto& impl = static_cast(*p); + impl.clear(); + impl.disarm_stop(); + impl.cancel(); + release(&impl); + } /** Construct the signal service. @@ -179,8 +192,6 @@ class BOOST_COROSIO_DECL win_signals final */ void shutdown() override; - /** Destroy a signal implementation. */ - void destroy_impl(win_signal& impl); /** Add a signal to a signal set. @@ -265,7 +276,7 @@ class BOOST_COROSIO_DECL win_signals final BOOST_COROSIO_MSVC_WARNING_PUSH BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // detail:: members, dll-interface win_mutex mutex_; - intrusive_list impl_list_; + object_pool pool_; BOOST_COROSIO_MSVC_WARNING_POP // Per-signal registration table for this service @@ -439,20 +450,34 @@ inline win_signals::~win_signals() inline void win_signals::shutdown() { - // Collected under the locks below and deleted after they are released: - // ~win_signal destroys an armed stop_cb_, and ~stop_callback blocks - // until a concurrently running token_canceller returns -- which takes - // mutex_. Deleting while still holding mutex_ would self-deadlock the - // same way disarm_stop() would if called inside the locked loop. - intrusive_list doomed; + // Collected under the locks below and released after they are + // released: ~win_signal destroys an armed stop_cb_, and + // ~stop_callback blocks until a concurrently running + // token_canceller returns -- which takes mutex_. Releasing while + // still holding mutex_ would self-deadlock the same way + // disarm_stop() would if called inside the locked loop. + // + // The acquire()+release() pair below nets to zero change on each + // impl's refs_ -- it exists only so the lock-held loop above can + // safely walk `doomed` without a concurrent recycle() tearing one + // down mid-walk. It does NOT drop these impls to zero: every impl + // here survives at refs_ == 1 (the service's own floor reference) + // until ~object_pool()'s unconditional sweep deletes whatever is + // still in `live_`, bypassing refs_ entirely. + std::vector doomed; { signal_detail::signal_state* state = signal_detail::get_signal_state(); std::lock_guard state_lock(state->mutex); std::lock_guard lock(mutex_); - for (auto* impl = impl_list_.pop_front(); impl != nullptr; - impl = impl_list_.pop_front()) + pool_.shutdown( + [&](win_signal* impl) + { + acquire(impl); + doomed.push_back(impl); + }); + for (auto* impl : doomed) { while (auto* reg = impl->signals_) { @@ -472,57 +497,31 @@ win_signals::shutdown() impl->signals_ = reg->next_in_set; delete reg; } - doomed.push_back(impl); } - // Every live registration hung off an implementation in impl_list_, - // so the whole table goes stale at once and can be dropped wholesale - // rather than node by node. It has to be dropped: deliver_signal() - // walks this service until the destructor unlinks it from the global - // list. + // Every live registration hung off an implementation this pool + // owns, so the whole table goes stale at once and can be dropped + // wholesale rather than node by node. It has to be dropped: + // deliver_signal() walks this service until the destructor + // unlinks it from the global list. for (int i = 0; i < max_signal_number; ++i) registrations_[i] = nullptr; } - for (auto* impl = doomed.pop_front(); impl != nullptr; - impl = doomed.pop_front()) - { - delete impl; - } + for (auto* impl : doomed) + release(impl); } inline io_object::implementation* win_signals::construct() { - auto* impl = new win_signal(*this); - - { - std::lock_guard lock(mutex_); - impl_list_.push_back(impl); - } - - return impl; -} - -inline void -win_signals::destroy(io_object::implementation* p) -{ - auto& impl = static_cast(*p); - impl.clear(); - impl.disarm_stop(); - impl.cancel(); - destroy_impl(impl); + return pool_.acquire(*this); } inline void -win_signals::destroy_impl(win_signal& impl) +win_signal::retire() noexcept { - { - std::lock_guard lock(mutex_); - impl_list_.remove(&impl); - } - - delete &impl; + svc_.pool_.recycle(this); } inline std::error_code diff --git a/include/boost/corosio/native/detail/iocp/win_stream_file.hpp b/include/boost/corosio/native/detail/iocp/win_stream_file.hpp index 3cb3a9bc8..a69b8e1ab 100644 --- a/include/boost/corosio/native/detail/iocp/win_stream_file.hpp +++ b/include/boost/corosio/native/detail/iocp/win_stream_file.hpp @@ -15,6 +15,7 @@ #if BOOST_COROSIO_HAS_IOCP #include +#include #include #include #include @@ -24,20 +25,18 @@ #include #include -#include namespace boost::corosio::detail { class win_file_service; -class win_stream_file_internal; +class win_stream_file; /** Read operation state for stream file IOCP I/O. */ struct file_read_op : overlapped_op { void* buf = nullptr; DWORD buf_len = 0; - win_stream_file_internal& file_; - std::shared_ptr file_ptr; + win_stream_file& file_; static void do_complete( void* owner, @@ -46,7 +45,7 @@ struct file_read_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit file_read_op(win_stream_file_internal& f) noexcept; + explicit file_read_op(win_stream_file& f) noexcept; }; /** Write operation state for stream file IOCP I/O. */ @@ -54,8 +53,7 @@ struct file_write_op : overlapped_op { void* buf = nullptr; DWORD buf_len = 0; - win_stream_file_internal& file_; - std::shared_ptr file_ptr; + win_stream_file& file_; static void do_complete( void* owner, @@ -64,22 +62,23 @@ struct file_write_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit file_write_op(win_stream_file_internal& f) noexcept; + explicit file_write_op(win_stream_file& f) noexcept; }; -/** Internal stream file state for IOCP-based I/O. +/** Stream file implementation for IOCP-based I/O. - Contains the actual state for a single file, including - the native handle, position tracking, and pending operations. - Derives from enable_shared_from_this so operations can extend - its lifetime. + Collapses the historical internal-state/wrapper split into one + pooled `io_object::implementation`: it both owns the native + handle and pending operations, and directly implements + `stream_file::implementation`. `win_file_service` recycles + instances through its pool instead of freeing them on every + close. */ -class win_stream_file_internal - : public intrusive_list::node - , public std::enable_shared_from_this +class win_stream_file final + : public stream_file::implementation + , public intrusive_list::node { friend class win_file_service; - friend class win_stream_file; friend struct file_read_op; friend struct file_write_op; @@ -90,56 +89,21 @@ class win_stream_file_internal std::uint64_t offset_ = 0; public: - explicit win_stream_file_internal(win_file_service& svc) noexcept; - ~win_stream_file_internal(); - - std::coroutine_handle<> read_some( - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - std::stop_token, - std::error_code*, - std::size_t*); - - std::coroutine_handle<> write_some( - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - std::stop_token, - std::error_code*, - std::size_t*); - - HANDLE native_handle() const noexcept; - bool is_open() const noexcept; - void cancel() noexcept; - void close_handle() noexcept; - - std::uint64_t size() const; - std::error_code resize(std::uint64_t new_size) noexcept; - std::error_code sync_data() noexcept; - std::error_code sync_all() noexcept; - native_handle_type release(); - std::error_code assign(native_handle_type handle) noexcept; - capy::io_result - seek(std::int64_t offset, file_base::seek_basis origin) noexcept; -}; + explicit win_stream_file(win_file_service& svc) noexcept; -/** Stream file implementation wrapper for IOCP-based I/O. + /// Recycle into the owning service's pool. Defined out-of-line + /// after `win_file_service` is complete (needs `svc_.pool_`). + void retire() noexcept override; - Public-facing implementation that holds a shared_ptr to - the internal state. Delegates all virtual calls. -*/ -class win_stream_file final - : public stream_file::implementation - , public intrusive_list::node -{ - std::shared_ptr internal_; + /** Reset recycled state for reuse. -public: - explicit win_stream_file( - std::shared_ptr internal) noexcept; + Each op's `OVERLAPPED` fields are re-seeded by + `overlapped_op::reset()` at its next submit, so they are not + zeroed here. - void close_internal() noexcept; + @pre refs_ == 0, handle closed, no op in flight. + */ + void reuse() noexcept; std::coroutine_handle<> read_some( std::coroutine_handle<> h, @@ -158,6 +122,7 @@ class win_stream_file final std::size_t* bytes) override; native_handle_type native_handle() const noexcept override; + bool is_open() const noexcept; void cancel() noexcept override; std::uint64_t size() const override; std::error_code resize(std::uint64_t new_size) noexcept override; @@ -168,7 +133,7 @@ class win_stream_file final capy::io_result seek(std::int64_t offset, file_base::seek_basis origin) noexcept override; - win_stream_file_internal* get_internal() const noexcept; + void close_handle() noexcept; }; } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/iocp/win_tcp_acceptor.hpp b/include/boost/corosio/native/detail/iocp/win_tcp_acceptor.hpp index 722ca55b0..756210911 100644 --- a/include/boost/corosio/native/detail/iocp/win_tcp_acceptor.hpp +++ b/include/boost/corosio/native/detail/iocp/win_tcp_acceptor.hpp @@ -19,12 +19,12 @@ #include #include #include +#include #include #include #include #include -#include #include #include @@ -34,14 +34,14 @@ namespace boost::corosio::detail { class win_tcp_acceptor_service; class win_tcp_service; class win_tcp_socket; -class win_tcp_acceptor_internal; +class win_tcp_acceptor; /** Accept operation state. */ struct accept_op : overlapped_op { SOCKET accepted_socket = INVALID_SOCKET; win_tcp_socket* peer_wrapper = nullptr; - std::shared_ptr acceptor_ptr; + win_tcp_acceptor& acceptor; SOCKET listen_socket = INVALID_SOCKET; io_object::implementation** impl_out = nullptr; char addr_buf[2 * (sizeof(sockaddr_in6) + 16)]; @@ -53,13 +53,13 @@ struct accept_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - accept_op() noexcept; + explicit accept_op(win_tcp_acceptor& acceptor_) noexcept; }; /** Readiness-wait operation state for an acceptor. */ struct acceptor_wait_op : overlapped_op { - std::shared_ptr acceptor_ptr; + win_tcp_acceptor& acceptor; SOCKET listen_socket = INVALID_SOCKET; static void do_complete( @@ -69,79 +69,57 @@ struct acceptor_wait_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - acceptor_wait_op() noexcept; + explicit acceptor_wait_op(win_tcp_acceptor& acceptor_) noexcept; }; -/** Internal acceptor state for IOCP-based I/O. +/** Acceptor implementation for IOCP-based I/O. - This class contains the actual state for a listening socket, including - the native socket handle and pending accept operation. + Collapses the historical internal-state/wrapper split into one + pooled `io_object::implementation`: it both owns the listening + socket and pending operations, and directly implements + `tcp_acceptor::implementation`. `win_tcp_service` recycles + instances through its acceptor pool instead of freeing them on + every close. @note Internal implementation detail. Users interact with acceptor class. */ -class win_tcp_acceptor_internal - : public intrusive_list::node - , public std::enable_shared_from_this +class win_tcp_acceptor final + : public tcp_acceptor::implementation + , public intrusive_list::node { friend class win_tcp_service; - friend class win_tcp_acceptor; - -public: - explicit win_tcp_acceptor_internal(win_tcp_service& svc) noexcept; - ~win_tcp_acceptor_internal(); - - /// Return the owning socket service. - win_tcp_service& socket_service() noexcept; - - std::coroutine_handle<> accept( - std::coroutine_handle<>, - capy::executor_ref, - std::stop_token, - std::error_code*, - io_object::implementation**); - - std::coroutine_handle<> wait( - std::coroutine_handle<>, - capy::executor_ref, - wait_type, - std::stop_token, - std::error_code*); - - SOCKET native_handle() const noexcept; - endpoint local_endpoint() const noexcept; - bool is_open() const noexcept; - void cancel() noexcept; - void close_socket() noexcept; - void set_local_endpoint(endpoint ep) noexcept; + friend struct accept_op; + friend struct acceptor_wait_op; + win_tcp_service& svc_; accept_op acc_; acceptor_wait_op wt_; - -private: - win_tcp_service& svc_; SOCKET socket_ = INVALID_SOCKET; int family_ = AF_UNSPEC; endpoint local_endpoint_; -}; -/** Acceptor implementation wrapper for IOCP-based I/O. +public: + explicit win_tcp_acceptor(win_tcp_service& svc) noexcept; - This class is the public-facing implementation that holds a shared_ptr - to the internal state. The shared_ptr is hidden from the public interface. + /// Recycle into the owning service's pool. Defined out-of-line + /// after `win_tcp_service` is complete (needs `svc_.acceptor_pool_`). + void retire() noexcept override; - @note Internal implementation detail. Users interact with acceptor class. -*/ -class win_tcp_acceptor final - : public tcp_acceptor::implementation - , public intrusive_list::node -{ - std::shared_ptr internal_; + /** Reset recycled state for reuse. -public: - explicit win_tcp_acceptor( - std::shared_ptr internal) noexcept; + `close_socket()` already drives the socket handle, family, + and cached endpoint to their closed values, and drops every + embedded op's `object_ref_` before `refs_` can reach zero, so + there is little left to reset here -- only to assert. Each + op's `OVERLAPPED` fields are re-seeded by `overlapped_op:: + reset()` at its next submit, so they are not zeroed here. - void close_internal() noexcept; + @pre refs_ == 0, socket closed, no op in flight. + */ + void reuse() noexcept; + + /// Return the owning socket service. + win_tcp_service& socket_service() noexcept; std::coroutine_handle<> accept( std::coroutine_handle<> h, @@ -167,7 +145,7 @@ class win_tcp_acceptor final { // getsockname fails with WSAEINVAL on an unbound socket, so // the family recorded at socket creation is authoritative - return to_family(internal_ ? internal_->family_ : AF_UNSPEC); + return to_family(family_); } native_handle_type release_socket() noexcept override; @@ -180,7 +158,8 @@ class win_tcp_acceptor final get_option(int level, int optname, void* data, std::size_t* size) const noexcept override; - win_tcp_acceptor_internal* get_internal() const noexcept; + void set_local_endpoint(endpoint ep) noexcept; + void close_socket() noexcept; }; } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/iocp/win_tcp_acceptor_service.hpp b/include/boost/corosio/native/detail/iocp/win_tcp_acceptor_service.hpp index 651de18ba..d6a2ae2e0 100644 --- a/include/boost/corosio/native/detail/iocp/win_tcp_acceptor_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_tcp_acceptor_service.hpp @@ -38,7 +38,8 @@ namespace boost::corosio::detail { /** IOCP acceptor service wrapping win_tcp_service for acceptor lifecycle. Provides io_service + acceptor_service interface for tcp_acceptor - on Windows. Delegates to win_tcp_service for actual socket operations. + on Windows. Delegates to win_tcp_service for actual socket operations + and for the acceptor pool itself. */ class BOOST_COROSIO_DECL win_tcp_acceptor_service final : public capy::execution_context::service @@ -79,1388 +80,248 @@ class BOOST_COROSIO_DECL win_tcp_acceptor_service final // Operation constructors -inline connect_op::connect_op(win_tcp_socket_internal& internal_) noexcept +inline accept_op::accept_op(win_tcp_acceptor& acceptor_) noexcept : overlapped_op(&do_complete) - , internal(internal_) + , acceptor(acceptor_) { cancel_func_ = &do_cancel_impl; } -inline read_op::read_op(win_tcp_socket_internal& internal_) noexcept +inline acceptor_wait_op::acceptor_wait_op(win_tcp_acceptor& acceptor_) noexcept : overlapped_op(&do_complete) - , internal(internal_) + , acceptor(acceptor_) { cancel_func_ = &do_cancel_impl; } -inline write_op::write_op(win_tcp_socket_internal& internal_) noexcept - : overlapped_op(&do_complete) - , internal(internal_) -{ - cancel_func_ = &do_cancel_impl; -} - -inline wait_op::wait_op(win_tcp_socket_internal& internal_) noexcept - : overlapped_op(&do_complete) - , internal(internal_) -{ - cancel_func_ = &do_cancel_impl; -} - -inline accept_op::accept_op() noexcept : overlapped_op(&do_complete) -{ - cancel_func_ = &do_cancel_impl; -} - -inline acceptor_wait_op::acceptor_wait_op() noexcept - : overlapped_op(&do_complete) -{ - cancel_func_ = &do_cancel_impl; -} - -// Cancellation functions - -inline void -connect_op::do_cancel_impl(overlapped_op* base) noexcept -{ - auto* op = static_cast(base); - if (op->internal.is_open()) - { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); - } -} - -inline void -read_op::do_cancel_impl(overlapped_op* base) noexcept -{ - auto* op = static_cast(base); - op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) - { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); - } -} - -inline void -write_op::do_cancel_impl(overlapped_op* base) noexcept -{ - auto* op = static_cast(base); - op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) - { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); - } -} - -inline void -wait_op::do_cancel_impl(overlapped_op* base) noexcept -{ - auto* op = static_cast(base); - op->cancelled.store(true, std::memory_order_release); - // Best-effort cancel of any pending zero-byte WSARecv issued for - // wait_type::read. ERROR_NOT_FOUND when nothing overlapped is - // pending is harmless. - if (op->internal.is_open()) - { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); - } - // wait_type::error parks the op in the auxiliary select reactor; - // wake it so the reactor can post a cancelled completion. A cancel - // for an op the reactor never registered finds nothing and returns. - op->internal.svc_.scheduler().cancel_wait(op); -} - -inline void -accept_op::do_cancel_impl(overlapped_op* base) noexcept -{ - auto* op = static_cast(base); - if (op->listen_socket != INVALID_SOCKET) - { - ::CancelIoEx(reinterpret_cast(op->listen_socket), op); - } -} - -inline void -acceptor_wait_op::do_cancel_impl(overlapped_op* base) noexcept -{ - auto* op = static_cast(base); - op->cancelled.store(true, std::memory_order_release); - if (op->listen_socket != INVALID_SOCKET) - { - ::CancelIoEx(reinterpret_cast(op->listen_socket), op); - } - if (op->acceptor_ptr) - { - op->acceptor_ptr->socket_service().scheduler().cancel_wait(op); - } -} - -// accept_op completion handler - -inline void -accept_op::do_complete( - void* owner, - scheduler_op* base, - std::uint32_t /*bytes*/, - std::uint32_t /*error*/) -{ - auto* op = static_cast(base); - - if (!owner) - { - if (op->accepted_socket != INVALID_SOCKET) - { - ::closesocket(op->accepted_socket); - op->accepted_socket = INVALID_SOCKET; - } - - if (op->peer_wrapper) - { - op->peer_wrapper->close_internal(); - op->peer_wrapper = nullptr; - } - - op->cleanup_only(); - op->acceptor_ptr.reset(); - return; - } - - op->stop_cb.reset(); - - bool success = - (op->dwError == 0 && !op->cancelled.load(std::memory_order_acquire)); - - if (op->ec_out) - { - if (op->cancelled.load(std::memory_order_acquire)) - *op->ec_out = capy::error::canceled; - else if (op->dwError != 0) - *op->ec_out = iocp_make_err(op->dwError, /*accept_path=*/true); - else - *op->ec_out = {}; - } - - if (success && op->accepted_socket != INVALID_SOCKET && op->peer_wrapper) - { - ::setsockopt( - op->accepted_socket, SOL_SOCKET, SO_UPDATE_ACCEPT_CONTEXT, - reinterpret_cast(&op->listen_socket), sizeof(SOCKET)); - - op->peer_wrapper->get_internal()->set_socket(op->accepted_socket); - - sockaddr_storage local_storage{}; - int local_len = sizeof(local_storage); - sockaddr_storage remote_storage{}; - int remote_len = sizeof(remote_storage); - - endpoint local_ep, remote_ep; - if (::getsockname( - op->accepted_socket, - reinterpret_cast(&local_storage), &local_len) == 0) - local_ep = from_sockaddr(local_storage); - if (::getpeername( - op->accepted_socket, - reinterpret_cast(&remote_storage), &remote_len) == 0) - remote_ep = from_sockaddr(remote_storage); - - op->peer_wrapper->get_internal()->set_endpoints(local_ep, remote_ep); - op->accepted_socket = INVALID_SOCKET; - - if (op->impl_out) - *op->impl_out = op->peer_wrapper; - } - else - { - if (op->accepted_socket != INVALID_SOCKET) - { - ::closesocket(op->accepted_socket); - op->accepted_socket = INVALID_SOCKET; - } - - if (op->peer_wrapper) - { - op->acceptor_ptr->socket_service().destroy(op->peer_wrapper); - op->peer_wrapper = nullptr; - } - - if (op->impl_out) - *op->impl_out = nullptr; - } - - op->cont.h = op->h; - auto saved_ex = op->ex; - auto prevent_premature_destruction = std::move(op->acceptor_ptr); - - dispatch_coro(saved_ex, op->cont).resume(); -} - -// acceptor_wait_op completion handler - -inline void -acceptor_wait_op::do_complete( - void* owner, - scheduler_op* base, - std::uint32_t /*bytes*/, - std::uint32_t /*error*/) -{ - auto* op = static_cast(base); - - if (!owner) - { - op->cleanup_only(); - op->acceptor_ptr.reset(); - return; - } - - auto prevent_premature_destruction = std::move(op->acceptor_ptr); - op->invoke_handler(); -} - -// connect_op completion handler - -inline void -connect_op::do_complete( - void* owner, - scheduler_op* base, - std::uint32_t /*bytes*/, - std::uint32_t /*error*/) -{ - auto* op = static_cast(base); - - if (!owner) - { - op->cleanup_only(); - op->internal_ptr.reset(); - return; - } - - bool success = - (op->dwError == 0 && !op->cancelled.load(std::memory_order_acquire)); - if (success && op->internal.is_open()) - { - // Required after ConnectEx to enable shutdown(), getsockname(), etc. - ::setsockopt( - op->internal.native_handle(), SOL_SOCKET, SO_UPDATE_CONNECT_CONTEXT, - nullptr, 0); - - endpoint local_ep; - sockaddr_storage local_storage{}; - int local_len = sizeof(local_storage); - if (::getsockname( - op->internal.native_handle(), - reinterpret_cast(&local_storage), &local_len) == 0) - local_ep = from_sockaddr(local_storage); - op->internal.set_endpoints(local_ep, op->target_endpoint); - } - - auto prevent_premature_destruction = std::move(op->internal_ptr); - op->invoke_handler(); -} - -// read_op completion handler - -inline void -read_op::do_complete( - void* owner, - scheduler_op* base, - std::uint32_t /*bytes*/, - std::uint32_t /*error*/) -{ - auto* op = static_cast(base); - - if (!owner) - { - op->cleanup_only(); - op->internal_ptr.reset(); - return; - } - - auto prevent_premature_destruction = std::move(op->internal_ptr); - op->invoke_handler(); -} - -// write_op completion handler - -inline void -write_op::do_complete( - void* owner, - scheduler_op* base, - std::uint32_t /*bytes*/, - std::uint32_t /*error*/) -{ - auto* op = static_cast(base); - - if (!owner) - { - op->cleanup_only(); - op->internal_ptr.reset(); - return; - } - - auto prevent_premature_destruction = std::move(op->internal_ptr); - op->invoke_handler(); -} - -// wait_op completion handler - -inline void -wait_op::do_complete( - void* owner, - scheduler_op* base, - std::uint32_t /*bytes*/, - std::uint32_t /*error*/) -{ - auto* op = static_cast(base); - - if (!owner) - { - op->cleanup_only(); - op->internal_ptr.reset(); - return; - } - - auto prevent_premature_destruction = std::move(op->internal_ptr); - op->invoke_handler(); -} - -// win_tcp_socket_internal - -inline win_tcp_socket_internal::win_tcp_socket_internal( - win_tcp_service& svc) noexcept - : svc_(svc) - , conn_(*this) - , rd_(*this) - , wr_(*this) - , wt_(*this) -{ -} - -inline win_tcp_socket_internal::~win_tcp_socket_internal() -{ - svc_.unregister_impl(*this); -} - -inline SOCKET -win_tcp_socket_internal::native_handle() const noexcept -{ - return socket_; -} - -inline endpoint -win_tcp_socket_internal::local_endpoint() const noexcept -{ - return local_endpoint_; -} - -inline endpoint -win_tcp_socket_internal::remote_endpoint() const noexcept -{ - return remote_endpoint_; -} - -inline bool -win_tcp_socket_internal::is_open() const noexcept -{ - return socket_ != INVALID_SOCKET; -} - -inline void -win_tcp_socket_internal::set_socket(SOCKET s) noexcept -{ - socket_ = s; -} - -inline void -win_tcp_socket_internal::set_endpoints(endpoint local, endpoint remote) noexcept -{ - local_endpoint_ = local; - remote_endpoint_ = remote; -} - -inline std::coroutine_handle<> -win_tcp_socket_internal::connect( - std::coroutine_handle<> h, - capy::executor_ref d, - endpoint ep, - std::stop_token token, - std::error_code* ec) -{ - // Keep internal alive during I/O - conn_.internal_ptr = shared_from_this(); - - auto& op = conn_; - op.reset(); - op.h = h; - op.ex = d; - op.ec_out = ec; - op.target_endpoint = ep; - op.start(token); - - svc_.work_started(); - - // ConnectEx requires the socket to be bound. Skip if already bound - // (e.g. the caller used tcp_socket::bind() before connect). - if (local_endpoint_ == endpoint{}) - { - sockaddr_storage bind_storage{}; - socklen_t bind_len; - if (family_ == AF_INET6) - { - sockaddr_in6 sa6{}; - sa6.sin6_family = AF_INET6; - sa6.sin6_port = 0; - sa6.sin6_addr = in6addr_any; - std::memcpy(&bind_storage, &sa6, sizeof(sa6)); - bind_len = sizeof(sa6); - } - else - { - sockaddr_in sa4{}; - sa4.sin_family = AF_INET; - sa4.sin_addr.s_addr = INADDR_ANY; - sa4.sin_port = 0; - std::memcpy(&bind_storage, &sa4, sizeof(sa4)); - bind_len = sizeof(sa4); - } - - if (::bind( - socket_, reinterpret_cast(&bind_storage), - bind_len) == SOCKET_ERROR) - { - svc_.on_completion(&op, ::WSAGetLastError(), 0); - return std::noop_coroutine(); - } - } - - auto connect_ex = svc_.connect_ex(); - if (!connect_ex) - { - svc_.on_completion(&op, WSAEOPNOTSUPP, 0); - return std::noop_coroutine(); - } - - sockaddr_storage storage{}; - socklen_t addrlen = detail::to_sockaddr(ep, family_, storage); - - BOOL result = connect_ex( - socket_, reinterpret_cast(&storage), - static_cast(addrlen), nullptr, 0, nullptr, &op); - - if (!result) - { - DWORD err = ::WSAGetLastError(); - if (err != ERROR_IO_PENDING) - { - svc_.on_completion(&op, err, 0); - return std::noop_coroutine(); - } - } - - svc_.on_pending(&op); - return std::noop_coroutine(); -} - -inline std::coroutine_handle<> -win_tcp_socket_internal::read_some( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param param, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes_out) -{ - // Keep internal alive during I/O - rd_.internal_ptr = shared_from_this(); - - auto& op = rd_; - op.reset(); - op.is_read = true; - op.h = h; - op.ex = d; - op.ec_out = ec; - op.bytes_out = bytes_out; - op.start(token); - - svc_.work_started(); - - // Closed-object contract: complete with bad_file_descriptor - // without touching the kernel. - if (socket_ == INVALID_SOCKET) - { - svc_.on_completion(&op, WSAEBADF, 0); - return std::noop_coroutine(); - } - - // Prepare buffers - capy::mutable_buffer bufs[read_op::max_buffers]; - op.wsabuf_count = - static_cast(param.copy_to(bufs, read_op::max_buffers)); - - // Handle empty buffer: complete with 0 bytes - if (op.wsabuf_count == 0) - { - op.empty_buffer = true; - svc_.on_completion(&op, 0, 0); - return std::noop_coroutine(); - } - - for (DWORD i = 0; i < op.wsabuf_count; ++i) - { - op.wsabufs[i].buf = static_cast(bufs[i].data()); - op.wsabufs[i].len = static_cast( - (std::min)(bufs[i].size(), std::size_t(0x7fffffff))); - } - - op.flags = 0; - - int result = ::WSARecv( - socket_, op.wsabufs, op.wsabuf_count, nullptr, &op.flags, &op, nullptr); - - if (result == SOCKET_ERROR) - { - DWORD err = ::WSAGetLastError(); - if (err != WSA_IO_PENDING) - { - svc_.on_completion(&op, err, 0); - return std::noop_coroutine(); - } - } - - svc_.on_pending(&op); - - // Re-check cancellation after I/O is pending - if (op.cancelled.load(std::memory_order_acquire)) - ::CancelIoEx(reinterpret_cast(socket_), &op); - - return std::noop_coroutine(); -} - -inline std::coroutine_handle<> -win_tcp_socket_internal::write_some( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param param, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes_out) -{ - // Keep internal alive during I/O - wr_.internal_ptr = shared_from_this(); - - auto& op = wr_; - op.reset(); - op.h = h; - op.ex = d; - op.ec_out = ec; - op.bytes_out = bytes_out; - op.start(token); - - svc_.work_started(); - - // Closed-object contract: complete with bad_file_descriptor - // without touching the kernel. - if (socket_ == INVALID_SOCKET) - { - svc_.on_completion(&op, WSAEBADF, 0); - return std::noop_coroutine(); - } - - // Prepare buffers - capy::mutable_buffer bufs[write_op::max_buffers]; - op.wsabuf_count = - static_cast(param.copy_to(bufs, write_op::max_buffers)); - - // Handle empty buffer: complete immediately with 0 bytes - if (op.wsabuf_count == 0) - { - svc_.on_completion(&op, 0, 0); - return std::noop_coroutine(); - } - - for (DWORD i = 0; i < op.wsabuf_count; ++i) - { - op.wsabufs[i].buf = static_cast(bufs[i].data()); - op.wsabufs[i].len = static_cast( - (std::min)(bufs[i].size(), std::size_t(0x7fffffff))); - } - - int result = ::WSASend( - socket_, op.wsabufs, op.wsabuf_count, nullptr, 0, &op, nullptr); - - if (result == SOCKET_ERROR) - { - DWORD err = ::WSAGetLastError(); - if (err != WSA_IO_PENDING) - { - svc_.on_completion(&op, err, 0); - return std::noop_coroutine(); - } - } - - svc_.on_pending(&op); - - // Re-check cancellation after I/O is pending - if (op.cancelled.load(std::memory_order_acquire)) - ::CancelIoEx(reinterpret_cast(socket_), &op); - - return std::noop_coroutine(); -} - -inline std::coroutine_handle<> -win_tcp_socket_internal::wait( - std::coroutine_handle<> h, - capy::executor_ref d, - wait_type w, - std::stop_token token, - std::error_code* ec) -{ - wt_.internal_ptr = shared_from_this(); - - auto& op = wt_; - op.reset(); - op.h = h; - op.ex = d; - op.ec_out = ec; - op.bytes_out = nullptr; - op.empty_buffer = true; // skip EOF translation in invoke_handler - op.start(token); - - svc_.work_started(); - - // Closed-object contract: complete with bad_file_descriptor - // without touching the kernel or the wait reactor. - if (socket_ == INVALID_SOCKET) - { - svc_.on_completion(&op, WSAEBADF, 0); - return std::noop_coroutine(); - } - - if (w == wait_type::read) - { - // Zero-byte WSARecv: kernel signals completion when data is - // available without consuming any bytes from the stream. This - // is the documented Winsock pattern for "is the socket - // readable" notifications. - op.wsabuf = WSABUF{0, nullptr}; - op.flags = 0; - - int result = - ::WSARecv(socket_, &op.wsabuf, 1, nullptr, &op.flags, &op, nullptr); - - if (result == SOCKET_ERROR) - { - DWORD err = ::WSAGetLastError(); - if (err != WSA_IO_PENDING) - { - svc_.on_completion(&op, err, 0); - return std::noop_coroutine(); - } - } - - svc_.on_pending(&op); - - // Re-check cancellation after I/O is pending. - if (op.cancelled.load(std::memory_order_acquire)) - ::CancelIoEx(reinterpret_cast(socket_), &op); - - return std::noop_coroutine(); - } - - // wait_type::write and wait_type::error: route through the - // auxiliary poll reactor. There is no overlapped primitive for - // "the send buffer has room" that does not also transfer bytes, - // and a write wait must report real writability. - svc_.scheduler().wait_reactor().register_wait(socket_, w, &op); - return std::noop_coroutine(); -} - -inline void -win_tcp_socket_internal::cancel() noexcept -{ - if (socket_ != INVALID_SOCKET) - { - ::CancelIoEx(reinterpret_cast(socket_), nullptr); - } - - conn_.request_cancel(); - rd_.request_cancel(); - wr_.request_cancel(); - wt_.request_cancel(); - // CancelIoEx covers overlapped I/O on the socket but cannot reach - // a wait op parked in the auxiliary reactor (no overlapped is - // outstanding). Route through the reactor explicitly. - svc_.scheduler().cancel_wait(&wt_); -} - -inline void -win_tcp_socket_internal::close_socket() noexcept -{ - // Flag every op cancelled before tearing down the handle. closesocket() - // can complete a pending overlapped op with ERROR_NETNAME_DELETED (not the - // MSDN-documented ERROR_OPERATION_ABORTED), which iocp_make_err now maps to - // connection_reset -- so a locally-closed op must be short-circuited to - // canceled via the flag rather than relying on the raw error, exactly as - // cancel() does. The store happens-before the completion is decoded. - conn_.request_cancel(); - rd_.request_cancel(); - wr_.request_cancel(); - // wt_ is parked in the aux reactor (no overlapped outstanding), so it also - // needs an explicit reactor deregister before the SOCKET handle is closed; - // otherwise the reactor would keep polling a dangling fd (and on a Winsock - // SOCKET-id reuse the wrong fd could be polled briefly). - wt_.request_cancel(); - svc_.scheduler().cancel_wait(&wt_); - - if (socket_ != INVALID_SOCKET) - { - ::CancelIoEx(reinterpret_cast(socket_), nullptr); - ::closesocket(socket_); - socket_ = INVALID_SOCKET; - } - - family_ = AF_UNSPEC; - - // Clear cached endpoints - local_endpoint_ = endpoint{}; - remote_endpoint_ = endpoint{}; -} - -// win_tcp_socket - -inline win_tcp_socket::win_tcp_socket( - std::shared_ptr internal) noexcept - : internal_(std::move(internal)) -{ -} - -inline void -win_tcp_socket::close_internal() noexcept -{ - if (internal_) - { - internal_->close_socket(); - internal_.reset(); - } -} - -inline std::coroutine_handle<> -win_tcp_socket::connect( - std::coroutine_handle<> h, - capy::executor_ref d, - endpoint ep, - std::stop_token token, - std::error_code* ec) -{ - return internal_->connect(h, d, ep, token, ec); -} - -inline std::coroutine_handle<> -win_tcp_socket::read_some( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->read_some(h, d, buf, token, ec, bytes); -} - -inline std::coroutine_handle<> -win_tcp_socket::write_some( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->write_some(h, d, buf, token, ec, bytes); -} - -inline std::coroutine_handle<> -win_tcp_socket::wait( - std::coroutine_handle<> h, - capy::executor_ref d, - wait_type w, - std::stop_token token, - std::error_code* ec) -{ - return internal_->wait(h, d, w, token, ec); -} - -inline std::error_code -win_tcp_socket::shutdown(tcp_socket::shutdown_type what) noexcept -{ - int how; - switch (what) - { - case tcp_socket::shutdown_receive: - how = SD_RECEIVE; - break; - case tcp_socket::shutdown_send: - how = SD_SEND; - break; - case tcp_socket::shutdown_both: - how = SD_BOTH; - break; - default: - return make_err(WSAEINVAL); - } - if (::shutdown(internal_->native_handle(), how) != 0) - return make_err(WSAGetLastError()); - return {}; -} - -inline native_handle_type -win_tcp_socket::native_handle() const noexcept -{ - return static_cast(internal_->native_handle()); -} - -inline native_handle_type -win_tcp_socket::release_socket() noexcept -{ - SOCKET s = internal_->socket_; - if (s != INVALID_SOCKET) - { - internal_->cancel(); - // Sever the port association so the descriptor can be - // adopted again; best-effort, the caller keeps a working - // socket either way. - dissociate_from_iocp(s); - internal_->socket_ = INVALID_SOCKET; - internal_->family_ = AF_UNSPEC; - internal_->local_endpoint_ = endpoint{}; - internal_->remote_endpoint_ = endpoint{}; - } - return static_cast(s); -} - -inline std::error_code -win_tcp_socket::set_option( - int level, int optname, void const* data, std::size_t size) noexcept -{ - if (::setsockopt( - internal_->native_handle(), level, optname, - reinterpret_cast(data), static_cast(size)) != 0) - return make_err(WSAGetLastError()); - return {}; -} - -inline std::error_code -win_tcp_socket::get_option( - int level, int optname, void* data, std::size_t* size) const noexcept -{ - int len = static_cast(*size); - if (::getsockopt( - internal_->native_handle(), level, optname, - reinterpret_cast(data), &len) != 0) - return make_err(WSAGetLastError()); - *size = static_cast(len); - return {}; -} - -inline endpoint -win_tcp_socket::local_endpoint() const noexcept -{ - return internal_->local_endpoint(); -} - -inline endpoint -win_tcp_socket::remote_endpoint() const noexcept -{ - return internal_->remote_endpoint(); -} - -inline void -win_tcp_socket::cancel() noexcept -{ - internal_->cancel(); -} - -inline win_tcp_socket_internal* -win_tcp_socket::get_internal() const noexcept -{ - return internal_.get(); -} - -// win_tcp_service - -inline win_tcp_service::win_tcp_service(capy::execution_context& ctx) - : sched_(ctx.use_service()) - , iocp_(sched_.native_handle()) -{ - load_extension_functions(); -} - -inline win_tcp_service::~win_tcp_service() -{ - // Delete wrappers that survived shutdown. This runs after - // win_scheduler is destroyed (reverse creation order), so - // all coroutine frames and their tcp_socket members are gone. - for (auto* w = socket_wrapper_list_.pop_front(); w != nullptr; - w = socket_wrapper_list_.pop_front()) - delete w; - - for (auto* w = acceptor_wrapper_list_.pop_front(); w != nullptr; - w = acceptor_wrapper_list_.pop_front()) - delete w; -} - -inline void -win_tcp_service::shutdown() -{ - std::lock_guard lock(mutex_); - - // Close all sockets to force pending I/O to complete via IOCP. - // Wrappers are NOT deleted here - coroutine frames destroyed - // during scheduler shutdown may still hold tcp_socket objects - // that reference them. Wrapper deletion is deferred to ~win_tcp_service - // after the scheduler has drained all outstanding operations. - for (auto* impl = socket_list_.pop_front(); impl != nullptr; - impl = socket_list_.pop_front()) - { - impl->close_socket(); - } - - for (auto* impl = acceptor_list_.pop_front(); impl != nullptr; - impl = acceptor_list_.pop_front()) - { - impl->close_socket(); - } -} - -inline io_object::implementation* -win_tcp_service::construct() -{ - auto internal = std::make_shared(*this); - - { - std::lock_guard lock(mutex_); - socket_list_.push_back(internal.get()); - } - - auto* wrapper = new win_tcp_socket(std::move(internal)); - - { - std::lock_guard lock(mutex_); - socket_wrapper_list_.push_back(wrapper); - } - - return wrapper; -} - -inline void -win_tcp_service::destroy(io_object::implementation* p) -{ - if (p) - { - auto& wrapper = static_cast(*p); - wrapper.close_internal(); - destroy_impl(wrapper); - } -} - -inline void -win_tcp_service::close(io_object::handle& h) -{ - auto& wrapper = static_cast(*h.get()); - wrapper.get_internal()->close_socket(); -} +// Cancellation functions inline void -win_tcp_service::destroy_impl(win_tcp_socket& impl) +accept_op::do_cancel_impl(overlapped_op* base) noexcept { + auto* op = static_cast(base); + if (op->listen_socket != INVALID_SOCKET) { - std::lock_guard lock(mutex_); - socket_wrapper_list_.remove(&impl); + ::CancelIoEx(reinterpret_cast(op->listen_socket), op); } - delete &impl; } inline void -win_tcp_service::unregister_impl(win_tcp_socket_internal& impl) -{ - std::lock_guard lock(mutex_); - socket_list_.remove(&impl); -} - -inline std::error_code -win_tcp_service::open_socket( - win_tcp_socket_internal& impl, int family, int type, int protocol) +acceptor_wait_op::do_cancel_impl(overlapped_op* base) noexcept { - impl.close_socket(); - - SOCKET sock = - ::WSASocketW(family, type, protocol, nullptr, 0, WSA_FLAG_OVERLAPPED); - - if (sock == INVALID_SOCKET) - return make_err(::WSAGetLastError()); - - if (family == AF_INET6) - { - DWORD one = 1; - ::setsockopt( - sock, IPPROTO_IPV6, IPV6_V6ONLY, reinterpret_cast(&one), - sizeof(one)); - } - - HANDLE result = ::CreateIoCompletionPort( - reinterpret_cast(sock), static_cast(iocp_), key_io, 0); - - if (result == nullptr) + auto* op = static_cast(base); + op->cancelled.store(true, std::memory_order_release); + if (op->listen_socket != INVALID_SOCKET) { - DWORD dwError = ::GetLastError(); - ::closesocket(sock); - return make_err(dwError); + ::CancelIoEx(reinterpret_cast(op->listen_socket), op); } - - impl.socket_ = sock; - impl.family_ = family; - return {}; -} - -inline std::error_code -win_tcp_service::assign_socket( - win_tcp_socket_internal& impl, native_handle_type fd) -{ - SOCKET sock = static_cast(fd); - if (sock == INVALID_SOCKET) - return make_err(WSAENOTSOCK); - if (sock == impl.socket_) - return std::make_error_code(std::errc::invalid_argument); - - // SO_PROTOCOL_INFOW works on an unbound socket, unlike getsockname - // (WSAEINVAL until bind/connect names it) -- an adopted socket may - // have reached connected state without an explicit bind. - WSAPROTOCOL_INFOW proto_info{}; - int proto_len = sizeof(proto_info); - if (::getsockopt( - sock, SOL_SOCKET, SO_PROTOCOL_INFOW, - reinterpret_cast(&proto_info), &proto_len) != 0) - return make_err(::WSAGetLastError()); - if (proto_info.iAddressFamily != AF_INET && - proto_info.iAddressFamily != AF_INET6) - return make_err(WSAEAFNOSUPPORT); - if (proto_info.iSocketType != SOCK_STREAM) - return make_err(WSAEPROTOTYPE); - - // Associate before releasing the held socket: on IOCP nothing - // shares descriptor state the way the reactor path does, so a - // failed association must not cost the caller their old socket. - HANDLE result = ::CreateIoCompletionPort( - reinterpret_cast(sock), static_cast(iocp_), key_io, 0); - if (result == nullptr) - return make_err(::GetLastError()); - - impl.close_socket(); - impl.socket_ = sock; - impl.family_ = proto_info.iAddressFamily; - - endpoint local_ep, remote_ep; - sockaddr_storage local_storage{}; - int local_len = sizeof(local_storage); - if (::getsockname( - sock, reinterpret_cast(&local_storage), &local_len) == 0) - local_ep = detail::from_sockaddr(local_storage); - sockaddr_storage remote_storage{}; - int remote_len = sizeof(remote_storage); - if (::getpeername( - sock, reinterpret_cast(&remote_storage), &remote_len) == - 0) - remote_ep = detail::from_sockaddr(remote_storage); - impl.set_endpoints(local_ep, remote_ep); - - return {}; -} - -inline std::error_code -win_tcp_service::bind_socket(win_tcp_socket_internal& impl, endpoint ep) -{ - SOCKET sock = impl.socket_; - - sockaddr_storage storage{}; - socklen_t addrlen = detail::to_sockaddr(ep, storage); - if (::bind( - sock, reinterpret_cast(&storage), - static_cast(addrlen)) == SOCKET_ERROR) - return make_err(::WSAGetLastError()); - - // Cache local endpoint (resolves ephemeral port) - sockaddr_storage local_storage{}; - int local_len = sizeof(local_storage); - if (::getsockname( - sock, reinterpret_cast(&local_storage), &local_len) == 0) - impl.local_endpoint_ = detail::from_sockaddr(local_storage); - - return {}; -} - -inline void* -win_tcp_service::native_handle() const noexcept -{ - return iocp_; -} - -inline void -win_tcp_service::post(overlapped_op* op) -{ - sched_.post(op); -} - -inline void -win_tcp_service::on_pending(overlapped_op* op) noexcept -{ - sched_.on_pending(op); + op->acceptor.socket_service().scheduler().cancel_wait(op); } -inline void -win_tcp_service::on_completion( - overlapped_op* op, DWORD error, DWORD bytes) noexcept -{ - sched_.on_completion(op, error, bytes); -} +// accept_op completion handler inline void -win_tcp_service::work_started() noexcept +accept_op::do_complete( + void* owner, + scheduler_op* base, + std::uint32_t /*bytes*/, + std::uint32_t /*error*/) { - sched_.work_started(); -} + auto* op = static_cast(base); -inline void -win_tcp_service::work_finished() noexcept -{ - sched_.work_finished(); -} + if (!owner) + { + if (op->accepted_socket != INVALID_SOCKET) + { + ::closesocket(op->accepted_socket); + op->accepted_socket = INVALID_SOCKET; + } -inline void -win_tcp_service::load_extension_functions() -{ - SOCKET sock = ::WSASocketW( - AF_INET, SOCK_STREAM, IPPROTO_TCP, nullptr, 0, WSA_FLAG_OVERLAPPED); + if (op->peer_wrapper) + { + op->peer_wrapper->close_socket(); + op->peer_wrapper = nullptr; + } - if (sock == INVALID_SOCKET) + op->cleanup_only(); + op->object_ref_.reset(); return; + } - DWORD bytes = 0; - - GUID connect_ex_guid = WSAID_CONNECTEX; - ::WSAIoctl( - sock, SIO_GET_EXTENSION_FUNCTION_POINTER, &connect_ex_guid, - sizeof(connect_ex_guid), &connect_ex_, sizeof(connect_ex_), &bytes, - nullptr, nullptr); - - GUID accept_ex_guid = WSAID_ACCEPTEX; - ::WSAIoctl( - sock, SIO_GET_EXTENSION_FUNCTION_POINTER, &accept_ex_guid, - sizeof(accept_ex_guid), &accept_ex_, sizeof(accept_ex_), &bytes, - nullptr, nullptr); + op->stop_cb.reset(); - ::closesocket(sock); -} + bool success = + (op->dwError == 0 && !op->cancelled.load(std::memory_order_acquire)); -inline void -win_tcp_service::destroy_acceptor_impl(win_tcp_acceptor& impl) -{ + if (op->ec_out) { - std::lock_guard lock(mutex_); - acceptor_wrapper_list_.remove(&impl); + if (op->cancelled.load(std::memory_order_acquire)) + *op->ec_out = capy::error::canceled; + else if (op->dwError != 0) + *op->ec_out = iocp_make_err(op->dwError, /*accept_path=*/true); + else + *op->ec_out = {}; } - delete &impl; -} - -inline void -win_tcp_service::unregister_acceptor_impl(win_tcp_acceptor_internal& impl) -{ - std::lock_guard lock(mutex_); - acceptor_list_.remove(&impl); -} -inline std::error_code -win_tcp_service::open_acceptor_socket( - win_tcp_acceptor_internal& impl, int family, int type, int protocol) -{ - impl.close_socket(); - - SOCKET sock = - ::WSASocketW(family, type, protocol, nullptr, 0, WSA_FLAG_OVERLAPPED); - - if (sock == INVALID_SOCKET) - return make_err(::WSAGetLastError()); - - if (family == AF_INET6) + if (success && op->accepted_socket != INVALID_SOCKET && op->peer_wrapper) { - DWORD val = 0; // dual-stack default ::setsockopt( - sock, IPPROTO_IPV6, IPV6_V6ONLY, reinterpret_cast(&val), - sizeof(val)); - } + op->accepted_socket, SOL_SOCKET, SO_UPDATE_ACCEPT_CONTEXT, + reinterpret_cast(&op->listen_socket), sizeof(SOCKET)); - HANDLE result = ::CreateIoCompletionPort( - reinterpret_cast(sock), static_cast(iocp_), key_io, 0); + op->peer_wrapper->set_socket(op->accepted_socket); - if (result == nullptr) - { - DWORD dwError = ::GetLastError(); - ::closesocket(sock); - return make_err(dwError); - } + sockaddr_storage local_storage{}; + int local_len = sizeof(local_storage); + sockaddr_storage remote_storage{}; + int remote_len = sizeof(remote_storage); - impl.socket_ = sock; - impl.family_ = family; - return {}; -} + endpoint local_ep, remote_ep; + if (::getsockname( + op->accepted_socket, + reinterpret_cast(&local_storage), &local_len) == 0) + local_ep = from_sockaddr(local_storage); + if (::getpeername( + op->accepted_socket, + reinterpret_cast(&remote_storage), &remote_len) == 0) + remote_ep = from_sockaddr(remote_storage); -inline std::error_code -win_tcp_service::assign_acceptor_socket( - win_tcp_acceptor_internal& impl, native_handle_type fd) -{ - SOCKET sock = static_cast(fd); - if (sock == INVALID_SOCKET) - return make_err(WSAENOTSOCK); - if (sock == impl.socket_) - return std::make_error_code(std::errc::invalid_argument); - - // SO_PROTOCOL_INFOW works on an unbound socket, unlike getsockname - // (WSAEINVAL until bind names it). - WSAPROTOCOL_INFOW proto_info{}; - int proto_len = sizeof(proto_info); - if (::getsockopt( - sock, SOL_SOCKET, SO_PROTOCOL_INFOW, - reinterpret_cast(&proto_info), &proto_len) != 0) - return make_err(::WSAGetLastError()); - if (proto_info.iAddressFamily != AF_INET && - proto_info.iAddressFamily != AF_INET6) - return make_err(WSAEAFNOSUPPORT); - if (proto_info.iSocketType != SOCK_STREAM) - return make_err(WSAEPROTOTYPE); - - // Associate before releasing the held socket: on IOCP nothing - // shares descriptor state the way the reactor path does, so a - // failed association must not cost the caller their old socket. - HANDLE result = ::CreateIoCompletionPort( - reinterpret_cast(sock), static_cast(iocp_), key_io, 0); - if (result == nullptr) - return make_err(::GetLastError()); + op->peer_wrapper->set_endpoints(local_ep, remote_ep); + op->accepted_socket = INVALID_SOCKET; - impl.close_socket(); - impl.socket_ = sock; - impl.family_ = proto_info.iAddressFamily; + if (op->impl_out) + *op->impl_out = op->peer_wrapper; + // Handed off to the caller: don't let a failed early return on + // the next accept() see this (now user-owned) pointer again. + op->peer_wrapper = nullptr; + } + else + { + if (op->accepted_socket != INVALID_SOCKET) + { + ::closesocket(op->accepted_socket); + op->accepted_socket = INVALID_SOCKET; + } - // AcceptEx sizes its address buffers from this cache, so an - // unseeded endpoint breaks accepts on an adopted v6 listener. - sockaddr_storage local_storage{}; - int local_len = sizeof(local_storage); - if (::getsockname( - sock, reinterpret_cast(&local_storage), &local_len) == 0) - impl.set_local_endpoint(detail::from_sockaddr(local_storage)); + if (op->peer_wrapper) + { + op->acceptor.socket_service().destroy(op->peer_wrapper); + op->peer_wrapper = nullptr; + } - return {}; -} + if (op->impl_out) + *op->impl_out = nullptr; + } -inline std::error_code -win_tcp_service::bind_acceptor(win_tcp_acceptor_internal& impl, endpoint ep) -{ - SOCKET sock = impl.socket_; - - sockaddr_storage storage{}; - socklen_t addrlen = detail::to_sockaddr(ep, storage); - if (::bind( - sock, reinterpret_cast(&storage), - static_cast(addrlen)) == SOCKET_ERROR) - return make_err(::WSAGetLastError()); - - // Cache local endpoint (resolves ephemeral port) - sockaddr_storage local_storage{}; - int local_len = sizeof(local_storage); - if (::getsockname( - sock, reinterpret_cast(&local_storage), &local_len) == 0) - impl.set_local_endpoint(detail::from_sockaddr(local_storage)); + op->cont.h = op->h; + auto saved_ex = op->ex; + auto prevent_premature_destruction = std::move(op->object_ref_); - return {}; + dispatch_coro(saved_ex, op->cont).resume(); } -inline std::error_code -win_tcp_service::listen_acceptor(win_tcp_acceptor_internal& impl, int backlog) +// acceptor_wait_op completion handler + +inline void +acceptor_wait_op::do_complete( + void* owner, + scheduler_op* base, + std::uint32_t /*bytes*/, + std::uint32_t /*error*/) { - SOCKET sock = impl.socket_; + auto* op = static_cast(base); - if (::listen(sock, backlog) == SOCKET_ERROR) - return make_err(::WSAGetLastError()); + if (!owner) + { + op->cleanup_only(); + op->object_ref_.reset(); + return; + } - return {}; + auto prevent_premature_destruction = std::move(op->object_ref_); + op->invoke_handler(); } -// win_tcp_acceptor_internal +// win_tcp_acceptor -inline win_tcp_acceptor_internal::win_tcp_acceptor_internal( - win_tcp_service& svc) noexcept +inline win_tcp_acceptor::win_tcp_acceptor(win_tcp_service& svc) noexcept : svc_(svc) + , acc_(*this) + , wt_(*this) { } -inline win_tcp_acceptor_internal::~win_tcp_acceptor_internal() +inline void +win_tcp_acceptor::reuse() noexcept { - svc_.unregister_acceptor_impl(*this); + BOOST_COROSIO_ASSERT(socket_ == INVALID_SOCKET); + BOOST_COROSIO_ASSERT(family_ == AF_UNSPEC); + BOOST_COROSIO_ASSERT(local_endpoint_ == endpoint{}); + BOOST_COROSIO_ASSERT(!acc_.stop_cb); + BOOST_COROSIO_ASSERT(!wt_.stop_cb); + BOOST_COROSIO_ASSERT(acc_.peer_wrapper == nullptr); + BOOST_COROSIO_ASSERT(acc_.accepted_socket == INVALID_SOCKET); } inline win_tcp_service& -win_tcp_acceptor_internal::socket_service() noexcept +win_tcp_acceptor::socket_service() noexcept { return svc_; } -inline SOCKET -win_tcp_acceptor_internal::native_handle() const noexcept +inline native_handle_type +win_tcp_acceptor::native_handle() const noexcept { - return socket_; + return static_cast(socket_); } inline endpoint -win_tcp_acceptor_internal::local_endpoint() const noexcept +win_tcp_acceptor::local_endpoint() const noexcept { return local_endpoint_; } inline bool -win_tcp_acceptor_internal::is_open() const noexcept +win_tcp_acceptor::is_open() const noexcept { return socket_ != INVALID_SOCKET; } inline void -win_tcp_acceptor_internal::set_local_endpoint(endpoint ep) noexcept +win_tcp_acceptor::set_local_endpoint(endpoint ep) noexcept { local_endpoint_ = ep; } inline std::coroutine_handle<> -win_tcp_acceptor_internal::accept( +win_tcp_acceptor::accept( std::coroutine_handle<> h, capy::executor_ref d, std::stop_token token, std::error_code* ec, io_object::implementation** impl_out) { - // Keep acceptor internal alive during I/O - acc_.acceptor_ptr = shared_from_this(); + // Keep this acceptor alive during I/O + acc_.object_ref_ = detail::object_ref(this); auto& op = acc_; op.reset(); - op.h = h; - op.ex = d; - op.ec_out = ec; - op.impl_out = impl_out; + // reset() doesn't touch these -- a value left over from a prior + // accept() (e.g. the socket already handed to the user via + // *impl_out) must not survive into a completion this call fails + // before reassigning; do_complete's failure branch would destroy() + // whatever stale pointer it finds here. + op.peer_wrapper = nullptr; + op.accepted_socket = INVALID_SOCKET; + op.h = h; + op.ex = d; + op.ec_out = ec; + op.impl_out = impl_out; op.start(token); svc_.work_started(); - // Create wrapper for the peer socket (service owns it) + // Create a (possibly recycled) peer socket; the service owns it. auto& peer_wrapper = static_cast(*svc_.construct()); // Derive AF from the listening socket's cached local endpoint @@ -1541,14 +402,14 @@ win_tcp_acceptor_internal::accept( } inline std::coroutine_handle<> -win_tcp_acceptor_internal::wait( +win_tcp_acceptor::wait( std::coroutine_handle<> h, capy::executor_ref d, wait_type w, std::stop_token token, std::error_code* ec) { - wt_.acceptor_ptr = shared_from_this(); + wt_.object_ref_ = detail::object_ref(this); wt_.listen_socket = socket_; auto& op = wt_; @@ -1584,7 +445,7 @@ win_tcp_acceptor_internal::wait( } inline void -win_tcp_acceptor_internal::cancel() noexcept +win_tcp_acceptor::cancel() noexcept { if (socket_ != INVALID_SOCKET) { @@ -1597,7 +458,7 @@ win_tcp_acceptor_internal::cancel() noexcept } inline void -win_tcp_acceptor_internal::close_socket() noexcept +win_tcp_acceptor::close_socket() noexcept { // Flag the accept op cancelled before closing so a closesocket-delivered // ERROR_NETNAME_DELETED is short-circuited to canceled rather than mapped @@ -1620,85 +481,17 @@ win_tcp_acceptor_internal::close_socket() noexcept local_endpoint_ = endpoint{}; } -// win_tcp_acceptor - -inline win_tcp_acceptor::win_tcp_acceptor( - std::shared_ptr internal) noexcept - : internal_(std::move(internal)) -{ -} - -inline void -win_tcp_acceptor::close_internal() noexcept -{ - if (internal_) - { - internal_->close_socket(); - internal_.reset(); - } -} - -inline std::coroutine_handle<> -win_tcp_acceptor::accept( - std::coroutine_handle<> h, - capy::executor_ref d, - std::stop_token token, - std::error_code* ec, - io_object::implementation** impl_out) -{ - return internal_->accept(h, d, token, ec, impl_out); -} - -inline std::coroutine_handle<> -win_tcp_acceptor::wait( - std::coroutine_handle<> h, - capy::executor_ref d, - wait_type w, - std::stop_token token, - std::error_code* ec) -{ - return internal_->wait(h, d, w, token, ec); -} - -inline endpoint -win_tcp_acceptor::local_endpoint() const noexcept -{ - return internal_->local_endpoint(); -} - -inline bool -win_tcp_acceptor::is_open() const noexcept -{ - return internal_ && internal_->is_open(); -} - -inline void -win_tcp_acceptor::cancel() noexcept -{ - internal_->cancel(); -} - -inline native_handle_type -win_tcp_acceptor::native_handle() const noexcept -{ - if (!internal_) - return static_cast(INVALID_SOCKET); - return static_cast(internal_->native_handle()); -} - inline native_handle_type win_tcp_acceptor::release_socket() noexcept { - if (!internal_) - return static_cast(INVALID_SOCKET); - SOCKET s = internal_->socket_; + SOCKET s = socket_; if (s != INVALID_SOCKET) { - internal_->cancel(); + cancel(); dissociate_from_iocp(s); - internal_->socket_ = INVALID_SOCKET; - internal_->family_ = AF_UNSPEC; - internal_->local_endpoint_ = endpoint{}; + socket_ = INVALID_SOCKET; + family_ = AF_UNSPEC; + local_endpoint_ = endpoint{}; } return static_cast(s); } @@ -1708,8 +501,8 @@ win_tcp_acceptor::set_option( int level, int optname, void const* data, std::size_t size) noexcept { if (::setsockopt( - internal_->native_handle(), level, optname, - reinterpret_cast(data), static_cast(size)) != 0) + socket_, level, optname, reinterpret_cast(data), + static_cast(size)) != 0) return make_err(WSAGetLastError()); return {}; } @@ -1720,17 +513,16 @@ win_tcp_acceptor::get_option( { int len = static_cast(*size); if (::getsockopt( - internal_->native_handle(), level, optname, - reinterpret_cast(data), &len) != 0) + socket_, level, optname, reinterpret_cast(data), &len) != 0) return make_err(WSAGetLastError()); *size = static_cast(len); return {}; } -inline win_tcp_acceptor_internal* -win_tcp_acceptor::get_internal() const noexcept +inline void +win_tcp_acceptor::retire() noexcept { - return internal_.get(); + svc_.acceptor_pool_.recycle(this); } // win_tcp_acceptor_service @@ -1744,77 +536,61 @@ inline win_tcp_acceptor_service::win_tcp_acceptor_service( inline io_object::implementation* win_tcp_acceptor_service::construct() { - auto internal = std::make_shared(svc_); - - { - std::lock_guard lock(svc_.mutex_); - svc_.acceptor_list_.push_back(internal.get()); - } - - auto* wrapper = new win_tcp_acceptor(std::move(internal)); - - { - std::lock_guard lock(svc_.mutex_); - svc_.acceptor_wrapper_list_.push_back(wrapper); - } - - return wrapper; + return svc_.acquire_acceptor_impl(); } inline void win_tcp_acceptor_service::destroy(io_object::implementation* p) { - if (p) - { - auto& wrapper = static_cast(*p); - wrapper.close_internal(); - svc_.destroy_acceptor_impl(wrapper); - } + if (!p) + return; + auto* a = static_cast(p); + a->close_socket(); + release(a); } inline void win_tcp_acceptor_service::close(io_object::handle& h) { - auto& wrapper = static_cast(*h.get()); - wrapper.get_internal()->close_socket(); + static_cast(h.get())->close_socket(); } inline std::error_code win_tcp_acceptor_service::open_acceptor_socket( tcp_acceptor::implementation& impl, int family, int type, int protocol) { - auto& wrapper = static_cast(impl); - return svc_.open_acceptor_socket( - *wrapper.get_internal(), family, type, protocol); + auto& acc = static_cast(impl); + return svc_.open_acceptor_socket(acc, family, type, protocol); } inline std::error_code win_tcp_acceptor_service::assign_socket( tcp_acceptor::implementation& impl, native_handle_type fd) { - auto& wrapper = static_cast(impl); - return svc_.assign_acceptor_socket(*wrapper.get_internal(), fd); + auto& acc = static_cast(impl); + return svc_.assign_acceptor_socket(acc, fd); } inline std::error_code win_tcp_acceptor_service::bind_acceptor( tcp_acceptor::implementation& impl, endpoint ep) { - auto& wrapper = static_cast(impl); - return svc_.bind_acceptor(*wrapper.get_internal(), ep); + auto& acc = static_cast(impl); + return svc_.bind_acceptor(acc, ep); } inline std::error_code win_tcp_acceptor_service::listen_acceptor( tcp_acceptor::implementation& impl, int backlog) { - auto& wrapper = static_cast(impl); - return svc_.listen_acceptor(*wrapper.get_internal(), backlog); + auto& acc = static_cast(impl); + return svc_.listen_acceptor(acc, backlog); } inline void win_tcp_acceptor_service::shutdown() { + // Socket/acceptor shutdown is handled by win_tcp_service::shutdown() } } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/iocp/win_tcp_service.hpp b/include/boost/corosio/native/detail/iocp/win_tcp_service.hpp index 2203b2fa0..aa57b196e 100644 --- a/include/boost/corosio/native/detail/iocp/win_tcp_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_tcp_service.hpp @@ -16,13 +16,18 @@ #if BOOST_COROSIO_HAS_IOCP #include +#include +#include #include #include -#include +#include #include #include +#include +#include #include +#include #include @@ -30,18 +35,18 @@ namespace boost::corosio::detail { class win_scheduler; class win_tcp_acceptor; -class win_tcp_acceptor_internal; class win_tcp_acceptor_service; /** Windows IOCP socket management service. - This service owns all socket implementations and coordinates their - lifecycle with the IOCP. It provides: + This service owns all socket and acceptor implementations and + coordinates their lifecycle with the IOCP. It provides: - - Socket implementation allocation and deallocation + - Socket/acceptor implementation recycling through `object_pool` - IOCP handle association for sockets - Function pointer loading for ConnectEx/AcceptEx - - Graceful shutdown - destroys all implementations when io_context stops + - Graceful shutdown - closes every live implementation when the + io_context stops; pool destruction frees whatever is left @par Thread Safety All public member functions are thread-safe. @@ -53,6 +58,9 @@ class BOOST_COROSIO_DECL win_tcp_service final , public capy::execution_context::service , public io_object::io_service { + friend class win_tcp_socket; + friend class win_tcp_acceptor; + public: using key_type = win_tcp_service; @@ -71,32 +79,19 @@ class BOOST_COROSIO_DECL win_tcp_service final */ explicit win_tcp_service(capy::execution_context& ctx); - /** Destroy the socket service. */ - ~win_tcp_service(); - win_tcp_service(win_tcp_service const&) = delete; win_tcp_service& operator=(win_tcp_service const&) = delete; /** Shut down the service. */ void shutdown() override; - /** Destroy a socket implementation wrapper. - Removes from tracking list and deletes. - */ - void destroy_impl(win_tcp_socket& impl); - - /** Unregister a socket implementation from the service list. - Called by the internal impl destructor. - */ - void unregister_impl(win_tcp_socket_internal& impl); - /** Create and register a socket with the IOCP. - @param impl The socket implementation internal to initialize. + @param impl The socket implementation to initialize. @return Error code, or success. */ std::error_code open_socket( - win_tcp_socket_internal& impl, int family, int type, int protocol); + win_tcp_socket& impl, int family, int type, int protocol); /** Adopt an existing socket handle into an implementation. @@ -105,30 +100,24 @@ class BOOST_COROSIO_DECL win_tcp_service final impl takes ownership and will close the handle; on failure the caller retains ownership. - @param impl The socket implementation internal to assign to. + @param impl The socket implementation to assign to. @param fd The native socket handle to adopt. @return Error code, or success. */ - std::error_code - assign_socket(win_tcp_socket_internal& impl, native_handle_type fd); + std::error_code assign_socket(win_tcp_socket& impl, native_handle_type fd); /** Bind a stream socket to a local endpoint. - @param impl The socket implementation internal to bind. + @param impl The socket implementation to bind. @param ep The local endpoint to bind to. @return Error code, or success. */ - std::error_code bind_socket(win_tcp_socket_internal& impl, endpoint ep); - - /** Destroy an acceptor implementation wrapper. - Removes from tracking list and deletes. - */ - void destroy_acceptor_impl(win_tcp_acceptor& impl); + std::error_code bind_socket(win_tcp_socket& impl, endpoint ep); - /** Unregister an acceptor implementation from the service list. - Called by the internal impl destructor. + /** Pop a recycled acceptor or news one, with the service reference + held. Used by `win_tcp_acceptor_service::construct()`. */ - void unregister_acceptor_impl(win_tcp_acceptor_internal& impl); + win_tcp_acceptor* acquire_acceptor_impl(); /** Create an acceptor socket without binding or listening. @@ -136,14 +125,14 @@ class BOOST_COROSIO_DECL win_tcp_service final For IPv6, dual-stack is enabled by default. Does not set SO_REUSEADDR. - @param impl The acceptor implementation internal to initialize. + @param impl The acceptor implementation to initialize. @param family Address family (e.g. `AF_INET`, `AF_INET6`). @param type Socket type (e.g. `SOCK_STREAM`). @param protocol Protocol number (e.g. `IPPROTO_TCP`). @return Error code, or success. */ std::error_code open_acceptor_socket( - win_tcp_acceptor_internal& impl, int family, int type, int protocol); + win_tcp_acceptor& impl, int family, int type, int protocol); /** Adopt an existing listening socket into an acceptor. @@ -151,30 +140,29 @@ class BOOST_COROSIO_DECL win_tcp_service final then releases the socket the acceptor already held. Listen state is not verified. - @param impl The acceptor implementation internal. + @param impl The acceptor implementation. @param fd The native socket to adopt. Ownership transfers only on success. @return Error code, or success. */ - std::error_code assign_acceptor_socket( - win_tcp_acceptor_internal& impl, native_handle_type fd); + std::error_code + assign_acceptor_socket(win_tcp_acceptor& impl, native_handle_type fd); /** Bind an open acceptor to a local endpoint. - @param impl The acceptor implementation internal. + @param impl The acceptor implementation. @param ep The local endpoint to bind to. @return Error code, or success. */ - std::error_code bind_acceptor(win_tcp_acceptor_internal& impl, endpoint ep); + std::error_code bind_acceptor(win_tcp_acceptor& impl, endpoint ep); /** Start listening for incoming connections. - @param impl The acceptor implementation internal. + @param impl The acceptor implementation. @param backlog The listen backlog. @return Error code, or success. */ - std::error_code - listen_acceptor(win_tcp_acceptor_internal& impl, int backlog); + std::error_code listen_acceptor(win_tcp_acceptor& impl, int backlog); /** Return the IOCP handle. */ void* native_handle() const noexcept; @@ -213,24 +201,1022 @@ class BOOST_COROSIO_DECL win_tcp_service final } private: - friend class win_tcp_acceptor_service; - void load_extension_functions(); win_scheduler& sched_; - BOOST_COROSIO_MSVC_WARNING_PUSH - BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // detail:: members, dll-interface - win_mutex mutex_; - intrusive_list socket_list_; - intrusive_list acceptor_list_; - intrusive_list socket_wrapper_list_; - intrusive_list acceptor_wrapper_list_; - BOOST_COROSIO_MSVC_WARNING_POP void* iocp_; LPFN_CONNECTEX connect_ex_ = nullptr; LPFN_ACCEPTEX accept_ex_ = nullptr; + BOOST_COROSIO_MSVC_WARNING_PUSH + BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // detail:: members, dll-interface + object_pool pool_; + object_pool acceptor_pool_; + BOOST_COROSIO_MSVC_WARNING_POP }; +// Operation constructors + +inline connect_op::connect_op(win_tcp_socket& internal_) noexcept + : overlapped_op(&do_complete) + , internal(internal_) +{ + cancel_func_ = &do_cancel_impl; +} + +inline read_op::read_op(win_tcp_socket& internal_) noexcept + : overlapped_op(&do_complete) + , internal(internal_) +{ + cancel_func_ = &do_cancel_impl; +} + +inline write_op::write_op(win_tcp_socket& internal_) noexcept + : overlapped_op(&do_complete) + , internal(internal_) +{ + cancel_func_ = &do_cancel_impl; +} + +inline wait_op::wait_op(win_tcp_socket& internal_) noexcept + : overlapped_op(&do_complete) + , internal(internal_) +{ + cancel_func_ = &do_cancel_impl; +} + +// Cancellation functions + +inline void +connect_op::do_cancel_impl(overlapped_op* base) noexcept +{ + auto* op = static_cast(base); + if (op->internal.socket_ != INVALID_SOCKET) + { + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); + } +} + +inline void +read_op::do_cancel_impl(overlapped_op* base) noexcept +{ + auto* op = static_cast(base); + op->cancelled.store(true, std::memory_order_release); + if (op->internal.socket_ != INVALID_SOCKET) + { + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); + } +} + +inline void +write_op::do_cancel_impl(overlapped_op* base) noexcept +{ + auto* op = static_cast(base); + op->cancelled.store(true, std::memory_order_release); + if (op->internal.socket_ != INVALID_SOCKET) + { + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); + } +} + +inline void +wait_op::do_cancel_impl(overlapped_op* base) noexcept +{ + auto* op = static_cast(base); + op->cancelled.store(true, std::memory_order_release); + // Best-effort cancel of any pending zero-byte WSARecv issued for + // wait_type::read. ERROR_NOT_FOUND when nothing overlapped is + // pending is harmless. + if (op->internal.socket_ != INVALID_SOCKET) + { + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); + } + // wait_type::error parks the op in the auxiliary select reactor; + // wake it so the reactor can post a cancelled completion. A cancel + // for an op the reactor never registered finds nothing and returns. + op->internal.svc_.scheduler().cancel_wait(op); +} + +// connect_op completion handler + +inline void +connect_op::do_complete( + void* owner, + scheduler_op* base, + std::uint32_t /*bytes*/, + std::uint32_t /*error*/) +{ + auto* op = static_cast(base); + + if (!owner) + { + op->cleanup_only(); + op->object_ref_.reset(); + return; + } + + bool success = + (op->dwError == 0 && !op->cancelled.load(std::memory_order_acquire)); + if (success && op->internal.socket_ != INVALID_SOCKET) + { + // Required after ConnectEx to enable shutdown(), getsockname(), etc. + ::setsockopt( + op->internal.socket_, SOL_SOCKET, SO_UPDATE_CONNECT_CONTEXT, + nullptr, 0); + + endpoint local_ep; + sockaddr_storage local_storage{}; + int local_len = sizeof(local_storage); + if (::getsockname( + op->internal.socket_, + reinterpret_cast(&local_storage), &local_len) == 0) + local_ep = from_sockaddr(local_storage); + op->internal.set_endpoints(local_ep, op->target_endpoint); + } + + auto prevent_premature_destruction = std::move(op->object_ref_); + op->invoke_handler(); +} + +// read_op completion handler + +inline void +read_op::do_complete( + void* owner, + scheduler_op* base, + std::uint32_t /*bytes*/, + std::uint32_t /*error*/) +{ + auto* op = static_cast(base); + + if (!owner) + { + op->cleanup_only(); + op->object_ref_.reset(); + return; + } + + auto prevent_premature_destruction = std::move(op->object_ref_); + op->invoke_handler(); +} + +// write_op completion handler + +inline void +write_op::do_complete( + void* owner, + scheduler_op* base, + std::uint32_t /*bytes*/, + std::uint32_t /*error*/) +{ + auto* op = static_cast(base); + + if (!owner) + { + op->cleanup_only(); + op->object_ref_.reset(); + return; + } + + auto prevent_premature_destruction = std::move(op->object_ref_); + op->invoke_handler(); +} + +// wait_op completion handler + +inline void +wait_op::do_complete( + void* owner, + scheduler_op* base, + std::uint32_t /*bytes*/, + std::uint32_t /*error*/) +{ + auto* op = static_cast(base); + + if (!owner) + { + op->cleanup_only(); + op->object_ref_.reset(); + return; + } + + auto prevent_premature_destruction = std::move(op->object_ref_); + op->invoke_handler(); +} + +// win_tcp_socket + +inline win_tcp_socket::win_tcp_socket(win_tcp_service& svc) noexcept + : svc_(svc) + , conn_(*this) + , rd_(*this) + , wr_(*this) + , wt_(*this) +{ +} + +inline void +win_tcp_socket::retire() noexcept +{ + svc_.pool_.recycle(this); +} + +inline void +win_tcp_socket::reuse() noexcept +{ + BOOST_COROSIO_ASSERT(socket_ == INVALID_SOCKET); + BOOST_COROSIO_ASSERT(family_ == AF_UNSPEC); + BOOST_COROSIO_ASSERT(local_endpoint_ == endpoint{}); + BOOST_COROSIO_ASSERT(remote_endpoint_ == endpoint{}); + BOOST_COROSIO_ASSERT(!conn_.stop_cb); + BOOST_COROSIO_ASSERT(!rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wr_.stop_cb); + BOOST_COROSIO_ASSERT(!wt_.stop_cb); +} + +inline void +win_tcp_socket::set_socket(SOCKET s) noexcept +{ + socket_ = s; +} + +inline void +win_tcp_socket::set_endpoints(endpoint local, endpoint remote) noexcept +{ + local_endpoint_ = local; + remote_endpoint_ = remote; +} + +inline std::coroutine_handle<> +win_tcp_socket::connect( + std::coroutine_handle<> h, + capy::executor_ref d, + endpoint ep, + std::stop_token token, + std::error_code* ec) +{ + // Keep this socket alive during I/O + conn_.object_ref_ = detail::object_ref(this); + + auto& op = conn_; + op.reset(); + op.h = h; + op.ex = d; + op.ec_out = ec; + op.target_endpoint = ep; + op.start(token); + + svc_.work_started(); + + // ConnectEx requires the socket to be bound. Skip if already bound + // (e.g. the caller used tcp_socket::bind() before connect). + if (local_endpoint_ == endpoint{}) + { + sockaddr_storage bind_storage{}; + socklen_t bind_len; + if (family_ == AF_INET6) + { + sockaddr_in6 sa6{}; + sa6.sin6_family = AF_INET6; + sa6.sin6_port = 0; + sa6.sin6_addr = in6addr_any; + std::memcpy(&bind_storage, &sa6, sizeof(sa6)); + bind_len = sizeof(sa6); + } + else + { + sockaddr_in sa4{}; + sa4.sin_family = AF_INET; + sa4.sin_addr.s_addr = INADDR_ANY; + sa4.sin_port = 0; + std::memcpy(&bind_storage, &sa4, sizeof(sa4)); + bind_len = sizeof(sa4); + } + + if (::bind( + socket_, reinterpret_cast(&bind_storage), + bind_len) == SOCKET_ERROR) + { + svc_.on_completion(&op, ::WSAGetLastError(), 0); + return std::noop_coroutine(); + } + } + + auto connect_ex = svc_.connect_ex(); + if (!connect_ex) + { + svc_.on_completion(&op, WSAEOPNOTSUPP, 0); + return std::noop_coroutine(); + } + + sockaddr_storage storage{}; + socklen_t addrlen = detail::to_sockaddr(ep, family_, storage); + + BOOL result = connect_ex( + socket_, reinterpret_cast(&storage), + static_cast(addrlen), nullptr, 0, nullptr, &op); + + if (!result) + { + DWORD err = ::WSAGetLastError(); + if (err != ERROR_IO_PENDING) + { + svc_.on_completion(&op, err, 0); + return std::noop_coroutine(); + } + } + + svc_.on_pending(&op); + return std::noop_coroutine(); +} + +inline std::coroutine_handle<> +win_tcp_socket::read_some( + std::coroutine_handle<> h, + capy::executor_ref d, + buffer_param param, + std::stop_token token, + std::error_code* ec, + std::size_t* bytes_out) +{ + // Keep this socket alive during I/O + rd_.object_ref_ = detail::object_ref(this); + + auto& op = rd_; + op.reset(); + op.is_read = true; + op.h = h; + op.ex = d; + op.ec_out = ec; + op.bytes_out = bytes_out; + op.start(token); + + svc_.work_started(); + + // Closed-object contract: complete with bad_file_descriptor + // without touching the kernel. + if (socket_ == INVALID_SOCKET) + { + svc_.on_completion(&op, WSAEBADF, 0); + return std::noop_coroutine(); + } + + // Prepare buffers + capy::mutable_buffer bufs[read_op::max_buffers]; + op.wsabuf_count = + static_cast(param.copy_to(bufs, read_op::max_buffers)); + + // Handle empty buffer: complete with 0 bytes + if (op.wsabuf_count == 0) + { + op.empty_buffer = true; + svc_.on_completion(&op, 0, 0); + return std::noop_coroutine(); + } + + for (DWORD i = 0; i < op.wsabuf_count; ++i) + { + op.wsabufs[i].buf = static_cast(bufs[i].data()); + op.wsabufs[i].len = static_cast( + (std::min)(bufs[i].size(), std::size_t(0x7fffffff))); + } + + op.flags = 0; + + int result = ::WSARecv( + socket_, op.wsabufs, op.wsabuf_count, nullptr, &op.flags, &op, nullptr); + + if (result == SOCKET_ERROR) + { + DWORD err = ::WSAGetLastError(); + if (err != WSA_IO_PENDING) + { + svc_.on_completion(&op, err, 0); + return std::noop_coroutine(); + } + } + + svc_.on_pending(&op); + + // Re-check cancellation after I/O is pending + if (op.cancelled.load(std::memory_order_acquire)) + ::CancelIoEx(reinterpret_cast(socket_), &op); + + return std::noop_coroutine(); +} + +inline std::coroutine_handle<> +win_tcp_socket::write_some( + std::coroutine_handle<> h, + capy::executor_ref d, + buffer_param param, + std::stop_token token, + std::error_code* ec, + std::size_t* bytes_out) +{ + // Keep this socket alive during I/O + wr_.object_ref_ = detail::object_ref(this); + + auto& op = wr_; + op.reset(); + op.h = h; + op.ex = d; + op.ec_out = ec; + op.bytes_out = bytes_out; + op.start(token); + + svc_.work_started(); + + // Closed-object contract: complete with bad_file_descriptor + // without touching the kernel. + if (socket_ == INVALID_SOCKET) + { + svc_.on_completion(&op, WSAEBADF, 0); + return std::noop_coroutine(); + } + + // Prepare buffers + capy::mutable_buffer bufs[write_op::max_buffers]; + op.wsabuf_count = + static_cast(param.copy_to(bufs, write_op::max_buffers)); + + // Handle empty buffer: complete immediately with 0 bytes + if (op.wsabuf_count == 0) + { + svc_.on_completion(&op, 0, 0); + return std::noop_coroutine(); + } + + for (DWORD i = 0; i < op.wsabuf_count; ++i) + { + op.wsabufs[i].buf = static_cast(bufs[i].data()); + op.wsabufs[i].len = static_cast( + (std::min)(bufs[i].size(), std::size_t(0x7fffffff))); + } + + int result = ::WSASend( + socket_, op.wsabufs, op.wsabuf_count, nullptr, 0, &op, nullptr); + + if (result == SOCKET_ERROR) + { + DWORD err = ::WSAGetLastError(); + if (err != WSA_IO_PENDING) + { + svc_.on_completion(&op, err, 0); + return std::noop_coroutine(); + } + } + + svc_.on_pending(&op); + + // Re-check cancellation after I/O is pending + if (op.cancelled.load(std::memory_order_acquire)) + ::CancelIoEx(reinterpret_cast(socket_), &op); + + return std::noop_coroutine(); +} + +inline std::coroutine_handle<> +win_tcp_socket::wait( + std::coroutine_handle<> h, + capy::executor_ref d, + wait_type w, + std::stop_token token, + std::error_code* ec) +{ + wt_.object_ref_ = detail::object_ref(this); + + auto& op = wt_; + op.reset(); + op.h = h; + op.ex = d; + op.ec_out = ec; + op.bytes_out = nullptr; + op.empty_buffer = true; // skip EOF translation in invoke_handler + op.start(token); + + svc_.work_started(); + + // Closed-object contract: complete with bad_file_descriptor + // without touching the kernel or the wait reactor. + if (socket_ == INVALID_SOCKET) + { + svc_.on_completion(&op, WSAEBADF, 0); + return std::noop_coroutine(); + } + + if (w == wait_type::read) + { + // Zero-byte WSARecv: kernel signals completion when data is + // available without consuming any bytes from the stream. This + // is the documented Winsock pattern for "is the socket + // readable" notifications. + op.wsabuf = WSABUF{0, nullptr}; + op.flags = 0; + + int result = + ::WSARecv(socket_, &op.wsabuf, 1, nullptr, &op.flags, &op, nullptr); + + if (result == SOCKET_ERROR) + { + DWORD err = ::WSAGetLastError(); + if (err != WSA_IO_PENDING) + { + svc_.on_completion(&op, err, 0); + return std::noop_coroutine(); + } + } + + svc_.on_pending(&op); + + // Re-check cancellation after I/O is pending. + if (op.cancelled.load(std::memory_order_acquire)) + ::CancelIoEx(reinterpret_cast(socket_), &op); + + return std::noop_coroutine(); + } + + // wait_type::write and wait_type::error: route through the + // auxiliary poll reactor. There is no overlapped primitive for + // "the send buffer has room" that does not also transfer bytes, + // and a write wait must report real writability. + svc_.scheduler().wait_reactor().register_wait(socket_, w, &op); + return std::noop_coroutine(); +} + +inline void +win_tcp_socket::cancel() noexcept +{ + if (socket_ != INVALID_SOCKET) + { + ::CancelIoEx(reinterpret_cast(socket_), nullptr); + } + + conn_.request_cancel(); + rd_.request_cancel(); + wr_.request_cancel(); + wt_.request_cancel(); + // CancelIoEx covers overlapped I/O on the socket but cannot reach + // a wait op parked in the auxiliary reactor (no overlapped is + // outstanding). Route through the reactor explicitly. + svc_.scheduler().cancel_wait(&wt_); +} + +inline void +win_tcp_socket::close_socket() noexcept +{ + // Flag every op cancelled before tearing down the handle. closesocket() + // can complete a pending overlapped op with ERROR_NETNAME_DELETED (not the + // MSDN-documented ERROR_OPERATION_ABORTED), which iocp_make_err now maps to + // connection_reset -- so a locally-closed op must be short-circuited to + // canceled via the flag rather than relying on the raw error, exactly as + // cancel() does. The store happens-before the completion is decoded. + conn_.request_cancel(); + rd_.request_cancel(); + wr_.request_cancel(); + // wt_ is parked in the aux reactor (no overlapped outstanding), so it also + // needs an explicit reactor deregister before the SOCKET handle is closed; + // otherwise the reactor would keep polling a dangling fd (and on a Winsock + // SOCKET-id reuse the wrong fd could be polled briefly). + wt_.request_cancel(); + svc_.scheduler().cancel_wait(&wt_); + + if (socket_ != INVALID_SOCKET) + { + ::CancelIoEx(reinterpret_cast(socket_), nullptr); + ::closesocket(socket_); + socket_ = INVALID_SOCKET; + } + + family_ = AF_UNSPEC; + + // Clear cached endpoints + local_endpoint_ = endpoint{}; + remote_endpoint_ = endpoint{}; +} + +inline std::error_code +win_tcp_socket::shutdown(tcp_socket::shutdown_type what) noexcept +{ + int how; + switch (what) + { + case tcp_socket::shutdown_receive: + how = SD_RECEIVE; + break; + case tcp_socket::shutdown_send: + how = SD_SEND; + break; + case tcp_socket::shutdown_both: + how = SD_BOTH; + break; + default: + return make_err(WSAEINVAL); + } + if (::shutdown(socket_, how) != 0) + return make_err(WSAGetLastError()); + return {}; +} + +inline native_handle_type +win_tcp_socket::native_handle() const noexcept +{ + return static_cast(socket_); +} + +inline native_handle_type +win_tcp_socket::release_socket() noexcept +{ + SOCKET s = socket_; + if (s != INVALID_SOCKET) + { + cancel(); + // Sever the port association so the descriptor can be + // adopted again; best-effort, the caller keeps a working + // socket either way. + dissociate_from_iocp(s); + socket_ = INVALID_SOCKET; + family_ = AF_UNSPEC; + local_endpoint_ = endpoint{}; + remote_endpoint_ = endpoint{}; + } + return static_cast(s); +} + +inline std::error_code +win_tcp_socket::set_option( + int level, int optname, void const* data, std::size_t size) noexcept +{ + if (::setsockopt( + socket_, level, optname, reinterpret_cast(data), + static_cast(size)) != 0) + return make_err(WSAGetLastError()); + return {}; +} + +inline std::error_code +win_tcp_socket::get_option( + int level, int optname, void* data, std::size_t* size) const noexcept +{ + int len = static_cast(*size); + if (::getsockopt( + socket_, level, optname, reinterpret_cast(data), &len) != 0) + return make_err(WSAGetLastError()); + *size = static_cast(len); + return {}; +} + +inline endpoint +win_tcp_socket::local_endpoint() const noexcept +{ + return local_endpoint_; +} + +inline endpoint +win_tcp_socket::remote_endpoint() const noexcept +{ + return remote_endpoint_; +} + +// win_tcp_service + +inline win_tcp_service::win_tcp_service(capy::execution_context& ctx) + : sched_(ctx.use_service()) + , iocp_(sched_.native_handle()) +{ + load_extension_functions(); +} + +inline void +win_tcp_service::shutdown() +{ + // Close all sockets/acceptors to force pending I/O to complete via + // IOCP. Not deleted here: in-flight ops hold their own object_ref, so + // each impl stays alive until its cancel completion drains; the + // pools' own destructors free whatever is still live once this + // service itself is torn down (after the scheduler, which drains + // every queued completion first). + pool_.shutdown([](win_tcp_socket* impl) { impl->close_socket(); }); + + acceptor_pool_.shutdown( + [](win_tcp_acceptor* impl) { impl->close_socket(); }); +} + +inline io_object::implementation* +win_tcp_service::construct() +{ + return pool_.acquire(*this); +} + +inline void +win_tcp_service::destroy(io_object::implementation* p) +{ + if (!p) + return; + auto* s = static_cast(p); + s->close_socket(); + release(s); +} + +inline void +win_tcp_service::close(io_object::handle& h) +{ + static_cast(h.get())->close_socket(); +} + +inline std::error_code +win_tcp_service::open_socket( + win_tcp_socket& impl, int family, int type, int protocol) +{ + impl.close_socket(); + + SOCKET sock = + ::WSASocketW(family, type, protocol, nullptr, 0, WSA_FLAG_OVERLAPPED); + + if (sock == INVALID_SOCKET) + return make_err(::WSAGetLastError()); + + if (family == AF_INET6) + { + DWORD one = 1; + ::setsockopt( + sock, IPPROTO_IPV6, IPV6_V6ONLY, reinterpret_cast(&one), + sizeof(one)); + } + + HANDLE result = ::CreateIoCompletionPort( + reinterpret_cast(sock), static_cast(iocp_), key_io, 0); + + if (result == nullptr) + { + DWORD dwError = ::GetLastError(); + ::closesocket(sock); + return make_err(dwError); + } + + impl.socket_ = sock; + impl.family_ = family; + return {}; +} + +inline std::error_code +win_tcp_service::assign_socket(win_tcp_socket& impl, native_handle_type fd) +{ + SOCKET sock = static_cast(fd); + if (sock == INVALID_SOCKET) + return make_err(WSAENOTSOCK); + if (sock == impl.socket_) + return std::make_error_code(std::errc::invalid_argument); + + // SO_PROTOCOL_INFOW works on an unbound socket, unlike getsockname + // (WSAEINVAL until bind/connect names it) -- an adopted socket may + // have reached connected state without an explicit bind. + WSAPROTOCOL_INFOW proto_info{}; + int proto_len = sizeof(proto_info); + if (::getsockopt( + sock, SOL_SOCKET, SO_PROTOCOL_INFOW, + reinterpret_cast(&proto_info), &proto_len) != 0) + return make_err(::WSAGetLastError()); + if (proto_info.iAddressFamily != AF_INET && + proto_info.iAddressFamily != AF_INET6) + return make_err(WSAEAFNOSUPPORT); + if (proto_info.iSocketType != SOCK_STREAM) + return make_err(WSAEPROTOTYPE); + + // Associate before releasing the held socket: on IOCP nothing + // shares descriptor state the way the reactor path does, so a + // failed association must not cost the caller their old socket. + HANDLE result = ::CreateIoCompletionPort( + reinterpret_cast(sock), static_cast(iocp_), key_io, 0); + if (result == nullptr) + return make_err(::GetLastError()); + + impl.close_socket(); + impl.socket_ = sock; + impl.family_ = proto_info.iAddressFamily; + + endpoint local_ep, remote_ep; + sockaddr_storage local_storage{}; + int local_len = sizeof(local_storage); + if (::getsockname( + sock, reinterpret_cast(&local_storage), &local_len) == 0) + local_ep = detail::from_sockaddr(local_storage); + sockaddr_storage remote_storage{}; + int remote_len = sizeof(remote_storage); + if (::getpeername( + sock, reinterpret_cast(&remote_storage), &remote_len) == + 0) + remote_ep = detail::from_sockaddr(remote_storage); + impl.set_endpoints(local_ep, remote_ep); + + return {}; +} + +inline std::error_code +win_tcp_service::bind_socket(win_tcp_socket& impl, endpoint ep) +{ + SOCKET sock = impl.socket_; + + sockaddr_storage storage{}; + socklen_t addrlen = detail::to_sockaddr(ep, storage); + if (::bind( + sock, reinterpret_cast(&storage), + static_cast(addrlen)) == SOCKET_ERROR) + return make_err(::WSAGetLastError()); + + // Cache local endpoint (resolves ephemeral port) + sockaddr_storage local_storage{}; + int local_len = sizeof(local_storage); + if (::getsockname( + sock, reinterpret_cast(&local_storage), &local_len) == 0) + impl.local_endpoint_ = detail::from_sockaddr(local_storage); + + return {}; +} + +inline void* +win_tcp_service::native_handle() const noexcept +{ + return iocp_; +} + +inline void +win_tcp_service::post(overlapped_op* op) +{ + sched_.post(op); +} + +inline void +win_tcp_service::on_pending(overlapped_op* op) noexcept +{ + sched_.on_pending(op); +} + +inline void +win_tcp_service::on_completion( + overlapped_op* op, DWORD error, DWORD bytes) noexcept +{ + sched_.on_completion(op, error, bytes); +} + +inline void +win_tcp_service::work_started() noexcept +{ + sched_.work_started(); +} + +inline void +win_tcp_service::work_finished() noexcept +{ + sched_.work_finished(); +} + +inline void +win_tcp_service::load_extension_functions() +{ + SOCKET sock = ::WSASocketW( + AF_INET, SOCK_STREAM, IPPROTO_TCP, nullptr, 0, WSA_FLAG_OVERLAPPED); + + if (sock == INVALID_SOCKET) + return; + + DWORD bytes = 0; + + GUID connect_ex_guid = WSAID_CONNECTEX; + ::WSAIoctl( + sock, SIO_GET_EXTENSION_FUNCTION_POINTER, &connect_ex_guid, + sizeof(connect_ex_guid), &connect_ex_, sizeof(connect_ex_), &bytes, + nullptr, nullptr); + + GUID accept_ex_guid = WSAID_ACCEPTEX; + ::WSAIoctl( + sock, SIO_GET_EXTENSION_FUNCTION_POINTER, &accept_ex_guid, + sizeof(accept_ex_guid), &accept_ex_, sizeof(accept_ex_), &bytes, + nullptr, nullptr); + + ::closesocket(sock); +} + +inline win_tcp_acceptor* +win_tcp_service::acquire_acceptor_impl() +{ + return acceptor_pool_.acquire(*this); +} + +inline std::error_code +win_tcp_service::open_acceptor_socket( + win_tcp_acceptor& impl, int family, int type, int protocol) +{ + impl.close_socket(); + + SOCKET sock = + ::WSASocketW(family, type, protocol, nullptr, 0, WSA_FLAG_OVERLAPPED); + + if (sock == INVALID_SOCKET) + return make_err(::WSAGetLastError()); + + if (family == AF_INET6) + { + DWORD val = 0; // dual-stack default + ::setsockopt( + sock, IPPROTO_IPV6, IPV6_V6ONLY, reinterpret_cast(&val), + sizeof(val)); + } + + HANDLE result = ::CreateIoCompletionPort( + reinterpret_cast(sock), static_cast(iocp_), key_io, 0); + + if (result == nullptr) + { + DWORD dwError = ::GetLastError(); + ::closesocket(sock); + return make_err(dwError); + } + + impl.socket_ = sock; + impl.family_ = family; + return {}; +} + +inline std::error_code +win_tcp_service::assign_acceptor_socket( + win_tcp_acceptor& impl, native_handle_type fd) +{ + SOCKET sock = static_cast(fd); + if (sock == INVALID_SOCKET) + return make_err(WSAENOTSOCK); + if (sock == impl.socket_) + return std::make_error_code(std::errc::invalid_argument); + + // SO_PROTOCOL_INFOW works on an unbound socket, unlike getsockname + // (WSAEINVAL until bind names it). + WSAPROTOCOL_INFOW proto_info{}; + int proto_len = sizeof(proto_info); + if (::getsockopt( + sock, SOL_SOCKET, SO_PROTOCOL_INFOW, + reinterpret_cast(&proto_info), &proto_len) != 0) + return make_err(::WSAGetLastError()); + if (proto_info.iAddressFamily != AF_INET && + proto_info.iAddressFamily != AF_INET6) + return make_err(WSAEAFNOSUPPORT); + if (proto_info.iSocketType != SOCK_STREAM) + return make_err(WSAEPROTOTYPE); + + // Associate before releasing the held socket: on IOCP nothing + // shares descriptor state the way the reactor path does, so a + // failed association must not cost the caller their old socket. + HANDLE result = ::CreateIoCompletionPort( + reinterpret_cast(sock), static_cast(iocp_), key_io, 0); + if (result == nullptr) + return make_err(::GetLastError()); + + impl.close_socket(); + impl.socket_ = sock; + impl.family_ = proto_info.iAddressFamily; + + // AcceptEx sizes its address buffers from this cache, so an + // unseeded endpoint breaks accepts on an adopted v6 listener. + sockaddr_storage local_storage{}; + int local_len = sizeof(local_storage); + if (::getsockname( + sock, reinterpret_cast(&local_storage), &local_len) == 0) + impl.set_local_endpoint(detail::from_sockaddr(local_storage)); + + return {}; +} + +inline std::error_code +win_tcp_service::bind_acceptor(win_tcp_acceptor& impl, endpoint ep) +{ + SOCKET sock = impl.socket_; + + sockaddr_storage storage{}; + socklen_t addrlen = detail::to_sockaddr(ep, storage); + if (::bind( + sock, reinterpret_cast(&storage), + static_cast(addrlen)) == SOCKET_ERROR) + return make_err(::WSAGetLastError()); + + // Cache local endpoint (resolves ephemeral port) + sockaddr_storage local_storage{}; + int local_len = sizeof(local_storage); + if (::getsockname( + sock, reinterpret_cast(&local_storage), &local_len) == 0) + impl.set_local_endpoint(detail::from_sockaddr(local_storage)); + + return {}; +} + +inline std::error_code +win_tcp_service::listen_acceptor(win_tcp_acceptor& impl, int backlog) +{ + SOCKET sock = impl.socket_; + + if (::listen(sock, backlog) == SOCKET_ERROR) + return make_err(::WSAGetLastError()); + + return {}; +} + } // namespace boost::corosio::detail #endif // BOOST_COROSIO_HAS_IOCP diff --git a/include/boost/corosio/native/detail/iocp/win_tcp_socket.hpp b/include/boost/corosio/native/detail/iocp/win_tcp_socket.hpp index 8b3f6ae67..2a8fd61f5 100644 --- a/include/boost/corosio/native/detail/iocp/win_tcp_socket.hpp +++ b/include/boost/corosio/native/detail/iocp/win_tcp_socket.hpp @@ -17,6 +17,7 @@ #if BOOST_COROSIO_HAS_IOCP #include +#include #include #include #include @@ -25,20 +26,18 @@ #include #include -#include #include namespace boost::corosio::detail { class win_tcp_service; -class win_tcp_socket_internal; +class win_tcp_socket; /** Connect operation state. */ struct connect_op : overlapped_op { - win_tcp_socket_internal& internal; - std::shared_ptr internal_ptr; + win_tcp_socket& internal; endpoint target_endpoint; static void do_complete( @@ -48,7 +47,7 @@ struct connect_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit connect_op(win_tcp_socket_internal& internal_) noexcept; + explicit connect_op(win_tcp_socket& internal_) noexcept; }; /** Read operation state with buffer descriptors. */ @@ -58,8 +57,7 @@ struct read_op : overlapped_op WSABUF wsabufs[max_buffers]; DWORD wsabuf_count = 0; DWORD flags = 0; - win_tcp_socket_internal& internal; - std::shared_ptr internal_ptr; + win_tcp_socket& internal; static void do_complete( void* owner, @@ -68,7 +66,7 @@ struct read_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit read_op(win_tcp_socket_internal& internal_) noexcept; + explicit read_op(win_tcp_socket& internal_) noexcept; }; /** Write operation state with buffer descriptors. */ @@ -77,8 +75,7 @@ struct write_op : overlapped_op static constexpr std::size_t max_buffers = 16; WSABUF wsabufs[max_buffers]; DWORD wsabuf_count = 0; - win_tcp_socket_internal& internal; - std::shared_ptr internal_ptr; + win_tcp_socket& internal; static void do_complete( void* owner, @@ -87,7 +84,7 @@ struct write_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit write_op(win_tcp_socket_internal& internal_) noexcept; + explicit write_op(win_tcp_socket& internal_) noexcept; }; /** Readiness-wait operation state. @@ -103,8 +100,7 @@ struct wait_op : overlapped_op { WSABUF wsabuf{}; DWORD flags = 0; - win_tcp_socket_internal& internal; - std::shared_ptr internal_ptr; + win_tcp_socket& internal; static void do_complete( void* owner, @@ -113,23 +109,26 @@ struct wait_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit wait_op(win_tcp_socket_internal& internal_) noexcept; + explicit wait_op(win_tcp_socket& internal_) noexcept; }; -/** Internal socket state for IOCP-based I/O. +/** Socket implementation for IOCP-based I/O. - This class contains the actual state for a single socket, including - the native socket handle and pending operations. It derives from - enable_shared_from_this so operations can extend its lifetime. + Collapses the historical internal-state/wrapper split into one + pooled `io_object::implementation`: it both owns the native socket + handle and pending operations, and directly implements + `tcp_socket::implementation`. `win_tcp_service` recycles instances + through its pool instead of freeing them on every close, so the + keepalive each embedded op holds is the intrusive `object_ref` from + `coro_op`, not a `shared_ptr`. @note Internal implementation detail. Users interact with socket class. */ -class win_tcp_socket_internal - : public intrusive_list::node - , public std::enable_shared_from_this +class win_tcp_socket final + : public tcp_socket::implementation + , public intrusive_list::node { friend class win_tcp_service; - friend class win_tcp_socket; friend struct read_op; friend struct write_op; friend struct connect_op; @@ -142,73 +141,28 @@ class win_tcp_socket_internal wait_op wt_; SOCKET socket_ = INVALID_SOCKET; int family_ = AF_UNSPEC; - -public: - explicit win_tcp_socket_internal(win_tcp_service& svc) noexcept; - ~win_tcp_socket_internal(); - - std::coroutine_handle<> connect( - std::coroutine_handle<>, - capy::executor_ref, - endpoint, - std::stop_token, - std::error_code*); - - std::coroutine_handle<> read_some( - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - std::stop_token, - std::error_code*, - std::size_t*); - - std::coroutine_handle<> write_some( - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - std::stop_token, - std::error_code*, - std::size_t*); - - std::coroutine_handle<> wait( - std::coroutine_handle<>, - capy::executor_ref, - wait_type, - std::stop_token, - std::error_code*); - - SOCKET native_handle() const noexcept; - endpoint local_endpoint() const noexcept; - endpoint remote_endpoint() const noexcept; - bool is_open() const noexcept; - void cancel() noexcept; - void close_socket() noexcept; - void set_socket(SOCKET s) noexcept; - void set_endpoints(endpoint local, endpoint remote) noexcept; - -private: endpoint local_endpoint_; endpoint remote_endpoint_; -}; -/** Socket implementation wrapper for IOCP-based I/O. +public: + explicit win_tcp_socket(win_tcp_service& svc) noexcept; - This class is the public-facing implementation that holds a shared_ptr - to the internal state. The shared_ptr is hidden from the public interface. + /// Recycle into the owning service's pool. Defined out-of-line + /// after `win_tcp_service` is complete (needs `svc_.pool_`). + void retire() noexcept override; - @note Internal implementation detail. Users interact with socket class. -*/ -class win_tcp_socket final - : public tcp_socket::implementation - , public intrusive_list::node -{ - std::shared_ptr internal_; + /** Reset recycled state for reuse. -public: - explicit win_tcp_socket( - std::shared_ptr internal) noexcept; + `close_socket()` already drives the socket handle, family, + and cached endpoints to their closed values, and drops every + embedded op's `object_ref_` before `refs_` can reach zero, so + there is little left to reset here -- only to assert. Each + op's `OVERLAPPED` fields are re-seeded by `overlapped_op:: + reset()` at its next submit, so they are not zeroed here. - void close_internal() noexcept; + @pre refs_ == 0, socket closed, no op in flight. + */ + void reuse() noexcept; std::coroutine_handle<> connect( std::coroutine_handle<> h, @@ -248,7 +202,7 @@ class win_tcp_socket final { // getsockname fails with WSAEINVAL on an unbound socket, so // the family recorded at socket creation is authoritative - return to_family(internal_ ? internal_->family_ : AF_UNSPEC); + return to_family(family_); } native_handle_type release_socket() noexcept override; @@ -266,7 +220,11 @@ class win_tcp_socket final endpoint remote_endpoint() const noexcept override; void cancel() noexcept override; - win_tcp_socket_internal* get_internal() const noexcept; + /// Used by the acceptor's `accept_op` to finish setting up the peer. + void set_socket(SOCKET s) noexcept; + void set_endpoints(endpoint local, endpoint remote) noexcept; + + void close_socket() noexcept; }; } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/iocp/win_udp_service.hpp b/include/boost/corosio/native/detail/iocp/win_udp_service.hpp index 280d0c5be..f89120bfc 100644 --- a/include/boost/corosio/native/detail/iocp/win_udp_service.hpp +++ b/include/boost/corosio/native/detail/iocp/win_udp_service.hpp @@ -15,13 +15,13 @@ #if BOOST_COROSIO_HAS_IOCP #include +#include #include #include #include #include #include -#include #include #include @@ -44,6 +44,8 @@ class BOOST_COROSIO_DECL win_udp_service final : private win_wsa_init , public udp_service { + friend class win_udp_socket; + public: io_object::implementation* construct() override; @@ -53,8 +55,6 @@ class BOOST_COROSIO_DECL win_udp_service final explicit win_udp_service(capy::execution_context& ctx); - ~win_udp_service(); - win_udp_service(win_udp_service const&) = delete; win_udp_service& operator=(win_udp_service const&) = delete; @@ -70,12 +70,8 @@ class BOOST_COROSIO_DECL win_udp_service final std::error_code bind_datagram(udp_socket::implementation& impl, endpoint ep) override; - void destroy_impl(win_udp_socket& impl); - - void unregister_impl(win_udp_socket_internal& impl); - std::error_code open_socket( - win_udp_socket_internal& impl, int family, int type, int protocol); + win_udp_socket& impl, int family, int type, int protocol); void post(overlapped_op* op); void on_pending(overlapped_op* op) noexcept; @@ -91,54 +87,51 @@ class BOOST_COROSIO_DECL win_udp_service final private: win_scheduler& sched_; + void* iocp_; BOOST_COROSIO_MSVC_WARNING_PUSH BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // detail:: members, dll-interface - win_mutex mutex_; - intrusive_list socket_list_; - intrusive_list wrapper_list_; + object_pool pool_; BOOST_COROSIO_MSVC_WARNING_POP - void* iocp_; }; // Operation constructors -inline send_to_op::send_to_op(win_udp_socket_internal& internal_) noexcept +inline send_to_op::send_to_op(win_udp_socket& internal_) noexcept : overlapped_op(&do_complete) , internal(internal_) { cancel_func_ = &do_cancel_impl; } -inline recv_from_op::recv_from_op(win_udp_socket_internal& internal_) noexcept +inline recv_from_op::recv_from_op(win_udp_socket& internal_) noexcept : overlapped_op(&do_complete) , internal(internal_) { cancel_func_ = &do_cancel_impl; } -inline udp_connect_op::udp_connect_op( - win_udp_socket_internal& internal_) noexcept +inline udp_connect_op::udp_connect_op(win_udp_socket& internal_) noexcept : overlapped_op(&do_complete) , internal(internal_) { cancel_func_ = &do_cancel_impl; } -inline udp_send_op::udp_send_op(win_udp_socket_internal& internal_) noexcept +inline udp_send_op::udp_send_op(win_udp_socket& internal_) noexcept : overlapped_op(&do_complete) , internal(internal_) { cancel_func_ = &do_cancel_impl; } -inline udp_recv_op::udp_recv_op(win_udp_socket_internal& internal_) noexcept +inline udp_recv_op::udp_recv_op(win_udp_socket& internal_) noexcept : overlapped_op(&do_complete) , internal(internal_) { cancel_func_ = &do_cancel_impl; } -inline udp_wait_op::udp_wait_op(win_udp_socket_internal& internal_) noexcept +inline udp_wait_op::udp_wait_op(win_udp_socket& internal_) noexcept : overlapped_op(&do_complete) , internal(internal_) { @@ -152,10 +145,9 @@ send_to_op::do_cancel_impl(overlapped_op* base) noexcept { auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) + if (op->internal.socket_ != INVALID_SOCKET) { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); } } @@ -164,10 +156,9 @@ recv_from_op::do_cancel_impl(overlapped_op* base) noexcept { auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) + if (op->internal.socket_ != INVALID_SOCKET) { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); } } @@ -185,11 +176,11 @@ send_to_op::do_complete( if (!owner) { op->cleanup_only(); - op->internal_ptr.reset(); + op->object_ref_.reset(); return; } - auto prevent_premature_destruction = std::move(op->internal_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } @@ -207,7 +198,7 @@ recv_from_op::do_complete( if (!owner) { op->cleanup_only(); - op->internal_ptr.reset(); + op->object_ref_.reset(); return; } @@ -219,7 +210,7 @@ recv_from_op::do_complete( *op->source_out = from_sockaddr(op->source_storage); } - auto prevent_premature_destruction = std::move(op->internal_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } @@ -237,10 +228,9 @@ udp_send_op::do_cancel_impl(overlapped_op* base) noexcept { auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) + if (op->internal.socket_ != INVALID_SOCKET) { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); } } @@ -249,10 +239,9 @@ udp_recv_op::do_cancel_impl(overlapped_op* base) noexcept { auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) + if (op->internal.socket_ != INVALID_SOCKET) { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); } } @@ -261,10 +250,9 @@ udp_wait_op::do_cancel_impl(overlapped_op* base) noexcept { auto* op = static_cast(base); op->cancelled.store(true, std::memory_order_release); - if (op->internal.is_open()) + if (op->internal.socket_ != INVALID_SOCKET) { - ::CancelIoEx( - reinterpret_cast(op->internal.native_handle()), op); + ::CancelIoEx(reinterpret_cast(op->internal.socket_), op); } op->internal.svc_.scheduler().cancel_wait(op); } @@ -283,7 +271,7 @@ udp_connect_op::do_complete( if (!owner) { op->cleanup_only(); - op->internal_ptr.reset(); + op->object_ref_.reset(); return; } @@ -301,7 +289,7 @@ udp_connect_op::do_complete( op->internal.remote_endpoint_ = op->target_endpoint; } - auto prevent_premature_destruction = std::move(op->internal_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } @@ -317,11 +305,11 @@ udp_send_op::do_complete( if (!owner) { op->cleanup_only(); - op->internal_ptr.reset(); + op->object_ref_.reset(); return; } - auto prevent_premature_destruction = std::move(op->internal_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } @@ -337,11 +325,11 @@ udp_recv_op::do_complete( if (!owner) { op->cleanup_only(); - op->internal_ptr.reset(); + op->object_ref_.reset(); return; } - auto prevent_premature_destruction = std::move(op->internal_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } @@ -357,18 +345,17 @@ udp_wait_op::do_complete( if (!owner) { op->cleanup_only(); - op->internal_ptr.reset(); + op->object_ref_.reset(); return; } - auto prevent_premature_destruction = std::move(op->internal_ptr); + auto prevent_premature_destruction = std::move(op->object_ref_); op->invoke_handler(); } -// win_udp_socket_internal +// win_udp_socket -inline win_udp_socket_internal::win_udp_socket_internal( - win_udp_service& svc) noexcept +inline win_udp_socket::win_udp_socket(win_udp_service& svc) noexcept : svc_(svc) , wr_(*this) , rd_(*this) @@ -379,37 +366,41 @@ inline win_udp_socket_internal::win_udp_socket_internal( { } -inline win_udp_socket_internal::~win_udp_socket_internal() +inline void +win_udp_socket::reuse() noexcept { - svc_.unregister_impl(*this); + BOOST_COROSIO_ASSERT(socket_ == INVALID_SOCKET); + BOOST_COROSIO_ASSERT(family_ == AF_UNSPEC); + BOOST_COROSIO_ASSERT(local_endpoint_ == endpoint{}); + BOOST_COROSIO_ASSERT(remote_endpoint_ == endpoint{}); + BOOST_COROSIO_ASSERT(!wr_.stop_cb); + BOOST_COROSIO_ASSERT(!rd_.stop_cb); + BOOST_COROSIO_ASSERT(!conn_.stop_cb); + BOOST_COROSIO_ASSERT(!send_wr_.stop_cb); + BOOST_COROSIO_ASSERT(!recv_rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wt_.stop_cb); } -inline SOCKET -win_udp_socket_internal::native_handle() const noexcept +inline native_handle_type +win_udp_socket::native_handle() const noexcept { - return socket_; + return static_cast(socket_); } inline endpoint -win_udp_socket_internal::local_endpoint() const noexcept +win_udp_socket::local_endpoint() const noexcept { return local_endpoint_; } inline endpoint -win_udp_socket_internal::remote_endpoint() const noexcept +win_udp_socket::remote_endpoint() const noexcept { return remote_endpoint_; } -inline bool -win_udp_socket_internal::is_open() const noexcept -{ - return socket_ != INVALID_SOCKET; -} - inline std::coroutine_handle<> -win_udp_socket_internal::send_to( +win_udp_socket::send_to( std::coroutine_handle<> h, capy::executor_ref d, buffer_param param, @@ -419,8 +410,8 @@ win_udp_socket_internal::send_to( std::error_code* ec, std::size_t* bytes_out) { - // Keep internal alive during I/O - wr_.internal_ptr = shared_from_this(); + // Keep this socket alive during I/O + wr_.object_ref_ = detail::object_ref(this); auto& op = wr_; op.reset(); @@ -472,7 +463,7 @@ win_udp_socket_internal::send_to( } inline std::coroutine_handle<> -win_udp_socket_internal::recv_from( +win_udp_socket::recv_from( std::coroutine_handle<> h, capy::executor_ref d, buffer_param param, @@ -482,8 +473,8 @@ win_udp_socket_internal::recv_from( std::error_code* ec, std::size_t* bytes_out) { - // Keep internal alive during I/O - rd_.internal_ptr = shared_from_this(); + // Keep this socket alive during I/O + rd_.object_ref_ = detail::object_ref(this); auto& op = rd_; op.reset(); @@ -545,14 +536,14 @@ win_udp_socket_internal::recv_from( // UDP connect is synchronous on Windows inline std::coroutine_handle<> -win_udp_socket_internal::connect( +win_udp_socket::connect( std::coroutine_handle<> h, capy::executor_ref d, endpoint ep, std::stop_token token, std::error_code* ec) { - conn_.internal_ptr = shared_from_this(); + conn_.object_ref_ = detail::object_ref(this); auto& op = conn_; op.reset(); @@ -579,7 +570,7 @@ win_udp_socket_internal::connect( } inline std::coroutine_handle<> -win_udp_socket_internal::send( +win_udp_socket::send( std::coroutine_handle<> h, capy::executor_ref d, buffer_param param, @@ -588,7 +579,7 @@ win_udp_socket_internal::send( std::error_code* ec, std::size_t* bytes_out) { - send_wr_.internal_ptr = shared_from_this(); + send_wr_.object_ref_ = detail::object_ref(this); auto& op = send_wr_; op.reset(); @@ -633,7 +624,7 @@ win_udp_socket_internal::send( } inline std::coroutine_handle<> -win_udp_socket_internal::recv( +win_udp_socket::recv( std::coroutine_handle<> h, capy::executor_ref d, buffer_param param, @@ -642,7 +633,7 @@ win_udp_socket_internal::recv( std::error_code* ec, std::size_t* bytes_out) { - recv_rd_.internal_ptr = shared_from_this(); + recv_rd_.object_ref_ = detail::object_ref(this); auto& op = recv_rd_; op.reset(); @@ -695,14 +686,14 @@ win_udp_socket_internal::recv( } inline std::coroutine_handle<> -win_udp_socket_internal::wait( +win_udp_socket::wait( std::coroutine_handle<> h, capy::executor_ref d, wait_type w, std::stop_token token, std::error_code* ec) { - wt_.internal_ptr = shared_from_this(); + wt_.object_ref_ = detail::object_ref(this); auto& op = wt_; op.reset(); @@ -733,7 +724,7 @@ win_udp_socket_internal::wait( } inline void -win_udp_socket_internal::cancel() noexcept +win_udp_socket::cancel() noexcept { if (socket_ != INVALID_SOCKET) { @@ -750,7 +741,7 @@ win_udp_socket_internal::cancel() noexcept } inline void -win_udp_socket_internal::close_socket() noexcept +win_udp_socket::close_socket() noexcept { // Flag every op cancelled before closing so a closesocket-delivered // ERROR_NETNAME_DELETED is short-circuited to canceled rather than mapped @@ -776,106 +767,6 @@ win_udp_socket_internal::close_socket() noexcept remote_endpoint_ = endpoint{}; } -// win_udp_socket - -inline win_udp_socket::win_udp_socket( - std::shared_ptr internal) noexcept - : internal_(std::move(internal)) -{ -} - -inline void -win_udp_socket::close_internal() noexcept -{ - if (internal_) - { - internal_->close_socket(); - internal_.reset(); - } -} - -inline std::coroutine_handle<> -win_udp_socket::send_to( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - endpoint dest, - int flags, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->send_to(h, d, buf, dest, flags, token, ec, bytes); -} - -inline std::coroutine_handle<> -win_udp_socket::recv_from( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - endpoint* source, - int flags, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->recv_from(h, d, buf, source, flags, token, ec, bytes); -} - -inline std::coroutine_handle<> -win_udp_socket::connect( - std::coroutine_handle<> h, - capy::executor_ref d, - endpoint ep, - std::stop_token token, - std::error_code* ec) -{ - return internal_->connect(h, d, ep, token, ec); -} - -inline std::coroutine_handle<> -win_udp_socket::send( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - int flags, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->send(h, d, buf, flags, token, ec, bytes); -} - -inline std::coroutine_handle<> -win_udp_socket::recv( - std::coroutine_handle<> h, - capy::executor_ref d, - buffer_param buf, - int flags, - std::stop_token token, - std::error_code* ec, - std::size_t* bytes) -{ - return internal_->recv(h, d, buf, flags, token, ec, bytes); -} - -inline std::coroutine_handle<> -win_udp_socket::wait( - std::coroutine_handle<> h, - capy::executor_ref d, - wait_type w, - std::stop_token token, - std::error_code* ec) -{ - return internal_->wait(h, d, w, token, ec); -} - -inline native_handle_type -win_udp_socket::native_handle() const noexcept -{ - return static_cast(internal_->native_handle()); -} - inline std::error_code win_udp_socket::shutdown(udp_socket::shutdown_type what) noexcept { @@ -894,7 +785,7 @@ win_udp_socket::shutdown(udp_socket::shutdown_type what) noexcept default: return make_err(WSAEINVAL); } - if (::shutdown(internal_->native_handle(), how) != 0) + if (::shutdown(socket_, how) != 0) return make_err(WSAGetLastError()); return {}; } @@ -902,15 +793,15 @@ win_udp_socket::shutdown(udp_socket::shutdown_type what) noexcept inline native_handle_type win_udp_socket::release_socket() noexcept { - SOCKET s = internal_->socket_; + SOCKET s = socket_; if (s != INVALID_SOCKET) { - internal_->cancel(); + cancel(); dissociate_from_iocp(s); - internal_->socket_ = INVALID_SOCKET; - internal_->family_ = AF_UNSPEC; - internal_->local_endpoint_ = endpoint{}; - internal_->remote_endpoint_ = endpoint{}; + socket_ = INVALID_SOCKET; + family_ = AF_UNSPEC; + local_endpoint_ = endpoint{}; + remote_endpoint_ = endpoint{}; } return static_cast(s); } @@ -920,8 +811,8 @@ win_udp_socket::set_option( int level, int optname, void const* data, std::size_t size) noexcept { if (::setsockopt( - internal_->native_handle(), level, optname, - reinterpret_cast(data), static_cast(size)) != 0) + socket_, level, optname, reinterpret_cast(data), + static_cast(size)) != 0) return make_err(WSAGetLastError()); return {}; } @@ -932,35 +823,16 @@ win_udp_socket::get_option( { int len = static_cast(*size); if (::getsockopt( - internal_->native_handle(), level, optname, - reinterpret_cast(data), &len) != 0) + socket_, level, optname, reinterpret_cast(data), &len) != 0) return make_err(WSAGetLastError()); *size = static_cast(len); return {}; } -inline endpoint -win_udp_socket::local_endpoint() const noexcept -{ - return internal_->local_endpoint(); -} - -inline endpoint -win_udp_socket::remote_endpoint() const noexcept -{ - return internal_->remote_endpoint(); -} - inline void -win_udp_socket::cancel() noexcept +win_udp_socket::retire() noexcept { - internal_->cancel(); -} - -inline win_udp_socket_internal* -win_udp_socket::get_internal() const noexcept -{ - return internal_.get(); + svc_.pool_.recycle(this); } // win_udp_service @@ -971,83 +843,37 @@ inline win_udp_service::win_udp_service(capy::execution_context& ctx) { } -inline win_udp_service::~win_udp_service() -{ - for (auto* w = wrapper_list_.pop_front(); w != nullptr; - w = wrapper_list_.pop_front()) - delete w; -} - inline void win_udp_service::shutdown() { - std::lock_guard lock(mutex_); - - for (auto* impl = socket_list_.pop_front(); impl != nullptr; - impl = socket_list_.pop_front()) - { - impl->close_socket(); - } + pool_.shutdown([](win_udp_socket* impl) { impl->close_socket(); }); } inline io_object::implementation* win_udp_service::construct() { - auto internal = std::make_shared(*this); - - { - std::lock_guard lock(mutex_); - socket_list_.push_back(internal.get()); - } - - auto* wrapper = new win_udp_socket(std::move(internal)); - - { - std::lock_guard lock(mutex_); - wrapper_list_.push_back(wrapper); - } - - return wrapper; + return pool_.acquire(*this); } inline void win_udp_service::destroy(io_object::implementation* p) { - if (p) - { - auto& wrapper = static_cast(*p); - wrapper.close_internal(); - destroy_impl(wrapper); - } + if (!p) + return; + auto* s = static_cast(p); + s->close_socket(); + release(s); } inline void win_udp_service::close(io_object::handle& h) { - auto& wrapper = static_cast(*h.get()); - wrapper.get_internal()->close_socket(); -} - -inline void -win_udp_service::destroy_impl(win_udp_socket& impl) -{ - { - std::lock_guard lock(mutex_); - wrapper_list_.remove(&impl); - } - delete &impl; -} - -inline void -win_udp_service::unregister_impl(win_udp_socket_internal& impl) -{ - std::lock_guard lock(mutex_); - socket_list_.remove(&impl); + static_cast(h.get())->close_socket(); } inline std::error_code win_udp_service::open_socket( - win_udp_socket_internal& impl, int family, int type, int protocol) + win_udp_socket& impl, int family, int type, int protocol) { impl.close_socket(); @@ -1084,21 +910,20 @@ inline std::error_code win_udp_service::open_datagram_socket( udp_socket::implementation& impl, int family, int type, int protocol) { - auto& wrapper = static_cast(impl); - return open_socket(*wrapper.get_internal(), family, type, protocol); + auto& sock = static_cast(impl); + return open_socket(sock, family, type, protocol); } inline std::error_code win_udp_service::assign_socket( udp_socket::implementation& impl, native_handle_type fd) { - auto& wrapper = static_cast(impl); - auto* internal = wrapper.get_internal(); + auto& sock = static_cast(impl); - SOCKET sock = static_cast(fd); - if (sock == INVALID_SOCKET) + SOCKET s = static_cast(fd); + if (s == INVALID_SOCKET) return make_err(WSAENOTSOCK); - if (sock == internal->socket_) + if (s == sock.socket_) return std::make_error_code(std::errc::invalid_argument); // SO_PROTOCOL_INFOW works on an unbound socket, unlike getsockname @@ -1106,7 +931,7 @@ win_udp_service::assign_socket( WSAPROTOCOL_INFOW proto_info{}; int proto_len = sizeof(proto_info); if (::getsockopt( - sock, SOL_SOCKET, SO_PROTOCOL_INFOW, + s, SOL_SOCKET, SO_PROTOCOL_INFOW, reinterpret_cast(&proto_info), &proto_len) != 0) return make_err(::WSAGetLastError()); if (proto_info.iAddressFamily != AF_INET && @@ -1119,28 +944,27 @@ win_udp_service::assign_socket( // shares descriptor state the way the reactor path does, so a // failed association must not cost the caller their old socket. HANDLE result = ::CreateIoCompletionPort( - reinterpret_cast(sock), static_cast(iocp_), key_io, 0); + reinterpret_cast(s), static_cast(iocp_), key_io, 0); if (result == nullptr) return make_err(::GetLastError()); - internal->close_socket(); - internal->socket_ = sock; - internal->family_ = proto_info.iAddressFamily; + sock.close_socket(); + sock.socket_ = s; + sock.family_ = proto_info.iAddressFamily; endpoint local_ep, remote_ep; sockaddr_storage local_storage{}; int local_len = sizeof(local_storage); if (::getsockname( - sock, reinterpret_cast(&local_storage), &local_len) == 0) + s, reinterpret_cast(&local_storage), &local_len) == 0) local_ep = detail::from_sockaddr(local_storage); sockaddr_storage remote_storage{}; int remote_len = sizeof(remote_storage); if (::getpeername( - sock, reinterpret_cast(&remote_storage), &remote_len) == - 0) + s, reinterpret_cast(&remote_storage), &remote_len) == 0) remote_ep = detail::from_sockaddr(remote_storage); - internal->local_endpoint_ = local_ep; - internal->remote_endpoint_ = remote_ep; + sock.local_endpoint_ = local_ep; + sock.remote_endpoint_ = remote_ep; return {}; } @@ -1148,14 +972,13 @@ win_udp_service::assign_socket( inline std::error_code win_udp_service::bind_datagram(udp_socket::implementation& impl, endpoint ep) { - auto& wrapper = static_cast(impl); - auto* internal = wrapper.get_internal(); - SOCKET sock = internal->socket_; + auto& sock = static_cast(impl); + SOCKET s = sock.socket_; sockaddr_storage storage{}; socklen_t addrlen = detail::to_sockaddr(ep, storage); if (::bind( - sock, reinterpret_cast(&storage), + s, reinterpret_cast(&storage), static_cast(addrlen)) == SOCKET_ERROR) return make_err(::WSAGetLastError()); @@ -1163,8 +986,8 @@ win_udp_service::bind_datagram(udp_socket::implementation& impl, endpoint ep) sockaddr_storage local_storage{}; int local_len = sizeof(local_storage); if (::getsockname( - sock, reinterpret_cast(&local_storage), &local_len) == 0) - internal->local_endpoint_ = detail::from_sockaddr(local_storage); + s, reinterpret_cast(&local_storage), &local_len) == 0) + sock.local_endpoint_ = detail::from_sockaddr(local_storage); return {}; } diff --git a/include/boost/corosio/native/detail/iocp/win_udp_socket.hpp b/include/boost/corosio/native/detail/iocp/win_udp_socket.hpp index 9679265d6..e489d58ab 100644 --- a/include/boost/corosio/native/detail/iocp/win_udp_socket.hpp +++ b/include/boost/corosio/native/detail/iocp/win_udp_socket.hpp @@ -16,6 +16,7 @@ #if BOOST_COROSIO_HAS_IOCP #include +#include #include #include #include @@ -24,12 +25,11 @@ #include #include -#include namespace boost::corosio::detail { class win_udp_service; -class win_udp_socket_internal; +class win_udp_socket; /** Send-to operation state with buffer and destination address. */ struct send_to_op : overlapped_op @@ -39,8 +39,7 @@ struct send_to_op : overlapped_op DWORD wsabuf_count = 0; sockaddr_storage dest_storage{}; int dest_len = 0; - win_udp_socket_internal& internal; - std::shared_ptr internal_ptr; + win_udp_socket& internal; static void do_complete( void* owner, @@ -49,7 +48,7 @@ struct send_to_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit send_to_op(win_udp_socket_internal& internal_) noexcept; + explicit send_to_op(win_udp_socket& internal_) noexcept; }; /** Recv-from operation state with buffer and source address capture. */ @@ -62,8 +61,7 @@ struct recv_from_op : overlapped_op sockaddr_storage source_storage{}; INT source_len = sizeof(sockaddr_storage); endpoint* source_out = nullptr; - win_udp_socket_internal& internal; - std::shared_ptr internal_ptr; + win_udp_socket& internal; static void do_complete( void* owner, @@ -72,15 +70,14 @@ struct recv_from_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit recv_from_op(win_udp_socket_internal& internal_) noexcept; + explicit recv_from_op(win_udp_socket& internal_) noexcept; }; /** Connect operation for connected-mode UDP on IOCP. */ struct udp_connect_op : overlapped_op { endpoint target_endpoint; - win_udp_socket_internal& internal; - std::shared_ptr internal_ptr; + win_udp_socket& internal; static void do_complete( void* owner, @@ -89,7 +86,7 @@ struct udp_connect_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit udp_connect_op(win_udp_socket_internal& internal_) noexcept; + explicit udp_connect_op(win_udp_socket& internal_) noexcept; }; /** Connected send operation (no destination address). */ @@ -98,8 +95,7 @@ struct udp_send_op : overlapped_op static constexpr std::size_t max_buffers = 16; WSABUF wsabufs[max_buffers]; DWORD wsabuf_count = 0; - win_udp_socket_internal& internal; - std::shared_ptr internal_ptr; + win_udp_socket& internal; static void do_complete( void* owner, @@ -108,7 +104,7 @@ struct udp_send_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit udp_send_op(win_udp_socket_internal& internal_) noexcept; + explicit udp_send_op(win_udp_socket& internal_) noexcept; }; /** Connected recv operation (no source address). */ @@ -118,8 +114,7 @@ struct udp_recv_op : overlapped_op WSABUF wsabufs[max_buffers]; DWORD wsabuf_count = 0; DWORD flags = 0; - win_udp_socket_internal& internal; - std::shared_ptr internal_ptr; + win_udp_socket& internal; static void do_complete( void* owner, @@ -128,14 +123,13 @@ struct udp_recv_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit udp_recv_op(win_udp_socket_internal& internal_) noexcept; + explicit udp_recv_op(win_udp_socket& internal_) noexcept; }; /** Readiness-wait operation (datagram socket). */ struct udp_wait_op : overlapped_op { - win_udp_socket_internal& internal; - std::shared_ptr internal_ptr; + win_udp_socket& internal; static void do_complete( void* owner, @@ -144,24 +138,24 @@ struct udp_wait_op : overlapped_op std::uint32_t error); static void do_cancel_impl(overlapped_op* op) noexcept; - explicit udp_wait_op(win_udp_socket_internal& internal_) noexcept; + explicit udp_wait_op(win_udp_socket& internal_) noexcept; }; -/** Internal datagram socket state for IOCP-based I/O. +/** Datagram socket implementation for IOCP-based I/O. - This class contains the actual state for a single UDP socket, - including the native socket handle and pending operations. It - derives from enable_shared_from_this so operations can extend - its lifetime. + Collapses the historical internal-state/wrapper split into one + pooled `io_object::implementation`: it both owns the native socket + handle and pending operations, and directly implements + `udp_socket::implementation`. `win_udp_service` recycles instances + through its pool instead of freeing them on every close. @note Internal implementation detail. */ -class win_udp_socket_internal - : public intrusive_list::node - , public std::enable_shared_from_this +class win_udp_socket final + : public udp_socket::implementation + , public intrusive_list::node { friend class win_udp_service; - friend class win_udp_socket; friend struct send_to_op; friend struct recv_from_op; friend struct udp_connect_op; @@ -178,94 +172,25 @@ class win_udp_socket_internal udp_wait_op wt_; SOCKET socket_ = INVALID_SOCKET; int family_ = AF_UNSPEC; - -public: - explicit win_udp_socket_internal(win_udp_service& svc) noexcept; - ~win_udp_socket_internal(); - - std::coroutine_handle<> send_to( - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - endpoint, - int flags, - std::stop_token, - std::error_code*, - std::size_t*); - - std::coroutine_handle<> recv_from( - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - endpoint*, - int flags, - std::stop_token, - std::error_code*, - std::size_t*); - - std::coroutine_handle<> connect( - std::coroutine_handle<>, - capy::executor_ref, - endpoint, - std::stop_token, - std::error_code*); - - std::coroutine_handle<> send( - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - int flags, - std::stop_token, - std::error_code*, - std::size_t*); - - std::coroutine_handle<> recv( - std::coroutine_handle<>, - capy::executor_ref, - buffer_param, - int flags, - std::stop_token, - std::error_code*, - std::size_t*); - - std::coroutine_handle<> wait( - std::coroutine_handle<>, - capy::executor_ref, - wait_type, - std::stop_token, - std::error_code*); - - SOCKET native_handle() const noexcept; - endpoint local_endpoint() const noexcept; - endpoint remote_endpoint() const noexcept; - bool is_open() const noexcept; - void cancel() noexcept; - void close_socket() noexcept; - -private: endpoint local_endpoint_; endpoint remote_endpoint_; -}; -/** Datagram socket implementation wrapper for IOCP-based I/O. +public: + explicit win_udp_socket(win_udp_service& svc) noexcept; - This class is the public-facing implementation that holds a - shared_ptr to the internal state. The shared_ptr is hidden - from the public interface. + /// Recycle into the owning service's pool. Defined out-of-line + /// after `win_udp_service` is complete (needs `svc_.pool_`). + void retire() noexcept override; - @note Internal implementation detail. -*/ -class win_udp_socket final - : public udp_socket::implementation - , public intrusive_list::node -{ - std::shared_ptr internal_; + /** Reset recycled state for reuse. -public: - explicit win_udp_socket( - std::shared_ptr internal) noexcept; + Each op's `OVERLAPPED` fields are re-seeded by + `overlapped_op::reset()` at its next submit, so they are not + zeroed here. - void close_internal() noexcept; + @pre refs_ == 0, socket closed, no op in flight. + */ + void reuse() noexcept; std::coroutine_handle<> send_to( std::coroutine_handle<> h, @@ -325,7 +250,7 @@ class win_udp_socket final { // getsockname fails with WSAEINVAL on an unbound socket, so // the family recorded at socket creation is authoritative - return to_family(internal_ ? internal_->family_ : AF_UNSPEC); + return to_family(family_); } std::error_code shutdown(udp_socket::shutdown_type what) noexcept override; @@ -345,7 +270,7 @@ class win_udp_socket final endpoint remote_endpoint() const noexcept override; void cancel() noexcept override; - win_udp_socket_internal* get_internal() const noexcept; + void close_socket() noexcept; }; } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/iocp/win_wait_reactor.hpp b/include/boost/corosio/native/detail/iocp/win_wait_reactor.hpp index 78a250f49..408a6e81b 100644 --- a/include/boost/corosio/native/detail/iocp/win_wait_reactor.hpp +++ b/include/boost/corosio/native/detail/iocp/win_wait_reactor.hpp @@ -476,6 +476,13 @@ win_wait_reactor::run() { std::vector pollfds; + // Hoisted like pollfds: the swap under the lock trades buffers + // with pending_register_/pending_cancel_, so the capacities + // ping-pong between these locals and the members and no wake + // allocates once both sides are warm. + std::vector to_add; + std::vector to_cancel; + // What the ops still parked here are told on the way out, and what // every later register_wait is told too. Ending on a poll the // provider refused is not a cancellation, and an op that reports @@ -485,8 +492,8 @@ win_wait_reactor::run() while (!stop_.load(std::memory_order_acquire)) { // Drain pending register/cancel under the lock. - std::vector to_add; - std::vector to_cancel; + to_add.clear(); + to_cancel.clear(); { std::lock_guard lock(mutex_); to_add.swap(pending_register_); diff --git a/include/boost/corosio/native/detail/kqueue/kqueue_scheduler.hpp b/include/boost/corosio/native/detail/kqueue/kqueue_scheduler.hpp index 4471646d2..4560bf559 100644 --- a/include/boost/corosio/native/detail/kqueue/kqueue_scheduler.hpp +++ b/include/boost/corosio/native/detail/kqueue/kqueue_scheduler.hpp @@ -249,7 +249,7 @@ kqueue_scheduler::register_descriptor( desc->ready_events_.store(0, std::memory_order_relaxed); conditionally_enabled_mutex::scoped_lock lock(desc->mutex); - desc->impl_ref_.reset(); + desc->object_ref_.reset(); desc->read_ready = false; desc->write_ready = false; return {}; diff --git a/include/boost/corosio/native/detail/native_socket_base.hpp b/include/boost/corosio/native/detail/native_socket_base.hpp index c067cd1ac..7fd6709e6 100644 --- a/include/boost/corosio/native/detail/native_socket_base.hpp +++ b/include/boost/corosio/native/detail/native_socket_base.hpp @@ -27,9 +27,10 @@ Holds the part of a socket impl that does not care whether the backend is readiness-based (epoll/kqueue/select, which park ops on a descriptor_state) or completion-based (io_uring, which submits SQEs): - the file descriptor, the cached local endpoint, the impl lifecycle - bases (enable_shared_from_this + the service's intrusive tracking node), - and the synchronous accessors (native_handle / options / bind). + the file descriptor, the cached local endpoint, and the synchronous + accessors (native_handle / options / bind). The impl lifecycle (the + intrusive refcount + the owning service's recycling pool) is carried + by `io_object::implementation`, not by this base. reactor_basic_socket derives from this and adds the readiness machinery (descriptor_state, register_op, the cancel/close that deregister from @@ -39,8 +40,7 @@ readiness-agnostic socket surface, while the on-EAGAIN action (park vs submit-SQE) and the op model stay backend-specific. - @tparam Derived The concrete socket type (CRTP, for shared_from_this - and the intrusive node). + @tparam Derived The concrete socket type (CRTP). @tparam ImplBase The public vtable base (tcp_socket::implementation, udp_socket::implementation, ...). @tparam Endpoint The endpoint type (endpoint or local_endpoint). @@ -51,7 +51,6 @@ namespace boost::corosio::detail { template class native_socket_base : public ImplBase - , public std::enable_shared_from_this { protected: // CRTP base: not publicly constructible. The check's preferred fix diff --git a/include/boost/corosio/native/detail/posix/posix_random_access_file.hpp b/include/boost/corosio/native/detail/posix/posix_random_access_file.hpp index c43431f47..1679ec31e 100644 --- a/include/boost/corosio/native/detail/posix/posix_random_access_file.hpp +++ b/include/boost/corosio/native/detail/posix/posix_random_access_file.hpp @@ -35,7 +35,6 @@ #include #include #include -#include #include #include #include @@ -51,12 +50,13 @@ POSIX Random-Access File Implementation ======================================== - Each async read/write heap-allocates an raf_op that serves - as both the thread-pool work item and the scheduler completion - op. This allows unlimited concurrent operations on the same - file object, matching Asio's per-op allocation model. + Each async read/write acquires an raf_op that serves as both the + thread-pool work item and the scheduler completion op. This + allows unlimited concurrent operations on the same file object. - The raf_op self-deletes on completion or shutdown. + Ops recycle through the file's free_ops_ list on completion or + shutdown, so steady-state reads and writes allocate nothing; the + file destructor frees the recycled storage. */ namespace boost::corosio::detail { @@ -67,7 +67,6 @@ class posix_random_access_file_service; /** Random-access file implementation for POSIX backends. */ class posix_random_access_file final : public random_access_file::implementation - , public std::enable_shared_from_this , public intrusive_list::node { friend class posix_random_access_file_service; @@ -75,13 +74,15 @@ class posix_random_access_file final public: static constexpr std::size_t max_buffers = 16; - /** Per-operation state, heap-allocated for each async call. + /** Per-operation state, acquired from the file's free list. Inherits from `coro_op` (for scheduler completion plus the shared coroutine, cancellation and keepalive machinery) and - `pool_work_item` (for thread-pool dispatch). Linked into the - file's outstanding_ops_ list for cancellation tracking. `coro_op` - leads the base list so a `scheduler_op*` round-trips. + `pool_work_item` (for thread-pool dispatch). The intrusive hook + links it into `outstanding_ops_` while in flight and `free_ops_` + once recycled — membership is strictly sequential, so one hook + serves both under `ops_mutex_`. `coro_op` leads the base list so + a `scheduler_op*` round-trips. */ struct raf_op final : coro_op @@ -95,7 +96,7 @@ class posix_random_access_file final int errn = 0; std::size_t bytes_transferred = 0; - // Raw back-pointer for the typed work; `impl_ptr` is the keepalive. + // Raw back-pointer for the typed work; `object_ref_` is the keepalive. posix_random_access_file* file_ = nullptr; void operator()() override; @@ -108,6 +109,34 @@ class posix_random_access_file final explicit posix_random_access_file( posix_random_access_file_service& svc) noexcept; + /// Recycle into the owning service's pool. Defined out-of-line + /// after posix_random_access_file_service for its complete type. + void retire() noexcept override; + + /** Reset for recycling. + + `close_file()` already drove fd_ to its closed value before + the refcount reached zero. Every `raf_op` holds its own + `object_ref` while in flight, so the refcount cannot reach + zero while one is outstanding — asserting `outstanding_ops_` + is empty is therefore a precondition check, not a defensive + one. `free_ops_` deliberately survives recycling: the storage + belongs to this impl and only the destructor frees it. + + @pre refs_ == 0, fd closed, no op in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(fd_ == -1); + BOOST_COROSIO_ASSERT(outstanding_ops_.empty()); + } + + ~posix_random_access_file() override + { + while (auto* op = free_ops_.pop_front()) + delete op; + } + // -- random_access_file::implementation -- std::coroutine_handle<> read_some_at( @@ -153,10 +182,32 @@ class posix_random_access_file final void close_file() noexcept; private: + /** Pop a recycled op, or allocate on the cold path. + + Per-use fields are filled by the caller; the fields `prepare`- + style reuse must not inherit from the previous run (`errn`, + `bytes_transferred`) are reset here. `start()` resets the + cancellation machinery. + */ + raf_op* acquire_op() + { + raf_op* op; + { + std::lock_guard lock(ops_mutex_); + op = free_ops_.pop_front(); + } + if (!op) + op = new raf_op(); + op->errn = 0; + op->bytes_transferred = 0; + return op; + } + posix_random_access_file_service& svc_; int fd_ = -1; std::mutex ops_mutex_; intrusive_list outstanding_ops_; + intrusive_list free_ops_; }; // --------------------------------------------------------------------------- @@ -291,16 +342,20 @@ posix_random_access_file::raf_op::operator()() errn != 0 ? make_err(errn) : std::error_code{}, is_read, bytes_transferred, /*empty_buffer=*/false); + // Copy out everything needed after recycling: once this op is on + // free_ops_ a concurrent initiation may pop and refill it. The + // keepalive drops after the push so a final release never runs + // under ops_mutex_. + auto keep = std::move(object_ref_); + auto coro = h; + auto exec = ex; { std::lock_guard lock(file_->ops_mutex_); file_->outstanding_ops_.remove(this); + file_->free_ops_.push_front(this); } - - impl_ptr.reset(); - - auto coro = h; - ex.on_work_finished(); - delete this; + keep.reset(); + exec.on_work_finished(); coro.resume(); } @@ -310,13 +365,15 @@ inline void posix_random_access_file::raf_op::destroy() { stop_cb.reset(); + auto keep = std::move(object_ref_); + auto exec = ex; { std::lock_guard lock(file_->ops_mutex_); file_->outstanding_ops_.remove(this); + file_->free_ops_.push_front(this); } - impl_ptr.reset(); - ex.on_work_finished(); - delete this; + keep.reset(); + exec.on_work_finished(); } } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/posix/posix_random_access_file_service.hpp b/include/boost/corosio/native/detail/posix/posix_random_access_file_service.hpp index 4729fab1d..2c90926e4 100644 --- a/include/boost/corosio/native/detail/posix/posix_random_access_file_service.hpp +++ b/include/boost/corosio/native/detail/posix/posix_random_access_file_service.hpp @@ -17,11 +17,12 @@ #include #include #include +#include +#include #include #include -#include -#include +#include namespace boost::corosio::detail { @@ -29,6 +30,8 @@ namespace boost::corosio::detail { class BOOST_COROSIO_DECL posix_random_access_file_service final : public random_access_file_service { + friend class posix_random_access_file; + public: explicit posix_random_access_file_service(capy::execution_context& ctx) : sched_(&get_scheduler(ctx)) @@ -45,16 +48,7 @@ class BOOST_COROSIO_DECL posix_random_access_file_service final io_object::implementation* construct() override { - auto ptr = std::make_shared(*this); - auto* impl = ptr.get(); - - { - std::lock_guard lock(mutex_); - file_list_.push_back(impl); - file_ptrs_[impl] = std::move(ptr); - } - - return impl; + return object_pool_.acquire(*this); } void destroy(io_object::implementation* p) override @@ -62,7 +56,7 @@ class BOOST_COROSIO_DECL posix_random_access_file_service final auto& impl = static_cast(*p); impl.cancel(); impl.close_file(); - destroy_impl(impl); + release(&impl); } void close(io_object::handle& h) override @@ -90,21 +84,24 @@ class BOOST_COROSIO_DECL posix_random_access_file_service final void shutdown() override { - std::lock_guard lock(mutex_); - for (auto* impl = file_list_.pop_front(); impl != nullptr; - impl = file_list_.pop_front()) + // See uring_socket_service_base::shutdown(): snapshot under an + // acquired reference, cancel+close without the pool lock held; + // shutdown() sets shutting-down and takes the snapshot in one + // critical section, so a close that drops the last ref deletes + // rather than recycles. + std::vector live; + object_pool_.shutdown( + [&](posix_random_access_file* f) + { + acquire(f); + live.push_back(f); + }); + for (auto* f : live) { - impl->cancel(); - impl->close_file(); + f->cancel(); + f->close_file(); + release(f); } - file_ptrs_.clear(); - } - - void destroy_impl(posix_random_access_file& impl) - { - std::lock_guard lock(mutex_); - file_list_.remove(&impl); - file_ptrs_.erase(&impl); } void post(scheduler_op* op) @@ -143,12 +140,7 @@ class BOOST_COROSIO_DECL posix_random_access_file_service final private: scheduler* sched_; thread_pool_ref pool_; - std::mutex mutex_; - intrusive_list file_list_; - std::unordered_map< - posix_random_access_file*, - std::shared_ptr> - file_ptrs_; + object_pool object_pool_; }; // --------------------------------------------------------------------------- @@ -183,7 +175,7 @@ posix_random_access_file::read_some_at( return h; } - auto* op = new raf_op(); + auto* op = acquire_op(); op->is_read = true; op->offset = offset; @@ -194,12 +186,12 @@ posix_random_access_file::read_some_at( op->iovecs[i].iov_len = bufs[i].size(); } - op->h = h; - op->ex = ex; - op->ec_out = ec; - op->bytes_out = bytes_out; - op->file_ = this; - op->impl_ptr = this->shared_from_this(); + op->h = h; + op->ex = ex; + op->ec_out = ec; + op->bytes_out = bytes_out; + op->file_ = this; + op->object_ref_ = detail::object_ref(this); op->start(token); op->ex.on_work_started(); @@ -217,7 +209,7 @@ posix_random_access_file::read_some_at( // like the closed-descriptor and zero-length exits above rather // than through a completion the scheduler has to carry back. // destroy() is the discard the op never reaching the queue - // needs: it unlinks, unwinds the work count and frees. + // needs: it unlinks, unwinds the work count and recycles. op->destroy(); *ec = pec; *bytes_out = 0; @@ -254,7 +246,7 @@ posix_random_access_file::write_some_at( return h; } - auto* op = new raf_op(); + auto* op = acquire_op(); op->is_read = false; op->offset = offset; @@ -265,12 +257,12 @@ posix_random_access_file::write_some_at( op->iovecs[i].iov_len = bufs[i].size(); } - op->h = h; - op->ex = ex; - op->ec_out = ec; - op->bytes_out = bytes_out; - op->file_ = this; - op->impl_ptr = this->shared_from_this(); + op->h = h; + op->ex = ex; + op->ec_out = ec; + op->bytes_out = bytes_out; + op->file_ = this; + op->object_ref_ = detail::object_ref(this); op->start(token); op->ex.on_work_started(); @@ -288,7 +280,7 @@ posix_random_access_file::write_some_at( // like the closed-descriptor and zero-length exits above rather // than through a completion the scheduler has to carry back. // destroy() is the discard the op never reaching the queue - // needs: it unlinks, unwinds the work count and frees. + // needs: it unlinks, unwinds the work count and recycles. op->destroy(); *ec = pec; *bytes_out = 0; @@ -356,6 +348,12 @@ posix_random_access_file::raf_op::do_work(pool_work_item* w) noexcept self->svc_.post(static_cast(op)); } +inline void +posix_random_access_file::retire() noexcept +{ + svc_.object_pool_.recycle(this); +} + } // namespace boost::corosio::detail #endif // BOOST_COROSIO_POSIX diff --git a/include/boost/corosio/native/detail/posix/posix_resolver.hpp b/include/boost/corosio/native/detail/posix/posix_resolver.hpp index a72010b79..0498b022d 100644 --- a/include/boost/corosio/native/detail/posix/posix_resolver.hpp +++ b/include/boost/corosio/native/detail/posix/posix_resolver.hpp @@ -57,11 +57,12 @@ Class Hierarchy --------------- - posix_resolver_service (execution_context service, one per context) - - Owns all posix_resolver instances via shared_ptr + - Owns all posix_resolver instances via a recycling object_pool - Stores scheduler* for posting completions - posix_resolver (one per resolver object) - Contains embedded resolve_op and reverse_resolve_op for reuse - - Uses shared_from_this to prevent premature destruction + - Each op holds an object_ref keepalive while its pool work is + in flight, preventing premature destruction - resolve_op (forward resolution state) - Uses getaddrinfo() to resolve host/service to endpoints - reverse_resolve_op (reverse resolution state) @@ -86,9 +87,12 @@ Shutdown -------- - The resolver service cancels all resolvers and clears the impl map. - The thread pool service shuts down separately via execution_context - service ordering, joining all worker threads. + The resolver service cancels all resolvers and releases the + service's own reference on each; the recycling pool frees (rather + than recycles) once each resolver's in-flight pool work also + releases its reference. The thread pool service shuts down + separately via execution_context service ordering, joining all + worker threads. */ namespace boost::corosio::detail { @@ -154,7 +158,6 @@ class posix_resolver_service; */ class posix_resolver final : public resolver::implementation - , public std::enable_shared_from_this , public intrusive_list::node { friend class posix_resolver_service; @@ -213,11 +216,34 @@ class posix_resolver final posix_resolver* resolver_ = nullptr; /// Prevent impl destruction while work is in flight. - std::shared_ptr ref_; + detail::object_ref ref_; }; explicit posix_resolver(posix_resolver_service& svc) noexcept; + /// Recycle into the owning service's pool. Defined out-of-line + /// after posix_resolver_service for its complete type. + void retire() noexcept override; + + /** Reset for recycling. + + Both ops' own `reset()` already clears `host`/`service`/ + `stored_host`/`stored_service` via `.clear()` (preserving their + allocated capacity — the whole point of recycling), run at the + top of the next `resolve()`/`reverse_resolve()`. refs_ cannot + reach zero while a resolve is in flight (the pool op's + `object_ref` holds a reference for the duration), so asserting + `stop_cb` disengaged on both ops is a precondition check, not + a defensive one. + + @pre refs_ == 0, no resolve in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(!op_.stop_cb); + BOOST_COROSIO_ASSERT(!reverse_op_.stop_cb); + } + std::coroutine_handle<> resolve( std::coroutine_handle<>, capy::executor_ref, diff --git a/include/boost/corosio/native/detail/posix/posix_resolver_service.hpp b/include/boost/corosio/native/detail/posix/posix_resolver_service.hpp index 83b90230a..7bcc216d2 100644 --- a/include/boost/corosio/native/detail/posix/posix_resolver_service.hpp +++ b/include/boost/corosio/native/detail/posix/posix_resolver_service.hpp @@ -17,9 +17,11 @@ #include #include +#include +#include #include -#include +#include namespace boost::corosio::detail { @@ -32,6 +34,8 @@ class BOOST_COROSIO_DECL posix_resolver_service final : public capy::execution_context::service , public io_object::io_service { + friend class posix_resolver; + public: using key_type = posix_resolver_service; @@ -52,11 +56,10 @@ class BOOST_COROSIO_DECL posix_resolver_service final { auto& impl = static_cast(*p); impl.cancel(); - destroy_impl(impl); + release(&impl); } void shutdown() override; - void destroy_impl(posix_resolver& impl); void post(scheduler_op* op); @@ -89,10 +92,7 @@ class BOOST_COROSIO_DECL posix_resolver_service final private: scheduler* sched_; thread_pool_ref pool_; - std::mutex mutex_; - intrusive_list resolver_list_; - std::unordered_map> - resolver_ptrs_; + object_pool object_pool_; }; // --------------------------------------------------------------------------- @@ -271,7 +271,7 @@ posix_resolver::resolve_op::operator()() // Hold the keepalive across the dispatch: it may be the last // reference to the implementation this op is embedded in. - auto prevent_destroy = std::move(impl_ptr); + auto prevent_destroy = std::move(object_ref_); ex.on_work_finished(); cont.h = h; dispatch_coro(ex, cont).resume(); @@ -283,7 +283,7 @@ posix_resolver::resolve_op::destroy() stop_cb.reset(); auto local_ex = ex; // May destroy the implementation, and with it this op. - impl_ptr.reset(); + object_ref_.reset(); local_ex.on_work_finished(); } @@ -328,7 +328,7 @@ posix_resolver::reverse_resolve_op::operator()() // Hold the keepalive across the dispatch: it may be the last // reference to the implementation this op is embedded in. - auto prevent_destroy = std::move(impl_ptr); + auto prevent_destroy = std::move(object_ref_); ex.on_work_finished(); cont.h = h; dispatch_coro(ex, cont).resume(); @@ -340,7 +340,7 @@ posix_resolver::reverse_resolve_op::destroy() stop_cb.reset(); auto local_ex = ex; // May destroy the implementation, and with it this op. - impl_ptr.reset(); + object_ref_.reset(); local_ex.on_work_finished(); } @@ -380,7 +380,7 @@ posix_resolver::resolve( // Prevent impl destruction while work is in flight resolve_pool_op_.resolver_ = this; - resolve_pool_op_.ref_ = this->shared_from_this(); + resolve_pool_op_.ref_ = detail::object_ref(this); resolve_pool_op_.func_ = &posix_resolver::do_resolve_work; if (auto pec = svc_.pool().post(&resolve_pool_op_)) { @@ -430,7 +430,7 @@ posix_resolver::reverse_resolve( // Prevent impl destruction while work is in flight reverse_pool_op_.resolver_ = this; - reverse_pool_op_.ref_ = this->shared_from_this(); + reverse_pool_op_.ref_ = detail::object_ref(this); reverse_pool_op_.func_ = &posix_resolver::do_reverse_resolve_work; if (auto pec = svc_.pool().post(&reverse_pool_op_)) { @@ -492,7 +492,7 @@ posix_resolver::do_resolve_work(pool_work_item* w) noexcept // Hand the keepalive to the op: the completion waits in the // scheduler's queue, and the implementation embedding it must // outlive that wait. Nothing may touch *self after the post. - self->op_.impl_ptr = std::move(pw->ref_); + self->op_.object_ref_ = std::move(pw->ref_); self->svc_.post(&self->op_); } @@ -543,7 +543,7 @@ posix_resolver::do_reverse_resolve_work(pool_work_item* w) noexcept // Hand the keepalive to the op: the completion waits in the // scheduler's queue, and the implementation embedding it must // outlive that wait. Nothing may touch *self after the post. - self->reverse_op_.impl_ptr = std::move(pw->ref_); + self->reverse_op_.object_ref_ = std::move(pw->ref_); self->svc_.post(&self->reverse_op_); } @@ -552,48 +552,42 @@ posix_resolver::do_reverse_resolve_work(pool_work_item* w) noexcept inline void posix_resolver_service::shutdown() { - std::lock_guard lock(mutex_); - - // Cancel all resolvers (sets cancelled flag checked by pool threads) - for (auto* impl = resolver_list_.pop_front(); impl != nullptr; - impl = resolver_list_.pop_front()) + // See uring_socket_service_base::shutdown(): snapshot under an + // acquired reference, cancel without the pool lock held; shutdown() + // sets shutting-down and takes the snapshot in one critical + // section, so a cancel that drops the last ref deletes rather than + // recycles. The thread pool service shuts down separately via + // execution_context ordering. + std::vector live; + object_pool_.shutdown( + [&](posix_resolver* r) + { + acquire(r); + live.push_back(r); + }); + for (auto* r : live) { - impl->cancel(); + r->cancel(); + release(r); } - - // Clear the map which releases shared_ptrs. - // The thread pool service shuts down separately via - // execution_context service ordering. - resolver_ptrs_.clear(); } inline io_object::implementation* posix_resolver_service::construct() { - auto ptr = std::make_shared(*this); - auto* impl = ptr.get(); - - { - std::lock_guard lock(mutex_); - resolver_list_.push_back(impl); - resolver_ptrs_[impl] = std::move(ptr); - } - - return impl; + return object_pool_.acquire(*this); } inline void -posix_resolver_service::destroy_impl(posix_resolver& impl) +posix_resolver_service::post(scheduler_op* op) { - std::lock_guard lock(mutex_); - resolver_list_.remove(&impl); - resolver_ptrs_.erase(&impl); + sched_->post(op); } inline void -posix_resolver_service::post(scheduler_op* op) +posix_resolver::retire() noexcept { - sched_->post(op); + svc_.object_pool_.recycle(this); } } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/posix/posix_signal.hpp b/include/boost/corosio/native/detail/posix/posix_signal.hpp index eef8a9cce..7ba5fc133 100644 --- a/include/boost/corosio/native/detail/posix/posix_signal.hpp +++ b/include/boost/corosio/native/detail/posix/posix_signal.hpp @@ -123,6 +123,31 @@ class posix_signal final public: explicit posix_signal(posix_signal_service& svc) noexcept; + /// Recycle into the owning service's pool. Defined out-of-line + /// after posix_signal_service for its complete type. + void retire() noexcept override; + + /** Reset for recycling. + + `destroy()` already calls `clear()` (empties the registration + list) and `disarm_stop()` before the refcount reaches zero, so + those are asserted rather than re-cleared. `destroy()` also + calls `cancel()`, which deliberately latches `cancelled_` to + abort any wait still unwinding — that latch is this object's + to clear for its next, unrelated use, not something reset() + can skip by asserting it already false. + + @pre refs_ == 0, registration list empty, no wait in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(signals_ == nullptr); + BOOST_COROSIO_ASSERT(!waiting_); + BOOST_COROSIO_ASSERT(!stop_cb_); + cancelled_ = false; + token_cancelled_ = false; + } + std::coroutine_handle<> wait( std::coroutine_handle<>, capy::executor_ref, diff --git a/include/boost/corosio/native/detail/posix/posix_signal_service.hpp b/include/boost/corosio/native/detail/posix/posix_signal_service.hpp index 420b24df5..847db35e7 100644 --- a/include/boost/corosio/native/detail/posix/posix_signal_service.hpp +++ b/include/boost/corosio/native/detail/posix/posix_signal_service.hpp @@ -18,6 +18,8 @@ #include #include +#include +#include #include #include #include @@ -25,6 +27,7 @@ #include #include +#include #include #include @@ -65,7 +68,7 @@ 2. posix_signal_service (one per execution_context) - Maintains registrations_[] table indexed by signal number - Each slot is a doubly-linked list of signal_registrations for that signal - - Also maintains impl_list_ of all posix_signal objects it owns + - Also owns a recycling pool of all posix_signal objects it owns 3. posix_signal (one per signal_set) - Owns a singly-linked list (sorted by signal number) of signal_registrations @@ -154,6 +157,8 @@ class BOOST_COROSIO_DECL posix_signal_service final : public capy::execution_context::service , public io_object::io_service { + friend class posix_signal; + public: using key_type = posix_signal_service; @@ -171,7 +176,7 @@ class BOOST_COROSIO_DECL posix_signal_service final [[maybe_unused]] auto n = impl.clear(); impl.disarm_stop(); impl.cancel(); - destroy_impl(impl); + release(&impl); } /** Shut down the service. @@ -181,8 +186,6 @@ class BOOST_COROSIO_DECL posix_signal_service final */ void shutdown() override; - void destroy_impl(posix_signal& impl); - std::error_code add_signal( posix_signal& impl, int signal_number, signal_set::flags_t flags); @@ -233,7 +236,7 @@ class BOOST_COROSIO_DECL posix_signal_service final std::mutex reader_mutex_; bool reader_registered_ = false; - intrusive_list impl_list_; + object_pool pool_; // Per-signal registration table signal_registration* registrations_[max_signal_number]; @@ -526,12 +529,25 @@ inline posix_signal_service::~posix_signal_service() inline void posix_signal_service::shutdown() { - // Collected under the locks below and deleted after they are released: - // ~posix_signal destroys an armed stop_cb_, and ~stop_callback blocks - // until a concurrently running token_canceller returns -- which takes - // mutex_. Deleting while still holding mutex_ would self-deadlock the - // same way disarm_stop() would if called inside the locked loop. - intrusive_list doomed; + // Collected under the locks below and released after they are + // released: ~posix_signal destroys an armed stop_cb_, and + // ~stop_callback blocks until a concurrently running + // token_canceller returns -- which takes mutex_. Releasing while + // still holding mutex_ would self-deadlock the same way + // disarm_stop() would if called inside the locked loop. + // + // The acquire()+release() pair below nets to zero change on each + // impl's refs_ -- it exists only so the lock-held loop above can + // safely walk `doomed` without a concurrent recycle() tearing one + // down mid-walk. It does NOT drop these impls to zero: that still + // only happens when the handle's own destroy() runs, which this + // context-wide shutdown does not do (shutdown() cancels/clears + // registrations service-wide; it never calls destroy() per impl). + // Every impl here survives at refs_ == 1 (the service's own floor + // reference) until ~object_pool()'s unconditional sweep deletes + // whatever is still in `live_`, bypassing refs_ entirely -- not + // through this release() reaching zero. + std::vector doomed; { posix_signal_detail::signal_state* state = @@ -539,8 +555,13 @@ posix_signal_service::shutdown() std::lock_guard state_lock(state->mutex); std::lock_guard lock(mutex_); - for (auto* impl = impl_list_.pop_front(); impl != nullptr; - impl = impl_list_.pop_front()) + pool_.shutdown( + [&](posix_signal* impl) + { + acquire(impl); + doomed.push_back(impl); + }); + for (auto* impl : doomed) { while (auto* reg = impl->signals_) { @@ -568,47 +589,31 @@ posix_signal_service::shutdown() impl->signals_ = reg->next_in_set; delete reg; } - doomed.push_back(impl); } - // Every live registration hung off an implementation in impl_list_, - // so the whole table goes stale at once and can be dropped wholesale - // rather than node by node. It has to be dropped: deliver_signal() - // walks this service until the destructor unlinks it from the global - // list. + // Every live registration hung off an implementation this pool + // owns, so the whole table goes stale at once and can be dropped + // wholesale rather than node by node. It has to be dropped: + // deliver_signal() walks this service until the destructor + // unlinks it from the global list. for (int i = 0; i < max_signal_number; ++i) registrations_[i] = nullptr; } - for (auto* impl = doomed.pop_front(); impl != nullptr; - impl = doomed.pop_front()) - { - delete impl; - } + for (auto* impl : doomed) + release(impl); } inline io_object::implementation* posix_signal_service::construct() { - auto* impl = new posix_signal(*this); - - { - std::lock_guard lock(mutex_); - impl_list_.push_back(impl); - } - - return impl; + return pool_.acquire(*this); } inline void -posix_signal_service::destroy_impl(posix_signal& impl) +posix_signal::retire() noexcept { - { - std::lock_guard lock(mutex_); - impl_list_.remove(&impl); - } - - delete &impl; + svc_.pool_.recycle(this); } inline std::error_code diff --git a/include/boost/corosio/native/detail/posix/posix_stream_file.hpp b/include/boost/corosio/native/detail/posix/posix_stream_file.hpp index 0f0d6c092..fcc7e1d01 100644 --- a/include/boost/corosio/native/detail/posix/posix_stream_file.hpp +++ b/include/boost/corosio/native/detail/posix/posix_stream_file.hpp @@ -56,7 +56,8 @@ to a shared thread pool, with completion posted back to the scheduler. This follows the same pattern as posix_resolver: pool_work_item for - dispatch, scheduler_op for completion, shared_from_this for lifetime. + dispatch, scheduler_op for completion, an object_ref keepalive on + each op for lifetime while its pool work is in flight. Completion Flow --------------- @@ -84,7 +85,6 @@ class posix_stream_file_service; */ class posix_stream_file final : public stream_file::implementation - , public std::enable_shared_from_this , public intrusive_list::node { friend class posix_stream_file_service; @@ -117,7 +117,7 @@ class posix_stream_file final is_read = false; cancelled.store(false, std::memory_order_relaxed); stop_cb.reset(); - impl_ptr.reset(); + object_ref_.reset(); ec_out = nullptr; bytes_out = nullptr; } @@ -130,11 +130,31 @@ class posix_stream_file final struct pool_op : pool_work_item { posix_stream_file* file_ = nullptr; - std::shared_ptr ref_; + detail::object_ref ref_; }; explicit posix_stream_file(posix_stream_file_service& svc) noexcept; + /// Recycle into the owning service's pool. Defined out-of-line + /// after posix_stream_file_service for its complete type. + void retire() noexcept override; + + /** Reset op slots for recycling. + + `close_file()` already drove fd_ to its closed value before + the refcount reached zero; each op's own `reset()` runs again + before its next use, so only the `stop_cb` invariant is worth + asserting here. + + @pre refs_ == 0, fd closed, no op in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(fd_ == -1); + BOOST_COROSIO_ASSERT(!read_op_.stop_cb); + BOOST_COROSIO_ASSERT(!write_op_.stop_cb); + } + // -- io_stream::implementation -- std::coroutine_handle<> read_some( @@ -385,10 +405,10 @@ posix_stream_file::file_op::operator()() errn != 0 ? make_err(errn) : std::error_code{}, is_read, bytes_transferred, /*empty_buffer=*/false); - // Move impl_ptr to a local so members remain valid through - // dispatch — impl_ptr may be the last shared_ptr keeping - // the parent posix_stream_file (which embeds this file_op) alive. - auto prevent_destroy = std::move(impl_ptr); + // Move object_ref_ to a local so members remain valid through + // dispatch — this may be the last reference keeping the parent + // posix_stream_file (which embeds this file_op) alive. + auto prevent_destroy = std::move(object_ref_); ex.on_work_finished(); cont.h = h; dispatch_coro(ex, cont).resume(); @@ -399,7 +419,7 @@ posix_stream_file::file_op::destroy() { stop_cb.reset(); auto local_ex = ex; - impl_ptr.reset(); + object_ref_.reset(); local_ex.on_work_finished(); } diff --git a/include/boost/corosio/native/detail/posix/posix_stream_file_service.hpp b/include/boost/corosio/native/detail/posix/posix_stream_file_service.hpp index 268143474..89ac9ee3a 100644 --- a/include/boost/corosio/native/detail/posix/posix_stream_file_service.hpp +++ b/include/boost/corosio/native/detail/posix/posix_stream_file_service.hpp @@ -17,10 +17,11 @@ #include #include #include +#include +#include #include -#include -#include +#include namespace boost::corosio::detail { @@ -31,6 +32,8 @@ namespace boost::corosio::detail { */ class BOOST_COROSIO_DECL posix_stream_file_service final : public file_service { + friend class posix_stream_file; + public: explicit posix_stream_file_service(capy::execution_context& ctx) : sched_(&get_scheduler(ctx)) @@ -46,16 +49,7 @@ class BOOST_COROSIO_DECL posix_stream_file_service final : public file_service io_object::implementation* construct() override { - auto ptr = std::make_shared(*this); - auto* impl = ptr.get(); - - { - std::lock_guard lock(mutex_); - file_list_.push_back(impl); - file_ptrs_[impl] = std::move(ptr); - } - - return impl; + return object_pool_.acquire(*this); } void destroy(io_object::implementation* p) override @@ -63,7 +57,7 @@ class BOOST_COROSIO_DECL posix_stream_file_service final : public file_service auto& impl = static_cast(*p); impl.cancel(); impl.close_file(); - destroy_impl(impl); + release(&impl); } void close(io_object::handle& h) override @@ -90,21 +84,24 @@ class BOOST_COROSIO_DECL posix_stream_file_service final : public file_service void shutdown() override { - std::lock_guard lock(mutex_); - for (auto* impl = file_list_.pop_front(); impl != nullptr; - impl = file_list_.pop_front()) + // See uring_socket_service_base::shutdown(): snapshot under an + // acquired reference, cancel+close without the pool lock held; + // shutdown() sets shutting-down and takes the snapshot in one + // critical section, so a close that drops the last ref deletes + // rather than recycles. + std::vector live; + object_pool_.shutdown( + [&](posix_stream_file* f) + { + acquire(f); + live.push_back(f); + }); + for (auto* f : live) { - impl->cancel(); - impl->close_file(); + f->cancel(); + f->close_file(); + release(f); } - file_ptrs_.clear(); - } - - void destroy_impl(posix_stream_file& impl) - { - std::lock_guard lock(mutex_); - file_list_.remove(&impl); - file_ptrs_.erase(&impl); } void post(scheduler_op* op) @@ -143,10 +140,7 @@ class BOOST_COROSIO_DECL posix_stream_file_service final : public file_service private: scheduler* sched_; thread_pool_ref pool_; - std::mutex mutex_; - intrusive_list file_list_; - std::unordered_map> - file_ptrs_; + object_pool object_pool_; }; // --------------------------------------------------------------------------- @@ -201,7 +195,7 @@ posix_stream_file::read_some( op.ex.on_work_started(); read_pool_op_.file_ = this; - read_pool_op_.ref_ = this->shared_from_this(); + read_pool_op_.ref_ = detail::object_ref(this); read_pool_op_.func_ = &posix_stream_file::do_read_work; if (auto pec = svc_.pool().post(&read_pool_op_)) { @@ -251,7 +245,7 @@ posix_stream_file::do_read_work(pool_work_item* w) noexcept } } - op.impl_ptr = std::move(pw->ref_); + op.object_ref_ = std::move(pw->ref_); self->svc_.post(&op); } @@ -303,7 +297,7 @@ posix_stream_file::write_some( op.ex.on_work_started(); write_pool_op_.file_ = this; - write_pool_op_.ref_ = this->shared_from_this(); + write_pool_op_.ref_ = detail::object_ref(this); write_pool_op_.func_ = &posix_stream_file::do_write_work; if (auto pec = svc_.pool().post(&write_pool_op_)) { @@ -353,10 +347,16 @@ posix_stream_file::do_write_work(pool_work_item* w) noexcept } } - op.impl_ptr = std::move(pw->ref_); + op.object_ref_ = std::move(pw->ref_); self->svc_.post(&op); } +inline void +posix_stream_file::retire() noexcept +{ + svc_.object_pool_.recycle(this); +} + } // namespace boost::corosio::detail #endif // BOOST_COROSIO_POSIX diff --git a/include/boost/corosio/native/detail/reactor/reactor_acceptor.hpp b/include/boost/corosio/native/detail/reactor/reactor_acceptor.hpp index d89595b76..69664c56f 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_acceptor.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_acceptor.hpp @@ -12,12 +12,14 @@ #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -58,7 +60,6 @@ template< class Endpoint = endpoint> class reactor_acceptor : public ImplBase - , public std::enable_shared_from_this , public intrusive_list::node { friend Derived; @@ -258,6 +259,64 @@ class reactor_acceptor /** Release the acceptor without closing the fd. */ native_handle_type do_release_socket() noexcept; + /** Reset descriptor state for recycling. + + Actively re-initializes every field it owns rather than + trusting `close_socket()`'s prior writes to still be there — + see reactor_basic_socket::reuse() for the same rationale. + `object_ref_` and `is_enqueued_` are asserted instead: `poison()` + never touches them, so they must already hold their + zero-action-complete state. + + @pre refs_ == 0, fd closed and deregistered, no op in flight. + */ + void reuse() noexcept + { + fd_ = -1; + local_endpoint_ = Endpoint{}; + desc_state_.fd = -1; + desc_state_.registered_events = 0; + desc_state_.read_op = nullptr; + desc_state_.wait_read_op = nullptr; + desc_state_.wait_write_op = nullptr; + desc_state_.wait_error_op = nullptr; + desc_state_.read_ready = false; + desc_state_.write_ready = false; + BOOST_COROSIO_ASSERT(!desc_state_.object_ref_); + BOOST_COROSIO_ASSERT(!desc_state_.is_enqueued_.load( + std::memory_order_relaxed)); + BOOST_COROSIO_ASSERT(!acc_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_wr_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_er_.stop_cb); + } + +#if !defined(NDEBUG) + /** Poison the fields `reuse()` re-initializes, before the pool + parks this impl on the free list. See + reactor_basic_socket::poison() for the full rationale and the + list of fields deliberately left untouched. + + @pre refs_ == 0, fd closed and deregistered, no op in flight. + */ + void poison() noexcept + { + auto smash = [](auto& field) { + std::memset(static_cast(&field), 0xDB, sizeof(field)); + }; + smash(fd_); + smash(local_endpoint_); + smash(desc_state_.fd); + smash(desc_state_.registered_events); + smash(desc_state_.read_op); + smash(desc_state_.wait_read_op); + smash(desc_state_.wait_write_op); + smash(desc_state_.wait_error_op); + smash(desc_state_.read_ready); + smash(desc_state_.write_ready); + } +#endif + /** Bind the acceptor socket to an endpoint. Caches the resolved local endpoint (including ephemeral @@ -300,10 +359,6 @@ reactor_acceptor< ImplBase, Endpoint>::cancel_single_op(Op& op) noexcept { - auto self = this->weak_from_this().lock(); - if (!self) - return; - op.request_cancel(); reactor_op_base* claimed = nullptr; @@ -320,7 +375,7 @@ reactor_acceptor< } if (claimed) { - op.impl_ptr = self; + op.object_ref_ = detail::object_ref(this); svc_.post(&op); svc_.work_finished(); } @@ -372,44 +427,40 @@ reactor_acceptor< ImplBase, Endpoint>::do_close_socket() noexcept { - auto self = this->weak_from_this().lock(); - if (self) + acc_.request_cancel(); + wait_rd_.request_cancel(); + wait_wr_.request_cancel(); + wait_er_.request_cancel(); + + reactor_op_base* claimed_acc = nullptr; + reactor_op_base* claimed_wr = nullptr; + reactor_op_base* claimed_ww = nullptr; + reactor_op_base* claimed_we = nullptr; { - acc_.request_cancel(); - wait_rd_.request_cancel(); - wait_wr_.request_cancel(); - wait_er_.request_cancel(); - - reactor_op_base* claimed_acc = nullptr; - reactor_op_base* claimed_wr = nullptr; - reactor_op_base* claimed_ww = nullptr; - reactor_op_base* claimed_we = nullptr; + std::lock_guard lock(desc_state_.mutex); + claimed_acc = std::exchange(desc_state_.read_op, nullptr); + claimed_wr = std::exchange(desc_state_.wait_read_op, nullptr); + claimed_ww = std::exchange(desc_state_.wait_write_op, nullptr); + claimed_we = std::exchange(desc_state_.wait_error_op, nullptr); + desc_state_.read_ready = false; + desc_state_.write_ready = false; + + if (desc_state_.is_enqueued_.load(std::memory_order_acquire)) + desc_state_.object_ref_ = detail::object_ref(this); + } + + auto repost = [&](reactor_op_base* claimed, reactor_op_base& op) { + if (claimed) { - std::lock_guard lock(desc_state_.mutex); - claimed_acc = std::exchange(desc_state_.read_op, nullptr); - claimed_wr = std::exchange(desc_state_.wait_read_op, nullptr); - claimed_ww = std::exchange(desc_state_.wait_write_op, nullptr); - claimed_we = std::exchange(desc_state_.wait_error_op, nullptr); - desc_state_.read_ready = false; - desc_state_.write_ready = false; - - if (desc_state_.is_enqueued_.load(std::memory_order_acquire)) - desc_state_.impl_ref_ = self; + op.object_ref_ = detail::object_ref(this); + svc_.post(&op); + svc_.work_finished(); } - - auto repost = [&](reactor_op_base* claimed, reactor_op_base& op) { - if (claimed) - { - op.impl_ptr = self; - svc_.post(&op); - svc_.work_finished(); - } - }; - repost(claimed_acc, acc_); - repost(claimed_wr, wait_rd_); - repost(claimed_ww, wait_wr_); - repost(claimed_we, wait_er_); - } + }; + repost(claimed_acc, acc_); + repost(claimed_wr, wait_rd_); + repost(claimed_ww, wait_wr_); + repost(claimed_we, wait_er_); if (fd_ >= 0) { @@ -445,44 +496,40 @@ reactor_acceptor< ImplBase, Endpoint>::do_release_socket() noexcept { - auto self = this->weak_from_this().lock(); - if (self) + acc_.request_cancel(); + wait_rd_.request_cancel(); + wait_wr_.request_cancel(); + wait_er_.request_cancel(); + + reactor_op_base* claimed_acc = nullptr; + reactor_op_base* claimed_wr = nullptr; + reactor_op_base* claimed_ww = nullptr; + reactor_op_base* claimed_we = nullptr; { - acc_.request_cancel(); - wait_rd_.request_cancel(); - wait_wr_.request_cancel(); - wait_er_.request_cancel(); - - reactor_op_base* claimed_acc = nullptr; - reactor_op_base* claimed_wr = nullptr; - reactor_op_base* claimed_ww = nullptr; - reactor_op_base* claimed_we = nullptr; + std::lock_guard lock(desc_state_.mutex); + claimed_acc = std::exchange(desc_state_.read_op, nullptr); + claimed_wr = std::exchange(desc_state_.wait_read_op, nullptr); + claimed_ww = std::exchange(desc_state_.wait_write_op, nullptr); + claimed_we = std::exchange(desc_state_.wait_error_op, nullptr); + desc_state_.read_ready = false; + desc_state_.write_ready = false; + + if (desc_state_.is_enqueued_.load(std::memory_order_acquire)) + desc_state_.object_ref_ = detail::object_ref(this); + } + + auto repost = [&](reactor_op_base* claimed, reactor_op_base& op) { + if (claimed) { - std::lock_guard lock(desc_state_.mutex); - claimed_acc = std::exchange(desc_state_.read_op, nullptr); - claimed_wr = std::exchange(desc_state_.wait_read_op, nullptr); - claimed_ww = std::exchange(desc_state_.wait_write_op, nullptr); - claimed_we = std::exchange(desc_state_.wait_error_op, nullptr); - desc_state_.read_ready = false; - desc_state_.write_ready = false; - - if (desc_state_.is_enqueued_.load(std::memory_order_acquire)) - desc_state_.impl_ref_ = self; + op.object_ref_ = detail::object_ref(this); + svc_.post(&op); + svc_.work_finished(); } - - auto repost = [&](reactor_op_base* claimed, reactor_op_base& op) { - if (claimed) - { - op.impl_ptr = self; - svc_.post(&op); - svc_.work_finished(); - } - }; - repost(claimed_acc, acc_); - repost(claimed_wr, wait_rd_); - repost(claimed_ww, wait_wr_); - repost(claimed_we, wait_er_); - } + }; + repost(claimed_acc, acc_); + repost(claimed_wr, wait_rd_); + repost(claimed_ww, wait_wr_); + repost(claimed_we, wait_er_); native_handle_type released = fd_; @@ -606,7 +653,7 @@ reactor_acceptor< op.ec_out = ec; op.fd = this->fd_; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(ENOTSUP, 0); svc_.post(&op); return std::noop_coroutine(); @@ -637,7 +684,7 @@ reactor_acceptor< op.ec_out = ec; op.fd = this->fd_; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); // A listener's readiness can predate the wait: an adopted or // shared descriptor has history the reactor never saw, and an diff --git a/include/boost/corosio/native/detail/reactor/reactor_acceptor_service.hpp b/include/boost/corosio/native/detail/reactor/reactor_acceptor_service.hpp index 5063161e0..7bd11e1c4 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_acceptor_service.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_acceptor_service.hpp @@ -11,12 +11,12 @@ #define BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_ACCEPTOR_SERVICE_HPP #include +#include #include #include #include #include -#include namespace boost::corosio::detail { @@ -45,6 +45,13 @@ template< class reactor_acceptor_service : public ServiceBase { friend Derived; + + // The intermediate CRTP template below (not Impl itself) is what + // actually reaches into state_->pool_ from retire() -- see + // reactor_socket_finals.hpp. + template + friend class reactor_acceptor_impl; + using state_type = reactor_service_state; protected: @@ -62,31 +69,23 @@ class reactor_acceptor_service : public ServiceBase void shutdown() override { - std::lock_guard lock(state_->mutex_); + state_->pool_.shutdown([](Impl* impl) { impl->close_socket(); }); - while (auto* impl = state_->impl_list_.pop_front()) - impl->close_socket(); + // See reactor_socket_service::shutdown(): queued ops hold their + // own reference; the scheduler's drain releases them after us, + // and shutting-down mode deletes rather than recycles at zero. } io_object::implementation* construct() override { - auto impl = std::make_shared(static_cast(*this)); - auto* raw = impl.get(); - - std::lock_guard lock(state_->mutex_); - state_->impl_ptrs_.emplace(raw, std::move(impl)); - state_->impl_list_.push_back(raw); - - return raw; + return state_->pool_.acquire(static_cast(*this)); } void destroy(io_object::implementation* impl) override { auto* typed = static_cast(impl); typed->close_socket(); - std::lock_guard lock(state_->mutex_); - state_->impl_list_.remove(typed); - state_->impl_ptrs_.erase(typed); + release(typed); } void close(io_object::handle& h) override diff --git a/include/boost/corosio/native/detail/reactor/reactor_backend.hpp b/include/boost/corosio/native/detail/reactor/reactor_backend.hpp index bf7b6ed9c..59adf8a26 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_backend.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_backend.hpp @@ -128,14 +128,14 @@ reactor_acceptor_impl< op.peer_storage = peer_storage; op.peer_addrlen = peer_addrlen; op.complete(0, 0); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->svc_.post(&op); return std::noop_coroutine(); } if (errno == EAGAIN || errno == EWOULDBLOCK) { - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->svc_.work_started(); std::lock_guard lock(this->desc_state_.mutex); @@ -162,7 +162,7 @@ reactor_acceptor_impl< } op.complete(errno, 0); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->svc_.post(&op); return std::noop_coroutine(); } diff --git a/include/boost/corosio/native/detail/reactor/reactor_basic_socket.hpp b/include/boost/corosio/native/detail/reactor/reactor_basic_socket.hpp index b3165cb00..eaf90bf16 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_basic_socket.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_basic_socket.hpp @@ -10,6 +10,7 @@ #ifndef BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_BASIC_SOCKET_HPP #define BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_BASIC_SOCKET_HPP +#include #include #include #include @@ -18,6 +19,7 @@ #include #include +#include #include #include #include @@ -174,6 +176,75 @@ class reactor_basic_socket Returns the fd so the caller can take ownership. */ native_handle_type do_release_socket() noexcept; + + /** Reset descriptor state for recycling. + + Called by the owning service's `construct()` when popping this + impl back off the free list. Actively re-initializes every + field it owns rather than trusting `close_socket()`'s prior + writes to still be there — `poison()` below depends on that, + and a future field added to either side without the other + would otherwise go unnoticed until this one re-sets it. + `object_ref_` and `is_enqueued_` are asserted instead: `poison()` + never touches them, so they must already hold their + zero-action-complete state. + + @pre refs_ == 0, fd closed and deregistered, no op in flight. + */ + void reuse() noexcept + { + fd_ = -1; + local_endpoint_ = Endpoint{}; + desc_state_.fd = -1; + desc_state_.registered_events = 0; + desc_state_.read_op = nullptr; + desc_state_.write_op = nullptr; + desc_state_.connect_op = nullptr; + desc_state_.wait_read_op = nullptr; + desc_state_.wait_write_op = nullptr; + desc_state_.wait_error_op = nullptr; + desc_state_.read_ready = false; + desc_state_.write_ready = false; + BOOST_COROSIO_ASSERT(!desc_state_.object_ref_); + BOOST_COROSIO_ASSERT(!desc_state_.is_enqueued_.load( + std::memory_order_relaxed)); + } + +#if !defined(NDEBUG) + /** Poison the fields `reuse()` re-initializes, before the pool + parks this impl on the free list. + + A field `reuse()` forgets to reset then reads back as this + unmistakable byte pattern instead of silently matching + whatever `close_socket()` happened to leave, turning a missed + reset into an immediate, loud failure rather than a latent + stale-state bug. Does not touch the mutex, `ready_events_` / + `is_enqueued_` (atomics), `scheduler_`, `object_ref_`, the + `intrusive_list` hook, or `refs_`: `recycle()` still reads + some of those after this call returns, and smashing the rest + would be undefined rather than diagnostic. + + @pre refs_ == 0, fd closed and deregistered, no op in flight. + */ + void poison() noexcept + { + auto smash = [](auto& field) { + std::memset(static_cast(&field), 0xDB, sizeof(field)); + }; + smash(fd_); + smash(local_endpoint_); + smash(desc_state_.fd); + smash(desc_state_.registered_events); + smash(desc_state_.read_op); + smash(desc_state_.write_op); + smash(desc_state_.connect_op); + smash(desc_state_.wait_read_op); + smash(desc_state_.wait_write_op); + smash(desc_state_.wait_error_op); + smash(desc_state_.read_ready); + smash(desc_state_.write_ready); + } +#endif }; template< @@ -238,10 +309,6 @@ void reactor_basic_socket:: cancel_single_op(Op& op) noexcept { - auto self = this->weak_from_this().lock(); - if (!self) - return; - op.request_cancel(); auto* d = static_cast(this); @@ -262,7 +329,7 @@ reactor_basic_socket:: } if (claimed) { - op.impl_ptr = self; + op.object_ref_ = detail::object_ref(this); svc_.post(&op); svc_.work_finished(); } @@ -279,10 +346,6 @@ void reactor_basic_socket:: do_cancel() noexcept { - auto self = this->weak_from_this().lock(); - if (!self) - return; - auto* d = static_cast(this); d->for_each_op([](auto& op) { op.request_cancel(); }); @@ -311,7 +374,7 @@ reactor_basic_socket:: for (int i = 0; i < count; ++i) { - claimed[i].base->impl_ptr = self; + claimed[i].base->object_ref_ = detail::object_ref(this); svc_.post(claimed[i].base); svc_.work_finished(); } @@ -327,44 +390,40 @@ void reactor_basic_socket:: do_close_socket() noexcept { - auto self = this->weak_from_this().lock(); - if (self) - { - auto* d = static_cast(this); + auto* d = static_cast(this); - d->for_each_op([](auto& op) { op.request_cancel(); }); + d->for_each_op([](auto& op) { op.request_cancel(); }); - struct claimed_entry - { - reactor_op_base* base = nullptr; - }; - claimed_entry claimed[8]; - int count = 0; + struct claimed_entry + { + reactor_op_base* base = nullptr; + }; + claimed_entry claimed[8]; + int count = 0; - { - std::lock_guard lock(desc_state_.mutex); - d->for_each_desc_entry( - [&](auto& /*op*/, reactor_op_base*& desc_slot) { - auto* c = std::exchange(desc_slot, nullptr); - if (c) - { - claimed[count].base = c; - ++count; - } - }); - desc_state_.read_ready = false; - desc_state_.write_ready = false; - - if (desc_state_.is_enqueued_.load(std::memory_order_acquire)) - desc_state_.impl_ref_ = self; - } + { + std::lock_guard lock(desc_state_.mutex); + d->for_each_desc_entry( + [&](auto& /*op*/, reactor_op_base*& desc_slot) { + auto* c = std::exchange(desc_slot, nullptr); + if (c) + { + claimed[count].base = c; + ++count; + } + }); + desc_state_.read_ready = false; + desc_state_.write_ready = false; + + if (desc_state_.is_enqueued_.load(std::memory_order_acquire)) + desc_state_.object_ref_ = detail::object_ref(this); + } - for (int i = 0; i < count; ++i) - { - claimed[i].base->impl_ptr = self; - svc_.post(claimed[i].base); - svc_.work_finished(); - } + for (int i = 0; i < count; ++i) + { + claimed[i].base->object_ref_ = detail::object_ref(this); + svc_.post(claimed[i].base); + svc_.work_finished(); } if (fd_ >= 0) @@ -392,44 +451,40 @@ reactor_basic_socket:: do_release_socket() noexcept { // Cancel pending ops (same as do_close_socket) - auto self = this->weak_from_this().lock(); - if (self) - { - auto* d = static_cast(this); + auto* d = static_cast(this); - d->for_each_op([](auto& op) { op.request_cancel(); }); + d->for_each_op([](auto& op) { op.request_cancel(); }); - struct claimed_entry - { - reactor_op_base* base = nullptr; - }; - claimed_entry claimed[8]; - int count = 0; + struct claimed_entry + { + reactor_op_base* base = nullptr; + }; + claimed_entry claimed[8]; + int count = 0; - { - std::lock_guard lock(desc_state_.mutex); - d->for_each_desc_entry( - [&](auto& /*op*/, reactor_op_base*& desc_slot) { - auto* c = std::exchange(desc_slot, nullptr); - if (c) - { - claimed[count].base = c; - ++count; - } - }); - desc_state_.read_ready = false; - desc_state_.write_ready = false; - - if (desc_state_.is_enqueued_.load(std::memory_order_acquire)) - desc_state_.impl_ref_ = self; - } + { + std::lock_guard lock(desc_state_.mutex); + d->for_each_desc_entry( + [&](auto& /*op*/, reactor_op_base*& desc_slot) { + auto* c = std::exchange(desc_slot, nullptr); + if (c) + { + claimed[count].base = c; + ++count; + } + }); + desc_state_.read_ready = false; + desc_state_.write_ready = false; + + if (desc_state_.is_enqueued_.load(std::memory_order_acquire)) + desc_state_.object_ref_ = detail::object_ref(this); + } - for (int i = 0; i < count; ++i) - { - claimed[i].base->impl_ptr = self; - svc_.post(claimed[i].base); - svc_.work_finished(); - } + for (int i = 0; i < count; ++i) + { + claimed[i].base->object_ref_ = detail::object_ref(this); + svc_.post(claimed[i].base); + svc_.work_finished(); } native_handle_type released = fd_; diff --git a/include/boost/corosio/native/detail/reactor/reactor_datagram_socket.hpp b/include/boost/corosio/native/detail/reactor/reactor_datagram_socket.hpp index b88afa9e9..e861e09b4 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_datagram_socket.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_datagram_socket.hpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -20,6 +21,7 @@ #include #include +#include #include #include @@ -325,6 +327,40 @@ class reactor_datagram_socket return fd; } + /** Reset op slots and the cached peer for recycling. + + See reactor_stream_socket::reuse() — same rationale: op state + is overwritten by the next do_* call, so only `remote_endpoint_` + needs an active reset here; every other slot only asserts + `stop_cb` disengaged. + + @pre refs_ == 0, fd closed and deregistered, no op in flight. + */ + void reuse() noexcept + { + base_type::reuse(); + remote_endpoint_ = Endpoint{}; + BOOST_COROSIO_ASSERT(!conn_.stop_cb); + BOOST_COROSIO_ASSERT(!wr_.stop_cb); + BOOST_COROSIO_ASSERT(!rd_.stop_cb); + BOOST_COROSIO_ASSERT(!send_wr_.stop_cb); + BOOST_COROSIO_ASSERT(!recv_rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_wr_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_er_.stop_cb); + } + +#if !defined(NDEBUG) + /// Extend base_type::poison() to the cached peer endpoint. + void poison() noexcept + { + base_type::poison(); + std::memset( + static_cast(&remote_endpoint_), 0xDB, + sizeof(remote_endpoint_)); + } +#endif + /** Shut down part or all of the full-duplex connection. Not an override — concrete backends forward here. @@ -497,7 +533,7 @@ reactor_datagram_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(err, bytes); this->svc_.post(&op); return std::noop_coroutine(); @@ -509,7 +545,7 @@ reactor_datagram_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->register_op( op, this->desc_state_.write_op, this->desc_state_.write_ready, true); @@ -567,7 +603,7 @@ reactor_datagram_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(0, 0); this->svc_.post(&op); return std::noop_coroutine(); @@ -619,7 +655,7 @@ reactor_datagram_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(err, bytes); this->svc_.post(&op); return std::noop_coroutine(); @@ -631,7 +667,7 @@ reactor_datagram_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->register_op( op, this->desc_state_.read_op, this->desc_state_.read_ready); @@ -708,7 +744,7 @@ reactor_datagram_socket< op.fd = this->fd_; op.target_endpoint = ep; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(err, 0); this->svc_.post(&op); return std::noop_coroutine(); @@ -800,7 +836,7 @@ reactor_datagram_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(err, bytes); this->svc_.post(&op); return std::noop_coroutine(); @@ -812,7 +848,7 @@ reactor_datagram_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->register_op( op, this->desc_state_.write_op, this->desc_state_.write_ready, true); @@ -868,7 +904,7 @@ reactor_datagram_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(0, 0); this->svc_.post(&op); return std::noop_coroutine(); @@ -912,7 +948,7 @@ reactor_datagram_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(err, bytes); this->svc_.post(&op); return std::noop_coroutine(); @@ -924,7 +960,7 @@ reactor_datagram_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->register_op( op, this->desc_state_.read_op, this->desc_state_.read_ready); @@ -1010,7 +1046,7 @@ reactor_datagram_socket< op.ec_out = ec; op.fd = this->fd_; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(perr, 0); this->svc_.post(&op); return std::noop_coroutine(); @@ -1023,7 +1059,7 @@ reactor_datagram_socket< op.ec_out = ec; op.fd = this->fd_; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); // Force register_op's ready path so the wait op re-probes under // the descriptor mutex before parking. An edge consumed between diff --git a/include/boost/corosio/native/detail/reactor/reactor_descriptor_state.hpp b/include/boost/corosio/native/detail/reactor/reactor_descriptor_state.hpp index 4dda0eb76..8789a68ab 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_descriptor_state.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_descriptor_state.hpp @@ -88,7 +88,7 @@ struct reactor_descriptor_state : scheduler_op reactor_scheduler const* scheduler_ = nullptr; /// Prevents impl destruction while queued in the scheduler. - std::shared_ptr impl_ref_; + detail::object_ref object_ref_; /// Add ready events atomically. /// Release pairs with the consumer's acquire exchange on @@ -106,11 +106,11 @@ struct reactor_descriptor_state : scheduler_op } /// Destroy without invoking. - /// Called during scheduler::shutdown() drain. Clear impl_ref_ to break + /// Called during scheduler::shutdown() drain. Clear object_ref_ to break /// the self-referential cycle set by close_socket(). void destroy() override { - impl_ref_.reset(); + object_ref_.reset(); } /** Perform deferred I/O and queue completions. @@ -125,18 +125,18 @@ struct reactor_descriptor_state : scheduler_op inline void reactor_descriptor_state::invoke_deferred_io() { - std::shared_ptr prevent_impl_destruction; + detail::object_ref prevent_impl_destruction; ready_queue local_ops; { conditionally_enabled_mutex::scoped_lock lock(mutex); - // Must clear is_enqueued_ and move impl_ref_ under the same + // Must clear is_enqueued_ and move object_ref_ under the same // lock that processes I/O. close_socket() checks is_enqueued_ // under this mutex — without atomicity between the flag store // and the ref move, close_socket() could see is_enqueued_==false, - // skip setting impl_ref_, and destroy the impl under us. - prevent_impl_destruction = std::move(impl_ref_); + // skip setting object_ref_, and destroy the impl under us. + prevent_impl_destruction = std::move(object_ref_); is_enqueued_.store(false, std::memory_order_release); std::uint32_t ev = ready_events_.exchange(0, std::memory_order_acquire); diff --git a/include/boost/corosio/native/detail/reactor/reactor_op.hpp b/include/boost/corosio/native/detail/reactor/reactor_op.hpp index 986e590f0..1f6ab425d 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_op.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_op.hpp @@ -38,7 +38,7 @@ namespace boost::corosio::detail { impl pointers. Fields shared across all backends (errn, bytes_transferred, - cancelled, impl_ptr, perform_io, complete) live in + cancelled, object_ref_, perform_io, complete) live in reactor_op_base so the scheduler and descriptor_state can access them without template instantiation. @@ -49,7 +49,7 @@ template struct reactor_op : reactor_op_base { // The op envelope — coroutine handle h, cont, executor ex, ec_out, - // bytes_out, cancelled, stop_cb (+ its canceller), impl_ptr — lives in + // bytes_out, cancelled, stop_cb (+ its canceller), object_ref_ — lives in // coro_op (via reactor_op_base) and is shared with io_uring/IOCP. // reactor_op adds only the reactor-specific routing state below. @@ -71,7 +71,7 @@ struct reactor_op : reactor_op_base errn = 0; bytes_transferred = 0; cancelled.store(false, std::memory_order_relaxed); - impl_ptr.reset(); + object_ref_.reset(); socket_impl_ = nullptr; acceptor_impl_ = nullptr; } diff --git a/include/boost/corosio/native/detail/reactor/reactor_op_base.hpp b/include/boost/corosio/native/detail/reactor/reactor_op_base.hpp index 0f91b9eaf..516a75108 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_op_base.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_op_base.hpp @@ -21,7 +21,7 @@ namespace boost::corosio::detail { Adds the reactor-specific result model (the recorded errno + bytes and the perform_io() readiness re-run) on top of coro_op, the shared op envelope (coroutine handle, executor, output pointers, stop_token wiring, - cancelled flag, impl_ptr keepalive) common to every backend. + cancelled flag, object_ref_ keepalive) common to every backend. Kept as a non-template layer so reactor_descriptor_state and the scheduler hot paths can touch op state without knowing the concrete @@ -37,7 +37,7 @@ struct reactor_op_base : coro_op /// Bytes transferred on success. std::size_t bytes_transferred = 0; - // cancelled, impl_ptr, and request_cancel() are inherited from coro_op. + // cancelled, object_ref_, and request_cancel() are inherited from coro_op. /// Record the result of an I/O attempt. void complete(int err, std::size_t bytes) noexcept @@ -49,10 +49,11 @@ struct reactor_op_base : coro_op /// Perform the I/O syscall (implemented by concrete op types). virtual void perform_io() noexcept = 0; - /// Destroy without invoking — drop the keepalive (impl_ptr from coro_op). + /// Destroy without invoking — drop the keepalive held in + /// coro_op's `object_ref_`. void destroy() override { - impl_ptr.reset(); + object_ref_.reset(); } }; diff --git a/include/boost/corosio/native/detail/reactor/reactor_service_state.hpp b/include/boost/corosio/native/detail/reactor/reactor_service_state.hpp index 636ca388d..e04965e01 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_service_state.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_service_state.hpp @@ -10,18 +10,15 @@ #ifndef BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_SERVICE_STATE_HPP #define BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_SERVICE_STATE_HPP +#include #include -#include -#include -#include - namespace boost::corosio::detail { /** Shared service state for reactor backends. - Holds the scheduler reference, service mutex, and per-impl - ownership tracking. Used by both socket and acceptor services. + Holds the scheduler reference and the impl recycling pool. Used + by both socket and acceptor services. @tparam Scheduler The backend's scheduler type. @tparam Impl The backend's socket or acceptor impl type. @@ -35,14 +32,8 @@ struct reactor_service_state /// Reference to the owning scheduler. Scheduler& sched_; - /// Protects `impl_list_` and `impl_ptrs_`. - std::mutex mutex_; - - /// All live impl objects for shutdown traversal. - intrusive_list impl_list_; - - /// Shared ownership of each impl, keyed by raw pointer. - std::unordered_map> impl_ptrs_; + /// Owns every impl: live for shutdown traversal, free for reuse. + object_pool pool_; }; } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/reactor/reactor_socket_finals.hpp b/include/boost/corosio/native/detail/reactor/reactor_socket_finals.hpp index aa405ff5c..5d37ce30a 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_socket_finals.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_socket_finals.hpp @@ -87,6 +87,15 @@ class reactor_stream_socket_impl ~reactor_stream_socket_impl() override = default; + /// Recycle into the owning service's pool at zero references. + void retire() noexcept override + { +#if !defined(NDEBUG) + this->poison(); +#endif + this->svc_.state_->pool_.recycle(static_cast(this)); + } + std::error_code set_option( int level, int optname, @@ -151,6 +160,15 @@ class reactor_dgram_socket_impl using impl_base_type = ImplBase; ~reactor_dgram_socket_impl() override = default; + + /// Recycle into the owning service's pool at zero references. + void retire() noexcept override + { +#if !defined(NDEBUG) + this->poison(); +#endif + this->svc_.state_->pool_.recycle(static_cast(this)); + } }; // ============================================================ @@ -197,6 +215,15 @@ class reactor_acceptor_impl ~reactor_acceptor_impl() override = default; + /// Recycle into the owning service's pool at zero references. + void retire() noexcept override + { +#if !defined(NDEBUG) + this->poison(); +#endif + this->svc_.state_->pool_.recycle(static_cast(this)); + } + std::coroutine_handle<> accept( std::coroutine_handle<>, capy::executor_ref, diff --git a/include/boost/corosio/native/detail/reactor/reactor_socket_service.hpp b/include/boost/corosio/native/detail/reactor/reactor_socket_service.hpp index 292254075..fa0790b21 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_socket_service.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_socket_service.hpp @@ -11,12 +11,12 @@ #define BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_SOCKET_SERVICE_HPP #include +#include #include #include #include #include -#include namespace boost::corosio::detail { @@ -36,6 +36,15 @@ template class reactor_socket_service : public ServiceBase { friend Derived; + + // The intermediate CRTP templates below (not Impl itself) are what + // actually reach into state_->pool_ from retire() -- see + // reactor_socket_finals.hpp. + template + friend class reactor_stream_socket_impl; + template + friend class reactor_dgram_socket_impl; + using state_type = reactor_service_state; protected: @@ -52,33 +61,23 @@ class reactor_socket_service : public ServiceBase void shutdown() override { - std::lock_guard lock(state_->mutex_); - - while (auto* impl = state_->impl_list_.pop_front()) - { - static_cast(this)->pre_shutdown(impl); - impl->close_socket(); - } - - // Don't clear impl_ptrs_ here. The scheduler shuts down after us - // and drains completed_ops_, calling destroy() on each queued op. - // Letting ~state_ release the ptrs (during service destruction, - // after scheduler shutdown) keeps every impl alive until all ops - // have been drained. + state_->pool_.shutdown( + [this](Impl* impl) + { + static_cast(this)->pre_shutdown(impl); + impl->close_socket(); + }); + + // Queued ops still hold references (their own object_ref copy); + // the scheduler shuts down after us and drains completed_ops_, + // each op's destroy() releasing its reference. Shutting-down + // mode makes that final release() delete instead of recycle, + // so every impl stays alive until its ops are gone, then frees. } io_object::implementation* construct() override { - auto impl = std::make_shared(static_cast(*this)); - auto* raw = impl.get(); - - { - std::lock_guard lock(state_->mutex_); - state_->impl_ptrs_.emplace(raw, std::move(impl)); - state_->impl_list_.push_back(raw); - } - - return raw; + return state_->pool_.acquire(static_cast(*this)); } void destroy(io_object::implementation* impl) override @@ -86,9 +85,7 @@ class reactor_socket_service : public ServiceBase auto* typed = static_cast(impl); static_cast(this)->pre_destroy(typed); typed->close_socket(); - std::lock_guard lock(state_->mutex_); - state_->impl_list_.remove(typed); - state_->impl_ptrs_.erase(typed); + release(typed); } void close(io_object::handle& h) override diff --git a/include/boost/corosio/native/detail/reactor/reactor_stream_socket.hpp b/include/boost/corosio/native/detail/reactor/reactor_stream_socket.hpp index e010b2e34..e350c959f 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_stream_socket.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_stream_socket.hpp @@ -13,12 +13,14 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include @@ -282,6 +284,41 @@ class reactor_stream_socket return fd; } + /** Reset op slots and the cached peer for recycling. + + Each op's own `reset()` runs again at its next `do_*` call + before any field is read, so only `remote_endpoint_` needs an + active reset here; `stop_cb` is checked instead of reset — a + still-armed callback would mean a stop_token outlived the op's + completion, which `reset()` always disengages, so asserting it + is what actually catches a missed completion rather than + papering over it. + + @pre refs_ == 0, fd closed and deregistered, no op in flight. + */ + void reuse() noexcept + { + base_type::reuse(); + remote_endpoint_ = Endpoint{}; + BOOST_COROSIO_ASSERT(!conn_.stop_cb); + BOOST_COROSIO_ASSERT(!rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wr_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_wr_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_er_.stop_cb); + } + +#if !defined(NDEBUG) + /// Extend base_type::poison() to the cached peer endpoint. + void poison() noexcept + { + base_type::poison(); + std::memset( + static_cast(&remote_endpoint_), 0xDB, + sizeof(remote_endpoint_)); + } +#endif + private: // CRTP callbacks for reactor_basic_socket cancel/close @@ -389,7 +426,7 @@ reactor_stream_socket< op.fd = this->fd_; op.target_endpoint = ep; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(err, 0); this->svc_.post(&op); return std::noop_coroutine(); @@ -403,7 +440,7 @@ reactor_stream_socket< op.fd = this->fd_; op.target_endpoint = ep; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->register_op( op, this->desc_state_.connect_op, this->desc_state_.write_ready, true); @@ -451,7 +488,7 @@ reactor_stream_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(EBADF, 0); this->svc_.post(&op); return std::noop_coroutine(); @@ -468,7 +505,7 @@ reactor_stream_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(0, 0); this->svc_.post(&op); return std::noop_coroutine(); @@ -522,7 +559,7 @@ reactor_stream_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(err, bytes); this->svc_.post(&op); return std::noop_coroutine(); @@ -535,7 +572,7 @@ reactor_stream_socket< op.bytes_out = bytes_out; op.fd = this->fd_; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->register_op( op, this->desc_state_.read_op, this->desc_state_.read_ready); @@ -583,7 +620,7 @@ reactor_stream_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(EBADF, 0); this->svc_.post(&op); return std::noop_coroutine(); @@ -600,7 +637,7 @@ reactor_stream_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(0, 0); this->svc_.post(&op); return std::noop_coroutine(); @@ -643,7 +680,7 @@ reactor_stream_socket< op.ec_out = ec; op.bytes_out = bytes_out; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(err, bytes); this->svc_.post(&op); return std::noop_coroutine(); @@ -656,7 +693,7 @@ reactor_stream_socket< op.bytes_out = bytes_out; op.fd = this->fd_; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); this->register_op( op, this->desc_state_.write_op, this->desc_state_.write_ready, true); @@ -737,7 +774,7 @@ reactor_stream_socket< op.ec_out = ec; op.fd = this->fd_; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); op.complete(perr, 0); this->svc_.post(&op); return std::noop_coroutine(); @@ -750,7 +787,7 @@ reactor_stream_socket< op.ec_out = ec; op.fd = this->fd_; op.start(token, static_cast(this)); - op.impl_ptr = this->shared_from_this(); + op.object_ref_ = detail::object_ref(this); // Force register_op's ready path so the wait op re-probes under // the descriptor mutex before parking. An edge consumed between diff --git a/include/boost/corosio/native/detail/select/select_scheduler.hpp b/include/boost/corosio/native/detail/select/select_scheduler.hpp index d1a8731d3..83363d638 100644 --- a/include/boost/corosio/native/detail/select/select_scheduler.hpp +++ b/include/boost/corosio/native/detail/select/select_scheduler.hpp @@ -225,7 +225,7 @@ select_scheduler::register_descriptor( { conditionally_enabled_mutex::scoped_lock lock(desc->mutex); - desc->impl_ref_.reset(); + desc->object_ref_.reset(); desc->read_ready = false; desc->write_ready = false; } diff --git a/include/boost/corosio/native/detail/speculative_state.hpp b/include/boost/corosio/native/detail/speculative_state.hpp index c29b8253b..9dbdd48bb 100644 --- a/include/boost/corosio/native/detail/speculative_state.hpp +++ b/include/boost/corosio/native/detail/speculative_state.hpp @@ -109,6 +109,20 @@ class speculative_state { try_write_.store(true, std::memory_order_relaxed); } + + /** Restore the default, fully-speculative state. + + The learned hint is scoped to one logical connection; a recycled + socket starts a new one and should not inherit the prior + connection's streak or perma-off latch. + */ + void reset() noexcept + { + try_read_.store(true, std::memory_order_relaxed); + try_write_.store(true, std::memory_order_relaxed); + read_eagain_streak_.store(0, std::memory_order_relaxed); + perma_off_read_.store(false, std::memory_order_relaxed); + } }; } // namespace boost::corosio::detail diff --git a/include/boost/corosio/native/detail/uring/uring_acceptor_ops.hpp b/include/boost/corosio/native/detail/uring/uring_acceptor_ops.hpp index 3473c2096..76cd42f0c 100644 --- a/include/boost/corosio/native/detail/uring/uring_acceptor_ops.hpp +++ b/include/boost/corosio/native/detail/uring/uring_acceptor_ops.hpp @@ -154,8 +154,23 @@ struct uring_accept_op : uring_op sockaddr_storage const& peer, socklen_t peer_len) noexcept = nullptr; + /** Disposal hook, run exactly once when the op is consumed. + + Heap-allocated ops delete themselves (the default); the + multishot acceptor's pooled nodes override this to return to + their acceptor's free list instead. Factoring disposal out of + `do_handler` is what lets one completion routine serve both + ownership models. + */ + void (*dispose)(uring_accept_op*) noexcept = &dispose_delete; + uring_accept_op() noexcept : uring_op(&do_handler, &do_cqe) {} + static void dispose_delete(uring_accept_op* op) noexcept + { + delete op; + } + // LCOV_EXCL_START: never receives a CQE; present for vtable // completeness. static void do_cqe(uring_op*, int, unsigned, ready_queue&) noexcept {} @@ -172,7 +187,7 @@ struct uring_accept_op : uring_op if (owner == nullptr) { - delete self; + self->dispose(self); return; } @@ -184,10 +199,7 @@ struct uring_accept_op : uring_op *self->ec_out = was_cancelled ? std::error_code(capy::error::canceled) : make_err(self->err); - self->cont.h = self->h; - auto next = dispatch_coro(self->ex, self->cont); - delete self; - next.resume(); + resume_and_dispose(self); return; } @@ -205,9 +217,21 @@ struct uring_accept_op : uring_op if (self->ec_out) *self->ec_out = {}; + resume_and_dispose(self); + } + + /** Dispose the op, then resume its awaiter. + + dispatch_coro does not retain the op's embedded continuation + (a deferring executor gets a frame-owned one), so disposing + before the resume is safe: a concurrent accept may refill the + recycled node immediately without touching anything queued. + */ + static void resume_and_dispose(uring_accept_op* self) + { self->cont.h = self->h; auto next = dispatch_coro(self->ex, self->cont); - delete self; + self->dispose(self); next.resume(); } }; diff --git a/include/boost/corosio/native/detail/uring/uring_dgram_ops.hpp b/include/boost/corosio/native/detail/uring/uring_dgram_ops.hpp index b4338e068..f4cdeaebd 100644 --- a/include/boost/corosio/native/detail/uring/uring_dgram_ops.hpp +++ b/include/boost/corosio/native/detail/uring/uring_dgram_ops.hpp @@ -69,7 +69,7 @@ struct uring_dgram_send_op : uring_op std::size_t* bytes, int file_descriptor, uring_scheduler* scheduler, - std::shared_ptr impl, + detail::object_ref impl, detail::speculative_state* spec, buffer_param buffers, socklen_t dest_addr_len, @@ -77,17 +77,17 @@ struct uring_dgram_send_op : uring_op int flags, std::stop_token const& token) noexcept { - h = handle; - ex = executor; - ec_out = ec; - bytes_out = bytes; - fd = file_descriptor; - sched_ = scheduler; - impl_ptr = std::move(impl); - spec_state = spec; - res = 0; - cqe_flags = 0; - msg_flags = flags; + h = handle; + ex = executor; + ec_out = ec; + bytes_out = bytes; + fd = file_descriptor; + sched_ = scheduler; + object_ref_ = std::move(impl); + spec_state = spec; + res = 0; + cqe_flags = 0; + msg_flags = flags; iovec_count = copy_to_iovec(buffers, iovecs); @@ -207,7 +207,7 @@ struct uring_dgram_recv_op : uring_op std::size_t* bytes, int file_descriptor, uring_scheduler* scheduler, - std::shared_ptr impl, + detail::object_ref impl, detail::speculative_state* spec, buffer_param buffers, void* source_ctx, @@ -215,17 +215,17 @@ struct uring_dgram_recv_op : uring_op int flags, std::stop_token const& token) noexcept { - h = handle; - ex = executor; - ec_out = ec; - bytes_out = bytes; - fd = file_descriptor; - sched_ = scheduler; - impl_ptr = std::move(impl); - spec_state = spec; - res = 0; - cqe_flags = 0; - msg_flags = flags; + h = handle; + ex = executor; + ec_out = ec; + bytes_out = bytes; + fd = file_descriptor; + sched_ = scheduler; + object_ref_ = std::move(impl); + spec_state = spec; + res = 0; + cqe_flags = 0; + msg_flags = flags; iovec_count = copy_to_iovec(buffers, iovecs); diff --git a/include/boost/corosio/native/detail/uring/uring_file_ops.hpp b/include/boost/corosio/native/detail/uring/uring_file_ops.hpp index e151a3ad5..6dbf9a2dc 100644 --- a/include/boost/corosio/native/detail/uring/uring_file_ops.hpp +++ b/include/boost/corosio/native/detail/uring/uring_file_ops.hpp @@ -71,7 +71,7 @@ struct uring_file_read_op_base : uring_op int file_descriptor, std::int64_t file_offset, uring_scheduler* scheduler, - std::shared_ptr impl, + detail::object_ref impl, buffer_param buffers, std::stop_token const& token) noexcept { @@ -82,7 +82,7 @@ struct uring_file_read_op_base : uring_op fd = file_descriptor; offset = file_offset; sched_ = scheduler; - impl_ptr = std::move(impl); + object_ref_ = std::move(impl); res = 0; cqe_flags = 0; iovec_count = copy_to_iovec(buffers, iovecs); @@ -217,7 +217,7 @@ struct uring_file_write_op_base : uring_op int file_descriptor, std::int64_t file_offset, uring_scheduler* scheduler, - std::shared_ptr impl, + detail::object_ref impl, buffer_param buffers, std::stop_token const& token) noexcept { @@ -228,7 +228,7 @@ struct uring_file_write_op_base : uring_op fd = file_descriptor; offset = file_offset; sched_ = scheduler; - impl_ptr = std::move(impl); + object_ref_ = std::move(impl); res = 0; cqe_flags = 0; iovec_count = copy_to_iovec(buffers, iovecs); diff --git a/include/boost/corosio/native/detail/uring/uring_file_service_base.hpp b/include/boost/corosio/native/detail/uring/uring_file_service_base.hpp index 2d9629839..0352e1e4e 100644 --- a/include/boost/corosio/native/detail/uring/uring_file_service_base.hpp +++ b/include/boost/corosio/native/detail/uring/uring_file_service_base.hpp @@ -14,33 +14,34 @@ #if BOOST_COROSIO_HAS_URING -#include +#include +#include #include #include -#include -#include -#include +#include /* Shared lifecycle plumbing for io_uring file services. uring_stream_file_service and uring_random_access_file_service were byte-for-byte identical apart from the impl type and open_file's parameter - type: both make_shared the file impl from the scheduler, track it in an - intrusive list + raw->shared_ptr map, and close every file on shutdown. - This base factors that out; the concrete services add only open_file. + type: both pop-or-new the file impl from a per-service recycling pool + and close every file on shutdown. This base factors that out; the + concrete services add only open_file. This is a separate base from uring_socket_service_base because file - services differ from socket services in three ways that match the reactor - socket service instead: they track via an intrusive list + map (sockets: - map only), they CLOSE files on shutdown (sockets: cancel only), and the - impl ctor takes just the scheduler (sockets: service + scheduler). See - tasks/proactor-dedup-decisions.md (#14). - - Requirements on File: derive from enable_shared_from_this and - intrusive_list::node, a `File(uring_scheduler&)` constructor, and - a `void close_file() noexcept` method (cancel in-flight ops + close fd). + services differ from socket services in two ways that match the reactor + socket service instead: they CLOSE files on shutdown (sockets: cancel + only), and the impl ctor takes just the scheduler (sockets: service + + scheduler). See tasks/proactor-dedup-decisions.md (#14). + + Requirements on File: derive from intrusive_list::node, a + `File(Derived&, uring_scheduler&)` constructor, a + `void close_file() noexcept` method (cancel in-flight ops + close fd), + a `void reuse() noexcept` method, and a `retire()` override that + recycles through the owning service's private `pool_` member + (reachable because `File` is a friend). @tparam Derived The concrete service (CRTP; unused today but kept for symmetry / future hooks). @@ -55,6 +56,7 @@ template class uring_file_service_base : public ServiceBase { friend Derived; + friend File; // Private CRTP ctor: only `Derived` (the concrete service, a friend) // constructs the base — prevents inheriting with the wrong Derived @@ -69,14 +71,7 @@ class uring_file_service_base : public ServiceBase io_object::implementation* construct() override { - auto ptr = std::make_shared(*sched_); - auto* impl = ptr.get(); - { - std::lock_guard lock(mutex_); - file_list_.push_back(impl); - file_ptrs_[impl] = std::move(ptr); - } - return impl; + return pool_.acquire(static_cast(*this), *sched_); } void destroy(io_object::implementation* p) override @@ -84,9 +79,7 @@ class uring_file_service_base : public ServiceBase // close_file() already does cancel_and_flush(fd_) before ::close. auto& impl = static_cast(*p); impl.close_file(); - std::lock_guard lock(mutex_); - file_list_.remove(&impl); - file_ptrs_.erase(&impl); + release(&impl); } void close(io_object::handle& h) override @@ -97,13 +90,23 @@ class uring_file_service_base : public ServiceBase void shutdown() override { - std::lock_guard lock(mutex_); - for (auto* impl = file_list_.pop_front(); impl != nullptr; - impl = file_list_.pop_front()) + // See uring_socket_service_base::shutdown(): snapshot under an + // acquired reference, close without the pool lock held; + // shutdown() sets shutting-down and takes the snapshot in one + // critical section, so a close that drops the last ref deletes + // rather than recycles. + std::vector live; + pool_.shutdown( + [&](File* f) + { + acquire(f); + live.push_back(f); + }); + for (auto* f : live) { - impl->close_file(); + f->close_file(); + release(f); } - file_ptrs_.clear(); } /// Return the scheduler used by files created by this service. @@ -114,9 +117,7 @@ class uring_file_service_base : public ServiceBase protected: uring_scheduler* sched_; - std::mutex mutex_; - intrusive_list file_list_; - std::unordered_map> file_ptrs_; + object_pool pool_; private: uring_file_service_base(uring_file_service_base const&) = delete; diff --git a/include/boost/corosio/native/detail/uring/uring_multishot_acceptor.hpp b/include/boost/corosio/native/detail/uring/uring_multishot_acceptor.hpp index 4ded4e8e9..1e40c6f8f 100644 --- a/include/boost/corosio/native/detail/uring/uring_multishot_acceptor.hpp +++ b/include/boost/corosio/native/detail/uring/uring_multishot_acceptor.hpp @@ -11,6 +11,7 @@ #define BOOST_COROSIO_NATIVE_DETAIL_URING_URING_MULTISHOT_ACCEPTOR_HPP #include +#include #include #if BOOST_COROSIO_HAS_URING @@ -32,7 +33,6 @@ #include #include #include -#include #include #include @@ -62,10 +62,15 @@ fd_is_listening(int fd) noexcept return accepting != 0; } -template +template< + class Derived, + class ImplBase, + class Endpoint, + class PeerService, + class AcceptorService> class uring_multishot_acceptor_base : public ImplBase - , public std::enable_shared_from_this + , public intrusive_list::node { protected: struct ready_fd_node : intrusive_list::node @@ -75,22 +80,34 @@ class uring_multishot_acceptor_base socklen_t peer_len = 0; }; - struct waiter_node; - - struct waiter_canceller - { - waiter_node* w; - void operator()() const noexcept; - }; - - struct waiter_node : intrusive_list::node + /** Pooled accept node: parked waiter and posted completion in one. + + A parked accept used to be a `waiter_node` whose fields were + copied into a freshly allocated `uring_accept_op` at every + completion boundary. The two had strictly sequential lifetimes + (a waiter *becomes* a completion), so this node is both at + once — the intrusive hook parks it in `waiters_`/`free_nodes_` + and the inherited op posts it to the scheduler. Nodes recycle + through the owning acceptor's `free_nodes_` via `dispose`, so + steady-state accepts allocate nothing. + + The inherited `cancelled` flag is the claim token: delivery, + arming-failure, and cancellation each claim the node with one + `exchange`, and the loser leaves completion to the winner. + Because the flag also decides how `do_handler` reports the + completion, every claiming path stores the final status after + `stop_cb.reset()` (which synchronizes with any in-flight + canceller) and before posting. + + While in flight — parked or posted — the node holds an + `object_ref` on its acceptor (the inherited `object_ref_` + slot), so the impl cannot retire out from under `dispose`. + */ + struct accept_node + : uring_accept_op + , intrusive_list::node { - std::coroutine_handle<> h; - capy::executor_ref ex; - std::error_code* ec_out = nullptr; - io_object::implementation** impl_out = nullptr; - Derived* owner = nullptr; - std::atomic cancelled{false}; + Derived* owner = nullptr; /// True once linked into `waiters_` (guarded by `mutex_`). /// The stop callback is armed before the node is queued, so /// cancel_waiter must not unlink a node it never queued. @@ -98,21 +115,50 @@ class uring_multishot_acceptor_base /// A readiness wait rather than an accept: completion /// observes a pending connection without consuming it. bool peek = false; - std::optional> stop_cb; + + accept_node() noexcept + { + this->dispose = &recycle_thunk; + } + + /// Claim the node for cancellation (the stop token fired); + /// losing the exchange means a delivery already owns it. + void on_cancel() noexcept override + { + if (this->cancelled.exchange(true, std::memory_order_acq_rel)) + return; + owner->cancel_waiter(this); + } + + static void recycle_thunk(uring_accept_op* op) noexcept + { + auto* n = static_cast(op); + n->owner->release_node(n); + } }; int fd_ = -1; uring_scheduler* sched_; PeerService* peer_service_; + /// Owning acceptor service; exposes the recycling pool to + /// retire(). Not used for anything else — the service already + /// drives shutdown/destroy from its own side. + AcceptorService* acceptor_svc_; Endpoint local_endpoint_{}; mutable std::mutex mutex_; intrusive_list ready_fds_; - intrusive_list waiters_; + intrusive_list waiters_; + /// Recycled accept nodes and ready-fd nodes (guarded by `mutex_`). + /// Both survive impl recycling — the storage belongs to this impl + /// and only the destructor frees it — so steady-state accepts + /// allocate nothing. + intrusive_list free_nodes_; + intrusive_list free_ready_; /// Single parked readiness wait (guarded by `mutex_`). Multishot /// accepting drains the kernel queue instantly, so a listener's /// readiness lives in `ready_fds_`, not in `poll()`; the wait is /// completed by the next delivery instead of a kernel poll. - waiter_node* read_wait_ = nullptr; + accept_node* read_wait_ = nullptr; std::unique_ptr multi_op_; bool closing_ = false; /// Non-zero once an arming failed to reach the kernel (guarded by @@ -136,49 +182,186 @@ class uring_multishot_acceptor_base // (clang-tidy bugprone-crtp-constructor-accessibility). friend Derived; uring_multishot_acceptor_base( - uring_scheduler& sched, PeerService& peer_svc) noexcept + AcceptorService& acceptor_svc, + uring_scheduler& sched, + PeerService& peer_svc) noexcept : sched_(&sched) , peer_service_(&peer_svc) + , acceptor_svc_(&acceptor_svc) { } -public: - ~uring_multishot_acceptor_base() override +protected: + /** Pop a recycled accept node, or allocate on the cold path. + + Resets every per-use field and arms the `object_ref` keepalive + on this acceptor; the caller fills the request fields and + either parks or posts the node. OOM on the cold path + terminates, matching every other noexcept initiation path. + */ + accept_node* acquire_node() noexcept { + accept_node* n; { std::lock_guard lk(mutex_); - closing_ = true; + n = free_nodes_.pop_front(); } - if (fd_ >= 0) + if (!n) { - sched_->submit_cancel_by_fd(fd_); - ::close(fd_); - fd_ = -1; + // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept initiation path: OOM => std::terminate is the intended behavior + n = new accept_node(); } + n->owner = static_cast(this); + n->queued = false; + n->peek = false; + n->err = 0; + n->accepted_fd = -1; + n->peer_len = 0; + n->ec_out = nullptr; + n->impl_out = nullptr; + n->peer_endpoint_out = nullptr; + n->peer_service = nullptr; + n->adopt_fn = nullptr; + n->cancelled.store(false, std::memory_order_relaxed); + n->object_ref_ = detail::object_ref(this); + return n; + } + + /** Return a consumed node to the free list; `dispose` target. + + The keepalive is moved out first and dropped after the lock, + so a final release (which may reenter this impl's retire()) + never runs under `mutex_`. + */ + void release_node(accept_node* n) noexcept + { + auto keep = std::move(n->object_ref_); + { + std::lock_guard lk(mutex_); + free_nodes_.push_front(n); + } + } + + /// Pop a recycled ready-fd node, or allocate on the cold path. + /// @pre `mutex_` is held. + ready_fd_node* acquire_ready_locked() noexcept + { + if (auto* r = free_ready_.pop_front()) + { + r->fd = -1; + r->peer_len = 0; + return r; + } + // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — CQE handler: noexcept, OOM => std::terminate is the intended behavior + return new ready_fd_node{}; + } - // Drain parked accepted-connection fds unconditionally. These are - // distinct from the listener fd and can be present even when the - // service close() path already closed and cleared fd_ — that path - // does not touch ready_fds_, so the drain must run here. +public: + /** Close and free every fd parked in `ready_fds_`. + + A multishot CQE can deliver a connection before any `accept()` + is outstanding to claim it; that connection's fd (and tracking + node) sits in `ready_fds_` until a later `accept()` consumes it + or the acceptor goes away. Idempotent: safe to call from both + `retire()` (recycling) and the destructor (pool-force-sweep + backstop) — the second call always sees an empty list. + */ + void drain_ready_fds() noexcept + { intrusive_list drained; { std::lock_guard lk(mutex_); while (auto* r = ready_fds_.pop_front()) drained.push_back(r); } + if (drained.empty()) + return; + intrusive_list recycled; while (auto* r = drained.pop_front()) { ::close(r->fd); - delete r; + recycled.push_back(r); } + std::lock_guard lk(mutex_); + while (auto* r = recycled.pop_front()) + free_ready_.push_front(r); + } - // Break the multi_op_ → impl_ptr (shared_ptr) cycle and + /** Recycle into the owning service's pool at zero references. + + Reaching zero used to mean the shared_ptr destructor was about + to run, which is where the ready_fds_ drain and the multi_op_ + cycle-break + CQE drain below used to live (the destructor + still runs them too, as a backstop for the pool's own + unconditional force-sweep at shutdown — both are idempotent). + Now reaching zero can instead mean this impl is about to go + back on the free list for reuse — its memory is never freed, + so the destructor never runs. Do the drain here, once, for + both outcomes: `close()` already reset `multi_op_->object_ref_` on + the normal path, so that reset is usually a no-op; it only + does real work when zero is reached some other way. Skipping + the ready_fds_ drain here specifically would leak every fd and + node still parked in it and trip `reuse()`'s `ready_fds_.empty()` + assert on the very next construct() that pops this impl back + out. + + `multi_op_` itself is freed here too, rather than kept for the + next session: `prepare_listen_arm()` treats a non-null + `multi_op_` with `closing_` false and `arm_err_` zero as + already armed and skips re-arming — true across a re-listen + on the same still-open acceptor, but not across reuse, where + the next `listen()` is a different logical acceptor on a + different fd. Leaving the old `multi_op_` in place would make + that listen() silently believe it was already armed and never + submit a multishot SQE for the new fd. + */ + void retire() noexcept override + { + drain_ready_fds(); + if (multi_op_) + { + multi_op_->object_ref_.reset(); + sched_->drain_cqes_for(multi_op_.get()); + multi_op_.reset(); + } + acceptor_svc_->pool_.recycle(static_cast(this)); + } + + ~uring_multishot_acceptor_base() override + { + { + std::lock_guard lk(mutex_); + closing_ = true; + } + if (fd_ >= 0) + { + sched_->submit_cancel_by_fd(fd_); + ::close(fd_); + fd_ = -1; + } + + // Backstop: retire() already drains this on the normal + // recycle path. This call only does real work when this impl + // is reached by the pool's unconditional force-sweep at + // shutdown without ever going through retire() first. + drain_ready_fds(); + + // Break the multi_op_ → object_ref_ (object_ref) cycle and // drain pending CQEs so unique_ptr can free safely. if (multi_op_) { - multi_op_->impl_ptr.reset(); + multi_op_->object_ref_.reset(); sched_->drain_cqes_for(multi_op_.get()); } + + // The recycled-node storage belongs to this impl and is only + // freed here; in-flight nodes hold an object_ref on this impl, + // so none can still be parked or posted once the count reached + // zero (or the pool's force-sweep ran after scheduler drain). + while (auto* n = free_nodes_.pop_front()) + delete n; + while (auto* r = free_ready_.pop_front()) + delete r; } Endpoint local_endpoint() const noexcept override @@ -204,7 +387,7 @@ class uring_multishot_acceptor_base native_handle_type release_socket() noexcept override { // Mirror the service close() path: cancel the multishot SQE and - // break the multi_op_ -> impl_ptr (shared_ptr) cycle that + // break the multi_op_ -> object_ref_ (object_ref) cycle that // start_multishot established. Without this, the cycle keeps the // acceptor and its multi_op_ alive after the caller takes the fd, // which LeakSanitizer reports on process exit. Caller still owns @@ -214,7 +397,7 @@ class uring_multishot_acceptor_base sched_->cancel_and_flush(fd_); drain_waiters_only(); if (multi_op_) - multi_op_->impl_ptr.reset(); + multi_op_->object_ref_.reset(); } int fd = fd_; fd_ = -1; @@ -229,13 +412,42 @@ class uring_multishot_acceptor_base sched_->submit_cancel_by_fd(fd_); } + /** Reset state for recycling. + + `close()`/`release_socket()` already drove fd_, local_endpoint_, + and the waiter/ready-fd queues to empty before the refcount + reached zero, so those are asserted rather than re-cleared. + `closing_` and `arm_err_` are the two fields that are NOT reset + by that close path (closing_ is only ever cleared by a re-listen + on a still-live acceptor, not by close) and must be explicitly + reset here or the next session would see this impl as still + shutting down. `multi_op_` is freed by `retire()` rather + than kept, so a fresh one is allocated the next time this impl + starts a multishot arming — see its comment for why keeping it + across reuse would leave the new session un-armed. + + @pre refs_ == 0, fd closed, no op or waiter in flight, + multi_op_ already freed. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(fd_ == -1); + BOOST_COROSIO_ASSERT(local_endpoint_ == Endpoint{}); + BOOST_COROSIO_ASSERT(ready_fds_.empty()); + BOOST_COROSIO_ASSERT(waiters_.empty()); + BOOST_COROSIO_ASSERT(read_wait_ == nullptr); + BOOST_COROSIO_ASSERT(!multi_op_); + closing_ = false; + arm_err_ = 0; + } + /// Drain queued waiters with operation_aborted but do NOT submit /// any kernel cancel for the fd. Used by service close() paths /// that have already submitted (or are about to submit) the /// cancel-by-fd themselves via `cancel_and_flush`. void drain_waiters_only() noexcept { - intrusive_list drained; + intrusive_list drained; { std::lock_guard lk(mutex_); closing_ = true; @@ -253,16 +465,11 @@ class uring_multishot_acceptor_base while (auto* w = drained.pop_front()) { + // reset() synchronizes with an in-flight canceller; after + // it the stored status is ours to set. w->stop_cb.reset(); - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept destructor path: OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); - op->h = w->h; - op->ex = w->ex; - op->ec_out = w->ec_out; - op->impl_out = w->impl_out; - op->cancelled.store(true, std::memory_order_release); - delete w; - sched_->post(op); + w->cancelled.store(true, std::memory_order_release); + sched_->post(w); sched_->work_finished(); } } @@ -280,6 +487,12 @@ class uring_multishot_acceptor_base std::stop_token const& token, std::error_code* ec) noexcept { + auto* w = acquire_node(); + w->h = h; + w->ex = ex; + w->ec_out = ec; + w->peek = true; + bool ready = false; bool aborted = false; { @@ -295,31 +508,17 @@ class uring_multishot_acceptor_base } if (ready || aborted) { - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept initiation path: OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); - op->h = h; - op->ex = ex; - op->ec_out = ec; if (aborted) - op->cancelled.store(true, std::memory_order_release); - sched_->post(op); + w->cancelled.store(true, std::memory_order_release); + sched_->post(w); return; } - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept initiation path: OOM => std::terminate is the intended behavior - auto* w = new waiter_node{}; - w->h = h; - w->ex = ex; - w->ec_out = ec; - w->owner = static_cast(this); - w->peek = true; - // Same protocol as accept parking: arm the callback before // the node is visible and outside `mutex_` (a pre-stopped // token invokes the canceller synchronously, and the // canceller takes `mutex_`). - if (token.stop_possible()) - w->stop_cb.emplace(token, waiter_canceller{w}); + w->start(token); bool was_cancelled = false; int arm_err = 0; @@ -348,16 +547,9 @@ class uring_multishot_acceptor_base } w->stop_cb.reset(); - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept initiation path: OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); - op->h = w->h; - op->ex = w->ex; - op->ec_out = w->ec_out; - op->err = arm_err; - if (was_cancelled) - op->cancelled.store(true, std::memory_order_release); - delete w; - sched_->post(op); + w->err = arm_err; + w->cancelled.store(was_cancelled, std::memory_order_release); + sched_->post(w); } std::error_code set_option( @@ -422,7 +614,7 @@ class uring_multishot_acceptor_base sched_->retire_op( multi_op_, [this](uring_multi_accept_op& op) noexcept { std::lock_guard lk(mutex_); - op.impl_ptr.reset(); + op.object_ref_.reset(); op.acceptor_impl = nullptr; op.on_cqe = nullptr; op.retire_func = &uring_multi_accept_op::do_retired_cqe; @@ -523,13 +715,13 @@ class uring_multishot_acceptor_base multi_op_->listen_fd = fd_; multi_op_->acceptor_impl = this; multi_op_->on_cqe = &uring_multishot_acceptor_base::on_accept_cqe; - multi_op_->impl_ptr = this->shared_from_this(); + multi_op_->object_ref_ = detail::object_ref(this); } else { // Reuse the existing op (re-arm path). Reset peer scratch // so the kernel writes into a clean slot. listen_fd and - // impl_ptr are re-seeded so the op can never carry state + // object_ref_ are re-seeded so the op can never carry state // from an arming that has since been torn down. `res` is // one of those: an arming that failed to submit left // -EAGAIN there, and the reader of `res` cannot tell a @@ -538,7 +730,7 @@ class uring_multishot_acceptor_base multi_op_->peer_len = sizeof(sockaddr_storage); multi_op_->res = 0; multi_op_->listen_fd = fd_; - multi_op_->impl_ptr = this->shared_from_this(); + multi_op_->object_ref_ = detail::object_ref(this); } auto* op = multi_op_.get(); @@ -569,7 +761,7 @@ class uring_multishot_acceptor_base */ void fail_arm(int err) noexcept { - intrusive_list claimed; + intrusive_list claimed; { std::lock_guard lk(mutex_); arm_err_ = err; @@ -577,7 +769,7 @@ class uring_multishot_acceptor_base // canceller already claimed belongs to it: cancel_waiter // is waiting on this mutex to unlink the node itself, so // it has to still be in the list when it gets in. - intrusive_list keep; + intrusive_list keep; while (auto* w = waiters_.pop_front()) { if (!w->cancelled.exchange(true, std::memory_order_acq_rel)) @@ -598,16 +790,13 @@ class uring_multishot_acceptor_base while (auto* w = claimed.pop_front()) { + // The exchange above was a claim, not the completion + // status: after reset() quiesces the canceller, restore + // the flag so the handler reports `err`, not canceled. w->stop_cb.reset(); - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept arming path: OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); - op->h = w->h; - op->ex = w->ex; - op->ec_out = w->ec_out; - op->impl_out = w->impl_out; - op->err = err; - delete w; - sched_->post(op); + w->err = err; + w->cancelled.store(false, std::memory_order_release); + sched_->post(w); sched_->work_finished(); // balance the waiter's work_started } } @@ -622,6 +811,12 @@ class uring_multishot_acceptor_base std::error_code* ec, io_object::implementation** impl_out) { + auto* w = acquire_node(); + w->h = h; + w->ex = ex; + w->ec_out = ec; + w->impl_out = impl_out; + sockaddr_storage peer_storage{}; socklen_t peer_len = sizeof(peer_storage); int accepted_fd = ::accept4( @@ -629,78 +824,55 @@ class uring_multishot_acceptor_base SOCK_NONBLOCK | SOCK_CLOEXEC); if (accepted_fd >= 0) { - auto* op = new uring_accept_op(); - op->h = h; - op->ex = ex; - op->ec_out = ec; - op->impl_out = impl_out; - op->peer_service = peer_service_; - op->adopt_fn = &Derived::adopt_thunk; - op->accepted_fd = accepted_fd; - op->peer_storage = peer_storage; - op->peer_len = peer_len; - sched_->post(op); + w->peer_service = peer_service_; + w->adopt_fn = &Derived::adopt_thunk; + w->accepted_fd = accepted_fd; + w->peer_storage = peer_storage; + w->peer_len = peer_len; + sched_->post(w); return; } // accept4 returned <0 — only EAGAIN/EWOULDBLOCK should fall // through to the parked/waiter path. Other errors (EBADF, etc.) // surface through the existing scheduler-completion path so the - // user sees them via the op's ec_out. Build an op with `err` - // set so do_handler delivers make_err(err). + // user sees them via the node's ec_out: `err` set means + // do_handler delivers make_err(err). if (errno != EAGAIN && errno != EWOULDBLOCK) { - int saved_errno = errno; - auto* op = new uring_accept_op(); - op->h = h; - op->ex = ex; - op->ec_out = ec; - op->impl_out = impl_out; - op->err = saved_errno; - sched_->post(op); + w->err = errno; + sched_->post(w); return; } - uring_accept_op* ready_op = nullptr; + bool have_ready = false; { std::lock_guard lk(mutex_); if (auto* r = ready_fds_.pop_front()) { - ready_op = new uring_accept_op(); - ready_op->h = h; - ready_op->ex = ex; - ready_op->ec_out = ec; - ready_op->impl_out = impl_out; - ready_op->peer_service = peer_service_; - ready_op->adopt_fn = &Derived::adopt_thunk; - ready_op->accepted_fd = r->fd; - ready_op->peer_storage = r->peer; - ready_op->peer_len = r->peer_len; - delete r; + w->peer_service = peer_service_; + w->adopt_fn = &Derived::adopt_thunk; + w->accepted_fd = r->fd; + w->peer_storage = r->peer; + w->peer_len = r->peer_len; + free_ready_.push_front(r); + have_ready = true; } } - if (ready_op) + if (have_ready) { // Post outside the lock — acceptor mutex_ must never be // held while dispatch_mutex_ is acquired by sched_->post(). - sched_->post(ready_op); + sched_->post(w); return; } - auto* w = new waiter_node{}; - w->h = h; - w->ex = ex; - w->ec_out = ec; - w->impl_out = impl_out; - w->owner = static_cast(this); - // Arm the stop callback before the node is visible in // `waiters_` and outside `mutex_`: an already-stopped token // invokes the canceller synchronously from emplace, and // cancel_waiter takes `mutex_` (self-deadlock if held). // Arming pre-queue also keeps the CQE handler from claiming - // and deleting a node whose callback is not yet constructed. - if (token.stop_possible()) - w->stop_cb.emplace(token, waiter_canceller{w}); + // a node whose callback is not yet constructed. + w->start(token); bool was_cancelled = false; { @@ -715,29 +887,18 @@ class uring_multishot_acceptor_base { // A connection arrived while the callback was armed; // prefer it over parking the waiter behind it. - ready_op = new uring_accept_op(); - ready_op->h = h; - ready_op->ex = ex; - ready_op->ec_out = ec; - ready_op->impl_out = impl_out; - ready_op->peer_service = peer_service_; - ready_op->adopt_fn = &Derived::adopt_thunk; - ready_op->accepted_fd = r->fd; - ready_op->peer_storage = r->peer; - ready_op->peer_len = r->peer_len; - delete r; + w->peer_service = peer_service_; + w->adopt_fn = &Derived::adopt_thunk; + w->accepted_fd = r->fd; + w->peer_storage = r->peer; + w->peer_len = r->peer_len; + free_ready_.push_front(r); } else if (arm_err_ != 0) { // No arming reached the kernel, so no CQE will deliver // a connection: parking here would park for good. - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept accept path: OOM => std::terminate is the intended behavior - ready_op = new uring_accept_op(); - ready_op->h = h; - ready_op->ex = ex; - ready_op->ec_out = ec; - ready_op->impl_out = impl_out; - ready_op->err = arm_err_; + w->err = arm_err_; } else { @@ -748,34 +909,20 @@ class uring_multishot_acceptor_base } } - if (was_cancelled) - { - auto* op = new uring_accept_op(); - op->h = w->h; - op->ex = w->ex; - op->ec_out = w->ec_out; - op->impl_out = w->impl_out; - op->cancelled.store(true, std::memory_order_release); - w->stop_cb.reset(); - delete w; - sched_->post(op); - return; - } - w->stop_cb.reset(); - delete w; - sched_->post(ready_op); + w->cancelled.store(was_cancelled, std::memory_order_release); + sched_->post(w); } - void cancel_waiter(waiter_node* w) noexcept + void cancel_waiter(accept_node* w) noexcept { { std::lock_guard lk(mutex_); if (closing_) - return; // on_accept_cqe_impl will drain with closing_ set + return; // drain_waiters_only will complete with closing_ set if (!w->queued) return; // not queued yet; the parking path observes - // `cancelled` and completes the op + // `cancelled` and completes the node if (w->peek) { if (read_wait_ != w) @@ -787,17 +934,13 @@ class uring_multishot_acceptor_base waiters_.remove(w); } } - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — stop-token callback: noexcept, OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); - op->h = w->h; - op->ex = w->ex; - op->ec_out = w->ec_out; - op->impl_out = w->impl_out; - op->cancelled.store(true, std::memory_order_release); - delete w; - // post() increments outstanding_work_; balances the work_started() - // from accept() when the waiter was queued. - sched_->post(op); + // `cancelled` is already true (the claim that routed here); + // that is also the completion status, so post as-is. The node's + // stop_cb is still engaged — do_handler resets it, and running + // that reset from the handler thread while this callback + // returns is the stop_callback destructor's documented + // wait-or-self case. + sched_->post(w); sched_->work_finished(); // balance the work_started() from accept() } @@ -812,9 +955,9 @@ class uring_multishot_acceptor_base void on_accept_cqe_impl(int new_fd, int err, bool more) noexcept { bool was_closing = false; - waiter_node* matched = nullptr; - waiter_node* claimed_peek = nullptr; - intrusive_list closing_waiters; + accept_node* matched = nullptr; + accept_node* claimed_peek = nullptr; + intrusive_list closing_waiters; { std::lock_guard lk(mutex_); was_closing = closing_; @@ -855,8 +998,7 @@ class uring_multishot_acceptor_base } else if (new_fd >= 0) { - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — CQE handler: noexcept, OOM => std::terminate is the intended behavior - auto* node = new ready_fd_node{}; + auto* node = acquire_ready_locked(); node->fd = new_fd; node->peer = multi_op_->peer_storage; node->peer_len = multi_op_->peer_len; @@ -865,8 +1007,7 @@ class uring_multishot_acceptor_base } else if (new_fd >= 0) { - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — CQE handler: noexcept, OOM => std::terminate is the intended behavior - auto* node = new ready_fd_node{}; + auto* node = acquire_ready_locked(); node->fd = new_fd; node->peer = multi_op_->peer_storage; node->peer_len = multi_op_->peer_len; @@ -874,57 +1015,42 @@ class uring_multishot_acceptor_base } } + // Each claim's exchange set `cancelled`; after reset() has + // quiesced any in-flight canceller, restore the flag to the + // real completion status before posting. if (claimed_peek) { claimed_peek->stop_cb.reset(); - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — CQE handler: noexcept, OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); - op->h = claimed_peek->h; - op->ex = claimed_peek->ex; - op->ec_out = claimed_peek->ec_out; - delete claimed_peek; - sched_->post(op); + claimed_peek->cancelled.store(false, std::memory_order_release); + sched_->post(claimed_peek); sched_->work_finished(); // balance the parking work_started } if (matched) { matched->stop_cb.reset(); - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — CQE handler: noexcept, OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); - op->h = matched->h; - op->ex = matched->ex; - op->ec_out = matched->ec_out; - op->impl_out = matched->impl_out; - op->peer_service = peer_service_; - op->adopt_fn = &Derived::adopt_thunk; + matched->peer_service = peer_service_; + matched->adopt_fn = &Derived::adopt_thunk; if (err) { - op->err = err; + matched->err = err; } else if (new_fd >= 0) { - op->accepted_fd = new_fd; - op->peer_storage = multi_op_->peer_storage; - op->peer_len = multi_op_->peer_len; + matched->accepted_fd = new_fd; + matched->peer_storage = multi_op_->peer_storage; + matched->peer_len = multi_op_->peer_len; } - delete matched; - sched_->post(op); + matched->cancelled.store(false, std::memory_order_release); + sched_->post(matched); sched_->work_finished(); // balance waiter's work_started } while (auto* w = closing_waiters.pop_front()) { w->stop_cb.reset(); - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — CQE handler shutdown path: noexcept, OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); - op->h = w->h; - op->ex = w->ex; - op->ec_out = w->ec_out; - op->impl_out = w->impl_out; - op->cancelled.store(true, std::memory_order_release); - delete w; - sched_->post(op); + w->cancelled.store(true, std::memory_order_release); + sched_->post(w); sched_->work_finished(); // balance waiter's work_started } @@ -933,19 +1059,23 @@ class uring_multishot_acceptor_base // Re-arm: kernel terminated multishot non-fatally. struct rearm_op final : scheduler_op { - std::shared_ptr self_; + detail::object_ref self_ref_; + Derived* self_; std::uint64_t generation_; rearm_op( - std::shared_ptr s, + detail::object_ref ref, + Derived* self, std::uint64_t generation) noexcept - : self_(std::move(s)) + : self_ref_(std::move(ref)) + , self_(self) , generation_(generation) { } void operator()() override { - auto self = std::move(self_); + auto self_ref = std::move(self_ref_); + auto* self = self_; auto generation = generation_; delete this; { @@ -972,22 +1102,13 @@ class uring_multishot_acceptor_base }; // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — CQE handler re-arm: noexcept, OOM => std::terminate is the intended behavior sched_->post(new rearm_op( - this->shared_from_this(), + detail::object_ref(this), + static_cast(this), arm_generation_.load(std::memory_order_acquire))); } } }; -template -inline void -uring_multishot_acceptor_base:: - waiter_canceller::operator()() const noexcept -{ - if (w->cancelled.exchange(true, std::memory_order_acq_rel)) - return; - w->owner->cancel_waiter(w); -} - } // namespace boost::corosio::detail #endif // BOOST_COROSIO_HAS_URING diff --git a/include/boost/corosio/native/detail/uring/uring_op.hpp b/include/boost/corosio/native/detail/uring/uring_op.hpp index 2128731de..027867797 100644 --- a/include/boost/corosio/native/detail/uring/uring_op.hpp +++ b/include/boost/corosio/native/detail/uring/uring_op.hpp @@ -33,7 +33,7 @@ namespace boost::corosio::detail { `coro_op` supplies the fields common to both proactor backends (coroutine handle, executor, output pointers, stop_token wiring, - impl_ptr keepalive). This type adds the CQE result (`res`/`cqe_flags`), + object_ref_ keepalive). This type adds the CQE result (`res`/`cqe_flags`), the ring-cancel visibility flag (`sqe_set`), and the two function pointers the run loop uses to prep an SQE and dispatch a CQE without template instantiation. diff --git a/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp b/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp index 773b049bf..06e1ea0e8 100644 --- a/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp +++ b/include/boost/corosio/native/detail/uring/uring_random_access_file.hpp @@ -15,6 +15,7 @@ #if BOOST_COROSIO_HAS_URING #include +#include #include #include #include @@ -25,7 +26,6 @@ #include #include #include -#include #include #include #include @@ -54,28 +54,138 @@ class uring_random_access_file_service; */ class BOOST_COROSIO_DECL uring_random_access_file final : public random_access_file::implementation - , public std::enable_shared_from_this , public intrusive_list::node { friend class uring_random_access_file_service; - int fd_ = -1; - uring_scheduler* sched_ = nullptr; + int fd_ = -1; + uring_scheduler* sched_ = nullptr; + uring_random_access_file_service* svc_ = nullptr; // Random-access files legitimately support concurrent ops at // different offsets on the same fd (e.g. parallel reads in // testConcurrentReads). Embedding a single slot would smash - // state across calls; ops are heap-allocated per submission. + // state across calls; ops recycle through per-file free lists + // so concurrency is unbounded but steady state allocates + // nothing. The storage belongs to this impl (it survives + // recycling) and only the destructor frees it. + + /** Pooled random-access op: completion recycles instead of + deleting. + + Reuses the heap variant's submission/finish machinery but + swaps the completion handler (the func-pointer convention + makes that one protected assignment) for one that returns the + op to its file's free list. The inherited `object_ref_` + keepalive is dropped only after the push, so a final release + never runs under `ops_mutex_`. + */ + template + struct pooled_op final + : Base + , intrusive_list>::node + { + uring_random_access_file* file_ = nullptr; + + pooled_op() noexcept + { + this->func_ = &do_pooled_handler; + } + + static void do_pooled_handler( + void* owner, + scheduler_op* base, + std::uint32_t /*bytes*/, + std::uint32_t /*error*/) noexcept + { + auto* self = static_cast(base); + self->stop_cb.reset(); + auto keep = std::move(self->object_ref_); + if (owner == nullptr) + { + self->file_->release_op(self); + return; + } + // finish() routes through dispatch_coro, which does not + // retain the op's embedded continuation, so releasing + // before the resume is safe even for a deferring + // executor. + auto next = Base::finish(self); + self->file_->release_op(self); + next.resume(); + } + }; + + using pooled_read_op = pooled_op; + using pooled_write_op = pooled_op; + + std::mutex ops_mutex_; + intrusive_list free_reads_; + intrusive_list free_writes_; + + /// Pop a recycled op of the given kind, or allocate on the cold + /// path; `prepare()` resets every per-use field. + template + Op* acquire_op(intrusive_list& free_list) + { + Op* op; + { + std::lock_guard lock(ops_mutex_); + op = free_list.pop_front(); + } + if (!op) + op = new Op(); + op->file_ = this; + return op; + } + + void release_op(pooled_read_op* op) noexcept + { + std::lock_guard lock(ops_mutex_); + free_reads_.push_front(op); + } + + void release_op(pooled_write_op* op) noexcept + { + std::lock_guard lock(ops_mutex_); + free_writes_.push_front(op); + } public: - explicit uring_random_access_file(uring_scheduler& sched) noexcept + explicit uring_random_access_file( + uring_random_access_file_service& svc, + uring_scheduler& sched) noexcept : sched_(&sched) + , svc_(&svc) { } ~uring_random_access_file() override { close_file(); + while (auto* op = free_reads_.pop_front()) + delete op; + while (auto* op = free_writes_.pop_front()) + delete op; + } + + /// Recycle into the owning service's pool. Defined out-of-line + /// after uring_random_access_file_service for its complete type. + void retire() noexcept override; + + /** Reset for recycling. + + `close_file()` already drove fd_ to its closed value before the + refcount reached zero; every op holds its own `object_ref` + while in flight, so none can be outstanding here. The free + lists deliberately survive recycling — the storage belongs to + this impl and only the destructor frees it. + + @pre refs_ == 0, fd closed, no op in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(fd_ == -1); } // -- random_access_file::implementation -- @@ -228,31 +338,31 @@ uring_random_access_file::read_some_at( std::error_code* ec, std::size_t* bytes) { - auto op_guard = std::make_unique(); - op_guard->prepare( + auto* op = acquire_op(free_reads_); + op->prepare( h, ex, ec, bytes, fd_, static_cast(user_offset), sched_, - shared_from_this(), buffers, token); + detail::object_ref(this), buffers, token); sched_->work_started(); // Closed-object contract outranks the zero-length no-op. if (fd_ < 0) { - op_guard->empty_buffer = false; - op_guard->res = -EBADF; + op->empty_buffer = false; + op->res = -EBADF; uring_scheduler::lock_type lock(sched_->dispatch_mutex()); - sched_->push_completed_locked(op_guard.release()); + sched_->push_completed_locked(op); return std::noop_coroutine(); } - if (op_guard->empty_buffer || - op_guard->cancelled.load(std::memory_order_acquire)) + if (op->empty_buffer || + op->cancelled.load(std::memory_order_acquire)) { uring_scheduler::lock_type lock(sched_->dispatch_mutex()); - sched_->push_completed_locked(op_guard.release()); + sched_->push_completed_locked(op); return std::noop_coroutine(); } - uring_submit_op(*sched_, op_guard.release()); + uring_submit_op(*sched_, op); return std::noop_coroutine(); } @@ -266,31 +376,31 @@ uring_random_access_file::write_some_at( std::error_code* ec, std::size_t* bytes) { - auto op_guard = std::make_unique(); - op_guard->prepare( + auto* op = acquire_op(free_writes_); + op->prepare( h, ex, ec, bytes, fd_, static_cast(user_offset), sched_, - shared_from_this(), buffers, token); + detail::object_ref(this), buffers, token); sched_->work_started(); // Closed-object contract outranks the zero-length no-op. if (fd_ < 0) { - op_guard->empty_buffer = false; - op_guard->res = -EBADF; + op->empty_buffer = false; + op->res = -EBADF; uring_scheduler::lock_type lock(sched_->dispatch_mutex()); - sched_->push_completed_locked(op_guard.release()); + sched_->push_completed_locked(op); return std::noop_coroutine(); } - if (op_guard->empty_buffer || - op_guard->cancelled.load(std::memory_order_acquire)) + if (op->empty_buffer || + op->cancelled.load(std::memory_order_acquire)) { uring_scheduler::lock_type lock(sched_->dispatch_mutex()); - sched_->push_completed_locked(op_guard.release()); + sched_->push_completed_locked(op); return std::noop_coroutine(); } - uring_submit_op(*sched_, op_guard.release()); + uring_submit_op(*sched_, op); return std::noop_coroutine(); } @@ -332,6 +442,12 @@ class BOOST_COROSIO_DECL uring_random_access_file_service final } }; +inline void +uring_random_access_file::retire() noexcept +{ + svc_->pool_.recycle(this); +} + } // namespace boost::corosio::detail #endif // BOOST_COROSIO_HAS_URING diff --git a/include/boost/corosio/native/detail/uring/uring_scheduler.hpp b/include/boost/corosio/native/detail/uring/uring_scheduler.hpp index 87c67b408..3cb753555 100644 --- a/include/boost/corosio/native/detail/uring/uring_scheduler.hpp +++ b/include/boost/corosio/native/detail/uring/uring_scheduler.hpp @@ -335,7 +335,7 @@ class BOOST_COROSIO_DECL uring_scheduler final Callers must hold neither. @pre After @p prepare runs, @p slot must hold no reference back - to its owner (`impl_ptr` cleared, back-pointers nulled). + to its owner (`object_ref_` cleared, back-pointers nulled). `release_retired_op` deletes the op from inside the CQE loop with `ring_mutex_` held, so its destructor must not re-enter the scheduler or touch the owner. @@ -742,8 +742,8 @@ uring_scheduler::shutdown() // for every live impl. The CQEs that result either land in // completed_ops_ (drained here as op->destroy()) or stay in the // kernel ring; ~scheduler's io_uring_queue_exit cleans the - // latter up at process teardown. Self-referential impl_ptr - // cycles (e.g. multishot acceptor's multi_op_->impl_ptr) are + // latter up at process teardown. Self-referential object_ref_ + // cycles (e.g. multishot acceptor's multi_op_->object_ref_) are // broken explicitly inside each service before the scheduler // shutdown runs. lock_type lock(dispatch_mutex_); diff --git a/include/boost/corosio/native/detail/uring/uring_socket_ops.hpp b/include/boost/corosio/native/detail/uring/uring_socket_ops.hpp index 0b564aa7d..bd850b93c 100644 --- a/include/boost/corosio/native/detail/uring/uring_socket_ops.hpp +++ b/include/boost/corosio/native/detail/uring/uring_socket_ops.hpp @@ -129,7 +129,7 @@ struct uring_read_op : uring_op std::size_t* bytes, int file_descriptor, uring_scheduler* scheduler, - std::shared_ptr impl, + detail::object_ref impl, detail::speculative_state* spec, buffer_param buffers, std::stop_token const& token) noexcept @@ -140,7 +140,7 @@ struct uring_read_op : uring_op bytes_out = bytes; fd = file_descriptor; sched_ = scheduler; - impl_ptr = std::move(impl); + object_ref_ = std::move(impl); spec_state = spec; res = 0; cqe_flags = 0; @@ -227,7 +227,7 @@ struct uring_write_op : uring_op std::size_t* bytes, int file_descriptor, uring_scheduler* scheduler, - std::shared_ptr impl, + detail::object_ref impl, detail::speculative_state* spec, buffer_param buffers, std::stop_token const& token) noexcept @@ -238,7 +238,7 @@ struct uring_write_op : uring_op bytes_out = bytes; fd = file_descriptor; sched_ = scheduler; - impl_ptr = std::move(impl); + object_ref_ = std::move(impl); spec_state = spec; res = 0; cqe_flags = 0; @@ -336,7 +336,7 @@ struct uring_connect_op : uring_op std::error_code* ec, int file_descriptor, uring_scheduler* scheduler, - std::shared_ptr impl, + detail::object_ref impl, endpoint target, endpoint* remote_out, endpoint* local_out, @@ -348,7 +348,7 @@ struct uring_connect_op : uring_op bytes_out = nullptr; fd = file_descriptor; sched_ = scheduler; - impl_ptr = std::move(impl); + object_ref_ = std::move(impl); res = 0; cqe_flags = 0; target_endpoint = target; @@ -586,20 +586,20 @@ struct uring_wait_op : uring_op std::error_code* ec, int file_descriptor, uring_scheduler* scheduler, - std::shared_ptr impl, + detail::object_ref impl, int flags, std::stop_token const& token) noexcept { - h = handle; - ex = executor; - ec_out = ec; - bytes_out = nullptr; - fd = file_descriptor; - sched_ = scheduler; - impl_ptr = std::move(impl); - poll_flags = flags; - res = 0; - cqe_flags = 0; + h = handle; + ex = executor; + ec_out = ec; + bytes_out = nullptr; + fd = file_descriptor; + sched_ = scheduler; + object_ref_ = std::move(impl); + poll_flags = flags; + res = 0; + cqe_flags = 0; start(token); } @@ -691,7 +691,7 @@ struct uring_local_connect_op : uring_op std::error_code* ec, int file_descriptor, uring_scheduler* scheduler, - std::shared_ptr impl, + detail::object_ref impl, corosio::local_endpoint target, corosio::local_endpoint* remote_out, corosio::local_endpoint* local_out, @@ -703,7 +703,7 @@ struct uring_local_connect_op : uring_op bytes_out = nullptr; fd = file_descriptor; sched_ = scheduler; - impl_ptr = std::move(impl); + object_ref_ = std::move(impl); res = 0; cqe_flags = 0; target_endpoint = target; diff --git a/include/boost/corosio/native/detail/uring/uring_socket_service_base.hpp b/include/boost/corosio/native/detail/uring/uring_socket_service_base.hpp index bf510575e..be8d30417 100644 --- a/include/boost/corosio/native/detail/uring/uring_socket_service_base.hpp +++ b/include/boost/corosio/native/detail/uring/uring_socket_service_base.hpp @@ -14,13 +14,12 @@ #if BOOST_COROSIO_HAS_URING +#include +#include #include #include #include -#include -#include -#include #include /* @@ -28,23 +27,23 @@ construct / destroy / shutdown / close / scheduler() are identical across uring_tcp_service, uring_udp_service, uring_local_stream_service, - and uring_local_datagram_service — they all make_shared the impl, track - it in a raw->shared_ptr map, cancel on shutdown, and close eagerly. This + and uring_local_datagram_service — they all pop-or-new the impl from a + per-service recycling pool, cancel on shutdown, and close eagerly. This base factors that out; the concrete services add only the protocol- specific open/bind/adopt. This is io_uring's own service base rather than a reuse of - reactor_socket_service: io_uring tracks impls in a map (the reactor uses - an intrusive list + map), cancels (not closes) on shutdown, and constructs - impls with a (service&, scheduler&) ctor. Reusing the reactor template - would force io_uring sockets to adopt the intrusive node, a close-on- - shutdown behavior change, and an Impl(Derived&) ctor — high churn and a - teardown behavior change for marginal extra sharing. See + reactor_socket_service: io_uring cancels (not closes) on shutdown and + constructs impls with a (service&, scheduler&) ctor, vs. the reactor's + close-on-shutdown and Impl(Derived&) ctor. Reusing the reactor template + would force a teardown behavior change for marginal extra sharing. See tasks/proactor-dedup-decisions.md (#13). Requirements on Socket: a `(Derived& service, uring_scheduler& sched)` - constructor and a `void close_socket() noexcept` method (cancel in-flight - ops + close fd + reset cached endpoints). + constructor, a `void close_socket() noexcept` method (cancel in-flight + ops + close fd + reset cached endpoints), a `void reuse() noexcept` + method, and derivation from `intrusive_list::node` (the pool's + live/free linkage). @tparam Derived The concrete service (CRTP, passed to the Socket ctor). @tparam ServiceBase The abstract service vtable base (tcp_service, ...). @@ -57,6 +56,7 @@ template class uring_socket_service_base : public ServiceBase { friend Derived; + friend Socket; // Private CRTP ctor: only `Derived` (the concrete service, a friend) // constructs the base — prevents inheriting with the wrong Derived @@ -71,41 +71,42 @@ class uring_socket_service_base : public ServiceBase void shutdown() override { - // Snapshot live impls, then cancel without the lock held to avoid - // inversion if cancel() ever re-enters the service. Impls stay owned - // by impls_ until ~service (after the scheduler drains its queue), - // keeping every impl alive while its cancel CQEs are processed. - std::vector> live; + // Snapshot live impls under an acquired reference, then cancel + // without the pool lock held to avoid inversion if cancel() ever + // re-enters the service. shutdown() sets shutting-down and + // takes the snapshot under one critical section, so each + // cancel's own release (if it drops the last ref) deletes + // rather than recycles. In-flight ops hold their own + // references, so every impl stays alive while its cancel CQEs + // are processed. + std::vector live; + pool_.shutdown( + [&](Socket* s) + { + acquire(s); + live.push_back(s); + }); + for (auto* s : live) { - std::lock_guard lk(mutex_); - live.reserve(impls_.size()); - for (auto& [_, p] : impls_) - live.push_back(p); + s->cancel(); + release(s); } - for (auto& p : live) - p->cancel(); } io_object::implementation* construct() override { - auto p = - std::make_shared(static_cast(*this), *sched_); - auto* raw = p.get(); - std::lock_guard lk(mutex_); - impls_.emplace(raw, std::move(p)); - return raw; + return acquire_impl(); } void destroy(io_object::implementation* p) override { if (!p) return; - std::lock_guard lk(mutex_); - impls_.erase(static_cast(p)); + release(static_cast(p)); } // Close the fd eagerly when the public close() is called, before - // destroy() drops the shared_ptr and the destructor runs. + // destroy() drops the service's reference and recycling runs. void close(io_object::handle& h) override { if (auto* sock = static_cast(h.get())) @@ -119,19 +120,18 @@ class uring_socket_service_base : public ServiceBase } protected: - /// Register an externally-built impl (used by adopt_fd on stream - /// services after accept(2)). Returns the raw pointer. - Socket* register_impl(std::shared_ptr p) + /** Pop a recycled impl or news one, with the service reference held. + + Used by both `construct()` and the `adopt_fd` accepted-connection + path (which additionally calls `assign_fd()` on the result). + */ + Socket* acquire_impl() { - auto* raw = p.get(); - std::lock_guard lk(mutex_); - impls_.emplace(raw, std::move(p)); - return raw; + return pool_.acquire(static_cast(*this), *sched_); } uring_scheduler* sched_; - std::mutex mutex_; - std::unordered_map> impls_; + object_pool pool_; private: uring_socket_service_base(uring_socket_service_base const&) = delete; diff --git a/include/boost/corosio/native/detail/uring/uring_stream_file.hpp b/include/boost/corosio/native/detail/uring/uring_stream_file.hpp index 05bb2ccf6..e893c6b62 100644 --- a/include/boost/corosio/native/detail/uring/uring_stream_file.hpp +++ b/include/boost/corosio/native/detail/uring/uring_stream_file.hpp @@ -15,6 +15,7 @@ #if BOOST_COROSIO_HAS_URING #include +#include #include #include #include @@ -62,13 +63,13 @@ class uring_stream_file_service; */ class BOOST_COROSIO_DECL uring_stream_file final : public stream_file::implementation - , public std::enable_shared_from_this , public intrusive_list::node { friend class uring_stream_file_service; - int fd_ = -1; - uring_scheduler* sched_ = nullptr; + int fd_ = -1; + uring_scheduler* sched_ = nullptr; + uring_stream_file_service* svc_ = nullptr; // Per-fd op slots — embedded to eliminate per-call heap allocation. // Single-pending invariant per slot. @@ -76,7 +77,10 @@ class BOOST_COROSIO_DECL uring_stream_file final uring_file_write_op wr_; public: - explicit uring_stream_file(uring_scheduler& sched) noexcept : sched_(&sched) + explicit uring_stream_file( + uring_stream_file_service& svc, uring_scheduler& sched) noexcept + : sched_(&sched) + , svc_(&svc) { } @@ -85,6 +89,26 @@ class BOOST_COROSIO_DECL uring_stream_file final close_file(); } + /// Recycle into the owning service's pool. Defined out-of-line + /// after uring_stream_file_service for its complete type. + void retire() noexcept override; + + /** Reset op slots for recycling. + + `close_file()` already drove fd_ to its closed value before the + refcount reached zero; each op's own `prepare()` overwrites its + state before the next use, so only the `stop_cb` invariant is + worth asserting here. + + @pre refs_ == 0, fd closed, no op in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(fd_ == -1); + BOOST_COROSIO_ASSERT(!rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wr_.stop_cb); + } + // -- io_stream::implementation -- std::coroutine_handle<> read_some( @@ -252,7 +276,7 @@ uring_stream_file::read_some( std::size_t* bytes) { rd_.prepare( - h, ex, ec, bytes, fd_, /*file_offset=*/-1, sched_, shared_from_this(), + h, ex, ec, bytes, fd_, /*file_offset=*/-1, sched_, detail::object_ref(this), buffers, token); sched_->work_started(); @@ -287,7 +311,7 @@ uring_stream_file::write_some( std::size_t* bytes) { wr_.prepare( - h, ex, ec, bytes, fd_, /*file_offset=*/-1, sched_, shared_from_this(), + h, ex, ec, bytes, fd_, /*file_offset=*/-1, sched_, detail::object_ref(this), buffers, token); sched_->work_started(); @@ -347,6 +371,12 @@ class BOOST_COROSIO_DECL uring_stream_file_service final } }; +inline void +uring_stream_file::retire() noexcept +{ + svc_->pool_.recycle(this); +} + } // namespace boost::corosio::detail #endif // BOOST_COROSIO_HAS_URING diff --git a/include/boost/corosio/native/detail/uring/uring_types.hpp b/include/boost/corosio/native/detail/uring/uring_types.hpp index f297004b9..a684326a2 100644 --- a/include/boost/corosio/native/detail/uring/uring_types.hpp +++ b/include/boost/corosio/native/detail/uring/uring_types.hpp @@ -15,6 +15,7 @@ #if BOOST_COROSIO_HAS_URING +#include #include #include #include @@ -69,8 +70,8 @@ class uring_local_datagram_service; read, write, and connect operations are submitted to the kernel via `uring_submit_op` and complete through the ring's CQE path. - The object is always owned by a `shared_ptr` managed by the service. - In-flight ops hold an additional `shared_ptr` copy (`impl_ptr`) so + The service holds one intrusive reference for as long as the impl + is live; in-flight ops hold an additional `object_ref_` keepalive so the kernel's user-data pointer remains valid until the CQE arrives. @par Thread Safety @@ -83,12 +84,13 @@ class BOOST_COROSIO_DECL uring_tcp_socket final uring_tcp_socket, tcp_socket::implementation, endpoint> + , public intrusive_list::node { friend uring_tcp_service; int family_ = AF_UNSPEC; // cached at open_socket uring_scheduler* sched_ = nullptr; - [[maybe_unused]] uring_tcp_service* svc_ = nullptr; + uring_tcp_service* svc_ = nullptr; // fd_ and local_endpoint_ are provided by native_socket_base (the // readiness/completion-agnostic socket base shared with the reactor @@ -207,7 +209,7 @@ class BOOST_COROSIO_DECL uring_tcp_socket final return dispatch_coro(ex, rd_.cont); } rd_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, token); if (stop_now) rd_.cancelled.store(true, std::memory_order_release); @@ -222,7 +224,7 @@ class BOOST_COROSIO_DECL uring_tcp_socket final } rd_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, buffers, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, token); sched_->work_started(); if (rd_.cancelled.load(std::memory_order_acquire)) @@ -286,7 +288,7 @@ class BOOST_COROSIO_DECL uring_tcp_socket final return dispatch_coro(ex, wr_.cont); } wr_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, token); if (stop_now) wr_.cancelled.store(true, std::memory_order_release); @@ -301,7 +303,7 @@ class BOOST_COROSIO_DECL uring_tcp_socket final } wr_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, buffers, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, token); sched_->work_started(); if (wr_.cancelled.load(std::memory_order_acquire)) @@ -337,7 +339,7 @@ class BOOST_COROSIO_DECL uring_tcp_socket final } conn_.addrlen = to_sockaddr(ep, family_, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), ep, + h, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, &local_endpoint_, token); conn_.cancelled.store(true, std::memory_order_release); sched_->work_started(); @@ -352,7 +354,7 @@ class BOOST_COROSIO_DECL uring_tcp_socket final // a subsequent IORING_OP_CONNECT would see EALREADY — avoid. conn_.addrlen = to_sockaddr(ep, family_, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), ep, &remote_endpoint_, + h, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, &local_endpoint_, token); sched_->work_started(); if (conn_.cancelled.load(std::memory_order_acquire)) @@ -386,7 +388,7 @@ class BOOST_COROSIO_DECL uring_tcp_socket final break; } wait_op_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), poll_flags, token); + h, ex, ec, fd_, sched_, detail::object_ref(this), poll_flags, token); sched_->work_started(); if (wait_op_.cancelled.load(std::memory_order_acquire)) { @@ -448,6 +450,45 @@ class BOOST_COROSIO_DECL uring_tcp_socket final endpoint_state::unresolved, std::memory_order_release); } + /** Recycle into the owning service's pool at zero references. + + Defined out-of-line after `uring_tcp_service` — its body needs + the service's complete type to reach the private `pool_` + member (reachable because this impl is a friend), and this + non-template member would otherwise require it right here, + before the service class exists in the file. + */ + void retire() noexcept override; + + /** Reset op slots, cached endpoints, and the speculation hint for + recycling. + + `close_socket()` already drove fd_, the endpoints, and the + endpoint-resolution state to their closed values before the + refcount reached zero; each op's own `prepare()` overwrites its + iovec/addr scratch before the next use. What is left: `family_` + (cached by `open_socket()`, but never touched by the accepted- + connection `adopt_fd()` path, so a socket recycled through + `adopt_fd()` would otherwise keep the prior session's family), + the speculation hint (a per-connection performance heuristic + that must not leak into the next logical connection), and the + `stop_cb` invariant on every op slot. + + @pre refs_ == 0, fd closed, no op in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(fd_ == -1); + BOOST_COROSIO_ASSERT(local_endpoint_ == endpoint{}); + BOOST_COROSIO_ASSERT(remote_endpoint_ == endpoint{}); + BOOST_COROSIO_ASSERT(!rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wr_.stop_cb); + BOOST_COROSIO_ASSERT(!conn_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_op_.stop_cb); + family_ = AF_UNSPEC; + spec_.reset(); + } + endpoint local_endpoint() const noexcept override { // Lazy resolution: only fire the getsockname syscall when @@ -484,9 +525,10 @@ class BOOST_COROSIO_DECL uring_tcp_socket final Satisfies the `tcp_service` interface so the generic `tcp_socket` front-end can call `open_socket` and `bind_socket` transparently. - Socket impls are reference-counted inside the service map; raw - pointers returned from `construct()` remain valid until `destroy()` - or `shutdown()` is called. + Socket impls live in the service's `object_pool`. `construct()` + hands out a pooled impl with one reference already held; `destroy()` + releases it, and `retire()` recycles it onto the free list + instead of freeing it. @par Thread Safety All public member functions are thread-safe. @@ -649,7 +691,7 @@ class BOOST_COROSIO_DECL uring_tcp_service final */ uring_tcp_socket* adopt_fd(int fd, endpoint const& peer) { - auto p = std::make_shared(*this, *sched_); + auto* p = this->acquire_impl(); p->fd_ = fd; p->remote_endpoint_ = peer; // Mark the local endpoint as authoritative-but-unresolved. @@ -660,10 +702,16 @@ class BOOST_COROSIO_DECL uring_tcp_service final uring_tcp_socket::endpoint_state::lazy_pending, std::memory_order_release); - return this->register_impl(std::move(p)); + return p; } }; +inline void +uring_tcp_socket::retire() noexcept +{ + svc_->pool_.recycle(this); +} + /** TCP acceptor implementation for io_uring. Inherits the multishot machinery (parked-fd queue, waiter queue, @@ -678,7 +726,8 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor final uring_tcp_acceptor, tcp_acceptor::implementation, endpoint, - uring_tcp_service> + uring_tcp_service, + uring_tcp_acceptor_service> { friend uring_tcp_acceptor_service; @@ -686,7 +735,8 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor final uring_tcp_acceptor, tcp_acceptor::implementation, endpoint, - uring_tcp_service>; + uring_tcp_service, + uring_tcp_acceptor_service>; // Readiness-wait slot. The multishot accept op delivers accepted // fds, but `wait()` reports raw poll readiness on the listening fd @@ -695,11 +745,21 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor final public: explicit uring_tcp_acceptor( - uring_tcp_acceptor_service&, + uring_tcp_acceptor_service& svc, uring_scheduler& sched, uring_tcp_service& peer_svc) noexcept - : base_type(sched, peer_svc) + : base_type(svc, sched, peer_svc) + { + } + + /** Extend the base reset with this acceptor's own `wait_op_` slot. + + @pre refs_ == 0, no wait in flight (see base_type::reuse()). + */ + void reuse() noexcept { + base_type::reuse(); + BOOST_COROSIO_ASSERT(!wait_op_.stop_cb); } std::coroutine_handle<> accept( @@ -724,8 +784,7 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor final // instead of parking a waiter no accept machinery will signal. if (this->fd_ < 0) { - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept-adjacent initiation path: OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); + auto* op = this->acquire_node(); op->h = h; op->ex = ex; op->ec_out = ec; @@ -745,8 +804,7 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor final // fail uniformly instead of never completing. if (w == wait_type::write) { - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept-adjacent initiation path: OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); + auto* op = this->acquire_node(); op->h = h; op->ex = ex; op->ec_out = ec; @@ -757,7 +815,7 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor final // Errors are not consumed by the accept machinery, so the // error wait still polls the descriptor. wait_op_.prepare( - h, ex, ec, this->fd_, this->sched_, this->shared_from_this(), + h, ex, ec, this->fd_, this->sched_, detail::object_ref(this), POLLPRI | POLLERR | POLLHUP, token); this->sched_->work_started(); if (wait_op_.cancelled.load(std::memory_order_acquire)) @@ -788,9 +846,10 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor final `tcp_acceptor` front-end can call `open_acceptor_socket`, `bind_acceptor`, and `listen_acceptor` transparently. - Acceptor impls are reference-counted inside the service map; raw - pointers returned from `construct()` remain valid until `destroy()` - or `shutdown()` is called. + Acceptor impls live in the service's `object_pool`. `construct()` + hands out a pooled impl with one reference already held; `destroy()` + releases it, and `retire()` recycles it onto the free list + instead of freeing it. @par Thread Safety All public member functions are thread-safe. @@ -798,6 +857,9 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor final class BOOST_COROSIO_DECL uring_tcp_acceptor_service final : public tcp_acceptor_service { + template + friend class uring_multishot_acceptor_base; + public: /// Identifies this service for `execution_context` lookup. using key_type = tcp_acceptor_service; @@ -815,39 +877,40 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor_service final void shutdown() override { - std::vector> live; + // See uring_socket_service_base::shutdown(): snapshot under an + // acquired reference, cancel without the pool lock held; + // shutdown() sets shutting-down and takes the snapshot in one + // critical section, so a cancel that drops the last ref deletes + // rather than recycles. + std::vector live; + pool_.shutdown( + [&](uring_tcp_acceptor* a) + { + acquire(a); + live.push_back(a); + }); + for (auto* a : live) { - std::lock_guard lk(mutex_); - live.reserve(impls_.size()); - for (auto& [_, p] : impls_) - live.push_back(p); + a->cancel(); + release(a); } - // Cancel without the lock held to avoid inversion if cancel() - // re-enters the service. - for (auto& p : live) - p->cancel(); } io_object::implementation* construct() override { - auto p = - std::make_shared(*this, *sched_, *peer_svc_); - auto* raw = p.get(); - std::lock_guard lk(mutex_); - impls_.emplace(raw, std::move(p)); - return raw; + return acquire_impl(); } void destroy(io_object::implementation* p) override { if (!p) return; - std::lock_guard lk(mutex_); - impls_.erase(static_cast(p)); + release(static_cast(p)); } // Close the fd eagerly when tcp_acceptor::close() is called, before - // destroy() drops the shared_ptr and the destructor runs. + // destroy() releases the service's reference and retire() may + // recycle this impl. void close(io_object::handle& h) override { auto* acc = static_cast(h.get()); @@ -860,15 +923,16 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor_service final sched_->cancel_and_flush(acc->fd_); acc->drain_waiters_only(); ::close(acc->fd_); - acc->fd_ = -1; + acc->fd_ = -1; + acc->local_endpoint_ = endpoint{}; - // Break the multi_op_ -> impl_ptr (shared_ptr) cycle + // Break the multi_op_ -> object_ref_ (object_ref) cycle // start_multishot established. The acceptor destructor's // drain_cqes_for(multi_op_.get()) is the safety net; here - // we just drop the cycle so the impl can be released when - // the user's last shared_ptr does. + // we just drop the cycle so the impl can be recycled once + // its last object_ref_ releases. if (acc->multi_op_) - acc->multi_op_->impl_ptr.reset(); + acc->multi_op_->object_ref_.reset(); } } @@ -1002,11 +1066,15 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor_service final } private: + /// Pop a recycled impl or news one, with the service reference held. + uring_tcp_acceptor* acquire_impl() + { + return pool_.acquire(*this, *sched_, *peer_svc_); + } + uring_scheduler* sched_; uring_tcp_service* peer_svc_; - std::mutex mutex_; - std::unordered_map> - impls_; + object_pool pool_; }; /** Unix domain stream socket implementation for io_uring. @@ -1016,8 +1084,8 @@ class BOOST_COROSIO_DECL uring_tcp_acceptor_service final kernel via `uring_submit_op` and complete through the ring's CQE path. - The object is always owned by a `shared_ptr` managed by the service. - In-flight ops hold an additional `shared_ptr` copy (`impl_ptr`) so + The service holds one intrusive reference for as long as the impl + is live; in-flight ops hold an additional `object_ref_` keepalive so the kernel's user-data pointer remains valid until the CQE arrives. @par Thread Safety @@ -1030,11 +1098,12 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final uring_local_stream_socket, local_stream_socket::implementation, corosio::local_endpoint> + , public intrusive_list::node { friend uring_local_stream_service; - uring_scheduler* sched_ = nullptr; - [[maybe_unused]] uring_local_stream_service* svc_ = nullptr; + uring_scheduler* sched_ = nullptr; + uring_local_stream_service* svc_ = nullptr; // fd_ and local_endpoint_ live in native_socket_base, which also // provides native_handle/is_open/set_option/get_option/local_endpoint. @@ -1129,7 +1198,7 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final return dispatch_coro(ex, rd_.cont); } rd_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, token); if (stop_now) rd_.cancelled.store(true, std::memory_order_release); @@ -1144,7 +1213,7 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final } rd_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, buffers, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, token); sched_->work_started(); if (rd_.cancelled.load(std::memory_order_acquire)) @@ -1208,7 +1277,7 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final return dispatch_coro(ex, wr_.cont); } wr_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, token); if (stop_now) wr_.cancelled.store(true, std::memory_order_release); @@ -1223,7 +1292,7 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final } wr_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, buffers, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, token); sched_->work_started(); if (wr_.cancelled.load(std::memory_order_acquire)) @@ -1259,7 +1328,7 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final } conn_.addrlen = to_sockaddr(ep, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), ep, + h, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, &local_endpoint_, token); conn_.cancelled.store(true, std::memory_order_release); sched_->work_started(); @@ -1274,7 +1343,7 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final // a subsequent IORING_OP_CONNECT would see EALREADY — avoid. conn_.addrlen = to_sockaddr(ep, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), ep, &remote_endpoint_, + h, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, &local_endpoint_, token); sched_->work_started(); if (conn_.cancelled.load(std::memory_order_acquire)) @@ -1308,7 +1377,7 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final break; } wait_op_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), poll_flags, token); + h, ex, ec, fd_, sched_, detail::object_ref(this), poll_flags, token); sched_->work_started(); if (wait_op_.cancelled.load(std::memory_order_acquire)) { @@ -1365,6 +1434,27 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final remote_endpoint_ = corosio::local_endpoint{}; } + /// Recycle into the owning service's pool. See + /// uring_tcp_socket::retire() for why this is out-of-line. + void retire() noexcept override; + + /** Reset op slots, cached endpoints, and the speculation hint for + recycling. See uring_tcp_socket::reuse() for the rationale. + + @pre refs_ == 0, fd closed, no op in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(fd_ == -1); + BOOST_COROSIO_ASSERT(local_endpoint_ == corosio::local_endpoint{}); + BOOST_COROSIO_ASSERT(remote_endpoint_ == corosio::local_endpoint{}); + BOOST_COROSIO_ASSERT(!rd_.stop_cb); + BOOST_COROSIO_ASSERT(!wr_.stop_cb); + BOOST_COROSIO_ASSERT(!conn_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_op_.stop_cb); + spec_.reset(); + } + corosio::local_endpoint remote_endpoint() const noexcept override { return remote_endpoint_; @@ -1378,9 +1468,10 @@ class BOOST_COROSIO_DECL uring_local_stream_socket final generic `local_stream_socket` front-end can call `open_socket` and `assign_socket` transparently. - Socket impls are reference-counted inside the service map; raw - pointers returned from `construct()` remain valid until `destroy()` - or `shutdown()` is called. + Socket impls live in the service's `object_pool`. `construct()` + hands out a pooled impl with one reference already held; `destroy()` + releases it, and `retire()` recycles it onto the free list + instead of freeing it. @par Thread Safety All public member functions are thread-safe. @@ -1504,7 +1595,7 @@ class BOOST_COROSIO_DECL uring_local_stream_service final uring_local_stream_socket* adopt_fd(int fd, corosio::local_endpoint const& peer) { - auto p = std::make_shared(*this, *sched_); + auto* p = this->acquire_impl(); p->fd_ = fd; p->remote_endpoint_ = peer; @@ -1513,10 +1604,16 @@ class BOOST_COROSIO_DECL uring_local_stream_service final if (::getsockname(fd, reinterpret_cast(&local), &len) == 0) p->local_endpoint_ = sockaddr_to_local_endpoint(local, len); - return this->register_impl(std::move(p)); + return p; } }; +inline void +uring_local_stream_socket::retire() noexcept +{ + svc_->pool_.recycle(this); +} + /** Local-stream (Unix domain) acceptor for io_uring. Inherits all multishot machinery (parked-fd queue, waiter queue, @@ -1530,7 +1627,8 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor final uring_local_stream_acceptor, local_stream_acceptor::implementation, corosio::local_endpoint, - uring_local_stream_service> + uring_local_stream_service, + uring_local_stream_acceptor_service> { friend uring_local_stream_acceptor_service; @@ -1538,18 +1636,29 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor final uring_local_stream_acceptor, local_stream_acceptor::implementation, corosio::local_endpoint, - uring_local_stream_service>; + uring_local_stream_service, + uring_local_stream_acceptor_service>; // Readiness-wait slot. See uring_tcp_acceptor::wait_op_. uring_wait_op wait_op_; public: explicit uring_local_stream_acceptor( - uring_local_stream_acceptor_service&, + uring_local_stream_acceptor_service& svc, uring_scheduler& sched, uring_local_stream_service& peer_svc) noexcept - : base_type(sched, peer_svc) + : base_type(svc, sched, peer_svc) + { + } + + /** Extend the base reset with this acceptor's own `wait_op_` slot. + + @pre refs_ == 0, no wait in flight (see base_type::reuse()). + */ + void reuse() noexcept { + base_type::reuse(); + BOOST_COROSIO_ASSERT(!wait_op_.stop_cb); } std::coroutine_handle<> accept( @@ -1574,8 +1683,7 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor final // instead of parking a waiter no accept machinery will signal. if (this->fd_ < 0) { - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept-adjacent initiation path: OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); + auto* op = this->acquire_node(); op->h = h; op->ex = ex; op->ec_out = ec; @@ -1595,8 +1703,7 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor final // fail uniformly instead of never completing. if (w == wait_type::write) { - // NOLINTNEXTLINE(bugprone-unhandled-exception-at-new) — noexcept-adjacent initiation path: OOM => std::terminate is the intended behavior - auto* op = new uring_accept_op(); + auto* op = this->acquire_node(); op->h = h; op->ex = ex; op->ec_out = ec; @@ -1607,7 +1714,7 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor final // Errors are not consumed by the accept machinery, so the // error wait still polls the descriptor. wait_op_.prepare( - h, ex, ec, this->fd_, this->sched_, this->shared_from_this(), + h, ex, ec, this->fd_, this->sched_, detail::object_ref(this), POLLPRI | POLLERR | POLLHUP, token); this->sched_->work_started(); if (wait_op_.cancelled.load(std::memory_order_acquire)) @@ -1639,9 +1746,10 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor final `open_acceptor_socket`, `bind_acceptor`, and `listen_acceptor` transparently. - Acceptor impls are reference-counted inside the service map; raw - pointers returned from `construct()` remain valid until `destroy()` - or `shutdown()` is called. + Acceptor impls live in the service's `object_pool`. `construct()` + hands out a pooled impl with one reference already held; `destroy()` + releases it, and `retire()` recycles it onto the free list + instead of freeing it. @par Thread Safety All public member functions are thread-safe. @@ -1649,6 +1757,9 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor final class BOOST_COROSIO_DECL uring_local_stream_acceptor_service final : public local_stream_acceptor_service { + template + friend class uring_multishot_acceptor_base; + public: /// Identifies this service for `execution_context` lookup. using key_type = local_stream_acceptor_service; @@ -1666,39 +1777,36 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor_service final void shutdown() override { - std::vector> live; + // See uring_tcp_acceptor_service::shutdown() for the ordering + // rationale. + std::vector live; + pool_.shutdown( + [&](uring_local_stream_acceptor* a) + { + acquire(a); + live.push_back(a); + }); + for (auto* a : live) { - std::lock_guard lk(mutex_); - live.reserve(impls_.size()); - for (auto& [_, p] : impls_) - live.push_back(p); + a->cancel(); + release(a); } - // Cancel without the lock held to avoid inversion if cancel() - // re-enters the service. - for (auto& p : live) - p->cancel(); } io_object::implementation* construct() override { - auto p = std::make_shared( - *this, *sched_, *peer_svc_); - auto* raw = p.get(); - std::lock_guard lk(mutex_); - impls_.emplace(raw, std::move(p)); - return raw; + return acquire_impl(); } void destroy(io_object::implementation* p) override { if (!p) return; - std::lock_guard lk(mutex_); - impls_.erase(static_cast(p)); + release(static_cast(p)); } // Close the fd eagerly when local_stream_acceptor::close() is called, - // before destroy() drops the shared_ptr and the destructor runs. + // before destroy() drops the service's reference and recycling runs. void close(io_object::handle& h) override { auto* acc = static_cast(h.get()); @@ -1709,13 +1817,14 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor_service final sched_->cancel_and_flush(acc->fd_); acc->drain_waiters_only(); ::close(acc->fd_); - acc->fd_ = -1; + acc->fd_ = -1; + acc->local_endpoint_ = corosio::local_endpoint{}; - // Break the multi_op_ -> impl_ptr (shared_ptr) cycle + // Break the multi_op_ -> object_ref_ (object_ref) cycle // start_multishot established. See the symmetric comment // in uring_tcp_acceptor_service::close. if (acc->multi_op_) - acc->multi_op_->impl_ptr.reset(); + acc->multi_op_->object_ref_.reset(); } } @@ -1844,13 +1953,15 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor_service final } private: + /// Pop a recycled impl or news one, with the service reference held. + uring_local_stream_acceptor* acquire_impl() + { + return pool_.acquire(*this, *sched_, *peer_svc_); + } + uring_scheduler* sched_; uring_local_stream_service* peer_svc_; - std::mutex mutex_; - std::unordered_map< - uring_local_stream_acceptor*, - std::shared_ptr> - impls_; + object_pool pool_; }; /** UDP socket implementation for io_uring. @@ -1860,8 +1971,8 @@ class BOOST_COROSIO_DECL uring_local_stream_acceptor_service final to the kernel via `uring_submit_op` and complete through the ring's CQE path. - The object is always owned by a `shared_ptr` managed by the service. - In-flight ops hold an additional `shared_ptr` copy (`impl_ptr`) so + The service holds one intrusive reference for as long as the impl + is live; in-flight ops hold an additional `object_ref_` keepalive so the kernel's user-data pointer remains valid until the CQE arrives. @par Thread Safety @@ -1874,12 +1985,13 @@ class BOOST_COROSIO_DECL uring_udp_socket final uring_udp_socket, udp_socket::implementation, corosio::endpoint> + , public intrusive_list::node { friend uring_udp_service; int family_ = AF_UNSPEC; // cached at open_socket uring_scheduler* sched_ = nullptr; - [[maybe_unused]] uring_udp_service* svc_ = nullptr; + uring_udp_service* svc_ = nullptr; // fd_ and local_endpoint_ live in native_socket_base, which also // provides native_handle/is_open/set_option/get_option/local_endpoint. @@ -1994,7 +2106,7 @@ class BOOST_COROSIO_DECL uring_udp_socket final } conn_.addrlen = to_sockaddr(ep, family_, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), ep, + h, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, &local_endpoint_, token); conn_.cancelled.store(true, std::memory_order_release); sched_->work_started(); @@ -2009,7 +2121,7 @@ class BOOST_COROSIO_DECL uring_udp_socket final // a prior speculative ::connect would leave EINPROGRESS → EALREADY. conn_.addrlen = to_sockaddr(ep, family_, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), ep, &remote_endpoint_, + h, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, &local_endpoint_, token); sched_->work_started(); if (conn_.cancelled.load(std::memory_order_acquire)) @@ -2043,7 +2155,7 @@ class BOOST_COROSIO_DECL uring_udp_socket final break; } wait_op_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), poll_flags, token); + h, ex, ec, fd_, sched_, detail::object_ref(this), poll_flags, token); sched_->work_started(); if (wait_op_.cancelled.load(std::memory_order_acquire)) { @@ -2099,6 +2211,33 @@ class BOOST_COROSIO_DECL uring_udp_socket final remote_endpoint_ = endpoint{}; } + /// Recycle into the owning service's pool. See + /// uring_tcp_socket::retire() for why this is out-of-line. + void retire() noexcept override; + + /** Reset op slots, cached endpoints, the speculation hint, and + `family_` for recycling. `family_` is set by `open_socket()`/ + `assign_socket()`, not by this reset, so resetting it to + `AF_UNSPEC` here is defensive — it guards any future entry + path that constructs a usable socket without going through + either of those, the same hazard `uring_tcp_socket::reuse()` + documents for its `adopt_fd()` path. + + @pre refs_ == 0, fd closed, no op in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(fd_ == -1); + BOOST_COROSIO_ASSERT(local_endpoint_ == endpoint{}); + BOOST_COROSIO_ASSERT(remote_endpoint_ == endpoint{}); + BOOST_COROSIO_ASSERT(!conn_.stop_cb); + BOOST_COROSIO_ASSERT(!send_.stop_cb); + BOOST_COROSIO_ASSERT(!recv_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_op_.stop_cb); + family_ = AF_UNSPEC; + spec_.reset(); + } + endpoint remote_endpoint() const noexcept override { return remote_endpoint_; @@ -2166,7 +2305,7 @@ class BOOST_COROSIO_DECL uring_udp_socket final return dispatch_coro(ex, send_.cont); } send_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, dest_len, dest_storage, to_native_msg_flags(flags), token); if (stop_now) @@ -2182,7 +2321,7 @@ class BOOST_COROSIO_DECL uring_udp_socket final } send_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, buffers, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, dest_len, dest_storage, to_native_msg_flags(flags), token); sched_->work_started(); if (send_.cancelled.load(std::memory_order_acquire)) @@ -2260,7 +2399,7 @@ class BOOST_COROSIO_DECL uring_udp_socket final return dispatch_coro(ex, recv_.cont); } recv_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, source_out, want_source ? &write_ip_source : nullptr, to_native_msg_flags(flags), token); if (stop_now) @@ -2286,7 +2425,7 @@ class BOOST_COROSIO_DECL uring_udp_socket final } recv_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, buffers, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, source_out, want_source ? &write_ip_source : nullptr, to_native_msg_flags(flags), token); sched_->work_started(); @@ -2316,9 +2455,10 @@ class BOOST_COROSIO_DECL uring_udp_socket final front-end can call `open_datagram_socket` and `bind_datagram` transparently. - Socket impls are reference-counted inside the service map; raw - pointers returned from `construct()` remain valid until `destroy()` - or `shutdown()` is called. + Socket impls live in the service's `object_pool`. `construct()` + hands out a pooled impl with one reference already held; `destroy()` + releases it, and `retire()` recycles it onto the free list + instead of freeing it. @par Thread Safety All public member functions are thread-safe. @@ -2461,6 +2601,12 @@ class BOOST_COROSIO_DECL uring_udp_service final } }; +inline void +uring_udp_socket::retire() noexcept +{ + svc_->pool_.recycle(this); +} + /** Unix domain datagram socket implementation for io_uring. Implements `local_datagram_socket::implementation` using a proactor @@ -2468,8 +2614,8 @@ class BOOST_COROSIO_DECL uring_udp_service final submitted to the kernel via `uring_submit_op` and complete through the ring's CQE path. - The object is always owned by a `shared_ptr` managed by the service. - In-flight ops hold an additional `shared_ptr` copy (`impl_ptr`) so + The service holds one intrusive reference for as long as the impl + is live; in-flight ops hold an additional `object_ref_` keepalive so the kernel's user-data pointer remains valid until the CQE arrives. @par Thread Safety @@ -2482,11 +2628,12 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final uring_local_datagram_socket, local_datagram_socket::implementation, corosio::local_endpoint> + , public intrusive_list::node { friend uring_local_datagram_service; - uring_scheduler* sched_ = nullptr; - [[maybe_unused]] uring_local_datagram_service* svc_ = nullptr; + uring_scheduler* sched_ = nullptr; + uring_local_datagram_service* svc_ = nullptr; // fd_ and local_endpoint_ live in native_socket_base, which also // provides native_handle/is_open/set_option/get_option/local_endpoint. @@ -2601,7 +2748,7 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final } conn_.addrlen = to_sockaddr(ep, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), ep, + h, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, &local_endpoint_, token); conn_.cancelled.store(true, std::memory_order_release); sched_->work_started(); @@ -2616,7 +2763,7 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final // a prior speculative ::connect would leave EINPROGRESS → EALREADY. conn_.addrlen = to_sockaddr(ep, conn_.addr); conn_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), ep, &remote_endpoint_, + h, ex, ec, fd_, sched_, detail::object_ref(this), ep, &remote_endpoint_, &local_endpoint_, token); sched_->work_started(); if (conn_.cancelled.load(std::memory_order_acquire)) @@ -2650,7 +2797,7 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final break; } wait_op_.prepare( - h, ex, ec, fd_, sched_, shared_from_this(), poll_flags, token); + h, ex, ec, fd_, sched_, detail::object_ref(this), poll_flags, token); sched_->work_started(); if (wait_op_.cancelled.load(std::memory_order_acquire)) { @@ -2707,6 +2854,27 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final remote_endpoint_ = corosio::local_endpoint{}; } + /// Recycle into the owning service's pool. See + /// uring_tcp_socket::retire() for why this is out-of-line. + void retire() noexcept override; + + /** Reset op slots, cached endpoints, and the speculation hint for + recycling. See uring_tcp_socket::reuse() for the rationale. + + @pre refs_ == 0, fd closed, no op in flight. + */ + void reuse() noexcept + { + BOOST_COROSIO_ASSERT(fd_ == -1); + BOOST_COROSIO_ASSERT(local_endpoint_ == corosio::local_endpoint{}); + BOOST_COROSIO_ASSERT(remote_endpoint_ == corosio::local_endpoint{}); + BOOST_COROSIO_ASSERT(!conn_.stop_cb); + BOOST_COROSIO_ASSERT(!send_.stop_cb); + BOOST_COROSIO_ASSERT(!recv_.stop_cb); + BOOST_COROSIO_ASSERT(!wait_op_.stop_cb); + spec_.reset(); + } + corosio::local_endpoint remote_endpoint() const noexcept override { return remote_endpoint_; @@ -2793,7 +2961,7 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final return dispatch_coro(ex, send_.cont); } send_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, dest_len, dest_storage, to_native_msg_flags(flags), token); if (stop_now) @@ -2809,7 +2977,7 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final } send_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, buffers, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, dest_len, dest_storage, to_native_msg_flags(flags), token); sched_->work_started(); if (send_.cancelled.load(std::memory_order_acquire)) @@ -2888,7 +3056,7 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final return dispatch_coro(ex, recv_.cont); } recv_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, source_out, want_source ? &write_local_source : nullptr, to_native_msg_flags(flags), token); @@ -2915,7 +3083,7 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final } recv_.prepare( - h, ex, ec, bytes, fd_, sched_, shared_from_this(), &spec_, buffers, + h, ex, ec, bytes, fd_, sched_, detail::object_ref(this), &spec_, buffers, source_out, want_source ? &write_local_source : nullptr, to_native_msg_flags(flags), token); sched_->work_started(); @@ -2945,9 +3113,10 @@ class BOOST_COROSIO_DECL uring_local_datagram_socket final generic `local_datagram_socket` front-end can call `open_socket` and `bind_socket` transparently. - Socket impls are reference-counted inside the service map; raw - pointers returned from `construct()` remain valid until `destroy()` - or `shutdown()` is called. + Socket impls live in the service's `object_pool`. `construct()` + hands out a pooled impl with one reference already held; `destroy()` + releases it, and `retire()` recycles it onto the free list + instead of freeing it. @par Thread Safety All public member functions are thread-safe. @@ -3083,6 +3252,12 @@ class BOOST_COROSIO_DECL uring_local_datagram_service final } }; +inline void +uring_local_datagram_socket::retire() noexcept +{ + svc_->pool_.recycle(this); +} + } // namespace boost::corosio::detail #endif // BOOST_COROSIO_HAS_URING diff --git a/include/boost/corosio/tcp_server.hpp b/include/boost/corosio/tcp_server.hpp index 494cde682..ac42314f8 100644 --- a/include/boost/corosio/tcp_server.hpp +++ b/include/boost/corosio/tcp_server.hpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -171,10 +172,16 @@ class BOOST_COROSIO_DECL tcp_server template struct launch_wrapper { - struct promise_type + // frame_alloc_mixin routes the frame through the thread-local + // recycling allocator, so a warmed per-connection launch does + // not hit the global allocator. + struct promise_type : capy::frame_alloc_mixin { Ex ex; // Executor stored directly in frame (outlives child tasks) capy::io_env env_; + /// Embedded post node: the start posts this instead of the + /// bare handle, which would heap-allocate a wrapper op. + capy::continuation cont_; // For regular coroutines: first arg is executor, second is stop token template @@ -526,15 +533,25 @@ class BOOST_COROSIO_DECL tcp_server } } guard{srv_, w}; - // Reset worker's stop source for this connection - w->stop_ = {}; - auto st = w->stop_.get_token(); + // A stop_source allocates shared state on construction; + // reuse the worker's across connections and replace it + // only once a stop has actually been delivered (the state + // is then latched for good). + if (w->stop_.stop_requested()) + w->stop_ = {}; + auto st = w->stop_.get_token(); auto wrapper = launch_coro{}(ex, st, srv_, std::move(task), w); - // Executor and stop token stored in promise via constructor - ex.post(std::exchange(wrapper.h, nullptr)); // Release before post + // Executor and stop token stored in promise via + // constructor. Post through the frame-embedded + // continuation — posting the bare handle would allocate a + // wrapper op. The frame stays suspended until the + // executor resumes it, so the node outlives the queue. + auto h = std::exchange(wrapper.h, nullptr); // Release before post + h.promise().cont_.h = h; + ex.post(h.promise().cont_); guard.w = nullptr; // Success - dismiss guard } }; diff --git a/src/corosio/src/tcp_socket.cpp b/src/corosio/src/tcp_socket.cpp index f01d9ae15..1c6b08666 100644 --- a/src/corosio/src/tcp_socket.cpp +++ b/src/corosio/src/tcp_socket.cpp @@ -59,10 +59,8 @@ tcp_socket::open_for_family(int family, int type, int protocol) noexcept { #if BOOST_COROSIO_HAS_IOCP auto& svc = static_cast(h_.service()); - auto& wrapper = static_cast(*h_.get()); - std::error_code ec = svc.open_socket( - *static_cast(wrapper).get_internal(), family, - type, protocol); + auto& sock = static_cast(*h_.get()); + std::error_code ec = svc.open_socket(sock, family, type, protocol); #else auto& svc = static_cast(h_.service()); std::error_code ec = svc.open_socket( @@ -77,9 +75,8 @@ tcp_socket::assign(native_handle_type fd) noexcept { #if BOOST_COROSIO_HAS_IOCP auto& svc = static_cast(h_.service()); - auto& wrapper = static_cast(*h_.get()); - std::error_code ec = svc.assign_socket( - *static_cast(wrapper).get_internal(), fd); + auto& sock = static_cast(*h_.get()); + std::error_code ec = svc.assign_socket(sock, fd); #else auto& svc = static_cast(h_.service()); std::error_code ec = svc.assign_socket( @@ -104,10 +101,9 @@ tcp_socket::bind(endpoint ep) noexcept if (!is_open()) return make_error_code(std::errc::bad_file_descriptor); #if BOOST_COROSIO_HAS_IOCP - auto& svc = static_cast(h_.service()); - auto& wrapper = static_cast(*h_.get()); - return svc.bind_socket( - *static_cast(wrapper).get_internal(), ep); + auto& svc = static_cast(h_.service()); + auto& sock = static_cast(*h_.get()); + return svc.bind_socket(sock, ep); #else auto& svc = static_cast(h_.service()); return svc.bind_socket( diff --git a/src/corosio/src/timer.cpp b/src/corosio/src/timer.cpp index d28e9389b..5612045f0 100644 --- a/src/corosio/src/timer.cpp +++ b/src/corosio/src/timer.cpp @@ -20,6 +20,15 @@ timer::timer(capy::execution_context& ctx) { } +// Not inline for the same reason as wait() below: retire() needs +// timer_service's complete type (to reach pool_), which timer.hpp only +// forward-declares. +void +timer::implementation::retire() noexcept +{ + svc_->pool_.recycle(this); +} + // Not inline: wait_awaitable::await_suspend (defined in timer.hpp) calls // this from translation units that may never include timer_service.hpp, // so this must be the one strong definition the linker can always find @@ -60,7 +69,23 @@ timer::implementation::publish(waiter_node& w) if (w.token_->stop_possible()) w.arm_stop_cb(); - svc_->insert_waiter(*this, &w); + // insert_waiter() grows the heap before publishing anything (see + // its own comment), so a throw out of it leaves w.impl_ null and + // the waiter never visible to the fire/cancel paths above -- but + // work_started() and arm_stop_cb() already ran and must be + // unwound here, or the escaping exception leaves a dangling + // stop-callback registration (UAF on a later cancel) and a work + // count that never balances. + try + { + svc_->insert_waiter(*this, &w); + } + catch (...) + { + w.reset_stop_cb(); + svc_->get_scheduler().work_finished(); + throw; + } return std::noop_coroutine(); } diff --git a/test/unit/Jamfile b/test/unit/Jamfile index 261697224..b5df005da 100644 --- a/test/unit/Jamfile +++ b/test/unit/Jamfile @@ -38,13 +38,20 @@ project boost/corosio/test/unit # applies the exclusion patterns to directory entries too and does not # descend into a match. Those targets need the hook translation unit # linked in, which only fault/Jamfile does. -for local f in [ glob-tree-ex . : *.cpp : openssl_stream.cpp wolfssl_stream.cpp cross_ssl_stream.cpp tls_stream.cpp tls_stream_stress.cpp iocp_shutdown.cpp iocp_error_map.cpp openssl_engine.cpp wolfssl_engine.cpp cross_engine.cpp fault ] +for local f in [ glob-tree-ex . : *.cpp : openssl_stream.cpp wolfssl_stream.cpp cross_ssl_stream.cpp tls_stream.cpp tls_stream_stress.cpp iocp_shutdown.cpp iocp_error_map.cpp openssl_engine.cpp wolfssl_engine.cpp cross_engine.cpp zero_alloc.cpp fault ] { run $(f) ; } build-project fault ; +# zero_alloc.cpp counts against alloc_counter.hpp's extern counters, whose +# one definition (plus the global operator new/delete override) lives in +# continuation_post.cpp. Unlike the CMake build, which links every test +# source into a single binary, each b2 `run` target is its own executable, +# so the extra translation unit must be named explicitly here. +run zero_alloc.cpp continuation_post.cpp ; + # IOCP-specific native tests (Windows host only; skip on other hosts to # avoid triggering a cross-compile attempt). if [ os.name ] = NT diff --git a/test/unit/alloc_counter.hpp b/test/unit/alloc_counter.hpp new file mode 100644 index 000000000..5b3f16ae7 --- /dev/null +++ b/test/unit/alloc_counter.hpp @@ -0,0 +1,85 @@ +// +// Copyright (c) 2026 Steve Gerbino +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// Official repository: https://github.com/cppalliance/corosio +// + +#ifndef BOOST_COROSIO_TEST_ALLOC_COUNTER_HPP +#define BOOST_COROSIO_TEST_ALLOC_COUNTER_HPP + +/* Shared global operator new/delete counting interposer. + + The replaceable global allocation functions are process-wide and + may be defined exactly once; continuation_post.cpp owns that one + definition (guarded against TSan, which replaces them itself). + Every other test that needs to assert an allocation count -- + zero_alloc.cpp included -- counts against these same extern + counters instead of layering on a second interposer, which would + be a link error. +*/ + +#include +#include +#include + +// TSan's runtime already replaces global operator new/delete, so a +// counting replacement cannot link under -fsanitize=thread (and the +// counts would reflect the sanitizer's allocator anyway). +#if defined(__SANITIZE_THREAD__) +#define COROSIO_TEST_HAS_TSAN 1 +#elif defined(__has_feature) +#if __has_feature(thread_sanitizer) +#define COROSIO_TEST_HAS_TSAN 1 +#endif +#endif + +namespace boost::corosio { + +#ifndef COROSIO_TEST_HAS_TSAN +/// True while allocations should be tallied into alloc_count. +extern std::atomic alloc_armed; +/// Running count of operator new calls while alloc_armed is set. +extern std::atomic alloc_count; + +/** Check that the counting interposer actually runs, loud-skipping if not. + + ASan and TSan are excluded at compile time, but Valgrind replaces + the global allocator at run time: the interposer never executes, + every count reads zero, and a gate would either fail spuriously + (a documented non-zero pin) or pass vacuously (a zero pin). The + probe tests the behavior rather than the tool, as the fault + harness's allocation probe does, so any runtime replacement is + caught; the skip names its reason on stderr, as the fault suites' + Valgrind skip does, so the leg passes with a visible cause. + + @param suite Name printed in the skip notice. + + @return `true` if allocations are observable and the gate may run. +*/ +inline bool +alloc_counter_is_live(char const* suite) noexcept +{ + // Called through a volatile pointer so the probe allocation + // cannot be elided. + void* (*volatile op_new)(std::size_t) = &::operator new; + alloc_count.store(0, std::memory_order_relaxed); + alloc_armed.store(true, std::memory_order_relaxed); + ::operator delete(op_new(1)); + alloc_armed.store(false, std::memory_order_relaxed); + if (alloc_count.load(std::memory_order_relaxed) != 0) + return true; + std::fprintf( + stderr, + "%s: the global allocator is replaced at run time (Valgrind?), " + "so allocation counts are unobservable; skipping this suite\n", + suite); + return false; +} +#endif + +} // namespace boost::corosio + +#endif diff --git a/test/unit/continuation_post.cpp b/test/unit/continuation_post.cpp index 10aff4586..017809d14 100644 --- a/test/unit/continuation_post.cpp +++ b/test/unit/continuation_post.cpp @@ -16,10 +16,12 @@ #include #include +#include #include #include #include +#include "alloc_counter.hpp" #include "context.hpp" #include "test_suite.hpp" @@ -53,32 +55,76 @@ bare_post_task(capy::continuation* cont, bool& ran) ran = true; } -// TSan's runtime already replaces global operator new/delete, so the -// counting replacements below cannot link under -fsanitize=thread (and -// the counts would reflect the sanitizer's allocator anyway). -#if defined(__SANITIZE_THREAD__) -#define COROSIO_TEST_HAS_TSAN 1 -#elif defined(__has_feature) -#if __has_feature(thread_sanitizer) -#define COROSIO_TEST_HAS_TSAN 1 -#endif -#endif +} // namespace +// Definitions for alloc_counter.hpp's extern declarations. Not in the +// anonymous namespace above: zero_alloc.cpp needs these by extern +// reference, and an unnamed namespace's members are unreachable from +// another translation unit even though formally external-linkage. #ifndef COROSIO_TEST_HAS_TSAN std::atomic alloc_armed{false}; std::atomic alloc_count{0}; -#endif + +namespace { + +inline void +count_alloc() noexcept +{ + if (alloc_armed.load(std::memory_order_relaxed)) + alloc_count.fetch_add(1, std::memory_order_relaxed); +} + +/* Over-aligned new/delete, implemented without a platform-specific + aligned allocator (aligned_alloc/posix_memalign/_aligned_malloc all + differ in availability or requirements): stash the raw malloc() + pointer just before the returned block so the matching delete can + recover it, the same trick a custom allocator uses. +*/ +void* +aligned_alloc_impl(std::size_t n, std::size_t align) +{ + if (align < alignof(void*)) + align = alignof(void*); + std::size_t total = (n ? n : 1) + align + sizeof(void*); + void* raw = std::malloc(total); + if (!raw) + return nullptr; + auto base = reinterpret_cast(raw) + sizeof(void*); + auto aligned = (base + align - 1) & ~(align - 1); + *reinterpret_cast(aligned - sizeof(void*)) = raw; + return reinterpret_cast(aligned); +} + +void +aligned_free_impl(void* p) noexcept +{ + if (!p) + return; + void* raw = *reinterpret_cast( + reinterpret_cast(p) - sizeof(void*)); + std::free(raw); +} } // namespace +#endif } // namespace boost::corosio #ifndef COROSIO_TEST_HAS_TSAN +// Plain forms void* operator new(std::size_t n) { - if (boost::corosio::alloc_armed.load(std::memory_order_relaxed)) - boost::corosio::alloc_count.fetch_add(1, std::memory_order_relaxed); + boost::corosio::count_alloc(); + if (void* p = std::malloc(n ? n : 1)) + return p; + throw std::bad_alloc{}; +} + +void* +operator new[](std::size_t n) +{ + boost::corosio::count_alloc(); if (void* p = std::malloc(n ? n : 1)) return p; throw std::bad_alloc{}; @@ -94,6 +140,111 @@ operator delete(void* p, std::size_t) noexcept { std::free(p); } +void +operator delete[](void* p) noexcept +{ + std::free(p); +} +void +operator delete[](void* p, std::size_t) noexcept +{ + std::free(p); +} + +// nothrow forms +void* +operator new(std::size_t n, std::nothrow_t const&) noexcept +{ + boost::corosio::count_alloc(); + return std::malloc(n ? n : 1); +} +void* +operator new[](std::size_t n, std::nothrow_t const&) noexcept +{ + boost::corosio::count_alloc(); + return std::malloc(n ? n : 1); +} +void +operator delete(void* p, std::nothrow_t const&) noexcept +{ + std::free(p); +} +void +operator delete[](void* p, std::nothrow_t const&) noexcept +{ + std::free(p); +} + +// Over-aligned forms (std::align_val_t) -- an aligned allocation on a +// measured path would otherwise bypass this interposer silently. +void* +operator new(std::size_t n, std::align_val_t align) +{ + boost::corosio::count_alloc(); + if (void* p = boost::corosio::aligned_alloc_impl( + n, static_cast(align))) + return p; + throw std::bad_alloc{}; +} +void* +operator new[](std::size_t n, std::align_val_t align) +{ + boost::corosio::count_alloc(); + if (void* p = boost::corosio::aligned_alloc_impl( + n, static_cast(align))) + return p; + throw std::bad_alloc{}; +} +void +operator delete(void* p, std::align_val_t) noexcept +{ + boost::corosio::aligned_free_impl(p); +} +void +operator delete(void* p, std::size_t, std::align_val_t) noexcept +{ + boost::corosio::aligned_free_impl(p); +} +void +operator delete[](void* p, std::align_val_t) noexcept +{ + boost::corosio::aligned_free_impl(p); +} +void +operator delete[](void* p, std::size_t, std::align_val_t) noexcept +{ + boost::corosio::aligned_free_impl(p); +} + +// Over-aligned nothrow forms +void* +operator new(std::size_t n, std::align_val_t align, std::nothrow_t const&) + noexcept +{ + boost::corosio::count_alloc(); + return boost::corosio::aligned_alloc_impl( + n, static_cast(align)); +} +void* +operator new[](std::size_t n, std::align_val_t align, std::nothrow_t const&) + noexcept +{ + boost::corosio::count_alloc(); + return boost::corosio::aligned_alloc_impl( + n, static_cast(align)); +} +void +operator delete( + void* p, std::align_val_t, std::nothrow_t const&) noexcept +{ + boost::corosio::aligned_free_impl(p); +} +void +operator delete[]( + void* p, std::align_val_t, std::nothrow_t const&) noexcept +{ + boost::corosio::aligned_free_impl(p); +} #endif namespace boost::corosio { @@ -150,6 +301,8 @@ struct continuation_zero_alloc_test void run() { + if (!alloc_counter_is_live("boost.corosio.continuation_post.zero_alloc")) + return; testPostIsZeroAlloc(); } }; diff --git a/test/unit/fault/fault_posix.cpp b/test/unit/fault/fault_posix.cpp index 24938fd81..eb5fcd82a 100644 --- a/test/unit/fault/fault_posix.cpp +++ b/test/unit/fault/fault_posix.cpp @@ -57,6 +57,77 @@ #define O_TMPFILE 0 #endif +// Sanitizer runtimes intercept part of the census for their own +// bookkeeping (ASan's thread registry, TSan's happens-before graph). +// When these shadows are linked into the executable they can bind +// ahead of the runtime's interceptors (observed with clang >= 23's +// static ASan runtime: `ASAN_OPTIONS=verbosity=1` reports "failed to +// intercept 'pthread_create'", and the first pthread_join aborts with +// "Joining already joined thread" because the thread was created +// behind the runtime's back). Each sanitizer exports its interceptor +// as `__interceptor_` precisely so an interposer like this one +// can cooperate: chaining into it keeps the bookkeeping intact and +// still reaches libc. The references are weak, so a name the runtime +// does not intercept resolves to null and falls back to the plain +// RTLD_NEXT lookup; in unsanitized builds every reference is null. +#if defined(__linux__) +#if defined(__SANITIZE_ADDRESS__) || defined(__SANITIZE_THREAD__) +#define COROSIO_FAULT_SANITIZER 1 +#elif defined(__has_feature) +#if __has_feature(address_sanitizer) || __has_feature(thread_sanitizer) || \ + __has_feature(memory_sanitizer) +#define COROSIO_FAULT_SANITIZER 1 +#endif +#endif +#endif +#if !defined(COROSIO_FAULT_SANITIZER) +#define COROSIO_FAULT_SANITIZER 0 +#endif + +#if COROSIO_FAULT_SANITIZER + +// The libc half of the census (the SSL and liburing entries have no +// sanitizer interceptors). A name listed here that the runtime does +// not actually intercept costs nothing: its weak reference is null. +#define COROSIO_FAULT_INTERCEPTED_SYMBOLS(X) \ + X(socket) X(socketpair) X(bind) X(listen) X(accept) X(connect) \ + X(getsockname) X(getpeername) X(getsockopt) X(setsockopt) \ + X(shutdown) X(close) X(read) X(write) X(readv) X(preadv) \ + X(pwritev) X(recv) X(send) X(recvmsg) X(sendmsg) X(poll) X(pipe) \ + X(fcntl) X(ioctl) X(open) X(fstat) X(ftruncate) X(fsync) \ + X(unlink) X(sigaction) X(pthread_create) X(getaddrinfo) \ + X(freeaddrinfo) X(getnameinfo) X(gethostname) X(fdatasync) \ + X(posix_fadvise) X(select) X(accept4) X(lseek) X(epoll_create1) \ + X(epoll_ctl) X(epoll_wait) X(eventfd) X(timerfd_create) \ + X(timerfd_settime) X(__read_chk) X(__recv_chk) \ + X(__recvfrom_chk) X(__poll_chk) X(__pread64_chk) X(__open_2) \ + X(__gethostname_chk) + +// NOLINTNEXTLINE(bugprone-macro-parentheses) — token-paste operand cannot be parenthesized +#define COROSIO_FAULT_DECLARE_INTERCEPTOR(name) \ + extern "C" void __interceptor_##name() __attribute__((weak)); +COROSIO_FAULT_INTERCEPTED_SYMBOLS(COROSIO_FAULT_DECLARE_INTERCEPTOR) +#undef COROSIO_FAULT_DECLARE_INTERCEPTOR + +namespace { + +struct sanitizer_interceptor +{ + char const* name; + void (*fn)(); +}; + +sanitizer_interceptor const sanitizer_interceptors[] = { +// NOLINTNEXTLINE(bugprone-macro-parentheses) — token-paste operand cannot be parenthesized +#define COROSIO_FAULT_INTERCEPTOR_ENTRY(name) {#name, &__interceptor_##name}, + COROSIO_FAULT_INTERCEPTED_SYMBOLS(COROSIO_FAULT_INTERCEPTOR_ENTRY) +#undef COROSIO_FAULT_INTERCEPTOR_ENTRY +}; + +} // namespace + +#endif // COROSIO_FAULT_SANITIZER + namespace boost::corosio::test::fault { thread_local cqe_slot tls_cqe; @@ -72,6 +143,13 @@ publish_error(int err) noexcept void* real_symbol(char const* name) noexcept { +#if COROSIO_FAULT_SANITIZER + // Chain into the sanitizer's own interceptor when it has one for + // this name — see the block comment above the table. + for (auto const& e : sanitizer_interceptors) + if (e.fn != nullptr && std::strcmp(e.name, name) == 0) + return reinterpret_cast(e.fn); +#endif void* p = ::dlsym(RTLD_NEXT, name); if (!p) { diff --git a/test/unit/object_pool.cpp b/test/unit/object_pool.cpp new file mode 100644 index 000000000..83ff0a3d4 --- /dev/null +++ b/test/unit/object_pool.cpp @@ -0,0 +1,216 @@ +#include +#include +#include +#include "test_suite.hpp" + +namespace boost::corosio::detail { + +namespace { + +struct pool_impl final + : io_object::implementation + , intrusive_list::node +{ + static inline int instances = 0; + + object_pool* pool = nullptr; + int reused_count = 0; + + pool_impl() { ++instances; } + ~pool_impl() override { --instances; } + + void reuse() noexcept { ++reused_count; } + void retire() noexcept override { pool->recycle(this); } +}; + +// Exercises the sanctioned-extension mechanism documented on +// object_pool: a derived pool re-exposing adopt()/remove() as public, +// the same way timer_object_pool does for the timer service. +struct test_pool : object_pool +{ + using object_pool::adopt; + using object_pool::remove; +}; + +// Fixture for the shutting-down-gated reentrant delete branch in +// recycle(): an impl holding an embedded object_ref to itself never +// reaches zero through ordinary release() calls, so the only thing +// that can retire it is ~object_pool()'s own sweep destroying that +// embedded ref as part of `delete impl` -- a reentrant recycle() +// call made from inside the outer delete. +struct self_ref_impl final + : io_object::implementation + , intrusive_list::node +{ + static inline int instances = 0; + + object_pool* pool = nullptr; + object_ref self_ref; + + self_ref_impl() { ++instances; } + ~self_ref_impl() override { --instances; } + + void reuse() noexcept { } + void retire() noexcept override { pool->recycle(this); } +}; + +} // namespace + +struct object_pool_test +{ + void run() + { + // acquire news on a cold pool and hands the same impl back, + // reuse()-reset, after it is recycled + { + object_pool pool; + auto* a = pool.acquire(); + a->pool = &pool; + BOOST_TEST_EQ(a->reused_count, 0); + release(a); // service ref -> recycle + BOOST_TEST_EQ(pool_impl::instances, 1); + auto* b = pool.acquire(); // warm: recycled, not newed + BOOST_TEST_EQ(b, a); + BOOST_TEST_EQ(b->reused_count, 1); + BOOST_TEST_EQ(pool_impl::instances, 1); + release(b); // back to free; dtor frees + } + BOOST_TEST_EQ(pool_impl::instances, 0); + // keepalive defers recycling until the last ref drops + { + object_pool pool; + auto* a = pool.acquire(); + a->pool = &pool; + object_ref op_ref(a); // simulated in-flight op + release(a); // service ref gone... + auto* b = pool.acquire(); // ...a not recycled: news + b->pool = &pool; + BOOST_TEST_NE(b, a); + BOOST_TEST_EQ(pool_impl::instances, 2); + op_ref.reset(); // op drains -> a recycles + auto* c = pool.acquire(); // warm again + BOOST_TEST_EQ(c, a); + release(b); + release(c); + } + BOOST_TEST_EQ(pool_impl::instances, 0); + // shutdown mode deletes instead of pooling + { + object_pool pool; + auto* a = pool.acquire(); + a->pool = &pool; + pool.shutdown([](pool_impl*) {}); + release(a); // deleted, not pooled + BOOST_TEST_EQ(pool_impl::instances, 0); + } + // remove() transfers ownership out of the pool entirely: the + // impl leaves live_ and the caller now owns it outright (a + // direct delete here must not be seen again by the pool's own + // destructor -- the scope exit below proves no double free). + { + test_pool pool; + auto* a = pool.acquire(); + a->pool = &pool; + BOOST_TEST(pool.remove(a)); // unlinked from live_ + delete a; // caller-owned now + // pool goes out of scope here with empty live_/free_: if + // remove() had failed to unlink `a`, ~object_pool() would + // delete it a second time. + } + BOOST_TEST_EQ(pool_impl::instances, 0); + // remove() is idempotent: a second call on an already-detached + // impl returns false instead of corrupting the list (mirrors + // intrusive_list::remove()'s own already-detached contract). + { + test_pool pool; + auto* a = pool.acquire(); + a->pool = &pool; + BOOST_TEST(pool.remove(a)); + BOOST_TEST(!pool.remove(a)); // already detached + delete a; + } + // Sentinel guard: remove() must refuse to touch a free_-parked + // impl (the same hazard recycle() guards against in reverse). + // The guard's debug-build path asserts and aborts before + // returning, so only its release-build-observable contract -- + // refused, false, free list left intact -- can be exercised + // here; it is skipped entirely when assertions are compiled in + // (the normal state of this test binary). +#ifdef NDEBUG + { + test_pool pool; + auto* a = pool.acquire(); + a->pool = &pool; + release(a); // parks a on free_ (refs_ == pooled_sentinel) + BOOST_TEST(!pool.remove(a)); // refused + auto* b = pool.acquire(); // free_ untouched: warm hit + BOOST_TEST_EQ(b, a); + release(b); // back to free_; pool dtor frees it + } +#endif + BOOST_TEST_EQ(pool_impl::instances, 0); + // recycle()'s own guards, release-build contract, mirroring + // the remove() block above: both skip entirely under + // assertions (the normal state of this test binary) because + // the guarded branch asserts and aborts before returning. +#ifdef NDEBUG + { + // Sentinel guard: a duplicate recycle() on an impl + // already parked in free_ (e.g. a double retire() for + // one zero-crossing) must refuse before touching either + // list, not corrupt free_ by removing it through live_'s + // boundary pointers. + test_pool pool; + auto* a = pool.acquire(); + a->pool = &pool; + release(a); // 1 -> 0: recycle() parks a on free_ + BOOST_TEST_EQ(pool_impl::instances, 1); + pool.recycle(a); // duplicate retire while parked: refused + auto* b = pool.acquire(); // free_ untouched: still a warm hit + BOOST_TEST_EQ(b, a); + release(b); + } + BOOST_TEST_EQ(pool_impl::instances, 0); + { + // Accounting-bug branch: an impl detached from live_ + // (not parked, not shutting down) reaching recycle() is + // deleted outright instead of being stranded (reachable + // through neither acquire() nor shutdown()'s visit while + // still allocated). + test_pool pool; + auto* a = pool.acquire(); + a->pool = &pool; + BOOST_TEST(pool.remove(a)); // detach from live_ without recycling + pool.recycle(a); // not live, not pooled: deleted + BOOST_TEST_EQ(pool_impl::instances, 0); + } +#endif + // Shutting-down-gated reentrant delete: a self-referential + // impl's refcount never reaches zero through release() + // alone, so the only path that can retire it is + // ~object_pool()'s own sweep destroying the embedded + // object_ref as part of `delete impl`. shutdown() must run + // first (the destructor's own precondition), so the + // reentrant recycle() call lands in the shutting-down branch + // instead of trying to re-park a half-destroyed impl on + // free_ or being mistaken for the accounting-bug case. + { + object_pool pool; + auto* a = pool.acquire(); + a->pool = &pool; + a->self_ref = object_ref(a); // refs_: 1 -> 2 + release(a); // refs_: 2 -> 1; self-ref keeps it alive + BOOST_TEST_EQ(self_ref_impl::instances, 1); + pool.shutdown([](self_ref_impl*) {}); + // pool destructs here: the live_ sweep deletes `a`; + // its destructor tears down self_ref, releasing the + // last reference and reentering recycle() -- exactly + // one delete must happen, no crash, no double free. + } + BOOST_TEST_EQ(self_ref_impl::instances, 0); + } +}; + +TEST_SUITE(object_pool_test, "boost.corosio.object_pool"); + +} // namespace boost::corosio::detail diff --git a/test/unit/object_ref.cpp b/test/unit/object_ref.cpp new file mode 100644 index 000000000..f99d7069e --- /dev/null +++ b/test/unit/object_ref.cpp @@ -0,0 +1,96 @@ +#include +#include +#include "test_suite.hpp" + +namespace boost::corosio::detail { + +namespace { + +struct counted_impl final : io_object::implementation +{ + int* last_ref_hits; + explicit counted_impl(int* hits) : last_ref_hits(hits) {} + void retire() noexcept override { ++*last_ref_hits; } +}; + +} // namespace + +struct object_ref_test +{ + void run() + { + // acquire/release pairs; zero-action fires exactly once + { + int hits = 0; + counted_impl impl(&hits); // refs_ starts at 1 + release(&impl); // 1 -> 0 + BOOST_TEST_EQ(hits, 1); + } + // object_ref RAII: ctor acquires, dtor releases + { + int hits = 0; + counted_impl impl(&hits); + { + object_ref r(&impl); // 1 -> 2 + BOOST_TEST(r); + object_ref r2 = r; // copy: 2 -> 3 + object_ref r3 = std::move(r2); // move: still 3 + BOOST_TEST(!r2); + BOOST_TEST_EQ(r3.get(), &impl); + } // back to 1 + BOOST_TEST_EQ(hits, 0); + release(&impl); + BOOST_TEST_EQ(hits, 1); + } + // reset releases early + { + int hits = 0; + counted_impl impl(&hits); + object_ref r(&impl); + release(&impl); // service ref gone, r holds it + BOOST_TEST_EQ(hits, 0); + r.reset(); + BOOST_TEST_EQ(hits, 1); + } + // copy-assignment: operator=(object_ref const&) is never + // otherwise executed anywhere in the tree. Verify it releases + // the old target and acquires the new one, and is safe under + // self-assignment. + { + int hits_a = 0; + int hits_b = 0; + counted_impl a(&hits_a); + counted_impl b(&hits_b); + + object_ref ra(&a); // a.refs_: 1 -> 2 + object_ref rb(&b); // b.refs_: 1 -> 2 + + rb = ra; // rb releases b, acquires a + BOOST_TEST_EQ(rb.get(), &a); + BOOST_TEST_EQ( + a.refs_.load(), 3u); // service + ra + rb + release(&b); // b's last (service) ref + BOOST_TEST_EQ(hits_b, 1); // retired: rb no longer holds it + + // Deliberate self-assignment: must not release before + // acquiring. Laundered through a named alias rather than + // `rb = rb;` -- clang's -Wself-assign-overloaded flags the + // literal spelling even when it is intentional. + object_ref& rb_alias = rb; + rb = rb_alias; + BOOST_TEST_EQ(rb.get(), &a); + BOOST_TEST_EQ(a.refs_.load(), 3u); + BOOST_TEST_EQ(hits_a, 0); // no spurious retire + + ra.reset(); // a.refs_: 3 -> 2 + release(&a); // service ref: 2 -> 1 + BOOST_TEST_EQ(hits_a, 0); + rb.reset(); // a.refs_: 1 -> 0 -> retire + BOOST_TEST_EQ(hits_a, 1); + } + } +}; + +TEST_SUITE(object_ref_test, "boost.corosio.object_ref"); + +} // namespace boost::corosio::detail diff --git a/test/unit/object_reuse.cpp b/test/unit/object_reuse.cpp new file mode 100644 index 000000000..54d918a4a --- /dev/null +++ b/test/unit/object_reuse.cpp @@ -0,0 +1,688 @@ +// +// Copyright (c) 2026 Steve Gerbino +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// Official repository: https://github.com/cppalliance/corosio +// + +// Keepalive-extension regression for the object_pool conversion: dropping +// a socket while an op is still parked must not use-after-free the +// impl, and a socket constructed afterward on the same context must +// get a fully working impl, whether the pool recycled the dropped one +// or allocated a new one -- the caller cannot tell which, and must not +// need to. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "context.hpp" +#include "temp_path.hpp" +#include "test_suite.hpp" + +namespace boost::corosio { + +template +struct object_reuse_test +{ + // Open a connected pair, park a read on one side with no writer + // on the other, then drop that socket while the read is still + // parked. close()/destroy() cancel the read and release the + // service's own reference; the read's own reference (its object_ref + // keepalive) keeps the impl alive until the cancellation actually + // drains, at which point the impl is recycled -- not freed. A + // sanitizer build is what actually validates no UAF happened; this + // test's own assertion is the behavioral half: the context keeps + // working afterward. + void testAbandonedReadThenReuse() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + // Declared at function scope, not inside the block below: if + // the abandoned read's cancellation is ever actually delivered + // (reactor backends always drain it; io_uring may instead + // abandon it in the kernel ring at shutdown), the coroutine + // resumes into this lambda and writes `resumed` — which must + // still be live stack memory whenever that happens, including + // after the block below has already exited. + char buf[16]; + bool resumed = false; + + { + auto [s1, s2] = test::make_socket_pair(ioc); + std::ignore = s2; // keeps the peer fd open; never written to + + capy::run_async(ex)( + [](tcp_socket& s, char* b, std::size_t n, + bool& done) -> capy::task<> { + std::ignore = + co_await s.read_some(capy::mutable_buffer(b, n)); + done = true; + }(s1, buf, sizeof(buf), resumed)); + + // Let the read actually register as parked before s1 (and + // its parked read) go out of scope at the end of this block. + std::ignore = ioc.run_one(); + } + + // Drain whatever the destructor's cancellation produced. This + // must not crash, hang, or corrupt the pool regardless of + // whether the cancellation's completion ever arrives (io_uring + // may abandon it in the kernel ring at io_context shutdown; + // the reactor backends always drain it here). + ioc.run(); + ioc.restart(); + + // A fresh socket on the same context must get a working impl. + tcp_socket probe(ioc); + BOOST_TEST(!probe.open()); + BOOST_TEST(probe.is_open()); + + auto [p1, p2] = test::make_socket_pair(ioc); + char const out = 'x'; + char in = 0; + bool wrote = false, read_ok = false; + capy::run_async(ex)( + [](tcp_socket& s, char const* b, bool& done) -> capy::task<> { + auto [ec, n] = co_await s.write_some(capy::const_buffer(b, 1)); + done = !ec && n == 1; + }(p1, &out, wrote)); + capy::run_async(ex)( + [](tcp_socket& s, char* b, bool& done) -> capy::task<> { + auto [ec, n] = co_await s.read_some(capy::mutable_buffer(b, 1)); + done = !ec && n == 1; + }(p2, &in, read_ok)); + ioc.restart(); + ioc.run(); + + BOOST_TEST(wrote); + BOOST_TEST(read_ok); + BOOST_TEST_EQ(in, out); + } + + // io_uring's multishot accept can deliver a connection before any + // accept() is outstanding to claim it; that connection's fd (and + // tracking node) parks in the acceptor impl's ready_fds_ list. If + // the acceptor is destroyed with a parked fd still there, draining + // it was originally destructor-only -- which never runs on the + // recycle path (reaching zero references usually means "back on + // the free list", not "actually freed"). Reproduces deterministically: + // destroy an acceptor holding one buffered connection, then + // construct a fresh acceptor on the same context -- that pops the + // just-recycled impl and, before the fix, immediately failed + // reuse()'s `ready_fds_.empty()` assert. On epoll/select there is + // no ready_fds_/multishot concept, so this is a plain acceptor + // recycle-and-reuse sanity check there; the bug this guards against + // is uring-only. + void testAcceptorRecycleDrainsBufferedConnection() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + { + tcp_acceptor acc(ioc); + BOOST_TEST(!acc.open()); + acc.set_option(socket_option::reuse_address(true)); + BOOST_TEST(!acc.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!acc.listen()); + auto port = acc.local_endpoint().port(); + + tcp_socket client(ioc); + bool connected = false; + capy::run_async(ex)( + [](tcp_socket& s, endpoint ep, bool& done) -> capy::task<> { + auto [ec] = co_await s.connect(ep); + done = !ec; + }(client, endpoint(ipv4_address::loopback(), port), + connected)); + + // No accept() is outstanding: on uring the connection's + // multishot CQE buffers in ready_fds_ instead of completing + // an accept. run() (not poll()) drives the connector fully + // to completion, which is also where the server's accept + // CQE lands -- same idiom as + // multishot_acceptor_test::testDestroyWithBufferedConnections. + ioc.run(); + BOOST_TEST(connected); + + client.close(); + + // acc drops here holding a buffered connection in + // ready_fds_ (uring) or nothing to hold (epoll/select). + } + + ioc.restart(); + + // A fresh acceptor on the same context pops the just-recycled + // impl. Before the fix this aborted on reuse()'s + // `ready_fds_.empty()` assert (uring only). + tcp_acceptor acc2(ioc); + BOOST_TEST(!acc2.open()); + acc2.set_option(socket_option::reuse_address(true)); + BOOST_TEST(!acc2.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!acc2.listen()); + auto port2 = acc2.local_endpoint().port(); + + // Full exercise: accept a real connection through the recycled + // (or freshly allocated) impl to prove it still works. + tcp_socket server2(ioc); + tcp_socket client2(ioc); + bool accept_done = false, connect_done = false; + capy::run_async(ex)( + [](tcp_acceptor& a, tcp_socket& s, bool& done) -> capy::task<> { + auto [ec] = co_await a.accept(s); + done = !ec; + }(acc2, server2, accept_done)); + capy::run_async(ex)( + [](tcp_socket& s, endpoint ep, bool& done) -> capy::task<> { + auto [ec] = co_await s.connect(ep); + done = !ec; + }(client2, endpoint(ipv4_address::loopback(), port2), + connect_done)); + ioc.run(); + + BOOST_TEST(accept_done); + BOOST_TEST(connect_done); + BOOST_TEST(server2.is_open()); + } + + void run() + { + testAbandonedReadThenReuse(); + testAcceptorRecycleDrainsBufferedConnection(); + } +}; + +// Churns every pooled object kind through 32 construct/destroy cycles, +// then runs one representative op per kind so the exercise below runs +// entirely on recycled (not freshly allocated) impls. Registered on +// the reactor backends plus io_uring -- not IOCP, which is covered by +// its own object_pool conversion separately. +template +struct churn_then_exercise_test +{ + static constexpr int churn_n = 32; + + void churnSockets(io_context& ioc) + { + for (int i = 0; i < churn_n; ++i) + { + tcp_socket s(ioc); + } + } + + void churnAcceptors(io_context& ioc) + { + for (int i = 0; i < churn_n; ++i) + { + tcp_acceptor a(ioc); + } + } + + void churnUdp(io_context& ioc) + { + for (int i = 0; i < churn_n; ++i) + { + udp_socket u(ioc); + } + } + + void churnResolvers(io_context& ioc) + { + for (int i = 0; i < churn_n; ++i) + { + resolver r(ioc); + } + } + + void churnSignalSets(io_context& ioc) + { +#if BOOST_COROSIO_POSIX + for (int i = 0; i < churn_n; ++i) + { + signal_set s(ioc, SIGUSR1); + } +#else + // SIGUSR1 doesn't exist on Windows; this template is never + // instantiated there (COROSIO_NON_IOCP_BACKEND_TESTS registers + // nothing on Windows), but non-dependent names inside it must + // still resolve at definition time regardless of instantiation. + (void)ioc; +#endif + } + + void churnFiles(io_context& ioc) + { + test::temp_file tmp("object_reuse_churn_"); + for (int i = 0; i < churn_n; ++i) + { + stream_file f(ioc); + std::ignore = f.open( + tmp.path, + file_base::read_write | file_base::create | + file_base::truncate); + } + } + + void churnTimers(io_context& ioc, capy::executor_ref ex) + { + for (int i = 0; i < churn_n; ++i) + { + bool done = false; + capy::run_async(ex)([](bool& d) -> capy::task<> { + auto [ec] = co_await delay(std::chrono::milliseconds(0)); + d = !ec; + }(done)); + ioc.run(); + ioc.restart(); + BOOST_TEST(done); + } + } + + // socket + acceptor: accept, connect, read, write, then cancel a + // fresh read via stop_token -- every op the brief names, run on + // the impls churnSockets()/churnAcceptors() just recycled. + void exerciseSocketAndAcceptor(io_context& ioc, capy::executor_ref ex) + { + tcp_acceptor acc(ioc); + BOOST_TEST(!acc.open()); + acc.set_option(socket_option::reuse_address(true)); + BOOST_TEST(!acc.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!acc.listen()); + auto port = acc.local_endpoint().port(); + + tcp_socket server(ioc); + tcp_socket client(ioc); + bool accept_done = false, connect_done = false; + + capy::run_async(ex)( + [](tcp_acceptor& a, tcp_socket& s, bool& done) -> capy::task<> { + auto [ec] = co_await a.accept(s); + done = !ec; + }(acc, server, accept_done)); + capy::run_async(ex)( + [](tcp_socket& s, endpoint ep, bool& done) -> capy::task<> { + auto [ec] = co_await s.connect(ep); + done = !ec; + }(client, endpoint(ipv4_address::loopback(), port), + connect_done)); + ioc.run(); + ioc.restart(); + BOOST_TEST(accept_done); + BOOST_TEST(connect_done); + + char const out = 'q'; + char in = 0; + bool wrote = false, read_ok = false; + capy::run_async(ex)( + [](tcp_socket& s, char const* b, bool& done) -> capy::task<> { + auto [ec, n] = co_await s.write_some(capy::const_buffer(b, 1)); + done = !ec && n == 1; + }(client, &out, wrote)); + capy::run_async(ex)( + [](tcp_socket& s, char* b, bool& done) -> capy::task<> { + auto [ec, n] = co_await s.read_some(capy::mutable_buffer(b, 1)); + done = !ec && n == 1; + }(server, &in, read_ok)); + ioc.run(); + ioc.restart(); + BOOST_TEST(wrote); + BOOST_TEST(read_ok); + BOOST_TEST_EQ(in, out); + + // Cancel-via-stop_token: park a second read with no writer, + // then cancel it through the token instead of close()/ + // destroy() -- a different code path through the same + // recycled impl's op machinery. + std::stop_source src; + bool canceled = false; + capy::run_async(ex, src.get_token())( + [](tcp_socket& s, bool& done) -> capy::task<> { + char buf[1]; + auto [ec, n] = + co_await s.read_some(capy::mutable_buffer(buf, 1)); + std::ignore = n; + done = (ec == capy::cond::canceled); + }(client, canceled)); + ioc.run_one(); + src.request_stop(); + ioc.run(); + ioc.restart(); + BOOST_TEST(canceled); + + client.close(); + server.close(); + acc.close(); + } + + // udp: send_to/recv_from on the recycled churnUdp() impls. + void exerciseUdp(io_context& ioc, capy::executor_ref ex) + { + udp_socket receiver(ioc); + BOOST_TEST(!receiver.open()); + BOOST_TEST(!receiver.bind(endpoint(ipv4_address::loopback(), 0))); + auto rport = receiver.local_endpoint().port(); + + udp_socket sender(ioc); + BOOST_TEST(!sender.open()); + BOOST_TEST(!sender.bind(endpoint(ipv4_address::loopback(), 0))); + + char const out = 'u'; + char in = 0; + bool sent = false, received = false; + endpoint dest(ipv4_address::loopback(), rport); + + capy::run_async(ex)( + [](udp_socket& s, char const* b, endpoint ep, + bool& done) -> capy::task<> { + auto [ec, n] = + co_await s.send_to(capy::const_buffer(b, 1), ep); + done = !ec && n == 1; + }(sender, &out, dest, sent)); + endpoint source; + capy::run_async(ex)( + [](udp_socket& s, char* b, endpoint& src, + bool& done) -> capy::task<> { + auto [ec, n] = + co_await s.recv_from(capy::mutable_buffer(b, 1), src); + done = !ec && n == 1; + }(receiver, &in, source, received)); + ioc.run(); + ioc.restart(); + + BOOST_TEST(sent); + BOOST_TEST(received); + BOOST_TEST_EQ(in, out); + } + + // resolver: resolve localhost on a recycled churnResolvers() impl. + void exerciseResolver(io_context& ioc, capy::executor_ref ex) + { + resolver r(ioc); + bool done = false; + std::error_code resolve_ec; + + capy::run_async(ex)( + [](resolver& r_ref, bool& done_out, + std::error_code& ec_out) -> capy::task<> { + auto [ec, results] = co_await r_ref.resolve("localhost", "80"); + std::ignore = results; + ec_out = ec; + done_out = true; + }(r, done, resolve_ec)); + ioc.run(); + ioc.restart(); + + BOOST_TEST(done); + BOOST_TEST(!resolve_ec); + } + + void run() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + churnSockets(ioc); + churnAcceptors(ioc); + churnUdp(ioc); + churnResolvers(ioc); + churnSignalSets(ioc); + churnFiles(ioc); + churnTimers(ioc, ex); + + exerciseSocketAndAcceptor(ioc, ex); + exerciseUdp(ioc, ex); + exerciseResolver(ioc, ex); + + // signal_set: wait() completed by raise() on a recycled + // churnSignalSets() impl. +#if BOOST_COROSIO_POSIX + { + signal_set sig(ioc, SIGUSR1); + bool done = false; + capy::run_async(ex)( + [](signal_set& s, bool& done_out) -> capy::task<> { + [[maybe_unused]] auto [ec, signum] = co_await s.wait(); + done_out = !ec; + }(sig, done)); + ioc.run_one(); + std::raise(SIGUSR1); + ioc.run(); + ioc.restart(); + BOOST_TEST(done); + } +#endif + + // files: write_some on a recycled churnFiles() impl. + { + test::temp_file tmp("object_reuse_exercise_"); + stream_file f(ioc); + BOOST_TEST(!f.open( + tmp.path, + file_base::read_write | file_base::create | + file_base::truncate)); + bool done = false; + char const data = 'f'; + capy::run_async(ex)( + [](stream_file& file, char const* b, + bool& done_out) -> capy::task<> { + auto [ec, n] = + co_await file.write_some(capy::const_buffer(b, 1)); + done_out = !ec && n == 1; + }(f, &data, done)); + ioc.run(); + ioc.restart(); + BOOST_TEST(done); + } + + // timer: one more delay() wait on a recycled churnTimers() impl. + { + bool done = false; + capy::run_async(ex)([](bool& d) -> capy::task<> { + auto [ec] = co_await delay(std::chrono::milliseconds(0)); + d = !ec; + }(done)); + ioc.run(); + ioc.restart(); + BOOST_TEST(done); + } + } +}; + +COROSIO_BACKEND_TESTS(object_reuse_test, "boost.corosio.object_reuse") +COROSIO_NON_IOCP_BACKEND_TESTS( + churn_then_exercise_test, "boost.corosio.object_reuse.churn") + +// AF_UNIX churn lane. local_stream_socket/local_stream_acceptor/ +// local_datagram_socket instantiate the exact same reactor_stream_ +// socket/reactor_acceptor/reactor_datagram_socket templates patched +// above, just parameterized on local_endpoint instead of endpoint -- +// local_endpoint_ is precisely the field poison() caught a missing +// reset for, and nothing else loops these types the way +// churn_then_exercise_test loops the IP-socket kinds. Windows +// loopback rules don't apply: this never binds an IP endpoint, only +// AF_UNIX paths, which are POSIX-only on epoll/select and also +// supported on Windows 10 1803+ -- irrelevant here since +// COROSIO_NON_IOCP_BACKEND_TESTS registers nothing on Windows anyway. +template +struct local_socket_churn_then_exercise_test +{ + static constexpr int churn_n = 32; + + void churnStreamSockets(io_context& ioc) + { + for (int i = 0; i < churn_n; ++i) + { + local_stream_socket s(ioc); + } + } + + void churnAcceptors(io_context& ioc) + { + for (int i = 0; i < churn_n; ++i) + { + local_stream_acceptor a(ioc); + } + } + + void churnDatagramSockets(io_context& ioc) + { +#if BOOST_COROSIO_POSIX + for (int i = 0; i < churn_n; ++i) + { + local_datagram_socket u(ioc); + } +#else + // local_datagram_socket is POSIX-only (local_datagram_socket.hpp). + // This template is never instantiated on Windows + // (COROSIO_NON_IOCP_BACKEND_TESTS registers nothing there), but + // non-dependent names inside it must still resolve at definition + // time regardless of instantiation. + (void)ioc; +#endif + } + + // local_stream_socket + local_stream_acceptor: accept, connect, + // echo one byte each way, then close -- the same shape as + // churn_then_exercise_test's TCP exercise, run on the AF_UNIX + // impls churnStreamSockets()/churnAcceptors() just recycled. + void exerciseStreamAndAcceptor(io_context& ioc, capy::executor_ref ex) + { + test::temp_socket_dir dir; + local_endpoint ep(dir.path()); + + local_stream_acceptor acc(ioc, ep); + BOOST_TEST(acc.is_open()); + + local_stream_socket server(ioc); + local_stream_socket client(ioc); + bool accept_done = false, connect_done = false; + + capy::run_async(ex)( + [](local_stream_acceptor& a, local_stream_socket& s, + bool& done) -> capy::task<> { + auto [ec] = co_await a.accept(s); + done = !ec; + }(acc, server, accept_done)); + capy::run_async(ex)( + [](local_stream_socket& s, local_endpoint ep_, + bool& done) -> capy::task<> { + auto [ec] = co_await s.connect(ep_); + done = !ec; + }(client, ep, connect_done)); + ioc.run(); + ioc.restart(); + BOOST_TEST(accept_done); + BOOST_TEST(connect_done); + + char const out = 'l'; + char in = 0; + bool wrote = false, read_ok = false; + capy::run_async(ex)( + [](local_stream_socket& s, char const* b, + bool& done) -> capy::task<> { + auto [ec, n] = co_await s.write_some(capy::const_buffer(b, 1)); + done = !ec && n == 1; + }(client, &out, wrote)); + capy::run_async(ex)( + [](local_stream_socket& s, char* b, bool& done) -> capy::task<> { + auto [ec, n] = + co_await s.read_some(capy::mutable_buffer(b, 1)); + done = !ec && n == 1; + }(server, &in, read_ok)); + ioc.run(); + ioc.restart(); + BOOST_TEST(wrote); + BOOST_TEST(read_ok); + BOOST_TEST_EQ(in, out); + + client.close(); + server.close(); + acc.close(); + } + + // local_datagram_socket: connect_pair() + connected-mode send()/ + // recv(), on the recycled impls churnDatagramSockets() just + // warmed. + void exerciseDatagram(io_context& ioc, capy::executor_ref ex) + { +#if BOOST_COROSIO_POSIX + local_datagram_socket a(ioc); + local_datagram_socket b(ioc); + BOOST_TEST(!connect_pair(a, b)); + + char const out = 'd'; + char in = 0; + bool sent = false, received = false; + + capy::run_async(ex)( + [](local_datagram_socket& s, char const* buf, + bool& done) -> capy::task<> { + auto [ec, n] = co_await s.send(capy::const_buffer(buf, 1)); + done = !ec && n == 1; + }(a, &out, sent)); + capy::run_async(ex)( + [](local_datagram_socket& s, char* buf, + bool& done) -> capy::task<> { + auto [ec, n] = co_await s.recv(capy::mutable_buffer(buf, 1)); + done = !ec && n == 1; + }(b, &in, received)); + ioc.run(); + ioc.restart(); + + BOOST_TEST(sent); + BOOST_TEST(received); + BOOST_TEST_EQ(in, out); + + a.close(); + b.close(); +#else + (void)ioc; + (void)ex; +#endif + } + + void run() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + churnStreamSockets(ioc); + churnAcceptors(ioc); + churnDatagramSockets(ioc); + + exerciseStreamAndAcceptor(ioc, ex); + exerciseDatagram(ioc, ex); + } +}; + +COROSIO_NON_IOCP_BACKEND_TESTS( + local_socket_churn_then_exercise_test, + "boost.corosio.object_reuse.churn_local") + +} // namespace boost::corosio diff --git a/test/unit/random_access_file.cpp b/test/unit/random_access_file.cpp index 8186aa653..346e56eae 100644 --- a/test/unit/random_access_file.cpp +++ b/test/unit/random_access_file.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include "context.hpp" @@ -36,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -719,6 +721,94 @@ struct random_access_file_test BOOST_TEST(!resumed); } + // A completion whose awaiting coroutine runs on a non-io_context + // executor takes dispatch_coro's deferring branch: the executor + // queues the op's EMBEDDED continuation by reference and returns + // noop, after which the handler recycles the op. A second + // initiation can then pop the recycled op and its own inline + // completion overwrites the continuation the foreign queue still + // points into — the queued resume then targets the wrong (already + // finished) coroutine and the first awaiter is lost. + // + // Sequencing is causal, not timed: a single-worker pool runs C1 + // up to its submission, then parks on a latch (FIFO guarantees + // C1 suspended first); the io_context thread processes C1's + // completion (queued into the parked pool), refills the op via + // C2, and only then releases the worker. + void testDeferredExecutorCompletion() + { + io_context ioc(Backend); + capy::thread_pool pool(1); + + temp_file tmp("raf_deferred_", "abcdefgh"); + random_access_file f(ioc); + BOOST_TEST(!f.open(tmp.path, file_base::read_only)); + + std::latch started(1); + std::latch hold(1); + std::atomic c1_done{false}; + std::atomic c2_done{false}; + char b1[4] = {}; + char b2[4] = {}; + + // C1: initiates from the pool worker; completion must be + // dispatched back to the pool executor (the deferring path). + capy::run_async(pool.get_executor())( + [](random_access_file& fl, char* b, + std::atomic& done) -> capy::task<> + { + auto [ec, n] = + co_await fl.read_some_at(0, capy::mutable_buffer(b, 4)); + done = !ec && n == 4; + }(f, b1, c1_done)); + + // Parked behind C1 on the single worker: when this runs, C1 + // has already submitted and suspended. + capy::run_async(pool.get_executor())( + [](std::latch& s, std::latch& h) -> capy::task<> + { + s.count_down(); + h.wait(); + co_return; + }(started, hold)); + + started.wait(); + + // Process C1's completion: the handler queues C1's resume + // into the (parked) pool and recycles the op. + ioc.run(); + ioc.restart(); + + // Refill: C2 pops the recycled op; its inline completion + // reuses the op's continuation while the pool still holds it. + capy::run_async(ioc.get_executor())( + [](random_access_file& fl, char* b, + std::atomic& done) -> capy::task<> + { + auto [ec, n] = + co_await fl.read_some_at(0, capy::mutable_buffer(b, 4)); + done = !ec && n == 4; + }(f, b2, c2_done)); + ioc.run(); + ioc.restart(); + BOOST_TEST(c2_done.load()); + + // Release the worker; it now dequeues what it believes is + // C1's continuation. Drain with a sentinel queued after it. + hold.count_down(); + std::latch drained(1); + capy::run_async(pool.get_executor())( + [](std::latch& d) -> capy::task<> + { + d.count_down(); + co_return; + }(drained)); + drained.wait(); + + BOOST_TEST(c1_done.load()); + pool.join(); + } + void run() { testConstruction(); @@ -757,6 +847,11 @@ struct random_access_file_test testClosedFileErrors(); testClosedAtOpsComplete(); testStopRaceReportsTransfer(); + // The recycling hazard under test is uring's (pooled op + + // deferring dispatch); the property asserted — an awaiter on + // a foreign executor resumes exactly once — holds on every + // backend, so all of them run it. + testDeferredExecutorCompletion(); #if BOOST_COROSIO_POSIX testSyncOnPipeFails(); testHugeOffsetFails(); diff --git a/test/unit/resolver.cpp b/test/unit/resolver.cpp index 85ea2d2db..c5476c8c2 100644 --- a/test/unit/resolver.cpp +++ b/test/unit/resolver.cpp @@ -1297,7 +1297,7 @@ struct resolver_test // Forward resolution on IOCP dispatches through GetAddrInfoExW and // posts its completion (resolve_op, embedded in the win_resolver) to // the scheduler queue. Unlike the reverse path and both POSIX paths, - // the forward op takes no shared_from_this()/impl_ptr keepalive, so + // the forward op takes no object_ref_ keepalive, so // destroying the resolver frees the win_resolver the queued // resolve_op lives in before teardown drains that op -- a // use-after-free that ASan catches. localhost resolves from the hosts @@ -1310,7 +1310,7 @@ struct resolver_test // only under ASan; without a sanitizer the freed read is silent. // // Contrast the reverse path, which holds the keepalive on - // reverse_op_.impl_ptr across the queued completion + // reverse_op_.object_ref_ across the queued completion // (win_resolver_service.hpp do_reverse_resolve_work / do_complete), // and so survives teardown intact. void testDestroyWithForwardResolveQueued() diff --git a/test/unit/tcp_acceptor.cpp b/test/unit/tcp_acceptor.cpp index 3efc2c261..ae7ab9f2c 100644 --- a/test/unit/tcp_acceptor.cpp +++ b/test/unit/tcp_acceptor.cpp @@ -19,9 +19,13 @@ #include #include #include +#include #include +#include #include +#include +#include #include #include #include @@ -1833,6 +1837,97 @@ struct tcp_acceptor_test BOOST_TEST(acceptOneThrough(ioc, acc, port, true)); } + // Accept-path twin of random_access_file's + // testDeferredExecutorCompletion: an accept awaited from a + // non-io_context executor takes dispatch_coro's deferring branch, + // where the queued continuation must survive the accept node's + // recycling. Sequencing is causal (latches + single-worker FIFO), + // not timed. + void testDeferredExecutorCompletion() + { + io_context ioc(Backend); + capy::thread_pool pool(1); + + tcp_acceptor acc(ioc); + BOOST_TEST(!acc.open()); + acc.set_option(socket_option::reuse_address(true)); + BOOST_TEST(!acc.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!acc.listen()); + endpoint ep(ipv4_address::loopback(), acc.local_endpoint().port()); + + std::latch started(1); + std::latch hold(1); + std::atomic c1_done{false}; + std::atomic c2_done{false}; + tcp_socket s1(ioc); + tcp_socket s2(ioc); + + // C1: accept awaited on the pool executor (deferring path). + capy::run_async(pool.get_executor())( + [](tcp_acceptor& a, tcp_socket& s, + std::atomic& done) -> capy::task<> + { + auto [ec] = co_await a.accept(s); + done = !ec; + }(acc, s1, c1_done)); + + // Parked behind C1 on the single worker: when this runs, C1 + // has already parked its accept and suspended. + capy::run_async(pool.get_executor())( + [](std::latch& s, std::latch& h) -> capy::task<> + { + s.count_down(); + h.wait(); + co_return; + }(started, hold)); + started.wait(); + + // Deliver C1's connection: the completion queues C1's resume + // into the parked pool and recycles the accept node. + tcp_socket client1(ioc); + capy::run_async(ioc.get_executor())( + [](tcp_socket& c, endpoint e) -> capy::task<> + { + std::ignore = co_await c.connect(e); + }(client1, ep)); + ioc.run(); + ioc.restart(); + + // Refill: C2 accepts on the io_context executor, popping the + // recycled node; its inline completion reuses the node's + // continuation while the pool still holds it. + tcp_socket client2(ioc); + capy::run_async(ioc.get_executor())( + [](tcp_acceptor& a, tcp_socket& s, + std::atomic& done) -> capy::task<> + { + auto [ec] = co_await a.accept(s); + done = !ec; + }(acc, s2, c2_done)); + capy::run_async(ioc.get_executor())( + [](tcp_socket& c, endpoint e) -> capy::task<> + { + std::ignore = co_await c.connect(e); + }(client2, ep)); + ioc.run(); + ioc.restart(); + BOOST_TEST(c2_done.load()); + + // Release the worker and drain with a sentinel. + hold.count_down(); + std::latch drained(1); + capy::run_async(pool.get_executor())( + [](std::latch& d) -> capy::task<> + { + d.count_down(); + co_return; + }(drained)); + drained.wait(); + + BOOST_TEST(c1_done.load()); + pool.join(); + } + void run() { testConstruction(); @@ -1841,6 +1936,11 @@ struct tcp_acceptor_test testMoveConstruct(); testMoveAssign(); + // The recycling hazard under test is uring's (pooled accept + // node + deferring dispatch); the exactly-once-resume + // property holds on every backend, so all of them run it. + testDeferredExecutorCompletion(); + // Cancellation testCancelAccept(); testCloseWhilePendingAccept(); diff --git a/test/unit/zero_alloc.cpp b/test/unit/zero_alloc.cpp new file mode 100644 index 000000000..1285b959d --- /dev/null +++ b/test/unit/zero_alloc.cpp @@ -0,0 +1,401 @@ +// +// Copyright (c) 2026 Steve Gerbino +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// Official repository: https://github.com/cppalliance/corosio +// + +// Acceptance gate for the object_pool conversion: steady-state +// construct()/destroy() and the accept/connect/read/write dispatch +// path must not call the global allocator once the per-service free +// list has been warmed. Counts against alloc_counter.hpp's shared +// interposer -- see that header for why this TU does not define its +// own operator new/delete. +// +// Excluded under ASan: ASan instruments its own replacement of the +// global allocation functions, and a second counting layer on top of +// that does not reliably observe the same calls. Excluded under TSan +// for the same reason alloc_counter.hpp's counters don't exist there +// (its runtime replaces operator new/delete itself). +#include "context.hpp" // COROSIO_TEST_HAS_ASAN +#include "alloc_counter.hpp" // COROSIO_TEST_HAS_TSAN, alloc_armed/alloc_count + +#if !COROSIO_TEST_HAS_ASAN && !defined(COROSIO_TEST_HAS_TSAN) + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "temp_path.hpp" +#include "test_suite.hpp" + +namespace boost::corosio { + +namespace { + +// Cycles warmed with counting off, before the assertion window opens. +constexpr int warmup_cycles = 8; + +// Cycles measured with counting on. +constexpr int measured_cycles = 8; + +} // namespace + +template +struct zero_alloc_test +{ + // One accept -> echo one buffer -> close round trip. `server` is + // constructed and destroyed by the caller on every cycle -- that + // construct()/destroy() pair is the thing under test, so it must + // not live across cycles the way `acc` and `client` do. + void runCycle( + io_context& ioc, + capy::executor_ref ex, + tcp_acceptor& acc, + tcp_socket& client, + endpoint ep) + { + tcp_socket server(ioc); + + bool accepted = false, read_ok = false; + bool connected = false, write_ok = false; + char in = 0; + char const out = 'x'; + + // Server side: accept into the reused socket, then echo the + // one byte the client sends back to it. + capy::run_async(ex)( + [](tcp_acceptor& a, tcp_socket& s, char* b, bool& got_accept, + bool& got_read) -> capy::task<> { + auto [ec1] = co_await a.accept(s); + got_accept = !ec1; + auto [ec2, n2] = co_await s.read_some( + capy::mutable_buffer(b, 1)); + got_read = !ec2 && n2 == 1; + }(acc, server, &in, accepted, read_ok)); + + // Client side: connect, then send the one byte. + capy::run_async(ex)( + [](tcp_socket& s, endpoint ep_, char const* b, + bool& got_connect, bool& got_write) -> capy::task<> { + auto [ec1] = co_await s.connect(ep_); + got_connect = !ec1; + auto [ec2, n2] = co_await s.write_some( + capy::const_buffer(b, 1)); + got_write = !ec2 && n2 == 1; + }(client, ep, &out, connected, write_ok)); + + ioc.run(); + ioc.restart(); + + BOOST_TEST(accepted); + BOOST_TEST(connected); + BOOST_TEST(write_ok); + BOOST_TEST(read_ok); + BOOST_TEST_EQ(in, out); + + client.close(); + + // server destructs here -- recycles its impl for the next + // cycle's construct() instead of freeing it. + } + + void testAcceptEchoCloseIsZeroAlloc() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + tcp_acceptor acc(ioc); + BOOST_TEST(!acc.open()); + acc.set_option(socket_option::reuse_address(true)); + BOOST_TEST(!acc.bind(endpoint(ipv4_address::loopback(), 0))); + BOOST_TEST(!acc.listen()); + auto port = acc.local_endpoint().port(); + endpoint ep(ipv4_address::loopback(), port); + + // client is pre-created and outlives every cycle: only the + // accept side's socket construct()/destroy() is under the + // gate here, matching the brief's accept(tcp_socket&) idiom. + tcp_socket client(ioc); + + for (int i = 0; i < warmup_cycles; ++i) + runCycle(ioc, ex, acc, client, ep); + + alloc_count.store(0, std::memory_order_relaxed); + alloc_armed.store(true, std::memory_order_relaxed); + for (int i = 0; i < measured_cycles; ++i) + runCycle(ioc, ex, acc, client, ep); + alloc_armed.store(false, std::memory_order_relaxed); + + // epoll: the reactor dispatch path (accept, connect, read, + // write, close) and the recycled server-socket impl are both + // allocation-free once warmed -- this is the task's primary + // claim. + // + // select: registered_descs_ (select_scheduler.hpp) is a + // std::map keyed by fd; register_descriptor()/ + // deregister_descriptor() insert/erase a node for each of the + // 2 fds this cycle creates (the accepted server fd, the + // reconnected client fd) every cycle, regardless of impl + // recycling. Documented out of scope for this task (design + // doc section 1); 2 allocations/cycle pins the known cost. + // + // uring: the multishot acceptor's accept nodes and ready-fd + // nodes recycle through per-acceptor free lists + // (uring_multishot_acceptor.hpp), so warmed accept churn is + // allocation-free like epoll. + // + // Either way, pinning the exact number (0, or the documented + // non-zero count) means a regression or an improvement both + // trip this assert instead of silently drifting. +#if BOOST_COROSIO_HAS_SELECT + if constexpr ( + std::is_same_v, select_t>) + BOOST_TEST_EQ( + alloc_count.load(std::memory_order_relaxed), + static_cast(2 * measured_cycles)); + else +#endif + BOOST_TEST_EQ(alloc_count.load(std::memory_order_relaxed), 0LL); + } + + // Reuse identity at the tcp_socket level: once the pool has one + // spare impl, a construct()/destroy()/construct() sequence pops + // and recycles that same impl both times instead of allocating. + void testReuseIdentityIsZeroAlloc() + { + io_context ioc(Backend); + + { + tcp_socket warm(ioc); // first-ever construct: pool is cold + } // destroy() recycles it + + alloc_count.store(0, std::memory_order_relaxed); + alloc_armed.store(true, std::memory_order_relaxed); + { + tcp_socket s1(ioc); // pop from free_ + } // destroy() -> recycle + { + tcp_socket s2(ioc); // pop the same recycled impl again + } + alloc_armed.store(false, std::memory_order_relaxed); + + BOOST_TEST_EQ(alloc_count.load(std::memory_order_relaxed), 0LL); + } + + // Timer churn: the thread-local single-slot tier plus capy's + // frame-embedded waiter should make a delay() await literally + // zero-alloc on every backend, not just the two the socket path + // tolerates a known gap on. + void testDelayChurnIsZeroAlloc() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + auto one_delay = [&] { + bool done = false; + capy::run_async(ex)([](bool& d) -> capy::task<> { + auto [ec] = co_await delay(std::chrono::milliseconds(0)); + d = !ec; + }(done)); + ioc.run(); + ioc.restart(); + BOOST_TEST(done); + }; + + // Warms both the timer_service pool and capy's coroutine + // frame pool -- the latter's own first-use allocation would + // otherwise show up as noise in the counted window below. + for (int i = 0; i < warmup_cycles; ++i) + one_delay(); + + alloc_count.store(0, std::memory_order_relaxed); + alloc_armed.store(true, std::memory_order_relaxed); + for (int i = 0; i < measured_cycles; ++i) + one_delay(); + alloc_armed.store(false, std::memory_order_relaxed); + + BOOST_TEST_EQ(alloc_count.load(std::memory_order_relaxed), 0LL); + } + + // Random-access file churn: each read_some_at/write_some_at used + // to heap-allocate its op; both now recycle through the file's + // free lists, so a warmed write+read round trip is zero-alloc on + // every backend (posix thread-pool path and native uring path). + void testRafChurnIsZeroAlloc() + { + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + test::temp_file tmp("zero_alloc_raf_", "warm"); + random_access_file f(ioc); + BOOST_TEST(!f.open(tmp.path, file_base::read_write)); + + char buf[4] = {}; + auto one_rw = [&] { + bool wrote = false, read = false; + capy::run_async(ex)( + [](random_access_file& file, char* b, bool& w, + bool& r) -> capy::task<> { + auto [ec1, n1] = co_await file.write_some_at( + 0, capy::const_buffer("abcd", 4)); + w = !ec1 && n1 == 4; + auto [ec2, n2] = co_await file.read_some_at( + 0, capy::mutable_buffer(b, 4)); + r = !ec2 && n2 == 4; + }(f, buf, wrote, read)); + ioc.run(); + ioc.restart(); + BOOST_TEST(wrote); + BOOST_TEST(read); + }; + + for (int i = 0; i < warmup_cycles; ++i) + one_rw(); + + alloc_count.store(0, std::memory_order_relaxed); + alloc_armed.store(true, std::memory_order_relaxed); + for (int i = 0; i < measured_cycles; ++i) + one_rw(); + alloc_armed.store(false, std::memory_order_relaxed); + + BOOST_TEST_EQ(alloc_count.load(std::memory_order_relaxed), 0LL); + } + + // tcp_server connection churn: launching a worker used to cost a + // fresh stop_source state, a globally allocated launch_wrapper + // frame, and a heap wrapper op for the bare-handle post. All + // three now recycle (reused stop state, frame_alloc_mixin, + // frame-embedded continuation), so a warmed connect/echo/close + // round trip through the server allocates nothing. Select keeps + // its documented 2/cycle registered_descs_ cost. + void testServerChurnIsZeroAlloc() + { + struct echo_worker : tcp_server::worker_base + { + io_context& ctx; + tcp_socket sock; + explicit echo_worker(io_context& c) : ctx(c), sock(c) {} + tcp_socket& socket() override + { + return sock; + } + void run(tcp_server::launcher launch) override + { + launch( + ctx.get_executor(), + [](tcp_socket* s) -> capy::task<> + { + char b[8]; + auto [ec, n] = co_await s->read_some( + capy::mutable_buffer(b, sizeof(b))); + if (!ec) + std::ignore = co_await s->write_some( + capy::const_buffer(b, n)); + s->close(); + }(&sock)); + } + }; + + io_context ioc(Backend); + auto ex = ioc.get_executor(); + + tcp_server srv(ioc, ex); + { + std::vector> v; + for (int i = 0; i < 2; ++i) + v.push_back(std::make_unique(ioc)); + srv.set_workers(std::move(v)); + } + BOOST_TEST(!srv.bind(endpoint(ipv4_address::loopback(), 0))); + auto port = srv.local_endpoint().port(); + endpoint ep(ipv4_address::loopback(), port); + srv.start(); + + auto one_conn = [&] { + bool echoed = false; + capy::run_async(ex)( + [](io_context& c, endpoint target, bool& ok) -> capy::task<> + { + tcp_socket s(c); + auto [ec1] = co_await s.connect(target); + if (ec1) + co_return; + char const out = 'z'; + auto [ec2, n2] = + co_await s.write_some(capy::const_buffer(&out, 1)); + if (ec2 || n2 != 1) + co_return; + char in = 0; + auto [ec3, n3] = + co_await s.read_some(capy::mutable_buffer(&in, 1)); + ok = !ec3 && n3 == 1 && in == out; + }(ioc, ep, echoed)); + // Pump one event at a time: the server's pending accept + // keeps the context busy forever, so a plain run() (or a + // full run_for window) would never return promptly. + while (!echoed && ioc.run_one_for(std::chrono::seconds(5)) != 0) + { + } + ioc.restart(); + BOOST_TEST(echoed); + }; + + for (int i = 0; i < warmup_cycles; ++i) + one_conn(); + + alloc_count.store(0, std::memory_order_relaxed); + alloc_armed.store(true, std::memory_order_relaxed); + for (int i = 0; i < measured_cycles; ++i) + one_conn(); + alloc_armed.store(false, std::memory_order_relaxed); + +#if BOOST_COROSIO_HAS_SELECT + if constexpr ( + std::is_same_v, select_t>) + BOOST_TEST_EQ( + alloc_count.load(std::memory_order_relaxed), + static_cast(2 * measured_cycles)); + else +#endif + BOOST_TEST_EQ(alloc_count.load(std::memory_order_relaxed), 0LL); + + srv.stop(); + ioc.run(); + } + + void run() + { + if (!alloc_counter_is_live("boost.corosio.zero_alloc")) + return; + testAcceptEchoCloseIsZeroAlloc(); + testReuseIdentityIsZeroAlloc(); + testDelayChurnIsZeroAlloc(); + testRafChurnIsZeroAlloc(); + testServerChurnIsZeroAlloc(); + } +}; + +COROSIO_TEST_EPOLL_(zero_alloc_test, "boost.corosio.zero_alloc") +COROSIO_TEST_SELECT_(zero_alloc_test, "boost.corosio.zero_alloc") +COROSIO_TEST_URING_(zero_alloc_test, "boost.corosio.zero_alloc") + +} // namespace boost::corosio + +#endif // !COROSIO_TEST_HAS_ASAN && !COROSIO_TEST_HAS_TSAN