. /include/boost/corosio /detail /object_pool.hpp
94.20% Lines (65/69) 100.00% Functions (10/10)
[ ] prev/next file n p prev/next uncovered line
TLA Baseline Branch
Line Hits Code Line Hits Code
1   // 1   //
2   // Copyright (c) 2025 Steve Gerbino (steve@gerbino.co) 2   // Copyright (c) 2025 Steve Gerbino (steve@gerbino.co)
3   // 3   //
4   // Distributed under the Boost Software License, Version 1.0. (See accompanying 4   // Distributed under the Boost Software License, Version 1.0. (See accompanying
5   // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) 5   // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6   // 6   //
7   // Official repository: https://github.com/cppalliance/corosio 7   // Official repository: https://github.com/cppalliance/corosio
8   // 8   //
9   9  
10   #ifndef BOOST_COROSIO_DETAIL_OBJECT_POOL_HPP 10   #ifndef BOOST_COROSIO_DETAIL_OBJECT_POOL_HPP
11   #define BOOST_COROSIO_DETAIL_OBJECT_POOL_HPP 11   #define BOOST_COROSIO_DETAIL_OBJECT_POOL_HPP
12   12  
13   #include <boost/corosio/detail/config.hpp> 13   #include <boost/corosio/detail/config.hpp>
14   #include <boost/corosio/detail/intrusive.hpp> 14   #include <boost/corosio/detail/intrusive.hpp>
15   #include <boost/corosio/detail/object_ref.hpp> 15   #include <boost/corosio/detail/object_ref.hpp>
16   #include <atomic> 16   #include <atomic>
17   #include <cstddef> 17   #include <cstddef>
18   #include <initializer_list> 18   #include <initializer_list>
19   #include <limits> 19   #include <limits>
20   #include <mutex> 20   #include <mutex>
21   #include <utility> 21   #include <utility>
22   #include <vector> 22   #include <vector>
23   23  
24   namespace boost::corosio::detail { 24   namespace boost::corosio::detail {
25   25  
26   /** Per-service recycling pool for I/O implementations. 26   /** Per-service recycling pool for I/O implementations.
27   27  
28   Replaces per-impl `shared_ptr` ownership: a service news an impl 28   Replaces per-impl `shared_ptr` ownership: a service news an impl
29   once, then `recycle()` (the impl's `retire()` action) parks 29   once, then `recycle()` (the impl's `retire()` action) parks
30   it on the free list and `acquire()` hands it back out, so steady-state 30   it on the free list and `acquire()` hands it back out, so steady-state
31   construct/destroy allocates nothing. Unbounded; memory is released 31   construct/destroy allocates nothing. Unbounded; memory is released
32   only on pool destruction. Debug poisoning, where an impl does it, 32   only on pool destruction. Debug poisoning, where an impl does it,
33   happens in that impl's `retire()`, not here (the pool cannot know 33   happens in that impl's `retire()`, not here (the pool cannot know
34   which bytes are resettable). 34   which bytes are resettable).
35   35  
36   A service tears down its impls by calling `shutdown()`, which 36   A service tears down its impls by calling `shutdown()`, which
37   enters shutting-down mode and visits every live impl in one 37   enters shutting-down mode and visits every live impl in one
38   critical section; subsequent zero-ref crossings delete instead of 38   critical section; subsequent zero-ref crossings delete instead of
39   recycling. 39   recycling.
40   40  
41   `adopt()` and `remove()` are protected ownership-transfer 41   `adopt()` and `remove()` are protected ownership-transfer
42   primitives for sanctioned extensions — a derived pool exposing 42   primitives for sanctioned extensions — a derived pool exposing
43   them to a caller that tracks some of its live impls through its 43   them to a caller that tracks some of its live impls through its
44   own structure instead of this pool's `live_` list (the timer 44   own structure instead of this pool's `live_` list (the timer
45   service, via its expiry heap) — rather than part of the public 45   service, via its expiry heap) — rather than part of the public
46   surface every ordinary service uses. 46   surface every ordinary service uses.
47   47  
48   @par Thread Safety 48   @par Thread Safety
49   Distinct objects: Safe. Shared objects: Safe; all operations lock 49   Distinct objects: Safe. Shared objects: Safe; all operations lock
50   an internal mutex. 50   an internal mutex.
51   */ 51   */
52   template<class Impl> 52   template<class Impl>
53   class object_pool 53   class object_pool
54   { 54   {
55   std::mutex mutex_; 55   std::mutex mutex_;
56   intrusive_list<Impl> live_; 56   intrusive_list<Impl> live_;
57   intrusive_list<Impl> free_; 57   intrusive_list<Impl> free_;
58   bool shutting_down_ = false; 58   bool shutting_down_ = false;
59   59  
60   /** Marks `refs_` on an impl currently parked in `free_`. 60   /** Marks `refs_` on an impl currently parked in `free_`.
61   61  
62   `intrusive_list::remove()` cannot distinguish "linked in 62   `intrusive_list::remove()` cannot distinguish "linked in
63   `free_`" from "linked in `live_`" by inspecting the node 63   `free_`" from "linked in `live_`" by inspecting the node
64   alone — both lists share one `next_`/`prev_` pair per impl — 64   alone — both lists share one `next_`/`prev_` pair per impl —
65   so `recycle()` cannot safely call `live_.remove()` on an impl 65   so `recycle()` cannot safely call `live_.remove()` on an impl
66   it does not already know is live. Stamping this sentinel into 66   it does not already know is live. Stamping this sentinel into
67   `refs_` the moment an impl joins `free_`, and checking it 67   `refs_` the moment an impl joins `free_`, and checking it
68   before touching either list, is what lets `recycle()` and 68   before touching either list, is what lets `recycle()` and
69   `remove()` tell the two cases apart in every build mode, not 69   `remove()` tell the two cases apart in every build mode, not
70   just under asserts. `acquire()` always overwrites it with 1 70   just under asserts. `acquire()` always overwrites it with 1
71   before the impl is usable again. 71   before the impl is usable again.
72   */ 72   */
73   static constexpr std::size_t pooled_sentinel = pooled_refs; 73   static constexpr std::size_t pooled_sentinel = pooled_refs;
74   74  
75   /// Check whether @p impl is parked in `free_` (`refs_` reads the 75   /// Check whether @p impl is parked in `free_` (`refs_` reads the
76   /// sentinel). Callers must check this before any list operation — 76   /// sentinel). Callers must check this before any list operation —
77   /// see the class-level comment on `pooled_sentinel`. 77   /// see the class-level comment on `pooled_sentinel`.
HITCBC 78   84948 static bool is_pooled(Impl const* impl) noexcept 78   85613 static bool is_pooled(Impl const* impl) noexcept
79   { 79   {
HITCBC 80   84948 return impl->refs_.load(std::memory_order_relaxed) == 80   85613 return impl->refs_.load(std::memory_order_relaxed) ==
HITCBC 81   84948 pooled_sentinel; 81   85613 pooled_sentinel;
82   } 82   }
83   83  
84   protected: 84   protected:
85   /** Track a fresh or reset impl as live. 85   /** Track a fresh or reset impl as live.
86   86  
87   @pre @p impl is not parked in `free_` (`refs_` must not read 87   @pre @p impl is not parked in `free_` (`refs_` must not read
88   `pooled_sentinel`). The same list-corruption hazard 88   `pooled_sentinel`). The same list-corruption hazard
89   `recycle()`'s class-level comment documents applies here: a 89   `recycle()`'s class-level comment documents applies here: a
90   free-listed impl pushed onto `live_` directly, bypassing 90   free-listed impl pushed onto `live_` directly, bypassing
91   `acquire()`, would end up linked in both lists through one shared 91   `acquire()`, would end up linked in both lists through one shared
92   `next_`/`prev_` pair. 92   `next_`/`prev_` pair.
93   93  
94   @pre The caller is not racing `shutdown()`. Every caller adopts 94   @pre The caller is not racing `shutdown()`. Every caller adopts
95   while constructing an object (`acquire()`, or a derived pool's 95   while constructing an object (`acquire()`, or a derived pool's
96   cache handing a parked impl back out), and construction is 96   cache handing a parked impl back out), and construction is
97   expected to stop before the service's own `shutdown()` runs; 97   expected to stop before the service's own `shutdown()` runs;
98   this is checked only in debug builds because the failure mode 98   this is checked only in debug builds because the failure mode
99   is a silently orphaned impl (no close callback), not memory 99   is a silently orphaned impl (no close callback), not memory
100   corruption. 100   corruption.
101   */ 101   */
HITCBC 102   31528 void adopt(Impl* impl) 102   31897 void adopt(Impl* impl)
103   { 103   {
HITCBC 104   31528 std::lock_guard lock(mutex_); 104   31897 std::lock_guard lock(mutex_);
HITCBC 105   31528 if (is_pooled(impl)) 105   31897 if (is_pooled(impl))
106   { 106   {
MISUBC 107   ✗ BOOST_COROSIO_ASSERT(false); 107   ✗ BOOST_COROSIO_ASSERT(false);
108   return; 108   return;
109   } 109   }
HITCBC 110   31528 BOOST_COROSIO_ASSERT(!shutting_down_); 110   31897 BOOST_COROSIO_ASSERT(!shutting_down_);
HITCBC 111   31528 live_.push_back(impl); 111   31897 live_.push_back(impl);
HITCBC 112   31528 } 112   31897 }
113   113  
114   /** Unlink a live impl without recycling (ownership transfer). 114   /** Unlink a live impl without recycling (ownership transfer).
115   115  
116   Used by a caller that is taking an impl out of this pool's 116   Used by a caller that is taking an impl out of this pool's
117   bookkeeping entirely (a thread-local cache slot, a direct 117   bookkeeping entirely (a thread-local cache slot, a direct
118   delete during shutdown) rather than parking it on `free_`. 118   delete during shutdown) rather than parking it on `free_`.
119   119  
120   @pre @p impl is not parked in `free_` (`refs_` must not read 120   @pre @p impl is not parked in `free_` (`refs_` must not read
121   `pooled_sentinel`). The same list-corruption hazard 121   `pooled_sentinel`). The same list-corruption hazard
122   `recycle()`'s class-level comment documents applies here: a 122   `recycle()`'s class-level comment documents applies here: a
123   free-listed impl would be spliced out using `live_`'s 123   free-listed impl would be spliced out using `live_`'s
124   boundary pointers instead of `free_`'s. 124   boundary pointers instead of `free_`'s.
125   125  
126   @return True if @p impl was linked in `live_` and has now 126   @return True if @p impl was linked in `live_` and has now
127   been removed; false if it was already detached. 127   been removed; false if it was already detached.
128   */ 128   */
HITCBC 129   18387 bool remove(Impl* impl) noexcept 129   18415 bool remove(Impl* impl) noexcept
130   { 130   {
HITCBC 131   18387 std::lock_guard lock(mutex_); 131   18415 std::lock_guard lock(mutex_);
HITCBC 132   18387 if (is_pooled(impl)) 132   18415 if (is_pooled(impl))
133   { 133   {
MISUBC 134   ✗ BOOST_COROSIO_ASSERT(false); 134   ✗ BOOST_COROSIO_ASSERT(false);
135   return false; 135   return false;
136   } 136   }
HITCBC 137   18387 return live_.remove(impl); 137   18415 return live_.remove(impl);
HITCBC 138   18387 } 138   18415 }
139   139  
140   public: 140   public:
141   /** Destroy the pool, freeing every impl still parked or live. 141   /** Destroy the pool, freeing every impl still parked or live.
142   142  
143   A live impl is deleted whatever its count. Keepalives it holds 143   A live impl is deleted whatever its count. Keepalives it holds
144   on itself (an embedded op's `object_ref`) are released by its 144   on itself (an embedded op's `object_ref`) are released by its
145   own destructor; the count is pinned first so none of those 145   own destructor; the count is pinned first so none of those
146   releases reaches zero and calls `retire()` on an object that 146   releases reaches zero and calls `retire()` on an object that
147   is already being destroyed. 147   is already being destroyed.
148   148  
149   @pre `shutdown()` has already run, or no impls are live. 149   @pre `shutdown()` has already run, or no impls are live.
150   */ 150   */
HITCBC 151   8916 ~object_pool() 151   8919 ~object_pool()
152   { 152   {
HITCBC 153   8916 BOOST_COROSIO_ASSERT(shutting_down_ || live_.empty()); 153   8919 BOOST_COROSIO_ASSERT(shutting_down_ || live_.empty());
HITCBC 154   26748 for (auto* list : {&free_, &live_}) 154   26757 for (auto* list : {&free_, &live_})
155   { 155   {
HITCBC 156   25035 while (auto* impl = list->pop_front()) 156   25368 while (auto* impl = list->pop_front())
157   { 157   {
HITCBC 158   7203 impl->refs_.store(pooled_sentinel, std::memory_order_relaxed); 158   7530 impl->refs_.store(pooled_sentinel, std::memory_order_relaxed);
HITCBC 159   7203 delete impl; 159   7530 delete impl;
160   } 160   }
161   } 161   }
HITCBC 162   8916 } 162   8919 }
163   163  
HITCBC 164   8906 object_pool() = default; 164   8909 object_pool() = default;
165   object_pool(object_pool const&) = delete; 165   object_pool(object_pool const&) = delete;
166   object_pool& operator=(object_pool const&) = delete; 166   object_pool& operator=(object_pool const&) = delete;
167   167  
168   /** Pop-or-create an impl with the acquisition protocol applied. 168   /** Pop-or-create an impl with the acquisition protocol applied.
169   169  
170   Recycled impls are `reuse()`-reset and their reference count 170   Recycled impls are `reuse()`-reset and their reference count
171   restored from the pooled sentinel; fresh impls are constructed 171   restored from the pooled sentinel; fresh impls are constructed
172   in place from `args` on the cold path. Either way the impl is 172   in place from `args` on the cold path. Either way the impl is
173   adopted as live. This is the sequence every service's 173   adopted as live. This is the sequence every service's
174   `construct()` used to hand-roll. Making the pool itself the 174   `construct()` used to hand-roll. Making the pool itself the
175   only place that calls `new Impl` gives the whole library one 175   only place that calls `new Impl` gives the whole library one
176   audit point for allocation policy. 176   audit point for allocation policy.
177   177  
178   @pre Not called concurrently with, or after, `shutdown()` on 178   @pre Not called concurrently with, or after, `shutdown()` on
179   this pool. Every caller reaches this through a service's 179   this pool. Every caller reaches this through a service's
180   `construct()`, and service shutdown order guarantees no 180   `construct()`, and service shutdown order guarantees no
181   `construct()` outlives that service's own `shutdown()`; the 181   `construct()` outlives that service's own `shutdown()`; the
182   window is only asserted in debug builds (see `adopt()`) 182   window is only asserted in debug builds (see `adopt()`)
183   because the failure mode is an orphaned impl, not corruption. 183   because the failure mode is an orphaned impl, not corruption.
184   184  
185   @param args Constructor arguments for `Impl` (cold path only). 185   @param args Constructor arguments for `Impl` (cold path only).
186   186  
187   @return The acquired impl, `refs_ == 1`, tracked live. 187   @return The acquired impl, `refs_ == 1`, tracked live.
188   */ 188   */
189   template<class... Args> 189   template<class... Args>
HITCBC 190   38774 Impl* acquire(Args&&... args) 190   39054 Impl* acquire(Args&&... args)
191   { 191   {
192   Impl* impl; 192   Impl* impl;
193   { 193   {
HITCBC 194   38774 std::lock_guard lock(mutex_); 194   39054 std::lock_guard lock(mutex_);
HITCBC 195   38774 BOOST_COROSIO_ASSERT(!shutting_down_); 195   39054 BOOST_COROSIO_ASSERT(!shutting_down_);
HITCBC 196   38774 impl = free_.pop_front(); 196   39054 impl = free_.pop_front();
HITCBC 197   38774 if (impl) 197   39054 if (impl)
198   { 198   {
HITCBC 199   21897 impl->refs_.store(1, std::memory_order_relaxed); 199   21833 impl->refs_.store(1, std::memory_order_relaxed);
HITCBC 200   21897 live_.push_back(impl); 200   21833 live_.push_back(impl);
201   } 201   }
HITCBC 202   38774 } 202   39054 }
203   // reuse() runs unlocked: acquire()'s precondition excludes a 203   // reuse() runs unlocked: acquire()'s precondition excludes a
204   // concurrent shutdown() visiting the impl mid-reset. 204   // concurrent shutdown() visiting the impl mid-reset.
HITCBC 205   38774 if (impl) 205   39054 if (impl)
206   { 206   {
HITCBC 207   21897 impl->reuse(); 207   21833 impl->reuse();
HITCBC 208   21897 return impl; 208   21833 return impl;
209   } 209   }
HITCBC 210   16877 impl = new Impl(std::forward<Args>(args)...); 210   17221 impl = new Impl(std::forward<Args>(args)...);
HITCBC 211   16877 adopt(impl); 211   17221 adopt(impl);
HITCBC 212   16877 return impl; 212   17221 return impl;
213   } 213   }
214   214  
215   /** Move a zero-ref impl from live to the free list. 215   /** Move a zero-ref impl from live to the free list.
216   216  
217   Called from `retire()`, possibly on a scheduler thread. 217   Called from `retire()`, possibly on a scheduler thread.
218   Deletes instead when shutting down. The impl must not be 218   Deletes instead when shutting down. The impl must not be
219   touched after this returns. 219   touched after this returns.
220   220  
221   Two accounting bugs are possible here, neither from this 221   Two accounting bugs are possible here, neither from this
222   pool's own bookkeeping but from a caller's refcounting (e.g. 222   pool's own bookkeeping but from a caller's refcounting (e.g.
223   a duplicate `retire()` for one zero-crossing). Both are 223   a duplicate `retire()` for one zero-crossing). Both are
224   made safe in every build mode, not just under asserts, because 224   made safe in every build mode, not just under asserts, because
225   neither can be told apart from the correct case by a 225   neither can be told apart from the correct case by a
226   `BOOST_COROSIO_ASSERT` alone that still runs the list 226   `BOOST_COROSIO_ASSERT` alone that still runs the list
227   operations around it: 227   operations around it:
228   228  
229   - `impl` is already parked in `free_` — the list-corruption 229   - `impl` is already parked in `free_` — the list-corruption
230   hazard the class-level `pooled_sentinel` comment documents. 230   hazard the class-level `pooled_sentinel` comment documents.
231   The `is_pooled()` check runs first and returns before either 231   The `is_pooled()` check runs first and returns before either
232   list is touched. 232   list is touched.
233   - `impl` is fully detached from `live_` and is not the 233   - `impl` is fully detached from `live_` and is not the
234   sentinel (not shutting down): `live_.remove()` correctly 234   sentinel (not shutting down): `live_.remove()` correctly
235   returns false, but then the impl is reachable through 235   returns false, but then the impl is reachable through
236   neither `acquire()` nor `shutdown()`'s visit while still 236   neither `acquire()` nor `shutdown()`'s visit while still
237   allocated — stranded. Deleted instead of left to leak. 237   allocated — stranded. Deleted instead of left to leak.
238   238  
239   While shutting down, an impl that is no longer in `live_` is 239   While shutting down, an impl that is no longer in `live_` is
240   left alone rather than deleted: something else already took 240   left alone rather than deleted: something else already took
241   it out of the pool's bookkeeping (a derived pool's owner, or a 241   it out of the pool's bookkeeping (a derived pool's owner, or a
242   duplicate retirement of an impl already deleted), and deleting 242   duplicate retirement of an impl already deleted), and deleting
243   it again here could be a double free. 243   it again here could be a double free.
244   */ 244   */
HITCBC 245   35033 void recycle(Impl* impl) 245   35301 void recycle(Impl* impl)
246   { 246   {
HITCBC 247   35033 bool do_delete = false; 247   35301 bool do_delete = false;
248   { 248   {
HITCBC 249   35033 std::lock_guard lock(mutex_); 249   35301 std::lock_guard lock(mutex_);
HITCBC 250   35033 if (is_pooled(impl)) 250   35301 if (is_pooled(impl))
251   { 251   {
MISUBC 252   ✗ BOOST_COROSIO_ASSERT(false); 252   ✗ BOOST_COROSIO_ASSERT(false);
253   return; 253   return;
254   } 254   }
255   255  
HITCBC 256   35033 bool const was_live = live_.remove(impl); 256   35301 bool const was_live = live_.remove(impl);
HITCBC 257   35033 if (shutting_down_) 257   35301 if (shutting_down_)
258   { 258   {
HITCBC 259   5939 do_delete = was_live; 259   5953 do_delete = was_live;
260   } 260   }
HITCBC 261   29094 else if (was_live) 261   29348 else if (was_live)
262   { 262   {
HITCBC 263   29094 impl->refs_.store( 263   29348 impl->refs_.store(
264   pooled_sentinel, std::memory_order_relaxed); 264   pooled_sentinel, std::memory_order_relaxed);
265   // LIFO: the most recently retired impl is the 265   // LIFO: the most recently retired impl is the
266   // cache-warmest candidate for the next acquire(). 266   // cache-warmest candidate for the next acquire().
HITCBC 267   29094 free_.push_front(impl); 267   29348 free_.push_front(impl);
268   } 268   }
269   else 269   else
270   { 270   {
MISUBC 271   ✗ BOOST_COROSIO_ASSERT(false); 271   ✗ BOOST_COROSIO_ASSERT(false);
272   do_delete = true; 272   do_delete = true;
273   } 273   }
HITCBC 274   35033 } 274   35301 }
HITCBC 275   35033 if (do_delete) 275   35301 if (do_delete)
HITCBC 276   5939 delete impl; 276   5953 delete impl;
277   } 277   }
278   278  
279   /** Enter shutdown mode and visit every live impl. 279   /** Enter shutdown mode and visit every live impl.
280   280  
281   Sets shutting-down (subsequent zero-crossings delete instead 281   Sets shutting-down (subsequent zero-crossings delete instead
282   of recycling) and runs `f` on each live impl, all under one 282   of recycling) and runs `f` on each live impl, all under one
283   critical section. Each visited impl is pinned with a reference 283   critical section. Each visited impl is pinned with a reference
284   for the call and released after the critical section ends, so 284   for the call and released after the critical section ends, so
285   `f` may hold on to it; an impl whose count already reached zero 285   `f` may hold on to it; an impl whose count already reached zero
286   is retiring and is skipped rather than revived. 286   is retiring and is skipped rather than revived.
287   287  
288   @note The callback runs under the pool's non-recursive mutex; 288   @note The callback runs under the pool's non-recursive mutex;
289   it must not call recycle(), adopt(), or remove() on this pool. 289   it must not call recycle(), adopt(), or remove() on this pool.
290   */ 290   */
291   template<class F> 291   template<class F>
HITCBC 292   13326 void shutdown(F&& f) 292   13329 void shutdown(F&& f)
293   { 293   {
HITCBC 294   13326 std::vector<Impl*> pinned; 294   13329 std::vector<Impl*> pinned;
295   { 295   {
HITCBC 296   13326 std::lock_guard lock(mutex_); 296   13329 std::lock_guard lock(mutex_);
HITCBC 297   13326 shutting_down_ = true; 297   13329 shutting_down_ = true;
298   // Sized before any pin, so a failure here leaves none held. 298   // Sized before any pin, so a failure here leaves none held.
HITCBC 299   13326 std::size_t n = 0; 299   13329 std::size_t n = 0;
HITCBC 300   22515 live_.for_each([&](Impl*) { ++n; }); 300   22545 live_.for_each([&](Impl*) { ++n; });
HITCBC 301   13326 pinned.reserve(n); 301   13329 pinned.reserve(n);
HITCBC 302   13326 live_.for_each( 302   13329 live_.for_each(
HITCBC 303   18384 [&](Impl* impl) 303   18438 [&](Impl* impl)
304   { 304   {
HITCBC 305   9192 if (try_acquire(impl)) 305   9219 if (try_acquire(impl))
306   { 306   {
HITCBC 307   9191 pinned.push_back(impl); 307   9218 pinned.push_back(impl);
HITCBC 308   9191 f(impl); 308   9218 f(impl);
309   } 309   }
310   }); 310   });
HITCBC 311   13326 } 311   13329 }
HITCBC 312   22517 for (auto* impl : pinned) 312   22547 for (auto* impl : pinned)
HITCBC 313   9191 release(impl); 313   9218 release(impl);
HITCBC 314   13326 } 314   13329 }
315   }; 315   };
316   316  
317   } // namespace boost::corosio::detail 317   } // namespace boost::corosio::detail
318   318  
319   #endif 319   #endif