. /include/boost/corosio /detail /object_pool.hpp
94.20% Lines (65/69) 100.00% Functions (10/10)
[ ] prev/next file n p prev/next uncovered line
TLA Baseline Branch
Line Hits Code Line Hits Code
1   // 1   //
2   // Copyright (c) 2025 Steve Gerbino (steve@gerbino.co) 2   // Copyright (c) 2025 Steve Gerbino (steve@gerbino.co)
3   // 3   //
4   // Distributed under the Boost Software License, Version 1.0. (See accompanying 4   // Distributed under the Boost Software License, Version 1.0. (See accompanying
5   // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) 5   // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6   // 6   //
7   // Official repository: https://github.com/cppalliance/corosio 7   // Official repository: https://github.com/cppalliance/corosio
8   // 8   //
9   9  
10   #ifndef BOOST_COROSIO_DETAIL_OBJECT_POOL_HPP 10   #ifndef BOOST_COROSIO_DETAIL_OBJECT_POOL_HPP
11   #define BOOST_COROSIO_DETAIL_OBJECT_POOL_HPP 11   #define BOOST_COROSIO_DETAIL_OBJECT_POOL_HPP
12   12  
13   #include <boost/corosio/detail/config.hpp> 13   #include <boost/corosio/detail/config.hpp>
14   #include <boost/corosio/detail/intrusive.hpp> 14   #include <boost/corosio/detail/intrusive.hpp>
15   #include <boost/corosio/detail/object_ref.hpp> 15   #include <boost/corosio/detail/object_ref.hpp>
16   #include <atomic> 16   #include <atomic>
17   #include <cstddef> 17   #include <cstddef>
18   #include <initializer_list> 18   #include <initializer_list>
19   #include <limits> 19   #include <limits>
20   #include <mutex> 20   #include <mutex>
21   #include <utility> 21   #include <utility>
22   #include <vector> 22   #include <vector>
23   23  
24   namespace boost::corosio::detail { 24   namespace boost::corosio::detail {
25   25  
26   /** Per-service recycling pool for I/O implementations. 26   /** Per-service recycling pool for I/O implementations.
27   27  
28   Replaces per-impl `shared_ptr` ownership: a service news an impl 28   Replaces per-impl `shared_ptr` ownership: a service news an impl
29   once, then `recycle()` (the impl's `retire()` action) parks 29   once, then `recycle()` (the impl's `retire()` action) parks
30   it on the free list and `acquire()` hands it back out, so steady-state 30   it on the free list and `acquire()` hands it back out, so steady-state
31   construct/destroy allocates nothing. Unbounded; memory is released 31   construct/destroy allocates nothing. Unbounded; memory is released
32   only on pool destruction. Debug poisoning, where an impl does it, 32   only on pool destruction. Debug poisoning, where an impl does it,
33   happens in that impl's `retire()`, not here (the pool cannot know 33   happens in that impl's `retire()`, not here (the pool cannot know
34   which bytes are resettable). 34   which bytes are resettable).
35   35  
36   A service tears down its impls by calling `shutdown()`, which 36   A service tears down its impls by calling `shutdown()`, which
37   enters shutting-down mode and visits every live impl in one 37   enters shutting-down mode and visits every live impl in one
38   critical section; subsequent zero-ref crossings delete instead of 38   critical section; subsequent zero-ref crossings delete instead of
39   recycling. 39   recycling.
40   40  
41   `adopt()` and `remove()` are protected ownership-transfer 41   `adopt()` and `remove()` are protected ownership-transfer
42   primitives for sanctioned extensions — a derived pool exposing 42   primitives for sanctioned extensions — a derived pool exposing
43   them to a caller that tracks some of its live impls through its 43   them to a caller that tracks some of its live impls through its
44   own structure instead of this pool's `live_` list (the timer 44   own structure instead of this pool's `live_` list (the timer
45   service, via its expiry heap) — rather than part of the public 45   service, via its expiry heap) — rather than part of the public
46   surface every ordinary service uses. 46   surface every ordinary service uses.
47   47  
48   @par Thread Safety 48   @par Thread Safety
49   Distinct objects: Safe. Shared objects: Safe; all operations lock 49   Distinct objects: Safe. Shared objects: Safe; all operations lock
50   an internal mutex. 50   an internal mutex.
51   */ 51   */
52   template<class Impl> 52   template<class Impl>
53   class object_pool 53   class object_pool
54   { 54   {
55   std::mutex mutex_; 55   std::mutex mutex_;
56   intrusive_list<Impl> live_; 56   intrusive_list<Impl> live_;
57   intrusive_list<Impl> free_; 57   intrusive_list<Impl> free_;
58   bool shutting_down_ = false; 58   bool shutting_down_ = false;
59   59  
60   /** Marks `refs_` on an impl currently parked in `free_`. 60   /** Marks `refs_` on an impl currently parked in `free_`.
61   61  
62   `intrusive_list::remove()` cannot distinguish "linked in 62   `intrusive_list::remove()` cannot distinguish "linked in
63   `free_`" from "linked in `live_`" by inspecting the node 63   `free_`" from "linked in `live_`" by inspecting the node
64   alone — both lists share one `next_`/`prev_` pair per impl — 64   alone — both lists share one `next_`/`prev_` pair per impl —
65   so `recycle()` cannot safely call `live_.remove()` on an impl 65   so `recycle()` cannot safely call `live_.remove()` on an impl
66   it does not already know is live. Stamping this sentinel into 66   it does not already know is live. Stamping this sentinel into
67   `refs_` the moment an impl joins `free_`, and checking it 67   `refs_` the moment an impl joins `free_`, and checking it
68   before touching either list, is what lets `recycle()` and 68   before touching either list, is what lets `recycle()` and
69   `remove()` tell the two cases apart in every build mode, not 69   `remove()` tell the two cases apart in every build mode, not
70   just under asserts. `acquire()` always overwrites it with 1 70   just under asserts. `acquire()` always overwrites it with 1
71   before the impl is usable again. 71   before the impl is usable again.
72   */ 72   */
73   static constexpr std::size_t pooled_sentinel = pooled_refs; 73   static constexpr std::size_t pooled_sentinel = pooled_refs;
74   74  
75   /// Check whether @p impl is parked in `free_` (`refs_` reads the 75   /// Check whether @p impl is parked in `free_` (`refs_` reads the
76   /// sentinel). Callers must check this before any list operation — 76   /// sentinel). Callers must check this before any list operation —
77   /// see the class-level comment on `pooled_sentinel`. 77   /// see the class-level comment on `pooled_sentinel`.
HITCBC 78   85778 static bool is_pooled(Impl const* impl) noexcept 78   83098 static bool is_pooled(Impl const* impl) noexcept
79   { 79   {
HITCBC 80   85778 return impl->refs_.load(std::memory_order_relaxed) == 80   83098 return impl->refs_.load(std::memory_order_relaxed) ==
HITCBC 81   85778 pooled_sentinel; 81   83098 pooled_sentinel;
82   } 82   }
83   83  
84   protected: 84   protected:
85   /** Track a fresh or reset impl as live. 85   /** Track a fresh or reset impl as live.
86   86  
87   @pre @p impl is not parked in `free_` (`refs_` must not read 87   @pre @p impl is not parked in `free_` (`refs_` must not read
88   `pooled_sentinel`). The same list-corruption hazard 88   `pooled_sentinel`). The same list-corruption hazard
89   `recycle()`'s class-level comment documents applies here: a 89   `recycle()`'s class-level comment documents applies here: a
90   free-listed impl pushed onto `live_` directly, bypassing 90   free-listed impl pushed onto `live_` directly, bypassing
91   `acquire()`, would end up linked in both lists through one shared 91   `acquire()`, would end up linked in both lists through one shared
92   `next_`/`prev_` pair. 92   `next_`/`prev_` pair.
93   93  
94   @pre The caller is not racing `shutdown()`. Every caller adopts 94   @pre The caller is not racing `shutdown()`. Every caller adopts
95   while constructing an object (`acquire()`, or a derived pool's 95   while constructing an object (`acquire()`, or a derived pool's
96   cache handing a parked impl back out), and construction is 96   cache handing a parked impl back out), and construction is
97   expected to stop before the service's own `shutdown()` runs; 97   expected to stop before the service's own `shutdown()` runs;
98   this is checked only in debug builds because the failure mode 98   this is checked only in debug builds because the failure mode
99   is a silently orphaned impl (no close callback), not memory 99   is a silently orphaned impl (no close callback), not memory
100   corruption. 100   corruption.
101   */ 101   */
HITCBC 102   32037 void adopt(Impl* impl) 102   31303 void adopt(Impl* impl)
103   { 103   {
HITCBC 104   32037 std::lock_guard lock(mutex_); 104   31303 std::lock_guard lock(mutex_);
HITCBC 105   32037 if (is_pooled(impl)) 105   31303 if (is_pooled(impl))
106   { 106   {
MISUBC 107   ✗ BOOST_COROSIO_ASSERT(false); 107   ✗ BOOST_COROSIO_ASSERT(false);
108   return; 108   return;
109   } 109   }
HITCBC 110   32037 BOOST_COROSIO_ASSERT(!shutting_down_); 110   31303 BOOST_COROSIO_ASSERT(!shutting_down_);
HITCBC 111   32037 live_.push_back(impl); 111   31303 live_.push_back(impl);
HITCBC 112   32037 } 112   31303 }
113   113  
114   /** Unlink a live impl without recycling (ownership transfer). 114   /** Unlink a live impl without recycling (ownership transfer).
115   115  
116   Used by a caller that is taking an impl out of this pool's 116   Used by a caller that is taking an impl out of this pool's
117   bookkeeping entirely (a thread-local cache slot, a direct 117   bookkeeping entirely (a thread-local cache slot, a direct
118   delete during shutdown) rather than parking it on `free_`. 118   delete during shutdown) rather than parking it on `free_`.
119   119  
120   @pre @p impl is not parked in `free_` (`refs_` must not read 120   @pre @p impl is not parked in `free_` (`refs_` must not read
121   `pooled_sentinel`). The same list-corruption hazard 121   `pooled_sentinel`). The same list-corruption hazard
122   `recycle()`'s class-level comment documents applies here: a 122   `recycle()`'s class-level comment documents applies here: a
123   free-listed impl would be spliced out using `live_`'s 123   free-listed impl would be spliced out using `live_`'s
124   boundary pointers instead of `free_`'s. 124   boundary pointers instead of `free_`'s.
125   125  
126   @return True if @p impl was linked in `live_` and has now 126   @return True if @p impl was linked in `live_` and has now
127   been removed; false if it was already detached. 127   been removed; false if it was already detached.
128   */ 128   */
HITCBC 129   18176 bool remove(Impl* impl) noexcept 129   17914 bool remove(Impl* impl) noexcept
130   { 130   {
HITCBC 131   18176 std::lock_guard lock(mutex_); 131   17914 std::lock_guard lock(mutex_);
HITCBC 132   18176 if (is_pooled(impl)) 132   17914 if (is_pooled(impl))
133   { 133   {
MISUBC 134   ✗ BOOST_COROSIO_ASSERT(false); 134   ✗ BOOST_COROSIO_ASSERT(false);
135   return false; 135   return false;
136   } 136   }
HITCBC 137   18176 return live_.remove(impl); 137   17914 return live_.remove(impl);
HITCBC 138   18176 } 138   17914 }
139   139  
140   public: 140   public:
141   /** Destroy the pool, freeing every impl still parked or live. 141   /** Destroy the pool, freeing every impl still parked or live.
142   142  
143   A live impl is deleted whatever its count. Keepalives it holds 143   A live impl is deleted whatever its count. Keepalives it holds
144   on itself (an embedded op's `object_ref`) are released by its 144   on itself (an embedded op's `object_ref`) are released by its
145   own destructor; the count is pinned first so none of those 145   own destructor; the count is pinned first so none of those
146   releases reaches zero and calls `retire()` on an object that 146   releases reaches zero and calls `retire()` on an object that
147   is already being destroyed. 147   is already being destroyed.
148   148  
149   @pre `shutdown()` has already run, or no impls are live. 149   @pre `shutdown()` has already run, or no impls are live.
150   */ 150   */
HITCBC 151   5574 ~object_pool() 151   5578 ~object_pool()
152   { 152   {
HITCBC 153   5574 BOOST_COROSIO_ASSERT(shutting_down_ || live_.empty()); 153   5578 BOOST_COROSIO_ASSERT(shutting_down_ || live_.empty());
HITCBC 154   16722 for (auto* list : {&free_, &live_}) 154   16734 for (auto* list : {&free_, &live_})
155   { 155   {
HITCBC 156   19053 while (auto* impl = list->pop_front()) 156   18593 while (auto* impl = list->pop_front())
157   { 157   {
HITCBC 158   7905 impl->refs_.store(pooled_sentinel, std::memory_order_relaxed); 158   7437 impl->refs_.store(pooled_sentinel, std::memory_order_relaxed);
HITCBC 159   7905 delete impl; 159   7437 delete impl;
160   } 160   }
161   } 161   }
HITCBC 162   5574 } 162   5578 }
163   163  
HITCBC 164   5564 object_pool() = default; 164   5568 object_pool() = default;
165   object_pool(object_pool const&) = delete; 165   object_pool(object_pool const&) = delete;
166   object_pool& operator=(object_pool const&) = delete; 166   object_pool& operator=(object_pool const&) = delete;
167   167  
168   /** Pop-or-create an impl with the acquisition protocol applied. 168   /** Pop-or-create an impl with the acquisition protocol applied.
169   169  
170   Recycled impls are `reuse()`-reset and their reference count 170   Recycled impls are `reuse()`-reset and their reference count
171   restored from the pooled sentinel; fresh impls are constructed 171   restored from the pooled sentinel; fresh impls are constructed
172   in place from `args` on the cold path. Either way the impl is 172   in place from `args` on the cold path. Either way the impl is
173   adopted as live. This is the sequence every service's 173   adopted as live. This is the sequence every service's
174   `construct()` used to hand-roll. Making the pool itself the 174   `construct()` used to hand-roll. Making the pool itself the
175   only place that calls `new Impl` gives the whole library one 175   only place that calls `new Impl` gives the whole library one
176   audit point for allocation policy. 176   audit point for allocation policy.
177   177  
178   @pre Not called concurrently with, or after, `shutdown()` on 178   @pre Not called concurrently with, or after, `shutdown()` on
179   this pool. Every caller reaches this through a service's 179   this pool. Every caller reaches this through a service's
180   `construct()`, and service shutdown order guarantees no 180   `construct()`, and service shutdown order guarantees no
181   `construct()` outlives that service's own `shutdown()`; the 181   `construct()` outlives that service's own `shutdown()`; the
182   window is only asserted in debug builds (see `adopt()`) 182   window is only asserted in debug builds (see `adopt()`)
183   because the failure mode is an orphaned impl, not corruption. 183   because the failure mode is an orphaned impl, not corruption.
184   184  
185   @param args Constructor arguments for `Impl` (cold path only). 185   @param args Constructor arguments for `Impl` (cold path only).
186   186  
187   @return The acquired impl, `refs_ == 1`, tracked live. 187   @return The acquired impl, `refs_ == 1`, tracked live.
188   */ 188   */
189   template<class... Args> 189   template<class... Args>
HITCBC 190   39325 Impl* acquire(Args&&... args) 190   37637 Impl* acquire(Args&&... args)
191   { 191   {
192   Impl* impl; 192   Impl* impl;
193   { 193   {
HITCBC 194   39325 std::lock_guard lock(mutex_); 194   37637 std::lock_guard lock(mutex_);
HITCBC 195   39325 BOOST_COROSIO_ASSERT(!shutting_down_); 195   37637 BOOST_COROSIO_ASSERT(!shutting_down_);
HITCBC 196   39325 impl = free_.pop_front(); 196   37637 impl = free_.pop_front();
HITCBC 197   39325 if (impl) 197   37637 if (impl)
198   { 198   {
HITCBC 199   21709 impl->refs_.store(1, std::memory_order_relaxed); 199   20497 impl->refs_.store(1, std::memory_order_relaxed);
HITCBC 200   21709 live_.push_back(impl); 200   20497 live_.push_back(impl);
201   } 201   }
HITCBC 202   39325 } 202   37637 }
203   // reuse() runs unlocked: acquire()'s precondition excludes a 203   // reuse() runs unlocked: acquire()'s precondition excludes a
204   // concurrent shutdown() visiting the impl mid-reset. 204   // concurrent shutdown() visiting the impl mid-reset.
HITCBC 205   39325 if (impl) 205   37637 if (impl)
206   { 206   {
HITCBC 207   21709 impl->reuse(); 207   20497 impl->reuse();
HITCBC 208   21709 return impl; 208   20497 return impl;
209   } 209   }
HITCBC 210   17616 impl = new Impl(std::forward<Args>(args)...); 210   17140 impl = new Impl(std::forward<Args>(args)...);
HITCBC 211   17616 adopt(impl); 211   17140 adopt(impl);
HITCBC 212   17616 return impl; 212   17140 return impl;
213   } 213   }
214   214  
215   /** Move a zero-ref impl from live to the free list. 215   /** Move a zero-ref impl from live to the free list.
216   216  
217   Called from `retire()`, possibly on a scheduler thread. 217   Called from `retire()`, possibly on a scheduler thread.
218   Deletes instead when shutting down. The impl must not be 218   Deletes instead when shutting down. The impl must not be
219   touched after this returns. 219   touched after this returns.
220   220  
221   Two accounting bugs are possible here, neither from this 221   Two accounting bugs are possible here, neither from this
222   pool's own bookkeeping but from a caller's refcounting (e.g. 222   pool's own bookkeeping but from a caller's refcounting (e.g.
223   a duplicate `retire()` for one zero-crossing). Both are 223   a duplicate `retire()` for one zero-crossing). Both are
224   made safe in every build mode, not just under asserts, because 224   made safe in every build mode, not just under asserts, because
225   neither can be told apart from the correct case by a 225   neither can be told apart from the correct case by a
226   `BOOST_COROSIO_ASSERT` alone that still runs the list 226   `BOOST_COROSIO_ASSERT` alone that still runs the list
227   operations around it: 227   operations around it:
228   228  
229   - `impl` is already parked in `free_` — the list-corruption 229   - `impl` is already parked in `free_` — the list-corruption
230   hazard the class-level `pooled_sentinel` comment documents. 230   hazard the class-level `pooled_sentinel` comment documents.
231   The `is_pooled()` check runs first and returns before either 231   The `is_pooled()` check runs first and returns before either
232   list is touched. 232   list is touched.
233   - `impl` is fully detached from `live_` and is not the 233   - `impl` is fully detached from `live_` and is not the
234   sentinel (not shutting down): `live_.remove()` correctly 234   sentinel (not shutting down): `live_.remove()` correctly
235   returns false, but then the impl is reachable through 235   returns false, but then the impl is reachable through
236   neither `acquire()` nor `shutdown()`'s visit while still 236   neither `acquire()` nor `shutdown()`'s visit while still
237   allocated — stranded. Deleted instead of left to leak. 237   allocated — stranded. Deleted instead of left to leak.
238   238  
239   While shutting down, an impl that is no longer in `live_` is 239   While shutting down, an impl that is no longer in `live_` is
240   left alone rather than deleted: something else already took 240   left alone rather than deleted: something else already took
241   it out of the pool's bookkeeping (a derived pool's owner, or a 241   it out of the pool's bookkeeping (a derived pool's owner, or a
242   duplicate retirement of an impl already deleted), and deleting 242   duplicate retirement of an impl already deleted), and deleting
243   it again here could be a double free. 243   it again here could be a double free.
244   */ 244   */
HITCBC 245   35565 void recycle(Impl* impl) 245   33881 void recycle(Impl* impl)
246   { 246   {
HITCBC 247   35565 bool do_delete = false; 247   33881 bool do_delete = false;
248   { 248   {
HITCBC 249   35565 std::lock_guard lock(mutex_); 249   33881 std::lock_guard lock(mutex_);
HITCBC 250   35565 if (is_pooled(impl)) 250   33881 if (is_pooled(impl))
251   { 251   {
MISUBC 252   ✗ BOOST_COROSIO_ASSERT(false); 252   ✗ BOOST_COROSIO_ASSERT(false);
253   return; 253   return;
254   } 254   }
255   255  
HITCBC 256   35565 bool const was_live = live_.remove(impl); 256   33881 bool const was_live = live_.remove(impl);
HITCBC 257   35565 if (shutting_down_) 257   33881 if (shutting_down_)
258   { 258   {
HITCBC 259   5957 do_delete = was_live; 259   5953 do_delete = was_live;
260   } 260   }
HITCBC 261   29608 else if (was_live) 261   27928 else if (was_live)
262   { 262   {
HITCBC 263   29608 impl->refs_.store( 263   27928 impl->refs_.store(
264   pooled_sentinel, std::memory_order_relaxed); 264   pooled_sentinel, std::memory_order_relaxed);
265   // LIFO: the most recently retired impl is the 265   // LIFO: the most recently retired impl is the
266   // cache-warmest candidate for the next acquire(). 266   // cache-warmest candidate for the next acquire().
HITCBC 267   29608 free_.push_front(impl); 267   27928 free_.push_front(impl);
268   } 268   }
269   else 269   else
270   { 270   {
MISUBC 271   ✗ BOOST_COROSIO_ASSERT(false); 271   ✗ BOOST_COROSIO_ASSERT(false);
272   do_delete = true; 272   do_delete = true;
273   } 273   }
HITCBC 274   35565 } 274   33881 }
HITCBC 275   35565 if (do_delete) 275   33881 if (do_delete)
HITCBC 276   5957 delete impl; 276   5953 delete impl;
277   } 277   }
278   278  
279   /** Enter shutdown mode and visit every live impl. 279   /** Enter shutdown mode and visit every live impl.
280   280  
281   Sets shutting-down (subsequent zero-crossings delete instead 281   Sets shutting-down (subsequent zero-crossings delete instead
282   of recycling) and runs `f` on each live impl, all under one 282   of recycling) and runs `f` on each live impl, all under one
283   critical section. Each visited impl is pinned with a reference 283   critical section. Each visited impl is pinned with a reference
284   for the call and released after the critical section ends, so 284   for the call and released after the critical section ends, so
285   `f` may hold on to it; an impl whose count already reached zero 285   `f` may hold on to it; an impl whose count already reached zero
286   is retiring and is skipped rather than revived. 286   is retiring and is skipped rather than revived.
287   287  
288   @note The callback runs under the pool's non-recursive mutex; 288   @note The callback runs under the pool's non-recursive mutex;
289   it must not call recycle(), adopt(), or remove() on this pool. 289   it must not call recycle(), adopt(), or remove() on this pool.
290   */ 290   */
291   template<class F> 291   template<class F>
HITCBC 292   5565 void shutdown(F&& f) 292   5569 void shutdown(F&& f)
293   { 293   {
HITCBC 294   5565 std::vector<Impl*> pinned; 294   5569 std::vector<Impl*> pinned;
295   { 295   {
HITCBC 296   5565 std::lock_guard lock(mutex_); 296   5569 std::lock_guard lock(mutex_);
HITCBC 297   5565 shutting_down_ = true; 297   5569 shutting_down_ = true;
298   // Sized before any pin, so a failure here leaves none held. 298   // Sized before any pin, so a failure here leaves none held.
HITCBC 299   5565 std::size_t n = 0; 299   5569 std::size_t n = 0;
HITCBC 300   11521 live_.for_each([&](Impl*) { ++n; }); 300   11521 live_.for_each([&](Impl*) { ++n; });
HITCBC 301   5565 pinned.reserve(n); 301   5569 pinned.reserve(n);
HITCBC 302   5565 live_.for_each( 302   5569 live_.for_each(
HITCBC 303   18420 [&](Impl* impl) 303   18412 [&](Impl* impl)
304   { 304   {
HITCBC 305   9210 if (try_acquire(impl)) 305   9206 if (try_acquire(impl))
306   { 306   {
HITCBC 307   9209 pinned.push_back(impl); 307   9205 pinned.push_back(impl);
HITCBC 308   9209 f(impl); 308   9205 f(impl);
309   } 309   }
310   }); 310   });
HITCBC 311   5565 } 311   5569 }
HITCBC 312   14774 for (auto* impl : pinned) 312   14774 for (auto* impl : pinned)
HITCBC 313   9209 release(impl); 313   9205 release(impl);
HITCBC 314   5565 } 314   5569 }
315   }; 315   };
316   316  
317   } // namespace boost::corosio::detail 317   } // namespace boost::corosio::detail
318   318  
319   #endif 319   #endif