. /include/boost/corosio /detail /timer_service.hpp
97.07% Lines (232/239) 100.00% Functions (28/28)
[ ] prev/next file n p prev/next uncovered line
TLA Baseline Branch
Line Hits Code Line Hits Code
1   // 1   //
2   // Copyright (c) 2025 Vinnie Falco (vinnie.falco@gmail.com) 2   // Copyright (c) 2025 Vinnie Falco (vinnie.falco@gmail.com)
3   // Copyright (c) 2026 Steve Gerbino 3   // Copyright (c) 2026 Steve Gerbino
4   // 4   //
5   // Distributed under the Boost Software License, Version 1.0. (See accompanying 5   // Distributed under the Boost Software License, Version 1.0. (See accompanying
6   // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) 6   // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
7   // 7   //
8   // Official repository: https://github.com/cppalliance/corosio 8   // Official repository: https://github.com/cppalliance/corosio
9   // 9   //
10   10  
11   #ifndef BOOST_COROSIO_DETAIL_TIMER_SERVICE_HPP 11   #ifndef BOOST_COROSIO_DETAIL_TIMER_SERVICE_HPP
12   #define BOOST_COROSIO_DETAIL_TIMER_SERVICE_HPP 12   #define BOOST_COROSIO_DETAIL_TIMER_SERVICE_HPP
13   13  
14   #include <boost/corosio/detail/timer.hpp> 14   #include <boost/corosio/detail/timer.hpp>
15   #include <boost/corosio/detail/scheduler.hpp> 15   #include <boost/corosio/detail/scheduler.hpp>
16   #include <boost/corosio/detail/scheduler_op.hpp> 16   #include <boost/corosio/detail/scheduler_op.hpp>
17   #include <boost/corosio/detail/object_pool.hpp> 17   #include <boost/corosio/detail/object_pool.hpp>
18   #include <boost/corosio/detail/object_ref.hpp> 18   #include <boost/corosio/detail/object_ref.hpp>
19   #include <boost/corosio/detail/intrusive.hpp> 19   #include <boost/corosio/detail/intrusive.hpp>
20   #include <boost/corosio/detail/thread_local_ptr.hpp> 20   #include <boost/corosio/detail/thread_local_ptr.hpp>
21   #include <boost/capy/error.hpp> 21   #include <boost/capy/error.hpp>
22   #include <boost/capy/ex/execution_context.hpp> 22   #include <boost/capy/ex/execution_context.hpp>
23   #include <boost/capy/ex/abandon.hpp> 23   #include <boost/capy/ex/abandon.hpp>
24   #include <boost/capy/ex/executor_ref.hpp> 24   #include <boost/capy/ex/executor_ref.hpp>
25   #include <system_error> 25   #include <system_error>
26   26  
27   #include <atomic> 27   #include <atomic>
28   #include <chrono> 28   #include <chrono>
29   #include <coroutine> 29   #include <coroutine>
30   #include <cstddef> 30   #include <cstddef>
31   #include <limits> 31   #include <limits>
32   #include <mutex> 32   #include <mutex>
33   #include <thread> 33   #include <thread>
34   #include <stop_token> 34   #include <stop_token>
35   #include <utility> 35   #include <utility>
36   #include <vector> 36   #include <vector>
37   37  
38   namespace boost::corosio::detail { 38   namespace boost::corosio::detail {
39   39  
40   class scheduler; 40   class scheduler;
41   41  
42   /* 42   /*
43   Timer Service 43   Timer Service
44   ============= 44   =============
45   45  
46   Data Structures 46   Data Structures
47   --------------- 47   ---------------
48   waiter_node (defined in timer.hpp) holds per-waiter state: 48   waiter_node (defined in timer.hpp) holds per-waiter state:
49   coroutine handle, executor, error output, embedded 49   coroutine handle, executor, error output, embedded
50   completion_op. Each concurrent co_await t.wait() embeds one 50   completion_op. Each concurrent co_await t.wait() embeds one
51   waiter_node in the awaitable on the suspended coroutine's 51   waiter_node in the awaitable on the suspended coroutine's
52   frame — waits perform no allocation. 52   frame — waits perform no allocation.
53   53  
54   timer::implementation holds per-timer state: expiry, heap 54   timer::implementation holds per-timer state: expiry, heap
55   index, and the single published waiter. Each timer holds 55   index, and the single published waiter. Each timer holds
56   at most one waiter; process_expired's local cross-timer drain 56   at most one waiter; process_expired's local cross-timer drain
57   list still threads waiters through their intrusive hooks when 57   list still threads waiters through their intrusive hooks when
58   collecting several timers' waiters past the lock. 58   collecting several timers' waiters past the lock.
59   59  
60   timer_service owns a min-heap of active timers and recycles 60   timer_service owns a min-heap of active timers and recycles
61   retired impls through a per-service object_pool, fronted by a 61   retired impls through a per-service object_pool, fronted by a
62   thread-local single-slot cache. The heap is ordered by expiry 62   thread-local single-slot cache. The heap is ordered by expiry
63   time; the scheduler queries nearest_expiry() to set the 63   time; the scheduler queries nearest_expiry() to set the
64   epoll/timerfd timeout. 64   epoll/timerfd timeout.
65   65  
66   Optimization Strategy 66   Optimization Strategy
67   --------------------- 67   ---------------------
68   1. Deferred heap insertion — expires_after() stores the expiry 68   1. Deferred heap insertion — expires_after() stores the expiry
69   but does not insert into the heap. Insertion happens in wait(). 69   but does not insert into the heap. Insertion happens in wait().
70   2. Thread-local impl cache — single-slot per-thread cache, in 70   2. Thread-local impl cache — single-slot per-thread cache, in
71   front of the object_pool fallback; keeps a thread's hot impl 71   front of the object_pool fallback; keeps a thread's hot impl
72   out of the shared free list. 72   out of the shared free list.
73   3. Frame-resident waiter_node with embedded completion_op — 73   3. Frame-resident waiter_node with embedded completion_op —
74   eliminates heap allocation per wait/fire/cancel. 74   eliminates heap allocation per wait/fire/cancel.
75   4. Cached nearest expiry — atomic avoids mutex in nearest_expiry(). 75   4. Cached nearest expiry — atomic avoids mutex in nearest_expiry().
76   5. might_have_pending_waits_ flag — skips lock when no wait issued. 76   5. might_have_pending_waits_ flag — skips lock when no wait issued.
77   77  
78   Concurrency 78   Concurrency
79   ----------- 79   -----------
80   stop_token callbacks can fire from any thread. The impl_ 80   stop_token callbacks can fire from any thread. The impl_
81   pointer on waiter_node is used as a "still in list" marker. 81   pointer on waiter_node is used as a "still in list" marker.
82   A waiter_node's storage is the suspended coroutine's frame: 82   A waiter_node's storage is the suspended coroutine's frame:
83   every completion path must finish touching the node before 83   every completion path must finish touching the node before
84   posting the continuation or destroying the handle. 84   posting the continuation or destroying the handle.
85   */ 85   */
86   86  
87   inline void timer_service_invalidate_cache() noexcept; 87   inline void timer_service_invalidate_cache() noexcept;
88   88  
89   /** The timer service's recycling pool. 89   /** The timer service's recycling pool.
90   90  
91   Adds no behavior of its own — it exists only to re-expose 91   Adds no behavior of its own — it exists only to re-expose
92   `adopt()` and `remove()` as public, so the timer service's 92   `adopt()` and `remove()` as public, so the timer service's
93   thread-local cache can transfer a cached impl's `live_` tracking 93   thread-local cache can transfer a cached impl's `live_` tracking
94   on both sides of the handoff: `construct()` re-adopts a 94   on both sides of the handoff: `construct()` re-adopts a
95   TL-cached impl as live, `shutdown()` detaches one before its 95   TL-cached impl as live, `shutdown()` detaches one before its
96   direct delete. 96   direct delete.
97   */ 97   */
98   class timer_object_pool : public object_pool<timer::implementation> 98   class timer_object_pool : public object_pool<timer::implementation>
99   { 99   {
100   public: 100   public:
101   using object_pool::adopt; 101   using object_pool::adopt;
102   using object_pool::remove; 102   using object_pool::remove;
103   }; 103   };
104   104  
105   // timer_service class body — member function definitions are 105   // timer_service class body — member function definitions are
106   // out-of-class (after implementation and waiter_node are complete) 106   // out-of-class (after implementation and waiter_node are complete)
107   class BOOST_COROSIO_DECL timer_service final 107   class BOOST_COROSIO_DECL timer_service final
108   : public capy::execution_context::service 108   : public capy::execution_context::service
109   , public io_object::io_service 109   , public io_object::io_service
110   { 110   {
111   friend struct timer::implementation; 111   friend struct timer::implementation;
112   112  
113   public: 113   public:
114   using clock_type = std::chrono::steady_clock; 114   using clock_type = std::chrono::steady_clock;
115   using time_point = clock_type::time_point; 115   using time_point = clock_type::time_point;
116   116  
117   /// Type-erased callback for earliest-expiry-changed notifications. 117   /// Type-erased callback for earliest-expiry-changed notifications.
118   class callback 118   class callback
119   { 119   {
120   void* ctx_ = nullptr; 120   void* ctx_ = nullptr;
121   void (*fn_)(void*) = nullptr; 121   void (*fn_)(void*) = nullptr;
122   122  
123   public: 123   public:
124   /// Construct an empty callback. 124   /// Construct an empty callback.
HITCBC 125   4415 callback() = default; 125   4415 callback() = default;
126   126  
127   /// Construct a callback with the given context and function. 127   /// Construct a callback with the given context and function.
HITCBC 128   4415 callback(void* ctx, void (*fn)(void*)) noexcept : ctx_(ctx), fn_(fn) {} 128   4415 callback(void* ctx, void (*fn)(void*)) noexcept : ctx_(ctx), fn_(fn) {}
129   129  
130   /// Return true if the callback is non-empty. 130   /// Return true if the callback is non-empty.
131   explicit operator bool() const noexcept 131   explicit operator bool() const noexcept
132   { 132   {
133   return fn_ != nullptr; 133   return fn_ != nullptr;
134   } 134   }
135   135  
136   /// Invoke the callback. 136   /// Invoke the callback.
HITCBC 137   8057 void operator()() const 137   8092 void operator()() const
138   { 138   {
HITCBC 139   8057 if (fn_) 139   8092 if (fn_)
HITCBC 140   8057 fn_(ctx_); 140   8092 fn_(ctx_);
HITCBC 141   8057 } 141   8092 }
142   }; 142   };
143   143  
144   private: 144   private:
145   struct heap_entry 145   struct heap_entry
146   { 146   {
147   time_point time_; 147   time_point time_;
148   timer::implementation* timer_; 148   timer::implementation* timer_;
149   }; 149   };
150   150  
151   scheduler* sched_ = nullptr; 151   scheduler* sched_ = nullptr;
152   BOOST_COROSIO_MSVC_WARNING_PUSH 152   BOOST_COROSIO_MSVC_WARNING_PUSH
153   BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // std:: members, dll-interface 153   BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // std:: members, dll-interface
154   mutable std::mutex mutex_; 154   mutable std::mutex mutex_;
155   std::vector<heap_entry> heap_; 155   std::vector<heap_entry> heap_;
156   timer_object_pool pool_; 156   timer_object_pool pool_;
157   callback on_earliest_changed_; 157   callback on_earliest_changed_;
158   bool shutting_down_ = false; 158   bool shutting_down_ = false;
159   159  
160   // Cancellers that claimed a waiter and have yet to post it; 160   // Cancellers that claimed a waiter and have yet to post it;
161   // shutdown waits them out so the scheduler's drain finds the post. 161   // shutdown waits them out so the scheduler's drain finds the post.
162   std::size_t cancels_posting_ = 0; // guarded by mutex_ 162   std::size_t cancels_posting_ = 0; // guarded by mutex_
163   // Avoids mutex in nearest_expiry() and empty() 163   // Avoids mutex in nearest_expiry() and empty()
164   mutable std::atomic<std::int64_t> cached_nearest_ns_{ 164   mutable std::atomic<std::int64_t> cached_nearest_ns_{
165   (std::numeric_limits<std::int64_t>::max)()}; 165   (std::numeric_limits<std::int64_t>::max)()};
166   BOOST_COROSIO_MSVC_WARNING_POP 166   BOOST_COROSIO_MSVC_WARNING_POP
167   167  
168   public: 168   public:
169   /// Construct the timer service bound to a scheduler. 169   /// Construct the timer service bound to a scheduler.
HITCBC 170   4415 inline timer_service(capy::execution_context&, scheduler& sched) 170   4415 inline timer_service(capy::execution_context&, scheduler& sched)
HITCBC 171   4415 : sched_(&sched) 171   4415 : sched_(&sched)
172   { 172   {
HITCBC 173   4415 } 173   4415 }
174   174  
175   /// Return the associated scheduler. 175   /// Return the associated scheduler.
HITCBC 176   34071 inline scheduler& get_scheduler() noexcept 176   34908 inline scheduler& get_scheduler() noexcept
177   { 177   {
HITCBC 178   34071 return *sched_; 178   34908 return *sched_;
179   } 179   }
180   180  
181   /// Destroy the timer service. 181   /// Destroy the timer service.
HITCBC 182   8830 ~timer_service() override = default; 182   8830 ~timer_service() override = default;
183   183  
184   timer_service(timer_service const&) = delete; 184   timer_service(timer_service const&) = delete;
185   timer_service& operator=(timer_service const&) = delete; 185   timer_service& operator=(timer_service const&) = delete;
186   186  
187   /// Register a callback invoked when the earliest expiry changes. 187   /// Register a callback invoked when the earliest expiry changes.
HITCBC 188   4415 inline void set_on_earliest_changed(callback cb) 188   4415 inline void set_on_earliest_changed(callback cb)
189   { 189   {
HITCBC 190   4415 on_earliest_changed_ = cb; 190   4415 on_earliest_changed_ = cb;
HITCBC 191   4415 } 191   4415 }
192   192  
193   /// Return true if no timers are in the heap. 193   /// Return true if no timers are in the heap.
194   inline bool empty() const noexcept 194   inline bool empty() const noexcept
195   { 195   {
196   return cached_nearest_ns_.load(std::memory_order_acquire) == 196   return cached_nearest_ns_.load(std::memory_order_acquire) ==
197   (std::numeric_limits<std::int64_t>::max)(); 197   (std::numeric_limits<std::int64_t>::max)();
198   } 198   }
199   199  
200   /// Return the nearest timer expiry without acquiring the mutex. 200   /// Return the nearest timer expiry without acquiring the mutex.
HITCBC 201   31661 inline time_point nearest_expiry() const noexcept 201   27007 inline time_point nearest_expiry() const noexcept
202   { 202   {
HITCBC 203   31661 auto ns = cached_nearest_ns_.load(std::memory_order_acquire); 203   27007 auto ns = cached_nearest_ns_.load(std::memory_order_acquire);
HITCBC 204   31661 return time_point(time_point::duration(ns)); 204   27007 return time_point(time_point::duration(ns));
205   } 205   }
206   206  
207   /// Cancel all pending timers and free cached resources. 207   /// Cancel all pending timers and free cached resources.
208   inline void shutdown() override; 208   inline void shutdown() override;
209   209  
210   /// Construct a new timer implementation. 210   /// Construct a new timer implementation.
211   inline io_object::implementation* construct() override; 211   inline io_object::implementation* construct() override;
212   212  
213   /// Destroy a timer implementation, cancelling pending waiters. 213   /// Destroy a timer implementation, cancelling pending waiters.
214   inline void destroy(io_object::implementation* p) override; 214   inline void destroy(io_object::implementation* p) override;
215   215  
216   /// Cancel and recycle a timer implementation. 216   /// Cancel and recycle a timer implementation.
217   inline void destroy_impl(timer::implementation& impl); 217   inline void destroy_impl(timer::implementation& impl);
218   218  
219   /// Publish the timer's waiter and insert the timer into the heap. 219   /// Publish the timer's waiter and insert the timer into the heap.
220   inline void insert_waiter(timer::implementation& impl, waiter_node* w); 220   inline void insert_waiter(timer::implementation& impl, waiter_node* w);
221   221  
222   /// Cancel the timer's published waiter, if any. 222   /// Cancel the timer's published waiter, if any.
223   inline void cancel_timer(timer::implementation& impl); 223   inline void cancel_timer(timer::implementation& impl);
224   224  
225   /// Cancel one specific waiter ( stop_token callback path ). 225   /// Cancel one specific waiter ( stop_token callback path ).
226   inline void cancel_waiter(waiter_node* w); 226   inline void cancel_waiter(waiter_node* w);
227   227  
228   /// Complete all waiters whose timers have expired. 228   /// Complete all waiters whose timers have expired.
229   inline std::size_t process_expired(); 229   inline std::size_t process_expired();
230   230  
231   private: 231   private:
HITCBC 232   181465 inline void refresh_cached_nearest() noexcept 232   179815 inline void refresh_cached_nearest() noexcept
233   { 233   {
HITCBC 234   181465 auto ns = heap_.empty() ? (std::numeric_limits<std::int64_t>::max)() 234   179815 auto ns = heap_.empty() ? (std::numeric_limits<std::int64_t>::max)()
HITCBC 235   60170 : heap_[0].time_.time_since_epoch().count(); 235   62724 : heap_[0].time_.time_since_epoch().count();
HITCBC 236   181465 cached_nearest_ns_.store(ns, std::memory_order_release); 236   179815 cached_nearest_ns_.store(ns, std::memory_order_release);
HITCBC 237   181465 } 237   179815 }
238   238  
239   inline void remove_timer_impl(timer::implementation& impl); 239   inline void remove_timer_impl(timer::implementation& impl);
240   inline void up_heap(std::size_t index); 240   inline void up_heap(std::size_t index);
241   inline void down_heap(std::size_t index); 241   inline void down_heap(std::size_t index);
242   inline void swap_heap(std::size_t i1, std::size_t i2); 242   inline void swap_heap(std::size_t i1, std::size_t i2);
243   }; 243   };
244   244  
245   // Thread-local single-slot impl cache, validated by comparing svc_. 245   // Thread-local single-slot impl cache, validated by comparing svc_.
246   // Keeps a hot impl out of the shared free list; both sides still 246   // Keeps a hot impl out of the shared free list; both sides still
247   // take the pool lock to move it in and out of live tracking. 247   // take the pool lock to move it in and out of live tracking.
248   // Cleared by timer_service_invalidate_cache() during shutdown. 248   // Cleared by timer_service_invalidate_cache() during shutdown.
249   249  
250   inline thread_local_ptr<timer::implementation> tl_cached_impl; 250   inline thread_local_ptr<timer::implementation> tl_cached_impl;
251   251  
252   // The POD TLS slot above never runs destructors, so a short-lived 252   // The POD TLS slot above never runs destructors, so a short-lived
253   // run() thread would leak its cached impl. Each push arms this 253   // run() thread would leak its cached impl. Each push arms this
254   // owner, whose destructor frees the slot at thread exit. A cached 254   // owner, whose destructor frees the slot at thread exit. A cached
255   // entry is a quiescent heap object (nothing in the heap or free 255   // entry is a quiescent heap object (nothing in the heap or free
256   // list) and deletion touches no service state, so it is safe after 256   // list) and deletion touches no service state, so it is safe after
257   // the owning service is gone (the stale-entry path in 257   // the owning service is gone (the stale-entry path in
258   // try_pop_tl_cache deletes the same way). 258   // try_pop_tl_cache deletes the same way).
259   struct tl_cache_owner 259   struct tl_cache_owner
260   { 260   {
HITCBC 261   47 ~tl_cache_owner() 261   55 ~tl_cache_owner()
262   { 262   {
HITCBC 263   47 delete tl_cached_impl.get(); 263   55 delete tl_cached_impl.get();
HITCBC 264   47 tl_cached_impl.set(nullptr); 264   55 tl_cached_impl.set(nullptr);
HITCBC 265   47 } 265   55 }
266   }; 266   };
267   267  
268   inline void 268   inline void
HITCBC 269   15141 arm_tl_cache_cleanup() noexcept 269   15174 arm_tl_cache_cleanup() noexcept
270   { 270   {
HITCBC 271   15141 [[maybe_unused]] thread_local tl_cache_owner owner; 271   15174 [[maybe_unused]] thread_local tl_cache_owner owner;
HITCBC 272   15141 } 272   15174 }
273   273  
274   inline timer::implementation* 274   inline timer::implementation*
HITCBC 275   19584 try_pop_tl_cache(timer_service* svc) noexcept 275   19999 try_pop_tl_cache(timer_service* svc) noexcept
276   { 276   {
HITCBC 277   19584 auto* impl = tl_cached_impl.get(); 277   19999 auto* impl = tl_cached_impl.get();
HITCBC 278   19584 if (impl) 278   19999 if (impl)
279   { 279   {
HITCBC 280   14651 tl_cached_impl.set(nullptr); 280   14676 tl_cached_impl.set(nullptr);
HITCBC 281   14651 if (impl->svc_ == svc) 281   14676 if (impl->svc_ == svc)
HITCBC 282   14651 return impl; 282   14676 return impl;
283   // Stale impl from a destroyed service 283   // Stale impl from a destroyed service
MISUBC 284   ✗ delete impl; 284   ✗ delete impl;
285   } 285   }
HITCBC 286   4933 return nullptr; 286   5323 return nullptr;
287   } 287   }
288   288  
289   inline bool 289   inline bool
HITCBC 290   16337 try_push_tl_cache(timer::implementation* impl) noexcept 290   16748 try_push_tl_cache(timer::implementation* impl) noexcept
291   { 291   {
HITCBC 292   16337 if (!tl_cached_impl.get()) 292   16748 if (!tl_cached_impl.get())
293   { 293   {
HITCBC 294   15141 arm_tl_cache_cleanup(); 294   15174 arm_tl_cache_cleanup();
HITCBC 295   15141 tl_cached_impl.set(impl); 295   15174 tl_cached_impl.set(impl);
HITCBC 296   15141 return true; 296   15174 return true;
297   } 297   }
HITCBC 298   1196 return false; 298   1574 return false;
299   } 299   }
300   300  
301   inline void 301   inline void
HITCBC 302   8830 timer_service_invalidate_cache() noexcept 302   8830 timer_service_invalidate_cache() noexcept
303   { 303   {
HITCBC 304   8830 delete tl_cached_impl.get(); 304   8830 delete tl_cached_impl.get();
HITCBC 305   8830 tl_cached_impl.set(nullptr); 305   8830 tl_cached_impl.set(nullptr);
HITCBC 306   8830 } 306   8830 }
307   307  
308   // timer_service out-of-class member function definitions 308   // timer_service out-of-class member function definitions
309   309  
310   inline void 310   inline void
HITCBC 311   8830 timer_service::shutdown() 311   8830 timer_service::shutdown()
312   { 312   {
HITCBC 313   8830 timer_service_invalidate_cache(); 313   8830 timer_service_invalidate_cache();
HITCBC 314   8830 shutting_down_ = true; 314   8830 shutting_down_ = true;
315   // Flag only; the real drain walks the expiry heap below, not 315   // Flag only; the real drain walks the expiry heap below, not
316   // object_pool's live_ list, so the visit callback is a no-op. 316   // object_pool's live_ list, so the visit callback is a no-op.
HITCBC 317   8830 pool_.shutdown([](timer::implementation*) {}); 317   8830 pool_.shutdown([](timer::implementation*) {});
318   318  
319   // Snapshot impls and detach them from the heap so that 319   // Snapshot impls and detach them from the heap so that
320   // coroutine-owned timer destructors (run by the unwinds the 320   // coroutine-owned timer destructors (run by the unwinds the
321   // abandons below start) cannot re-enter remove_timer_impl() and 321   // abandons below start) cannot re-enter remove_timer_impl() and
322   // mutate the vector during iteration. A stop requested on another thread 322   // mutate the vector during iteration. A stop requested on another thread
323   // reaches cancel_waiter() under the same lock: whichever side 323   // reaches cancel_waiter() under the same lock: whichever side
324   // claims a waiter first owns it, and a waiter its canceller 324   // claims a waiter first owns it, and a waiter its canceller
325   // claimed is posted, so the scheduler's drain abandons it. 325   // claimed is posted, so the scheduler's drain abandons it.
HITCBC 326   8830 std::vector<timer::implementation*> impls; 326   8830 std::vector<timer::implementation*> impls;
HITCBC 327   8830 std::vector<waiter_node*> waiters; 327   8830 std::vector<waiter_node*> waiters;
328   { 328   {
HITCBC 329   8830 std::lock_guard lock(mutex_); 329   8830 std::lock_guard lock(mutex_);
HITCBC 330   8830 impls.reserve(heap_.size()); 330   8830 impls.reserve(heap_.size());
HITCBC 331   8830 waiters.reserve(heap_.size()); 331   8830 waiters.reserve(heap_.size());
HITCBC 332   12073 for (auto& entry : heap_) 332   12068 for (auto& entry : heap_)
333   { 333   {
HITCBC 334   3243 entry.timer_->heap_index_.store( 334   3238 entry.timer_->heap_index_.store(
335   (std::numeric_limits<std::size_t>::max)(), 335   (std::numeric_limits<std::size_t>::max)(),
336   std::memory_order_relaxed); 336   std::memory_order_relaxed);
HITCBC 337   3243 impls.push_back(entry.timer_); 337   3238 impls.push_back(entry.timer_);
HITCBC 338   3243 if (auto* w = std::exchange(entry.timer_->waiter_, nullptr)) 338   3238 if (auto* w = std::exchange(entry.timer_->waiter_, nullptr))
339   { 339   {
HITCBC 340   3243 w->impl_ = nullptr; 340   3238 w->impl_ = nullptr;
HITCBC 341   3243 waiters.push_back(w); 341   3238 waiters.push_back(w);
342   } 342   }
343   } 343   }
HITCBC 344   8830 heap_.clear(); 344   8830 heap_.clear();
HITCBC 345   8830 cached_nearest_ns_.store( 345   8830 cached_nearest_ns_.store(
346   (std::numeric_limits<std::int64_t>::max)(), 346   (std::numeric_limits<std::int64_t>::max)(),
347   std::memory_order_release); 347   std::memory_order_release);
HITCBC 348   8830 } 348   8830 }
349   349  
350   // A stop on another thread may have claimed its waiter before the 350   // A stop on another thread may have claimed its waiter before the
351   // pass above and still be posting it; the scheduler drains next, 351   // pass above and still be posting it; the scheduler drains next,
352   // and must find that post. 352   // and must find that post.
353   for (;;) 353   for (;;)
354   { 354   {
355   { 355   {
HITCBC 356   8830 std::lock_guard lock(mutex_); 356   11518 std::lock_guard lock(mutex_);
HITCBC 357   8830 if (cancels_posting_ == 0) 357   11518 if (cancels_posting_ == 0)
HITCBC 358   8830 break; 358   8830 break;
HITCBC 359   8830 } 359   11518 }
HITGBC 360   ✗ std::this_thread::yield(); 360   2688 std::this_thread::yield();
HITGBC 361   ✗ } 361   2688 }
362   362  
363   // Cancel waiting timers. Each waiter called work_started() 363   // Cancel waiting timers. Each waiter called work_started()
364   // in implementation::wait(); a waiter given up here never 364   // in implementation::wait(); a waiter given up here never
365   // completes, so its work is finished here to keep the count 365   // completes, so its work is finished here to keep the count
366   // balanced. Resetting the stop callback outside the lock 366   // balanced. Resetting the stop callback outside the lock
367   // waits out a canceller already running, which finds the waiter 367   // waits out a canceller already running, which finds the waiter
368   // claimed. 368   // claimed.
HITCBC 369   12073 for (auto* w : waiters) 369   12068 for (auto* w : waiters)
370   { 370   {
HITCBC 371   3243 w->reset_stop_cb(); 371   3238 w->reset_stop_cb();
HITCBC 372   3243 auto c = std::exchange(w->cont_, {}); 372   3238 auto c = std::exchange(w->cont_, {});
HITCBC 373   3243 sched_->work_finished(); 373   3238 sched_->work_finished();
374   // Unwinding the frame also ends the node's storage 374   // Unwinding the frame also ends the node's storage
HITCBC 375   3243 if (c.h) 375   3238 if (c.h)
HITCBC 376   3243 capy::abandon(c); 376   3238 capy::abandon(c);
377   } 377   }
378   378  
HITCBC 379   12073 for (auto* impl : impls) 379   12068 for (auto* impl : impls)
380   { 380   {
381   // Unlink from the pool's live_ list before the direct delete 381   // Unlink from the pool's live_ list before the direct delete
382   // below, or ~object_pool()'s unconditional sweep would delete 382   // below, or ~object_pool()'s unconditional sweep would delete
383   // this impl a second time. 383   // this impl a second time.
HITCBC 384   3243 [[maybe_unused]] bool const removed = pool_.remove(impl); 384   3238 [[maybe_unused]] bool const removed = pool_.remove(impl);
HITCBC 385   3243 BOOST_COROSIO_ASSERT(removed); 385   3238 BOOST_COROSIO_ASSERT(removed);
HITCBC 386   3243 delete impl; 386   3238 delete impl;
387   } 387   }
388   388  
389   // Anything still parked in pool_ (free-listed or live with no 389   // Anything still parked in pool_ (free-listed or live with no
390   // pending wait at shutdown) is freed by object_pool's own destructor. 390   // pending wait at shutdown) is freed by object_pool's own destructor.
HITCBC 391   8830 } 391   8830 }
392   392  
393   inline io_object::implementation* 393   inline io_object::implementation*
HITCBC 394   19584 timer_service::construct() 394   19999 timer_service::construct()
395   { 395   {
HITCBC 396   19584 timer::implementation* impl = try_pop_tl_cache(this); 396   19999 timer::implementation* impl = try_pop_tl_cache(this);
HITCBC 397   19584 if (impl) 397   19999 if (impl)
398   { 398   {
HITCBC 399   14651 impl->svc_ = this; 399   14676 impl->svc_ = this;
HITCBC 400   14651 impl->reuse(); 400   14676 impl->reuse();
HITCBC 401   14651 impl->refs_.store(1, std::memory_order_relaxed); 401   14676 impl->refs_.store(1, std::memory_order_relaxed);
402   // The TL push that parked impl here also removed it from 402   // The TL push that parked impl here also removed it from
403   // live_ (see destroy_impl) — re-adopt it now that it is live 403   // live_ (see destroy_impl) — re-adopt it now that it is live
404   // again. 404   // again.
HITCBC 405   14651 pool_.adopt(impl); 405   14676 pool_.adopt(impl);
HITCBC 406   14651 return impl; 406   14676 return impl;
407   } 407   }
408   408  
HITCBC 409   4933 return pool_.acquire(*this); 409   5323 return pool_.acquire(*this);
410   } 410   }
411   411  
412   inline void 412   inline void
HITCBC 413   19584 timer_service::destroy(io_object::implementation* p) 413   19999 timer_service::destroy(io_object::implementation* p)
414   { 414   {
415   // During shutdown the drain loop owns every impl and deletes 415   // During shutdown the drain loop owns every impl and deletes
416   // them directly. A frame destroyed by that loop can unwind a 416   // them directly. A frame destroyed by that loop can unwind a
417   // handle whose impl was freed in an earlier iteration (a 417   // handle whose impl was freed in an earlier iteration (a
418   // timeout's parent frame owns the timeout timer while 418   // timeout's parent frame owns the timeout timer while
419   // suspended on the inner delay's timer), so bail out before 419   // suspended on the inner delay's timer), so bail out before
420   // even downcasting the pointer. 420   // even downcasting the pointer.
HITCBC 421   19584 if (shutting_down_) 421   19999 if (shutting_down_)
HITCBC 422   3247 return; 422   3251 return;
HITCBC 423   16337 destroy_impl(static_cast<timer::implementation&>(*p)); 423   16748 destroy_impl(static_cast<timer::implementation&>(*p));
424   } 424   }
425   425  
426   inline void 426   inline void
HITCBC 427   16337 timer_service::destroy_impl(timer::implementation& impl) 427   16748 timer_service::destroy_impl(timer::implementation& impl)
428   { 428   {
429   // During shutdown the impl is owned by the shutdown loop. 429   // During shutdown the impl is owned by the shutdown loop.
430   // Re-entering here (from a coroutine-owned timer destructor 430   // Re-entering here (from a coroutine-owned timer destructor
431   // triggered by h.destroy()) must not modify the heap or 431   // triggered by h.destroy()) must not modify the heap or
432   // recycle the impl — shutdown deletes it directly. 432   // recycle the impl — shutdown deletes it directly.
HITCBC 433   16337 if (shutting_down_) 433   16748 if (shutting_down_)
MISUBC 434   ✗ return; 434   ✗ return;
435   435  
HITCBC 436   16337 cancel_timer(impl); 436   16748 cancel_timer(impl);
437   437  
HITCBC 438   32674 if (impl.heap_index_.load(std::memory_order_relaxed) != 438   33496 if (impl.heap_index_.load(std::memory_order_relaxed) !=
HITCBC 439   16337 (std::numeric_limits<std::size_t>::max)()) 439   16748 (std::numeric_limits<std::size_t>::max)())
440   { 440   {
MISUBC 441   ✗ std::lock_guard lock(mutex_); 441   ✗ std::lock_guard lock(mutex_);
MISUBC 442   ✗ remove_timer_impl(impl); 442   ✗ remove_timer_impl(impl);
MISUBC 443   ✗ refresh_cached_nearest(); 443   ✗ refresh_cached_nearest();
MISUBC 444   ✗ } 444   ✗ }
445   445  
HITCBC 446   16337 if (try_push_tl_cache(&impl)) 446   16748 if (try_push_tl_cache(&impl))
447   { 447   {
448   // The TL slot takes sole ownership: it is deleted directly by 448   // The TL slot takes sole ownership: it is deleted directly by
449   // tl_cache_owner / the stale-service path, never through this 449   // tl_cache_owner / the stale-service path, never through this
450   // pool, so it must leave live_ now or ~object_pool() would see 450   // pool, so it must leave live_ now or ~object_pool() would see
451   // it as abandoned (and, at the wrong moment, double-delete it 451   // it as abandoned (and, at the wrong moment, double-delete it
452   // after a later construct() re-adopts it). 452   // after a later construct() re-adopts it).
HITCBC 453   15141 [[maybe_unused]] bool const removed = pool_.remove(&impl); 453   15174 [[maybe_unused]] bool const removed = pool_.remove(&impl);
HITCBC 454   15141 BOOST_COROSIO_ASSERT(removed); 454   15174 BOOST_COROSIO_ASSERT(removed);
HITCBC 455   15141 return; 455   15174 return;
456   } 456   }
457   457  
458   // No op keepalives on a timer impl: this is the only reference, 458   // No op keepalives on a timer impl: this is the only reference,
459   // so release() always recycles immediately via retire(). 459   // so release() always recycles immediately via retire().
HITCBC 460   1196 release(&impl); 460   1574 release(&impl);
461   } 461   }
462   462  
463   inline void 463   inline void
HITCBC 464   22918 timer_service::insert_waiter(timer::implementation& impl, waiter_node* w) 464   23801 timer_service::insert_waiter(timer::implementation& impl, waiter_node* w)
465   { 465   {
HITCBC 466   22918 bool notify = false; 466   23801 bool notify = false;
HITCBC 467   22918 bool lost_cancel = false; 467   23801 bool lost_cancel = false;
468   { 468   {
HITCBC 469   22918 std::lock_guard lock(mutex_); 469   23801 std::lock_guard lock(mutex_);
470   // Grow before publishing anything, so the push_back below 470   // Grow before publishing anything, so the push_back below
471   // cannot throw: a failure here leaves the waiter untouched, 471   // cannot throw: a failure here leaves the waiter untouched,
472   // the strong guarantee rearm_wait's recovery relies on. 472   // the strong guarantee rearm_wait's recovery relies on.
HITCBC 473   22918 if (impl.heap_index_.load(std::memory_order_relaxed) == 473   23801 if (impl.heap_index_.load(std::memory_order_relaxed) ==
HITCBC 474   45836 (std::numeric_limits<std::size_t>::max)() && 474   47602 (std::numeric_limits<std::size_t>::max)() &&
HITCBC 475   22918 heap_.size() == heap_.capacity()) 475   23801 heap_.size() == heap_.capacity())
HITCBC 476   916 heap_.reserve(heap_.capacity() == 0 ? 16 : 2 * heap_.capacity()); 476   934 heap_.reserve(heap_.capacity() == 0 ? 16 : 2 * heap_.capacity());
477   // Publish: from here the waiter is visible to the fire path and 477   // Publish: from here the waiter is visible to the fire path and
478   // to its own stop callback (impl_ non-null enables cancel_waiter). 478   // to its own stop callback (impl_ non-null enables cancel_waiter).
HITCBC 479   22918 w->impl_ = &impl; 479   23801 w->impl_ = &impl;
HITCBC 480   45836 if (impl.heap_index_.load(std::memory_order_relaxed) == 480   47602 if (impl.heap_index_.load(std::memory_order_relaxed) ==
HITCBC 481   22918 (std::numeric_limits<std::size_t>::max)()) 481   23801 (std::numeric_limits<std::size_t>::max)())
482   { 482   {
HITCBC 483   22918 impl.heap_index_.store(heap_.size(), std::memory_order_relaxed); 483   23801 impl.heap_index_.store(heap_.size(), std::memory_order_relaxed);
HITCBC 484   22918 heap_.push_back({impl.expiry_, &impl}); 484   23801 heap_.push_back({impl.expiry_, &impl});
HITCBC 485   22918 up_heap(heap_.size() - 1); 485   23801 up_heap(heap_.size() - 1);
HITCBC 486   22918 notify = (impl.heap_index_.load(std::memory_order_relaxed) == 0); 486   23801 notify = (impl.heap_index_.load(std::memory_order_relaxed) == 0);
HITCBC 487   22918 refresh_cached_nearest(); 487   23801 refresh_cached_nearest();
488   } 488   }
HITCBC 489   22918 BOOST_COROSIO_ASSERT(impl.waiter_ == nullptr); 489   23801 BOOST_COROSIO_ASSERT(impl.waiter_ == nullptr);
HITCBC 490   22918 impl.waiter_ = w; 490   23801 impl.waiter_ = w;
491   491  
492   // Lost-cancel re-check: a stop requested after the canceller was 492   // Lost-cancel re-check: a stop requested after the canceller was
493   // armed in wait() but before this publication found impl_ null 493   // armed in wait() but before this publication found impl_ null
494   // and returned a no-op. Observe it now and undo the insertion. 494   // and returned a no-op. Observe it now and undo the insertion.
HITCBC 495   22918 if (w->token_->stop_requested()) 495   23801 if (w->token_->stop_requested())
496   { 496   {
HITCBC 497   2 w->impl_ = nullptr; 497   97 w->impl_ = nullptr;
HITCBC 498   2 impl.waiter_ = nullptr; 498   97 impl.waiter_ = nullptr;
HITCBC 499   2 remove_timer_impl(impl); 499   97 remove_timer_impl(impl);
HITCBC 500   2 impl.might_have_pending_waits_.store( 500   97 impl.might_have_pending_waits_.store(
501   false, std::memory_order_release); 501   false, std::memory_order_release);
HITCBC 502   2 refresh_cached_nearest(); 502   97 refresh_cached_nearest();
HITCBC 503   2 lost_cancel = true; 503   97 lost_cancel = true;
HITCBC 504   2 notify = false; // insertion undone; nearest unchanged 504   97 notify = false; // insertion undone; nearest unchanged
505   } 505   }
HITCBC 506   22918 } 506   23801 }
HITCBC 507   22918 if (notify) 507   23801 if (notify)
HITCBC 508   8057 on_earliest_changed_(); 508   8092 on_earliest_changed_();
HITCBC 509   22918 if (lost_cancel) 509   23801 if (lost_cancel)
510   { 510   {
HITCBC 511   2 w->ec_ = make_error_code(capy::error::canceled); 511   97 w->ec_ = make_error_code(capy::error::canceled);
HITCBC 512   2 sched_->post(&w->op_); 512   97 sched_->post(&w->op_);
513   } 513   }
HITCBC 514   22918 } 514   23801 }
515   515  
516   inline void 516   inline void
HITCBC 517   16337 timer_service::cancel_timer(timer::implementation& impl) 517   16748 timer_service::cancel_timer(timer::implementation& impl)
518   { 518   {
519   // Acquire pairs with the drains' release: a caller that skips the 519   // Acquire pairs with the drains' release: a caller that skips the
520   // lock may recycle the impl next, so the drain's writes to it must 520   // lock may recycle the impl next, so the drain's writes to it must
521   // happen before. 521   // happen before.
HITCBC 522   16337 if (!impl.might_have_pending_waits_.load(std::memory_order_acquire)) 522   16748 if (!impl.might_have_pending_waits_.load(std::memory_order_acquire))
HITCBC 523   16335 return; 523   16746 return;
524   524  
525   // No unlocked already-done fast-out here: it would need the 525   // No unlocked already-done fast-out here: it would need the
526   // non-atomic waiter_ (a race with concurrent drains), and an 526   // non-atomic waiter_ (a race with concurrent drains), and an
527   // index-only check is lifetime-unsafe because npos is stored 527   // index-only check is lifetime-unsafe because npos is stored
528   // before the drain finishes touching the impl. A stale-true 528   // before the drain finishes touching the impl. A stale-true
529   // flag is rare with the stateless API; the locked path below 529   // flag is rare with the stateless API; the locked path below
530   // re-validates. 530   // re-validates.
531   531  
HITCBC 532   2 waiter_node* canceled = nullptr; 532   2 waiter_node* canceled = nullptr;
533   533  
534   { 534   {
HITCBC 535   2 std::lock_guard lock(mutex_); 535   2 std::lock_guard lock(mutex_);
HITCBC 536   2 remove_timer_impl(impl); 536   2 remove_timer_impl(impl);
HITCBC 537   2 canceled = std::exchange(impl.waiter_, nullptr); 537   2 canceled = std::exchange(impl.waiter_, nullptr);
HITCBC 538   2 if (canceled) 538   2 if (canceled)
HITCBC 539   2 canceled->impl_ = nullptr; 539   2 canceled->impl_ = nullptr;
540   // Store false as the final touch of the impl under the lock so 540   // Store false as the final touch of the impl under the lock so
541   // a pre-lock false-flag check trusts it unqualified. 541   // a pre-lock false-flag check trusts it unqualified.
HITCBC 542   2 impl.might_have_pending_waits_.store(false, std::memory_order_release); 542   2 impl.might_have_pending_waits_.store(false, std::memory_order_release);
HITCBC 543   2 refresh_cached_nearest(); 543   2 refresh_cached_nearest();
HITCBC 544   2 } 544   2 }
545   545  
HITCBC 546   2 if (canceled) 546   2 if (canceled)
547   { 547   {
HITCBC 548   2 canceled->ec_ = make_error_code(capy::error::canceled); 548   2 canceled->ec_ = make_error_code(capy::error::canceled);
HITCBC 549   2 sched_->post(&canceled->op_); 549   2 sched_->post(&canceled->op_);
550   } 550   }
551   } 551   }
552   552  
553   inline void 553   inline void
HITCBC 554   1703 timer_service::cancel_waiter(waiter_node* w) 554   2215 timer_service::cancel_waiter(waiter_node* w)
555   { 555   {
556   { 556   {
HITCBC 557   1703 std::lock_guard lock(mutex_); 557   2215 std::lock_guard lock(mutex_);
558   // Already removed by another drain: cancel_timer, 558   // Already removed by another drain: cancel_timer,
559   // process_expired, or insert_waiter's lost-cancel recheck 559   // process_expired, or insert_waiter's lost-cancel recheck
HITCBC 560   1703 if (!w->impl_) 560   2215 if (!w->impl_)
HITCBC 561   65 return; 561   308 return;
HITCBC 562   1638 auto* impl = w->impl_; 562   1907 auto* impl = w->impl_;
HITCBC 563   1638 w->impl_ = nullptr; 563   1907 w->impl_ = nullptr;
HITCBC 564   1638 impl->waiter_ = nullptr; 564   1907 impl->waiter_ = nullptr;
HITCBC 565   1638 remove_timer_impl(*impl); 565   1907 remove_timer_impl(*impl);
HITCBC 566   1638 impl->might_have_pending_waits_.store(false, std::memory_order_release); 566   1907 impl->might_have_pending_waits_.store(false, std::memory_order_release);
HITCBC 567   1638 refresh_cached_nearest(); 567   1907 refresh_cached_nearest();
HITCBC 568   1638 ++cancels_posting_; 568   1907 ++cancels_posting_;
HITCBC 569   1703 } 569   2215 }
570   570  
HITCBC 571   1638 w->ec_ = make_error_code(capy::error::canceled); 571   1907 w->ec_ = make_error_code(capy::error::canceled);
HITCBC 572   1638 sched_->post(&w->op_); 572   1907 sched_->post(&w->op_);
573   573  
HITCBC 574   1638 std::lock_guard lock(mutex_); 574   1907 std::lock_guard lock(mutex_);
HITCBC 575   1638 --cancels_posting_; 575   1907 --cancels_posting_;
HITCBC 576   1638 } 576   1907 }
577   577  
578   inline std::size_t 578   inline std::size_t
HITCBC 579   156905 timer_service::process_expired() 579   154008 timer_service::process_expired()
580   { 580   {
HITCBC 581   156905 intrusive_list<waiter_node> expired; 581   154008 intrusive_list<waiter_node> expired;
582   582  
583   { 583   {
HITCBC 584   156905 std::lock_guard lock(mutex_); 584   154008 std::lock_guard lock(mutex_);
HITCBC 585   156905 auto now = clock_type::now(); 585   154008 auto now = clock_type::now();
586   586  
HITCBC 587   174938 while (!heap_.empty() && heap_[0].time_ <= now) 587   172565 while (!heap_.empty() && heap_[0].time_ <= now)
588   { 588   {
HITCBC 589   18033 timer::implementation* t = heap_[0].timer_; 589   18557 timer::implementation* t = heap_[0].timer_;
HITCBC 590   18033 remove_timer_impl(*t); 590   18557 remove_timer_impl(*t);
HITCBC 591   18033 if (auto* w = std::exchange(t->waiter_, nullptr)) 591   18557 if (auto* w = std::exchange(t->waiter_, nullptr))
592   { 592   {
HITCBC 593   18033 w->impl_ = nullptr; 593   18557 w->impl_ = nullptr;
HITCBC 594   18033 w->ec_ = {}; 594   18557 w->ec_ = {};
HITCBC 595   18033 expired.push_back(w); 595   18557 expired.push_back(w);
596   } 596   }
HITCBC 597   18033 t->might_have_pending_waits_.store( 597   18557 t->might_have_pending_waits_.store(
598   false, std::memory_order_release); 598   false, std::memory_order_release);
599   } 599   }
600   600  
HITCBC 601   156905 refresh_cached_nearest(); 601   154008 refresh_cached_nearest();
HITCBC 602   156905 } 602   154008 }
603   603  
HITCBC 604   156905 std::size_t count = 0; 604   154008 std::size_t count = 0;
HITCBC 605   174938 while (auto* w = expired.pop_front()) 605   172565 while (auto* w = expired.pop_front())
606   { 606   {
HITCBC 607   18033 sched_->post(&w->op_); 607   18557 sched_->post(&w->op_);
HITCBC 608   18033 ++count; 608   18557 ++count;
HITCBC 609   18033 } 609   18557 }
610   610  
HITCBC 611   156905 return count; 611   154008 return count;
612   } 612   }
613   613  
614   inline void 614   inline void
HITCBC 615   19675 timer_service::remove_timer_impl(timer::implementation& impl) 615   20563 timer_service::remove_timer_impl(timer::implementation& impl)
616   { 616   {
HITCBC 617   19675 std::size_t index = impl.heap_index_.load(std::memory_order_relaxed); 617   20563 std::size_t index = impl.heap_index_.load(std::memory_order_relaxed);
HITCBC 618   19675 if (index >= heap_.size()) 618   20563 if (index >= heap_.size())
MISUBC 619   ✗ return; // Not in heap 619   ✗ return; // Not in heap
620   620  
HITCBC 621   19675 if (index == heap_.size() - 1) 621   20563 if (index == heap_.size() - 1)
622   { 622   {
623   // Last element, just pop 623   // Last element, just pop
HITCBC 624   2782 impl.heap_index_.store( 624   2968 impl.heap_index_.store(
625   (std::numeric_limits<std::size_t>::max)(), 625   (std::numeric_limits<std::size_t>::max)(),
626   std::memory_order_relaxed); 626   std::memory_order_relaxed);
HITCBC 627   2782 heap_.pop_back(); 627   2968 heap_.pop_back();
628   } 628   }
629   else 629   else
630   { 630   {
631   // Swap with last and reheapify 631   // Swap with last and reheapify
HITCBC 632   16893 swap_heap(index, heap_.size() - 1); 632   17595 swap_heap(index, heap_.size() - 1);
HITCBC 633   16893 impl.heap_index_.store( 633   17595 impl.heap_index_.store(
634   (std::numeric_limits<std::size_t>::max)(), 634   (std::numeric_limits<std::size_t>::max)(),
635   std::memory_order_relaxed); 635   std::memory_order_relaxed);
HITCBC 636   16893 heap_.pop_back(); 636   17595 heap_.pop_back();
637   637  
HITCBC 638   16893 if (index > 0 && heap_[index].time_ < heap_[(index - 1) / 2].time_) 638   17595 if (index > 0 && heap_[index].time_ < heap_[(index - 1) / 2].time_)
HITGBC 639   ✗ up_heap(index); 639   3 up_heap(index);
640   else 640   else
HITCBC 641   16893 down_heap(index); 641   17592 down_heap(index);
642   } 642   }
643   } 643   }
644   644  
645   inline void 645   inline void
HITCBC 646   22918 timer_service::up_heap(std::size_t index) 646   23804 timer_service::up_heap(std::size_t index)
647   { 647   {
HITCBC 648   30641 while (index > 0) 648   31618 while (index > 0)
649   { 649   {
HITCBC 650   22582 std::size_t parent = (index - 1) / 2; 650   23429 std::size_t parent = (index - 1) / 2;
HITCBC 651   22582 if (!(heap_[index].time_ < heap_[parent].time_)) 651   23429 if (!(heap_[index].time_ < heap_[parent].time_))
HITCBC 652   14859 break; 652   15615 break;
HITCBC 653   7723 swap_heap(index, parent); 653   7814 swap_heap(index, parent);
HITCBC 654   7723 index = parent; 654   7814 index = parent;
655   } 655   }
HITCBC 656   22918 } 656   23804 }
657   657  
658   inline void 658   inline void
HITCBC 659   16893 timer_service::down_heap(std::size_t index) 659   17592 timer_service::down_heap(std::size_t index)
660   { 660   {
HITCBC 661   16893 std::size_t child = index * 2 + 1; 661   17592 std::size_t child = index * 2 + 1;
HITCBC 662   35781 while (child < heap_.size()) 662   37421 while (child < heap_.size())
663   { 663   {
HITCBC 664   21157 std::size_t min_child = (child + 1 == heap_.size() || 664   22351 std::size_t min_child = (child + 1 == heap_.size() ||
HITCBC 665   17876 heap_[child].time_ < heap_[child + 1].time_) 665   18171 heap_[child].time_ < heap_[child + 1].time_)
HITCBC 666   39033 ? child 666   40522 ? child
HITCBC 667   21157 : child + 1; 667   22351 : child + 1;
668   668  
HITCBC 669   21157 if (heap_[index].time_ < heap_[min_child].time_) 669   22351 if (heap_[index].time_ < heap_[min_child].time_)
HITCBC 670   2269 break; 670   2522 break;
671   671  
HITCBC 672   18888 swap_heap(index, min_child); 672   19829 swap_heap(index, min_child);
HITCBC 673   18888 index = min_child; 673   19829 index = min_child;
HITCBC 674   18888 child = index * 2 + 1; 674   19829 child = index * 2 + 1;
675   } 675   }
HITCBC 676   16893 } 676   17592 }
677   677  
678   inline void 678   inline void
HITCBC 679   43504 timer_service::swap_heap(std::size_t i1, std::size_t i2) 679   45238 timer_service::swap_heap(std::size_t i1, std::size_t i2)
680   { 680   {
HITCBC 681   43504 heap_entry tmp = heap_[i1]; 681   45238 heap_entry tmp = heap_[i1];
HITCBC 682   43504 heap_[i1] = heap_[i2]; 682   45238 heap_[i1] = heap_[i2];
HITCBC 683   43504 heap_[i2] = tmp; 683   45238 heap_[i2] = tmp;
HITCBC 684   43504 heap_[i1].timer_->heap_index_.store(i1, std::memory_order_relaxed); 684   45238 heap_[i1].timer_->heap_index_.store(i1, std::memory_order_relaxed);
HITCBC 685   43504 heap_[i2].timer_->heap_index_.store(i2, std::memory_order_relaxed); 685   45238 heap_[i2].timer_->heap_index_.store(i2, std::memory_order_relaxed);
HITCBC 686   43504 } 686   45238 }
687   687  
688   // waiter_node's completion_op and canceller members are defined in 688   // waiter_node's completion_op and canceller members are defined in
689   // timer.cpp alongside implementation::wait(), for the same reason 689   // timer.cpp alongside implementation::wait(), for the same reason
690   // wait() lives there (see below). 690   // wait() lives there (see below).
691   691  
692   // timer::implementation::wait() is defined in timer.cpp, not here. 692   // timer::implementation::wait() is defined in timer.cpp, not here.
693   // It must be a non-inline definition in a translation unit that is 693   // It must be a non-inline definition in a translation unit that is
694   // always pulled into the link whenever detail::timer is used (every 694   // always pulled into the link whenever detail::timer is used (every
695   // consumer needs timer's constructors from that same object file). 695   // consumer needs timer's constructors from that same object file).
696   // An inline definition in this header would only be emitted in 696   // An inline definition in this header would only be emitted in
697   // translation units that happen to also include this header, which 697   // translation units that happen to also include this header, which
698   // is not guaranteed for every caller of wait_awaitable::await_suspend 698   // is not guaranteed for every caller of wait_awaitable::await_suspend
699   // in timer.hpp (e.g. code that only reaches timer.hpp through 699   // in timer.hpp (e.g. code that only reaches timer.hpp through
700   // delay.hpp, without transitively including a scheduler header). 700   // delay.hpp, without transitively including a scheduler header).
701   701  
702   // Free functions 702   // Free functions
703   703  
704   inline timer_service& 704   inline timer_service&
HITCBC 705   4415 get_timer_service(capy::execution_context& ctx, scheduler& sched) 705   4415 get_timer_service(capy::execution_context& ctx, scheduler& sched)
706   { 706   {
HITCBC 707   4415 return ctx.make_service<timer_service>(sched); 707   4415 return ctx.make_service<timer_service>(sched);
708   } 708   }
709   709  
710   } // namespace boost::corosio::detail 710   } // namespace boost::corosio::detail
711   711  
712   #endif 712   #endif