[KV Offload] Add on_schedule_end() hook to separate step lifecycle from event draining (#44206)

Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
This commit is contained in:
Ronen Schaffer
2026-06-02 13:42:52 +03:00
committed by GitHub
parent 689b0eeb9e
commit 2a2b5ca791
5 changed files with 55 additions and 40 deletions
@@ -897,6 +897,7 @@ class OffloadingConnectorScheduler:
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
self._update_req_states(scheduler_output)
self.manager.on_schedule_end()
# Flush jobs for preempted requests.
for req_id in scheduler_output.preempted_req_ids or ():
+8
View File
@@ -257,6 +257,14 @@ class OffloadingManager(ABC):
"""
return ()
def on_schedule_end(self) -> None:
"""Called once at the end of each scheduler step.
Managers may override this to flush deferred work accumulated
during the step (e.g., batched promotions).
"""
return
def reset_cache(self) -> None:
"""Evict all tracked blocks and reset internal state."""
return
+8
View File
@@ -185,6 +185,14 @@ class SecondaryTierManager(ABC):
"""
return
def on_schedule_end(self) -> None:
"""Called once at the end of each scheduler step.
Secondary tiers may override this for per-step cleanup or
deferred work submission.
"""
return
def shutdown(self) -> None:
"""Release resources held by this tier (threads, connections, etc.)."""
return
+22 -28
View File
@@ -152,7 +152,7 @@ class TieringOffloadingManager(OffloadingManager):
self._transfer_jobs: dict[JobId, JobMetadata] = {}
# Pending promotion requests accumulated during lookup() calls; flushed
# as one batched submit_load() per (tier, request) in take_events().
# as one batched submit_load() per (tier, request) in on_schedule_end().
# Outer key: tier. Inner key: req_context.req_id — the same ReqContext
# object is reused for all block lookups of a given request per engine step.
self._pending_load_submissions: dict[
@@ -160,7 +160,7 @@ class TieringOffloadingManager(OffloadingManager):
] = {}
# Gate for once-per-step execution of _maybe_process_finished_jobs().
# Reset at the end of each step in take_events().
# Reset at the end of each step in on_schedule_end().
self._processed_jobs_this_step: bool = False
# Per-request set of secondary tiers that requested REQUEST_LEVEL
@@ -182,7 +182,7 @@ class TieringOffloadingManager(OffloadingManager):
Guarded by _processed_jobs_this_step: the first call in an engine step
does the actual polling; subsequent calls are no-ops. The flag is reset
in take_events() at the end of each step.
in on_schedule_end() at the end of each step.
"""
if self._processed_jobs_this_step:
return
@@ -304,8 +304,8 @@ class TieringOffloadingManager(OffloadingManager):
store_spec = primary_write_result.store_spec
assert isinstance(store_spec, CPULoadStoreSpec)
# Defer submit_load to take_events(). Group by (tier, request) so each
# request's blocks are submitted as one batched job per tier.
# Defer submit_load to on_schedule_end(). Group by (tier, request) so
# each request's blocks are submitted as one batched job per tier.
tier_pending = self._pending_load_submissions.setdefault(tier, {})
ctx_id = req_context.req_id
if ctx_id not in tier_pending:
@@ -320,8 +320,8 @@ class TieringOffloadingManager(OffloadingManager):
def _flush_pending_promotions(self) -> None:
"""Submit one batched submit_load() per (tier, request).
Called from take_events() at the end of each engine step, flushing
all promotion requests deferred during lookup().
Called from on_schedule_end() at the end of each scheduler step,
flushing all promotion requests deferred during lookup().
"""
if not self._pending_load_submissions:
return
@@ -564,32 +564,26 @@ class TieringOffloadingManager(OffloadingManager):
self._request_level_tiers.pop(req_context.req_id, None)
@override
def take_events(self) -> Iterable[OffloadingEvent]:
"""
End-of-step hook: flush deferred work, yield events, reset per-step state.
def on_schedule_end(self) -> None:
"""End-of-schedule hook: process finished jobs, flush deferred
promotions, and reset the per-step gate.
Called once per engine step from Scheduler.update_from_output() →
connector.take_events(). Ensures _maybe_process_finished_jobs() has run
at least once this step, flushes pending promotions, yields collected
events, and resets the per-step flag.
Called once per scheduler step from
OffloadingConnectorScheduler.build_connector_meta().
"""
self._maybe_process_finished_jobs()
self._processed_jobs_this_step = False
self._flush_pending_promotions()
for tier in self.secondary_tiers:
tier.on_schedule_end()
@override
def take_events(self) -> Iterable[OffloadingEvent]:
"""Yield offloading events collected since the last call.
Yields:
New OffloadingEvents collected since the last call.
"""
# TODO: Move _flush_pending_promotions() to a dedicated end_of_batch()
# hook once one exists. For now, take_events() serves as the flush
# point under the assumption that it is called at the end of each
# engine step (Scheduler.update_from_output() → connector.take_events()).
# When the dedicated hook is added, update tests that rely on
# take_events() to signal end of step.
self._maybe_process_finished_jobs()
self._flush_pending_promotions()
# Reset the per-step gate so next step's first call does real work.
self._processed_jobs_this_step = False
if self.events is not None:
yield from self.events
self.events.clear()