Fix KV offload request-finished lifecycle contract (#46284)

Signed-off-by: test test <2260891073@qq.com>
Signed-off-by: Rui Yin <2260891073@qq.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Rui Yin
2026-06-24 15:42:40 +03:00
committed by GitHub
co-authored by Or Ozeri
parent d4448b511d
commit cf9fd6457e
5 changed files with 334 additions and 112 deletions
@@ -1140,11 +1140,6 @@ class OffloadingConnectorScheduler:
del self._jobs[job_id]
req_status.transfer_jobs.remove(job_id)
if not req_status.transfer_jobs and req_status.req.is_finished():
# Deferred from request_finished: the request's last in-flight
# job is now done, so fire the finalize hook here, after the
# final complete_store/complete_load above (and any submit_store
# the complete_store cascade issued).
self.manager.on_request_finished(req_status.req_context)
del self._req_status[job_status.req_id]
def get_stats(self) -> OffloadingConnectorStats | None:
@@ -1181,20 +1176,23 @@ class OffloadingConnectorScheduler:
if req_status is None:
# Untracked request (offloading never started): no in-flight jobs,
# nothing was deferred, so finalize immediately.
self.manager.on_request_finished(_create_req_context(request))
req_context = _create_req_context(request)
self.manager.on_new_request(req_context)
self.manager.on_request_finished(req_context)
return False, None
self.manager.on_request_finished(req_status.req_context)
if not req_status.transfer_jobs:
# No in-flight jobs: all per-request calls are done, finalize now.
self.manager.on_request_finished(req_status.req_context)
# No in-flight jobs: no later complete_store()/complete_load() calls
# need this request's state.
del self._req_status[request.request_id]
return False, None
# In-flight jobs remain, so defer on_request_finished to
# update_connector_output, which fires it once the last job completes
# (after the final complete_store and any cascade submit_store it
# issues). These pending stores outlive the request's block ownership;
# register them so future reuse of those blocks triggers a flush.
# In-flight jobs remain after the request stopped. Their completion may
# still call manager.complete_store()/complete_load(), so keep req_status.
# Pending stores outlive the request's block ownership; register them so
# future reuse of those blocks triggers a flush.
for job_id in req_status.transfer_jobs:
job_status = self._jobs[job_id]
for bid in job_status.non_sliding_window_block_ids or ():
@@ -1225,14 +1223,8 @@ class OffloadingConnectorScheduler:
# Flush all in-flight jobs
self._current_batch_jobs_to_flush.update(self._jobs.keys())
# A finished request may still be tracked here with in-flight jobs that
# this reset discards, so its deferred on_request_finished() would never
# fire (completions are skipped as stale) and its _req_status entry would
# leak. Finalize such requests now, before resetting the manager.
# list() snapshots because we delete while iterating.
for req_id, status in list(self._req_status.items()):
if status.req.is_finished():
self.manager.on_request_finished(status.req_context)
del self._req_status[req_id]
# Reset offloading manager cache
+9 -9
View File
@@ -276,16 +276,16 @@ class OffloadingManager(ABC):
"""
Called when a request has finished.
By the time this is called, all per-request offload calls for this
request (prepare_store/complete_store, prepare_load/complete_load,
touch, lookup) have already been issued, and none will follow. The
scheduler defers this call until the request is finished and has no
in-flight transfer jobs.
By the time this is called, the scheduler will issue no more
submit-side calls for this request, such as prepare_store() and
prepare_load(). Completion callbacks for already-submitted transfers
(complete_store() and complete_load()) may still arrive afterward.
Note this signals only that no further calls will be made; it does NOT
imply the data has been persisted. Asynchronous transfers already
submitted for this request (e.g. CPU->secondary cascades) may still be
in flight. This is the right place to release per-request bookkeeping.
This hook does NOT imply the data has been persisted. Asynchronous
transfers already submitted for this request may still be in flight.
Managers that cascade to lower tiers should delay those tiers'
on_request_finished() calls until no more lower-tier submit calls can
be issued for this request.
Args:
req_context: per-request context.
+84 -40
View File
@@ -20,7 +20,6 @@ Key Design Principles:
protecting blocks from eviction until complete_read() is called
"""
from collections import defaultdict
from collections.abc import Collection, Iterable, Sequence
from dataclasses import dataclass, field
@@ -62,6 +61,14 @@ class PendingPromotion:
block_ids: list[int] = field(default_factory=list)
@dataclass(slots=True)
class RequestState:
req_context: ReqContext
pending_primary_stores: int = 0
is_finished: bool = False
request_level_tiers: set[SecondaryTierManager] | None = None
class CPUPrimaryTierOffloadingManager(CPUOffloadingManager):
"""CPUOffloadingManager with a primary/secondary transfer interface.
@@ -166,12 +173,10 @@ class TieringOffloadingManager(OffloadingManager):
# Reset at the end of each step in on_schedule_end().
self._processed_jobs_this_step: bool = False
# Per-request set of secondary tiers that requested REQUEST_LEVEL
# policy. Populated in on_new_request(),
# cleaned up in on_request_finished().
self._request_level_tiers: defaultdict[str, set[SecondaryTierManager]] = (
defaultdict(set)
)
# Per-request state for prepared GPU->primary stores and finalization.
# Secondary tiers are finalized only after pending primary stores reach
# complete_store(), since complete_store() can still submit cascades.
self._req_state: dict[str, RequestState] = {}
def _next_job_id(self) -> JobId:
"""Generate a unique job ID for async transfer tracking."""
@@ -432,9 +437,13 @@ class TieringOffloadingManager(OffloadingManager):
if primary_result is None:
return None
if primary_result.keys_to_store:
state = self._req_state[req_context.req_id]
state.pending_primary_stores += 1
# Step 3: For request-level tiers, cascade blocks already in primary
request_level_tiers = self._request_level_tiers.get(req_context.req_id)
if request_level_tiers is not None:
request_level_tiers = self._req_state[req_context.req_id].request_level_tiers
if request_level_tiers:
keys_to_store_set = set(primary_result.keys_to_store)
keys_already_in_primary = tuple(
k for k in keys if k not in keys_to_store_set
@@ -508,36 +517,38 @@ class TieringOffloadingManager(OffloadingManager):
# Step 1: Complete store in primary tier (makes blocks loadable)
self.primary_tier.complete_store(keys, req_context, success)
if not success:
# If GPU→Primary transfer failed, don't cascade to secondary tiers
return
if success:
# Step 2: Cascade to ALL secondary tiers
# For each secondary tier, call primary.prepare_read() to get the
# LoadStoreSpec AND to increment ref_cnt (protecting blocks from
# eviction during the async transfer). One prepare_read() call per
# secondary tier.
for tier in self.secondary_tiers:
primary_blocks_spec = self.primary_tier.prepare_read(keys, req_context)
# Step 2: Cascade to ALL secondary tiers
# For each secondary tier, call primary.prepare_read() to get the
# LoadStoreSpec AND to increment ref_cnt (protecting blocks from
# eviction during the async transfer). One prepare_read() call per
# secondary tier.
for tier in self.secondary_tiers:
primary_blocks_spec = self.primary_tier.prepare_read(keys, req_context)
# Submit async store job: primary→secondary
job_id = self._next_job_id()
# Submit async store job: primary→secondary
job_id = self._next_job_id()
# Track this store job
assert isinstance(primary_blocks_spec, CPULoadStoreSpec)
job_metadata = JobMetadata(
job_id=job_id,
keys=keys,
block_ids=primary_blocks_spec.block_ids,
is_promotion=False,
req_context=req_context,
)
self._transfer_jobs[job_id] = job_metadata
# Track this store job
assert isinstance(primary_blocks_spec, CPULoadStoreSpec)
job_metadata = JobMetadata(
job_id=job_id,
keys=keys,
block_ids=primary_blocks_spec.block_ids,
is_promotion=False,
req_context=req_context,
)
self._transfer_jobs[job_id] = job_metadata
tier.submit_store(job_metadata)
tier.submit_store(job_metadata)
# Note: The async transfers are now in flight. Their completion is
# tracked via get_finished_jobs() / _maybe_process_finished_jobs().
req_id = req_context.req_id
state = self._req_state[req_id]
assert state.pending_primary_stores > 0
state.pending_primary_stores -= 1
self._maybe_finalize_request(req_id)
@override
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
@@ -547,14 +558,18 @@ class TieringOffloadingManager(OffloadingManager):
Returns REQUEST_LEVEL if ANY secondary tier wants request-level.
Only stores REQUEST_LEVEL tier decisions for use in prepare_store.
"""
state = RequestState(req_context=req_context)
for tier in self.secondary_tiers:
tier_ctx = tier.on_new_request(req_context)
if tier_ctx.policy == OffloadPolicy.REQUEST_LEVEL:
self._request_level_tiers[req_context.req_id].add(tier)
if state.request_level_tiers is None:
state.request_level_tiers = set()
state.request_level_tiers.add(tier)
self._req_state[req_context.req_id] = state
policy = (
OffloadPolicy.REQUEST_LEVEL
if req_context.req_id in self._request_level_tiers
if state.request_level_tiers
else OffloadPolicy.BLOCK_LEVEL
)
return RequestOffloadingContext(policy=policy)
@@ -562,9 +577,26 @@ class TieringOffloadingManager(OffloadingManager):
@override
def on_request_finished(self, req_context: ReqContext) -> None:
self.primary_tier.on_request_finished(req_context)
state = self._req_state[req_context.req_id]
state.is_finished = True
self._maybe_finalize_request(req_context.req_id)
def _maybe_finalize_request(self, req_id: str) -> None:
"""Finalize secondary tiers once no more store cascades can be submitted.
Finalization means forwarding on_request_finished() to secondary tiers.
It is delayed until pending GPU->primary stores finish, since their
complete_store() callbacks may still submit primary->secondary stores.
"""
state = self._req_state[req_id]
if not state.is_finished:
return
if state.pending_primary_stores != 0:
return
for tier in self.secondary_tiers:
tier.on_request_finished(req_context)
self._request_level_tiers.pop(req_context.req_id, None)
tier.on_request_finished(state.req_context)
del self._req_state[req_id]
@override
def on_schedule_end(self) -> None:
@@ -604,7 +636,7 @@ class TieringOffloadingManager(OffloadingManager):
@override
def reset_cache(self) -> None:
"""Drop all tracked state in the orchestrator and primary tier.
"""Reset transfer bookkeeping and primary-tier cache.
Called during sleep, weight update, or resume. Each secondary tier
drains its in-flight transfers via drain_jobs() so no tier I/O is
@@ -613,7 +645,9 @@ class TieringOffloadingManager(OffloadingManager):
from reusing primary slots while a transfer is mid-copy.
Secondary tiers are intentionally not reset: persistent stores
(FS, network) keep their data across resets.
(FS, network) keep their data across resets. Active request state is
retained so those requests can continue after the reset; finished
requests are finalized and removed.
"""
for tier in self.secondary_tiers:
tier.drain_jobs()
@@ -626,9 +660,19 @@ class TieringOffloadingManager(OffloadingManager):
# called so no tier I/O is touching that memory.
self._pending_load_submissions.clear()
finished_req_ids = []
for req_id, state in self._req_state.items():
state.pending_primary_stores = 0
if not state.is_finished:
continue
for tier in self.secondary_tiers:
tier.on_request_finished(state.req_context)
finished_req_ids.append(req_id)
self.primary_tier.reset_cache()
self._request_level_tiers.clear()
for req_id in finished_req_ids:
del self._req_state[req_id]
self._processed_jobs_this_step = False
@override