Fix KV offload request-finished lifecycle contract (#46284)
Signed-off-by: test test <2260891073@qq.com> Signed-off-by: Rui Yin <2260891073@qq.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -1140,11 +1140,6 @@ class OffloadingConnectorScheduler:
|
||||
del self._jobs[job_id]
|
||||
req_status.transfer_jobs.remove(job_id)
|
||||
if not req_status.transfer_jobs and req_status.req.is_finished():
|
||||
# Deferred from request_finished: the request's last in-flight
|
||||
# job is now done, so fire the finalize hook here, after the
|
||||
# final complete_store/complete_load above (and any submit_store
|
||||
# the complete_store cascade issued).
|
||||
self.manager.on_request_finished(req_status.req_context)
|
||||
del self._req_status[job_status.req_id]
|
||||
|
||||
def get_stats(self) -> OffloadingConnectorStats | None:
|
||||
@@ -1181,20 +1176,23 @@ class OffloadingConnectorScheduler:
|
||||
if req_status is None:
|
||||
# Untracked request (offloading never started): no in-flight jobs,
|
||||
# nothing was deferred, so finalize immediately.
|
||||
self.manager.on_request_finished(_create_req_context(request))
|
||||
req_context = _create_req_context(request)
|
||||
self.manager.on_new_request(req_context)
|
||||
self.manager.on_request_finished(req_context)
|
||||
return False, None
|
||||
|
||||
self.manager.on_request_finished(req_status.req_context)
|
||||
|
||||
if not req_status.transfer_jobs:
|
||||
# No in-flight jobs: all per-request calls are done, finalize now.
|
||||
self.manager.on_request_finished(req_status.req_context)
|
||||
# No in-flight jobs: no later complete_store()/complete_load() calls
|
||||
# need this request's state.
|
||||
del self._req_status[request.request_id]
|
||||
return False, None
|
||||
|
||||
# In-flight jobs remain, so defer on_request_finished to
|
||||
# update_connector_output, which fires it once the last job completes
|
||||
# (after the final complete_store and any cascade submit_store it
|
||||
# issues). These pending stores outlive the request's block ownership;
|
||||
# register them so future reuse of those blocks triggers a flush.
|
||||
# In-flight jobs remain after the request stopped. Their completion may
|
||||
# still call manager.complete_store()/complete_load(), so keep req_status.
|
||||
# Pending stores outlive the request's block ownership; register them so
|
||||
# future reuse of those blocks triggers a flush.
|
||||
for job_id in req_status.transfer_jobs:
|
||||
job_status = self._jobs[job_id]
|
||||
for bid in job_status.non_sliding_window_block_ids or ():
|
||||
@@ -1225,14 +1223,8 @@ class OffloadingConnectorScheduler:
|
||||
# Flush all in-flight jobs
|
||||
self._current_batch_jobs_to_flush.update(self._jobs.keys())
|
||||
|
||||
# A finished request may still be tracked here with in-flight jobs that
|
||||
# this reset discards, so its deferred on_request_finished() would never
|
||||
# fire (completions are skipped as stale) and its _req_status entry would
|
||||
# leak. Finalize such requests now, before resetting the manager.
|
||||
# list() snapshots because we delete while iterating.
|
||||
for req_id, status in list(self._req_status.items()):
|
||||
if status.req.is_finished():
|
||||
self.manager.on_request_finished(status.req_context)
|
||||
del self._req_status[req_id]
|
||||
|
||||
# Reset offloading manager cache
|
||||
|
||||
@@ -276,16 +276,16 @@ class OffloadingManager(ABC):
|
||||
"""
|
||||
Called when a request has finished.
|
||||
|
||||
By the time this is called, all per-request offload calls for this
|
||||
request (prepare_store/complete_store, prepare_load/complete_load,
|
||||
touch, lookup) have already been issued, and none will follow. The
|
||||
scheduler defers this call until the request is finished and has no
|
||||
in-flight transfer jobs.
|
||||
By the time this is called, the scheduler will issue no more
|
||||
submit-side calls for this request, such as prepare_store() and
|
||||
prepare_load(). Completion callbacks for already-submitted transfers
|
||||
(complete_store() and complete_load()) may still arrive afterward.
|
||||
|
||||
Note this signals only that no further calls will be made; it does NOT
|
||||
imply the data has been persisted. Asynchronous transfers already
|
||||
submitted for this request (e.g. CPU->secondary cascades) may still be
|
||||
in flight. This is the right place to release per-request bookkeeping.
|
||||
This hook does NOT imply the data has been persisted. Asynchronous
|
||||
transfers already submitted for this request may still be in flight.
|
||||
Managers that cascade to lower tiers should delay those tiers'
|
||||
on_request_finished() calls until no more lower-tier submit calls can
|
||||
be issued for this request.
|
||||
|
||||
Args:
|
||||
req_context: per-request context.
|
||||
|
||||
@@ -20,7 +20,6 @@ Key Design Principles:
|
||||
protecting blocks from eviction until complete_read() is called
|
||||
"""
|
||||
|
||||
from collections import defaultdict
|
||||
from collections.abc import Collection, Iterable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
@@ -62,6 +61,14 @@ class PendingPromotion:
|
||||
block_ids: list[int] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class RequestState:
|
||||
req_context: ReqContext
|
||||
pending_primary_stores: int = 0
|
||||
is_finished: bool = False
|
||||
request_level_tiers: set[SecondaryTierManager] | None = None
|
||||
|
||||
|
||||
class CPUPrimaryTierOffloadingManager(CPUOffloadingManager):
|
||||
"""CPUOffloadingManager with a primary/secondary transfer interface.
|
||||
|
||||
@@ -166,12 +173,10 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
# Reset at the end of each step in on_schedule_end().
|
||||
self._processed_jobs_this_step: bool = False
|
||||
|
||||
# Per-request set of secondary tiers that requested REQUEST_LEVEL
|
||||
# policy. Populated in on_new_request(),
|
||||
# cleaned up in on_request_finished().
|
||||
self._request_level_tiers: defaultdict[str, set[SecondaryTierManager]] = (
|
||||
defaultdict(set)
|
||||
)
|
||||
# Per-request state for prepared GPU->primary stores and finalization.
|
||||
# Secondary tiers are finalized only after pending primary stores reach
|
||||
# complete_store(), since complete_store() can still submit cascades.
|
||||
self._req_state: dict[str, RequestState] = {}
|
||||
|
||||
def _next_job_id(self) -> JobId:
|
||||
"""Generate a unique job ID for async transfer tracking."""
|
||||
@@ -432,9 +437,13 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
if primary_result is None:
|
||||
return None
|
||||
|
||||
if primary_result.keys_to_store:
|
||||
state = self._req_state[req_context.req_id]
|
||||
state.pending_primary_stores += 1
|
||||
|
||||
# Step 3: For request-level tiers, cascade blocks already in primary
|
||||
request_level_tiers = self._request_level_tiers.get(req_context.req_id)
|
||||
if request_level_tiers is not None:
|
||||
request_level_tiers = self._req_state[req_context.req_id].request_level_tiers
|
||||
if request_level_tiers:
|
||||
keys_to_store_set = set(primary_result.keys_to_store)
|
||||
keys_already_in_primary = tuple(
|
||||
k for k in keys if k not in keys_to_store_set
|
||||
@@ -508,36 +517,38 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
# Step 1: Complete store in primary tier (makes blocks loadable)
|
||||
self.primary_tier.complete_store(keys, req_context, success)
|
||||
|
||||
if not success:
|
||||
# If GPU→Primary transfer failed, don't cascade to secondary tiers
|
||||
return
|
||||
if success:
|
||||
# Step 2: Cascade to ALL secondary tiers
|
||||
# For each secondary tier, call primary.prepare_read() to get the
|
||||
# LoadStoreSpec AND to increment ref_cnt (protecting blocks from
|
||||
# eviction during the async transfer). One prepare_read() call per
|
||||
# secondary tier.
|
||||
for tier in self.secondary_tiers:
|
||||
primary_blocks_spec = self.primary_tier.prepare_read(keys, req_context)
|
||||
|
||||
# Step 2: Cascade to ALL secondary tiers
|
||||
# For each secondary tier, call primary.prepare_read() to get the
|
||||
# LoadStoreSpec AND to increment ref_cnt (protecting blocks from
|
||||
# eviction during the async transfer). One prepare_read() call per
|
||||
# secondary tier.
|
||||
for tier in self.secondary_tiers:
|
||||
primary_blocks_spec = self.primary_tier.prepare_read(keys, req_context)
|
||||
# Submit async store job: primary→secondary
|
||||
job_id = self._next_job_id()
|
||||
|
||||
# Submit async store job: primary→secondary
|
||||
job_id = self._next_job_id()
|
||||
# Track this store job
|
||||
assert isinstance(primary_blocks_spec, CPULoadStoreSpec)
|
||||
job_metadata = JobMetadata(
|
||||
job_id=job_id,
|
||||
keys=keys,
|
||||
block_ids=primary_blocks_spec.block_ids,
|
||||
is_promotion=False,
|
||||
req_context=req_context,
|
||||
)
|
||||
self._transfer_jobs[job_id] = job_metadata
|
||||
|
||||
# Track this store job
|
||||
assert isinstance(primary_blocks_spec, CPULoadStoreSpec)
|
||||
job_metadata = JobMetadata(
|
||||
job_id=job_id,
|
||||
keys=keys,
|
||||
block_ids=primary_blocks_spec.block_ids,
|
||||
is_promotion=False,
|
||||
req_context=req_context,
|
||||
)
|
||||
self._transfer_jobs[job_id] = job_metadata
|
||||
|
||||
tier.submit_store(job_metadata)
|
||||
tier.submit_store(job_metadata)
|
||||
|
||||
# Note: The async transfers are now in flight. Their completion is
|
||||
# tracked via get_finished_jobs() / _maybe_process_finished_jobs().
|
||||
req_id = req_context.req_id
|
||||
state = self._req_state[req_id]
|
||||
assert state.pending_primary_stores > 0
|
||||
state.pending_primary_stores -= 1
|
||||
self._maybe_finalize_request(req_id)
|
||||
|
||||
@override
|
||||
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
|
||||
@@ -547,14 +558,18 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
Returns REQUEST_LEVEL if ANY secondary tier wants request-level.
|
||||
Only stores REQUEST_LEVEL tier decisions for use in prepare_store.
|
||||
"""
|
||||
state = RequestState(req_context=req_context)
|
||||
for tier in self.secondary_tiers:
|
||||
tier_ctx = tier.on_new_request(req_context)
|
||||
if tier_ctx.policy == OffloadPolicy.REQUEST_LEVEL:
|
||||
self._request_level_tiers[req_context.req_id].add(tier)
|
||||
if state.request_level_tiers is None:
|
||||
state.request_level_tiers = set()
|
||||
state.request_level_tiers.add(tier)
|
||||
self._req_state[req_context.req_id] = state
|
||||
|
||||
policy = (
|
||||
OffloadPolicy.REQUEST_LEVEL
|
||||
if req_context.req_id in self._request_level_tiers
|
||||
if state.request_level_tiers
|
||||
else OffloadPolicy.BLOCK_LEVEL
|
||||
)
|
||||
return RequestOffloadingContext(policy=policy)
|
||||
@@ -562,9 +577,26 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
@override
|
||||
def on_request_finished(self, req_context: ReqContext) -> None:
|
||||
self.primary_tier.on_request_finished(req_context)
|
||||
state = self._req_state[req_context.req_id]
|
||||
state.is_finished = True
|
||||
self._maybe_finalize_request(req_context.req_id)
|
||||
|
||||
def _maybe_finalize_request(self, req_id: str) -> None:
|
||||
"""Finalize secondary tiers once no more store cascades can be submitted.
|
||||
|
||||
Finalization means forwarding on_request_finished() to secondary tiers.
|
||||
It is delayed until pending GPU->primary stores finish, since their
|
||||
complete_store() callbacks may still submit primary->secondary stores.
|
||||
"""
|
||||
state = self._req_state[req_id]
|
||||
if not state.is_finished:
|
||||
return
|
||||
if state.pending_primary_stores != 0:
|
||||
return
|
||||
|
||||
for tier in self.secondary_tiers:
|
||||
tier.on_request_finished(req_context)
|
||||
self._request_level_tiers.pop(req_context.req_id, None)
|
||||
tier.on_request_finished(state.req_context)
|
||||
del self._req_state[req_id]
|
||||
|
||||
@override
|
||||
def on_schedule_end(self) -> None:
|
||||
@@ -604,7 +636,7 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
|
||||
@override
|
||||
def reset_cache(self) -> None:
|
||||
"""Drop all tracked state in the orchestrator and primary tier.
|
||||
"""Reset transfer bookkeeping and primary-tier cache.
|
||||
|
||||
Called during sleep, weight update, or resume. Each secondary tier
|
||||
drains its in-flight transfers via drain_jobs() so no tier I/O is
|
||||
@@ -613,7 +645,9 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
from reusing primary slots while a transfer is mid-copy.
|
||||
|
||||
Secondary tiers are intentionally not reset: persistent stores
|
||||
(FS, network) keep their data across resets.
|
||||
(FS, network) keep their data across resets. Active request state is
|
||||
retained so those requests can continue after the reset; finished
|
||||
requests are finalized and removed.
|
||||
"""
|
||||
for tier in self.secondary_tiers:
|
||||
tier.drain_jobs()
|
||||
@@ -626,9 +660,19 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
# called so no tier I/O is touching that memory.
|
||||
self._pending_load_submissions.clear()
|
||||
|
||||
finished_req_ids = []
|
||||
for req_id, state in self._req_state.items():
|
||||
state.pending_primary_stores = 0
|
||||
if not state.is_finished:
|
||||
continue
|
||||
for tier in self.secondary_tiers:
|
||||
tier.on_request_finished(state.req_context)
|
||||
finished_req_ids.append(req_id)
|
||||
|
||||
self.primary_tier.reset_cache()
|
||||
|
||||
self._request_level_tiers.clear()
|
||||
for req_id in finished_req_ids:
|
||||
del self._req_state[req_id]
|
||||
self._processed_jobs_this_step = False
|
||||
|
||||
@override
|
||||
|
||||
Reference in New Issue
Block a user