[KV Offload] Add ParentManager ABC for secondary tier callbacks (#47274)

Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Ronen Schaffer
2026-07-07 13:51:18 +03:00
committed by GitHub
co-authored by Or Ozeri
parent 3354dba381
commit 48fcfc926c
2 changed files with 169 additions and 44 deletions
+48 -2
View File
@@ -49,6 +49,43 @@ class JobResult:
success: bool
class ParentManager(ABC):
"""Interface for secondary tiers to call back into the tiering manager.
Passed to secondary tiers via serve_external_requests() each step.
The _SecondaryTierFacingParent wrapper implements this, automatically
excluding the calling tier from fan-out operations.
Required call sequence for each remote request:
1. on_new_request(req_context) — set up per-request state
2. lookup(key, req_context) — check block availability
(repeat per block)
3. create_store_job(keys, req_context) — pin blocks and get a
job handle
4. on_request_finished(req_context) — clean up per-request state
Steps 2-3 may be interleaved. Step 4 must be called even if no
blocks were found, to avoid leaking async lookup state (e.g. in
the fs tier's AsyncLookupManager).
"""
@abstractmethod
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext: ...
@abstractmethod
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult: ...
@abstractmethod
def create_store_job(
self,
keys: Collection[OffloadKey],
req_context: ReqContext,
) -> JobMetadata: ...
@abstractmethod
def on_request_finished(self, req_context: ReqContext) -> None: ...
class SecondaryTierManager(ABC):
"""
Abstract interface for managing a single non-primary offloading tier.
@@ -210,11 +247,20 @@ class SecondaryTierManager(ABC):
"""
return
def serve_external_requests(self, parent: ParentManager) -> None:
"""Process remotely-originated requests using the parent manager.
Called once per scheduler step, BEFORE _flush_pending_promotions().
The parent handle is valid only for the duration of this call.
Tiers that don't serve external requests leave this as a no-op.
"""
return
def on_schedule_end(self, context: ScheduleEndContext) -> None:
"""Called once at the end of each scheduler step.
Secondary tiers may override this for per-step cleanup or
deferred work submission.
Args:
context: Per-step context from the scheduler.
"""
return
+121 -42
View File
@@ -48,6 +48,7 @@ from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
from vllm.v1.kv_offload.tiering.base import (
JobId,
JobMetadata,
ParentManager,
SecondaryTierManager,
)
@@ -120,6 +121,35 @@ class CPUPrimaryTierOffloadingManager(CPUOffloadingManager):
self._mmap_region.cleanup()
class _SecondaryTierFacingParent(ParentManager):
"""Wrapper that implements ParentManager by delegating to the
TieringOffloadingManager with exclude_tier set to the origin tier."""
__slots__ = ("_m", "_origin")
def __init__(
self,
manager: "TieringOffloadingManager",
tier: SecondaryTierManager,
):
self._m = manager
self._origin = tier
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
return self._m.on_new_request(req_context, exclude_tier=self._origin)
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
return self._m.lookup(key, req_context, exclude_tier=self._origin)
def create_store_job(
self, keys: Collection[OffloadKey], req_context: ReqContext
) -> JobMetadata:
return self._m.create_store_job(keys, req_context)
def on_request_finished(self, req_context: ReqContext) -> None:
return self._m.on_request_finished(req_context, exclude_tier=self._origin)
class TieringOffloadingManager(OffloadingManager):
"""
Orchestrates multi-tier KV cache offloading.
@@ -180,6 +210,12 @@ class TieringOffloadingManager(OffloadingManager):
# complete_store(), since complete_store() can still submit cascades.
self._req_state: dict[str, RequestState] = {}
# Cached ParentManager wrappers for each secondary tier.
self._tier_parents: dict[SecondaryTierManager, _SecondaryTierFacingParent] = {
tier: _SecondaryTierFacingParent(self, tier)
for tier in self.secondary_tiers
}
def _next_job_id(self) -> JobId:
"""Generate a unique job ID for async transfer tracking."""
job_id = self._job_id_counter
@@ -235,7 +271,13 @@ class TieringOffloadingManager(OffloadingManager):
)
@override
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
def lookup(
self,
key: OffloadKey,
req_context: ReqContext,
*,
exclude_tier: SecondaryTierManager | None = None,
) -> LookupResult:
"""
Check whether a single block is offloaded and ready.
@@ -257,6 +299,10 @@ class TieringOffloadingManager(OffloadingManager):
MISS — block not found in any tier, or primary is full
and cannot accept a promotion.
"""
# Poll first so a promotion that finished since the last call is
# already reflected as HIT (not stale HIT_PENDING/MISS) below, and
# so blocks freed by cascade or promotion completions are evictable
# in time for a promotion this lookup may initiate.
self._maybe_process_finished_jobs()
primary_hit = self.primary_tier.lookup(key, req_context)
@@ -267,6 +313,8 @@ class TieringOffloadingManager(OffloadingManager):
any_retry = False
for tier in self.secondary_tiers:
if tier is exclude_tier:
continue
result = tier.lookup(key, req_context)
if result is LookupResult.HIT:
if not self._initiate_promotion(tier, key, req_context):
@@ -359,8 +407,8 @@ class TieringOffloadingManager(OffloadingManager):
"""
Prepare blocks to be loaded from primary tier to GPU.
CRITICAL: This method calls _maybe_process_finished_jobs() FIRST to ensure
that any completed promotions have been finalized and blocks are ready.
Callers only pass keys already confirmed HIT by lookup() earlier this
step.
This increments ref_cnt on the blocks in the primary tier, protecting
them from eviction during the transfer.
@@ -372,9 +420,6 @@ class TieringOffloadingManager(OffloadingManager):
Returns:
LoadStoreSpec for reading from primary tier.
"""
# Process completed promotions to ensure blocks are ready
self._maybe_process_finished_jobs()
return self.primary_tier.prepare_load(keys, req_context)
@override
@@ -427,8 +472,15 @@ class TieringOffloadingManager(OffloadingManager):
evicted, or None if store cannot proceed.
"""
# Step 1: Poll for completed async jobs FIRST
# This decrements ref_cnt on primary blocks that have been
# successfully transferred to secondary tiers.
# _process_finished_jobs() handles two kinds of completions here:
# - Cascade completions (store to a secondary tier, either a local
# cascade or a store job created for a remote requester via
# create_store_job()): decrements ref_cnt on the primary blocks
# that were read, making them evictable again once ref_cnt hits 0.
# - Promotion completions (secondary->primary loads): sets a
# not-yet-ready block's ref_cnt from -1 to 0 via complete_write(),
# making it evictable for the first time.
# Both must be accounted for before the eviction decision below.
self._maybe_process_finished_jobs()
# Step 2: Store to primary tier (new blocks only).
@@ -478,20 +530,7 @@ class TieringOffloadingManager(OffloadingManager):
return
for tier in request_level_tiers:
primary_blocks_spec = self.primary_tier.prepare_read(
ready_keys, req_context
)
job_id = self._next_job_id()
assert isinstance(primary_blocks_spec, CPULoadStoreSpec)
job_metadata = JobMetadata(
job_id=job_id,
keys=ready_keys,
block_ids=primary_blocks_spec.block_ids,
is_promotion=False,
req_context=req_context,
)
self._transfer_jobs[job_id] = job_metadata
job_metadata = self.create_store_job(ready_keys, req_context)
tier.submit_store(job_metadata)
@override
@@ -529,22 +568,7 @@ class TieringOffloadingManager(OffloadingManager):
# eviction during the async transfer). One prepare_read() call per
# secondary tier.
for tier in self.secondary_tiers:
primary_blocks_spec = self.primary_tier.prepare_read(keys, req_context)
# Submit async store job: primary→secondary
job_id = self._next_job_id()
# Track this store job
assert isinstance(primary_blocks_spec, CPULoadStoreSpec)
job_metadata = JobMetadata(
job_id=job_id,
keys=keys,
block_ids=primary_blocks_spec.block_ids,
is_promotion=False,
req_context=req_context,
)
self._transfer_jobs[job_id] = job_metadata
job_metadata = self.create_store_job(keys, req_context)
tier.submit_store(job_metadata)
# Note: The async transfers are now in flight. Their completion is
@@ -555,8 +579,40 @@ class TieringOffloadingManager(OffloadingManager):
state.pending_primary_stores -= 1
self._maybe_finalize_request(req_id)
def create_store_job(
self,
keys: Collection[OffloadKey],
req_context: ReqContext,
) -> JobMetadata:
"""Pin blocks in the primary tier and create a tracked store job.
Calls prepare_read() to increment ref_cnt (protecting blocks
from eviction during the async transfer), allocates a job ID,
and registers the job in _transfer_jobs.
The caller is responsible for the actual data transfer and
reporting completion via get_finished_jobs().
"""
primary_blocks_spec = self.primary_tier.prepare_read(keys, req_context)
assert isinstance(primary_blocks_spec, CPULoadStoreSpec)
job_id = self._next_job_id()
job_metadata = JobMetadata(
job_id=job_id,
keys=keys,
block_ids=primary_blocks_spec.block_ids,
is_promotion=False,
req_context=req_context,
)
self._transfer_jobs[job_id] = job_metadata
return job_metadata
@override
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
def on_new_request(
self,
req_context: ReqContext,
*,
exclude_tier: SecondaryTierManager | None = None,
) -> RequestOffloadingContext:
"""
Query each secondary tier for its offload policy preference.
@@ -565,6 +621,8 @@ class TieringOffloadingManager(OffloadingManager):
"""
state = RequestState(req_context=req_context)
for tier in self.secondary_tiers:
if tier is exclude_tier:
continue
tier_ctx = tier.on_new_request(req_context)
if tier_ctx.policy == OffloadPolicy.REQUEST_LEVEL:
if state.request_level_tiers is None:
@@ -580,13 +638,22 @@ class TieringOffloadingManager(OffloadingManager):
return RequestOffloadingContext(policy=policy)
@override
def on_request_finished(self, req_context: ReqContext) -> None:
def on_request_finished(
self,
req_context: ReqContext,
*,
exclude_tier: SecondaryTierManager | None = None,
) -> None:
self.primary_tier.on_request_finished(req_context)
state = self._req_state[req_context.req_id]
state.is_finished = True
self._maybe_finalize_request(req_context.req_id)
self._maybe_finalize_request(req_context.req_id, exclude_tier)
def _maybe_finalize_request(self, req_id: str) -> None:
def _maybe_finalize_request(
self,
req_id: str,
exclude_tier: SecondaryTierManager | None = None,
) -> None:
"""Finalize secondary tiers once no more store cascades can be submitted.
Finalization means forwarding on_request_finished() to secondary tiers.
@@ -600,6 +667,8 @@ class TieringOffloadingManager(OffloadingManager):
return
for tier in self.secondary_tiers:
if tier is exclude_tier:
continue
tier.on_request_finished(state.req_context)
del self._req_state[req_id]
@@ -611,8 +680,18 @@ class TieringOffloadingManager(OffloadingManager):
Called once per scheduler step from
OffloadingConnectorScheduler.build_connector_meta().
"""
# Catch-all poll: guarantees jobs are processed even on steps where
# lookup()/prepare_store() were never called (e.g. no requests
# scheduled but a tier still has_pending_work()).
self._maybe_process_finished_jobs()
for tier in self.secondary_tiers:
tier.serve_external_requests(self._tier_parents[tier])
# Reset the per-step gate AFTER serve_external_requests so that
# lookup() calls within it skip redundant _process_finished_jobs().
self._processed_jobs_this_step = False
self._flush_pending_promotions()
for tier in self.secondary_tiers:
tier.on_schedule_end(context)