From 48fcfc926c0a6e62a8f46557a59163702b7403b9 Mon Sep 17 00:00:00 2001 From: Ronen Schaffer Date: Tue, 7 Jul 2026 13:51:18 +0300 Subject: [PATCH] [KV Offload] Add `ParentManager` ABC for secondary tier callbacks (#47274) Signed-off-by: Ronen Schaffer Co-authored-by: Or Ozeri --- vllm/v1/kv_offload/tiering/base.py | 50 +++++++- vllm/v1/kv_offload/tiering/manager.py | 163 +++++++++++++++++++------- 2 files changed, 169 insertions(+), 44 deletions(-) diff --git a/vllm/v1/kv_offload/tiering/base.py b/vllm/v1/kv_offload/tiering/base.py index b022e1f2f8..9e3d19644e 100644 --- a/vllm/v1/kv_offload/tiering/base.py +++ b/vllm/v1/kv_offload/tiering/base.py @@ -49,6 +49,43 @@ class JobResult: success: bool +class ParentManager(ABC): + """Interface for secondary tiers to call back into the tiering manager. + + Passed to secondary tiers via serve_external_requests() each step. + The _SecondaryTierFacingParent wrapper implements this, automatically + excluding the calling tier from fan-out operations. + + Required call sequence for each remote request: + 1. on_new_request(req_context) — set up per-request state + 2. lookup(key, req_context) — check block availability + (repeat per block) + 3. create_store_job(keys, req_context) — pin blocks and get a + job handle + 4. on_request_finished(req_context) — clean up per-request state + + Steps 2-3 may be interleaved. Step 4 must be called even if no + blocks were found, to avoid leaking async lookup state (e.g. in + the fs tier's AsyncLookupManager). + """ + + @abstractmethod + def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext: ... + + @abstractmethod + def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult: ... + + @abstractmethod + def create_store_job( + self, + keys: Collection[OffloadKey], + req_context: ReqContext, + ) -> JobMetadata: ... + + @abstractmethod + def on_request_finished(self, req_context: ReqContext) -> None: ... + + class SecondaryTierManager(ABC): """ Abstract interface for managing a single non-primary offloading tier. @@ -210,11 +247,20 @@ class SecondaryTierManager(ABC): """ return + def serve_external_requests(self, parent: ParentManager) -> None: + """Process remotely-originated requests using the parent manager. + + Called once per scheduler step, BEFORE _flush_pending_promotions(). + The parent handle is valid only for the duration of this call. + Tiers that don't serve external requests leave this as a no-op. + """ + return + def on_schedule_end(self, context: ScheduleEndContext) -> None: """Called once at the end of each scheduler step. - Secondary tiers may override this for per-step cleanup or - deferred work submission. + Args: + context: Per-step context from the scheduler. """ return diff --git a/vllm/v1/kv_offload/tiering/manager.py b/vllm/v1/kv_offload/tiering/manager.py index 81151a7c0c..156f5a96e1 100644 --- a/vllm/v1/kv_offload/tiering/manager.py +++ b/vllm/v1/kv_offload/tiering/manager.py @@ -48,6 +48,7 @@ from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion from vllm.v1.kv_offload.tiering.base import ( JobId, JobMetadata, + ParentManager, SecondaryTierManager, ) @@ -120,6 +121,35 @@ class CPUPrimaryTierOffloadingManager(CPUOffloadingManager): self._mmap_region.cleanup() +class _SecondaryTierFacingParent(ParentManager): + """Wrapper that implements ParentManager by delegating to the + TieringOffloadingManager with exclude_tier set to the origin tier.""" + + __slots__ = ("_m", "_origin") + + def __init__( + self, + manager: "TieringOffloadingManager", + tier: SecondaryTierManager, + ): + self._m = manager + self._origin = tier + + def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext: + return self._m.on_new_request(req_context, exclude_tier=self._origin) + + def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult: + return self._m.lookup(key, req_context, exclude_tier=self._origin) + + def create_store_job( + self, keys: Collection[OffloadKey], req_context: ReqContext + ) -> JobMetadata: + return self._m.create_store_job(keys, req_context) + + def on_request_finished(self, req_context: ReqContext) -> None: + return self._m.on_request_finished(req_context, exclude_tier=self._origin) + + class TieringOffloadingManager(OffloadingManager): """ Orchestrates multi-tier KV cache offloading. @@ -180,6 +210,12 @@ class TieringOffloadingManager(OffloadingManager): # complete_store(), since complete_store() can still submit cascades. self._req_state: dict[str, RequestState] = {} + # Cached ParentManager wrappers for each secondary tier. + self._tier_parents: dict[SecondaryTierManager, _SecondaryTierFacingParent] = { + tier: _SecondaryTierFacingParent(self, tier) + for tier in self.secondary_tiers + } + def _next_job_id(self) -> JobId: """Generate a unique job ID for async transfer tracking.""" job_id = self._job_id_counter @@ -235,7 +271,13 @@ class TieringOffloadingManager(OffloadingManager): ) @override - def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult: + def lookup( + self, + key: OffloadKey, + req_context: ReqContext, + *, + exclude_tier: SecondaryTierManager | None = None, + ) -> LookupResult: """ Check whether a single block is offloaded and ready. @@ -257,6 +299,10 @@ class TieringOffloadingManager(OffloadingManager): MISS — block not found in any tier, or primary is full and cannot accept a promotion. """ + # Poll first so a promotion that finished since the last call is + # already reflected as HIT (not stale HIT_PENDING/MISS) below, and + # so blocks freed by cascade or promotion completions are evictable + # in time for a promotion this lookup may initiate. self._maybe_process_finished_jobs() primary_hit = self.primary_tier.lookup(key, req_context) @@ -267,6 +313,8 @@ class TieringOffloadingManager(OffloadingManager): any_retry = False for tier in self.secondary_tiers: + if tier is exclude_tier: + continue result = tier.lookup(key, req_context) if result is LookupResult.HIT: if not self._initiate_promotion(tier, key, req_context): @@ -359,8 +407,8 @@ class TieringOffloadingManager(OffloadingManager): """ Prepare blocks to be loaded from primary tier to GPU. - CRITICAL: This method calls _maybe_process_finished_jobs() FIRST to ensure - that any completed promotions have been finalized and blocks are ready. + Callers only pass keys already confirmed HIT by lookup() earlier this + step. This increments ref_cnt on the blocks in the primary tier, protecting them from eviction during the transfer. @@ -372,9 +420,6 @@ class TieringOffloadingManager(OffloadingManager): Returns: LoadStoreSpec for reading from primary tier. """ - # Process completed promotions to ensure blocks are ready - self._maybe_process_finished_jobs() - return self.primary_tier.prepare_load(keys, req_context) @override @@ -427,8 +472,15 @@ class TieringOffloadingManager(OffloadingManager): evicted, or None if store cannot proceed. """ # Step 1: Poll for completed async jobs FIRST - # This decrements ref_cnt on primary blocks that have been - # successfully transferred to secondary tiers. + # _process_finished_jobs() handles two kinds of completions here: + # - Cascade completions (store to a secondary tier, either a local + # cascade or a store job created for a remote requester via + # create_store_job()): decrements ref_cnt on the primary blocks + # that were read, making them evictable again once ref_cnt hits 0. + # - Promotion completions (secondary->primary loads): sets a + # not-yet-ready block's ref_cnt from -1 to 0 via complete_write(), + # making it evictable for the first time. + # Both must be accounted for before the eviction decision below. self._maybe_process_finished_jobs() # Step 2: Store to primary tier (new blocks only). @@ -478,20 +530,7 @@ class TieringOffloadingManager(OffloadingManager): return for tier in request_level_tiers: - primary_blocks_spec = self.primary_tier.prepare_read( - ready_keys, req_context - ) - - job_id = self._next_job_id() - assert isinstance(primary_blocks_spec, CPULoadStoreSpec) - job_metadata = JobMetadata( - job_id=job_id, - keys=ready_keys, - block_ids=primary_blocks_spec.block_ids, - is_promotion=False, - req_context=req_context, - ) - self._transfer_jobs[job_id] = job_metadata + job_metadata = self.create_store_job(ready_keys, req_context) tier.submit_store(job_metadata) @override @@ -529,22 +568,7 @@ class TieringOffloadingManager(OffloadingManager): # eviction during the async transfer). One prepare_read() call per # secondary tier. for tier in self.secondary_tiers: - primary_blocks_spec = self.primary_tier.prepare_read(keys, req_context) - - # Submit async store job: primary→secondary - job_id = self._next_job_id() - - # Track this store job - assert isinstance(primary_blocks_spec, CPULoadStoreSpec) - job_metadata = JobMetadata( - job_id=job_id, - keys=keys, - block_ids=primary_blocks_spec.block_ids, - is_promotion=False, - req_context=req_context, - ) - self._transfer_jobs[job_id] = job_metadata - + job_metadata = self.create_store_job(keys, req_context) tier.submit_store(job_metadata) # Note: The async transfers are now in flight. Their completion is @@ -555,8 +579,40 @@ class TieringOffloadingManager(OffloadingManager): state.pending_primary_stores -= 1 self._maybe_finalize_request(req_id) + def create_store_job( + self, + keys: Collection[OffloadKey], + req_context: ReqContext, + ) -> JobMetadata: + """Pin blocks in the primary tier and create a tracked store job. + + Calls prepare_read() to increment ref_cnt (protecting blocks + from eviction during the async transfer), allocates a job ID, + and registers the job in _transfer_jobs. + + The caller is responsible for the actual data transfer and + reporting completion via get_finished_jobs(). + """ + primary_blocks_spec = self.primary_tier.prepare_read(keys, req_context) + assert isinstance(primary_blocks_spec, CPULoadStoreSpec) + job_id = self._next_job_id() + job_metadata = JobMetadata( + job_id=job_id, + keys=keys, + block_ids=primary_blocks_spec.block_ids, + is_promotion=False, + req_context=req_context, + ) + self._transfer_jobs[job_id] = job_metadata + return job_metadata + @override - def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext: + def on_new_request( + self, + req_context: ReqContext, + *, + exclude_tier: SecondaryTierManager | None = None, + ) -> RequestOffloadingContext: """ Query each secondary tier for its offload policy preference. @@ -565,6 +621,8 @@ class TieringOffloadingManager(OffloadingManager): """ state = RequestState(req_context=req_context) for tier in self.secondary_tiers: + if tier is exclude_tier: + continue tier_ctx = tier.on_new_request(req_context) if tier_ctx.policy == OffloadPolicy.REQUEST_LEVEL: if state.request_level_tiers is None: @@ -580,13 +638,22 @@ class TieringOffloadingManager(OffloadingManager): return RequestOffloadingContext(policy=policy) @override - def on_request_finished(self, req_context: ReqContext) -> None: + def on_request_finished( + self, + req_context: ReqContext, + *, + exclude_tier: SecondaryTierManager | None = None, + ) -> None: self.primary_tier.on_request_finished(req_context) state = self._req_state[req_context.req_id] state.is_finished = True - self._maybe_finalize_request(req_context.req_id) + self._maybe_finalize_request(req_context.req_id, exclude_tier) - def _maybe_finalize_request(self, req_id: str) -> None: + def _maybe_finalize_request( + self, + req_id: str, + exclude_tier: SecondaryTierManager | None = None, + ) -> None: """Finalize secondary tiers once no more store cascades can be submitted. Finalization means forwarding on_request_finished() to secondary tiers. @@ -600,6 +667,8 @@ class TieringOffloadingManager(OffloadingManager): return for tier in self.secondary_tiers: + if tier is exclude_tier: + continue tier.on_request_finished(state.req_context) del self._req_state[req_id] @@ -611,8 +680,18 @@ class TieringOffloadingManager(OffloadingManager): Called once per scheduler step from OffloadingConnectorScheduler.build_connector_meta(). """ + # Catch-all poll: guarantees jobs are processed even on steps where + # lookup()/prepare_store() were never called (e.g. no requests + # scheduled but a tier still has_pending_work()). self._maybe_process_finished_jobs() + + for tier in self.secondary_tiers: + tier.serve_external_requests(self._tier_parents[tier]) + + # Reset the per-step gate AFTER serve_external_requests so that + # lookup() calls within it skip redundant _process_finished_jobs(). self._processed_jobs_this_step = False + self._flush_pending_promotions() for tier in self.secondary_tiers: tier.on_schedule_end(context)