# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """TieringOffloadingManager: Multi-tier KV cache offloading orchestrator. This manager coordinates between a CPU primary tier (with direct GPU access) and zero or more secondary tiers (Storage, Network, etc.) to provide hierarchical KV cache offloading. Key Design Principles: 1. Always offload to all tiers — When a chunk is stored to the primary tier, it is cascaded to ALL secondary tiers 2. Primary tier is the gateway — Secondary tiers cannot access GPU memory directly; all data flows through the CPU primary tier 3. Staged promotion — Chunks in secondary tiers must be promoted to the primary tier before GPU can access them 4. Transparent retry mechanism — Return None from lookup() to signal "data is being promoted, try later" 5. ref_cnt as eviction protection — primary.prepare_read() increments ref_cnt, protecting chunks from eviction until complete_read() is called """ import threading import time from collections.abc import Collection, Iterable, Sequence from dataclasses import dataclass, field from typing import NamedTuple import numpy as np from typing_extensions import override from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import ( OffloadingConnectorStats, ) from vllm.logger import init_logger from vllm.v1.kv_offload.base import ( ConfigInfoMapping, LoadStoreSpec, LookupResult, OffloadingEvent, OffloadingManager, OffloadKey, OffloadPolicy, PrepareStoreOutput, ReqContext, RequestOffloadingContext, ScheduleEndContext, ) from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion from vllm.v1.kv_offload.tiering.base import ( TIER_LABEL, JobId, JobResult, ParentManager, SecondaryTierManager, TransferJob, ) from vllm.v1.kv_offload.tiering.metrics import TieringMetricsTracker from vllm.v1.metrics.cache_hit_source import CacheHitSource logger = init_logger(__name__) # Default pause between tier-poll rounds; the user-facing default of the # ``tier_poll_interval_s`` extra-config key. A round is a non-blocking sweep, so # this sets how long a peer's request can sit unserved; 1 ms is three orders of # magnitude below the model step it hides, and matches the pause the p2p tier's # own drain loops and EngineCoreProc's GIL-yield sleep already use. DEFAULT_TIER_POLL_INTERVAL_S = 0.001 # How long a round waits for the manager lock before giving up and retrying. # Bounded so the thread stays responsive to shutdown even if a caller holds the # lock for a long time, or leaked it. _TIER_POLL_LOCK_TIMEOUT_S = 1.0 # Warn after this much continuous failure to acquire the lock. Above the 5 s # warning drain_jobs() emits, so a slow reset_cache() does not warn twice. _TIER_POLL_STALL_WARN_S = 30.0 # Give up after rounds have failed continuously for this long. Counted in time, # not rounds: at the default 1 ms interval a round budget is spent in a blink, # so a transient fault (a peer reconnecting) would turn the thread off for # good. on_schedule_end() still serves the tiers, so the fallback is per-step # servicing rather than none. _TIER_POLL_GIVE_UP_AFTER_S = 30.0 # Minimum spacing between logged tracebacks for a run of failing rounds, which # would otherwise log one per interval. _TIER_POLL_ERROR_LOG_INTERVAL_S = 1.0 # How long shutdown() waits for the thread to finish its round. _TIER_POLL_JOIN_TIMEOUT_S = 5.0 @dataclass class PendingPromotion: """Accumulator for chunks awaiting submit_load() for one (tier, request).""" req_context: ReqContext keys: list[OffloadKey] = field(default_factory=list) chunk_ids: list[int] = field(default_factory=list) @dataclass(slots=True) class RequestState: req_context: ReqContext pending_primary_stores: int = 0 is_finished: bool = False request_level_tiers: set[int] | None = None pending_cascade_keys: list[OffloadKey] = field(default_factory=list) class JobMetadata(NamedTuple): transfer_job: TransferJob tier_idx: int class CPUPrimaryTierOffloadingManager(CPUOffloadingManager): """CPUOffloadingManager with a primary/secondary transfer interface. The inherited prepare_store/complete_store/prepare_load/complete_load are the GPU-facing OffloadingManager interface. These aliases expose the same operations from the secondary tier perspective, where read/write refers to secondary accessing primary. This avoids confusion when reading TieringOffloadingManager code (e.g. calling prepare_load inside a cascade/store path would be misleading). """ def __init__( self, num_chunks: int, mmap_region: SharedOffloadRegion, cache_policy: str = "lru", cache_policy_module_path: str | None = None, enable_events: bool = False, ): super().__init__( num_chunks=num_chunks, cache_policy=cache_policy, cache_policy_module_path=cache_policy_module_path, enable_events=enable_events, ) self._mmap_region = mmap_region # read/write is for CPU<->secondary transfers, # load/store is for CPU<->GPU transfers. # These aliases avoid calling prepare_load inside a store path. self.complete_read = self.complete_load self.prepare_write = self.prepare_store self.complete_write = self.complete_store self._kv_memoryview = mmap_region.create_kv_memoryview() def prepare_read( self, keys: Collection[OffloadKey], req_context: ReqContext ) -> LoadStoreSpec: """Pin chunks for a CPU-to-secondary transfer. Cascade reads are implementation details of tiering, not additional request accesses, so they must not alter request-scoped recency. """ return self._prepare_load(keys, req_context, record_access=False) def get_kv_memoryview(self) -> memoryview: """Return the memoryview over the primary tier's KV cache buffer. The view has shape (num_chunks, row_stride_bytes) and is backed by the SharedOffloadRegion mmap. Secondary tiers address chunk *c* as ``view[c]``. """ return self._kv_memoryview @override def shutdown(self) -> None: super().shutdown() self._kv_memoryview.release() self._mmap_region.cleanup() class _SecondaryTierFacingParent(ParentManager): """Wrapper that implements ParentManager by delegating to the TieringOffloadingManager with exclude_tier_idx set to the origin tier.""" __slots__ = ("_m", "_origin_idx") def __init__( self, manager: "TieringOffloadingManager", tier_idx: int, ): self._m = manager self._origin_idx = tier_idx def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext: return self._m.on_new_request(req_context, exclude_tier_idx=self._origin_idx) def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult: return self._m.lookup(key, req_context, exclude_tier_idx=self._origin_idx) def create_store_job( self, keys: Collection[OffloadKey], req_context: ReqContext ) -> TransferJob: return self._m.create_store_job(keys, req_context, self._origin_idx) def on_request_finished(self, req_context: ReqContext) -> None: return self._m.on_request_finished( req_context, exclude_tier_idx=self._origin_idx ) class TieringOffloadingManager(OffloadingManager): """Orchestrates multi-tier KV cache offloading. This manager coordinates between a CPU primary tier (with direct GPU access) and zero or more secondary tiers (Storage, Network, etc.) to provide hierarchical KV cache offloading. Key internal state: - Minimal state tracking; relies on secondary tiers to report completion via get_finished_jobs() - Secondary tiers return JobResult objects containing all necessary information - job_id_counter: monotonically increasing counter for job IDs """ def __init__( self, primary_tier: CPUPrimaryTierOffloadingManager, secondary_tiers: list[SecondaryTierManager] | None = None, tier_poll_interval_s: float = DEFAULT_TIER_POLL_INTERVAL_S, ): """Initialize the TieringOffloadingManager. Args: primary_tier: The primary tier manager (CPU-based). secondary_tiers: List of secondary tier managers (e.g., Storage, Network). Can be None or empty list. tier_poll_interval_s: Pause between tier-poll rounds for tiers that set tolerates_off_thread_polling. Zero or negative disables the thread, leaving those tiers serviced once per engine step as before. """ # Guards this manager and every tier it composes, the primary tier # included. A real lock because the tier-poll thread enters the # manager too; the primary tier has none of its own. self._lock = threading.Lock() self.primary_tier: CPUPrimaryTierOffloadingManager = primary_tier self.secondary_tiers = secondary_tiers or [] self._job_id_counter: int = 0 # Job tracking: maps job_id to metadata for all in-flight transfers. # TransferJob.is_promotion distinguishes direction: # True: secondary → primary (promotion) # False: primary → secondary (cascade) self._jobs: dict[JobId, JobMetadata] = {} primary_view = self.primary_tier.get_kv_memoryview() assert primary_view.strides is not None self._metrics = TieringMetricsTracker( tier_types=[tier.tier_type for tier in self.secondary_tiers], num_primary_chunks=self.primary_tier._num_chunks, primary_chunk_size=primary_view.strides[0], ) # Pending promotion requests accumulated during lookup() calls; flushed # as one batched submit_load() per (tier, request) in on_schedule_end(). # Outer key: tier index. Inner key: req_context.req_id — the same ReqContext # object is reused for all chunk lookups of a given request per engine step. self._pending_load_submissions: dict[int, dict[str, PendingPromotion]] = {} # Gate for once-per-step execution of _maybe_process_finished_jobs(). # Reset at the end of each step in on_schedule_end(). self._processed_jobs_this_step: bool = False # Per-request state for prepared GPU->primary stores and finalization. # Secondary tiers are finalized only after pending primary stores reach # complete_store(), since complete_store() can still submit cascades. self._req_state: dict[str, RequestState] = {} # Preserve the original tier for this request's cache-hit metrics, # even after its KV blocks are promoted into host memory # (otherwise a disk or P2P hit would be mislabeled as a host hit). self._request_load_sources: dict[str, dict[OffloadKey, CacheHitSource]] = {} # Cached ParentManager wrappers for each secondary tier. self._tier_parents: dict[SecondaryTierManager, _SecondaryTierFacingParent] = { tier: _SecondaryTierFacingParent(self, tier_idx) for tier_idx, tier in enumerate(self.secondary_tiers) } self._tier_index: dict[SecondaryTierManager, int] = { tier: i for i, tier in enumerate(self.secondary_tiers) } # Tiers that serve external requests, and so must also be serviced # between engine steps. self._polled_tiers: list[SecondaryTierManager] = [ tier for tier in self.secondary_tiers if tier.tolerates_off_thread_polling ] self._tier_poll_interval_s = tier_poll_interval_s self._tier_poll_stop = threading.Event() self._tier_poll_thread: threading.Thread | None = None self._start_tier_poll_thread() @property @override def lock(self) -> threading.Lock: return self._lock # ------------------------------------------------------------------ # Tier polling # ------------------------------------------------------------------ def _start_tier_poll_thread(self) -> None: """Start the tier-poll thread, if any tier asked for one.""" if not self._polled_tiers or self._tier_poll_interval_s <= 0: return self._tier_poll_thread = threading.Thread( target=self._tier_poll_loop, name="vllm_offload_tier_poll", daemon=True, ) self._tier_poll_thread.start() logger.info( "KV offload tier-poll thread started for tier(s) %s, interval %.3fs", [tier.tier_type for tier in self._polled_tiers], self._tier_poll_interval_s, ) def _stop_tier_poll_thread(self) -> None: """Stop and join the tier-poll thread. Must run before any tier teardown: the thread drives tier transports, and closing one underneath it can crash the process rather than raise. """ self._tier_poll_stop.set() thread = self._tier_poll_thread if thread is None: return thread.join(timeout=_TIER_POLL_JOIN_TIMEOUT_S) if thread.is_alive(): logger.error( "KV offload tier-poll thread did not exit within %.1fs", _TIER_POLL_JOIN_TIMEOUT_S, ) else: self._tier_poll_thread = None def _tier_poll_loop(self) -> None: stalled_since: float | None = None # Start of the current run of failed rounds, and when one last logged. failing_since: float | None = None last_error_log = 0.0 while not self._tier_poll_stop.is_set(): if not self._lock.acquire(timeout=_TIER_POLL_LOCK_TIMEOUT_S): now = time.monotonic() if stalled_since is None: stalled_since = now elif now - stalled_since > _TIER_POLL_STALL_WARN_S: logger.warning( "KV offload tier-poll thread has not acquired the " "manager lock for %.0fs; peer requests are only being " "served at engine-step boundaries.", now - stalled_since, ) stalled_since = None continue stalled_since = None try: self.poll_tiers() failing_since = None except Exception: # Never let a round kill the thread silently: on_schedule_end() # still serves these tiers, so persistent failure degrades to # per-step servicing instead of stopping the tier-poll thread. now = time.monotonic() if failing_since is None: failing_since = now if now - last_error_log >= _TIER_POLL_ERROR_LOG_INTERVAL_S: last_error_log = now logger.exception("KV offload tier-poll round failed") finally: self._lock.release() if ( failing_since is not None and time.monotonic() - failing_since >= _TIER_POLL_GIVE_UP_AFTER_S ): logger.error( "KV offload tier-poll thread stopping after %.0fs of " "consecutive failures; falling back to per-step servicing.", _TIER_POLL_GIVE_UP_AFTER_S, ) return # Yield unconditionally, and only after release(). Python locks are # not fair, so a release-then-reacquire loop could starve the # scheduler thread; this pause is what bounds its wait to one round. self._tier_poll_stop.wait(self._tier_poll_interval_s) def poll_tiers(self) -> None: """Advance the control plane of tiers that cannot wait for a step. Polls those tiers for finished jobs and lets them serve whatever their counterpart has asked for -- the same two steps on_schedule_end() runs, narrowed to the tiers that opted in. Excludes _flush_pending_promotions() and _flush_pending_cascades() by design: both are per-step batching points, and running them here would fragment batches the scheduler thread accumulated during schedule(). A promotion that a lookup() from this path initiates therefore waits for the next on_schedule_end() to be submitted. The caller must hold self.lock for the whole call. serve_external_requests() relies on that: it establishes lookup() HITs and then pins them, and the two must not be separated by an eviction. Serving can look keys up through the parent, and lookup() runs the step's once-per-step poll. From here that poll would reach every tier off-thread, re-enter the serving tier's get_finished_jobs() from inside its own serve, and leave the gate set so the next step skips its poll. The round has just polled its tiers itself, so it holds the gate shut for its duration and then puts it back untouched. """ gate = self._processed_jobs_this_step self._processed_jobs_this_step = True try: self._process_finished_jobs(self._polled_tiers) for tier in self._polled_tiers: tier.serve_external_requests(self._tier_parents[tier]) finally: self._processed_jobs_this_step = gate @property def _transfer_jobs(self) -> dict[JobId, JobMetadata]: return self._jobs def _next_job_id(self) -> JobId: """Generate a unique job ID for async transfer tracking.""" job_id = self._job_id_counter self._job_id_counter += 1 return job_id def _register_job(self, transfer_job: TransferJob, tier_idx: int) -> None: job_metadata = JobMetadata(transfer_job, tier_idx) self._jobs[transfer_job.job_id] = job_metadata self._metrics.on_job_registered(job_metadata) def _pop_job(self, job_id: JobId) -> JobMetadata | None: return self._jobs.pop(job_id, None) def _maybe_process_finished_jobs(self): """Poll secondary tiers for completed jobs (at most once per step). Guarded by _processed_jobs_this_step: the first call in an engine step does the actual polling; subsequent calls are no-ops. The flag is reset in on_schedule_end() at the end of each step. """ if self._processed_jobs_this_step: return self._processed_jobs_this_step = True self._process_finished_jobs() def _complete_promotion( self, job_metadata: JobMetadata, completed_job: JobResult ) -> None: transfer_job = job_metadata.transfer_job successful_keys = completed_job.successful_keys failed_keys: Collection[OffloadKey] if completed_job.success: successful_keys = transfer_job.keys failed_keys = () elif successful_keys: failed_keys_set = set(transfer_job.keys) assert failed_keys_set.issuperset(successful_keys), ( f"Finished promotion job_id {completed_job.job_id} " "reported unknown successful keys" ) failed_keys_set.difference_update(successful_keys) failed_keys = failed_keys_set else: successful_keys = () failed_keys = transfer_job.keys load_sources = self._request_load_sources.get(transfer_job.req_context.req_id) if load_sources is not None: source = self.secondary_tiers[job_metadata.tier_idx].cache_hit_source for key in failed_keys: if load_sources.get(key) == source: del load_sources[key] if successful_keys: self.primary_tier.complete_write( successful_keys, transfer_job.req_context, True, ) if failed_keys: self.primary_tier.complete_write( failed_keys, transfer_job.req_context, False, ) def _process_finished_jobs( self, tiers: Sequence[SecondaryTierManager] | None = None, ): """Unconditionally poll secondary tiers for completed jobs. This method: 1. Calls get_finished_jobs() on each secondary tier 2. For completed stores (primary→secondary): calls primary.complete_read() to decrement ref_cnt 3. For completed loads (secondary→primary): calls primary.complete_write() to make chunks available Args: tiers: Tiers to poll. Defaults to every secondary tier. Pass a subset to poll only the tiers a caller is responsible for, so a tier that expects to be polled on the scheduler thread is not dragged elsewhere. """ if tiers is None: tiers = self.secondary_tiers for tier in tiers: i = self._tier_index[tier] for completed_job in tier.get_finished_jobs(): job_id = completed_job.job_id job_metadata = self._pop_job(job_id) assert job_metadata is not None, ( f"Finished job_id {job_id} from tier #{i}" f" ({tier.tier_type}) not in _jobs" ) assert job_metadata.tier_idx == i, ( f"Finished job_id {job_id} reported by tier #{i}" f" but belongs to tier #{job_metadata.tier_idx}" ) transfer_job = job_metadata.transfer_job self._metrics.on_job_finished(job_metadata, completed_job) if transfer_job.is_promotion: # secondary→primary transfer (promotion) completed. # Make chunks available in primary tier. self._complete_promotion(job_metadata, completed_job) else: # primary→secondary transfer completed. # Decrement ref_cnt on primary chunks. self.primary_tier.complete_read( transfer_job.keys, transfer_job.req_context ) if completed_job.success: self._update_backpressure(tier, job_metadata, completed_job) def _should_store_to_tier( self, tier: SecondaryTierManager, num_blocks: int ) -> bool: detector = tier.bp_detector if detector is None: return True return detector.should_store(num_blocks) def _update_backpressure( self, tier: SecondaryTierManager, job_metadata: JobMetadata, completed_job: JobResult, ) -> None: detector = tier.bp_detector if detector is None: return was_under_pressure = detector.is_under_pressure() tj = job_metadata.transfer_job num_bytes = ( completed_job.transfer_bytes if completed_job.transfer_bytes is not None else len(tj.keys) * tier.block_size_bytes ) detector.update(tj.submit_time, num_bytes) if detector.is_under_pressure() != was_under_pressure: tier_idx = self._tier_index[tier] logger.info( "Tier #%d (%s) back-pressure %s (stats=%s)", tier_idx, tier.tier_type, "activated" if detector.is_under_pressure() else "cleared", detector.stats, ) @override def lookup( self, key: OffloadKey, req_context: ReqContext, *, exclude_tier_idx: int | None = None, ) -> LookupResult: """Check whether a single chunk is offloaded and ready. Algorithm: 1. Process any completed async jobs first. 2. Query primary tier — short-circuit on hit or in-flight. 3. On primary miss, query secondary tiers — stop on first hit and initiate promotion. Args: key: Chunk hash to look up. req_context: Per-request context. exclude_tier_idx: Skip this tier index during the lookup. Returns: HIT — chunk is ready in the primary tier. HIT_PENDING — chunk found but not yet readable (write in-flight on the primary tier). RETRY — promotion started or a secondary tier is busy. MISS — chunk not found in any tier, or primary is full and cannot accept a promotion. """ # Poll first so a promotion that finished since the last call is # already reflected as HIT (not stale HIT_PENDING/MISS) below, and # so chunks freed by cascade or promotion completions are evictable # in time for a promotion this lookup may initiate. self._maybe_process_finished_jobs() start_time = time.monotonic() primary_hit = self.primary_tier.lookup(key, req_context) lookup_duration = time.monotonic() - start_time self._metrics.on_lookup( req_context, key, self._metrics.primary_tier_label, primary_hit, lookup_duration, ) if primary_hit is LookupResult.HIT: return LookupResult.HIT if primary_hit is LookupResult.HIT_PENDING: return LookupResult.HIT_PENDING any_retry = False for i, tier in enumerate(self.secondary_tiers): if i == exclude_tier_idx: continue if not req_context.load_tier_filter.allows(tier.medium, tier.locality): continue labelvalues = self._metrics.tier_label(i) start_time = time.monotonic() result = tier.lookup(key, req_context) lookup_duration = time.monotonic() - start_time if result is LookupResult.HIT: self._metrics.on_lookup( req_context, key, labelvalues, result, lookup_duration, ) promoted = self._initiate_promotion(i, key, req_context) return LookupResult.MISS if not promoted else LookupResult.HIT_PENDING if result is LookupResult.RETRY: any_retry = True self._metrics.on_lookup( req_context, key, labelvalues, result, lookup_duration, ) if any_retry: return LookupResult.RETRY return LookupResult.MISS @override def get_load_source( self, key: OffloadKey, req_context: ReqContext ) -> CacheHitSource: load_sources = self._request_load_sources.get(req_context.req_id) if load_sources is not None and key in load_sources: return load_sources[key] return self.primary_tier.get_load_source(key, req_context) def _initiate_promotion( self, tier_idx: int, key: OffloadKey, req_context: ReqContext, ) -> bool: """Queue a chunk for promotion from a secondary tier to the primary tier. Allocates space in the primary tier immediately (sets ref_cnt=-1 so subsequent lookups within the same step see the slot as in-flight), then defers the actual submit_load() call to _flush_pending_promotions() so all chunks queued during one engine step are submitted as a single batched job. Args: tier_idx: The secondary tier index to promote from key: Chunk to promote req_context: Per-request context forwarded to primary.prepare_write(). Returns: True if promotion was initiated, False if primary tier is full. """ # Allocate space in primary tier for promoted chunk. # Must happen immediately so primary.lookup() returns None (in-flight) # for this key on any subsequent lookup() call within the same step, # preventing duplicate promotion attempts. primary_write_result = self.primary_tier.prepare_write([key], req_context) if primary_write_result is None: # Primary tier is full; caller should treat the chunk as unavailable # rather than retrying indefinitely. self._metrics.on_promotion_allocation_failure() return False store_spec = primary_write_result.store_spec assert isinstance(store_spec, CPULoadStoreSpec) load_sources = self._request_load_sources.setdefault(req_context.req_id, {}) source = self.secondary_tiers[tier_idx].cache_hit_source for promoted_key in primary_write_result.keys_to_store: load_sources[promoted_key] = source # Defer submit_load to on_schedule_end(). Group by (tier, request) so # each request's chunks are submitted as one batched job per tier. tier_pending = self._pending_load_submissions.setdefault(tier_idx, {}) ctx_id = req_context.req_id if ctx_id not in tier_pending: tier_pending[ctx_id] = PendingPromotion( keys=[], chunk_ids=[], req_context=req_context ) entry = tier_pending[ctx_id] entry.keys.extend(primary_write_result.keys_to_store) entry.chunk_ids.extend(store_spec.chunk_ids) return True def _flush_pending_promotions(self) -> None: """Submit one batched submit_load() per (tier, request). Called from on_schedule_end() at the end of each scheduler step, flushing all promotion requests deferred during lookup(). """ if not self._pending_load_submissions: return for tier_idx, pending_by_ctx in self._pending_load_submissions.items(): tier = self.secondary_tiers[tier_idx] for entry in pending_by_ctx.values(): job_id = self._next_job_id() job_metadata = TransferJob( job_id=job_id, keys=entry.keys, chunk_ids=np.array(entry.chunk_ids, dtype=np.int32), is_promotion=True, req_context=entry.req_context, ) self._register_job(job_metadata, tier_idx) tier.submit_load(job_metadata) self._pending_load_submissions.clear() @override def prepare_load( self, keys: Collection[OffloadKey], req_context: ReqContext ) -> LoadStoreSpec: """Prepare chunks to be loaded from primary tier to GPU. Callers only pass keys already confirmed HIT by lookup() earlier this step. This increments ref_cnt on the chunks in the primary tier, protecting them from eviction during the transfer. Args: keys: Chunks to prepare for loading. req_context: Per-request context. Returns: LoadStoreSpec for reading from primary tier. """ return self.primary_tier.prepare_load(keys, req_context) @override def touch(self, keys: Collection[OffloadKey], req_context: ReqContext): """Mark chunks as recently used in all tiers. Args: keys: Chunks to mark as recently used. req_context: Per-request context. """ self.primary_tier.touch(keys, req_context) for tier in self.secondary_tiers: tier.touch(keys, req_context) @override def complete_load(self, keys: Collection[OffloadKey], req_context: ReqContext): """Mark chunks as done loading from primary tier to GPU. This decrements ref_cnt on the chunks in the primary tier, allowing them to be evicted again. Args: keys: Chunks that finished loading. req_context: Per-request context. """ self.primary_tier.complete_load(keys, req_context) @override def prepare_store( self, keys: Collection[OffloadKey], req_context: ReqContext ) -> PrepareStoreOutput | None: """Prepare chunks to be stored from GPU to primary tier. CRITICAL: This method polls for finished jobs FIRST to ensure that any completed async transfers have their ref_cnt decremented before the primary tier makes eviction decisions. For request-level tiers, chunks already present in the primary tier are immediately cascaded via submit_store(). Args: keys: Chunks to prepare for storing. req_context: Per-request context. Returns: PrepareStoreOutput describing where to store chunks and what was evicted, or None if store cannot proceed. """ # Step 1: Poll for completed async jobs FIRST # _process_finished_jobs() handles two kinds of completions here: # - Cascade completions (store to a secondary tier, either a local # cascade or a store job created for a remote requester via # create_store_job()): decrements ref_cnt on the primary chunks # that were read, making them evictable again once ref_cnt hits 0. # - Promotion completions (secondary->primary loads): sets a # not-yet-ready chunk's ref_cnt from -1 to 0 via complete_write(), # making it evictable for the first time. # Both must be accounted for before the eviction decision below. # Unconditional, not _maybe_process_finished_jobs(): eviction must see # the freshest completions, and on_schedule_end now runs at the very end # of the step, so the once-per-step gate would otherwise still be set by # this step's first lookup() and skip the poll. self._processed_jobs_this_step = True self._process_finished_jobs() # Step 2: Store to primary tier (new chunks only). # Cascading of these newly-stored chunks to ALL secondary tiers # happens later in complete_store(), after the GPU→Primary transfer # completes. primary_result = self.primary_tier.prepare_store(keys, req_context) if primary_result is None: return None if primary_result.keys_to_store: state = self._req_state[req_context.req_id] state.pending_primary_stores += 1 # Step 3: For request-level tiers, cascade chunks already in primary request_level_tiers = self._req_state[req_context.req_id].request_level_tiers if request_level_tiers: keys_to_store_set = set(primary_result.keys_to_store) keys_already_in_primary = tuple( k for k in keys if k not in keys_to_store_set ) if keys_already_in_primary: self._cascade_existing_chunks_to_request_level_tiers( keys_already_in_primary, req_context, request_level_tiers ) return primary_result def _cascade_existing_chunks_to_request_level_tiers( self, keys: Sequence[OffloadKey], req_context: ReqContext, request_level_tiers: set[int], ) -> None: """For tiers that requested request-level policy, submit_store() for chunks that are already present in the primary tier. A key whose primary write is still in flight (HIT_PENDING) cannot be dropped: prepare_store already excluded it as present, and the scheduler advances past its chunk, so no path offers it again. Park it instead. MISS keys are dropped, since nothing is there to read. The primary tier resolves every key it holds, so RETRY cannot reach here. Parking on it would have no guarantee of ever draining, which is what makes parking HIT_PENDING safe, so it is rejected rather than guessed at. """ state = self._req_state[req_context.req_id] ready_keys = [] for key in keys: result = self.primary_tier.lookup(key, req_context) if result is LookupResult.HIT: ready_keys.append(key) elif result is LookupResult.HIT_PENDING: state.pending_cascade_keys.append(key) else: assert result is LookupResult.MISS, ( f"primary tier returned {result} for a cascade key" ) if not ready_keys: return for tier_idx in request_level_tiers: tier = self.secondary_tiers[tier_idx] if not self._should_store_to_tier(tier, len(ready_keys)): continue job_metadata = self.create_store_job(ready_keys, req_context, tier_idx) tier.submit_store(job_metadata) def _flush_pending_cascades(self) -> None: """Retry request-level cascades parked on an in-flight primary write. A parked key always resolves, to HIT or to MISS, so the set drains and a request cannot be held from finalization forever. """ for req_id, state in list(self._req_state.items()): if not state.pending_cascade_keys: continue assert state.request_level_tiers keys, state.pending_cascade_keys = state.pending_cascade_keys, [] self._cascade_existing_chunks_to_request_level_tiers( keys, state.req_context, state.request_level_tiers ) self._maybe_finalize_request(req_id) @override def complete_store( self, keys: Collection[OffloadKey], req_context: ReqContext, success: bool = True, ) -> None: """Mark chunks as done storing from GPU to primary tier. This is where secondary tier cascading happens — after chunks are confirmed to be in the primary tier, they are cascaded to ALL secondary tiers. For each secondary tier: 1. Call primary.prepare_read() to get LoadStoreSpec AND increment ref_cnt (protecting chunks during async transfer) 2. Call tier.submit_store() to start async transfer: primary→secondary 3. Track the job in _store_jobs dictionary Args: keys: Chunks that finished storing. success: Whether the GPU→primary transfer succeeded. req_context: Per-request context forwarded to primary.prepare_read(). """ # Step 1: Complete store in primary tier (makes chunks loadable) self.primary_tier.complete_store(keys, req_context, success) if success: # Step 2: Cascade to ALL secondary tiers # For each secondary tier, call primary.prepare_read() to get the # LoadStoreSpec AND to increment ref_cnt (protecting chunks from # eviction during the async transfer). One prepare_read() call per # secondary tier. for tier_idx, tier in enumerate(self.secondary_tiers): if not self._should_store_to_tier(tier, len(keys)): continue job_metadata = self.create_store_job(keys, req_context, tier_idx) tier.submit_store(job_metadata) # Note: The async transfers are now in flight. Their completion is # tracked via get_finished_jobs() / _maybe_process_finished_jobs(). req_id = req_context.req_id state = self._req_state[req_id] assert state.pending_primary_stores > 0 state.pending_primary_stores -= 1 self._maybe_finalize_request(req_id) def create_store_job( self, keys: Collection[OffloadKey], req_context: ReqContext, tier_idx: int = 0, ) -> TransferJob: """Pin chunks in the primary tier and create a tracked store job. Calls prepare_read() to increment ref_cnt (protecting chunks from eviction during the async transfer), allocates a job ID, and registers the job in _jobs. The caller is responsible for the actual data transfer and reporting completion via get_finished_jobs(). """ primary_chunks_spec = self.primary_tier.prepare_read(keys, req_context) assert isinstance(primary_chunks_spec, CPULoadStoreSpec) job_id = self._next_job_id() job_metadata = TransferJob( job_id=job_id, keys=keys, chunk_ids=primary_chunks_spec.chunk_ids, is_promotion=False, req_context=req_context, ) self._register_job(job_metadata, tier_idx) return job_metadata @override def on_new_request( self, req_context: ReqContext, *, exclude_tier_idx: int | None = None, ) -> RequestOffloadingContext: """Query each secondary tier for its offload policy preference. Returns REQUEST_LEVEL if ANY secondary tier wants request-level. Only stores REQUEST_LEVEL tier decisions for use in prepare_store. """ state = RequestState(req_context=req_context) self._metrics.on_new_request(req_context) for tier_idx, tier in enumerate(self.secondary_tiers): if tier_idx == exclude_tier_idx: continue tier_ctx = tier.on_new_request(req_context) if tier_ctx.policy == OffloadPolicy.REQUEST_LEVEL: if state.request_level_tiers is None: state.request_level_tiers = set() state.request_level_tiers.add(tier_idx) self._req_state[req_context.req_id] = state policy = ( OffloadPolicy.REQUEST_LEVEL if state.request_level_tiers else OffloadPolicy.CHUNK_LEVEL ) return RequestOffloadingContext(policy=policy) @override def on_request_finished( self, req_context: ReqContext, *, exclude_tier_idx: int | None = None, ) -> None: self.primary_tier.on_request_finished(req_context) state = self._req_state[req_context.req_id] state.is_finished = True self._maybe_finalize_request(req_context.req_id, exclude_tier_idx) def _maybe_finalize_request( self, req_id: str, exclude_tier_idx: int | None = None, ) -> None: """Finalize secondary tiers once no more cascades can be submitted. Their finalization is delayed until pending GPU->primary stores finish, since those callbacks may still submit secondary stores. """ state = self._req_state[req_id] if not state.is_finished: return if state.pending_primary_stores != 0: return if state.pending_cascade_keys: return for tier_idx, tier in enumerate(self.secondary_tiers): if tier_idx == exclude_tier_idx: continue tier.on_request_finished(state.req_context) self._metrics.on_request_finished(state.req_context) self._request_load_sources.pop(req_id, None) del self._req_state[req_id] @override def on_schedule_end(self, context: ScheduleEndContext) -> None: """End-of-schedule hook: process finished jobs, flush deferred promotions, and reset the per-step gate. Called once per scheduler step, as the last manager call of the step, from OffloadingConnectorScheduler.build_connector_meta(). """ # Catch-all poll: guarantees jobs are processed even on steps where # lookup()/prepare_store() were never called (e.g. no requests # scheduled but a tier still has_pending_work()). self._maybe_process_finished_jobs() for tier in self.secondary_tiers: tier.serve_external_requests(self._tier_parents[tier]) # Reset the per-step gate AFTER serve_external_requests so that # lookup() calls within it skip redundant _process_finished_jobs(). self._processed_jobs_this_step = False self._flush_pending_promotions() # Keys parked by this step's prepare_store cannot have completed yet, so # they are simply re-parked here and retried on the next step. That is # safe: _maybe_finalize_request() holds off while pending_cascade_keys is # non-empty, so nothing finalizes early and the list still drains. self._flush_pending_cascades() for tier in self.secondary_tiers: tier.on_schedule_end(context) for req_id in context.new_req_ids: state = self._req_state.get(req_id) if state is None: continue self._metrics.on_request_allocated(state.req_context) @override def has_pending_work(self) -> bool: # In-flight primary<->secondary transfers (pending promotions are # translated to transfer jobs in on_schedule_end), plus any work the # secondary tiers themselves still have outstanding. return ( bool(self._jobs) or any(state.pending_cascade_keys for state in self._req_state.values()) or any(tier.has_pending_work() for tier in self.secondary_tiers) ) @override def take_events(self) -> Iterable[OffloadingEvent]: """Yield events owned by the primary and secondary tiers. Yields: New OffloadingEvents collected by each tier since the last call. """ yield from self.primary_tier.take_events() for tier in self.secondary_tiers: yield from tier.take_events() @override def reset_cache(self) -> None: """Reset transfer bookkeeping and primary-tier cache. Called during sleep, weight update, or resume. Each secondary tier drains its in-flight transfers via drain_jobs() so no tier I/O is touching primary memory before the primary tier is reset. A stuck tier will block here visibly — preferable to silent corruption from reusing primary slots while a transfer is mid-copy. Secondary tiers are intentionally not reset: persistent stores (FS, network) keep their data across resets. Active request state is retained so those requests can continue after the reset; finished requests are finalized and removed. """ for tier in self.secondary_tiers: tier.drain_jobs() # All tier I/O has stopped; consume their completion notifications # so manager bookkeeping is consistent before the primary reset. self._process_finished_jobs() assert not self._jobs # Deferred promotion submissions reserve primary slots that the # reset below invalidates; their submit_load() has not yet been # called so no tier I/O is touching that memory. self._pending_load_submissions.clear() self._request_load_sources.clear() self._metrics.assert_idle() finished_req_ids = [] for req_id, state in self._req_state.items(): state.pending_primary_stores = 0 state.pending_cascade_keys.clear() if not state.is_finished: continue for tier in self.secondary_tiers: tier.on_request_finished(state.req_context) self._metrics.on_request_finished(state.req_context) finished_req_ids.append(req_id) self.primary_tier.reset_cache() for req_id in finished_req_ids: del self._req_state[req_id] self._processed_jobs_this_step = False for tier in self.secondary_tiers: if tier.bp_detector is not None: tier.bp_detector.reset() @override def get_stats(self) -> OffloadingConnectorStats | None: stats = self.primary_tier.get_stats() if stats is not None and stats.is_empty(): stats = None self._metrics.record_backpressure(self.secondary_tiers) metrics_stats = self._metrics.take_stats() if metrics_stats is not None: if stats is None: stats = metrics_stats else: stats.aggregate(metrics_stats) for tier in self.secondary_tiers: tier_stats = tier.get_stats() if tier_stats is None or tier_stats.is_empty(): continue if stats is None: stats = tier_stats else: stats.aggregate(tier_stats) return stats @override def config_info(self) -> Sequence[ConfigInfoMapping]: """Return one mapping for each tier, primary tier first, with TIER_LABEL. Raises: ValueError: If the primary tier returns more than one mapping. """ (primary_info,) = self.primary_tier.config_info() primary_tier_info = dict(primary_info) primary_tier_info[TIER_LABEL] = self._metrics.primary_tier_label[0] tier_infos = [primary_tier_info] for tier in self.secondary_tiers: tier_info = dict(tier.config_info()) tier_info[TIER_LABEL] = self._metrics.tier_label(self._tier_index[tier])[0] tier_infos.append(tier_info) return tier_infos @override def shutdown(self) -> None: """Shut down secondary tiers before releasing primary resources. Stops the tier-poll thread first: it drives tier transports, and tearing one down underneath it (closing a socket, destroying a ZMQ context) can take the process down rather than raise. If the thread will not exit, hold the manager lock across tier teardown so it cannot be mid-round, and raise if even that is unavailable -- tearing a tier down under a live round risks a crash, and skipping it silently would leak the primary's shared memory region. Unlike every other manager method, call this WITHOUT holding self.lock: it takes the lock itself, which is not reentrant, and the tier-poll thread needs the lock free to notice the stop request and exit. A caller that does hold it only gets the RuntimeError above, after waiting out the timeout. Every secondary tier is given a shutdown attempt. If any shutdown fails, preserve the primary mmap because a failed tier may still use it. """ self._stop_tier_poll_thread() if self._tier_poll_thread is not None: if not self._lock.acquire(timeout=_TIER_POLL_JOIN_TIMEOUT_S): raise RuntimeError( "KV offload tier-poll thread is still running and the " "manager lock is unavailable; secondary tiers were not " "shut down. shutdown() must be called without holding " "manager.lock." ) try: self._shutdown_tiers() finally: self._lock.release() return self._shutdown_tiers() def _shutdown_tiers(self) -> None: shutdown_error: Exception | None = None for tier_idx, tier in enumerate(self.secondary_tiers): try: tier.shutdown() except Exception as exc: shutdown_error = exc logger.exception( "Failed to shut down secondary tier #%d " "(tier_type=%s, impl_class=%s)", tier_idx, tier.tier_type, type(tier).__name__, ) if shutdown_error is not None: raise shutdown_error self.primary_tier.shutdown()