# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """TieringOffloadingSpec: Spec for multi-tier KV cache offloading. This spec creates a TieringOffloadingManager with a CPU primary tier and configurable secondary tiers (e.g., Storage, Network). Configuration via kv_connector_extra_config: - cpu_bytes_to_use: (required) Bytes to allocate for CPU primary tier - block_size: (optional) Tokens per offloaded chunk (default: GPU block size) - eviction_policy: (optional) Primary tier eviction policy: built-in "lru"/ "arc", or the name of a policy registered via CachePolicyFactory, or an out-of-tree CachePolicy class name paired with cache_policy_module_path (default: "lru") - cache_policy_module_path: (optional) Python import path to load eviction_policy from when it names an out-of-tree CachePolicy not registered via CachePolicyFactory - secondary_tiers: (optional) List of secondary tier configurations Each secondary tier config is a dict with: - type: (required) Type of secondary tier (e.g., "example", "fs", "p2p", "obj"), or the class name of an out-of-tree SecondaryTierManager paired with module_path. - module_path: (optional) Python import path to load 'type' from when it names an out-of-tree SecondaryTierManager not registered via SecondaryTierFactory.register_tier() - Additional tier-specific parameters are passed directly to the tier constructor. See each tier's documentation for supported parameters. Example configuration: { "cpu_bytes_to_use": 10737418240, # 10 GB "block_size": 16, "eviction_policy": "lru", "secondary_tiers": [ { "type": "example", "custom_param": 67 } ] } Example out-of-tree tier configuration: { "cpu_bytes_to_use": 10737418240, "secondary_tiers": [ { "type": "MyCustomTier", "module_path": "my_package.my_module", "custom_param": "value" } ] } """ from typing import Any from typing_extensions import override from vllm.logger import init_logger from vllm.v1.kv_offload.base import ( CanonicalKVCaches, OffloadingCounterMetadata, OffloadingGaugeMetadata, OffloadingHistogramMetadata, OffloadingManager, OffloadingMetricMetadata, ) from vllm.v1.kv_offload.config import OffloadingConfig from vllm.v1.kv_offload.cpu.gpu_worker import CPUOffloadingWorker from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec from vllm.v1.kv_offload.tiering.base import ( TIER_LABEL, SecondaryTierManager, TieringOffloadingMetrics, ) from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory from vllm.v1.kv_offload.tiering.manager import ( DEFAULT_TIER_POLL_INTERVAL_S, CPUPrimaryTierOffloadingManager, TieringOffloadingManager, ) logger = init_logger(__name__) class TieringOffloadingSpec(CPUOffloadingSpec): """Spec for multi-tier KV cache offloading. Creates a TieringOffloadingManager with: - Primary tier: CPU (LRU or ARC eviction policy) - Secondary tiers: Configurable via extra_config The CPU primary tier has direct GPU access and serves as the gateway for all GPU↔offload operations. Secondary tiers cannot directly access GPU memory and must transfer data through the primary tier. """ BLOCK_SIZE_ALIGNMENT = SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT @classmethod @override def build_metric_definitions( cls, extra_config: dict[str, Any] ) -> dict[str, OffloadingMetricMetadata]: metrics = super().build_metric_definitions(extra_config) metrics[TieringOffloadingMetrics.LOOKUP_SYNC_DELAY] = ( OffloadingHistogramMetadata( documentation=( "Histogram of blocking time spent in a per-chunk tier lookup " "that resolved as a hit or miss, labeled by tier, in seconds." ), labelnames=(TIER_LABEL,), buckets=( 0.00001, 0.00005, 0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, ), ) ) metrics[TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY] = ( OffloadingHistogramMetadata( documentation=( "Histogram of wall-clock time from a per-chunk tier lookup " "first returning retry until that same tier lookup resolves " "as a hit or miss, labeled by tier, in seconds." ), labelnames=(TIER_LABEL,), buckets=( 0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10, ), ) ) metrics[TieringOffloadingMetrics.READ_BYTES] = OffloadingCounterMetadata( documentation=( "Total bytes read from secondary tiers into the primary tier, " "labeled by tier." ), labelnames=(TIER_LABEL,), ) metrics[TieringOffloadingMetrics.READ_TIME] = OffloadingCounterMetadata( documentation=( "Total time spent reading from secondary tiers into the primary " "tier, in seconds, labeled by tier. Tiers that do not report " "per-job transfer time have no series here. Use " "vllm:kv_offload_tiering_promotion_latency_seconds for " "tier-independent promotion timing." ), labelnames=(TIER_LABEL,), ) metrics[TieringOffloadingMetrics.WRITE_BYTES] = OffloadingCounterMetadata( documentation=( "Total bytes written from the primary tier to secondary tiers, " "labeled by tier." ), labelnames=(TIER_LABEL,), ) metrics[TieringOffloadingMetrics.WRITE_TIME] = OffloadingCounterMetadata( documentation=( "Total time spent writing from the primary tier to secondary " "tiers, in seconds, labeled by tier. Tiers that do not report " "per-job transfer time have no series here." ), labelnames=(TIER_LABEL,), ) metrics[TieringOffloadingMetrics.PROMOTION_JOB_FAILURES] = ( OffloadingCounterMetadata( documentation=( "Number of failed secondary-tier promotion jobs, labeled by tier." ), labelnames=(TIER_LABEL,), ) ) metrics[TieringOffloadingMetrics.CASCADE_JOB_FAILURES] = ( OffloadingCounterMetadata( documentation=( "Number of failed secondary-tier cascade jobs, labeled by tier." ), labelnames=(TIER_LABEL,), ) ) metrics[TieringOffloadingMetrics.CHUNK_QUERIES] = OffloadingCounterMetadata( documentation=( "Number of chunk lookup queries sent to a tier, labeled by tier." ), labelnames=(TIER_LABEL,), ) metrics[TieringOffloadingMetrics.CHUNK_HITS] = OffloadingCounterMetadata( documentation="Number of chunk lookup hits in a tier, labeled by tier.", labelnames=(TIER_LABEL,), ) metrics[TieringOffloadingMetrics.PROMOTION_ALLOCATION_FAILURES] = ( OffloadingCounterMetadata( documentation=( "Number of promotion attempts that failed because the " "primary tier could not allocate space." ), ) ) metrics[TieringOffloadingMetrics.PRIMARY_WRITE_USAGE_PERC] = ( OffloadingGaugeMetadata( documentation=( "Current fraction of primary-tier space used by writes from " "secondary tiers, labeled by tier." ), labelnames=(TIER_LABEL,), ) ) metrics[TieringOffloadingMetrics.PRIMARY_READ_USAGE_PERC] = ( OffloadingGaugeMetadata( documentation=( "Current fraction of primary-tier space used by reads to " "secondary tiers, labeled by tier." ), labelnames=(TIER_LABEL,), ) ) metrics[TieringOffloadingMetrics.ACTIVE_PROMOTION_JOBS] = ( OffloadingGaugeMetadata( documentation=( "Number of active secondary-tier promotion jobs, labeled by tier." ), labelnames=(TIER_LABEL,), ) ) metrics[TieringOffloadingMetrics.ACTIVE_CASCADE_JOBS] = OffloadingGaugeMetadata( documentation=( "Number of active secondary-tier cascade jobs, labeled by tier." ), labelnames=(TIER_LABEL,), ) metrics[TieringOffloadingMetrics.PROMOTION_LATENCY] = ( OffloadingHistogramMetadata( documentation=( "Histogram of per-job latency for secondary-tier promotion " "attempts, measured from job creation until the tiering " "manager observes completion so that tier queueing is " "included, labeled by tier and result, in seconds. Includes " "failed attempts." ), labelnames=(TIER_LABEL, "result"), buckets=( 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, ), ) ) for tier_config, tier_cls in cls._get_secondary_tiers(extra_config): metrics.update(tier_cls.build_metric_definitions(tier_config)) metrics[TieringOffloadingMetrics.BACKPRESSURE_STORE_LATENCY_EMA] = ( OffloadingGaugeMetadata( documentation=( "Exponential moving average of store latency " "for back-pressure detection, in s/MiB." ), labelnames=(TIER_LABEL,), ) ) metrics[TieringOffloadingMetrics.BACKPRESSURE_STORES_DROPPED] = ( OffloadingCounterMetadata( documentation=( "Number of store operations dropped due to " "back-pressure on a secondary tier." ), labelnames=(TIER_LABEL,), ) ) metrics[TieringOffloadingMetrics.BACKPRESSURE_BLOCKS_DROPPED] = ( OffloadingCounterMetadata( documentation=( "Number of blocks dropped due to back-pressure on a secondary tier." ), labelnames=(TIER_LABEL,), ) ) return metrics def __init__(self, config: OffloadingConfig): super().__init__(config) # Redeclare for mypy: parent sets this but `--follow-imports skip` hides it self._manager: OffloadingManager | None = None # Parse secondary tier configurations self.secondary_tier_configs = self.extra_config.get("secondary_tiers", []) if not isinstance(self.secondary_tier_configs, list): raise ValueError("secondary_tiers must be a list of tier configurations") # Pause between tier-poll rounds for tiers that need servicing # between engine steps. Zero or negative disables the thread, which # restores per-step servicing. self.tier_poll_interval_s = float( self.extra_config.get("tier_poll_interval_s", DEFAULT_TIER_POLL_INTERVAL_S) ) # Backpressure config is merged field-by-field in priority order # (highest first): # 1. Per-tier ``backpressure`` dict in the tier config # 2. Top-level ``backpressure`` in kv_connector_extra_config # Merging per field (rather than per whole dict) means a partial # tier override still inherits missing fields from the top-level # default, so the resolved dict reaching the factory is complete. # Within each tier's resolved dict, tier-type-aware water marks # are filled in last so a bare ``"backpressure": {}`` picks up # sensible thresholds for the storage medium. bp_defaults = self.extra_config.get("backpressure") for tier_cfg in self.secondary_tier_configs: tier_override = tier_cfg.get("backpressure") # Overlay from lowest to highest precedence so higher-precedence # fields win while lower-precedence ones fill in the gaps. merged: dict[str, Any] = {} for source in (bp_defaults, tier_override): if source: merged.update(source) # Only set a resolved dict when at least one source contributed # (or an explicit ``backpressure`` key was present, e.g. ``{}``), # so tiers without any backpressure config stay unconfigured. if merged or "backpressure" in tier_cfg: tier_cfg["backpressure"] = merged # Scheduler-side mmap (rank=None); kept for cleanup self._scheduler_mmap: SharedOffloadRegion | None = None # Set by create_worker when canonical_layout is enabled: True when # every layer's canonical bytes are parallelism-agnostic (portable), # False when some layers use the opaque fallback (exact-topology only) self.all_layers_portable: bool | None = None # engine_id is unique per DP replica (suffixed with _dp{rank} in both # the Ray and multiprocessing paths), so it names a per-replica offload # region. self._engine_id = config.engine_id @override def get_manager(self) -> OffloadingManager: """Get the TieringOffloadingManager. Creates a TieringOffloadingManager with: - Primary tier: CPU (LRU or ARC) - Secondary tiers: As configured in extra_config Returns: TieringOffloadingManager instance """ if not self._manager: if int(self.extra_config.get("store_threshold", 0)) >= 2: raise ValueError( "store_threshold is not supported for TieringOffloadingSpec" ) scheduler_mmap: SharedOffloadRegion | None = None primary_tier: CPUPrimaryTierOffloadingManager | None = None secondary_tiers = [] try: # Create scheduler-side SharedOffloadRegion (rank=None) so the # primary tier can eagerly create a memoryview over _base. scheduler_mmap = SharedOffloadRegion( engine_id=self._engine_id, num_chunks=self.num_chunks, rank=None, kv_bytes_per_chunk=self.kv_bytes_per_chunk, cpu_page_size=self.cpu_page_size_per_worker, ) self._scheduler_mmap = scheduler_mmap # Create primary tier (CPU-based) primary_tier = CPUPrimaryTierOffloadingManager( num_chunks=self.num_chunks, cache_policy=self.eviction_policy, cache_policy_module_path=self.cache_policy_module_path, enable_events=self.kv_events_config.enable_kv_cache_events, mmap_region=scheduler_mmap, ) # Create secondary tiers primary_kv_view = primary_tier.get_kv_memoryview() for i, tier_config in enumerate(self.secondary_tier_configs): tier = SecondaryTierFactory.create_secondary_tier( tier_config, primary_kv_view, self ) secondary_tiers.append(tier) logger.info( "Created secondary tier #%d (%s)", i, tier.tier_type, ) # Create TieringOffloadingManager. GPU↔CPU transfers use the inherited # get_worker(). Secondary tier transfers are handled by the # secondary tier managers and need no additional workers here. tiering_manager = TieringOffloadingManager( primary_tier=primary_tier, secondary_tiers=secondary_tiers, tier_poll_interval_s=self.tier_poll_interval_s, ) self._manager = tiering_manager except Exception: for tier in reversed(secondary_tiers): try: tier.shutdown() except Exception: logger.exception( "Failed to shut down secondary tier during " "initialization cleanup" ) if primary_tier is not None: try: primary_tier.shutdown() except Exception: logger.exception( "Failed to shut down primary tier during " "initialization cleanup" ) elif scheduler_mmap is not None: try: scheduler_mmap.cleanup() except Exception: logger.exception( "Failed to clean up scheduler mmap during " "initialization cleanup" ) self._scheduler_mmap = None raise logger.info( "Created TieringOffloadingManager with primary tier " "(%s, %s chunks) and %s secondary tier(s)", self.eviction_policy, self.num_chunks, len(secondary_tiers), ) return self._manager @classmethod @override def config_info_keys(cls, extra_config: dict[str, Any]) -> tuple[str, ...]: """Return TIER_LABEL and the label names of all tiers, with no repeat.""" # dict.fromkeys() keeps one copy of a name that two tier types share. keys = dict.fromkeys(super().config_info_keys(extra_config)) keys[TIER_LABEL] = None for tier_config, tier_cls in cls._get_secondary_tiers(extra_config): tier_keys = tier_cls.config_info_keys(tier_config) assert TIER_LABEL not in tier_keys, ( f"{tier_cls.__name__} must not declare the reserved label " f"{TIER_LABEL!r}" ) keys.update(dict.fromkeys(tier_keys)) return tuple(keys) @classmethod def _get_secondary_tiers( cls, extra_config: dict[str, Any] ) -> list[tuple[dict[str, Any], type[SecondaryTierManager]]]: """Resolve every configured secondary tier to its manager class. Args: extra_config: kv_connector_extra_config of this instance. Returns: One (tier config, tier class) pair for each secondary tier, in configuration order. That order sets the tier index. Raises: ValueError: If secondary_tiers is not a list. """ secondary_tier_configs = extra_config.get("secondary_tiers", []) if not isinstance(secondary_tier_configs, list): raise ValueError("secondary_tiers must be a list of tier configurations") tiers = [] for tier_config in secondary_tier_configs: assert isinstance(tier_config, dict) tier_cls = SecondaryTierFactory.get_tier_class(tier_config) tiers.append((tier_config, tier_cls)) return tiers @override def _uses_shared_region(self) -> bool: # Tiering always allocates on the shared region (every platform), so the # replicated-layout gate must not be narrowed by the CPU spec's # CUDA-alike check. return True @override def create_worker(self, kv_caches: CanonicalKVCaches) -> CPUOffloadingWorker: rank = 0 if self.replicated_layout else self.config.parallel.rank worker_mmap = SharedOffloadRegion( engine_id=self._engine_id, num_chunks=self.num_chunks, rank=rank, kv_bytes_per_chunk=self.kv_bytes_per_chunk, cpu_page_size=self.cpu_page_size_per_worker, ) try: if self.config.canonical_layout: self._validate_canonical_refs(kv_caches) return CPUOffloadingWorker( kv_caches=kv_caches, blocks_per_chunk=self.blocks_per_chunk, num_cpu_chunks=self.num_chunks, mmap_region=worker_mmap, canonical_layout=self.config.canonical_layout, ) except Exception: worker_mmap.cleanup() raise def _validate_canonical_refs(self, kv_caches: CanonicalKVCaches) -> None: """Require a mapping on every ref and record layer portability. Fails loudly rather than persist direct-layout bytes under a canonical format identity.""" all_refs = [ ref for group_refs in kv_caches.group_data_refs for ref in group_refs ] if any(ref.mapping is None for ref in all_refs): raise RuntimeError( "canonical_layout was requested but the KV cache layout " "could not be certified for canonical offload (offload " "workers must be exactly the TP group, and packed / " "cross-layer KV layouts are not supported). Remove " "canonical_layout from kv_connector_extra_config." ) self.all_layers_portable = all( ref.mapping is not None and ref.mapping.parallelism_agnostic for ref in all_refs ) if self.config.parallel.is_parallelism_agnostic and not ( self.all_layers_portable ): # The scheduler-side storage namespace was already collapsed on # the static portability claim; opaque per-topology bytes must # not land in it. raise RuntimeError( "canonical_layout could not certify every layer as " "parallelism-agnostic, but the storage namespace is shared " "across topologies. Remove canonical_layout from " "kv_connector_extra_config or disable parallel-agnostic " "secondary tiers." ) logger.info( "Canonical KV layout enabled (all_layers_portable=%s)", self.all_layers_portable, )