[KV Offloading] Back-pressure detection and remediation (#50045)

Signed-off-by: Bill Nell <bnell@redhat.com>
Signed-off-by: bnellnm <49004751+bnellnm@users.noreply.github.com>
Co-authored-by: Itay Etelis <92247226+Etelis@users.noreply.github.com>
This commit is contained in:
bnellnm
2026-09-18 14:47:48 -04:00
committed by GitHub
co-authored by Itay Etelis
parent 71fc70d3ae
commit 23e26e0588
13 changed files with 1464 additions and 5 deletions
+384
View File
@@ -0,0 +1,384 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import time
from abc import ABC, abstractmethod
class BackpressurePolicy(ABC):
"""Decides what action to take when a detector signals pressure."""
@abstractmethod
def should_store(self, detector: BackpressureDetector) -> bool: ...
@abstractmethod
def on_store_skipped(self, num_blocks: int) -> None: ...
@abstractmethod
def pop_stores_dropped(self) -> tuple[int, int]:
"""Return and reset (stores_dropped, blocks_dropped)."""
...
@abstractmethod
def reset(self) -> None: ...
class DropAccountingPolicy(BackpressurePolicy):
"""Base policy that tracks dropped store/block counters.
Subclasses implement ``should_store``; the shared drop accounting
(``on_store_skipped``/``pop_stores_dropped``/``reset``) lives here so
every policy reports these counters identically. Subclasses that add
their own state override ``reset`` and call ``super().reset()``.
"""
def __init__(self):
self._stores_dropped: int = 0
self._blocks_dropped: int = 0
def on_store_skipped(self, num_blocks: int) -> None:
self._stores_dropped += 1
self._blocks_dropped += num_blocks
def pop_stores_dropped(self) -> tuple[int, int]:
stores, blocks = self._stores_dropped, self._blocks_dropped
self._stores_dropped = 0
self._blocks_dropped = 0
return stores, blocks
def reset(self) -> None:
self._stores_dropped = 0
self._blocks_dropped = 0
class DropStorePolicy(DropAccountingPolicy):
"""Silently drop stores to pressured tiers."""
def should_store(self, detector) -> bool:
return not detector.is_under_pressure()
class ThrottledDropPolicy(DropAccountingPolicy):
"""Drop stores proportionally to pressure severity.
Instead of dropping all stores when pressure is detected, the drop
rate scales with how far the EMA exceeds the high watermark::
overshoot = (ema - high_water) / high_water
drop_rate = min(1.0, overshoot / ramp_factor)
With the default ``ramp_factor=2.0``, stores ramp from 0% drop at
the high watermark to 100% drop at 3x the high watermark. This
keeps secondary tiers populated under moderate pressure, reducing
cache misses (and therefore recomputation / ITL regression) while
still protecting TTFT under severe congestion.
"""
def __init__(self, ramp_factor: float = 2.0):
super().__init__()
self._call_count: int = 0
self._ramp_factor = ramp_factor
def should_store(self, detector: BackpressureDetector) -> bool:
# Proportional throttling needs the EMA and high watermark, which
# only the EMA detector exposes.
assert isinstance(detector, EMABackpressureDetector), (
"ThrottledDropPolicy requires an EMABackpressureDetector"
)
if not detector.is_under_pressure():
return True
ema = detector.store_latency_ema
high = detector._high
if high <= 0:
return False
overshoot = max(0.0, ema - high) / high
drop_rate = min(1.0, overshoot / self._ramp_factor)
if drop_rate <= 0:
return True
if drop_rate >= 1.0:
return False
self._call_count += 1
period = max(2, round(1.0 / drop_rate))
return (self._call_count % period) != 0
def reset(self) -> None:
super().reset()
self._call_count = 0
class BackpressureDetector(ABC):
"""Observes store completion signals and determines pressure state."""
def __init__(
self,
policy: BackpressurePolicy | None = None,
):
self._policy = policy or ThrottledDropPolicy()
@property
def policy(self) -> BackpressurePolicy:
return self._policy
@abstractmethod
def on_store_completed(self, elapsed_s: float, num_bytes: int) -> None: ...
@abstractmethod
def is_under_pressure(self) -> bool: ...
@abstractmethod
def reset(self) -> None: ...
def update(self, submit_time: float, num_bytes: int) -> None:
"""Update pressure state from a completed store job.
Args:
submit_time: ``time.monotonic()`` when the job was submitted.
num_bytes: Total bytes written (num_blocks * block_size_bytes).
"""
if num_bytes <= 0:
return
elapsed = time.monotonic() - submit_time
self.on_store_completed(elapsed, num_bytes)
def should_store(self, num_blocks: int) -> bool:
"""Check policy and record skip if rejected."""
if not self._policy.should_store(self):
self._policy.on_store_skipped(num_blocks)
return False
return True
@classmethod
def default_config(
cls, tier_type: str, *, locality: str | None = None
) -> dict | None:
"""Return default constructor kwargs for ``tier_type``, or None.
Args:
tier_type: The tier type string (e.g. ``"fs"``, ``"obj"``).
locality: Optional locality hint from the tier config
(``"LOCAL"`` or ``"REMOTE"``). When ``"REMOTE"``, tiers
that would normally get local-storage watermarks (e.g.
``"fs"``) receive network watermarks instead.
"""
return None
@property
def stats(self) -> dict[str, float]:
return {}
class EMABackpressureDetector(BackpressureDetector):
"""EMA of store latency normalized by transfer size.
The EMA tracks seconds per megabyte (s/MiB) so that the metric is
comparable regardless of how many blocks are in a job or how large
each block is. Water marks are in the same unit.
Default water marks are derived from fio benchmarks on the WDC H100
cluster. Two presets are provided:
LOCAL (NVMe/SSD): NVMe sustains ~5 GB/s writes.
high=0.005 s/MiB (~200 MB/s) catches severe congestion;
low=0.001 s/MiB (~1 GB/s) requires meaningful recovery.
NETWORK (CephFS, object store, ``obj``tiers, or any tier
with ``"locality": "REMOTE"``): CephFS sustains ~1.5 GB/s.
high=0.020 s/MiB (~50 MB/s); low=0.005 s/MiB (~200 MB/s).
``obj`` tiers get NETWORK defaults automatically. An
``fs`` tier defaults to LOCAL (appropriate for NVMe/SSD); set
``"locality": "REMOTE"`` in the tier config for network-backed
filesystems like CephFS.
"""
_MIB = 1 << 20
# EMA smoothing factor: higher values (→1) react faster to latency
# spikes but are noisier; lower values (→0) smooth more but lag.
DEFAULT_ALPHA = 0.15
# Number of store completions to collect before seeding the EMA.
# During warmup, pressure is never signalled; the EMA is initialized
# to the mean of the warmup samples to avoid cold-start false positives.
DEFAULT_WARMUP_COMPLETIONS = 10
# When under pressure and no completions arrive, the EMA decays
# toward zero so the detector can eventually recover. The half-life
# controls how quickly: after this many seconds of silence the EMA
# halves. With a 2 s half-life and high_water=0.005, an EMA frozen
# at 0.010 reaches low_water=0.001 in ~6.6 s.
DEFAULT_DECAY_HALF_LIFE_S = 2.0
# After pressure clears, stores are throttled to 50% for this many
# seconds to prevent a burst of stores from re-overwhelming the tier.
DEFAULT_COOLDOWN_S = 1.0
# Number of consecutive sub-low-watermark completions before the
# detector enters a "healthy" fast-path that bypasses policy checks.
_HEALTHY_THRESHOLD = 10
LOCAL_HIGH_WATER_S = 0.010
LOCAL_LOW_WATER_S = 0.002
NETWORK_HIGH_WATER_S = 0.050
NETWORK_LOW_WATER_S = 0.010
_NETWORK_TIER_TYPES = frozenset({"obj"})
@classmethod
def default_config(
cls, tier_type: str, *, locality: str | None = None
) -> dict | None:
is_remote = locality is not None and locality.upper() == "REMOTE"
if is_remote or tier_type in cls._NETWORK_TIER_TYPES:
return {
"high_water_s": cls.NETWORK_HIGH_WATER_S,
"low_water_s": cls.NETWORK_LOW_WATER_S,
}
return {
"high_water_s": cls.LOCAL_HIGH_WATER_S,
"low_water_s": cls.LOCAL_LOW_WATER_S,
}
def __init__(
self,
high_water_s: float,
low_water_s: float,
alpha: float = DEFAULT_ALPHA,
warmup_completions: int = DEFAULT_WARMUP_COMPLETIONS,
decay_half_life_s: float = DEFAULT_DECAY_HALF_LIFE_S,
cooldown_s: float = DEFAULT_COOLDOWN_S,
policy: BackpressurePolicy | None = None,
):
super().__init__(policy=policy)
if not (0 < alpha <= 1):
raise ValueError(f"alpha must be in (0, 1], got {alpha}")
if low_water_s > high_water_s:
raise ValueError(
f"low_water_s ({low_water_s}) must be <= high_water_s ({high_water_s})"
)
self._alpha = alpha
self._high = high_water_s
self._low = low_water_s
self._warmup_completions = warmup_completions
self._decay_half_life_s = decay_half_life_s
self._cooldown_s = cooldown_s
# Exponential moving average of store latency in seconds.
self._ema: float = 0.0
# Whether the tier is currently considered under pressure.
self._under_pressure: bool = False
# Number of store completions observed (for warmup gating).
self._completions: int = 0
# Samples collected during warmup; used to seed the EMA.
self._warmup_samples: list[float] = []
# Monotonic timestamp of the last EMA update; used for idle decay.
self._last_update: float = 0.0
# Consecutive completions with EMA below low watermark.
self._healthy_streak: int = 0
# Monotonic timestamp when pressure was last cleared (for cooldown).
self._pressure_cleared_at: float = 0.0
self._cooldown_count: int = 0
def _apply_idle_decay(self) -> None:
"""Decay the EMA toward zero based on elapsed idle time.
Uses the formula ``ema * 0.5^(dt / half_life)`` so the EMA
halves every ``decay_half_life_s`` seconds of silence. This
lets the detector recover from pressure when no stores are
attempted (and therefore no completions arrive to update the
EMA organically).
"""
now = time.monotonic()
dt = now - self._last_update
if dt <= 0 or self._decay_half_life_s <= 0:
return
self._ema *= 0.5 ** (dt / self._decay_half_life_s)
self._last_update = now
def on_store_completed(self, elapsed_s: float, num_bytes: int) -> None:
now = time.monotonic()
s_per_mib = elapsed_s / (num_bytes / self._MIB)
self._completions += 1
if self._completions <= self._warmup_completions:
self._warmup_samples.append(s_per_mib)
if self._completions == self._warmup_completions:
self._ema = sum(self._warmup_samples) / len(self._warmup_samples)
self._last_update = now
return
self._ema = self._alpha * s_per_mib + (1 - self._alpha) * self._ema
self._last_update = now
if self._ema > self._high:
self._under_pressure = True
self._pressure_cleared_at = 0.0
self._healthy_streak = 0
elif self._ema < self._low:
if self._under_pressure:
self._pressure_cleared_at = now
self._cooldown_count = 0
self._under_pressure = False
self._healthy_streak += 1
else:
self._healthy_streak = 0
def is_under_pressure(self) -> bool:
if self._under_pressure and self._completions >= self._warmup_completions:
self._apply_idle_decay()
if self._ema < self._low:
self._under_pressure = False
self._pressure_cleared_at = time.monotonic()
self._cooldown_count = 0
return self._under_pressure
def is_healthy(self) -> bool:
"""True when the tier has been consistently fast.
After ``_HEALTHY_THRESHOLD`` consecutive store completions with
EMA below the low watermark (and no active pressure), the
detector enters a healthy fast-path: ``should_store()`` returns
``True`` immediately, skipping the policy check entirely.
"""
return (
not self._under_pressure and self._healthy_streak >= self._HEALTHY_THRESHOLD
)
def should_store(self, num_blocks: int) -> bool:
under_pressure = self.is_under_pressure()
if self.is_healthy():
return True
if (
not under_pressure
and self._pressure_cleared_at > 0
and time.monotonic() - self._pressure_cleared_at < self._cooldown_s
):
self._cooldown_count += 1
if self._cooldown_count % 2 == 0:
self._policy.on_store_skipped(num_blocks)
return False
return super().should_store(num_blocks)
def reset(self) -> None:
self._ema = 0.0
self._under_pressure = False
self._completions = 0
self._warmup_samples.clear()
self._last_update = 0.0
self._healthy_streak = 0
self._pressure_cleared_at = 0.0
self._cooldown_count = 0
self._policy.reset()
@property
def store_latency_ema(self) -> float:
return self._ema
@store_latency_ema.setter
def store_latency_ema(self, value: float) -> None:
self._ema = value
@property
def stats(self) -> dict[str, float]:
return {"store_latency_ema": self._ema}
+27 -1
View File
@@ -2,9 +2,10 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Abstract interfaces and data types for the secondary tiering layer."""
import time
from abc import ABC, abstractmethod
from collections.abc import Collection, Iterable
from dataclasses import dataclass
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, ClassVar
import numpy as np
@@ -20,6 +21,9 @@ from vllm.v1.kv_offload.base import (
RequestOffloadingContext,
ScheduleEndContext,
)
from vllm.v1.kv_offload.tiering.backpressure import (
BackpressureDetector,
)
if TYPE_CHECKING:
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
@@ -52,6 +56,11 @@ class TieringOffloadingMetrics:
)
ACTIVE_PROMOTION_JOBS = "vllm:kv_offload_tiering_active_promotion_jobs"
ACTIVE_CASCADE_JOBS = "vllm:kv_offload_tiering_active_cascade_jobs"
BACKPRESSURE_STORE_LATENCY_EMA = (
"vllm:kv_offload_tiering_backpressure_store_latency_ema"
)
BACKPRESSURE_STORES_DROPPED = "vllm:kv_offload_tiering_backpressure_stores_dropped"
BACKPRESSURE_BLOCKS_DROPPED = "vllm:kv_offload_tiering_backpressure_blocks_dropped"
@dataclass
@@ -63,6 +72,7 @@ class TransferJob:
chunk_ids: np.ndarray
is_promotion: bool
req_context: ReqContext
submit_time: float = field(default_factory=time.monotonic)
@dataclass
@@ -77,6 +87,7 @@ class JobResult:
# indicated by `success`. Must be a subset of the job's original keys.
successful_keys: Collection[OffloadKey] | None = None
transfer_time: float | None = None
transfer_bytes: int | None = None
class ParentManager(ABC):
@@ -136,18 +147,33 @@ class SecondaryTierManager(ABC):
offloading_spec: "OffloadingSpec",
primary_kv_view: memoryview,
tier_type: str,
backpressure_detector: BackpressureDetector | None = None,
) -> None:
"""Args:
offloading_spec: Offloading configuration.
primary_kv_view: Memoryview of the primary tier's CPU KV cache.
tier_type: Tier type identifier, set by SecondaryTierFactory
from the registered tier type.
backpressure_detector: Optional `BackpressureDetector`.
"""
self._offloading_spec = offloading_spec
self._primary_kv_view: memoryview = primary_kv_view
assert primary_kv_view.strides is not None, (
"primary_kv_view.strides cannot be None"
)
self._block_size_bytes: int = primary_kv_view.strides[0]
self.tier_type = tier_type
self.locality: Locality | None = None
self._bp_detector = backpressure_detector
@property
def block_size_bytes(self) -> int:
return self._block_size_bytes
@property
def bp_detector(self) -> BackpressureDetector | None:
return self._bp_detector
@abstractmethod
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
@@ -31,6 +31,7 @@ logger = logging.getLogger(__name__)
if TYPE_CHECKING:
from vllm.v1.kv_offload.base import OffloadingSpec
from vllm.v1.kv_offload.tiering.backpressure import BackpressureDetector
class ExampleSecondaryTierManager(SecondaryTierManager):
@@ -49,6 +50,7 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
primary_kv_view: memoryview,
tier_type: str,
custom_param: int = 0,
backpressure_detector: "BackpressureDetector | None" = None,
):
"""Initialize the example secondary tier.
@@ -57,12 +59,14 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
primary_kv_view: Memoryview over the primary tier's KV buffer.
tier_type: Name identifying this tier type.
custom_param: Dummy parameter demonstrating custom args.
backpressure_detector: Optional backpressure detector.
"""
super().__init__(
offloading_spec=offloading_spec,
primary_kv_view=primary_kv_view,
tier_type=tier_type,
backpressure_detector=backpressure_detector,
)
logger.info(
+42
View File
@@ -8,11 +8,33 @@ from vllm.logger import init_logger
from vllm.v1.kv_offload.tiering.base import SecondaryTierManager
if TYPE_CHECKING:
from vllm.v1.kv_offload.backpressure import BackpressureDetector
from vllm.v1.kv_offload.base import OffloadingSpec
logger = init_logger(__name__)
_DEFAULT_PACKAGE = "vllm.v1.kv_offload.tiering.backpressure"
def _import_class(name: str) -> type:
"""Import a class by name.
If ``name`` contains a dot it is treated as a fully qualified path
(e.g. ``'my_package.MyDetector'``). Otherwise it is looked up in
the default backpressure module
(``vllm.v1.kv_offload.tiering.backpressure``), so short names like
``'DropStorePolicy'`` or ``'EMABackpressureDetector'`` work out of
the box.
"""
if "." in name:
module_path, _, class_name = name.rpartition(".")
else:
module_path, class_name = _DEFAULT_PACKAGE, name
module = importlib.import_module(module_path)
return getattr(module, class_name)
class SecondaryTierFactory:
"""Registry for SecondaryTierManager implementations, resolved by type.
@@ -47,10 +69,30 @@ class SecondaryTierFactory:
config = tier_config.copy()
tier_type = config.pop("type")
config.pop("module_path", None)
locality = config.get("locality")
bp_config = config.pop("backpressure", None)
bp_detector = None
if bp_config is not None:
bp_config = bp_config.copy()
bp_cls_name = bp_config.pop("backpressure_cls", None)
if bp_cls_name is None:
raise ValueError(
"backpressure config must include 'backpressure_cls' "
"(e.g. 'EMABackpressureDetector')"
)
policy_cls_name = bp_config.pop("policy_cls", None)
policy = _import_class(policy_cls_name)() if policy_cls_name else None
detector_cls: type[BackpressureDetector] = _import_class(bp_cls_name)
defaults = detector_cls.default_config(tier_type, locality=locality)
if defaults:
for k, v in defaults.items():
bp_config.setdefault(k, v)
bp_detector = detector_cls(policy=policy, **bp_config)
return tier_cls(
offloading_spec=offloading_spec,
primary_kv_view=primary_kv_view,
tier_type=tier_type,
backpressure_detector=bp_detector,
**config,
)
+14 -1
View File
@@ -40,6 +40,7 @@ from vllm.v1.kv_offload.base import (
)
from vllm.v1.kv_offload.file_mapper import FileMapper
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager
from vllm.v1.kv_offload.tiering.backpressure import BackpressureDetector
from vllm.v1.kv_offload.tiering.base import (
JobId,
JobResult,
@@ -115,6 +116,7 @@ class FileSystemTierManager(SecondaryTierManager):
n_write_threads: int = 16,
enable_kv_events: bool = False,
locality: str | None = None,
backpressure_detector: BackpressureDetector | None = None,
):
"""Args:
offloading_spec: Contains normalized offloading configuration and
@@ -129,9 +131,12 @@ class FileSystemTierManager(SecondaryTierManager):
cache events are enabled globally (kv_events_config).
locality: Whether this tier's storage is LOCAL or REMOTE relative
to the publishing vLLM instance.
backpressure_detector: Optional backpressure detector.
"""
super().__init__(offloading_spec, primary_kv_view, tier_type)
super().__init__(
offloading_spec, primary_kv_view, tier_type, backpressure_detector
)
self.locality = Locality(locality) if locality is not None else None
self.events: list[OffloadingEvent] | None = None
@@ -150,6 +155,8 @@ class FileSystemTierManager(SecondaryTierManager):
# Keys of in-flight load (promotion) jobs, so a failed load can mark
# its own cached lookup verdicts False (see get_finished_jobs).
self._load_job_keys: dict[JobId, list[OffloadKey]] = {}
# Block count per in-flight job, used to report transfer_bytes.
self._job_block_counts: dict[JobId, int] = {}
# Per load job: how many blocks loaded before a failure (partial keep).
# Written by the pool worker inside the load task before it raises (so
# before task_done publishes the job); read on the scheduler thread in
@@ -225,6 +232,7 @@ class FileSystemTierManager(SecondaryTierManager):
self._block_size,
self._use_o_direct,
)
self._job_block_counts[job_metadata.job_id] = len(keys)
self._pool.enqueue_store(job_metadata.job_id, 1, [task])
@override
@@ -234,6 +242,7 @@ class FileSystemTierManager(SecondaryTierManager):
# keys as a miss (see get_finished_jobs).
keys = list(job_metadata.keys)
self._load_job_keys[job_id] = keys
self._job_block_counts[job_id] = len(keys)
paths = [self.file_mapper.get_file_name(key) for key in keys]
offsets = [int(cid) * self._block_size for cid in job_metadata.chunk_ids]
@@ -272,6 +281,8 @@ class FileSystemTierManager(SecondaryTierManager):
as a miss here (scheduler thread)."""
results = []
for job_id, success, transfer_time in self._pool.get_finished():
block_count = self._job_block_counts.pop(job_id, 0)
transfer_bytes = block_count * self._block_size if block_count else None
if self.events is not None:
keys = self._store_job_keys.pop(job_id, None)
if success and keys:
@@ -299,6 +310,7 @@ class FileSystemTierManager(SecondaryTierManager):
success=False,
successful_keys=tuple(successful) if successful else None,
transfer_time=transfer_time,
transfer_bytes=transfer_bytes,
)
)
continue
@@ -307,6 +319,7 @@ class FileSystemTierManager(SecondaryTierManager):
job_id=job_id,
success=success,
transfer_time=transfer_time,
transfer_bytes=transfer_bytes,
)
)
return results
+51 -1
View File
@@ -238,6 +238,10 @@ class TieringOffloadingManager(OffloadingManager):
for tier_idx, tier in enumerate(self.secondary_tiers)
}
self._tier_index: dict[SecondaryTierManager, int] = {
tier: i for i, tier in enumerate(self.secondary_tiers)
}
@property
def _transfer_jobs(self) -> dict[JobId, JobMetadata]:
return self._jobs
@@ -337,6 +341,43 @@ class TieringOffloadingManager(OffloadingManager):
self.primary_tier.complete_read(
transfer_job.keys, transfer_job.req_context
)
if completed_job.success:
self._update_backpressure(tier, job_metadata, completed_job)
def _should_store_to_tier(
self, tier: SecondaryTierManager, num_blocks: int
) -> bool:
detector = tier.bp_detector
if detector is None:
return True
return detector.should_store(num_blocks)
def _update_backpressure(
self,
tier: SecondaryTierManager,
job_metadata: JobMetadata,
completed_job: JobResult,
) -> None:
detector = tier.bp_detector
if detector is None:
return
was_under_pressure = detector.is_under_pressure()
tj = job_metadata.transfer_job
num_bytes = (
completed_job.transfer_bytes
if completed_job.transfer_bytes is not None
else len(tj.keys) * tier.block_size_bytes
)
detector.update(tj.submit_time, num_bytes)
if detector.is_under_pressure() != was_under_pressure:
tier_idx = self._tier_index[tier]
logger.info(
"Tier #%d (%s) back-pressure %s (stats=%s)",
tier_idx,
tier.tier_type,
"activated" if detector.is_under_pressure() else "cleared",
detector.stats,
)
@override
def lookup(
@@ -643,8 +684,10 @@ class TieringOffloadingManager(OffloadingManager):
return
for tier_idx in request_level_tiers:
job_metadata = self.create_store_job(ready_keys, req_context, tier_idx)
tier = self.secondary_tiers[tier_idx]
if not self._should_store_to_tier(tier, len(ready_keys)):
continue
job_metadata = self.create_store_job(ready_keys, req_context, tier_idx)
tier.submit_store(job_metadata)
def _flush_pending_cascades(self) -> None:
@@ -698,6 +741,8 @@ class TieringOffloadingManager(OffloadingManager):
# eviction during the async transfer). One prepare_read() call per
# secondary tier.
for tier_idx, tier in enumerate(self.secondary_tiers):
if not self._should_store_to_tier(tier, len(keys)):
continue
job_metadata = self.create_store_job(keys, req_context, tier_idx)
tier.submit_store(job_metadata)
@@ -904,6 +949,10 @@ class TieringOffloadingManager(OffloadingManager):
del self._req_state[req_id]
self._processed_jobs_this_step = False
for tier in self.secondary_tiers:
if tier.bp_detector is not None:
tier.bp_detector.reset()
@override
def get_stats(self) -> OffloadingConnectorStats | None:
stats = self.primary_tier.get_stats()
@@ -911,6 +960,7 @@ class TieringOffloadingManager(OffloadingManager):
if stats is not None and stats.is_empty():
stats = None
self._metrics.record_backpressure(self.secondary_tiers)
metrics_stats = self._metrics.take_stats()
if metrics_stats is not None:
if stats is None:
+31
View File
@@ -12,6 +12,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
from vllm.v1.kv_offload.base import LookupResult, OffloadKey, ReqContext
from vllm.v1.kv_offload.tiering.base import (
JobResult,
SecondaryTierManager,
TieringOffloadingMetrics,
TransferJob,
)
@@ -130,6 +131,36 @@ class TieringMetricsTracker:
TieringOffloadingMetrics.PROMOTION_ALLOCATION_FAILURES
)
def record_backpressure(
self,
tiers: list[SecondaryTierManager],
) -> None:
for i, tier in enumerate(tiers):
detector = tier.bp_detector
if detector is None:
continue
label = self.tier_label(i)
ema = detector.stats.get("store_latency_ema")
if ema is not None:
self._stats.set_gauge(
TieringOffloadingMetrics.BACKPRESSURE_STORE_LATENCY_EMA,
ema,
labelvalues=label,
)
stores_dropped, blocks_dropped = detector.policy.pop_stores_dropped()
if stores_dropped > 0:
self._stats.increase_counter(
TieringOffloadingMetrics.BACKPRESSURE_STORES_DROPPED,
stores_dropped,
labelvalues=label,
)
if blocks_dropped > 0:
self._stats.increase_counter(
TieringOffloadingMetrics.BACKPRESSURE_BLOCKS_DROPPED,
blocks_dropped,
labelvalues=label,
)
def take_stats(self) -> OffloadingConnectorStats | None:
active_transfer_stats = OffloadingConnectorStats()
self._observe_active_transfer_stats(active_transfer_stats)
+9 -1
View File
@@ -22,6 +22,7 @@ from vllm.v1.kv_offload.base import (
)
from vllm.v1.kv_offload.file_mapper import FileMapper
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager
from vllm.v1.kv_offload.tiering.backpressure import BackpressureDetector
from vllm.v1.kv_offload.tiering.base import (
JobId,
JobResult,
@@ -112,6 +113,7 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
io_threads: int = 4,
enable_kv_events: bool = False,
locality: str | None = None,
backpressure_detector: BackpressureDetector | None = None,
):
"""Args:
offloading_spec: Offloading configuration.
@@ -125,9 +127,15 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
cache events are enabled globally (kv_events_config).
locality: Whether this tier's storage is LOCAL or REMOTE relative
to the publishing vLLM instance.
backpressure_detector: Optional backpressure detector.
"""
super().__init__(offloading_spec, primary_kv_view, tier_type)
super().__init__(
offloading_spec,
primary_kv_view,
tier_type,
backpressure_detector,
)
self.locality = Locality(locality) if locality is not None else None
self.events: list[OffloadingEvent] | None = None
+19 -1
View File
@@ -265,7 +265,25 @@ class P2PSecondaryTierManager(SecondaryTierManager):
number, or anything convertible to one.
"""
super().__init__(offloading_spec, primary_kv_view, tier_type)
backpressure_detector = kwargs.pop("backpressure_detector", None)
if backpressure_detector is not None:
# The generic (store-latency) detector is unreliable for P2P: in
# PD mode dropping a store can leave the decoder waiting until the
# load timeout instead of failing fast, and rendezvous time makes
# store latency a poor pressure signal. Reject it until a
# P2P-specific fail-fast path exists.
raise ValueError(
"Backpressure is not supported for the P2P secondary tier. "
"The generic store-latency detector cannot fail fast in PD "
"mode and rendezvous time makes store latency an unreliable "
"pressure signal. Remove the 'backpressure' config from the "
"p2p tier."
)
super().__init__(
offloading_spec,
primary_kv_view,
tier_type,
)
try:
timeout_s = float(unbound_store_timeout_s)
except (TypeError, ValueError):
+54
View File
@@ -247,6 +247,34 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
assert isinstance(tier_config, dict)
tier_cls = SecondaryTierFactory.get_tier_class(tier_config)
metrics.update(tier_cls.build_metric_definitions(tier_config))
metrics[TieringOffloadingMetrics.BACKPRESSURE_STORE_LATENCY_EMA] = (
OffloadingGaugeMetadata(
documentation=(
"Exponential moving average of store latency "
"for back-pressure detection, in s/MiB."
),
labelnames=("tier",),
)
)
metrics[TieringOffloadingMetrics.BACKPRESSURE_STORES_DROPPED] = (
OffloadingCounterMetadata(
documentation=(
"Number of store operations dropped due to "
"back-pressure on a secondary tier."
),
labelnames=("tier",),
)
)
metrics[TieringOffloadingMetrics.BACKPRESSURE_BLOCKS_DROPPED] = (
OffloadingCounterMetadata(
documentation=(
"Number of blocks dropped due to back-pressure on a secondary tier."
),
labelnames=("tier",),
)
)
return metrics
def __init__(self, config: OffloadingConfig):
@@ -259,6 +287,32 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
if not isinstance(self.secondary_tier_configs, list):
raise ValueError("secondary_tiers must be a list of tier configurations")
# Backpressure config is merged field-by-field in priority order
# (highest first):
# 1. Per-tier ``backpressure`` dict in the tier config
# 2. Top-level ``backpressure`` in kv_connector_extra_config
# Merging per field (rather than per whole dict) means a partial
# tier override still inherits missing fields from the top-level
# default, so the resolved dict reaching the factory is complete.
# Within each tier's resolved dict, tier-type-aware water marks
# are filled in last so a bare ``"backpressure": {}`` picks up
# sensible thresholds for the storage medium.
bp_defaults = self.extra_config.get("backpressure")
for tier_cfg in self.secondary_tier_configs:
tier_override = tier_cfg.get("backpressure")
# Overlay from lowest to highest precedence so higher-precedence
# fields win while lower-precedence ones fill in the gaps.
merged: dict[str, Any] = {}
for source in (bp_defaults, tier_override):
if source:
merged.update(source)
# Only set a resolved dict when at least one source contributed
# (or an explicit ``backpressure`` key was present, e.g. ``{}``),
# so tiers without any backpressure config stay unconfigured.
if merged or "backpressure" in tier_cfg:
tier_cfg["backpressure"] = merged
# Scheduler-side mmap (rank=None); kept for cleanup
self._scheduler_mmap: SharedOffloadRegion | None = None