[KV Offloading] Back-pressure detection and remediation (#50045)
Signed-off-by: Bill Nell <bnell@redhat.com> Signed-off-by: bnellnm <49004751+bnellnm@users.noreply.github.com> Co-authored-by: Itay Etelis <92247226+Etelis@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,384 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
|
||||
class BackpressurePolicy(ABC):
|
||||
"""Decides what action to take when a detector signals pressure."""
|
||||
|
||||
@abstractmethod
|
||||
def should_store(self, detector: BackpressureDetector) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
def on_store_skipped(self, num_blocks: int) -> None: ...
|
||||
|
||||
@abstractmethod
|
||||
def pop_stores_dropped(self) -> tuple[int, int]:
|
||||
"""Return and reset (stores_dropped, blocks_dropped)."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def reset(self) -> None: ...
|
||||
|
||||
|
||||
class DropAccountingPolicy(BackpressurePolicy):
|
||||
"""Base policy that tracks dropped store/block counters.
|
||||
|
||||
Subclasses implement ``should_store``; the shared drop accounting
|
||||
(``on_store_skipped``/``pop_stores_dropped``/``reset``) lives here so
|
||||
every policy reports these counters identically. Subclasses that add
|
||||
their own state override ``reset`` and call ``super().reset()``.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._stores_dropped: int = 0
|
||||
self._blocks_dropped: int = 0
|
||||
|
||||
def on_store_skipped(self, num_blocks: int) -> None:
|
||||
self._stores_dropped += 1
|
||||
self._blocks_dropped += num_blocks
|
||||
|
||||
def pop_stores_dropped(self) -> tuple[int, int]:
|
||||
stores, blocks = self._stores_dropped, self._blocks_dropped
|
||||
self._stores_dropped = 0
|
||||
self._blocks_dropped = 0
|
||||
return stores, blocks
|
||||
|
||||
def reset(self) -> None:
|
||||
self._stores_dropped = 0
|
||||
self._blocks_dropped = 0
|
||||
|
||||
|
||||
class DropStorePolicy(DropAccountingPolicy):
|
||||
"""Silently drop stores to pressured tiers."""
|
||||
|
||||
def should_store(self, detector) -> bool:
|
||||
return not detector.is_under_pressure()
|
||||
|
||||
|
||||
class ThrottledDropPolicy(DropAccountingPolicy):
|
||||
"""Drop stores proportionally to pressure severity.
|
||||
|
||||
Instead of dropping all stores when pressure is detected, the drop
|
||||
rate scales with how far the EMA exceeds the high watermark::
|
||||
|
||||
overshoot = (ema - high_water) / high_water
|
||||
drop_rate = min(1.0, overshoot / ramp_factor)
|
||||
|
||||
With the default ``ramp_factor=2.0``, stores ramp from 0% drop at
|
||||
the high watermark to 100% drop at 3x the high watermark. This
|
||||
keeps secondary tiers populated under moderate pressure, reducing
|
||||
cache misses (and therefore recomputation / ITL regression) while
|
||||
still protecting TTFT under severe congestion.
|
||||
"""
|
||||
|
||||
def __init__(self, ramp_factor: float = 2.0):
|
||||
super().__init__()
|
||||
self._call_count: int = 0
|
||||
self._ramp_factor = ramp_factor
|
||||
|
||||
def should_store(self, detector: BackpressureDetector) -> bool:
|
||||
# Proportional throttling needs the EMA and high watermark, which
|
||||
# only the EMA detector exposes.
|
||||
assert isinstance(detector, EMABackpressureDetector), (
|
||||
"ThrottledDropPolicy requires an EMABackpressureDetector"
|
||||
)
|
||||
if not detector.is_under_pressure():
|
||||
return True
|
||||
ema = detector.store_latency_ema
|
||||
high = detector._high
|
||||
if high <= 0:
|
||||
return False
|
||||
overshoot = max(0.0, ema - high) / high
|
||||
drop_rate = min(1.0, overshoot / self._ramp_factor)
|
||||
if drop_rate <= 0:
|
||||
return True
|
||||
if drop_rate >= 1.0:
|
||||
return False
|
||||
self._call_count += 1
|
||||
period = max(2, round(1.0 / drop_rate))
|
||||
return (self._call_count % period) != 0
|
||||
|
||||
def reset(self) -> None:
|
||||
super().reset()
|
||||
self._call_count = 0
|
||||
|
||||
|
||||
class BackpressureDetector(ABC):
|
||||
"""Observes store completion signals and determines pressure state."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
policy: BackpressurePolicy | None = None,
|
||||
):
|
||||
self._policy = policy or ThrottledDropPolicy()
|
||||
|
||||
@property
|
||||
def policy(self) -> BackpressurePolicy:
|
||||
return self._policy
|
||||
|
||||
@abstractmethod
|
||||
def on_store_completed(self, elapsed_s: float, num_bytes: int) -> None: ...
|
||||
|
||||
@abstractmethod
|
||||
def is_under_pressure(self) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
def reset(self) -> None: ...
|
||||
|
||||
def update(self, submit_time: float, num_bytes: int) -> None:
|
||||
"""Update pressure state from a completed store job.
|
||||
|
||||
Args:
|
||||
submit_time: ``time.monotonic()`` when the job was submitted.
|
||||
num_bytes: Total bytes written (num_blocks * block_size_bytes).
|
||||
|
||||
"""
|
||||
if num_bytes <= 0:
|
||||
return
|
||||
elapsed = time.monotonic() - submit_time
|
||||
self.on_store_completed(elapsed, num_bytes)
|
||||
|
||||
def should_store(self, num_blocks: int) -> bool:
|
||||
"""Check policy and record skip if rejected."""
|
||||
if not self._policy.should_store(self):
|
||||
self._policy.on_store_skipped(num_blocks)
|
||||
return False
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def default_config(
|
||||
cls, tier_type: str, *, locality: str | None = None
|
||||
) -> dict | None:
|
||||
"""Return default constructor kwargs for ``tier_type``, or None.
|
||||
|
||||
Args:
|
||||
tier_type: The tier type string (e.g. ``"fs"``, ``"obj"``).
|
||||
locality: Optional locality hint from the tier config
|
||||
(``"LOCAL"`` or ``"REMOTE"``). When ``"REMOTE"``, tiers
|
||||
that would normally get local-storage watermarks (e.g.
|
||||
``"fs"``) receive network watermarks instead.
|
||||
|
||||
"""
|
||||
return None
|
||||
|
||||
@property
|
||||
def stats(self) -> dict[str, float]:
|
||||
return {}
|
||||
|
||||
|
||||
class EMABackpressureDetector(BackpressureDetector):
|
||||
"""EMA of store latency normalized by transfer size.
|
||||
|
||||
The EMA tracks seconds per megabyte (s/MiB) so that the metric is
|
||||
comparable regardless of how many blocks are in a job or how large
|
||||
each block is. Water marks are in the same unit.
|
||||
|
||||
Default water marks are derived from fio benchmarks on the WDC H100
|
||||
cluster. Two presets are provided:
|
||||
|
||||
LOCAL (NVMe/SSD): NVMe sustains ~5 GB/s writes.
|
||||
high=0.005 s/MiB (~200 MB/s) catches severe congestion;
|
||||
low=0.001 s/MiB (~1 GB/s) requires meaningful recovery.
|
||||
|
||||
NETWORK (CephFS, object store, ``obj``tiers, or any tier
|
||||
with ``"locality": "REMOTE"``): CephFS sustains ~1.5 GB/s.
|
||||
high=0.020 s/MiB (~50 MB/s); low=0.005 s/MiB (~200 MB/s).
|
||||
|
||||
``obj`` tiers get NETWORK defaults automatically. An
|
||||
``fs`` tier defaults to LOCAL (appropriate for NVMe/SSD); set
|
||||
``"locality": "REMOTE"`` in the tier config for network-backed
|
||||
filesystems like CephFS.
|
||||
"""
|
||||
|
||||
_MIB = 1 << 20
|
||||
|
||||
# EMA smoothing factor: higher values (→1) react faster to latency
|
||||
# spikes but are noisier; lower values (→0) smooth more but lag.
|
||||
DEFAULT_ALPHA = 0.15
|
||||
|
||||
# Number of store completions to collect before seeding the EMA.
|
||||
# During warmup, pressure is never signalled; the EMA is initialized
|
||||
# to the mean of the warmup samples to avoid cold-start false positives.
|
||||
DEFAULT_WARMUP_COMPLETIONS = 10
|
||||
|
||||
# When under pressure and no completions arrive, the EMA decays
|
||||
# toward zero so the detector can eventually recover. The half-life
|
||||
# controls how quickly: after this many seconds of silence the EMA
|
||||
# halves. With a 2 s half-life and high_water=0.005, an EMA frozen
|
||||
# at 0.010 reaches low_water=0.001 in ~6.6 s.
|
||||
DEFAULT_DECAY_HALF_LIFE_S = 2.0
|
||||
|
||||
# After pressure clears, stores are throttled to 50% for this many
|
||||
# seconds to prevent a burst of stores from re-overwhelming the tier.
|
||||
DEFAULT_COOLDOWN_S = 1.0
|
||||
|
||||
# Number of consecutive sub-low-watermark completions before the
|
||||
# detector enters a "healthy" fast-path that bypasses policy checks.
|
||||
_HEALTHY_THRESHOLD = 10
|
||||
|
||||
LOCAL_HIGH_WATER_S = 0.010
|
||||
LOCAL_LOW_WATER_S = 0.002
|
||||
|
||||
NETWORK_HIGH_WATER_S = 0.050
|
||||
NETWORK_LOW_WATER_S = 0.010
|
||||
|
||||
_NETWORK_TIER_TYPES = frozenset({"obj"})
|
||||
|
||||
@classmethod
|
||||
def default_config(
|
||||
cls, tier_type: str, *, locality: str | None = None
|
||||
) -> dict | None:
|
||||
is_remote = locality is not None and locality.upper() == "REMOTE"
|
||||
if is_remote or tier_type in cls._NETWORK_TIER_TYPES:
|
||||
return {
|
||||
"high_water_s": cls.NETWORK_HIGH_WATER_S,
|
||||
"low_water_s": cls.NETWORK_LOW_WATER_S,
|
||||
}
|
||||
return {
|
||||
"high_water_s": cls.LOCAL_HIGH_WATER_S,
|
||||
"low_water_s": cls.LOCAL_LOW_WATER_S,
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
high_water_s: float,
|
||||
low_water_s: float,
|
||||
alpha: float = DEFAULT_ALPHA,
|
||||
warmup_completions: int = DEFAULT_WARMUP_COMPLETIONS,
|
||||
decay_half_life_s: float = DEFAULT_DECAY_HALF_LIFE_S,
|
||||
cooldown_s: float = DEFAULT_COOLDOWN_S,
|
||||
policy: BackpressurePolicy | None = None,
|
||||
):
|
||||
super().__init__(policy=policy)
|
||||
if not (0 < alpha <= 1):
|
||||
raise ValueError(f"alpha must be in (0, 1], got {alpha}")
|
||||
if low_water_s > high_water_s:
|
||||
raise ValueError(
|
||||
f"low_water_s ({low_water_s}) must be <= high_water_s ({high_water_s})"
|
||||
)
|
||||
|
||||
self._alpha = alpha
|
||||
self._high = high_water_s
|
||||
self._low = low_water_s
|
||||
self._warmup_completions = warmup_completions
|
||||
self._decay_half_life_s = decay_half_life_s
|
||||
self._cooldown_s = cooldown_s
|
||||
# Exponential moving average of store latency in seconds.
|
||||
self._ema: float = 0.0
|
||||
# Whether the tier is currently considered under pressure.
|
||||
self._under_pressure: bool = False
|
||||
# Number of store completions observed (for warmup gating).
|
||||
self._completions: int = 0
|
||||
# Samples collected during warmup; used to seed the EMA.
|
||||
self._warmup_samples: list[float] = []
|
||||
# Monotonic timestamp of the last EMA update; used for idle decay.
|
||||
self._last_update: float = 0.0
|
||||
# Consecutive completions with EMA below low watermark.
|
||||
self._healthy_streak: int = 0
|
||||
# Monotonic timestamp when pressure was last cleared (for cooldown).
|
||||
self._pressure_cleared_at: float = 0.0
|
||||
self._cooldown_count: int = 0
|
||||
|
||||
def _apply_idle_decay(self) -> None:
|
||||
"""Decay the EMA toward zero based on elapsed idle time.
|
||||
|
||||
Uses the formula ``ema * 0.5^(dt / half_life)`` so the EMA
|
||||
halves every ``decay_half_life_s`` seconds of silence. This
|
||||
lets the detector recover from pressure when no stores are
|
||||
attempted (and therefore no completions arrive to update the
|
||||
EMA organically).
|
||||
"""
|
||||
now = time.monotonic()
|
||||
dt = now - self._last_update
|
||||
if dt <= 0 or self._decay_half_life_s <= 0:
|
||||
return
|
||||
self._ema *= 0.5 ** (dt / self._decay_half_life_s)
|
||||
self._last_update = now
|
||||
|
||||
def on_store_completed(self, elapsed_s: float, num_bytes: int) -> None:
|
||||
now = time.monotonic()
|
||||
s_per_mib = elapsed_s / (num_bytes / self._MIB)
|
||||
self._completions += 1
|
||||
if self._completions <= self._warmup_completions:
|
||||
self._warmup_samples.append(s_per_mib)
|
||||
if self._completions == self._warmup_completions:
|
||||
self._ema = sum(self._warmup_samples) / len(self._warmup_samples)
|
||||
self._last_update = now
|
||||
return
|
||||
self._ema = self._alpha * s_per_mib + (1 - self._alpha) * self._ema
|
||||
self._last_update = now
|
||||
if self._ema > self._high:
|
||||
self._under_pressure = True
|
||||
self._pressure_cleared_at = 0.0
|
||||
self._healthy_streak = 0
|
||||
elif self._ema < self._low:
|
||||
if self._under_pressure:
|
||||
self._pressure_cleared_at = now
|
||||
self._cooldown_count = 0
|
||||
self._under_pressure = False
|
||||
self._healthy_streak += 1
|
||||
else:
|
||||
self._healthy_streak = 0
|
||||
|
||||
def is_under_pressure(self) -> bool:
|
||||
if self._under_pressure and self._completions >= self._warmup_completions:
|
||||
self._apply_idle_decay()
|
||||
if self._ema < self._low:
|
||||
self._under_pressure = False
|
||||
self._pressure_cleared_at = time.monotonic()
|
||||
self._cooldown_count = 0
|
||||
return self._under_pressure
|
||||
|
||||
def is_healthy(self) -> bool:
|
||||
"""True when the tier has been consistently fast.
|
||||
|
||||
After ``_HEALTHY_THRESHOLD`` consecutive store completions with
|
||||
EMA below the low watermark (and no active pressure), the
|
||||
detector enters a healthy fast-path: ``should_store()`` returns
|
||||
``True`` immediately, skipping the policy check entirely.
|
||||
"""
|
||||
return (
|
||||
not self._under_pressure and self._healthy_streak >= self._HEALTHY_THRESHOLD
|
||||
)
|
||||
|
||||
def should_store(self, num_blocks: int) -> bool:
|
||||
under_pressure = self.is_under_pressure()
|
||||
if self.is_healthy():
|
||||
return True
|
||||
if (
|
||||
not under_pressure
|
||||
and self._pressure_cleared_at > 0
|
||||
and time.monotonic() - self._pressure_cleared_at < self._cooldown_s
|
||||
):
|
||||
self._cooldown_count += 1
|
||||
if self._cooldown_count % 2 == 0:
|
||||
self._policy.on_store_skipped(num_blocks)
|
||||
return False
|
||||
return super().should_store(num_blocks)
|
||||
|
||||
def reset(self) -> None:
|
||||
self._ema = 0.0
|
||||
self._under_pressure = False
|
||||
self._completions = 0
|
||||
self._warmup_samples.clear()
|
||||
self._last_update = 0.0
|
||||
self._healthy_streak = 0
|
||||
self._pressure_cleared_at = 0.0
|
||||
self._cooldown_count = 0
|
||||
self._policy.reset()
|
||||
|
||||
@property
|
||||
def store_latency_ema(self) -> float:
|
||||
return self._ema
|
||||
|
||||
@store_latency_ema.setter
|
||||
def store_latency_ema(self, value: float) -> None:
|
||||
self._ema = value
|
||||
|
||||
@property
|
||||
def stats(self) -> dict[str, float]:
|
||||
return {"store_latency_ema": self._ema}
|
||||
@@ -2,9 +2,10 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Abstract interfaces and data types for the secondary tiering layer."""
|
||||
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Collection, Iterable
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
|
||||
import numpy as np
|
||||
@@ -20,6 +21,9 @@ from vllm.v1.kv_offload.base import (
|
||||
RequestOffloadingContext,
|
||||
ScheduleEndContext,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.backpressure import (
|
||||
BackpressureDetector,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
@@ -52,6 +56,11 @@ class TieringOffloadingMetrics:
|
||||
)
|
||||
ACTIVE_PROMOTION_JOBS = "vllm:kv_offload_tiering_active_promotion_jobs"
|
||||
ACTIVE_CASCADE_JOBS = "vllm:kv_offload_tiering_active_cascade_jobs"
|
||||
BACKPRESSURE_STORE_LATENCY_EMA = (
|
||||
"vllm:kv_offload_tiering_backpressure_store_latency_ema"
|
||||
)
|
||||
BACKPRESSURE_STORES_DROPPED = "vllm:kv_offload_tiering_backpressure_stores_dropped"
|
||||
BACKPRESSURE_BLOCKS_DROPPED = "vllm:kv_offload_tiering_backpressure_blocks_dropped"
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -63,6 +72,7 @@ class TransferJob:
|
||||
chunk_ids: np.ndarray
|
||||
is_promotion: bool
|
||||
req_context: ReqContext
|
||||
submit_time: float = field(default_factory=time.monotonic)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -77,6 +87,7 @@ class JobResult:
|
||||
# indicated by `success`. Must be a subset of the job's original keys.
|
||||
successful_keys: Collection[OffloadKey] | None = None
|
||||
transfer_time: float | None = None
|
||||
transfer_bytes: int | None = None
|
||||
|
||||
|
||||
class ParentManager(ABC):
|
||||
@@ -136,18 +147,33 @@ class SecondaryTierManager(ABC):
|
||||
offloading_spec: "OffloadingSpec",
|
||||
primary_kv_view: memoryview,
|
||||
tier_type: str,
|
||||
backpressure_detector: BackpressureDetector | None = None,
|
||||
) -> None:
|
||||
"""Args:
|
||||
offloading_spec: Offloading configuration.
|
||||
primary_kv_view: Memoryview of the primary tier's CPU KV cache.
|
||||
tier_type: Tier type identifier, set by SecondaryTierFactory
|
||||
from the registered tier type.
|
||||
backpressure_detector: Optional `BackpressureDetector`.
|
||||
|
||||
"""
|
||||
self._offloading_spec = offloading_spec
|
||||
self._primary_kv_view: memoryview = primary_kv_view
|
||||
assert primary_kv_view.strides is not None, (
|
||||
"primary_kv_view.strides cannot be None"
|
||||
)
|
||||
self._block_size_bytes: int = primary_kv_view.strides[0]
|
||||
self.tier_type = tier_type
|
||||
self.locality: Locality | None = None
|
||||
self._bp_detector = backpressure_detector
|
||||
|
||||
@property
|
||||
def block_size_bytes(self) -> int:
|
||||
return self._block_size_bytes
|
||||
|
||||
@property
|
||||
def bp_detector(self) -> BackpressureDetector | None:
|
||||
return self._bp_detector
|
||||
|
||||
@abstractmethod
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
||||
|
||||
@@ -31,6 +31,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.kv_offload.base import OffloadingSpec
|
||||
from vllm.v1.kv_offload.tiering.backpressure import BackpressureDetector
|
||||
|
||||
|
||||
class ExampleSecondaryTierManager(SecondaryTierManager):
|
||||
@@ -49,6 +50,7 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
|
||||
primary_kv_view: memoryview,
|
||||
tier_type: str,
|
||||
custom_param: int = 0,
|
||||
backpressure_detector: "BackpressureDetector | None" = None,
|
||||
):
|
||||
"""Initialize the example secondary tier.
|
||||
|
||||
@@ -57,12 +59,14 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
|
||||
primary_kv_view: Memoryview over the primary tier's KV buffer.
|
||||
tier_type: Name identifying this tier type.
|
||||
custom_param: Dummy parameter demonstrating custom args.
|
||||
backpressure_detector: Optional backpressure detector.
|
||||
|
||||
"""
|
||||
super().__init__(
|
||||
offloading_spec=offloading_spec,
|
||||
primary_kv_view=primary_kv_view,
|
||||
tier_type=tier_type,
|
||||
backpressure_detector=backpressure_detector,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
|
||||
@@ -8,11 +8,33 @@ from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.tiering.base import SecondaryTierManager
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.kv_offload.backpressure import BackpressureDetector
|
||||
from vllm.v1.kv_offload.base import OffloadingSpec
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
_DEFAULT_PACKAGE = "vllm.v1.kv_offload.tiering.backpressure"
|
||||
|
||||
|
||||
def _import_class(name: str) -> type:
|
||||
"""Import a class by name.
|
||||
|
||||
If ``name`` contains a dot it is treated as a fully qualified path
|
||||
(e.g. ``'my_package.MyDetector'``). Otherwise it is looked up in
|
||||
the default backpressure module
|
||||
(``vllm.v1.kv_offload.tiering.backpressure``), so short names like
|
||||
``'DropStorePolicy'`` or ``'EMABackpressureDetector'`` work out of
|
||||
the box.
|
||||
"""
|
||||
if "." in name:
|
||||
module_path, _, class_name = name.rpartition(".")
|
||||
else:
|
||||
module_path, class_name = _DEFAULT_PACKAGE, name
|
||||
module = importlib.import_module(module_path)
|
||||
return getattr(module, class_name)
|
||||
|
||||
|
||||
class SecondaryTierFactory:
|
||||
"""Registry for SecondaryTierManager implementations, resolved by type.
|
||||
|
||||
@@ -47,10 +69,30 @@ class SecondaryTierFactory:
|
||||
config = tier_config.copy()
|
||||
tier_type = config.pop("type")
|
||||
config.pop("module_path", None)
|
||||
locality = config.get("locality")
|
||||
bp_config = config.pop("backpressure", None)
|
||||
bp_detector = None
|
||||
if bp_config is not None:
|
||||
bp_config = bp_config.copy()
|
||||
bp_cls_name = bp_config.pop("backpressure_cls", None)
|
||||
if bp_cls_name is None:
|
||||
raise ValueError(
|
||||
"backpressure config must include 'backpressure_cls' "
|
||||
"(e.g. 'EMABackpressureDetector')"
|
||||
)
|
||||
policy_cls_name = bp_config.pop("policy_cls", None)
|
||||
policy = _import_class(policy_cls_name)() if policy_cls_name else None
|
||||
detector_cls: type[BackpressureDetector] = _import_class(bp_cls_name)
|
||||
defaults = detector_cls.default_config(tier_type, locality=locality)
|
||||
if defaults:
|
||||
for k, v in defaults.items():
|
||||
bp_config.setdefault(k, v)
|
||||
bp_detector = detector_cls(policy=policy, **bp_config)
|
||||
return tier_cls(
|
||||
offloading_spec=offloading_spec,
|
||||
primary_kv_view=primary_kv_view,
|
||||
tier_type=tier_type,
|
||||
backpressure_detector=bp_detector,
|
||||
**config,
|
||||
)
|
||||
|
||||
|
||||
@@ -40,6 +40,7 @@ from vllm.v1.kv_offload.base import (
|
||||
)
|
||||
from vllm.v1.kv_offload.file_mapper import FileMapper
|
||||
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager
|
||||
from vllm.v1.kv_offload.tiering.backpressure import BackpressureDetector
|
||||
from vllm.v1.kv_offload.tiering.base import (
|
||||
JobId,
|
||||
JobResult,
|
||||
@@ -115,6 +116,7 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
n_write_threads: int = 16,
|
||||
enable_kv_events: bool = False,
|
||||
locality: str | None = None,
|
||||
backpressure_detector: BackpressureDetector | None = None,
|
||||
):
|
||||
"""Args:
|
||||
offloading_spec: Contains normalized offloading configuration and
|
||||
@@ -129,9 +131,12 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
cache events are enabled globally (kv_events_config).
|
||||
locality: Whether this tier's storage is LOCAL or REMOTE relative
|
||||
to the publishing vLLM instance.
|
||||
backpressure_detector: Optional backpressure detector.
|
||||
|
||||
"""
|
||||
super().__init__(offloading_spec, primary_kv_view, tier_type)
|
||||
super().__init__(
|
||||
offloading_spec, primary_kv_view, tier_type, backpressure_detector
|
||||
)
|
||||
self.locality = Locality(locality) if locality is not None else None
|
||||
|
||||
self.events: list[OffloadingEvent] | None = None
|
||||
@@ -150,6 +155,8 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
# Keys of in-flight load (promotion) jobs, so a failed load can mark
|
||||
# its own cached lookup verdicts False (see get_finished_jobs).
|
||||
self._load_job_keys: dict[JobId, list[OffloadKey]] = {}
|
||||
# Block count per in-flight job, used to report transfer_bytes.
|
||||
self._job_block_counts: dict[JobId, int] = {}
|
||||
# Per load job: how many blocks loaded before a failure (partial keep).
|
||||
# Written by the pool worker inside the load task before it raises (so
|
||||
# before task_done publishes the job); read on the scheduler thread in
|
||||
@@ -225,6 +232,7 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
self._block_size,
|
||||
self._use_o_direct,
|
||||
)
|
||||
self._job_block_counts[job_metadata.job_id] = len(keys)
|
||||
self._pool.enqueue_store(job_metadata.job_id, 1, [task])
|
||||
|
||||
@override
|
||||
@@ -234,6 +242,7 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
# keys as a miss (see get_finished_jobs).
|
||||
keys = list(job_metadata.keys)
|
||||
self._load_job_keys[job_id] = keys
|
||||
self._job_block_counts[job_id] = len(keys)
|
||||
paths = [self.file_mapper.get_file_name(key) for key in keys]
|
||||
offsets = [int(cid) * self._block_size for cid in job_metadata.chunk_ids]
|
||||
|
||||
@@ -272,6 +281,8 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
as a miss here (scheduler thread)."""
|
||||
results = []
|
||||
for job_id, success, transfer_time in self._pool.get_finished():
|
||||
block_count = self._job_block_counts.pop(job_id, 0)
|
||||
transfer_bytes = block_count * self._block_size if block_count else None
|
||||
if self.events is not None:
|
||||
keys = self._store_job_keys.pop(job_id, None)
|
||||
if success and keys:
|
||||
@@ -299,6 +310,7 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
success=False,
|
||||
successful_keys=tuple(successful) if successful else None,
|
||||
transfer_time=transfer_time,
|
||||
transfer_bytes=transfer_bytes,
|
||||
)
|
||||
)
|
||||
continue
|
||||
@@ -307,6 +319,7 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
job_id=job_id,
|
||||
success=success,
|
||||
transfer_time=transfer_time,
|
||||
transfer_bytes=transfer_bytes,
|
||||
)
|
||||
)
|
||||
return results
|
||||
|
||||
@@ -238,6 +238,10 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
for tier_idx, tier in enumerate(self.secondary_tiers)
|
||||
}
|
||||
|
||||
self._tier_index: dict[SecondaryTierManager, int] = {
|
||||
tier: i for i, tier in enumerate(self.secondary_tiers)
|
||||
}
|
||||
|
||||
@property
|
||||
def _transfer_jobs(self) -> dict[JobId, JobMetadata]:
|
||||
return self._jobs
|
||||
@@ -337,6 +341,43 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
self.primary_tier.complete_read(
|
||||
transfer_job.keys, transfer_job.req_context
|
||||
)
|
||||
if completed_job.success:
|
||||
self._update_backpressure(tier, job_metadata, completed_job)
|
||||
|
||||
def _should_store_to_tier(
|
||||
self, tier: SecondaryTierManager, num_blocks: int
|
||||
) -> bool:
|
||||
detector = tier.bp_detector
|
||||
if detector is None:
|
||||
return True
|
||||
return detector.should_store(num_blocks)
|
||||
|
||||
def _update_backpressure(
|
||||
self,
|
||||
tier: SecondaryTierManager,
|
||||
job_metadata: JobMetadata,
|
||||
completed_job: JobResult,
|
||||
) -> None:
|
||||
detector = tier.bp_detector
|
||||
if detector is None:
|
||||
return
|
||||
was_under_pressure = detector.is_under_pressure()
|
||||
tj = job_metadata.transfer_job
|
||||
num_bytes = (
|
||||
completed_job.transfer_bytes
|
||||
if completed_job.transfer_bytes is not None
|
||||
else len(tj.keys) * tier.block_size_bytes
|
||||
)
|
||||
detector.update(tj.submit_time, num_bytes)
|
||||
if detector.is_under_pressure() != was_under_pressure:
|
||||
tier_idx = self._tier_index[tier]
|
||||
logger.info(
|
||||
"Tier #%d (%s) back-pressure %s (stats=%s)",
|
||||
tier_idx,
|
||||
tier.tier_type,
|
||||
"activated" if detector.is_under_pressure() else "cleared",
|
||||
detector.stats,
|
||||
)
|
||||
|
||||
@override
|
||||
def lookup(
|
||||
@@ -643,8 +684,10 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
return
|
||||
|
||||
for tier_idx in request_level_tiers:
|
||||
job_metadata = self.create_store_job(ready_keys, req_context, tier_idx)
|
||||
tier = self.secondary_tiers[tier_idx]
|
||||
if not self._should_store_to_tier(tier, len(ready_keys)):
|
||||
continue
|
||||
job_metadata = self.create_store_job(ready_keys, req_context, tier_idx)
|
||||
tier.submit_store(job_metadata)
|
||||
|
||||
def _flush_pending_cascades(self) -> None:
|
||||
@@ -698,6 +741,8 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
# eviction during the async transfer). One prepare_read() call per
|
||||
# secondary tier.
|
||||
for tier_idx, tier in enumerate(self.secondary_tiers):
|
||||
if not self._should_store_to_tier(tier, len(keys)):
|
||||
continue
|
||||
job_metadata = self.create_store_job(keys, req_context, tier_idx)
|
||||
tier.submit_store(job_metadata)
|
||||
|
||||
@@ -904,6 +949,10 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
del self._req_state[req_id]
|
||||
self._processed_jobs_this_step = False
|
||||
|
||||
for tier in self.secondary_tiers:
|
||||
if tier.bp_detector is not None:
|
||||
tier.bp_detector.reset()
|
||||
|
||||
@override
|
||||
def get_stats(self) -> OffloadingConnectorStats | None:
|
||||
stats = self.primary_tier.get_stats()
|
||||
@@ -911,6 +960,7 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
if stats is not None and stats.is_empty():
|
||||
stats = None
|
||||
|
||||
self._metrics.record_backpressure(self.secondary_tiers)
|
||||
metrics_stats = self._metrics.take_stats()
|
||||
if metrics_stats is not None:
|
||||
if stats is None:
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
from vllm.v1.kv_offload.base import LookupResult, OffloadKey, ReqContext
|
||||
from vllm.v1.kv_offload.tiering.base import (
|
||||
JobResult,
|
||||
SecondaryTierManager,
|
||||
TieringOffloadingMetrics,
|
||||
TransferJob,
|
||||
)
|
||||
@@ -130,6 +131,36 @@ class TieringMetricsTracker:
|
||||
TieringOffloadingMetrics.PROMOTION_ALLOCATION_FAILURES
|
||||
)
|
||||
|
||||
def record_backpressure(
|
||||
self,
|
||||
tiers: list[SecondaryTierManager],
|
||||
) -> None:
|
||||
for i, tier in enumerate(tiers):
|
||||
detector = tier.bp_detector
|
||||
if detector is None:
|
||||
continue
|
||||
label = self.tier_label(i)
|
||||
ema = detector.stats.get("store_latency_ema")
|
||||
if ema is not None:
|
||||
self._stats.set_gauge(
|
||||
TieringOffloadingMetrics.BACKPRESSURE_STORE_LATENCY_EMA,
|
||||
ema,
|
||||
labelvalues=label,
|
||||
)
|
||||
stores_dropped, blocks_dropped = detector.policy.pop_stores_dropped()
|
||||
if stores_dropped > 0:
|
||||
self._stats.increase_counter(
|
||||
TieringOffloadingMetrics.BACKPRESSURE_STORES_DROPPED,
|
||||
stores_dropped,
|
||||
labelvalues=label,
|
||||
)
|
||||
if blocks_dropped > 0:
|
||||
self._stats.increase_counter(
|
||||
TieringOffloadingMetrics.BACKPRESSURE_BLOCKS_DROPPED,
|
||||
blocks_dropped,
|
||||
labelvalues=label,
|
||||
)
|
||||
|
||||
def take_stats(self) -> OffloadingConnectorStats | None:
|
||||
active_transfer_stats = OffloadingConnectorStats()
|
||||
self._observe_active_transfer_stats(active_transfer_stats)
|
||||
|
||||
@@ -22,6 +22,7 @@ from vllm.v1.kv_offload.base import (
|
||||
)
|
||||
from vllm.v1.kv_offload.file_mapper import FileMapper
|
||||
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager
|
||||
from vllm.v1.kv_offload.tiering.backpressure import BackpressureDetector
|
||||
from vllm.v1.kv_offload.tiering.base import (
|
||||
JobId,
|
||||
JobResult,
|
||||
@@ -112,6 +113,7 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
|
||||
io_threads: int = 4,
|
||||
enable_kv_events: bool = False,
|
||||
locality: str | None = None,
|
||||
backpressure_detector: BackpressureDetector | None = None,
|
||||
):
|
||||
"""Args:
|
||||
offloading_spec: Offloading configuration.
|
||||
@@ -125,9 +127,15 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
|
||||
cache events are enabled globally (kv_events_config).
|
||||
locality: Whether this tier's storage is LOCAL or REMOTE relative
|
||||
to the publishing vLLM instance.
|
||||
backpressure_detector: Optional backpressure detector.
|
||||
|
||||
"""
|
||||
super().__init__(offloading_spec, primary_kv_view, tier_type)
|
||||
super().__init__(
|
||||
offloading_spec,
|
||||
primary_kv_view,
|
||||
tier_type,
|
||||
backpressure_detector,
|
||||
)
|
||||
self.locality = Locality(locality) if locality is not None else None
|
||||
|
||||
self.events: list[OffloadingEvent] | None = None
|
||||
|
||||
@@ -265,7 +265,25 @@ class P2PSecondaryTierManager(SecondaryTierManager):
|
||||
number, or anything convertible to one.
|
||||
|
||||
"""
|
||||
super().__init__(offloading_spec, primary_kv_view, tier_type)
|
||||
backpressure_detector = kwargs.pop("backpressure_detector", None)
|
||||
if backpressure_detector is not None:
|
||||
# The generic (store-latency) detector is unreliable for P2P: in
|
||||
# PD mode dropping a store can leave the decoder waiting until the
|
||||
# load timeout instead of failing fast, and rendezvous time makes
|
||||
# store latency a poor pressure signal. Reject it until a
|
||||
# P2P-specific fail-fast path exists.
|
||||
raise ValueError(
|
||||
"Backpressure is not supported for the P2P secondary tier. "
|
||||
"The generic store-latency detector cannot fail fast in PD "
|
||||
"mode and rendezvous time makes store latency an unreliable "
|
||||
"pressure signal. Remove the 'backpressure' config from the "
|
||||
"p2p tier."
|
||||
)
|
||||
super().__init__(
|
||||
offloading_spec,
|
||||
primary_kv_view,
|
||||
tier_type,
|
||||
)
|
||||
try:
|
||||
timeout_s = float(unbound_store_timeout_s)
|
||||
except (TypeError, ValueError):
|
||||
|
||||
@@ -247,6 +247,34 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
assert isinstance(tier_config, dict)
|
||||
tier_cls = SecondaryTierFactory.get_tier_class(tier_config)
|
||||
metrics.update(tier_cls.build_metric_definitions(tier_config))
|
||||
|
||||
metrics[TieringOffloadingMetrics.BACKPRESSURE_STORE_LATENCY_EMA] = (
|
||||
OffloadingGaugeMetadata(
|
||||
documentation=(
|
||||
"Exponential moving average of store latency "
|
||||
"for back-pressure detection, in s/MiB."
|
||||
),
|
||||
labelnames=("tier",),
|
||||
)
|
||||
)
|
||||
metrics[TieringOffloadingMetrics.BACKPRESSURE_STORES_DROPPED] = (
|
||||
OffloadingCounterMetadata(
|
||||
documentation=(
|
||||
"Number of store operations dropped due to "
|
||||
"back-pressure on a secondary tier."
|
||||
),
|
||||
labelnames=("tier",),
|
||||
)
|
||||
)
|
||||
metrics[TieringOffloadingMetrics.BACKPRESSURE_BLOCKS_DROPPED] = (
|
||||
OffloadingCounterMetadata(
|
||||
documentation=(
|
||||
"Number of blocks dropped due to back-pressure on a secondary tier."
|
||||
),
|
||||
labelnames=("tier",),
|
||||
)
|
||||
)
|
||||
|
||||
return metrics
|
||||
|
||||
def __init__(self, config: OffloadingConfig):
|
||||
@@ -259,6 +287,32 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
if not isinstance(self.secondary_tier_configs, list):
|
||||
raise ValueError("secondary_tiers must be a list of tier configurations")
|
||||
|
||||
# Backpressure config is merged field-by-field in priority order
|
||||
# (highest first):
|
||||
# 1. Per-tier ``backpressure`` dict in the tier config
|
||||
# 2. Top-level ``backpressure`` in kv_connector_extra_config
|
||||
# Merging per field (rather than per whole dict) means a partial
|
||||
# tier override still inherits missing fields from the top-level
|
||||
# default, so the resolved dict reaching the factory is complete.
|
||||
# Within each tier's resolved dict, tier-type-aware water marks
|
||||
# are filled in last so a bare ``"backpressure": {}`` picks up
|
||||
# sensible thresholds for the storage medium.
|
||||
bp_defaults = self.extra_config.get("backpressure")
|
||||
|
||||
for tier_cfg in self.secondary_tier_configs:
|
||||
tier_override = tier_cfg.get("backpressure")
|
||||
# Overlay from lowest to highest precedence so higher-precedence
|
||||
# fields win while lower-precedence ones fill in the gaps.
|
||||
merged: dict[str, Any] = {}
|
||||
for source in (bp_defaults, tier_override):
|
||||
if source:
|
||||
merged.update(source)
|
||||
# Only set a resolved dict when at least one source contributed
|
||||
# (or an explicit ``backpressure`` key was present, e.g. ``{}``),
|
||||
# so tiers without any backpressure config stay unconfigured.
|
||||
if merged or "backpressure" in tier_cfg:
|
||||
tier_cfg["backpressure"] = merged
|
||||
|
||||
# Scheduler-side mmap (rank=None); kept for cleanup
|
||||
self._scheduler_mmap: SharedOffloadRegion | None = None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user