[KV Offloading] Back-pressure detection and remediation (#50045)

Signed-off-by: Bill Nell <bnell@redhat.com>
Signed-off-by: bnellnm <49004751+bnellnm@users.noreply.github.com>
Co-authored-by: Itay Etelis <92247226+Etelis@users.noreply.github.com>
This commit is contained in:
bnellnm
2026-09-18 14:47:48 -04:00
committed by GitHub
co-authored by Itay Etelis
parent 71fc70d3ae
commit 23e26e0588
13 changed files with 1464 additions and 5 deletions
@@ -682,3 +682,92 @@ def test_tiering_accepts_self_describing_kv_events():
assert spec.kv_events_config.enable_kv_cache_events
assert spec.kv_events_config.self_describing_kv_events
assert tracker.self_describing_enabled
def _build_tiering_spec(secondary_tiers, *, top_level_backpressure=None):
vllm_config = create_vllm_config(
block_size=4,
max_num_batched_tokens=16,
disable_hybrid_kv_cache_manager=False,
)
extra_config = {
"spec_name": "TieringOffloadingSpec",
"cpu_bytes_to_use": 1 << 20,
"secondary_tiers": secondary_tiers,
}
if top_level_backpressure is not None:
extra_config["backpressure"] = top_level_backpressure
vllm_config.kv_transfer_config = KVTransferConfig(
kv_connector="OffloadingConnector",
kv_role="kv_both",
kv_connector_extra_config=extra_config,
)
kv_cache_config = KVCacheConfig(
num_blocks=0,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["layer"],
FullAttentionSpec(
block_size=4,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
)
],
)
return TieringOffloadingSpec(build_offloading_config(vllm_config, kv_cache_config))
def test_partial_tier_backpressure_inherits_top_level_defaults():
"""A partial tier override still inherits missing fields.
Field-by-field merge means an override that sets only ``high_water_s``
keeps ``backpressure_cls`` (and any other field) from the top-level
default, so the resolved dict reaching the factory is complete.
"""
spec = _build_tiering_spec(
[{"type": "example", "backpressure": {"high_water_s": 0.1}}],
top_level_backpressure={
"backpressure_cls": "EMABackpressureDetector",
"high_water_s": 0.5,
"low_water_s": 0.05,
},
)
resolved = spec.secondary_tier_configs[0]["backpressure"]
# Tier override wins for the field it sets.
assert resolved["high_water_s"] == 0.1
# Missing fields fall back to the top-level default.
assert resolved["backpressure_cls"] == "EMABackpressureDetector"
assert resolved["low_water_s"] == 0.05
def test_tier_override_wins_over_top_level():
"""Precedence is tier override > top-level default, field-by-field."""
spec = _build_tiering_spec(
[
{
"type": "example",
"backpressure": {"high_water_s": 0.1, "low_water_s": 0.02},
}
],
top_level_backpressure={
"backpressure_cls": "EMABackpressureDetector",
"high_water_s": 0.5,
"low_water_s": 0.05,
},
)
resolved = spec.secondary_tier_configs[0]["backpressure"]
# Tier override wins for the fields it sets.
assert resolved["high_water_s"] == 0.1
assert resolved["low_water_s"] == 0.02
# Fields only in top-level still fill in.
assert resolved["backpressure_cls"] == "EMABackpressureDetector"
def test_tier_without_backpressure_stays_unconfigured():
spec = _build_tiering_spec([{"type": "example"}])
assert "backpressure" not in spec.secondary_tier_configs[0]
@@ -180,6 +180,40 @@ class TestInitHashSeed:
assert mgr._get_hash_seed() == "random-seed-abc"
class TestBackpressureRejected:
def _patch_transports(self, monkeypatch) -> None:
monkeypatch.setattr(manager_module, "NixlTransport", lambda *a, **k: object())
monkeypatch.setattr(manager_module, "ZmqTransport", lambda *a, **k: object())
monkeypatch.setattr(
manager_module.FileMapper,
"from_offloading_spec",
lambda **k: SimpleNamespace(get_run_config=lambda: {}),
)
def test_explicit_backpressure_detector_rejected(self, monkeypatch):
"""An explicitly configured detector must fail fast for P2P.
The generic store-latency detector cannot fail fast in PD mode and
rendezvous time makes store latency an unreliable pressure signal,
so P2P rejects it rather than degrading silently.
"""
self._patch_transports(monkeypatch)
with pytest.raises(ValueError, match="not supported for the P2P"):
P2PSecondaryTierManager(
offloading_spec=_init_offloading_spec(),
primary_kv_view=memoryview(bytearray(16)),
backpressure_detector=object(),
)
def test_no_detector_constructs_normally(self, monkeypatch):
self._patch_transports(monkeypatch)
mgr = P2PSecondaryTierManager(
offloading_spec=_init_offloading_spec(),
primary_kv_view=memoryview(bytearray(16)),
)
assert mgr.bp_detector is None
# ---------------------------------------------------------------------------
# Tests for _peer_id_from_params
# ---------------------------------------------------------------------------
@@ -0,0 +1,706 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for back-pressure detection in TieringOffloadingManager.
These tests use a DelayedSecondaryTierManager that holds completed jobs
until explicitly released, allowing precise control over when the manager
observes store completions and their apparent latency.
"""
import time
from collections.abc import Iterable
from unittest.mock import MagicMock, patch
import pytest
import torch
from vllm.v1.kv_offload.base import (
LookupResult,
OffloadKey,
OffloadPolicy,
ReqContext,
RequestOffloadingContext,
ScheduleEndContext,
make_offload_key,
)
from vllm.v1.kv_offload.tiering.backpressure import (
DropAccountingPolicy,
DropStorePolicy,
EMABackpressureDetector,
ThrottledDropPolicy,
)
from vllm.v1.kv_offload.tiering.base import (
JobResult,
SecondaryTierManager,
TieringOffloadingMetrics,
TransferJob,
)
from vllm.v1.kv_offload.tiering.manager import (
CPUPrimaryTierOffloadingManager,
TieringOffloadingManager,
)
_BP_EMA_ALPHA = EMABackpressureDetector.DEFAULT_ALPHA
_BP_WARMUP = EMABackpressureDetector.DEFAULT_WARMUP_COMPLETIONS
# Water marks in s/MiB, scaled for the test's tiny 16-byte blocks.
_BP_HIGH_WATER_S = 1000.0
_BP_LOW_WATER_S = 500.0
_CTX = ReqContext(req_id="test")
_MOCK_OFFLOADING_SPEC = MagicMock()
def _mock_mmap_region(num_blocks: int, row_bytes: int = 16):
mock = MagicMock()
view = memoryview(torch.zeros((num_blocks, row_bytes), dtype=torch.int8).numpy())
mock.create_kv_memoryview.return_value = view
return mock
def to_keys(int_ids: Iterable[int]) -> list[OffloadKey]:
return [make_offload_key(str(i).encode(), 0) for i in int_ids]
class DelayedSecondaryTierManager(SecondaryTierManager):
"""Secondary tier that holds completed store jobs until released.
Jobs are stored immediately but their completion is not reported
by get_finished_jobs() until release_jobs() is called. This lets
tests control the apparent latency of store operations by adjusting
submit_time on the JobMetadata before triggering the completion poll.
"""
def __init__(
self, offloading_spec, primary_kv_view, tier_type, backpressure_detector=None
):
super().__init__(
offloading_spec,
primary_kv_view,
tier_type,
backpressure_detector=backpressure_detector,
)
self.blocks: dict[OffloadKey, bool] = {}
self._held_jobs: list[JobResult] = []
self._released_jobs: list[JobResult] = []
self._request_policy: OffloadPolicy | None = None
def lookup(self, key, req_context):
return LookupResult.HIT if key in self.blocks else LookupResult.MISS
def submit_store(self, job_metadata: TransferJob) -> None:
for key in job_metadata.keys:
self.blocks[key] = True
self._held_jobs.append(JobResult(job_id=job_metadata.job_id, success=True))
def submit_load(self, job_metadata: TransferJob) -> None:
self._released_jobs.append(JobResult(job_id=job_metadata.job_id, success=True))
def get_finished_jobs(self) -> Iterable[JobResult]:
result = self._released_jobs
self._released_jobs = []
return result
def release_jobs(self):
self._released_jobs.extend(self._held_jobs)
self._held_jobs.clear()
def on_new_request(self, req_context):
if self._request_policy is not None:
return RequestOffloadingContext(policy=self._request_policy)
return RequestOffloadingContext()
def drain_jobs(self):
self.release_jobs()
def has_pending_work(self):
return bool(self._held_jobs)
def get_num_blocks(self):
return len(self.blocks)
class TestDefaultConfig:
def test_fs_gets_local_watermarks(self):
cfg = EMABackpressureDetector.default_config("fs")
assert cfg["high_water_s"] == EMABackpressureDetector.LOCAL_HIGH_WATER_S
assert cfg["low_water_s"] == EMABackpressureDetector.LOCAL_LOW_WATER_S
def test_obj_gets_network_watermarks(self):
cfg = EMABackpressureDetector.default_config("obj")
assert cfg["high_water_s"] == EMABackpressureDetector.NETWORK_HIGH_WATER_S
assert cfg["low_water_s"] == EMABackpressureDetector.NETWORK_LOW_WATER_S
def test_p2p_gets_local_watermarks(self):
cfg = EMABackpressureDetector.default_config("p2p")
assert cfg["high_water_s"] == EMABackpressureDetector.LOCAL_HIGH_WATER_S
assert cfg["low_water_s"] == EMABackpressureDetector.LOCAL_LOW_WATER_S
def test_fs_with_remote_locality_gets_network_watermarks(self):
cfg = EMABackpressureDetector.default_config("fs", locality="REMOTE")
assert cfg["high_water_s"] == EMABackpressureDetector.NETWORK_HIGH_WATER_S
assert cfg["low_water_s"] == EMABackpressureDetector.NETWORK_LOW_WATER_S
def test_fs_with_local_locality_gets_local_watermarks(self):
cfg = EMABackpressureDetector.default_config("fs", locality="LOCAL")
assert cfg["high_water_s"] == EMABackpressureDetector.LOCAL_HIGH_WATER_S
assert cfg["low_water_s"] == EMABackpressureDetector.LOCAL_LOW_WATER_S
class TestIdleDecay:
"""Tests for EMA idle decay — recovery from pressure without completions."""
def _make_detector(self, **kwargs):
defaults = dict(
high_water_s=_BP_HIGH_WATER_S,
low_water_s=_BP_LOW_WATER_S,
decay_half_life_s=2.0,
)
defaults.update(kwargs)
return EMABackpressureDetector(**defaults)
def test_ema_decays_while_under_pressure(self):
bp = self._make_detector()
bp._completions = _BP_WARMUP
bp._ema = _BP_HIGH_WATER_S * 2
bp._under_pressure = True
bp._last_update = time.monotonic()
with patch("time.monotonic", return_value=bp._last_update + 4.1):
# ~2.05 half-lives → EMA * 0.24, well below low water (500)
assert bp.is_under_pressure() is False
assert bp.store_latency_ema < _BP_LOW_WATER_S
def test_ema_stays_pressured_if_not_enough_decay(self):
bp = self._make_detector()
bp._completions = _BP_WARMUP
bp._ema = _BP_HIGH_WATER_S * 4
bp._under_pressure = True
bp._last_update = time.monotonic()
with patch("time.monotonic", return_value=bp._last_update + 2.0):
# 2s = 1 half-life → EMA * 0.5 = HIGH * 2, still above LOW
assert bp.is_under_pressure() is True
assert bp.store_latency_ema == pytest.approx(_BP_HIGH_WATER_S * 2, rel=0.01)
def test_no_decay_before_warmup(self):
bp = self._make_detector()
bp._ema = _BP_HIGH_WATER_S * 2
bp._under_pressure = True
bp._last_update = time.monotonic()
with patch("time.monotonic", return_value=bp._last_update + 10.0):
# Still in warmup (_completions=0 < _warmup_completions),
# so decay should not be applied in is_under_pressure.
assert bp.is_under_pressure() is True
assert bp.store_latency_ema == _BP_HIGH_WATER_S * 2
def test_no_decay_applied_during_new_sample(self):
"""on_store_completed applies the new sample directly without
idle decay — decay only runs in is_under_pressure()."""
bp = self._make_detector()
bp._completions = _BP_WARMUP
bp._ema = _BP_HIGH_WATER_S * 2
bp._under_pressure = True
t0 = time.monotonic()
bp._last_update = t0
block_bytes = 16
with patch("time.monotonic", return_value=t0 + 4.0):
bp.on_store_completed(0.0, block_bytes)
# No decay in on_store_completed: EMA = alpha*0 + (1-alpha)*2000
expected = (1 - _BP_EMA_ALPHA) * _BP_HIGH_WATER_S * 2
assert bp.store_latency_ema == pytest.approx(expected, rel=0.01)
def test_reset_clears_last_update(self):
bp = self._make_detector()
bp._last_update = time.monotonic()
bp.reset()
assert bp._last_update == 0.0
class TestBackpressure:
@pytest.fixture
def setup(self):
mock_region = _mock_mmap_region(20)
self.primary = CPUPrimaryTierOffloadingManager(
num_chunks=20, mmap_region=mock_region
)
mock_view = mock_region.create_kv_memoryview()
self.tier = DelayedSecondaryTierManager(
offloading_spec=_MOCK_OFFLOADING_SPEC,
primary_kv_view=mock_view,
tier_type="delayed",
backpressure_detector=EMABackpressureDetector(
high_water_s=_BP_HIGH_WATER_S,
low_water_s=_BP_LOW_WATER_S,
),
)
self.manager = TieringOffloadingManager(
primary_tier=self.primary,
secondary_tiers=[self.tier],
)
def _start_request(self, ctx=_CTX):
if ctx.req_id not in self.manager._req_state:
self.manager.on_new_request(ctx)
def _store_blocks(self, keys, ctx=_CTX):
self._start_request(ctx)
result = self.manager.prepare_store(keys, ctx)
assert result is not None
self.manager.complete_store(keys, ctx, success=True)
def _simulate_on_schedule_end(self):
ctx = ScheduleEndContext(new_req_ids=[], preempted_req_ids=())
self.manager.on_schedule_end(ctx)
def _backdate_held_jobs(self, age_s: float):
"""Set submit_time on all in-flight store jobs so they appear
to have taken ``age_s`` seconds when completed."""
self.manager._processed_jobs_this_step = False
now = time.monotonic()
for job_id, meta in self.manager._jobs.items():
tj = meta.transfer_job
if not tj.is_promotion and tj.submit_time > 0:
tj.submit_time = now - age_s
def test_ema_updates_on_store_completion(self, setup):
bp = self.tier.bp_detector
fast_latency = 0.01
# EMA is in s/MiB; with 16-byte blocks, scale = MiB / block_bytes.
scale = EMABackpressureDetector._MIB / self.tier.block_size_bytes
# During warmup, EMA stays at 0.
for i in range(_BP_WARMUP - 1):
keys = to_keys([300 + i])
self._store_blocks(keys)
self._backdate_held_jobs(fast_latency)
self.tier.release_jobs()
self._simulate_on_schedule_end()
assert bp.store_latency_ema == 0.0
# Final warmup sample seeds EMA with the mean.
keys = to_keys([400])
self._store_blocks(keys)
self._backdate_held_jobs(fast_latency)
self.tier.release_jobs()
self._simulate_on_schedule_end()
fast_s_per_mib = fast_latency * scale
assert bp.store_latency_ema == pytest.approx(fast_s_per_mib, rel=0.1)
# After warmup, EMA updates normally.
new_latency = 0.05
keys = to_keys([401])
self._store_blocks(keys)
self._backdate_held_jobs(new_latency)
self.tier.release_jobs()
self._simulate_on_schedule_end()
new_s_per_mib = new_latency * scale
expected = _BP_EMA_ALPHA * new_s_per_mib + (1 - _BP_EMA_ALPHA) * fast_s_per_mib
assert bp.store_latency_ema == pytest.approx(expected, rel=0.1)
def test_pressure_activates_above_high_water(self, setup):
bp = self.tier.bp_detector
# Warm up the detector with fast completions first.
for i in range(_BP_WARMUP):
keys = to_keys([200 + i])
self._store_blocks(keys)
self._backdate_held_jobs(0.001)
self.tier.release_jobs()
self._simulate_on_schedule_end()
keys = to_keys([100])
self._store_blocks(keys)
self._simulate_on_schedule_end()
# 0.1s for a single 16-byte block → ~6553 s/MiB, well above high water.
self._backdate_held_jobs(0.1)
self.tier.release_jobs()
self._simulate_on_schedule_end()
assert bp.store_latency_ema > _BP_HIGH_WATER_S
assert bp.is_under_pressure() is True
def test_pressure_clears_below_low_water(self, setup):
bp = self.tier.bp_detector
bp.store_latency_ema = _BP_HIGH_WATER_S * 2
bp._under_pressure = True
bp._completions = _BP_WARMUP
# Drive several fast completions to bring EMA below low water.
block_id = 100
for _ in range(50):
if bp.store_latency_ema < _BP_LOW_WATER_S:
break
keys = to_keys([block_id])
block_id += 1
self._store_blocks(keys)
# Pressure is on, so the cascade was skipped. Force-submit
# a store job manually to get a completion to feed the EMA.
job_meta = self.manager.create_store_job(keys, _CTX)
job_meta.submit_time = time.monotonic() - 0.001
self.tier.submit_store(job_meta)
self.tier.release_jobs()
self._simulate_on_schedule_end()
else:
pytest.fail(f"EMA did not recover below low water: {bp.store_latency_ema}")
assert bp.is_under_pressure() is False
def test_hysteresis_prevents_oscillation(self, setup):
bp = self.tier.bp_detector
bp._completions = _BP_WARMUP
block_bytes = self.tier.block_size_bytes
# Set EMA between low and high water marks (in s/MiB).
mid = (_BP_LOW_WATER_S + _BP_HIGH_WATER_S) / 2
bp.store_latency_ema = mid
bp._last_update = time.monotonic()
# Raw seconds that produce `mid` s/MiB for a given number of blocks.
def mid_latency_s(num_blocks: int) -> float:
return mid * (num_blocks * block_bytes / EMABackpressureDetector._MIB)
# When not under pressure, mid-range EMA should not activate.
bp._under_pressure = False
keys = to_keys(range(2))
self._store_blocks(keys)
self._backdate_held_jobs(mid_latency_s(len(keys)))
self.tier.release_jobs()
self._simulate_on_schedule_end()
assert bp.is_under_pressure() is False
# When under pressure, mid-range EMA should not deactivate.
bp._under_pressure = True
bp._last_update = time.monotonic()
keys2 = to_keys(range(10, 12))
# Force-submit since pressure is on (cascade would skip).
for k in keys2:
self.primary.prepare_store([k], _CTX)
self.primary.complete_store([k], _CTX)
job_meta = self.manager.create_store_job(keys2, _CTX)
job_meta.submit_time = time.monotonic() - mid_latency_s(len(keys2))
self.tier.submit_store(job_meta)
self.tier.release_jobs()
self._simulate_on_schedule_end()
assert bp.is_under_pressure() is True
def test_stores_skipped_under_pressure(self, setup):
bp = self.tier.bp_detector
bp._under_pressure = True
bp.store_latency_ema = _BP_HIGH_WATER_S * 4
initial_blocks = self.tier.get_num_blocks()
keys = to_keys(range(50, 53))
self._store_blocks(keys)
# Blocks should be in primary but NOT in secondary.
for k in keys:
assert self.primary.lookup(k, _CTX) is LookupResult.HIT
assert self.tier.get_num_blocks() == initial_blocks
def test_stores_resume_after_pressure_clears(self, setup):
bp = self.tier.bp_detector
# Start under pressure — stores skipped.
bp._under_pressure = True
bp.store_latency_ema = _BP_HIGH_WATER_S * 4
keys1 = to_keys(range(60, 62))
self._store_blocks(keys1)
assert all(k not in self.tier.blocks for k in keys1)
# Clear pressure — next store should cascade.
bp._under_pressure = False
keys2 = to_keys(range(70, 72))
self._store_blocks(keys2)
assert all(k in self.tier.blocks for k in keys2)
def test_dropped_store_count_tracked(self, setup):
bp = self.tier.bp_detector
bp._under_pressure = True
bp.store_latency_ema = _BP_HIGH_WATER_S * 4
policy = bp.policy
assert policy.pop_stores_dropped() == (0, 0)
self._store_blocks(to_keys([80]))
assert policy._stores_dropped == 1
assert policy._blocks_dropped == 1
self._store_blocks(to_keys([81, 82]))
assert policy._stores_dropped == 2
assert policy._blocks_dropped == 3
def test_metrics_reported_via_get_stats(self, setup):
bp = self.tier.bp_detector
bp._under_pressure = True
bp.store_latency_ema = 2.5
policy = bp.policy
policy._stores_dropped = 5
policy._blocks_dropped = 12
stats = self.manager.get_stats()
assert stats is not None
reduced = stats.reduce()
ema_key = TieringOffloadingMetrics.BACKPRESSURE_STORE_LATENCY_EMA
stores_key = TieringOffloadingMetrics.BACKPRESSURE_STORES_DROPPED
blocks_key = TieringOffloadingMetrics.BACKPRESSURE_BLOCKS_DROPPED
assert reduced[f"{ema_key}:('1:delayed',)"] == pytest.approx(2.5)
assert reduced[f"{stores_key}:('1:delayed',)"] == 5
assert reduced[f"{blocks_key}:('1:delayed',)"] == 12
# Dropped counts reset after get_stats.
assert policy._stores_dropped == 0
assert policy._blocks_dropped == 0
def test_reset_cache_clears_backpressure(self, setup):
bp = self.tier.bp_detector
bp.store_latency_ema = 5.0
bp._under_pressure = True
self.manager.reset_cache()
assert bp.store_latency_ema == 0.0
assert bp.is_under_pressure() is False
def test_request_level_tier_respects_backpressure(self, setup):
"""Request-level cascade also skips pressured tiers."""
# Store blocks in primary first.
existing = to_keys(range(3))
self._store_blocks(existing)
self.tier.release_jobs()
self._simulate_on_schedule_end()
self._simulate_on_schedule_end()
# Make tier request-level for a new request.
self.tier._request_policy = OffloadPolicy.REQUEST_LEVEL
ctx = ReqContext(req_id="req_rl_bp")
self.manager.on_new_request(ctx)
# Activate pressure.
bp = self.tier.bp_detector
bp._under_pressure = True
bp.store_latency_ema = _BP_HIGH_WATER_S * 4
initial_blocks = self.tier.get_num_blocks()
# prepare_store with existing + new blocks.
new = to_keys(range(10, 12))
all_keys = existing + new
result = self.manager.prepare_store(all_keys, ctx)
assert result is not None
# Existing blocks would normally cascade to request-level tier,
# but should be skipped under pressure.
assert self.tier.get_num_blocks() == initial_blocks
def test_loads_continue_under_pressure(self, setup):
"""Loads from a pressured tier still work."""
blocks = to_keys(range(3))
for b in blocks:
self.tier.blocks[b] = True
bp = self.tier.bp_detector
bp._under_pressure = True
bp.store_latency_ema = _BP_HIGH_WATER_S * 4
# Lookups should still initiate promotion.
for b in blocks:
result = self.manager.lookup(b, _CTX)
assert result is LookupResult.HIT_PENDING
class TestThrottledDropPolicy:
"""Tests for proportional throttling under pressure."""
def _make_detector(self, **kwargs):
defaults = dict(
high_water_s=_BP_HIGH_WATER_S,
low_water_s=_BP_LOW_WATER_S,
cooldown_s=0.0,
)
defaults.update(kwargs)
return EMABackpressureDetector(**defaults)
def test_no_drop_when_not_under_pressure(self):
bp = self._make_detector()
for _ in range(20):
assert bp.should_store(1) is True
def test_no_drop_at_exactly_high_water(self):
bp = self._make_detector()
bp._under_pressure = True
bp.store_latency_ema = _BP_HIGH_WATER_S
for _ in range(20):
assert bp.should_store(1) is True
def test_partial_drop_at_moderate_pressure(self):
bp = self._make_detector()
bp._under_pressure = True
bp.store_latency_ema = _BP_HIGH_WATER_S * 2
results = [bp.should_store(1) for _ in range(20)]
assert any(results), "some stores should be allowed"
assert not all(results), "some stores should be dropped"
def test_full_drop_at_severe_pressure(self):
bp = self._make_detector()
bp._under_pressure = True
bp.store_latency_ema = _BP_HIGH_WATER_S * 4
for _ in range(20):
assert bp.should_store(1) is False
def test_drop_rate_increases_with_severity(self):
"""Higher EMA → higher drop rate."""
drop_counts = []
for multiplier in [1.5, 2.0, 2.5, 3.5]:
bp = self._make_detector()
bp._under_pressure = True
bp.store_latency_ema = _BP_HIGH_WATER_S * multiplier
drops = sum(1 for _ in range(100) if not bp.should_store(1))
drop_counts.append(drops)
for i in range(len(drop_counts) - 1):
assert drop_counts[i] <= drop_counts[i + 1]
def test_explicit_drop_store_policy_still_drops_all(self):
"""DropStorePolicy (explicit) still drops everything under pressure."""
bp = EMABackpressureDetector(
high_water_s=_BP_HIGH_WATER_S,
low_water_s=_BP_LOW_WATER_S,
policy=DropStorePolicy(),
cooldown_s=0.0,
)
bp._under_pressure = True
bp.store_latency_ema = _BP_HIGH_WATER_S * 1.5
for _ in range(20):
assert bp.should_store(1) is False
class TestSharedDropAccounting:
"""Both policies share drop accounting via DropAccountingPolicy."""
@pytest.mark.parametrize("policy_cls", [DropStorePolicy, ThrottledDropPolicy])
def test_shared_base(self, policy_cls):
assert issubclass(policy_cls, DropAccountingPolicy)
@pytest.mark.parametrize("policy_cls", [DropStorePolicy, ThrottledDropPolicy])
def test_pop_returns_and_resets(self, policy_cls):
policy = policy_cls()
policy.on_store_skipped(3)
policy.on_store_skipped(2)
assert policy.pop_stores_dropped() == (2, 5)
assert policy.pop_stores_dropped() == (0, 0)
@pytest.mark.parametrize("policy_cls", [DropStorePolicy, ThrottledDropPolicy])
def test_reset_clears_counters(self, policy_cls):
policy = policy_cls()
policy.on_store_skipped(4)
policy.reset()
assert policy.pop_stores_dropped() == (0, 0)
def test_throttled_reset_also_clears_call_count(self):
policy = ThrottledDropPolicy()
policy._call_count = 7
policy.reset()
assert policy._call_count == 0
class TestHealthyBypass:
"""Tests for the fast-path bypass when a tier is consistently healthy."""
def _make_detector(self, **kwargs):
defaults = dict(
high_water_s=_BP_HIGH_WATER_S,
low_water_s=_BP_LOW_WATER_S,
cooldown_s=0.0,
)
defaults.update(kwargs)
return EMABackpressureDetector(**defaults)
def test_not_healthy_initially(self):
bp = self._make_detector()
assert bp.is_healthy() is False
def test_healthy_after_streak(self):
bp = self._make_detector()
bp._completions = _BP_WARMUP
bp._last_update = time.monotonic()
block_bytes = 16
fast_s_per_mib = _BP_LOW_WATER_S * 0.5
fast_elapsed = fast_s_per_mib * (block_bytes / EMABackpressureDetector._MIB)
for _ in range(EMABackpressureDetector._HEALTHY_THRESHOLD + _BP_WARMUP):
bp.on_store_completed(fast_elapsed, block_bytes)
assert bp.is_healthy() is True
def test_healthy_resets_on_high_latency(self):
bp = self._make_detector()
bp._completions = _BP_WARMUP
bp._healthy_streak = EMABackpressureDetector._HEALTHY_THRESHOLD
bp._ema = _BP_LOW_WATER_S
bp._last_update = time.monotonic()
block_bytes = 16
slow_s_per_mib = _BP_HIGH_WATER_S * 2
slow_elapsed = slow_s_per_mib * (block_bytes / EMABackpressureDetector._MIB)
bp.on_store_completed(slow_elapsed, block_bytes)
assert bp.is_healthy() is False
def test_healthy_bypasses_policy(self):
bp = self._make_detector()
bp._completions = _BP_WARMUP
bp._healthy_streak = EMABackpressureDetector._HEALTHY_THRESHOLD
assert bp.should_store(1) is True
def test_reset_clears_healthy_streak(self):
bp = self._make_detector()
bp._healthy_streak = EMABackpressureDetector._HEALTHY_THRESHOLD
bp.reset()
assert bp._healthy_streak == 0
assert bp.is_healthy() is False
class TestCooldown:
"""Tests for the post-pressure cooldown period."""
def _make_detector(self, **kwargs):
defaults = dict(
high_water_s=_BP_HIGH_WATER_S,
low_water_s=_BP_LOW_WATER_S,
cooldown_s=1.0,
)
defaults.update(kwargs)
return EMABackpressureDetector(**defaults)
def test_cooldown_throttles_stores(self):
bp = self._make_detector()
bp._completions = _BP_WARMUP
bp._under_pressure = True
bp._ema = _BP_HIGH_WATER_S * 2
bp._last_update = time.monotonic()
with patch("time.monotonic") as mock_time:
t0 = bp._last_update + 10.0
mock_time.return_value = t0
assert bp.is_under_pressure() is False
assert bp._pressure_cleared_at > 0
mock_time.return_value = t0 + 0.1
results = [bp.should_store(1) for _ in range(10)]
assert sum(results) == 5, "cooldown should allow ~50%"
def test_no_cooldown_when_never_pressured(self):
bp = self._make_detector()
for _ in range(20):
assert bp.should_store(1) is True
def test_no_cooldown_after_window_expires(self):
bp = self._make_detector()
bp._pressure_cleared_at = time.monotonic() - 2.0
for _ in range(20):
assert bp.should_store(1) is True
def test_reset_clears_cooldown(self):
bp = self._make_detector()
bp._pressure_cleared_at = time.monotonic()
bp._cooldown_count = 5
bp.reset()
assert bp._pressure_cleared_at == 0.0
assert bp._cooldown_count == 0
+384
View File
@@ -0,0 +1,384 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import time
from abc import ABC, abstractmethod
class BackpressurePolicy(ABC):
"""Decides what action to take when a detector signals pressure."""
@abstractmethod
def should_store(self, detector: BackpressureDetector) -> bool: ...
@abstractmethod
def on_store_skipped(self, num_blocks: int) -> None: ...
@abstractmethod
def pop_stores_dropped(self) -> tuple[int, int]:
"""Return and reset (stores_dropped, blocks_dropped)."""
...
@abstractmethod
def reset(self) -> None: ...
class DropAccountingPolicy(BackpressurePolicy):
"""Base policy that tracks dropped store/block counters.
Subclasses implement ``should_store``; the shared drop accounting
(``on_store_skipped``/``pop_stores_dropped``/``reset``) lives here so
every policy reports these counters identically. Subclasses that add
their own state override ``reset`` and call ``super().reset()``.
"""
def __init__(self):
self._stores_dropped: int = 0
self._blocks_dropped: int = 0
def on_store_skipped(self, num_blocks: int) -> None:
self._stores_dropped += 1
self._blocks_dropped += num_blocks
def pop_stores_dropped(self) -> tuple[int, int]:
stores, blocks = self._stores_dropped, self._blocks_dropped
self._stores_dropped = 0
self._blocks_dropped = 0
return stores, blocks
def reset(self) -> None:
self._stores_dropped = 0
self._blocks_dropped = 0
class DropStorePolicy(DropAccountingPolicy):
"""Silently drop stores to pressured tiers."""
def should_store(self, detector) -> bool:
return not detector.is_under_pressure()
class ThrottledDropPolicy(DropAccountingPolicy):
"""Drop stores proportionally to pressure severity.
Instead of dropping all stores when pressure is detected, the drop
rate scales with how far the EMA exceeds the high watermark::
overshoot = (ema - high_water) / high_water
drop_rate = min(1.0, overshoot / ramp_factor)
With the default ``ramp_factor=2.0``, stores ramp from 0% drop at
the high watermark to 100% drop at 3x the high watermark. This
keeps secondary tiers populated under moderate pressure, reducing
cache misses (and therefore recomputation / ITL regression) while
still protecting TTFT under severe congestion.
"""
def __init__(self, ramp_factor: float = 2.0):
super().__init__()
self._call_count: int = 0
self._ramp_factor = ramp_factor
def should_store(self, detector: BackpressureDetector) -> bool:
# Proportional throttling needs the EMA and high watermark, which
# only the EMA detector exposes.
assert isinstance(detector, EMABackpressureDetector), (
"ThrottledDropPolicy requires an EMABackpressureDetector"
)
if not detector.is_under_pressure():
return True
ema = detector.store_latency_ema
high = detector._high
if high <= 0:
return False
overshoot = max(0.0, ema - high) / high
drop_rate = min(1.0, overshoot / self._ramp_factor)
if drop_rate <= 0:
return True
if drop_rate >= 1.0:
return False
self._call_count += 1
period = max(2, round(1.0 / drop_rate))
return (self._call_count % period) != 0
def reset(self) -> None:
super().reset()
self._call_count = 0
class BackpressureDetector(ABC):
"""Observes store completion signals and determines pressure state."""
def __init__(
self,
policy: BackpressurePolicy | None = None,
):
self._policy = policy or ThrottledDropPolicy()
@property
def policy(self) -> BackpressurePolicy:
return self._policy
@abstractmethod
def on_store_completed(self, elapsed_s: float, num_bytes: int) -> None: ...
@abstractmethod
def is_under_pressure(self) -> bool: ...
@abstractmethod
def reset(self) -> None: ...
def update(self, submit_time: float, num_bytes: int) -> None:
"""Update pressure state from a completed store job.
Args:
submit_time: ``time.monotonic()`` when the job was submitted.
num_bytes: Total bytes written (num_blocks * block_size_bytes).
"""
if num_bytes <= 0:
return
elapsed = time.monotonic() - submit_time
self.on_store_completed(elapsed, num_bytes)
def should_store(self, num_blocks: int) -> bool:
"""Check policy and record skip if rejected."""
if not self._policy.should_store(self):
self._policy.on_store_skipped(num_blocks)
return False
return True
@classmethod
def default_config(
cls, tier_type: str, *, locality: str | None = None
) -> dict | None:
"""Return default constructor kwargs for ``tier_type``, or None.
Args:
tier_type: The tier type string (e.g. ``"fs"``, ``"obj"``).
locality: Optional locality hint from the tier config
(``"LOCAL"`` or ``"REMOTE"``). When ``"REMOTE"``, tiers
that would normally get local-storage watermarks (e.g.
``"fs"``) receive network watermarks instead.
"""
return None
@property
def stats(self) -> dict[str, float]:
return {}
class EMABackpressureDetector(BackpressureDetector):
"""EMA of store latency normalized by transfer size.
The EMA tracks seconds per megabyte (s/MiB) so that the metric is
comparable regardless of how many blocks are in a job or how large
each block is. Water marks are in the same unit.
Default water marks are derived from fio benchmarks on the WDC H100
cluster. Two presets are provided:
LOCAL (NVMe/SSD): NVMe sustains ~5 GB/s writes.
high=0.005 s/MiB (~200 MB/s) catches severe congestion;
low=0.001 s/MiB (~1 GB/s) requires meaningful recovery.
NETWORK (CephFS, object store, ``obj``tiers, or any tier
with ``"locality": "REMOTE"``): CephFS sustains ~1.5 GB/s.
high=0.020 s/MiB (~50 MB/s); low=0.005 s/MiB (~200 MB/s).
``obj`` tiers get NETWORK defaults automatically. An
``fs`` tier defaults to LOCAL (appropriate for NVMe/SSD); set
``"locality": "REMOTE"`` in the tier config for network-backed
filesystems like CephFS.
"""
_MIB = 1 << 20
# EMA smoothing factor: higher values (→1) react faster to latency
# spikes but are noisier; lower values (→0) smooth more but lag.
DEFAULT_ALPHA = 0.15
# Number of store completions to collect before seeding the EMA.
# During warmup, pressure is never signalled; the EMA is initialized
# to the mean of the warmup samples to avoid cold-start false positives.
DEFAULT_WARMUP_COMPLETIONS = 10
# When under pressure and no completions arrive, the EMA decays
# toward zero so the detector can eventually recover. The half-life
# controls how quickly: after this many seconds of silence the EMA
# halves. With a 2 s half-life and high_water=0.005, an EMA frozen
# at 0.010 reaches low_water=0.001 in ~6.6 s.
DEFAULT_DECAY_HALF_LIFE_S = 2.0
# After pressure clears, stores are throttled to 50% for this many
# seconds to prevent a burst of stores from re-overwhelming the tier.
DEFAULT_COOLDOWN_S = 1.0
# Number of consecutive sub-low-watermark completions before the
# detector enters a "healthy" fast-path that bypasses policy checks.
_HEALTHY_THRESHOLD = 10
LOCAL_HIGH_WATER_S = 0.010
LOCAL_LOW_WATER_S = 0.002
NETWORK_HIGH_WATER_S = 0.050
NETWORK_LOW_WATER_S = 0.010
_NETWORK_TIER_TYPES = frozenset({"obj"})
@classmethod
def default_config(
cls, tier_type: str, *, locality: str | None = None
) -> dict | None:
is_remote = locality is not None and locality.upper() == "REMOTE"
if is_remote or tier_type in cls._NETWORK_TIER_TYPES:
return {
"high_water_s": cls.NETWORK_HIGH_WATER_S,
"low_water_s": cls.NETWORK_LOW_WATER_S,
}
return {
"high_water_s": cls.LOCAL_HIGH_WATER_S,
"low_water_s": cls.LOCAL_LOW_WATER_S,
}
def __init__(
self,
high_water_s: float,
low_water_s: float,
alpha: float = DEFAULT_ALPHA,
warmup_completions: int = DEFAULT_WARMUP_COMPLETIONS,
decay_half_life_s: float = DEFAULT_DECAY_HALF_LIFE_S,
cooldown_s: float = DEFAULT_COOLDOWN_S,
policy: BackpressurePolicy | None = None,
):
super().__init__(policy=policy)
if not (0 < alpha <= 1):
raise ValueError(f"alpha must be in (0, 1], got {alpha}")
if low_water_s > high_water_s:
raise ValueError(
f"low_water_s ({low_water_s}) must be <= high_water_s ({high_water_s})"
)
self._alpha = alpha
self._high = high_water_s
self._low = low_water_s
self._warmup_completions = warmup_completions
self._decay_half_life_s = decay_half_life_s
self._cooldown_s = cooldown_s
# Exponential moving average of store latency in seconds.
self._ema: float = 0.0
# Whether the tier is currently considered under pressure.
self._under_pressure: bool = False
# Number of store completions observed (for warmup gating).
self._completions: int = 0
# Samples collected during warmup; used to seed the EMA.
self._warmup_samples: list[float] = []
# Monotonic timestamp of the last EMA update; used for idle decay.
self._last_update: float = 0.0
# Consecutive completions with EMA below low watermark.
self._healthy_streak: int = 0
# Monotonic timestamp when pressure was last cleared (for cooldown).
self._pressure_cleared_at: float = 0.0
self._cooldown_count: int = 0
def _apply_idle_decay(self) -> None:
"""Decay the EMA toward zero based on elapsed idle time.
Uses the formula ``ema * 0.5^(dt / half_life)`` so the EMA
halves every ``decay_half_life_s`` seconds of silence. This
lets the detector recover from pressure when no stores are
attempted (and therefore no completions arrive to update the
EMA organically).
"""
now = time.monotonic()
dt = now - self._last_update
if dt <= 0 or self._decay_half_life_s <= 0:
return
self._ema *= 0.5 ** (dt / self._decay_half_life_s)
self._last_update = now
def on_store_completed(self, elapsed_s: float, num_bytes: int) -> None:
now = time.monotonic()
s_per_mib = elapsed_s / (num_bytes / self._MIB)
self._completions += 1
if self._completions <= self._warmup_completions:
self._warmup_samples.append(s_per_mib)
if self._completions == self._warmup_completions:
self._ema = sum(self._warmup_samples) / len(self._warmup_samples)
self._last_update = now
return
self._ema = self._alpha * s_per_mib + (1 - self._alpha) * self._ema
self._last_update = now
if self._ema > self._high:
self._under_pressure = True
self._pressure_cleared_at = 0.0
self._healthy_streak = 0
elif self._ema < self._low:
if self._under_pressure:
self._pressure_cleared_at = now
self._cooldown_count = 0
self._under_pressure = False
self._healthy_streak += 1
else:
self._healthy_streak = 0
def is_under_pressure(self) -> bool:
if self._under_pressure and self._completions >= self._warmup_completions:
self._apply_idle_decay()
if self._ema < self._low:
self._under_pressure = False
self._pressure_cleared_at = time.monotonic()
self._cooldown_count = 0
return self._under_pressure
def is_healthy(self) -> bool:
"""True when the tier has been consistently fast.
After ``_HEALTHY_THRESHOLD`` consecutive store completions with
EMA below the low watermark (and no active pressure), the
detector enters a healthy fast-path: ``should_store()`` returns
``True`` immediately, skipping the policy check entirely.
"""
return (
not self._under_pressure and self._healthy_streak >= self._HEALTHY_THRESHOLD
)
def should_store(self, num_blocks: int) -> bool:
under_pressure = self.is_under_pressure()
if self.is_healthy():
return True
if (
not under_pressure
and self._pressure_cleared_at > 0
and time.monotonic() - self._pressure_cleared_at < self._cooldown_s
):
self._cooldown_count += 1
if self._cooldown_count % 2 == 0:
self._policy.on_store_skipped(num_blocks)
return False
return super().should_store(num_blocks)
def reset(self) -> None:
self._ema = 0.0
self._under_pressure = False
self._completions = 0
self._warmup_samples.clear()
self._last_update = 0.0
self._healthy_streak = 0
self._pressure_cleared_at = 0.0
self._cooldown_count = 0
self._policy.reset()
@property
def store_latency_ema(self) -> float:
return self._ema
@store_latency_ema.setter
def store_latency_ema(self, value: float) -> None:
self._ema = value
@property
def stats(self) -> dict[str, float]:
return {"store_latency_ema": self._ema}
+27 -1
View File
@@ -2,9 +2,10 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Abstract interfaces and data types for the secondary tiering layer."""
import time
from abc import ABC, abstractmethod
from collections.abc import Collection, Iterable
from dataclasses import dataclass
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, ClassVar
import numpy as np
@@ -20,6 +21,9 @@ from vllm.v1.kv_offload.base import (
RequestOffloadingContext,
ScheduleEndContext,
)
from vllm.v1.kv_offload.tiering.backpressure import (
BackpressureDetector,
)
if TYPE_CHECKING:
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
@@ -52,6 +56,11 @@ class TieringOffloadingMetrics:
)
ACTIVE_PROMOTION_JOBS = "vllm:kv_offload_tiering_active_promotion_jobs"
ACTIVE_CASCADE_JOBS = "vllm:kv_offload_tiering_active_cascade_jobs"
BACKPRESSURE_STORE_LATENCY_EMA = (
"vllm:kv_offload_tiering_backpressure_store_latency_ema"
)
BACKPRESSURE_STORES_DROPPED = "vllm:kv_offload_tiering_backpressure_stores_dropped"
BACKPRESSURE_BLOCKS_DROPPED = "vllm:kv_offload_tiering_backpressure_blocks_dropped"
@dataclass
@@ -63,6 +72,7 @@ class TransferJob:
chunk_ids: np.ndarray
is_promotion: bool
req_context: ReqContext
submit_time: float = field(default_factory=time.monotonic)
@dataclass
@@ -77,6 +87,7 @@ class JobResult:
# indicated by `success`. Must be a subset of the job's original keys.
successful_keys: Collection[OffloadKey] | None = None
transfer_time: float | None = None
transfer_bytes: int | None = None
class ParentManager(ABC):
@@ -136,18 +147,33 @@ class SecondaryTierManager(ABC):
offloading_spec: "OffloadingSpec",
primary_kv_view: memoryview,
tier_type: str,
backpressure_detector: BackpressureDetector | None = None,
) -> None:
"""Args:
offloading_spec: Offloading configuration.
primary_kv_view: Memoryview of the primary tier's CPU KV cache.
tier_type: Tier type identifier, set by SecondaryTierFactory
from the registered tier type.
backpressure_detector: Optional `BackpressureDetector`.
"""
self._offloading_spec = offloading_spec
self._primary_kv_view: memoryview = primary_kv_view
assert primary_kv_view.strides is not None, (
"primary_kv_view.strides cannot be None"
)
self._block_size_bytes: int = primary_kv_view.strides[0]
self.tier_type = tier_type
self.locality: Locality | None = None
self._bp_detector = backpressure_detector
@property
def block_size_bytes(self) -> int:
return self._block_size_bytes
@property
def bp_detector(self) -> BackpressureDetector | None:
return self._bp_detector
@abstractmethod
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
@@ -31,6 +31,7 @@ logger = logging.getLogger(__name__)
if TYPE_CHECKING:
from vllm.v1.kv_offload.base import OffloadingSpec
from vllm.v1.kv_offload.tiering.backpressure import BackpressureDetector
class ExampleSecondaryTierManager(SecondaryTierManager):
@@ -49,6 +50,7 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
primary_kv_view: memoryview,
tier_type: str,
custom_param: int = 0,
backpressure_detector: "BackpressureDetector | None" = None,
):
"""Initialize the example secondary tier.
@@ -57,12 +59,14 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
primary_kv_view: Memoryview over the primary tier's KV buffer.
tier_type: Name identifying this tier type.
custom_param: Dummy parameter demonstrating custom args.
backpressure_detector: Optional backpressure detector.
"""
super().__init__(
offloading_spec=offloading_spec,
primary_kv_view=primary_kv_view,
tier_type=tier_type,
backpressure_detector=backpressure_detector,
)
logger.info(
+42
View File
@@ -8,11 +8,33 @@ from vllm.logger import init_logger
from vllm.v1.kv_offload.tiering.base import SecondaryTierManager
if TYPE_CHECKING:
from vllm.v1.kv_offload.backpressure import BackpressureDetector
from vllm.v1.kv_offload.base import OffloadingSpec
logger = init_logger(__name__)
_DEFAULT_PACKAGE = "vllm.v1.kv_offload.tiering.backpressure"
def _import_class(name: str) -> type:
"""Import a class by name.
If ``name`` contains a dot it is treated as a fully qualified path
(e.g. ``'my_package.MyDetector'``). Otherwise it is looked up in
the default backpressure module
(``vllm.v1.kv_offload.tiering.backpressure``), so short names like
``'DropStorePolicy'`` or ``'EMABackpressureDetector'`` work out of
the box.
"""
if "." in name:
module_path, _, class_name = name.rpartition(".")
else:
module_path, class_name = _DEFAULT_PACKAGE, name
module = importlib.import_module(module_path)
return getattr(module, class_name)
class SecondaryTierFactory:
"""Registry for SecondaryTierManager implementations, resolved by type.
@@ -47,10 +69,30 @@ class SecondaryTierFactory:
config = tier_config.copy()
tier_type = config.pop("type")
config.pop("module_path", None)
locality = config.get("locality")
bp_config = config.pop("backpressure", None)
bp_detector = None
if bp_config is not None:
bp_config = bp_config.copy()
bp_cls_name = bp_config.pop("backpressure_cls", None)
if bp_cls_name is None:
raise ValueError(
"backpressure config must include 'backpressure_cls' "
"(e.g. 'EMABackpressureDetector')"
)
policy_cls_name = bp_config.pop("policy_cls", None)
policy = _import_class(policy_cls_name)() if policy_cls_name else None
detector_cls: type[BackpressureDetector] = _import_class(bp_cls_name)
defaults = detector_cls.default_config(tier_type, locality=locality)
if defaults:
for k, v in defaults.items():
bp_config.setdefault(k, v)
bp_detector = detector_cls(policy=policy, **bp_config)
return tier_cls(
offloading_spec=offloading_spec,
primary_kv_view=primary_kv_view,
tier_type=tier_type,
backpressure_detector=bp_detector,
**config,
)
+14 -1
View File
@@ -40,6 +40,7 @@ from vllm.v1.kv_offload.base import (
)
from vllm.v1.kv_offload.file_mapper import FileMapper
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager
from vllm.v1.kv_offload.tiering.backpressure import BackpressureDetector
from vllm.v1.kv_offload.tiering.base import (
JobId,
JobResult,
@@ -115,6 +116,7 @@ class FileSystemTierManager(SecondaryTierManager):
n_write_threads: int = 16,
enable_kv_events: bool = False,
locality: str | None = None,
backpressure_detector: BackpressureDetector | None = None,
):
"""Args:
offloading_spec: Contains normalized offloading configuration and
@@ -129,9 +131,12 @@ class FileSystemTierManager(SecondaryTierManager):
cache events are enabled globally (kv_events_config).
locality: Whether this tier's storage is LOCAL or REMOTE relative
to the publishing vLLM instance.
backpressure_detector: Optional backpressure detector.
"""
super().__init__(offloading_spec, primary_kv_view, tier_type)
super().__init__(
offloading_spec, primary_kv_view, tier_type, backpressure_detector
)
self.locality = Locality(locality) if locality is not None else None
self.events: list[OffloadingEvent] | None = None
@@ -150,6 +155,8 @@ class FileSystemTierManager(SecondaryTierManager):
# Keys of in-flight load (promotion) jobs, so a failed load can mark
# its own cached lookup verdicts False (see get_finished_jobs).
self._load_job_keys: dict[JobId, list[OffloadKey]] = {}
# Block count per in-flight job, used to report transfer_bytes.
self._job_block_counts: dict[JobId, int] = {}
# Per load job: how many blocks loaded before a failure (partial keep).
# Written by the pool worker inside the load task before it raises (so
# before task_done publishes the job); read on the scheduler thread in
@@ -225,6 +232,7 @@ class FileSystemTierManager(SecondaryTierManager):
self._block_size,
self._use_o_direct,
)
self._job_block_counts[job_metadata.job_id] = len(keys)
self._pool.enqueue_store(job_metadata.job_id, 1, [task])
@override
@@ -234,6 +242,7 @@ class FileSystemTierManager(SecondaryTierManager):
# keys as a miss (see get_finished_jobs).
keys = list(job_metadata.keys)
self._load_job_keys[job_id] = keys
self._job_block_counts[job_id] = len(keys)
paths = [self.file_mapper.get_file_name(key) for key in keys]
offsets = [int(cid) * self._block_size for cid in job_metadata.chunk_ids]
@@ -272,6 +281,8 @@ class FileSystemTierManager(SecondaryTierManager):
as a miss here (scheduler thread)."""
results = []
for job_id, success, transfer_time in self._pool.get_finished():
block_count = self._job_block_counts.pop(job_id, 0)
transfer_bytes = block_count * self._block_size if block_count else None
if self.events is not None:
keys = self._store_job_keys.pop(job_id, None)
if success and keys:
@@ -299,6 +310,7 @@ class FileSystemTierManager(SecondaryTierManager):
success=False,
successful_keys=tuple(successful) if successful else None,
transfer_time=transfer_time,
transfer_bytes=transfer_bytes,
)
)
continue
@@ -307,6 +319,7 @@ class FileSystemTierManager(SecondaryTierManager):
job_id=job_id,
success=success,
transfer_time=transfer_time,
transfer_bytes=transfer_bytes,
)
)
return results
+51 -1
View File
@@ -238,6 +238,10 @@ class TieringOffloadingManager(OffloadingManager):
for tier_idx, tier in enumerate(self.secondary_tiers)
}
self._tier_index: dict[SecondaryTierManager, int] = {
tier: i for i, tier in enumerate(self.secondary_tiers)
}
@property
def _transfer_jobs(self) -> dict[JobId, JobMetadata]:
return self._jobs
@@ -337,6 +341,43 @@ class TieringOffloadingManager(OffloadingManager):
self.primary_tier.complete_read(
transfer_job.keys, transfer_job.req_context
)
if completed_job.success:
self._update_backpressure(tier, job_metadata, completed_job)
def _should_store_to_tier(
self, tier: SecondaryTierManager, num_blocks: int
) -> bool:
detector = tier.bp_detector
if detector is None:
return True
return detector.should_store(num_blocks)
def _update_backpressure(
self,
tier: SecondaryTierManager,
job_metadata: JobMetadata,
completed_job: JobResult,
) -> None:
detector = tier.bp_detector
if detector is None:
return
was_under_pressure = detector.is_under_pressure()
tj = job_metadata.transfer_job
num_bytes = (
completed_job.transfer_bytes
if completed_job.transfer_bytes is not None
else len(tj.keys) * tier.block_size_bytes
)
detector.update(tj.submit_time, num_bytes)
if detector.is_under_pressure() != was_under_pressure:
tier_idx = self._tier_index[tier]
logger.info(
"Tier #%d (%s) back-pressure %s (stats=%s)",
tier_idx,
tier.tier_type,
"activated" if detector.is_under_pressure() else "cleared",
detector.stats,
)
@override
def lookup(
@@ -643,8 +684,10 @@ class TieringOffloadingManager(OffloadingManager):
return
for tier_idx in request_level_tiers:
job_metadata = self.create_store_job(ready_keys, req_context, tier_idx)
tier = self.secondary_tiers[tier_idx]
if not self._should_store_to_tier(tier, len(ready_keys)):
continue
job_metadata = self.create_store_job(ready_keys, req_context, tier_idx)
tier.submit_store(job_metadata)
def _flush_pending_cascades(self) -> None:
@@ -698,6 +741,8 @@ class TieringOffloadingManager(OffloadingManager):
# eviction during the async transfer). One prepare_read() call per
# secondary tier.
for tier_idx, tier in enumerate(self.secondary_tiers):
if not self._should_store_to_tier(tier, len(keys)):
continue
job_metadata = self.create_store_job(keys, req_context, tier_idx)
tier.submit_store(job_metadata)
@@ -904,6 +949,10 @@ class TieringOffloadingManager(OffloadingManager):
del self._req_state[req_id]
self._processed_jobs_this_step = False
for tier in self.secondary_tiers:
if tier.bp_detector is not None:
tier.bp_detector.reset()
@override
def get_stats(self) -> OffloadingConnectorStats | None:
stats = self.primary_tier.get_stats()
@@ -911,6 +960,7 @@ class TieringOffloadingManager(OffloadingManager):
if stats is not None and stats.is_empty():
stats = None
self._metrics.record_backpressure(self.secondary_tiers)
metrics_stats = self._metrics.take_stats()
if metrics_stats is not None:
if stats is None:
+31
View File
@@ -12,6 +12,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
from vllm.v1.kv_offload.base import LookupResult, OffloadKey, ReqContext
from vllm.v1.kv_offload.tiering.base import (
JobResult,
SecondaryTierManager,
TieringOffloadingMetrics,
TransferJob,
)
@@ -130,6 +131,36 @@ class TieringMetricsTracker:
TieringOffloadingMetrics.PROMOTION_ALLOCATION_FAILURES
)
def record_backpressure(
self,
tiers: list[SecondaryTierManager],
) -> None:
for i, tier in enumerate(tiers):
detector = tier.bp_detector
if detector is None:
continue
label = self.tier_label(i)
ema = detector.stats.get("store_latency_ema")
if ema is not None:
self._stats.set_gauge(
TieringOffloadingMetrics.BACKPRESSURE_STORE_LATENCY_EMA,
ema,
labelvalues=label,
)
stores_dropped, blocks_dropped = detector.policy.pop_stores_dropped()
if stores_dropped > 0:
self._stats.increase_counter(
TieringOffloadingMetrics.BACKPRESSURE_STORES_DROPPED,
stores_dropped,
labelvalues=label,
)
if blocks_dropped > 0:
self._stats.increase_counter(
TieringOffloadingMetrics.BACKPRESSURE_BLOCKS_DROPPED,
blocks_dropped,
labelvalues=label,
)
def take_stats(self) -> OffloadingConnectorStats | None:
active_transfer_stats = OffloadingConnectorStats()
self._observe_active_transfer_stats(active_transfer_stats)
+9 -1
View File
@@ -22,6 +22,7 @@ from vllm.v1.kv_offload.base import (
)
from vllm.v1.kv_offload.file_mapper import FileMapper
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager
from vllm.v1.kv_offload.tiering.backpressure import BackpressureDetector
from vllm.v1.kv_offload.tiering.base import (
JobId,
JobResult,
@@ -112,6 +113,7 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
io_threads: int = 4,
enable_kv_events: bool = False,
locality: str | None = None,
backpressure_detector: BackpressureDetector | None = None,
):
"""Args:
offloading_spec: Offloading configuration.
@@ -125,9 +127,15 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
cache events are enabled globally (kv_events_config).
locality: Whether this tier's storage is LOCAL or REMOTE relative
to the publishing vLLM instance.
backpressure_detector: Optional backpressure detector.
"""
super().__init__(offloading_spec, primary_kv_view, tier_type)
super().__init__(
offloading_spec,
primary_kv_view,
tier_type,
backpressure_detector,
)
self.locality = Locality(locality) if locality is not None else None
self.events: list[OffloadingEvent] | None = None
+19 -1
View File
@@ -265,7 +265,25 @@ class P2PSecondaryTierManager(SecondaryTierManager):
number, or anything convertible to one.
"""
super().__init__(offloading_spec, primary_kv_view, tier_type)
backpressure_detector = kwargs.pop("backpressure_detector", None)
if backpressure_detector is not None:
# The generic (store-latency) detector is unreliable for P2P: in
# PD mode dropping a store can leave the decoder waiting until the
# load timeout instead of failing fast, and rendezvous time makes
# store latency a poor pressure signal. Reject it until a
# P2P-specific fail-fast path exists.
raise ValueError(
"Backpressure is not supported for the P2P secondary tier. "
"The generic store-latency detector cannot fail fast in PD "
"mode and rendezvous time makes store latency an unreliable "
"pressure signal. Remove the 'backpressure' config from the "
"p2p tier."
)
super().__init__(
offloading_spec,
primary_kv_view,
tier_type,
)
try:
timeout_s = float(unbound_store_timeout_s)
except (TypeError, ValueError):
+54
View File
@@ -247,6 +247,34 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
assert isinstance(tier_config, dict)
tier_cls = SecondaryTierFactory.get_tier_class(tier_config)
metrics.update(tier_cls.build_metric_definitions(tier_config))
metrics[TieringOffloadingMetrics.BACKPRESSURE_STORE_LATENCY_EMA] = (
OffloadingGaugeMetadata(
documentation=(
"Exponential moving average of store latency "
"for back-pressure detection, in s/MiB."
),
labelnames=("tier",),
)
)
metrics[TieringOffloadingMetrics.BACKPRESSURE_STORES_DROPPED] = (
OffloadingCounterMetadata(
documentation=(
"Number of store operations dropped due to "
"back-pressure on a secondary tier."
),
labelnames=("tier",),
)
)
metrics[TieringOffloadingMetrics.BACKPRESSURE_BLOCKS_DROPPED] = (
OffloadingCounterMetadata(
documentation=(
"Number of blocks dropped due to back-pressure on a secondary tier."
),
labelnames=("tier",),
)
)
return metrics
def __init__(self, config: OffloadingConfig):
@@ -259,6 +287,32 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
if not isinstance(self.secondary_tier_configs, list):
raise ValueError("secondary_tiers must be a list of tier configurations")
# Backpressure config is merged field-by-field in priority order
# (highest first):
# 1. Per-tier ``backpressure`` dict in the tier config
# 2. Top-level ``backpressure`` in kv_connector_extra_config
# Merging per field (rather than per whole dict) means a partial
# tier override still inherits missing fields from the top-level
# default, so the resolved dict reaching the factory is complete.
# Within each tier's resolved dict, tier-type-aware water marks
# are filled in last so a bare ``"backpressure": {}`` picks up
# sensible thresholds for the storage medium.
bp_defaults = self.extra_config.get("backpressure")
for tier_cfg in self.secondary_tier_configs:
tier_override = tier_cfg.get("backpressure")
# Overlay from lowest to highest precedence so higher-precedence
# fields win while lower-precedence ones fill in the gaps.
merged: dict[str, Any] = {}
for source in (bp_defaults, tier_override):
if source:
merged.update(source)
# Only set a resolved dict when at least one source contributed
# (or an explicit ``backpressure`` key was present, e.g. ``{}``),
# so tiers without any backpressure config stay unconfigured.
if merged or "backpressure" in tier_cfg:
tier_cfg["backpressure"] = merged
# Scheduler-side mmap (rank=None); kept for cleanup
self._scheduler_mmap: SharedOffloadRegion | None = None