[KV Offloading] Back-pressure detection and remediation (#50045)
Signed-off-by: Bill Nell <bnell@redhat.com> Signed-off-by: bnellnm <49004751+bnellnm@users.noreply.github.com> Co-authored-by: Itay Etelis <92247226+Etelis@users.noreply.github.com>
This commit is contained in:
@@ -682,3 +682,92 @@ def test_tiering_accepts_self_describing_kv_events():
|
||||
assert spec.kv_events_config.enable_kv_cache_events
|
||||
assert spec.kv_events_config.self_describing_kv_events
|
||||
assert tracker.self_describing_enabled
|
||||
|
||||
|
||||
def _build_tiering_spec(secondary_tiers, *, top_level_backpressure=None):
|
||||
vllm_config = create_vllm_config(
|
||||
block_size=4,
|
||||
max_num_batched_tokens=16,
|
||||
disable_hybrid_kv_cache_manager=False,
|
||||
)
|
||||
extra_config = {
|
||||
"spec_name": "TieringOffloadingSpec",
|
||||
"cpu_bytes_to_use": 1 << 20,
|
||||
"secondary_tiers": secondary_tiers,
|
||||
}
|
||||
if top_level_backpressure is not None:
|
||||
extra_config["backpressure"] = top_level_backpressure
|
||||
vllm_config.kv_transfer_config = KVTransferConfig(
|
||||
kv_connector="OffloadingConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config=extra_config,
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=0,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["layer"],
|
||||
FullAttentionSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
)
|
||||
],
|
||||
)
|
||||
return TieringOffloadingSpec(build_offloading_config(vllm_config, kv_cache_config))
|
||||
|
||||
|
||||
def test_partial_tier_backpressure_inherits_top_level_defaults():
|
||||
"""A partial tier override still inherits missing fields.
|
||||
|
||||
Field-by-field merge means an override that sets only ``high_water_s``
|
||||
keeps ``backpressure_cls`` (and any other field) from the top-level
|
||||
default, so the resolved dict reaching the factory is complete.
|
||||
"""
|
||||
spec = _build_tiering_spec(
|
||||
[{"type": "example", "backpressure": {"high_water_s": 0.1}}],
|
||||
top_level_backpressure={
|
||||
"backpressure_cls": "EMABackpressureDetector",
|
||||
"high_water_s": 0.5,
|
||||
"low_water_s": 0.05,
|
||||
},
|
||||
)
|
||||
|
||||
resolved = spec.secondary_tier_configs[0]["backpressure"]
|
||||
# Tier override wins for the field it sets.
|
||||
assert resolved["high_water_s"] == 0.1
|
||||
# Missing fields fall back to the top-level default.
|
||||
assert resolved["backpressure_cls"] == "EMABackpressureDetector"
|
||||
assert resolved["low_water_s"] == 0.05
|
||||
|
||||
|
||||
def test_tier_override_wins_over_top_level():
|
||||
"""Precedence is tier override > top-level default, field-by-field."""
|
||||
spec = _build_tiering_spec(
|
||||
[
|
||||
{
|
||||
"type": "example",
|
||||
"backpressure": {"high_water_s": 0.1, "low_water_s": 0.02},
|
||||
}
|
||||
],
|
||||
top_level_backpressure={
|
||||
"backpressure_cls": "EMABackpressureDetector",
|
||||
"high_water_s": 0.5,
|
||||
"low_water_s": 0.05,
|
||||
},
|
||||
)
|
||||
|
||||
resolved = spec.secondary_tier_configs[0]["backpressure"]
|
||||
# Tier override wins for the fields it sets.
|
||||
assert resolved["high_water_s"] == 0.1
|
||||
assert resolved["low_water_s"] == 0.02
|
||||
# Fields only in top-level still fill in.
|
||||
assert resolved["backpressure_cls"] == "EMABackpressureDetector"
|
||||
|
||||
|
||||
def test_tier_without_backpressure_stays_unconfigured():
|
||||
spec = _build_tiering_spec([{"type": "example"}])
|
||||
assert "backpressure" not in spec.secondary_tier_configs[0]
|
||||
|
||||
@@ -180,6 +180,40 @@ class TestInitHashSeed:
|
||||
assert mgr._get_hash_seed() == "random-seed-abc"
|
||||
|
||||
|
||||
class TestBackpressureRejected:
|
||||
def _patch_transports(self, monkeypatch) -> None:
|
||||
monkeypatch.setattr(manager_module, "NixlTransport", lambda *a, **k: object())
|
||||
monkeypatch.setattr(manager_module, "ZmqTransport", lambda *a, **k: object())
|
||||
monkeypatch.setattr(
|
||||
manager_module.FileMapper,
|
||||
"from_offloading_spec",
|
||||
lambda **k: SimpleNamespace(get_run_config=lambda: {}),
|
||||
)
|
||||
|
||||
def test_explicit_backpressure_detector_rejected(self, monkeypatch):
|
||||
"""An explicitly configured detector must fail fast for P2P.
|
||||
|
||||
The generic store-latency detector cannot fail fast in PD mode and
|
||||
rendezvous time makes store latency an unreliable pressure signal,
|
||||
so P2P rejects it rather than degrading silently.
|
||||
"""
|
||||
self._patch_transports(monkeypatch)
|
||||
with pytest.raises(ValueError, match="not supported for the P2P"):
|
||||
P2PSecondaryTierManager(
|
||||
offloading_spec=_init_offloading_spec(),
|
||||
primary_kv_view=memoryview(bytearray(16)),
|
||||
backpressure_detector=object(),
|
||||
)
|
||||
|
||||
def test_no_detector_constructs_normally(self, monkeypatch):
|
||||
self._patch_transports(monkeypatch)
|
||||
mgr = P2PSecondaryTierManager(
|
||||
offloading_spec=_init_offloading_spec(),
|
||||
primary_kv_view=memoryview(bytearray(16)),
|
||||
)
|
||||
assert mgr.bp_detector is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests for _peer_id_from_params
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,706 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for back-pressure detection in TieringOffloadingManager.
|
||||
|
||||
These tests use a DelayedSecondaryTierManager that holds completed jobs
|
||||
until explicitly released, allowing precise control over when the manager
|
||||
observes store completions and their apparent latency.
|
||||
"""
|
||||
|
||||
import time
|
||||
from collections.abc import Iterable
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LookupResult,
|
||||
OffloadKey,
|
||||
OffloadPolicy,
|
||||
ReqContext,
|
||||
RequestOffloadingContext,
|
||||
ScheduleEndContext,
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.backpressure import (
|
||||
DropAccountingPolicy,
|
||||
DropStorePolicy,
|
||||
EMABackpressureDetector,
|
||||
ThrottledDropPolicy,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.base import (
|
||||
JobResult,
|
||||
SecondaryTierManager,
|
||||
TieringOffloadingMetrics,
|
||||
TransferJob,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.manager import (
|
||||
CPUPrimaryTierOffloadingManager,
|
||||
TieringOffloadingManager,
|
||||
)
|
||||
|
||||
_BP_EMA_ALPHA = EMABackpressureDetector.DEFAULT_ALPHA
|
||||
_BP_WARMUP = EMABackpressureDetector.DEFAULT_WARMUP_COMPLETIONS
|
||||
# Water marks in s/MiB, scaled for the test's tiny 16-byte blocks.
|
||||
_BP_HIGH_WATER_S = 1000.0
|
||||
_BP_LOW_WATER_S = 500.0
|
||||
|
||||
_CTX = ReqContext(req_id="test")
|
||||
_MOCK_OFFLOADING_SPEC = MagicMock()
|
||||
|
||||
|
||||
def _mock_mmap_region(num_blocks: int, row_bytes: int = 16):
|
||||
mock = MagicMock()
|
||||
view = memoryview(torch.zeros((num_blocks, row_bytes), dtype=torch.int8).numpy())
|
||||
mock.create_kv_memoryview.return_value = view
|
||||
return mock
|
||||
|
||||
|
||||
def to_keys(int_ids: Iterable[int]) -> list[OffloadKey]:
|
||||
return [make_offload_key(str(i).encode(), 0) for i in int_ids]
|
||||
|
||||
|
||||
class DelayedSecondaryTierManager(SecondaryTierManager):
|
||||
"""Secondary tier that holds completed store jobs until released.
|
||||
|
||||
Jobs are stored immediately but their completion is not reported
|
||||
by get_finished_jobs() until release_jobs() is called. This lets
|
||||
tests control the apparent latency of store operations by adjusting
|
||||
submit_time on the JobMetadata before triggering the completion poll.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, offloading_spec, primary_kv_view, tier_type, backpressure_detector=None
|
||||
):
|
||||
super().__init__(
|
||||
offloading_spec,
|
||||
primary_kv_view,
|
||||
tier_type,
|
||||
backpressure_detector=backpressure_detector,
|
||||
)
|
||||
self.blocks: dict[OffloadKey, bool] = {}
|
||||
self._held_jobs: list[JobResult] = []
|
||||
self._released_jobs: list[JobResult] = []
|
||||
self._request_policy: OffloadPolicy | None = None
|
||||
|
||||
def lookup(self, key, req_context):
|
||||
return LookupResult.HIT if key in self.blocks else LookupResult.MISS
|
||||
|
||||
def submit_store(self, job_metadata: TransferJob) -> None:
|
||||
for key in job_metadata.keys:
|
||||
self.blocks[key] = True
|
||||
self._held_jobs.append(JobResult(job_id=job_metadata.job_id, success=True))
|
||||
|
||||
def submit_load(self, job_metadata: TransferJob) -> None:
|
||||
self._released_jobs.append(JobResult(job_id=job_metadata.job_id, success=True))
|
||||
|
||||
def get_finished_jobs(self) -> Iterable[JobResult]:
|
||||
result = self._released_jobs
|
||||
self._released_jobs = []
|
||||
return result
|
||||
|
||||
def release_jobs(self):
|
||||
self._released_jobs.extend(self._held_jobs)
|
||||
self._held_jobs.clear()
|
||||
|
||||
def on_new_request(self, req_context):
|
||||
if self._request_policy is not None:
|
||||
return RequestOffloadingContext(policy=self._request_policy)
|
||||
return RequestOffloadingContext()
|
||||
|
||||
def drain_jobs(self):
|
||||
self.release_jobs()
|
||||
|
||||
def has_pending_work(self):
|
||||
return bool(self._held_jobs)
|
||||
|
||||
def get_num_blocks(self):
|
||||
return len(self.blocks)
|
||||
|
||||
|
||||
class TestDefaultConfig:
|
||||
def test_fs_gets_local_watermarks(self):
|
||||
cfg = EMABackpressureDetector.default_config("fs")
|
||||
assert cfg["high_water_s"] == EMABackpressureDetector.LOCAL_HIGH_WATER_S
|
||||
assert cfg["low_water_s"] == EMABackpressureDetector.LOCAL_LOW_WATER_S
|
||||
|
||||
def test_obj_gets_network_watermarks(self):
|
||||
cfg = EMABackpressureDetector.default_config("obj")
|
||||
assert cfg["high_water_s"] == EMABackpressureDetector.NETWORK_HIGH_WATER_S
|
||||
assert cfg["low_water_s"] == EMABackpressureDetector.NETWORK_LOW_WATER_S
|
||||
|
||||
def test_p2p_gets_local_watermarks(self):
|
||||
cfg = EMABackpressureDetector.default_config("p2p")
|
||||
assert cfg["high_water_s"] == EMABackpressureDetector.LOCAL_HIGH_WATER_S
|
||||
assert cfg["low_water_s"] == EMABackpressureDetector.LOCAL_LOW_WATER_S
|
||||
|
||||
def test_fs_with_remote_locality_gets_network_watermarks(self):
|
||||
cfg = EMABackpressureDetector.default_config("fs", locality="REMOTE")
|
||||
assert cfg["high_water_s"] == EMABackpressureDetector.NETWORK_HIGH_WATER_S
|
||||
assert cfg["low_water_s"] == EMABackpressureDetector.NETWORK_LOW_WATER_S
|
||||
|
||||
def test_fs_with_local_locality_gets_local_watermarks(self):
|
||||
cfg = EMABackpressureDetector.default_config("fs", locality="LOCAL")
|
||||
assert cfg["high_water_s"] == EMABackpressureDetector.LOCAL_HIGH_WATER_S
|
||||
assert cfg["low_water_s"] == EMABackpressureDetector.LOCAL_LOW_WATER_S
|
||||
|
||||
|
||||
class TestIdleDecay:
|
||||
"""Tests for EMA idle decay — recovery from pressure without completions."""
|
||||
|
||||
def _make_detector(self, **kwargs):
|
||||
defaults = dict(
|
||||
high_water_s=_BP_HIGH_WATER_S,
|
||||
low_water_s=_BP_LOW_WATER_S,
|
||||
decay_half_life_s=2.0,
|
||||
)
|
||||
defaults.update(kwargs)
|
||||
return EMABackpressureDetector(**defaults)
|
||||
|
||||
def test_ema_decays_while_under_pressure(self):
|
||||
bp = self._make_detector()
|
||||
bp._completions = _BP_WARMUP
|
||||
bp._ema = _BP_HIGH_WATER_S * 2
|
||||
bp._under_pressure = True
|
||||
bp._last_update = time.monotonic()
|
||||
|
||||
with patch("time.monotonic", return_value=bp._last_update + 4.1):
|
||||
# ~2.05 half-lives → EMA * 0.24, well below low water (500)
|
||||
assert bp.is_under_pressure() is False
|
||||
assert bp.store_latency_ema < _BP_LOW_WATER_S
|
||||
|
||||
def test_ema_stays_pressured_if_not_enough_decay(self):
|
||||
bp = self._make_detector()
|
||||
bp._completions = _BP_WARMUP
|
||||
bp._ema = _BP_HIGH_WATER_S * 4
|
||||
bp._under_pressure = True
|
||||
bp._last_update = time.monotonic()
|
||||
|
||||
with patch("time.monotonic", return_value=bp._last_update + 2.0):
|
||||
# 2s = 1 half-life → EMA * 0.5 = HIGH * 2, still above LOW
|
||||
assert bp.is_under_pressure() is True
|
||||
assert bp.store_latency_ema == pytest.approx(_BP_HIGH_WATER_S * 2, rel=0.01)
|
||||
|
||||
def test_no_decay_before_warmup(self):
|
||||
bp = self._make_detector()
|
||||
bp._ema = _BP_HIGH_WATER_S * 2
|
||||
bp._under_pressure = True
|
||||
bp._last_update = time.monotonic()
|
||||
|
||||
with patch("time.monotonic", return_value=bp._last_update + 10.0):
|
||||
# Still in warmup (_completions=0 < _warmup_completions),
|
||||
# so decay should not be applied in is_under_pressure.
|
||||
assert bp.is_under_pressure() is True
|
||||
assert bp.store_latency_ema == _BP_HIGH_WATER_S * 2
|
||||
|
||||
def test_no_decay_applied_during_new_sample(self):
|
||||
"""on_store_completed applies the new sample directly without
|
||||
idle decay — decay only runs in is_under_pressure()."""
|
||||
bp = self._make_detector()
|
||||
bp._completions = _BP_WARMUP
|
||||
bp._ema = _BP_HIGH_WATER_S * 2
|
||||
bp._under_pressure = True
|
||||
t0 = time.monotonic()
|
||||
bp._last_update = t0
|
||||
|
||||
block_bytes = 16
|
||||
with patch("time.monotonic", return_value=t0 + 4.0):
|
||||
bp.on_store_completed(0.0, block_bytes)
|
||||
# No decay in on_store_completed: EMA = alpha*0 + (1-alpha)*2000
|
||||
expected = (1 - _BP_EMA_ALPHA) * _BP_HIGH_WATER_S * 2
|
||||
assert bp.store_latency_ema == pytest.approx(expected, rel=0.01)
|
||||
|
||||
def test_reset_clears_last_update(self):
|
||||
bp = self._make_detector()
|
||||
bp._last_update = time.monotonic()
|
||||
bp.reset()
|
||||
assert bp._last_update == 0.0
|
||||
|
||||
|
||||
class TestBackpressure:
|
||||
@pytest.fixture
|
||||
def setup(self):
|
||||
mock_region = _mock_mmap_region(20)
|
||||
self.primary = CPUPrimaryTierOffloadingManager(
|
||||
num_chunks=20, mmap_region=mock_region
|
||||
)
|
||||
mock_view = mock_region.create_kv_memoryview()
|
||||
self.tier = DelayedSecondaryTierManager(
|
||||
offloading_spec=_MOCK_OFFLOADING_SPEC,
|
||||
primary_kv_view=mock_view,
|
||||
tier_type="delayed",
|
||||
backpressure_detector=EMABackpressureDetector(
|
||||
high_water_s=_BP_HIGH_WATER_S,
|
||||
low_water_s=_BP_LOW_WATER_S,
|
||||
),
|
||||
)
|
||||
self.manager = TieringOffloadingManager(
|
||||
primary_tier=self.primary,
|
||||
secondary_tiers=[self.tier],
|
||||
)
|
||||
|
||||
def _start_request(self, ctx=_CTX):
|
||||
if ctx.req_id not in self.manager._req_state:
|
||||
self.manager.on_new_request(ctx)
|
||||
|
||||
def _store_blocks(self, keys, ctx=_CTX):
|
||||
self._start_request(ctx)
|
||||
result = self.manager.prepare_store(keys, ctx)
|
||||
assert result is not None
|
||||
self.manager.complete_store(keys, ctx, success=True)
|
||||
|
||||
def _simulate_on_schedule_end(self):
|
||||
ctx = ScheduleEndContext(new_req_ids=[], preempted_req_ids=())
|
||||
self.manager.on_schedule_end(ctx)
|
||||
|
||||
def _backdate_held_jobs(self, age_s: float):
|
||||
"""Set submit_time on all in-flight store jobs so they appear
|
||||
to have taken ``age_s`` seconds when completed."""
|
||||
self.manager._processed_jobs_this_step = False
|
||||
now = time.monotonic()
|
||||
for job_id, meta in self.manager._jobs.items():
|
||||
tj = meta.transfer_job
|
||||
if not tj.is_promotion and tj.submit_time > 0:
|
||||
tj.submit_time = now - age_s
|
||||
|
||||
def test_ema_updates_on_store_completion(self, setup):
|
||||
bp = self.tier.bp_detector
|
||||
fast_latency = 0.01
|
||||
# EMA is in s/MiB; with 16-byte blocks, scale = MiB / block_bytes.
|
||||
scale = EMABackpressureDetector._MIB / self.tier.block_size_bytes
|
||||
|
||||
# During warmup, EMA stays at 0.
|
||||
for i in range(_BP_WARMUP - 1):
|
||||
keys = to_keys([300 + i])
|
||||
self._store_blocks(keys)
|
||||
self._backdate_held_jobs(fast_latency)
|
||||
self.tier.release_jobs()
|
||||
self._simulate_on_schedule_end()
|
||||
assert bp.store_latency_ema == 0.0
|
||||
|
||||
# Final warmup sample seeds EMA with the mean.
|
||||
keys = to_keys([400])
|
||||
self._store_blocks(keys)
|
||||
self._backdate_held_jobs(fast_latency)
|
||||
self.tier.release_jobs()
|
||||
self._simulate_on_schedule_end()
|
||||
fast_s_per_mib = fast_latency * scale
|
||||
assert bp.store_latency_ema == pytest.approx(fast_s_per_mib, rel=0.1)
|
||||
|
||||
# After warmup, EMA updates normally.
|
||||
new_latency = 0.05
|
||||
keys = to_keys([401])
|
||||
self._store_blocks(keys)
|
||||
self._backdate_held_jobs(new_latency)
|
||||
self.tier.release_jobs()
|
||||
self._simulate_on_schedule_end()
|
||||
new_s_per_mib = new_latency * scale
|
||||
expected = _BP_EMA_ALPHA * new_s_per_mib + (1 - _BP_EMA_ALPHA) * fast_s_per_mib
|
||||
assert bp.store_latency_ema == pytest.approx(expected, rel=0.1)
|
||||
|
||||
def test_pressure_activates_above_high_water(self, setup):
|
||||
bp = self.tier.bp_detector
|
||||
|
||||
# Warm up the detector with fast completions first.
|
||||
for i in range(_BP_WARMUP):
|
||||
keys = to_keys([200 + i])
|
||||
self._store_blocks(keys)
|
||||
self._backdate_held_jobs(0.001)
|
||||
self.tier.release_jobs()
|
||||
self._simulate_on_schedule_end()
|
||||
|
||||
keys = to_keys([100])
|
||||
self._store_blocks(keys)
|
||||
self._simulate_on_schedule_end()
|
||||
|
||||
# 0.1s for a single 16-byte block → ~6553 s/MiB, well above high water.
|
||||
self._backdate_held_jobs(0.1)
|
||||
self.tier.release_jobs()
|
||||
self._simulate_on_schedule_end()
|
||||
|
||||
assert bp.store_latency_ema > _BP_HIGH_WATER_S
|
||||
assert bp.is_under_pressure() is True
|
||||
|
||||
def test_pressure_clears_below_low_water(self, setup):
|
||||
bp = self.tier.bp_detector
|
||||
bp.store_latency_ema = _BP_HIGH_WATER_S * 2
|
||||
bp._under_pressure = True
|
||||
bp._completions = _BP_WARMUP
|
||||
|
||||
# Drive several fast completions to bring EMA below low water.
|
||||
block_id = 100
|
||||
for _ in range(50):
|
||||
if bp.store_latency_ema < _BP_LOW_WATER_S:
|
||||
break
|
||||
keys = to_keys([block_id])
|
||||
block_id += 1
|
||||
self._store_blocks(keys)
|
||||
# Pressure is on, so the cascade was skipped. Force-submit
|
||||
# a store job manually to get a completion to feed the EMA.
|
||||
job_meta = self.manager.create_store_job(keys, _CTX)
|
||||
job_meta.submit_time = time.monotonic() - 0.001
|
||||
self.tier.submit_store(job_meta)
|
||||
self.tier.release_jobs()
|
||||
self._simulate_on_schedule_end()
|
||||
else:
|
||||
pytest.fail(f"EMA did not recover below low water: {bp.store_latency_ema}")
|
||||
|
||||
assert bp.is_under_pressure() is False
|
||||
|
||||
def test_hysteresis_prevents_oscillation(self, setup):
|
||||
bp = self.tier.bp_detector
|
||||
bp._completions = _BP_WARMUP
|
||||
block_bytes = self.tier.block_size_bytes
|
||||
|
||||
# Set EMA between low and high water marks (in s/MiB).
|
||||
mid = (_BP_LOW_WATER_S + _BP_HIGH_WATER_S) / 2
|
||||
bp.store_latency_ema = mid
|
||||
bp._last_update = time.monotonic()
|
||||
|
||||
# Raw seconds that produce `mid` s/MiB for a given number of blocks.
|
||||
def mid_latency_s(num_blocks: int) -> float:
|
||||
return mid * (num_blocks * block_bytes / EMABackpressureDetector._MIB)
|
||||
|
||||
# When not under pressure, mid-range EMA should not activate.
|
||||
bp._under_pressure = False
|
||||
keys = to_keys(range(2))
|
||||
self._store_blocks(keys)
|
||||
self._backdate_held_jobs(mid_latency_s(len(keys)))
|
||||
self.tier.release_jobs()
|
||||
self._simulate_on_schedule_end()
|
||||
assert bp.is_under_pressure() is False
|
||||
|
||||
# When under pressure, mid-range EMA should not deactivate.
|
||||
bp._under_pressure = True
|
||||
bp._last_update = time.monotonic()
|
||||
keys2 = to_keys(range(10, 12))
|
||||
# Force-submit since pressure is on (cascade would skip).
|
||||
for k in keys2:
|
||||
self.primary.prepare_store([k], _CTX)
|
||||
self.primary.complete_store([k], _CTX)
|
||||
job_meta = self.manager.create_store_job(keys2, _CTX)
|
||||
job_meta.submit_time = time.monotonic() - mid_latency_s(len(keys2))
|
||||
self.tier.submit_store(job_meta)
|
||||
self.tier.release_jobs()
|
||||
self._simulate_on_schedule_end()
|
||||
assert bp.is_under_pressure() is True
|
||||
|
||||
def test_stores_skipped_under_pressure(self, setup):
|
||||
bp = self.tier.bp_detector
|
||||
bp._under_pressure = True
|
||||
bp.store_latency_ema = _BP_HIGH_WATER_S * 4
|
||||
initial_blocks = self.tier.get_num_blocks()
|
||||
|
||||
keys = to_keys(range(50, 53))
|
||||
self._store_blocks(keys)
|
||||
|
||||
# Blocks should be in primary but NOT in secondary.
|
||||
for k in keys:
|
||||
assert self.primary.lookup(k, _CTX) is LookupResult.HIT
|
||||
assert self.tier.get_num_blocks() == initial_blocks
|
||||
|
||||
def test_stores_resume_after_pressure_clears(self, setup):
|
||||
bp = self.tier.bp_detector
|
||||
|
||||
# Start under pressure — stores skipped.
|
||||
bp._under_pressure = True
|
||||
bp.store_latency_ema = _BP_HIGH_WATER_S * 4
|
||||
keys1 = to_keys(range(60, 62))
|
||||
self._store_blocks(keys1)
|
||||
assert all(k not in self.tier.blocks for k in keys1)
|
||||
|
||||
# Clear pressure — next store should cascade.
|
||||
bp._under_pressure = False
|
||||
keys2 = to_keys(range(70, 72))
|
||||
self._store_blocks(keys2)
|
||||
assert all(k in self.tier.blocks for k in keys2)
|
||||
|
||||
def test_dropped_store_count_tracked(self, setup):
|
||||
bp = self.tier.bp_detector
|
||||
bp._under_pressure = True
|
||||
bp.store_latency_ema = _BP_HIGH_WATER_S * 4
|
||||
policy = bp.policy
|
||||
assert policy.pop_stores_dropped() == (0, 0)
|
||||
|
||||
self._store_blocks(to_keys([80]))
|
||||
assert policy._stores_dropped == 1
|
||||
assert policy._blocks_dropped == 1
|
||||
|
||||
self._store_blocks(to_keys([81, 82]))
|
||||
assert policy._stores_dropped == 2
|
||||
assert policy._blocks_dropped == 3
|
||||
|
||||
def test_metrics_reported_via_get_stats(self, setup):
|
||||
bp = self.tier.bp_detector
|
||||
bp._under_pressure = True
|
||||
bp.store_latency_ema = 2.5
|
||||
policy = bp.policy
|
||||
policy._stores_dropped = 5
|
||||
policy._blocks_dropped = 12
|
||||
|
||||
stats = self.manager.get_stats()
|
||||
assert stats is not None
|
||||
reduced = stats.reduce()
|
||||
|
||||
ema_key = TieringOffloadingMetrics.BACKPRESSURE_STORE_LATENCY_EMA
|
||||
stores_key = TieringOffloadingMetrics.BACKPRESSURE_STORES_DROPPED
|
||||
blocks_key = TieringOffloadingMetrics.BACKPRESSURE_BLOCKS_DROPPED
|
||||
assert reduced[f"{ema_key}:('1:delayed',)"] == pytest.approx(2.5)
|
||||
assert reduced[f"{stores_key}:('1:delayed',)"] == 5
|
||||
assert reduced[f"{blocks_key}:('1:delayed',)"] == 12
|
||||
|
||||
# Dropped counts reset after get_stats.
|
||||
assert policy._stores_dropped == 0
|
||||
assert policy._blocks_dropped == 0
|
||||
|
||||
def test_reset_cache_clears_backpressure(self, setup):
|
||||
bp = self.tier.bp_detector
|
||||
bp.store_latency_ema = 5.0
|
||||
bp._under_pressure = True
|
||||
|
||||
self.manager.reset_cache()
|
||||
|
||||
assert bp.store_latency_ema == 0.0
|
||||
assert bp.is_under_pressure() is False
|
||||
|
||||
def test_request_level_tier_respects_backpressure(self, setup):
|
||||
"""Request-level cascade also skips pressured tiers."""
|
||||
# Store blocks in primary first.
|
||||
existing = to_keys(range(3))
|
||||
self._store_blocks(existing)
|
||||
self.tier.release_jobs()
|
||||
self._simulate_on_schedule_end()
|
||||
self._simulate_on_schedule_end()
|
||||
|
||||
# Make tier request-level for a new request.
|
||||
self.tier._request_policy = OffloadPolicy.REQUEST_LEVEL
|
||||
ctx = ReqContext(req_id="req_rl_bp")
|
||||
self.manager.on_new_request(ctx)
|
||||
|
||||
# Activate pressure.
|
||||
bp = self.tier.bp_detector
|
||||
bp._under_pressure = True
|
||||
bp.store_latency_ema = _BP_HIGH_WATER_S * 4
|
||||
initial_blocks = self.tier.get_num_blocks()
|
||||
|
||||
# prepare_store with existing + new blocks.
|
||||
new = to_keys(range(10, 12))
|
||||
all_keys = existing + new
|
||||
result = self.manager.prepare_store(all_keys, ctx)
|
||||
assert result is not None
|
||||
|
||||
# Existing blocks would normally cascade to request-level tier,
|
||||
# but should be skipped under pressure.
|
||||
assert self.tier.get_num_blocks() == initial_blocks
|
||||
|
||||
def test_loads_continue_under_pressure(self, setup):
|
||||
"""Loads from a pressured tier still work."""
|
||||
blocks = to_keys(range(3))
|
||||
for b in blocks:
|
||||
self.tier.blocks[b] = True
|
||||
|
||||
bp = self.tier.bp_detector
|
||||
bp._under_pressure = True
|
||||
bp.store_latency_ema = _BP_HIGH_WATER_S * 4
|
||||
|
||||
# Lookups should still initiate promotion.
|
||||
for b in blocks:
|
||||
result = self.manager.lookup(b, _CTX)
|
||||
assert result is LookupResult.HIT_PENDING
|
||||
|
||||
|
||||
class TestThrottledDropPolicy:
|
||||
"""Tests for proportional throttling under pressure."""
|
||||
|
||||
def _make_detector(self, **kwargs):
|
||||
defaults = dict(
|
||||
high_water_s=_BP_HIGH_WATER_S,
|
||||
low_water_s=_BP_LOW_WATER_S,
|
||||
cooldown_s=0.0,
|
||||
)
|
||||
defaults.update(kwargs)
|
||||
return EMABackpressureDetector(**defaults)
|
||||
|
||||
def test_no_drop_when_not_under_pressure(self):
|
||||
bp = self._make_detector()
|
||||
for _ in range(20):
|
||||
assert bp.should_store(1) is True
|
||||
|
||||
def test_no_drop_at_exactly_high_water(self):
|
||||
bp = self._make_detector()
|
||||
bp._under_pressure = True
|
||||
bp.store_latency_ema = _BP_HIGH_WATER_S
|
||||
for _ in range(20):
|
||||
assert bp.should_store(1) is True
|
||||
|
||||
def test_partial_drop_at_moderate_pressure(self):
|
||||
bp = self._make_detector()
|
||||
bp._under_pressure = True
|
||||
bp.store_latency_ema = _BP_HIGH_WATER_S * 2
|
||||
results = [bp.should_store(1) for _ in range(20)]
|
||||
assert any(results), "some stores should be allowed"
|
||||
assert not all(results), "some stores should be dropped"
|
||||
|
||||
def test_full_drop_at_severe_pressure(self):
|
||||
bp = self._make_detector()
|
||||
bp._under_pressure = True
|
||||
bp.store_latency_ema = _BP_HIGH_WATER_S * 4
|
||||
for _ in range(20):
|
||||
assert bp.should_store(1) is False
|
||||
|
||||
def test_drop_rate_increases_with_severity(self):
|
||||
"""Higher EMA → higher drop rate."""
|
||||
drop_counts = []
|
||||
for multiplier in [1.5, 2.0, 2.5, 3.5]:
|
||||
bp = self._make_detector()
|
||||
bp._under_pressure = True
|
||||
bp.store_latency_ema = _BP_HIGH_WATER_S * multiplier
|
||||
drops = sum(1 for _ in range(100) if not bp.should_store(1))
|
||||
drop_counts.append(drops)
|
||||
for i in range(len(drop_counts) - 1):
|
||||
assert drop_counts[i] <= drop_counts[i + 1]
|
||||
|
||||
def test_explicit_drop_store_policy_still_drops_all(self):
|
||||
"""DropStorePolicy (explicit) still drops everything under pressure."""
|
||||
bp = EMABackpressureDetector(
|
||||
high_water_s=_BP_HIGH_WATER_S,
|
||||
low_water_s=_BP_LOW_WATER_S,
|
||||
policy=DropStorePolicy(),
|
||||
cooldown_s=0.0,
|
||||
)
|
||||
bp._under_pressure = True
|
||||
bp.store_latency_ema = _BP_HIGH_WATER_S * 1.5
|
||||
for _ in range(20):
|
||||
assert bp.should_store(1) is False
|
||||
|
||||
|
||||
class TestSharedDropAccounting:
|
||||
"""Both policies share drop accounting via DropAccountingPolicy."""
|
||||
|
||||
@pytest.mark.parametrize("policy_cls", [DropStorePolicy, ThrottledDropPolicy])
|
||||
def test_shared_base(self, policy_cls):
|
||||
assert issubclass(policy_cls, DropAccountingPolicy)
|
||||
|
||||
@pytest.mark.parametrize("policy_cls", [DropStorePolicy, ThrottledDropPolicy])
|
||||
def test_pop_returns_and_resets(self, policy_cls):
|
||||
policy = policy_cls()
|
||||
policy.on_store_skipped(3)
|
||||
policy.on_store_skipped(2)
|
||||
assert policy.pop_stores_dropped() == (2, 5)
|
||||
assert policy.pop_stores_dropped() == (0, 0)
|
||||
|
||||
@pytest.mark.parametrize("policy_cls", [DropStorePolicy, ThrottledDropPolicy])
|
||||
def test_reset_clears_counters(self, policy_cls):
|
||||
policy = policy_cls()
|
||||
policy.on_store_skipped(4)
|
||||
policy.reset()
|
||||
assert policy.pop_stores_dropped() == (0, 0)
|
||||
|
||||
def test_throttled_reset_also_clears_call_count(self):
|
||||
policy = ThrottledDropPolicy()
|
||||
policy._call_count = 7
|
||||
policy.reset()
|
||||
assert policy._call_count == 0
|
||||
|
||||
|
||||
class TestHealthyBypass:
|
||||
"""Tests for the fast-path bypass when a tier is consistently healthy."""
|
||||
|
||||
def _make_detector(self, **kwargs):
|
||||
defaults = dict(
|
||||
high_water_s=_BP_HIGH_WATER_S,
|
||||
low_water_s=_BP_LOW_WATER_S,
|
||||
cooldown_s=0.0,
|
||||
)
|
||||
defaults.update(kwargs)
|
||||
return EMABackpressureDetector(**defaults)
|
||||
|
||||
def test_not_healthy_initially(self):
|
||||
bp = self._make_detector()
|
||||
assert bp.is_healthy() is False
|
||||
|
||||
def test_healthy_after_streak(self):
|
||||
bp = self._make_detector()
|
||||
bp._completions = _BP_WARMUP
|
||||
bp._last_update = time.monotonic()
|
||||
block_bytes = 16
|
||||
fast_s_per_mib = _BP_LOW_WATER_S * 0.5
|
||||
fast_elapsed = fast_s_per_mib * (block_bytes / EMABackpressureDetector._MIB)
|
||||
for _ in range(EMABackpressureDetector._HEALTHY_THRESHOLD + _BP_WARMUP):
|
||||
bp.on_store_completed(fast_elapsed, block_bytes)
|
||||
assert bp.is_healthy() is True
|
||||
|
||||
def test_healthy_resets_on_high_latency(self):
|
||||
bp = self._make_detector()
|
||||
bp._completions = _BP_WARMUP
|
||||
bp._healthy_streak = EMABackpressureDetector._HEALTHY_THRESHOLD
|
||||
bp._ema = _BP_LOW_WATER_S
|
||||
bp._last_update = time.monotonic()
|
||||
block_bytes = 16
|
||||
slow_s_per_mib = _BP_HIGH_WATER_S * 2
|
||||
slow_elapsed = slow_s_per_mib * (block_bytes / EMABackpressureDetector._MIB)
|
||||
bp.on_store_completed(slow_elapsed, block_bytes)
|
||||
assert bp.is_healthy() is False
|
||||
|
||||
def test_healthy_bypasses_policy(self):
|
||||
bp = self._make_detector()
|
||||
bp._completions = _BP_WARMUP
|
||||
bp._healthy_streak = EMABackpressureDetector._HEALTHY_THRESHOLD
|
||||
assert bp.should_store(1) is True
|
||||
|
||||
def test_reset_clears_healthy_streak(self):
|
||||
bp = self._make_detector()
|
||||
bp._healthy_streak = EMABackpressureDetector._HEALTHY_THRESHOLD
|
||||
bp.reset()
|
||||
assert bp._healthy_streak == 0
|
||||
assert bp.is_healthy() is False
|
||||
|
||||
|
||||
class TestCooldown:
|
||||
"""Tests for the post-pressure cooldown period."""
|
||||
|
||||
def _make_detector(self, **kwargs):
|
||||
defaults = dict(
|
||||
high_water_s=_BP_HIGH_WATER_S,
|
||||
low_water_s=_BP_LOW_WATER_S,
|
||||
cooldown_s=1.0,
|
||||
)
|
||||
defaults.update(kwargs)
|
||||
return EMABackpressureDetector(**defaults)
|
||||
|
||||
def test_cooldown_throttles_stores(self):
|
||||
bp = self._make_detector()
|
||||
bp._completions = _BP_WARMUP
|
||||
bp._under_pressure = True
|
||||
bp._ema = _BP_HIGH_WATER_S * 2
|
||||
bp._last_update = time.monotonic()
|
||||
|
||||
with patch("time.monotonic") as mock_time:
|
||||
t0 = bp._last_update + 10.0
|
||||
mock_time.return_value = t0
|
||||
assert bp.is_under_pressure() is False
|
||||
assert bp._pressure_cleared_at > 0
|
||||
|
||||
mock_time.return_value = t0 + 0.1
|
||||
results = [bp.should_store(1) for _ in range(10)]
|
||||
assert sum(results) == 5, "cooldown should allow ~50%"
|
||||
|
||||
def test_no_cooldown_when_never_pressured(self):
|
||||
bp = self._make_detector()
|
||||
for _ in range(20):
|
||||
assert bp.should_store(1) is True
|
||||
|
||||
def test_no_cooldown_after_window_expires(self):
|
||||
bp = self._make_detector()
|
||||
bp._pressure_cleared_at = time.monotonic() - 2.0
|
||||
for _ in range(20):
|
||||
assert bp.should_store(1) is True
|
||||
|
||||
def test_reset_clears_cooldown(self):
|
||||
bp = self._make_detector()
|
||||
bp._pressure_cleared_at = time.monotonic()
|
||||
bp._cooldown_count = 5
|
||||
bp.reset()
|
||||
assert bp._pressure_cleared_at == 0.0
|
||||
assert bp._cooldown_count == 0
|
||||
@@ -0,0 +1,384 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
|
||||
class BackpressurePolicy(ABC):
|
||||
"""Decides what action to take when a detector signals pressure."""
|
||||
|
||||
@abstractmethod
|
||||
def should_store(self, detector: BackpressureDetector) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
def on_store_skipped(self, num_blocks: int) -> None: ...
|
||||
|
||||
@abstractmethod
|
||||
def pop_stores_dropped(self) -> tuple[int, int]:
|
||||
"""Return and reset (stores_dropped, blocks_dropped)."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def reset(self) -> None: ...
|
||||
|
||||
|
||||
class DropAccountingPolicy(BackpressurePolicy):
|
||||
"""Base policy that tracks dropped store/block counters.
|
||||
|
||||
Subclasses implement ``should_store``; the shared drop accounting
|
||||
(``on_store_skipped``/``pop_stores_dropped``/``reset``) lives here so
|
||||
every policy reports these counters identically. Subclasses that add
|
||||
their own state override ``reset`` and call ``super().reset()``.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._stores_dropped: int = 0
|
||||
self._blocks_dropped: int = 0
|
||||
|
||||
def on_store_skipped(self, num_blocks: int) -> None:
|
||||
self._stores_dropped += 1
|
||||
self._blocks_dropped += num_blocks
|
||||
|
||||
def pop_stores_dropped(self) -> tuple[int, int]:
|
||||
stores, blocks = self._stores_dropped, self._blocks_dropped
|
||||
self._stores_dropped = 0
|
||||
self._blocks_dropped = 0
|
||||
return stores, blocks
|
||||
|
||||
def reset(self) -> None:
|
||||
self._stores_dropped = 0
|
||||
self._blocks_dropped = 0
|
||||
|
||||
|
||||
class DropStorePolicy(DropAccountingPolicy):
|
||||
"""Silently drop stores to pressured tiers."""
|
||||
|
||||
def should_store(self, detector) -> bool:
|
||||
return not detector.is_under_pressure()
|
||||
|
||||
|
||||
class ThrottledDropPolicy(DropAccountingPolicy):
|
||||
"""Drop stores proportionally to pressure severity.
|
||||
|
||||
Instead of dropping all stores when pressure is detected, the drop
|
||||
rate scales with how far the EMA exceeds the high watermark::
|
||||
|
||||
overshoot = (ema - high_water) / high_water
|
||||
drop_rate = min(1.0, overshoot / ramp_factor)
|
||||
|
||||
With the default ``ramp_factor=2.0``, stores ramp from 0% drop at
|
||||
the high watermark to 100% drop at 3x the high watermark. This
|
||||
keeps secondary tiers populated under moderate pressure, reducing
|
||||
cache misses (and therefore recomputation / ITL regression) while
|
||||
still protecting TTFT under severe congestion.
|
||||
"""
|
||||
|
||||
def __init__(self, ramp_factor: float = 2.0):
|
||||
super().__init__()
|
||||
self._call_count: int = 0
|
||||
self._ramp_factor = ramp_factor
|
||||
|
||||
def should_store(self, detector: BackpressureDetector) -> bool:
|
||||
# Proportional throttling needs the EMA and high watermark, which
|
||||
# only the EMA detector exposes.
|
||||
assert isinstance(detector, EMABackpressureDetector), (
|
||||
"ThrottledDropPolicy requires an EMABackpressureDetector"
|
||||
)
|
||||
if not detector.is_under_pressure():
|
||||
return True
|
||||
ema = detector.store_latency_ema
|
||||
high = detector._high
|
||||
if high <= 0:
|
||||
return False
|
||||
overshoot = max(0.0, ema - high) / high
|
||||
drop_rate = min(1.0, overshoot / self._ramp_factor)
|
||||
if drop_rate <= 0:
|
||||
return True
|
||||
if drop_rate >= 1.0:
|
||||
return False
|
||||
self._call_count += 1
|
||||
period = max(2, round(1.0 / drop_rate))
|
||||
return (self._call_count % period) != 0
|
||||
|
||||
def reset(self) -> None:
|
||||
super().reset()
|
||||
self._call_count = 0
|
||||
|
||||
|
||||
class BackpressureDetector(ABC):
|
||||
"""Observes store completion signals and determines pressure state."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
policy: BackpressurePolicy | None = None,
|
||||
):
|
||||
self._policy = policy or ThrottledDropPolicy()
|
||||
|
||||
@property
|
||||
def policy(self) -> BackpressurePolicy:
|
||||
return self._policy
|
||||
|
||||
@abstractmethod
|
||||
def on_store_completed(self, elapsed_s: float, num_bytes: int) -> None: ...
|
||||
|
||||
@abstractmethod
|
||||
def is_under_pressure(self) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
def reset(self) -> None: ...
|
||||
|
||||
def update(self, submit_time: float, num_bytes: int) -> None:
|
||||
"""Update pressure state from a completed store job.
|
||||
|
||||
Args:
|
||||
submit_time: ``time.monotonic()`` when the job was submitted.
|
||||
num_bytes: Total bytes written (num_blocks * block_size_bytes).
|
||||
|
||||
"""
|
||||
if num_bytes <= 0:
|
||||
return
|
||||
elapsed = time.monotonic() - submit_time
|
||||
self.on_store_completed(elapsed, num_bytes)
|
||||
|
||||
def should_store(self, num_blocks: int) -> bool:
|
||||
"""Check policy and record skip if rejected."""
|
||||
if not self._policy.should_store(self):
|
||||
self._policy.on_store_skipped(num_blocks)
|
||||
return False
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def default_config(
|
||||
cls, tier_type: str, *, locality: str | None = None
|
||||
) -> dict | None:
|
||||
"""Return default constructor kwargs for ``tier_type``, or None.
|
||||
|
||||
Args:
|
||||
tier_type: The tier type string (e.g. ``"fs"``, ``"obj"``).
|
||||
locality: Optional locality hint from the tier config
|
||||
(``"LOCAL"`` or ``"REMOTE"``). When ``"REMOTE"``, tiers
|
||||
that would normally get local-storage watermarks (e.g.
|
||||
``"fs"``) receive network watermarks instead.
|
||||
|
||||
"""
|
||||
return None
|
||||
|
||||
@property
|
||||
def stats(self) -> dict[str, float]:
|
||||
return {}
|
||||
|
||||
|
||||
class EMABackpressureDetector(BackpressureDetector):
|
||||
"""EMA of store latency normalized by transfer size.
|
||||
|
||||
The EMA tracks seconds per megabyte (s/MiB) so that the metric is
|
||||
comparable regardless of how many blocks are in a job or how large
|
||||
each block is. Water marks are in the same unit.
|
||||
|
||||
Default water marks are derived from fio benchmarks on the WDC H100
|
||||
cluster. Two presets are provided:
|
||||
|
||||
LOCAL (NVMe/SSD): NVMe sustains ~5 GB/s writes.
|
||||
high=0.005 s/MiB (~200 MB/s) catches severe congestion;
|
||||
low=0.001 s/MiB (~1 GB/s) requires meaningful recovery.
|
||||
|
||||
NETWORK (CephFS, object store, ``obj``tiers, or any tier
|
||||
with ``"locality": "REMOTE"``): CephFS sustains ~1.5 GB/s.
|
||||
high=0.020 s/MiB (~50 MB/s); low=0.005 s/MiB (~200 MB/s).
|
||||
|
||||
``obj`` tiers get NETWORK defaults automatically. An
|
||||
``fs`` tier defaults to LOCAL (appropriate for NVMe/SSD); set
|
||||
``"locality": "REMOTE"`` in the tier config for network-backed
|
||||
filesystems like CephFS.
|
||||
"""
|
||||
|
||||
_MIB = 1 << 20
|
||||
|
||||
# EMA smoothing factor: higher values (→1) react faster to latency
|
||||
# spikes but are noisier; lower values (→0) smooth more but lag.
|
||||
DEFAULT_ALPHA = 0.15
|
||||
|
||||
# Number of store completions to collect before seeding the EMA.
|
||||
# During warmup, pressure is never signalled; the EMA is initialized
|
||||
# to the mean of the warmup samples to avoid cold-start false positives.
|
||||
DEFAULT_WARMUP_COMPLETIONS = 10
|
||||
|
||||
# When under pressure and no completions arrive, the EMA decays
|
||||
# toward zero so the detector can eventually recover. The half-life
|
||||
# controls how quickly: after this many seconds of silence the EMA
|
||||
# halves. With a 2 s half-life and high_water=0.005, an EMA frozen
|
||||
# at 0.010 reaches low_water=0.001 in ~6.6 s.
|
||||
DEFAULT_DECAY_HALF_LIFE_S = 2.0
|
||||
|
||||
# After pressure clears, stores are throttled to 50% for this many
|
||||
# seconds to prevent a burst of stores from re-overwhelming the tier.
|
||||
DEFAULT_COOLDOWN_S = 1.0
|
||||
|
||||
# Number of consecutive sub-low-watermark completions before the
|
||||
# detector enters a "healthy" fast-path that bypasses policy checks.
|
||||
_HEALTHY_THRESHOLD = 10
|
||||
|
||||
LOCAL_HIGH_WATER_S = 0.010
|
||||
LOCAL_LOW_WATER_S = 0.002
|
||||
|
||||
NETWORK_HIGH_WATER_S = 0.050
|
||||
NETWORK_LOW_WATER_S = 0.010
|
||||
|
||||
_NETWORK_TIER_TYPES = frozenset({"obj"})
|
||||
|
||||
@classmethod
|
||||
def default_config(
|
||||
cls, tier_type: str, *, locality: str | None = None
|
||||
) -> dict | None:
|
||||
is_remote = locality is not None and locality.upper() == "REMOTE"
|
||||
if is_remote or tier_type in cls._NETWORK_TIER_TYPES:
|
||||
return {
|
||||
"high_water_s": cls.NETWORK_HIGH_WATER_S,
|
||||
"low_water_s": cls.NETWORK_LOW_WATER_S,
|
||||
}
|
||||
return {
|
||||
"high_water_s": cls.LOCAL_HIGH_WATER_S,
|
||||
"low_water_s": cls.LOCAL_LOW_WATER_S,
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
high_water_s: float,
|
||||
low_water_s: float,
|
||||
alpha: float = DEFAULT_ALPHA,
|
||||
warmup_completions: int = DEFAULT_WARMUP_COMPLETIONS,
|
||||
decay_half_life_s: float = DEFAULT_DECAY_HALF_LIFE_S,
|
||||
cooldown_s: float = DEFAULT_COOLDOWN_S,
|
||||
policy: BackpressurePolicy | None = None,
|
||||
):
|
||||
super().__init__(policy=policy)
|
||||
if not (0 < alpha <= 1):
|
||||
raise ValueError(f"alpha must be in (0, 1], got {alpha}")
|
||||
if low_water_s > high_water_s:
|
||||
raise ValueError(
|
||||
f"low_water_s ({low_water_s}) must be <= high_water_s ({high_water_s})"
|
||||
)
|
||||
|
||||
self._alpha = alpha
|
||||
self._high = high_water_s
|
||||
self._low = low_water_s
|
||||
self._warmup_completions = warmup_completions
|
||||
self._decay_half_life_s = decay_half_life_s
|
||||
self._cooldown_s = cooldown_s
|
||||
# Exponential moving average of store latency in seconds.
|
||||
self._ema: float = 0.0
|
||||
# Whether the tier is currently considered under pressure.
|
||||
self._under_pressure: bool = False
|
||||
# Number of store completions observed (for warmup gating).
|
||||
self._completions: int = 0
|
||||
# Samples collected during warmup; used to seed the EMA.
|
||||
self._warmup_samples: list[float] = []
|
||||
# Monotonic timestamp of the last EMA update; used for idle decay.
|
||||
self._last_update: float = 0.0
|
||||
# Consecutive completions with EMA below low watermark.
|
||||
self._healthy_streak: int = 0
|
||||
# Monotonic timestamp when pressure was last cleared (for cooldown).
|
||||
self._pressure_cleared_at: float = 0.0
|
||||
self._cooldown_count: int = 0
|
||||
|
||||
def _apply_idle_decay(self) -> None:
|
||||
"""Decay the EMA toward zero based on elapsed idle time.
|
||||
|
||||
Uses the formula ``ema * 0.5^(dt / half_life)`` so the EMA
|
||||
halves every ``decay_half_life_s`` seconds of silence. This
|
||||
lets the detector recover from pressure when no stores are
|
||||
attempted (and therefore no completions arrive to update the
|
||||
EMA organically).
|
||||
"""
|
||||
now = time.monotonic()
|
||||
dt = now - self._last_update
|
||||
if dt <= 0 or self._decay_half_life_s <= 0:
|
||||
return
|
||||
self._ema *= 0.5 ** (dt / self._decay_half_life_s)
|
||||
self._last_update = now
|
||||
|
||||
def on_store_completed(self, elapsed_s: float, num_bytes: int) -> None:
|
||||
now = time.monotonic()
|
||||
s_per_mib = elapsed_s / (num_bytes / self._MIB)
|
||||
self._completions += 1
|
||||
if self._completions <= self._warmup_completions:
|
||||
self._warmup_samples.append(s_per_mib)
|
||||
if self._completions == self._warmup_completions:
|
||||
self._ema = sum(self._warmup_samples) / len(self._warmup_samples)
|
||||
self._last_update = now
|
||||
return
|
||||
self._ema = self._alpha * s_per_mib + (1 - self._alpha) * self._ema
|
||||
self._last_update = now
|
||||
if self._ema > self._high:
|
||||
self._under_pressure = True
|
||||
self._pressure_cleared_at = 0.0
|
||||
self._healthy_streak = 0
|
||||
elif self._ema < self._low:
|
||||
if self._under_pressure:
|
||||
self._pressure_cleared_at = now
|
||||
self._cooldown_count = 0
|
||||
self._under_pressure = False
|
||||
self._healthy_streak += 1
|
||||
else:
|
||||
self._healthy_streak = 0
|
||||
|
||||
def is_under_pressure(self) -> bool:
|
||||
if self._under_pressure and self._completions >= self._warmup_completions:
|
||||
self._apply_idle_decay()
|
||||
if self._ema < self._low:
|
||||
self._under_pressure = False
|
||||
self._pressure_cleared_at = time.monotonic()
|
||||
self._cooldown_count = 0
|
||||
return self._under_pressure
|
||||
|
||||
def is_healthy(self) -> bool:
|
||||
"""True when the tier has been consistently fast.
|
||||
|
||||
After ``_HEALTHY_THRESHOLD`` consecutive store completions with
|
||||
EMA below the low watermark (and no active pressure), the
|
||||
detector enters a healthy fast-path: ``should_store()`` returns
|
||||
``True`` immediately, skipping the policy check entirely.
|
||||
"""
|
||||
return (
|
||||
not self._under_pressure and self._healthy_streak >= self._HEALTHY_THRESHOLD
|
||||
)
|
||||
|
||||
def should_store(self, num_blocks: int) -> bool:
|
||||
under_pressure = self.is_under_pressure()
|
||||
if self.is_healthy():
|
||||
return True
|
||||
if (
|
||||
not under_pressure
|
||||
and self._pressure_cleared_at > 0
|
||||
and time.monotonic() - self._pressure_cleared_at < self._cooldown_s
|
||||
):
|
||||
self._cooldown_count += 1
|
||||
if self._cooldown_count % 2 == 0:
|
||||
self._policy.on_store_skipped(num_blocks)
|
||||
return False
|
||||
return super().should_store(num_blocks)
|
||||
|
||||
def reset(self) -> None:
|
||||
self._ema = 0.0
|
||||
self._under_pressure = False
|
||||
self._completions = 0
|
||||
self._warmup_samples.clear()
|
||||
self._last_update = 0.0
|
||||
self._healthy_streak = 0
|
||||
self._pressure_cleared_at = 0.0
|
||||
self._cooldown_count = 0
|
||||
self._policy.reset()
|
||||
|
||||
@property
|
||||
def store_latency_ema(self) -> float:
|
||||
return self._ema
|
||||
|
||||
@store_latency_ema.setter
|
||||
def store_latency_ema(self, value: float) -> None:
|
||||
self._ema = value
|
||||
|
||||
@property
|
||||
def stats(self) -> dict[str, float]:
|
||||
return {"store_latency_ema": self._ema}
|
||||
@@ -2,9 +2,10 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Abstract interfaces and data types for the secondary tiering layer."""
|
||||
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Collection, Iterable
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
|
||||
import numpy as np
|
||||
@@ -20,6 +21,9 @@ from vllm.v1.kv_offload.base import (
|
||||
RequestOffloadingContext,
|
||||
ScheduleEndContext,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.backpressure import (
|
||||
BackpressureDetector,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
@@ -52,6 +56,11 @@ class TieringOffloadingMetrics:
|
||||
)
|
||||
ACTIVE_PROMOTION_JOBS = "vllm:kv_offload_tiering_active_promotion_jobs"
|
||||
ACTIVE_CASCADE_JOBS = "vllm:kv_offload_tiering_active_cascade_jobs"
|
||||
BACKPRESSURE_STORE_LATENCY_EMA = (
|
||||
"vllm:kv_offload_tiering_backpressure_store_latency_ema"
|
||||
)
|
||||
BACKPRESSURE_STORES_DROPPED = "vllm:kv_offload_tiering_backpressure_stores_dropped"
|
||||
BACKPRESSURE_BLOCKS_DROPPED = "vllm:kv_offload_tiering_backpressure_blocks_dropped"
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -63,6 +72,7 @@ class TransferJob:
|
||||
chunk_ids: np.ndarray
|
||||
is_promotion: bool
|
||||
req_context: ReqContext
|
||||
submit_time: float = field(default_factory=time.monotonic)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -77,6 +87,7 @@ class JobResult:
|
||||
# indicated by `success`. Must be a subset of the job's original keys.
|
||||
successful_keys: Collection[OffloadKey] | None = None
|
||||
transfer_time: float | None = None
|
||||
transfer_bytes: int | None = None
|
||||
|
||||
|
||||
class ParentManager(ABC):
|
||||
@@ -136,18 +147,33 @@ class SecondaryTierManager(ABC):
|
||||
offloading_spec: "OffloadingSpec",
|
||||
primary_kv_view: memoryview,
|
||||
tier_type: str,
|
||||
backpressure_detector: BackpressureDetector | None = None,
|
||||
) -> None:
|
||||
"""Args:
|
||||
offloading_spec: Offloading configuration.
|
||||
primary_kv_view: Memoryview of the primary tier's CPU KV cache.
|
||||
tier_type: Tier type identifier, set by SecondaryTierFactory
|
||||
from the registered tier type.
|
||||
backpressure_detector: Optional `BackpressureDetector`.
|
||||
|
||||
"""
|
||||
self._offloading_spec = offloading_spec
|
||||
self._primary_kv_view: memoryview = primary_kv_view
|
||||
assert primary_kv_view.strides is not None, (
|
||||
"primary_kv_view.strides cannot be None"
|
||||
)
|
||||
self._block_size_bytes: int = primary_kv_view.strides[0]
|
||||
self.tier_type = tier_type
|
||||
self.locality: Locality | None = None
|
||||
self._bp_detector = backpressure_detector
|
||||
|
||||
@property
|
||||
def block_size_bytes(self) -> int:
|
||||
return self._block_size_bytes
|
||||
|
||||
@property
|
||||
def bp_detector(self) -> BackpressureDetector | None:
|
||||
return self._bp_detector
|
||||
|
||||
@abstractmethod
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
||||
|
||||
@@ -31,6 +31,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.kv_offload.base import OffloadingSpec
|
||||
from vllm.v1.kv_offload.tiering.backpressure import BackpressureDetector
|
||||
|
||||
|
||||
class ExampleSecondaryTierManager(SecondaryTierManager):
|
||||
@@ -49,6 +50,7 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
|
||||
primary_kv_view: memoryview,
|
||||
tier_type: str,
|
||||
custom_param: int = 0,
|
||||
backpressure_detector: "BackpressureDetector | None" = None,
|
||||
):
|
||||
"""Initialize the example secondary tier.
|
||||
|
||||
@@ -57,12 +59,14 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
|
||||
primary_kv_view: Memoryview over the primary tier's KV buffer.
|
||||
tier_type: Name identifying this tier type.
|
||||
custom_param: Dummy parameter demonstrating custom args.
|
||||
backpressure_detector: Optional backpressure detector.
|
||||
|
||||
"""
|
||||
super().__init__(
|
||||
offloading_spec=offloading_spec,
|
||||
primary_kv_view=primary_kv_view,
|
||||
tier_type=tier_type,
|
||||
backpressure_detector=backpressure_detector,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
|
||||
@@ -8,11 +8,33 @@ from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.tiering.base import SecondaryTierManager
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.kv_offload.backpressure import BackpressureDetector
|
||||
from vllm.v1.kv_offload.base import OffloadingSpec
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
_DEFAULT_PACKAGE = "vllm.v1.kv_offload.tiering.backpressure"
|
||||
|
||||
|
||||
def _import_class(name: str) -> type:
|
||||
"""Import a class by name.
|
||||
|
||||
If ``name`` contains a dot it is treated as a fully qualified path
|
||||
(e.g. ``'my_package.MyDetector'``). Otherwise it is looked up in
|
||||
the default backpressure module
|
||||
(``vllm.v1.kv_offload.tiering.backpressure``), so short names like
|
||||
``'DropStorePolicy'`` or ``'EMABackpressureDetector'`` work out of
|
||||
the box.
|
||||
"""
|
||||
if "." in name:
|
||||
module_path, _, class_name = name.rpartition(".")
|
||||
else:
|
||||
module_path, class_name = _DEFAULT_PACKAGE, name
|
||||
module = importlib.import_module(module_path)
|
||||
return getattr(module, class_name)
|
||||
|
||||
|
||||
class SecondaryTierFactory:
|
||||
"""Registry for SecondaryTierManager implementations, resolved by type.
|
||||
|
||||
@@ -47,10 +69,30 @@ class SecondaryTierFactory:
|
||||
config = tier_config.copy()
|
||||
tier_type = config.pop("type")
|
||||
config.pop("module_path", None)
|
||||
locality = config.get("locality")
|
||||
bp_config = config.pop("backpressure", None)
|
||||
bp_detector = None
|
||||
if bp_config is not None:
|
||||
bp_config = bp_config.copy()
|
||||
bp_cls_name = bp_config.pop("backpressure_cls", None)
|
||||
if bp_cls_name is None:
|
||||
raise ValueError(
|
||||
"backpressure config must include 'backpressure_cls' "
|
||||
"(e.g. 'EMABackpressureDetector')"
|
||||
)
|
||||
policy_cls_name = bp_config.pop("policy_cls", None)
|
||||
policy = _import_class(policy_cls_name)() if policy_cls_name else None
|
||||
detector_cls: type[BackpressureDetector] = _import_class(bp_cls_name)
|
||||
defaults = detector_cls.default_config(tier_type, locality=locality)
|
||||
if defaults:
|
||||
for k, v in defaults.items():
|
||||
bp_config.setdefault(k, v)
|
||||
bp_detector = detector_cls(policy=policy, **bp_config)
|
||||
return tier_cls(
|
||||
offloading_spec=offloading_spec,
|
||||
primary_kv_view=primary_kv_view,
|
||||
tier_type=tier_type,
|
||||
backpressure_detector=bp_detector,
|
||||
**config,
|
||||
)
|
||||
|
||||
|
||||
@@ -40,6 +40,7 @@ from vllm.v1.kv_offload.base import (
|
||||
)
|
||||
from vllm.v1.kv_offload.file_mapper import FileMapper
|
||||
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager
|
||||
from vllm.v1.kv_offload.tiering.backpressure import BackpressureDetector
|
||||
from vllm.v1.kv_offload.tiering.base import (
|
||||
JobId,
|
||||
JobResult,
|
||||
@@ -115,6 +116,7 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
n_write_threads: int = 16,
|
||||
enable_kv_events: bool = False,
|
||||
locality: str | None = None,
|
||||
backpressure_detector: BackpressureDetector | None = None,
|
||||
):
|
||||
"""Args:
|
||||
offloading_spec: Contains normalized offloading configuration and
|
||||
@@ -129,9 +131,12 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
cache events are enabled globally (kv_events_config).
|
||||
locality: Whether this tier's storage is LOCAL or REMOTE relative
|
||||
to the publishing vLLM instance.
|
||||
backpressure_detector: Optional backpressure detector.
|
||||
|
||||
"""
|
||||
super().__init__(offloading_spec, primary_kv_view, tier_type)
|
||||
super().__init__(
|
||||
offloading_spec, primary_kv_view, tier_type, backpressure_detector
|
||||
)
|
||||
self.locality = Locality(locality) if locality is not None else None
|
||||
|
||||
self.events: list[OffloadingEvent] | None = None
|
||||
@@ -150,6 +155,8 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
# Keys of in-flight load (promotion) jobs, so a failed load can mark
|
||||
# its own cached lookup verdicts False (see get_finished_jobs).
|
||||
self._load_job_keys: dict[JobId, list[OffloadKey]] = {}
|
||||
# Block count per in-flight job, used to report transfer_bytes.
|
||||
self._job_block_counts: dict[JobId, int] = {}
|
||||
# Per load job: how many blocks loaded before a failure (partial keep).
|
||||
# Written by the pool worker inside the load task before it raises (so
|
||||
# before task_done publishes the job); read on the scheduler thread in
|
||||
@@ -225,6 +232,7 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
self._block_size,
|
||||
self._use_o_direct,
|
||||
)
|
||||
self._job_block_counts[job_metadata.job_id] = len(keys)
|
||||
self._pool.enqueue_store(job_metadata.job_id, 1, [task])
|
||||
|
||||
@override
|
||||
@@ -234,6 +242,7 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
# keys as a miss (see get_finished_jobs).
|
||||
keys = list(job_metadata.keys)
|
||||
self._load_job_keys[job_id] = keys
|
||||
self._job_block_counts[job_id] = len(keys)
|
||||
paths = [self.file_mapper.get_file_name(key) for key in keys]
|
||||
offsets = [int(cid) * self._block_size for cid in job_metadata.chunk_ids]
|
||||
|
||||
@@ -272,6 +281,8 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
as a miss here (scheduler thread)."""
|
||||
results = []
|
||||
for job_id, success, transfer_time in self._pool.get_finished():
|
||||
block_count = self._job_block_counts.pop(job_id, 0)
|
||||
transfer_bytes = block_count * self._block_size if block_count else None
|
||||
if self.events is not None:
|
||||
keys = self._store_job_keys.pop(job_id, None)
|
||||
if success and keys:
|
||||
@@ -299,6 +310,7 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
success=False,
|
||||
successful_keys=tuple(successful) if successful else None,
|
||||
transfer_time=transfer_time,
|
||||
transfer_bytes=transfer_bytes,
|
||||
)
|
||||
)
|
||||
continue
|
||||
@@ -307,6 +319,7 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
job_id=job_id,
|
||||
success=success,
|
||||
transfer_time=transfer_time,
|
||||
transfer_bytes=transfer_bytes,
|
||||
)
|
||||
)
|
||||
return results
|
||||
|
||||
@@ -238,6 +238,10 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
for tier_idx, tier in enumerate(self.secondary_tiers)
|
||||
}
|
||||
|
||||
self._tier_index: dict[SecondaryTierManager, int] = {
|
||||
tier: i for i, tier in enumerate(self.secondary_tiers)
|
||||
}
|
||||
|
||||
@property
|
||||
def _transfer_jobs(self) -> dict[JobId, JobMetadata]:
|
||||
return self._jobs
|
||||
@@ -337,6 +341,43 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
self.primary_tier.complete_read(
|
||||
transfer_job.keys, transfer_job.req_context
|
||||
)
|
||||
if completed_job.success:
|
||||
self._update_backpressure(tier, job_metadata, completed_job)
|
||||
|
||||
def _should_store_to_tier(
|
||||
self, tier: SecondaryTierManager, num_blocks: int
|
||||
) -> bool:
|
||||
detector = tier.bp_detector
|
||||
if detector is None:
|
||||
return True
|
||||
return detector.should_store(num_blocks)
|
||||
|
||||
def _update_backpressure(
|
||||
self,
|
||||
tier: SecondaryTierManager,
|
||||
job_metadata: JobMetadata,
|
||||
completed_job: JobResult,
|
||||
) -> None:
|
||||
detector = tier.bp_detector
|
||||
if detector is None:
|
||||
return
|
||||
was_under_pressure = detector.is_under_pressure()
|
||||
tj = job_metadata.transfer_job
|
||||
num_bytes = (
|
||||
completed_job.transfer_bytes
|
||||
if completed_job.transfer_bytes is not None
|
||||
else len(tj.keys) * tier.block_size_bytes
|
||||
)
|
||||
detector.update(tj.submit_time, num_bytes)
|
||||
if detector.is_under_pressure() != was_under_pressure:
|
||||
tier_idx = self._tier_index[tier]
|
||||
logger.info(
|
||||
"Tier #%d (%s) back-pressure %s (stats=%s)",
|
||||
tier_idx,
|
||||
tier.tier_type,
|
||||
"activated" if detector.is_under_pressure() else "cleared",
|
||||
detector.stats,
|
||||
)
|
||||
|
||||
@override
|
||||
def lookup(
|
||||
@@ -643,8 +684,10 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
return
|
||||
|
||||
for tier_idx in request_level_tiers:
|
||||
job_metadata = self.create_store_job(ready_keys, req_context, tier_idx)
|
||||
tier = self.secondary_tiers[tier_idx]
|
||||
if not self._should_store_to_tier(tier, len(ready_keys)):
|
||||
continue
|
||||
job_metadata = self.create_store_job(ready_keys, req_context, tier_idx)
|
||||
tier.submit_store(job_metadata)
|
||||
|
||||
def _flush_pending_cascades(self) -> None:
|
||||
@@ -698,6 +741,8 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
# eviction during the async transfer). One prepare_read() call per
|
||||
# secondary tier.
|
||||
for tier_idx, tier in enumerate(self.secondary_tiers):
|
||||
if not self._should_store_to_tier(tier, len(keys)):
|
||||
continue
|
||||
job_metadata = self.create_store_job(keys, req_context, tier_idx)
|
||||
tier.submit_store(job_metadata)
|
||||
|
||||
@@ -904,6 +949,10 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
del self._req_state[req_id]
|
||||
self._processed_jobs_this_step = False
|
||||
|
||||
for tier in self.secondary_tiers:
|
||||
if tier.bp_detector is not None:
|
||||
tier.bp_detector.reset()
|
||||
|
||||
@override
|
||||
def get_stats(self) -> OffloadingConnectorStats | None:
|
||||
stats = self.primary_tier.get_stats()
|
||||
@@ -911,6 +960,7 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
if stats is not None and stats.is_empty():
|
||||
stats = None
|
||||
|
||||
self._metrics.record_backpressure(self.secondary_tiers)
|
||||
metrics_stats = self._metrics.take_stats()
|
||||
if metrics_stats is not None:
|
||||
if stats is None:
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
from vllm.v1.kv_offload.base import LookupResult, OffloadKey, ReqContext
|
||||
from vllm.v1.kv_offload.tiering.base import (
|
||||
JobResult,
|
||||
SecondaryTierManager,
|
||||
TieringOffloadingMetrics,
|
||||
TransferJob,
|
||||
)
|
||||
@@ -130,6 +131,36 @@ class TieringMetricsTracker:
|
||||
TieringOffloadingMetrics.PROMOTION_ALLOCATION_FAILURES
|
||||
)
|
||||
|
||||
def record_backpressure(
|
||||
self,
|
||||
tiers: list[SecondaryTierManager],
|
||||
) -> None:
|
||||
for i, tier in enumerate(tiers):
|
||||
detector = tier.bp_detector
|
||||
if detector is None:
|
||||
continue
|
||||
label = self.tier_label(i)
|
||||
ema = detector.stats.get("store_latency_ema")
|
||||
if ema is not None:
|
||||
self._stats.set_gauge(
|
||||
TieringOffloadingMetrics.BACKPRESSURE_STORE_LATENCY_EMA,
|
||||
ema,
|
||||
labelvalues=label,
|
||||
)
|
||||
stores_dropped, blocks_dropped = detector.policy.pop_stores_dropped()
|
||||
if stores_dropped > 0:
|
||||
self._stats.increase_counter(
|
||||
TieringOffloadingMetrics.BACKPRESSURE_STORES_DROPPED,
|
||||
stores_dropped,
|
||||
labelvalues=label,
|
||||
)
|
||||
if blocks_dropped > 0:
|
||||
self._stats.increase_counter(
|
||||
TieringOffloadingMetrics.BACKPRESSURE_BLOCKS_DROPPED,
|
||||
blocks_dropped,
|
||||
labelvalues=label,
|
||||
)
|
||||
|
||||
def take_stats(self) -> OffloadingConnectorStats | None:
|
||||
active_transfer_stats = OffloadingConnectorStats()
|
||||
self._observe_active_transfer_stats(active_transfer_stats)
|
||||
|
||||
@@ -22,6 +22,7 @@ from vllm.v1.kv_offload.base import (
|
||||
)
|
||||
from vllm.v1.kv_offload.file_mapper import FileMapper
|
||||
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager
|
||||
from vllm.v1.kv_offload.tiering.backpressure import BackpressureDetector
|
||||
from vllm.v1.kv_offload.tiering.base import (
|
||||
JobId,
|
||||
JobResult,
|
||||
@@ -112,6 +113,7 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
|
||||
io_threads: int = 4,
|
||||
enable_kv_events: bool = False,
|
||||
locality: str | None = None,
|
||||
backpressure_detector: BackpressureDetector | None = None,
|
||||
):
|
||||
"""Args:
|
||||
offloading_spec: Offloading configuration.
|
||||
@@ -125,9 +127,15 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
|
||||
cache events are enabled globally (kv_events_config).
|
||||
locality: Whether this tier's storage is LOCAL or REMOTE relative
|
||||
to the publishing vLLM instance.
|
||||
backpressure_detector: Optional backpressure detector.
|
||||
|
||||
"""
|
||||
super().__init__(offloading_spec, primary_kv_view, tier_type)
|
||||
super().__init__(
|
||||
offloading_spec,
|
||||
primary_kv_view,
|
||||
tier_type,
|
||||
backpressure_detector,
|
||||
)
|
||||
self.locality = Locality(locality) if locality is not None else None
|
||||
|
||||
self.events: list[OffloadingEvent] | None = None
|
||||
|
||||
@@ -265,7 +265,25 @@ class P2PSecondaryTierManager(SecondaryTierManager):
|
||||
number, or anything convertible to one.
|
||||
|
||||
"""
|
||||
super().__init__(offloading_spec, primary_kv_view, tier_type)
|
||||
backpressure_detector = kwargs.pop("backpressure_detector", None)
|
||||
if backpressure_detector is not None:
|
||||
# The generic (store-latency) detector is unreliable for P2P: in
|
||||
# PD mode dropping a store can leave the decoder waiting until the
|
||||
# load timeout instead of failing fast, and rendezvous time makes
|
||||
# store latency a poor pressure signal. Reject it until a
|
||||
# P2P-specific fail-fast path exists.
|
||||
raise ValueError(
|
||||
"Backpressure is not supported for the P2P secondary tier. "
|
||||
"The generic store-latency detector cannot fail fast in PD "
|
||||
"mode and rendezvous time makes store latency an unreliable "
|
||||
"pressure signal. Remove the 'backpressure' config from the "
|
||||
"p2p tier."
|
||||
)
|
||||
super().__init__(
|
||||
offloading_spec,
|
||||
primary_kv_view,
|
||||
tier_type,
|
||||
)
|
||||
try:
|
||||
timeout_s = float(unbound_store_timeout_s)
|
||||
except (TypeError, ValueError):
|
||||
|
||||
@@ -247,6 +247,34 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
assert isinstance(tier_config, dict)
|
||||
tier_cls = SecondaryTierFactory.get_tier_class(tier_config)
|
||||
metrics.update(tier_cls.build_metric_definitions(tier_config))
|
||||
|
||||
metrics[TieringOffloadingMetrics.BACKPRESSURE_STORE_LATENCY_EMA] = (
|
||||
OffloadingGaugeMetadata(
|
||||
documentation=(
|
||||
"Exponential moving average of store latency "
|
||||
"for back-pressure detection, in s/MiB."
|
||||
),
|
||||
labelnames=("tier",),
|
||||
)
|
||||
)
|
||||
metrics[TieringOffloadingMetrics.BACKPRESSURE_STORES_DROPPED] = (
|
||||
OffloadingCounterMetadata(
|
||||
documentation=(
|
||||
"Number of store operations dropped due to "
|
||||
"back-pressure on a secondary tier."
|
||||
),
|
||||
labelnames=("tier",),
|
||||
)
|
||||
)
|
||||
metrics[TieringOffloadingMetrics.BACKPRESSURE_BLOCKS_DROPPED] = (
|
||||
OffloadingCounterMetadata(
|
||||
documentation=(
|
||||
"Number of blocks dropped due to back-pressure on a secondary tier."
|
||||
),
|
||||
labelnames=("tier",),
|
||||
)
|
||||
)
|
||||
|
||||
return metrics
|
||||
|
||||
def __init__(self, config: OffloadingConfig):
|
||||
@@ -259,6 +287,32 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
if not isinstance(self.secondary_tier_configs, list):
|
||||
raise ValueError("secondary_tiers must be a list of tier configurations")
|
||||
|
||||
# Backpressure config is merged field-by-field in priority order
|
||||
# (highest first):
|
||||
# 1. Per-tier ``backpressure`` dict in the tier config
|
||||
# 2. Top-level ``backpressure`` in kv_connector_extra_config
|
||||
# Merging per field (rather than per whole dict) means a partial
|
||||
# tier override still inherits missing fields from the top-level
|
||||
# default, so the resolved dict reaching the factory is complete.
|
||||
# Within each tier's resolved dict, tier-type-aware water marks
|
||||
# are filled in last so a bare ``"backpressure": {}`` picks up
|
||||
# sensible thresholds for the storage medium.
|
||||
bp_defaults = self.extra_config.get("backpressure")
|
||||
|
||||
for tier_cfg in self.secondary_tier_configs:
|
||||
tier_override = tier_cfg.get("backpressure")
|
||||
# Overlay from lowest to highest precedence so higher-precedence
|
||||
# fields win while lower-precedence ones fill in the gaps.
|
||||
merged: dict[str, Any] = {}
|
||||
for source in (bp_defaults, tier_override):
|
||||
if source:
|
||||
merged.update(source)
|
||||
# Only set a resolved dict when at least one source contributed
|
||||
# (or an explicit ``backpressure`` key was present, e.g. ``{}``),
|
||||
# so tiers without any backpressure config stay unconfigured.
|
||||
if merged or "backpressure" in tier_cfg:
|
||||
tier_cfg["backpressure"] = merged
|
||||
|
||||
# Scheduler-side mmap (rank=None); kept for cleanup
|
||||
self._scheduler_mmap: SharedOffloadRegion | None = None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user