From 23e26e058839fb2a3e77c78fbf2184f563593227 Mon Sep 17 00:00:00 2001 From: bnellnm <49004751+bnellnm@users.noreply.github.com> Date: Fri, 18 Sep 2026 14:47:48 -0400 Subject: [PATCH] [KV Offloading] Back-pressure detection and remediation (#50045) Signed-off-by: Bill Nell Signed-off-by: bnellnm <49004751+bnellnm@users.noreply.github.com> Co-authored-by: Itay Etelis <92247226+Etelis@users.noreply.github.com> --- .../unit/offloading_connector/test_events.py | 89 +++ .../v1/kv_offload/tiering/p2p/test_manager.py | 34 + .../kv_offload/tiering/test_backpressure.py | 706 ++++++++++++++++++ vllm/v1/kv_offload/tiering/backpressure.py | 384 ++++++++++ vllm/v1/kv_offload/tiering/base.py | 28 +- vllm/v1/kv_offload/tiering/example/manager.py | 4 + vllm/v1/kv_offload/tiering/factory.py | 42 ++ vllm/v1/kv_offload/tiering/fs/manager.py | 15 +- vllm/v1/kv_offload/tiering/manager.py | 52 +- vllm/v1/kv_offload/tiering/metrics.py | 31 + vllm/v1/kv_offload/tiering/obj/manager.py | 10 +- vllm/v1/kv_offload/tiering/p2p/manager.py | 20 +- vllm/v1/kv_offload/tiering/spec.py | 54 ++ 13 files changed, 1464 insertions(+), 5 deletions(-) create mode 100644 tests/v1/kv_offload/tiering/test_backpressure.py create mode 100644 vllm/v1/kv_offload/tiering/backpressure.py diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_events.py b/tests/v1/kv_connector/unit/offloading_connector/test_events.py index da0ccb02b1..c409fd125d 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_events.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_events.py @@ -682,3 +682,92 @@ def test_tiering_accepts_self_describing_kv_events(): assert spec.kv_events_config.enable_kv_cache_events assert spec.kv_events_config.self_describing_kv_events assert tracker.self_describing_enabled + + +def _build_tiering_spec(secondary_tiers, *, top_level_backpressure=None): + vllm_config = create_vllm_config( + block_size=4, + max_num_batched_tokens=16, + disable_hybrid_kv_cache_manager=False, + ) + extra_config = { + "spec_name": "TieringOffloadingSpec", + "cpu_bytes_to_use": 1 << 20, + "secondary_tiers": secondary_tiers, + } + if top_level_backpressure is not None: + extra_config["backpressure"] = top_level_backpressure + vllm_config.kv_transfer_config = KVTransferConfig( + kv_connector="OffloadingConnector", + kv_role="kv_both", + kv_connector_extra_config=extra_config, + ) + kv_cache_config = KVCacheConfig( + num_blocks=0, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec( + ["layer"], + FullAttentionSpec( + block_size=4, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + ) + ], + ) + return TieringOffloadingSpec(build_offloading_config(vllm_config, kv_cache_config)) + + +def test_partial_tier_backpressure_inherits_top_level_defaults(): + """A partial tier override still inherits missing fields. + + Field-by-field merge means an override that sets only ``high_water_s`` + keeps ``backpressure_cls`` (and any other field) from the top-level + default, so the resolved dict reaching the factory is complete. + """ + spec = _build_tiering_spec( + [{"type": "example", "backpressure": {"high_water_s": 0.1}}], + top_level_backpressure={ + "backpressure_cls": "EMABackpressureDetector", + "high_water_s": 0.5, + "low_water_s": 0.05, + }, + ) + + resolved = spec.secondary_tier_configs[0]["backpressure"] + # Tier override wins for the field it sets. + assert resolved["high_water_s"] == 0.1 + # Missing fields fall back to the top-level default. + assert resolved["backpressure_cls"] == "EMABackpressureDetector" + assert resolved["low_water_s"] == 0.05 + + +def test_tier_override_wins_over_top_level(): + """Precedence is tier override > top-level default, field-by-field.""" + spec = _build_tiering_spec( + [ + { + "type": "example", + "backpressure": {"high_water_s": 0.1, "low_water_s": 0.02}, + } + ], + top_level_backpressure={ + "backpressure_cls": "EMABackpressureDetector", + "high_water_s": 0.5, + "low_water_s": 0.05, + }, + ) + + resolved = spec.secondary_tier_configs[0]["backpressure"] + # Tier override wins for the fields it sets. + assert resolved["high_water_s"] == 0.1 + assert resolved["low_water_s"] == 0.02 + # Fields only in top-level still fill in. + assert resolved["backpressure_cls"] == "EMABackpressureDetector" + + +def test_tier_without_backpressure_stays_unconfigured(): + spec = _build_tiering_spec([{"type": "example"}]) + assert "backpressure" not in spec.secondary_tier_configs[0] diff --git a/tests/v1/kv_offload/tiering/p2p/test_manager.py b/tests/v1/kv_offload/tiering/p2p/test_manager.py index aceec1cd8f..8e806582fd 100644 --- a/tests/v1/kv_offload/tiering/p2p/test_manager.py +++ b/tests/v1/kv_offload/tiering/p2p/test_manager.py @@ -180,6 +180,40 @@ class TestInitHashSeed: assert mgr._get_hash_seed() == "random-seed-abc" +class TestBackpressureRejected: + def _patch_transports(self, monkeypatch) -> None: + monkeypatch.setattr(manager_module, "NixlTransport", lambda *a, **k: object()) + monkeypatch.setattr(manager_module, "ZmqTransport", lambda *a, **k: object()) + monkeypatch.setattr( + manager_module.FileMapper, + "from_offloading_spec", + lambda **k: SimpleNamespace(get_run_config=lambda: {}), + ) + + def test_explicit_backpressure_detector_rejected(self, monkeypatch): + """An explicitly configured detector must fail fast for P2P. + + The generic store-latency detector cannot fail fast in PD mode and + rendezvous time makes store latency an unreliable pressure signal, + so P2P rejects it rather than degrading silently. + """ + self._patch_transports(monkeypatch) + with pytest.raises(ValueError, match="not supported for the P2P"): + P2PSecondaryTierManager( + offloading_spec=_init_offloading_spec(), + primary_kv_view=memoryview(bytearray(16)), + backpressure_detector=object(), + ) + + def test_no_detector_constructs_normally(self, monkeypatch): + self._patch_transports(monkeypatch) + mgr = P2PSecondaryTierManager( + offloading_spec=_init_offloading_spec(), + primary_kv_view=memoryview(bytearray(16)), + ) + assert mgr.bp_detector is None + + # --------------------------------------------------------------------------- # Tests for _peer_id_from_params # --------------------------------------------------------------------------- diff --git a/tests/v1/kv_offload/tiering/test_backpressure.py b/tests/v1/kv_offload/tiering/test_backpressure.py new file mode 100644 index 0000000000..b809bc4dc4 --- /dev/null +++ b/tests/v1/kv_offload/tiering/test_backpressure.py @@ -0,0 +1,706 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for back-pressure detection in TieringOffloadingManager. + +These tests use a DelayedSecondaryTierManager that holds completed jobs +until explicitly released, allowing precise control over when the manager +observes store completions and their apparent latency. +""" + +import time +from collections.abc import Iterable +from unittest.mock import MagicMock, patch + +import pytest +import torch + +from vllm.v1.kv_offload.base import ( + LookupResult, + OffloadKey, + OffloadPolicy, + ReqContext, + RequestOffloadingContext, + ScheduleEndContext, + make_offload_key, +) +from vllm.v1.kv_offload.tiering.backpressure import ( + DropAccountingPolicy, + DropStorePolicy, + EMABackpressureDetector, + ThrottledDropPolicy, +) +from vllm.v1.kv_offload.tiering.base import ( + JobResult, + SecondaryTierManager, + TieringOffloadingMetrics, + TransferJob, +) +from vllm.v1.kv_offload.tiering.manager import ( + CPUPrimaryTierOffloadingManager, + TieringOffloadingManager, +) + +_BP_EMA_ALPHA = EMABackpressureDetector.DEFAULT_ALPHA +_BP_WARMUP = EMABackpressureDetector.DEFAULT_WARMUP_COMPLETIONS +# Water marks in s/MiB, scaled for the test's tiny 16-byte blocks. +_BP_HIGH_WATER_S = 1000.0 +_BP_LOW_WATER_S = 500.0 + +_CTX = ReqContext(req_id="test") +_MOCK_OFFLOADING_SPEC = MagicMock() + + +def _mock_mmap_region(num_blocks: int, row_bytes: int = 16): + mock = MagicMock() + view = memoryview(torch.zeros((num_blocks, row_bytes), dtype=torch.int8).numpy()) + mock.create_kv_memoryview.return_value = view + return mock + + +def to_keys(int_ids: Iterable[int]) -> list[OffloadKey]: + return [make_offload_key(str(i).encode(), 0) for i in int_ids] + + +class DelayedSecondaryTierManager(SecondaryTierManager): + """Secondary tier that holds completed store jobs until released. + + Jobs are stored immediately but their completion is not reported + by get_finished_jobs() until release_jobs() is called. This lets + tests control the apparent latency of store operations by adjusting + submit_time on the JobMetadata before triggering the completion poll. + """ + + def __init__( + self, offloading_spec, primary_kv_view, tier_type, backpressure_detector=None + ): + super().__init__( + offloading_spec, + primary_kv_view, + tier_type, + backpressure_detector=backpressure_detector, + ) + self.blocks: dict[OffloadKey, bool] = {} + self._held_jobs: list[JobResult] = [] + self._released_jobs: list[JobResult] = [] + self._request_policy: OffloadPolicy | None = None + + def lookup(self, key, req_context): + return LookupResult.HIT if key in self.blocks else LookupResult.MISS + + def submit_store(self, job_metadata: TransferJob) -> None: + for key in job_metadata.keys: + self.blocks[key] = True + self._held_jobs.append(JobResult(job_id=job_metadata.job_id, success=True)) + + def submit_load(self, job_metadata: TransferJob) -> None: + self._released_jobs.append(JobResult(job_id=job_metadata.job_id, success=True)) + + def get_finished_jobs(self) -> Iterable[JobResult]: + result = self._released_jobs + self._released_jobs = [] + return result + + def release_jobs(self): + self._released_jobs.extend(self._held_jobs) + self._held_jobs.clear() + + def on_new_request(self, req_context): + if self._request_policy is not None: + return RequestOffloadingContext(policy=self._request_policy) + return RequestOffloadingContext() + + def drain_jobs(self): + self.release_jobs() + + def has_pending_work(self): + return bool(self._held_jobs) + + def get_num_blocks(self): + return len(self.blocks) + + +class TestDefaultConfig: + def test_fs_gets_local_watermarks(self): + cfg = EMABackpressureDetector.default_config("fs") + assert cfg["high_water_s"] == EMABackpressureDetector.LOCAL_HIGH_WATER_S + assert cfg["low_water_s"] == EMABackpressureDetector.LOCAL_LOW_WATER_S + + def test_obj_gets_network_watermarks(self): + cfg = EMABackpressureDetector.default_config("obj") + assert cfg["high_water_s"] == EMABackpressureDetector.NETWORK_HIGH_WATER_S + assert cfg["low_water_s"] == EMABackpressureDetector.NETWORK_LOW_WATER_S + + def test_p2p_gets_local_watermarks(self): + cfg = EMABackpressureDetector.default_config("p2p") + assert cfg["high_water_s"] == EMABackpressureDetector.LOCAL_HIGH_WATER_S + assert cfg["low_water_s"] == EMABackpressureDetector.LOCAL_LOW_WATER_S + + def test_fs_with_remote_locality_gets_network_watermarks(self): + cfg = EMABackpressureDetector.default_config("fs", locality="REMOTE") + assert cfg["high_water_s"] == EMABackpressureDetector.NETWORK_HIGH_WATER_S + assert cfg["low_water_s"] == EMABackpressureDetector.NETWORK_LOW_WATER_S + + def test_fs_with_local_locality_gets_local_watermarks(self): + cfg = EMABackpressureDetector.default_config("fs", locality="LOCAL") + assert cfg["high_water_s"] == EMABackpressureDetector.LOCAL_HIGH_WATER_S + assert cfg["low_water_s"] == EMABackpressureDetector.LOCAL_LOW_WATER_S + + +class TestIdleDecay: + """Tests for EMA idle decay — recovery from pressure without completions.""" + + def _make_detector(self, **kwargs): + defaults = dict( + high_water_s=_BP_HIGH_WATER_S, + low_water_s=_BP_LOW_WATER_S, + decay_half_life_s=2.0, + ) + defaults.update(kwargs) + return EMABackpressureDetector(**defaults) + + def test_ema_decays_while_under_pressure(self): + bp = self._make_detector() + bp._completions = _BP_WARMUP + bp._ema = _BP_HIGH_WATER_S * 2 + bp._under_pressure = True + bp._last_update = time.monotonic() + + with patch("time.monotonic", return_value=bp._last_update + 4.1): + # ~2.05 half-lives → EMA * 0.24, well below low water (500) + assert bp.is_under_pressure() is False + assert bp.store_latency_ema < _BP_LOW_WATER_S + + def test_ema_stays_pressured_if_not_enough_decay(self): + bp = self._make_detector() + bp._completions = _BP_WARMUP + bp._ema = _BP_HIGH_WATER_S * 4 + bp._under_pressure = True + bp._last_update = time.monotonic() + + with patch("time.monotonic", return_value=bp._last_update + 2.0): + # 2s = 1 half-life → EMA * 0.5 = HIGH * 2, still above LOW + assert bp.is_under_pressure() is True + assert bp.store_latency_ema == pytest.approx(_BP_HIGH_WATER_S * 2, rel=0.01) + + def test_no_decay_before_warmup(self): + bp = self._make_detector() + bp._ema = _BP_HIGH_WATER_S * 2 + bp._under_pressure = True + bp._last_update = time.monotonic() + + with patch("time.monotonic", return_value=bp._last_update + 10.0): + # Still in warmup (_completions=0 < _warmup_completions), + # so decay should not be applied in is_under_pressure. + assert bp.is_under_pressure() is True + assert bp.store_latency_ema == _BP_HIGH_WATER_S * 2 + + def test_no_decay_applied_during_new_sample(self): + """on_store_completed applies the new sample directly without + idle decay — decay only runs in is_under_pressure().""" + bp = self._make_detector() + bp._completions = _BP_WARMUP + bp._ema = _BP_HIGH_WATER_S * 2 + bp._under_pressure = True + t0 = time.monotonic() + bp._last_update = t0 + + block_bytes = 16 + with patch("time.monotonic", return_value=t0 + 4.0): + bp.on_store_completed(0.0, block_bytes) + # No decay in on_store_completed: EMA = alpha*0 + (1-alpha)*2000 + expected = (1 - _BP_EMA_ALPHA) * _BP_HIGH_WATER_S * 2 + assert bp.store_latency_ema == pytest.approx(expected, rel=0.01) + + def test_reset_clears_last_update(self): + bp = self._make_detector() + bp._last_update = time.monotonic() + bp.reset() + assert bp._last_update == 0.0 + + +class TestBackpressure: + @pytest.fixture + def setup(self): + mock_region = _mock_mmap_region(20) + self.primary = CPUPrimaryTierOffloadingManager( + num_chunks=20, mmap_region=mock_region + ) + mock_view = mock_region.create_kv_memoryview() + self.tier = DelayedSecondaryTierManager( + offloading_spec=_MOCK_OFFLOADING_SPEC, + primary_kv_view=mock_view, + tier_type="delayed", + backpressure_detector=EMABackpressureDetector( + high_water_s=_BP_HIGH_WATER_S, + low_water_s=_BP_LOW_WATER_S, + ), + ) + self.manager = TieringOffloadingManager( + primary_tier=self.primary, + secondary_tiers=[self.tier], + ) + + def _start_request(self, ctx=_CTX): + if ctx.req_id not in self.manager._req_state: + self.manager.on_new_request(ctx) + + def _store_blocks(self, keys, ctx=_CTX): + self._start_request(ctx) + result = self.manager.prepare_store(keys, ctx) + assert result is not None + self.manager.complete_store(keys, ctx, success=True) + + def _simulate_on_schedule_end(self): + ctx = ScheduleEndContext(new_req_ids=[], preempted_req_ids=()) + self.manager.on_schedule_end(ctx) + + def _backdate_held_jobs(self, age_s: float): + """Set submit_time on all in-flight store jobs so they appear + to have taken ``age_s`` seconds when completed.""" + self.manager._processed_jobs_this_step = False + now = time.monotonic() + for job_id, meta in self.manager._jobs.items(): + tj = meta.transfer_job + if not tj.is_promotion and tj.submit_time > 0: + tj.submit_time = now - age_s + + def test_ema_updates_on_store_completion(self, setup): + bp = self.tier.bp_detector + fast_latency = 0.01 + # EMA is in s/MiB; with 16-byte blocks, scale = MiB / block_bytes. + scale = EMABackpressureDetector._MIB / self.tier.block_size_bytes + + # During warmup, EMA stays at 0. + for i in range(_BP_WARMUP - 1): + keys = to_keys([300 + i]) + self._store_blocks(keys) + self._backdate_held_jobs(fast_latency) + self.tier.release_jobs() + self._simulate_on_schedule_end() + assert bp.store_latency_ema == 0.0 + + # Final warmup sample seeds EMA with the mean. + keys = to_keys([400]) + self._store_blocks(keys) + self._backdate_held_jobs(fast_latency) + self.tier.release_jobs() + self._simulate_on_schedule_end() + fast_s_per_mib = fast_latency * scale + assert bp.store_latency_ema == pytest.approx(fast_s_per_mib, rel=0.1) + + # After warmup, EMA updates normally. + new_latency = 0.05 + keys = to_keys([401]) + self._store_blocks(keys) + self._backdate_held_jobs(new_latency) + self.tier.release_jobs() + self._simulate_on_schedule_end() + new_s_per_mib = new_latency * scale + expected = _BP_EMA_ALPHA * new_s_per_mib + (1 - _BP_EMA_ALPHA) * fast_s_per_mib + assert bp.store_latency_ema == pytest.approx(expected, rel=0.1) + + def test_pressure_activates_above_high_water(self, setup): + bp = self.tier.bp_detector + + # Warm up the detector with fast completions first. + for i in range(_BP_WARMUP): + keys = to_keys([200 + i]) + self._store_blocks(keys) + self._backdate_held_jobs(0.001) + self.tier.release_jobs() + self._simulate_on_schedule_end() + + keys = to_keys([100]) + self._store_blocks(keys) + self._simulate_on_schedule_end() + + # 0.1s for a single 16-byte block → ~6553 s/MiB, well above high water. + self._backdate_held_jobs(0.1) + self.tier.release_jobs() + self._simulate_on_schedule_end() + + assert bp.store_latency_ema > _BP_HIGH_WATER_S + assert bp.is_under_pressure() is True + + def test_pressure_clears_below_low_water(self, setup): + bp = self.tier.bp_detector + bp.store_latency_ema = _BP_HIGH_WATER_S * 2 + bp._under_pressure = True + bp._completions = _BP_WARMUP + + # Drive several fast completions to bring EMA below low water. + block_id = 100 + for _ in range(50): + if bp.store_latency_ema < _BP_LOW_WATER_S: + break + keys = to_keys([block_id]) + block_id += 1 + self._store_blocks(keys) + # Pressure is on, so the cascade was skipped. Force-submit + # a store job manually to get a completion to feed the EMA. + job_meta = self.manager.create_store_job(keys, _CTX) + job_meta.submit_time = time.monotonic() - 0.001 + self.tier.submit_store(job_meta) + self.tier.release_jobs() + self._simulate_on_schedule_end() + else: + pytest.fail(f"EMA did not recover below low water: {bp.store_latency_ema}") + + assert bp.is_under_pressure() is False + + def test_hysteresis_prevents_oscillation(self, setup): + bp = self.tier.bp_detector + bp._completions = _BP_WARMUP + block_bytes = self.tier.block_size_bytes + + # Set EMA between low and high water marks (in s/MiB). + mid = (_BP_LOW_WATER_S + _BP_HIGH_WATER_S) / 2 + bp.store_latency_ema = mid + bp._last_update = time.monotonic() + + # Raw seconds that produce `mid` s/MiB for a given number of blocks. + def mid_latency_s(num_blocks: int) -> float: + return mid * (num_blocks * block_bytes / EMABackpressureDetector._MIB) + + # When not under pressure, mid-range EMA should not activate. + bp._under_pressure = False + keys = to_keys(range(2)) + self._store_blocks(keys) + self._backdate_held_jobs(mid_latency_s(len(keys))) + self.tier.release_jobs() + self._simulate_on_schedule_end() + assert bp.is_under_pressure() is False + + # When under pressure, mid-range EMA should not deactivate. + bp._under_pressure = True + bp._last_update = time.monotonic() + keys2 = to_keys(range(10, 12)) + # Force-submit since pressure is on (cascade would skip). + for k in keys2: + self.primary.prepare_store([k], _CTX) + self.primary.complete_store([k], _CTX) + job_meta = self.manager.create_store_job(keys2, _CTX) + job_meta.submit_time = time.monotonic() - mid_latency_s(len(keys2)) + self.tier.submit_store(job_meta) + self.tier.release_jobs() + self._simulate_on_schedule_end() + assert bp.is_under_pressure() is True + + def test_stores_skipped_under_pressure(self, setup): + bp = self.tier.bp_detector + bp._under_pressure = True + bp.store_latency_ema = _BP_HIGH_WATER_S * 4 + initial_blocks = self.tier.get_num_blocks() + + keys = to_keys(range(50, 53)) + self._store_blocks(keys) + + # Blocks should be in primary but NOT in secondary. + for k in keys: + assert self.primary.lookup(k, _CTX) is LookupResult.HIT + assert self.tier.get_num_blocks() == initial_blocks + + def test_stores_resume_after_pressure_clears(self, setup): + bp = self.tier.bp_detector + + # Start under pressure — stores skipped. + bp._under_pressure = True + bp.store_latency_ema = _BP_HIGH_WATER_S * 4 + keys1 = to_keys(range(60, 62)) + self._store_blocks(keys1) + assert all(k not in self.tier.blocks for k in keys1) + + # Clear pressure — next store should cascade. + bp._under_pressure = False + keys2 = to_keys(range(70, 72)) + self._store_blocks(keys2) + assert all(k in self.tier.blocks for k in keys2) + + def test_dropped_store_count_tracked(self, setup): + bp = self.tier.bp_detector + bp._under_pressure = True + bp.store_latency_ema = _BP_HIGH_WATER_S * 4 + policy = bp.policy + assert policy.pop_stores_dropped() == (0, 0) + + self._store_blocks(to_keys([80])) + assert policy._stores_dropped == 1 + assert policy._blocks_dropped == 1 + + self._store_blocks(to_keys([81, 82])) + assert policy._stores_dropped == 2 + assert policy._blocks_dropped == 3 + + def test_metrics_reported_via_get_stats(self, setup): + bp = self.tier.bp_detector + bp._under_pressure = True + bp.store_latency_ema = 2.5 + policy = bp.policy + policy._stores_dropped = 5 + policy._blocks_dropped = 12 + + stats = self.manager.get_stats() + assert stats is not None + reduced = stats.reduce() + + ema_key = TieringOffloadingMetrics.BACKPRESSURE_STORE_LATENCY_EMA + stores_key = TieringOffloadingMetrics.BACKPRESSURE_STORES_DROPPED + blocks_key = TieringOffloadingMetrics.BACKPRESSURE_BLOCKS_DROPPED + assert reduced[f"{ema_key}:('1:delayed',)"] == pytest.approx(2.5) + assert reduced[f"{stores_key}:('1:delayed',)"] == 5 + assert reduced[f"{blocks_key}:('1:delayed',)"] == 12 + + # Dropped counts reset after get_stats. + assert policy._stores_dropped == 0 + assert policy._blocks_dropped == 0 + + def test_reset_cache_clears_backpressure(self, setup): + bp = self.tier.bp_detector + bp.store_latency_ema = 5.0 + bp._under_pressure = True + + self.manager.reset_cache() + + assert bp.store_latency_ema == 0.0 + assert bp.is_under_pressure() is False + + def test_request_level_tier_respects_backpressure(self, setup): + """Request-level cascade also skips pressured tiers.""" + # Store blocks in primary first. + existing = to_keys(range(3)) + self._store_blocks(existing) + self.tier.release_jobs() + self._simulate_on_schedule_end() + self._simulate_on_schedule_end() + + # Make tier request-level for a new request. + self.tier._request_policy = OffloadPolicy.REQUEST_LEVEL + ctx = ReqContext(req_id="req_rl_bp") + self.manager.on_new_request(ctx) + + # Activate pressure. + bp = self.tier.bp_detector + bp._under_pressure = True + bp.store_latency_ema = _BP_HIGH_WATER_S * 4 + initial_blocks = self.tier.get_num_blocks() + + # prepare_store with existing + new blocks. + new = to_keys(range(10, 12)) + all_keys = existing + new + result = self.manager.prepare_store(all_keys, ctx) + assert result is not None + + # Existing blocks would normally cascade to request-level tier, + # but should be skipped under pressure. + assert self.tier.get_num_blocks() == initial_blocks + + def test_loads_continue_under_pressure(self, setup): + """Loads from a pressured tier still work.""" + blocks = to_keys(range(3)) + for b in blocks: + self.tier.blocks[b] = True + + bp = self.tier.bp_detector + bp._under_pressure = True + bp.store_latency_ema = _BP_HIGH_WATER_S * 4 + + # Lookups should still initiate promotion. + for b in blocks: + result = self.manager.lookup(b, _CTX) + assert result is LookupResult.HIT_PENDING + + +class TestThrottledDropPolicy: + """Tests for proportional throttling under pressure.""" + + def _make_detector(self, **kwargs): + defaults = dict( + high_water_s=_BP_HIGH_WATER_S, + low_water_s=_BP_LOW_WATER_S, + cooldown_s=0.0, + ) + defaults.update(kwargs) + return EMABackpressureDetector(**defaults) + + def test_no_drop_when_not_under_pressure(self): + bp = self._make_detector() + for _ in range(20): + assert bp.should_store(1) is True + + def test_no_drop_at_exactly_high_water(self): + bp = self._make_detector() + bp._under_pressure = True + bp.store_latency_ema = _BP_HIGH_WATER_S + for _ in range(20): + assert bp.should_store(1) is True + + def test_partial_drop_at_moderate_pressure(self): + bp = self._make_detector() + bp._under_pressure = True + bp.store_latency_ema = _BP_HIGH_WATER_S * 2 + results = [bp.should_store(1) for _ in range(20)] + assert any(results), "some stores should be allowed" + assert not all(results), "some stores should be dropped" + + def test_full_drop_at_severe_pressure(self): + bp = self._make_detector() + bp._under_pressure = True + bp.store_latency_ema = _BP_HIGH_WATER_S * 4 + for _ in range(20): + assert bp.should_store(1) is False + + def test_drop_rate_increases_with_severity(self): + """Higher EMA → higher drop rate.""" + drop_counts = [] + for multiplier in [1.5, 2.0, 2.5, 3.5]: + bp = self._make_detector() + bp._under_pressure = True + bp.store_latency_ema = _BP_HIGH_WATER_S * multiplier + drops = sum(1 for _ in range(100) if not bp.should_store(1)) + drop_counts.append(drops) + for i in range(len(drop_counts) - 1): + assert drop_counts[i] <= drop_counts[i + 1] + + def test_explicit_drop_store_policy_still_drops_all(self): + """DropStorePolicy (explicit) still drops everything under pressure.""" + bp = EMABackpressureDetector( + high_water_s=_BP_HIGH_WATER_S, + low_water_s=_BP_LOW_WATER_S, + policy=DropStorePolicy(), + cooldown_s=0.0, + ) + bp._under_pressure = True + bp.store_latency_ema = _BP_HIGH_WATER_S * 1.5 + for _ in range(20): + assert bp.should_store(1) is False + + +class TestSharedDropAccounting: + """Both policies share drop accounting via DropAccountingPolicy.""" + + @pytest.mark.parametrize("policy_cls", [DropStorePolicy, ThrottledDropPolicy]) + def test_shared_base(self, policy_cls): + assert issubclass(policy_cls, DropAccountingPolicy) + + @pytest.mark.parametrize("policy_cls", [DropStorePolicy, ThrottledDropPolicy]) + def test_pop_returns_and_resets(self, policy_cls): + policy = policy_cls() + policy.on_store_skipped(3) + policy.on_store_skipped(2) + assert policy.pop_stores_dropped() == (2, 5) + assert policy.pop_stores_dropped() == (0, 0) + + @pytest.mark.parametrize("policy_cls", [DropStorePolicy, ThrottledDropPolicy]) + def test_reset_clears_counters(self, policy_cls): + policy = policy_cls() + policy.on_store_skipped(4) + policy.reset() + assert policy.pop_stores_dropped() == (0, 0) + + def test_throttled_reset_also_clears_call_count(self): + policy = ThrottledDropPolicy() + policy._call_count = 7 + policy.reset() + assert policy._call_count == 0 + + +class TestHealthyBypass: + """Tests for the fast-path bypass when a tier is consistently healthy.""" + + def _make_detector(self, **kwargs): + defaults = dict( + high_water_s=_BP_HIGH_WATER_S, + low_water_s=_BP_LOW_WATER_S, + cooldown_s=0.0, + ) + defaults.update(kwargs) + return EMABackpressureDetector(**defaults) + + def test_not_healthy_initially(self): + bp = self._make_detector() + assert bp.is_healthy() is False + + def test_healthy_after_streak(self): + bp = self._make_detector() + bp._completions = _BP_WARMUP + bp._last_update = time.monotonic() + block_bytes = 16 + fast_s_per_mib = _BP_LOW_WATER_S * 0.5 + fast_elapsed = fast_s_per_mib * (block_bytes / EMABackpressureDetector._MIB) + for _ in range(EMABackpressureDetector._HEALTHY_THRESHOLD + _BP_WARMUP): + bp.on_store_completed(fast_elapsed, block_bytes) + assert bp.is_healthy() is True + + def test_healthy_resets_on_high_latency(self): + bp = self._make_detector() + bp._completions = _BP_WARMUP + bp._healthy_streak = EMABackpressureDetector._HEALTHY_THRESHOLD + bp._ema = _BP_LOW_WATER_S + bp._last_update = time.monotonic() + block_bytes = 16 + slow_s_per_mib = _BP_HIGH_WATER_S * 2 + slow_elapsed = slow_s_per_mib * (block_bytes / EMABackpressureDetector._MIB) + bp.on_store_completed(slow_elapsed, block_bytes) + assert bp.is_healthy() is False + + def test_healthy_bypasses_policy(self): + bp = self._make_detector() + bp._completions = _BP_WARMUP + bp._healthy_streak = EMABackpressureDetector._HEALTHY_THRESHOLD + assert bp.should_store(1) is True + + def test_reset_clears_healthy_streak(self): + bp = self._make_detector() + bp._healthy_streak = EMABackpressureDetector._HEALTHY_THRESHOLD + bp.reset() + assert bp._healthy_streak == 0 + assert bp.is_healthy() is False + + +class TestCooldown: + """Tests for the post-pressure cooldown period.""" + + def _make_detector(self, **kwargs): + defaults = dict( + high_water_s=_BP_HIGH_WATER_S, + low_water_s=_BP_LOW_WATER_S, + cooldown_s=1.0, + ) + defaults.update(kwargs) + return EMABackpressureDetector(**defaults) + + def test_cooldown_throttles_stores(self): + bp = self._make_detector() + bp._completions = _BP_WARMUP + bp._under_pressure = True + bp._ema = _BP_HIGH_WATER_S * 2 + bp._last_update = time.monotonic() + + with patch("time.monotonic") as mock_time: + t0 = bp._last_update + 10.0 + mock_time.return_value = t0 + assert bp.is_under_pressure() is False + assert bp._pressure_cleared_at > 0 + + mock_time.return_value = t0 + 0.1 + results = [bp.should_store(1) for _ in range(10)] + assert sum(results) == 5, "cooldown should allow ~50%" + + def test_no_cooldown_when_never_pressured(self): + bp = self._make_detector() + for _ in range(20): + assert bp.should_store(1) is True + + def test_no_cooldown_after_window_expires(self): + bp = self._make_detector() + bp._pressure_cleared_at = time.monotonic() - 2.0 + for _ in range(20): + assert bp.should_store(1) is True + + def test_reset_clears_cooldown(self): + bp = self._make_detector() + bp._pressure_cleared_at = time.monotonic() + bp._cooldown_count = 5 + bp.reset() + assert bp._pressure_cleared_at == 0.0 + assert bp._cooldown_count == 0 diff --git a/vllm/v1/kv_offload/tiering/backpressure.py b/vllm/v1/kv_offload/tiering/backpressure.py new file mode 100644 index 0000000000..deba37c94b --- /dev/null +++ b/vllm/v1/kv_offload/tiering/backpressure.py @@ -0,0 +1,384 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from __future__ import annotations + +import time +from abc import ABC, abstractmethod + + +class BackpressurePolicy(ABC): + """Decides what action to take when a detector signals pressure.""" + + @abstractmethod + def should_store(self, detector: BackpressureDetector) -> bool: ... + + @abstractmethod + def on_store_skipped(self, num_blocks: int) -> None: ... + + @abstractmethod + def pop_stores_dropped(self) -> tuple[int, int]: + """Return and reset (stores_dropped, blocks_dropped).""" + ... + + @abstractmethod + def reset(self) -> None: ... + + +class DropAccountingPolicy(BackpressurePolicy): + """Base policy that tracks dropped store/block counters. + + Subclasses implement ``should_store``; the shared drop accounting + (``on_store_skipped``/``pop_stores_dropped``/``reset``) lives here so + every policy reports these counters identically. Subclasses that add + their own state override ``reset`` and call ``super().reset()``. + """ + + def __init__(self): + self._stores_dropped: int = 0 + self._blocks_dropped: int = 0 + + def on_store_skipped(self, num_blocks: int) -> None: + self._stores_dropped += 1 + self._blocks_dropped += num_blocks + + def pop_stores_dropped(self) -> tuple[int, int]: + stores, blocks = self._stores_dropped, self._blocks_dropped + self._stores_dropped = 0 + self._blocks_dropped = 0 + return stores, blocks + + def reset(self) -> None: + self._stores_dropped = 0 + self._blocks_dropped = 0 + + +class DropStorePolicy(DropAccountingPolicy): + """Silently drop stores to pressured tiers.""" + + def should_store(self, detector) -> bool: + return not detector.is_under_pressure() + + +class ThrottledDropPolicy(DropAccountingPolicy): + """Drop stores proportionally to pressure severity. + + Instead of dropping all stores when pressure is detected, the drop + rate scales with how far the EMA exceeds the high watermark:: + + overshoot = (ema - high_water) / high_water + drop_rate = min(1.0, overshoot / ramp_factor) + + With the default ``ramp_factor=2.0``, stores ramp from 0% drop at + the high watermark to 100% drop at 3x the high watermark. This + keeps secondary tiers populated under moderate pressure, reducing + cache misses (and therefore recomputation / ITL regression) while + still protecting TTFT under severe congestion. + """ + + def __init__(self, ramp_factor: float = 2.0): + super().__init__() + self._call_count: int = 0 + self._ramp_factor = ramp_factor + + def should_store(self, detector: BackpressureDetector) -> bool: + # Proportional throttling needs the EMA and high watermark, which + # only the EMA detector exposes. + assert isinstance(detector, EMABackpressureDetector), ( + "ThrottledDropPolicy requires an EMABackpressureDetector" + ) + if not detector.is_under_pressure(): + return True + ema = detector.store_latency_ema + high = detector._high + if high <= 0: + return False + overshoot = max(0.0, ema - high) / high + drop_rate = min(1.0, overshoot / self._ramp_factor) + if drop_rate <= 0: + return True + if drop_rate >= 1.0: + return False + self._call_count += 1 + period = max(2, round(1.0 / drop_rate)) + return (self._call_count % period) != 0 + + def reset(self) -> None: + super().reset() + self._call_count = 0 + + +class BackpressureDetector(ABC): + """Observes store completion signals and determines pressure state.""" + + def __init__( + self, + policy: BackpressurePolicy | None = None, + ): + self._policy = policy or ThrottledDropPolicy() + + @property + def policy(self) -> BackpressurePolicy: + return self._policy + + @abstractmethod + def on_store_completed(self, elapsed_s: float, num_bytes: int) -> None: ... + + @abstractmethod + def is_under_pressure(self) -> bool: ... + + @abstractmethod + def reset(self) -> None: ... + + def update(self, submit_time: float, num_bytes: int) -> None: + """Update pressure state from a completed store job. + + Args: + submit_time: ``time.monotonic()`` when the job was submitted. + num_bytes: Total bytes written (num_blocks * block_size_bytes). + + """ + if num_bytes <= 0: + return + elapsed = time.monotonic() - submit_time + self.on_store_completed(elapsed, num_bytes) + + def should_store(self, num_blocks: int) -> bool: + """Check policy and record skip if rejected.""" + if not self._policy.should_store(self): + self._policy.on_store_skipped(num_blocks) + return False + return True + + @classmethod + def default_config( + cls, tier_type: str, *, locality: str | None = None + ) -> dict | None: + """Return default constructor kwargs for ``tier_type``, or None. + + Args: + tier_type: The tier type string (e.g. ``"fs"``, ``"obj"``). + locality: Optional locality hint from the tier config + (``"LOCAL"`` or ``"REMOTE"``). When ``"REMOTE"``, tiers + that would normally get local-storage watermarks (e.g. + ``"fs"``) receive network watermarks instead. + + """ + return None + + @property + def stats(self) -> dict[str, float]: + return {} + + +class EMABackpressureDetector(BackpressureDetector): + """EMA of store latency normalized by transfer size. + + The EMA tracks seconds per megabyte (s/MiB) so that the metric is + comparable regardless of how many blocks are in a job or how large + each block is. Water marks are in the same unit. + + Default water marks are derived from fio benchmarks on the WDC H100 + cluster. Two presets are provided: + + LOCAL (NVMe/SSD): NVMe sustains ~5 GB/s writes. + high=0.005 s/MiB (~200 MB/s) catches severe congestion; + low=0.001 s/MiB (~1 GB/s) requires meaningful recovery. + + NETWORK (CephFS, object store, ``obj``tiers, or any tier + with ``"locality": "REMOTE"``): CephFS sustains ~1.5 GB/s. + high=0.020 s/MiB (~50 MB/s); low=0.005 s/MiB (~200 MB/s). + + ``obj`` tiers get NETWORK defaults automatically. An + ``fs`` tier defaults to LOCAL (appropriate for NVMe/SSD); set + ``"locality": "REMOTE"`` in the tier config for network-backed + filesystems like CephFS. + """ + + _MIB = 1 << 20 + + # EMA smoothing factor: higher values (→1) react faster to latency + # spikes but are noisier; lower values (→0) smooth more but lag. + DEFAULT_ALPHA = 0.15 + + # Number of store completions to collect before seeding the EMA. + # During warmup, pressure is never signalled; the EMA is initialized + # to the mean of the warmup samples to avoid cold-start false positives. + DEFAULT_WARMUP_COMPLETIONS = 10 + + # When under pressure and no completions arrive, the EMA decays + # toward zero so the detector can eventually recover. The half-life + # controls how quickly: after this many seconds of silence the EMA + # halves. With a 2 s half-life and high_water=0.005, an EMA frozen + # at 0.010 reaches low_water=0.001 in ~6.6 s. + DEFAULT_DECAY_HALF_LIFE_S = 2.0 + + # After pressure clears, stores are throttled to 50% for this many + # seconds to prevent a burst of stores from re-overwhelming the tier. + DEFAULT_COOLDOWN_S = 1.0 + + # Number of consecutive sub-low-watermark completions before the + # detector enters a "healthy" fast-path that bypasses policy checks. + _HEALTHY_THRESHOLD = 10 + + LOCAL_HIGH_WATER_S = 0.010 + LOCAL_LOW_WATER_S = 0.002 + + NETWORK_HIGH_WATER_S = 0.050 + NETWORK_LOW_WATER_S = 0.010 + + _NETWORK_TIER_TYPES = frozenset({"obj"}) + + @classmethod + def default_config( + cls, tier_type: str, *, locality: str | None = None + ) -> dict | None: + is_remote = locality is not None and locality.upper() == "REMOTE" + if is_remote or tier_type in cls._NETWORK_TIER_TYPES: + return { + "high_water_s": cls.NETWORK_HIGH_WATER_S, + "low_water_s": cls.NETWORK_LOW_WATER_S, + } + return { + "high_water_s": cls.LOCAL_HIGH_WATER_S, + "low_water_s": cls.LOCAL_LOW_WATER_S, + } + + def __init__( + self, + high_water_s: float, + low_water_s: float, + alpha: float = DEFAULT_ALPHA, + warmup_completions: int = DEFAULT_WARMUP_COMPLETIONS, + decay_half_life_s: float = DEFAULT_DECAY_HALF_LIFE_S, + cooldown_s: float = DEFAULT_COOLDOWN_S, + policy: BackpressurePolicy | None = None, + ): + super().__init__(policy=policy) + if not (0 < alpha <= 1): + raise ValueError(f"alpha must be in (0, 1], got {alpha}") + if low_water_s > high_water_s: + raise ValueError( + f"low_water_s ({low_water_s}) must be <= high_water_s ({high_water_s})" + ) + + self._alpha = alpha + self._high = high_water_s + self._low = low_water_s + self._warmup_completions = warmup_completions + self._decay_half_life_s = decay_half_life_s + self._cooldown_s = cooldown_s + # Exponential moving average of store latency in seconds. + self._ema: float = 0.0 + # Whether the tier is currently considered under pressure. + self._under_pressure: bool = False + # Number of store completions observed (for warmup gating). + self._completions: int = 0 + # Samples collected during warmup; used to seed the EMA. + self._warmup_samples: list[float] = [] + # Monotonic timestamp of the last EMA update; used for idle decay. + self._last_update: float = 0.0 + # Consecutive completions with EMA below low watermark. + self._healthy_streak: int = 0 + # Monotonic timestamp when pressure was last cleared (for cooldown). + self._pressure_cleared_at: float = 0.0 + self._cooldown_count: int = 0 + + def _apply_idle_decay(self) -> None: + """Decay the EMA toward zero based on elapsed idle time. + + Uses the formula ``ema * 0.5^(dt / half_life)`` so the EMA + halves every ``decay_half_life_s`` seconds of silence. This + lets the detector recover from pressure when no stores are + attempted (and therefore no completions arrive to update the + EMA organically). + """ + now = time.monotonic() + dt = now - self._last_update + if dt <= 0 or self._decay_half_life_s <= 0: + return + self._ema *= 0.5 ** (dt / self._decay_half_life_s) + self._last_update = now + + def on_store_completed(self, elapsed_s: float, num_bytes: int) -> None: + now = time.monotonic() + s_per_mib = elapsed_s / (num_bytes / self._MIB) + self._completions += 1 + if self._completions <= self._warmup_completions: + self._warmup_samples.append(s_per_mib) + if self._completions == self._warmup_completions: + self._ema = sum(self._warmup_samples) / len(self._warmup_samples) + self._last_update = now + return + self._ema = self._alpha * s_per_mib + (1 - self._alpha) * self._ema + self._last_update = now + if self._ema > self._high: + self._under_pressure = True + self._pressure_cleared_at = 0.0 + self._healthy_streak = 0 + elif self._ema < self._low: + if self._under_pressure: + self._pressure_cleared_at = now + self._cooldown_count = 0 + self._under_pressure = False + self._healthy_streak += 1 + else: + self._healthy_streak = 0 + + def is_under_pressure(self) -> bool: + if self._under_pressure and self._completions >= self._warmup_completions: + self._apply_idle_decay() + if self._ema < self._low: + self._under_pressure = False + self._pressure_cleared_at = time.monotonic() + self._cooldown_count = 0 + return self._under_pressure + + def is_healthy(self) -> bool: + """True when the tier has been consistently fast. + + After ``_HEALTHY_THRESHOLD`` consecutive store completions with + EMA below the low watermark (and no active pressure), the + detector enters a healthy fast-path: ``should_store()`` returns + ``True`` immediately, skipping the policy check entirely. + """ + return ( + not self._under_pressure and self._healthy_streak >= self._HEALTHY_THRESHOLD + ) + + def should_store(self, num_blocks: int) -> bool: + under_pressure = self.is_under_pressure() + if self.is_healthy(): + return True + if ( + not under_pressure + and self._pressure_cleared_at > 0 + and time.monotonic() - self._pressure_cleared_at < self._cooldown_s + ): + self._cooldown_count += 1 + if self._cooldown_count % 2 == 0: + self._policy.on_store_skipped(num_blocks) + return False + return super().should_store(num_blocks) + + def reset(self) -> None: + self._ema = 0.0 + self._under_pressure = False + self._completions = 0 + self._warmup_samples.clear() + self._last_update = 0.0 + self._healthy_streak = 0 + self._pressure_cleared_at = 0.0 + self._cooldown_count = 0 + self._policy.reset() + + @property + def store_latency_ema(self) -> float: + return self._ema + + @store_latency_ema.setter + def store_latency_ema(self, value: float) -> None: + self._ema = value + + @property + def stats(self) -> dict[str, float]: + return {"store_latency_ema": self._ema} diff --git a/vllm/v1/kv_offload/tiering/base.py b/vllm/v1/kv_offload/tiering/base.py index 77300fa707..a90d33eb11 100644 --- a/vllm/v1/kv_offload/tiering/base.py +++ b/vllm/v1/kv_offload/tiering/base.py @@ -2,9 +2,10 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Abstract interfaces and data types for the secondary tiering layer.""" +import time from abc import ABC, abstractmethod from collections.abc import Collection, Iterable -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, ClassVar import numpy as np @@ -20,6 +21,9 @@ from vllm.v1.kv_offload.base import ( RequestOffloadingContext, ScheduleEndContext, ) +from vllm.v1.kv_offload.tiering.backpressure import ( + BackpressureDetector, +) if TYPE_CHECKING: from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import ( @@ -52,6 +56,11 @@ class TieringOffloadingMetrics: ) ACTIVE_PROMOTION_JOBS = "vllm:kv_offload_tiering_active_promotion_jobs" ACTIVE_CASCADE_JOBS = "vllm:kv_offload_tiering_active_cascade_jobs" + BACKPRESSURE_STORE_LATENCY_EMA = ( + "vllm:kv_offload_tiering_backpressure_store_latency_ema" + ) + BACKPRESSURE_STORES_DROPPED = "vllm:kv_offload_tiering_backpressure_stores_dropped" + BACKPRESSURE_BLOCKS_DROPPED = "vllm:kv_offload_tiering_backpressure_blocks_dropped" @dataclass @@ -63,6 +72,7 @@ class TransferJob: chunk_ids: np.ndarray is_promotion: bool req_context: ReqContext + submit_time: float = field(default_factory=time.monotonic) @dataclass @@ -77,6 +87,7 @@ class JobResult: # indicated by `success`. Must be a subset of the job's original keys. successful_keys: Collection[OffloadKey] | None = None transfer_time: float | None = None + transfer_bytes: int | None = None class ParentManager(ABC): @@ -136,18 +147,33 @@ class SecondaryTierManager(ABC): offloading_spec: "OffloadingSpec", primary_kv_view: memoryview, tier_type: str, + backpressure_detector: BackpressureDetector | None = None, ) -> None: """Args: offloading_spec: Offloading configuration. primary_kv_view: Memoryview of the primary tier's CPU KV cache. tier_type: Tier type identifier, set by SecondaryTierFactory from the registered tier type. + backpressure_detector: Optional `BackpressureDetector`. """ self._offloading_spec = offloading_spec self._primary_kv_view: memoryview = primary_kv_view + assert primary_kv_view.strides is not None, ( + "primary_kv_view.strides cannot be None" + ) + self._block_size_bytes: int = primary_kv_view.strides[0] self.tier_type = tier_type self.locality: Locality | None = None + self._bp_detector = backpressure_detector + + @property + def block_size_bytes(self) -> int: + return self._block_size_bytes + + @property + def bp_detector(self) -> BackpressureDetector | None: + return self._bp_detector @abstractmethod def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult: diff --git a/vllm/v1/kv_offload/tiering/example/manager.py b/vllm/v1/kv_offload/tiering/example/manager.py index 5ec13f62da..e3f7c93d7a 100644 --- a/vllm/v1/kv_offload/tiering/example/manager.py +++ b/vllm/v1/kv_offload/tiering/example/manager.py @@ -31,6 +31,7 @@ logger = logging.getLogger(__name__) if TYPE_CHECKING: from vllm.v1.kv_offload.base import OffloadingSpec + from vllm.v1.kv_offload.tiering.backpressure import BackpressureDetector class ExampleSecondaryTierManager(SecondaryTierManager): @@ -49,6 +50,7 @@ class ExampleSecondaryTierManager(SecondaryTierManager): primary_kv_view: memoryview, tier_type: str, custom_param: int = 0, + backpressure_detector: "BackpressureDetector | None" = None, ): """Initialize the example secondary tier. @@ -57,12 +59,14 @@ class ExampleSecondaryTierManager(SecondaryTierManager): primary_kv_view: Memoryview over the primary tier's KV buffer. tier_type: Name identifying this tier type. custom_param: Dummy parameter demonstrating custom args. + backpressure_detector: Optional backpressure detector. """ super().__init__( offloading_spec=offloading_spec, primary_kv_view=primary_kv_view, tier_type=tier_type, + backpressure_detector=backpressure_detector, ) logger.info( diff --git a/vllm/v1/kv_offload/tiering/factory.py b/vllm/v1/kv_offload/tiering/factory.py index 086392a406..6a460724de 100644 --- a/vllm/v1/kv_offload/tiering/factory.py +++ b/vllm/v1/kv_offload/tiering/factory.py @@ -8,11 +8,33 @@ from vllm.logger import init_logger from vllm.v1.kv_offload.tiering.base import SecondaryTierManager if TYPE_CHECKING: + from vllm.v1.kv_offload.backpressure import BackpressureDetector from vllm.v1.kv_offload.base import OffloadingSpec logger = init_logger(__name__) +_DEFAULT_PACKAGE = "vllm.v1.kv_offload.tiering.backpressure" + + +def _import_class(name: str) -> type: + """Import a class by name. + + If ``name`` contains a dot it is treated as a fully qualified path + (e.g. ``'my_package.MyDetector'``). Otherwise it is looked up in + the default backpressure module + (``vllm.v1.kv_offload.tiering.backpressure``), so short names like + ``'DropStorePolicy'`` or ``'EMABackpressureDetector'`` work out of + the box. + """ + if "." in name: + module_path, _, class_name = name.rpartition(".") + else: + module_path, class_name = _DEFAULT_PACKAGE, name + module = importlib.import_module(module_path) + return getattr(module, class_name) + + class SecondaryTierFactory: """Registry for SecondaryTierManager implementations, resolved by type. @@ -47,10 +69,30 @@ class SecondaryTierFactory: config = tier_config.copy() tier_type = config.pop("type") config.pop("module_path", None) + locality = config.get("locality") + bp_config = config.pop("backpressure", None) + bp_detector = None + if bp_config is not None: + bp_config = bp_config.copy() + bp_cls_name = bp_config.pop("backpressure_cls", None) + if bp_cls_name is None: + raise ValueError( + "backpressure config must include 'backpressure_cls' " + "(e.g. 'EMABackpressureDetector')" + ) + policy_cls_name = bp_config.pop("policy_cls", None) + policy = _import_class(policy_cls_name)() if policy_cls_name else None + detector_cls: type[BackpressureDetector] = _import_class(bp_cls_name) + defaults = detector_cls.default_config(tier_type, locality=locality) + if defaults: + for k, v in defaults.items(): + bp_config.setdefault(k, v) + bp_detector = detector_cls(policy=policy, **bp_config) return tier_cls( offloading_spec=offloading_spec, primary_kv_view=primary_kv_view, tier_type=tier_type, + backpressure_detector=bp_detector, **config, ) diff --git a/vllm/v1/kv_offload/tiering/fs/manager.py b/vllm/v1/kv_offload/tiering/fs/manager.py index 3d51d95774..6522cfd236 100644 --- a/vllm/v1/kv_offload/tiering/fs/manager.py +++ b/vllm/v1/kv_offload/tiering/fs/manager.py @@ -40,6 +40,7 @@ from vllm.v1.kv_offload.base import ( ) from vllm.v1.kv_offload.file_mapper import FileMapper from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager +from vllm.v1.kv_offload.tiering.backpressure import BackpressureDetector from vllm.v1.kv_offload.tiering.base import ( JobId, JobResult, @@ -115,6 +116,7 @@ class FileSystemTierManager(SecondaryTierManager): n_write_threads: int = 16, enable_kv_events: bool = False, locality: str | None = None, + backpressure_detector: BackpressureDetector | None = None, ): """Args: offloading_spec: Contains normalized offloading configuration and @@ -129,9 +131,12 @@ class FileSystemTierManager(SecondaryTierManager): cache events are enabled globally (kv_events_config). locality: Whether this tier's storage is LOCAL or REMOTE relative to the publishing vLLM instance. + backpressure_detector: Optional backpressure detector. """ - super().__init__(offloading_spec, primary_kv_view, tier_type) + super().__init__( + offloading_spec, primary_kv_view, tier_type, backpressure_detector + ) self.locality = Locality(locality) if locality is not None else None self.events: list[OffloadingEvent] | None = None @@ -150,6 +155,8 @@ class FileSystemTierManager(SecondaryTierManager): # Keys of in-flight load (promotion) jobs, so a failed load can mark # its own cached lookup verdicts False (see get_finished_jobs). self._load_job_keys: dict[JobId, list[OffloadKey]] = {} + # Block count per in-flight job, used to report transfer_bytes. + self._job_block_counts: dict[JobId, int] = {} # Per load job: how many blocks loaded before a failure (partial keep). # Written by the pool worker inside the load task before it raises (so # before task_done publishes the job); read on the scheduler thread in @@ -225,6 +232,7 @@ class FileSystemTierManager(SecondaryTierManager): self._block_size, self._use_o_direct, ) + self._job_block_counts[job_metadata.job_id] = len(keys) self._pool.enqueue_store(job_metadata.job_id, 1, [task]) @override @@ -234,6 +242,7 @@ class FileSystemTierManager(SecondaryTierManager): # keys as a miss (see get_finished_jobs). keys = list(job_metadata.keys) self._load_job_keys[job_id] = keys + self._job_block_counts[job_id] = len(keys) paths = [self.file_mapper.get_file_name(key) for key in keys] offsets = [int(cid) * self._block_size for cid in job_metadata.chunk_ids] @@ -272,6 +281,8 @@ class FileSystemTierManager(SecondaryTierManager): as a miss here (scheduler thread).""" results = [] for job_id, success, transfer_time in self._pool.get_finished(): + block_count = self._job_block_counts.pop(job_id, 0) + transfer_bytes = block_count * self._block_size if block_count else None if self.events is not None: keys = self._store_job_keys.pop(job_id, None) if success and keys: @@ -299,6 +310,7 @@ class FileSystemTierManager(SecondaryTierManager): success=False, successful_keys=tuple(successful) if successful else None, transfer_time=transfer_time, + transfer_bytes=transfer_bytes, ) ) continue @@ -307,6 +319,7 @@ class FileSystemTierManager(SecondaryTierManager): job_id=job_id, success=success, transfer_time=transfer_time, + transfer_bytes=transfer_bytes, ) ) return results diff --git a/vllm/v1/kv_offload/tiering/manager.py b/vllm/v1/kv_offload/tiering/manager.py index 6e0bf90a33..ce518042b6 100644 --- a/vllm/v1/kv_offload/tiering/manager.py +++ b/vllm/v1/kv_offload/tiering/manager.py @@ -238,6 +238,10 @@ class TieringOffloadingManager(OffloadingManager): for tier_idx, tier in enumerate(self.secondary_tiers) } + self._tier_index: dict[SecondaryTierManager, int] = { + tier: i for i, tier in enumerate(self.secondary_tiers) + } + @property def _transfer_jobs(self) -> dict[JobId, JobMetadata]: return self._jobs @@ -337,6 +341,43 @@ class TieringOffloadingManager(OffloadingManager): self.primary_tier.complete_read( transfer_job.keys, transfer_job.req_context ) + if completed_job.success: + self._update_backpressure(tier, job_metadata, completed_job) + + def _should_store_to_tier( + self, tier: SecondaryTierManager, num_blocks: int + ) -> bool: + detector = tier.bp_detector + if detector is None: + return True + return detector.should_store(num_blocks) + + def _update_backpressure( + self, + tier: SecondaryTierManager, + job_metadata: JobMetadata, + completed_job: JobResult, + ) -> None: + detector = tier.bp_detector + if detector is None: + return + was_under_pressure = detector.is_under_pressure() + tj = job_metadata.transfer_job + num_bytes = ( + completed_job.transfer_bytes + if completed_job.transfer_bytes is not None + else len(tj.keys) * tier.block_size_bytes + ) + detector.update(tj.submit_time, num_bytes) + if detector.is_under_pressure() != was_under_pressure: + tier_idx = self._tier_index[tier] + logger.info( + "Tier #%d (%s) back-pressure %s (stats=%s)", + tier_idx, + tier.tier_type, + "activated" if detector.is_under_pressure() else "cleared", + detector.stats, + ) @override def lookup( @@ -643,8 +684,10 @@ class TieringOffloadingManager(OffloadingManager): return for tier_idx in request_level_tiers: - job_metadata = self.create_store_job(ready_keys, req_context, tier_idx) tier = self.secondary_tiers[tier_idx] + if not self._should_store_to_tier(tier, len(ready_keys)): + continue + job_metadata = self.create_store_job(ready_keys, req_context, tier_idx) tier.submit_store(job_metadata) def _flush_pending_cascades(self) -> None: @@ -698,6 +741,8 @@ class TieringOffloadingManager(OffloadingManager): # eviction during the async transfer). One prepare_read() call per # secondary tier. for tier_idx, tier in enumerate(self.secondary_tiers): + if not self._should_store_to_tier(tier, len(keys)): + continue job_metadata = self.create_store_job(keys, req_context, tier_idx) tier.submit_store(job_metadata) @@ -904,6 +949,10 @@ class TieringOffloadingManager(OffloadingManager): del self._req_state[req_id] self._processed_jobs_this_step = False + for tier in self.secondary_tiers: + if tier.bp_detector is not None: + tier.bp_detector.reset() + @override def get_stats(self) -> OffloadingConnectorStats | None: stats = self.primary_tier.get_stats() @@ -911,6 +960,7 @@ class TieringOffloadingManager(OffloadingManager): if stats is not None and stats.is_empty(): stats = None + self._metrics.record_backpressure(self.secondary_tiers) metrics_stats = self._metrics.take_stats() if metrics_stats is not None: if stats is None: diff --git a/vllm/v1/kv_offload/tiering/metrics.py b/vllm/v1/kv_offload/tiering/metrics.py index 55d5699939..240ea57dd7 100644 --- a/vllm/v1/kv_offload/tiering/metrics.py +++ b/vllm/v1/kv_offload/tiering/metrics.py @@ -12,6 +12,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import ( from vllm.v1.kv_offload.base import LookupResult, OffloadKey, ReqContext from vllm.v1.kv_offload.tiering.base import ( JobResult, + SecondaryTierManager, TieringOffloadingMetrics, TransferJob, ) @@ -130,6 +131,36 @@ class TieringMetricsTracker: TieringOffloadingMetrics.PROMOTION_ALLOCATION_FAILURES ) + def record_backpressure( + self, + tiers: list[SecondaryTierManager], + ) -> None: + for i, tier in enumerate(tiers): + detector = tier.bp_detector + if detector is None: + continue + label = self.tier_label(i) + ema = detector.stats.get("store_latency_ema") + if ema is not None: + self._stats.set_gauge( + TieringOffloadingMetrics.BACKPRESSURE_STORE_LATENCY_EMA, + ema, + labelvalues=label, + ) + stores_dropped, blocks_dropped = detector.policy.pop_stores_dropped() + if stores_dropped > 0: + self._stats.increase_counter( + TieringOffloadingMetrics.BACKPRESSURE_STORES_DROPPED, + stores_dropped, + labelvalues=label, + ) + if blocks_dropped > 0: + self._stats.increase_counter( + TieringOffloadingMetrics.BACKPRESSURE_BLOCKS_DROPPED, + blocks_dropped, + labelvalues=label, + ) + def take_stats(self) -> OffloadingConnectorStats | None: active_transfer_stats = OffloadingConnectorStats() self._observe_active_transfer_stats(active_transfer_stats) diff --git a/vllm/v1/kv_offload/tiering/obj/manager.py b/vllm/v1/kv_offload/tiering/obj/manager.py index 3d6c66c483..1886e3b811 100644 --- a/vllm/v1/kv_offload/tiering/obj/manager.py +++ b/vllm/v1/kv_offload/tiering/obj/manager.py @@ -22,6 +22,7 @@ from vllm.v1.kv_offload.base import ( ) from vllm.v1.kv_offload.file_mapper import FileMapper from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager +from vllm.v1.kv_offload.tiering.backpressure import BackpressureDetector from vllm.v1.kv_offload.tiering.base import ( JobId, JobResult, @@ -112,6 +113,7 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager): io_threads: int = 4, enable_kv_events: bool = False, locality: str | None = None, + backpressure_detector: BackpressureDetector | None = None, ): """Args: offloading_spec: Offloading configuration. @@ -125,9 +127,15 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager): cache events are enabled globally (kv_events_config). locality: Whether this tier's storage is LOCAL or REMOTE relative to the publishing vLLM instance. + backpressure_detector: Optional backpressure detector. """ - super().__init__(offloading_spec, primary_kv_view, tier_type) + super().__init__( + offloading_spec, + primary_kv_view, + tier_type, + backpressure_detector, + ) self.locality = Locality(locality) if locality is not None else None self.events: list[OffloadingEvent] | None = None diff --git a/vllm/v1/kv_offload/tiering/p2p/manager.py b/vllm/v1/kv_offload/tiering/p2p/manager.py index 703c632899..59b9c32069 100644 --- a/vllm/v1/kv_offload/tiering/p2p/manager.py +++ b/vllm/v1/kv_offload/tiering/p2p/manager.py @@ -265,7 +265,25 @@ class P2PSecondaryTierManager(SecondaryTierManager): number, or anything convertible to one. """ - super().__init__(offloading_spec, primary_kv_view, tier_type) + backpressure_detector = kwargs.pop("backpressure_detector", None) + if backpressure_detector is not None: + # The generic (store-latency) detector is unreliable for P2P: in + # PD mode dropping a store can leave the decoder waiting until the + # load timeout instead of failing fast, and rendezvous time makes + # store latency a poor pressure signal. Reject it until a + # P2P-specific fail-fast path exists. + raise ValueError( + "Backpressure is not supported for the P2P secondary tier. " + "The generic store-latency detector cannot fail fast in PD " + "mode and rendezvous time makes store latency an unreliable " + "pressure signal. Remove the 'backpressure' config from the " + "p2p tier." + ) + super().__init__( + offloading_spec, + primary_kv_view, + tier_type, + ) try: timeout_s = float(unbound_store_timeout_s) except (TypeError, ValueError): diff --git a/vllm/v1/kv_offload/tiering/spec.py b/vllm/v1/kv_offload/tiering/spec.py index 9adac0f787..93c6cbabcb 100644 --- a/vllm/v1/kv_offload/tiering/spec.py +++ b/vllm/v1/kv_offload/tiering/spec.py @@ -247,6 +247,34 @@ class TieringOffloadingSpec(CPUOffloadingSpec): assert isinstance(tier_config, dict) tier_cls = SecondaryTierFactory.get_tier_class(tier_config) metrics.update(tier_cls.build_metric_definitions(tier_config)) + + metrics[TieringOffloadingMetrics.BACKPRESSURE_STORE_LATENCY_EMA] = ( + OffloadingGaugeMetadata( + documentation=( + "Exponential moving average of store latency " + "for back-pressure detection, in s/MiB." + ), + labelnames=("tier",), + ) + ) + metrics[TieringOffloadingMetrics.BACKPRESSURE_STORES_DROPPED] = ( + OffloadingCounterMetadata( + documentation=( + "Number of store operations dropped due to " + "back-pressure on a secondary tier." + ), + labelnames=("tier",), + ) + ) + metrics[TieringOffloadingMetrics.BACKPRESSURE_BLOCKS_DROPPED] = ( + OffloadingCounterMetadata( + documentation=( + "Number of blocks dropped due to back-pressure on a secondary tier." + ), + labelnames=("tier",), + ) + ) + return metrics def __init__(self, config: OffloadingConfig): @@ -259,6 +287,32 @@ class TieringOffloadingSpec(CPUOffloadingSpec): if not isinstance(self.secondary_tier_configs, list): raise ValueError("secondary_tiers must be a list of tier configurations") + # Backpressure config is merged field-by-field in priority order + # (highest first): + # 1. Per-tier ``backpressure`` dict in the tier config + # 2. Top-level ``backpressure`` in kv_connector_extra_config + # Merging per field (rather than per whole dict) means a partial + # tier override still inherits missing fields from the top-level + # default, so the resolved dict reaching the factory is complete. + # Within each tier's resolved dict, tier-type-aware water marks + # are filled in last so a bare ``"backpressure": {}`` picks up + # sensible thresholds for the storage medium. + bp_defaults = self.extra_config.get("backpressure") + + for tier_cfg in self.secondary_tier_configs: + tier_override = tier_cfg.get("backpressure") + # Overlay from lowest to highest precedence so higher-precedence + # fields win while lower-precedence ones fill in the gaps. + merged: dict[str, Any] = {} + for source in (bp_defaults, tier_override): + if source: + merged.update(source) + # Only set a resolved dict when at least one source contributed + # (or an explicit ``backpressure`` key was present, e.g. ``{}``), + # so tiers without any backpressure config stay unconfigured. + if merged or "backpressure" in tier_cfg: + tier_cfg["backpressure"] = merged + # Scheduler-side mmap (rank=None); kept for cleanup self._scheduler_mmap: SharedOffloadRegion | None = None