[kv_offload] Emit tier-owned BlockStored events from FS/OBJ secondary tiers (#47923)

Signed-off-by: Change72 <changg@nvidia.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Chang Guo
2026-07-10 06:17:23 +03:00
committed by GitHub
co-authored by Claude Fable 5 Or Ozeri
parent 88e5e2c57b
commit 2d814a0082
6 changed files with 433 additions and 9 deletions
+196
View File
@@ -18,8 +18,10 @@ import numpy as np
import pytest
import torch
from vllm.distributed.kv_events import MEDIUM_FS
from vllm.v1.kv_offload.base import (
LookupResult,
OffloadingEvent,
OffloadKey,
ReqContext,
ScheduleEndContext,
@@ -58,6 +60,16 @@ _MOCK_OFFLOADING_SPEC.kv_cache_config = _MOCK_KV_CACHE_CONFIG
_MOCK_OFFLOADING_SPEC.block_size_factor = 1
def _make_offloading_spec(enable_kv_cache_events: bool) -> MagicMock:
"""Mock spec with an explicit global KV events flag."""
spec = MagicMock()
spec.vllm_config = _MOCK_VLLM_CONFIG
spec.kv_cache_config = _MOCK_KV_CACHE_CONFIG
spec.block_size_factor = 1
spec.kv_events_config.enable_kv_cache_events = enable_kv_cache_events
return spec
def key(n: int) -> OffloadKey:
return make_offload_key(n.to_bytes(8, "big"), 0)
@@ -164,6 +176,23 @@ def fs_tier(tmp_path):
tier.shutdown()
@pytest.fixture
def fs_tier_with_events(tmp_path):
tensor = _page_aligned_zero_tensor(4, _BLOCK_ELEMENTS)
mock_view = memoryview(tensor.numpy())
tier = FileSystemTierManager(
offloading_spec=_make_offloading_spec(enable_kv_cache_events=True),
primary_kv_view=mock_view,
tier_type="fs",
root_dir=str(tmp_path),
n_read_threads=4,
n_write_threads=4,
enable_kv_events=True,
)
yield tier
tier.shutdown()
# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------
@@ -396,3 +425,170 @@ def test_batch_lookup_dispatch(fs_tier, monkeypatch, use_c_ext):
results = lookup_and_wait(tier, [key(1), key(2)])
assert results == [LookupResult.HIT, LookupResult.MISS]
# ---------------------------------------------------------------------------
# KV events
# ---------------------------------------------------------------------------
def test_successful_store_emits_stored_event(fs_tier_with_events):
"""A completed store job emits one stored event with the job's keys."""
tier = fs_tier_with_events
keys = [key(1), key(2)]
tier.submit_store(make_job(1, keys, [0, 1]))
assert all(r.success for r in drain(tier))
events = list(tier.take_events())
assert len(events) == 1
assert events[0].keys == keys
# Literal medium pins the wire contract, not just the constant choice.
assert events[0].medium == "FS"
assert not events[0].removed
# take_events drains the buffer.
assert list(tier.take_events()) == []
def test_load_job_emits_no_event(fs_tier_with_events):
tier = fs_tier_with_events
tier.submit_store(make_job(1, [key(1)], [0]))
results = drain(tier)
assert len(results) == 1
assert results[0].success
list(tier.take_events())
tier.submit_load(make_job(2, [key(1)], [1], is_promotion=True))
results = drain(tier)
assert len(results) == 1
assert results[0].success
assert list(tier.take_events()) == []
def test_mixed_job_results_emit_event_only_for_successful_job(
fs_tier_with_events, monkeypatch
):
"""With a failed and a successful store job in flight, exactly one event
is emitted and its keys belong to the successful job."""
import vllm.v1.kv_offload.tiering.fs.manager as mgr_mod
tier = fs_tier_with_events
failing_path = tier.file_mapper.get_file_name(key(1))
original_store_block = mgr_mod.store_block
def flaky_store_block(dest_path, *args, **kwargs):
if dest_path == failing_path:
raise OSError("injected store failure")
return original_store_block(dest_path, *args, **kwargs)
monkeypatch.setattr(mgr_mod, "store_block", flaky_store_block)
tier.submit_store(make_job(1, [key(1)], [0]))
tier.submit_store(make_job(2, [key(2)], [1]))
results = drain(tier)
assert len(results) == 2
by_id = {r.job_id: r for r in results}
assert not by_id[1].success
assert by_id[2].success
events = list(tier.take_events())
assert len(events) == 1
assert events[0].keys == [key(2)]
def test_partially_failed_store_emits_no_event(fs_tier_with_events, monkeypatch):
"""A store job with any failed block emits no event for the whole job."""
import vllm.v1.kv_offload.tiering.fs.manager as mgr_mod
tier = fs_tier_with_events
failing_path = tier.file_mapper.get_file_name(key(2))
original_store_block = mgr_mod.store_block
def flaky_store_block(dest_path, *args, **kwargs):
if dest_path == failing_path:
raise OSError("injected store failure")
return original_store_block(dest_path, *args, **kwargs)
monkeypatch.setattr(mgr_mod, "store_block", flaky_store_block)
tier.submit_store(make_job(1, [key(1), key(2)], [0, 1]))
results = drain(tier)
assert len(results) == 1
assert not results[0].success
assert list(tier.take_events()) == []
assert tier._store_job_keys == {}
def test_events_disabled_by_default(fs_tier):
tier, _ = fs_tier
tier.submit_store(make_job(1, [key(1)], [0]))
results = drain(tier)
assert len(results) == 1
assert results[0].success
assert tier.events is None
assert tier._store_job_keys == {}
assert list(tier.take_events()) == []
def test_events_require_global_kv_events_flag(tmp_path):
"""Tier-level opt-in alone is not enough; the global flag gates events."""
tensor = _page_aligned_zero_tensor(4, _BLOCK_ELEMENTS)
tier = FileSystemTierManager(
offloading_spec=_make_offloading_spec(enable_kv_cache_events=False),
primary_kv_view=memoryview(tensor.numpy()),
tier_type="fs",
root_dir=str(tmp_path),
enable_kv_events=True,
)
try:
assert tier.events is None
tier.submit_store(make_job(1, [key(1)], [0]))
results = drain(tier)
assert len(results) == 1
assert results[0].success
assert list(tier.take_events()) == []
assert tier._store_job_keys == {}
finally:
tier.shutdown()
def test_cascade_store_emits_fs_event_through_tiering_manager(tmp_path):
"""A GPU->CPU->fs cascade surfaces the tier-owned FS stored event via the
TieringOffloadingManager's aggregated take_events()."""
from vllm.v1.kv_offload.tiering.manager import (
CPUPrimaryTierOffloadingManager,
TieringOffloadingManager,
)
tensor = _page_aligned_zero_tensor(4, _BLOCK_ELEMENTS)
view = memoryview(tensor.numpy())
mock_region = MagicMock()
mock_region.create_kv_memoryview.return_value = view
primary = CPUPrimaryTierOffloadingManager(num_blocks=4, mmap_region=mock_region)
tier = FileSystemTierManager(
offloading_spec=_make_offloading_spec(enable_kv_cache_events=True),
primary_kv_view=primary.get_kv_memoryview(),
tier_type="fs",
root_dir=str(tmp_path),
enable_kv_events=True,
)
manager = TieringOffloadingManager(primary_tier=primary, secondary_tiers=[tier])
try:
keys = [key(1), key(2)]
manager.on_new_request(_CTX)
assert manager.prepare_store(keys, _CTX) is not None
manager.complete_store(keys, _CTX) # cascades to the fs tier
events: list[OffloadingEvent] = []
ctx = ScheduleEndContext(new_req_ids=[], preempted_req_ids=())
deadline = time.monotonic() + 5.0
while time.monotonic() < deadline and not events:
manager.on_schedule_end(ctx)
events.extend(manager.take_events())
time.sleep(0.01)
fs_events = [e for e in events if e.medium == MEDIUM_FS]
assert len(fs_events) == 1
assert set(fs_events[0].keys) == set(keys)
assert not fs_events[0].removed
finally:
tier.shutdown()
+107 -1
View File
@@ -179,8 +179,19 @@ class MockNixlAgent:
# ---------------------------------------------------------------------------
def _make_events_spec(enable_kv_cache_events: bool) -> SimpleNamespace:
"""Offloading spec stub with an explicit global KV events flag."""
return SimpleNamespace(
vllm_config=_make_vllm_config(),
kv_cache_config=SimpleNamespace(kv_cache_groups=[]),
kv_events_config=SimpleNamespace(enable_kv_cache_events=enable_kv_cache_events),
)
def _make_tier(
num_blocks: int = 4,
offloading_spec: SimpleNamespace = _OFFLOADING_SPEC,
**tier_kwargs,
) -> tuple[ObjectStoreSecondaryTierManager, MockNixlAgent]:
"""Create a tier backed by a fresh MockNixlAgent."""
mock_agent = MockNixlAgent()
@@ -194,11 +205,12 @@ def _make_tier(
),
):
tier = ObjectStoreSecondaryTierManager(
offloading_spec=_OFFLOADING_SPEC,
offloading_spec=offloading_spec,
primary_kv_view=view,
tier_type="obj",
store_config=_STORE_CONFIG,
prefix=_RUN_PREFIX,
**tier_kwargs,
)
return tier, mock_agent
@@ -421,6 +433,100 @@ class TestMockObjTierShutdown:
tier.shutdown() # must not raise
class TestObjTierKVEvents:
def setup_method(self):
self.tier, self.agent = _make_tier(
offloading_spec=_make_events_spec(enable_kv_cache_events=True),
enable_kv_events=True,
)
def test_successful_store_emits_stored_event(self):
"""A completed store transfer emits one stored event with the job's keys."""
keys = [key(1), key(2)]
self.tier.submit_store(make_job(1, keys, [0, 1]))
assert all(r.success for r in drain(self.tier))
events = list(self.tier.take_events())
assert len(events) == 1
assert events[0].keys == keys
# Literal medium pins the wire contract, not just the constant choice.
assert events[0].medium == "OBJ"
assert not events[0].removed
# take_events drains the buffer.
assert list(self.tier.take_events()) == []
def test_mixed_job_results_emit_event_only_for_successful_job(self):
"""With a failed and a successful store job resolving in the same
poll, exactly one event is emitted and its keys belong to the
successful job."""
original = self.agent.check_xfer_state
self.agent.check_xfer_state = lambda h: "ERR" if h._id == 0 else original(h)
self.tier.submit_store(make_job(1, [key(1)], [0])) # handle 0: fails
self.tier.submit_store(make_job(2, [key(2)], [1])) # handle 1: succeeds
results = drain(self.tier)
by_id = {r.job_id: r for r in results}
assert not by_id[1].success
assert by_id[2].success
events = list(self.tier.take_events())
assert len(events) == 1
assert events[0].keys == [key(2)]
assert self.tier._store_job_keys == {}
def test_load_job_emits_no_event(self):
self.tier.submit_store(make_job(1, [key(1)], [0]))
results = drain(self.tier)
assert len(results) == 1
assert results[0].success
list(self.tier.take_events())
self.tier.submit_load(make_job(2, [key(1)], [0]))
results = drain(self.tier)
assert len(results) == 1
assert results[0].success
assert list(self.tier.take_events()) == []
def test_failed_transfer_emits_no_event(self):
self.agent.check_xfer_state = lambda h: "ERR"
self.tier.submit_store(make_job(1, [key(1)], [0]))
results = drain(self.tier)
assert not results[0].success
assert list(self.tier.take_events()) == []
assert self.tier._store_job_keys == {}
def test_submission_failure_emits_no_event(self):
self.agent.make_prepped_xfer = lambda *a, **k: None
self.tier.submit_store(make_job(1, [key(1)], [0]))
results = list(self.tier.get_finished_jobs())
assert not results[0].success
assert list(self.tier.take_events()) == []
assert self.tier._store_job_keys == {}
def test_events_disabled_by_default(self):
tier, _ = _make_tier()
tier.submit_store(make_job(1, [key(1)], [0]))
results = drain(tier)
assert len(results) == 1
assert results[0].success
assert tier.events is None
assert tier._store_job_keys == {}
assert list(tier.take_events()) == []
def test_events_require_global_kv_events_flag(self):
"""Tier-level opt-in alone is not enough; the global flag gates events."""
tier, _ = _make_tier(
offloading_spec=_make_events_spec(enable_kv_cache_events=False),
enable_kv_events=True,
)
tier.submit_store(make_job(1, [key(1)], [0]))
results = drain(tier)
assert len(results) == 1
assert results[0].success
assert tier.events is None
assert tier._store_job_keys == {}
assert list(tier.take_events()) == []
class TestObjStoreConfig:
def test_explicit_credentials_included(self):
cfg = ObjStoreConfig(