[kv_offload] Emit tier-owned BlockStored events from FS/OBJ secondary tiers (#47923)
Signed-off-by: Change72 <changg@nvidia.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
Claude Fable 5
Or Ozeri
parent
88e5e2c57b
commit
2d814a0082
@@ -18,8 +18,10 @@ import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.distributed.kv_events import MEDIUM_FS
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LookupResult,
|
||||
OffloadingEvent,
|
||||
OffloadKey,
|
||||
ReqContext,
|
||||
ScheduleEndContext,
|
||||
@@ -58,6 +60,16 @@ _MOCK_OFFLOADING_SPEC.kv_cache_config = _MOCK_KV_CACHE_CONFIG
|
||||
_MOCK_OFFLOADING_SPEC.block_size_factor = 1
|
||||
|
||||
|
||||
def _make_offloading_spec(enable_kv_cache_events: bool) -> MagicMock:
|
||||
"""Mock spec with an explicit global KV events flag."""
|
||||
spec = MagicMock()
|
||||
spec.vllm_config = _MOCK_VLLM_CONFIG
|
||||
spec.kv_cache_config = _MOCK_KV_CACHE_CONFIG
|
||||
spec.block_size_factor = 1
|
||||
spec.kv_events_config.enable_kv_cache_events = enable_kv_cache_events
|
||||
return spec
|
||||
|
||||
|
||||
def key(n: int) -> OffloadKey:
|
||||
return make_offload_key(n.to_bytes(8, "big"), 0)
|
||||
|
||||
@@ -164,6 +176,23 @@ def fs_tier(tmp_path):
|
||||
tier.shutdown()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fs_tier_with_events(tmp_path):
|
||||
tensor = _page_aligned_zero_tensor(4, _BLOCK_ELEMENTS)
|
||||
mock_view = memoryview(tensor.numpy())
|
||||
tier = FileSystemTierManager(
|
||||
offloading_spec=_make_offloading_spec(enable_kv_cache_events=True),
|
||||
primary_kv_view=mock_view,
|
||||
tier_type="fs",
|
||||
root_dir=str(tmp_path),
|
||||
n_read_threads=4,
|
||||
n_write_threads=4,
|
||||
enable_kv_events=True,
|
||||
)
|
||||
yield tier
|
||||
tier.shutdown()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -396,3 +425,170 @@ def test_batch_lookup_dispatch(fs_tier, monkeypatch, use_c_ext):
|
||||
|
||||
results = lookup_and_wait(tier, [key(1), key(2)])
|
||||
assert results == [LookupResult.HIT, LookupResult.MISS]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# KV events
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_successful_store_emits_stored_event(fs_tier_with_events):
|
||||
"""A completed store job emits one stored event with the job's keys."""
|
||||
tier = fs_tier_with_events
|
||||
keys = [key(1), key(2)]
|
||||
tier.submit_store(make_job(1, keys, [0, 1]))
|
||||
assert all(r.success for r in drain(tier))
|
||||
|
||||
events = list(tier.take_events())
|
||||
assert len(events) == 1
|
||||
assert events[0].keys == keys
|
||||
# Literal medium pins the wire contract, not just the constant choice.
|
||||
assert events[0].medium == "FS"
|
||||
assert not events[0].removed
|
||||
# take_events drains the buffer.
|
||||
assert list(tier.take_events()) == []
|
||||
|
||||
|
||||
def test_load_job_emits_no_event(fs_tier_with_events):
|
||||
tier = fs_tier_with_events
|
||||
tier.submit_store(make_job(1, [key(1)], [0]))
|
||||
results = drain(tier)
|
||||
assert len(results) == 1
|
||||
assert results[0].success
|
||||
list(tier.take_events())
|
||||
|
||||
tier.submit_load(make_job(2, [key(1)], [1], is_promotion=True))
|
||||
results = drain(tier)
|
||||
assert len(results) == 1
|
||||
assert results[0].success
|
||||
assert list(tier.take_events()) == []
|
||||
|
||||
|
||||
def test_mixed_job_results_emit_event_only_for_successful_job(
|
||||
fs_tier_with_events, monkeypatch
|
||||
):
|
||||
"""With a failed and a successful store job in flight, exactly one event
|
||||
is emitted and its keys belong to the successful job."""
|
||||
import vllm.v1.kv_offload.tiering.fs.manager as mgr_mod
|
||||
|
||||
tier = fs_tier_with_events
|
||||
failing_path = tier.file_mapper.get_file_name(key(1))
|
||||
original_store_block = mgr_mod.store_block
|
||||
|
||||
def flaky_store_block(dest_path, *args, **kwargs):
|
||||
if dest_path == failing_path:
|
||||
raise OSError("injected store failure")
|
||||
return original_store_block(dest_path, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(mgr_mod, "store_block", flaky_store_block)
|
||||
|
||||
tier.submit_store(make_job(1, [key(1)], [0]))
|
||||
tier.submit_store(make_job(2, [key(2)], [1]))
|
||||
results = drain(tier)
|
||||
assert len(results) == 2
|
||||
by_id = {r.job_id: r for r in results}
|
||||
assert not by_id[1].success
|
||||
assert by_id[2].success
|
||||
|
||||
events = list(tier.take_events())
|
||||
assert len(events) == 1
|
||||
assert events[0].keys == [key(2)]
|
||||
|
||||
|
||||
def test_partially_failed_store_emits_no_event(fs_tier_with_events, monkeypatch):
|
||||
"""A store job with any failed block emits no event for the whole job."""
|
||||
import vllm.v1.kv_offload.tiering.fs.manager as mgr_mod
|
||||
|
||||
tier = fs_tier_with_events
|
||||
failing_path = tier.file_mapper.get_file_name(key(2))
|
||||
original_store_block = mgr_mod.store_block
|
||||
|
||||
def flaky_store_block(dest_path, *args, **kwargs):
|
||||
if dest_path == failing_path:
|
||||
raise OSError("injected store failure")
|
||||
return original_store_block(dest_path, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(mgr_mod, "store_block", flaky_store_block)
|
||||
|
||||
tier.submit_store(make_job(1, [key(1), key(2)], [0, 1]))
|
||||
results = drain(tier)
|
||||
assert len(results) == 1
|
||||
assert not results[0].success
|
||||
assert list(tier.take_events()) == []
|
||||
assert tier._store_job_keys == {}
|
||||
|
||||
|
||||
def test_events_disabled_by_default(fs_tier):
|
||||
tier, _ = fs_tier
|
||||
tier.submit_store(make_job(1, [key(1)], [0]))
|
||||
results = drain(tier)
|
||||
assert len(results) == 1
|
||||
assert results[0].success
|
||||
assert tier.events is None
|
||||
assert tier._store_job_keys == {}
|
||||
assert list(tier.take_events()) == []
|
||||
|
||||
|
||||
def test_events_require_global_kv_events_flag(tmp_path):
|
||||
"""Tier-level opt-in alone is not enough; the global flag gates events."""
|
||||
tensor = _page_aligned_zero_tensor(4, _BLOCK_ELEMENTS)
|
||||
tier = FileSystemTierManager(
|
||||
offloading_spec=_make_offloading_spec(enable_kv_cache_events=False),
|
||||
primary_kv_view=memoryview(tensor.numpy()),
|
||||
tier_type="fs",
|
||||
root_dir=str(tmp_path),
|
||||
enable_kv_events=True,
|
||||
)
|
||||
try:
|
||||
assert tier.events is None
|
||||
tier.submit_store(make_job(1, [key(1)], [0]))
|
||||
results = drain(tier)
|
||||
assert len(results) == 1
|
||||
assert results[0].success
|
||||
assert list(tier.take_events()) == []
|
||||
assert tier._store_job_keys == {}
|
||||
finally:
|
||||
tier.shutdown()
|
||||
|
||||
|
||||
def test_cascade_store_emits_fs_event_through_tiering_manager(tmp_path):
|
||||
"""A GPU->CPU->fs cascade surfaces the tier-owned FS stored event via the
|
||||
TieringOffloadingManager's aggregated take_events()."""
|
||||
from vllm.v1.kv_offload.tiering.manager import (
|
||||
CPUPrimaryTierOffloadingManager,
|
||||
TieringOffloadingManager,
|
||||
)
|
||||
|
||||
tensor = _page_aligned_zero_tensor(4, _BLOCK_ELEMENTS)
|
||||
view = memoryview(tensor.numpy())
|
||||
mock_region = MagicMock()
|
||||
mock_region.create_kv_memoryview.return_value = view
|
||||
primary = CPUPrimaryTierOffloadingManager(num_blocks=4, mmap_region=mock_region)
|
||||
tier = FileSystemTierManager(
|
||||
offloading_spec=_make_offloading_spec(enable_kv_cache_events=True),
|
||||
primary_kv_view=primary.get_kv_memoryview(),
|
||||
tier_type="fs",
|
||||
root_dir=str(tmp_path),
|
||||
enable_kv_events=True,
|
||||
)
|
||||
manager = TieringOffloadingManager(primary_tier=primary, secondary_tiers=[tier])
|
||||
try:
|
||||
keys = [key(1), key(2)]
|
||||
manager.on_new_request(_CTX)
|
||||
assert manager.prepare_store(keys, _CTX) is not None
|
||||
manager.complete_store(keys, _CTX) # cascades to the fs tier
|
||||
|
||||
events: list[OffloadingEvent] = []
|
||||
ctx = ScheduleEndContext(new_req_ids=[], preempted_req_ids=())
|
||||
deadline = time.monotonic() + 5.0
|
||||
while time.monotonic() < deadline and not events:
|
||||
manager.on_schedule_end(ctx)
|
||||
events.extend(manager.take_events())
|
||||
time.sleep(0.01)
|
||||
|
||||
fs_events = [e for e in events if e.medium == MEDIUM_FS]
|
||||
assert len(fs_events) == 1
|
||||
assert set(fs_events[0].keys) == set(keys)
|
||||
assert not fs_events[0].removed
|
||||
finally:
|
||||
tier.shutdown()
|
||||
|
||||
@@ -179,8 +179,19 @@ class MockNixlAgent:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_events_spec(enable_kv_cache_events: bool) -> SimpleNamespace:
|
||||
"""Offloading spec stub with an explicit global KV events flag."""
|
||||
return SimpleNamespace(
|
||||
vllm_config=_make_vllm_config(),
|
||||
kv_cache_config=SimpleNamespace(kv_cache_groups=[]),
|
||||
kv_events_config=SimpleNamespace(enable_kv_cache_events=enable_kv_cache_events),
|
||||
)
|
||||
|
||||
|
||||
def _make_tier(
|
||||
num_blocks: int = 4,
|
||||
offloading_spec: SimpleNamespace = _OFFLOADING_SPEC,
|
||||
**tier_kwargs,
|
||||
) -> tuple[ObjectStoreSecondaryTierManager, MockNixlAgent]:
|
||||
"""Create a tier backed by a fresh MockNixlAgent."""
|
||||
mock_agent = MockNixlAgent()
|
||||
@@ -194,11 +205,12 @@ def _make_tier(
|
||||
),
|
||||
):
|
||||
tier = ObjectStoreSecondaryTierManager(
|
||||
offloading_spec=_OFFLOADING_SPEC,
|
||||
offloading_spec=offloading_spec,
|
||||
primary_kv_view=view,
|
||||
tier_type="obj",
|
||||
store_config=_STORE_CONFIG,
|
||||
prefix=_RUN_PREFIX,
|
||||
**tier_kwargs,
|
||||
)
|
||||
return tier, mock_agent
|
||||
|
||||
@@ -421,6 +433,100 @@ class TestMockObjTierShutdown:
|
||||
tier.shutdown() # must not raise
|
||||
|
||||
|
||||
class TestObjTierKVEvents:
|
||||
def setup_method(self):
|
||||
self.tier, self.agent = _make_tier(
|
||||
offloading_spec=_make_events_spec(enable_kv_cache_events=True),
|
||||
enable_kv_events=True,
|
||||
)
|
||||
|
||||
def test_successful_store_emits_stored_event(self):
|
||||
"""A completed store transfer emits one stored event with the job's keys."""
|
||||
keys = [key(1), key(2)]
|
||||
self.tier.submit_store(make_job(1, keys, [0, 1]))
|
||||
assert all(r.success for r in drain(self.tier))
|
||||
|
||||
events = list(self.tier.take_events())
|
||||
assert len(events) == 1
|
||||
assert events[0].keys == keys
|
||||
# Literal medium pins the wire contract, not just the constant choice.
|
||||
assert events[0].medium == "OBJ"
|
||||
assert not events[0].removed
|
||||
# take_events drains the buffer.
|
||||
assert list(self.tier.take_events()) == []
|
||||
|
||||
def test_mixed_job_results_emit_event_only_for_successful_job(self):
|
||||
"""With a failed and a successful store job resolving in the same
|
||||
poll, exactly one event is emitted and its keys belong to the
|
||||
successful job."""
|
||||
original = self.agent.check_xfer_state
|
||||
self.agent.check_xfer_state = lambda h: "ERR" if h._id == 0 else original(h)
|
||||
self.tier.submit_store(make_job(1, [key(1)], [0])) # handle 0: fails
|
||||
self.tier.submit_store(make_job(2, [key(2)], [1])) # handle 1: succeeds
|
||||
results = drain(self.tier)
|
||||
by_id = {r.job_id: r for r in results}
|
||||
assert not by_id[1].success
|
||||
assert by_id[2].success
|
||||
|
||||
events = list(self.tier.take_events())
|
||||
assert len(events) == 1
|
||||
assert events[0].keys == [key(2)]
|
||||
assert self.tier._store_job_keys == {}
|
||||
|
||||
def test_load_job_emits_no_event(self):
|
||||
self.tier.submit_store(make_job(1, [key(1)], [0]))
|
||||
results = drain(self.tier)
|
||||
assert len(results) == 1
|
||||
assert results[0].success
|
||||
list(self.tier.take_events())
|
||||
|
||||
self.tier.submit_load(make_job(2, [key(1)], [0]))
|
||||
results = drain(self.tier)
|
||||
assert len(results) == 1
|
||||
assert results[0].success
|
||||
assert list(self.tier.take_events()) == []
|
||||
|
||||
def test_failed_transfer_emits_no_event(self):
|
||||
self.agent.check_xfer_state = lambda h: "ERR"
|
||||
self.tier.submit_store(make_job(1, [key(1)], [0]))
|
||||
results = drain(self.tier)
|
||||
assert not results[0].success
|
||||
assert list(self.tier.take_events()) == []
|
||||
assert self.tier._store_job_keys == {}
|
||||
|
||||
def test_submission_failure_emits_no_event(self):
|
||||
self.agent.make_prepped_xfer = lambda *a, **k: None
|
||||
self.tier.submit_store(make_job(1, [key(1)], [0]))
|
||||
results = list(self.tier.get_finished_jobs())
|
||||
assert not results[0].success
|
||||
assert list(self.tier.take_events()) == []
|
||||
assert self.tier._store_job_keys == {}
|
||||
|
||||
def test_events_disabled_by_default(self):
|
||||
tier, _ = _make_tier()
|
||||
tier.submit_store(make_job(1, [key(1)], [0]))
|
||||
results = drain(tier)
|
||||
assert len(results) == 1
|
||||
assert results[0].success
|
||||
assert tier.events is None
|
||||
assert tier._store_job_keys == {}
|
||||
assert list(tier.take_events()) == []
|
||||
|
||||
def test_events_require_global_kv_events_flag(self):
|
||||
"""Tier-level opt-in alone is not enough; the global flag gates events."""
|
||||
tier, _ = _make_tier(
|
||||
offloading_spec=_make_events_spec(enable_kv_cache_events=False),
|
||||
enable_kv_events=True,
|
||||
)
|
||||
tier.submit_store(make_job(1, [key(1)], [0]))
|
||||
results = drain(tier)
|
||||
assert len(results) == 1
|
||||
assert results[0].success
|
||||
assert tier.events is None
|
||||
assert tier._store_job_keys == {}
|
||||
assert list(tier.take_events()) == []
|
||||
|
||||
|
||||
class TestObjStoreConfig:
|
||||
def test_explicit_credentials_included(self):
|
||||
cfg = ObjStoreConfig(
|
||||
|
||||
Reference in New Issue
Block a user