[KV Offload] Emit self-describing events for partial recurrent blocks (#51243)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Chauncey
2026-08-09 11:28:44 +03:00
committed by GitHub
co-authored by Or Ozeri
parent 7f6432cc0d
commit c423998642
5 changed files with 181 additions and 6 deletions
@@ -208,6 +208,58 @@ def test_take_events_forwards_locality_to_placeholder_store():
assert events[0].locality == "REMOTE"
def test_partial_tail_event_describes_hash_aligned_physical_block_prefix():
tracker = _tracker()
group_config = _group_config(block_size=16, blocks_per_chunk=1)._replace(
hashes_per_chunk=4
)
req = _request(block_hashes=[_hash(i) for i in range(8)], token_count=32)
key = make_offload_key(req.block_hashes[6], group_config.group_idx)
tracker.record_partial_store(req, group_config, 28, key)
[event] = tracker.take_events([_stored_event([key])])
assert isinstance(event, BlockStored)
assert event.block_hashes == [_wire_hash(_hash(i)) for i in range(4, 7)]
assert event.parent_block_hash == _wire_hash(_hash(3))
assert event.token_ids == list(range(17, 29))
assert event.block_size == 4
def test_partial_tail_lookup_does_not_overwrite_store_metadata():
tracker = _tracker()
group_config = _group_config()
stored_req = _request(block_hashes=[_hash(0)], token_count=4)
lookup_req = _request(block_hashes=[_hash(0)], token_count=4)
lookup_req.all_token_ids = [9, 9, 9, 9]
key = make_offload_key(stored_req.block_hashes[0], group_config.group_idx)
tracker.record_partial_store(stored_req, group_config, 4, key)
tracker.record_partial_lookup(lookup_req, group_config, 4, key)
[event] = tracker.take_events([_stored_event([key])])
assert isinstance(event, BlockStored)
assert event.token_ids == [1, 2, 3, 4]
@pytest.mark.parametrize(
"record_method", ["record_partial_store", "record_partial_lookup"]
)
def test_partial_tail_sliding_window_event_uses_placeholder(record_method):
tracker = _tracker()
group_config = _group_config(sliding_window_size_in_chunks=1)
req = _request(block_hashes=[_hash(0)], token_count=4)
key = make_offload_key(req.block_hashes[0], group_config.group_idx)
getattr(tracker, record_method)(req, group_config, 4, key)
[event] = tracker.take_events([_stored_event([key])])
assert isinstance(event, BlockStored)
assert event.block_hashes == [_wire_hash(_hash(0))]
assert event.token_ids == []
assert event.block_size == 0
def test_take_events_forwards_locality_to_remove():
tracker = _tracker()
req = _request(block_hashes=[_hash(0)], token_count=4)
@@ -16,6 +16,7 @@ from tests.v1.kv_connector.unit.offloading_connector.utils import (
to_keys,
)
from tests.v1.kv_connector.unit.utils import EOS_TOKEN_ID
from vllm.config import KVEventsConfig
from vllm.distributed.kv_events import MEDIUM_CPU, BlockRemoved, BlockStored
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
OffloadingConnectorMetadata,
@@ -60,9 +61,12 @@ from vllm.v1.request import RequestStatus
def _make_partial_tail_scheduler() -> OffloadingConnectorScheduler:
vllm_config = _make_vllm_config()
vllm_config = _make_vllm_config(extra_config={"self_describing_kv_events": True})
vllm_config.cache_config.prefix_match_unit = 4
vllm_config.speculative_config = None
vllm_config.kv_events_config = KVEventsConfig(
enable_kv_cache_events=True, publisher="null"
)
kv_cache_config = _make_mamba_hybrid_kv_cache_config()
spec = MockOffloadingSpec(build_offloading_config(vllm_config, kv_cache_config))
return OffloadingConnectorScheduler(spec, vllm_config, kv_cache_config)
@@ -77,6 +81,8 @@ def _make_partial_tail_request(
request.num_prompt_tokens = 30
request.num_tokens = 30
request.block_hashes = [BlockHash(f"h{i}".encode()) for i in range(7)]
request.all_token_ids = list(range(30))
request.lora_request = None
request.is_finished.return_value = False
scheduler.on_new_request(request)
return request
@@ -119,6 +125,34 @@ def test_partial_tail_store_uses_attention_and_recurrent_cow_sources():
12: {job_id},
99: {job_id},
}
assert scheduler.config.supports_partial_tail
events = list(
scheduler._events_tracker.take_events(
[
OffloadingEvent(
keys=list(scheduler._jobs[job_id].keys),
medium=Medium.CPU,
removed=False,
)
]
)
)
assert len(events) == 2
assert all(isinstance(event, BlockStored) for event in events)
assert {event.group_idx for event in events} == {0, 1}
events_by_group = {event.group_idx: event for event in events}
full_attention_event = events_by_group[0]
assert full_attention_event.block_size == 4
assert full_attention_event.token_ids == list(range(16, 28))
assert len(full_attention_event.block_hashes) == 3
assert full_attention_event.parent_block_hash is not None
recurrent_event = events_by_group[1]
assert recurrent_event.block_size == 0
assert recurrent_event.token_ids == []
assert len(recurrent_event.block_hashes) == 1
assert recurrent_event.parent_block_hash is None
def test_partial_lookup_returns_exact_boundary_and_group_load_keys():