[KV Offload] Emit self-describing events for partial recurrent blocks (#51243)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -208,6 +208,58 @@ def test_take_events_forwards_locality_to_placeholder_store():
|
||||
assert events[0].locality == "REMOTE"
|
||||
|
||||
|
||||
def test_partial_tail_event_describes_hash_aligned_physical_block_prefix():
|
||||
tracker = _tracker()
|
||||
group_config = _group_config(block_size=16, blocks_per_chunk=1)._replace(
|
||||
hashes_per_chunk=4
|
||||
)
|
||||
req = _request(block_hashes=[_hash(i) for i in range(8)], token_count=32)
|
||||
key = make_offload_key(req.block_hashes[6], group_config.group_idx)
|
||||
|
||||
tracker.record_partial_store(req, group_config, 28, key)
|
||||
[event] = tracker.take_events([_stored_event([key])])
|
||||
|
||||
assert isinstance(event, BlockStored)
|
||||
assert event.block_hashes == [_wire_hash(_hash(i)) for i in range(4, 7)]
|
||||
assert event.parent_block_hash == _wire_hash(_hash(3))
|
||||
assert event.token_ids == list(range(17, 29))
|
||||
assert event.block_size == 4
|
||||
|
||||
|
||||
def test_partial_tail_lookup_does_not_overwrite_store_metadata():
|
||||
tracker = _tracker()
|
||||
group_config = _group_config()
|
||||
stored_req = _request(block_hashes=[_hash(0)], token_count=4)
|
||||
lookup_req = _request(block_hashes=[_hash(0)], token_count=4)
|
||||
lookup_req.all_token_ids = [9, 9, 9, 9]
|
||||
key = make_offload_key(stored_req.block_hashes[0], group_config.group_idx)
|
||||
|
||||
tracker.record_partial_store(stored_req, group_config, 4, key)
|
||||
tracker.record_partial_lookup(lookup_req, group_config, 4, key)
|
||||
[event] = tracker.take_events([_stored_event([key])])
|
||||
|
||||
assert isinstance(event, BlockStored)
|
||||
assert event.token_ids == [1, 2, 3, 4]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"record_method", ["record_partial_store", "record_partial_lookup"]
|
||||
)
|
||||
def test_partial_tail_sliding_window_event_uses_placeholder(record_method):
|
||||
tracker = _tracker()
|
||||
group_config = _group_config(sliding_window_size_in_chunks=1)
|
||||
req = _request(block_hashes=[_hash(0)], token_count=4)
|
||||
key = make_offload_key(req.block_hashes[0], group_config.group_idx)
|
||||
|
||||
getattr(tracker, record_method)(req, group_config, 4, key)
|
||||
[event] = tracker.take_events([_stored_event([key])])
|
||||
|
||||
assert isinstance(event, BlockStored)
|
||||
assert event.block_hashes == [_wire_hash(_hash(0))]
|
||||
assert event.token_ids == []
|
||||
assert event.block_size == 0
|
||||
|
||||
|
||||
def test_take_events_forwards_locality_to_remove():
|
||||
tracker = _tracker()
|
||||
req = _request(block_hashes=[_hash(0)], token_count=4)
|
||||
|
||||
@@ -16,6 +16,7 @@ from tests.v1.kv_connector.unit.offloading_connector.utils import (
|
||||
to_keys,
|
||||
)
|
||||
from tests.v1.kv_connector.unit.utils import EOS_TOKEN_ID
|
||||
from vllm.config import KVEventsConfig
|
||||
from vllm.distributed.kv_events import MEDIUM_CPU, BlockRemoved, BlockStored
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
|
||||
OffloadingConnectorMetadata,
|
||||
@@ -60,9 +61,12 @@ from vllm.v1.request import RequestStatus
|
||||
|
||||
|
||||
def _make_partial_tail_scheduler() -> OffloadingConnectorScheduler:
|
||||
vllm_config = _make_vllm_config()
|
||||
vllm_config = _make_vllm_config(extra_config={"self_describing_kv_events": True})
|
||||
vllm_config.cache_config.prefix_match_unit = 4
|
||||
vllm_config.speculative_config = None
|
||||
vllm_config.kv_events_config = KVEventsConfig(
|
||||
enable_kv_cache_events=True, publisher="null"
|
||||
)
|
||||
kv_cache_config = _make_mamba_hybrid_kv_cache_config()
|
||||
spec = MockOffloadingSpec(build_offloading_config(vllm_config, kv_cache_config))
|
||||
return OffloadingConnectorScheduler(spec, vllm_config, kv_cache_config)
|
||||
@@ -77,6 +81,8 @@ def _make_partial_tail_request(
|
||||
request.num_prompt_tokens = 30
|
||||
request.num_tokens = 30
|
||||
request.block_hashes = [BlockHash(f"h{i}".encode()) for i in range(7)]
|
||||
request.all_token_ids = list(range(30))
|
||||
request.lora_request = None
|
||||
request.is_finished.return_value = False
|
||||
scheduler.on_new_request(request)
|
||||
return request
|
||||
@@ -119,6 +125,34 @@ def test_partial_tail_store_uses_attention_and_recurrent_cow_sources():
|
||||
12: {job_id},
|
||||
99: {job_id},
|
||||
}
|
||||
assert scheduler.config.supports_partial_tail
|
||||
|
||||
events = list(
|
||||
scheduler._events_tracker.take_events(
|
||||
[
|
||||
OffloadingEvent(
|
||||
keys=list(scheduler._jobs[job_id].keys),
|
||||
medium=Medium.CPU,
|
||||
removed=False,
|
||||
)
|
||||
]
|
||||
)
|
||||
)
|
||||
assert len(events) == 2
|
||||
assert all(isinstance(event, BlockStored) for event in events)
|
||||
assert {event.group_idx for event in events} == {0, 1}
|
||||
events_by_group = {event.group_idx: event for event in events}
|
||||
full_attention_event = events_by_group[0]
|
||||
assert full_attention_event.block_size == 4
|
||||
assert full_attention_event.token_ids == list(range(16, 28))
|
||||
assert len(full_attention_event.block_hashes) == 3
|
||||
assert full_attention_event.parent_block_hash is not None
|
||||
|
||||
recurrent_event = events_by_group[1]
|
||||
assert recurrent_event.block_size == 0
|
||||
assert recurrent_event.token_ids == []
|
||||
assert len(recurrent_event.block_hashes) == 1
|
||||
assert recurrent_event.parent_block_hash is None
|
||||
|
||||
|
||||
def test_partial_lookup_returns_exact_boundary_and_group_load_keys():
|
||||
|
||||
Reference in New Issue
Block a user