[KV Offload] Emit self-describing events for partial recurrent blocks (#51243)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -76,7 +76,7 @@ vllm serve <model> \
|
||||
| `max_tracker_size` | no | `64000` | single-tier | Max entries in the lookup tracker. |
|
||||
| `secondary_tiers` | no | `[]` | multi-tier | List of secondary tier configs (see below). |
|
||||
| `offload_prompt_only` | no | `true` | both | If `true`, only prompt (prefill) blocks are offloaded; decode blocks are skipped. |
|
||||
| `self_describing_kv_events` | no | `false` | both | Opt-in. When `true` *and* KV cache events are enabled (`--kv-events-config` with `enable_kv_cache_events`), the connector emits self-describing block-granular `BlockStored`/`BlockRemoved` payloads (constituent block hashes, whole-chunk `token_ids`, per-block `block_size`, parent hash, LoRA + group/cache-spec metadata) instead of the placeholder fallback, so external KV-event consumers can index offloaded blocks. Inert unless events are enabled. With `TieringOffloadingSpec`, a CPU promotion is self-describing when a local request observes its primary-tier `HIT` before event translation; otherwise its stored event may retain the placeholder, while a later `HIT` can backfill metadata for removal. Pending-removal/re-promotion races and externally initiated promotions may also produce placeholders, and consumers must ignore removals for unknown hashes. Full-attention groups only; sliding-window/SSM groups keep the placeholder fallback. In chunk mode (`block_size` > GPU block size, or `blocks_per_chunk` > 1), overlapping chunks re-announce shared per-block hashes, so consumers must reference-count (deduplicate) repeated store/remove announcements. |
|
||||
| `self_describing_kv_events` | no | `false` | both | Opt-in. When `true` *and* KV cache events are enabled (`--kv-events-config` with `enable_kv_cache_events`), the connector emits self-describing block-granular `BlockStored`/`BlockRemoved` payloads (constituent block hashes, whole-chunk `token_ids`, per-block `block_size`, parent hash, LoRA + group/cache-spec metadata) instead of the placeholder fallback, so external KV-event consumers can index offloaded blocks. Inert unless events are enabled. With `TieringOffloadingSpec`, a CPU promotion is self-describing when a local request observes its primary-tier `HIT` before event translation; otherwise its stored event may retain the placeholder, while a later `HIT` can backfill metadata for removal. Pending-removal/re-promotion races and externally initiated promotions may also produce placeholders, and consumers must ignore removals for unknown hashes. Partial recurrent tails emit the hash-aligned portion from the physical block start through the tail boundary. Other sliding-window/SSM chunks keep the placeholder fallback. In chunk mode (`block_size` > GPU block size, or `blocks_per_chunk` > 1), overlapping chunks re-announce shared per-block hashes, so consumers must reference-count (deduplicate) repeated store/remove announcements. |
|
||||
| `spec_module_path` | no | — | both | Python import path for a custom `OffloadingSpec` not in the built-in registry. Required only when `spec_name` is not built-in (advanced). |
|
||||
|
||||
## Custom Eviction Policies
|
||||
|
||||
@@ -208,6 +208,58 @@ def test_take_events_forwards_locality_to_placeholder_store():
|
||||
assert events[0].locality == "REMOTE"
|
||||
|
||||
|
||||
def test_partial_tail_event_describes_hash_aligned_physical_block_prefix():
|
||||
tracker = _tracker()
|
||||
group_config = _group_config(block_size=16, blocks_per_chunk=1)._replace(
|
||||
hashes_per_chunk=4
|
||||
)
|
||||
req = _request(block_hashes=[_hash(i) for i in range(8)], token_count=32)
|
||||
key = make_offload_key(req.block_hashes[6], group_config.group_idx)
|
||||
|
||||
tracker.record_partial_store(req, group_config, 28, key)
|
||||
[event] = tracker.take_events([_stored_event([key])])
|
||||
|
||||
assert isinstance(event, BlockStored)
|
||||
assert event.block_hashes == [_wire_hash(_hash(i)) for i in range(4, 7)]
|
||||
assert event.parent_block_hash == _wire_hash(_hash(3))
|
||||
assert event.token_ids == list(range(17, 29))
|
||||
assert event.block_size == 4
|
||||
|
||||
|
||||
def test_partial_tail_lookup_does_not_overwrite_store_metadata():
|
||||
tracker = _tracker()
|
||||
group_config = _group_config()
|
||||
stored_req = _request(block_hashes=[_hash(0)], token_count=4)
|
||||
lookup_req = _request(block_hashes=[_hash(0)], token_count=4)
|
||||
lookup_req.all_token_ids = [9, 9, 9, 9]
|
||||
key = make_offload_key(stored_req.block_hashes[0], group_config.group_idx)
|
||||
|
||||
tracker.record_partial_store(stored_req, group_config, 4, key)
|
||||
tracker.record_partial_lookup(lookup_req, group_config, 4, key)
|
||||
[event] = tracker.take_events([_stored_event([key])])
|
||||
|
||||
assert isinstance(event, BlockStored)
|
||||
assert event.token_ids == [1, 2, 3, 4]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"record_method", ["record_partial_store", "record_partial_lookup"]
|
||||
)
|
||||
def test_partial_tail_sliding_window_event_uses_placeholder(record_method):
|
||||
tracker = _tracker()
|
||||
group_config = _group_config(sliding_window_size_in_chunks=1)
|
||||
req = _request(block_hashes=[_hash(0)], token_count=4)
|
||||
key = make_offload_key(req.block_hashes[0], group_config.group_idx)
|
||||
|
||||
getattr(tracker, record_method)(req, group_config, 4, key)
|
||||
[event] = tracker.take_events([_stored_event([key])])
|
||||
|
||||
assert isinstance(event, BlockStored)
|
||||
assert event.block_hashes == [_wire_hash(_hash(0))]
|
||||
assert event.token_ids == []
|
||||
assert event.block_size == 0
|
||||
|
||||
|
||||
def test_take_events_forwards_locality_to_remove():
|
||||
tracker = _tracker()
|
||||
req = _request(block_hashes=[_hash(0)], token_count=4)
|
||||
|
||||
@@ -16,6 +16,7 @@ from tests.v1.kv_connector.unit.offloading_connector.utils import (
|
||||
to_keys,
|
||||
)
|
||||
from tests.v1.kv_connector.unit.utils import EOS_TOKEN_ID
|
||||
from vllm.config import KVEventsConfig
|
||||
from vllm.distributed.kv_events import MEDIUM_CPU, BlockRemoved, BlockStored
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
|
||||
OffloadingConnectorMetadata,
|
||||
@@ -60,9 +61,12 @@ from vllm.v1.request import RequestStatus
|
||||
|
||||
|
||||
def _make_partial_tail_scheduler() -> OffloadingConnectorScheduler:
|
||||
vllm_config = _make_vllm_config()
|
||||
vllm_config = _make_vllm_config(extra_config={"self_describing_kv_events": True})
|
||||
vllm_config.cache_config.prefix_match_unit = 4
|
||||
vllm_config.speculative_config = None
|
||||
vllm_config.kv_events_config = KVEventsConfig(
|
||||
enable_kv_cache_events=True, publisher="null"
|
||||
)
|
||||
kv_cache_config = _make_mamba_hybrid_kv_cache_config()
|
||||
spec = MockOffloadingSpec(build_offloading_config(vllm_config, kv_cache_config))
|
||||
return OffloadingConnectorScheduler(spec, vllm_config, kv_cache_config)
|
||||
@@ -77,6 +81,8 @@ def _make_partial_tail_request(
|
||||
request.num_prompt_tokens = 30
|
||||
request.num_tokens = 30
|
||||
request.block_hashes = [BlockHash(f"h{i}".encode()) for i in range(7)]
|
||||
request.all_token_ids = list(range(30))
|
||||
request.lora_request = None
|
||||
request.is_finished.return_value = False
|
||||
scheduler.on_new_request(request)
|
||||
return request
|
||||
@@ -119,6 +125,34 @@ def test_partial_tail_store_uses_attention_and_recurrent_cow_sources():
|
||||
12: {job_id},
|
||||
99: {job_id},
|
||||
}
|
||||
assert scheduler.config.supports_partial_tail
|
||||
|
||||
events = list(
|
||||
scheduler._events_tracker.take_events(
|
||||
[
|
||||
OffloadingEvent(
|
||||
keys=list(scheduler._jobs[job_id].keys),
|
||||
medium=Medium.CPU,
|
||||
removed=False,
|
||||
)
|
||||
]
|
||||
)
|
||||
)
|
||||
assert len(events) == 2
|
||||
assert all(isinstance(event, BlockStored) for event in events)
|
||||
assert {event.group_idx for event in events} == {0, 1}
|
||||
events_by_group = {event.group_idx: event for event in events}
|
||||
full_attention_event = events_by_group[0]
|
||||
assert full_attention_event.block_size == 4
|
||||
assert full_attention_event.token_ids == list(range(16, 28))
|
||||
assert len(full_attention_event.block_hashes) == 3
|
||||
assert full_attention_event.parent_block_hash is not None
|
||||
|
||||
recurrent_event = events_by_group[1]
|
||||
assert recurrent_event.block_size == 0
|
||||
assert recurrent_event.token_ids == []
|
||||
assert len(recurrent_event.block_hashes) == 1
|
||||
assert recurrent_event.parent_block_hash is None
|
||||
|
||||
|
||||
def test_partial_lookup_returns_exact_boundary_and_group_load_keys():
|
||||
|
||||
@@ -145,6 +145,85 @@ class OffloadingEventsTracker:
|
||||
req, group_config, chunk_idx
|
||||
)
|
||||
|
||||
def record_partial_store(
|
||||
self,
|
||||
req: Request,
|
||||
group_config: "GroupOffloadConfig",
|
||||
boundary_tokens: int,
|
||||
offload_key: OffloadKey,
|
||||
) -> None:
|
||||
"""Snapshot metadata for a newly stored partial recurrent tail."""
|
||||
if group_config.sliding_window_size_in_chunks is not None:
|
||||
return
|
||||
self._record_partial_tail(req, group_config, boundary_tokens, offload_key)
|
||||
|
||||
def record_partial_lookup(
|
||||
self,
|
||||
req: Request,
|
||||
group_config: "GroupOffloadConfig",
|
||||
boundary_tokens: int,
|
||||
offload_key: OffloadKey,
|
||||
) -> None:
|
||||
"""Backfill metadata for a partial recurrent tail lookup hit."""
|
||||
if group_config.sliding_window_size_in_chunks is not None:
|
||||
return
|
||||
if offload_key not in self._pending_event_metadata:
|
||||
self._record_partial_tail(req, group_config, boundary_tokens, offload_key)
|
||||
|
||||
def _record_partial_tail(
|
||||
self,
|
||||
req: Request,
|
||||
group_config: "GroupOffloadConfig",
|
||||
boundary_tokens: int,
|
||||
offload_key: OffloadKey,
|
||||
) -> None:
|
||||
"""Build metadata for the valid prefix of one physical cache block.
|
||||
|
||||
A partial recurrent tail ends on a hash boundary but before its
|
||||
physical cache block is full. The event describes only the valid
|
||||
hashes and tokens, not the unused remainder of that physical block.
|
||||
"""
|
||||
if not self.self_describing_enabled:
|
||||
return
|
||||
|
||||
tokens_per_hash = group_config.tokens_per_chunk // group_config.hashes_per_chunk
|
||||
# Subtract one so the boundary token itself cannot select the next
|
||||
# physical block when the boundary lies exactly on a block edge.
|
||||
chunk_start = (
|
||||
(boundary_tokens - 1) // group_config.tokens_per_chunk
|
||||
) * group_config.tokens_per_chunk
|
||||
first_hash_idx = chunk_start // tokens_per_hash
|
||||
last_hash_idx = boundary_tokens // tokens_per_hash
|
||||
assert chunk_start < boundary_tokens
|
||||
assert boundary_tokens % tokens_per_hash == 0
|
||||
assert last_hash_idx <= len(req.block_hashes)
|
||||
|
||||
# Unlike a complete chunk, a partial tail contains only the hashes
|
||||
# between its physical block start and its valid token boundary.
|
||||
maybe_block_hashes = req.block_hashes[first_hash_idx:last_hash_idx]
|
||||
block_hashes = tuple(
|
||||
block_hash for block_hash in maybe_block_hashes if block_hash is not None
|
||||
)
|
||||
assert block_hashes and len(block_hashes) == len(maybe_block_hashes)
|
||||
parent_block_hash = (
|
||||
req.block_hashes[first_hash_idx - 1] if first_hash_idx > 0 else None
|
||||
)
|
||||
assert first_hash_idx == 0 or parent_block_hash is not None
|
||||
|
||||
lora_id = req.lora_request.adapter_id if req.lora_request is not None else None
|
||||
lora_name = req.lora_request.name if req.lora_request is not None else None
|
||||
self._pending_event_metadata[offload_key] = _OffloadEventMetadata(
|
||||
block_hashes=block_hashes,
|
||||
parent_block_hash=parent_block_hash,
|
||||
token_ids=tuple(req.all_token_ids[chunk_start:boundary_tokens]),
|
||||
block_size=tokens_per_hash,
|
||||
lora_id=lora_id,
|
||||
lora_name=lora_name,
|
||||
extra_keys=None,
|
||||
group_idx=group_config.group_idx,
|
||||
kv_cache_spec=group_config.kv_event_group_spec,
|
||||
)
|
||||
|
||||
def take_events(self, events: Iterable[OffloadingEvent]) -> Iterable[KVCacheEvent]:
|
||||
"""Translate raw OffloadingEvents into self-describing KV events.
|
||||
|
||||
|
||||
@@ -280,10 +280,6 @@ class SchedulerOffloadConfig(NamedTuple):
|
||||
spec.blocks_per_chunk == 1
|
||||
and len(group_block_sizes) == 1
|
||||
and has_partial_recurrent_group
|
||||
and not (
|
||||
spec.kv_events_config.enable_kv_cache_events
|
||||
and spec.kv_events_config.self_describing_kv_events
|
||||
)
|
||||
and all(
|
||||
config.sliding_window_size_in_chunks is None
|
||||
or config.requires_cow_source
|
||||
@@ -897,10 +893,12 @@ class OffloadingConnectorScheduler:
|
||||
for boundary in range(max_boundary, complete_boundary, -tokens_per_hash):
|
||||
boundary_pending = False
|
||||
boundary_missed = False
|
||||
boundary_keys = []
|
||||
for group_config in self.config.kv_group_configs:
|
||||
key = self._make_boundary_key(
|
||||
req_status.req, group_config.group_idx, boundary
|
||||
)
|
||||
boundary_keys.append(key)
|
||||
result = self.manager.lookup(key, req_status.req_context)
|
||||
if result is LookupResult.MISS:
|
||||
boundary_missed = True
|
||||
@@ -910,6 +908,12 @@ class OffloadingConnectorScheduler:
|
||||
|
||||
pending |= boundary_pending
|
||||
if not boundary_missed and not boundary_pending:
|
||||
for group_config, key in zip(
|
||||
self.config.kv_group_configs, boundary_keys
|
||||
):
|
||||
self._events_tracker.record_partial_lookup(
|
||||
req_status.req, group_config, boundary, key
|
||||
)
|
||||
req_status.partial_tail_boundary = boundary
|
||||
return boundary - local_tokens
|
||||
|
||||
@@ -1203,6 +1207,12 @@ class OffloadingConnectorScheduler:
|
||||
if not store_output.keys_to_store:
|
||||
continue
|
||||
|
||||
for group_config, key in zip(self.config.kv_group_configs, keys):
|
||||
if key in store_output.keys_to_store:
|
||||
self._events_tracker.record_partial_store(
|
||||
req, group_config, boundary, key
|
||||
)
|
||||
|
||||
group_by_key = {key: idx for idx, key in enumerate(keys)}
|
||||
accepted_groups = [group_by_key[key] for key in store_output.keys_to_store]
|
||||
group_sizes = [0] * len(self.config.kv_group_configs)
|
||||
|
||||
Reference in New Issue
Block a user