[KV Offload] Emit self-describing events for partial recurrent blocks (#51243)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Chauncey
2026-08-09 11:28:44 +03:00
committed by GitHub
co-authored by Or Ozeri
parent 7f6432cc0d
commit c423998642
5 changed files with 181 additions and 6 deletions
+1 -1
View File
@@ -76,7 +76,7 @@ vllm serve <model> \
| `max_tracker_size` | no | `64000` | single-tier | Max entries in the lookup tracker. |
| `secondary_tiers` | no | `[]` | multi-tier | List of secondary tier configs (see below). |
| `offload_prompt_only` | no | `true` | both | If `true`, only prompt (prefill) blocks are offloaded; decode blocks are skipped. |
| `self_describing_kv_events` | no | `false` | both | Opt-in. When `true` *and* KV cache events are enabled (`--kv-events-config` with `enable_kv_cache_events`), the connector emits self-describing block-granular `BlockStored`/`BlockRemoved` payloads (constituent block hashes, whole-chunk `token_ids`, per-block `block_size`, parent hash, LoRA + group/cache-spec metadata) instead of the placeholder fallback, so external KV-event consumers can index offloaded blocks. Inert unless events are enabled. With `TieringOffloadingSpec`, a CPU promotion is self-describing when a local request observes its primary-tier `HIT` before event translation; otherwise its stored event may retain the placeholder, while a later `HIT` can backfill metadata for removal. Pending-removal/re-promotion races and externally initiated promotions may also produce placeholders, and consumers must ignore removals for unknown hashes. Full-attention groups only; sliding-window/SSM groups keep the placeholder fallback. In chunk mode (`block_size` > GPU block size, or `blocks_per_chunk` > 1), overlapping chunks re-announce shared per-block hashes, so consumers must reference-count (deduplicate) repeated store/remove announcements. |
| `self_describing_kv_events` | no | `false` | both | Opt-in. When `true` *and* KV cache events are enabled (`--kv-events-config` with `enable_kv_cache_events`), the connector emits self-describing block-granular `BlockStored`/`BlockRemoved` payloads (constituent block hashes, whole-chunk `token_ids`, per-block `block_size`, parent hash, LoRA + group/cache-spec metadata) instead of the placeholder fallback, so external KV-event consumers can index offloaded blocks. Inert unless events are enabled. With `TieringOffloadingSpec`, a CPU promotion is self-describing when a local request observes its primary-tier `HIT` before event translation; otherwise its stored event may retain the placeholder, while a later `HIT` can backfill metadata for removal. Pending-removal/re-promotion races and externally initiated promotions may also produce placeholders, and consumers must ignore removals for unknown hashes. Partial recurrent tails emit the hash-aligned portion from the physical block start through the tail boundary. Other sliding-window/SSM chunks keep the placeholder fallback. In chunk mode (`block_size` > GPU block size, or `blocks_per_chunk` > 1), overlapping chunks re-announce shared per-block hashes, so consumers must reference-count (deduplicate) repeated store/remove announcements. |
| `spec_module_path` | no | — | both | Python import path for a custom `OffloadingSpec` not in the built-in registry. Required only when `spec_name` is not built-in (advanced). |
## Custom Eviction Policies
@@ -208,6 +208,58 @@ def test_take_events_forwards_locality_to_placeholder_store():
assert events[0].locality == "REMOTE"
def test_partial_tail_event_describes_hash_aligned_physical_block_prefix():
tracker = _tracker()
group_config = _group_config(block_size=16, blocks_per_chunk=1)._replace(
hashes_per_chunk=4
)
req = _request(block_hashes=[_hash(i) for i in range(8)], token_count=32)
key = make_offload_key(req.block_hashes[6], group_config.group_idx)
tracker.record_partial_store(req, group_config, 28, key)
[event] = tracker.take_events([_stored_event([key])])
assert isinstance(event, BlockStored)
assert event.block_hashes == [_wire_hash(_hash(i)) for i in range(4, 7)]
assert event.parent_block_hash == _wire_hash(_hash(3))
assert event.token_ids == list(range(17, 29))
assert event.block_size == 4
def test_partial_tail_lookup_does_not_overwrite_store_metadata():
tracker = _tracker()
group_config = _group_config()
stored_req = _request(block_hashes=[_hash(0)], token_count=4)
lookup_req = _request(block_hashes=[_hash(0)], token_count=4)
lookup_req.all_token_ids = [9, 9, 9, 9]
key = make_offload_key(stored_req.block_hashes[0], group_config.group_idx)
tracker.record_partial_store(stored_req, group_config, 4, key)
tracker.record_partial_lookup(lookup_req, group_config, 4, key)
[event] = tracker.take_events([_stored_event([key])])
assert isinstance(event, BlockStored)
assert event.token_ids == [1, 2, 3, 4]
@pytest.mark.parametrize(
"record_method", ["record_partial_store", "record_partial_lookup"]
)
def test_partial_tail_sliding_window_event_uses_placeholder(record_method):
tracker = _tracker()
group_config = _group_config(sliding_window_size_in_chunks=1)
req = _request(block_hashes=[_hash(0)], token_count=4)
key = make_offload_key(req.block_hashes[0], group_config.group_idx)
getattr(tracker, record_method)(req, group_config, 4, key)
[event] = tracker.take_events([_stored_event([key])])
assert isinstance(event, BlockStored)
assert event.block_hashes == [_wire_hash(_hash(0))]
assert event.token_ids == []
assert event.block_size == 0
def test_take_events_forwards_locality_to_remove():
tracker = _tracker()
req = _request(block_hashes=[_hash(0)], token_count=4)
@@ -16,6 +16,7 @@ from tests.v1.kv_connector.unit.offloading_connector.utils import (
to_keys,
)
from tests.v1.kv_connector.unit.utils import EOS_TOKEN_ID
from vllm.config import KVEventsConfig
from vllm.distributed.kv_events import MEDIUM_CPU, BlockRemoved, BlockStored
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
OffloadingConnectorMetadata,
@@ -60,9 +61,12 @@ from vllm.v1.request import RequestStatus
def _make_partial_tail_scheduler() -> OffloadingConnectorScheduler:
vllm_config = _make_vllm_config()
vllm_config = _make_vllm_config(extra_config={"self_describing_kv_events": True})
vllm_config.cache_config.prefix_match_unit = 4
vllm_config.speculative_config = None
vllm_config.kv_events_config = KVEventsConfig(
enable_kv_cache_events=True, publisher="null"
)
kv_cache_config = _make_mamba_hybrid_kv_cache_config()
spec = MockOffloadingSpec(build_offloading_config(vllm_config, kv_cache_config))
return OffloadingConnectorScheduler(spec, vllm_config, kv_cache_config)
@@ -77,6 +81,8 @@ def _make_partial_tail_request(
request.num_prompt_tokens = 30
request.num_tokens = 30
request.block_hashes = [BlockHash(f"h{i}".encode()) for i in range(7)]
request.all_token_ids = list(range(30))
request.lora_request = None
request.is_finished.return_value = False
scheduler.on_new_request(request)
return request
@@ -119,6 +125,34 @@ def test_partial_tail_store_uses_attention_and_recurrent_cow_sources():
12: {job_id},
99: {job_id},
}
assert scheduler.config.supports_partial_tail
events = list(
scheduler._events_tracker.take_events(
[
OffloadingEvent(
keys=list(scheduler._jobs[job_id].keys),
medium=Medium.CPU,
removed=False,
)
]
)
)
assert len(events) == 2
assert all(isinstance(event, BlockStored) for event in events)
assert {event.group_idx for event in events} == {0, 1}
events_by_group = {event.group_idx: event for event in events}
full_attention_event = events_by_group[0]
assert full_attention_event.block_size == 4
assert full_attention_event.token_ids == list(range(16, 28))
assert len(full_attention_event.block_hashes) == 3
assert full_attention_event.parent_block_hash is not None
recurrent_event = events_by_group[1]
assert recurrent_event.block_size == 0
assert recurrent_event.token_ids == []
assert len(recurrent_event.block_hashes) == 1
assert recurrent_event.parent_block_hash is None
def test_partial_lookup_returns_exact_boundary_and_group_load_keys():
@@ -145,6 +145,85 @@ class OffloadingEventsTracker:
req, group_config, chunk_idx
)
def record_partial_store(
self,
req: Request,
group_config: "GroupOffloadConfig",
boundary_tokens: int,
offload_key: OffloadKey,
) -> None:
"""Snapshot metadata for a newly stored partial recurrent tail."""
if group_config.sliding_window_size_in_chunks is not None:
return
self._record_partial_tail(req, group_config, boundary_tokens, offload_key)
def record_partial_lookup(
self,
req: Request,
group_config: "GroupOffloadConfig",
boundary_tokens: int,
offload_key: OffloadKey,
) -> None:
"""Backfill metadata for a partial recurrent tail lookup hit."""
if group_config.sliding_window_size_in_chunks is not None:
return
if offload_key not in self._pending_event_metadata:
self._record_partial_tail(req, group_config, boundary_tokens, offload_key)
def _record_partial_tail(
self,
req: Request,
group_config: "GroupOffloadConfig",
boundary_tokens: int,
offload_key: OffloadKey,
) -> None:
"""Build metadata for the valid prefix of one physical cache block.
A partial recurrent tail ends on a hash boundary but before its
physical cache block is full. The event describes only the valid
hashes and tokens, not the unused remainder of that physical block.
"""
if not self.self_describing_enabled:
return
tokens_per_hash = group_config.tokens_per_chunk // group_config.hashes_per_chunk
# Subtract one so the boundary token itself cannot select the next
# physical block when the boundary lies exactly on a block edge.
chunk_start = (
(boundary_tokens - 1) // group_config.tokens_per_chunk
) * group_config.tokens_per_chunk
first_hash_idx = chunk_start // tokens_per_hash
last_hash_idx = boundary_tokens // tokens_per_hash
assert chunk_start < boundary_tokens
assert boundary_tokens % tokens_per_hash == 0
assert last_hash_idx <= len(req.block_hashes)
# Unlike a complete chunk, a partial tail contains only the hashes
# between its physical block start and its valid token boundary.
maybe_block_hashes = req.block_hashes[first_hash_idx:last_hash_idx]
block_hashes = tuple(
block_hash for block_hash in maybe_block_hashes if block_hash is not None
)
assert block_hashes and len(block_hashes) == len(maybe_block_hashes)
parent_block_hash = (
req.block_hashes[first_hash_idx - 1] if first_hash_idx > 0 else None
)
assert first_hash_idx == 0 or parent_block_hash is not None
lora_id = req.lora_request.adapter_id if req.lora_request is not None else None
lora_name = req.lora_request.name if req.lora_request is not None else None
self._pending_event_metadata[offload_key] = _OffloadEventMetadata(
block_hashes=block_hashes,
parent_block_hash=parent_block_hash,
token_ids=tuple(req.all_token_ids[chunk_start:boundary_tokens]),
block_size=tokens_per_hash,
lora_id=lora_id,
lora_name=lora_name,
extra_keys=None,
group_idx=group_config.group_idx,
kv_cache_spec=group_config.kv_event_group_spec,
)
def take_events(self, events: Iterable[OffloadingEvent]) -> Iterable[KVCacheEvent]:
"""Translate raw OffloadingEvents into self-describing KV events.
@@ -280,10 +280,6 @@ class SchedulerOffloadConfig(NamedTuple):
spec.blocks_per_chunk == 1
and len(group_block_sizes) == 1
and has_partial_recurrent_group
and not (
spec.kv_events_config.enable_kv_cache_events
and spec.kv_events_config.self_describing_kv_events
)
and all(
config.sliding_window_size_in_chunks is None
or config.requires_cow_source
@@ -897,10 +893,12 @@ class OffloadingConnectorScheduler:
for boundary in range(max_boundary, complete_boundary, -tokens_per_hash):
boundary_pending = False
boundary_missed = False
boundary_keys = []
for group_config in self.config.kv_group_configs:
key = self._make_boundary_key(
req_status.req, group_config.group_idx, boundary
)
boundary_keys.append(key)
result = self.manager.lookup(key, req_status.req_context)
if result is LookupResult.MISS:
boundary_missed = True
@@ -910,6 +908,12 @@ class OffloadingConnectorScheduler:
pending |= boundary_pending
if not boundary_missed and not boundary_pending:
for group_config, key in zip(
self.config.kv_group_configs, boundary_keys
):
self._events_tracker.record_partial_lookup(
req_status.req, group_config, boundary, key
)
req_status.partial_tail_boundary = boundary
return boundary - local_tokens
@@ -1203,6 +1207,12 @@ class OffloadingConnectorScheduler:
if not store_output.keys_to_store:
continue
for group_config, key in zip(self.config.kv_group_configs, keys):
if key in store_output.keys_to_store:
self._events_tracker.record_partial_store(
req, group_config, boundary, key
)
group_by_key = {key: idx for idx, key in enumerate(keys)}
accepted_groups = [group_by_key[key] for key in store_output.keys_to_store]
group_sizes = [0] * len(self.config.kv_group_configs)