From c4239986427a40389fd79c1a4f6afd48aea6512e Mon Sep 17 00:00:00 2001 From: Chauncey Date: Sun, 9 Aug 2026 16:28:44 +0800 Subject: [PATCH] [KV Offload] Emit self-describing events for partial recurrent blocks (#51243) Signed-off-by: chaunceyjiang Co-authored-by: Or Ozeri --- docs/features/kv_offloading_usage.md | 2 +- .../unit/offloading_connector/test_events.py | 52 ++++++++++++ .../offloading_connector/test_scheduler.py | 36 ++++++++- .../kv_connector/v1/offloading/events.py | 79 +++++++++++++++++++ .../kv_connector/v1/offloading/scheduler.py | 18 ++++- 5 files changed, 181 insertions(+), 6 deletions(-) diff --git a/docs/features/kv_offloading_usage.md b/docs/features/kv_offloading_usage.md index 0167b45930..83c98c3d01 100644 --- a/docs/features/kv_offloading_usage.md +++ b/docs/features/kv_offloading_usage.md @@ -76,7 +76,7 @@ vllm serve \ | `max_tracker_size` | no | `64000` | single-tier | Max entries in the lookup tracker. | | `secondary_tiers` | no | `[]` | multi-tier | List of secondary tier configs (see below). | | `offload_prompt_only` | no | `true` | both | If `true`, only prompt (prefill) blocks are offloaded; decode blocks are skipped. | -| `self_describing_kv_events` | no | `false` | both | Opt-in. When `true` *and* KV cache events are enabled (`--kv-events-config` with `enable_kv_cache_events`), the connector emits self-describing block-granular `BlockStored`/`BlockRemoved` payloads (constituent block hashes, whole-chunk `token_ids`, per-block `block_size`, parent hash, LoRA + group/cache-spec metadata) instead of the placeholder fallback, so external KV-event consumers can index offloaded blocks. Inert unless events are enabled. With `TieringOffloadingSpec`, a CPU promotion is self-describing when a local request observes its primary-tier `HIT` before event translation; otherwise its stored event may retain the placeholder, while a later `HIT` can backfill metadata for removal. Pending-removal/re-promotion races and externally initiated promotions may also produce placeholders, and consumers must ignore removals for unknown hashes. Full-attention groups only; sliding-window/SSM groups keep the placeholder fallback. In chunk mode (`block_size` > GPU block size, or `blocks_per_chunk` > 1), overlapping chunks re-announce shared per-block hashes, so consumers must reference-count (deduplicate) repeated store/remove announcements. | +| `self_describing_kv_events` | no | `false` | both | Opt-in. When `true` *and* KV cache events are enabled (`--kv-events-config` with `enable_kv_cache_events`), the connector emits self-describing block-granular `BlockStored`/`BlockRemoved` payloads (constituent block hashes, whole-chunk `token_ids`, per-block `block_size`, parent hash, LoRA + group/cache-spec metadata) instead of the placeholder fallback, so external KV-event consumers can index offloaded blocks. Inert unless events are enabled. With `TieringOffloadingSpec`, a CPU promotion is self-describing when a local request observes its primary-tier `HIT` before event translation; otherwise its stored event may retain the placeholder, while a later `HIT` can backfill metadata for removal. Pending-removal/re-promotion races and externally initiated promotions may also produce placeholders, and consumers must ignore removals for unknown hashes. Partial recurrent tails emit the hash-aligned portion from the physical block start through the tail boundary. Other sliding-window/SSM chunks keep the placeholder fallback. In chunk mode (`block_size` > GPU block size, or `blocks_per_chunk` > 1), overlapping chunks re-announce shared per-block hashes, so consumers must reference-count (deduplicate) repeated store/remove announcements. | | `spec_module_path` | no | — | both | Python import path for a custom `OffloadingSpec` not in the built-in registry. Required only when `spec_name` is not built-in (advanced). | ## Custom Eviction Policies diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_events.py b/tests/v1/kv_connector/unit/offloading_connector/test_events.py index a6bab14e06..6d191bdb70 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_events.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_events.py @@ -208,6 +208,58 @@ def test_take_events_forwards_locality_to_placeholder_store(): assert events[0].locality == "REMOTE" +def test_partial_tail_event_describes_hash_aligned_physical_block_prefix(): + tracker = _tracker() + group_config = _group_config(block_size=16, blocks_per_chunk=1)._replace( + hashes_per_chunk=4 + ) + req = _request(block_hashes=[_hash(i) for i in range(8)], token_count=32) + key = make_offload_key(req.block_hashes[6], group_config.group_idx) + + tracker.record_partial_store(req, group_config, 28, key) + [event] = tracker.take_events([_stored_event([key])]) + + assert isinstance(event, BlockStored) + assert event.block_hashes == [_wire_hash(_hash(i)) for i in range(4, 7)] + assert event.parent_block_hash == _wire_hash(_hash(3)) + assert event.token_ids == list(range(17, 29)) + assert event.block_size == 4 + + +def test_partial_tail_lookup_does_not_overwrite_store_metadata(): + tracker = _tracker() + group_config = _group_config() + stored_req = _request(block_hashes=[_hash(0)], token_count=4) + lookup_req = _request(block_hashes=[_hash(0)], token_count=4) + lookup_req.all_token_ids = [9, 9, 9, 9] + key = make_offload_key(stored_req.block_hashes[0], group_config.group_idx) + + tracker.record_partial_store(stored_req, group_config, 4, key) + tracker.record_partial_lookup(lookup_req, group_config, 4, key) + [event] = tracker.take_events([_stored_event([key])]) + + assert isinstance(event, BlockStored) + assert event.token_ids == [1, 2, 3, 4] + + +@pytest.mark.parametrize( + "record_method", ["record_partial_store", "record_partial_lookup"] +) +def test_partial_tail_sliding_window_event_uses_placeholder(record_method): + tracker = _tracker() + group_config = _group_config(sliding_window_size_in_chunks=1) + req = _request(block_hashes=[_hash(0)], token_count=4) + key = make_offload_key(req.block_hashes[0], group_config.group_idx) + + getattr(tracker, record_method)(req, group_config, 4, key) + [event] = tracker.take_events([_stored_event([key])]) + + assert isinstance(event, BlockStored) + assert event.block_hashes == [_wire_hash(_hash(0))] + assert event.token_ids == [] + assert event.block_size == 0 + + def test_take_events_forwards_locality_to_remove(): tracker = _tracker() req = _request(block_hashes=[_hash(0)], token_count=4) diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py index 7632b26ea4..ca5dd26575 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py @@ -16,6 +16,7 @@ from tests.v1.kv_connector.unit.offloading_connector.utils import ( to_keys, ) from tests.v1.kv_connector.unit.utils import EOS_TOKEN_ID +from vllm.config import KVEventsConfig from vllm.distributed.kv_events import MEDIUM_CPU, BlockRemoved, BlockStored from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import ( OffloadingConnectorMetadata, @@ -60,9 +61,12 @@ from vllm.v1.request import RequestStatus def _make_partial_tail_scheduler() -> OffloadingConnectorScheduler: - vllm_config = _make_vllm_config() + vllm_config = _make_vllm_config(extra_config={"self_describing_kv_events": True}) vllm_config.cache_config.prefix_match_unit = 4 vllm_config.speculative_config = None + vllm_config.kv_events_config = KVEventsConfig( + enable_kv_cache_events=True, publisher="null" + ) kv_cache_config = _make_mamba_hybrid_kv_cache_config() spec = MockOffloadingSpec(build_offloading_config(vllm_config, kv_cache_config)) return OffloadingConnectorScheduler(spec, vllm_config, kv_cache_config) @@ -77,6 +81,8 @@ def _make_partial_tail_request( request.num_prompt_tokens = 30 request.num_tokens = 30 request.block_hashes = [BlockHash(f"h{i}".encode()) for i in range(7)] + request.all_token_ids = list(range(30)) + request.lora_request = None request.is_finished.return_value = False scheduler.on_new_request(request) return request @@ -119,6 +125,34 @@ def test_partial_tail_store_uses_attention_and_recurrent_cow_sources(): 12: {job_id}, 99: {job_id}, } + assert scheduler.config.supports_partial_tail + + events = list( + scheduler._events_tracker.take_events( + [ + OffloadingEvent( + keys=list(scheduler._jobs[job_id].keys), + medium=Medium.CPU, + removed=False, + ) + ] + ) + ) + assert len(events) == 2 + assert all(isinstance(event, BlockStored) for event in events) + assert {event.group_idx for event in events} == {0, 1} + events_by_group = {event.group_idx: event for event in events} + full_attention_event = events_by_group[0] + assert full_attention_event.block_size == 4 + assert full_attention_event.token_ids == list(range(16, 28)) + assert len(full_attention_event.block_hashes) == 3 + assert full_attention_event.parent_block_hash is not None + + recurrent_event = events_by_group[1] + assert recurrent_event.block_size == 0 + assert recurrent_event.token_ids == [] + assert len(recurrent_event.block_hashes) == 1 + assert recurrent_event.parent_block_hash is None def test_partial_lookup_returns_exact_boundary_and_group_load_keys(): diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py index 67ada00e03..daeb1b224c 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py @@ -145,6 +145,85 @@ class OffloadingEventsTracker: req, group_config, chunk_idx ) + def record_partial_store( + self, + req: Request, + group_config: "GroupOffloadConfig", + boundary_tokens: int, + offload_key: OffloadKey, + ) -> None: + """Snapshot metadata for a newly stored partial recurrent tail.""" + if group_config.sliding_window_size_in_chunks is not None: + return + self._record_partial_tail(req, group_config, boundary_tokens, offload_key) + + def record_partial_lookup( + self, + req: Request, + group_config: "GroupOffloadConfig", + boundary_tokens: int, + offload_key: OffloadKey, + ) -> None: + """Backfill metadata for a partial recurrent tail lookup hit.""" + if group_config.sliding_window_size_in_chunks is not None: + return + if offload_key not in self._pending_event_metadata: + self._record_partial_tail(req, group_config, boundary_tokens, offload_key) + + def _record_partial_tail( + self, + req: Request, + group_config: "GroupOffloadConfig", + boundary_tokens: int, + offload_key: OffloadKey, + ) -> None: + """Build metadata for the valid prefix of one physical cache block. + + A partial recurrent tail ends on a hash boundary but before its + physical cache block is full. The event describes only the valid + hashes and tokens, not the unused remainder of that physical block. + """ + if not self.self_describing_enabled: + return + + tokens_per_hash = group_config.tokens_per_chunk // group_config.hashes_per_chunk + # Subtract one so the boundary token itself cannot select the next + # physical block when the boundary lies exactly on a block edge. + chunk_start = ( + (boundary_tokens - 1) // group_config.tokens_per_chunk + ) * group_config.tokens_per_chunk + first_hash_idx = chunk_start // tokens_per_hash + last_hash_idx = boundary_tokens // tokens_per_hash + assert chunk_start < boundary_tokens + assert boundary_tokens % tokens_per_hash == 0 + assert last_hash_idx <= len(req.block_hashes) + + # Unlike a complete chunk, a partial tail contains only the hashes + # between its physical block start and its valid token boundary. + maybe_block_hashes = req.block_hashes[first_hash_idx:last_hash_idx] + block_hashes = tuple( + block_hash for block_hash in maybe_block_hashes if block_hash is not None + ) + assert block_hashes and len(block_hashes) == len(maybe_block_hashes) + parent_block_hash = ( + req.block_hashes[first_hash_idx - 1] if first_hash_idx > 0 else None + ) + assert first_hash_idx == 0 or parent_block_hash is not None + + lora_id = req.lora_request.adapter_id if req.lora_request is not None else None + lora_name = req.lora_request.name if req.lora_request is not None else None + self._pending_event_metadata[offload_key] = _OffloadEventMetadata( + block_hashes=block_hashes, + parent_block_hash=parent_block_hash, + token_ids=tuple(req.all_token_ids[chunk_start:boundary_tokens]), + block_size=tokens_per_hash, + lora_id=lora_id, + lora_name=lora_name, + extra_keys=None, + group_idx=group_config.group_idx, + kv_cache_spec=group_config.kv_event_group_spec, + ) + def take_events(self, events: Iterable[OffloadingEvent]) -> Iterable[KVCacheEvent]: """Translate raw OffloadingEvents into self-describing KV events. diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index 1644d9c72e..ebc9b87e12 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -280,10 +280,6 @@ class SchedulerOffloadConfig(NamedTuple): spec.blocks_per_chunk == 1 and len(group_block_sizes) == 1 and has_partial_recurrent_group - and not ( - spec.kv_events_config.enable_kv_cache_events - and spec.kv_events_config.self_describing_kv_events - ) and all( config.sliding_window_size_in_chunks is None or config.requires_cow_source @@ -897,10 +893,12 @@ class OffloadingConnectorScheduler: for boundary in range(max_boundary, complete_boundary, -tokens_per_hash): boundary_pending = False boundary_missed = False + boundary_keys = [] for group_config in self.config.kv_group_configs: key = self._make_boundary_key( req_status.req, group_config.group_idx, boundary ) + boundary_keys.append(key) result = self.manager.lookup(key, req_status.req_context) if result is LookupResult.MISS: boundary_missed = True @@ -910,6 +908,12 @@ class OffloadingConnectorScheduler: pending |= boundary_pending if not boundary_missed and not boundary_pending: + for group_config, key in zip( + self.config.kv_group_configs, boundary_keys + ): + self._events_tracker.record_partial_lookup( + req_status.req, group_config, boundary, key + ) req_status.partial_tail_boundary = boundary return boundary - local_tokens @@ -1203,6 +1207,12 @@ class OffloadingConnectorScheduler: if not store_output.keys_to_store: continue + for group_config, key in zip(self.config.kv_group_configs, keys): + if key in store_output.keys_to_store: + self._events_tracker.record_partial_store( + req, group_config, boundary, key + ) + group_by_key = {key: idx for idx, key in enumerate(keys)} accepted_groups = [group_by_key[key] for key in store_output.keys_to_store] group_sizes = [0] * len(self.config.kv_group_configs)