[feature][kv_offload] Self-describing KV events for OffloadingConnector (#43468)

Signed-off-by: Change72 <changg@nvidia.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Change72
2026-06-22 07:27:46 +00:00
committed by GitHub
co-authored by Claude
parent d14e551a53
commit a9f7b2d41c
9 changed files with 720 additions and 62 deletions
@@ -0,0 +1,286 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Self-describing KV cache events for the offloading connector.
The OffloadingManager identifies an offloaded chunk only by its OffloadKey,
so its raw events carry no token ids, parent hash, or block size.
:class:`OffloadingEventsTracker` snapshots each chunk's full ``BlockStored``
payload while the ``Request`` is alive and publishes stores as block-granular
payloads: a chunk event may carry multiple constituent per-block hashes, and
evictions fan out to the same hashes. Chunks overlapping a non-chunk-aligned
shared prefix re-announce the shared hashes once per chunk; consumers are
expected to deduplicate (reference-count) repeated store/remove announcements
of the same hash. Opt-in via
``kv_connector_extra_config["self_describing_kv_events"]``; inert unless
KV cache events are enabled. See the PR description for the full design.
"""
from collections.abc import Iterable
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, NamedTuple
from vllm.distributed.kv_events import BlockRemoved, BlockStored, KVCacheEvent
from vllm.logger import init_logger
from vllm.v1.core.kv_cache_utils import BlockHash, maybe_convert_block_hash
from vllm.v1.kv_cache_interface import (
KVCacheGroupSpec,
get_kv_cache_spec_kind,
get_kv_cache_spec_sliding_window,
)
from vllm.v1.kv_offload.base import (
OffloadingEvent,
OffloadingKVEventsConfig,
OffloadKey,
get_offload_block_hash,
get_offload_group_idx,
)
from vllm.v1.request import Request
if TYPE_CHECKING:
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import (
GroupOffloadConfig,
)
logger = init_logger(__name__)
class OffloadingEventGroupSpec(NamedTuple):
kv_cache_spec_kind: str | None
kv_cache_spec_sliding_window: int | None
def get_offloading_event_group_spec(
kv_cache_group: KVCacheGroupSpec,
) -> OffloadingEventGroupSpec:
kv_cache_spec = kv_cache_group.kv_cache_spec
return OffloadingEventGroupSpec(
kv_cache_spec_kind=get_kv_cache_spec_kind(kv_cache_spec).value,
kv_cache_spec_sliding_window=get_kv_cache_spec_sliding_window(kv_cache_spec),
)
@dataclass(slots=True)
class _OffloadEventMetadata:
"""BlockStored payload snapshot for one OffloadKey, captured at store
time and kept until the matching eviction event. ``medium`` is forwarded
from the OffloadingEvent."""
# The chunk's constituent block hashes; the last one is the OffloadKey.
block_hashes: tuple[BlockHash, ...]
parent_block_hash: BlockHash | None
token_ids: tuple[int, ...]
block_size: int
lora_id: int | None
lora_name: str | None
# Deferred: needs the same incremental curr_mm_idx handling as GPU events.
extra_keys: tuple[tuple[Any, ...] | None, ...] | None
group_idx: int
kv_cache_spec: OffloadingEventGroupSpec
class OffloadingEventsTracker:
"""Tracks offloaded chunks' KV event payloads from store to eviction.
The scheduler calls :meth:`record_store` from ``_build_store_jobs``
while the ``Request`` is available, and routes the manager's raw
:class:`OffloadingEvent` stream through :meth:`take_events`. All state
is bounded by the CPU pool capacity and cleared by :meth:`reset`.
"""
def __init__(self, config: OffloadingKVEventsConfig):
self.config = config
self.self_describing_enabled = (
config.enable_kv_cache_events and config.self_describing_kv_events
)
# OffloadKey -> payload snapshot, kept until the eviction event so
# BlockRemoved can fan out. Bounded: one entry per offloaded chunk.
self._pending_event_metadata: dict[OffloadKey, _OffloadEventMetadata] = {}
def record_store(
self,
req: Request,
group_config: "GroupOffloadConfig",
offload_block_idx: int,
offload_key: OffloadKey,
) -> None:
"""Snapshot the KV cache event payload for one offloaded chunk.
No-op when self-describing event capture is disabled or for
sliding-window / SSM groups, which keep the legacy placeholder payload.
"""
if not self.self_describing_enabled:
return
if group_config.sliding_window_size_in_blocks is not None:
return
meta = self._build_event_metadata(req, group_config, offload_block_idx)
self._pending_event_metadata[offload_key] = meta
def take_events(self, events: Iterable[OffloadingEvent]) -> Iterable[KVCacheEvent]:
"""Translate raw OffloadingEvents into self-describing KV events.
Complete metadata is available only for full-attention groups when
the tracker is enabled. Other shapes retain the legacy placeholder
payload so consumers can ignore them.
Yields:
``BlockStored`` or ``BlockRemoved`` events corresponding to
the underlying :class:`OffloadingEvent` stream.
"""
for event in events:
if event.removed:
yield from self._take_removed_event(event)
else:
yield from self._take_stored_event(event)
def reset(self) -> None:
"""Drop all tracked state; pending payloads are stale after a
manager cache reset."""
self._pending_event_metadata.clear()
def _build_event_metadata(
self,
req: Request,
group_config: "GroupOffloadConfig",
offload_block_idx: int,
) -> _OffloadEventMetadata:
"""Build the payload snapshot for one offloaded chunk: its
constituent per-block hashes, the whole chunk's tokens, and the
per-block ``block_size``."""
hbf = group_config.hash_block_size_factor
assert hbf > 0
assert offload_block_idx >= 0
# per-block token count (= the GPU/hash block size)
sub_block_size = group_config.offloaded_block_size // hbf
# chunk c covers hash-blocks [c*hbf, (c+1)*hbf); its tail block's hash
# is the chunk's OffloadKey.
first_hash_idx = offload_block_idx * hbf
last_hash_idx = first_hash_idx + hbf
assert first_hash_idx >= 0
assert last_hash_idx <= len(req.block_hashes)
chunk_hashes: list[BlockHash] = []
for block_hash in req.block_hashes[first_hash_idx:last_hash_idx]:
assert block_hash is not None
chunk_hashes.append(block_hash)
assert len(chunk_hashes) == hbf
if group_config.sliding_window_size_in_blocks is not None:
# record_store filters these out before calling this helper.
raise AssertionError("self-describing events only support full attention")
parent_block_hash: BlockHash | None
if first_hash_idx == 0:
parent_block_hash = None
else:
parent_block_hash = req.block_hashes[first_hash_idx - 1]
assert parent_block_hash is not None
tok_start = offload_block_idx * group_config.offloaded_block_size
tok_end = tok_start + group_config.offloaded_block_size
assert tok_end <= len(req.all_token_ids)
token_ids = tuple(req.all_token_ids[tok_start:tok_end])
lora_id: int | None = None
lora_name: str | None = None
if req.lora_request is not None:
lora_id = req.lora_request.adapter_id
lora_name = req.lora_request.name
return _OffloadEventMetadata(
block_hashes=tuple(chunk_hashes),
parent_block_hash=parent_block_hash,
token_ids=token_ids,
block_size=sub_block_size,
lora_id=lora_id,
lora_name=lora_name,
extra_keys=None,
group_idx=group_config.group_idx,
kv_cache_spec=group_config.kv_event_group_spec,
)
def _placeholder_stored(self, key: OffloadKey, medium: str) -> BlockStored:
return BlockStored(
block_hashes=[
maybe_convert_block_hash(BlockHash(get_offload_block_hash(key)))
],
parent_block_hash=None,
token_ids=[],
lora_id=None,
block_size=0,
medium=medium,
lora_name=None,
group_idx=get_offload_group_idx(key),
)
def _take_stored_event(self, event: OffloadingEvent) -> Iterable[KVCacheEvent]:
# Metadata is read, NOT popped: the entry must survive until the
# eviction event so BlockRemoved can fan out to the same hashes.
# Events are self-contained (own parent), so key order is free.
for key in event.keys:
meta = self._pending_event_metadata.get(key)
if meta is None:
if self.self_describing_enabled:
# Expected for unsupported shapes; warn once only.
logger.warning_once(
"OffloadingEventsTracker: no event metadata for "
"offload key during BlockStored emission; emitting a "
"placeholder payload. Expected for non-full-attention "
"groups; otherwise indicates a missing populate path."
)
yield self._placeholder_stored(key, event.medium)
continue
yield BlockStored(
block_hashes=list(
maybe_convert_block_hash(h) for h in meta.block_hashes
),
parent_block_hash=(
maybe_convert_block_hash(meta.parent_block_hash)
if meta.parent_block_hash is not None
else None
),
token_ids=list(meta.token_ids),
block_size=meta.block_size,
lora_id=meta.lora_id,
medium=event.medium,
lora_name=meta.lora_name,
extra_keys=(
list(meta.extra_keys) if meta.extra_keys is not None else None
),
group_idx=meta.group_idx,
kv_cache_spec_kind=meta.kv_cache_spec.kv_cache_spec_kind,
kv_cache_spec_sliding_window=(
meta.kv_cache_spec.kv_cache_spec_sliding_window
),
)
def _take_removed_event(self, event: OffloadingEvent) -> Iterable[KVCacheEvent]:
# Keep group_idx unambiguous if a manager batch spans groups.
by_group: dict[int, list] = {}
for key in event.keys:
meta = self._pending_event_metadata.pop(key, None)
if meta is not None:
group_idx = meta.group_idx
by_group.setdefault(group_idx, []).extend(
maybe_convert_block_hash(h) for h in meta.block_hashes
)
else:
if self.self_describing_enabled:
logger.warning_once(
"OffloadingEventsTracker: no event metadata for "
"offload key during BlockRemoved emission; emitting a "
"placeholder removal. Expected if the matching store "
"used the legacy placeholder payload; otherwise "
"indicates missing store metadata."
)
group_idx = get_offload_group_idx(key)
by_group.setdefault(group_idx, []).append(
maybe_convert_block_hash(BlockHash(get_offload_block_hash(key)))
)
for group_idx, hashes in by_group.items():
yield BlockRemoved(
block_hashes=hashes,
medium=event.medium,
group_idx=group_idx,
)
@@ -5,7 +5,7 @@ from dataclasses import dataclass, field
from itertools import islice
from typing import Any, NamedTuple
from vllm.distributed.kv_events import BlockRemoved, BlockStored, KVCacheEvent
from vllm.distributed.kv_events import KVCacheEvent
from vllm.distributed.kv_transfer.kv_connector.utils import yield_req_data
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
@@ -14,6 +14,11 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
ReqId,
TransferJob,
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.events import (
OffloadingEventGroupSpec,
OffloadingEventsTracker,
get_offloading_event_group_spec,
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
OffloadingConnectorStats,
_TransferMetricName,
@@ -36,7 +41,6 @@ from vllm.v1.kv_offload.base import (
OffloadPolicy,
ReqContext,
RequestOffloadingContext,
get_offload_block_hash,
make_offload_key,
)
from vllm.v1.outputs import KVConnectorOutput
@@ -69,6 +73,9 @@ class GroupOffloadConfig(NamedTuple):
gpu_block_size: int
offloaded_block_size: int
hash_block_size_factor: int
# KV cache spec metadata propagated onto emitted BlockStored events so
# KV-aware consumers can classify and filter the group.
kv_event_group_spec: OffloadingEventGroupSpec
# None below means full attention
sliding_window_size_in_blocks: int | None
# Number of this group's offloaded blocks per full-attention alignment
@@ -200,6 +207,9 @@ class SchedulerOffloadConfig(NamedTuple):
alignment_block_count=_alignment_block_count(
gpu_block_size * spec.block_size_factor, sw
),
kv_event_group_spec=get_offloading_event_group_spec(
spec.kv_cache_config.kv_cache_groups[idx]
),
is_eagle_group=idx in eagle_groups,
)
for idx, gpu_block_size in enumerate(spec.gpu_block_size)
@@ -361,6 +371,8 @@ class OffloadingConnectorScheduler:
# be freed before a request finishes).
self._block_id_to_pending_jobs: dict[int, set[int]] = {}
self._events_tracker = OffloadingEventsTracker(spec.kv_events_config)
def _generate_job_id(self) -> int:
job_id = self._job_counter
self._job_counter += 1
@@ -934,6 +946,11 @@ class OffloadingConnectorScheduler:
continue
offloaded_block_idx = start_block_idx + idx
self._events_tracker.record_store(
req, group_config, offloaded_block_idx, offload_key
)
gpu_block_idx = offloaded_block_idx * block_size_factor
for i in range(block_size_factor):
block_id = block_ids[gpu_block_idx + i]
@@ -1184,25 +1201,17 @@ class OffloadingConnectorScheduler:
return False, None
def take_events(self) -> Iterable[KVCacheEvent]:
"""Take the KV cache events from the connector.
"""Drain pending KV cache events.
Returns:
A list of KV cache events.
Complete metadata is available only when self-describing KV events
are enabled, and only for full-attention groups. Other shapes retain
the previous placeholder payload so consumers can ignore them.
Yields:
``BlockStored`` or ``BlockRemoved`` events corresponding to
the underlying :class:`OffloadingEvent` stream.
"""
for event in self.manager.take_events():
block_hashes = [get_offload_block_hash(key) for key in event.keys]
if event.removed:
yield BlockRemoved(block_hashes=block_hashes, medium=event.medium)
else:
yield BlockStored(
block_hashes=block_hashes,
parent_block_hash=None,
token_ids=[],
lora_id=None,
block_size=0,
medium=event.medium,
lora_name=None,
)
yield from self._events_tracker.take_events(self.manager.take_events())
def reset_cache(self) -> None:
"""Reset the offloading manager cache, evicting all stored blocks."""
@@ -1238,6 +1247,10 @@ class OffloadingConnectorScheduler:
self._jobs.clear()
self._block_id_to_pending_jobs.clear()
# The manager pool is empty; pending event payloads and announced
# reference counts are stale.
self._events_tracker.reset()
# Note: _current_batch_jobs_to_flush is intentionally NOT cleared.
# The load flush IDs collected above must be delivered to workers.
if self._blocks_being_loaded is not None: