[KV Offload] Emit self-describing events for partial recurrent blocks (#51243)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Chauncey
2026-08-09 11:28:44 +03:00
committed by GitHub
co-authored by Or Ozeri
parent 7f6432cc0d
commit c423998642
5 changed files with 181 additions and 6 deletions
@@ -145,6 +145,85 @@ class OffloadingEventsTracker:
req, group_config, chunk_idx
)
def record_partial_store(
self,
req: Request,
group_config: "GroupOffloadConfig",
boundary_tokens: int,
offload_key: OffloadKey,
) -> None:
"""Snapshot metadata for a newly stored partial recurrent tail."""
if group_config.sliding_window_size_in_chunks is not None:
return
self._record_partial_tail(req, group_config, boundary_tokens, offload_key)
def record_partial_lookup(
self,
req: Request,
group_config: "GroupOffloadConfig",
boundary_tokens: int,
offload_key: OffloadKey,
) -> None:
"""Backfill metadata for a partial recurrent tail lookup hit."""
if group_config.sliding_window_size_in_chunks is not None:
return
if offload_key not in self._pending_event_metadata:
self._record_partial_tail(req, group_config, boundary_tokens, offload_key)
def _record_partial_tail(
self,
req: Request,
group_config: "GroupOffloadConfig",
boundary_tokens: int,
offload_key: OffloadKey,
) -> None:
"""Build metadata for the valid prefix of one physical cache block.
A partial recurrent tail ends on a hash boundary but before its
physical cache block is full. The event describes only the valid
hashes and tokens, not the unused remainder of that physical block.
"""
if not self.self_describing_enabled:
return
tokens_per_hash = group_config.tokens_per_chunk // group_config.hashes_per_chunk
# Subtract one so the boundary token itself cannot select the next
# physical block when the boundary lies exactly on a block edge.
chunk_start = (
(boundary_tokens - 1) // group_config.tokens_per_chunk
) * group_config.tokens_per_chunk
first_hash_idx = chunk_start // tokens_per_hash
last_hash_idx = boundary_tokens // tokens_per_hash
assert chunk_start < boundary_tokens
assert boundary_tokens % tokens_per_hash == 0
assert last_hash_idx <= len(req.block_hashes)
# Unlike a complete chunk, a partial tail contains only the hashes
# between its physical block start and its valid token boundary.
maybe_block_hashes = req.block_hashes[first_hash_idx:last_hash_idx]
block_hashes = tuple(
block_hash for block_hash in maybe_block_hashes if block_hash is not None
)
assert block_hashes and len(block_hashes) == len(maybe_block_hashes)
parent_block_hash = (
req.block_hashes[first_hash_idx - 1] if first_hash_idx > 0 else None
)
assert first_hash_idx == 0 or parent_block_hash is not None
lora_id = req.lora_request.adapter_id if req.lora_request is not None else None
lora_name = req.lora_request.name if req.lora_request is not None else None
self._pending_event_metadata[offload_key] = _OffloadEventMetadata(
block_hashes=block_hashes,
parent_block_hash=parent_block_hash,
token_ids=tuple(req.all_token_ids[chunk_start:boundary_tokens]),
block_size=tokens_per_hash,
lora_id=lora_id,
lora_name=lora_name,
extra_keys=None,
group_idx=group_config.group_idx,
kv_cache_spec=group_config.kv_event_group_spec,
)
def take_events(self, events: Iterable[OffloadingEvent]) -> Iterable[KVCacheEvent]:
"""Translate raw OffloadingEvents into self-describing KV events.
@@ -280,10 +280,6 @@ class SchedulerOffloadConfig(NamedTuple):
spec.blocks_per_chunk == 1
and len(group_block_sizes) == 1
and has_partial_recurrent_group
and not (
spec.kv_events_config.enable_kv_cache_events
and spec.kv_events_config.self_describing_kv_events
)
and all(
config.sliding_window_size_in_chunks is None
or config.requires_cow_source
@@ -897,10 +893,12 @@ class OffloadingConnectorScheduler:
for boundary in range(max_boundary, complete_boundary, -tokens_per_hash):
boundary_pending = False
boundary_missed = False
boundary_keys = []
for group_config in self.config.kv_group_configs:
key = self._make_boundary_key(
req_status.req, group_config.group_idx, boundary
)
boundary_keys.append(key)
result = self.manager.lookup(key, req_status.req_context)
if result is LookupResult.MISS:
boundary_missed = True
@@ -910,6 +908,12 @@ class OffloadingConnectorScheduler:
pending |= boundary_pending
if not boundary_missed and not boundary_pending:
for group_config, key in zip(
self.config.kv_group_configs, boundary_keys
):
self._events_tracker.record_partial_lookup(
req_status.req, group_config, boundary, key
)
req_status.partial_tail_boundary = boundary
return boundary - local_tokens
@@ -1203,6 +1207,12 @@ class OffloadingConnectorScheduler:
if not store_output.keys_to_store:
continue
for group_config, key in zip(self.config.kv_group_configs, keys):
if key in store_output.keys_to_store:
self._events_tracker.record_partial_store(
req, group_config, boundary, key
)
group_by_key = {key: idx for idx, key in enumerate(keys)}
accepted_groups = [group_by_key[key] for key in store_output.keys_to_store]
group_sizes = [0] * len(self.config.kv_group_configs)