[KV Offload] Emit self-describing events for partial recurrent blocks (#51243)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -145,6 +145,85 @@ class OffloadingEventsTracker:
|
||||
req, group_config, chunk_idx
|
||||
)
|
||||
|
||||
def record_partial_store(
|
||||
self,
|
||||
req: Request,
|
||||
group_config: "GroupOffloadConfig",
|
||||
boundary_tokens: int,
|
||||
offload_key: OffloadKey,
|
||||
) -> None:
|
||||
"""Snapshot metadata for a newly stored partial recurrent tail."""
|
||||
if group_config.sliding_window_size_in_chunks is not None:
|
||||
return
|
||||
self._record_partial_tail(req, group_config, boundary_tokens, offload_key)
|
||||
|
||||
def record_partial_lookup(
|
||||
self,
|
||||
req: Request,
|
||||
group_config: "GroupOffloadConfig",
|
||||
boundary_tokens: int,
|
||||
offload_key: OffloadKey,
|
||||
) -> None:
|
||||
"""Backfill metadata for a partial recurrent tail lookup hit."""
|
||||
if group_config.sliding_window_size_in_chunks is not None:
|
||||
return
|
||||
if offload_key not in self._pending_event_metadata:
|
||||
self._record_partial_tail(req, group_config, boundary_tokens, offload_key)
|
||||
|
||||
def _record_partial_tail(
|
||||
self,
|
||||
req: Request,
|
||||
group_config: "GroupOffloadConfig",
|
||||
boundary_tokens: int,
|
||||
offload_key: OffloadKey,
|
||||
) -> None:
|
||||
"""Build metadata for the valid prefix of one physical cache block.
|
||||
|
||||
A partial recurrent tail ends on a hash boundary but before its
|
||||
physical cache block is full. The event describes only the valid
|
||||
hashes and tokens, not the unused remainder of that physical block.
|
||||
"""
|
||||
if not self.self_describing_enabled:
|
||||
return
|
||||
|
||||
tokens_per_hash = group_config.tokens_per_chunk // group_config.hashes_per_chunk
|
||||
# Subtract one so the boundary token itself cannot select the next
|
||||
# physical block when the boundary lies exactly on a block edge.
|
||||
chunk_start = (
|
||||
(boundary_tokens - 1) // group_config.tokens_per_chunk
|
||||
) * group_config.tokens_per_chunk
|
||||
first_hash_idx = chunk_start // tokens_per_hash
|
||||
last_hash_idx = boundary_tokens // tokens_per_hash
|
||||
assert chunk_start < boundary_tokens
|
||||
assert boundary_tokens % tokens_per_hash == 0
|
||||
assert last_hash_idx <= len(req.block_hashes)
|
||||
|
||||
# Unlike a complete chunk, a partial tail contains only the hashes
|
||||
# between its physical block start and its valid token boundary.
|
||||
maybe_block_hashes = req.block_hashes[first_hash_idx:last_hash_idx]
|
||||
block_hashes = tuple(
|
||||
block_hash for block_hash in maybe_block_hashes if block_hash is not None
|
||||
)
|
||||
assert block_hashes and len(block_hashes) == len(maybe_block_hashes)
|
||||
parent_block_hash = (
|
||||
req.block_hashes[first_hash_idx - 1] if first_hash_idx > 0 else None
|
||||
)
|
||||
assert first_hash_idx == 0 or parent_block_hash is not None
|
||||
|
||||
lora_id = req.lora_request.adapter_id if req.lora_request is not None else None
|
||||
lora_name = req.lora_request.name if req.lora_request is not None else None
|
||||
self._pending_event_metadata[offload_key] = _OffloadEventMetadata(
|
||||
block_hashes=block_hashes,
|
||||
parent_block_hash=parent_block_hash,
|
||||
token_ids=tuple(req.all_token_ids[chunk_start:boundary_tokens]),
|
||||
block_size=tokens_per_hash,
|
||||
lora_id=lora_id,
|
||||
lora_name=lora_name,
|
||||
extra_keys=None,
|
||||
group_idx=group_config.group_idx,
|
||||
kv_cache_spec=group_config.kv_event_group_spec,
|
||||
)
|
||||
|
||||
def take_events(self, events: Iterable[OffloadingEvent]) -> Iterable[KVCacheEvent]:
|
||||
"""Translate raw OffloadingEvents into self-describing KV events.
|
||||
|
||||
|
||||
@@ -280,10 +280,6 @@ class SchedulerOffloadConfig(NamedTuple):
|
||||
spec.blocks_per_chunk == 1
|
||||
and len(group_block_sizes) == 1
|
||||
and has_partial_recurrent_group
|
||||
and not (
|
||||
spec.kv_events_config.enable_kv_cache_events
|
||||
and spec.kv_events_config.self_describing_kv_events
|
||||
)
|
||||
and all(
|
||||
config.sliding_window_size_in_chunks is None
|
||||
or config.requires_cow_source
|
||||
@@ -897,10 +893,12 @@ class OffloadingConnectorScheduler:
|
||||
for boundary in range(max_boundary, complete_boundary, -tokens_per_hash):
|
||||
boundary_pending = False
|
||||
boundary_missed = False
|
||||
boundary_keys = []
|
||||
for group_config in self.config.kv_group_configs:
|
||||
key = self._make_boundary_key(
|
||||
req_status.req, group_config.group_idx, boundary
|
||||
)
|
||||
boundary_keys.append(key)
|
||||
result = self.manager.lookup(key, req_status.req_context)
|
||||
if result is LookupResult.MISS:
|
||||
boundary_missed = True
|
||||
@@ -910,6 +908,12 @@ class OffloadingConnectorScheduler:
|
||||
|
||||
pending |= boundary_pending
|
||||
if not boundary_missed and not boundary_pending:
|
||||
for group_config, key in zip(
|
||||
self.config.kv_group_configs, boundary_keys
|
||||
):
|
||||
self._events_tracker.record_partial_lookup(
|
||||
req_status.req, group_config, boundary, key
|
||||
)
|
||||
req_status.partial_tail_boundary = boundary
|
||||
return boundary - local_tokens
|
||||
|
||||
@@ -1203,6 +1207,12 @@ class OffloadingConnectorScheduler:
|
||||
if not store_output.keys_to_store:
|
||||
continue
|
||||
|
||||
for group_config, key in zip(self.config.kv_group_configs, keys):
|
||||
if key in store_output.keys_to_store:
|
||||
self._events_tracker.record_partial_store(
|
||||
req, group_config, boundary, key
|
||||
)
|
||||
|
||||
group_by_key = {key: idx for idx, key in enumerate(keys)}
|
||||
accepted_groups = [group_by_key[key] for key in store_output.keys_to_store]
|
||||
group_sizes = [0] * len(self.config.kv_group_configs)
|
||||
|
||||
Reference in New Issue
Block a user