[Core] Fix ValueError on KV load failure with a hybrid KV cache (#50388)

Signed-off-by: Kebe <mail@kebe7jun.com>
This commit is contained in:
Kebe
2026-09-13 21:03:05 +08:00
committed by GitHub
parent fa1b3b1922
commit c351fd3c64
3 changed files with 380 additions and 4 deletions
+38 -2
View File
@@ -3058,6 +3058,11 @@ class Scheduler(SchedulerInterface):
For observability, it also accumulates the total number of tokens that
will need to be recomputed across all affected requests.
A hybrid model spreads one token span over KV cache groups that disagree
on block size, so a block index maps to a different token position in
each group and there is no single longest valid prefix. Those requests
recompute in full instead.
Args:
requests: The set of requests to scan for invalid blocks.
invalid_block_ids: IDs of invalid blocks.
@@ -3081,18 +3086,49 @@ class Scheduler(SchedulerInterface):
# these requests must be rescheduled, but only the first will recompute
# it. This set tracks blocks already marked for recomputation.
marked_invalid_block_ids: set[int] = set()
# Positions a group holds no KV for -- skipped sliding-window blocks,
# and every Mamba block but the aligned snapshot -- all point at this
# one block, shared by every request. It carries no request's data, so
# it can neither fail for a request nor be evicted on its behalf.
null_block_id = self.kv_cache_manager.block_pool.null_block.block_id
for request in requests:
is_affected = False
marked_invalid_block = False
req_id = request.request_id
# TODO (davidb): add support for hybrid memory allocator
(req_block_ids,) = self.kv_cache_manager.get_block_ids(req_id)
# One block-id list per KV cache group.
req_block_ids_per_group = self.kv_cache_manager.get_block_ids(req_id)
# We iterate only over blocks that may contain externally computed
# tokens
req_num_computed_tokens = (
request.num_computed_tokens - num_scheduled_tokens.get(req_id, 0)
)
if len(req_block_ids_per_group) > 1:
# No single truncation point exists across groups, so the whole
# request is recomputed. Blocks are still marked, so a request
# sharing one of them is rescheduled too.
request_invalid_block_ids = {
block_id
for req_block_ids in req_block_ids_per_group
for block_id in req_block_ids
if block_id != null_block_id and block_id in invalid_block_ids
}
if request_invalid_block_ids:
marked_invalid_block_ids |= request_invalid_block_ids
total_affected_tokens += req_num_computed_tokens
request.num_computed_tokens = 0
if evict_blocks:
for req_block_ids in req_block_ids_per_group:
blocks_to_evict.update(
block_id
for block_id in req_block_ids
if block_id != null_block_id
)
affected_req_ids.add(req_id)
continue
(req_block_ids,) = req_block_ids_per_group
req_num_computed_blocks = (
req_num_computed_tokens + self.block_size - 1
) // self.block_size