[Bugfix][KV Offload] Do not let a recurrent group's unhashed block truncate the load boundary (#52807)

Signed-off-by: Yifan Jiang <19356972+yifjiang@users.noreply.github.com>
This commit is contained in:
Yifan Jiang
2026-09-03 19:08:46 +03:00
committed by GitHub
parent 4ae6228284
commit da8ec28268
2 changed files with 81 additions and 15 deletions
@@ -1025,18 +1025,23 @@ class OffloadingConnectorScheduler:
num_gpu_blocks = cdiv(num_cached_tokens, tokens_per_block)
assert len(group_blocks) >= num_gpu_blocks
num_locally_computed_gpu_blocks = num_gpu_blocks
# Skip null placeholder blocks (used for sliding window or mamba padding).
for i, block in enumerate(group_blocks[:num_gpu_blocks]):
# ``load_start_gpu_block_idx``: the index in ``group_blocks`` where the
# load region begins -- a slice bound, not a count of computed blocks.
# Scan from the computed boundary, not 0: sparse groups (Mamba,
# SWA) legitimately hold non-null unhashed blocks below it.
# Skip nulls (sentinel / out-of-retention padding).
first_fresh_gpu_block_idx = cdiv(
num_locally_computed_tokens, tokens_per_block
)
load_start_gpu_block_idx = num_gpu_blocks
for i in range(first_fresh_gpu_block_idx, num_gpu_blocks):
block = group_blocks[i]
if not block.is_null and block.block_hash is None:
num_locally_computed_gpu_blocks = i
load_start_gpu_block_idx = i
break
assert (
num_locally_computed_tokens
<= num_locally_computed_gpu_blocks * tokens_per_block
)
num_pending_gpu_blocks = num_gpu_blocks - num_locally_computed_gpu_blocks
assert num_locally_computed_tokens % tokens_per_block == 0
num_pending_gpu_blocks = num_gpu_blocks - load_start_gpu_block_idx
if group_config.sliding_window_size_in_chunks is not None:
assert (
@@ -1049,7 +1054,7 @@ class OffloadingConnectorScheduler:
num_chunks = cdiv(num_cached_tokens, tokens_per_chunk)
if num_pending_gpu_blocks:
start_chunk_idx = (
num_locally_computed_gpu_blocks // self.config.blocks_per_chunk
load_start_gpu_block_idx // self.config.blocks_per_chunk
)
end_chunk_idx = num_chunks - (partial_tail_boundary is not None)
assert len(offload_keys) >= end_chunk_idx
@@ -1063,12 +1068,10 @@ class OffloadingConnectorScheduler:
dst_block_ids.extend(
block.block_id
for block in group_blocks[
num_locally_computed_gpu_blocks:num_gpu_blocks
]
for block in group_blocks[load_start_gpu_block_idx:num_gpu_blocks]
)
group_sizes.append(num_pending_gpu_blocks)
block_indices.append(num_locally_computed_gpu_blocks)
block_indices.append(load_start_gpu_block_idx)
# Skip prefix-hit chunks for block-level policy; for
# request-level, next_stored_chunk_idx stays at 0 so all