[Bugfix][KV offload] Store interior chunk-boundary blocks under MTP/Eagle (#46972)

Signed-off-by: Mikhail Kostryukov <mike@triptrack.net>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
drakosha
2026-07-07 13:16:52 +03:00
committed by GitHub
co-authored by Claude Or Ozeri
parent cbb5f045be
commit 3354dba381
2 changed files with 131 additions and 27 deletions
@@ -298,12 +298,39 @@ class RequestOffloadState:
for group_state, new_blocks in zip(self.group_states, new_block_id_groups):
group_state.block_ids.extend(new_blocks)
def storable_blocks(
self, group_config: "GroupOffloadConfig", num_offloadable_tokens: int
) -> int:
"""Number of leading offloaded blocks eligible for store.
For eagle/MTP groups the volatile trailing block of the offloadable
range is excluded while decoding: the draft-layer KV of the last
accepted position may be rewritten after spec-token rejection. During
prefill the trailing block is stable (the draft input for a chunk's
last position is the next prompt token), so it is stored immediately.
The exclusion must be applied consistently everywhere
``next_stored_block_idx`` is derived: otherwise the trailing block of
each step is skipped on collection but jumped over by
``next_stored_block_idx``, so it is never re-considered and a
permanent hole breaks prefix-reuse lookup.
"""
num_blocks = num_offloadable_tokens // group_config.offloaded_block_size
is_decoding = num_offloadable_tokens > self.req.num_prompt_tokens
if group_config.is_eagle_group and is_decoding:
num_blocks = max(0, num_blocks - 1)
return num_blocks
def advance_stored_idx(self, num_offloadable_tokens: int) -> None:
# max(): at the prefill->decode transition of a block-aligned prompt,
# storable_blocks drops by one (the eagle exclusion kicks in), and the
# index must not move backwards past already-stored blocks.
for group_config, group_state in zip(
self.config.kv_group_configs, self.group_states
):
num_blocks = num_offloadable_tokens // group_config.offloaded_block_size
group_state.next_stored_block_idx = num_blocks
group_state.next_stored_block_idx = max(
group_state.next_stored_block_idx,
self.storable_blocks(group_config, num_offloadable_tokens),
)
def update_num_hit_blocks(self, num_cached_tokens: int) -> None:
for group_config, group_state in zip(
@@ -903,9 +930,9 @@ class OffloadingConnectorScheduler:
for group_config, group_state in zip(
self.config.kv_group_configs, req_status.group_states
):
num_blocks = num_offloadable_tokens // group_config.offloaded_block_size
if group_config.is_eagle_group:
num_blocks = max(0, num_blocks - 1)
num_blocks = req_status.storable_blocks(
group_config, num_offloadable_tokens
)
start_block_idx = group_state.next_stored_block_idx
if num_blocks <= start_block_idx:
@@ -977,7 +1004,9 @@ class OffloadingConnectorScheduler:
is_sliding_window = (
group_config.sliding_window_size_in_blocks is not None
)
num_blocks = num_offloadable_tokens // group_config.offloaded_block_size
num_blocks = req_status.storable_blocks(
group_config, num_offloadable_tokens
)
start_block_idx = group_state.next_stored_block_idx
block_ids = group_state.block_ids
num_group_blocks = 0
@@ -1010,7 +1039,9 @@ class OffloadingConnectorScheduler:
group_sizes.append(num_group_blocks)
block_indices.append(start_gpu_block_idx or 0)
group_state.next_stored_block_idx = num_blocks
group_state.next_stored_block_idx = max(
group_state.next_stored_block_idx, num_blocks
)
src_spec = GPULoadStoreSpec(
src_block_ids, group_sizes=group_sizes, block_indices=block_indices