[Bugfix][KV offload] Store interior chunk-boundary blocks under MTP/Eagle (#46972)
Signed-off-by: Mikhail Kostryukov <mike@triptrack.net> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
Claude
Or Ozeri
parent
cbb5f045be
commit
3354dba381
@@ -298,12 +298,39 @@ class RequestOffloadState:
|
||||
for group_state, new_blocks in zip(self.group_states, new_block_id_groups):
|
||||
group_state.block_ids.extend(new_blocks)
|
||||
|
||||
def storable_blocks(
|
||||
self, group_config: "GroupOffloadConfig", num_offloadable_tokens: int
|
||||
) -> int:
|
||||
"""Number of leading offloaded blocks eligible for store.
|
||||
|
||||
For eagle/MTP groups the volatile trailing block of the offloadable
|
||||
range is excluded while decoding: the draft-layer KV of the last
|
||||
accepted position may be rewritten after spec-token rejection. During
|
||||
prefill the trailing block is stable (the draft input for a chunk's
|
||||
last position is the next prompt token), so it is stored immediately.
|
||||
The exclusion must be applied consistently everywhere
|
||||
``next_stored_block_idx`` is derived: otherwise the trailing block of
|
||||
each step is skipped on collection but jumped over by
|
||||
``next_stored_block_idx``, so it is never re-considered and a
|
||||
permanent hole breaks prefix-reuse lookup.
|
||||
"""
|
||||
num_blocks = num_offloadable_tokens // group_config.offloaded_block_size
|
||||
is_decoding = num_offloadable_tokens > self.req.num_prompt_tokens
|
||||
if group_config.is_eagle_group and is_decoding:
|
||||
num_blocks = max(0, num_blocks - 1)
|
||||
return num_blocks
|
||||
|
||||
def advance_stored_idx(self, num_offloadable_tokens: int) -> None:
|
||||
# max(): at the prefill->decode transition of a block-aligned prompt,
|
||||
# storable_blocks drops by one (the eagle exclusion kicks in), and the
|
||||
# index must not move backwards past already-stored blocks.
|
||||
for group_config, group_state in zip(
|
||||
self.config.kv_group_configs, self.group_states
|
||||
):
|
||||
num_blocks = num_offloadable_tokens // group_config.offloaded_block_size
|
||||
group_state.next_stored_block_idx = num_blocks
|
||||
group_state.next_stored_block_idx = max(
|
||||
group_state.next_stored_block_idx,
|
||||
self.storable_blocks(group_config, num_offloadable_tokens),
|
||||
)
|
||||
|
||||
def update_num_hit_blocks(self, num_cached_tokens: int) -> None:
|
||||
for group_config, group_state in zip(
|
||||
@@ -903,9 +930,9 @@ class OffloadingConnectorScheduler:
|
||||
for group_config, group_state in zip(
|
||||
self.config.kv_group_configs, req_status.group_states
|
||||
):
|
||||
num_blocks = num_offloadable_tokens // group_config.offloaded_block_size
|
||||
if group_config.is_eagle_group:
|
||||
num_blocks = max(0, num_blocks - 1)
|
||||
num_blocks = req_status.storable_blocks(
|
||||
group_config, num_offloadable_tokens
|
||||
)
|
||||
|
||||
start_block_idx = group_state.next_stored_block_idx
|
||||
if num_blocks <= start_block_idx:
|
||||
@@ -977,7 +1004,9 @@ class OffloadingConnectorScheduler:
|
||||
is_sliding_window = (
|
||||
group_config.sliding_window_size_in_blocks is not None
|
||||
)
|
||||
num_blocks = num_offloadable_tokens // group_config.offloaded_block_size
|
||||
num_blocks = req_status.storable_blocks(
|
||||
group_config, num_offloadable_tokens
|
||||
)
|
||||
start_block_idx = group_state.next_stored_block_idx
|
||||
block_ids = group_state.block_ids
|
||||
num_group_blocks = 0
|
||||
@@ -1010,7 +1039,9 @@ class OffloadingConnectorScheduler:
|
||||
|
||||
group_sizes.append(num_group_blocks)
|
||||
block_indices.append(start_gpu_block_idx or 0)
|
||||
group_state.next_stored_block_idx = num_blocks
|
||||
group_state.next_stored_block_idx = max(
|
||||
group_state.next_stored_block_idx, num_blocks
|
||||
)
|
||||
|
||||
src_spec = GPULoadStoreSpec(
|
||||
src_block_ids, group_sizes=group_sizes, block_indices=block_indices
|
||||
|
||||
Reference in New Issue
Block a user