[BugFix] Retain both replay boundaries so an EAGLE resend of a block-aligned prompt still hits (#54713)

Signed-off-by: tobymao <toby.mao@gmail.com>
This commit is contained in:
Toby Mao
2026-09-10 16:39:24 +08:00
committed by GitHub
parent 1768273c13
commit b28c3e1568
4 changed files with 130 additions and 31 deletions
+19 -20
View File
@@ -307,27 +307,26 @@ class KVCacheCoordinator(ABC):
for manager in self.single_type_managers
)
def get_replay_boundary(self, request: Request) -> int:
"""Return the position a later request replaying this prompt resumes at.
def get_replay_boundaries(self, request: Request) -> tuple[int, ...]:
"""Positions a later request replaying this prompt can resume at.
A cache hit is the shortest hit across all groups, so this is a
model-level position: every group has to retain state here, whether or
not it is the group that drops. Groups differ only in how much they
keep around it -- EAGLE groups also keep the block above, which they
A hit is the shortest across all groups, so every group retains state
at each position; EAGLE groups also keep the block above, which they
match and drop back from (see ``reachable_block_mask``).
Under EAGLE that block must exist, so the boundary sits one alignment
unit below the prompt's last aligned position; every group's block size
divides the alignment, so the block above always fits in the prompt.
Two positions are reachable: a resend of the identical prompt is capped
at ``num_tokens - 1`` (its last token is recomputed for logits), a
longer sibling matches the final aligned block. They differ only on a
block-aligned prompt, where retaining just the higher one collapses the
resend's hit to 0. The alignment is the scheduler block size, not the
finer hash granularity, which would over-estimate the reach.
"""
if not self.eagle_group_ids:
return request.num_prompt_tokens - 1
aligned = (
request.num_prompt_tokens
// self.scheduler_block_size
* self.scheduler_block_size
)
return max(aligned - self.scheduler_block_size, 0)
return (request.num_prompt_tokens - 1,)
block = self.scheduler_block_size
resend = (request.num_prompt_tokens - 1) // block * block
extension = request.num_prompt_tokens // block * block
return tuple(sorted({max(resend - block, 0), max(extension - block, 0)}))
def cache_blocks(self, request: Request, num_computed_tokens: int) -> None:
"""
@@ -339,7 +338,7 @@ class KVCacheCoordinator(ABC):
that need to be cached
(including tokens that are already cached).
"""
replay_boundary = self.get_replay_boundary(request)
boundaries = self.get_replay_boundaries(request)
for manager in self.single_type_managers:
# Only cache tokens with finalized KV. The last num_reprefillable_tokens
# tokens can be re-prefilled during multi-module MTP.
@@ -350,7 +349,7 @@ class KVCacheCoordinator(ABC):
request,
num_tokens_to_cache,
retention_interval=self.retention_interval,
replay_boundary=replay_boundary,
replay_boundaries=boundaries,
)
def free(self, request_id: str) -> None:
@@ -775,7 +774,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
def cache_blocks(self, request: Request, num_computed_tokens: int) -> None:
cached_num_computed_tokens = self._align_cacheable(num_computed_tokens)
replay_boundary = self.get_replay_boundary(request)
boundaries = self.get_replay_boundaries(request)
for manager in self.single_type_managers:
num_tokens_to_cache = cached_num_computed_tokens
# EAGLE groups match one block past each aligned boundary and drop
@@ -802,7 +801,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
request,
num_tokens_to_cache,
retention_interval=self.retention_interval,
replay_boundary=replay_boundary,
replay_boundaries=boundaries,
)
def find_longest_cache_hit(
+12 -10
View File
@@ -452,7 +452,7 @@ class SingleTypeKVCacheManager(ABC):
num_tokens: int,
retention_interval: int | None = None,
*,
replay_boundary: int,
replay_boundaries: Sequence[int],
) -> None:
"""
Cache the blocks for the request.
@@ -465,6 +465,8 @@ class SingleTypeKVCacheManager(ABC):
keeps dense checkpointing; ``0`` keeps only the latest replay
boundary; a positive multiple of ``scheduler_block_size`` keeps
a tail once per that-sized segment. Only SWA acts on it.
replay_boundaries: Positions a later request replaying this prompt
can resume at, from ``get_replay_boundaries``.
"""
num_cached_blocks = self.num_cached_block.get(request.request_id, 0)
num_full_blocks = num_tokens // self.block_size
@@ -473,9 +475,9 @@ class SingleTypeKVCacheManager(ABC):
return
# Token boundaries whose reachable tail must be retained under sparse
# retention: the replay boundary (``num_prompt - 1``, capped by
# ``get_computed_blocks``) and any detected shared-prefix junction.
reachable_boundaries = [replay_boundary]
# retention: every position a replaying sibling can resume at (see
# ``get_replay_boundaries``) and any detected shared-prefix junction.
reachable_boundaries = [*replay_boundaries]
if request.shared_prefix_boundary:
reachable_boundaries.append(request.shared_prefix_boundary)
@@ -818,13 +820,13 @@ class FullAttentionManager(SingleTypeKVCacheManager):
num_tokens: int,
retention_interval: int | None = None,
*,
replay_boundary: int,
replay_boundaries: Sequence[int],
) -> None:
super().cache_blocks(
request,
num_tokens,
retention_interval=retention_interval,
replay_boundary=replay_boundary,
replay_boundaries=replay_boundaries,
)
hash_block_size = self.block_pool.hash_block_size
if self.block_size == hash_block_size:
@@ -1228,7 +1230,7 @@ class CircularBufferManager(FullAttentionManager):
num_tokens: int,
retention_interval: int | None = None,
*,
replay_boundary: int,
replay_boundaries: Sequence[int],
) -> None:
return
@@ -1940,14 +1942,14 @@ class MambaManager(SingleTypeKVCacheManager):
num_tokens: int,
retention_interval: int | None = None,
*,
replay_boundary: int,
replay_boundaries: Sequence[int],
) -> None:
num_cached_blocks_before = self.num_cached_block.get(request.request_id, 0)
super().cache_blocks(
request,
num_tokens,
retention_interval=retention_interval,
replay_boundary=replay_boundary,
replay_boundaries=replay_boundaries,
)
num_cached_blocks_after = self.num_cached_block.get(request.request_id, 0)
if self.mamba_cache_mode == "align":
@@ -2090,7 +2092,7 @@ class CrossAttentionManager(SingleTypeKVCacheManager):
num_tokens: int,
retention_interval: int | None = None,
*,
replay_boundary: int,
replay_boundaries: Sequence[int],
) -> None:
# We do not cache blocks for cross-attention to be shared between
# requests, so this method is not relevant.