[BugFix] Retain both replay boundaries so an EAGLE resend of a block-aligned prompt still hits (#54713)

Signed-off-by: tobymao <toby.mao@gmail.com>
This commit is contained in:
Toby Mao
2026-09-10 16:39:24 +08:00
committed by GitHub
parent 1768273c13
commit b28c3e1568
4 changed files with 130 additions and 31 deletions
+98
View File
@@ -3641,6 +3641,104 @@ def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary():
assert [len(blocks) for blocks in computed_blocks.blocks] == [2, 8]
def test_hybrid_mamba_retention_mtp_resend_of_aligned_prompt():
"""An identical resend and a longer sibling resume at DIFFERENT positions.
How far a lookup matches depends on who is asking. A resend of the same
prompt caps its lookup at ``num_tokens - 1`` (the last token is recomputed
for logits), while a sibling whose prompt merely starts with this one caps
above the prompt. The two coincide unless the prompt length is an exact
multiple of the alignment -- there they differ by one alignment unit, and
under the EAGLE drop BOTH are reachable.
Retaining only the higher one leaves the resend with every retained state
above every candidate its lookup can produce, and the reconciled hit
collapses to 0 -- the same zero-hit failure sparse retention already fixes
at unaligned prompt lengths.
"""
block_size = 32
num_spec = 3
kv_cache_config = KVCacheConfig(
num_blocks=100,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float16,
),
),
KVCacheGroupSpec(
["mamba_mtp"],
MambaSpec(
block_size=block_size,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
num_speculative_blocks=num_spec,
),
),
],
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
retention_interval=0,
use_eagle=True,
)
# 128 tokens, an exact multiple of the 32-token alignment. A longer sibling
# matches 128 and drops to 96; this prompt's own resend caps at 127, matches
# 96 and drops to 64. Both states must survive retention.
token_ids = [i for i in range(4) for _ in range(block_size)]
req0 = make_request("0", token_ids, block_size, sha256)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
assert num_computed_tokens == 0
# Prefill in block-aligned chunks the way the align-mode scheduler does: a
# state only materializes as a chunk's running-state block, so a
# single-shot prefill could not retain the lower one.
for chunk_end in (32, 64, 96, 128):
blocks = manager.allocate_slots(
req0,
chunk_end - req0.num_computed_tokens,
num_computed_tokens,
computed_blocks,
num_lookahead_tokens=num_spec,
)
assert blocks is not None
req0.num_computed_tokens = chunk_end
# Block ``i`` ends at token ``(i + 1) * 32``, so positions 64 and 96 are
# mamba blocks 1 and 2.
pool = manager.block_pool
expected_mamba_cached = {1, 2}
for i in range(4):
cached = pool.get_cached_block(req0.block_hashes[i], kv_cache_group_ids=[1])
if i in expected_mamba_cached:
assert cached is not None, f"mamba hash {i} should be cached"
else:
assert cached is None, f"mamba hash {i} should not be cached"
manager.free(req0)
# The identical resend: full attention matches blocks 0-2 (96 tokens, capped
# by num_tokens - 1) and the EAGLE drop caps the candidate at 64. Without
# the lower state retained the reconciled hit would be 0.
req1 = make_request("1", token_ids, block_size, sha256)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
assert num_computed_tokens == 2 * block_size
assert [len(blocks) for blocks in computed_blocks.blocks] == [2, 2]
# The longer sibling resumes one alignment unit higher, off the same prompt.
longer = make_request("2", token_ids + [9] * block_size, block_size, sha256)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(longer)
assert num_computed_tokens == 3 * block_size
def test_block_lookup_cache_single_block_per_key():
cache = BlockHashToBlockMap()
key0 = BlockHashWithGroupId(b"hash0")
@@ -192,7 +192,7 @@ def test_circular_buffer_allocates_one_block_for_the_request_lifetime():
)
assert manager.allocate_new_blocks(request_id, num_tokens, num_tokens) == []
manager.cache_blocks(request_id, 1024, replay_boundary=0)
manager.cache_blocks(request_id, 1024, replay_boundaries=(0,))
manager.remove_skipped_blocks(request_id, 1024)
assert manager.get_num_common_prefix_blocks(request_id) == 0
assert manager.get_num_skipped_tokens(1024) == 0