[BugFix] Retain both replay boundaries so an EAGLE resend of a block-aligned prompt still hits (#54713)
Signed-off-by: tobymao <toby.mao@gmail.com>
This commit is contained in:
@@ -3641,6 +3641,104 @@ def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary():
|
||||
assert [len(blocks) for blocks in computed_blocks.blocks] == [2, 8]
|
||||
|
||||
|
||||
def test_hybrid_mamba_retention_mtp_resend_of_aligned_prompt():
|
||||
"""An identical resend and a longer sibling resume at DIFFERENT positions.
|
||||
|
||||
How far a lookup matches depends on who is asking. A resend of the same
|
||||
prompt caps its lookup at ``num_tokens - 1`` (the last token is recomputed
|
||||
for logits), while a sibling whose prompt merely starts with this one caps
|
||||
above the prompt. The two coincide unless the prompt length is an exact
|
||||
multiple of the alignment -- there they differ by one alignment unit, and
|
||||
under the EAGLE drop BOTH are reachable.
|
||||
|
||||
Retaining only the higher one leaves the resend with every retained state
|
||||
above every candidate its lookup can produce, and the reconciled hit
|
||||
collapses to 0 -- the same zero-hit failure sparse retention already fixes
|
||||
at unaligned prompt lengths.
|
||||
"""
|
||||
block_size = 32
|
||||
num_spec = 3
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=100,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float16,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba_mtp"],
|
||||
MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
num_speculative_blocks=num_spec,
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
retention_interval=0,
|
||||
use_eagle=True,
|
||||
)
|
||||
|
||||
# 128 tokens, an exact multiple of the 32-token alignment. A longer sibling
|
||||
# matches 128 and drops to 96; this prompt's own resend caps at 127, matches
|
||||
# 96 and drops to 64. Both states must survive retention.
|
||||
token_ids = [i for i in range(4) for _ in range(block_size)]
|
||||
req0 = make_request("0", token_ids, block_size, sha256)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
|
||||
assert num_computed_tokens == 0
|
||||
# Prefill in block-aligned chunks the way the align-mode scheduler does: a
|
||||
# state only materializes as a chunk's running-state block, so a
|
||||
# single-shot prefill could not retain the lower one.
|
||||
for chunk_end in (32, 64, 96, 128):
|
||||
blocks = manager.allocate_slots(
|
||||
req0,
|
||||
chunk_end - req0.num_computed_tokens,
|
||||
num_computed_tokens,
|
||||
computed_blocks,
|
||||
num_lookahead_tokens=num_spec,
|
||||
)
|
||||
assert blocks is not None
|
||||
req0.num_computed_tokens = chunk_end
|
||||
|
||||
# Block ``i`` ends at token ``(i + 1) * 32``, so positions 64 and 96 are
|
||||
# mamba blocks 1 and 2.
|
||||
pool = manager.block_pool
|
||||
expected_mamba_cached = {1, 2}
|
||||
for i in range(4):
|
||||
cached = pool.get_cached_block(req0.block_hashes[i], kv_cache_group_ids=[1])
|
||||
if i in expected_mamba_cached:
|
||||
assert cached is not None, f"mamba hash {i} should be cached"
|
||||
else:
|
||||
assert cached is None, f"mamba hash {i} should not be cached"
|
||||
manager.free(req0)
|
||||
|
||||
# The identical resend: full attention matches blocks 0-2 (96 tokens, capped
|
||||
# by num_tokens - 1) and the EAGLE drop caps the candidate at 64. Without
|
||||
# the lower state retained the reconciled hit would be 0.
|
||||
req1 = make_request("1", token_ids, block_size, sha256)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
|
||||
assert num_computed_tokens == 2 * block_size
|
||||
assert [len(blocks) for blocks in computed_blocks.blocks] == [2, 2]
|
||||
|
||||
# The longer sibling resumes one alignment unit higher, off the same prompt.
|
||||
longer = make_request("2", token_ids + [9] * block_size, block_size, sha256)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(longer)
|
||||
assert num_computed_tokens == 3 * block_size
|
||||
|
||||
|
||||
def test_block_lookup_cache_single_block_per_key():
|
||||
cache = BlockHashToBlockMap()
|
||||
key0 = BlockHashWithGroupId(b"hash0")
|
||||
|
||||
@@ -192,7 +192,7 @@ def test_circular_buffer_allocates_one_block_for_the_request_lifetime():
|
||||
)
|
||||
assert manager.allocate_new_blocks(request_id, num_tokens, num_tokens) == []
|
||||
|
||||
manager.cache_blocks(request_id, 1024, replay_boundary=0)
|
||||
manager.cache_blocks(request_id, 1024, replay_boundaries=(0,))
|
||||
manager.remove_skipped_blocks(request_id, 1024)
|
||||
assert manager.get_num_common_prefix_blocks(request_id) == 0
|
||||
assert manager.get_num_skipped_tokens(1024) == 0
|
||||
|
||||
Reference in New Issue
Block a user