[Bugfix][Mamba] Restore prompt-tail prefix-cache hits with MTP (#58368)
Signed-off-by: Netanel Haber <58652339+netanel-haber@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Benjamin Chislett
parent
4f72bd0152
commit
d5051abaf1
@@ -196,6 +196,28 @@ def test_sibling_resumes_below_the_block_grid_when_the_prefix_ends_early():
|
||||
assert hit == resume, f"expected the resume point at {resume}, got {hit}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mtp_draft_group_identified", [False, True])
|
||||
def test_mamba_prefix_cache_drops_hash_block(mtp_draft_group_identified):
|
||||
block_size, hash_block_size = 1024, 64
|
||||
manager = _manager(
|
||||
block_size,
|
||||
hash_block_size,
|
||||
eagle_group=0 if mtp_draft_group_identified else None,
|
||||
)
|
||||
prompt_a = PREFIX[:1600]
|
||||
hit_after_mtp_block_drop = len(prompt_a) - hash_block_size
|
||||
|
||||
_prefill(
|
||||
manager,
|
||||
_stub(manager, block_size, hash_block_size),
|
||||
make_request("A", prompt_a, hash_block_size, sha256),
|
||||
)
|
||||
|
||||
tokens_after_a = [-1] * 500
|
||||
hit = _sibling_hit(manager, len(prompt_a), tokens_after_a, hash_block_size)
|
||||
assert hit == hit_after_mtp_block_drop
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Invariants the two clauses have to keep
|
||||
|
||||
|
||||
@@ -2078,7 +2078,7 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
latest_prompt_hash_boundary = (
|
||||
request.num_prompt_tokens // hash_block_size
|
||||
) * hash_block_size
|
||||
if self.use_eagle:
|
||||
if self.drop_eagle_checkpoint_block:
|
||||
# Eagle groups match one hash unit past the candidate and drop it,
|
||||
# so register the tail one unit lower.
|
||||
latest_prompt_hash_boundary = max(
|
||||
|
||||
Reference in New Issue
Block a user