[Bugfix][Mamba] Keep the prompt-end prefill checkpoint under sparse retention (#59146)

Signed-off-by: Jared Wen <jaredwen@inferact.ai>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
Jared Wen
2026-09-29 21:00:16 +00:00
committed by GitHub
co-authored by Claude Opus 5.5 Jiangyun Zhu
parent 1d331f8341
commit d882bddbea
2 changed files with 46 additions and 0 deletions
@@ -2531,3 +2531,37 @@ def test_boundary_states_offered_past_prompt_for_resumed_prefill():
offered = [b for _, _, b in drain_boundary_state_offloads(manager).get("0", [])]
assert 2 * block_size in offered
assert req0.num_prompt_tokens < 2 * block_size
def test_prompt_end_checkpoint_survives_sparse_retention_with_coarse_hash():
"""With hash_block_size == mamba block size (no fine-grained matching), the
hybrid coordinator caches the final chunk only up to its last full block,
so the managers see ``num_tokens < num_prompt_tokens`` there too. The
prompt-end checkpoint must still be published under retention_interval=0,
otherwise the next request that extends this prompt misses the prefix.
"""
block_size = 32
manager = make_full_mamba_manager(
dcp_world_size=1,
hash_block_size=block_size,
full_block_size=block_size,
mamba_block_size=block_size,
num_prefill_checkpoint_blocks=1,
)
manager.coordinator.retention_interval = 0
# 240 tokens: not block aligned, prompt-end checkpoint at 224.
request = make_request("producer", list(range(240)), block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(request)
assert manager.allocate_slots(request, 128, num_computed, computed_blocks)
request.num_computed_tokens = 128
manager.new_step_starts()
assert manager.allocate_slots(request, 112) is not None
end_hash = request.block_hashes[224 // block_size - 1]
assert manager.block_pool.get_cached_block(end_hash, [1]) is not None
# A request extending the prompt resumes from that checkpoint.
follower = make_request("follower", list(range(240)) + [7] * 40, block_size, sha256)
_, num_hit, _ = manager.get_computed_blocks(follower)
assert num_hit == 224
@@ -2042,9 +2042,21 @@ class MambaManager(SingleTypeKVCacheManager):
blocks = self.req_to_blocks[request.request_id]
assert 0 <= checkpoint_idx < len(blocks)
checkpoint_block = blocks[checkpoint_idx]
# The prompt's final chunk caches only up to its last full block,
# so ``num_tokens < num_prompt_tokens`` holds there too; its
# checkpoint is the prompt's reusable boundary, not a transient one.
is_prompt_checkpoint = (
checkpoint_position
== get_mamba_prefill_checkpoint_position(
request.num_prompt_tokens,
hash_block_size,
self.drop_eagle_checkpoint_block,
)
)
if (
retention_interval == 0
and num_tokens < request.num_prompt_tokens
and not is_prompt_checkpoint
and checkpoint_position != request.shared_prefix_boundary
):
# retention_interval == 0 keeps this transient checkpoint