[Bugfix] Seed hybrid mamba state index with mamba_block_size on prefix-cache hits (#55601)

Signed-off-by: pondzikk <pondzik@gmail.com>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: ptorsten <1652174+ptorsten@users.noreply.github.com>
This commit is contained in:
pondzikk
2026-10-07 17:09:50 +00:00
committed by GitHub
co-authored by Claude Fable 5.1 zjy0516 ptorsten
parent a8260cbc5c
commit 3d6853b32c
2 changed files with 22 additions and 1 deletions
@@ -67,6 +67,22 @@ def test_prepare_attn_forwards_positions(monkeypatch: pytest.MonkeyPatch) -> Non
assert build_attn_metadata.call_args.kwargs["positions"] is positions
def test_add_request_seeds_state_idx_in_mamba_blocks() -> None:
"""A drafter group can lower cache_config.block_size below the mamba
block size; the seeded column must still be in mamba blocks."""
state = object.__new__(MambaHybridModelState)
state.cache_config = SimpleNamespace(block_size=16, mamba_block_size=880)
state._align_mode = True
state.rope_state = None
state.prompt_embeds_state = None
state.num_accepted_tokens_gpu = torch.ones(2, dtype=torch.int32)
state._mamba_state_idx_gpu = torch.zeros(2, dtype=torch.int32)
state.add_request(1, SimpleNamespace(num_computed_tokens=107_360))
assert state._mamba_state_idx_gpu[1] == 121
def test_padded_prompt_tail_builds_as_spec_decode(
monkeypatch: pytest.MonkeyPatch,
) -> None:
@@ -115,8 +115,13 @@ class MambaHybridModelState(DefaultModelState):
self.num_accepted_tokens_gpu[req_index].fill_(1)
if self._align_mode:
# Seed the running state block from the resumed/prefilled position.
# The column is in mamba blocks. cache_config.block_size is the
# min over prefix-cacheable groups, so a drafter group with its own
# smaller block (e.g. DFlash) pulls it below mamba_block_size.
mamba_block_size = self.cache_config.mamba_block_size
assert mamba_block_size is not None
self._mamba_state_idx_gpu[req_index].fill_(
(new_req_data.num_computed_tokens - 1) // self.cache_config.block_size
(new_req_data.num_computed_tokens - 1) // mamba_block_size
)
def _get_mamba_group_info(