[Bugfix] Seed hybrid mamba state index with mamba_block_size on prefix-cache hits (#55601)
Signed-off-by: pondzikk <pondzik@gmail.com> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: ptorsten <1652174+ptorsten@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
zjy0516
ptorsten
parent
a8260cbc5c
commit
3d6853b32c
@@ -67,6 +67,22 @@ def test_prepare_attn_forwards_positions(monkeypatch: pytest.MonkeyPatch) -> Non
|
||||
assert build_attn_metadata.call_args.kwargs["positions"] is positions
|
||||
|
||||
|
||||
def test_add_request_seeds_state_idx_in_mamba_blocks() -> None:
|
||||
"""A drafter group can lower cache_config.block_size below the mamba
|
||||
block size; the seeded column must still be in mamba blocks."""
|
||||
state = object.__new__(MambaHybridModelState)
|
||||
state.cache_config = SimpleNamespace(block_size=16, mamba_block_size=880)
|
||||
state._align_mode = True
|
||||
state.rope_state = None
|
||||
state.prompt_embeds_state = None
|
||||
state.num_accepted_tokens_gpu = torch.ones(2, dtype=torch.int32)
|
||||
state._mamba_state_idx_gpu = torch.zeros(2, dtype=torch.int32)
|
||||
|
||||
state.add_request(1, SimpleNamespace(num_computed_tokens=107_360))
|
||||
|
||||
assert state._mamba_state_idx_gpu[1] == 121
|
||||
|
||||
|
||||
def test_padded_prompt_tail_builds_as_spec_decode(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
|
||||
@@ -115,8 +115,13 @@ class MambaHybridModelState(DefaultModelState):
|
||||
self.num_accepted_tokens_gpu[req_index].fill_(1)
|
||||
if self._align_mode:
|
||||
# Seed the running state block from the resumed/prefilled position.
|
||||
# The column is in mamba blocks. cache_config.block_size is the
|
||||
# min over prefix-cacheable groups, so a drafter group with its own
|
||||
# smaller block (e.g. DFlash) pulls it below mamba_block_size.
|
||||
mamba_block_size = self.cache_config.mamba_block_size
|
||||
assert mamba_block_size is not None
|
||||
self._mamba_state_idx_gpu[req_index].fill_(
|
||||
(new_req_data.num_computed_tokens - 1) // self.cache_config.block_size
|
||||
(new_req_data.num_computed_tokens - 1) // mamba_block_size
|
||||
)
|
||||
|
||||
def _get_mamba_group_info(
|
||||
|
||||
Reference in New Issue
Block a user