[ModelRunnerV2] Fix scalar Mamba state update with int32 mappings (#50327)

Signed-off-by: Varun Shenoy <varun.vinayak.shenoy@oracle.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Varun Vinayak Shenoy
2026-08-03 22:05:12 +00:00
committed by GitHub
co-authored by Nick Hill
parent f43e1d26e3
commit e578de311c
2 changed files with 58 additions and 16 deletions
@@ -0,0 +1,29 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.platforms import current_platform
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
@pytest.mark.parametrize(("num_sampled", "expected_value"), [(0, 1), (3, 3)])
def test_postprocess_state_scalar_with_int32_mapping(
num_sampled: int, expected_value: int
) -> None:
state = object.__new__(MambaHybridModelState)
state.num_accepted_tokens_gpu = torch.full(
(4,), 9, dtype=torch.int32, device="cuda"
)
state._align_mode = False
state._mamba_ctx = None
idx_mapping = torch.tensor([2, -1, 0], dtype=torch.int32, device="cuda")
state.postprocess_state(idx_mapping, num_sampled)
expected = torch.tensor(
[expected_value, 9, expected_value, 9], dtype=torch.int32, device="cuda"
)
torch.testing.assert_close(state.num_accepted_tokens_gpu, expected)
+29 -16
View File
@@ -294,18 +294,20 @@ class MambaHybridModelState(DefaultModelState):
) -> None:
# Chunked prefill does not sample a token, so num_sampled can be 0.
# Mamba treats num_accepted_tokens=1 as the neutral non-spec value.
num_reqs = idx_mapping.shape[0]
if not num_reqs:
return
if not isinstance(num_sampled, int):
# idx_mapping may contain -1 sentinels (filtered rows) under PP; the
# kernel skips them rather than scattering with a host-side gather.
n = idx_mapping.shape[0]
if n:
_scatter_num_accepted_kernel[(n,)](
idx_mapping, num_sampled, self.num_accepted_tokens_gpu
)
_scatter_num_accepted_kernel[(num_reqs,)](
idx_mapping, num_sampled, self.num_accepted_tokens_gpu
)
else:
# Fill with single value.
self.num_accepted_tokens_gpu.index_fill_(
0, idx_mapping, max(num_sampled, 1)
_fill_num_accepted_kernel[(num_reqs,)](
idx_mapping, self.num_accepted_tokens_gpu, max(num_sampled, 1)
)
# Align: save the running state to the block-aligned position when
@@ -317,15 +319,13 @@ class MambaHybridModelState(DefaultModelState):
and num_computed_tokens is not None
and self._mamba_ctx is not None
):
num_reqs = idx_mapping.shape[0]
if num_reqs:
self._mamba_ctx.run_fused_postprocess_align(
num_reqs,
self.num_accepted_tokens_gpu,
self._mamba_state_idx_gpu,
num_computed_tokens,
idx_mapping,
)
self._mamba_ctx.run_fused_postprocess_align(
num_reqs,
self.num_accepted_tokens_gpu,
self._mamba_state_idx_gpu,
num_computed_tokens,
idx_mapping,
)
@triton.jit
@@ -340,3 +340,16 @@ def _scatter_num_accepted_kernel(
return
num_sampled = tl.load(num_sampled_ptr + row)
tl.store(num_accepted_ptr + req_state_idx, tl.maximum(num_sampled, 1))
@triton.jit
def _fill_num_accepted_kernel(
idx_mapping_ptr, # [num_reqs] batch_idx -> req_state_idx (-1 to skip)
num_accepted_ptr, # [max_num_reqs]
num_sampled,
):
row = tl.program_id(0)
req_state_idx = tl.load(idx_mapping_ptr + row)
if req_state_idx < 0:
return
tl.store(num_accepted_ptr + req_state_idx, num_sampled)