diff --git a/tests/v1/worker/test_mamba_hybrid_model_state.py b/tests/v1/worker/test_mamba_hybrid_model_state.py new file mode 100644 index 0000000000..545d23f909 --- /dev/null +++ b/tests/v1/worker/test_mamba_hybrid_model_state.py @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch + +from vllm.platforms import current_platform +from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState + + +@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA") +@pytest.mark.parametrize(("num_sampled", "expected_value"), [(0, 1), (3, 3)]) +def test_postprocess_state_scalar_with_int32_mapping( + num_sampled: int, expected_value: int +) -> None: + state = object.__new__(MambaHybridModelState) + state.num_accepted_tokens_gpu = torch.full( + (4,), 9, dtype=torch.int32, device="cuda" + ) + state._align_mode = False + state._mamba_ctx = None + idx_mapping = torch.tensor([2, -1, 0], dtype=torch.int32, device="cuda") + + state.postprocess_state(idx_mapping, num_sampled) + + expected = torch.tensor( + [expected_value, 9, expected_value, 9], dtype=torch.int32, device="cuda" + ) + torch.testing.assert_close(state.num_accepted_tokens_gpu, expected) diff --git a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py index d7487afaaa..2715f790dc 100644 --- a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py +++ b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py @@ -294,18 +294,20 @@ class MambaHybridModelState(DefaultModelState): ) -> None: # Chunked prefill does not sample a token, so num_sampled can be 0. # Mamba treats num_accepted_tokens=1 as the neutral non-spec value. + num_reqs = idx_mapping.shape[0] + if not num_reqs: + return + if not isinstance(num_sampled, int): # idx_mapping may contain -1 sentinels (filtered rows) under PP; the # kernel skips them rather than scattering with a host-side gather. - n = idx_mapping.shape[0] - if n: - _scatter_num_accepted_kernel[(n,)]( - idx_mapping, num_sampled, self.num_accepted_tokens_gpu - ) + _scatter_num_accepted_kernel[(num_reqs,)]( + idx_mapping, num_sampled, self.num_accepted_tokens_gpu + ) else: # Fill with single value. - self.num_accepted_tokens_gpu.index_fill_( - 0, idx_mapping, max(num_sampled, 1) + _fill_num_accepted_kernel[(num_reqs,)]( + idx_mapping, self.num_accepted_tokens_gpu, max(num_sampled, 1) ) # Align: save the running state to the block-aligned position when @@ -317,15 +319,13 @@ class MambaHybridModelState(DefaultModelState): and num_computed_tokens is not None and self._mamba_ctx is not None ): - num_reqs = idx_mapping.shape[0] - if num_reqs: - self._mamba_ctx.run_fused_postprocess_align( - num_reqs, - self.num_accepted_tokens_gpu, - self._mamba_state_idx_gpu, - num_computed_tokens, - idx_mapping, - ) + self._mamba_ctx.run_fused_postprocess_align( + num_reqs, + self.num_accepted_tokens_gpu, + self._mamba_state_idx_gpu, + num_computed_tokens, + idx_mapping, + ) @triton.jit @@ -340,3 +340,16 @@ def _scatter_num_accepted_kernel( return num_sampled = tl.load(num_sampled_ptr + row) tl.store(num_accepted_ptr + req_state_idx, tl.maximum(num_sampled, 1)) + + +@triton.jit +def _fill_num_accepted_kernel( + idx_mapping_ptr, # [num_reqs] batch_idx -> req_state_idx (-1 to skip) + num_accepted_ptr, # [max_num_reqs] + num_sampled, +): + row = tl.program_id(0) + req_state_idx = tl.load(idx_mapping_ptr + row) + if req_state_idx < 0: + return + tl.store(num_accepted_ptr + req_state_idx, num_sampled)