[ModelRunnerV2] Fix scalar Mamba state update with int32 mappings (#50327)
Signed-off-by: Varun Shenoy <varun.vinayak.shenoy@oracle.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Nick Hill
parent
f43e1d26e3
commit
e578de311c
@@ -0,0 +1,29 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
|
||||
@pytest.mark.parametrize(("num_sampled", "expected_value"), [(0, 1), (3, 3)])
|
||||
def test_postprocess_state_scalar_with_int32_mapping(
|
||||
num_sampled: int, expected_value: int
|
||||
) -> None:
|
||||
state = object.__new__(MambaHybridModelState)
|
||||
state.num_accepted_tokens_gpu = torch.full(
|
||||
(4,), 9, dtype=torch.int32, device="cuda"
|
||||
)
|
||||
state._align_mode = False
|
||||
state._mamba_ctx = None
|
||||
idx_mapping = torch.tensor([2, -1, 0], dtype=torch.int32, device="cuda")
|
||||
|
||||
state.postprocess_state(idx_mapping, num_sampled)
|
||||
|
||||
expected = torch.tensor(
|
||||
[expected_value, 9, expected_value, 9], dtype=torch.int32, device="cuda"
|
||||
)
|
||||
torch.testing.assert_close(state.num_accepted_tokens_gpu, expected)
|
||||
@@ -294,18 +294,20 @@ class MambaHybridModelState(DefaultModelState):
|
||||
) -> None:
|
||||
# Chunked prefill does not sample a token, so num_sampled can be 0.
|
||||
# Mamba treats num_accepted_tokens=1 as the neutral non-spec value.
|
||||
num_reqs = idx_mapping.shape[0]
|
||||
if not num_reqs:
|
||||
return
|
||||
|
||||
if not isinstance(num_sampled, int):
|
||||
# idx_mapping may contain -1 sentinels (filtered rows) under PP; the
|
||||
# kernel skips them rather than scattering with a host-side gather.
|
||||
n = idx_mapping.shape[0]
|
||||
if n:
|
||||
_scatter_num_accepted_kernel[(n,)](
|
||||
idx_mapping, num_sampled, self.num_accepted_tokens_gpu
|
||||
)
|
||||
_scatter_num_accepted_kernel[(num_reqs,)](
|
||||
idx_mapping, num_sampled, self.num_accepted_tokens_gpu
|
||||
)
|
||||
else:
|
||||
# Fill with single value.
|
||||
self.num_accepted_tokens_gpu.index_fill_(
|
||||
0, idx_mapping, max(num_sampled, 1)
|
||||
_fill_num_accepted_kernel[(num_reqs,)](
|
||||
idx_mapping, self.num_accepted_tokens_gpu, max(num_sampled, 1)
|
||||
)
|
||||
|
||||
# Align: save the running state to the block-aligned position when
|
||||
@@ -317,15 +319,13 @@ class MambaHybridModelState(DefaultModelState):
|
||||
and num_computed_tokens is not None
|
||||
and self._mamba_ctx is not None
|
||||
):
|
||||
num_reqs = idx_mapping.shape[0]
|
||||
if num_reqs:
|
||||
self._mamba_ctx.run_fused_postprocess_align(
|
||||
num_reqs,
|
||||
self.num_accepted_tokens_gpu,
|
||||
self._mamba_state_idx_gpu,
|
||||
num_computed_tokens,
|
||||
idx_mapping,
|
||||
)
|
||||
self._mamba_ctx.run_fused_postprocess_align(
|
||||
num_reqs,
|
||||
self.num_accepted_tokens_gpu,
|
||||
self._mamba_state_idx_gpu,
|
||||
num_computed_tokens,
|
||||
idx_mapping,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -340,3 +340,16 @@ def _scatter_num_accepted_kernel(
|
||||
return
|
||||
num_sampled = tl.load(num_sampled_ptr + row)
|
||||
tl.store(num_accepted_ptr + req_state_idx, tl.maximum(num_sampled, 1))
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fill_num_accepted_kernel(
|
||||
idx_mapping_ptr, # [num_reqs] batch_idx -> req_state_idx (-1 to skip)
|
||||
num_accepted_ptr, # [max_num_reqs]
|
||||
num_sampled,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
req_state_idx = tl.load(idx_mapping_ptr + row)
|
||||
if req_state_idx < 0:
|
||||
return
|
||||
tl.store(num_accepted_ptr + req_state_idx, num_sampled)
|
||||
|
||||
Reference in New Issue
Block a user