[Kernel][Test] Make kernel tests for mamba dual-HW (CUDA + XPU) (#42736)

Signed-off-by: Dobrzyniewicz, Agata <agata.dobrzyniewicz@intel.com>
This commit is contained in:
Agata Dobrzyniewicz
2026-06-08 08:22:47 +08:00
committed by GitHub
parent 4dcd10eb0d
commit 2ed0a9627b
3 changed files with 49 additions and 14 deletions
+12 -4
View File
@@ -11,9 +11,17 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
causal_conv1d_fn,
causal_conv1d_update,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
DEVICE = current_platform.device_type
pytestmark = pytest.mark.skipif(
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
reason="causal_conv1d Triton kernels require CUDA-alike or XPU",
)
def causal_conv1d_ref(
x: torch.Tensor,
@@ -149,7 +157,7 @@ def causal_conv1d_opcheck_fn(
@pytest.mark.parametrize("width", [4])
@pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096])
def test_causal_conv1d_update(dim, width, seqlen, has_bias, silu_activation, itype):
device = "cuda"
device = DEVICE
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3)
if itype == torch.bfloat16:
rtol, atol = 1e-2, 5e-2
@@ -196,7 +204,7 @@ def test_causal_conv1d_update(dim, width, seqlen, has_bias, silu_activation, ity
def test_causal_conv1d_update_with_batch_gather(
batch_size, with_padding, dim, width, seqlen, has_bias, silu_activation, itype
):
device = "cuda"
device = DEVICE
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3)
if itype == torch.bfloat16:
rtol, atol = 1e-2, 5e-2
@@ -275,7 +283,7 @@ def test_causal_conv1d_update_with_batch_gather(
def test_causal_conv1d_varlen(
batch, with_padding, dim, seqlen, width, has_bias, silu_activation, itype
):
device = "cuda"
device = DEVICE
torch.accelerator.empty_cache()
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3)
if itype == torch.bfloat16:
@@ -341,7 +349,7 @@ def test_causal_conv1d_varlen(
weight,
bias=bias,
conv_states=final_states,
query_start_loc=cumsum.cuda(),
query_start_loc=cumsum.to(device),
cache_indices=padded_state_indices,
has_initial_state=has_initial_states,
activation=activation,
+24 -7
View File
@@ -17,6 +17,21 @@ from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
DEVICE = current_platform.device_type
pytestmark = pytest.mark.skipif(
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
reason="mamba_ssm kernels require CUDA-alike or XPU",
)
# selective_scan_fn is backed by the CUDA-only `ops.selective_scan_fwd` C++ op,
# so tests exercising it must be skipped on XPU. selective_state_update is
# pure Triton and runs on both CUDA-alike and XPU.
skip_unless_cuda_alike = pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="selective_scan_fn uses CUDA-only custom op",
)
def selective_scan_ref(
u,
@@ -181,6 +196,7 @@ def selective_scan_opcheck_fn(
@pytest.mark.parametrize("is_variable_C", [True])
@pytest.mark.parametrize("is_variable_B", [True])
@pytest.mark.parametrize("scan_chunks", [1, 3])
@skip_unless_cuda_alike
def test_selective_scan(
is_variable_B,
is_variable_C,
@@ -327,7 +343,7 @@ def test_selective_scan(
@pytest.mark.parametrize("dstate", [16, 64])
@pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096])
def test_selective_state_update(dim, dstate, has_z, itype):
device = "cuda"
device = DEVICE
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2)
if itype == torch.bfloat16:
rtol, atol = 1e-2, 5e-2
@@ -370,7 +386,7 @@ def test_selective_state_update(dim, dstate, has_z, itype):
" on compute capability 10.0 CUDA devices.",
)
def test_selective_state_update_stochastic_rounding(dim, dstate, has_z, philox_rounds):
device = "cuda"
device = DEVICE
rtol, atol = 5e-3, 1e-1
# set seed
set_random_seed(0)
@@ -417,7 +433,7 @@ def test_selective_state_update_stochastic_rounding(dim, dstate, has_z, philox_r
@pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096])
@pytest.mark.parametrize("max_seq_len", [1, 2, 4])
def test_selective_state_update_varlen(dim, dstate, has_z, itype, max_seq_len):
device = "cuda"
device = DEVICE
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2)
if itype == torch.bfloat16:
rtol, atol = 5e-2, 1.5e-1
@@ -498,6 +514,7 @@ def test_selective_state_update_varlen(dim, dstate, has_z, itype, max_seq_len):
@pytest.mark.parametrize("is_variable_B", [True])
# tests correctness in case subset of the sequences are padded
@pytest.mark.parametrize("with_padding", [False, True])
@skip_unless_cuda_alike
def test_selective_scan_varlen(
with_padding,
is_variable_B,
@@ -679,7 +696,7 @@ def test_selective_scan_varlen(
def test_selective_state_update_with_batch_indices(
with_padding, dim, dstate, has_z, itype
):
device = "cuda"
device = DEVICE
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2)
if itype == torch.bfloat16:
rtol, atol = 1e-1, 1e-1
@@ -771,7 +788,7 @@ def test_selective_state_update_with_batch_indices(
def test_selective_state_update_with_heads_with_batch_indices(
dim, dstate, ngroups, has_z, tie_hdim, itype
):
device = "cuda"
device = DEVICE
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 3e-2)
if itype == torch.bfloat16:
rtol, atol = 1e-1, 1e-1
@@ -844,7 +861,7 @@ def test_selective_state_update_with_heads_with_batch_indices(
def test_selective_state_update_with_num_accepted_tokens(
dim, dstate, has_z, itype, max_seq_len
):
device = "cuda"
device = DEVICE
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2)
if itype == torch.bfloat16:
rtol, atol = 5e-2, 1.5e-1
@@ -970,7 +987,7 @@ def test_selective_state_update_with_num_accepted_tokens(
def test_selective_state_update_varlen_with_num_accepted(
dim, dstate, has_z, itype, max_seq_len
):
device = "cuda"
device = DEVICE
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2)
if itype == torch.bfloat16:
rtol, atol = 5e-2, 1.5e-1
+13 -3
View File
@@ -9,9 +9,19 @@ from einops import rearrange, repeat
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
mamba_chunk_scan_combined_varlen,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.backends.mamba2_attn import compute_varlen_chunk_metadata
# All kernels exercised here are pure Triton, so they run on any backend
# that the vLLM platform layer treats as a CUDA-alike device or as XPU.
DEVICE = current_platform.device_type
pytestmark = pytest.mark.skipif(
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
reason="Mamba2 SSD Triton kernels require a CUDA-alike or XPU device.",
)
# Added by the IBM Team, 2024
# Adapted from https://github.com/state-spaces/mamba/blob/v2.2.4/mamba_ssm/modules/ssd_minimal.py
@@ -81,7 +91,7 @@ def ssd_minimal_discrete(X, A, B, C, block_len, initial_states=None):
return Y, final_state
def generate_random_inputs(batch_size, seqlen, n_heads, d_head, itype, device="cuda"):
def generate_random_inputs(batch_size, seqlen, n_heads, d_head, itype, device=DEVICE):
set_random_seed(0)
A = -torch.exp(torch.rand(n_heads, dtype=itype, device=device))
dt = F.softplus(
@@ -103,7 +113,7 @@ def generate_continuous_batched_examples(
n_heads,
d_head,
itype,
device="cuda",
device=DEVICE,
return_naive_ref=True,
):
# this function generates a random examples of certain length
@@ -215,7 +225,7 @@ def test_mamba_chunk_scan_single_example(d_head, n_heads, seq_len_chunk_size, it
X * dt.unsqueeze(-1), A * dt, B, C, chunk_size
)
cu_seqlens = torch.tensor((0, seqlen), device="cuda").cumsum(dim=0)
cu_seqlens = torch.tensor((0, seqlen), device=DEVICE).cumsum(dim=0)
cu_chunk_seqlens, last_chunk_indices, seq_idx_chunks = (
compute_varlen_chunk_metadata(cu_seqlens, chunk_size)
)