diff --git a/tests/kernels/mamba/test_causal_conv1d.py b/tests/kernels/mamba/test_causal_conv1d.py index 0ebc527d54..c6554f131f 100644 --- a/tests/kernels/mamba/test_causal_conv1d.py +++ b/tests/kernels/mamba/test_causal_conv1d.py @@ -11,9 +11,17 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import ( causal_conv1d_fn, causal_conv1d_update, ) +from vllm.platforms import current_platform from vllm.utils.torch_utils import set_random_seed from vllm.v1.attention.backends.utils import NULL_BLOCK_ID +DEVICE = current_platform.device_type + +pytestmark = pytest.mark.skipif( + not (current_platform.is_cuda_alike() or current_platform.is_xpu()), + reason="causal_conv1d Triton kernels require CUDA-alike or XPU", +) + def causal_conv1d_ref( x: torch.Tensor, @@ -149,7 +157,7 @@ def causal_conv1d_opcheck_fn( @pytest.mark.parametrize("width", [4]) @pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096]) def test_causal_conv1d_update(dim, width, seqlen, has_bias, silu_activation, itype): - device = "cuda" + device = DEVICE rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3) if itype == torch.bfloat16: rtol, atol = 1e-2, 5e-2 @@ -196,7 +204,7 @@ def test_causal_conv1d_update(dim, width, seqlen, has_bias, silu_activation, ity def test_causal_conv1d_update_with_batch_gather( batch_size, with_padding, dim, width, seqlen, has_bias, silu_activation, itype ): - device = "cuda" + device = DEVICE rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3) if itype == torch.bfloat16: rtol, atol = 1e-2, 5e-2 @@ -275,7 +283,7 @@ def test_causal_conv1d_update_with_batch_gather( def test_causal_conv1d_varlen( batch, with_padding, dim, seqlen, width, has_bias, silu_activation, itype ): - device = "cuda" + device = DEVICE torch.accelerator.empty_cache() rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3) if itype == torch.bfloat16: @@ -341,7 +349,7 @@ def test_causal_conv1d_varlen( weight, bias=bias, conv_states=final_states, - query_start_loc=cumsum.cuda(), + query_start_loc=cumsum.to(device), cache_indices=padded_state_indices, has_initial_state=has_initial_states, activation=activation, diff --git a/tests/kernels/mamba/test_mamba_ssm.py b/tests/kernels/mamba/test_mamba_ssm.py index d812242cba..e3d35b44ff 100644 --- a/tests/kernels/mamba/test_mamba_ssm.py +++ b/tests/kernels/mamba/test_mamba_ssm.py @@ -17,6 +17,21 @@ from vllm.platforms import current_platform from vllm.utils.torch_utils import set_random_seed from vllm.v1.attention.backends.utils import NULL_BLOCK_ID +DEVICE = current_platform.device_type + +pytestmark = pytest.mark.skipif( + not (current_platform.is_cuda_alike() or current_platform.is_xpu()), + reason="mamba_ssm kernels require CUDA-alike or XPU", +) + +# selective_scan_fn is backed by the CUDA-only `ops.selective_scan_fwd` C++ op, +# so tests exercising it must be skipped on XPU. selective_state_update is +# pure Triton and runs on both CUDA-alike and XPU. +skip_unless_cuda_alike = pytest.mark.skipif( + not current_platform.is_cuda_alike(), + reason="selective_scan_fn uses CUDA-only custom op", +) + def selective_scan_ref( u, @@ -181,6 +196,7 @@ def selective_scan_opcheck_fn( @pytest.mark.parametrize("is_variable_C", [True]) @pytest.mark.parametrize("is_variable_B", [True]) @pytest.mark.parametrize("scan_chunks", [1, 3]) +@skip_unless_cuda_alike def test_selective_scan( is_variable_B, is_variable_C, @@ -327,7 +343,7 @@ def test_selective_scan( @pytest.mark.parametrize("dstate", [16, 64]) @pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096]) def test_selective_state_update(dim, dstate, has_z, itype): - device = "cuda" + device = DEVICE rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2) if itype == torch.bfloat16: rtol, atol = 1e-2, 5e-2 @@ -370,7 +386,7 @@ def test_selective_state_update(dim, dstate, has_z, itype): " on compute capability 10.0 CUDA devices.", ) def test_selective_state_update_stochastic_rounding(dim, dstate, has_z, philox_rounds): - device = "cuda" + device = DEVICE rtol, atol = 5e-3, 1e-1 # set seed set_random_seed(0) @@ -417,7 +433,7 @@ def test_selective_state_update_stochastic_rounding(dim, dstate, has_z, philox_r @pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096]) @pytest.mark.parametrize("max_seq_len", [1, 2, 4]) def test_selective_state_update_varlen(dim, dstate, has_z, itype, max_seq_len): - device = "cuda" + device = DEVICE rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2) if itype == torch.bfloat16: rtol, atol = 5e-2, 1.5e-1 @@ -498,6 +514,7 @@ def test_selective_state_update_varlen(dim, dstate, has_z, itype, max_seq_len): @pytest.mark.parametrize("is_variable_B", [True]) # tests correctness in case subset of the sequences are padded @pytest.mark.parametrize("with_padding", [False, True]) +@skip_unless_cuda_alike def test_selective_scan_varlen( with_padding, is_variable_B, @@ -679,7 +696,7 @@ def test_selective_scan_varlen( def test_selective_state_update_with_batch_indices( with_padding, dim, dstate, has_z, itype ): - device = "cuda" + device = DEVICE rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2) if itype == torch.bfloat16: rtol, atol = 1e-1, 1e-1 @@ -771,7 +788,7 @@ def test_selective_state_update_with_batch_indices( def test_selective_state_update_with_heads_with_batch_indices( dim, dstate, ngroups, has_z, tie_hdim, itype ): - device = "cuda" + device = DEVICE rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 3e-2) if itype == torch.bfloat16: rtol, atol = 1e-1, 1e-1 @@ -844,7 +861,7 @@ def test_selective_state_update_with_heads_with_batch_indices( def test_selective_state_update_with_num_accepted_tokens( dim, dstate, has_z, itype, max_seq_len ): - device = "cuda" + device = DEVICE rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2) if itype == torch.bfloat16: rtol, atol = 5e-2, 1.5e-1 @@ -970,7 +987,7 @@ def test_selective_state_update_with_num_accepted_tokens( def test_selective_state_update_varlen_with_num_accepted( dim, dstate, has_z, itype, max_seq_len ): - device = "cuda" + device = DEVICE rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2) if itype == torch.bfloat16: rtol, atol = 5e-2, 1.5e-1 diff --git a/tests/kernels/mamba/test_mamba_ssm_ssd.py b/tests/kernels/mamba/test_mamba_ssm_ssd.py index 40aa3d017d..1de25780ea 100644 --- a/tests/kernels/mamba/test_mamba_ssm_ssd.py +++ b/tests/kernels/mamba/test_mamba_ssm_ssd.py @@ -9,9 +9,19 @@ from einops import rearrange, repeat from vllm.model_executor.layers.mamba.ops.ssd_combined import ( mamba_chunk_scan_combined_varlen, ) +from vllm.platforms import current_platform from vllm.utils.torch_utils import set_random_seed from vllm.v1.attention.backends.mamba2_attn import compute_varlen_chunk_metadata +# All kernels exercised here are pure Triton, so they run on any backend +# that the vLLM platform layer treats as a CUDA-alike device or as XPU. +DEVICE = current_platform.device_type + +pytestmark = pytest.mark.skipif( + not (current_platform.is_cuda_alike() or current_platform.is_xpu()), + reason="Mamba2 SSD Triton kernels require a CUDA-alike or XPU device.", +) + # Added by the IBM Team, 2024 # Adapted from https://github.com/state-spaces/mamba/blob/v2.2.4/mamba_ssm/modules/ssd_minimal.py @@ -81,7 +91,7 @@ def ssd_minimal_discrete(X, A, B, C, block_len, initial_states=None): return Y, final_state -def generate_random_inputs(batch_size, seqlen, n_heads, d_head, itype, device="cuda"): +def generate_random_inputs(batch_size, seqlen, n_heads, d_head, itype, device=DEVICE): set_random_seed(0) A = -torch.exp(torch.rand(n_heads, dtype=itype, device=device)) dt = F.softplus( @@ -103,7 +113,7 @@ def generate_continuous_batched_examples( n_heads, d_head, itype, - device="cuda", + device=DEVICE, return_naive_ref=True, ): # this function generates a random examples of certain length @@ -215,7 +225,7 @@ def test_mamba_chunk_scan_single_example(d_head, n_heads, seq_len_chunk_size, it X * dt.unsqueeze(-1), A * dt, B, C, chunk_size ) - cu_seqlens = torch.tensor((0, seqlen), device="cuda").cumsum(dim=0) + cu_seqlens = torch.tensor((0, seqlen), device=DEVICE).cumsum(dim=0) cu_chunk_seqlens, last_chunk_indices, seq_idx_chunks = ( compute_varlen_chunk_metadata(cu_seqlens, chunk_size) )