diff --git a/tests/conftest.py b/tests/conftest.py index 53d448b9b5..a2d910c694 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -284,19 +284,18 @@ def cleanup_fixture(should_do_global_cleanup_after_test: bool): def workspace_init(): """Initialize the workspace manager for tests that need it. - This fixture initializes the workspace manager with a CUDA device - if available, and resets it after the test completes. Tests that - create a full vLLM engine should NOT use this fixture as the engine - will initialize the workspace manager itself. + This fixture initializes the workspace manager with the current + platform's accelerator device if available, and resets it after the test + completes. Tests that create a full vLLM engine should NOT use this + fixture as the engine will initialize the workspace manager itself. """ from vllm.v1.worker.workspace import ( init_workspace_manager, reset_workspace_manager, ) - if torch.cuda.is_available(): - device = torch.device("cuda:0") - init_workspace_manager(device) + if torch.accelerator.is_available(): + init_workspace_manager(torch.device(0)) yield reset_workspace_manager() diff --git a/tests/kernels/attention/test_merge_attn_states.py b/tests/kernels/attention/test_merge_attn_states.py index eeadf0da0b..1b89e4db66 100644 --- a/tests/kernels/attention/test_merge_attn_states.py +++ b/tests/kernels/attention/test_merge_attn_states.py @@ -5,7 +5,7 @@ import pytest import torch from vllm._custom_ops import ( - merge_attn_states as merge_attn_states_cuda, + merge_attn_states as merge_attn_states_native, ) from vllm._custom_ops import ( scaled_fp8_quant, @@ -15,11 +15,13 @@ from vllm.v1.attention.ops.triton_merge_attn_states import ( merge_attn_states as merge_attn_states_triton, ) +DEVICE = current_platform.device_type + pytestmark = [ pytest.mark.skip_global_cleanup, pytest.mark.skipif( - not current_platform.is_cuda_alike(), - reason="merge_attn_states kernels require CUDA or ROCm.", + not (current_platform.is_cuda_alike() or current_platform.is_xpu()), + reason="merge_attn_states kernels require CUDA, ROCm or XPU.", ), ] @@ -81,20 +83,22 @@ DTYPES = [torch.float32, torch.half, torch.bfloat16] all_case_info: list[tuple] = [] -@pytest.mark.parametrize("merge_fn", [merge_attn_states_cuda, merge_attn_states_triton]) +@pytest.mark.parametrize( + "merge_fn", [merge_attn_states_native, merge_attn_states_triton] +) @pytest.mark.parametrize("output_dtype", [torch.float32, torch.half, torch.bfloat16]) def test_merge_attn_states_both_empty(merge_fn, output_dtype) -> None: """When a token is empty on both sides (both LSE -inf), the 0/0 softmax scales must not surface as NaN in the merged output.""" num_tokens, num_heads, head_size = 6, 8, 128 prefix_output = torch.zeros( - num_tokens, num_heads, head_size, device="cuda", dtype=output_dtype + num_tokens, num_heads, head_size, device=DEVICE, dtype=output_dtype ) - prefix_lse = torch.randn(num_heads, num_tokens, device="cuda") + prefix_lse = torch.randn(num_heads, num_tokens, device=DEVICE) suffix_output = torch.zeros( - num_tokens, num_heads, head_size, device="cuda", dtype=output_dtype + num_tokens, num_heads, head_size, device=DEVICE, dtype=output_dtype ) - suffix_lse = torch.randn(num_heads, num_tokens, device="cuda") + suffix_lse = torch.randn(num_heads, num_tokens, device=DEVICE) # Tokens 2 and 3 are empty on both sides. empty = slice(2, 4) @@ -111,7 +115,7 @@ def generate_markdown_table(): global all_case_info table_header = ( "| tokens | heads | headsize | dtype " - "| device | torch | triton | cuda | speedup |" + "| device | torch | triton | native | speedup |" ) table_separator = "| --- | --- | --- | --- | --- | --- | --- | --- | --- |" @@ -132,7 +136,7 @@ def generate_markdown_table(): device, avg_time_torch_kernel, avg_time_triton_kernel, - avg_time_cuda_kernel, + avg_time_native_kernel, performance_improved, ) = info dtype = shortly_dtype(dtype) @@ -141,7 +145,7 @@ def generate_markdown_table(): f"| {num_tokens} | {num_heads} | {head_size} " f"| {dtype} | {device} | {avg_time_torch_kernel:.5f}ms " f"| {avg_time_triton_kernel:.5f}ms " - f"| {avg_time_cuda_kernel:.5f}ms " + f"| {avg_time_native_kernel:.5f}ms " f"| {performance_improved:.4f}x |" ) @@ -171,7 +175,7 @@ def test_merge_attn_states( output_scale = None if use_fp8: output_dtype = current_platform.fp8_dtype() - output_scale = torch.tensor([0.05], dtype=torch.float32, device="cuda") + output_scale = torch.tensor([0.05], dtype=torch.float32, device=DEVICE) print( f"\nNUM_TOKENS:{NUM_TOKENS}, NUM_HEADS:{NUM_HEADS}, " @@ -182,8 +186,8 @@ def test_merge_attn_states( ) # prefix_lse and suffix_lse contain inf and normal values - prefix_lse = torch.randn(NUM_HEADS, NUM_TOKENS, dtype=torch.float32, device="cuda") - suffix_lse = torch.randn(NUM_HEADS, NUM_TOKENS, dtype=torch.float32, device="cuda") + prefix_lse = torch.randn(NUM_HEADS, NUM_TOKENS, dtype=torch.float32, device=DEVICE) + suffix_lse = torch.randn(NUM_HEADS, NUM_TOKENS, dtype=torch.float32, device=DEVICE) # Generate boolean masks mask_prefix = torch.rand(NUM_HEADS, NUM_TOKENS) < 0.1 @@ -199,16 +203,16 @@ def test_merge_attn_states( # Other input tensors (need to be initialized but # no actual calculation needed) output = torch.zeros( - (NUM_TOKENS, NUM_HEADS, HEAD_SIZE), dtype=output_dtype, device="cuda" + (NUM_TOKENS, NUM_HEADS, HEAD_SIZE), dtype=output_dtype, device=DEVICE ) output_lse = torch.zeros( - (NUM_HEADS, NUM_TOKENS), dtype=torch.float32, device="cuda" + (NUM_HEADS, NUM_TOKENS), dtype=torch.float32, device=DEVICE ) prefix_output = torch.randn( - (NUM_TOKENS, NUM_HEADS, HEAD_SIZE), dtype=input_dtype, device="cuda" + (NUM_TOKENS, NUM_HEADS, HEAD_SIZE), dtype=input_dtype, device=DEVICE ) suffix_output = torch.randn( - (NUM_TOKENS, NUM_HEADS, HEAD_SIZE), dtype=input_dtype, device="cuda" + (NUM_TOKENS, NUM_HEADS, HEAD_SIZE), dtype=input_dtype, device=DEVICE ) warmup_times = 2 @@ -293,19 +297,19 @@ def test_merge_attn_states( avg_time_triton_kernel = total_time_triton_kernel / repeat_times - # 2. Run the CUDA kernel - total_time_cuda_kernel = 0 - output_cuda = output.clone() - output_lse_cuda = output_lse.clone() + # 2. Run the native (CUDA/XPU) kernel + total_time_native_kernel = 0 + output_native = output.clone() + output_lse_native = output_lse.clone() for _ in range(warmup_times): - merge_attn_states_cuda( - output_cuda, + merge_attn_states_native( + output_native, prefix_output, prefix_lse, suffix_output, suffix_lse, - output_lse_cuda, + output_lse_native, prefill_tokens_with_context, output_scale, ) @@ -313,28 +317,28 @@ def test_merge_attn_states( for _ in range(repeat_times): start.record() - merge_attn_states_cuda( - output_cuda, + merge_attn_states_native( + output_native, prefix_output, prefix_lse, suffix_output, suffix_lse, - output_lse_cuda, + output_lse_native, prefill_tokens_with_context, output_scale, ) end.record() torch.accelerator.synchronize() - total_time_cuda_kernel += start.elapsed_time(end) + total_time_native_kernel += start.elapsed_time(end) - avg_time_cuda_kernel = total_time_cuda_kernel / repeat_times + avg_time_native_kernel = total_time_native_kernel / repeat_times # 3. Performance compare - performance_improved = avg_time_triton_kernel / avg_time_cuda_kernel + performance_improved = avg_time_triton_kernel / avg_time_native_kernel print(f" Torch time: {avg_time_torch_kernel:.6f}ms") print(f"Triton time: {avg_time_triton_kernel:.6f}ms") print( - f" CUDA time: {avg_time_cuda_kernel:.6f}ms, " + f"Native time: {avg_time_native_kernel:.6f}ms, " f"Performance: {performance_improved:.5f}x" ) print("-" * 100) @@ -362,12 +366,12 @@ def test_merge_attn_states( return max_diff # Use Triton output as reference because we want to replace - # the Triton kernel with custom CUDA kernel for merge attn + # the Triton kernel with the custom native kernel for merge attn # states operation. output_ref = output_ref_triton output_lse_ref = output_lse_ref_triton torch.testing.assert_close( - output_cuda.float() * scale, + output_native.float() * scale, output_ref.float() * scale, atol=atol, rtol=rtol, @@ -379,19 +383,19 @@ def test_merge_attn_states( ) _diff = diff(output_ref.float() * scale, output_torch.float() * scale) print(f"(Triton vs Torch) : {_diff}") - _diff = diff(output_torch.float() * scale, output_cuda.float() * scale) - print(f" (CUDA vs Torch) : {_diff}") - _diff = diff(output_ref.float() * scale, output_cuda.float() * scale) - print(f" (CUDA vs Triton): {_diff}") + _diff = diff(output_torch.float() * scale, output_native.float() * scale) + print(f"(Native vs Torch) : {_diff}") + _diff = diff(output_ref.float() * scale, output_native.float() * scale) + print(f"(Native vs Triton): {_diff}") print("-" * 100) torch.testing.assert_close( - output_lse_cuda.float(), output_lse_ref.float(), atol=atol, rtol=rtol + output_lse_native.float(), output_lse_ref.float(), atol=atol, rtol=rtol ) print("Output LSE all match, max abs diff:") print(f"(Triton vs Torch) : {diff(output_lse_torch, output_lse_ref)}") - print(f" (CUDA vs Torch) : {diff(output_lse_torch, output_lse_cuda)}") - print(f" (CUDA vs Triton): {diff(output_lse_ref, output_lse_cuda)}") + print(f"(Native vs Torch) : {diff(output_lse_torch, output_lse_native)}") + print(f"(Native vs Triton): {diff(output_lse_ref, output_lse_native)}") print("-" * 100) print( @@ -410,7 +414,7 @@ def test_merge_attn_states( device, avg_time_torch_kernel, avg_time_triton_kernel, - avg_time_cuda_kernel, + avg_time_native_kernel, performance_improved, ) ) diff --git a/tests/kernels/mamba/test_replayssm_prefill_decode_equivalence_mamba2.py b/tests/kernels/mamba/test_replayssm_prefill_decode_equivalence_mamba2.py index 5319d8964d..063ff17dd3 100644 --- a/tests/kernels/mamba/test_replayssm_prefill_decode_equivalence_mamba2.py +++ b/tests/kernels/mamba/test_replayssm_prefill_decode_equivalence_mamba2.py @@ -36,6 +36,7 @@ from vllm.model_executor.layers.mamba.ops.selective_state_update_replayssm_outpu from vllm.model_executor.layers.mamba.ops.ssd_combined import ( mamba_chunk_scan_combined_varlen, ) +from vllm.platforms import current_platform from vllm.utils.torch_utils import set_random_seed from vllm.v1.attention.backends.mamba2_attn import compute_varlen_chunk_metadata @@ -68,7 +69,7 @@ def _run_prefill_decode_equivalence( flow (raw dt + softplus + a per-head dt_bias), so this also checks prefill and decode apply the softplus/bias preprocessing consistently. ``state_dtype`` is the recurrent-state precision; ``act_dtype`` the activation/buffer one.""" - device = "cuda" + device = current_platform.device_type rtol, atol = _prefill_tolerances(act_dtype) set_random_seed(seed) @@ -203,7 +204,10 @@ def _run_prefill_decode_equivalence( ) -@pytest.mark.skipif(not torch.cuda.is_available(), reason="Need CUDA device") +@pytest.mark.skipif( + not (current_platform.is_cuda_alike() or current_platform.is_xpu()), + reason="Mamba2 ReplaySSM kernels require a CUDA-alike or XPU device.", +) @pytest.mark.parametrize( "precision", # fp32 state is the default; bf16/fp16 are reduced-footprint configs. fp16 diff --git a/tests/kernels/moe/test_batched_moe.py b/tests/kernels/moe/test_batched_moe.py index 3c605ced68..19375e6ce9 100644 --- a/tests/kernels/moe/test_batched_moe.py +++ b/tests/kernels/moe/test_batched_moe.py @@ -23,6 +23,13 @@ from vllm.platforms import current_platform from vllm.triton_utils import tl from vllm.utils.torch_utils import set_random_seed +DEVICE = current_platform.device_type + +pytestmark = pytest.mark.skipif( + not (current_platform.is_cuda_alike() or current_platform.is_xpu()), + reason="Triton MoE kernels require CUDA/ROCm/XPU.", +) + MNK_FACTORS = [ (1, 128, 128), (1, 512, 512), @@ -71,19 +78,19 @@ class BatchedMMTensors: A = ( torch.randn( (config.num_experts, config.max_tokens_per_expert, config.K), - device="cuda", + device=DEVICE, dtype=config.in_dtype, ) / 10 ) B = torch.randn( (config.num_experts, config.N, config.K), - device="cuda", + device=DEVICE, dtype=config.in_dtype, ) C = torch.zeros( (config.num_experts, config.max_tokens_per_expert, config.N), - device="cuda", + device=DEVICE, dtype=config.out_dtype, ) @@ -91,7 +98,7 @@ class BatchedMMTensors: low=0, high=config.max_tokens_per_expert, size=(config.num_experts,), - device="cuda", + device=DEVICE, dtype=torch.int32, ) @@ -120,8 +127,10 @@ def test_batched_mm( use_fp8_w8a8 = dtype == torch.float8_e4m3fn - if (dtype == torch.float8_e4m3fn) and not current_platform.has_device_capability( - 89 + if ( + dtype == torch.float8_e4m3fn + and current_platform.is_cuda_alike() + and not current_platform.has_device_capability(89) ): pytest.skip( "Triton limitation: fp8e4nv data type is not supported on CUDA arch < 89" @@ -144,7 +153,7 @@ def test_batched_mm( low=0, high=max_tokens_per_expert, size=(num_experts,), - device="cuda", + device=DEVICE, dtype=torch.int32, ) @@ -169,9 +178,9 @@ def test_batched_mm( ) out_shape = (num_experts, max_tokens_per_expert, N) - test_output = torch.zeros(out_shape, dtype=act_dtype, device="cuda") - ref_output = torch.zeros(out_shape, dtype=act_dtype, device="cuda") - q_ref_output = torch.zeros(out_shape, dtype=act_dtype, device="cuda") + test_output = torch.zeros(out_shape, dtype=act_dtype, device=DEVICE) + ref_output = torch.zeros(out_shape, dtype=act_dtype, device=DEVICE) + q_ref_output = torch.zeros(out_shape, dtype=act_dtype, device=DEVICE) compute_tl_dtype = { torch.float16: tl.float16, @@ -257,8 +266,10 @@ def test_fused_moe_batched_experts( use_fp8_w8a8 = dtype == torch.float8_e4m3fn - if (dtype == torch.float8_e4m3fn) and not current_platform.has_device_capability( - 89 + if ( + dtype == torch.float8_e4m3fn + and current_platform.is_cuda_alike() + and not current_platform.has_device_capability(89) ): pytest.skip( "Triton limitation: fp8e4nv data type is not supported on CUDA arch < 89" @@ -273,8 +284,8 @@ def test_fused_moe_batched_experts( if per_act_token_quant and block_shape is not None: pytest.skip("Skip illegal quantization test.") - a = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) / 10 - score = torch.randn((m, e), device="cuda", dtype=torch.bfloat16) + a = torch.randn((m, k), device=DEVICE, dtype=torch.bfloat16) / 10 + score = torch.randn((m, e), device=DEVICE, dtype=torch.bfloat16) if dtype.itemsize == 1: act_dtype = torch.bfloat16 @@ -294,8 +305,8 @@ def test_fused_moe_batched_experts( ) if input_scales and quant_dtype is not None: - a1_scale = torch.tensor(1, device="cuda", dtype=torch.float32) - a2_scale = torch.tensor(1, device="cuda", dtype=torch.float32) + a1_scale = torch.tensor(1, device=DEVICE, dtype=torch.float32) + a2_scale = torch.tensor(1, device=DEVICE, dtype=torch.float32) else: a1_scale = None a2_scale = None @@ -489,10 +500,7 @@ def test_batched_triton_experts_supports_current_device(): BatchedTritonExperts, ) - if current_platform.is_xpu() or current_platform.is_cuda_alike(): - assert BatchedTritonExperts._supports_current_device() - else: - pytest.skip("No GPU device available") + assert BatchedTritonExperts._supports_current_device() @pytest.mark.parametrize("m,n,k,e,topk", [(32, 512, 512, 8, 2), (45, 1024, 128, 8, 1)]) @@ -500,9 +508,6 @@ def test_batched_experts_end_to_end(m, n, k, e, topk): """End-to-end BatchedTritonExperts via the reference (no-comms) BatchedPrepareAndFinalize, validated against a torch reference. Exercises the device-enablement path.""" - if not (current_platform.is_xpu() or current_platform.is_cuda_alike()): - pytest.skip("No GPU device available") - from vllm.v1.worker.workspace import init_workspace_manager set_random_seed(7) diff --git a/tests/kernels/moe/test_count_expert_num_tokens.py b/tests/kernels/moe/test_count_expert_num_tokens.py index 39138be83b..8014076192 100644 --- a/tests/kernels/moe/test_count_expert_num_tokens.py +++ b/tests/kernels/moe/test_count_expert_num_tokens.py @@ -10,6 +10,14 @@ import pytest import torch from vllm.model_executor.layers.fused_moe.utils import count_expert_num_tokens +from vllm.platforms import current_platform + +DEVICE = current_platform.device_type + +pytestmark = pytest.mark.skipif( + not (current_platform.is_cuda_alike() or current_platform.is_xpu()), + reason="Triton MoE kernels require CUDA/ROCm/XPU.", +) @dataclasses.dataclass @@ -88,9 +96,9 @@ def do_test_compute_expert_num_tokens( (num_local_experts), device="cpu", dtype=torch.int32 ) ref_impl(tt_rank, ref_expert_num_tokens) - ref_expert_num_tokens = ref_expert_num_tokens.to("cuda") + ref_expert_num_tokens = ref_expert_num_tokens.to(DEVICE) - tt_rank.to_device("cuda") + tt_rank.to_device(DEVICE) # Test with expert_map triton_expert_num_tokens_w_emap = count_expert_num_tokens( tt_rank.topk_ids, num_local_experts, tt_rank.expert_map diff --git a/tests/kernels/moe/test_silu_mul_per_token_group_quant_fp8_colmajor.py b/tests/kernels/moe/test_silu_mul_per_token_group_quant_fp8_colmajor.py index cb01db44f8..bfe763b44a 100644 --- a/tests/kernels/moe/test_silu_mul_per_token_group_quant_fp8_colmajor.py +++ b/tests/kernels/moe/test_silu_mul_per_token_group_quant_fp8_colmajor.py @@ -13,6 +13,7 @@ from vllm.triton_utils import triton from vllm.utils.deep_gemm import is_deep_gemm_e8m0_used from vllm.utils.torch_utils import set_random_seed +DEVICE = current_platform.device_type FLOAT8_DTYPE = torch.float8_e4m3fn GROUP_SIZE = 128 @@ -61,7 +62,7 @@ def reference_quant(x: torch.Tensor, use_ue8m0: bool): def reference(x: torch.Tensor, use_ue8m0: bool) -> tuple[torch.Tensor, torch.Tensor]: T, N = x.size() - ref_act_out = torch.empty((T, N // 2), dtype=torch.bfloat16, device="cuda") + ref_act_out = torch.empty((T, N // 2), dtype=torch.bfloat16, device=DEVICE) torch.ops._C.silu_and_mul(ref_act_out, x) return reference_quant(ref_act_out, use_ue8m0) @@ -93,7 +94,7 @@ def reference_with_clamp( def test_silu_mul_fp8_quant_deep_gemm(T: int, N: int): set_random_seed(42) - input = torch.rand((T, N), dtype=torch.bfloat16, device="cuda") + input = torch.rand((T, N), dtype=torch.bfloat16, device=DEVICE) use_ue8m0 = is_deep_gemm_e8m0_used() @@ -122,7 +123,7 @@ def test_silu_mul_fp8_quant_deep_gemm_clamp(T: int, N: int, clamp_limit: float): # Use a wide distribution so values routinely exceed both clamp limits and # the clamp branch is actually exercised (uniform [0, 1) inputs would never # trigger it). - input = torch.randn((T, N), dtype=torch.bfloat16, device="cuda") * 8.0 + input = torch.randn((T, N), dtype=torch.bfloat16, device=DEVICE) * 8.0 use_ue8m0 = is_deep_gemm_e8m0_used() diff --git a/tests/kernels/moe/test_triton_moe_no_act_mul.py b/tests/kernels/moe/test_triton_moe_no_act_mul.py index e9f6f51c22..1171061a6f 100644 --- a/tests/kernels/moe/test_triton_moe_no_act_mul.py +++ b/tests/kernels/moe/test_triton_moe_no_act_mul.py @@ -23,6 +23,20 @@ from vllm.model_executor.layers.fused_moe.config import ( from vllm.model_executor.layers.fused_moe.experts.triton_moe import TritonExperts from vllm.platforms import current_platform +DEVICE = current_platform.device_type + +pytestmark = [ + pytest.mark.skipif( + not (current_platform.is_cuda_alike() or current_platform.is_xpu()), + reason="Triton MoE kernels require a CUDA-alike or XPU device.", + ), + pytest.mark.skipif( + current_platform.is_cuda_alike() + and not current_platform.has_device_capability(80), + reason="Triton MoE kernels require compute capability >= 8.0 on CUDA/ROCm.", + ), +] + # Test parameters M_SIZES = [1, 16, 64] N_SIZES = [128, 256] @@ -86,7 +100,7 @@ def make_test_tensors( num_experts: int, topk: int, dtype: torch.dtype = torch.bfloat16, - device: str = "cuda", + device: str = DEVICE, ): """Create test tensors for MoE with non-gated activation. @@ -106,10 +120,6 @@ def make_test_tensors( return hidden_states, w1, w2, topk_weights, topk_ids -@pytest.mark.skipif( - not current_platform.has_device_capability(80), - reason="Requires compute capability >= 8.0", -) @pytest.mark.parametrize("m", M_SIZES) @pytest.mark.parametrize("n", N_SIZES) @pytest.mark.parametrize("k", K_SIZES) @@ -192,10 +202,6 @@ def test_triton_experts_no_mul_activation( assert output.abs().sum() > 0, "Output is all zeros" -@pytest.mark.skipif( - not current_platform.has_device_capability(80), - reason="Requires compute capability >= 8.0", -) @torch.inference_mode() def test_workspace_shapes_no_mul_vs_gated(): """Test that workspace shapes differ correctly between gated and non-gated.""" @@ -233,10 +239,6 @@ def test_workspace_shapes_no_mul_vs_gated(): assert out_no_mul == out_gated == (M, K) -@pytest.mark.skipif( - not current_platform.has_device_capability(80), - reason="Requires compute capability >= 8.0", -) @torch.inference_mode() def test_adjust_n_for_activation(): """Test the adjust_N_for_activation method.""" diff --git a/tests/kernels/moe/utils.py b/tests/kernels/moe/utils.py index ce6e07f122..e82d315715 100644 --- a/tests/kernels/moe/utils.py +++ b/tests/kernels/moe/utils.py @@ -40,6 +40,8 @@ from vllm.platforms import current_platform from vllm.utils.deep_gemm import per_block_cast_to_fp8 from vllm.utils.math_utils import round_up +DEVICE = current_platform.device_type + def shuffle_weight(w: torch.Tensor) -> torch.Tensor: """Fold weights to adjacent locations for Triton MoE / SwiGLU kernel layout.""" @@ -80,7 +82,7 @@ def make_dummy_moe_config( moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(), activation=activation, in_dtype=in_dtype, - device="cuda", + device=DEVICE, routing_method=RoutingMethodType.TopK, max_num_tokens=max_num_tokens, ) @@ -238,7 +240,7 @@ def make_quantized_test_activations( block_shape: list[int] | None = None, per_act_token_quant: bool = False, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]: - a = torch.randn((E, m, k), device="cuda", dtype=in_dtype) / 10 + a = torch.randn((E, m, k), device=DEVICE, dtype=in_dtype) / 10 a_q = a a_scale = None @@ -369,7 +371,7 @@ def make_test_weight( block_shape: list[int] | None = None, per_out_ch_quant: bool = False, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None, torch.Tensor | None]: - w_16 = torch.randn((e, rows, cols), device="cuda", dtype=in_dtype) / 15 + w_16 = torch.randn((e, rows, cols), device=DEVICE, dtype=in_dtype) / 15 if quant_dtype is not None: w, w_s, w_gs = moe_quantize_weights( @@ -449,8 +451,8 @@ def make_test_quant_config( a1_gscale: torch.Tensor | None = None a2_gscale: torch.Tensor | None = None if quant_dtype == "nvfp4": - a1_gscale = torch.ones((e,), device="cuda", dtype=torch.float32) - a2_gscale = torch.ones((e,), device="cuda", dtype=torch.float32) + a1_gscale = torch.ones((e,), device=DEVICE, dtype=torch.float32) + a2_gscale = torch.ones((e,), device=DEVICE, dtype=torch.float32) a1_scale = a1_gscale a2_scale = a2_gscale else: @@ -551,8 +553,8 @@ def make_naive_shared_experts( K: int, in_dtype: torch.dtype = torch.bfloat16, ) -> torch.nn.Module: - w1 = torch.randn((K, N * 2), device="cuda", dtype=in_dtype) / 15 - w2 = torch.randn((N, K), device="cuda", dtype=in_dtype) / 15 + w1 = torch.randn((K, N * 2), device=DEVICE, dtype=in_dtype) / 15 + w2 = torch.randn((N, K), device=DEVICE, dtype=in_dtype) / 15 return TestMLP(w1, w2, out_dtype=in_dtype) diff --git a/tests/kernels/quant_utils.py b/tests/kernels/quant_utils.py index a67cb8fd78..7e14ec95d3 100644 --- a/tests/kernels/quant_utils.py +++ b/tests/kernels/quant_utils.py @@ -13,10 +13,11 @@ from vllm.utils.deep_gemm import _ceil_to_ue8m0, is_deep_gemm_e8m0_used from vllm.utils.math_utils import round_up FP8_DTYPE = current_platform.fp8_dtype() +DEVICE = current_platform.device_type def as_float32_tensor(x: float | torch.Tensor) -> torch.Tensor: - return torch.as_tensor(x, dtype=torch.float32, device="cuda") + return torch.as_tensor(x, dtype=torch.float32, device=DEVICE) def ref_dynamic_per_token_quant( diff --git a/tests/kernels/quantization/test_per_token_group_quant.py b/tests/kernels/quantization/test_per_token_group_quant.py index 0d9b6c0c3e..623c9b6f13 100644 --- a/tests/kernels/quantization/test_per_token_group_quant.py +++ b/tests/kernels/quantization/test_per_token_group_quant.py @@ -18,19 +18,20 @@ from vllm.platforms import current_platform @pytest.mark.parametrize("scale_ue8m0", [False, True]) @pytest.mark.parametrize("group_size", [64, 128]) @pytest.mark.skipif( - not current_platform.is_cuda_alike(), reason="Only test on CUDA/ROCm." + not (current_platform.is_cuda_alike() or current_platform.is_xpu()), + reason="Only test on CUDA/ROCm/XPU.", ) def test_per_token_group_quant_fp8( shape, column_major: bool, tma_aligned: bool, scale_ue8m0: bool, group_size: int ): - device = "cuda" + device = current_platform.device_type torch.manual_seed(42) num_tokens, hidden_dim = shape x = torch.randn((num_tokens, hidden_dim), device=device, dtype=torch.bfloat16) * 8 - # cuda path + # native kernel path out_q, scale = fp8_utils.per_token_group_quant_fp8( x, group_size, @@ -40,7 +41,10 @@ def test_per_token_group_quant_fp8( ) # triton ref - with patch("vllm.platforms.current_platform.is_cuda_alike", return_value=False): + with ( + patch("vllm.platforms.current_platform.is_cuda_alike", return_value=False), + patch("vllm.platforms.current_platform.is_xpu", return_value=False), + ): ref_q, ref_s = fp8_utils.per_token_group_quant_fp8( x, group_size, diff --git a/tests/kernels/test_fused_recurrent_packed_decode.py b/tests/kernels/test_fused_recurrent_packed_decode.py index 33a66034b4..5e83537d94 100644 --- a/tests/kernels/test_fused_recurrent_packed_decode.py +++ b/tests/kernels/test_fused_recurrent_packed_decode.py @@ -4,13 +4,20 @@ import pytest import torch +from vllm.platforms import current_platform from vllm.third_party.flash_linear_attention.ops import ( fused_recurrent_gated_delta_rule, fused_recurrent_gated_delta_rule_packed_decode, ) +DEVICE = current_platform.device_type + +pytestmark = pytest.mark.skipif( + not (current_platform.is_cuda_alike() or current_platform.is_xpu()), + reason="Gated delta rule Triton kernels require a CUDA-alike or XPU device.", +) + -@pytest.mark.skipif(not torch.cuda.is_available(), reason="Need CUDA device") @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32]) @pytest.mark.parametrize("strided_mixed_qkv", [False, True]) def test_fused_recurrent_packed_decode_matches_reference( @@ -26,7 +33,7 @@ def test_fused_recurrent_packed_decode_matches_reference( V = 128 qkv_dim = 2 * (H * K) + (HV * V) - device = torch.device("cuda") + device = torch.device(DEVICE) if strided_mixed_qkv: # Simulate a packed view into a larger projection buffer: @@ -102,10 +109,9 @@ def test_fused_recurrent_packed_decode_matches_reference( torch.testing.assert_close(state_packed, state_ref, rtol=rtol, atol=atol) -@pytest.mark.skipif(not torch.cuda.is_available(), reason="Need CUDA device") def test_packed_decode_supports_large_batch_head_grid(): B, H, HV, K, V = 1024, 8, 64, 1, 1 - device = torch.device("cuda") + device = torch.device(DEVICE) gates = torch.empty((B, HV), device=device) params = torch.empty((HV,), device=device) out = torch.empty((B, 1, HV, V), device=device) diff --git a/vllm/model_executor/layers/fused_moe/experts/fused_batched_moe.py b/vllm/model_executor/layers/fused_moe/experts/fused_batched_moe.py index d35c3d0d71..40d17b14dc 100644 --- a/vllm/model_executor/layers/fused_moe/experts/fused_batched_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/fused_batched_moe.py @@ -37,9 +37,10 @@ from vllm.triton_utils.allocation import set_triton_allocator def _is_capturing_or_compiling() -> bool: - # torch.cuda.is_current_stream_capturing() is unavailable on non-CUDA (XPU) torch. - return torch.compiler.is_compiling() or ( - current_platform.is_cuda_alike() and torch.cuda.is_current_stream_capturing() + # The accelerator API dispatches to the active backend. + return ( + torch.compiler.is_compiling() + or torch.accelerator.current_stream().is_capturing() )