[XPU][Tests] Make tests device-agnostic (#51968)

Signed-off-by: pmanczak <pawel.manczak@intel.com>
This commit is contained in:
pmanczak
2026-08-20 02:25:41 -07:00
committed by GitHub
parent 38e9cefdef
commit 44cf3f0466
12 changed files with 148 additions and 111 deletions
+6 -7
View File
@@ -284,19 +284,18 @@ def cleanup_fixture(should_do_global_cleanup_after_test: bool):
def workspace_init():
"""Initialize the workspace manager for tests that need it.
This fixture initializes the workspace manager with a CUDA device
if available, and resets it after the test completes. Tests that
create a full vLLM engine should NOT use this fixture as the engine
will initialize the workspace manager itself.
This fixture initializes the workspace manager with the current
platform's accelerator device if available, and resets it after the test
completes. Tests that create a full vLLM engine should NOT use this
fixture as the engine will initialize the workspace manager itself.
"""
from vllm.v1.worker.workspace import (
init_workspace_manager,
reset_workspace_manager,
)
if torch.cuda.is_available():
device = torch.device("cuda:0")
init_workspace_manager(device)
if torch.accelerator.is_available():
init_workspace_manager(torch.device(0))
yield
reset_workspace_manager()
@@ -5,7 +5,7 @@ import pytest
import torch
from vllm._custom_ops import (
merge_attn_states as merge_attn_states_cuda,
merge_attn_states as merge_attn_states_native,
)
from vllm._custom_ops import (
scaled_fp8_quant,
@@ -15,11 +15,13 @@ from vllm.v1.attention.ops.triton_merge_attn_states import (
merge_attn_states as merge_attn_states_triton,
)
DEVICE = current_platform.device_type
pytestmark = [
pytest.mark.skip_global_cleanup,
pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="merge_attn_states kernels require CUDA or ROCm.",
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
reason="merge_attn_states kernels require CUDA, ROCm or XPU.",
),
]
@@ -81,20 +83,22 @@ DTYPES = [torch.float32, torch.half, torch.bfloat16]
all_case_info: list[tuple] = []
@pytest.mark.parametrize("merge_fn", [merge_attn_states_cuda, merge_attn_states_triton])
@pytest.mark.parametrize(
"merge_fn", [merge_attn_states_native, merge_attn_states_triton]
)
@pytest.mark.parametrize("output_dtype", [torch.float32, torch.half, torch.bfloat16])
def test_merge_attn_states_both_empty(merge_fn, output_dtype) -> None:
"""When a token is empty on both sides (both LSE -inf), the 0/0 softmax
scales must not surface as NaN in the merged output."""
num_tokens, num_heads, head_size = 6, 8, 128
prefix_output = torch.zeros(
num_tokens, num_heads, head_size, device="cuda", dtype=output_dtype
num_tokens, num_heads, head_size, device=DEVICE, dtype=output_dtype
)
prefix_lse = torch.randn(num_heads, num_tokens, device="cuda")
prefix_lse = torch.randn(num_heads, num_tokens, device=DEVICE)
suffix_output = torch.zeros(
num_tokens, num_heads, head_size, device="cuda", dtype=output_dtype
num_tokens, num_heads, head_size, device=DEVICE, dtype=output_dtype
)
suffix_lse = torch.randn(num_heads, num_tokens, device="cuda")
suffix_lse = torch.randn(num_heads, num_tokens, device=DEVICE)
# Tokens 2 and 3 are empty on both sides.
empty = slice(2, 4)
@@ -111,7 +115,7 @@ def generate_markdown_table():
global all_case_info
table_header = (
"| tokens | heads | headsize | dtype "
"| device | torch | triton | cuda | speedup |"
"| device | torch | triton | native | speedup |"
)
table_separator = "| --- | --- | --- | --- | --- | --- | --- | --- | --- |"
@@ -132,7 +136,7 @@ def generate_markdown_table():
device,
avg_time_torch_kernel,
avg_time_triton_kernel,
avg_time_cuda_kernel,
avg_time_native_kernel,
performance_improved,
) = info
dtype = shortly_dtype(dtype)
@@ -141,7 +145,7 @@ def generate_markdown_table():
f"| {num_tokens} | {num_heads} | {head_size} "
f"| {dtype} | {device} | {avg_time_torch_kernel:.5f}ms "
f"| {avg_time_triton_kernel:.5f}ms "
f"| {avg_time_cuda_kernel:.5f}ms "
f"| {avg_time_native_kernel:.5f}ms "
f"| {performance_improved:.4f}x |"
)
@@ -171,7 +175,7 @@ def test_merge_attn_states(
output_scale = None
if use_fp8:
output_dtype = current_platform.fp8_dtype()
output_scale = torch.tensor([0.05], dtype=torch.float32, device="cuda")
output_scale = torch.tensor([0.05], dtype=torch.float32, device=DEVICE)
print(
f"\nNUM_TOKENS:{NUM_TOKENS}, NUM_HEADS:{NUM_HEADS}, "
@@ -182,8 +186,8 @@ def test_merge_attn_states(
)
# prefix_lse and suffix_lse contain inf and normal values
prefix_lse = torch.randn(NUM_HEADS, NUM_TOKENS, dtype=torch.float32, device="cuda")
suffix_lse = torch.randn(NUM_HEADS, NUM_TOKENS, dtype=torch.float32, device="cuda")
prefix_lse = torch.randn(NUM_HEADS, NUM_TOKENS, dtype=torch.float32, device=DEVICE)
suffix_lse = torch.randn(NUM_HEADS, NUM_TOKENS, dtype=torch.float32, device=DEVICE)
# Generate boolean masks
mask_prefix = torch.rand(NUM_HEADS, NUM_TOKENS) < 0.1
@@ -199,16 +203,16 @@ def test_merge_attn_states(
# Other input tensors (need to be initialized but
# no actual calculation needed)
output = torch.zeros(
(NUM_TOKENS, NUM_HEADS, HEAD_SIZE), dtype=output_dtype, device="cuda"
(NUM_TOKENS, NUM_HEADS, HEAD_SIZE), dtype=output_dtype, device=DEVICE
)
output_lse = torch.zeros(
(NUM_HEADS, NUM_TOKENS), dtype=torch.float32, device="cuda"
(NUM_HEADS, NUM_TOKENS), dtype=torch.float32, device=DEVICE
)
prefix_output = torch.randn(
(NUM_TOKENS, NUM_HEADS, HEAD_SIZE), dtype=input_dtype, device="cuda"
(NUM_TOKENS, NUM_HEADS, HEAD_SIZE), dtype=input_dtype, device=DEVICE
)
suffix_output = torch.randn(
(NUM_TOKENS, NUM_HEADS, HEAD_SIZE), dtype=input_dtype, device="cuda"
(NUM_TOKENS, NUM_HEADS, HEAD_SIZE), dtype=input_dtype, device=DEVICE
)
warmup_times = 2
@@ -293,19 +297,19 @@ def test_merge_attn_states(
avg_time_triton_kernel = total_time_triton_kernel / repeat_times
# 2. Run the CUDA kernel
total_time_cuda_kernel = 0
output_cuda = output.clone()
output_lse_cuda = output_lse.clone()
# 2. Run the native (CUDA/XPU) kernel
total_time_native_kernel = 0
output_native = output.clone()
output_lse_native = output_lse.clone()
for _ in range(warmup_times):
merge_attn_states_cuda(
output_cuda,
merge_attn_states_native(
output_native,
prefix_output,
prefix_lse,
suffix_output,
suffix_lse,
output_lse_cuda,
output_lse_native,
prefill_tokens_with_context,
output_scale,
)
@@ -313,28 +317,28 @@ def test_merge_attn_states(
for _ in range(repeat_times):
start.record()
merge_attn_states_cuda(
output_cuda,
merge_attn_states_native(
output_native,
prefix_output,
prefix_lse,
suffix_output,
suffix_lse,
output_lse_cuda,
output_lse_native,
prefill_tokens_with_context,
output_scale,
)
end.record()
torch.accelerator.synchronize()
total_time_cuda_kernel += start.elapsed_time(end)
total_time_native_kernel += start.elapsed_time(end)
avg_time_cuda_kernel = total_time_cuda_kernel / repeat_times
avg_time_native_kernel = total_time_native_kernel / repeat_times
# 3. Performance compare
performance_improved = avg_time_triton_kernel / avg_time_cuda_kernel
performance_improved = avg_time_triton_kernel / avg_time_native_kernel
print(f" Torch time: {avg_time_torch_kernel:.6f}ms")
print(f"Triton time: {avg_time_triton_kernel:.6f}ms")
print(
f" CUDA time: {avg_time_cuda_kernel:.6f}ms, "
f"Native time: {avg_time_native_kernel:.6f}ms, "
f"Performance: {performance_improved:.5f}x"
)
print("-" * 100)
@@ -362,12 +366,12 @@ def test_merge_attn_states(
return max_diff
# Use Triton output as reference because we want to replace
# the Triton kernel with custom CUDA kernel for merge attn
# the Triton kernel with the custom native kernel for merge attn
# states operation.
output_ref = output_ref_triton
output_lse_ref = output_lse_ref_triton
torch.testing.assert_close(
output_cuda.float() * scale,
output_native.float() * scale,
output_ref.float() * scale,
atol=atol,
rtol=rtol,
@@ -379,19 +383,19 @@ def test_merge_attn_states(
)
_diff = diff(output_ref.float() * scale, output_torch.float() * scale)
print(f"(Triton vs Torch) : {_diff}")
_diff = diff(output_torch.float() * scale, output_cuda.float() * scale)
print(f" (CUDA vs Torch) : {_diff}")
_diff = diff(output_ref.float() * scale, output_cuda.float() * scale)
print(f" (CUDA vs Triton): {_diff}")
_diff = diff(output_torch.float() * scale, output_native.float() * scale)
print(f"(Native vs Torch) : {_diff}")
_diff = diff(output_ref.float() * scale, output_native.float() * scale)
print(f"(Native vs Triton): {_diff}")
print("-" * 100)
torch.testing.assert_close(
output_lse_cuda.float(), output_lse_ref.float(), atol=atol, rtol=rtol
output_lse_native.float(), output_lse_ref.float(), atol=atol, rtol=rtol
)
print("Output LSE all match, max abs diff:")
print(f"(Triton vs Torch) : {diff(output_lse_torch, output_lse_ref)}")
print(f" (CUDA vs Torch) : {diff(output_lse_torch, output_lse_cuda)}")
print(f" (CUDA vs Triton): {diff(output_lse_ref, output_lse_cuda)}")
print(f"(Native vs Torch) : {diff(output_lse_torch, output_lse_native)}")
print(f"(Native vs Triton): {diff(output_lse_ref, output_lse_native)}")
print("-" * 100)
print(
@@ -410,7 +414,7 @@ def test_merge_attn_states(
device,
avg_time_torch_kernel,
avg_time_triton_kernel,
avg_time_cuda_kernel,
avg_time_native_kernel,
performance_improved,
)
)
@@ -36,6 +36,7 @@ from vllm.model_executor.layers.mamba.ops.selective_state_update_replayssm_outpu
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
mamba_chunk_scan_combined_varlen,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.backends.mamba2_attn import compute_varlen_chunk_metadata
@@ -68,7 +69,7 @@ def _run_prefill_decode_equivalence(
flow (raw dt + softplus + a per-head dt_bias), so this also checks prefill
and decode apply the softplus/bias preprocessing consistently. ``state_dtype``
is the recurrent-state precision; ``act_dtype`` the activation/buffer one."""
device = "cuda"
device = current_platform.device_type
rtol, atol = _prefill_tolerances(act_dtype)
set_random_seed(seed)
@@ -203,7 +204,10 @@ def _run_prefill_decode_equivalence(
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Need CUDA device")
@pytest.mark.skipif(
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
reason="Mamba2 ReplaySSM kernels require a CUDA-alike or XPU device.",
)
@pytest.mark.parametrize(
"precision",
# fp32 state is the default; bf16/fp16 are reduced-footprint configs. fp16
+28 -23
View File
@@ -23,6 +23,13 @@ from vllm.platforms import current_platform
from vllm.triton_utils import tl
from vllm.utils.torch_utils import set_random_seed
DEVICE = current_platform.device_type
pytestmark = pytest.mark.skipif(
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
reason="Triton MoE kernels require CUDA/ROCm/XPU.",
)
MNK_FACTORS = [
(1, 128, 128),
(1, 512, 512),
@@ -71,19 +78,19 @@ class BatchedMMTensors:
A = (
torch.randn(
(config.num_experts, config.max_tokens_per_expert, config.K),
device="cuda",
device=DEVICE,
dtype=config.in_dtype,
)
/ 10
)
B = torch.randn(
(config.num_experts, config.N, config.K),
device="cuda",
device=DEVICE,
dtype=config.in_dtype,
)
C = torch.zeros(
(config.num_experts, config.max_tokens_per_expert, config.N),
device="cuda",
device=DEVICE,
dtype=config.out_dtype,
)
@@ -91,7 +98,7 @@ class BatchedMMTensors:
low=0,
high=config.max_tokens_per_expert,
size=(config.num_experts,),
device="cuda",
device=DEVICE,
dtype=torch.int32,
)
@@ -120,8 +127,10 @@ def test_batched_mm(
use_fp8_w8a8 = dtype == torch.float8_e4m3fn
if (dtype == torch.float8_e4m3fn) and not current_platform.has_device_capability(
89
if (
dtype == torch.float8_e4m3fn
and current_platform.is_cuda_alike()
and not current_platform.has_device_capability(89)
):
pytest.skip(
"Triton limitation: fp8e4nv data type is not supported on CUDA arch < 89"
@@ -144,7 +153,7 @@ def test_batched_mm(
low=0,
high=max_tokens_per_expert,
size=(num_experts,),
device="cuda",
device=DEVICE,
dtype=torch.int32,
)
@@ -169,9 +178,9 @@ def test_batched_mm(
)
out_shape = (num_experts, max_tokens_per_expert, N)
test_output = torch.zeros(out_shape, dtype=act_dtype, device="cuda")
ref_output = torch.zeros(out_shape, dtype=act_dtype, device="cuda")
q_ref_output = torch.zeros(out_shape, dtype=act_dtype, device="cuda")
test_output = torch.zeros(out_shape, dtype=act_dtype, device=DEVICE)
ref_output = torch.zeros(out_shape, dtype=act_dtype, device=DEVICE)
q_ref_output = torch.zeros(out_shape, dtype=act_dtype, device=DEVICE)
compute_tl_dtype = {
torch.float16: tl.float16,
@@ -257,8 +266,10 @@ def test_fused_moe_batched_experts(
use_fp8_w8a8 = dtype == torch.float8_e4m3fn
if (dtype == torch.float8_e4m3fn) and not current_platform.has_device_capability(
89
if (
dtype == torch.float8_e4m3fn
and current_platform.is_cuda_alike()
and not current_platform.has_device_capability(89)
):
pytest.skip(
"Triton limitation: fp8e4nv data type is not supported on CUDA arch < 89"
@@ -273,8 +284,8 @@ def test_fused_moe_batched_experts(
if per_act_token_quant and block_shape is not None:
pytest.skip("Skip illegal quantization test.")
a = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) / 10
score = torch.randn((m, e), device="cuda", dtype=torch.bfloat16)
a = torch.randn((m, k), device=DEVICE, dtype=torch.bfloat16) / 10
score = torch.randn((m, e), device=DEVICE, dtype=torch.bfloat16)
if dtype.itemsize == 1:
act_dtype = torch.bfloat16
@@ -294,8 +305,8 @@ def test_fused_moe_batched_experts(
)
if input_scales and quant_dtype is not None:
a1_scale = torch.tensor(1, device="cuda", dtype=torch.float32)
a2_scale = torch.tensor(1, device="cuda", dtype=torch.float32)
a1_scale = torch.tensor(1, device=DEVICE, dtype=torch.float32)
a2_scale = torch.tensor(1, device=DEVICE, dtype=torch.float32)
else:
a1_scale = None
a2_scale = None
@@ -489,10 +500,7 @@ def test_batched_triton_experts_supports_current_device():
BatchedTritonExperts,
)
if current_platform.is_xpu() or current_platform.is_cuda_alike():
assert BatchedTritonExperts._supports_current_device()
else:
pytest.skip("No GPU device available")
assert BatchedTritonExperts._supports_current_device()
@pytest.mark.parametrize("m,n,k,e,topk", [(32, 512, 512, 8, 2), (45, 1024, 128, 8, 1)])
@@ -500,9 +508,6 @@ def test_batched_experts_end_to_end(m, n, k, e, topk):
"""End-to-end BatchedTritonExperts via the reference (no-comms)
BatchedPrepareAndFinalize, validated against a torch reference. Exercises
the device-enablement path."""
if not (current_platform.is_xpu() or current_platform.is_cuda_alike()):
pytest.skip("No GPU device available")
from vllm.v1.worker.workspace import init_workspace_manager
set_random_seed(7)
@@ -10,6 +10,14 @@ import pytest
import torch
from vllm.model_executor.layers.fused_moe.utils import count_expert_num_tokens
from vllm.platforms import current_platform
DEVICE = current_platform.device_type
pytestmark = pytest.mark.skipif(
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
reason="Triton MoE kernels require CUDA/ROCm/XPU.",
)
@dataclasses.dataclass
@@ -88,9 +96,9 @@ def do_test_compute_expert_num_tokens(
(num_local_experts), device="cpu", dtype=torch.int32
)
ref_impl(tt_rank, ref_expert_num_tokens)
ref_expert_num_tokens = ref_expert_num_tokens.to("cuda")
ref_expert_num_tokens = ref_expert_num_tokens.to(DEVICE)
tt_rank.to_device("cuda")
tt_rank.to_device(DEVICE)
# Test with expert_map
triton_expert_num_tokens_w_emap = count_expert_num_tokens(
tt_rank.topk_ids, num_local_experts, tt_rank.expert_map
@@ -13,6 +13,7 @@ from vllm.triton_utils import triton
from vllm.utils.deep_gemm import is_deep_gemm_e8m0_used
from vllm.utils.torch_utils import set_random_seed
DEVICE = current_platform.device_type
FLOAT8_DTYPE = torch.float8_e4m3fn
GROUP_SIZE = 128
@@ -61,7 +62,7 @@ def reference_quant(x: torch.Tensor, use_ue8m0: bool):
def reference(x: torch.Tensor, use_ue8m0: bool) -> tuple[torch.Tensor, torch.Tensor]:
T, N = x.size()
ref_act_out = torch.empty((T, N // 2), dtype=torch.bfloat16, device="cuda")
ref_act_out = torch.empty((T, N // 2), dtype=torch.bfloat16, device=DEVICE)
torch.ops._C.silu_and_mul(ref_act_out, x)
return reference_quant(ref_act_out, use_ue8m0)
@@ -93,7 +94,7 @@ def reference_with_clamp(
def test_silu_mul_fp8_quant_deep_gemm(T: int, N: int):
set_random_seed(42)
input = torch.rand((T, N), dtype=torch.bfloat16, device="cuda")
input = torch.rand((T, N), dtype=torch.bfloat16, device=DEVICE)
use_ue8m0 = is_deep_gemm_e8m0_used()
@@ -122,7 +123,7 @@ def test_silu_mul_fp8_quant_deep_gemm_clamp(T: int, N: int, clamp_limit: float):
# Use a wide distribution so values routinely exceed both clamp limits and
# the clamp branch is actually exercised (uniform [0, 1) inputs would never
# trigger it).
input = torch.randn((T, N), dtype=torch.bfloat16, device="cuda") * 8.0
input = torch.randn((T, N), dtype=torch.bfloat16, device=DEVICE) * 8.0
use_ue8m0 = is_deep_gemm_e8m0_used()
+15 -13
View File
@@ -23,6 +23,20 @@ from vllm.model_executor.layers.fused_moe.config import (
from vllm.model_executor.layers.fused_moe.experts.triton_moe import TritonExperts
from vllm.platforms import current_platform
DEVICE = current_platform.device_type
pytestmark = [
pytest.mark.skipif(
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
reason="Triton MoE kernels require a CUDA-alike or XPU device.",
),
pytest.mark.skipif(
current_platform.is_cuda_alike()
and not current_platform.has_device_capability(80),
reason="Triton MoE kernels require compute capability >= 8.0 on CUDA/ROCm.",
),
]
# Test parameters
M_SIZES = [1, 16, 64]
N_SIZES = [128, 256]
@@ -86,7 +100,7 @@ def make_test_tensors(
num_experts: int,
topk: int,
dtype: torch.dtype = torch.bfloat16,
device: str = "cuda",
device: str = DEVICE,
):
"""Create test tensors for MoE with non-gated activation.
@@ -106,10 +120,6 @@ def make_test_tensors(
return hidden_states, w1, w2, topk_weights, topk_ids
@pytest.mark.skipif(
not current_platform.has_device_capability(80),
reason="Requires compute capability >= 8.0",
)
@pytest.mark.parametrize("m", M_SIZES)
@pytest.mark.parametrize("n", N_SIZES)
@pytest.mark.parametrize("k", K_SIZES)
@@ -192,10 +202,6 @@ def test_triton_experts_no_mul_activation(
assert output.abs().sum() > 0, "Output is all zeros"
@pytest.mark.skipif(
not current_platform.has_device_capability(80),
reason="Requires compute capability >= 8.0",
)
@torch.inference_mode()
def test_workspace_shapes_no_mul_vs_gated():
"""Test that workspace shapes differ correctly between gated and non-gated."""
@@ -233,10 +239,6 @@ def test_workspace_shapes_no_mul_vs_gated():
assert out_no_mul == out_gated == (M, K)
@pytest.mark.skipif(
not current_platform.has_device_capability(80),
reason="Requires compute capability >= 8.0",
)
@torch.inference_mode()
def test_adjust_n_for_activation():
"""Test the adjust_N_for_activation method."""
+9 -7
View File
@@ -40,6 +40,8 @@ from vllm.platforms import current_platform
from vllm.utils.deep_gemm import per_block_cast_to_fp8
from vllm.utils.math_utils import round_up
DEVICE = current_platform.device_type
def shuffle_weight(w: torch.Tensor) -> torch.Tensor:
"""Fold weights to adjacent locations for Triton MoE / SwiGLU kernel layout."""
@@ -80,7 +82,7 @@ def make_dummy_moe_config(
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
activation=activation,
in_dtype=in_dtype,
device="cuda",
device=DEVICE,
routing_method=RoutingMethodType.TopK,
max_num_tokens=max_num_tokens,
)
@@ -238,7 +240,7 @@ def make_quantized_test_activations(
block_shape: list[int] | None = None,
per_act_token_quant: bool = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
a = torch.randn((E, m, k), device="cuda", dtype=in_dtype) / 10
a = torch.randn((E, m, k), device=DEVICE, dtype=in_dtype) / 10
a_q = a
a_scale = None
@@ -369,7 +371,7 @@ def make_test_weight(
block_shape: list[int] | None = None,
per_out_ch_quant: bool = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None, torch.Tensor | None]:
w_16 = torch.randn((e, rows, cols), device="cuda", dtype=in_dtype) / 15
w_16 = torch.randn((e, rows, cols), device=DEVICE, dtype=in_dtype) / 15
if quant_dtype is not None:
w, w_s, w_gs = moe_quantize_weights(
@@ -449,8 +451,8 @@ def make_test_quant_config(
a1_gscale: torch.Tensor | None = None
a2_gscale: torch.Tensor | None = None
if quant_dtype == "nvfp4":
a1_gscale = torch.ones((e,), device="cuda", dtype=torch.float32)
a2_gscale = torch.ones((e,), device="cuda", dtype=torch.float32)
a1_gscale = torch.ones((e,), device=DEVICE, dtype=torch.float32)
a2_gscale = torch.ones((e,), device=DEVICE, dtype=torch.float32)
a1_scale = a1_gscale
a2_scale = a2_gscale
else:
@@ -551,8 +553,8 @@ def make_naive_shared_experts(
K: int,
in_dtype: torch.dtype = torch.bfloat16,
) -> torch.nn.Module:
w1 = torch.randn((K, N * 2), device="cuda", dtype=in_dtype) / 15
w2 = torch.randn((N, K), device="cuda", dtype=in_dtype) / 15
w1 = torch.randn((K, N * 2), device=DEVICE, dtype=in_dtype) / 15
w2 = torch.randn((N, K), device=DEVICE, dtype=in_dtype) / 15
return TestMLP(w1, w2, out_dtype=in_dtype)
+2 -1
View File
@@ -13,10 +13,11 @@ from vllm.utils.deep_gemm import _ceil_to_ue8m0, is_deep_gemm_e8m0_used
from vllm.utils.math_utils import round_up
FP8_DTYPE = current_platform.fp8_dtype()
DEVICE = current_platform.device_type
def as_float32_tensor(x: float | torch.Tensor) -> torch.Tensor:
return torch.as_tensor(x, dtype=torch.float32, device="cuda")
return torch.as_tensor(x, dtype=torch.float32, device=DEVICE)
def ref_dynamic_per_token_quant(
@@ -18,19 +18,20 @@ from vllm.platforms import current_platform
@pytest.mark.parametrize("scale_ue8m0", [False, True])
@pytest.mark.parametrize("group_size", [64, 128])
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Only test on CUDA/ROCm."
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
reason="Only test on CUDA/ROCm/XPU.",
)
def test_per_token_group_quant_fp8(
shape, column_major: bool, tma_aligned: bool, scale_ue8m0: bool, group_size: int
):
device = "cuda"
device = current_platform.device_type
torch.manual_seed(42)
num_tokens, hidden_dim = shape
x = torch.randn((num_tokens, hidden_dim), device=device, dtype=torch.bfloat16) * 8
# cuda path
# native kernel path
out_q, scale = fp8_utils.per_token_group_quant_fp8(
x,
group_size,
@@ -40,7 +41,10 @@ def test_per_token_group_quant_fp8(
)
# triton ref
with patch("vllm.platforms.current_platform.is_cuda_alike", return_value=False):
with (
patch("vllm.platforms.current_platform.is_cuda_alike", return_value=False),
patch("vllm.platforms.current_platform.is_xpu", return_value=False),
):
ref_q, ref_s = fp8_utils.per_token_group_quant_fp8(
x,
group_size,
@@ -4,13 +4,20 @@
import pytest
import torch
from vllm.platforms import current_platform
from vllm.third_party.flash_linear_attention.ops import (
fused_recurrent_gated_delta_rule,
fused_recurrent_gated_delta_rule_packed_decode,
)
DEVICE = current_platform.device_type
pytestmark = pytest.mark.skipif(
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
reason="Gated delta rule Triton kernels require a CUDA-alike or XPU device.",
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Need CUDA device")
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
@pytest.mark.parametrize("strided_mixed_qkv", [False, True])
def test_fused_recurrent_packed_decode_matches_reference(
@@ -26,7 +33,7 @@ def test_fused_recurrent_packed_decode_matches_reference(
V = 128
qkv_dim = 2 * (H * K) + (HV * V)
device = torch.device("cuda")
device = torch.device(DEVICE)
if strided_mixed_qkv:
# Simulate a packed view into a larger projection buffer:
@@ -102,10 +109,9 @@ def test_fused_recurrent_packed_decode_matches_reference(
torch.testing.assert_close(state_packed, state_ref, rtol=rtol, atol=atol)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Need CUDA device")
def test_packed_decode_supports_large_batch_head_grid():
B, H, HV, K, V = 1024, 8, 64, 1, 1
device = torch.device("cuda")
device = torch.device(DEVICE)
gates = torch.empty((B, HV), device=device)
params = torch.empty((HV,), device=device)
out = torch.empty((B, 1, HV, V), device=device)
@@ -37,9 +37,10 @@ from vllm.triton_utils.allocation import set_triton_allocator
def _is_capturing_or_compiling() -> bool:
# torch.cuda.is_current_stream_capturing() is unavailable on non-CUDA (XPU) torch.
return torch.compiler.is_compiling() or (
current_platform.is_cuda_alike() and torch.cuda.is_current_stream_capturing()
# The accelerator API dispatches to the active backend.
return (
torch.compiler.is_compiling()
or torch.accelerator.current_stream().is_capturing()
)