[XPU][Tests] Make tests device-agnostic (#51968)

Signed-off-by: pmanczak <pawel.manczak@intel.com>
This commit is contained in:
pmanczak
2026-08-20 02:25:41 -07:00
committed by GitHub
parent 38e9cefdef
commit 44cf3f0466
12 changed files with 148 additions and 111 deletions
@@ -5,7 +5,7 @@ import pytest
import torch
from vllm._custom_ops import (
merge_attn_states as merge_attn_states_cuda,
merge_attn_states as merge_attn_states_native,
)
from vllm._custom_ops import (
scaled_fp8_quant,
@@ -15,11 +15,13 @@ from vllm.v1.attention.ops.triton_merge_attn_states import (
merge_attn_states as merge_attn_states_triton,
)
DEVICE = current_platform.device_type
pytestmark = [
pytest.mark.skip_global_cleanup,
pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="merge_attn_states kernels require CUDA or ROCm.",
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
reason="merge_attn_states kernels require CUDA, ROCm or XPU.",
),
]
@@ -81,20 +83,22 @@ DTYPES = [torch.float32, torch.half, torch.bfloat16]
all_case_info: list[tuple] = []
@pytest.mark.parametrize("merge_fn", [merge_attn_states_cuda, merge_attn_states_triton])
@pytest.mark.parametrize(
"merge_fn", [merge_attn_states_native, merge_attn_states_triton]
)
@pytest.mark.parametrize("output_dtype", [torch.float32, torch.half, torch.bfloat16])
def test_merge_attn_states_both_empty(merge_fn, output_dtype) -> None:
"""When a token is empty on both sides (both LSE -inf), the 0/0 softmax
scales must not surface as NaN in the merged output."""
num_tokens, num_heads, head_size = 6, 8, 128
prefix_output = torch.zeros(
num_tokens, num_heads, head_size, device="cuda", dtype=output_dtype
num_tokens, num_heads, head_size, device=DEVICE, dtype=output_dtype
)
prefix_lse = torch.randn(num_heads, num_tokens, device="cuda")
prefix_lse = torch.randn(num_heads, num_tokens, device=DEVICE)
suffix_output = torch.zeros(
num_tokens, num_heads, head_size, device="cuda", dtype=output_dtype
num_tokens, num_heads, head_size, device=DEVICE, dtype=output_dtype
)
suffix_lse = torch.randn(num_heads, num_tokens, device="cuda")
suffix_lse = torch.randn(num_heads, num_tokens, device=DEVICE)
# Tokens 2 and 3 are empty on both sides.
empty = slice(2, 4)
@@ -111,7 +115,7 @@ def generate_markdown_table():
global all_case_info
table_header = (
"| tokens | heads | headsize | dtype "
"| device | torch | triton | cuda | speedup |"
"| device | torch | triton | native | speedup |"
)
table_separator = "| --- | --- | --- | --- | --- | --- | --- | --- | --- |"
@@ -132,7 +136,7 @@ def generate_markdown_table():
device,
avg_time_torch_kernel,
avg_time_triton_kernel,
avg_time_cuda_kernel,
avg_time_native_kernel,
performance_improved,
) = info
dtype = shortly_dtype(dtype)
@@ -141,7 +145,7 @@ def generate_markdown_table():
f"| {num_tokens} | {num_heads} | {head_size} "
f"| {dtype} | {device} | {avg_time_torch_kernel:.5f}ms "
f"| {avg_time_triton_kernel:.5f}ms "
f"| {avg_time_cuda_kernel:.5f}ms "
f"| {avg_time_native_kernel:.5f}ms "
f"| {performance_improved:.4f}x |"
)
@@ -171,7 +175,7 @@ def test_merge_attn_states(
output_scale = None
if use_fp8:
output_dtype = current_platform.fp8_dtype()
output_scale = torch.tensor([0.05], dtype=torch.float32, device="cuda")
output_scale = torch.tensor([0.05], dtype=torch.float32, device=DEVICE)
print(
f"\nNUM_TOKENS:{NUM_TOKENS}, NUM_HEADS:{NUM_HEADS}, "
@@ -182,8 +186,8 @@ def test_merge_attn_states(
)
# prefix_lse and suffix_lse contain inf and normal values
prefix_lse = torch.randn(NUM_HEADS, NUM_TOKENS, dtype=torch.float32, device="cuda")
suffix_lse = torch.randn(NUM_HEADS, NUM_TOKENS, dtype=torch.float32, device="cuda")
prefix_lse = torch.randn(NUM_HEADS, NUM_TOKENS, dtype=torch.float32, device=DEVICE)
suffix_lse = torch.randn(NUM_HEADS, NUM_TOKENS, dtype=torch.float32, device=DEVICE)
# Generate boolean masks
mask_prefix = torch.rand(NUM_HEADS, NUM_TOKENS) < 0.1
@@ -199,16 +203,16 @@ def test_merge_attn_states(
# Other input tensors (need to be initialized but
# no actual calculation needed)
output = torch.zeros(
(NUM_TOKENS, NUM_HEADS, HEAD_SIZE), dtype=output_dtype, device="cuda"
(NUM_TOKENS, NUM_HEADS, HEAD_SIZE), dtype=output_dtype, device=DEVICE
)
output_lse = torch.zeros(
(NUM_HEADS, NUM_TOKENS), dtype=torch.float32, device="cuda"
(NUM_HEADS, NUM_TOKENS), dtype=torch.float32, device=DEVICE
)
prefix_output = torch.randn(
(NUM_TOKENS, NUM_HEADS, HEAD_SIZE), dtype=input_dtype, device="cuda"
(NUM_TOKENS, NUM_HEADS, HEAD_SIZE), dtype=input_dtype, device=DEVICE
)
suffix_output = torch.randn(
(NUM_TOKENS, NUM_HEADS, HEAD_SIZE), dtype=input_dtype, device="cuda"
(NUM_TOKENS, NUM_HEADS, HEAD_SIZE), dtype=input_dtype, device=DEVICE
)
warmup_times = 2
@@ -293,19 +297,19 @@ def test_merge_attn_states(
avg_time_triton_kernel = total_time_triton_kernel / repeat_times
# 2. Run the CUDA kernel
total_time_cuda_kernel = 0
output_cuda = output.clone()
output_lse_cuda = output_lse.clone()
# 2. Run the native (CUDA/XPU) kernel
total_time_native_kernel = 0
output_native = output.clone()
output_lse_native = output_lse.clone()
for _ in range(warmup_times):
merge_attn_states_cuda(
output_cuda,
merge_attn_states_native(
output_native,
prefix_output,
prefix_lse,
suffix_output,
suffix_lse,
output_lse_cuda,
output_lse_native,
prefill_tokens_with_context,
output_scale,
)
@@ -313,28 +317,28 @@ def test_merge_attn_states(
for _ in range(repeat_times):
start.record()
merge_attn_states_cuda(
output_cuda,
merge_attn_states_native(
output_native,
prefix_output,
prefix_lse,
suffix_output,
suffix_lse,
output_lse_cuda,
output_lse_native,
prefill_tokens_with_context,
output_scale,
)
end.record()
torch.accelerator.synchronize()
total_time_cuda_kernel += start.elapsed_time(end)
total_time_native_kernel += start.elapsed_time(end)
avg_time_cuda_kernel = total_time_cuda_kernel / repeat_times
avg_time_native_kernel = total_time_native_kernel / repeat_times
# 3. Performance compare
performance_improved = avg_time_triton_kernel / avg_time_cuda_kernel
performance_improved = avg_time_triton_kernel / avg_time_native_kernel
print(f" Torch time: {avg_time_torch_kernel:.6f}ms")
print(f"Triton time: {avg_time_triton_kernel:.6f}ms")
print(
f" CUDA time: {avg_time_cuda_kernel:.6f}ms, "
f"Native time: {avg_time_native_kernel:.6f}ms, "
f"Performance: {performance_improved:.5f}x"
)
print("-" * 100)
@@ -362,12 +366,12 @@ def test_merge_attn_states(
return max_diff
# Use Triton output as reference because we want to replace
# the Triton kernel with custom CUDA kernel for merge attn
# the Triton kernel with the custom native kernel for merge attn
# states operation.
output_ref = output_ref_triton
output_lse_ref = output_lse_ref_triton
torch.testing.assert_close(
output_cuda.float() * scale,
output_native.float() * scale,
output_ref.float() * scale,
atol=atol,
rtol=rtol,
@@ -379,19 +383,19 @@ def test_merge_attn_states(
)
_diff = diff(output_ref.float() * scale, output_torch.float() * scale)
print(f"(Triton vs Torch) : {_diff}")
_diff = diff(output_torch.float() * scale, output_cuda.float() * scale)
print(f" (CUDA vs Torch) : {_diff}")
_diff = diff(output_ref.float() * scale, output_cuda.float() * scale)
print(f" (CUDA vs Triton): {_diff}")
_diff = diff(output_torch.float() * scale, output_native.float() * scale)
print(f"(Native vs Torch) : {_diff}")
_diff = diff(output_ref.float() * scale, output_native.float() * scale)
print(f"(Native vs Triton): {_diff}")
print("-" * 100)
torch.testing.assert_close(
output_lse_cuda.float(), output_lse_ref.float(), atol=atol, rtol=rtol
output_lse_native.float(), output_lse_ref.float(), atol=atol, rtol=rtol
)
print("Output LSE all match, max abs diff:")
print(f"(Triton vs Torch) : {diff(output_lse_torch, output_lse_ref)}")
print(f" (CUDA vs Torch) : {diff(output_lse_torch, output_lse_cuda)}")
print(f" (CUDA vs Triton): {diff(output_lse_ref, output_lse_cuda)}")
print(f"(Native vs Torch) : {diff(output_lse_torch, output_lse_native)}")
print(f"(Native vs Triton): {diff(output_lse_ref, output_lse_native)}")
print("-" * 100)
print(
@@ -410,7 +414,7 @@ def test_merge_attn_states(
device,
avg_time_torch_kernel,
avg_time_triton_kernel,
avg_time_cuda_kernel,
avg_time_native_kernel,
performance_improved,
)
)
@@ -36,6 +36,7 @@ from vllm.model_executor.layers.mamba.ops.selective_state_update_replayssm_outpu
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
mamba_chunk_scan_combined_varlen,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.backends.mamba2_attn import compute_varlen_chunk_metadata
@@ -68,7 +69,7 @@ def _run_prefill_decode_equivalence(
flow (raw dt + softplus + a per-head dt_bias), so this also checks prefill
and decode apply the softplus/bias preprocessing consistently. ``state_dtype``
is the recurrent-state precision; ``act_dtype`` the activation/buffer one."""
device = "cuda"
device = current_platform.device_type
rtol, atol = _prefill_tolerances(act_dtype)
set_random_seed(seed)
@@ -203,7 +204,10 @@ def _run_prefill_decode_equivalence(
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Need CUDA device")
@pytest.mark.skipif(
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
reason="Mamba2 ReplaySSM kernels require a CUDA-alike or XPU device.",
)
@pytest.mark.parametrize(
"precision",
# fp32 state is the default; bf16/fp16 are reduced-footprint configs. fp16
+28 -23
View File
@@ -23,6 +23,13 @@ from vllm.platforms import current_platform
from vllm.triton_utils import tl
from vllm.utils.torch_utils import set_random_seed
DEVICE = current_platform.device_type
pytestmark = pytest.mark.skipif(
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
reason="Triton MoE kernels require CUDA/ROCm/XPU.",
)
MNK_FACTORS = [
(1, 128, 128),
(1, 512, 512),
@@ -71,19 +78,19 @@ class BatchedMMTensors:
A = (
torch.randn(
(config.num_experts, config.max_tokens_per_expert, config.K),
device="cuda",
device=DEVICE,
dtype=config.in_dtype,
)
/ 10
)
B = torch.randn(
(config.num_experts, config.N, config.K),
device="cuda",
device=DEVICE,
dtype=config.in_dtype,
)
C = torch.zeros(
(config.num_experts, config.max_tokens_per_expert, config.N),
device="cuda",
device=DEVICE,
dtype=config.out_dtype,
)
@@ -91,7 +98,7 @@ class BatchedMMTensors:
low=0,
high=config.max_tokens_per_expert,
size=(config.num_experts,),
device="cuda",
device=DEVICE,
dtype=torch.int32,
)
@@ -120,8 +127,10 @@ def test_batched_mm(
use_fp8_w8a8 = dtype == torch.float8_e4m3fn
if (dtype == torch.float8_e4m3fn) and not current_platform.has_device_capability(
89
if (
dtype == torch.float8_e4m3fn
and current_platform.is_cuda_alike()
and not current_platform.has_device_capability(89)
):
pytest.skip(
"Triton limitation: fp8e4nv data type is not supported on CUDA arch < 89"
@@ -144,7 +153,7 @@ def test_batched_mm(
low=0,
high=max_tokens_per_expert,
size=(num_experts,),
device="cuda",
device=DEVICE,
dtype=torch.int32,
)
@@ -169,9 +178,9 @@ def test_batched_mm(
)
out_shape = (num_experts, max_tokens_per_expert, N)
test_output = torch.zeros(out_shape, dtype=act_dtype, device="cuda")
ref_output = torch.zeros(out_shape, dtype=act_dtype, device="cuda")
q_ref_output = torch.zeros(out_shape, dtype=act_dtype, device="cuda")
test_output = torch.zeros(out_shape, dtype=act_dtype, device=DEVICE)
ref_output = torch.zeros(out_shape, dtype=act_dtype, device=DEVICE)
q_ref_output = torch.zeros(out_shape, dtype=act_dtype, device=DEVICE)
compute_tl_dtype = {
torch.float16: tl.float16,
@@ -257,8 +266,10 @@ def test_fused_moe_batched_experts(
use_fp8_w8a8 = dtype == torch.float8_e4m3fn
if (dtype == torch.float8_e4m3fn) and not current_platform.has_device_capability(
89
if (
dtype == torch.float8_e4m3fn
and current_platform.is_cuda_alike()
and not current_platform.has_device_capability(89)
):
pytest.skip(
"Triton limitation: fp8e4nv data type is not supported on CUDA arch < 89"
@@ -273,8 +284,8 @@ def test_fused_moe_batched_experts(
if per_act_token_quant and block_shape is not None:
pytest.skip("Skip illegal quantization test.")
a = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) / 10
score = torch.randn((m, e), device="cuda", dtype=torch.bfloat16)
a = torch.randn((m, k), device=DEVICE, dtype=torch.bfloat16) / 10
score = torch.randn((m, e), device=DEVICE, dtype=torch.bfloat16)
if dtype.itemsize == 1:
act_dtype = torch.bfloat16
@@ -294,8 +305,8 @@ def test_fused_moe_batched_experts(
)
if input_scales and quant_dtype is not None:
a1_scale = torch.tensor(1, device="cuda", dtype=torch.float32)
a2_scale = torch.tensor(1, device="cuda", dtype=torch.float32)
a1_scale = torch.tensor(1, device=DEVICE, dtype=torch.float32)
a2_scale = torch.tensor(1, device=DEVICE, dtype=torch.float32)
else:
a1_scale = None
a2_scale = None
@@ -489,10 +500,7 @@ def test_batched_triton_experts_supports_current_device():
BatchedTritonExperts,
)
if current_platform.is_xpu() or current_platform.is_cuda_alike():
assert BatchedTritonExperts._supports_current_device()
else:
pytest.skip("No GPU device available")
assert BatchedTritonExperts._supports_current_device()
@pytest.mark.parametrize("m,n,k,e,topk", [(32, 512, 512, 8, 2), (45, 1024, 128, 8, 1)])
@@ -500,9 +508,6 @@ def test_batched_experts_end_to_end(m, n, k, e, topk):
"""End-to-end BatchedTritonExperts via the reference (no-comms)
BatchedPrepareAndFinalize, validated against a torch reference. Exercises
the device-enablement path."""
if not (current_platform.is_xpu() or current_platform.is_cuda_alike()):
pytest.skip("No GPU device available")
from vllm.v1.worker.workspace import init_workspace_manager
set_random_seed(7)
@@ -10,6 +10,14 @@ import pytest
import torch
from vllm.model_executor.layers.fused_moe.utils import count_expert_num_tokens
from vllm.platforms import current_platform
DEVICE = current_platform.device_type
pytestmark = pytest.mark.skipif(
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
reason="Triton MoE kernels require CUDA/ROCm/XPU.",
)
@dataclasses.dataclass
@@ -88,9 +96,9 @@ def do_test_compute_expert_num_tokens(
(num_local_experts), device="cpu", dtype=torch.int32
)
ref_impl(tt_rank, ref_expert_num_tokens)
ref_expert_num_tokens = ref_expert_num_tokens.to("cuda")
ref_expert_num_tokens = ref_expert_num_tokens.to(DEVICE)
tt_rank.to_device("cuda")
tt_rank.to_device(DEVICE)
# Test with expert_map
triton_expert_num_tokens_w_emap = count_expert_num_tokens(
tt_rank.topk_ids, num_local_experts, tt_rank.expert_map
@@ -13,6 +13,7 @@ from vllm.triton_utils import triton
from vllm.utils.deep_gemm import is_deep_gemm_e8m0_used
from vllm.utils.torch_utils import set_random_seed
DEVICE = current_platform.device_type
FLOAT8_DTYPE = torch.float8_e4m3fn
GROUP_SIZE = 128
@@ -61,7 +62,7 @@ def reference_quant(x: torch.Tensor, use_ue8m0: bool):
def reference(x: torch.Tensor, use_ue8m0: bool) -> tuple[torch.Tensor, torch.Tensor]:
T, N = x.size()
ref_act_out = torch.empty((T, N // 2), dtype=torch.bfloat16, device="cuda")
ref_act_out = torch.empty((T, N // 2), dtype=torch.bfloat16, device=DEVICE)
torch.ops._C.silu_and_mul(ref_act_out, x)
return reference_quant(ref_act_out, use_ue8m0)
@@ -93,7 +94,7 @@ def reference_with_clamp(
def test_silu_mul_fp8_quant_deep_gemm(T: int, N: int):
set_random_seed(42)
input = torch.rand((T, N), dtype=torch.bfloat16, device="cuda")
input = torch.rand((T, N), dtype=torch.bfloat16, device=DEVICE)
use_ue8m0 = is_deep_gemm_e8m0_used()
@@ -122,7 +123,7 @@ def test_silu_mul_fp8_quant_deep_gemm_clamp(T: int, N: int, clamp_limit: float):
# Use a wide distribution so values routinely exceed both clamp limits and
# the clamp branch is actually exercised (uniform [0, 1) inputs would never
# trigger it).
input = torch.randn((T, N), dtype=torch.bfloat16, device="cuda") * 8.0
input = torch.randn((T, N), dtype=torch.bfloat16, device=DEVICE) * 8.0
use_ue8m0 = is_deep_gemm_e8m0_used()
+15 -13
View File
@@ -23,6 +23,20 @@ from vllm.model_executor.layers.fused_moe.config import (
from vllm.model_executor.layers.fused_moe.experts.triton_moe import TritonExperts
from vllm.platforms import current_platform
DEVICE = current_platform.device_type
pytestmark = [
pytest.mark.skipif(
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
reason="Triton MoE kernels require a CUDA-alike or XPU device.",
),
pytest.mark.skipif(
current_platform.is_cuda_alike()
and not current_platform.has_device_capability(80),
reason="Triton MoE kernels require compute capability >= 8.0 on CUDA/ROCm.",
),
]
# Test parameters
M_SIZES = [1, 16, 64]
N_SIZES = [128, 256]
@@ -86,7 +100,7 @@ def make_test_tensors(
num_experts: int,
topk: int,
dtype: torch.dtype = torch.bfloat16,
device: str = "cuda",
device: str = DEVICE,
):
"""Create test tensors for MoE with non-gated activation.
@@ -106,10 +120,6 @@ def make_test_tensors(
return hidden_states, w1, w2, topk_weights, topk_ids
@pytest.mark.skipif(
not current_platform.has_device_capability(80),
reason="Requires compute capability >= 8.0",
)
@pytest.mark.parametrize("m", M_SIZES)
@pytest.mark.parametrize("n", N_SIZES)
@pytest.mark.parametrize("k", K_SIZES)
@@ -192,10 +202,6 @@ def test_triton_experts_no_mul_activation(
assert output.abs().sum() > 0, "Output is all zeros"
@pytest.mark.skipif(
not current_platform.has_device_capability(80),
reason="Requires compute capability >= 8.0",
)
@torch.inference_mode()
def test_workspace_shapes_no_mul_vs_gated():
"""Test that workspace shapes differ correctly between gated and non-gated."""
@@ -233,10 +239,6 @@ def test_workspace_shapes_no_mul_vs_gated():
assert out_no_mul == out_gated == (M, K)
@pytest.mark.skipif(
not current_platform.has_device_capability(80),
reason="Requires compute capability >= 8.0",
)
@torch.inference_mode()
def test_adjust_n_for_activation():
"""Test the adjust_N_for_activation method."""
+9 -7
View File
@@ -40,6 +40,8 @@ from vllm.platforms import current_platform
from vllm.utils.deep_gemm import per_block_cast_to_fp8
from vllm.utils.math_utils import round_up
DEVICE = current_platform.device_type
def shuffle_weight(w: torch.Tensor) -> torch.Tensor:
"""Fold weights to adjacent locations for Triton MoE / SwiGLU kernel layout."""
@@ -80,7 +82,7 @@ def make_dummy_moe_config(
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
activation=activation,
in_dtype=in_dtype,
device="cuda",
device=DEVICE,
routing_method=RoutingMethodType.TopK,
max_num_tokens=max_num_tokens,
)
@@ -238,7 +240,7 @@ def make_quantized_test_activations(
block_shape: list[int] | None = None,
per_act_token_quant: bool = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
a = torch.randn((E, m, k), device="cuda", dtype=in_dtype) / 10
a = torch.randn((E, m, k), device=DEVICE, dtype=in_dtype) / 10
a_q = a
a_scale = None
@@ -369,7 +371,7 @@ def make_test_weight(
block_shape: list[int] | None = None,
per_out_ch_quant: bool = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None, torch.Tensor | None]:
w_16 = torch.randn((e, rows, cols), device="cuda", dtype=in_dtype) / 15
w_16 = torch.randn((e, rows, cols), device=DEVICE, dtype=in_dtype) / 15
if quant_dtype is not None:
w, w_s, w_gs = moe_quantize_weights(
@@ -449,8 +451,8 @@ def make_test_quant_config(
a1_gscale: torch.Tensor | None = None
a2_gscale: torch.Tensor | None = None
if quant_dtype == "nvfp4":
a1_gscale = torch.ones((e,), device="cuda", dtype=torch.float32)
a2_gscale = torch.ones((e,), device="cuda", dtype=torch.float32)
a1_gscale = torch.ones((e,), device=DEVICE, dtype=torch.float32)
a2_gscale = torch.ones((e,), device=DEVICE, dtype=torch.float32)
a1_scale = a1_gscale
a2_scale = a2_gscale
else:
@@ -551,8 +553,8 @@ def make_naive_shared_experts(
K: int,
in_dtype: torch.dtype = torch.bfloat16,
) -> torch.nn.Module:
w1 = torch.randn((K, N * 2), device="cuda", dtype=in_dtype) / 15
w2 = torch.randn((N, K), device="cuda", dtype=in_dtype) / 15
w1 = torch.randn((K, N * 2), device=DEVICE, dtype=in_dtype) / 15
w2 = torch.randn((N, K), device=DEVICE, dtype=in_dtype) / 15
return TestMLP(w1, w2, out_dtype=in_dtype)
+2 -1
View File
@@ -13,10 +13,11 @@ from vllm.utils.deep_gemm import _ceil_to_ue8m0, is_deep_gemm_e8m0_used
from vllm.utils.math_utils import round_up
FP8_DTYPE = current_platform.fp8_dtype()
DEVICE = current_platform.device_type
def as_float32_tensor(x: float | torch.Tensor) -> torch.Tensor:
return torch.as_tensor(x, dtype=torch.float32, device="cuda")
return torch.as_tensor(x, dtype=torch.float32, device=DEVICE)
def ref_dynamic_per_token_quant(
@@ -18,19 +18,20 @@ from vllm.platforms import current_platform
@pytest.mark.parametrize("scale_ue8m0", [False, True])
@pytest.mark.parametrize("group_size", [64, 128])
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Only test on CUDA/ROCm."
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
reason="Only test on CUDA/ROCm/XPU.",
)
def test_per_token_group_quant_fp8(
shape, column_major: bool, tma_aligned: bool, scale_ue8m0: bool, group_size: int
):
device = "cuda"
device = current_platform.device_type
torch.manual_seed(42)
num_tokens, hidden_dim = shape
x = torch.randn((num_tokens, hidden_dim), device=device, dtype=torch.bfloat16) * 8
# cuda path
# native kernel path
out_q, scale = fp8_utils.per_token_group_quant_fp8(
x,
group_size,
@@ -40,7 +41,10 @@ def test_per_token_group_quant_fp8(
)
# triton ref
with patch("vllm.platforms.current_platform.is_cuda_alike", return_value=False):
with (
patch("vllm.platforms.current_platform.is_cuda_alike", return_value=False),
patch("vllm.platforms.current_platform.is_xpu", return_value=False),
):
ref_q, ref_s = fp8_utils.per_token_group_quant_fp8(
x,
group_size,
@@ -4,13 +4,20 @@
import pytest
import torch
from vllm.platforms import current_platform
from vllm.third_party.flash_linear_attention.ops import (
fused_recurrent_gated_delta_rule,
fused_recurrent_gated_delta_rule_packed_decode,
)
DEVICE = current_platform.device_type
pytestmark = pytest.mark.skipif(
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
reason="Gated delta rule Triton kernels require a CUDA-alike or XPU device.",
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Need CUDA device")
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
@pytest.mark.parametrize("strided_mixed_qkv", [False, True])
def test_fused_recurrent_packed_decode_matches_reference(
@@ -26,7 +33,7 @@ def test_fused_recurrent_packed_decode_matches_reference(
V = 128
qkv_dim = 2 * (H * K) + (HV * V)
device = torch.device("cuda")
device = torch.device(DEVICE)
if strided_mixed_qkv:
# Simulate a packed view into a larger projection buffer:
@@ -102,10 +109,9 @@ def test_fused_recurrent_packed_decode_matches_reference(
torch.testing.assert_close(state_packed, state_ref, rtol=rtol, atol=atol)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Need CUDA device")
def test_packed_decode_supports_large_batch_head_grid():
B, H, HV, K, V = 1024, 8, 64, 1, 1
device = torch.device("cuda")
device = torch.device(DEVICE)
gates = torch.empty((B, HV), device=device)
params = torch.empty((HV,), device=device)
out = torch.empty((B, 1, HV, V), device=device)