[XPU][Tests] Make tests device-agnostic (#51968)
Signed-off-by: pmanczak <pawel.manczak@intel.com>
This commit is contained in:
+6
-7
@@ -284,19 +284,18 @@ def cleanup_fixture(should_do_global_cleanup_after_test: bool):
|
||||
def workspace_init():
|
||||
"""Initialize the workspace manager for tests that need it.
|
||||
|
||||
This fixture initializes the workspace manager with a CUDA device
|
||||
if available, and resets it after the test completes. Tests that
|
||||
create a full vLLM engine should NOT use this fixture as the engine
|
||||
will initialize the workspace manager itself.
|
||||
This fixture initializes the workspace manager with the current
|
||||
platform's accelerator device if available, and resets it after the test
|
||||
completes. Tests that create a full vLLM engine should NOT use this
|
||||
fixture as the engine will initialize the workspace manager itself.
|
||||
"""
|
||||
from vllm.v1.worker.workspace import (
|
||||
init_workspace_manager,
|
||||
reset_workspace_manager,
|
||||
)
|
||||
|
||||
if torch.cuda.is_available():
|
||||
device = torch.device("cuda:0")
|
||||
init_workspace_manager(device)
|
||||
if torch.accelerator.is_available():
|
||||
init_workspace_manager(torch.device(0))
|
||||
yield
|
||||
reset_workspace_manager()
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm._custom_ops import (
|
||||
merge_attn_states as merge_attn_states_cuda,
|
||||
merge_attn_states as merge_attn_states_native,
|
||||
)
|
||||
from vllm._custom_ops import (
|
||||
scaled_fp8_quant,
|
||||
@@ -15,11 +15,13 @@ from vllm.v1.attention.ops.triton_merge_attn_states import (
|
||||
merge_attn_states as merge_attn_states_triton,
|
||||
)
|
||||
|
||||
DEVICE = current_platform.device_type
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.skip_global_cleanup,
|
||||
pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="merge_attn_states kernels require CUDA or ROCm.",
|
||||
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
|
||||
reason="merge_attn_states kernels require CUDA, ROCm or XPU.",
|
||||
),
|
||||
]
|
||||
|
||||
@@ -81,20 +83,22 @@ DTYPES = [torch.float32, torch.half, torch.bfloat16]
|
||||
all_case_info: list[tuple] = []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("merge_fn", [merge_attn_states_cuda, merge_attn_states_triton])
|
||||
@pytest.mark.parametrize(
|
||||
"merge_fn", [merge_attn_states_native, merge_attn_states_triton]
|
||||
)
|
||||
@pytest.mark.parametrize("output_dtype", [torch.float32, torch.half, torch.bfloat16])
|
||||
def test_merge_attn_states_both_empty(merge_fn, output_dtype) -> None:
|
||||
"""When a token is empty on both sides (both LSE -inf), the 0/0 softmax
|
||||
scales must not surface as NaN in the merged output."""
|
||||
num_tokens, num_heads, head_size = 6, 8, 128
|
||||
prefix_output = torch.zeros(
|
||||
num_tokens, num_heads, head_size, device="cuda", dtype=output_dtype
|
||||
num_tokens, num_heads, head_size, device=DEVICE, dtype=output_dtype
|
||||
)
|
||||
prefix_lse = torch.randn(num_heads, num_tokens, device="cuda")
|
||||
prefix_lse = torch.randn(num_heads, num_tokens, device=DEVICE)
|
||||
suffix_output = torch.zeros(
|
||||
num_tokens, num_heads, head_size, device="cuda", dtype=output_dtype
|
||||
num_tokens, num_heads, head_size, device=DEVICE, dtype=output_dtype
|
||||
)
|
||||
suffix_lse = torch.randn(num_heads, num_tokens, device="cuda")
|
||||
suffix_lse = torch.randn(num_heads, num_tokens, device=DEVICE)
|
||||
|
||||
# Tokens 2 and 3 are empty on both sides.
|
||||
empty = slice(2, 4)
|
||||
@@ -111,7 +115,7 @@ def generate_markdown_table():
|
||||
global all_case_info
|
||||
table_header = (
|
||||
"| tokens | heads | headsize | dtype "
|
||||
"| device | torch | triton | cuda | speedup |"
|
||||
"| device | torch | triton | native | speedup |"
|
||||
)
|
||||
table_separator = "| --- | --- | --- | --- | --- | --- | --- | --- | --- |"
|
||||
|
||||
@@ -132,7 +136,7 @@ def generate_markdown_table():
|
||||
device,
|
||||
avg_time_torch_kernel,
|
||||
avg_time_triton_kernel,
|
||||
avg_time_cuda_kernel,
|
||||
avg_time_native_kernel,
|
||||
performance_improved,
|
||||
) = info
|
||||
dtype = shortly_dtype(dtype)
|
||||
@@ -141,7 +145,7 @@ def generate_markdown_table():
|
||||
f"| {num_tokens} | {num_heads} | {head_size} "
|
||||
f"| {dtype} | {device} | {avg_time_torch_kernel:.5f}ms "
|
||||
f"| {avg_time_triton_kernel:.5f}ms "
|
||||
f"| {avg_time_cuda_kernel:.5f}ms "
|
||||
f"| {avg_time_native_kernel:.5f}ms "
|
||||
f"| {performance_improved:.4f}x |"
|
||||
)
|
||||
|
||||
@@ -171,7 +175,7 @@ def test_merge_attn_states(
|
||||
output_scale = None
|
||||
if use_fp8:
|
||||
output_dtype = current_platform.fp8_dtype()
|
||||
output_scale = torch.tensor([0.05], dtype=torch.float32, device="cuda")
|
||||
output_scale = torch.tensor([0.05], dtype=torch.float32, device=DEVICE)
|
||||
|
||||
print(
|
||||
f"\nNUM_TOKENS:{NUM_TOKENS}, NUM_HEADS:{NUM_HEADS}, "
|
||||
@@ -182,8 +186,8 @@ def test_merge_attn_states(
|
||||
)
|
||||
|
||||
# prefix_lse and suffix_lse contain inf and normal values
|
||||
prefix_lse = torch.randn(NUM_HEADS, NUM_TOKENS, dtype=torch.float32, device="cuda")
|
||||
suffix_lse = torch.randn(NUM_HEADS, NUM_TOKENS, dtype=torch.float32, device="cuda")
|
||||
prefix_lse = torch.randn(NUM_HEADS, NUM_TOKENS, dtype=torch.float32, device=DEVICE)
|
||||
suffix_lse = torch.randn(NUM_HEADS, NUM_TOKENS, dtype=torch.float32, device=DEVICE)
|
||||
|
||||
# Generate boolean masks
|
||||
mask_prefix = torch.rand(NUM_HEADS, NUM_TOKENS) < 0.1
|
||||
@@ -199,16 +203,16 @@ def test_merge_attn_states(
|
||||
# Other input tensors (need to be initialized but
|
||||
# no actual calculation needed)
|
||||
output = torch.zeros(
|
||||
(NUM_TOKENS, NUM_HEADS, HEAD_SIZE), dtype=output_dtype, device="cuda"
|
||||
(NUM_TOKENS, NUM_HEADS, HEAD_SIZE), dtype=output_dtype, device=DEVICE
|
||||
)
|
||||
output_lse = torch.zeros(
|
||||
(NUM_HEADS, NUM_TOKENS), dtype=torch.float32, device="cuda"
|
||||
(NUM_HEADS, NUM_TOKENS), dtype=torch.float32, device=DEVICE
|
||||
)
|
||||
prefix_output = torch.randn(
|
||||
(NUM_TOKENS, NUM_HEADS, HEAD_SIZE), dtype=input_dtype, device="cuda"
|
||||
(NUM_TOKENS, NUM_HEADS, HEAD_SIZE), dtype=input_dtype, device=DEVICE
|
||||
)
|
||||
suffix_output = torch.randn(
|
||||
(NUM_TOKENS, NUM_HEADS, HEAD_SIZE), dtype=input_dtype, device="cuda"
|
||||
(NUM_TOKENS, NUM_HEADS, HEAD_SIZE), dtype=input_dtype, device=DEVICE
|
||||
)
|
||||
|
||||
warmup_times = 2
|
||||
@@ -293,19 +297,19 @@ def test_merge_attn_states(
|
||||
|
||||
avg_time_triton_kernel = total_time_triton_kernel / repeat_times
|
||||
|
||||
# 2. Run the CUDA kernel
|
||||
total_time_cuda_kernel = 0
|
||||
output_cuda = output.clone()
|
||||
output_lse_cuda = output_lse.clone()
|
||||
# 2. Run the native (CUDA/XPU) kernel
|
||||
total_time_native_kernel = 0
|
||||
output_native = output.clone()
|
||||
output_lse_native = output_lse.clone()
|
||||
|
||||
for _ in range(warmup_times):
|
||||
merge_attn_states_cuda(
|
||||
output_cuda,
|
||||
merge_attn_states_native(
|
||||
output_native,
|
||||
prefix_output,
|
||||
prefix_lse,
|
||||
suffix_output,
|
||||
suffix_lse,
|
||||
output_lse_cuda,
|
||||
output_lse_native,
|
||||
prefill_tokens_with_context,
|
||||
output_scale,
|
||||
)
|
||||
@@ -313,28 +317,28 @@ def test_merge_attn_states(
|
||||
|
||||
for _ in range(repeat_times):
|
||||
start.record()
|
||||
merge_attn_states_cuda(
|
||||
output_cuda,
|
||||
merge_attn_states_native(
|
||||
output_native,
|
||||
prefix_output,
|
||||
prefix_lse,
|
||||
suffix_output,
|
||||
suffix_lse,
|
||||
output_lse_cuda,
|
||||
output_lse_native,
|
||||
prefill_tokens_with_context,
|
||||
output_scale,
|
||||
)
|
||||
end.record()
|
||||
torch.accelerator.synchronize()
|
||||
total_time_cuda_kernel += start.elapsed_time(end)
|
||||
total_time_native_kernel += start.elapsed_time(end)
|
||||
|
||||
avg_time_cuda_kernel = total_time_cuda_kernel / repeat_times
|
||||
avg_time_native_kernel = total_time_native_kernel / repeat_times
|
||||
|
||||
# 3. Performance compare
|
||||
performance_improved = avg_time_triton_kernel / avg_time_cuda_kernel
|
||||
performance_improved = avg_time_triton_kernel / avg_time_native_kernel
|
||||
print(f" Torch time: {avg_time_torch_kernel:.6f}ms")
|
||||
print(f"Triton time: {avg_time_triton_kernel:.6f}ms")
|
||||
print(
|
||||
f" CUDA time: {avg_time_cuda_kernel:.6f}ms, "
|
||||
f"Native time: {avg_time_native_kernel:.6f}ms, "
|
||||
f"Performance: {performance_improved:.5f}x"
|
||||
)
|
||||
print("-" * 100)
|
||||
@@ -362,12 +366,12 @@ def test_merge_attn_states(
|
||||
return max_diff
|
||||
|
||||
# Use Triton output as reference because we want to replace
|
||||
# the Triton kernel with custom CUDA kernel for merge attn
|
||||
# the Triton kernel with the custom native kernel for merge attn
|
||||
# states operation.
|
||||
output_ref = output_ref_triton
|
||||
output_lse_ref = output_lse_ref_triton
|
||||
torch.testing.assert_close(
|
||||
output_cuda.float() * scale,
|
||||
output_native.float() * scale,
|
||||
output_ref.float() * scale,
|
||||
atol=atol,
|
||||
rtol=rtol,
|
||||
@@ -379,19 +383,19 @@ def test_merge_attn_states(
|
||||
)
|
||||
_diff = diff(output_ref.float() * scale, output_torch.float() * scale)
|
||||
print(f"(Triton vs Torch) : {_diff}")
|
||||
_diff = diff(output_torch.float() * scale, output_cuda.float() * scale)
|
||||
print(f" (CUDA vs Torch) : {_diff}")
|
||||
_diff = diff(output_ref.float() * scale, output_cuda.float() * scale)
|
||||
print(f" (CUDA vs Triton): {_diff}")
|
||||
_diff = diff(output_torch.float() * scale, output_native.float() * scale)
|
||||
print(f"(Native vs Torch) : {_diff}")
|
||||
_diff = diff(output_ref.float() * scale, output_native.float() * scale)
|
||||
print(f"(Native vs Triton): {_diff}")
|
||||
print("-" * 100)
|
||||
|
||||
torch.testing.assert_close(
|
||||
output_lse_cuda.float(), output_lse_ref.float(), atol=atol, rtol=rtol
|
||||
output_lse_native.float(), output_lse_ref.float(), atol=atol, rtol=rtol
|
||||
)
|
||||
print("Output LSE all match, max abs diff:")
|
||||
print(f"(Triton vs Torch) : {diff(output_lse_torch, output_lse_ref)}")
|
||||
print(f" (CUDA vs Torch) : {diff(output_lse_torch, output_lse_cuda)}")
|
||||
print(f" (CUDA vs Triton): {diff(output_lse_ref, output_lse_cuda)}")
|
||||
print(f"(Native vs Torch) : {diff(output_lse_torch, output_lse_native)}")
|
||||
print(f"(Native vs Triton): {diff(output_lse_ref, output_lse_native)}")
|
||||
print("-" * 100)
|
||||
|
||||
print(
|
||||
@@ -410,7 +414,7 @@ def test_merge_attn_states(
|
||||
device,
|
||||
avg_time_torch_kernel,
|
||||
avg_time_triton_kernel,
|
||||
avg_time_cuda_kernel,
|
||||
avg_time_native_kernel,
|
||||
performance_improved,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -36,6 +36,7 @@ from vllm.model_executor.layers.mamba.ops.selective_state_update_replayssm_outpu
|
||||
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
|
||||
mamba_chunk_scan_combined_varlen,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.backends.mamba2_attn import compute_varlen_chunk_metadata
|
||||
|
||||
@@ -68,7 +69,7 @@ def _run_prefill_decode_equivalence(
|
||||
flow (raw dt + softplus + a per-head dt_bias), so this also checks prefill
|
||||
and decode apply the softplus/bias preprocessing consistently. ``state_dtype``
|
||||
is the recurrent-state precision; ``act_dtype`` the activation/buffer one."""
|
||||
device = "cuda"
|
||||
device = current_platform.device_type
|
||||
rtol, atol = _prefill_tolerances(act_dtype)
|
||||
set_random_seed(seed)
|
||||
|
||||
@@ -203,7 +204,10 @@ def _run_prefill_decode_equivalence(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Need CUDA device")
|
||||
@pytest.mark.skipif(
|
||||
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
|
||||
reason="Mamba2 ReplaySSM kernels require a CUDA-alike or XPU device.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"precision",
|
||||
# fp32 state is the default; bf16/fp16 are reduced-footprint configs. fp16
|
||||
|
||||
@@ -23,6 +23,13 @@ from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
DEVICE = current_platform.device_type
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
|
||||
reason="Triton MoE kernels require CUDA/ROCm/XPU.",
|
||||
)
|
||||
|
||||
MNK_FACTORS = [
|
||||
(1, 128, 128),
|
||||
(1, 512, 512),
|
||||
@@ -71,19 +78,19 @@ class BatchedMMTensors:
|
||||
A = (
|
||||
torch.randn(
|
||||
(config.num_experts, config.max_tokens_per_expert, config.K),
|
||||
device="cuda",
|
||||
device=DEVICE,
|
||||
dtype=config.in_dtype,
|
||||
)
|
||||
/ 10
|
||||
)
|
||||
B = torch.randn(
|
||||
(config.num_experts, config.N, config.K),
|
||||
device="cuda",
|
||||
device=DEVICE,
|
||||
dtype=config.in_dtype,
|
||||
)
|
||||
C = torch.zeros(
|
||||
(config.num_experts, config.max_tokens_per_expert, config.N),
|
||||
device="cuda",
|
||||
device=DEVICE,
|
||||
dtype=config.out_dtype,
|
||||
)
|
||||
|
||||
@@ -91,7 +98,7 @@ class BatchedMMTensors:
|
||||
low=0,
|
||||
high=config.max_tokens_per_expert,
|
||||
size=(config.num_experts,),
|
||||
device="cuda",
|
||||
device=DEVICE,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
@@ -120,8 +127,10 @@ def test_batched_mm(
|
||||
|
||||
use_fp8_w8a8 = dtype == torch.float8_e4m3fn
|
||||
|
||||
if (dtype == torch.float8_e4m3fn) and not current_platform.has_device_capability(
|
||||
89
|
||||
if (
|
||||
dtype == torch.float8_e4m3fn
|
||||
and current_platform.is_cuda_alike()
|
||||
and not current_platform.has_device_capability(89)
|
||||
):
|
||||
pytest.skip(
|
||||
"Triton limitation: fp8e4nv data type is not supported on CUDA arch < 89"
|
||||
@@ -144,7 +153,7 @@ def test_batched_mm(
|
||||
low=0,
|
||||
high=max_tokens_per_expert,
|
||||
size=(num_experts,),
|
||||
device="cuda",
|
||||
device=DEVICE,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
@@ -169,9 +178,9 @@ def test_batched_mm(
|
||||
)
|
||||
|
||||
out_shape = (num_experts, max_tokens_per_expert, N)
|
||||
test_output = torch.zeros(out_shape, dtype=act_dtype, device="cuda")
|
||||
ref_output = torch.zeros(out_shape, dtype=act_dtype, device="cuda")
|
||||
q_ref_output = torch.zeros(out_shape, dtype=act_dtype, device="cuda")
|
||||
test_output = torch.zeros(out_shape, dtype=act_dtype, device=DEVICE)
|
||||
ref_output = torch.zeros(out_shape, dtype=act_dtype, device=DEVICE)
|
||||
q_ref_output = torch.zeros(out_shape, dtype=act_dtype, device=DEVICE)
|
||||
|
||||
compute_tl_dtype = {
|
||||
torch.float16: tl.float16,
|
||||
@@ -257,8 +266,10 @@ def test_fused_moe_batched_experts(
|
||||
|
||||
use_fp8_w8a8 = dtype == torch.float8_e4m3fn
|
||||
|
||||
if (dtype == torch.float8_e4m3fn) and not current_platform.has_device_capability(
|
||||
89
|
||||
if (
|
||||
dtype == torch.float8_e4m3fn
|
||||
and current_platform.is_cuda_alike()
|
||||
and not current_platform.has_device_capability(89)
|
||||
):
|
||||
pytest.skip(
|
||||
"Triton limitation: fp8e4nv data type is not supported on CUDA arch < 89"
|
||||
@@ -273,8 +284,8 @@ def test_fused_moe_batched_experts(
|
||||
if per_act_token_quant and block_shape is not None:
|
||||
pytest.skip("Skip illegal quantization test.")
|
||||
|
||||
a = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) / 10
|
||||
score = torch.randn((m, e), device="cuda", dtype=torch.bfloat16)
|
||||
a = torch.randn((m, k), device=DEVICE, dtype=torch.bfloat16) / 10
|
||||
score = torch.randn((m, e), device=DEVICE, dtype=torch.bfloat16)
|
||||
|
||||
if dtype.itemsize == 1:
|
||||
act_dtype = torch.bfloat16
|
||||
@@ -294,8 +305,8 @@ def test_fused_moe_batched_experts(
|
||||
)
|
||||
|
||||
if input_scales and quant_dtype is not None:
|
||||
a1_scale = torch.tensor(1, device="cuda", dtype=torch.float32)
|
||||
a2_scale = torch.tensor(1, device="cuda", dtype=torch.float32)
|
||||
a1_scale = torch.tensor(1, device=DEVICE, dtype=torch.float32)
|
||||
a2_scale = torch.tensor(1, device=DEVICE, dtype=torch.float32)
|
||||
else:
|
||||
a1_scale = None
|
||||
a2_scale = None
|
||||
@@ -489,10 +500,7 @@ def test_batched_triton_experts_supports_current_device():
|
||||
BatchedTritonExperts,
|
||||
)
|
||||
|
||||
if current_platform.is_xpu() or current_platform.is_cuda_alike():
|
||||
assert BatchedTritonExperts._supports_current_device()
|
||||
else:
|
||||
pytest.skip("No GPU device available")
|
||||
assert BatchedTritonExperts._supports_current_device()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,n,k,e,topk", [(32, 512, 512, 8, 2), (45, 1024, 128, 8, 1)])
|
||||
@@ -500,9 +508,6 @@ def test_batched_experts_end_to_end(m, n, k, e, topk):
|
||||
"""End-to-end BatchedTritonExperts via the reference (no-comms)
|
||||
BatchedPrepareAndFinalize, validated against a torch reference. Exercises
|
||||
the device-enablement path."""
|
||||
if not (current_platform.is_xpu() or current_platform.is_cuda_alike()):
|
||||
pytest.skip("No GPU device available")
|
||||
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
set_random_seed(7)
|
||||
|
||||
@@ -10,6 +10,14 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.utils import count_expert_num_tokens
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
DEVICE = current_platform.device_type
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
|
||||
reason="Triton MoE kernels require CUDA/ROCm/XPU.",
|
||||
)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
@@ -88,9 +96,9 @@ def do_test_compute_expert_num_tokens(
|
||||
(num_local_experts), device="cpu", dtype=torch.int32
|
||||
)
|
||||
ref_impl(tt_rank, ref_expert_num_tokens)
|
||||
ref_expert_num_tokens = ref_expert_num_tokens.to("cuda")
|
||||
ref_expert_num_tokens = ref_expert_num_tokens.to(DEVICE)
|
||||
|
||||
tt_rank.to_device("cuda")
|
||||
tt_rank.to_device(DEVICE)
|
||||
# Test with expert_map
|
||||
triton_expert_num_tokens_w_emap = count_expert_num_tokens(
|
||||
tt_rank.topk_ids, num_local_experts, tt_rank.expert_map
|
||||
|
||||
@@ -13,6 +13,7 @@ from vllm.triton_utils import triton
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_e8m0_used
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
DEVICE = current_platform.device_type
|
||||
FLOAT8_DTYPE = torch.float8_e4m3fn
|
||||
GROUP_SIZE = 128
|
||||
|
||||
@@ -61,7 +62,7 @@ def reference_quant(x: torch.Tensor, use_ue8m0: bool):
|
||||
|
||||
def reference(x: torch.Tensor, use_ue8m0: bool) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
T, N = x.size()
|
||||
ref_act_out = torch.empty((T, N // 2), dtype=torch.bfloat16, device="cuda")
|
||||
ref_act_out = torch.empty((T, N // 2), dtype=torch.bfloat16, device=DEVICE)
|
||||
torch.ops._C.silu_and_mul(ref_act_out, x)
|
||||
return reference_quant(ref_act_out, use_ue8m0)
|
||||
|
||||
@@ -93,7 +94,7 @@ def reference_with_clamp(
|
||||
def test_silu_mul_fp8_quant_deep_gemm(T: int, N: int):
|
||||
set_random_seed(42)
|
||||
|
||||
input = torch.rand((T, N), dtype=torch.bfloat16, device="cuda")
|
||||
input = torch.rand((T, N), dtype=torch.bfloat16, device=DEVICE)
|
||||
|
||||
use_ue8m0 = is_deep_gemm_e8m0_used()
|
||||
|
||||
@@ -122,7 +123,7 @@ def test_silu_mul_fp8_quant_deep_gemm_clamp(T: int, N: int, clamp_limit: float):
|
||||
# Use a wide distribution so values routinely exceed both clamp limits and
|
||||
# the clamp branch is actually exercised (uniform [0, 1) inputs would never
|
||||
# trigger it).
|
||||
input = torch.randn((T, N), dtype=torch.bfloat16, device="cuda") * 8.0
|
||||
input = torch.randn((T, N), dtype=torch.bfloat16, device=DEVICE) * 8.0
|
||||
|
||||
use_ue8m0 = is_deep_gemm_e8m0_used()
|
||||
|
||||
|
||||
@@ -23,6 +23,20 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_moe import TritonExperts
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
DEVICE = current_platform.device_type
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.skipif(
|
||||
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
|
||||
reason="Triton MoE kernels require a CUDA-alike or XPU device.",
|
||||
),
|
||||
pytest.mark.skipif(
|
||||
current_platform.is_cuda_alike()
|
||||
and not current_platform.has_device_capability(80),
|
||||
reason="Triton MoE kernels require compute capability >= 8.0 on CUDA/ROCm.",
|
||||
),
|
||||
]
|
||||
|
||||
# Test parameters
|
||||
M_SIZES = [1, 16, 64]
|
||||
N_SIZES = [128, 256]
|
||||
@@ -86,7 +100,7 @@ def make_test_tensors(
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
dtype: torch.dtype = torch.bfloat16,
|
||||
device: str = "cuda",
|
||||
device: str = DEVICE,
|
||||
):
|
||||
"""Create test tensors for MoE with non-gated activation.
|
||||
|
||||
@@ -106,10 +120,6 @@ def make_test_tensors(
|
||||
return hidden_states, w1, w2, topk_weights, topk_ids
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.has_device_capability(80),
|
||||
reason="Requires compute capability >= 8.0",
|
||||
)
|
||||
@pytest.mark.parametrize("m", M_SIZES)
|
||||
@pytest.mark.parametrize("n", N_SIZES)
|
||||
@pytest.mark.parametrize("k", K_SIZES)
|
||||
@@ -192,10 +202,6 @@ def test_triton_experts_no_mul_activation(
|
||||
assert output.abs().sum() > 0, "Output is all zeros"
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.has_device_capability(80),
|
||||
reason="Requires compute capability >= 8.0",
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_workspace_shapes_no_mul_vs_gated():
|
||||
"""Test that workspace shapes differ correctly between gated and non-gated."""
|
||||
@@ -233,10 +239,6 @@ def test_workspace_shapes_no_mul_vs_gated():
|
||||
assert out_no_mul == out_gated == (M, K)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.has_device_capability(80),
|
||||
reason="Requires compute capability >= 8.0",
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_adjust_n_for_activation():
|
||||
"""Test the adjust_N_for_activation method."""
|
||||
|
||||
@@ -40,6 +40,8 @@ from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import per_block_cast_to_fp8
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
DEVICE = current_platform.device_type
|
||||
|
||||
|
||||
def shuffle_weight(w: torch.Tensor) -> torch.Tensor:
|
||||
"""Fold weights to adjacent locations for Triton MoE / SwiGLU kernel layout."""
|
||||
@@ -80,7 +82,7 @@ def make_dummy_moe_config(
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
activation=activation,
|
||||
in_dtype=in_dtype,
|
||||
device="cuda",
|
||||
device=DEVICE,
|
||||
routing_method=RoutingMethodType.TopK,
|
||||
max_num_tokens=max_num_tokens,
|
||||
)
|
||||
@@ -238,7 +240,7 @@ def make_quantized_test_activations(
|
||||
block_shape: list[int] | None = None,
|
||||
per_act_token_quant: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
|
||||
a = torch.randn((E, m, k), device="cuda", dtype=in_dtype) / 10
|
||||
a = torch.randn((E, m, k), device=DEVICE, dtype=in_dtype) / 10
|
||||
a_q = a
|
||||
a_scale = None
|
||||
|
||||
@@ -369,7 +371,7 @@ def make_test_weight(
|
||||
block_shape: list[int] | None = None,
|
||||
per_out_ch_quant: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None, torch.Tensor | None]:
|
||||
w_16 = torch.randn((e, rows, cols), device="cuda", dtype=in_dtype) / 15
|
||||
w_16 = torch.randn((e, rows, cols), device=DEVICE, dtype=in_dtype) / 15
|
||||
|
||||
if quant_dtype is not None:
|
||||
w, w_s, w_gs = moe_quantize_weights(
|
||||
@@ -449,8 +451,8 @@ def make_test_quant_config(
|
||||
a1_gscale: torch.Tensor | None = None
|
||||
a2_gscale: torch.Tensor | None = None
|
||||
if quant_dtype == "nvfp4":
|
||||
a1_gscale = torch.ones((e,), device="cuda", dtype=torch.float32)
|
||||
a2_gscale = torch.ones((e,), device="cuda", dtype=torch.float32)
|
||||
a1_gscale = torch.ones((e,), device=DEVICE, dtype=torch.float32)
|
||||
a2_gscale = torch.ones((e,), device=DEVICE, dtype=torch.float32)
|
||||
a1_scale = a1_gscale
|
||||
a2_scale = a2_gscale
|
||||
else:
|
||||
@@ -551,8 +553,8 @@ def make_naive_shared_experts(
|
||||
K: int,
|
||||
in_dtype: torch.dtype = torch.bfloat16,
|
||||
) -> torch.nn.Module:
|
||||
w1 = torch.randn((K, N * 2), device="cuda", dtype=in_dtype) / 15
|
||||
w2 = torch.randn((N, K), device="cuda", dtype=in_dtype) / 15
|
||||
w1 = torch.randn((K, N * 2), device=DEVICE, dtype=in_dtype) / 15
|
||||
w2 = torch.randn((N, K), device=DEVICE, dtype=in_dtype) / 15
|
||||
return TestMLP(w1, w2, out_dtype=in_dtype)
|
||||
|
||||
|
||||
|
||||
@@ -13,10 +13,11 @@ from vllm.utils.deep_gemm import _ceil_to_ue8m0, is_deep_gemm_e8m0_used
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
DEVICE = current_platform.device_type
|
||||
|
||||
|
||||
def as_float32_tensor(x: float | torch.Tensor) -> torch.Tensor:
|
||||
return torch.as_tensor(x, dtype=torch.float32, device="cuda")
|
||||
return torch.as_tensor(x, dtype=torch.float32, device=DEVICE)
|
||||
|
||||
|
||||
def ref_dynamic_per_token_quant(
|
||||
|
||||
@@ -18,19 +18,20 @@ from vllm.platforms import current_platform
|
||||
@pytest.mark.parametrize("scale_ue8m0", [False, True])
|
||||
@pytest.mark.parametrize("group_size", [64, 128])
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="Only test on CUDA/ROCm."
|
||||
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
|
||||
reason="Only test on CUDA/ROCm/XPU.",
|
||||
)
|
||||
def test_per_token_group_quant_fp8(
|
||||
shape, column_major: bool, tma_aligned: bool, scale_ue8m0: bool, group_size: int
|
||||
):
|
||||
device = "cuda"
|
||||
device = current_platform.device_type
|
||||
|
||||
torch.manual_seed(42)
|
||||
num_tokens, hidden_dim = shape
|
||||
|
||||
x = torch.randn((num_tokens, hidden_dim), device=device, dtype=torch.bfloat16) * 8
|
||||
|
||||
# cuda path
|
||||
# native kernel path
|
||||
out_q, scale = fp8_utils.per_token_group_quant_fp8(
|
||||
x,
|
||||
group_size,
|
||||
@@ -40,7 +41,10 @@ def test_per_token_group_quant_fp8(
|
||||
)
|
||||
|
||||
# triton ref
|
||||
with patch("vllm.platforms.current_platform.is_cuda_alike", return_value=False):
|
||||
with (
|
||||
patch("vllm.platforms.current_platform.is_cuda_alike", return_value=False),
|
||||
patch("vllm.platforms.current_platform.is_xpu", return_value=False),
|
||||
):
|
||||
ref_q, ref_s = fp8_utils.per_token_group_quant_fp8(
|
||||
x,
|
||||
group_size,
|
||||
|
||||
@@ -4,13 +4,20 @@
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.third_party.flash_linear_attention.ops import (
|
||||
fused_recurrent_gated_delta_rule,
|
||||
fused_recurrent_gated_delta_rule_packed_decode,
|
||||
)
|
||||
|
||||
DEVICE = current_platform.device_type
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
|
||||
reason="Gated delta rule Triton kernels require a CUDA-alike or XPU device.",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Need CUDA device")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
|
||||
@pytest.mark.parametrize("strided_mixed_qkv", [False, True])
|
||||
def test_fused_recurrent_packed_decode_matches_reference(
|
||||
@@ -26,7 +33,7 @@ def test_fused_recurrent_packed_decode_matches_reference(
|
||||
V = 128
|
||||
qkv_dim = 2 * (H * K) + (HV * V)
|
||||
|
||||
device = torch.device("cuda")
|
||||
device = torch.device(DEVICE)
|
||||
|
||||
if strided_mixed_qkv:
|
||||
# Simulate a packed view into a larger projection buffer:
|
||||
@@ -102,10 +109,9 @@ def test_fused_recurrent_packed_decode_matches_reference(
|
||||
torch.testing.assert_close(state_packed, state_ref, rtol=rtol, atol=atol)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Need CUDA device")
|
||||
def test_packed_decode_supports_large_batch_head_grid():
|
||||
B, H, HV, K, V = 1024, 8, 64, 1, 1
|
||||
device = torch.device("cuda")
|
||||
device = torch.device(DEVICE)
|
||||
gates = torch.empty((B, HV), device=device)
|
||||
params = torch.empty((HV,), device=device)
|
||||
out = torch.empty((B, 1, HV, V), device=device)
|
||||
|
||||
@@ -37,9 +37,10 @@ from vllm.triton_utils.allocation import set_triton_allocator
|
||||
|
||||
|
||||
def _is_capturing_or_compiling() -> bool:
|
||||
# torch.cuda.is_current_stream_capturing() is unavailable on non-CUDA (XPU) torch.
|
||||
return torch.compiler.is_compiling() or (
|
||||
current_platform.is_cuda_alike() and torch.cuda.is_current_stream_capturing()
|
||||
# The accelerator API dispatches to the active backend.
|
||||
return (
|
||||
torch.compiler.is_compiling()
|
||||
or torch.accelerator.current_stream().is_capturing()
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user