[Feature] Support batch invariance on ROCm (#52231)

Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Matt
2026-10-12 10:53:26 -05:00
committed by GitHub
co-authored by Claude Opus 5.5 Andreas Karatzas
parent 50b21a2667
commit e0317b28ce
31 changed files with 1106 additions and 122 deletions
@@ -32,11 +32,9 @@ def torch_scaled_mm(
out = torch.mm(a.to(torch.float32), b.to(torch.float32))
out = scale_a * out
out = scale_b.T * out
out = out.to(out_dtype)
if bias is not None:
out = out + bias
return out
out = out + bias.float()
return out.to(out_dtype)
def get_8bit_types():
@@ -125,6 +123,25 @@ def test_scaled_mm(
torch.testing.assert_close(c_check, c_actual, rtol=1e-1, atol=1e-1)
@pytest.mark.parametrize("in_dtype", get_8bit_types())
@pytest.mark.parametrize("use_heuristic", [True, False])
def test_scaled_mm_adds_bias_before_rounding(in_dtype, use_heuristic):
# The accumulator is 1 + 3 * 2**-10 and the bias 2**-8. Rounding to bf16
# first gives 1.0 (1 + 2**-8 is a tie, to even); adding first, 1 + 2**-7.
a = torch.ones((1, 32), device=device).to(in_dtype)
col = torch.tensor([64.0] * 16 + [2.0, 1.0] + [0.0] * 14, device=device)
b = col[:, None].expand(32, 16).to(in_dtype).contiguous()
scale_a = torch.ones((1, 1), device=device)
scale_b = torch.full((1, 1), 2**-10, device=device)
bias = torch.full((16,), 2**-8, device=device, dtype=torch.bfloat16)
args = (a, b, scale_a, scale_b, torch.bfloat16, bias)
out = triton_scaled_mm(*args, use_heuristic=use_heuristic)
expected = torch_scaled_mm(*args)
assert (expected == 1 + 2**-7).all()
torch.testing.assert_close(out, expected, rtol=0, atol=0)
# TD operand loads must be bit-exact vs the plain masked-load path.
@pytest.mark.skipif(
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
@@ -180,6 +180,7 @@ def test_logprobs_bitwise_batch_invariance_bs1_vs_bsN(
llm = LLM(
model=TEST_MODEL,
tensor_parallel_size=tp_size,
disable_custom_all_reduce=os.getenv("VLLM_TEST_DISABLE_CUSTOM_AR") == "1",
max_num_seqs=128,
max_model_len=8192,
dtype="auto", # not everything is supported
@@ -9,7 +9,7 @@ import numpy as np
import pytest
import torch
from PIL import Image
from utils import _extract_step_logprobs, skip_if_not_cuda
from utils import _extract_step_logprobs, skip_if_not_cuda_alike
from vllm import LLM, SamplingParams
from vllm.inputs import TextPrompt
@@ -89,7 +89,7 @@ def _assert_batch_invariant(llm: LLM, inputs: list[TextPrompt], sampling) -> Non
)
@skip_if_not_cuda
@skip_if_not_cuda_alike
@pytest.mark.parametrize("input_type", ["image", "video"])
@pytest.mark.parametrize("mm_encoder_attn_backend", ["FLASH_ATTN", "TORCH_SDPA"])
def test_vlm_batch_invariance_bs1_vs_bsN(
@@ -101,7 +101,7 @@ def test_vlm_batch_invariance_bs1_vs_bsN(
_run_vlm_batch_invariance(input_type, mm_encoder_attn_backend, enforce_eager=True)
@skip_if_not_cuda
@skip_if_not_cuda_alike
def test_vlm_batch_invariance_default_execution(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("VLLM_USE_FLASHINFER_SAMPLER", "0")
_run_vlm_batch_invariance("image", "FLASH_ATTN", enforce_eager=False)
@@ -19,8 +19,10 @@ from tests.kernels.moe.utils import make_dummy_moe_config, make_test_quant_confi
from tests.kernels.quantization.nvfp4_utils import get_nvfp4_global_scale
from tests.utils import TestFP8Layer, requires_fp8
from vllm import _custom_ops as ops
from vllm.model_executor.kernels.linear.scaled_mm.cutlass import (
from vllm.model_executor.kernels.linear import fp8_scaled_mm_is_batch_invariant
from vllm.model_executor.kernels.linear.scaled_mm import (
CutlassFP8ScaledMMLinearKernel,
TritonFP8ScaledMMLinearKernel,
)
from vllm.model_executor.layers.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
@@ -31,8 +33,11 @@ from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
CutlassExpertsFp4,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
kFp8StaticTokenSym,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
@@ -87,16 +92,42 @@ _NVFP4_MOE_BATCH_INVARIANT_CASES = (
@pytest.fixture(autouse=True)
def setup_cuda():
if not current_platform.is_cuda():
pytest.skip("CUTLASS FP8 kernels require CUDA.")
if not current_platform.is_cuda_alike():
pytest.skip("Requires CUDA or ROCm.")
torch.set_default_device("cuda")
@requires_fp8
@pytest.mark.parametrize(
"kernel",
[
pytest.param(
CutlassFP8ScaledMMLinearKernel,
marks=pytest.mark.skipif(
not current_platform.is_cuda(), reason="CUTLASS requires CUDA."
),
),
TritonFP8ScaledMMLinearKernel,
],
)
@pytest.mark.parametrize(
"act_key,weight_key",
[
(kFp8DynamicTokenSym, kFp8StaticTensorSym),
(kFp8DynamicTokenSym, kFp8StaticChannelSym),
(kFp8DynamicTokenSym, kFp8StaticTokenSym),
(kFp8StaticTensorSym, kFp8StaticTensorSym),
# An amax over the whole batch: the GEMM is invariant, the layer is not.
(kFp8DynamicTensorSym, kFp8StaticTensorSym),
],
)
@pytest.mark.parametrize("weight_shape", [(1024, 2048), (4608, 4096)])
@pytest.mark.parametrize("batch_size", [1, 16, 17, 32, 64, 65, 256, 257])
@torch.inference_mode()
def test_cutlass_fp8_batch_invariant_fixed_config(
def test_fp8_scaled_mm_batch_invariant_fixed_config(
kernel: type,
act_key,
weight_key,
weight_shape: tuple[int, int],
batch_size: int,
default_vllm_config,
@@ -108,14 +139,16 @@ def test_cutlass_fp8_batch_invariant_fixed_config(
torch.manual_seed(0)
layer = TestFP8Layer(
weight_shape=weight_shape,
activation_quant_key=kFp8DynamicTokenSym,
weight_quant_key=kFp8StaticTensorSym,
activation_quant_key=act_key,
weight_quant_key=weight_key,
input_dtype=torch.bfloat16,
out_dtype=torch.bfloat16,
device=torch.device("cuda"),
force_kernel=CutlassFP8ScaledMMLinearKernel,
force_kernel=kernel,
)
assert isinstance(layer.kernel, CutlassFP8ScaledMMLinearKernel)
assert isinstance(layer.kernel, kernel)
dynamic_per_tensor = act_key == kFp8DynamicTensorSym
assert fp8_scaled_mm_is_batch_invariant(layer.kernel) != dynamic_per_tensor
in_features = weight_shape[1]
needle = torch.randn((1, in_features), device="cuda", dtype=torch.bfloat16)
@@ -124,6 +157,12 @@ def test_cutlass_fp8_batch_invariant_fixed_config(
filler = torch.randn(
(max(batch_size - 1, 0), in_features), device="cuda", dtype=torch.bfloat16
)
if dynamic_per_tensor:
if batch_size == 1:
pytest.skip("no other rows to move the scale")
front_output = layer(torch.cat([needle, 100 * filler], dim=0))[0]
assert not torch.equal(front_output, baseline)
return
front_batch = torch.cat([needle, filler], dim=0)
back_batch = torch.cat([filler, needle], dim=0)
@@ -135,6 +174,69 @@ def test_cutlass_fp8_batch_invariant_fixed_config(
torch.testing.assert_close(back_output, baseline, rtol=0, atol=0)
@requires_fp8
@pytest.mark.parametrize("scheme", ["fp8_per_tensor", "fp8_per_channel"])
@torch.inference_mode()
def test_online_fp8_linear_batch_invariant(
scheme, monkeypatch, default_vllm_config, dist_init
):
from vllm.config.model import ModelConfig
from vllm.config.quantization import resolve_quantization_config
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.quantization.online.base import (
OnlineQuantizationConfig,
)
from vllm.utils.torch_utils import set_default_torch_dtype
monkeypatch.setattr(envs, "VLLM_BATCH_INVARIANT", True)
default_vllm_config.model_config = ModelConfig(dtype="bfloat16")
quant_config = OnlineQuantizationConfig(resolve_quantization_config(scheme, None))
set_random_seed(0)
with set_default_torch_dtype(torch.bfloat16), torch.device("cuda"):
layer = ReplicatedLinear(
1024, 768, bias=False, quant_config=quant_config, prefix="mlp.down_proj"
)
layer.weight.weight_loader(layer.weight, torch.randn(768, 1024))
needle = torch.randn(1, 1024)
filler = 100 * torch.randn(63, 1024)
baseline = layer(needle)[0]
assert torch.equal(layer(torch.cat([needle, filler]))[0][:1], baseline)
assert torch.equal(layer(torch.cat([filler, needle]))[0][-1:], baseline)
@requires_fp8
@torch.inference_mode()
def test_block_fp8_batch_invariant_across_tuned_block_size_k(monkeypatch):
"""The block-FP8 kernel applies the block scales once per K tile, so a tuned
BLOCK_SIZE_K below the scale group size reorders the sum."""
from vllm.model_executor.layers.quantization.utils import fp8_utils
base = dict(
BLOCK_SIZE_M=64, BLOCK_SIZE_N=128, GROUP_SIZE_M=1, num_warps=4, num_stages=2
)
table = {256: {**base, "BLOCK_SIZE_K": 128}, 512: {**base, "BLOCK_SIZE_K": 64}}
monkeypatch.setattr(fp8_utils, "get_w8a8_block_fp8_configs", lambda *_: table)
monkeypatch.setattr(envs, "VLLM_BATCH_INVARIANT", True)
N, K, block = 12288, 2048, [128, 128]
set_random_seed(0)
fp8 = current_platform.fp8_dtype()
A = torch.randn(512, K).to(fp8)
B = torch.randn(N, K).to(fp8)
As = torch.rand(512, K // 128) / 100
Bs = torch.rand(N // 128, K // 128) / 100
def run(m):
return fp8_utils.w8a8_triton_block_scaled_mm(
A[:m], B, As[:m], Bs, block, torch.bfloat16
)[:256]
assert torch.equal(run(256), run(512))
monkeypatch.setattr(envs, "VLLM_BATCH_INVARIANT", False)
assert not torch.equal(run(256), run(512))
@_NVFP4_REQUIRES_SM100
@pytest.mark.parametrize("dtype", _NVFP4_LINEAR_DTYPES)
@pytest.mark.parametrize("shape", _NVFP4_LINEAR_CONSISTENCY_SHAPES)
@@ -4,7 +4,7 @@
import pytest
import torch
from utils import skip_if_not_cuda
from utils import skip_if_not_cuda_alike
import vllm.lora.ops.triton_ops.lora_shrink_op as shrink_op
import vllm.lora.ops.triton_ops.utils as lora_utils
@@ -39,7 +39,7 @@ def _shrink(inputs, weights, mapping, scaling):
return out
@skip_if_not_cuda
@skip_if_not_cuda_alike
@pytest.mark.parametrize(
"hidden_size,rank,nslices,dtype",
[
@@ -68,7 +68,7 @@ def test_lora_shrink_matches_reference(hidden_size, rank, nslices, dtype):
torch.testing.assert_close(out, ref, rtol=5e-3, atol=5e-3)
@skip_if_not_cuda
@skip_if_not_cuda_alike
@pytest.mark.parametrize("nslices", [1, 3])
def test_lora_shrink_invariant_across_batch_sizes(monkeypatch, nslices):
reduce_kernel, reduce_grids = shrink_op._lora_shrink_reduce_kernel, []
@@ -10,7 +10,11 @@ import pytest
import torch
from utils import skip_unsupported
from vllm.model_executor.determinism.batch_invariant import matmul_batch_invariant
import vllm.model_executor.determinism.batch_invariant as bi
from vllm.model_executor.determinism.batch_invariant import (
enable_batch_invariant_mode,
matmul_batch_invariant,
)
from vllm.model_executor.determinism.batch_invariant_configs import (
_BATCH_INVARIANT_MATMUL_TUNED_CONFIGS,
_get_tuned_matmul_arch_family,
@@ -129,3 +133,49 @@ def test_matmul_batch_invariance_across_tuned_m_buckets(m, transpose_b):
batch_output = matmul_batch_invariant(a, b)
assert torch.equal(single_output[0], batch_output[0])
@pytest.fixture
def batch_invariant_mode():
# The overrides are process-global: remove them so later tests compare
# against the stock aten kernels again.
enable_batch_invariant_mode()
yield
assert bi._batch_invariant_LIB is not None
bi._batch_invariant_LIB._destroy()
bi._batch_invariant_MODE = False
@skip_unsupported
@pytest.mark.usefixtures("batch_invariant_mode")
@pytest.mark.parametrize("op", ["mm", "addmm", "bmm"])
def test_compiled_matmul_reaches_batch_invariant_out_overloads(op):
"""Inductor lowers mm/addmm/bmm to ``extern_kernels.<op>(..., out=buf)``,
i.e. the ``.out`` overload, which eager never dispatches."""
if (
op != "bmm"
and current_platform.is_cuda()
and not current_platform.is_device_capability_family(80)
):
pytest.skip("mm/addmm are not overridden on SM90+")
torch.manual_seed(0)
dtype, k, n, needle = torch.bfloat16, 1024, 768, 37
w = torch.randn(4, k, n, device=DEVICE_TYPE, dtype=dtype)
bias = torch.randn(n, device=DEVICE_TYPE, dtype=dtype)
fn = {
"mm": lambda x: torch.mm(x[0], w[0]),
# Bare addmm: a following pointwise op would make Inductor unfuse the bias.
"addmm": lambda x: torch.addmm(bias, x[0], w[0]),
"bmm": lambda x: torch.bmm(x, w),
}[op]
compiled = torch.compile(fn, fullgraph=True)
x = torch.randn(4, 256, k, device=DEVICE_TYPE, dtype=dtype)
row = None
for m in (64, 256, 128):
xs = x[:, :m].clone()
# With dynamic=True, a specialization on M would silently recompile.
torch._dynamo.mark_dynamic(xs, 1)
out = compiled(xs)
assert torch.equal(out, fn(xs)), f"compiled {op} bypassed the BI kernel (M={m})"
row = out[..., needle, :] if row is None else row
assert torch.equal(out[..., needle, :], row)
@@ -0,0 +1,165 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The ROCm native MX linear kernels must not depend on the number of rows."""
import pytest
import torch
from utils import requires_mx
from vllm.utils.torch_utils import set_random_seed
SEED = 0
# E8M0 exponent half-width. With 4-bit operands and a narrow spread the fp32
# partial sums are exact, so a reordering of them would be invisible.
SCALE_SPREAD = 15
# (N, K, use_asm_gemm)
MXFP4_CASES = [
(4096, 2048, False),
(2048, 6144, False),
(4608, 7168, False),
(8192, 2048, True),
(8192, 1024, True),
(8192, 14336, True),
]
MXFP8_CASES = [(4096, 2048), (2048, 6144), (1024, 768), (1536, 1024)]
# The untuned AITER MXFP4 configs switch at M = 9/33/65/129/257/513.
MXFP4_TOKEN_COUNTS = [
1,
2,
8,
9,
16,
32,
33,
64,
65,
128,
129,
256,
257,
512,
513,
1024,
2048,
]
MXFP8_TOKEN_COUNTS = [1, 32, 64, 65, 128, 129, 256, 257, 512, 1024, 1025, 2048]
# Row 0 lands in the first tile of every decomposition, so it can stay
# invariant when the rest of the output does not.
CHECK_ROWS = [0, 1, 2, 3, 7, 15, 31]
def _probe(out: torch.Tensor) -> torch.Tensor:
"""The leading rows of an output, kept so the sweep does not hold every GEMM."""
return out[: CHECK_ROWS[-1] + 1].clone()
def _variant_rows(probes: dict[int, torch.Tensor]) -> list[str]:
"""Rows whose contents changed with the number of rows in the launch."""
failures = []
for row in CHECK_ROWS:
counts = [n for n in sorted(probes) if n > row]
if not counts:
continue
reference = probes[counts[0]][row]
variant = [n for n in counts if not torch.equal(probes[n][row], reference)]
if variant:
failures.append(f"row {row} changed at row counts {variant}")
return failures
def _mxfp4_weights(n: int, k: int, use_asm_gemm: bool):
"""Weights laid out as AiterMxfp4LinearKernel.process_weights_after_loading."""
weight = torch.randint(0, 255, (n, k // 2), dtype=torch.uint8, device="cuda")
weight_scale = torch.randint(
127 - SCALE_SPREAD,
128 + SCALE_SPREAD,
(n, k // 32),
dtype=torch.uint8,
device="cuda",
)
if not use_asm_gemm:
return weight, weight_scale.T.contiguous()
from aiter.ops.shuffle import shuffle_weight
sm, sn = weight_scale.shape
weight_scale = (
weight_scale.view(sm // 32, 2, 16, sn // 8, 2, 4, 1)
.permute(0, 3, 5, 2, 4, 1, 6)
.contiguous()
.view(sm, sn)
)
return shuffle_weight(weight, layout=(16, 16)), weight_scale
def _mxfp8_weights(n: int, k: int):
"""E4M3 weights with E8M0 block scales, as the MXFP8 layer stores them."""
weight = (torch.randn(n, k, device="cuda") / 8).to(torch.float8_e4m3fn)
weight_scale = torch.randint(
127 - SCALE_SPREAD,
128 + SCALE_SPREAD,
(n, k // 32),
dtype=torch.uint8,
device="cuda",
)
return weight, weight_scale
@requires_mx
@pytest.mark.parametrize("n,k,use_asm_gemm", MXFP4_CASES)
def test_mxfp4_linear_is_batch_invariant(n: int, k: int, use_asm_gemm: bool):
pytest.importorskip("aiter")
# Importing the module registers torch.ops.vllm.gemm_with_dynamic_quant.
import vllm.model_executor.kernels.linear.mxfp4.aiter # noqa: F401
set_random_seed(SEED)
weight, weight_scale = _mxfp4_weights(n, k, use_asm_gemm)
x = torch.randn(max(MXFP4_TOKEN_COUNTS), k, device="cuda", dtype=torch.bfloat16)
probes = {
num_tokens: _probe(
torch.ops.vllm.gemm_with_dynamic_quant(
x[:num_tokens], weight, weight_scale, use_asm_gemm, torch.bfloat16
)
)
for num_tokens in MXFP4_TOKEN_COUNTS
}
failures = _variant_rows(probes)
assert not failures, (
f"MXFP4 linear depends on the row count "
f"(N={n}, K={k}, asm={use_asm_gemm}):\n " + "\n ".join(failures)
)
@requires_mx
@pytest.mark.parametrize("n,k", MXFP8_CASES)
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
def test_mxfp8_linear_is_batch_invariant(n: int, k: int, dtype: torch.dtype):
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
_mxfp8_dot_scaled_linear,
)
set_random_seed(SEED)
weight, weight_scale = _mxfp8_weights(n, k)
x = torch.randn(max(MXFP8_TOKEN_COUNTS), k, device="cuda", dtype=dtype)
probes = {
num_tokens: _probe(
_mxfp8_dot_scaled_linear(x[:num_tokens], weight, weight_scale)
)
for num_tokens in MXFP8_TOKEN_COUNTS
}
failures = _variant_rows(probes)
assert not failures, (
f"MXFP8 linear depends on the row count (N={n}, K={k}, {dtype}):\n "
+ "\n ".join(failures)
)
@@ -9,10 +9,14 @@ import torch
from utils import (
_extract_step_logprobs,
_random_prompt,
skip_if_not_cuda,
skip_if_not_cuda_alike,
)
from vllm import LLM, SamplingParams
from vllm.platforms import current_platform
# FLASH_ATTN is CUDA-only.
E2E_BACKENDS = ["TRITON_ATTN"] if current_platform.is_rocm() else ["FLASH_ATTN"]
pytestmark = pytest.mark.skipif(
not hasattr(torch, "float8_e4m3fn"),
@@ -40,8 +44,8 @@ def _make_llm(max_num_seqs: int, backend: str) -> LLM:
)
@skip_if_not_cuda
@pytest.mark.parametrize("backend", ["FLASH_ATTN"])
@skip_if_not_cuda_alike
@pytest.mark.parametrize("backend", E2E_BACKENDS)
def test_dense_nvfp4_generation_is_deterministic_across_batch_sizes_e2e(backend):
seed = int(os.getenv("VLLM_TEST_SEED", "12345"))
random.seed(seed)
@@ -6,6 +6,7 @@ vLLM server and compare BS=1 vs BS=N results (tokens and per-step logprobs).
Environment variables:
- VLLM_TEST_MODEL: served model name (e.g., Qwen/Qwen3-1.7B / DeepSeek-R1)
- VLLM_TP_SIZE: tensor parallelism size (e.g., 4)
- VLLM_TEST_SERVER_ARGS: extra server args (e.g., "--data-parallel-size 4")
"""
@@ -16,7 +17,7 @@ from typing import Any
import openai
import pytest
from utils import BACKENDS, TEST_MODEL, _random_prompt, skip_if_not_cuda
from utils import BACKENDS, TEST_MODEL, _random_prompt, skip_if_not_cuda_alike
from tests.utils import RemoteOpenAIServer
@@ -132,7 +133,7 @@ def _compare_bs1_vs_bsn_single_process(
)
@skip_if_not_cuda
@skip_if_not_cuda_alike
@pytest.mark.parametrize("backend", BACKENDS)
def test_logprobs_bitwise_batch_invariance_bs1_vs_bsN(
backend: str,
@@ -156,6 +157,7 @@ def test_logprobs_bitwise_batch_invariance_bs1_vs_bsN(
]
if tp_size:
server_args += ["-tp", tp_size]
server_args += os.getenv("VLLM_TEST_SERVER_ARGS", "").split()
with RemoteOpenAIServer(TEST_MODEL, server_args) as server:
client = server.get_client()
@@ -4,7 +4,7 @@
import pytest
import torch
from utils import skip_if_not_cuda, skip_unsupported
from utils import skip_if_not_cuda_alike, skip_unsupported
from vllm.model_executor.determinism.batch_invariant import (
rms_norm_batch_invariant,
@@ -27,7 +27,7 @@ def _rms_norm_reference(
return (output * weight.float()).to(input_tensor.dtype)
@skip_if_not_cuda
@skip_if_not_cuda_alike
@pytest.mark.parametrize("batch_size", [1, 4, 64, 300])
@pytest.mark.parametrize("hidden_size", [512, 2048, 4096, 8192])
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@@ -76,7 +76,7 @@ def test_rms_norm_batch_invariant_vs_reference(
)
@skip_if_not_cuda
@skip_if_not_cuda_alike
@pytest.mark.parametrize("hidden_size", [512, 4096])
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("eps", [1e-6])
@@ -189,7 +189,7 @@ def _assert_rows_bit_identical(small, large, msg):
torch.testing.assert_close(small, large, rtol=0.0, atol=0.0, msg=msg)
@skip_if_not_cuda
@skip_if_not_cuda_alike
@pytest.mark.parametrize("hidden_size", [512, 4096])
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("seed", list(range(4)))
@@ -221,7 +221,7 @@ def test_rms_norm_batch_invariant_nonresidual_kernel(
)
@skip_if_not_cuda
@skip_if_not_cuda_alike
@pytest.mark.parametrize("hidden_size", [512, 4096])
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("seed", list(range(4)))
@@ -267,7 +267,7 @@ def test_rms_norm_static_fp8_quant_batch_invariant(
)
@skip_if_not_cuda
@skip_if_not_cuda_alike
@pytest.mark.parametrize("hidden_size", [512, 4096])
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("seed", list(range(4)))
@@ -303,7 +303,7 @@ def test_rms_norm_per_block_quant_batch_invariant(
)
@skip_if_not_cuda
@skip_if_not_cuda_alike
@pytest.mark.parametrize("batch_size", [1, 16, 128])
@pytest.mark.parametrize("seq_len", [1, 32, 512])
@pytest.mark.parametrize("hidden_size", [2048, 4096])
@@ -343,7 +343,7 @@ def test_rms_norm_3d_input(
)
@skip_if_not_cuda
@skip_if_not_cuda_alike
def test_rms_norm_numerical_stability(default_vllm_config):
"""Test RMS norm numerical stability with extreme values.
@@ -431,7 +431,7 @@ def test_rms_norm_formula(default_vllm_config):
)
@skip_if_not_cuda
@skip_if_not_cuda_alike
@pytest.mark.parametrize("hidden_size", [128, 1024, 4096, 16384])
def test_rms_norm_different_hidden_sizes(default_vllm_config, hidden_size: int):
"""Test RMS norm with various hidden sizes to ensure block size handling.
+20 -2
View File
@@ -34,12 +34,16 @@ DEVICE_BACKENDS: dict[str, DeviceConfig] = {
available=current_platform.is_xpu() and HAS_TRITON,
backends=["TRITON_ATTN"],
),
"rocm": DeviceConfig(
available=current_platform.is_rocm() and HAS_TRITON,
backends=["TRITON_ATTN"],
),
}
DEFAULT_MODEL = "Qwen/Qwen3-1.7B"
TEST_MODEL = os.getenv("VLLM_TEST_MODEL", DEFAULT_MODEL)
# Override backends for MLA models (MLA only supported on CUDA).
# Override backends for MLA models.
if os.getenv("VLLM_TEST_MODEL"):
config = get_config(TEST_MODEL, trust_remote_code=False)
if ModelArchConfigConvertorBase(config, config.get_text_config()).is_deepseek_mla():
@@ -52,6 +56,10 @@ if os.getenv("VLLM_TEST_MODEL"):
available=DEVICE_BACKENDS["xpu"].available,
backends=[],
)
DEVICE_BACKENDS["rocm"] = DeviceConfig(
available=DEVICE_BACKENDS["rocm"].available,
backends=["TRITON_MLA"],
)
# Only include backends for devices that are actually available.
BACKENDS: list[str] = sorted(
@@ -60,7 +68,7 @@ BACKENDS: list[str] = sorted(
skip_unsupported = pytest.mark.skipif(
not any(cfg.available for cfg in DEVICE_BACKENDS.values()),
reason="Requires CUDA >= Ampere (SM80) or Intel XPU with Triton",
reason="Requires CUDA >= Ampere (SM80), ROCm, or Intel XPU with Triton",
)
skip_if_not_cuda = pytest.mark.skipif(
@@ -68,6 +76,16 @@ skip_if_not_cuda = pytest.mark.skipif(
reason="Requires CUDA >= Ampere (SM80)",
)
skip_if_not_cuda_alike = pytest.mark.skipif(
not (DEVICE_BACKENDS["cuda"].available or DEVICE_BACKENDS["rocm"].available),
reason="Requires CUDA >= Ampere (SM80) or ROCm",
)
requires_mx = pytest.mark.skipif(
not (current_platform.is_rocm() and current_platform.supports_mx()),
reason="requires a ROCm device with native MX support (gfx95x)",
)
def _random_prompt(min_words: int = 1024, max_words: int = 1024 * 2) -> str:
# Generate more realistic prompts that will actually produce varied tokens