[Feature] Support batch invariance on ROCm (#52231)
Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Andreas Karatzas
parent
50b21a2667
commit
e0317b28ce
@@ -32,11 +32,9 @@ def torch_scaled_mm(
|
||||
out = torch.mm(a.to(torch.float32), b.to(torch.float32))
|
||||
out = scale_a * out
|
||||
out = scale_b.T * out
|
||||
out = out.to(out_dtype)
|
||||
if bias is not None:
|
||||
out = out + bias
|
||||
|
||||
return out
|
||||
out = out + bias.float()
|
||||
return out.to(out_dtype)
|
||||
|
||||
|
||||
def get_8bit_types():
|
||||
@@ -125,6 +123,25 @@ def test_scaled_mm(
|
||||
torch.testing.assert_close(c_check, c_actual, rtol=1e-1, atol=1e-1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("in_dtype", get_8bit_types())
|
||||
@pytest.mark.parametrize("use_heuristic", [True, False])
|
||||
def test_scaled_mm_adds_bias_before_rounding(in_dtype, use_heuristic):
|
||||
# The accumulator is 1 + 3 * 2**-10 and the bias 2**-8. Rounding to bf16
|
||||
# first gives 1.0 (1 + 2**-8 is a tie, to even); adding first, 1 + 2**-7.
|
||||
a = torch.ones((1, 32), device=device).to(in_dtype)
|
||||
col = torch.tensor([64.0] * 16 + [2.0, 1.0] + [0.0] * 14, device=device)
|
||||
b = col[:, None].expand(32, 16).to(in_dtype).contiguous()
|
||||
scale_a = torch.ones((1, 1), device=device)
|
||||
scale_b = torch.full((1, 1), 2**-10, device=device)
|
||||
bias = torch.full((16,), 2**-8, device=device, dtype=torch.bfloat16)
|
||||
args = (a, b, scale_a, scale_b, torch.bfloat16, bias)
|
||||
|
||||
out = triton_scaled_mm(*args, use_heuristic=use_heuristic)
|
||||
expected = torch_scaled_mm(*args)
|
||||
assert (expected == 1 + 2**-7).all()
|
||||
torch.testing.assert_close(out, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
# TD operand loads must be bit-exact vs the plain masked-load path.
|
||||
@pytest.mark.skipif(
|
||||
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
|
||||
|
||||
@@ -180,6 +180,7 @@ def test_logprobs_bitwise_batch_invariance_bs1_vs_bsN(
|
||||
llm = LLM(
|
||||
model=TEST_MODEL,
|
||||
tensor_parallel_size=tp_size,
|
||||
disable_custom_all_reduce=os.getenv("VLLM_TEST_DISABLE_CUSTOM_AR") == "1",
|
||||
max_num_seqs=128,
|
||||
max_model_len=8192,
|
||||
dtype="auto", # not everything is supported
|
||||
|
||||
@@ -9,7 +9,7 @@ import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
from utils import _extract_step_logprobs, skip_if_not_cuda
|
||||
from utils import _extract_step_logprobs, skip_if_not_cuda_alike
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.inputs import TextPrompt
|
||||
@@ -89,7 +89,7 @@ def _assert_batch_invariant(llm: LLM, inputs: list[TextPrompt], sampling) -> Non
|
||||
)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@skip_if_not_cuda_alike
|
||||
@pytest.mark.parametrize("input_type", ["image", "video"])
|
||||
@pytest.mark.parametrize("mm_encoder_attn_backend", ["FLASH_ATTN", "TORCH_SDPA"])
|
||||
def test_vlm_batch_invariance_bs1_vs_bsN(
|
||||
@@ -101,7 +101,7 @@ def test_vlm_batch_invariance_bs1_vs_bsN(
|
||||
_run_vlm_batch_invariance(input_type, mm_encoder_attn_backend, enforce_eager=True)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@skip_if_not_cuda_alike
|
||||
def test_vlm_batch_invariance_default_execution(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setenv("VLLM_USE_FLASHINFER_SAMPLER", "0")
|
||||
_run_vlm_batch_invariance("image", "FLASH_ATTN", enforce_eager=False)
|
||||
|
||||
@@ -19,8 +19,10 @@ from tests.kernels.moe.utils import make_dummy_moe_config, make_test_quant_confi
|
||||
from tests.kernels.quantization.nvfp4_utils import get_nvfp4_global_scale
|
||||
from tests.utils import TestFP8Layer, requires_fp8
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.cutlass import (
|
||||
from vllm.model_executor.kernels.linear import fp8_scaled_mm_is_batch_invariant
|
||||
from vllm.model_executor.kernels.linear.scaled_mm import (
|
||||
CutlassFP8ScaledMMLinearKernel,
|
||||
TritonFP8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
@@ -31,8 +33,11 @@ from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
|
||||
CutlassExpertsFp4,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
kFp8StaticTokenSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
@@ -87,16 +92,42 @@ _NVFP4_MOE_BATCH_INVARIANT_CASES = (
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_cuda():
|
||||
if not current_platform.is_cuda():
|
||||
pytest.skip("CUTLASS FP8 kernels require CUDA.")
|
||||
if not current_platform.is_cuda_alike():
|
||||
pytest.skip("Requires CUDA or ROCm.")
|
||||
torch.set_default_device("cuda")
|
||||
|
||||
|
||||
@requires_fp8
|
||||
@pytest.mark.parametrize(
|
||||
"kernel",
|
||||
[
|
||||
pytest.param(
|
||||
CutlassFP8ScaledMMLinearKernel,
|
||||
marks=pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="CUTLASS requires CUDA."
|
||||
),
|
||||
),
|
||||
TritonFP8ScaledMMLinearKernel,
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"act_key,weight_key",
|
||||
[
|
||||
(kFp8DynamicTokenSym, kFp8StaticTensorSym),
|
||||
(kFp8DynamicTokenSym, kFp8StaticChannelSym),
|
||||
(kFp8DynamicTokenSym, kFp8StaticTokenSym),
|
||||
(kFp8StaticTensorSym, kFp8StaticTensorSym),
|
||||
# An amax over the whole batch: the GEMM is invariant, the layer is not.
|
||||
(kFp8DynamicTensorSym, kFp8StaticTensorSym),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("weight_shape", [(1024, 2048), (4608, 4096)])
|
||||
@pytest.mark.parametrize("batch_size", [1, 16, 17, 32, 64, 65, 256, 257])
|
||||
@torch.inference_mode()
|
||||
def test_cutlass_fp8_batch_invariant_fixed_config(
|
||||
def test_fp8_scaled_mm_batch_invariant_fixed_config(
|
||||
kernel: type,
|
||||
act_key,
|
||||
weight_key,
|
||||
weight_shape: tuple[int, int],
|
||||
batch_size: int,
|
||||
default_vllm_config,
|
||||
@@ -108,14 +139,16 @@ def test_cutlass_fp8_batch_invariant_fixed_config(
|
||||
torch.manual_seed(0)
|
||||
layer = TestFP8Layer(
|
||||
weight_shape=weight_shape,
|
||||
activation_quant_key=kFp8DynamicTokenSym,
|
||||
weight_quant_key=kFp8StaticTensorSym,
|
||||
activation_quant_key=act_key,
|
||||
weight_quant_key=weight_key,
|
||||
input_dtype=torch.bfloat16,
|
||||
out_dtype=torch.bfloat16,
|
||||
device=torch.device("cuda"),
|
||||
force_kernel=CutlassFP8ScaledMMLinearKernel,
|
||||
force_kernel=kernel,
|
||||
)
|
||||
assert isinstance(layer.kernel, CutlassFP8ScaledMMLinearKernel)
|
||||
assert isinstance(layer.kernel, kernel)
|
||||
dynamic_per_tensor = act_key == kFp8DynamicTensorSym
|
||||
assert fp8_scaled_mm_is_batch_invariant(layer.kernel) != dynamic_per_tensor
|
||||
|
||||
in_features = weight_shape[1]
|
||||
needle = torch.randn((1, in_features), device="cuda", dtype=torch.bfloat16)
|
||||
@@ -124,6 +157,12 @@ def test_cutlass_fp8_batch_invariant_fixed_config(
|
||||
filler = torch.randn(
|
||||
(max(batch_size - 1, 0), in_features), device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
if dynamic_per_tensor:
|
||||
if batch_size == 1:
|
||||
pytest.skip("no other rows to move the scale")
|
||||
front_output = layer(torch.cat([needle, 100 * filler], dim=0))[0]
|
||||
assert not torch.equal(front_output, baseline)
|
||||
return
|
||||
|
||||
front_batch = torch.cat([needle, filler], dim=0)
|
||||
back_batch = torch.cat([filler, needle], dim=0)
|
||||
@@ -135,6 +174,69 @@ def test_cutlass_fp8_batch_invariant_fixed_config(
|
||||
torch.testing.assert_close(back_output, baseline, rtol=0, atol=0)
|
||||
|
||||
|
||||
@requires_fp8
|
||||
@pytest.mark.parametrize("scheme", ["fp8_per_tensor", "fp8_per_channel"])
|
||||
@torch.inference_mode()
|
||||
def test_online_fp8_linear_batch_invariant(
|
||||
scheme, monkeypatch, default_vllm_config, dist_init
|
||||
):
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.config.quantization import resolve_quantization_config
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
OnlineQuantizationConfig,
|
||||
)
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
|
||||
monkeypatch.setattr(envs, "VLLM_BATCH_INVARIANT", True)
|
||||
default_vllm_config.model_config = ModelConfig(dtype="bfloat16")
|
||||
quant_config = OnlineQuantizationConfig(resolve_quantization_config(scheme, None))
|
||||
set_random_seed(0)
|
||||
with set_default_torch_dtype(torch.bfloat16), torch.device("cuda"):
|
||||
layer = ReplicatedLinear(
|
||||
1024, 768, bias=False, quant_config=quant_config, prefix="mlp.down_proj"
|
||||
)
|
||||
layer.weight.weight_loader(layer.weight, torch.randn(768, 1024))
|
||||
|
||||
needle = torch.randn(1, 1024)
|
||||
filler = 100 * torch.randn(63, 1024)
|
||||
baseline = layer(needle)[0]
|
||||
assert torch.equal(layer(torch.cat([needle, filler]))[0][:1], baseline)
|
||||
assert torch.equal(layer(torch.cat([filler, needle]))[0][-1:], baseline)
|
||||
|
||||
|
||||
@requires_fp8
|
||||
@torch.inference_mode()
|
||||
def test_block_fp8_batch_invariant_across_tuned_block_size_k(monkeypatch):
|
||||
"""The block-FP8 kernel applies the block scales once per K tile, so a tuned
|
||||
BLOCK_SIZE_K below the scale group size reorders the sum."""
|
||||
from vllm.model_executor.layers.quantization.utils import fp8_utils
|
||||
|
||||
base = dict(
|
||||
BLOCK_SIZE_M=64, BLOCK_SIZE_N=128, GROUP_SIZE_M=1, num_warps=4, num_stages=2
|
||||
)
|
||||
table = {256: {**base, "BLOCK_SIZE_K": 128}, 512: {**base, "BLOCK_SIZE_K": 64}}
|
||||
monkeypatch.setattr(fp8_utils, "get_w8a8_block_fp8_configs", lambda *_: table)
|
||||
monkeypatch.setattr(envs, "VLLM_BATCH_INVARIANT", True)
|
||||
N, K, block = 12288, 2048, [128, 128]
|
||||
|
||||
set_random_seed(0)
|
||||
fp8 = current_platform.fp8_dtype()
|
||||
A = torch.randn(512, K).to(fp8)
|
||||
B = torch.randn(N, K).to(fp8)
|
||||
As = torch.rand(512, K // 128) / 100
|
||||
Bs = torch.rand(N // 128, K // 128) / 100
|
||||
|
||||
def run(m):
|
||||
return fp8_utils.w8a8_triton_block_scaled_mm(
|
||||
A[:m], B, As[:m], Bs, block, torch.bfloat16
|
||||
)[:256]
|
||||
|
||||
assert torch.equal(run(256), run(512))
|
||||
monkeypatch.setattr(envs, "VLLM_BATCH_INVARIANT", False)
|
||||
assert not torch.equal(run(256), run(512))
|
||||
|
||||
|
||||
@_NVFP4_REQUIRES_SM100
|
||||
@pytest.mark.parametrize("dtype", _NVFP4_LINEAR_DTYPES)
|
||||
@pytest.mark.parametrize("shape", _NVFP4_LINEAR_CONSISTENCY_SHAPES)
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from utils import skip_if_not_cuda
|
||||
from utils import skip_if_not_cuda_alike
|
||||
|
||||
import vllm.lora.ops.triton_ops.lora_shrink_op as shrink_op
|
||||
import vllm.lora.ops.triton_ops.utils as lora_utils
|
||||
@@ -39,7 +39,7 @@ def _shrink(inputs, weights, mapping, scaling):
|
||||
return out
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@skip_if_not_cuda_alike
|
||||
@pytest.mark.parametrize(
|
||||
"hidden_size,rank,nslices,dtype",
|
||||
[
|
||||
@@ -68,7 +68,7 @@ def test_lora_shrink_matches_reference(hidden_size, rank, nslices, dtype):
|
||||
torch.testing.assert_close(out, ref, rtol=5e-3, atol=5e-3)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@skip_if_not_cuda_alike
|
||||
@pytest.mark.parametrize("nslices", [1, 3])
|
||||
def test_lora_shrink_invariant_across_batch_sizes(monkeypatch, nslices):
|
||||
reduce_kernel, reduce_grids = shrink_op._lora_shrink_reduce_kernel, []
|
||||
|
||||
@@ -10,7 +10,11 @@ import pytest
|
||||
import torch
|
||||
from utils import skip_unsupported
|
||||
|
||||
from vllm.model_executor.determinism.batch_invariant import matmul_batch_invariant
|
||||
import vllm.model_executor.determinism.batch_invariant as bi
|
||||
from vllm.model_executor.determinism.batch_invariant import (
|
||||
enable_batch_invariant_mode,
|
||||
matmul_batch_invariant,
|
||||
)
|
||||
from vllm.model_executor.determinism.batch_invariant_configs import (
|
||||
_BATCH_INVARIANT_MATMUL_TUNED_CONFIGS,
|
||||
_get_tuned_matmul_arch_family,
|
||||
@@ -129,3 +133,49 @@ def test_matmul_batch_invariance_across_tuned_m_buckets(m, transpose_b):
|
||||
batch_output = matmul_batch_invariant(a, b)
|
||||
|
||||
assert torch.equal(single_output[0], batch_output[0])
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def batch_invariant_mode():
|
||||
# The overrides are process-global: remove them so later tests compare
|
||||
# against the stock aten kernels again.
|
||||
enable_batch_invariant_mode()
|
||||
yield
|
||||
assert bi._batch_invariant_LIB is not None
|
||||
bi._batch_invariant_LIB._destroy()
|
||||
bi._batch_invariant_MODE = False
|
||||
|
||||
|
||||
@skip_unsupported
|
||||
@pytest.mark.usefixtures("batch_invariant_mode")
|
||||
@pytest.mark.parametrize("op", ["mm", "addmm", "bmm"])
|
||||
def test_compiled_matmul_reaches_batch_invariant_out_overloads(op):
|
||||
"""Inductor lowers mm/addmm/bmm to ``extern_kernels.<op>(..., out=buf)``,
|
||||
i.e. the ``.out`` overload, which eager never dispatches."""
|
||||
if (
|
||||
op != "bmm"
|
||||
and current_platform.is_cuda()
|
||||
and not current_platform.is_device_capability_family(80)
|
||||
):
|
||||
pytest.skip("mm/addmm are not overridden on SM90+")
|
||||
torch.manual_seed(0)
|
||||
dtype, k, n, needle = torch.bfloat16, 1024, 768, 37
|
||||
w = torch.randn(4, k, n, device=DEVICE_TYPE, dtype=dtype)
|
||||
bias = torch.randn(n, device=DEVICE_TYPE, dtype=dtype)
|
||||
fn = {
|
||||
"mm": lambda x: torch.mm(x[0], w[0]),
|
||||
# Bare addmm: a following pointwise op would make Inductor unfuse the bias.
|
||||
"addmm": lambda x: torch.addmm(bias, x[0], w[0]),
|
||||
"bmm": lambda x: torch.bmm(x, w),
|
||||
}[op]
|
||||
compiled = torch.compile(fn, fullgraph=True)
|
||||
x = torch.randn(4, 256, k, device=DEVICE_TYPE, dtype=dtype)
|
||||
row = None
|
||||
for m in (64, 256, 128):
|
||||
xs = x[:, :m].clone()
|
||||
# With dynamic=True, a specialization on M would silently recompile.
|
||||
torch._dynamo.mark_dynamic(xs, 1)
|
||||
out = compiled(xs)
|
||||
assert torch.equal(out, fn(xs)), f"compiled {op} bypassed the BI kernel (M={m})"
|
||||
row = out[..., needle, :] if row is None else row
|
||||
assert torch.equal(out[..., needle, :], row)
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The ROCm native MX linear kernels must not depend on the number of rows."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from utils import requires_mx
|
||||
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
SEED = 0
|
||||
|
||||
# E8M0 exponent half-width. With 4-bit operands and a narrow spread the fp32
|
||||
# partial sums are exact, so a reordering of them would be invisible.
|
||||
SCALE_SPREAD = 15
|
||||
|
||||
# (N, K, use_asm_gemm)
|
||||
MXFP4_CASES = [
|
||||
(4096, 2048, False),
|
||||
(2048, 6144, False),
|
||||
(4608, 7168, False),
|
||||
(8192, 2048, True),
|
||||
(8192, 1024, True),
|
||||
(8192, 14336, True),
|
||||
]
|
||||
|
||||
MXFP8_CASES = [(4096, 2048), (2048, 6144), (1024, 768), (1536, 1024)]
|
||||
|
||||
# The untuned AITER MXFP4 configs switch at M = 9/33/65/129/257/513.
|
||||
MXFP4_TOKEN_COUNTS = [
|
||||
1,
|
||||
2,
|
||||
8,
|
||||
9,
|
||||
16,
|
||||
32,
|
||||
33,
|
||||
64,
|
||||
65,
|
||||
128,
|
||||
129,
|
||||
256,
|
||||
257,
|
||||
512,
|
||||
513,
|
||||
1024,
|
||||
2048,
|
||||
]
|
||||
|
||||
MXFP8_TOKEN_COUNTS = [1, 32, 64, 65, 128, 129, 256, 257, 512, 1024, 1025, 2048]
|
||||
|
||||
# Row 0 lands in the first tile of every decomposition, so it can stay
|
||||
# invariant when the rest of the output does not.
|
||||
CHECK_ROWS = [0, 1, 2, 3, 7, 15, 31]
|
||||
|
||||
|
||||
def _probe(out: torch.Tensor) -> torch.Tensor:
|
||||
"""The leading rows of an output, kept so the sweep does not hold every GEMM."""
|
||||
return out[: CHECK_ROWS[-1] + 1].clone()
|
||||
|
||||
|
||||
def _variant_rows(probes: dict[int, torch.Tensor]) -> list[str]:
|
||||
"""Rows whose contents changed with the number of rows in the launch."""
|
||||
failures = []
|
||||
for row in CHECK_ROWS:
|
||||
counts = [n for n in sorted(probes) if n > row]
|
||||
if not counts:
|
||||
continue
|
||||
reference = probes[counts[0]][row]
|
||||
variant = [n for n in counts if not torch.equal(probes[n][row], reference)]
|
||||
if variant:
|
||||
failures.append(f"row {row} changed at row counts {variant}")
|
||||
return failures
|
||||
|
||||
|
||||
def _mxfp4_weights(n: int, k: int, use_asm_gemm: bool):
|
||||
"""Weights laid out as AiterMxfp4LinearKernel.process_weights_after_loading."""
|
||||
weight = torch.randint(0, 255, (n, k // 2), dtype=torch.uint8, device="cuda")
|
||||
weight_scale = torch.randint(
|
||||
127 - SCALE_SPREAD,
|
||||
128 + SCALE_SPREAD,
|
||||
(n, k // 32),
|
||||
dtype=torch.uint8,
|
||||
device="cuda",
|
||||
)
|
||||
if not use_asm_gemm:
|
||||
return weight, weight_scale.T.contiguous()
|
||||
|
||||
from aiter.ops.shuffle import shuffle_weight
|
||||
|
||||
sm, sn = weight_scale.shape
|
||||
weight_scale = (
|
||||
weight_scale.view(sm // 32, 2, 16, sn // 8, 2, 4, 1)
|
||||
.permute(0, 3, 5, 2, 4, 1, 6)
|
||||
.contiguous()
|
||||
.view(sm, sn)
|
||||
)
|
||||
return shuffle_weight(weight, layout=(16, 16)), weight_scale
|
||||
|
||||
|
||||
def _mxfp8_weights(n: int, k: int):
|
||||
"""E4M3 weights with E8M0 block scales, as the MXFP8 layer stores them."""
|
||||
weight = (torch.randn(n, k, device="cuda") / 8).to(torch.float8_e4m3fn)
|
||||
weight_scale = torch.randint(
|
||||
127 - SCALE_SPREAD,
|
||||
128 + SCALE_SPREAD,
|
||||
(n, k // 32),
|
||||
dtype=torch.uint8,
|
||||
device="cuda",
|
||||
)
|
||||
return weight, weight_scale
|
||||
|
||||
|
||||
@requires_mx
|
||||
@pytest.mark.parametrize("n,k,use_asm_gemm", MXFP4_CASES)
|
||||
def test_mxfp4_linear_is_batch_invariant(n: int, k: int, use_asm_gemm: bool):
|
||||
pytest.importorskip("aiter")
|
||||
|
||||
# Importing the module registers torch.ops.vllm.gemm_with_dynamic_quant.
|
||||
import vllm.model_executor.kernels.linear.mxfp4.aiter # noqa: F401
|
||||
|
||||
set_random_seed(SEED)
|
||||
weight, weight_scale = _mxfp4_weights(n, k, use_asm_gemm)
|
||||
x = torch.randn(max(MXFP4_TOKEN_COUNTS), k, device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
probes = {
|
||||
num_tokens: _probe(
|
||||
torch.ops.vllm.gemm_with_dynamic_quant(
|
||||
x[:num_tokens], weight, weight_scale, use_asm_gemm, torch.bfloat16
|
||||
)
|
||||
)
|
||||
for num_tokens in MXFP4_TOKEN_COUNTS
|
||||
}
|
||||
|
||||
failures = _variant_rows(probes)
|
||||
assert not failures, (
|
||||
f"MXFP4 linear depends on the row count "
|
||||
f"(N={n}, K={k}, asm={use_asm_gemm}):\n " + "\n ".join(failures)
|
||||
)
|
||||
|
||||
|
||||
@requires_mx
|
||||
@pytest.mark.parametrize("n,k", MXFP8_CASES)
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
def test_mxfp8_linear_is_batch_invariant(n: int, k: int, dtype: torch.dtype):
|
||||
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
|
||||
_mxfp8_dot_scaled_linear,
|
||||
)
|
||||
|
||||
set_random_seed(SEED)
|
||||
weight, weight_scale = _mxfp8_weights(n, k)
|
||||
x = torch.randn(max(MXFP8_TOKEN_COUNTS), k, device="cuda", dtype=dtype)
|
||||
|
||||
probes = {
|
||||
num_tokens: _probe(
|
||||
_mxfp8_dot_scaled_linear(x[:num_tokens], weight, weight_scale)
|
||||
)
|
||||
for num_tokens in MXFP8_TOKEN_COUNTS
|
||||
}
|
||||
|
||||
failures = _variant_rows(probes)
|
||||
assert not failures, (
|
||||
f"MXFP8 linear depends on the row count (N={n}, K={k}, {dtype}):\n "
|
||||
+ "\n ".join(failures)
|
||||
)
|
||||
@@ -9,10 +9,14 @@ import torch
|
||||
from utils import (
|
||||
_extract_step_logprobs,
|
||||
_random_prompt,
|
||||
skip_if_not_cuda,
|
||||
skip_if_not_cuda_alike,
|
||||
)
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# FLASH_ATTN is CUDA-only.
|
||||
E2E_BACKENDS = ["TRITON_ATTN"] if current_platform.is_rocm() else ["FLASH_ATTN"]
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not hasattr(torch, "float8_e4m3fn"),
|
||||
@@ -40,8 +44,8 @@ def _make_llm(max_num_seqs: int, backend: str) -> LLM:
|
||||
)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@pytest.mark.parametrize("backend", ["FLASH_ATTN"])
|
||||
@skip_if_not_cuda_alike
|
||||
@pytest.mark.parametrize("backend", E2E_BACKENDS)
|
||||
def test_dense_nvfp4_generation_is_deterministic_across_batch_sizes_e2e(backend):
|
||||
seed = int(os.getenv("VLLM_TEST_SEED", "12345"))
|
||||
random.seed(seed)
|
||||
|
||||
@@ -6,6 +6,7 @@ vLLM server and compare BS=1 vs BS=N results (tokens and per-step logprobs).
|
||||
Environment variables:
|
||||
- VLLM_TEST_MODEL: served model name (e.g., Qwen/Qwen3-1.7B / DeepSeek-R1)
|
||||
- VLLM_TP_SIZE: tensor parallelism size (e.g., 4)
|
||||
- VLLM_TEST_SERVER_ARGS: extra server args (e.g., "--data-parallel-size 4")
|
||||
|
||||
"""
|
||||
|
||||
@@ -16,7 +17,7 @@ from typing import Any
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
from utils import BACKENDS, TEST_MODEL, _random_prompt, skip_if_not_cuda
|
||||
from utils import BACKENDS, TEST_MODEL, _random_prompt, skip_if_not_cuda_alike
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
@@ -132,7 +133,7 @@ def _compare_bs1_vs_bsn_single_process(
|
||||
)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@skip_if_not_cuda_alike
|
||||
@pytest.mark.parametrize("backend", BACKENDS)
|
||||
def test_logprobs_bitwise_batch_invariance_bs1_vs_bsN(
|
||||
backend: str,
|
||||
@@ -156,6 +157,7 @@ def test_logprobs_bitwise_batch_invariance_bs1_vs_bsN(
|
||||
]
|
||||
if tp_size:
|
||||
server_args += ["-tp", tp_size]
|
||||
server_args += os.getenv("VLLM_TEST_SERVER_ARGS", "").split()
|
||||
|
||||
with RemoteOpenAIServer(TEST_MODEL, server_args) as server:
|
||||
client = server.get_client()
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from utils import skip_if_not_cuda, skip_unsupported
|
||||
from utils import skip_if_not_cuda_alike, skip_unsupported
|
||||
|
||||
from vllm.model_executor.determinism.batch_invariant import (
|
||||
rms_norm_batch_invariant,
|
||||
@@ -27,7 +27,7 @@ def _rms_norm_reference(
|
||||
return (output * weight.float()).to(input_tensor.dtype)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@skip_if_not_cuda_alike
|
||||
@pytest.mark.parametrize("batch_size", [1, 4, 64, 300])
|
||||
@pytest.mark.parametrize("hidden_size", [512, 2048, 4096, 8192])
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@@ -76,7 +76,7 @@ def test_rms_norm_batch_invariant_vs_reference(
|
||||
)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@skip_if_not_cuda_alike
|
||||
@pytest.mark.parametrize("hidden_size", [512, 4096])
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("eps", [1e-6])
|
||||
@@ -189,7 +189,7 @@ def _assert_rows_bit_identical(small, large, msg):
|
||||
torch.testing.assert_close(small, large, rtol=0.0, atol=0.0, msg=msg)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@skip_if_not_cuda_alike
|
||||
@pytest.mark.parametrize("hidden_size", [512, 4096])
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("seed", list(range(4)))
|
||||
@@ -221,7 +221,7 @@ def test_rms_norm_batch_invariant_nonresidual_kernel(
|
||||
)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@skip_if_not_cuda_alike
|
||||
@pytest.mark.parametrize("hidden_size", [512, 4096])
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("seed", list(range(4)))
|
||||
@@ -267,7 +267,7 @@ def test_rms_norm_static_fp8_quant_batch_invariant(
|
||||
)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@skip_if_not_cuda_alike
|
||||
@pytest.mark.parametrize("hidden_size", [512, 4096])
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("seed", list(range(4)))
|
||||
@@ -303,7 +303,7 @@ def test_rms_norm_per_block_quant_batch_invariant(
|
||||
)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@skip_if_not_cuda_alike
|
||||
@pytest.mark.parametrize("batch_size", [1, 16, 128])
|
||||
@pytest.mark.parametrize("seq_len", [1, 32, 512])
|
||||
@pytest.mark.parametrize("hidden_size", [2048, 4096])
|
||||
@@ -343,7 +343,7 @@ def test_rms_norm_3d_input(
|
||||
)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@skip_if_not_cuda_alike
|
||||
def test_rms_norm_numerical_stability(default_vllm_config):
|
||||
"""Test RMS norm numerical stability with extreme values.
|
||||
|
||||
@@ -431,7 +431,7 @@ def test_rms_norm_formula(default_vllm_config):
|
||||
)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@skip_if_not_cuda_alike
|
||||
@pytest.mark.parametrize("hidden_size", [128, 1024, 4096, 16384])
|
||||
def test_rms_norm_different_hidden_sizes(default_vllm_config, hidden_size: int):
|
||||
"""Test RMS norm with various hidden sizes to ensure block size handling.
|
||||
|
||||
@@ -34,12 +34,16 @@ DEVICE_BACKENDS: dict[str, DeviceConfig] = {
|
||||
available=current_platform.is_xpu() and HAS_TRITON,
|
||||
backends=["TRITON_ATTN"],
|
||||
),
|
||||
"rocm": DeviceConfig(
|
||||
available=current_platform.is_rocm() and HAS_TRITON,
|
||||
backends=["TRITON_ATTN"],
|
||||
),
|
||||
}
|
||||
|
||||
DEFAULT_MODEL = "Qwen/Qwen3-1.7B"
|
||||
TEST_MODEL = os.getenv("VLLM_TEST_MODEL", DEFAULT_MODEL)
|
||||
|
||||
# Override backends for MLA models (MLA only supported on CUDA).
|
||||
# Override backends for MLA models.
|
||||
if os.getenv("VLLM_TEST_MODEL"):
|
||||
config = get_config(TEST_MODEL, trust_remote_code=False)
|
||||
if ModelArchConfigConvertorBase(config, config.get_text_config()).is_deepseek_mla():
|
||||
@@ -52,6 +56,10 @@ if os.getenv("VLLM_TEST_MODEL"):
|
||||
available=DEVICE_BACKENDS["xpu"].available,
|
||||
backends=[],
|
||||
)
|
||||
DEVICE_BACKENDS["rocm"] = DeviceConfig(
|
||||
available=DEVICE_BACKENDS["rocm"].available,
|
||||
backends=["TRITON_MLA"],
|
||||
)
|
||||
|
||||
# Only include backends for devices that are actually available.
|
||||
BACKENDS: list[str] = sorted(
|
||||
@@ -60,7 +68,7 @@ BACKENDS: list[str] = sorted(
|
||||
|
||||
skip_unsupported = pytest.mark.skipif(
|
||||
not any(cfg.available for cfg in DEVICE_BACKENDS.values()),
|
||||
reason="Requires CUDA >= Ampere (SM80) or Intel XPU with Triton",
|
||||
reason="Requires CUDA >= Ampere (SM80), ROCm, or Intel XPU with Triton",
|
||||
)
|
||||
|
||||
skip_if_not_cuda = pytest.mark.skipif(
|
||||
@@ -68,6 +76,16 @@ skip_if_not_cuda = pytest.mark.skipif(
|
||||
reason="Requires CUDA >= Ampere (SM80)",
|
||||
)
|
||||
|
||||
skip_if_not_cuda_alike = pytest.mark.skipif(
|
||||
not (DEVICE_BACKENDS["cuda"].available or DEVICE_BACKENDS["rocm"].available),
|
||||
reason="Requires CUDA >= Ampere (SM80) or ROCm",
|
||||
)
|
||||
|
||||
requires_mx = pytest.mark.skipif(
|
||||
not (current_platform.is_rocm() and current_platform.supports_mx()),
|
||||
reason="requires a ROCm device with native MX support (gfx95x)",
|
||||
)
|
||||
|
||||
|
||||
def _random_prompt(min_words: int = 1024, max_words: int = 1024 * 2) -> str:
|
||||
# Generate more realistic prompts that will actually produce varied tokens
|
||||
|
||||
Reference in New Issue
Block a user