[Mini-VLLM] Repair test suite after SM86 cut

- Restore example connector, test helpers, and dropped patch tests
- Guard MLA metadata import in the example connector
- Point the PLE test at common.ops.ple after upstream #60021 move
- Cut tests for removed features: tools, DRY, epd proxy, STT batch,
  diffusion scheduler, INC/Quark PLE cases, mamba-2 mixer

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 01:06:30 +07:00
co-authored by Hinata
parent 0152af1336
commit 54a396586e
112 changed files with 55726 additions and 10371 deletions
@@ -1,90 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.utils.import_utils import has_helion
if not has_helion():
pytest.skip("Helion is not installed", allow_module_level=True)
from tools import benchmark_helion_kernels
check_correctness = benchmark_helion_kernels.check_correctness
class _FakeKernel:
helion_settings = None
def __init__(self, offset: float):
self.calls = 0
self.offset = offset
def __call__(self, output: torch.Tensor, input: torch.Tensor) -> torch.Tensor:
self.calls += 1
output.copy_(input + self.offset)
return input * 2
def test_check_correctness_runs_once_without_mutating_benchmark_inputs():
kernel = _FakeKernel(offset=1)
baseline_calls = 0
def baseline(output: torch.Tensor, input: torch.Tensor) -> torch.Tensor:
nonlocal baseline_calls
baseline_calls += 1
output.copy_(input + 1)
return input * 2
output = torch.zeros(4)
inputs = (output, torch.arange(4))
check_correctness(kernel, baseline, inputs, "matching")
assert kernel.calls == 1
assert baseline_calls == 1
torch.testing.assert_close(output, torch.zeros(4))
def test_check_correctness_reports_mutated_input_mismatch():
kernel = _FakeKernel(offset=2)
def baseline(output: torch.Tensor, input: torch.Tensor) -> torch.Tensor:
output.copy_(input + 1)
return input * 2
inputs = (torch.zeros(4), torch.arange(4))
with pytest.raises(AssertionError, match="Numerics check failed for case bad"):
check_correctness(kernel, baseline, inputs, "bad")
def test_check_correctness_reports_return_value_mismatch():
kernel = _FakeKernel(offset=1)
def baseline(output: torch.Tensor, input: torch.Tensor) -> torch.Tensor:
output.copy_(input + 1)
return input * 3
inputs = (torch.zeros(4), torch.arange(4))
with pytest.raises(AssertionError, match="Numerics check failed for case bad"):
check_correctness(kernel, baseline, inputs, "bad")
def test_log_versions(monkeypatch):
messages = []
def record(message: str, *args):
messages.append(message % args)
monkeypatch.setattr(benchmark_helion_kernels.logger, "info", record)
benchmark_helion_kernels.log_versions()
assert [message.split(":", 1)[0] for message in messages] == [
"torch",
"helion",
"triton",
]
-140
View File
@@ -1,140 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import unittest
import pytest
import torch
from tests.utils import ensure_current_vllm_config, multi_gpu_test
from vllm.distributed.parallel_state import (
init_distributed_environment,
initialize_model_parallel,
)
from vllm.model_executor.layers.mamba.mamba_mixer2 import Mixer2RMSNormGated
from vllm.utils.system_utils import update_environment_variables
from vllm.utils.torch_utils import set_random_seed
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize("batch_size", [8])
@pytest.mark.parametrize("seq_len", [128])
@pytest.mark.parametrize(
"hidden_size_n_groups",
[
(64, 1),
(64, 2),
(64, 4), # hidden_size be divisible by num_gpus
],
)
@pytest.mark.parametrize("dtype", [torch.float16])
def test_mixer2_gated_norm_multi_gpu(
batch_size: int,
seq_len: int,
hidden_size_n_groups: tuple[int, int],
dtype: torch.dtype,
device: str = "cuda",
):
hidden_size, n_groups = hidden_size_n_groups
num_processes = 2
def run_torch_spawn(fn, nprocs):
# need to use torch.mp.spawn otherwise will have problems with
# torch.distributed and cuda
torch.multiprocessing.spawn(
fn,
args=(
num_processes,
batch_size,
seq_len,
hidden_size,
n_groups,
dtype,
device,
),
nprocs=nprocs,
)
run_torch_spawn(mixer2_gated_norm_tensor_parallel, 2)
def mixer2_gated_norm_tensor_parallel(
local_rank: int,
world_size: int,
batch_size: int,
seq_len: int,
hidden_size: int,
n_groups: int,
dtype: torch.dtype,
device: str,
):
set_random_seed(0)
device = torch.device(f"cuda:{local_rank}")
torch.accelerator.set_device_index(device)
torch.set_default_device(device)
torch.set_default_dtype(dtype)
update_environment_variables(
{
"RANK": str(local_rank),
"LOCAL_RANK": str(local_rank),
"WORLD_SIZE": str(world_size),
"MASTER_ADDR": "localhost",
"MASTER_PORT": "12345",
}
)
# initialize distributed
init_distributed_environment()
with ensure_current_vllm_config():
initialize_model_parallel(tensor_model_parallel_size=world_size)
# create random weights an inputs
weight = torch.rand((hidden_size,), dtype=dtype, device=device)
hidden_states = torch.randn(batch_size, seq_len, hidden_size)
gate_states = torch.randn(batch_size, seq_len, hidden_size)
# create gated-norm with TP
with ensure_current_vllm_config():
mixer = Mixer2RMSNormGated(
full_hidden_size=hidden_size,
full_n_groups=n_groups,
)
mixer.weight.weight_loader(mixer.weight, weight) # load
# create gated-norm without TP to compute reference
# - utilize mock patching to disable TP when
with (
ensure_current_vllm_config(),
unittest.mock.patch(
"vllm.model_executor.layers.mamba.mamba_mixer2."
"get_tensor_model_parallel_world_size",
return_value=1,
),
unittest.mock.patch(
"vllm.model_executor.layers.mamba.mamba_mixer2."
"get_tensor_model_parallel_rank",
return_value=0,
),
):
mixer_single_gpu = Mixer2RMSNormGated(
full_hidden_size=hidden_size,
full_n_groups=n_groups,
)
# assign weight to single-gpu mixer
mixer_single_gpu.weight.data = weight
# generate and compare
N = hidden_size // world_size
output = mixer(
hidden_states[..., local_rank * N : (local_rank + 1) * N],
gate_states[..., local_rank * N : (local_rank + 1) * N],
)
ref_output = mixer_single_gpu(hidden_states, gate_states)
torch.testing.assert_close(
output,
ref_output[..., local_rank * N : (local_rank + 1) * N],
atol=5e-3,
rtol=1e-3,
)
+1 -1
View File
@@ -13,7 +13,7 @@ import pytest
import torch
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
from vllm.models.qwen4_exp.nvidia.ops.ple import ple_conv
from vllm.models.qwen4_exp.common.ops.ple import ple_conv
from vllm.v1.attention.backends.ple_recoverssm import _PleConvCommit
C, KSIZE, DIL, SQ, NB = 512, 4, 3, 4, 8
+634
View File
@@ -0,0 +1,634 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit-test DeepGEMM FP8 and FP4 kernels (no DeepEP).
Compare DeepGEMM path against the Triton fallback inside vLLM's fused_experts.
"""
import importlib
import math
import pytest
import torch
# vLLM fused-expert reference (Triton fallback + DeepGEMM option)
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from tests.kernels.moe.utils import make_dummy_moe_config
from vllm.model_executor.layers.fused_moe.activation import (
MoEActivation,
)
from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
FusedMoEQuantDesc,
fp8_w8a8_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.deep_gemm_utils import (
deepgemm_moe_permute,
ep_gather,
)
from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import (
_fp8_workspace_shape,
)
from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import (
TritonOrDeepGemmExperts,
)
from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
per_token_group_quant_fp8,
)
from vllm.utils.deep_gemm import (
calc_diff,
is_deep_gemm_supported,
per_block_cast_to_fp8,
)
BLOCK_SIZE = [128, 128]
def test_ep_gather_uses_64_bit_row_offsets():
hidden_size = 4096
source_row = 1 << 18
input_tensor = torch.empty(
(source_row + 1, hidden_size),
device="cuda",
dtype=torch.bfloat16,
)
input_tensor[source_row].fill_(1)
output = torch.empty((1, hidden_size), device="cuda", dtype=torch.bfloat16)
ep_gather(
input_tensor=input_tensor,
recv_topk_ids=torch.zeros((1, 1), device="cuda", dtype=torch.int64),
recv_topk_weight=torch.ones((1, 1), device="cuda", dtype=torch.float32),
input_index=torch.full((1, 1), source_row, device="cuda", dtype=torch.int32),
expert_map=None,
output_tensor=output,
)
torch.testing.assert_close(output, torch.ones_like(output), rtol=0, atol=0)
def test_ep_gather_uses_64_bit_output_row_offsets():
hidden_size = 6144
output_row = (1 << 31) // hidden_size + 1
num_rows = output_row + 1
output = torch.empty(
(num_rows, hidden_size),
device="cuda",
dtype=torch.bfloat16,
)
ep_gather(
input_tensor=torch.ones((1, hidden_size), device="cuda", dtype=torch.bfloat16),
recv_topk_ids=torch.zeros((num_rows, 1), device="cuda", dtype=torch.int64),
recv_topk_weight=torch.ones((num_rows, 1), device="cuda", dtype=torch.float32),
input_index=torch.zeros((num_rows, 1), device="cuda", dtype=torch.int32),
expert_map=None,
output_tensor=output,
)
torch.testing.assert_close(output, torch.ones_like(output), rtol=0, atol=0)
@pytest.mark.parametrize("workspace_dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("num_columns", [2048, 6144, 6145])
def test_fp8_workspace_shape(workspace_dtype, num_columns):
num_rows = 17
shape = _fp8_workspace_shape(num_rows, num_columns, workspace_dtype)
allocated_bytes = math.prod(shape) * workspace_dtype.itemsize
required_bytes = num_rows * num_columns * torch.float8_e4m3fn.itemsize
assert allocated_bytes >= required_bytes
assert allocated_bytes - required_bytes < num_rows * workspace_dtype.itemsize
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
def test_deepgemm_moe_permute_initializes_padding_scales(workspace_init):
hidden_states = torch.randn(2, 128, device="cuda", dtype=torch.bfloat16)
activations, scales = per_token_group_quant_fp8(
hidden_states,
group_size=128,
use_ue8m0=True,
)
topk_ids = torch.tensor([[0], [1]], device="cuda", dtype=torch.int64)
_, permuted_scales, expert_ids, _, _ = deepgemm_moe_permute(
aq=activations,
aq_scale=scales,
topk_ids=topk_ids,
local_num_experts=2,
expert_map=None,
expert_tokens_meta=None,
)
padding = expert_ids < 0
assert padding.any()
torch.testing.assert_close(
permuted_scales[padding],
torch.zeros_like(permuted_scales[padding]),
rtol=0,
atol=0,
)
def make_block_quant_fp8_weights(
e: int,
n: int,
k: int,
block_size: list[int],
):
"""Generate (w1, w2) expert weights and their per-block scale tensors
in FP8 block-quantized format.
w1 shape: (E, 2N, K)
w2 shape: (E, K, N)
"""
dtype = torch.bfloat16
fp8_max, fp8_min = (
torch.finfo(torch.float8_e4m3fn).max,
torch.finfo(torch.float8_e4m3fn).min,
)
# bf16 reference weights
w1_bf16 = torch.randn(e, 2 * n, k, device="cuda", dtype=dtype) / 10
w2_bf16 = torch.randn(e, k, n, device="cuda", dtype=dtype) / 10
w1_bf16.clamp_(fp8_min, fp8_max)
w2_bf16.clamp_(fp8_min, fp8_max)
block_n, block_k = block_size
n_tiles_w1 = math.ceil((2 * n) / block_n)
k_tiles_w1 = math.ceil(k / block_k)
n_tiles_w2 = math.ceil(k / block_n)
k_tiles_w2 = math.ceil(n / block_k)
w1 = torch.empty_like(w1_bf16, dtype=torch.float8_e4m3fn)
w2 = torch.empty_like(w2_bf16, dtype=torch.float8_e4m3fn)
w1_s = torch.empty(e, n_tiles_w1, k_tiles_w1, device="cuda", dtype=torch.float32)
w2_s = torch.empty(e, n_tiles_w2, k_tiles_w2, device="cuda", dtype=torch.float32)
for i in range(e):
w1[i], w1_s[i] = per_block_cast_to_fp8(
w1_bf16[i], block_size=block_size, use_ue8m0=True
)
w2[i], w2_s[i] = per_block_cast_to_fp8(
w2_bf16[i], block_size=block_size, use_ue8m0=True
)
return w1, w2, w1_s, w2_s
@pytest.mark.parametrize("num_tokens", [16, 129, 512])
@pytest.mark.parametrize("with_metadata", [False, True])
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
def test_block_fp8_padding_on_graph_replay(num_tokens, with_metadata, monkeypatch):
"""Both FP8 GEMMs consume only live rows as routing changes on replay."""
from unittest.mock import Mock
import vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe as impl
from vllm.utils.deep_gemm import DeepGemmQuantScaleFMT
if not (
impl.current_platform.is_device_capability_family(90)
or (
impl.current_platform.is_device_capability_family(100)
and DeepGemmQuantScaleFMT.from_oracle() == DeepGemmQuantScaleFMT.UE8M0
)
):
pytest.skip("Prefix-sum FP8 dispatch requires SM90 or SM10.x")
torch.manual_seed(0)
e, n, k, topk = 8, 1024, 512, 2
x = torch.randn(num_tokens, k, device="cuda", dtype=torch.bfloat16) / 10
aq, scales = per_token_group_quant_fp8(x, 128, use_ue8m0=True)
w1, w2, s1, s2 = make_block_quant_fp8_weights(e, n, k, BLOCK_SIZE)
quant = fp8_w8a8_moe_quant_config(w1_scale=s1, w2_scale=s2, block_shape=BLOCK_SIZE)
experts = impl.DeepGemmExperts(make_dummy_moe_config(), quant)
ids = torch.zeros(num_tokens, topk, device="cuda", dtype=torch.int64)
weights = torch.full((num_tokens, topk), 0.5, device="cuda")
counts = torch.zeros(e, device="cuda", dtype=torch.int32)
metadata = mk.ExpertTokensMetadata(counts, None) if with_metadata else None
shapes = experts.workspace_shapes(
num_tokens, 2 * n, k, topk, e, e, metadata, MoEActivation.SILU
)
def route(mode):
ids[:, 0] = torch.arange(num_tokens, device="cuda") % e
ids[:, 1] = (ids[:, 0] + 1) % e
if mode == 1:
ids.fill_(-1)
ids[:1, 0] = e - 1
elif mode == 2:
ids.fill_(-1)
counts.copy_(torch.bincount(ids[ids >= 0], minlength=e).int())
route(0)
variants = []
for dense in (True, False):
ws1, ws2, output = [
torch.empty(shape, device="cuda", dtype=torch.bfloat16) for shape in shapes
]
def invoke(output=output, ws1=ws1, ws2=ws2):
experts.apply(
output,
aq,
w1,
w2,
weights,
ids,
MoEActivation.SILU,
e,
None,
scales,
None,
ws1,
ws2,
metadata,
False,
)
with monkeypatch.context() as mp:
if dense:
platform = Mock(wraps=impl.current_platform)
platform.is_device_capability_family.return_value = False
mp.setattr(impl, "current_platform", platform)
gemm = impl.m_grouped_fp8_gemm_nt_contiguous
calls: list[int] = []
def check_layout(*args, dense=dense, calls=calls, gemm=gemm, **kwargs):
assert kwargs.get("use_psum_layout", False) == (not dense)
calls.append(args[3].numel())
return gemm(*args, **kwargs)
mp.setattr(impl, "m_grouped_fp8_gemm_nt_contiguous", check_layout)
invoke()
assert len(calls) == 2
if not dense:
assert calls == [e, e]
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
invoke()
variants.append((graph, output, ws1, ws2))
for mode in (0, 1, 2, 0):
route(mode)
for graph, _, ws1, ws2 in variants:
ws1.fill_(float("nan"))
ws2.fill_(float("nan"))
graph.replay()
torch.accelerator.synchronize()
torch.testing.assert_close(variants[0][1], variants[1][1], rtol=0, atol=0)
assert torch.isfinite(variants[1][1]).all()
@pytest.mark.parametrize(("n", "k"), [(2048, 512), (512, 1024)])
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
def test_deepgemm_prefix_sum_leaves_unused_capacity_untouched(n, k):
"""Both projection shapes skip unused tiles as counts change on replay."""
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import (
m_grouped_fp8_gemm_nt_contiguous,
mk_alignment_scope,
)
if not (
current_platform.is_device_capability_family(90)
or current_platform.is_device_capability_family(100)
):
pytest.skip("Requires SM90 or SM10.x")
capacity, experts, alignment = 512, 4, 128
x = torch.randn(capacity, k, device="cuda", dtype=torch.bfloat16) / 10
aq, scales = per_token_group_quant_fp8(x, 128, use_ue8m0=True)
w, _, ws, _ = make_block_quant_fp8_weights(experts, n // 2, k, BLOCK_SIZE)
ends = torch.tensor([0, 1, 128, 129], device="cuda", dtype=torch.int32)
output = torch.empty(capacity, n, device="cuda", dtype=torch.bfloat16)
def invoke():
with mk_alignment_scope(alignment):
m_grouped_fp8_gemm_nt_contiguous(
(aq, scales), (w, ws), output, ends, use_psum_layout=True
)
invoke()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
invoke()
for bounds, allocated_end in (([0, 1, 128, 129], 256), ([0] * 4, 0)):
ends.copy_(torch.tensor(bounds, device="cuda", dtype=torch.int32))
output.fill_(91)
graph.replay()
torch.accelerator.synchronize()
assert torch.all(output[allocated_end:] == 91)
if allocated_end:
assert torch.all(output[[0, 128]] != 91)
def run_single_case(m, n, k, topk, num_experts, block_size):
"""Run one (M,N,K) configuration on a single GPU and assert DeepGEMM ==
Triton baseline within tolerance.
"""
tokens_bf16 = (
torch.randn(m, k, device="cuda", dtype=torch.bfloat16)
.clamp_min_(-1)
.clamp_max_(1)
)
_, a1_scale = per_token_group_quant_fp8(tokens_bf16, block_size[1])
# expert weight tensors
w1, w2, w1_s, w2_s = make_block_quant_fp8_weights(num_experts, n, k, block_size)
router_logits = torch.randn(m, num_experts, device="cuda", dtype=torch.float32)
topk_weights, topk_ids = torch.topk(router_logits, k=topk, dim=-1)
topk_weights = torch.nn.functional.softmax(topk_weights, dim=-1)
quant_config = fp8_w8a8_moe_quant_config(
w1_scale=w1_s,
w2_scale=w2_s,
a1_scale=a1_scale,
block_shape=block_size,
)
moe_config = make_dummy_moe_config()
deep_gemm_experts = mk.FusedMoEKernel(
prepare_finalize=maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
use_monolithic=False,
),
fused_experts=TritonOrDeepGemmExperts(
moe_config=moe_config,
quant_config=quant_config,
),
)
# triton reference
out_triton = fused_experts(
hidden_states=tokens_bf16,
w1=w1,
w2=w2,
topk_weights=topk_weights,
topk_ids=topk_ids,
quant_config=quant_config,
)
# DeepGemm
out_deepgemm = deep_gemm_experts.apply(
hidden_states=tokens_bf16,
w1=w1,
w2=w2,
topk_weights=topk_weights,
topk_ids=topk_ids,
global_num_experts=num_experts,
activation=MoEActivation.SILU,
apply_router_weight_on_input=False,
expert_map=None,
)
diff = calc_diff(out_deepgemm, out_triton)
assert diff < 0.001, f"Diff exceeded 1%: {diff}"
# Note: N <= 512 will disable the deepgemm path due to performance issues.
MNKs = [
(1024, 768, 128),
(2048, 768, 512),
(512, 1024, 1024),
(4096, 4096, 1024),
]
TOPKS = [2, 6]
NUM_EXPERTS = [32]
@pytest.mark.parametrize(("m", "n", "k"), MNKs)
@pytest.mark.parametrize("topk", TOPKS)
@pytest.mark.parametrize("num_experts", NUM_EXPERTS)
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
def test_deepgemm_vs_triton(m, n, k, topk, num_experts, monkeypatch, workspace_init):
with monkeypatch.context() as mp:
mp.setenv("VLLM_USE_DEEP_GEMM", "1")
_DeepGemmExperts = importlib.import_module(
"vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe"
).DeepGemmExperts
call_counter = {"cnt": 0}
orig_fn = _DeepGemmExperts.apply
def _spy_apply(*args, **kwargs):
call_counter["cnt"] += 1
return orig_fn(*args, **kwargs)
monkeypatch.setattr(_DeepGemmExperts, "apply", _spy_apply)
if topk > num_experts:
pytest.skip(f"topk={topk} > num_experts={num_experts}")
run_single_case(
m=m,
n=n,
k=k,
topk=topk,
num_experts=num_experts,
block_size=BLOCK_SIZE,
)
# ensure that the DeepGEMM path was indeed taken.
assert call_counter["cnt"] == 1, (
f"DeepGEMM path was not executed during the test. "
f"Call counter: {call_counter['cnt']}"
)
# ---------------------------------------------------------------------------
# FP4 weight tests (DeepGEMM m_grouped_fp8_fp4_gemm_nt_contiguous)
# ---------------------------------------------------------------------------
def make_mxfp4_weights(
e: int,
n: int,
k: int,
):
"""Generate (w1, w2) expert weights in MXFP4 packed format with float32 scales,
plus BF16 reference weights for validation.
w1 shape: (E, 2N, K//2) uint8 — packed FP4
w2 shape: (E, K, N//2) uint8 — packed FP4
w1_s shape: (E, 2N, K//32) float32 — per-row block-32 scales
w2_s shape: (E, K, N//32) float32 — per-row block-32 scales
w1_bf16: (E, 2N, K) — original BF16 for reference
w2_bf16: (E, K, N) — original BF16 for reference
"""
from deep_gemm.utils.math import per_token_cast_to_fp4
dtype = torch.bfloat16
gran_k = 32 # MXFP4 block size
# bf16 reference weights — scale by 1/sqrt(dim) for numerical stability
w1_bf16 = torch.randn(e, 2 * n, k, device="cuda", dtype=dtype) * (k**-0.5)
w2_bf16 = torch.randn(e, k, n, device="cuda", dtype=dtype) * (n**-0.5)
# Quantize per-expert to FP4
w1 = torch.empty(e, 2 * n, k // 2, device="cuda", dtype=torch.uint8)
w2 = torch.empty(e, k, n // 2, device="cuda", dtype=torch.uint8)
w1_s = torch.empty(
e, 2 * n, math.ceil(k / gran_k), device="cuda", dtype=torch.float32
)
w2_s = torch.empty(e, k, math.ceil(n / gran_k), device="cuda", dtype=torch.float32)
for i in range(e):
w1[i], w1_s[i] = per_token_cast_to_fp4(
w1_bf16[i].float(), use_ue8m0=True, gran_k=gran_k
)
w2[i], w2_s[i] = per_token_cast_to_fp4(
w2_bf16[i].float(), use_ue8m0=True, gran_k=gran_k
)
return w1, w2, w1_s, w2_s, w1_bf16, w2_bf16
def _bf16_moe_reference(x, w1, w2, topk_weights, topk_ids):
"""BF16 token-loop MoE reference for correctness testing."""
import torch.nn.functional as F
num_tokens, hidden_size = x.shape
intermediate = w1.shape[1] // 2
top_k = topk_ids.shape[1]
output = torch.zeros(num_tokens, hidden_size, dtype=torch.float32, device=x.device)
for t in range(num_tokens):
for kk in range(top_k):
e = topk_ids[t, kk].item()
w = topk_weights[t, kk].item()
fc1 = x[t : t + 1].float() @ w1[e].float().T
linear = fc1[:, :intermediate]
gate = fc1[:, intermediate:]
act = F.silu(gate) * linear
fc2 = act @ w2[e].float().T
output[t] += w * fc2[0]
return output.to(torch.bfloat16)
def run_single_fp4_case(m, n, k, topk, num_experts):
"""Run one (M,N,K) configuration with FP4 weights on DeepGEMM and assert
DeepGEMM FP4 == BF16 reference within tolerance.
"""
tokens_bf16 = torch.randn(m, k, device="cuda", dtype=torch.bfloat16) * (k**-0.5)
# FP4 expert weight tensors + BF16 originals for reference
w1, w2, w1_s, w2_s, w1_bf16, w2_bf16 = make_mxfp4_weights(num_experts, n, k)
router_logits = torch.randn(m, num_experts, device="cuda", dtype=torch.float32)
topk_weights, topk_ids = torch.topk(router_logits, k=topk, dim=-1)
topk_weights = torch.nn.functional.softmax(topk_weights, dim=-1)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
)
from vllm.platforms import current_platform
_fp8_dtype = current_platform.fp8_dtype()
_block_shape = GroupShape(128, 128)
quant_config = FusedMoEQuantConfig(
_a1=FusedMoEQuantDesc(_fp8_dtype, _block_shape, None, None, None, None),
_a2=FusedMoEQuantDesc(_fp8_dtype, _block_shape, None, None, None, None),
_w1=FusedMoEQuantDesc("mxfp4", None, w1_s, None, None, None),
_w2=FusedMoEQuantDesc("mxfp4", None, w2_s, None, None, None),
)
moe_config = make_dummy_moe_config()
from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import (
DeepGemmFP4Experts,
)
deep_gemm_fp4_experts = mk.FusedMoEKernel(
prepare_finalize=maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
use_monolithic=False,
),
fused_experts=DeepGemmFP4Experts(
moe_config=moe_config,
quant_config=quant_config,
),
)
# DeepGEMM FP4 path
out_deepgemm_fp4 = deep_gemm_fp4_experts.apply(
hidden_states=tokens_bf16,
w1=w1,
w2=w2,
topk_weights=topk_weights,
topk_ids=topk_ids,
global_num_experts=num_experts,
activation=MoEActivation.SILU,
apply_router_weight_on_input=False,
expert_map=None,
)
# BF16 reference using the same original weights
out_ref = _bf16_moe_reference(tokens_bf16, w1_bf16, w2_bf16, topk_weights, topk_ids)
# FP4 vs BF16 reference: quantization error from FP4 weights + FP8 activations
diff = calc_diff(out_deepgemm_fp4, out_ref)
assert diff < 0.05, f"FP4 diff exceeded 5%: {diff}"
# DeepSeek V4 dims: H=4096, I=2048, so N=2*I=4096, K=H=4096.
# FP4 quantization with block_k=32 needs large K for good accuracy.
FP4_MNKs = [
(128, 4096, 4096), # DeepSeek V4 shape
(256, 2048, 2048), # Half-size variant
(128, 384, 3584), # Kimi-K3 TP8 latent-MoE shape
]
FP4_TOPKS = [2]
FP4_NUM_EXPERTS = [8]
@pytest.mark.parametrize(("m", "n", "k"), FP4_MNKs)
@pytest.mark.parametrize("topk", FP4_TOPKS)
@pytest.mark.parametrize("num_experts", FP4_NUM_EXPERTS)
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
def test_deepgemm_fp4_vs_triton(
m, n, k, topk, num_experts, monkeypatch, workspace_init
):
pytest.importorskip("deep_gemm.utils.math")
with monkeypatch.context() as mp:
mp.setenv("VLLM_USE_DEEP_GEMM", "1")
_DeepGemmFP4Experts = importlib.import_module(
"vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe"
).DeepGemmFP4Experts
call_counter = {"cnt": 0}
orig_fn = _DeepGemmFP4Experts.apply
def _spy_apply(*args, **kwargs):
call_counter["cnt"] += 1
return orig_fn(*args, **kwargs)
monkeypatch.setattr(_DeepGemmFP4Experts, "apply", _spy_apply)
if topk > num_experts:
pytest.skip(f"topk={topk} > num_experts={num_experts}")
run_single_fp4_case(
m=m,
n=n,
k=k,
topk=topk,
num_experts=num_experts,
)
# ensure that the DeepGEMM FP4 path was indeed taken.
assert call_counter["cnt"] == 1, (
f"DeepGEMM FP4 path was not executed during the test. "
f"Call counter: {call_counter['cnt']}"
)
+658
View File
@@ -0,0 +1,658 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import torch
import vllm._custom_ops as ops
from tests.kernels.quant_utils import per_block_cast_to_int8
from tests.kernels.quantization.nvfp4_utils import FLOAT4_E2M1_MAX, FLOAT8_E4M3_MAX
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
BatchedTritonExperts,
NaiveBatchedExperts,
)
from vllm.model_executor.layers.fused_moe.experts.triton_moe import (
TritonExperts,
)
from vllm.model_executor.layers.fused_moe.fused_moe import (
fused_experts,
)
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import moe_unpermute
from vllm.model_executor.layers.fused_moe.prepare_finalize.batched import (
BatchedPrepareAndFinalize,
)
from vllm.model_executor.layers.fused_moe.router.fused_topk_router import fused_topk
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
from vllm.model_executor.layers.quantization.utils.fp8_utils import is_fp8
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
ref_nvfp4_quant,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import per_block_cast_to_fp8
from vllm.utils.math_utils import round_up
DEVICE = current_platform.device_type
def shuffle_weight(w: torch.Tensor) -> torch.Tensor:
"""Fold weights to adjacent locations for Triton MoE / SwiGLU kernel layout."""
shape = w.shape
n = shape[-1]
first = w[..., : n // 2]
second = w[..., n // 2 :]
stacked = torch.stack((first, second), dim=-1)
return stacked.reshape(shape)
def make_dummy_moe_config(
num_experts: int = 1,
num_local_experts: int | None = None,
experts_per_token: int = 1,
hidden_dim: int = 1,
intermediate_size: int = 1,
in_dtype: torch.dtype = torch.bfloat16,
max_num_tokens: int = 512,
activation: MoEActivation = MoEActivation.SILU,
) -> FusedMoEConfig:
"""This is a dummy config for the mk constructor interface
as most kernels like DeepGEMM, CUTLASSFp4, Triton, MARLIN
do not actually use this config.
CUTLASSFp8 needs to set some params for workshapes.
"""
return FusedMoEConfig(
num_experts=num_experts,
experts_per_token=experts_per_token,
hidden_dim=hidden_dim,
intermediate_size=intermediate_size,
num_local_experts=num_local_experts
if num_local_experts is not None
else num_experts,
num_logical_experts=num_experts,
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
activation=activation,
in_dtype=in_dtype,
device=DEVICE,
routing_method=RoutingMethodType.TopK,
max_num_tokens=max_num_tokens,
)
def triton_moe(
a: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weight: torch.Tensor,
topk_ids: torch.Tensor,
w1_scale: torch.Tensor | None = None,
w2_scale: torch.Tensor | None = None,
a1_scale: torch.Tensor | None = None,
a2_scale: torch.Tensor | None = None,
quant_dtype: torch.dtype | None = None,
per_act_token_quant=False,
block_shape: list[int] | None = None,
) -> torch.Tensor:
quant_config = FusedMoEQuantConfig.make(
quant_dtype,
per_act_token_quant=per_act_token_quant,
block_shape=block_shape,
w1_scale=w1_scale,
w2_scale=w2_scale,
a1_scale=a1_scale,
a2_scale=a2_scale,
)
return fused_experts(a, w1, w2, topk_weight, topk_ids, quant_config=quant_config)
def batched_moe(
a: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weight: torch.Tensor,
topk_ids: torch.Tensor,
w1_scale: torch.Tensor | None = None,
w2_scale: torch.Tensor | None = None,
a1_scale: torch.Tensor | None = None,
a2_scale: torch.Tensor | None = None,
quant_dtype: torch.dtype | None = None,
per_act_token_quant: bool = False,
block_shape: list[int] | None = None,
moe_config: FusedMoEConfig | None = None,
) -> torch.Tensor:
max_num_tokens = round_up(a.shape[0], 64)
quant_config = FusedMoEQuantConfig.make(
quant_dtype,
per_act_token_quant=per_act_token_quant,
block_shape=block_shape,
w1_scale=w1_scale,
w2_scale=w2_scale,
a1_scale=a1_scale,
a2_scale=a2_scale,
)
if moe_config is None:
moe_config = make_dummy_moe_config()
fused_experts = FusedMoEKernel(
BatchedPrepareAndFinalize(
max_num_tokens, num_dispatchers=1, num_local_experts=w1.shape[0], rank=0
),
BatchedTritonExperts(
max_num_tokens=max_num_tokens,
num_dispatchers=1,
quant_config=quant_config,
moe_config=moe_config,
),
)
return fused_experts.apply(
a,
w1,
w2,
topk_weight,
topk_ids,
global_num_experts=w1.shape[0],
activation=moe_config.activation,
apply_router_weight_on_input=False,
expert_map=None,
)
def naive_batched_moe(
a: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weight: torch.Tensor,
topk_ids: torch.Tensor,
w1_scale: torch.Tensor | None = None,
w2_scale: torch.Tensor | None = None,
a1_scale: torch.Tensor | None = None,
a2_scale: torch.Tensor | None = None,
quant_dtype: torch.dtype | None = None,
per_act_token_quant: bool = False,
block_shape: list[int] | None = None,
) -> torch.Tensor:
max_num_tokens = round_up(a.shape[0], 64)
quant_config = FusedMoEQuantConfig.make(
quant_dtype,
per_act_token_quant=per_act_token_quant,
block_shape=block_shape,
w1_scale=w1_scale,
w2_scale=w2_scale,
a1_scale=a1_scale,
a2_scale=a2_scale,
)
moe_config = make_dummy_moe_config()
fused_experts = FusedMoEKernel(
BatchedPrepareAndFinalize(
max_num_tokens, num_dispatchers=1, num_local_experts=w1.shape[0], rank=0
),
NaiveBatchedExperts(
max_num_tokens=max_num_tokens,
num_dispatchers=1,
quant_config=quant_config,
moe_config=moe_config,
),
)
return fused_experts.apply(
a,
w1,
w2,
topk_weight,
topk_ids,
global_num_experts=w1.shape[0],
activation=moe_config.activation,
apply_router_weight_on_input=False,
expert_map=None,
)
def chunk_scales(
scales: torch.Tensor | None, start: int, end: int
) -> torch.Tensor | None:
if scales is not None:
if scales.numel() == 1:
return scales
else:
return scales[start:end]
return None
def make_quantized_test_activations(
E: int,
m: int,
k: int,
in_dtype: torch.dtype,
quant_dtype: torch.dtype | None = None,
block_shape: list[int] | None = None,
per_act_token_quant: bool = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
a = torch.randn((E, m, k), device=DEVICE, dtype=in_dtype) / 10
a_q = a
a_scale = None
if quant_dtype is not None:
assert quant_dtype == torch.float8_e4m3fn or quant_dtype == torch.int8, (
"only fp8/int8 supported"
)
a_q = torch.zeros_like(a, dtype=quant_dtype)
a_scale_l = [None] * E
for e in range(E):
a_q[e], a_scale_l[e] = moe_kernel_quantize_input(
a[e], None, quant_dtype, per_act_token_quant, block_shape
)
a_scale = torch.stack(a_scale_l)
if not per_act_token_quant and block_shape is None:
a_scale = a_scale.view(E, 1, 1)
return a, a_q, a_scale
def moe_quantize_weights_2d(
w: torch.Tensor,
w_s: torch.Tensor | None,
quant_dtype: torch.dtype | str | None,
per_token_quant: bool,
block_shape: list[int] | None,
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None]:
is_fp8_dtype = is_fp8(quant_dtype)
assert is_fp8_dtype or quant_dtype == torch.int8 or quant_dtype == "nvfp4", (
"only fp8/int8/nvfp4 supported"
)
w_gs = None
if block_shape is not None:
assert not per_token_quant
if quant_dtype == torch.int8:
w, w_s = per_block_cast_to_int8(w, block_shape)
elif is_fp8_dtype:
w, w_s = per_block_cast_to_fp8(w, block_shape)
elif quant_dtype == "nvfp4":
raise RuntimeError("blocked quantization not supported for nvfp4")
else:
raise RuntimeError(f"Unsupported quant type {quant_dtype}")
else:
if quant_dtype == torch.int8:
w, w_s = ops.scaled_int8_quant(
w, w_s, use_per_token_if_dynamic=per_token_quant
)
elif is_fp8_dtype:
w, w_s = ops.scaled_fp8_quant(
w, w_s, use_per_token_if_dynamic=per_token_quant
)
elif quant_dtype == "nvfp4":
assert not per_token_quant
w_amax = torch.abs(w).max().to(torch.float32)
w_gs = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / w_amax
if current_platform.is_rocm():
w, w_s = _scaled_fp4_quant_emulated(w, w_gs)
else:
w, w_s = ops.scaled_fp4_quant(w, w_gs)
else:
raise RuntimeError(f"Unsupported quant type {quant_dtype}")
return w, w_s, w_gs
def _pack_e2m1_fp4(fp4_values: torch.Tensor) -> torch.Tensor:
assert fp4_values.shape[-1] % 2 == 0
abs_values = fp4_values.abs()
codes = torch.empty_like(abs_values, dtype=torch.uint8)
for code, value in enumerate((0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0)):
codes[abs_values == value] = code
codes = codes | ((fp4_values < 0).to(torch.uint8) << 3)
return codes[..., 0::2] | (codes[..., 1::2] << 4)
def _scaled_fp4_quant_emulated(
w: torch.Tensor, w_gs: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
fp4_values, w_s = ref_nvfp4_quant(w, w_gs, block_size=16)
return _pack_e2m1_fp4(fp4_values), w_s.to(torch.float8_e4m3fn)
def moe_quantize_weights(
w: torch.Tensor,
w_s: torch.Tensor | None,
quant_dtype: torch.dtype | str | None,
per_token_quant: bool,
block_shape: list[int] | None,
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None]:
assert w.dim() == 3
e, rows, cols = w.shape
w_l = [None] * e
w_s_l = [None] * e
w_gs_l = [None] * e
for idx in range(e):
w_l[idx], w_s_l[idx], w_gs_l[idx] = moe_quantize_weights_2d(
w[idx], None, quant_dtype, per_token_quant, block_shape
)
w = torch.stack(w_l)
w_s = torch.stack(w_s_l)
w_gs = torch.stack(w_gs_l) if e > 0 and w_gs_l[0] is not None else None
if w_s.ndim == 2:
assert w_s.shape[-1] == 1
w_s = w_s.view(-1, 1, 1)
if block_shape is not None:
block_n, block_k = block_shape
n_tiles = (rows + block_n - 1) // block_n
k_tiles = (cols + block_k - 1) // block_k
assert w_s.shape == (e, n_tiles, k_tiles)
return w, w_s, w_gs
def make_test_weight(
e: int,
rows: int,
cols: int,
in_dtype: torch.dtype = torch.bfloat16,
quant_dtype: torch.dtype | str | None = None,
block_shape: list[int] | None = None,
per_out_ch_quant: bool = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None, torch.Tensor | None]:
w_16 = torch.randn((e, rows, cols), device=DEVICE, dtype=in_dtype) / 15
if quant_dtype is not None:
w, w_s, w_gs = moe_quantize_weights(
w_16, None, quant_dtype, per_out_ch_quant, block_shape
)
else:
w = w_16
w_s = None
w_gs = None
return w_16, w, w_s, w_gs
def make_test_weights(
e: int,
n: int,
k: int,
in_dtype: torch.dtype = torch.bfloat16,
quant_dtype: torch.dtype | str | None = None,
block_shape: list[int] | None = None,
per_out_ch_quant: bool = False,
make_gate: bool = True,
) -> tuple[
tuple[torch.Tensor, torch.Tensor, torch.Tensor | None, torch.Tensor | None],
tuple[torch.Tensor, torch.Tensor, torch.Tensor | None, torch.Tensor | None],
]:
return (
make_test_weight(
e,
(2 if make_gate else 1) * n,
k,
in_dtype,
quant_dtype,
block_shape,
per_out_ch_quant,
),
make_test_weight(e, k, n, in_dtype, quant_dtype, block_shape, per_out_ch_quant),
)
def per_token_cast_to_fp8(
x: torch.Tensor, block_size: int = 128
) -> tuple[torch.Tensor, torch.Tensor]:
assert x.dim() == 2
m, n = x.shape
pad_size = (block_size - (n % block_size)) % block_size
x = torch.nn.functional.pad(x, (0, pad_size), value=0) if pad_size > 0 else x
x_view = x.view(m, -1, block_size)
x_amax = x_view.abs().float().amax(dim=2).view(m, -1).clamp(1e-4)
fp8_data = (x_view * (448.0 / x_amax.unsqueeze(2))).to(torch.float8_e4m3fn)
return fp8_data.view(m, n + pad_size)[:, :n], (x_amax / 448.0).view(m, -1)
def make_test_quant_config(
e: int,
n: int,
k: int,
in_dtype: torch.dtype,
quant_dtype: torch.dtype | str | None = None,
per_act_token_quant: bool = False,
block_shape: list[int] | None = None,
make_gate: bool = True,
is_scale_swizzled: bool = True,
) -> tuple[torch.Tensor, torch.Tensor, FusedMoEQuantConfig]:
(_, w1, w1_s, w1_gs), (_, w2, w2_s, w2_gs) = make_test_weights(
e,
n,
k,
in_dtype,
quant_dtype,
per_out_ch_quant=per_act_token_quant,
block_shape=block_shape,
make_gate=make_gate,
)
# Hacky/trivial scales for nvfp4.
a1_gscale: torch.Tensor | None = None
a2_gscale: torch.Tensor | None = None
if quant_dtype == "nvfp4":
a1_gscale = torch.ones((e,), device=DEVICE, dtype=torch.float32)
a2_gscale = torch.ones((e,), device=DEVICE, dtype=torch.float32)
a1_scale = a1_gscale
a2_scale = a2_gscale
else:
a1_scale = None
a2_scale = None
return (
w1,
w2,
FusedMoEQuantConfig.make(
quant_dtype,
per_act_token_quant=per_act_token_quant,
block_shape=block_shape,
w1_scale=w1_s,
w2_scale=w2_s,
a1_gscale=a1_gscale,
a2_gscale=a2_gscale,
a1_scale=a1_scale,
a2_scale=a2_scale,
# TODO: make sure this is handled properly
g1_alphas=(1 / w1_gs) if w1_gs is not None else None,
g2_alphas=(1 / w2_gs) if w2_gs is not None else None,
is_scale_swizzled=is_scale_swizzled,
),
)
def fused_moe(
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
score: torch.Tensor,
topk: int,
renormalize: bool = False,
quant_config: FusedMoEQuantConfig | None = None,
global_num_experts: int = -1,
expert_map: torch.Tensor | None = None,
) -> torch.Tensor:
topk_weights, topk_ids, _ = fused_topk(
hidden_states, score.float(), topk, renormalize
)
return fused_experts(
hidden_states,
w1,
w2,
topk_weights,
topk_ids,
global_num_experts=global_num_experts,
expert_map=expert_map,
quant_config=quant_config,
)
class BaselineMM(torch.nn.Module):
def __init__(
self,
b: torch.Tensor,
out_dtype: torch.dtype,
):
super().__init__()
self.b = torch.nn.Parameter(b.to(dtype=torch.float32))
self.out_dtype = out_dtype
def forward(self, a: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor | None]:
return torch.mm(a.to(dtype=torch.float32), self.b).to(self.out_dtype), None
class BaselineSiluAndMul(torch.nn.Module):
def __init__(self):
super().__init__()
def forward(self, x: torch.Tensor) -> torch.Tensor:
d = x.shape[-1] // 2
return torch.nn.functional.silu(x[..., :d]) * x[..., d:]
class TestMLP(torch.nn.Module):
def __init__(
self,
w1: torch.Tensor,
w2: torch.Tensor,
out_dtype: torch.dtype,
):
super().__init__()
self.gate_up_proj = BaselineMM(w1, out_dtype)
self.down_proj = BaselineMM(w2, out_dtype)
self.act_fn = BaselineSiluAndMul()
def forward(self, x):
x, _ = self.gate_up_proj(x)
x = self.act_fn(x)
x, _ = self.down_proj(x)
return x
def modular_triton_fused_moe(
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
) -> FusedMoEKernel:
return FusedMoEKernel(
maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
use_monolithic=False,
),
TritonExperts(moe_config, quant_config),
)
def check_accuracy(a, b, atol, rtol, percent):
"""Allow a mismatch percentage of 1 - percent."""
if torch.any(torch.isnan(a)):
raise Exception("NaN in reference output")
if torch.any(torch.isnan(b)):
raise Exception("NaN in actual output")
if torch.any(torch.isinf(a)):
raise Exception("Inf in reference output")
if torch.any(torch.isinf(b)):
raise Exception("Inf in actual output")
assert a.shape == b.shape, f"Shape mismatch: {a.shape} vs {b.shape}"
left = torch.abs(a - b)
right = atol + rtol * torch.abs(b)
count = torch.sum(left > right)
mismatch_percent = count / a.numel()
if mismatch_percent > 1 - percent:
raise Exception(
f"Mismatch percentage is {mismatch_percent:.4f} for rtol {rtol} "
f"(threshold: {1 - percent:.4f})"
)
def check_deferred_moe_finalize(
moe_config: FusedMoEConfig,
run: Callable[[], torch.Tensor | UnfinalizedMoEOutput],
router_weights: torch.Tensor | None = None,
chunked: bool = False,
) -> None:
"""Check a kernel that defers its finalize against the finalize it skips.
``run`` calls the kernel on fixed inputs, first as built and then with
``moe_config`` asking to defer, which ``should_defer_moe_finalize`` must
report truthfully. A deferred output reduced by ``moe_unpermute``, the
TRT-LLM finalize kernel, must give the kernel's own finalized output bit for
bit, and modular experts hand the router's weights back as-is. A call the
experts split across kernel launches finalizes instead.
"""
# The deferred output views the router's buffer, so compare with a copy.
expected_weights = None if router_weights is None else router_weights.clone()
finalized = run()
assert isinstance(finalized, torch.Tensor)
moe_config.defer_moe_finalize()
output = run()
assert moe_config.should_defer_moe_finalize(finalized.shape[0]) != chunked
if chunked:
torch.testing.assert_close(output, finalized, atol=0, rtol=0)
return
assert isinstance(output, UnfinalizedMoEOutput)
if expected_weights is not None:
torch.testing.assert_close(
output.expert_weights, expected_weights, atol=0, rtol=0
)
reference = torch.empty_like(finalized)
moe_unpermute(
reference,
output.gemm2_permuted,
output.expert_weights.float(),
output.expanded_idx_to_permuted_idx,
# The kernel reads its valid-row count through this pointer either way.
expert_first_token_offset=output.gemm2_permuted.new_full(
(1,), output.gemm2_permuted.shape[0], dtype=torch.int64
),
)
torch.testing.assert_close(reference, finalized, atol=0, rtol=0)
def mxfp4_w_layouts(mx_axis: int, num_warps: int = 8):
"""Weight/scale layouts for mxfp4 MoE, as (layout, opts) pairs.
triton_kernels 3.8 returns layout instances; earlier versions return a
(layout, opts) tuple.
"""
from triton_kernels.tensor_details import layout
from vllm.utils.import_utils import get_triton_kernels_version
if get_triton_kernels_version() == "3.8":
w = layout.make_default_matmul_mxfp4_w_layout(mx_axis=mx_axis)
s = layout.make_default_matmul_mxfp4_w_scale_layout(
mx_axis=mx_axis, num_warps=num_warps
)
return w, {}, s, {}
w, w_opts = layout.make_default_matmul_mxfp4_w_layout(mx_axis=mx_axis)
s, s_opts = layout.make_default_matmul_mxfp4_w_scale_layout(
mx_axis=mx_axis, num_warps=num_warps
)
return w, w_opts, s, s_opts
+290
View File
@@ -0,0 +1,290 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_fp8_min_max,
group_broadcast,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import _ceil_to_ue8m0, is_deep_gemm_e8m0_used
from vllm.utils.math_utils import round_up
FP8_DTYPE = current_platform.fp8_dtype()
DEVICE = current_platform.device_type
def as_float32_tensor(x: float | torch.Tensor) -> torch.Tensor:
return torch.as_tensor(x, dtype=torch.float32, device=DEVICE)
def ref_dynamic_per_token_quant(
x: torch.Tensor, quant_dtype: torch.dtype, scale_ub: torch.Tensor | None = None
) -> tuple[torch.Tensor, torch.Tensor]:
assert quant_dtype in [torch.int8, FP8_DTYPE]
if scale_ub is not None:
assert quant_dtype == FP8_DTYPE
if quant_dtype == torch.int8:
qtype_traits = torch.iinfo(quant_dtype)
qtype_traits_min = qtype_traits.min
qtype_traits_max = qtype_traits.max
else:
qtype_traits_min, qtype_traits_max = get_fp8_min_max()
qtype_max = as_float32_tensor(qtype_traits_max)
s_1 = as_float32_tensor(1.0)
s_512 = as_float32_tensor(512.0)
# For fp8, in order to match the cuda kernel output, we have to do exactly
# the same operations as in the corresponding fp8 kernel to prevent
# rounding errors.
# Compute scales
x_token_max, _ = x.abs().max(dim=-1)
x_token_max = as_float32_tensor(x_token_max)
if scale_ub is not None:
x_token_max = x_token_max.clamp(max=scale_ub)
scales = (x_token_max / qtype_max)[:, None]
# Quant
if quant_dtype == torch.int8:
iscales = as_float32_tensor(s_1 / scales)
torch_out = as_float32_tensor(x) * iscales
torch_out = torch_out.round()
torch_out = torch_out.clamp(qtype_traits_min, qtype_traits_max).to(quant_dtype)
else:
assert quant_dtype == FP8_DTYPE
min_scaling_factor = s_1 / (qtype_max * s_512)
scales = scales.clamp(min=min_scaling_factor)
torch_out = as_float32_tensor(x) / scales
torch_out = torch_out.clamp(qtype_traits_min, qtype_traits_max).to(quant_dtype)
return torch_out, scales
# The int8 version is very similar. Incorporate the int8 version, like in
# ref_dynamic_per_token_quant, when we have a dynamic_per_tensor int8 quant
# kernel
def ref_dynamic_per_tensor_fp8_quant(
x: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
fp8_traits_min, fp8_traits_max = get_fp8_min_max()
fp8_max = as_float32_tensor(fp8_traits_max)
one = as_float32_tensor(1.0)
# For fp8, in order to match the cuda kernel output, we have to do exactly
# the same operations as in the corresponding fp8 kernel to prevent
# rounding errors.
x_max = as_float32_tensor(x.abs().max())
ref_scale = x_max / fp8_max
ref_iscale = one / ref_scale
ref_out = (
(as_float32_tensor(x) * ref_iscale)
.clamp(fp8_traits_min, fp8_traits_max)
.to(FP8_DTYPE)
)
return ref_out, ref_scale.view(1)
def native_w8a8_block_matmul(
A: torch.Tensor,
B: torch.Tensor,
As: torch.Tensor,
Bs: torch.Tensor,
block_size: list[int],
output_dtype: torch.dtype,
compute_type: torch.dtype = torch.float32,
) -> torch.Tensor:
"""This function performs matrix multiplication with block-wise
quantization using native torch.
It is agnostic to the input data type and can be used for both int8 and
fp8 data types.
It takes two input tensors `A` and `B` (int8) with scales `As` and
`Bs` (float32).
The output is returned in the specified `output_dtype`.
"""
A = A.to(compute_type)
B = B.to(compute_type)
assert A.shape[-1] == B.shape[-1]
assert B.ndim == 2 and B.is_contiguous() and Bs.ndim == 2
assert len(block_size) == 2
block_n, block_k = block_size[0], block_size[1]
assert (A.shape[-1] + block_k - 1) // block_k == As.shape[-1]
assert A.shape[:-1] == As.shape[:-1]
M = A.numel() // A.shape[-1]
N, K = B.shape
origin_C_shape = A.shape[:-1] + (N,)
A = A.reshape(M, A.shape[-1])
As = As.reshape(M, As.shape[-1])
n_tiles = (N + block_n - 1) // block_n
k_tiles = (K + block_k - 1) // block_k
assert n_tiles == Bs.shape[0], f"{n_tiles} == {Bs.shape[0]}"
assert k_tiles == Bs.shape[1], f"{k_tiles} == {Bs.shape[1]}"
C_shape = (M, N)
C = torch.zeros(C_shape, dtype=compute_type, device=A.device)
A_tiles = [A[:, i * block_k : min((i + 1) * block_k, K)] for i in range(k_tiles)]
B_tiles = [
[
B[
j * block_n : min((j + 1) * block_n, N),
i * block_k : min((i + 1) * block_k, K),
]
for i in range(k_tiles)
]
for j in range(n_tiles)
]
C_tiles = [C[:, j * block_n : min((j + 1) * block_n, N)] for j in range(n_tiles)]
As_tiles = [As[:, i : i + 1] for i in range(k_tiles)]
for i in range(k_tiles):
for j in range(n_tiles):
a = A_tiles[i]
b = B_tiles[j][i]
c = C_tiles[j]
s = As_tiles[i] * Bs[j][i]
c[:, :] += torch.matmul(a, b.t()) * s
C = C.reshape(origin_C_shape).to(output_dtype)
return C
def native_per_token_group_quant_fp8(
x, group_size, eps=1e-10, dtype=torch.float8_e4m3fn
):
"""Function to perform per-token-group quantization on an input tensor
`x` using native torch."""
assert x.shape[-1] % group_size == 0, (
"the last dimension of `x` must be divisible by `group_size`"
)
assert x.is_contiguous(), "`x` is not contiguous"
finfo = torch.finfo(dtype)
fp8_min = finfo.min
fp8_max = finfo.max
x_ = x.reshape(x.numel() // group_size, group_size)
amax = x_.abs().max(dim=-1, keepdim=True)[0].clamp(min=eps).to(torch.float32)
x_s = amax / fp8_max
if is_deep_gemm_e8m0_used():
x_s = _ceil_to_ue8m0(x_s)
x_q = (x_ / x_s).clamp(min=fp8_min, max=fp8_max).to(dtype)
x_q = x_q.reshape(x.shape)
x_s = x_s.reshape(x.shape[:-1] + (x.shape[-1] // group_size,))
return x_q, x_s
def native_per_token_group_quant_int8(
x, group_size, eps=1e-10, dtype=torch.int8, *, round_to_nearest=True
):
"""Function to perform per-token-group quantization on an input tensor
`x` using native torch.
It converts the tensor values into int8 values and returns the
quantized tensor along with the scaling factor used for quantization.
Per-token quantization rounds to nearest; block quantization truncates.
"""
assert x.shape[-1] % group_size == 0, (
"the last dimension of `x` must be divisible by `group_size`"
)
assert x.is_contiguous(), "`x` is not contiguous"
iinfo = torch.iinfo(dtype)
int8_min = iinfo.min
int8_max = iinfo.max
x_ = x.reshape(x.numel() // group_size, group_size)
# Use float32 for scale calculation for stability
amax = x_.abs().max(dim=-1, keepdim=True)[0].clamp(min=eps).to(torch.float32)
x_s = amax / int8_max
x_q = x_.to(torch.float32) / x_s
if round_to_nearest:
x_q = x_q.round()
x_q = x_q.clamp(min=int8_min, max=int8_max).to(dtype)
x_q = x_q.reshape(x.shape)
x_s = x_s.reshape(x.shape[:-1] + (x.shape[-1] // group_size,))
return x_q, x_s
DEFAULT_BLOCK_SHAPE = [128, 128]
def per_block_cast_to_int8(
x: torch.Tensor,
block_shape: list[int] = DEFAULT_BLOCK_SHAPE,
) -> tuple[torch.Tensor, torch.Tensor]:
block_m, block_n = block_shape
assert x.dim() == 2
m, n = x.shape
x_padded = torch.zeros(
(round_up(m, block_m), round_up(n, block_n)), dtype=x.dtype, device=x.device
)
x_padded[:m, :n] = x
x_view = x_padded.view(-1, block_m, x_padded.size(1) // block_n, block_n)
x_amax = x_view.abs().float().amax(dim=(1, 3), keepdim=True).clamp(1e-4)
x_scaled = (x_view * (256.0 / x_amax)).to(torch.int8)
x_scaled_sub = x_scaled.view_as(x_padded)[:m, :n].contiguous()
scales = (x_amax / 256.0).view(x_view.size(0), x_view.size(2))
return x_scaled_sub, scales
def dequant(
t: torch.Tensor,
scale: torch.Tensor | None,
block_shape: list[int] | None,
per_act_token_quant: bool,
out_dtype: torch.dtype | None = torch.float32,
) -> torch.Tensor:
if scale is not None:
f32 = torch.float32
if per_act_token_quant or block_shape is None:
return (t.to(f32) * scale).to(out_dtype)
else:
return (t.to(f32) * group_broadcast(scale, t.shape)).to(out_dtype)
else:
return t.to(out_dtype)
def native_batched_masked_quant_matmul(
A: torch.Tensor,
B: torch.Tensor,
C: torch.Tensor,
num_expert_tokens: torch.Tensor,
A_scale: torch.Tensor | None = None,
B_scale: torch.Tensor | None = None,
block_shape: list[int] | None = None,
per_act_token_quant: bool = False,
) -> torch.Tensor:
num_expert_tokens_cpu = num_expert_tokens.clone()
num_expert_tokens_cpu = num_expert_tokens_cpu.to(device="cpu")
num_experts = num_expert_tokens.size(0)
for e in range(num_experts):
num_tokens = num_expert_tokens_cpu[e]
if A.dtype.itemsize == 1 and block_shape is not None:
assert A_scale is not None and B_scale is not None
tmp = native_w8a8_block_matmul(
A[e], B[e], A_scale[e], B_scale[e], block_shape, C.dtype
)
C[e, :num_tokens, :] = tmp[:num_tokens, :]
elif A.dtype.itemsize == 1 and block_shape is None:
assert A_scale is not None and B_scale is not None
A_dq = dequant(A[e], A_scale[e], block_shape, per_act_token_quant)
B_dq = dequant(B[e], B_scale[e], block_shape, per_act_token_quant)
C[e, :num_tokens, :] = (A_dq[:num_tokens] @ B_dq.transpose(0, 1)).to(
C.dtype
)
else:
assert A_scale is None
assert B_scale is None
C[e, :num_tokens, :] = A[e, :num_tokens, :] @ B[e].transpose(0, 1)
return C