[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
@@ -1,855 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass, replace
from types import SimpleNamespace
from typing import Any
import torch
import vllm._custom_ops as ops
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from tests.kernels.moe.utils import make_test_weights, per_token_cast_to_fp8
from tests.kernels.quantization.nvfp4_utils import (
FLOAT4_E2M1_MAX,
FLOAT8_E4M3_MAX,
dequantize_nvfp4_to_dtype,
)
from tests.kernels.utils import torch_experts
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig
from vllm.distributed import (
get_dp_group,
get_pcp_group,
get_tensor_model_parallel_world_size,
)
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.quantization.utils.fp8_utils import is_fp8
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8Dynamic128Sym,
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8Static128BlockSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
)
from vllm.utils.import_utils import (
has_aiter,
has_deep_ep,
has_deep_ep_v2,
has_deep_gemm,
has_mori,
)
from vllm.utils.math_utils import next_power_of_2
from .mk_objects import (
TestMoEQuantConfig,
expert_info,
make_fused_experts,
prepare_finalize_info,
)
from .parallel_utils import ProcessGroupInfo
def _describe_tensor(t: torch.Tensor | None, name: str) -> str:
if t is None:
return f"{name} : None"
else:
return f"{name} : {t.shape} {t.dtype} {t.device}"
@dataclass
class Config:
Ms: list[int] | int
K: int
N: int
E: int
topks: list[int] | int
dtype: torch.dtype
quant_config: TestMoEQuantConfig | None
prepare_finalize_type: mk.FusedMoEPrepareAndFinalize
fused_experts_type: mk.FusedMoEExperts
world_size: int
activation: MoEActivation = MoEActivation.SILU
torch_trace_dir_path: str | None = None
# Force AiterExperts's hidden_pad/intermediate_pad computation
# (`experts/rocm_aiter_moe.py`) to diverge from the padded K/N sizes above.
# None (default) preserves today's behavior: FusedMoEConfig defaults both
# to the (unpadded) K/intermediate_size_per_partition, so hidden_pad and
# intermediate_pad come out to 0.
# See https://github.com/vllm-project/vllm/issues/54966 ("Test padding").
hidden_dim_unpadded: int | None = None
intermediate_size_per_partition_unpadded: int | None = None
def __post_init__(self):
if self.quant_config is None:
self.quant_config = TestMoEQuantConfig(None, False, False, None)
def describe(self) -> str:
s = ""
s += "== Config:\n"
s += f" world_size={self.world_size}\n"
s += f" PF={self.prepare_finalize_type.__name__}\n"
s += f" FE={self.fused_experts_type.__name__}\n"
s += f" E={self.E}\n"
s += f" Ms={self.Ms}\n"
s += f" N={self.N}\n"
s += f" K={self.K}\n"
s += f" topk={self.topks}\n"
s += f" dtype={self.dtype}\n"
s += " Quant:\n"
if self.quant_config is not None:
s += f" q_dtype={self.quant_dtype}\n"
s += f" q_block_shape={self.quant_block_shape}\n"
s += f" q_per_out_ch_quant={self.is_per_out_ch_quant}\n"
s += f" q_per_act_token={self.is_per_act_token_quant}\n"
else:
s += " quant=None\n"
return s
@property
def M(self) -> int:
assert isinstance(self.Ms, int)
return self.Ms
@property
def quant_dtype(self) -> torch.dtype | str | None:
assert self.quant_config is not None
return self.quant_config.quant_dtype
@property
def is_per_act_token_quant(self) -> bool:
assert self.quant_config is not None
return self.quant_config.per_act_token_quant
@property
def is_per_tensor_act_quant(self) -> bool:
return not self.is_per_act_token_quant and self.quant_block_shape is None
@property
def is_per_out_ch_quant(self) -> bool:
assert self.quant_config is not None
return self.quant_config.per_out_ch_quant
@property
def quant_block_shape(self) -> list[int] | None:
assert self.quant_config is not None
return self.quant_config.block_shape
@property
def topk(self) -> int:
assert isinstance(self.topks, int)
return self.topks
@property
def num_local_experts(self) -> int:
return self.E // self.world_size
def make_env_data(self) -> tuple[VllmConfig, dict[Any, Any]]:
"""Make env data for vllm launch."""
vllm_config = VllmConfig()
vllm_config.model_config = SimpleNamespace(
enforce_eager=True,
is_moe=True,
enable_sleep_mode=False,
sleep_mode_backend="cumem",
sleep_mode_offload_cudagraph=False,
)
vllm_config.parallel_config.data_parallel_size = self.world_size
vllm_config.parallel_config.enable_expert_parallel = True
env_dict = {
"VLLM_USE_DEEP_GEMM": str(int(self.needs_deep_gemm())),
}
vllm_config.parallel_config.all2all_backend = self.all2all_backend()
return vllm_config, env_dict
def fp8_quant_key_pair(self) -> tuple[QuantKey, QuantKey]:
"""Derive the (weight_quant_key, activation_quant_key) pair an FP8
quant config of this shape corresponds to (either OCP or FNUZ FP8,
see ``current_platform.fp8_dtype()``)."""
if self.quant_block_shape is not None:
return kFp8Static128BlockSym, kFp8Dynamic128Sym
if self.is_per_out_ch_quant:
return (
kFp8StaticChannelSym,
kFp8DynamicTokenSym
if self.is_per_act_token_quant
else kFp8StaticTensorSym,
)
return (
kFp8StaticTensorSym,
kFp8DynamicTensorSym
if self.is_per_act_token_quant
else kFp8StaticTensorSym,
)
def fe_supports_quant_scheme(self) -> bool:
"""Check if the fused experts class supports this quant config.
See https://github.com/ROCm/aiter/issues/2419 for AITER gaps."""
if self.quant_config is None or self.quant_dtype is None:
return True
if not is_fp8(self.quant_dtype):
return True
w_key, a_key = self.fp8_quant_key_pair()
fe_cls = self.fused_experts_type
if hasattr(fe_cls, "_supports_quant_scheme"):
try:
return fe_cls._supports_quant_scheme(w_key, a_key)
except NotImplementedError:
pass
return True
def is_fp8_block_quantized(self):
return is_fp8(self.quant_dtype) and self.quant_block_shape is not None
def is_batched_prepare_finalize(self):
info = prepare_finalize_info(self.prepare_finalize_type)
return mk.FusedMoEActivationFormat.BatchedExperts == info.activation_format
def is_batched_fused_experts(self):
info = expert_info(self.fused_experts_type)
return mk.FusedMoEActivationFormat.BatchedExperts == info.activation_format
def is_standard_fused_experts(self):
info = expert_info(self.fused_experts_type)
return mk.FusedMoEActivationFormat.Standard == info.activation_format
def fe_supported_types(self):
info = expert_info(self.fused_experts_type)
return info.supported_dtypes
def pf_supported_types(self):
info = prepare_finalize_info(self.prepare_finalize_type)
return info.supported_dtypes
def is_block_quant_supported(self):
info = expert_info(self.fused_experts_type)
return info.blocked_quantization_support
def supports_apply_weight_on_input(self):
info = prepare_finalize_info(self.prepare_finalize_type)
return info.supports_apply_weight_on_input
def needs_deep_gemm(self):
info = expert_info(self.fused_experts_type)
return info.needs_deep_gemm
def needs_deep_ep(self):
info = prepare_finalize_info(self.prepare_finalize_type)
return (
info.backend == "deepep_high_throughput"
or info.backend == "deepep_low_latency"
)
def needs_deep_ep_v2(self):
info = prepare_finalize_info(self.prepare_finalize_type)
return info.backend == "deepep_v2"
def needs_aiter(self):
info = expert_info(self.fused_experts_type)
return info.needs_aiter
def needs_mori(self):
info = prepare_finalize_info(self.prepare_finalize_type)
return info.backend in ("mori_high_throughput", "mori_low_latency")
def all2all_backend(self):
info = prepare_finalize_info(self.prepare_finalize_type)
return info.backend
def is_valid(self) -> tuple[bool, str | None]:
# Check prepare-finalize and fused-experts compatibility
if self.is_batched_prepare_finalize():
if not self.is_batched_fused_experts():
return False, "Mismatched format."
else:
if not self.is_standard_fused_experts():
return False, "Mismatched format."
# Check quantization sanity
if (
int(self.is_per_act_token_quant)
+ int(self.is_per_tensor_act_quant)
+ int(self.quant_block_shape is not None)
) > 1:
# invalid quant config
return False, f"Bad quant_config {self.quant_config}."
# check type support
if self.quant_dtype is None:
if (
self.dtype not in self.pf_supported_types()
or self.dtype not in self.fe_supported_types()
):
return False, (
f"Unsupported type {self.dtype} not in "
f"{self.pf_supported_types()} and "
f"{self.fe_supported_types()}."
)
else:
if (
self.quant_dtype not in self.pf_supported_types()
or self.quant_dtype not in self.fe_supported_types()
):
return False, (
f"Unsupported quant type {self.quant_dtype} "
f"not in {self.pf_supported_types()} and "
f"{self.fe_supported_types()}."
)
# Check quant scheme compatibility with fused experts class
if not self.fe_supports_quant_scheme():
return False, (
f"FE {self.fused_experts_type.__name__} does not support "
f"quant scheme (per_out_ch={self.is_per_out_ch_quant}, "
f"per_act_token={self.is_per_act_token_quant}, "
f"block={self.quant_block_shape})"
)
# Check activation support; NotImplementedError means no opinion.
try:
if not self.fused_experts_type._supports_activation(self.activation):
return False, (
f"FE {self.fused_experts_type.__name__} does not support "
f"activation {self.activation}"
)
except NotImplementedError:
pass
# Check block quantization support
is_block_quantized = self.quant_block_shape is not None
if is_block_quantized and self.quant_dtype is None:
return False, "No block quantization support."
if is_block_quantized and not self.is_block_quant_supported():
return False, "Mismatched block quantization support."
# deep_gemm only works with block-quantized
if self.needs_deep_gemm() and not is_block_quantized:
return False, "Needs DeepGEMM but not block quantized."
# Check dependencies (turn into asserts?)
if self.needs_deep_ep() and not has_deep_ep():
return False, "Needs DeepEP, but DeepEP not available."
if self.needs_deep_ep_v2() and not has_deep_ep_v2():
return False, "Needs DeepEP v2, but DeepEP v2 not available."
if self.needs_deep_gemm() and not has_deep_gemm():
return (
False,
"Needs DeepGEMM, but the current vLLM environment does not provide it.",
)
if self.needs_aiter() and not has_aiter(): # noqa: SIM103
return False, "Needs Aiter, but Aiter not available."
if self.needs_mori() and not has_mori(): # noqa: SIM103
return False, "Needs MoRI, but MoRI not available."
if self.needs_mori() and not rocm_aiter_ops.is_fused_moe_enabled():
return False, (
"Mori requires AITER's fused-moe backend to be enabled "
"(VLLM_ROCM_USE_AITER=1 and VLLM_ROCM_USE_AITER_MOE=1)."
)
try:
if not self.fused_experts_type._supports_current_device():
return (
False,
f"{self.fused_experts_type} not supported on the current device.",
)
except NotImplementedError:
pass
return True, None
@dataclass
class WeightTensors:
w1: torch.Tensor
w2: torch.Tensor
w1_scale: torch.Tensor | None
w2_scale: torch.Tensor | None
w1_gs: torch.Tensor | None = None
w2_gs: torch.Tensor | None = None
def describe(self):
s = ""
s += "== Weight Tensors: \n"
s += f" - {_describe_tensor(self.w1, 'w1')} \n"
s += f" - {_describe_tensor(self.w2, 'w2')} \n"
s += f" - {_describe_tensor(self.w1_scale, 'w1_scale')} \n"
s += f" - {_describe_tensor(self.w2_scale, 'w2_scale')} \n"
s += f" - {_describe_tensor(self.w1_gs, 'w1_gs')} \n"
s += f" - {_describe_tensor(self.w2_gs, 'w2_gs')} \n"
return s
def is_quantized(self) -> bool:
# or w1_scale is not None?
return (
is_fp8(self.w1.dtype)
or self.w1.dtype == torch.uint8
or self.w1.dtype == torch.int8
)
def to_current_device(self):
device = torch.accelerator.current_device_index()
self.w1 = self.w1.to(device=device)
self.w2 = self.w2.to(device=device)
if self.w1_scale is not None:
self.w1_scale = self.w1_scale.to(device=device)
if self.w2_scale is not None:
self.w2_scale = self.w2_scale.to(device=device)
if self.w1_gs is not None:
self.w1_gs = self.w1_gs.to(device=device)
if self.w2_gs is not None:
self.w2_gs = self.w2_gs.to(device=device)
def slice_weights(self, rank: int, num_local_experts: int) -> "WeightTensors":
s = rank * num_local_experts
e = s + num_local_experts
w1 = self.w1[s:e, :, :]
w2 = self.w2[s:e, :, :]
w1_scale = self.w1_scale[s:e, :, :] if self.w1_scale is not None else None
w2_scale = self.w2_scale[s:e, :, :] if self.w2_scale is not None else None
w1_gs = self.w1_gs[s:e] if self.w1_gs is not None else None
w2_gs = self.w2_gs[s:e] if self.w2_gs is not None else None
return WeightTensors(w1, w2, w1_scale, w2_scale, w1_gs, w2_gs)
@staticmethod
def make(config: Config) -> "WeightTensors":
(_, w1, w1_scale, w1_gs), (_, w2, w2_scale, w2_gs) = make_test_weights(
e=config.E,
n=config.N,
k=config.K,
in_dtype=config.dtype,
quant_dtype=config.quant_dtype,
block_shape=config.quant_block_shape,
# or config.is_per_out_ch_quant
per_out_ch_quant=config.is_per_act_token_quant,
)
return WeightTensors(
w1=w1, w2=w2, w1_scale=w1_scale, w2_scale=w2_scale, w1_gs=w1_gs, w2_gs=w2_gs
)
@dataclass
class RankTensors:
hidden_states: torch.Tensor
hidden_states_scale: torch.Tensor | None
topk_weights: torch.Tensor
topk_ids: torch.Tensor
expert_map: torch.Tensor | None
def describe(self):
s = ""
s += "== Rank Tensors: \n"
s += f" - {_describe_tensor(self.hidden_states, 'HS')} \n"
s += f" - {_describe_tensor(self.hidden_states_scale, 'HS_scale')} \n"
s += f" - {_describe_tensor(self.topk_weights, 'topk_weights')} \n"
s += f" - {_describe_tensor(self.topk_ids, 'topk_ids')} \n"
s += f" - {_describe_tensor(self.expert_map, 'expert_map')} \n"
return s
@staticmethod
def make_hidden_states(
config: Config,
) -> tuple[torch.Tensor, torch.Tensor | None]:
"""Return hidden_states."""
m, k, dtype = (config.M, config.K, config.dtype)
device = torch.accelerator.current_device_index()
a = torch.randn((m, k), device=device, dtype=dtype) / 15.0
if config.quant_dtype is None:
return a, None
# We dequant and use that as hidden_states so the tests are stable.
# quantizing and dequantizing yield slightly different results
# depending on the hardware. Here we, quantize and dequantize
# first - so further quantize and dequantize will yield the same
# values.
if config.is_per_tensor_act_quant:
a_q, a_scales = ops.scaled_fp8_quant(a, use_per_token_if_dynamic=False)
return a_q.float().mul(a_scales).to(dtype), a_scales
if config.is_per_act_token_quant:
a_q, a_scales = ops.scaled_fp8_quant(a, use_per_token_if_dynamic=True)
return a_q.float().mul(a_scales).to(dtype), None
assert config.quant_block_shape is not None
block_k = config.quant_block_shape[1]
a_q, a_scales = per_token_cast_to_fp8(a, block_size=block_k)
return a_q.float().view((-1, block_k)).mul(a_scales.view(-1, 1)).view(m, k).to(
dtype
), None
@staticmethod
def make(config: Config, pgi: ProcessGroupInfo):
dtype = config.dtype
topk, m, _ = (config.topk, config.M, config.K)
hidden_states, hidden_states_scale = RankTensors.make_hidden_states(config)
num_local_experts, global_num_experts = (config.num_local_experts, config.E)
score = torch.randn((m, global_num_experts), device="cuda", dtype=dtype)
topk_weights, topk_ids, _ = fused_topk(hidden_states, score, topk, False)
# distribute topk_ids evenly
device = torch.accelerator.current_device_index()
for mi in range(m):
topk_ids[mi] = torch.randperm(config.E)[:topk]
topk_ids = topk_ids.to(device=device)
expert_map = None
if config.world_size > 1:
expert_map = torch.full(
(global_num_experts,), fill_value=-1, dtype=torch.int32
)
s = pgi.rank * num_local_experts
e = s + num_local_experts
expert_map[s:e] = torch.tensor(list(range(num_local_experts)))
expert_map = expert_map.to(device=device, dtype=torch.int32)
return RankTensors(
hidden_states=hidden_states,
hidden_states_scale=hidden_states_scale,
topk_weights=topk_weights,
topk_ids=topk_ids,
expert_map=expert_map,
)
def reference_moe_impl(
config: Config, weights: WeightTensors, rank_tensors: RankTensors
) -> torch.Tensor:
if config.quant_dtype == "nvfp4":
quant_blocksize = 16
dtype = config.dtype
w1_q = weights.w1
w1_blockscale = weights.w1_scale
w1_gs = weights.w1_gs
w2_q = weights.w2
w2_blockscale = weights.w2_scale
w2_gs = weights.w2_gs
a_global_scale = (
(FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX)
/ torch.amax(rank_tensors.hidden_states.flatten(), dim=-1)
).to(torch.float32)
assert w1_gs is not None
assert w2_gs is not None
assert w1_blockscale is not None
assert w2_blockscale is not None
assert w1_blockscale.shape[1] % 128 == 0
assert w1_blockscale.shape[2] % 4 == 0
assert w2_blockscale.shape[1] % 128 == 0
assert w2_blockscale.shape[2] % 4 == 0
a_fp4, a_scale_interleaved = ops.scaled_fp4_quant(
rank_tensors.hidden_states, a_global_scale
)
a = dequantize_nvfp4_to_dtype(
a_fp4,
a_scale_interleaved,
a_global_scale,
dtype=dtype,
device=a_fp4.device,
block_size=quant_blocksize,
)
e = w1_q.shape[0]
n = w1_q.shape[1] // 2
k = w2_q.shape[1]
w1 = torch.zeros((e, 2 * n, k), device="cuda", dtype=dtype)
w2 = torch.zeros((e, k, n), device="cuda", dtype=dtype)
for idx in range(0, e):
w1[idx] = dequantize_nvfp4_to_dtype(
w1_q[idx],
w1_blockscale[idx],
w1_gs[idx],
dtype=dtype,
device=w1_q.device,
block_size=quant_blocksize,
)
w2[idx] = dequantize_nvfp4_to_dtype(
w2_q[idx],
w2_blockscale[idx],
w2_gs[idx],
dtype=dtype,
device=w2_q.device,
block_size=quant_blocksize,
)
a_scale = None
w1_scale = None
w2_scale = None
quant_dtype = None
per_act_token_quant = False
block_shape = None
else:
a = rank_tensors.hidden_states
a_scale = rank_tensors.hidden_states_scale
w1 = weights.w1
w1_scale = weights.w1_scale
w2 = weights.w2
w2_scale = weights.w2_scale
quant_dtype = config.quant_dtype
per_act_token_quant = config.is_per_act_token_quant
block_shape = config.quant_block_shape
return torch_experts(
a=a,
w1=w1,
w2=w2,
topk_weight=rank_tensors.topk_weights,
topk_ids=rank_tensors.topk_ids,
global_num_experts=config.E,
expert_map=None,
activation=config.activation,
w1_scale=w1_scale,
w2_scale=w2_scale,
a1_scale=a_scale,
quant_dtype=quant_dtype,
per_act_token_quant=per_act_token_quant,
block_shape=block_shape,
apply_router_weights_on_input=config.topk == 1
and config.supports_apply_weight_on_input(),
)
def _make_gscale(num_experts: int) -> torch.Tensor:
return torch.ones(
(num_experts,),
device=torch.accelerator.current_device_index(),
dtype=torch.float32,
)
def make_modular_kernel(
config: Config,
vllm_config: VllmConfig,
quant_config: FusedMoEQuantConfig,
) -> mk.FusedMoEKernel:
# make moe config
moe_parallel_config: FusedMoEParallelConfig = FusedMoEParallelConfig.make(
tp_size_=get_tensor_model_parallel_world_size(),
pcp_size_=get_pcp_group().world_size,
dp_size_=get_dp_group().world_size,
sp_size_=1,
vllm_parallel_config=vllm_config.parallel_config,
)
moe = FusedMoEConfig(
num_experts=config.E,
experts_per_token=config.topk,
hidden_dim=config.K,
intermediate_size=config.N,
num_local_experts=config.num_local_experts,
num_logical_experts=config.E,
moe_parallel_config=moe_parallel_config,
in_dtype=config.dtype,
max_num_tokens=next_power_of_2(config.M),
activation=config.activation,
device=vllm_config.device_config.device,
routing_method=RoutingMethodType.DeepSeekV3,
hidden_dim_unpadded=config.hidden_dim_unpadded,
intermediate_size_per_partition_unpadded=(
config.intermediate_size_per_partition_unpadded
),
)
prepare_finalize = maybe_make_prepare_finalize(
moe=moe,
quant_config=quant_config,
allow_new_interface=True,
)
assert prepare_finalize is not None
fused_experts = make_fused_experts(
config.fused_experts_type,
moe,
quant_config,
prepare_finalize.num_dispatchers(),
config.N,
)
modular_kernel = mk.FusedMoEKernel(
prepare_finalize=prepare_finalize,
fused_experts=fused_experts,
)
return modular_kernel
def _shuffle_weights_for_aiter(rank_weights: WeightTensors) -> WeightTensors:
"""Pre-shuffle weights so AITER selects its prebuilt `preshuffle_on` module.
Production shuffles in `process_weights_after_loading`; this harness builds its
tensors directly, so without this every call asks for a `preshuffle_off` module
that is not prebuilt, and JIT-compiles a kernel the image already ships.
"""
from vllm._aiter_ops import rocm_aiter_ops
w1, w2 = rocm_aiter_ops.shuffle_weights(rank_weights.w1, rank_weights.w2)
w1.is_shuffled = True
w2.is_shuffled = True
return replace(rank_weights, w1=w1, w2=w2)
def _maybe_convert_weights_for_experts(
config: Config,
rank_weights: WeightTensors,
) -> WeightTensors:
"""Convert weights to expert-specific format (e.g., TrtLLM BlockMajorK)."""
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
AiterExperts,
)
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
Fp8MoeBackend,
convert_to_fp8_moe_kernel_format,
)
fe_type = config.fused_experts_type
fe_name = getattr(fe_type, "__name__", "")
# AITER's prebuilt modules carry a gfx950 instance table even on gfx942 and reject
# intermediate sizes that are not a multiple of 256, but only for the fp8 per-tensor
# and per-token schemes. Those keep JIT-compiling until
# https://github.com/ROCm/aiter/issues/5766 is fixed.
if issubclass(fe_type, AiterExperts) and (
config.quant_dtype is None or config.quant_block_shape is not None
):
return _shuffle_weights_for_aiter(rank_weights)
backend: Fp8MoeBackend | None = None
if fe_name == "TrtLlmFp8ExpertsModular":
backend = Fp8MoeBackend.FLASHINFER_TRTLLM
elif fe_name == "FlashInferExperts":
backend = Fp8MoeBackend.FLASHINFER_CUTLASS
if backend is None or not rank_weights.is_quantized():
return rank_weights
mock_layer = SimpleNamespace(
weight_block_size=config.quant_block_shape,
moe_config=SimpleNamespace(
is_act_and_mul=True,
intermediate_size_per_partition=config.N,
),
activation=SimpleNamespace(is_gated=True),
)
w1, w2, w1_scale, w2_scale = convert_to_fp8_moe_kernel_format(
fp8_backend=backend,
layer=mock_layer,
w13=rank_weights.w1,
w2=rank_weights.w2,
w13_scale=rank_weights.w1_scale,
w2_scale=rank_weights.w2_scale,
w13_input_scale=None,
w2_input_scale=None,
)
return WeightTensors(
w1=w1,
w2=w2,
w1_scale=w1_scale,
w2_scale=w2_scale,
w1_gs=rank_weights.w1_gs,
w2_gs=rank_weights.w2_gs,
)
def run_modular_kernel(
pgi: ProcessGroupInfo,
vllm_config: VllmConfig,
config: Config,
weights: WeightTensors,
rank_tensors: RankTensors,
) -> torch.Tensor:
assert isinstance(config.Ms, int)
assert isinstance(config.topks, int)
# weights for rank
rank_weights = weights.slice_weights(pgi.rank, config.num_local_experts)
rank_weights = _maybe_convert_weights_for_experts(config, rank_weights)
if config.quant_dtype == "nvfp4":
gscale = _make_gscale(config.num_local_experts)
else:
gscale = None
quant_config = FusedMoEQuantConfig.make(
config.quant_dtype,
w1_scale=rank_weights.w1_scale,
w2_scale=rank_weights.w2_scale,
a1_scale=rank_tensors.hidden_states_scale,
g1_alphas=(1 / rank_weights.w1_gs) if rank_weights.w1_gs is not None else None,
g2_alphas=(1 / rank_weights.w2_gs) if rank_weights.w2_gs is not None else None,
a1_gscale=gscale,
a2_gscale=gscale,
block_shape=config.quant_block_shape,
per_act_token_quant=config.is_per_act_token_quant,
per_out_ch_quant=config.is_per_out_ch_quant,
)
mk = make_modular_kernel(config, vllm_config, quant_config)
# impls might update the tensor in place
hidden_states = rank_tensors.hidden_states.clone()
topk_ids = rank_tensors.topk_ids.to(mk.prepare_finalize.topk_indices_dtype())
mk_kwargs = {
"hidden_states": hidden_states,
"w1": rank_weights.w1,
"w2": rank_weights.w2,
"topk_weights": rank_tensors.topk_weights,
"topk_ids": topk_ids,
"activation": config.activation,
"expert_map": rank_tensors.expert_map,
"global_num_experts": config.E,
"apply_router_weight_on_input": config.topk == 1
and config.supports_apply_weight_on_input(),
}
num_tokens = rank_tensors.hidden_states.shape[0]
num_tokens_across_dp = torch.tensor(
[num_tokens] * config.world_size, device="cuda", dtype=torch.int
)
torch.distributed.barrier()
with set_forward_context(
None,
vllm_config,
num_tokens=num_tokens,
num_tokens_across_dp=num_tokens_across_dp,
):
out = mk.apply(**mk_kwargs)
return out
@@ -1,499 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
import torch
# Fused experts and PrepareFinalize imports
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe import TritonExperts
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.experts.batched_deep_gemm_moe import (
BatchedDeepGemmExperts,
)
from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import DeepGemmExperts
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
BatchedTritonExperts,
NaiveBatchedExperts,
)
from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import (
TritonOrDeepGemmExperts,
)
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
MoEPrepareAndFinalizeNoDPEPModular,
)
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
cutlass_fp4_supported,
)
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
cutlass_fp8_supported,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import is_deep_gemm_supported
from vllm.utils.flashinfer import (
has_flashinfer_cutlass_fused_moe,
has_flashinfer_nvlink_one_sided,
has_flashinfer_trtllm_fused_moe,
)
from vllm.utils.import_utils import (
has_aiter,
has_deep_ep,
has_deep_ep_v2,
has_deep_gemm,
has_mori,
)
@dataclass
class TestMoEQuantConfig:
quant_dtype: torch.dtype | str | None
per_out_ch_quant: bool
per_act_token_quant: bool
block_shape: list[int] | None
@dataclass
class PrepareFinalizeInfo:
activation_format: mk.FusedMoEActivationFormat
supported_dtypes: list[torch.dtype | str]
blocked_quantization_support: bool
backend: str | None
supports_apply_weight_on_input: bool = True
@dataclass
class ExpertInfo:
activation_format: mk.FusedMoEActivationFormat
supported_dtypes: list[torch.dtype | str]
blocked_quantization_support: bool
needs_matching_quant: bool = False
needs_deep_gemm: bool = False
needs_aiter: bool = False
PREPARE_FINALIZE_INFO: dict[
mk.FusedMoEPrepareAndFinalizeModular, PrepareFinalizeInfo
] = {}
EXPERT_INFO: dict[mk.FusedMoEExpertsModular, ExpertInfo] = {}
MK_ALL_PREPARE_FINALIZE_TYPES: list[mk.FusedMoEPrepareAndFinalizeModular] = []
MK_MULTI_GPU_PREPARE_FINALIZE_TYPES: list[mk.FusedMoEPrepareAndFinalizeModular] = []
MK_SINGLE_GPU_PREPARE_FINALIZE_TYPES: list[mk.FusedMoEPrepareAndFinalizeModular] = []
MK_FUSED_EXPERT_TYPES: list[mk.FusedMoEExpertsModular] = []
standard_format = mk.FusedMoEActivationFormat.Standard
batched_format = mk.FusedMoEActivationFormat.BatchedExperts
common_float_types: list[torch.dtype | str] = [
current_platform.fp8_dtype(),
torch.bfloat16,
torch.float16,
torch.float32,
]
common_float_and_int_types = common_float_types + [torch.int8]
nvfp4_types = ["nvfp4"]
mxfp4_types = ["mxfp4"]
fp8_types = [current_platform.fp8_dtype()]
def register_prepare_and_finalize(
kind,
activation_format: mk.FusedMoEActivationFormat,
supported_dtypes: list[torch.dtype | str],
blocked_quantization_support: bool,
backend: str | None,
force_multigpu: bool = False,
supports_apply_weight_on_input: bool = True,
):
global PREPARE_FINALIZE_INFO
global MK_ALL_PREPARE_FINALIZE_TYPES
global MK_MULTI_GPU_PREPARE_FINALIZE_TYPES
global MK_SINGLE_GPU_PREPARE_FINALIZE_TYPES
assert kind not in PREPARE_FINALIZE_INFO
PREPARE_FINALIZE_INFO[kind] = PrepareFinalizeInfo(
activation_format,
supported_dtypes,
blocked_quantization_support,
backend,
supports_apply_weight_on_input,
)
MK_ALL_PREPARE_FINALIZE_TYPES.append(kind)
if backend is not None or force_multigpu:
MK_MULTI_GPU_PREPARE_FINALIZE_TYPES.append(kind)
else:
MK_SINGLE_GPU_PREPARE_FINALIZE_TYPES.append(kind)
def register_experts(
kind,
activation_format: mk.FusedMoEActivationFormat,
supported_dtypes: list[torch.dtype | str],
blocked_quantization_support: bool,
needs_matching_quant: bool = False,
needs_deep_gemm: bool = False,
needs_aiter: bool = False,
):
global EXPERT_INFO
global MK_FUSED_EXPERT_TYPES
assert kind not in EXPERT_INFO
EXPERT_INFO[kind] = ExpertInfo(
activation_format,
supported_dtypes,
blocked_quantization_support,
needs_matching_quant,
needs_deep_gemm,
needs_aiter,
)
MK_FUSED_EXPERT_TYPES.append(kind)
def prepare_finalize_info(kind) -> PrepareFinalizeInfo:
info = PREPARE_FINALIZE_INFO.get(kind)
assert info is not None
return info
def expert_info(kind) -> ExpertInfo:
info = EXPERT_INFO.get(kind)
assert info is not None
return info
register_prepare_and_finalize(
MoEPrepareAndFinalizeNoDPEPModular,
standard_format,
common_float_types,
blocked_quantization_support=True,
backend=None,
)
register_experts(
BatchedTritonExperts,
batched_format,
common_float_types,
blocked_quantization_support=True,
needs_matching_quant=True,
)
register_experts(
TritonExperts,
standard_format,
common_float_and_int_types,
blocked_quantization_support=True,
needs_matching_quant=True,
)
register_experts(
NaiveBatchedExperts,
batched_format,
common_float_and_int_types,
blocked_quantization_support=True,
)
# Disable on blackwell for now
if has_deep_ep() and not current_platform.has_device_capability(100):
from vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_ht import (
DeepEPHTPrepareAndFinalize,
)
from vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_ll import (
DeepEPLLPrepareAndFinalize,
)
register_prepare_and_finalize(
DeepEPHTPrepareAndFinalize,
standard_format,
common_float_types,
blocked_quantization_support=True,
backend="deepep_high_throughput",
)
register_prepare_and_finalize(
DeepEPLLPrepareAndFinalize,
batched_format,
common_float_types,
blocked_quantization_support=True,
backend="deepep_low_latency",
)
if has_deep_ep_v2() and current_platform.has_device_capability(100):
from vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_v2 import (
DeepEPV2PrepareAndFinalize,
)
register_prepare_and_finalize(
DeepEPV2PrepareAndFinalize,
standard_format,
common_float_types,
blocked_quantization_support=True,
backend="deepep_v2",
)
if has_mori():
from vllm.model_executor.layers.fused_moe.prepare_finalize.mori import (
MoriPrepareAndFinalize,
)
register_prepare_and_finalize(
MoriPrepareAndFinalize,
standard_format,
fp8_types,
blocked_quantization_support=True,
backend="mori_high_throughput",
supports_apply_weight_on_input=False,
)
if has_flashinfer_cutlass_fused_moe() and current_platform.has_device_capability(100):
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import (
FlashInferExperts,
)
from vllm.model_executor.layers.fused_moe.prepare_finalize.flashinfer_nvlink_two_sided import ( # noqa: E501
FlashInferNVLinkTwoSidedPrepareAndFinalize,
)
register_prepare_and_finalize(
FlashInferNVLinkTwoSidedPrepareAndFinalize,
standard_format,
nvfp4_types + fp8_types,
blocked_quantization_support=True,
backend=None,
force_multigpu=True,
supports_apply_weight_on_input=False,
)
register_experts(
FlashInferExperts,
standard_format,
nvfp4_types + fp8_types,
blocked_quantization_support=True,
# Note: this is a hack to get it to run for now
)
else:
FlashInferCutlassMoEPrepareAndFinalize = None
FlashInferExperts = None
if (
has_flashinfer_nvlink_one_sided()
and has_flashinfer_cutlass_fused_moe()
and current_platform.has_device_capability(100)
):
from vllm.model_executor.layers.fused_moe.prepare_finalize.flashinfer_nvlink_one_sided import ( # noqa: E501
FlashInferNVLinkOneSidedPrepareAndFinalize,
)
register_prepare_and_finalize(
FlashInferNVLinkOneSidedPrepareAndFinalize,
standard_format,
nvfp4_types,
blocked_quantization_support=False,
backend="flashinfer_nvlink_one_sided",
supports_apply_weight_on_input=False,
)
if has_flashinfer_cutlass_fused_moe() and current_platform.has_device_capability(100):
from vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe import (
TrtLlmNvFp4ExpertsModular,
)
register_experts(
TrtLlmNvFp4ExpertsModular,
standard_format,
nvfp4_types,
blocked_quantization_support=False,
)
if has_flashinfer_trtllm_fused_moe() and current_platform.has_device_capability(100):
from vllm.model_executor.layers.fused_moe.experts.trtllm_fp8_moe import (
TrtLlmFp8ExpertsModular,
)
register_experts(
TrtLlmFp8ExpertsModular,
standard_format,
fp8_types,
blocked_quantization_support=True,
)
if has_aiter():
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
AiterFp8BlockExperts,
AiterFp8ChannelExperts,
AiterFp8TensorExperts,
AiterMxfp4Experts,
AiterUnquantizedExperts,
)
# Registration dtypes gate which quant configs the modular-kernel test
# harness will ever pair with a given class (see Config.is_valid()):
# they must match what each subclass's _supports_quant_scheme() actually
# declares, or the harness will generate a case the class rejects.
for aiter_experts_cls, aiter_supported_dtypes in (
(AiterUnquantizedExperts, common_float_types),
(AiterFp8BlockExperts, fp8_types),
(AiterFp8TensorExperts, fp8_types),
(AiterFp8ChannelExperts, fp8_types),
(AiterMxfp4Experts, mxfp4_types),
):
register_experts(
aiter_experts_cls,
standard_format,
aiter_supported_dtypes,
blocked_quantization_support=True,
needs_aiter=True,
)
else:
AiterUnquantizedExperts = None
AiterFp8BlockExperts = None
AiterFp8TensorExperts = None
AiterFp8ChannelExperts = None
AiterMxfp4Experts = None
if has_deep_gemm() and is_deep_gemm_supported():
register_experts(
BatchedDeepGemmExperts,
batched_format,
fp8_types,
blocked_quantization_support=True,
needs_matching_quant=False,
needs_deep_gemm=True,
)
register_experts(
DeepGemmExperts,
standard_format,
fp8_types,
blocked_quantization_support=True,
needs_matching_quant=False,
needs_deep_gemm=True,
)
register_experts(
TritonOrDeepGemmExperts,
standard_format,
common_float_and_int_types,
blocked_quantization_support=True,
needs_matching_quant=True,
needs_deep_gemm=True,
)
if cutlass_fp8_supported():
from vllm.model_executor.layers.fused_moe import (
CutlassBatchedExpertsFp8,
CutlassExpertsFp8,
)
register_experts(
CutlassExpertsFp8,
standard_format,
fp8_types,
blocked_quantization_support=False,
)
register_experts(
CutlassBatchedExpertsFp8,
batched_format,
fp8_types,
blocked_quantization_support=False,
)
else:
CutlassBatchedExpertsFp8 = None
CutlassExpertsFp8 = None
if cutlass_fp4_supported():
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
CutlassExpertsFp4,
)
register_experts(
CutlassExpertsFp4,
standard_format,
nvfp4_types,
blocked_quantization_support=True,
)
else:
CutlassExpertsFp4 = None
MK_QUANT_CONFIGS: list[TestMoEQuantConfig | None] = [
None,
# per-channel / per-column weights and per-tensor activations
TestMoEQuantConfig(
quant_dtype=current_platform.fp8_dtype(),
per_out_ch_quant=True,
per_act_token_quant=False,
block_shape=None,
),
# per-channel / per-column weights and per-token activations
TestMoEQuantConfig(
quant_dtype=current_platform.fp8_dtype(),
per_out_ch_quant=True,
per_act_token_quant=True,
block_shape=None,
),
# per-tensor weights and per-tensor activations
TestMoEQuantConfig(
quant_dtype=current_platform.fp8_dtype(),
per_out_ch_quant=False,
per_act_token_quant=False,
block_shape=None,
),
# per-tensor weights and per-token activations
TestMoEQuantConfig(
quant_dtype=current_platform.fp8_dtype(),
per_out_ch_quant=False,
per_act_token_quant=True,
block_shape=None,
),
# block-quantized weights and 128 block per-token activations
TestMoEQuantConfig(
quant_dtype=current_platform.fp8_dtype(),
per_out_ch_quant=False,
per_act_token_quant=False,
block_shape=[128, 128],
),
# TODO (varun) : Should we test the following combinations ?
# block-quantized weights and per-token activations
# block-quantized weights and per-tensor activations
]
if cutlass_fp4_supported() or has_flashinfer_cutlass_fused_moe():
MK_QUANT_CONFIGS += [
TestMoEQuantConfig(
quant_dtype="nvfp4",
per_out_ch_quant=False,
per_act_token_quant=False,
block_shape=None,
),
]
def make_fused_experts(
fused_experts_type: mk.FusedMoEExpertsModular,
moe: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
num_dispatchers: int,
N: int,
) -> mk.FusedMoEExpertsModular:
if (
fused_experts_type.activation_format()
== mk.FusedMoEActivationFormat.BatchedExperts
):
kwargs = {
"moe_config": moe,
"quant_config": quant_config,
"max_num_tokens": moe.max_num_tokens,
"num_dispatchers": num_dispatchers,
}
else:
kwargs = {
"moe_config": moe,
"quant_config": quant_config,
}
torch.set_printoptions(threshold=0, edgeitems=0, linewidth=10000)
print(f"Making {fused_experts_type.__class__.__name__} {kwargs} ...")
experts = fused_experts_type(**kwargs)
torch.set_printoptions(threshold=1000, edgeitems=5, linewidth=80)
return experts
@@ -1,183 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import dataclasses
import os
import time
import traceback
from collections.abc import Callable
from typing import Any, Concatenate
import torch
from torch.multiprocessing import spawn # pyright: ignore[reportPrivateImportUsage]
from typing_extensions import ParamSpec
import vllm.envs as envs
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.distributed import (
cleanup_dist_env_and_memory,
init_distributed_environment,
initialize_model_parallel,
)
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
## Parallel Processes Utils
P = ParamSpec("P")
# On ROCm the first AITER CK MoE launch JIT-compiles two stage modules that the
# AITER wheel does not prebuild, ~10 min each (ROCm/aiter#5766).
LAUNCH_TIMEOUT_S = 1800 if current_platform.is_rocm() else 900
@dataclasses.dataclass
class ProcessGroupInfo:
world_size: int
world_local_size: int
rank: int
node_rank: int
local_rank: int
device: torch.device
def _set_vllm_config(
vllm_config: VllmConfig, world_size: int, rank: int, local_rank: int
):
import tempfile
temp_file = tempfile.mkstemp()[1]
# When DP is enabled, processes are organized as:
# rank = dp_rank * tp_pp_world_size + tp_pp_rank
tp_pp_world_size = vllm_config.parallel_config.world_size
vllm_config.parallel_config.data_parallel_rank = rank // tp_pp_world_size
tp_pp_rank = rank % tp_pp_world_size
vllm_config.parallel_config.rank = tp_pp_rank
with set_current_vllm_config(vllm_config):
init_distributed_environment(
world_size=tp_pp_world_size,
rank=tp_pp_rank,
distributed_init_method=f"file://{temp_file}",
local_rank=local_rank,
backend="nccl",
)
initialize_model_parallel(
tensor_model_parallel_size=vllm_config.parallel_config.tensor_parallel_size,
pipeline_model_parallel_size=vllm_config.parallel_config.pipeline_parallel_size,
)
if envs.VLLM_DISTRIBUTED_USE_SPLIT_GROUP:
cpu_group = torch.distributed.split_group(
split_ranks=[list(range(world_size))],
group_desc="moe_test_cpu",
)
else:
cpu_group = torch.distributed.new_group(
list(range(world_size)), backend="gloo"
)
return cpu_group
def _worker_parallel_launch(
local_rank: int,
world_size: int,
world_local_size: int,
node_rank: int,
init_method: str,
worker: Callable[..., None],
vllm_config: VllmConfig | None,
env_dict: dict | None,
worker_kwargs: dict[str, Any],
*args: Any,
) -> None:
rank = node_rank * world_local_size + local_rank
device = torch.device("cuda", local_rank)
torch.accelerator.set_device_index(device)
torch.set_default_device(device)
torch.distributed.init_process_group(
backend="cpu:gloo,cuda:nccl",
init_method=init_method,
rank=rank,
world_size=world_size,
device_id=device,
)
barrier = torch.tensor([rank], device=device)
torch.distributed.all_reduce(barrier)
if env_dict is not None:
os.environ.update(env_dict)
cpu_group = None
if vllm_config is not None:
cpu_group = _set_vllm_config(vllm_config, world_size, rank, local_rank)
def _run_worker():
worker(
ProcessGroupInfo(
world_size=world_size,
world_local_size=world_local_size,
rank=rank,
node_rank=node_rank,
local_rank=local_rank,
device=device,
),
vllm_config,
cpu_group,
*args,
**worker_kwargs,
)
try:
if vllm_config is not None:
with set_current_vllm_config(vllm_config):
_run_worker()
else:
_run_worker()
except Exception as ex:
print(ex)
traceback.print_exc()
raise
finally:
torch.accelerator.synchronize()
if vllm_config is not None:
cleanup_dist_env_and_memory()
else:
torch.distributed.destroy_process_group()
def parallel_launch_with_config(
world_size: int,
worker: Callable[Concatenate[ProcessGroupInfo, VllmConfig, Any, P], None],
vllm_config: VllmConfig,
env_dict: dict[Any, Any] | None,
*args: P.args,
**kwargs: P.kwargs,
) -> None:
context = spawn(
_worker_parallel_launch,
args=(
world_size,
world_size,
0,
f"tcp://{os.getenv('LOCALHOST', 'localhost')}:{get_open_port()}",
worker,
vllm_config,
env_dict,
kwargs,
)
+ args,
nprocs=world_size,
join=False,
)
# A worker stuck in communicator setup never exits, so bound the wait.
deadline = time.monotonic() + LAUNCH_TIMEOUT_S
while not context.join(timeout=max(0.0, deadline - time.monotonic())):
if time.monotonic() >= deadline:
for process in context.processes:
process.kill()
for process in context.processes:
process.join()
raise TimeoutError(
f"Parallel workers did not finish within {LAUNCH_TIMEOUT_S}s"
)
File diff suppressed because it is too large Load Diff
-690
View File
@@ -1,690 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
import pytest
import torch
from tests.kernels.moe.utils import (
batched_moe,
make_dummy_moe_config,
make_quantized_test_activations,
make_test_weights,
naive_batched_moe,
)
from tests.kernels.quant_utils import native_batched_masked_quant_matmul
from tests.kernels.utils import torch_experts
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe.activation import (
ApplyMoEActivationConfig,
MoEActivation,
apply_moe_activation,
)
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
BatchedTritonExperts,
invoke_moe_batched_triton_kernel,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl
from vllm.utils.torch_utils import set_random_seed
DEVICE = current_platform.device_type
pytestmark = pytest.mark.skipif(
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
reason="Triton MoE kernels require CUDA/ROCm/XPU.",
)
MNK_FACTORS = [
(1, 128, 128),
(1, 512, 512),
(1, 1024, 2048),
(32, 128, 128),
(32, 512, 512),
(32, 1024, 2048),
(45, 128, 2048),
(45, 1024, 128),
(64, 512, 512),
(64, 1024, 2048),
(222, 128, 2048),
(222, 1024, 2048),
]
NUM_EXPERTS = [8, 64]
TOP_KS = [1, 2, 6]
DTYPES = [torch.bfloat16]
if not current_platform.is_fp8_fnuz():
DTYPES.append(torch.float8_e4m3fn)
vllm_config = VllmConfig()
@dataclass
class BatchedMMConfig:
in_dtype: torch.dtype
quant_dtype: torch.dtype | None
out_dtype: torch.dtype
num_experts: int
max_tokens_per_expert: int
K: int
N: int
@dataclass
class BatchedMMTensors:
A: torch.Tensor # [E, max_tokens, K]
B: torch.Tensor # [E, K, N] - column major
C: torch.Tensor # [E, max_tokens, N]
num_expert_tokens: torch.Tensor # [E]
@staticmethod
def make_tensors(config: BatchedMMConfig):
A = (
torch.randn(
(config.num_experts, config.max_tokens_per_expert, config.K),
device=DEVICE,
dtype=config.in_dtype,
)
/ 10
)
B = torch.randn(
(config.num_experts, config.N, config.K),
device=DEVICE,
dtype=config.in_dtype,
)
C = torch.zeros(
(config.num_experts, config.max_tokens_per_expert, config.N),
device=DEVICE,
dtype=config.out_dtype,
)
num_expert_tokens = torch.randint(
low=0,
high=config.max_tokens_per_expert,
size=(config.num_experts,),
device=DEVICE,
dtype=torch.int32,
)
return BatchedMMTensors(A, B, C, num_expert_tokens)
@pytest.mark.parametrize("num_experts", [8, 32])
@pytest.mark.parametrize("max_tokens_per_expert", [32, 224, 512])
@pytest.mark.parametrize("K", [128, 1024])
@pytest.mark.parametrize("N", [128, 1024])
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("block_shape", [None, [128, 128]])
@pytest.mark.parametrize("per_act_token_quant", [False, True])
# Tuned configs can pick a BLOCK_SIZE_K wider than the quantization block.
@pytest.mark.parametrize("fp8_block_k", [32, 256])
def test_batched_mm(
num_experts: int,
max_tokens_per_expert: int,
K: int,
N: int,
dtype: torch.dtype,
block_shape: list[int] | None,
per_act_token_quant: bool,
fp8_block_k: int,
):
"""Note: float8_e4m3fn is not supported on CUDA architecture < 89,
and those tests will be skipped on unsupported hardware."""
set_random_seed(7)
use_fp8_w8a8 = dtype == torch.float8_e4m3fn
if (
dtype == torch.float8_e4m3fn
and current_platform.is_cuda_alike()
and not current_platform.has_device_capability(89)
):
pytest.skip(
"Triton limitation: fp8e4nv data type is not supported on CUDA arch < 89"
)
if (per_act_token_quant or block_shape is not None) and not use_fp8_w8a8:
pytest.skip("Don't test blocking for non-quantized types.")
if per_act_token_quant and block_shape is not None:
pytest.skip("Skip illegal quantization test.")
if fp8_block_k != 32 and not use_fp8_w8a8:
pytest.skip("BLOCK_SIZE_K is only varied for fp8.")
if dtype.itemsize == 1:
act_dtype = torch.bfloat16
quant_dtype = dtype
else:
act_dtype = dtype
quant_dtype = None
num_expert_tokens = torch.randint(
low=0,
high=max_tokens_per_expert,
size=(num_experts,),
device=DEVICE,
dtype=torch.int32,
)
A, A_q, A_scale = make_quantized_test_activations(
num_experts,
max_tokens_per_expert,
K,
in_dtype=act_dtype,
quant_dtype=quant_dtype,
block_shape=block_shape,
per_act_token_quant=per_act_token_quant,
)
(B, B_q, B_scale, _), _ = make_test_weights(
num_experts,
N // 2,
K,
in_dtype=act_dtype,
quant_dtype=quant_dtype,
block_shape=block_shape,
per_out_ch_quant=per_act_token_quant,
)
out_shape = (num_experts, max_tokens_per_expert, N)
test_output = torch.zeros(out_shape, dtype=act_dtype, device=DEVICE)
ref_output = torch.zeros(out_shape, dtype=act_dtype, device=DEVICE)
q_ref_output = torch.zeros(out_shape, dtype=act_dtype, device=DEVICE)
compute_tl_dtype = {
torch.float16: tl.float16,
torch.bfloat16: tl.bfloat16,
torch.float32: tl.float32,
}[test_output.dtype]
assert A_q.dtype == B_q.dtype
invoke_moe_batched_triton_kernel(
A_q,
B_q,
test_output,
num_expert_tokens,
compute_tl_dtype,
# Quantization data
A_scale,
B_scale,
None,
# Quantization schemes
use_fp8_w8a8,
False,
False,
config={
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 16,
"BLOCK_SIZE_K": 16 if dtype.itemsize > 1 else fp8_block_k,
},
per_act_token_quant=per_act_token_quant,
block_shape=block_shape,
)
ref_output = native_batched_masked_quant_matmul(
A,
B,
ref_output,
num_expert_tokens,
)
q_ref_output = native_batched_masked_quant_matmul(
A_q,
B_q,
q_ref_output,
num_expert_tokens,
A_scale,
B_scale,
block_shape,
per_act_token_quant,
)
rtol, atol = {
torch.float16: (6e-2, 6e-2),
torch.bfloat16: (6e-2, 6e-2),
torch.float32: (1e-2, 1e-2),
}[test_output.dtype]
torch.testing.assert_close(ref_output, q_ref_output, atol=atol, rtol=rtol)
torch.testing.assert_close(test_output, q_ref_output, atol=atol, rtol=rtol)
@pytest.mark.parametrize(("m", "n", "k"), MNK_FACTORS)
@pytest.mark.parametrize("e", NUM_EXPERTS)
@pytest.mark.parametrize("topk", TOP_KS)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("per_act_token_quant", [False, True])
@pytest.mark.parametrize("block_shape", [None, [128, 128]])
@pytest.mark.parametrize("input_scales", [False])
def test_fused_moe_batched_experts(
m: int,
n: int,
k: int,
e: int,
topk: int,
dtype: torch.dtype,
per_act_token_quant: bool,
block_shape: list[int] | None,
input_scales: bool,
workspace_init,
):
"""Note: float8_e4m3fn is not supported on CUDA architecture < 89,
and those tests will be skipped on unsupported hardware."""
set_random_seed(7)
use_fp8_w8a8 = dtype == torch.float8_e4m3fn
if (
dtype == torch.float8_e4m3fn
and current_platform.is_cuda_alike()
and not current_platform.has_device_capability(89)
):
pytest.skip(
"Triton limitation: fp8e4nv data type is not supported on CUDA arch < 89"
)
if topk > e:
pytest.skip("topk > e")
if not use_fp8_w8a8 and (per_act_token_quant or block_shape is not None):
pytest.skip("Skip quantization test for non-quantized type")
if per_act_token_quant and block_shape is not None:
pytest.skip("Skip illegal quantization test.")
a = torch.randn((m, k), device=DEVICE, dtype=torch.bfloat16) / 10
score = torch.randn((m, e), device=DEVICE, dtype=torch.bfloat16)
if dtype.itemsize == 1:
act_dtype = torch.bfloat16
quant_dtype = dtype
else:
act_dtype = dtype
quant_dtype = None
(w1_16, w1, w1_s, _), (w2_16, w2, w2_s, _) = make_test_weights(
e,
n,
k,
block_shape=block_shape,
in_dtype=act_dtype,
quant_dtype=quant_dtype,
per_out_ch_quant=per_act_token_quant,
)
if input_scales and quant_dtype is not None:
a1_scale = torch.tensor(1, device=DEVICE, dtype=torch.float32)
a2_scale = torch.tensor(1, device=DEVICE, dtype=torch.float32)
else:
a1_scale = None
a2_scale = None
with set_current_vllm_config(vllm_config):
topk_weight, topk_ids, _ = fused_topk(a, score, topk, False)
baseline_output = torch_experts(
a,
w1,
w2,
topk_weight,
topk_ids,
w1_scale=w1_s,
w2_scale=w2_s,
a1_scale=a1_scale,
a2_scale=a2_scale,
quant_dtype=quant_dtype,
per_act_token_quant=per_act_token_quant,
block_shape=block_shape,
)
batched_output = naive_batched_moe(
a,
w1,
w2,
topk_weight,
topk_ids,
w1_scale=w1_s,
w2_scale=w2_s,
a1_scale=a1_scale,
a2_scale=a2_scale,
quant_dtype=quant_dtype,
per_act_token_quant=per_act_token_quant,
block_shape=block_shape,
)
triton_output = batched_moe(
a,
w1,
w2,
topk_weight,
topk_ids,
w1_scale=w1_s,
w2_scale=w2_s,
a1_scale=a1_scale,
a2_scale=a2_scale,
quant_dtype=quant_dtype,
per_act_token_quant=per_act_token_quant,
block_shape=block_shape,
)
torch.testing.assert_close(batched_output, baseline_output, atol=3e-2, rtol=2e-2)
torch.testing.assert_close(triton_output, batched_output, atol=2e-2, rtol=2e-2)
# USE_TD in moe_mmk
# K % 8 == 0 (bf16, 16-byte alignment)
_TD_SHAPES = [
(4, 1, 128, 256),
(8, 64, 512, 512),
(4, 256, 1024, 2048),
]
def _td_supported() -> bool:
return current_platform.is_xpu() or (
current_platform.is_cuda() and current_platform.has_device_capability(90)
)
def _run_td(A, B, num_expert_tokens, use_td: bool):
out = torch.zeros(
A.shape[0],
A.shape[1],
B.shape[1],
dtype=torch.bfloat16,
device=A.device,
)
import triton
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
batched_triton_kernel,
)
if use_td:
from vllm.triton_utils.allocation import set_triton_allocator
set_triton_allocator(A.device)
E = A.shape[0]
max_tokens = A.shape[1]
K = A.shape[2]
N = B.shape[1]
BM = BN = BK = 64
grid = (E, triton.cdiv(max_tokens, BM) * triton.cdiv(N, BN))
batched_triton_kernel[grid](
A,
B,
out,
num_expert_tokens,
tl.bfloat16,
max_tokens,
K,
N,
None,
None,
None,
A.stride(0),
A.stride(1),
A.stride(2),
B.stride(0),
B.stride(2),
B.stride(1),
out.stride(0),
out.stride(1),
out.stride(2),
0,
0,
0,
0,
0,
0,
0,
0,
False,
False,
False,
BLOCK_M=BM,
BLOCK_N=BN,
BLOCK_K=BK,
USE_TD=use_td,
)
return out
@pytest.mark.parametrize("num_experts,max_tokens_per_expert,K,N", _TD_SHAPES)
def test_batched_mm_td_matches_plain(num_experts, max_tokens_per_expert, K, N):
if not _td_supported():
pytest.skip("TD requires XPU or CUDA sm_90+")
set_random_seed(42)
device = current_platform.device_type
A = (
torch.randn(
num_experts, max_tokens_per_expert, K, device=device, dtype=torch.bfloat16
)
/ 10
)
B = torch.randn(num_experts, N, K, device=device, dtype=torch.bfloat16)
num_expert_tokens = torch.randint(
1,
max_tokens_per_expert + 1,
size=(num_experts,),
device=device,
dtype=torch.int32,
)
out_plain = _run_td(A, B, num_expert_tokens, False)
out_td = _run_td(A, B, num_expert_tokens, True)
torch.testing.assert_close(out_td, out_plain, atol=6e-2, rtol=6e-2)
def test_batched_mm_td_zero_expert_tokens():
if not _td_supported():
pytest.skip("TD requires XPU or CUDA sm_90+")
set_random_seed(42)
device = current_platform.device_type
E, M, K, N = 8, 32, 256, 256
A = torch.randn(E, M, K, device=device, dtype=torch.bfloat16) / 10
B = torch.randn(E, N, K, device=device, dtype=torch.bfloat16)
num_expert_tokens = torch.zeros(E, device=device, dtype=torch.int32)
num_expert_tokens[::2] = M
out_plain = _run_td(A, B, num_expert_tokens, False)
out_td = _run_td(A, B, num_expert_tokens, True)
for e in range(E):
if num_expert_tokens[e].item() == 0:
assert out_plain[e].abs().max().item() == 0.0
assert out_td[e].abs().max().item() == 0.0
torch.testing.assert_close(out_td, out_plain, atol=6e-2, rtol=6e-2)
# BatchedTritonExperts device enablement (XPU)
def test_batched_triton_experts_supports_current_device():
assert BatchedTritonExperts._supports_current_device()
@pytest.mark.parametrize(
"activation",
[
MoEActivation.SITU,
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
MoEActivation.SWIGLUSTEP,
],
)
def test_batched_triton_supports_masked_activations(
monkeypatch, activation: MoEActivation
):
monkeypatch.setattr(current_platform, "is_xpu", lambda: False)
assert BatchedTritonExperts._supports_activation(activation)
monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
assert not BatchedTritonExperts._supports_activation(activation)
def _torch_experts_with_shared_activation(
a: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: MoEActivation,
activation_config: ApplyMoEActivationConfig,
) -> torch.Tensor:
num_tokens, hidden_dim = a.shape
topk = topk_ids.shape[1]
routed_input = a[:, None, :].expand(-1, topk, -1).reshape(-1, hidden_dim)
routed_experts = topk_ids.reshape(-1)
routed_output = torch.zeros_like(routed_input)
for expert in range(w1.shape[0]):
expert_mask = routed_experts == expert
if not expert_mask.any():
continue
projected = routed_input[expert_mask] @ w1[expert].T
activated = torch.empty(
projected.shape[0], w2.shape[2], dtype=a.dtype, device=a.device
)
apply_moe_activation(
activation,
activated,
projected,
activation_config=activation_config,
)
routed_output[expert_mask] = activated @ w2[expert].T
return (
(
routed_output.view(num_tokens, topk, hidden_dim).float()
* topk_weights[..., None]
)
.sum(dim=1)
.to(a.dtype)
)
_MASKED_BATCHED_ACTIVATION_CASES = [
pytest.param(
MoEActivation.SITU,
ApplyMoEActivationConfig(
activation_situ_beta=1.5,
activation_situ_linear_beta=2.0,
),
id="situ",
),
pytest.param(
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
ApplyMoEActivationConfig(clamp_limit=3.0, alpha=1.3, beta=0.5),
id="swigluoai-uninterleave",
),
pytest.param(
MoEActivation.SWIGLUSTEP,
ApplyMoEActivationConfig(),
id="swiglustep",
),
]
@pytest.mark.parametrize(
("activation", "activation_config"), _MASKED_BATCHED_ACTIVATION_CASES
)
@torch.inference_mode()
def test_batched_experts_masked_activation_end_to_end(
activation: MoEActivation,
activation_config: ApplyMoEActivationConfig,
):
if current_platform.is_xpu():
pytest.skip("Masked activations are not enabled on XPU")
from vllm.v1.worker.workspace import init_workspace_manager
set_random_seed(7)
device = current_platform.device_type
init_workspace_manager(torch.device(f"{device}:0"))
m, n, k, e, topk = 16, 128, 128, 8, 2
a = torch.randn((m, k), device=device, dtype=torch.bfloat16) / 10
score = torch.randn((m, e), device=device, dtype=torch.bfloat16)
w1 = torch.randn((e, 2 * n, k), device=device, dtype=torch.bfloat16) / 15
w2 = torch.randn((e, k, n), device=device, dtype=torch.bfloat16) / 15
moe_config = make_dummy_moe_config(
num_experts=e,
experts_per_token=topk,
hidden_dim=k,
intermediate_size=n,
in_dtype=a.dtype,
activation=activation,
)
moe_config.swiglu_limit = activation_config.clamp_limit
moe_config.swiglu_alpha = activation_config.alpha
moe_config.swiglu_beta = activation_config.beta
moe_config.activation_situ_beta = activation_config.activation_situ_beta
moe_config.activation_situ_linear_beta = (
activation_config.activation_situ_linear_beta
)
routing_weights = torch.softmax(score.float(), dim=-1)
topk_weights, topk_ids = torch.topk(routing_weights, topk, dim=-1)
topk_ids = topk_ids.to(torch.int32)
with set_current_vllm_config(vllm_config):
baseline_output = _torch_experts_with_shared_activation(
a,
w1,
w2,
topk_weights,
topk_ids,
activation,
activation_config,
)
triton_output = batched_moe(
a, w1, w2, topk_weights, topk_ids, moe_config=moe_config
)
torch.testing.assert_close(triton_output, baseline_output, atol=3e-2, rtol=2e-2)
@pytest.mark.parametrize("m,n,k,e,topk", [(32, 512, 512, 8, 2), (45, 1024, 128, 8, 1)])
def test_batched_experts_end_to_end(m, n, k, e, topk):
"""End-to-end BatchedTritonExperts via the reference (no-comms)
BatchedPrepareAndFinalize, validated against a torch reference. Exercises
the device-enablement path."""
from vllm.v1.worker.workspace import init_workspace_manager
set_random_seed(7)
device = current_platform.device_type
init_workspace_manager(torch.device(f"{device}:0"))
a = torch.randn((m, k), device=device, dtype=torch.bfloat16) / 10
score = torch.randn((m, e), device=device, dtype=torch.bfloat16)
# w1 is the fused gate+up projection [E, 2N, K]; w2 is [E, K, N].
w1 = torch.randn((e, 2 * n, k), device=device, dtype=torch.bfloat16) / 15
w2 = torch.randn((e, k, n), device=device, dtype=torch.bfloat16) / 15
with set_current_vllm_config(vllm_config):
topk_weight, topk_ids, _ = fused_topk(a, score, topk, False)
baseline_output = torch_experts(a, w1, w2, topk_weight, topk_ids)
triton_output = batched_moe(a, w1, w2, topk_weight, topk_ids)
torch.testing.assert_close(triton_output, baseline_output, atol=3e-2, rtol=2e-2)
def test_batched_triton_backend_mapping():
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
UnquantizedMoeBackend,
map_unquantized_backend,
)
assert (
map_unquantized_backend("batched_triton")
== UnquantizedMoeBackend.BATCHED_TRITON
)
assert map_unquantized_backend("triton") == UnquantizedMoeBackend.TRITON
-572
View File
@@ -1,572 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from tests.kernels.moe.utils import (
make_dummy_moe_config,
make_test_quant_config,
make_test_weights,
modular_triton_fused_moe,
)
from tests.kernels.quant_utils import native_w8a8_block_matmul
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.fused_moe import (
fused_experts,
fused_topk,
)
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.config import (
fp8_w8a8_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts import triton_moe
from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import (
_valid_deep_gemm_shape,
)
from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import (
TritonOrDeepGemmExperts,
)
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
per_token_group_quant_fp8,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import (
get_mk_alignment_for_contiguous_layout,
is_deep_gemm_e8m0_used,
)
from vllm.utils.import_utils import has_deep_gemm
dg_available = has_deep_gemm()
if current_platform.get_device_capability() < (9, 0):
pytest.skip("FP8 Triton requires CUDA 9.0 or higher", allow_module_level=True)
if current_platform.is_fp8_fnuz():
pytest.skip(
"Tests in this file require float8_e4m3fn and platform does not support",
allow_module_level=True,
)
vllm_config = VllmConfig()
# Test configurations
DTYPES = [torch.bfloat16] # [torch.half, torch.bfloat16, torch.float32]
# Deepseek-V3's intermediate size 18432, so N is 18432*2/8=4608 at TP8
# and its hidden size is 7168.
MNK_FACTORS = [
(1, 128, 128),
(1, 128, 7168),
(1, 1024, 7168),
(1, 4608, 128),
(1, 4608, 7168),
(83, 128, 128),
(83, 512, 512),
(83, 4608, 512),
(83, 4608, 7168),
(128, 512, 512),
(128, 1024, 7168),
(128, 4608, 7168),
(2048, 128, 128),
(2048, 1024, 7168),
(2048, 4608, 512),
(2048, 4608, 7168),
(8192, 128, 128),
(8192, 128, 7168),
(8192, 1024, 7168),
(8192, 4608, 7168),
]
MNK_FACTORS_DG = [
(128, 128, 128),
(128, 128, 7168),
(128, 1024, 7168),
(128, 4608, 128),
(128, 4608, 7168),
(192, 512, 512),
(192, 1024, 7168),
(192, 4608, 7168),
(1335, 128, 128),
(1335, 1024, 7168),
(1335, 4608, 512),
(1335, 4608, 7168),
(2048, 128, 128),
(2048, 128, 7168),
(2048, 1024, 7168),
(2048, 4608, 7168),
]
BLOCK_SIZE = [[128, 128]]
E = [2, 8, 16] # [128, 256]
TOP_KS = [1, 2, 6]
SEEDS = [0]
def torch_w8a8_block_fp8_moe(
a,
w1,
w2,
w1_s,
w2_s,
topk_weight,
topk_ids,
block_shape,
silu_fp32=False,
expert_map=None,
):
"""Fused MoE with block-wise fp8 quantization using native torch.
silu_fp32=True computes the intermediate SiLU in fp32 and quantizes it
directly, matching the modular Helion silu_and_mul_per_block_quant kernel;
False rounds the SiLU output to bf16 first, matching fused_experts. Each
kernel is checked against the reference variant matching its precision.
"""
B, D = a.shape
topk = topk_ids.size(1)
a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device)
topk_weight = topk_weight.view(-1)
topk_ids = topk_ids.view(-1)
if expert_map is not None:
# w1/w2 hold this rank's experts; slots mapped to -1 contribute nothing.
topk_ids = expert_map[topk_ids]
_, block_k = block_shape[0], block_shape[1]
# Quantize with the production per-token-group fp8 kernel (same HIP/CUDA op the
# kernels use) so the reference is bit-identical here; removes ~0.06-0.10% fp8
# boundary-flip divergence that otherwise accumulates over K. Matmul stays fp32.
a_q, a_s = per_token_group_quant_fp8(a, block_k, dtype=current_platform.fp8_dtype())
a_q = a_q.to(torch.float32)
for i in range(w1.shape[0]):
mask = topk_ids == i
if mask.sum():
inter_out = native_w8a8_block_matmul(
a_q[mask], w1[i], a_s[mask], w1_s[i], block_shape, output_dtype=a.dtype
)
act_out = SiluAndMul().forward_native(
inter_out.float() if silu_fp32 else inter_out
)
act_out_q, act_out_s = per_token_group_quant_fp8(
act_out, block_k, dtype=current_platform.fp8_dtype()
)
out[mask] = native_w8a8_block_matmul(
act_out_q, w2[i], act_out_s, w2_s[i], block_shape, output_dtype=a.dtype
)
return (
out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype)
).sum(dim=1)
# Skip all tests if CUDA is not available
pytest.importorskip("torch.cuda")
@pytest.fixture(autouse=True)
def setup_cuda():
torch.set_default_device("cuda")
@pytest.mark.parametrize(("M", "N", "K"), MNK_FACTORS)
@pytest.mark.parametrize("E", E)
@pytest.mark.parametrize("topk", TOP_KS)
@pytest.mark.parametrize("block_size", BLOCK_SIZE)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("seed", SEEDS)
@torch.inference_mode()
def test_w8a8_block_fp8_fused_moe(
M, N, K, E, topk, block_size, dtype, seed, monkeypatch, workspace_init
):
if topk > E:
pytest.skip(f"Skipping test; topk={topk} > E={E}")
torch.manual_seed(seed)
a = torch.randn((M, K), dtype=dtype) / 10
score = torch.randn((M, E), dtype=dtype)
w1, w2, quant_config = make_test_quant_config(
E,
N,
K,
dtype,
quant_dtype=torch.float8_e4m3fn,
per_act_token_quant=False,
block_shape=block_size,
)
m_fused_moe = modular_triton_fused_moe(make_dummy_moe_config(), quant_config)
topk_weights, topk_ids, _ = fused_topk(a, score.float(), topk, False)
# Set the context to avoid lots of warning spam.
with set_current_vllm_config(vllm_config):
ref_out = torch_w8a8_block_fp8_moe(
a,
w1,
w2,
quant_config.w1_scale,
quant_config.w2_scale,
topk_weights,
topk_ids,
block_size,
)
out = fused_experts(
a, w1, w2, topk_weights, topk_ids, quant_config=quant_config
)
m_out = m_fused_moe.apply(
a,
w1,
w2,
topk_weights,
topk_ids,
activation=MoEActivation.SILU,
apply_router_weight_on_input=False,
expert_map=None,
global_num_experts=w1.shape[0],
)
# 0.039 only needed for M >= 8192
tol = 0.035 if M < 8192 else 0.039
# The modular path fuses SiLU+quant in fp32 (silu_and_mul_per_block_quant),
# while fused_experts/the reference round SiLU to bf16 first. On large K/N this
# ~1-ULP gap pushes m_out past the base tol, so validate m_out against an
# fp32-SiLU reference — keeping the tight base tolerance, no widened override.
if current_platform.is_rocm() and K >= 4096 and N >= 1024:
with set_current_vllm_config(vllm_config):
ref_out_m = torch_w8a8_block_fp8_moe(
a,
w1,
w2,
quant_config.w1_scale,
quant_config.w2_scale,
topk_weights,
topk_ids,
block_size,
silu_fp32=True,
)
else:
ref_out_m = ref_out
torch.testing.assert_close(out, ref_out, atol=tol, rtol=tol)
torch.testing.assert_close(m_out, ref_out_m, atol=tol, rtol=tol)
@pytest.mark.parametrize("clamp_limit", [None, 10.0])
@torch.inference_mode()
def test_w8a8_block_fp8_clamp_skips_fused_silu_quant(
clamp_limit, monkeypatch, workspace_init, disable_deepgemm_ue8m0
):
"""A SwiGLU clamp must bypass the fused SiLU+quant kernel, which cannot clamp."""
torch.manual_seed(0)
M, N, K, E, topk = 16, 256, 512, 8, 2
block_size = [128, 128]
a = torch.randn((M, K), dtype=torch.bfloat16) / 10
score = torch.randn((M, E), dtype=torch.bfloat16)
(_, w1, w1_s, _), (_, w2, w2_s, _) = make_test_weights(
E, N, K, torch.bfloat16, torch.float8_e4m3fn, block_shape=block_size
)
quant_config = fp8_w8a8_moe_quant_config(
w1_scale=w1_s,
w2_scale=w2_s,
block_shape=block_size,
gemm1_clamp_limit=clamp_limit,
)
m_fused_moe = modular_triton_fused_moe(make_dummy_moe_config(), quant_config)
topk_weights, topk_ids, _ = fused_topk(a, score.float(), topk, False)
fused_calls = 0
fused_op = triton_moe.ops.silu_and_mul_per_block_quant
def counting_fused_op(*args, **kwargs):
nonlocal fused_calls
fused_calls += 1
return fused_op(*args, **kwargs)
monkeypatch.setattr(
triton_moe.ops, "silu_and_mul_per_block_quant", counting_fused_op
)
with set_current_vllm_config(vllm_config):
m_fused_moe.apply(
a,
w1,
w2,
topk_weights,
topk_ids,
activation=MoEActivation.SILU,
apply_router_weight_on_input=False,
expert_map=None,
global_num_experts=E,
)
assert (fused_calls > 0) == (clamp_limit is None), (
f"fused fast path ran {fused_calls} times with clamp_limit={clamp_limit}"
)
@pytest.mark.parametrize(("M", "N", "K"), [(83, 512, 512), (2048, 4608, 512)])
@pytest.mark.parametrize("topk", [2, 6])
@pytest.mark.parametrize("ep_size", [2, 4])
@torch.inference_mode()
def test_w8a8_block_fp8_fused_moe_expert_map(M, N, K, topk, ep_size, workspace_init):
"""One EP rank: local expert shard plus an expert_map with -1 entries."""
E, block_size, dtype = 16, [128, 128], torch.bfloat16
torch.manual_seed(0)
a = torch.randn((M, K), dtype=dtype) / 10
score = torch.randn((M, E), dtype=dtype)
w1, w2, quant_config = make_test_quant_config(
E, N, K, dtype, quant_dtype=torch.float8_e4m3fn, block_shape=block_size
)
local_e = E // ep_size
e_ids = torch.randperm(E, dtype=torch.int32)[:local_e]
e_map = torch.full((E,), -1, dtype=torch.int32)
e_map[e_ids] = torch.arange(local_e, dtype=torch.int32)
w1, w2 = w1[e_ids].contiguous(), w2[e_ids].contiguous()
w1_s = quant_config.w1_scale[e_ids].contiguous()
w2_s = quant_config.w2_scale[e_ids].contiguous()
quant_config = fp8_w8a8_moe_quant_config(
w1_scale=w1_s, w2_scale=w2_s, block_shape=block_size
)
m_fused_moe = modular_triton_fused_moe(make_dummy_moe_config(), quant_config)
topk_weights, topk_ids, _ = fused_topk(a, score.float(), topk, False)
with set_current_vllm_config(vllm_config):
ref_out = torch_w8a8_block_fp8_moe(
a, w1, w2, w1_s, w2_s, topk_weights, topk_ids, block_size, expert_map=e_map
)
m_out = m_fused_moe.apply(
a,
w1,
w2,
topk_weights,
topk_ids,
activation=MoEActivation.SILU,
apply_router_weight_on_input=False,
expert_map=e_map,
global_num_experts=E,
)
torch.testing.assert_close(m_out, ref_out, atol=0.035, rtol=0.035)
@pytest.mark.parametrize(("M", "N", "K"), MNK_FACTORS_DG)
@pytest.mark.parametrize("E", E)
@pytest.mark.parametrize("topk", TOP_KS)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.skipif(not dg_available, reason="DeepGemm kernels not available.")
@pytest.mark.skipif(is_deep_gemm_e8m0_used(), reason="Not E8M0 scale MOE")
@torch.inference_mode()
def test_w8a8_block_fp8_deep_gemm_fused_moe(M, N, K, E, topk, seed, monkeypatch):
if topk > E:
pytest.skip(f"Skipping test: topk={topk} > E={E}")
if not _valid_deep_gemm_shape(M, N, K):
pytest.skip(f"Skipping test: invalid size m={M}, n={N}, k={K}")
torch.manual_seed(seed)
block_size = get_mk_alignment_for_contiguous_layout()
dtype = torch.bfloat16
a = torch.randn((M, K), dtype=dtype) / 10
score = torch.randn((M, E), dtype=dtype)
(_, w1, w1_s, _), (_, w2, w2_s, _) = make_test_weights(
E,
N,
K,
dtype,
torch.float8_e4m3fn,
per_out_ch_quant=False,
block_shape=block_size,
)
# Note: for now use_compile will error out if the problem size is
# large enough to trigger chunking. I'm leaving the flag and
# setup code in case we are able to revisit this later.
use_compile = False
use_cudagraph = N >= 1024 and K >= 1024 and current_platform.is_cuda_alike()
topk_weights, topk_ids, _ = fused_topk(a, score.float(), topk, False)
quant_config = fp8_w8a8_moe_quant_config(
w1_scale=w1_s,
w2_scale=w2_s,
block_shape=block_size,
)
moe_config = make_dummy_moe_config()
deep_gemm_experts = mk.FusedMoEKernel(
prepare_finalize=maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
use_monolithic=False,
),
fused_experts=TritonOrDeepGemmExperts(
moe_config=moe_config,
quant_config=quant_config,
),
)
def deep_gemm_moe_fp8(a, w1, w2, w1_s, w2_s, topk_weights, topk_ids):
return deep_gemm_experts.apply(
hidden_states=a,
w1=w1,
w2=w2,
topk_weights=topk_weights,
topk_ids=topk_ids,
global_num_experts=E,
activation=MoEActivation.SILU,
apply_router_weight_on_input=False,
expert_map=False,
)
# Set the context to avoid lots of warning spam.
with set_current_vllm_config(vllm_config):
ref_out = torch_w8a8_block_fp8_moe(
a, w1, w2, w1_s, w2_s, topk_weights, topk_ids, block_size
)
if use_compile:
deep_gemm_moe_fp8_fn = torch.compile(
deep_gemm_moe_fp8, backend="inductor", fullgraph=True
)
torch._dynamo.mark_dynamic(a, 0)
torch._dynamo.mark_dynamic(topk_weights, 0)
torch._dynamo.mark_dynamic(topk_ids, 0)
else:
deep_gemm_moe_fp8_fn = deep_gemm_moe_fp8
out = deep_gemm_moe_fp8_fn(a, w1, w2, w1_s, w2_s, topk_weights, topk_ids)
if use_cudagraph:
out.fill_(0)
stream = torch.cuda.Stream()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=stream):
out = deep_gemm_moe_fp8_fn(
a, w1, w2, w1_s, w2_s, topk_weights, topk_ids
)
torch.accelerator.synchronize()
graph.replay()
torch.accelerator.synchronize()
torch.testing.assert_close(out, ref_out, atol=0.035, rtol=0.035)
def _dequant_block_fp8(
w_fp8: torch.Tensor, w_s: torch.Tensor, block_shape: list[int]
) -> torch.Tensor:
"""Dequantize a [N, K] fp8 weight with [Nb, Kb] block scales to fp32."""
n, k = w_fp8.shape
bn, bk = block_shape
s = w_s.repeat_interleave(bn, dim=0).repeat_interleave(bk, dim=1)
return w_fp8.to(torch.float32) * s[:n, :k]
@pytest.mark.parametrize("tp_rank", [0, 1, 2, 3])
@torch.inference_mode()
def test_w8a8_block_fp8_fused_moe_refined_block_scales(tp_rank, workspace_init):
"""TP-misaligned blockwise FP8 MoE (e.g. Qwen4Exp, intermediate
640 at TP=4 -> 160 per rank, not divisible by the checkpoint's 128 block).
The per-rank scale grid is refined from 128 to 32 (lossless: each 32-block
lies inside one global 128-block) so the Triton kernel can consume exact
per-shard scales. Simulates one TP rank's shard and checks:
1. the refined+sharded scales dequantize exactly like the global scales;
2. the Triton fused MoE kernel with block_shape=[32, 32] matches the
native-torch blockwise reference on the same shard.
"""
torch.manual_seed(0)
dtype = torch.bfloat16
M, E, topk = 8, 4, 2
hidden = 256
inter_full = 640
tp_size = 4
n_shard = inter_full // tp_size # 160
ckpt_block = [128, 128]
refined_block = [32, 32]
factor = ckpt_block[0] // refined_block[0] # 4
a = torch.randn((M, hidden), dtype=dtype) / 10
score = torch.randn((M, E), dtype=dtype)
(_, w1_full, w1_s_full, _), (_, w2_full, w2_s_full, _) = make_test_weights(
E,
inter_full,
hidden,
dtype,
torch.float8_e4m3fn,
per_out_ch_quant=False,
block_shape=ckpt_block,
)
assert w1_s_full is not None
assert w2_s_full is not None
# TP shard the weights (exact slicing, no scale involvement).
lo, hi = tp_rank * n_shard, (tp_rank + 1) * n_shard
w1 = torch.cat(
[w1_full[:, lo:hi], w1_full[:, inter_full + lo : inter_full + hi]], dim=1
)
w2 = w2_full[:, :, lo:hi]
# Refine the global 128-block scales to 32 blocks, then take the shard's
# slice -- mirrors the Fp8MoEMethod/RoutedExperts loading path.
w1_s32 = w1_s_full.repeat_interleave(factor, dim=-2).repeat_interleave(
factor, dim=-1
)
nb = n_shard // refined_block[0] # 5 local 32-blocks per projection
gate_hi = w1_s32.shape[1] // 2
w1_s = torch.cat(
[
w1_s32[:, tp_rank * nb : (tp_rank + 1) * nb],
w1_s32[:, gate_hi + tp_rank * nb : gate_hi + (tp_rank + 1) * nb],
],
dim=1,
)
w2_s32 = w2_s_full.repeat_interleave(factor, dim=-2).repeat_interleave(
factor, dim=-1
)
w2_s = w2_s32[:, :, tp_rank * nb : (tp_rank + 1) * nb]
# The refined per-shard scales must reproduce the global-scale dequant
# exactly (dequant full-width, then slice to the shard).
for e in range(E):
d32 = _dequant_block_fp8(w1[e, :n_shard], w1_s[e, :nb], refined_block)
d128 = _dequant_block_fp8(w1_full[e], w1_s_full[e], ckpt_block)[lo:hi]
assert torch.equal(d32, d128)
d32 = _dequant_block_fp8(w2[e], w2_s[e], refined_block)
d128 = _dequant_block_fp8(w2_full[e], w2_s_full[e], ckpt_block)[:, lo:hi]
assert torch.equal(d32, d128)
quant_config = fp8_w8a8_moe_quant_config(
w1_scale=w1_s,
w2_scale=w2_s,
block_shape=refined_block,
)
topk_weights, topk_ids, _ = fused_topk(a, score.float(), topk, False)
with set_current_vllm_config(vllm_config):
ref_out = torch_w8a8_block_fp8_moe(
a, w1, w2, w1_s, w2_s, topk_weights, topk_ids, refined_block
)
out = fused_experts(
a, w1, w2, topk_weights, topk_ids, quant_config=quant_config
)
torch.testing.assert_close(out, ref_out, atol=0.035, rtol=0.035)
-163
View File
@@ -1,163 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from tests.kernels.moe.utils import make_test_quant_config
from tests.kernels.quant_utils import (
native_per_token_group_quant_int8,
native_w8a8_block_matmul,
)
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.fused_moe import fused_experts, fused_topk
from vllm.model_executor.layers.quantization.utils.int8_utils import (
per_token_group_quant_int8,
)
from vllm.platforms import current_platform
if current_platform.get_device_capability() < (7, 0):
pytest.skip("INT8 Triton requires CUDA 7.0 or higher", allow_module_level=True)
vllm_config = VllmConfig()
DTYPES = [torch.bfloat16]
MNK_FACTORS = [
(1, 128, 128),
(1, 128, 7168),
(1, 1024, 7168),
(1, 4096, 512),
(1, 4096, 7168),
(33, 512, 512),
(33, 128, 7168),
(33, 1024, 7168),
(33, 4096, 128),
(33, 4096, 7168),
(128, 128, 128),
(128, 1024, 7168),
(128, 4096, 512),
(128, 4096, 7168),
(222, 512, 512),
(222, 1024, 7168),
(222, 4096, 7168),
(2048, 128, 128),
(2048, 1024, 7168),
(2048, 4096, 4096),
]
E = [8, 24]
TOP_KS = [2, 6]
# BLOCK_SIZE = [[64, 64], [64, 128], [128, 64], [128, 128]]
BLOCK_SIZE = [[128, 128]]
SEEDS = [0]
# For test
def torch_w8a8_block_int8_moe(a, w1, w2, w1_s, w2_s, score, topk, block_shape):
"""This function performs fused moe with block-wise quantization using
native torch."""
B, D = a.shape
a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device)
score = torch.softmax(score, dim=-1, dtype=torch.float32)
topk_weight, topk_ids = torch.topk(score, topk)
topk_weight = topk_weight.view(-1)
topk_ids = topk_ids.view(-1)
_, block_k = block_shape[0], block_shape[1]
a_q, a_s = native_per_token_group_quant_int8(a, block_k, round_to_nearest=False)
for i in range(w1.shape[0]):
mask = topk_ids == i
if mask.sum():
inter_out = native_w8a8_block_matmul(
a_q[mask], w1[i], a_s[mask], w1_s[i], block_shape, output_dtype=a.dtype
)
act_out = SiluAndMul().forward_native(inter_out)
act_out_q, act_out_s = native_per_token_group_quant_int8(
act_out, block_k, round_to_nearest=False
)
act_out = act_out.to(torch.float32)
out[mask] = native_w8a8_block_matmul(
act_out_q, w2[i], act_out_s, w2_s[i], block_shape, output_dtype=a.dtype
)
return (
out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype)
).sum(dim=1)
@pytest.fixture(autouse=True)
def setup_cuda():
"""Sets the default CUDA device before each test in this module."""
torch.set_default_device("cuda")
@pytest.mark.parametrize(("M", "N", "K"), MNK_FACTORS)
@pytest.mark.parametrize("E", E)
@pytest.mark.parametrize("topk", TOP_KS)
@pytest.mark.parametrize("block_size", BLOCK_SIZE)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("seed", SEEDS)
@torch.inference_mode()
def test_w8a8_block_int8_fused_moe(M, N, K, E, topk, block_size, dtype, seed):
"""Tests the fused_moe kernel with W8A8 INT8 block quantization against a
native torch reference."""
torch.manual_seed(seed)
a = torch.randn((M, K), dtype=dtype) / 10
score = torch.randn((M, E), dtype=dtype)
topk_weights, topk_ids, _ = fused_topk(a, score.float(), topk, False)
w1, w2, quant_config = make_test_quant_config(
E,
N,
K,
dtype,
quant_dtype=torch.int8,
per_act_token_quant=False,
block_shape=block_size,
)
# Set the context to avoid lots of warning spam.
with set_current_vllm_config(vllm_config):
out = fused_experts(
a, w1, w2, topk_weights, topk_ids, quant_config=quant_config
)
ref_out = torch_w8a8_block_int8_moe(
a,
w1,
w2,
quant_config.w1_scale,
quant_config.w2_scale,
score,
topk,
block_size,
)
# Check results
torch.testing.assert_close(out, ref_out, atol=0.065, rtol=0.065)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
def test_block_int8_quantization_truncates_fractional_values(dtype):
"""Block INT8 quantization truncates, unlike per-token round-to-nearest."""
device = current_platform.device_type
x = torch.tensor(
[[-127, -1.75, -0.75, 0.75, 1.75, 127] + [0] * 122],
dtype=dtype,
device=device,
)
expected = torch.tensor(
[[-127, -1, 0, 0, 1, 127] + [0] * 122], dtype=torch.int8, device=device
)
out_q, scale = per_token_group_quant_int8(x, group_size=128)
ref_q, ref_s = native_per_token_group_quant_int8(
x, group_size=128, round_to_nearest=False
)
torch.testing.assert_close(out_q, expected, atol=0, rtol=0)
torch.testing.assert_close(ref_q, expected, atol=0, rtol=0)
torch.testing.assert_close(scale, torch.ones_like(scale), atol=0, rtol=0)
torch.testing.assert_close(ref_s, scale, atol=0, rtol=0)
@@ -1,149 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests compute_expert_num_tokens kernels."""
import dataclasses
import pytest
import torch
from vllm.model_executor.layers.fused_moe.utils import count_expert_num_tokens
from vllm.platforms import current_platform
DEVICE = current_platform.device_type
pytestmark = pytest.mark.skipif(
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
reason="Triton MoE kernels require CUDA/ROCm/XPU.",
)
@dataclasses.dataclass
class TestTensors:
topk_ids: torch.Tensor
expert_map: torch.Tensor | None = None
def to_device(self, device: str):
self.topk_ids = self.topk_ids.to(device=device)
if self.expert_map is not None:
self.expert_map = self.expert_map.to(device=device)
@staticmethod
def make(
num_tokens: int,
num_topk: int,
num_experts: int,
device: str,
topk_ids_dtype: torch.dtype,
) -> "TestTensors":
# make topk ids
topk_ids = torch.empty((num_tokens, num_topk), device=device, dtype=torch.int64)
for x in range(num_tokens):
topk_ids[x] = torch.randperm(num_experts)[:num_topk]
topk_ids = topk_ids.to(dtype=torch.int64)
return TestTensors(topk_ids=topk_ids)
def with_ep_rank(
self, ep_rank: int, num_global_experts: int, num_local_experts: int, device: str
):
# make an expert map
expert_map = torch.empty((num_global_experts), device=device, dtype=torch.int32)
expert_map.fill_(-1)
s = ep_rank * num_local_experts
e = s + num_local_experts
expert_map[s:e] = torch.tensor(list(range(num_local_experts)), device=device)
return TestTensors(topk_ids=self.topk_ids.clone(), expert_map=expert_map)
def ref_impl(tt: TestTensors, expert_num_tokens: torch.Tensor):
# do the reference in cpu
tt.to_device("cpu")
expert_ids, counts = tt.topk_ids.unique(return_counts=True)
for eid, count in zip(expert_ids, counts):
if eid != -1 and tt.expert_map is not None:
eid = tt.expert_map[eid]
if eid == -1:
continue
expert_num_tokens[eid] += count
def do_test_compute_expert_num_tokens(
num_tokens: int,
num_topk: int,
num_experts: int,
ep_size: int,
topk_ids_dtype: torch.dtype,
):
assert num_topk <= num_experts
tt = TestTensors.make(
num_tokens, num_topk, num_experts, topk_ids_dtype=topk_ids_dtype, device="cpu"
)
num_global_experts = num_experts
assert num_global_experts % ep_size == 0
num_local_experts = num_global_experts // ep_size
for ep_rank in range(ep_size):
tt_rank = tt.with_ep_rank(ep_rank, num_global_experts, num_local_experts, "cpu")
ref_expert_num_tokens = torch.zeros(
(num_local_experts), device="cpu", dtype=torch.int32
)
ref_impl(tt_rank, ref_expert_num_tokens)
ref_expert_num_tokens = ref_expert_num_tokens.to(DEVICE)
tt_rank.to_device(DEVICE)
# Test with expert_map
triton_expert_num_tokens_w_emap = count_expert_num_tokens(
tt_rank.topk_ids, num_local_experts, tt_rank.expert_map
)
# Test without expert map
topk_ids = tt_rank.expert_map[tt_rank.topk_ids].to(topk_ids_dtype)
triton_expert_num_tokens_wo_emap = count_expert_num_tokens(
topk_ids, num_local_experts, expert_map=None
)
torch.testing.assert_close(
ref_expert_num_tokens, triton_expert_num_tokens_w_emap, atol=0, rtol=0
)
torch.testing.assert_close(
ref_expert_num_tokens, triton_expert_num_tokens_wo_emap, atol=0, rtol=0
)
@pytest.mark.parametrize("num_tokens", [1, 4, 8, 11, 127, 128, 3333, 7317])
@pytest.mark.parametrize("num_topk", [2, 6, 8])
@pytest.mark.parametrize("num_experts", [64])
@pytest.mark.parametrize("ep_size", [1, 2, 4])
@pytest.mark.parametrize("topk_ids_dtype", [torch.int64])
def test_compute_expert_num_tokens(
num_tokens: int,
num_topk: int,
num_experts: int,
ep_size: int,
topk_ids_dtype: torch.dtype,
):
do_test_compute_expert_num_tokens(
num_tokens, num_topk, num_experts, ep_size, topk_ids_dtype
)
@pytest.mark.parametrize("numel", list(range(1, 8192, 111)))
@pytest.mark.parametrize("num_experts", [32])
@pytest.mark.parametrize("ep_size", [2])
@pytest.mark.parametrize("topk_ids_dtype", [torch.int64])
def test_compute_expert_num_tokens_from_numel(
numel: int, num_experts: int, ep_size: int, topk_ids_dtype: torch.dtype
):
do_test_compute_expert_num_tokens(
num_tokens=numel,
num_topk=1,
num_experts=num_experts,
ep_size=ep_size,
topk_ids_dtype=topk_ids_dtype,
)
-608
View File
@@ -1,608 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from tests.kernels.allclose_default import get_default_atol, get_default_rtol
from vllm._custom_ops import (
cpu_fused_moe,
cpu_fused_moe_int8,
cpu_prepack_moe_weight,
cpu_prepack_moe_weight_int8,
)
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
RoutingMethodType,
biased_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
ArmCPUUnquantizedExperts,
CPUUnquantizedExperts,
PowerCPUUnquantizedExperts,
X86CPUUnquantizedExperts,
)
from vllm.model_executor.layers.fused_moe.router.cpu_router import _softmax_topk
from vllm.platforms import CpuArchEnum, current_platform
from vllm.utils.torch_utils import set_random_seed
if not current_platform.is_cpu():
pytest.skip("skipping CPU-only tests", allow_module_level=True)
EXPERT_NUM = [
8,
]
HIDDEN_DIM = [128, 2880]
INTERMEDIATE_DIM = [128, 2880]
BATCH_SIZE = [1, 64, 256]
ACT = [
MoEActivation.SILU,
MoEActivation.SWIGLUOAI,
MoEActivation.GELU,
MoEActivation.GELU_TANH,
]
USE_BIAS = [False, True]
ISA = ["vec"]
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
ISA.append("neon")
if torch.cpu._is_amx_tile_supported():
ISA.append("amx")
if current_platform.get_cpu_architecture() == CpuArchEnum.POWERPC:
ISA.append("vsx")
DTYPE = [torch.bfloat16]
def _ref_moe_activation(
input: torch.Tensor,
activation: MoEActivation,
) -> torch.Tensor:
if activation == MoEActivation.SWIGLUOAI:
gate = input[..., ::2].clamp(max=7.0)
up = input[..., 1::2].clamp(min=-7.0, max=7.0)
glu = gate * torch.sigmoid(gate * 1.702)
return (up + 1) * glu
d = input.shape[-1] // 2
gate, up = input[..., :d], input[..., d:]
if activation == MoEActivation.SILU:
return torch.nn.functional.silu(gate) * up
if activation == MoEActivation.GELU:
return torch.nn.functional.gelu(gate, approximate="none") * up
if activation == MoEActivation.GELU_TANH:
return torch.nn.functional.gelu(gate, approximate="tanh") * up
raise ValueError(f"Unsupported activation: {activation}")
def ref_fused_moe(
input: torch.Tensor,
w13: torch.Tensor,
w2: torch.Tensor,
w13_bias: torch.Tensor | None,
w2_bias: torch.Tensor | None,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: MoEActivation,
) -> torch.Tensor:
len_experts = w13.size(0)
cnts = topk_ids.new_zeros((topk_ids.shape[0], len_experts))
cnts.scatter_(1, topk_ids.to(torch.int64), 1)
tokens_per_expert = cnts.sum(dim=0)
idxs = topk_ids.view(-1).argsort()
sorted_tokens = input[idxs // topk_ids.shape[1]]
tokens_per_expert = tokens_per_expert.cpu().numpy()
outputs = []
start_idx = 0
for i, num_tokens in enumerate(tokens_per_expert):
end_idx = start_idx + num_tokens
if num_tokens == 0:
continue
tokens_for_this_expert = sorted_tokens[start_idx:end_idx].float()
curr_w13 = w13[i].float()
curr_w2 = w2[i].float()
curr_w13_bias = None
if w13_bias is not None:
curr_w13_bias = w13_bias[i].float()
curr_w2_bias = None
if w2_bias is not None:
curr_w2_bias = w2_bias[i].float()
gate_up = torch.nn.functional.linear(
tokens_for_this_expert, curr_w13, curr_w13_bias
)
# Note: to simulate the kernel implementation
gate_up = _ref_moe_activation(gate_up, activation)
gate_up = gate_up.to(dtype=input.dtype).float()
expert_out = torch.nn.functional.linear(gate_up, curr_w2, curr_w2_bias)
outputs.append(expert_out)
start_idx = end_idx
outs = torch.cat(outputs, dim=0) if len(outputs) else sorted_tokens.new_empty(0)
new_x = torch.empty_like(outs)
new_x[idxs] = outs
final_out = (
new_x.view(*topk_ids.shape, -1)
.mul_(topk_weights.unsqueeze(dim=-1))
.sum(dim=1)
.type(input.dtype)
)
return final_out
def quantize_per_channel(
weight: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Symmetrically quantize each weight output channel."""
weight_fp32 = weight.float()
scale = weight_fp32.abs().amax(dim=-1).clamp_min(1e-12) / 127.0
quantized = (
(weight_fp32 / scale.unsqueeze(-1)).round().clamp(-127, 127).to(torch.int8)
)
return quantized, scale
def quantize_per_token(
input: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Symmetrically quantize each input token."""
input_fp32 = input.float()
scale = input_fp32.abs().amax(dim=-1, keepdim=True).clamp_min(1e-7) / 127.0
quantized = (input_fp32 / scale).round().clamp(-127, 127).to(torch.int8)
return quantized, scale
def ref_fused_moe_int8(
input: torch.Tensor,
w13: torch.Tensor,
w2: torch.Tensor,
w13_scale: torch.Tensor,
w2_scale: torch.Tensor,
w13_bias: torch.Tensor | None,
w2_bias: torch.Tensor | None,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: MoEActivation,
) -> torch.Tensor:
"""Reference the two dynamically quantized INT8 GEMMs."""
input_int8, input_scale = quantize_per_token(input)
expert_num = w13.size(0)
counts = topk_ids.new_zeros((topk_ids.size(0), expert_num))
counts.scatter_(1, topk_ids.to(torch.int64), 1)
tokens_per_expert = counts.sum(dim=0).tolist()
sorted_route_ids = topk_ids.view(-1).argsort()
sorted_token_ids = sorted_route_ids // topk_ids.size(1)
outputs = []
start_idx = 0
for expert_idx, token_count in enumerate(tokens_per_expert):
end_idx = start_idx + token_count
if token_count == 0:
continue
token_ids = sorted_token_ids[start_idx:end_idx]
gate_up = torch.matmul(
input_int8[token_ids].float(),
w13[expert_idx].float().T,
)
gate_up *= input_scale[token_ids] * w13_scale[expert_idx]
if w13_bias is not None:
gate_up += w13_bias[expert_idx].float()
intermediate = _ref_moe_activation(gate_up, activation).to(input.dtype)
intermediate_int8, intermediate_scale = quantize_per_token(intermediate)
output = torch.matmul(
intermediate_int8.float(),
w2[expert_idx].float().T,
)
output *= intermediate_scale * w2_scale[expert_idx]
if w2_bias is not None:
output += w2_bias[expert_idx].float()
outputs.append(output)
start_idx = end_idx
sorted_output = torch.cat(outputs, dim=0)
routed_output = torch.empty_like(sorted_output)
routed_output[sorted_route_ids] = sorted_output
return (
routed_output.view(*topk_ids.shape, -1)
.mul_(topk_weights.unsqueeze(-1))
.sum(dim=1)
.to(input.dtype)
)
@pytest.mark.parametrize("batch_size", BATCH_SIZE)
@pytest.mark.parametrize("expert_num", EXPERT_NUM)
@pytest.mark.parametrize("hidden_size", HIDDEN_DIM)
@pytest.mark.parametrize("intermediate_size", INTERMEDIATE_DIM)
@pytest.mark.parametrize("use_bias", USE_BIAS)
@pytest.mark.parametrize("dtype", DTYPE)
@pytest.mark.parametrize("act", ACT)
@pytest.mark.parametrize("isa", ISA)
def test_cpu_fused_moe(
default_vllm_config,
batch_size: int,
expert_num: int,
hidden_size: int,
intermediate_size: int,
use_bias: bool,
dtype: torch.dtype,
act: MoEActivation,
isa: str,
):
set_random_seed(0)
topk_num = max(expert_num // 2, 1)
up_dim = 2 * intermediate_size
input = torch.randn((batch_size, hidden_size), dtype=dtype) / (
0.5 * hidden_size**0.5
)
w13 = torch.randn((expert_num, up_dim, hidden_size), dtype=dtype) / (
0.5 * hidden_size**0.5
)
w2 = torch.randn((expert_num, hidden_size, intermediate_size), dtype=dtype) / (
0.5 * intermediate_size**0.5
)
router_logits = torch.randn((batch_size, expert_num), dtype=dtype)
w13_bias = None
w2_bias = None
if use_bias:
w13_bias = torch.randn((expert_num, up_dim), dtype=dtype) / (0.5 * up_dim**0.5)
w2_bias = torch.randn((expert_num, hidden_size), dtype=dtype) / (
0.5 * hidden_size**0.5
)
score = torch.softmax(router_logits, dim=-1, dtype=torch.float32)
topk_weight, topk_ids = torch.topk(score, topk_num)
topk_ids = topk_ids.to(torch.int32)
ref_output = ref_fused_moe(
input,
w13,
w2,
w13_bias,
w2_bias,
topk_weight,
topk_ids,
act,
)
packed_w13 = cpu_prepack_moe_weight(w13, isa)
packed_w2 = cpu_prepack_moe_weight(w2, isa)
output = torch.empty_like(input)
cpu_fused_moe(
output,
input,
packed_w13,
packed_w2,
w13_bias,
w2_bias,
topk_weight,
topk_ids,
act.value,
isa,
)
atol, rtol = get_default_atol(output), get_default_rtol(output)
(
torch.testing.assert_close(output, ref_output, atol=atol, rtol=rtol),
f"{torch.max(torch.abs(output - ref_output))}",
)
@pytest.mark.skipif(
current_platform.get_cpu_architecture()
not in (CpuArchEnum.ARM, CpuArchEnum.POWERPC),
reason="Requires Arm or POWER CPU",
)
@pytest.mark.parametrize("batch_size", BATCH_SIZE)
@pytest.mark.parametrize("expert_num", EXPERT_NUM)
@pytest.mark.parametrize("hidden_size", HIDDEN_DIM)
@pytest.mark.parametrize("intermediate_size", INTERMEDIATE_DIM)
@pytest.mark.parametrize("use_bias", USE_BIAS)
@pytest.mark.parametrize(
"dtype",
(
[torch.float32, torch.float16, torch.bfloat16]
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM
else [torch.float32, torch.bfloat16]
),
)
@pytest.mark.parametrize("act", ACT)
@pytest.mark.parametrize(
"isa",
["neon"] if current_platform.get_cpu_architecture() == CpuArchEnum.ARM else ["vsx"],
)
def test_cpu_fused_moe_int8(
batch_size: int,
expert_num: int,
hidden_size: int,
intermediate_size: int,
use_bias: bool,
dtype: torch.dtype,
act: MoEActivation,
isa: str,
):
set_random_seed(0)
topk_num = max(expert_num // 2, 1)
up_dim = 2 * intermediate_size
input = torch.randn((batch_size, hidden_size), dtype=dtype) / (
0.5 * hidden_size**0.5
)
w13_fp = torch.randn((expert_num, up_dim, hidden_size), dtype=dtype) / (
0.5 * hidden_size**0.5
)
w2_fp = torch.randn((expert_num, hidden_size, intermediate_size), dtype=dtype) / (
0.5 * intermediate_size**0.5
)
w13, w13_scale = quantize_per_channel(w13_fp)
w2, w2_scale = quantize_per_channel(w2_fp)
w13_bias = None
w2_bias = None
if use_bias:
w13_bias = torch.randn((expert_num, up_dim), dtype=dtype) / (0.5 * up_dim**0.5)
w2_bias = torch.randn((expert_num, hidden_size), dtype=dtype) / (
0.5 * hidden_size**0.5
)
router_logits = torch.randn((batch_size, expert_num), dtype=dtype)
score = torch.softmax(router_logits, dim=-1, dtype=torch.float32)
topk_weights, topk_ids = torch.topk(score, topk_num)
topk_ids = topk_ids.to(torch.int32)
ref_output = ref_fused_moe_int8(
input,
w13,
w2,
w13_scale,
w2_scale,
w13_bias,
w2_bias,
topk_weights,
topk_ids,
act,
)
packed_w13 = cpu_prepack_moe_weight_int8(w13, isa)
packed_w2 = cpu_prepack_moe_weight_int8(w2, isa)
output = torch.empty_like(input)
cpu_fused_moe_int8(
output,
input,
packed_w13,
packed_w2,
w13_scale,
w2_scale,
w13_bias,
w2_bias,
topk_weights,
topk_ids,
act.value,
isa,
)
atol = rtol = 1e-1 if isa == "vsx" else 2e-2
torch.testing.assert_close(output, ref_output, atol=atol, rtol=rtol)
# moe_intermediate_size not a multiple of 32, e.g. what tensor-parallel
# sharding of moe_intermediate_size=704 at tp=4 produces (704 // 4 == 176).
UNALIGNED_INTERMEDIATE_DIM = 176
class _StubMoELayer(torch.nn.Module):
"""Minimal stand-in for the real MoE layer module, exposing just what
the unquantized CPU experts read/replace (w13_weight, w2_weight, the
router configuration, and optionally w13_bias/w2_bias)."""
def __init__(
self,
w13_weight: torch.Tensor,
w2_weight: torch.Tensor,
activation: MoEActivation,
w13_bias: torch.Tensor | None = None,
w2_bias: torch.Tensor | None = None,
):
super().__init__()
self.w13_weight = torch.nn.Parameter(w13_weight, requires_grad=False)
self.w2_weight = torch.nn.Parameter(w2_weight, requires_grad=False)
self.activation = activation
if w13_bias is not None:
self.w13_bias = torch.nn.Parameter(w13_bias, requires_grad=False)
if w2_bias is not None:
self.w2_bias = torch.nn.Parameter(w2_bias, requires_grad=False)
self.use_grouped_topk = False
self.renormalize = False
self.scoring_func = "softmax"
self.custom_routing_function = None
def _make_moe_config(
expert_num: int,
hidden_size: int,
intermediate_size: int,
topk_num: int,
dtype: torch.dtype,
act: MoEActivation,
has_bias: bool = False,
) -> FusedMoEConfig:
return FusedMoEConfig(
num_experts=expert_num,
experts_per_token=topk_num,
hidden_dim=hidden_size,
intermediate_size=intermediate_size,
num_local_experts=expert_num,
num_logical_experts=expert_num,
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
activation=act,
in_dtype=dtype,
device="cpu",
routing_method=RoutingMethodType.Default,
has_bias=has_bias,
)
@pytest.mark.parametrize(
("hidden_size", "intermediate_size", "act", "expected"),
[
(128, 128, MoEActivation.SILU, True),
# Hidden size is not 32-aligned.
(112, 128, MoEActivation.SILU, False),
(128, 176, MoEActivation.SILU, True),
# SwiGLUOAI's interleaved gate/up layout cannot be padded.
(128, 176, MoEActivation.SWIGLUOAI, False),
],
)
def test_cpu_vec_fused_moe_shape_support(
hidden_size: int,
intermediate_size: int,
act: MoEActivation,
expected: bool,
):
moe_config = _make_moe_config(
expert_num=8,
hidden_size=hidden_size,
intermediate_size=intermediate_size,
topk_num=4,
dtype=torch.bfloat16,
act=act,
)
supported, _ = CPUUnquantizedExperts.is_supported_config(
CPUUnquantizedExperts,
moe_config,
None,
None,
mk.FusedMoEActivationFormat.Standard,
)
assert supported is expected
@pytest.mark.parametrize("expert_num", EXPERT_NUM)
@pytest.mark.parametrize("hidden_size", HIDDEN_DIM)
@pytest.mark.parametrize("use_bias", USE_BIAS)
@pytest.mark.parametrize("dtype", DTYPE)
@pytest.mark.parametrize("isa", ISA)
@pytest.mark.parametrize(
"act",
[MoEActivation.SILU, MoEActivation.GELU, MoEActivation.GELU_TANH],
)
def test_cpu_fused_moe_unaligned_intermediate_size(
default_vllm_config,
expert_num: int,
hidden_size: int,
use_bias: bool,
dtype: torch.dtype,
isa: str,
act: MoEActivation,
):
"""CPU kernels handle unaligned intermediate sizes by zero-padding the
weights before prepacking."""
set_random_seed(0)
batch_size = 64
intermediate_size = UNALIGNED_INTERMEDIATE_DIM
topk_num = max(expert_num // 2, 1)
up_dim = 2 * intermediate_size
input = torch.randn((batch_size, hidden_size), dtype=dtype) / (
0.5 * hidden_size**0.5
)
w13 = torch.randn((expert_num, up_dim, hidden_size), dtype=dtype) / (
0.5 * hidden_size**0.5
)
w2 = torch.randn((expert_num, hidden_size, intermediate_size), dtype=dtype) / (
0.5 * intermediate_size**0.5
)
router_logits = torch.randn((batch_size, expert_num), dtype=dtype)
w13_bias = None
w2_bias = None
if use_bias:
w13_bias = torch.randn((expert_num, up_dim), dtype=dtype) / (0.5 * up_dim**0.5)
w2_bias = torch.randn((expert_num, hidden_size), dtype=dtype) / (
0.5 * hidden_size**0.5
)
# Route with the same helper the router uses internally, so the
# reference only differs from the kernel in how the experts are
# evaluated.
topk_weight, topk_ids = _softmax_topk(router_logits, topk_num, False)
ref_output = ref_fused_moe(
input, w13, w2, w13_bias, w2_bias, topk_weight, topk_ids, act
)
layer = _StubMoELayer(
w13.clone(),
w2.clone(),
act,
w13_bias.clone() if w13_bias is not None else None,
w2_bias.clone() if w2_bias is not None else None,
)
moe_config = _make_moe_config(
expert_num, hidden_size, intermediate_size, topk_num, dtype, act, use_bias
)
experts_cls = {
"vec": CPUUnquantizedExperts,
"amx": X86CPUUnquantizedExperts,
"neon": ArmCPUUnquantizedExperts,
"vsx": PowerCPUUnquantizedExperts,
}[isa]
supported, reason = experts_cls.is_supported_config(
experts_cls,
moe_config,
None,
None,
mk.FusedMoEActivationFormat.Standard,
)
assert supported, (
f"expected the padded intermediate size to be supported, got {reason}"
)
# Mirror UnquantizedFusedMoEMethod._setup_kernel: the quant config is
# built from the layer first, then the experts shuffle the layer into the
# kernel's runtime format.
quant_config = (
biased_moe_quant_config(layer.w13_bias, layer.w2_bias)
if use_bias
else FusedMoEQuantConfig.make()
)
experts = experts_cls(moe_config, quant_config)
assert experts.isa == isa
experts.process_weights_after_loading(layer)
output = torch.empty_like(input)
experts.apply(
output=output,
hidden_states=input,
w1=layer.w13_weight,
w2=layer.w2_weight,
topk_weights=topk_weight,
topk_ids=topk_ids,
activation=act,
global_num_experts=expert_num,
expert_map=None,
a1q_scale=None,
a2_scale=None,
workspace13=torch.empty(0),
workspace2=torch.empty(0),
expert_tokens_meta=None,
apply_router_weight_on_input=False,
)
atol, rtol = get_default_atol(output), get_default_rtol(output)
torch.testing.assert_close(output, ref_output, atol=atol, rtol=rtol)
-227
View File
@@ -1,227 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for CPU INT4 W4A8 dynamic quantized fused MoE kernel (CPUExpertsInt4)."""
import sys
import pytest
import torch
import torch.nn.functional as F
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.experts.cpu_int4_moe import (
CPUExpertsInt4,
)
from vllm.model_executor.layers.fused_moe.oracle.w4a8_int8 import (
convert_to_w4a8_int8_moe_format,
)
from vllm.platforms import CpuArchEnum, current_platform
from vllm.utils.torch_utils import set_random_seed
if (
not current_platform.is_cpu()
or current_platform.get_cpu_architecture() != CpuArchEnum.ARM
):
pytest.skip("skipping Arm CPU-only tests", allow_module_level=True)
# Tolerance for INT4 W4A8
INT4_W4A8_ATOL = 2e-2
INT4_W4A8_RTOL = 2e-2
def _silu_and_mul(x: torch.Tensor) -> torch.Tensor:
"""SwiGLU activation: SiLU(gate) * up."""
d = x.shape[-1] // 2
return F.silu(x[..., :d]) * x[..., d:]
def _make_int4_moe_weights(
E: int,
N: int,
K: int,
group_size: int,
has_bias: bool = False,
) -> tuple[
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor | None,
torch.Tensor | None,
]:
"""Generate random INT4 MoE weights with random scales.
Args:
E: Number of experts
N: Intermediate size
K: Hidden size
group_size: Quantization group size (-1 for channel-wise)
has_bias: Whether to include bias
Returns:
(w13_packed, w2_packed, w13_ref, w2_ref, w13_bias, w2_bias)
where *_ref are the dequantized float reference weights
"""
# Generate INT4 weights as int8 values in [-8, 7]
w13_int4 = torch.randint(-8, 8, (E, 2 * N, K), dtype=torch.int8)
w2_int4 = torch.randint(-8, 8, (E, K, N), dtype=torch.int8)
# Determine number of scale columns
def _n_scale_cols(in_features: int) -> int:
return 1 if group_size == -1 else (in_features // group_size)
# Generate random scales
scale_dtype = torch.float32 if group_size == -1 else torch.bfloat16
w13_scales = torch.rand(E, 2 * N, _n_scale_cols(K), dtype=scale_dtype) * 0.01
w2_scales = torch.rand(E, K, _n_scale_cols(N), dtype=scale_dtype) * 0.01
# Generate biases if needed
w13_bias = None
w2_bias = None
if has_bias:
w13_bias = torch.randn(E, 2 * N, dtype=torch.float32) * 0.01
w2_bias = torch.randn(E, K, dtype=torch.float32) * 0.01
w13_packed, w2_packed, *_ = convert_to_w4a8_int8_moe_format(
w13_weight=w13_int4,
w2_weight=w2_int4,
w13_weight_scale=w13_scales,
w2_weight_scale=w2_scales,
group_size=group_size,
w13_bias=w13_bias if has_bias else None,
w2_bias=w2_bias if has_bias else None,
)
if group_size == -1:
w13_scale = w13_scales.float()
w2_scale = w2_scales.float()
else:
w13_scale = w13_scales.float().repeat_interleave(group_size, dim=-1)
w2_scale = w2_scales.float().repeat_interleave(group_size, dim=-1)
w13_ref = w13_int4.float() * w13_scale
w2_ref = w2_int4.float() * w2_scale
if has_bias and w13_bias is not None:
w13_ref = w13_ref + w13_bias.float().unsqueeze(-1)
if has_bias and w2_bias is not None:
w2_ref = w2_ref + w2_bias.float().unsqueeze(-1)
return w13_packed, w2_packed, w13_ref, w2_ref, w13_bias, w2_bias
def ref_int4_moe(
a: torch.Tensor,
w13_ref: torch.Tensor,
w2_ref: torch.Tensor,
topk_weight: torch.Tensor,
topk_ids: torch.Tensor,
) -> torch.Tensor:
"""Reference INT4 W4A8 fused MoE using dequantized weights.
Steps:
1. Use dequantized float weights
2. For each expert: matmul → SwiGLU → matmul
3. Weighted sum across top-k experts
"""
B, D = a.shape
topk = topk_ids.size(1)
a_exp = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D).float()
out = torch.zeros(B * topk, w2_ref.shape[1], dtype=torch.float32)
topk_weight_flat = topk_weight.view(-1)
topk_ids_flat = topk_ids.view(-1)
for i in range(w13_ref.shape[0]):
mask = topk_ids_flat == i
if mask.sum():
# w13: [2N, K], input: [B, K] -> output: [B, 2N]
gate_up = torch.matmul(a_exp[mask], w13_ref[i].transpose(0, 1))
# SwiGLU activation
hidden = _silu_and_mul(gate_up)
# w2: [K, N], hidden: [B, N] -> output: [B, K]
out[mask] = torch.matmul(hidden, w2_ref[i].transpose(0, 1))
return (
(out.view(B, -1, w2_ref.shape[1]) * topk_weight_flat.view(B, -1, 1))
.sum(dim=1)
.to(a.dtype)
)
NUM_TOKENS = [1, 2, 64, 128]
# (intermediate_size N, hidden_size K, num_experts E, topk, group_size)
MoE_CONFIGS = [
(256, 512, 8, 2, 128),
(256, 512, 8, 2, 64),
(256, 512, 8, 2, -1), # channel-wise
(512, 256, 8, 4, 128),
(512, 512, 8, 2, 128),
(768, 2048, 8, 2, 128),
(768, 2048, 16, 4, 64),
]
SEEDS = [0, 42]
ACTIVATION_DTYPES = [torch.float32, torch.bfloat16, torch.float16]
@pytest.mark.parametrize("M", NUM_TOKENS)
@pytest.mark.parametrize("N,K,E,topk,group_size", MoE_CONFIGS)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("activation_dtype", ACTIVATION_DTYPES)
def test_cpu_int4_moe_kernel(M, N, K, E, topk, group_size, seed, activation_dtype):
"""Test dynamic_4bit_int_moe kernel against dequantized torch reference."""
set_random_seed(seed)
activation = MoEActivation.SILU
# Generate input activations
a = torch.randn(M, K, dtype=activation_dtype) / (K**0.5)
# Generate INT4 weights
w13_packed, w2_packed, w13_ref, w2_ref, w13_bias, w2_bias = _make_int4_moe_weights(
E, N, K, group_size, has_bias=False
)
# Generate router logits and topk
score = torch.randn(M, E, dtype=torch.bfloat16)
score = torch.softmax(score, dim=-1, dtype=torch.float32)
topk_weight, topk_ids = torch.topk(score, topk)
topk_ids = topk_ids.to(torch.long)
# Reference output using dequantized weights
ref_out = ref_int4_moe(
a,
w13_ref,
w2_ref,
topk_weight,
topk_ids,
)
# Test dynamic_4bit_int_moe kernel
apply_router_weight_on_input = False
out = torch.ops._C.dynamic_4bit_int_moe(
a,
topk_ids,
topk_weight,
w13_packed,
w2_packed,
K, # H (hidden_size / w2_out_features)
N, # I (intermediate_size / w2_in_features)
group_size,
apply_router_weight_on_input,
CPUExpertsInt4._activation_kind(activation),
)
assert out.dtype == activation_dtype
torch.testing.assert_close(
ref_out,
out,
atol=INT4_W4A8_ATOL,
rtol=INT4_W4A8_RTOL,
)
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-v"]))
File diff suppressed because it is too large Load Diff
-586
View File
@@ -1,586 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.platforms import current_platform
if not current_platform.is_device_capability_family(100):
pytest.skip(
reason="Nvfp4 Requires compute capability of 10 or above.",
allow_module_level=True,
)
import torch
from flashinfer import fp4_quantize
from torch.nn import functional as F
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutedsl_batched_moe import ( # noqa: E501
flashinfer_cutedsl_moe_masked,
)
from vllm.utils.flashinfer import (
flashinfer_cutedsl_grouped_gemm_nt_masked as cutedsl_gmm_masked,
)
from vllm.utils.flashinfer import (
scaled_fp4_grouped_quantize,
)
kE2M1ToFloat = torch.tensor(
[0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0], dtype=torch.float32
)
FLOAT8_E4M3_MAX = 448.0
FLOAT4_E2M1_MAX = 6.0
def convert_swizzled_to_linear(a_sf_swizzled: torch.Tensor, m, k, block_size):
m_tiles = (m + 128 - 1) // 128
f = block_size * 4
k_tiles = (k + f - 1) // f
tmp = torch.reshape(a_sf_swizzled, (1, m_tiles, k_tiles, 32, 4, 4))
tmp = torch.permute(tmp, (0, 1, 4, 3, 2, 5))
out = tmp.reshape(m_tiles * 128, k_tiles * f // block_size)
return out[0:m, 0:k]
def dequantize_nvfp4_to_dtype(
tensor_fp4, tensor_sf, global_scale, dtype, device, block_size=16
):
"""Dequantize the fp4 tensor back to high precision."""
# Two fp4 values are packed into one uint8.
assert tensor_fp4.dtype == torch.uint8
m, packed_k = tensor_fp4.shape
k = packed_k * 2
tensor_f32 = break_fp4_bytes(tensor_fp4, dtype)
tensor_f32 = tensor_f32.reshape(m, k // block_size, block_size)
tensor_sf = tensor_sf.view(torch.float8_e4m3fn)
tensor_sf = convert_swizzled_to_linear(tensor_sf, m, k, block_size)
tensor_sf_dtype = tensor_sf.to(torch.float32) / global_scale
# scale the tensor
out = (tensor_f32 * tensor_sf_dtype.unsqueeze(-1)).reshape(m, k)
return out.to(dtype=dtype)
def break_fp4_bytes(a, dtype):
assert a.dtype == torch.uint8
m, n = a.shape
# Vectorized nibble processing
a_flat = a.flatten()
high = (a_flat & 0xF0) >> 4 # Upper nibbles
low = a_flat & 0x0F # Lower nibbles
# Combine nibbles for batch processing
combined = torch.stack((low, high), dim=1).flatten()
# Vectorized sign and magnitude extraction
signs = (combined & 0x08).to(torch.bool) # Sign bits
abs_vals = (combined & 0x07).to(torch.long) # Magnitude indices
# Device-aware lookup and sign application
kE2M1 = kE2M1ToFloat.to(device=a.device)
values = kE2M1[abs_vals] * torch.where(signs, -1.0, 1.0)
# Reshape to final form
return values.reshape(m, n * 2).to(dtype=dtype)
def generate_balanced_routing(
hidden_states: torch.Tensor, num_experts: int, top_k: int
):
"""Generate routing weights and topk indices such that every expert is active.
Returns routing_weights, topk_idx
"""
num_tokens, hidden_dim = hidden_states.shape
# num_tokens = batch_size * seq_len
# First, assign at least one token per expert
tokens_per_expert = torch.arange(num_tokens) % num_experts
tokens_per_expert = tokens_per_expert[torch.randperm(num_tokens)] # shuffle
# Each token has top_k experts — start with one guaranteed expert
topk_idx = torch.full((num_tokens, top_k), -1, dtype=torch.long)
topk_idx[:, 0] = tokens_per_expert
# For remaining top_k - 1 experts, pick randomly (allowing repeats)
if top_k > 1:
random_choices = torch.randint(0, num_experts, (num_tokens, top_k - 1))
topk_idx[:, 1:] = random_choices
# Normalize routing weights so each token's weights sum to 1
routing_weights = torch.rand(num_tokens, top_k)
routing_weights /= routing_weights.sum(dim=-1, keepdim=True)
# Reshape back if needed
routing_weights = routing_weights.view(num_tokens, top_k)
topk_idx = topk_idx.view(num_tokens, top_k)
return routing_weights, topk_idx
def prepare_inputs(
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
num_experts: int,
topk: int,
):
routing_weights, topk_idx = generate_balanced_routing(
router_logits, num_experts, topk
)
masked_m = []
for i in range(num_experts):
mask = topk_idx.view(-1) == i
masked_m.append(mask.sum())
masked_m = torch.tensor(masked_m, dtype=torch.int32)
# Initialize the hidden_states_3d with ones instead of empty to avoid nan
# issue.
hidden_states_3d = torch.ones(
(num_experts, max(masked_m), hidden_states.shape[1]),
dtype=hidden_states.dtype,
device=hidden_states.device,
)
for i in range(num_experts):
hidden_states_3d[i, : masked_m[i], :] = hidden_states[topk_idx.view(-1) == i]
return hidden_states_3d, masked_m, topk_idx, routing_weights
MNK_FACTORS = [
(2, 1024, 1024),
(2, 1024, 1536),
(2, 3072, 1024),
(2, 3072, 1536),
(64, 1024, 1024),
(64, 1024, 1536),
(64, 3072, 1024),
(64, 2048, 1024),
(224, 1024, 1024),
(224, 1024, 1536),
]
# Reference implementation of torch_moe
def torch_moe(a, w1, w2, score, topk, expert_map):
B, D = a.shape
a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device)
score = torch.softmax(score, dim=-1, dtype=torch.float32)
topk_weight, topk_ids = torch.topk(score, topk)
topk_weight = topk_weight.view(-1)
topk_ids = topk_ids.view(-1)
if expert_map is not None:
topk_ids = expert_map[topk_ids]
for i in range(w1.shape[0]):
mask = topk_ids == i
if mask.sum():
out[mask] = SiluAndMul()(a[mask] @ w1[i].transpose(0, 1)) @ w2[i].transpose(
0, 1
)
return (
out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype)
).sum(dim=1)
def torch_moe_nvfp4(a, w1, w2, topk, topk_weight, topk_ids):
B, D = a.shape
a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device)
topk_weight = topk_weight.view(-1)
topk_ids = topk_ids.view(-1)
for i in range(w1.shape[0]):
mask = topk_ids == i
if mask.sum():
m = w1[i].shape[0]
assert m % 2 == 0
# Note: w1 and w3 are swapped!
w3_expert, w1_expert = w1[i][m // 2 :, :], w1[i][: m // 2, :]
inter = F.silu(a[mask] @ w1_expert.t()) * (a[mask] @ w3_expert.t())
inter_gs = torch.tensor(1.0).cuda()
inter_q, inter_blockscale = fp4_quantize(inter, inter_gs)
inter = dequantize_nvfp4_to_dtype(
inter_q,
inter_blockscale,
inter_gs,
dtype=inter.dtype,
device=inter.device,
block_size=16,
).cuda()
out[mask] = inter @ w2[i].transpose(0, 1)
return (
out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype)
).sum(dim=1)
def grouped_gemm_ref(
hidden_states_expanded: torch.Tensor,
hidden_states_3d: torch.Tensor,
weights: torch.Tensor,
topk_idx: torch.Tensor,
masked_m: torch.Tensor,
B: int,
topk: int,
num_experts: int,
*,
block_size: int = 16,
) -> torch.Tensor:
"""Computes the reference grouped GEMM (fp4 quantized per-expert loop),
computes flashinfer grouped GEMM (for scale consistency),
and returns ONLY the repacked reference output: out_ref.
Returns:
out_ref: Tensor [num_experts, max_m, n_out]
"""
device_hs = hidden_states_expanded.device
device_w = weights.device
out_dtype = weights.dtype
n_out = weights.shape[1]
# Flattened reference output (B*topk, n_out)
out = torch.zeros((B * topk, n_out), dtype=out_dtype, device=device_w)
# Per-expert reference compute loop
for i in range(num_experts):
mask = topk_idx.view(-1) == i
if mask.any():
lhs = hidden_states_expanded[mask]
rhs = weights[i]
a_amax = lhs.abs().max().to(torch.float32).to(device_hs)
b_amax = rhs.abs().max().to(torch.float32).to(device_w)
a_gs = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / a_amax
b_gs = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / b_amax
lhsq, lhsq_sf = fp4_quantize(lhs, a_gs)
rhsq, rhsq_sf = fp4_quantize(rhs, b_gs)
lhs_in_dtype = dequantize_nvfp4_to_dtype(
lhsq,
lhsq_sf,
a_gs,
dtype=lhs.dtype,
device=device_hs,
block_size=block_size,
)
rhs_in_dtype = dequantize_nvfp4_to_dtype(
rhsq,
rhsq_sf,
b_gs,
dtype=rhs.dtype,
device=device_w,
block_size=block_size,
)
out[mask] = lhs_in_dtype @ rhs_in_dtype.t()
# Determine per-expert max_m
max_m_val = int(masked_m.max().item())
# Repack into [num_experts, max_m, n_out]
out_ref = torch.zeros(
(num_experts, max_m_val, n_out),
dtype=out.dtype,
device=out.device,
)
expert_slot = [0] * num_experts
for i, expert_id in enumerate(topk_idx.view(-1).tolist()):
slot = expert_slot[expert_id]
if slot < max_m_val:
out_ref[expert_id, slot, :] = out[i]
expert_slot[expert_id] += 1
else:
raise IndexError(
f"Expert {expert_id} exceeded max slots ({max_m_val}). "
"Increase max_m or check masked_m."
)
return out_ref
def flashinfer_cutedsl_grouped_gemm_nt_masked(
hidden_states: torch.Tensor, # 3d
input_global_scale: torch.Tensor, # (l,)
weights: torch.Tensor,
w_global_scale: torch.Tensor, # (l,)
masked_m: torch.Tensor,
):
# hidden_states: [l, m, k]
# weights: [l, n, k]
aq, aq_sf = scaled_fp4_grouped_quantize(
hidden_states,
masked_m.to(hidden_states.device),
input_global_scale,
)
num_experts, n, k = weights.shape
bq, bq_sf = scaled_fp4_grouped_quantize(
weights,
torch.full((num_experts,), n, device=weights.device, dtype=torch.int32),
w_global_scale,
)
out = torch.zeros(
(num_experts, max(masked_m), n), dtype=weights.dtype, device=aq.device
)
out = out.permute(1, 2, 0) # requirement of kernel
sf_vec_size = 16
ab_dtype = "float4_e2m1fn"
sf_dtype = "float8_e4m3fn"
c_dtype = "bfloat16"
alpha = 1.0 / (input_global_scale * w_global_scale).to(out.dtype).view(
1, 1, num_experts
)
def get_cute_dtype(input: torch.Tensor) -> str:
if input.dtype == torch.bfloat16:
return "bfloat16"
elif input.dtype == torch.float16:
return "float16"
elif input.dtype == torch.float32:
return "float32"
else:
raise ValueError(f"Unsupported cute dtype {input.dtype}")
cutedsl_gmm_masked(
(aq, aq_sf),
(bq, bq_sf),
out,
masked_m.to(aq.device),
ab_dtype=ab_dtype,
sf_dtype=sf_dtype,
c_dtype=c_dtype,
sf_vec_size=sf_vec_size,
alpha=alpha,
alpha_dtype=get_cute_dtype(alpha),
)
return out
@pytest.mark.parametrize("bs, hidden_dim, inter_dim", [(2, 128, 256), (16, 128, 512)])
@pytest.mark.parametrize("topk", [1, 2, 4])
@torch.inference_mode()
def test_flashinfer_cutedsl_moe_masked(
bs: int, hidden_dim: int, inter_dim: int, topk: int
):
torch.manual_seed(42)
device = "cuda"
num_experts = 8
hidden_states = (
torch.randn(bs, hidden_dim, dtype=torch.bfloat16, device=device) / 5.0
)
w1 = (
torch.randn(
num_experts, 2 * inter_dim, hidden_dim, dtype=torch.bfloat16, device=device
)
/ 10.0
)
w2 = (
torch.randn(
num_experts, hidden_dim, inter_dim, dtype=torch.bfloat16, device=device
)
/ 10.0
)
router_logits = torch.randn(bs, num_experts, dtype=torch.float32)
hidden_states_expanded = (
hidden_states.view(bs, -1, hidden_dim)
.repeat(1, topk, 1)
.reshape(-1, hidden_dim)
)
hidden_states_3d, masked_m, topk_idx, routing_weights = prepare_inputs(
hidden_states_expanded, router_logits, num_experts, topk
)
w1_amax = w1.abs().amax(dim=(1, 2)).to(torch.float32).to(w1.device)
w2_amax = w2.abs().amax(dim=(1, 2)).to(torch.float32).to(w2.device)
input_global_scale = torch.ones(
(num_experts,), dtype=torch.float32, device=hidden_states.device
)
w1_global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / w1_amax
w2_global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / w2_amax
a2_global_scale = torch.ones(
(num_experts,), dtype=torch.float32, device=hidden_states.device
) # assume intermediate scale is 1.0
w1_fp4, w1_blockscale = scaled_fp4_grouped_quantize(
w1,
torch.ones(num_experts, dtype=torch.int32, device=w1.device) * 2 * inter_dim,
w1_global_scale,
)
w2_fp4, w2_blockscale = scaled_fp4_grouped_quantize(
w2,
torch.ones(num_experts, dtype=torch.int32, device=w2.device) * hidden_dim,
w2_global_scale,
)
w1_alpha = 1.0 / (input_global_scale * w1_global_scale)
w2_alpha = 1.0 / (a2_global_scale * w2_global_scale)
out = torch.empty_like(hidden_states_3d, device=hidden_states.device)
# Note: the 1st dim shouldn't be bs
wk = torch.empty(
num_experts,
hidden_states_3d.shape[1],
inter_dim * 2,
dtype=hidden_states_3d.dtype,
device=hidden_states.device,
)
flashinfer_cutedsl_moe_masked(
hidden_states_3d.to(hidden_states.device),
input_global_scale,
w1_fp4.permute(2, 0, 1),
w1_blockscale,
w1_alpha,
w2_fp4.permute(2, 0, 1),
a2_global_scale,
w2_blockscale,
w2_alpha,
masked_m.to(hidden_states.device),
wk,
out,
)
# reference
# input_global_scale is per-expert ([num_experts]); fp4_quantize and
# dequantize_nvfp4_to_dtype are non-grouped APIs that expect [1] or
# [num_tokens]. Use a single element since all values are uniform here.
a_global = input_global_scale[:1].contiguous()
a_fp4, a_scale_interleaved = fp4_quantize(hidden_states, a_global)
a_in_dtype = dequantize_nvfp4_to_dtype(
a_fp4,
a_scale_interleaved,
a_global,
dtype=hidden_states.dtype,
device=hidden_states.device,
block_size=16,
)
w1_d = torch.empty(
(num_experts, 2 * inter_dim, hidden_dim), device=w1.device, dtype=w1.dtype
)
w2_d = torch.empty(
(num_experts, hidden_dim, inter_dim), device=w2.device, dtype=w2.dtype
)
for idx in range(0, num_experts):
w1_fp4_sliced, w1_blockscale_sliced = fp4_quantize(
w1[idx], w1_global_scale[idx]
)
w2_fp4_sliced, w2_blockscale_sliced = fp4_quantize(
w2[idx], w2_global_scale[idx]
)
w1_d[idx] = dequantize_nvfp4_to_dtype(
w1_fp4_sliced,
w1_blockscale_sliced,
w1_global_scale[idx],
dtype=w1.dtype,
device=w1.device,
block_size=16,
)
w2_d[idx] = dequantize_nvfp4_to_dtype(
w2_fp4_sliced,
w2_blockscale_sliced,
w2_global_scale[idx],
dtype=w2.dtype,
device=w2.device,
block_size=16,
)
ref_output = torch_moe_nvfp4(
a_in_dtype,
w1_d,
w2_d,
topk,
routing_weights.to(a_in_dtype.device),
topk_idx.to(a_in_dtype.device),
)
out_weighted = torch.zeros_like(ref_output, device=out.device, dtype=out.dtype)
positions = torch.nonzero(masked_m[topk_idx], as_tuple=False)
rows, cols = positions[:, 0], positions[:, 1]
experts = topk_idx[rows, cols]
for i in range(num_experts):
mask = experts == i
if mask.any():
idx = torch.nonzero(mask, as_tuple=False).squeeze(-1)
r, c = rows[idx], cols[idx]
out_weighted[r] += out[i, : len(r), :] * routing_weights[r, c].to(
out.device
).unsqueeze(-1)
torch.testing.assert_close(
out_weighted.cpu(), ref_output.cpu(), atol=2e-1, rtol=2e-1
)
@pytest.mark.parametrize(
"bs, hidden_dim, inter_dim, topk", [(2, 128, 256, 2), (16, 128, 512, 5)]
)
@torch.inference_mode()
def test_grouped_gemm_nt_masked(
bs: int, hidden_dim: int, inter_dim: int, topk: int
) -> None:
torch.manual_seed(42)
B = bs
D = hidden_dim
N = inter_dim
# CuteDSL group gemm has issue when not all experts are active.
# i.e. masked = [2, 3, 0, 0, 1] where the 2nd and 3rd experts are inactive
# see https://github.com/flashinfer-ai/flashinfer/issues/1856
num_experts = bs
hidden_states = torch.randn(B, D, dtype=torch.bfloat16, device="cuda")
weights = torch.randn(num_experts, N, D, dtype=torch.bfloat16, device="cuda")
router_logits = torch.randn(B, num_experts, dtype=torch.float32)
hidden_states_expanded = (
hidden_states.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
)
hidden_states_3d, masked_m, topk_idx, _ = prepare_inputs(
hidden_states_expanded, router_logits, num_experts, topk
)
a_amax = (
hidden_states_3d.abs()
.amax(dim=(1, 2))
.to(torch.float32)
.to(hidden_states.device)
)
b_amax = weights.abs().amax(dim=(1, 2)).to(torch.float32).to(weights.device)
a_gs = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / a_amax
b_gs = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / b_amax
out_flashinfer = flashinfer_cutedsl_grouped_gemm_nt_masked(
hidden_states_3d.to(hidden_states.device), a_gs, weights, b_gs, masked_m
)
# reference
out_ref = grouped_gemm_ref(
hidden_states_expanded=hidden_states_expanded,
hidden_states_3d=hidden_states_3d,
weights=weights,
topk_idx=topk_idx,
masked_m=masked_m,
B=B,
topk=topk,
num_experts=num_experts,
)
# Note: just to compare the masked position due to cutedsl may write nan
# into unmasked position.
for i in range(num_experts):
torch.testing.assert_close(
out_flashinfer.permute(2, 0, 1)[i, : masked_m[i]],
out_ref.to(out_flashinfer.device)[i, : masked_m[i]],
atol=1e-1,
rtol=1e-1,
)
if __name__ == "__main__":
test_flashinfer_cutedsl_moe_masked(16, 128, 512, 4)
test_grouped_gemm_nt_masked(16, 128, 512, 4)
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
-570
View File
@@ -1,570 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Test deepep dispatch-combine logic."""
import dataclasses
import pytest
import torch.distributed
from torch.distributed import ProcessGroup
import vllm.envs as envs
from tests.kernels.moe.utils import make_dummy_moe_config
from vllm import _custom_ops as ops
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.fused_moe import TritonExperts
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
BatchedTritonExperts,
)
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
per_token_group_quant_fp8,
)
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_deep_ep
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.worker.workspace import init_workspace_manager
from ...utils import multi_gpu_test
from .parallel_utils import ProcessGroupInfo, parallel_launch
if has_deep_ep():
from vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_ht import (
DeepEPHTPrepareAndFinalize,
)
from vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_ll import (
DeepEPLLPrepareAndFinalize,
)
from .parallel_utils import DeepEPHTArgs, DeepEPLLArgs, make_deepep_a2a
requires_deep_ep = pytest.mark.skipif(
not has_deep_ep(),
reason="Requires deep_ep kernels",
)
MAX_TOKENS_PER_RANK = 64
def make_weights(
e, n, k, dtype
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Return weights w1, w2, w1_scale, w2_scale."""
if dtype in [torch.float16, torch.bfloat16]:
w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10
w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10
return w1, w2, None, None
# per-out-channel weight quantization
assert dtype == current_platform.fp8_dtype()
# Keep FP8 inputs finite and bounded. torch.empty made this test depend on
# allocator contents, while larger values exceed its fixed W8A8 tolerance.
w1 = torch.randn(
(e, 2 * n, k), device=current_platform.device_type, dtype=torch.float16
).div_(100)
w2 = torch.randn(
(e, k, n), device=current_platform.device_type, dtype=torch.float16
).div_(100)
n_b_scales = 2 * n
k_b_scales = k
w1_q = torch.empty_like(w1, dtype=dtype)
w2_q = torch.empty_like(w2, dtype=dtype)
w1_scale = torch.empty((e, n_b_scales, 1), device="cuda", dtype=torch.float32)
w2_scale = torch.empty((e, k_b_scales, 1), device="cuda", dtype=torch.float32)
for expert in range(e):
w1_q[expert], w1_scale[expert] = ops.scaled_fp8_quant(
w1[expert], use_per_token_if_dynamic=True
)
w2_q[expert], w2_scale[expert] = ops.scaled_fp8_quant(
w2[expert], use_per_token_if_dynamic=True
)
return w1_q, w2_q, w1_scale, w2_scale
@dataclasses.dataclass
class TestConfig:
dtype: torch.dtype
topk: int
m: int
k: int
n: int
num_experts: int
@dataclasses.dataclass
class TestTensors:
rank_tokens: torch.Tensor # all ranks make this many tokens
rank_token_scales: torch.Tensor | None
topk: torch.Tensor
topk_weights: torch.Tensor
config: TestConfig
@staticmethod
def make(config: TestConfig, low_latency_mode: bool) -> "TestTensors":
# TODO (varun) - check that float16 works ?
assert config.dtype in [torch.bfloat16, current_platform.fp8_dtype()]
token_dtype = (
torch.bfloat16
if config.dtype == current_platform.fp8_dtype()
else config.dtype
)
rank_tokens = (
torch.randn((config.m, config.k), device="cuda", dtype=token_dtype) / 10
)
rank_token_scales = None
topk = torch.randint(
low=0, high=config.num_experts, size=(config.m, config.topk), device="cuda"
).to(dtype=torch.int64)
topk_weights = torch.randn(topk.shape, dtype=torch.float32, device="cuda")
return TestTensors(
rank_tokens=rank_tokens,
rank_token_scales=rank_token_scales,
topk=topk,
topk_weights=topk_weights,
config=config,
)
def make_modular_kernel(
pg: ProcessGroup,
pgi: ProcessGroupInfo,
low_latency_mode: bool,
hidden_size: int,
dp_size: int,
num_experts: int,
num_local_experts: int,
q_dtype: torch.dtype | None,
use_fp8_dispatch: bool,
quant_config: FusedMoEQuantConfig,
) -> FusedMoEKernel:
ht_args: DeepEPHTArgs | None = None
ll_args: DeepEPLLArgs | None = None
if low_latency_mode:
ll_args = DeepEPLLArgs(
max_tokens_per_rank=MAX_TOKENS_PER_RANK,
hidden_size=hidden_size,
num_experts=num_experts,
use_fp8_dispatch=use_fp8_dispatch,
)
else:
assert not use_fp8_dispatch, (
"FP8 Dispatch is valid only for low-latency kernels"
)
ht_args = DeepEPHTArgs(num_local_experts=num_local_experts)
a2a: DeepEPHTPrepareAndFinalize | DeepEPLLPrepareAndFinalize = make_deepep_a2a(
pg=pg,
pgi=pgi,
dp_size=dp_size,
q_dtype=q_dtype,
block_shape=None,
deepep_ht_args=ht_args,
deepep_ll_args=ll_args,
)
num_dispatchers = pgi.world_size // dp_size
moe_config = make_dummy_moe_config()
if low_latency_mode:
assert not quant_config.per_act_token_quant, "not supported in ll mode"
fused_experts = BatchedTritonExperts(
max_num_tokens=MAX_TOKENS_PER_RANK,
num_dispatchers=num_dispatchers,
moe_config=moe_config,
quant_config=quant_config,
)
else:
fused_experts = TritonExperts(
moe_config=moe_config,
quant_config=quant_config,
)
mk = FusedMoEKernel(
prepare_finalize=a2a,
fused_experts=fused_experts,
)
return mk
def deep_ep_moe_impl(
pg: ProcessGroup,
pgi: ProcessGroupInfo,
low_latency_mode: bool,
dp_size: int,
test_tensors: TestTensors,
w1: torch.Tensor,
w2: torch.Tensor,
w1_scale: torch.Tensor | None,
w2_scale: torch.Tensor | None,
num_experts: int,
use_fp8_dispatch: bool,
per_act_token_quant: bool,
) -> torch.Tensor:
num_local_experts = w1.size(0)
def build_expert_map():
num_local_experts = w1.size(0)
expert_map = torch.full((num_experts,), fill_value=-1, dtype=torch.int32)
s = pgi.rank * num_local_experts
e = s + num_local_experts
expert_map[s:e] = torch.tensor(list(range(num_local_experts)))
device = torch.accelerator.current_device_index()
return expert_map.to(device=device, dtype=torch.int32)
hidden_size = test_tensors.rank_tokens.size(1)
is_quantized = w1.dtype == current_platform.fp8_dtype()
q_dtype = None
if is_quantized:
q_dtype = current_platform.fp8_dtype()
out_hidden_states = torch.empty_like(test_tensors.rank_tokens)
total_num_tokens = test_tensors.rank_tokens.size(0)
quant_config = FusedMoEQuantConfig.make(
q_dtype,
w1_scale=w1_scale,
w2_scale=w2_scale,
per_act_token_quant=per_act_token_quant,
a1_scale=test_tensors.rank_token_scales,
)
# Build the kernel (and its DeepEP buffer) once and reuse it across chunks.
# Re-creating it per chunk re-inits rocSHMEM, which only allows one
# allocation per process on ROCm. The buffer is sized by max_tokens_per_rank
# so it is valid for every chunk (mirrors production's cached all2all handle).
mk: FusedMoEKernel = make_modular_kernel(
pg,
pgi,
low_latency_mode,
hidden_size,
dp_size,
num_experts,
num_local_experts,
q_dtype,
use_fp8_dispatch,
quant_config,
)
def process_chunk(chunk_start, chunk_end, skip_result_store=False):
rank_tokens_chunk = test_tensors.rank_tokens[chunk_start:chunk_end]
topk_weights_chunk = test_tensors.topk_weights[chunk_start:chunk_end]
topk_chunk = test_tensors.topk[chunk_start:chunk_end]
if low_latency_mode:
# Reusing one buffer leaves it dirty; the low-latency kernels need
# the zero-initialized regions reset before each dispatch.
mk.prepare_finalize.buffer.clean_low_latency_buffer(
MAX_TOKENS_PER_RANK,
hidden_size,
num_experts,
)
out = mk.apply(
hidden_states=rank_tokens_chunk,
w1=w1,
w2=w2,
topk_weights=topk_weights_chunk,
topk_ids=topk_chunk,
activation=MoEActivation.SILU,
global_num_experts=num_experts,
expert_map=build_expert_map(),
apply_router_weight_on_input=False,
)
if not skip_result_store:
out_hidden_states[chunk_start:chunk_end, :].copy_(out, non_blocking=True)
max_num_tokens_per_dp = (
MAX_TOKENS_PER_RANK if low_latency_mode else total_num_tokens
)
for chunk_start_ in range(0, total_num_tokens, max_num_tokens_per_dp):
chunk_start = chunk_start_
chunk_end = min(chunk_start + max_num_tokens_per_dp, total_num_tokens)
# clamp start and end
chunk_start = min(chunk_start, total_num_tokens - 1)
chunk_end = min(chunk_end, total_num_tokens)
process_chunk(
chunk_start, chunk_end, skip_result_store=chunk_start_ >= total_num_tokens
)
return out_hidden_states
def torch_moe_impl(
test_tensors: TestTensors,
w1: torch.Tensor,
w2: torch.Tensor,
w1_scale: torch.Tensor | None,
w2_scale: torch.Tensor | None,
using_fp8_dispatch: bool,
per_act_token_quant: bool,
):
a, topk_ids, topk_weights = (
test_tensors.rank_tokens,
test_tensors.topk,
test_tensors.topk_weights,
)
if using_fp8_dispatch:
# The DeepEP implementation is requested to dispatch using FP8.
# For numerical stability for testing, emulate the fp8 dispatch by
# blockwise quant and de-quant.
assert not per_act_token_quant
a = test_tensors.rank_tokens
aq, aq_scale = per_token_group_quant_fp8(a, 128, use_ue8m0=False)
a = (
(aq.view(-1, 128).to(torch.float32) * aq_scale.view(-1, 1))
.view(a.shape)
.to(a.dtype)
)
is_quantized = w1.dtype == current_platform.fp8_dtype()
a_dtype = a.dtype
if is_quantized:
w1 = w1.to(dtype=torch.float32) * w1_scale
w2 = w2.to(dtype=torch.float32) * w2_scale
a = a.to(dtype=torch.float32)
m, _ = a.shape
topk = topk_ids.size(1)
out = torch.zeros_like(a)
for i in range(m):
a_i = a[i]
o_i = out[i]
for j in range(topk):
e = topk_ids[i][j]
e_w = topk_weights[i][j]
w1_e = w1[e]
w2_e = w2[e]
o_i += (
SiluAndMul()(a_i @ w1_e.transpose(0, 1)) @ w2_e.transpose(0, 1)
) * e_w
if is_quantized:
out = out.to(dtype=a_dtype)
return out
def assert_deepep_close(
expected: torch.Tensor,
actual: torch.Tensor,
k: int,
use_fp8_dispatch: bool,
) -> None:
torch.testing.assert_close(
expected,
actual,
atol=6e-2,
rtol=6e-2,
)
def _deep_ep_moe(
pgi: ProcessGroupInfo,
low_latency_mode: bool,
dp_size: int,
config: TestConfig,
w1: torch.Tensor,
w2: torch.Tensor,
w1_scale: torch.Tensor | None,
w2_scale: torch.Tensor | None,
use_fp8_dispatch: bool,
per_act_token_quant: bool,
):
# Set seed in worker process for deterministic tensor generation.
set_random_seed(7)
device = torch.device(f"cuda:{pgi.local_rank}")
init_workspace_manager(device)
if not low_latency_mode:
assert not use_fp8_dispatch, (
"FP8 dispatch interface is available only in low-latency mode"
)
is_quantized = w1.dtype == current_platform.fp8_dtype()
device_idx = torch.accelerator.current_device_index()
w1 = w1.to(device=device_idx)
w2 = w2.to(device=device_idx)
if is_quantized:
assert w1_scale is not None and w2_scale is not None
w1_scale = w1_scale.to(device=device_idx)
w2_scale = w2_scale.to(device=device_idx)
if envs.VLLM_DISTRIBUTED_USE_SPLIT_GROUP:
pg = torch.distributed.split_group(
split_ranks=[list(range(pgi.world_size))],
group_desc="deepep_test",
)
else:
pg = torch.distributed.new_group(list(range(pgi.world_size)))
test_tensors = TestTensors.make(config, low_latency_mode)
with set_current_vllm_config(VllmConfig()):
# Reference
torch_combined = torch_moe_impl(
test_tensors,
w1,
w2,
w1_scale,
w2_scale,
use_fp8_dispatch,
per_act_token_quant,
)
# Splice experts for this rank.
num_local_experts = config.num_experts // pgi.world_size
e_start = num_local_experts * pgi.rank
e_end = e_start + num_local_experts
w1_ep = w1[e_start:e_end]
w2_ep = w2[e_start:e_end]
w1_scale_ep, w2_scale_ep = None, None
if is_quantized:
w1_scale_ep = w1_scale[e_start:e_end] # type: ignore
w2_scale_ep = w2_scale[e_start:e_end] # type: ignore
deepep_combined = deep_ep_moe_impl(
pg,
pgi,
low_latency_mode,
dp_size,
test_tensors,
w1_ep,
w2_ep,
w1_scale_ep,
w2_scale_ep,
config.num_experts,
use_fp8_dispatch,
per_act_token_quant,
)
assert_deepep_close(torch_combined, deepep_combined, config.k, use_fp8_dispatch)
MNKs = [
(1, 128, 128),
(2, 128, 512),
(3, 1024, 2048),
(32, 128, 1024),
(45, 512, 2048),
(64, 1024, 1024),
(222, 1024, 2048),
]
DTYPES = [torch.bfloat16, current_platform.fp8_dtype()]
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("m,n,k", MNKs)
@pytest.mark.parametrize("num_experts", [32])
@pytest.mark.parametrize("topk", [6])
@pytest.mark.parametrize("world_dp_size", [(2, 1)])
@pytest.mark.parametrize("per_act_token_quant", [False, True])
@multi_gpu_test(num_gpus=2)
@requires_deep_ep
def test_deep_ep_moe(
dtype: torch.dtype,
m: int,
n: int,
k: int,
num_experts: int,
topk: int,
world_dp_size: tuple[int, int],
per_act_token_quant: bool,
workspace_init,
):
low_latency_mode = False
use_fp8_dispatch = False
set_random_seed(7)
world_size, dp_size = world_dp_size
config = TestConfig(dtype=dtype, topk=topk, m=m, k=k, n=n, num_experts=num_experts)
w1, w2, w1_scale, w2_scale = make_weights(num_experts, n, k, dtype)
parallel_launch(
world_size,
_deep_ep_moe,
low_latency_mode,
dp_size,
config,
w1,
w2,
w1_scale,
w2_scale,
use_fp8_dispatch,
per_act_token_quant,
)
MNKs = [
(1, 128, 2560),
(2, 128, 2560),
(3, 1024, 2560),
(32, 128, 2560),
(45, 512, 2560),
(64, 1024, 2560),
(222, 1024, 2560),
]
DTYPES = [current_platform.fp8_dtype(), torch.bfloat16]
USE_FP8_DISPATCH = [True, False]
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("m,n,k", MNKs)
@pytest.mark.parametrize("num_experts", [32])
@pytest.mark.parametrize("topk", [6])
@pytest.mark.parametrize("world_dp_size", [(2, 1)])
@pytest.mark.parametrize("use_fp8_dispatch", USE_FP8_DISPATCH)
@multi_gpu_test(num_gpus=2)
@requires_deep_ep
def test_low_latency_deep_ep_moe(
dtype: torch.dtype,
m: int,
n: int,
k: int,
num_experts: int,
topk: int,
world_dp_size: tuple[int, int],
use_fp8_dispatch: bool,
workspace_init,
):
low_latency_mode = True
if low_latency_mode and k not in DeepEPLLPrepareAndFinalize.SUPPORTED_HIDDEN_SIZES:
pytest.skip(
f"Skipping test as hidden size {k} is not in list of supported "
f"hidden sizes {DeepEPLLPrepareAndFinalize.SUPPORTED_HIDDEN_SIZES}"
)
set_random_seed(7)
world_size, dp_size = world_dp_size
config = TestConfig(dtype=dtype, topk=topk, m=m, k=k, n=n, num_experts=num_experts)
w1, w2, w1_scale, w2_scale = make_weights(num_experts, n, k, dtype)
parallel_launch(
world_size,
_deep_ep_moe,
low_latency_mode,
dp_size,
config,
w1,
w2,
w1_scale,
w2_scale,
use_fp8_dispatch,
False,
)
-655
View File
@@ -1,655 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Test DeepEP v2 (ElasticBuffer) dispatch-combine logic.
Compares against a pure-PyTorch reference MoE implementation.
"""
import dataclasses
import pytest
import torch.distributed
from torch.distributed import ProcessGroup
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from tests.kernels.moe.utils import make_dummy_moe_config, make_test_weights
from tests.kernels.utils import torch_experts
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.fused_moe import TritonExperts
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer
from vllm.utils.import_utils import has_deep_ep_v2
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.worker.workspace import init_workspace_manager
from ...utils import multi_gpu_test
from .parallel_utils import ProcessGroupInfo, parallel_launch
if has_deep_ep_v2():
from .parallel_utils import DeepEPV2Args, make_deepep_v2_a2a
requires_deep_ep_v2 = pytest.mark.skipif(
not has_deep_ep_v2(),
reason="Requires DeepEP v2 (ElasticBuffer)",
)
def assert_fp8_close(actual: torch.Tensor, expected: torch.Tensor) -> None:
close = torch.isclose(actual, expected, atol=2e-1, rtol=2e-1)
close_fraction = close.float().mean().item()
assert close_fraction > 0.99, (
f"Only {close_fraction:.1%} of FP8 outputs are within tolerance"
)
@dataclasses.dataclass
class TestConfig:
dtype: torch.dtype
topk: int
m: int
k: int
n: int
num_experts: int
@dataclasses.dataclass
class TestTensors:
rank_tokens: torch.Tensor
rank_token_scales: torch.Tensor | None
intermediate_scales: torch.Tensor | None
topk: torch.Tensor
topk_weights: torch.Tensor
config: TestConfig
@staticmethod
def make(config: TestConfig) -> "TestTensors":
assert config.dtype in [torch.bfloat16, torch.float8_e4m3fn]
token_dtype = (
torch.bfloat16 if config.dtype == torch.float8_e4m3fn else config.dtype
)
rank_tokens = (
torch.randn((config.m, config.k), device="cuda", dtype=token_dtype) / 10
)
if config.dtype == torch.float8_e4m3fn:
rank_token_scales = torch.tensor(1 / 448, device="cuda")
intermediate_scales = torch.tensor(8 / 448, device="cuda")
else:
rank_token_scales = None
intermediate_scales = None
topk = torch.stack(
[
torch.randperm(config.num_experts, device="cuda")[: config.topk]
for _ in range(config.m)
]
).to(dtype=torch.int64)
topk_weights = torch.randn(topk.shape, dtype=torch.float32, device="cuda")
return TestTensors(
rank_tokens=rank_tokens,
rank_token_scales=rank_token_scales,
intermediate_scales=intermediate_scales,
topk=topk,
topk_weights=topk_weights,
config=config,
)
def make_modular_kernel(
pg: ProcessGroup,
pgi: ProcessGroupInfo,
dp_size: int,
hidden_size: int,
num_experts: int,
num_local_experts: int,
topk: int,
q_dtype: torch.dtype | None,
use_fp8_dispatch: bool,
quant_config: FusedMoEQuantConfig,
use_cudagraph: bool = False,
) -> FusedMoEKernel:
v2_args = DeepEPV2Args(
num_local_experts=num_local_experts,
num_experts=num_experts,
num_topk=topk,
hidden_size=hidden_size,
max_tokens_per_rank=8192,
use_fp8_dispatch=use_fp8_dispatch,
)
a2a = make_deepep_v2_a2a(
pg=pg,
pgi=pgi,
dp_size=dp_size,
v2_args=v2_args,
use_cudagraph=use_cudagraph,
)
moe_config = make_dummy_moe_config(
num_experts=num_local_experts,
experts_per_token=topk,
hidden_dim=hidden_size,
)
fused_experts = TritonExperts(
moe_config=moe_config,
quant_config=quant_config,
)
mk = FusedMoEKernel(
prepare_finalize=a2a,
fused_experts=fused_experts,
)
return mk
def deepep_v2_moe_impl(
pg: ProcessGroup,
pgi: ProcessGroupInfo,
dp_size: int,
test_tensors: TestTensors,
w1: torch.Tensor,
w2: torch.Tensor,
w1_scale: torch.Tensor | None,
w2_scale: torch.Tensor | None,
num_experts: int,
topk: int,
use_fp8_dispatch: bool,
per_act_token_quant: bool,
) -> torch.Tensor:
num_local_experts = w1.size(0)
def build_expert_map():
expert_map = torch.full((num_experts,), fill_value=-1, dtype=torch.int32)
s = pgi.rank * num_local_experts
e = s + num_local_experts
expert_map[s:e] = torch.tensor(list(range(num_local_experts)))
device = torch.accelerator.current_device_index()
return expert_map.to(device=device, dtype=torch.int32)
is_quantized = w1.dtype == torch.float8_e4m3fn
q_dtype = torch.float8_e4m3fn if is_quantized else None
quant_config = FusedMoEQuantConfig.make(
q_dtype,
w1_scale=w1_scale,
w2_scale=w2_scale,
per_act_token_quant=per_act_token_quant,
a1_scale=test_tensors.rank_token_scales,
a2_scale=test_tensors.intermediate_scales,
)
hidden_size = test_tensors.rank_tokens.size(1)
mk: FusedMoEKernel = make_modular_kernel(
pg,
pgi,
dp_size,
hidden_size,
num_experts,
num_local_experts,
topk,
q_dtype,
use_fp8_dispatch,
quant_config,
)
out = mk.apply(
hidden_states=test_tensors.rank_tokens,
w1=w1,
w2=w2,
topk_weights=test_tensors.topk_weights,
topk_ids=test_tensors.topk,
activation=MoEActivation.SILU,
global_num_experts=num_experts,
expert_map=build_expert_map(),
apply_router_weight_on_input=False,
)
return out
def _deep_ep_v2_moe(
pgi: ProcessGroupInfo,
dp_size: int,
config: TestConfig,
w1: torch.Tensor,
w2: torch.Tensor,
w1_scale: torch.Tensor | None,
w2_scale: torch.Tensor | None,
use_fp8_dispatch: bool,
per_act_token_quant: bool,
):
device = torch.device(f"cuda:{pgi.local_rank}")
init_workspace_manager(device)
is_quantized = w1.dtype == torch.float8_e4m3fn
device_idx = torch.accelerator.current_device_index()
w1 = w1.to(device=device_idx)
w2 = w2.to(device=device_idx)
if is_quantized:
assert w1_scale is not None and w2_scale is not None
w1_scale = w1_scale.to(device=device_idx)
w2_scale = w2_scale.to(device=device_idx)
pg = torch.distributed.new_group(list(range(pgi.world_size)))
# The caller's set_random_seed() only seeds the parent; spawn() gives each
# worker a fresh unseeded RNG, so seed here too or the inputs below differ
# every run. Offset by rank to keep the ranks' data distinct.
set_random_seed(7 + pgi.rank)
test_tensors = TestTensors.make(config)
with set_current_vllm_config(VllmConfig()):
# Reference
q_dtype = torch.float8_e4m3fn if is_quantized else None
torch_combined = torch_experts(
test_tensors.rank_tokens,
w1,
w2,
test_tensors.topk_weights,
test_tensors.topk,
w1_scale=w1_scale,
w2_scale=w2_scale,
a1_scale=test_tensors.rank_token_scales,
a2_scale=test_tensors.intermediate_scales,
quant_dtype=q_dtype,
per_act_token_quant=per_act_token_quant,
)
# Splice experts for this rank
num_local_experts = config.num_experts // pgi.world_size
e_start = num_local_experts * pgi.rank
e_end = e_start + num_local_experts
w1_ep = w1[e_start:e_end]
w2_ep = w2[e_start:e_end]
w1_scale_ep, w2_scale_ep = None, None
if is_quantized:
w1_scale_ep = w1_scale[e_start:e_end] # type: ignore
w2_scale_ep = w2_scale[e_start:e_end] # type: ignore
deepep_combined = deepep_v2_moe_impl(
pg,
pgi,
dp_size,
test_tensors,
w1_ep,
w2_ep,
w1_scale_ep,
w2_scale_ep,
config.num_experts,
config.topk,
use_fp8_dispatch,
per_act_token_quant,
)
if is_quantized:
assert_fp8_close(torch_combined, deepep_combined)
else:
torch.testing.assert_close(
torch_combined,
deepep_combined,
atol=6e-2,
rtol=6e-2,
)
MNKs = [
(1, 256, 256),
(2, 256, 512),
(3, 1024, 2048),
(32, 256, 1024),
(45, 512, 2048),
(64, 1024, 1024),
(222, 1024, 2048),
]
DTYPES = [torch.bfloat16, torch.float8_e4m3fn]
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("m,n,k", MNKs)
@pytest.mark.parametrize("num_experts", [32])
@pytest.mark.parametrize("topk", [6])
@pytest.mark.parametrize("world_dp_size", [(2, 1)])
@multi_gpu_test(num_gpus=2)
@requires_deep_ep_v2
def test_deep_ep_v2_moe(
dtype: torch.dtype,
m: int,
n: int,
k: int,
num_experts: int,
topk: int,
world_dp_size: tuple[int, int],
workspace_init,
):
per_act_token_quant = False
use_fp8_dispatch = False
set_random_seed(7)
world_size, dp_size = world_dp_size
config = TestConfig(dtype=dtype, topk=topk, m=m, k=k, n=n, num_experts=num_experts)
quant_dtype = dtype if dtype == torch.float8_e4m3fn else None
(_, w1, w1_scale, _), (_, w2, w2_scale, _) = make_test_weights(
num_experts,
n,
k,
quant_dtype=quant_dtype,
per_out_ch_quant=True,
)
parallel_launch(
world_size,
_deep_ep_v2_moe,
dp_size,
config,
w1,
w2,
w1_scale,
w2_scale,
use_fp8_dispatch,
per_act_token_quant,
)
EXPERTS_BACKENDS = [
"flashinfer_trtllm",
"flashinfer_cutlass",
"trtllm_fp8",
]
def _make_experts(
experts_backend: str,
config: TestConfig,
moe_config,
num_local_experts: int,
rank: int,
w1_bf16: torch.Tensor,
w2_bf16: torch.Tensor,
test_tensors: TestTensors,
):
e_start = num_local_experts * rank
e_end = e_start + num_local_experts
if experts_backend != "trtllm_fp8":
from vllm.model_executor.layers.fused_moe.config import (
FUSED_MOE_UNQUANTIZED_CONFIG,
)
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
backend_to_kernel_cls,
convert_to_unquantized_kernel_format,
map_unquantized_backend,
)
torch_combined = torch_experts(
test_tensors.rank_tokens,
w1_bf16,
w2_bf16,
test_tensors.topk_weights,
test_tensors.topk,
)
backend = map_unquantized_backend(experts_backend)
w1_ep, w2_ep = convert_to_unquantized_kernel_format(
backend,
moe_config,
w1_bf16[e_start:e_end],
w2_bf16[e_start:e_end],
)
experts_cls = next(
cls
for cls in backend_to_kernel_cls(backend)
if issubclass(cls, mk.FusedMoEExpertsModular)
)
fused_experts = experts_cls(
moe_config=moe_config,
quant_config=FUSED_MOE_UNQUANTIZED_CONFIG,
)
return fused_experts, w1_ep, w2_ep, torch_combined, 1e-1, 2e-1
from tests.kernels.moe.test_moe_layer import _quantize_fp8_halves
from vllm.model_executor.layers.fused_moe.experts.trtllm_fp8_moe import (
TrtLlmFp8ExpertsModular,
)
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
Fp8MoeBackend,
convert_to_fp8_moe_kernel_format,
)
w1_ref = w1_bf16.to(torch.float8_e4m3fn).to(torch.bfloat16)
w2_ref = w2_bf16.to(torch.float8_e4m3fn).to(torch.bfloat16)
block_shape = [128, 128]
qw = _quantize_fp8_halves(w1_ref, w2_ref, block_shape)
assert qw.w13_weight_scale is not None
assert qw.w2_weight_scale is not None
reference_topk_weights = test_tensors.topk_weights.to(torch.bfloat16).to(
torch.float32
)
torch_combined = torch_experts(
test_tensors.rank_tokens,
qw.w13_weight,
qw.w2_weight,
reference_topk_weights,
test_tensors.topk,
w1_scale=qw.w13_weight_scale,
w2_scale=qw.w2_weight_scale,
quant_dtype=torch.float8_e4m3fn,
block_shape=block_shape,
)
class _MockLayer:
weight_block_size = block_shape
class moe_config:
is_act_and_mul = True
intermediate_size_per_partition = config.n
class activation:
is_gated = True
w1_ep, w2_ep, w1_scale_ep, w2_scale_ep = convert_to_fp8_moe_kernel_format(
fp8_backend=Fp8MoeBackend.FLASHINFER_TRTLLM,
layer=_MockLayer(),
w13=qw.w13_weight[e_start:e_end],
w2=qw.w2_weight[e_start:e_end],
w13_scale=qw.w13_weight_scale[e_start:e_end],
w2_scale=qw.w2_weight_scale[e_start:e_end],
w13_input_scale=None,
w2_input_scale=None,
)
fused_experts = TrtLlmFp8ExpertsModular(
moe_config=moe_config,
quant_config=FusedMoEQuantConfig.make(
torch.float8_e4m3fn,
block_shape=block_shape,
w1_scale=w1_scale_ep,
w2_scale=w2_scale_ep,
),
)
return fused_experts, w1_ep, w2_ep, torch_combined, 6e-2, 6e-2
def _deep_ep_v2_moe_backends(
pgi: ProcessGroupInfo,
dp_size: int,
config: TestConfig,
use_cudagraph: bool,
experts_backend: str,
):
import tempfile
from vllm.config import KernelConfig
from vllm.distributed import (
init_distributed_environment,
initialize_model_parallel,
)
device = torch.device(f"cuda:{pgi.local_rank}")
init_workspace_manager(device)
pg = torch.distributed.new_group(list(range(pgi.world_size)))
set_random_seed(7 + pgi.rank)
test_tensors = TestTensors.make(config)
num_local_experts = config.num_experts // pgi.world_size
hidden_size = config.k
# All ranks must use the same global weights before taking their EP slice.
w1_bf16 = (
torch.randn(
(config.num_experts, 2 * config.n, config.k),
device="cuda",
dtype=torch.bfloat16,
)
/ 15
)
w2_bf16 = (
torch.randn(
(config.num_experts, config.k, config.n),
device="cuda",
dtype=torch.bfloat16,
)
/ 15
)
torch.distributed.broadcast(w1_bf16, src=0, group=pg)
torch.distributed.broadcast(w2_bf16, src=0, group=pg)
vllm_cfg = VllmConfig()
vllm_cfg.kernel_config = KernelConfig(moe_backend="flashinfer_trtllm")
with set_current_vllm_config(vllm_cfg):
temp_file = tempfile.mktemp()
init_distributed_environment(
world_size=pgi.world_size,
rank=pgi.rank,
distributed_init_method=f"file://{temp_file}",
local_rank=pgi.local_rank,
backend="nccl",
)
initialize_model_parallel(tensor_model_parallel_size=1)
moe_config = make_dummy_moe_config(
num_experts=config.num_experts,
num_local_experts=num_local_experts,
experts_per_token=config.topk,
hidden_dim=hidden_size,
intermediate_size=config.n,
)
moe_config = dataclasses.replace(
moe_config,
moe_parallel_config=dataclasses.replace(
moe_config.moe_parallel_config,
ep_size=pgi.world_size,
ep_rank=pgi.rank,
use_ep=True,
all2all_backend="deepep_v2",
),
)
(
fused_experts,
w1_ep,
w2_ep,
torch_combined,
atol,
rtol,
) = _make_experts(
experts_backend,
config,
moe_config,
num_local_experts,
pgi.rank,
w1_bf16,
w2_bf16,
test_tensors,
)
v2_args = DeepEPV2Args(
num_local_experts=num_local_experts,
num_experts=config.num_experts,
num_topk=config.topk,
hidden_size=hidden_size,
max_tokens_per_rank=8192,
use_fp8_dispatch=False,
)
a2a = make_deepep_v2_a2a(
pg=pg,
pgi=pgi,
dp_size=dp_size,
v2_args=v2_args,
use_cudagraph=use_cudagraph,
)
mk_kernel = FusedMoEKernel(
prepare_finalize=a2a,
fused_experts=fused_experts,
)
with set_forward_context(None, vllm_cfg):
for _ in range(3):
out = mk_kernel.apply(
hidden_states=test_tensors.rank_tokens,
w1=w1_ep,
w2=w2_ep,
topk_weights=test_tensors.topk_weights,
topk_ids=test_tensors.topk,
activation=MoEActivation.SILU,
global_num_experts=config.num_experts,
expert_map=None,
apply_router_weight_on_input=False,
)
torch.testing.assert_close(torch_combined, out, atol=atol, rtol=rtol)
@pytest.mark.parametrize("m,n,k", [(32, 256, 1024)])
@pytest.mark.parametrize("num_experts", [32])
@pytest.mark.parametrize("topk", [6])
@pytest.mark.parametrize("world_dp_size", [(2, 1)])
@pytest.mark.parametrize("experts_backend", EXPERTS_BACKENDS)
@pytest.mark.parametrize("use_cudagraph", [True, False])
@multi_gpu_test(num_gpus=2)
@requires_deep_ep_v2
@pytest.mark.skipif(
not has_flashinfer() or not current_platform.has_device_capability(100),
reason="Requires FlashInfer TRTLLM fused MoE (SM100)",
)
def test_deep_ep_v2_moe_backends(
m: int,
n: int,
k: int,
num_experts: int,
topk: int,
world_dp_size: tuple[int, int],
experts_backend: str,
use_cudagraph: bool,
workspace_init,
):
set_random_seed(7)
world_size, dp_size = world_dp_size
config = TestConfig(
dtype=torch.float8_e4m3fn
if experts_backend == "trtllm_fp8"
else torch.bfloat16,
topk=topk,
m=m,
k=k,
n=n,
num_experts=num_experts,
)
parallel_launch(
world_size,
_deep_ep_v2_moe_backends,
dp_size,
config,
use_cudagraph,
experts_backend,
)
-634
View File
@@ -1,634 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit-test DeepGEMM FP8 and FP4 kernels (no DeepEP).
Compare DeepGEMM path against the Triton fallback inside vLLM's fused_experts.
"""
import importlib
import math
import pytest
import torch
# vLLM fused-expert reference (Triton fallback + DeepGEMM option)
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from tests.kernels.moe.utils import make_dummy_moe_config
from vllm.model_executor.layers.fused_moe.activation import (
MoEActivation,
)
from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
FusedMoEQuantDesc,
fp8_w8a8_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.deep_gemm_utils import (
deepgemm_moe_permute,
ep_gather,
)
from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import (
_fp8_workspace_shape,
)
from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import (
TritonOrDeepGemmExperts,
)
from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
per_token_group_quant_fp8,
)
from vllm.utils.deep_gemm import (
calc_diff,
is_deep_gemm_supported,
per_block_cast_to_fp8,
)
BLOCK_SIZE = [128, 128]
def test_ep_gather_uses_64_bit_row_offsets():
hidden_size = 4096
source_row = 1 << 18
input_tensor = torch.empty(
(source_row + 1, hidden_size),
device="cuda",
dtype=torch.bfloat16,
)
input_tensor[source_row].fill_(1)
output = torch.empty((1, hidden_size), device="cuda", dtype=torch.bfloat16)
ep_gather(
input_tensor=input_tensor,
recv_topk_ids=torch.zeros((1, 1), device="cuda", dtype=torch.int64),
recv_topk_weight=torch.ones((1, 1), device="cuda", dtype=torch.float32),
input_index=torch.full((1, 1), source_row, device="cuda", dtype=torch.int32),
expert_map=None,
output_tensor=output,
)
torch.testing.assert_close(output, torch.ones_like(output), rtol=0, atol=0)
def test_ep_gather_uses_64_bit_output_row_offsets():
hidden_size = 6144
output_row = (1 << 31) // hidden_size + 1
num_rows = output_row + 1
output = torch.empty(
(num_rows, hidden_size),
device="cuda",
dtype=torch.bfloat16,
)
ep_gather(
input_tensor=torch.ones((1, hidden_size), device="cuda", dtype=torch.bfloat16),
recv_topk_ids=torch.zeros((num_rows, 1), device="cuda", dtype=torch.int64),
recv_topk_weight=torch.ones((num_rows, 1), device="cuda", dtype=torch.float32),
input_index=torch.zeros((num_rows, 1), device="cuda", dtype=torch.int32),
expert_map=None,
output_tensor=output,
)
torch.testing.assert_close(output, torch.ones_like(output), rtol=0, atol=0)
@pytest.mark.parametrize("workspace_dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("num_columns", [2048, 6144, 6145])
def test_fp8_workspace_shape(workspace_dtype, num_columns):
num_rows = 17
shape = _fp8_workspace_shape(num_rows, num_columns, workspace_dtype)
allocated_bytes = math.prod(shape) * workspace_dtype.itemsize
required_bytes = num_rows * num_columns * torch.float8_e4m3fn.itemsize
assert allocated_bytes >= required_bytes
assert allocated_bytes - required_bytes < num_rows * workspace_dtype.itemsize
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
def test_deepgemm_moe_permute_initializes_padding_scales(workspace_init):
hidden_states = torch.randn(2, 128, device="cuda", dtype=torch.bfloat16)
activations, scales = per_token_group_quant_fp8(
hidden_states,
group_size=128,
use_ue8m0=True,
)
topk_ids = torch.tensor([[0], [1]], device="cuda", dtype=torch.int64)
_, permuted_scales, expert_ids, _, _ = deepgemm_moe_permute(
aq=activations,
aq_scale=scales,
topk_ids=topk_ids,
local_num_experts=2,
expert_map=None,
expert_tokens_meta=None,
)
padding = expert_ids < 0
assert padding.any()
torch.testing.assert_close(
permuted_scales[padding],
torch.zeros_like(permuted_scales[padding]),
rtol=0,
atol=0,
)
def make_block_quant_fp8_weights(
e: int,
n: int,
k: int,
block_size: list[int],
):
"""Generate (w1, w2) expert weights and their per-block scale tensors
in FP8 block-quantized format.
w1 shape: (E, 2N, K)
w2 shape: (E, K, N)
"""
dtype = torch.bfloat16
fp8_max, fp8_min = (
torch.finfo(torch.float8_e4m3fn).max,
torch.finfo(torch.float8_e4m3fn).min,
)
# bf16 reference weights
w1_bf16 = torch.randn(e, 2 * n, k, device="cuda", dtype=dtype) / 10
w2_bf16 = torch.randn(e, k, n, device="cuda", dtype=dtype) / 10
w1_bf16.clamp_(fp8_min, fp8_max)
w2_bf16.clamp_(fp8_min, fp8_max)
block_n, block_k = block_size
n_tiles_w1 = math.ceil((2 * n) / block_n)
k_tiles_w1 = math.ceil(k / block_k)
n_tiles_w2 = math.ceil(k / block_n)
k_tiles_w2 = math.ceil(n / block_k)
w1 = torch.empty_like(w1_bf16, dtype=torch.float8_e4m3fn)
w2 = torch.empty_like(w2_bf16, dtype=torch.float8_e4m3fn)
w1_s = torch.empty(e, n_tiles_w1, k_tiles_w1, device="cuda", dtype=torch.float32)
w2_s = torch.empty(e, n_tiles_w2, k_tiles_w2, device="cuda", dtype=torch.float32)
for i in range(e):
w1[i], w1_s[i] = per_block_cast_to_fp8(
w1_bf16[i], block_size=block_size, use_ue8m0=True
)
w2[i], w2_s[i] = per_block_cast_to_fp8(
w2_bf16[i], block_size=block_size, use_ue8m0=True
)
return w1, w2, w1_s, w2_s
@pytest.mark.parametrize("num_tokens", [16, 129, 512])
@pytest.mark.parametrize("with_metadata", [False, True])
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
def test_block_fp8_padding_on_graph_replay(num_tokens, with_metadata, monkeypatch):
"""Both FP8 GEMMs consume only live rows as routing changes on replay."""
from unittest.mock import Mock
import vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe as impl
from vllm.utils.deep_gemm import DeepGemmQuantScaleFMT
if not (
impl.current_platform.is_device_capability_family(90)
or (
impl.current_platform.is_device_capability_family(100)
and DeepGemmQuantScaleFMT.from_oracle() == DeepGemmQuantScaleFMT.UE8M0
)
):
pytest.skip("Prefix-sum FP8 dispatch requires SM90 or SM10.x")
torch.manual_seed(0)
e, n, k, topk = 8, 1024, 512, 2
x = torch.randn(num_tokens, k, device="cuda", dtype=torch.bfloat16) / 10
aq, scales = per_token_group_quant_fp8(x, 128, use_ue8m0=True)
w1, w2, s1, s2 = make_block_quant_fp8_weights(e, n, k, BLOCK_SIZE)
quant = fp8_w8a8_moe_quant_config(w1_scale=s1, w2_scale=s2, block_shape=BLOCK_SIZE)
experts = impl.DeepGemmExperts(make_dummy_moe_config(), quant)
ids = torch.zeros(num_tokens, topk, device="cuda", dtype=torch.int64)
weights = torch.full((num_tokens, topk), 0.5, device="cuda")
counts = torch.zeros(e, device="cuda", dtype=torch.int32)
metadata = mk.ExpertTokensMetadata(counts, None) if with_metadata else None
shapes = experts.workspace_shapes(
num_tokens, 2 * n, k, topk, e, e, metadata, MoEActivation.SILU
)
def route(mode):
ids[:, 0] = torch.arange(num_tokens, device="cuda") % e
ids[:, 1] = (ids[:, 0] + 1) % e
if mode == 1:
ids.fill_(-1)
ids[:1, 0] = e - 1
elif mode == 2:
ids.fill_(-1)
counts.copy_(torch.bincount(ids[ids >= 0], minlength=e).int())
route(0)
variants = []
for dense in (True, False):
ws1, ws2, output = [
torch.empty(shape, device="cuda", dtype=torch.bfloat16) for shape in shapes
]
def invoke(output=output, ws1=ws1, ws2=ws2):
experts.apply(
output,
aq,
w1,
w2,
weights,
ids,
MoEActivation.SILU,
e,
None,
scales,
None,
ws1,
ws2,
metadata,
False,
)
with monkeypatch.context() as mp:
if dense:
platform = Mock(wraps=impl.current_platform)
platform.is_device_capability_family.return_value = False
mp.setattr(impl, "current_platform", platform)
gemm = impl.m_grouped_fp8_gemm_nt_contiguous
calls: list[int] = []
def check_layout(*args, dense=dense, calls=calls, gemm=gemm, **kwargs):
assert kwargs.get("use_psum_layout", False) == (not dense)
calls.append(args[3].numel())
return gemm(*args, **kwargs)
mp.setattr(impl, "m_grouped_fp8_gemm_nt_contiguous", check_layout)
invoke()
assert len(calls) == 2
if not dense:
assert calls == [e, e]
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
invoke()
variants.append((graph, output, ws1, ws2))
for mode in (0, 1, 2, 0):
route(mode)
for graph, _, ws1, ws2 in variants:
ws1.fill_(float("nan"))
ws2.fill_(float("nan"))
graph.replay()
torch.accelerator.synchronize()
torch.testing.assert_close(variants[0][1], variants[1][1], rtol=0, atol=0)
assert torch.isfinite(variants[1][1]).all()
@pytest.mark.parametrize(("n", "k"), [(2048, 512), (512, 1024)])
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
def test_deepgemm_prefix_sum_leaves_unused_capacity_untouched(n, k):
"""Both projection shapes skip unused tiles as counts change on replay."""
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import (
m_grouped_fp8_gemm_nt_contiguous,
mk_alignment_scope,
)
if not (
current_platform.is_device_capability_family(90)
or current_platform.is_device_capability_family(100)
):
pytest.skip("Requires SM90 or SM10.x")
capacity, experts, alignment = 512, 4, 128
x = torch.randn(capacity, k, device="cuda", dtype=torch.bfloat16) / 10
aq, scales = per_token_group_quant_fp8(x, 128, use_ue8m0=True)
w, _, ws, _ = make_block_quant_fp8_weights(experts, n // 2, k, BLOCK_SIZE)
ends = torch.tensor([0, 1, 128, 129], device="cuda", dtype=torch.int32)
output = torch.empty(capacity, n, device="cuda", dtype=torch.bfloat16)
def invoke():
with mk_alignment_scope(alignment):
m_grouped_fp8_gemm_nt_contiguous(
(aq, scales), (w, ws), output, ends, use_psum_layout=True
)
invoke()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
invoke()
for bounds, allocated_end in (([0, 1, 128, 129], 256), ([0] * 4, 0)):
ends.copy_(torch.tensor(bounds, device="cuda", dtype=torch.int32))
output.fill_(91)
graph.replay()
torch.accelerator.synchronize()
assert torch.all(output[allocated_end:] == 91)
if allocated_end:
assert torch.all(output[[0, 128]] != 91)
def run_single_case(m, n, k, topk, num_experts, block_size):
"""Run one (M,N,K) configuration on a single GPU and assert DeepGEMM ==
Triton baseline within tolerance.
"""
tokens_bf16 = (
torch.randn(m, k, device="cuda", dtype=torch.bfloat16)
.clamp_min_(-1)
.clamp_max_(1)
)
_, a1_scale = per_token_group_quant_fp8(tokens_bf16, block_size[1])
# expert weight tensors
w1, w2, w1_s, w2_s = make_block_quant_fp8_weights(num_experts, n, k, block_size)
router_logits = torch.randn(m, num_experts, device="cuda", dtype=torch.float32)
topk_weights, topk_ids = torch.topk(router_logits, k=topk, dim=-1)
topk_weights = torch.nn.functional.softmax(topk_weights, dim=-1)
quant_config = fp8_w8a8_moe_quant_config(
w1_scale=w1_s,
w2_scale=w2_s,
a1_scale=a1_scale,
block_shape=block_size,
)
moe_config = make_dummy_moe_config()
deep_gemm_experts = mk.FusedMoEKernel(
prepare_finalize=maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
use_monolithic=False,
),
fused_experts=TritonOrDeepGemmExperts(
moe_config=moe_config,
quant_config=quant_config,
),
)
# triton reference
out_triton = fused_experts(
hidden_states=tokens_bf16,
w1=w1,
w2=w2,
topk_weights=topk_weights,
topk_ids=topk_ids,
quant_config=quant_config,
)
# DeepGemm
out_deepgemm = deep_gemm_experts.apply(
hidden_states=tokens_bf16,
w1=w1,
w2=w2,
topk_weights=topk_weights,
topk_ids=topk_ids,
global_num_experts=num_experts,
activation=MoEActivation.SILU,
apply_router_weight_on_input=False,
expert_map=None,
)
diff = calc_diff(out_deepgemm, out_triton)
assert diff < 0.001, f"Diff exceeded 1%: {diff}"
# Note: N <= 512 will disable the deepgemm path due to performance issues.
MNKs = [
(1024, 768, 128),
(2048, 768, 512),
(512, 1024, 1024),
(4096, 4096, 1024),
]
TOPKS = [2, 6]
NUM_EXPERTS = [32]
@pytest.mark.parametrize(("m", "n", "k"), MNKs)
@pytest.mark.parametrize("topk", TOPKS)
@pytest.mark.parametrize("num_experts", NUM_EXPERTS)
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
def test_deepgemm_vs_triton(m, n, k, topk, num_experts, monkeypatch, workspace_init):
with monkeypatch.context() as mp:
mp.setenv("VLLM_USE_DEEP_GEMM", "1")
_DeepGemmExperts = importlib.import_module(
"vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe"
).DeepGemmExperts
call_counter = {"cnt": 0}
orig_fn = _DeepGemmExperts.apply
def _spy_apply(*args, **kwargs):
call_counter["cnt"] += 1
return orig_fn(*args, **kwargs)
monkeypatch.setattr(_DeepGemmExperts, "apply", _spy_apply)
if topk > num_experts:
pytest.skip(f"topk={topk} > num_experts={num_experts}")
run_single_case(
m=m,
n=n,
k=k,
topk=topk,
num_experts=num_experts,
block_size=BLOCK_SIZE,
)
# ensure that the DeepGEMM path was indeed taken.
assert call_counter["cnt"] == 1, (
f"DeepGEMM path was not executed during the test. "
f"Call counter: {call_counter['cnt']}"
)
# ---------------------------------------------------------------------------
# FP4 weight tests (DeepGEMM m_grouped_fp8_fp4_gemm_nt_contiguous)
# ---------------------------------------------------------------------------
def make_mxfp4_weights(
e: int,
n: int,
k: int,
):
"""Generate (w1, w2) expert weights in MXFP4 packed format with float32 scales,
plus BF16 reference weights for validation.
w1 shape: (E, 2N, K//2) uint8 — packed FP4
w2 shape: (E, K, N//2) uint8 — packed FP4
w1_s shape: (E, 2N, K//32) float32 — per-row block-32 scales
w2_s shape: (E, K, N//32) float32 — per-row block-32 scales
w1_bf16: (E, 2N, K) — original BF16 for reference
w2_bf16: (E, K, N) — original BF16 for reference
"""
from deep_gemm.utils.math import per_token_cast_to_fp4
dtype = torch.bfloat16
gran_k = 32 # MXFP4 block size
# bf16 reference weights — scale by 1/sqrt(dim) for numerical stability
w1_bf16 = torch.randn(e, 2 * n, k, device="cuda", dtype=dtype) * (k**-0.5)
w2_bf16 = torch.randn(e, k, n, device="cuda", dtype=dtype) * (n**-0.5)
# Quantize per-expert to FP4
w1 = torch.empty(e, 2 * n, k // 2, device="cuda", dtype=torch.uint8)
w2 = torch.empty(e, k, n // 2, device="cuda", dtype=torch.uint8)
w1_s = torch.empty(
e, 2 * n, math.ceil(k / gran_k), device="cuda", dtype=torch.float32
)
w2_s = torch.empty(e, k, math.ceil(n / gran_k), device="cuda", dtype=torch.float32)
for i in range(e):
w1[i], w1_s[i] = per_token_cast_to_fp4(
w1_bf16[i].float(), use_ue8m0=True, gran_k=gran_k
)
w2[i], w2_s[i] = per_token_cast_to_fp4(
w2_bf16[i].float(), use_ue8m0=True, gran_k=gran_k
)
return w1, w2, w1_s, w2_s, w1_bf16, w2_bf16
def _bf16_moe_reference(x, w1, w2, topk_weights, topk_ids):
"""BF16 token-loop MoE reference for correctness testing."""
import torch.nn.functional as F
num_tokens, hidden_size = x.shape
intermediate = w1.shape[1] // 2
top_k = topk_ids.shape[1]
output = torch.zeros(num_tokens, hidden_size, dtype=torch.float32, device=x.device)
for t in range(num_tokens):
for kk in range(top_k):
e = topk_ids[t, kk].item()
w = topk_weights[t, kk].item()
fc1 = x[t : t + 1].float() @ w1[e].float().T
linear = fc1[:, :intermediate]
gate = fc1[:, intermediate:]
act = F.silu(gate) * linear
fc2 = act @ w2[e].float().T
output[t] += w * fc2[0]
return output.to(torch.bfloat16)
def run_single_fp4_case(m, n, k, topk, num_experts):
"""Run one (M,N,K) configuration with FP4 weights on DeepGEMM and assert
DeepGEMM FP4 == BF16 reference within tolerance.
"""
tokens_bf16 = torch.randn(m, k, device="cuda", dtype=torch.bfloat16) * (k**-0.5)
# FP4 expert weight tensors + BF16 originals for reference
w1, w2, w1_s, w2_s, w1_bf16, w2_bf16 = make_mxfp4_weights(num_experts, n, k)
router_logits = torch.randn(m, num_experts, device="cuda", dtype=torch.float32)
topk_weights, topk_ids = torch.topk(router_logits, k=topk, dim=-1)
topk_weights = torch.nn.functional.softmax(topk_weights, dim=-1)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
)
from vllm.platforms import current_platform
_fp8_dtype = current_platform.fp8_dtype()
_block_shape = GroupShape(128, 128)
quant_config = FusedMoEQuantConfig(
_a1=FusedMoEQuantDesc(_fp8_dtype, _block_shape, None, None, None, None),
_a2=FusedMoEQuantDesc(_fp8_dtype, _block_shape, None, None, None, None),
_w1=FusedMoEQuantDesc("mxfp4", None, w1_s, None, None, None),
_w2=FusedMoEQuantDesc("mxfp4", None, w2_s, None, None, None),
)
moe_config = make_dummy_moe_config()
from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import (
DeepGemmFP4Experts,
)
deep_gemm_fp4_experts = mk.FusedMoEKernel(
prepare_finalize=maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
use_monolithic=False,
),
fused_experts=DeepGemmFP4Experts(
moe_config=moe_config,
quant_config=quant_config,
),
)
# DeepGEMM FP4 path
out_deepgemm_fp4 = deep_gemm_fp4_experts.apply(
hidden_states=tokens_bf16,
w1=w1,
w2=w2,
topk_weights=topk_weights,
topk_ids=topk_ids,
global_num_experts=num_experts,
activation=MoEActivation.SILU,
apply_router_weight_on_input=False,
expert_map=None,
)
# BF16 reference using the same original weights
out_ref = _bf16_moe_reference(tokens_bf16, w1_bf16, w2_bf16, topk_weights, topk_ids)
# FP4 vs BF16 reference: quantization error from FP4 weights + FP8 activations
diff = calc_diff(out_deepgemm_fp4, out_ref)
assert diff < 0.05, f"FP4 diff exceeded 5%: {diff}"
# DeepSeek V4 dims: H=4096, I=2048, so N=2*I=4096, K=H=4096.
# FP4 quantization with block_k=32 needs large K for good accuracy.
FP4_MNKs = [
(128, 4096, 4096), # DeepSeek V4 shape
(256, 2048, 2048), # Half-size variant
(128, 384, 3584), # Kimi-K3 TP8 latent-MoE shape
]
FP4_TOPKS = [2]
FP4_NUM_EXPERTS = [8]
@pytest.mark.parametrize(("m", "n", "k"), FP4_MNKs)
@pytest.mark.parametrize("topk", FP4_TOPKS)
@pytest.mark.parametrize("num_experts", FP4_NUM_EXPERTS)
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
def test_deepgemm_fp4_vs_triton(
m, n, k, topk, num_experts, monkeypatch, workspace_init
):
pytest.importorskip("deep_gemm.utils.math")
with monkeypatch.context() as mp:
mp.setenv("VLLM_USE_DEEP_GEMM", "1")
_DeepGemmFP4Experts = importlib.import_module(
"vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe"
).DeepGemmFP4Experts
call_counter = {"cnt": 0}
orig_fn = _DeepGemmFP4Experts.apply
def _spy_apply(*args, **kwargs):
call_counter["cnt"] += 1
return orig_fn(*args, **kwargs)
monkeypatch.setattr(_DeepGemmFP4Experts, "apply", _spy_apply)
if topk > num_experts:
pytest.skip(f"topk={topk} > num_experts={num_experts}")
run_single_fp4_case(
m=m,
n=n,
k=k,
topk=topk,
num_experts=num_experts,
)
# ensure that the DeepGEMM FP4 path was indeed taken.
assert call_counter["cnt"] == 1, (
f"DeepGEMM FP4 path was not executed during the test. "
f"Call counter: {call_counter['cnt']}"
)
@@ -1,354 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from vllm.platforms import current_platform
if not current_platform.is_device_capability_family(120):
pytest.skip(
reason="FlashInfer B12x MoE requires SM120 (RTX Pro 6000 / DGX Spark).",
allow_module_level=True,
)
from vllm.utils.flashinfer import has_flashinfer_b12x_moe
if not has_flashinfer_b12x_moe():
pytest.skip(
reason=(
"FlashInfer B12xMoEWrapper not available in installed "
"FlashInfer (needs PR #3080)."
),
allow_module_level=True,
)
# Import fp4_quantize after the skip guard — FlashInfer must be installed.
from flashinfer.fp4_quantization import fp4_quantize
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from tests.kernels.moe.utils import make_dummy_moe_config
from tests.kernels.utils import torch_moe
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.config import nvfp4_moe_quant_config
from vllm.model_executor.layers.fused_moe.experts.flashinfer_b12x_moe import (
FlashInferB12xExperts,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
reorder_w1w3_to_w3w1,
)
from vllm.utils.torch_utils import set_random_seed
# Dimensions chosen to satisfy FP4 alignment requirements (k multiple of 256,
# n multiple of 128) while keeping tests fast.
MNK_FACTORS = [
(2, 128, 256),
(2, 256, 512),
(16, 128, 256),
(64, 256, 512),
]
def _process_b12x_weights(
experts: FlashInferB12xExperts,
w1_scale: torch.Tensor,
w2_scale: torch.Tensor,
w1_scale_2: torch.Tensor,
w2_scale_2: torch.Tensor,
) -> None:
layer = SimpleNamespace(
w13_weight_scale=w1_scale,
w13_weight_scale_2=w1_scale_2,
w2_weight_scale=w2_scale,
w2_weight_scale_2=w2_scale_2,
)
experts.process_weights_after_loading(layer)
@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
@pytest.mark.parametrize("e", [8, 16])
@pytest.mark.parametrize("topk", [1, 2, 4])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@torch.inference_mode()
def test_flashinfer_b12x_moe(
m: int,
n: int,
k: int,
e: int,
topk: int,
dtype: torch.dtype,
workspace_init,
):
"""Test FlashInferB12xExperts against a BF16 torch reference.
The SM12x kernel takes BF16 hidden states directly and fuses token
dispatch, W1 GEMM, SwiGLU, and W2 GEMM into one call. We verify
correctness against ``torch_moe`` using generous tolerances to account
for the internal FP4 quantization of activations and weights.
Scale convention
----------------
The SM12x kernel uses ``w1_alpha`` as *both* the activation-quantisation
global scale and the weight dequantisation factor. These two roles are
conflated into a single parameter in ``launch_sm120_moe``, so they must
equal the same value. We use ``global_scale = 1.0`` for
``fp4_quantize`` so that ``w1_alpha = ones`` satisfies both roles
simultaneously. The alternative — vLLM's convention of baking a large
``w_gs`` into block-scale values and compensating with
``g1_alphas = 1/w_gs`` — is incompatible with this kernel.
"""
set_random_seed(7)
with set_current_vllm_config(
VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
):
a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
# Generate BF16 reference weights in [gate, up] order.
# Shape: w1=(e, 2n, k), w2=(e, k, n).
w1_bf16 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 15
w2_bf16 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 15
# ------------------------------------------------------------------ #
# Quantise weights for the SM12x kernel using FlashInfer's convention:
# global_scale = 1.0 → block_scale = max_abs_block / fp4_max
# w1_alpha = 1.0 (no extra global factor to compensate)
#
# The scale factors returned by fp4_quantize(..., is_sf_swizzled_layout=True)
# are already in the swizzled 2D layout expected by convert_sf_to_mma_layout.
# No additional swizzle_blockscale() call is needed.
# ------------------------------------------------------------------ #
gs = torch.ones(1, device="cuda", dtype=torch.float32)
sf_vec_size = 16
# W1: reorder BF16 from [gate, up] → [up, gate], then quantise.
# Note: in reorder_w1w3_to_w3w1, "w1" refers to the gate projection
# and "w3" refers to the up projection.
# A dummy scale is passed and discarded; real scales come from
# fp4_quantize after reordering.
w1_reordered, _ = reorder_w1w3_to_w3w1(
w1_bf16.clone(),
torch.ones((e, 2 * n, 1), device="cuda", dtype=torch.float32),
)
w1_flat = w1_reordered.reshape(e * 2 * n, k)
w1_q_flat, w1_sf_flat = fp4_quantize(
w1_flat,
global_scale=gs,
sf_vec_size=sf_vec_size,
is_sf_swizzled_layout=True,
)
w1_q = w1_q_flat.view(e, 2 * n, k // 2) # uint8, packed FP4
w1_blockscale = w1_sf_flat.view(e, 2 * n, w1_sf_flat.shape[1]) # float8
# W2: no row reordering needed for the down-projection.
w2_flat = w2_bf16.reshape(e * k, n)
w2_q_flat, w2_sf_flat = fp4_quantize(
w2_flat,
global_scale=gs,
sf_vec_size=sf_vec_size,
is_sf_swizzled_layout=True,
)
w2_q = w2_q_flat.view(e, k, n // 2) # uint8, packed FP4
w2_blockscale = w2_sf_flat.view(e, k, w2_sf_flat.shape[1]) # float8
# All per-expert alphas are 1.0 (global_scale = 1.0, no compensation).
ones_e = torch.ones(e, device="cuda", dtype=torch.float32)
quant_config = nvfp4_moe_quant_config(
g1_alphas=ones_e,
g2_alphas=ones_e,
a1_gscale=ones_e,
a2_gscale=ones_e,
w1_scale=w1_blockscale,
w2_scale=w2_blockscale,
)
moe_config = make_dummy_moe_config(
num_experts=e,
experts_per_token=topk,
hidden_dim=k,
intermediate_size=n,
in_dtype=dtype,
)
experts = FlashInferB12xExperts(
moe_config=moe_config,
quant_config=quant_config,
)
_process_b12x_weights(
experts,
w1_blockscale,
w2_blockscale,
ones_e,
ones_e,
)
kernel = mk.FusedMoEKernel(
maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
use_monolithic=False,
),
experts,
)
score = torch.randn((m, e), device="cuda", dtype=dtype)
topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False)
sm12x_output = kernel.apply(
hidden_states=a,
w1=w1_q,
w2=w2_q,
topk_weights=topk_weights,
topk_ids=topk_ids,
global_num_experts=e,
activation=MoEActivation.SILU,
apply_router_weight_on_input=False,
expert_map=None,
)
# Reference: BF16 torch MoE using original [gate, up] BF16 weights.
# torch_moe's SiluAndMul expects [gate, up] order, matching w1_bf16.
torch_output = torch_moe(a, w1_bf16, w2_bf16, score, topk)
torch.testing.assert_close(sm12x_output, torch_output, atol=2e-1, rtol=2e-1)
@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
@pytest.mark.parametrize("e", [8, 16])
@pytest.mark.parametrize("topk", [1, 2, 4])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@torch.inference_mode()
def test_flashinfer_b12x_moe_relu2(
m: int,
n: int,
k: int,
e: int,
topk: int,
dtype: torch.dtype,
workspace_init,
):
"""Test FlashInferB12xExperts with ReLU2 (non-gated) activation.
ReLU2 is used by Nemotron-H style models. Unlike the gated SiLU
path, w1 has shape [E, N, K] (not [E, 2N, K]) and the activation
is relu(x)^2 without a gate/up split.
"""
set_random_seed(7)
with set_current_vllm_config(
VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
):
a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
# Non-gated: w1 shape is (e, n, k), not (e, 2n, k).
w1_bf16 = torch.randn((e, n, k), device="cuda", dtype=dtype) / 15
w2_bf16 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 15
gs = torch.ones(1, device="cuda", dtype=torch.float32)
sf_vec_size = 16
# W1: no gate/up reordering for non-gated.
w1_flat = w1_bf16.reshape(e * n, k)
w1_q_flat, w1_sf_flat = fp4_quantize(
w1_flat,
global_scale=gs,
sf_vec_size=sf_vec_size,
is_sf_swizzled_layout=True,
)
w1_q = w1_q_flat.view(e, n, k // 2)
w1_blockscale = w1_sf_flat.view(e, n, w1_sf_flat.shape[1])
w2_flat = w2_bf16.reshape(e * k, n)
w2_q_flat, w2_sf_flat = fp4_quantize(
w2_flat,
global_scale=gs,
sf_vec_size=sf_vec_size,
is_sf_swizzled_layout=True,
)
w2_q = w2_q_flat.view(e, k, n // 2)
w2_blockscale = w2_sf_flat.view(e, k, w2_sf_flat.shape[1])
ones_e = torch.ones(e, device="cuda", dtype=torch.float32)
quant_config = nvfp4_moe_quant_config(
g1_alphas=ones_e,
g2_alphas=ones_e,
a1_gscale=ones_e,
a2_gscale=ones_e,
w1_scale=w1_blockscale,
w2_scale=w2_blockscale,
)
moe_config = make_dummy_moe_config(
num_experts=e,
experts_per_token=topk,
hidden_dim=k,
intermediate_size=n,
in_dtype=dtype,
activation=MoEActivation.RELU2_NO_MUL,
)
experts = FlashInferB12xExperts(
moe_config=moe_config,
quant_config=quant_config,
)
_process_b12x_weights(
experts,
w1_blockscale,
w2_blockscale,
ones_e,
ones_e,
)
kernel = mk.FusedMoEKernel(
maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
use_monolithic=False,
),
experts,
)
score = torch.randn((m, e), device="cuda", dtype=dtype)
topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False)
b12x_output = kernel.apply(
hidden_states=a,
w1=w1_q,
w2=w2_q,
topk_weights=topk_weights,
topk_ids=topk_ids,
global_num_experts=e,
activation=MoEActivation.RELU2_NO_MUL,
apply_router_weight_on_input=False,
expert_map=None,
)
torch_output = torch_moe(
a,
w1_bf16,
w2_bf16,
score,
topk,
activation=MoEActivation.RELU2_NO_MUL,
)
torch.testing.assert_close(
b12x_output,
torch_output,
atol=2e-1,
rtol=2e-1,
)
if __name__ == "__main__":
test_flashinfer_b12x_moe(16, 128, 256, 8, 2, torch.bfloat16)
@@ -1,357 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for FlashInfer CuTeDSL NVFP4 MoE."""
from types import SimpleNamespace
import pytest
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from tests.kernels.quantization.nvfp4_utils import (
FLOAT4_E2M1_MAX,
FLOAT8_E4M3_MAX,
break_fp4_bytes,
)
from vllm import _custom_ops as ops
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.activation import (
SiluAndMul,
SiluAndMulWithClamp,
SwigluOAIAndMul,
)
from vllm.model_executor.layers.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutedsl_moe import (
FlashInferCuteDSLExperts,
)
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
NvFp4MoeBackend,
make_nvfp4_moe_quant_config,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
prepare_nvfp4_moe_layer_for_flashinfer_cutedsl,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
quantize_nvfp4_per_token_input,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer_cutedsl_moe_nvfp4
from vllm.utils.math_utils import next_power_of_2
from vllm.utils.torch_utils import set_random_seed
if not has_flashinfer_cutedsl_moe_nvfp4() or not (
current_platform.is_device_capability_family(100)
):
pytest.skip(
"Requires FlashInfer CuTeDSL NVFP4 MoE on SM100",
allow_module_level=True,
)
_SWIGLU_ALPHA = 1.702
_SWIGLU_BETA = 1.0
# The GEMM1 output is O(0.5) at these tensor scales, so the OAI default of 7.0
# would never clamp anything. Pick a limit that actually bites.
_SWIGLU_LIMIT = 0.3
_ACT_CASES = [
pytest.param(MoEActivation.SILU, None, None, None, id="silu"),
pytest.param(MoEActivation.SILU, None, None, _SWIGLU_LIMIT, id="silu-clamp"),
pytest.param(MoEActivation.RELU2_NO_MUL, None, None, None, id="relu2_no_mul"),
pytest.param(
MoEActivation.SWIGLUOAI,
_SWIGLU_ALPHA,
_SWIGLU_BETA,
_SWIGLU_LIMIT,
id="swigluoai",
),
pytest.param(
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
_SWIGLU_ALPHA,
_SWIGLU_BETA,
_SWIGLU_LIMIT,
id="swigluoai_uninterleave",
),
]
def _reference_activation(
activation: MoEActivation,
alpha: float | None,
beta: float | None,
limit: float | None,
):
"""VLLM's own op for this activation, so the reference is not re-derived."""
if activation == MoEActivation.RELU2_NO_MUL:
return lambda x: torch.square(torch.relu(x))
if activation == MoEActivation.SWIGLUOAI:
# SwigluOAIAndMul hardcodes beta=1 and reads gate/up interleaved.
assert beta == 1.0
return SwigluOAIAndMul(alpha=alpha, limit=limit)
if activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
return SiluAndMulWithClamp(limit, alpha, beta, compile_native=False)
if limit is not None:
return SiluAndMulWithClamp(limit, compile_native=False)
return SiluAndMul()
def _torch_moe_reference(
a: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
score: torch.Tensor,
topk: int,
act_fn,
) -> torch.Tensor:
m = a.shape[0]
weights, ids = torch.topk(torch.softmax(score, dim=-1, dtype=torch.float32), topk)
x = a.view(m, 1, -1).repeat(1, topk, 1).reshape(m * topk, -1)
flat_ids = ids.reshape(-1)
out = torch.zeros(m * topk, w2.shape[1], dtype=a.dtype, device=a.device)
for expert in range(w1.shape[0]):
mask = flat_ids == expert
if mask.any():
acc = act_fn(x[mask] @ w1[expert].transpose(0, 1)).to(a.dtype)
out[mask] = acc @ w2[expert].transpose(0, 1)
return (
(out.view(m, topk, -1).float() * weights.view(m, topk, 1))
.sum(dim=1)
.to(a.dtype)
)
def _quantize_nvfp4_linear(
weight: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
weights_q = []
scales = []
global_scales = []
for expert_weight in weight:
global_scale = (
FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / expert_weight.abs().max()
).to(torch.float32)
weight_q, scale = ops.scaled_fp4_quant(
expert_weight,
global_scale,
is_sf_swizzled_layout=False,
)
weights_q.append(weight_q)
scales.append(scale)
global_scales.append(global_scale)
return torch.stack(weights_q), torch.stack(scales), torch.stack(global_scales)
def _dequantize_nvfp4_linear(
tensor_fp4: torch.Tensor,
tensor_sf: torch.Tensor,
global_scale: torch.Tensor,
dtype: torch.dtype,
) -> torch.Tensor:
assert tensor_fp4.dtype == torch.uint8
m, packed_k = tensor_fp4.shape
k = packed_k * 2
tensor_f32 = break_fp4_bytes(tensor_fp4, torch.float32)
tensor_f32 = tensor_f32.reshape(m, k // 16, 16)
tensor_sf = tensor_sf.view(torch.float8_e4m3fn).to(torch.float32)
tensor_sf = tensor_sf[:, : k // 16] / global_scale
return (tensor_f32 * tensor_sf.unsqueeze(-1)).reshape(m, k).to(dtype)
@pytest.mark.parametrize(
"m,n,k,e,topk",
[(16, 128, 512, 4, 2), (16, 192, 512, 4, 2), (16, 192, 2688, 4, 2)],
)
@pytest.mark.parametrize("activation,alpha,beta,limit", _ACT_CASES)
@pytest.mark.parametrize("per_token_activation", [False, True])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@torch.inference_mode()
def test_flashinfer_cutedsl_fp4_moe(
m: int,
n: int,
k: int,
e: int,
topk: int,
activation: MoEActivation,
alpha: float | None,
beta: float | None,
limit: float | None,
per_token_activation: bool,
dtype: torch.dtype,
workspace_init,
):
set_random_seed(7)
with set_current_vllm_config(
VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
):
hidden_states = torch.randn((m, k), device="cuda", dtype=dtype) / 10
# Keep GEMM1 activation variance stable as K changes, so the fixed
# tolerance against the BF16 GEMM2 reference measures comparable FP4
# quantization error at each hidden width.
hidden_states *= (512 / k) ** 0.5
w1_rows = 2 * n if activation.is_gated else n
w1 = torch.randn((e, w1_rows, k), device="cuda", dtype=dtype) / 15
w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 15
w1_q, w1_scale, w1_global_scale = _quantize_nvfp4_linear(w1)
w2_q, w2_scale, w2_global_scale = _quantize_nvfp4_linear(w2)
score = torch.randn((m, e), device="cuda", dtype=dtype)
topk_weights, topk_ids, _ = fused_topk(
hidden_states, score, topk, renormalize=False
)
moe_config = FusedMoEConfig(
num_experts=e,
experts_per_token=topk,
hidden_dim=k,
intermediate_size=n,
num_local_experts=e,
num_logical_experts=e,
activation=activation,
device="cuda",
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
in_dtype=dtype,
routing_method=RoutingMethodType.TopK,
max_num_tokens=next_power_of_2(m),
)
fake_layer = SimpleNamespace(activation=activation, moe_config=moe_config)
a1_scale = torch.ones(1, device="cuda", dtype=torch.float32)
a2_scale = torch.ones(1, device="cuda", dtype=torch.float32)
(
w1_cutedsl,
w1_scale_cutedsl,
w1_alpha,
a1_scale,
w2_cutedsl,
w2_scale_cutedsl,
w2_alpha,
a2_scale,
) = prepare_nvfp4_moe_layer_for_flashinfer_cutedsl(
layer=fake_layer,
w13=w1_q,
w13_scale=w1_scale,
w13_scale_2=(1.0 / w1_global_scale),
a13_scale=a1_scale,
w2=w2_q,
w2_scale=w2_scale,
w2_scale_2=(1.0 / w2_global_scale),
a2_scale=a2_scale,
)
# I=192 must remain unpadded for gated experts, but become 256 for
# non-gated experts. The reference below uses the checkpoint shapes.
expected_n = n if activation.is_gated else (n + 127) // 128 * 128
assert moe_config.intermediate_size_per_partition == expected_n
assert w1_cutedsl.shape[1] == expected_n * (2 if activation.is_gated else 1)
assert w2_cutedsl.shape[2] * 2 == expected_n
expected_k = (k + 255) // 256 * 256
assert moe_config.hidden_dim == expected_k
assert moe_config.hidden_dim_unpadded == k
assert w1_cutedsl.shape[2] * 2 == expected_k
assert w2_cutedsl.shape[1] == expected_k
quant_config = make_nvfp4_moe_quant_config(
backend=NvFp4MoeBackend.FLASHINFER_CUTEDSL,
w13_scale_2=w1_alpha,
w2_scale_2=w2_alpha,
a13_scale=a1_scale,
a2_scale=a2_scale,
w13_scale=w1_scale_cutedsl,
w2_scale=w2_scale_cutedsl,
# Unset params must be omitted rather than forwarded as None into
# the kernel's float-typed SwiGLU arguments.
swiglu_alpha=alpha,
swiglu_beta=beta,
swiglu_limit=limit,
)
cutedsl_experts = mk.FusedMoEKernel(
maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
use_monolithic=False,
),
FlashInferCuteDSLExperts(
moe_config=moe_config,
quant_config=quant_config,
per_token_activation=per_token_activation,
),
)
cutedsl_output = cutedsl_experts.apply(
# The model runner pads inputs and trims outputs around this kernel.
hidden_states=torch.nn.functional.pad(hidden_states, (0, expected_k - k)),
w1=w1_cutedsl,
w2=w2_cutedsl,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=activation,
global_num_experts=e,
expert_map=None,
apply_router_weight_on_input=False,
)
cutedsl_output = cutedsl_output[:, :k]
a_global_scale = torch.ones(1, device="cuda", dtype=torch.float32)
if per_token_activation:
a_q, a_scale, a_row_scale = quantize_nvfp4_per_token_input(hidden_states)
else:
a_q, a_scale = ops.scaled_fp4_quant(
hidden_states,
a_global_scale,
is_sf_swizzled_layout=False,
)
a_in_dtype = _dequantize_nvfp4_linear(
a_q,
a_scale,
a_global_scale,
dtype=dtype,
)
if per_token_activation:
a_in_dtype = (a_in_dtype.float() * a_row_scale.view(-1, 1)).to(dtype)
w1_d = torch.empty((e, w1_rows, k), device="cuda", dtype=dtype)
w2_d = torch.empty((e, k, n), device="cuda", dtype=dtype)
for idx in range(e):
w1_d[idx] = _dequantize_nvfp4_linear(
w1_q[idx],
w1_scale[idx],
w1_global_scale[idx],
dtype=dtype,
)
w2_d[idx] = _dequantize_nvfp4_linear(
w2_q[idx],
w2_scale[idx],
w2_global_scale[idx],
dtype=dtype,
)
torch_output = _torch_moe_reference(
a_in_dtype,
w1_d,
w2_d,
score,
topk,
_reference_activation(activation, alpha, beta, limit),
)
torch.testing.assert_close(
torch_output,
cutedsl_output,
atol=3e-2,
rtol=2e-1,
)
# Outputs here are O(1e-2) while NVFP4 noise is O(1e-3), so an absolute
# tolerance loose enough for the quantization error also accepts a zero
# tensor. Compare direction too, which dropped SwiGLU params or a wrong
# w13 layout would break.
cosine = torch.nn.functional.cosine_similarity(
cutedsl_output.flatten().float(), torch_output.flatten().float(), dim=0
)
assert cosine > 0.99, f"cosine similarity {cosine:.4f} below 0.99"
-250
View File
@@ -1,250 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from tests.kernels.moe.utils import make_test_quant_config
from tests.kernels.quantization.nvfp4_utils import (
FLOAT4_E2M1_MAX,
FLOAT8_E4M3_MAX,
dequantize_nvfp4_to_dtype,
)
from tests.kernels.utils import torch_moe
from vllm import _custom_ops as ops
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import (
FlashInferExperts,
is_valid_flashinfer_cutlass_fused_moe,
)
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer_cutlass_fused_moe
from vllm.utils.math_utils import next_power_of_2
from vllm.utils.torch_utils import set_random_seed
if not has_flashinfer_cutlass_fused_moe() or not current_platform.has_device_capability(
100
):
pytest.skip(
"Requires flashinfer_cutlass_fused_moe and nvfp4 support",
allow_module_level=True,
)
MNK_FACTORS = [
(2, 1024, 1024),
(2, 3072, 1024),
(2, 3072, 1536),
(64, 1024, 1536),
(64, 3072, 1024),
(64, 2048, 1536),
(224, 1024, 1024),
(224, 1024, 1536),
]
@pytest.mark.parametrize(
"activation",
[MoEActivation.SWIGLUOAI, MoEActivation.SWIGLUOAI_UNINTERLEAVE],
)
def test_flashinfer_swigluoai_params_are_forwarded(activation, monkeypatch):
from flashinfer.fused_moe.core import ActivationType
moe_config = FusedMoEConfig(
num_experts=2,
experts_per_token=1,
hidden_dim=128,
intermediate_size=128,
num_local_experts=2,
num_logical_experts=2,
activation=activation,
device="cuda",
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
in_dtype=torch.bfloat16,
routing_method=RoutingMethodType.TopK,
)
quant_config = FusedMoEQuantConfig.make(
gemm1_alpha=1.702,
gemm1_beta=1.0,
gemm1_clamp_limit=7.0,
)
experts = FlashInferExperts(moe_config=moe_config, quant_config=quant_config)
call_args = {}
def fake_flashinfer_cutlass_fused_moe(**kwargs):
call_args.update(kwargs)
monkeypatch.setattr(
"vllm.model_executor.layers.fused_moe.experts."
"flashinfer_cutlass_moe.flashinfer_cutlass_fused_moe",
fake_flashinfer_cutlass_fused_moe,
)
experts.apply(
output=torch.empty((1, 128), device="cuda", dtype=torch.bfloat16),
hidden_states=torch.empty((1, 128), device="cuda", dtype=torch.bfloat16),
w1=torch.empty((2, 256, 128), device="cuda", dtype=torch.bfloat16),
w2=torch.empty((2, 128, 128), device="cuda", dtype=torch.bfloat16),
topk_weights=torch.ones((1, 1), device="cuda", dtype=torch.float32),
topk_ids=torch.zeros((1, 1), device="cuda", dtype=torch.int64),
activation=activation,
global_num_experts=2,
expert_map=None,
a1q_scale=None,
a2_scale=None,
workspace13=None,
workspace2=None,
expert_tokens_meta=None,
apply_router_weight_on_input=False,
)
assert experts._supports_activation(activation)
assert call_args["activation_type"] == ActivationType.Swiglu
for name, value in (
("swiglu_alpha", 1.702),
("swiglu_beta", 1.0),
("swiglu_limit", 7.0),
):
torch.testing.assert_close(
call_args[name],
torch.full((2,), value, device="cuda", dtype=torch.float32),
)
@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
@pytest.mark.parametrize("e", [40, 64, 256])
@pytest.mark.parametrize("topk", [1, 6, 8])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("activation", [MoEActivation.SILU, MoEActivation.RELU2_NO_MUL])
@torch.inference_mode()
def test_flashinfer_fp4_moe_no_graph(
m: int,
n: int,
k: int,
e: int,
topk: int,
dtype: torch.dtype,
activation: MoEActivation,
workspace_init,
):
set_random_seed(7)
with set_current_vllm_config(
VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
):
a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
quant_blocksize = 16
is_gated_act = activation.is_gated
w1_q, w2_q, quant_config = make_test_quant_config(
e,
n,
k,
in_dtype=dtype,
quant_dtype="nvfp4",
block_shape=None,
per_act_token_quant=False,
make_gate=is_gated_act,
)
score = torch.randn((m, e), device="cuda", dtype=dtype)
topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False)
assert is_valid_flashinfer_cutlass_fused_moe(a, w1_q, w2_q)
moe_config = FusedMoEConfig(
num_experts=e,
experts_per_token=topk,
hidden_dim=k,
intermediate_size=n,
num_local_experts=e,
num_logical_experts=e,
activation=activation,
device="cuda",
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
in_dtype=dtype,
routing_method=RoutingMethodType.TopK,
max_num_tokens=next_power_of_2(m),
)
flashinfer_experts = FusedMoEKernel(
maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
use_monolithic=False,
),
FlashInferExperts(moe_config=moe_config, quant_config=quant_config),
)
flashinfer_output = flashinfer_experts.apply(
hidden_states=a,
w1=w1_q,
w2=w2_q,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=activation,
global_num_experts=e,
expert_map=None,
apply_router_weight_on_input=False,
)
# Reference check:
a_global_scale = (
(FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.amax(a.flatten(), dim=-1)
).to(torch.float32)
a_fp4, a_scale_interleaved = ops.scaled_fp4_quant(a, a_global_scale)
_, m_k = a_fp4.shape
a_in_dtype = dequantize_nvfp4_to_dtype(
a_fp4,
a_scale_interleaved,
a_global_scale,
dtype=a.dtype,
device=a.device,
block_size=quant_blocksize,
)
w1_d = torch.empty(
(e, (2 if is_gated_act else 1) * n, k), device="cuda", dtype=dtype
)
w2_d = torch.empty((e, k, n), device="cuda", dtype=dtype)
for idx in range(0, e):
w1_d[idx] = dequantize_nvfp4_to_dtype(
w1_q[idx],
quant_config.w1_scale[idx],
(1 / quant_config.g1_alphas[idx]),
dtype=dtype,
device=w1_q.device,
block_size=quant_blocksize,
)
w2_d[idx] = dequantize_nvfp4_to_dtype(
w2_q[idx],
quant_config.w2_scale[idx],
(1 / quant_config.g2_alphas[idx]),
dtype=dtype,
device=w2_q.device,
block_size=quant_blocksize,
)
torch_output = torch_moe(
a_in_dtype, w1_d, w2_d, score, topk, activation=activation
)
torch.testing.assert_close(
torch_output, flashinfer_output, atol=1e-1, rtol=1e-1
)
if __name__ == "__main__":
test_flashinfer_fp4_moe_no_graph((2, 1024, 1024), 40, 1, torch.half)
-369
View File
@@ -1,369 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""End-to-end tests for the FlashInfer MoE-EP adapter on real Blackwell GPUs.
Each test spawns one process per GPU, bootstraps vLLM's expert-parallel
groups, builds a ``FlashInferMoeEp`` adapter from bf16 expert weights at the
MoE geometry of a real model, warms it up and runs a forward whose inputs are
chosen so that every quantization step in the megakernel is exact, for the
CuTeDSL kernel (NVFP4 activations and weights, E4M3 scales per 16) as well as
the DeepGEMM kernel (FP8 activations and FP4 weights, power-of-two scales per
32). The expected output is then known in closed form and compared bit for
bit.
A test that needs no SM100 also checks the adapter's kwargs against the
installed FlashInfer's config types, so config drift fails on any CI runner
with FlashInfer installed.
"""
from dataclasses import dataclass
from types import SimpleNamespace
import pytest
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from tests.distributed.eplb_utils import distributed_run, set_env_vars_and_device
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.config.kernel import FLASHINFER_MOE_EP_CUTEDSL, FLASHINFER_MOE_EP_DEEP_GEMM
from vllm.distributed.parallel_state import ensure_model_parallel_initialized
from vllm.model_executor.layers.fused_moe import flashinfer_moe_ep as fi_ep
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.experts.flashinfer_moe_ep import (
FlashInferMoeEpExperts,
)
from vllm.model_executor.layers.fused_moe.flashinfer_moe_ep import (
FlashInferMoeEp,
FlashInferMoeEpWeights,
apply_topk_in_fc1,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Static
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_deep_gemm
@dataclass(frozen=True)
class Geometry:
name: str
hidden_size: int
intermediate_size: int
num_experts: int
top_k: int
# Routed-expert geometry of the models the branch was evaluated on. DeepGEMM
# keeps one UE8M0 scale byte per 32 elements and requires the per-token scale
# rows to be TMA aligned, so hidden and intermediate sizes must be multiples
# of 512 (FlashInfer's own validation only checks 128); all four qualify.
GEOMETRIES = (
Geometry("dsv4-pro", 7168, 3072, 384, 6),
Geometry("dsv4-flash", 4096, 2048, 256, 6),
Geometry("mistral-small-4", 4096, 2048, 128, 4),
Geometry("mistral-large-3", 7168, 4096, 128, 4),
)
BACKENDS = (FLASHINFER_MOE_EP_CUTEDSL, FLASHINFER_MOE_EP_DEEP_GEMM)
NUM_TOKENS_PER_RANK = 4
MAX_TOKENS_PER_RANK = 128
# Every block scale below is either 1 or a power of two, so the FP4 codes
# and the FP8 values survive quantization exactly: NVFP4 scales a 16-block by
# absmax / 6 in E4M3, DeepGEMM scales a 32-block by absmax / 6 (FP4) or
# absmax / 448 (FP8) rounded up to a power of two. A 6.0 in a block pins the
# FP4 scale to 1; 2.0, 6.0 and 144 are exact FP8 values at any such scale.
#
# Token t carries X_SIGNAL at hidden column 16 * t and a 6.0 sentinel next to
# it; every weight column the sentinel meets is zero.
X_SIGNAL = 2.0
X_SENTINEL = 6.0
# Gate and up rows are identical, so h = silu(12) * 12 = 144 = 6 * 24
# whichever half the kernel treats as the gate, and 144 requantizes exactly.
W13_DIAG = 6.0
H_SIGNAL = 144.0
# Expert e writes E2M1_CODES[e % 7] * H_SIGNAL into output channel e. The 6.0
# sentinel sits next to a zero intermediate channel and pins the block scale.
W2_SENTINEL = 6.0
E2M1_CODES = (0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0)
def _expert_out(expert: int) -> float:
return E2M1_CODES[expert % len(E2M1_CODES)]
def _topk_weights(top_k: int) -> tuple[float, ...]:
# The DeepSeek-V4 routing policy applies the router weight before fc2, so
# weight * 144 must requantize exactly too: powers of two do.
return tuple(2.0 ** -(k + 1) for k in range(top_k))
def _routing_method(backend: str) -> RoutingMethodType:
if backend == FLASHINFER_MOE_EP_DEEP_GEMM:
return RoutingMethodType.DeepseekV4
return RoutingMethodType.Default
def _routing(
geometry: Geometry, rank: int, device: torch.device
) -> tuple[torch.Tensor, torch.Tensor]:
"""top_k distinct experts per token, strided across ranks so dispatch goes
remote."""
base = (
torch.arange(NUM_TOKENS_PER_RANK, device=device) + rank * NUM_TOKENS_PER_RANK
) % geometry.num_experts
stride = geometry.num_experts // geometry.top_k
offsets = torch.arange(geometry.top_k, device=device) * stride
topk_ids = ((base[:, None] + offsets[None, :]) % geometry.num_experts).to(
torch.int32
)
topk_weights = torch.tensor(
[_topk_weights(geometry.top_k)] * NUM_TOKENS_PER_RANK,
dtype=torch.float32,
device=device,
)
return topk_ids, topk_weights
def _hidden_states(geometry: Geometry, device: torch.device) -> torch.Tensor:
x = torch.zeros(
NUM_TOKENS_PER_RANK, geometry.hidden_size, dtype=torch.bfloat16, device=device
)
for t in range(NUM_TOKENS_PER_RANK):
x[t, 16 * t] = X_SIGNAL
x[t, 16 * t + 1] = X_SENTINEL
return x
def _local_expert_weights(
geometry: Geometry, rank: int, num_local_experts: int, device: torch.device
) -> FlashInferMoeEpWeights:
w13 = torch.zeros(
num_local_experts,
2 * geometry.intermediate_size,
geometry.hidden_size,
dtype=torch.bfloat16,
device=device,
)
w2 = torch.zeros(
num_local_experts,
geometry.hidden_size,
geometry.intermediate_size,
dtype=torch.bfloat16,
device=device,
)
token_columns = [16 * t for t in range(NUM_TOKENS_PER_RANK)]
for local in range(num_local_experts):
expert = rank * num_local_experts + local
w13[local, 0, token_columns] = W13_DIAG
w13[local, geometry.intermediate_size, token_columns] = W13_DIAG
w2[local, expert, 0] = _expert_out(expert)
w2[local, expert, 2] = W2_SENTINEL
return FlashInferMoeEpWeights(w13=w13, w2=w2)
def _expected_output(
geometry: Geometry, topk_ids: torch.Tensor, topk_weights: torch.Tensor
) -> torch.Tensor:
expected = torch.zeros(
NUM_TOKENS_PER_RANK,
geometry.hidden_size,
dtype=torch.float32,
device=topk_ids.device,
)
for t in range(NUM_TOKENS_PER_RANK):
for k in range(geometry.top_k):
expert = int(topk_ids[t, k])
weight = float(topk_weights[t, k])
expected[t, expert] += weight * _expert_out(expert) * H_SIGNAL
return expected.to(torch.bfloat16)
def _moe_config(
vllm_config: VllmConfig,
geometry: Geometry,
world_size: int,
device: torch.device,
backend: str,
) -> FusedMoEConfig:
parallel = FusedMoEParallelConfig.make(
tp_size_=1,
pcp_size_=1,
dp_size_=world_size,
sp_size_=1,
vllm_parallel_config=vllm_config.parallel_config,
)
return FusedMoEConfig(
num_experts=geometry.num_experts,
experts_per_token=geometry.top_k,
hidden_dim=geometry.hidden_size,
intermediate_size=geometry.intermediate_size,
num_local_experts=geometry.num_experts // world_size,
num_logical_experts=geometry.num_experts,
activation=MoEActivation.SILU,
device=device,
routing_method=_routing_method(backend),
moe_parallel_config=parallel,
in_dtype=torch.bfloat16,
moe_backend=backend,
max_num_tokens=MAX_TOKENS_PER_RANK,
)
def _check_expert_views_alias_kernel_memory(
adapter: FlashInferMoeEp,
geometry: Geometry,
rank: int,
num_local_experts: int,
device: torch.device,
) -> None:
"""EPLB moves experts through ``expert_weight_views``; the kernel must see it.
Swap local experts 0 and 1 in place on every rank, then route through the
swapped physical slots: the output must be the one the unswapped logical
experts would have produced.
"""
for view in adapter.expert_weight_views():
first = view[0].clone()
view[0].copy_(view[1])
view[1].copy_(first)
torch.distributed.barrier()
topk_ids, topk_weights = _routing(geometry, rank, device)
slot = topk_ids % num_local_experts
physical_ids = torch.where(slot == 0, topk_ids + 1, topk_ids)
physical_ids = torch.where(slot == 1, topk_ids - 1, physical_ids)
output = adapter(_hidden_states(geometry, device), physical_ids, topk_weights)
torch.accelerator.synchronize()
assert torch.equal(output, _expected_output(geometry, topk_ids, topk_weights))
def _init_warmup_forward(
env: dict[str, str], world_size: int, geometry: Geometry, backend: str
) -> None:
set_env_vars_and_device(env)
vllm_config = VllmConfig()
vllm_config.parallel_config.data_parallel_size = world_size
vllm_config.parallel_config.enable_expert_parallel = True
vllm_config.kernel_config.moe_backend = backend
with set_current_vllm_config(vllm_config):
ensure_model_parallel_initialized(
tensor_model_parallel_size=1, pipeline_model_parallel_size=1
)
rank = torch.distributed.get_rank()
device = torch.device(f"cuda:{rank}")
moe = _moe_config(vllm_config, geometry, world_size, device, backend)
supported, reason = FlashInferMoeEpExperts.is_supported_config(
FlashInferMoeEpExperts,
moe,
kMxfp4Static,
None,
mk.FusedMoEActivationFormat.Standard,
)
assert supported, reason
adapter = FlashInferMoeEp(
moe,
_local_expert_weights(geometry, rank, moe.num_local_experts, device),
apply_topk_in_fc1=apply_topk_in_fc1(moe),
)
try:
adapter.warmup()
topk_ids, topk_weights = _routing(geometry, rank, device)
output = adapter(_hidden_states(geometry, device), topk_ids, topk_weights)
torch.accelerator.synchronize()
expected = _expected_output(geometry, topk_ids, topk_weights)
assert output.dtype == expected.dtype and output.shape == expected.shape
assert torch.equal(output, expected), (
f"rank {rank}: max abs diff "
f"{(output.float() - expected.float()).abs().max().item()}, "
f"nonzero output channels per token "
f"{[torch.nonzero(row).flatten().tolist() for row in output]}"
)
if backend == FLASHINFER_MOE_EP_CUTEDSL:
_check_expert_views_alias_kernel_memory(
adapter, geometry, rank, moe.num_local_experts, device
)
finally:
adapter.destroy()
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="The FlashInfer MoE-EP megakernels require SM100",
)
@pytest.mark.parametrize("geometry", GEOMETRIES, ids=[g.name for g in GEOMETRIES])
@pytest.mark.parametrize("backend", BACKENDS, ids=("cutedsl", "deep_gemm"))
@pytest.mark.parametrize("world_size", [2, 4])
def test_init_warmup_forward_is_exact_on_grid_inputs(
world_size: int, backend: str, geometry: Geometry
):
pytest.importorskip("flashinfer.moe_ep")
if backend == FLASHINFER_MOE_EP_DEEP_GEMM and not has_deep_gemm():
pytest.skip("DeepGEMM is not available")
if torch.accelerator.device_count() < world_size:
pytest.skip(f"Need at least {world_size} GPUs")
distributed_run(_init_warmup_forward, world_size, geometry, backend)
@pytest.mark.parametrize("backend", BACKENDS, ids=("cutedsl", "deep_gemm"))
def test_adapter_kwargs_match_installed_flashinfer(
monkeypatch: pytest.MonkeyPatch, backend: str
):
"""The adapter's kwargs construct the installed FlashInfer's config types.
Needs no SM100, so config kwarg drift fails in CI rather than at model load.
Fakes only the megakernel layer (symmetric memory), the EP group, the device
index and the DeepGEMM import.
"""
moe_ep = pytest.importorskip("flashinfer.moe_ep")
layer_args: list[tuple] = []
class FakeMegaLayer:
def __init__(self, *args) -> None:
layer_args.append(args)
def destroy(self) -> None:
pass
monkeypatch.setattr(moe_ep, "MoEEpMegaLayer", FakeMegaLayer)
monkeypatch.setattr(fi_ep, "_expose_deep_gemm_to_flashinfer", lambda: None)
monkeypatch.setattr(
fi_ep,
"get_ep_group",
lambda: SimpleNamespace(world_size=2, rank_in_group=0, device_group=None),
)
monkeypatch.setattr(torch.accelerator, "current_device_index", lambda: 0)
geometry = GEOMETRIES[0]
moe = SimpleNamespace(
moe_backend=backend,
num_experts=geometry.num_experts,
experts_per_token=geometry.top_k,
hidden_dim=geometry.hidden_size,
intermediate_size=geometry.intermediate_size,
max_num_tokens=MAX_TOKENS_PER_RANK,
swiglu_limit=10.0,
routing_method=_routing_method(backend),
)
weight = torch.zeros(1, dtype=torch.bfloat16)
FlashInferMoeEp(
moe,
FlashInferMoeEpWeights(w13=weight, w2=weight),
apply_topk_in_fc1=apply_topk_in_fc1(moe),
)
((_, _, _, mega_config),) = layer_args
expected = (
moe_ep.Nvfp4CutedslMegaMoeConfig
if backend == FLASHINFER_MOE_EP_CUTEDSL
else moe_ep.DeepGemmMegaMoeConfig
)
assert isinstance(mega_config.megakernel, expected)
-227
View File
@@ -1,227 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright (c) 2025 FlyDSL Project Contributors
import importlib.util
import pytest
import torch
from vllm.model_executor.layers.fused_moe import fused_experts
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
int4_w4a16_moe_quant_config,
)
from vllm.platforms import current_platform
from vllm.platforms.rocm import on_gfx950
from vllm.utils.torch_utils import set_random_seed
if not (current_platform.is_rocm() and on_gfx950()):
pytest.skip("This test can only run on ROCm and gfx950.", allow_module_level=True)
aiter_available = importlib.util.find_spec("aiter") is not None
if not aiter_available:
pytest.skip("These tests require AITER to run.", allow_module_level=True)
from vllm.model_executor.layers.fused_moe.fused_flydsl_moe import ( # noqa: E402
fused_flydsl_moe,
)
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa: E402, E501
compressed_tensors_moe_w4a16_flydsl,
)
RoutingBuffers = tuple[
torch.Tensor, # sorted_token_ids
torch.Tensor, # sorted_weights
torch.Tensor, # sorted_expert_ids
torch.Tensor, # num_valid_ids (shape [1], i32)
int, # sorted_size
int, # blocks
]
def _assert_flydsl_matches_reference(
actual: torch.Tensor,
expected: torch.Tensor,
*,
atol: float = 0.5,
rtol: float = 0.1,
max_mismatch_fraction: float = 1e-5,
max_error_ratio: float = 3.0,
) -> None:
"""Compare kernels without making accuracy depend on output size.
The FlyDSL and reference kernels accumulate BF16 values in different
orders. A strict allclose makes one expected rounding outlier fail an
otherwise accurate output, which becomes increasingly likely for the
largest token counts (up to 117 million output elements here).
Keep the original elementwise tolerance for 99.999% of values, while also
bounding every tolerated outlier to three times its elementwise tolerance.
"""
assert actual.shape == expected.shape
mismatch = ~torch.isclose(actual, expected, atol=atol, rtol=rtol)
mismatch_count = int(mismatch.sum().item())
total = actual.numel()
allowed_mismatches = int(total * max_mismatch_fraction)
mismatch_msg = (
f"FlyDSL/reference mismatch: {mismatch_count}/{total} values "
f"({mismatch_count / total:.6%}) exceed atol={atol}, rtol={rtol}; "
f"allowed <= {allowed_mismatches}/{total} "
f"({max_mismatch_fraction:.6%})"
)
assert mismatch_count <= allowed_mismatches, mismatch_msg
if mismatch_count:
actual_mismatch = actual[mismatch].float()
expected_mismatch = expected[mismatch].float()
abs_error = (actual_mismatch - expected_mismatch).abs()
tolerance = atol + rtol * expected_mismatch.abs()
worst_error_ratio = (abs_error / tolerance).max().item()
assert worst_error_ratio <= max_error_ratio, (
f"{mismatch_msg}; worst error is {worst_error_ratio:.4f}x its "
f"elementwise tolerance (allowed <= {max_error_ratio:.4f}x)"
)
@pytest.mark.parametrize(
"num_tokens", [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384]
)
@pytest.mark.parametrize("inter_dim", [256, 512])
def test_flydsl_moe(num_tokens: int, inter_dim: int):
set_random_seed(0)
device = "cuda"
topk = 8
num_experts = 384
hidden_size = 7168
packed_factor = 8
w13_num_shards = 2
params_dtype = torch.bfloat16
group_size = 32
w2_scales_size = inter_dim
scale_factor = 0.01
num_groups_w2 = w2_scales_size // group_size
num_groups_w13 = hidden_size // group_size
w13_weight = torch.randint(
0,
255,
(num_experts, hidden_size // packed_factor, w13_num_shards * inter_dim),
dtype=torch.int32,
device=device,
)
w2_weight = torch.randint(
0,
255,
(num_experts, inter_dim // packed_factor, hidden_size),
dtype=torch.int32,
device=device,
)
w13_scale = scale_factor * torch.randn(
num_experts,
num_groups_w13,
w13_num_shards * inter_dim,
dtype=params_dtype,
device=device,
)
w2_scale = scale_factor * torch.randn(
num_experts, num_groups_w2, hidden_size, dtype=params_dtype, device=device
)
w13_weight_packed = w13_weight.transpose(1, 2).contiguous().view(torch.uint8)
w2_weight_packed = w2_weight.transpose(1, 2).contiguous().view(torch.uint8)
w13_weight_scale = w13_scale.transpose(1, 2).contiguous()
w2_weight_scale = w2_scale.transpose(1, 2).contiguous()
moe_quant_config = int4_w4a16_moe_quant_config(
w1_scale=w13_weight_scale,
w2_scale=w2_weight_scale,
w1_zp=None,
w2_zp=None,
block_shape=[0, group_size],
)
score = torch.rand((num_tokens, num_experts), device=device, dtype=torch.float32)
topk_vals, topk_ids = torch.topk(score, k=topk, dim=1)
topk_weights = torch.softmax(topk_vals, dim=1).to(torch.float32)
x = torch.randn((num_tokens, hidden_size), dtype=torch.bfloat16, device=device)
out_ref = fused_experts(
x,
w13_weight_packed,
w2_weight_packed,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=MoEActivation.SILU,
apply_router_weight_on_input=False,
global_num_experts=num_experts,
expert_map=None,
quant_config=moe_quant_config,
)
w13 = w13_weight
w13 = compressed_tensors_moe_w4a16_flydsl._gptq_int32_to_flydsl_packed(w13)
w13 = w13.view(-1).contiguous()
w2 = w2_weight
w2 = compressed_tensors_moe_w4a16_flydsl._gptq_int32_to_flydsl_packed(w2)
w2 = w2.view(-1).contiguous()
w13_scale_flydsl = w13_scale
w2_scale_flydsl = w2_scale
if group_size > 0 and w13_scale.dim() == 3 and w13_scale.shape[1] > 1:
E, G, N = w13_scale.shape
w13_scale_flydsl = (
w13_scale_flydsl.view(E, G // 2, 2, N)
.permute(0, 1, 3, 2)
.contiguous()
.view(-1)
.contiguous()
)
elif w13_scale.dim() == 3 and w13_scale.shape[1] == 1:
w13_scale_flydsl = w13_scale_flydsl.squeeze(1)
if group_size > 0 and w2_scale.dim() == 3 and w2_scale.shape[1] > 1:
E, G, N = w2_scale.shape
w2_scale_flydsl = (
w2_scale_flydsl.view(E, G // 2, 2, N)
.permute(0, 1, 3, 2)
.contiguous()
.view(-1)
.contiguous()
)
elif w2_scale.dim() == 3 and w2_scale.shape[1] == 1:
w2_scale_flydsl = w2_scale_flydsl.squeeze(1)
w13_scale_flydsl = w13_scale_flydsl.contiguous()
w2_scale_flydsl = w2_scale_flydsl.contiguous()
w13.is_shuffled = True
w2.is_shuffled = True
out = fused_flydsl_moe(
x,
w13,
w2,
num_experts,
inter_dim,
topk_weights,
topk_ids,
w1_scale=w13_scale_flydsl,
w2_scale=w2_scale_flydsl,
topk=topk_weights.shape[-1],
group_size=group_size,
doweight_stage1=False,
scale_is_bf16=True,
)
_assert_flydsl_matches_reference(out, out_ref)
if __name__ == "__main__":
test_flydsl_moe(512, 256)
-311
View File
@@ -1,311 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
import vllm.distributed as distributed
from tests.kernels.moe.utils import make_dummy_moe_config
from vllm.model_executor.layers.fused_moe import (
unquantized_fused_moe_method as unquantized_module,
)
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
from vllm.model_executor.layers.quantization import fp8 as fp8_module
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
def _make_fp8_tp_experts(
monkeypatch,
tp_size,
tp_rank,
backend="flashinfer_trtllm",
*,
mock_backend=True,
num_experts=2,
hidden_dim=256,
intermediate_size=640,
quant_config=None,
config_overrides=None,
):
# Exercise allocation and the public weight loader without selecting a GPU kernel.
if mock_backend:
monkeypatch.setattr(
fp8_module, "select_fp8_moe_backend", lambda **kwargs: (None, None)
)
monkeypatch.setattr(
unquantized_module,
"select_unquantized_moe_backend",
lambda **kwargs: (None, None),
)
monkeypatch.setattr(
fp8_module, "get_tensor_model_parallel_world_size", lambda: tp_size
)
# The block-shape validator imports this function from vllm.distributed.
monkeypatch.setattr(
distributed, "get_tensor_model_parallel_world_size", lambda: tp_size
)
config = make_dummy_moe_config(
num_experts=num_experts,
hidden_dim=hidden_dim,
intermediate_size=intermediate_size,
experts_per_token=min(num_experts, 10),
)
config.routing_method = RoutingMethodType.RenormalizeNaive
config.moe_parallel_config.tp_size = tp_size
config.moe_parallel_config.tp_rank = tp_rank
config.intermediate_size_per_partition = intermediate_size // tp_size
config.intermediate_size_per_partition_unpadded = intermediate_size // tp_size
config.moe_backend = backend
for field, value in (config_overrides or {}).items():
setattr(config, field, value)
return RoutedExperts(
"model.layers.0.mlp.experts",
torch.bfloat16,
config,
quant_config
or Fp8Config(is_checkpoint_fp8_serialized=True, weight_block_size=[128, 128]),
expert_map_manager=SimpleNamespace(
local_num_experts=num_experts,
placement_strategy="linear",
expert_map=None,
expert_mask=None,
routing_tables=None,
map_global_to_local=lambda index: index,
),
)
@pytest.mark.parametrize("tp_size", [2, 4, 8])
@pytest.mark.parametrize("batched", [False, True])
@pytest.mark.parametrize("intermediate_size", [640, 896])
def test_fp8_block_aligned_tp_preserves_checkpoint(
monkeypatch, tp_size, batched, intermediate_size
):
"""All ranks reconstruct the original dequantized projections, including
padding-only ranks. Reloading must clear stale weights and scales.
"""
generator = torch.Generator().manual_seed(42)
weights = {
"w1": torch.randn(2, intermediate_size, 256, generator=generator).to(
torch.float8_e4m3fn
),
"w3": torch.randn(2, intermediate_size, 256, generator=generator).to(
torch.float8_e4m3fn
),
"w2": torch.randn(2, 256, intermediate_size, generator=generator).to(
torch.float8_e4m3fn
),
}
scales = {
name: torch.rand(2, w.shape[1] // 128, w.shape[2] // 128, generator=generator)
+ 0.1
for name, w in weights.items()
}
reconstructed: dict[str, list[torch.Tensor]] = {name: [] for name in weights}
num_blocks = intermediate_size // 128
for rank in range(tp_size):
layer = _make_fp8_tp_experts(
monkeypatch, tp_size, rank, intermediate_size=intermediate_size
)
assert layer.quant_method.weight_scale_refine is None
assert layer.quant_method.moe_block_shape == [128, 128]
width = layer.moe_config.intermediate_size_per_partition
assert width == ((num_blocks + tp_size - 1) // tp_size) * 128
for name in weights:
prefix = "w2" if name == "w2" else "w13"
for suffix, checkpoint in (
("weight", weights[name]),
("weight_scale_inv", scales[name]),
):
param_name = f"{prefix}_{suffix}"
param = getattr(layer, param_name)
# Also reload after dirtying the destination, to catch padding leaks.
for _ in range(2):
if name != "w3":
param.data.fill_(7)
if batched:
assert param.weight_loader(
param, checkpoint, param_name, name, 0, return_success=True
)
else:
for expert in range(2):
assert param.weight_loader(
param,
checkpoint[expert],
param_name,
name,
expert,
return_success=True,
)
w = getattr(layer, f"{prefix}_weight").float()
s = getattr(layer, f"{prefix}_weight_scale_inv")
if name in ("w1", "w3"):
half = 0 if name == "w1" else 1
w = w.chunk(2, dim=1)[half]
s = s.chunk(2, dim=1)[half]
dim = 2 if name == "w2" else 1
valid = max(0, min(width, intermediate_size - rank * width))
assert torch.count_nonzero(w.narrow(dim, valid, width - valid)) == 0
assert torch.all(s.narrow(dim, valid // 128, (width - valid) // 128) == 1)
dequant = w * s.repeat_interleave(128, dim=1).repeat_interleave(128, dim=2)
reconstructed[name].append(dequant.narrow(dim, 0, valid))
for name, parts in reconstructed.items():
expected = weights[name].float() * scales[name].repeat_interleave(
128, dim=1
).repeat_interleave(128, dim=2)
assert torch.equal(torch.cat(parts, dim=2 if name == "w2" else 1), expected)
@pytest.mark.parametrize("backend", ["auto", "triton"])
@pytest.mark.parametrize("tp_size,block", [(2, 64), (4, 32)])
def test_fp8_tp_default_keeps_refined_layout(monkeypatch, backend, tp_size, block):
layer = _make_fp8_tp_experts(monkeypatch, tp_size, 0, backend)
assert layer.moe_config.intermediate_size_per_partition == 640 // tp_size
assert layer.quant_method.moe_block_shape == [block, block]
@pytest.mark.parametrize(
"field,value",
[
("hidden_dim", 192),
("intermediate_size", 672),
("is_lora_enabled", True),
("has_bias", True),
],
)
def test_fp8_block_aligned_tp_rejects_unsupported_layout(monkeypatch, field, value):
with pytest.raises(ValueError, match="Block-aligned FP8 TP sharding requires"):
_make_fp8_tp_experts(monkeypatch, 4, 0, config_overrides={field: value})
@pytest.mark.parametrize("tp_size", [1, 5])
def test_fp8_aligned_tp_keeps_original_layout(monkeypatch, tp_size):
layer = _make_fp8_tp_experts(monkeypatch, tp_size, 0)
assert layer.moe_config.intermediate_size_per_partition == 640 // tp_size
assert not layer.moe_config.tp_shard_with_padding
assert layer.quant_method.weight_scale_refine is None
def test_fp8_skipped_layer_keeps_original_tp_layout(monkeypatch, default_vllm_config):
layer = _make_fp8_tp_experts(
monkeypatch,
4,
0,
quant_config=Fp8Config(
is_checkpoint_fp8_serialized=True,
weight_block_size=[128, 128],
ignored_layers=["model.layers.0.mlp.experts"],
),
)
assert isinstance(layer.quant_method, unquantized_module.UnquantizedFusedMoEMethod)
assert not layer.moe_config.tp_shard_with_padding
assert layer.moe_config.intermediate_size_per_partition == 160
assert layer.w13_weight.shape == (2, 320, 256)
assert layer.w2_weight.shape == (2, 256, 160)
def test_fp8_block_aligned_tp_rejects_presharded_weights(monkeypatch):
layer = _make_fp8_tp_experts(monkeypatch, 4, 0)
with pytest.raises(ValueError, match="unsharded checkpoint"):
layer.w13_weight.weight_loader(
layer.w13_weight,
torch.zeros(160, 256, dtype=torch.float8_e4m3fn),
"w13_weight",
"w1",
0,
)
@pytest.mark.parametrize("tp_size", [2, 4, 8])
@pytest.mark.parametrize("num_tokens", [1, 17])
@torch.inference_mode()
def test_fp8_block_aligned_tp_flashinfer_matches_unsharded(
monkeypatch,
tp_size,
num_tokens,
dist_init,
workspace_init,
):
"""Compare sharded FI with unsharded FI, reporting the cross-backend
difference separately because activation quantization can differ.
"""
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe.config import fp8_w8a8_moe_quant_config
from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts
from vllm.platforms import current_platform
if not current_platform.is_device_capability_family(100):
pytest.skip("Requires datacenter Blackwell and FlashInfer TRTLLM")
torch.manual_seed(42)
e, h, n, topk = 16, 2560, 640, 10
with torch.device("cuda"), set_current_vllm_config(VllmConfig()):
x = torch.randn(num_tokens, h, dtype=torch.bfloat16) / 10
logits = torch.randn(num_tokens, e, dtype=torch.float32)
probabilities = logits.softmax(dim=-1)
topk_weights, topk_ids = probabilities.topk(topk, dim=-1)
topk_weights /= topk_weights.sum(dim=-1, keepdim=True)
weights = {
"w1": torch.randn(e, n, h).to(torch.float8_e4m3fn),
"w3": torch.randn(e, n, h).to(torch.float8_e4m3fn),
"w2": torch.randn(e, h, n).to(torch.float8_e4m3fn),
}
scales = {
name: (torch.rand(e, w.shape[1] // 128, w.shape[2] // 128) + 0.5) / 32
for name, w in weights.items()
}
ref = fused_experts(
x,
torch.cat([weights["w1"], weights["w3"]], dim=1),
weights["w2"],
topk_weights,
topk_ids,
quant_config=fp8_w8a8_moe_quant_config(
w1_scale=torch.cat([scales["w1"], scales["w3"]], dim=1),
w2_scale=scales["w2"],
block_shape=[128, 128],
),
)
total = torch.zeros_like(ref, dtype=torch.float32)
fi_reference = None
for rank in range(-1, tp_size):
layer = _make_fp8_tp_experts(
monkeypatch,
1 if rank == -1 else tp_size,
max(rank, 0),
mock_backend=False,
num_experts=e,
hidden_dim=h,
)
for name in weights:
prefix = "w2" if name == "w2" else "w13"
for suffix, checkpoint in (
("weight", weights[name]),
("weight_scale_inv", scales[name]),
):
param_name = f"{prefix}_{suffix}"
param = getattr(layer, param_name)
param.weight_loader(param, checkpoint, param_name, name, 0)
method = layer.quant_method
method.process_weights_after_loading(layer)
assert method.is_monolithic
out = method.apply_monolithic(layer, x, logits)
assert torch.isfinite(out).all()
if rank == -1:
fi_reference = out.float().clone()
else:
total += out.float()
# Different FP8 GEMM reduction orders need not be bitwise identical.
assert fi_reference is not None
relative_l2 = (total - fi_reference).norm() / fi_reference.norm().clamp_min(
1e-8
)
cross_backend_l2 = (
fi_reference - ref.float()
).norm() / ref.float().norm().clamp_min(1e-8)
print(
f"TP{tp_size} M={num_tokens}: shard_l2={relative_l2.item():.6f}, "
f"unsharded_fi_vs_triton_l2={cross_backend_l2.item():.6f}"
)
assert relative_l2 < 0.01, relative_l2.item()
@@ -1,229 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness of the fused globalize + align + sort decode metadata kernel.
The fused op replaces the DeepEP-v2 INDEXED cudagraph-decode chain
(_globalize_recv_topk_idx + moe_align_block_size + count_and_sort). It is
checked against that exact reference chain over a grid of shapes / EP ranks /
fill distributions.
"""
import pytest
import torch
from vllm.model_executor.layers.fused_moe.fused_globalize_align_block_size import (
fused_globalize_align_block_size,
)
from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
moe_align_block_size,
)
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_deep_ep_v2
if current_platform.is_cuda() and has_deep_ep_v2():
from vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_v2 import (
_globalize_recv_topk_idx,
)
else:
pytest.skip("Requires CUDA and DeepEP v2", allow_module_level=True)
def _expert_map(global_num_experts, rank_expert_offset, local_num_experts, device):
expert_map = torch.full((global_num_experts,), -1, device=device, dtype=torch.int32)
rng = torch.arange(local_num_experts, device=device, dtype=torch.int32)
expert_map[rank_expert_offset : rank_expert_offset + local_num_experts] = rng
return expert_map
def _make_case(n, topk, ep_size, local_e, block_size, seed, frac_recv, rank, fill):
torch.manual_seed(seed)
device = "cuda"
global_num_experts = local_e * ep_size
rank_idx = {"first": 0, "mid": ep_size // 2, "last": ep_size - 1}[rank]
rank_expert_offset = rank_idx * local_e
if fill == "all_nonlocal":
local_idx = torch.full((n, topk), -1, device=device, dtype=torch.int64)
elif fill == "single":
local_idx = torch.zeros((n, topk), device=device, dtype=torch.int64)
elif fill == "all_local":
local_idx = torch.randint(
0, local_e, (n, topk), device=device, dtype=torch.int64
)
else:
local_idx = torch.randint(
-1, local_e, (n, topk), device=device, dtype=torch.int64
)
num_recv = max(0, min(n, int(round(n * frac_recv))))
if num_recv < n:
local_idx[num_recv:] = torch.randint(
-1, local_e, (n - num_recv, topk), device=device, dtype=torch.int64
)
psum = torch.zeros(4, device=device, dtype=torch.int32)
psum[-1] = num_recv
return local_idx, psum, rank_expert_offset, global_num_experts
def _group_by_expert(sorted_ids, expert_ids, block_size, valid_len, total):
groups: dict[int, list[int]] = {}
for b in range(valid_len // block_size):
eid = int(expert_ids[b].item())
blk = sorted_ids[b * block_size : (b + 1) * block_size]
groups.setdefault(eid, []).extend(blk[blk < total].tolist())
return groups
@pytest.mark.parametrize("n", [1, 7, 64, 257])
# topk=16 with local_num_experts=28 is the Kimi K3 decode config. topk=6 and
# block_size=16, 128 are outside the old template dispatch table: they guard
# against a regression to fixed (topk, block_size) specialization.
@pytest.mark.parametrize("topk", [1, 6, 8, 16])
@pytest.mark.parametrize("block_size", [16, 32, 64, 128])
@pytest.mark.parametrize("local_e", [28, 32])
@pytest.mark.parametrize("rank", ["first", "mid", "last"])
@pytest.mark.parametrize("fill", ["rand", "all_local", "all_nonlocal", "single"])
def test_fused_matches_reference(n, topk, block_size, local_e, rank, fill):
ep_size = 4
local_idx, psum, reo, gne = _make_case(
n,
topk,
ep_size,
local_e,
block_size,
seed=n * 131 + topk,
frac_recv=0.75,
rank=rank,
fill=fill,
)
ref_global = _globalize_recv_topk_idx(local_idx.clone(), psum, reo, gne)
ref_sorted, ref_expert_ids, ref_num = moe_align_block_size(
topk_ids=ref_global,
block_size=block_size,
num_experts=gne,
expert_map=_expert_map(gne, reo, local_e, local_idx.device),
ignore_invalid_experts=True,
)
fused_global, fused_sorted, fused_expert_ids, fused_num = (
fused_globalize_align_block_size(
local_idx.clone(), psum, reo, gne, local_e, block_size
)
)
total = n * topk
assert torch.equal(fused_global, ref_global)
assert torch.equal(fused_num, ref_num)
valid = int(fused_num.item())
nb = valid // block_size
assert torch.equal(fused_expert_ids[:nb], ref_expert_ids[:nb])
assert bool((fused_expert_ids[nb:] == -1).all())
assert bool((fused_sorted[valid:] == total).all())
ref_groups = _group_by_expert(
ref_sorted, ref_expert_ids, block_size, int(ref_num.item()), total
)
fused_groups = _group_by_expert(
fused_sorted, fused_expert_ids, block_size, valid, total
)
assert set(ref_groups) == set(fused_groups)
for eid in ref_groups:
assert sorted(ref_groups[eid]) == sorted(fused_groups[eid])
@pytest.mark.parametrize("n", [7, 257])
def test_fused_scatter_idx_matches_reference(n):
"""The optional scatter output masks invalid and padded routes."""
local_e = 28
local_idx, psum, reo, gne = _make_case(
n, 16, 4, local_e, 64, seed=n, frac_recv=0.75, rank="mid", fill="rand"
)
ref_global = _globalize_recv_topk_idx(local_idx.clone(), psum, reo, gne)
*_, ref_scatter = moe_align_block_size(
topk_ids=ref_global,
block_size=64,
num_experts=gne,
expert_map=_expert_map(gne, reo, local_e, local_idx.device),
ignore_invalid_experts=True,
return_scatter_idx=True,
)
fused_global, *_, fused_scatter = fused_globalize_align_block_size(
local_idx.clone(), psum, reo, gne, local_e, 64, return_scatter_idx=True
)
assert torch.equal(fused_global, ref_global)
assert torch.equal(fused_scatter, ref_scatter)
# These have effective_work 1504 (single block) and 6068 (cooperative).
@pytest.mark.parametrize("n", [1, 64])
def test_fused_cuda_graph_replay(n):
"""Both launch paths remain capturable and consume replay-time inputs."""
topk = 16
local_e = 28
block_size = 64
ep_size = 4
local_idx, psum, reo, gne = _make_case(
n,
topk,
ep_size,
local_e,
block_size,
seed=n,
frac_recv=0.75,
rank="mid",
fill="rand",
)
graph_idx = torch.empty_like(local_idx)
graph_psum = torch.empty_like(psum)
# Initialize lazy CUDA state before capture.
fused_globalize_align_block_size(
local_idx.clone(), psum, reo, gne, local_e, block_size
)
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
graph_idx.copy_(local_idx)
graph_psum.copy_(psum)
graph_outputs = fused_globalize_align_block_size(
graph_idx, graph_psum, reo, gne, local_e, block_size
)
for frac_recv in (0.75, 0.5):
psum[-1] = int(round(n * frac_recv))
ref_idx = _globalize_recv_topk_idx(local_idx.clone(), psum, reo, gne)
ref_sorted, ref_expert_ids, ref_num = moe_align_block_size(
topk_ids=ref_idx,
block_size=block_size,
num_experts=gne,
expert_map=_expert_map(gne, reo, local_e, local_idx.device),
ignore_invalid_experts=True,
)
graph.replay()
torch.accelerator.synchronize()
fused_idx, fused_sorted, fused_expert_ids, fused_num = graph_outputs
assert torch.equal(fused_idx, ref_idx)
assert torch.equal(fused_num, ref_num)
valid = int(fused_num.item())
num_blocks = valid // block_size
assert torch.equal(fused_expert_ids[:num_blocks], ref_expert_ids[:num_blocks])
fused_groups = _group_by_expert(
fused_sorted, fused_expert_ids, block_size, valid, n * topk
)
ref_groups = _group_by_expert(
ref_sorted,
ref_expert_ids,
block_size,
int(ref_num.item()),
n * topk,
)
assert set(fused_groups) == set(ref_groups)
for eid in ref_groups:
assert sorted(fused_groups[eid]) == sorted(ref_groups[eid])
-277
View File
@@ -1,277 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the MoE fused topk kernel.
Run `pytest tests/kernels/moe/test_fused_topk.py`.
"""
import pytest
import torch
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
fused_topk_bias,
)
from vllm.model_executor.layers.fused_moe.router.fused_topk_router import fused_topk
from vllm.platforms import current_platform
def torch_topk(
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
e_score_correction_bias: torch.Tensor = None,
scoring_func: str = "softmax",
):
if scoring_func == "softmax":
scores = torch.softmax(gating_output.float(), dim=-1)
else:
assert scoring_func == "sigmoid"
scores = torch.sigmoid(gating_output.float())
if e_score_correction_bias is not None:
num_experts = gating_output.shape[-1]
scores_for_choice = scores.view(
-1, num_experts
) + e_score_correction_bias.unsqueeze(0)
_, topk_ids = torch.topk(scores_for_choice, k=topk, dim=-1)
topk_weights = scores.gather(1, topk_ids)
else:
topk_weights, topk_ids = torch.topk(scores, k=topk, dim=-1)
if renormalize:
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
return topk_weights, topk_ids
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="This test is skipped on non-CUDA platform.",
)
@pytest.mark.parametrize("num_tokens", [1, 33, 56])
@pytest.mark.parametrize("hidden_size", [1024, 2048])
@pytest.mark.parametrize("num_experts", [6, 16])
@pytest.mark.parametrize("topk", [3, 4])
@pytest.mark.parametrize("renormalize", [True, False])
@pytest.mark.parametrize("scoring_func", ["softmax", "sigmoid"])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half, torch.float32])
def test_fused_topk(
num_tokens: int,
hidden_size: int,
num_experts: int,
topk: int,
renormalize: bool,
scoring_func: str,
dtype: torch.dtype,
):
torch.manual_seed(0)
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
topk_weights_ref, topk_ids_ref = torch_topk(
gating_output=gating_output,
topk=topk,
renormalize=renormalize,
scoring_func=scoring_func,
)
topk_weights, topk_ids, _ = fused_topk(
hidden_states=hidden_states,
gating_output=gating_output,
topk=topk,
renormalize=renormalize,
scoring_func=scoring_func,
)
torch.testing.assert_close(
topk_weights_ref.to(torch.float32), topk_weights, atol=1e-2, rtol=1e-2
)
torch.testing.assert_close(topk_ids_ref.to(torch.int32), topk_ids, atol=0, rtol=0)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="This test is skipped on non-CUDA platform.",
)
@pytest.mark.parametrize("num_tokens", [1, 33, 56])
@pytest.mark.parametrize("hidden_size", [1024, 2048])
@pytest.mark.parametrize("num_experts", [6, 16])
@pytest.mark.parametrize("topk", [3, 4])
@pytest.mark.parametrize("renormalize", [True, False])
@pytest.mark.parametrize("scoring_func", ["softmax", "sigmoid"])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half, torch.float32])
def test_fused_topk_bias(
num_tokens: int,
hidden_size: int,
num_experts: int,
topk: int,
renormalize: bool,
scoring_func: str,
dtype: torch.dtype,
):
torch.manual_seed(0)
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
e_score_correction_bias = torch.randn(
(num_experts,), dtype=torch.float32, device="cuda"
)
topk_weights_ref, topk_ids_ref = torch_topk(
gating_output=gating_output,
topk=topk,
renormalize=renormalize,
e_score_correction_bias=e_score_correction_bias,
scoring_func=scoring_func,
)
topk_weights, topk_ids = fused_topk_bias(
hidden_states=hidden_states,
gating_output=gating_output,
e_score_correction_bias=e_score_correction_bias,
topk=topk,
renormalize=renormalize,
scoring_func=scoring_func,
)
torch.testing.assert_close(
topk_weights_ref.to(torch.float32), topk_weights, atol=1e-2, rtol=1e-2
)
torch.testing.assert_close(topk_ids_ref.to(torch.int32), topk_ids, atol=0, rtol=0)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="This test is skipped on non-CUDA platform.",
)
@pytest.mark.parametrize("num_experts", [6, 8, 16])
@pytest.mark.parametrize("topk", [3, 4])
@pytest.mark.parametrize("scoring_func", ["softmax", "sigmoid"])
@pytest.mark.parametrize("bad_value", [float("nan"), float("inf")])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half, torch.float32])
def test_fused_topk_nan_inf_clamp(
num_experts: int,
topk: int,
scoring_func: str,
bad_value: float,
dtype: torch.dtype,
):
"""Regression test for the NaN/Inf clamp in topk_softmax_kernels.cu.
Degenerate hidden states (e.g., from CUDA graph padding) can produce
NaN/Inf gating logits. Without the clamp, softmax/sigmoid outputs are
NaN and the argmax loop picks expert 0 for every top-k slot (since
"NaN > NaN" is false per IEEE 754), yielding duplicate expert IDs that
crash downstream MoE sort kernels. The fix clamps NaN/Inf to 0 before
argmax so index tie-breaking selects unique experts [0, 1, ..., k-1].
"""
torch.manual_seed(0)
num_tokens = 4
hidden_size = 1024
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
# Row 0: all normal. Rows 1-3: fully poisoned with NaN or Inf.
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
gating_output[1:, :] = bad_value
topk_weights, topk_ids, _ = fused_topk(
hidden_states=hidden_states,
gating_output=gating_output,
topk=topk,
renormalize=False,
scoring_func=scoring_func,
)
# Normal row must still match the torch reference.
ref_weights, ref_ids = torch_topk(
gating_output=gating_output[:1],
topk=topk,
renormalize=False,
scoring_func=scoring_func,
)
torch.testing.assert_close(
ref_weights.to(torch.float32), topk_weights[:1], atol=1e-2, rtol=1e-2
)
torch.testing.assert_close(ref_ids.to(torch.int32), topk_ids[:1], atol=0, rtol=0)
# Poisoned rows: IDs must be unique (no duplicates) and weights must be
# finite (no NaN/Inf propagation into downstream MoE kernels).
for row in range(1, num_tokens):
row_ids = topk_ids[row]
assert row_ids.unique().numel() == topk, (
f"Row {row} has duplicate expert IDs {row_ids.tolist()} "
f"(bad_value={bad_value}, scoring_func={scoring_func})"
)
assert torch.isfinite(topk_weights[row]).all(), (
f"Row {row} has non-finite weights {topk_weights[row].tolist()} "
f"(bad_value={bad_value}, scoring_func={scoring_func})"
)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="This test is skipped on non-CUDA platform.",
)
@pytest.mark.parametrize("num_experts", [6, 8, 16])
@pytest.mark.parametrize("topk", [3, 4])
@pytest.mark.parametrize("scoring_func", ["softmax", "sigmoid"])
@pytest.mark.parametrize("bad_value", [float("nan"), float("inf")])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half, torch.float32])
def test_fused_topk_bias_nan_inf_clamp(
num_experts: int,
topk: int,
scoring_func: str,
bad_value: float,
dtype: torch.dtype,
):
"""Regression test: NaN/Inf in gating logits must not produce duplicate
expert IDs or non-finite weights when e_score_correction_bias is present.
Same scenario as test_fused_topk_nan_inf_clamp but exercising the bias
path (fused_topk_bias) so the fix in topk_softmax_kernels.cu is covered
for that entry point as well.
"""
torch.manual_seed(0)
num_tokens = 4
hidden_size = 1024
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
e_score_correction_bias = torch.randn(
(num_experts,), dtype=torch.float32, device="cuda"
)
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
gating_output[1:, :] = bad_value
topk_weights, topk_ids = fused_topk_bias(
hidden_states=hidden_states,
gating_output=gating_output,
e_score_correction_bias=e_score_correction_bias,
topk=topk,
renormalize=False,
scoring_func=scoring_func,
)
# Normal row must still match the torch reference.
ref_weights, ref_ids = torch_topk(
gating_output=gating_output[:1],
topk=topk,
renormalize=False,
e_score_correction_bias=e_score_correction_bias,
scoring_func=scoring_func,
)
torch.testing.assert_close(
ref_weights.to(torch.float32), topk_weights[:1], atol=1e-2, rtol=1e-2
)
torch.testing.assert_close(ref_ids.to(torch.int32), topk_ids[:1], atol=0, rtol=0)
# Poisoned rows: IDs must be unique (no duplicates) and weights must be
# finite (no NaN/Inf propagation into downstream MoE kernels).
for row in range(1, num_tokens):
row_ids = topk_ids[row]
assert row_ids.unique().numel() == topk, (
f"Row {row} has duplicate expert IDs {row_ids.tolist()} "
f"(bad_value={bad_value}, scoring_func={scoring_func})"
)
assert torch.isfinite(topk_weights[row]).all(), (
f"Row {row} has non-finite weights {topk_weights[row].tolist()} "
f"(bad_value={bad_value}, scoring_func={scoring_func})"
)
@@ -1,513 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass, fields, is_dataclass
import pytest
import torch
import torch.nn.functional as F
from vllm.platforms import current_platform
from vllm.utils.import_utils import get_triton_kernels_version, has_triton_kernels
if not has_triton_kernels():
pytest.skip(
"triton_kernels not found, skipping all related tests",
allow_module_level=True,
)
import triton_kernels.swiglu
if get_triton_kernels_version() == "3.8":
# 3.8: matmul_ogs -> matmul, matmul_ogs_details -> matmul_details
import triton_kernels.matmul_details.opt_flags as opt_flags
from triton_kernels.matmul import FlexCtx, PrecisionConfig
else:
import triton_kernels.matmul_ogs_details.opt_flags as opt_flags
from triton_kernels.matmul_ogs import FlexCtx, PrecisionConfig
from triton_kernels.numerics import InFlexData
from triton_kernels.numerics_details.mxfp import downcast_to_mxfp, upcast_from_mxfp
from triton_kernels.tensor import FP4, convert_layout, wrap_torch_tensor
from triton_kernels.tensor_details import layout
from triton_kernels.testing import assert_close
from vllm.model_executor.layers.fused_moe.config import mxfp4_w4a16_moe_quant_config
from vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe import (
_sm12x_block_m,
triton_kernel_moe_forward,
)
from vllm.model_executor.layers.fused_moe.oracle import mxfp4 as mxfp4_oracle
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import mx_scale_kwargs
from vllm.utils.math_utils import round_up
from .utils import mxfp4_w_layouts, shuffle_weight
def deshuffle(w: torch.Tensor):
first = w[..., ::2]
second = w[..., 1::2]
deshuffled = torch.concat((first, second), dim=-1)
return deshuffled
def init_compute_data(M, K, N, E, a_dtype: str, w_dtype: str, num_warps: int):
randbits = [torch.randperm(E) for _ in range(M)]
x_list = [
(-1) ** i
* ((16384 + ((i * 512) % 4096) + bits).to(torch.int16).view(torch.bfloat16))
for i, bits in enumerate(randbits)
]
exp_data = torch.stack(x_list).to(device="cuda") # simulating gate_output (M, E)
# create input tensor
x = torch.randn((M, K), dtype=torch.bfloat16, device="cuda")
w1 = torch.randn((E, 2 * N, K), dtype=torch.bfloat16, device="cuda")
w1_bias = torch.randn((E, 2 * N), dtype=torch.bfloat16, device="cuda")
w2 = torch.randn((E, K, N), dtype=torch.bfloat16, device="cuda")
w2_bias = torch.randn((E, K), dtype=torch.bfloat16, device="cuda")
exp_data_tri = exp_data.clone()
x_tri = x.clone()
w1_tri = w1.clone()
w2_tri = w2.clone()
w1_bias_tri = w1_bias.clone()
w2_bias_tri = w2_bias.clone()
w1_bias_tri = w1_bias_tri.to(torch.float32)
w2_bias_tri = w2_bias_tri.to(torch.float32)
dtype_dict = {
"bf16": torch.bfloat16,
"fp8_e4m3": torch.float8_e4m3fn,
"fp8_e5m2": torch.float8_e5m2,
}
x = x.to(dtype_dict[a_dtype]).to(torch.bfloat16)
if w_dtype != "mx4":
# simulate quantization support on reference impl
w1 = w1.to(dtype_dict[w_dtype]).to(torch.bfloat16)
w2 = w2.to(dtype_dict[w_dtype]).to(torch.bfloat16)
# triton moe kernel use transposed shape for matmul
w1_tri = w1_tri.transpose(-2, -1)
w2_tri = w2_tri.transpose(-2, -1)
# shuffle weights
w1_tri = shuffle_weight(w1_tri)
w1_bias_tri = shuffle_weight(w1_bias_tri)
# quant triton_weights
x_tri = x.to(dtype_dict[a_dtype])
if w_dtype != "mx4":
pytest.skip("NYI")
else: # quantize to mx4
# Padding alignment depends on the platform. On CDNA4 the scale
# swizzle requires SCALE_K % 8 == 0 (K % 256) and
# SCALE_N % 32 == 0 (2*N % 512), matching the production
# alignment in mxfp4_round_up_hidden_size_and_intermediate_size.
# On CUDA (Hopper) the scale layout pads internally, so the
# original 64/128 alignment is sufficient.
if current_platform.is_rocm():
k_align, n2_align = 256, 512
else:
k_align, n2_align = 64, 128
w1_bottom_pad = round_up(w1_tri.shape[1], k_align) - w1_tri.shape[1]
w1_right_pad = round_up(w1_tri.shape[2], n2_align) - w1_tri.shape[2]
w2_bottom_pad = w1_right_pad // 2
w2_right_pad = w1_bottom_pad
x_pad = w1_bottom_pad
w1_tri = F.pad(
w1_tri,
(0, w1_right_pad, 0, w1_bottom_pad, 0, 0),
mode="constant",
value=0,
)
w2_tri = F.pad(
w2_tri,
(0, w2_right_pad, 0, w2_bottom_pad, 0, 0),
mode="constant",
value=0,
)
w1_bias_tri = F.pad(
w1_bias_tri, (0, w1_right_pad, 0, 0), mode="constant", value=0
)
w2_bias_tri = F.pad(
w2_bias_tri, (0, w2_right_pad, 0, 0), mode="constant", value=0
)
x_tri = F.pad(x_tri, (0, x_pad, 0, 0), mode="constant", value=0)
(
w_layout,
w_layout_opts,
w_scale_layout,
w_scale_layout_opts,
) = mxfp4_w_layouts(mx_axis=1, num_warps=num_warps)
w1_tri, w1_scale_tri = downcast_to_mxfp(w1_tri, torch.uint8, axis=1)
w1 = upcast_from_mxfp(w1_tri, w1_scale_tri, torch.bfloat16, axis=1)
w2_tri, w2_scale_tri = downcast_to_mxfp(w2_tri, torch.uint8, axis=1)
w2 = upcast_from_mxfp(w2_tri, w2_scale_tri, torch.bfloat16, axis=1)
w1_tri = convert_layout(
wrap_torch_tensor(w1_tri, FP4), w_layout, **w_layout_opts
)
w1_scale_tri = convert_layout(
wrap_torch_tensor(w1_scale_tri),
w_scale_layout,
**w_scale_layout_opts,
)
w2_tri = convert_layout(
wrap_torch_tensor(w2_tri, FP4), w_layout, **w_layout_opts
)
w2_scale_tri = convert_layout(
wrap_torch_tensor(w2_scale_tri),
w_scale_layout,
**w_scale_layout_opts,
)
pc1 = PrecisionConfig(
**mx_scale_kwargs(w1_scale_tri), flex_ctx=FlexCtx(rhs_data=InFlexData())
)
pc2 = PrecisionConfig(
**mx_scale_kwargs(w2_scale_tri), flex_ctx=FlexCtx(rhs_data=InFlexData())
)
# tucuate so the rest can run properly
w1 = w1[..., :K, : 2 * N]
w2 = w2[..., :N, :K]
w1 = deshuffle(w1)
w1 = w1.transpose(-1, -2).contiguous()
w2 = w2.transpose(-1, -2).contiguous()
return (
x,
w1,
w1_bias,
w2,
w2_bias,
exp_data,
x_tri,
w1_tri,
w2_tri,
exp_data_tri,
w1_bias_tri,
w2_bias_tri,
pc1,
pc2,
)
@dataclass
class ModelConfig:
num_hidden_layers: int = 36
num_experts: int = 128
experts_per_token: int = 4
vocab_size: int = 201088
hidden_size: int = 2880
intermediate_size: int = 2880
head_dim: int = 64
num_attention_heads: int = 64
num_key_value_heads: int = 8
sliding_window: int = 128
initial_context_length: int = 4096
rope_theta: float = 150000.0
rope_parameters_factor: float = 32.0
rope_ntk_alpha: float = 1.0
rope_ntk_beta: float = 32.0
def swiglu(x, alpha: float = 1.702, limit: float = 1.0):
# Note we add an extra bias of 1 to the linear layer
x_glu, x_linear = torch.chunk(x, 2, dim=-1)
if limit is not None:
x_glu = x_glu.clamp(max=limit)
out_glu = x_glu * torch.sigmoid(alpha * x_glu)
if limit is not None:
x_linear = x_linear.clamp(min=-limit, max=limit)
return out_glu * (x_linear + 1)
def oai_moe_forward(
hidden_states: torch.Tensor, # (M, K)
w1: torch.Tensor, # (E, 2N)
w1_bias: torch.Tensor, # (E, 2N, K)
w2: torch.Tensor, # (E, K, N)
w2_bias: torch.Tensor, # (E, N)
gating_output: torch.Tensor, # (M, E)
topk: int,
):
# model.py 309:330, assuming gating and norm
t = hidden_states
experts = torch.topk(gating_output, k=topk, dim=-1, sorted=True)
expert_weights = torch.nn.functional.softmax(experts.values, dim=1)
expert_indices = experts.indices
# MLP #1
mlp1_weight = w1[expert_indices, ...]
mlp1_bias = w1_bias[expert_indices, ...]
t = torch.einsum("beck,bk->bec", mlp1_weight, t) + mlp1_bias
t = swiglu(t, limit=7)
# MLP #2
mlp2_weight = w2[expert_indices, ...]
mlp2_bias = w2_bias[expert_indices, ...]
t = torch.einsum("beck,bek->bec", mlp2_weight, t)
t += mlp2_bias
# Weighted sum of experts
t = torch.einsum("bec,be->bc", t, expert_weights)
return t
@dataclass
class Case:
a_dtype: str
w_dtype: str
@pytest.mark.parametrize(
", ".join(f.name for f in fields(Case)),
[
tuple(getattr(case, f.name) for f in fields(Case))
for case in [
# Case(a_dtype="bf16", w_dtype="bf16"),
# Case(a_dtype="fp8_e4m3", w_dtype="fp8_e5m2"),
Case(a_dtype="bf16", w_dtype="mx4")
]
],
)
@pytest.mark.parametrize("num_token", [2])
@pytest.mark.parametrize("tp", [1, 2, 4, 8])
def test_equiv(num_token, a_dtype, w_dtype, tp, workspace_init):
if not hasattr(layout, "make_default_matmul_mxfp4_w_layout"):
pytest.skip("make_default_matmul_mxfp4_w_layout not available")
M = num_token
E = ModelConfig.num_experts
K = ModelConfig.hidden_size
N = ModelConfig.intermediate_size // tp
topk = ModelConfig.experts_per_token
(
x,
w1,
w1_bias,
w2,
w2_bias,
exp_data,
x_tri,
w1_tri,
w2_tri,
exp_data_tri,
w1_bias_tri,
w2_bias_tri,
pc1,
pc2,
) = init_compute_data(M, K, N, E, a_dtype, w_dtype, num_warps=8)
if current_platform.is_device_capability_family(
100
) or current_platform.is_device_capability_family(120):
# Mirrors `_swizzle_mxfp4`; SM12x also needs epilogue_subtile=1.
constraints = {
"is_persistent": True,
"epilogue_subtile": 1,
}
opt_flags.update_opt_flags_constraints(constraints)
if a_dtype == "bf16" and w_dtype == "mx4":
quant_config = mxfp4_w4a16_moe_quant_config(
w1_scale=pc1,
w2_scale=pc2,
w1_bias=w1_bias_tri,
w2_bias=w2_bias_tri,
)
else:
raise NotImplementedError(
f"Quantization configuration for activation={a_dtype} and weight={w_dtype} "
f"has not been implemented."
)
out_triton_monolithic = triton_kernel_moe_forward(
hidden_states=x_tri,
w1=w1_tri,
w2=w2_tri,
gating_output=exp_data_tri,
topk=topk,
renormalize=True,
quant_config=quant_config,
)
out_triton_monolithic = out_triton_monolithic[..., :K]
out_ref = oai_moe_forward(
hidden_states=x,
w1=w1,
w1_bias=w1_bias,
w2=w2,
w2_bias=w2_bias,
gating_output=exp_data,
topk=topk,
)
assert_close(ref=out_ref, tri=out_triton_monolithic, maxtol=0.025, rmstol=0.005)
@pytest.mark.parametrize(
("num_rows", "num_experts", "expected"),
[
(4, 32, 16), # bs=1 decode: below the 16-row floor
(32 * 20, 32, 32), # 20 tokens/expert rounds up to 32
(32 * 33, 32, 32), # heuristic would pick 64; capped for 99KB smem
(12000, 32, 32), # prefill: heuristic would pick 128
],
)
def test_sm12x_block_m(num_rows, num_experts, expected):
assert _sm12x_block_m(num_rows, num_experts) == expected
@pytest.mark.parametrize("sm12x", [True, False])
def test_gpt_oss_priority_prefers_marlin_on_sm12x(
monkeypatch: pytest.MonkeyPatch, sm12x: bool
):
"""SM12x runs the OAI Triton kernels at small tiles only, so MARLIN must
stay the default there while TRITON remains selectable explicitly."""
monkeypatch.setattr(mxfp4_oracle.current_platform, "is_cuda", lambda: True)
monkeypatch.setattr(
mxfp4_oracle.current_platform,
"is_device_capability_family",
lambda family: sm12x and family == 120,
)
order = mxfp4_oracle._get_priority_backends_for_gpt_oss()
triton_first = order.index(Mxfp4MoeBackend.TRITON) < order.index(
Mxfp4MoeBackend.MARLIN
)
assert triton_first is not sm12x
def test_unit_shuffle():
N = ModelConfig.intermediate_size
K = ModelConfig.hidden_size
m = torch.randn((K, 2 * N), dtype=torch.bfloat16, device="cuda")
x = torch.randn(K, dtype=torch.bfloat16, device="cuda")
m_shuffled = shuffle_weight(m)
out_ref = x @ m
out_ref = swiglu(out_ref, limit=1.0)
out = x @ m_shuffled
out = triton_kernels.swiglu.swiglu_torch(
out,
alpha=1.702,
precision_config=triton_kernels.swiglu.PrecisionConfig(limit=1.0),
)
assert_close(ref=out_ref, tri=out)
@pytest.mark.parametrize("n_tokens", [1, 33, 512])
@pytest.mark.parametrize("n_experts,topk", [(32, 4), (128, 4)])
def test_routing_data_from_sparse_topk_parity(n_tokens, n_experts, topk):
"""routing_data_from_sparse_topk must produce routing structures
identical to make_routing_data for the same topk result."""
from vllm.model_executor.layers.fused_moe.experts import (
gpt_oss_triton_kernels_moe as gptoss_moe,
)
make_routing_data = gptoss_moe.make_routing_data
routing_data_from_sparse_topk = gptoss_moe.routing_data_from_sparse_topk
if gptoss_moe.triton_kernels_version == "3.5.1":
pytest.skip("SparseMatrix path requires triton_kernels v3.6.0+")
from triton_kernels.topk import topk as topk_fn
torch.manual_seed(0)
logits = torch.randn(n_tokens, n_experts, dtype=torch.bfloat16, device="cuda")
sparse_topk = topk_fn(logits, topk, apply_softmax=True)
assert not isinstance(sparse_topk, tuple)
rd_new, gather_new, scatter_new = routing_data_from_sparse_topk(
sparse_topk, n_experts
)
rd_ref, gather_ref, scatter_ref = make_routing_data(
sparse_topk.indx.to(torch.long), sparse_topk.vals, n_experts
)
def assert_equivalent(new, ref, path="RoutingData"):
"""Recursively compare tensor-valued leaves of routing structures."""
if isinstance(ref, torch.Tensor):
torch.testing.assert_close(new, ref, msg=lambda m: f"{path}: {m}")
elif isinstance(ref, (int, float, bool, str)) or ref is None:
assert new == ref, f"{path}: {new!r} != {ref!r}"
elif is_dataclass(ref):
for f in fields(ref):
assert_equivalent(
getattr(new, f.name), getattr(ref, f.name), f"{path}.{f.name}"
)
elif callable(ref):
assert callable(new), f"{path}: callable vs {type(new)}"
elif hasattr(ref, "__dict__"):
for k, v in vars(ref).items():
assert_equivalent(getattr(new, k), v, f"{path}.{k}")
elif hasattr(type(ref), "__slots__"):
for k in type(ref).__slots__:
assert_equivalent(getattr(new, k), getattr(ref, k), f"{path}.{k}")
else:
assert type(new) is type(ref), f"{path}: {type(new)} vs {type(ref)}"
assert_equivalent(rd_new, rd_ref)
if isinstance(gather_ref, torch.Tensor):
torch.testing.assert_close(gather_new, gather_ref)
torch.testing.assert_close(scatter_new, scatter_ref)
return
torch.testing.assert_close(gather_new.src_indx, gather_ref.src_indx)
torch.testing.assert_close(gather_new.dst_indx, gather_ref.dst_indx)
torch.testing.assert_close(scatter_new.src_indx, scatter_ref.src_indx)
torch.testing.assert_close(scatter_new.dst_indx, scatter_ref.dst_indx)
@pytest.mark.parametrize("n_tokens", [1, 33, 4099])
@pytest.mark.parametrize("n_experts,topk", [(128, 4), (256, 6)])
def test_make_routing_data_tk38_matches_reference(n_tokens, n_experts, topk):
from vllm.model_executor.layers.fused_moe.experts import (
gpt_oss_triton_kernels_moe as gptoss_moe,
)
if gptoss_moe.triton_kernels_version != "3.8":
pytest.skip("requires triton_kernels 3.8")
torch.manual_seed(0)
topk_ids = torch.stack(
[torch.randperm(n_experts, device="cuda")[:topk] for _ in range(n_tokens)]
).to(torch.int32)
topk_ids[torch.rand(topk_ids.shape, device="cuda") < 0.5] = -1
topk_weights = torch.rand(topk_ids.shape, device="cuda")
routing, gather_tok, scatter_tok = gptoss_moe.make_routing_data(
topk_ids, topk_weights, n_experts
)
flat = topk_ids.reshape(-1).to(torch.int64)
order = torch.argsort(flat.clamp(min=0), stable=True)
counts = torch.bincount(flat.clamp(min=0), minlength=n_experts).to(torch.int32)
weights = topk_weights.reshape(-1) * (flat >= 0).to(torch.float32)
assert torch.equal(routing.ragged.slice_sizes, counts)
assert torch.equal(gather_tok, (order // topk).to(torch.int32))
assert torch.equal(scatter_tok, gather_tok)
assert torch.equal(routing.gate_scal, weights[order])
assert routing.n_expts_act == topk
assert routing.n_valid == flat.numel()
-401
View File
@@ -1,401 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the MoE grouped topk kernel.
Run `pytest tests/kernels/moe/test_grouped_topk.py`.
"""
import pytest
import torch
import vllm.envs as envs
from vllm.config import (
CompilationConfig,
VllmConfig,
get_cached_compilation_config,
set_current_vllm_config,
)
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.fused_moe.router.grouped_topk_router import (
GroupedTopk,
GroupedTopKRouter,
fused_grouped_topk,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
def _run_single_group_topk(
logits: torch.Tensor,
bias: torch.Tensor,
topk: int,
*,
scoring_func: str,
renormalize: bool,
routed_scaling_factor: float = 1.0,
) -> tuple[torch.Tensor, torch.Tensor]:
return fused_grouped_topk(
hidden_states=torch.empty(
(logits.shape[0], 0), dtype=logits.dtype, device=logits.device
),
gating_output=logits,
topk=topk,
renormalize=renormalize,
e_score_correction_bias=bias,
num_expert_group=1,
topk_group=1,
scoring_func=scoring_func,
routed_scaling_factor=routed_scaling_factor,
)
def _single_group_reference(
logits: torch.Tensor,
bias: torch.Tensor,
topk: int,
*,
scoring_func: str,
renormalize: bool,
routed_scaling_factor: float = 1.0,
) -> tuple[torch.Tensor, torch.Tensor]:
if scoring_func == "sigmoid":
scores = 0.5 * torch.tanh(0.5 * logits.float()) + 0.5
else:
scores = torch.softmax(logits, dim=-1).float()
indices = torch.argsort(
scores + bias.float(), dim=-1, descending=True, stable=True
)[:, :topk]
values = scores.gather(1, indices)
if renormalize:
values /= values.sum(dim=-1, keepdim=True) + 1e-20
values *= routed_scaling_factor
return values, indices.to(torch.int32)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
)
@pytest.mark.parametrize("skip_padding", [False, True])
def test_kimi_grouped_topk_masks_padding_when_enabled(skip_padding: bool):
hidden_states = torch.randn((2, 16), device="cuda")
gating_output = torch.randn((2, 896), device="cuda")
is_padding = torch.tensor([False, True], device="cuda")
router = GroupedTopKRouter(
top_k=16,
global_num_experts=896,
num_expert_group=1,
topk_group=1,
skip_padding=skip_padding,
)
with set_forward_context(None, VllmConfig(), is_padding=is_padding):
topk_weights, topk_ids = router._compute_routing(
hidden_states, gating_output, torch.int32
)
assert torch.all(topk_ids[0] >= 0)
if skip_padding:
assert torch.all(topk_ids[1] == -1)
assert torch.all(topk_weights[1] == 0)
else:
assert torch.all(topk_ids[1] >= 0)
assert torch.all(topk_weights[1] > 0)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
)
@pytest.mark.parametrize("n_token", [1, 33, 64])
@pytest.mark.parametrize("n_hidden", [1024, 2048])
@pytest.mark.parametrize(
"n_expert,topk,num_expert_group,topk_group",
[
(16, 2, 8, 2),
(128, 2, 8, 2),
(256, 8, 8, 4),
(384, 8, 1, 1),
(512, 22, 1, 1),
],
)
@pytest.mark.parametrize("renormalize", [True, False])
@pytest.mark.parametrize("scoring_func", ["softmax", "sigmoid"])
@pytest.mark.parametrize("routed_scaling_factor", [1.0, 2.5])
@pytest.mark.parametrize("input_dtype", [torch.bfloat16, torch.float32])
@pytest.mark.parametrize("bias_dtype", [torch.float32])
def test_grouped_topk(
monkeypatch: pytest.MonkeyPatch,
n_token: int,
n_hidden: int,
n_expert: int,
topk: int,
num_expert_group: int,
topk_group: int,
renormalize: bool,
scoring_func: str,
routed_scaling_factor: float,
input_dtype: torch.dtype,
bias_dtype: torch.dtype,
):
vllm_config = VllmConfig(
compilation_config=CompilationConfig(custom_ops=["all", "+grouped_topk"])
)
get_cached_compilation_config.cache_clear()
set_random_seed(0)
hidden_states = torch.randn((n_token, n_hidden), dtype=input_dtype, device="cuda")
gating_output = torch.randn((n_token, n_expert), dtype=input_dtype, device="cuda")
e_score_correction_bias = torch.randn((n_expert,), dtype=bias_dtype, device="cuda")
with set_current_vllm_config(vllm_config), monkeypatch.context() as m:
m.setenv("VLLM_USE_FUSED_MOE_GROUPED_TOPK", "0")
m.setattr(envs, "VLLM_BATCH_INVARIANT", True)
grouped_topk = GroupedTopk(
topk=topk,
renormalize=renormalize,
num_expert_group=num_expert_group,
topk_group=topk_group,
scoring_func=scoring_func,
routed_scaling_factor=routed_scaling_factor,
)
assert grouped_topk._forward_method.__name__ == "forward_cuda"
baseline_topk_weights, baseline_topk_ids = grouped_topk(
hidden_states=hidden_states,
gating_output=gating_output,
e_score_correction_bias=e_score_correction_bias,
)
test_topk_weights, test_topk_ids = fused_grouped_topk(
hidden_states=hidden_states,
gating_output=gating_output,
topk=topk,
renormalize=renormalize,
num_expert_group=num_expert_group,
topk_group=topk_group,
scoring_func=scoring_func,
routed_scaling_factor=routed_scaling_factor,
e_score_correction_bias=e_score_correction_bias,
)
torch.testing.assert_close(
baseline_topk_weights, test_topk_weights, atol=2e-2, rtol=0
)
torch.testing.assert_close(baseline_topk_ids, test_topk_ids, atol=0, rtol=0)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
)
def test_grouped_topk_single_group_large_batch():
set_random_seed(0)
logits = torch.randn((1536, 896), dtype=torch.bfloat16, device="cuda")
bias = torch.randn((896,), dtype=torch.float32, device="cuda")
expected_values, expected_ids = _single_group_reference(
logits, bias, 16, scoring_func="sigmoid", renormalize=True
)
actual_values, actual_ids = _run_single_group_topk(
logits, bias, 16, scoring_func="sigmoid", renormalize=True
)
torch.testing.assert_close(actual_ids, expected_ids)
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
)
@pytest.mark.parametrize(
"num_experts,topk,input_dtype,bias_dtype",
[
(512, 9, torch.bfloat16, torch.float32),
(512, 16, torch.float16, torch.float16),
(513, 9, torch.float32, torch.bfloat16),
(513, 16, torch.bfloat16, torch.float32),
(895, 9, torch.float16, torch.bfloat16),
(896, 16, torch.float32, torch.float16),
(897, 9, torch.bfloat16, torch.bfloat16),
(897, 16, torch.float16, torch.float32),
(1024, 9, torch.float32, torch.bfloat16),
(1024, 16, torch.bfloat16, torch.float16),
],
)
@pytest.mark.parametrize(
"scoring_func,renormalize,routed_scaling_factor",
[
("sigmoid", True, 1.0),
("sigmoid", False, 2.5),
("softmax", True, 2.5),
("softmax", False, 1.0),
],
)
def test_grouped_topk_single_group_tiers(
num_experts: int,
topk: int,
input_dtype: torch.dtype,
bias_dtype: torch.dtype,
scoring_func: str,
renormalize: bool,
routed_scaling_factor: float,
):
set_random_seed(7)
logits = torch.randn((17, num_experts), dtype=input_dtype, device="cuda")
bias = torch.randn((num_experts,), dtype=bias_dtype, device="cuda")
expected_values, expected_ids = _single_group_reference(
logits,
bias,
topk,
scoring_func=scoring_func,
renormalize=renormalize,
routed_scaling_factor=routed_scaling_factor,
)
actual_values, actual_ids = _run_single_group_topk(
logits,
bias,
topk,
scoring_func=scoring_func,
renormalize=renormalize,
routed_scaling_factor=routed_scaling_factor,
)
torch.testing.assert_close(actual_ids, expected_ids)
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
)
@pytest.mark.parametrize(
"num_experts,topk,scoring_func",
[
(128, 8, "sigmoid"),
(129, 8, "sigmoid"),
(257, 8, "sigmoid"),
(385, 8, "sigmoid"),
(512, 9, "sigmoid"),
(513, 9, "sigmoid"),
(769, 9, "sigmoid"),
(897, 16, "sigmoid"),
(1024, 16, "sigmoid"),
(128, 4, "softmax"),
(128, 5, "softmax"),
(129, 8, "softmax"),
(161, 8, "softmax"),
(256, 9, "softmax"),
(257, 8, "softmax"),
(512, 9, "softmax"),
(512, 17, "softmax"),
(512, 23, "softmax"),
(513, 8, "softmax"),
(577, 9, "softmax"),
(769, 9, "softmax"),
(897, 9, "softmax"),
(1024, 16, "softmax"),
],
)
def test_grouped_topk_single_group_capacity_tiers(
num_experts: int,
topk: int,
scoring_func: str,
):
set_random_seed(11)
logits = torch.randn((3, num_experts), dtype=torch.bfloat16, device="cuda")
bias = torch.randn((num_experts,), dtype=torch.float32, device="cuda")
expected_values, expected_ids = _single_group_reference(
logits,
bias,
topk,
scoring_func=scoring_func,
renormalize=True,
routed_scaling_factor=2.5,
)
actual_values, actual_ids = _run_single_group_topk(
logits,
bias,
topk,
scoring_func=scoring_func,
renormalize=True,
routed_scaling_factor=2.5,
)
torch.testing.assert_close(actual_ids, expected_ids)
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
)
@pytest.mark.parametrize("num_experts", [512, 896, 1024])
def test_grouped_topk_single_group_stable_ties(num_experts: int):
logits = torch.zeros((1, num_experts), dtype=torch.bfloat16, device="cuda")
bias = torch.zeros((num_experts,), dtype=torch.float32, device="cuda")
actual_values, actual_ids = _run_single_group_topk(
logits,
bias,
16,
scoring_func="sigmoid",
renormalize=True,
routed_scaling_factor=2.5,
)
expected_ids = torch.arange(16, dtype=torch.int32, device="cuda")[None]
expected_values = torch.full((1, 16), 2.5 / 16, dtype=torch.float32, device="cuda")
torch.testing.assert_close(actual_ids, expected_ids)
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
)
@pytest.mark.parametrize("num_experts", [512, 896, 1024])
@pytest.mark.parametrize("num_finite", [0, 15])
@pytest.mark.parametrize("renormalize", [False, True])
def test_grouped_topk_single_group_nonfinite_scores(
num_experts: int, num_finite: int, renormalize: bool
):
logits = torch.full(
(1, num_experts), float("nan"), dtype=torch.bfloat16, device="cuda"
)
if num_finite:
logits[0, :num_finite] = torch.arange(
num_finite, dtype=torch.bfloat16, device="cuda"
)
logits[0, num_finite] = torch.inf
logits[0, num_finite + 1] = -torch.inf
bias = torch.zeros((num_experts,), dtype=torch.float32, device="cuda")
actual_values, actual_ids = _run_single_group_topk(
logits,
bias,
16,
scoring_func="sigmoid",
renormalize=renormalize,
routed_scaling_factor=2.5,
)
if num_finite == 0:
expected_ids = torch.arange(16, dtype=torch.int32, device="cuda")[None]
if renormalize:
expected_values = torch.full(
(1, 16), 1 / 16, dtype=torch.float32, device="cuda"
)
else:
expected_values = torch.zeros((1, 16), dtype=torch.float32, device="cuda")
else:
expected_ids = torch.cat(
(
torch.arange(num_finite - 1, -1, -1, dtype=torch.int32, device="cuda"),
torch.tensor([num_finite], dtype=torch.int32, device="cuda"),
)
)[None]
finite_values = logits[0, :num_finite].float().sigmoid().flip(0)
if renormalize:
finite_values /= finite_values.sum()
finite_values *= 2.5
expected_values = torch.cat(
(finite_values, torch.zeros(1, dtype=torch.float32, device="cuda"))
)[None]
torch.testing.assert_close(actual_ids, expected_ids)
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
@@ -1,373 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Test comparing Marlin INT4 MoE vs FlashInfer TRT-LLM MXINT4 MoE."""
import pytest
import torch
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
fused_marlin_moe,
)
from vllm.model_executor.layers.fused_moe.experts.trtllm_mxint4_moe import (
TrtLlmMxint4ExpertsMonolithic,
)
from vllm.model_executor.layers.fused_moe.router.grouped_topk_router import (
grouped_topk,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_mxint4_moe import (
prepare_static_weights_for_trtllm_mxint4_moe,
)
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
from vllm.utils.torch_utils import set_random_seed
def mxint4_quantize(
x: torch.Tensor, sf_vec_size: int = 32
) -> tuple[torch.Tensor, torch.Tensor]:
"""Quantize BF16 tensor to MXINT4 with block scaling (group_size=sf_vec_size).
Returns:
- uint8 packed (2 INT4/byte): [..., k//2] - stores SIGNED INT4 [-8, 7]
- scales in BF16: [..., k//sf_vec_size]
"""
x_reshaped = x.reshape(-1, sf_vec_size)
x_max = x_reshaped.max(dim=-1, keepdim=True)[0].to(torch.float32)
x_min = x_reshaped.min(dim=-1, keepdim=True)[0].to(torch.float32)
x_max = x_max * 8.0 / 7.0
amax = torch.where(x_max > -x_min, x_max, -x_min)
scales = amax / 8.0
x_scaled = x_reshaped * scales.reciprocal()
x_int8 = (
x_scaled.round().clamp(-8, 7).to(torch.int8).reshape(-1, sf_vec_size // 2, 2)
)
x_int4 = (x_int8[..., 0] & 0x0F) | ((x_int8[..., 1] & 0x0F) << 4)
return (
x_int4.to(torch.uint8).reshape(*x.shape[:-1], x.shape[-1] // 2),
scales.to(x.dtype).reshape(*x.shape[:-1], x.shape[-1] // sf_vec_size),
)
def mxint4_quantize_moe_weights(
weights_bf16: torch.Tensor, group_size: int = 32
) -> tuple[torch.Tensor, torch.Tensor]:
"""Quantize MoE weights [e, n, k] to MxInt4 format.
Args:
weights_bf16: BF16 weights of shape [num_experts, out_features, in_features]
group_size: Quantization group size (default: 32)
Returns:
- weights_mxint4: Quantized weights [e, n, k//2] uint8
- scales_mxint4: Quantization scales [e, n, k//group_size] bf16
"""
e = weights_bf16.shape[0]
weight_list = []
scale_list = []
for i in range(e):
w_q, w_s = mxint4_quantize(weights_bf16[i], sf_vec_size=group_size)
weight_list.append(w_q)
scale_list.append(w_s)
return torch.stack(weight_list), torch.stack(scale_list)
__all__ = [
"mxint4_quantize",
"mxint4_quantize_moe_weights",
"marlin_quantize_moe_weights",
]
def test_trtllm_mxint4_activation_supports_vllm_gated_silu():
assert TrtLlmMxint4ExpertsMonolithic._supports_activation(MoEActivation.SILU)
assert TrtLlmMxint4ExpertsMonolithic._supports_activation(MoEActivation.SWIGLUOAI)
assert not TrtLlmMxint4ExpertsMonolithic._supports_activation(
MoEActivation.RELU2_NO_MUL
)
def marlin_quantize_moe_weights(
weights_bf16: torch.Tensor, group_size: int = 32
) -> tuple[torch.Tensor, torch.Tensor]:
"""Quantize MoE weights [e, n, k] to Marlin INT4 format.
Args:
weights_bf16: BF16 weights of shape [num_experts, out_features, in_features]
group_size: Quantization group size (default: 32)
Returns:
- weights_marlin: Marlin quantized weights [e, k//8, n] int32
- scales_marlin: Marlin quantization scales [e, k//group_size, n] bf16
"""
from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
marlin_quantize,
)
e, n, k = weights_bf16.shape
weight_list = []
scale_list = []
for i in range(e):
# Transpose for Marlin: [n, k] → [k, n]
w_t = weights_bf16[i].T.contiguous()
_, w_q, w_s = marlin_quantize(w_t, scalar_types.uint4b8, group_size)
weight_list.append(w_q)
scale_list.append(w_s)
# Stack to get [e, ...] shape
weights_marlin = torch.stack(weight_list) # [e, k // 8, n]
scales_marlin = torch.stack(scale_list) # [e, k // group_size, n]
return weights_marlin, scales_marlin
TRTLLM_GEN_AVAILABLE = (
current_platform.is_cuda() and current_platform.is_device_capability_family(100)
)
@pytest.mark.skipif(not TRTLLM_GEN_AVAILABLE, reason="Skip for non SM100")
@pytest.mark.parametrize("m", [1, 33])
@pytest.mark.parametrize("n", [7168])
@pytest.mark.parametrize("k", [512])
@pytest.mark.parametrize("e", [384])
@pytest.mark.parametrize("topk", [8])
@pytest.mark.parametrize("group_size", [32])
def test_marlin_vs_trtllm_mxint4_moe_kimik2(monkeypatch, m, n, k, e, topk, group_size):
"""Compare Marlin INT4 MoE vs FlashInfer TRT-LLM MXINT4 MoE.
Uses mxint4_quantize() to generate common INT4 weights + BF16 scales,
then runs both Marlin and TRT-LLM kernels and compares outputs.
"""
pytest.importorskip("flashinfer")
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_INT4", "1")
set_random_seed(0)
dtype = torch.bfloat16
# DeepSeekV3 routing config (from Kimi-K2-Thinking config.json)
n_group = 1 # n_group from model config
topk_group = 1 # topk_group from model config
routed_scaling = 2.827 # routed_scaling_factor from model config
# Input - realistic activation range for LLM (after LayerNorm: mean~0, std~1)
a = torch.randn((m, k), device="cuda", dtype=dtype) * 0.5
# Generate routing logits and bias (DeepSeekV3 expects float logits)
# Realistic ranges: logits typically [-3, 3], bias [-2, 2]
routing_logits = torch.randn((m, e), device="cuda", dtype=torch.float32) * 1.5
routing_bias = torch.randn(e, device="cuda", dtype=torch.float32) * 0.8
# 1. Generate BF16 weights (SHARED between both paths)
# Realistic weight initialization: Xavier/Glorot uniform scaling
# std = sqrt(2 / (fan_in + fan_out))
std_w1 = (2.0 / (k + 2 * n)) ** 0.5
std_w2 = (2.0 / (n + k)) ** 0.5
w1_bf16 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) * std_w1
w2_bf16 = torch.randn((e, k, n), device="cuda", dtype=dtype) * std_w2
# === Path 1: TRT-LLM FlashInfer MXINT4 MoE ===
# Similar to: if self.use_flashinfer_mxint4_moe
# Quantize using MXINT4 method (signed INT4)
w1_int4, w1_scales = mxint4_quantize_moe_weights(w1_bf16, group_size)
w2_int4, w2_scales = mxint4_quantize_moe_weights(w2_bf16, group_size)
trtllm_weights = prepare_static_weights_for_trtllm_mxint4_moe(
gemm1_weights=w1_int4,
gemm1_scales=w1_scales,
gemm2_weights=w2_int4,
gemm2_scales=w2_scales,
)
from flashinfer import RoutingMethodType
from flashinfer.fused_moe import trtllm_mxint4_block_scale_moe
# Routing handled internally by trtllm_mxint4_block_scale_moe
trtllm_output = trtllm_mxint4_block_scale_moe(
routing_logits=routing_logits,
routing_bias=routing_bias.to(torch.bfloat16),
hidden_states=a,
gemm1_weights=trtllm_weights["gemm1_weights"],
gemm1_weights_scale=trtllm_weights["gemm1_scales"],
gemm1_alpha=None,
gemm1_beta=None,
gemm1_clamp_limit=None,
gemm2_weights=trtllm_weights["gemm2_weights"],
gemm2_weights_scale=trtllm_weights["gemm2_scales"],
num_experts=e,
top_k=topk,
n_group=n_group,
topk_group=topk_group,
intermediate_size=n,
local_expert_offset=0,
local_num_experts=e,
routed_scaling_factor=routed_scaling,
routing_method_type=RoutingMethodType.DeepSeekV3,
enable_pdl=None,
output=None,
tune_max_num_tokens=8192,
).to(dtype)
# === Path 2: Marlin INT4 MoE ===
# Similar to: else (non-flashinfer path)
# Quantize using Marlin's method (UINT4b8)
w1_marlin, w1_scales_marlin = marlin_quantize_moe_weights(w1_bf16, group_size)
w2_marlin, w2_scales_marlin = marlin_quantize_moe_weights(w2_bf16, group_size)
# Use production routing kernel (same as router.select_experts internally uses)
topk_weights, topk_ids = grouped_topk(
hidden_states=a,
gating_output=routing_logits,
topk=topk,
renormalize=False, # DeepSeekV3 doesn't renormalize
num_expert_group=n_group,
topk_group=topk_group,
scoring_func="sigmoid", # DeepSeekV3 uses sigmoid
routed_scaling_factor=routed_scaling,
e_score_correction_bias=routing_bias,
)
marlin_output = fused_marlin_moe(
hidden_states=a,
w1=w1_marlin,
w2=w2_marlin,
bias1=None,
bias2=None,
w1_scale=w1_scales_marlin,
w2_scale=w2_scales_marlin,
topk_weights=topk_weights,
topk_ids=topk_ids,
quant_type_id=scalar_types.uint4b8.id,
global_num_experts=e,
expert_map=None,
global_scale1=None,
global_scale2=None,
input_global_scale1=None,
input_global_scale2=None,
w1_zeros=None,
w2_zeros=None,
input_dtype=dtype,
)
# Sanity check: manually compute BF16 reference for comparison
# Use same routing as Marlin path for consistency
bf16_output = torch.zeros((m, k), device="cuda", dtype=dtype)
for token_idx in range(m):
for expert_rank in range(topk):
expert_id = topk_ids[token_idx, expert_rank].item()
weight = topk_weights[token_idx, expert_rank].item()
# w1: [2*n, k] @ [k] -> [2*n]
up_gate = a[token_idx] @ w1_bf16[expert_id].T # [2*n]
gate, up = up_gate.chunk(2, dim=0)
intermediate = torch.nn.functional.silu(gate) * up # [n]
# w2: [k, n] @ [n] -> [k]
expert_out = intermediate @ w2_bf16[expert_id].T # [k]
bf16_output[token_idx] += weight * expert_out
# Compare against BF16 reference.
torch.testing.assert_close(marlin_output, bf16_output, atol=0.3, rtol=1.0)
torch.testing.assert_close(trtllm_output, bf16_output, atol=0.3, rtol=1.0)
# Compare against each other for sanity.
# Note: Different quantization schemes (UINT4b8 vs signed MXINT4) cause
# some differences
torch.testing.assert_close(marlin_output, trtllm_output, atol=0.3, rtol=6.0)
@pytest.mark.skipif(not TRTLLM_GEN_AVAILABLE, reason="Skip for non SM100")
@pytest.mark.parametrize("m", [1, 33])
@pytest.mark.parametrize("n", [7168])
@pytest.mark.parametrize("k", [512])
@pytest.mark.parametrize("e", [384])
@pytest.mark.parametrize("topk", [8])
@torch.inference_mode()
def test_flashinfer_trtllm_mxint4_moe_wrapper(m, n, k, e, topk):
"""Test that the flashinfer_trtllm_mxint4_moe wrapper matches the raw
trtllm_mxint4_block_scale_moe kernel call."""
pytest.importorskip("flashinfer")
from flashinfer import RoutingMethodType
from flashinfer.fused_moe import trtllm_mxint4_block_scale_moe
from vllm.model_executor.layers.quantization.utils.flashinfer_mxint4_moe import (
flashinfer_trtllm_mxint4_moe,
)
set_random_seed(0)
dtype = torch.bfloat16
a = torch.randn((m, k), device="cuda", dtype=dtype) * 0.5
router_logits = torch.randn((m, e), device="cuda", dtype=torch.float32) * 1.5
routing_bias = torch.randn(e, device="cuda", dtype=torch.float32) * 0.8
std_w1 = (2.0 / (k + 2 * n)) ** 0.5
std_w2 = (2.0 / (n + k)) ** 0.5
w1_bf16 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) * std_w1
w2_bf16 = torch.randn((e, k, n), device="cuda", dtype=dtype) * std_w2
w1_int4, w1_scales = mxint4_quantize_moe_weights(w1_bf16)
w2_int4, w2_scales = mxint4_quantize_moe_weights(w2_bf16)
prepared = prepare_static_weights_for_trtllm_mxint4_moe(
gemm1_weights=w1_int4,
gemm1_scales=w1_scales,
gemm2_weights=w2_int4,
gemm2_scales=w2_scales,
)
# Raw kernel call (reference)
raw_out = trtllm_mxint4_block_scale_moe(
routing_logits=router_logits.to(torch.float32),
routing_bias=routing_bias.to(torch.bfloat16),
hidden_states=a,
gemm1_weights=prepared["gemm1_weights"].data,
gemm1_weights_scale=prepared["gemm1_scales"].data,
gemm1_alpha=None,
gemm1_beta=None,
gemm1_clamp_limit=None,
gemm2_weights=prepared["gemm2_weights"].data,
gemm2_weights_scale=prepared["gemm2_scales"].data,
num_experts=e,
top_k=topk,
n_group=1,
topk_group=1,
intermediate_size=n,
local_expert_offset=0,
local_num_experts=e,
routed_scaling_factor=None,
routing_method_type=RoutingMethodType.DeepSeekV3,
enable_pdl=None,
output=None,
tune_max_num_tokens=8192,
)
if not isinstance(raw_out, torch.Tensor):
raw_out = raw_out[0]
raw_out = raw_out.to(dtype)
# Wrapper call
wrapper_out = flashinfer_trtllm_mxint4_moe(
x=a,
router_logits=router_logits,
w13_weight_packed=prepared["gemm1_weights"],
w13_weight_scale=prepared["gemm1_scales"],
w2_weight_packed=prepared["gemm2_weights"],
w2_weight_scale=prepared["gemm2_scales"],
global_num_experts=e,
top_k=topk,
intermediate_size_per_partition=n,
local_num_experts=e,
ep_rank=0,
num_expert_group=1,
topk_group=1,
e_score_correction_bias=routing_bias,
routing_method_type=RoutingMethodType.DeepSeekV3,
)
assert wrapper_out.shape == (m, k)
assert wrapper_out.dtype == dtype
torch.testing.assert_close(wrapper_out, raw_out, atol=0.0, rtol=0.0)
File diff suppressed because it is too large Load Diff
@@ -1,414 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Test modular OAI Triton MoE."""
from __future__ import annotations
import pytest
import torch
import torch.nn.functional as F
from tests.utils import wait_for_gpu_memory_to_clear
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.utils.import_utils import get_triton_kernels_version, has_triton_kernels
if not has_triton_kernels():
pytest.skip(
"triton_kernels not found, skipping all related tests",
allow_module_level=True,
)
if get_triton_kernels_version() == "3.8":
# 3.8: matmul_ogs -> matmul
from triton_kernels.matmul import FlexCtx, PrecisionConfig
else:
from triton_kernels.matmul_ogs import FlexCtx, PrecisionConfig
from triton_kernels.numerics import InFlexData
from triton_kernels.numerics_details.mxfp import downcast_to_mxfp, upcast_from_mxfp
from triton_kernels.tensor import FP4, convert_layout, wrap_torch_tensor
from triton_kernels.testing import assert_close
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.config import mxfp4_w4a16_moe_quant_config
from vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe import (
OAITritonExperts,
UnfusedOAITritonExperts,
)
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import mx_scale_kwargs
from vllm.platforms import current_platform
from vllm.utils.math_utils import round_up
from vllm.utils.torch_utils import set_random_seed
from .utils import make_dummy_moe_config, mxfp4_w_layouts, shuffle_weight
MNK = [
(1, 512, 384),
(1, 2880, 2880),
(2, 512, 384),
(2, 2880, 2880),
(16, 2880, 2880),
]
def deepseek_v4_flash_moe_topology():
"""MoE sizes for the DeepSeek-V4-Flash`.
Default weights from: ``deepseek-ai/DeepSeek-V4-Flash`.
``moe_intermediate_size``, ``n_routed_experts``, and ``num_experts_per_tok``.
"""
defaults = {
"hidden_size": 4096,
"moe_intermediate_size": 2048,
"n_routed_experts": 256,
"num_experts_per_tok": 6,
}
return defaults
def scaled_deepseek_v4_flash_problem(
*,
dim_scale: int = 8,
expert_scale: int = 8,
):
"""Smaller K/N/E for kernel tests; keeps production top_k and K:N ratio (~2:1)."""
t = deepseek_v4_flash_moe_topology()
k = max(128, t["hidden_size"] // dim_scale)
n = max(64, t["moe_intermediate_size"] // dim_scale)
num_experts = max(t["num_experts_per_tok"], t["n_routed_experts"] // expert_scale)
topk = t["num_experts_per_tok"]
return k, n, num_experts, topk
def unshuffle_weight(w: torch.Tensor):
first = w[..., ::2]
second = w[..., 1::2]
return torch.concat((first, second), dim=-1)
def make_weights(dtype, k, n, e):
w1 = torch.randn((e, k, 2 * n), dtype=dtype, device="cuda")
w1_bias = torch.randn((e, 2 * n), dtype=dtype, device="cuda")
w2 = torch.randn((e, n, k), dtype=dtype, device="cuda")
w2_bias = torch.randn((e, k), dtype=dtype, device="cuda")
w1_tri = w1.clone()
w2_tri = w2.clone()
w1_bias_tri = w1_bias.clone()
w2_bias_tri = w2_bias.clone()
w1_bias_tri = w1_bias_tri.to(torch.float32)
w2_bias_tri = w2_bias_tri.to(torch.float32)
# shuffle weights
w1_tri = shuffle_weight(w1_tri)
w1_bias_tri = shuffle_weight(w1_bias_tri)
if current_platform.is_rocm():
k_align, n2_align = 256, 512
else:
k_align, n2_align = 64, 128
w1_bottom_pad = round_up(w1_tri.shape[1], k_align) - w1_tri.shape[1]
w1_right_pad = round_up(w1_tri.shape[2], n2_align) - w1_tri.shape[2]
w2_bottom_pad = w1_right_pad // 2
w2_right_pad = w1_bottom_pad
w1_tri = F.pad(w1_tri, (0, w1_right_pad, 0, w1_bottom_pad, 0, 0))
w2_tri = F.pad(w2_tri, (0, w2_right_pad, 0, w2_bottom_pad, 0, 0))
w1_bias_tri = F.pad(w1_bias_tri, (0, w1_right_pad, 0, 0))
w2_bias_tri = F.pad(w2_bias_tri, (0, w2_right_pad, 0, 0))
# quant triton_weights
w1_tri, w1_scale_tri = downcast_to_mxfp(w1_tri, torch.uint8, axis=1)
w1 = upcast_from_mxfp(w1_tri, w1_scale_tri, dtype, axis=1)
w1 = w1[..., :k, : 2 * n]
w1 = unshuffle_weight(w1)
w2_tri, w2_scale_tri = downcast_to_mxfp(w2_tri, torch.uint8, axis=1)
w2 = upcast_from_mxfp(w2_tri, w2_scale_tri, dtype, axis=1)
w2 = w2[..., :n, :k]
num_warps = 8
(
w_layout,
w_layout_opts,
w_scale_layout,
w_scale_layout_opts,
) = mxfp4_w_layouts(mx_axis=1, num_warps=num_warps)
w1_tri = convert_layout(wrap_torch_tensor(w1_tri, FP4), w_layout, **w_layout_opts)
w1_scale_tri = convert_layout(
wrap_torch_tensor(w1_scale_tri),
w_scale_layout,
**w_scale_layout_opts,
)
w2_tri = convert_layout(wrap_torch_tensor(w2_tri, FP4), w_layout, **w_layout_opts)
w2_scale_tri = convert_layout(
wrap_torch_tensor(w2_scale_tri),
w_scale_layout,
**w_scale_layout_opts,
)
w1_precision_config = PrecisionConfig(
**mx_scale_kwargs(w1_scale_tri), flex_ctx=FlexCtx(rhs_data=InFlexData())
)
w2_precision_config = PrecisionConfig(
**mx_scale_kwargs(w2_scale_tri), flex_ctx=FlexCtx(rhs_data=InFlexData())
)
return (
w1,
w2,
w1_bias,
w2_bias,
w1_tri,
w2_tri,
w1_bias_tri,
w2_bias_tri,
w1_precision_config,
w2_precision_config,
w1_bottom_pad,
)
def swiglu(x, alpha: float = 1.702, limit: float = 1.0):
# Note we add an extra bias of 1 to the linear layer
x_glu, x_linear = torch.chunk(x, 2, dim=-1)
if limit is not None:
x_glu = x_glu.clamp(max=limit)
out_glu = x_glu * torch.sigmoid(alpha * x_glu)
if limit is not None:
x_linear = x_linear.clamp(min=-limit, max=limit)
return out_glu * (x_linear + 1)
def torch_moe_impl(
hidden_states: torch.Tensor, # (M, K)
w1: torch.Tensor, # (E, K, 2N)
w2: torch.Tensor, # (E, N, K)
w1_bias: torch.Tensor, # (E, 2N)
w2_bias: torch.Tensor, # (E, K)
topk_weights: torch.Tensor, # (M, topk)
topk_ids: torch.Tensor, # (M, topk)
):
w1 = w1[topk_ids, ...]
w1_bias = w1_bias[topk_ids, ...]
hidden_states = torch.einsum("bekc,bk->bec", w1, hidden_states) + w1_bias
hidden_states = swiglu(hidden_states, limit=7)
w2 = w2[topk_ids, ...]
w2_bias = w2_bias[topk_ids, ...]
hidden_states = torch.einsum("bekc,bek->bec", w2, hidden_states) + w2_bias
# Weighted sum of experts
hidden_states = torch.einsum("bec,be->bc", hidden_states, topk_weights)
return hidden_states
def oai_triton_moe_impl(
x: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
w1_scale: PrecisionConfig,
w2_scale: PrecisionConfig,
w1_bias: torch.Tensor | None,
w2_bias: torch.Tensor | None,
num_experts: int,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
unfused: bool = False,
) -> torch.Tensor:
quant_config = mxfp4_w4a16_moe_quant_config(
w1_bias=w1_bias,
w2_bias=w2_bias,
w1_scale=w1_scale,
w2_scale=w2_scale,
)
moe_config = make_dummy_moe_config()
if unfused:
fused_experts = UnfusedOAITritonExperts(moe_config, quant_config)
else:
fused_experts = OAITritonExperts(moe_config, quant_config)
mk = FusedMoEKernel(
maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
use_monolithic=False,
),
fused_experts,
)
return mk.apply(
hidden_states=x,
w1=w1,
w2=w2,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=MoEActivation.SWIGLUOAI,
global_num_experts=num_experts,
expert_map=None,
apply_router_weight_on_input=False,
)
@pytest.mark.skipif(
not OAITritonExperts._supports_current_device(),
reason="OAI Triton MoE is not supported on this device.",
)
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("m,n,k", MNK)
@pytest.mark.parametrize("num_experts", [32, 128])
@pytest.mark.parametrize("topk", [4])
@pytest.mark.parametrize("unfused", [True, False])
def test_oai_triton_moe(
dtype: torch.dtype,
m: int,
n: int,
k: int,
num_experts: int,
topk: int,
unfused: bool,
workspace_init,
):
wait_for_gpu_memory_to_clear(devices=[0], threshold_ratio=0.1)
set_random_seed(0)
(
w1,
w2,
w1_bias,
w2_bias,
w1_tri,
w2_tri,
w1_bias_tri,
w2_bias_tri,
w1_precision_config,
w2_precision_config,
x_pad,
) = make_weights(dtype, k, n, num_experts)
x = torch.randn((m, k), dtype=dtype, device="cuda")
x_tri = F.pad(x, (0, x_pad, 0, 0))
router_logits = torch.randn(m, num_experts, device="cuda", dtype=dtype)
topk_weights, topk_ids = torch.topk(router_logits, k=topk, dim=-1, sorted=True)
topk_weights = torch.nn.functional.softmax(topk_weights, dim=-1)
with set_current_vllm_config(VllmConfig()):
out_ref = torch_moe_impl(x, w1, w2, w1_bias, w2_bias, topk_weights, topk_ids)
out = oai_triton_moe_impl(
x_tri,
w1_tri,
w2_tri,
w1_precision_config,
w2_precision_config,
w1_bias_tri,
w2_bias_tri,
num_experts,
topk_weights,
topk_ids,
unfused,
)
out = out[..., :k]
assert_close(ref=out_ref, tri=out, maxtol=0.025, rmstol=0.005)
@pytest.mark.skipif(
not UnfusedOAITritonExperts._supports_current_device(),
reason="Unfused OAI Triton MoE is not supported on this device.",
)
def test_unfused_oai_triton_experts_apply_direct_deepseek_v4_topology(workspace_init):
"""Exercise ``UnfusedOAITritonExperts.apply`` with explicit workspaces.
Same MoE topology as ``launch_dsv4.sh`` / DeepSeek-V4-Flash ``config.json``,
with linear dimensions and expert count scaled down for test GPU memory.
"""
wait_for_gpu_memory_to_clear(devices=[0], threshold_ratio=0.1)
set_random_seed(0)
k, n, num_experts, topk = scaled_deepseek_v4_flash_problem()
m = 7
dtype = torch.bfloat16
(
w1,
w2,
w1_bias,
w2_bias,
w1_tri,
w2_tri,
w1_bias_tri,
w2_bias_tri,
w1_precision_config,
w2_precision_config,
x_pad,
) = make_weights(dtype, k, n, num_experts)
x = torch.randn((m, k), dtype=dtype, device="cuda")
x_tri = F.pad(x, (0, x_pad, 0, 0))
router_logits = torch.randn(m, num_experts, device="cuda", dtype=dtype)
topk_weights, topk_ids = torch.topk(router_logits, k=topk, dim=-1, sorted=True)
topk_weights = torch.nn.functional.softmax(topk_weights, dim=-1)
quant_config = mxfp4_w4a16_moe_quant_config(
w1_bias=w1_bias_tri,
w2_bias=w2_bias_tri,
w1_scale=w1_precision_config,
w2_scale=w2_precision_config,
)
moe_config = make_dummy_moe_config(
num_experts=num_experts,
experts_per_token=topk,
hidden_dim=k,
intermediate_size=n,
)
experts = UnfusedOAITritonExperts(moe_config, quant_config)
_, _, N, K, top_k = experts.moe_problem_size(x_tri, w1_tri, w2_tri, topk_ids)
assert top_k == topk
ws13_shape, ws2_shape, out_shape = experts.workspace_shapes(
m,
N,
K,
topk,
num_experts,
num_experts,
None,
MoEActivation.SWIGLUOAI,
)
workspace13 = torch.empty(ws13_shape, dtype=dtype, device="cuda")
workspace2 = torch.empty(ws2_shape, dtype=dtype, device="cuda")
output = torch.empty(out_shape, dtype=dtype, device="cuda")
with set_current_vllm_config(VllmConfig()):
out_ref = torch_moe_impl(x, w1, w2, w1_bias, w2_bias, topk_weights, topk_ids)
experts.apply(
output=output,
hidden_states=x_tri,
w1=w1_tri,
w2=w2_tri,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=MoEActivation.SWIGLUOAI,
global_num_experts=num_experts,
expert_map=None,
a1q_scale=None,
a2_scale=None,
workspace13=workspace13,
workspace2=workspace2,
expert_tokens_meta=None,
apply_router_weight_on_input=False,
)
output = output[..., :k]
assert_close(ref=out_ref, tri=output, maxtol=0.025, rmstol=0.005)
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -1,512 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the MOE permute/unpermute kernel.
Run `pytest tests/kernels/test_moe_permute_unpermute.py`.
"""
import numpy as np
import pytest
import torch
from vllm.model_executor.layers.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe.expert_map_manager import (
determine_expert_map,
)
from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import (
MoEPermuteScratch,
get_moe_permute_scratch,
moe_permute,
moe_permute_unpermute_supported,
moe_prepare_scatter,
moe_unpermute,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.worker.workspace import current_workspace_manager
NUM_EXPERTS = [16, 64, 256]
TOP_KS = [2, 6, 8]
EP_SIZE = [1, 4, 16]
set_random_seed(0)
if current_platform.is_rocm():
pytest.skip(
"moe_permute_unpermute_supported is not defined for ROCm",
allow_module_level=True,
)
def torch_permute(
hidden_states: torch.Tensor,
topk_ids: torch.Tensor,
# token_expert_indices: torch.Tensor,
topk: int,
n_expert: int,
n_local_expert: int,
start_expert: int,
expert_map: torch.Tensor | None = None,
) -> list[torch.Tensor]:
n_token = hidden_states.shape[0]
if expert_map is not None:
is_local_expert = expert_map[topk_ids] != -1
not_local_expert = expert_map[topk_ids] == -1
topk_ids = is_local_expert * (topk_ids - start_expert) + not_local_expert * (
topk_ids + n_expert
)
token_expert_indices = torch.arange(
0, n_token * topk, dtype=torch.int32, device=hidden_states.device
).reshape((n_token, topk))
sorted_topk_ids, sorted_indices = torch.sort(topk_ids.flatten(), stable=True)
dst_row_id2src_row_id_map = token_expert_indices.flatten()[sorted_indices]
expert_first_token_offset = torch.zeros(
n_local_expert + 1, dtype=torch.int64, device="cuda"
)
idx = 0
for i in range(0, n_local_expert):
cnt = 0
while idx < sorted_topk_ids.numel() and sorted_topk_ids[idx] == i:
cnt += 1
idx += 1
expert_first_token_offset[i + 1] = expert_first_token_offset[i] + cnt
_, src2dst_idx = torch.sort(dst_row_id2src_row_id_map)
valid_row_idx = []
permuted_hidden_states = hidden_states[dst_row_id2src_row_id_map // topk, ...]
src_row_id2dst_row_id_map = torch.arange(
0, n_token * topk, device="cuda", dtype=torch.int32
)[src2dst_idx].reshape((n_token, topk))
valid_row_idx += [i for i in range(expert_first_token_offset[-1])]
dst_row_id2src_row_id_map[expert_first_token_offset[-1] :] = n_token * topk
return [
permuted_hidden_states,
expert_first_token_offset,
src_row_id2dst_row_id_map,
dst_row_id2src_row_id_map,
valid_row_idx,
]
def torch_unpermute(
permuted_hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
token_expert_indices: torch.Tensor,
src_row_id2dst_row_id_map: torch.Tensor,
valid_row_idx: torch.Tensor,
topk: int,
n_expert: int,
) -> torch.Tensor:
# ignore invalid row
n_hidden = permuted_hidden_states.shape[1]
mask = torch.zeros(permuted_hidden_states.shape[0], dtype=bool, device="cuda")
mask[valid_row_idx] = True
permuted_hidden_states[~mask] = 0
permuted_hidden_states = permuted_hidden_states[
src_row_id2dst_row_id_map.flatten(), ...
]
permuted_hidden_states = permuted_hidden_states.view(-1, topk, n_hidden)
output = (
(permuted_hidden_states * topk_weights.unsqueeze(2))
.sum(1)
.to(permuted_hidden_states.dtype)
)
return output
@pytest.mark.parametrize("n_token", [1, 33, 1024, 5000])
@pytest.mark.parametrize("n_hidden", [2048, 7168])
@pytest.mark.parametrize("n_expert", NUM_EXPERTS)
@pytest.mark.parametrize("topk", TOP_KS)
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("ep_size", EP_SIZE)
@pytest.mark.parametrize("use_scratch", [False, True])
def test_moe_permute_unpermute(
n_token: int,
n_hidden: int,
topk: int,
n_expert: int,
ep_size: int,
dtype: torch.dtype,
use_scratch: bool,
workspace_init,
):
if not moe_permute_unpermute_supported():
pytest.skip("moe_permute_unpermute is not supported on this platform.")
ep_rank = np.random.randint(0, ep_size)
expert_map = None
n_local_expert = n_expert
if ep_size != 1:
n_local_expert, expert_map, _ = determine_expert_map(ep_size, ep_rank, n_expert)
expert_map = expert_map.cuda()
start_expert = n_local_expert * ep_rank
set_random_seed(0)
hidden_states = torch.randn((n_token, n_hidden), device="cuda").to(dtype)
gating_output = torch.randn((n_token, n_expert), device="cuda").to(dtype)
topk_weights, topk_ids, token_expert_indices = fused_topk(
hidden_states, gating_output, topk, False
)
(
gold_permuted_hidden_states,
gold_expert_first_token_offset,
gold_inv_permuted_idx,
gold_permuted_idx,
valid_row_idx,
) = torch_permute(
hidden_states,
topk_ids,
# token_expert_indices,
topk,
n_expert,
n_local_expert,
start_expert,
expert_map=expert_map,
)
scratch = None
if use_scratch:
scratch = get_moe_permute_scratch(
num_experts=n_expert,
num_local_experts=n_local_expert,
device=hidden_states.device,
)
(
permuted_hidden_states,
_,
expert_first_token_offset,
inv_permuted_idx,
_,
) = moe_permute(
hidden_states=hidden_states,
a1q_scale=None,
topk_ids=topk_ids,
n_expert=n_expert,
n_local_expert=n_local_expert,
expert_map=expert_map,
scratch=scratch,
)
# check expert_first_token_offset
torch.testing.assert_close(
gold_expert_first_token_offset, expert_first_token_offset, atol=0, rtol=0
)
# check src_row_id2dst_row_id_map
torch.testing.assert_close(
gold_inv_permuted_idx.flatten(), inv_permuted_idx, atol=0, rtol=0
)
# check permuted_hidden_states, only valid token
torch.testing.assert_close(
gold_permuted_hidden_states[valid_row_idx],
permuted_hidden_states[valid_row_idx],
atol=0,
rtol=0,
)
# add a random tensor to simulate group gemm
result0 = 0.5 * permuted_hidden_states + torch.randn_like(permuted_hidden_states)
result4 = torch.empty_like(hidden_states)
moe_unpermute(
result4, result0, topk_weights, inv_permuted_idx, expert_first_token_offset
)
gold4 = torch_unpermute(
result0,
topk_weights,
topk_ids,
token_expert_indices,
inv_permuted_idx,
valid_row_idx,
topk,
n_local_expert,
)
# check unpermuted hidden
torch.testing.assert_close(result4, gold4, atol=2e-2, rtol=0)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("n_token", [1, 33, 128])
@pytest.mark.parametrize("topk", [1, 6, 8])
def test_moe_permute_reuses_scratch_buffers(
dtype: torch.dtype, n_token: int, topk: int, workspace_init
):
if not moe_permute_unpermute_supported():
pytest.skip("moe_permute_unpermute is not supported on this platform.")
n_hidden = 2048
n_expert = 16
hidden_states = torch.randn((n_token, n_hidden), device="cuda").to(dtype)
gating_output = torch.randn((n_token, n_expert), device="cuda").to(dtype)
_, topk_ids, _ = fused_topk(hidden_states, gating_output, topk, False)
scratch_config = dict(
num_experts=n_expert,
num_local_experts=n_expert,
device=hidden_states.device,
)
scratch = get_moe_permute_scratch(**scratch_config)
first = moe_permute(
hidden_states=hidden_states,
a1q_scale=None,
topk_ids=topk_ids,
n_expert=n_expert,
scratch=scratch,
)
current_workspace_manager().lock()
assert get_moe_permute_scratch(**scratch_config) is scratch
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
second = moe_permute(
hidden_states=hidden_states,
a1q_scale=None,
topk_ids=topk_ids,
n_expert=n_expert,
scratch=get_moe_permute_scratch(**scratch_config),
)
for _ in range(2):
hidden_states.add_(1)
topk_ids.copy_(topk_ids.roll(1, 0))
graph.replay()
expected = moe_permute(hidden_states, None, topk_ids, n_expert)
for actual, reference in zip(second, expected):
if actual is not None:
torch.testing.assert_close(actual, reference)
(
permuted_hidden_states_1,
_,
expert_first_token_offset_1,
inv_permuted_idx_1,
permuted_idx_1,
) = first
(
permuted_hidden_states_2,
_,
expert_first_token_offset_2,
inv_permuted_idx_2,
permuted_idx_2,
) = second
torch.testing.assert_close(expert_first_token_offset_1, expert_first_token_offset_2)
torch.testing.assert_close(inv_permuted_idx_1, inv_permuted_idx_2)
torch.testing.assert_close(permuted_idx_1, permuted_idx_2)
assert (
expert_first_token_offset_1.untyped_storage().data_ptr()
== expert_first_token_offset_2.untyped_storage().data_ptr()
)
assert (
inv_permuted_idx_1.untyped_storage().data_ptr()
== scratch.inv_permuted_idx.untyped_storage().data_ptr()
)
assert (
permuted_idx_1.untyped_storage().data_ptr()
== scratch.permuted_idx.untyped_storage().data_ptr()
)
def test_moe_permute_scratch_reused_across_graph_sizes(workspace_init) -> None:
"""Switching graph sizes must not expose metadata left by a larger batch."""
if not moe_permute_unpermute_supported():
pytest.skip("moe_permute_unpermute is not supported on this platform.")
device = torch.device("cuda:0")
scratch = get_moe_permute_scratch(
num_experts=16,
num_local_experts=16,
device=device,
)
scratch.reserve(128 * 6)
current_workspace_manager().lock()
runs = []
for n_token in (128, 1, 33):
hidden = torch.randn((n_token, 2048), dtype=torch.bfloat16, device=device)
topk_ids = torch.randint(16, (n_token, 6), device=device)
moe_permute(hidden, None, topk_ids, 16, scratch=scratch)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
output = moe_permute(hidden, None, topk_ids, 16, scratch=scratch)
runs.append((graph, hidden, topk_ids, output))
for graph, hidden, topk_ids, output in runs + runs[::-1]:
hidden.add_(1)
topk_ids.copy_((topk_ids + 1) % 16)
graph.replay()
expected = moe_permute(hidden, None, topk_ids, 16)
for actual, reference in zip(output, expected):
if actual is not None:
torch.testing.assert_close(actual, reference)
def test_moe_permute_scratch_isolated_across_execution_slots(monkeypatch) -> None:
"""Concurrent graph replays in different ubatches/lanes cannot share scratch."""
if not moe_permute_unpermute_supported():
pytest.skip("moe_permute_unpermute is not supported on this platform.")
from vllm.v1.worker import workspace
device = torch.device("cuda:0")
manager = workspace.WorkspaceManager(device, num_ubatches=2, num_lanes=2)
monkeypatch.setattr(workspace, "_manager", manager)
ubatch = 0
monkeypatch.setattr(workspace, "dbo_current_ubatch_id", lambda: ubatch)
scratch_config = dict(
num_experts=16,
num_local_experts=16,
device=device,
)
runs = []
for i, n_token in enumerate((1, 7, 33, 64)):
ubatch = i // 2
stream = torch.cuda.Stream()
stream.wait_stream(torch.cuda.current_stream())
with workspace.use_workspace_lane(i % 2), torch.cuda.stream(stream):
hidden = torch.randn((n_token, 2048), dtype=torch.bfloat16, device=device)
topk_ids = torch.randint(16, (n_token, 6), device=device)
scratch = get_moe_permute_scratch(**scratch_config)
moe_permute(hidden, None, topk_ids, 16, scratch=scratch)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=stream):
output = moe_permute(hidden, None, topk_ids, 16, scratch=scratch)
runs.append((stream, graph, hidden, topk_ids, output, scratch))
torch.cuda.current_stream().wait_stream(stream)
assert len({run[-1].permuted_idx.data_ptr() for run in runs}) == 4
manager.lock()
for _ in range(3):
for i, (stream, graph, hidden, topk_ids, _, scratch) in enumerate(runs):
ubatch = i // 2
with workspace.use_workspace_lane(i % 2), torch.cuda.stream(stream):
assert get_moe_permute_scratch(**scratch_config) is scratch
hidden.add_(1)
topk_ids.copy_((topk_ids + 1) % 16)
graph.replay()
for stream, _, hidden, topk_ids, output, _ in runs:
torch.cuda.current_stream().wait_stream(stream)
expected = moe_permute(hidden, None, topk_ids, 16)
for actual, reference in zip(output, expected):
if actual is not None:
torch.testing.assert_close(actual, reference)
def test_moe_permute_scratch_without_manager(monkeypatch) -> None:
"""Standalone calls get independent scratch with initialized row indices."""
if not moe_permute_unpermute_supported():
pytest.skip("moe_permute_unpermute is not supported on this platform.")
from vllm.v1.worker import workspace
monkeypatch.setattr(workspace, "_manager", None)
config = dict(
num_experts=4,
num_local_experts=4,
device=torch.device("cuda"),
)
first = get_moe_permute_scratch(**config)
second = get_moe_permute_scratch(**config)
first.reserve(8)
second.reserve(8)
assert (
first.token_expert_indices.data_ptr() != second.token_expert_indices.data_ptr()
)
torch.testing.assert_close(
first.token_expert_indices, torch.arange(8, dtype=torch.int32, device="cuda")
)
def test_moe_permute_ignores_invalid_expert_ids_with_scratch() -> None:
if not moe_permute_unpermute_supported():
pytest.skip("moe_permute_unpermute is not supported on this platform.")
hidden_states = torch.arange(5 * 16, dtype=torch.bfloat16, device="cuda").view(
5, 16
)
topk_ids = torch.tensor([[0], [-1], [1], [4], [2]], device="cuda")
expert_map = torch.tensor([0, 1, -1, -1], dtype=torch.int32, device="cuda")
scratch = MoEPermuteScratch(
num_experts=4,
num_local_experts=2,
device=hidden_states.device,
)
permuted, _, expert_offsets, inverse, _ = moe_permute(
hidden_states=hidden_states,
a1q_scale=None,
topk_ids=topk_ids,
n_expert=4,
n_local_expert=2,
expert_map=expert_map,
scratch=scratch,
)
torch.testing.assert_close(
expert_offsets,
torch.tensor([0, 1, 2], dtype=torch.int64, device="cuda"),
)
torch.testing.assert_close(permuted[:2], hidden_states[[0, 2]])
assert torch.all(inverse[[1, 3, 4]] >= expert_offsets[-1])
expected = torch.zeros_like(hidden_states)
expected[[0, 2]] = hidden_states[[0, 2]]
output = torch.empty_like(hidden_states)
moe_unpermute(
output, permuted, torch.ones(5, 1, device="cuda"), inverse, expert_offsets
)
torch.testing.assert_close(output, expected)
expected_inverse = inverse.clone()
expected_offsets = expert_offsets.clone()
expert_offsets, indices = moe_prepare_scatter(topk_ids, expert_map, scratch)
torch.testing.assert_close(indices.flatten(), expected_inverse)
torch.testing.assert_close(expert_offsets, expected_offsets)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
expert_offsets, indices = moe_prepare_scatter(topk_ids, expert_map, scratch)
topk_ids.zero_()
graph.replay()
expected_inverse = torch.arange(5, dtype=torch.int32, device="cuda")
expected_offsets = torch.tensor([0, 5, 5], dtype=torch.int64, device="cuda")
torch.testing.assert_close(indices.flatten(), expected_inverse)
torch.testing.assert_close(expert_offsets, expected_offsets)
def test_moe_permute_scratch_grows_until_workspace_lock(workspace_init) -> None:
"""Capacity follows the inputs seen in warmup, whatever their topk or size."""
if not moe_permute_unpermute_supported():
pytest.skip("moe_permute_unpermute is not supported on this platform.")
device = torch.device("cuda:0")
scratch = get_moe_permute_scratch(
num_experts=16, num_local_experts=16, device=device
)
def run(n_token: int, topk: int, dtype: torch.dtype) -> None:
hidden = torch.randn((n_token, 64), device=device).to(dtype)
topk_ids = torch.randint(16, (n_token, topk), device=device)
actual = moe_permute(hidden, None, topk_ids, 16, scratch=scratch)
expected = moe_permute(hidden, None, topk_ids, 16)
for a, e in zip(actual, expected):
if a is not None:
torch.testing.assert_close(a.float(), e.float())
run(8, 6, torch.bfloat16)
run(64, 6, torch.bfloat16)
run(384, 1, torch.float8_e4m3fn)
assert scratch.max_expanded_rows == 384
current_workspace_manager().lock()
run(64, 6, torch.bfloat16)
with pytest.raises(AssertionError, match="Scratch growth is not allowed"):
run(65, 6, torch.bfloat16)
# Experts rebuilt after the lock (e.g. a weight reload) keep the capacity.
rebuilt = get_moe_permute_scratch(
num_experts=16, num_local_experts=16, device=device
)
assert rebuilt.max_expanded_rows == 384
-247
View File
@@ -1,247 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Two-rank MoRI/AITER numerical counterpart to the DeepEP v2 MoE matrix."""
from dataclasses import replace
import pytest
import torch
from tests.kernels.moe.modular_kernel_tools.parallel_utils import (
parallel_launch_with_config,
)
from tests.kernels.moe.utils import make_dummy_moe_config, make_test_weights
from tests.kernels.utils import torch_experts
from tests.utils import multi_gpu_test
from vllm.config import ParallelConfig, SchedulerConfig, VllmConfig
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEParallelConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_mori
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.worker.workspace import init_workspace_manager
MNKS = [
(1, 256, 256),
(2, 256, 512),
(3, 1024, 2048),
(32, 256, 1024),
(45, 512, 2048),
(64, 1024, 1024),
(222, 1024, 2048),
]
def _mxfp4_aiter_weights(w1, w2, start, end):
"""MXFP4-quantize experts [start, end) into AITER's W4A4 kernel layout."""
from aiter.ops.triton.quant import dynamic_mxfp4_quant
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
Mxfp4MoeBackend,
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format,
make_mxfp4_moe_quant_config,
)
def quantize(w):
w = w[start:end]
q, s = dynamic_mxfp4_quant(w.flatten(0, 1))
return q.view(*w.shape[:2], -1), s.view(*w.shape[:2], -1)
backend = Mxfp4MoeBackend.AITER_MXFP4_MXFP4
(q1, s1), (q2, s2) = quantize(w1), quantize(w2)
q1, q2, s1, s2, _, _ = convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
backend, None, q1, q2, s1, s2
)
return q1, q2, make_mxfp4_moe_quant_config(backend, s1, s2)
def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, scheme, graph):
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
AiterFp8ChannelExperts,
AiterMxfp4Experts,
AiterUnquantizedExperts,
rocm_aiter_fused_experts,
)
from vllm.model_executor.layers.fused_moe.prepare_finalize.mori import (
MoriPrepareAndFinalize,
)
rocm_aiter_ops.refresh_env_variables()
init_workspace_manager(pgi.device)
set_random_seed(47)
fp8 = scheme == "fp8_token_channel"
mxfp4 = scheme == "mxfp4"
dtype = current_platform.fp8_dtype() if fp8 else None
(_, w1, s1, _), (_, w2, s2, _) = make_test_weights(
32, n, k, quant_dtype=dtype, per_out_ch_quant=fp8
)
# Share global expert weights, but give each rank different token inputs.
set_random_seed(101 + pgi.rank)
x = torch.randn(m, k, device=pgi.device, dtype=torch.bfloat16) / 10
routing = torch.randn(m, 32, device=pgi.device)
# Guarantee a route to both ranks, including the one-token case.
routing[:, 0] = routing[:, 16] = float("inf")
ids = routing.topk(6, dim=-1).indices.to(torch.int32)
weights = torch.randn(m, 6, device=pgi.device, dtype=torch.float32)
parallel = replace(
FusedMoEParallelConfig.make_no_parallel(),
dp_size=2,
dp_rank=pgi.rank,
ep_size=2,
ep_rank=pgi.rank,
use_ep=True,
all2all_backend="mori_high_throughput",
)
config = replace(
make_dummy_moe_config(
num_experts=32,
num_local_experts=16,
experts_per_token=6,
hidden_dim=k,
intermediate_size=n,
max_num_tokens=256,
),
moe_parallel_config=parallel,
)
start, end = pgi.rank * 16, (pgi.rank + 1) * 16
if mxfp4:
local_w1, local_w2, quant = _mxfp4_aiter_weights(w1, w2, start, end)
ref_w1, ref_w2, ref_quant = _mxfp4_aiter_weights(w1, w2, 0, 32)
else:
quant = FusedMoEQuantConfig.make(
dtype,
per_act_token_quant=fp8,
per_out_ch_quant=fp8,
w1_scale=None if s1 is None else s1[start:end].contiguous(),
w2_scale=None if s2 is None else s2[start:end].contiguous(),
)
local_w1, local_w2 = rocm_aiter_ops.shuffle_weights(
w1[start:end].contiguous(), w2[start:end].contiguous()
)
local_w1.is_shuffled = local_w2.is_shuffled = True
prepare = maybe_make_prepare_finalize(config, quant)
assert isinstance(prepare, MoriPrepareAndFinalize)
assert prepare.use_fp8_dispatch == fp8
assert prepare.use_fp4_dispatch == mxfp4
assert prepare.num_dispatchers() == 2
experts_cls = {
"bf16": AiterUnquantizedExperts,
"fp8_token_channel": AiterFp8ChannelExperts,
"mxfp4": AiterMxfp4Experts,
}[scheme]
experts = experts_cls(config, quant)
assert not experts.expects_unquantized_inputs
kernel = FusedMoEKernel(prepare, experts)
expert_mask = torch.zeros(32, dtype=torch.int32, device=pgi.device)
expert_mask[start:end] = 1
def reference(route_weights):
if mxfp4:
# AITER's non-EP MXFP4 path quantizes the BF16 input the same way
# as FP4 dispatch, so the BF16 tolerances below still apply.
return rocm_aiter_fused_experts(
x, ref_w1, ref_w2, route_weights, ids, config, quant_config=ref_quant
)
return torch_experts(
x,
w1,
w2,
route_weights,
ids,
w1_scale=s1,
w2_scale=s2,
quant_dtype=dtype,
per_act_token_quant=fp8,
)
def run():
with set_forward_context(None, vllm_config):
return kernel.apply(
x,
local_w1,
local_w2,
weights,
ids,
activation=MoEActivation.SILU,
global_num_experts=32,
expert_map=expert_mask,
apply_router_weight_on_input=False,
)
def check(actual):
expected = reference(weights)
assert actual.shape == expected.shape
assert actual.dtype == expected.dtype
local_weights = weights * ((ids >= start) & (ids < end))
missing_remote = reference(local_weights)
norm = expected.float().norm()
assert norm > 0
norm_budget = 0.2 if fp8 else 0.06
assert (actual.float() - expected.float()).norm() / norm < norm_budget
# The norm check also rejects low-signal zero or rank-local results.
for incorrect in (torch.zeros_like(expected), missing_remote):
assert (incorrect.float() - expected.float()).norm() / norm > norm_budget
if fp8:
assert (
torch.isclose(actual, expected, atol=0.2, rtol=0.2)
.float()
.mean()
.item()
> 0.99
)
else:
torch.testing.assert_close(actual, expected, atol=0.06, rtol=0.06)
assert torch.isfinite(actual).all()
actual = run()
check(actual)
if graph:
saved = actual.clone()
run()
capture = torch.cuda.CUDAGraph()
with torch.cuda.graph(capture):
captured = run()
for factor in (-0.5, 1.5):
x.mul_(factor)
weights.mul_(-0.75)
capture.replay()
torch.accelerator.synchronize()
check(captured)
torch.testing.assert_close(actual, saved, atol=0, rtol=0)
@pytest.mark.skipif(
not current_platform.is_rocm() or not has_mori(), reason="Requires ROCm MoRI"
)
@pytest.mark.parametrize("m,n,k", MNKS)
@pytest.mark.parametrize("scheme", ["bf16", "fp8_token_channel", "mxfp4"])
@multi_gpu_test(num_gpus=2)
def test_mori_moe(m, n, k, scheme):
if scheme == "mxfp4":
from vllm.platforms.rocm import on_gfx950
if not on_gfx950():
pytest.skip("AITER MXFP4 W4A4 MoE requires gfx950")
config = VllmConfig(
parallel_config=ParallelConfig(
data_parallel_size=2,
enable_expert_parallel=True,
all2all_backend="mori_high_throughput",
),
scheduler_config=SchedulerConfig.default_factory(max_num_batched_tokens=256),
)
env = {
"VLLM_ROCM_USE_AITER": "1",
"VLLM_ROCM_USE_AITER_MOE": "1",
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "0",
}
parallel_launch_with_config(2, _mori_worker, config, env, m, n, k, scheme, m == 32)
-462
View File
@@ -1,462 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for SM100 CUTLASS MXFP4 x MXFP4 grouped MoE kernels."""
import random
import pytest
import torch
from tests.kernels.utils import torch_moe_single
from vllm import _custom_ops as ops
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
random.seed(42)
set_random_seed(42)
MXFP4_BLOCK_SIZE = 32
def align(val: int, alignment: int = 128) -> int:
return int((val + alignment - 1) // alignment * alignment)
def calc_diff(x, y):
x, y = x.double(), y.double()
denominator = (x * x + y * y).sum()
sim = 2 * (x * y).sum() / denominator
return 1 - sim
def is_sm100_supported() -> bool:
return current_platform.is_cuda() and current_platform.is_device_capability_family(
100
)
def compute_ref_output(
input_tensor: torch.Tensor,
weight_list: list[torch.Tensor],
expert_offsets: list[int],
expert_offset: int,
num_experts: int,
) -> torch.Tensor:
"""Reference output using torch_moe_single with top-1 routing."""
score = torch.full(
(expert_offset, num_experts),
-1e9,
device=input_tensor.device,
dtype=torch.float32,
)
for g in range(num_experts):
start = expert_offsets[g]
end = expert_offsets[g + 1] if g + 1 < num_experts else expert_offset
score[start:end, g] = 0.0
return torch_moe_single(
input_tensor, torch.stack(weight_list, dim=0), score, topk=1
)
@pytest.mark.skipif(
not is_sm100_supported(),
reason="cutlass_mxfp4_group_mm requires CUDA SM100",
)
@pytest.mark.parametrize("num_experts", [8, 16, 32])
@pytest.mark.parametrize("out_dtype", [torch.bfloat16])
def test_cutlass_mxfp4_grouped_mm(num_experts, out_dtype):
"""Test the MXFP4 grouped GEMM kernel by:
1. Creating random per-expert inputs and weights
2. Quantizing both to MXFP4 using the CUDA kernel
3. Running the CUTLASS grouped GEMM
4. Comparing against BF16 reference
"""
device = "cuda"
alignment = 128
# N and K must be multiples of 128 for clean swizzle layout
n_g = random.randint(1, 16) * alignment
k_g = random.randint(1, 16) * alignment
expert_offset = 0
expert_offsets_input = []
problem_sizes = []
input_list = []
weight_list = []
for g in range(num_experts):
m_g = random.randint(1, 256)
expert_offsets_input.append(expert_offset)
expert_offset += m_g
problem_sizes.append([m_g, n_g, k_g])
input_list.append(
torch.normal(0.0, std=0.5, size=(m_g, k_g), device=device, dtype=out_dtype)
)
weight_list.append(
torch.normal(0.0, std=0.5, size=(n_g, k_g), device=device, dtype=out_dtype)
)
input_tensor = torch.concat(input_list, dim=0) # [M_total, K]
# --- Quantize INPUTS via mxfp4_experts_quant ---
input_bs_offsets = []
tot = 0
for g in range(num_experts):
input_bs_offsets.append(tot)
tot += align(problem_sizes[g][0], 128)
input_bs_offsets.append(tot)
_inp_expert_offsets = torch.tensor(
expert_offsets_input + [expert_offset], device=device, dtype=torch.int32
)
_inp_bs_offsets = torch.tensor(input_bs_offsets, device=device, dtype=torch.int32)
input_quant, input_sf = ops.mxfp4_experts_quant(
input_tensor,
_inp_expert_offsets,
_inp_bs_offsets,
num_experts,
topk=1,
)
# --- Quantize WEIGHTS via mxfp4_experts_quant ---
# Treat each expert's N weight rows as an "expert" with N tokens
weight_tensor = torch.concat(weight_list, dim=0) # [E*N, K]
weight_expert_offsets = [g * n_g for g in range(num_experts)] + [num_experts * n_g]
# N is always multiple of 128, so blockscale offsets are clean
weight_bs_offsets = [g * n_g for g in range(num_experts)] + [num_experts * n_g]
_wt_expert_offsets = torch.tensor(
weight_expert_offsets, device=device, dtype=torch.int32
)
_wt_bs_offsets = torch.tensor(weight_bs_offsets, device=device, dtype=torch.int32)
weight_quant, weight_sf = ops.mxfp4_experts_quant(
weight_tensor,
_wt_expert_offsets,
_wt_bs_offsets,
num_experts,
topk=1,
)
# Reshape weight quantized data to [E, N, K//2]
weight_quant = weight_quant[: num_experts * n_g].view(num_experts, n_g, k_g // 2)
# Reshape weight scale factors to [E, N, K//32]
# The quant kernel produces uint8 SF buffer. Each row has K//32 SFs.
scales_per_row = k_g // MXFP4_BLOCK_SIZE
weight_sf_flat = weight_sf.view(-1)[: num_experts * n_g * scales_per_row]
weight_sf_3d = weight_sf_flat.view(num_experts, n_g, scales_per_row)
# Output
output = torch.empty((expert_offset, n_g), device=device, dtype=out_dtype)
_problem_sizes = torch.tensor(problem_sizes, device=device, dtype=torch.int32)
_expert_offsets = torch.tensor(
expert_offsets_input, device=device, dtype=torch.int32
)
_input_bs = torch.tensor(input_bs_offsets[:-1], device=device, dtype=torch.int32)
# Run the MXFP4 grouped GEMM
ops.cutlass_mxfp4_moe_mm(
output,
input_quant,
weight_quant,
input_sf,
weight_sf_3d,
_problem_sizes,
_expert_offsets,
_input_bs,
)
# Reference: BF16 matmul
ref_output = compute_ref_output(
input_tensor=input_tensor,
weight_list=weight_list,
expert_offsets=expert_offsets_input,
expert_offset=expert_offset,
num_experts=num_experts,
)
# Compare per-expert
for g in range(num_experts):
start = expert_offsets_input[g]
end = expert_offsets_input[g + 1] if g + 1 < num_experts else expert_offset
if start == end:
continue
baseline = ref_output[start:end]
actual = output[start:end]
diff = calc_diff(actual, baseline)
print(
f"m_g={end - start} n_g={n_g} k_g={k_g} "
f"num_experts={num_experts}, "
f"out_dtype={out_dtype}, diff={diff:.5f}"
)
# FP4 quantization is very lossy (~4 bits precision)
# Comparing quantized vs full-precision gives cosine diff of 0.05-0.15
assert diff < 0.15, f"Expert {g}: diff={diff:.5f} exceeds threshold"
@pytest.mark.skipif(
not is_sm100_supported(),
reason="mxfp4_experts_quant requires CUDA SM100",
)
def test_mxfp4_experts_quant_basic():
"""Basic smoke test for the MXFP4 experts quantization kernel."""
device = "cuda"
num_experts = 4
k = 256
tokens_per_expert = 16
total_tokens = tokens_per_expert * num_experts
input_tensor = torch.randn(total_tokens, k, device=device, dtype=torch.bfloat16) / 5
expert_offsets = [i * tokens_per_expert for i in range(num_experts + 1)]
blockscale_offsets = [
align(i * tokens_per_expert, 128) for i in range(num_experts + 1)
]
_expert_offsets = torch.tensor(expert_offsets, device=device, dtype=torch.int32)
_blockscale_offsets = torch.tensor(
blockscale_offsets, device=device, dtype=torch.int32
)
output, output_sf = ops.mxfp4_experts_quant(
input_tensor,
_expert_offsets,
_blockscale_offsets,
num_experts,
topk=1,
)
assert output.shape == (total_tokens, k // 2)
assert output.dtype == torch.uint8
assert output_sf.dtype == torch.uint8
assert output.any(), "Quantized output is all zeros"
print(
f"MXFP4 experts quant: output shape={output.shape}, sf shape={output_sf.shape}"
)
print("PASSED")
def untile_cutlass_scale(scale_raw: torch.Tensor, rows: int, K: int) -> torch.Tensor:
"""Convert CUTLASS tiled scale back to flat [M, K//32] layout.
CUTLASS tiled layout: [numMTiles, numKTiles, 32(outerM), 4(innerM), 4(innerK)]
Produced by: padded.reshape(numMTiles, 4, 32, numKTiles, 4).permute(0,3,2,1,4)
To undo: tiled.permute(0, 3, 2, 1, 4).reshape(padded_M, padded_sK)
"""
num_scale_cols = K // MXFP4_BLOCK_SIZE
num_m_tiles = (rows + 127) // 128
num_k_tiles = (num_scale_cols + 3) // 4
padded_M = num_m_tiles * 128
padded_sK = num_k_tiles * 4
scale_bytes = scale_raw.view(torch.uint8).flatten()
total_bytes = padded_M * padded_sK
tiled = scale_bytes[:total_bytes].reshape(num_m_tiles, num_k_tiles, 32, 4, 4)
undone = tiled.permute(0, 3, 2, 1, 4).contiguous()
return undone.reshape(padded_M, padded_sK)[:rows, :num_scale_cols]
def compute_reference_e8m0_scale(block_max: float) -> int:
"""Compute the expected OCP MX spec E8M0 scale for a given block max.
The CUTLASS kernel uses round-to-nearest on the mantissa:
rounded_bits = (float_bits + (1 << 21)) & 0xFF800000
biased_exp = (rounded_bits >> 23) & 0xFF
scale_exp = max(biased_exp - 2, 0)
This ensures max_val / scale <= 6.0 for most inputs.
"""
import struct
if block_max <= 0:
return 0
# Replicate the kernel's rounding logic in Python
float_bytes = struct.pack("f", block_max)
max_bits = struct.unpack("I", float_bytes)[0]
rounded_bits = (max_bits + (1 << 21)) & 0xFF800000
biased_exp = (rounded_bits >> 23) & 0xFF
scale_exp = max(int(biased_exp) - 2, 0)
scale_exp = min(scale_exp, 254)
return scale_exp
@pytest.mark.skipif(
not is_sm100_supported(),
reason="mxfp4_experts_quant requires CUDA SM100",
)
@pytest.mark.parametrize("k", [256, 7168])
@pytest.mark.parametrize("m", [16, 64])
def test_mxfp4_experts_quant_e8m0_scale_correctness(m, k):
"""Test that mxfp4_experts_quant computes E8M0 block scales correctly
per OCP MX spec (not the NVFP4 formula).
The old buggy kernel used: floor(log2(max/6)) + 127
The fixed kernel uses: round_nearest_exp(max) - 2
This test verifies:
1. Scales match the expected OCP MX formula for all blocks
2. No block max exceeds the representable range (no unexpected saturation)
3. Reconstruction error is within expected bounds for MXFP4
"""
device = "cuda"
# Generate input with controlled range
input_tensor = torch.randn(m, k, device=device, dtype=torch.bfloat16) * 0.5
# Quantize
num_experts = 1
expert_offsets = torch.tensor([0, m], device=device, dtype=torch.int32)
num_k_tiles = (k // MXFP4_BLOCK_SIZE + 3) // 4
blockscale_offsets = torch.tensor(
[0, align(m, 128) * num_k_tiles], device=device, dtype=torch.int32
)
output_fp4, output_sf = ops.mxfp4_experts_quant(
input_tensor, expert_offsets, blockscale_offsets, num_experts, topk=1
)
# Untile scale to flat layout for verification
scale_flat = untile_cutlass_scale(output_sf, m, k)
assert scale_flat.shape == (m, k // MXFP4_BLOCK_SIZE)
# Verify each block's scale matches the OCP MX spec formula
num_blocks = k // MXFP4_BLOCK_SIZE
mismatches = 0
buggy_pattern = 0 # count blocks where scale is 1-2 lower than expected
for row in range(m):
for blk in range(num_blocks):
block_start = blk * MXFP4_BLOCK_SIZE
block_end = block_start + MXFP4_BLOCK_SIZE
block_max = (
input_tensor[row, block_start:block_end].float().abs().max().item()
)
actual_scale = scale_flat[row, blk].item()
expected_scale = compute_reference_e8m0_scale(block_max)
if actual_scale != expected_scale:
mismatches += 1
if actual_scale < expected_scale:
buggy_pattern += 1
total_blocks = m * num_blocks
match_rate = (total_blocks - mismatches) / total_blocks
print(
f" m={m}, k={k}: scale match rate = {match_rate * 100:.2f}% "
f"({mismatches}/{total_blocks} mismatches)"
)
# The fixed kernel should match the reference formula exactly
assert match_rate > 0.99, (
f"E8M0 scale match rate too low: {match_rate * 100:.2f}%. "
f"Buggy pattern (scale too low): {buggy_pattern}/{mismatches}. "
f"This suggests the NVFP4 formula bug is present."
)
# Extra check: if most mismatches show scale < expected, it's the old bug
if mismatches > 0:
assert buggy_pattern / mismatches < 0.5, (
f"Most scale mismatches show scale too LOW ({buggy_pattern}/{mismatches}). "
"This is the signature of the NVFP4 formula bug in nvfp4_utils.cuh."
)
# Verify reconstruction error is within MXFP4 expected bounds
# Dequantize and check cosine similarity
fp4_lut = torch.tensor(
[0, 0.5, 1, 1.5, 2, 3, 4, 6, 0, -0.5, -1, -1.5, -2, -3, -4, -6],
device=device,
dtype=torch.float32,
)
lo = (output_fp4 & 0x0F).long()
hi = ((output_fp4 >> 4) & 0x0F).long()
unpacked = torch.stack([lo, hi], dim=-1).reshape(m, k)
fp4_vals = fp4_lut[unpacked]
scales_expanded = 2.0 ** (scale_flat.float() - 127.0)
scales_expanded = scales_expanded.unsqueeze(-1).expand(-1, -1, MXFP4_BLOCK_SIZE)
scales_expanded = scales_expanded.reshape(m, k)
recon = (fp4_vals * scales_expanded).bfloat16()
# Cosine similarity should be > 0.99 for well-behaved MXFP4 quantization
cos_sim = torch.nn.functional.cosine_similarity(
recon.float().flatten().unsqueeze(0),
input_tensor.float().flatten().unsqueeze(0),
).item()
max_abs_diff = (recon.float() - input_tensor.float()).abs().max().item()
print(
f" Reconstruction: cosine_sim={cos_sim:.6f}, max_abs_diff={max_abs_diff:.4f}"
)
assert cos_sim > 0.99, (
f"Reconstruction cosine similarity too low: {cos_sim:.6f}. "
f"Expected > 0.99 for correct MXFP4 quantization."
)
# With correct E8M0, max abs diff should be bounded by scale * 6
# (worst case: value just below threshold rounds to wrong FP4 code)
assert max_abs_diff < 1.0, (
f"Max reconstruction error too large: {max_abs_diff:.4f}. "
"Likely caused by incorrect E8M0 scale (values saturating to ±6)."
)
@pytest.mark.skipif(
not is_sm100_supported(),
reason="mxfp4_experts_quant requires CUDA SM100",
)
def test_mxfp4_experts_quant_no_saturation():
"""Test that the E8M0 scale is large enough to avoid unexpected saturation.
With the buggy NVFP4 formula, the scale was too small causing most values
to saturate to ±6 in FP4. The fixed OCP MX formula should ensure that
block_max / scale <= 6.0 (the max E2M1 value) in almost all cases.
"""
device = "cuda"
m, k = 128, 1024
# Use inputs with known range to make saturation detectable
input_tensor = torch.randn(m, k, device=device, dtype=torch.bfloat16) * 0.5
num_experts = 1
expert_offsets = torch.tensor([0, m], device=device, dtype=torch.int32)
num_k_tiles = (k // MXFP4_BLOCK_SIZE + 3) // 4
blockscale_offsets = torch.tensor(
[0, align(m, 128) * num_k_tiles], device=device, dtype=torch.int32
)
output_fp4, output_sf = ops.mxfp4_experts_quant(
input_tensor, expert_offsets, blockscale_offsets, num_experts, topk=1
)
# Check saturation rate: count FP4 values that are ±6 (codes 7 and 15)
lo = output_fp4 & 0x0F
hi = (output_fp4 >> 4) & 0x0F
# Code 7 = +6.0, code 15 = -6.0
saturated = ((lo == 7) | (lo == 15) | (hi == 7) | (hi == 15)).sum().item()
total_values = m * k
saturation_rate = saturated / total_values
print(
f" Saturation rate: {saturation_rate * 100:.2f}% "
f"({saturated}/{total_values} values at ±6)"
)
# For Gaussian input with std=0.5, saturation should be very rare
# (±6 * scale is far from the typical range).
# The buggy kernel had ~30-50% saturation; fixed should be < 5%.
assert saturation_rate < 0.05, (
f"FP4 saturation rate too high: {saturation_rate * 100:.2f}%. "
"This suggests the E8M0 scale is too small (NVFP4 formula bug). "
"Expected < 5% for Gaussian(0, 0.5) input with correct OCP MX scale."
)
if __name__ == "__main__":
pytest.main([__file__, "-v", "-s"])
@@ -1,176 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MXFP8 MoE backend selection for the AITER FlyDSL kernel (gfx950).
GPU-free: mocks the platform (gfx950) and the aiter fused-MoE enable flag, then
exercises the oracle so the FlyDSL backend is auto-picked when usable (including
under expert parallelism, since apply() forwards the expert_map as aiter's
expert_mask) and skipped (native fallback) when the device is unsupported or the
aiter runtime is disabled.
"""
import dataclasses
from unittest.mock import patch
import pytest
from vllm.platforms import current_platform
if not current_platform.is_rocm():
pytest.skip("This test can only run on ROCm.", allow_module_level=True)
from tests.kernels.moe.utils import make_dummy_moe_config # noqa: E402
from vllm.model_executor.layers.fused_moe.activation import ( # noqa: E402
MoEActivation,
)
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp8_moe import ( # noqa: E402
_AITER_SWIGLU_ALPHA,
_AITER_SWIGLU_BETA,
AiterMxfp8Experts,
)
from vllm.model_executor.layers.fused_moe.experts.mxfp8_emulation_moe import ( # noqa: E402
Mxfp8EmulationTritonExperts,
)
from vllm.model_executor.layers.fused_moe.modular_kernel import ( # noqa: E402
FusedMoEActivationFormat,
)
from vllm.model_executor.layers.fused_moe.oracle.fp8 import ( # noqa: E402
Fp8MoeBackend,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import ( # noqa: E402
_BACKEND_NAME_MAP,
_SUPPORTED_BACKENDS,
_mxfp8_backend_to_kernel_cls,
_select_kernel_cls,
select_mxfp8_moe_backend,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import ( # noqa: E402
kMxfp8Dynamic,
kMxfp8Static,
)
from vllm._aiter_ops import rocm_aiter_ops # noqa: E402 # isort: skip
_AITER_MOD = "vllm.model_executor.layers.fused_moe.experts.aiter_mxfp8_moe"
def _config(ep_size: int = 1):
# AiterMxfp8Experts hardcodes SwiGLU-OAI: match its required activation and
# alpha/beta so is_supported_config doesn't reject the config on those grounds.
cfg = make_dummy_moe_config(
num_experts=128,
experts_per_token=4,
hidden_dim=6144,
activation=MoEActivation.SWIGLUOAI_UNINTERLEAVE,
)
cfg = dataclasses.replace(
cfg, swiglu_alpha=_AITER_SWIGLU_ALPHA, swiglu_beta=_AITER_SWIGLU_BETA
)
if ep_size != 1:
cfg = dataclasses.replace(
cfg,
moe_parallel_config=dataclasses.replace(
cfg.moe_parallel_config, ep_size=ep_size, use_ep=True
),
)
return cfg
def _gfx950():
"""Patch the platform so the device gate (gfx950 / MX) passes off-ROCm."""
return patch.multiple(
f"{_AITER_MOD}.current_platform",
is_rocm=lambda: True,
supports_mx=lambda: True,
)
def _aiter_moe_enabled(present: bool):
return patch.object(rocm_aiter_ops, "is_fused_moe_enabled", return_value=present)
def test_aiter_mxfp8_registered():
"""The FlyDSL backend is auto-selectable and reachable via --moe-backend aiter."""
assert Fp8MoeBackend.AITER_MXFP8 in _SUPPORTED_BACKENDS
assert _BACKEND_NAME_MAP["aiter"] is Fp8MoeBackend.AITER_MXFP8
assert _mxfp8_backend_to_kernel_cls(Fp8MoeBackend.AITER_MXFP8) == [
AiterMxfp8Experts
]
@pytest.mark.parametrize("ep_size", [1, 2])
def test_ep_supported(ep_size):
"""FlyDSL accepts both TP and EP: apply() forwards expert_map as expert_mask."""
assert (
AiterMxfp8Experts._supports_parallel_config(
_config(ep_size).moe_parallel_config
)
is True
)
@pytest.mark.parametrize(
"present,ep_size,supported,reason_substr",
[
(True, 1, True, None), # gfx950 + aiter MoE + TP -> selectable
(True, 2, True, None), # gfx950 + aiter MoE + EP -> selectable (expert_mask)
(
False,
1,
False,
"does not support current device",
), # disabled -> not selected
],
)
def test_is_supported_config(present, ep_size, supported, reason_substr):
with _gfx950(), _aiter_moe_enabled(present):
ok, reason = AiterMxfp8Experts.is_supported_config(
AiterMxfp8Experts,
_config(ep_size),
kMxfp8Static,
kMxfp8Dynamic,
FusedMoEActivationFormat.Standard,
)
assert ok is supported
if reason_substr is not None:
assert reason_substr in reason
def test_explicit_moe_backend_aiter():
"""--moe-backend aiter: returns FlyDSL when usable (TP or EP), else a clear
ValueError when the aiter MoE runtime is disabled."""
with _gfx950(), _aiter_moe_enabled(True):
assert (
_select_kernel_cls(Fp8MoeBackend.AITER_MXFP8, _config(1))
is AiterMxfp8Experts
)
assert (
_select_kernel_cls(Fp8MoeBackend.AITER_MXFP8, _config(2))
is AiterMxfp8Experts
)
with (
_gfx950(),
_aiter_moe_enabled(False),
pytest.raises(ValueError, match="does not support current device"),
):
_select_kernel_cls(Fp8MoeBackend.AITER_MXFP8, _config(1))
def test_gfx950_picks_aiter():
"""Auto-select on real ROCm hardware with aiter MoE enabled -> FlyDSL wins."""
with (
patch(f"{_AITER_MOD}.current_platform.supports_mx", return_value=True),
_aiter_moe_enabled(True),
):
backend, experts_cls = select_mxfp8_moe_backend(_config())
assert backend is Fp8MoeBackend.AITER_MXFP8
assert experts_cls is AiterMxfp8Experts
def test_gfx942_picks_emulation():
"""Flydsl unusable (e.g. gfx942, no FlyDSL support) -> native Triton
dot_scaled backend wins instead."""
with patch(f"{_AITER_MOD}.current_platform.supports_mx", return_value=False):
backend, experts_cls = select_mxfp8_moe_backend(_config())
assert backend is Fp8MoeBackend.EMULATION
assert experts_cls is Mxfp8EmulationTritonExperts
-291
View File
@@ -1,291 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from tests.kernels.moe.utils import make_dummy_moe_config, make_test_weights
from tests.kernels.quantization.nvfp4_utils import (
FLOAT4_E2M1_MAX,
FLOAT8_E4M3_MAX,
dequantize_nvfp4_to_dtype,
)
from tests.kernels.utils import torch_moe
from vllm import _custom_ops as ops
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.config import nvfp4_moe_quant_config
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
CutlassExpertsFp4,
)
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
make_moe_prepare_and_finalize_no_dp_ep,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
if not current_platform.has_device_capability(100):
pytest.skip(
"Nvfp4 Requires compute capability of 10 or above.", allow_module_level=True
)
MNK_FACTORS = [
(2, 1024, 1024),
(2, 1024, 1536),
(2, 3072, 1024),
(64, 1024, 1024),
(64, 3072, 1024),
(64, 2048, 1536),
(224, 1024, 1024),
(224, 1024, 1536),
]
@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
@pytest.mark.parametrize("e", [40, 64, 256])
@pytest.mark.parametrize("topk", [1, 6, 8])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@torch.inference_mode()
def test_cutlass_fp4_moe_no_graph(
m: int, n: int, k: int, e: int, topk: int, dtype: torch.dtype, workspace_init
):
set_random_seed(7)
with set_current_vllm_config(
VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
):
quant_blocksize = 16
a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
(_, w1_q, w1_blockscale, w1_gs), (_, w2_q, w2_blockscale, w2_gs) = (
make_test_weights(
e,
n,
k,
in_dtype=dtype,
quant_dtype="nvfp4",
block_shape=None, # use quant_blocksize?
per_out_ch_quant=False,
)
)
score = torch.randn((m, e), device="cuda", dtype=dtype)
topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False)
a1_gs = torch.ones((e,), device="cuda", dtype=torch.float32)
a2_gs = torch.ones((e,), device="cuda", dtype=torch.float32)
assert w1_gs is not None
assert w2_gs is not None
assert w1_blockscale is not None
assert w2_blockscale is not None
quant_config = nvfp4_moe_quant_config(
g1_alphas=(1 / w1_gs),
g2_alphas=(1 / w2_gs),
a1_gscale=a1_gs,
a2_gscale=a2_gs,
w1_scale=w1_blockscale,
w2_scale=w2_blockscale,
)
moe_config = make_dummy_moe_config()
kernel = mk.FusedMoEKernel(
maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
use_monolithic=False,
),
CutlassExpertsFp4(
moe_config=moe_config,
quant_config=quant_config,
),
)
cutlass_output = kernel.apply(
hidden_states=a,
w1=w1_q,
w2=w2_q,
topk_weights=topk_weights,
topk_ids=topk_ids,
global_num_experts=e,
activation=mk.MoEActivation.SILU,
apply_router_weight_on_input=False,
expert_map=None,
)
# Reference check:
a_global_scale = (
(FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.amax(a.flatten(), dim=-1)
).to(torch.float32)
a_fp4, a_scale_interleaved = ops.scaled_fp4_quant(a, a_global_scale)
a_in_dtype = dequantize_nvfp4_to_dtype(
a_fp4,
a_scale_interleaved,
a_global_scale,
dtype=a.dtype,
device=a.device,
block_size=quant_blocksize,
)
w1_d = torch.empty((e, 2 * n, k), device="cuda", dtype=dtype)
w2_d = torch.empty((e, k, n), device="cuda", dtype=dtype)
for idx in range(0, e):
w1_d[idx] = dequantize_nvfp4_to_dtype(
w1_q[idx],
w1_blockscale[idx],
w1_gs[idx],
dtype=dtype,
device=w1_q.device,
block_size=quant_blocksize,
)
w2_d[idx] = dequantize_nvfp4_to_dtype(
w2_q[idx],
w2_blockscale[idx],
w2_gs[idx],
dtype=dtype,
device=w2_q.device,
block_size=quant_blocksize,
)
torch_output = torch_moe(a_in_dtype, w1_d, w2_d, score, topk)
torch.testing.assert_close(torch_output, cutlass_output, atol=1e-1, rtol=1e-1)
# step3.5-flash uses swiglustep activation (clipped SwiGLU with limit=7.0)
# for MoE layers 43-44. This tests the non-fused activation fallback path
# in run_cutlass_moe_fp4 (apply_moe_activation + separate fp4 quantization).
# Model dims: e=288, topk=8, n=1280 (moe_intermediate_size), k=4096 (hidden)
SWIGLUSTEP_MNK_FACTORS = [
(2, 1280, 4096),
(64, 1280, 4096),
(224, 1280, 4096),
]
@pytest.mark.parametrize("m,n,k", SWIGLUSTEP_MNK_FACTORS)
@pytest.mark.parametrize("e", [64, 288])
@pytest.mark.parametrize("topk", [1, 8])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@torch.inference_mode()
def test_cutlass_fp4_moe_swiglustep(
m: int, n: int, k: int, e: int, topk: int, dtype: torch.dtype, workspace_init
):
set_random_seed(7)
with set_current_vllm_config(
VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
):
quant_blocksize = 16
a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
(_, w1_q, w1_blockscale, w1_gs), (_, w2_q, w2_blockscale, w2_gs) = (
make_test_weights(
e,
n,
k,
in_dtype=dtype,
quant_dtype="nvfp4",
block_shape=None,
per_out_ch_quant=False,
)
)
score = torch.randn((m, e), device="cuda", dtype=dtype)
topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False)
a1_gs = torch.ones((e,), device="cuda", dtype=torch.float32)
a2_gs = torch.ones((e,), device="cuda", dtype=torch.float32)
assert w1_gs is not None
assert w2_gs is not None
assert w1_blockscale is not None
assert w2_blockscale is not None
quant_config = nvfp4_moe_quant_config(
g1_alphas=(1 / w1_gs),
g2_alphas=(1 / w2_gs),
a1_gscale=a1_gs,
a2_gscale=a2_gs,
w1_scale=w1_blockscale,
w2_scale=w2_blockscale,
)
kernel = mk.FusedMoEKernel(
make_moe_prepare_and_finalize_no_dp_ep(use_monolithic=False),
CutlassExpertsFp4(
moe_config=make_dummy_moe_config(),
quant_config=quant_config,
),
)
cutlass_output = kernel.apply(
hidden_states=a,
w1=w1_q,
w2=w2_q,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=MoEActivation.SWIGLUSTEP,
global_num_experts=e,
expert_map=None,
apply_router_weight_on_input=False,
)
# Reference: dequantize everything and run torch_moe with swiglustep
a_global_scale = (
(FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.amax(a.flatten(), dim=-1)
).to(torch.float32)
a_fp4, a_scale_interleaved = ops.scaled_fp4_quant(a, a_global_scale)
a_in_dtype = dequantize_nvfp4_to_dtype(
a_fp4,
a_scale_interleaved,
a_global_scale,
dtype=a.dtype,
device=a.device,
block_size=quant_blocksize,
)
w1_d = torch.empty((e, 2 * n, k), device="cuda", dtype=dtype)
w2_d = torch.empty((e, k, n), device="cuda", dtype=dtype)
for idx in range(0, e):
w1_d[idx] = dequantize_nvfp4_to_dtype(
w1_q[idx],
w1_blockscale[idx],
w1_gs[idx],
dtype=dtype,
device=w1_q.device,
block_size=quant_blocksize,
)
w2_d[idx] = dequantize_nvfp4_to_dtype(
w2_q[idx],
w2_blockscale[idx],
w2_gs[idx],
dtype=dtype,
device=w2_q.device,
block_size=quant_blocksize,
)
torch_output = torch_moe(
a_in_dtype,
w1_d,
w2_d,
score,
topk,
activation=MoEActivation.SWIGLUSTEP,
)
torch.testing.assert_close(torch_output, cutlass_output, atol=1e-1, rtol=1e-1)
if __name__ == "__main__":
test_cutlass_fp4_moe_no_graph((2, 1024, 1024), 40, 1, torch.half)
File diff suppressed because it is too large Load Diff
@@ -1,38 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.model_executor.layers.fused_moe import TritonExperts
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
MoEPrepareAndFinalizeNoDPEPModular,
)
from .modular_kernel_tools.common import Config
from .modular_kernel_tools.profile_modular_kernel import run
@pytest.mark.skipif(
not torch.cuda.is_available(),
reason="profile_modular_kernel requires a CUDA device",
)
def test_profile_modular_kernel_smoke(tmp_path):
config = Config(
Ms=[16],
K=128,
N=256,
E=4,
topks=[2],
dtype=torch.bfloat16,
quant_config=None,
prepare_finalize_type=MoEPrepareAndFinalizeNoDPEPModular,
fused_experts_type=TritonExperts,
world_size=1,
torch_trace_dir_path=str(tmp_path),
)
run(config)
traces = list(tmp_path.glob("m*_*_trace.json"))
assert traces, "profile_modular_kernel.run did not emit any chrome traces"
File diff suppressed because it is too large Load Diff
@@ -1,136 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the AITER biased_grouped_topk expert-group count.
The AITER kernel is only instantiated for ``NUM_GRP`` in {1, 2, 4, 8}, so the
group count derived from ``num_experts`` has to be rounded to a supported
value that still divides ``num_experts``. Grouping is a no-op on this path
(``topk_group == num_expert_group``), so any such value routes identically;
the constraint is purely about which kernel exists.
"""
import pytest
import torch
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
AITER_MAX_EXPERTS_PER_GROUP as MAX_EXPERTS_PER_GROUP,
)
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
AITER_SUPPORTED_NUM_GRP as SUPPORTED_NUM_GRP,
)
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
_aiter_can_use_biased_grouped_topk,
_aiter_get_num_expert_group,
)
from vllm.model_executor.layers.fused_moe.router.grouped_topk_router import (
grouped_topk,
)
from vllm.platforms import current_platform
from vllm.platforms.rocm import on_cdna
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(), reason="ROCm-specific tests"
)
@pytest.mark.parametrize(
("num_experts", "expected"),
[
(8, 1), # fits in a single group
(32, 1), # exactly at the per-group limit
(64, 2),
(128, 4),
(256, 8),
(72, 8), # naive ceil gives 3 -> unsupported, rounds up to 8
(96, 8), # naive ceil gives 3 -> unsupported, rounds up to 8
(160, 8), # naive ceil gives 5 -> unsupported, rounds up to 8
(192, 8), # naive ceil gives 6 -> unsupported, rounds up to 8
# No supported NUM_GRP divides these within the group-size limit, so
# the naive value is kept and the call site declines the kernel.
(33, 3),
(129, 43),
(257, 257),
(320, 10),
(384, 12),
],
)
def test_group_count_and_guard(num_experts, expected):
g = _aiter_get_num_expert_group(num_experts)
assert g == expected
# The router asserts both unconditionally; rounding must never break them.
assert num_experts % g == 0
assert num_experts // g <= MAX_EXPERTS_PER_GROUP
supported = g in SUPPORTED_NUM_GRP
# Ample topk, so only the membership check can decline.
assert _aiter_can_use_biased_grouped_topk(num_experts, topk=64) == supported
if supported:
assert _aiter_can_use_biased_grouped_topk(num_experts, topk=g)
assert not _aiter_can_use_biased_grouped_topk(num_experts, topk=g - 1)
@pytest.mark.skipif(not on_cdna(), reason="CDNA ROCm only")
@pytest.mark.parametrize(
"num_experts",
[
64, # g == 2, not rounded -- covers the no-op claim itself
128, # g == 4, not rounded
256, # g == 8, not rounded
96, # naive 3 -> rounded to 8
192, # naive 6 -> rounded to 8
],
)
def test_rounded_group_count_routes_like_the_reference(num_experts):
"""The rounded group count must route identically to ``grouped_topk``.
Gating is continuous random, so exact score ties -- the one place the two
may legitimately differ -- have measure zero.
"""
torch.manual_seed(num_experts)
device = "cuda"
topk = 8
num_tokens = 83 # not a multiple of any warp/tile size
g = _aiter_get_num_expert_group(num_experts)
assert _aiter_can_use_biased_grouped_topk(num_experts, topk)
gating = torch.randn(num_tokens, num_experts, dtype=torch.float32, device=device)
bias = torch.randn(num_experts, dtype=torch.float32, device=device)
ref_weights, ref_ids = grouped_topk(
hidden_states=torch.empty(num_tokens, 1, device=device),
gating_output=gating,
topk=topk,
renormalize=True,
num_expert_group=g,
topk_group=g,
scoring_func="sigmoid",
e_score_correction_bias=bias,
)
weights = torch.empty(num_tokens, topk, dtype=torch.float32, device=device)
ids = torch.empty(num_tokens, topk, dtype=torch.int32, device=device)
rocm_aiter_ops.biased_grouped_topk(
gating,
bias,
weights,
ids,
num_expert_group=g,
topk_group=g,
need_renorm=True,
)
# Neither side promises an order within a token's top-k, so sort by expert
# id and carry the weights through the same permutation -- sorting the two
# independently would not catch a weight attached to the wrong expert.
order = ids.argsort(dim=-1)
ref_order = ref_ids.argsort(dim=-1)
torch.testing.assert_close(
ids.gather(1, order), ref_ids.to(torch.int32).gather(1, ref_order)
)
torch.testing.assert_close(
weights.gather(1, order),
ref_weights.gather(1, ref_order),
atol=1e-4,
rtol=1e-4,
)
-387
View File
@@ -1,387 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# This is a test for the AITER ops.
# It tests if the AITER ops are
# 1. correctly registered as custom ops
# 2. correctly defined the relationship between
# implementation and fake function
# 3. can be used with torch.compile
# This file will be skipped if AITER is not installed
# and the platform is not ROCm.
import importlib.util
import pytest
import torch
from vllm.platforms import current_platform
if not current_platform.is_rocm():
pytest.skip("This test can only run on ROCm.", allow_module_level=True)
# this import statement is needed to ensure the ops are registered
import vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe # noqa: F401
# need to import once to ensure the ops are registered
# Check if aiter package is installed
aiter_available = importlib.util.find_spec("aiter") is not None
if not aiter_available:
pytest.skip("These tests require AITER to run.", allow_module_level=True)
def test_rocm_aiter_biased_grouped_topk_custom_op_registration():
"""Test that the custom op is correctly registered."""
# Check if the op exists in torch.ops.vllm
assert hasattr(torch.ops.vllm, "rocm_aiter_biased_grouped_topk")
# Check if the op is callable
assert callable(torch.ops.vllm.rocm_aiter_biased_grouped_topk)
def test_rocm_aiter_grouped_topk_custom_op_registration():
"""Test that the custom op is correctly registered."""
# Check if the op exists in torch.ops.vllm
assert hasattr(torch.ops.vllm, "rocm_aiter_grouped_topk")
# Check if the op is callable
assert callable(torch.ops.vllm.rocm_aiter_grouped_topk)
def test_rocm_aiter_biased_grouped_topk_torch_compile_compatibility():
"""Test that the op can be used with torch.compile."""
# Create test tensors
token = 64
expert = 256
num_expert_group = 8
topk = 8
topk_group = 4
renormalize = True
scale_factor = 1.0
gating_output = torch.randn((token, expert), dtype=torch.bfloat16, device="cuda")
e_score_correction_bias = torch.randn(
(expert,), dtype=torch.bfloat16, device="cuda"
)
device = gating_output.device
topk_ids = torch.empty((token, topk), dtype=torch.int32, device=device)
topk_weights = torch.empty((token, topk), dtype=torch.float32, device=device)
# Define a function that uses the op
def biased_grouped_topk_fn(
gating_output, e_score_correction_bias, topk_weights, topk_ids
):
return torch.ops.vllm.rocm_aiter_biased_grouped_topk(
gating_output,
e_score_correction_bias,
topk_weights,
topk_ids,
num_expert_group,
topk_group,
renormalize,
scale_factor,
)
# Verify the op's fake implementation
torch.library.opcheck(
torch.ops.vllm.rocm_aiter_biased_grouped_topk,
(gating_output, e_score_correction_bias, topk_weights, topk_ids),
kwargs={
"num_expert_group": num_expert_group,
"topk_group": topk_group,
"need_renorm": renormalize,
"routed_scaling_factor": scale_factor,
},
test_utils=("test_faketensor"),
)
# Compile the function with appropriate settings
compiled_fn = torch.compile(
biased_grouped_topk_fn,
fullgraph=True,
backend="inductor",
mode="reduce-overhead",
dynamic=False,
)
topk_weights_original = torch.empty(
(token, topk), dtype=torch.float32, device=device
)
topk_ids_original = torch.empty((token, topk), dtype=torch.int32, device=device)
topk_weights_compiled = torch.empty(
(token, topk), dtype=torch.float32, device=device
)
topk_ids_compiled = torch.empty((token, topk), dtype=torch.int32, device=device)
# Run both compiled (V1 graph mode) and uncompiled versions (V1 eager mode)
biased_grouped_topk_fn(
gating_output, e_score_correction_bias, topk_weights_original, topk_ids_original
)
compiled_fn(
gating_output, e_score_correction_bias, topk_weights_compiled, topk_ids_compiled
)
# Sort the results for comparison since the order might not be deterministic
topk_ids_original, indices_original = torch.sort(topk_ids_original)
topk_weights_original = torch.gather(topk_weights_original, 1, indices_original)
topk_ids_compiled, indices_compiled = torch.sort(topk_ids_compiled)
topk_weights_compiled = torch.gather(topk_weights_compiled, 1, indices_compiled)
# Verify results match
assert torch.allclose(
topk_weights_original, topk_weights_compiled, rtol=1e-2, atol=1e-2
)
assert torch.allclose(topk_ids_original, topk_ids_compiled)
def test_rocm_aiter_grouped_topk_torch_compile_compatibility():
"""Test that the op can be used with torch.compile."""
# Create test tensors
token = 64
expert = 256
num_expert_group = 8
topk = 8
topk_group = 4
renormalize = True
scoring_func = "softmax"
scale_factor = 1.0
gating_output = torch.randn((token, expert), dtype=torch.bfloat16, device="cuda")
device = gating_output.device
topk_ids = torch.empty((token, topk), dtype=torch.int32, device=device)
topk_weights = torch.empty((token, topk), dtype=torch.float32, device=device)
# Define a function that uses the op
def grouped_topk_fn(gating_output, topk_weights, topk_ids, scoring_func):
return torch.ops.vllm.rocm_aiter_grouped_topk(
gating_output,
topk_weights,
topk_ids,
num_expert_group,
topk_group,
renormalize,
scoring_func,
scale_factor,
)
# Verify the op's fake implementation
torch.library.opcheck(
torch.ops.vllm.rocm_aiter_grouped_topk,
(gating_output, topk_weights, topk_ids),
kwargs={
"num_expert_group": num_expert_group,
"topk_group": topk_group,
"need_renorm": renormalize,
"scoring_func": scoring_func,
"routed_scaling_factor": scale_factor,
},
test_utils=("test_faketensor"),
)
# Compile the function with appropriate settings
compiled_fn = torch.compile(
grouped_topk_fn,
fullgraph=True,
backend="inductor",
mode="reduce-overhead",
dynamic=False,
)
topk_weights_original = torch.empty(
(token, topk), dtype=torch.float32, device=device
)
topk_ids_original = torch.empty((token, topk), dtype=torch.int32, device=device)
topk_weights_compiled = torch.empty(
(token, topk), dtype=torch.float32, device=device
)
topk_ids_compiled = torch.empty((token, topk), dtype=torch.int32, device=device)
# Run both compiled (V1 graph mode) and uncompiled versions (V1 eager mode)
grouped_topk_fn(
gating_output, topk_weights_original, topk_ids_original, scoring_func
)
compiled_fn(gating_output, topk_weights_compiled, topk_ids_compiled, scoring_func)
# Sort the results for comparison since the order might not be deterministic
topk_ids_original, indices_original = torch.sort(topk_ids_original)
topk_weights_original = torch.gather(topk_weights_original, 1, indices_original)
topk_ids_compiled, indices_compiled = torch.sort(topk_ids_compiled)
topk_weights_compiled = torch.gather(topk_weights_compiled, 1, indices_compiled)
# Verify results match
assert torch.allclose(
topk_weights_original, topk_weights_compiled, rtol=1e-2, atol=1e-2
)
assert torch.allclose(topk_ids_original, topk_ids_compiled)
def test_rocm_aiter_topk_gating_custom_op_registration():
assert callable(torch.ops.vllm.rocm_aiter_topk_gating)
def _sort_routing(weights: torch.Tensor, ids: torch.Tensor):
ids_sorted, order = torch.sort(ids, dim=-1)
weights_sorted = torch.gather(weights, 1, order)
return weights_sorted, ids_sorted
@pytest.mark.parametrize("num_tokens", [1, 4, 16, 64])
@pytest.mark.parametrize("num_experts", [256, 512])
@pytest.mark.parametrize("topk", [8, 10])
@pytest.mark.parametrize("renormalize", [True, False])
def test_rocm_aiter_topk_gating_matches_softmax_reference(
num_tokens: int,
num_experts: int,
topk: int,
renormalize: bool,
):
"""Numerical check vs softmax + topk, including Qwen3.8 (E=512, k=10)."""
torch.manual_seed(0)
gating_output = torch.randn(
(num_tokens, num_experts), dtype=torch.bfloat16, device="cuda"
)
scores = torch.softmax(gating_output.float(), dim=-1)
cutoff = torch.topk(scores, k=topk, dim=-1).values[:, -1:]
topk_weights = torch.empty((num_tokens, topk), dtype=torch.float32, device="cuda")
topk_ids = torch.empty((num_tokens, topk), dtype=torch.int32, device="cuda")
token_expert_indices = torch.empty(
(num_tokens, topk), dtype=torch.int32, device="cuda"
)
torch.ops.vllm.rocm_aiter_topk_gating(
topk_weights,
topk_ids,
token_expert_indices,
gating_output,
renormalize,
0,
"",
)
# BF16 logits can tie at the top-k boundary, so a different tied expert is
# valid. Check selection against the cutoff and weights against the scores
# of the experts the kernel actually selected.
assert ((topk_ids >= 0) & (topk_ids < num_experts)).all()
sorted_ids = torch.sort(topk_ids, dim=-1).values
assert (sorted_ids[:, 1:] != sorted_ids[:, :-1]).all()
selected_scores = torch.gather(scores, 1, topk_ids.long())
assert (selected_scores >= cutoff).all()
expected_weights = selected_scores
if renormalize:
expected_weights = expected_weights / expected_weights.sum(dim=-1, keepdim=True)
torch.testing.assert_close(topk_weights, expected_weights, atol=2e-2, rtol=2e-2)
def test_rocm_aiter_topk_gating_shared_expert_suffix():
"""Fused shared expert: softmax top-k on the prefix, sigmoid appended.
Matches the Qwen3.8 FSE launch: 512 routed experts, top-10, one shared
column, weights wide enough for the suffix, and the shared id prefilled.
"""
torch.manual_seed(0)
num_tokens, num_experts, topk, num_shared = 4, 512, 10, 1
routed = torch.randn((num_tokens, num_experts), dtype=torch.bfloat16, device="cuda")
shared = torch.randn((num_tokens, num_shared), dtype=torch.bfloat16, device="cuda")
shared[0, 0] = 40
gating_output = torch.cat([routed, shared], dim=-1).contiguous()
width = topk + num_shared
shared_ids = torch.arange(
num_experts, num_experts + num_shared, device="cuda", dtype=torch.int32
)
topk_ids = torch.full((num_tokens, width), -1, dtype=torch.int32, device="cuda")
topk_ids[:, topk:] = shared_ids
topk_weights = torch.empty((num_tokens, width), dtype=torch.float32, device="cuda")
token_expert_indices = torch.empty(
(num_tokens, topk), dtype=torch.int32, device="cuda"
)
torch.ops.vllm.rocm_aiter_topk_gating(
topk_weights,
topk_ids[:, :topk],
token_expert_indices,
gating_output,
True,
num_shared,
"sigmoid",
)
scores = torch.softmax(routed.float(), dim=-1)
cutoff = torch.topk(scores, k=topk, dim=-1).values[:, -1:]
routed_ids = topk_ids[:, :topk]
assert ((routed_ids >= 0) & (routed_ids < num_experts)).all()
sorted_ids = torch.sort(routed_ids, dim=-1).values
assert (sorted_ids[:, 1:] != sorted_ids[:, :-1]).all()
selected = torch.gather(scores, 1, routed_ids.long())
assert (selected >= cutoff).all()
expected = selected / selected.sum(dim=-1, keepdim=True)
torch.testing.assert_close(topk_weights[:, :topk], expected, atol=2e-2, rtol=2e-2)
torch.testing.assert_close(
topk_weights[:, topk:],
torch.sigmoid(shared.float()),
atol=2e-2,
rtol=2e-2,
)
assert torch.equal(topk_ids[:, topk:], shared_ids.expand_as(topk_ids[:, topk:]))
def test_rocm_aiter_topk_gating_torch_compile_compatibility():
token, expert, topk, renormalize = 4, 512, 10, True
gating_output = torch.randn((token, expert), dtype=torch.bfloat16, device="cuda")
device = gating_output.device
def gating_fn(gating_output, topk_weights, topk_ids, token_expert_indices):
return torch.ops.vllm.rocm_aiter_topk_gating(
topk_weights,
topk_ids,
token_expert_indices,
gating_output,
renormalize,
0,
"",
)
torch.library.opcheck(
torch.ops.vllm.rocm_aiter_topk_gating,
(
torch.empty((token, topk), dtype=torch.float32, device=device),
torch.empty((token, topk), dtype=torch.int32, device=device),
torch.empty((token, topk), dtype=torch.int32, device=device),
gating_output,
),
kwargs={
"renormalize": renormalize,
"num_shared_experts": 0,
"shared_expert_scoring_func": "",
},
test_utils=("test_faketensor"),
)
compiled_fn = torch.compile(
gating_fn,
fullgraph=True,
backend="inductor",
mode="reduce-overhead",
dynamic=False,
)
def alloc():
return (
torch.empty((token, topk), dtype=torch.float32, device=device),
torch.empty((token, topk), dtype=torch.int32, device=device),
torch.empty((token, topk), dtype=torch.int32, device=device),
)
w0, i0, t0 = alloc()
w1, i1, t1 = alloc()
gating_fn(gating_output, w0, i0, t0)
compiled_fn(gating_output, w1, i1, t1)
w0, i0 = _sort_routing(w0, i0)
w1, i1 = _sort_routing(w1, i1)
assert torch.allclose(w0, w1, rtol=1e-2, atol=1e-2)
assert torch.equal(i0, i1)
@@ -1,303 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Routed-expert capture on the monolithic (fused router + experts) MoE path.
A monolithic kernel that ``supports_routing_replay_capture`` must write, into
the leading rows of ``routing_replay_out``, exactly the experts it routed each
token to. The router logits plant an unambiguous top-k per token, so the test
checks the reported experts against that answer and not just their range.
"""
from __future__ import annotations
from collections.abc import Callable
import pytest
import torch
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
RoutingMethodType,
fp8_w8a8_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import (
TrtLlmBf16ExpertsMonolithic,
)
from vllm.model_executor.layers.fused_moe.experts.trtllm_fp8_moe import (
TrtLlmFp8ExpertsMonolithic,
)
from vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe import (
TrtLlmNvFp4ExpertsMonolithic,
)
from vllm.model_executor.layers.fused_moe.modular_kernel import (
FusedMoEExpertsMonolithic,
)
from vllm.platforms import current_platform
try:
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
except ImportError:
pytest.skip("flashinfer not available", allow_module_level=True)
if not has_flashinfer_trtllm_fused_moe() or not current_platform.is_cuda():
pytest.skip(
"Requires FlashInfer TRT-LLM fused MoE on CUDA",
allow_module_level=True,
)
if not current_platform.is_device_capability_family(100):
pytest.skip(
"TRT-LLM fused MoE kernels require SM100+",
allow_module_level=True,
)
NUM_EXPERTS = 32
NUM_GROUPS = 4 # DeepSeekV3 grouped routing
TOPK_GROUPS = 2
HIDDEN = 1024
INTERMEDIATE = 1024
DEVICE = torch.device("cuda:0")
# A kernel under test, and its call: (router_logits, routing_replay_out) -> None.
Kernel = tuple[FusedMoEExpertsMonolithic, Callable[[torch.Tensor, torch.Tensor], None]]
def _planted_routing(num_tokens: int, top_k: int) -> tuple[torch.Tensor, torch.Tensor]:
"""Router logits with one unambiguous answer under every routing method.
Each token gives +4 to ``top_k`` experts drawn from at most two expert
groups and -4 to the rest, so softmax top-k, sigmoid top-k and DeepSeekV3
grouped top-k all pick the same experts, with margins no rounding can flip.
Returns the logits and the planted experts of each token, sorted.
"""
group_size = NUM_EXPERTS // NUM_GROUPS
planted = []
for _ in range(num_tokens):
groups = torch.randperm(NUM_GROUPS)[:TOPK_GROUPS]
candidates = (groups[:, None] * group_size + torch.arange(group_size)).flatten()
planted.append(candidates[torch.randperm(len(candidates))[:top_k]])
experts = torch.stack(planted)
logits = torch.full((num_tokens, NUM_EXPERTS), -4.0).scatter_(1, experts, 4.0)
return logits.to(DEVICE), experts.sort(dim=-1).values.to(DEVICE)
def _routing_kwargs(routing_method: RoutingMethodType) -> dict:
if routing_method != RoutingMethodType.DeepSeekV3:
return {}
return dict(
num_expert_group=NUM_GROUPS,
topk_group=TOPK_GROUPS,
e_score_correction_bias=torch.zeros(
NUM_EXPERTS, device=DEVICE, dtype=torch.bfloat16
),
routed_scaling_factor=1.0,
)
def _moe_config(top_k: int, routing_method: RoutingMethodType) -> FusedMoEConfig:
return FusedMoEConfig(
num_experts=NUM_EXPERTS,
experts_per_token=top_k,
hidden_dim=HIDDEN,
intermediate_size=INTERMEDIATE,
num_local_experts=NUM_EXPERTS,
num_logical_experts=NUM_EXPERTS,
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
in_dtype=torch.bfloat16,
activation=MoEActivation.SILU,
device=DEVICE,
routing_method=routing_method,
max_num_tokens=16,
)
def _bf16_kernel(top_k: int, routing_method: RoutingMethodType) -> Kernel:
from flashinfer import shuffle_matrix_a
from flashinfer.fused_moe import convert_to_block_layout
def block_major_k(w: torch.Tensor) -> torch.Tensor:
"""(E, M, K) -> the BlockMajorK layout ``trtllm_bf16_moe`` expects."""
return torch.stack(
[
convert_to_block_layout(shuffle_matrix_a(e.view(torch.uint8), 64), 128)
for e in w
]
).view(torch.bfloat16)
experts = TrtLlmBf16ExpertsMonolithic(
moe_config=_moe_config(top_k, routing_method),
quant_config=FusedMoEQuantConfig.make(
quant_dtype=None,
per_act_token_quant=False,
per_out_ch_quant=False,
block_shape=None,
),
)
w13 = block_major_k(
torch.randn(NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN, device=DEVICE).bfloat16()
* 0.1
)
w2 = block_major_k(
torch.randn(NUM_EXPERTS, HIDDEN, INTERMEDIATE, device=DEVICE).bfloat16() * 0.1
)
def forward(router_logits, routing_replay_out):
hidden_states = torch.randn(len(router_logits), HIDDEN, device=DEVICE) * 0.1
experts.apply(
hidden_states=hidden_states.bfloat16(),
w1=w13,
w2=w2,
router_logits=router_logits,
activation=MoEActivation.SILU,
global_num_experts=NUM_EXPERTS,
expert_map=None,
a1q_scale=None,
apply_router_weight_on_input=False,
routing_replay_out=routing_replay_out,
**_routing_kwargs(routing_method),
)
return experts, forward
def _fp8_block_kernel(top_k: int, routing_method: RoutingMethodType) -> Kernel:
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
_shuffle_deepseek_fp8_moe_weights,
)
block = 128
w13, w2 = _shuffle_deepseek_fp8_moe_weights(
torch.randn(NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN, device=DEVICE).to(
torch.float8_e4m3fn
),
torch.randn(NUM_EXPERTS, HIDDEN, INTERMEDIATE, device=DEVICE).to(
torch.float8_e4m3fn
),
)
experts = TrtLlmFp8ExpertsMonolithic(
moe_config=_moe_config(top_k, routing_method),
quant_config=fp8_w8a8_moe_quant_config(
w1_scale=torch.ones(
NUM_EXPERTS, 2 * INTERMEDIATE // block, HIDDEN // block, device=DEVICE
),
w2_scale=torch.ones(
NUM_EXPERTS, HIDDEN // block, INTERMEDIATE // block, device=DEVICE
),
block_shape=[block, block],
per_act_token_quant=False,
),
)
def forward(router_logits, routing_replay_out):
num_tokens = len(router_logits)
hidden_states = torch.randn(num_tokens, HIDDEN, device=DEVICE) * 0.1
experts.apply(
hidden_states=hidden_states.to(torch.float8_e4m3fn),
w1=w13,
w2=w2,
router_logits=router_logits,
activation=MoEActivation.SILU,
global_num_experts=NUM_EXPERTS,
expert_map=None,
# Per-block activation scales, (num_tokens, hidden / 128).
a1q_scale=torch.ones(num_tokens, HIDDEN // block, device=DEVICE),
apply_router_weight_on_input=False,
routing_replay_out=routing_replay_out,
**_routing_kwargs(routing_method),
)
return experts, forward
def _nvfp4_kernel(top_k: int, routing_method: RoutingMethodType) -> Kernel:
from flashinfer import fp4_quantize
block = 16
global_scale = torch.tensor(1.0, device=DEVICE)
def quantize(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
"""Packed fp4 values and their fp8 per-16 block scales, (E, M, K/16)."""
q, scale = fp4_quantize(
w, global_scale, block, sf_use_ue8m0=False, is_sf_swizzled_layout=False
)
return q, scale.view(torch.float8_e4m3fn).reshape(*w.shape[:2], -1)
w13, w13_scale = quantize(
torch.randn(NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN, device=DEVICE).bfloat16()
)
w2, w2_scale = quantize(
torch.randn(NUM_EXPERTS, HIDDEN, INTERMEDIATE, device=DEVICE).bfloat16()
)
ones = torch.ones(NUM_EXPERTS, device=DEVICE)
experts = TrtLlmNvFp4ExpertsMonolithic(
moe_config=_moe_config(top_k, routing_method),
quant_config=FusedMoEQuantConfig.make(
quant_dtype="nvfp4",
per_act_token_quant=False,
per_out_ch_quant=False,
block_shape=None,
w1_scale=w13_scale,
w2_scale=w2_scale,
g1_alphas=ones,
g2_alphas=ones,
a1_gscale=global_scale,
a2_gscale=ones,
),
)
def forward(router_logits, routing_replay_out):
hidden_states = torch.randn(len(router_logits), HIDDEN, device=DEVICE) * 0.1
# apply() reinterprets the packed uint8 activation scales itself.
hidden_q, hidden_scale = fp4_quantize(
hidden_states.bfloat16(),
global_scale,
block,
sf_use_ue8m0=False,
is_sf_swizzled_layout=False,
)
experts.apply(
hidden_states=hidden_q,
w1=w13,
w2=w2,
router_logits=router_logits,
activation=MoEActivation.SILU,
global_num_experts=NUM_EXPERTS,
expert_map=None,
a1q_scale=hidden_scale,
apply_router_weight_on_input=False,
routing_replay_out=routing_replay_out,
**_routing_kwargs(routing_method),
)
return experts, forward
KERNELS = {"bf16": _bf16_kernel, "fp8_block": _fp8_block_kernel, "nvfp4": _nvfp4_kernel}
@pytest.mark.parametrize("kernel", list(KERNELS))
@pytest.mark.parametrize(
"routing_method", [RoutingMethodType.Renormalize, RoutingMethodType.DeepSeekV3]
)
@pytest.mark.parametrize("top_k", [2, 4])
@pytest.mark.parametrize("num_tokens", [2, 7, 16])
def test_routing_replay_reports_the_routed_experts(
kernel: str, routing_method: RoutingMethodType, top_k: int, num_tokens: int
) -> None:
torch.manual_seed(0)
experts, forward = KERNELS[kernel](top_k, routing_method)
assert experts.supports_routing_replay_capture()
router_logits, planted = _planted_routing(num_tokens, top_k)
# One spare row: the kernel must write the batch's rows and nothing else.
routing_replay_out = torch.full(
(num_tokens + 1, top_k), -1, dtype=torch.int16, device=DEVICE
)
forward(router_logits, routing_replay_out)
reported = routing_replay_out[:num_tokens].long().sort(dim=-1).values
assert torch.equal(reported, planted), (reported, planted)
assert (routing_replay_out[num_tokens:] == -1).all()
-209
View File
@@ -1,209 +0,0 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Test script for the token-to-expert routing simulator.
This script demonstrates how to use the routing simulator to test
different routing strategies and analyze their performance, including
integration tests with FusedMoEFactory layer.
"""
import tempfile
import pytest
import torch
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.distributed import (
init_distributed_environment,
initialize_model_parallel,
)
from vllm.model_executor.layers.fused_moe.router.routing_simulator_router import (
DistributionBasedRouting,
RoutingSimulator,
)
@pytest.fixture
def device():
"""Fixture to provide the appropriate device for testing."""
return torch.device("cuda" if torch.cuda.is_available() else "cpu")
@pytest.mark.parametrize("num_tokens", [1, 16, 256])
@pytest.mark.parametrize("hidden_size", [64, 1024])
@pytest.mark.parametrize("num_experts", [16, 128])
@pytest.mark.parametrize("top_k", [1, 4])
def test_basic_functionality(
num_tokens: int,
hidden_size: int,
num_experts: int,
top_k: int,
device,
):
"""Test basic functionality of the routing simulator."""
# Test each routing strategy
strategies = RoutingSimulator.get_available_strategies()
hidden_states = torch.randn(num_tokens, hidden_size, device=device)
router_logits = torch.randn(num_tokens, num_experts, device=device)
for strategy in strategies:
# Simulate routing
topk_weights, topk_ids = RoutingSimulator.simulate_routing(
hidden_states=hidden_states,
router_logits=router_logits,
strategy_name=strategy,
top_k=top_k,
)
# Check output shapes
assert topk_weights.shape == (
num_tokens,
top_k,
), f"Wrong weights shape for {strategy}"
assert topk_ids.shape == (
num_tokens,
top_k,
), f"Wrong ids shape for {strategy}"
# Check that expert IDs are valid
assert topk_ids.min() >= 0, f"Invalid expert ID (negative) for {strategy}"
assert topk_ids.max() < num_experts, (
f"Invalid expert ID (too large) for {strategy}"
)
def test_routing_strategy_integration(monkeypatch, device):
"""Test that the routing strategy environment variable works with
FusedMoEFactory."""
pytest.importorskip("vllm.model_executor.layers.fused_moe.layer")
import vllm.envs as envs
from vllm.model_executor.layers.fused_moe.layer import FusedMoEFactory
# Test parameters
num_tokens = 32
hidden_size = 16
num_experts = 4
top_k = 2
# Create test data
hidden_states = torch.randn(num_tokens, hidden_size, device=device)
router_logits = torch.randn(num_tokens, num_experts, device=device)
# Test different routing strategies
strategies = RoutingSimulator.get_available_strategies()
vllm_config = VllmConfig()
with set_current_vllm_config(vllm_config):
temp_file = tempfile.mkstemp()[1]
init_distributed_environment(
world_size=1,
rank=0,
local_rank=0,
distributed_init_method=f"file://{temp_file}",
)
initialize_model_parallel(
tensor_model_parallel_size=1,
pipeline_model_parallel_size=1,
)
for strategy in strategies:
fused_moe = FusedMoEFactory(
num_experts=num_experts,
top_k=top_k,
hidden_size=hidden_size,
intermediate_size=0,
use_grouped_topk=False,
renormalize=True,
prefix=strategy,
)
# Set environment variable
env_name = "VLLM_MOE_ROUTING_SIMULATION_STRATEGY"
monkeypatch.setenv(env_name, strategy)
# Temporarily override the envs lookup so the router factory
# reads the monkeypatched value instead of the module-load-time
# default. Use monkeypatch.setitem so the original lambda is
# restored automatically at teardown.
monkeypatch.setitem(
envs.environment_variables,
env_name,
lambda s=strategy: s,
)
# Test the select_experts method
topk_weights, topk_ids = fused_moe.router.select_experts(
hidden_states=hidden_states,
router_logits=router_logits,
)
# Verify output shapes
assert topk_weights.shape == (num_tokens, top_k), (
f"Wrong weights shape for {strategy}"
)
assert topk_ids.shape == (num_tokens, top_k), (
f"Wrong ids shape for {strategy}"
)
# Verify expert IDs are valid
assert topk_ids.min() >= 0, f"Invalid expert ID (negative) for {strategy}"
assert topk_ids.max() < num_experts, (
f"Invalid expert ID (too large) for {strategy}"
)
def test_distribution_based_routing_with_custom_strategy():
"""Test registering and using DistributionBasedRouting with custom
parameters."""
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Register custom distribution-based strategy
custom_strategy = DistributionBasedRouting(distribution="normal", mean=2.0, std=0.5)
RoutingSimulator.register_strategy("custom_normal", custom_strategy)
# Test data
num_tokens = 60
hidden_size = 48
num_experts = 6
top_k = 3
hidden_states = torch.randn(num_tokens, hidden_size, device=device)
router_logits = torch.randn(num_tokens, num_experts, device=device)
# Use the custom strategy
topk_weights, topk_ids = RoutingSimulator.simulate_routing(
hidden_states=hidden_states,
router_logits=router_logits,
strategy_name="custom_normal",
top_k=top_k,
)
# Check output shapes
assert topk_weights.shape == (num_tokens, top_k)
assert topk_ids.shape == (num_tokens, top_k)
# Check that expert IDs are valid
assert topk_ids.min() >= 0
assert topk_ids.max() < num_experts
def test_instance_compatibility():
"""Test that static methods work correctly."""
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Test static method directly
hidden_states = torch.randn(10, 8, device=device)
router_logits = torch.randn(10, 4, device=device)
topk_weights, topk_ids = RoutingSimulator.simulate_routing(
hidden_states=hidden_states,
router_logits=router_logits,
strategy_name="uniform_random",
top_k=2,
)
assert topk_weights.shape == (10, 2)
assert topk_ids.shape == (10, 2)
@@ -1,228 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for FusedMoEFactory with routed_input_transform.
Verifies that applying routed_input_transform inside FusedMoEFactory
produces the same results as applying the transform manually outside.
"""
import pytest
import torch
import torch.nn as nn
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.platforms import current_platform
from vllm.utils.torch_utils import is_torch_equal_or_newer, set_random_seed
class SimpleLinear(nn.Module):
"""A simple linear transform mimicking latent projection in latent MoE."""
def __init__(self, in_features: int, out_features: int, dtype: torch.dtype):
super().__init__()
self.weight = nn.Parameter(
torch.randn(out_features, in_features, device="cuda", dtype=dtype) / 10
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
return nn.functional.linear(x, self.weight)
class SimpleSharedExperts(nn.Module):
"""A simple 2-layer MLP mimicking shared experts."""
def __init__(self, hidden_size: int, intermediate_size: int, dtype: torch.dtype):
super().__init__()
self.up = nn.Linear(
hidden_size, intermediate_size * 2, bias=False, device="cuda", dtype=dtype
)
self.down = nn.Linear(
intermediate_size, hidden_size, bias=False, device="cuda", dtype=dtype
)
with torch.no_grad():
self.up.weight.div_(10)
self.down.weight.div_(10)
def forward(self, x: torch.Tensor) -> torch.Tensor:
gate_up = self.up(x)
gate, up = gate_up.chunk(2, dim=-1)
return self.down(nn.functional.silu(gate) * up)
def _assert_close(
actual: torch.Tensor,
expected: torch.Tensor,
atol: float,
rtol: float,
label: str,
) -> None:
"""assert_close that prints diff diagnostics on both success and failure."""
actual_nans = int(actual.isnan().sum().item())
expected_nans = int(expected.isnan().sum().item())
actual_zeros = int((actual == 0).sum().item())
expected_zeros = int((expected == 0).sum().item())
n_total = actual.numel()
diff = (actual - expected).abs()
max_diff = diff.max().item()
mean_diff = diff.mean().item()
n_exceed = int((diff > atol).sum().item())
pct_exceed = n_exceed / n_total * 100
print(
f"[{label}] "
f"shape={list(actual.shape)}, "
f"max_diff={max_diff:.6e}, "
f"mean_diff={mean_diff:.6e}, "
f"exceed_atol({atol})={n_exceed}/{n_total} ({pct_exceed:.2f}%), "
f"actual=[{actual.min().item():.4f}, {actual.max().item():.4f}], "
f"expected=[{expected.min().item():.4f}, {expected.max().item():.4f}], "
f"nan(actual/expected)={actual_nans}/{expected_nans}, "
f"zeros(actual/expected)={actual_zeros}/{expected_zeros}"
)
assert actual_nans == 0, (
f"{label}: actual has {actual_nans}/{n_total} NaN values "
f"(expected has {expected_nans}). "
f"This indicates a kernel bug, not a precision issue."
)
assert expected_nans == 0, (
f"{label}: expected has {expected_nans}/{n_total} NaN values. "
f"This indicates a kernel bug, not a precision issue."
)
torch.testing.assert_close(
actual,
expected,
atol=atol,
rtol=rtol,
msg=(
f"{label}: max_diff={max_diff:.6e}, mean_diff={mean_diff:.6e}, "
f"exceed_atol({atol})={n_exceed}/{n_total} ({pct_exceed:.2f}%)"
),
)
@pytest.fixture(autouse=True)
def setup_cuda():
if not torch.cuda.is_available():
pytest.skip("CUDA not available")
torch.set_default_device("cuda")
@pytest.mark.parametrize("num_tokens", [1, 32])
@pytest.mark.parametrize("hidden_size,latent_size", [(256, 128), (128, 64)])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize(
"use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
)
@pytest.mark.skipif(
is_torch_equal_or_newer("2.10.0"),
reason="Test fails with PyTorch 2.10.0 see: https://github.com/vllm-project/vllm/issues/33995",
)
def test_routed_input_transform_inside_vs_outside(
num_tokens: int,
hidden_size: int,
latent_size: int,
dtype: torch.dtype,
use_rocm_aiter: bool,
dist_init,
workspace_init,
monkeypatch,
):
"""Compare FusedMoEFactory with transform inside vs manually applying outside.
Method A (inside): FusedMoEFactory with routed_input_transform
Method B (outside): Manually transform, then FusedMoEFactory without transform
"""
if current_platform.is_rocm():
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1" if use_rocm_aiter else "0")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "1" if use_rocm_aiter else "0")
from vllm._aiter_ops import rocm_aiter_ops
rocm_aiter_ops.refresh_env_variables()
set_random_seed(42)
num_experts = 8
top_k = 2
intermediate_size = hidden_size * 2
vllm_config = VllmConfig()
vllm_config.compilation_config.static_forward_context = dict()
shared_experts = SimpleSharedExperts(hidden_size, intermediate_size, dtype)
routed_transform = SimpleLinear(hidden_size, latent_size, dtype)
with set_current_vllm_config(vllm_config):
# Method A: FusedMoEFactory WITH routed_input_transform
moe_with_transform = FusedMoEFactory(
shared_experts=shared_experts,
routed_input_transform=routed_transform,
num_experts=num_experts,
top_k=top_k,
hidden_size=latent_size,
intermediate_size=intermediate_size,
renormalize=True,
params_dtype=dtype,
tp_size=1,
dp_size=1,
pcp_size=1,
prefix="moe_with_transform",
)
# Method B: FusedMoEFactory WITHOUT routed_input_transform
# Note: shared_experts=None because when transform is done outside,
moe_without_transform = FusedMoEFactory(
shared_experts=None,
routed_input_transform=None,
num_experts=num_experts,
top_k=top_k,
hidden_size=latent_size,
intermediate_size=intermediate_size,
renormalize=True,
params_dtype=dtype,
tp_size=1,
dp_size=1,
pcp_size=1,
prefix="moe_without_transform",
)
# Weights are created via torch.empty (uninitialized).
# Initialize with seeded random values for reproducibility.
with torch.no_grad():
moe_with_transform.w13_weight.normal_()
moe_with_transform.w13_weight.div_(10)
moe_with_transform.w2_weight.normal_()
moe_with_transform.w2_weight.div_(10)
moe_without_transform.w13_weight.copy_(moe_with_transform.w13_weight)
moe_without_transform.w2_weight.copy_(moe_with_transform.w2_weight)
moe_with_transform.quant_method.process_weights_after_loading(
moe_with_transform
)
moe_without_transform.quant_method.process_weights_after_loading(
moe_without_transform
)
hidden_states = torch.randn(num_tokens, hidden_size, device="cuda", dtype=dtype)
router_logits = torch.randn(num_tokens, num_experts, device="cuda", dtype=dtype)
with set_forward_context(None, vllm_config, num_tokens=num_tokens):
# Method A: combined output (shared + routed)
combined_A = moe_with_transform(hidden_states, router_logits)
# Method B: manually transform, get routed output, add shared
transformed_hidden = routed_transform(hidden_states)
routed_out_B = moe_without_transform(transformed_hidden, router_logits)
shared_out_B = shared_experts(hidden_states)
combined_B = shared_out_B + routed_out_B
torch.testing.assert_close(
combined_A,
combined_B,
atol=1e-3,
rtol=1e-3,
msg="Combined output should match: transform inside vs outside",
)
@@ -1,318 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import random
import pytest
import torch
from vllm.model_executor.layers.fused_moe.experts.batched_deep_gemm_moe import (
persistent_masked_m_silu_mul_quant,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_fp8_min_max,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import (
DeepGemmQuantScaleFMT,
has_deep_gemm,
transform_sf_into_required_layout,
)
from vllm.utils.math_utils import cdiv, round_up
from vllm.utils.torch_utils import set_random_seed
fp8_dtype = current_platform.fp8_dtype()
DEVICE = current_platform.device_type
pytestmark = pytest.mark.skipif(
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
reason="persistent_masked_m_silu_mul_quant requires a CUDA-alike or XPU device.",
)
CASES = [
(1, 1, 128, fp8_dtype),
(1, 4, 128 * 1, fp8_dtype),
(2, 4, 128 * 2, fp8_dtype),
(1, 4, 128 * 3, fp8_dtype),
(8, 16, 128 * 4, fp8_dtype),
(8, 16, 128 * 5, fp8_dtype),
(8, 16, 128 * 6, fp8_dtype),
(8, 16, 128 * 7, fp8_dtype),
(8, 16, 128 * 8, fp8_dtype),
(8, 16, 128 * 9, fp8_dtype),
(8, 64, 7168, fp8_dtype),
(8, 128, 128 * 33, fp8_dtype),
(1, 4, 128 * 10, fp8_dtype),
(8, 128, 7168, fp8_dtype),
(8, 512, 7168, fp8_dtype),
(8, 1024, 7168, fp8_dtype),
(17, 31, 768, fp8_dtype),
(32, 64, 256, fp8_dtype),
(256, 8, 7168, fp8_dtype),
(256, 32, 7168, fp8_dtype),
(256, 64, 7168, fp8_dtype),
# Only add a few fnuz tests to help with long CI times.
(8, 512, 7168, torch.float8_e4m3fnuz),
(8, 1024, 7168, torch.float8_e4m3fnuz),
]
def as_uint8(x) -> torch.Tensor:
return (
torch.empty(x.shape, dtype=x.dtype, device=x.device).copy_(x).view(torch.uint8)
)
def silu(x: torch.Tensor) -> torch.Tensor:
x_f32 = x.to(torch.float32)
act_f32 = x_f32 / (1.0 + torch.exp(-x_f32))
if current_platform.is_cuda():
# C++ kernel returns bf16
return act_f32.to(torch.bfloat16)
# Triton fallback stays in f32
return act_f32
def do_quant(x: torch.Tensor, group_size: int, ceil_ue8m0: bool):
fp8_min_val, fp8_max_val = get_fp8_min_max()
eps_bf16 = torch.tensor([1e-10], device=x.device, dtype=torch.bfloat16)
one_bf16 = torch.tensor([1.0], device=x.device, dtype=torch.bfloat16)
fp8_max_bf16 = torch.tensor([fp8_max_val], device=x.device, dtype=torch.bfloat16)
fp8_min_bf16 = torch.tensor([fp8_min_val], device=x.device, dtype=torch.bfloat16)
fp8_max_inv = one_bf16 / fp8_max_bf16
assert fp8_max_inv.dtype == torch.bfloat16
assert x.size(-1) % group_size == 0
num_groups = x.numel() // group_size
x_og_shape = x.shape
if current_platform.is_cuda():
# C++ kernel computes entirely in bf16
x = x.to(torch.bfloat16)
x = x.view((-1, group_size))
amax = x.abs().amax(dim=1).clamp(min=eps_bf16)
assert amax.dtype == torch.bfloat16
s = amax * fp8_max_inv
if ceil_ue8m0:
s = torch.exp2(
torch.ceil(torch.log2(s).to(torch.bfloat16)).to(torch.bfloat16)
).to(torch.bfloat16)
inv_s = one_bf16 / s
inv_s = inv_s.view((num_groups, 1))
xq = torch.clamp(
x * inv_s, min=fp8_min_bf16.item(), max=fp8_max_bf16.item()
).to(fp8_dtype)
else:
# Triton fallback computes in f32. Use multiply-by-reciprocal
# to match Triton's constexpr evaluation of 1.0/fp8_max.
fp8_min_f, fp8_max_f = get_fp8_min_max()
x = x.to(torch.float32).view((-1, group_size))
amax = x.abs().amax(dim=1).clamp(min=1e-10)
s = amax * (1.0 / fp8_max_f)
if ceil_ue8m0:
s = torch.exp2(torch.ceil(torch.log2(s)))
inv_s = (1.0 / s).view((num_groups, 1))
xq = torch.clamp(x * inv_s, min=fp8_min_f, max=fp8_max_f).to(fp8_dtype)
xq = xq.view(x_og_shape)
xs = s.view((-1, xq.size(-1) // group_size))
return xq, xs
def silu_mul_quant(
gate: torch.Tensor, up: torch.Tensor, group_size: int, ceil_ue8m0: bool
) -> tuple[torch.Tensor, torch.Tensor]:
assert gate.size(-1) % group_size == 0
assert up.size(-1) % group_size == 0
assert gate.dtype == torch.bfloat16
assert up.dtype == torch.bfloat16
act = silu(gate)
# act & mul
a_m = act * up
q, s = do_quant(a_m, group_size, ceil_ue8m0)
return q, s
def pack_scales(x: torch.Tensor, tokens_per_expert: torch.Tensor) -> torch.Tensor:
"""Pack float32 scales into a int32 tensor."""
assert x.dtype == torch.float32
E, T, G = x.size()
# Add i32_padding here so we can view it as a i32 tensor later on.
i32_padding = round_up(G, 4) - G
ref_s_i8 = torch.empty((E, T, G + i32_padding), dtype=torch.uint8, device=DEVICE)
for e in range(E):
nt = tokens_per_expert[e].item()
ref_s_i8[e, :nt, :G] = x[e, :nt].view(torch.int32) >> 23
ref_s_i32 = ref_s_i8.view(torch.int32)
return ref_s_i32
def ref_with_scale_fmt(
E: int,
T: int,
H: int,
group_size: int,
tokens_per_expert: torch.Tensor,
gate: torch.Tensor,
up: torch.Tensor,
scale_fmt: DeepGemmQuantScaleFMT,
) -> tuple[torch.Tensor, torch.Tensor]:
"""The precision types of the operations triggered by this function
match closely with the kernel implementation so we compare more
accurately.
"""
scale_dtype = (
torch.int32 if scale_fmt == DeepGemmQuantScaleFMT.UE8M0 else torch.float32
)
ceil_ue8m0 = scale_fmt in [
DeepGemmQuantScaleFMT.UE8M0,
DeepGemmQuantScaleFMT.FLOAT32_CEIL_UE8M0,
]
ref_q = torch.empty((E, T, H), dtype=fp8_dtype, device=DEVICE)
ref_s_f32 = torch.empty(
(E, T, cdiv(H, group_size)), dtype=torch.float32, device=DEVICE
)
for e in range(E):
nt = tokens_per_expert[e].item()
if nt == 0:
continue
ref_q[e, :nt], ref_s_f32[e, :nt] = silu_mul_quant(
gate[e, :nt], up[e, :nt], group_size, ceil_ue8m0=ceil_ue8m0
)
if scale_dtype == torch.float32:
return ref_q, ref_s_f32
assert scale_dtype == torch.int32
return ref_q, pack_scales(ref_s_f32, tokens_per_expert)
def token_random(E, T, H2, tokens_per_expert):
"""Initialize each token in a random range so we test a range of
scale values.
"""
y = torch.empty((E, T, H2), dtype=torch.bfloat16, device=DEVICE)
for e in range(E):
for t in range(tokens_per_expert[e].item()):
exp = random.choice(range(1, 20))
y[e, t].uniform_(-(2**exp), 2**exp)
return y
@pytest.mark.parametrize("E,T,H,fp8_type", CASES)
@torch.inference_mode()
def test_silu_mul_fp8_quant_deep_gemm(E: int, T: int, H: int, fp8_type: torch.dtype):
group_size = 128
set_random_seed(42)
tokens_per_expert = torch.randint(
low=0,
high=T,
size=(E,),
dtype=torch.int32,
device=DEVICE,
)
# Input tensor of shape (E, T, 2*H)
y = token_random(E, T, 2 * H, tokens_per_expert)
gate = y[..., :H].to(torch.bfloat16)
up = y[..., H:].to(torch.bfloat16)
scale_fmts = [
DeepGemmQuantScaleFMT.FLOAT32,
DeepGemmQuantScaleFMT.FLOAT32_CEIL_UE8M0,
DeepGemmQuantScaleFMT.UE8M0,
]
# Run the SiLU V2 kernel
for scale_fmt in scale_fmts:
y_q, y_s = persistent_masked_m_silu_mul_quant(
y,
tokens_per_expert,
group_size=group_size,
quant_scale_fmt=scale_fmt,
)
ref_y_q, ref_y_s = ref_with_scale_fmt(
E, T, H, group_size, tokens_per_expert, gate, up, scale_fmt=scale_fmt
)
# deepgemm scales transform
dg_scales = None
if (
has_deep_gemm()
and current_platform.has_device_capability(100)
and scale_fmt == DeepGemmQuantScaleFMT.UE8M0
):
_q, _s = ref_with_scale_fmt(
E,
T,
H,
group_size,
tokens_per_expert,
gate,
up,
scale_fmt=DeepGemmQuantScaleFMT.FLOAT32_CEIL_UE8M0,
)
dg_scales = transform_sf_into_required_layout(
sf=_s,
mn=_q.size(1),
k=_q.size(2),
recipe=(1, 128, 128),
num_groups=_q.size(0),
is_sfa=True,
)
expected_scale_dtype = (
torch.int32 if scale_fmt == DeepGemmQuantScaleFMT.UE8M0 else torch.float32
)
assert y_s.dtype == expected_scale_dtype
assert ref_y_s.dtype == expected_scale_dtype
for e in range(E):
nt = tokens_per_expert[e].item()
if current_platform.is_rocm() or current_platform.is_xpu():
# ROCm and XPU both run the Triton fallback kernel, whose f32
# math intrinsics (tl.exp) may differ from PyTorch's torch.exp
# by 1 ULP. At FP8 quantization boundaries this can flip one
# representable value. Allow 1 FP8 quantum of tolerance.
torch.testing.assert_close(
y_q[e, :nt].to(torch.float32),
ref_y_q[e, :nt].to(torch.float32),
atol=32.0,
rtol=0.2,
)
else:
torch.testing.assert_close(
y_q[e, :nt].to(torch.float32),
ref_y_q[e, :nt].to(torch.float32),
)
if scale_fmt == DeepGemmQuantScaleFMT.UE8M0:
G = H // group_size
y_s_sliced = as_uint8(y_s[e])
ref_s_sliced = as_uint8(ref_y_s[e])
torch.testing.assert_close(y_s_sliced[:nt, :G], ref_s_sliced[:nt, :G])
if dg_scales is not None:
dg_sliced = as_uint8(dg_scales[e])
torch.testing.assert_close(y_s_sliced[:nt, :G], dg_sliced[:nt, :G])
else:
torch.testing.assert_close(
y_s[e, :nt],
ref_y_s[e, :nt],
)
@@ -1,230 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
_per_token_group_quant_fp8_colmajor,
silu_mul_per_token_group_quant_fp8_colmajor,
silu_mul_quant_fp8_packed_triton,
)
from vllm.platforms import current_platform
from vllm.triton_utils import triton
from vllm.utils.deep_gemm import is_deep_gemm_e8m0_used
from vllm.utils.torch_utils import set_random_seed
DEVICE = current_platform.device_type
FLOAT8_DTYPE = torch.float8_e4m3fn
GROUP_SIZE = 128
def reference_quant(x: torch.Tensor, use_ue8m0: bool):
"""Reference triton quant kernel from,
vllm.model_executor.layers.quantization.utils.fp8_utils
"""
x_q = torch.empty_like(x, device=x.device, dtype=FLOAT8_DTYPE)
# Allocate the scale tensor in column-major format.
shape = (x.shape[-1] // GROUP_SIZE,) + x.shape[:-1]
x_s = torch.empty(shape, device=x.device, dtype=torch.float32).permute(-1, -2)
M = x.numel() // GROUP_SIZE
N = GROUP_SIZE
BLOCK = triton.next_power_of_2(N)
# heuristics for number of warps
num_warps = min(max(BLOCK // 256, 1), 8)
num_stages = 1
finfo = torch.finfo(FLOAT8_DTYPE)
fp8_min = finfo.min
fp8_max = finfo.max
_per_token_group_quant_fp8_colmajor[(M,)](
x,
x_q,
x_s,
GROUP_SIZE,
x.shape[1],
x.stride(0),
x_s.stride(1),
eps=1e-10,
fp8_min=fp8_min,
fp8_max=fp8_max,
use_ue8m0=use_ue8m0,
BLOCK=BLOCK,
num_warps=num_warps,
num_stages=num_stages,
)
return x_q, x_s
def reference(x: torch.Tensor, use_ue8m0: bool) -> tuple[torch.Tensor, torch.Tensor]:
T, N = x.size()
ref_act_out = torch.empty((T, N // 2), dtype=torch.bfloat16, device=DEVICE)
torch.ops._C.silu_and_mul(ref_act_out, x)
return reference_quant(ref_act_out, use_ue8m0)
def reference_with_clamp(
x: torch.Tensor, use_ue8m0: bool, clamp_limit: float
) -> tuple[torch.Tensor, torch.Tensor]:
"""Pre-clamp inputs (gate from above, up symmetric) at the input dtype to
match the C++ compute() template, then run the standard silu_and_mul +
quant reference."""
N_2 = x.size(1) // 2
dtype = x.dtype
gate = x[..., :N_2].to(torch.float32).clamp(max=clamp_limit).to(dtype)
up = (
x[..., N_2:]
.to(torch.float32)
.clamp(min=-clamp_limit, max=clamp_limit)
.to(dtype)
)
return reference(torch.cat([gate, up], dim=-1), use_ue8m0)
@pytest.mark.parametrize("T", [128, 256, 512])
@pytest.mark.parametrize("N", [128 * 2, 256 * 2, 768 * 2, 2048 * 2, 7168 * 2])
@pytest.mark.skipif(
current_platform.is_rocm(),
reason="ROCm does not support DeepGemm.",
)
def test_silu_mul_fp8_quant_deep_gemm(T: int, N: int):
set_random_seed(42)
input = torch.rand((T, N), dtype=torch.bfloat16, device=DEVICE)
use_ue8m0 = is_deep_gemm_e8m0_used()
# Test
output, output_scales = silu_mul_per_token_group_quant_fp8_colmajor(
input, use_ue8m0=use_ue8m0
)
# Reference
ref_output, ref_output_scales = reference(input, use_ue8m0)
torch.testing.assert_close(output.to(torch.float32), ref_output.to(torch.float32))
torch.testing.assert_close(output_scales, ref_output_scales)
@pytest.mark.parametrize(
"counts", [[0, 1, 127, 128, 129, 0, 257, 13], [128] * 8, [0] * 8]
)
@pytest.mark.parametrize("group_size", [32, 128])
@pytest.mark.parametrize("scale_format", ["packed", "float32", "ceil_ue8m0"])
@pytest.mark.parametrize("hidden_size", [256, 2048])
@pytest.mark.parametrize("clamp_limit", [None, 10.0])
@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="CUDA/ROCm graph test")
def test_silu_quant_skips_expert_padding_on_graph_replay(
counts: list[int],
group_size: int,
scale_format: str,
hidden_size: int,
clamp_limit: float | None,
):
"""Live rows match dense quantization; gaps stay untouched as counts change."""
alignment = 128
capacity = sum(triton.cdiv(n, alignment) * alignment for n in counts)
# Exercise unused capacity as well as the gaps between experts.
capacity = max(capacity, alignment)
if counts != [128] * 8:
capacity += alignment
x = torch.randn((capacity, 2 * hidden_size), device=DEVICE, dtype=torch.bfloat16)
x *= 16
clean_x = x.clone()
ends = torch.empty(len(counts), device=DEVICE, dtype=torch.int32)
output_dtype = (
FLOAT8_DTYPE if scale_format == "packed" else current_platform.fp8_dtype()
)
output = torch.empty((capacity, hidden_size), device=DEVICE, dtype=output_dtype)
def set_counts(values):
positions = []
mask = torch.zeros(capacity, device=DEVICE, dtype=torch.bool)
start = 0
for count in values:
mask[start : start + count] = True
positions.append(start + count)
start += triton.cdiv(count, alignment) * alignment
ends.copy_(torch.tensor(positions, device=DEVICE, dtype=torch.int32))
return mask
def quantize(x, output=None, **kwargs):
if scale_format == "packed":
return silu_mul_quant_fp8_packed_triton(
x,
group_size=group_size,
output_q=output,
clamp_limit=clamp_limit,
**kwargs,
)
return silu_mul_per_token_group_quant_fp8_colmajor(
x,
group_size=group_size,
output=output,
clamp_limit=clamp_limit,
use_ue8m0=scale_format == "ceil_ue8m0",
**kwargs,
)
def invoke():
return quantize(
x,
output=output,
expert_ends=ends,
expert_alignment=alignment,
)
set_counts(counts)
invoke()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
actual_q, actual_s = invoke()
for values in (counts, counts[::-1]):
live = set_counts(values)
x.copy_(clean_x)
x[~live] = float("nan")
output.view(torch.uint8).fill_(91)
graph.replay()
reference_q, reference_s = quantize(x)
assert actual_q.dtype == reference_q.dtype
torch.testing.assert_close(
actual_q.view(torch.uint8)[live],
reference_q.view(torch.uint8)[live],
rtol=0,
atol=0,
)
torch.testing.assert_close(actual_s[live], reference_s[live], rtol=0, atol=0)
assert torch.all(actual_q.view(torch.uint8)[~live] == 91)
@pytest.mark.parametrize("T", [128, 256, 512])
@pytest.mark.parametrize("N", [128 * 2, 256 * 2, 768 * 2, 2048 * 2, 7168 * 2])
@pytest.mark.parametrize("clamp_limit", [7.0, 10.0])
@pytest.mark.skipif(
current_platform.is_rocm(),
reason="ROCm does not support DeepGemm.",
)
def test_silu_mul_fp8_quant_deep_gemm_clamp(T: int, N: int, clamp_limit: float):
set_random_seed(42)
# Use a wide distribution so values routinely exceed both clamp limits and
# the clamp branch is actually exercised (uniform [0, 1) inputs would never
# trigger it).
input = torch.randn((T, N), dtype=torch.bfloat16, device=DEVICE) * 8.0
use_ue8m0 = is_deep_gemm_e8m0_used()
# Test
output, output_scales = silu_mul_per_token_group_quant_fp8_colmajor(
input, use_ue8m0=use_ue8m0, clamp_limit=clamp_limit
)
# Reference
ref_output, ref_output_scales = reference_with_clamp(input, use_ue8m0, clamp_limit)
torch.testing.assert_close(output.to(torch.float32), ref_output.to(torch.float32))
torch.testing.assert_close(output_scales, ref_output_scales)
@@ -1,224 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness tests for the persistent cp.async-pipelined SITU + block-FP8
quant kernel (``situ_and_mul_quant``).
Mirrors ``test_silu_mul_fp8_quant_deep_gemm.py`` in structure: build a reference
activation + block-FP8 quant, drive random per-row scale magnitudes, exercise
the masked contiguous layout, and compare per valid row.
The kernel computes SITU in fp32 and converts to fp8 with hardware rounding,
which torch cannot reproduce bit-exactly. It is therefore validated against the
*ideal* FP8 quantizer of the exact fp64 activation: the difference must stay
within the unavoidable FP8 block-quant error (within 1 fp8 code for ~all
elements, never off by more than 2).
"""
import pytest
import torch
from vllm.model_executor.layers.fused_moe.activation import situ_and_mul_quant
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
fp8_dtype = current_platform.fp8_dtype()
DEVICE = current_platform.device_type
FP8_MAX = torch.finfo(fp8_dtype).max
# The kernel uses fp8 e4m3 conversion, so it is CUDA-only and needs SM89+
# (Ada/Hopper). There is no Triton fallback.
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda() or not current_platform.has_device_capability(89),
reason="situ_and_mul_quant persistent kernel requires CUDA SM89+ (fp8 e4m3).",
)
# Kimi-K3 baked SITU params. Only (d=3072, beta=4.0, linear_beta=25.0,
# group_size=128, fp16/bf16) routes to the persistent cp.async-pipelined kernel;
# anything else falls back to the scalar group kernel.
SITU_BETA = 4.0
SITU_LINEAR_BETA = 25.0
SITU_D = 3072
GROUP_SIZE = 128
# (num_tokens, num_valid_tokens). None means all rows are valid. 2048/4096 rows
# span multiple grid-stride waves of the persistent grid (GRID_DIM = 132*8).
CASES = [
(1, None),
(17, None),
(128, None),
(2048, None),
(4096, None),
(2048, 1500), # DeepEP v2 contiguous padding
(2048, 1),
(4096, 4095),
(512, 0), # whole tensor is padding
]
def token_random(num_tokens: int, twod: int, dtype: torch.dtype) -> torch.Tensor:
"""Per-row random exponent so groups span many scale magnitudes."""
base = torch.randn(num_tokens, twod, dtype=torch.float32, device=DEVICE)
exps = torch.randint(1, 13, (num_tokens, 1), device=DEVICE).float()
return (base * (2.0**exps)).to(dtype)
def ref_situ_block_fp8_quant(
inp: torch.Tensor, beta: float, linear_beta: float, group_size: int
) -> tuple[torch.Tensor, torch.Tensor]:
"""Ideal FP8 block-quant of the exact fp64 SITU activation.
The kernel rounds the activation to ``scalar_t`` before quantizing (it stores
``(float)(scalar_t)situ(...)``), so the reference does the same before taking
the per-group absmax.
"""
d = inp.shape[-1] // 2
gate, up = inp.double().chunk(2, dim=-1)
gate_out = beta * torch.tanh(gate / beta) * torch.sigmoid(gate)
up_out = linear_beta * torch.tanh(up / linear_beta) if linear_beta > 0 else up
act = (gate_out * up_out).to(inp.dtype).float()
m = act.shape[0]
ng = d // group_size
a = act.view(m, ng, group_size)
scale = a.abs().amax(dim=2).clamp_min(1e-30) / FP8_MAX
q = (a / scale.unsqueeze(2)).clamp(-FP8_MAX, FP8_MAX).to(fp8_dtype)
return q.view(m, d), scale
def fp8_code_dist(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
"""Signed-magnitude fp8 code distance (valid for e4m3fn and e4m3fnuz)."""
def ordered(x: torch.Tensor) -> torch.Tensor:
x = x.view(torch.uint8).to(torch.int32)
return torch.where((x >> 7) == 1, -(x & 0x7F), x & 0x7F)
return (ordered(a) - ordered(b)).abs()
@pytest.mark.parametrize("num_tokens,valid", CASES)
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half])
@torch.inference_mode()
def test_situ_and_mul_quant_pipelined(
num_tokens: int, valid: int | None, dtype: torch.dtype
) -> None:
set_random_seed(42)
d = SITU_D
inp = token_random(num_tokens, 2 * d, dtype)
# Sentinels: padding rows must keep their bytes (out) and get scale 1.0.
out = torch.full((num_tokens, d), 17.0, device=DEVICE).to(fp8_dtype)
scale = torch.full(
(num_tokens, d // GROUP_SIZE), -1.0, dtype=torch.float32, device=DEVICE
)
out_sentinel = out.clone()
num_valid_tokens = (
None if valid is None else torch.tensor(valid, dtype=torch.int32, device=DEVICE)
)
situ_and_mul_quant(
out,
scale,
inp,
beta=SITU_BETA,
linear_beta=SITU_LINEAR_BETA,
group_size=GROUP_SIZE,
num_valid_tokens=num_valid_tokens,
)
# Padding contract: skipped rows keep their bytes; their scale becomes 1.0.
if valid is not None and valid < num_tokens:
assert torch.equal(scale[valid:], torch.ones_like(scale[valid:]))
assert torch.equal(
out[valid:].view(torch.uint8), out_sentinel[valid:].view(torch.uint8)
)
rows = num_tokens if valid is None else valid
if rows == 0:
return
ref_q, ref_s = ref_situ_block_fp8_quant(
inp[:rows], SITU_BETA, SITU_LINEAR_BETA, GROUP_SIZE
)
assert torch.isfinite(scale[:rows]).all()
assert not torch.isnan(out[:rows].float()).any()
# Scales: the kernel's fp32 SITU shifts the per-group absmax only
# marginally vs the fp64 reference.
torch.testing.assert_close(scale[:rows], ref_s, rtol=3e-2, atol=1e-5)
# FP8 codes: must land within 1 code of the ideal quantizer for ~all
# elements, never off by more than 2.
dist = fp8_code_dist(out[:rows], ref_q)
assert dist.max().item() <= 2
assert (dist <= 1).float().mean().item() >= 0.99
@pytest.mark.parametrize("tokens,topk", [(300, 8), (1, 4), (256, 6)])
@torch.inference_mode()
def test_situ_and_mul_quant_topk_row_bound(tokens: int, topk: int) -> None:
"""The kernel expands num_valid_tokens by topk to bound rows."""
set_random_seed(7)
d = SITU_D
num_tokens = tokens * topk + 37 # padding tail past the valid rows
inp = token_random(num_tokens, 2 * d, torch.bfloat16)
out = torch.full((num_tokens, d), 17.0, device=DEVICE).to(fp8_dtype)
scale = torch.full(
(num_tokens, d // GROUP_SIZE), -1.0, dtype=torch.float32, device=DEVICE
)
out_sentinel = out.clone()
num_valid_tokens = torch.tensor(tokens, dtype=torch.int32, device=DEVICE)
situ_and_mul_quant(
out,
scale,
inp,
beta=SITU_BETA,
linear_beta=SITU_LINEAR_BETA,
group_size=GROUP_SIZE,
num_valid_tokens=num_valid_tokens,
topk=topk,
)
rows = tokens * topk
assert torch.equal(scale[rows:], torch.ones_like(scale[rows:]))
assert torch.equal(
out[rows:].view(torch.uint8), out_sentinel[rows:].view(torch.uint8)
)
ref_q, ref_s = ref_situ_block_fp8_quant(
inp[:rows], SITU_BETA, SITU_LINEAR_BETA, GROUP_SIZE
)
torch.testing.assert_close(scale[:rows], ref_s, rtol=3e-2, atol=1e-5)
assert fp8_code_dist(out[:rows], ref_q).max().item() <= 2
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half])
@torch.inference_mode()
def test_situ_and_mul_quant_pipelined_deterministic(dtype: torch.dtype) -> None:
"""Two runs on the same input must be byte-identical."""
set_random_seed(0)
d = SITU_D
num_tokens = 2048
inp = token_random(num_tokens, 2 * d, dtype)
def run() -> tuple[torch.Tensor, torch.Tensor]:
out = torch.empty(num_tokens, d, dtype=fp8_dtype, device=DEVICE)
scale = torch.empty(
num_tokens, d // GROUP_SIZE, dtype=torch.float32, device=DEVICE
)
situ_and_mul_quant(
out,
scale,
inp,
beta=SITU_BETA,
linear_beta=SITU_LINEAR_BETA,
group_size=GROUP_SIZE,
)
return out, scale
out1, s1 = run()
out2, s2 = run()
assert torch.equal(out1.view(torch.uint8), out2.view(torch.uint8))
assert torch.equal(s1, s2)
@@ -1,740 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.nn.functional as F
import vllm._custom_ops as ops
from vllm.model_executor.layers.fused_moe.config import (
RoutingMethodType,
get_routing_method_type,
)
from vllm.model_executor.layers.fused_moe.router.dsv4_topk import dsv4_topk
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
fused_topk_bias,
)
from vllm.platforms import current_platform
def _torch_topk_softplus_sqrt(
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
routed_scaling_factor: float,
e_score_correction_bias: torch.Tensor | None = None,
input_ids: torch.Tensor | None = None,
hash_indices_table: torch.Tensor | None = None,
bias_vl: torch.Tensor | None = None,
image_sentinel_lo: int = 0,
):
scores = F.softplus(gating_output.float()).sqrt()
original_scores = scores
if hash_indices_table is not None:
assert input_ids is not None
if bias_vl is None or image_sentinel_lo == 0:
topk_ids = hash_indices_table[input_ids.long()]
else:
image_mask = (input_ids.long() >= image_sentinel_lo) & (
input_ids.long() < image_sentinel_lo + 5
)
topk_ids = hash_indices_table[input_ids.long()]
vl_ids = (scores + bias_vl).topk(topk, dim=-1).indices
topk_ids = torch.where(
image_mask.unsqueeze(-1), vl_ids.to(topk_ids.dtype), topk_ids
)
else:
if bias_vl is not None and image_sentinel_lo > 0:
# Image tokens carry five consecutive in-vocab sentinel ids
# starting at image_sentinel_lo and select experts with bias_vl
# instead of the regular bias.
assert input_ids is not None
assert e_score_correction_bias is not None
image_mask = (
(input_ids.long() >= image_sentinel_lo)
& (input_ids.long() < image_sentinel_lo + 5)
).unsqueeze(-1)
row_bias = torch.where(
image_mask,
bias_vl.unsqueeze(0),
e_score_correction_bias.unsqueeze(0),
)
scores_for_choice = scores + row_bias
elif e_score_correction_bias is not None:
scores_for_choice = scores + e_score_correction_bias.unsqueeze(0)
else:
scores_for_choice = scores
# Match the fused kernel's deterministic tie-break: lower expert ids
# win when scores are equal. torch.topk does not guarantee which tied
# index it selects at the k-th boundary.
topk_ids = torch.argsort(
scores_for_choice, dim=-1, descending=True, stable=True
)[:, :topk]
topk_weights = original_scores.gather(1, topk_ids.long())
if renormalize:
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
if routed_scaling_factor != 1.0:
topk_weights = topk_weights * routed_scaling_factor
return topk_weights.to(torch.float32), topk_ids.to(torch.int32)
def _dsv4_moe_config(monkeypatch, default_vllm_config, has_hash_routing, backend):
from dataclasses import replace
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.layer import FusedMoEFactory
from vllm.platforms import rocm
# Exercise both native MX and monolithic AITER selection on any ROCm device.
monkeypatch.setattr(rocm, "on_gfx950", lambda: True)
monkeypatch.setattr(rocm, "on_gfx1250", lambda: False)
monkeypatch.setattr(rocm_aiter_ops, "is_enabled", lambda: True)
monkeypatch.setattr(rocm_aiter_ops, "is_fused_moe_enabled", lambda: True)
default_vllm_config.kernel_config.moe_backend = "triton"
layer = FusedMoEFactory(
num_experts=8,
top_k=2,
hidden_size=256,
intermediate_size=256,
params_dtype=torch.bfloat16,
scoring_func="sqrtsoftplus",
e_score_correction_bias=torch.zeros(8),
hash_indices_table=(
torch.tensor([[0, 1], [2, 3]], dtype=torch.int32)
if has_hash_routing
else None
),
prefix="hash_routing_selection",
)
return layer, replace(layer.routed_experts.moe_config, moe_backend=backend)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm backend selection")
@pytest.mark.parametrize("has_hash_routing", [False, True])
@pytest.mark.parametrize(
"backend,expected_experts",
[
("aiter_triton_mxfp4_bf16", "AiterW4A16ExpertsMonolithic"),
("aiter", "AiterMxfp4Experts"),
("triton_unfused", "UnfusedOAITritonExperts"),
],
)
def test_hash_routing_backend_selection(
dist_init,
default_vllm_config,
monkeypatch,
has_hash_routing,
backend,
expected_experts,
):
"""Hash tables reach backend selection and hash-capable kernels."""
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
select_deepseek_v4_mxfp4_moe_backend,
)
layer, config = _dsv4_moe_config(
monkeypatch, default_vllm_config, has_hash_routing, backend
)
_, experts_cls = select_deepseek_v4_mxfp4_moe_backend(config)
assert experts_cls.__name__ == expected_experts
# Monolithic kernels route internally, so they read the table from the layer.
assert (layer.routed_experts.hash_indices_table is not None) == has_hash_routing
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm backend selection")
def test_hash_routing_rejects_monolithic_without_support(
dist_init, default_vllm_config, monkeypatch
):
"""A monolithic kernel that does not implement hash routing is rejected."""
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a16_moe import (
AiterW4A16ExpertsMonolithic,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
select_deepseek_v4_mxfp4_moe_backend,
)
monkeypatch.setattr(
AiterW4A16ExpertsMonolithic,
"_supports_hash_routing",
staticmethod(lambda: False),
)
_, config = _dsv4_moe_config(
monkeypatch, default_vllm_config, True, "aiter_triton_mxfp4_bf16"
)
with pytest.raises(ValueError, match="hash routing"):
select_deepseek_v4_mxfp4_moe_backend(config)
def test_torch_topk_softplus_sqrt_breaks_ties_by_expert_id():
gating_output = torch.tensor([[2.0, 1.0, 1.0, 0.0]])
_, topk_ids = _torch_topk_softplus_sqrt(
gating_output,
topk=2,
renormalize=False,
routed_scaling_factor=1.0,
)
assert topk_ids.tolist() == [[0, 1]]
def test_sqrtsoftplus_bias_uses_deepseek_v4_routing_method():
assert (
get_routing_method_type(
scoring_func="sqrtsoftplus",
top_k=8,
renormalize=True,
num_expert_group=None,
has_e_score_bias=True,
)
== RoutingMethodType.DeepseekV4
)
assert (
get_routing_method_type(
scoring_func="sqrtsoftplus",
top_k=8,
renormalize=False,
num_expert_group=None,
has_e_score_bias=True,
)
== RoutingMethodType.Unspecified
)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="This test is skipped on non-CUDA platform.",
)
@pytest.mark.parametrize("num_tokens", [1, 33, 128])
@pytest.mark.parametrize("hidden_size", [1024, 2048])
@pytest.mark.parametrize("num_experts", [128, 256, 384, 512])
@pytest.mark.parametrize("topk", [6, 8, 16])
@pytest.mark.parametrize("renormalize", [True, False])
@pytest.mark.parametrize("routed_scaling_factor", [1.0, 1.5])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half, torch.float32])
def test_fused_topk_softplus_sqrt(
num_tokens: int,
hidden_size: int,
num_experts: int,
topk: int,
renormalize: bool,
routed_scaling_factor: float,
dtype: torch.dtype,
):
torch.manual_seed(0)
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
e_score_correction_bias = torch.randn(
(num_experts,), dtype=torch.float32, device="cuda"
)
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
gating_output=gating_output,
topk=topk,
renormalize=renormalize,
routed_scaling_factor=routed_scaling_factor,
e_score_correction_bias=e_score_correction_bias,
)
topk_weights, topk_ids = fused_topk_bias(
hidden_states=hidden_states,
gating_output=gating_output,
scoring_func="sqrtsoftplus",
e_score_correction_bias=e_score_correction_bias,
topk=topk,
renormalize=renormalize,
routed_scaling_factor=routed_scaling_factor,
)
# Different kernels may return the topk experts in different orders when
# scores tie; sort by expert id before comparing.
sorted_ref_ids, idx_ref = topk_ids_ref.sort(dim=-1)
sorted_ids, idx_ops = topk_ids.sort(dim=-1)
torch.testing.assert_close(sorted_ref_ids, sorted_ids, atol=0, rtol=0)
sorted_w_ref = topk_weights_ref.gather(1, idx_ref)
sorted_w = topk_weights.gather(1, idx_ops)
torch.testing.assert_close(sorted_w_ref, sorted_w, atol=2e-2, rtol=1e-2)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="This test is skipped on non-CUDA platform.",
)
@pytest.mark.parametrize("num_tokens", [1, 33, 128])
@pytest.mark.parametrize("hidden_size", [1024, 2048])
@pytest.mark.parametrize("num_experts", [256, 384, 512])
@pytest.mark.parametrize("topk", [6, 8, 16])
@pytest.mark.parametrize("renormalize", [True, False])
@pytest.mark.parametrize("routed_scaling_factor", [1.0, 2.5])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half, torch.float32])
def test_fused_topk_softplus_sqrt_hash(
num_tokens: int,
hidden_size: int,
num_experts: int,
topk: int,
renormalize: bool,
routed_scaling_factor: float,
dtype: torch.dtype,
):
torch.manual_seed(0)
vocab_size = 1024
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
# Per-token fixed expert selection: for each vocab id pick `topk` distinct
# experts.
hash_indices_table = torch.stack(
[torch.randperm(num_experts)[:topk] for _ in range(vocab_size)]
).to(device="cuda", dtype=torch.long)
input_ids = torch.randint(
0, vocab_size, (num_tokens,), dtype=torch.long, device="cuda"
)
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
gating_output=gating_output,
topk=topk,
renormalize=renormalize,
routed_scaling_factor=routed_scaling_factor,
input_ids=input_ids,
hash_indices_table=hash_indices_table,
)
topk_weights, topk_ids = fused_topk_bias(
hidden_states=hidden_states,
gating_output=gating_output,
scoring_func="sqrtsoftplus",
e_score_correction_bias=None,
topk=topk,
renormalize=renormalize,
input_tokens=input_ids,
hash_indices_table=hash_indices_table,
routed_scaling_factor=routed_scaling_factor,
)
sorted_ref_ids, idx_ref = topk_ids_ref.sort(dim=-1)
sorted_ids, idx_ops = topk_ids.sort(dim=-1)
torch.testing.assert_close(sorted_ref_ids, sorted_ids, atol=0, rtol=0)
sorted_w_ref = topk_weights_ref.gather(1, idx_ref)
sorted_w = topk_weights.gather(1, idx_ops)
torch.testing.assert_close(sorted_w_ref, sorted_w, atol=2e-2, rtol=1e-2)
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="The DeepSeek V4 fast path is CUDA-only.",
)
@pytest.mark.parametrize(
("num_tokens", "num_experts", "indices_type"),
[
(0, 256, torch.int32),
(17, 256, torch.int32),
(17, 384, torch.int64),
],
)
@pytest.mark.parametrize("use_padding", [False, True])
def test_dsv4_fast_topk(
num_tokens: int,
num_experts: int,
indices_type: torch.dtype,
use_padding: bool,
monkeypatch: pytest.MonkeyPatch,
):
torch.manual_seed(0)
gating_output = torch.randn(
(num_tokens, num_experts), dtype=torch.float32, device="cuda"
)
correction_bias = torch.randn(num_experts, dtype=torch.float32, device="cuda")
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
gating_output=gating_output,
topk=6,
renormalize=True,
routed_scaling_factor=1.5,
e_score_correction_bias=correction_bias,
)
is_padding = torch.arange(num_tokens, device="cuda") % 2 == 1
if use_padding:
monkeypatch.setattr(
"vllm.model_executor.layers.fused_moe.router."
"fused_topk_bias_router._get_padding_mask",
lambda _: is_padding,
)
gating_output[is_padding] = float("nan")
topk_weights_ref[is_padding] = 0
topk_ids_ref[is_padding] = -1
topk_weights, topk_ids = fused_topk_bias(
gating_output,
gating_output,
"sqrtsoftplus",
correction_bias,
6,
True,
indices_type=indices_type,
routed_scaling_factor=1.5,
)
assert topk_ids.dtype == indices_type
torch.testing.assert_close(topk_ids_ref.to(indices_type), topk_ids, atol=0, rtol=0)
torch.testing.assert_close(
topk_weights_ref,
topk_weights,
atol=2e-5,
rtol=2e-5,
)
if use_padding and num_tokens:
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
weights, ids = dsv4_topk(
gating_output, correction_bias, indices_type, 1.5, is_padding=is_padding
)
is_padding.fill_(True)
graph.replay()
assert (ids == -1).all()
assert (weights == 0).all()
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="This test is skipped on non-CUDA platform.",
)
@pytest.mark.parametrize("use_hash", [False, True])
@pytest.mark.parametrize("use_bias", [False, True])
@pytest.mark.parametrize("use_padding_mask", [False, True])
@pytest.mark.parametrize("pad_with_nan", [False, True])
@pytest.mark.parametrize("num_experts", [128, 256, 384])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half, torch.float32])
def test_fused_topk_softplus_sqrt_padding(
use_hash: bool,
use_bias: bool,
use_padding_mask: bool,
pad_with_nan: bool,
num_experts: int,
dtype: torch.dtype,
):
"""Verify explicit padding and NaN-padded rows do not affect real rows."""
torch.manual_seed(0)
num_tokens = 8
topk = 6
indices_dtype = torch.int32
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
# Exercise top-k cutoff ties without relying on random low-precision collisions.
gating_output[0] = 0
padding_rows = torch.zeros(num_tokens, dtype=torch.bool, device="cuda")
padding_rows[1::2] = True
if pad_with_nan:
gating_output[padding_rows] = float("nan")
is_padding = padding_rows if use_padding_mask else None
# A negative correction bias makes explicit pad rows look selectable unless
# the kernel uses the is_padding guard.
e_score_correction_bias = None
if use_bias:
e_score_correction_bias = (
-torch.rand((num_experts,), dtype=torch.float32, device="cuda") - 1.0
)
input_ids = None
hash_indices_table = None
if use_hash:
vocab_size = 64
hash_indices_table = torch.stack(
[torch.randperm(num_experts)[:topk] for _ in range(vocab_size)]
).to(device="cuda", dtype=indices_dtype)
input_ids = torch.randint(
0, vocab_size, (num_tokens,), dtype=indices_dtype, device="cuda"
)
topk_weights = torch.empty(num_tokens, topk, dtype=torch.float32, device="cuda")
topk_ids = torch.empty(num_tokens, topk, dtype=indices_dtype, device="cuda")
token_expert_indices = torch.empty(
num_tokens, topk, dtype=torch.int32, device="cuda"
)
ops.topk_hash_softplus_sqrt(
topk_weights,
topk_ids,
token_expert_indices,
gating_output,
renormalize=True,
routed_scaling_factor=1.0,
e_score_correction_bias=e_score_correction_bias,
input_tokens=input_ids,
hash_indices_table=hash_indices_table,
is_padding=is_padding,
)
if use_padding_mask:
pad_ids = topk_ids[padding_rows]
pad_weights = topk_weights[padding_rows]
assert torch.equal(pad_ids, torch.full_like(pad_ids, -1)), (
f"Explicit pad rows should contain only -1 ids, got {pad_ids.tolist()}"
)
assert (pad_weights == 0).all(), (
"Explicit pad rows should have all-zero weights, "
f"got {pad_weights.tolist()}"
)
if pad_with_nan:
nan_pad_weights = topk_weights[padding_rows]
assert torch.isfinite(nan_pad_weights).all(), (
f"NaN-padded rows have non-finite weights, got {nan_pad_weights.tolist()}"
)
assert (nan_pad_weights == 0).all(), (
"NaN-padded rows should have all-zero weights, "
f"got {nan_pad_weights.tolist()}"
)
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
gating_output=gating_output,
topk=topk,
renormalize=True,
routed_scaling_factor=1.0,
e_score_correction_bias=e_score_correction_bias,
input_ids=input_ids,
hash_indices_table=hash_indices_table,
)
rows_to_compare = torch.ones(num_tokens, dtype=torch.bool, device="cuda")
if use_padding_mask or pad_with_nan:
rows_to_compare = ~padding_rows
sorted_ref_ids, idx_ref = topk_ids_ref[rows_to_compare].sort(dim=-1)
sorted_ids, idx_ops = topk_ids[rows_to_compare].sort(dim=-1)
torch.testing.assert_close(
sorted_ref_ids, sorted_ids.to(sorted_ref_ids.dtype), atol=0, rtol=0
)
sorted_w_ref = topk_weights_ref[rows_to_compare].gather(1, idx_ref)
sorted_w = topk_weights[rows_to_compare].gather(1, idx_ops)
torch.testing.assert_close(sorted_w_ref, sorted_w, atol=2e-2, rtol=1e-2)
def _make_mixed_input_ids(
num_tokens: int, image_sentinel_lo: int, dtype: torch.dtype = torch.long
) -> torch.Tensor:
"""Mix of text, image-sentinel, and above-sentinel ids.
Every third token is an image sentinel id (cycling through the five slots
[image_sentinel_lo, image_sentinel_lo + 5)); every sixth token carries an
id just above the sentinel block, simulating the named special tokens
that follow it — those must route as regular text tokens.
"""
input_ids = torch.randint(
0, image_sentinel_lo, (num_tokens,), dtype=torch.long, device="cuda"
)
pos = torch.arange(num_tokens, device="cuda")
image_rows = pos % 3 == 0
input_ids[image_rows] = image_sentinel_lo + (pos[image_rows] // 3) % 5
above_rows = pos % 6 == 3
input_ids[above_rows] = image_sentinel_lo + 5
return input_ids.to(dtype)
def _assert_topk_matches(
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
topk_weights_ref: torch.Tensor,
topk_ids_ref: torch.Tensor,
) -> None:
# Different kernels may return the topk experts in different orders when
# scores tie; sort by expert id before comparing.
sorted_ref_ids, idx_ref = topk_ids_ref.sort(dim=-1)
sorted_ids, idx_ops = topk_ids.sort(dim=-1)
torch.testing.assert_close(
sorted_ref_ids, sorted_ids.to(sorted_ref_ids.dtype), atol=0, rtol=0
)
sorted_w_ref = topk_weights_ref.gather(1, idx_ref)
sorted_w = topk_weights.gather(1, idx_ops.to(torch.long))
torch.testing.assert_close(sorted_w_ref, sorted_w, atol=2e-2, rtol=1e-2)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="This test is skipped on non-CUDA platform.",
)
@pytest.mark.parametrize(
("num_experts", "topk", "renormalize", "dtype"),
[
(256, 6, True, torch.float32), # Triton dsv4 fast path
(256, 8, True, torch.bfloat16), # CUDA topk kernel
(384, 6, False, torch.half),
(512, 16, True, torch.float32),
],
)
def test_fused_topk_softplus_sqrt_bias_vl(
num_experts: int,
topk: int,
renormalize: bool,
dtype: torch.dtype,
):
"""Image sentinel tokens must select experts with bias_vl.
Rows with ids just above the sentinel block (image_sentinel_lo + 5)
simulate the named special tokens that follow it and must route through
the regular bias path; the reference comparison covers them.
"""
torch.manual_seed(0)
num_tokens = 64
hidden_size = 1024
vocab_size = 128
image_sentinel_lo = vocab_size - 8
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
e_score_correction_bias = torch.randn(
(num_experts,), dtype=torch.float32, device="cuda"
)
bias_vl = torch.randn((num_experts,), dtype=torch.float32, device="cuda")
input_ids = _make_mixed_input_ids(num_tokens, image_sentinel_lo)
assert (input_ids == image_sentinel_lo + 5).any()
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
gating_output=gating_output,
topk=topk,
renormalize=renormalize,
routed_scaling_factor=1.5,
e_score_correction_bias=e_score_correction_bias,
input_ids=input_ids,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
)
topk_weights, topk_ids = fused_topk_bias(
hidden_states=hidden_states,
gating_output=gating_output,
scoring_func="sqrtsoftplus",
e_score_correction_bias=e_score_correction_bias,
topk=topk,
renormalize=renormalize,
routed_scaling_factor=1.5,
input_tokens=input_ids,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
)
_assert_topk_matches(topk_weights, topk_ids, topk_weights_ref, topk_ids_ref)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="This test is skipped on non-CUDA platform.",
)
@pytest.mark.parametrize(
("num_experts", "topk", "renormalize", "dtype", "indices_dtype"),
[
(256, 6, True, torch.float32, torch.long), # specialized hash kernel
(384, 6, True, torch.float32, torch.int32), # specialized hash kernel
(384, 8, False, torch.bfloat16, torch.long), # generic hash kernel
(512, 6, True, torch.float32, torch.long), # generic hash kernel
],
)
def test_fused_topk_softplus_sqrt_hash_bias_vl(
num_experts: int,
topk: int,
renormalize: bool,
dtype: torch.dtype,
indices_dtype: torch.dtype,
):
"""Hash MoE: text rows use tid2eid, image rows use topk(score + bias_vl).
Sentinel ids are in-vocab; rows with ids above the sentinel block
(image_sentinel_lo + 5) must still come straight from the hash table.
"""
torch.manual_seed(0)
num_tokens = 64
hidden_size = 1024
vocab_size = 64
image_sentinel_lo = vocab_size - 8
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
hash_indices_table = torch.stack(
[torch.randperm(num_experts)[:topk] for _ in range(vocab_size)]
).to(device="cuda", dtype=indices_dtype)
bias_vl = torch.randn((num_experts,), dtype=torch.float32, device="cuda")
input_ids = _make_mixed_input_ids(num_tokens, image_sentinel_lo)
image_rows = (input_ids >= image_sentinel_lo) & (input_ids < image_sentinel_lo + 5)
assert image_rows.any() and (~image_rows).any()
assert (input_ids == image_sentinel_lo + 5).any()
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
gating_output=gating_output,
topk=topk,
renormalize=renormalize,
routed_scaling_factor=2.5,
input_ids=input_ids,
hash_indices_table=hash_indices_table,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
)
topk_weights, topk_ids = fused_topk_bias(
hidden_states=hidden_states,
gating_output=gating_output,
scoring_func="sqrtsoftplus",
e_score_correction_bias=None,
topk=topk,
renormalize=renormalize,
input_tokens=input_ids,
hash_indices_table=hash_indices_table,
routed_scaling_factor=2.5,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
)
_assert_topk_matches(topk_weights, topk_ids, topk_weights_ref, topk_ids_ref)
# Text rows must come straight from the hash table, image rows from
# topk(score + bias_vl).
text_ids = hash_indices_table[input_ids[~image_rows].long()]
torch.testing.assert_close(
topk_ids[~image_rows].to(text_ids.dtype), text_ids, atol=0, rtol=0
)
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="The DeepSeek V4 fast path is CUDA-only.",
)
def test_dsv4_fast_topk_bias_vl():
torch.manual_seed(0)
num_tokens = 33
num_experts = 256
vocab_size = 128
image_sentinel_lo = vocab_size - 8
gating_output = torch.randn(
(num_tokens, num_experts), dtype=torch.float32, device="cuda"
)
correction_bias = torch.randn(num_experts, dtype=torch.float32, device="cuda")
bias_vl = torch.randn(num_experts, dtype=torch.float32, device="cuda")
input_ids = _make_mixed_input_ids(num_tokens, image_sentinel_lo)
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
gating_output=gating_output,
topk=6,
renormalize=True,
routed_scaling_factor=1.5,
e_score_correction_bias=correction_bias,
input_ids=input_ids,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
)
topk_weights, topk_ids = dsv4_topk(
gating_output,
correction_bias,
torch.int64,
1.5,
input_ids=input_ids,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
)
assert topk_ids.dtype == torch.int64
torch.testing.assert_close(topk_ids_ref.to(torch.int64), topk_ids, atol=0, rtol=0)
torch.testing.assert_close(topk_weights_ref, topk_weights, atol=2e-5, rtol=2e-5)
@@ -1,261 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for MoE with non-gated activations (*_no_mul).
These tests verify that MoE layers work correctly with activations like
silu_no_mul, gelu_no_mul, relu2_no_mul where the activation output dimension
equals N (not N // 2 like gated activations).
"""
import pytest
import torch
from tests.kernels.moe.utils import make_dummy_moe_config
from vllm.model_executor.layers.fused_moe.activation import (
ApplyMoEActivationConfig,
MoEActivation,
apply_moe_activation,
apply_moe_activation_supported,
)
from vllm.model_executor.layers.fused_moe.config import (
FUSED_MOE_UNQUANTIZED_CONFIG,
)
from vllm.model_executor.layers.fused_moe.experts.triton_moe import TritonExperts
from vllm.platforms import current_platform
DEVICE = current_platform.device_type
pytestmark = [
pytest.mark.skipif(
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
reason="Triton MoE kernels require a CUDA-alike or XPU device.",
),
pytest.mark.skipif(
current_platform.is_cuda_alike()
and not current_platform.has_device_capability(80),
reason="Triton MoE kernels require compute capability >= 8.0 on CUDA/ROCm.",
),
]
# Test parameters
M_SIZES = [1, 16, 64]
N_SIZES = [128, 256]
K_SIZES = [64, 128]
TOPK_VALUES = [1, 2]
NUM_EXPERTS = 8
NO_MUL_ACTIVATIONS = [
MoEActivation.SILU_NO_MUL,
MoEActivation.GELU_NO_MUL,
MoEActivation.RELU2_NO_MUL,
]
def test_apply_moe_activation_supported_contract():
supported = {
activation
for activation in MoEActivation
if apply_moe_activation_supported(activation)
}
assert supported == set(MoEActivation) - {MoEActivation.RELU2}
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
@pytest.mark.parametrize(
"activation",
[
activation
for activation in MoEActivation
if apply_moe_activation_supported(activation)
],
)
@torch.inference_mode()
def test_supported_apply_moe_activation_executes(activation: MoEActivation):
output_width = 64
input_width = output_width * 2 if activation.is_gated else output_width
input = torch.randn(2, input_width, device="cuda", dtype=torch.bfloat16)
output = torch.empty(2, output_width, device="cuda", dtype=torch.bfloat16)
activation_config = ApplyMoEActivationConfig()
if activation == MoEActivation.SITU:
activation_config = ApplyMoEActivationConfig(activation_situ_beta=1.0)
elif activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
activation_config = ApplyMoEActivationConfig(clamp_limit=7.0)
apply_moe_activation(activation, output, input, activation_config=activation_config)
assert torch.isfinite(output).all()
@pytest.mark.parametrize("activation", list(MoEActivation))
def test_triton_activation_metadata_tracks_shared_apply(activation: MoEActivation):
assert TritonExperts._supports_activation(
activation
) == apply_moe_activation_supported(activation)
def make_test_tensors(
m: int,
n: int,
k: int,
num_experts: int,
topk: int,
dtype: torch.dtype = torch.bfloat16,
device: str = DEVICE,
):
"""Create test tensors for MoE with non-gated activation.
For non-gated activations (*_no_mul):
- w1: (E, N, K) - projects from K to N
- w2: (E, K, N) - projects from N back to K (note: N, not N//2)
"""
hidden_states = torch.randn(m, k, dtype=dtype, device=device)
# For non-gated: w1 projects K -> N, w2 projects N -> K
w1 = torch.randn(num_experts, n, k, dtype=dtype, device=device) * 0.1
w2 = torch.randn(num_experts, k, n, dtype=dtype, device=device) * 0.1
topk_weights = torch.ones(m, topk, dtype=torch.float32, device=device) / topk
topk_ids = torch.randint(0, num_experts, (m, topk), device=device)
return hidden_states, w1, w2, topk_weights, topk_ids
@pytest.mark.parametrize("m", M_SIZES)
@pytest.mark.parametrize("n", N_SIZES)
@pytest.mark.parametrize("k", K_SIZES)
@pytest.mark.parametrize("topk", TOPK_VALUES)
@pytest.mark.parametrize("activation", NO_MUL_ACTIVATIONS)
@torch.inference_mode()
def test_triton_experts_no_mul_activation(
m: int,
n: int,
k: int,
topk: int,
activation: MoEActivation,
):
hidden_states, w1, w2, topk_weights, topk_ids = make_test_tensors(
m, n, k, NUM_EXPERTS, topk
)
experts = TritonExperts(
moe_config=make_dummy_moe_config(),
quant_config=FUSED_MOE_UNQUANTIZED_CONFIG,
)
ws1_shape, ws2_shape, out_shape = experts.workspace_shapes(
M=m,
N=n,
K=k,
topk=topk,
global_num_experts=NUM_EXPERTS,
local_num_experts=NUM_EXPERTS,
expert_tokens_meta=None,
activation=activation,
)
# Verify workspace shapes are correct for no_mul activation
# workspace1 should handle activation_out_dim = N (not N//2)
assert ws1_shape == (m, topk, max(n, k)), (
f"workspace1 shape mismatch: expected {(m, topk, max(n, k))}, got {ws1_shape}"
)
# workspace2 should handle max(N, K) for intermediate_cache1/cache3
assert ws2_shape == (m, topk, max(n, k)), (
f"workspace2 shape mismatch: expected {(m, topk, max(n, k))}, got {ws2_shape}"
)
assert out_shape == (m, k), (
f"output shape mismatch: expected {(m, k)}, got {out_shape}"
)
workspace1 = torch.empty(
ws1_shape[0] * ws1_shape[1] * ws1_shape[2],
dtype=hidden_states.dtype,
device=hidden_states.device,
)
workspace2 = torch.empty(
ws2_shape[0] * ws2_shape[1] * ws2_shape[2],
dtype=hidden_states.dtype,
device=hidden_states.device,
)
output = torch.zeros(m, k, dtype=hidden_states.dtype, device=hidden_states.device)
experts.apply(
output=output,
hidden_states=hidden_states,
w1=w1,
w2=w2,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=activation,
global_num_experts=NUM_EXPERTS,
expert_map=None,
a1q_scale=None,
a2_scale=None,
workspace13=workspace1,
workspace2=workspace2,
expert_tokens_meta=None,
apply_router_weight_on_input=False,
)
assert output.shape == (m, k), f"Expected shape {(m, k)}, got {output.shape}"
assert not torch.isnan(output).any(), "Output contains NaN"
assert not torch.isinf(output).any(), "Output contains Inf"
assert output.abs().sum() > 0, "Output is all zeros"
@torch.inference_mode()
def test_workspace_shapes_no_mul_vs_gated():
"""Test that workspace shapes differ correctly between gated and non-gated."""
from vllm.model_executor.layers.fused_moe.experts.triton_moe import TritonExperts
M, N, K, topk = 64, 256, 128, 2
experts = TritonExperts(
moe_config=make_dummy_moe_config(),
quant_config=FUSED_MOE_UNQUANTIZED_CONFIG,
)
ws1_no_mul, _, out_no_mul = experts.workspace_shapes(
M, N, K, topk, 8, 8, None, MoEActivation.SILU_NO_MUL
)
ws1_gated, _, out_gated = experts.workspace_shapes(
M, N, K, topk, 8, 8, None, MoEActivation.SILU
)
# For no_mul: activation_out_dim = N
# For gated: activation_out_dim = N // 2
# workspace1 should use max(activation_out_dim, K)
activation_out_dim_no_mul = N
activation_out_dim_gated = N // 2
assert ws1_no_mul[2] == max(activation_out_dim_no_mul, K), (
f"no_mul workspace1 last dim should be max({activation_out_dim_no_mul}, {K})"
)
assert ws1_gated[2] == max(activation_out_dim_gated, K), (
f"gated workspace1 last dim should be max({activation_out_dim_gated}, {K})"
)
# Output shapes should be the same
assert out_no_mul == out_gated == (M, K)
@torch.inference_mode()
def test_adjust_n_for_activation():
"""Test the adjust_N_for_activation method."""
from vllm.model_executor.layers.fused_moe.experts.triton_moe import TritonExperts
experts = TritonExperts(
moe_config=make_dummy_moe_config(),
quant_config=FUSED_MOE_UNQUANTIZED_CONFIG,
)
N = 256
# Gated activations should return N // 2
assert experts.adjust_N_for_activation(N, MoEActivation.SILU) == N // 2
assert experts.adjust_N_for_activation(N, MoEActivation.GELU) == N // 2
# Non-gated activations should return N
assert experts.adjust_N_for_activation(N, MoEActivation.SILU_NO_MUL) == N
assert experts.adjust_N_for_activation(N, MoEActivation.GELU_NO_MUL) == N
assert experts.adjust_N_for_activation(N, MoEActivation.RELU2_NO_MUL) == N
@@ -1,169 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from https://github.com/sgl-project/sglang/blob/main/test/srt/test_triton_moe_channel_fp8_kernel.py
import itertools
import pytest
import torch
from tests.kernels.moe.utils import fused_moe
from vllm import _custom_ops as ops
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.fused_moe.config import fp8_w8a8_moe_quant_config
from vllm.platforms import current_platform
if current_platform.get_device_capability() < (9, 0):
pytest.skip("FP8 Triton requires CUDA 9.0 or higher", allow_module_level=True)
vllm_config = VllmConfig()
if current_platform.is_fp8_fnuz():
pytest.skip(
"Tests in this file require float8_e4m3fn and platform does not support",
allow_module_level=True,
)
def native_w8a8_per_token_matmul(A, B, As, Bs, output_dtype=torch.float16):
"""Matrix multiplication function that supports per-token input
quantization and per-column weight quantization"""
A = A.to(torch.float32)
B = B.to(torch.float32)
assert A.shape[-1] == B.shape[-1], "Dimension mismatch"
assert B.ndim == 2 and B.is_contiguous(), "B must be a 2D contiguous tensor"
# Reshape input
M = A.numel() // A.shape[-1]
B = B.t() # Transpose weight matrix
N, K = B.shape
origin_C_shape = A.shape[:-1] + (K,)
A = A.reshape(M, N)
# As is per-token [M, 1], Bs is per-column [1, K]
C = torch.matmul(A, B) # [M, K]
C = As * C * Bs.view(1, -1) # Broadcast per-column scale
return C.reshape(origin_C_shape).to(output_dtype)
def fp8_mask(a, mask):
dtype = a.dtype
return a.view(torch.int8)[mask].view(dtype)
def torch_w8a8_per_column_moe(a, w1, w2, w1_s, w2_s, score, topk):
"""This function performs fused moe with per-column int8
quantization using native torch."""
B, D = a.shape
# Perform per-token quantization
a_q, a_s = ops.scaled_fp8_quant(a, use_per_token_if_dynamic=True)
# Repeat tokens to match topk
a_q = a_q.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
# Also repeat the scale
a_s = a_s.view(B, -1, 1).repeat(1, topk, 1).reshape(-1, 1) # [B*topk, 1]
out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device)
# Calculate routing
score = torch.softmax(score, dim=-1, dtype=torch.float32)
topk_weight, topk_ids = torch.topk(score, topk)
topk_weight = topk_weight.view(-1)
topk_ids = topk_ids.view(-1)
# Process each expert
for i in range(w1.shape[0]):
mask = topk_ids == i
if mask.sum():
# First MLP layer: note that a_s is now per-token
inter_out = native_w8a8_per_token_matmul(
fp8_mask(a_q, mask),
w1[i],
fp8_mask(a_s, mask),
w1_s[i],
output_dtype=a.dtype,
)
# Activation function
act_out = SiluAndMul().forward_native(inter_out)
# Quantize activation output with per-token
act_out_q, act_out_s = ops.scaled_fp8_quant(
act_out, use_per_token_if_dynamic=True
)
# Second MLP layer
out[mask] = native_w8a8_per_token_matmul(
act_out_q, w2[i], act_out_s, w2_s[i], output_dtype=a.dtype
)
# Apply routing weights and sum
return (
out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype)
).sum(dim=1)
@pytest.fixture(autouse=True)
def setup_cuda():
"""Sets the default CUDA device before each test in this module."""
torch.set_default_device("cuda")
DTYPES = [torch.half, torch.bfloat16]
M = [1, 33]
N = [128, 1024]
K = [256, 4096]
E = [8]
TOP_KS = [2, 6]
SEEDS = [0]
@pytest.mark.parametrize(
"M, N, K, E, topk, dtype, seed",
itertools.product(M, N, K, E, TOP_KS, DTYPES, SEEDS),
)
@torch.inference_mode()
def test_w8a8_fp8_fused_moe(M, N, K, E, topk, dtype, seed):
torch.manual_seed(seed)
# Initialize int8 quantization parameters
factor_for_scale = 1e-2
finfo = torch.finfo(torch.float8_e4m3fn)
fp8_max = finfo.max
fp8_min = finfo.min
# Input tensor
# M * K
a = torch.randn((M, K), dtype=dtype) / 10
# Generate int8 weights
w1_fp32 = (torch.rand((E, 2 * N, K), dtype=torch.float32) - 0.5) * 2
w1 = (w1_fp32 * fp8_max).clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn)
w2_fp32 = (torch.rand((E, K, N), dtype=torch.float32) - 0.5) * 2
w2 = (w2_fp32 * fp8_max).clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn)
# Generate scale for each column (per-column quantization)
w1_s = torch.rand(E, 2 * N, device=w1_fp32.device) * factor_for_scale
w2_s = torch.rand(E, K, device=w2_fp32.device) * factor_for_scale
score = torch.randn((M, E), dtype=dtype)
with set_current_vllm_config(vllm_config):
ref_out = torch_w8a8_per_column_moe(a, w1, w2, w1_s, w2_s, score, topk)
out = fused_moe(
a,
w1,
w2,
score,
topk,
renormalize=False,
quant_config=fp8_w8a8_moe_quant_config(
per_act_token_quant=True,
w1_scale=w1_s,
w2_scale=w2_s,
block_shape=None, # Not using block quantization
),
)
# Check results
rel_diff = torch.mean(
torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))
) / torch.mean(torch.abs(ref_out.to(torch.float32)))
assert rel_diff < 0.05
-201
View File
@@ -1,201 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the FlashInfer TRTLLM BF16 MoE backend
(`TrtLlmBf16ExpertsModular`).
This mirrors the TRTLLM NvFP4 modular test shape: construct the modular
expert wrapper directly, pass production-format BlockMajorK weights, and
compare against a torch MoE reference using the original BF16 weights.
"""
import pytest
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from tests.kernels.moe.utils import make_dummy_moe_config
from tests.kernels.utils import torch_moe
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.config import (
FUSED_MOE_UNQUANTIZED_CONFIG,
FusedMoEConfig,
FusedMoEParallelConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import (
TrtLlmBf16ExpertsModular,
)
from vllm.model_executor.layers.fused_moe.experts.trtllm_lora_moe import (
TrtLlmBf16LoRAExperts,
)
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
UnquantizedMoeBackend,
convert_to_unquantized_kernel_format,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
from vllm.utils.math_utils import next_power_of_2
from vllm.utils.torch_utils import set_random_seed
if pytest and (
not has_flashinfer_trtllm_fused_moe()
or not current_platform.is_device_capability_family(100)
):
pytest.skip(
"Requires flashinfer TRTLLM fused MoE BF16 backend (SM100)",
allow_module_level=True,
)
# (m, n, k) = (tokens, intermediate_size_per_partition, hidden_dim).
MNK_FACTORS = [
(2, 160, 2560),
(2, 1024, 1024),
(64, 2048, 1536),
(64, 1024, 4096),
]
@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
@pytest.mark.parametrize("e", [128])
@pytest.mark.parametrize("topk", [8])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@torch.inference_mode()
def test_trtllm_bf16_moe_modular_no_graph(
m: int,
n: int,
k: int,
e: int,
topk: int,
dtype: torch.dtype,
workspace_init,
):
set_random_seed(7)
with set_current_vllm_config(
VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
):
a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10
w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10
# The FlashInfer conversion may rewrite unpadded input storage in place.
# Preserve the original layout for the independent torch reference.
reference_w1 = w1.clone()
reference_w2 = w2.clone()
score = torch.randn((m, e), device="cuda", dtype=dtype)
scores = torch.softmax(score, dim=-1, dtype=torch.float32)
topk_weights, topk_ids = torch.topk(scores, topk)
topk_weights = topk_weights.contiguous()
topk_ids = topk_ids.to(torch.int32).contiguous()
moe_config = FusedMoEConfig(
num_experts=e,
experts_per_token=topk,
hidden_dim=k,
intermediate_size=n,
num_local_experts=e,
num_logical_experts=e,
activation=MoEActivation.SILU,
device="cuda",
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
in_dtype=dtype,
routing_method=RoutingMethodType.TopK,
max_num_tokens=next_power_of_2(m),
)
trtllm_w1, trtllm_w2 = convert_to_unquantized_kernel_format(
UnquantizedMoeBackend.FLASHINFER_TRTLLM,
moe_config,
w1,
w2,
)
expected_n = (n + 127) // 128 * 128
assert moe_config.intermediate_size_per_partition == expected_n
assert trtllm_w2.numel() == e * k * expected_n
trtllm_experts = mk.FusedMoEKernel(
maybe_make_prepare_finalize(
moe=moe_config,
quant_config=FUSED_MOE_UNQUANTIZED_CONFIG,
allow_new_interface=True,
use_monolithic=False,
),
TrtLlmBf16ExpertsModular(
moe_config=moe_config,
quant_config=FUSED_MOE_UNQUANTIZED_CONFIG,
),
)
trtllm_output = trtllm_experts.apply(
hidden_states=a,
w1=trtllm_w1,
w2=trtllm_w2,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=MoEActivation.SILU,
global_num_experts=e,
expert_map=None,
apply_router_weight_on_input=False,
)
torch_output = torch_moe(
a,
reference_w1,
reference_w2,
score,
topk,
activation=MoEActivation.SILU,
)
torch.testing.assert_close(
torch_output,
trtllm_output,
atol=1e-1,
rtol=2e-1,
)
@pytest.mark.parametrize("has_lora_delta", [False, True])
@torch.inference_mode()
def test_trtllm_bf16_lora_accepts_checkpoint_shaped_weights(has_lora_delta):
"""LoRA dispatch accepts both 3D parameters and legacy 4D packed views."""
config = make_dummy_moe_config(
num_experts=128, hidden_dim=256, intermediate_size=128
)
experts = TrtLlmBf16LoRAExperts(config, FUSED_MOE_UNQUANTIZED_CONFIG)
x = torch.randn(2, 256, device="cuda", dtype=torch.bfloat16) / 10
w1, w2 = convert_to_unquantized_kernel_format(
UnquantizedMoeBackend.FLASHINFER_TRTLLM,
config,
torch.randn(128, 256, 256, device="cuda", dtype=torch.bfloat16) / 10,
torch.randn(128, 256, 128, device="cuda", dtype=torch.bfloat16) / 10,
)
assert w1.ndim == w2.ndim == 3
topk = (
torch.tensor([[0], [1]], device="cuda", dtype=torch.int32),
torch.ones(2, 1, device="cuda", dtype=torch.float32),
)
delta = torch.randn(2, 1, 256, device="cuda", dtype=torch.bfloat16)
def invoke(w1, w2):
result = experts.invoke_routed_moe(
hidden_states=x,
w1=w1,
w2=w2,
topk_ids_and_weights=topk,
gemm1_lora_delta=delta if has_lora_delta else None,
global_num_experts=128,
a1q_scale=None,
output=torch.empty_like(x),
)
if has_lora_delta:
# Only compare rows belonging to real tokens, excluding padding.
indices = result[2].flatten().long()
return result[0][indices], result[3][indices]
return result
actual = invoke(w1, w2)
expected = invoke(w1.view(128, 4, 256, 64), w2.view(128, 2, 256, 64))
for actual_tensor, expected_tensor in zip(actual, expected):
torch.testing.assert_close(actual_tensor, expected_tensor)
-514
View File
@@ -1,514 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the FlashInfer TRTLLM NvFP4 MoE backend
(`TrtLlmNvFp4ExpertsModular`).
Covers the activations the wrapper claims to support — SiLU, RELU^2 (non-gated),
and GELU — including a Gemma4-shaped case (128 experts, top-k 8,
intermediate_size 704) that exercises the non-256-aligned padding path.
"""
from dataclasses import replace
import pytest
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from tests.kernels.moe.utils import check_deferred_moe_finalize, make_test_quant_config
from tests.kernels.quantization.nvfp4_utils import (
convert_swizzled_to_linear,
dequantize_nvfp4_to_dtype,
)
from tests.kernels.utils import torch_moe
from vllm import _custom_ops as ops
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
from vllm.model_executor.custom_op import CustomOp, op_registry
from vllm.model_executor.layers.activation import SiluAndMulWithClamp, SituAndMul
from vllm.model_executor.layers.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe import (
TrtLlmNvFp4ExpertsModular,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
prepare_static_weights_for_trtllm_fp4_moe,
reorder_w1w3_to_w3w1,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
NVFP4_PER_TOKEN_BASE_GLOBAL_SCALE,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
from vllm.utils.math_utils import next_power_of_2
from vllm.utils.torch_utils import set_random_seed
if pytest and (
not has_flashinfer_trtllm_fused_moe()
or not current_platform.is_device_capability_family(100)
):
pytest.skip(
"Requires flashinfer TRTLLM fused MoE and NvFP4 (SM100)",
allow_module_level=True,
)
# (m, n, k) = (tokens, intermediate_size_per_partition, hidden_dim).
# The (64, 704, 4096) row matches Gemma4's MoE shape and exercises the
# non-256-aligned intermediate (padded inside the wrapper).
MNK_FACTORS = [
(2, 1024, 1024),
(64, 2048, 1536),
(64, 704, 4096),
]
_SWIGLU_LIMIT = 0.1
_LARGE_OUTPUT1_SCALE = 32768.0
_CLAMP_OP_NAME = "test_silu_and_mul_with_clamp"
_SITU_OP_NAME = "test_situ_and_mul"
# Test-only fixed-limit clamp. ``custom_op_name`` makes the class itself
# valid as an ``activation=`` argument to ``torch_moe`` (which only looks
# up ``activation.custom_op_name`` in ``op_registry``), so no
# ``MoEActivation`` enum extension is needed.
if _CLAMP_OP_NAME not in op_registry:
@CustomOp.register(_CLAMP_OP_NAME)
class _SiluAndMulWithClampTest(SiluAndMulWithClamp):
custom_op_name = _CLAMP_OP_NAME
def __init__(self, *, compile_native: bool = True) -> None:
super().__init__(_SWIGLU_LIMIT, compile_native=compile_native)
if _SITU_OP_NAME not in op_registry:
@CustomOp.register(_SITU_OP_NAME)
class _SituAndMulTest(SituAndMul):
custom_op_name = _SITU_OP_NAME
def __init__(self, *, compile_native: bool = True) -> None:
super().__init__(4.0, 25.0, compile_native=compile_native)
SILU_WITH_CLAMP = op_registry[_CLAMP_OP_NAME]
SITU = op_registry[_SITU_OP_NAME]
def _make_trtllm_fp4_moe_kernel(
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
per_token_activation: bool = False,
) -> mk.FusedMoEKernel:
experts = TrtLlmNvFp4ExpertsModular(
moe_config=moe_config,
quant_config=quant_config,
per_token_activation=per_token_activation,
)
# Mimic the production weight-loader path so per-expert tensors that
# are normally precomputed in process_weights_after_loading (g1_scale_c
# and the rescaled gemm1_clamp_limit) get materialized. The test's
# synthetic quant_config has g1_alphas/g2_alphas already at their
# post-fusion values, so we set w13_weight_scale_2 to alias g1_alphas
# (same tensor) and use input_scale=1 to make the in-place
# weight_scale_2 *= input_scale step a no-op.
fake_layer = torch.nn.Module()
fake_layer.w13_weight_scale_2 = quant_config.g1_alphas
fake_layer.w2_weight_scale_2 = quant_config.g2_alphas
fake_layer.w13_input_scale = torch.ones_like(quant_config.g1_alphas)
fake_layer.w2_input_scale = torch.ones_like(quant_config.g2_alphas)
experts.process_weights_after_loading(fake_layer)
return mk.FusedMoEKernel(
maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
use_monolithic=False,
),
experts,
)
ACTIVATION_CASES = [
pytest.param(MoEActivation.SILU, MoEActivation.SILU, None, False, id="silu"),
pytest.param(
MoEActivation.SILU,
MoEActivation.SILU,
None,
True,
id="silu_per_token",
),
pytest.param(
MoEActivation.SILU,
SILU_WITH_CLAMP,
_SWIGLU_LIMIT,
False,
id="silu_clamp",
),
pytest.param(MoEActivation.SITU, SITU, None, False, id="situ"),
pytest.param(
MoEActivation.RELU2_NO_MUL,
MoEActivation.RELU2_NO_MUL,
None,
False,
id="relu2_no_mul",
),
pytest.param(
MoEActivation.RELU2_NO_MUL,
MoEActivation.RELU2_NO_MUL,
None,
True,
id="relu2_no_mul_per_token",
),
pytest.param(MoEActivation.GELU, MoEActivation.GELU, None, False, id="gelu"),
]
@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
@pytest.mark.parametrize("e", [128])
@pytest.mark.parametrize("topk", [8])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize(
"activation,torch_activation,swiglu_limit,per_token_activation",
ACTIVATION_CASES,
)
@torch.inference_mode()
def test_trtllm_fp4_moe_no_graph(
m: int,
n: int,
k: int,
e: int,
topk: int,
dtype: torch.dtype,
activation: MoEActivation,
torch_activation: MoEActivation | type[SiluAndMulWithClamp] | type[SituAndMul],
swiglu_limit: float | None,
per_token_activation: bool,
workspace_init,
):
# FlashInfer's trtllm_batched_gemm_runner has no precompiled tile
# config for non-gated RELU^2 at non-256-aligned intermediate_size
# (e.g. Gemma4's 704). Other activations (SiLU/GELU) work at the
# same shape. Tracked upstream in FlashInfer; unrelated to this
# PR's GELU enablement (Gemma4 uses GeGLU, not non-gated RELU^2).
if activation == MoEActivation.RELU2_NO_MUL and (m, n, k) == (64, 704, 4096):
pytest.skip(
"FlashInfer trtllm_batched_gemm_runner: no valid tile config "
"for non-gated RELU^2 at intermediate_size=704 "
"(getValidConfigIndices throws). Tracked upstream."
)
set_random_seed(7)
with set_current_vllm_config(
VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
):
a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
quant_blocksize = 16
is_gated_act = activation.is_gated
w1_q, w2_q, quant_config = make_test_quant_config(
e,
n,
k,
in_dtype=dtype,
quant_dtype="nvfp4",
block_shape=None,
per_act_token_quant=False,
make_gate=is_gated_act,
# The TRT-LLM FP4 MoE kernel rejects swizzled (padded) activation
# scales — its numel-based vec_size check requires numel == M*K/16.
# Match what oracle/nvfp4.py does for this backend.
is_scale_swizzled=False,
)
quant_config.gemm1_clamp_limit = swiglu_limit
if swiglu_limit is not None:
assert quant_config.g1_alphas is not None
assert quant_config.a2_gscale is not None
assert torch.all(quant_config.a2_gscale == 1)
# With a2_gscale == 1, g1_alphas is the TRTLLM
# output1_scale_gate_scalar. Make it large enough to catch
# clamp/output-scale coupling in the FlashInfer kernel wrapper.
quant_config.g1_alphas.fill_(_LARGE_OUTPUT1_SCALE)
w1_q_linear = w1_q
w2_q_linear = w2_q
w1_scale_reference = quant_config.w1_scale
w2_scale_reference = quant_config.w2_scale
assert w1_scale_reference is not None
assert w2_scale_reference is not None
w1_scale_linear = torch.stack(
[
convert_swizzled_to_linear(scale, w1_q.size(1), k, quant_blocksize)
for scale in w1_scale_reference
]
)
w2_scale_linear = torch.stack(
[
convert_swizzled_to_linear(scale, w2_q.size(1), n, quant_blocksize)
for scale in w2_scale_reference
]
)
w1_q_for_kernel = w1_q
if is_gated_act:
w1_q_for_kernel, w1_scale_linear = reorder_w1w3_to_w3w1(
w1_q.clone(), w1_scale_linear
)
w1_q, w1_scale, w2_q, w2_scale = prepare_static_weights_for_trtllm_fp4_moe(
w1_q_for_kernel,
w2_q,
w1_scale_linear,
w2_scale_linear,
hidden_size=k,
intermediate_size=n,
num_experts=e,
is_gated_activation=is_gated_act,
)
quant_config._w1.scale = w1_scale
quant_config._w2.scale = w2_scale
score = torch.randn((m, e), device="cuda", dtype=dtype)
topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False)
moe_config = FusedMoEConfig(
num_experts=e,
experts_per_token=topk,
hidden_dim=k,
intermediate_size=n,
num_local_experts=e,
num_logical_experts=e,
activation=activation,
device="cuda",
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
in_dtype=dtype,
routing_method=RoutingMethodType.TopK,
max_num_tokens=next_power_of_2(m),
activation_situ_beta=4.0 if activation == MoEActivation.SITU else None,
activation_situ_linear_beta=(
25.0 if activation == MoEActivation.SITU else None
),
)
trtllm_experts = _make_trtllm_fp4_moe_kernel(
moe_config,
quant_config,
per_token_activation=per_token_activation,
)
if activation == MoEActivation.SITU:
torch.testing.assert_close(
trtllm_experts.fused_experts.g1_scale_c, quant_config.a2_gscale
)
trtllm_output = trtllm_experts.apply(
hidden_states=a,
w1=w1_q,
w2=w2_q,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=activation,
global_num_experts=e,
expert_map=None,
apply_router_weight_on_input=False,
)
# Reference: round-trip activations and weights through FP4
# quant/dequant so the comparison isolates kernel/activation behavior
# from quantization error.
if per_token_activation:
from flashinfer import SfLayout, nvfp4_quantize
a_fp4, a_scale_interleaved, per_token_scale = nvfp4_quantize(
a,
NVFP4_PER_TOKEN_BASE_GLOBAL_SCALE,
sfLayout=SfLayout.layout_linear,
per_token_activation=True,
)
a_global_scale = 1.0 / per_token_scale.unsqueeze(1)
else:
assert quant_config.a1_gscale is not None
a_global_scale = quant_config.a1_gscale[0]
a_fp4, a_scale_interleaved = ops.scaled_fp4_quant(a, a_global_scale)
a_in_dtype = dequantize_nvfp4_to_dtype(
a_fp4,
a_scale_interleaved,
a_global_scale,
dtype=a.dtype,
device=a.device,
block_size=quant_blocksize,
is_sf_linear_layout=per_token_activation,
)
w1_d = torch.empty(
(e, (2 if is_gated_act else 1) * n, k), device="cuda", dtype=dtype
)
w2_d = torch.empty((e, k, n), device="cuda", dtype=dtype)
for idx in range(e):
w1_d[idx] = dequantize_nvfp4_to_dtype(
w1_q_linear[idx],
w1_scale_reference[idx],
(1 / quant_config.g1_alphas[idx]),
dtype=dtype,
device=w1_q.device,
block_size=quant_blocksize,
)
w2_d[idx] = dequantize_nvfp4_to_dtype(
w2_q_linear[idx],
w2_scale_reference[idx],
(1 / quant_config.g2_alphas[idx]),
dtype=dtype,
device=w2_q.device,
block_size=quant_blocksize,
)
torch_output = torch_moe(
a_in_dtype, w1_d, w2_d, score, topk, activation=torch_activation
)
if swiglu_limit is None:
cosine = torch.nn.functional.cosine_similarity(
torch_output.float().flatten(),
trtllm_output.float().flatten(),
dim=0,
)
assert cosine > 0.99
torch.testing.assert_close(torch_output, trtllm_output, atol=2e-1, rtol=2e-1)
@torch.inference_mode()
def test_trtllm_fp4_moe_reprocess_does_not_refold_swiglu_params():
"""Weight reloads rerun post-processing on the same experts object; the
g1_alphas fold of clamp/beta must start from the unfolded values."""
e, n, k = 8, 256, 256
clamp, beta = 7.0, 1.0
with set_current_vllm_config(
VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
):
_, _, quant_config = make_test_quant_config(
e,
n,
k,
in_dtype=torch.bfloat16,
quant_dtype="nvfp4",
block_shape=None,
per_act_token_quant=False,
is_scale_swizzled=False,
)
quant_config.gemm1_clamp_limit = clamp
quant_config.gemm1_beta = beta
moe_config = FusedMoEConfig(
num_experts=e,
experts_per_token=2,
hidden_dim=k,
intermediate_size=n,
num_local_experts=e,
num_logical_experts=e,
activation=MoEActivation.SILU,
device="cuda",
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
in_dtype=torch.bfloat16,
routing_method=RoutingMethodType.TopK,
max_num_tokens=16,
)
experts = TrtLlmNvFp4ExpertsModular(
moe_config=moe_config, quant_config=quant_config
)
g1_alphas = quant_config.g1_alphas
assert g1_alphas is not None
layer = torch.nn.Module()
layer.w13_weight_scale_2 = g1_alphas
layer.w2_weight_scale_2 = quant_config.g2_alphas
layer.w13_input_scale = torch.ones_like(g1_alphas)
layer.w2_input_scale = torch.ones_like(g1_alphas)
for new_alphas in (0.5, 0.25):
g1_alphas.fill_(new_alphas)
experts.process_weights_after_loading(layer)
expected = torch.full_like(g1_alphas, 1.0 / new_alphas)
torch.testing.assert_close(experts.gemm1_clamp_limit, clamp * expected)
torch.testing.assert_close(experts.gemm1_beta, beta * expected)
@pytest.mark.parametrize("m,chunk_size", [(1, None), (16, None), (16, 5)])
@torch.inference_mode()
def test_trtllm_fp4_moe_deferred_finalize(
m: int,
chunk_size: int | None,
monkeypatch: pytest.MonkeyPatch,
workspace_init,
):
"""TRTLLM-Gen NvFP4 modular experts can leave the top-k finalize to the caller."""
e, topk, n, k = 128, 8, 1024, 1024
dtype = torch.bfloat16
if chunk_size is not None:
monkeypatch.setattr(
TrtLlmNvFp4ExpertsModular, "_get_chunk_size", lambda self: chunk_size
)
set_random_seed(7)
with set_current_vllm_config(
VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
):
a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
w1_q, w2_q, quant_config = make_test_quant_config(
e, n, k, in_dtype=dtype, quant_dtype="nvfp4", is_scale_swizzled=False
)
score = torch.randn((m, e), device="cuda", dtype=dtype)
topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False)
# One rank of a TP group, which deferral needs.
moe_config = FusedMoEConfig(
num_experts=e,
experts_per_token=topk,
hidden_dim=k,
intermediate_size=2 * n,
num_local_experts=e,
num_logical_experts=e,
activation=MoEActivation.SILU,
device="cuda",
moe_parallel_config=replace(
FusedMoEParallelConfig.make_no_parallel(), tp_size=2
),
in_dtype=dtype,
routing_method=RoutingMethodType.TopK,
max_num_tokens=next_power_of_2(m),
)
kernel = _make_trtllm_fp4_moe_kernel(moe_config, quant_config)
check_deferred_moe_finalize(
moe_config,
lambda: kernel.apply(
hidden_states=a,
w1=w1_q,
w2=w2_q,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=MoEActivation.SILU,
global_num_experts=e,
expert_map=None,
apply_router_weight_on_input=False,
),
router_weights=topk_weights,
chunked=chunk_size is not None,
)
if __name__ == "__main__":
test_trtllm_fp4_moe_no_graph(
64,
704,
4096,
128,
8,
torch.bfloat16,
MoEActivation.GELU,
MoEActivation.GELU,
None,
False,
None,
)
@@ -1,642 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import patch
import pytest
import torch
from tests.kernels.moe.utils import make_dummy_moe_config
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
UnquantizedMoeBackend,
backend_to_kernel_cls,
select_unquantized_moe_backend,
)
from vllm.platforms import CpuArchEnum, current_platform
skipif_not_cuda_rocm = pytest.mark.skipif(
not (current_platform.is_cuda() or current_platform.is_rocm()),
reason="Only supported on CUDA/ROCm platforms.",
)
@pytest.mark.parametrize(
("amx_supported", "in_dtype", "expect_amx_kernel"),
[
(True, torch.bfloat16, True),
(False, torch.bfloat16, False),
(True, torch.float16, False),
],
)
def test_x86_cpu_unquantized_kernel_selection(
amx_supported: bool,
in_dtype: torch.dtype,
expect_amx_kernel: bool,
):
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
CPUUnquantizedExperts,
X86CPUUnquantizedExperts,
)
with (
patch.object(current_platform, "is_cpu", return_value=True),
patch.object(
current_platform,
"get_cpu_architecture",
return_value=CpuArchEnum.X86,
),
patch("torch.cpu._is_amx_tile_supported", return_value=amx_supported),
):
moe_config = make_dummy_moe_config(
hidden_dim=128,
intermediate_size=128,
in_dtype=in_dtype,
)
kernel_cls = next(
cls
for cls in backend_to_kernel_cls(UnquantizedMoeBackend.CPU)
if cls.is_supported_config(
cls,
moe_config,
None,
None,
CPUUnquantizedExperts.activation_format(),
)[0]
)
expected_kernel_cls = (
X86CPUUnquantizedExperts if expect_amx_kernel else CPUUnquantizedExperts
)
assert kernel_cls is expected_kernel_cls
@pytest.mark.parametrize(
("platform", "in_dtype", "expect_arm_kernel"),
[
("linux", torch.bfloat16, True),
("linux", torch.float16, False),
("darwin", torch.bfloat16, False),
],
)
def test_arm_cpu_unquantized_kernel_selection(
platform: str,
in_dtype: torch.dtype,
expect_arm_kernel: bool,
):
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
ArmCPUUnquantizedExperts,
CPUUnquantizedExperts,
)
with (
patch.object(current_platform, "is_cpu", return_value=True),
patch.object(
current_platform,
"get_cpu_architecture",
return_value=CpuArchEnum.ARM,
),
patch(
"vllm.model_executor.layers.fused_moe.experts.cpu_moe.sys.platform",
platform,
),
):
moe_config = make_dummy_moe_config(
hidden_dim=128,
intermediate_size=128,
in_dtype=in_dtype,
)
kernel_cls = next(
cls
for cls in backend_to_kernel_cls(UnquantizedMoeBackend.CPU)
if cls.is_supported_config(
cls,
moe_config,
None,
None,
CPUUnquantizedExperts.activation_format(),
)[0]
)
expected_kernel_cls = (
ArmCPUUnquantizedExperts if expect_arm_kernel else CPUUnquantizedExperts
)
assert kernel_cls is expected_kernel_cls
@pytest.mark.parametrize(
"platform_method,expected_backend",
[
("is_cuda", UnquantizedMoeBackend.TRITON), # Default CUDA without FlashInfer
("is_rocm", UnquantizedMoeBackend.TRITON), # ROCm without AITER
("is_cpu", UnquantizedMoeBackend.CPU),
("is_xpu", UnquantizedMoeBackend.XPU),
("is_tpu", UnquantizedMoeBackend.TPU),
("is_out_of_tree", UnquantizedMoeBackend.OOT),
],
)
@patch(
"vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe."
"has_flashinfer_cutlass_fused_moe",
return_value=False,
)
@patch(
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe."
"has_flashinfer_trtllm_fused_moe",
return_value=False,
)
@patch(
"vllm.model_executor.layers.fused_moe.oracle.unquantized.rocm_aiter_ops.is_fused_moe_enabled",
return_value=False,
)
def test_select_default_backend_by_platform(
mock_aiter_enabled,
mock_has_flashinfer_trtllm,
mock_has_flashinfer_cutlass,
monkeypatch,
platform_method,
expected_backend,
):
"""Test default backend selection per platform with all optional
accelerators (FlashInfer, AITER) disabled.
Patch the feature probes directly because their cached results can outlive
changes to has_flashinfer().
"""
with patch(
"vllm.model_executor.layers.fused_moe.oracle.unquantized.current_platform"
) as mock_platform:
# Set all platform checks to False
mock_platform.is_cuda.return_value = False
mock_platform.is_rocm.return_value = False
mock_platform.is_cpu.return_value = False
mock_platform.is_xpu.return_value = False
mock_platform.is_tpu.return_value = False
mock_platform.is_out_of_tree.return_value = False
# Set only the specified platform to True
getattr(mock_platform, platform_method).return_value = True
with (
patch.object(current_platform, "is_cuda", return_value=False),
patch.object(current_platform, "is_rocm", return_value=False),
patch.object(current_platform, "is_cpu", return_value=False),
patch.object(current_platform, "is_xpu", return_value=False),
patch.object(current_platform, "is_tpu", return_value=False),
patch.object(current_platform, "is_out_of_tree", return_value=False),
patch.object(current_platform, platform_method, return_value=True),
):
# CPU's grouped-gemm kernels require hidden/intermediate sizes
# aligned to 32; the size-1 defaults only work for backends that
# don't check shapes at selection time.
moe_config = (
make_dummy_moe_config(hidden_dim=128, intermediate_size=128)
if expected_backend == UnquantizedMoeBackend.CPU
else make_dummy_moe_config()
)
selected_backend, expert_cls = select_unquantized_moe_backend(
moe_config=moe_config
)
assert selected_backend == expected_backend
if expected_backend in [
UnquantizedMoeBackend.OOT,
UnquantizedMoeBackend.TPU,
]:
assert expert_cls is None
else:
assert expert_cls is not None
@patch(
"vllm.utils.flashinfer.has_flashinfer",
return_value=False,
)
@patch(
"vllm.model_executor.layers.fused_moe.oracle.unquantized.rocm_aiter_ops.is_fused_moe_enabled",
return_value=True,
)
@pytest.mark.skipif(
not current_platform.is_rocm(), reason="ROCm-specific backend selection test"
)
def test_select_rocm_aiter_backend(mock_aiter_enabled, mock_has_flashinfer):
"""Test ROCm backend selection when AITER is available."""
with patch(
"vllm.model_executor.layers.fused_moe.oracle.unquantized.current_platform"
) as mock_platform:
mock_platform.is_cuda.return_value = False
mock_platform.is_rocm.return_value = True
mock_platform.is_cpu.return_value = False
mock_platform.is_xpu.return_value = False
mock_platform.is_tpu.return_value = False
mock_platform.is_out_of_tree.return_value = False
moe_config = make_dummy_moe_config()
selected_backend, expert_cls = select_unquantized_moe_backend(
moe_config=moe_config,
)
assert selected_backend == UnquantizedMoeBackend.AITER
assert expert_cls is not None
@patch(
"vllm.utils.flashinfer.has_flashinfer",
return_value=False,
)
@patch(
"vllm.model_executor.layers.fused_moe.oracle.unquantized.rocm_aiter_ops."
"is_fused_moe_enabled",
return_value=True,
)
@patch(
"vllm.model_executor.layers.fused_moe.oracle.unquantized.rocm_aiter_ops."
"is_rdna_aiter_enabled",
return_value=False,
)
@pytest.mark.skipif(
not current_platform.is_rocm(), reason="ROCm-specific backend selection test"
)
def test_select_rocm_aiter_backend_non_gated_activation_falls_back(
mock_rdna_disabled, mock_aiter_enabled, mock_has_flashinfer, monkeypatch
):
"""Test ROCm backend selection falls back (not raises) for non-gated MoE."""
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
with patch(
"vllm.model_executor.layers.fused_moe.oracle.unquantized.current_platform"
) as mock_platform:
mock_platform.is_cuda.return_value = False
mock_platform.is_rocm.return_value = True
mock_platform.is_cpu.return_value = False
mock_platform.is_xpu.return_value = False
mock_platform.is_tpu.return_value = False
mock_platform.is_out_of_tree.return_value = False
moe_config = make_dummy_moe_config(activation=MoEActivation.SILU_NO_MUL)
assert moe_config.is_act_and_mul is False
selected_backend, expert_cls = select_unquantized_moe_backend(
moe_config=moe_config,
)
assert selected_backend != UnquantizedMoeBackend.AITER
assert expert_cls is not None
@pytest.mark.skipif(
not current_platform.is_rocm(), reason="ROCm-specific backend selection test"
)
def test_explicit_aiter_backend_non_gated_activation_still_raises():
"""Explicit `--moe-backend aiter` still raises for non-gated MoE;
only the env-var opt-in path falls back."""
moe_config = make_dummy_moe_config(activation=MoEActivation.SILU_NO_MUL)
moe_config.moe_backend = "aiter" # explicit pin, not "auto"
assert moe_config.is_act_and_mul is False
with pytest.raises(ValueError):
select_unquantized_moe_backend(moe_config=moe_config)
@patch(
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsMonolithic.is_supported_config",
return_value=(True, None),
)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="Only supported on NVIDIA platforms."
)
def test_select_cuda_flashinfer_trtllm_backend(
mock_is_supported_trtllm_monolithic,
):
"""Test CUDA backend selection when FlashInfer TRTLLM is available and enabled."""
with (
patch.object(current_platform, "is_cuda", return_value=True),
patch.object(current_platform, "is_rocm", return_value=False),
patch.object(current_platform, "is_cpu", return_value=False),
patch.object(current_platform, "is_xpu", return_value=False),
patch.object(current_platform, "is_tpu", return_value=False),
patch.object(current_platform, "is_out_of_tree", return_value=False),
patch.object(current_platform, "has_device_capability", return_value=True),
):
moe_config = make_dummy_moe_config()
moe_config.moe_backend = "flashinfer_trtllm"
# TRTLLM requires EP and does not support DP
moe_config.moe_parallel_config.use_ep = True
moe_config.moe_parallel_config.use_dp = False
selected_backend, experts_cls = select_unquantized_moe_backend(
moe_config=moe_config
)
assert selected_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM
assert experts_cls is not None
assert experts_cls.__name__ == "TrtLlmBf16ExpertsMonolithic"
@patch(
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsMonolithic.is_supported_config",
return_value=(False, "monolithic unsupported"),
)
@patch(
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsModular.is_supported_config",
return_value=(True, None),
)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="Only supported on NVIDIA platforms."
)
def test_select_cuda_flashinfer_trtllm_modular_backend(
mock_is_supported_trtllm_modular,
mock_is_supported_trtllm_monolithic,
):
"""Test CUDA backend selection falls back to FlashInfer TRTLLM modular."""
with (
patch.object(current_platform, "is_cuda", return_value=True),
patch.object(current_platform, "is_rocm", return_value=False),
patch.object(current_platform, "is_cpu", return_value=False),
patch.object(current_platform, "is_xpu", return_value=False),
patch.object(current_platform, "is_tpu", return_value=False),
patch.object(current_platform, "is_out_of_tree", return_value=False),
patch.object(current_platform, "has_device_capability", return_value=True),
):
moe_config = make_dummy_moe_config()
moe_config.moe_backend = "flashinfer_trtllm"
moe_config.moe_parallel_config.use_ep = True
moe_config.moe_parallel_config.use_dp = False
selected_backend, experts_cls = select_unquantized_moe_backend(
moe_config=moe_config
)
assert selected_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM
assert experts_cls is not None
assert experts_cls.__name__ == "TrtLlmBf16ExpertsModular"
@patch(
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsBase._supports_current_device",
return_value=True,
)
@pytest.mark.parametrize(
"all2all_backend",
[
"mori_high_throughput",
"mori_low_latency",
"flashinfer_nvlink_two_sided",
"flashinfer_nvlink_one_sided",
],
)
def test_select_cuda_flashinfer_trtllm_modular_for_standard_all2all(
mock_supports_current_device,
all2all_backend,
):
"""Test non-AG/RS standard-format all2all backends select modular BF16."""
with (
patch.object(current_platform, "is_cuda", return_value=True),
patch.object(current_platform, "is_rocm", return_value=False),
patch.object(current_platform, "is_cpu", return_value=False),
patch.object(current_platform, "is_xpu", return_value=False),
patch.object(current_platform, "is_tpu", return_value=False),
patch.object(current_platform, "is_out_of_tree", return_value=False),
patch.object(
current_platform, "is_device_capability_family", return_value=False
),
):
moe_config = make_dummy_moe_config(num_experts=4, num_local_experts=2)
moe_config.moe_backend = "flashinfer_trtllm"
moe_config.moe_parallel_config.use_ep = True
moe_config.moe_parallel_config.dp_size = 2
moe_config.moe_parallel_config.ep_size = 2
moe_config.moe_parallel_config.all2all_backend = all2all_backend
selected_backend, experts_cls = select_unquantized_moe_backend(
moe_config=moe_config
)
assert selected_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM
assert experts_cls is not None
assert experts_cls.__name__ == "TrtLlmBf16ExpertsModular"
@patch(
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsBase._supports_current_device",
return_value=True,
)
@pytest.mark.parametrize("require_decomposed_backend", [False, True])
def test_require_decomposed_backend_rejects_monolithic_kernels(
mock_supports_current_device,
require_decomposed_backend,
):
"""Expert substitution adjusts routes, so the kernel must not route itself."""
from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import (
TrtLlmBf16ExpertsMonolithic,
)
from vllm.model_executor.layers.fused_moe.modular_kernel import (
FusedMoEActivationFormat,
)
moe_config = make_dummy_moe_config()
moe_config.routing_method = RoutingMethodType.Renormalize
moe_config.require_decomposed_backend = require_decomposed_backend
supported, reason = TrtLlmBf16ExpertsMonolithic.is_supported_config(
TrtLlmBf16ExpertsMonolithic,
moe_config,
None,
None,
FusedMoEActivationFormat.Standard,
)
assert supported != require_decomposed_backend
assert (reason is not None and "routes adjusted" in reason) == (
require_decomposed_backend
)
@patch(
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsBase._supports_current_device",
return_value=True,
)
def test_select_cuda_deepep_ht_falls_back_from_trtllm(
mock_supports_current_device,
):
"""Test DeepEP HT avoids the unsupported BF16 TRTLLM modular path."""
with (
patch.object(current_platform, "is_cuda", return_value=True),
patch.object(current_platform, "is_rocm", return_value=False),
patch.object(current_platform, "is_cpu", return_value=False),
patch.object(current_platform, "is_xpu", return_value=False),
patch.object(current_platform, "is_tpu", return_value=False),
patch.object(current_platform, "is_out_of_tree", return_value=False),
patch.object(
current_platform, "is_device_capability_family", return_value=False
),
):
moe_config = make_dummy_moe_config(num_experts=4, num_local_experts=2)
moe_config.moe_backend = "auto"
moe_config.moe_parallel_config.use_ep = True
moe_config.moe_parallel_config.dp_size = 2
moe_config.moe_parallel_config.ep_size = 2
moe_config.moe_parallel_config.all2all_backend = "deepep_high_throughput"
selected_backend, experts_cls = select_unquantized_moe_backend(
moe_config=moe_config
)
assert selected_backend == UnquantizedMoeBackend.TRITON
assert experts_cls is not None
assert experts_cls.__name__ == "TritonExperts"
@patch(
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsBase._supports_current_device",
return_value=True,
)
@pytest.mark.parametrize(
"routing_method",
[RoutingMethodType.Renormalize, RoutingMethodType.MiniMax2],
)
@pytest.mark.parametrize("moe_backend", ["auto", "flashinfer_trtllm"])
def test_select_cuda_flashinfer_trtllm_ag_rs_uses_monolithic(
mock_supports_current_device,
routing_method,
moe_backend,
):
"""Test AG/RS stays on BF16 TRTLLM monolithic when TRTLLM is supported."""
with (
patch.object(current_platform, "is_cuda", return_value=True),
patch.object(current_platform, "is_rocm", return_value=False),
patch.object(current_platform, "is_cpu", return_value=False),
patch.object(current_platform, "is_xpu", return_value=False),
patch.object(current_platform, "is_tpu", return_value=False),
patch.object(current_platform, "is_out_of_tree", return_value=False),
patch.object(
current_platform, "is_device_capability_family", return_value=False
),
):
moe_config = make_dummy_moe_config(num_experts=4, num_local_experts=2)
moe_config.moe_backend = moe_backend
moe_config.routing_method = routing_method
moe_config.moe_parallel_config.use_ep = True
moe_config.moe_parallel_config.dp_size = 2
moe_config.moe_parallel_config.ep_size = 2
moe_config.moe_parallel_config.all2all_backend = "allgather_reducescatter"
selected_backend, experts_cls = select_unquantized_moe_backend(
moe_config=moe_config
)
assert selected_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM
assert experts_cls is not None
assert experts_cls.__name__ == "TrtLlmBf16ExpertsMonolithic"
@patch(
"vllm.utils.flashinfer.has_flashinfer",
return_value=True,
)
@patch(
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsMonolithic.is_supported_config",
return_value=(False, None),
)
@patch(
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsModular.is_supported_config",
return_value=(False, None),
)
@patch(
"vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe.FlashInferExperts.is_supported_config",
return_value=(True, None),
)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="Only supported on NVIDIA platforms."
)
def test_select_cuda_flashinfer_cutlass_backend(
mock_is_supported_cutlass,
mock_is_supported_trtllm_modular,
mock_is_supported_trtllm_monolithic,
mock_has_flashinfer,
):
"""Test CUDA backend selection when FlashInfer TRTLLM is not available
and FlashInfer CUTLASS is available."""
with (
patch.object(current_platform, "is_cuda", return_value=True),
patch.object(current_platform, "is_rocm", return_value=False),
patch.object(current_platform, "is_cpu", return_value=False),
patch.object(current_platform, "is_xpu", return_value=False),
patch.object(current_platform, "is_tpu", return_value=False),
patch.object(current_platform, "is_out_of_tree", return_value=False),
patch.object(current_platform, "has_device_capability", return_value=True),
):
moe_config = make_dummy_moe_config()
# Select FlashInfer CUTLASS explicitly
moe_config.moe_backend = "flashinfer_cutlass"
# CUTLASS requires EP and does not support DP
moe_config.moe_parallel_config.use_ep = True
moe_config.moe_parallel_config.use_dp = False
selected_backend, experts_cls = select_unquantized_moe_backend(
moe_config=moe_config
)
assert selected_backend == UnquantizedMoeBackend.FLASHINFER_CUTLASS
assert experts_cls is not None
@skipif_not_cuda_rocm
def test_select_lora_backend_prefers_triton():
"""LoRA-enabled unquantized MoE should select Triton backend."""
moe_config = make_dummy_moe_config()
moe_config.is_lora_enabled = True
selected_backend, experts_cls = select_unquantized_moe_backend(
moe_config=moe_config
)
assert selected_backend == UnquantizedMoeBackend.TRITON
assert experts_cls is not None
@skipif_not_cuda_rocm
def test_select_lora_explicit_non_triton_backend():
"""LoRA should override explicit non-Triton backend to Triton."""
moe_config = make_dummy_moe_config()
moe_config.is_lora_enabled = True
# Use string from mapping in function map_unquantized_backend()
moe_config.moe_backend = "flashinfer_cutlass"
selected_backend, experts_cls = select_unquantized_moe_backend(
moe_config=moe_config
)
assert selected_backend == UnquantizedMoeBackend.TRITON
assert experts_cls is not None
@skipif_not_cuda_rocm
@pytest.mark.parametrize("is_lora_enabled", [False, True])
def test_select_explicit_triton_backend(is_lora_enabled):
"""Explicit triton backend selection should return Triton."""
moe_config = make_dummy_moe_config()
moe_config.is_lora_enabled = is_lora_enabled
moe_config.moe_backend = "triton"
selected_backend, experts_cls = select_unquantized_moe_backend(
moe_config=moe_config
)
assert selected_backend == UnquantizedMoeBackend.TRITON
assert experts_cls is not None
@skipif_not_cuda_rocm
@pytest.mark.parametrize("moe_backend", ["humming", "marlin"])
def test_quantization_only_backend_falls_back_to_auto(moe_backend):
"""A quantization-only --moe-backend also reaches the unquantized MoE
layers of a quantized model (e.g. a BF16 MTP draft), which must use the
auto-selected backend instead of failing."""
moe_config = make_dummy_moe_config()
moe_config.moe_backend = moe_backend
assert select_unquantized_moe_backend(
moe_config=moe_config
) == select_unquantized_moe_backend(moe_config=make_dummy_moe_config())
-211
View File
@@ -1,211 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for ZenTorch CPU fused MoE dispatch and forward."""
import pytest
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from tests.kernels.allclose_default import get_default_atol, get_default_rtol
from tests.kernels.moe.test_cpu_fused_moe import (
UNALIGNED_INTERMEDIATE_DIM,
_make_moe_config,
_StubMoELayer,
ref_fused_moe,
)
from vllm.model_executor.kernels.linear.zentorch_utils import (
has_zentorch_op,
is_zentorch_moe_config_supported,
)
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEQuantConfig,
biased_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
CPUUnquantizedExperts,
)
from vllm.model_executor.layers.fused_moe.router.cpu_router import _softmax_topk
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
if not current_platform.is_cpu() or not current_platform.is_zen_cpu():
pytest.skip("skipping non-Zen CPU tests", allow_module_level=True)
if not has_zentorch_op(["zentorch_fused_moe"]):
pytest.skip(
"skipping tests: zentorch_fused_moe op not available",
allow_module_level=True,
)
ZENTORCH_ACT = [
MoEActivation.SILU,
MoEActivation.GELU,
MoEActivation.GELU_TANH,
MoEActivation.SWIGLUOAI,
]
USE_BIAS = [False, True]
def _make_layer_and_config(
act: MoEActivation,
*,
hidden_size: int = 128,
intermediate_size: int = 128,
expert_num: int = 8,
topk_num: int = 4,
use_bias: bool = False,
dtype: torch.dtype = torch.bfloat16,
) -> tuple[_StubMoELayer, FusedMoEConfig]:
up_dim = 2 * intermediate_size
w13 = torch.randn((expert_num, up_dim, hidden_size), dtype=dtype) / (
0.5 * hidden_size**0.5
)
w2 = torch.randn((expert_num, hidden_size, intermediate_size), dtype=dtype) / (
0.5 * intermediate_size**0.5
)
w13_bias = None
w2_bias = None
if use_bias:
w13_bias = torch.randn((expert_num, up_dim), dtype=dtype) / (0.5 * up_dim**0.5)
w2_bias = torch.randn((expert_num, hidden_size), dtype=dtype) / (
0.5 * hidden_size**0.5
)
layer = _StubMoELayer(
w13,
w2,
act,
w13_bias,
w2_bias,
)
moe_config = _make_moe_config(
expert_num,
hidden_size,
intermediate_size,
topk_num,
dtype,
act,
use_bias,
)
return layer, moe_config
@pytest.mark.parametrize("act", ZENTORCH_ACT)
def test_zen_cpu_fused_moe_config_supported(act: MoEActivation):
"""CPUUnquantizedExperts is selected when zentorch MoE is supported."""
_, moe_config = _make_layer_and_config(act)
assert is_zentorch_moe_config_supported(moe_config)
supported, reason = CPUUnquantizedExperts.is_supported_config(
CPUUnquantizedExperts,
moe_config,
None,
None,
mk.FusedMoEActivationFormat.Standard,
)
assert supported, reason
def test_zen_cpu_fused_moe_config_supported_unaligned_intermediate():
"""Zentorch accepts shapes the grouped-gemm kernel rejects."""
act = MoEActivation.SILU
moe_config = _make_moe_config(
expert_num=8,
hidden_size=128,
intermediate_size=UNALIGNED_INTERMEDIATE_DIM,
topk_num=4,
dtype=torch.bfloat16,
act=act,
)
assert is_zentorch_moe_config_supported(moe_config)
supported, reason = CPUUnquantizedExperts.is_supported_config(
CPUUnquantizedExperts,
moe_config,
None,
None,
mk.FusedMoEActivationFormat.Standard,
)
assert supported, reason
@pytest.mark.parametrize("act", ZENTORCH_ACT)
@pytest.mark.parametrize("use_bias", USE_BIAS)
def test_zen_cpu_fused_moe_forward(
default_vllm_config,
act: MoEActivation,
use_bias: bool,
):
"""The zentorch forward runs end-to-end and matches the reference."""
set_random_seed(0)
expert_num = 8
hidden_size = 128
batch_size = 64
dtype = torch.bfloat16
topk_num = expert_num // 2
layer, moe_config = _make_layer_and_config(
act,
expert_num=expert_num,
topk_num=topk_num,
use_bias=use_bias,
)
input = torch.randn((batch_size, hidden_size), dtype=dtype) / (
0.5 * hidden_size**0.5
)
router_logits = torch.randn((batch_size, expert_num), dtype=dtype)
quant_config = (
biased_moe_quant_config(layer.w13_bias, layer.w2_bias)
if use_bias
else FusedMoEQuantConfig.make()
)
experts = CPUUnquantizedExperts(moe_config, quant_config)
experts.process_weights_after_loading(layer)
assert experts._use_zentorch is True
w13_before = layer.w13_weight.detach().clone()
w2_before = layer.w2_weight.detach().clone()
topk_weight, topk_ids = _softmax_topk(router_logits, topk_num, False)
output = torch.empty_like(input)
experts.apply(
output=output,
hidden_states=input,
w1=layer.w13_weight,
w2=layer.w2_weight,
topk_weights=topk_weight,
topk_ids=topk_ids,
activation=act,
global_num_experts=expert_num,
expert_map=None,
a1q_scale=None,
a2_scale=None,
workspace13=torch.empty(0),
workspace2=torch.empty(0),
expert_tokens_meta=None,
apply_router_weight_on_input=False,
)
# zentorch needs no prepacking; weights are consumed as loaded.
torch.testing.assert_close(layer.w13_weight, w13_before)
torch.testing.assert_close(layer.w2_weight, w2_before)
ref_output = ref_fused_moe(
input,
layer.w13_weight,
layer.w2_weight,
getattr(layer, "w13_bias", None),
getattr(layer, "w2_bias", None),
topk_weight,
topk_ids,
act,
)
atol, rtol = get_default_atol(output), get_default_rtol(output)
torch.testing.assert_close(output, ref_output, atol=atol, rtol=rtol)
-252
View File
@@ -1,252 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the Zen CPU INT8 MoE experts (ZenCPUExpertsInt8)."""
import pytest
import torch
from tests.kernels.moe.test_cpu_fused_moe import (
_make_moe_config,
quantize_per_channel,
ref_fused_moe_int8,
)
from vllm.model_executor.kernels.linear.zentorch_utils import (
_ZENTORCH_MOE_ACTIVATIONS,
)
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEParallelConfig,
int8_w8a8_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import ZenCPUExpertsInt8
from vllm.model_executor.layers.fused_moe.oracle.int8 import (
Int8MoeBackend,
backend_to_kernel_cls,
)
from vllm.model_executor.layers.fused_moe.router.cpu_router import _softmax_topk
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kInt8DynamicTokenSym,
kInt8StaticChannelSym,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
if not current_platform.is_cpu():
pytest.skip("skipping CPU-only tests", allow_module_level=True)
EXPERT_NUM = 8
HIDDEN_SIZE = 128
INTERMEDIATE_SIZE = 128
BATCH_SIZE = 32
# zentorch names its activations as strings; the experts API is enum-typed.
ZENTORCH_ACT = [act for act in MoEActivation if act.value in _ZENTORCH_MOE_ACTIVATIONS]
@pytest.fixture
def mock_zentorch_fused_moe():
"""Reference zentorch_fused_moe, registered when zentorch is absent."""
if hasattr(torch.ops.zentorch, "zentorch_fused_moe"):
yield None
return
calls: dict[str, dict] = {}
def _fused_moe(
output,
input,
w13,
w2,
w13_bias,
w2_bias,
topk_weights,
topk_id,
skip_weighted,
act,
w13_scales=None,
w2_scales=None,
zentorch_op_name="",
):
calls["fused_moe"] = {
"w13": w13,
"w2": w2,
"w13_bias": w13_bias,
"w2_bias": w2_bias,
"topk_weights": topk_weights,
"topk_id": topk_id,
"skip_weighted": skip_weighted,
"act": act,
"w13_scales": w13_scales,
"w2_scales": w2_scales,
}
output.copy_(
ref_fused_moe_int8(
input,
w13,
w2,
w13_scales.float(),
w2_scales.float(),
None if w13_bias is None else w13_bias.float(),
None if w2_bias is None else w2_bias.float(),
topk_weights,
topk_id,
MoEActivation(act),
)
)
lib_def = torch.library.Library("zentorch", "DEF")
lib_def.define(
"zentorch_fused_moe(Tensor(a!) output, Tensor input, Tensor w13, "
"Tensor w2, Tensor? w13_bias, Tensor? w2_bias, Tensor topk_weights, "
"Tensor topk_id, bool skip_weighted, str act, "
"Tensor? w13_scales=None, Tensor? w2_scales=None, *, "
"str zentorch_op_name='zentorch::zentorch_fused_moe') -> ()"
)
lib_impl = torch.library.Library("zentorch", "IMPL", "CPU")
lib_impl.impl("zentorch_fused_moe", _fused_moe)
yield calls
lib_impl._destroy()
lib_def._destroy()
def _make_int8_moe_weights(use_bias: bool, dtype: torch.dtype = torch.bfloat16):
up_dim = 2 * INTERMEDIATE_SIZE
w13_fp = torch.randn((EXPERT_NUM, up_dim, HIDDEN_SIZE), dtype=dtype) / (
0.5 * HIDDEN_SIZE**0.5
)
w2_fp = torch.randn((EXPERT_NUM, HIDDEN_SIZE, INTERMEDIATE_SIZE), dtype=dtype) / (
0.5 * INTERMEDIATE_SIZE**0.5
)
w13, w13_scale = quantize_per_channel(w13_fp)
w2, w2_scale = quantize_per_channel(w2_fp)
w13_scale = w13_scale.to(torch.bfloat16).float()
w2_scale = w2_scale.to(torch.bfloat16).float()
w13_bias = w2_bias = None
if use_bias:
w13_bias = torch.randn((EXPERT_NUM, up_dim), dtype=dtype) / (0.5 * up_dim**0.5)
w2_bias = torch.randn((EXPERT_NUM, HIDDEN_SIZE), dtype=dtype) / (
0.5 * HIDDEN_SIZE**0.5
)
return w13, w2, w13_scale, w2_scale, w13_bias, w2_bias
def test_zen_int8_first_in_cpu_backend_priority():
"""The Zen experts are preferred over the generic CPU int8 experts."""
assert backend_to_kernel_cls(Int8MoeBackend.CPU)[0] is ZenCPUExpertsInt8
def test_zen_int8_support_predicates(mock_zentorch_fused_moe, monkeypatch):
monkeypatch.setattr(current_platform, "is_zen_cpu", lambda: False)
assert not ZenCPUExpertsInt8._supports_current_device()
monkeypatch.setattr(current_platform, "is_zen_cpu", lambda: True)
assert ZenCPUExpertsInt8._supports_current_device()
assert ZenCPUExpertsInt8._supports_quant_scheme(
kInt8StaticChannelSym, kInt8DynamicTokenSym
)
assert not ZenCPUExpertsInt8._supports_quant_scheme(kInt8StaticChannelSym, None)
for act in ZENTORCH_ACT:
assert ZenCPUExpertsInt8._supports_activation(act)
assert not ZenCPUExpertsInt8._supports_activation(MoEActivation.RELU2)
assert not ZenCPUExpertsInt8._supports_activation(
MoEActivation.SWIGLUOAI_UNINTERLEAVE
)
assert not ZenCPUExpertsInt8._supports_no_act_and_mul()
assert ZenCPUExpertsInt8._supports_parallel_config(
FusedMoEParallelConfig.make_no_parallel()
)
@pytest.mark.parametrize("act", ZENTORCH_ACT)
@pytest.mark.parametrize("use_bias", [False, True])
def test_zen_int8_dispatch_contract(
default_vllm_config,
mock_zentorch_fused_moe,
act: MoEActivation,
use_bias: bool,
):
"""apply() hands zentorch the weights, scales and biases it expects."""
if mock_zentorch_fused_moe is None:
pytest.skip("real zentorch is installed; the recorded call is unavailable")
set_random_seed(0)
topk_num = EXPERT_NUM // 2
w13, w2, w13_scale, w2_scale, w13_bias, w2_bias = _make_int8_moe_weights(use_bias)
moe_config = _make_moe_config(
EXPERT_NUM,
HIDDEN_SIZE,
INTERMEDIATE_SIZE,
topk_num,
torch.bfloat16,
act,
use_bias,
)
# A layer carries per-channel scales as [E, N, 1].
quant_config = int8_w8a8_moe_quant_config(
w1_scale=w13_scale.unsqueeze(-1),
w2_scale=w2_scale.unsqueeze(-1),
a1_scale=None,
a2_scale=None,
w1_bias=w13_bias,
w2_bias=w2_bias,
per_act_token_quant=True,
)
input = torch.randn((BATCH_SIZE, HIDDEN_SIZE), dtype=torch.bfloat16) / (
0.5 * HIDDEN_SIZE**0.5
)
router_logits = torch.randn((BATCH_SIZE, EXPERT_NUM), dtype=torch.bfloat16)
topk_weights, topk_ids = _softmax_topk(router_logits, topk_num, False)
experts = ZenCPUExpertsInt8(moe_config, quant_config)
# zentorch_fused_moe takes no expert_map.
assert not experts.supports_expert_map()
output = torch.empty_like(input)
experts.apply(
output=output,
hidden_states=input,
w1=w13,
w2=w2,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=act,
global_num_experts=EXPERT_NUM,
expert_map=None,
a1q_scale=None,
a2_scale=None,
workspace13=torch.empty(0),
workspace2=torch.empty(0),
expert_tokens_meta=None,
apply_router_weight_on_input=False,
)
call = mock_zentorch_fused_moe["fused_moe"]
assert call["w13_scales"].shape == (EXPERT_NUM, 2 * INTERMEDIATE_SIZE)
assert call["w2_scales"].shape == (EXPERT_NUM, HIDDEN_SIZE)
# f32 scales keep the layer on zentorch's fast routed MoE executor.
assert call["w13_scales"].dtype == torch.float32
assert call["w2_scales"].dtype == torch.float32
assert call["act"] == act.value.lower()
assert call["skip_weighted"] is False
assert call["topk_weights"].dtype == torch.float32
assert call["topk_id"].dtype == torch.int32
if use_bias:
assert call["w13_bias"].shape == (EXPERT_NUM, 2 * INTERMEDIATE_SIZE)
assert call["w2_bias"].shape == (EXPERT_NUM, HIDDEN_SIZE)
else:
assert call["w13_bias"] is None and call["w2_bias"] is None
ref = ref_fused_moe_int8(
input,
w13,
w2,
w13_scale,
w2_scale,
None if w13_bias is None else w13_bias.float(),
None if w2_bias is None else w2_bias.float(),
topk_weights.to(torch.float32),
topk_ids.to(torch.int32),
act,
)
torch.testing.assert_close(output, ref)
-658
View File
@@ -1,658 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import torch
import vllm._custom_ops as ops
from tests.kernels.quant_utils import per_block_cast_to_int8
from tests.kernels.quantization.nvfp4_utils import FLOAT4_E2M1_MAX, FLOAT8_E4M3_MAX
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
BatchedTritonExperts,
NaiveBatchedExperts,
)
from vllm.model_executor.layers.fused_moe.experts.triton_moe import (
TritonExperts,
)
from vllm.model_executor.layers.fused_moe.fused_moe import (
fused_experts,
)
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import moe_unpermute
from vllm.model_executor.layers.fused_moe.prepare_finalize.batched import (
BatchedPrepareAndFinalize,
)
from vllm.model_executor.layers.fused_moe.router.fused_topk_router import fused_topk
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
from vllm.model_executor.layers.quantization.utils.fp8_utils import is_fp8
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
ref_nvfp4_quant,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import per_block_cast_to_fp8
from vllm.utils.math_utils import round_up
DEVICE = current_platform.device_type
def shuffle_weight(w: torch.Tensor) -> torch.Tensor:
"""Fold weights to adjacent locations for Triton MoE / SwiGLU kernel layout."""
shape = w.shape
n = shape[-1]
first = w[..., : n // 2]
second = w[..., n // 2 :]
stacked = torch.stack((first, second), dim=-1)
return stacked.reshape(shape)
def make_dummy_moe_config(
num_experts: int = 1,
num_local_experts: int | None = None,
experts_per_token: int = 1,
hidden_dim: int = 1,
intermediate_size: int = 1,
in_dtype: torch.dtype = torch.bfloat16,
max_num_tokens: int = 512,
activation: MoEActivation = MoEActivation.SILU,
) -> FusedMoEConfig:
"""This is a dummy config for the mk constructor interface
as most kernels like DeepGEMM, CUTLASSFp4, Triton, MARLIN
do not actually use this config.
CUTLASSFp8 needs to set some params for workshapes.
"""
return FusedMoEConfig(
num_experts=num_experts,
experts_per_token=experts_per_token,
hidden_dim=hidden_dim,
intermediate_size=intermediate_size,
num_local_experts=num_local_experts
if num_local_experts is not None
else num_experts,
num_logical_experts=num_experts,
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
activation=activation,
in_dtype=in_dtype,
device=DEVICE,
routing_method=RoutingMethodType.TopK,
max_num_tokens=max_num_tokens,
)
def triton_moe(
a: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weight: torch.Tensor,
topk_ids: torch.Tensor,
w1_scale: torch.Tensor | None = None,
w2_scale: torch.Tensor | None = None,
a1_scale: torch.Tensor | None = None,
a2_scale: torch.Tensor | None = None,
quant_dtype: torch.dtype | None = None,
per_act_token_quant=False,
block_shape: list[int] | None = None,
) -> torch.Tensor:
quant_config = FusedMoEQuantConfig.make(
quant_dtype,
per_act_token_quant=per_act_token_quant,
block_shape=block_shape,
w1_scale=w1_scale,
w2_scale=w2_scale,
a1_scale=a1_scale,
a2_scale=a2_scale,
)
return fused_experts(a, w1, w2, topk_weight, topk_ids, quant_config=quant_config)
def batched_moe(
a: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weight: torch.Tensor,
topk_ids: torch.Tensor,
w1_scale: torch.Tensor | None = None,
w2_scale: torch.Tensor | None = None,
a1_scale: torch.Tensor | None = None,
a2_scale: torch.Tensor | None = None,
quant_dtype: torch.dtype | None = None,
per_act_token_quant: bool = False,
block_shape: list[int] | None = None,
moe_config: FusedMoEConfig | None = None,
) -> torch.Tensor:
max_num_tokens = round_up(a.shape[0], 64)
quant_config = FusedMoEQuantConfig.make(
quant_dtype,
per_act_token_quant=per_act_token_quant,
block_shape=block_shape,
w1_scale=w1_scale,
w2_scale=w2_scale,
a1_scale=a1_scale,
a2_scale=a2_scale,
)
if moe_config is None:
moe_config = make_dummy_moe_config()
fused_experts = FusedMoEKernel(
BatchedPrepareAndFinalize(
max_num_tokens, num_dispatchers=1, num_local_experts=w1.shape[0], rank=0
),
BatchedTritonExperts(
max_num_tokens=max_num_tokens,
num_dispatchers=1,
quant_config=quant_config,
moe_config=moe_config,
),
)
return fused_experts.apply(
a,
w1,
w2,
topk_weight,
topk_ids,
global_num_experts=w1.shape[0],
activation=moe_config.activation,
apply_router_weight_on_input=False,
expert_map=None,
)
def naive_batched_moe(
a: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weight: torch.Tensor,
topk_ids: torch.Tensor,
w1_scale: torch.Tensor | None = None,
w2_scale: torch.Tensor | None = None,
a1_scale: torch.Tensor | None = None,
a2_scale: torch.Tensor | None = None,
quant_dtype: torch.dtype | None = None,
per_act_token_quant: bool = False,
block_shape: list[int] | None = None,
) -> torch.Tensor:
max_num_tokens = round_up(a.shape[0], 64)
quant_config = FusedMoEQuantConfig.make(
quant_dtype,
per_act_token_quant=per_act_token_quant,
block_shape=block_shape,
w1_scale=w1_scale,
w2_scale=w2_scale,
a1_scale=a1_scale,
a2_scale=a2_scale,
)
moe_config = make_dummy_moe_config()
fused_experts = FusedMoEKernel(
BatchedPrepareAndFinalize(
max_num_tokens, num_dispatchers=1, num_local_experts=w1.shape[0], rank=0
),
NaiveBatchedExperts(
max_num_tokens=max_num_tokens,
num_dispatchers=1,
quant_config=quant_config,
moe_config=moe_config,
),
)
return fused_experts.apply(
a,
w1,
w2,
topk_weight,
topk_ids,
global_num_experts=w1.shape[0],
activation=moe_config.activation,
apply_router_weight_on_input=False,
expert_map=None,
)
def chunk_scales(
scales: torch.Tensor | None, start: int, end: int
) -> torch.Tensor | None:
if scales is not None:
if scales.numel() == 1:
return scales
else:
return scales[start:end]
return None
def make_quantized_test_activations(
E: int,
m: int,
k: int,
in_dtype: torch.dtype,
quant_dtype: torch.dtype | None = None,
block_shape: list[int] | None = None,
per_act_token_quant: bool = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
a = torch.randn((E, m, k), device=DEVICE, dtype=in_dtype) / 10
a_q = a
a_scale = None
if quant_dtype is not None:
assert quant_dtype == torch.float8_e4m3fn or quant_dtype == torch.int8, (
"only fp8/int8 supported"
)
a_q = torch.zeros_like(a, dtype=quant_dtype)
a_scale_l = [None] * E
for e in range(E):
a_q[e], a_scale_l[e] = moe_kernel_quantize_input(
a[e], None, quant_dtype, per_act_token_quant, block_shape
)
a_scale = torch.stack(a_scale_l)
if not per_act_token_quant and block_shape is None:
a_scale = a_scale.view(E, 1, 1)
return a, a_q, a_scale
def moe_quantize_weights_2d(
w: torch.Tensor,
w_s: torch.Tensor | None,
quant_dtype: torch.dtype | str | None,
per_token_quant: bool,
block_shape: list[int] | None,
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None]:
is_fp8_dtype = is_fp8(quant_dtype)
assert is_fp8_dtype or quant_dtype == torch.int8 or quant_dtype == "nvfp4", (
"only fp8/int8/nvfp4 supported"
)
w_gs = None
if block_shape is not None:
assert not per_token_quant
if quant_dtype == torch.int8:
w, w_s = per_block_cast_to_int8(w, block_shape)
elif is_fp8_dtype:
w, w_s = per_block_cast_to_fp8(w, block_shape)
elif quant_dtype == "nvfp4":
raise RuntimeError("blocked quantization not supported for nvfp4")
else:
raise RuntimeError(f"Unsupported quant type {quant_dtype}")
else:
if quant_dtype == torch.int8:
w, w_s = ops.scaled_int8_quant(
w, w_s, use_per_token_if_dynamic=per_token_quant
)
elif is_fp8_dtype:
w, w_s = ops.scaled_fp8_quant(
w, w_s, use_per_token_if_dynamic=per_token_quant
)
elif quant_dtype == "nvfp4":
assert not per_token_quant
w_amax = torch.abs(w).max().to(torch.float32)
w_gs = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / w_amax
if current_platform.is_rocm():
w, w_s = _scaled_fp4_quant_emulated(w, w_gs)
else:
w, w_s = ops.scaled_fp4_quant(w, w_gs)
else:
raise RuntimeError(f"Unsupported quant type {quant_dtype}")
return w, w_s, w_gs
def _pack_e2m1_fp4(fp4_values: torch.Tensor) -> torch.Tensor:
assert fp4_values.shape[-1] % 2 == 0
abs_values = fp4_values.abs()
codes = torch.empty_like(abs_values, dtype=torch.uint8)
for code, value in enumerate((0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0)):
codes[abs_values == value] = code
codes = codes | ((fp4_values < 0).to(torch.uint8) << 3)
return codes[..., 0::2] | (codes[..., 1::2] << 4)
def _scaled_fp4_quant_emulated(
w: torch.Tensor, w_gs: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
fp4_values, w_s = ref_nvfp4_quant(w, w_gs, block_size=16)
return _pack_e2m1_fp4(fp4_values), w_s.to(torch.float8_e4m3fn)
def moe_quantize_weights(
w: torch.Tensor,
w_s: torch.Tensor | None,
quant_dtype: torch.dtype | str | None,
per_token_quant: bool,
block_shape: list[int] | None,
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None]:
assert w.dim() == 3
e, rows, cols = w.shape
w_l = [None] * e
w_s_l = [None] * e
w_gs_l = [None] * e
for idx in range(e):
w_l[idx], w_s_l[idx], w_gs_l[idx] = moe_quantize_weights_2d(
w[idx], None, quant_dtype, per_token_quant, block_shape
)
w = torch.stack(w_l)
w_s = torch.stack(w_s_l)
w_gs = torch.stack(w_gs_l) if e > 0 and w_gs_l[0] is not None else None
if w_s.ndim == 2:
assert w_s.shape[-1] == 1
w_s = w_s.view(-1, 1, 1)
if block_shape is not None:
block_n, block_k = block_shape
n_tiles = (rows + block_n - 1) // block_n
k_tiles = (cols + block_k - 1) // block_k
assert w_s.shape == (e, n_tiles, k_tiles)
return w, w_s, w_gs
def make_test_weight(
e: int,
rows: int,
cols: int,
in_dtype: torch.dtype = torch.bfloat16,
quant_dtype: torch.dtype | str | None = None,
block_shape: list[int] | None = None,
per_out_ch_quant: bool = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None, torch.Tensor | None]:
w_16 = torch.randn((e, rows, cols), device=DEVICE, dtype=in_dtype) / 15
if quant_dtype is not None:
w, w_s, w_gs = moe_quantize_weights(
w_16, None, quant_dtype, per_out_ch_quant, block_shape
)
else:
w = w_16
w_s = None
w_gs = None
return w_16, w, w_s, w_gs
def make_test_weights(
e: int,
n: int,
k: int,
in_dtype: torch.dtype = torch.bfloat16,
quant_dtype: torch.dtype | str | None = None,
block_shape: list[int] | None = None,
per_out_ch_quant: bool = False,
make_gate: bool = True,
) -> tuple[
tuple[torch.Tensor, torch.Tensor, torch.Tensor | None, torch.Tensor | None],
tuple[torch.Tensor, torch.Tensor, torch.Tensor | None, torch.Tensor | None],
]:
return (
make_test_weight(
e,
(2 if make_gate else 1) * n,
k,
in_dtype,
quant_dtype,
block_shape,
per_out_ch_quant,
),
make_test_weight(e, k, n, in_dtype, quant_dtype, block_shape, per_out_ch_quant),
)
def per_token_cast_to_fp8(
x: torch.Tensor, block_size: int = 128
) -> tuple[torch.Tensor, torch.Tensor]:
assert x.dim() == 2
m, n = x.shape
pad_size = (block_size - (n % block_size)) % block_size
x = torch.nn.functional.pad(x, (0, pad_size), value=0) if pad_size > 0 else x
x_view = x.view(m, -1, block_size)
x_amax = x_view.abs().float().amax(dim=2).view(m, -1).clamp(1e-4)
fp8_data = (x_view * (448.0 / x_amax.unsqueeze(2))).to(torch.float8_e4m3fn)
return fp8_data.view(m, n + pad_size)[:, :n], (x_amax / 448.0).view(m, -1)
def make_test_quant_config(
e: int,
n: int,
k: int,
in_dtype: torch.dtype,
quant_dtype: torch.dtype | str | None = None,
per_act_token_quant: bool = False,
block_shape: list[int] | None = None,
make_gate: bool = True,
is_scale_swizzled: bool = True,
) -> tuple[torch.Tensor, torch.Tensor, FusedMoEQuantConfig]:
(_, w1, w1_s, w1_gs), (_, w2, w2_s, w2_gs) = make_test_weights(
e,
n,
k,
in_dtype,
quant_dtype,
per_out_ch_quant=per_act_token_quant,
block_shape=block_shape,
make_gate=make_gate,
)
# Hacky/trivial scales for nvfp4.
a1_gscale: torch.Tensor | None = None
a2_gscale: torch.Tensor | None = None
if quant_dtype == "nvfp4":
a1_gscale = torch.ones((e,), device=DEVICE, dtype=torch.float32)
a2_gscale = torch.ones((e,), device=DEVICE, dtype=torch.float32)
a1_scale = a1_gscale
a2_scale = a2_gscale
else:
a1_scale = None
a2_scale = None
return (
w1,
w2,
FusedMoEQuantConfig.make(
quant_dtype,
per_act_token_quant=per_act_token_quant,
block_shape=block_shape,
w1_scale=w1_s,
w2_scale=w2_s,
a1_gscale=a1_gscale,
a2_gscale=a2_gscale,
a1_scale=a1_scale,
a2_scale=a2_scale,
# TODO: make sure this is handled properly
g1_alphas=(1 / w1_gs) if w1_gs is not None else None,
g2_alphas=(1 / w2_gs) if w2_gs is not None else None,
is_scale_swizzled=is_scale_swizzled,
),
)
def fused_moe(
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
score: torch.Tensor,
topk: int,
renormalize: bool = False,
quant_config: FusedMoEQuantConfig | None = None,
global_num_experts: int = -1,
expert_map: torch.Tensor | None = None,
) -> torch.Tensor:
topk_weights, topk_ids, _ = fused_topk(
hidden_states, score.float(), topk, renormalize
)
return fused_experts(
hidden_states,
w1,
w2,
topk_weights,
topk_ids,
global_num_experts=global_num_experts,
expert_map=expert_map,
quant_config=quant_config,
)
class BaselineMM(torch.nn.Module):
def __init__(
self,
b: torch.Tensor,
out_dtype: torch.dtype,
):
super().__init__()
self.b = torch.nn.Parameter(b.to(dtype=torch.float32))
self.out_dtype = out_dtype
def forward(self, a: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor | None]:
return torch.mm(a.to(dtype=torch.float32), self.b).to(self.out_dtype), None
class BaselineSiluAndMul(torch.nn.Module):
def __init__(self):
super().__init__()
def forward(self, x: torch.Tensor) -> torch.Tensor:
d = x.shape[-1] // 2
return torch.nn.functional.silu(x[..., :d]) * x[..., d:]
class TestMLP(torch.nn.Module):
def __init__(
self,
w1: torch.Tensor,
w2: torch.Tensor,
out_dtype: torch.dtype,
):
super().__init__()
self.gate_up_proj = BaselineMM(w1, out_dtype)
self.down_proj = BaselineMM(w2, out_dtype)
self.act_fn = BaselineSiluAndMul()
def forward(self, x):
x, _ = self.gate_up_proj(x)
x = self.act_fn(x)
x, _ = self.down_proj(x)
return x
def modular_triton_fused_moe(
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
) -> FusedMoEKernel:
return FusedMoEKernel(
maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
use_monolithic=False,
),
TritonExperts(moe_config, quant_config),
)
def check_accuracy(a, b, atol, rtol, percent):
"""Allow a mismatch percentage of 1 - percent."""
if torch.any(torch.isnan(a)):
raise Exception("NaN in reference output")
if torch.any(torch.isnan(b)):
raise Exception("NaN in actual output")
if torch.any(torch.isinf(a)):
raise Exception("Inf in reference output")
if torch.any(torch.isinf(b)):
raise Exception("Inf in actual output")
assert a.shape == b.shape, f"Shape mismatch: {a.shape} vs {b.shape}"
left = torch.abs(a - b)
right = atol + rtol * torch.abs(b)
count = torch.sum(left > right)
mismatch_percent = count / a.numel()
if mismatch_percent > 1 - percent:
raise Exception(
f"Mismatch percentage is {mismatch_percent:.4f} for rtol {rtol} "
f"(threshold: {1 - percent:.4f})"
)
def check_deferred_moe_finalize(
moe_config: FusedMoEConfig,
run: Callable[[], torch.Tensor | UnfinalizedMoEOutput],
router_weights: torch.Tensor | None = None,
chunked: bool = False,
) -> None:
"""Check a kernel that defers its finalize against the finalize it skips.
``run`` calls the kernel on fixed inputs, first as built and then with
``moe_config`` asking to defer, which ``should_defer_moe_finalize`` must
report truthfully. A deferred output reduced by ``moe_unpermute``, the
TRT-LLM finalize kernel, must give the kernel's own finalized output bit for
bit, and modular experts hand the router's weights back as-is. A call the
experts split across kernel launches finalizes instead.
"""
# The deferred output views the router's buffer, so compare with a copy.
expected_weights = None if router_weights is None else router_weights.clone()
finalized = run()
assert isinstance(finalized, torch.Tensor)
moe_config.defer_moe_finalize()
output = run()
assert moe_config.should_defer_moe_finalize(finalized.shape[0]) != chunked
if chunked:
torch.testing.assert_close(output, finalized, atol=0, rtol=0)
return
assert isinstance(output, UnfinalizedMoEOutput)
if expected_weights is not None:
torch.testing.assert_close(
output.expert_weights, expected_weights, atol=0, rtol=0
)
reference = torch.empty_like(finalized)
moe_unpermute(
reference,
output.gemm2_permuted,
output.expert_weights.float(),
output.expanded_idx_to_permuted_idx,
# The kernel reads its valid-row count through this pointer either way.
expert_first_token_offset=output.gemm2_permuted.new_full(
(1,), output.gemm2_permuted.shape[0], dtype=torch.int64
),
)
torch.testing.assert_close(reference, finalized, atol=0, rtol=0)
def mxfp4_w_layouts(mx_axis: int, num_warps: int = 8):
"""Weight/scale layouts for mxfp4 MoE, as (layout, opts) pairs.
triton_kernels 3.8 returns layout instances; earlier versions return a
(layout, opts) tuple.
"""
from triton_kernels.tensor_details import layout
from vllm.utils.import_utils import get_triton_kernels_version
if get_triton_kernels_version() == "3.8":
w = layout.make_default_matmul_mxfp4_w_layout(mx_axis=mx_axis)
s = layout.make_default_matmul_mxfp4_w_scale_layout(
mx_axis=mx_axis, num_warps=num_warps
)
return w, {}, s, {}
w, w_opts = layout.make_default_matmul_mxfp4_w_layout(mx_axis=mx_axis)
s, s_opts = layout.make_default_matmul_mxfp4_w_scale_layout(
mx_axis=mx_axis, num_warps=num_warps
)
return w, w_opts, s, s_opts