[Model] Add MiniMax M3 support (#45381)

Signed-off-by: youkaichao <youkaichao@gmail.com>
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Signed-off-by: functionstackx <47992694+functionstackx@users.noreply.github.com>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: functionstackx <47992694+functionstackx@users.noreply.github.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
youkaichao
2026-06-16 01:01:25 +08:00
committed by GitHub
co-authored by OpenAI Codex Isotr0py Thien Tran Bugen Zhao Jee Jee Li Roger Wang functionstackx Yongye Zhu Jee Jee Li
parent a3195fab7b
commit 0a1c5034f5
108 changed files with 14734 additions and 311 deletions
@@ -93,6 +93,9 @@ from vllm.model_executor.kernels.linear.mxfp8.flashinfer import (
from vllm.model_executor.kernels.linear.mxfp8.marlin import (
MarlinMxfp8LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
RocmDotScaledMxfp8LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp8.xpu import (
XPUMxFp8LinearKernel,
)
@@ -383,6 +386,9 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = {
EmulationMxfp8LinearKernel,
],
PlatformEnum.ROCM: [
# Native CDNA4 (gfx950) MX linear; is_supported() gates to gfx95x and
# falls through to BF16 emulation (hipBLASLt) elsewhere / on regression.
RocmDotScaledMxfp8LinearKernel,
EmulationMxfp8LinearKernel,
],
PlatformEnum.XPU: [
@@ -33,6 +33,17 @@ class EmulationMxfp8LinearKernel(Mxfp8LinearKernel):
weight_scale = layer.weight_scale.data[:N, :scale_k].contiguous()
# Dequantize MXFP8 -> BF16 ONCE here, at load time, so apply_weights runs
# a plain BF16 linear with no per-step dequant -- i.e. run as if from a
# BF16 checkpoint. The 1-byte MXFP8 weight is replaced by BF16 (2x its
# size, but linear weights are small vs the MoE experts); the tiny E8M0
# scale is kept for the dtype/ndim asserts but is otherwise unused.
# Opt out (VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD=0) to keep the MXFP8
# weight and dequant per-step in apply_weights instead.
import vllm.envs as envs
if envs.VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD:
weight = dequant_mxfp8_to_bf16(weight.contiguous(), weight_scale)
layer.weight = Parameter(weight.contiguous(), requires_grad=False)
layer.weight_scale = Parameter(weight_scale, requires_grad=False)
@@ -42,6 +53,17 @@ class EmulationMxfp8LinearKernel(Mxfp8LinearKernel):
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
weight = layer.weight
# Load-time dequant path: weights are already BF16/FP16 (>= 2-byte), so
# run a plain linear -- no per-step dequant. (MXFP8 weights are 1-byte.)
if weight.element_size() >= 2:
# F.linear requires x and weight share a dtype; .to() is a no-op when
# they already match (e.g. both BF16).
output = torch.nn.functional.linear(x, weight.to(x.dtype), bias)
return output.to(x.dtype)
# Fallback: weights still in MXFP8 -- dequant on the fly (other archs /
# if a future caller skips the load-time conversion above).
weight_scale = layer.weight_scale
if weight_scale.dtype != MXFP8_SCALE_DTYPE:
raise ValueError(
@@ -55,6 +77,8 @@ class EmulationMxfp8LinearKernel(Mxfp8LinearKernel):
f"Ensure process_weights_after_loading was called."
)
weight_bf16 = dequant_mxfp8_to_bf16(layer.weight, weight_scale)
# Cast to x's dtype: dequant yields BF16, but F.linear needs both operands
# to match (e.g. an FP16 model). No-op when x is already BF16.
weight_bf16 = dequant_mxfp8_to_bf16(weight, weight_scale).to(x.dtype)
output = torch.nn.functional.linear(x, weight_bf16, bias)
return output.to(x.dtype)
@@ -56,8 +56,6 @@ class FlashInferCutlassMxfp8LinearKernel(Mxfp8LinearKernel):
input_shape = x.shape
input_2d = x.view(-1, K)
M_orig = input_2d.shape[0]
min_dim = 128
assert min_dim <= K, (
@@ -72,11 +70,6 @@ class FlashInferCutlassMxfp8LinearKernel(Mxfp8LinearKernel):
f"out_features is too small for mm_mxfp8."
)
M_padded = ((M_orig + min_dim - 1) // min_dim) * min_dim
if M_padded != M_orig:
pad_rows = M_padded - M_orig
input_2d = torch.nn.functional.pad(input_2d, (0, 0, 0, pad_rows))
input_mxfp8, input_scale = mxfp8_e4m3_quantize(
input_2d, is_sf_swizzled_layout=True
)
@@ -93,9 +86,6 @@ class FlashInferCutlassMxfp8LinearKernel(Mxfp8LinearKernel):
backend="cutlass",
)
if M_padded != M_orig:
output = output[:M_orig, :]
if bias is not None:
output = output + bias
@@ -0,0 +1,171 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Native MXFP8 linear GEMM for AMD CDNA4 (gfx950) via Triton ``tl.dot_scaled``.
Consumes the FP8 E4M3 weights + E8M0 block scales directly (no dequant-to-BF16);
activations are MXFP8-quantized per token. Uses the CDNA4 hardware microscaling
matrix cores. Falls back (via the kernel selector) to the BF16
``EmulationMxfp8LinearKernel`` on archs without native MX or for shapes with
``K % 128 != 0``.
"""
import torch
from torch.nn.parameter import Parameter
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
MXFP8_BLOCK_SIZE,
MXFP8_SCALE_DTYPE,
dequant_mxfp8_to_bf16,
mxfp8_e4m3_quantize,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
@triton.jit
def _mxfp8_linear_kernel(
x_ptr,
xs_ptr,
w_ptr,
ws_ptr,
out_ptr,
M,
N,
K,
stride_xm,
stride_xk,
stride_xsm,
stride_xsk,
stride_wn,
stride_wk,
stride_wsn,
stride_wsk,
stride_om,
stride_on,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
BLOCK_K: tl.constexpr,
):
pid_m = tl.program_id(0)
pid_n = tl.program_id(1)
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
offs_k = tl.arange(0, BLOCK_K)
offs_sk = tl.arange(0, BLOCK_K // 32)
m_mask = offs_m < M
n_mask = offs_n < N
x_ptrs = x_ptr + offs_m[:, None] * stride_xm + offs_k[None, :] * stride_xk
xs_ptrs = xs_ptr + offs_m[:, None] * stride_xsm + offs_sk[None, :] * stride_xsk
w_ptrs = w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :] * stride_wk
ws_ptrs = ws_ptr + offs_n[:, None] * stride_wsn + offs_sk[None, :] * stride_wsk
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
for _ in range(0, tl.cdiv(K, BLOCK_K)):
x = tl.load(x_ptrs, mask=m_mask[:, None], other=0.0)
w = tl.load(w_ptrs, mask=n_mask[:, None], other=0.0)
xs = tl.load(xs_ptrs, mask=m_mask[:, None], other=0)
ws = tl.load(ws_ptrs, mask=n_mask[:, None], other=0)
acc += tl.dot_scaled(x, xs, "e4m3", w.T, ws, "e4m3")
x_ptrs += BLOCK_K * stride_xk
w_ptrs += BLOCK_K * stride_wk
xs_ptrs += (BLOCK_K // 32) * stride_xsk
ws_ptrs += (BLOCK_K // 32) * stride_wsk
o_ptrs = out_ptr + offs_m[:, None] * stride_om + offs_n[None, :] * stride_on
tl.store(
o_ptrs, acc.to(out_ptr.dtype.element_ty), mask=m_mask[:, None] & n_mask[None, :]
)
def _mxfp8_dot_scaled_linear(
x: torch.Tensor, # [M, K] bf16/fp16
w: torch.Tensor, # [N, K] fp8 e4m3
w_scale: torch.Tensor, # [N, K//32] uint8 (E8M0)
) -> torch.Tensor:
M, K = x.shape
N = w.shape[0]
x_q, x_scale = mxfp8_e4m3_quantize(x)
out = torch.empty((M, N), dtype=x.dtype, device=x.device)
BLOCK_M, BLOCK_N, BLOCK_K = 64, 128, 128
grid = (triton.cdiv(M, BLOCK_M), triton.cdiv(N, BLOCK_N))
_mxfp8_linear_kernel[grid](
x_q,
x_scale,
w,
w_scale,
out,
M,
N,
K,
x_q.stride(0),
x_q.stride(1),
x_scale.stride(0),
x_scale.stride(1),
w.stride(0),
w.stride(1),
w_scale.stride(0),
w_scale.stride(1),
out.stride(0),
out.stride(1),
BLOCK_M=BLOCK_M,
BLOCK_N=BLOCK_N,
BLOCK_K=BLOCK_K,
num_warps=8,
)
return out
class RocmDotScaledMxfp8LinearKernel(Mxfp8LinearKernel):
"""Native CDNA4 (gfx950) MXFP8 linear via Triton ``tl.dot_scaled``."""
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_rocm():
return False, "not ROCm"
# supports_mx() == gfx95x (CDNA4 native microscaling hardware). On other
# archs dot_scaled would upcast to BF16, so the kernel selector falls
# through to the BF16 emulation (hipBLASLt) path instead.
if not current_platform.supports_mx():
return False, "native MX requires CDNA4 (gfx95x)"
return True, None
@classmethod
def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]:
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
weight = layer.weight.data # [N, K] fp8
N, K = weight.shape
scale_k = K // MXFP8_BLOCK_SIZE
weight_scale = layer.weight_scale.data[:N, :scale_k].contiguous()
layer.weight = Parameter(weight.contiguous(), requires_grad=False)
layer.weight_scale = Parameter(weight_scale, requires_grad=False)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
if layer.weight_scale.dtype != MXFP8_SCALE_DTYPE:
raise ValueError(
f"Expected {MXFP8_SCALE_DTYPE} weight_scale, got "
f"{layer.weight_scale.dtype}."
)
out_shape = (*x.shape[:-1], layer.weight.shape[0])
x2d = x.reshape(-1, x.shape[-1])
if x2d.shape[-1] % 128 == 0:
out = _mxfp8_dot_scaled_linear(x2d, layer.weight, layer.weight_scale)
else:
# dot_scaled tiling needs K % 128 == 0; dequantize fallback otherwise.
w_bf16 = dequant_mxfp8_to_bf16(layer.weight, layer.weight_scale)
out = torch.nn.functional.linear(x2d, w_bf16).to(x.dtype)
out = out.reshape(out_shape)
if bias is not None:
out = out + bias
return out
+22 -5
View File
@@ -158,17 +158,28 @@ class SiluAndMulWithClamp(CustomOp):
Computes:
gate = clamp(x[..., :d], max=swiglu_limit)
up = clamp(x[..., d:], min=-swiglu_limit, max=swiglu_limit)
out = silu(gate) * up
where d = x.shape[-1] // 2.
out = gate * sigmoid(alpha * gate) * (up + beta)
where d = x.shape[-1] // 2. The defaults alpha=1.0, beta=0.0 reduce this to
``silu(gate) * up``; SwiGLU-OAI style models pass alpha (sigmoid scale) and
beta=1.0 (up bias).
Shapes:
x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d)
return: (num_tokens, d) or (batch_size, seq_len, d)
"""
def __init__(self, swiglu_limit: float, *, compile_native: bool = True):
def __init__(
self,
swiglu_limit: float,
alpha: float = 1.0,
beta: float = 0.0,
*,
compile_native: bool = True,
):
super().__init__(compile_native=compile_native)
self.swiglu_limit = float(swiglu_limit)
self.alpha = float(alpha)
self.beta = float(beta)
if current_platform.is_rocm() or current_platform.is_xpu():
self._forward_method = self.forward_native
elif current_platform.is_cuda_alike():
@@ -180,18 +191,24 @@ class SiluAndMulWithClamp(CustomOp):
d = x.shape[-1] // 2
gate = torch.clamp(x[..., :d], max=self.swiglu_limit)
up = torch.clamp(x[..., d:], min=-self.swiglu_limit, max=self.swiglu_limit)
return F.silu(gate) * up
return gate * torch.sigmoid(self.alpha * gate) * (up + self.beta)
def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
d = x.shape[-1] // 2
output_shape = x.shape[:-1] + (d,)
out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
self.op(out, x, self.swiglu_limit)
self.op(out, x, self.swiglu_limit, self.alpha, self.beta)
return out
def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
return self.forward_native(x)
def extra_repr(self) -> str:
return (
f"swiglu_limit={self.swiglu_limit!r}, "
f"alpha={self.alpha!r}, beta={self.beta!r}"
)
# --8<-- [start:mul_and_silu]
@CustomOp.register("mul_and_silu")
@@ -7,6 +7,7 @@ import torch
import torch.nn as nn
import vllm.envs as envs
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CacheConfig, get_current_vllm_config
from vllm.config.vllm import VllmConfig
from vllm.forward_context import ForwardContext, get_forward_context
@@ -730,6 +731,7 @@ direct_register_custom_op(
)
@eager_break_during_capture
@maybe_transfer_kv_layer
def unified_attention_with_output(
query: torch.Tensor,
@@ -0,0 +1,143 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Manual fusion of tensor-parallel all-reduce with the following GemmaRMSNorm.
Under tensor parallelism a ``RowParallelLinear`` (e.g. attention ``o_proj``)
produces a per-rank partial sum that is all-reduced, and the result is then fed
into a ``GemmaRMSNorm`` that adds the residual and normalizes. flashinfer ships a
kernel that fuses all-reduce + residual-add + RMSNorm into a single launch; this
helper drives it directly (no torch.compile pass) for models that run eager.
Scope: attention output only, no quantization. When the flashinfer fast path is
not applicable (TP==1, flashinfer/NVSwitch unavailable, unsupported dtype, or an
oversize batch) it falls back to ``all_reduce`` + ``GemmaRMSNorm``, which is
numerically identical to the unfused model path.
"""
import torch
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce
from vllm.distributed.parallel_state import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
get_tp_group,
)
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
MiB = 1024 * 1024
# flashinfer fused all-reduce + RMSNorm is wired as a registered custom op in
# allreduce_rms_fusion; both that op and the workspace helpers only exist when
# flashinfer.comm.allreduce_fusion is importable.
try:
from vllm.compilation.passes.fusion.allreduce_rms_fusion import (
flashinfer_trtllm_fused_allreduce_norm,
)
from vllm.distributed.device_communicators.flashinfer_all_reduce import (
flashinfer_comm,
get_fi_ar_workspace,
)
_AR_RESIDUAL_RMS_NORM = (
flashinfer_comm.AllReduceFusionPattern.kARResidualRMSNorm
if flashinfer_comm is not None
else None
)
except ImportError:
flashinfer_trtllm_fused_allreduce_norm = None # type: ignore[assignment]
get_fi_ar_workspace = None # type: ignore[assignment]
_AR_RESIDUAL_RMS_NORM = None
_FI_SUPPORTED_DTYPES = (torch.bfloat16, torch.float16)
def _max_token_num(tp_size: int, hidden_size: int, dtype: torch.dtype) -> int | None:
"""Workspace token budget for flashinfer fused all-reduce, or None if the
current world size / device is unsupported. Mirrors ``FlashInferAllReduce``."""
from vllm.config.compilation import PassConfig
max_size_mb = PassConfig.default_fi_allreduce_fusion_max_size_mb().get(tp_size)
if not max_size_mb:
return None
element_size = torch.tensor([], dtype=dtype).element_size()
return int(max_size_mb * MiB) // (hidden_size * element_size)
def _can_use_flashinfer(hidden_states: torch.Tensor, tp_size: int) -> tuple[bool, int]:
"""Whether the flashinfer fused path applies; returns (ok, max_token_num)."""
if (
flashinfer_trtllm_fused_allreduce_norm is None
or get_fi_ar_workspace is None
or _AR_RESIDUAL_RMS_NORM is None
):
return False, 0
if (
not hidden_states.is_cuda
or hidden_states.dim() != 2
or not hidden_states.is_contiguous()
or hidden_states.dtype not in _FI_SUPPORTED_DTYPES
):
return False, 0
num_tokens, hidden_size = hidden_states.shape
max_token_num = _max_token_num(tp_size, hidden_size, hidden_states.dtype)
if max_token_num is None or num_tokens > max_token_num:
return False, 0
# Lazily create / fetch the (globally cached) workspace; returns None on
# GPUs without NVSwitch, in which case we fall back gracefully.
workspace = get_fi_ar_workspace(
world_size=tp_size,
rank=get_tensor_model_parallel_rank(),
max_token_num=max_token_num,
hidden_dim=hidden_size,
dtype=hidden_states.dtype,
group=get_tp_group().device_group,
)
if workspace is None:
return False, 0
return True, max_token_num
def fused_allreduce_gemma_rms_norm(
hidden_states: torch.Tensor,
residual: torch.Tensor,
norm: GemmaRMSNorm,
) -> tuple[torch.Tensor, torch.Tensor]:
"""All-reduce ``hidden_states`` + add ``residual`` + GemmaRMSNorm, fused.
``hidden_states`` is the per-rank *partial* (un-reduced) output of a
row-parallel linear; ``norm`` is the GemmaRMSNorm applied right after.
Returns ``(normed_output, new_residual)``, equivalent to
``norm(all_reduce(hidden_states), residual)``.
"""
tp_size = get_tensor_model_parallel_world_size()
if tp_size == 1:
# No all-reduce needed; identical to the unfused path.
return norm(hidden_states, residual)
ok, max_token_num = _can_use_flashinfer(hidden_states, tp_size)
if ok:
norm_out = torch.empty_like(hidden_states)
# With norm_out provided, the kernel writes the new residual
# (all_reduce(hidden_states) + residual) into the hidden_states buffer
# and the normalized result into norm_out, leaving `residual` untouched.
flashinfer_trtllm_fused_allreduce_norm(
allreduce_in=hidden_states,
residual=residual,
rms_gamma=norm.weight,
rms_eps=norm.variance_epsilon,
world_size=tp_size,
weight_bias=1.0, # GemmaRMSNorm-style
launch_with_pdl=True,
fp32_acc=True,
max_token_num=max_token_num,
pattern_code=_AR_RESIDUAL_RMS_NORM,
norm_out=norm_out,
)
return norm_out, hidden_states
# Fallback: explicit all-reduce + GemmaRMSNorm (matches the unfused model).
reduced = tensor_model_parallel_all_reduce(hidden_states)
return norm(reduced, residual)
@@ -17,7 +17,12 @@ class MoEActivation(Enum):
GELU = "gelu"
GELU_TANH = "gelu_tanh"
RELU2 = "relu2"
# SWIGLUOAI expects gate/up *interleaved* in w13 ([gate0, up0, gate1, ...]),
# as in gpt-oss checkpoints. SWIGLUOAI_UNINTERLEAVE has identical math but
# expects the *packed* layout ([all gates; all ups]), as produced by a
# MergedColumnParallelLinear gate_up_proj (e.g. MiniMax-M3).
SWIGLUOAI = "swigluoai"
SWIGLUOAI_UNINTERLEAVE = "swigluoai_uninterleave"
SWIGLUSTEP = "swiglustep"
# Non-gated activations (no mul with gate) expect input of shape [..., d]
@@ -73,6 +78,7 @@ _CUSTOM_OP_NAMES: dict[MoEActivation, str] = {
MoEActivation.GELU: "gelu_and_mul",
MoEActivation.GELU_TANH: "gelu_tanh_and_mul",
MoEActivation.SWIGLUOAI: "swigluoai_and_mul",
MoEActivation.SWIGLUOAI_UNINTERLEAVE: "silu_and_mul_with_clamp",
MoEActivation.SWIGLUSTEP: "swiglustep_and_mul",
MoEActivation.RELU2: "relu2",
MoEActivation.SILU_NO_MUL: "silu_and_mul",
@@ -105,8 +111,17 @@ def apply_moe_activation(
activation: MoEActivation,
output: torch.Tensor,
input: torch.Tensor,
*,
clamp_limit: float | None = None,
alpha: float = 1.0,
beta: float = 0.0,
) -> torch.Tensor:
"""Apply MoE activation function."""
"""Apply MoE activation function.
``clamp_limit``/``alpha``/``beta`` (from the quant config) drive the clamped
SwiGLU kernels: ``SILU`` + ``clamp_limit`` and ``SWIGLUOAI_UNINTERLEAVE`` both
map to ``silu_and_mul_with_clamp``. Other activations ignore them.
"""
assert input.dim() == 2, "Input must be 2D"
assert output.dim() == 2, "Output must be 2D"
if activation.is_gated:
@@ -122,13 +137,21 @@ def apply_moe_activation(
# Activations with gated multiplication (gate × activation(up))
if activation == MoEActivation.SILU:
torch.ops._C.silu_and_mul(output, input)
if clamp_limit is not None:
# Fused silu(clamp(gate)) * clamp(up); equivalent to swiglu_limit_func.
torch.ops._C.silu_and_mul_with_clamp(output, input, clamp_limit, 1.0, 0.0)
else:
torch.ops._C.silu_and_mul(output, input)
elif activation == MoEActivation.GELU:
torch.ops._C.gelu_and_mul(output, input)
elif activation == MoEActivation.GELU_TANH:
torch.ops._C.gelu_tanh_and_mul(output, input)
elif activation == MoEActivation.SWIGLUOAI:
torch.ops._C.swigluoai_and_mul(output, input)
elif activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
# SwiGLU-OAI on packed w13 (gate = first half, up = second half).
assert clamp_limit is not None, "SWIGLUOAI_UNINTERLEAVE requires clamp_limit"
torch.ops._C.silu_and_mul_with_clamp(output, input, clamp_limit, alpha, beta)
elif activation == MoEActivation.SWIGLUSTEP:
from vllm.model_executor.layers.activation import swiglustep_and_mul_triton
@@ -900,6 +900,9 @@ def fp8_w8a16_moe_quant_config(
w1_bias: torch.Tensor | None = None,
w2_bias: torch.Tensor | None = None,
block_shape: list[int] | None = None,
gemm1_alpha: float | None = None,
gemm1_beta: float | None = None,
gemm1_clamp_limit: float | None = None,
) -> FusedMoEQuantConfig:
"""
Construct a quant config for 16-bit float activations and fp8 weights.
@@ -925,6 +928,9 @@ def fp8_w8a16_moe_quant_config(
None,
w2_bias,
),
gemm1_alpha=gemm1_alpha,
gemm1_beta=gemm1_beta,
gemm1_clamp_limit=gemm1_clamp_limit,
)
@@ -979,15 +985,24 @@ def int4_w4afp8_moe_quant_config(
def biased_moe_quant_config(
w1_bias: torch.Tensor | None,
w2_bias: torch.Tensor | None,
gemm1_alpha: float | None = None,
gemm1_beta: float | None = None,
gemm1_clamp_limit: float | None = None,
) -> FusedMoEQuantConfig:
"""
Construct a quant config for unquantized activations with biases.
gemm1_alpha/gemm1_beta/gemm1_clamp_limit carry the SwiGLU gate params
through to the fused activation kernel (e.g. swigluoai_uninterleave).
"""
return FusedMoEQuantConfig(
_a1=FusedMoEQuantDesc(),
_a2=FusedMoEQuantDesc(),
_w1=FusedMoEQuantDesc(bias=w1_bias),
_w2=FusedMoEQuantDesc(bias=w2_bias),
gemm1_alpha=gemm1_alpha,
gemm1_beta=gemm1_beta,
gemm1_clamp_limit=gemm1_clamp_limit,
)
@@ -1268,6 +1283,8 @@ class FusedMoEConfig:
# are filtered out by `FusedMoEExperts.is_supported_config` so the oracle
# cannot silently select one and drop the clamp.
swiglu_limit: float | None = None
swiglu_alpha: float | None = None
swiglu_beta: float | None = None
max_capture_size: int = 0
@@ -130,6 +130,9 @@ def _fwd_kernel_ep_scatter_2(
HIDDEN_SIZE_PAD: tl.constexpr,
SCALE_HIDDEN_SIZE: tl.constexpr,
SCALE_HIDDEN_SIZE_PAD: tl.constexpr,
PACK_UE8M0: tl.constexpr,
SCALE_PACKED_SIZE: tl.constexpr,
SCALE_PACKED_SIZE_PAD: tl.constexpr,
):
start_token_id = tl.program_id(0)
grid_num = tl.num_programs(0)
@@ -137,16 +140,47 @@ def _fwd_kernel_ep_scatter_2(
offset_in = tl.arange(0, HIDDEN_SIZE_PAD)
mask = offset_in < HIDDEN_SIZE
offset_in_s = tl.arange(0, SCALE_HIDDEN_SIZE_PAD)
mask_s = offset_in_s < SCALE_HIDDEN_SIZE
output_tensor_stride0 = output_tensor_stride0.to(tl.int64)
if PACK_UE8M0:
# One int32 per 4 consecutive 32-wide UE8M0 groups, stored MN-major.
offs_pk = tl.arange(0, SCALE_PACKED_SIZE_PAD)
mask_pk = offs_pk < SCALE_PACKED_SIZE
else:
offset_in_s = tl.arange(0, SCALE_HIDDEN_SIZE_PAD)
mask_s = offset_in_s < SCALE_HIDDEN_SIZE
for token_id in range(start_token_id, total_token_num, grid_num):
to_copy = tl.load(recv_x + token_id * recv_x_stride0 + offset_in, mask=mask)
to_copy_s = tl.load(
recv_x_scale + token_id * recv_x_scale_stride0 + offset_in_s, mask=mask_s
)
if PACK_UE8M0:
# Pack 4 UE8M0 bytes into one int32 (byte j = group 4*pk+j).
base_s = recv_x_scale + token_id * recv_x_scale_stride0
g0, g1 = offs_pk * 4, offs_pk * 4 + 1
g2, g3 = offs_pk * 4 + 2, offs_pk * 4 + 3
b0 = tl.load(
base_s + g0 * recv_x_scale_stride1, mask=g0 < SCALE_HIDDEN_SIZE
)
b1 = tl.load(
base_s + g1 * recv_x_scale_stride1, mask=g1 < SCALE_HIDDEN_SIZE
)
b2 = tl.load(
base_s + g2 * recv_x_scale_stride1, mask=g2 < SCALE_HIDDEN_SIZE
)
b3 = tl.load(
base_s + g3 * recv_x_scale_stride1, mask=g3 < SCALE_HIDDEN_SIZE
)
packed_s = (
b0.to(tl.int32)
| (b1.to(tl.int32) << 8)
| (b2.to(tl.int32) << 16)
| (b3.to(tl.int32) << 24)
)
else:
to_copy_s = tl.load(
recv_x_scale + token_id * recv_x_scale_stride0 + offset_in_s,
mask=mask_s,
)
for topk_index in tl.range(0, topk_num, 1, num_stages=4):
expert_id = tl.load(recv_topk + token_id * recv_topk_stride0 + topk_index)
@@ -164,11 +198,21 @@ def _fwd_kernel_ep_scatter_2(
output_tensor_ptr = (
output_tensor + dest_token_index_i64 * output_tensor_stride0
)
tl.store(output_tensor_ptr + offset_in, to_copy, mask=mask)
output_tensor_scale_ptr = (
output_tensor_scale + dest_token_index * output_tensor_scale_stride0
)
tl.store(output_tensor_ptr + offset_in, to_copy, mask=mask)
tl.store(output_tensor_scale_ptr + offset_in_s, to_copy_s, mask=mask_s)
if PACK_UE8M0:
tl.store(
output_tensor_scale_ptr + offs_pk * output_tensor_scale_stride1,
packed_s,
mask=mask_pk,
)
else:
tl.store(
output_tensor_scale_ptr + offset_in_s, to_copy_s, mask=mask_s
)
@torch.no_grad()
@@ -183,9 +227,11 @@ def ep_scatter(
output_tensor_scale: torch.Tensor,
m_indices: torch.Tensor,
output_index: torch.Tensor,
block_size: int = 128,
pack_ue8m0: bool = False,
):
BLOCK_E = 128 # token num of per expert is aligned to 128
BLOCK_D = 128 # block size of quantization
BLOCK_D = block_size # block size of activation-scale quantization
num_warps = 8
num_experts = num_recv_tokens_per_expert.shape[0]
hidden_size = recv_x.shape[1]
@@ -195,6 +241,10 @@ def ep_scatter(
assert m_indices.shape[0] % BLOCK_E == 0
assert expert_start_loc.shape[0] == num_experts
# pack_ue8m0: scatter packs 4 UE8M0 bytes per int32; else copies scales as-is.
scale_hidden_size = hidden_size // BLOCK_D
scale_packed_size = (scale_hidden_size + 3) // 4 if pack_ue8m0 else 1
_fwd_kernel_ep_scatter_1[(grid,)](
num_recv_tokens_per_expert,
expert_start_loc,
@@ -234,8 +284,11 @@ def ep_scatter(
num_warps=num_warps,
HIDDEN_SIZE=hidden_size,
HIDDEN_SIZE_PAD=triton.next_power_of_2(hidden_size),
SCALE_HIDDEN_SIZE=hidden_size // BLOCK_D,
SCALE_HIDDEN_SIZE_PAD=triton.next_power_of_2(hidden_size // BLOCK_D),
SCALE_HIDDEN_SIZE=scale_hidden_size,
SCALE_HIDDEN_SIZE_PAD=triton.next_power_of_2(scale_hidden_size),
PACK_UE8M0=pack_ue8m0,
SCALE_PACKED_SIZE=scale_packed_size,
SCALE_PACKED_SIZE_PAD=triton.next_power_of_2(scale_packed_size),
)
return
@@ -352,6 +405,7 @@ def deepgemm_moe_permute(
expert_map: torch.Tensor | None,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
aq_out: torch.Tensor | None = None,
block_size: int | None = None,
):
assert aq.ndim == 2
assert topk_ids.dtype.is_signed, "The kernel uses -1 to represent invalid topk_ids"
@@ -359,6 +413,10 @@ def deepgemm_moe_permute(
device = aq.device
block_m, block_k = get_mk_alignment_for_contiguous_layout()
# The activation-scale group size may differ from the M/K tile alignment
# (e.g. MXFP8 uses a 32-element scale group while block_k stays 128).
if block_size is not None:
block_k = block_size
M_sum = compute_aligned_M(
M=topk_ids.size(0),
@@ -376,9 +434,21 @@ def deepgemm_moe_permute(
if aq_out is None:
aq_out = torch.empty((M_sum, H), device=device, dtype=aq.dtype)
aq_scale_out = torch.empty(
(M_sum, H // block_k), device=device, dtype=torch.float32
)
# uint8 UE8M0 (MXFP8) -> scatter packs into DeepGEMM's int32 MN-major
# TMA-aligned layout; float32 (FP8/FP4) scattered row-major as-is.
pack_ue8m0 = aq_scale.dtype == torch.uint8
sf_k = H // block_k
if pack_ue8m0:
packed_sf_k = (sf_k + 3) // 4
tma_aligned_mn = round_up(M_sum, 4)
aq_scale_out = torch.empty_strided(
(M_sum, packed_sf_k),
(1, tma_aligned_mn),
device=device,
dtype=torch.int32,
)
else:
aq_scale_out = torch.empty((M_sum, sf_k), device=device, dtype=torch.float32)
# DeepGEMM uses negative values in m_indices (here expert_ids) to mark
# completely invalid / padded blocks that should be skipped. We always
@@ -412,6 +482,8 @@ def deepgemm_moe_permute(
output_tensor_scale=aq_scale_out,
m_indices=expert_ids,
output_index=inv_perm,
block_size=block_k,
pack_ue8m0=pack_ue8m0,
)
return aq_out, aq_scale_out, expert_ids, inv_perm
@@ -33,7 +33,10 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Dynamic128Sym,
kFp8Static128BlockSym,
kMxfp4Static,
kMxfp8Dynamic,
kMxfp8Static,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import (
DeepGemmQuantScaleFMT,
get_mk_alignment_for_contiguous_layout,
@@ -123,12 +126,26 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
def __init__(self, moe_config: FusedMoEConfig, quant_config: FusedMoEQuantConfig):
super().__init__(moe_config=moe_config, quant_config=quant_config)
assert quant_config.block_shape == get_mk_alignment_for_contiguous_layout()
assert quant_config.quant_dtype == torch.float8_e4m3fn
# MXFP8: FP8 e4m3 values + UE8M0 1x32 block scales (Blackwell). Reuses
# the same grouped GEMM (aliased to fp8_fp4) with recipe (1, 32).
self.mxfp8 = quant_config.block_shape == [1, 32]
if self.mxfp8:
assert quant_config.quant_dtype == "mxfp8"
else:
assert quant_config.block_shape == get_mk_alignment_for_contiguous_layout()
assert quant_config.quant_dtype == torch.float8_e4m3fn
assert not quant_config.per_act_token_quant
assert not quant_config.per_out_ch_quant
self.gemm1_clamp_limit = quant_config.gemm1_clamp_limit
# Gated-activation params: silu == swigluoai with alpha=1, beta=0.
# FP8 (silu) configs leave these None, reproducing plain silu.
self.gemm1_alpha = (
quant_config.gemm1_alpha if quant_config.gemm1_alpha is not None else 1.0
)
self.gemm1_beta = (
quant_config.gemm1_beta if quant_config.gemm1_beta is not None else 0.0
)
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
@@ -147,14 +164,25 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
SUPPORTED_W_A = [
(kFp8Static128BlockSym, kFp8Dynamic128Sym),
]
return (weight_key, activation_key) in SUPPORTED_W_A
if (weight_key, activation_key) == (kFp8Static128BlockSym, kFp8Dynamic128Sym):
return True
# MXFP8 1x32 uses the fp8_fp4 grouped GEMM with recipe (1, 32) — only
# available on Blackwell (SM100).
if (weight_key, activation_key) == (kMxfp8Static, kMxfp8Dynamic):
return current_platform.is_device_capability_family(100)
return False
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation in [MoEActivation.SILU, MoEActivation.SWIGLUSTEP]
# silu/swigluoai go through the fused alpha/beta kernel; swiglustep
# uses the unfused activation path. The fused kernel reads packed w13
# (gate = first half, up = second half), so it implements the
# *uninterleaved* SwiGLU-OAI variant.
return activation in [
MoEActivation.SILU,
MoEActivation.SWIGLUSTEP,
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
]
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
@@ -179,7 +207,9 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
activation: MoEActivation,
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
assert self.block_shape is not None
block_m = self.block_shape[0]
# Use the contiguous-layout M alignment (matches apply()); block_shape[0]
# is the quant block (1 for MXFP8) and would under-size the workspace.
block_m = get_mk_alignment_for_contiguous_layout()[0]
M_sum = compute_aligned_M(
M, topk, local_num_experts, block_m, expert_tokens_meta
)
@@ -201,14 +231,24 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
M_sum, N = input.size()
activation_out_dim = self.adjust_N_for_activation(N, activation)
# 1. DeepGemm UE8M0: fused SiLU+mul+clamp+quant+pack
# silu and swigluoai are both expressible by the fused gated kernel via
# (alpha, beta): silu uses alpha=1, beta=0; swigluoai uses config values.
# The fused kernel reads packed w13, hence SWIGLUOAI_UNINTERLEAVE.
fused_gated = activation in (
MoEActivation.SILU,
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
)
# 1. DeepGemm UE8M0: fused gate+mul+clamp+quant+pack
if scale_fmt == DeepGemmQuantScaleFMT.UE8M0:
if activation == MoEActivation.SILU:
if fused_gated:
return fused_silu_mul_fp8_quant_packed(
input=input,
output_q=output,
group_size=block_k,
clamp_limit=self.gemm1_clamp_limit,
alpha=self.gemm1_alpha,
beta=self.gemm1_beta,
)
act_out = torch.empty(
(M_sum, activation_out_dim), dtype=input.dtype, device=input.device
@@ -221,14 +261,17 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
)
return a2q, a2q_scale
# 2. Hopper / non‑E8M0: prefer the fused SiLU+mul+quant kernel
if activation == MoEActivation.SILU:
# 2. Hopper / non‑E8M0: prefer the fused gate+mul+quant kernel
if fused_gated:
use_ue8m0 = scale_fmt == DeepGemmQuantScaleFMT.FLOAT32_CEIL_UE8M0
return silu_mul_per_token_group_quant_fp8_colmajor(
input=input,
output=output,
use_ue8m0=use_ue8m0,
clamp_limit=self.gemm1_clamp_limit,
group_size=block_k,
alpha=self.gemm1_alpha,
beta=self.gemm1_beta,
)
# 3. fallback path for non-SiLU activations in non‑UE8M0 cases.
@@ -292,12 +335,23 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
expert_map=expert_map,
expert_tokens_meta=expert_tokens_meta,
aq_out=a1q_perm,
# MXFP8 uses a 32-element activation-scale group (block_shape[1]);
# FP8-block keeps the default (128) alignment.
block_size=self.block_shape[1] if self.mxfp8 else None,
)
assert a1q.size(0) == M_sum
# MXFP8 (1x32) drives the fp8_fp4-aliased grouped GEMM with recipe
# (1, 32); the FP8 block path keeps the default (128) recipe.
gemm_kwargs = (
{"recipe_a": (1, self.block_shape[1]), "recipe_b": (1, self.block_shape[1])}
if self.mxfp8
else {}
)
mm1_out = _resize_cache(workspace2, (M_sum, N))
m_grouped_fp8_gemm_nt_contiguous(
(a1q, a1q_scale), (w1, self.w1_scale), mm1_out, expert_ids
(a1q, a1q_scale), (w1, self.w1_scale), mm1_out, expert_ids, **gemm_kwargs
)
activation_out_dim = self.adjust_N_for_activation(N, activation)
@@ -310,7 +364,7 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
mm2_out = _resize_cache(workspace2, (M_sum, K))
m_grouped_fp8_gemm_nt_contiguous(
(a2q, a2q_scale), (w2, self.w2_scale), mm2_out, expert_ids
(a2q, a2q_scale), (w2, self.w2_scale), mm2_out, expert_ids, **gemm_kwargs
)
if apply_router_weight_on_input:
@@ -801,7 +801,11 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular):
return TopKWeightAndReduceDelegate()
def activation(
self, activation: MoEActivation, output: torch.Tensor, input: torch.Tensor
self,
activation: MoEActivation,
output: torch.Tensor,
input: torch.Tensor,
**kwargs,
) -> None:
gemm1_clamp_limit = self.quant_config.gemm1_clamp_limit
if activation == MoEActivation.SILU and gemm1_clamp_limit is not None:
@@ -787,6 +787,7 @@ class UnfusedOAITritonExperts(LoRAExpertsMixin, BaseOAITritonExperts):
activation: MoEActivation,
output: torch.Tensor,
input: torch.Tensor,
**kwargs,
) -> None:
quant_config = self.quant_config or FUSED_MOE_UNQUANTIZED_CONFIG
if activation == MoEActivation.SWIGLUOAI:
@@ -28,10 +28,7 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceDelegate,
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.fused_moe.utils import (
_resize_cache,
swiglu_limit_func,
)
from vllm.model_executor.layers.fused_moe.utils import _resize_cache
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
get_marlin_input_dtype,
marlin_make_workspace_new,
@@ -74,9 +71,7 @@ def _fused_marlin_moe(
expert_ids: torch.Tensor,
num_tokens_post_padded: torch.Tensor,
activation: MoEActivation = MoEActivation.SILU,
activation_func: Callable[
[MoEActivation, torch.Tensor, torch.Tensor], None
] = apply_moe_activation,
activation_func: Callable[..., None] = apply_moe_activation,
input_global_scale1: torch.Tensor | None = None,
input_global_scale2: torch.Tensor | None = None,
global_scale1: torch.Tensor | None = None,
@@ -94,6 +89,8 @@ def _fused_marlin_moe(
input_dtype: torch.dtype | None = None,
is_k_full: bool = True,
clamp_limit: float | None = None,
gemm1_alpha: float = 1.0,
gemm1_beta: float = 0.0,
) -> torch.Tensor:
assert hidden_states.ndim == 2
M, K = hidden_states.size()
@@ -161,18 +158,16 @@ def _fused_marlin_moe(
use_fp32_reduce=True,
is_zp_float=False,
)
if clamp_limit is not None and activation == MoEActivation.SILU:
swiglu_limit_func(
intermediate_cache2,
intermediate_cache1.view(-1, w13_num_shards * N),
clamp_limit,
)
else:
activation_func(
activation,
intermediate_cache2,
intermediate_cache1.view(-1, w13_num_shards * N),
)
# apply_moe_activation fuses the clamp/gate params: SILU + clamp_limit and
# SWIGLUOAI_UNINTERLEAVE both map to the silu_and_mul_with_clamp kernel.
activation_func(
activation,
intermediate_cache2,
intermediate_cache1.view(-1, w13_num_shards * N),
clamp_limit=clamp_limit,
alpha=gemm1_alpha,
beta=gemm1_beta,
)
if output is None:
output = intermediate_cache3
@@ -238,9 +233,7 @@ def fused_marlin_moe(
apply_router_weight_on_input: bool = False,
global_num_experts: int = -1,
activation: MoEActivation = MoEActivation.SILU,
activation_func: Callable[
[MoEActivation, torch.Tensor, torch.Tensor], None
] = apply_moe_activation,
activation_func: Callable[..., None] = apply_moe_activation,
moe_sum: Callable[[torch.Tensor, torch.Tensor], None] | None = None,
expert_map: torch.Tensor | None = None,
input_global_scale1: torch.Tensor | None = None,
@@ -260,6 +253,8 @@ def fused_marlin_moe(
output: torch.Tensor | None = None,
input_dtype: torch.dtype | None = None,
clamp_limit: float | None = None,
gemm1_alpha: float = 1.0,
gemm1_beta: float = 0.0,
) -> torch.Tensor:
"""
This function computes a Mixture of Experts (MoE) layer using two sets of
@@ -373,6 +368,8 @@ def fused_marlin_moe(
input_dtype=input_dtype,
is_k_full=is_k_full,
clamp_limit=clamp_limit,
gemm1_alpha=gemm1_alpha,
gemm1_beta=gemm1_beta,
).view(-1, topk, K)
if output is None:
@@ -415,6 +412,8 @@ def batched_fused_marlin_moe(
output: torch.Tensor | None = None,
input_dtype: torch.dtype | None = None,
clamp_limit: float | None = None,
gemm1_alpha: float = 1.0,
gemm1_beta: float = 0.0,
) -> torch.Tensor:
"""
This function massages the inputs so the batched hidden_states can be
@@ -544,6 +543,8 @@ def batched_fused_marlin_moe(
input_dtype=input_dtype,
is_k_full=is_k_full,
clamp_limit=clamp_limit,
gemm1_alpha=gemm1_alpha,
gemm1_beta=gemm1_beta,
)
output = output.view(B, BATCH_TOKENS_MAX, K)
@@ -579,6 +580,15 @@ class MarlinExpertsBase(mk.FusedMoEExpertsModular):
self.is_k_full = is_k_full
self.input_dtype = get_marlin_input_dtype()
self.gemm1_clamp_limit = quant_config.gemm1_clamp_limit
# Gated-activation params (used by SWIGLUOAI_UNINTERLEAVE on packed w13).
# silu == swigluoai with alpha=1, beta=0; configs that don't set these
# (plain silu) fall back to the silu identity.
self.gemm1_alpha = (
quant_config.gemm1_alpha if quant_config.gemm1_alpha is not None else 1.0
)
self.gemm1_beta = (
quant_config.gemm1_beta if quant_config.gemm1_beta is not None else 0.0
)
super().__init__(
moe_config=moe_config,
@@ -627,6 +637,7 @@ class MarlinExpertsBase(mk.FusedMoEExpertsModular):
MoEActivation.GELU,
MoEActivation.GELU_TANH,
MoEActivation.SWIGLUOAI,
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
MoEActivation.SWIGLUSTEP,
MoEActivation.SILU_NO_MUL,
MoEActivation.GELU_NO_MUL,
@@ -787,6 +798,8 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
is_k_full=self.is_k_full,
input_dtype=self.input_dtype,
clamp_limit=self.gemm1_clamp_limit,
gemm1_alpha=self.gemm1_alpha,
gemm1_beta=self.gemm1_beta,
)
return
@@ -805,6 +818,10 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
act_enum: MoEActivation,
act_output: torch.Tensor,
act_input: torch.Tensor,
*,
clamp_limit: float | None = None,
alpha: float = 1.0,
beta: float = 0.0,
) -> None:
# act_input = intermediate_cache1 (M*topk, 2N for gated)
# act_output = intermediate_cache2 (M*topk, N)
@@ -834,7 +851,14 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
"tlm": token_lora_mapping,
}
)
self.activation(act_enum, act_output, act_input)
self.activation(
act_enum,
act_output,
act_input,
clamp_limit=clamp_limit,
alpha=alpha,
beta=beta,
)
lora_state["cache2"] = act_output
def moe_sum_with_lora(moe_out: torch.Tensor, out: torch.Tensor) -> None:
@@ -888,6 +912,8 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
is_k_full=self.is_k_full,
input_dtype=self.input_dtype,
clamp_limit=self.gemm1_clamp_limit,
gemm1_alpha=self.gemm1_alpha,
gemm1_beta=self.gemm1_beta,
)
def moe_sum(self, input: torch.Tensor, output: torch.Tensor) -> None:
@@ -996,4 +1022,6 @@ class BatchedMarlinExperts(MarlinExpertsBase):
input_dtype=self.input_dtype,
is_k_full=self.is_k_full,
clamp_limit=self.gemm1_clamp_limit,
gemm1_alpha=self.gemm1_alpha,
gemm1_beta=self.gemm1_beta,
)
@@ -0,0 +1,176 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MXFP8 (1x32 block, E8M0 scale) MoE experts on Triton.
``Mxfp8TritonExpertsBase`` stashes E8M0 weight scales for checkpoint layout.
``Mxfp8EmulationTritonExperts`` dequantizes to BF16 and runs ``TritonExperts``
for devices without a native MXFP8 MoE kernel (e.g. ROCm gfx942 / MI300).
"""
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.experts.triton_moe import TritonExperts
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
dequant_mxfp8_to_bf16,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp8Dynamic,
kMxfp8Static,
)
logger = init_logger(__name__)
class Mxfp8TritonExpertsBase(TritonExperts):
"""Shared MXFP8 MoE setup: stash E8M0 scales, clear scales on ``quant_config``."""
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
):
super().__init__(moe_config, quant_config)
self.w1_scale_val = self.quant_config.w1_scale
self.w2_scale_val = self.quant_config.w2_scale
self.quant_config._w1.scale = None
self.quant_config._w2.scale = None
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return (weight_key, activation_key) == (kMxfp8Static, kMxfp8Dynamic)
@staticmethod
def _supports_activation(activation) -> bool:
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
if activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
return True
return TritonExperts._supports_activation(activation)
class Mxfp8EmulationTritonExperts(Mxfp8TritonExpertsBase):
"""Dequantize MXFP8 weights to BF16 on the fly and run ``TritonExperts``."""
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
):
super().__init__(moe_config, quant_config)
logger.warning_once(
"Using Mxfp8EmulationTritonExperts MoE backend. Weights are "
"dequantized to BF16 on the fly; this is slower than a native "
"MXFP8 MoE kernel and is intended for devices without one."
)
@property
def quant_dtype(self) -> torch.dtype | str | None:
# BF16 fallback: do not MXFP8-quantize activations in ``TritonExperts``.
return None
@property
def block_shape(self) -> list[int] | None:
return None
@property
def expects_unquantized_inputs(self) -> bool:
return True
@staticmethod
def _supports_current_device() -> bool:
return True
def activation(
self,
activation,
output: torch.Tensor,
input: torch.Tensor,
**kwargs,
):
"""Apply GEMM1 activation with quant-config alpha/beta/clamp."""
from vllm.model_executor.layers.fused_moe.activation import (
MoEActivation,
apply_moe_activation,
)
if activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
limit = self.quant_config.gemm1_clamp_limit
if limit is None:
raise ValueError("SWIGLUOAI_UNINTERLEAVE requires gemm1_clamp_limit")
alpha = self.quant_config.gemm1_alpha
alpha = 1.702 if alpha is None else float(alpha)
beta = self.quant_config.gemm1_beta
beta = 1.0 if beta is None else float(beta)
apply_moe_activation(
activation,
output,
input,
clamp_limit=float(limit),
alpha=alpha,
beta=beta,
)
return
super().activation(activation, output, input)
def apply(
self,
output: torch.Tensor,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
a2_scale: torch.Tensor | None,
workspace13: torch.Tensor,
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
):
# If the weights were already dequantized to BF16 at load time
# (process_weights_after_loading on devices without a native MXFP8 MoE
# kernel), use them directly -- no per-step dequant. MXFP8 weights are
# 1-byte FP8 (element_size 1); BF16/FP16 are >= 2 bytes.
if w1.element_size() >= 2:
# tl.dot requires w and activations share a dtype; .to() is a no-op
# when they already match (e.g. both BF16).
w1_bf16 = w1.to(hidden_states.dtype)
w2_bf16 = w2.to(hidden_states.dtype)
else:
w1_bf16 = dequant_mxfp8_to_bf16(w1, self.w1_scale_val).to(
hidden_states.dtype
)
w2_bf16 = dequant_mxfp8_to_bf16(w2, self.w2_scale_val).to(
hidden_states.dtype
)
super().apply(
output=output,
hidden_states=hidden_states,
w1=w1_bf16,
w2=w2_bf16,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=activation,
global_num_experts=global_num_experts,
expert_map=expert_map,
a1q_scale=None,
a2_scale=None,
workspace13=workspace13,
workspace2=workspace2,
expert_tokens_meta=expert_tokens_meta,
apply_router_weight_on_input=apply_router_weight_on_input,
)
@@ -0,0 +1,326 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Native MXFP8 (1x32 block, E8M0 scale) MoE for AMD CDNA4 (gfx950) via Triton
``tl.dot_scaled`` (hardware microscaling matmul).
The expert GEMMs consume the FP8 E4M3 weights and their E8M0 block scales
directly (no dequant-to-BF16), and activations are MXFP8-quantized per token.
On CDNA4 ``dot_scaled`` maps to the native MX matrix-core ops; on other archs
Triton upcasts to BF16 (so this stays correct, just not faster) — but the
oracle only selects this path on gfx950 and routes everything else to the
BF16 ``Mxfp8EmulationTritonExperts`` fallback.
Structure mirrors vLLM's ``fused_moe_kernel``: tokens are sorted by expert
(``moe_align_block_size``); each program computes a ``[BLOCK_M, BLOCK_N]`` tile
for one expert, accumulating over K with ``dot_scaled``. SwiGLU-OAI activation
and the top-k weighted reduction run in PyTorch between/after the two GEMMs.
"""
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.experts.mxfp8_emulation_moe import (
Mxfp8TritonExpertsBase,
)
from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
moe_align_block_size,
)
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
mxfp8_e4m3_quantize,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
logger = init_logger(__name__)
@triton.jit
def _mxfp8_grouped_gemm_kernel(
a_ptr,
a_scale_ptr,
b_ptr,
b_scale_ptr,
c_ptr,
topk_weights_ptr,
sorted_token_ids_ptr,
expert_ids_ptr,
num_tokens_post_padded_ptr,
N,
K,
num_valid_tokens,
top_k,
stride_am,
stride_ak,
stride_asm,
stride_ask,
stride_be,
stride_bn,
stride_bk,
stride_bse,
stride_bsn,
stride_bsk,
stride_cm,
stride_cn,
A_DIV: tl.constexpr,
MUL_WEIGHT: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
BLOCK_K: tl.constexpr,
):
pid_m = tl.program_id(0)
pid_n = tl.program_id(1)
num_post = tl.load(num_tokens_post_padded_ptr)
if pid_m * BLOCK_M >= num_post:
return
offs_tid = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
offs_token = tl.load(sorted_token_ids_ptr + offs_tid).to(tl.int64)
token_mask = offs_token < num_valid_tokens
off_e = tl.load(expert_ids_ptr + pid_m).to(tl.int64)
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
offs_k = tl.arange(0, BLOCK_K)
offs_sk = tl.arange(0, BLOCK_K // 32)
a_row = offs_token // A_DIV
a_ptrs = a_ptr + a_row[:, None] * stride_am + offs_k[None, :] * stride_ak
as_ptrs = a_scale_ptr + a_row[:, None] * stride_asm + offs_sk[None, :] * stride_ask
b_ptrs = (
b_ptr
+ off_e * stride_be
+ offs_n[:, None] * stride_bn
+ offs_k[None, :] * stride_bk
)
bs_ptrs = (
b_scale_ptr
+ off_e * stride_bse
+ offs_n[:, None] * stride_bsn
+ offs_sk[None, :] * stride_bsk
)
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
n_mask = offs_n < N
for _ in range(0, tl.cdiv(K, BLOCK_K)):
a = tl.load(a_ptrs, mask=token_mask[:, None], other=0.0)
b = tl.load(b_ptrs, mask=n_mask[:, None], other=0.0)
asc = tl.load(as_ptrs, mask=token_mask[:, None], other=0)
bsc = tl.load(bs_ptrs, mask=n_mask[:, None], other=0)
acc += tl.dot_scaled(a, asc, "e4m3", b.T, bsc, "e4m3")
a_ptrs += BLOCK_K * stride_ak
b_ptrs += BLOCK_K * stride_bk
as_ptrs += (BLOCK_K // 32) * stride_ask
bs_ptrs += (BLOCK_K // 32) * stride_bsk
if MUL_WEIGHT:
w = tl.load(topk_weights_ptr + offs_token, mask=token_mask, other=0.0)
acc = acc * w[:, None]
c_ptrs = c_ptr + offs_token[:, None] * stride_cm + offs_n[None, :] * stride_cn
tl.store(
c_ptrs,
acc.to(c_ptr.dtype.element_ty),
mask=token_mask[:, None] & n_mask[None, :],
)
def _grouped_gemm_mxfp8(
a_q: torch.Tensor, # [M, K] fp8 e4m3
a_scale: torch.Tensor, # [M, K//32] uint8 (E8M0)
w: torch.Tensor, # [E, N, K] fp8 e4m3
w_scale: torch.Tensor, # [E, N, K//32] uint8 (E8M0)
sorted_token_ids: torch.Tensor,
expert_ids: torch.Tensor,
num_tokens_post_padded: torch.Tensor,
num_valid_tokens: int,
top_k: int,
block_m: int,
out_dtype: torch.dtype,
a_div: int,
mul_weight_by: torch.Tensor | None = None,
expert_map: torch.Tensor | None = None,
) -> torch.Tensor:
M_routed = num_valid_tokens
E, N, K = w.shape
assert K % 128 == 0, f"MXFP8 native MoE requires K%128==0, got K={K}"
# Under expert parallelism (expert_map set) tokens routed to non-local
# experts are dropped from sorted_token_ids, so their output rows are never
# written — zero them so the downstream reduction ignores their garbage.
alloc = torch.zeros if expert_map is not None else torch.empty
out = alloc((M_routed, N), dtype=out_dtype, device=a_q.device)
BLOCK_N = 128
BLOCK_K = 128
grid = (triton.cdiv(sorted_token_ids.shape[0], block_m), triton.cdiv(N, BLOCK_N))
_mxfp8_grouped_gemm_kernel[grid](
a_q,
a_scale,
w,
w_scale,
out,
mul_weight_by if mul_weight_by is not None else a_q,
sorted_token_ids,
expert_ids,
num_tokens_post_padded,
N,
K,
num_valid_tokens,
top_k,
a_q.stride(0),
a_q.stride(1),
a_scale.stride(0),
a_scale.stride(1),
w.stride(0),
w.stride(1),
w.stride(2),
w_scale.stride(0),
w_scale.stride(1),
w_scale.stride(2),
out.stride(0),
out.stride(1),
A_DIV=a_div,
MUL_WEIGHT=mul_weight_by is not None,
BLOCK_M=block_m,
BLOCK_N=BLOCK_N,
BLOCK_K=BLOCK_K,
num_warps=8,
)
return out
def fused_moe_mxfp8_native(
hidden_states: torch.Tensor, # [T, H] bf16
w13: torch.Tensor, # [E, 2I, H] fp8
w13_scale: torch.Tensor, # [E, 2I, H//32] uint8
w2: torch.Tensor, # [E, H, I] fp8
w2_scale: torch.Tensor, # [E, H, I//32] uint8
topk_weights: torch.Tensor, # [T, top_k]
topk_ids: torch.Tensor, # [T, top_k] (global expert ids)
*,
alpha: float,
beta: float,
limit: float | None,
global_num_experts: int,
expert_map: torch.Tensor | None,
) -> torch.Tensor:
T, H = hidden_states.shape
top_k = topk_ids.shape[1]
M = T * top_k
block_m = 64
sorted_ids, expert_ids, num_post = moe_align_block_size(
topk_ids,
block_m,
global_num_experts,
expert_map,
ignore_invalid_experts=expert_map is not None,
)
# GEMM1: x (mxfp8) @ w13^T -> [M, 2I]
a_q, a_s = mxfp8_e4m3_quantize(hidden_states)
g1 = _grouped_gemm_mxfp8(
a_q,
a_s,
w13,
w13_scale,
sorted_ids,
expert_ids,
num_post,
M,
top_k,
block_m,
hidden_states.dtype,
a_div=top_k,
expert_map=expert_map,
) # [M, 2I]
# SwiGLU-OAI (split layout: gate=g1[:, :I], up=g1[:, I:]) FUSED with the
# GEMM2 MXFP8 activation-quant in one fp32 Triton pass — no bf16 ``act``
# round-trip to HBM. Bit-exact vs the unfused swiglu+quant chain on measured
# MoE shapes, and ~1.2-1.9x faster on that step in isolation. (Not the #22
# ``silu_and_mul_with_clamp`` op: it rounds intermediates to bf16, rel ~3e-3.)
# Lazy import: the amd.ops package pulls in the minimax_m3 platform dispatch,
# only resolvable after the model module finishes loading.
from vllm.models.minimax_m3.amd.ops import swiglu_oai_quantize_mxfp8
# GEMM2: act (mxfp8) @ w2^T -> [M, H], weighted by topk_weights, then reduce.
act_q, act_s = swiglu_oai_quantize_mxfp8(g1, alpha=alpha, beta=beta, limit=limit)
g2 = _grouped_gemm_mxfp8(
act_q,
act_s,
w2,
w2_scale,
sorted_ids,
expert_ids,
num_post,
M,
top_k,
block_m,
torch.float32,
a_div=1,
mul_weight_by=topk_weights.reshape(-1).to(torch.float32),
expert_map=expert_map,
) # [M, H] == [T*top_k, H]
return g2.view(T, top_k, H).sum(dim=1).to(hidden_states.dtype)
class Mxfp8NativeTritonExperts(Mxfp8TritonExpertsBase):
"""Native MXFP8 MoE (CDNA4 ``dot_scaled``) on gfx950."""
@property
def quant_dtype(self) -> torch.dtype | str | None:
return self.quant_config.quant_dtype
@property
def block_shape(self) -> list[int] | None:
return self.quant_config.block_shape
@property
def expects_unquantized_inputs(self) -> bool:
# Activations are MXFP8-quantized inside ``fused_moe_mxfp8_native``.
return True
@staticmethod
def _supports_current_device() -> bool:
return current_platform.is_rocm() and current_platform.supports_mx()
def apply(
self,
output: torch.Tensor,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
a2_scale: torch.Tensor | None,
workspace13: torch.Tensor,
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
):
alpha = self.quant_config.gemm1_alpha
alpha = 1.702 if alpha is None else float(alpha)
beta = self.quant_config.gemm1_beta
beta = 1.0 if beta is None else float(beta)
limit = self.quant_config.gemm1_clamp_limit
limit = None if limit is None else float(limit)
out = fused_moe_mxfp8_native(
hidden_states,
w1,
self.w1_scale_val,
w2,
self.w2_scale_val,
topk_weights,
topk_ids,
alpha=alpha,
beta=beta,
limit=limit,
global_num_experts=global_num_experts,
expert_map=expert_map,
)
output.copy_(out)
@@ -64,6 +64,15 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
self.quantization_emulation = False
super().__init__(moe_config, quant_config)
self.gemm1_clamp_limit = quant_config.gemm1_clamp_limit
# Gated-activation params: silu == swigluoai with alpha=1, beta=0.
self.gemm1_alpha = (
quant_config.gemm1_alpha if quant_config.gemm1_alpha is not None else 1.0
)
self.gemm1_beta = (
quant_config.gemm1_beta if quant_config.gemm1_beta is not None else 0.0
)
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@@ -107,6 +116,7 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
MoEActivation.GELU,
MoEActivation.GELU_TANH,
MoEActivation.SWIGLUOAI,
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
MoEActivation.SWIGLUSTEP,
MoEActivation.SILU_NO_MUL,
MoEActivation.GELU_NO_MUL,
@@ -129,14 +139,34 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
return TopKWeightAndReduceNoOP()
def activation(
self, activation: MoEActivation, output: torch.Tensor, input: torch.Tensor
self,
activation: MoEActivation,
output: torch.Tensor,
input: torch.Tensor,
**kwargs,
) -> None:
gemm1_clamp_limit = self.quant_config.gemm1_clamp_limit
if activation == MoEActivation.SILU and gemm1_clamp_limit is not None:
swiglu_limit_func(output, input, float(gemm1_clamp_limit))
return
super().activation(activation, output, input)
# SWIGLUOAI_UNINTERLEAVE routes to the silu_and_mul_with_clamp kernel and
# needs the clamped-SwiGLU params (gemm1_clamp_limit/alpha/beta read from
# the quant config in __init__) forwarded; without a clamp_limit it
# asserts. Other activations ignore alpha/beta/clamp_limit.
if activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
assert gemm1_clamp_limit is not None, (
"SWIGLUOAI_UNINTERLEAVE requires gemm1_clamp_limit"
)
super().activation(
activation,
output,
input,
clamp_limit=gemm1_clamp_limit,
alpha=self.gemm1_alpha,
beta=self.gemm1_beta,
)
def workspace_shapes(
self,
@@ -120,6 +120,8 @@ def FusedMoE(
scoring_func: str = "softmax",
routed_scaling_factor: float = 1.0,
swiglu_limit: float | None = None,
swiglu_alpha: float | None = None,
swiglu_beta: float | None = None,
e_score_correction_bias: torch.Tensor | None = None,
apply_router_weight_on_input: bool = False,
activation: str = "silu",
@@ -322,6 +324,8 @@ def FusedMoE(
device=vllm_config.device_config.device,
routing_method=router.routing_method_type, # Not ideal
swiglu_limit=swiglu_limit,
swiglu_alpha=swiglu_alpha,
swiglu_beta=swiglu_beta,
max_capture_size=vllm_config.compilation_config.max_cudagraph_capture_size,
)
@@ -353,6 +357,8 @@ def FusedMoE(
if not apply_routed_scale_to_output
else 1.0,
swiglu_limit=swiglu_limit,
swiglu_alpha=swiglu_alpha,
swiglu_beta=swiglu_beta,
# TODO get from router? needs to be truncated?
e_score_correction_bias=e_score_correction_bias,
apply_router_weight_on_input=apply_router_weight_on_input,
@@ -880,9 +880,18 @@ class FusedMoEExpertsModular(FusedMoEExperts):
return N if not activation.is_gated else N // 2
def activation(
self, activation: MoEActivation, output: torch.Tensor, input: torch.Tensor
self,
activation: MoEActivation,
output: torch.Tensor,
input: torch.Tensor,
*,
clamp_limit: float | None = None,
alpha: float = 1.0,
beta: float = 0.0,
) -> None:
apply_moe_activation(activation, output, input)
apply_moe_activation(
activation, output, input, clamp_limit=clamp_limit, alpha=alpha, beta=beta
)
@abstractmethod
def finalize_weight_and_reduce_impl(self) -> TopKWeightAndReduce:
@@ -52,6 +52,13 @@ class Fp8MoeBackend(Enum):
BATCHED_VLLM_CUTLASS = "BATCHED_VLLM_CUTLASS"
XPU = "XPU"
CPU = "CPU"
# Dequantize-to-BF16 emulation for MXFP8 on devices without a native
# MXFP8 MoE kernel (e.g. ROCm). Weights pass through unchanged here.
EMULATION = "EMULATION"
# MXFP8 MoE via a Triton ``dot_scaled`` kernel that lowers to CDNA4
# (gfx950) native MX matrix-core ops. Weights stay in MXFP8 (no load-time
# format conversion); the FP8 values + E8M0 scales are consumed directly.
NATIVE_MXFP8 = "NATIVE_MXFP8"
def _get_priority_backends(
@@ -463,6 +470,10 @@ def convert_to_fp8_moe_kernel_format(
Fp8MoeBackend.VLLM_CUTLASS,
Fp8MoeBackend.BATCHED_VLLM_CUTLASS,
Fp8MoeBackend.XPU,
# EMULATION dequantizes weights at runtime; NATIVE_MXFP8 consumes
# the MXFP8 weights as-is — neither needs a load-time layout change.
Fp8MoeBackend.EMULATION,
Fp8MoeBackend.NATIVE_MXFP8,
]:
raise ValueError(f"Unsupported FP8 MoE backend: {fp8_backend.value}")
@@ -481,6 +492,8 @@ def make_fp8_moe_quant_config(
per_act_token_quant: bool = False,
per_out_ch_quant: bool = False,
swiglu_limit: float | None = None,
gemm1_alpha: float | None = None,
gemm1_beta: float | None = None,
) -> FusedMoEQuantConfig:
"""
Create FusedMoEQuantConfig for the specified FP8 Backend.
@@ -503,6 +516,9 @@ def make_fp8_moe_quant_config(
w1_bias=w1_bias,
w2_bias=w2_bias,
block_shape=block_shape,
gemm1_alpha=gemm1_alpha,
gemm1_beta=gemm1_beta,
gemm1_clamp_limit=swiglu_limit,
)
# Flashinfer CUTLASS per-tensor uses single dq scale
@@ -522,10 +538,9 @@ def make_fp8_moe_quant_config(
g2_alphas=(w2_scale * a2_scale).squeeze(),
gemm1_clamp_limit=swiglu_limit,
)
# MXFP8 uses "mxfp8" quant_dtype so the prepare step dispatches to
# _mxfp8_e4m3_quantize rather than standard FP8 block quantization.
# Non-swizzled layout is required since the TRTLLM kernel expects
# scales in (num_tokens, hidden_dim // 32) format.
# MXFP8 (block [1, 32]) dispatches to the mxfp8 activation quant. Scales are
# the non-swizzled (num_tokens, hidden_dim // 32) uint8 UE8M0 layout for all
# backends; the DeepGEMM expert permute repacks them for the grouped GEMM.
if block_shape == [1, 32]:
return FusedMoEQuantConfig.make(
"mxfp8",
@@ -537,6 +552,8 @@ def make_fp8_moe_quant_config(
a2_scale=a2_scale,
block_shape=block_shape,
is_scale_swizzled=False,
gemm1_alpha=gemm1_alpha,
gemm1_beta=gemm1_beta,
gemm1_clamp_limit=swiglu_limit,
)
@@ -12,22 +12,43 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kMxfp8Dynamic,
kMxfp8Static,
)
from vllm.platforms import current_platform
logger = init_logger(__name__)
_SUPPORTED_BACKENDS = (
Fp8MoeBackend.FLASHINFER_TRTLLM,
Fp8MoeBackend.DEEPGEMM,
Fp8MoeBackend.MARLIN,
Fp8MoeBackend.XPU,
)
_BACKEND_NAME_MAP: dict[str, Fp8MoeBackend] = {
"flashinfer_trtllm": Fp8MoeBackend.FLASHINFER_TRTLLM,
"deep_gemm": Fp8MoeBackend.DEEPGEMM,
"marlin": Fp8MoeBackend.MARLIN,
"xpu": Fp8MoeBackend.XPU,
}
def _mxfp8_backend_to_kernel_cls(
backend: Fp8MoeBackend,
) -> list[type[mk.FusedMoEExperts]]:
"""Resolve the MXFP8 expert classes for a backend.
DeepGEMM resolves directly to ``DeepGemmExperts`` (not the
``TritonOrDeepGemmExperts`` wrapper, whose Triton fallback cannot handle the
MXFP8 1x32 scheme); all other backends defer to the FP8 resolver.
"""
if backend == Fp8MoeBackend.DEEPGEMM:
from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import (
DeepGemmExperts,
)
return [DeepGemmExperts]
return backend_to_kernel_cls(backend)
def _select_kernel_cls(
backend: Fp8MoeBackend,
config: FusedMoEConfig,
@@ -39,7 +60,7 @@ def _select_kernel_cls(
else mk.FusedMoEActivationFormat.Standard
)
last_reason: str | None = None
for cls in backend_to_kernel_cls(backend):
for cls in _mxfp8_backend_to_kernel_cls(backend):
supported, reason = cls.is_supported_config(
cls,
config,
@@ -55,6 +76,29 @@ def _select_kernel_cls(
)
def _select_rocm_mxfp8_backend() -> tuple[Fp8MoeBackend, type[mk.FusedMoEExperts]]:
"""ROCm fallback when vendor MXFP8 backends are unavailable."""
if current_platform.supports_mx():
from vllm.model_executor.layers.fused_moe.experts.mxfp8_native_moe import (
Mxfp8NativeTritonExperts,
)
logger.info_once("Using native CDNA4 (gfx950) MXFP8 dot_scaled MoE backend.")
return Fp8MoeBackend.NATIVE_MXFP8, Mxfp8NativeTritonExperts
from vllm.model_executor.layers.fused_moe.experts.mxfp8_emulation_moe import (
Mxfp8EmulationTritonExperts,
)
logger.info_once(
"No native MXFP8 MoE backend available on this device; "
"MXFP8 weights will be dequantized to BF16 once at load time and the "
"MoE will run in BF16 (no per-step dequant)."
)
return Fp8MoeBackend.EMULATION, Mxfp8EmulationTritonExperts
def select_mxfp8_moe_backend(
config: FusedMoEConfig,
) -> tuple[Fp8MoeBackend, type[mk.FusedMoEExperts]]:
@@ -88,4 +132,8 @@ def select_mxfp8_moe_backend(
logger.info_once("Using '%s' MxFp8 MoE backend.", backend.value)
return backend, experts_cls
# simplify the logic for rocm, refactor later when more backends are supported
if current_platform.is_rocm():
return _select_rocm_mxfp8_backend()
raise ValueError("No MXFP8 MoE backends available.")
@@ -72,6 +72,8 @@ class RoutedExperts(PluggableLayer):
scoring_func: str = "softmax",
routed_scaling_factor: float = 1.0,
swiglu_limit: float | None = None,
swiglu_alpha: float | None = None,
swiglu_beta: float | None = None,
e_score_correction_bias: torch.Tensor | None = None,
apply_router_weight_on_input: bool = False,
):
@@ -103,6 +105,8 @@ class RoutedExperts(PluggableLayer):
self.scoring_func = scoring_func
self.routed_scaling_factor = routed_scaling_factor
self.swiglu_limit = swiglu_limit
self.swiglu_alpha = swiglu_alpha
self.swiglu_beta = swiglu_beta
self.e_score_correction_bias = e_score_correction_bias
self.apply_router_weight_on_input = apply_router_weight_on_input
# End random parameters
@@ -29,9 +29,9 @@ class GateLinear(ReplicatedLinear):
DSV3_SUPPORTED_NUM_EXPERTS = [256, 384]
DSV3_SUPPORTED_HIDDEN_SIZES = [7168]
# Dimensions supported by the fp32 specialized kernel
FP32_SUPPORTED_NUM_EXPERTS = [256]
FP32_SUPPORTED_HIDDEN_SIZES = [3072]
# (hidden_size, num_experts) pairs with an instantiated fp32 kernel:
# (3072, 256) -> MiniMax-M2/M2.5, (6144, 128) -> MiniMax-M3
FP32_SUPPORTED_SHAPES = {(3072, 256), (6144, 128)}
FP32_MAX_TOKENS = 32
def __init__(
@@ -82,8 +82,7 @@ class GateLinear(ReplicatedLinear):
and self.weight.dtype == torch.float32
and current_platform.is_cuda()
and (is_hopper or is_blackwell)
and output_size in self.FP32_SUPPORTED_NUM_EXPERTS
and input_size in self.FP32_SUPPORTED_HIDDEN_SIZES
and (input_size, output_size) in self.FP32_SUPPORTED_SHAPES
)
# cuBLAS bf16→fp32 eligibility
@@ -11,7 +11,6 @@ import vllm.envs as envs
from vllm.logger import init_logger
from vllm.model_executor.custom_op import CustomOp
from vllm.model_executor.layers.fused_moe.config import (
FUSED_MOE_UNQUANTIZED_CONFIG,
FusedMoEConfig,
FusedMoEQuantConfig,
biased_moe_quant_config,
@@ -184,11 +183,10 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
if not is_weight_update:
# Setup moe kernel only on the first call. For the unquantized
# method, moe_quant_config is either the constant
# FUSED_MOE_UNQUANTIZED_CONFIG or biased_moe_quant_config(...)
# which references layer.w{13,2}_bias; since weight updates
# mutate those bias tensors in place, the kernel does not need
# to be re-built.
# method, moe_quant_config carries no quantized scales -- only
# optional w{13,2}_bias references and SwiGLU gate params. Since
# weight updates mutate those bias tensors in place, the kernel
# does not need to be re-built.
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
assert self.moe_quant_config is not None
assert self.experts_cls is not None
@@ -272,13 +270,27 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
)
def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig:
# SwiGLU/swigluoai gate params live on the layer; plumb them into the
# quant config so the fused activation (e.g. swigluoai_uninterleave on
# MiniMax-M3) receives gemm1_clamp_limit/alpha/beta.
gemm1_alpha = getattr(layer, "swiglu_alpha", None)
gemm1_beta = getattr(layer, "swiglu_beta", None)
gemm1_clamp_limit = getattr(layer, "swiglu_limit", None)
if self.moe.has_bias:
return biased_moe_quant_config(
layer.w13_bias,
layer.w2_bias,
gemm1_alpha=gemm1_alpha,
gemm1_beta=gemm1_beta,
gemm1_clamp_limit=gemm1_clamp_limit,
)
else:
return FUSED_MOE_UNQUANTIZED_CONFIG
return FusedMoEQuantConfig.make(
gemm1_alpha=gemm1_alpha,
gemm1_beta=gemm1_beta,
gemm1_clamp_limit=gemm1_clamp_limit,
)
def apply(
self,
@@ -313,6 +313,8 @@ def moe_kernel_quantize_input(
"moe_kernel_quantize_input does not support quant_dtype='mxfp8' MOE "
"quantization emulation. Please open an issue."
)
# Non-swizzled (M, K/32) uint8 UE8M0 scales; deepgemm_moe_permute packs
# them for DeepGEMM, TRTLLM takes them as-is.
return _mxfp8_e4m3_quantize(
A,
A_scale,
+186
View File
@@ -1105,6 +1105,7 @@ class QKVParallelLinear(ColumnParallelLinear):
shard_offset = self._get_shard_offset_mapping(loaded_shard_id)
shard_size = self._get_shard_size_mapping(loaded_shard_id)
assert shard_offset is not None and shard_size is not None
if isinstance(param, BlockQuantScaleParameter):
weight_block_size = getattr(self, "weight_block_size", None)
@@ -1302,6 +1303,191 @@ class QKVParallelLinear(ColumnParallelLinear):
param_data.copy_(loaded_weight)
class MinimaxM3QKVParallelLinearWithIndexer(QKVParallelLinear):
"""QKV projection fused with a lightning-indexer's index_q/index_k.
NOTE: MiniMax-M3-specific. This is tailored to the M3 sparse-attention
layers (it assumes the indexer's head count equals the KV head count and
shares the main head_dim); it is not a general-purpose linear layer. It
lives here only to sit alongside QKVParallelLinear, whose sharding /
weight-loading machinery it reuses.
A single column-parallel GEMM emits, per rank::
[q | k | v | index_q | index_k]
``index_q`` must have the same head count as the KV heads
(``total_num_index_heads == total_num_kv_heads``) and ``index_head_size ==
head_size``, so it shards exactly like K/V -- including the KV-head
*replication* path when ``tp_size > total_num_kv_heads`` (this is what makes
a TP size greater than the KV-head count work). ``index_k`` is a single
shared head, replicated to every rank.
"""
def __init__(
self,
hidden_size: int,
head_size: int,
total_num_heads: int,
total_num_kv_heads: int,
total_num_index_heads: int,
index_head_size: int,
bias: bool = False,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
# index_q rides the KV-head sharding/replication path, so its head count
# must match the KV heads.
assert total_num_index_heads == total_num_kv_heads, (
"MinimaxM3QKVParallelLinearWithIndexer requires "
"total_num_index_heads == total_num_kv_heads"
)
self.hidden_size = hidden_size
self.head_size = head_size
self.v_head_size = head_size
self.total_num_heads = total_num_heads
self.total_num_kv_heads = total_num_kv_heads
self.total_num_index_heads = total_num_index_heads
self.index_head_size = index_head_size
tp_size = get_tensor_model_parallel_world_size()
self.num_heads = divide(self.total_num_heads, tp_size)
if tp_size >= self.total_num_kv_heads:
self.num_kv_heads = 1
self.num_kv_head_replicas = divide(tp_size, self.total_num_kv_heads)
else:
self.num_kv_heads = divide(self.total_num_kv_heads, tp_size)
self.num_kv_head_replicas = 1
# index_q shards identically to the KV heads.
self.num_index_heads = self.num_kv_heads
# Global per-group sizes (replicated groups counted x tp_size, matching
# the QKVParallelLinear convention). index_k is a single replicated head.
q = self.num_heads * self.head_size
kv = self.num_kv_heads * self.head_size
iq = self.num_index_heads * self.index_head_size
ik = self.index_head_size
self.output_sizes = [
q * tp_size, # q
kv * tp_size, # k
kv * tp_size, # v
iq * tp_size, # index_q
ik * tp_size, # index_k (replicated)
]
# Skip QKVParallelLinear.__init__ (3-group layout); build the 5-group
# column-parallel weight directly.
ColumnParallelLinear.__init__(
self,
input_size=self.hidden_size,
output_size=sum(self.output_sizes),
bias=bias,
gather_output=False,
quant_config=quant_config,
prefix=prefix,
)
def validate_shard_id(self, loaded_shard_id: str | None) -> None:
if loaded_shard_id is None:
return
if loaded_shard_id not in ("q", "k", "v", "index_q", "index_k"):
raise ValueError(
"Shard id for MinimaxM3QKVParallelLinearWithIndexer must be one of "
"'q', 'k', 'v', 'index_q', 'index_k'; got "
f"{loaded_shard_id}."
)
def _get_shard_offset_mapping(self, loaded_shard_id: str) -> int | None:
h = self.head_size
nq, nkv, nidx = self.num_heads, self.num_kv_heads, self.num_index_heads
return {
"q": 0,
"k": nq * h,
"v": (nq + nkv) * h,
"index_q": (nq + 2 * nkv) * h,
"index_k": (nq + 2 * nkv + nidx) * h,
}.get(loaded_shard_id)
def _get_shard_size_mapping(self, loaded_shard_id: str) -> int | None:
h = self.head_size
return {
"q": self.num_heads * h,
"k": self.num_kv_heads * h,
"v": self.num_kv_heads * h,
"index_q": self.num_index_heads * h,
"index_k": self.index_head_size,
}.get(loaded_shard_id)
def weight_loader_v2(
self,
param: BasevLLMParameter,
loaded_weight: torch.Tensor,
loaded_shard_id: str | None = None,
) -> None:
self.validate_shard_id(loaded_shard_id)
# Index checkpoints are never pre-fused on disk; a shard id is always given.
assert loaded_shard_id in ("q", "k", "v", "index_q", "index_k")
shard_offset = self._get_shard_offset_mapping(loaded_shard_id)
shard_size = self._get_shard_size_mapping(loaded_shard_id)
assert shard_offset is not None and shard_size is not None
if isinstance(param, BlockQuantScaleParameter):
weight_block_size = getattr(self, "weight_block_size", None)
shard_size, shard_offset = adjust_block_scale_shard(
weight_block_size, shard_size, shard_offset
)
# index_k is fully replicated: num_heads == tp_size makes
# load_qkv_weight pick shard_id_int == 0 on every rank. q/k/v/index_q ride
# the KV-head replication factor.
num_heads = (
self.tp_size if loaded_shard_id == "index_k" else self.num_kv_head_replicas
)
param.load_qkv_weight(
loaded_weight=loaded_weight,
num_heads=num_heads,
shard_id=loaded_shard_id,
shard_offset=shard_offset,
shard_size=shard_size,
tp_rank=self.tp_rank,
)
def weight_loader(
self,
param: Parameter,
loaded_weight: torch.Tensor,
loaded_shard_id: str | None = None,
) -> None:
# Unquantized (bf16) path. MXFP8 checkpoints use weight_loader_v2; this
# keeps an unquantized load correct too.
self.validate_shard_id(loaded_shard_id)
assert loaded_shard_id in ("q", "k", "v", "index_q", "index_k")
output_dim = getattr(param, "output_dim", None)
assert output_dim is not None
shard_offset = self._get_shard_offset_mapping(loaded_shard_id)
shard_size = self._get_shard_size_mapping(loaded_shard_id)
assert shard_offset is not None and shard_size is not None
if isinstance(param, BlockQuantScaleParameter):
weight_block_size = getattr(self, "weight_block_size", None)
shard_size, shard_offset = adjust_block_scale_shard(
weight_block_size, shard_size, shard_offset
)
param_data = param.data.narrow(output_dim, shard_offset, shard_size)
if loaded_shard_id == "q":
shard_rank = self.tp_rank
elif loaded_shard_id == "index_k":
shard_rank = 0 # replicated to every rank
else:
shard_rank = self.tp_rank // self.num_kv_head_replicas
loaded_weight = loaded_weight.narrow(
output_dim, shard_rank * shard_size, shard_size
)
assert param_data.shape == loaded_weight.shape
param_data.copy_(loaded_weight)
# --8<-- [start:row_parallel_linear]
@PluggableLayer.register("row_parallel_linear")
class RowParallelLinear(LinearBase):
@@ -163,17 +163,18 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
"deepseek_v4_fp8": DeepseekV4FP8Config,
"humming": HummingConfig,
"online": OnlineQuantizationConfig,
# MiniMax-style checkpoints tag `quant_method: "mxfp8"`; load with the
# ModelOpt MXFP8 config (same format). The "mxfp8" online shorthand
# below only applies to the `--quantization mxfp8` CLI path.
"mxfp8": ModelOptMxFp8Config,
}
# Register online shorthands as quantization methods so the user can
# specify "LLM(..., quantization='fp8_per_tensor')" as shorthand for
# creating a more complicated online quant config object.
# Register online shorthands (e.g. "fp8_per_tensor") as quant methods.
# setdefault so a shorthand that is also a checkpoint method (e.g. "mxfp8")
# keeps its checkpoint config; the shorthand still works via the
# `--quantization` CLI path in `resolve_quantization_config`.
for shorthand in _ONLINE_SHORTHANDS:
assert shorthand not in method_to_config, (
f"Online quant shorthand {shorthand!r} conflicts with an "
f"existing quantization method"
)
method_to_config[shorthand] = OnlineQuantizationConfig
method_to_config.setdefault(shorthand, OnlineQuantizationConfig)
# Update the `method_to_config` with customized quantization methods.
method_to_config.update(_CUSTOMIZED_METHOD_TO_QUANT_CONFIG)
@@ -2,11 +2,12 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from fnmatch import fnmatch
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, cast
import torch
from torch.nn.parameter import Parameter
import vllm.envs as envs
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.config import get_current_vllm_config
from vllm.logger import init_logger
@@ -27,6 +28,7 @@ from vllm.model_executor.layers.fused_moe import (
SharedExperts,
)
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
Fp8MoeBackend,
convert_to_fp8_moe_kernel_format,
make_fp8_moe_kernel,
make_fp8_moe_quant_config,
@@ -1720,6 +1722,22 @@ class ModelOptMxFp8Config(ModelOptQuantConfigBase):
return "modelopt_mxfp8"
return None
@classmethod
def from_config(cls, config: dict[str, Any]) -> "ModelOptMxFp8Config":
# MiniMax-style checkpoints tag `quant_method: "mxfp8"` + `ignored_layers`
# (same on-disk format as ModelOpt MXFP8); normalize to the ModelOpt
# schema and reuse the shared parser.
if "quantization" not in config and not config.get("quant_algo"):
config = {
"quant_method": "modelopt",
"quantization": {
"quant_algo": "MXFP8",
"kv_cache_quant_algo": config.get("kv_cache_quant_algo"),
"exclude_modules": config.get("ignored_layers", []) or [],
},
}
return cast("ModelOptMxFp8Config", super().from_config(config))
@classmethod
def _from_config(
cls,
@@ -1823,6 +1841,12 @@ class ModelOptMxFp8LinearMethod(LinearMethodBase):
layer.register_parameter("weight_scale", weight_scale)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# Idempotent: the emulation kernel may dequant the weight to BF16 at load
# time (>=2-byte). If already converted, there is nothing left to do --
# avoid re-running the MXFP8-only validation/conversion below.
if layer.weight.element_size() >= 2:
return
# Validate weight tensor
if layer.weight.ndim != 2:
raise ValueError(
@@ -2065,6 +2089,44 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
torch.stack(w2_scale_shuffled).contiguous(),
)
def _dequant_mxfp8_weights_to_bf16(self, layer: RoutedExperts) -> None:
"""One-time MXFP8->BF16 weight dequant for the emulation path.
On devices without a native MXFP8 MoE kernel (e.g. gfx942 / MI300),
``Mxfp8EmulationTritonExperts`` otherwise dequantizes every expert
weight to BF16 on *every* forward step -- the dominant cost (conc1
~1.3 tok/s). Doing the dequant once here and replacing the MXFP8
parameters with BF16 makes the MoE run exactly like a plain BF16
checkpoint (full precision, no per-step dequant); SwiGLU-OAI is still
applied by the experts' ``activation()`` override. The MXFP8 weights
are freed by ``replace_parameter`` (BF16 is 2x their size; the small
E8M0 scale tensors are left in place, unused).
"""
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
dequant_mxfp8_to_bf16,
)
target_dtype = getattr(layer, "orig_dtype", torch.bfloat16)
num_experts = layer.w13_weight.shape[0]
# dequant_mxfp8_to_bf16 handles arbitrary leading dims (*x.shape[:-1]),
# so dequant the whole [E, N, K] weight in one vectorized call.
w13_bf16 = dequant_mxfp8_to_bf16(layer.w13_weight, layer.w13_weight_scale).to(
target_dtype
)
w2_bf16 = dequant_mxfp8_to_bf16(layer.w2_weight, layer.w2_weight_scale).to(
target_dtype
)
replace_parameter(layer, "w13_weight", w13_bf16)
replace_parameter(layer, "w2_weight", w2_bf16)
logger.info_once(
"MXFP8->BF16 load-time dequant complete (%d experts/layer); MoE "
"now runs in BF16 with no per-step dequant.",
num_experts,
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
# TODO(bnell): why is this required only for mxfp8?
if getattr(layer, "_already_called_process_weights_after_loading", False):
@@ -2102,6 +2164,17 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
routing_tables=layer._expert_routing_tables(),
)
# No native MXFP8 MoE kernel on this device (e.g. gfx942): the emulation
# experts would dequant MXFP8->BF16 every forward step. Convert the
# weights to BF16 once, here, so the MoE runs like a BF16 checkpoint.
# Opt out (VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD=0) to keep the 1-byte
# MXFP8 weights and dequant per-step (~half the memory, much slower).
if (
self.mxfp8_backend == Fp8MoeBackend.EMULATION
and envs.VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD
):
self._dequant_mxfp8_weights_to_bf16(layer)
def maybe_make_prepare_finalize(
self,
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
@@ -2131,6 +2204,9 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
a1_scale=None,
a2_scale=None,
block_shape=self.weight_block_size,
swiglu_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
)
def apply_monolithic(
@@ -159,75 +159,104 @@ def _silu_mul_quant_fp8_packed_kernel(
output_q_stride_m,
output_scale_stride_k,
clamp_limit,
alpha,
beta,
N: tl.constexpr,
NUM_GROUPS: tl.constexpr,
GROUPS_PER_ROW: tl.constexpr,
PACKS_PER_ROW: tl.constexpr,
fp8_min: tl.constexpr,
fp8_max: tl.constexpr,
GROUP_SIZE: tl.constexpr,
PACKS_PER_CTA: tl.constexpr,
BLOCK_M: tl.constexpr,
HAS_CLAMP: tl.constexpr,
):
N_2: tl.constexpr = N // 2
GROUPS_PER_PACK: tl.constexpr = 4
hidden_size: tl.constexpr = N // 2
pid_pack = tl.program_id(0)
pid_m = tl.program_id(1)
m_offset = pid_m.to(tl.int64) * BLOCK_M
pack_tile = tl.program_id(0)
row_start = tl.program_id(1).to(tl.int64) * BLOCK_M
row_step = tl.num_programs(1).to(tl.int64) * BLOCK_M
if m_offset >= M:
return
groups_per_cta: tl.constexpr = PACKS_PER_CTA * GROUPS_PER_PACK
elems_per_cta: tl.constexpr = groups_per_cta * GROUP_SIZE
col_start = pack_tile * elems_per_cta
col_offsets = tl.arange(0, elems_per_cta)
row_offsets = tl.arange(0, BLOCK_M)
pack_offsets = tl.arange(0, PACKS_PER_CTA)
offs_m = tl.arange(0, BLOCK_M)
offs_n = tl.arange(0, GROUP_SIZE)
row_mask = (m_offset + offs_m) < M
col_mask = (col_start + col_offsets) < (GROUPS_PER_ROW * GROUP_SIZE)
base_row_offset = (m_offset + offs_m[:, None]) * input_stride_m
base_out_offset = (m_offset + offs_m[:, None]) * output_q_stride_m
# persistent with grid_m-stride loop
while row_start < M:
rows = row_start + row_offsets
row_mask = rows < M
input_row_start = rows[:, None] * input_stride_m
output_row_start = rows[:, None] * output_q_stride_m
packed_scale = tl.zeros((BLOCK_M,), dtype=tl.int32)
gate_flat = tl.load(
input_ptr + input_row_start + col_start + col_offsets[None, :],
mask=row_mask[:, None] & col_mask[None, :],
other=0.0,
)
up_flat = tl.load(
input_ptr
+ input_row_start
+ hidden_size
+ col_start
+ col_offsets[None, :],
mask=row_mask[:, None] & col_mask[None, :],
other=0.0,
)
for pack_idx in tl.static_range(4):
group_id = pid_pack * 4 + pack_idx
gate = tl.reshape(gate_flat, (BLOCK_M, groups_per_cta, GROUP_SIZE)).to(
tl.float32
)
up = tl.reshape(up_flat, (BLOCK_M, groups_per_cta, GROUP_SIZE)).to(tl.float32)
if group_id < NUM_GROUPS:
n_offset = group_id * GROUP_SIZE
if HAS_CLAMP:
gate = tl.minimum(gate, clamp_limit)
up = tl.clamp(up, -clamp_limit, clamp_limit)
act_ptrs = input_ptr + base_row_offset + n_offset + offs_n[None, :]
act_in = tl.load(act_ptrs, mask=row_mask[:, None], other=0.0)
# Unified gated activation: silu == swigluoai with alpha=1, beta=0.
# glu = gate * sigmoid(alpha * gate); y = (up + beta) * glu
glu = gate / (1.0 + tl.exp(-gate * alpha))
y = (up + beta) * glu
# Round through bf16 to match unfused precision path
y = y.to(tl.bfloat16).to(tl.float32)
mul_ptrs = act_ptrs + N_2
mul_in = tl.load(mul_ptrs, mask=row_mask[:, None], other=0.0)
absmax = tl.max(tl.abs(y), axis=2)
scale_raw = tl.maximum(absmax / fp8_max, 1e-10)
exponent = tl.ceil(tl.log2(scale_raw))
scale = tl.math.exp2(exponent)
act_f32 = act_in.to(tl.float32)
mul_f32 = mul_in.to(tl.float32)
y_q = tl.clamp(y / scale[:, :, None], fp8_min, fp8_max)
if HAS_CLAMP:
act_f32 = tl.minimum(act_f32, clamp_limit)
mul_f32 = tl.clamp(mul_f32, -clamp_limit, clamp_limit)
y_q_flat = tl.reshape(y_q, (BLOCK_M, elems_per_cta))
tl.store(
output_q_ptr + output_row_start + col_start + col_offsets[None, :],
y_q_flat.to(output_q_ptr.dtype.element_ty),
mask=row_mask[:, None] & col_mask[None, :],
)
y = (act_f32 / (1.0 + tl.exp(-act_f32))) * mul_f32
# Round through bf16 to match unfused precision path
y = y.to(tl.bfloat16).to(tl.float32)
scale_byte = tl.clamp(exponent + 127.0, 0.0, 255.0).to(tl.int32)
scale_bytes = tl.reshape(scale_byte, (BLOCK_M, PACKS_PER_CTA, GROUPS_PER_PACK))
shifts = tl.arange(0, GROUPS_PER_PACK) * 8
packed_scale = tl.sum(scale_bytes << shifts[None, None, :], axis=2)
absmax = tl.max(tl.abs(y), axis=1)
scale_pack = pack_tile * PACKS_PER_CTA + pack_offsets
scale_ptrs = (
output_scale_ptr
+ scale_pack[None, :] * output_scale_stride_k
+ rows[:, None]
)
tl.store(
scale_ptrs,
packed_scale,
mask=row_mask[:, None] & (scale_pack[None, :] < PACKS_PER_ROW),
)
scale_raw = tl.maximum(absmax / fp8_max, 1e-10)
exponent = tl.ceil(tl.log2(scale_raw))
scale = tl.math.exp2(exponent)
y_q = tl.clamp(y / scale[:, None], fp8_min, fp8_max)
out_q_ptrs = output_q_ptr + base_out_offset + n_offset + offs_n[None, :]
tl.store(
out_q_ptrs,
y_q.to(output_q_ptr.dtype.element_ty),
mask=row_mask[:, None],
)
exponent_biased = tl.clamp(exponent + 127.0, 0.0, 255.0).to(tl.int32)
packed_scale = packed_scale | (exponent_biased << (pack_idx * 8))
scale_ptrs = output_scale_ptr + pid_pack * output_scale_stride_k + m_offset + offs_m
tl.store(scale_ptrs, packed_scale, mask=row_mask)
row_start += row_step
def silu_mul_quant_fp8_packed_triton(
@@ -235,37 +264,48 @@ def silu_mul_quant_fp8_packed_triton(
group_size: int = 128,
output_q: torch.Tensor | None = None,
clamp_limit: float | None = None,
alpha: float = 1.0,
beta: float = 0.0,
) -> tuple[torch.Tensor, torch.Tensor]:
assert input.dim() == 2
assert input.is_contiguous()
M, N = input.shape
N_2 = N // 2
hidden_size = N // 2
assert N_2 % group_size == 0
assert hidden_size % group_size == 0
fp8_dtype = torch.float8_e4m3fn
finfo = torch.finfo(fp8_dtype)
fp8_min, fp8_max = finfo.min, finfo.max
num_groups_per_row = N_2 // group_size
num_packed_groups = (num_groups_per_row + 3) // 4
tma_aligned_M = ((M + 3) // 4) * 4
groups_per_row = hidden_size // group_size
groups_per_pack = 4 # pack 4 UE8M0 scales to a single INT32
packs_per_row = triton.cdiv(groups_per_row, groups_per_pack)
if output_q is None:
output_q = torch.empty((M, N_2), dtype=fp8_dtype, device=input.device)
output_q = torch.empty((M, hidden_size), dtype=fp8_dtype, device=input.device)
aligned_m = triton.cdiv(M, 4) * 4
output_scale_packed = torch.empty(
(num_packed_groups, tma_aligned_M),
(packs_per_row, aligned_m),
dtype=torch.int32,
device=input.device,
).T[:M, :]
BLOCK_M = 8
grid = (num_packed_groups, (M + BLOCK_M - 1) // BLOCK_M)
num_warps = max(4, group_size // 32)
# Tuned for group_size=32 (MXFP8) and group_size=128 (DeepSeek-V4)
num_warps = 4
num_stages = 2
if group_size < 128:
BM = 1
packs_per_cta = 8
else:
BM = 1 if M < 512 else 4
packs_per_cta = 2 if M < 512 else 1
grid_n = triton.cdiv(packs_per_row, packs_per_cta)
grid_m = min(triton.cdiv(M, BM), 4096)
grid = (grid_n, grid_m)
has_clamp = clamp_limit is not None
_silu_mul_quant_fp8_packed_kernel[grid](
@@ -277,12 +317,16 @@ def silu_mul_quant_fp8_packed_triton(
output_q.stride(0),
output_scale_packed.stride(1),
clamp_limit if has_clamp else 0.0,
alpha,
beta,
N=N,
NUM_GROUPS=num_groups_per_row,
GROUPS_PER_ROW=groups_per_row,
PACKS_PER_ROW=packs_per_row,
fp8_min=fp8_min,
fp8_max=fp8_max,
GROUP_SIZE=group_size,
BLOCK_M=BLOCK_M,
PACKS_PER_CTA=packs_per_cta,
BLOCK_M=BM,
HAS_CLAMP=has_clamp,
num_warps=num_warps,
num_stages=num_stages,
@@ -303,6 +347,8 @@ def _silu_mul_per_token_group_quant_fp8_colmajor(
# Information for float8
eps,
clamp_limit,
alpha,
beta,
fp8_min: tl.constexpr,
fp8_max: tl.constexpr,
use_ue8m0: tl.constexpr,
@@ -348,10 +394,14 @@ def _silu_mul_per_token_group_quant_fp8_colmajor(
mul_in = tl.clamp(mul_in.to(tl.float32), -clamp_limit, clamp_limit).to(
y_ptr.dtype.element_ty
)
# Unified gated activation: silu == swigluoai with alpha=1, beta=0.
# glu = gate * sigmoid(alpha * gate); y = (up + beta) * glu
# Keep glu/up at input precision (narrow before the mul) so the alpha=1,
# beta=0 defaults match the C++ silu_and_mul path bit-for-bit.
act_in = act_in.to(tl.float32)
one_f32 = tl.cast(1, tl.float32)
silu_out = (act_in / (one_f32 + tl.exp(-act_in))).to(y_ptr.dtype.element_ty)
y = (silu_out * mul_in).to(tl.float32)
glu = (act_in / (1.0 + tl.exp(-act_in * alpha))).to(y_ptr.dtype.element_ty)
up = (mul_in.to(tl.float32) + beta).to(y_ptr.dtype.element_ty)
y = (glu * up).to(tl.float32)
# quant
_absmax = tl.maximum(tl.max(tl.abs(y), axis=1), eps)
@@ -379,11 +429,15 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
use_ue8m0: bool | None = None,
eps: float = 1e-10,
clamp_limit: float | None = None,
group_size: int = 128,
alpha: float = 1.0,
beta: float = 0.0,
):
"""
silu+mul + block-fp8 quant with group size 128.
Gated activation + block-fp8 quant. ``alpha``/``beta`` select the gate
(silu: alpha=1, beta=0; swigluoai: alpha, beta from config).
"""
GROUP_SIZE = 128
GROUP_SIZE = group_size
assert input.ndim == 2
if output is not None:
assert output.ndim == 2
@@ -431,6 +485,8 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
output_scales.stride(-1),
eps,
clamp_limit if has_clamp else 0.0,
alpha,
beta,
fp8_min,
fp8_max,
use_ue8m0,
@@ -1015,9 +1071,10 @@ def deepgemm_post_process_fp8_weight_block(
f"to be torch.float8_e4m3fn, got {wq.dtype} instead."
)
if ws.dtype == torch.float8_e8m0fnu:
# Scales already in E8M0 from checkpoint — upcast to fp32
# and skip requantization (weights already have power-of-two scales).
if ws.dtype in (torch.float8_e8m0fnu, torch.uint8):
# Scales already in E8M0 from checkpoint (float8_e8m0fnu, or raw E8M0
# bits as uint8 for MXFP8) — upcast to fp32 and skip requantization
# (weights already have power-of-two scales).
ws = _upcast_e8m0_to_fp32(ws)
else:
assert ws.dtype == torch.float32, (
@@ -1057,7 +1114,8 @@ def deepgemm_post_process_fp8_weight_block(
ws = ws.unsqueeze(0)
# From https://github.com/deepseek-ai/DeepGEMM/blob/c9f8b34dcdacc20aa746b786f983492c51072870/csrc/utils/layout.hpp#L46
recipe = (1, 128, 128)
# (1, block_n, block_k): (1, 128, 128) for FP8 block, (1, 1, 32) for MXFP8.
recipe = (1, quant_block_shape[0], quant_block_shape[1])
# Ref : https://github.com/deepseek-ai/DeepGEMM/blob/c9f8b34dcdacc20aa746b786f983492c51072870/csrc/apis/gemm.hpp
# DeepGemm uses the `transform_sf_into_required_layout` function to
@@ -84,6 +84,92 @@ def _mxfp8_e4m3_quantize_torch(
return x_fp8, scales_uint8
def _mxfp8_quant_triton_kernel():
"""Lazily-built Triton kernel: per-32-block E8M0 scale + FP8-E4M3 quant.
Fuses what ``_mxfp8_e4m3_quantize_torch`` does in several elementwise passes
into one launch. Each program handles ``[BLOCK_M, 32]`` (one MX block).
"""
from vllm.triton_utils import tl, triton
@triton.jit
def _kernel(
x_ptr,
xq_ptr,
s_ptr,
M,
K,
sxm,
sxk,
sqm,
sqk,
ssm,
ssk,
BLOCK_M: tl.constexpr,
):
pid_m = tl.program_id(0)
pid_b = tl.program_id(1) # which 32-element block along K
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
offs_k = pid_b * 32 + tl.arange(0, 32)
m_mask = offs_m < M
x = tl.load(
x_ptr + offs_m[:, None] * sxm + offs_k[None, :] * sxk,
mask=m_mask[:, None],
other=0.0,
).to(tl.float32)
amax = tl.maximum(tl.max(tl.abs(x), axis=1), 1e-30) # [BLOCK_M]
sb = tl.floor(tl.log2(amax)) + 127.0
sb = tl.minimum(tl.maximum(sb, 0.0), 254.0)
descale = tl.exp2(sb - 127.0)
xq = (x / descale[:, None]).to(xq_ptr.dtype.element_ty)
tl.store(
xq_ptr + offs_m[:, None] * sqm + offs_k[None, :] * sqk,
xq,
mask=m_mask[:, None],
)
tl.store(s_ptr + offs_m * ssm + pid_b * ssk, sb.to(tl.uint8), mask=m_mask)
return _kernel
_MXFP8_QUANT_KERNEL = None
def _mxfp8_e4m3_quantize_triton(
x: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Fused 2D MXFP8 quant (non-swizzled, row-major [M, K//32] scales)."""
from vllm.triton_utils import triton
global _MXFP8_QUANT_KERNEL
if _MXFP8_QUANT_KERNEL is None:
_MXFP8_QUANT_KERNEL = _mxfp8_quant_triton_kernel()
M, K = x.shape
x = x.contiguous()
xq = torch.empty((M, K), dtype=MXFP8_VALUE_DTYPE, device=x.device)
scales = torch.empty(
(M, K // MXFP8_BLOCK_SIZE), dtype=MXFP8_SCALE_DTYPE, device=x.device
)
BLOCK_M = 64
grid = (triton.cdiv(M, BLOCK_M), K // MXFP8_BLOCK_SIZE)
_MXFP8_QUANT_KERNEL[grid](
x,
xq,
scales,
M,
K,
x.stride(0),
x.stride(1),
xq.stride(0),
xq.stride(1),
scales.stride(0),
scales.stride(1),
BLOCK_M=BLOCK_M,
)
return xq, scales
def _mxfp8_e4m3_quantize_impl(
x: torch.Tensor,
is_sf_swizzled_layout: bool = False,
@@ -103,6 +189,17 @@ def _mxfp8_e4m3_quantize_impl(
x_scales = x_scales.view(x.size(0), -1)
return x_q, x_scales
# ROCm: a single fused Triton kernel beats the multi-pass torch path for the
# common 2D, non-swizzled activation-quant case (used by the native MX
# linear/MoE). Falls back to torch otherwise (3D weights, swizzled layout).
if (
current_platform.is_rocm()
and not is_sf_swizzled_layout
and x.ndim == 2
and x.shape[-1] % MXFP8_BLOCK_SIZE == 0
):
return _mxfp8_e4m3_quantize_triton(x)
return _mxfp8_e4m3_quantize_torch(x, is_sf_swizzled_layout)
+9
View File
@@ -164,6 +164,10 @@ _TEXT_GENERATION_MODELS = {
"MiniMaxText01ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
"MiniMaxM1ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
"MiniMaxM2ForCausalLM": ("minimax_m2", "MiniMaxM2ForCausalLM"),
"MiniMaxM3SparseForCausalLM": (
"vllm.models.minimax_m3",
"MiniMaxM3SparseForCausalLM",
),
"Ministral3ForCausalLM": ("mistral", "MistralForCausalLM"),
"MistralForCausalLM": ("mistral", "MistralForCausalLM"),
"MistralLarge3ForCausalLM": ("mistral_large_3", "MistralLarge3ForCausalLM"),
@@ -483,6 +487,10 @@ _MULTIMODAL_MODELS = {
"MantisForConditionalGeneration": ("llava", "MantisForConditionalGeneration"),
"MiDashengLMModel": ("midashenglm", "MiDashengLMModel"),
"MiMoV2OmniForCausalLM": ("mimo_v2_omni", "MiMoV2OmniForCausalLM"),
"MiniMaxM3SparseForConditionalGeneration": (
"vllm.models.minimax_m3",
"MiniMaxM3SparseForConditionalGeneration",
),
"MiniMaxVL01ForConditionalGeneration": (
"minimax_vl_01",
"MiniMaxVL01ForConditionalGeneration",
@@ -620,6 +628,7 @@ _SPECULATIVE_DECODING_MODELS = {
"EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"),
"DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
"DeepSeekV4MTPModel": ("vllm.models.deepseek_v4", "DeepSeekV4MTP"),
"MiniMaxM3MTP": ("vllm.models.minimax_m3", "MiniMaxM3MTP"),
"Gemma4MTPModel": ("gemma4_mtp", "Gemma4MTP"),
"ErnieMTPModel": ("ernie_mtp", "ErnieMTP"),
"ExaoneMoeMTP": ("exaone_moe_mtp", "ExaoneMoeMTP"),
@@ -53,6 +53,10 @@ def _resolve_flashinfer_autotune_file(runner: "GPUModelRunner") -> Path:
def kernel_warmup(worker: "Worker"):
from vllm.model_executor.warmup.minimax_m3_msa_warmup import (
minimax_m3_msa_warmup,
)
# Deep GEMM warmup
do_deep_gemm_warmup = (
envs.VLLM_USE_DEEP_GEMM
@@ -64,6 +68,8 @@ def kernel_warmup(worker: "Worker"):
max_tokens = worker.scheduler_config.max_num_batched_tokens
deep_gemm_warmup(model, max_tokens)
minimax_m3_msa_warmup(worker)
enable_flashinfer_autotune = (
worker.vllm_config.kernel_config.enable_flashinfer_autotune
)
@@ -0,0 +1,43 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from vllm.logger import init_logger
from vllm.models.minimax_m3.nvidia.model import MiniMaxM3SparseAttention
from vllm.platforms import current_platform
from vllm.tracing import instrument
if TYPE_CHECKING:
from vllm.v1.worker.gpu_worker import Worker
logger = init_logger(__name__)
@instrument(span_name="MiniMax M3 MSA warmup")
def minimax_m3_msa_warmup(worker: "Worker") -> None:
sparse_module = next(
(
module
for module in worker.get_model().modules()
if isinstance(module, MiniMaxM3SparseAttention)
),
None,
)
if sparse_module is None:
return
if not (
current_platform.is_cuda() and current_platform.is_device_capability_family(100)
):
return
logger.info("Warming up MiniMax M3 MSA kernels.")
# Cover sparse prefill through the normal model path.
worker.model_runner._dummy_run(
num_tokens=16,
skip_eplb=True,
is_profile=True,
force_attention=True,
create_mixed_batch=True,
)