[Model] Add MiniMax M3 support (#45381)
Signed-off-by: youkaichao <youkaichao@gmail.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: functionstackx <47992694+functionstackx@users.noreply.github.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: functionstackx <47992694+functionstackx@users.noreply.github.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
co-authored by
OpenAI Codex
Isotr0py
Thien Tran
Bugen Zhao
Jee Jee Li
Roger Wang
functionstackx
Yongye Zhu
Jee Jee Li
parent
a3195fab7b
commit
0a1c5034f5
@@ -93,6 +93,9 @@ from vllm.model_executor.kernels.linear.mxfp8.flashinfer import (
|
||||
from vllm.model_executor.kernels.linear.mxfp8.marlin import (
|
||||
MarlinMxfp8LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
|
||||
RocmDotScaledMxfp8LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp8.xpu import (
|
||||
XPUMxFp8LinearKernel,
|
||||
)
|
||||
@@ -383,6 +386,9 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = {
|
||||
EmulationMxfp8LinearKernel,
|
||||
],
|
||||
PlatformEnum.ROCM: [
|
||||
# Native CDNA4 (gfx950) MX linear; is_supported() gates to gfx95x and
|
||||
# falls through to BF16 emulation (hipBLASLt) elsewhere / on regression.
|
||||
RocmDotScaledMxfp8LinearKernel,
|
||||
EmulationMxfp8LinearKernel,
|
||||
],
|
||||
PlatformEnum.XPU: [
|
||||
|
||||
@@ -33,6 +33,17 @@ class EmulationMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
|
||||
weight_scale = layer.weight_scale.data[:N, :scale_k].contiguous()
|
||||
|
||||
# Dequantize MXFP8 -> BF16 ONCE here, at load time, so apply_weights runs
|
||||
# a plain BF16 linear with no per-step dequant -- i.e. run as if from a
|
||||
# BF16 checkpoint. The 1-byte MXFP8 weight is replaced by BF16 (2x its
|
||||
# size, but linear weights are small vs the MoE experts); the tiny E8M0
|
||||
# scale is kept for the dtype/ndim asserts but is otherwise unused.
|
||||
# Opt out (VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD=0) to keep the MXFP8
|
||||
# weight and dequant per-step in apply_weights instead.
|
||||
import vllm.envs as envs
|
||||
|
||||
if envs.VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD:
|
||||
weight = dequant_mxfp8_to_bf16(weight.contiguous(), weight_scale)
|
||||
layer.weight = Parameter(weight.contiguous(), requires_grad=False)
|
||||
layer.weight_scale = Parameter(weight_scale, requires_grad=False)
|
||||
|
||||
@@ -42,6 +53,17 @@ class EmulationMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
weight = layer.weight
|
||||
# Load-time dequant path: weights are already BF16/FP16 (>= 2-byte), so
|
||||
# run a plain linear -- no per-step dequant. (MXFP8 weights are 1-byte.)
|
||||
if weight.element_size() >= 2:
|
||||
# F.linear requires x and weight share a dtype; .to() is a no-op when
|
||||
# they already match (e.g. both BF16).
|
||||
output = torch.nn.functional.linear(x, weight.to(x.dtype), bias)
|
||||
return output.to(x.dtype)
|
||||
|
||||
# Fallback: weights still in MXFP8 -- dequant on the fly (other archs /
|
||||
# if a future caller skips the load-time conversion above).
|
||||
weight_scale = layer.weight_scale
|
||||
if weight_scale.dtype != MXFP8_SCALE_DTYPE:
|
||||
raise ValueError(
|
||||
@@ -55,6 +77,8 @@ class EmulationMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
f"Ensure process_weights_after_loading was called."
|
||||
)
|
||||
|
||||
weight_bf16 = dequant_mxfp8_to_bf16(layer.weight, weight_scale)
|
||||
# Cast to x's dtype: dequant yields BF16, but F.linear needs both operands
|
||||
# to match (e.g. an FP16 model). No-op when x is already BF16.
|
||||
weight_bf16 = dequant_mxfp8_to_bf16(weight, weight_scale).to(x.dtype)
|
||||
output = torch.nn.functional.linear(x, weight_bf16, bias)
|
||||
return output.to(x.dtype)
|
||||
|
||||
@@ -56,8 +56,6 @@ class FlashInferCutlassMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
|
||||
input_shape = x.shape
|
||||
input_2d = x.view(-1, K)
|
||||
M_orig = input_2d.shape[0]
|
||||
|
||||
min_dim = 128
|
||||
|
||||
assert min_dim <= K, (
|
||||
@@ -72,11 +70,6 @@ class FlashInferCutlassMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
f"out_features is too small for mm_mxfp8."
|
||||
)
|
||||
|
||||
M_padded = ((M_orig + min_dim - 1) // min_dim) * min_dim
|
||||
if M_padded != M_orig:
|
||||
pad_rows = M_padded - M_orig
|
||||
input_2d = torch.nn.functional.pad(input_2d, (0, 0, 0, pad_rows))
|
||||
|
||||
input_mxfp8, input_scale = mxfp8_e4m3_quantize(
|
||||
input_2d, is_sf_swizzled_layout=True
|
||||
)
|
||||
@@ -93,9 +86,6 @@ class FlashInferCutlassMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
backend="cutlass",
|
||||
)
|
||||
|
||||
if M_padded != M_orig:
|
||||
output = output[:M_orig, :]
|
||||
|
||||
if bias is not None:
|
||||
output = output + bias
|
||||
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Native MXFP8 linear GEMM for AMD CDNA4 (gfx950) via Triton ``tl.dot_scaled``.
|
||||
|
||||
Consumes the FP8 E4M3 weights + E8M0 block scales directly (no dequant-to-BF16);
|
||||
activations are MXFP8-quantized per token. Uses the CDNA4 hardware microscaling
|
||||
matrix cores. Falls back (via the kernel selector) to the BF16
|
||||
``EmulationMxfp8LinearKernel`` on archs without native MX or for shapes with
|
||||
``K % 128 != 0``.
|
||||
"""
|
||||
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
MXFP8_BLOCK_SIZE,
|
||||
MXFP8_SCALE_DTYPE,
|
||||
dequant_mxfp8_to_bf16,
|
||||
mxfp8_e4m3_quantize,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _mxfp8_linear_kernel(
|
||||
x_ptr,
|
||||
xs_ptr,
|
||||
w_ptr,
|
||||
ws_ptr,
|
||||
out_ptr,
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
stride_xm,
|
||||
stride_xk,
|
||||
stride_xsm,
|
||||
stride_xsk,
|
||||
stride_wn,
|
||||
stride_wk,
|
||||
stride_wsn,
|
||||
stride_wsk,
|
||||
stride_om,
|
||||
stride_on,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
):
|
||||
pid_m = tl.program_id(0)
|
||||
pid_n = tl.program_id(1)
|
||||
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
offs_k = tl.arange(0, BLOCK_K)
|
||||
offs_sk = tl.arange(0, BLOCK_K // 32)
|
||||
m_mask = offs_m < M
|
||||
n_mask = offs_n < N
|
||||
|
||||
x_ptrs = x_ptr + offs_m[:, None] * stride_xm + offs_k[None, :] * stride_xk
|
||||
xs_ptrs = xs_ptr + offs_m[:, None] * stride_xsm + offs_sk[None, :] * stride_xsk
|
||||
w_ptrs = w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :] * stride_wk
|
||||
ws_ptrs = ws_ptr + offs_n[:, None] * stride_wsn + offs_sk[None, :] * stride_wsk
|
||||
|
||||
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
|
||||
for _ in range(0, tl.cdiv(K, BLOCK_K)):
|
||||
x = tl.load(x_ptrs, mask=m_mask[:, None], other=0.0)
|
||||
w = tl.load(w_ptrs, mask=n_mask[:, None], other=0.0)
|
||||
xs = tl.load(xs_ptrs, mask=m_mask[:, None], other=0)
|
||||
ws = tl.load(ws_ptrs, mask=n_mask[:, None], other=0)
|
||||
acc += tl.dot_scaled(x, xs, "e4m3", w.T, ws, "e4m3")
|
||||
x_ptrs += BLOCK_K * stride_xk
|
||||
w_ptrs += BLOCK_K * stride_wk
|
||||
xs_ptrs += (BLOCK_K // 32) * stride_xsk
|
||||
ws_ptrs += (BLOCK_K // 32) * stride_wsk
|
||||
|
||||
o_ptrs = out_ptr + offs_m[:, None] * stride_om + offs_n[None, :] * stride_on
|
||||
tl.store(
|
||||
o_ptrs, acc.to(out_ptr.dtype.element_ty), mask=m_mask[:, None] & n_mask[None, :]
|
||||
)
|
||||
|
||||
|
||||
def _mxfp8_dot_scaled_linear(
|
||||
x: torch.Tensor, # [M, K] bf16/fp16
|
||||
w: torch.Tensor, # [N, K] fp8 e4m3
|
||||
w_scale: torch.Tensor, # [N, K//32] uint8 (E8M0)
|
||||
) -> torch.Tensor:
|
||||
M, K = x.shape
|
||||
N = w.shape[0]
|
||||
x_q, x_scale = mxfp8_e4m3_quantize(x)
|
||||
out = torch.empty((M, N), dtype=x.dtype, device=x.device)
|
||||
BLOCK_M, BLOCK_N, BLOCK_K = 64, 128, 128
|
||||
grid = (triton.cdiv(M, BLOCK_M), triton.cdiv(N, BLOCK_N))
|
||||
_mxfp8_linear_kernel[grid](
|
||||
x_q,
|
||||
x_scale,
|
||||
w,
|
||||
w_scale,
|
||||
out,
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
x_q.stride(0),
|
||||
x_q.stride(1),
|
||||
x_scale.stride(0),
|
||||
x_scale.stride(1),
|
||||
w.stride(0),
|
||||
w.stride(1),
|
||||
w_scale.stride(0),
|
||||
w_scale.stride(1),
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
BLOCK_M=BLOCK_M,
|
||||
BLOCK_N=BLOCK_N,
|
||||
BLOCK_K=BLOCK_K,
|
||||
num_warps=8,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
class RocmDotScaledMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
"""Native CDNA4 (gfx950) MXFP8 linear via Triton ``tl.dot_scaled``."""
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_rocm():
|
||||
return False, "not ROCm"
|
||||
# supports_mx() == gfx95x (CDNA4 native microscaling hardware). On other
|
||||
# archs dot_scaled would upcast to BF16, so the kernel selector falls
|
||||
# through to the BF16 emulation (hipBLASLt) path instead.
|
||||
if not current_platform.supports_mx():
|
||||
return False, "native MX requires CDNA4 (gfx95x)"
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
weight = layer.weight.data # [N, K] fp8
|
||||
N, K = weight.shape
|
||||
scale_k = K // MXFP8_BLOCK_SIZE
|
||||
weight_scale = layer.weight_scale.data[:N, :scale_k].contiguous()
|
||||
layer.weight = Parameter(weight.contiguous(), requires_grad=False)
|
||||
layer.weight_scale = Parameter(weight_scale, requires_grad=False)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if layer.weight_scale.dtype != MXFP8_SCALE_DTYPE:
|
||||
raise ValueError(
|
||||
f"Expected {MXFP8_SCALE_DTYPE} weight_scale, got "
|
||||
f"{layer.weight_scale.dtype}."
|
||||
)
|
||||
out_shape = (*x.shape[:-1], layer.weight.shape[0])
|
||||
x2d = x.reshape(-1, x.shape[-1])
|
||||
if x2d.shape[-1] % 128 == 0:
|
||||
out = _mxfp8_dot_scaled_linear(x2d, layer.weight, layer.weight_scale)
|
||||
else:
|
||||
# dot_scaled tiling needs K % 128 == 0; dequantize fallback otherwise.
|
||||
w_bf16 = dequant_mxfp8_to_bf16(layer.weight, layer.weight_scale)
|
||||
out = torch.nn.functional.linear(x2d, w_bf16).to(x.dtype)
|
||||
out = out.reshape(out_shape)
|
||||
if bias is not None:
|
||||
out = out + bias
|
||||
return out
|
||||
@@ -158,17 +158,28 @@ class SiluAndMulWithClamp(CustomOp):
|
||||
Computes:
|
||||
gate = clamp(x[..., :d], max=swiglu_limit)
|
||||
up = clamp(x[..., d:], min=-swiglu_limit, max=swiglu_limit)
|
||||
out = silu(gate) * up
|
||||
where d = x.shape[-1] // 2.
|
||||
out = gate * sigmoid(alpha * gate) * (up + beta)
|
||||
where d = x.shape[-1] // 2. The defaults alpha=1.0, beta=0.0 reduce this to
|
||||
``silu(gate) * up``; SwiGLU-OAI style models pass alpha (sigmoid scale) and
|
||||
beta=1.0 (up bias).
|
||||
|
||||
Shapes:
|
||||
x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d)
|
||||
return: (num_tokens, d) or (batch_size, seq_len, d)
|
||||
"""
|
||||
|
||||
def __init__(self, swiglu_limit: float, *, compile_native: bool = True):
|
||||
def __init__(
|
||||
self,
|
||||
swiglu_limit: float,
|
||||
alpha: float = 1.0,
|
||||
beta: float = 0.0,
|
||||
*,
|
||||
compile_native: bool = True,
|
||||
):
|
||||
super().__init__(compile_native=compile_native)
|
||||
self.swiglu_limit = float(swiglu_limit)
|
||||
self.alpha = float(alpha)
|
||||
self.beta = float(beta)
|
||||
if current_platform.is_rocm() or current_platform.is_xpu():
|
||||
self._forward_method = self.forward_native
|
||||
elif current_platform.is_cuda_alike():
|
||||
@@ -180,18 +191,24 @@ class SiluAndMulWithClamp(CustomOp):
|
||||
d = x.shape[-1] // 2
|
||||
gate = torch.clamp(x[..., :d], max=self.swiglu_limit)
|
||||
up = torch.clamp(x[..., d:], min=-self.swiglu_limit, max=self.swiglu_limit)
|
||||
return F.silu(gate) * up
|
||||
return gate * torch.sigmoid(self.alpha * gate) * (up + self.beta)
|
||||
|
||||
def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
|
||||
d = x.shape[-1] // 2
|
||||
output_shape = x.shape[:-1] + (d,)
|
||||
out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
|
||||
self.op(out, x, self.swiglu_limit)
|
||||
self.op(out, x, self.swiglu_limit, self.alpha, self.beta)
|
||||
return out
|
||||
|
||||
def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return self.forward_native(x)
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
return (
|
||||
f"swiglu_limit={self.swiglu_limit!r}, "
|
||||
f"alpha={self.alpha!r}, beta={self.beta!r}"
|
||||
)
|
||||
|
||||
|
||||
# --8<-- [start:mul_and_silu]
|
||||
@CustomOp.register("mul_and_silu")
|
||||
|
||||
@@ -7,6 +7,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CacheConfig, get_current_vllm_config
|
||||
from vllm.config.vllm import VllmConfig
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
@@ -730,6 +731,7 @@ direct_register_custom_op(
|
||||
)
|
||||
|
||||
|
||||
@eager_break_during_capture
|
||||
@maybe_transfer_kv_layer
|
||||
def unified_attention_with_output(
|
||||
query: torch.Tensor,
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Manual fusion of tensor-parallel all-reduce with the following GemmaRMSNorm.
|
||||
|
||||
Under tensor parallelism a ``RowParallelLinear`` (e.g. attention ``o_proj``)
|
||||
produces a per-rank partial sum that is all-reduced, and the result is then fed
|
||||
into a ``GemmaRMSNorm`` that adds the residual and normalizes. flashinfer ships a
|
||||
kernel that fuses all-reduce + residual-add + RMSNorm into a single launch; this
|
||||
helper drives it directly (no torch.compile pass) for models that run eager.
|
||||
|
||||
Scope: attention output only, no quantization. When the flashinfer fast path is
|
||||
not applicable (TP==1, flashinfer/NVSwitch unavailable, unsupported dtype, or an
|
||||
oversize batch) it falls back to ``all_reduce`` + ``GemmaRMSNorm``, which is
|
||||
numerically identical to the unfused model path.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
get_tp_group,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
|
||||
|
||||
MiB = 1024 * 1024
|
||||
|
||||
# flashinfer fused all-reduce + RMSNorm is wired as a registered custom op in
|
||||
# allreduce_rms_fusion; both that op and the workspace helpers only exist when
|
||||
# flashinfer.comm.allreduce_fusion is importable.
|
||||
try:
|
||||
from vllm.compilation.passes.fusion.allreduce_rms_fusion import (
|
||||
flashinfer_trtllm_fused_allreduce_norm,
|
||||
)
|
||||
from vllm.distributed.device_communicators.flashinfer_all_reduce import (
|
||||
flashinfer_comm,
|
||||
get_fi_ar_workspace,
|
||||
)
|
||||
|
||||
_AR_RESIDUAL_RMS_NORM = (
|
||||
flashinfer_comm.AllReduceFusionPattern.kARResidualRMSNorm
|
||||
if flashinfer_comm is not None
|
||||
else None
|
||||
)
|
||||
except ImportError:
|
||||
flashinfer_trtllm_fused_allreduce_norm = None # type: ignore[assignment]
|
||||
get_fi_ar_workspace = None # type: ignore[assignment]
|
||||
_AR_RESIDUAL_RMS_NORM = None
|
||||
|
||||
|
||||
_FI_SUPPORTED_DTYPES = (torch.bfloat16, torch.float16)
|
||||
|
||||
|
||||
def _max_token_num(tp_size: int, hidden_size: int, dtype: torch.dtype) -> int | None:
|
||||
"""Workspace token budget for flashinfer fused all-reduce, or None if the
|
||||
current world size / device is unsupported. Mirrors ``FlashInferAllReduce``."""
|
||||
from vllm.config.compilation import PassConfig
|
||||
|
||||
max_size_mb = PassConfig.default_fi_allreduce_fusion_max_size_mb().get(tp_size)
|
||||
if not max_size_mb:
|
||||
return None
|
||||
element_size = torch.tensor([], dtype=dtype).element_size()
|
||||
return int(max_size_mb * MiB) // (hidden_size * element_size)
|
||||
|
||||
|
||||
def _can_use_flashinfer(hidden_states: torch.Tensor, tp_size: int) -> tuple[bool, int]:
|
||||
"""Whether the flashinfer fused path applies; returns (ok, max_token_num)."""
|
||||
if (
|
||||
flashinfer_trtllm_fused_allreduce_norm is None
|
||||
or get_fi_ar_workspace is None
|
||||
or _AR_RESIDUAL_RMS_NORM is None
|
||||
):
|
||||
return False, 0
|
||||
if (
|
||||
not hidden_states.is_cuda
|
||||
or hidden_states.dim() != 2
|
||||
or not hidden_states.is_contiguous()
|
||||
or hidden_states.dtype not in _FI_SUPPORTED_DTYPES
|
||||
):
|
||||
return False, 0
|
||||
|
||||
num_tokens, hidden_size = hidden_states.shape
|
||||
max_token_num = _max_token_num(tp_size, hidden_size, hidden_states.dtype)
|
||||
if max_token_num is None or num_tokens > max_token_num:
|
||||
return False, 0
|
||||
|
||||
# Lazily create / fetch the (globally cached) workspace; returns None on
|
||||
# GPUs without NVSwitch, in which case we fall back gracefully.
|
||||
workspace = get_fi_ar_workspace(
|
||||
world_size=tp_size,
|
||||
rank=get_tensor_model_parallel_rank(),
|
||||
max_token_num=max_token_num,
|
||||
hidden_dim=hidden_size,
|
||||
dtype=hidden_states.dtype,
|
||||
group=get_tp_group().device_group,
|
||||
)
|
||||
if workspace is None:
|
||||
return False, 0
|
||||
return True, max_token_num
|
||||
|
||||
|
||||
def fused_allreduce_gemma_rms_norm(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
norm: GemmaRMSNorm,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""All-reduce ``hidden_states`` + add ``residual`` + GemmaRMSNorm, fused.
|
||||
|
||||
``hidden_states`` is the per-rank *partial* (un-reduced) output of a
|
||||
row-parallel linear; ``norm`` is the GemmaRMSNorm applied right after.
|
||||
Returns ``(normed_output, new_residual)``, equivalent to
|
||||
``norm(all_reduce(hidden_states), residual)``.
|
||||
"""
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
if tp_size == 1:
|
||||
# No all-reduce needed; identical to the unfused path.
|
||||
return norm(hidden_states, residual)
|
||||
|
||||
ok, max_token_num = _can_use_flashinfer(hidden_states, tp_size)
|
||||
if ok:
|
||||
norm_out = torch.empty_like(hidden_states)
|
||||
# With norm_out provided, the kernel writes the new residual
|
||||
# (all_reduce(hidden_states) + residual) into the hidden_states buffer
|
||||
# and the normalized result into norm_out, leaving `residual` untouched.
|
||||
flashinfer_trtllm_fused_allreduce_norm(
|
||||
allreduce_in=hidden_states,
|
||||
residual=residual,
|
||||
rms_gamma=norm.weight,
|
||||
rms_eps=norm.variance_epsilon,
|
||||
world_size=tp_size,
|
||||
weight_bias=1.0, # GemmaRMSNorm-style
|
||||
launch_with_pdl=True,
|
||||
fp32_acc=True,
|
||||
max_token_num=max_token_num,
|
||||
pattern_code=_AR_RESIDUAL_RMS_NORM,
|
||||
norm_out=norm_out,
|
||||
)
|
||||
return norm_out, hidden_states
|
||||
|
||||
# Fallback: explicit all-reduce + GemmaRMSNorm (matches the unfused model).
|
||||
reduced = tensor_model_parallel_all_reduce(hidden_states)
|
||||
return norm(reduced, residual)
|
||||
@@ -17,7 +17,12 @@ class MoEActivation(Enum):
|
||||
GELU = "gelu"
|
||||
GELU_TANH = "gelu_tanh"
|
||||
RELU2 = "relu2"
|
||||
# SWIGLUOAI expects gate/up *interleaved* in w13 ([gate0, up0, gate1, ...]),
|
||||
# as in gpt-oss checkpoints. SWIGLUOAI_UNINTERLEAVE has identical math but
|
||||
# expects the *packed* layout ([all gates; all ups]), as produced by a
|
||||
# MergedColumnParallelLinear gate_up_proj (e.g. MiniMax-M3).
|
||||
SWIGLUOAI = "swigluoai"
|
||||
SWIGLUOAI_UNINTERLEAVE = "swigluoai_uninterleave"
|
||||
SWIGLUSTEP = "swiglustep"
|
||||
|
||||
# Non-gated activations (no mul with gate) expect input of shape [..., d]
|
||||
@@ -73,6 +78,7 @@ _CUSTOM_OP_NAMES: dict[MoEActivation, str] = {
|
||||
MoEActivation.GELU: "gelu_and_mul",
|
||||
MoEActivation.GELU_TANH: "gelu_tanh_and_mul",
|
||||
MoEActivation.SWIGLUOAI: "swigluoai_and_mul",
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE: "silu_and_mul_with_clamp",
|
||||
MoEActivation.SWIGLUSTEP: "swiglustep_and_mul",
|
||||
MoEActivation.RELU2: "relu2",
|
||||
MoEActivation.SILU_NO_MUL: "silu_and_mul",
|
||||
@@ -105,8 +111,17 @@ def apply_moe_activation(
|
||||
activation: MoEActivation,
|
||||
output: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
*,
|
||||
clamp_limit: float | None = None,
|
||||
alpha: float = 1.0,
|
||||
beta: float = 0.0,
|
||||
) -> torch.Tensor:
|
||||
"""Apply MoE activation function."""
|
||||
"""Apply MoE activation function.
|
||||
|
||||
``clamp_limit``/``alpha``/``beta`` (from the quant config) drive the clamped
|
||||
SwiGLU kernels: ``SILU`` + ``clamp_limit`` and ``SWIGLUOAI_UNINTERLEAVE`` both
|
||||
map to ``silu_and_mul_with_clamp``. Other activations ignore them.
|
||||
"""
|
||||
assert input.dim() == 2, "Input must be 2D"
|
||||
assert output.dim() == 2, "Output must be 2D"
|
||||
if activation.is_gated:
|
||||
@@ -122,13 +137,21 @@ def apply_moe_activation(
|
||||
|
||||
# Activations with gated multiplication (gate × activation(up))
|
||||
if activation == MoEActivation.SILU:
|
||||
torch.ops._C.silu_and_mul(output, input)
|
||||
if clamp_limit is not None:
|
||||
# Fused silu(clamp(gate)) * clamp(up); equivalent to swiglu_limit_func.
|
||||
torch.ops._C.silu_and_mul_with_clamp(output, input, clamp_limit, 1.0, 0.0)
|
||||
else:
|
||||
torch.ops._C.silu_and_mul(output, input)
|
||||
elif activation == MoEActivation.GELU:
|
||||
torch.ops._C.gelu_and_mul(output, input)
|
||||
elif activation == MoEActivation.GELU_TANH:
|
||||
torch.ops._C.gelu_tanh_and_mul(output, input)
|
||||
elif activation == MoEActivation.SWIGLUOAI:
|
||||
torch.ops._C.swigluoai_and_mul(output, input)
|
||||
elif activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
|
||||
# SwiGLU-OAI on packed w13 (gate = first half, up = second half).
|
||||
assert clamp_limit is not None, "SWIGLUOAI_UNINTERLEAVE requires clamp_limit"
|
||||
torch.ops._C.silu_and_mul_with_clamp(output, input, clamp_limit, alpha, beta)
|
||||
elif activation == MoEActivation.SWIGLUSTEP:
|
||||
from vllm.model_executor.layers.activation import swiglustep_and_mul_triton
|
||||
|
||||
|
||||
@@ -900,6 +900,9 @@ def fp8_w8a16_moe_quant_config(
|
||||
w1_bias: torch.Tensor | None = None,
|
||||
w2_bias: torch.Tensor | None = None,
|
||||
block_shape: list[int] | None = None,
|
||||
gemm1_alpha: float | None = None,
|
||||
gemm1_beta: float | None = None,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
Construct a quant config for 16-bit float activations and fp8 weights.
|
||||
@@ -925,6 +928,9 @@ def fp8_w8a16_moe_quant_config(
|
||||
None,
|
||||
w2_bias,
|
||||
),
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
gemm1_clamp_limit=gemm1_clamp_limit,
|
||||
)
|
||||
|
||||
|
||||
@@ -979,15 +985,24 @@ def int4_w4afp8_moe_quant_config(
|
||||
def biased_moe_quant_config(
|
||||
w1_bias: torch.Tensor | None,
|
||||
w2_bias: torch.Tensor | None,
|
||||
gemm1_alpha: float | None = None,
|
||||
gemm1_beta: float | None = None,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
Construct a quant config for unquantized activations with biases.
|
||||
|
||||
gemm1_alpha/gemm1_beta/gemm1_clamp_limit carry the SwiGLU gate params
|
||||
through to the fused activation kernel (e.g. swigluoai_uninterleave).
|
||||
"""
|
||||
return FusedMoEQuantConfig(
|
||||
_a1=FusedMoEQuantDesc(),
|
||||
_a2=FusedMoEQuantDesc(),
|
||||
_w1=FusedMoEQuantDesc(bias=w1_bias),
|
||||
_w2=FusedMoEQuantDesc(bias=w2_bias),
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
gemm1_clamp_limit=gemm1_clamp_limit,
|
||||
)
|
||||
|
||||
|
||||
@@ -1268,6 +1283,8 @@ class FusedMoEConfig:
|
||||
# are filtered out by `FusedMoEExperts.is_supported_config` so the oracle
|
||||
# cannot silently select one and drop the clamp.
|
||||
swiglu_limit: float | None = None
|
||||
swiglu_alpha: float | None = None
|
||||
swiglu_beta: float | None = None
|
||||
|
||||
max_capture_size: int = 0
|
||||
|
||||
|
||||
@@ -130,6 +130,9 @@ def _fwd_kernel_ep_scatter_2(
|
||||
HIDDEN_SIZE_PAD: tl.constexpr,
|
||||
SCALE_HIDDEN_SIZE: tl.constexpr,
|
||||
SCALE_HIDDEN_SIZE_PAD: tl.constexpr,
|
||||
PACK_UE8M0: tl.constexpr,
|
||||
SCALE_PACKED_SIZE: tl.constexpr,
|
||||
SCALE_PACKED_SIZE_PAD: tl.constexpr,
|
||||
):
|
||||
start_token_id = tl.program_id(0)
|
||||
grid_num = tl.num_programs(0)
|
||||
@@ -137,16 +140,47 @@ def _fwd_kernel_ep_scatter_2(
|
||||
offset_in = tl.arange(0, HIDDEN_SIZE_PAD)
|
||||
mask = offset_in < HIDDEN_SIZE
|
||||
|
||||
offset_in_s = tl.arange(0, SCALE_HIDDEN_SIZE_PAD)
|
||||
mask_s = offset_in_s < SCALE_HIDDEN_SIZE
|
||||
|
||||
output_tensor_stride0 = output_tensor_stride0.to(tl.int64)
|
||||
|
||||
if PACK_UE8M0:
|
||||
# One int32 per 4 consecutive 32-wide UE8M0 groups, stored MN-major.
|
||||
offs_pk = tl.arange(0, SCALE_PACKED_SIZE_PAD)
|
||||
mask_pk = offs_pk < SCALE_PACKED_SIZE
|
||||
else:
|
||||
offset_in_s = tl.arange(0, SCALE_HIDDEN_SIZE_PAD)
|
||||
mask_s = offset_in_s < SCALE_HIDDEN_SIZE
|
||||
|
||||
for token_id in range(start_token_id, total_token_num, grid_num):
|
||||
to_copy = tl.load(recv_x + token_id * recv_x_stride0 + offset_in, mask=mask)
|
||||
to_copy_s = tl.load(
|
||||
recv_x_scale + token_id * recv_x_scale_stride0 + offset_in_s, mask=mask_s
|
||||
)
|
||||
|
||||
if PACK_UE8M0:
|
||||
# Pack 4 UE8M0 bytes into one int32 (byte j = group 4*pk+j).
|
||||
base_s = recv_x_scale + token_id * recv_x_scale_stride0
|
||||
g0, g1 = offs_pk * 4, offs_pk * 4 + 1
|
||||
g2, g3 = offs_pk * 4 + 2, offs_pk * 4 + 3
|
||||
b0 = tl.load(
|
||||
base_s + g0 * recv_x_scale_stride1, mask=g0 < SCALE_HIDDEN_SIZE
|
||||
)
|
||||
b1 = tl.load(
|
||||
base_s + g1 * recv_x_scale_stride1, mask=g1 < SCALE_HIDDEN_SIZE
|
||||
)
|
||||
b2 = tl.load(
|
||||
base_s + g2 * recv_x_scale_stride1, mask=g2 < SCALE_HIDDEN_SIZE
|
||||
)
|
||||
b3 = tl.load(
|
||||
base_s + g3 * recv_x_scale_stride1, mask=g3 < SCALE_HIDDEN_SIZE
|
||||
)
|
||||
packed_s = (
|
||||
b0.to(tl.int32)
|
||||
| (b1.to(tl.int32) << 8)
|
||||
| (b2.to(tl.int32) << 16)
|
||||
| (b3.to(tl.int32) << 24)
|
||||
)
|
||||
else:
|
||||
to_copy_s = tl.load(
|
||||
recv_x_scale + token_id * recv_x_scale_stride0 + offset_in_s,
|
||||
mask=mask_s,
|
||||
)
|
||||
|
||||
for topk_index in tl.range(0, topk_num, 1, num_stages=4):
|
||||
expert_id = tl.load(recv_topk + token_id * recv_topk_stride0 + topk_index)
|
||||
@@ -164,11 +198,21 @@ def _fwd_kernel_ep_scatter_2(
|
||||
output_tensor_ptr = (
|
||||
output_tensor + dest_token_index_i64 * output_tensor_stride0
|
||||
)
|
||||
tl.store(output_tensor_ptr + offset_in, to_copy, mask=mask)
|
||||
|
||||
output_tensor_scale_ptr = (
|
||||
output_tensor_scale + dest_token_index * output_tensor_scale_stride0
|
||||
)
|
||||
tl.store(output_tensor_ptr + offset_in, to_copy, mask=mask)
|
||||
tl.store(output_tensor_scale_ptr + offset_in_s, to_copy_s, mask=mask_s)
|
||||
if PACK_UE8M0:
|
||||
tl.store(
|
||||
output_tensor_scale_ptr + offs_pk * output_tensor_scale_stride1,
|
||||
packed_s,
|
||||
mask=mask_pk,
|
||||
)
|
||||
else:
|
||||
tl.store(
|
||||
output_tensor_scale_ptr + offset_in_s, to_copy_s, mask=mask_s
|
||||
)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
@@ -183,9 +227,11 @@ def ep_scatter(
|
||||
output_tensor_scale: torch.Tensor,
|
||||
m_indices: torch.Tensor,
|
||||
output_index: torch.Tensor,
|
||||
block_size: int = 128,
|
||||
pack_ue8m0: bool = False,
|
||||
):
|
||||
BLOCK_E = 128 # token num of per expert is aligned to 128
|
||||
BLOCK_D = 128 # block size of quantization
|
||||
BLOCK_D = block_size # block size of activation-scale quantization
|
||||
num_warps = 8
|
||||
num_experts = num_recv_tokens_per_expert.shape[0]
|
||||
hidden_size = recv_x.shape[1]
|
||||
@@ -195,6 +241,10 @@ def ep_scatter(
|
||||
assert m_indices.shape[0] % BLOCK_E == 0
|
||||
assert expert_start_loc.shape[0] == num_experts
|
||||
|
||||
# pack_ue8m0: scatter packs 4 UE8M0 bytes per int32; else copies scales as-is.
|
||||
scale_hidden_size = hidden_size // BLOCK_D
|
||||
scale_packed_size = (scale_hidden_size + 3) // 4 if pack_ue8m0 else 1
|
||||
|
||||
_fwd_kernel_ep_scatter_1[(grid,)](
|
||||
num_recv_tokens_per_expert,
|
||||
expert_start_loc,
|
||||
@@ -234,8 +284,11 @@ def ep_scatter(
|
||||
num_warps=num_warps,
|
||||
HIDDEN_SIZE=hidden_size,
|
||||
HIDDEN_SIZE_PAD=triton.next_power_of_2(hidden_size),
|
||||
SCALE_HIDDEN_SIZE=hidden_size // BLOCK_D,
|
||||
SCALE_HIDDEN_SIZE_PAD=triton.next_power_of_2(hidden_size // BLOCK_D),
|
||||
SCALE_HIDDEN_SIZE=scale_hidden_size,
|
||||
SCALE_HIDDEN_SIZE_PAD=triton.next_power_of_2(scale_hidden_size),
|
||||
PACK_UE8M0=pack_ue8m0,
|
||||
SCALE_PACKED_SIZE=scale_packed_size,
|
||||
SCALE_PACKED_SIZE_PAD=triton.next_power_of_2(scale_packed_size),
|
||||
)
|
||||
return
|
||||
|
||||
@@ -352,6 +405,7 @@ def deepgemm_moe_permute(
|
||||
expert_map: torch.Tensor | None,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
aq_out: torch.Tensor | None = None,
|
||||
block_size: int | None = None,
|
||||
):
|
||||
assert aq.ndim == 2
|
||||
assert topk_ids.dtype.is_signed, "The kernel uses -1 to represent invalid topk_ids"
|
||||
@@ -359,6 +413,10 @@ def deepgemm_moe_permute(
|
||||
device = aq.device
|
||||
|
||||
block_m, block_k = get_mk_alignment_for_contiguous_layout()
|
||||
# The activation-scale group size may differ from the M/K tile alignment
|
||||
# (e.g. MXFP8 uses a 32-element scale group while block_k stays 128).
|
||||
if block_size is not None:
|
||||
block_k = block_size
|
||||
|
||||
M_sum = compute_aligned_M(
|
||||
M=topk_ids.size(0),
|
||||
@@ -376,9 +434,21 @@ def deepgemm_moe_permute(
|
||||
if aq_out is None:
|
||||
aq_out = torch.empty((M_sum, H), device=device, dtype=aq.dtype)
|
||||
|
||||
aq_scale_out = torch.empty(
|
||||
(M_sum, H // block_k), device=device, dtype=torch.float32
|
||||
)
|
||||
# uint8 UE8M0 (MXFP8) -> scatter packs into DeepGEMM's int32 MN-major
|
||||
# TMA-aligned layout; float32 (FP8/FP4) scattered row-major as-is.
|
||||
pack_ue8m0 = aq_scale.dtype == torch.uint8
|
||||
sf_k = H // block_k
|
||||
if pack_ue8m0:
|
||||
packed_sf_k = (sf_k + 3) // 4
|
||||
tma_aligned_mn = round_up(M_sum, 4)
|
||||
aq_scale_out = torch.empty_strided(
|
||||
(M_sum, packed_sf_k),
|
||||
(1, tma_aligned_mn),
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
else:
|
||||
aq_scale_out = torch.empty((M_sum, sf_k), device=device, dtype=torch.float32)
|
||||
|
||||
# DeepGEMM uses negative values in m_indices (here expert_ids) to mark
|
||||
# completely invalid / padded blocks that should be skipped. We always
|
||||
@@ -412,6 +482,8 @@ def deepgemm_moe_permute(
|
||||
output_tensor_scale=aq_scale_out,
|
||||
m_indices=expert_ids,
|
||||
output_index=inv_perm,
|
||||
block_size=block_k,
|
||||
pack_ue8m0=pack_ue8m0,
|
||||
)
|
||||
|
||||
return aq_out, aq_scale_out, expert_ids, inv_perm
|
||||
|
||||
@@ -33,7 +33,10 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8Static128BlockSym,
|
||||
kMxfp4Static,
|
||||
kMxfp8Dynamic,
|
||||
kMxfp8Static,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import (
|
||||
DeepGemmQuantScaleFMT,
|
||||
get_mk_alignment_for_contiguous_layout,
|
||||
@@ -123,12 +126,26 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
|
||||
def __init__(self, moe_config: FusedMoEConfig, quant_config: FusedMoEQuantConfig):
|
||||
super().__init__(moe_config=moe_config, quant_config=quant_config)
|
||||
assert quant_config.block_shape == get_mk_alignment_for_contiguous_layout()
|
||||
assert quant_config.quant_dtype == torch.float8_e4m3fn
|
||||
# MXFP8: FP8 e4m3 values + UE8M0 1x32 block scales (Blackwell). Reuses
|
||||
# the same grouped GEMM (aliased to fp8_fp4) with recipe (1, 32).
|
||||
self.mxfp8 = quant_config.block_shape == [1, 32]
|
||||
if self.mxfp8:
|
||||
assert quant_config.quant_dtype == "mxfp8"
|
||||
else:
|
||||
assert quant_config.block_shape == get_mk_alignment_for_contiguous_layout()
|
||||
assert quant_config.quant_dtype == torch.float8_e4m3fn
|
||||
assert not quant_config.per_act_token_quant
|
||||
assert not quant_config.per_out_ch_quant
|
||||
|
||||
self.gemm1_clamp_limit = quant_config.gemm1_clamp_limit
|
||||
# Gated-activation params: silu == swigluoai with alpha=1, beta=0.
|
||||
# FP8 (silu) configs leave these None, reproducing plain silu.
|
||||
self.gemm1_alpha = (
|
||||
quant_config.gemm1_alpha if quant_config.gemm1_alpha is not None else 1.0
|
||||
)
|
||||
self.gemm1_beta = (
|
||||
quant_config.gemm1_beta if quant_config.gemm1_beta is not None else 0.0
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
@@ -147,14 +164,25 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
SUPPORTED_W_A = [
|
||||
(kFp8Static128BlockSym, kFp8Dynamic128Sym),
|
||||
]
|
||||
return (weight_key, activation_key) in SUPPORTED_W_A
|
||||
if (weight_key, activation_key) == (kFp8Static128BlockSym, kFp8Dynamic128Sym):
|
||||
return True
|
||||
# MXFP8 1x32 uses the fp8_fp4 grouped GEMM with recipe (1, 32) — only
|
||||
# available on Blackwell (SM100).
|
||||
if (weight_key, activation_key) == (kMxfp8Static, kMxfp8Dynamic):
|
||||
return current_platform.is_device_capability_family(100)
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return activation in [MoEActivation.SILU, MoEActivation.SWIGLUSTEP]
|
||||
# silu/swigluoai go through the fused alpha/beta kernel; swiglustep
|
||||
# uses the unfused activation path. The fused kernel reads packed w13
|
||||
# (gate = first half, up = second half), so it implements the
|
||||
# *uninterleaved* SwiGLU-OAI variant.
|
||||
return activation in [
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.SWIGLUSTEP,
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
|
||||
@@ -179,7 +207,9 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
activation: MoEActivation,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
|
||||
assert self.block_shape is not None
|
||||
block_m = self.block_shape[0]
|
||||
# Use the contiguous-layout M alignment (matches apply()); block_shape[0]
|
||||
# is the quant block (1 for MXFP8) and would under-size the workspace.
|
||||
block_m = get_mk_alignment_for_contiguous_layout()[0]
|
||||
M_sum = compute_aligned_M(
|
||||
M, topk, local_num_experts, block_m, expert_tokens_meta
|
||||
)
|
||||
@@ -201,14 +231,24 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
M_sum, N = input.size()
|
||||
activation_out_dim = self.adjust_N_for_activation(N, activation)
|
||||
|
||||
# 1. DeepGemm UE8M0: fused SiLU+mul+clamp+quant+pack
|
||||
# silu and swigluoai are both expressible by the fused gated kernel via
|
||||
# (alpha, beta): silu uses alpha=1, beta=0; swigluoai uses config values.
|
||||
# The fused kernel reads packed w13, hence SWIGLUOAI_UNINTERLEAVE.
|
||||
fused_gated = activation in (
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
)
|
||||
|
||||
# 1. DeepGemm UE8M0: fused gate+mul+clamp+quant+pack
|
||||
if scale_fmt == DeepGemmQuantScaleFMT.UE8M0:
|
||||
if activation == MoEActivation.SILU:
|
||||
if fused_gated:
|
||||
return fused_silu_mul_fp8_quant_packed(
|
||||
input=input,
|
||||
output_q=output,
|
||||
group_size=block_k,
|
||||
clamp_limit=self.gemm1_clamp_limit,
|
||||
alpha=self.gemm1_alpha,
|
||||
beta=self.gemm1_beta,
|
||||
)
|
||||
act_out = torch.empty(
|
||||
(M_sum, activation_out_dim), dtype=input.dtype, device=input.device
|
||||
@@ -221,14 +261,17 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
)
|
||||
return a2q, a2q_scale
|
||||
|
||||
# 2. Hopper / non‑E8M0: prefer the fused SiLU+mul+quant kernel
|
||||
if activation == MoEActivation.SILU:
|
||||
# 2. Hopper / non‑E8M0: prefer the fused gate+mul+quant kernel
|
||||
if fused_gated:
|
||||
use_ue8m0 = scale_fmt == DeepGemmQuantScaleFMT.FLOAT32_CEIL_UE8M0
|
||||
return silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
input=input,
|
||||
output=output,
|
||||
use_ue8m0=use_ue8m0,
|
||||
clamp_limit=self.gemm1_clamp_limit,
|
||||
group_size=block_k,
|
||||
alpha=self.gemm1_alpha,
|
||||
beta=self.gemm1_beta,
|
||||
)
|
||||
|
||||
# 3. fallback path for non-SiLU activations in non‑UE8M0 cases.
|
||||
@@ -292,12 +335,23 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
expert_map=expert_map,
|
||||
expert_tokens_meta=expert_tokens_meta,
|
||||
aq_out=a1q_perm,
|
||||
# MXFP8 uses a 32-element activation-scale group (block_shape[1]);
|
||||
# FP8-block keeps the default (128) alignment.
|
||||
block_size=self.block_shape[1] if self.mxfp8 else None,
|
||||
)
|
||||
assert a1q.size(0) == M_sum
|
||||
|
||||
# MXFP8 (1x32) drives the fp8_fp4-aliased grouped GEMM with recipe
|
||||
# (1, 32); the FP8 block path keeps the default (128) recipe.
|
||||
gemm_kwargs = (
|
||||
{"recipe_a": (1, self.block_shape[1]), "recipe_b": (1, self.block_shape[1])}
|
||||
if self.mxfp8
|
||||
else {}
|
||||
)
|
||||
|
||||
mm1_out = _resize_cache(workspace2, (M_sum, N))
|
||||
m_grouped_fp8_gemm_nt_contiguous(
|
||||
(a1q, a1q_scale), (w1, self.w1_scale), mm1_out, expert_ids
|
||||
(a1q, a1q_scale), (w1, self.w1_scale), mm1_out, expert_ids, **gemm_kwargs
|
||||
)
|
||||
|
||||
activation_out_dim = self.adjust_N_for_activation(N, activation)
|
||||
@@ -310,7 +364,7 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
|
||||
mm2_out = _resize_cache(workspace2, (M_sum, K))
|
||||
m_grouped_fp8_gemm_nt_contiguous(
|
||||
(a2q, a2q_scale), (w2, self.w2_scale), mm2_out, expert_ids
|
||||
(a2q, a2q_scale), (w2, self.w2_scale), mm2_out, expert_ids, **gemm_kwargs
|
||||
)
|
||||
|
||||
if apply_router_weight_on_input:
|
||||
|
||||
@@ -801,7 +801,11 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular):
|
||||
return TopKWeightAndReduceDelegate()
|
||||
|
||||
def activation(
|
||||
self, activation: MoEActivation, output: torch.Tensor, input: torch.Tensor
|
||||
self,
|
||||
activation: MoEActivation,
|
||||
output: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
gemm1_clamp_limit = self.quant_config.gemm1_clamp_limit
|
||||
if activation == MoEActivation.SILU and gemm1_clamp_limit is not None:
|
||||
|
||||
@@ -787,6 +787,7 @@ class UnfusedOAITritonExperts(LoRAExpertsMixin, BaseOAITritonExperts):
|
||||
activation: MoEActivation,
|
||||
output: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
quant_config = self.quant_config or FUSED_MOE_UNQUANTIZED_CONFIG
|
||||
if activation == MoEActivation.SWIGLUOAI:
|
||||
|
||||
@@ -28,10 +28,7 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceDelegate,
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
_resize_cache,
|
||||
swiglu_limit_func,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import _resize_cache
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
get_marlin_input_dtype,
|
||||
marlin_make_workspace_new,
|
||||
@@ -74,9 +71,7 @@ def _fused_marlin_moe(
|
||||
expert_ids: torch.Tensor,
|
||||
num_tokens_post_padded: torch.Tensor,
|
||||
activation: MoEActivation = MoEActivation.SILU,
|
||||
activation_func: Callable[
|
||||
[MoEActivation, torch.Tensor, torch.Tensor], None
|
||||
] = apply_moe_activation,
|
||||
activation_func: Callable[..., None] = apply_moe_activation,
|
||||
input_global_scale1: torch.Tensor | None = None,
|
||||
input_global_scale2: torch.Tensor | None = None,
|
||||
global_scale1: torch.Tensor | None = None,
|
||||
@@ -94,6 +89,8 @@ def _fused_marlin_moe(
|
||||
input_dtype: torch.dtype | None = None,
|
||||
is_k_full: bool = True,
|
||||
clamp_limit: float | None = None,
|
||||
gemm1_alpha: float = 1.0,
|
||||
gemm1_beta: float = 0.0,
|
||||
) -> torch.Tensor:
|
||||
assert hidden_states.ndim == 2
|
||||
M, K = hidden_states.size()
|
||||
@@ -161,18 +158,16 @@ def _fused_marlin_moe(
|
||||
use_fp32_reduce=True,
|
||||
is_zp_float=False,
|
||||
)
|
||||
if clamp_limit is not None and activation == MoEActivation.SILU:
|
||||
swiglu_limit_func(
|
||||
intermediate_cache2,
|
||||
intermediate_cache1.view(-1, w13_num_shards * N),
|
||||
clamp_limit,
|
||||
)
|
||||
else:
|
||||
activation_func(
|
||||
activation,
|
||||
intermediate_cache2,
|
||||
intermediate_cache1.view(-1, w13_num_shards * N),
|
||||
)
|
||||
# apply_moe_activation fuses the clamp/gate params: SILU + clamp_limit and
|
||||
# SWIGLUOAI_UNINTERLEAVE both map to the silu_and_mul_with_clamp kernel.
|
||||
activation_func(
|
||||
activation,
|
||||
intermediate_cache2,
|
||||
intermediate_cache1.view(-1, w13_num_shards * N),
|
||||
clamp_limit=clamp_limit,
|
||||
alpha=gemm1_alpha,
|
||||
beta=gemm1_beta,
|
||||
)
|
||||
|
||||
if output is None:
|
||||
output = intermediate_cache3
|
||||
@@ -238,9 +233,7 @@ def fused_marlin_moe(
|
||||
apply_router_weight_on_input: bool = False,
|
||||
global_num_experts: int = -1,
|
||||
activation: MoEActivation = MoEActivation.SILU,
|
||||
activation_func: Callable[
|
||||
[MoEActivation, torch.Tensor, torch.Tensor], None
|
||||
] = apply_moe_activation,
|
||||
activation_func: Callable[..., None] = apply_moe_activation,
|
||||
moe_sum: Callable[[torch.Tensor, torch.Tensor], None] | None = None,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
input_global_scale1: torch.Tensor | None = None,
|
||||
@@ -260,6 +253,8 @@ def fused_marlin_moe(
|
||||
output: torch.Tensor | None = None,
|
||||
input_dtype: torch.dtype | None = None,
|
||||
clamp_limit: float | None = None,
|
||||
gemm1_alpha: float = 1.0,
|
||||
gemm1_beta: float = 0.0,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
This function computes a Mixture of Experts (MoE) layer using two sets of
|
||||
@@ -373,6 +368,8 @@ def fused_marlin_moe(
|
||||
input_dtype=input_dtype,
|
||||
is_k_full=is_k_full,
|
||||
clamp_limit=clamp_limit,
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
).view(-1, topk, K)
|
||||
|
||||
if output is None:
|
||||
@@ -415,6 +412,8 @@ def batched_fused_marlin_moe(
|
||||
output: torch.Tensor | None = None,
|
||||
input_dtype: torch.dtype | None = None,
|
||||
clamp_limit: float | None = None,
|
||||
gemm1_alpha: float = 1.0,
|
||||
gemm1_beta: float = 0.0,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
This function massages the inputs so the batched hidden_states can be
|
||||
@@ -544,6 +543,8 @@ def batched_fused_marlin_moe(
|
||||
input_dtype=input_dtype,
|
||||
is_k_full=is_k_full,
|
||||
clamp_limit=clamp_limit,
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
)
|
||||
|
||||
output = output.view(B, BATCH_TOKENS_MAX, K)
|
||||
@@ -579,6 +580,15 @@ class MarlinExpertsBase(mk.FusedMoEExpertsModular):
|
||||
self.is_k_full = is_k_full
|
||||
self.input_dtype = get_marlin_input_dtype()
|
||||
self.gemm1_clamp_limit = quant_config.gemm1_clamp_limit
|
||||
# Gated-activation params (used by SWIGLUOAI_UNINTERLEAVE on packed w13).
|
||||
# silu == swigluoai with alpha=1, beta=0; configs that don't set these
|
||||
# (plain silu) fall back to the silu identity.
|
||||
self.gemm1_alpha = (
|
||||
quant_config.gemm1_alpha if quant_config.gemm1_alpha is not None else 1.0
|
||||
)
|
||||
self.gemm1_beta = (
|
||||
quant_config.gemm1_beta if quant_config.gemm1_beta is not None else 0.0
|
||||
)
|
||||
|
||||
super().__init__(
|
||||
moe_config=moe_config,
|
||||
@@ -627,6 +637,7 @@ class MarlinExpertsBase(mk.FusedMoEExpertsModular):
|
||||
MoEActivation.GELU,
|
||||
MoEActivation.GELU_TANH,
|
||||
MoEActivation.SWIGLUOAI,
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
MoEActivation.SWIGLUSTEP,
|
||||
MoEActivation.SILU_NO_MUL,
|
||||
MoEActivation.GELU_NO_MUL,
|
||||
@@ -787,6 +798,8 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
|
||||
is_k_full=self.is_k_full,
|
||||
input_dtype=self.input_dtype,
|
||||
clamp_limit=self.gemm1_clamp_limit,
|
||||
gemm1_alpha=self.gemm1_alpha,
|
||||
gemm1_beta=self.gemm1_beta,
|
||||
)
|
||||
return
|
||||
|
||||
@@ -805,6 +818,10 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
|
||||
act_enum: MoEActivation,
|
||||
act_output: torch.Tensor,
|
||||
act_input: torch.Tensor,
|
||||
*,
|
||||
clamp_limit: float | None = None,
|
||||
alpha: float = 1.0,
|
||||
beta: float = 0.0,
|
||||
) -> None:
|
||||
# act_input = intermediate_cache1 (M*topk, 2N for gated)
|
||||
# act_output = intermediate_cache2 (M*topk, N)
|
||||
@@ -834,7 +851,14 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
|
||||
"tlm": token_lora_mapping,
|
||||
}
|
||||
)
|
||||
self.activation(act_enum, act_output, act_input)
|
||||
self.activation(
|
||||
act_enum,
|
||||
act_output,
|
||||
act_input,
|
||||
clamp_limit=clamp_limit,
|
||||
alpha=alpha,
|
||||
beta=beta,
|
||||
)
|
||||
lora_state["cache2"] = act_output
|
||||
|
||||
def moe_sum_with_lora(moe_out: torch.Tensor, out: torch.Tensor) -> None:
|
||||
@@ -888,6 +912,8 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
|
||||
is_k_full=self.is_k_full,
|
||||
input_dtype=self.input_dtype,
|
||||
clamp_limit=self.gemm1_clamp_limit,
|
||||
gemm1_alpha=self.gemm1_alpha,
|
||||
gemm1_beta=self.gemm1_beta,
|
||||
)
|
||||
|
||||
def moe_sum(self, input: torch.Tensor, output: torch.Tensor) -> None:
|
||||
@@ -996,4 +1022,6 @@ class BatchedMarlinExperts(MarlinExpertsBase):
|
||||
input_dtype=self.input_dtype,
|
||||
is_k_full=self.is_k_full,
|
||||
clamp_limit=self.gemm1_clamp_limit,
|
||||
gemm1_alpha=self.gemm1_alpha,
|
||||
gemm1_beta=self.gemm1_beta,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MXFP8 (1x32 block, E8M0 scale) MoE experts on Triton.
|
||||
|
||||
``Mxfp8TritonExpertsBase`` stashes E8M0 weight scales for checkpoint layout.
|
||||
``Mxfp8EmulationTritonExperts`` dequantizes to BF16 and runs ``TritonExperts``
|
||||
for devices without a native MXFP8 MoE kernel (e.g. ROCm gfx942 / MI300).
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_moe import TritonExperts
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
dequant_mxfp8_to_bf16,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kMxfp8Dynamic,
|
||||
kMxfp8Static,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class Mxfp8TritonExpertsBase(TritonExperts):
|
||||
"""Shared MXFP8 MoE setup: stash E8M0 scales, clear scales on ``quant_config``."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
):
|
||||
super().__init__(moe_config, quant_config)
|
||||
self.w1_scale_val = self.quant_config.w1_scale
|
||||
self.w2_scale_val = self.quant_config.w2_scale
|
||||
self.quant_config._w1.scale = None
|
||||
self.quant_config._w2.scale = None
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
return (weight_key, activation_key) == (kMxfp8Static, kMxfp8Dynamic)
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation) -> bool:
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
|
||||
if activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
|
||||
return True
|
||||
return TritonExperts._supports_activation(activation)
|
||||
|
||||
|
||||
class Mxfp8EmulationTritonExperts(Mxfp8TritonExpertsBase):
|
||||
"""Dequantize MXFP8 weights to BF16 on the fly and run ``TritonExperts``."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
):
|
||||
super().__init__(moe_config, quant_config)
|
||||
logger.warning_once(
|
||||
"Using Mxfp8EmulationTritonExperts MoE backend. Weights are "
|
||||
"dequantized to BF16 on the fly; this is slower than a native "
|
||||
"MXFP8 MoE kernel and is intended for devices without one."
|
||||
)
|
||||
|
||||
@property
|
||||
def quant_dtype(self) -> torch.dtype | str | None:
|
||||
# BF16 fallback: do not MXFP8-quantize activations in ``TritonExperts``.
|
||||
return None
|
||||
|
||||
@property
|
||||
def block_shape(self) -> list[int] | None:
|
||||
return None
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
return True
|
||||
|
||||
def activation(
|
||||
self,
|
||||
activation,
|
||||
output: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
**kwargs,
|
||||
):
|
||||
"""Apply GEMM1 activation with quant-config alpha/beta/clamp."""
|
||||
from vllm.model_executor.layers.fused_moe.activation import (
|
||||
MoEActivation,
|
||||
apply_moe_activation,
|
||||
)
|
||||
|
||||
if activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
|
||||
limit = self.quant_config.gemm1_clamp_limit
|
||||
if limit is None:
|
||||
raise ValueError("SWIGLUOAI_UNINTERLEAVE requires gemm1_clamp_limit")
|
||||
alpha = self.quant_config.gemm1_alpha
|
||||
alpha = 1.702 if alpha is None else float(alpha)
|
||||
beta = self.quant_config.gemm1_beta
|
||||
beta = 1.0 if beta is None else float(beta)
|
||||
apply_moe_activation(
|
||||
activation,
|
||||
output,
|
||||
input,
|
||||
clamp_limit=float(limit),
|
||||
alpha=alpha,
|
||||
beta=beta,
|
||||
)
|
||||
return
|
||||
super().activation(activation, output, input)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
workspace13: torch.Tensor,
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
):
|
||||
# If the weights were already dequantized to BF16 at load time
|
||||
# (process_weights_after_loading on devices without a native MXFP8 MoE
|
||||
# kernel), use them directly -- no per-step dequant. MXFP8 weights are
|
||||
# 1-byte FP8 (element_size 1); BF16/FP16 are >= 2 bytes.
|
||||
if w1.element_size() >= 2:
|
||||
# tl.dot requires w and activations share a dtype; .to() is a no-op
|
||||
# when they already match (e.g. both BF16).
|
||||
w1_bf16 = w1.to(hidden_states.dtype)
|
||||
w2_bf16 = w2.to(hidden_states.dtype)
|
||||
else:
|
||||
w1_bf16 = dequant_mxfp8_to_bf16(w1, self.w1_scale_val).to(
|
||||
hidden_states.dtype
|
||||
)
|
||||
w2_bf16 = dequant_mxfp8_to_bf16(w2, self.w2_scale_val).to(
|
||||
hidden_states.dtype
|
||||
)
|
||||
|
||||
super().apply(
|
||||
output=output,
|
||||
hidden_states=hidden_states,
|
||||
w1=w1_bf16,
|
||||
w2=w2_bf16,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=activation,
|
||||
global_num_experts=global_num_experts,
|
||||
expert_map=expert_map,
|
||||
a1q_scale=None,
|
||||
a2_scale=None,
|
||||
workspace13=workspace13,
|
||||
workspace2=workspace2,
|
||||
expert_tokens_meta=expert_tokens_meta,
|
||||
apply_router_weight_on_input=apply_router_weight_on_input,
|
||||
)
|
||||
@@ -0,0 +1,326 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Native MXFP8 (1x32 block, E8M0 scale) MoE for AMD CDNA4 (gfx950) via Triton
|
||||
``tl.dot_scaled`` (hardware microscaling matmul).
|
||||
|
||||
The expert GEMMs consume the FP8 E4M3 weights and their E8M0 block scales
|
||||
directly (no dequant-to-BF16), and activations are MXFP8-quantized per token.
|
||||
On CDNA4 ``dot_scaled`` maps to the native MX matrix-core ops; on other archs
|
||||
Triton upcasts to BF16 (so this stays correct, just not faster) — but the
|
||||
oracle only selects this path on gfx950 and routes everything else to the
|
||||
BF16 ``Mxfp8EmulationTritonExperts`` fallback.
|
||||
|
||||
Structure mirrors vLLM's ``fused_moe_kernel``: tokens are sorted by expert
|
||||
(``moe_align_block_size``); each program computes a ``[BLOCK_M, BLOCK_N]`` tile
|
||||
for one expert, accumulating over K with ``dot_scaled``. SwiGLU-OAI activation
|
||||
and the top-k weighted reduction run in PyTorch between/after the two GEMMs.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.experts.mxfp8_emulation_moe import (
|
||||
Mxfp8TritonExpertsBase,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
|
||||
moe_align_block_size,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
mxfp8_e4m3_quantize,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _mxfp8_grouped_gemm_kernel(
|
||||
a_ptr,
|
||||
a_scale_ptr,
|
||||
b_ptr,
|
||||
b_scale_ptr,
|
||||
c_ptr,
|
||||
topk_weights_ptr,
|
||||
sorted_token_ids_ptr,
|
||||
expert_ids_ptr,
|
||||
num_tokens_post_padded_ptr,
|
||||
N,
|
||||
K,
|
||||
num_valid_tokens,
|
||||
top_k,
|
||||
stride_am,
|
||||
stride_ak,
|
||||
stride_asm,
|
||||
stride_ask,
|
||||
stride_be,
|
||||
stride_bn,
|
||||
stride_bk,
|
||||
stride_bse,
|
||||
stride_bsn,
|
||||
stride_bsk,
|
||||
stride_cm,
|
||||
stride_cn,
|
||||
A_DIV: tl.constexpr,
|
||||
MUL_WEIGHT: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
):
|
||||
pid_m = tl.program_id(0)
|
||||
pid_n = tl.program_id(1)
|
||||
num_post = tl.load(num_tokens_post_padded_ptr)
|
||||
if pid_m * BLOCK_M >= num_post:
|
||||
return
|
||||
|
||||
offs_tid = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
offs_token = tl.load(sorted_token_ids_ptr + offs_tid).to(tl.int64)
|
||||
token_mask = offs_token < num_valid_tokens
|
||||
off_e = tl.load(expert_ids_ptr + pid_m).to(tl.int64)
|
||||
|
||||
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
offs_k = tl.arange(0, BLOCK_K)
|
||||
offs_sk = tl.arange(0, BLOCK_K // 32)
|
||||
a_row = offs_token // A_DIV
|
||||
|
||||
a_ptrs = a_ptr + a_row[:, None] * stride_am + offs_k[None, :] * stride_ak
|
||||
as_ptrs = a_scale_ptr + a_row[:, None] * stride_asm + offs_sk[None, :] * stride_ask
|
||||
b_ptrs = (
|
||||
b_ptr
|
||||
+ off_e * stride_be
|
||||
+ offs_n[:, None] * stride_bn
|
||||
+ offs_k[None, :] * stride_bk
|
||||
)
|
||||
bs_ptrs = (
|
||||
b_scale_ptr
|
||||
+ off_e * stride_bse
|
||||
+ offs_n[:, None] * stride_bsn
|
||||
+ offs_sk[None, :] * stride_bsk
|
||||
)
|
||||
|
||||
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
|
||||
n_mask = offs_n < N
|
||||
for _ in range(0, tl.cdiv(K, BLOCK_K)):
|
||||
a = tl.load(a_ptrs, mask=token_mask[:, None], other=0.0)
|
||||
b = tl.load(b_ptrs, mask=n_mask[:, None], other=0.0)
|
||||
asc = tl.load(as_ptrs, mask=token_mask[:, None], other=0)
|
||||
bsc = tl.load(bs_ptrs, mask=n_mask[:, None], other=0)
|
||||
acc += tl.dot_scaled(a, asc, "e4m3", b.T, bsc, "e4m3")
|
||||
|
||||
a_ptrs += BLOCK_K * stride_ak
|
||||
b_ptrs += BLOCK_K * stride_bk
|
||||
as_ptrs += (BLOCK_K // 32) * stride_ask
|
||||
bs_ptrs += (BLOCK_K // 32) * stride_bsk
|
||||
|
||||
if MUL_WEIGHT:
|
||||
w = tl.load(topk_weights_ptr + offs_token, mask=token_mask, other=0.0)
|
||||
acc = acc * w[:, None]
|
||||
|
||||
c_ptrs = c_ptr + offs_token[:, None] * stride_cm + offs_n[None, :] * stride_cn
|
||||
tl.store(
|
||||
c_ptrs,
|
||||
acc.to(c_ptr.dtype.element_ty),
|
||||
mask=token_mask[:, None] & n_mask[None, :],
|
||||
)
|
||||
|
||||
|
||||
def _grouped_gemm_mxfp8(
|
||||
a_q: torch.Tensor, # [M, K] fp8 e4m3
|
||||
a_scale: torch.Tensor, # [M, K//32] uint8 (E8M0)
|
||||
w: torch.Tensor, # [E, N, K] fp8 e4m3
|
||||
w_scale: torch.Tensor, # [E, N, K//32] uint8 (E8M0)
|
||||
sorted_token_ids: torch.Tensor,
|
||||
expert_ids: torch.Tensor,
|
||||
num_tokens_post_padded: torch.Tensor,
|
||||
num_valid_tokens: int,
|
||||
top_k: int,
|
||||
block_m: int,
|
||||
out_dtype: torch.dtype,
|
||||
a_div: int,
|
||||
mul_weight_by: torch.Tensor | None = None,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
M_routed = num_valid_tokens
|
||||
E, N, K = w.shape
|
||||
assert K % 128 == 0, f"MXFP8 native MoE requires K%128==0, got K={K}"
|
||||
# Under expert parallelism (expert_map set) tokens routed to non-local
|
||||
# experts are dropped from sorted_token_ids, so their output rows are never
|
||||
# written — zero them so the downstream reduction ignores their garbage.
|
||||
alloc = torch.zeros if expert_map is not None else torch.empty
|
||||
out = alloc((M_routed, N), dtype=out_dtype, device=a_q.device)
|
||||
BLOCK_N = 128
|
||||
BLOCK_K = 128
|
||||
grid = (triton.cdiv(sorted_token_ids.shape[0], block_m), triton.cdiv(N, BLOCK_N))
|
||||
_mxfp8_grouped_gemm_kernel[grid](
|
||||
a_q,
|
||||
a_scale,
|
||||
w,
|
||||
w_scale,
|
||||
out,
|
||||
mul_weight_by if mul_weight_by is not None else a_q,
|
||||
sorted_token_ids,
|
||||
expert_ids,
|
||||
num_tokens_post_padded,
|
||||
N,
|
||||
K,
|
||||
num_valid_tokens,
|
||||
top_k,
|
||||
a_q.stride(0),
|
||||
a_q.stride(1),
|
||||
a_scale.stride(0),
|
||||
a_scale.stride(1),
|
||||
w.stride(0),
|
||||
w.stride(1),
|
||||
w.stride(2),
|
||||
w_scale.stride(0),
|
||||
w_scale.stride(1),
|
||||
w_scale.stride(2),
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
A_DIV=a_div,
|
||||
MUL_WEIGHT=mul_weight_by is not None,
|
||||
BLOCK_M=block_m,
|
||||
BLOCK_N=BLOCK_N,
|
||||
BLOCK_K=BLOCK_K,
|
||||
num_warps=8,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def fused_moe_mxfp8_native(
|
||||
hidden_states: torch.Tensor, # [T, H] bf16
|
||||
w13: torch.Tensor, # [E, 2I, H] fp8
|
||||
w13_scale: torch.Tensor, # [E, 2I, H//32] uint8
|
||||
w2: torch.Tensor, # [E, H, I] fp8
|
||||
w2_scale: torch.Tensor, # [E, H, I//32] uint8
|
||||
topk_weights: torch.Tensor, # [T, top_k]
|
||||
topk_ids: torch.Tensor, # [T, top_k] (global expert ids)
|
||||
*,
|
||||
alpha: float,
|
||||
beta: float,
|
||||
limit: float | None,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
T, H = hidden_states.shape
|
||||
top_k = topk_ids.shape[1]
|
||||
M = T * top_k
|
||||
|
||||
block_m = 64
|
||||
sorted_ids, expert_ids, num_post = moe_align_block_size(
|
||||
topk_ids,
|
||||
block_m,
|
||||
global_num_experts,
|
||||
expert_map,
|
||||
ignore_invalid_experts=expert_map is not None,
|
||||
)
|
||||
|
||||
# GEMM1: x (mxfp8) @ w13^T -> [M, 2I]
|
||||
a_q, a_s = mxfp8_e4m3_quantize(hidden_states)
|
||||
g1 = _grouped_gemm_mxfp8(
|
||||
a_q,
|
||||
a_s,
|
||||
w13,
|
||||
w13_scale,
|
||||
sorted_ids,
|
||||
expert_ids,
|
||||
num_post,
|
||||
M,
|
||||
top_k,
|
||||
block_m,
|
||||
hidden_states.dtype,
|
||||
a_div=top_k,
|
||||
expert_map=expert_map,
|
||||
) # [M, 2I]
|
||||
|
||||
# SwiGLU-OAI (split layout: gate=g1[:, :I], up=g1[:, I:]) FUSED with the
|
||||
# GEMM2 MXFP8 activation-quant in one fp32 Triton pass — no bf16 ``act``
|
||||
# round-trip to HBM. Bit-exact vs the unfused swiglu+quant chain on measured
|
||||
# MoE shapes, and ~1.2-1.9x faster on that step in isolation. (Not the #22
|
||||
# ``silu_and_mul_with_clamp`` op: it rounds intermediates to bf16, rel ~3e-3.)
|
||||
# Lazy import: the amd.ops package pulls in the minimax_m3 platform dispatch,
|
||||
# only resolvable after the model module finishes loading.
|
||||
from vllm.models.minimax_m3.amd.ops import swiglu_oai_quantize_mxfp8
|
||||
|
||||
# GEMM2: act (mxfp8) @ w2^T -> [M, H], weighted by topk_weights, then reduce.
|
||||
act_q, act_s = swiglu_oai_quantize_mxfp8(g1, alpha=alpha, beta=beta, limit=limit)
|
||||
g2 = _grouped_gemm_mxfp8(
|
||||
act_q,
|
||||
act_s,
|
||||
w2,
|
||||
w2_scale,
|
||||
sorted_ids,
|
||||
expert_ids,
|
||||
num_post,
|
||||
M,
|
||||
top_k,
|
||||
block_m,
|
||||
torch.float32,
|
||||
a_div=1,
|
||||
mul_weight_by=topk_weights.reshape(-1).to(torch.float32),
|
||||
expert_map=expert_map,
|
||||
) # [M, H] == [T*top_k, H]
|
||||
|
||||
return g2.view(T, top_k, H).sum(dim=1).to(hidden_states.dtype)
|
||||
|
||||
|
||||
class Mxfp8NativeTritonExperts(Mxfp8TritonExpertsBase):
|
||||
"""Native MXFP8 MoE (CDNA4 ``dot_scaled``) on gfx950."""
|
||||
|
||||
@property
|
||||
def quant_dtype(self) -> torch.dtype | str | None:
|
||||
return self.quant_config.quant_dtype
|
||||
|
||||
@property
|
||||
def block_shape(self) -> list[int] | None:
|
||||
return self.quant_config.block_shape
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
# Activations are MXFP8-quantized inside ``fused_moe_mxfp8_native``.
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
return current_platform.is_rocm() and current_platform.supports_mx()
|
||||
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
workspace13: torch.Tensor,
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
):
|
||||
alpha = self.quant_config.gemm1_alpha
|
||||
alpha = 1.702 if alpha is None else float(alpha)
|
||||
beta = self.quant_config.gemm1_beta
|
||||
beta = 1.0 if beta is None else float(beta)
|
||||
limit = self.quant_config.gemm1_clamp_limit
|
||||
limit = None if limit is None else float(limit)
|
||||
out = fused_moe_mxfp8_native(
|
||||
hidden_states,
|
||||
w1,
|
||||
self.w1_scale_val,
|
||||
w2,
|
||||
self.w2_scale_val,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
alpha=alpha,
|
||||
beta=beta,
|
||||
limit=limit,
|
||||
global_num_experts=global_num_experts,
|
||||
expert_map=expert_map,
|
||||
)
|
||||
output.copy_(out)
|
||||
@@ -64,6 +64,15 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
||||
self.quantization_emulation = False
|
||||
super().__init__(moe_config, quant_config)
|
||||
|
||||
self.gemm1_clamp_limit = quant_config.gemm1_clamp_limit
|
||||
# Gated-activation params: silu == swigluoai with alpha=1, beta=0.
|
||||
self.gemm1_alpha = (
|
||||
quant_config.gemm1_alpha if quant_config.gemm1_alpha is not None else 1.0
|
||||
)
|
||||
self.gemm1_beta = (
|
||||
quant_config.gemm1_beta if quant_config.gemm1_beta is not None else 0.0
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
@@ -107,6 +116,7 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
||||
MoEActivation.GELU,
|
||||
MoEActivation.GELU_TANH,
|
||||
MoEActivation.SWIGLUOAI,
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
MoEActivation.SWIGLUSTEP,
|
||||
MoEActivation.SILU_NO_MUL,
|
||||
MoEActivation.GELU_NO_MUL,
|
||||
@@ -129,14 +139,34 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
||||
return TopKWeightAndReduceNoOP()
|
||||
|
||||
def activation(
|
||||
self, activation: MoEActivation, output: torch.Tensor, input: torch.Tensor
|
||||
self,
|
||||
activation: MoEActivation,
|
||||
output: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
gemm1_clamp_limit = self.quant_config.gemm1_clamp_limit
|
||||
if activation == MoEActivation.SILU and gemm1_clamp_limit is not None:
|
||||
swiglu_limit_func(output, input, float(gemm1_clamp_limit))
|
||||
return
|
||||
|
||||
super().activation(activation, output, input)
|
||||
# SWIGLUOAI_UNINTERLEAVE routes to the silu_and_mul_with_clamp kernel and
|
||||
# needs the clamped-SwiGLU params (gemm1_clamp_limit/alpha/beta read from
|
||||
# the quant config in __init__) forwarded; without a clamp_limit it
|
||||
# asserts. Other activations ignore alpha/beta/clamp_limit.
|
||||
if activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
|
||||
assert gemm1_clamp_limit is not None, (
|
||||
"SWIGLUOAI_UNINTERLEAVE requires gemm1_clamp_limit"
|
||||
)
|
||||
|
||||
super().activation(
|
||||
activation,
|
||||
output,
|
||||
input,
|
||||
clamp_limit=gemm1_clamp_limit,
|
||||
alpha=self.gemm1_alpha,
|
||||
beta=self.gemm1_beta,
|
||||
)
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
|
||||
@@ -120,6 +120,8 @@ def FusedMoE(
|
||||
scoring_func: str = "softmax",
|
||||
routed_scaling_factor: float = 1.0,
|
||||
swiglu_limit: float | None = None,
|
||||
swiglu_alpha: float | None = None,
|
||||
swiglu_beta: float | None = None,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
apply_router_weight_on_input: bool = False,
|
||||
activation: str = "silu",
|
||||
@@ -322,6 +324,8 @@ def FusedMoE(
|
||||
device=vllm_config.device_config.device,
|
||||
routing_method=router.routing_method_type, # Not ideal
|
||||
swiglu_limit=swiglu_limit,
|
||||
swiglu_alpha=swiglu_alpha,
|
||||
swiglu_beta=swiglu_beta,
|
||||
max_capture_size=vllm_config.compilation_config.max_cudagraph_capture_size,
|
||||
)
|
||||
|
||||
@@ -353,6 +357,8 @@ def FusedMoE(
|
||||
if not apply_routed_scale_to_output
|
||||
else 1.0,
|
||||
swiglu_limit=swiglu_limit,
|
||||
swiglu_alpha=swiglu_alpha,
|
||||
swiglu_beta=swiglu_beta,
|
||||
# TODO get from router? needs to be truncated?
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
apply_router_weight_on_input=apply_router_weight_on_input,
|
||||
|
||||
@@ -880,9 +880,18 @@ class FusedMoEExpertsModular(FusedMoEExperts):
|
||||
return N if not activation.is_gated else N // 2
|
||||
|
||||
def activation(
|
||||
self, activation: MoEActivation, output: torch.Tensor, input: torch.Tensor
|
||||
self,
|
||||
activation: MoEActivation,
|
||||
output: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
*,
|
||||
clamp_limit: float | None = None,
|
||||
alpha: float = 1.0,
|
||||
beta: float = 0.0,
|
||||
) -> None:
|
||||
apply_moe_activation(activation, output, input)
|
||||
apply_moe_activation(
|
||||
activation, output, input, clamp_limit=clamp_limit, alpha=alpha, beta=beta
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
def finalize_weight_and_reduce_impl(self) -> TopKWeightAndReduce:
|
||||
|
||||
@@ -52,6 +52,13 @@ class Fp8MoeBackend(Enum):
|
||||
BATCHED_VLLM_CUTLASS = "BATCHED_VLLM_CUTLASS"
|
||||
XPU = "XPU"
|
||||
CPU = "CPU"
|
||||
# Dequantize-to-BF16 emulation for MXFP8 on devices without a native
|
||||
# MXFP8 MoE kernel (e.g. ROCm). Weights pass through unchanged here.
|
||||
EMULATION = "EMULATION"
|
||||
# MXFP8 MoE via a Triton ``dot_scaled`` kernel that lowers to CDNA4
|
||||
# (gfx950) native MX matrix-core ops. Weights stay in MXFP8 (no load-time
|
||||
# format conversion); the FP8 values + E8M0 scales are consumed directly.
|
||||
NATIVE_MXFP8 = "NATIVE_MXFP8"
|
||||
|
||||
|
||||
def _get_priority_backends(
|
||||
@@ -463,6 +470,10 @@ def convert_to_fp8_moe_kernel_format(
|
||||
Fp8MoeBackend.VLLM_CUTLASS,
|
||||
Fp8MoeBackend.BATCHED_VLLM_CUTLASS,
|
||||
Fp8MoeBackend.XPU,
|
||||
# EMULATION dequantizes weights at runtime; NATIVE_MXFP8 consumes
|
||||
# the MXFP8 weights as-is — neither needs a load-time layout change.
|
||||
Fp8MoeBackend.EMULATION,
|
||||
Fp8MoeBackend.NATIVE_MXFP8,
|
||||
]:
|
||||
raise ValueError(f"Unsupported FP8 MoE backend: {fp8_backend.value}")
|
||||
|
||||
@@ -481,6 +492,8 @@ def make_fp8_moe_quant_config(
|
||||
per_act_token_quant: bool = False,
|
||||
per_out_ch_quant: bool = False,
|
||||
swiglu_limit: float | None = None,
|
||||
gemm1_alpha: float | None = None,
|
||||
gemm1_beta: float | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
Create FusedMoEQuantConfig for the specified FP8 Backend.
|
||||
@@ -503,6 +516,9 @@ def make_fp8_moe_quant_config(
|
||||
w1_bias=w1_bias,
|
||||
w2_bias=w2_bias,
|
||||
block_shape=block_shape,
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
gemm1_clamp_limit=swiglu_limit,
|
||||
)
|
||||
|
||||
# Flashinfer CUTLASS per-tensor uses single dq scale
|
||||
@@ -522,10 +538,9 @@ def make_fp8_moe_quant_config(
|
||||
g2_alphas=(w2_scale * a2_scale).squeeze(),
|
||||
gemm1_clamp_limit=swiglu_limit,
|
||||
)
|
||||
# MXFP8 uses "mxfp8" quant_dtype so the prepare step dispatches to
|
||||
# _mxfp8_e4m3_quantize rather than standard FP8 block quantization.
|
||||
# Non-swizzled layout is required since the TRTLLM kernel expects
|
||||
# scales in (num_tokens, hidden_dim // 32) format.
|
||||
# MXFP8 (block [1, 32]) dispatches to the mxfp8 activation quant. Scales are
|
||||
# the non-swizzled (num_tokens, hidden_dim // 32) uint8 UE8M0 layout for all
|
||||
# backends; the DeepGEMM expert permute repacks them for the grouped GEMM.
|
||||
if block_shape == [1, 32]:
|
||||
return FusedMoEQuantConfig.make(
|
||||
"mxfp8",
|
||||
@@ -537,6 +552,8 @@ def make_fp8_moe_quant_config(
|
||||
a2_scale=a2_scale,
|
||||
block_shape=block_shape,
|
||||
is_scale_swizzled=False,
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
gemm1_clamp_limit=swiglu_limit,
|
||||
)
|
||||
|
||||
|
||||
@@ -12,22 +12,43 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kMxfp8Dynamic,
|
||||
kMxfp8Static,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_SUPPORTED_BACKENDS = (
|
||||
Fp8MoeBackend.FLASHINFER_TRTLLM,
|
||||
Fp8MoeBackend.DEEPGEMM,
|
||||
Fp8MoeBackend.MARLIN,
|
||||
Fp8MoeBackend.XPU,
|
||||
)
|
||||
|
||||
_BACKEND_NAME_MAP: dict[str, Fp8MoeBackend] = {
|
||||
"flashinfer_trtllm": Fp8MoeBackend.FLASHINFER_TRTLLM,
|
||||
"deep_gemm": Fp8MoeBackend.DEEPGEMM,
|
||||
"marlin": Fp8MoeBackend.MARLIN,
|
||||
"xpu": Fp8MoeBackend.XPU,
|
||||
}
|
||||
|
||||
|
||||
def _mxfp8_backend_to_kernel_cls(
|
||||
backend: Fp8MoeBackend,
|
||||
) -> list[type[mk.FusedMoEExperts]]:
|
||||
"""Resolve the MXFP8 expert classes for a backend.
|
||||
|
||||
DeepGEMM resolves directly to ``DeepGemmExperts`` (not the
|
||||
``TritonOrDeepGemmExperts`` wrapper, whose Triton fallback cannot handle the
|
||||
MXFP8 1x32 scheme); all other backends defer to the FP8 resolver.
|
||||
"""
|
||||
if backend == Fp8MoeBackend.DEEPGEMM:
|
||||
from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import (
|
||||
DeepGemmExperts,
|
||||
)
|
||||
|
||||
return [DeepGemmExperts]
|
||||
return backend_to_kernel_cls(backend)
|
||||
|
||||
|
||||
def _select_kernel_cls(
|
||||
backend: Fp8MoeBackend,
|
||||
config: FusedMoEConfig,
|
||||
@@ -39,7 +60,7 @@ def _select_kernel_cls(
|
||||
else mk.FusedMoEActivationFormat.Standard
|
||||
)
|
||||
last_reason: str | None = None
|
||||
for cls in backend_to_kernel_cls(backend):
|
||||
for cls in _mxfp8_backend_to_kernel_cls(backend):
|
||||
supported, reason = cls.is_supported_config(
|
||||
cls,
|
||||
config,
|
||||
@@ -55,6 +76,29 @@ def _select_kernel_cls(
|
||||
)
|
||||
|
||||
|
||||
def _select_rocm_mxfp8_backend() -> tuple[Fp8MoeBackend, type[mk.FusedMoEExperts]]:
|
||||
"""ROCm fallback when vendor MXFP8 backends are unavailable."""
|
||||
|
||||
if current_platform.supports_mx():
|
||||
from vllm.model_executor.layers.fused_moe.experts.mxfp8_native_moe import (
|
||||
Mxfp8NativeTritonExperts,
|
||||
)
|
||||
|
||||
logger.info_once("Using native CDNA4 (gfx950) MXFP8 dot_scaled MoE backend.")
|
||||
return Fp8MoeBackend.NATIVE_MXFP8, Mxfp8NativeTritonExperts
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.experts.mxfp8_emulation_moe import (
|
||||
Mxfp8EmulationTritonExperts,
|
||||
)
|
||||
|
||||
logger.info_once(
|
||||
"No native MXFP8 MoE backend available on this device; "
|
||||
"MXFP8 weights will be dequantized to BF16 once at load time and the "
|
||||
"MoE will run in BF16 (no per-step dequant)."
|
||||
)
|
||||
return Fp8MoeBackend.EMULATION, Mxfp8EmulationTritonExperts
|
||||
|
||||
|
||||
def select_mxfp8_moe_backend(
|
||||
config: FusedMoEConfig,
|
||||
) -> tuple[Fp8MoeBackend, type[mk.FusedMoEExperts]]:
|
||||
@@ -88,4 +132,8 @@ def select_mxfp8_moe_backend(
|
||||
logger.info_once("Using '%s' MxFp8 MoE backend.", backend.value)
|
||||
return backend, experts_cls
|
||||
|
||||
# simplify the logic for rocm, refactor later when more backends are supported
|
||||
if current_platform.is_rocm():
|
||||
return _select_rocm_mxfp8_backend()
|
||||
|
||||
raise ValueError("No MXFP8 MoE backends available.")
|
||||
|
||||
@@ -72,6 +72,8 @@ class RoutedExperts(PluggableLayer):
|
||||
scoring_func: str = "softmax",
|
||||
routed_scaling_factor: float = 1.0,
|
||||
swiglu_limit: float | None = None,
|
||||
swiglu_alpha: float | None = None,
|
||||
swiglu_beta: float | None = None,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
apply_router_weight_on_input: bool = False,
|
||||
):
|
||||
@@ -103,6 +105,8 @@ class RoutedExperts(PluggableLayer):
|
||||
self.scoring_func = scoring_func
|
||||
self.routed_scaling_factor = routed_scaling_factor
|
||||
self.swiglu_limit = swiglu_limit
|
||||
self.swiglu_alpha = swiglu_alpha
|
||||
self.swiglu_beta = swiglu_beta
|
||||
self.e_score_correction_bias = e_score_correction_bias
|
||||
self.apply_router_weight_on_input = apply_router_weight_on_input
|
||||
# End random parameters
|
||||
|
||||
@@ -29,9 +29,9 @@ class GateLinear(ReplicatedLinear):
|
||||
DSV3_SUPPORTED_NUM_EXPERTS = [256, 384]
|
||||
DSV3_SUPPORTED_HIDDEN_SIZES = [7168]
|
||||
|
||||
# Dimensions supported by the fp32 specialized kernel
|
||||
FP32_SUPPORTED_NUM_EXPERTS = [256]
|
||||
FP32_SUPPORTED_HIDDEN_SIZES = [3072]
|
||||
# (hidden_size, num_experts) pairs with an instantiated fp32 kernel:
|
||||
# (3072, 256) -> MiniMax-M2/M2.5, (6144, 128) -> MiniMax-M3
|
||||
FP32_SUPPORTED_SHAPES = {(3072, 256), (6144, 128)}
|
||||
FP32_MAX_TOKENS = 32
|
||||
|
||||
def __init__(
|
||||
@@ -82,8 +82,7 @@ class GateLinear(ReplicatedLinear):
|
||||
and self.weight.dtype == torch.float32
|
||||
and current_platform.is_cuda()
|
||||
and (is_hopper or is_blackwell)
|
||||
and output_size in self.FP32_SUPPORTED_NUM_EXPERTS
|
||||
and input_size in self.FP32_SUPPORTED_HIDDEN_SIZES
|
||||
and (input_size, output_size) in self.FP32_SUPPORTED_SHAPES
|
||||
)
|
||||
|
||||
# cuBLAS bf16→fp32 eligibility
|
||||
|
||||
@@ -11,7 +11,6 @@ import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FUSED_MOE_UNQUANTIZED_CONFIG,
|
||||
FusedMoEConfig,
|
||||
FusedMoEQuantConfig,
|
||||
biased_moe_quant_config,
|
||||
@@ -184,11 +183,10 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
|
||||
if not is_weight_update:
|
||||
# Setup moe kernel only on the first call. For the unquantized
|
||||
# method, moe_quant_config is either the constant
|
||||
# FUSED_MOE_UNQUANTIZED_CONFIG or biased_moe_quant_config(...)
|
||||
# which references layer.w{13,2}_bias; since weight updates
|
||||
# mutate those bias tensors in place, the kernel does not need
|
||||
# to be re-built.
|
||||
# method, moe_quant_config carries no quantized scales -- only
|
||||
# optional w{13,2}_bias references and SwiGLU gate params. Since
|
||||
# weight updates mutate those bias tensors in place, the kernel
|
||||
# does not need to be re-built.
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
assert self.moe_quant_config is not None
|
||||
assert self.experts_cls is not None
|
||||
@@ -272,13 +270,27 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig:
|
||||
# SwiGLU/swigluoai gate params live on the layer; plumb them into the
|
||||
# quant config so the fused activation (e.g. swigluoai_uninterleave on
|
||||
# MiniMax-M3) receives gemm1_clamp_limit/alpha/beta.
|
||||
gemm1_alpha = getattr(layer, "swiglu_alpha", None)
|
||||
gemm1_beta = getattr(layer, "swiglu_beta", None)
|
||||
gemm1_clamp_limit = getattr(layer, "swiglu_limit", None)
|
||||
|
||||
if self.moe.has_bias:
|
||||
return biased_moe_quant_config(
|
||||
layer.w13_bias,
|
||||
layer.w2_bias,
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
gemm1_clamp_limit=gemm1_clamp_limit,
|
||||
)
|
||||
else:
|
||||
return FUSED_MOE_UNQUANTIZED_CONFIG
|
||||
|
||||
return FusedMoEQuantConfig.make(
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
gemm1_clamp_limit=gemm1_clamp_limit,
|
||||
)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
|
||||
@@ -313,6 +313,8 @@ def moe_kernel_quantize_input(
|
||||
"moe_kernel_quantize_input does not support quant_dtype='mxfp8' MOE "
|
||||
"quantization emulation. Please open an issue."
|
||||
)
|
||||
# Non-swizzled (M, K/32) uint8 UE8M0 scales; deepgemm_moe_permute packs
|
||||
# them for DeepGEMM, TRTLLM takes them as-is.
|
||||
return _mxfp8_e4m3_quantize(
|
||||
A,
|
||||
A_scale,
|
||||
|
||||
@@ -1105,6 +1105,7 @@ class QKVParallelLinear(ColumnParallelLinear):
|
||||
|
||||
shard_offset = self._get_shard_offset_mapping(loaded_shard_id)
|
||||
shard_size = self._get_shard_size_mapping(loaded_shard_id)
|
||||
assert shard_offset is not None and shard_size is not None
|
||||
|
||||
if isinstance(param, BlockQuantScaleParameter):
|
||||
weight_block_size = getattr(self, "weight_block_size", None)
|
||||
@@ -1302,6 +1303,191 @@ class QKVParallelLinear(ColumnParallelLinear):
|
||||
param_data.copy_(loaded_weight)
|
||||
|
||||
|
||||
class MinimaxM3QKVParallelLinearWithIndexer(QKVParallelLinear):
|
||||
"""QKV projection fused with a lightning-indexer's index_q/index_k.
|
||||
|
||||
NOTE: MiniMax-M3-specific. This is tailored to the M3 sparse-attention
|
||||
layers (it assumes the indexer's head count equals the KV head count and
|
||||
shares the main head_dim); it is not a general-purpose linear layer. It
|
||||
lives here only to sit alongside QKVParallelLinear, whose sharding /
|
||||
weight-loading machinery it reuses.
|
||||
|
||||
A single column-parallel GEMM emits, per rank::
|
||||
|
||||
[q | k | v | index_q | index_k]
|
||||
|
||||
``index_q`` must have the same head count as the KV heads
|
||||
(``total_num_index_heads == total_num_kv_heads``) and ``index_head_size ==
|
||||
head_size``, so it shards exactly like K/V -- including the KV-head
|
||||
*replication* path when ``tp_size > total_num_kv_heads`` (this is what makes
|
||||
a TP size greater than the KV-head count work). ``index_k`` is a single
|
||||
shared head, replicated to every rank.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
head_size: int,
|
||||
total_num_heads: int,
|
||||
total_num_kv_heads: int,
|
||||
total_num_index_heads: int,
|
||||
index_head_size: int,
|
||||
bias: bool = False,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
# index_q rides the KV-head sharding/replication path, so its head count
|
||||
# must match the KV heads.
|
||||
assert total_num_index_heads == total_num_kv_heads, (
|
||||
"MinimaxM3QKVParallelLinearWithIndexer requires "
|
||||
"total_num_index_heads == total_num_kv_heads"
|
||||
)
|
||||
self.hidden_size = hidden_size
|
||||
self.head_size = head_size
|
||||
self.v_head_size = head_size
|
||||
self.total_num_heads = total_num_heads
|
||||
self.total_num_kv_heads = total_num_kv_heads
|
||||
self.total_num_index_heads = total_num_index_heads
|
||||
self.index_head_size = index_head_size
|
||||
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
self.num_heads = divide(self.total_num_heads, tp_size)
|
||||
if tp_size >= self.total_num_kv_heads:
|
||||
self.num_kv_heads = 1
|
||||
self.num_kv_head_replicas = divide(tp_size, self.total_num_kv_heads)
|
||||
else:
|
||||
self.num_kv_heads = divide(self.total_num_kv_heads, tp_size)
|
||||
self.num_kv_head_replicas = 1
|
||||
# index_q shards identically to the KV heads.
|
||||
self.num_index_heads = self.num_kv_heads
|
||||
|
||||
# Global per-group sizes (replicated groups counted x tp_size, matching
|
||||
# the QKVParallelLinear convention). index_k is a single replicated head.
|
||||
q = self.num_heads * self.head_size
|
||||
kv = self.num_kv_heads * self.head_size
|
||||
iq = self.num_index_heads * self.index_head_size
|
||||
ik = self.index_head_size
|
||||
self.output_sizes = [
|
||||
q * tp_size, # q
|
||||
kv * tp_size, # k
|
||||
kv * tp_size, # v
|
||||
iq * tp_size, # index_q
|
||||
ik * tp_size, # index_k (replicated)
|
||||
]
|
||||
|
||||
# Skip QKVParallelLinear.__init__ (3-group layout); build the 5-group
|
||||
# column-parallel weight directly.
|
||||
ColumnParallelLinear.__init__(
|
||||
self,
|
||||
input_size=self.hidden_size,
|
||||
output_size=sum(self.output_sizes),
|
||||
bias=bias,
|
||||
gather_output=False,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
)
|
||||
|
||||
def validate_shard_id(self, loaded_shard_id: str | None) -> None:
|
||||
if loaded_shard_id is None:
|
||||
return
|
||||
if loaded_shard_id not in ("q", "k", "v", "index_q", "index_k"):
|
||||
raise ValueError(
|
||||
"Shard id for MinimaxM3QKVParallelLinearWithIndexer must be one of "
|
||||
"'q', 'k', 'v', 'index_q', 'index_k'; got "
|
||||
f"{loaded_shard_id}."
|
||||
)
|
||||
|
||||
def _get_shard_offset_mapping(self, loaded_shard_id: str) -> int | None:
|
||||
h = self.head_size
|
||||
nq, nkv, nidx = self.num_heads, self.num_kv_heads, self.num_index_heads
|
||||
return {
|
||||
"q": 0,
|
||||
"k": nq * h,
|
||||
"v": (nq + nkv) * h,
|
||||
"index_q": (nq + 2 * nkv) * h,
|
||||
"index_k": (nq + 2 * nkv + nidx) * h,
|
||||
}.get(loaded_shard_id)
|
||||
|
||||
def _get_shard_size_mapping(self, loaded_shard_id: str) -> int | None:
|
||||
h = self.head_size
|
||||
return {
|
||||
"q": self.num_heads * h,
|
||||
"k": self.num_kv_heads * h,
|
||||
"v": self.num_kv_heads * h,
|
||||
"index_q": self.num_index_heads * h,
|
||||
"index_k": self.index_head_size,
|
||||
}.get(loaded_shard_id)
|
||||
|
||||
def weight_loader_v2(
|
||||
self,
|
||||
param: BasevLLMParameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
loaded_shard_id: str | None = None,
|
||||
) -> None:
|
||||
self.validate_shard_id(loaded_shard_id)
|
||||
# Index checkpoints are never pre-fused on disk; a shard id is always given.
|
||||
assert loaded_shard_id in ("q", "k", "v", "index_q", "index_k")
|
||||
|
||||
shard_offset = self._get_shard_offset_mapping(loaded_shard_id)
|
||||
shard_size = self._get_shard_size_mapping(loaded_shard_id)
|
||||
assert shard_offset is not None and shard_size is not None
|
||||
if isinstance(param, BlockQuantScaleParameter):
|
||||
weight_block_size = getattr(self, "weight_block_size", None)
|
||||
shard_size, shard_offset = adjust_block_scale_shard(
|
||||
weight_block_size, shard_size, shard_offset
|
||||
)
|
||||
|
||||
# index_k is fully replicated: num_heads == tp_size makes
|
||||
# load_qkv_weight pick shard_id_int == 0 on every rank. q/k/v/index_q ride
|
||||
# the KV-head replication factor.
|
||||
num_heads = (
|
||||
self.tp_size if loaded_shard_id == "index_k" else self.num_kv_head_replicas
|
||||
)
|
||||
param.load_qkv_weight(
|
||||
loaded_weight=loaded_weight,
|
||||
num_heads=num_heads,
|
||||
shard_id=loaded_shard_id,
|
||||
shard_offset=shard_offset,
|
||||
shard_size=shard_size,
|
||||
tp_rank=self.tp_rank,
|
||||
)
|
||||
|
||||
def weight_loader(
|
||||
self,
|
||||
param: Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
loaded_shard_id: str | None = None,
|
||||
) -> None:
|
||||
# Unquantized (bf16) path. MXFP8 checkpoints use weight_loader_v2; this
|
||||
# keeps an unquantized load correct too.
|
||||
self.validate_shard_id(loaded_shard_id)
|
||||
assert loaded_shard_id in ("q", "k", "v", "index_q", "index_k")
|
||||
output_dim = getattr(param, "output_dim", None)
|
||||
assert output_dim is not None
|
||||
|
||||
shard_offset = self._get_shard_offset_mapping(loaded_shard_id)
|
||||
shard_size = self._get_shard_size_mapping(loaded_shard_id)
|
||||
assert shard_offset is not None and shard_size is not None
|
||||
if isinstance(param, BlockQuantScaleParameter):
|
||||
weight_block_size = getattr(self, "weight_block_size", None)
|
||||
shard_size, shard_offset = adjust_block_scale_shard(
|
||||
weight_block_size, shard_size, shard_offset
|
||||
)
|
||||
|
||||
param_data = param.data.narrow(output_dim, shard_offset, shard_size)
|
||||
if loaded_shard_id == "q":
|
||||
shard_rank = self.tp_rank
|
||||
elif loaded_shard_id == "index_k":
|
||||
shard_rank = 0 # replicated to every rank
|
||||
else:
|
||||
shard_rank = self.tp_rank // self.num_kv_head_replicas
|
||||
loaded_weight = loaded_weight.narrow(
|
||||
output_dim, shard_rank * shard_size, shard_size
|
||||
)
|
||||
assert param_data.shape == loaded_weight.shape
|
||||
param_data.copy_(loaded_weight)
|
||||
|
||||
|
||||
# --8<-- [start:row_parallel_linear]
|
||||
@PluggableLayer.register("row_parallel_linear")
|
||||
class RowParallelLinear(LinearBase):
|
||||
|
||||
@@ -163,17 +163,18 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
|
||||
"deepseek_v4_fp8": DeepseekV4FP8Config,
|
||||
"humming": HummingConfig,
|
||||
"online": OnlineQuantizationConfig,
|
||||
# MiniMax-style checkpoints tag `quant_method: "mxfp8"`; load with the
|
||||
# ModelOpt MXFP8 config (same format). The "mxfp8" online shorthand
|
||||
# below only applies to the `--quantization mxfp8` CLI path.
|
||||
"mxfp8": ModelOptMxFp8Config,
|
||||
}
|
||||
|
||||
# Register online shorthands as quantization methods so the user can
|
||||
# specify "LLM(..., quantization='fp8_per_tensor')" as shorthand for
|
||||
# creating a more complicated online quant config object.
|
||||
# Register online shorthands (e.g. "fp8_per_tensor") as quant methods.
|
||||
# setdefault so a shorthand that is also a checkpoint method (e.g. "mxfp8")
|
||||
# keeps its checkpoint config; the shorthand still works via the
|
||||
# `--quantization` CLI path in `resolve_quantization_config`.
|
||||
for shorthand in _ONLINE_SHORTHANDS:
|
||||
assert shorthand not in method_to_config, (
|
||||
f"Online quant shorthand {shorthand!r} conflicts with an "
|
||||
f"existing quantization method"
|
||||
)
|
||||
method_to_config[shorthand] = OnlineQuantizationConfig
|
||||
method_to_config.setdefault(shorthand, OnlineQuantizationConfig)
|
||||
|
||||
# Update the `method_to_config` with customized quantization methods.
|
||||
method_to_config.update(_CUSTOMIZED_METHOD_TO_QUANT_CONFIG)
|
||||
|
||||
@@ -2,11 +2,12 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from fnmatch import fnmatch
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
import vllm.envs as envs
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
@@ -27,6 +28,7 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
SharedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
Fp8MoeBackend,
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
make_fp8_moe_kernel,
|
||||
make_fp8_moe_quant_config,
|
||||
@@ -1720,6 +1722,22 @@ class ModelOptMxFp8Config(ModelOptQuantConfigBase):
|
||||
return "modelopt_mxfp8"
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: dict[str, Any]) -> "ModelOptMxFp8Config":
|
||||
# MiniMax-style checkpoints tag `quant_method: "mxfp8"` + `ignored_layers`
|
||||
# (same on-disk format as ModelOpt MXFP8); normalize to the ModelOpt
|
||||
# schema and reuse the shared parser.
|
||||
if "quantization" not in config and not config.get("quant_algo"):
|
||||
config = {
|
||||
"quant_method": "modelopt",
|
||||
"quantization": {
|
||||
"quant_algo": "MXFP8",
|
||||
"kv_cache_quant_algo": config.get("kv_cache_quant_algo"),
|
||||
"exclude_modules": config.get("ignored_layers", []) or [],
|
||||
},
|
||||
}
|
||||
return cast("ModelOptMxFp8Config", super().from_config(config))
|
||||
|
||||
@classmethod
|
||||
def _from_config(
|
||||
cls,
|
||||
@@ -1823,6 +1841,12 @@ class ModelOptMxFp8LinearMethod(LinearMethodBase):
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
# Idempotent: the emulation kernel may dequant the weight to BF16 at load
|
||||
# time (>=2-byte). If already converted, there is nothing left to do --
|
||||
# avoid re-running the MXFP8-only validation/conversion below.
|
||||
if layer.weight.element_size() >= 2:
|
||||
return
|
||||
|
||||
# Validate weight tensor
|
||||
if layer.weight.ndim != 2:
|
||||
raise ValueError(
|
||||
@@ -2065,6 +2089,44 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
|
||||
torch.stack(w2_scale_shuffled).contiguous(),
|
||||
)
|
||||
|
||||
def _dequant_mxfp8_weights_to_bf16(self, layer: RoutedExperts) -> None:
|
||||
"""One-time MXFP8->BF16 weight dequant for the emulation path.
|
||||
|
||||
On devices without a native MXFP8 MoE kernel (e.g. gfx942 / MI300),
|
||||
``Mxfp8EmulationTritonExperts`` otherwise dequantizes every expert
|
||||
weight to BF16 on *every* forward step -- the dominant cost (conc1
|
||||
~1.3 tok/s). Doing the dequant once here and replacing the MXFP8
|
||||
parameters with BF16 makes the MoE run exactly like a plain BF16
|
||||
checkpoint (full precision, no per-step dequant); SwiGLU-OAI is still
|
||||
applied by the experts' ``activation()`` override. The MXFP8 weights
|
||||
are freed by ``replace_parameter`` (BF16 is 2x their size; the small
|
||||
E8M0 scale tensors are left in place, unused).
|
||||
"""
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
dequant_mxfp8_to_bf16,
|
||||
)
|
||||
|
||||
target_dtype = getattr(layer, "orig_dtype", torch.bfloat16)
|
||||
num_experts = layer.w13_weight.shape[0]
|
||||
|
||||
# dequant_mxfp8_to_bf16 handles arbitrary leading dims (*x.shape[:-1]),
|
||||
# so dequant the whole [E, N, K] weight in one vectorized call.
|
||||
w13_bf16 = dequant_mxfp8_to_bf16(layer.w13_weight, layer.w13_weight_scale).to(
|
||||
target_dtype
|
||||
)
|
||||
w2_bf16 = dequant_mxfp8_to_bf16(layer.w2_weight, layer.w2_weight_scale).to(
|
||||
target_dtype
|
||||
)
|
||||
|
||||
replace_parameter(layer, "w13_weight", w13_bf16)
|
||||
replace_parameter(layer, "w2_weight", w2_bf16)
|
||||
|
||||
logger.info_once(
|
||||
"MXFP8->BF16 load-time dequant complete (%d experts/layer); MoE "
|
||||
"now runs in BF16 with no per-step dequant.",
|
||||
num_experts,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
# TODO(bnell): why is this required only for mxfp8?
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
@@ -2102,6 +2164,17 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
|
||||
# No native MXFP8 MoE kernel on this device (e.g. gfx942): the emulation
|
||||
# experts would dequant MXFP8->BF16 every forward step. Convert the
|
||||
# weights to BF16 once, here, so the MoE runs like a BF16 checkpoint.
|
||||
# Opt out (VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD=0) to keep the 1-byte
|
||||
# MXFP8 weights and dequant per-step (~half the memory, much slower).
|
||||
if (
|
||||
self.mxfp8_backend == Fp8MoeBackend.EMULATION
|
||||
and envs.VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD
|
||||
):
|
||||
self._dequant_mxfp8_weights_to_bf16(layer)
|
||||
|
||||
def maybe_make_prepare_finalize(
|
||||
self,
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
@@ -2131,6 +2204,9 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
|
||||
a1_scale=None,
|
||||
a2_scale=None,
|
||||
block_shape=self.weight_block_size,
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
)
|
||||
|
||||
def apply_monolithic(
|
||||
|
||||
@@ -159,75 +159,104 @@ def _silu_mul_quant_fp8_packed_kernel(
|
||||
output_q_stride_m,
|
||||
output_scale_stride_k,
|
||||
clamp_limit,
|
||||
alpha,
|
||||
beta,
|
||||
N: tl.constexpr,
|
||||
NUM_GROUPS: tl.constexpr,
|
||||
GROUPS_PER_ROW: tl.constexpr,
|
||||
PACKS_PER_ROW: tl.constexpr,
|
||||
fp8_min: tl.constexpr,
|
||||
fp8_max: tl.constexpr,
|
||||
GROUP_SIZE: tl.constexpr,
|
||||
PACKS_PER_CTA: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
HAS_CLAMP: tl.constexpr,
|
||||
):
|
||||
N_2: tl.constexpr = N // 2
|
||||
GROUPS_PER_PACK: tl.constexpr = 4
|
||||
hidden_size: tl.constexpr = N // 2
|
||||
|
||||
pid_pack = tl.program_id(0)
|
||||
pid_m = tl.program_id(1)
|
||||
m_offset = pid_m.to(tl.int64) * BLOCK_M
|
||||
pack_tile = tl.program_id(0)
|
||||
row_start = tl.program_id(1).to(tl.int64) * BLOCK_M
|
||||
row_step = tl.num_programs(1).to(tl.int64) * BLOCK_M
|
||||
|
||||
if m_offset >= M:
|
||||
return
|
||||
groups_per_cta: tl.constexpr = PACKS_PER_CTA * GROUPS_PER_PACK
|
||||
elems_per_cta: tl.constexpr = groups_per_cta * GROUP_SIZE
|
||||
col_start = pack_tile * elems_per_cta
|
||||
col_offsets = tl.arange(0, elems_per_cta)
|
||||
row_offsets = tl.arange(0, BLOCK_M)
|
||||
pack_offsets = tl.arange(0, PACKS_PER_CTA)
|
||||
|
||||
offs_m = tl.arange(0, BLOCK_M)
|
||||
offs_n = tl.arange(0, GROUP_SIZE)
|
||||
row_mask = (m_offset + offs_m) < M
|
||||
col_mask = (col_start + col_offsets) < (GROUPS_PER_ROW * GROUP_SIZE)
|
||||
|
||||
base_row_offset = (m_offset + offs_m[:, None]) * input_stride_m
|
||||
base_out_offset = (m_offset + offs_m[:, None]) * output_q_stride_m
|
||||
# persistent with grid_m-stride loop
|
||||
while row_start < M:
|
||||
rows = row_start + row_offsets
|
||||
row_mask = rows < M
|
||||
input_row_start = rows[:, None] * input_stride_m
|
||||
output_row_start = rows[:, None] * output_q_stride_m
|
||||
|
||||
packed_scale = tl.zeros((BLOCK_M,), dtype=tl.int32)
|
||||
gate_flat = tl.load(
|
||||
input_ptr + input_row_start + col_start + col_offsets[None, :],
|
||||
mask=row_mask[:, None] & col_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
up_flat = tl.load(
|
||||
input_ptr
|
||||
+ input_row_start
|
||||
+ hidden_size
|
||||
+ col_start
|
||||
+ col_offsets[None, :],
|
||||
mask=row_mask[:, None] & col_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
|
||||
for pack_idx in tl.static_range(4):
|
||||
group_id = pid_pack * 4 + pack_idx
|
||||
gate = tl.reshape(gate_flat, (BLOCK_M, groups_per_cta, GROUP_SIZE)).to(
|
||||
tl.float32
|
||||
)
|
||||
up = tl.reshape(up_flat, (BLOCK_M, groups_per_cta, GROUP_SIZE)).to(tl.float32)
|
||||
|
||||
if group_id < NUM_GROUPS:
|
||||
n_offset = group_id * GROUP_SIZE
|
||||
if HAS_CLAMP:
|
||||
gate = tl.minimum(gate, clamp_limit)
|
||||
up = tl.clamp(up, -clamp_limit, clamp_limit)
|
||||
|
||||
act_ptrs = input_ptr + base_row_offset + n_offset + offs_n[None, :]
|
||||
act_in = tl.load(act_ptrs, mask=row_mask[:, None], other=0.0)
|
||||
# Unified gated activation: silu == swigluoai with alpha=1, beta=0.
|
||||
# glu = gate * sigmoid(alpha * gate); y = (up + beta) * glu
|
||||
glu = gate / (1.0 + tl.exp(-gate * alpha))
|
||||
y = (up + beta) * glu
|
||||
# Round through bf16 to match unfused precision path
|
||||
y = y.to(tl.bfloat16).to(tl.float32)
|
||||
|
||||
mul_ptrs = act_ptrs + N_2
|
||||
mul_in = tl.load(mul_ptrs, mask=row_mask[:, None], other=0.0)
|
||||
absmax = tl.max(tl.abs(y), axis=2)
|
||||
scale_raw = tl.maximum(absmax / fp8_max, 1e-10)
|
||||
exponent = tl.ceil(tl.log2(scale_raw))
|
||||
scale = tl.math.exp2(exponent)
|
||||
|
||||
act_f32 = act_in.to(tl.float32)
|
||||
mul_f32 = mul_in.to(tl.float32)
|
||||
y_q = tl.clamp(y / scale[:, :, None], fp8_min, fp8_max)
|
||||
|
||||
if HAS_CLAMP:
|
||||
act_f32 = tl.minimum(act_f32, clamp_limit)
|
||||
mul_f32 = tl.clamp(mul_f32, -clamp_limit, clamp_limit)
|
||||
y_q_flat = tl.reshape(y_q, (BLOCK_M, elems_per_cta))
|
||||
tl.store(
|
||||
output_q_ptr + output_row_start + col_start + col_offsets[None, :],
|
||||
y_q_flat.to(output_q_ptr.dtype.element_ty),
|
||||
mask=row_mask[:, None] & col_mask[None, :],
|
||||
)
|
||||
|
||||
y = (act_f32 / (1.0 + tl.exp(-act_f32))) * mul_f32
|
||||
# Round through bf16 to match unfused precision path
|
||||
y = y.to(tl.bfloat16).to(tl.float32)
|
||||
scale_byte = tl.clamp(exponent + 127.0, 0.0, 255.0).to(tl.int32)
|
||||
scale_bytes = tl.reshape(scale_byte, (BLOCK_M, PACKS_PER_CTA, GROUPS_PER_PACK))
|
||||
shifts = tl.arange(0, GROUPS_PER_PACK) * 8
|
||||
packed_scale = tl.sum(scale_bytes << shifts[None, None, :], axis=2)
|
||||
|
||||
absmax = tl.max(tl.abs(y), axis=1)
|
||||
scale_pack = pack_tile * PACKS_PER_CTA + pack_offsets
|
||||
scale_ptrs = (
|
||||
output_scale_ptr
|
||||
+ scale_pack[None, :] * output_scale_stride_k
|
||||
+ rows[:, None]
|
||||
)
|
||||
tl.store(
|
||||
scale_ptrs,
|
||||
packed_scale,
|
||||
mask=row_mask[:, None] & (scale_pack[None, :] < PACKS_PER_ROW),
|
||||
)
|
||||
|
||||
scale_raw = tl.maximum(absmax / fp8_max, 1e-10)
|
||||
exponent = tl.ceil(tl.log2(scale_raw))
|
||||
scale = tl.math.exp2(exponent)
|
||||
|
||||
y_q = tl.clamp(y / scale[:, None], fp8_min, fp8_max)
|
||||
|
||||
out_q_ptrs = output_q_ptr + base_out_offset + n_offset + offs_n[None, :]
|
||||
tl.store(
|
||||
out_q_ptrs,
|
||||
y_q.to(output_q_ptr.dtype.element_ty),
|
||||
mask=row_mask[:, None],
|
||||
)
|
||||
|
||||
exponent_biased = tl.clamp(exponent + 127.0, 0.0, 255.0).to(tl.int32)
|
||||
packed_scale = packed_scale | (exponent_biased << (pack_idx * 8))
|
||||
|
||||
scale_ptrs = output_scale_ptr + pid_pack * output_scale_stride_k + m_offset + offs_m
|
||||
tl.store(scale_ptrs, packed_scale, mask=row_mask)
|
||||
row_start += row_step
|
||||
|
||||
|
||||
def silu_mul_quant_fp8_packed_triton(
|
||||
@@ -235,37 +264,48 @@ def silu_mul_quant_fp8_packed_triton(
|
||||
group_size: int = 128,
|
||||
output_q: torch.Tensor | None = None,
|
||||
clamp_limit: float | None = None,
|
||||
alpha: float = 1.0,
|
||||
beta: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert input.dim() == 2
|
||||
assert input.is_contiguous()
|
||||
|
||||
M, N = input.shape
|
||||
N_2 = N // 2
|
||||
hidden_size = N // 2
|
||||
|
||||
assert N_2 % group_size == 0
|
||||
assert hidden_size % group_size == 0
|
||||
|
||||
fp8_dtype = torch.float8_e4m3fn
|
||||
finfo = torch.finfo(fp8_dtype)
|
||||
fp8_min, fp8_max = finfo.min, finfo.max
|
||||
|
||||
num_groups_per_row = N_2 // group_size
|
||||
num_packed_groups = (num_groups_per_row + 3) // 4
|
||||
tma_aligned_M = ((M + 3) // 4) * 4
|
||||
groups_per_row = hidden_size // group_size
|
||||
groups_per_pack = 4 # pack 4 UE8M0 scales to a single INT32
|
||||
packs_per_row = triton.cdiv(groups_per_row, groups_per_pack)
|
||||
|
||||
if output_q is None:
|
||||
output_q = torch.empty((M, N_2), dtype=fp8_dtype, device=input.device)
|
||||
output_q = torch.empty((M, hidden_size), dtype=fp8_dtype, device=input.device)
|
||||
|
||||
aligned_m = triton.cdiv(M, 4) * 4
|
||||
output_scale_packed = torch.empty(
|
||||
(num_packed_groups, tma_aligned_M),
|
||||
(packs_per_row, aligned_m),
|
||||
dtype=torch.int32,
|
||||
device=input.device,
|
||||
).T[:M, :]
|
||||
|
||||
BLOCK_M = 8
|
||||
grid = (num_packed_groups, (M + BLOCK_M - 1) // BLOCK_M)
|
||||
|
||||
num_warps = max(4, group_size // 32)
|
||||
# Tuned for group_size=32 (MXFP8) and group_size=128 (DeepSeek-V4)
|
||||
num_warps = 4
|
||||
num_stages = 2
|
||||
if group_size < 128:
|
||||
BM = 1
|
||||
packs_per_cta = 8
|
||||
else:
|
||||
BM = 1 if M < 512 else 4
|
||||
packs_per_cta = 2 if M < 512 else 1
|
||||
|
||||
grid_n = triton.cdiv(packs_per_row, packs_per_cta)
|
||||
grid_m = min(triton.cdiv(M, BM), 4096)
|
||||
grid = (grid_n, grid_m)
|
||||
|
||||
has_clamp = clamp_limit is not None
|
||||
_silu_mul_quant_fp8_packed_kernel[grid](
|
||||
@@ -277,12 +317,16 @@ def silu_mul_quant_fp8_packed_triton(
|
||||
output_q.stride(0),
|
||||
output_scale_packed.stride(1),
|
||||
clamp_limit if has_clamp else 0.0,
|
||||
alpha,
|
||||
beta,
|
||||
N=N,
|
||||
NUM_GROUPS=num_groups_per_row,
|
||||
GROUPS_PER_ROW=groups_per_row,
|
||||
PACKS_PER_ROW=packs_per_row,
|
||||
fp8_min=fp8_min,
|
||||
fp8_max=fp8_max,
|
||||
GROUP_SIZE=group_size,
|
||||
BLOCK_M=BLOCK_M,
|
||||
PACKS_PER_CTA=packs_per_cta,
|
||||
BLOCK_M=BM,
|
||||
HAS_CLAMP=has_clamp,
|
||||
num_warps=num_warps,
|
||||
num_stages=num_stages,
|
||||
@@ -303,6 +347,8 @@ def _silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
# Information for float8
|
||||
eps,
|
||||
clamp_limit,
|
||||
alpha,
|
||||
beta,
|
||||
fp8_min: tl.constexpr,
|
||||
fp8_max: tl.constexpr,
|
||||
use_ue8m0: tl.constexpr,
|
||||
@@ -348,10 +394,14 @@ def _silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
mul_in = tl.clamp(mul_in.to(tl.float32), -clamp_limit, clamp_limit).to(
|
||||
y_ptr.dtype.element_ty
|
||||
)
|
||||
# Unified gated activation: silu == swigluoai with alpha=1, beta=0.
|
||||
# glu = gate * sigmoid(alpha * gate); y = (up + beta) * glu
|
||||
# Keep glu/up at input precision (narrow before the mul) so the alpha=1,
|
||||
# beta=0 defaults match the C++ silu_and_mul path bit-for-bit.
|
||||
act_in = act_in.to(tl.float32)
|
||||
one_f32 = tl.cast(1, tl.float32)
|
||||
silu_out = (act_in / (one_f32 + tl.exp(-act_in))).to(y_ptr.dtype.element_ty)
|
||||
y = (silu_out * mul_in).to(tl.float32)
|
||||
glu = (act_in / (1.0 + tl.exp(-act_in * alpha))).to(y_ptr.dtype.element_ty)
|
||||
up = (mul_in.to(tl.float32) + beta).to(y_ptr.dtype.element_ty)
|
||||
y = (glu * up).to(tl.float32)
|
||||
|
||||
# quant
|
||||
_absmax = tl.maximum(tl.max(tl.abs(y), axis=1), eps)
|
||||
@@ -379,11 +429,15 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
use_ue8m0: bool | None = None,
|
||||
eps: float = 1e-10,
|
||||
clamp_limit: float | None = None,
|
||||
group_size: int = 128,
|
||||
alpha: float = 1.0,
|
||||
beta: float = 0.0,
|
||||
):
|
||||
"""
|
||||
silu+mul + block-fp8 quant with group size 128.
|
||||
Gated activation + block-fp8 quant. ``alpha``/``beta`` select the gate
|
||||
(silu: alpha=1, beta=0; swigluoai: alpha, beta from config).
|
||||
"""
|
||||
GROUP_SIZE = 128
|
||||
GROUP_SIZE = group_size
|
||||
assert input.ndim == 2
|
||||
if output is not None:
|
||||
assert output.ndim == 2
|
||||
@@ -431,6 +485,8 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
output_scales.stride(-1),
|
||||
eps,
|
||||
clamp_limit if has_clamp else 0.0,
|
||||
alpha,
|
||||
beta,
|
||||
fp8_min,
|
||||
fp8_max,
|
||||
use_ue8m0,
|
||||
@@ -1015,9 +1071,10 @@ def deepgemm_post_process_fp8_weight_block(
|
||||
f"to be torch.float8_e4m3fn, got {wq.dtype} instead."
|
||||
)
|
||||
|
||||
if ws.dtype == torch.float8_e8m0fnu:
|
||||
# Scales already in E8M0 from checkpoint — upcast to fp32
|
||||
# and skip requantization (weights already have power-of-two scales).
|
||||
if ws.dtype in (torch.float8_e8m0fnu, torch.uint8):
|
||||
# Scales already in E8M0 from checkpoint (float8_e8m0fnu, or raw E8M0
|
||||
# bits as uint8 for MXFP8) — upcast to fp32 and skip requantization
|
||||
# (weights already have power-of-two scales).
|
||||
ws = _upcast_e8m0_to_fp32(ws)
|
||||
else:
|
||||
assert ws.dtype == torch.float32, (
|
||||
@@ -1057,7 +1114,8 @@ def deepgemm_post_process_fp8_weight_block(
|
||||
ws = ws.unsqueeze(0)
|
||||
|
||||
# From https://github.com/deepseek-ai/DeepGEMM/blob/c9f8b34dcdacc20aa746b786f983492c51072870/csrc/utils/layout.hpp#L46
|
||||
recipe = (1, 128, 128)
|
||||
# (1, block_n, block_k): (1, 128, 128) for FP8 block, (1, 1, 32) for MXFP8.
|
||||
recipe = (1, quant_block_shape[0], quant_block_shape[1])
|
||||
|
||||
# Ref : https://github.com/deepseek-ai/DeepGEMM/blob/c9f8b34dcdacc20aa746b786f983492c51072870/csrc/apis/gemm.hpp
|
||||
# DeepGemm uses the `transform_sf_into_required_layout` function to
|
||||
|
||||
@@ -84,6 +84,92 @@ def _mxfp8_e4m3_quantize_torch(
|
||||
return x_fp8, scales_uint8
|
||||
|
||||
|
||||
def _mxfp8_quant_triton_kernel():
|
||||
"""Lazily-built Triton kernel: per-32-block E8M0 scale + FP8-E4M3 quant.
|
||||
|
||||
Fuses what ``_mxfp8_e4m3_quantize_torch`` does in several elementwise passes
|
||||
into one launch. Each program handles ``[BLOCK_M, 32]`` (one MX block).
|
||||
"""
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
@triton.jit
|
||||
def _kernel(
|
||||
x_ptr,
|
||||
xq_ptr,
|
||||
s_ptr,
|
||||
M,
|
||||
K,
|
||||
sxm,
|
||||
sxk,
|
||||
sqm,
|
||||
sqk,
|
||||
ssm,
|
||||
ssk,
|
||||
BLOCK_M: tl.constexpr,
|
||||
):
|
||||
pid_m = tl.program_id(0)
|
||||
pid_b = tl.program_id(1) # which 32-element block along K
|
||||
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
offs_k = pid_b * 32 + tl.arange(0, 32)
|
||||
m_mask = offs_m < M
|
||||
x = tl.load(
|
||||
x_ptr + offs_m[:, None] * sxm + offs_k[None, :] * sxk,
|
||||
mask=m_mask[:, None],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
amax = tl.maximum(tl.max(tl.abs(x), axis=1), 1e-30) # [BLOCK_M]
|
||||
sb = tl.floor(tl.log2(amax)) + 127.0
|
||||
sb = tl.minimum(tl.maximum(sb, 0.0), 254.0)
|
||||
descale = tl.exp2(sb - 127.0)
|
||||
xq = (x / descale[:, None]).to(xq_ptr.dtype.element_ty)
|
||||
tl.store(
|
||||
xq_ptr + offs_m[:, None] * sqm + offs_k[None, :] * sqk,
|
||||
xq,
|
||||
mask=m_mask[:, None],
|
||||
)
|
||||
tl.store(s_ptr + offs_m * ssm + pid_b * ssk, sb.to(tl.uint8), mask=m_mask)
|
||||
|
||||
return _kernel
|
||||
|
||||
|
||||
_MXFP8_QUANT_KERNEL = None
|
||||
|
||||
|
||||
def _mxfp8_e4m3_quantize_triton(
|
||||
x: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Fused 2D MXFP8 quant (non-swizzled, row-major [M, K//32] scales)."""
|
||||
from vllm.triton_utils import triton
|
||||
|
||||
global _MXFP8_QUANT_KERNEL
|
||||
if _MXFP8_QUANT_KERNEL is None:
|
||||
_MXFP8_QUANT_KERNEL = _mxfp8_quant_triton_kernel()
|
||||
|
||||
M, K = x.shape
|
||||
x = x.contiguous()
|
||||
xq = torch.empty((M, K), dtype=MXFP8_VALUE_DTYPE, device=x.device)
|
||||
scales = torch.empty(
|
||||
(M, K // MXFP8_BLOCK_SIZE), dtype=MXFP8_SCALE_DTYPE, device=x.device
|
||||
)
|
||||
BLOCK_M = 64
|
||||
grid = (triton.cdiv(M, BLOCK_M), K // MXFP8_BLOCK_SIZE)
|
||||
_MXFP8_QUANT_KERNEL[grid](
|
||||
x,
|
||||
xq,
|
||||
scales,
|
||||
M,
|
||||
K,
|
||||
x.stride(0),
|
||||
x.stride(1),
|
||||
xq.stride(0),
|
||||
xq.stride(1),
|
||||
scales.stride(0),
|
||||
scales.stride(1),
|
||||
BLOCK_M=BLOCK_M,
|
||||
)
|
||||
return xq, scales
|
||||
|
||||
|
||||
def _mxfp8_e4m3_quantize_impl(
|
||||
x: torch.Tensor,
|
||||
is_sf_swizzled_layout: bool = False,
|
||||
@@ -103,6 +189,17 @@ def _mxfp8_e4m3_quantize_impl(
|
||||
x_scales = x_scales.view(x.size(0), -1)
|
||||
return x_q, x_scales
|
||||
|
||||
# ROCm: a single fused Triton kernel beats the multi-pass torch path for the
|
||||
# common 2D, non-swizzled activation-quant case (used by the native MX
|
||||
# linear/MoE). Falls back to torch otherwise (3D weights, swizzled layout).
|
||||
if (
|
||||
current_platform.is_rocm()
|
||||
and not is_sf_swizzled_layout
|
||||
and x.ndim == 2
|
||||
and x.shape[-1] % MXFP8_BLOCK_SIZE == 0
|
||||
):
|
||||
return _mxfp8_e4m3_quantize_triton(x)
|
||||
|
||||
return _mxfp8_e4m3_quantize_torch(x, is_sf_swizzled_layout)
|
||||
|
||||
|
||||
|
||||
@@ -164,6 +164,10 @@ _TEXT_GENERATION_MODELS = {
|
||||
"MiniMaxText01ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
|
||||
"MiniMaxM1ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
|
||||
"MiniMaxM2ForCausalLM": ("minimax_m2", "MiniMaxM2ForCausalLM"),
|
||||
"MiniMaxM3SparseForCausalLM": (
|
||||
"vllm.models.minimax_m3",
|
||||
"MiniMaxM3SparseForCausalLM",
|
||||
),
|
||||
"Ministral3ForCausalLM": ("mistral", "MistralForCausalLM"),
|
||||
"MistralForCausalLM": ("mistral", "MistralForCausalLM"),
|
||||
"MistralLarge3ForCausalLM": ("mistral_large_3", "MistralLarge3ForCausalLM"),
|
||||
@@ -483,6 +487,10 @@ _MULTIMODAL_MODELS = {
|
||||
"MantisForConditionalGeneration": ("llava", "MantisForConditionalGeneration"),
|
||||
"MiDashengLMModel": ("midashenglm", "MiDashengLMModel"),
|
||||
"MiMoV2OmniForCausalLM": ("mimo_v2_omni", "MiMoV2OmniForCausalLM"),
|
||||
"MiniMaxM3SparseForConditionalGeneration": (
|
||||
"vllm.models.minimax_m3",
|
||||
"MiniMaxM3SparseForConditionalGeneration",
|
||||
),
|
||||
"MiniMaxVL01ForConditionalGeneration": (
|
||||
"minimax_vl_01",
|
||||
"MiniMaxVL01ForConditionalGeneration",
|
||||
@@ -620,6 +628,7 @@ _SPECULATIVE_DECODING_MODELS = {
|
||||
"EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"),
|
||||
"DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
|
||||
"DeepSeekV4MTPModel": ("vllm.models.deepseek_v4", "DeepSeekV4MTP"),
|
||||
"MiniMaxM3MTP": ("vllm.models.minimax_m3", "MiniMaxM3MTP"),
|
||||
"Gemma4MTPModel": ("gemma4_mtp", "Gemma4MTP"),
|
||||
"ErnieMTPModel": ("ernie_mtp", "ErnieMTP"),
|
||||
"ExaoneMoeMTP": ("exaone_moe_mtp", "ExaoneMoeMTP"),
|
||||
|
||||
@@ -53,6 +53,10 @@ def _resolve_flashinfer_autotune_file(runner: "GPUModelRunner") -> Path:
|
||||
|
||||
|
||||
def kernel_warmup(worker: "Worker"):
|
||||
from vllm.model_executor.warmup.minimax_m3_msa_warmup import (
|
||||
minimax_m3_msa_warmup,
|
||||
)
|
||||
|
||||
# Deep GEMM warmup
|
||||
do_deep_gemm_warmup = (
|
||||
envs.VLLM_USE_DEEP_GEMM
|
||||
@@ -64,6 +68,8 @@ def kernel_warmup(worker: "Worker"):
|
||||
max_tokens = worker.scheduler_config.max_num_batched_tokens
|
||||
deep_gemm_warmup(model, max_tokens)
|
||||
|
||||
minimax_m3_msa_warmup(worker)
|
||||
|
||||
enable_flashinfer_autotune = (
|
||||
worker.vllm_config.kernel_config.enable_flashinfer_autotune
|
||||
)
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.models.minimax_m3.nvidia.model import MiniMaxM3SparseAttention
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.tracing import instrument
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@instrument(span_name="MiniMax M3 MSA warmup")
|
||||
def minimax_m3_msa_warmup(worker: "Worker") -> None:
|
||||
sparse_module = next(
|
||||
(
|
||||
module
|
||||
for module in worker.get_model().modules()
|
||||
if isinstance(module, MiniMaxM3SparseAttention)
|
||||
),
|
||||
None,
|
||||
)
|
||||
if sparse_module is None:
|
||||
return
|
||||
if not (
|
||||
current_platform.is_cuda() and current_platform.is_device_capability_family(100)
|
||||
):
|
||||
return
|
||||
|
||||
logger.info("Warming up MiniMax M3 MSA kernels.")
|
||||
|
||||
# Cover sparse prefill through the normal model path.
|
||||
worker.model_runner._dummy_run(
|
||||
num_tokens=16,
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
force_attention=True,
|
||||
create_mixed_batch=True,
|
||||
)
|
||||
Reference in New Issue
Block a user