[Model] Add MiniMax M3 support (#45381)

Signed-off-by: youkaichao <youkaichao@gmail.com>
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Signed-off-by: functionstackx <47992694+functionstackx@users.noreply.github.com>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: functionstackx <47992694+functionstackx@users.noreply.github.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
youkaichao
2026-06-16 01:01:25 +08:00
committed by GitHub
co-authored by OpenAI Codex Isotr0py Thien Tran Bugen Zhao Jee Jee Li Roger Wang functionstackx Yongye Zhu Jee Jee Li
parent a3195fab7b
commit 0a1c5034f5
108 changed files with 14734 additions and 311 deletions
@@ -159,75 +159,104 @@ def _silu_mul_quant_fp8_packed_kernel(
output_q_stride_m,
output_scale_stride_k,
clamp_limit,
alpha,
beta,
N: tl.constexpr,
NUM_GROUPS: tl.constexpr,
GROUPS_PER_ROW: tl.constexpr,
PACKS_PER_ROW: tl.constexpr,
fp8_min: tl.constexpr,
fp8_max: tl.constexpr,
GROUP_SIZE: tl.constexpr,
PACKS_PER_CTA: tl.constexpr,
BLOCK_M: tl.constexpr,
HAS_CLAMP: tl.constexpr,
):
N_2: tl.constexpr = N // 2
GROUPS_PER_PACK: tl.constexpr = 4
hidden_size: tl.constexpr = N // 2
pid_pack = tl.program_id(0)
pid_m = tl.program_id(1)
m_offset = pid_m.to(tl.int64) * BLOCK_M
pack_tile = tl.program_id(0)
row_start = tl.program_id(1).to(tl.int64) * BLOCK_M
row_step = tl.num_programs(1).to(tl.int64) * BLOCK_M
if m_offset >= M:
return
groups_per_cta: tl.constexpr = PACKS_PER_CTA * GROUPS_PER_PACK
elems_per_cta: tl.constexpr = groups_per_cta * GROUP_SIZE
col_start = pack_tile * elems_per_cta
col_offsets = tl.arange(0, elems_per_cta)
row_offsets = tl.arange(0, BLOCK_M)
pack_offsets = tl.arange(0, PACKS_PER_CTA)
offs_m = tl.arange(0, BLOCK_M)
offs_n = tl.arange(0, GROUP_SIZE)
row_mask = (m_offset + offs_m) < M
col_mask = (col_start + col_offsets) < (GROUPS_PER_ROW * GROUP_SIZE)
base_row_offset = (m_offset + offs_m[:, None]) * input_stride_m
base_out_offset = (m_offset + offs_m[:, None]) * output_q_stride_m
# persistent with grid_m-stride loop
while row_start < M:
rows = row_start + row_offsets
row_mask = rows < M
input_row_start = rows[:, None] * input_stride_m
output_row_start = rows[:, None] * output_q_stride_m
packed_scale = tl.zeros((BLOCK_M,), dtype=tl.int32)
gate_flat = tl.load(
input_ptr + input_row_start + col_start + col_offsets[None, :],
mask=row_mask[:, None] & col_mask[None, :],
other=0.0,
)
up_flat = tl.load(
input_ptr
+ input_row_start
+ hidden_size
+ col_start
+ col_offsets[None, :],
mask=row_mask[:, None] & col_mask[None, :],
other=0.0,
)
for pack_idx in tl.static_range(4):
group_id = pid_pack * 4 + pack_idx
gate = tl.reshape(gate_flat, (BLOCK_M, groups_per_cta, GROUP_SIZE)).to(
tl.float32
)
up = tl.reshape(up_flat, (BLOCK_M, groups_per_cta, GROUP_SIZE)).to(tl.float32)
if group_id < NUM_GROUPS:
n_offset = group_id * GROUP_SIZE
if HAS_CLAMP:
gate = tl.minimum(gate, clamp_limit)
up = tl.clamp(up, -clamp_limit, clamp_limit)
act_ptrs = input_ptr + base_row_offset + n_offset + offs_n[None, :]
act_in = tl.load(act_ptrs, mask=row_mask[:, None], other=0.0)
# Unified gated activation: silu == swigluoai with alpha=1, beta=0.
# glu = gate * sigmoid(alpha * gate); y = (up + beta) * glu
glu = gate / (1.0 + tl.exp(-gate * alpha))
y = (up + beta) * glu
# Round through bf16 to match unfused precision path
y = y.to(tl.bfloat16).to(tl.float32)
mul_ptrs = act_ptrs + N_2
mul_in = tl.load(mul_ptrs, mask=row_mask[:, None], other=0.0)
absmax = tl.max(tl.abs(y), axis=2)
scale_raw = tl.maximum(absmax / fp8_max, 1e-10)
exponent = tl.ceil(tl.log2(scale_raw))
scale = tl.math.exp2(exponent)
act_f32 = act_in.to(tl.float32)
mul_f32 = mul_in.to(tl.float32)
y_q = tl.clamp(y / scale[:, :, None], fp8_min, fp8_max)
if HAS_CLAMP:
act_f32 = tl.minimum(act_f32, clamp_limit)
mul_f32 = tl.clamp(mul_f32, -clamp_limit, clamp_limit)
y_q_flat = tl.reshape(y_q, (BLOCK_M, elems_per_cta))
tl.store(
output_q_ptr + output_row_start + col_start + col_offsets[None, :],
y_q_flat.to(output_q_ptr.dtype.element_ty),
mask=row_mask[:, None] & col_mask[None, :],
)
y = (act_f32 / (1.0 + tl.exp(-act_f32))) * mul_f32
# Round through bf16 to match unfused precision path
y = y.to(tl.bfloat16).to(tl.float32)
scale_byte = tl.clamp(exponent + 127.0, 0.0, 255.0).to(tl.int32)
scale_bytes = tl.reshape(scale_byte, (BLOCK_M, PACKS_PER_CTA, GROUPS_PER_PACK))
shifts = tl.arange(0, GROUPS_PER_PACK) * 8
packed_scale = tl.sum(scale_bytes << shifts[None, None, :], axis=2)
absmax = tl.max(tl.abs(y), axis=1)
scale_pack = pack_tile * PACKS_PER_CTA + pack_offsets
scale_ptrs = (
output_scale_ptr
+ scale_pack[None, :] * output_scale_stride_k
+ rows[:, None]
)
tl.store(
scale_ptrs,
packed_scale,
mask=row_mask[:, None] & (scale_pack[None, :] < PACKS_PER_ROW),
)
scale_raw = tl.maximum(absmax / fp8_max, 1e-10)
exponent = tl.ceil(tl.log2(scale_raw))
scale = tl.math.exp2(exponent)
y_q = tl.clamp(y / scale[:, None], fp8_min, fp8_max)
out_q_ptrs = output_q_ptr + base_out_offset + n_offset + offs_n[None, :]
tl.store(
out_q_ptrs,
y_q.to(output_q_ptr.dtype.element_ty),
mask=row_mask[:, None],
)
exponent_biased = tl.clamp(exponent + 127.0, 0.0, 255.0).to(tl.int32)
packed_scale = packed_scale | (exponent_biased << (pack_idx * 8))
scale_ptrs = output_scale_ptr + pid_pack * output_scale_stride_k + m_offset + offs_m
tl.store(scale_ptrs, packed_scale, mask=row_mask)
row_start += row_step
def silu_mul_quant_fp8_packed_triton(
@@ -235,37 +264,48 @@ def silu_mul_quant_fp8_packed_triton(
group_size: int = 128,
output_q: torch.Tensor | None = None,
clamp_limit: float | None = None,
alpha: float = 1.0,
beta: float = 0.0,
) -> tuple[torch.Tensor, torch.Tensor]:
assert input.dim() == 2
assert input.is_contiguous()
M, N = input.shape
N_2 = N // 2
hidden_size = N // 2
assert N_2 % group_size == 0
assert hidden_size % group_size == 0
fp8_dtype = torch.float8_e4m3fn
finfo = torch.finfo(fp8_dtype)
fp8_min, fp8_max = finfo.min, finfo.max
num_groups_per_row = N_2 // group_size
num_packed_groups = (num_groups_per_row + 3) // 4
tma_aligned_M = ((M + 3) // 4) * 4
groups_per_row = hidden_size // group_size
groups_per_pack = 4 # pack 4 UE8M0 scales to a single INT32
packs_per_row = triton.cdiv(groups_per_row, groups_per_pack)
if output_q is None:
output_q = torch.empty((M, N_2), dtype=fp8_dtype, device=input.device)
output_q = torch.empty((M, hidden_size), dtype=fp8_dtype, device=input.device)
aligned_m = triton.cdiv(M, 4) * 4
output_scale_packed = torch.empty(
(num_packed_groups, tma_aligned_M),
(packs_per_row, aligned_m),
dtype=torch.int32,
device=input.device,
).T[:M, :]
BLOCK_M = 8
grid = (num_packed_groups, (M + BLOCK_M - 1) // BLOCK_M)
num_warps = max(4, group_size // 32)
# Tuned for group_size=32 (MXFP8) and group_size=128 (DeepSeek-V4)
num_warps = 4
num_stages = 2
if group_size < 128:
BM = 1
packs_per_cta = 8
else:
BM = 1 if M < 512 else 4
packs_per_cta = 2 if M < 512 else 1
grid_n = triton.cdiv(packs_per_row, packs_per_cta)
grid_m = min(triton.cdiv(M, BM), 4096)
grid = (grid_n, grid_m)
has_clamp = clamp_limit is not None
_silu_mul_quant_fp8_packed_kernel[grid](
@@ -277,12 +317,16 @@ def silu_mul_quant_fp8_packed_triton(
output_q.stride(0),
output_scale_packed.stride(1),
clamp_limit if has_clamp else 0.0,
alpha,
beta,
N=N,
NUM_GROUPS=num_groups_per_row,
GROUPS_PER_ROW=groups_per_row,
PACKS_PER_ROW=packs_per_row,
fp8_min=fp8_min,
fp8_max=fp8_max,
GROUP_SIZE=group_size,
BLOCK_M=BLOCK_M,
PACKS_PER_CTA=packs_per_cta,
BLOCK_M=BM,
HAS_CLAMP=has_clamp,
num_warps=num_warps,
num_stages=num_stages,
@@ -303,6 +347,8 @@ def _silu_mul_per_token_group_quant_fp8_colmajor(
# Information for float8
eps,
clamp_limit,
alpha,
beta,
fp8_min: tl.constexpr,
fp8_max: tl.constexpr,
use_ue8m0: tl.constexpr,
@@ -348,10 +394,14 @@ def _silu_mul_per_token_group_quant_fp8_colmajor(
mul_in = tl.clamp(mul_in.to(tl.float32), -clamp_limit, clamp_limit).to(
y_ptr.dtype.element_ty
)
# Unified gated activation: silu == swigluoai with alpha=1, beta=0.
# glu = gate * sigmoid(alpha * gate); y = (up + beta) * glu
# Keep glu/up at input precision (narrow before the mul) so the alpha=1,
# beta=0 defaults match the C++ silu_and_mul path bit-for-bit.
act_in = act_in.to(tl.float32)
one_f32 = tl.cast(1, tl.float32)
silu_out = (act_in / (one_f32 + tl.exp(-act_in))).to(y_ptr.dtype.element_ty)
y = (silu_out * mul_in).to(tl.float32)
glu = (act_in / (1.0 + tl.exp(-act_in * alpha))).to(y_ptr.dtype.element_ty)
up = (mul_in.to(tl.float32) + beta).to(y_ptr.dtype.element_ty)
y = (glu * up).to(tl.float32)
# quant
_absmax = tl.maximum(tl.max(tl.abs(y), axis=1), eps)
@@ -379,11 +429,15 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
use_ue8m0: bool | None = None,
eps: float = 1e-10,
clamp_limit: float | None = None,
group_size: int = 128,
alpha: float = 1.0,
beta: float = 0.0,
):
"""
silu+mul + block-fp8 quant with group size 128.
Gated activation + block-fp8 quant. ``alpha``/``beta`` select the gate
(silu: alpha=1, beta=0; swigluoai: alpha, beta from config).
"""
GROUP_SIZE = 128
GROUP_SIZE = group_size
assert input.ndim == 2
if output is not None:
assert output.ndim == 2
@@ -431,6 +485,8 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
output_scales.stride(-1),
eps,
clamp_limit if has_clamp else 0.0,
alpha,
beta,
fp8_min,
fp8_max,
use_ue8m0,
@@ -1015,9 +1071,10 @@ def deepgemm_post_process_fp8_weight_block(
f"to be torch.float8_e4m3fn, got {wq.dtype} instead."
)
if ws.dtype == torch.float8_e8m0fnu:
# Scales already in E8M0 from checkpoint — upcast to fp32
# and skip requantization (weights already have power-of-two scales).
if ws.dtype in (torch.float8_e8m0fnu, torch.uint8):
# Scales already in E8M0 from checkpoint (float8_e8m0fnu, or raw E8M0
# bits as uint8 for MXFP8) — upcast to fp32 and skip requantization
# (weights already have power-of-two scales).
ws = _upcast_e8m0_to_fp32(ws)
else:
assert ws.dtype == torch.float32, (
@@ -1057,7 +1114,8 @@ def deepgemm_post_process_fp8_weight_block(
ws = ws.unsqueeze(0)
# From https://github.com/deepseek-ai/DeepGEMM/blob/c9f8b34dcdacc20aa746b786f983492c51072870/csrc/utils/layout.hpp#L46
recipe = (1, 128, 128)
# (1, block_n, block_k): (1, 128, 128) for FP8 block, (1, 1, 32) for MXFP8.
recipe = (1, quant_block_shape[0], quant_block_shape[1])
# Ref : https://github.com/deepseek-ai/DeepGEMM/blob/c9f8b34dcdacc20aa746b786f983492c51072870/csrc/apis/gemm.hpp
# DeepGemm uses the `transform_sf_into_required_layout` function to
@@ -84,6 +84,92 @@ def _mxfp8_e4m3_quantize_torch(
return x_fp8, scales_uint8
def _mxfp8_quant_triton_kernel():
"""Lazily-built Triton kernel: per-32-block E8M0 scale + FP8-E4M3 quant.
Fuses what ``_mxfp8_e4m3_quantize_torch`` does in several elementwise passes
into one launch. Each program handles ``[BLOCK_M, 32]`` (one MX block).
"""
from vllm.triton_utils import tl, triton
@triton.jit
def _kernel(
x_ptr,
xq_ptr,
s_ptr,
M,
K,
sxm,
sxk,
sqm,
sqk,
ssm,
ssk,
BLOCK_M: tl.constexpr,
):
pid_m = tl.program_id(0)
pid_b = tl.program_id(1) # which 32-element block along K
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
offs_k = pid_b * 32 + tl.arange(0, 32)
m_mask = offs_m < M
x = tl.load(
x_ptr + offs_m[:, None] * sxm + offs_k[None, :] * sxk,
mask=m_mask[:, None],
other=0.0,
).to(tl.float32)
amax = tl.maximum(tl.max(tl.abs(x), axis=1), 1e-30) # [BLOCK_M]
sb = tl.floor(tl.log2(amax)) + 127.0
sb = tl.minimum(tl.maximum(sb, 0.0), 254.0)
descale = tl.exp2(sb - 127.0)
xq = (x / descale[:, None]).to(xq_ptr.dtype.element_ty)
tl.store(
xq_ptr + offs_m[:, None] * sqm + offs_k[None, :] * sqk,
xq,
mask=m_mask[:, None],
)
tl.store(s_ptr + offs_m * ssm + pid_b * ssk, sb.to(tl.uint8), mask=m_mask)
return _kernel
_MXFP8_QUANT_KERNEL = None
def _mxfp8_e4m3_quantize_triton(
x: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Fused 2D MXFP8 quant (non-swizzled, row-major [M, K//32] scales)."""
from vllm.triton_utils import triton
global _MXFP8_QUANT_KERNEL
if _MXFP8_QUANT_KERNEL is None:
_MXFP8_QUANT_KERNEL = _mxfp8_quant_triton_kernel()
M, K = x.shape
x = x.contiguous()
xq = torch.empty((M, K), dtype=MXFP8_VALUE_DTYPE, device=x.device)
scales = torch.empty(
(M, K // MXFP8_BLOCK_SIZE), dtype=MXFP8_SCALE_DTYPE, device=x.device
)
BLOCK_M = 64
grid = (triton.cdiv(M, BLOCK_M), K // MXFP8_BLOCK_SIZE)
_MXFP8_QUANT_KERNEL[grid](
x,
xq,
scales,
M,
K,
x.stride(0),
x.stride(1),
xq.stride(0),
xq.stride(1),
scales.stride(0),
scales.stride(1),
BLOCK_M=BLOCK_M,
)
return xq, scales
def _mxfp8_e4m3_quantize_impl(
x: torch.Tensor,
is_sf_swizzled_layout: bool = False,
@@ -103,6 +189,17 @@ def _mxfp8_e4m3_quantize_impl(
x_scales = x_scales.view(x.size(0), -1)
return x_q, x_scales
# ROCm: a single fused Triton kernel beats the multi-pass torch path for the
# common 2D, non-swizzled activation-quant case (used by the native MX
# linear/MoE). Falls back to torch otherwise (3D weights, swizzled layout).
if (
current_platform.is_rocm()
and not is_sf_swizzled_layout
and x.ndim == 2
and x.shape[-1] % MXFP8_BLOCK_SIZE == 0
):
return _mxfp8_e4m3_quantize_triton(x)
return _mxfp8_e4m3_quantize_torch(x, is_sf_swizzled_layout)