[Model] Add MiniMax M3 support (#45381)

Signed-off-by: youkaichao <youkaichao@gmail.com>
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Signed-off-by: functionstackx <47992694+functionstackx@users.noreply.github.com>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: functionstackx <47992694+functionstackx@users.noreply.github.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
youkaichao
2026-06-16 01:01:25 +08:00
committed by GitHub
co-authored by OpenAI Codex Isotr0py Thien Tran Bugen Zhao Jee Jee Li Roger Wang functionstackx Yongye Zhu Jee Jee Li
parent a3195fab7b
commit 0a1c5034f5
108 changed files with 14734 additions and 311 deletions
@@ -163,17 +163,18 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
"deepseek_v4_fp8": DeepseekV4FP8Config,
"humming": HummingConfig,
"online": OnlineQuantizationConfig,
# MiniMax-style checkpoints tag `quant_method: "mxfp8"`; load with the
# ModelOpt MXFP8 config (same format). The "mxfp8" online shorthand
# below only applies to the `--quantization mxfp8` CLI path.
"mxfp8": ModelOptMxFp8Config,
}
# Register online shorthands as quantization methods so the user can
# specify "LLM(..., quantization='fp8_per_tensor')" as shorthand for
# creating a more complicated online quant config object.
# Register online shorthands (e.g. "fp8_per_tensor") as quant methods.
# setdefault so a shorthand that is also a checkpoint method (e.g. "mxfp8")
# keeps its checkpoint config; the shorthand still works via the
# `--quantization` CLI path in `resolve_quantization_config`.
for shorthand in _ONLINE_SHORTHANDS:
assert shorthand not in method_to_config, (
f"Online quant shorthand {shorthand!r} conflicts with an "
f"existing quantization method"
)
method_to_config[shorthand] = OnlineQuantizationConfig
method_to_config.setdefault(shorthand, OnlineQuantizationConfig)
# Update the `method_to_config` with customized quantization methods.
method_to_config.update(_CUSTOMIZED_METHOD_TO_QUANT_CONFIG)
@@ -2,11 +2,12 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from fnmatch import fnmatch
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, cast
import torch
from torch.nn.parameter import Parameter
import vllm.envs as envs
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.config import get_current_vllm_config
from vllm.logger import init_logger
@@ -27,6 +28,7 @@ from vllm.model_executor.layers.fused_moe import (
SharedExperts,
)
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
Fp8MoeBackend,
convert_to_fp8_moe_kernel_format,
make_fp8_moe_kernel,
make_fp8_moe_quant_config,
@@ -1720,6 +1722,22 @@ class ModelOptMxFp8Config(ModelOptQuantConfigBase):
return "modelopt_mxfp8"
return None
@classmethod
def from_config(cls, config: dict[str, Any]) -> "ModelOptMxFp8Config":
# MiniMax-style checkpoints tag `quant_method: "mxfp8"` + `ignored_layers`
# (same on-disk format as ModelOpt MXFP8); normalize to the ModelOpt
# schema and reuse the shared parser.
if "quantization" not in config and not config.get("quant_algo"):
config = {
"quant_method": "modelopt",
"quantization": {
"quant_algo": "MXFP8",
"kv_cache_quant_algo": config.get("kv_cache_quant_algo"),
"exclude_modules": config.get("ignored_layers", []) or [],
},
}
return cast("ModelOptMxFp8Config", super().from_config(config))
@classmethod
def _from_config(
cls,
@@ -1823,6 +1841,12 @@ class ModelOptMxFp8LinearMethod(LinearMethodBase):
layer.register_parameter("weight_scale", weight_scale)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# Idempotent: the emulation kernel may dequant the weight to BF16 at load
# time (>=2-byte). If already converted, there is nothing left to do --
# avoid re-running the MXFP8-only validation/conversion below.
if layer.weight.element_size() >= 2:
return
# Validate weight tensor
if layer.weight.ndim != 2:
raise ValueError(
@@ -2065,6 +2089,44 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
torch.stack(w2_scale_shuffled).contiguous(),
)
def _dequant_mxfp8_weights_to_bf16(self, layer: RoutedExperts) -> None:
"""One-time MXFP8->BF16 weight dequant for the emulation path.
On devices without a native MXFP8 MoE kernel (e.g. gfx942 / MI300),
``Mxfp8EmulationTritonExperts`` otherwise dequantizes every expert
weight to BF16 on *every* forward step -- the dominant cost (conc1
~1.3 tok/s). Doing the dequant once here and replacing the MXFP8
parameters with BF16 makes the MoE run exactly like a plain BF16
checkpoint (full precision, no per-step dequant); SwiGLU-OAI is still
applied by the experts' ``activation()`` override. The MXFP8 weights
are freed by ``replace_parameter`` (BF16 is 2x their size; the small
E8M0 scale tensors are left in place, unused).
"""
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
dequant_mxfp8_to_bf16,
)
target_dtype = getattr(layer, "orig_dtype", torch.bfloat16)
num_experts = layer.w13_weight.shape[0]
# dequant_mxfp8_to_bf16 handles arbitrary leading dims (*x.shape[:-1]),
# so dequant the whole [E, N, K] weight in one vectorized call.
w13_bf16 = dequant_mxfp8_to_bf16(layer.w13_weight, layer.w13_weight_scale).to(
target_dtype
)
w2_bf16 = dequant_mxfp8_to_bf16(layer.w2_weight, layer.w2_weight_scale).to(
target_dtype
)
replace_parameter(layer, "w13_weight", w13_bf16)
replace_parameter(layer, "w2_weight", w2_bf16)
logger.info_once(
"MXFP8->BF16 load-time dequant complete (%d experts/layer); MoE "
"now runs in BF16 with no per-step dequant.",
num_experts,
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
# TODO(bnell): why is this required only for mxfp8?
if getattr(layer, "_already_called_process_weights_after_loading", False):
@@ -2102,6 +2164,17 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
routing_tables=layer._expert_routing_tables(),
)
# No native MXFP8 MoE kernel on this device (e.g. gfx942): the emulation
# experts would dequant MXFP8->BF16 every forward step. Convert the
# weights to BF16 once, here, so the MoE runs like a BF16 checkpoint.
# Opt out (VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD=0) to keep the 1-byte
# MXFP8 weights and dequant per-step (~half the memory, much slower).
if (
self.mxfp8_backend == Fp8MoeBackend.EMULATION
and envs.VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD
):
self._dequant_mxfp8_weights_to_bf16(layer)
def maybe_make_prepare_finalize(
self,
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
@@ -2131,6 +2204,9 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
a1_scale=None,
a2_scale=None,
block_shape=self.weight_block_size,
swiglu_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
)
def apply_monolithic(
@@ -159,75 +159,104 @@ def _silu_mul_quant_fp8_packed_kernel(
output_q_stride_m,
output_scale_stride_k,
clamp_limit,
alpha,
beta,
N: tl.constexpr,
NUM_GROUPS: tl.constexpr,
GROUPS_PER_ROW: tl.constexpr,
PACKS_PER_ROW: tl.constexpr,
fp8_min: tl.constexpr,
fp8_max: tl.constexpr,
GROUP_SIZE: tl.constexpr,
PACKS_PER_CTA: tl.constexpr,
BLOCK_M: tl.constexpr,
HAS_CLAMP: tl.constexpr,
):
N_2: tl.constexpr = N // 2
GROUPS_PER_PACK: tl.constexpr = 4
hidden_size: tl.constexpr = N // 2
pid_pack = tl.program_id(0)
pid_m = tl.program_id(1)
m_offset = pid_m.to(tl.int64) * BLOCK_M
pack_tile = tl.program_id(0)
row_start = tl.program_id(1).to(tl.int64) * BLOCK_M
row_step = tl.num_programs(1).to(tl.int64) * BLOCK_M
if m_offset >= M:
return
groups_per_cta: tl.constexpr = PACKS_PER_CTA * GROUPS_PER_PACK
elems_per_cta: tl.constexpr = groups_per_cta * GROUP_SIZE
col_start = pack_tile * elems_per_cta
col_offsets = tl.arange(0, elems_per_cta)
row_offsets = tl.arange(0, BLOCK_M)
pack_offsets = tl.arange(0, PACKS_PER_CTA)
offs_m = tl.arange(0, BLOCK_M)
offs_n = tl.arange(0, GROUP_SIZE)
row_mask = (m_offset + offs_m) < M
col_mask = (col_start + col_offsets) < (GROUPS_PER_ROW * GROUP_SIZE)
base_row_offset = (m_offset + offs_m[:, None]) * input_stride_m
base_out_offset = (m_offset + offs_m[:, None]) * output_q_stride_m
# persistent with grid_m-stride loop
while row_start < M:
rows = row_start + row_offsets
row_mask = rows < M
input_row_start = rows[:, None] * input_stride_m
output_row_start = rows[:, None] * output_q_stride_m
packed_scale = tl.zeros((BLOCK_M,), dtype=tl.int32)
gate_flat = tl.load(
input_ptr + input_row_start + col_start + col_offsets[None, :],
mask=row_mask[:, None] & col_mask[None, :],
other=0.0,
)
up_flat = tl.load(
input_ptr
+ input_row_start
+ hidden_size
+ col_start
+ col_offsets[None, :],
mask=row_mask[:, None] & col_mask[None, :],
other=0.0,
)
for pack_idx in tl.static_range(4):
group_id = pid_pack * 4 + pack_idx
gate = tl.reshape(gate_flat, (BLOCK_M, groups_per_cta, GROUP_SIZE)).to(
tl.float32
)
up = tl.reshape(up_flat, (BLOCK_M, groups_per_cta, GROUP_SIZE)).to(tl.float32)
if group_id < NUM_GROUPS:
n_offset = group_id * GROUP_SIZE
if HAS_CLAMP:
gate = tl.minimum(gate, clamp_limit)
up = tl.clamp(up, -clamp_limit, clamp_limit)
act_ptrs = input_ptr + base_row_offset + n_offset + offs_n[None, :]
act_in = tl.load(act_ptrs, mask=row_mask[:, None], other=0.0)
# Unified gated activation: silu == swigluoai with alpha=1, beta=0.
# glu = gate * sigmoid(alpha * gate); y = (up + beta) * glu
glu = gate / (1.0 + tl.exp(-gate * alpha))
y = (up + beta) * glu
# Round through bf16 to match unfused precision path
y = y.to(tl.bfloat16).to(tl.float32)
mul_ptrs = act_ptrs + N_2
mul_in = tl.load(mul_ptrs, mask=row_mask[:, None], other=0.0)
absmax = tl.max(tl.abs(y), axis=2)
scale_raw = tl.maximum(absmax / fp8_max, 1e-10)
exponent = tl.ceil(tl.log2(scale_raw))
scale = tl.math.exp2(exponent)
act_f32 = act_in.to(tl.float32)
mul_f32 = mul_in.to(tl.float32)
y_q = tl.clamp(y / scale[:, :, None], fp8_min, fp8_max)
if HAS_CLAMP:
act_f32 = tl.minimum(act_f32, clamp_limit)
mul_f32 = tl.clamp(mul_f32, -clamp_limit, clamp_limit)
y_q_flat = tl.reshape(y_q, (BLOCK_M, elems_per_cta))
tl.store(
output_q_ptr + output_row_start + col_start + col_offsets[None, :],
y_q_flat.to(output_q_ptr.dtype.element_ty),
mask=row_mask[:, None] & col_mask[None, :],
)
y = (act_f32 / (1.0 + tl.exp(-act_f32))) * mul_f32
# Round through bf16 to match unfused precision path
y = y.to(tl.bfloat16).to(tl.float32)
scale_byte = tl.clamp(exponent + 127.0, 0.0, 255.0).to(tl.int32)
scale_bytes = tl.reshape(scale_byte, (BLOCK_M, PACKS_PER_CTA, GROUPS_PER_PACK))
shifts = tl.arange(0, GROUPS_PER_PACK) * 8
packed_scale = tl.sum(scale_bytes << shifts[None, None, :], axis=2)
absmax = tl.max(tl.abs(y), axis=1)
scale_pack = pack_tile * PACKS_PER_CTA + pack_offsets
scale_ptrs = (
output_scale_ptr
+ scale_pack[None, :] * output_scale_stride_k
+ rows[:, None]
)
tl.store(
scale_ptrs,
packed_scale,
mask=row_mask[:, None] & (scale_pack[None, :] < PACKS_PER_ROW),
)
scale_raw = tl.maximum(absmax / fp8_max, 1e-10)
exponent = tl.ceil(tl.log2(scale_raw))
scale = tl.math.exp2(exponent)
y_q = tl.clamp(y / scale[:, None], fp8_min, fp8_max)
out_q_ptrs = output_q_ptr + base_out_offset + n_offset + offs_n[None, :]
tl.store(
out_q_ptrs,
y_q.to(output_q_ptr.dtype.element_ty),
mask=row_mask[:, None],
)
exponent_biased = tl.clamp(exponent + 127.0, 0.0, 255.0).to(tl.int32)
packed_scale = packed_scale | (exponent_biased << (pack_idx * 8))
scale_ptrs = output_scale_ptr + pid_pack * output_scale_stride_k + m_offset + offs_m
tl.store(scale_ptrs, packed_scale, mask=row_mask)
row_start += row_step
def silu_mul_quant_fp8_packed_triton(
@@ -235,37 +264,48 @@ def silu_mul_quant_fp8_packed_triton(
group_size: int = 128,
output_q: torch.Tensor | None = None,
clamp_limit: float | None = None,
alpha: float = 1.0,
beta: float = 0.0,
) -> tuple[torch.Tensor, torch.Tensor]:
assert input.dim() == 2
assert input.is_contiguous()
M, N = input.shape
N_2 = N // 2
hidden_size = N // 2
assert N_2 % group_size == 0
assert hidden_size % group_size == 0
fp8_dtype = torch.float8_e4m3fn
finfo = torch.finfo(fp8_dtype)
fp8_min, fp8_max = finfo.min, finfo.max
num_groups_per_row = N_2 // group_size
num_packed_groups = (num_groups_per_row + 3) // 4
tma_aligned_M = ((M + 3) // 4) * 4
groups_per_row = hidden_size // group_size
groups_per_pack = 4 # pack 4 UE8M0 scales to a single INT32
packs_per_row = triton.cdiv(groups_per_row, groups_per_pack)
if output_q is None:
output_q = torch.empty((M, N_2), dtype=fp8_dtype, device=input.device)
output_q = torch.empty((M, hidden_size), dtype=fp8_dtype, device=input.device)
aligned_m = triton.cdiv(M, 4) * 4
output_scale_packed = torch.empty(
(num_packed_groups, tma_aligned_M),
(packs_per_row, aligned_m),
dtype=torch.int32,
device=input.device,
).T[:M, :]
BLOCK_M = 8
grid = (num_packed_groups, (M + BLOCK_M - 1) // BLOCK_M)
num_warps = max(4, group_size // 32)
# Tuned for group_size=32 (MXFP8) and group_size=128 (DeepSeek-V4)
num_warps = 4
num_stages = 2
if group_size < 128:
BM = 1
packs_per_cta = 8
else:
BM = 1 if M < 512 else 4
packs_per_cta = 2 if M < 512 else 1
grid_n = triton.cdiv(packs_per_row, packs_per_cta)
grid_m = min(triton.cdiv(M, BM), 4096)
grid = (grid_n, grid_m)
has_clamp = clamp_limit is not None
_silu_mul_quant_fp8_packed_kernel[grid](
@@ -277,12 +317,16 @@ def silu_mul_quant_fp8_packed_triton(
output_q.stride(0),
output_scale_packed.stride(1),
clamp_limit if has_clamp else 0.0,
alpha,
beta,
N=N,
NUM_GROUPS=num_groups_per_row,
GROUPS_PER_ROW=groups_per_row,
PACKS_PER_ROW=packs_per_row,
fp8_min=fp8_min,
fp8_max=fp8_max,
GROUP_SIZE=group_size,
BLOCK_M=BLOCK_M,
PACKS_PER_CTA=packs_per_cta,
BLOCK_M=BM,
HAS_CLAMP=has_clamp,
num_warps=num_warps,
num_stages=num_stages,
@@ -303,6 +347,8 @@ def _silu_mul_per_token_group_quant_fp8_colmajor(
# Information for float8
eps,
clamp_limit,
alpha,
beta,
fp8_min: tl.constexpr,
fp8_max: tl.constexpr,
use_ue8m0: tl.constexpr,
@@ -348,10 +394,14 @@ def _silu_mul_per_token_group_quant_fp8_colmajor(
mul_in = tl.clamp(mul_in.to(tl.float32), -clamp_limit, clamp_limit).to(
y_ptr.dtype.element_ty
)
# Unified gated activation: silu == swigluoai with alpha=1, beta=0.
# glu = gate * sigmoid(alpha * gate); y = (up + beta) * glu
# Keep glu/up at input precision (narrow before the mul) so the alpha=1,
# beta=0 defaults match the C++ silu_and_mul path bit-for-bit.
act_in = act_in.to(tl.float32)
one_f32 = tl.cast(1, tl.float32)
silu_out = (act_in / (one_f32 + tl.exp(-act_in))).to(y_ptr.dtype.element_ty)
y = (silu_out * mul_in).to(tl.float32)
glu = (act_in / (1.0 + tl.exp(-act_in * alpha))).to(y_ptr.dtype.element_ty)
up = (mul_in.to(tl.float32) + beta).to(y_ptr.dtype.element_ty)
y = (glu * up).to(tl.float32)
# quant
_absmax = tl.maximum(tl.max(tl.abs(y), axis=1), eps)
@@ -379,11 +429,15 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
use_ue8m0: bool | None = None,
eps: float = 1e-10,
clamp_limit: float | None = None,
group_size: int = 128,
alpha: float = 1.0,
beta: float = 0.0,
):
"""
silu+mul + block-fp8 quant with group size 128.
Gated activation + block-fp8 quant. ``alpha``/``beta`` select the gate
(silu: alpha=1, beta=0; swigluoai: alpha, beta from config).
"""
GROUP_SIZE = 128
GROUP_SIZE = group_size
assert input.ndim == 2
if output is not None:
assert output.ndim == 2
@@ -431,6 +485,8 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
output_scales.stride(-1),
eps,
clamp_limit if has_clamp else 0.0,
alpha,
beta,
fp8_min,
fp8_max,
use_ue8m0,
@@ -1015,9 +1071,10 @@ def deepgemm_post_process_fp8_weight_block(
f"to be torch.float8_e4m3fn, got {wq.dtype} instead."
)
if ws.dtype == torch.float8_e8m0fnu:
# Scales already in E8M0 from checkpoint — upcast to fp32
# and skip requantization (weights already have power-of-two scales).
if ws.dtype in (torch.float8_e8m0fnu, torch.uint8):
# Scales already in E8M0 from checkpoint (float8_e8m0fnu, or raw E8M0
# bits as uint8 for MXFP8) — upcast to fp32 and skip requantization
# (weights already have power-of-two scales).
ws = _upcast_e8m0_to_fp32(ws)
else:
assert ws.dtype == torch.float32, (
@@ -1057,7 +1114,8 @@ def deepgemm_post_process_fp8_weight_block(
ws = ws.unsqueeze(0)
# From https://github.com/deepseek-ai/DeepGEMM/blob/c9f8b34dcdacc20aa746b786f983492c51072870/csrc/utils/layout.hpp#L46
recipe = (1, 128, 128)
# (1, block_n, block_k): (1, 128, 128) for FP8 block, (1, 1, 32) for MXFP8.
recipe = (1, quant_block_shape[0], quant_block_shape[1])
# Ref : https://github.com/deepseek-ai/DeepGEMM/blob/c9f8b34dcdacc20aa746b786f983492c51072870/csrc/apis/gemm.hpp
# DeepGemm uses the `transform_sf_into_required_layout` function to
@@ -84,6 +84,92 @@ def _mxfp8_e4m3_quantize_torch(
return x_fp8, scales_uint8
def _mxfp8_quant_triton_kernel():
"""Lazily-built Triton kernel: per-32-block E8M0 scale + FP8-E4M3 quant.
Fuses what ``_mxfp8_e4m3_quantize_torch`` does in several elementwise passes
into one launch. Each program handles ``[BLOCK_M, 32]`` (one MX block).
"""
from vllm.triton_utils import tl, triton
@triton.jit
def _kernel(
x_ptr,
xq_ptr,
s_ptr,
M,
K,
sxm,
sxk,
sqm,
sqk,
ssm,
ssk,
BLOCK_M: tl.constexpr,
):
pid_m = tl.program_id(0)
pid_b = tl.program_id(1) # which 32-element block along K
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
offs_k = pid_b * 32 + tl.arange(0, 32)
m_mask = offs_m < M
x = tl.load(
x_ptr + offs_m[:, None] * sxm + offs_k[None, :] * sxk,
mask=m_mask[:, None],
other=0.0,
).to(tl.float32)
amax = tl.maximum(tl.max(tl.abs(x), axis=1), 1e-30) # [BLOCK_M]
sb = tl.floor(tl.log2(amax)) + 127.0
sb = tl.minimum(tl.maximum(sb, 0.0), 254.0)
descale = tl.exp2(sb - 127.0)
xq = (x / descale[:, None]).to(xq_ptr.dtype.element_ty)
tl.store(
xq_ptr + offs_m[:, None] * sqm + offs_k[None, :] * sqk,
xq,
mask=m_mask[:, None],
)
tl.store(s_ptr + offs_m * ssm + pid_b * ssk, sb.to(tl.uint8), mask=m_mask)
return _kernel
_MXFP8_QUANT_KERNEL = None
def _mxfp8_e4m3_quantize_triton(
x: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Fused 2D MXFP8 quant (non-swizzled, row-major [M, K//32] scales)."""
from vllm.triton_utils import triton
global _MXFP8_QUANT_KERNEL
if _MXFP8_QUANT_KERNEL is None:
_MXFP8_QUANT_KERNEL = _mxfp8_quant_triton_kernel()
M, K = x.shape
x = x.contiguous()
xq = torch.empty((M, K), dtype=MXFP8_VALUE_DTYPE, device=x.device)
scales = torch.empty(
(M, K // MXFP8_BLOCK_SIZE), dtype=MXFP8_SCALE_DTYPE, device=x.device
)
BLOCK_M = 64
grid = (triton.cdiv(M, BLOCK_M), K // MXFP8_BLOCK_SIZE)
_MXFP8_QUANT_KERNEL[grid](
x,
xq,
scales,
M,
K,
x.stride(0),
x.stride(1),
xq.stride(0),
xq.stride(1),
scales.stride(0),
scales.stride(1),
BLOCK_M=BLOCK_M,
)
return xq, scales
def _mxfp8_e4m3_quantize_impl(
x: torch.Tensor,
is_sf_swizzled_layout: bool = False,
@@ -103,6 +189,17 @@ def _mxfp8_e4m3_quantize_impl(
x_scales = x_scales.view(x.size(0), -1)
return x_q, x_scales
# ROCm: a single fused Triton kernel beats the multi-pass torch path for the
# common 2D, non-swizzled activation-quant case (used by the native MX
# linear/MoE). Falls back to torch otherwise (3D weights, swizzled layout).
if (
current_platform.is_rocm()
and not is_sf_swizzled_layout
and x.ndim == 2
and x.shape[-1] % MXFP8_BLOCK_SIZE == 0
):
return _mxfp8_e4m3_quantize_triton(x)
return _mxfp8_e4m3_quantize_torch(x, is_sf_swizzled_layout)