[Model] Add MiniMax M3 support (#45381)
Signed-off-by: youkaichao <youkaichao@gmail.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: functionstackx <47992694+functionstackx@users.noreply.github.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: functionstackx <47992694+functionstackx@users.noreply.github.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
co-authored by
OpenAI Codex
Isotr0py
Thien Tran
Bugen Zhao
Jee Jee Li
Roger Wang
functionstackx
Yongye Zhu
Jee Jee Li
parent
a3195fab7b
commit
0a1c5034f5
@@ -163,17 +163,18 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
|
||||
"deepseek_v4_fp8": DeepseekV4FP8Config,
|
||||
"humming": HummingConfig,
|
||||
"online": OnlineQuantizationConfig,
|
||||
# MiniMax-style checkpoints tag `quant_method: "mxfp8"`; load with the
|
||||
# ModelOpt MXFP8 config (same format). The "mxfp8" online shorthand
|
||||
# below only applies to the `--quantization mxfp8` CLI path.
|
||||
"mxfp8": ModelOptMxFp8Config,
|
||||
}
|
||||
|
||||
# Register online shorthands as quantization methods so the user can
|
||||
# specify "LLM(..., quantization='fp8_per_tensor')" as shorthand for
|
||||
# creating a more complicated online quant config object.
|
||||
# Register online shorthands (e.g. "fp8_per_tensor") as quant methods.
|
||||
# setdefault so a shorthand that is also a checkpoint method (e.g. "mxfp8")
|
||||
# keeps its checkpoint config; the shorthand still works via the
|
||||
# `--quantization` CLI path in `resolve_quantization_config`.
|
||||
for shorthand in _ONLINE_SHORTHANDS:
|
||||
assert shorthand not in method_to_config, (
|
||||
f"Online quant shorthand {shorthand!r} conflicts with an "
|
||||
f"existing quantization method"
|
||||
)
|
||||
method_to_config[shorthand] = OnlineQuantizationConfig
|
||||
method_to_config.setdefault(shorthand, OnlineQuantizationConfig)
|
||||
|
||||
# Update the `method_to_config` with customized quantization methods.
|
||||
method_to_config.update(_CUSTOMIZED_METHOD_TO_QUANT_CONFIG)
|
||||
|
||||
@@ -2,11 +2,12 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from fnmatch import fnmatch
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
import vllm.envs as envs
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
@@ -27,6 +28,7 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
SharedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
Fp8MoeBackend,
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
make_fp8_moe_kernel,
|
||||
make_fp8_moe_quant_config,
|
||||
@@ -1720,6 +1722,22 @@ class ModelOptMxFp8Config(ModelOptQuantConfigBase):
|
||||
return "modelopt_mxfp8"
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: dict[str, Any]) -> "ModelOptMxFp8Config":
|
||||
# MiniMax-style checkpoints tag `quant_method: "mxfp8"` + `ignored_layers`
|
||||
# (same on-disk format as ModelOpt MXFP8); normalize to the ModelOpt
|
||||
# schema and reuse the shared parser.
|
||||
if "quantization" not in config and not config.get("quant_algo"):
|
||||
config = {
|
||||
"quant_method": "modelopt",
|
||||
"quantization": {
|
||||
"quant_algo": "MXFP8",
|
||||
"kv_cache_quant_algo": config.get("kv_cache_quant_algo"),
|
||||
"exclude_modules": config.get("ignored_layers", []) or [],
|
||||
},
|
||||
}
|
||||
return cast("ModelOptMxFp8Config", super().from_config(config))
|
||||
|
||||
@classmethod
|
||||
def _from_config(
|
||||
cls,
|
||||
@@ -1823,6 +1841,12 @@ class ModelOptMxFp8LinearMethod(LinearMethodBase):
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
# Idempotent: the emulation kernel may dequant the weight to BF16 at load
|
||||
# time (>=2-byte). If already converted, there is nothing left to do --
|
||||
# avoid re-running the MXFP8-only validation/conversion below.
|
||||
if layer.weight.element_size() >= 2:
|
||||
return
|
||||
|
||||
# Validate weight tensor
|
||||
if layer.weight.ndim != 2:
|
||||
raise ValueError(
|
||||
@@ -2065,6 +2089,44 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
|
||||
torch.stack(w2_scale_shuffled).contiguous(),
|
||||
)
|
||||
|
||||
def _dequant_mxfp8_weights_to_bf16(self, layer: RoutedExperts) -> None:
|
||||
"""One-time MXFP8->BF16 weight dequant for the emulation path.
|
||||
|
||||
On devices without a native MXFP8 MoE kernel (e.g. gfx942 / MI300),
|
||||
``Mxfp8EmulationTritonExperts`` otherwise dequantizes every expert
|
||||
weight to BF16 on *every* forward step -- the dominant cost (conc1
|
||||
~1.3 tok/s). Doing the dequant once here and replacing the MXFP8
|
||||
parameters with BF16 makes the MoE run exactly like a plain BF16
|
||||
checkpoint (full precision, no per-step dequant); SwiGLU-OAI is still
|
||||
applied by the experts' ``activation()`` override. The MXFP8 weights
|
||||
are freed by ``replace_parameter`` (BF16 is 2x their size; the small
|
||||
E8M0 scale tensors are left in place, unused).
|
||||
"""
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
dequant_mxfp8_to_bf16,
|
||||
)
|
||||
|
||||
target_dtype = getattr(layer, "orig_dtype", torch.bfloat16)
|
||||
num_experts = layer.w13_weight.shape[0]
|
||||
|
||||
# dequant_mxfp8_to_bf16 handles arbitrary leading dims (*x.shape[:-1]),
|
||||
# so dequant the whole [E, N, K] weight in one vectorized call.
|
||||
w13_bf16 = dequant_mxfp8_to_bf16(layer.w13_weight, layer.w13_weight_scale).to(
|
||||
target_dtype
|
||||
)
|
||||
w2_bf16 = dequant_mxfp8_to_bf16(layer.w2_weight, layer.w2_weight_scale).to(
|
||||
target_dtype
|
||||
)
|
||||
|
||||
replace_parameter(layer, "w13_weight", w13_bf16)
|
||||
replace_parameter(layer, "w2_weight", w2_bf16)
|
||||
|
||||
logger.info_once(
|
||||
"MXFP8->BF16 load-time dequant complete (%d experts/layer); MoE "
|
||||
"now runs in BF16 with no per-step dequant.",
|
||||
num_experts,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
# TODO(bnell): why is this required only for mxfp8?
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
@@ -2102,6 +2164,17 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
|
||||
# No native MXFP8 MoE kernel on this device (e.g. gfx942): the emulation
|
||||
# experts would dequant MXFP8->BF16 every forward step. Convert the
|
||||
# weights to BF16 once, here, so the MoE runs like a BF16 checkpoint.
|
||||
# Opt out (VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD=0) to keep the 1-byte
|
||||
# MXFP8 weights and dequant per-step (~half the memory, much slower).
|
||||
if (
|
||||
self.mxfp8_backend == Fp8MoeBackend.EMULATION
|
||||
and envs.VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD
|
||||
):
|
||||
self._dequant_mxfp8_weights_to_bf16(layer)
|
||||
|
||||
def maybe_make_prepare_finalize(
|
||||
self,
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
@@ -2131,6 +2204,9 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
|
||||
a1_scale=None,
|
||||
a2_scale=None,
|
||||
block_shape=self.weight_block_size,
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
)
|
||||
|
||||
def apply_monolithic(
|
||||
|
||||
@@ -159,75 +159,104 @@ def _silu_mul_quant_fp8_packed_kernel(
|
||||
output_q_stride_m,
|
||||
output_scale_stride_k,
|
||||
clamp_limit,
|
||||
alpha,
|
||||
beta,
|
||||
N: tl.constexpr,
|
||||
NUM_GROUPS: tl.constexpr,
|
||||
GROUPS_PER_ROW: tl.constexpr,
|
||||
PACKS_PER_ROW: tl.constexpr,
|
||||
fp8_min: tl.constexpr,
|
||||
fp8_max: tl.constexpr,
|
||||
GROUP_SIZE: tl.constexpr,
|
||||
PACKS_PER_CTA: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
HAS_CLAMP: tl.constexpr,
|
||||
):
|
||||
N_2: tl.constexpr = N // 2
|
||||
GROUPS_PER_PACK: tl.constexpr = 4
|
||||
hidden_size: tl.constexpr = N // 2
|
||||
|
||||
pid_pack = tl.program_id(0)
|
||||
pid_m = tl.program_id(1)
|
||||
m_offset = pid_m.to(tl.int64) * BLOCK_M
|
||||
pack_tile = tl.program_id(0)
|
||||
row_start = tl.program_id(1).to(tl.int64) * BLOCK_M
|
||||
row_step = tl.num_programs(1).to(tl.int64) * BLOCK_M
|
||||
|
||||
if m_offset >= M:
|
||||
return
|
||||
groups_per_cta: tl.constexpr = PACKS_PER_CTA * GROUPS_PER_PACK
|
||||
elems_per_cta: tl.constexpr = groups_per_cta * GROUP_SIZE
|
||||
col_start = pack_tile * elems_per_cta
|
||||
col_offsets = tl.arange(0, elems_per_cta)
|
||||
row_offsets = tl.arange(0, BLOCK_M)
|
||||
pack_offsets = tl.arange(0, PACKS_PER_CTA)
|
||||
|
||||
offs_m = tl.arange(0, BLOCK_M)
|
||||
offs_n = tl.arange(0, GROUP_SIZE)
|
||||
row_mask = (m_offset + offs_m) < M
|
||||
col_mask = (col_start + col_offsets) < (GROUPS_PER_ROW * GROUP_SIZE)
|
||||
|
||||
base_row_offset = (m_offset + offs_m[:, None]) * input_stride_m
|
||||
base_out_offset = (m_offset + offs_m[:, None]) * output_q_stride_m
|
||||
# persistent with grid_m-stride loop
|
||||
while row_start < M:
|
||||
rows = row_start + row_offsets
|
||||
row_mask = rows < M
|
||||
input_row_start = rows[:, None] * input_stride_m
|
||||
output_row_start = rows[:, None] * output_q_stride_m
|
||||
|
||||
packed_scale = tl.zeros((BLOCK_M,), dtype=tl.int32)
|
||||
gate_flat = tl.load(
|
||||
input_ptr + input_row_start + col_start + col_offsets[None, :],
|
||||
mask=row_mask[:, None] & col_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
up_flat = tl.load(
|
||||
input_ptr
|
||||
+ input_row_start
|
||||
+ hidden_size
|
||||
+ col_start
|
||||
+ col_offsets[None, :],
|
||||
mask=row_mask[:, None] & col_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
|
||||
for pack_idx in tl.static_range(4):
|
||||
group_id = pid_pack * 4 + pack_idx
|
||||
gate = tl.reshape(gate_flat, (BLOCK_M, groups_per_cta, GROUP_SIZE)).to(
|
||||
tl.float32
|
||||
)
|
||||
up = tl.reshape(up_flat, (BLOCK_M, groups_per_cta, GROUP_SIZE)).to(tl.float32)
|
||||
|
||||
if group_id < NUM_GROUPS:
|
||||
n_offset = group_id * GROUP_SIZE
|
||||
if HAS_CLAMP:
|
||||
gate = tl.minimum(gate, clamp_limit)
|
||||
up = tl.clamp(up, -clamp_limit, clamp_limit)
|
||||
|
||||
act_ptrs = input_ptr + base_row_offset + n_offset + offs_n[None, :]
|
||||
act_in = tl.load(act_ptrs, mask=row_mask[:, None], other=0.0)
|
||||
# Unified gated activation: silu == swigluoai with alpha=1, beta=0.
|
||||
# glu = gate * sigmoid(alpha * gate); y = (up + beta) * glu
|
||||
glu = gate / (1.0 + tl.exp(-gate * alpha))
|
||||
y = (up + beta) * glu
|
||||
# Round through bf16 to match unfused precision path
|
||||
y = y.to(tl.bfloat16).to(tl.float32)
|
||||
|
||||
mul_ptrs = act_ptrs + N_2
|
||||
mul_in = tl.load(mul_ptrs, mask=row_mask[:, None], other=0.0)
|
||||
absmax = tl.max(tl.abs(y), axis=2)
|
||||
scale_raw = tl.maximum(absmax / fp8_max, 1e-10)
|
||||
exponent = tl.ceil(tl.log2(scale_raw))
|
||||
scale = tl.math.exp2(exponent)
|
||||
|
||||
act_f32 = act_in.to(tl.float32)
|
||||
mul_f32 = mul_in.to(tl.float32)
|
||||
y_q = tl.clamp(y / scale[:, :, None], fp8_min, fp8_max)
|
||||
|
||||
if HAS_CLAMP:
|
||||
act_f32 = tl.minimum(act_f32, clamp_limit)
|
||||
mul_f32 = tl.clamp(mul_f32, -clamp_limit, clamp_limit)
|
||||
y_q_flat = tl.reshape(y_q, (BLOCK_M, elems_per_cta))
|
||||
tl.store(
|
||||
output_q_ptr + output_row_start + col_start + col_offsets[None, :],
|
||||
y_q_flat.to(output_q_ptr.dtype.element_ty),
|
||||
mask=row_mask[:, None] & col_mask[None, :],
|
||||
)
|
||||
|
||||
y = (act_f32 / (1.0 + tl.exp(-act_f32))) * mul_f32
|
||||
# Round through bf16 to match unfused precision path
|
||||
y = y.to(tl.bfloat16).to(tl.float32)
|
||||
scale_byte = tl.clamp(exponent + 127.0, 0.0, 255.0).to(tl.int32)
|
||||
scale_bytes = tl.reshape(scale_byte, (BLOCK_M, PACKS_PER_CTA, GROUPS_PER_PACK))
|
||||
shifts = tl.arange(0, GROUPS_PER_PACK) * 8
|
||||
packed_scale = tl.sum(scale_bytes << shifts[None, None, :], axis=2)
|
||||
|
||||
absmax = tl.max(tl.abs(y), axis=1)
|
||||
scale_pack = pack_tile * PACKS_PER_CTA + pack_offsets
|
||||
scale_ptrs = (
|
||||
output_scale_ptr
|
||||
+ scale_pack[None, :] * output_scale_stride_k
|
||||
+ rows[:, None]
|
||||
)
|
||||
tl.store(
|
||||
scale_ptrs,
|
||||
packed_scale,
|
||||
mask=row_mask[:, None] & (scale_pack[None, :] < PACKS_PER_ROW),
|
||||
)
|
||||
|
||||
scale_raw = tl.maximum(absmax / fp8_max, 1e-10)
|
||||
exponent = tl.ceil(tl.log2(scale_raw))
|
||||
scale = tl.math.exp2(exponent)
|
||||
|
||||
y_q = tl.clamp(y / scale[:, None], fp8_min, fp8_max)
|
||||
|
||||
out_q_ptrs = output_q_ptr + base_out_offset + n_offset + offs_n[None, :]
|
||||
tl.store(
|
||||
out_q_ptrs,
|
||||
y_q.to(output_q_ptr.dtype.element_ty),
|
||||
mask=row_mask[:, None],
|
||||
)
|
||||
|
||||
exponent_biased = tl.clamp(exponent + 127.0, 0.0, 255.0).to(tl.int32)
|
||||
packed_scale = packed_scale | (exponent_biased << (pack_idx * 8))
|
||||
|
||||
scale_ptrs = output_scale_ptr + pid_pack * output_scale_stride_k + m_offset + offs_m
|
||||
tl.store(scale_ptrs, packed_scale, mask=row_mask)
|
||||
row_start += row_step
|
||||
|
||||
|
||||
def silu_mul_quant_fp8_packed_triton(
|
||||
@@ -235,37 +264,48 @@ def silu_mul_quant_fp8_packed_triton(
|
||||
group_size: int = 128,
|
||||
output_q: torch.Tensor | None = None,
|
||||
clamp_limit: float | None = None,
|
||||
alpha: float = 1.0,
|
||||
beta: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert input.dim() == 2
|
||||
assert input.is_contiguous()
|
||||
|
||||
M, N = input.shape
|
||||
N_2 = N // 2
|
||||
hidden_size = N // 2
|
||||
|
||||
assert N_2 % group_size == 0
|
||||
assert hidden_size % group_size == 0
|
||||
|
||||
fp8_dtype = torch.float8_e4m3fn
|
||||
finfo = torch.finfo(fp8_dtype)
|
||||
fp8_min, fp8_max = finfo.min, finfo.max
|
||||
|
||||
num_groups_per_row = N_2 // group_size
|
||||
num_packed_groups = (num_groups_per_row + 3) // 4
|
||||
tma_aligned_M = ((M + 3) // 4) * 4
|
||||
groups_per_row = hidden_size // group_size
|
||||
groups_per_pack = 4 # pack 4 UE8M0 scales to a single INT32
|
||||
packs_per_row = triton.cdiv(groups_per_row, groups_per_pack)
|
||||
|
||||
if output_q is None:
|
||||
output_q = torch.empty((M, N_2), dtype=fp8_dtype, device=input.device)
|
||||
output_q = torch.empty((M, hidden_size), dtype=fp8_dtype, device=input.device)
|
||||
|
||||
aligned_m = triton.cdiv(M, 4) * 4
|
||||
output_scale_packed = torch.empty(
|
||||
(num_packed_groups, tma_aligned_M),
|
||||
(packs_per_row, aligned_m),
|
||||
dtype=torch.int32,
|
||||
device=input.device,
|
||||
).T[:M, :]
|
||||
|
||||
BLOCK_M = 8
|
||||
grid = (num_packed_groups, (M + BLOCK_M - 1) // BLOCK_M)
|
||||
|
||||
num_warps = max(4, group_size // 32)
|
||||
# Tuned for group_size=32 (MXFP8) and group_size=128 (DeepSeek-V4)
|
||||
num_warps = 4
|
||||
num_stages = 2
|
||||
if group_size < 128:
|
||||
BM = 1
|
||||
packs_per_cta = 8
|
||||
else:
|
||||
BM = 1 if M < 512 else 4
|
||||
packs_per_cta = 2 if M < 512 else 1
|
||||
|
||||
grid_n = triton.cdiv(packs_per_row, packs_per_cta)
|
||||
grid_m = min(triton.cdiv(M, BM), 4096)
|
||||
grid = (grid_n, grid_m)
|
||||
|
||||
has_clamp = clamp_limit is not None
|
||||
_silu_mul_quant_fp8_packed_kernel[grid](
|
||||
@@ -277,12 +317,16 @@ def silu_mul_quant_fp8_packed_triton(
|
||||
output_q.stride(0),
|
||||
output_scale_packed.stride(1),
|
||||
clamp_limit if has_clamp else 0.0,
|
||||
alpha,
|
||||
beta,
|
||||
N=N,
|
||||
NUM_GROUPS=num_groups_per_row,
|
||||
GROUPS_PER_ROW=groups_per_row,
|
||||
PACKS_PER_ROW=packs_per_row,
|
||||
fp8_min=fp8_min,
|
||||
fp8_max=fp8_max,
|
||||
GROUP_SIZE=group_size,
|
||||
BLOCK_M=BLOCK_M,
|
||||
PACKS_PER_CTA=packs_per_cta,
|
||||
BLOCK_M=BM,
|
||||
HAS_CLAMP=has_clamp,
|
||||
num_warps=num_warps,
|
||||
num_stages=num_stages,
|
||||
@@ -303,6 +347,8 @@ def _silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
# Information for float8
|
||||
eps,
|
||||
clamp_limit,
|
||||
alpha,
|
||||
beta,
|
||||
fp8_min: tl.constexpr,
|
||||
fp8_max: tl.constexpr,
|
||||
use_ue8m0: tl.constexpr,
|
||||
@@ -348,10 +394,14 @@ def _silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
mul_in = tl.clamp(mul_in.to(tl.float32), -clamp_limit, clamp_limit).to(
|
||||
y_ptr.dtype.element_ty
|
||||
)
|
||||
# Unified gated activation: silu == swigluoai with alpha=1, beta=0.
|
||||
# glu = gate * sigmoid(alpha * gate); y = (up + beta) * glu
|
||||
# Keep glu/up at input precision (narrow before the mul) so the alpha=1,
|
||||
# beta=0 defaults match the C++ silu_and_mul path bit-for-bit.
|
||||
act_in = act_in.to(tl.float32)
|
||||
one_f32 = tl.cast(1, tl.float32)
|
||||
silu_out = (act_in / (one_f32 + tl.exp(-act_in))).to(y_ptr.dtype.element_ty)
|
||||
y = (silu_out * mul_in).to(tl.float32)
|
||||
glu = (act_in / (1.0 + tl.exp(-act_in * alpha))).to(y_ptr.dtype.element_ty)
|
||||
up = (mul_in.to(tl.float32) + beta).to(y_ptr.dtype.element_ty)
|
||||
y = (glu * up).to(tl.float32)
|
||||
|
||||
# quant
|
||||
_absmax = tl.maximum(tl.max(tl.abs(y), axis=1), eps)
|
||||
@@ -379,11 +429,15 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
use_ue8m0: bool | None = None,
|
||||
eps: float = 1e-10,
|
||||
clamp_limit: float | None = None,
|
||||
group_size: int = 128,
|
||||
alpha: float = 1.0,
|
||||
beta: float = 0.0,
|
||||
):
|
||||
"""
|
||||
silu+mul + block-fp8 quant with group size 128.
|
||||
Gated activation + block-fp8 quant. ``alpha``/``beta`` select the gate
|
||||
(silu: alpha=1, beta=0; swigluoai: alpha, beta from config).
|
||||
"""
|
||||
GROUP_SIZE = 128
|
||||
GROUP_SIZE = group_size
|
||||
assert input.ndim == 2
|
||||
if output is not None:
|
||||
assert output.ndim == 2
|
||||
@@ -431,6 +485,8 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
output_scales.stride(-1),
|
||||
eps,
|
||||
clamp_limit if has_clamp else 0.0,
|
||||
alpha,
|
||||
beta,
|
||||
fp8_min,
|
||||
fp8_max,
|
||||
use_ue8m0,
|
||||
@@ -1015,9 +1071,10 @@ def deepgemm_post_process_fp8_weight_block(
|
||||
f"to be torch.float8_e4m3fn, got {wq.dtype} instead."
|
||||
)
|
||||
|
||||
if ws.dtype == torch.float8_e8m0fnu:
|
||||
# Scales already in E8M0 from checkpoint — upcast to fp32
|
||||
# and skip requantization (weights already have power-of-two scales).
|
||||
if ws.dtype in (torch.float8_e8m0fnu, torch.uint8):
|
||||
# Scales already in E8M0 from checkpoint (float8_e8m0fnu, or raw E8M0
|
||||
# bits as uint8 for MXFP8) — upcast to fp32 and skip requantization
|
||||
# (weights already have power-of-two scales).
|
||||
ws = _upcast_e8m0_to_fp32(ws)
|
||||
else:
|
||||
assert ws.dtype == torch.float32, (
|
||||
@@ -1057,7 +1114,8 @@ def deepgemm_post_process_fp8_weight_block(
|
||||
ws = ws.unsqueeze(0)
|
||||
|
||||
# From https://github.com/deepseek-ai/DeepGEMM/blob/c9f8b34dcdacc20aa746b786f983492c51072870/csrc/utils/layout.hpp#L46
|
||||
recipe = (1, 128, 128)
|
||||
# (1, block_n, block_k): (1, 128, 128) for FP8 block, (1, 1, 32) for MXFP8.
|
||||
recipe = (1, quant_block_shape[0], quant_block_shape[1])
|
||||
|
||||
# Ref : https://github.com/deepseek-ai/DeepGEMM/blob/c9f8b34dcdacc20aa746b786f983492c51072870/csrc/apis/gemm.hpp
|
||||
# DeepGemm uses the `transform_sf_into_required_layout` function to
|
||||
|
||||
@@ -84,6 +84,92 @@ def _mxfp8_e4m3_quantize_torch(
|
||||
return x_fp8, scales_uint8
|
||||
|
||||
|
||||
def _mxfp8_quant_triton_kernel():
|
||||
"""Lazily-built Triton kernel: per-32-block E8M0 scale + FP8-E4M3 quant.
|
||||
|
||||
Fuses what ``_mxfp8_e4m3_quantize_torch`` does in several elementwise passes
|
||||
into one launch. Each program handles ``[BLOCK_M, 32]`` (one MX block).
|
||||
"""
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
@triton.jit
|
||||
def _kernel(
|
||||
x_ptr,
|
||||
xq_ptr,
|
||||
s_ptr,
|
||||
M,
|
||||
K,
|
||||
sxm,
|
||||
sxk,
|
||||
sqm,
|
||||
sqk,
|
||||
ssm,
|
||||
ssk,
|
||||
BLOCK_M: tl.constexpr,
|
||||
):
|
||||
pid_m = tl.program_id(0)
|
||||
pid_b = tl.program_id(1) # which 32-element block along K
|
||||
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
offs_k = pid_b * 32 + tl.arange(0, 32)
|
||||
m_mask = offs_m < M
|
||||
x = tl.load(
|
||||
x_ptr + offs_m[:, None] * sxm + offs_k[None, :] * sxk,
|
||||
mask=m_mask[:, None],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
amax = tl.maximum(tl.max(tl.abs(x), axis=1), 1e-30) # [BLOCK_M]
|
||||
sb = tl.floor(tl.log2(amax)) + 127.0
|
||||
sb = tl.minimum(tl.maximum(sb, 0.0), 254.0)
|
||||
descale = tl.exp2(sb - 127.0)
|
||||
xq = (x / descale[:, None]).to(xq_ptr.dtype.element_ty)
|
||||
tl.store(
|
||||
xq_ptr + offs_m[:, None] * sqm + offs_k[None, :] * sqk,
|
||||
xq,
|
||||
mask=m_mask[:, None],
|
||||
)
|
||||
tl.store(s_ptr + offs_m * ssm + pid_b * ssk, sb.to(tl.uint8), mask=m_mask)
|
||||
|
||||
return _kernel
|
||||
|
||||
|
||||
_MXFP8_QUANT_KERNEL = None
|
||||
|
||||
|
||||
def _mxfp8_e4m3_quantize_triton(
|
||||
x: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Fused 2D MXFP8 quant (non-swizzled, row-major [M, K//32] scales)."""
|
||||
from vllm.triton_utils import triton
|
||||
|
||||
global _MXFP8_QUANT_KERNEL
|
||||
if _MXFP8_QUANT_KERNEL is None:
|
||||
_MXFP8_QUANT_KERNEL = _mxfp8_quant_triton_kernel()
|
||||
|
||||
M, K = x.shape
|
||||
x = x.contiguous()
|
||||
xq = torch.empty((M, K), dtype=MXFP8_VALUE_DTYPE, device=x.device)
|
||||
scales = torch.empty(
|
||||
(M, K // MXFP8_BLOCK_SIZE), dtype=MXFP8_SCALE_DTYPE, device=x.device
|
||||
)
|
||||
BLOCK_M = 64
|
||||
grid = (triton.cdiv(M, BLOCK_M), K // MXFP8_BLOCK_SIZE)
|
||||
_MXFP8_QUANT_KERNEL[grid](
|
||||
x,
|
||||
xq,
|
||||
scales,
|
||||
M,
|
||||
K,
|
||||
x.stride(0),
|
||||
x.stride(1),
|
||||
xq.stride(0),
|
||||
xq.stride(1),
|
||||
scales.stride(0),
|
||||
scales.stride(1),
|
||||
BLOCK_M=BLOCK_M,
|
||||
)
|
||||
return xq, scales
|
||||
|
||||
|
||||
def _mxfp8_e4m3_quantize_impl(
|
||||
x: torch.Tensor,
|
||||
is_sf_swizzled_layout: bool = False,
|
||||
@@ -103,6 +189,17 @@ def _mxfp8_e4m3_quantize_impl(
|
||||
x_scales = x_scales.view(x.size(0), -1)
|
||||
return x_q, x_scales
|
||||
|
||||
# ROCm: a single fused Triton kernel beats the multi-pass torch path for the
|
||||
# common 2D, non-swizzled activation-quant case (used by the native MX
|
||||
# linear/MoE). Falls back to torch otherwise (3D weights, swizzled layout).
|
||||
if (
|
||||
current_platform.is_rocm()
|
||||
and not is_sf_swizzled_layout
|
||||
and x.ndim == 2
|
||||
and x.shape[-1] % MXFP8_BLOCK_SIZE == 0
|
||||
):
|
||||
return _mxfp8_e4m3_quantize_triton(x)
|
||||
|
||||
return _mxfp8_e4m3_quantize_torch(x, is_sf_swizzled_layout)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user