[Bugfix][ROCm] Fix MiniMax-M3 fused MXFP8 block scale (#53674)
Signed-off-by: maithilijoshi20 <97733343+maithilijoshi20@users.noreply.github.com>
This commit is contained in:
@@ -42,6 +42,7 @@ from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( # noqa:
|
|||||||
from vllm.models.minimax_m3.amd.ops import ( # noqa: E402
|
from vllm.models.minimax_m3.amd.ops import ( # noqa: E402
|
||||||
gemma_fused_add_rmsnorm,
|
gemma_fused_add_rmsnorm,
|
||||||
gemma_rmsnorm,
|
gemma_rmsnorm,
|
||||||
|
swiglu_oai_quantize_mxfp8,
|
||||||
swiglu_oai_split,
|
swiglu_oai_split,
|
||||||
)
|
)
|
||||||
from vllm.models.minimax_m3.amd.ops.gemma_rmsnorm import _num_warps # noqa: E402
|
from vllm.models.minimax_m3.amd.ops.gemma_rmsnorm import _num_warps # noqa: E402
|
||||||
@@ -167,6 +168,25 @@ def test_swiglu_oai_split(m, inter, limit, dtype):
|
|||||||
assert _relerr(got, ref) < 5e-3
|
assert _relerr(got, ref) < 5e-3
|
||||||
|
|
||||||
|
|
||||||
|
@torch.inference_mode()
|
||||||
|
def test_swiglu_oai_quantize_mxfp8_uses_e4m3_range_for_scale():
|
||||||
|
# Keep a small value in two MX blocks next to their maxima. The scale must
|
||||||
|
# use E4M3's finite range (448), otherwise that value underflows to zero.
|
||||||
|
# Keep the third block empty to cover the reference's finite tiny clamp.
|
||||||
|
m, inter = 3, 96
|
||||||
|
gate = torch.full((m, inter), 2.0, device=DEVICE, dtype=torch.float16)
|
||||||
|
up = torch.full((m, inter), 2**-10, device=DEVICE, dtype=torch.float16)
|
||||||
|
up[:, :64:32] = 1.0
|
||||||
|
up[:, 64:] = 0.0
|
||||||
|
gate_up = torch.cat((gate, up), dim=-1)
|
||||||
|
|
||||||
|
got_q, got_s = swiglu_oai_quantize_mxfp8(gate_up, alpha=0.0, beta=0.0, limit=None)
|
||||||
|
ref_q, ref_s = _mxfp8_e4m3_quantize_torch(up, is_sf_swizzled_layout=False)
|
||||||
|
|
||||||
|
assert torch.equal(got_s, ref_s)
|
||||||
|
assert torch.equal(got_q, ref_q)
|
||||||
|
|
||||||
|
|
||||||
# --------------------------------------------------------------------------- #
|
# --------------------------------------------------------------------------- #
|
||||||
# Fused MXFP8 activation quant (Triton vs torch reference)
|
# Fused MXFP8 activation quant (Triton vs torch reference)
|
||||||
# --------------------------------------------------------------------------- #
|
# --------------------------------------------------------------------------- #
|
||||||
@@ -179,8 +199,8 @@ def test_mxfp8_quant_triton_matches_torch(shape, dtype):
|
|||||||
xq_t, s_t = _mxfp8_e4m3_quantize_torch(x, is_sf_swizzled_layout=False)
|
xq_t, s_t = _mxfp8_e4m3_quantize_torch(x, is_sf_swizzled_layout=False)
|
||||||
xq_k, s_k = _mxfp8_e4m3_quantize_triton(x)
|
xq_k, s_k = _mxfp8_e4m3_quantize_triton(x)
|
||||||
assert s_k.shape == s_t.shape == (shape[0], shape[1] // 32)
|
assert s_k.shape == s_t.shape == (shape[0], shape[1] // 32)
|
||||||
# E8M0 block exponents share the floor(log2(amax))+127 algorithm; allow at
|
# Both paths use the E4M3-aware scale calculation; allow a 1-step
|
||||||
# most a 1-step difference at exact powers of two.
|
# difference at exact powers of two due to floating-point rounding.
|
||||||
assert (s_k.int() - s_t.int()).abs().max().item() <= 1
|
assert (s_k.int() - s_t.int()).abs().max().item() <= 1
|
||||||
# Dequantized values agree to fp8 granularity.
|
# Dequantized values agree to fp8 granularity.
|
||||||
deq_t = dequant_mxfp8_to_bf16(xq_t, s_t)
|
deq_t = dequant_mxfp8_to_bf16(xq_t, s_t)
|
||||||
|
|||||||
@@ -83,6 +83,8 @@ def _swiglu_oai_quant_kernel(
|
|||||||
limit,
|
limit,
|
||||||
HAS_LIMIT: tl.constexpr,
|
HAS_LIMIT: tl.constexpr,
|
||||||
BLOCK_M: tl.constexpr,
|
BLOCK_M: tl.constexpr,
|
||||||
|
FP8_MAX: tl.constexpr,
|
||||||
|
TINY: tl.constexpr,
|
||||||
):
|
):
|
||||||
"""SwiGLU-OAI (split layout) fused with per-32-block MXFP8 (E4M3 + E8M0)
|
"""SwiGLU-OAI (split layout) fused with per-32-block MXFP8 (E4M3 + E8M0)
|
||||||
quant. Each program handles ``[BLOCK_M, 32]`` of the ``[M, I]`` output (one
|
quant. Each program handles ``[BLOCK_M, 32]`` of the ``[M, I]`` output (one
|
||||||
@@ -109,8 +111,8 @@ def _swiglu_oai_quant_kernel(
|
|||||||
gate = tl.minimum(gate, limit)
|
gate = tl.minimum(gate, limit)
|
||||||
up = tl.minimum(tl.maximum(up, -limit), limit)
|
up = tl.minimum(tl.maximum(up, -limit), limit)
|
||||||
act = gate * tl.sigmoid(alpha * gate) * (up + beta) # [BLOCK_M, 32] fp32
|
act = gate * tl.sigmoid(alpha * gate) * (up + beta) # [BLOCK_M, 32] fp32
|
||||||
amax = tl.maximum(tl.max(tl.abs(act), axis=1), 1e-30) # [BLOCK_M]
|
amax = tl.maximum(tl.max(tl.abs(act), axis=1), TINY) # [BLOCK_M]
|
||||||
sb = tl.minimum(tl.maximum(tl.floor(tl.log2(amax)) + 127.0, 0.0), 254.0)
|
sb = tl.minimum(tl.maximum(tl.ceil(tl.log2(amax / FP8_MAX)) + 127.0, 0.0), 254.0)
|
||||||
descale = tl.exp2(sb - 127.0)
|
descale = tl.exp2(sb - 127.0)
|
||||||
aq = (act / descale[:, None]).to(aq_ptr.dtype.element_ty)
|
aq = (act / descale[:, None]).to(aq_ptr.dtype.element_ty)
|
||||||
tl.store(
|
tl.store(
|
||||||
@@ -173,6 +175,8 @@ def swiglu_oai_quantize_mxfp8(
|
|||||||
0.0 if limit is None else float(limit),
|
0.0 if limit is None else float(limit),
|
||||||
HAS_LIMIT=limit is not None,
|
HAS_LIMIT=limit is not None,
|
||||||
BLOCK_M=block_m,
|
BLOCK_M=block_m,
|
||||||
|
FP8_MAX=float(torch.finfo(MXFP8_VALUE_DTYPE).max),
|
||||||
|
TINY=float(torch.finfo(torch.float32).tiny),
|
||||||
num_warps=4,
|
num_warps=4,
|
||||||
)
|
)
|
||||||
return aq, asc
|
return aq, asc
|
||||||
|
|||||||
Reference in New Issue
Block a user