Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
79 lines
2.9 KiB
Python
79 lines
2.9 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
|
|
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
|
|
# atom/mono/device/mx.py
|
|
"""FP8 / MX arithmetic the mono kernels share: the E4M3 range, E8M0 scale
|
|
codes, the byte of a scale in aiter's ``shuffle_scale`` layout, an FP4 weight
|
|
tile's operand loads and the scaled MFMA that takes them.
|
|
|
|
An E8M0 code is a biased exponent; ``pow2(code)`` is the scale it stands for."""
|
|
|
|
import flydsl.expr as fx
|
|
from aiter.ops.flydsl.kernels import buffer_ops as bo
|
|
from flydsl.expr import rocdl
|
|
from flydsl.expr.typing import T
|
|
|
|
from vllm.models.deepseek_v41.amd.mono.common.ops import CM_NT
|
|
|
|
FP8_MAX = 448.0
|
|
# the f8f6f4 MFMA's operand formats (cbsz / blgp)
|
|
FP8, FP4 = 0, 4
|
|
UNIT_SCALE = 127 # the E8M0 code of 1.0
|
|
|
|
|
|
def clamp_fp8(v):
|
|
"""``v`` clamped to the E4M3 range."""
|
|
return fx.max(fx.min(v, fx.Float32(FP8_MAX)), fx.Float32(-FP8_MAX))
|
|
|
|
|
|
def code_ceil(v):
|
|
"""Biased exponent of the smallest power of two >= ``v`` (v >= 0, finite),
|
|
clamped to [1, 254]: the codes a scale may take."""
|
|
bits = v.bitcast(fx.Int32)
|
|
e = ((bits >> 23) & 0xFF) + ((bits & 0x7FFFFF) != 0).select(
|
|
fx.Int32(1), fx.Int32(0)
|
|
)
|
|
return fx.max(fx.min(e, fx.Int32(254)), fx.Int32(1))
|
|
|
|
|
|
def rcp_pow2(code):
|
|
"""1 / ``pow2(code)``, exact (code in [1, 253]): x times it is x / scale
|
|
bit for bit, a multiply where the division is a dozen instructions."""
|
|
return ((254 - code) << 23).bitcast(fx.Float32)
|
|
|
|
|
|
def scale_index(row, col, cols):
|
|
"""Byte of e8m0 scale (row, col) in aiter's ``shuffle_scale`` layout:
|
|
``view(rows/32, 2, 16, cols/8, 2, 4).permute(0, 3, 5, 2, 4, 1)``."""
|
|
r32, a, b = row // 32, (row // 16) % 2, row % 16
|
|
c8, d, e = col // 8, (col // 4) % 2, col % 4
|
|
return ((((r32 * (cols // 8) + c8) * 4 + e) * 16 + b) * 2 + d) * 2 + a
|
|
|
|
|
|
def fp4_tile_load(r_w, base, rg, k, st, at):
|
|
"""A lane's 16 B of an FP4 weight's (16, 16)-shuffled 16-row group ``rg``
|
|
(K = ``k``) at 128-k step ``st``: dword ``base`` + the group's 2 k + 256 a
|
|
step + ``at`` (``lane * 4``: 32 fp4, K block lane / 16 of row lane % 16).
|
|
Nontemporal: a weight a step reads once."""
|
|
return fx.Vector(
|
|
bo.buffer_load(
|
|
r_w,
|
|
base + rg * (2 * k) + st * 256 + at,
|
|
vec_width=4,
|
|
dtype=T.i32,
|
|
cache_modifier=CM_NT,
|
|
)
|
|
)
|
|
|
|
|
|
def mfma_scaled(a, b, c, sa, sb, a_fmt=FP8, b_fmt=FP8, sa_byte=0, sb_byte=0):
|
|
"""One 16x16x128 scaled MFMA (gfx950 f8f6f4): A ``a`` (``a_fmt``) with E8M0
|
|
``sa`` (its byte ``sa_byte``), B ``b`` (``b_fmt``) with E8M0 ``sb`` (its
|
|
byte ``sb_byte``)."""
|
|
return fx.Vector(
|
|
rocdl.mfma_scale_f32_16x16x128_f8f6f4(
|
|
T.vec(4, T.f32), [a, b, c, a_fmt, b_fmt, sa_byte, sa, sb_byte, sb]
|
|
)
|
|
)
|