Files

79 lines
2.9 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
# atom/mono/device/mx.py
"""FP8 / MX arithmetic the mono kernels share: the E4M3 range, E8M0 scale
codes, the byte of a scale in aiter's ``shuffle_scale`` layout, an FP4 weight
tile's operand loads and the scaled MFMA that takes them.
An E8M0 code is a biased exponent; ``pow2(code)`` is the scale it stands for."""
import flydsl.expr as fx
from aiter.ops.flydsl.kernels import buffer_ops as bo
from flydsl.expr import rocdl
from flydsl.expr.typing import T
from vllm.models.deepseek_v41.amd.mono.common.ops import CM_NT
FP8_MAX = 448.0
# the f8f6f4 MFMA's operand formats (cbsz / blgp)
FP8, FP4 = 0, 4
UNIT_SCALE = 127 # the E8M0 code of 1.0
def clamp_fp8(v):
"""``v`` clamped to the E4M3 range."""
return fx.max(fx.min(v, fx.Float32(FP8_MAX)), fx.Float32(-FP8_MAX))
def code_ceil(v):
"""Biased exponent of the smallest power of two >= ``v`` (v >= 0, finite),
clamped to [1, 254]: the codes a scale may take."""
bits = v.bitcast(fx.Int32)
e = ((bits >> 23) & 0xFF) + ((bits & 0x7FFFFF) != 0).select(
fx.Int32(1), fx.Int32(0)
)
return fx.max(fx.min(e, fx.Int32(254)), fx.Int32(1))
def rcp_pow2(code):
"""1 / ``pow2(code)``, exact (code in [1, 253]): x times it is x / scale
bit for bit, a multiply where the division is a dozen instructions."""
return ((254 - code) << 23).bitcast(fx.Float32)
def scale_index(row, col, cols):
"""Byte of e8m0 scale (row, col) in aiter's ``shuffle_scale`` layout:
``view(rows/32, 2, 16, cols/8, 2, 4).permute(0, 3, 5, 2, 4, 1)``."""
r32, a, b = row // 32, (row // 16) % 2, row % 16
c8, d, e = col // 8, (col // 4) % 2, col % 4
return ((((r32 * (cols // 8) + c8) * 4 + e) * 16 + b) * 2 + d) * 2 + a
def fp4_tile_load(r_w, base, rg, k, st, at):
"""A lane's 16 B of an FP4 weight's (16, 16)-shuffled 16-row group ``rg``
(K = ``k``) at 128-k step ``st``: dword ``base`` + the group's 2 k + 256 a
step + ``at`` (``lane * 4``: 32 fp4, K block lane / 16 of row lane % 16).
Nontemporal: a weight a step reads once."""
return fx.Vector(
bo.buffer_load(
r_w,
base + rg * (2 * k) + st * 256 + at,
vec_width=4,
dtype=T.i32,
cache_modifier=CM_NT,
)
)
def mfma_scaled(a, b, c, sa, sb, a_fmt=FP8, b_fmt=FP8, sa_byte=0, sb_byte=0):
"""One 16x16x128 scaled MFMA (gfx950 f8f6f4): A ``a`` (``a_fmt``) with E8M0
``sa`` (its byte ``sa_byte``), B ``b`` (``b_fmt``) with E8M0 ``sb`` (its
byte ``sb_byte``)."""
return fx.Vector(
rocdl.mfma_scale_f32_16x16x128_f8f6f4(
T.vec(4, T.f32), [a, b, c, a_fmt, b_fmt, sa_byte, sa, sb_byte, sb]
)
)