Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
505 lines
15 KiB
Python
505 lines
15 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Device primitives of the DeepSeek-V4.1 attention mega kernel.
|
|
|
|
Wave reductions and shuffles, hardware math, bf16 / fp8 / MX packing, the
|
|
scaled MFMA, buffer and 64-bit global accesses, and the tagged-pair mailbox.
|
|
Adapted from ATOM's mono decode framework (``atom/mono/device``), itself ported
|
|
from the FlyDSL GLM-5 layer kernel.
|
|
"""
|
|
|
|
import flydsl.expr as fx
|
|
from aiter.ops.flydsl.kernels import buffer_ops as bo
|
|
from aiter.ops.flydsl.kernels.dpp_utils import update_dpp_i32
|
|
from aiter.ops.flydsl.kernels.kernels_common import kernel_signature
|
|
from flydsl._mlir import ir
|
|
from flydsl._mlir.dialects import llvm
|
|
from flydsl.compiler.ast_rewriter import ASTRewriter
|
|
from flydsl.expr import const_expr, gpu, range_constexpr, rocdl
|
|
from flydsl.expr import math as fmath
|
|
from flydsl.expr.typing import T, as_ir_value
|
|
|
|
# The execution model: one CTA a CU of an MI355X, all resident.
|
|
BLOCKS = 256
|
|
THREADS = 512
|
|
WAVES = THREADS // 64
|
|
|
|
# gfx95x cache policy bits (LLVM CPol): SC0 = 1, SC1 = 16, NT = 2. SC1 alone is
|
|
# device scope (past the per-XCD L2s); NT streams data read once.
|
|
CM_DEV = 16
|
|
CM_NT = 2
|
|
|
|
FP8_MAX = 448.0
|
|
FP8, FP4 = 0, 4 # f8f6f4 MFMA operand formats
|
|
UNIT_SCALE = 127 # E8M0 code of 1.0
|
|
FLT_MIN = 1.1754943508222875e-38
|
|
|
|
POLL_MAX = 12 # mailbox pairs polled a batch
|
|
|
|
|
|
def traced(fn):
|
|
"""FlyDSL's AST rewrite for a helper, so its ``if`` / ``while`` / ``for`` on
|
|
traced values lower to scf ops (kernels rewrite only their own body)."""
|
|
return ASTRewriter.transform(fn)
|
|
|
|
|
|
def kernel_symbol(stem, **params):
|
|
"""Kernel symbol that profilers demangle to ``aiter::<stem>_<params>``."""
|
|
name = f"{stem}_{kernel_signature(**params)}"
|
|
return f"_ZN5aiter{len(name)}{name}E"
|
|
|
|
|
|
def first_task(bid, base):
|
|
"""CTA ``bid``'s first task of a stage placed from CTA ``base``, then every
|
|
BLOCKS-th."""
|
|
return (bid - base % BLOCKS + BLOCKS) % BLOCKS
|
|
|
|
|
|
# ----------------------------------------------------------------------------- memory
|
|
|
|
|
|
def rsrc(addr, nbytes=None):
|
|
return bo.create_buffer_resource_from_addr(addr, num_records_bytes=nbytes)
|
|
|
|
|
|
def ld_i32(ptr, i, cm=0):
|
|
return fx.Int32(
|
|
bo.buffer_load(rsrc(ptr), i, vec_width=1, dtype=T.i32, cache_modifier=cm)
|
|
)
|
|
|
|
|
|
def ld_f32(ptr, i):
|
|
return fx.Float32(bo.buffer_load(rsrc(ptr), i, vec_width=1, dtype=T.f32))
|
|
|
|
|
|
def ld_bf(ptr, i):
|
|
return fx.Float32(
|
|
fx.BFloat16(bo.buffer_load(rsrc(ptr), i, vec_width=1, dtype=T.bf16))
|
|
)
|
|
|
|
|
|
def ld_i64(ptr, i):
|
|
"""Element i of an i64 buffer (two dwords)."""
|
|
w = fx.Vector(bo.buffer_load(rsrc(ptr), i * 2, vec_width=2, dtype=T.i32))
|
|
return (fx.Int64(w[1]) << 32) | fx.Int64(fx.Uint32(w[0]))
|
|
|
|
|
|
def _gptr(addr):
|
|
return llvm.IntToPtrOp(
|
|
ir.Type.parse("!llvm.ptr<1>"), as_ir_value(fx.Int64(addr))
|
|
).result
|
|
|
|
|
|
def gload(addr, words=4, nt=False):
|
|
"""``words`` i32 (1, 2 or 4) at a 64-bit global address: a cache past 4 GiB is
|
|
reachable only this way (a buffer offset is 32 bits)."""
|
|
i32 = ir.IntegerType.get_signless(32)
|
|
ty = i32 if words == 1 else ir.VectorType.get([words], i32)
|
|
v = llvm.LoadOp(ty, _gptr(addr), alignment=4 * words, nontemporal=nt or None).result
|
|
return fx.Int32(v) if words == 1 else fx.Vector(v)
|
|
|
|
|
|
def atomic_add_dev(addr, value):
|
|
"""Device-scope atomic add on the global i32 at ``addr``; the old value.
|
|
Relaxed: an acquire / release here would write back and invalidate L2."""
|
|
return fx.Int32(
|
|
fx.atomic_add(
|
|
_gptr(addr),
|
|
fx.Int32(value),
|
|
syncscope="agent",
|
|
ordering=fx.AtomicOrdering.Monotonic,
|
|
)
|
|
)
|
|
|
|
|
|
def gstore(addr, value, words=4):
|
|
"""Store ``words`` i32 (an Int32 or a Vector) at a 64-bit global address."""
|
|
v = as_ir_value(fx.Int32(value) if words == 1 else value)
|
|
llvm.StoreOp(v, _gptr(addr), alignment=4 * words)
|
|
|
|
|
|
# ----------------------------------------------------------------------------- math
|
|
|
|
|
|
def fresh(v):
|
|
"""``v`` through an opaque move, so the lane math built on it is computed at
|
|
its use instead of being hoisted out of a task loop and held in registers."""
|
|
return fx.Int32(
|
|
llvm.inline_asm(
|
|
T.i32,
|
|
[fx.Int32(v).ir_value()],
|
|
"v_mov_b32 $0, $1",
|
|
"=v,v",
|
|
has_side_effects=True,
|
|
)
|
|
)
|
|
|
|
|
|
def tile_map(tid, width):
|
|
"""(tid // width, tid % width) for 0 <= tid < THREADS without a division:
|
|
shifts for a power of two, else a multiply-shift exact on that range."""
|
|
if width & (width - 1) == 0:
|
|
sh = width.bit_length() - 1
|
|
return tid >> sh, tid & (width - 1)
|
|
m = -(-(1 << 20) // width)
|
|
assert all((t * m) >> 20 == t // width for t in range(THREADS)), width
|
|
row = (tid * m) >> 20
|
|
return row, tid - row * width
|
|
|
|
|
|
def uniform(v):
|
|
return fx.Int32(rocdl.readfirstlane(T.i32, fx.Int32(v).ir_value()))
|
|
|
|
|
|
def _hw_f32(name, x):
|
|
return fx.Float32(
|
|
llvm.call_intrinsic(T.f32, name, [fx.Float32(x).ir_value()], [], [])
|
|
)
|
|
|
|
|
|
def hw_rsq(x):
|
|
return _hw_f32("llvm.amdgcn.rsq.f32", x)
|
|
|
|
|
|
def hw_rcp(x):
|
|
return _hw_f32("llvm.amdgcn.rcp.f32", x)
|
|
|
|
|
|
def hw_exp2(x):
|
|
return _hw_f32("llvm.amdgcn.exp2.f32", x)
|
|
|
|
|
|
def fma(a, b, c):
|
|
return fx.Float32(fmath.fma(fx.Float32(a), fx.Float32(b), fx.Float32(c)))
|
|
|
|
|
|
def div_rn(x, d, r):
|
|
"""``x / d`` rounded to nearest (IEEE), given ``r = RN(1 / d)``."""
|
|
q = x * r
|
|
return fma(fma(-q, d, x), r, q)
|
|
|
|
|
|
def permlane_swap(off, x, y):
|
|
swap = rocdl.permlane32_swap if off == 32 else rocdl.permlane16_swap
|
|
pr = swap(
|
|
llvm.StructType.get_literal([T.i32, T.i32]),
|
|
as_ir_value(fx.Int32(x)),
|
|
as_ir_value(fx.Int32(y)),
|
|
False,
|
|
False,
|
|
)
|
|
return tuple(fx.Int32(llvm.extractvalue(T.i32, pr, [j])) for j in range(2))
|
|
|
|
|
|
def lane_gather(v, src_lane):
|
|
"""Lane ``src_lane``'s value of v (ds_bpermute)."""
|
|
return fx.Int32(
|
|
rocdl.ds_bpermute(T.i32, fx.Int32(src_lane) * 4, fx.Int32(v).ir_value())
|
|
)
|
|
|
|
|
|
def lane_f32(v, src_lane):
|
|
return lane_gather(v.bitcast(fx.Int32), src_lane).bitcast(fx.Float32)
|
|
|
|
|
|
def readlane(v, src_lane):
|
|
bits = v.bitcast(fx.Int32) if isinstance(v, fx.Float32) else fx.Int32(v)
|
|
r = fx.Int32(rocdl.readlane(T.i32, bits, fx.Int32(src_lane)))
|
|
return r.bitcast(fx.Float32) if isinstance(v, fx.Float32) else r
|
|
|
|
|
|
def xshfl(v, off):
|
|
"""Value of lane ``lane ^ off`` (DPP in a row, permlane swaps across)."""
|
|
assert off in (1, 2, 4, 8, 16, 32), off
|
|
if off >= 16:
|
|
return v.shuffle_xor(off, 64)
|
|
is_f = isinstance(v, fx.Float32)
|
|
x = v.bitcast(fx.Int32) if is_f else fx.Int32(v)
|
|
if off == 8:
|
|
y = fx.Int32(update_dpp_i32(x, x, 0x118, 0xF, 0xC, False))
|
|
y = fx.Int32(update_dpp_i32(y, x, 0x108, 0xF, 0x3, False))
|
|
elif off == 4:
|
|
y = fx.Int32(update_dpp_i32(x, x, 0x114, 0xF, 0xA, False))
|
|
y = fx.Int32(update_dpp_i32(y, x, 0x104, 0xF, 0x5, False))
|
|
elif off == 2:
|
|
y = fx.Int32(update_dpp_i32(x, x, 0x4E, 0xF, 0xF, False))
|
|
else:
|
|
y = fx.Int32(update_dpp_i32(x, x, 0xB1, 0xF, 0xF, False))
|
|
return y.bitcast(fx.Float32) if is_f else y
|
|
|
|
|
|
def xred(v, off, op):
|
|
if off < 16:
|
|
return op(v, xshfl(v, off))
|
|
is_f = isinstance(v, fx.Float32)
|
|
x = v.bitcast(fx.Int32) if is_f else fx.Int32(v)
|
|
a, b = permlane_swap(off, x, x)
|
|
if is_f:
|
|
return op(a.bitcast(fx.Float32), b.bitcast(fx.Float32))
|
|
return op(type(v)(a), type(v)(b))
|
|
|
|
|
|
def _add(a, b):
|
|
return a + b
|
|
|
|
|
|
def butterfly(v, offsets, op=_add):
|
|
"""Butterfly reduction over the xor ``offsets``, in that order."""
|
|
for off in offsets:
|
|
v = xred(v, off, op)
|
|
return v
|
|
|
|
|
|
def wave_sum(v):
|
|
return butterfly(v, (32, 16, 8, 4, 2, 1))
|
|
|
|
|
|
def wave_max(v):
|
|
return butterfly(v, (32, 16, 8, 4, 2, 1), fx.max)
|
|
|
|
|
|
def row_shr(v, lane, k):
|
|
"""Lane - k of the same 16-lane row, else 0 (DPP row_shr)."""
|
|
moved = lane_gather(v.bitcast(fx.Int32), fx.max(lane - k, 0)).bitcast(fx.Float32)
|
|
return fx.Float32((lane % 16 >= k).select(moved, fx.Float32(0.0)))
|
|
|
|
|
|
# ----------------------------------------------------------------------------- packing
|
|
|
|
|
|
def bf16_round(a):
|
|
return fx.Float32(fx.Float32(a).to(fx.BFloat16))
|
|
|
|
|
|
def bf16_pair(a, b):
|
|
"""Two f32 -> one f32-typed word holding (bf16(a), bf16(b))."""
|
|
return (
|
|
fx.Vector.from_elements([a, b], fx.Float32)
|
|
.to(fx.BFloat16)
|
|
.bitcast(fx.Float32)[0]
|
|
)
|
|
|
|
|
|
def bf_lo(w):
|
|
return (w << 16).bitcast(fx.Float32)
|
|
|
|
|
|
def bf_hi(w):
|
|
return (w & fx.Int32(-65536)).bitcast(fx.Float32)
|
|
|
|
|
|
def fp8_pack4(a, b, c, d):
|
|
"""Four f32 (in range) -> one dword of four E4M3 bytes, RNE."""
|
|
lo = fx.Int32(rocdl.cvt_pk_fp8_f32(T.i32, a, b, fx.Int32(0), False))
|
|
return fx.Int32(rocdl.cvt_pk_fp8_f32(T.i32, c, d, lo, True))
|
|
|
|
|
|
def fp8x8_bf16(w0, w1, scale):
|
|
"""Eight E4M3 of two dwords times an f32 ``scale`` -> eight bf16 as a
|
|
4-dword Vector (exact for a power-of-two scale in range)."""
|
|
out = []
|
|
for w in (w0, w1):
|
|
for hi in (False, True):
|
|
p = rocdl.cvt_scalef32_pk_bf16_fp8(
|
|
T.bf16x2, as_ir_value(fx.Int32(w)), as_ir_value(fx.Float32(scale)), hi
|
|
)
|
|
out.append(fx.Vector(p).bitcast(fx.Int32)[0])
|
|
return fx.Vector.from_elements(out, fx.Int32)
|
|
|
|
|
|
def clamp_fp8(v):
|
|
return fx.max(fx.min(v, fx.Float32(FP8_MAX)), fx.Float32(-FP8_MAX))
|
|
|
|
|
|
def pow2(code):
|
|
"""The fp32 power of two a biased exponent (1..254) stands for."""
|
|
return (code << 23).bitcast(fx.Float32)
|
|
|
|
|
|
def rcp_pow2(code):
|
|
"""1 / ``pow2(code)``, exact for code in [1, 253]."""
|
|
return ((254 - code) << 23).bitcast(fx.Float32)
|
|
|
|
|
|
def ceil_exp(v):
|
|
"""Biased exponent of the smallest power of two >= v (v > 0 finite, normal)."""
|
|
bits = v.bitcast(fx.Int32)
|
|
return ((bits >> 23) & 0xFF) + ((bits & 0x7FFFFF) != 0).select(
|
|
fx.Int32(1), fx.Int32(0)
|
|
)
|
|
|
|
|
|
def mx_code(amax):
|
|
"""VLLM's ROCm MXFP8 scale code of a 32-group: clamp(ceil(log2(amax / 448))
|
|
+ 127, 0, 254), amax floored at FLT_MIN."""
|
|
a = fx.max(amax, fx.Float32(FLT_MIN))
|
|
q = div_rn(a, fx.Float32(FP8_MAX), fx.Float32(1.0 / FP8_MAX))
|
|
return fx.min(fx.max(ceil_exp(q), fx.Int32(0)), fx.Int32(254))
|
|
|
|
|
|
def mx_mul(code):
|
|
"""2^(127 - code) as fp32, for code in [0, 254] (the quant multiplier)."""
|
|
return ((254 - code) << 23).bitcast(fx.Float32)
|
|
|
|
|
|
# ----------------------------------------------------------------------------- MFMA
|
|
|
|
|
|
def mfma_scaled(a, b, c, sa, sb, a_fmt=FP8, b_fmt=FP8):
|
|
"""16x16x128 scaled MFMA (gfx950 f8f6f4): lane l (row l % 16) holds K bytes
|
|
16 (l // 16) .. +16 and 64 + 16 (l // 16) .. +16 of the 128-step, and the
|
|
E8M0 of K block l // 16."""
|
|
return fx.Vector(
|
|
rocdl.mfma_scale_f32_16x16x128_f8f6f4(
|
|
T.vec(4, T.f32), [a, b, c, a_fmt, b_fmt, 0, sa, 0, sb]
|
|
)
|
|
)
|
|
|
|
|
|
def mfma_bf16(a, b, c):
|
|
"""16x16x32 bf16 MFMA: lane l (row / col l % 16) holds K 8 (l // 16) .. +8."""
|
|
return fx.Vector(rocdl.mfma_f32_16x16x32_bf16(T.vec(4, T.f32), [a, b, c]))
|
|
|
|
|
|
# ----------------------------------------------------------------------------- mailbox
|
|
|
|
|
|
@traced
|
|
def _spin(load_all, pending):
|
|
v = load_all()
|
|
while pending(v):
|
|
rocdl.s_sleep(1)
|
|
v = load_all()
|
|
return v
|
|
|
|
|
|
def spin_until(load, pending):
|
|
"""``load()`` again (s_sleep between) while ``pending(value)``; the value."""
|
|
return _spin(load, pending)
|
|
|
|
|
|
class Mailbox:
|
|
"""Tagged-pair hand-off between the CTAs of one launch (device scope).
|
|
|
|
Every 32-bit value is stored next to the launch's tag in one 8-byte store; a
|
|
consumer polls until every tag matches, so a hand-off is one round trip with
|
|
no separate flag. The tag is ``epoch * 64 + layer + 1`` (``epoch`` a device
|
|
counter bumped once a step): a pair from an earlier layer or step never
|
|
carries a live tag, and nothing is cleared between launches."""
|
|
|
|
def __init__(self, tag):
|
|
self.tag = tag
|
|
|
|
def put(self, base, i, v):
|
|
bits = v.bitcast(fx.Int32) if isinstance(v, fx.Float32) else fx.Int32(v)
|
|
bo.buffer_store(
|
|
fx.Vector.from_elements([bits, self.tag], fx.Int32),
|
|
rsrc(base),
|
|
i * 2,
|
|
cache_modifier=CM_DEV,
|
|
)
|
|
|
|
def put_bf(self, base, i, vs):
|
|
"""Elements i .. i + len(vs) (2 or 4, i aligned) as bf16 pairs, one store."""
|
|
words = []
|
|
for j in range_constexpr(len(vs) // 2):
|
|
words += [bf16_pair(vs[2 * j], vs[2 * j + 1]).bitcast(fx.Int32), self.tag]
|
|
bo.buffer_store(
|
|
fx.Vector.from_elements(words, fx.Int32),
|
|
rsrc(base),
|
|
i,
|
|
cache_modifier=CM_DEV,
|
|
)
|
|
|
|
def put_words(self, base, i, words):
|
|
"""Raw 32-bit words (1 or 2) at pairs i.. (fp8 words must come this way)."""
|
|
vals = []
|
|
for w in words:
|
|
vals += [fx.Int32(w), self.tag]
|
|
bo.buffer_store(
|
|
fx.Vector.from_elements(vals, fx.Int32),
|
|
rsrc(base),
|
|
i * 2,
|
|
cache_modifier=CM_DEV,
|
|
)
|
|
|
|
def poll(self, specs, batch=POLL_MAX):
|
|
"""Batched poll of ``specs`` = [(base, pair index, npairs in {1, 2})]: all
|
|
re-loaded while any tag is stale. Returns the value words."""
|
|
if const_expr(len(specs) == 0):
|
|
return []
|
|
if const_expr(len(specs) > batch):
|
|
return self.poll(specs[:batch], batch) + self.poll(specs[batch:], batch)
|
|
tag = self.tag
|
|
|
|
def load_all():
|
|
words = []
|
|
for b, i, n in specs:
|
|
w = fx.Vector(
|
|
bo.buffer_load(
|
|
rsrc(b),
|
|
fx.Int32(i) * 2,
|
|
vec_width=2 * n,
|
|
dtype=T.i32,
|
|
cache_modifier=CM_DEV,
|
|
)
|
|
)
|
|
words += [w[e] for e in range(2 * n)]
|
|
return fx.Vector.from_elements(words, fx.Int32)
|
|
|
|
nw = sum(2 * n for _, _, n in specs)
|
|
|
|
def pending(v):
|
|
bad = v[1] != tag
|
|
for e in range_constexpr(3, nw, 2):
|
|
bad = bad | (v[e] != tag)
|
|
return bad
|
|
|
|
v = _spin(load_all, pending)
|
|
outs, e = [], 0
|
|
for _, _, n in specs:
|
|
outs.append([v[e + 2 * q] for q in range(n)])
|
|
e += 2 * n
|
|
return outs
|
|
|
|
|
|
LOAD_BATCH = 4
|
|
|
|
|
|
@traced
|
|
def batched_rounds(tid, n, load, use, batch=LOAD_BATCH):
|
|
"""A CTA's strided loop over v = tid + THREADS i < n: ``batch`` rounds' loads
|
|
issued before their uses; a round past n loads element n - 1, uses nothing."""
|
|
rounds = -(-n // THREADS)
|
|
for b0 in range_constexpr(0, rounds, batch):
|
|
idx = range(b0, min(b0 + batch, rounds))
|
|
got = [load(fx.min(tid + THREADS * i, n - 1)) for i in idx]
|
|
for i, g in zip(idx, got):
|
|
v = tid + THREADS * i
|
|
if v < n:
|
|
use(v, g)
|
|
|
|
|
|
def barrier():
|
|
gpu.barrier()
|
|
|
|
|
|
# ----------------------------------------------------------------------------- timeline
|
|
|
|
|
|
def memrealtime():
|
|
"""s_memrealtime: the 100 MHz device clock."""
|
|
return fx.Int64(llvm.call_intrinsic(T.i64, "llvm.amdgcn.s.memrealtime", [], [], []))
|
|
|
|
|
|
@traced
|
|
def stamp(c, point):
|
|
"""A timeline build's stamp: thread 0 of the CTA stores the clock at
|
|
``point`` (``c["tl"]`` [BLOCKS, points] i64; None: not a timeline build)."""
|
|
if const_expr(c.get("tl") is None):
|
|
return
|
|
if c["tid"] == 0:
|
|
t = memrealtime()
|
|
w = fx.Vector.from_elements([fx.Int32(t), fx.Int32(t >> 32)], fx.Int32)
|
|
bo.buffer_store(w, rsrc(c["tl"]), (c["bid"] * c["tl_points"] + point) * 2)
|