Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1019 lines
28 KiB
Python
1019 lines
28 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""The mono decode layer's two launches (README): K1 = the attention seam
|
|
(``stages.seam``: slice, gate, the norm into vLLM's MXFP8) + the attention front
|
|
(``attention.front``); K2 = the attention back (``attention.back``, wo_b pushed
|
|
to every rank) + the FFN seam (the TP sum folded in) + the MoE (``stages.moe``,
|
|
its all-reduce in-kernel too). A layer whose attention vLLM runs takes the FFN
|
|
launch instead: K2 with the attention back replaced by a push of vLLM's
|
|
unreduced wo_b output (``seam.stage_push``).
|
|
|
|
Tags: K1's hand-offs carry 2 e + 1, K2's 2 e + 2 (e: the launch pair's epoch,
|
|
moved on by K2's CTA 0 at its end), so K2's seam may reuse K1's seam regions.
|
|
Peer regions alternate by e's parity; every rank runs the same launches, so
|
|
the epochs agree across ranks.
|
|
"""
|
|
|
|
from dataclasses import dataclass
|
|
|
|
import flydsl.compiler as flyc
|
|
import flydsl.expr as fx
|
|
from aiter.ops.flydsl.kernels import buffer_ops as bo
|
|
from flydsl.expr import const_expr, gpu, range_constexpr, rocdl
|
|
from flydsl.expr.typing import Int32, Int64, T
|
|
|
|
from .attention import back as mb_back
|
|
from .attention import front as mb_front
|
|
from .attention.device import BLOCKS, CM_DEV, THREADS, gstore, kernel_symbol, rsrc
|
|
from .attention.plan import HIDDEN, Dims, back_scratch, front_scratch
|
|
from .common.mx import clamp_fp8
|
|
from .common.ops import butterfly, fp8_pack4, peer_bases, traced
|
|
from .common.plan import WAVES, first_task, key_tuple
|
|
from .common.sync import Mailbox, preg, publish, sreg
|
|
from .sources import SOURCES
|
|
from .stages import moe, seam
|
|
from .stages.dims import Dims as StageDims
|
|
from .stages.moe_shape import SORT_NETS, MoeBuild, route_shape
|
|
|
|
_STREAM = fx.Stream(None)
|
|
MAX_TOKENS = 48
|
|
X_WORDS = HIDDEN // 4
|
|
X_GROUPS = HIDDEN // 32
|
|
SLICES = seam.SLICES # 160
|
|
# K1's MXFP8 x and the MoE's: plain words in both, the same size, so the two
|
|
# launches share the MoE's (placed later); any other name is one region, so a
|
|
# word is never a tag to one launch and data to another
|
|
SHARED_REGIONS = ("x8", "x8s")
|
|
COUNTER_WORDS = 2 * 256 # the MoE's ug queue and down counts, a word a slot
|
|
COUNTERS = ("ugq", "dq")
|
|
# the epoch buffer, in words: the epoch, a mark a CTA (back.EPOCH_MARKS), then on
|
|
# lines of their own the MoE's counters
|
|
EPOCH_COUNTERS = -(-(mb_back.EPOCH_MARKS + BLOCKS) // 64) * 64
|
|
EPOCH_WORDS = EPOCH_COUNTERS + COUNTER_WORDS
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class MonoBuild:
|
|
tokens: int
|
|
tp: int
|
|
ratio: int # the layer's compress ratio (its attention's key sources)
|
|
timeline: bool = False
|
|
# K1's seam folds the previous layer's output into the residual; off for a
|
|
# residual that arrives settled (the first layer, after an Engram)
|
|
fold: bool = True
|
|
# the layer selects its own top-k (an index source): K1 hands its normed
|
|
# input and qr out for vLLM's indexer, K2 builds the key table
|
|
index: bool = False
|
|
|
|
|
|
def _moe_key(key: MonoBuild) -> MoeBuild:
|
|
return MoeBuild(tokens=key.tokens, tp=key.tp)
|
|
|
|
|
|
def _front_key(key: MonoBuild):
|
|
return mb_front.FrontBuild(key.tokens, key.tp, key.ratio, key.timeline)
|
|
|
|
|
|
def _back_key(key: MonoBuild):
|
|
return mb_back.BackBuild(key.tokens, key.tp, key.ratio, key.timeline)
|
|
|
|
|
|
def scratch_layout(s: int, tp: int) -> dict:
|
|
"""Every region of step width s's launches -> (byte offset, bytes), disjoint:
|
|
K1's front and seam, K2's back, the MoE's regions, the normed rows and their
|
|
flags. Each width has a scratch of its own (``DSV41MonoLayer.scratch``):
|
|
under another width's layout a mailbox pair's tag word could be plain data,
|
|
which a poll may take for a current tag. The MoE's counters, whose slots
|
|
outlive a step width, are in the epoch buffer."""
|
|
d = Dims(tp)
|
|
out: dict = {}
|
|
off = 0
|
|
|
|
def place(regions):
|
|
nonlocal off
|
|
base = off
|
|
for name, (o, n) in regions.items():
|
|
assert name not in out or name in SHARED_REGIONS, name
|
|
out[name] = (base + o, n)
|
|
off = max(off, base + max(o + n for o, n in regions.values()))
|
|
off = -(-off // 256) * 256
|
|
|
|
place(front_scratch(s, d))
|
|
place(seam.scratch_layout(s))
|
|
place(back_scratch(s, d, start=0))
|
|
regions = moe.scratch_layout(_moe_key(MonoBuild(s, tp, 1))).items()
|
|
mr = {n: v for n, v in regions if n not in COUNTERS}
|
|
lo = min(o for o, _ in mr.values())
|
|
place({n: (o - lo, n_) for n, (o, n_) in mr.items()})
|
|
place({"normed": (0, s * HIDDEN * 2), "xrdy_moe": (s * HIDDEN * 2 + 256, s * 8)})
|
|
place({"krdy": (0, s * 8)})
|
|
return out
|
|
|
|
|
|
def scratch_bytes(s: int = MAX_TOKENS, tp: int = 2) -> int:
|
|
return max(o + n for o, n in scratch_layout(s, tp).values())
|
|
|
|
|
|
def peer_half_bytes(tp: int) -> int:
|
|
"""One parity's peer regions: the attention's partials, then the MoE's."""
|
|
return seam.attn_peer_bytes(MAX_TOKENS, tp) + moe.peer_bytes(MAX_TOKENS, tp)
|
|
|
|
|
|
def _epoch(epoch):
|
|
return fx.Int32(bo.buffer_load(rsrc(epoch), 0, vec_width=1, dtype=T.i32))
|
|
|
|
|
|
# ---------------------------------------------------------------- K1
|
|
|
|
|
|
@traced
|
|
def store_x8(c, t, col, ys, live):
|
|
"""The attention seam's norm output (``seam.stage_norm``'s 8 columns at
|
|
``col``) -> vLLM's MXFP8 of each 32 (the attention's input quant) -> the
|
|
front's X8 / X8S, plain at device scope."""
|
|
amax = abs(ys[0])
|
|
for y in ys[1:]:
|
|
amax = fx.max(amax, abs(y))
|
|
code = moe.vllm_mx_code(butterfly(amax, (1, 2), fx.max))
|
|
mul = ((254 - code) << 23).bitcast(fx.Float32)
|
|
w0 = fp8_pack4(*[clamp_fp8(ys[i] * mul) for i in range(4)])
|
|
w1 = fp8_pack4(*[clamp_fp8(ys[4 + i] * mul) for i in range(4)])
|
|
if live:
|
|
bo.buffer_store(
|
|
fx.Vector.from_elements([w0, w1], fx.Int32),
|
|
rsrc(c["x8"]),
|
|
t * X_WORDS + col // 4,
|
|
cache_modifier=CM_DEV,
|
|
)
|
|
if c["tid"] % 4 == 0:
|
|
bo.buffer_store(
|
|
code, rsrc(c["x8s"]), t * X_GROUPS + col // 32, cache_modifier=CM_DEV
|
|
)
|
|
|
|
|
|
@traced
|
|
def publish_x8(c, t):
|
|
"""Token t's two XRDY flags (one a wqkv K half) once its X8 / X8S landed."""
|
|
rocdl.s_waitcnt(vmcnt=0)
|
|
gpu.barrier()
|
|
if c["tid"] < 2:
|
|
c["mb"].put(c["xrdy"], 2 * t + c["tid"], fx.Int32(1))
|
|
gpu.barrier()
|
|
|
|
|
|
def build_mono_k1(key: MonoBuild):
|
|
s, tp = key.tokens, key.tp
|
|
assert 1 <= s <= MAX_TOKENS
|
|
layout = scratch_layout(s, tp)
|
|
fkey = _front_key(key)
|
|
FrontLds = mb_front.front_lds(s, Dims(tp))
|
|
NORM0 = SLICES # the norms on the CTAs past the slices: the wqkv waits on them
|
|
GATE0 = SLICES + s
|
|
assert GATE0 + s <= BLOCKS
|
|
|
|
@fx.struct
|
|
class SeamLds:
|
|
red: fx.Array[fx.Float32, WAVES * 2 * 64 * 4, 16]
|
|
rl: fx.Array[fx.Float32, s * seam.KT, 16]
|
|
fl: fx.Array[fx.Float32, seam.MIX * seam.KT, 16]
|
|
|
|
@fx.union
|
|
class K1Lds:
|
|
seam: SeamLds
|
|
front: FrontLds # type: ignore[valid-type]
|
|
|
|
name = kernel_symbol(
|
|
"dsv41_mono_k1",
|
|
s=s,
|
|
tp=tp,
|
|
r=key.ratio,
|
|
tl=key.timeline,
|
|
f=key.fold,
|
|
i=key.index,
|
|
)
|
|
keyed = key_tuple(key, SOURCES)
|
|
|
|
@flyc.kernel(name=name, known_block_size=[THREADS, 1, 1])
|
|
def k1(
|
|
res_in: Int64,
|
|
pend: Int64,
|
|
post_in: Int64,
|
|
comb_in: Int64,
|
|
pre_in: Int64,
|
|
hc_fn: Int64,
|
|
hc_scale: Int64,
|
|
hc_base: Int64,
|
|
attn_w: Int64,
|
|
res_out: Int64,
|
|
post_out: Int64,
|
|
comb_out: Int64,
|
|
pre_out: Int64,
|
|
wqkv: Int64,
|
|
wqkv_s: Int64,
|
|
qn_w: Int64,
|
|
kvn_w: Int64,
|
|
wqb: Int64,
|
|
wqb_s: Int64,
|
|
cos_sin: Int64,
|
|
pos: Int64,
|
|
slot: Int64,
|
|
swa: Int64,
|
|
swa_stride: Int64,
|
|
swa_block: Int32,
|
|
q: Int64,
|
|
swa_idx: Int64,
|
|
swa_lens: Int64,
|
|
comp_block: Int32,
|
|
comp_bt: Int64,
|
|
bt_stride: Int32,
|
|
topk: Int64,
|
|
t2r: Int64,
|
|
kt: Int64,
|
|
klen: Int64,
|
|
normed_out: Int64,
|
|
qr_out: Int64,
|
|
scratch: Int64,
|
|
epoch: Int64,
|
|
tl: Int64,
|
|
):
|
|
_ = keyed
|
|
bid = fx.block_idx.x
|
|
lds = fx.SharedAllocator().allocate(K1Lds)
|
|
sl = lds.seam.peek()
|
|
ep = _epoch(epoch)
|
|
tag = (ep << 1) + 1
|
|
fargs = {
|
|
"x": 0,
|
|
"x_stride": 0,
|
|
"wqkv": wqkv,
|
|
"wqkv_s": wqkv_s,
|
|
"qn_w": qn_w,
|
|
"kvn_w": kvn_w,
|
|
"wqb": wqb,
|
|
"wqb_s": wqb_s,
|
|
"cos_sin": cos_sin,
|
|
"pos": pos,
|
|
"slot": slot,
|
|
"swa": swa,
|
|
"swa_stride": swa_stride,
|
|
"swa_block": swa_block,
|
|
"q": q,
|
|
"swa_idx": swa_idx,
|
|
"swa_lens": swa_lens,
|
|
"comp_block": comp_block,
|
|
"comp_bt": comp_bt,
|
|
"bt_stride": bt_stride,
|
|
"topk": topk,
|
|
"t2r": t2r,
|
|
"kt": kt,
|
|
"klen": klen,
|
|
"qr_out": qr_out,
|
|
}
|
|
flayout = {n: layout[n] for n in front_scratch(s, Dims(tp))}
|
|
c = mb_front.front_context(fkey, lds.front.peek(), fargs, scratch, tag, flayout)
|
|
c["index"] = key.index
|
|
amb = Mailbox(tag)
|
|
tid = fx.thread_idx.x
|
|
ca = {
|
|
"S": s,
|
|
"tid": tid,
|
|
"bid": bid,
|
|
"lane": tid % 64,
|
|
"wave": tid // 64,
|
|
"rl": sl.rl.ptr,
|
|
"fl": sl.fl.ptr,
|
|
"red": sl.red.ptr,
|
|
"put": amb.put,
|
|
"put_bf": amb.put_bf,
|
|
"put_words": amb.put_words,
|
|
"poll": amb.poll,
|
|
"args": {
|
|
"res_in": res_in,
|
|
"pend": pend,
|
|
"post_in": post_in,
|
|
"comb_in": comb_in,
|
|
"pre_in": pre_in,
|
|
"hc_fn": hc_fn,
|
|
"hc_scale": hc_scale,
|
|
"hc_base": hc_base,
|
|
"norm_w": attn_w,
|
|
"res_out": res_out,
|
|
"post_out": post_out,
|
|
"comb_out": comb_out,
|
|
"pre_out": pre_out,
|
|
},
|
|
"d": StageDims(tp),
|
|
"fold": key.fold,
|
|
}
|
|
for region in ("lin", "pmix"):
|
|
ca[region] = sreg(scratch, layout[region][0], region)
|
|
if const_expr(key.timeline):
|
|
c["tl"], c["tl_points"] = tl, mb_front.FRONT_POINTS
|
|
|
|
def seam_slice():
|
|
for task in range(first_task(bid, 0), SLICES, BLOCKS):
|
|
seam.stage_slice(ca, task)
|
|
gpu.barrier()
|
|
|
|
def store(*v):
|
|
store_x8(c, *v)
|
|
if const_expr(key.index):
|
|
store_normed(normed_out, *v)
|
|
|
|
def seam_rest():
|
|
for t in range(first_task(bid, NORM0), s, BLOCKS):
|
|
seam.stage_norm(ca, t, store)
|
|
publish_x8(c, t)
|
|
for t in range(first_task(bid, GATE0), s, BLOCKS):
|
|
seam.stage_gate(ca, t)
|
|
gpu.barrier()
|
|
|
|
mb_front.run_front(
|
|
c, fkey, bid, x8_given=True, before_wqkv=seam_slice, after_wqkv=seam_rest
|
|
)
|
|
|
|
@flyc.jit
|
|
def launch(
|
|
res_in: Int64,
|
|
pend: Int64,
|
|
post_in: Int64,
|
|
comb_in: Int64,
|
|
pre_in: Int64,
|
|
hc_fn: Int64,
|
|
hc_scale: Int64,
|
|
hc_base: Int64,
|
|
attn_w: Int64,
|
|
res_out: Int64,
|
|
post_out: Int64,
|
|
comb_out: Int64,
|
|
pre_out: Int64,
|
|
wqkv: Int64,
|
|
wqkv_s: Int64,
|
|
qn_w: Int64,
|
|
kvn_w: Int64,
|
|
wqb: Int64,
|
|
wqb_s: Int64,
|
|
cos_sin: Int64,
|
|
pos: Int64,
|
|
slot: Int64,
|
|
swa: Int64,
|
|
swa_stride: Int64,
|
|
swa_block: Int32,
|
|
q: Int64,
|
|
swa_idx: Int64,
|
|
swa_lens: Int64,
|
|
comp_block: Int32,
|
|
comp_bt: Int64,
|
|
bt_stride: Int32,
|
|
topk: Int64,
|
|
t2r: Int64,
|
|
kt: Int64,
|
|
klen: Int64,
|
|
normed_out: Int64,
|
|
qr_out: Int64,
|
|
scratch: Int64,
|
|
epoch: Int64,
|
|
tl: Int64,
|
|
stream: fx.Stream = _STREAM,
|
|
):
|
|
_ = keyed
|
|
k1(
|
|
res_in,
|
|
pend,
|
|
post_in,
|
|
comb_in,
|
|
pre_in,
|
|
hc_fn,
|
|
hc_scale,
|
|
hc_base,
|
|
attn_w,
|
|
res_out,
|
|
post_out,
|
|
comb_out,
|
|
pre_out,
|
|
wqkv,
|
|
wqkv_s,
|
|
qn_w,
|
|
kvn_w,
|
|
wqb,
|
|
wqb_s,
|
|
cos_sin,
|
|
pos,
|
|
slot,
|
|
swa,
|
|
swa_stride,
|
|
swa_block,
|
|
q,
|
|
swa_idx,
|
|
swa_lens,
|
|
comp_block,
|
|
comp_bt,
|
|
bt_stride,
|
|
topk,
|
|
t2r,
|
|
kt,
|
|
klen,
|
|
normed_out,
|
|
qr_out,
|
|
scratch,
|
|
epoch,
|
|
tl,
|
|
).launch(grid=(BLOCKS,), block=(THREADS,), stream=stream)
|
|
|
|
return launch
|
|
|
|
|
|
# ---------------------------------------------------------------- K2
|
|
|
|
|
|
@traced
|
|
def store_normed(normed, t, col, ys, live):
|
|
"""The FFN seam's norm output -> the MoE input row (bf16, device scope:
|
|
this launch reads it)."""
|
|
if live:
|
|
for j in range_constexpr(8):
|
|
bo.buffer_store(
|
|
ys[j].to(fx.BFloat16),
|
|
rsrc(normed),
|
|
t * HIDDEN + col + j,
|
|
cache_modifier=CM_DEV,
|
|
)
|
|
|
|
|
|
def _seam_lds(s):
|
|
@fx.struct
|
|
class SeamLds:
|
|
red: fx.Array[fx.Float32, WAVES * 2 * 64 * 4, 16]
|
|
rl: fx.Array[fx.Float32, s * seam.KT, 16]
|
|
fl: fx.Array[fx.Float32, seam.MIX * seam.KT, 16]
|
|
pl: fx.Array[fx.Float32, s * seam.COLS, 16]
|
|
|
|
return SeamLds
|
|
|
|
|
|
@traced
|
|
def run_ffn(key: MonoBuild, lds, a: dict, amb, bases, own, rank, ep, part=None):
|
|
"""K2's FFN half on this CTA: the FFN seam (the attention's TP partials --
|
|
``part`` pushed to every rank's ATTN region first -- summed and folded into
|
|
the residual), the MoE and its all-reduce into ``a["out"]``, then the
|
|
launch pair's epoch moved on."""
|
|
s, tp = key.tokens, key.tp
|
|
layout = scratch_layout(s, tp)
|
|
mkey = _moe_key(key)
|
|
rs = route_shape(mkey)
|
|
GATE0 = SLICES
|
|
NORM0 = SLICES + s
|
|
bid = fx.block_idx.x
|
|
tid = fx.thread_idx.x
|
|
sl = lds.seam.peek()
|
|
scratch = a["scratch"]
|
|
seam_args = (
|
|
"res_in",
|
|
"post_in",
|
|
"comb_in",
|
|
"pre_in",
|
|
"hc_fn",
|
|
"hc_scale",
|
|
"hc_base",
|
|
"res_out",
|
|
"post_out",
|
|
"comb_out",
|
|
"pre_out",
|
|
)
|
|
ca = {
|
|
"S": s,
|
|
"tid": tid,
|
|
"bid": bid,
|
|
"lane": tid % 64,
|
|
"wave": tid // 64,
|
|
"rl": sl.rl.ptr,
|
|
"fl": sl.fl.ptr,
|
|
"red": sl.red.ptr,
|
|
"pend_lds": sl.pl.ptr,
|
|
"put": amb.put,
|
|
"put_bf": amb.put_bf,
|
|
"put_words": amb.put_words,
|
|
"poll": amb.poll,
|
|
"peer_addr": lambda p: bases[p],
|
|
"rank": rank,
|
|
"sym": own,
|
|
"args": {n: a[n] for n in seam_args} | {"norm_w": a["ffn_w"]},
|
|
"d": StageDims(tp),
|
|
}
|
|
normed = scratch + fx.Int64(layout["normed"][0])
|
|
for region in ("lin", "pmix"):
|
|
ca[region] = sreg(scratch, layout[region][0], region)
|
|
mlayout = {n: layout[n] for n in moe.scratch_layout(mkey) if n not in COUNTERS}
|
|
mlayout["xrdy"] = layout["xrdy_moe"]
|
|
margs = moe.moe_args(
|
|
normed,
|
|
a["gate_w"],
|
|
a["bias"],
|
|
a["w13"],
|
|
a["w13_s"],
|
|
a["w2"],
|
|
a["w2_s"],
|
|
a["sgu"],
|
|
a["sgu_s"],
|
|
a["sw2"],
|
|
a["sw2_s"],
|
|
a["out"],
|
|
)
|
|
cb = moe.moe_context(
|
|
s, rs, lds.moe.peek(), amb, bases, mlayout, scratch, margs, rank, own
|
|
)
|
|
cb["x_cm"], cb["x_ready"] = CM_DEV, True
|
|
cb["tag"] = ep & 255
|
|
for i, region in enumerate(COUNTERS):
|
|
cb[region] = sreg(a["epoch"], 4 * (EPOCH_COUNTERS + i * moe.TAGS), region)
|
|
for task in range(first_task(bid, 0), SLICES, BLOCKS):
|
|
if const_expr(part is not None):
|
|
seam.stage_push(ca, task, part)
|
|
seam.stage_reduce(ca, task)
|
|
seam.stage_slice(ca, task)
|
|
for t in range(first_task(bid, GATE0), s, BLOCKS):
|
|
seam.stage_gate(ca, t)
|
|
for t in range(first_task(bid, NORM0), s, BLOCKS):
|
|
seam.stage_norm(ca, t, lambda *v: store_normed(normed, *v))
|
|
publish(cb["put"], cb["xrdy"], t, 1, tid == 0)
|
|
gpu.barrier()
|
|
|
|
# ---- the MoE, its all-reduce into ``out``
|
|
moe.run_moe(cb, mkey, bid, 0, 0)
|
|
|
|
def reset():
|
|
# the MoE counter slot 128 launch pairs ahead: its last use long
|
|
# done, its next use far off
|
|
slot = (ep + 128) & 255
|
|
for region in COUNTERS:
|
|
gstore(cb[region].value + fx.Int64(slot * 4), fx.Int32(0), words=1)
|
|
|
|
mb_back.epoch_end({"bid": bid, "tid": tid}, a["epoch"], ep, reset)
|
|
|
|
|
|
def build_mono_k2(key: MonoBuild):
|
|
s, tp = key.tokens, key.tp
|
|
assert 1 <= s <= MAX_TOKENS
|
|
layout = scratch_layout(s, tp)
|
|
bkey = _back_key(key)
|
|
mkey = _moe_key(key)
|
|
rs = route_shape(mkey)
|
|
assert rs.experts // 64 in SORT_NETS
|
|
BackM = mb_back.back_lds_members(s, Dims(tp))
|
|
SplitLds, GemvLds = BackM["split"], BackM["gemv"]
|
|
MoeLds = moe.moe_smem(s, rs, False)
|
|
half = peer_half_bytes(tp)
|
|
assert SLICES + 2 * s <= BLOCKS
|
|
SeamLds = _seam_lds(s)
|
|
|
|
@fx.union
|
|
class K2Lds:
|
|
split: SplitLds # type: ignore[valid-type]
|
|
gemv: GemvLds # type: ignore[valid-type]
|
|
seam: SeamLds # type: ignore[valid-type]
|
|
moe: MoeLds # type: ignore[valid-type]
|
|
|
|
name = kernel_symbol(
|
|
"dsv41_mono_k2", s=s, tp=tp, r=key.ratio, tl=key.timeline, i=key.index
|
|
)
|
|
keyed = key_tuple(key, SOURCES)
|
|
|
|
@flyc.kernel(name=name, known_block_size=[THREADS, 1, 1])
|
|
def k2(
|
|
q: Int64,
|
|
swa: Int64,
|
|
swa_stride: Int64,
|
|
swa_block: Int32,
|
|
comp: Int64,
|
|
comp_stride: Int64,
|
|
comp_block: Int32,
|
|
kt: Int64,
|
|
klen: Int64,
|
|
slot: Int64,
|
|
swa_idx: Int64,
|
|
swa_lens: Int64,
|
|
t2r: Int64,
|
|
topk: Int64,
|
|
comp_bt: Int64,
|
|
bt_stride: Int32,
|
|
pos: Int64,
|
|
sink: Int64,
|
|
qk_scale: Int32,
|
|
cos_sin: Int64,
|
|
woa: Int64,
|
|
woa_s: Int64,
|
|
wob: Int64,
|
|
wob_s: Int64,
|
|
zrec: Int64,
|
|
res_in: Int64,
|
|
post_in: Int64,
|
|
comb_in: Int64,
|
|
pre_in: Int64,
|
|
hc_fn: Int64,
|
|
hc_scale: Int64,
|
|
hc_base: Int64,
|
|
ffn_w: Int64,
|
|
res_out: Int64,
|
|
post_out: Int64,
|
|
comb_out: Int64,
|
|
pre_out: Int64,
|
|
gate_w: Int64,
|
|
bias: Int64,
|
|
w13: Int64,
|
|
w13_s: Int64,
|
|
w2: Int64,
|
|
w2_s: Int64,
|
|
sgu: Int64,
|
|
sgu_s: Int64,
|
|
sw2: Int64,
|
|
sw2_s: Int64,
|
|
out: Int64,
|
|
scratch: Int64,
|
|
sym: Int64,
|
|
peers: Int64,
|
|
rank: Int32,
|
|
epoch: Int64,
|
|
tl: Int64,
|
|
):
|
|
_ = keyed
|
|
bid = fx.block_idx.x
|
|
lds = fx.SharedAllocator().allocate(K2Lds)
|
|
ep = _epoch(epoch)
|
|
tag = (ep << 1) + 2
|
|
par = fx.Int64(ep & 1) * fx.Int64(half)
|
|
bases = [b + par for b in peer_bases(peers, tp)]
|
|
own = sym + par
|
|
bargs = {
|
|
"q": q,
|
|
"swa": swa,
|
|
"swa_stride": swa_stride,
|
|
"swa_block": swa_block,
|
|
"comp": comp,
|
|
"comp_stride": comp_stride,
|
|
"comp_block": comp_block,
|
|
"kt": kt,
|
|
"klen": klen,
|
|
"pos": pos,
|
|
"sink": sink,
|
|
"qk_scale": qk_scale,
|
|
"cos_sin": cos_sin,
|
|
"woa": woa,
|
|
"woa_s": woa_s,
|
|
"wob": wob,
|
|
"wob_s": wob_s,
|
|
"out": 0,
|
|
"out_stride": 0,
|
|
"zrec": zrec,
|
|
}
|
|
blayout = {n: layout[n] for n in back_scratch(s, Dims(tp), start=0)}
|
|
views = {"split": lds.split.peek(), "gemv": lds.gemv.peek()}
|
|
c = mb_back.back_context(bkey, views, bargs, scratch, tag, blayout)
|
|
amb = Mailbox(tag)
|
|
|
|
def push(t, col, v0, v1):
|
|
# this rank's bf16 partial pair to every rank's ATTN region
|
|
for p in range_constexpr(tp):
|
|
dst = preg(bases[p], 0, "attn")
|
|
amb.put_bf(dst, 2 * seam.attn_region_pair(rank, t, s, col), [v0, v1])
|
|
|
|
c["wob_out"] = push
|
|
if const_expr(key.timeline):
|
|
c["tl"], c["tl_points"] = tl, mb_back.BACK_POINTS
|
|
mb_back.epoch_begin(c, epoch, ep)
|
|
if const_expr(key.index):
|
|
# the key table, from the top-k vLLM's indexer selected after K1,
|
|
# on the last CTAs: each split task waits on its token's
|
|
ckt = {
|
|
"S": s,
|
|
"tid": c["tid"],
|
|
"dev": True,
|
|
"ratio": key.ratio,
|
|
"args": {
|
|
"slot": slot,
|
|
"swa_idx": swa_idx,
|
|
"swa_lens": swa_lens,
|
|
"pos": pos,
|
|
"t2r": t2r,
|
|
"topk": topk,
|
|
"comp_block": comp_block,
|
|
"comp_bt": comp_bt,
|
|
"bt_stride": bt_stride,
|
|
"kt": kt,
|
|
"klen": klen,
|
|
},
|
|
}
|
|
c["krdy"] = sreg(scratch, layout["krdy"][0], "krdy")
|
|
for t in range(first_task(bid, BLOCKS - s), s, BLOCKS):
|
|
mb_front.stage_kt(ckt, t)
|
|
mb_back.publish(c, c["krdy"], t)
|
|
mb_back.run_back(c, bkey, bid)
|
|
gpu.barrier()
|
|
|
|
ffn = dict(
|
|
res_in=res_in,
|
|
post_in=post_in,
|
|
comb_in=comb_in,
|
|
pre_in=pre_in,
|
|
hc_fn=hc_fn,
|
|
hc_scale=hc_scale,
|
|
hc_base=hc_base,
|
|
ffn_w=ffn_w,
|
|
res_out=res_out,
|
|
post_out=post_out,
|
|
comb_out=comb_out,
|
|
pre_out=pre_out,
|
|
gate_w=gate_w,
|
|
bias=bias,
|
|
w13=w13,
|
|
w13_s=w13_s,
|
|
w2=w2,
|
|
w2_s=w2_s,
|
|
sgu=sgu,
|
|
sgu_s=sgu_s,
|
|
sw2=sw2,
|
|
sw2_s=sw2_s,
|
|
out=out,
|
|
scratch=scratch,
|
|
epoch=epoch,
|
|
)
|
|
run_ffn(key, lds, ffn, amb, bases, own, rank, ep)
|
|
|
|
@flyc.jit
|
|
def launch(
|
|
q: Int64,
|
|
swa: Int64,
|
|
swa_stride: Int64,
|
|
swa_block: Int32,
|
|
comp: Int64,
|
|
comp_stride: Int64,
|
|
comp_block: Int32,
|
|
kt: Int64,
|
|
klen: Int64,
|
|
slot: Int64,
|
|
swa_idx: Int64,
|
|
swa_lens: Int64,
|
|
t2r: Int64,
|
|
topk: Int64,
|
|
comp_bt: Int64,
|
|
bt_stride: Int32,
|
|
pos: Int64,
|
|
sink: Int64,
|
|
qk_scale: Int32,
|
|
cos_sin: Int64,
|
|
woa: Int64,
|
|
woa_s: Int64,
|
|
wob: Int64,
|
|
wob_s: Int64,
|
|
zrec: Int64,
|
|
res_in: Int64,
|
|
post_in: Int64,
|
|
comb_in: Int64,
|
|
pre_in: Int64,
|
|
hc_fn: Int64,
|
|
hc_scale: Int64,
|
|
hc_base: Int64,
|
|
ffn_w: Int64,
|
|
res_out: Int64,
|
|
post_out: Int64,
|
|
comb_out: Int64,
|
|
pre_out: Int64,
|
|
gate_w: Int64,
|
|
bias: Int64,
|
|
w13: Int64,
|
|
w13_s: Int64,
|
|
w2: Int64,
|
|
w2_s: Int64,
|
|
sgu: Int64,
|
|
sgu_s: Int64,
|
|
sw2: Int64,
|
|
sw2_s: Int64,
|
|
out: Int64,
|
|
scratch: Int64,
|
|
sym: Int64,
|
|
peers: Int64,
|
|
rank: Int32,
|
|
epoch: Int64,
|
|
tl: Int64,
|
|
stream: fx.Stream = _STREAM,
|
|
):
|
|
_ = keyed
|
|
k2(
|
|
q,
|
|
swa,
|
|
swa_stride,
|
|
swa_block,
|
|
comp,
|
|
comp_stride,
|
|
comp_block,
|
|
kt,
|
|
klen,
|
|
slot,
|
|
swa_idx,
|
|
swa_lens,
|
|
t2r,
|
|
topk,
|
|
comp_bt,
|
|
bt_stride,
|
|
pos,
|
|
sink,
|
|
qk_scale,
|
|
cos_sin,
|
|
woa,
|
|
woa_s,
|
|
wob,
|
|
wob_s,
|
|
zrec,
|
|
res_in,
|
|
post_in,
|
|
comb_in,
|
|
pre_in,
|
|
hc_fn,
|
|
hc_scale,
|
|
hc_base,
|
|
ffn_w,
|
|
res_out,
|
|
post_out,
|
|
comb_out,
|
|
pre_out,
|
|
gate_w,
|
|
bias,
|
|
w13,
|
|
w13_s,
|
|
w2,
|
|
w2_s,
|
|
sgu,
|
|
sgu_s,
|
|
sw2,
|
|
sw2_s,
|
|
out,
|
|
scratch,
|
|
sym,
|
|
peers,
|
|
rank,
|
|
epoch,
|
|
tl,
|
|
).launch(grid=(BLOCKS,), block=(THREADS,), stream=stream)
|
|
|
|
return launch
|
|
|
|
|
|
# ---------------------------------------------------------------- the FFN launch
|
|
|
|
|
|
def build_mono_ffn(key: MonoBuild):
|
|
"""The FFN launch of a layer whose attention vLLM ran (``part``: this rank's
|
|
unreduced wo_b output, bf16 [S, HIDDEN]): K2's FFN half, the partial pushed
|
|
to every rank first. It takes a launch pair's epoch, as K2 does."""
|
|
s, tp = key.tokens, key.tp
|
|
assert 1 <= s <= MAX_TOKENS
|
|
rs = route_shape(_moe_key(key))
|
|
assert rs.experts // 64 in SORT_NETS
|
|
MoeLds = moe.moe_smem(s, rs, False)
|
|
half = peer_half_bytes(tp)
|
|
assert SLICES + 2 * s <= BLOCKS
|
|
SeamLds = _seam_lds(s)
|
|
|
|
@fx.union
|
|
class FfnLds:
|
|
seam: SeamLds # type: ignore[valid-type]
|
|
moe: MoeLds # type: ignore[valid-type]
|
|
|
|
name = kernel_symbol("dsv41_mono_ffn", s=s, tp=tp)
|
|
keyed = key_tuple(key, SOURCES)
|
|
|
|
@flyc.kernel(name=name, known_block_size=[THREADS, 1, 1])
|
|
def kffn(
|
|
part: Int64,
|
|
res_in: Int64,
|
|
post_in: Int64,
|
|
comb_in: Int64,
|
|
pre_in: Int64,
|
|
hc_fn: Int64,
|
|
hc_scale: Int64,
|
|
hc_base: Int64,
|
|
ffn_w: Int64,
|
|
res_out: Int64,
|
|
post_out: Int64,
|
|
comb_out: Int64,
|
|
pre_out: Int64,
|
|
gate_w: Int64,
|
|
bias: Int64,
|
|
w13: Int64,
|
|
w13_s: Int64,
|
|
w2: Int64,
|
|
w2_s: Int64,
|
|
sgu: Int64,
|
|
sgu_s: Int64,
|
|
sw2: Int64,
|
|
sw2_s: Int64,
|
|
out: Int64,
|
|
scratch: Int64,
|
|
sym: Int64,
|
|
peers: Int64,
|
|
rank: Int32,
|
|
epoch: Int64,
|
|
):
|
|
_ = keyed
|
|
lds = fx.SharedAllocator().allocate(FfnLds)
|
|
ep = _epoch(epoch)
|
|
par = fx.Int64(ep & 1) * fx.Int64(half)
|
|
bases = [b + par for b in peer_bases(peers, tp)]
|
|
mb_back.epoch_begin({"bid": fx.block_idx.x, "tid": fx.thread_idx.x}, epoch, ep)
|
|
a = dict(
|
|
res_in=res_in,
|
|
post_in=post_in,
|
|
comb_in=comb_in,
|
|
pre_in=pre_in,
|
|
hc_fn=hc_fn,
|
|
hc_scale=hc_scale,
|
|
hc_base=hc_base,
|
|
ffn_w=ffn_w,
|
|
res_out=res_out,
|
|
post_out=post_out,
|
|
comb_out=comb_out,
|
|
pre_out=pre_out,
|
|
gate_w=gate_w,
|
|
bias=bias,
|
|
w13=w13,
|
|
w13_s=w13_s,
|
|
w2=w2,
|
|
w2_s=w2_s,
|
|
sgu=sgu,
|
|
sgu_s=sgu_s,
|
|
sw2=sw2,
|
|
sw2_s=sw2_s,
|
|
out=out,
|
|
scratch=scratch,
|
|
epoch=epoch,
|
|
)
|
|
amb = Mailbox((ep << 1) + 2)
|
|
run_ffn(key, lds, a, amb, bases, sym + par, rank, ep, part=part)
|
|
|
|
@flyc.jit
|
|
def launch(
|
|
part: Int64,
|
|
res_in: Int64,
|
|
post_in: Int64,
|
|
comb_in: Int64,
|
|
pre_in: Int64,
|
|
hc_fn: Int64,
|
|
hc_scale: Int64,
|
|
hc_base: Int64,
|
|
ffn_w: Int64,
|
|
res_out: Int64,
|
|
post_out: Int64,
|
|
comb_out: Int64,
|
|
pre_out: Int64,
|
|
gate_w: Int64,
|
|
bias: Int64,
|
|
w13: Int64,
|
|
w13_s: Int64,
|
|
w2: Int64,
|
|
w2_s: Int64,
|
|
sgu: Int64,
|
|
sgu_s: Int64,
|
|
sw2: Int64,
|
|
sw2_s: Int64,
|
|
out: Int64,
|
|
scratch: Int64,
|
|
sym: Int64,
|
|
peers: Int64,
|
|
rank: Int32,
|
|
epoch: Int64,
|
|
stream: fx.Stream = _STREAM,
|
|
):
|
|
_ = keyed
|
|
kffn(
|
|
part,
|
|
res_in,
|
|
post_in,
|
|
comb_in,
|
|
pre_in,
|
|
hc_fn,
|
|
hc_scale,
|
|
hc_base,
|
|
ffn_w,
|
|
res_out,
|
|
post_out,
|
|
comb_out,
|
|
pre_out,
|
|
gate_w,
|
|
bias,
|
|
w13,
|
|
w13_s,
|
|
w2,
|
|
w2_s,
|
|
sgu,
|
|
sgu_s,
|
|
sw2,
|
|
sw2_s,
|
|
out,
|
|
scratch,
|
|
sym,
|
|
peers,
|
|
rank,
|
|
epoch,
|
|
).launch(grid=(BLOCKS,), block=(THREADS,), stream=stream)
|
|
|
|
return launch
|