Files
2026-10-13 10:16:50 +08:00

1019 lines
28 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The mono decode layer's two launches (README): K1 = the attention seam
(``stages.seam``: slice, gate, the norm into vLLM's MXFP8) + the attention front
(``attention.front``); K2 = the attention back (``attention.back``, wo_b pushed
to every rank) + the FFN seam (the TP sum folded in) + the MoE (``stages.moe``,
its all-reduce in-kernel too). A layer whose attention vLLM runs takes the FFN
launch instead: K2 with the attention back replaced by a push of vLLM's
unreduced wo_b output (``seam.stage_push``).
Tags: K1's hand-offs carry 2 e + 1, K2's 2 e + 2 (e: the launch pair's epoch,
moved on by K2's CTA 0 at its end), so K2's seam may reuse K1's seam regions.
Peer regions alternate by e's parity; every rank runs the same launches, so
the epochs agree across ranks.
"""
from dataclasses import dataclass
import flydsl.compiler as flyc
import flydsl.expr as fx
from aiter.ops.flydsl.kernels import buffer_ops as bo
from flydsl.expr import const_expr, gpu, range_constexpr, rocdl
from flydsl.expr.typing import Int32, Int64, T
from .attention import back as mb_back
from .attention import front as mb_front
from .attention.device import BLOCKS, CM_DEV, THREADS, gstore, kernel_symbol, rsrc
from .attention.plan import HIDDEN, Dims, back_scratch, front_scratch
from .common.mx import clamp_fp8
from .common.ops import butterfly, fp8_pack4, peer_bases, traced
from .common.plan import WAVES, first_task, key_tuple
from .common.sync import Mailbox, preg, publish, sreg
from .sources import SOURCES
from .stages import moe, seam
from .stages.dims import Dims as StageDims
from .stages.moe_shape import SORT_NETS, MoeBuild, route_shape
_STREAM = fx.Stream(None)
MAX_TOKENS = 48
X_WORDS = HIDDEN // 4
X_GROUPS = HIDDEN // 32
SLICES = seam.SLICES # 160
# K1's MXFP8 x and the MoE's: plain words in both, the same size, so the two
# launches share the MoE's (placed later); any other name is one region, so a
# word is never a tag to one launch and data to another
SHARED_REGIONS = ("x8", "x8s")
COUNTER_WORDS = 2 * 256 # the MoE's ug queue and down counts, a word a slot
COUNTERS = ("ugq", "dq")
# the epoch buffer, in words: the epoch, a mark a CTA (back.EPOCH_MARKS), then on
# lines of their own the MoE's counters
EPOCH_COUNTERS = -(-(mb_back.EPOCH_MARKS + BLOCKS) // 64) * 64
EPOCH_WORDS = EPOCH_COUNTERS + COUNTER_WORDS
@dataclass(frozen=True)
class MonoBuild:
tokens: int
tp: int
ratio: int # the layer's compress ratio (its attention's key sources)
timeline: bool = False
# K1's seam folds the previous layer's output into the residual; off for a
# residual that arrives settled (the first layer, after an Engram)
fold: bool = True
# the layer selects its own top-k (an index source): K1 hands its normed
# input and qr out for vLLM's indexer, K2 builds the key table
index: bool = False
def _moe_key(key: MonoBuild) -> MoeBuild:
return MoeBuild(tokens=key.tokens, tp=key.tp)
def _front_key(key: MonoBuild):
return mb_front.FrontBuild(key.tokens, key.tp, key.ratio, key.timeline)
def _back_key(key: MonoBuild):
return mb_back.BackBuild(key.tokens, key.tp, key.ratio, key.timeline)
def scratch_layout(s: int, tp: int) -> dict:
"""Every region of step width s's launches -> (byte offset, bytes), disjoint:
K1's front and seam, K2's back, the MoE's regions, the normed rows and their
flags. Each width has a scratch of its own (``DSV41MonoLayer.scratch``):
under another width's layout a mailbox pair's tag word could be plain data,
which a poll may take for a current tag. The MoE's counters, whose slots
outlive a step width, are in the epoch buffer."""
d = Dims(tp)
out: dict = {}
off = 0
def place(regions):
nonlocal off
base = off
for name, (o, n) in regions.items():
assert name not in out or name in SHARED_REGIONS, name
out[name] = (base + o, n)
off = max(off, base + max(o + n for o, n in regions.values()))
off = -(-off // 256) * 256
place(front_scratch(s, d))
place(seam.scratch_layout(s))
place(back_scratch(s, d, start=0))
regions = moe.scratch_layout(_moe_key(MonoBuild(s, tp, 1))).items()
mr = {n: v for n, v in regions if n not in COUNTERS}
lo = min(o for o, _ in mr.values())
place({n: (o - lo, n_) for n, (o, n_) in mr.items()})
place({"normed": (0, s * HIDDEN * 2), "xrdy_moe": (s * HIDDEN * 2 + 256, s * 8)})
place({"krdy": (0, s * 8)})
return out
def scratch_bytes(s: int = MAX_TOKENS, tp: int = 2) -> int:
return max(o + n for o, n in scratch_layout(s, tp).values())
def peer_half_bytes(tp: int) -> int:
"""One parity's peer regions: the attention's partials, then the MoE's."""
return seam.attn_peer_bytes(MAX_TOKENS, tp) + moe.peer_bytes(MAX_TOKENS, tp)
def _epoch(epoch):
return fx.Int32(bo.buffer_load(rsrc(epoch), 0, vec_width=1, dtype=T.i32))
# ---------------------------------------------------------------- K1
@traced
def store_x8(c, t, col, ys, live):
"""The attention seam's norm output (``seam.stage_norm``'s 8 columns at
``col``) -> vLLM's MXFP8 of each 32 (the attention's input quant) -> the
front's X8 / X8S, plain at device scope."""
amax = abs(ys[0])
for y in ys[1:]:
amax = fx.max(amax, abs(y))
code = moe.vllm_mx_code(butterfly(amax, (1, 2), fx.max))
mul = ((254 - code) << 23).bitcast(fx.Float32)
w0 = fp8_pack4(*[clamp_fp8(ys[i] * mul) for i in range(4)])
w1 = fp8_pack4(*[clamp_fp8(ys[4 + i] * mul) for i in range(4)])
if live:
bo.buffer_store(
fx.Vector.from_elements([w0, w1], fx.Int32),
rsrc(c["x8"]),
t * X_WORDS + col // 4,
cache_modifier=CM_DEV,
)
if c["tid"] % 4 == 0:
bo.buffer_store(
code, rsrc(c["x8s"]), t * X_GROUPS + col // 32, cache_modifier=CM_DEV
)
@traced
def publish_x8(c, t):
"""Token t's two XRDY flags (one a wqkv K half) once its X8 / X8S landed."""
rocdl.s_waitcnt(vmcnt=0)
gpu.barrier()
if c["tid"] < 2:
c["mb"].put(c["xrdy"], 2 * t + c["tid"], fx.Int32(1))
gpu.barrier()
def build_mono_k1(key: MonoBuild):
s, tp = key.tokens, key.tp
assert 1 <= s <= MAX_TOKENS
layout = scratch_layout(s, tp)
fkey = _front_key(key)
FrontLds = mb_front.front_lds(s, Dims(tp))
NORM0 = SLICES # the norms on the CTAs past the slices: the wqkv waits on them
GATE0 = SLICES + s
assert GATE0 + s <= BLOCKS
@fx.struct
class SeamLds:
red: fx.Array[fx.Float32, WAVES * 2 * 64 * 4, 16]
rl: fx.Array[fx.Float32, s * seam.KT, 16]
fl: fx.Array[fx.Float32, seam.MIX * seam.KT, 16]
@fx.union
class K1Lds:
seam: SeamLds
front: FrontLds # type: ignore[valid-type]
name = kernel_symbol(
"dsv41_mono_k1",
s=s,
tp=tp,
r=key.ratio,
tl=key.timeline,
f=key.fold,
i=key.index,
)
keyed = key_tuple(key, SOURCES)
@flyc.kernel(name=name, known_block_size=[THREADS, 1, 1])
def k1(
res_in: Int64,
pend: Int64,
post_in: Int64,
comb_in: Int64,
pre_in: Int64,
hc_fn: Int64,
hc_scale: Int64,
hc_base: Int64,
attn_w: Int64,
res_out: Int64,
post_out: Int64,
comb_out: Int64,
pre_out: Int64,
wqkv: Int64,
wqkv_s: Int64,
qn_w: Int64,
kvn_w: Int64,
wqb: Int64,
wqb_s: Int64,
cos_sin: Int64,
pos: Int64,
slot: Int64,
swa: Int64,
swa_stride: Int64,
swa_block: Int32,
q: Int64,
swa_idx: Int64,
swa_lens: Int64,
comp_block: Int32,
comp_bt: Int64,
bt_stride: Int32,
topk: Int64,
t2r: Int64,
kt: Int64,
klen: Int64,
normed_out: Int64,
qr_out: Int64,
scratch: Int64,
epoch: Int64,
tl: Int64,
):
_ = keyed
bid = fx.block_idx.x
lds = fx.SharedAllocator().allocate(K1Lds)
sl = lds.seam.peek()
ep = _epoch(epoch)
tag = (ep << 1) + 1
fargs = {
"x": 0,
"x_stride": 0,
"wqkv": wqkv,
"wqkv_s": wqkv_s,
"qn_w": qn_w,
"kvn_w": kvn_w,
"wqb": wqb,
"wqb_s": wqb_s,
"cos_sin": cos_sin,
"pos": pos,
"slot": slot,
"swa": swa,
"swa_stride": swa_stride,
"swa_block": swa_block,
"q": q,
"swa_idx": swa_idx,
"swa_lens": swa_lens,
"comp_block": comp_block,
"comp_bt": comp_bt,
"bt_stride": bt_stride,
"topk": topk,
"t2r": t2r,
"kt": kt,
"klen": klen,
"qr_out": qr_out,
}
flayout = {n: layout[n] for n in front_scratch(s, Dims(tp))}
c = mb_front.front_context(fkey, lds.front.peek(), fargs, scratch, tag, flayout)
c["index"] = key.index
amb = Mailbox(tag)
tid = fx.thread_idx.x
ca = {
"S": s,
"tid": tid,
"bid": bid,
"lane": tid % 64,
"wave": tid // 64,
"rl": sl.rl.ptr,
"fl": sl.fl.ptr,
"red": sl.red.ptr,
"put": amb.put,
"put_bf": amb.put_bf,
"put_words": amb.put_words,
"poll": amb.poll,
"args": {
"res_in": res_in,
"pend": pend,
"post_in": post_in,
"comb_in": comb_in,
"pre_in": pre_in,
"hc_fn": hc_fn,
"hc_scale": hc_scale,
"hc_base": hc_base,
"norm_w": attn_w,
"res_out": res_out,
"post_out": post_out,
"comb_out": comb_out,
"pre_out": pre_out,
},
"d": StageDims(tp),
"fold": key.fold,
}
for region in ("lin", "pmix"):
ca[region] = sreg(scratch, layout[region][0], region)
if const_expr(key.timeline):
c["tl"], c["tl_points"] = tl, mb_front.FRONT_POINTS
def seam_slice():
for task in range(first_task(bid, 0), SLICES, BLOCKS):
seam.stage_slice(ca, task)
gpu.barrier()
def store(*v):
store_x8(c, *v)
if const_expr(key.index):
store_normed(normed_out, *v)
def seam_rest():
for t in range(first_task(bid, NORM0), s, BLOCKS):
seam.stage_norm(ca, t, store)
publish_x8(c, t)
for t in range(first_task(bid, GATE0), s, BLOCKS):
seam.stage_gate(ca, t)
gpu.barrier()
mb_front.run_front(
c, fkey, bid, x8_given=True, before_wqkv=seam_slice, after_wqkv=seam_rest
)
@flyc.jit
def launch(
res_in: Int64,
pend: Int64,
post_in: Int64,
comb_in: Int64,
pre_in: Int64,
hc_fn: Int64,
hc_scale: Int64,
hc_base: Int64,
attn_w: Int64,
res_out: Int64,
post_out: Int64,
comb_out: Int64,
pre_out: Int64,
wqkv: Int64,
wqkv_s: Int64,
qn_w: Int64,
kvn_w: Int64,
wqb: Int64,
wqb_s: Int64,
cos_sin: Int64,
pos: Int64,
slot: Int64,
swa: Int64,
swa_stride: Int64,
swa_block: Int32,
q: Int64,
swa_idx: Int64,
swa_lens: Int64,
comp_block: Int32,
comp_bt: Int64,
bt_stride: Int32,
topk: Int64,
t2r: Int64,
kt: Int64,
klen: Int64,
normed_out: Int64,
qr_out: Int64,
scratch: Int64,
epoch: Int64,
tl: Int64,
stream: fx.Stream = _STREAM,
):
_ = keyed
k1(
res_in,
pend,
post_in,
comb_in,
pre_in,
hc_fn,
hc_scale,
hc_base,
attn_w,
res_out,
post_out,
comb_out,
pre_out,
wqkv,
wqkv_s,
qn_w,
kvn_w,
wqb,
wqb_s,
cos_sin,
pos,
slot,
swa,
swa_stride,
swa_block,
q,
swa_idx,
swa_lens,
comp_block,
comp_bt,
bt_stride,
topk,
t2r,
kt,
klen,
normed_out,
qr_out,
scratch,
epoch,
tl,
).launch(grid=(BLOCKS,), block=(THREADS,), stream=stream)
return launch
# ---------------------------------------------------------------- K2
@traced
def store_normed(normed, t, col, ys, live):
"""The FFN seam's norm output -> the MoE input row (bf16, device scope:
this launch reads it)."""
if live:
for j in range_constexpr(8):
bo.buffer_store(
ys[j].to(fx.BFloat16),
rsrc(normed),
t * HIDDEN + col + j,
cache_modifier=CM_DEV,
)
def _seam_lds(s):
@fx.struct
class SeamLds:
red: fx.Array[fx.Float32, WAVES * 2 * 64 * 4, 16]
rl: fx.Array[fx.Float32, s * seam.KT, 16]
fl: fx.Array[fx.Float32, seam.MIX * seam.KT, 16]
pl: fx.Array[fx.Float32, s * seam.COLS, 16]
return SeamLds
@traced
def run_ffn(key: MonoBuild, lds, a: dict, amb, bases, own, rank, ep, part=None):
"""K2's FFN half on this CTA: the FFN seam (the attention's TP partials --
``part`` pushed to every rank's ATTN region first -- summed and folded into
the residual), the MoE and its all-reduce into ``a["out"]``, then the
launch pair's epoch moved on."""
s, tp = key.tokens, key.tp
layout = scratch_layout(s, tp)
mkey = _moe_key(key)
rs = route_shape(mkey)
GATE0 = SLICES
NORM0 = SLICES + s
bid = fx.block_idx.x
tid = fx.thread_idx.x
sl = lds.seam.peek()
scratch = a["scratch"]
seam_args = (
"res_in",
"post_in",
"comb_in",
"pre_in",
"hc_fn",
"hc_scale",
"hc_base",
"res_out",
"post_out",
"comb_out",
"pre_out",
)
ca = {
"S": s,
"tid": tid,
"bid": bid,
"lane": tid % 64,
"wave": tid // 64,
"rl": sl.rl.ptr,
"fl": sl.fl.ptr,
"red": sl.red.ptr,
"pend_lds": sl.pl.ptr,
"put": amb.put,
"put_bf": amb.put_bf,
"put_words": amb.put_words,
"poll": amb.poll,
"peer_addr": lambda p: bases[p],
"rank": rank,
"sym": own,
"args": {n: a[n] for n in seam_args} | {"norm_w": a["ffn_w"]},
"d": StageDims(tp),
}
normed = scratch + fx.Int64(layout["normed"][0])
for region in ("lin", "pmix"):
ca[region] = sreg(scratch, layout[region][0], region)
mlayout = {n: layout[n] for n in moe.scratch_layout(mkey) if n not in COUNTERS}
mlayout["xrdy"] = layout["xrdy_moe"]
margs = moe.moe_args(
normed,
a["gate_w"],
a["bias"],
a["w13"],
a["w13_s"],
a["w2"],
a["w2_s"],
a["sgu"],
a["sgu_s"],
a["sw2"],
a["sw2_s"],
a["out"],
)
cb = moe.moe_context(
s, rs, lds.moe.peek(), amb, bases, mlayout, scratch, margs, rank, own
)
cb["x_cm"], cb["x_ready"] = CM_DEV, True
cb["tag"] = ep & 255
for i, region in enumerate(COUNTERS):
cb[region] = sreg(a["epoch"], 4 * (EPOCH_COUNTERS + i * moe.TAGS), region)
for task in range(first_task(bid, 0), SLICES, BLOCKS):
if const_expr(part is not None):
seam.stage_push(ca, task, part)
seam.stage_reduce(ca, task)
seam.stage_slice(ca, task)
for t in range(first_task(bid, GATE0), s, BLOCKS):
seam.stage_gate(ca, t)
for t in range(first_task(bid, NORM0), s, BLOCKS):
seam.stage_norm(ca, t, lambda *v: store_normed(normed, *v))
publish(cb["put"], cb["xrdy"], t, 1, tid == 0)
gpu.barrier()
# ---- the MoE, its all-reduce into ``out``
moe.run_moe(cb, mkey, bid, 0, 0)
def reset():
# the MoE counter slot 128 launch pairs ahead: its last use long
# done, its next use far off
slot = (ep + 128) & 255
for region in COUNTERS:
gstore(cb[region].value + fx.Int64(slot * 4), fx.Int32(0), words=1)
mb_back.epoch_end({"bid": bid, "tid": tid}, a["epoch"], ep, reset)
def build_mono_k2(key: MonoBuild):
s, tp = key.tokens, key.tp
assert 1 <= s <= MAX_TOKENS
layout = scratch_layout(s, tp)
bkey = _back_key(key)
mkey = _moe_key(key)
rs = route_shape(mkey)
assert rs.experts // 64 in SORT_NETS
BackM = mb_back.back_lds_members(s, Dims(tp))
SplitLds, GemvLds = BackM["split"], BackM["gemv"]
MoeLds = moe.moe_smem(s, rs, False)
half = peer_half_bytes(tp)
assert SLICES + 2 * s <= BLOCKS
SeamLds = _seam_lds(s)
@fx.union
class K2Lds:
split: SplitLds # type: ignore[valid-type]
gemv: GemvLds # type: ignore[valid-type]
seam: SeamLds # type: ignore[valid-type]
moe: MoeLds # type: ignore[valid-type]
name = kernel_symbol(
"dsv41_mono_k2", s=s, tp=tp, r=key.ratio, tl=key.timeline, i=key.index
)
keyed = key_tuple(key, SOURCES)
@flyc.kernel(name=name, known_block_size=[THREADS, 1, 1])
def k2(
q: Int64,
swa: Int64,
swa_stride: Int64,
swa_block: Int32,
comp: Int64,
comp_stride: Int64,
comp_block: Int32,
kt: Int64,
klen: Int64,
slot: Int64,
swa_idx: Int64,
swa_lens: Int64,
t2r: Int64,
topk: Int64,
comp_bt: Int64,
bt_stride: Int32,
pos: Int64,
sink: Int64,
qk_scale: Int32,
cos_sin: Int64,
woa: Int64,
woa_s: Int64,
wob: Int64,
wob_s: Int64,
zrec: Int64,
res_in: Int64,
post_in: Int64,
comb_in: Int64,
pre_in: Int64,
hc_fn: Int64,
hc_scale: Int64,
hc_base: Int64,
ffn_w: Int64,
res_out: Int64,
post_out: Int64,
comb_out: Int64,
pre_out: Int64,
gate_w: Int64,
bias: Int64,
w13: Int64,
w13_s: Int64,
w2: Int64,
w2_s: Int64,
sgu: Int64,
sgu_s: Int64,
sw2: Int64,
sw2_s: Int64,
out: Int64,
scratch: Int64,
sym: Int64,
peers: Int64,
rank: Int32,
epoch: Int64,
tl: Int64,
):
_ = keyed
bid = fx.block_idx.x
lds = fx.SharedAllocator().allocate(K2Lds)
ep = _epoch(epoch)
tag = (ep << 1) + 2
par = fx.Int64(ep & 1) * fx.Int64(half)
bases = [b + par for b in peer_bases(peers, tp)]
own = sym + par
bargs = {
"q": q,
"swa": swa,
"swa_stride": swa_stride,
"swa_block": swa_block,
"comp": comp,
"comp_stride": comp_stride,
"comp_block": comp_block,
"kt": kt,
"klen": klen,
"pos": pos,
"sink": sink,
"qk_scale": qk_scale,
"cos_sin": cos_sin,
"woa": woa,
"woa_s": woa_s,
"wob": wob,
"wob_s": wob_s,
"out": 0,
"out_stride": 0,
"zrec": zrec,
}
blayout = {n: layout[n] for n in back_scratch(s, Dims(tp), start=0)}
views = {"split": lds.split.peek(), "gemv": lds.gemv.peek()}
c = mb_back.back_context(bkey, views, bargs, scratch, tag, blayout)
amb = Mailbox(tag)
def push(t, col, v0, v1):
# this rank's bf16 partial pair to every rank's ATTN region
for p in range_constexpr(tp):
dst = preg(bases[p], 0, "attn")
amb.put_bf(dst, 2 * seam.attn_region_pair(rank, t, s, col), [v0, v1])
c["wob_out"] = push
if const_expr(key.timeline):
c["tl"], c["tl_points"] = tl, mb_back.BACK_POINTS
mb_back.epoch_begin(c, epoch, ep)
if const_expr(key.index):
# the key table, from the top-k vLLM's indexer selected after K1,
# on the last CTAs: each split task waits on its token's
ckt = {
"S": s,
"tid": c["tid"],
"dev": True,
"ratio": key.ratio,
"args": {
"slot": slot,
"swa_idx": swa_idx,
"swa_lens": swa_lens,
"pos": pos,
"t2r": t2r,
"topk": topk,
"comp_block": comp_block,
"comp_bt": comp_bt,
"bt_stride": bt_stride,
"kt": kt,
"klen": klen,
},
}
c["krdy"] = sreg(scratch, layout["krdy"][0], "krdy")
for t in range(first_task(bid, BLOCKS - s), s, BLOCKS):
mb_front.stage_kt(ckt, t)
mb_back.publish(c, c["krdy"], t)
mb_back.run_back(c, bkey, bid)
gpu.barrier()
ffn = dict(
res_in=res_in,
post_in=post_in,
comb_in=comb_in,
pre_in=pre_in,
hc_fn=hc_fn,
hc_scale=hc_scale,
hc_base=hc_base,
ffn_w=ffn_w,
res_out=res_out,
post_out=post_out,
comb_out=comb_out,
pre_out=pre_out,
gate_w=gate_w,
bias=bias,
w13=w13,
w13_s=w13_s,
w2=w2,
w2_s=w2_s,
sgu=sgu,
sgu_s=sgu_s,
sw2=sw2,
sw2_s=sw2_s,
out=out,
scratch=scratch,
epoch=epoch,
)
run_ffn(key, lds, ffn, amb, bases, own, rank, ep)
@flyc.jit
def launch(
q: Int64,
swa: Int64,
swa_stride: Int64,
swa_block: Int32,
comp: Int64,
comp_stride: Int64,
comp_block: Int32,
kt: Int64,
klen: Int64,
slot: Int64,
swa_idx: Int64,
swa_lens: Int64,
t2r: Int64,
topk: Int64,
comp_bt: Int64,
bt_stride: Int32,
pos: Int64,
sink: Int64,
qk_scale: Int32,
cos_sin: Int64,
woa: Int64,
woa_s: Int64,
wob: Int64,
wob_s: Int64,
zrec: Int64,
res_in: Int64,
post_in: Int64,
comb_in: Int64,
pre_in: Int64,
hc_fn: Int64,
hc_scale: Int64,
hc_base: Int64,
ffn_w: Int64,
res_out: Int64,
post_out: Int64,
comb_out: Int64,
pre_out: Int64,
gate_w: Int64,
bias: Int64,
w13: Int64,
w13_s: Int64,
w2: Int64,
w2_s: Int64,
sgu: Int64,
sgu_s: Int64,
sw2: Int64,
sw2_s: Int64,
out: Int64,
scratch: Int64,
sym: Int64,
peers: Int64,
rank: Int32,
epoch: Int64,
tl: Int64,
stream: fx.Stream = _STREAM,
):
_ = keyed
k2(
q,
swa,
swa_stride,
swa_block,
comp,
comp_stride,
comp_block,
kt,
klen,
slot,
swa_idx,
swa_lens,
t2r,
topk,
comp_bt,
bt_stride,
pos,
sink,
qk_scale,
cos_sin,
woa,
woa_s,
wob,
wob_s,
zrec,
res_in,
post_in,
comb_in,
pre_in,
hc_fn,
hc_scale,
hc_base,
ffn_w,
res_out,
post_out,
comb_out,
pre_out,
gate_w,
bias,
w13,
w13_s,
w2,
w2_s,
sgu,
sgu_s,
sw2,
sw2_s,
out,
scratch,
sym,
peers,
rank,
epoch,
tl,
).launch(grid=(BLOCKS,), block=(THREADS,), stream=stream)
return launch
# ---------------------------------------------------------------- the FFN launch
def build_mono_ffn(key: MonoBuild):
"""The FFN launch of a layer whose attention vLLM ran (``part``: this rank's
unreduced wo_b output, bf16 [S, HIDDEN]): K2's FFN half, the partial pushed
to every rank first. It takes a launch pair's epoch, as K2 does."""
s, tp = key.tokens, key.tp
assert 1 <= s <= MAX_TOKENS
rs = route_shape(_moe_key(key))
assert rs.experts // 64 in SORT_NETS
MoeLds = moe.moe_smem(s, rs, False)
half = peer_half_bytes(tp)
assert SLICES + 2 * s <= BLOCKS
SeamLds = _seam_lds(s)
@fx.union
class FfnLds:
seam: SeamLds # type: ignore[valid-type]
moe: MoeLds # type: ignore[valid-type]
name = kernel_symbol("dsv41_mono_ffn", s=s, tp=tp)
keyed = key_tuple(key, SOURCES)
@flyc.kernel(name=name, known_block_size=[THREADS, 1, 1])
def kffn(
part: Int64,
res_in: Int64,
post_in: Int64,
comb_in: Int64,
pre_in: Int64,
hc_fn: Int64,
hc_scale: Int64,
hc_base: Int64,
ffn_w: Int64,
res_out: Int64,
post_out: Int64,
comb_out: Int64,
pre_out: Int64,
gate_w: Int64,
bias: Int64,
w13: Int64,
w13_s: Int64,
w2: Int64,
w2_s: Int64,
sgu: Int64,
sgu_s: Int64,
sw2: Int64,
sw2_s: Int64,
out: Int64,
scratch: Int64,
sym: Int64,
peers: Int64,
rank: Int32,
epoch: Int64,
):
_ = keyed
lds = fx.SharedAllocator().allocate(FfnLds)
ep = _epoch(epoch)
par = fx.Int64(ep & 1) * fx.Int64(half)
bases = [b + par for b in peer_bases(peers, tp)]
mb_back.epoch_begin({"bid": fx.block_idx.x, "tid": fx.thread_idx.x}, epoch, ep)
a = dict(
res_in=res_in,
post_in=post_in,
comb_in=comb_in,
pre_in=pre_in,
hc_fn=hc_fn,
hc_scale=hc_scale,
hc_base=hc_base,
ffn_w=ffn_w,
res_out=res_out,
post_out=post_out,
comb_out=comb_out,
pre_out=pre_out,
gate_w=gate_w,
bias=bias,
w13=w13,
w13_s=w13_s,
w2=w2,
w2_s=w2_s,
sgu=sgu,
sgu_s=sgu_s,
sw2=sw2,
sw2_s=sw2_s,
out=out,
scratch=scratch,
epoch=epoch,
)
amb = Mailbox((ep << 1) + 2)
run_ffn(key, lds, a, amb, bases, sym + par, rank, ep, part=part)
@flyc.jit
def launch(
part: Int64,
res_in: Int64,
post_in: Int64,
comb_in: Int64,
pre_in: Int64,
hc_fn: Int64,
hc_scale: Int64,
hc_base: Int64,
ffn_w: Int64,
res_out: Int64,
post_out: Int64,
comb_out: Int64,
pre_out: Int64,
gate_w: Int64,
bias: Int64,
w13: Int64,
w13_s: Int64,
w2: Int64,
w2_s: Int64,
sgu: Int64,
sgu_s: Int64,
sw2: Int64,
sw2_s: Int64,
out: Int64,
scratch: Int64,
sym: Int64,
peers: Int64,
rank: Int32,
epoch: Int64,
stream: fx.Stream = _STREAM,
):
_ = keyed
kffn(
part,
res_in,
post_in,
comb_in,
pre_in,
hc_fn,
hc_scale,
hc_base,
ffn_w,
res_out,
post_out,
comb_out,
pre_out,
gate_w,
bias,
w13,
w13_s,
w2,
w2_s,
sgu,
sgu_s,
sw2,
sw2_s,
out,
scratch,
sym,
peers,
rank,
epoch,
).launch(grid=(BLOCKS,), block=(THREADS,), stream=stream)
return launch