Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
174 lines
4.6 KiB
Python
174 lines
4.6 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Geometry of the DeepSeek-V4.1 attention mega kernel: model widths, a TP
|
|
rank's share of them, task counts and the scratch (mailbox) layout."""
|
|
|
|
from dataclasses import dataclass
|
|
|
|
HIDDEN = 5120
|
|
Q_RANK = 1280
|
|
KV_DIM = 512
|
|
QKV_ROWS = Q_RANK + KV_DIM # 1792
|
|
HEAD_DIM = 512
|
|
NOPE = 448
|
|
ROPE = 64
|
|
HALF = ROPE // 2
|
|
N_HEADS = 64
|
|
O_GROUPS = 8
|
|
O_RANK = 1024
|
|
WINDOW = 128
|
|
TOPK = 512
|
|
EPS = 1e-20
|
|
|
|
ROWS = 16 # GEMV rows a task
|
|
TILE = 16 # tokens a GEMV pass (an MFMA's N)
|
|
HEAD_TILE = 16 # heads an attention MFMA takes
|
|
WOA_ROWS = 32 # a wo_a task's rows: one MX group of its output
|
|
MAX_TOKENS = 48
|
|
|
|
# KV record (V4 fp8_ds_mla): 576 B data rows (448 e4m3 | 64 bf16), then 8 B of
|
|
# scales a token (7 UE8M0 + pad) after the block's rows
|
|
DATA = 576
|
|
SCALE_BYTES = 8
|
|
|
|
# attention: a split is 16 keys; a token has at most TOPK + WINDOW keys
|
|
BK = 16
|
|
SPLITS = 64 # split slots a (token, head tile): 1024 keys >= TOPK + WINDOW
|
|
COLS = 32 # PV columns a group (one MX group of the output)
|
|
GROUPS = HEAD_DIM // COLS # 16
|
|
|
|
PAIR = 8 # a mailbox (value, tag) pair
|
|
|
|
|
|
def cdiv(a, b):
|
|
return -(-a // b)
|
|
|
|
|
|
def round_up(a, b):
|
|
return cdiv(a, b) * b
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Dims:
|
|
tp: int
|
|
|
|
def __post_init__(self):
|
|
assert self.tp in (2, 4), self.tp
|
|
|
|
@property
|
|
def heads(self) -> int:
|
|
return N_HEADS // self.tp
|
|
|
|
@property
|
|
def head_tiles(self) -> int:
|
|
return self.heads // HEAD_TILE
|
|
|
|
@property
|
|
def groups(self) -> int:
|
|
return O_GROUPS // self.tp
|
|
|
|
@property
|
|
def group_k(self) -> int:
|
|
"""wo_a's K a group: its eight heads' outputs."""
|
|
return self.heads // self.groups * HEAD_DIM # 4096
|
|
|
|
@property
|
|
def o_rows(self) -> int:
|
|
return self.groups * O_RANK
|
|
|
|
@property
|
|
def q_rows(self) -> int:
|
|
return self.heads * HEAD_DIM
|
|
|
|
@property
|
|
def wqb_tasks(self) -> int:
|
|
return self.q_rows // ROWS
|
|
|
|
@property
|
|
def woa_tasks(self) -> int:
|
|
return self.o_rows // WOA_ROWS
|
|
|
|
@property
|
|
def wob_tasks(self) -> int:
|
|
return HIDDEN // ROWS # 320
|
|
|
|
|
|
WQKV_TASKS = QKV_ROWS // ROWS # 112
|
|
WQKV_KPARTS = 2 # wqkv's K split: 224 (row tile, K half) parts
|
|
KEYS = TOPK + WINDOW # a token's attention keys at most
|
|
|
|
|
|
def token_tiles(s):
|
|
return [(t0, min(TILE, s - t0)) for t0 in range(0, s, TILE)]
|
|
|
|
|
|
def tile_rows(s):
|
|
return min(s, TILE)
|
|
|
|
|
|
def pair_layout(items, start=0, align=256):
|
|
out, off = {}, start
|
|
for name, pairs in items:
|
|
off = round_up(off, align)
|
|
out[name] = (off, pairs * PAIR)
|
|
off += pairs * PAIR
|
|
return out
|
|
|
|
|
|
def byte_layout(items, start=0, align=256):
|
|
out, off = {}, start
|
|
for name, nbytes in items:
|
|
off = round_up(off, align)
|
|
out[name] = (off, nbytes)
|
|
off += nbytes
|
|
return out
|
|
|
|
|
|
def front_scratch(s: int, d: Dims) -> dict:
|
|
"""K1's scratch regions -> (byte offset, bytes): published data (plain
|
|
words behind a flag) and mailbox pairs."""
|
|
return byte_layout(
|
|
[
|
|
("x8", s * HIDDEN), # e4m3 words (S > 16: quantized once, shared)
|
|
("x8s", s * HIDDEN // 32 * 4), # an i32 code a group
|
|
("xrdy", WQKV_KPARTS * s * PAIR), # a flag a (token, K half)
|
|
("qkvp", WQKV_KPARTS * s * QKV_ROWS * PAIR),
|
|
("qx8", s * Q_RANK), # e4m3 words
|
|
("qx8s", s * Q_RANK // 32 * 4), # an i32 code a group
|
|
("qrdy", s * PAIR),
|
|
]
|
|
)
|
|
|
|
|
|
def back_scratch(s: int, d: Dims, start: int = 0) -> dict:
|
|
"""K2's scratch regions, laid out past ``start``: published data (plain)
|
|
and flag pairs."""
|
|
ck = (
|
|
64 if d.head_tiles == 2 and s * cdiv(KEYS, 64) <= 256 else 128
|
|
) # back.chunk_keys
|
|
nchunk = cdiv(KEYS, ck)
|
|
return byte_layout(
|
|
[
|
|
("part", s * nchunk * d.heads * HEAD_DIM * 2), # bf16 partials
|
|
("pstat", s * nchunk * d.heads * 8), # (max, sum) f32
|
|
("srdy", s * nchunk * PAIR),
|
|
("xo", s * d.q_rows), # e4m3 words
|
|
("xos", s * d.q_rows // 32 * 4), # an i32 code a group
|
|
("crdy", s * d.heads // 8 * PAIR),
|
|
("x8b", s * d.o_rows),
|
|
("x8bs", s * d.o_rows // 32 * 4),
|
|
("ardy", d.woa_tasks * PAIR),
|
|
],
|
|
start=start,
|
|
)
|
|
|
|
|
|
def front_end(s: int, d: Dims) -> int:
|
|
"""Where K2's regions start: past K1's, which share the launch tag."""
|
|
return round_up(max(o + n for o, n in front_scratch(s, d).values()), 256)
|
|
|
|
|
|
def scratch_bytes(s: int, d: Dims) -> int:
|
|
b = back_scratch(s, d, start=front_end(s, d))
|
|
return round_up(max(o + n for o, n in b.values()), 4096)
|