[ROCm][Perf][DSv4.1] Mono decode on every backbone layer (#61082)

Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Fangzhou Ai
2026-10-13 10:16:50 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent f47792af7f
commit cc2a6eea1a
13 changed files with 379 additions and 111 deletions
+69 -14
View File
@@ -27,21 +27,25 @@ class _Runner:
self.calls.append((args, kwargs))
return ("mono outputs",)
def first_pre_mix(self, rows):
return torch.zeros(rows, 4)
def ffn(self, *args):
self.calls.append((args, {}))
return ("ffn outputs",)
def _layer(comp="comp"):
def _layer(comp="comp", indexer=None):
cache = lambda: torch.zeros(4, 128 * md.RECORD, dtype=torch.uint8) # noqa: E731
attn = SimpleNamespace(
swa_cache_layer=SimpleNamespace(prefix="swa", kv_cache=cache()),
compressed_cache_prefix=comp,
topk_indices_buffer=torch.zeros(64, 512, dtype=torch.int32),
compress_ratio=1,
indexer=indexer,
_compressed_kv_cache=cache,
)
return SimpleNamespace(attn=attn)
return SimpleNamespace(attn=attn, engram=None, hc_mult=4)
def _swa(**overrides):
@@ -77,7 +81,12 @@ def run(monkeypatch):
``mode`` and ``metadata``; returns (result, runner calls)."""
def _run(
mode=CUDAGraphMode.FULL, metadata=None, ffn_only=False, entry=None, **inputs
mode=CUDAGraphMode.FULL,
metadata=None,
ffn_only=False,
entry=None,
layer=None,
**inputs,
):
if metadata is None:
metadata = {
@@ -97,7 +106,7 @@ def run(monkeypatch):
# has no compressed cache
part, _ = args.pop("x"), args.pop("positions")
return mono.ffn(_layer(comp=None), part, **args), runner.calls
return mono(_layer(), **args), runner.calls
return mono(layer or _layer(), **args), runner.calls
return _run
@@ -114,6 +123,23 @@ def test_decode_step_runs_mono(run, mode):
1,
)
assert kwargs["comp_cache"].shape == (4, 128, md.RECORD)
assert kwargs["fold"] and "select" not in kwargs
def test_first_layer_runs_mono_from_the_embedding(run):
"""The first layer's residual is the embedding broadcast to the four
streams, which K1 takes as it is (no owed post to fold)."""
out, calls = run(residual=None, post_mix=None, res_mix=None, pre_mix=None)
assert out == ("mono outputs",) and len(calls) == 1
args, kwargs = calls[0]
assert args[2].shape == (M, 4, 5120) and not kwargs["fold"]
def test_index_source_selects_between_the_kernels(run):
"""An index source hands the runner its compressor and indexer to run
between K1 and K2."""
out, calls = run(layer=_layer(indexer=object()))
assert out == ("mono outputs",) and callable(calls[0][1]["select"])
def test_padded_decode_step_runs_mono(run):
@@ -149,7 +175,9 @@ def test_padded_decode_step_runs_mono(run):
),
dict(metadata={}), # a profile / dummy run: no attention metadata
dict(**_inputs(md.MAX_ROWS + 6)),
dict(residual=None), # the first layer's seam broadcasts the embedding
# a later pipeline stage's first layer: its stream arrives without
# the mixes
dict(residual=None, x=torch.zeros(M, 4, 5120, dtype=torch.bfloat16)),
],
ids=[
"piecewise",
@@ -158,7 +186,7 @@ def test_padded_decode_step_runs_mono(run):
"tokens-past-rows",
"no-metadata",
"rows",
"first-layer",
"stage-input",
],
)
def test_other_steps_take_the_original_path(run, case):
@@ -202,22 +230,41 @@ def _decoder_layer(engram=None, **attn):
)
def _indexer():
from vllm.models.deepseek_v41.amd.rocm import DeepseekV41RocmMxfp4Indexer
return object.__new__(DeepseekV41RocmMxfp4Indexer)
@pytest.mark.parametrize(
"layer, path",
[
(_decoder_layer(), "whole"),
(_decoder_layer(compress_ratio=0), "ffn"),
(_decoder_layer(engram=object()), "ffn"),
(_decoder_layer(compressor=object(), indexer=object()), "ffn"),
(_decoder_layer(compress_ratio=0), "whole"),
(_decoder_layer(engram=object()), "whole"),
(_decoder_layer(compressor=object(), indexer=_indexer()), "whole"),
(_decoder_layer(indexer=_indexer()), "whole"),
(_decoder_layer(compressor=object()), "ffn"),
(_decoder_layer(indexer=object()), "ffn"),
(_decoder_layer(kv_cache_dtype="auto"), "ffn"),
(_decoder_layer(layer_id=40), None),
],
ids=["standard", "first", "engram", "kv-source", "index-source", "kv", "draft"],
ids=[
"standard",
"window",
"engram",
"kv-source",
"index-source",
"compressor-only",
"other-indexer",
"kv",
"draft",
],
)
def test_layers_take_whole_or_ffn_launch(monkeypatch, layer, path):
"""The whole-layer kernels serve the standard layers; every other backbone
layer keeps vLLM's attention and runs its FFN half as one launch."""
"""The whole-layer kernels serve every backbone layer of the checkpoint (an
index source's compressor and indexer run between them); a layer outside
them keeps vLLM's attention and runs its FFN half as one launch."""
_deployment(monkeypatch)
mono = md.MonoDecodeLayer.create(layer, _config())
got = None if mono is None else "ffn" if mono.ffn_only else "whole"
@@ -254,7 +301,7 @@ def test_other_steps_reduce_wo_b_themselves(run, case):
assert out is None and not calls
def _config(**parallel):
def _config(monokernel=True, **parallel):
parallel = (
dict(enable_expert_parallel=False, enable_eplb=False, data_parallel_size=1)
| parallel
@@ -262,6 +309,7 @@ def _config(**parallel):
return SimpleNamespace(
model_config=SimpleNamespace(hf_config=SimpleNamespace(num_hidden_layers=40)),
parallel_config=SimpleNamespace(**parallel),
kernel_config=SimpleNamespace(monokernel=monokernel),
)
@@ -269,7 +317,6 @@ def _deployment(monkeypatch, cdna=4, cus=256):
"""A deployment the kernels take (the runner module stubbed: no FlyDSL)."""
import vllm.platforms.rocm as rocm
monkeypatch.setenv("VLLM_ROCM_MONO_DECODE", "1")
monkeypatch.delenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4", raising=False)
monkeypatch.setattr(rocm, "get_cdna_version", lambda: cdna)
monkeypatch.setattr(md, "get_tensor_model_parallel_world_size", lambda: 2)
@@ -280,6 +327,14 @@ def _deployment(monkeypatch, cdna=4, cus=256):
monkeypatch.setitem(sys.modules, name, runner)
def test_off_without_the_flag(monkeypatch):
"""No --monokernel: no mono layer, whatever the deployment."""
_deployment(monkeypatch)
assert (
md.MonoDecodeLayer.create(_decoder_layer(), _config(monokernel=False)) is None
)
@pytest.mark.parametrize("cdna", [3, 5])
def test_opt_in_outside_the_kernels_cdna_raises(monkeypatch, cdna):
_deployment(monkeypatch, cdna=cdna)
+30 -25
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The DeepSeek-V4.1 mono decode path through the real model: a five-layer
DeepSeek-V4.1-Flash at TP2 and TP4 on CDNA4, served with ``VLLM_ROCM_MONO_DECODE`` on
DeepSeek-V4.1-Flash at TP2 and TP4 on CDNA4, served with ``--monokernel`` on
and off on the same random weights: every decoder layer's outputs at every
decode step must match. Unlike ``test_dsv41_mono_numerics.py``, which
checks the kernels against vLLM's ops one stage at a time, this runs
@@ -10,12 +10,13 @@ the per-step path choice, the weights taken off the layer, the metadata and
the indexer's top-k -- so a change to the decoder layer that the kernels do
not follow fails here.
The five layers hold both kinds the mono path takes:
- 0: sliding window only -> the FFN launch
- 1: ratio-2 KV and index source -> the FFN launch
- 2: ratio-2 consumer -> the whole layer (K1 + K2)
- 3: ratio-1 KV, index and candidate source -> the FFN launch
- 4: ratio-1 consumer -> the whole layer
Every layer takes the whole-layer path (K1 + K2); the five hold each variant:
- 0: the first layer, sliding window only (K1 takes the embedding settled)
- 1: ratio-2 KV and index source (vLLM's compressor and indexer between K1
and K2)
- 2: ratio-2 consumer with an Engram (K1 takes the residual settled)
- 3: ratio-1 KV, index and candidate source
- 4: ratio-1 index source over 3's candidates and KV
"""
import zlib
@@ -34,13 +35,17 @@ TRUNCATED = {
# one entry a layer, the three MTP layers' included
"compress_ratios": [0, 2, 2, 1, 1, 0, 0, 0],
"kv_source_layer_ids": [1, 3],
"index_source_layer_ids": [1, 3],
# 4: an indexer of its own over 3's candidates (a REINDEX layer)
"index_source_layer_ids": [1, 3, 4],
"candidate_source_layer_id": 3,
"engram_layer_ids": [],
"engram_num_embeddings": [],
# an Engram on a whole layer (K1 then takes its residual settled), its
# tables shrunk: 24 prime-sized ranges from 1000
"engram_layer_ids": [2],
"engram_num_embeddings": [32768],
"engram_vocab_size": 1000,
"dspark_target_layer_ids": [2, 3, 4],
}
WHOLE, FFN = (2, 4), (0, 1, 3)
WHOLE, FFN = (0, 1, 2, 3, 4), ()
STEPS = 5 # decode steps after the prefill's token
@@ -227,7 +232,6 @@ def _decode(vllm_runner, monkeypatch, tp: int, mono: bool, prompts, path: str):
from vllm import SamplingParams
from vllm.inputs import TokensPrompt
monkeypatch.setenv("VLLM_ROCM_MONO_DECODE", "1" if mono else "0")
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
with vllm_runner(
@@ -237,6 +241,7 @@ def _decode(vllm_runner, monkeypatch, tp: int, mono: bool, prompts, path: str):
hf_overrides=TRUNCATED,
worker_extension_cls=f"{__name__}.MonoProbe",
moe_backend="aiter",
monokernel=mono,
language_model_only=True,
tokenizer_mode="deepseek_v41",
attention_config={"indexer_kv_dtype": "mxfp4", "indexer_sparse_logits": True},
@@ -300,15 +305,16 @@ def test_mono_decode_matches_the_model(vllm_runner, monkeypatch, tmp_path, tp):
same.add((len(prompt), len(prompt) + step))
# each layer's outputs for those tokens, mono against vLLM's path, token by
# token. Layer 0 sees the same inputs on both paths at a request's first
# decode token: there the bound is test_dsv41_mono_numerics' (about 3x the
# kernels' 0.4%); its later tokens attend over KV each path wrote, 3%. Each
# later layer takes the previous one's slightly different output, which
# random weights amplify (2-4% by layer 4): 8%. A token whose top-6 routing
# is a near-tie on vLLM's path (6th / 7th score within 5e-3: a 0.4% input
# difference moves scores near 0.8 by about 3e-3) may pick another expert;
# once it has, its residual stream differs and the later layers are exempt
# for it. The residual stream: 3% (0.3% seen). A layer the kernels no
# token. Layer 0 sees the same inputs on both paths, but its FFN input
# differs by up to test_dsv41_mono_numerics' 1.2%, which random weights
# amplify through the MoE: 3%. Each later layer takes the previous one's
# slightly different output, which random weights amplify (2-4% by layer
# 4): 8%. A token whose top-6 routing is a near-tie on vLLM's path may pick
# another expert: a 0.4% input difference moves scores near 0.8 by about
# 3e-3, so 6th / 7th scores within 5e-3 at layer 0 and, past it, where the
# input carries the earlier layers' differences (0.4-0.7%), within 1e-2.
# Once it has, its residual stream differs and the later layers are exempt
# for it. The residual stream: 3% (0.5% seen). A layer the kernels no
# longer follow is off by far more, on every token.
for rank in range(tp):
got, _ = torch.load(f"{tmp_path}/on.rank{rank}")
@@ -317,8 +323,6 @@ def test_mono_decode_matches_the_model(vllm_runner, monkeypatch, tmp_path, tp):
same & {k[1:] for k in got} & {k[1:] for k in want}
)
assert len(keys) >= len(prompts), keys
# the bound that sees a 3% error: a request's first decode token
assert any(p == n for n, p in keys), keys
flipped: set = set()
for i in sorted(WHOLE + FFN):
for k in keys:
@@ -327,9 +331,10 @@ def test_mono_decode_matches_the_model(vllm_runner, monkeypatch, tmp_path, tp):
(x, res), (x_ref, res_ref) = got[(i, *k)], want[(i, *k)]
err = _rel(res, res_ref)
assert err < 0.03, (rank, i, k, "residual", err)
bound = (0.015 if k[1] == k[0] else 0.03) if i == 0 else 0.08
bound = 0.03 if i == 0 else 0.08
err = _rel(x, x_ref)
if err >= bound:
assert gaps[(i, *k)] < 5e-3, (rank, i, k, "x", err)
tie = 5e-3 if i == 0 else 1e-2
assert gaps[(i, *k)] < tie, (rank, i, k, "x", err)
flipped.add(k)
assert len(flipped) <= len(keys) // 4, (rank, sorted(flipped))
+6
View File
@@ -291,6 +291,12 @@ class KernelConfig:
enable_jit_warmup: bool = True
"""If True, run JIT compile warmup during kernel warmup."""
monokernel: bool = False
"""If True, run the eligible layers of decode steps on the model's
MonoKernel: a persistent kernel that runs most or all of one decoder layer
in a single launch (DeepSeek-V4.1 on MI355X). A deployment that cannot run
it is refused at startup."""
moe_backend: MoEBackend = "auto"
"""Backend for MoE expert computation kernels. Available options:
+4
View File
@@ -758,6 +758,7 @@ class EngineArgs:
enable_flashinfer_autotune: bool = get_field(
KernelConfig, "enable_flashinfer_autotune"
)
monokernel: bool = get_field(KernelConfig, "monokernel")
worker_cls: str = ParallelConfig.worker_cls
worker_extension_cls: str = ParallelConfig.worker_extension_cls
@@ -1802,6 +1803,7 @@ class EngineArgs:
"--enable-flashinfer-autotune",
**kernel_kwargs["enable_flashinfer_autotune"],
)
kernel_group.add_argument("--monokernel", **kernel_kwargs["monokernel"])
moe_backend_kwargs = kernel_kwargs["moe_backend"]
moe_backend_kwargs["type"] = lambda s: s.lower().replace("-", "_")
kernel_group.add_argument("--moe-backend", **moe_backend_kwargs)
@@ -2754,6 +2756,8 @@ class EngineArgs:
"are mutually exclusive"
)
kernel_config.enable_flashinfer_autotune = self.enable_flashinfer_autotune
if self.monokernel:
kernel_config.monokernel = True
if self.moe_backend != "auto":
kernel_config.moe_backend = self.moe_backend
if self.linear_backend != "auto":
-7
View File
@@ -156,7 +156,6 @@ if TYPE_CHECKING:
VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION: bool = False
VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: bool = False
VLLM_ROCM_AITER_MEGA_MOE_FUSE_SHARED_EXPERTS: bool = False
VLLM_ROCM_MONO_DECODE: bool = False
VLLM_ROCM_USE_AITER_TRITON_GEMM: bool = True
VLLM_ROCM_USE_SKINNY_GEMM: bool = True
VLLM_ROCM_FP8_PADDING: bool = True
@@ -1398,12 +1397,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
os.getenv("VLLM_ROCM_AITER_MEGA_MOE_FUSE_SHARED_EXPERTS", "False").lower()
in ("true", "1")
),
# Run the eligible layers of decode steps on the model's mono decode
# kernels: persistent launches a layer, the TP all-reduces in-kernel
# (DeepSeek-V4.1 on CDNA4). By default is disabled.
"VLLM_ROCM_MONO_DECODE": lambda: (
os.getenv("VLLM_ROCM_MONO_DECODE", "False").lower() in ("true", "1")
),
# Whether to use aiter triton kernels for gemm ops.
# By default is enabled.
"VLLM_ROCM_USE_AITER_TRITON_GEMM": lambda: (
+11 -1
View File
@@ -282,7 +282,17 @@ class DeepseekV4DecoderLayer(nn.Module):
engram_mask: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
if self.mono is not None:
out = self.mono(self, x, positions, residual, post_mix, res_mix, pre_mix)
out = self.mono(
self,
x,
positions,
residual,
post_mix,
res_mix,
pre_mix,
engram_hashes,
engram_mask,
)
if out is not None:
return out
# Layer 0's attention seam projects the 2-D embedding with the folded
+13 -10
View File
@@ -2,9 +2,10 @@
One fused decoder layer for vLLM's decode steps on MI355X: the attention seam,
the attention, the attention all-reduce, the FFN seam, the MoE and the MoE
all-reduce, in two persistent launches a layer. The other backbone layers keep
vLLM's attention and run the rest -- the attention all-reduce, the FFN seam,
the MoE and its all-reduce -- as one launch (the FFN launch). ATOM's V4.1 mono decode
all-reduce, in two persistent launches a layer. A layer outside the whole-layer
kernels keeps vLLM's attention and runs the rest -- the attention all-reduce,
the FFN seam, the MoE and its all-reduce -- as one launch (the FFN launch).
ATOM's V4.1 mono decode
(ROCm/ATOM `atom/models/deepseek_v41/mono`) is the reference design: the seam
and MoE stages (`stages/`) and the framework under them (`common/`) are adapted
from it, each file naming its origin; the attention stages (`attention/`) are
@@ -12,13 +13,15 @@ new, reading vLLM's KV records and weights as loaded.
## Scope
- Layers: the whole layer on every backbone layer with the standard seam and no
compressor / indexer: 3-7, 9-13, 15-19 (compress ratio 2) and 21-23, 25-27,
29-31, 33-35, 37-39 (ratio 1), 30 of 40. The FFN launch on the other 10:
layer 0 (embedding-broadcast seam), layer 1 (Engram at the seam) and the
compressor / indexer layers 2, 8, 14 (also Engram), 20, 24, 28, 32, 36,
whose wo_b leaves its TP reduce to it. The DSpark draft layers keep vLLM's
path. The paths interleave freely (same tensors at the layer boundary).
- Layers: the whole layer on all 40 backbone layers. K1 takes the residual
settled, without folding the previous output in, on layer 0 (the embedding
broadcast to the four streams) and after an Engram write (layers 1 and 14,
vLLM's Engram running before K1). On the index sources (2, 8, 14, 20 with
their compressor; 24, 28, 32, 36) K1 also hands out its normed input and bf16
qr, vLLM's compressor and indexer run between K1 and K2, and K2 builds the
key table from that top-k. The FFN launch serves a layer outside these (an
MXFP8 KV cache, another indexer); the DSpark draft layers keep vLLM's path.
The paths interleave freely (same tensors at the layer boundary).
- Steps: decode only (FULL graph or eager), M <= 48 rows (8 requests x 6
DSpark tokens), TP2 / TP4, `fp8_ds_mla` KV records (ROCm), text tokens (the
routing bias is `e_score_correction_bias`).
@@ -151,7 +151,9 @@ def key_rec(c, t, k):
table: slot | 1 << 31 for a compressed row, a window slot otherwise, -1 for
none (it reads a zero record: its scores are masked, its values zeros)."""
a, ratio = c["args"], c["ratio"]
val = ld_i32(a["kt"], t * KEYS + fx.min(fx.max(k, 0), KEYS - 1))
# a table this launch wrote (``krdy``) is read past the CU's cache
cm = CM_DEV if const_expr("krdy" in c) else 0
val = ld_i32(a["kt"], t * KEYS + fx.min(fx.max(k, 0), KEYS - 1), cm)
ok = val != fx.Int32(-1)
slot = ok.select(val & fx.Int32(0x7FFFFFFF), fx.Int32(0))
if const_expr(ratio > 0):
@@ -209,9 +211,11 @@ def stage_split(c, unit):
g = lane // 16
r16 = lane % 16
k0 = ch * ck
if const_expr("krdy" in c):
await_flags(c, c["krdy"], 1, lambda u: t)
# the key count, this lane's key and q: independent loads, out together
# (K1's key table holds -1 past a token's keys)
kv_len = ld_i32(a["klen"], t)
# (the key table holds -1 past a token's keys)
kv_len = ld_i32(a["klen"], t, CM_DEV if const_expr("krdy" in c) else 0)
key = k0 + kt * 16 + r16
data, sc, okv = key_rec(c, t, key)
qrow = d.heads * HEAD_DIM // 8
@@ -188,6 +188,8 @@ def stage_kt(c, t):
ntopk."""
tid = c["tid"]
a, ratio = c["args"], c["ratio"]
# device scope where this launch reads the table (K2's)
cm = CM_DEV if const_expr(c.get("dev", False)) else 0
slot = ld_i32(a["slot"], 2 * t)
live = slot >= 0
nswa = live.select(ld_i32(a["swa_lens"], t), fx.Int32(0))
@@ -212,10 +214,10 @@ def stage_kt(c, t):
c_val = (local >= 0).select(c_slot, fx.Int32(-1))
val = (k < ntopk).select(c_val, val)
if k < KEYS:
bo.buffer_store(val, rsrc(a["kt"]), t * KEYS + k)
bo.buffer_store(val, rsrc(a["kt"]), t * KEYS + k, cache_modifier=cm)
if tid == 0:
bo.buffer_store(kv_len, rsrc(a["klen"]), t)
bo.buffer_store(ntopk, rsrc(a["klen"]), c["S"] + t)
bo.buffer_store(kv_len, rsrc(a["klen"]), t, cache_modifier=cm)
bo.buffer_store(ntopk, rsrc(a["klen"]), c["S"] + t, cache_modifier=cm)
# ---------------------------------------------------------------- wqkv_a
@@ -357,6 +359,20 @@ def stage_qkv(c, t):
amax = fx.max(amax, abs(y))
code = mx_code(butterfly(amax, (1, 2), fx.max))
mul = mx_mul(code)
if const_expr(c.get("index", False)): # noqa: SIM102
# the bf16 qr, vLLM's indexer's input
if mine & (tt * 8 < Q_RANK):
words = [
fx.Vector.from_elements([ys[2 * i], ys[2 * i + 1]], fx.Float32)
.to(fx.BFloat16)
.bitcast(fx.Int32)[0]
for i in range(4)
]
bo.buffer_store(
fx.Vector.from_elements(words, fx.Int32),
rsrc(a["qr_out"]),
t * (Q_RANK // 2) + 4 * tt,
)
if mine & (tt * 8 < Q_RANK):
w0 = fp8_pack4(*[clamp_fp8(ys[i] * mul) for i in range(4)])
w1 = fp8_pack4(*[clamp_fp8(ys[4 + i] * mul) for i in range(4)])
@@ -597,8 +613,10 @@ def run_front(c, key, bid, x8_given=False, before_wqkv=None, after_wqkv=None):
u = free + n_free * tid
if u < xq_units:
c["mb"].put(c["xrdy"], u, fx.Int32(1))
for t in range(is_free.select(free, fx.Int32(s)), s, n_free):
stage_kt(c, t)
if const_expr(not c.get("index", False)):
# an index source's top-k comes after this launch: K2 builds its table
for t in range(is_free.select(free, fx.Int32(s)), s, n_free):
stage_kt(c, t)
stamp(c, 1)
poss = [wqb_pos(c, t0, n) for t0, n in tiles]
+81 -3
View File
@@ -58,6 +58,12 @@ class MonoBuild:
tp: int
ratio: int # the layer's compress ratio (its attention's key sources)
timeline: bool = False
# K1's seam folds the previous layer's output into the residual; off for a
# residual that arrives settled (the first layer, after an Engram)
fold: bool = True
# the layer selects its own top-k (an index source): K1 hands its normed
# input and qr out for vLLM's indexer, K2 builds the key table
index: bool = False
def _moe_key(key: MonoBuild) -> MoeBuild:
@@ -100,6 +106,7 @@ def scratch_layout(s: int, tp: int) -> dict:
lo = min(o for o, _ in mr.values())
place({n: (o - lo, n_) for n, (o, n_) in mr.items()})
place({"normed": (0, s * HIDDEN * 2), "xrdy_moe": (s * HIDDEN * 2 + 256, s * 8)})
place({"krdy": (0, s * 8)})
return out
@@ -175,7 +182,15 @@ def build_mono_k1(key: MonoBuild):
seam: SeamLds
front: FrontLds # type: ignore[valid-type]
name = kernel_symbol("dsv41_mono_k1", s=s, tp=tp, r=key.ratio, tl=key.timeline)
name = kernel_symbol(
"dsv41_mono_k1",
s=s,
tp=tp,
r=key.ratio,
tl=key.timeline,
f=key.fold,
i=key.index,
)
keyed = key_tuple(key, SOURCES)
@flyc.kernel(name=name, known_block_size=[THREADS, 1, 1])
@@ -215,6 +230,8 @@ def build_mono_k1(key: MonoBuild):
t2r: Int64,
kt: Int64,
klen: Int64,
normed_out: Int64,
qr_out: Int64,
scratch: Int64,
epoch: Int64,
tl: Int64,
@@ -250,9 +267,11 @@ def build_mono_k1(key: MonoBuild):
"t2r": t2r,
"kt": kt,
"klen": klen,
"qr_out": qr_out,
}
flayout = {n: layout[n] for n in front_scratch(s, Dims(tp))}
c = mb_front.front_context(fkey, lds.front.peek(), fargs, scratch, tag, flayout)
c["index"] = key.index
amb = Mailbox(tag)
tid = fx.thread_idx.x
ca = {
@@ -284,6 +303,7 @@ def build_mono_k1(key: MonoBuild):
"pre_out": pre_out,
},
"d": StageDims(tp),
"fold": key.fold,
}
for region in ("lin", "pmix"):
ca[region] = sreg(scratch, layout[region][0], region)
@@ -295,9 +315,14 @@ def build_mono_k1(key: MonoBuild):
seam.stage_slice(ca, task)
gpu.barrier()
def store(*v):
store_x8(c, *v)
if const_expr(key.index):
store_normed(normed_out, *v)
def seam_rest():
for t in range(first_task(bid, NORM0), s, BLOCKS):
seam.stage_norm(ca, t, lambda *v: store_x8(c, *v))
seam.stage_norm(ca, t, store)
publish_x8(c, t)
for t in range(first_task(bid, GATE0), s, BLOCKS):
seam.stage_gate(ca, t)
@@ -344,6 +369,8 @@ def build_mono_k1(key: MonoBuild):
t2r: Int64,
kt: Int64,
klen: Int64,
normed_out: Int64,
qr_out: Int64,
scratch: Int64,
epoch: Int64,
tl: Int64,
@@ -386,6 +413,8 @@ def build_mono_k1(key: MonoBuild):
t2r,
kt,
klen,
normed_out,
qr_out,
scratch,
epoch,
tl,
@@ -544,7 +573,9 @@ def build_mono_k2(key: MonoBuild):
seam: SeamLds # type: ignore[valid-type]
moe: MoeLds # type: ignore[valid-type]
name = kernel_symbol("dsv41_mono_k2", s=s, tp=tp, r=key.ratio, tl=key.timeline)
name = kernel_symbol(
"dsv41_mono_k2", s=s, tp=tp, r=key.ratio, tl=key.timeline, i=key.index
)
keyed = key_tuple(key, SOURCES)
@flyc.kernel(name=name, known_block_size=[THREADS, 1, 1])
@@ -558,6 +589,13 @@ def build_mono_k2(key: MonoBuild):
comp_block: Int32,
kt: Int64,
klen: Int64,
slot: Int64,
swa_idx: Int64,
swa_lens: Int64,
t2r: Int64,
topk: Int64,
comp_bt: Int64,
bt_stride: Int32,
pos: Int64,
sink: Int64,
qk_scale: Int32,
@@ -642,6 +680,32 @@ def build_mono_k2(key: MonoBuild):
if const_expr(key.timeline):
c["tl"], c["tl_points"] = tl, mb_back.BACK_POINTS
mb_back.epoch_begin(c, epoch, ep)
if const_expr(key.index):
# the key table, from the top-k vLLM's indexer selected after K1,
# on the last CTAs: each split task waits on its token's
ckt = {
"S": s,
"tid": c["tid"],
"dev": True,
"ratio": key.ratio,
"args": {
"slot": slot,
"swa_idx": swa_idx,
"swa_lens": swa_lens,
"pos": pos,
"t2r": t2r,
"topk": topk,
"comp_block": comp_block,
"comp_bt": comp_bt,
"bt_stride": bt_stride,
"kt": kt,
"klen": klen,
},
}
c["krdy"] = sreg(scratch, layout["krdy"][0], "krdy")
for t in range(first_task(bid, BLOCKS - s), s, BLOCKS):
mb_front.stage_kt(ckt, t)
mb_back.publish(c, c["krdy"], t)
mb_back.run_back(c, bkey, bid)
gpu.barrier()
@@ -685,6 +749,13 @@ def build_mono_k2(key: MonoBuild):
comp_block: Int32,
kt: Int64,
klen: Int64,
slot: Int64,
swa_idx: Int64,
swa_lens: Int64,
t2r: Int64,
topk: Int64,
comp_bt: Int64,
bt_stride: Int32,
pos: Int64,
sink: Int64,
qk_scale: Int32,
@@ -736,6 +807,13 @@ def build_mono_k2(key: MonoBuild):
comp_block,
kt,
klen,
slot,
swa_idx,
swa_lens,
t2r,
topk,
comp_bt,
bt_stride,
pos,
sink,
qk_scale,
+33 -6
View File
@@ -16,12 +16,13 @@ the same launches, so the ranks' epochs agree.
from __future__ import annotations
import os
from collections.abc import Callable
from dataclasses import dataclass
from pathlib import Path
import torch
from .attention.plan import HEAD_DIM, HIDDEN, KEYS, Dims
from .attention.plan import HEAD_DIM, HIDDEN, KEYS, Q_RANK, Dims
from .common.plan import BLOCKS
from .layer import (
EPOCH_WORDS,
@@ -170,6 +171,12 @@ class DSV41MonoLayer:
self.klen = torch.zeros(2 * MAX_TOKENS, dtype=torch.int32, device=dev)
self._dummy = torch.zeros(4, dtype=torch.int32, device=dev)
self._zero_rec = torch.zeros(1024, dtype=torch.uint8, device=dev)
# the first layer's incoming pre-mix: stream 0 of the broadcast embedding
self._first_pre = torch.zeros(MAX_TOKENS, HC, dtype=torch.float32, device=dev)
self._first_pre[:, 0] = 1.0
# an index source's K1 outputs, vLLM's indexer's inputs
self.normed = torch.zeros(MAX_TOKENS, HIDDEN, dtype=torch.bfloat16, device=dev)
self.qr = torch.zeros(MAX_TOKENS, Q_RANK, dtype=torch.bfloat16, device=dev)
# the attention seam's outputs, K1 -> K2
self.res_mid = torch.zeros(
MAX_TOKENS, HC, HIDDEN, dtype=torch.bfloat16, device=dev
@@ -203,14 +210,17 @@ class DSV41MonoLayer:
self._scratch[tokens] = buf
return buf
def first_pre_mix(self, tokens: int) -> torch.Tensor:
return self._first_pre[:tokens]
@staticmethod
def supports(tokens: int) -> bool:
return 1 <= tokens <= MAX_TOKENS
def kernels(self, tokens: int, ratio: int):
key = (tokens, ratio)
def kernels(self, tokens: int, ratio: int, fold: bool = True, index: bool = False):
key = (tokens, ratio, fold, index)
if key not in self._kernels:
b = MonoBuild(tokens, self.tp, ratio)
b = MonoBuild(tokens, self.tp, ratio, fold=fold, index=index)
self._kernels[key] = (build_mono_k1(b), build_mono_k2(b))
return self._kernels[key]
@@ -249,15 +259,21 @@ class DSV41MonoLayer:
comp_cache: torch.Tensor | None = None,
comp_block_table: torch.Tensor | None = None,
outs: tuple | None = None,
fold: bool = True,
select: Callable[[torch.Tensor, torch.Tensor], None] | None = None,
):
"""-> (out, residual, post_mix, res_mix, pre_mix): the layer's outputs
(``outs``: buffers to write them into)."""
(``outs``: buffers to write them into). ``select(normed, qr)``: an index
source's top-k selection into ``topk_indices``, run between the
kernels on K1's normed input and qr."""
M = x.shape[0]
assert self.supports(M), M
a = w.attn
assert a is not None
ratio = a.ratio
k1, k2 = self.kernels(M, ratio)
index = select is not None
assert not index or ratio
k1, k2 = self.kernels(M, ratio, fold, index)
st = torch.cuda.current_stream()
if outs is None:
outs = self._outs(M, residual)
@@ -324,11 +340,15 @@ class DSV41MonoLayer:
token_to_req.data_ptr(),
self.kt.data_ptr(),
self.klen.data_ptr(),
self.normed.data_ptr(),
self.qr.data_ptr(),
self.scratch(M).data_ptr(),
self.epoch.data_ptr(),
0,
stream=st,
)
if select is not None:
select(self.normed[:M], self.qr[:M])
k2(
self.q.data_ptr(),
swa_cache.data_ptr(),
@@ -339,6 +359,13 @@ class DSV41MonoLayer:
comp_block,
self.kt.data_ptr(),
self.klen.data_ptr(),
slot_mapping.data_ptr(),
swa_indices.data_ptr(),
swa_lens.data_ptr(),
token_to_req.data_ptr(),
topk.data_ptr(),
bt.data_ptr(),
bt_stride,
positions.data_ptr(),
a.attn_sink.data_ptr(),
self._qk_scale,
@@ -141,6 +141,8 @@ def slice_loads(c, c0, e):
"""Element e's loads of ``stage_slice`` (e clamped into the step's)."""
a = c["args"]
t, o, col = slice_at(c0, e)
if const_expr(not c.get("fold", True)):
return (ld_bf(a["res_in"], (t * HC + o) * HIDDEN + col),)
r = [ld_bf(a["res_in"], (t * HC + h) * HIDDEN + col) for h in range(HC)]
cm = [ld_f32(a["comb_in"], (t * HC + h) * HC + o) for h in range(HC)]
return r, cm, ld_f32(a["post_in"], t * HC + o)
@@ -150,15 +152,19 @@ def slice_row(c, c0, e, got):
"""Element e of ``stage_slice``'s R_new from ``slice_loads``' values."""
a = c["args"]
t, o, col = slice_at(c0, e)
r, cm, post = got
# aiter's mhc_fused_post_pre_delayed_rmsnorm: ((post x + c0 r0)
# + c1 r1) + c2 r2, each product rounded, then fma(c3, r3, ..)
v = post * pend_value(c, t, col, e % COLS) + (r[0] * cm[0])
for h in range_constexpr(1, HC - 1):
v = v + cm[h] * r[h]
# bf16 R_new is what the collapse, the sums of squares and the
# mix projection all read
v = bf16_round(fma(cm[HC - 1], r[HC - 1], v))
if const_expr(not c.get("fold", True)):
# a settled residual (no owed post): stream o as it arrives
v = got[0]
else:
r, cm, post = got
# aiter's mhc_fused_post_pre_delayed_rmsnorm: ((post x + c0 r0)
# + c1 r1) + c2 r2, each product rounded, then fma(c3, r3, ..)
v = post * pend_value(c, t, col, e % COLS) + (r[0] * cm[0])
for h in range_constexpr(1, HC - 1):
v = v + cm[h] * r[h]
# bf16 R_new is what the collapse, the sums of squares and the
# mix projection all read
v = bf16_round(fma(cm[HC - 1], r[HC - 1], v))
bo.buffer_store(v.to(fx.BFloat16), rsrc(a["res_out"]), (t * HC + o) * HIDDEN + col)
fx.ptr_store(v, c["rl"] + (t * KT + o * COLS + e % COLS))
+87 -28
View File
@@ -1,15 +1,16 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DeepSeek-V4.1 mono decode layer on ROCm CDNA4 (``VLLM_ROCM_MONO_DECODE=1``).
"""DeepSeek-V4.1 mono decode layer on ROCm CDNA4 (``--monokernel``).
A decode step's backbone layer -- the attention seam, the attention, its TP
all-reduce, the FFN seam, the MoE and its all-reduce -- runs as two persistent
FlyDSL launches (``mono``, this directory) on the layer's loaded
weights, the TP reductions inside the kernels. A layer runs whole when it has
the standard seam and no compressor or indexer of its own (compress ratio 1 or
2) over ``fp8_ds_mla`` records. Any other backbone layer (the first, the Engram
layers, the compressor / indexer layers) keeps vLLM's attention seam and
attention and runs the rest as one launch, the FFN launch, from wo_b's
weights, the TP reductions inside the kernels. Every backbone layer of the
checkpoint runs whole (compress ratio 0, 1 or 2 over ``fp8_ds_mla`` records):
the first layer and the Engram layers hand K1 a settled residual, and an index
source runs vLLM's compressor and indexer between the two launches. A layer
outside that (an MXFP8 KV cache, another indexer) keeps vLLM's attention seam
and attention and runs the rest as one launch, the FFN launch, from wo_b's
unreduced output. A step takes the kernels when it is decode only, at most
``MAX_ROWS`` rows (eager or a FULL CUDA graph: the model is not torch.compiled);
a whole layer also needs causal SWA windows. Anything else runs the layer as
@@ -20,7 +21,7 @@ Every TP rank must take the same path at every layer (the kernels wait on each
other's pushes): the conditions read only metadata every rank holds alike.
"""
from typing import TYPE_CHECKING, cast
from typing import TYPE_CHECKING, Any, cast
import torch
@@ -62,7 +63,7 @@ def _mono_runner(device: torch.device):
# its peer memory is allocated and exchanged eagerly; vLLM warms
# every graph up eagerly before capturing it
raise RuntimeError(
"VLLM_ROCM_MONO_DECODE was first reached inside a CUDA graph "
"--monokernel was first reached inside a CUDA graph "
"capture, before an eager decode step."
)
from vllm.models.deepseek_v41.amd.mono.runner import MAX_TOKENS, DSV41MonoLayer
@@ -113,6 +114,35 @@ def _unsupported_moe(layer: "DeepseekV4DecoderLayer", vllm_config) -> str | None
return None
def _select(
layer: "DeepseekV4DecoderLayer",
positions: torch.Tensor,
normed: torch.Tensor,
qr: torch.Tensor,
) -> None:
"""An index source's compressor and top-k selection, from K1's normed input
and bf16 qr: ``_forward_csa2_full`` / ``_forward_csa2_reindex`` and the
indexer op, serial."""
attn = layer.attn
comp, indexer = attn.compressor, attn.indexer
assert indexer is not None
if comp is not None:
kv_score = torch.mm(
normed, comp.fused_wkv_wgate.weight.T, out_dtype=torch.float32
)
latent = comp(kv_score, positions)
comp.insert_cache(latent, positions, attn.rotary_emb)
indexer._produce_k(latent, positions, attn.indexer_rotary_emb)
weights, _ = indexer.weights_proj(normed)
q, q_scale, weights = indexer.forward_q(
qr, None, weights, positions, attn.indexer_rotary_emb
)
assert weights is not None
indexer.indexer_op(
normed, (q, q_scale) if q_scale is not None else q, None, weights
)
class MonoDecodeLayer:
"""One decoder layer's mono path (``create``: None when the layer is not
eligible): the whole layer (``__call__``) or, with ``ffn_only``, the FFN
@@ -127,7 +157,7 @@ class MonoDecodeLayer:
def create(
layer: "DeepseekV4DecoderLayer", vllm_config
) -> "MonoDecodeLayer | None":
if not envs.VLLM_ROCM_MONO_DECODE:
if not vllm_config.kernel_config.monokernel:
return None
from vllm.platforms.rocm import get_cdna_version
@@ -150,7 +180,7 @@ class MonoDecodeLayer:
if cus < BLOCKS:
why = f"needs {BLOCKS} compute units, the GPU has {cus}"
if why is not None:
raise ValueError(f"VLLM_ROCM_MONO_DECODE {why}.")
raise ValueError(f"--monokernel {why}.")
# the layer: the kernels serve the backbone's seams and MoE
config = vllm_config.model_config.hf_config
attn, ffn = layer.attn, layer.ffn
@@ -174,12 +204,17 @@ class MonoDecodeLayer:
if why is not None:
logger.info_once("DSv4.1 mono decode off for some layers: %s", why)
return None
# the whole layer: the kernels' attention takes the standard layers only
if layer.engram is not None:
why = "Engram"
elif attn.compressor is not None or attn.indexer is not None:
why = "a compressor / indexer"
elif attn.compress_ratio not in (1, 2):
# the whole layer: the kernels' attention takes the standard layers; an
# index source's compressor and indexer run between its two launches
from vllm.models.deepseek_v41.amd.rocm import DeepseekV41RocmMxfp4Indexer
if attn.compressor is not None and attn.indexer is None:
why = "a compressor without an indexer"
elif attn.indexer is not None and not isinstance(
attn.indexer, DeepseekV41RocmMxfp4Indexer
):
why = f"a {type(attn.indexer).__name__}"
elif attn.compress_ratio not in (0, 1, 2):
why = f"compress ratio {attn.compress_ratio}"
elif attn.kv_mxfp8 or attn.kv_cache_dtype != "fp8_ds_mla":
why = f"a {attn.kv_cache_dtype} KV cache"
@@ -297,20 +332,23 @@ class MonoDecodeLayer:
post_mix: torch.Tensor | None,
res_mix: torch.Tensor | None,
pre_mix: torch.Tensor | None,
engram_hashes: torch.Tensor | None = None,
engram_mask: torch.Tensor | None = None,
) -> tuple[torch.Tensor, ...] | None:
"""The layer's outputs (x, residual, post_mix, res_mix, pre_mix) by the
mono kernels, or None when this step takes the original path (always for
an ``ffn_only`` layer)."""
if self.ffn_only:
return None
if residual is None or x.dim() != 2:
return self._skip("first layer")
first = residual is None
if first and x.dim() != 2:
return self._skip("first layer of a pipeline stage")
why, swa, comp = self._decode_step(layer, x.shape[0])
attn = layer.attn
tensors = (x, residual, post_mix, res_mix, pre_mix)
tensors = (x,) if first else (x, residual, post_mix, res_mix, pre_mix)
if why is None:
# the kernels' attention: causal SWA windows, the compressed cache
if comp is None:
if comp is None and attn.compress_ratio:
why = "layer metadata missing"
elif swa.decode_swa_width != SWA_WIDTH:
why = f"SWA width {swa.decode_swa_width}"
@@ -324,12 +362,37 @@ class MonoDecodeLayer:
why = "non-contiguous layer inputs"
if why is not None:
return self._skip(why)
assert swa is not None and comp is not None
assert swa is not None
runner = _mono_runner(x.device)
logger.info_once(
"DSv4.1 mono decode layer active (decode steps of <= %d rows)", MAX_ROWS
)
out = runner.forward(
# K1's seam folds the owed post into the residual, except where the
# residual arrives settled: the embedding, or after the Engram write
fold = True
if first:
residual = x.unsqueeze(1).expand(-1, layer.hc_mult, -1).contiguous()
pre_mix = runner.first_pre_mix(x.shape[0])
post_mix = res_mix = pre_mix # unread without the fold
fold = False
elif layer.engram is not None and engram_hashes is not None:
residual = layer.mhc_post(x, residual, post_mix, res_mix)
hashes = engram_hashes[:, layer.engram.layer_hash_index]
residual = layer.engram(residual, hashes, engram_mask)
fold = False
kw: dict[str, Any] = {}
if attn.indexer is not None:
kw["select"] = lambda normed, qr: _select(layer, positions, normed, qr)
if attn.compress_ratio:
assert comp is not None
kw.update(
topk_indices=attn.topk_indices_buffer,
comp_cache=_record_view(
attn._compressed_kv_cache(), comp.block_size // attn.compress_ratio
),
comp_block_table=comp.block_table,
)
return runner.forward(
self.weights(layer),
x,
residual,
@@ -342,13 +405,9 @@ class MonoDecodeLayer:
swa.decode_swa_indices,
swa.decode_swa_lens,
swa.token_to_req_indices,
topk_indices=attn.topk_indices_buffer,
comp_cache=_record_view(
attn._compressed_kv_cache(), comp.block_size // attn.compress_ratio
),
comp_block_table=comp.block_table,
fold=fold,
**kw,
)
return out
def ffn(
self,