[ROCm][Perf][DSv4.1] Mono decode on every backbone layer (#61082)

Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Fangzhou Ai
2026-10-13 10:16:50 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent f47792af7f
commit cc2a6eea1a
13 changed files with 379 additions and 111 deletions
+69 -14
View File
@@ -27,21 +27,25 @@ class _Runner:
self.calls.append((args, kwargs))
return ("mono outputs",)
def first_pre_mix(self, rows):
return torch.zeros(rows, 4)
def ffn(self, *args):
self.calls.append((args, {}))
return ("ffn outputs",)
def _layer(comp="comp"):
def _layer(comp="comp", indexer=None):
cache = lambda: torch.zeros(4, 128 * md.RECORD, dtype=torch.uint8) # noqa: E731
attn = SimpleNamespace(
swa_cache_layer=SimpleNamespace(prefix="swa", kv_cache=cache()),
compressed_cache_prefix=comp,
topk_indices_buffer=torch.zeros(64, 512, dtype=torch.int32),
compress_ratio=1,
indexer=indexer,
_compressed_kv_cache=cache,
)
return SimpleNamespace(attn=attn)
return SimpleNamespace(attn=attn, engram=None, hc_mult=4)
def _swa(**overrides):
@@ -77,7 +81,12 @@ def run(monkeypatch):
``mode`` and ``metadata``; returns (result, runner calls)."""
def _run(
mode=CUDAGraphMode.FULL, metadata=None, ffn_only=False, entry=None, **inputs
mode=CUDAGraphMode.FULL,
metadata=None,
ffn_only=False,
entry=None,
layer=None,
**inputs,
):
if metadata is None:
metadata = {
@@ -97,7 +106,7 @@ def run(monkeypatch):
# has no compressed cache
part, _ = args.pop("x"), args.pop("positions")
return mono.ffn(_layer(comp=None), part, **args), runner.calls
return mono(_layer(), **args), runner.calls
return mono(layer or _layer(), **args), runner.calls
return _run
@@ -114,6 +123,23 @@ def test_decode_step_runs_mono(run, mode):
1,
)
assert kwargs["comp_cache"].shape == (4, 128, md.RECORD)
assert kwargs["fold"] and "select" not in kwargs
def test_first_layer_runs_mono_from_the_embedding(run):
"""The first layer's residual is the embedding broadcast to the four
streams, which K1 takes as it is (no owed post to fold)."""
out, calls = run(residual=None, post_mix=None, res_mix=None, pre_mix=None)
assert out == ("mono outputs",) and len(calls) == 1
args, kwargs = calls[0]
assert args[2].shape == (M, 4, 5120) and not kwargs["fold"]
def test_index_source_selects_between_the_kernels(run):
"""An index source hands the runner its compressor and indexer to run
between K1 and K2."""
out, calls = run(layer=_layer(indexer=object()))
assert out == ("mono outputs",) and callable(calls[0][1]["select"])
def test_padded_decode_step_runs_mono(run):
@@ -149,7 +175,9 @@ def test_padded_decode_step_runs_mono(run):
),
dict(metadata={}), # a profile / dummy run: no attention metadata
dict(**_inputs(md.MAX_ROWS + 6)),
dict(residual=None), # the first layer's seam broadcasts the embedding
# a later pipeline stage's first layer: its stream arrives without
# the mixes
dict(residual=None, x=torch.zeros(M, 4, 5120, dtype=torch.bfloat16)),
],
ids=[
"piecewise",
@@ -158,7 +186,7 @@ def test_padded_decode_step_runs_mono(run):
"tokens-past-rows",
"no-metadata",
"rows",
"first-layer",
"stage-input",
],
)
def test_other_steps_take_the_original_path(run, case):
@@ -202,22 +230,41 @@ def _decoder_layer(engram=None, **attn):
)
def _indexer():
from vllm.models.deepseek_v41.amd.rocm import DeepseekV41RocmMxfp4Indexer
return object.__new__(DeepseekV41RocmMxfp4Indexer)
@pytest.mark.parametrize(
"layer, path",
[
(_decoder_layer(), "whole"),
(_decoder_layer(compress_ratio=0), "ffn"),
(_decoder_layer(engram=object()), "ffn"),
(_decoder_layer(compressor=object(), indexer=object()), "ffn"),
(_decoder_layer(compress_ratio=0), "whole"),
(_decoder_layer(engram=object()), "whole"),
(_decoder_layer(compressor=object(), indexer=_indexer()), "whole"),
(_decoder_layer(indexer=_indexer()), "whole"),
(_decoder_layer(compressor=object()), "ffn"),
(_decoder_layer(indexer=object()), "ffn"),
(_decoder_layer(kv_cache_dtype="auto"), "ffn"),
(_decoder_layer(layer_id=40), None),
],
ids=["standard", "first", "engram", "kv-source", "index-source", "kv", "draft"],
ids=[
"standard",
"window",
"engram",
"kv-source",
"index-source",
"compressor-only",
"other-indexer",
"kv",
"draft",
],
)
def test_layers_take_whole_or_ffn_launch(monkeypatch, layer, path):
"""The whole-layer kernels serve the standard layers; every other backbone
layer keeps vLLM's attention and runs its FFN half as one launch."""
"""The whole-layer kernels serve every backbone layer of the checkpoint (an
index source's compressor and indexer run between them); a layer outside
them keeps vLLM's attention and runs its FFN half as one launch."""
_deployment(monkeypatch)
mono = md.MonoDecodeLayer.create(layer, _config())
got = None if mono is None else "ffn" if mono.ffn_only else "whole"
@@ -254,7 +301,7 @@ def test_other_steps_reduce_wo_b_themselves(run, case):
assert out is None and not calls
def _config(**parallel):
def _config(monokernel=True, **parallel):
parallel = (
dict(enable_expert_parallel=False, enable_eplb=False, data_parallel_size=1)
| parallel
@@ -262,6 +309,7 @@ def _config(**parallel):
return SimpleNamespace(
model_config=SimpleNamespace(hf_config=SimpleNamespace(num_hidden_layers=40)),
parallel_config=SimpleNamespace(**parallel),
kernel_config=SimpleNamespace(monokernel=monokernel),
)
@@ -269,7 +317,6 @@ def _deployment(monkeypatch, cdna=4, cus=256):
"""A deployment the kernels take (the runner module stubbed: no FlyDSL)."""
import vllm.platforms.rocm as rocm
monkeypatch.setenv("VLLM_ROCM_MONO_DECODE", "1")
monkeypatch.delenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4", raising=False)
monkeypatch.setattr(rocm, "get_cdna_version", lambda: cdna)
monkeypatch.setattr(md, "get_tensor_model_parallel_world_size", lambda: 2)
@@ -280,6 +327,14 @@ def _deployment(monkeypatch, cdna=4, cus=256):
monkeypatch.setitem(sys.modules, name, runner)
def test_off_without_the_flag(monkeypatch):
"""No --monokernel: no mono layer, whatever the deployment."""
_deployment(monkeypatch)
assert (
md.MonoDecodeLayer.create(_decoder_layer(), _config(monokernel=False)) is None
)
@pytest.mark.parametrize("cdna", [3, 5])
def test_opt_in_outside_the_kernels_cdna_raises(monkeypatch, cdna):
_deployment(monkeypatch, cdna=cdna)
+30 -25
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The DeepSeek-V4.1 mono decode path through the real model: a five-layer
DeepSeek-V4.1-Flash at TP2 and TP4 on CDNA4, served with ``VLLM_ROCM_MONO_DECODE`` on
DeepSeek-V4.1-Flash at TP2 and TP4 on CDNA4, served with ``--monokernel`` on
and off on the same random weights: every decoder layer's outputs at every
decode step must match. Unlike ``test_dsv41_mono_numerics.py``, which
checks the kernels against vLLM's ops one stage at a time, this runs
@@ -10,12 +10,13 @@ the per-step path choice, the weights taken off the layer, the metadata and
the indexer's top-k -- so a change to the decoder layer that the kernels do
not follow fails here.
The five layers hold both kinds the mono path takes:
- 0: sliding window only -> the FFN launch
- 1: ratio-2 KV and index source -> the FFN launch
- 2: ratio-2 consumer -> the whole layer (K1 + K2)
- 3: ratio-1 KV, index and candidate source -> the FFN launch
- 4: ratio-1 consumer -> the whole layer
Every layer takes the whole-layer path (K1 + K2); the five hold each variant:
- 0: the first layer, sliding window only (K1 takes the embedding settled)
- 1: ratio-2 KV and index source (vLLM's compressor and indexer between K1
and K2)
- 2: ratio-2 consumer with an Engram (K1 takes the residual settled)
- 3: ratio-1 KV, index and candidate source
- 4: ratio-1 index source over 3's candidates and KV
"""
import zlib
@@ -34,13 +35,17 @@ TRUNCATED = {
# one entry a layer, the three MTP layers' included
"compress_ratios": [0, 2, 2, 1, 1, 0, 0, 0],
"kv_source_layer_ids": [1, 3],
"index_source_layer_ids": [1, 3],
# 4: an indexer of its own over 3's candidates (a REINDEX layer)
"index_source_layer_ids": [1, 3, 4],
"candidate_source_layer_id": 3,
"engram_layer_ids": [],
"engram_num_embeddings": [],
# an Engram on a whole layer (K1 then takes its residual settled), its
# tables shrunk: 24 prime-sized ranges from 1000
"engram_layer_ids": [2],
"engram_num_embeddings": [32768],
"engram_vocab_size": 1000,
"dspark_target_layer_ids": [2, 3, 4],
}
WHOLE, FFN = (2, 4), (0, 1, 3)
WHOLE, FFN = (0, 1, 2, 3, 4), ()
STEPS = 5 # decode steps after the prefill's token
@@ -227,7 +232,6 @@ def _decode(vllm_runner, monkeypatch, tp: int, mono: bool, prompts, path: str):
from vllm import SamplingParams
from vllm.inputs import TokensPrompt
monkeypatch.setenv("VLLM_ROCM_MONO_DECODE", "1" if mono else "0")
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
with vllm_runner(
@@ -237,6 +241,7 @@ def _decode(vllm_runner, monkeypatch, tp: int, mono: bool, prompts, path: str):
hf_overrides=TRUNCATED,
worker_extension_cls=f"{__name__}.MonoProbe",
moe_backend="aiter",
monokernel=mono,
language_model_only=True,
tokenizer_mode="deepseek_v41",
attention_config={"indexer_kv_dtype": "mxfp4", "indexer_sparse_logits": True},
@@ -300,15 +305,16 @@ def test_mono_decode_matches_the_model(vllm_runner, monkeypatch, tmp_path, tp):
same.add((len(prompt), len(prompt) + step))
# each layer's outputs for those tokens, mono against vLLM's path, token by
# token. Layer 0 sees the same inputs on both paths at a request's first
# decode token: there the bound is test_dsv41_mono_numerics' (about 3x the
# kernels' 0.4%); its later tokens attend over KV each path wrote, 3%. Each
# later layer takes the previous one's slightly different output, which
# random weights amplify (2-4% by layer 4): 8%. A token whose top-6 routing
# is a near-tie on vLLM's path (6th / 7th score within 5e-3: a 0.4% input
# difference moves scores near 0.8 by about 3e-3) may pick another expert;
# once it has, its residual stream differs and the later layers are exempt
# for it. The residual stream: 3% (0.3% seen). A layer the kernels no
# token. Layer 0 sees the same inputs on both paths, but its FFN input
# differs by up to test_dsv41_mono_numerics' 1.2%, which random weights
# amplify through the MoE: 3%. Each later layer takes the previous one's
# slightly different output, which random weights amplify (2-4% by layer
# 4): 8%. A token whose top-6 routing is a near-tie on vLLM's path may pick
# another expert: a 0.4% input difference moves scores near 0.8 by about
# 3e-3, so 6th / 7th scores within 5e-3 at layer 0 and, past it, where the
# input carries the earlier layers' differences (0.4-0.7%), within 1e-2.
# Once it has, its residual stream differs and the later layers are exempt
# for it. The residual stream: 3% (0.5% seen). A layer the kernels no
# longer follow is off by far more, on every token.
for rank in range(tp):
got, _ = torch.load(f"{tmp_path}/on.rank{rank}")
@@ -317,8 +323,6 @@ def test_mono_decode_matches_the_model(vllm_runner, monkeypatch, tmp_path, tp):
same & {k[1:] for k in got} & {k[1:] for k in want}
)
assert len(keys) >= len(prompts), keys
# the bound that sees a 3% error: a request's first decode token
assert any(p == n for n, p in keys), keys
flipped: set = set()
for i in sorted(WHOLE + FFN):
for k in keys:
@@ -327,9 +331,10 @@ def test_mono_decode_matches_the_model(vllm_runner, monkeypatch, tmp_path, tp):
(x, res), (x_ref, res_ref) = got[(i, *k)], want[(i, *k)]
err = _rel(res, res_ref)
assert err < 0.03, (rank, i, k, "residual", err)
bound = (0.015 if k[1] == k[0] else 0.03) if i == 0 else 0.08
bound = 0.03 if i == 0 else 0.08
err = _rel(x, x_ref)
if err >= bound:
assert gaps[(i, *k)] < 5e-3, (rank, i, k, "x", err)
tie = 5e-3 if i == 0 else 1e-2
assert gaps[(i, *k)] < tie, (rank, i, k, "x", err)
flipped.add(k)
assert len(flipped) <= len(keys) // 4, (rank, sorted(flipped))