[ROCm][Perf][DSv4.1] Mono decode on every backbone layer (#61082)

Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Fangzhou Ai
2026-10-13 10:16:50 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent f47792af7f
commit cc2a6eea1a
13 changed files with 379 additions and 111 deletions
+69 -14
View File
@@ -27,21 +27,25 @@ class _Runner:
self.calls.append((args, kwargs))
return ("mono outputs",)
def first_pre_mix(self, rows):
return torch.zeros(rows, 4)
def ffn(self, *args):
self.calls.append((args, {}))
return ("ffn outputs",)
def _layer(comp="comp"):
def _layer(comp="comp", indexer=None):
cache = lambda: torch.zeros(4, 128 * md.RECORD, dtype=torch.uint8) # noqa: E731
attn = SimpleNamespace(
swa_cache_layer=SimpleNamespace(prefix="swa", kv_cache=cache()),
compressed_cache_prefix=comp,
topk_indices_buffer=torch.zeros(64, 512, dtype=torch.int32),
compress_ratio=1,
indexer=indexer,
_compressed_kv_cache=cache,
)
return SimpleNamespace(attn=attn)
return SimpleNamespace(attn=attn, engram=None, hc_mult=4)
def _swa(**overrides):
@@ -77,7 +81,12 @@ def run(monkeypatch):
``mode`` and ``metadata``; returns (result, runner calls)."""
def _run(
mode=CUDAGraphMode.FULL, metadata=None, ffn_only=False, entry=None, **inputs
mode=CUDAGraphMode.FULL,
metadata=None,
ffn_only=False,
entry=None,
layer=None,
**inputs,
):
if metadata is None:
metadata = {
@@ -97,7 +106,7 @@ def run(monkeypatch):
# has no compressed cache
part, _ = args.pop("x"), args.pop("positions")
return mono.ffn(_layer(comp=None), part, **args), runner.calls
return mono(_layer(), **args), runner.calls
return mono(layer or _layer(), **args), runner.calls
return _run
@@ -114,6 +123,23 @@ def test_decode_step_runs_mono(run, mode):
1,
)
assert kwargs["comp_cache"].shape == (4, 128, md.RECORD)
assert kwargs["fold"] and "select" not in kwargs
def test_first_layer_runs_mono_from_the_embedding(run):
"""The first layer's residual is the embedding broadcast to the four
streams, which K1 takes as it is (no owed post to fold)."""
out, calls = run(residual=None, post_mix=None, res_mix=None, pre_mix=None)
assert out == ("mono outputs",) and len(calls) == 1
args, kwargs = calls[0]
assert args[2].shape == (M, 4, 5120) and not kwargs["fold"]
def test_index_source_selects_between_the_kernels(run):
"""An index source hands the runner its compressor and indexer to run
between K1 and K2."""
out, calls = run(layer=_layer(indexer=object()))
assert out == ("mono outputs",) and callable(calls[0][1]["select"])
def test_padded_decode_step_runs_mono(run):
@@ -149,7 +175,9 @@ def test_padded_decode_step_runs_mono(run):
),
dict(metadata={}), # a profile / dummy run: no attention metadata
dict(**_inputs(md.MAX_ROWS + 6)),
dict(residual=None), # the first layer's seam broadcasts the embedding
# a later pipeline stage's first layer: its stream arrives without
# the mixes
dict(residual=None, x=torch.zeros(M, 4, 5120, dtype=torch.bfloat16)),
],
ids=[
"piecewise",
@@ -158,7 +186,7 @@ def test_padded_decode_step_runs_mono(run):
"tokens-past-rows",
"no-metadata",
"rows",
"first-layer",
"stage-input",
],
)
def test_other_steps_take_the_original_path(run, case):
@@ -202,22 +230,41 @@ def _decoder_layer(engram=None, **attn):
)
def _indexer():
from vllm.models.deepseek_v41.amd.rocm import DeepseekV41RocmMxfp4Indexer
return object.__new__(DeepseekV41RocmMxfp4Indexer)
@pytest.mark.parametrize(
"layer, path",
[
(_decoder_layer(), "whole"),
(_decoder_layer(compress_ratio=0), "ffn"),
(_decoder_layer(engram=object()), "ffn"),
(_decoder_layer(compressor=object(), indexer=object()), "ffn"),
(_decoder_layer(compress_ratio=0), "whole"),
(_decoder_layer(engram=object()), "whole"),
(_decoder_layer(compressor=object(), indexer=_indexer()), "whole"),
(_decoder_layer(indexer=_indexer()), "whole"),
(_decoder_layer(compressor=object()), "ffn"),
(_decoder_layer(indexer=object()), "ffn"),
(_decoder_layer(kv_cache_dtype="auto"), "ffn"),
(_decoder_layer(layer_id=40), None),
],
ids=["standard", "first", "engram", "kv-source", "index-source", "kv", "draft"],
ids=[
"standard",
"window",
"engram",
"kv-source",
"index-source",
"compressor-only",
"other-indexer",
"kv",
"draft",
],
)
def test_layers_take_whole_or_ffn_launch(monkeypatch, layer, path):
"""The whole-layer kernels serve the standard layers; every other backbone
layer keeps vLLM's attention and runs its FFN half as one launch."""
"""The whole-layer kernels serve every backbone layer of the checkpoint (an
index source's compressor and indexer run between them); a layer outside
them keeps vLLM's attention and runs its FFN half as one launch."""
_deployment(monkeypatch)
mono = md.MonoDecodeLayer.create(layer, _config())
got = None if mono is None else "ffn" if mono.ffn_only else "whole"
@@ -254,7 +301,7 @@ def test_other_steps_reduce_wo_b_themselves(run, case):
assert out is None and not calls
def _config(**parallel):
def _config(monokernel=True, **parallel):
parallel = (
dict(enable_expert_parallel=False, enable_eplb=False, data_parallel_size=1)
| parallel
@@ -262,6 +309,7 @@ def _config(**parallel):
return SimpleNamespace(
model_config=SimpleNamespace(hf_config=SimpleNamespace(num_hidden_layers=40)),
parallel_config=SimpleNamespace(**parallel),
kernel_config=SimpleNamespace(monokernel=monokernel),
)
@@ -269,7 +317,6 @@ def _deployment(monkeypatch, cdna=4, cus=256):
"""A deployment the kernels take (the runner module stubbed: no FlyDSL)."""
import vllm.platforms.rocm as rocm
monkeypatch.setenv("VLLM_ROCM_MONO_DECODE", "1")
monkeypatch.delenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4", raising=False)
monkeypatch.setattr(rocm, "get_cdna_version", lambda: cdna)
monkeypatch.setattr(md, "get_tensor_model_parallel_world_size", lambda: 2)
@@ -280,6 +327,14 @@ def _deployment(monkeypatch, cdna=4, cus=256):
monkeypatch.setitem(sys.modules, name, runner)
def test_off_without_the_flag(monkeypatch):
"""No --monokernel: no mono layer, whatever the deployment."""
_deployment(monkeypatch)
assert (
md.MonoDecodeLayer.create(_decoder_layer(), _config(monokernel=False)) is None
)
@pytest.mark.parametrize("cdna", [3, 5])
def test_opt_in_outside_the_kernels_cdna_raises(monkeypatch, cdna):
_deployment(monkeypatch, cdna=cdna)