[ROCm][Perf][DSv4.1] Mono decode on every backbone layer (#61082)
Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
f47792af7f
commit
cc2a6eea1a
@@ -27,21 +27,25 @@ class _Runner:
|
||||
self.calls.append((args, kwargs))
|
||||
return ("mono outputs",)
|
||||
|
||||
def first_pre_mix(self, rows):
|
||||
return torch.zeros(rows, 4)
|
||||
|
||||
def ffn(self, *args):
|
||||
self.calls.append((args, {}))
|
||||
return ("ffn outputs",)
|
||||
|
||||
|
||||
def _layer(comp="comp"):
|
||||
def _layer(comp="comp", indexer=None):
|
||||
cache = lambda: torch.zeros(4, 128 * md.RECORD, dtype=torch.uint8) # noqa: E731
|
||||
attn = SimpleNamespace(
|
||||
swa_cache_layer=SimpleNamespace(prefix="swa", kv_cache=cache()),
|
||||
compressed_cache_prefix=comp,
|
||||
topk_indices_buffer=torch.zeros(64, 512, dtype=torch.int32),
|
||||
compress_ratio=1,
|
||||
indexer=indexer,
|
||||
_compressed_kv_cache=cache,
|
||||
)
|
||||
return SimpleNamespace(attn=attn)
|
||||
return SimpleNamespace(attn=attn, engram=None, hc_mult=4)
|
||||
|
||||
|
||||
def _swa(**overrides):
|
||||
@@ -77,7 +81,12 @@ def run(monkeypatch):
|
||||
``mode`` and ``metadata``; returns (result, runner calls)."""
|
||||
|
||||
def _run(
|
||||
mode=CUDAGraphMode.FULL, metadata=None, ffn_only=False, entry=None, **inputs
|
||||
mode=CUDAGraphMode.FULL,
|
||||
metadata=None,
|
||||
ffn_only=False,
|
||||
entry=None,
|
||||
layer=None,
|
||||
**inputs,
|
||||
):
|
||||
if metadata is None:
|
||||
metadata = {
|
||||
@@ -97,7 +106,7 @@ def run(monkeypatch):
|
||||
# has no compressed cache
|
||||
part, _ = args.pop("x"), args.pop("positions")
|
||||
return mono.ffn(_layer(comp=None), part, **args), runner.calls
|
||||
return mono(_layer(), **args), runner.calls
|
||||
return mono(layer or _layer(), **args), runner.calls
|
||||
|
||||
return _run
|
||||
|
||||
@@ -114,6 +123,23 @@ def test_decode_step_runs_mono(run, mode):
|
||||
1,
|
||||
)
|
||||
assert kwargs["comp_cache"].shape == (4, 128, md.RECORD)
|
||||
assert kwargs["fold"] and "select" not in kwargs
|
||||
|
||||
|
||||
def test_first_layer_runs_mono_from_the_embedding(run):
|
||||
"""The first layer's residual is the embedding broadcast to the four
|
||||
streams, which K1 takes as it is (no owed post to fold)."""
|
||||
out, calls = run(residual=None, post_mix=None, res_mix=None, pre_mix=None)
|
||||
assert out == ("mono outputs",) and len(calls) == 1
|
||||
args, kwargs = calls[0]
|
||||
assert args[2].shape == (M, 4, 5120) and not kwargs["fold"]
|
||||
|
||||
|
||||
def test_index_source_selects_between_the_kernels(run):
|
||||
"""An index source hands the runner its compressor and indexer to run
|
||||
between K1 and K2."""
|
||||
out, calls = run(layer=_layer(indexer=object()))
|
||||
assert out == ("mono outputs",) and callable(calls[0][1]["select"])
|
||||
|
||||
|
||||
def test_padded_decode_step_runs_mono(run):
|
||||
@@ -149,7 +175,9 @@ def test_padded_decode_step_runs_mono(run):
|
||||
),
|
||||
dict(metadata={}), # a profile / dummy run: no attention metadata
|
||||
dict(**_inputs(md.MAX_ROWS + 6)),
|
||||
dict(residual=None), # the first layer's seam broadcasts the embedding
|
||||
# a later pipeline stage's first layer: its stream arrives without
|
||||
# the mixes
|
||||
dict(residual=None, x=torch.zeros(M, 4, 5120, dtype=torch.bfloat16)),
|
||||
],
|
||||
ids=[
|
||||
"piecewise",
|
||||
@@ -158,7 +186,7 @@ def test_padded_decode_step_runs_mono(run):
|
||||
"tokens-past-rows",
|
||||
"no-metadata",
|
||||
"rows",
|
||||
"first-layer",
|
||||
"stage-input",
|
||||
],
|
||||
)
|
||||
def test_other_steps_take_the_original_path(run, case):
|
||||
@@ -202,22 +230,41 @@ def _decoder_layer(engram=None, **attn):
|
||||
)
|
||||
|
||||
|
||||
def _indexer():
|
||||
from vllm.models.deepseek_v41.amd.rocm import DeepseekV41RocmMxfp4Indexer
|
||||
|
||||
return object.__new__(DeepseekV41RocmMxfp4Indexer)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"layer, path",
|
||||
[
|
||||
(_decoder_layer(), "whole"),
|
||||
(_decoder_layer(compress_ratio=0), "ffn"),
|
||||
(_decoder_layer(engram=object()), "ffn"),
|
||||
(_decoder_layer(compressor=object(), indexer=object()), "ffn"),
|
||||
(_decoder_layer(compress_ratio=0), "whole"),
|
||||
(_decoder_layer(engram=object()), "whole"),
|
||||
(_decoder_layer(compressor=object(), indexer=_indexer()), "whole"),
|
||||
(_decoder_layer(indexer=_indexer()), "whole"),
|
||||
(_decoder_layer(compressor=object()), "ffn"),
|
||||
(_decoder_layer(indexer=object()), "ffn"),
|
||||
(_decoder_layer(kv_cache_dtype="auto"), "ffn"),
|
||||
(_decoder_layer(layer_id=40), None),
|
||||
],
|
||||
ids=["standard", "first", "engram", "kv-source", "index-source", "kv", "draft"],
|
||||
ids=[
|
||||
"standard",
|
||||
"window",
|
||||
"engram",
|
||||
"kv-source",
|
||||
"index-source",
|
||||
"compressor-only",
|
||||
"other-indexer",
|
||||
"kv",
|
||||
"draft",
|
||||
],
|
||||
)
|
||||
def test_layers_take_whole_or_ffn_launch(monkeypatch, layer, path):
|
||||
"""The whole-layer kernels serve the standard layers; every other backbone
|
||||
layer keeps vLLM's attention and runs its FFN half as one launch."""
|
||||
"""The whole-layer kernels serve every backbone layer of the checkpoint (an
|
||||
index source's compressor and indexer run between them); a layer outside
|
||||
them keeps vLLM's attention and runs its FFN half as one launch."""
|
||||
_deployment(monkeypatch)
|
||||
mono = md.MonoDecodeLayer.create(layer, _config())
|
||||
got = None if mono is None else "ffn" if mono.ffn_only else "whole"
|
||||
@@ -254,7 +301,7 @@ def test_other_steps_reduce_wo_b_themselves(run, case):
|
||||
assert out is None and not calls
|
||||
|
||||
|
||||
def _config(**parallel):
|
||||
def _config(monokernel=True, **parallel):
|
||||
parallel = (
|
||||
dict(enable_expert_parallel=False, enable_eplb=False, data_parallel_size=1)
|
||||
| parallel
|
||||
@@ -262,6 +309,7 @@ def _config(**parallel):
|
||||
return SimpleNamespace(
|
||||
model_config=SimpleNamespace(hf_config=SimpleNamespace(num_hidden_layers=40)),
|
||||
parallel_config=SimpleNamespace(**parallel),
|
||||
kernel_config=SimpleNamespace(monokernel=monokernel),
|
||||
)
|
||||
|
||||
|
||||
@@ -269,7 +317,6 @@ def _deployment(monkeypatch, cdna=4, cus=256):
|
||||
"""A deployment the kernels take (the runner module stubbed: no FlyDSL)."""
|
||||
import vllm.platforms.rocm as rocm
|
||||
|
||||
monkeypatch.setenv("VLLM_ROCM_MONO_DECODE", "1")
|
||||
monkeypatch.delenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4", raising=False)
|
||||
monkeypatch.setattr(rocm, "get_cdna_version", lambda: cdna)
|
||||
monkeypatch.setattr(md, "get_tensor_model_parallel_world_size", lambda: 2)
|
||||
@@ -280,6 +327,14 @@ def _deployment(monkeypatch, cdna=4, cus=256):
|
||||
monkeypatch.setitem(sys.modules, name, runner)
|
||||
|
||||
|
||||
def test_off_without_the_flag(monkeypatch):
|
||||
"""No --monokernel: no mono layer, whatever the deployment."""
|
||||
_deployment(monkeypatch)
|
||||
assert (
|
||||
md.MonoDecodeLayer.create(_decoder_layer(), _config(monokernel=False)) is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cdna", [3, 5])
|
||||
def test_opt_in_outside_the_kernels_cdna_raises(monkeypatch, cdna):
|
||||
_deployment(monkeypatch, cdna=cdna)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The DeepSeek-V4.1 mono decode path through the real model: a five-layer
|
||||
DeepSeek-V4.1-Flash at TP2 and TP4 on CDNA4, served with ``VLLM_ROCM_MONO_DECODE`` on
|
||||
DeepSeek-V4.1-Flash at TP2 and TP4 on CDNA4, served with ``--monokernel`` on
|
||||
and off on the same random weights: every decoder layer's outputs at every
|
||||
decode step must match. Unlike ``test_dsv41_mono_numerics.py``, which
|
||||
checks the kernels against vLLM's ops one stage at a time, this runs
|
||||
@@ -10,12 +10,13 @@ the per-step path choice, the weights taken off the layer, the metadata and
|
||||
the indexer's top-k -- so a change to the decoder layer that the kernels do
|
||||
not follow fails here.
|
||||
|
||||
The five layers hold both kinds the mono path takes:
|
||||
- 0: sliding window only -> the FFN launch
|
||||
- 1: ratio-2 KV and index source -> the FFN launch
|
||||
- 2: ratio-2 consumer -> the whole layer (K1 + K2)
|
||||
- 3: ratio-1 KV, index and candidate source -> the FFN launch
|
||||
- 4: ratio-1 consumer -> the whole layer
|
||||
Every layer takes the whole-layer path (K1 + K2); the five hold each variant:
|
||||
- 0: the first layer, sliding window only (K1 takes the embedding settled)
|
||||
- 1: ratio-2 KV and index source (vLLM's compressor and indexer between K1
|
||||
and K2)
|
||||
- 2: ratio-2 consumer with an Engram (K1 takes the residual settled)
|
||||
- 3: ratio-1 KV, index and candidate source
|
||||
- 4: ratio-1 index source over 3's candidates and KV
|
||||
"""
|
||||
|
||||
import zlib
|
||||
@@ -34,13 +35,17 @@ TRUNCATED = {
|
||||
# one entry a layer, the three MTP layers' included
|
||||
"compress_ratios": [0, 2, 2, 1, 1, 0, 0, 0],
|
||||
"kv_source_layer_ids": [1, 3],
|
||||
"index_source_layer_ids": [1, 3],
|
||||
# 4: an indexer of its own over 3's candidates (a REINDEX layer)
|
||||
"index_source_layer_ids": [1, 3, 4],
|
||||
"candidate_source_layer_id": 3,
|
||||
"engram_layer_ids": [],
|
||||
"engram_num_embeddings": [],
|
||||
# an Engram on a whole layer (K1 then takes its residual settled), its
|
||||
# tables shrunk: 24 prime-sized ranges from 1000
|
||||
"engram_layer_ids": [2],
|
||||
"engram_num_embeddings": [32768],
|
||||
"engram_vocab_size": 1000,
|
||||
"dspark_target_layer_ids": [2, 3, 4],
|
||||
}
|
||||
WHOLE, FFN = (2, 4), (0, 1, 3)
|
||||
WHOLE, FFN = (0, 1, 2, 3, 4), ()
|
||||
STEPS = 5 # decode steps after the prefill's token
|
||||
|
||||
|
||||
@@ -227,7 +232,6 @@ def _decode(vllm_runner, monkeypatch, tp: int, mono: bool, prompts, path: str):
|
||||
from vllm import SamplingParams
|
||||
from vllm.inputs import TokensPrompt
|
||||
|
||||
monkeypatch.setenv("VLLM_ROCM_MONO_DECODE", "1" if mono else "0")
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
|
||||
with vllm_runner(
|
||||
@@ -237,6 +241,7 @@ def _decode(vllm_runner, monkeypatch, tp: int, mono: bool, prompts, path: str):
|
||||
hf_overrides=TRUNCATED,
|
||||
worker_extension_cls=f"{__name__}.MonoProbe",
|
||||
moe_backend="aiter",
|
||||
monokernel=mono,
|
||||
language_model_only=True,
|
||||
tokenizer_mode="deepseek_v41",
|
||||
attention_config={"indexer_kv_dtype": "mxfp4", "indexer_sparse_logits": True},
|
||||
@@ -300,15 +305,16 @@ def test_mono_decode_matches_the_model(vllm_runner, monkeypatch, tmp_path, tp):
|
||||
same.add((len(prompt), len(prompt) + step))
|
||||
|
||||
# each layer's outputs for those tokens, mono against vLLM's path, token by
|
||||
# token. Layer 0 sees the same inputs on both paths at a request's first
|
||||
# decode token: there the bound is test_dsv41_mono_numerics' (about 3x the
|
||||
# kernels' 0.4%); its later tokens attend over KV each path wrote, 3%. Each
|
||||
# later layer takes the previous one's slightly different output, which
|
||||
# random weights amplify (2-4% by layer 4): 8%. A token whose top-6 routing
|
||||
# is a near-tie on vLLM's path (6th / 7th score within 5e-3: a 0.4% input
|
||||
# difference moves scores near 0.8 by about 3e-3) may pick another expert;
|
||||
# once it has, its residual stream differs and the later layers are exempt
|
||||
# for it. The residual stream: 3% (0.3% seen). A layer the kernels no
|
||||
# token. Layer 0 sees the same inputs on both paths, but its FFN input
|
||||
# differs by up to test_dsv41_mono_numerics' 1.2%, which random weights
|
||||
# amplify through the MoE: 3%. Each later layer takes the previous one's
|
||||
# slightly different output, which random weights amplify (2-4% by layer
|
||||
# 4): 8%. A token whose top-6 routing is a near-tie on vLLM's path may pick
|
||||
# another expert: a 0.4% input difference moves scores near 0.8 by about
|
||||
# 3e-3, so 6th / 7th scores within 5e-3 at layer 0 and, past it, where the
|
||||
# input carries the earlier layers' differences (0.4-0.7%), within 1e-2.
|
||||
# Once it has, its residual stream differs and the later layers are exempt
|
||||
# for it. The residual stream: 3% (0.5% seen). A layer the kernels no
|
||||
# longer follow is off by far more, on every token.
|
||||
for rank in range(tp):
|
||||
got, _ = torch.load(f"{tmp_path}/on.rank{rank}")
|
||||
@@ -317,8 +323,6 @@ def test_mono_decode_matches_the_model(vllm_runner, monkeypatch, tmp_path, tp):
|
||||
same & {k[1:] for k in got} & {k[1:] for k in want}
|
||||
)
|
||||
assert len(keys) >= len(prompts), keys
|
||||
# the bound that sees a 3% error: a request's first decode token
|
||||
assert any(p == n for n, p in keys), keys
|
||||
flipped: set = set()
|
||||
for i in sorted(WHOLE + FFN):
|
||||
for k in keys:
|
||||
@@ -327,9 +331,10 @@ def test_mono_decode_matches_the_model(vllm_runner, monkeypatch, tmp_path, tp):
|
||||
(x, res), (x_ref, res_ref) = got[(i, *k)], want[(i, *k)]
|
||||
err = _rel(res, res_ref)
|
||||
assert err < 0.03, (rank, i, k, "residual", err)
|
||||
bound = (0.015 if k[1] == k[0] else 0.03) if i == 0 else 0.08
|
||||
bound = 0.03 if i == 0 else 0.08
|
||||
err = _rel(x, x_ref)
|
||||
if err >= bound:
|
||||
assert gaps[(i, *k)] < 5e-3, (rank, i, k, "x", err)
|
||||
tie = 5e-3 if i == 0 else 1e-2
|
||||
assert gaps[(i, *k)] < tie, (rank, i, k, "x", err)
|
||||
flipped.add(k)
|
||||
assert len(flipped) <= len(keys) // 4, (rank, sorted(flipped))
|
||||
|
||||
Reference in New Issue
Block a user