[ROCm][Perf][DSv4.1] Mono decode on every backbone layer (#61082)
Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
f47792af7f
commit
cc2a6eea1a
@@ -27,21 +27,25 @@ class _Runner:
|
||||
self.calls.append((args, kwargs))
|
||||
return ("mono outputs",)
|
||||
|
||||
def first_pre_mix(self, rows):
|
||||
return torch.zeros(rows, 4)
|
||||
|
||||
def ffn(self, *args):
|
||||
self.calls.append((args, {}))
|
||||
return ("ffn outputs",)
|
||||
|
||||
|
||||
def _layer(comp="comp"):
|
||||
def _layer(comp="comp", indexer=None):
|
||||
cache = lambda: torch.zeros(4, 128 * md.RECORD, dtype=torch.uint8) # noqa: E731
|
||||
attn = SimpleNamespace(
|
||||
swa_cache_layer=SimpleNamespace(prefix="swa", kv_cache=cache()),
|
||||
compressed_cache_prefix=comp,
|
||||
topk_indices_buffer=torch.zeros(64, 512, dtype=torch.int32),
|
||||
compress_ratio=1,
|
||||
indexer=indexer,
|
||||
_compressed_kv_cache=cache,
|
||||
)
|
||||
return SimpleNamespace(attn=attn)
|
||||
return SimpleNamespace(attn=attn, engram=None, hc_mult=4)
|
||||
|
||||
|
||||
def _swa(**overrides):
|
||||
@@ -77,7 +81,12 @@ def run(monkeypatch):
|
||||
``mode`` and ``metadata``; returns (result, runner calls)."""
|
||||
|
||||
def _run(
|
||||
mode=CUDAGraphMode.FULL, metadata=None, ffn_only=False, entry=None, **inputs
|
||||
mode=CUDAGraphMode.FULL,
|
||||
metadata=None,
|
||||
ffn_only=False,
|
||||
entry=None,
|
||||
layer=None,
|
||||
**inputs,
|
||||
):
|
||||
if metadata is None:
|
||||
metadata = {
|
||||
@@ -97,7 +106,7 @@ def run(monkeypatch):
|
||||
# has no compressed cache
|
||||
part, _ = args.pop("x"), args.pop("positions")
|
||||
return mono.ffn(_layer(comp=None), part, **args), runner.calls
|
||||
return mono(_layer(), **args), runner.calls
|
||||
return mono(layer or _layer(), **args), runner.calls
|
||||
|
||||
return _run
|
||||
|
||||
@@ -114,6 +123,23 @@ def test_decode_step_runs_mono(run, mode):
|
||||
1,
|
||||
)
|
||||
assert kwargs["comp_cache"].shape == (4, 128, md.RECORD)
|
||||
assert kwargs["fold"] and "select" not in kwargs
|
||||
|
||||
|
||||
def test_first_layer_runs_mono_from_the_embedding(run):
|
||||
"""The first layer's residual is the embedding broadcast to the four
|
||||
streams, which K1 takes as it is (no owed post to fold)."""
|
||||
out, calls = run(residual=None, post_mix=None, res_mix=None, pre_mix=None)
|
||||
assert out == ("mono outputs",) and len(calls) == 1
|
||||
args, kwargs = calls[0]
|
||||
assert args[2].shape == (M, 4, 5120) and not kwargs["fold"]
|
||||
|
||||
|
||||
def test_index_source_selects_between_the_kernels(run):
|
||||
"""An index source hands the runner its compressor and indexer to run
|
||||
between K1 and K2."""
|
||||
out, calls = run(layer=_layer(indexer=object()))
|
||||
assert out == ("mono outputs",) and callable(calls[0][1]["select"])
|
||||
|
||||
|
||||
def test_padded_decode_step_runs_mono(run):
|
||||
@@ -149,7 +175,9 @@ def test_padded_decode_step_runs_mono(run):
|
||||
),
|
||||
dict(metadata={}), # a profile / dummy run: no attention metadata
|
||||
dict(**_inputs(md.MAX_ROWS + 6)),
|
||||
dict(residual=None), # the first layer's seam broadcasts the embedding
|
||||
# a later pipeline stage's first layer: its stream arrives without
|
||||
# the mixes
|
||||
dict(residual=None, x=torch.zeros(M, 4, 5120, dtype=torch.bfloat16)),
|
||||
],
|
||||
ids=[
|
||||
"piecewise",
|
||||
@@ -158,7 +186,7 @@ def test_padded_decode_step_runs_mono(run):
|
||||
"tokens-past-rows",
|
||||
"no-metadata",
|
||||
"rows",
|
||||
"first-layer",
|
||||
"stage-input",
|
||||
],
|
||||
)
|
||||
def test_other_steps_take_the_original_path(run, case):
|
||||
@@ -202,22 +230,41 @@ def _decoder_layer(engram=None, **attn):
|
||||
)
|
||||
|
||||
|
||||
def _indexer():
|
||||
from vllm.models.deepseek_v41.amd.rocm import DeepseekV41RocmMxfp4Indexer
|
||||
|
||||
return object.__new__(DeepseekV41RocmMxfp4Indexer)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"layer, path",
|
||||
[
|
||||
(_decoder_layer(), "whole"),
|
||||
(_decoder_layer(compress_ratio=0), "ffn"),
|
||||
(_decoder_layer(engram=object()), "ffn"),
|
||||
(_decoder_layer(compressor=object(), indexer=object()), "ffn"),
|
||||
(_decoder_layer(compress_ratio=0), "whole"),
|
||||
(_decoder_layer(engram=object()), "whole"),
|
||||
(_decoder_layer(compressor=object(), indexer=_indexer()), "whole"),
|
||||
(_decoder_layer(indexer=_indexer()), "whole"),
|
||||
(_decoder_layer(compressor=object()), "ffn"),
|
||||
(_decoder_layer(indexer=object()), "ffn"),
|
||||
(_decoder_layer(kv_cache_dtype="auto"), "ffn"),
|
||||
(_decoder_layer(layer_id=40), None),
|
||||
],
|
||||
ids=["standard", "first", "engram", "kv-source", "index-source", "kv", "draft"],
|
||||
ids=[
|
||||
"standard",
|
||||
"window",
|
||||
"engram",
|
||||
"kv-source",
|
||||
"index-source",
|
||||
"compressor-only",
|
||||
"other-indexer",
|
||||
"kv",
|
||||
"draft",
|
||||
],
|
||||
)
|
||||
def test_layers_take_whole_or_ffn_launch(monkeypatch, layer, path):
|
||||
"""The whole-layer kernels serve the standard layers; every other backbone
|
||||
layer keeps vLLM's attention and runs its FFN half as one launch."""
|
||||
"""The whole-layer kernels serve every backbone layer of the checkpoint (an
|
||||
index source's compressor and indexer run between them); a layer outside
|
||||
them keeps vLLM's attention and runs its FFN half as one launch."""
|
||||
_deployment(monkeypatch)
|
||||
mono = md.MonoDecodeLayer.create(layer, _config())
|
||||
got = None if mono is None else "ffn" if mono.ffn_only else "whole"
|
||||
@@ -254,7 +301,7 @@ def test_other_steps_reduce_wo_b_themselves(run, case):
|
||||
assert out is None and not calls
|
||||
|
||||
|
||||
def _config(**parallel):
|
||||
def _config(monokernel=True, **parallel):
|
||||
parallel = (
|
||||
dict(enable_expert_parallel=False, enable_eplb=False, data_parallel_size=1)
|
||||
| parallel
|
||||
@@ -262,6 +309,7 @@ def _config(**parallel):
|
||||
return SimpleNamespace(
|
||||
model_config=SimpleNamespace(hf_config=SimpleNamespace(num_hidden_layers=40)),
|
||||
parallel_config=SimpleNamespace(**parallel),
|
||||
kernel_config=SimpleNamespace(monokernel=monokernel),
|
||||
)
|
||||
|
||||
|
||||
@@ -269,7 +317,6 @@ def _deployment(monkeypatch, cdna=4, cus=256):
|
||||
"""A deployment the kernels take (the runner module stubbed: no FlyDSL)."""
|
||||
import vllm.platforms.rocm as rocm
|
||||
|
||||
monkeypatch.setenv("VLLM_ROCM_MONO_DECODE", "1")
|
||||
monkeypatch.delenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4", raising=False)
|
||||
monkeypatch.setattr(rocm, "get_cdna_version", lambda: cdna)
|
||||
monkeypatch.setattr(md, "get_tensor_model_parallel_world_size", lambda: 2)
|
||||
@@ -280,6 +327,14 @@ def _deployment(monkeypatch, cdna=4, cus=256):
|
||||
monkeypatch.setitem(sys.modules, name, runner)
|
||||
|
||||
|
||||
def test_off_without_the_flag(monkeypatch):
|
||||
"""No --monokernel: no mono layer, whatever the deployment."""
|
||||
_deployment(monkeypatch)
|
||||
assert (
|
||||
md.MonoDecodeLayer.create(_decoder_layer(), _config(monokernel=False)) is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cdna", [3, 5])
|
||||
def test_opt_in_outside_the_kernels_cdna_raises(monkeypatch, cdna):
|
||||
_deployment(monkeypatch, cdna=cdna)
|
||||
|
||||
Reference in New Issue
Block a user