[ROCm][Perf][DSv4.1] Mono decode layer for DeepSeek-V4.1-Flash on MI355X (#60397)

Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Fangzhou Ai
2026-10-09 20:53:18 -05:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 3c802af3cf
commit 193922d6ff
33 changed files with 9119 additions and 2 deletions
+317
View File
@@ -0,0 +1,317 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The DeepSeek-V4.1 mono decode layer runs only where its kernels are exact:
decode-only steps of at most MAX_ROWS rows with causal SWA windows, eager or in
a FULL graph. A PIECEWISE capture is replayed for mixed batches, so it must
never record the mono launches. Layers outside the whole-layer kernels run the
FFN launch on the same steps, from wo_b's unreduced output."""
import sys
from types import ModuleType, SimpleNamespace
import pytest
import torch
from vllm.config import CUDAGraphMode
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend
from vllm.models.deepseek_v41.amd import mono_decode as md
M = 6
class _Runner:
def __init__(self):
self.calls = []
def forward(self, *args, **kwargs):
self.calls.append((args, kwargs))
return ("mono outputs",)
def ffn(self, *args):
self.calls.append((args, {}))
return ("ffn outputs",)
def _layer(comp="comp"):
cache = lambda: torch.zeros(4, 128 * md.RECORD, dtype=torch.uint8) # noqa: E731
attn = SimpleNamespace(
swa_cache_layer=SimpleNamespace(prefix="swa", kv_cache=cache()),
compressed_cache_prefix=comp,
topk_indices_buffer=torch.zeros(64, 512, dtype=torch.int32),
compress_ratio=1,
_compressed_kv_cache=cache,
)
return SimpleNamespace(attn=attn)
def _swa(**overrides):
fields = dict(
num_prefills=0,
num_decodes=1,
num_decode_tokens=M,
decode_swa_width=md.SWA_WIDTH,
decode_swa_indices=torch.zeros(64, 1, md.SWA_WIDTH, dtype=torch.int32),
decode_swa_lens=torch.zeros(64, dtype=torch.int32),
token_to_req_indices=torch.zeros(64, dtype=torch.int32),
slot_mapping=torch.zeros(64, dtype=torch.int64),
block_size=32,
)
fields.update(overrides)
return SimpleNamespace(**fields)
def _inputs(rows=M):
return dict(
x=torch.zeros(rows, 5120, dtype=torch.bfloat16),
positions=torch.zeros(rows, dtype=torch.int64),
residual=torch.zeros(rows, 4, 5120, dtype=torch.bfloat16),
post_mix=torch.zeros(rows, 4, 1),
res_mix=torch.zeros(rows, 4, 4),
pre_mix=torch.zeros(rows, 4),
)
@pytest.fixture
def run(monkeypatch):
"""Call the mono path for one layer under a forward context built from
``mode`` and ``metadata``; returns (result, runner calls)."""
def _run(
mode=CUDAGraphMode.FULL, metadata=None, ffn_only=False, entry=None, **inputs
):
if metadata is None:
metadata = {
"swa": _swa(),
"comp": SimpleNamespace(block_size=128, block_table=None),
}
fc = SimpleNamespace(cudagraph_runtime_mode=mode, attn_metadata=metadata)
runner = _Runner()
monkeypatch.setattr(md, "is_forward_context_available", lambda: True)
monkeypatch.setattr(md, "get_forward_context", lambda: fc)
monkeypatch.setattr(md, "_mono_runner", lambda device: runner)
monkeypatch.setattr(md.MonoDecodeLayer, "weights", lambda self, layer: None)
args = {**_inputs(), **inputs}
mono = md.MonoDecodeLayer(ffn_only)
if (entry or ("ffn" if ffn_only else "layer")) == "ffn":
# x stands for wo_b's unreduced output; the first layer's attention
# has no compressed cache
part, _ = args.pop("x"), args.pop("positions")
return mono.ffn(_layer(comp=None), part, **args), runner.calls
return mono(_layer(), **args), runner.calls
return _run
@pytest.mark.parametrize("mode", [CUDAGraphMode.NONE, CUDAGraphMode.FULL])
def test_decode_step_runs_mono(run, mode):
out, calls = run(mode)
assert out == ("mono outputs",) and len(calls) == 1
args, kwargs = calls[0]
# the caches reach the kernels as [blocks, block, record] bytes
assert args[8].shape == (4, 32, md.RECORD) and args[8].stride() == (
128 * md.RECORD,
md.RECORD,
1,
)
assert kwargs["comp_cache"].shape == (4, 128, md.RECORD)
def test_padded_decode_step_runs_mono(run):
"""A FULL graph pads the step to its capture size: the kernels take it and
skip the pad rows (their slot is -1)."""
swa = _swa(num_decode_tokens=M - 2)
comp = SimpleNamespace(block_size=128, block_table=None)
out, calls = run(metadata={"swa": swa, "comp": comp})
assert out == ("mono outputs",) and len(calls) == 1
@pytest.mark.parametrize(
"case",
[
dict(mode=CUDAGraphMode.PIECEWISE),
dict(
metadata={
"swa": _swa(num_prefills=1),
"comp": SimpleNamespace(block_size=128),
}
),
dict(
metadata={
"swa": _swa(decode_swa_width=256),
"comp": SimpleNamespace(block_size=128),
}
),
dict(
metadata={
"swa": _swa(num_decode_tokens=M + 1),
"comp": SimpleNamespace(block_size=128),
}
),
dict(metadata={}), # a profile / dummy run: no attention metadata
dict(**_inputs(md.MAX_ROWS + 6)),
dict(residual=None), # the first layer's seam broadcasts the embedding
],
ids=[
"piecewise",
"prefill",
"noncausal-window",
"tokens-past-rows",
"no-metadata",
"rows",
"first-layer",
],
)
def test_other_steps_take_the_original_path(run, case):
out, calls = run(**case)
assert out is None and not calls
def _decoder_layer(engram=None, **attn):
attn = (
dict(
layer_id=21,
compressor=None,
indexer=None,
compress_ratio=1,
kv_mxfp8=False,
kv_cache_dtype="fp8_ds_mla",
)
| attn
)
backend = Mxfp4MoeBackend.AITER_MXFP4_BF16
quant_method = SimpleNamespace(mxfp4_backend=backend)
ffn = SimpleNamespace(
experts=SimpleNamespace(
routed_experts=SimpleNamespace(quant_method=quant_method)
),
shared_experts=object(),
gate=SimpleNamespace(tid2eid=None),
n_routed_experts=384,
n_activated_experts=6,
routed_scaling_factor=1.5,
swiglu_limit=10.0,
scoring_func="sqrtsoftplus",
renormalize=True,
)
return SimpleNamespace(
attn=SimpleNamespace(**attn),
ffn=ffn,
engram=engram,
use_sequence_parallel=False,
fuse_seam_norm=True,
)
@pytest.mark.parametrize(
"layer, path",
[
(_decoder_layer(), "whole"),
(_decoder_layer(compress_ratio=0), "ffn"),
(_decoder_layer(engram=object()), "ffn"),
(_decoder_layer(compressor=object(), indexer=object()), "ffn"),
(_decoder_layer(indexer=object()), "ffn"),
(_decoder_layer(kv_cache_dtype="auto"), "ffn"),
(_decoder_layer(layer_id=40), None),
],
ids=["standard", "first", "engram", "kv-source", "index-source", "kv", "draft"],
)
def test_layers_take_whole_or_ffn_launch(monkeypatch, layer, path):
"""The whole-layer kernels serve the standard layers; every other backbone
layer keeps vLLM's attention and runs its FFN half as one launch."""
_deployment(monkeypatch)
mono = md.MonoDecodeLayer.create(layer, _config())
got = None if mono is None else "ffn" if mono.ffn_only else "whole"
assert got == path
@pytest.mark.parametrize("mode", [CUDAGraphMode.NONE, CUDAGraphMode.FULL])
def test_decode_step_runs_ffn_launch(run, mode):
out, calls = run(mode, ffn_only=True, metadata={"swa": _swa()})
assert out == ("ffn outputs",) and len(calls) == 1
assert calls[0][0][1].shape == (M, 5120) # wo_b's unreduced output
def test_ffn_layer_never_runs_whole(run):
out, calls = run(ffn_only=True, entry="layer")
assert out is None and not calls
@pytest.mark.parametrize(
"case",
[
dict(mode=CUDAGraphMode.PIECEWISE),
dict(metadata={"swa": _swa(num_prefills=1)}),
dict(metadata={"swa": _swa(num_decode_tokens=M + 1)}),
dict(metadata={}),
dict(**_inputs(md.MAX_ROWS + 6)),
],
ids=["piecewise", "prefill", "tokens-past-rows", "no-metadata", "rows"],
)
def test_other_steps_reduce_wo_b_themselves(run, case):
"""None: the layer all-reduces wo_b's partial and runs the FFN as before."""
case.setdefault("metadata", {"swa": _swa()})
out, calls = run(ffn_only=True, **case)
assert out is None and not calls
def _config(**parallel):
parallel = (
dict(enable_expert_parallel=False, enable_eplb=False, data_parallel_size=1)
| parallel
)
return SimpleNamespace(
model_config=SimpleNamespace(hf_config=SimpleNamespace(num_hidden_layers=40)),
parallel_config=SimpleNamespace(**parallel),
)
def _deployment(monkeypatch, cdna=4, cus=256):
"""A deployment the kernels take (the runner module stubbed: no FlyDSL)."""
import vllm.platforms.rocm as rocm
monkeypatch.setenv("VLLM_ROCM_MONO_DECODE", "1")
monkeypatch.delenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4", raising=False)
monkeypatch.setattr(rocm, "get_cdna_version", lambda: cdna)
monkeypatch.setattr(md, "get_tensor_model_parallel_world_size", lambda: 2)
monkeypatch.setattr(md, "_compute_units", lambda: cus)
name = "vllm.models.deepseek_v41.amd.mono.runner"
runner = ModuleType(name)
runner.BLOCKS = 256 # type: ignore[attr-defined]
monkeypatch.setitem(sys.modules, name, runner)
@pytest.mark.parametrize("cdna", [3, 5])
def test_opt_in_outside_the_kernels_cdna_raises(monkeypatch, cdna):
_deployment(monkeypatch, cdna=cdna)
with pytest.raises(ValueError, match="needs CDNA4"):
md.MonoDecodeLayer.create(_decoder_layer(), _config())
def _other_backend():
layer = _decoder_layer()
experts = layer.ffn.experts.routed_experts
experts.quant_method.mxfp4_backend = Mxfp4MoeBackend.TRITON
return layer
@pytest.mark.parametrize(
"case, match",
[
(dict(parallel=dict(enable_expert_parallel=True)), "no expert"),
(dict(parallel=dict(data_parallel_size=2)), "no expert"),
(dict(parallel=dict(enable_eplb=True)), "no expert"),
(dict(a4w4=True), "A8W4"),
(dict(layer=_other_backend), "AITER_MXFP4_BF16"),
(dict(cus=128), "256 compute units"),
],
ids=["expert-parallel", "data-parallel", "eplb", "a4w4", "moe-backend", "cus"],
)
def test_opt_in_outside_the_kernels_moe_raises(monkeypatch, case, match):
"""The kernels read all 384 experts on each rank in AITER's A8W4 layout, with
all 256 CTAs resident: any other deployment is an error, not garbage."""
_deployment(monkeypatch, cus=case.get("cus", 256))
if case.get("a4w4"):
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4", "1")
layer = case.get("layer", _decoder_layer)()
with pytest.raises(ValueError, match=match):
md.MonoDecodeLayer.create(layer, _config(**case.get("parallel", {})))
+335
View File
@@ -0,0 +1,335 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The DeepSeek-V4.1 mono decode path through the real model: a five-layer
DeepSeek-V4.1-Flash at TP2 and TP4 on CDNA4, served with ``VLLM_ROCM_MONO_DECODE`` on
and off on the same random weights: every decoder layer's outputs at every
decode step must match. Unlike ``test_dsv41_mono_numerics.py``, which
checks the kernels against vLLM's ops one stage at a time, this runs
``DeepseekV4DecoderLayer.forward`` as the engine does -- the model's hooks,
the per-step path choice, the weights taken off the layer, the metadata and
the indexer's top-k -- so a change to the decoder layer that the kernels do
not follow fails here.
The five layers hold both kinds the mono path takes:
- 0: sliding window only -> the FFN launch
- 1: ratio-2 KV and index source -> the FFN launch
- 2: ratio-2 consumer -> the whole layer (K1 + K2)
- 3: ratio-1 KV, index and candidate source -> the FFN launch
- 4: ratio-1 consumer -> the whole layer
"""
import zlib
import pytest
import torch
from vllm.model_executor.model_loader import register_model_loader
from vllm.model_executor.model_loader.dummy_loader import DummyModelLoader
from vllm.platforms import current_platform
MODEL = "deepseek-ai/DeepSeek-V4.1-Flash"
LOAD_FORMAT = "dsv41_mono_random"
TRUNCATED = {
"num_hidden_layers": 5,
# one entry a layer, the three MTP layers' included
"compress_ratios": [0, 2, 2, 1, 1, 0, 0, 0],
"kv_source_layer_ids": [1, 3],
"index_source_layer_ids": [1, 3],
"candidate_source_layer_id": 3,
"engram_layer_ids": [],
"engram_num_embeddings": [],
"dspark_target_layer_ids": [2, 3, 4],
}
WHOLE, FFN = (2, 4), (0, 1, 3)
STEPS = 5 # decode steps after the prefill's token
def _on_cdna4() -> bool:
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import get_cdna_version
return get_cdna_version() == 4
@torch.no_grad()
def _fill(name: str, t: torch.Tensor) -> None:
"""A random value for every weight, seeded by its name (alike on every
rank and in every run), in the ranges the checkpoint's formats hold."""
if t.device.type == "meta":
return # materialized and filled by the dummy loader's own path
g = torch.Generator(device=t.device).manual_seed(zlib.crc32(name.encode()))
def randn(scale=1.0, shift=0.0):
return torch.randn(t.shape, generator=g, device=t.device) * scale + shift
if t.dtype == torch.uint8 and "scale" in name:
# E8M0 2^-12 .. 2^-8, alike over each 32 rows as in the checkpoint (the
# loader folds per-row scales into 32 x 32 blocks only then). Larger
# random layers make the stack chaotic: atomics' order alone moves
# vLLM's own logits run to run.
rows = t.shape[-2] if t.dim() > 1 else 1
shape = (*t.shape[:-2], -(-rows // 32), t.shape[-1]) if t.dim() > 1 else t.shape
e8m0 = torch.randint(115, 120, shape, generator=g, device=t.device)
if t.dim() > 1:
e8m0 = e8m0.repeat_interleave(32, dim=-2)[..., :rows, :]
t.copy_(e8m0)
elif t.dtype in (torch.uint8, torch.float4_e2m1fn_x2):
u8 = t.view(torch.uint8)
u8.copy_(torch.randint(0, 256, u8.shape, generator=g, device=t.device))
elif t.dtype == torch.float8_e4m3fn:
t.copy_(randn().clamp(-6, 6))
elif not torch.is_floating_point(t):
return
elif "norm" in name:
t.copy_(randn(0.1, 1.0))
elif name.endswith(("hc_attn_scale", "hc_ffn_scale")):
t.copy_(randn(0.5).abs() + 0.5)
elif name.endswith(("hc_attn_base", "hc_ffn_base", "attn_sink")):
t.copy_(randn(0.1))
elif name.endswith("embed_tokens.weight"):
t.copy_(randn())
else:
t.copy_(randn(0.02))
@register_model_loader(LOAD_FORMAT)
class RandomWeightLoader(DummyModelLoader):
"""Dummy loading with weights in the checkpoint's ranges, before post-load
processing, so every processed copy follows (plain dummy weights leave the
E8M0 scales zero, and both paths would agree on zeros)."""
def load_weights(self, model, model_config) -> None:
from vllm.distributed import get_tensor_model_parallel_world_size
super().load_weights(model, model_config)
for name, t in model.state_dict().items():
_fill(name, t)
# the routed experts' intermediate past this rank's share is the
# loader's zero padding (576 -> 640 at TP4): the kernels skip it
inter = model_config.hf_text_config.moe_intermediate_size
inter //= get_tensor_model_parallel_world_size()
for m in model.modules():
w13, w2 = getattr(m, "w13_weight", None), getattr(m, "w2_weight", None)
if w13 is None or w2 is None or w13.device.type == "meta":
continue
padded = w13.shape[1] // 2
if padded > inter:
w13, w2 = w13.data.view(torch.uint8), w2.data.view(torch.uint8)
for half in (0, padded):
w13[:, half + inter : half + padded] = 0
w2[..., inter // 2 :] = 0
class MonoProbe:
"""Worker extension (``worker_extension_cls``): it also brings this module,
and so the loader, into the workers."""
def _decoder_layers(self) -> dict:
from vllm.models.deepseek_v41.amd.model import DeepseekV4DecoderLayer
model = self.model_runner.model # type: ignore[attr-defined]
return {
m.attn.layer_id: m
for m in model.modules()
if isinstance(m, DeepseekV4DecoderLayer)
}
def capture_steps(self, lens: list[int]) -> None:
"""Keep each decoder layer's outputs (x, residual) for every decode
token taken in a decode-only step (the steps the mono path takes),
keyed by (layer, prompt length, position): a request's step and row
can differ between engines (requests reach the engine core while it
runs), its prompt length and position cannot. ``lens``: the prompt
lengths, all distinct; a row's request is the longest prompt not past
its position."""
from vllm.forward_context import get_forward_context
self._captured: dict = {}
lens = sorted(lens)
def hook(layer_id):
def keep(module, args, kwargs, out):
md = get_forward_context().attn_metadata
if not isinstance(md, dict):
return
swa = md.get(module.attn.swa_cache_layer.prefix)
if swa is None or swa.num_prefills != 0:
return
pos = kwargs.get("positions", args[1] if len(args) > 1 else None)
for row, p in enumerate(pos.tolist()):
n = max(n for n in lens if n <= p)
kept = [t[row].float().cpu() for t in out[:2]]
self._captured.setdefault((layer_id, n, p), kept)
return keep
def gap_hook(layer_id, layer):
def keep(ffn, args, kwargs):
# vLLM's path only (the mono kernels route inside): each row's
# top-6 margin, sqrt(softplus(logits)) + bias as vLLM routes
md = get_forward_context().attn_metadata
if not isinstance(md, dict):
return
swa = md.get(layer.attn.swa_cache_layer.prefix)
if swa is None or swa.num_prefills != 0:
return
h = args[0] if args else kwargs["hidden_states"]
logits = h.float() @ ffn.gate.weight.float().T
score = torch.nn.functional.softplus(logits).sqrt()
score = score + ffn.gate.e_score_correction_bias.float()
top = score.topk(7, dim=-1).values
gap = (top[:, 5] - top[:, 6]).tolist()
pos = self._positions
for row, p in enumerate(pos.tolist()):
n = max(n for n in lens if n <= p)
self._gaps.setdefault((layer_id, n, p), gap[row])
return keep
def pos_hook(module, args, kwargs):
self._positions = kwargs.get(
"positions", args[1] if len(args) > 1 else None
)
self._gaps: dict = {}
for i, layer in self._decoder_layers().items():
layer.register_forward_hook(hook(i), with_kwargs=True)
layer.register_forward_pre_hook(pos_hook, with_kwargs=True)
gap = gap_hook(i, layer)
layer.ffn.register_forward_pre_hook(gap, with_kwargs=True)
def save_capture(self, path: str) -> None:
rank = self.rank # type: ignore[attr-defined]
torch.save((self._captured, self._gaps), f"{path}.rank{rank}")
def mono_layers(self) -> dict:
from vllm.models.deepseek_v41.amd import mono_decode
layers = {
i: None
if m.mono is None
else ["ffn" if m.mono.ffn_only else "whole", m.mono._weights is not None]
for i, m in self._decoder_layers().items()
}
runner = mono_decode._runner
return {
"layers": layers,
"epoch": -1 if runner is None else int(runner.epoch[0]),
}
def _decode(vllm_runner, monkeypatch, tp: int, mono: bool, prompts, path: str):
"""Greedy decode of ``prompts`` with the mono path on or off: (each
prompt's output, the workers' mono layers); each decoder layer's outputs
for the decode tokens of decode-only steps, and vLLM's routing margins,
saved under ``path``."""
from vllm import SamplingParams
from vllm.inputs import TokensPrompt
monkeypatch.setenv("VLLM_ROCM_MONO_DECODE", "1" if mono else "0")
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
with vllm_runner(
MODEL,
tensor_parallel_size=tp,
load_format=LOAD_FORMAT,
hf_overrides=TRUNCATED,
worker_extension_cls=f"{__name__}.MonoProbe",
moe_backend="aiter",
language_model_only=True,
tokenizer_mode="deepseek_v41",
attention_config={"indexer_kv_dtype": "mxfp4", "indexer_sparse_logits": True},
block_size=128,
enforce_eager=True,
max_model_len=4096,
max_num_seqs=len(prompts),
# a prompt prefills in one step, so its decode tokens never share one
max_num_batched_tokens=16384,
enable_prefix_caching=False,
gpu_memory_utilization=0.5,
) as runner:
runner.llm.collective_rpc("capture_steps", args=([len(p) for p in prompts],))
params = SamplingParams(temperature=0.0, max_tokens=STEPS + 1)
outs = runner.llm.generate(
[TokensPrompt(prompt_token_ids=p) for p in prompts], params
)
runner.llm.collective_rpc("save_capture", args=(path,))
probe = runner.llm.collective_rpc("mono_layers")
return [o.outputs[0] for o in outs], probe
def _rel(a: torch.Tensor, b: torch.Tensor) -> float:
return ((a - b).norm() / b.norm()).item()
@pytest.mark.skipif(not _on_cdna4(), reason="the mono kernels target CDNA4")
@pytest.mark.parametrize("tp", [2, 4])
def test_mono_decode_matches_the_model(vllm_runner, monkeypatch, tmp_path, tp):
if torch.accelerator.device_count() < tp:
pytest.skip(f"needs {tp} GPUs")
pytest.importorskip("flydsl")
g = torch.Generator().manual_seed(0)
# contexts past both ratios' 512 compressed entries, so top-k selects
lens = torch.randperm(1700, generator=g)[:8].add(300).tolist() # distinct
prompts = [torch.randint(1000, 60000, (n,), generator=g).tolist() for n in lens]
on, probe_on = _decode(
vllm_runner, monkeypatch, tp, True, prompts, f"{tmp_path}/on"
)
off, probe_off = _decode(
vllm_runner, monkeypatch, tp, False, prompts, f"{tmp_path}/off"
)
# both layer kinds took the mono path on every rank, at every decode step
launches = len(WHOLE + FFN) # one launch (pair) a layer a step
for rank in probe_on:
assert rank["layers"] == {
i: ["whole" if i in WHOLE else "ffn", True] for i in WHOLE + FFN
}, rank
assert rank["epoch"] >= launches * STEPS, rank
assert rank["epoch"] % launches == 0, rank
assert all(set(r["layers"].values()) == {None} for r in probe_off)
# each decode token's input matches between the runs while the greedy
# tokens before it do: (prompt length, position) pairs to compare
same: set[tuple[int, int]] = set()
for prompt, a, b in zip(prompts, on, off):
for step in range(STEPS):
if a.token_ids[: step + 1] != b.token_ids[: step + 1]:
break
same.add((len(prompt), len(prompt) + step))
# each layer's outputs for those tokens, mono against vLLM's path, token by
# token. Layer 0 sees the same inputs on both paths at a request's first
# decode token: there the bound is test_dsv41_mono_numerics' (about 3x the
# kernels' 0.4%); its later tokens attend over KV each path wrote, 3%. Each
# later layer takes the previous one's slightly different output, which
# random weights amplify (2-4% by layer 4): 8%. A token whose top-6 routing
# is a near-tie on vLLM's path (6th / 7th score within 5e-3: a 0.4% input
# difference moves scores near 0.8 by about 3e-3) may pick another expert;
# once it has, its residual stream differs and the later layers are exempt
# for it. The residual stream: 3% (0.3% seen). A layer the kernels no
# longer follow is off by far more, on every token.
for rank in range(tp):
got, _ = torch.load(f"{tmp_path}/on.rank{rank}")
want, gaps = torch.load(f"{tmp_path}/off.rank{rank}")
keys: list[tuple[int, int]] = sorted(
same & {k[1:] for k in got} & {k[1:] for k in want}
)
assert len(keys) >= len(prompts), keys
# the bound that sees a 3% error: a request's first decode token
assert any(p == n for n, p in keys), keys
flipped: set = set()
for i in sorted(WHOLE + FFN):
for k in keys:
if k in flipped:
continue
(x, res), (x_ref, res_ref) = got[(i, *k)], want[(i, *k)]
err = _rel(res, res_ref)
assert err < 0.03, (rank, i, k, "residual", err)
bound = (0.015 if k[1] == k[0] else 0.03) if i == 0 else 0.08
err = _rel(x, x_ref)
if err >= bound:
assert gaps[(i, *k)] < 5e-3, (rank, i, k, "x", err)
flipped.add(k)
assert len(flipped) <= len(keys) // 4, (rank, sorted(flipped))
+529
View File
@@ -0,0 +1,529 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Numerics of the DeepSeek-V4.1 mono decode kernels at TP2 on CDNA4 against the
ops vLLM runs for the same decode step: random weights in the checkpoint's
formats, the step's KV caches as vLLM's insert writes them.
- The whole layer (K1 + K2) against the mHC seams, the ROCm attention op chain,
the TP all-reduce and ``DeepseekV4MoE``.
- The FFN launch against the same FFN half, from wo_b's unreduced output.
Each stage is compared on its own input, so a failure names the stage; the MoE
is checked on the kernels' own FFN input, where routing cannot differ."""
import pytest
import ray
import torch
from vllm.platforms import current_platform
from ..utils import (
ensure_model_parallel_initialized,
init_test_distributed_environment,
multi_process_parallel,
)
MODEL = "deepseek-ai/DeepSeek-V4.1-Flash"
HIDDEN, Q_RANK, HEAD_DIM, NOPE, ROPE = 5120, 1280, 512, 448, 64
N_HEADS, O_GROUPS, O_RANK = 64, 8, 1024
WINDOW, TOPK, SWA_BLOCK, CACHE_BLOCK = 128, 512, 32, 128
RECORD, DATA, ALIGN = 584, 576, 576
REPLAYS = 300
HC = 4
def _on_cdna4() -> bool:
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import get_cdna_version
return get_cdna_version() == 4
def _cos(a: torch.Tensor, b: torch.Tensor) -> float:
a, b = a.float().reshape(-1), b.float().reshape(-1)
return torch.nn.functional.cosine_similarity(a, b, dim=0).item()
def _rel(a: torch.Tensor, b: torch.Tensor) -> float:
"""Relative L2 error of ``a`` against ``b``: unlike the cosine, it sees a
wrong scale."""
a, b = a.float().reshape(-1), b.float().reshape(-1)
return ((a - b).norm() / b.norm()).item()
# ---------------------------------------------------------------- caches
def _paged_cache(num_blocks: int, block: int, device) -> torch.Tensor:
"""[blocks, block, 584] u8 over pages of round_up(block * 584, 576) bytes,
the way vLLM binds a packed V4 cache."""
page = -(-block * RECORD // ALIGN) * ALIGN
raw = torch.zeros(num_blocks * page, dtype=torch.uint8, device=device)
return torch.as_strided(raw, (num_blocks, block, RECORD), (page, RECORD, 1))
def _rope(kv: torch.Tensor, positions: torch.Tensor, cos_sin: torch.Tensor):
"""GPT-J RoPE of the last 64 dims (fp32 math, bf16 out)."""
cs = cos_sin[positions.long()]
cos, sin = cs[:, :32], cs[:, 32:]
x = kv.float()
e, o = x[:, NOPE::2].clone(), x[:, NOPE + 1 :: 2].clone()
x[:, NOPE::2] = e * cos - o * sin
x[:, NOPE + 1 :: 2] = e * sin + o * cos
return x.to(torch.bfloat16)
def _write_records(cache, block: int, slots, kv):
"""fp8_ds_mla records of roped bf16 kv rows at ``slots``: the 448 nope dims
in e4m3 with a UE8M0 per 64, the rope dims in bf16, as vLLM's insert."""
nope = kv.float()[:, :NOPE].view(-1, 7, 64)
exp = torch.ceil(torch.log2(nope.abs().amax(-1).clamp_min(1e-4) / 448.0))
q = (nope * torch.exp2(-exp)[..., None]).clamp(-448, 448).reshape(-1, NOPE)
data = torch.cat(
[
q.to(torch.float8_e4m3fn).view(torch.uint8),
kv[:, NOPE:].contiguous().view(torch.uint8),
],
1,
)
scales = torch.zeros(kv.shape[0], 8, dtype=torch.uint8, device=kv.device)
scales[:, :7] = (exp + 127).clamp(0, 255).to(torch.uint8)
n = cache.untyped_storage().nbytes() - cache.storage_offset()
flat = torch.as_strided(cache, (n,), (1,))
slots = slots.to(kv.device).long()
b, o = slots // block, slots % block
d0 = b * cache.stride(0) + o * DATA
s0 = b * cache.stride(0) + block * DATA + o * 8
ar = torch.arange(DATA, device=kv.device)
flat[(d0[:, None] + ar).reshape(-1)] = data.reshape(-1)
ar = torch.arange(8, device=kv.device)
flat[(s0[:, None] + ar).reshape(-1)] = scales.reshape(-1)
def _normed_kv(n: int, kv_norm, g, device):
x = torch.randn(n, HEAD_DIM, generator=g).to(device)
x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True)) * kv_norm.float()
return x.to(torch.bfloat16)
def _decode_step(ratio: int, R: int, T: int, kv_norm, cos_sin, seed: int):
"""R requests x T tokens past random contexts: the step's metadata as
vLLM's builders lay it out, its window rows and (ratio > 0) the compressed
cache filled, a random top-512 a token."""
g = torch.Generator().manual_seed(seed)
dev = kv_norm.device
ctx = [int(c) for c in torch.randint(2000, 40000, (R,), generator=g)]
M = R * T
spans = [
(max(0, c - WINDOW - T) // SWA_BLOCK, (c + T - 1) // SWA_BLOCK) for c in ctx
]
n_swa = 1 + sum(hi - lo + 1 for lo, hi in spans)
perm = (torch.randperm(n_swa - 1, generator=g) + 1).tolist()
bt = torch.zeros(R, max(hi for _, hi in spans) + 1, dtype=torch.int64)
for r, (lo, hi) in enumerate(spans):
for lb in range(lo, hi + 1):
bt[r, lb] = perm.pop()
swa = _paged_cache(n_swa, SWA_BLOCK, dev)
def slot(r, p):
return int(bt[r, p // SWA_BLOCK]) * SWA_BLOCK + p % SWA_BLOCK
pos = [ctx[r] + t for r in range(R) for t in range(T)]
t2r = [r for r in range(R) for _ in range(T)]
swa_idx = torch.full((M, WINDOW), -1, dtype=torch.int32)
swa_len = torch.zeros(M, dtype=torch.int32)
for i in range(M):
ps = range(max(pos[i] - WINDOW + 1, 0), pos[i] + 1)
swa_idx[i, : len(ps)] = torch.tensor([slot(t2r[i], p) for p in ps])
swa_len[i] = len(ps)
hist = [
(r, p) for r in range(R) for p in range(max(0, ctx[r] - WINDOW - T), ctx[r])
]
hp = torch.tensor([p for _, p in hist], device=dev)
kv = _rope(_normed_kv(len(hist), kv_norm, g, dev), hp, cos_sin)
_write_records(swa, SWA_BLOCK, torch.tensor([slot(r, p) for r, p in hist]), kv)
step = dict(
M=M,
positions=torch.tensor(pos, dtype=torch.int64, device=dev),
slot_mapping=torch.tensor(
[slot(t2r[i], pos[i]) for i in range(M)], dtype=torch.int64, device=dev
),
token_to_req=torch.tensor(t2r, dtype=torch.int32, device=dev),
swa_indices=swa_idx.to(dev),
swa_lens=swa_len.to(dev),
swa_cache=swa,
topk=None,
comp_cache=None,
comp_block_table=None,
comp_entries=0,
)
if ratio:
entries = CACHE_BLOCK // ratio
lens = [(c + T) // ratio for c in ctx]
nblk = [-(-n // entries) for n in lens]
perm = (torch.randperm(sum(nblk), generator=g) + 1).tolist()
cbt = torch.zeros(R, max(nblk), dtype=torch.int32)
for r in range(R):
for b in range(nblk[r]):
cbt[r, b] = perm.pop()
comp = _paged_cache(1 + sum(nblk), entries, dev)
for r in range(R):
idx = torch.arange(lens[r])
slots = cbt[r, idx // entries].long() * entries + idx % entries
kv = _rope(
_normed_kv(lens[r], kv_norm, g, dev), (idx * ratio).to(dev), cos_sin
)
_write_records(comp, entries, slots, kv)
topk = torch.full((M, TOPK), -1, dtype=torch.int32)
for i in range(M):
cand = (pos[i] + 1) // ratio
n = min(cand, TOPK)
topk[i, :n] = torch.randperm(cand, generator=g)[:n].to(torch.int32)
step.update(
topk=topk.to(dev),
comp_cache=comp,
comp_block_table=cbt.to(dev),
comp_entries=entries,
)
return step
# ---------------------------------------------------------------- weights
def _mxfp8(n: int, k: int, g, device):
"""A random MXFP8 weight [n, k] e4m3 and its E8M0 32 x 32 block scales."""
w = torch.randn(n, k, generator=g).clamp(-6, 6).to(torch.float8_e4m3fn)
s = torch.randint(118, 123, (n // 32, k // 32), generator=g, dtype=torch.uint8)
return w.to(device), s.to(device)
def _random_moe(moe, gd, gr, inter: int) -> None:
"""Random parameters in the loaded layout, before post-load processing: this
rank's shards (``gd``, a device generator) of the MXFP4 routed experts
(random nibbles, E8M0 scales; the intermediate past ``inter`` zero, as the
loader pads it) and the MXFP8 shared expert (one E8M0 a 32 x 32 block); the
replicated router from ``gr``, seeded alike on every rank."""
u8 = torch.uint8
for lin in (moe.shared_experts.gate_up_proj, moe.shared_experts.down_proj):
n = lin.weight.shape[0]
w = torch.randn(lin.weight.shape, generator=gd, device=gd.device)
lin.weight.data.copy_(w.clamp(-6, 6).to(lin.weight.dtype))
sc = lin.weight_scale.data.view(u8)
rows = -(-n // 32) if sc.shape[0] == n else sc.shape[0]
blocks = torch.randint(
118, 123, (rows, sc.shape[1]), generator=gd, device=gd.device, dtype=u8
)
sc.copy_(blocks.repeat_interleave(32, 0)[:n] if sc.shape[0] == n else blocks)
e = moe.experts.routed_experts
for name, lo, hi in (("weight", 0, 256), ("weight_scale", 118, 122)):
for w in ("w13", "w2"):
t = getattr(e, f"{w}_{name}").data.view(u8)
t.copy_(
torch.randint(lo, hi, t.shape, generator=gd, device=gd.device, dtype=u8)
)
padded = e.w13_weight.shape[1] // 2
if padded > inter:
w13, w2 = e.w13_weight.data.view(u8), e.w2_weight.data.view(u8)
for half in (0, padded):
w13[:, half + inter : half + padded] = 0
w2[..., inter // 2 :] = 0
for t in (moe.gate.weight, moe.gate.e_score_correction_bias):
t.data.copy_(torch.randn(t.shape, generator=gr, device=gr.device) * 0.02)
# ---------------------------------------------------------------- the worker
@ray.remote(num_gpus=1, max_calls=1)
def _mono_numerics(monkeypatch, tp_size, pp_size, rank, distributed_init_port):
from vllm._aiter_ops import rocm_aiter_ops
from vllm.v1.worker.workspace import init_workspace_manager
with monkeypatch.context() as m:
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
# the checkpoint's MXFP4 experts run on AITER's MoE; its flags are read
# at import, so refresh them
m.setenv("VLLM_ROCM_USE_AITER", "1")
m.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
rocm_aiter_ops.refresh_env_variables()
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
ensure_model_parallel_initialized(tp_size, pp_size)
init_workspace_manager(device)
_check_layers(rank, tp_size, device)
def _check_layers(rank: int, tp: int, device) -> None:
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.distributed import get_tp_group, tensor_model_parallel_all_reduce
from vllm.engine.arg_utils import EngineArgs
from vllm.forward_context import set_forward_context
from vllm.model_executor.kernels.linear.mxfp8.rocm_block32_gemm import (
rocm_mxfp8_block32_gemm,
)
from vllm.model_executor.layers.mhc import MHCPreDelayedOp
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
mxfp8_e4m3_quantize,
)
from vllm.model_executor.model_loader.utils import process_weights_after_loading
from vllm.models.common.ops import fused_q_kv_rmsnorm
from vllm.models.deepseek_v41.amd.model import DeepseekV4MoE
from vllm.models.deepseek_v41.amd.mono.layer import scratch_layout
from vllm.models.deepseek_v41.amd.mono.runner import (
AttnWeights,
DSV41MonoLayer,
MonoLayerWeights,
)
from vllm.models.deepseek_v41.amd.rocm import (
compute_global_topk_ragged_indices_and_indptr,
)
from vllm.models.deepseek_v41.common.rope import build_deepseek_v4_rope
from vllm.utils.torch_utils import set_default_torch_dtype
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
build_ragged_indices_from_dense,
rocm_inverse_rope_mxfp8_rows,
)
vc = EngineArgs(
model=MODEL,
tensor_parallel_size=tp,
moe_backend="aiter",
load_format="dummy",
max_model_len=65536,
language_model_only=True,
attention_config={"indexer_kv_dtype": "mxfp4", "indexer_sparse_logits": True},
).create_engine_config()
cfg = vc.model_config.hf_config
cfg = getattr(cfg, "text_config", cfg)
# replicated tensors alike on every rank; each rank's own TP shards
g = torch.Generator().manual_seed(1000)
gs = torch.Generator().manual_seed(1001 + rank)
H, G = N_HEADS // tp, O_GROUPS // tp
bf = torch.bfloat16
with set_current_vllm_config(vc), set_default_torch_dtype(bf), device:
moe = DeepseekV4MoE(vc, prefix="model.layers.21.ffn")
mhc = MHCPreDelayedOp()
_random_moe(
moe,
torch.Generator(device=device).manual_seed(2001 + rank),
torch.Generator(device=device).manual_seed(2000),
inter=2304 // tp,
)
process_weights_after_loading(moe, vc.model_config, device)
def f32(*shape, s=1.0, gen=g):
return (torch.randn(*shape, generator=gen) * s).to(device)
mix = (2 + HC) * HC
seams = {
sub: dict(
fn=f32(mix, HC * HIDDEN, s=0.01),
scale=f32(3, s=0.5).abs() + 0.5,
base=f32(mix, s=0.1),
norm=(1 + f32(HIDDEN, s=0.1)).to(bf),
)
for sub in ("attn", "ffn")
}
def seam(sub, residual, x, post, comb, pre):
s = seams[sub]
return mhc(
residual, s["fn"], s["scale"], s["base"], cfg.rms_norm_eps,
cfg.hc_eps, cfg.hc_eps, 2.0, cfg.hc_sinkhorn_iters,
pre_mix=pre, sublayer_out=x, post_layer_mix=post, comb_res_mix=comb,
norm_weight=s["norm"], norm_eps=cfg.rms_norm_eps,
) # fmt: skip
def moe_out(x):
with set_forward_context(None, vc):
ids = torch.zeros(x.shape[0], dtype=torch.int64, device=device)
return moe(x.clone(), ids)
wqkv, wqkv_s = _mxfp8(1792, HIDDEN, g, device)
wq_b, wq_b_s = _mxfp8(H * HEAD_DIM, Q_RANK, gs, device)
wo_a, wo_a_s = _mxfp8(G * O_RANK, H // G * HEAD_DIM, gs, device)
wo_b, wo_b_s = _mxfp8(HIDDEN, G * O_RANK, gs, device)
q_norm, kv_norm = (1 + f32(Q_RANK, s=0.1)).to(bf), (1 + f32(HEAD_DIM, s=0.1)).to(bf)
sink = f32(H, gen=gs)
e, sh = moe.experts.routed_experts, moe.shared_experts
u8 = lambda t: t.view(torch.uint8) # noqa: E731
runner = DSV41MonoLayer(tp, rank, get_tp_group().cpu_group, device)
gw, bias = moe.gate.weight.float(), moe.gate.e_score_correction_bias.float()
def top6(x):
"""The top-6 experts as vLLM routes: sqrt(softplus(logits)) + bias."""
sc = torch.nn.functional.softplus(x.float() @ gw.T).sqrt() + bias
return sc.topk(6, dim=1).indices.sort(1).values
steps = [] # each case's launches, on fixed buffers, for the replay stress
for ratio, R, T in ((1, 1, 6), (2, 8, 6)):
with set_current_vllm_config(VllmConfig()):
rope = build_deepseek_v4_rope(
cfg, head_dim=HEAD_DIM, rope_head_dim=ROPE,
max_position_embeddings=cfg.max_position_embeddings,
compress_ratio=ratio,
) # fmt: skip
cos_sin = rope.cos_sin_cache.to(device=device, dtype=torch.float32)
s = _decode_step(ratio, R, T, kv_norm, cos_sin, seed=7 + ratio)
M = s["M"]
mw = MonoLayerWeights(
attn=AttnWeights(
layer_id=21, wqkv=wqkv, wqkv_scale=wqkv_s, q_norm=q_norm,
kv_norm=kv_norm, wq_b=wq_b, wq_b_scale=wq_b_s, wo_a=wo_a,
wo_a_scale=wo_a_s, wo_b=wo_b, wo_b_scale=wo_b_s, attn_sink=sink,
cos_sin=cos_sin, ratio=ratio,
),
hc_attn_fn=seams["attn"]["fn"], hc_attn_scale=seams["attn"]["scale"],
hc_attn_base=seams["attn"]["base"], attn_norm=seams["attn"]["norm"],
hc_ffn_fn=seams["ffn"]["fn"], hc_ffn_scale=seams["ffn"]["scale"],
hc_ffn_base=seams["ffn"]["base"], ffn_norm=seams["ffn"]["norm"],
gate_w=moe.gate.weight, bias=moe.gate.e_score_correction_bias,
w13=e.w13_weight, w13_s=e.w13_weight_scale, w2=e.w2_weight,
w2_s=e.w2_weight_scale, sgu=sh.gate_up_proj.weight,
sgu_s=u8(sh.gate_up_proj.weight_scale), sw2=sh.down_proj.weight,
sw2_s=u8(sh.down_proj.weight_scale),
) # fmt: skip
# the layer's inputs at the attention seam (the same on every rank)
gi = torch.Generator().manual_seed(11)
res0 = torch.randn(M, HC, HIDDEN, generator=gi)
res0 = (
(res0 * torch.rsqrt(res0.pow(2).mean(-1, keepdim=True))).to(bf).to(device)
)
x0 = (torch.randn(M, HIDDEN, generator=gi) * 0.5).to(bf).to(device)
comb0 = torch.rand(M, HC, HC, generator=gi) + 0.1
for _ in range(20):
comb0 = comb0 / comb0.sum(-1, keepdim=True)
comb0 = comb0 / comb0.sum(-2, keepdim=True)
comb0 = comb0.to(device)
post0 = (torch.sigmoid(torch.randn(M, HC, 1, generator=gi)) * 2).to(device)
pre0 = (torch.sigmoid(torch.randn(M, HC, generator=gi)) + 1e-6).to(device)
# vLLM's decode chain
res, post, comb, xa, attn_pre = seam("attn", res0, x0, post0, comb0, pre0)
swa_ind, swa_ptr = build_ragged_indices_from_dense(
s["swa_indices"].reshape(M, -1), s["swa_lens"]
)
top_ind = top_ptr = None
if ratio:
valid = torch.ones(M, dtype=torch.bool, device=device)
top_ind, top_ptr, _ = compute_global_topk_ragged_indices_and_indptr(
s["topk"], s["token_to_req"], s["comp_block_table"],
s["comp_entries"], valid,
) # fmt: skip
x8, xs = mxfp8_e4m3_quantize(xa)
qr_kv = rocm_mxfp8_block32_gemm(x8, xs, wqkv, wqkv_s, bf)
qr, kv = fused_q_kv_rmsnorm(
qr_kv[:, :Q_RANK], qr_kv[:, Q_RANK:], q_norm, kv_norm, cfg.rms_norm_eps
)
q8, qs = mxfp8_e4m3_quantize(qr)
q = rocm_mxfp8_block32_gemm(q8, qs, wq_b, wq_b_s, bf).view(M, H, HEAD_DIM)
swa2d = s["swa_cache"].view(s["swa_cache"].shape[0], -1)
q_roped = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
q, kv, swa2d, s["slot_mapping"], s["positions"], cos_sin, H,
cfg.rms_norm_eps, SWA_BLOCK, False, False, True, False,
) # fmt: skip
attn = torch.empty(M, H, HEAD_DIM, dtype=bf, device=device)
rocm_aiter_ops.triton_sparse_mla_fwd(
q_roped, s["swa_cache"], attn, HEAD_DIM**-0.5, swa_ptr, swa_ind,
attn_sink=sink, extra_kv_buffer=s["comp_cache"],
extra_kv_indptr=top_ptr, extra_kv_indices=top_ind,
) # fmt: skip
a8 = torch.empty(M, H * HEAD_DIM, dtype=torch.float8_e4m3fn, device=device)
a_s = torch.empty(M, H * HEAD_DIM // 32, dtype=torch.uint8, device=device)
rocm_inverse_rope_mxfp8_rows(attn, s["positions"], cos_sin, ROPE, a8, a_s)
z = torch.ops.vllm.rocm_dsv41_mxfp8_wo_a_bmm(a8, a_s, wo_a, wo_a_s, G, O_RANK)
z8, zs = mxfp8_e4m3_quantize(z)
part = rocm_mxfp8_block32_gemm(z8, zs, wo_b, wo_b_s, bf).contiguous()
a = tensor_model_parallel_all_reduce(part.clone())
res2, post2, comb2, xf, ffn_pre = seam("ffn", res, a, post, comb, attn_pre)
out = moe_out(xf)
# ---- the whole layer: K1 + K2
o = runner.forward(
mw, x0, res0, post0, comb0, pre0, s["positions"], s["slot_mapping"],
s["swa_cache"], s["swa_indices"], s["swa_lens"], s["token_to_req"],
topk_indices=s["topk"], comp_cache=s["comp_cache"],
comp_block_table=s["comp_block_table"],
) # fmt: skip
off = scratch_layout(M, tp)["normed"][0]
normed = runner.scratch(M)[off : off + M * HIDDEN * 2].view(bf).view(M, HIDDEN)
tag = f"ratio {ratio}, {R} x {T}"
assert _cos(runner.res_mid[:M], res) > 0.99999, tag
for got, want in (
(runner.post_a, post),
(runner.comb_a, comb),
(runner.pre_a, attn_pre),
):
torch.testing.assert_close(
got[:M].view_as(want), want, rtol=1e-4, atol=1e-5
)
same_in = moe_out(normed)
# relative errors about 3x what the kernels show (0.3-0.5%)
assert _cos(o[1], res2) > 0.9999 and _rel(o[1], res2) < 0.01, tag
assert _cos(normed, xf) > 0.999 and _rel(normed, xf) < 0.012, tag
assert _cos(o[0], same_in) > 0.9999 and _rel(o[0], same_in) < 0.015, tag
# per token, where both FFN inputs pick the same experts
same = (top6(xf) == top6(normed)).all(1)
assert same.float().mean() >= 0.5, tag
tok = torch.nn.functional.cosine_similarity(o[0].float(), out.float(), dim=1)
assert tok[same].min() > 0.99, tag
# ---- the FFN launch, from the same unreduced wo_b output
f = runner.ffn(mw, part, res, post, comb, attn_pre)
torch.testing.assert_close(f[1], res2, rtol=2**-8, atol=1e-6)
torch.testing.assert_close(normed, xf, rtol=2**-7, atol=1e-6)
assert _cos(f[0], out) > 0.9999 and _rel(f[0], out) < 0.015, tag
torch.accelerator.synchronize()
whole, ffn = runner._outs(M, res0), runner._outs(M, res)
args = (s["positions"], s["slot_mapping"], s["swa_cache"], s["swa_indices"])
meta = dict(
topk_indices=s["topk"], comp_cache=s["comp_cache"],
comp_block_table=s["comp_block_table"],
) # fmt: skip
def step(mw=mw, x0=x0, res0=res0, post0=post0, comb0=comb0, pre0=pre0,
args=args, s=s, meta=meta, whole=whole, part=part, res=res,
post=post, comb=comb, attn_pre=attn_pre, ffn=ffn): # fmt: skip
runner.forward(
mw, x0, res0, post0, comb0, pre0, *args, s["swa_lens"],
s["token_to_req"], **meta, outs=whole,
) # fmt: skip
runner.ffn(mw, part, res, post, comb, attn_pre, outs=ffn)
return [*whole, *ffn]
steps.append(step)
# ---- the hand-off protocol under back-to-back launches: both step widths
# alternating in one graph, every replay bit for bit the first's
outs = [t for step in steps for t in step()]
torch.accelerator.synchronize()
ref = [t.clone() for t in outs]
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
for step in steps:
step()
for i in range(REPLAYS):
graph.replay()
if i % 50 == 49 or i == REPLAYS - 1:
torch.accelerator.synchronize()
bad = [
j for j, (t, r) in enumerate(zip(outs, ref)) if not torch.equal(t, r)
]
assert not bad, f"replay {i}: outputs {bad} differ from the first run"
@pytest.mark.skipif(not _on_cdna4(), reason="the mono kernels target CDNA4")
@pytest.mark.parametrize("tp", [2, 4])
def test_mono_decode_layer_matches_vllm(monkeypatch: pytest.MonkeyPatch, tp: int):
if torch.accelerator.device_count() < tp:
pytest.skip(f"needs {tp} GPUs")
pytest.importorskip("flydsl")
multi_process_parallel(monkeypatch, tp, 1, _mono_numerics)