[ROCm][Perf][DSv4.1] Mono decode layer for DeepSeek-V4.1-Flash on MI355X (#60397)
Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
3c802af3cf
commit
193922d6ff
@@ -0,0 +1,317 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The DeepSeek-V4.1 mono decode layer runs only where its kernels are exact:
|
||||
decode-only steps of at most MAX_ROWS rows with causal SWA windows, eager or in
|
||||
a FULL graph. A PIECEWISE capture is replayed for mixed batches, so it must
|
||||
never record the mono launches. Layers outside the whole-layer kernels run the
|
||||
FFN launch on the same steps, from wo_b's unreduced output."""
|
||||
|
||||
import sys
|
||||
from types import ModuleType, SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend
|
||||
from vllm.models.deepseek_v41.amd import mono_decode as md
|
||||
|
||||
M = 6
|
||||
|
||||
|
||||
class _Runner:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def forward(self, *args, **kwargs):
|
||||
self.calls.append((args, kwargs))
|
||||
return ("mono outputs",)
|
||||
|
||||
def ffn(self, *args):
|
||||
self.calls.append((args, {}))
|
||||
return ("ffn outputs",)
|
||||
|
||||
|
||||
def _layer(comp="comp"):
|
||||
cache = lambda: torch.zeros(4, 128 * md.RECORD, dtype=torch.uint8) # noqa: E731
|
||||
attn = SimpleNamespace(
|
||||
swa_cache_layer=SimpleNamespace(prefix="swa", kv_cache=cache()),
|
||||
compressed_cache_prefix=comp,
|
||||
topk_indices_buffer=torch.zeros(64, 512, dtype=torch.int32),
|
||||
compress_ratio=1,
|
||||
_compressed_kv_cache=cache,
|
||||
)
|
||||
return SimpleNamespace(attn=attn)
|
||||
|
||||
|
||||
def _swa(**overrides):
|
||||
fields = dict(
|
||||
num_prefills=0,
|
||||
num_decodes=1,
|
||||
num_decode_tokens=M,
|
||||
decode_swa_width=md.SWA_WIDTH,
|
||||
decode_swa_indices=torch.zeros(64, 1, md.SWA_WIDTH, dtype=torch.int32),
|
||||
decode_swa_lens=torch.zeros(64, dtype=torch.int32),
|
||||
token_to_req_indices=torch.zeros(64, dtype=torch.int32),
|
||||
slot_mapping=torch.zeros(64, dtype=torch.int64),
|
||||
block_size=32,
|
||||
)
|
||||
fields.update(overrides)
|
||||
return SimpleNamespace(**fields)
|
||||
|
||||
|
||||
def _inputs(rows=M):
|
||||
return dict(
|
||||
x=torch.zeros(rows, 5120, dtype=torch.bfloat16),
|
||||
positions=torch.zeros(rows, dtype=torch.int64),
|
||||
residual=torch.zeros(rows, 4, 5120, dtype=torch.bfloat16),
|
||||
post_mix=torch.zeros(rows, 4, 1),
|
||||
res_mix=torch.zeros(rows, 4, 4),
|
||||
pre_mix=torch.zeros(rows, 4),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def run(monkeypatch):
|
||||
"""Call the mono path for one layer under a forward context built from
|
||||
``mode`` and ``metadata``; returns (result, runner calls)."""
|
||||
|
||||
def _run(
|
||||
mode=CUDAGraphMode.FULL, metadata=None, ffn_only=False, entry=None, **inputs
|
||||
):
|
||||
if metadata is None:
|
||||
metadata = {
|
||||
"swa": _swa(),
|
||||
"comp": SimpleNamespace(block_size=128, block_table=None),
|
||||
}
|
||||
fc = SimpleNamespace(cudagraph_runtime_mode=mode, attn_metadata=metadata)
|
||||
runner = _Runner()
|
||||
monkeypatch.setattr(md, "is_forward_context_available", lambda: True)
|
||||
monkeypatch.setattr(md, "get_forward_context", lambda: fc)
|
||||
monkeypatch.setattr(md, "_mono_runner", lambda device: runner)
|
||||
monkeypatch.setattr(md.MonoDecodeLayer, "weights", lambda self, layer: None)
|
||||
args = {**_inputs(), **inputs}
|
||||
mono = md.MonoDecodeLayer(ffn_only)
|
||||
if (entry or ("ffn" if ffn_only else "layer")) == "ffn":
|
||||
# x stands for wo_b's unreduced output; the first layer's attention
|
||||
# has no compressed cache
|
||||
part, _ = args.pop("x"), args.pop("positions")
|
||||
return mono.ffn(_layer(comp=None), part, **args), runner.calls
|
||||
return mono(_layer(), **args), runner.calls
|
||||
|
||||
return _run
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", [CUDAGraphMode.NONE, CUDAGraphMode.FULL])
|
||||
def test_decode_step_runs_mono(run, mode):
|
||||
out, calls = run(mode)
|
||||
assert out == ("mono outputs",) and len(calls) == 1
|
||||
args, kwargs = calls[0]
|
||||
# the caches reach the kernels as [blocks, block, record] bytes
|
||||
assert args[8].shape == (4, 32, md.RECORD) and args[8].stride() == (
|
||||
128 * md.RECORD,
|
||||
md.RECORD,
|
||||
1,
|
||||
)
|
||||
assert kwargs["comp_cache"].shape == (4, 128, md.RECORD)
|
||||
|
||||
|
||||
def test_padded_decode_step_runs_mono(run):
|
||||
"""A FULL graph pads the step to its capture size: the kernels take it and
|
||||
skip the pad rows (their slot is -1)."""
|
||||
swa = _swa(num_decode_tokens=M - 2)
|
||||
comp = SimpleNamespace(block_size=128, block_table=None)
|
||||
out, calls = run(metadata={"swa": swa, "comp": comp})
|
||||
assert out == ("mono outputs",) and len(calls) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
[
|
||||
dict(mode=CUDAGraphMode.PIECEWISE),
|
||||
dict(
|
||||
metadata={
|
||||
"swa": _swa(num_prefills=1),
|
||||
"comp": SimpleNamespace(block_size=128),
|
||||
}
|
||||
),
|
||||
dict(
|
||||
metadata={
|
||||
"swa": _swa(decode_swa_width=256),
|
||||
"comp": SimpleNamespace(block_size=128),
|
||||
}
|
||||
),
|
||||
dict(
|
||||
metadata={
|
||||
"swa": _swa(num_decode_tokens=M + 1),
|
||||
"comp": SimpleNamespace(block_size=128),
|
||||
}
|
||||
),
|
||||
dict(metadata={}), # a profile / dummy run: no attention metadata
|
||||
dict(**_inputs(md.MAX_ROWS + 6)),
|
||||
dict(residual=None), # the first layer's seam broadcasts the embedding
|
||||
],
|
||||
ids=[
|
||||
"piecewise",
|
||||
"prefill",
|
||||
"noncausal-window",
|
||||
"tokens-past-rows",
|
||||
"no-metadata",
|
||||
"rows",
|
||||
"first-layer",
|
||||
],
|
||||
)
|
||||
def test_other_steps_take_the_original_path(run, case):
|
||||
out, calls = run(**case)
|
||||
assert out is None and not calls
|
||||
|
||||
|
||||
def _decoder_layer(engram=None, **attn):
|
||||
attn = (
|
||||
dict(
|
||||
layer_id=21,
|
||||
compressor=None,
|
||||
indexer=None,
|
||||
compress_ratio=1,
|
||||
kv_mxfp8=False,
|
||||
kv_cache_dtype="fp8_ds_mla",
|
||||
)
|
||||
| attn
|
||||
)
|
||||
backend = Mxfp4MoeBackend.AITER_MXFP4_BF16
|
||||
quant_method = SimpleNamespace(mxfp4_backend=backend)
|
||||
ffn = SimpleNamespace(
|
||||
experts=SimpleNamespace(
|
||||
routed_experts=SimpleNamespace(quant_method=quant_method)
|
||||
),
|
||||
shared_experts=object(),
|
||||
gate=SimpleNamespace(tid2eid=None),
|
||||
n_routed_experts=384,
|
||||
n_activated_experts=6,
|
||||
routed_scaling_factor=1.5,
|
||||
swiglu_limit=10.0,
|
||||
scoring_func="sqrtsoftplus",
|
||||
renormalize=True,
|
||||
)
|
||||
return SimpleNamespace(
|
||||
attn=SimpleNamespace(**attn),
|
||||
ffn=ffn,
|
||||
engram=engram,
|
||||
use_sequence_parallel=False,
|
||||
fuse_seam_norm=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"layer, path",
|
||||
[
|
||||
(_decoder_layer(), "whole"),
|
||||
(_decoder_layer(compress_ratio=0), "ffn"),
|
||||
(_decoder_layer(engram=object()), "ffn"),
|
||||
(_decoder_layer(compressor=object(), indexer=object()), "ffn"),
|
||||
(_decoder_layer(indexer=object()), "ffn"),
|
||||
(_decoder_layer(kv_cache_dtype="auto"), "ffn"),
|
||||
(_decoder_layer(layer_id=40), None),
|
||||
],
|
||||
ids=["standard", "first", "engram", "kv-source", "index-source", "kv", "draft"],
|
||||
)
|
||||
def test_layers_take_whole_or_ffn_launch(monkeypatch, layer, path):
|
||||
"""The whole-layer kernels serve the standard layers; every other backbone
|
||||
layer keeps vLLM's attention and runs its FFN half as one launch."""
|
||||
_deployment(monkeypatch)
|
||||
mono = md.MonoDecodeLayer.create(layer, _config())
|
||||
got = None if mono is None else "ffn" if mono.ffn_only else "whole"
|
||||
assert got == path
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", [CUDAGraphMode.NONE, CUDAGraphMode.FULL])
|
||||
def test_decode_step_runs_ffn_launch(run, mode):
|
||||
out, calls = run(mode, ffn_only=True, metadata={"swa": _swa()})
|
||||
assert out == ("ffn outputs",) and len(calls) == 1
|
||||
assert calls[0][0][1].shape == (M, 5120) # wo_b's unreduced output
|
||||
|
||||
|
||||
def test_ffn_layer_never_runs_whole(run):
|
||||
out, calls = run(ffn_only=True, entry="layer")
|
||||
assert out is None and not calls
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
[
|
||||
dict(mode=CUDAGraphMode.PIECEWISE),
|
||||
dict(metadata={"swa": _swa(num_prefills=1)}),
|
||||
dict(metadata={"swa": _swa(num_decode_tokens=M + 1)}),
|
||||
dict(metadata={}),
|
||||
dict(**_inputs(md.MAX_ROWS + 6)),
|
||||
],
|
||||
ids=["piecewise", "prefill", "tokens-past-rows", "no-metadata", "rows"],
|
||||
)
|
||||
def test_other_steps_reduce_wo_b_themselves(run, case):
|
||||
"""None: the layer all-reduces wo_b's partial and runs the FFN as before."""
|
||||
case.setdefault("metadata", {"swa": _swa()})
|
||||
out, calls = run(ffn_only=True, **case)
|
||||
assert out is None and not calls
|
||||
|
||||
|
||||
def _config(**parallel):
|
||||
parallel = (
|
||||
dict(enable_expert_parallel=False, enable_eplb=False, data_parallel_size=1)
|
||||
| parallel
|
||||
)
|
||||
return SimpleNamespace(
|
||||
model_config=SimpleNamespace(hf_config=SimpleNamespace(num_hidden_layers=40)),
|
||||
parallel_config=SimpleNamespace(**parallel),
|
||||
)
|
||||
|
||||
|
||||
def _deployment(monkeypatch, cdna=4, cus=256):
|
||||
"""A deployment the kernels take (the runner module stubbed: no FlyDSL)."""
|
||||
import vllm.platforms.rocm as rocm
|
||||
|
||||
monkeypatch.setenv("VLLM_ROCM_MONO_DECODE", "1")
|
||||
monkeypatch.delenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4", raising=False)
|
||||
monkeypatch.setattr(rocm, "get_cdna_version", lambda: cdna)
|
||||
monkeypatch.setattr(md, "get_tensor_model_parallel_world_size", lambda: 2)
|
||||
monkeypatch.setattr(md, "_compute_units", lambda: cus)
|
||||
name = "vllm.models.deepseek_v41.amd.mono.runner"
|
||||
runner = ModuleType(name)
|
||||
runner.BLOCKS = 256 # type: ignore[attr-defined]
|
||||
monkeypatch.setitem(sys.modules, name, runner)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cdna", [3, 5])
|
||||
def test_opt_in_outside_the_kernels_cdna_raises(monkeypatch, cdna):
|
||||
_deployment(monkeypatch, cdna=cdna)
|
||||
with pytest.raises(ValueError, match="needs CDNA4"):
|
||||
md.MonoDecodeLayer.create(_decoder_layer(), _config())
|
||||
|
||||
|
||||
def _other_backend():
|
||||
layer = _decoder_layer()
|
||||
experts = layer.ffn.experts.routed_experts
|
||||
experts.quant_method.mxfp4_backend = Mxfp4MoeBackend.TRITON
|
||||
return layer
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case, match",
|
||||
[
|
||||
(dict(parallel=dict(enable_expert_parallel=True)), "no expert"),
|
||||
(dict(parallel=dict(data_parallel_size=2)), "no expert"),
|
||||
(dict(parallel=dict(enable_eplb=True)), "no expert"),
|
||||
(dict(a4w4=True), "A8W4"),
|
||||
(dict(layer=_other_backend), "AITER_MXFP4_BF16"),
|
||||
(dict(cus=128), "256 compute units"),
|
||||
],
|
||||
ids=["expert-parallel", "data-parallel", "eplb", "a4w4", "moe-backend", "cus"],
|
||||
)
|
||||
def test_opt_in_outside_the_kernels_moe_raises(monkeypatch, case, match):
|
||||
"""The kernels read all 384 experts on each rank in AITER's A8W4 layout, with
|
||||
all 256 CTAs resident: any other deployment is an error, not garbage."""
|
||||
_deployment(monkeypatch, cus=case.get("cus", 256))
|
||||
if case.get("a4w4"):
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4", "1")
|
||||
layer = case.get("layer", _decoder_layer)()
|
||||
with pytest.raises(ValueError, match=match):
|
||||
md.MonoDecodeLayer.create(layer, _config(**case.get("parallel", {})))
|
||||
@@ -0,0 +1,335 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The DeepSeek-V4.1 mono decode path through the real model: a five-layer
|
||||
DeepSeek-V4.1-Flash at TP2 and TP4 on CDNA4, served with ``VLLM_ROCM_MONO_DECODE`` on
|
||||
and off on the same random weights: every decoder layer's outputs at every
|
||||
decode step must match. Unlike ``test_dsv41_mono_numerics.py``, which
|
||||
checks the kernels against vLLM's ops one stage at a time, this runs
|
||||
``DeepseekV4DecoderLayer.forward`` as the engine does -- the model's hooks,
|
||||
the per-step path choice, the weights taken off the layer, the metadata and
|
||||
the indexer's top-k -- so a change to the decoder layer that the kernels do
|
||||
not follow fails here.
|
||||
|
||||
The five layers hold both kinds the mono path takes:
|
||||
- 0: sliding window only -> the FFN launch
|
||||
- 1: ratio-2 KV and index source -> the FFN launch
|
||||
- 2: ratio-2 consumer -> the whole layer (K1 + K2)
|
||||
- 3: ratio-1 KV, index and candidate source -> the FFN launch
|
||||
- 4: ratio-1 consumer -> the whole layer
|
||||
"""
|
||||
|
||||
import zlib
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.model_loader import register_model_loader
|
||||
from vllm.model_executor.model_loader.dummy_loader import DummyModelLoader
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
MODEL = "deepseek-ai/DeepSeek-V4.1-Flash"
|
||||
LOAD_FORMAT = "dsv41_mono_random"
|
||||
TRUNCATED = {
|
||||
"num_hidden_layers": 5,
|
||||
# one entry a layer, the three MTP layers' included
|
||||
"compress_ratios": [0, 2, 2, 1, 1, 0, 0, 0],
|
||||
"kv_source_layer_ids": [1, 3],
|
||||
"index_source_layer_ids": [1, 3],
|
||||
"candidate_source_layer_id": 3,
|
||||
"engram_layer_ids": [],
|
||||
"engram_num_embeddings": [],
|
||||
"dspark_target_layer_ids": [2, 3, 4],
|
||||
}
|
||||
WHOLE, FFN = (2, 4), (0, 1, 3)
|
||||
STEPS = 5 # decode steps after the prefill's token
|
||||
|
||||
|
||||
def _on_cdna4() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
|
||||
return get_cdna_version() == 4
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def _fill(name: str, t: torch.Tensor) -> None:
|
||||
"""A random value for every weight, seeded by its name (alike on every
|
||||
rank and in every run), in the ranges the checkpoint's formats hold."""
|
||||
if t.device.type == "meta":
|
||||
return # materialized and filled by the dummy loader's own path
|
||||
g = torch.Generator(device=t.device).manual_seed(zlib.crc32(name.encode()))
|
||||
|
||||
def randn(scale=1.0, shift=0.0):
|
||||
return torch.randn(t.shape, generator=g, device=t.device) * scale + shift
|
||||
|
||||
if t.dtype == torch.uint8 and "scale" in name:
|
||||
# E8M0 2^-12 .. 2^-8, alike over each 32 rows as in the checkpoint (the
|
||||
# loader folds per-row scales into 32 x 32 blocks only then). Larger
|
||||
# random layers make the stack chaotic: atomics' order alone moves
|
||||
# vLLM's own logits run to run.
|
||||
rows = t.shape[-2] if t.dim() > 1 else 1
|
||||
shape = (*t.shape[:-2], -(-rows // 32), t.shape[-1]) if t.dim() > 1 else t.shape
|
||||
e8m0 = torch.randint(115, 120, shape, generator=g, device=t.device)
|
||||
if t.dim() > 1:
|
||||
e8m0 = e8m0.repeat_interleave(32, dim=-2)[..., :rows, :]
|
||||
t.copy_(e8m0)
|
||||
elif t.dtype in (torch.uint8, torch.float4_e2m1fn_x2):
|
||||
u8 = t.view(torch.uint8)
|
||||
u8.copy_(torch.randint(0, 256, u8.shape, generator=g, device=t.device))
|
||||
elif t.dtype == torch.float8_e4m3fn:
|
||||
t.copy_(randn().clamp(-6, 6))
|
||||
elif not torch.is_floating_point(t):
|
||||
return
|
||||
elif "norm" in name:
|
||||
t.copy_(randn(0.1, 1.0))
|
||||
elif name.endswith(("hc_attn_scale", "hc_ffn_scale")):
|
||||
t.copy_(randn(0.5).abs() + 0.5)
|
||||
elif name.endswith(("hc_attn_base", "hc_ffn_base", "attn_sink")):
|
||||
t.copy_(randn(0.1))
|
||||
elif name.endswith("embed_tokens.weight"):
|
||||
t.copy_(randn())
|
||||
else:
|
||||
t.copy_(randn(0.02))
|
||||
|
||||
|
||||
@register_model_loader(LOAD_FORMAT)
|
||||
class RandomWeightLoader(DummyModelLoader):
|
||||
"""Dummy loading with weights in the checkpoint's ranges, before post-load
|
||||
processing, so every processed copy follows (plain dummy weights leave the
|
||||
E8M0 scales zero, and both paths would agree on zeros)."""
|
||||
|
||||
def load_weights(self, model, model_config) -> None:
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
|
||||
super().load_weights(model, model_config)
|
||||
for name, t in model.state_dict().items():
|
||||
_fill(name, t)
|
||||
# the routed experts' intermediate past this rank's share is the
|
||||
# loader's zero padding (576 -> 640 at TP4): the kernels skip it
|
||||
inter = model_config.hf_text_config.moe_intermediate_size
|
||||
inter //= get_tensor_model_parallel_world_size()
|
||||
for m in model.modules():
|
||||
w13, w2 = getattr(m, "w13_weight", None), getattr(m, "w2_weight", None)
|
||||
if w13 is None or w2 is None or w13.device.type == "meta":
|
||||
continue
|
||||
padded = w13.shape[1] // 2
|
||||
if padded > inter:
|
||||
w13, w2 = w13.data.view(torch.uint8), w2.data.view(torch.uint8)
|
||||
for half in (0, padded):
|
||||
w13[:, half + inter : half + padded] = 0
|
||||
w2[..., inter // 2 :] = 0
|
||||
|
||||
|
||||
class MonoProbe:
|
||||
"""Worker extension (``worker_extension_cls``): it also brings this module,
|
||||
and so the loader, into the workers."""
|
||||
|
||||
def _decoder_layers(self) -> dict:
|
||||
from vllm.models.deepseek_v41.amd.model import DeepseekV4DecoderLayer
|
||||
|
||||
model = self.model_runner.model # type: ignore[attr-defined]
|
||||
return {
|
||||
m.attn.layer_id: m
|
||||
for m in model.modules()
|
||||
if isinstance(m, DeepseekV4DecoderLayer)
|
||||
}
|
||||
|
||||
def capture_steps(self, lens: list[int]) -> None:
|
||||
"""Keep each decoder layer's outputs (x, residual) for every decode
|
||||
token taken in a decode-only step (the steps the mono path takes),
|
||||
keyed by (layer, prompt length, position): a request's step and row
|
||||
can differ between engines (requests reach the engine core while it
|
||||
runs), its prompt length and position cannot. ``lens``: the prompt
|
||||
lengths, all distinct; a row's request is the longest prompt not past
|
||||
its position."""
|
||||
from vllm.forward_context import get_forward_context
|
||||
|
||||
self._captured: dict = {}
|
||||
lens = sorted(lens)
|
||||
|
||||
def hook(layer_id):
|
||||
def keep(module, args, kwargs, out):
|
||||
md = get_forward_context().attn_metadata
|
||||
if not isinstance(md, dict):
|
||||
return
|
||||
swa = md.get(module.attn.swa_cache_layer.prefix)
|
||||
if swa is None or swa.num_prefills != 0:
|
||||
return
|
||||
pos = kwargs.get("positions", args[1] if len(args) > 1 else None)
|
||||
for row, p in enumerate(pos.tolist()):
|
||||
n = max(n for n in lens if n <= p)
|
||||
kept = [t[row].float().cpu() for t in out[:2]]
|
||||
self._captured.setdefault((layer_id, n, p), kept)
|
||||
|
||||
return keep
|
||||
|
||||
def gap_hook(layer_id, layer):
|
||||
def keep(ffn, args, kwargs):
|
||||
# vLLM's path only (the mono kernels route inside): each row's
|
||||
# top-6 margin, sqrt(softplus(logits)) + bias as vLLM routes
|
||||
md = get_forward_context().attn_metadata
|
||||
if not isinstance(md, dict):
|
||||
return
|
||||
swa = md.get(layer.attn.swa_cache_layer.prefix)
|
||||
if swa is None or swa.num_prefills != 0:
|
||||
return
|
||||
h = args[0] if args else kwargs["hidden_states"]
|
||||
logits = h.float() @ ffn.gate.weight.float().T
|
||||
score = torch.nn.functional.softplus(logits).sqrt()
|
||||
score = score + ffn.gate.e_score_correction_bias.float()
|
||||
top = score.topk(7, dim=-1).values
|
||||
gap = (top[:, 5] - top[:, 6]).tolist()
|
||||
pos = self._positions
|
||||
for row, p in enumerate(pos.tolist()):
|
||||
n = max(n for n in lens if n <= p)
|
||||
self._gaps.setdefault((layer_id, n, p), gap[row])
|
||||
|
||||
return keep
|
||||
|
||||
def pos_hook(module, args, kwargs):
|
||||
self._positions = kwargs.get(
|
||||
"positions", args[1] if len(args) > 1 else None
|
||||
)
|
||||
|
||||
self._gaps: dict = {}
|
||||
for i, layer in self._decoder_layers().items():
|
||||
layer.register_forward_hook(hook(i), with_kwargs=True)
|
||||
layer.register_forward_pre_hook(pos_hook, with_kwargs=True)
|
||||
gap = gap_hook(i, layer)
|
||||
layer.ffn.register_forward_pre_hook(gap, with_kwargs=True)
|
||||
|
||||
def save_capture(self, path: str) -> None:
|
||||
rank = self.rank # type: ignore[attr-defined]
|
||||
torch.save((self._captured, self._gaps), f"{path}.rank{rank}")
|
||||
|
||||
def mono_layers(self) -> dict:
|
||||
from vllm.models.deepseek_v41.amd import mono_decode
|
||||
|
||||
layers = {
|
||||
i: None
|
||||
if m.mono is None
|
||||
else ["ffn" if m.mono.ffn_only else "whole", m.mono._weights is not None]
|
||||
for i, m in self._decoder_layers().items()
|
||||
}
|
||||
runner = mono_decode._runner
|
||||
return {
|
||||
"layers": layers,
|
||||
"epoch": -1 if runner is None else int(runner.epoch[0]),
|
||||
}
|
||||
|
||||
|
||||
def _decode(vllm_runner, monkeypatch, tp: int, mono: bool, prompts, path: str):
|
||||
"""Greedy decode of ``prompts`` with the mono path on or off: (each
|
||||
prompt's output, the workers' mono layers); each decoder layer's outputs
|
||||
for the decode tokens of decode-only steps, and vLLM's routing margins,
|
||||
saved under ``path``."""
|
||||
from vllm import SamplingParams
|
||||
from vllm.inputs import TokensPrompt
|
||||
|
||||
monkeypatch.setenv("VLLM_ROCM_MONO_DECODE", "1" if mono else "0")
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
|
||||
with vllm_runner(
|
||||
MODEL,
|
||||
tensor_parallel_size=tp,
|
||||
load_format=LOAD_FORMAT,
|
||||
hf_overrides=TRUNCATED,
|
||||
worker_extension_cls=f"{__name__}.MonoProbe",
|
||||
moe_backend="aiter",
|
||||
language_model_only=True,
|
||||
tokenizer_mode="deepseek_v41",
|
||||
attention_config={"indexer_kv_dtype": "mxfp4", "indexer_sparse_logits": True},
|
||||
block_size=128,
|
||||
enforce_eager=True,
|
||||
max_model_len=4096,
|
||||
max_num_seqs=len(prompts),
|
||||
# a prompt prefills in one step, so its decode tokens never share one
|
||||
max_num_batched_tokens=16384,
|
||||
enable_prefix_caching=False,
|
||||
gpu_memory_utilization=0.5,
|
||||
) as runner:
|
||||
runner.llm.collective_rpc("capture_steps", args=([len(p) for p in prompts],))
|
||||
params = SamplingParams(temperature=0.0, max_tokens=STEPS + 1)
|
||||
outs = runner.llm.generate(
|
||||
[TokensPrompt(prompt_token_ids=p) for p in prompts], params
|
||||
)
|
||||
runner.llm.collective_rpc("save_capture", args=(path,))
|
||||
probe = runner.llm.collective_rpc("mono_layers")
|
||||
return [o.outputs[0] for o in outs], probe
|
||||
|
||||
|
||||
def _rel(a: torch.Tensor, b: torch.Tensor) -> float:
|
||||
return ((a - b).norm() / b.norm()).item()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _on_cdna4(), reason="the mono kernels target CDNA4")
|
||||
@pytest.mark.parametrize("tp", [2, 4])
|
||||
def test_mono_decode_matches_the_model(vllm_runner, monkeypatch, tmp_path, tp):
|
||||
if torch.accelerator.device_count() < tp:
|
||||
pytest.skip(f"needs {tp} GPUs")
|
||||
pytest.importorskip("flydsl")
|
||||
g = torch.Generator().manual_seed(0)
|
||||
# contexts past both ratios' 512 compressed entries, so top-k selects
|
||||
lens = torch.randperm(1700, generator=g)[:8].add(300).tolist() # distinct
|
||||
prompts = [torch.randint(1000, 60000, (n,), generator=g).tolist() for n in lens]
|
||||
|
||||
on, probe_on = _decode(
|
||||
vllm_runner, monkeypatch, tp, True, prompts, f"{tmp_path}/on"
|
||||
)
|
||||
off, probe_off = _decode(
|
||||
vllm_runner, monkeypatch, tp, False, prompts, f"{tmp_path}/off"
|
||||
)
|
||||
|
||||
# both layer kinds took the mono path on every rank, at every decode step
|
||||
launches = len(WHOLE + FFN) # one launch (pair) a layer a step
|
||||
for rank in probe_on:
|
||||
assert rank["layers"] == {
|
||||
i: ["whole" if i in WHOLE else "ffn", True] for i in WHOLE + FFN
|
||||
}, rank
|
||||
assert rank["epoch"] >= launches * STEPS, rank
|
||||
assert rank["epoch"] % launches == 0, rank
|
||||
assert all(set(r["layers"].values()) == {None} for r in probe_off)
|
||||
# each decode token's input matches between the runs while the greedy
|
||||
# tokens before it do: (prompt length, position) pairs to compare
|
||||
same: set[tuple[int, int]] = set()
|
||||
for prompt, a, b in zip(prompts, on, off):
|
||||
for step in range(STEPS):
|
||||
if a.token_ids[: step + 1] != b.token_ids[: step + 1]:
|
||||
break
|
||||
same.add((len(prompt), len(prompt) + step))
|
||||
|
||||
# each layer's outputs for those tokens, mono against vLLM's path, token by
|
||||
# token. Layer 0 sees the same inputs on both paths at a request's first
|
||||
# decode token: there the bound is test_dsv41_mono_numerics' (about 3x the
|
||||
# kernels' 0.4%); its later tokens attend over KV each path wrote, 3%. Each
|
||||
# later layer takes the previous one's slightly different output, which
|
||||
# random weights amplify (2-4% by layer 4): 8%. A token whose top-6 routing
|
||||
# is a near-tie on vLLM's path (6th / 7th score within 5e-3: a 0.4% input
|
||||
# difference moves scores near 0.8 by about 3e-3) may pick another expert;
|
||||
# once it has, its residual stream differs and the later layers are exempt
|
||||
# for it. The residual stream: 3% (0.3% seen). A layer the kernels no
|
||||
# longer follow is off by far more, on every token.
|
||||
for rank in range(tp):
|
||||
got, _ = torch.load(f"{tmp_path}/on.rank{rank}")
|
||||
want, gaps = torch.load(f"{tmp_path}/off.rank{rank}")
|
||||
keys: list[tuple[int, int]] = sorted(
|
||||
same & {k[1:] for k in got} & {k[1:] for k in want}
|
||||
)
|
||||
assert len(keys) >= len(prompts), keys
|
||||
# the bound that sees a 3% error: a request's first decode token
|
||||
assert any(p == n for n, p in keys), keys
|
||||
flipped: set = set()
|
||||
for i in sorted(WHOLE + FFN):
|
||||
for k in keys:
|
||||
if k in flipped:
|
||||
continue
|
||||
(x, res), (x_ref, res_ref) = got[(i, *k)], want[(i, *k)]
|
||||
err = _rel(res, res_ref)
|
||||
assert err < 0.03, (rank, i, k, "residual", err)
|
||||
bound = (0.015 if k[1] == k[0] else 0.03) if i == 0 else 0.08
|
||||
err = _rel(x, x_ref)
|
||||
if err >= bound:
|
||||
assert gaps[(i, *k)] < 5e-3, (rank, i, k, "x", err)
|
||||
flipped.add(k)
|
||||
assert len(flipped) <= len(keys) // 4, (rank, sorted(flipped))
|
||||
@@ -0,0 +1,529 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Numerics of the DeepSeek-V4.1 mono decode kernels at TP2 on CDNA4 against the
|
||||
ops vLLM runs for the same decode step: random weights in the checkpoint's
|
||||
formats, the step's KV caches as vLLM's insert writes them.
|
||||
|
||||
- The whole layer (K1 + K2) against the mHC seams, the ROCm attention op chain,
|
||||
the TP all-reduce and ``DeepseekV4MoE``.
|
||||
- The FFN launch against the same FFN half, from wo_b's unreduced output.
|
||||
|
||||
Each stage is compared on its own input, so a failure names the stage; the MoE
|
||||
is checked on the kernels' own FFN input, where routing cannot differ."""
|
||||
|
||||
import pytest
|
||||
import ray
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ..utils import (
|
||||
ensure_model_parallel_initialized,
|
||||
init_test_distributed_environment,
|
||||
multi_process_parallel,
|
||||
)
|
||||
|
||||
MODEL = "deepseek-ai/DeepSeek-V4.1-Flash"
|
||||
HIDDEN, Q_RANK, HEAD_DIM, NOPE, ROPE = 5120, 1280, 512, 448, 64
|
||||
N_HEADS, O_GROUPS, O_RANK = 64, 8, 1024
|
||||
WINDOW, TOPK, SWA_BLOCK, CACHE_BLOCK = 128, 512, 32, 128
|
||||
RECORD, DATA, ALIGN = 584, 576, 576
|
||||
REPLAYS = 300
|
||||
HC = 4
|
||||
|
||||
|
||||
def _on_cdna4() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
|
||||
return get_cdna_version() == 4
|
||||
|
||||
|
||||
def _cos(a: torch.Tensor, b: torch.Tensor) -> float:
|
||||
a, b = a.float().reshape(-1), b.float().reshape(-1)
|
||||
return torch.nn.functional.cosine_similarity(a, b, dim=0).item()
|
||||
|
||||
|
||||
def _rel(a: torch.Tensor, b: torch.Tensor) -> float:
|
||||
"""Relative L2 error of ``a`` against ``b``: unlike the cosine, it sees a
|
||||
wrong scale."""
|
||||
a, b = a.float().reshape(-1), b.float().reshape(-1)
|
||||
return ((a - b).norm() / b.norm()).item()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- caches
|
||||
|
||||
|
||||
def _paged_cache(num_blocks: int, block: int, device) -> torch.Tensor:
|
||||
"""[blocks, block, 584] u8 over pages of round_up(block * 584, 576) bytes,
|
||||
the way vLLM binds a packed V4 cache."""
|
||||
page = -(-block * RECORD // ALIGN) * ALIGN
|
||||
raw = torch.zeros(num_blocks * page, dtype=torch.uint8, device=device)
|
||||
return torch.as_strided(raw, (num_blocks, block, RECORD), (page, RECORD, 1))
|
||||
|
||||
|
||||
def _rope(kv: torch.Tensor, positions: torch.Tensor, cos_sin: torch.Tensor):
|
||||
"""GPT-J RoPE of the last 64 dims (fp32 math, bf16 out)."""
|
||||
cs = cos_sin[positions.long()]
|
||||
cos, sin = cs[:, :32], cs[:, 32:]
|
||||
x = kv.float()
|
||||
e, o = x[:, NOPE::2].clone(), x[:, NOPE + 1 :: 2].clone()
|
||||
x[:, NOPE::2] = e * cos - o * sin
|
||||
x[:, NOPE + 1 :: 2] = e * sin + o * cos
|
||||
return x.to(torch.bfloat16)
|
||||
|
||||
|
||||
def _write_records(cache, block: int, slots, kv):
|
||||
"""fp8_ds_mla records of roped bf16 kv rows at ``slots``: the 448 nope dims
|
||||
in e4m3 with a UE8M0 per 64, the rope dims in bf16, as vLLM's insert."""
|
||||
nope = kv.float()[:, :NOPE].view(-1, 7, 64)
|
||||
exp = torch.ceil(torch.log2(nope.abs().amax(-1).clamp_min(1e-4) / 448.0))
|
||||
q = (nope * torch.exp2(-exp)[..., None]).clamp(-448, 448).reshape(-1, NOPE)
|
||||
data = torch.cat(
|
||||
[
|
||||
q.to(torch.float8_e4m3fn).view(torch.uint8),
|
||||
kv[:, NOPE:].contiguous().view(torch.uint8),
|
||||
],
|
||||
1,
|
||||
)
|
||||
scales = torch.zeros(kv.shape[0], 8, dtype=torch.uint8, device=kv.device)
|
||||
scales[:, :7] = (exp + 127).clamp(0, 255).to(torch.uint8)
|
||||
n = cache.untyped_storage().nbytes() - cache.storage_offset()
|
||||
flat = torch.as_strided(cache, (n,), (1,))
|
||||
slots = slots.to(kv.device).long()
|
||||
b, o = slots // block, slots % block
|
||||
d0 = b * cache.stride(0) + o * DATA
|
||||
s0 = b * cache.stride(0) + block * DATA + o * 8
|
||||
ar = torch.arange(DATA, device=kv.device)
|
||||
flat[(d0[:, None] + ar).reshape(-1)] = data.reshape(-1)
|
||||
ar = torch.arange(8, device=kv.device)
|
||||
flat[(s0[:, None] + ar).reshape(-1)] = scales.reshape(-1)
|
||||
|
||||
|
||||
def _normed_kv(n: int, kv_norm, g, device):
|
||||
x = torch.randn(n, HEAD_DIM, generator=g).to(device)
|
||||
x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True)) * kv_norm.float()
|
||||
return x.to(torch.bfloat16)
|
||||
|
||||
|
||||
def _decode_step(ratio: int, R: int, T: int, kv_norm, cos_sin, seed: int):
|
||||
"""R requests x T tokens past random contexts: the step's metadata as
|
||||
vLLM's builders lay it out, its window rows and (ratio > 0) the compressed
|
||||
cache filled, a random top-512 a token."""
|
||||
g = torch.Generator().manual_seed(seed)
|
||||
dev = kv_norm.device
|
||||
ctx = [int(c) for c in torch.randint(2000, 40000, (R,), generator=g)]
|
||||
M = R * T
|
||||
spans = [
|
||||
(max(0, c - WINDOW - T) // SWA_BLOCK, (c + T - 1) // SWA_BLOCK) for c in ctx
|
||||
]
|
||||
n_swa = 1 + sum(hi - lo + 1 for lo, hi in spans)
|
||||
perm = (torch.randperm(n_swa - 1, generator=g) + 1).tolist()
|
||||
bt = torch.zeros(R, max(hi for _, hi in spans) + 1, dtype=torch.int64)
|
||||
for r, (lo, hi) in enumerate(spans):
|
||||
for lb in range(lo, hi + 1):
|
||||
bt[r, lb] = perm.pop()
|
||||
swa = _paged_cache(n_swa, SWA_BLOCK, dev)
|
||||
|
||||
def slot(r, p):
|
||||
return int(bt[r, p // SWA_BLOCK]) * SWA_BLOCK + p % SWA_BLOCK
|
||||
|
||||
pos = [ctx[r] + t for r in range(R) for t in range(T)]
|
||||
t2r = [r for r in range(R) for _ in range(T)]
|
||||
swa_idx = torch.full((M, WINDOW), -1, dtype=torch.int32)
|
||||
swa_len = torch.zeros(M, dtype=torch.int32)
|
||||
for i in range(M):
|
||||
ps = range(max(pos[i] - WINDOW + 1, 0), pos[i] + 1)
|
||||
swa_idx[i, : len(ps)] = torch.tensor([slot(t2r[i], p) for p in ps])
|
||||
swa_len[i] = len(ps)
|
||||
hist = [
|
||||
(r, p) for r in range(R) for p in range(max(0, ctx[r] - WINDOW - T), ctx[r])
|
||||
]
|
||||
hp = torch.tensor([p for _, p in hist], device=dev)
|
||||
kv = _rope(_normed_kv(len(hist), kv_norm, g, dev), hp, cos_sin)
|
||||
_write_records(swa, SWA_BLOCK, torch.tensor([slot(r, p) for r, p in hist]), kv)
|
||||
|
||||
step = dict(
|
||||
M=M,
|
||||
positions=torch.tensor(pos, dtype=torch.int64, device=dev),
|
||||
slot_mapping=torch.tensor(
|
||||
[slot(t2r[i], pos[i]) for i in range(M)], dtype=torch.int64, device=dev
|
||||
),
|
||||
token_to_req=torch.tensor(t2r, dtype=torch.int32, device=dev),
|
||||
swa_indices=swa_idx.to(dev),
|
||||
swa_lens=swa_len.to(dev),
|
||||
swa_cache=swa,
|
||||
topk=None,
|
||||
comp_cache=None,
|
||||
comp_block_table=None,
|
||||
comp_entries=0,
|
||||
)
|
||||
if ratio:
|
||||
entries = CACHE_BLOCK // ratio
|
||||
lens = [(c + T) // ratio for c in ctx]
|
||||
nblk = [-(-n // entries) for n in lens]
|
||||
perm = (torch.randperm(sum(nblk), generator=g) + 1).tolist()
|
||||
cbt = torch.zeros(R, max(nblk), dtype=torch.int32)
|
||||
for r in range(R):
|
||||
for b in range(nblk[r]):
|
||||
cbt[r, b] = perm.pop()
|
||||
comp = _paged_cache(1 + sum(nblk), entries, dev)
|
||||
for r in range(R):
|
||||
idx = torch.arange(lens[r])
|
||||
slots = cbt[r, idx // entries].long() * entries + idx % entries
|
||||
kv = _rope(
|
||||
_normed_kv(lens[r], kv_norm, g, dev), (idx * ratio).to(dev), cos_sin
|
||||
)
|
||||
_write_records(comp, entries, slots, kv)
|
||||
topk = torch.full((M, TOPK), -1, dtype=torch.int32)
|
||||
for i in range(M):
|
||||
cand = (pos[i] + 1) // ratio
|
||||
n = min(cand, TOPK)
|
||||
topk[i, :n] = torch.randperm(cand, generator=g)[:n].to(torch.int32)
|
||||
step.update(
|
||||
topk=topk.to(dev),
|
||||
comp_cache=comp,
|
||||
comp_block_table=cbt.to(dev),
|
||||
comp_entries=entries,
|
||||
)
|
||||
return step
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- weights
|
||||
|
||||
|
||||
def _mxfp8(n: int, k: int, g, device):
|
||||
"""A random MXFP8 weight [n, k] e4m3 and its E8M0 32 x 32 block scales."""
|
||||
w = torch.randn(n, k, generator=g).clamp(-6, 6).to(torch.float8_e4m3fn)
|
||||
s = torch.randint(118, 123, (n // 32, k // 32), generator=g, dtype=torch.uint8)
|
||||
return w.to(device), s.to(device)
|
||||
|
||||
|
||||
def _random_moe(moe, gd, gr, inter: int) -> None:
|
||||
"""Random parameters in the loaded layout, before post-load processing: this
|
||||
rank's shards (``gd``, a device generator) of the MXFP4 routed experts
|
||||
(random nibbles, E8M0 scales; the intermediate past ``inter`` zero, as the
|
||||
loader pads it) and the MXFP8 shared expert (one E8M0 a 32 x 32 block); the
|
||||
replicated router from ``gr``, seeded alike on every rank."""
|
||||
u8 = torch.uint8
|
||||
for lin in (moe.shared_experts.gate_up_proj, moe.shared_experts.down_proj):
|
||||
n = lin.weight.shape[0]
|
||||
w = torch.randn(lin.weight.shape, generator=gd, device=gd.device)
|
||||
lin.weight.data.copy_(w.clamp(-6, 6).to(lin.weight.dtype))
|
||||
sc = lin.weight_scale.data.view(u8)
|
||||
rows = -(-n // 32) if sc.shape[0] == n else sc.shape[0]
|
||||
blocks = torch.randint(
|
||||
118, 123, (rows, sc.shape[1]), generator=gd, device=gd.device, dtype=u8
|
||||
)
|
||||
sc.copy_(blocks.repeat_interleave(32, 0)[:n] if sc.shape[0] == n else blocks)
|
||||
e = moe.experts.routed_experts
|
||||
for name, lo, hi in (("weight", 0, 256), ("weight_scale", 118, 122)):
|
||||
for w in ("w13", "w2"):
|
||||
t = getattr(e, f"{w}_{name}").data.view(u8)
|
||||
t.copy_(
|
||||
torch.randint(lo, hi, t.shape, generator=gd, device=gd.device, dtype=u8)
|
||||
)
|
||||
padded = e.w13_weight.shape[1] // 2
|
||||
if padded > inter:
|
||||
w13, w2 = e.w13_weight.data.view(u8), e.w2_weight.data.view(u8)
|
||||
for half in (0, padded):
|
||||
w13[:, half + inter : half + padded] = 0
|
||||
w2[..., inter // 2 :] = 0
|
||||
for t in (moe.gate.weight, moe.gate.e_score_correction_bias):
|
||||
t.data.copy_(torch.randn(t.shape, generator=gr, device=gr.device) * 0.02)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the worker
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def _mono_numerics(monkeypatch, tp_size, pp_size, rank, distributed_init_port):
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
with monkeypatch.context() as m:
|
||||
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
# the checkpoint's MXFP4 experts run on AITER's MoE; its flags are read
|
||||
# at import, so refresh them
|
||||
m.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
m.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
|
||||
ensure_model_parallel_initialized(tp_size, pp_size)
|
||||
init_workspace_manager(device)
|
||||
_check_layers(rank, tp_size, device)
|
||||
|
||||
|
||||
def _check_layers(rank: int, tp: int, device) -> None:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.distributed import get_tp_group, tensor_model_parallel_all_reduce
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.kernels.linear.mxfp8.rocm_block32_gemm import (
|
||||
rocm_mxfp8_block32_gemm,
|
||||
)
|
||||
from vllm.model_executor.layers.mhc import MHCPreDelayedOp
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
mxfp8_e4m3_quantize,
|
||||
)
|
||||
from vllm.model_executor.model_loader.utils import process_weights_after_loading
|
||||
from vllm.models.common.ops import fused_q_kv_rmsnorm
|
||||
from vllm.models.deepseek_v41.amd.model import DeepseekV4MoE
|
||||
from vllm.models.deepseek_v41.amd.mono.layer import scratch_layout
|
||||
from vllm.models.deepseek_v41.amd.mono.runner import (
|
||||
AttnWeights,
|
||||
DSV41MonoLayer,
|
||||
MonoLayerWeights,
|
||||
)
|
||||
from vllm.models.deepseek_v41.amd.rocm import (
|
||||
compute_global_topk_ragged_indices_and_indptr,
|
||||
)
|
||||
from vllm.models.deepseek_v41.common.rope import build_deepseek_v4_rope
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
build_ragged_indices_from_dense,
|
||||
rocm_inverse_rope_mxfp8_rows,
|
||||
)
|
||||
|
||||
vc = EngineArgs(
|
||||
model=MODEL,
|
||||
tensor_parallel_size=tp,
|
||||
moe_backend="aiter",
|
||||
load_format="dummy",
|
||||
max_model_len=65536,
|
||||
language_model_only=True,
|
||||
attention_config={"indexer_kv_dtype": "mxfp4", "indexer_sparse_logits": True},
|
||||
).create_engine_config()
|
||||
cfg = vc.model_config.hf_config
|
||||
cfg = getattr(cfg, "text_config", cfg)
|
||||
# replicated tensors alike on every rank; each rank's own TP shards
|
||||
g = torch.Generator().manual_seed(1000)
|
||||
gs = torch.Generator().manual_seed(1001 + rank)
|
||||
H, G = N_HEADS // tp, O_GROUPS // tp
|
||||
bf = torch.bfloat16
|
||||
|
||||
with set_current_vllm_config(vc), set_default_torch_dtype(bf), device:
|
||||
moe = DeepseekV4MoE(vc, prefix="model.layers.21.ffn")
|
||||
mhc = MHCPreDelayedOp()
|
||||
_random_moe(
|
||||
moe,
|
||||
torch.Generator(device=device).manual_seed(2001 + rank),
|
||||
torch.Generator(device=device).manual_seed(2000),
|
||||
inter=2304 // tp,
|
||||
)
|
||||
process_weights_after_loading(moe, vc.model_config, device)
|
||||
|
||||
def f32(*shape, s=1.0, gen=g):
|
||||
return (torch.randn(*shape, generator=gen) * s).to(device)
|
||||
|
||||
mix = (2 + HC) * HC
|
||||
seams = {
|
||||
sub: dict(
|
||||
fn=f32(mix, HC * HIDDEN, s=0.01),
|
||||
scale=f32(3, s=0.5).abs() + 0.5,
|
||||
base=f32(mix, s=0.1),
|
||||
norm=(1 + f32(HIDDEN, s=0.1)).to(bf),
|
||||
)
|
||||
for sub in ("attn", "ffn")
|
||||
}
|
||||
|
||||
def seam(sub, residual, x, post, comb, pre):
|
||||
s = seams[sub]
|
||||
return mhc(
|
||||
residual, s["fn"], s["scale"], s["base"], cfg.rms_norm_eps,
|
||||
cfg.hc_eps, cfg.hc_eps, 2.0, cfg.hc_sinkhorn_iters,
|
||||
pre_mix=pre, sublayer_out=x, post_layer_mix=post, comb_res_mix=comb,
|
||||
norm_weight=s["norm"], norm_eps=cfg.rms_norm_eps,
|
||||
) # fmt: skip
|
||||
|
||||
def moe_out(x):
|
||||
with set_forward_context(None, vc):
|
||||
ids = torch.zeros(x.shape[0], dtype=torch.int64, device=device)
|
||||
return moe(x.clone(), ids)
|
||||
|
||||
wqkv, wqkv_s = _mxfp8(1792, HIDDEN, g, device)
|
||||
wq_b, wq_b_s = _mxfp8(H * HEAD_DIM, Q_RANK, gs, device)
|
||||
wo_a, wo_a_s = _mxfp8(G * O_RANK, H // G * HEAD_DIM, gs, device)
|
||||
wo_b, wo_b_s = _mxfp8(HIDDEN, G * O_RANK, gs, device)
|
||||
q_norm, kv_norm = (1 + f32(Q_RANK, s=0.1)).to(bf), (1 + f32(HEAD_DIM, s=0.1)).to(bf)
|
||||
sink = f32(H, gen=gs)
|
||||
e, sh = moe.experts.routed_experts, moe.shared_experts
|
||||
u8 = lambda t: t.view(torch.uint8) # noqa: E731
|
||||
runner = DSV41MonoLayer(tp, rank, get_tp_group().cpu_group, device)
|
||||
gw, bias = moe.gate.weight.float(), moe.gate.e_score_correction_bias.float()
|
||||
|
||||
def top6(x):
|
||||
"""The top-6 experts as vLLM routes: sqrt(softplus(logits)) + bias."""
|
||||
sc = torch.nn.functional.softplus(x.float() @ gw.T).sqrt() + bias
|
||||
return sc.topk(6, dim=1).indices.sort(1).values
|
||||
|
||||
steps = [] # each case's launches, on fixed buffers, for the replay stress
|
||||
for ratio, R, T in ((1, 1, 6), (2, 8, 6)):
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
rope = build_deepseek_v4_rope(
|
||||
cfg, head_dim=HEAD_DIM, rope_head_dim=ROPE,
|
||||
max_position_embeddings=cfg.max_position_embeddings,
|
||||
compress_ratio=ratio,
|
||||
) # fmt: skip
|
||||
cos_sin = rope.cos_sin_cache.to(device=device, dtype=torch.float32)
|
||||
s = _decode_step(ratio, R, T, kv_norm, cos_sin, seed=7 + ratio)
|
||||
M = s["M"]
|
||||
mw = MonoLayerWeights(
|
||||
attn=AttnWeights(
|
||||
layer_id=21, wqkv=wqkv, wqkv_scale=wqkv_s, q_norm=q_norm,
|
||||
kv_norm=kv_norm, wq_b=wq_b, wq_b_scale=wq_b_s, wo_a=wo_a,
|
||||
wo_a_scale=wo_a_s, wo_b=wo_b, wo_b_scale=wo_b_s, attn_sink=sink,
|
||||
cos_sin=cos_sin, ratio=ratio,
|
||||
),
|
||||
hc_attn_fn=seams["attn"]["fn"], hc_attn_scale=seams["attn"]["scale"],
|
||||
hc_attn_base=seams["attn"]["base"], attn_norm=seams["attn"]["norm"],
|
||||
hc_ffn_fn=seams["ffn"]["fn"], hc_ffn_scale=seams["ffn"]["scale"],
|
||||
hc_ffn_base=seams["ffn"]["base"], ffn_norm=seams["ffn"]["norm"],
|
||||
gate_w=moe.gate.weight, bias=moe.gate.e_score_correction_bias,
|
||||
w13=e.w13_weight, w13_s=e.w13_weight_scale, w2=e.w2_weight,
|
||||
w2_s=e.w2_weight_scale, sgu=sh.gate_up_proj.weight,
|
||||
sgu_s=u8(sh.gate_up_proj.weight_scale), sw2=sh.down_proj.weight,
|
||||
sw2_s=u8(sh.down_proj.weight_scale),
|
||||
) # fmt: skip
|
||||
|
||||
# the layer's inputs at the attention seam (the same on every rank)
|
||||
gi = torch.Generator().manual_seed(11)
|
||||
res0 = torch.randn(M, HC, HIDDEN, generator=gi)
|
||||
res0 = (
|
||||
(res0 * torch.rsqrt(res0.pow(2).mean(-1, keepdim=True))).to(bf).to(device)
|
||||
)
|
||||
x0 = (torch.randn(M, HIDDEN, generator=gi) * 0.5).to(bf).to(device)
|
||||
comb0 = torch.rand(M, HC, HC, generator=gi) + 0.1
|
||||
for _ in range(20):
|
||||
comb0 = comb0 / comb0.sum(-1, keepdim=True)
|
||||
comb0 = comb0 / comb0.sum(-2, keepdim=True)
|
||||
comb0 = comb0.to(device)
|
||||
post0 = (torch.sigmoid(torch.randn(M, HC, 1, generator=gi)) * 2).to(device)
|
||||
pre0 = (torch.sigmoid(torch.randn(M, HC, generator=gi)) + 1e-6).to(device)
|
||||
|
||||
# vLLM's decode chain
|
||||
res, post, comb, xa, attn_pre = seam("attn", res0, x0, post0, comb0, pre0)
|
||||
swa_ind, swa_ptr = build_ragged_indices_from_dense(
|
||||
s["swa_indices"].reshape(M, -1), s["swa_lens"]
|
||||
)
|
||||
top_ind = top_ptr = None
|
||||
if ratio:
|
||||
valid = torch.ones(M, dtype=torch.bool, device=device)
|
||||
top_ind, top_ptr, _ = compute_global_topk_ragged_indices_and_indptr(
|
||||
s["topk"], s["token_to_req"], s["comp_block_table"],
|
||||
s["comp_entries"], valid,
|
||||
) # fmt: skip
|
||||
x8, xs = mxfp8_e4m3_quantize(xa)
|
||||
qr_kv = rocm_mxfp8_block32_gemm(x8, xs, wqkv, wqkv_s, bf)
|
||||
qr, kv = fused_q_kv_rmsnorm(
|
||||
qr_kv[:, :Q_RANK], qr_kv[:, Q_RANK:], q_norm, kv_norm, cfg.rms_norm_eps
|
||||
)
|
||||
q8, qs = mxfp8_e4m3_quantize(qr)
|
||||
q = rocm_mxfp8_block32_gemm(q8, qs, wq_b, wq_b_s, bf).view(M, H, HEAD_DIM)
|
||||
swa2d = s["swa_cache"].view(s["swa_cache"].shape[0], -1)
|
||||
q_roped = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
q, kv, swa2d, s["slot_mapping"], s["positions"], cos_sin, H,
|
||||
cfg.rms_norm_eps, SWA_BLOCK, False, False, True, False,
|
||||
) # fmt: skip
|
||||
attn = torch.empty(M, H, HEAD_DIM, dtype=bf, device=device)
|
||||
rocm_aiter_ops.triton_sparse_mla_fwd(
|
||||
q_roped, s["swa_cache"], attn, HEAD_DIM**-0.5, swa_ptr, swa_ind,
|
||||
attn_sink=sink, extra_kv_buffer=s["comp_cache"],
|
||||
extra_kv_indptr=top_ptr, extra_kv_indices=top_ind,
|
||||
) # fmt: skip
|
||||
a8 = torch.empty(M, H * HEAD_DIM, dtype=torch.float8_e4m3fn, device=device)
|
||||
a_s = torch.empty(M, H * HEAD_DIM // 32, dtype=torch.uint8, device=device)
|
||||
rocm_inverse_rope_mxfp8_rows(attn, s["positions"], cos_sin, ROPE, a8, a_s)
|
||||
z = torch.ops.vllm.rocm_dsv41_mxfp8_wo_a_bmm(a8, a_s, wo_a, wo_a_s, G, O_RANK)
|
||||
z8, zs = mxfp8_e4m3_quantize(z)
|
||||
part = rocm_mxfp8_block32_gemm(z8, zs, wo_b, wo_b_s, bf).contiguous()
|
||||
a = tensor_model_parallel_all_reduce(part.clone())
|
||||
res2, post2, comb2, xf, ffn_pre = seam("ffn", res, a, post, comb, attn_pre)
|
||||
out = moe_out(xf)
|
||||
|
||||
# ---- the whole layer: K1 + K2
|
||||
o = runner.forward(
|
||||
mw, x0, res0, post0, comb0, pre0, s["positions"], s["slot_mapping"],
|
||||
s["swa_cache"], s["swa_indices"], s["swa_lens"], s["token_to_req"],
|
||||
topk_indices=s["topk"], comp_cache=s["comp_cache"],
|
||||
comp_block_table=s["comp_block_table"],
|
||||
) # fmt: skip
|
||||
off = scratch_layout(M, tp)["normed"][0]
|
||||
normed = runner.scratch(M)[off : off + M * HIDDEN * 2].view(bf).view(M, HIDDEN)
|
||||
tag = f"ratio {ratio}, {R} x {T}"
|
||||
assert _cos(runner.res_mid[:M], res) > 0.99999, tag
|
||||
for got, want in (
|
||||
(runner.post_a, post),
|
||||
(runner.comb_a, comb),
|
||||
(runner.pre_a, attn_pre),
|
||||
):
|
||||
torch.testing.assert_close(
|
||||
got[:M].view_as(want), want, rtol=1e-4, atol=1e-5
|
||||
)
|
||||
same_in = moe_out(normed)
|
||||
# relative errors about 3x what the kernels show (0.3-0.5%)
|
||||
assert _cos(o[1], res2) > 0.9999 and _rel(o[1], res2) < 0.01, tag
|
||||
assert _cos(normed, xf) > 0.999 and _rel(normed, xf) < 0.012, tag
|
||||
assert _cos(o[0], same_in) > 0.9999 and _rel(o[0], same_in) < 0.015, tag
|
||||
# per token, where both FFN inputs pick the same experts
|
||||
same = (top6(xf) == top6(normed)).all(1)
|
||||
assert same.float().mean() >= 0.5, tag
|
||||
tok = torch.nn.functional.cosine_similarity(o[0].float(), out.float(), dim=1)
|
||||
assert tok[same].min() > 0.99, tag
|
||||
|
||||
# ---- the FFN launch, from the same unreduced wo_b output
|
||||
f = runner.ffn(mw, part, res, post, comb, attn_pre)
|
||||
torch.testing.assert_close(f[1], res2, rtol=2**-8, atol=1e-6)
|
||||
torch.testing.assert_close(normed, xf, rtol=2**-7, atol=1e-6)
|
||||
assert _cos(f[0], out) > 0.9999 and _rel(f[0], out) < 0.015, tag
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
whole, ffn = runner._outs(M, res0), runner._outs(M, res)
|
||||
args = (s["positions"], s["slot_mapping"], s["swa_cache"], s["swa_indices"])
|
||||
meta = dict(
|
||||
topk_indices=s["topk"], comp_cache=s["comp_cache"],
|
||||
comp_block_table=s["comp_block_table"],
|
||||
) # fmt: skip
|
||||
|
||||
def step(mw=mw, x0=x0, res0=res0, post0=post0, comb0=comb0, pre0=pre0,
|
||||
args=args, s=s, meta=meta, whole=whole, part=part, res=res,
|
||||
post=post, comb=comb, attn_pre=attn_pre, ffn=ffn): # fmt: skip
|
||||
runner.forward(
|
||||
mw, x0, res0, post0, comb0, pre0, *args, s["swa_lens"],
|
||||
s["token_to_req"], **meta, outs=whole,
|
||||
) # fmt: skip
|
||||
runner.ffn(mw, part, res, post, comb, attn_pre, outs=ffn)
|
||||
return [*whole, *ffn]
|
||||
|
||||
steps.append(step)
|
||||
|
||||
# ---- the hand-off protocol under back-to-back launches: both step widths
|
||||
# alternating in one graph, every replay bit for bit the first's
|
||||
outs = [t for step in steps for t in step()]
|
||||
torch.accelerator.synchronize()
|
||||
ref = [t.clone() for t in outs]
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
for step in steps:
|
||||
step()
|
||||
for i in range(REPLAYS):
|
||||
graph.replay()
|
||||
if i % 50 == 49 or i == REPLAYS - 1:
|
||||
torch.accelerator.synchronize()
|
||||
bad = [
|
||||
j for j, (t, r) in enumerate(zip(outs, ref)) if not torch.equal(t, r)
|
||||
]
|
||||
assert not bad, f"replay {i}: outputs {bad} differ from the first run"
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _on_cdna4(), reason="the mono kernels target CDNA4")
|
||||
@pytest.mark.parametrize("tp", [2, 4])
|
||||
def test_mono_decode_layer_matches_vllm(monkeypatch: pytest.MonkeyPatch, tp: int):
|
||||
if torch.accelerator.device_count() < tp:
|
||||
pytest.skip(f"needs {tp} GPUs")
|
||||
pytest.importorskip("flydsl")
|
||||
multi_process_parallel(monkeypatch, tp, 1, _mono_numerics)
|
||||
Reference in New Issue
Block a user