[ROCm][Perf][DSv4.1] Mono decode on every backbone layer (#61082)

Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Fangzhou Ai
2026-10-13 10:16:50 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent f47792af7f
commit cc2a6eea1a
13 changed files with 379 additions and 111 deletions
+69 -14
View File
@@ -27,21 +27,25 @@ class _Runner:
self.calls.append((args, kwargs)) self.calls.append((args, kwargs))
return ("mono outputs",) return ("mono outputs",)
def first_pre_mix(self, rows):
return torch.zeros(rows, 4)
def ffn(self, *args): def ffn(self, *args):
self.calls.append((args, {})) self.calls.append((args, {}))
return ("ffn outputs",) return ("ffn outputs",)
def _layer(comp="comp"): def _layer(comp="comp", indexer=None):
cache = lambda: torch.zeros(4, 128 * md.RECORD, dtype=torch.uint8) # noqa: E731 cache = lambda: torch.zeros(4, 128 * md.RECORD, dtype=torch.uint8) # noqa: E731
attn = SimpleNamespace( attn = SimpleNamespace(
swa_cache_layer=SimpleNamespace(prefix="swa", kv_cache=cache()), swa_cache_layer=SimpleNamespace(prefix="swa", kv_cache=cache()),
compressed_cache_prefix=comp, compressed_cache_prefix=comp,
topk_indices_buffer=torch.zeros(64, 512, dtype=torch.int32), topk_indices_buffer=torch.zeros(64, 512, dtype=torch.int32),
compress_ratio=1, compress_ratio=1,
indexer=indexer,
_compressed_kv_cache=cache, _compressed_kv_cache=cache,
) )
return SimpleNamespace(attn=attn) return SimpleNamespace(attn=attn, engram=None, hc_mult=4)
def _swa(**overrides): def _swa(**overrides):
@@ -77,7 +81,12 @@ def run(monkeypatch):
``mode`` and ``metadata``; returns (result, runner calls).""" ``mode`` and ``metadata``; returns (result, runner calls)."""
def _run( def _run(
mode=CUDAGraphMode.FULL, metadata=None, ffn_only=False, entry=None, **inputs mode=CUDAGraphMode.FULL,
metadata=None,
ffn_only=False,
entry=None,
layer=None,
**inputs,
): ):
if metadata is None: if metadata is None:
metadata = { metadata = {
@@ -97,7 +106,7 @@ def run(monkeypatch):
# has no compressed cache # has no compressed cache
part, _ = args.pop("x"), args.pop("positions") part, _ = args.pop("x"), args.pop("positions")
return mono.ffn(_layer(comp=None), part, **args), runner.calls return mono.ffn(_layer(comp=None), part, **args), runner.calls
return mono(_layer(), **args), runner.calls return mono(layer or _layer(), **args), runner.calls
return _run return _run
@@ -114,6 +123,23 @@ def test_decode_step_runs_mono(run, mode):
1, 1,
) )
assert kwargs["comp_cache"].shape == (4, 128, md.RECORD) assert kwargs["comp_cache"].shape == (4, 128, md.RECORD)
assert kwargs["fold"] and "select" not in kwargs
def test_first_layer_runs_mono_from_the_embedding(run):
"""The first layer's residual is the embedding broadcast to the four
streams, which K1 takes as it is (no owed post to fold)."""
out, calls = run(residual=None, post_mix=None, res_mix=None, pre_mix=None)
assert out == ("mono outputs",) and len(calls) == 1
args, kwargs = calls[0]
assert args[2].shape == (M, 4, 5120) and not kwargs["fold"]
def test_index_source_selects_between_the_kernels(run):
"""An index source hands the runner its compressor and indexer to run
between K1 and K2."""
out, calls = run(layer=_layer(indexer=object()))
assert out == ("mono outputs",) and callable(calls[0][1]["select"])
def test_padded_decode_step_runs_mono(run): def test_padded_decode_step_runs_mono(run):
@@ -149,7 +175,9 @@ def test_padded_decode_step_runs_mono(run):
), ),
dict(metadata={}), # a profile / dummy run: no attention metadata dict(metadata={}), # a profile / dummy run: no attention metadata
dict(**_inputs(md.MAX_ROWS + 6)), dict(**_inputs(md.MAX_ROWS + 6)),
dict(residual=None), # the first layer's seam broadcasts the embedding # a later pipeline stage's first layer: its stream arrives without
# the mixes
dict(residual=None, x=torch.zeros(M, 4, 5120, dtype=torch.bfloat16)),
], ],
ids=[ ids=[
"piecewise", "piecewise",
@@ -158,7 +186,7 @@ def test_padded_decode_step_runs_mono(run):
"tokens-past-rows", "tokens-past-rows",
"no-metadata", "no-metadata",
"rows", "rows",
"first-layer", "stage-input",
], ],
) )
def test_other_steps_take_the_original_path(run, case): def test_other_steps_take_the_original_path(run, case):
@@ -202,22 +230,41 @@ def _decoder_layer(engram=None, **attn):
) )
def _indexer():
from vllm.models.deepseek_v41.amd.rocm import DeepseekV41RocmMxfp4Indexer
return object.__new__(DeepseekV41RocmMxfp4Indexer)
@pytest.mark.parametrize( @pytest.mark.parametrize(
"layer, path", "layer, path",
[ [
(_decoder_layer(), "whole"), (_decoder_layer(), "whole"),
(_decoder_layer(compress_ratio=0), "ffn"), (_decoder_layer(compress_ratio=0), "whole"),
(_decoder_layer(engram=object()), "ffn"), (_decoder_layer(engram=object()), "whole"),
(_decoder_layer(compressor=object(), indexer=object()), "ffn"), (_decoder_layer(compressor=object(), indexer=_indexer()), "whole"),
(_decoder_layer(indexer=_indexer()), "whole"),
(_decoder_layer(compressor=object()), "ffn"),
(_decoder_layer(indexer=object()), "ffn"), (_decoder_layer(indexer=object()), "ffn"),
(_decoder_layer(kv_cache_dtype="auto"), "ffn"), (_decoder_layer(kv_cache_dtype="auto"), "ffn"),
(_decoder_layer(layer_id=40), None), (_decoder_layer(layer_id=40), None),
], ],
ids=["standard", "first", "engram", "kv-source", "index-source", "kv", "draft"], ids=[
"standard",
"window",
"engram",
"kv-source",
"index-source",
"compressor-only",
"other-indexer",
"kv",
"draft",
],
) )
def test_layers_take_whole_or_ffn_launch(monkeypatch, layer, path): def test_layers_take_whole_or_ffn_launch(monkeypatch, layer, path):
"""The whole-layer kernels serve the standard layers; every other backbone """The whole-layer kernels serve every backbone layer of the checkpoint (an
layer keeps vLLM's attention and runs its FFN half as one launch.""" index source's compressor and indexer run between them); a layer outside
them keeps vLLM's attention and runs its FFN half as one launch."""
_deployment(monkeypatch) _deployment(monkeypatch)
mono = md.MonoDecodeLayer.create(layer, _config()) mono = md.MonoDecodeLayer.create(layer, _config())
got = None if mono is None else "ffn" if mono.ffn_only else "whole" got = None if mono is None else "ffn" if mono.ffn_only else "whole"
@@ -254,7 +301,7 @@ def test_other_steps_reduce_wo_b_themselves(run, case):
assert out is None and not calls assert out is None and not calls
def _config(**parallel): def _config(monokernel=True, **parallel):
parallel = ( parallel = (
dict(enable_expert_parallel=False, enable_eplb=False, data_parallel_size=1) dict(enable_expert_parallel=False, enable_eplb=False, data_parallel_size=1)
| parallel | parallel
@@ -262,6 +309,7 @@ def _config(**parallel):
return SimpleNamespace( return SimpleNamespace(
model_config=SimpleNamespace(hf_config=SimpleNamespace(num_hidden_layers=40)), model_config=SimpleNamespace(hf_config=SimpleNamespace(num_hidden_layers=40)),
parallel_config=SimpleNamespace(**parallel), parallel_config=SimpleNamespace(**parallel),
kernel_config=SimpleNamespace(monokernel=monokernel),
) )
@@ -269,7 +317,6 @@ def _deployment(monkeypatch, cdna=4, cus=256):
"""A deployment the kernels take (the runner module stubbed: no FlyDSL).""" """A deployment the kernels take (the runner module stubbed: no FlyDSL)."""
import vllm.platforms.rocm as rocm import vllm.platforms.rocm as rocm
monkeypatch.setenv("VLLM_ROCM_MONO_DECODE", "1")
monkeypatch.delenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4", raising=False) monkeypatch.delenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4", raising=False)
monkeypatch.setattr(rocm, "get_cdna_version", lambda: cdna) monkeypatch.setattr(rocm, "get_cdna_version", lambda: cdna)
monkeypatch.setattr(md, "get_tensor_model_parallel_world_size", lambda: 2) monkeypatch.setattr(md, "get_tensor_model_parallel_world_size", lambda: 2)
@@ -280,6 +327,14 @@ def _deployment(monkeypatch, cdna=4, cus=256):
monkeypatch.setitem(sys.modules, name, runner) monkeypatch.setitem(sys.modules, name, runner)
def test_off_without_the_flag(monkeypatch):
"""No --monokernel: no mono layer, whatever the deployment."""
_deployment(monkeypatch)
assert (
md.MonoDecodeLayer.create(_decoder_layer(), _config(monokernel=False)) is None
)
@pytest.mark.parametrize("cdna", [3, 5]) @pytest.mark.parametrize("cdna", [3, 5])
def test_opt_in_outside_the_kernels_cdna_raises(monkeypatch, cdna): def test_opt_in_outside_the_kernels_cdna_raises(monkeypatch, cdna):
_deployment(monkeypatch, cdna=cdna) _deployment(monkeypatch, cdna=cdna)
+30 -25
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The DeepSeek-V4.1 mono decode path through the real model: a five-layer """The DeepSeek-V4.1 mono decode path through the real model: a five-layer
DeepSeek-V4.1-Flash at TP2 and TP4 on CDNA4, served with ``VLLM_ROCM_MONO_DECODE`` on DeepSeek-V4.1-Flash at TP2 and TP4 on CDNA4, served with ``--monokernel`` on
and off on the same random weights: every decoder layer's outputs at every and off on the same random weights: every decoder layer's outputs at every
decode step must match. Unlike ``test_dsv41_mono_numerics.py``, which decode step must match. Unlike ``test_dsv41_mono_numerics.py``, which
checks the kernels against vLLM's ops one stage at a time, this runs checks the kernels against vLLM's ops one stage at a time, this runs
@@ -10,12 +10,13 @@ the per-step path choice, the weights taken off the layer, the metadata and
the indexer's top-k -- so a change to the decoder layer that the kernels do the indexer's top-k -- so a change to the decoder layer that the kernels do
not follow fails here. not follow fails here.
The five layers hold both kinds the mono path takes: Every layer takes the whole-layer path (K1 + K2); the five hold each variant:
- 0: sliding window only -> the FFN launch - 0: the first layer, sliding window only (K1 takes the embedding settled)
- 1: ratio-2 KV and index source -> the FFN launch - 1: ratio-2 KV and index source (vLLM's compressor and indexer between K1
- 2: ratio-2 consumer -> the whole layer (K1 + K2) and K2)
- 3: ratio-1 KV, index and candidate source -> the FFN launch - 2: ratio-2 consumer with an Engram (K1 takes the residual settled)
- 4: ratio-1 consumer -> the whole layer - 3: ratio-1 KV, index and candidate source
- 4: ratio-1 index source over 3's candidates and KV
""" """
import zlib import zlib
@@ -34,13 +35,17 @@ TRUNCATED = {
# one entry a layer, the three MTP layers' included # one entry a layer, the three MTP layers' included
"compress_ratios": [0, 2, 2, 1, 1, 0, 0, 0], "compress_ratios": [0, 2, 2, 1, 1, 0, 0, 0],
"kv_source_layer_ids": [1, 3], "kv_source_layer_ids": [1, 3],
"index_source_layer_ids": [1, 3], # 4: an indexer of its own over 3's candidates (a REINDEX layer)
"index_source_layer_ids": [1, 3, 4],
"candidate_source_layer_id": 3, "candidate_source_layer_id": 3,
"engram_layer_ids": [], # an Engram on a whole layer (K1 then takes its residual settled), its
"engram_num_embeddings": [], # tables shrunk: 24 prime-sized ranges from 1000
"engram_layer_ids": [2],
"engram_num_embeddings": [32768],
"engram_vocab_size": 1000,
"dspark_target_layer_ids": [2, 3, 4], "dspark_target_layer_ids": [2, 3, 4],
} }
WHOLE, FFN = (2, 4), (0, 1, 3) WHOLE, FFN = (0, 1, 2, 3, 4), ()
STEPS = 5 # decode steps after the prefill's token STEPS = 5 # decode steps after the prefill's token
@@ -227,7 +232,6 @@ def _decode(vllm_runner, monkeypatch, tp: int, mono: bool, prompts, path: str):
from vllm import SamplingParams from vllm import SamplingParams
from vllm.inputs import TokensPrompt from vllm.inputs import TokensPrompt
monkeypatch.setenv("VLLM_ROCM_MONO_DECODE", "1" if mono else "0")
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1") monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "1") monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
with vllm_runner( with vllm_runner(
@@ -237,6 +241,7 @@ def _decode(vllm_runner, monkeypatch, tp: int, mono: bool, prompts, path: str):
hf_overrides=TRUNCATED, hf_overrides=TRUNCATED,
worker_extension_cls=f"{__name__}.MonoProbe", worker_extension_cls=f"{__name__}.MonoProbe",
moe_backend="aiter", moe_backend="aiter",
monokernel=mono,
language_model_only=True, language_model_only=True,
tokenizer_mode="deepseek_v41", tokenizer_mode="deepseek_v41",
attention_config={"indexer_kv_dtype": "mxfp4", "indexer_sparse_logits": True}, attention_config={"indexer_kv_dtype": "mxfp4", "indexer_sparse_logits": True},
@@ -300,15 +305,16 @@ def test_mono_decode_matches_the_model(vllm_runner, monkeypatch, tmp_path, tp):
same.add((len(prompt), len(prompt) + step)) same.add((len(prompt), len(prompt) + step))
# each layer's outputs for those tokens, mono against vLLM's path, token by # each layer's outputs for those tokens, mono against vLLM's path, token by
# token. Layer 0 sees the same inputs on both paths at a request's first # token. Layer 0 sees the same inputs on both paths, but its FFN input
# decode token: there the bound is test_dsv41_mono_numerics' (about 3x the # differs by up to test_dsv41_mono_numerics' 1.2%, which random weights
# kernels' 0.4%); its later tokens attend over KV each path wrote, 3%. Each # amplify through the MoE: 3%. Each later layer takes the previous one's
# later layer takes the previous one's slightly different output, which # slightly different output, which random weights amplify (2-4% by layer
# random weights amplify (2-4% by layer 4): 8%. A token whose top-6 routing # 4): 8%. A token whose top-6 routing is a near-tie on vLLM's path may pick
# is a near-tie on vLLM's path (6th / 7th score within 5e-3: a 0.4% input # another expert: a 0.4% input difference moves scores near 0.8 by about
# difference moves scores near 0.8 by about 3e-3) may pick another expert; # 3e-3, so 6th / 7th scores within 5e-3 at layer 0 and, past it, where the
# once it has, its residual stream differs and the later layers are exempt # input carries the earlier layers' differences (0.4-0.7%), within 1e-2.
# for it. The residual stream: 3% (0.3% seen). A layer the kernels no # Once it has, its residual stream differs and the later layers are exempt
# for it. The residual stream: 3% (0.5% seen). A layer the kernels no
# longer follow is off by far more, on every token. # longer follow is off by far more, on every token.
for rank in range(tp): for rank in range(tp):
got, _ = torch.load(f"{tmp_path}/on.rank{rank}") got, _ = torch.load(f"{tmp_path}/on.rank{rank}")
@@ -317,8 +323,6 @@ def test_mono_decode_matches_the_model(vllm_runner, monkeypatch, tmp_path, tp):
same & {k[1:] for k in got} & {k[1:] for k in want} same & {k[1:] for k in got} & {k[1:] for k in want}
) )
assert len(keys) >= len(prompts), keys assert len(keys) >= len(prompts), keys
# the bound that sees a 3% error: a request's first decode token
assert any(p == n for n, p in keys), keys
flipped: set = set() flipped: set = set()
for i in sorted(WHOLE + FFN): for i in sorted(WHOLE + FFN):
for k in keys: for k in keys:
@@ -327,9 +331,10 @@ def test_mono_decode_matches_the_model(vllm_runner, monkeypatch, tmp_path, tp):
(x, res), (x_ref, res_ref) = got[(i, *k)], want[(i, *k)] (x, res), (x_ref, res_ref) = got[(i, *k)], want[(i, *k)]
err = _rel(res, res_ref) err = _rel(res, res_ref)
assert err < 0.03, (rank, i, k, "residual", err) assert err < 0.03, (rank, i, k, "residual", err)
bound = (0.015 if k[1] == k[0] else 0.03) if i == 0 else 0.08 bound = 0.03 if i == 0 else 0.08
err = _rel(x, x_ref) err = _rel(x, x_ref)
if err >= bound: if err >= bound:
assert gaps[(i, *k)] < 5e-3, (rank, i, k, "x", err) tie = 5e-3 if i == 0 else 1e-2
assert gaps[(i, *k)] < tie, (rank, i, k, "x", err)
flipped.add(k) flipped.add(k)
assert len(flipped) <= len(keys) // 4, (rank, sorted(flipped)) assert len(flipped) <= len(keys) // 4, (rank, sorted(flipped))
+6
View File
@@ -291,6 +291,12 @@ class KernelConfig:
enable_jit_warmup: bool = True enable_jit_warmup: bool = True
"""If True, run JIT compile warmup during kernel warmup.""" """If True, run JIT compile warmup during kernel warmup."""
monokernel: bool = False
"""If True, run the eligible layers of decode steps on the model's
MonoKernel: a persistent kernel that runs most or all of one decoder layer
in a single launch (DeepSeek-V4.1 on MI355X). A deployment that cannot run
it is refused at startup."""
moe_backend: MoEBackend = "auto" moe_backend: MoEBackend = "auto"
"""Backend for MoE expert computation kernels. Available options: """Backend for MoE expert computation kernels. Available options:
+4
View File
@@ -758,6 +758,7 @@ class EngineArgs:
enable_flashinfer_autotune: bool = get_field( enable_flashinfer_autotune: bool = get_field(
KernelConfig, "enable_flashinfer_autotune" KernelConfig, "enable_flashinfer_autotune"
) )
monokernel: bool = get_field(KernelConfig, "monokernel")
worker_cls: str = ParallelConfig.worker_cls worker_cls: str = ParallelConfig.worker_cls
worker_extension_cls: str = ParallelConfig.worker_extension_cls worker_extension_cls: str = ParallelConfig.worker_extension_cls
@@ -1802,6 +1803,7 @@ class EngineArgs:
"--enable-flashinfer-autotune", "--enable-flashinfer-autotune",
**kernel_kwargs["enable_flashinfer_autotune"], **kernel_kwargs["enable_flashinfer_autotune"],
) )
kernel_group.add_argument("--monokernel", **kernel_kwargs["monokernel"])
moe_backend_kwargs = kernel_kwargs["moe_backend"] moe_backend_kwargs = kernel_kwargs["moe_backend"]
moe_backend_kwargs["type"] = lambda s: s.lower().replace("-", "_") moe_backend_kwargs["type"] = lambda s: s.lower().replace("-", "_")
kernel_group.add_argument("--moe-backend", **moe_backend_kwargs) kernel_group.add_argument("--moe-backend", **moe_backend_kwargs)
@@ -2754,6 +2756,8 @@ class EngineArgs:
"are mutually exclusive" "are mutually exclusive"
) )
kernel_config.enable_flashinfer_autotune = self.enable_flashinfer_autotune kernel_config.enable_flashinfer_autotune = self.enable_flashinfer_autotune
if self.monokernel:
kernel_config.monokernel = True
if self.moe_backend != "auto": if self.moe_backend != "auto":
kernel_config.moe_backend = self.moe_backend kernel_config.moe_backend = self.moe_backend
if self.linear_backend != "auto": if self.linear_backend != "auto":
-7
View File
@@ -156,7 +156,6 @@ if TYPE_CHECKING:
VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION: bool = False VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION: bool = False
VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: bool = False VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: bool = False
VLLM_ROCM_AITER_MEGA_MOE_FUSE_SHARED_EXPERTS: bool = False VLLM_ROCM_AITER_MEGA_MOE_FUSE_SHARED_EXPERTS: bool = False
VLLM_ROCM_MONO_DECODE: bool = False
VLLM_ROCM_USE_AITER_TRITON_GEMM: bool = True VLLM_ROCM_USE_AITER_TRITON_GEMM: bool = True
VLLM_ROCM_USE_SKINNY_GEMM: bool = True VLLM_ROCM_USE_SKINNY_GEMM: bool = True
VLLM_ROCM_FP8_PADDING: bool = True VLLM_ROCM_FP8_PADDING: bool = True
@@ -1398,12 +1397,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
os.getenv("VLLM_ROCM_AITER_MEGA_MOE_FUSE_SHARED_EXPERTS", "False").lower() os.getenv("VLLM_ROCM_AITER_MEGA_MOE_FUSE_SHARED_EXPERTS", "False").lower()
in ("true", "1") in ("true", "1")
), ),
# Run the eligible layers of decode steps on the model's mono decode
# kernels: persistent launches a layer, the TP all-reduces in-kernel
# (DeepSeek-V4.1 on CDNA4). By default is disabled.
"VLLM_ROCM_MONO_DECODE": lambda: (
os.getenv("VLLM_ROCM_MONO_DECODE", "False").lower() in ("true", "1")
),
# Whether to use aiter triton kernels for gemm ops. # Whether to use aiter triton kernels for gemm ops.
# By default is enabled. # By default is enabled.
"VLLM_ROCM_USE_AITER_TRITON_GEMM": lambda: ( "VLLM_ROCM_USE_AITER_TRITON_GEMM": lambda: (
+11 -1
View File
@@ -282,7 +282,17 @@ class DeepseekV4DecoderLayer(nn.Module):
engram_mask: torch.Tensor | None = None, engram_mask: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
if self.mono is not None: if self.mono is not None:
out = self.mono(self, x, positions, residual, post_mix, res_mix, pre_mix) out = self.mono(
self,
x,
positions,
residual,
post_mix,
res_mix,
pre_mix,
engram_hashes,
engram_mask,
)
if out is not None: if out is not None:
return out return out
# Layer 0's attention seam projects the 2-D embedding with the folded # Layer 0's attention seam projects the 2-D embedding with the folded
+13 -10
View File
@@ -2,9 +2,10 @@
One fused decoder layer for vLLM's decode steps on MI355X: the attention seam, One fused decoder layer for vLLM's decode steps on MI355X: the attention seam,
the attention, the attention all-reduce, the FFN seam, the MoE and the MoE the attention, the attention all-reduce, the FFN seam, the MoE and the MoE
all-reduce, in two persistent launches a layer. The other backbone layers keep all-reduce, in two persistent launches a layer. A layer outside the whole-layer
vLLM's attention and run the rest -- the attention all-reduce, the FFN seam, kernels keeps vLLM's attention and runs the rest -- the attention all-reduce,
the MoE and its all-reduce -- as one launch (the FFN launch). ATOM's V4.1 mono decode the FFN seam, the MoE and its all-reduce -- as one launch (the FFN launch).
ATOM's V4.1 mono decode
(ROCm/ATOM `atom/models/deepseek_v41/mono`) is the reference design: the seam (ROCm/ATOM `atom/models/deepseek_v41/mono`) is the reference design: the seam
and MoE stages (`stages/`) and the framework under them (`common/`) are adapted and MoE stages (`stages/`) and the framework under them (`common/`) are adapted
from it, each file naming its origin; the attention stages (`attention/`) are from it, each file naming its origin; the attention stages (`attention/`) are
@@ -12,13 +13,15 @@ new, reading vLLM's KV records and weights as loaded.
## Scope ## Scope
- Layers: the whole layer on every backbone layer with the standard seam and no - Layers: the whole layer on all 40 backbone layers. K1 takes the residual
compressor / indexer: 3-7, 9-13, 15-19 (compress ratio 2) and 21-23, 25-27, settled, without folding the previous output in, on layer 0 (the embedding
29-31, 33-35, 37-39 (ratio 1), 30 of 40. The FFN launch on the other 10: broadcast to the four streams) and after an Engram write (layers 1 and 14,
layer 0 (embedding-broadcast seam), layer 1 (Engram at the seam) and the vLLM's Engram running before K1). On the index sources (2, 8, 14, 20 with
compressor / indexer layers 2, 8, 14 (also Engram), 20, 24, 28, 32, 36, their compressor; 24, 28, 32, 36) K1 also hands out its normed input and bf16
whose wo_b leaves its TP reduce to it. The DSpark draft layers keep vLLM's qr, vLLM's compressor and indexer run between K1 and K2, and K2 builds the
path. The paths interleave freely (same tensors at the layer boundary). key table from that top-k. The FFN launch serves a layer outside these (an
MXFP8 KV cache, another indexer); the DSpark draft layers keep vLLM's path.
The paths interleave freely (same tensors at the layer boundary).
- Steps: decode only (FULL graph or eager), M <= 48 rows (8 requests x 6 - Steps: decode only (FULL graph or eager), M <= 48 rows (8 requests x 6
DSpark tokens), TP2 / TP4, `fp8_ds_mla` KV records (ROCm), text tokens (the DSpark tokens), TP2 / TP4, `fp8_ds_mla` KV records (ROCm), text tokens (the
routing bias is `e_score_correction_bias`). routing bias is `e_score_correction_bias`).
@@ -151,7 +151,9 @@ def key_rec(c, t, k):
table: slot | 1 << 31 for a compressed row, a window slot otherwise, -1 for table: slot | 1 << 31 for a compressed row, a window slot otherwise, -1 for
none (it reads a zero record: its scores are masked, its values zeros).""" none (it reads a zero record: its scores are masked, its values zeros)."""
a, ratio = c["args"], c["ratio"] a, ratio = c["args"], c["ratio"]
val = ld_i32(a["kt"], t * KEYS + fx.min(fx.max(k, 0), KEYS - 1)) # a table this launch wrote (``krdy``) is read past the CU's cache
cm = CM_DEV if const_expr("krdy" in c) else 0
val = ld_i32(a["kt"], t * KEYS + fx.min(fx.max(k, 0), KEYS - 1), cm)
ok = val != fx.Int32(-1) ok = val != fx.Int32(-1)
slot = ok.select(val & fx.Int32(0x7FFFFFFF), fx.Int32(0)) slot = ok.select(val & fx.Int32(0x7FFFFFFF), fx.Int32(0))
if const_expr(ratio > 0): if const_expr(ratio > 0):
@@ -209,9 +211,11 @@ def stage_split(c, unit):
g = lane // 16 g = lane // 16
r16 = lane % 16 r16 = lane % 16
k0 = ch * ck k0 = ch * ck
if const_expr("krdy" in c):
await_flags(c, c["krdy"], 1, lambda u: t)
# the key count, this lane's key and q: independent loads, out together # the key count, this lane's key and q: independent loads, out together
# (K1's key table holds -1 past a token's keys) # (the key table holds -1 past a token's keys)
kv_len = ld_i32(a["klen"], t) kv_len = ld_i32(a["klen"], t, CM_DEV if const_expr("krdy" in c) else 0)
key = k0 + kt * 16 + r16 key = k0 + kt * 16 + r16
data, sc, okv = key_rec(c, t, key) data, sc, okv = key_rec(c, t, key)
qrow = d.heads * HEAD_DIM // 8 qrow = d.heads * HEAD_DIM // 8
@@ -188,6 +188,8 @@ def stage_kt(c, t):
ntopk.""" ntopk."""
tid = c["tid"] tid = c["tid"]
a, ratio = c["args"], c["ratio"] a, ratio = c["args"], c["ratio"]
# device scope where this launch reads the table (K2's)
cm = CM_DEV if const_expr(c.get("dev", False)) else 0
slot = ld_i32(a["slot"], 2 * t) slot = ld_i32(a["slot"], 2 * t)
live = slot >= 0 live = slot >= 0
nswa = live.select(ld_i32(a["swa_lens"], t), fx.Int32(0)) nswa = live.select(ld_i32(a["swa_lens"], t), fx.Int32(0))
@@ -212,10 +214,10 @@ def stage_kt(c, t):
c_val = (local >= 0).select(c_slot, fx.Int32(-1)) c_val = (local >= 0).select(c_slot, fx.Int32(-1))
val = (k < ntopk).select(c_val, val) val = (k < ntopk).select(c_val, val)
if k < KEYS: if k < KEYS:
bo.buffer_store(val, rsrc(a["kt"]), t * KEYS + k) bo.buffer_store(val, rsrc(a["kt"]), t * KEYS + k, cache_modifier=cm)
if tid == 0: if tid == 0:
bo.buffer_store(kv_len, rsrc(a["klen"]), t) bo.buffer_store(kv_len, rsrc(a["klen"]), t, cache_modifier=cm)
bo.buffer_store(ntopk, rsrc(a["klen"]), c["S"] + t) bo.buffer_store(ntopk, rsrc(a["klen"]), c["S"] + t, cache_modifier=cm)
# ---------------------------------------------------------------- wqkv_a # ---------------------------------------------------------------- wqkv_a
@@ -357,6 +359,20 @@ def stage_qkv(c, t):
amax = fx.max(amax, abs(y)) amax = fx.max(amax, abs(y))
code = mx_code(butterfly(amax, (1, 2), fx.max)) code = mx_code(butterfly(amax, (1, 2), fx.max))
mul = mx_mul(code) mul = mx_mul(code)
if const_expr(c.get("index", False)): # noqa: SIM102
# the bf16 qr, vLLM's indexer's input
if mine & (tt * 8 < Q_RANK):
words = [
fx.Vector.from_elements([ys[2 * i], ys[2 * i + 1]], fx.Float32)
.to(fx.BFloat16)
.bitcast(fx.Int32)[0]
for i in range(4)
]
bo.buffer_store(
fx.Vector.from_elements(words, fx.Int32),
rsrc(a["qr_out"]),
t * (Q_RANK // 2) + 4 * tt,
)
if mine & (tt * 8 < Q_RANK): if mine & (tt * 8 < Q_RANK):
w0 = fp8_pack4(*[clamp_fp8(ys[i] * mul) for i in range(4)]) w0 = fp8_pack4(*[clamp_fp8(ys[i] * mul) for i in range(4)])
w1 = fp8_pack4(*[clamp_fp8(ys[4 + i] * mul) for i in range(4)]) w1 = fp8_pack4(*[clamp_fp8(ys[4 + i] * mul) for i in range(4)])
@@ -597,8 +613,10 @@ def run_front(c, key, bid, x8_given=False, before_wqkv=None, after_wqkv=None):
u = free + n_free * tid u = free + n_free * tid
if u < xq_units: if u < xq_units:
c["mb"].put(c["xrdy"], u, fx.Int32(1)) c["mb"].put(c["xrdy"], u, fx.Int32(1))
for t in range(is_free.select(free, fx.Int32(s)), s, n_free): if const_expr(not c.get("index", False)):
stage_kt(c, t) # an index source's top-k comes after this launch: K2 builds its table
for t in range(is_free.select(free, fx.Int32(s)), s, n_free):
stage_kt(c, t)
stamp(c, 1) stamp(c, 1)
poss = [wqb_pos(c, t0, n) for t0, n in tiles] poss = [wqb_pos(c, t0, n) for t0, n in tiles]
+81 -3
View File
@@ -58,6 +58,12 @@ class MonoBuild:
tp: int tp: int
ratio: int # the layer's compress ratio (its attention's key sources) ratio: int # the layer's compress ratio (its attention's key sources)
timeline: bool = False timeline: bool = False
# K1's seam folds the previous layer's output into the residual; off for a
# residual that arrives settled (the first layer, after an Engram)
fold: bool = True
# the layer selects its own top-k (an index source): K1 hands its normed
# input and qr out for vLLM's indexer, K2 builds the key table
index: bool = False
def _moe_key(key: MonoBuild) -> MoeBuild: def _moe_key(key: MonoBuild) -> MoeBuild:
@@ -100,6 +106,7 @@ def scratch_layout(s: int, tp: int) -> dict:
lo = min(o for o, _ in mr.values()) lo = min(o for o, _ in mr.values())
place({n: (o - lo, n_) for n, (o, n_) in mr.items()}) place({n: (o - lo, n_) for n, (o, n_) in mr.items()})
place({"normed": (0, s * HIDDEN * 2), "xrdy_moe": (s * HIDDEN * 2 + 256, s * 8)}) place({"normed": (0, s * HIDDEN * 2), "xrdy_moe": (s * HIDDEN * 2 + 256, s * 8)})
place({"krdy": (0, s * 8)})
return out return out
@@ -175,7 +182,15 @@ def build_mono_k1(key: MonoBuild):
seam: SeamLds seam: SeamLds
front: FrontLds # type: ignore[valid-type] front: FrontLds # type: ignore[valid-type]
name = kernel_symbol("dsv41_mono_k1", s=s, tp=tp, r=key.ratio, tl=key.timeline) name = kernel_symbol(
"dsv41_mono_k1",
s=s,
tp=tp,
r=key.ratio,
tl=key.timeline,
f=key.fold,
i=key.index,
)
keyed = key_tuple(key, SOURCES) keyed = key_tuple(key, SOURCES)
@flyc.kernel(name=name, known_block_size=[THREADS, 1, 1]) @flyc.kernel(name=name, known_block_size=[THREADS, 1, 1])
@@ -215,6 +230,8 @@ def build_mono_k1(key: MonoBuild):
t2r: Int64, t2r: Int64,
kt: Int64, kt: Int64,
klen: Int64, klen: Int64,
normed_out: Int64,
qr_out: Int64,
scratch: Int64, scratch: Int64,
epoch: Int64, epoch: Int64,
tl: Int64, tl: Int64,
@@ -250,9 +267,11 @@ def build_mono_k1(key: MonoBuild):
"t2r": t2r, "t2r": t2r,
"kt": kt, "kt": kt,
"klen": klen, "klen": klen,
"qr_out": qr_out,
} }
flayout = {n: layout[n] for n in front_scratch(s, Dims(tp))} flayout = {n: layout[n] for n in front_scratch(s, Dims(tp))}
c = mb_front.front_context(fkey, lds.front.peek(), fargs, scratch, tag, flayout) c = mb_front.front_context(fkey, lds.front.peek(), fargs, scratch, tag, flayout)
c["index"] = key.index
amb = Mailbox(tag) amb = Mailbox(tag)
tid = fx.thread_idx.x tid = fx.thread_idx.x
ca = { ca = {
@@ -284,6 +303,7 @@ def build_mono_k1(key: MonoBuild):
"pre_out": pre_out, "pre_out": pre_out,
}, },
"d": StageDims(tp), "d": StageDims(tp),
"fold": key.fold,
} }
for region in ("lin", "pmix"): for region in ("lin", "pmix"):
ca[region] = sreg(scratch, layout[region][0], region) ca[region] = sreg(scratch, layout[region][0], region)
@@ -295,9 +315,14 @@ def build_mono_k1(key: MonoBuild):
seam.stage_slice(ca, task) seam.stage_slice(ca, task)
gpu.barrier() gpu.barrier()
def store(*v):
store_x8(c, *v)
if const_expr(key.index):
store_normed(normed_out, *v)
def seam_rest(): def seam_rest():
for t in range(first_task(bid, NORM0), s, BLOCKS): for t in range(first_task(bid, NORM0), s, BLOCKS):
seam.stage_norm(ca, t, lambda *v: store_x8(c, *v)) seam.stage_norm(ca, t, store)
publish_x8(c, t) publish_x8(c, t)
for t in range(first_task(bid, GATE0), s, BLOCKS): for t in range(first_task(bid, GATE0), s, BLOCKS):
seam.stage_gate(ca, t) seam.stage_gate(ca, t)
@@ -344,6 +369,8 @@ def build_mono_k1(key: MonoBuild):
t2r: Int64, t2r: Int64,
kt: Int64, kt: Int64,
klen: Int64, klen: Int64,
normed_out: Int64,
qr_out: Int64,
scratch: Int64, scratch: Int64,
epoch: Int64, epoch: Int64,
tl: Int64, tl: Int64,
@@ -386,6 +413,8 @@ def build_mono_k1(key: MonoBuild):
t2r, t2r,
kt, kt,
klen, klen,
normed_out,
qr_out,
scratch, scratch,
epoch, epoch,
tl, tl,
@@ -544,7 +573,9 @@ def build_mono_k2(key: MonoBuild):
seam: SeamLds # type: ignore[valid-type] seam: SeamLds # type: ignore[valid-type]
moe: MoeLds # type: ignore[valid-type] moe: MoeLds # type: ignore[valid-type]
name = kernel_symbol("dsv41_mono_k2", s=s, tp=tp, r=key.ratio, tl=key.timeline) name = kernel_symbol(
"dsv41_mono_k2", s=s, tp=tp, r=key.ratio, tl=key.timeline, i=key.index
)
keyed = key_tuple(key, SOURCES) keyed = key_tuple(key, SOURCES)
@flyc.kernel(name=name, known_block_size=[THREADS, 1, 1]) @flyc.kernel(name=name, known_block_size=[THREADS, 1, 1])
@@ -558,6 +589,13 @@ def build_mono_k2(key: MonoBuild):
comp_block: Int32, comp_block: Int32,
kt: Int64, kt: Int64,
klen: Int64, klen: Int64,
slot: Int64,
swa_idx: Int64,
swa_lens: Int64,
t2r: Int64,
topk: Int64,
comp_bt: Int64,
bt_stride: Int32,
pos: Int64, pos: Int64,
sink: Int64, sink: Int64,
qk_scale: Int32, qk_scale: Int32,
@@ -642,6 +680,32 @@ def build_mono_k2(key: MonoBuild):
if const_expr(key.timeline): if const_expr(key.timeline):
c["tl"], c["tl_points"] = tl, mb_back.BACK_POINTS c["tl"], c["tl_points"] = tl, mb_back.BACK_POINTS
mb_back.epoch_begin(c, epoch, ep) mb_back.epoch_begin(c, epoch, ep)
if const_expr(key.index):
# the key table, from the top-k vLLM's indexer selected after K1,
# on the last CTAs: each split task waits on its token's
ckt = {
"S": s,
"tid": c["tid"],
"dev": True,
"ratio": key.ratio,
"args": {
"slot": slot,
"swa_idx": swa_idx,
"swa_lens": swa_lens,
"pos": pos,
"t2r": t2r,
"topk": topk,
"comp_block": comp_block,
"comp_bt": comp_bt,
"bt_stride": bt_stride,
"kt": kt,
"klen": klen,
},
}
c["krdy"] = sreg(scratch, layout["krdy"][0], "krdy")
for t in range(first_task(bid, BLOCKS - s), s, BLOCKS):
mb_front.stage_kt(ckt, t)
mb_back.publish(c, c["krdy"], t)
mb_back.run_back(c, bkey, bid) mb_back.run_back(c, bkey, bid)
gpu.barrier() gpu.barrier()
@@ -685,6 +749,13 @@ def build_mono_k2(key: MonoBuild):
comp_block: Int32, comp_block: Int32,
kt: Int64, kt: Int64,
klen: Int64, klen: Int64,
slot: Int64,
swa_idx: Int64,
swa_lens: Int64,
t2r: Int64,
topk: Int64,
comp_bt: Int64,
bt_stride: Int32,
pos: Int64, pos: Int64,
sink: Int64, sink: Int64,
qk_scale: Int32, qk_scale: Int32,
@@ -736,6 +807,13 @@ def build_mono_k2(key: MonoBuild):
comp_block, comp_block,
kt, kt,
klen, klen,
slot,
swa_idx,
swa_lens,
t2r,
topk,
comp_bt,
bt_stride,
pos, pos,
sink, sink,
qk_scale, qk_scale,
+33 -6
View File
@@ -16,12 +16,13 @@ the same launches, so the ranks' epochs agree.
from __future__ import annotations from __future__ import annotations
import os import os
from collections.abc import Callable
from dataclasses import dataclass from dataclasses import dataclass
from pathlib import Path from pathlib import Path
import torch import torch
from .attention.plan import HEAD_DIM, HIDDEN, KEYS, Dims from .attention.plan import HEAD_DIM, HIDDEN, KEYS, Q_RANK, Dims
from .common.plan import BLOCKS from .common.plan import BLOCKS
from .layer import ( from .layer import (
EPOCH_WORDS, EPOCH_WORDS,
@@ -170,6 +171,12 @@ class DSV41MonoLayer:
self.klen = torch.zeros(2 * MAX_TOKENS, dtype=torch.int32, device=dev) self.klen = torch.zeros(2 * MAX_TOKENS, dtype=torch.int32, device=dev)
self._dummy = torch.zeros(4, dtype=torch.int32, device=dev) self._dummy = torch.zeros(4, dtype=torch.int32, device=dev)
self._zero_rec = torch.zeros(1024, dtype=torch.uint8, device=dev) self._zero_rec = torch.zeros(1024, dtype=torch.uint8, device=dev)
# the first layer's incoming pre-mix: stream 0 of the broadcast embedding
self._first_pre = torch.zeros(MAX_TOKENS, HC, dtype=torch.float32, device=dev)
self._first_pre[:, 0] = 1.0
# an index source's K1 outputs, vLLM's indexer's inputs
self.normed = torch.zeros(MAX_TOKENS, HIDDEN, dtype=torch.bfloat16, device=dev)
self.qr = torch.zeros(MAX_TOKENS, Q_RANK, dtype=torch.bfloat16, device=dev)
# the attention seam's outputs, K1 -> K2 # the attention seam's outputs, K1 -> K2
self.res_mid = torch.zeros( self.res_mid = torch.zeros(
MAX_TOKENS, HC, HIDDEN, dtype=torch.bfloat16, device=dev MAX_TOKENS, HC, HIDDEN, dtype=torch.bfloat16, device=dev
@@ -203,14 +210,17 @@ class DSV41MonoLayer:
self._scratch[tokens] = buf self._scratch[tokens] = buf
return buf return buf
def first_pre_mix(self, tokens: int) -> torch.Tensor:
return self._first_pre[:tokens]
@staticmethod @staticmethod
def supports(tokens: int) -> bool: def supports(tokens: int) -> bool:
return 1 <= tokens <= MAX_TOKENS return 1 <= tokens <= MAX_TOKENS
def kernels(self, tokens: int, ratio: int): def kernels(self, tokens: int, ratio: int, fold: bool = True, index: bool = False):
key = (tokens, ratio) key = (tokens, ratio, fold, index)
if key not in self._kernels: if key not in self._kernels:
b = MonoBuild(tokens, self.tp, ratio) b = MonoBuild(tokens, self.tp, ratio, fold=fold, index=index)
self._kernels[key] = (build_mono_k1(b), build_mono_k2(b)) self._kernels[key] = (build_mono_k1(b), build_mono_k2(b))
return self._kernels[key] return self._kernels[key]
@@ -249,15 +259,21 @@ class DSV41MonoLayer:
comp_cache: torch.Tensor | None = None, comp_cache: torch.Tensor | None = None,
comp_block_table: torch.Tensor | None = None, comp_block_table: torch.Tensor | None = None,
outs: tuple | None = None, outs: tuple | None = None,
fold: bool = True,
select: Callable[[torch.Tensor, torch.Tensor], None] | None = None,
): ):
"""-> (out, residual, post_mix, res_mix, pre_mix): the layer's outputs """-> (out, residual, post_mix, res_mix, pre_mix): the layer's outputs
(``outs``: buffers to write them into).""" (``outs``: buffers to write them into). ``select(normed, qr)``: an index
source's top-k selection into ``topk_indices``, run between the
kernels on K1's normed input and qr."""
M = x.shape[0] M = x.shape[0]
assert self.supports(M), M assert self.supports(M), M
a = w.attn a = w.attn
assert a is not None assert a is not None
ratio = a.ratio ratio = a.ratio
k1, k2 = self.kernels(M, ratio) index = select is not None
assert not index or ratio
k1, k2 = self.kernels(M, ratio, fold, index)
st = torch.cuda.current_stream() st = torch.cuda.current_stream()
if outs is None: if outs is None:
outs = self._outs(M, residual) outs = self._outs(M, residual)
@@ -324,11 +340,15 @@ class DSV41MonoLayer:
token_to_req.data_ptr(), token_to_req.data_ptr(),
self.kt.data_ptr(), self.kt.data_ptr(),
self.klen.data_ptr(), self.klen.data_ptr(),
self.normed.data_ptr(),
self.qr.data_ptr(),
self.scratch(M).data_ptr(), self.scratch(M).data_ptr(),
self.epoch.data_ptr(), self.epoch.data_ptr(),
0, 0,
stream=st, stream=st,
) )
if select is not None:
select(self.normed[:M], self.qr[:M])
k2( k2(
self.q.data_ptr(), self.q.data_ptr(),
swa_cache.data_ptr(), swa_cache.data_ptr(),
@@ -339,6 +359,13 @@ class DSV41MonoLayer:
comp_block, comp_block,
self.kt.data_ptr(), self.kt.data_ptr(),
self.klen.data_ptr(), self.klen.data_ptr(),
slot_mapping.data_ptr(),
swa_indices.data_ptr(),
swa_lens.data_ptr(),
token_to_req.data_ptr(),
topk.data_ptr(),
bt.data_ptr(),
bt_stride,
positions.data_ptr(), positions.data_ptr(),
a.attn_sink.data_ptr(), a.attn_sink.data_ptr(),
self._qk_scale, self._qk_scale,
@@ -141,6 +141,8 @@ def slice_loads(c, c0, e):
"""Element e's loads of ``stage_slice`` (e clamped into the step's).""" """Element e's loads of ``stage_slice`` (e clamped into the step's)."""
a = c["args"] a = c["args"]
t, o, col = slice_at(c0, e) t, o, col = slice_at(c0, e)
if const_expr(not c.get("fold", True)):
return (ld_bf(a["res_in"], (t * HC + o) * HIDDEN + col),)
r = [ld_bf(a["res_in"], (t * HC + h) * HIDDEN + col) for h in range(HC)] r = [ld_bf(a["res_in"], (t * HC + h) * HIDDEN + col) for h in range(HC)]
cm = [ld_f32(a["comb_in"], (t * HC + h) * HC + o) for h in range(HC)] cm = [ld_f32(a["comb_in"], (t * HC + h) * HC + o) for h in range(HC)]
return r, cm, ld_f32(a["post_in"], t * HC + o) return r, cm, ld_f32(a["post_in"], t * HC + o)
@@ -150,15 +152,19 @@ def slice_row(c, c0, e, got):
"""Element e of ``stage_slice``'s R_new from ``slice_loads``' values.""" """Element e of ``stage_slice``'s R_new from ``slice_loads``' values."""
a = c["args"] a = c["args"]
t, o, col = slice_at(c0, e) t, o, col = slice_at(c0, e)
r, cm, post = got if const_expr(not c.get("fold", True)):
# aiter's mhc_fused_post_pre_delayed_rmsnorm: ((post x + c0 r0) # a settled residual (no owed post): stream o as it arrives
# + c1 r1) + c2 r2, each product rounded, then fma(c3, r3, ..) v = got[0]
v = post * pend_value(c, t, col, e % COLS) + (r[0] * cm[0]) else:
for h in range_constexpr(1, HC - 1): r, cm, post = got
v = v + cm[h] * r[h] # aiter's mhc_fused_post_pre_delayed_rmsnorm: ((post x + c0 r0)
# bf16 R_new is what the collapse, the sums of squares and the # + c1 r1) + c2 r2, each product rounded, then fma(c3, r3, ..)
# mix projection all read v = post * pend_value(c, t, col, e % COLS) + (r[0] * cm[0])
v = bf16_round(fma(cm[HC - 1], r[HC - 1], v)) for h in range_constexpr(1, HC - 1):
v = v + cm[h] * r[h]
# bf16 R_new is what the collapse, the sums of squares and the
# mix projection all read
v = bf16_round(fma(cm[HC - 1], r[HC - 1], v))
bo.buffer_store(v.to(fx.BFloat16), rsrc(a["res_out"]), (t * HC + o) * HIDDEN + col) bo.buffer_store(v.to(fx.BFloat16), rsrc(a["res_out"]), (t * HC + o) * HIDDEN + col)
fx.ptr_store(v, c["rl"] + (t * KT + o * COLS + e % COLS)) fx.ptr_store(v, c["rl"] + (t * KT + o * COLS + e % COLS))
+87 -28
View File
@@ -1,15 +1,16 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DeepSeek-V4.1 mono decode layer on ROCm CDNA4 (``VLLM_ROCM_MONO_DECODE=1``). """DeepSeek-V4.1 mono decode layer on ROCm CDNA4 (``--monokernel``).
A decode step's backbone layer -- the attention seam, the attention, its TP A decode step's backbone layer -- the attention seam, the attention, its TP
all-reduce, the FFN seam, the MoE and its all-reduce -- runs as two persistent all-reduce, the FFN seam, the MoE and its all-reduce -- runs as two persistent
FlyDSL launches (``mono``, this directory) on the layer's loaded FlyDSL launches (``mono``, this directory) on the layer's loaded
weights, the TP reductions inside the kernels. A layer runs whole when it has weights, the TP reductions inside the kernels. Every backbone layer of the
the standard seam and no compressor or indexer of its own (compress ratio 1 or checkpoint runs whole (compress ratio 0, 1 or 2 over ``fp8_ds_mla`` records):
2) over ``fp8_ds_mla`` records. Any other backbone layer (the first, the Engram the first layer and the Engram layers hand K1 a settled residual, and an index
layers, the compressor / indexer layers) keeps vLLM's attention seam and source runs vLLM's compressor and indexer between the two launches. A layer
attention and runs the rest as one launch, the FFN launch, from wo_b's outside that (an MXFP8 KV cache, another indexer) keeps vLLM's attention seam
and attention and runs the rest as one launch, the FFN launch, from wo_b's
unreduced output. A step takes the kernels when it is decode only, at most unreduced output. A step takes the kernels when it is decode only, at most
``MAX_ROWS`` rows (eager or a FULL CUDA graph: the model is not torch.compiled); ``MAX_ROWS`` rows (eager or a FULL CUDA graph: the model is not torch.compiled);
a whole layer also needs causal SWA windows. Anything else runs the layer as a whole layer also needs causal SWA windows. Anything else runs the layer as
@@ -20,7 +21,7 @@ Every TP rank must take the same path at every layer (the kernels wait on each
other's pushes): the conditions read only metadata every rank holds alike. other's pushes): the conditions read only metadata every rank holds alike.
""" """
from typing import TYPE_CHECKING, cast from typing import TYPE_CHECKING, Any, cast
import torch import torch
@@ -62,7 +63,7 @@ def _mono_runner(device: torch.device):
# its peer memory is allocated and exchanged eagerly; vLLM warms # its peer memory is allocated and exchanged eagerly; vLLM warms
# every graph up eagerly before capturing it # every graph up eagerly before capturing it
raise RuntimeError( raise RuntimeError(
"VLLM_ROCM_MONO_DECODE was first reached inside a CUDA graph " "--monokernel was first reached inside a CUDA graph "
"capture, before an eager decode step." "capture, before an eager decode step."
) )
from vllm.models.deepseek_v41.amd.mono.runner import MAX_TOKENS, DSV41MonoLayer from vllm.models.deepseek_v41.amd.mono.runner import MAX_TOKENS, DSV41MonoLayer
@@ -113,6 +114,35 @@ def _unsupported_moe(layer: "DeepseekV4DecoderLayer", vllm_config) -> str | None
return None return None
def _select(
layer: "DeepseekV4DecoderLayer",
positions: torch.Tensor,
normed: torch.Tensor,
qr: torch.Tensor,
) -> None:
"""An index source's compressor and top-k selection, from K1's normed input
and bf16 qr: ``_forward_csa2_full`` / ``_forward_csa2_reindex`` and the
indexer op, serial."""
attn = layer.attn
comp, indexer = attn.compressor, attn.indexer
assert indexer is not None
if comp is not None:
kv_score = torch.mm(
normed, comp.fused_wkv_wgate.weight.T, out_dtype=torch.float32
)
latent = comp(kv_score, positions)
comp.insert_cache(latent, positions, attn.rotary_emb)
indexer._produce_k(latent, positions, attn.indexer_rotary_emb)
weights, _ = indexer.weights_proj(normed)
q, q_scale, weights = indexer.forward_q(
qr, None, weights, positions, attn.indexer_rotary_emb
)
assert weights is not None
indexer.indexer_op(
normed, (q, q_scale) if q_scale is not None else q, None, weights
)
class MonoDecodeLayer: class MonoDecodeLayer:
"""One decoder layer's mono path (``create``: None when the layer is not """One decoder layer's mono path (``create``: None when the layer is not
eligible): the whole layer (``__call__``) or, with ``ffn_only``, the FFN eligible): the whole layer (``__call__``) or, with ``ffn_only``, the FFN
@@ -127,7 +157,7 @@ class MonoDecodeLayer:
def create( def create(
layer: "DeepseekV4DecoderLayer", vllm_config layer: "DeepseekV4DecoderLayer", vllm_config
) -> "MonoDecodeLayer | None": ) -> "MonoDecodeLayer | None":
if not envs.VLLM_ROCM_MONO_DECODE: if not vllm_config.kernel_config.monokernel:
return None return None
from vllm.platforms.rocm import get_cdna_version from vllm.platforms.rocm import get_cdna_version
@@ -150,7 +180,7 @@ class MonoDecodeLayer:
if cus < BLOCKS: if cus < BLOCKS:
why = f"needs {BLOCKS} compute units, the GPU has {cus}" why = f"needs {BLOCKS} compute units, the GPU has {cus}"
if why is not None: if why is not None:
raise ValueError(f"VLLM_ROCM_MONO_DECODE {why}.") raise ValueError(f"--monokernel {why}.")
# the layer: the kernels serve the backbone's seams and MoE # the layer: the kernels serve the backbone's seams and MoE
config = vllm_config.model_config.hf_config config = vllm_config.model_config.hf_config
attn, ffn = layer.attn, layer.ffn attn, ffn = layer.attn, layer.ffn
@@ -174,12 +204,17 @@ class MonoDecodeLayer:
if why is not None: if why is not None:
logger.info_once("DSv4.1 mono decode off for some layers: %s", why) logger.info_once("DSv4.1 mono decode off for some layers: %s", why)
return None return None
# the whole layer: the kernels' attention takes the standard layers only # the whole layer: the kernels' attention takes the standard layers; an
if layer.engram is not None: # index source's compressor and indexer run between its two launches
why = "Engram" from vllm.models.deepseek_v41.amd.rocm import DeepseekV41RocmMxfp4Indexer
elif attn.compressor is not None or attn.indexer is not None:
why = "a compressor / indexer" if attn.compressor is not None and attn.indexer is None:
elif attn.compress_ratio not in (1, 2): why = "a compressor without an indexer"
elif attn.indexer is not None and not isinstance(
attn.indexer, DeepseekV41RocmMxfp4Indexer
):
why = f"a {type(attn.indexer).__name__}"
elif attn.compress_ratio not in (0, 1, 2):
why = f"compress ratio {attn.compress_ratio}" why = f"compress ratio {attn.compress_ratio}"
elif attn.kv_mxfp8 or attn.kv_cache_dtype != "fp8_ds_mla": elif attn.kv_mxfp8 or attn.kv_cache_dtype != "fp8_ds_mla":
why = f"a {attn.kv_cache_dtype} KV cache" why = f"a {attn.kv_cache_dtype} KV cache"
@@ -297,20 +332,23 @@ class MonoDecodeLayer:
post_mix: torch.Tensor | None, post_mix: torch.Tensor | None,
res_mix: torch.Tensor | None, res_mix: torch.Tensor | None,
pre_mix: torch.Tensor | None, pre_mix: torch.Tensor | None,
engram_hashes: torch.Tensor | None = None,
engram_mask: torch.Tensor | None = None,
) -> tuple[torch.Tensor, ...] | None: ) -> tuple[torch.Tensor, ...] | None:
"""The layer's outputs (x, residual, post_mix, res_mix, pre_mix) by the """The layer's outputs (x, residual, post_mix, res_mix, pre_mix) by the
mono kernels, or None when this step takes the original path (always for mono kernels, or None when this step takes the original path (always for
an ``ffn_only`` layer).""" an ``ffn_only`` layer)."""
if self.ffn_only: if self.ffn_only:
return None return None
if residual is None or x.dim() != 2: first = residual is None
return self._skip("first layer") if first and x.dim() != 2:
return self._skip("first layer of a pipeline stage")
why, swa, comp = self._decode_step(layer, x.shape[0]) why, swa, comp = self._decode_step(layer, x.shape[0])
attn = layer.attn attn = layer.attn
tensors = (x, residual, post_mix, res_mix, pre_mix) tensors = (x,) if first else (x, residual, post_mix, res_mix, pre_mix)
if why is None: if why is None:
# the kernels' attention: causal SWA windows, the compressed cache # the kernels' attention: causal SWA windows, the compressed cache
if comp is None: if comp is None and attn.compress_ratio:
why = "layer metadata missing" why = "layer metadata missing"
elif swa.decode_swa_width != SWA_WIDTH: elif swa.decode_swa_width != SWA_WIDTH:
why = f"SWA width {swa.decode_swa_width}" why = f"SWA width {swa.decode_swa_width}"
@@ -324,12 +362,37 @@ class MonoDecodeLayer:
why = "non-contiguous layer inputs" why = "non-contiguous layer inputs"
if why is not None: if why is not None:
return self._skip(why) return self._skip(why)
assert swa is not None and comp is not None assert swa is not None
runner = _mono_runner(x.device) runner = _mono_runner(x.device)
logger.info_once( logger.info_once(
"DSv4.1 mono decode layer active (decode steps of <= %d rows)", MAX_ROWS "DSv4.1 mono decode layer active (decode steps of <= %d rows)", MAX_ROWS
) )
out = runner.forward( # K1's seam folds the owed post into the residual, except where the
# residual arrives settled: the embedding, or after the Engram write
fold = True
if first:
residual = x.unsqueeze(1).expand(-1, layer.hc_mult, -1).contiguous()
pre_mix = runner.first_pre_mix(x.shape[0])
post_mix = res_mix = pre_mix # unread without the fold
fold = False
elif layer.engram is not None and engram_hashes is not None:
residual = layer.mhc_post(x, residual, post_mix, res_mix)
hashes = engram_hashes[:, layer.engram.layer_hash_index]
residual = layer.engram(residual, hashes, engram_mask)
fold = False
kw: dict[str, Any] = {}
if attn.indexer is not None:
kw["select"] = lambda normed, qr: _select(layer, positions, normed, qr)
if attn.compress_ratio:
assert comp is not None
kw.update(
topk_indices=attn.topk_indices_buffer,
comp_cache=_record_view(
attn._compressed_kv_cache(), comp.block_size // attn.compress_ratio
),
comp_block_table=comp.block_table,
)
return runner.forward(
self.weights(layer), self.weights(layer),
x, x,
residual, residual,
@@ -342,13 +405,9 @@ class MonoDecodeLayer:
swa.decode_swa_indices, swa.decode_swa_indices,
swa.decode_swa_lens, swa.decode_swa_lens,
swa.token_to_req_indices, swa.token_to_req_indices,
topk_indices=attn.topk_indices_buffer, fold=fold,
comp_cache=_record_view( **kw,
attn._compressed_kv_cache(), comp.block_size // attn.compress_ratio
),
comp_block_table=comp.block_table,
) )
return out
def ffn( def ffn(
self, self,