[ROCm][Perf][DSv4.1] Mono decode on every backbone layer (#61082)
Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
f47792af7f
commit
cc2a6eea1a
@@ -27,21 +27,25 @@ class _Runner:
|
||||
self.calls.append((args, kwargs))
|
||||
return ("mono outputs",)
|
||||
|
||||
def first_pre_mix(self, rows):
|
||||
return torch.zeros(rows, 4)
|
||||
|
||||
def ffn(self, *args):
|
||||
self.calls.append((args, {}))
|
||||
return ("ffn outputs",)
|
||||
|
||||
|
||||
def _layer(comp="comp"):
|
||||
def _layer(comp="comp", indexer=None):
|
||||
cache = lambda: torch.zeros(4, 128 * md.RECORD, dtype=torch.uint8) # noqa: E731
|
||||
attn = SimpleNamespace(
|
||||
swa_cache_layer=SimpleNamespace(prefix="swa", kv_cache=cache()),
|
||||
compressed_cache_prefix=comp,
|
||||
topk_indices_buffer=torch.zeros(64, 512, dtype=torch.int32),
|
||||
compress_ratio=1,
|
||||
indexer=indexer,
|
||||
_compressed_kv_cache=cache,
|
||||
)
|
||||
return SimpleNamespace(attn=attn)
|
||||
return SimpleNamespace(attn=attn, engram=None, hc_mult=4)
|
||||
|
||||
|
||||
def _swa(**overrides):
|
||||
@@ -77,7 +81,12 @@ def run(monkeypatch):
|
||||
``mode`` and ``metadata``; returns (result, runner calls)."""
|
||||
|
||||
def _run(
|
||||
mode=CUDAGraphMode.FULL, metadata=None, ffn_only=False, entry=None, **inputs
|
||||
mode=CUDAGraphMode.FULL,
|
||||
metadata=None,
|
||||
ffn_only=False,
|
||||
entry=None,
|
||||
layer=None,
|
||||
**inputs,
|
||||
):
|
||||
if metadata is None:
|
||||
metadata = {
|
||||
@@ -97,7 +106,7 @@ def run(monkeypatch):
|
||||
# has no compressed cache
|
||||
part, _ = args.pop("x"), args.pop("positions")
|
||||
return mono.ffn(_layer(comp=None), part, **args), runner.calls
|
||||
return mono(_layer(), **args), runner.calls
|
||||
return mono(layer or _layer(), **args), runner.calls
|
||||
|
||||
return _run
|
||||
|
||||
@@ -114,6 +123,23 @@ def test_decode_step_runs_mono(run, mode):
|
||||
1,
|
||||
)
|
||||
assert kwargs["comp_cache"].shape == (4, 128, md.RECORD)
|
||||
assert kwargs["fold"] and "select" not in kwargs
|
||||
|
||||
|
||||
def test_first_layer_runs_mono_from_the_embedding(run):
|
||||
"""The first layer's residual is the embedding broadcast to the four
|
||||
streams, which K1 takes as it is (no owed post to fold)."""
|
||||
out, calls = run(residual=None, post_mix=None, res_mix=None, pre_mix=None)
|
||||
assert out == ("mono outputs",) and len(calls) == 1
|
||||
args, kwargs = calls[0]
|
||||
assert args[2].shape == (M, 4, 5120) and not kwargs["fold"]
|
||||
|
||||
|
||||
def test_index_source_selects_between_the_kernels(run):
|
||||
"""An index source hands the runner its compressor and indexer to run
|
||||
between K1 and K2."""
|
||||
out, calls = run(layer=_layer(indexer=object()))
|
||||
assert out == ("mono outputs",) and callable(calls[0][1]["select"])
|
||||
|
||||
|
||||
def test_padded_decode_step_runs_mono(run):
|
||||
@@ -149,7 +175,9 @@ def test_padded_decode_step_runs_mono(run):
|
||||
),
|
||||
dict(metadata={}), # a profile / dummy run: no attention metadata
|
||||
dict(**_inputs(md.MAX_ROWS + 6)),
|
||||
dict(residual=None), # the first layer's seam broadcasts the embedding
|
||||
# a later pipeline stage's first layer: its stream arrives without
|
||||
# the mixes
|
||||
dict(residual=None, x=torch.zeros(M, 4, 5120, dtype=torch.bfloat16)),
|
||||
],
|
||||
ids=[
|
||||
"piecewise",
|
||||
@@ -158,7 +186,7 @@ def test_padded_decode_step_runs_mono(run):
|
||||
"tokens-past-rows",
|
||||
"no-metadata",
|
||||
"rows",
|
||||
"first-layer",
|
||||
"stage-input",
|
||||
],
|
||||
)
|
||||
def test_other_steps_take_the_original_path(run, case):
|
||||
@@ -202,22 +230,41 @@ def _decoder_layer(engram=None, **attn):
|
||||
)
|
||||
|
||||
|
||||
def _indexer():
|
||||
from vllm.models.deepseek_v41.amd.rocm import DeepseekV41RocmMxfp4Indexer
|
||||
|
||||
return object.__new__(DeepseekV41RocmMxfp4Indexer)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"layer, path",
|
||||
[
|
||||
(_decoder_layer(), "whole"),
|
||||
(_decoder_layer(compress_ratio=0), "ffn"),
|
||||
(_decoder_layer(engram=object()), "ffn"),
|
||||
(_decoder_layer(compressor=object(), indexer=object()), "ffn"),
|
||||
(_decoder_layer(compress_ratio=0), "whole"),
|
||||
(_decoder_layer(engram=object()), "whole"),
|
||||
(_decoder_layer(compressor=object(), indexer=_indexer()), "whole"),
|
||||
(_decoder_layer(indexer=_indexer()), "whole"),
|
||||
(_decoder_layer(compressor=object()), "ffn"),
|
||||
(_decoder_layer(indexer=object()), "ffn"),
|
||||
(_decoder_layer(kv_cache_dtype="auto"), "ffn"),
|
||||
(_decoder_layer(layer_id=40), None),
|
||||
],
|
||||
ids=["standard", "first", "engram", "kv-source", "index-source", "kv", "draft"],
|
||||
ids=[
|
||||
"standard",
|
||||
"window",
|
||||
"engram",
|
||||
"kv-source",
|
||||
"index-source",
|
||||
"compressor-only",
|
||||
"other-indexer",
|
||||
"kv",
|
||||
"draft",
|
||||
],
|
||||
)
|
||||
def test_layers_take_whole_or_ffn_launch(monkeypatch, layer, path):
|
||||
"""The whole-layer kernels serve the standard layers; every other backbone
|
||||
layer keeps vLLM's attention and runs its FFN half as one launch."""
|
||||
"""The whole-layer kernels serve every backbone layer of the checkpoint (an
|
||||
index source's compressor and indexer run between them); a layer outside
|
||||
them keeps vLLM's attention and runs its FFN half as one launch."""
|
||||
_deployment(monkeypatch)
|
||||
mono = md.MonoDecodeLayer.create(layer, _config())
|
||||
got = None if mono is None else "ffn" if mono.ffn_only else "whole"
|
||||
@@ -254,7 +301,7 @@ def test_other_steps_reduce_wo_b_themselves(run, case):
|
||||
assert out is None and not calls
|
||||
|
||||
|
||||
def _config(**parallel):
|
||||
def _config(monokernel=True, **parallel):
|
||||
parallel = (
|
||||
dict(enable_expert_parallel=False, enable_eplb=False, data_parallel_size=1)
|
||||
| parallel
|
||||
@@ -262,6 +309,7 @@ def _config(**parallel):
|
||||
return SimpleNamespace(
|
||||
model_config=SimpleNamespace(hf_config=SimpleNamespace(num_hidden_layers=40)),
|
||||
parallel_config=SimpleNamespace(**parallel),
|
||||
kernel_config=SimpleNamespace(monokernel=monokernel),
|
||||
)
|
||||
|
||||
|
||||
@@ -269,7 +317,6 @@ def _deployment(monkeypatch, cdna=4, cus=256):
|
||||
"""A deployment the kernels take (the runner module stubbed: no FlyDSL)."""
|
||||
import vllm.platforms.rocm as rocm
|
||||
|
||||
monkeypatch.setenv("VLLM_ROCM_MONO_DECODE", "1")
|
||||
monkeypatch.delenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4", raising=False)
|
||||
monkeypatch.setattr(rocm, "get_cdna_version", lambda: cdna)
|
||||
monkeypatch.setattr(md, "get_tensor_model_parallel_world_size", lambda: 2)
|
||||
@@ -280,6 +327,14 @@ def _deployment(monkeypatch, cdna=4, cus=256):
|
||||
monkeypatch.setitem(sys.modules, name, runner)
|
||||
|
||||
|
||||
def test_off_without_the_flag(monkeypatch):
|
||||
"""No --monokernel: no mono layer, whatever the deployment."""
|
||||
_deployment(monkeypatch)
|
||||
assert (
|
||||
md.MonoDecodeLayer.create(_decoder_layer(), _config(monokernel=False)) is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cdna", [3, 5])
|
||||
def test_opt_in_outside_the_kernels_cdna_raises(monkeypatch, cdna):
|
||||
_deployment(monkeypatch, cdna=cdna)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The DeepSeek-V4.1 mono decode path through the real model: a five-layer
|
||||
DeepSeek-V4.1-Flash at TP2 and TP4 on CDNA4, served with ``VLLM_ROCM_MONO_DECODE`` on
|
||||
DeepSeek-V4.1-Flash at TP2 and TP4 on CDNA4, served with ``--monokernel`` on
|
||||
and off on the same random weights: every decoder layer's outputs at every
|
||||
decode step must match. Unlike ``test_dsv41_mono_numerics.py``, which
|
||||
checks the kernels against vLLM's ops one stage at a time, this runs
|
||||
@@ -10,12 +10,13 @@ the per-step path choice, the weights taken off the layer, the metadata and
|
||||
the indexer's top-k -- so a change to the decoder layer that the kernels do
|
||||
not follow fails here.
|
||||
|
||||
The five layers hold both kinds the mono path takes:
|
||||
- 0: sliding window only -> the FFN launch
|
||||
- 1: ratio-2 KV and index source -> the FFN launch
|
||||
- 2: ratio-2 consumer -> the whole layer (K1 + K2)
|
||||
- 3: ratio-1 KV, index and candidate source -> the FFN launch
|
||||
- 4: ratio-1 consumer -> the whole layer
|
||||
Every layer takes the whole-layer path (K1 + K2); the five hold each variant:
|
||||
- 0: the first layer, sliding window only (K1 takes the embedding settled)
|
||||
- 1: ratio-2 KV and index source (vLLM's compressor and indexer between K1
|
||||
and K2)
|
||||
- 2: ratio-2 consumer with an Engram (K1 takes the residual settled)
|
||||
- 3: ratio-1 KV, index and candidate source
|
||||
- 4: ratio-1 index source over 3's candidates and KV
|
||||
"""
|
||||
|
||||
import zlib
|
||||
@@ -34,13 +35,17 @@ TRUNCATED = {
|
||||
# one entry a layer, the three MTP layers' included
|
||||
"compress_ratios": [0, 2, 2, 1, 1, 0, 0, 0],
|
||||
"kv_source_layer_ids": [1, 3],
|
||||
"index_source_layer_ids": [1, 3],
|
||||
# 4: an indexer of its own over 3's candidates (a REINDEX layer)
|
||||
"index_source_layer_ids": [1, 3, 4],
|
||||
"candidate_source_layer_id": 3,
|
||||
"engram_layer_ids": [],
|
||||
"engram_num_embeddings": [],
|
||||
# an Engram on a whole layer (K1 then takes its residual settled), its
|
||||
# tables shrunk: 24 prime-sized ranges from 1000
|
||||
"engram_layer_ids": [2],
|
||||
"engram_num_embeddings": [32768],
|
||||
"engram_vocab_size": 1000,
|
||||
"dspark_target_layer_ids": [2, 3, 4],
|
||||
}
|
||||
WHOLE, FFN = (2, 4), (0, 1, 3)
|
||||
WHOLE, FFN = (0, 1, 2, 3, 4), ()
|
||||
STEPS = 5 # decode steps after the prefill's token
|
||||
|
||||
|
||||
@@ -227,7 +232,6 @@ def _decode(vllm_runner, monkeypatch, tp: int, mono: bool, prompts, path: str):
|
||||
from vllm import SamplingParams
|
||||
from vllm.inputs import TokensPrompt
|
||||
|
||||
monkeypatch.setenv("VLLM_ROCM_MONO_DECODE", "1" if mono else "0")
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
|
||||
with vllm_runner(
|
||||
@@ -237,6 +241,7 @@ def _decode(vllm_runner, monkeypatch, tp: int, mono: bool, prompts, path: str):
|
||||
hf_overrides=TRUNCATED,
|
||||
worker_extension_cls=f"{__name__}.MonoProbe",
|
||||
moe_backend="aiter",
|
||||
monokernel=mono,
|
||||
language_model_only=True,
|
||||
tokenizer_mode="deepseek_v41",
|
||||
attention_config={"indexer_kv_dtype": "mxfp4", "indexer_sparse_logits": True},
|
||||
@@ -300,15 +305,16 @@ def test_mono_decode_matches_the_model(vllm_runner, monkeypatch, tmp_path, tp):
|
||||
same.add((len(prompt), len(prompt) + step))
|
||||
|
||||
# each layer's outputs for those tokens, mono against vLLM's path, token by
|
||||
# token. Layer 0 sees the same inputs on both paths at a request's first
|
||||
# decode token: there the bound is test_dsv41_mono_numerics' (about 3x the
|
||||
# kernels' 0.4%); its later tokens attend over KV each path wrote, 3%. Each
|
||||
# later layer takes the previous one's slightly different output, which
|
||||
# random weights amplify (2-4% by layer 4): 8%. A token whose top-6 routing
|
||||
# is a near-tie on vLLM's path (6th / 7th score within 5e-3: a 0.4% input
|
||||
# difference moves scores near 0.8 by about 3e-3) may pick another expert;
|
||||
# once it has, its residual stream differs and the later layers are exempt
|
||||
# for it. The residual stream: 3% (0.3% seen). A layer the kernels no
|
||||
# token. Layer 0 sees the same inputs on both paths, but its FFN input
|
||||
# differs by up to test_dsv41_mono_numerics' 1.2%, which random weights
|
||||
# amplify through the MoE: 3%. Each later layer takes the previous one's
|
||||
# slightly different output, which random weights amplify (2-4% by layer
|
||||
# 4): 8%. A token whose top-6 routing is a near-tie on vLLM's path may pick
|
||||
# another expert: a 0.4% input difference moves scores near 0.8 by about
|
||||
# 3e-3, so 6th / 7th scores within 5e-3 at layer 0 and, past it, where the
|
||||
# input carries the earlier layers' differences (0.4-0.7%), within 1e-2.
|
||||
# Once it has, its residual stream differs and the later layers are exempt
|
||||
# for it. The residual stream: 3% (0.5% seen). A layer the kernels no
|
||||
# longer follow is off by far more, on every token.
|
||||
for rank in range(tp):
|
||||
got, _ = torch.load(f"{tmp_path}/on.rank{rank}")
|
||||
@@ -317,8 +323,6 @@ def test_mono_decode_matches_the_model(vllm_runner, monkeypatch, tmp_path, tp):
|
||||
same & {k[1:] for k in got} & {k[1:] for k in want}
|
||||
)
|
||||
assert len(keys) >= len(prompts), keys
|
||||
# the bound that sees a 3% error: a request's first decode token
|
||||
assert any(p == n for n, p in keys), keys
|
||||
flipped: set = set()
|
||||
for i in sorted(WHOLE + FFN):
|
||||
for k in keys:
|
||||
@@ -327,9 +331,10 @@ def test_mono_decode_matches_the_model(vllm_runner, monkeypatch, tmp_path, tp):
|
||||
(x, res), (x_ref, res_ref) = got[(i, *k)], want[(i, *k)]
|
||||
err = _rel(res, res_ref)
|
||||
assert err < 0.03, (rank, i, k, "residual", err)
|
||||
bound = (0.015 if k[1] == k[0] else 0.03) if i == 0 else 0.08
|
||||
bound = 0.03 if i == 0 else 0.08
|
||||
err = _rel(x, x_ref)
|
||||
if err >= bound:
|
||||
assert gaps[(i, *k)] < 5e-3, (rank, i, k, "x", err)
|
||||
tie = 5e-3 if i == 0 else 1e-2
|
||||
assert gaps[(i, *k)] < tie, (rank, i, k, "x", err)
|
||||
flipped.add(k)
|
||||
assert len(flipped) <= len(keys) // 4, (rank, sorted(flipped))
|
||||
|
||||
@@ -291,6 +291,12 @@ class KernelConfig:
|
||||
enable_jit_warmup: bool = True
|
||||
"""If True, run JIT compile warmup during kernel warmup."""
|
||||
|
||||
monokernel: bool = False
|
||||
"""If True, run the eligible layers of decode steps on the model's
|
||||
MonoKernel: a persistent kernel that runs most or all of one decoder layer
|
||||
in a single launch (DeepSeek-V4.1 on MI355X). A deployment that cannot run
|
||||
it is refused at startup."""
|
||||
|
||||
moe_backend: MoEBackend = "auto"
|
||||
"""Backend for MoE expert computation kernels. Available options:
|
||||
|
||||
|
||||
@@ -758,6 +758,7 @@ class EngineArgs:
|
||||
enable_flashinfer_autotune: bool = get_field(
|
||||
KernelConfig, "enable_flashinfer_autotune"
|
||||
)
|
||||
monokernel: bool = get_field(KernelConfig, "monokernel")
|
||||
worker_cls: str = ParallelConfig.worker_cls
|
||||
worker_extension_cls: str = ParallelConfig.worker_extension_cls
|
||||
|
||||
@@ -1802,6 +1803,7 @@ class EngineArgs:
|
||||
"--enable-flashinfer-autotune",
|
||||
**kernel_kwargs["enable_flashinfer_autotune"],
|
||||
)
|
||||
kernel_group.add_argument("--monokernel", **kernel_kwargs["monokernel"])
|
||||
moe_backend_kwargs = kernel_kwargs["moe_backend"]
|
||||
moe_backend_kwargs["type"] = lambda s: s.lower().replace("-", "_")
|
||||
kernel_group.add_argument("--moe-backend", **moe_backend_kwargs)
|
||||
@@ -2754,6 +2756,8 @@ class EngineArgs:
|
||||
"are mutually exclusive"
|
||||
)
|
||||
kernel_config.enable_flashinfer_autotune = self.enable_flashinfer_autotune
|
||||
if self.monokernel:
|
||||
kernel_config.monokernel = True
|
||||
if self.moe_backend != "auto":
|
||||
kernel_config.moe_backend = self.moe_backend
|
||||
if self.linear_backend != "auto":
|
||||
|
||||
@@ -156,7 +156,6 @@ if TYPE_CHECKING:
|
||||
VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION: bool = False
|
||||
VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: bool = False
|
||||
VLLM_ROCM_AITER_MEGA_MOE_FUSE_SHARED_EXPERTS: bool = False
|
||||
VLLM_ROCM_MONO_DECODE: bool = False
|
||||
VLLM_ROCM_USE_AITER_TRITON_GEMM: bool = True
|
||||
VLLM_ROCM_USE_SKINNY_GEMM: bool = True
|
||||
VLLM_ROCM_FP8_PADDING: bool = True
|
||||
@@ -1398,12 +1397,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
os.getenv("VLLM_ROCM_AITER_MEGA_MOE_FUSE_SHARED_EXPERTS", "False").lower()
|
||||
in ("true", "1")
|
||||
),
|
||||
# Run the eligible layers of decode steps on the model's mono decode
|
||||
# kernels: persistent launches a layer, the TP all-reduces in-kernel
|
||||
# (DeepSeek-V4.1 on CDNA4). By default is disabled.
|
||||
"VLLM_ROCM_MONO_DECODE": lambda: (
|
||||
os.getenv("VLLM_ROCM_MONO_DECODE", "False").lower() in ("true", "1")
|
||||
),
|
||||
# Whether to use aiter triton kernels for gemm ops.
|
||||
# By default is enabled.
|
||||
"VLLM_ROCM_USE_AITER_TRITON_GEMM": lambda: (
|
||||
|
||||
@@ -282,7 +282,17 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
engram_mask: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
if self.mono is not None:
|
||||
out = self.mono(self, x, positions, residual, post_mix, res_mix, pre_mix)
|
||||
out = self.mono(
|
||||
self,
|
||||
x,
|
||||
positions,
|
||||
residual,
|
||||
post_mix,
|
||||
res_mix,
|
||||
pre_mix,
|
||||
engram_hashes,
|
||||
engram_mask,
|
||||
)
|
||||
if out is not None:
|
||||
return out
|
||||
# Layer 0's attention seam projects the 2-D embedding with the folded
|
||||
|
||||
@@ -2,9 +2,10 @@
|
||||
|
||||
One fused decoder layer for vLLM's decode steps on MI355X: the attention seam,
|
||||
the attention, the attention all-reduce, the FFN seam, the MoE and the MoE
|
||||
all-reduce, in two persistent launches a layer. The other backbone layers keep
|
||||
vLLM's attention and run the rest -- the attention all-reduce, the FFN seam,
|
||||
the MoE and its all-reduce -- as one launch (the FFN launch). ATOM's V4.1 mono decode
|
||||
all-reduce, in two persistent launches a layer. A layer outside the whole-layer
|
||||
kernels keeps vLLM's attention and runs the rest -- the attention all-reduce,
|
||||
the FFN seam, the MoE and its all-reduce -- as one launch (the FFN launch).
|
||||
ATOM's V4.1 mono decode
|
||||
(ROCm/ATOM `atom/models/deepseek_v41/mono`) is the reference design: the seam
|
||||
and MoE stages (`stages/`) and the framework under them (`common/`) are adapted
|
||||
from it, each file naming its origin; the attention stages (`attention/`) are
|
||||
@@ -12,13 +13,15 @@ new, reading vLLM's KV records and weights as loaded.
|
||||
|
||||
## Scope
|
||||
|
||||
- Layers: the whole layer on every backbone layer with the standard seam and no
|
||||
compressor / indexer: 3-7, 9-13, 15-19 (compress ratio 2) and 21-23, 25-27,
|
||||
29-31, 33-35, 37-39 (ratio 1), 30 of 40. The FFN launch on the other 10:
|
||||
layer 0 (embedding-broadcast seam), layer 1 (Engram at the seam) and the
|
||||
compressor / indexer layers 2, 8, 14 (also Engram), 20, 24, 28, 32, 36,
|
||||
whose wo_b leaves its TP reduce to it. The DSpark draft layers keep vLLM's
|
||||
path. The paths interleave freely (same tensors at the layer boundary).
|
||||
- Layers: the whole layer on all 40 backbone layers. K1 takes the residual
|
||||
settled, without folding the previous output in, on layer 0 (the embedding
|
||||
broadcast to the four streams) and after an Engram write (layers 1 and 14,
|
||||
vLLM's Engram running before K1). On the index sources (2, 8, 14, 20 with
|
||||
their compressor; 24, 28, 32, 36) K1 also hands out its normed input and bf16
|
||||
qr, vLLM's compressor and indexer run between K1 and K2, and K2 builds the
|
||||
key table from that top-k. The FFN launch serves a layer outside these (an
|
||||
MXFP8 KV cache, another indexer); the DSpark draft layers keep vLLM's path.
|
||||
The paths interleave freely (same tensors at the layer boundary).
|
||||
- Steps: decode only (FULL graph or eager), M <= 48 rows (8 requests x 6
|
||||
DSpark tokens), TP2 / TP4, `fp8_ds_mla` KV records (ROCm), text tokens (the
|
||||
routing bias is `e_score_correction_bias`).
|
||||
|
||||
@@ -151,7 +151,9 @@ def key_rec(c, t, k):
|
||||
table: slot | 1 << 31 for a compressed row, a window slot otherwise, -1 for
|
||||
none (it reads a zero record: its scores are masked, its values zeros)."""
|
||||
a, ratio = c["args"], c["ratio"]
|
||||
val = ld_i32(a["kt"], t * KEYS + fx.min(fx.max(k, 0), KEYS - 1))
|
||||
# a table this launch wrote (``krdy``) is read past the CU's cache
|
||||
cm = CM_DEV if const_expr("krdy" in c) else 0
|
||||
val = ld_i32(a["kt"], t * KEYS + fx.min(fx.max(k, 0), KEYS - 1), cm)
|
||||
ok = val != fx.Int32(-1)
|
||||
slot = ok.select(val & fx.Int32(0x7FFFFFFF), fx.Int32(0))
|
||||
if const_expr(ratio > 0):
|
||||
@@ -209,9 +211,11 @@ def stage_split(c, unit):
|
||||
g = lane // 16
|
||||
r16 = lane % 16
|
||||
k0 = ch * ck
|
||||
if const_expr("krdy" in c):
|
||||
await_flags(c, c["krdy"], 1, lambda u: t)
|
||||
# the key count, this lane's key and q: independent loads, out together
|
||||
# (K1's key table holds -1 past a token's keys)
|
||||
kv_len = ld_i32(a["klen"], t)
|
||||
# (the key table holds -1 past a token's keys)
|
||||
kv_len = ld_i32(a["klen"], t, CM_DEV if const_expr("krdy" in c) else 0)
|
||||
key = k0 + kt * 16 + r16
|
||||
data, sc, okv = key_rec(c, t, key)
|
||||
qrow = d.heads * HEAD_DIM // 8
|
||||
|
||||
@@ -188,6 +188,8 @@ def stage_kt(c, t):
|
||||
ntopk."""
|
||||
tid = c["tid"]
|
||||
a, ratio = c["args"], c["ratio"]
|
||||
# device scope where this launch reads the table (K2's)
|
||||
cm = CM_DEV if const_expr(c.get("dev", False)) else 0
|
||||
slot = ld_i32(a["slot"], 2 * t)
|
||||
live = slot >= 0
|
||||
nswa = live.select(ld_i32(a["swa_lens"], t), fx.Int32(0))
|
||||
@@ -212,10 +214,10 @@ def stage_kt(c, t):
|
||||
c_val = (local >= 0).select(c_slot, fx.Int32(-1))
|
||||
val = (k < ntopk).select(c_val, val)
|
||||
if k < KEYS:
|
||||
bo.buffer_store(val, rsrc(a["kt"]), t * KEYS + k)
|
||||
bo.buffer_store(val, rsrc(a["kt"]), t * KEYS + k, cache_modifier=cm)
|
||||
if tid == 0:
|
||||
bo.buffer_store(kv_len, rsrc(a["klen"]), t)
|
||||
bo.buffer_store(ntopk, rsrc(a["klen"]), c["S"] + t)
|
||||
bo.buffer_store(kv_len, rsrc(a["klen"]), t, cache_modifier=cm)
|
||||
bo.buffer_store(ntopk, rsrc(a["klen"]), c["S"] + t, cache_modifier=cm)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- wqkv_a
|
||||
@@ -357,6 +359,20 @@ def stage_qkv(c, t):
|
||||
amax = fx.max(amax, abs(y))
|
||||
code = mx_code(butterfly(amax, (1, 2), fx.max))
|
||||
mul = mx_mul(code)
|
||||
if const_expr(c.get("index", False)): # noqa: SIM102
|
||||
# the bf16 qr, vLLM's indexer's input
|
||||
if mine & (tt * 8 < Q_RANK):
|
||||
words = [
|
||||
fx.Vector.from_elements([ys[2 * i], ys[2 * i + 1]], fx.Float32)
|
||||
.to(fx.BFloat16)
|
||||
.bitcast(fx.Int32)[0]
|
||||
for i in range(4)
|
||||
]
|
||||
bo.buffer_store(
|
||||
fx.Vector.from_elements(words, fx.Int32),
|
||||
rsrc(a["qr_out"]),
|
||||
t * (Q_RANK // 2) + 4 * tt,
|
||||
)
|
||||
if mine & (tt * 8 < Q_RANK):
|
||||
w0 = fp8_pack4(*[clamp_fp8(ys[i] * mul) for i in range(4)])
|
||||
w1 = fp8_pack4(*[clamp_fp8(ys[4 + i] * mul) for i in range(4)])
|
||||
@@ -597,8 +613,10 @@ def run_front(c, key, bid, x8_given=False, before_wqkv=None, after_wqkv=None):
|
||||
u = free + n_free * tid
|
||||
if u < xq_units:
|
||||
c["mb"].put(c["xrdy"], u, fx.Int32(1))
|
||||
for t in range(is_free.select(free, fx.Int32(s)), s, n_free):
|
||||
stage_kt(c, t)
|
||||
if const_expr(not c.get("index", False)):
|
||||
# an index source's top-k comes after this launch: K2 builds its table
|
||||
for t in range(is_free.select(free, fx.Int32(s)), s, n_free):
|
||||
stage_kt(c, t)
|
||||
stamp(c, 1)
|
||||
poss = [wqb_pos(c, t0, n) for t0, n in tiles]
|
||||
|
||||
|
||||
@@ -58,6 +58,12 @@ class MonoBuild:
|
||||
tp: int
|
||||
ratio: int # the layer's compress ratio (its attention's key sources)
|
||||
timeline: bool = False
|
||||
# K1's seam folds the previous layer's output into the residual; off for a
|
||||
# residual that arrives settled (the first layer, after an Engram)
|
||||
fold: bool = True
|
||||
# the layer selects its own top-k (an index source): K1 hands its normed
|
||||
# input and qr out for vLLM's indexer, K2 builds the key table
|
||||
index: bool = False
|
||||
|
||||
|
||||
def _moe_key(key: MonoBuild) -> MoeBuild:
|
||||
@@ -100,6 +106,7 @@ def scratch_layout(s: int, tp: int) -> dict:
|
||||
lo = min(o for o, _ in mr.values())
|
||||
place({n: (o - lo, n_) for n, (o, n_) in mr.items()})
|
||||
place({"normed": (0, s * HIDDEN * 2), "xrdy_moe": (s * HIDDEN * 2 + 256, s * 8)})
|
||||
place({"krdy": (0, s * 8)})
|
||||
return out
|
||||
|
||||
|
||||
@@ -175,7 +182,15 @@ def build_mono_k1(key: MonoBuild):
|
||||
seam: SeamLds
|
||||
front: FrontLds # type: ignore[valid-type]
|
||||
|
||||
name = kernel_symbol("dsv41_mono_k1", s=s, tp=tp, r=key.ratio, tl=key.timeline)
|
||||
name = kernel_symbol(
|
||||
"dsv41_mono_k1",
|
||||
s=s,
|
||||
tp=tp,
|
||||
r=key.ratio,
|
||||
tl=key.timeline,
|
||||
f=key.fold,
|
||||
i=key.index,
|
||||
)
|
||||
keyed = key_tuple(key, SOURCES)
|
||||
|
||||
@flyc.kernel(name=name, known_block_size=[THREADS, 1, 1])
|
||||
@@ -215,6 +230,8 @@ def build_mono_k1(key: MonoBuild):
|
||||
t2r: Int64,
|
||||
kt: Int64,
|
||||
klen: Int64,
|
||||
normed_out: Int64,
|
||||
qr_out: Int64,
|
||||
scratch: Int64,
|
||||
epoch: Int64,
|
||||
tl: Int64,
|
||||
@@ -250,9 +267,11 @@ def build_mono_k1(key: MonoBuild):
|
||||
"t2r": t2r,
|
||||
"kt": kt,
|
||||
"klen": klen,
|
||||
"qr_out": qr_out,
|
||||
}
|
||||
flayout = {n: layout[n] for n in front_scratch(s, Dims(tp))}
|
||||
c = mb_front.front_context(fkey, lds.front.peek(), fargs, scratch, tag, flayout)
|
||||
c["index"] = key.index
|
||||
amb = Mailbox(tag)
|
||||
tid = fx.thread_idx.x
|
||||
ca = {
|
||||
@@ -284,6 +303,7 @@ def build_mono_k1(key: MonoBuild):
|
||||
"pre_out": pre_out,
|
||||
},
|
||||
"d": StageDims(tp),
|
||||
"fold": key.fold,
|
||||
}
|
||||
for region in ("lin", "pmix"):
|
||||
ca[region] = sreg(scratch, layout[region][0], region)
|
||||
@@ -295,9 +315,14 @@ def build_mono_k1(key: MonoBuild):
|
||||
seam.stage_slice(ca, task)
|
||||
gpu.barrier()
|
||||
|
||||
def store(*v):
|
||||
store_x8(c, *v)
|
||||
if const_expr(key.index):
|
||||
store_normed(normed_out, *v)
|
||||
|
||||
def seam_rest():
|
||||
for t in range(first_task(bid, NORM0), s, BLOCKS):
|
||||
seam.stage_norm(ca, t, lambda *v: store_x8(c, *v))
|
||||
seam.stage_norm(ca, t, store)
|
||||
publish_x8(c, t)
|
||||
for t in range(first_task(bid, GATE0), s, BLOCKS):
|
||||
seam.stage_gate(ca, t)
|
||||
@@ -344,6 +369,8 @@ def build_mono_k1(key: MonoBuild):
|
||||
t2r: Int64,
|
||||
kt: Int64,
|
||||
klen: Int64,
|
||||
normed_out: Int64,
|
||||
qr_out: Int64,
|
||||
scratch: Int64,
|
||||
epoch: Int64,
|
||||
tl: Int64,
|
||||
@@ -386,6 +413,8 @@ def build_mono_k1(key: MonoBuild):
|
||||
t2r,
|
||||
kt,
|
||||
klen,
|
||||
normed_out,
|
||||
qr_out,
|
||||
scratch,
|
||||
epoch,
|
||||
tl,
|
||||
@@ -544,7 +573,9 @@ def build_mono_k2(key: MonoBuild):
|
||||
seam: SeamLds # type: ignore[valid-type]
|
||||
moe: MoeLds # type: ignore[valid-type]
|
||||
|
||||
name = kernel_symbol("dsv41_mono_k2", s=s, tp=tp, r=key.ratio, tl=key.timeline)
|
||||
name = kernel_symbol(
|
||||
"dsv41_mono_k2", s=s, tp=tp, r=key.ratio, tl=key.timeline, i=key.index
|
||||
)
|
||||
keyed = key_tuple(key, SOURCES)
|
||||
|
||||
@flyc.kernel(name=name, known_block_size=[THREADS, 1, 1])
|
||||
@@ -558,6 +589,13 @@ def build_mono_k2(key: MonoBuild):
|
||||
comp_block: Int32,
|
||||
kt: Int64,
|
||||
klen: Int64,
|
||||
slot: Int64,
|
||||
swa_idx: Int64,
|
||||
swa_lens: Int64,
|
||||
t2r: Int64,
|
||||
topk: Int64,
|
||||
comp_bt: Int64,
|
||||
bt_stride: Int32,
|
||||
pos: Int64,
|
||||
sink: Int64,
|
||||
qk_scale: Int32,
|
||||
@@ -642,6 +680,32 @@ def build_mono_k2(key: MonoBuild):
|
||||
if const_expr(key.timeline):
|
||||
c["tl"], c["tl_points"] = tl, mb_back.BACK_POINTS
|
||||
mb_back.epoch_begin(c, epoch, ep)
|
||||
if const_expr(key.index):
|
||||
# the key table, from the top-k vLLM's indexer selected after K1,
|
||||
# on the last CTAs: each split task waits on its token's
|
||||
ckt = {
|
||||
"S": s,
|
||||
"tid": c["tid"],
|
||||
"dev": True,
|
||||
"ratio": key.ratio,
|
||||
"args": {
|
||||
"slot": slot,
|
||||
"swa_idx": swa_idx,
|
||||
"swa_lens": swa_lens,
|
||||
"pos": pos,
|
||||
"t2r": t2r,
|
||||
"topk": topk,
|
||||
"comp_block": comp_block,
|
||||
"comp_bt": comp_bt,
|
||||
"bt_stride": bt_stride,
|
||||
"kt": kt,
|
||||
"klen": klen,
|
||||
},
|
||||
}
|
||||
c["krdy"] = sreg(scratch, layout["krdy"][0], "krdy")
|
||||
for t in range(first_task(bid, BLOCKS - s), s, BLOCKS):
|
||||
mb_front.stage_kt(ckt, t)
|
||||
mb_back.publish(c, c["krdy"], t)
|
||||
mb_back.run_back(c, bkey, bid)
|
||||
gpu.barrier()
|
||||
|
||||
@@ -685,6 +749,13 @@ def build_mono_k2(key: MonoBuild):
|
||||
comp_block: Int32,
|
||||
kt: Int64,
|
||||
klen: Int64,
|
||||
slot: Int64,
|
||||
swa_idx: Int64,
|
||||
swa_lens: Int64,
|
||||
t2r: Int64,
|
||||
topk: Int64,
|
||||
comp_bt: Int64,
|
||||
bt_stride: Int32,
|
||||
pos: Int64,
|
||||
sink: Int64,
|
||||
qk_scale: Int32,
|
||||
@@ -736,6 +807,13 @@ def build_mono_k2(key: MonoBuild):
|
||||
comp_block,
|
||||
kt,
|
||||
klen,
|
||||
slot,
|
||||
swa_idx,
|
||||
swa_lens,
|
||||
t2r,
|
||||
topk,
|
||||
comp_bt,
|
||||
bt_stride,
|
||||
pos,
|
||||
sink,
|
||||
qk_scale,
|
||||
|
||||
@@ -16,12 +16,13 @@ the same launches, so the ranks' epochs agree.
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
|
||||
from .attention.plan import HEAD_DIM, HIDDEN, KEYS, Dims
|
||||
from .attention.plan import HEAD_DIM, HIDDEN, KEYS, Q_RANK, Dims
|
||||
from .common.plan import BLOCKS
|
||||
from .layer import (
|
||||
EPOCH_WORDS,
|
||||
@@ -170,6 +171,12 @@ class DSV41MonoLayer:
|
||||
self.klen = torch.zeros(2 * MAX_TOKENS, dtype=torch.int32, device=dev)
|
||||
self._dummy = torch.zeros(4, dtype=torch.int32, device=dev)
|
||||
self._zero_rec = torch.zeros(1024, dtype=torch.uint8, device=dev)
|
||||
# the first layer's incoming pre-mix: stream 0 of the broadcast embedding
|
||||
self._first_pre = torch.zeros(MAX_TOKENS, HC, dtype=torch.float32, device=dev)
|
||||
self._first_pre[:, 0] = 1.0
|
||||
# an index source's K1 outputs, vLLM's indexer's inputs
|
||||
self.normed = torch.zeros(MAX_TOKENS, HIDDEN, dtype=torch.bfloat16, device=dev)
|
||||
self.qr = torch.zeros(MAX_TOKENS, Q_RANK, dtype=torch.bfloat16, device=dev)
|
||||
# the attention seam's outputs, K1 -> K2
|
||||
self.res_mid = torch.zeros(
|
||||
MAX_TOKENS, HC, HIDDEN, dtype=torch.bfloat16, device=dev
|
||||
@@ -203,14 +210,17 @@ class DSV41MonoLayer:
|
||||
self._scratch[tokens] = buf
|
||||
return buf
|
||||
|
||||
def first_pre_mix(self, tokens: int) -> torch.Tensor:
|
||||
return self._first_pre[:tokens]
|
||||
|
||||
@staticmethod
|
||||
def supports(tokens: int) -> bool:
|
||||
return 1 <= tokens <= MAX_TOKENS
|
||||
|
||||
def kernels(self, tokens: int, ratio: int):
|
||||
key = (tokens, ratio)
|
||||
def kernels(self, tokens: int, ratio: int, fold: bool = True, index: bool = False):
|
||||
key = (tokens, ratio, fold, index)
|
||||
if key not in self._kernels:
|
||||
b = MonoBuild(tokens, self.tp, ratio)
|
||||
b = MonoBuild(tokens, self.tp, ratio, fold=fold, index=index)
|
||||
self._kernels[key] = (build_mono_k1(b), build_mono_k2(b))
|
||||
return self._kernels[key]
|
||||
|
||||
@@ -249,15 +259,21 @@ class DSV41MonoLayer:
|
||||
comp_cache: torch.Tensor | None = None,
|
||||
comp_block_table: torch.Tensor | None = None,
|
||||
outs: tuple | None = None,
|
||||
fold: bool = True,
|
||||
select: Callable[[torch.Tensor, torch.Tensor], None] | None = None,
|
||||
):
|
||||
"""-> (out, residual, post_mix, res_mix, pre_mix): the layer's outputs
|
||||
(``outs``: buffers to write them into)."""
|
||||
(``outs``: buffers to write them into). ``select(normed, qr)``: an index
|
||||
source's top-k selection into ``topk_indices``, run between the
|
||||
kernels on K1's normed input and qr."""
|
||||
M = x.shape[0]
|
||||
assert self.supports(M), M
|
||||
a = w.attn
|
||||
assert a is not None
|
||||
ratio = a.ratio
|
||||
k1, k2 = self.kernels(M, ratio)
|
||||
index = select is not None
|
||||
assert not index or ratio
|
||||
k1, k2 = self.kernels(M, ratio, fold, index)
|
||||
st = torch.cuda.current_stream()
|
||||
if outs is None:
|
||||
outs = self._outs(M, residual)
|
||||
@@ -324,11 +340,15 @@ class DSV41MonoLayer:
|
||||
token_to_req.data_ptr(),
|
||||
self.kt.data_ptr(),
|
||||
self.klen.data_ptr(),
|
||||
self.normed.data_ptr(),
|
||||
self.qr.data_ptr(),
|
||||
self.scratch(M).data_ptr(),
|
||||
self.epoch.data_ptr(),
|
||||
0,
|
||||
stream=st,
|
||||
)
|
||||
if select is not None:
|
||||
select(self.normed[:M], self.qr[:M])
|
||||
k2(
|
||||
self.q.data_ptr(),
|
||||
swa_cache.data_ptr(),
|
||||
@@ -339,6 +359,13 @@ class DSV41MonoLayer:
|
||||
comp_block,
|
||||
self.kt.data_ptr(),
|
||||
self.klen.data_ptr(),
|
||||
slot_mapping.data_ptr(),
|
||||
swa_indices.data_ptr(),
|
||||
swa_lens.data_ptr(),
|
||||
token_to_req.data_ptr(),
|
||||
topk.data_ptr(),
|
||||
bt.data_ptr(),
|
||||
bt_stride,
|
||||
positions.data_ptr(),
|
||||
a.attn_sink.data_ptr(),
|
||||
self._qk_scale,
|
||||
|
||||
@@ -141,6 +141,8 @@ def slice_loads(c, c0, e):
|
||||
"""Element e's loads of ``stage_slice`` (e clamped into the step's)."""
|
||||
a = c["args"]
|
||||
t, o, col = slice_at(c0, e)
|
||||
if const_expr(not c.get("fold", True)):
|
||||
return (ld_bf(a["res_in"], (t * HC + o) * HIDDEN + col),)
|
||||
r = [ld_bf(a["res_in"], (t * HC + h) * HIDDEN + col) for h in range(HC)]
|
||||
cm = [ld_f32(a["comb_in"], (t * HC + h) * HC + o) for h in range(HC)]
|
||||
return r, cm, ld_f32(a["post_in"], t * HC + o)
|
||||
@@ -150,15 +152,19 @@ def slice_row(c, c0, e, got):
|
||||
"""Element e of ``stage_slice``'s R_new from ``slice_loads``' values."""
|
||||
a = c["args"]
|
||||
t, o, col = slice_at(c0, e)
|
||||
r, cm, post = got
|
||||
# aiter's mhc_fused_post_pre_delayed_rmsnorm: ((post x + c0 r0)
|
||||
# + c1 r1) + c2 r2, each product rounded, then fma(c3, r3, ..)
|
||||
v = post * pend_value(c, t, col, e % COLS) + (r[0] * cm[0])
|
||||
for h in range_constexpr(1, HC - 1):
|
||||
v = v + cm[h] * r[h]
|
||||
# bf16 R_new is what the collapse, the sums of squares and the
|
||||
# mix projection all read
|
||||
v = bf16_round(fma(cm[HC - 1], r[HC - 1], v))
|
||||
if const_expr(not c.get("fold", True)):
|
||||
# a settled residual (no owed post): stream o as it arrives
|
||||
v = got[0]
|
||||
else:
|
||||
r, cm, post = got
|
||||
# aiter's mhc_fused_post_pre_delayed_rmsnorm: ((post x + c0 r0)
|
||||
# + c1 r1) + c2 r2, each product rounded, then fma(c3, r3, ..)
|
||||
v = post * pend_value(c, t, col, e % COLS) + (r[0] * cm[0])
|
||||
for h in range_constexpr(1, HC - 1):
|
||||
v = v + cm[h] * r[h]
|
||||
# bf16 R_new is what the collapse, the sums of squares and the
|
||||
# mix projection all read
|
||||
v = bf16_round(fma(cm[HC - 1], r[HC - 1], v))
|
||||
bo.buffer_store(v.to(fx.BFloat16), rsrc(a["res_out"]), (t * HC + o) * HIDDEN + col)
|
||||
fx.ptr_store(v, c["rl"] + (t * KT + o * COLS + e % COLS))
|
||||
|
||||
|
||||
@@ -1,15 +1,16 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek-V4.1 mono decode layer on ROCm CDNA4 (``VLLM_ROCM_MONO_DECODE=1``).
|
||||
"""DeepSeek-V4.1 mono decode layer on ROCm CDNA4 (``--monokernel``).
|
||||
|
||||
A decode step's backbone layer -- the attention seam, the attention, its TP
|
||||
all-reduce, the FFN seam, the MoE and its all-reduce -- runs as two persistent
|
||||
FlyDSL launches (``mono``, this directory) on the layer's loaded
|
||||
weights, the TP reductions inside the kernels. A layer runs whole when it has
|
||||
the standard seam and no compressor or indexer of its own (compress ratio 1 or
|
||||
2) over ``fp8_ds_mla`` records. Any other backbone layer (the first, the Engram
|
||||
layers, the compressor / indexer layers) keeps vLLM's attention seam and
|
||||
attention and runs the rest as one launch, the FFN launch, from wo_b's
|
||||
weights, the TP reductions inside the kernels. Every backbone layer of the
|
||||
checkpoint runs whole (compress ratio 0, 1 or 2 over ``fp8_ds_mla`` records):
|
||||
the first layer and the Engram layers hand K1 a settled residual, and an index
|
||||
source runs vLLM's compressor and indexer between the two launches. A layer
|
||||
outside that (an MXFP8 KV cache, another indexer) keeps vLLM's attention seam
|
||||
and attention and runs the rest as one launch, the FFN launch, from wo_b's
|
||||
unreduced output. A step takes the kernels when it is decode only, at most
|
||||
``MAX_ROWS`` rows (eager or a FULL CUDA graph: the model is not torch.compiled);
|
||||
a whole layer also needs causal SWA windows. Anything else runs the layer as
|
||||
@@ -20,7 +21,7 @@ Every TP rank must take the same path at every layer (the kernels wait on each
|
||||
other's pushes): the conditions read only metadata every rank holds alike.
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING, cast
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import torch
|
||||
|
||||
@@ -62,7 +63,7 @@ def _mono_runner(device: torch.device):
|
||||
# its peer memory is allocated and exchanged eagerly; vLLM warms
|
||||
# every graph up eagerly before capturing it
|
||||
raise RuntimeError(
|
||||
"VLLM_ROCM_MONO_DECODE was first reached inside a CUDA graph "
|
||||
"--monokernel was first reached inside a CUDA graph "
|
||||
"capture, before an eager decode step."
|
||||
)
|
||||
from vllm.models.deepseek_v41.amd.mono.runner import MAX_TOKENS, DSV41MonoLayer
|
||||
@@ -113,6 +114,35 @@ def _unsupported_moe(layer: "DeepseekV4DecoderLayer", vllm_config) -> str | None
|
||||
return None
|
||||
|
||||
|
||||
def _select(
|
||||
layer: "DeepseekV4DecoderLayer",
|
||||
positions: torch.Tensor,
|
||||
normed: torch.Tensor,
|
||||
qr: torch.Tensor,
|
||||
) -> None:
|
||||
"""An index source's compressor and top-k selection, from K1's normed input
|
||||
and bf16 qr: ``_forward_csa2_full`` / ``_forward_csa2_reindex`` and the
|
||||
indexer op, serial."""
|
||||
attn = layer.attn
|
||||
comp, indexer = attn.compressor, attn.indexer
|
||||
assert indexer is not None
|
||||
if comp is not None:
|
||||
kv_score = torch.mm(
|
||||
normed, comp.fused_wkv_wgate.weight.T, out_dtype=torch.float32
|
||||
)
|
||||
latent = comp(kv_score, positions)
|
||||
comp.insert_cache(latent, positions, attn.rotary_emb)
|
||||
indexer._produce_k(latent, positions, attn.indexer_rotary_emb)
|
||||
weights, _ = indexer.weights_proj(normed)
|
||||
q, q_scale, weights = indexer.forward_q(
|
||||
qr, None, weights, positions, attn.indexer_rotary_emb
|
||||
)
|
||||
assert weights is not None
|
||||
indexer.indexer_op(
|
||||
normed, (q, q_scale) if q_scale is not None else q, None, weights
|
||||
)
|
||||
|
||||
|
||||
class MonoDecodeLayer:
|
||||
"""One decoder layer's mono path (``create``: None when the layer is not
|
||||
eligible): the whole layer (``__call__``) or, with ``ffn_only``, the FFN
|
||||
@@ -127,7 +157,7 @@ class MonoDecodeLayer:
|
||||
def create(
|
||||
layer: "DeepseekV4DecoderLayer", vllm_config
|
||||
) -> "MonoDecodeLayer | None":
|
||||
if not envs.VLLM_ROCM_MONO_DECODE:
|
||||
if not vllm_config.kernel_config.monokernel:
|
||||
return None
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
|
||||
@@ -150,7 +180,7 @@ class MonoDecodeLayer:
|
||||
if cus < BLOCKS:
|
||||
why = f"needs {BLOCKS} compute units, the GPU has {cus}"
|
||||
if why is not None:
|
||||
raise ValueError(f"VLLM_ROCM_MONO_DECODE {why}.")
|
||||
raise ValueError(f"--monokernel {why}.")
|
||||
# the layer: the kernels serve the backbone's seams and MoE
|
||||
config = vllm_config.model_config.hf_config
|
||||
attn, ffn = layer.attn, layer.ffn
|
||||
@@ -174,12 +204,17 @@ class MonoDecodeLayer:
|
||||
if why is not None:
|
||||
logger.info_once("DSv4.1 mono decode off for some layers: %s", why)
|
||||
return None
|
||||
# the whole layer: the kernels' attention takes the standard layers only
|
||||
if layer.engram is not None:
|
||||
why = "Engram"
|
||||
elif attn.compressor is not None or attn.indexer is not None:
|
||||
why = "a compressor / indexer"
|
||||
elif attn.compress_ratio not in (1, 2):
|
||||
# the whole layer: the kernels' attention takes the standard layers; an
|
||||
# index source's compressor and indexer run between its two launches
|
||||
from vllm.models.deepseek_v41.amd.rocm import DeepseekV41RocmMxfp4Indexer
|
||||
|
||||
if attn.compressor is not None and attn.indexer is None:
|
||||
why = "a compressor without an indexer"
|
||||
elif attn.indexer is not None and not isinstance(
|
||||
attn.indexer, DeepseekV41RocmMxfp4Indexer
|
||||
):
|
||||
why = f"a {type(attn.indexer).__name__}"
|
||||
elif attn.compress_ratio not in (0, 1, 2):
|
||||
why = f"compress ratio {attn.compress_ratio}"
|
||||
elif attn.kv_mxfp8 or attn.kv_cache_dtype != "fp8_ds_mla":
|
||||
why = f"a {attn.kv_cache_dtype} KV cache"
|
||||
@@ -297,20 +332,23 @@ class MonoDecodeLayer:
|
||||
post_mix: torch.Tensor | None,
|
||||
res_mix: torch.Tensor | None,
|
||||
pre_mix: torch.Tensor | None,
|
||||
engram_hashes: torch.Tensor | None = None,
|
||||
engram_mask: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, ...] | None:
|
||||
"""The layer's outputs (x, residual, post_mix, res_mix, pre_mix) by the
|
||||
mono kernels, or None when this step takes the original path (always for
|
||||
an ``ffn_only`` layer)."""
|
||||
if self.ffn_only:
|
||||
return None
|
||||
if residual is None or x.dim() != 2:
|
||||
return self._skip("first layer")
|
||||
first = residual is None
|
||||
if first and x.dim() != 2:
|
||||
return self._skip("first layer of a pipeline stage")
|
||||
why, swa, comp = self._decode_step(layer, x.shape[0])
|
||||
attn = layer.attn
|
||||
tensors = (x, residual, post_mix, res_mix, pre_mix)
|
||||
tensors = (x,) if first else (x, residual, post_mix, res_mix, pre_mix)
|
||||
if why is None:
|
||||
# the kernels' attention: causal SWA windows, the compressed cache
|
||||
if comp is None:
|
||||
if comp is None and attn.compress_ratio:
|
||||
why = "layer metadata missing"
|
||||
elif swa.decode_swa_width != SWA_WIDTH:
|
||||
why = f"SWA width {swa.decode_swa_width}"
|
||||
@@ -324,12 +362,37 @@ class MonoDecodeLayer:
|
||||
why = "non-contiguous layer inputs"
|
||||
if why is not None:
|
||||
return self._skip(why)
|
||||
assert swa is not None and comp is not None
|
||||
assert swa is not None
|
||||
runner = _mono_runner(x.device)
|
||||
logger.info_once(
|
||||
"DSv4.1 mono decode layer active (decode steps of <= %d rows)", MAX_ROWS
|
||||
)
|
||||
out = runner.forward(
|
||||
# K1's seam folds the owed post into the residual, except where the
|
||||
# residual arrives settled: the embedding, or after the Engram write
|
||||
fold = True
|
||||
if first:
|
||||
residual = x.unsqueeze(1).expand(-1, layer.hc_mult, -1).contiguous()
|
||||
pre_mix = runner.first_pre_mix(x.shape[0])
|
||||
post_mix = res_mix = pre_mix # unread without the fold
|
||||
fold = False
|
||||
elif layer.engram is not None and engram_hashes is not None:
|
||||
residual = layer.mhc_post(x, residual, post_mix, res_mix)
|
||||
hashes = engram_hashes[:, layer.engram.layer_hash_index]
|
||||
residual = layer.engram(residual, hashes, engram_mask)
|
||||
fold = False
|
||||
kw: dict[str, Any] = {}
|
||||
if attn.indexer is not None:
|
||||
kw["select"] = lambda normed, qr: _select(layer, positions, normed, qr)
|
||||
if attn.compress_ratio:
|
||||
assert comp is not None
|
||||
kw.update(
|
||||
topk_indices=attn.topk_indices_buffer,
|
||||
comp_cache=_record_view(
|
||||
attn._compressed_kv_cache(), comp.block_size // attn.compress_ratio
|
||||
),
|
||||
comp_block_table=comp.block_table,
|
||||
)
|
||||
return runner.forward(
|
||||
self.weights(layer),
|
||||
x,
|
||||
residual,
|
||||
@@ -342,13 +405,9 @@ class MonoDecodeLayer:
|
||||
swa.decode_swa_indices,
|
||||
swa.decode_swa_lens,
|
||||
swa.token_to_req_indices,
|
||||
topk_indices=attn.topk_indices_buffer,
|
||||
comp_cache=_record_view(
|
||||
attn._compressed_kv_cache(), comp.block_size // attn.compress_ratio
|
||||
),
|
||||
comp_block_table=comp.block_table,
|
||||
fold=fold,
|
||||
**kw,
|
||||
)
|
||||
return out
|
||||
|
||||
def ffn(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user