diff --git a/tests/models/test_dsv41_mono_decode.py b/tests/models/test_dsv41_mono_decode.py index 684e72c004..309d278bcb 100644 --- a/tests/models/test_dsv41_mono_decode.py +++ b/tests/models/test_dsv41_mono_decode.py @@ -27,21 +27,25 @@ class _Runner: self.calls.append((args, kwargs)) return ("mono outputs",) + def first_pre_mix(self, rows): + return torch.zeros(rows, 4) + def ffn(self, *args): self.calls.append((args, {})) return ("ffn outputs",) -def _layer(comp="comp"): +def _layer(comp="comp", indexer=None): cache = lambda: torch.zeros(4, 128 * md.RECORD, dtype=torch.uint8) # noqa: E731 attn = SimpleNamespace( swa_cache_layer=SimpleNamespace(prefix="swa", kv_cache=cache()), compressed_cache_prefix=comp, topk_indices_buffer=torch.zeros(64, 512, dtype=torch.int32), compress_ratio=1, + indexer=indexer, _compressed_kv_cache=cache, ) - return SimpleNamespace(attn=attn) + return SimpleNamespace(attn=attn, engram=None, hc_mult=4) def _swa(**overrides): @@ -77,7 +81,12 @@ def run(monkeypatch): ``mode`` and ``metadata``; returns (result, runner calls).""" def _run( - mode=CUDAGraphMode.FULL, metadata=None, ffn_only=False, entry=None, **inputs + mode=CUDAGraphMode.FULL, + metadata=None, + ffn_only=False, + entry=None, + layer=None, + **inputs, ): if metadata is None: metadata = { @@ -97,7 +106,7 @@ def run(monkeypatch): # has no compressed cache part, _ = args.pop("x"), args.pop("positions") return mono.ffn(_layer(comp=None), part, **args), runner.calls - return mono(_layer(), **args), runner.calls + return mono(layer or _layer(), **args), runner.calls return _run @@ -114,6 +123,23 @@ def test_decode_step_runs_mono(run, mode): 1, ) assert kwargs["comp_cache"].shape == (4, 128, md.RECORD) + assert kwargs["fold"] and "select" not in kwargs + + +def test_first_layer_runs_mono_from_the_embedding(run): + """The first layer's residual is the embedding broadcast to the four + streams, which K1 takes as it is (no owed post to fold).""" + out, calls = run(residual=None, post_mix=None, res_mix=None, pre_mix=None) + assert out == ("mono outputs",) and len(calls) == 1 + args, kwargs = calls[0] + assert args[2].shape == (M, 4, 5120) and not kwargs["fold"] + + +def test_index_source_selects_between_the_kernels(run): + """An index source hands the runner its compressor and indexer to run + between K1 and K2.""" + out, calls = run(layer=_layer(indexer=object())) + assert out == ("mono outputs",) and callable(calls[0][1]["select"]) def test_padded_decode_step_runs_mono(run): @@ -149,7 +175,9 @@ def test_padded_decode_step_runs_mono(run): ), dict(metadata={}), # a profile / dummy run: no attention metadata dict(**_inputs(md.MAX_ROWS + 6)), - dict(residual=None), # the first layer's seam broadcasts the embedding + # a later pipeline stage's first layer: its stream arrives without + # the mixes + dict(residual=None, x=torch.zeros(M, 4, 5120, dtype=torch.bfloat16)), ], ids=[ "piecewise", @@ -158,7 +186,7 @@ def test_padded_decode_step_runs_mono(run): "tokens-past-rows", "no-metadata", "rows", - "first-layer", + "stage-input", ], ) def test_other_steps_take_the_original_path(run, case): @@ -202,22 +230,41 @@ def _decoder_layer(engram=None, **attn): ) +def _indexer(): + from vllm.models.deepseek_v41.amd.rocm import DeepseekV41RocmMxfp4Indexer + + return object.__new__(DeepseekV41RocmMxfp4Indexer) + + @pytest.mark.parametrize( "layer, path", [ (_decoder_layer(), "whole"), - (_decoder_layer(compress_ratio=0), "ffn"), - (_decoder_layer(engram=object()), "ffn"), - (_decoder_layer(compressor=object(), indexer=object()), "ffn"), + (_decoder_layer(compress_ratio=0), "whole"), + (_decoder_layer(engram=object()), "whole"), + (_decoder_layer(compressor=object(), indexer=_indexer()), "whole"), + (_decoder_layer(indexer=_indexer()), "whole"), + (_decoder_layer(compressor=object()), "ffn"), (_decoder_layer(indexer=object()), "ffn"), (_decoder_layer(kv_cache_dtype="auto"), "ffn"), (_decoder_layer(layer_id=40), None), ], - ids=["standard", "first", "engram", "kv-source", "index-source", "kv", "draft"], + ids=[ + "standard", + "window", + "engram", + "kv-source", + "index-source", + "compressor-only", + "other-indexer", + "kv", + "draft", + ], ) def test_layers_take_whole_or_ffn_launch(monkeypatch, layer, path): - """The whole-layer kernels serve the standard layers; every other backbone - layer keeps vLLM's attention and runs its FFN half as one launch.""" + """The whole-layer kernels serve every backbone layer of the checkpoint (an + index source's compressor and indexer run between them); a layer outside + them keeps vLLM's attention and runs its FFN half as one launch.""" _deployment(monkeypatch) mono = md.MonoDecodeLayer.create(layer, _config()) got = None if mono is None else "ffn" if mono.ffn_only else "whole" @@ -254,7 +301,7 @@ def test_other_steps_reduce_wo_b_themselves(run, case): assert out is None and not calls -def _config(**parallel): +def _config(monokernel=True, **parallel): parallel = ( dict(enable_expert_parallel=False, enable_eplb=False, data_parallel_size=1) | parallel @@ -262,6 +309,7 @@ def _config(**parallel): return SimpleNamespace( model_config=SimpleNamespace(hf_config=SimpleNamespace(num_hidden_layers=40)), parallel_config=SimpleNamespace(**parallel), + kernel_config=SimpleNamespace(monokernel=monokernel), ) @@ -269,7 +317,6 @@ def _deployment(monkeypatch, cdna=4, cus=256): """A deployment the kernels take (the runner module stubbed: no FlyDSL).""" import vllm.platforms.rocm as rocm - monkeypatch.setenv("VLLM_ROCM_MONO_DECODE", "1") monkeypatch.delenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4", raising=False) monkeypatch.setattr(rocm, "get_cdna_version", lambda: cdna) monkeypatch.setattr(md, "get_tensor_model_parallel_world_size", lambda: 2) @@ -280,6 +327,14 @@ def _deployment(monkeypatch, cdna=4, cus=256): monkeypatch.setitem(sys.modules, name, runner) +def test_off_without_the_flag(monkeypatch): + """No --monokernel: no mono layer, whatever the deployment.""" + _deployment(monkeypatch) + assert ( + md.MonoDecodeLayer.create(_decoder_layer(), _config(monokernel=False)) is None + ) + + @pytest.mark.parametrize("cdna", [3, 5]) def test_opt_in_outside_the_kernels_cdna_raises(monkeypatch, cdna): _deployment(monkeypatch, cdna=cdna) diff --git a/tests/models/test_dsv41_mono_layers.py b/tests/models/test_dsv41_mono_layers.py index 580462a823..a924cfda50 100644 --- a/tests/models/test_dsv41_mono_layers.py +++ b/tests/models/test_dsv41_mono_layers.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """The DeepSeek-V4.1 mono decode path through the real model: a five-layer -DeepSeek-V4.1-Flash at TP2 and TP4 on CDNA4, served with ``VLLM_ROCM_MONO_DECODE`` on +DeepSeek-V4.1-Flash at TP2 and TP4 on CDNA4, served with ``--monokernel`` on and off on the same random weights: every decoder layer's outputs at every decode step must match. Unlike ``test_dsv41_mono_numerics.py``, which checks the kernels against vLLM's ops one stage at a time, this runs @@ -10,12 +10,13 @@ the per-step path choice, the weights taken off the layer, the metadata and the indexer's top-k -- so a change to the decoder layer that the kernels do not follow fails here. -The five layers hold both kinds the mono path takes: -- 0: sliding window only -> the FFN launch -- 1: ratio-2 KV and index source -> the FFN launch -- 2: ratio-2 consumer -> the whole layer (K1 + K2) -- 3: ratio-1 KV, index and candidate source -> the FFN launch -- 4: ratio-1 consumer -> the whole layer +Every layer takes the whole-layer path (K1 + K2); the five hold each variant: +- 0: the first layer, sliding window only (K1 takes the embedding settled) +- 1: ratio-2 KV and index source (vLLM's compressor and indexer between K1 + and K2) +- 2: ratio-2 consumer with an Engram (K1 takes the residual settled) +- 3: ratio-1 KV, index and candidate source +- 4: ratio-1 index source over 3's candidates and KV """ import zlib @@ -34,13 +35,17 @@ TRUNCATED = { # one entry a layer, the three MTP layers' included "compress_ratios": [0, 2, 2, 1, 1, 0, 0, 0], "kv_source_layer_ids": [1, 3], - "index_source_layer_ids": [1, 3], + # 4: an indexer of its own over 3's candidates (a REINDEX layer) + "index_source_layer_ids": [1, 3, 4], "candidate_source_layer_id": 3, - "engram_layer_ids": [], - "engram_num_embeddings": [], + # an Engram on a whole layer (K1 then takes its residual settled), its + # tables shrunk: 24 prime-sized ranges from 1000 + "engram_layer_ids": [2], + "engram_num_embeddings": [32768], + "engram_vocab_size": 1000, "dspark_target_layer_ids": [2, 3, 4], } -WHOLE, FFN = (2, 4), (0, 1, 3) +WHOLE, FFN = (0, 1, 2, 3, 4), () STEPS = 5 # decode steps after the prefill's token @@ -227,7 +232,6 @@ def _decode(vllm_runner, monkeypatch, tp: int, mono: bool, prompts, path: str): from vllm import SamplingParams from vllm.inputs import TokensPrompt - monkeypatch.setenv("VLLM_ROCM_MONO_DECODE", "1" if mono else "0") monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1") monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "1") with vllm_runner( @@ -237,6 +241,7 @@ def _decode(vllm_runner, monkeypatch, tp: int, mono: bool, prompts, path: str): hf_overrides=TRUNCATED, worker_extension_cls=f"{__name__}.MonoProbe", moe_backend="aiter", + monokernel=mono, language_model_only=True, tokenizer_mode="deepseek_v41", attention_config={"indexer_kv_dtype": "mxfp4", "indexer_sparse_logits": True}, @@ -300,15 +305,16 @@ def test_mono_decode_matches_the_model(vllm_runner, monkeypatch, tmp_path, tp): same.add((len(prompt), len(prompt) + step)) # each layer's outputs for those tokens, mono against vLLM's path, token by - # token. Layer 0 sees the same inputs on both paths at a request's first - # decode token: there the bound is test_dsv41_mono_numerics' (about 3x the - # kernels' 0.4%); its later tokens attend over KV each path wrote, 3%. Each - # later layer takes the previous one's slightly different output, which - # random weights amplify (2-4% by layer 4): 8%. A token whose top-6 routing - # is a near-tie on vLLM's path (6th / 7th score within 5e-3: a 0.4% input - # difference moves scores near 0.8 by about 3e-3) may pick another expert; - # once it has, its residual stream differs and the later layers are exempt - # for it. The residual stream: 3% (0.3% seen). A layer the kernels no + # token. Layer 0 sees the same inputs on both paths, but its FFN input + # differs by up to test_dsv41_mono_numerics' 1.2%, which random weights + # amplify through the MoE: 3%. Each later layer takes the previous one's + # slightly different output, which random weights amplify (2-4% by layer + # 4): 8%. A token whose top-6 routing is a near-tie on vLLM's path may pick + # another expert: a 0.4% input difference moves scores near 0.8 by about + # 3e-3, so 6th / 7th scores within 5e-3 at layer 0 and, past it, where the + # input carries the earlier layers' differences (0.4-0.7%), within 1e-2. + # Once it has, its residual stream differs and the later layers are exempt + # for it. The residual stream: 3% (0.5% seen). A layer the kernels no # longer follow is off by far more, on every token. for rank in range(tp): got, _ = torch.load(f"{tmp_path}/on.rank{rank}") @@ -317,8 +323,6 @@ def test_mono_decode_matches_the_model(vllm_runner, monkeypatch, tmp_path, tp): same & {k[1:] for k in got} & {k[1:] for k in want} ) assert len(keys) >= len(prompts), keys - # the bound that sees a 3% error: a request's first decode token - assert any(p == n for n, p in keys), keys flipped: set = set() for i in sorted(WHOLE + FFN): for k in keys: @@ -327,9 +331,10 @@ def test_mono_decode_matches_the_model(vllm_runner, monkeypatch, tmp_path, tp): (x, res), (x_ref, res_ref) = got[(i, *k)], want[(i, *k)] err = _rel(res, res_ref) assert err < 0.03, (rank, i, k, "residual", err) - bound = (0.015 if k[1] == k[0] else 0.03) if i == 0 else 0.08 + bound = 0.03 if i == 0 else 0.08 err = _rel(x, x_ref) if err >= bound: - assert gaps[(i, *k)] < 5e-3, (rank, i, k, "x", err) + tie = 5e-3 if i == 0 else 1e-2 + assert gaps[(i, *k)] < tie, (rank, i, k, "x", err) flipped.add(k) assert len(flipped) <= len(keys) // 4, (rank, sorted(flipped)) diff --git a/vllm/config/kernel.py b/vllm/config/kernel.py index fa3ecf5df9..5c4b877fd7 100644 --- a/vllm/config/kernel.py +++ b/vllm/config/kernel.py @@ -291,6 +291,12 @@ class KernelConfig: enable_jit_warmup: bool = True """If True, run JIT compile warmup during kernel warmup.""" + monokernel: bool = False + """If True, run the eligible layers of decode steps on the model's + MonoKernel: a persistent kernel that runs most or all of one decoder layer + in a single launch (DeepSeek-V4.1 on MI355X). A deployment that cannot run + it is refused at startup.""" + moe_backend: MoEBackend = "auto" """Backend for MoE expert computation kernels. Available options: diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index ea628467e0..86199e5f00 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -758,6 +758,7 @@ class EngineArgs: enable_flashinfer_autotune: bool = get_field( KernelConfig, "enable_flashinfer_autotune" ) + monokernel: bool = get_field(KernelConfig, "monokernel") worker_cls: str = ParallelConfig.worker_cls worker_extension_cls: str = ParallelConfig.worker_extension_cls @@ -1802,6 +1803,7 @@ class EngineArgs: "--enable-flashinfer-autotune", **kernel_kwargs["enable_flashinfer_autotune"], ) + kernel_group.add_argument("--monokernel", **kernel_kwargs["monokernel"]) moe_backend_kwargs = kernel_kwargs["moe_backend"] moe_backend_kwargs["type"] = lambda s: s.lower().replace("-", "_") kernel_group.add_argument("--moe-backend", **moe_backend_kwargs) @@ -2754,6 +2756,8 @@ class EngineArgs: "are mutually exclusive" ) kernel_config.enable_flashinfer_autotune = self.enable_flashinfer_autotune + if self.monokernel: + kernel_config.monokernel = True if self.moe_backend != "auto": kernel_config.moe_backend = self.moe_backend if self.linear_backend != "auto": diff --git a/vllm/envs.py b/vllm/envs.py index a2b4603077..bb1c03a9da 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -156,7 +156,6 @@ if TYPE_CHECKING: VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION: bool = False VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: bool = False VLLM_ROCM_AITER_MEGA_MOE_FUSE_SHARED_EXPERTS: bool = False - VLLM_ROCM_MONO_DECODE: bool = False VLLM_ROCM_USE_AITER_TRITON_GEMM: bool = True VLLM_ROCM_USE_SKINNY_GEMM: bool = True VLLM_ROCM_FP8_PADDING: bool = True @@ -1398,12 +1397,6 @@ environment_variables: dict[str, Callable[[], Any]] = { os.getenv("VLLM_ROCM_AITER_MEGA_MOE_FUSE_SHARED_EXPERTS", "False").lower() in ("true", "1") ), - # Run the eligible layers of decode steps on the model's mono decode - # kernels: persistent launches a layer, the TP all-reduces in-kernel - # (DeepSeek-V4.1 on CDNA4). By default is disabled. - "VLLM_ROCM_MONO_DECODE": lambda: ( - os.getenv("VLLM_ROCM_MONO_DECODE", "False").lower() in ("true", "1") - ), # Whether to use aiter triton kernels for gemm ops. # By default is enabled. "VLLM_ROCM_USE_AITER_TRITON_GEMM": lambda: ( diff --git a/vllm/models/deepseek_v41/amd/model.py b/vllm/models/deepseek_v41/amd/model.py index 5de173a870..4c5f158587 100644 --- a/vllm/models/deepseek_v41/amd/model.py +++ b/vllm/models/deepseek_v41/amd/model.py @@ -282,7 +282,17 @@ class DeepseekV4DecoderLayer(nn.Module): engram_mask: torch.Tensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: if self.mono is not None: - out = self.mono(self, x, positions, residual, post_mix, res_mix, pre_mix) + out = self.mono( + self, + x, + positions, + residual, + post_mix, + res_mix, + pre_mix, + engram_hashes, + engram_mask, + ) if out is not None: return out # Layer 0's attention seam projects the 2-D embedding with the folded diff --git a/vllm/models/deepseek_v41/amd/mono/README.md b/vllm/models/deepseek_v41/amd/mono/README.md index 88e40d73dd..462f7bdecd 100644 --- a/vllm/models/deepseek_v41/amd/mono/README.md +++ b/vllm/models/deepseek_v41/amd/mono/README.md @@ -2,9 +2,10 @@ One fused decoder layer for vLLM's decode steps on MI355X: the attention seam, the attention, the attention all-reduce, the FFN seam, the MoE and the MoE -all-reduce, in two persistent launches a layer. The other backbone layers keep -vLLM's attention and run the rest -- the attention all-reduce, the FFN seam, -the MoE and its all-reduce -- as one launch (the FFN launch). ATOM's V4.1 mono decode +all-reduce, in two persistent launches a layer. A layer outside the whole-layer +kernels keeps vLLM's attention and runs the rest -- the attention all-reduce, +the FFN seam, the MoE and its all-reduce -- as one launch (the FFN launch). +ATOM's V4.1 mono decode (ROCm/ATOM `atom/models/deepseek_v41/mono`) is the reference design: the seam and MoE stages (`stages/`) and the framework under them (`common/`) are adapted from it, each file naming its origin; the attention stages (`attention/`) are @@ -12,13 +13,15 @@ new, reading vLLM's KV records and weights as loaded. ## Scope -- Layers: the whole layer on every backbone layer with the standard seam and no - compressor / indexer: 3-7, 9-13, 15-19 (compress ratio 2) and 21-23, 25-27, - 29-31, 33-35, 37-39 (ratio 1), 30 of 40. The FFN launch on the other 10: - layer 0 (embedding-broadcast seam), layer 1 (Engram at the seam) and the - compressor / indexer layers 2, 8, 14 (also Engram), 20, 24, 28, 32, 36, - whose wo_b leaves its TP reduce to it. The DSpark draft layers keep vLLM's - path. The paths interleave freely (same tensors at the layer boundary). +- Layers: the whole layer on all 40 backbone layers. K1 takes the residual + settled, without folding the previous output in, on layer 0 (the embedding + broadcast to the four streams) and after an Engram write (layers 1 and 14, + vLLM's Engram running before K1). On the index sources (2, 8, 14, 20 with + their compressor; 24, 28, 32, 36) K1 also hands out its normed input and bf16 + qr, vLLM's compressor and indexer run between K1 and K2, and K2 builds the + key table from that top-k. The FFN launch serves a layer outside these (an + MXFP8 KV cache, another indexer); the DSpark draft layers keep vLLM's path. + The paths interleave freely (same tensors at the layer boundary). - Steps: decode only (FULL graph or eager), M <= 48 rows (8 requests x 6 DSpark tokens), TP2 / TP4, `fp8_ds_mla` KV records (ROCm), text tokens (the routing bias is `e_score_correction_bias`). diff --git a/vllm/models/deepseek_v41/amd/mono/attention/back.py b/vllm/models/deepseek_v41/amd/mono/attention/back.py index 4526f8eda7..5cd4793be2 100644 --- a/vllm/models/deepseek_v41/amd/mono/attention/back.py +++ b/vllm/models/deepseek_v41/amd/mono/attention/back.py @@ -151,7 +151,9 @@ def key_rec(c, t, k): table: slot | 1 << 31 for a compressed row, a window slot otherwise, -1 for none (it reads a zero record: its scores are masked, its values zeros).""" a, ratio = c["args"], c["ratio"] - val = ld_i32(a["kt"], t * KEYS + fx.min(fx.max(k, 0), KEYS - 1)) + # a table this launch wrote (``krdy``) is read past the CU's cache + cm = CM_DEV if const_expr("krdy" in c) else 0 + val = ld_i32(a["kt"], t * KEYS + fx.min(fx.max(k, 0), KEYS - 1), cm) ok = val != fx.Int32(-1) slot = ok.select(val & fx.Int32(0x7FFFFFFF), fx.Int32(0)) if const_expr(ratio > 0): @@ -209,9 +211,11 @@ def stage_split(c, unit): g = lane // 16 r16 = lane % 16 k0 = ch * ck + if const_expr("krdy" in c): + await_flags(c, c["krdy"], 1, lambda u: t) # the key count, this lane's key and q: independent loads, out together - # (K1's key table holds -1 past a token's keys) - kv_len = ld_i32(a["klen"], t) + # (the key table holds -1 past a token's keys) + kv_len = ld_i32(a["klen"], t, CM_DEV if const_expr("krdy" in c) else 0) key = k0 + kt * 16 + r16 data, sc, okv = key_rec(c, t, key) qrow = d.heads * HEAD_DIM // 8 diff --git a/vllm/models/deepseek_v41/amd/mono/attention/front.py b/vllm/models/deepseek_v41/amd/mono/attention/front.py index 3ac9e3fe9b..23c009739d 100644 --- a/vllm/models/deepseek_v41/amd/mono/attention/front.py +++ b/vllm/models/deepseek_v41/amd/mono/attention/front.py @@ -188,6 +188,8 @@ def stage_kt(c, t): ntopk.""" tid = c["tid"] a, ratio = c["args"], c["ratio"] + # device scope where this launch reads the table (K2's) + cm = CM_DEV if const_expr(c.get("dev", False)) else 0 slot = ld_i32(a["slot"], 2 * t) live = slot >= 0 nswa = live.select(ld_i32(a["swa_lens"], t), fx.Int32(0)) @@ -212,10 +214,10 @@ def stage_kt(c, t): c_val = (local >= 0).select(c_slot, fx.Int32(-1)) val = (k < ntopk).select(c_val, val) if k < KEYS: - bo.buffer_store(val, rsrc(a["kt"]), t * KEYS + k) + bo.buffer_store(val, rsrc(a["kt"]), t * KEYS + k, cache_modifier=cm) if tid == 0: - bo.buffer_store(kv_len, rsrc(a["klen"]), t) - bo.buffer_store(ntopk, rsrc(a["klen"]), c["S"] + t) + bo.buffer_store(kv_len, rsrc(a["klen"]), t, cache_modifier=cm) + bo.buffer_store(ntopk, rsrc(a["klen"]), c["S"] + t, cache_modifier=cm) # ---------------------------------------------------------------- wqkv_a @@ -357,6 +359,20 @@ def stage_qkv(c, t): amax = fx.max(amax, abs(y)) code = mx_code(butterfly(amax, (1, 2), fx.max)) mul = mx_mul(code) + if const_expr(c.get("index", False)): # noqa: SIM102 + # the bf16 qr, vLLM's indexer's input + if mine & (tt * 8 < Q_RANK): + words = [ + fx.Vector.from_elements([ys[2 * i], ys[2 * i + 1]], fx.Float32) + .to(fx.BFloat16) + .bitcast(fx.Int32)[0] + for i in range(4) + ] + bo.buffer_store( + fx.Vector.from_elements(words, fx.Int32), + rsrc(a["qr_out"]), + t * (Q_RANK // 2) + 4 * tt, + ) if mine & (tt * 8 < Q_RANK): w0 = fp8_pack4(*[clamp_fp8(ys[i] * mul) for i in range(4)]) w1 = fp8_pack4(*[clamp_fp8(ys[4 + i] * mul) for i in range(4)]) @@ -597,8 +613,10 @@ def run_front(c, key, bid, x8_given=False, before_wqkv=None, after_wqkv=None): u = free + n_free * tid if u < xq_units: c["mb"].put(c["xrdy"], u, fx.Int32(1)) - for t in range(is_free.select(free, fx.Int32(s)), s, n_free): - stage_kt(c, t) + if const_expr(not c.get("index", False)): + # an index source's top-k comes after this launch: K2 builds its table + for t in range(is_free.select(free, fx.Int32(s)), s, n_free): + stage_kt(c, t) stamp(c, 1) poss = [wqb_pos(c, t0, n) for t0, n in tiles] diff --git a/vllm/models/deepseek_v41/amd/mono/layer.py b/vllm/models/deepseek_v41/amd/mono/layer.py index c1be9e3428..dcd39ae2b7 100644 --- a/vllm/models/deepseek_v41/amd/mono/layer.py +++ b/vllm/models/deepseek_v41/amd/mono/layer.py @@ -58,6 +58,12 @@ class MonoBuild: tp: int ratio: int # the layer's compress ratio (its attention's key sources) timeline: bool = False + # K1's seam folds the previous layer's output into the residual; off for a + # residual that arrives settled (the first layer, after an Engram) + fold: bool = True + # the layer selects its own top-k (an index source): K1 hands its normed + # input and qr out for vLLM's indexer, K2 builds the key table + index: bool = False def _moe_key(key: MonoBuild) -> MoeBuild: @@ -100,6 +106,7 @@ def scratch_layout(s: int, tp: int) -> dict: lo = min(o for o, _ in mr.values()) place({n: (o - lo, n_) for n, (o, n_) in mr.items()}) place({"normed": (0, s * HIDDEN * 2), "xrdy_moe": (s * HIDDEN * 2 + 256, s * 8)}) + place({"krdy": (0, s * 8)}) return out @@ -175,7 +182,15 @@ def build_mono_k1(key: MonoBuild): seam: SeamLds front: FrontLds # type: ignore[valid-type] - name = kernel_symbol("dsv41_mono_k1", s=s, tp=tp, r=key.ratio, tl=key.timeline) + name = kernel_symbol( + "dsv41_mono_k1", + s=s, + tp=tp, + r=key.ratio, + tl=key.timeline, + f=key.fold, + i=key.index, + ) keyed = key_tuple(key, SOURCES) @flyc.kernel(name=name, known_block_size=[THREADS, 1, 1]) @@ -215,6 +230,8 @@ def build_mono_k1(key: MonoBuild): t2r: Int64, kt: Int64, klen: Int64, + normed_out: Int64, + qr_out: Int64, scratch: Int64, epoch: Int64, tl: Int64, @@ -250,9 +267,11 @@ def build_mono_k1(key: MonoBuild): "t2r": t2r, "kt": kt, "klen": klen, + "qr_out": qr_out, } flayout = {n: layout[n] for n in front_scratch(s, Dims(tp))} c = mb_front.front_context(fkey, lds.front.peek(), fargs, scratch, tag, flayout) + c["index"] = key.index amb = Mailbox(tag) tid = fx.thread_idx.x ca = { @@ -284,6 +303,7 @@ def build_mono_k1(key: MonoBuild): "pre_out": pre_out, }, "d": StageDims(tp), + "fold": key.fold, } for region in ("lin", "pmix"): ca[region] = sreg(scratch, layout[region][0], region) @@ -295,9 +315,14 @@ def build_mono_k1(key: MonoBuild): seam.stage_slice(ca, task) gpu.barrier() + def store(*v): + store_x8(c, *v) + if const_expr(key.index): + store_normed(normed_out, *v) + def seam_rest(): for t in range(first_task(bid, NORM0), s, BLOCKS): - seam.stage_norm(ca, t, lambda *v: store_x8(c, *v)) + seam.stage_norm(ca, t, store) publish_x8(c, t) for t in range(first_task(bid, GATE0), s, BLOCKS): seam.stage_gate(ca, t) @@ -344,6 +369,8 @@ def build_mono_k1(key: MonoBuild): t2r: Int64, kt: Int64, klen: Int64, + normed_out: Int64, + qr_out: Int64, scratch: Int64, epoch: Int64, tl: Int64, @@ -386,6 +413,8 @@ def build_mono_k1(key: MonoBuild): t2r, kt, klen, + normed_out, + qr_out, scratch, epoch, tl, @@ -544,7 +573,9 @@ def build_mono_k2(key: MonoBuild): seam: SeamLds # type: ignore[valid-type] moe: MoeLds # type: ignore[valid-type] - name = kernel_symbol("dsv41_mono_k2", s=s, tp=tp, r=key.ratio, tl=key.timeline) + name = kernel_symbol( + "dsv41_mono_k2", s=s, tp=tp, r=key.ratio, tl=key.timeline, i=key.index + ) keyed = key_tuple(key, SOURCES) @flyc.kernel(name=name, known_block_size=[THREADS, 1, 1]) @@ -558,6 +589,13 @@ def build_mono_k2(key: MonoBuild): comp_block: Int32, kt: Int64, klen: Int64, + slot: Int64, + swa_idx: Int64, + swa_lens: Int64, + t2r: Int64, + topk: Int64, + comp_bt: Int64, + bt_stride: Int32, pos: Int64, sink: Int64, qk_scale: Int32, @@ -642,6 +680,32 @@ def build_mono_k2(key: MonoBuild): if const_expr(key.timeline): c["tl"], c["tl_points"] = tl, mb_back.BACK_POINTS mb_back.epoch_begin(c, epoch, ep) + if const_expr(key.index): + # the key table, from the top-k vLLM's indexer selected after K1, + # on the last CTAs: each split task waits on its token's + ckt = { + "S": s, + "tid": c["tid"], + "dev": True, + "ratio": key.ratio, + "args": { + "slot": slot, + "swa_idx": swa_idx, + "swa_lens": swa_lens, + "pos": pos, + "t2r": t2r, + "topk": topk, + "comp_block": comp_block, + "comp_bt": comp_bt, + "bt_stride": bt_stride, + "kt": kt, + "klen": klen, + }, + } + c["krdy"] = sreg(scratch, layout["krdy"][0], "krdy") + for t in range(first_task(bid, BLOCKS - s), s, BLOCKS): + mb_front.stage_kt(ckt, t) + mb_back.publish(c, c["krdy"], t) mb_back.run_back(c, bkey, bid) gpu.barrier() @@ -685,6 +749,13 @@ def build_mono_k2(key: MonoBuild): comp_block: Int32, kt: Int64, klen: Int64, + slot: Int64, + swa_idx: Int64, + swa_lens: Int64, + t2r: Int64, + topk: Int64, + comp_bt: Int64, + bt_stride: Int32, pos: Int64, sink: Int64, qk_scale: Int32, @@ -736,6 +807,13 @@ def build_mono_k2(key: MonoBuild): comp_block, kt, klen, + slot, + swa_idx, + swa_lens, + t2r, + topk, + comp_bt, + bt_stride, pos, sink, qk_scale, diff --git a/vllm/models/deepseek_v41/amd/mono/runner.py b/vllm/models/deepseek_v41/amd/mono/runner.py index eb37990275..ef9a1978e4 100644 --- a/vllm/models/deepseek_v41/amd/mono/runner.py +++ b/vllm/models/deepseek_v41/amd/mono/runner.py @@ -16,12 +16,13 @@ the same launches, so the ranks' epochs agree. from __future__ import annotations import os +from collections.abc import Callable from dataclasses import dataclass from pathlib import Path import torch -from .attention.plan import HEAD_DIM, HIDDEN, KEYS, Dims +from .attention.plan import HEAD_DIM, HIDDEN, KEYS, Q_RANK, Dims from .common.plan import BLOCKS from .layer import ( EPOCH_WORDS, @@ -170,6 +171,12 @@ class DSV41MonoLayer: self.klen = torch.zeros(2 * MAX_TOKENS, dtype=torch.int32, device=dev) self._dummy = torch.zeros(4, dtype=torch.int32, device=dev) self._zero_rec = torch.zeros(1024, dtype=torch.uint8, device=dev) + # the first layer's incoming pre-mix: stream 0 of the broadcast embedding + self._first_pre = torch.zeros(MAX_TOKENS, HC, dtype=torch.float32, device=dev) + self._first_pre[:, 0] = 1.0 + # an index source's K1 outputs, vLLM's indexer's inputs + self.normed = torch.zeros(MAX_TOKENS, HIDDEN, dtype=torch.bfloat16, device=dev) + self.qr = torch.zeros(MAX_TOKENS, Q_RANK, dtype=torch.bfloat16, device=dev) # the attention seam's outputs, K1 -> K2 self.res_mid = torch.zeros( MAX_TOKENS, HC, HIDDEN, dtype=torch.bfloat16, device=dev @@ -203,14 +210,17 @@ class DSV41MonoLayer: self._scratch[tokens] = buf return buf + def first_pre_mix(self, tokens: int) -> torch.Tensor: + return self._first_pre[:tokens] + @staticmethod def supports(tokens: int) -> bool: return 1 <= tokens <= MAX_TOKENS - def kernels(self, tokens: int, ratio: int): - key = (tokens, ratio) + def kernels(self, tokens: int, ratio: int, fold: bool = True, index: bool = False): + key = (tokens, ratio, fold, index) if key not in self._kernels: - b = MonoBuild(tokens, self.tp, ratio) + b = MonoBuild(tokens, self.tp, ratio, fold=fold, index=index) self._kernels[key] = (build_mono_k1(b), build_mono_k2(b)) return self._kernels[key] @@ -249,15 +259,21 @@ class DSV41MonoLayer: comp_cache: torch.Tensor | None = None, comp_block_table: torch.Tensor | None = None, outs: tuple | None = None, + fold: bool = True, + select: Callable[[torch.Tensor, torch.Tensor], None] | None = None, ): """-> (out, residual, post_mix, res_mix, pre_mix): the layer's outputs - (``outs``: buffers to write them into).""" + (``outs``: buffers to write them into). ``select(normed, qr)``: an index + source's top-k selection into ``topk_indices``, run between the + kernels on K1's normed input and qr.""" M = x.shape[0] assert self.supports(M), M a = w.attn assert a is not None ratio = a.ratio - k1, k2 = self.kernels(M, ratio) + index = select is not None + assert not index or ratio + k1, k2 = self.kernels(M, ratio, fold, index) st = torch.cuda.current_stream() if outs is None: outs = self._outs(M, residual) @@ -324,11 +340,15 @@ class DSV41MonoLayer: token_to_req.data_ptr(), self.kt.data_ptr(), self.klen.data_ptr(), + self.normed.data_ptr(), + self.qr.data_ptr(), self.scratch(M).data_ptr(), self.epoch.data_ptr(), 0, stream=st, ) + if select is not None: + select(self.normed[:M], self.qr[:M]) k2( self.q.data_ptr(), swa_cache.data_ptr(), @@ -339,6 +359,13 @@ class DSV41MonoLayer: comp_block, self.kt.data_ptr(), self.klen.data_ptr(), + slot_mapping.data_ptr(), + swa_indices.data_ptr(), + swa_lens.data_ptr(), + token_to_req.data_ptr(), + topk.data_ptr(), + bt.data_ptr(), + bt_stride, positions.data_ptr(), a.attn_sink.data_ptr(), self._qk_scale, diff --git a/vllm/models/deepseek_v41/amd/mono/stages/seam.py b/vllm/models/deepseek_v41/amd/mono/stages/seam.py index 82abee9f46..ea73e2ef54 100644 --- a/vllm/models/deepseek_v41/amd/mono/stages/seam.py +++ b/vllm/models/deepseek_v41/amd/mono/stages/seam.py @@ -141,6 +141,8 @@ def slice_loads(c, c0, e): """Element e's loads of ``stage_slice`` (e clamped into the step's).""" a = c["args"] t, o, col = slice_at(c0, e) + if const_expr(not c.get("fold", True)): + return (ld_bf(a["res_in"], (t * HC + o) * HIDDEN + col),) r = [ld_bf(a["res_in"], (t * HC + h) * HIDDEN + col) for h in range(HC)] cm = [ld_f32(a["comb_in"], (t * HC + h) * HC + o) for h in range(HC)] return r, cm, ld_f32(a["post_in"], t * HC + o) @@ -150,15 +152,19 @@ def slice_row(c, c0, e, got): """Element e of ``stage_slice``'s R_new from ``slice_loads``' values.""" a = c["args"] t, o, col = slice_at(c0, e) - r, cm, post = got - # aiter's mhc_fused_post_pre_delayed_rmsnorm: ((post x + c0 r0) - # + c1 r1) + c2 r2, each product rounded, then fma(c3, r3, ..) - v = post * pend_value(c, t, col, e % COLS) + (r[0] * cm[0]) - for h in range_constexpr(1, HC - 1): - v = v + cm[h] * r[h] - # bf16 R_new is what the collapse, the sums of squares and the - # mix projection all read - v = bf16_round(fma(cm[HC - 1], r[HC - 1], v)) + if const_expr(not c.get("fold", True)): + # a settled residual (no owed post): stream o as it arrives + v = got[0] + else: + r, cm, post = got + # aiter's mhc_fused_post_pre_delayed_rmsnorm: ((post x + c0 r0) + # + c1 r1) + c2 r2, each product rounded, then fma(c3, r3, ..) + v = post * pend_value(c, t, col, e % COLS) + (r[0] * cm[0]) + for h in range_constexpr(1, HC - 1): + v = v + cm[h] * r[h] + # bf16 R_new is what the collapse, the sums of squares and the + # mix projection all read + v = bf16_round(fma(cm[HC - 1], r[HC - 1], v)) bo.buffer_store(v.to(fx.BFloat16), rsrc(a["res_out"]), (t * HC + o) * HIDDEN + col) fx.ptr_store(v, c["rl"] + (t * KT + o * COLS + e % COLS)) diff --git a/vllm/models/deepseek_v41/amd/mono_decode.py b/vllm/models/deepseek_v41/amd/mono_decode.py index b404a42de3..ddeb6b2631 100644 --- a/vllm/models/deepseek_v41/amd/mono_decode.py +++ b/vllm/models/deepseek_v41/amd/mono_decode.py @@ -1,15 +1,16 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""DeepSeek-V4.1 mono decode layer on ROCm CDNA4 (``VLLM_ROCM_MONO_DECODE=1``). +"""DeepSeek-V4.1 mono decode layer on ROCm CDNA4 (``--monokernel``). A decode step's backbone layer -- the attention seam, the attention, its TP all-reduce, the FFN seam, the MoE and its all-reduce -- runs as two persistent FlyDSL launches (``mono``, this directory) on the layer's loaded -weights, the TP reductions inside the kernels. A layer runs whole when it has -the standard seam and no compressor or indexer of its own (compress ratio 1 or -2) over ``fp8_ds_mla`` records. Any other backbone layer (the first, the Engram -layers, the compressor / indexer layers) keeps vLLM's attention seam and -attention and runs the rest as one launch, the FFN launch, from wo_b's +weights, the TP reductions inside the kernels. Every backbone layer of the +checkpoint runs whole (compress ratio 0, 1 or 2 over ``fp8_ds_mla`` records): +the first layer and the Engram layers hand K1 a settled residual, and an index +source runs vLLM's compressor and indexer between the two launches. A layer +outside that (an MXFP8 KV cache, another indexer) keeps vLLM's attention seam +and attention and runs the rest as one launch, the FFN launch, from wo_b's unreduced output. A step takes the kernels when it is decode only, at most ``MAX_ROWS`` rows (eager or a FULL CUDA graph: the model is not torch.compiled); a whole layer also needs causal SWA windows. Anything else runs the layer as @@ -20,7 +21,7 @@ Every TP rank must take the same path at every layer (the kernels wait on each other's pushes): the conditions read only metadata every rank holds alike. """ -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING, Any, cast import torch @@ -62,7 +63,7 @@ def _mono_runner(device: torch.device): # its peer memory is allocated and exchanged eagerly; vLLM warms # every graph up eagerly before capturing it raise RuntimeError( - "VLLM_ROCM_MONO_DECODE was first reached inside a CUDA graph " + "--monokernel was first reached inside a CUDA graph " "capture, before an eager decode step." ) from vllm.models.deepseek_v41.amd.mono.runner import MAX_TOKENS, DSV41MonoLayer @@ -113,6 +114,35 @@ def _unsupported_moe(layer: "DeepseekV4DecoderLayer", vllm_config) -> str | None return None +def _select( + layer: "DeepseekV4DecoderLayer", + positions: torch.Tensor, + normed: torch.Tensor, + qr: torch.Tensor, +) -> None: + """An index source's compressor and top-k selection, from K1's normed input + and bf16 qr: ``_forward_csa2_full`` / ``_forward_csa2_reindex`` and the + indexer op, serial.""" + attn = layer.attn + comp, indexer = attn.compressor, attn.indexer + assert indexer is not None + if comp is not None: + kv_score = torch.mm( + normed, comp.fused_wkv_wgate.weight.T, out_dtype=torch.float32 + ) + latent = comp(kv_score, positions) + comp.insert_cache(latent, positions, attn.rotary_emb) + indexer._produce_k(latent, positions, attn.indexer_rotary_emb) + weights, _ = indexer.weights_proj(normed) + q, q_scale, weights = indexer.forward_q( + qr, None, weights, positions, attn.indexer_rotary_emb + ) + assert weights is not None + indexer.indexer_op( + normed, (q, q_scale) if q_scale is not None else q, None, weights + ) + + class MonoDecodeLayer: """One decoder layer's mono path (``create``: None when the layer is not eligible): the whole layer (``__call__``) or, with ``ffn_only``, the FFN @@ -127,7 +157,7 @@ class MonoDecodeLayer: def create( layer: "DeepseekV4DecoderLayer", vllm_config ) -> "MonoDecodeLayer | None": - if not envs.VLLM_ROCM_MONO_DECODE: + if not vllm_config.kernel_config.monokernel: return None from vllm.platforms.rocm import get_cdna_version @@ -150,7 +180,7 @@ class MonoDecodeLayer: if cus < BLOCKS: why = f"needs {BLOCKS} compute units, the GPU has {cus}" if why is not None: - raise ValueError(f"VLLM_ROCM_MONO_DECODE {why}.") + raise ValueError(f"--monokernel {why}.") # the layer: the kernels serve the backbone's seams and MoE config = vllm_config.model_config.hf_config attn, ffn = layer.attn, layer.ffn @@ -174,12 +204,17 @@ class MonoDecodeLayer: if why is not None: logger.info_once("DSv4.1 mono decode off for some layers: %s", why) return None - # the whole layer: the kernels' attention takes the standard layers only - if layer.engram is not None: - why = "Engram" - elif attn.compressor is not None or attn.indexer is not None: - why = "a compressor / indexer" - elif attn.compress_ratio not in (1, 2): + # the whole layer: the kernels' attention takes the standard layers; an + # index source's compressor and indexer run between its two launches + from vllm.models.deepseek_v41.amd.rocm import DeepseekV41RocmMxfp4Indexer + + if attn.compressor is not None and attn.indexer is None: + why = "a compressor without an indexer" + elif attn.indexer is not None and not isinstance( + attn.indexer, DeepseekV41RocmMxfp4Indexer + ): + why = f"a {type(attn.indexer).__name__}" + elif attn.compress_ratio not in (0, 1, 2): why = f"compress ratio {attn.compress_ratio}" elif attn.kv_mxfp8 or attn.kv_cache_dtype != "fp8_ds_mla": why = f"a {attn.kv_cache_dtype} KV cache" @@ -297,20 +332,23 @@ class MonoDecodeLayer: post_mix: torch.Tensor | None, res_mix: torch.Tensor | None, pre_mix: torch.Tensor | None, + engram_hashes: torch.Tensor | None = None, + engram_mask: torch.Tensor | None = None, ) -> tuple[torch.Tensor, ...] | None: """The layer's outputs (x, residual, post_mix, res_mix, pre_mix) by the mono kernels, or None when this step takes the original path (always for an ``ffn_only`` layer).""" if self.ffn_only: return None - if residual is None or x.dim() != 2: - return self._skip("first layer") + first = residual is None + if first and x.dim() != 2: + return self._skip("first layer of a pipeline stage") why, swa, comp = self._decode_step(layer, x.shape[0]) attn = layer.attn - tensors = (x, residual, post_mix, res_mix, pre_mix) + tensors = (x,) if first else (x, residual, post_mix, res_mix, pre_mix) if why is None: # the kernels' attention: causal SWA windows, the compressed cache - if comp is None: + if comp is None and attn.compress_ratio: why = "layer metadata missing" elif swa.decode_swa_width != SWA_WIDTH: why = f"SWA width {swa.decode_swa_width}" @@ -324,12 +362,37 @@ class MonoDecodeLayer: why = "non-contiguous layer inputs" if why is not None: return self._skip(why) - assert swa is not None and comp is not None + assert swa is not None runner = _mono_runner(x.device) logger.info_once( "DSv4.1 mono decode layer active (decode steps of <= %d rows)", MAX_ROWS ) - out = runner.forward( + # K1's seam folds the owed post into the residual, except where the + # residual arrives settled: the embedding, or after the Engram write + fold = True + if first: + residual = x.unsqueeze(1).expand(-1, layer.hc_mult, -1).contiguous() + pre_mix = runner.first_pre_mix(x.shape[0]) + post_mix = res_mix = pre_mix # unread without the fold + fold = False + elif layer.engram is not None and engram_hashes is not None: + residual = layer.mhc_post(x, residual, post_mix, res_mix) + hashes = engram_hashes[:, layer.engram.layer_hash_index] + residual = layer.engram(residual, hashes, engram_mask) + fold = False + kw: dict[str, Any] = {} + if attn.indexer is not None: + kw["select"] = lambda normed, qr: _select(layer, positions, normed, qr) + if attn.compress_ratio: + assert comp is not None + kw.update( + topk_indices=attn.topk_indices_buffer, + comp_cache=_record_view( + attn._compressed_kv_cache(), comp.block_size // attn.compress_ratio + ), + comp_block_table=comp.block_table, + ) + return runner.forward( self.weights(layer), x, residual, @@ -342,13 +405,9 @@ class MonoDecodeLayer: swa.decode_swa_indices, swa.decode_swa_lens, swa.token_to_req_indices, - topk_indices=attn.topk_indices_buffer, - comp_cache=_record_view( - attn._compressed_kv_cache(), comp.block_size // attn.compress_ratio - ), - comp_block_table=comp.block_table, + fold=fold, + **kw, ) - return out def ffn( self,