From b3079e6e46a21109a8641c6d48641453a9c7cb86 Mon Sep 17 00:00:00 2001 From: leo <92263211+liuyao0322@users.noreply.github.com> Date: Thu, 17 Sep 2026 15:04:53 +0800 Subject: [PATCH] [Perf][DSpark] Stack DeepSeek V4 context WKV projections (#54674) Signed-off-by: liuyao0322 Co-authored-by: OpenAI Codex --- tests/models/test_dspark_mla.py | 73 ++++++++++++++++++++++++ vllm/models/deepseek_v4/nvidia/dspark.py | 52 +++++++++++++++-- 2 files changed, 119 insertions(+), 6 deletions(-) diff --git a/tests/models/test_dspark_mla.py b/tests/models/test_dspark_mla.py index d30eb9e7c9..3e70450f2c 100644 --- a/tests/models/test_dspark_mla.py +++ b/tests/models/test_dspark_mla.py @@ -11,6 +11,7 @@ from vllm.compilation.wrapper import TorchCompileWithNoGuardsWrapper from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.models.qwen3_dspark import DSparkMarkovHead from vllm.model_executor.models.registry import ModelRegistry +from vllm.models.deepseek_v4.nvidia import dspark as dsv4_dspark from vllm.models.kimi_k3.nvidia import dspark_mla from vllm.models.kimi_k3.nvidia.dspark_mla import K3DSparkForCausalLM, K3DSparkModel @@ -262,3 +263,75 @@ def test_v41_dspark_loads_linear_scales( assert loaded == {runtime_name} assert shards == [() if shard_id is None else (shard_id,)] torch.testing.assert_close(param, checkpoint_scale) + + +def test_dsv4_context_wkv_weights_are_duplicated_by_draft_layer(): + weights = [ + ("mtp.0.attn.wkv.weight", torch.arange(4)), + ("mtp.1.attn.wq_a.weight", torch.arange(3)), + ("mtp.2.attn.wkv.scale", torch.tensor(0.5)), + ("mtp.3.attn.wkv.weight", torch.arange(2)), + ] + + duplicated = list(dsv4_dspark._duplicate_context_wkv_weights(weights, 3)) + + assert [name for name, _ in duplicated] == [ + "mtp.0.attn.wkv.weight", + "context_wkv_proj.weight", + "mtp.1.attn.wq_a.weight", + "mtp.2.attn.wkv.scale", + "context_wkv_proj.scale", + "mtp.3.attn.wkv.weight", + ] + assert duplicated[1][1].shard_id == 0 + assert duplicated[4][1].shard_id == 2 + assert duplicated[0][1].data_ptr() == duplicated[1][1].data_ptr() + assert duplicated[3][1].data_ptr() == duplicated[4][1].data_ptr() + + +def test_dsv4_context_kv_uses_one_stacked_wkv_projection(monkeypatch): + calls = [] + stacked_output = torch.arange(24, dtype=torch.float32).view(2, 12) + + class StackedProjection: + def __init__(self): + self.calls = 0 + + def __call__(self, main_x): + self.calls += 1 + assert main_x.shape == (2, 5) + return stacked_output + + projection = StackedProjection() + layers = [ + SimpleNamespace(attn=SimpleNamespace(kv_norm=lambda kv, offset=i: kv + offset)) + for i in range(3) + ] + model = SimpleNamespace( + config=SimpleNamespace(head_dim=4), + context_wkv_proj=projection, + layers=layers, + num_dspark_layers=3, + ) + slot_mappings = [torch.tensor([0, 1]), None, torch.tensor([4, 5])] + monkeypatch.setattr( + dsv4_dspark, + "_insert_context_kv", + lambda attn, kv, positions, slots: calls.append( + (attn, kv.clone(), positions, slots) + ), + ) + + dsv4_dspark.DSparkDeepseekV4Model.precompute_and_store_context_kv( + model, + torch.zeros(2, 5), + torch.tensor([7, 8]), + slot_mappings, + ) + + assert projection.calls == 1 + assert len(calls) == 2 + assert torch.equal(calls[0][1], stacked_output.view(2, 3, 4)[:, 0]) + assert torch.equal(calls[1][1], stacked_output.view(2, 3, 4)[:, 2] + 2) + assert calls[0][3] is slot_mappings[0] + assert calls[1][3] is slot_mappings[2] diff --git a/vllm/models/deepseek_v4/nvidia/dspark.py b/vllm/models/deepseek_v4/nvidia/dspark.py index 1a3c2c780d..52783889d6 100644 --- a/vllm/models/deepseek_v4/nvidia/dspark.py +++ b/vllm/models/deepseek_v4/nvidia/dspark.py @@ -31,7 +31,10 @@ from vllm.model_executor.layers.fused_moe import ( fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.layernorm import RMSNorm -from vllm.model_executor.layers.linear import ReplicatedLinear +from vllm.model_executor.layers.linear import ( + MergedColumnParallelLinear, + ReplicatedLinear, +) from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, @@ -61,6 +64,24 @@ logger = init_logger(__name__) # MoE expert scale suffix differs by expert dtype (mirrors deepseek_v4 loaders): # fp4 experts register ``.weight_scale``; block-fp8 experts ``.weight_scale_inv``. _EXPERT_SCALE_RE = re.compile(r"\.experts\.\d+\.w[123]\.scale$") +_CONTEXT_WKV_RE = re.compile(r"^mtp\.(\d+)\.attn\.wkv\.(.+)$") + + +def _duplicate_context_wkv_weights( + weights: Iterable[tuple[str, torch.Tensor]], num_layers: int +) -> Iterable[tuple[str, torch.Tensor]]: + """Load every draft layer's WKV into the cross-layer projection.""" + for name, weight in weights: + yield name, weight + match = _CONTEXT_WKV_RE.fullmatch(name) + if match is None: + continue + layer_idx = int(match.group(1)) + if layer_idx >= num_layers: + continue + stacked_weight = weight.detach() + stacked_weight.shard_id = layer_idx + yield f"context_wkv_proj.{match.group(2)}", stacked_weight class DSparkDeepseekV4Model(nn.Module): @@ -113,6 +134,15 @@ class DSparkDeepseekV4Model(nn.Module): for i in range(self.num_dspark_layers) ] ) + self.context_wkv_proj = MergedColumnParallelLinear( + config.hidden_size, + [config.head_dim] * self.num_dspark_layers, + bias=False, + return_bias=False, + quant_config=vllm_config.quant_config, + prefix=maybe_prefix(prefix, "context_wkv_proj"), + disable_tp=True, + ) # Heads: final norm + hc_head, and the Markov + confidence heads # Loaded from the "final" MTP layer weights (mtp.*) in the target checkpoint @@ -173,15 +203,16 @@ class DSparkDeepseekV4Model(nn.Module): place draft layers in different groups). ``None`` (or a ``None`` entry) runs the projection to reserve workspace but writes nothing (profiling). """ - for i, layer in enumerate(self.layers): + all_kv = self.context_wkv_proj(main_x).view( + main_x.shape[0], self.num_dspark_layers, self.config.head_dim + ) + for i, (layer, kv) in enumerate( + zip(self.layers, all_kv.unbind(1), strict=True) + ): slot_mapping = ( None if context_slot_mappings is None else context_slot_mappings[i] ) attn = layer.attn - # Optimized DSV4 MLA path: wkv part of the fused wq_a|wkv projection - # (q_lora part discarded), then RoPE/quant/insert via the fused op. - qr_kv, _ = attn.fused_wqa_wkv(main_x) - kv = qr_kv[..., attn.q_lora_rank :] kv = attn.kv_norm(kv) if slot_mapping is None: continue @@ -429,6 +460,7 @@ class DSparkDeepseekV4ForCausalLM(nn.Module): head_start = n_local_head * tp_rank head_end = n_local_head * (tp_rank + 1) + weights = _duplicate_context_wkv_weights(weights, len(self.model.layers)) for name, loaded_weight in weights: mapped = self._remap_dspark_name(name) if mapped is None: @@ -452,6 +484,12 @@ class DSparkDeepseekV4ForCausalLM(nn.Module): self.quant_config, name, loaded_weight ) + if name.startswith("model.context_wkv_proj."): + param = params_dict[name] + param.weight_loader(param, loaded_weight, loaded_weight.shard_id) + loaded_params.add(name) + continue + # E8M0 expert scales: keep raw exponent bytes. if ".experts." in name: if ( @@ -522,6 +560,8 @@ class DSparkDeepseekV4ForCausalLM(nn.Module): Returns None for non-mtp weights (owned by the target model). """ + if name.startswith("context_wkv_proj."): + return f"model.{name}" m = re.match(r"mtp\.(\d+)\.(.*)", name) if m is None: return None