[Perf][DSpark] Stack DeepSeek V4 context WKV projections (#54674)

Signed-off-by: liuyao0322 <yaoliu548926@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
leo
2026-09-17 00:04:53 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent 7bbce752b8
commit b3079e6e46
2 changed files with 119 additions and 6 deletions
+73
View File
@@ -11,6 +11,7 @@ from vllm.compilation.wrapper import TorchCompileWithNoGuardsWrapper
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.models.qwen3_dspark import DSparkMarkovHead
from vllm.model_executor.models.registry import ModelRegistry
from vllm.models.deepseek_v4.nvidia import dspark as dsv4_dspark
from vllm.models.kimi_k3.nvidia import dspark_mla
from vllm.models.kimi_k3.nvidia.dspark_mla import K3DSparkForCausalLM, K3DSparkModel
@@ -262,3 +263,75 @@ def test_v41_dspark_loads_linear_scales(
assert loaded == {runtime_name}
assert shards == [() if shard_id is None else (shard_id,)]
torch.testing.assert_close(param, checkpoint_scale)
def test_dsv4_context_wkv_weights_are_duplicated_by_draft_layer():
weights = [
("mtp.0.attn.wkv.weight", torch.arange(4)),
("mtp.1.attn.wq_a.weight", torch.arange(3)),
("mtp.2.attn.wkv.scale", torch.tensor(0.5)),
("mtp.3.attn.wkv.weight", torch.arange(2)),
]
duplicated = list(dsv4_dspark._duplicate_context_wkv_weights(weights, 3))
assert [name for name, _ in duplicated] == [
"mtp.0.attn.wkv.weight",
"context_wkv_proj.weight",
"mtp.1.attn.wq_a.weight",
"mtp.2.attn.wkv.scale",
"context_wkv_proj.scale",
"mtp.3.attn.wkv.weight",
]
assert duplicated[1][1].shard_id == 0
assert duplicated[4][1].shard_id == 2
assert duplicated[0][1].data_ptr() == duplicated[1][1].data_ptr()
assert duplicated[3][1].data_ptr() == duplicated[4][1].data_ptr()
def test_dsv4_context_kv_uses_one_stacked_wkv_projection(monkeypatch):
calls = []
stacked_output = torch.arange(24, dtype=torch.float32).view(2, 12)
class StackedProjection:
def __init__(self):
self.calls = 0
def __call__(self, main_x):
self.calls += 1
assert main_x.shape == (2, 5)
return stacked_output
projection = StackedProjection()
layers = [
SimpleNamespace(attn=SimpleNamespace(kv_norm=lambda kv, offset=i: kv + offset))
for i in range(3)
]
model = SimpleNamespace(
config=SimpleNamespace(head_dim=4),
context_wkv_proj=projection,
layers=layers,
num_dspark_layers=3,
)
slot_mappings = [torch.tensor([0, 1]), None, torch.tensor([4, 5])]
monkeypatch.setattr(
dsv4_dspark,
"_insert_context_kv",
lambda attn, kv, positions, slots: calls.append(
(attn, kv.clone(), positions, slots)
),
)
dsv4_dspark.DSparkDeepseekV4Model.precompute_and_store_context_kv(
model,
torch.zeros(2, 5),
torch.tensor([7, 8]),
slot_mappings,
)
assert projection.calls == 1
assert len(calls) == 2
assert torch.equal(calls[0][1], stacked_output.view(2, 3, 4)[:, 0])
assert torch.equal(calls[1][1], stacked_output.view(2, 3, 4)[:, 2] + 2)
assert calls[0][3] is slot_mappings[0]
assert calls[1][3] is slot_mappings[2]
+46 -6
View File
@@ -31,7 +31,10 @@ from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
ReplicatedLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
@@ -61,6 +64,24 @@ logger = init_logger(__name__)
# MoE expert scale suffix differs by expert dtype (mirrors deepseek_v4 loaders):
# fp4 experts register ``.weight_scale``; block-fp8 experts ``.weight_scale_inv``.
_EXPERT_SCALE_RE = re.compile(r"\.experts\.\d+\.w[123]\.scale$")
_CONTEXT_WKV_RE = re.compile(r"^mtp\.(\d+)\.attn\.wkv\.(.+)$")
def _duplicate_context_wkv_weights(
weights: Iterable[tuple[str, torch.Tensor]], num_layers: int
) -> Iterable[tuple[str, torch.Tensor]]:
"""Load every draft layer's WKV into the cross-layer projection."""
for name, weight in weights:
yield name, weight
match = _CONTEXT_WKV_RE.fullmatch(name)
if match is None:
continue
layer_idx = int(match.group(1))
if layer_idx >= num_layers:
continue
stacked_weight = weight.detach()
stacked_weight.shard_id = layer_idx
yield f"context_wkv_proj.{match.group(2)}", stacked_weight
class DSparkDeepseekV4Model(nn.Module):
@@ -113,6 +134,15 @@ class DSparkDeepseekV4Model(nn.Module):
for i in range(self.num_dspark_layers)
]
)
self.context_wkv_proj = MergedColumnParallelLinear(
config.hidden_size,
[config.head_dim] * self.num_dspark_layers,
bias=False,
return_bias=False,
quant_config=vllm_config.quant_config,
prefix=maybe_prefix(prefix, "context_wkv_proj"),
disable_tp=True,
)
# Heads: final norm + hc_head, and the Markov + confidence heads
# Loaded from the "final" MTP layer weights (mtp.*) in the target checkpoint
@@ -173,15 +203,16 @@ class DSparkDeepseekV4Model(nn.Module):
place draft layers in different groups). ``None`` (or a ``None`` entry)
runs the projection to reserve workspace but writes nothing (profiling).
"""
for i, layer in enumerate(self.layers):
all_kv = self.context_wkv_proj(main_x).view(
main_x.shape[0], self.num_dspark_layers, self.config.head_dim
)
for i, (layer, kv) in enumerate(
zip(self.layers, all_kv.unbind(1), strict=True)
):
slot_mapping = (
None if context_slot_mappings is None else context_slot_mappings[i]
)
attn = layer.attn
# Optimized DSV4 MLA path: wkv part of the fused wq_a|wkv projection
# (q_lora part discarded), then RoPE/quant/insert via the fused op.
qr_kv, _ = attn.fused_wqa_wkv(main_x)
kv = qr_kv[..., attn.q_lora_rank :]
kv = attn.kv_norm(kv)
if slot_mapping is None:
continue
@@ -429,6 +460,7 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
head_start = n_local_head * tp_rank
head_end = n_local_head * (tp_rank + 1)
weights = _duplicate_context_wkv_weights(weights, len(self.model.layers))
for name, loaded_weight in weights:
mapped = self._remap_dspark_name(name)
if mapped is None:
@@ -452,6 +484,12 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
self.quant_config, name, loaded_weight
)
if name.startswith("model.context_wkv_proj."):
param = params_dict[name]
param.weight_loader(param, loaded_weight, loaded_weight.shard_id)
loaded_params.add(name)
continue
# E8M0 expert scales: keep raw exponent bytes.
if ".experts." in name:
if (
@@ -522,6 +560,8 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
Returns None for non-mtp weights (owned by the target model).
"""
if name.startswith("context_wkv_proj."):
return f"model.{name}"
m = re.match(r"mtp\.(\d+)\.(.*)", name)
if m is None:
return None