[Perf][DSpark] Stack DeepSeek V4 context WKV projections (#54674)
Signed-off-by: liuyao0322 <yaoliu548926@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
@@ -11,6 +11,7 @@ from vllm.compilation.wrapper import TorchCompileWithNoGuardsWrapper
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.models.qwen3_dspark import DSparkMarkovHead
|
||||
from vllm.model_executor.models.registry import ModelRegistry
|
||||
from vllm.models.deepseek_v4.nvidia import dspark as dsv4_dspark
|
||||
from vllm.models.kimi_k3.nvidia import dspark_mla
|
||||
from vllm.models.kimi_k3.nvidia.dspark_mla import K3DSparkForCausalLM, K3DSparkModel
|
||||
|
||||
@@ -262,3 +263,75 @@ def test_v41_dspark_loads_linear_scales(
|
||||
assert loaded == {runtime_name}
|
||||
assert shards == [() if shard_id is None else (shard_id,)]
|
||||
torch.testing.assert_close(param, checkpoint_scale)
|
||||
|
||||
|
||||
def test_dsv4_context_wkv_weights_are_duplicated_by_draft_layer():
|
||||
weights = [
|
||||
("mtp.0.attn.wkv.weight", torch.arange(4)),
|
||||
("mtp.1.attn.wq_a.weight", torch.arange(3)),
|
||||
("mtp.2.attn.wkv.scale", torch.tensor(0.5)),
|
||||
("mtp.3.attn.wkv.weight", torch.arange(2)),
|
||||
]
|
||||
|
||||
duplicated = list(dsv4_dspark._duplicate_context_wkv_weights(weights, 3))
|
||||
|
||||
assert [name for name, _ in duplicated] == [
|
||||
"mtp.0.attn.wkv.weight",
|
||||
"context_wkv_proj.weight",
|
||||
"mtp.1.attn.wq_a.weight",
|
||||
"mtp.2.attn.wkv.scale",
|
||||
"context_wkv_proj.scale",
|
||||
"mtp.3.attn.wkv.weight",
|
||||
]
|
||||
assert duplicated[1][1].shard_id == 0
|
||||
assert duplicated[4][1].shard_id == 2
|
||||
assert duplicated[0][1].data_ptr() == duplicated[1][1].data_ptr()
|
||||
assert duplicated[3][1].data_ptr() == duplicated[4][1].data_ptr()
|
||||
|
||||
|
||||
def test_dsv4_context_kv_uses_one_stacked_wkv_projection(monkeypatch):
|
||||
calls = []
|
||||
stacked_output = torch.arange(24, dtype=torch.float32).view(2, 12)
|
||||
|
||||
class StackedProjection:
|
||||
def __init__(self):
|
||||
self.calls = 0
|
||||
|
||||
def __call__(self, main_x):
|
||||
self.calls += 1
|
||||
assert main_x.shape == (2, 5)
|
||||
return stacked_output
|
||||
|
||||
projection = StackedProjection()
|
||||
layers = [
|
||||
SimpleNamespace(attn=SimpleNamespace(kv_norm=lambda kv, offset=i: kv + offset))
|
||||
for i in range(3)
|
||||
]
|
||||
model = SimpleNamespace(
|
||||
config=SimpleNamespace(head_dim=4),
|
||||
context_wkv_proj=projection,
|
||||
layers=layers,
|
||||
num_dspark_layers=3,
|
||||
)
|
||||
slot_mappings = [torch.tensor([0, 1]), None, torch.tensor([4, 5])]
|
||||
monkeypatch.setattr(
|
||||
dsv4_dspark,
|
||||
"_insert_context_kv",
|
||||
lambda attn, kv, positions, slots: calls.append(
|
||||
(attn, kv.clone(), positions, slots)
|
||||
),
|
||||
)
|
||||
|
||||
dsv4_dspark.DSparkDeepseekV4Model.precompute_and_store_context_kv(
|
||||
model,
|
||||
torch.zeros(2, 5),
|
||||
torch.tensor([7, 8]),
|
||||
slot_mappings,
|
||||
)
|
||||
|
||||
assert projection.calls == 1
|
||||
assert len(calls) == 2
|
||||
assert torch.equal(calls[0][1], stacked_output.view(2, 3, 4)[:, 0])
|
||||
assert torch.equal(calls[1][1], stacked_output.view(2, 3, 4)[:, 2] + 2)
|
||||
assert calls[0][3] is slot_mappings[0]
|
||||
assert calls[1][3] is slot_mappings[2]
|
||||
|
||||
@@ -31,7 +31,10 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
@@ -61,6 +64,24 @@ logger = init_logger(__name__)
|
||||
# MoE expert scale suffix differs by expert dtype (mirrors deepseek_v4 loaders):
|
||||
# fp4 experts register ``.weight_scale``; block-fp8 experts ``.weight_scale_inv``.
|
||||
_EXPERT_SCALE_RE = re.compile(r"\.experts\.\d+\.w[123]\.scale$")
|
||||
_CONTEXT_WKV_RE = re.compile(r"^mtp\.(\d+)\.attn\.wkv\.(.+)$")
|
||||
|
||||
|
||||
def _duplicate_context_wkv_weights(
|
||||
weights: Iterable[tuple[str, torch.Tensor]], num_layers: int
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
"""Load every draft layer's WKV into the cross-layer projection."""
|
||||
for name, weight in weights:
|
||||
yield name, weight
|
||||
match = _CONTEXT_WKV_RE.fullmatch(name)
|
||||
if match is None:
|
||||
continue
|
||||
layer_idx = int(match.group(1))
|
||||
if layer_idx >= num_layers:
|
||||
continue
|
||||
stacked_weight = weight.detach()
|
||||
stacked_weight.shard_id = layer_idx
|
||||
yield f"context_wkv_proj.{match.group(2)}", stacked_weight
|
||||
|
||||
|
||||
class DSparkDeepseekV4Model(nn.Module):
|
||||
@@ -113,6 +134,15 @@ class DSparkDeepseekV4Model(nn.Module):
|
||||
for i in range(self.num_dspark_layers)
|
||||
]
|
||||
)
|
||||
self.context_wkv_proj = MergedColumnParallelLinear(
|
||||
config.hidden_size,
|
||||
[config.head_dim] * self.num_dspark_layers,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=vllm_config.quant_config,
|
||||
prefix=maybe_prefix(prefix, "context_wkv_proj"),
|
||||
disable_tp=True,
|
||||
)
|
||||
|
||||
# Heads: final norm + hc_head, and the Markov + confidence heads
|
||||
# Loaded from the "final" MTP layer weights (mtp.*) in the target checkpoint
|
||||
@@ -173,15 +203,16 @@ class DSparkDeepseekV4Model(nn.Module):
|
||||
place draft layers in different groups). ``None`` (or a ``None`` entry)
|
||||
runs the projection to reserve workspace but writes nothing (profiling).
|
||||
"""
|
||||
for i, layer in enumerate(self.layers):
|
||||
all_kv = self.context_wkv_proj(main_x).view(
|
||||
main_x.shape[0], self.num_dspark_layers, self.config.head_dim
|
||||
)
|
||||
for i, (layer, kv) in enumerate(
|
||||
zip(self.layers, all_kv.unbind(1), strict=True)
|
||||
):
|
||||
slot_mapping = (
|
||||
None if context_slot_mappings is None else context_slot_mappings[i]
|
||||
)
|
||||
attn = layer.attn
|
||||
# Optimized DSV4 MLA path: wkv part of the fused wq_a|wkv projection
|
||||
# (q_lora part discarded), then RoPE/quant/insert via the fused op.
|
||||
qr_kv, _ = attn.fused_wqa_wkv(main_x)
|
||||
kv = qr_kv[..., attn.q_lora_rank :]
|
||||
kv = attn.kv_norm(kv)
|
||||
if slot_mapping is None:
|
||||
continue
|
||||
@@ -429,6 +460,7 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
head_start = n_local_head * tp_rank
|
||||
head_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
weights = _duplicate_context_wkv_weights(weights, len(self.model.layers))
|
||||
for name, loaded_weight in weights:
|
||||
mapped = self._remap_dspark_name(name)
|
||||
if mapped is None:
|
||||
@@ -452,6 +484,12 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
self.quant_config, name, loaded_weight
|
||||
)
|
||||
|
||||
if name.startswith("model.context_wkv_proj."):
|
||||
param = params_dict[name]
|
||||
param.weight_loader(param, loaded_weight, loaded_weight.shard_id)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
|
||||
# E8M0 expert scales: keep raw exponent bytes.
|
||||
if ".experts." in name:
|
||||
if (
|
||||
@@ -522,6 +560,8 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
|
||||
Returns None for non-mtp weights (owned by the target model).
|
||||
"""
|
||||
if name.startswith("context_wkv_proj."):
|
||||
return f"model.{name}"
|
||||
m = re.match(r"mtp\.(\d+)\.(.*)", name)
|
||||
if m is None:
|
||||
return None
|
||||
|
||||
Reference in New Issue
Block a user