[Spec][V2] Support MTP speculative decoding under pipeline parallelism (#46994)

Signed-off-by: Chris Eastwood <chris.eastwood@pwn4g3.dev>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Chris Eastwood
2026-09-11 17:55:21 +00:00
committed by GitHub
co-authored by Nick Hill mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 9a35c081e8
commit 6fe67cbbf3
12 changed files with 167 additions and 49 deletions
+16 -2
View File
@@ -32,8 +32,10 @@ from vllm.model_executor.models.deepseek_v2 import (
_try_load_fp8_indexer_wk,
get_spec_layer_idx_from_weight_name,
)
from vllm.model_executor.models.interfaces import SupportsPP
from vllm.model_executor.models.utils import (
get_pp_missing_layer_names,
make_empty_intermediate_tensors_factory,
maybe_prefix,
)
from vllm.models.deepseek_v32.common.kernels import fused_eh_norm
@@ -160,7 +162,7 @@ class DeepseekV32MultiTokenPredictor(nn.Module):
return self.logits_processor(mtp_layer.shared_head.head, hidden_states)
class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts, SupportsPP):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
self.config = vllm_config.model_config.hf_config
@@ -169,6 +171,9 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
self.set_moe_parameters()
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states", "residual"], self.config.hidden_size
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.model.layers.values(),
DeepseekV2MoE,
@@ -192,7 +197,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
def forward( # type: ignore[override]
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
@@ -268,6 +273,15 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
continue
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
if spec_layer is None:
# A tied top-level embed_tokens has no spec layer to rewrite
# from; the draft needs its own copy under PP.
param = params_dict.get(name) if "embed_tokens" in name else None
if param is not None:
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
continue
is_fusion_moe_shared_experts_layer = (
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)