[Spec][V2] Support MTP speculative decoding under pipeline parallelism (#46994)
Signed-off-by: Chris Eastwood <chris.eastwood@pwn4g3.dev> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Nick Hill
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
9a35c081e8
commit
6fe67cbbf3
@@ -32,8 +32,10 @@ from vllm.model_executor.models.deepseek_v2 import (
|
||||
_try_load_fp8_indexer_wk,
|
||||
get_spec_layer_idx_from_weight_name,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import SupportsPP
|
||||
from vllm.model_executor.models.utils import (
|
||||
get_pp_missing_layer_names,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.models.deepseek_v32.common.kernels import fused_eh_norm
|
||||
@@ -160,7 +162,7 @@ class DeepseekV32MultiTokenPredictor(nn.Module):
|
||||
return self.logits_processor(mtp_layer.shared_head.head, hidden_states)
|
||||
|
||||
|
||||
class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts, SupportsPP):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.config = vllm_config.model_config.hf_config
|
||||
@@ -169,6 +171,9 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
self.set_moe_parameters()
|
||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||
["hidden_states", "residual"], self.config.hidden_size
|
||||
)
|
||||
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
|
||||
self.model.layers.values(),
|
||||
DeepseekV2MoE,
|
||||
@@ -192,7 +197,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
def forward( # type: ignore[override]
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
@@ -268,6 +273,15 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
continue
|
||||
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
|
||||
if spec_layer is None:
|
||||
# A tied top-level embed_tokens has no spec layer to rewrite
|
||||
# from; the draft needs its own copy under PP.
|
||||
param = params_dict.get(name) if "embed_tokens" in name else None
|
||||
if param is not None:
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
is_fusion_moe_shared_experts_layer = (
|
||||
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
|
||||
|
||||
@@ -182,7 +182,8 @@ class DeepseekV32Model(torch.nn.Module):
|
||||
# DSA is always sparse (has index_topk); allocate the shared top-k
|
||||
# buffer the indexer writes and the sparse MLA backend reads.
|
||||
self.is_v32 = True
|
||||
topk_indices_buffer = torch.empty(
|
||||
# On the model, not a local: the MTP proposer shares it with the draft.
|
||||
self.topk_indices_buffer = torch.empty(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
config.index_topk,
|
||||
dtype=torch.int32,
|
||||
@@ -207,7 +208,7 @@ class DeepseekV32Model(torch.nn.Module):
|
||||
lambda prefix: DeepseekV32DecoderLayer(
|
||||
vllm_config=vllm_config,
|
||||
prefix=prefix,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
topk_indices_buffer=self.topk_indices_buffer,
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
|
||||
@@ -33,8 +33,10 @@ from vllm.model_executor.models.deepseek_v2 import (
|
||||
_try_load_fp8_indexer_wk,
|
||||
get_spec_layer_idx_from_weight_name,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import SupportsPP
|
||||
from vllm.model_executor.models.utils import (
|
||||
get_pp_missing_layer_names,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.models.common.ops.fused_allreduce_rms_norm import fused_allreduce_rms_norm
|
||||
@@ -260,7 +262,7 @@ class DeepseekV32MultiTokenPredictor(nn.Module):
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts, SupportsPP):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.config = vllm_config.model_config.hf_config
|
||||
@@ -271,6 +273,9 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
if self.config.model_type == "glm_moe_dsa":
|
||||
enable_glm52_low_latency_gemm(self, vllm_config.model_config.dtype)
|
||||
self.set_moe_parameters()
|
||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||
["hidden_states", "residual"], self.config.hidden_size
|
||||
)
|
||||
|
||||
def set_moe_parameters(self):
|
||||
self.num_moe_layers = self.config.num_nextn_predict_layers
|
||||
@@ -289,7 +294,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
def forward( # type: ignore[override]
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
@@ -368,6 +373,15 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
continue
|
||||
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
|
||||
if spec_layer is None:
|
||||
# A tied top-level embed_tokens has no spec layer to rewrite
|
||||
# from; the draft needs its own copy under PP.
|
||||
param = params_dict.get(name) if "embed_tokens" in name else None
|
||||
if param is not None:
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
name = self._rewrite_spec_layer_name(spec_layer, name)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user