[EPLB] Add EPLB support to dspark draft models (DeepSeekV4) (#57549)
Signed-off-by: Markov Ilya <markovilya197@gmail.com>
This commit is contained in:
@@ -0,0 +1,50 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared EPLB MoE registration helpers for DeepSeek V4 family models."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
|
||||
|
||||
def dspark_draft_supports_eplb(draft_model_config: ModelConfig) -> bool:
|
||||
"""Return whether a DSpark draft can share EPLB state with the target.
|
||||
|
||||
Only DeepSeek-V4 DSpark drafts reuse the target expert layout. V4.1 drafts
|
||||
use a smaller routed-expert count and cannot share EPLB state.
|
||||
"""
|
||||
return getattr(draft_model_config.hf_config, "model_type", None) == "deepseek_v4"
|
||||
|
||||
|
||||
def collect_moe_layers(
|
||||
moe_model: Any,
|
||||
layers: Iterable[nn.Module],
|
||||
config: object,
|
||||
*,
|
||||
decoder_layer_type: type[nn.Module],
|
||||
moe_type: type[nn.Module],
|
||||
) -> None:
|
||||
"""Populate ``MixtureOfExperts`` fields from decoder layers for EPLB.
|
||||
|
||||
Works for both target backbones and DSpark draft stacks. Draft models pass
|
||||
their shorter ``model.layers`` list; pipeline-parallel placeholder layers
|
||||
are skipped via ``decoder_layer_type``.
|
||||
"""
|
||||
moe_model.num_expert_groups = getattr(config, "n_group", 1)
|
||||
moe_model.moe_layers = []
|
||||
moe_model.moe_mlp_layers = []
|
||||
example_moe = None
|
||||
for layer in layers:
|
||||
if not isinstance(layer, decoder_layer_type):
|
||||
continue
|
||||
if not isinstance(layer.ffn, moe_type):
|
||||
continue
|
||||
example_moe = layer.ffn
|
||||
moe_model.moe_mlp_layers.append(layer.ffn)
|
||||
moe_model.moe_layers.append(layer.ffn.experts)
|
||||
|
||||
moe_model.num_moe_layers = len(moe_model.moe_layers)
|
||||
moe_model.extract_moe_parameters(example_moe)
|
||||
@@ -52,11 +52,14 @@ from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_padding_mask,
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.models.deepseek_v4.common.eplb_util import collect_moe_layers
|
||||
from vllm.models.deepseek_v4.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
|
||||
|
||||
from .model import (
|
||||
DeepseekV4DecoderLayer,
|
||||
DeepseekV4MixtureOfExperts,
|
||||
DeepseekV4Model,
|
||||
DeepseekV4MoE,
|
||||
_use_sequence_parallel,
|
||||
make_deepseek_v4_expert_params_mapping,
|
||||
prepare_mega_gate_routing_metadata,
|
||||
@@ -346,7 +349,7 @@ def _insert_context_kv(
|
||||
)
|
||||
|
||||
|
||||
class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
class DSparkDeepseekV4ForCausalLM(nn.Module, DeepseekV4MixtureOfExperts):
|
||||
# Draft weights ship in the target checkpoint (mtp.*) without embed/head, so
|
||||
# load_dspark_model always aliases the target's.
|
||||
has_own_embed_tokens = False
|
||||
@@ -373,6 +376,16 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
self.logits_processor = LogitsProcessor(self.config.vocab_size)
|
||||
self.set_moe_parameters()
|
||||
|
||||
def set_moe_parameters(self) -> None:
|
||||
collect_moe_layers(
|
||||
self,
|
||||
self.model.layers,
|
||||
self.config,
|
||||
decoder_layer_type=DeepseekV4DecoderLayer,
|
||||
moe_type=DeepseekV4MoE,
|
||||
)
|
||||
|
||||
# --- Hooks used by the speculator -------------------------------------
|
||||
|
||||
@@ -451,6 +464,7 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
num_experts=self.config.n_routed_experts,
|
||||
num_redundant_experts=first_layer.ffn.n_redundant_experts,
|
||||
)
|
||||
expert_scale_suffix = (
|
||||
".weight_scale"
|
||||
|
||||
@@ -81,6 +81,7 @@ from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.models.deepseek_v4.attention import DeepseekV4Attention
|
||||
from vllm.models.deepseek_v4.common.eplb_util import collect_moe_layers
|
||||
from vllm.models.deepseek_v4.nvidia.flashinfer_sparse import (
|
||||
DeepseekV4FlashInferMLAAttention,
|
||||
DeepseekV4FlashInferSM120Attention,
|
||||
@@ -1841,23 +1842,13 @@ class DeepseekV4ForCausalLM(
|
||||
self.set_moe_parameters()
|
||||
|
||||
def set_moe_parameters(self) -> None:
|
||||
self.num_expert_groups = getattr(self.config, "n_group", 1)
|
||||
self.num_moe_layers = self.config.num_hidden_layers
|
||||
self.moe_layers: list[nn.Module] = []
|
||||
self.moe_mlp_layers: list[DeepseekV4MoE] = []
|
||||
example_moe: DeepseekV4MoE | None = None
|
||||
for layer in self.model.layers:
|
||||
if isinstance(layer, PPMissingLayer):
|
||||
continue
|
||||
if not isinstance(layer, DeepseekV4DecoderLayer):
|
||||
continue
|
||||
if isinstance(layer.ffn, DeepseekV4MoE):
|
||||
example_moe = layer.ffn
|
||||
self.moe_mlp_layers.append(layer.ffn)
|
||||
self.moe_layers.append(layer.ffn.experts)
|
||||
|
||||
self.num_moe_layers = len(self.moe_layers)
|
||||
self.extract_moe_parameters(example_moe)
|
||||
collect_moe_layers(
|
||||
self,
|
||||
self.model.layers,
|
||||
self.config,
|
||||
decoder_layer_type=DeepseekV4DecoderLayer,
|
||||
moe_type=DeepseekV4MoE,
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
Reference in New Issue
Block a user