[Model] Add Qwen3-Omni DSpark support (#52560)
Signed-off-by: Zhou248 <630563665@qq.com>
This commit is contained in:
@@ -377,8 +377,6 @@ class DFlashQwen3DecoderLayer(nn.Module):
|
||||
|
||||
@support_torch_compile
|
||||
class DFlashQwen3Model(nn.Module):
|
||||
decoder_layer_cls = DFlashQwen3DecoderLayer
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_substr={
|
||||
"midlayer.": "layers.0.",
|
||||
@@ -412,10 +410,10 @@ class DFlashQwen3Model(nn.Module):
|
||||
drafter_config = getattr(self.config, "eagle_config", {})
|
||||
drafter_config.update(getattr(self.config, "dflash_config", {}))
|
||||
|
||||
if drafter_config is not None and "use_aux_hidden_state" in drafter_config:
|
||||
self.use_aux_hidden_state = drafter_config["use_aux_hidden_state"]
|
||||
else:
|
||||
self.use_aux_hidden_state = True
|
||||
self.use_aux_hidden_state = drafter_config.get(
|
||||
"use_aux_hidden_state",
|
||||
getattr(self.config, "use_aux_hidden_state", True),
|
||||
)
|
||||
|
||||
current_vllm_config = get_current_vllm_config()
|
||||
|
||||
@@ -430,7 +428,9 @@ class DFlashQwen3Model(nn.Module):
|
||||
# at that slot id. Some checkpoints (XiaomiMiMo/MiMo-V2.5-Pro-FP4-DFlash) ship
|
||||
# with a separate mask embedding tensor to use instead. When present, we load it
|
||||
# and substitute it for embed_tokens[mask_token_id] when computing embeddings.
|
||||
self.mask_token_id = drafter_config.get("mask_token_id")
|
||||
self.mask_token_id = drafter_config.get(
|
||||
"mask_token_id", getattr(self.config, "mask_token_id", None)
|
||||
)
|
||||
self.mask_embedding = nn.Parameter(
|
||||
torch.zeros(self.config.hidden_size, dtype=vllm_config.model_config.dtype),
|
||||
requires_grad=False,
|
||||
@@ -439,7 +439,7 @@ class DFlashQwen3Model(nn.Module):
|
||||
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
self.decoder_layer_cls(
|
||||
DFlashQwen3DecoderLayer(
|
||||
current_vllm_config,
|
||||
config=self.config,
|
||||
layer_idx=layer_idx,
|
||||
|
||||
@@ -200,8 +200,8 @@ class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM):
|
||||
self.logits_processor = LogitsProcessor(
|
||||
self.config.draft_vocab_size, scale=logit_scale
|
||||
)
|
||||
target_vocab_size = vllm_config.model_config.get_vocab_size()
|
||||
if self.config.draft_vocab_size != target_vocab_size:
|
||||
self.target_vocab_size = vllm_config.model_config.get_vocab_size()
|
||||
if self.config.draft_vocab_size != self.target_vocab_size:
|
||||
self.draft_id_to_target_id = nn.Parameter(
|
||||
torch.zeros(self.config.draft_vocab_size, dtype=torch.long),
|
||||
requires_grad=False,
|
||||
@@ -280,6 +280,24 @@ class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM):
|
||||
# mask_embedding is an unused placeholder param; DSpark masks via the vocab row.
|
||||
# embed_tokens / lm_head are optional; when omitted they are shared from
|
||||
# the target by load_dspark_model, so skip the unloaded params here.
|
||||
uses_expanded_input_vocab = self.config.vocab_size > self.target_vocab_size
|
||||
uses_reduced_vocab = self.config.draft_vocab_size != self.target_vocab_size
|
||||
if uses_expanded_input_vocab and not includes_embed_tokens:
|
||||
raise ValueError(
|
||||
"Qwen3 DSpark checkpoints whose input vocab_size is larger than "
|
||||
"the target vocabulary must include embed_tokens weights."
|
||||
)
|
||||
if uses_reduced_vocab and not includes_lm_head:
|
||||
raise ValueError(
|
||||
"Reduced-vocabulary Qwen3 DSpark checkpoints must include "
|
||||
"lm_head weights; the full target lm_head cannot be shared."
|
||||
)
|
||||
if uses_reduced_vocab and not includes_draft_id_mapping:
|
||||
raise ValueError(
|
||||
"Reduced-vocabulary Qwen3 DSpark checkpoints must include a "
|
||||
"d2t mapping so sampled draft ids can be converted to target ids."
|
||||
)
|
||||
|
||||
orig_to_new_substr = {"mask_embedding": None}
|
||||
if not includes_embed_tokens:
|
||||
orig_to_new_substr["embed_tokens"] = None
|
||||
|
||||
@@ -83,6 +83,7 @@ from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
from .interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsEagle3,
|
||||
SupportsMRoPE,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
@@ -1067,7 +1068,7 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
deepstack_input_embeds: IntermediateTensors | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]:
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
hidden_states = inputs_embeds
|
||||
@@ -1078,6 +1079,9 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
residual = intermediate_tensors["residual"]
|
||||
aux_hidden_states = self._maybe_add_hidden_state(
|
||||
[], self.start_layer, hidden_states, residual
|
||||
)
|
||||
for layer_idx, layer in enumerate(
|
||||
self.layers[self.start_layer : self.end_layer]
|
||||
):
|
||||
@@ -1096,12 +1100,17 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
|
||||
hidden_states
|
||||
+ deepstack_input_embeds[f"deepstack_input_embeds_{layer_idx}"]
|
||||
)
|
||||
self._maybe_add_hidden_state(
|
||||
aux_hidden_states, layer_idx + 1, hidden_states, residual
|
||||
)
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
if aux_hidden_states:
|
||||
return hidden_states, aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -1577,6 +1586,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
SupportsMRoPE,
|
||||
SupportsEagle3,
|
||||
Qwen3OmniMoeConditionalGenerationMixin,
|
||||
SupportsTranscription,
|
||||
):
|
||||
|
||||
@@ -643,6 +643,7 @@ _SPECULATIVE_DECODING_MODELS = {
|
||||
"DFlashMuseGlimmerAssistantModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"),
|
||||
"DSparkDraftModel": ("vllm.models.deepseek_v4", "DSparkDeepseekV4ForCausalLM"),
|
||||
"Qwen3DSparkModel": ("qwen3_dspark", "Qwen3DSparkForCausalLM"),
|
||||
"Qwen3OmniDSparkModel": ("qwen3_dspark", "Qwen3DSparkForCausalLM"),
|
||||
"K3DSparkModel": (
|
||||
"vllm.models.kimi_k3.nvidia.dspark_mla",
|
||||
"K3DSparkForCausalLM",
|
||||
|
||||
Reference in New Issue
Block a user