[Model] Add Qwen3-Omni DSpark support (#52560)

Signed-off-by: Zhou248 <630563665@qq.com>
This commit is contained in:
Zhou248
2026-08-22 15:35:22 -07:00
committed by GitHub
parent a014e35f38
commit 2f55ef254c
11 changed files with 731 additions and 22 deletions
+8 -8
View File
@@ -377,8 +377,6 @@ class DFlashQwen3DecoderLayer(nn.Module):
@support_torch_compile
class DFlashQwen3Model(nn.Module):
decoder_layer_cls = DFlashQwen3DecoderLayer
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_substr={
"midlayer.": "layers.0.",
@@ -412,10 +410,10 @@ class DFlashQwen3Model(nn.Module):
drafter_config = getattr(self.config, "eagle_config", {})
drafter_config.update(getattr(self.config, "dflash_config", {}))
if drafter_config is not None and "use_aux_hidden_state" in drafter_config:
self.use_aux_hidden_state = drafter_config["use_aux_hidden_state"]
else:
self.use_aux_hidden_state = True
self.use_aux_hidden_state = drafter_config.get(
"use_aux_hidden_state",
getattr(self.config, "use_aux_hidden_state", True),
)
current_vllm_config = get_current_vllm_config()
@@ -430,7 +428,9 @@ class DFlashQwen3Model(nn.Module):
# at that slot id. Some checkpoints (XiaomiMiMo/MiMo-V2.5-Pro-FP4-DFlash) ship
# with a separate mask embedding tensor to use instead. When present, we load it
# and substitute it for embed_tokens[mask_token_id] when computing embeddings.
self.mask_token_id = drafter_config.get("mask_token_id")
self.mask_token_id = drafter_config.get(
"mask_token_id", getattr(self.config, "mask_token_id", None)
)
self.mask_embedding = nn.Parameter(
torch.zeros(self.config.hidden_size, dtype=vllm_config.model_config.dtype),
requires_grad=False,
@@ -439,7 +439,7 @@ class DFlashQwen3Model(nn.Module):
self.layers = nn.ModuleList(
[
self.decoder_layer_cls(
DFlashQwen3DecoderLayer(
current_vllm_config,
config=self.config,
layer_idx=layer_idx,
+20 -2
View File
@@ -200,8 +200,8 @@ class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM):
self.logits_processor = LogitsProcessor(
self.config.draft_vocab_size, scale=logit_scale
)
target_vocab_size = vllm_config.model_config.get_vocab_size()
if self.config.draft_vocab_size != target_vocab_size:
self.target_vocab_size = vllm_config.model_config.get_vocab_size()
if self.config.draft_vocab_size != self.target_vocab_size:
self.draft_id_to_target_id = nn.Parameter(
torch.zeros(self.config.draft_vocab_size, dtype=torch.long),
requires_grad=False,
@@ -280,6 +280,24 @@ class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM):
# mask_embedding is an unused placeholder param; DSpark masks via the vocab row.
# embed_tokens / lm_head are optional; when omitted they are shared from
# the target by load_dspark_model, so skip the unloaded params here.
uses_expanded_input_vocab = self.config.vocab_size > self.target_vocab_size
uses_reduced_vocab = self.config.draft_vocab_size != self.target_vocab_size
if uses_expanded_input_vocab and not includes_embed_tokens:
raise ValueError(
"Qwen3 DSpark checkpoints whose input vocab_size is larger than "
"the target vocabulary must include embed_tokens weights."
)
if uses_reduced_vocab and not includes_lm_head:
raise ValueError(
"Reduced-vocabulary Qwen3 DSpark checkpoints must include "
"lm_head weights; the full target lm_head cannot be shared."
)
if uses_reduced_vocab and not includes_draft_id_mapping:
raise ValueError(
"Reduced-vocabulary Qwen3 DSpark checkpoints must include a "
"d2t mapping so sampled draft ids can be converted to target ids."
)
orig_to_new_substr = {"mask_embedding": None}
if not includes_embed_tokens:
orig_to_new_substr["embed_tokens"] = None
@@ -83,6 +83,7 @@ from vllm.v1.attention.backends.registry import AttentionBackendEnum
from .interfaces import (
MultiModalEmbeddings,
SupportsEagle3,
SupportsMRoPE,
SupportsMultiModal,
SupportsPP,
@@ -1067,7 +1068,7 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
deepstack_input_embeds: IntermediateTensors | None = None,
) -> torch.Tensor | IntermediateTensors:
) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -1078,6 +1079,9 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
residual = intermediate_tensors["residual"]
aux_hidden_states = self._maybe_add_hidden_state(
[], self.start_layer, hidden_states, residual
)
for layer_idx, layer in enumerate(
self.layers[self.start_layer : self.end_layer]
):
@@ -1096,12 +1100,17 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
hidden_states
+ deepstack_input_embeds[f"deepstack_input_embeds_{layer_idx}"]
)
self._maybe_add_hidden_state(
aux_hidden_states, layer_idx + 1, hidden_states, residual
)
if not get_pp_group().is_last_rank:
return IntermediateTensors(
{"hidden_states": hidden_states, "residual": residual}
)
hidden_states, _ = self.norm(hidden_states, residual)
if aux_hidden_states:
return hidden_states, aux_hidden_states
return hidden_states
@@ -1577,6 +1586,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
SupportsMultiModal,
SupportsPP,
SupportsMRoPE,
SupportsEagle3,
Qwen3OmniMoeConditionalGenerationMixin,
SupportsTranscription,
):
+1
View File
@@ -643,6 +643,7 @@ _SPECULATIVE_DECODING_MODELS = {
"DFlashMuseGlimmerAssistantModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"),
"DSparkDraftModel": ("vllm.models.deepseek_v4", "DSparkDeepseekV4ForCausalLM"),
"Qwen3DSparkModel": ("qwen3_dspark", "Qwen3DSparkForCausalLM"),
"Qwen3OmniDSparkModel": ("qwen3_dspark", "Qwen3DSparkForCausalLM"),
"K3DSparkModel": (
"vllm.models.kimi_k3.nvidia.dspark_mla",
"K3DSparkForCausalLM",