[Model] Add Qwen3-Omni DSpark support (#52560)

Signed-off-by: Zhou248 <630563665@qq.com>
This commit is contained in:
Zhou248
2026-08-22 15:35:22 -07:00
committed by GitHub
parent a014e35f38
commit 2f55ef254c
11 changed files with 731 additions and 22 deletions
+171 -1
View File
@@ -1,9 +1,13 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import Mock
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import Mock, patch
import pytest
import torch
import torch.nn as nn
from transformers import PretrainedConfig
from vllm.multimodal.processing import InputProcessingContext
@@ -218,5 +222,171 @@ def test_qwen3_omni_get_updates_use_audio_in_video(
)
@pytest.mark.skip_global_cleanup
def test_qwen3_omni_exposes_eagle3_to_its_text_backbone():
from vllm.model_executor.models.interfaces import EagleModelMixin, supports_eagle3
from vllm.model_executor.models.qwen3_omni_moe_thinker import (
Qwen3OmniMoeThinkerForConditionalGeneration,
)
class DummyBackbone(nn.Module, EagleModelMixin):
def __init__(self):
super().__init__()
self.layers = nn.ModuleList([nn.Identity(), nn.Identity()])
class DummyLanguageModel(nn.Module):
def __init__(self):
super().__init__()
self.model = DummyBackbone()
def embed_input_ids(self, input_ids):
return input_ids
model = Qwen3OmniMoeThinkerForConditionalGeneration.__new__(
Qwen3OmniMoeThinkerForConditionalGeneration
)
nn.Module.__init__(model)
model.language_model = DummyLanguageModel()
assert supports_eagle3(model)
model.set_aux_hidden_state_layers((1, 2))
assert model.language_model.model.aux_hidden_state_layers == (1, 2)
@pytest.mark.skip_global_cleanup
def test_qwen3_omni_text_model_collects_post_deepstack_aux_hidden_states():
from vllm.model_executor.models.qwen3_omni_moe_thinker import Qwen3MoeLLMModel
class DummyLayer(nn.Module):
def forward(self, positions, hidden_states, residual):
return hidden_states + 1, torch.full_like(hidden_states, 10)
class DummyNorm(nn.Module):
def forward(self, hidden_states, residual):
return hidden_states + residual, None
model = Qwen3MoeLLMModel.__new__(Qwen3MoeLLMModel)
nn.Module.__init__(model)
model.start_layer = 0
model.end_layer = 1
model.layers = nn.ModuleList([DummyLayer()])
model.norm = DummyNorm()
model.aux_hidden_state_layers = (1,)
pp_group = Mock(is_first_rank=True, is_last_rank=True)
inputs_embeds = torch.tensor([[1.0]])
deepstack_inputs = {"deepstack_input_embeds_0": torch.tensor([[3.0]])}
with patch(
"vllm.model_executor.models.qwen3_omni_moe_thinker.get_pp_group",
return_value=pp_group,
):
output, aux_hidden_states = model.forward(
input_ids=None,
positions=torch.tensor([0]),
inputs_embeds=inputs_embeds,
deepstack_input_embeds=deepstack_inputs,
)
torch.testing.assert_close(output, torch.tensor([[15.0]]))
assert len(aux_hidden_states) == 1
torch.testing.assert_close(aux_hidden_states[0], torch.tensor([[15.0]]))
@pytest.mark.skip_global_cleanup
@pytest.mark.parametrize(
("input_vocab_size", "draft_vocab_size", "weights", "error"),
[
(101, 100, [], "must include embed_tokens weights"),
(99, 99, [], "must include lm_head weights"),
(100, 40, [], "must include lm_head weights"),
(
100,
40,
[("lm_head.weight", torch.empty(40, 8))],
"must include a d2t mapping",
),
],
)
def test_qwen3_dspark_rejects_incomplete_vocab_weights(
input_vocab_size, draft_vocab_size, weights, error
):
from vllm.model_executor.models.qwen3_dspark import Qwen3DSparkForCausalLM
model = Qwen3DSparkForCausalLM.__new__(Qwen3DSparkForCausalLM)
nn.Module.__init__(model)
object.__setattr__(
model,
"config",
SimpleNamespace(
vocab_size=input_vocab_size,
draft_vocab_size=draft_vocab_size,
),
)
object.__setattr__(model, "target_vocab_size", 100)
with pytest.raises(ValueError, match=error):
model.load_weights(weights)
@pytest.mark.skip_global_cleanup
def test_dspark_shares_target_embedding_with_smaller_draft_vocabulary():
from vllm.v1.worker.gpu.spec_decode.dspark import utils as dspark_utils
target_embedding = nn.Embedding(100, 8)
draft_embedding = nn.Embedding(99, 8)
target_model = SimpleNamespace(model=SimpleNamespace(embed_tokens=target_embedding))
draft_model = SimpleNamespace(
model=SimpleNamespace(embed_tokens=draft_embedding),
has_own_embed_tokens=False,
)
draft_model_config = SimpleNamespace(
hf_config=SimpleNamespace(model_type="qwen3"),
get_vocab_size=Mock(return_value=99),
)
vllm_config = SimpleNamespace(
speculative_config=SimpleNamespace(
draft_model_config=draft_model_config,
attention_backend=None,
kv_cache_dtype=None,
),
attention_config=SimpleNamespace(backend=None),
cache_config=SimpleNamespace(),
model_config=SimpleNamespace(get_vocab_size=Mock(return_value=100)),
)
def fake_replace(config, **changes):
values = vars(config).copy()
values.update(changes)
return SimpleNamespace(**values)
with (
patch.object(dspark_utils, "replace", side_effect=fake_replace),
patch.object(
dspark_utils,
"get_pp_group",
return_value=SimpleNamespace(world_size=1),
),
patch(
"vllm.compilation.backends.set_model_tag",
return_value=nullcontext(),
),
patch(
"vllm.model_executor.model_loader.get_model",
return_value=draft_model,
),
patch(
"vllm.model_executor.models.qwen3_dflash.dflash_has_any_non_causal",
return_value=False,
),
patch(
"vllm.model_executor.models.utils.get_draft_quant_config",
return_value=None,
),
):
loaded_model = dspark_utils.load_dspark_model(target_model, vllm_config)
assert loaded_model.model.embed_tokens is target_embedding
if __name__ == "__main__":
pytest.main([__file__, "-v"])
+7
View File
@@ -1557,6 +1557,13 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
is_available_online=False,
use_original_num_layers=True, # DSpark backbone requires all layers
),
"Qwen3OmniDSparkModel": _HfExamplesInfo(
"Qwen/Qwen3-Omni-30B-A3B-Instruct",
speculative_model="Qwen/Qwen3-Omni-30B-A3B-DSpark",
is_available_online=False,
use_original_num_layers=True,
min_transformers_version="4.57",
),
"Gemma4DSparkModel": _HfExamplesInfo(
"google/gemma-4-12B-it",
speculative_model="deepseek-ai/dspark_gemma4_12b_block7",
+121
View File
@@ -32,6 +32,7 @@ from vllm.config.compilation import CompilationMode, CUDAGraphMode
from vllm.config.kernel import IrOpPriorityConfig
from vllm.config.load import LoadConfig
from vllm.config.mamba import MambaBackendEnum
from vllm.config.speculative import _validate_qwen3_omni_dspark
from vllm.config.utils import get_field
from vllm.config.vllm import OPTIMIZATION_LEVEL_TO_CONFIG, OptimizationLevel
from vllm.platforms import current_platform
@@ -2100,6 +2101,126 @@ def test_mtp_draft_uses_model_weights_not_local_cache(mock_model_config_cls):
assert actual_model == s3_url
def _make_qwen3_omni_dspark_configs():
text_config = SimpleNamespace(
hidden_size=2048,
num_hidden_layers=48,
num_attention_heads=32,
num_key_value_heads=4,
head_dim=128,
vocab_size=152064,
)
target_model_config = SimpleNamespace(
hf_config=SimpleNamespace(
model_type="qwen3_omni_moe",
architectures=["Qwen3OmniMoeForConditionalGeneration"],
thinker_config=SimpleNamespace(text_config=text_config),
),
hf_text_config=text_config,
architectures=["Qwen3OmniMoeForConditionalGeneration"],
get_hidden_size=lambda: 2048,
get_total_num_hidden_layers=lambda: 48,
get_vocab_size=lambda: 152064,
)
draft_hf_config = SimpleNamespace(
model_type="qwen3",
architectures=["Qwen3OmniDSparkModel"],
hidden_size=2048,
num_attention_heads=32,
num_key_value_heads=4,
head_dim=128,
block_size=7,
target_hidden_size=2048,
target_layer_ids=[1, 9, 17, 25, 33],
use_aux_hidden_state=True,
markov_rank=64,
markov_head_type="vanilla",
sample_from_anchor=True,
dspark_bonus_anchor=False,
vocab_size=152064,
draft_vocab_size=32000,
mask_token_id=151669,
rope_parameters={"rope_type": "default", "rope_theta": 1000000.0},
)
draft_model_config = SimpleNamespace(
hf_config=draft_hf_config,
architectures=["Qwen3OmniDSparkModel"],
)
return target_model_config, draft_model_config
@pytest.mark.skip_global_cleanup
def test_qwen3_omni_dspark_checkpoint_contract_is_accepted():
target_config, draft_config = _make_qwen3_omni_dspark_configs()
_validate_qwen3_omni_dspark(target_config, draft_config, 7)
@pytest.mark.skip_global_cleanup
def test_qwen3_omni_dspark_rejects_generic_qwen3_architecture():
target_config, draft_config = _make_qwen3_omni_dspark_configs()
draft_config.hf_config.architectures = ["Qwen3DSparkModel"]
draft_config.architectures = ["Qwen3DSparkModel"]
with pytest.raises(ValueError, match="must be converted first"):
_validate_qwen3_omni_dspark(target_config, draft_config, 7)
@pytest.mark.parametrize(
("field", "value", "error"),
[
("block_size", 5, "trained block_size"),
("target_hidden_size", 4096, "target_hidden_size"),
("hidden_size", 4096, "draft hidden_size"),
("num_attention_heads", 16, "num_attention_heads"),
("num_key_value_heads", 8, "num_key_value_heads"),
("head_dim", 64, "head_dim"),
("target_layer_ids", [7, 48], "zero-based text-layer"),
("target_layer_ids", [23, 7], "strictly increasing"),
("use_aux_hidden_state", False, "use_aux_hidden_state=true"),
("markov_rank", 0, "markov_rank"),
("markov_head_type", "gated", "markov_head_type='vanilla'"),
("sample_from_anchor", False, "sample_from_anchor=true"),
("dspark_bonus_anchor", True, "dspark_bonus_anchor=false"),
("vocab_size", 0, "input vocab_size must be a positive integer"),
],
)
@pytest.mark.skip_global_cleanup
def test_qwen3_omni_dspark_rejects_incompatible_checkpoint_fields(field, value, error):
target_config, draft_config = _make_qwen3_omni_dspark_configs()
setattr(draft_config.hf_config, field, value)
with pytest.raises(ValueError, match=error):
_validate_qwen3_omni_dspark(target_config, draft_config, 7)
@pytest.mark.skip_global_cleanup
def test_qwen3_omni_dspark_rejects_mrope_draft_positions():
target_config, draft_config = _make_qwen3_omni_dspark_configs()
draft_config.hf_config.rope_parameters["mrope_section"] = [24, 20, 20]
with pytest.raises(ValueError, match="logical 1-D RoPE"):
_validate_qwen3_omni_dspark(target_config, draft_config, 7)
@pytest.mark.skip_global_cleanup
def test_qwen3_omni_dspark_allows_draft_only_noise_token_row():
target_config, draft_config = _make_qwen3_omni_dspark_configs()
draft_config.hf_config.vocab_size = 152065
draft_config.hf_config.draft_vocab_size = 152064
draft_config.hf_config.mask_token_id = 152064
_validate_qwen3_omni_dspark(target_config, draft_config, 7)
@pytest.mark.skip_global_cleanup
def test_qwen3_omni_dspark_allows_smaller_input_vocabulary():
target_config, draft_config = _make_qwen3_omni_dspark_configs()
draft_config.hf_config.vocab_size = 151936
draft_config.hf_config.mask_token_id = 151669
_validate_qwen3_omni_dspark(target_config, draft_config, 7)
def test_ir_op_priority_default():
"""Test that IR op priority defaults are set correctly."""
from vllm.config.kernel import IrOpPriorityConfig
@@ -0,0 +1,58 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.transformers_utils.configs.speculators.base import SpeculatorsConfig
pytestmark = pytest.mark.skip_global_cleanup
def _make_dspark_config(architecture: str) -> dict:
return {
"speculators_model_type": "dspark",
"architectures": [architecture],
"sample_from_anchor": True,
"use_aux_hidden_state": True,
"draft_vocab_size": 32,
"target_hidden_size": 16,
"mask_token_id": 31,
"markov_rank": 4,
"markov_head_type": "vanilla",
"block_size": 3,
"aux_hidden_state_layer_ids": [2, 5],
"transformer_layer_config": {
"model_type": "qwen3",
"hidden_size": 16,
},
}
def test_dspark_updater_preserves_qwen3_omni_contract() -> None:
config = SpeculatorsConfig.extract_transformers_pre_trained_config(
_make_dspark_config("Qwen3OmniDSparkModel")
)
assert config["architectures"] == ["Qwen3OmniDSparkModel"]
assert config["sample_from_anchor"] is True
assert config["dspark_bonus_anchor"] is False
assert config["use_aux_hidden_state"] is True
assert config["target_layer_ids"] == [1, 4]
def test_dspark_updater_keeps_legacy_qwen3_checkpoint_loadable() -> None:
config = SpeculatorsConfig.extract_transformers_pre_trained_config(
_make_dspark_config("DSparkSpeculator")
)
assert config["architectures"] == ["Qwen3DSparkModel"]
def test_dspark_updater_maps_bonus_anchor_semantics() -> None:
outer_config = _make_dspark_config("Qwen3DSparkModel")
outer_config["sample_from_anchor"] = False
config = SpeculatorsConfig.extract_transformers_pre_trained_config(outer_config)
assert config["sample_from_anchor"] is False
assert config["dspark_bonus_anchor"] is True