[Model] Add Qwen3-Omni DSpark support (#52560)
Signed-off-by: Zhou248 <630563665@qq.com>
This commit is contained in:
@@ -1,9 +1,13 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import Mock
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.multimodal.processing import InputProcessingContext
|
||||
@@ -218,5 +222,171 @@ def test_qwen3_omni_get_updates_use_audio_in_video(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_qwen3_omni_exposes_eagle3_to_its_text_backbone():
|
||||
from vllm.model_executor.models.interfaces import EagleModelMixin, supports_eagle3
|
||||
from vllm.model_executor.models.qwen3_omni_moe_thinker import (
|
||||
Qwen3OmniMoeThinkerForConditionalGeneration,
|
||||
)
|
||||
|
||||
class DummyBackbone(nn.Module, EagleModelMixin):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.layers = nn.ModuleList([nn.Identity(), nn.Identity()])
|
||||
|
||||
class DummyLanguageModel(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.model = DummyBackbone()
|
||||
|
||||
def embed_input_ids(self, input_ids):
|
||||
return input_ids
|
||||
|
||||
model = Qwen3OmniMoeThinkerForConditionalGeneration.__new__(
|
||||
Qwen3OmniMoeThinkerForConditionalGeneration
|
||||
)
|
||||
nn.Module.__init__(model)
|
||||
model.language_model = DummyLanguageModel()
|
||||
|
||||
assert supports_eagle3(model)
|
||||
model.set_aux_hidden_state_layers((1, 2))
|
||||
assert model.language_model.model.aux_hidden_state_layers == (1, 2)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_qwen3_omni_text_model_collects_post_deepstack_aux_hidden_states():
|
||||
from vllm.model_executor.models.qwen3_omni_moe_thinker import Qwen3MoeLLMModel
|
||||
|
||||
class DummyLayer(nn.Module):
|
||||
def forward(self, positions, hidden_states, residual):
|
||||
return hidden_states + 1, torch.full_like(hidden_states, 10)
|
||||
|
||||
class DummyNorm(nn.Module):
|
||||
def forward(self, hidden_states, residual):
|
||||
return hidden_states + residual, None
|
||||
|
||||
model = Qwen3MoeLLMModel.__new__(Qwen3MoeLLMModel)
|
||||
nn.Module.__init__(model)
|
||||
model.start_layer = 0
|
||||
model.end_layer = 1
|
||||
model.layers = nn.ModuleList([DummyLayer()])
|
||||
model.norm = DummyNorm()
|
||||
model.aux_hidden_state_layers = (1,)
|
||||
|
||||
pp_group = Mock(is_first_rank=True, is_last_rank=True)
|
||||
inputs_embeds = torch.tensor([[1.0]])
|
||||
deepstack_inputs = {"deepstack_input_embeds_0": torch.tensor([[3.0]])}
|
||||
with patch(
|
||||
"vllm.model_executor.models.qwen3_omni_moe_thinker.get_pp_group",
|
||||
return_value=pp_group,
|
||||
):
|
||||
output, aux_hidden_states = model.forward(
|
||||
input_ids=None,
|
||||
positions=torch.tensor([0]),
|
||||
inputs_embeds=inputs_embeds,
|
||||
deepstack_input_embeds=deepstack_inputs,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(output, torch.tensor([[15.0]]))
|
||||
assert len(aux_hidden_states) == 1
|
||||
torch.testing.assert_close(aux_hidden_states[0], torch.tensor([[15.0]]))
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.parametrize(
|
||||
("input_vocab_size", "draft_vocab_size", "weights", "error"),
|
||||
[
|
||||
(101, 100, [], "must include embed_tokens weights"),
|
||||
(99, 99, [], "must include lm_head weights"),
|
||||
(100, 40, [], "must include lm_head weights"),
|
||||
(
|
||||
100,
|
||||
40,
|
||||
[("lm_head.weight", torch.empty(40, 8))],
|
||||
"must include a d2t mapping",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_qwen3_dspark_rejects_incomplete_vocab_weights(
|
||||
input_vocab_size, draft_vocab_size, weights, error
|
||||
):
|
||||
from vllm.model_executor.models.qwen3_dspark import Qwen3DSparkForCausalLM
|
||||
|
||||
model = Qwen3DSparkForCausalLM.__new__(Qwen3DSparkForCausalLM)
|
||||
nn.Module.__init__(model)
|
||||
object.__setattr__(
|
||||
model,
|
||||
"config",
|
||||
SimpleNamespace(
|
||||
vocab_size=input_vocab_size,
|
||||
draft_vocab_size=draft_vocab_size,
|
||||
),
|
||||
)
|
||||
object.__setattr__(model, "target_vocab_size", 100)
|
||||
|
||||
with pytest.raises(ValueError, match=error):
|
||||
model.load_weights(weights)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_dspark_shares_target_embedding_with_smaller_draft_vocabulary():
|
||||
from vllm.v1.worker.gpu.spec_decode.dspark import utils as dspark_utils
|
||||
|
||||
target_embedding = nn.Embedding(100, 8)
|
||||
draft_embedding = nn.Embedding(99, 8)
|
||||
target_model = SimpleNamespace(model=SimpleNamespace(embed_tokens=target_embedding))
|
||||
draft_model = SimpleNamespace(
|
||||
model=SimpleNamespace(embed_tokens=draft_embedding),
|
||||
has_own_embed_tokens=False,
|
||||
)
|
||||
draft_model_config = SimpleNamespace(
|
||||
hf_config=SimpleNamespace(model_type="qwen3"),
|
||||
get_vocab_size=Mock(return_value=99),
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
speculative_config=SimpleNamespace(
|
||||
draft_model_config=draft_model_config,
|
||||
attention_backend=None,
|
||||
kv_cache_dtype=None,
|
||||
),
|
||||
attention_config=SimpleNamespace(backend=None),
|
||||
cache_config=SimpleNamespace(),
|
||||
model_config=SimpleNamespace(get_vocab_size=Mock(return_value=100)),
|
||||
)
|
||||
|
||||
def fake_replace(config, **changes):
|
||||
values = vars(config).copy()
|
||||
values.update(changes)
|
||||
return SimpleNamespace(**values)
|
||||
|
||||
with (
|
||||
patch.object(dspark_utils, "replace", side_effect=fake_replace),
|
||||
patch.object(
|
||||
dspark_utils,
|
||||
"get_pp_group",
|
||||
return_value=SimpleNamespace(world_size=1),
|
||||
),
|
||||
patch(
|
||||
"vllm.compilation.backends.set_model_tag",
|
||||
return_value=nullcontext(),
|
||||
),
|
||||
patch(
|
||||
"vllm.model_executor.model_loader.get_model",
|
||||
return_value=draft_model,
|
||||
),
|
||||
patch(
|
||||
"vllm.model_executor.models.qwen3_dflash.dflash_has_any_non_causal",
|
||||
return_value=False,
|
||||
),
|
||||
patch(
|
||||
"vllm.model_executor.models.utils.get_draft_quant_config",
|
||||
return_value=None,
|
||||
),
|
||||
):
|
||||
loaded_model = dspark_utils.load_dspark_model(target_model, vllm_config)
|
||||
|
||||
assert loaded_model.model.embed_tokens is target_embedding
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
|
||||
@@ -1557,6 +1557,13 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
is_available_online=False,
|
||||
use_original_num_layers=True, # DSpark backbone requires all layers
|
||||
),
|
||||
"Qwen3OmniDSparkModel": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
speculative_model="Qwen/Qwen3-Omni-30B-A3B-DSpark",
|
||||
is_available_online=False,
|
||||
use_original_num_layers=True,
|
||||
min_transformers_version="4.57",
|
||||
),
|
||||
"Gemma4DSparkModel": _HfExamplesInfo(
|
||||
"google/gemma-4-12B-it",
|
||||
speculative_model="deepseek-ai/dspark_gemma4_12b_block7",
|
||||
|
||||
@@ -32,6 +32,7 @@ from vllm.config.compilation import CompilationMode, CUDAGraphMode
|
||||
from vllm.config.kernel import IrOpPriorityConfig
|
||||
from vllm.config.load import LoadConfig
|
||||
from vllm.config.mamba import MambaBackendEnum
|
||||
from vllm.config.speculative import _validate_qwen3_omni_dspark
|
||||
from vllm.config.utils import get_field
|
||||
from vllm.config.vllm import OPTIMIZATION_LEVEL_TO_CONFIG, OptimizationLevel
|
||||
from vllm.platforms import current_platform
|
||||
@@ -2100,6 +2101,126 @@ def test_mtp_draft_uses_model_weights_not_local_cache(mock_model_config_cls):
|
||||
assert actual_model == s3_url
|
||||
|
||||
|
||||
def _make_qwen3_omni_dspark_configs():
|
||||
text_config = SimpleNamespace(
|
||||
hidden_size=2048,
|
||||
num_hidden_layers=48,
|
||||
num_attention_heads=32,
|
||||
num_key_value_heads=4,
|
||||
head_dim=128,
|
||||
vocab_size=152064,
|
||||
)
|
||||
target_model_config = SimpleNamespace(
|
||||
hf_config=SimpleNamespace(
|
||||
model_type="qwen3_omni_moe",
|
||||
architectures=["Qwen3OmniMoeForConditionalGeneration"],
|
||||
thinker_config=SimpleNamespace(text_config=text_config),
|
||||
),
|
||||
hf_text_config=text_config,
|
||||
architectures=["Qwen3OmniMoeForConditionalGeneration"],
|
||||
get_hidden_size=lambda: 2048,
|
||||
get_total_num_hidden_layers=lambda: 48,
|
||||
get_vocab_size=lambda: 152064,
|
||||
)
|
||||
draft_hf_config = SimpleNamespace(
|
||||
model_type="qwen3",
|
||||
architectures=["Qwen3OmniDSparkModel"],
|
||||
hidden_size=2048,
|
||||
num_attention_heads=32,
|
||||
num_key_value_heads=4,
|
||||
head_dim=128,
|
||||
block_size=7,
|
||||
target_hidden_size=2048,
|
||||
target_layer_ids=[1, 9, 17, 25, 33],
|
||||
use_aux_hidden_state=True,
|
||||
markov_rank=64,
|
||||
markov_head_type="vanilla",
|
||||
sample_from_anchor=True,
|
||||
dspark_bonus_anchor=False,
|
||||
vocab_size=152064,
|
||||
draft_vocab_size=32000,
|
||||
mask_token_id=151669,
|
||||
rope_parameters={"rope_type": "default", "rope_theta": 1000000.0},
|
||||
)
|
||||
draft_model_config = SimpleNamespace(
|
||||
hf_config=draft_hf_config,
|
||||
architectures=["Qwen3OmniDSparkModel"],
|
||||
)
|
||||
return target_model_config, draft_model_config
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_qwen3_omni_dspark_checkpoint_contract_is_accepted():
|
||||
target_config, draft_config = _make_qwen3_omni_dspark_configs()
|
||||
_validate_qwen3_omni_dspark(target_config, draft_config, 7)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_qwen3_omni_dspark_rejects_generic_qwen3_architecture():
|
||||
target_config, draft_config = _make_qwen3_omni_dspark_configs()
|
||||
draft_config.hf_config.architectures = ["Qwen3DSparkModel"]
|
||||
draft_config.architectures = ["Qwen3DSparkModel"]
|
||||
|
||||
with pytest.raises(ValueError, match="must be converted first"):
|
||||
_validate_qwen3_omni_dspark(target_config, draft_config, 7)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "value", "error"),
|
||||
[
|
||||
("block_size", 5, "trained block_size"),
|
||||
("target_hidden_size", 4096, "target_hidden_size"),
|
||||
("hidden_size", 4096, "draft hidden_size"),
|
||||
("num_attention_heads", 16, "num_attention_heads"),
|
||||
("num_key_value_heads", 8, "num_key_value_heads"),
|
||||
("head_dim", 64, "head_dim"),
|
||||
("target_layer_ids", [7, 48], "zero-based text-layer"),
|
||||
("target_layer_ids", [23, 7], "strictly increasing"),
|
||||
("use_aux_hidden_state", False, "use_aux_hidden_state=true"),
|
||||
("markov_rank", 0, "markov_rank"),
|
||||
("markov_head_type", "gated", "markov_head_type='vanilla'"),
|
||||
("sample_from_anchor", False, "sample_from_anchor=true"),
|
||||
("dspark_bonus_anchor", True, "dspark_bonus_anchor=false"),
|
||||
("vocab_size", 0, "input vocab_size must be a positive integer"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_qwen3_omni_dspark_rejects_incompatible_checkpoint_fields(field, value, error):
|
||||
target_config, draft_config = _make_qwen3_omni_dspark_configs()
|
||||
setattr(draft_config.hf_config, field, value)
|
||||
|
||||
with pytest.raises(ValueError, match=error):
|
||||
_validate_qwen3_omni_dspark(target_config, draft_config, 7)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_qwen3_omni_dspark_rejects_mrope_draft_positions():
|
||||
target_config, draft_config = _make_qwen3_omni_dspark_configs()
|
||||
draft_config.hf_config.rope_parameters["mrope_section"] = [24, 20, 20]
|
||||
|
||||
with pytest.raises(ValueError, match="logical 1-D RoPE"):
|
||||
_validate_qwen3_omni_dspark(target_config, draft_config, 7)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_qwen3_omni_dspark_allows_draft_only_noise_token_row():
|
||||
target_config, draft_config = _make_qwen3_omni_dspark_configs()
|
||||
draft_config.hf_config.vocab_size = 152065
|
||||
draft_config.hf_config.draft_vocab_size = 152064
|
||||
draft_config.hf_config.mask_token_id = 152064
|
||||
|
||||
_validate_qwen3_omni_dspark(target_config, draft_config, 7)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_qwen3_omni_dspark_allows_smaller_input_vocabulary():
|
||||
target_config, draft_config = _make_qwen3_omni_dspark_configs()
|
||||
draft_config.hf_config.vocab_size = 151936
|
||||
draft_config.hf_config.mask_token_id = 151669
|
||||
|
||||
_validate_qwen3_omni_dspark(target_config, draft_config, 7)
|
||||
|
||||
|
||||
def test_ir_op_priority_default():
|
||||
"""Test that IR op priority defaults are set correctly."""
|
||||
from vllm.config.kernel import IrOpPriorityConfig
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.transformers_utils.configs.speculators.base import SpeculatorsConfig
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
|
||||
def _make_dspark_config(architecture: str) -> dict:
|
||||
return {
|
||||
"speculators_model_type": "dspark",
|
||||
"architectures": [architecture],
|
||||
"sample_from_anchor": True,
|
||||
"use_aux_hidden_state": True,
|
||||
"draft_vocab_size": 32,
|
||||
"target_hidden_size": 16,
|
||||
"mask_token_id": 31,
|
||||
"markov_rank": 4,
|
||||
"markov_head_type": "vanilla",
|
||||
"block_size": 3,
|
||||
"aux_hidden_state_layer_ids": [2, 5],
|
||||
"transformer_layer_config": {
|
||||
"model_type": "qwen3",
|
||||
"hidden_size": 16,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_dspark_updater_preserves_qwen3_omni_contract() -> None:
|
||||
config = SpeculatorsConfig.extract_transformers_pre_trained_config(
|
||||
_make_dspark_config("Qwen3OmniDSparkModel")
|
||||
)
|
||||
|
||||
assert config["architectures"] == ["Qwen3OmniDSparkModel"]
|
||||
assert config["sample_from_anchor"] is True
|
||||
assert config["dspark_bonus_anchor"] is False
|
||||
assert config["use_aux_hidden_state"] is True
|
||||
assert config["target_layer_ids"] == [1, 4]
|
||||
|
||||
|
||||
def test_dspark_updater_keeps_legacy_qwen3_checkpoint_loadable() -> None:
|
||||
config = SpeculatorsConfig.extract_transformers_pre_trained_config(
|
||||
_make_dspark_config("DSparkSpeculator")
|
||||
)
|
||||
|
||||
assert config["architectures"] == ["Qwen3DSparkModel"]
|
||||
|
||||
|
||||
def test_dspark_updater_maps_bonus_anchor_semantics() -> None:
|
||||
outer_config = _make_dspark_config("Qwen3DSparkModel")
|
||||
outer_config["sample_from_anchor"] = False
|
||||
|
||||
config = SpeculatorsConfig.extract_transformers_pre_trained_config(outer_config)
|
||||
|
||||
assert config["sample_from_anchor"] is False
|
||||
assert config["dspark_bonus_anchor"] is True
|
||||
Reference in New Issue
Block a user