[LoRA] Add Qwen3-Omni multimodal LoRA support (#52786)

Signed-off-by: linitra24 <Joy25810@foxmail.com>
This commit is contained in:
linitra24
2026-08-24 05:25:23 +00:00
committed by GitHub
parent e6e1af4ca1
commit 460c08bc8a
@@ -66,7 +66,11 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.models.module_mapping import MultiModelKeys from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.model_executor.models.qwen2_audio import Qwen2AudioProcessingInfo from vllm.model_executor.models.qwen2_audio import Qwen2AudioProcessingInfo
from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalKwargsItems from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
MultiModalKwargsItem,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataItems from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataItems
from vllm.multimodal.processing.processor import ( from vllm.multimodal.processing.processor import (
MultiModalPromptUpdates, MultiModalPromptUpdates,
@@ -84,6 +88,7 @@ from vllm.v1.attention.backends.registry import AttentionBackendEnum
from .interfaces import ( from .interfaces import (
MultiModalEmbeddings, MultiModalEmbeddings,
SupportsEagle3, SupportsEagle3,
SupportsLoRA,
SupportsMRoPE, SupportsMRoPE,
SupportsMultiModal, SupportsMultiModal,
SupportsPP, SupportsPP,
@@ -202,13 +207,13 @@ class Qwen3OmniMoeAudioAttention(nn.Module):
self.scaling = self.head_dim**-0.5 self.scaling = self.head_dim**-0.5
self.qkv = QKVParallelLinear( self.qkv_proj = QKVParallelLinear(
hidden_size=self.embed_dim, hidden_size=self.embed_dim,
head_size=self.head_dim, head_size=self.head_dim,
total_num_heads=self.num_heads, total_num_heads=self.num_heads,
total_num_kv_heads=self.num_heads, total_num_kv_heads=self.num_heads,
bias=True, bias=True,
prefix=f"{prefix}.qkv", prefix=f"{prefix}.qkv_proj",
) )
self.out_proj = RowParallelLinear( self.out_proj = RowParallelLinear(
@@ -232,7 +237,7 @@ class Qwen3OmniMoeAudioAttention(nn.Module):
max_seqlen: torch.Tensor | None, max_seqlen: torch.Tensor | None,
) -> torch.Tensor: ) -> torch.Tensor:
seq_length, _ = hidden_states.size() seq_length, _ = hidden_states.size()
qkv, _ = self.qkv(hidden_states) qkv, _ = self.qkv_proj(hidden_states)
q, k, v = qkv.chunk(3, dim=-1) q, k, v = qkv.chunk(3, dim=-1)
q = q.view(1, seq_length, -1, self.head_dim) q = q.view(1, seq_length, -1, self.head_dim)
k = k.view(1, seq_length, -1, self.head_dim) k = k.view(1, seq_length, -1, self.head_dim)
@@ -323,9 +328,9 @@ class Qwen3OmniMoeAudioEncoder(nn.Module):
hf_to_vllm_mapper = WeightsMapper( hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={ orig_to_new_stacked={
".self_attn.q_proj.": (".self_attn.qkv.", "q"), ".self_attn.q_proj.": (".self_attn.qkv_proj.", "q"),
".self_attn.k_proj.": (".self_attn.qkv.", "k"), ".self_attn.k_proj.": (".self_attn.qkv_proj.", "k"),
".self_attn.v_proj.": (".self_attn.qkv.", "v"), ".self_attn.v_proj.": (".self_attn.qkv_proj.", "v"),
} }
) )
@@ -1584,12 +1589,14 @@ class Qwen3OmniMoeConditionalGenerationMixin(Qwen2_5OmniConditionalGenerationMix
class Qwen3OmniMoeThinkerForConditionalGeneration( class Qwen3OmniMoeThinkerForConditionalGeneration(
nn.Module, nn.Module,
SupportsMultiModal, SupportsMultiModal,
SupportsLoRA,
SupportsPP, SupportsPP,
SupportsMRoPE, SupportsMRoPE,
SupportsEagle3, SupportsEagle3,
Qwen3OmniMoeConditionalGenerationMixin, Qwen3OmniMoeConditionalGenerationMixin,
SupportsTranscription, SupportsTranscription,
): ):
is_3d_moe_weight: bool = True
hf_to_vllm_mapper = WeightsMapper( hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={ orig_to_new_prefix={
"thinker.lm_head.": "language_model.lm_head.", "thinker.lm_head.": "language_model.lm_head.",
@@ -1597,6 +1604,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
"thinker.": "", "thinker.": "",
"talker.": None, "talker.": None,
"code2wav.": None, "code2wav.": None,
# Adapter trained/saved directly from the thinker model
"model.": "language_model.model.",
"lm_head.": "language_model.lm_head.",
} }
) )
@@ -1610,6 +1620,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
"gate_proj", "gate_proj",
"up_proj", "up_proj",
], ],
"qkv": ["qkv"], # for visual encoder
} }
supported_languages = ISO639_1_SUPPORTED_LANGS supported_languages = ISO639_1_SUPPORTED_LANGS
@@ -1631,6 +1642,8 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
thinker_config: Qwen3OmniMoeThinkerConfig = ( thinker_config: Qwen3OmniMoeThinkerConfig = (
vllm_config.model_config.hf_config.thinker_config vllm_config.model_config.hf_config.thinker_config
) )
# MoE LoRA uses the architecture to determine its weight layout.
thinker_config.architectures = vllm_config.model_config.hf_config.architectures
quant_config = vllm_config.quant_config quant_config = vllm_config.quant_config
multimodal_config = vllm_config.model_config.multimodal_config multimodal_config = vllm_config.model_config.multimodal_config
self.config = thinker_config self.config = thinker_config
@@ -2299,3 +2312,23 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
connector="visual.merger", connector="visual.merger",
tower_model=["visual.", "audio_tower."], tower_model=["visual.", "audio_tower."],
) )
def get_mm_lora_token_counts(
self,
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
if modality in ("image", "video"):
hf_config = self.config
vision_config = hf_config.vision_config
merge_size = vision_config.spatial_merge_size
tower_tokens = num_mm_embeds * merge_size**2
connector_tokens = num_mm_embeds
if modality == "audio":
tower_tokens = num_mm_embeds
connector_tokens = num_mm_embeds
return tower_tokens, connector_tokens