[Model] Implement LoRA support for Qwen3ASRForConditionalGeneration (#37247)
Signed-off-by: Peter Nguyen <petern0408@gmail.com>
This commit is contained in:
@@ -37,6 +37,7 @@ from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TokensProm
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsLoRA,
|
||||
SupportsMRoPE,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
@@ -266,7 +267,21 @@ class Qwen3ASRForConditionalGeneration(
|
||||
SupportsPP,
|
||||
SupportsMRoPE,
|
||||
SupportsTranscription,
|
||||
SupportsLoRA,
|
||||
):
|
||||
# LoRA support
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": [
|
||||
"q_proj",
|
||||
"k_proj",
|
||||
"v_proj",
|
||||
],
|
||||
"gate_up_proj": [
|
||||
"gate_proj",
|
||||
"up_proj",
|
||||
],
|
||||
}
|
||||
|
||||
supported_languages = ISO639_1_SUPPORTED_LANGS
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
@@ -513,6 +528,17 @@ class Qwen3ASRForConditionalGeneration(
|
||||
tower_model=["audio_tower."],
|
||||
)
|
||||
|
||||
def get_num_mm_encoder_tokens(self, num_audio_tokens: int) -> int:
|
||||
"""Return the number of tokens processed by the audio tower encoder.
|
||||
|
||||
Required for LoRA support on the tower module.
|
||||
"""
|
||||
# For Qwen3-ASR, the audio tower produces one embedding per audio
|
||||
# placeholder token inserted into the prompt (no additional
|
||||
# merge/downsample step like vision towers). Therefore, the encoder
|
||||
# token budget is identity.
|
||||
return num_audio_tokens
|
||||
|
||||
@classmethod
|
||||
def get_speech_to_text_config(
|
||||
cls, model_config: ModelConfig, task_type: str
|
||||
|
||||
@@ -57,6 +57,7 @@ from vllm.model_executor.layers.conv import Conv3dLayer
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
@@ -357,7 +358,13 @@ class Qwen3OmniMoeAudioEncoder(nn.Module):
|
||||
conv_out_dim = config.downsample_hidden_size * (
|
||||
(((config.num_mel_bins + 1) // 2 + 1) // 2 + 1) // 2
|
||||
)
|
||||
self.conv_out = nn.Linear(conv_out_dim, config.d_model, bias=False)
|
||||
self.conv_out = ReplicatedLinear(
|
||||
conv_out_dim,
|
||||
config.d_model,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
prefix=f"{prefix}.conv_out",
|
||||
)
|
||||
|
||||
# Transformer encoder layers
|
||||
self.layers = nn.ModuleList(
|
||||
@@ -372,9 +379,21 @@ class Qwen3OmniMoeAudioEncoder(nn.Module):
|
||||
|
||||
# Output layers
|
||||
self.ln_post = nn.LayerNorm(config.d_model)
|
||||
self.proj1 = nn.Linear(config.d_model, config.d_model)
|
||||
self.proj1 = ReplicatedLinear(
|
||||
config.d_model,
|
||||
config.d_model,
|
||||
bias=True,
|
||||
return_bias=False,
|
||||
prefix=f"{prefix}.proj1",
|
||||
)
|
||||
self.act = _ACTIVATION_REGISTRY[config.activation_function]
|
||||
self.proj2 = nn.Linear(config.d_model, config.output_dim)
|
||||
self.proj2 = ReplicatedLinear(
|
||||
config.d_model,
|
||||
config.output_dim,
|
||||
bias=True,
|
||||
return_bias=False,
|
||||
prefix=f"{prefix}.proj2",
|
||||
)
|
||||
|
||||
# Get attention backend
|
||||
self.attn_backend = get_vit_attn_backend(
|
||||
|
||||
Reference in New Issue
Block a user