[Model] Ultravox: Support Llama 4 and Gemma 3 backends (#17818)

Signed-off-by: Farzad Abdolhosseini <farzad@fixie.ai>
Signed-off-by: Patrick Li <patrick8289@gmail.com>
Co-authored-by: Patrick Li <patrick8289@gmail.com>
This commit is contained in:
Farzad Abdolhosseini
2025-07-25 18:12:31 -07:00
committed by GitHub
co-authored by Patrick Li
parent 7ae75fa6d0
commit 62965de5fe
4 changed files with 39 additions and 24 deletions
+13 -9
View File
@@ -45,6 +45,7 @@ class UltravoxConfig(transformers.PretrainedConfig):
"""
model_type = "ultravox"
audio_token = "<|audio|>"
is_composition = False
def __init__(
@@ -80,29 +81,32 @@ class UltravoxConfig(transformers.PretrainedConfig):
# Avoid circular import
from vllm.transformers_utils.config import get_config
self.text_config = get_config(text_model_id,
trust_remote_code=False)
text_config_obj = get_config(text_model_id,
trust_remote_code=False)
else:
text_config = text_config or {}
self.text_config = transformers.CONFIG_MAPPING[text_config.get(
text_config_obj = transformers.CONFIG_MAPPING[text_config.get(
"model_type", "llama")](**text_config)
inner_text_config = text_config_obj.get_text_config()
if audio_model_id is not None:
# Avoid circular import
from vllm.transformers_utils.config import get_config
self.audio_config = get_config(audio_model_id,
trust_remote_code=False)
audio_config = get_config(audio_model_id, trust_remote_code=False)
else:
audio_config = audio_config or {}
self.audio_config = transformers.CONFIG_MAPPING[audio_config.get(
audio_config = transformers.CONFIG_MAPPING[audio_config.get(
"model_type", "whisper")](**audio_config)
self.text_config = text_config_obj
self.audio_config = audio_config
self.text_model_lora_config = text_model_lora_config or {}
self.audio_model_lora_config = audio_model_lora_config or {}
self.vocab_size = self.text_config.vocab_size
self.initializer_range = self.text_config.initializer_range
self.vocab_size = inner_text_config.vocab_size
self.initializer_range = inner_text_config.initializer_range
self.text_hidden_size = inner_text_config.hidden_size
super().__init__(**kwargs)