[Model] Ultravox: Support Llama 4 and Gemma 3 backends (#17818)
Signed-off-by: Farzad Abdolhosseini <farzad@fixie.ai> Signed-off-by: Patrick Li <patrick8289@gmail.com> Co-authored-by: Patrick Li <patrick8289@gmail.com>
This commit is contained in:
co-authored by
Patrick Li
parent
7ae75fa6d0
commit
62965de5fe
@@ -45,6 +45,7 @@ class UltravoxConfig(transformers.PretrainedConfig):
|
||||
"""
|
||||
|
||||
model_type = "ultravox"
|
||||
audio_token = "<|audio|>"
|
||||
is_composition = False
|
||||
|
||||
def __init__(
|
||||
@@ -80,29 +81,32 @@ class UltravoxConfig(transformers.PretrainedConfig):
|
||||
# Avoid circular import
|
||||
from vllm.transformers_utils.config import get_config
|
||||
|
||||
self.text_config = get_config(text_model_id,
|
||||
trust_remote_code=False)
|
||||
text_config_obj = get_config(text_model_id,
|
||||
trust_remote_code=False)
|
||||
else:
|
||||
text_config = text_config or {}
|
||||
self.text_config = transformers.CONFIG_MAPPING[text_config.get(
|
||||
text_config_obj = transformers.CONFIG_MAPPING[text_config.get(
|
||||
"model_type", "llama")](**text_config)
|
||||
|
||||
inner_text_config = text_config_obj.get_text_config()
|
||||
|
||||
if audio_model_id is not None:
|
||||
# Avoid circular import
|
||||
from vllm.transformers_utils.config import get_config
|
||||
|
||||
self.audio_config = get_config(audio_model_id,
|
||||
trust_remote_code=False)
|
||||
audio_config = get_config(audio_model_id, trust_remote_code=False)
|
||||
else:
|
||||
audio_config = audio_config or {}
|
||||
self.audio_config = transformers.CONFIG_MAPPING[audio_config.get(
|
||||
audio_config = transformers.CONFIG_MAPPING[audio_config.get(
|
||||
"model_type", "whisper")](**audio_config)
|
||||
|
||||
self.text_config = text_config_obj
|
||||
self.audio_config = audio_config
|
||||
self.text_model_lora_config = text_model_lora_config or {}
|
||||
self.audio_model_lora_config = audio_model_lora_config or {}
|
||||
|
||||
self.vocab_size = self.text_config.vocab_size
|
||||
|
||||
self.initializer_range = self.text_config.initializer_range
|
||||
self.vocab_size = inner_text_config.vocab_size
|
||||
self.initializer_range = inner_text_config.initializer_range
|
||||
self.text_hidden_size = inner_text_config.hidden_size
|
||||
|
||||
super().__init__(**kwargs)
|
||||
|
||||
Reference in New Issue
Block a user