[Model] Ultravox: Support Llama 4 and Gemma 3 backends (#17818)

Signed-off-by: Farzad Abdolhosseini <farzad@fixie.ai>
Signed-off-by: Patrick Li <patrick8289@gmail.com>
Co-authored-by: Patrick Li <patrick8289@gmail.com>
This commit is contained in:
Farzad Abdolhosseini
2025-07-25 18:12:31 -07:00
committed by GitHub
co-authored by Patrick Li
parent 7ae75fa6d0
commit 62965de5fe
4 changed files with 39 additions and 24 deletions
+1
View File
@@ -89,6 +89,7 @@ _TEXT_GENERATION_MODELS = {
"JAISLMHeadModel": ("jais", "JAISLMHeadModel"),
"JambaForCausalLM": ("jamba", "JambaForCausalLM"),
"LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
"Llama4ForCausalLM": ("llama4", "Llama4ForCausalLM"), # noqa: E501
# For decapoda-research/llama-*
"LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
"MambaForCausalLM": ("mamba", "MambaForCausalLM"),
+23 -15
View File
@@ -39,9 +39,7 @@ from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
merge_multimodal_embeddings,
merge_multimodal_embeddings_from_map)
_AUDIO_PLACEHOLDER_OVERRIDE = "<|reserved_special_token_0|>"
_AUDIO_PLACEHOLDER_TOKEN = 128002
_AUDIO_TOKENS_PER_SECOND = 6.25
_AUDIO_PLACEHOLDER_OVERRIDE = "<|audio|>"
_MAX_ENCODER_BATCH_SIZE = 16
@@ -80,14 +78,15 @@ class UltravoxProcessingInfo(BaseProcessingInfo):
sampling_rate: Optional[int] = None,
**kwargs: object,
) -> ProcessorMixin:
config = self.ctx.model_config.hf_config
hf_processor = self.ctx.get_hf_processor(**kwargs)
# NOTE: Ultravox processing definition uses '<|eot_id|>' as the
# placeholder that will cause confusion with the actual end of turn
# token, thus we override placeholder with a reserved special
# token.
# token, thus we override placeholder with a reserved token.
hf_processor.audio_token_replacement = _AUDIO_PLACEHOLDER_OVERRIDE
hf_processor.audio_replacement_token_id = _AUDIO_PLACEHOLDER_TOKEN
hf_processor.audio_replacement_token_id = config.audio_token_index
return hf_processor
def get_feature_extractor(
@@ -274,7 +273,7 @@ class UltravoxProjector(nn.Module):
else:
self.act = get_act_fn(config.projector_act)
dim_out = config.text_config.hidden_size
dim_out = config.text_hidden_size
self.linear_2 = nn.Linear(dim_mid, dim_out, bias=False)
# Ultravox v0.4.1 and below use layer_norm after the second linear layer
@@ -572,9 +571,14 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
input_ids: torch.Tensor,
multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
) -> torch.Tensor:
inputs_embeds = self.language_model.get_input_embeddings(input_ids)
if multimodal_embeddings is not None \
and len(multimodal_embeddings) != 0:
# The audio token index is not included in the embedding table
# We need to remove it before embedding lookup
safe_input_ids = input_ids.clone()
safe_input_ids[safe_input_ids == self.config.audio_token_index] = 0
inputs_embeds = self.language_model.get_input_embeddings(
safe_input_ids)
if multimodal_embeddings is not None and len(
multimodal_embeddings) > 0:
# TODO(ywang96): remove this block after v0 is deprecated.
if not envs.VLLM_USE_V1:
@@ -585,7 +589,7 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
else:
inputs_embeds = merge_multimodal_embeddings(
input_ids, inputs_embeds, multimodal_embeddings,
_AUDIO_PLACEHOLDER_TOKEN)
self.config.audio_token_index)
return inputs_embeds
def forward(self,
@@ -623,10 +627,14 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
multimodal_embeddings)
input_ids = None
hidden_states = self.language_model.model(input_ids,
positions,
intermediate_tensors,
inputs_embeds=inputs_embeds)
language_model = self.language_model
if hasattr(language_model, "language_model"):
language_model = language_model.language_model
hidden_states = language_model.model(input_ids,
positions,
intermediate_tensors,
inputs_embeds=inputs_embeds)
return hidden_states
def compute_logits(self, hidden_states: torch.Tensor,