From 0730414999343e722590ace615d5814c7e5b6827 Mon Sep 17 00:00:00 2001 From: jeremyteboul <80506730+jeremyteboul@users.noreply.github.com> Date: Thu, 20 Nov 2025 19:39:47 -0800 Subject: [PATCH] [Core] Add audio_embeds support to chat completions (#29059) Signed-off-by: Jeremy Teboul Co-authored-by: Jeremy Teboul --- docs/features/multimodal_inputs.md | 32 ++++++ tests/entrypoints/test_chat_utils.py | 145 ++++++++++++++++++++++++++ vllm/entrypoints/chat_utils.py | 149 ++++++++++++++++++++++++++- vllm/multimodal/audio.py | 24 +++++ vllm/multimodal/utils.py | 13 ++- 5 files changed, 360 insertions(+), 3 deletions(-) diff --git a/docs/features/multimodal_inputs.md b/docs/features/multimodal_inputs.md index 5f684604e6..4656ee43ea 100644 --- a/docs/features/multimodal_inputs.md +++ b/docs/features/multimodal_inputs.md @@ -365,6 +365,8 @@ You must enable this feature via `enable_mm_embeds=True`. The vLLM engine may crash if incorrect shape of embeddings is passed. Only enable this flag for trusted users! +#### Image Embeddings + ??? code ```python @@ -441,6 +443,36 @@ For Qwen2-VL and MiniCPM-V, we accept additional parameters alongside the embedd print(generated_text) ``` +#### Audio Embeddings + +You can pass pre-computed audio embeddings similar to image embeddings: + +??? code + + ```python + from vllm import LLM + import torch + + # Enable audio embeddings support + llm = LLM(model="fixie-ai/ultravox-v0_5-llama-3_2-1b", enable_mm_embeds=True) + + # Refer to the HuggingFace repo for the correct format to use + prompt = "USER: