[CI] Stabilize the multi-audio OpenAI server path (#44051)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-06-07 15:54:32 +08:00
committed by GitHub
parent 15652a6b70
commit f0f6805d8a
3 changed files with 62 additions and 59 deletions
+58 -58
View File
@@ -406,6 +406,7 @@ ModalityStr = Literal[
"prompt_embeds",
]
_T = TypeVar("_T")
_AsyncMultiModalItem: TypeAlias = Callable[[], Awaitable[tuple[object, str | None]]]
# Backward compatibility for single item input
@@ -596,10 +597,27 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
or model-specific placeholder logic. The corresponding placeholder string is
managed by the parser via `_add_placeholder`, so we return None here.
"""
if modality == "prompt_embeds":
add_info = self._validate_add(modality)
if add_info is None:
self._items_by_modality["prompt_embeds"].append(item)
return None
input_modality, original_modality, use_vision_chunk, num_items = add_info
# Track original modality for vision_chunk items
if use_vision_chunk:
self._items_by_modality[input_modality].append(item) # type: ignore
self._modality_order["vision_chunk"].append(original_modality)
else:
self._items_by_modality[original_modality].append(item)
return self.model_cls.get_placeholder_str(modality, num_items)
def _validate_add(self, modality: ModalityStr) -> tuple[str, str, bool, int] | None:
"""Validate that one more item of the modality can be tracked."""
if modality == "prompt_embeds":
return None
input_modality = modality.replace("_embeds", "")
original_modality = modality
use_vision_chunk = (
@@ -630,14 +648,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
else:
self.mm_processor.info.validate_num_items(input_modality, num_items)
# Track original modality for vision_chunk items
if use_vision_chunk:
self._items_by_modality[input_modality].append(item) # type: ignore
self._modality_order["vision_chunk"].append(original_modality)
else:
self._items_by_modality[original_modality].append(item)
return self.model_cls.get_placeholder_str(modality, num_items)
return input_modality, original_modality, use_vision_chunk, num_items
@abstractmethod
def create_parser(
@@ -803,9 +814,7 @@ class MultiModalItemTracker(BaseMultiModalItemTracker[tuple[object, str | None]]
return MultiModalContentParser(self, mm_processor_kwargs=mm_processor_kwargs)
class AsyncMultiModalItemTracker(
BaseMultiModalItemTracker[Awaitable[tuple[object, str | None]]]
):
class AsyncMultiModalItemTracker(BaseMultiModalItemTracker[_AsyncMultiModalItem]):
async def resolve_items(
self,
) -> tuple[MultiModalDataDict | None, MultiModalUUIDDict | None]:
@@ -813,8 +822,8 @@ class AsyncMultiModalItemTracker(
return None, None
resolved_items_by_modality = {
modality: await asyncio.gather(*coros)
for modality, coros in self._items_by_modality.items()
modality: await asyncio.gather(*(item() for item in items))
for modality, items in self._items_by_modality.items()
}
mm_processor = (
@@ -1074,6 +1083,9 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
def model_config(self) -> ModelConfig:
return self._tracker.model_config
async def _item_with_uuid_async(self, item: object, uuid: str | None):
return item, uuid
@override
def parse_prompt_embeds(self, data: str) -> None:
"""Schedule async prompt embeds decode and store the coroutine in the tracker.
@@ -1085,8 +1097,9 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
if not self.model_config.enable_prompt_embeds:
raise ValueError(_ENABLE_PROMPT_EMBEDS_ERROR)
coro = self._load_prompt_embeds_async(data.encode())
self._tracker.add("prompt_embeds", coro)
self._tracker.add(
"prompt_embeds", partial(self._load_prompt_embeds_async, data.encode())
)
self._add_placeholder("prompt_embeds", PROMPT_EMBEDS_PLACEHOLDER_TOKEN)
async def _load_prompt_embeds_async(
@@ -1104,9 +1117,9 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
return image, uuid
def parse_image(self, image_url: str | None, uuid: str | None = None) -> None:
coro = self._image_with_uuid_async(image_url, uuid)
placeholder = self._tracker.add("image", coro)
placeholder = self._tracker.add(
"image", partial(self._image_with_uuid_async, image_url, uuid)
)
self._add_placeholder("image", placeholder)
def parse_image_embeds(
@@ -1120,25 +1133,20 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
"You must set `--enable-mm-embeds` to input `image_embeds`"
)
future = asyncio.Future[
tuple[torch.Tensor | dict[str, torch.Tensor] | None, str | None]
]()
if isinstance(image_embeds, dict):
embeds = {
k: self._connector.fetch_image_embedding(v)
for k, v in image_embeds.items()
}
future.set_result((embeds, uuid))
if isinstance(image_embeds, str):
elif isinstance(image_embeds, str):
embedding = self._connector.fetch_image_embedding(image_embeds)
future.set_result((embedding, uuid))
embeds = embedding
else:
embeds = None
if image_embeds is None:
future.set_result((None, uuid))
placeholder = self._tracker.add("image_embeds", future)
placeholder = self._tracker.add(
"image_embeds", partial(self._item_with_uuid_async, embeds, uuid)
)
self._add_placeholder("image", placeholder)
def parse_audio_embeds(
@@ -1152,25 +1160,20 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
"You must set `--enable-mm-embeds` to input `audio_embeds`"
)
future = asyncio.Future[
tuple[torch.Tensor | dict[str, torch.Tensor] | None, str | None]
]()
if isinstance(audio_embeds, dict):
embeds = {
k: self._connector.fetch_audio_embedding(v)
for k, v in audio_embeds.items()
}
future.set_result((embeds, uuid))
if isinstance(audio_embeds, str):
elif isinstance(audio_embeds, str):
embedding = self._connector.fetch_audio_embedding(audio_embeds)
future.set_result((embedding, uuid))
embeds = embedding
else:
embeds = None
if audio_embeds is None:
future.set_result((None, uuid))
placeholder = self._tracker.add("audio_embeds", future)
placeholder = self._tracker.add(
"audio_embeds", partial(self._item_with_uuid_async, embeds, uuid)
)
self._add_placeholder("audio", placeholder)
def parse_image_pil(
@@ -1178,13 +1181,9 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
image_pil: Image.Image | None,
uuid: str | None = None,
) -> None:
future = asyncio.Future[tuple[Image.Image | None, str | None]]()
if image_pil:
future.set_result((image_pil, uuid))
else:
future.set_result((None, uuid))
placeholder = self._tracker.add("image", future)
placeholder = self._tracker.add(
"image", partial(self._item_with_uuid_async, image_pil, uuid)
)
self._add_placeholder("image", placeholder)
async def _audio_with_uuid_async(self, audio_url: str | None, uuid: str | None):
@@ -1194,9 +1193,9 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
return audio, uuid
def parse_audio(self, audio_url: str | None, uuid: str | None = None) -> None:
coro = self._audio_with_uuid_async(audio_url, uuid)
placeholder = self._tracker.add("audio", coro)
placeholder = self._tracker.add(
"audio", partial(self._audio_with_uuid_async, audio_url, uuid)
)
self._add_placeholder("audio", placeholder)
def parse_input_audio(
@@ -1227,9 +1226,9 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
return video, uuid
def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
coro = self._video_with_uuid_async(video_url, uuid)
placeholder = self._tracker.add("video", coro)
placeholder = self._tracker.add(
"video", partial(self._video_with_uuid_async, video_url, uuid)
)
self._add_placeholder("video", placeholder)
# Extract audio from video if use_audio_in_video is True
@@ -1238,8 +1237,9 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
and self._mm_processor_kwargs
and self._mm_processor_kwargs.get("use_audio_in_video", False)
):
audio_coro = self._audio_with_uuid_async(video_url, uuid)
audio_placeholder = self._tracker.add("audio", audio_coro)
audio_placeholder = self._tracker.add(
"audio", partial(self._audio_with_uuid_async, video_url, uuid)
)
self._add_placeholder("audio", audio_placeholder)
+2
View File
@@ -52,6 +52,7 @@ from vllm.entrypoints.serve.utils.server_utils import (
log_response,
validation_exception_handler,
)
from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from vllm.reasoning import ReasoningParserManager
from vllm.tasks import POOLING_TASKS, SupportedTask
@@ -250,6 +251,7 @@ def build_app(
app.exception_handler(EngineGenerateError)(engine_error_handler)
app.exception_handler(EngineDeadError)(engine_error_handler)
app.exception_handler(GenerationError)(generation_error_handler)
app.exception_handler(VLLMValidationError)(exception_handler)
app.exception_handler(Exception)(exception_handler)
# Ensure --api-key option from CLI takes precedence over VLLM_API_KEY
+2 -1
View File
@@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Any, overload
import torch
from typing_extensions import TypeVar
from vllm.exceptions import VLLMValidationError
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from vllm.multimodal.parse import (
@@ -424,7 +425,7 @@ class BaseProcessingInfo:
if num_items <= supported_limit:
msg += " Set `--limit-mm-per-prompt` to increase this limit."
raise ValueError(msg)
raise VLLMValidationError(msg, parameter=modality)
def parse_mm_data(
self,