[CI] Stabilize the multi-audio OpenAI server path (#44051)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -406,6 +406,7 @@ ModalityStr = Literal[
|
||||
"prompt_embeds",
|
||||
]
|
||||
_T = TypeVar("_T")
|
||||
_AsyncMultiModalItem: TypeAlias = Callable[[], Awaitable[tuple[object, str | None]]]
|
||||
|
||||
|
||||
# Backward compatibility for single item input
|
||||
@@ -596,10 +597,27 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
|
||||
or model-specific placeholder logic. The corresponding placeholder string is
|
||||
managed by the parser via `_add_placeholder`, so we return None here.
|
||||
"""
|
||||
if modality == "prompt_embeds":
|
||||
add_info = self._validate_add(modality)
|
||||
if add_info is None:
|
||||
self._items_by_modality["prompt_embeds"].append(item)
|
||||
return None
|
||||
|
||||
input_modality, original_modality, use_vision_chunk, num_items = add_info
|
||||
|
||||
# Track original modality for vision_chunk items
|
||||
if use_vision_chunk:
|
||||
self._items_by_modality[input_modality].append(item) # type: ignore
|
||||
self._modality_order["vision_chunk"].append(original_modality)
|
||||
else:
|
||||
self._items_by_modality[original_modality].append(item)
|
||||
|
||||
return self.model_cls.get_placeholder_str(modality, num_items)
|
||||
|
||||
def _validate_add(self, modality: ModalityStr) -> tuple[str, str, bool, int] | None:
|
||||
"""Validate that one more item of the modality can be tracked."""
|
||||
if modality == "prompt_embeds":
|
||||
return None
|
||||
|
||||
input_modality = modality.replace("_embeds", "")
|
||||
original_modality = modality
|
||||
use_vision_chunk = (
|
||||
@@ -630,14 +648,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
|
||||
else:
|
||||
self.mm_processor.info.validate_num_items(input_modality, num_items)
|
||||
|
||||
# Track original modality for vision_chunk items
|
||||
if use_vision_chunk:
|
||||
self._items_by_modality[input_modality].append(item) # type: ignore
|
||||
self._modality_order["vision_chunk"].append(original_modality)
|
||||
else:
|
||||
self._items_by_modality[original_modality].append(item)
|
||||
|
||||
return self.model_cls.get_placeholder_str(modality, num_items)
|
||||
return input_modality, original_modality, use_vision_chunk, num_items
|
||||
|
||||
@abstractmethod
|
||||
def create_parser(
|
||||
@@ -803,9 +814,7 @@ class MultiModalItemTracker(BaseMultiModalItemTracker[tuple[object, str | None]]
|
||||
return MultiModalContentParser(self, mm_processor_kwargs=mm_processor_kwargs)
|
||||
|
||||
|
||||
class AsyncMultiModalItemTracker(
|
||||
BaseMultiModalItemTracker[Awaitable[tuple[object, str | None]]]
|
||||
):
|
||||
class AsyncMultiModalItemTracker(BaseMultiModalItemTracker[_AsyncMultiModalItem]):
|
||||
async def resolve_items(
|
||||
self,
|
||||
) -> tuple[MultiModalDataDict | None, MultiModalUUIDDict | None]:
|
||||
@@ -813,8 +822,8 @@ class AsyncMultiModalItemTracker(
|
||||
return None, None
|
||||
|
||||
resolved_items_by_modality = {
|
||||
modality: await asyncio.gather(*coros)
|
||||
for modality, coros in self._items_by_modality.items()
|
||||
modality: await asyncio.gather(*(item() for item in items))
|
||||
for modality, items in self._items_by_modality.items()
|
||||
}
|
||||
|
||||
mm_processor = (
|
||||
@@ -1074,6 +1083,9 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
def model_config(self) -> ModelConfig:
|
||||
return self._tracker.model_config
|
||||
|
||||
async def _item_with_uuid_async(self, item: object, uuid: str | None):
|
||||
return item, uuid
|
||||
|
||||
@override
|
||||
def parse_prompt_embeds(self, data: str) -> None:
|
||||
"""Schedule async prompt embeds decode and store the coroutine in the tracker.
|
||||
@@ -1085,8 +1097,9 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
if not self.model_config.enable_prompt_embeds:
|
||||
raise ValueError(_ENABLE_PROMPT_EMBEDS_ERROR)
|
||||
|
||||
coro = self._load_prompt_embeds_async(data.encode())
|
||||
self._tracker.add("prompt_embeds", coro)
|
||||
self._tracker.add(
|
||||
"prompt_embeds", partial(self._load_prompt_embeds_async, data.encode())
|
||||
)
|
||||
self._add_placeholder("prompt_embeds", PROMPT_EMBEDS_PLACEHOLDER_TOKEN)
|
||||
|
||||
async def _load_prompt_embeds_async(
|
||||
@@ -1104,9 +1117,9 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
return image, uuid
|
||||
|
||||
def parse_image(self, image_url: str | None, uuid: str | None = None) -> None:
|
||||
coro = self._image_with_uuid_async(image_url, uuid)
|
||||
|
||||
placeholder = self._tracker.add("image", coro)
|
||||
placeholder = self._tracker.add(
|
||||
"image", partial(self._image_with_uuid_async, image_url, uuid)
|
||||
)
|
||||
self._add_placeholder("image", placeholder)
|
||||
|
||||
def parse_image_embeds(
|
||||
@@ -1120,25 +1133,20 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
"You must set `--enable-mm-embeds` to input `image_embeds`"
|
||||
)
|
||||
|
||||
future = asyncio.Future[
|
||||
tuple[torch.Tensor | dict[str, torch.Tensor] | None, str | None]
|
||||
]()
|
||||
|
||||
if isinstance(image_embeds, dict):
|
||||
embeds = {
|
||||
k: self._connector.fetch_image_embedding(v)
|
||||
for k, v in image_embeds.items()
|
||||
}
|
||||
future.set_result((embeds, uuid))
|
||||
|
||||
if isinstance(image_embeds, str):
|
||||
elif isinstance(image_embeds, str):
|
||||
embedding = self._connector.fetch_image_embedding(image_embeds)
|
||||
future.set_result((embedding, uuid))
|
||||
embeds = embedding
|
||||
else:
|
||||
embeds = None
|
||||
|
||||
if image_embeds is None:
|
||||
future.set_result((None, uuid))
|
||||
|
||||
placeholder = self._tracker.add("image_embeds", future)
|
||||
placeholder = self._tracker.add(
|
||||
"image_embeds", partial(self._item_with_uuid_async, embeds, uuid)
|
||||
)
|
||||
self._add_placeholder("image", placeholder)
|
||||
|
||||
def parse_audio_embeds(
|
||||
@@ -1152,25 +1160,20 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
"You must set `--enable-mm-embeds` to input `audio_embeds`"
|
||||
)
|
||||
|
||||
future = asyncio.Future[
|
||||
tuple[torch.Tensor | dict[str, torch.Tensor] | None, str | None]
|
||||
]()
|
||||
|
||||
if isinstance(audio_embeds, dict):
|
||||
embeds = {
|
||||
k: self._connector.fetch_audio_embedding(v)
|
||||
for k, v in audio_embeds.items()
|
||||
}
|
||||
future.set_result((embeds, uuid))
|
||||
|
||||
if isinstance(audio_embeds, str):
|
||||
elif isinstance(audio_embeds, str):
|
||||
embedding = self._connector.fetch_audio_embedding(audio_embeds)
|
||||
future.set_result((embedding, uuid))
|
||||
embeds = embedding
|
||||
else:
|
||||
embeds = None
|
||||
|
||||
if audio_embeds is None:
|
||||
future.set_result((None, uuid))
|
||||
|
||||
placeholder = self._tracker.add("audio_embeds", future)
|
||||
placeholder = self._tracker.add(
|
||||
"audio_embeds", partial(self._item_with_uuid_async, embeds, uuid)
|
||||
)
|
||||
self._add_placeholder("audio", placeholder)
|
||||
|
||||
def parse_image_pil(
|
||||
@@ -1178,13 +1181,9 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
image_pil: Image.Image | None,
|
||||
uuid: str | None = None,
|
||||
) -> None:
|
||||
future = asyncio.Future[tuple[Image.Image | None, str | None]]()
|
||||
if image_pil:
|
||||
future.set_result((image_pil, uuid))
|
||||
else:
|
||||
future.set_result((None, uuid))
|
||||
|
||||
placeholder = self._tracker.add("image", future)
|
||||
placeholder = self._tracker.add(
|
||||
"image", partial(self._item_with_uuid_async, image_pil, uuid)
|
||||
)
|
||||
self._add_placeholder("image", placeholder)
|
||||
|
||||
async def _audio_with_uuid_async(self, audio_url: str | None, uuid: str | None):
|
||||
@@ -1194,9 +1193,9 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
return audio, uuid
|
||||
|
||||
def parse_audio(self, audio_url: str | None, uuid: str | None = None) -> None:
|
||||
coro = self._audio_with_uuid_async(audio_url, uuid)
|
||||
|
||||
placeholder = self._tracker.add("audio", coro)
|
||||
placeholder = self._tracker.add(
|
||||
"audio", partial(self._audio_with_uuid_async, audio_url, uuid)
|
||||
)
|
||||
self._add_placeholder("audio", placeholder)
|
||||
|
||||
def parse_input_audio(
|
||||
@@ -1227,9 +1226,9 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
return video, uuid
|
||||
|
||||
def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
|
||||
coro = self._video_with_uuid_async(video_url, uuid)
|
||||
|
||||
placeholder = self._tracker.add("video", coro)
|
||||
placeholder = self._tracker.add(
|
||||
"video", partial(self._video_with_uuid_async, video_url, uuid)
|
||||
)
|
||||
self._add_placeholder("video", placeholder)
|
||||
|
||||
# Extract audio from video if use_audio_in_video is True
|
||||
@@ -1238,8 +1237,9 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
and self._mm_processor_kwargs
|
||||
and self._mm_processor_kwargs.get("use_audio_in_video", False)
|
||||
):
|
||||
audio_coro = self._audio_with_uuid_async(video_url, uuid)
|
||||
audio_placeholder = self._tracker.add("audio", audio_coro)
|
||||
audio_placeholder = self._tracker.add(
|
||||
"audio", partial(self._audio_with_uuid_async, video_url, uuid)
|
||||
)
|
||||
self._add_placeholder("audio", audio_placeholder)
|
||||
|
||||
|
||||
|
||||
@@ -52,6 +52,7 @@ from vllm.entrypoints.serve.utils.server_utils import (
|
||||
log_response,
|
||||
validation_exception_handler,
|
||||
)
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.reasoning import ReasoningParserManager
|
||||
from vllm.tasks import POOLING_TASKS, SupportedTask
|
||||
@@ -250,6 +251,7 @@ def build_app(
|
||||
app.exception_handler(EngineGenerateError)(engine_error_handler)
|
||||
app.exception_handler(EngineDeadError)(engine_error_handler)
|
||||
app.exception_handler(GenerationError)(generation_error_handler)
|
||||
app.exception_handler(VLLMValidationError)(exception_handler)
|
||||
app.exception_handler(Exception)(exception_handler)
|
||||
|
||||
# Ensure --api-key option from CLI takes precedence over VLLM_API_KEY
|
||||
|
||||
@@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Any, overload
|
||||
import torch
|
||||
from typing_extensions import TypeVar
|
||||
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.logger import init_logger
|
||||
from vllm.multimodal.parse import (
|
||||
@@ -424,7 +425,7 @@ class BaseProcessingInfo:
|
||||
if num_items <= supported_limit:
|
||||
msg += " Set `--limit-mm-per-prompt` to increase this limit."
|
||||
|
||||
raise ValueError(msg)
|
||||
raise VLLMValidationError(msg, parameter=modality)
|
||||
|
||||
def parse_mm_data(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user