[Frontend] Add video embeds input support (#54242)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-08-31 03:20:47 +00:00
committed by GitHub
parent 5707355209
commit 555ea65e8c
8 changed files with 461 additions and 24 deletions
+115
View File
@@ -178,6 +178,22 @@ class ChatCompletionContentPartAudioEmbedsParam(TypedDict, total=False):
"""
class ChatCompletionContentPartVideoEmbedsParam(TypedDict, total=False):
video_embeds: str | dict[str, str] | None
"""
The video embeddings. It can be either:
- A single base64 string representing a serialized torch tensor.
- A dictionary where each value is a base64 string.
"""
type: Required[Literal["video_embeds"]]
"""The type of the content part."""
uuid: str | None
"""
User-provided UUID of a media. User must guarantee that it is properly
generated and unique for different medias.
"""
class ChatCompletionContentPartPromptEmbedsParam(TypedDict, total=False):
data: Required[str]
"""
@@ -324,6 +340,7 @@ ChatCompletionContentPartParam: TypeAlias = (
| CustomChatCompletionContentSimpleImageParam
| ChatCompletionContentPartImageEmbedsParam
| ChatCompletionContentPartAudioEmbedsParam
| ChatCompletionContentPartVideoEmbedsParam
| ChatCompletionContentPartPromptEmbedsParam
| CustomChatCompletionContentSimpleAudioParam
| CustomChatCompletionContentSimpleVideoParam
@@ -415,6 +432,7 @@ ModalityStr = Literal[
"video",
"image_embeds",
"audio_embeds",
"video_embeds",
"vision_chunk",
"prompt_embeds",
]
@@ -755,6 +773,11 @@ def _resolve_items(
"Mixing raw audio and embedding inputs is not allowed",
parameter="audio_embeds",
)
if "video" in items_by_modality and "video_embeds" in items_by_modality:
raise VLLMValidationError(
"Mixing raw video and embedding inputs is not allowed",
parameter="video_embeds",
)
# `prompt_embeds` bypasses HF MM processors. Every other modality requires one.
processor_modalities = items_by_modality.keys() - {"prompt_embeds"}
if processor_modalities and mm_processor is None:
@@ -789,6 +812,14 @@ def _resolve_items(
if "video" in items_by_modality:
mm_data["video"] = [data for data, uuid in items_by_modality["video"]]
mm_uuids["video"] = [uuid for data, uuid in items_by_modality["video"]]
if "video_embeds" in items_by_modality:
assert mm_processor is not None
mm_data["video"] = _get_embeds_data(
"video",
[data for data, uuid in items_by_modality["video_embeds"]],
mm_processor,
)
mm_uuids["video"] = [uuid for data, uuid in items_by_modality["video_embeds"]]
if "vision_chunk" in items_by_modality:
assert mm_processor is not None
# Process vision_chunk items - extract from (data, modality) tuples
@@ -941,6 +972,14 @@ class BaseMultiModalContentParser(ABC):
def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
raise NotImplementedError
@abstractmethod
def parse_video_embeds(
self,
video_embeds: str | dict[str, str] | None,
uuid: str | None = None,
) -> None:
raise NotImplementedError
class MultiModalContentParser(BaseMultiModalContentParser):
def __init__(
@@ -1096,6 +1135,32 @@ class MultiModalContentParser(BaseMultiModalContentParser):
audio_placeholder = self._tracker.add("audio", (audio, uuid))
self._add_placeholder("audio", audio_placeholder)
def parse_video_embeds(
self,
video_embeds: str | dict[str, str] | None,
uuid: str | None = None,
) -> None:
mm_config = self.model_config.get_multimodal_config()
if not mm_config.enable_mm_embeds:
raise VLLMValidationError(
"You must set `--enable-mm-embeds` to input `video_embeds`",
parameter="video_embeds",
)
if isinstance(video_embeds, dict):
embeds = {
k: self._connector.fetch_video_embedding(v)
for k, v in video_embeds.items()
}
placeholder = self._tracker.add("video_embeds", (embeds, uuid))
elif isinstance(video_embeds, str):
embedding = self._connector.fetch_video_embedding(video_embeds)
placeholder = self._tracker.add("video_embeds", (embedding, uuid))
else:
placeholder = self._tracker.add("video_embeds", (None, uuid))
self._add_placeholder("video", placeholder)
class AsyncMultiModalContentParser(BaseMultiModalContentParser):
def __init__(
@@ -1304,6 +1369,43 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
)
self._add_placeholder("audio", audio_placeholder)
def parse_video_embeds(
self,
video_embeds: str | dict[str, str] | None,
uuid: str | None = None,
) -> None:
mm_config = self.model_config.get_multimodal_config()
if not mm_config.enable_mm_embeds:
raise VLLMValidationError(
"You must set `--enable-mm-embeds` to input `video_embeds`",
parameter="video_embeds",
)
placeholder = self._tracker.add(
"video_embeds",
partial(self._video_embeds_with_uuid_async, video_embeds, uuid),
)
self._add_placeholder("video", placeholder)
async def _video_embeds_with_uuid_async(
self,
video_embeds: str | dict[str, str] | None,
uuid: str | None,
):
if isinstance(video_embeds, dict):
tensors = await asyncio.gather(
*(
self._connector.fetch_video_embedding_async(v)
for v in video_embeds.values()
)
)
embeds = dict(zip(video_embeds, tensors))
elif isinstance(video_embeds, str):
embeds = await self._connector.fetch_video_embedding_async(video_embeds)
else:
embeds = None
return embeds, uuid
@dataclass
class ChatTemplateConfig:
@@ -1481,6 +1583,7 @@ def _get_full_multimodal_text_prompt(
_TextParser = partial(cast, ChatCompletionContentPartTextParam)
_ImageEmbedsParser = partial(cast, ChatCompletionContentPartImageEmbedsParam)
_AudioEmbedsParser = partial(cast, ChatCompletionContentPartAudioEmbedsParam)
_VideoEmbedsParser = partial(cast, ChatCompletionContentPartVideoEmbedsParam)
_PromptEmbedsParser = partial(cast, ChatCompletionContentPartPromptEmbedsParam)
_InputAudioParser = partial(cast, ChatCompletionContentPartInputAudioParam)
_RefusalParser = partial(cast, ChatCompletionContentPartRefusalParam)
@@ -1507,6 +1610,7 @@ MM_PARSER_MAP: dict[
"image_url": lambda part: _ImageParser(part).get("image_url", {}).get("url", None),
"image_embeds": lambda part: _ImageEmbedsParser(part).get("image_embeds", None),
"audio_embeds": lambda part: _AudioEmbedsParser(part).get("audio_embeds", None),
"video_embeds": lambda part: _VideoEmbedsParser(part).get("video_embeds", None),
"prompt_embeds": lambda part: _PromptEmbedsParser(part).get("data", None),
"image_pil": lambda part: _PILImageParser(part).get("image_pil", None),
"audio_url": lambda part: _AudioParser(part).get("audio_url", {}).get("url", None),
@@ -1605,6 +1709,13 @@ def _parse_chat_message_content_mm_part(
)
audio_embeds = audio_params.get("audio_embeds", None)
return "audio_embeds", audio_embeds
if "video_embeds" in part:
# "video_embeds" could be None if UUID is provided.
video_embeds_params = cast( # type: ignore[assignment]
ChatCompletionContentPartVideoEmbedsParam, part
)
video_embeds = video_embeds_params.get("video_embeds", None)
return "video_embeds", video_embeds
if "prompt_embeds" in part:
prompt_embeds_params = cast( # type: ignore[assignment]
ChatCompletionContentPartPromptEmbedsParam, part
@@ -1780,6 +1891,10 @@ def _parse_chat_message_content_part(
content = cast(str | dict[str, str], content) if content is not None else None
mm_parser.parse_audio_embeds(content, uuid)
modality = "audio"
elif part_type == "video_embeds":
content = cast(str | dict[str, str], content) if content is not None else None
mm_parser.parse_video_embeds(content, uuid)
modality = "video"
elif part_type == "prompt_embeds":
if not content:
raise VLLMValidationError(