[Frontend] Add video embeds input support (#54242)
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -10,7 +10,7 @@ clusters:
|
||||
• decode (language-model inference)
|
||||
|
||||
For MM input we:
|
||||
1. Extract *every* image/audio item.
|
||||
1. Extract *every* image/audio/video item.
|
||||
2. Fire N concurrent requests to the encoder cluster
|
||||
(one request per item, with **all text removed**).
|
||||
3. Wait for all of them to succeed.
|
||||
@@ -61,7 +61,16 @@ encoder_rr_lock = asyncio.Lock()
|
||||
###############################################################################
|
||||
|
||||
|
||||
MM_TYPES = {"image_url", "audio_url", "input_audio"}
|
||||
MM_TYPES = {"image_url", "audio_url", "input_audio", "video_url"}
|
||||
|
||||
# The embeds content type each MM item is rewritten to once the encoder has
|
||||
# published its embedding out of band.
|
||||
EMBEDS_TYPES = {
|
||||
"image_url": "image_embeds",
|
||||
"audio_url": "audio_embeds",
|
||||
"input_audio": "audio_embeds",
|
||||
"video_url": "video_embeds",
|
||||
}
|
||||
|
||||
|
||||
def encoder_rr_assignment(
|
||||
@@ -94,7 +103,9 @@ def content_uuid(item: dict) -> str:
|
||||
the unmodified path, which hashes the content, would keep it. That asymmetry
|
||||
silently biases any comparison between the two.
|
||||
"""
|
||||
url = (item.get("image_url") or item.get("audio_url") or {}).get("url") or ""
|
||||
url = (
|
||||
item.get("image_url") or item.get("audio_url") or item.get("video_url") or {}
|
||||
).get("url") or ""
|
||||
payload = url or json.dumps(item, sort_keys=True)
|
||||
return hashlib.sha256(payload.encode()).hexdigest()
|
||||
|
||||
@@ -103,19 +114,22 @@ def _b64_tensor(values: list) -> str:
|
||||
import torch
|
||||
|
||||
buf = io.BytesIO()
|
||||
grid = torch.tensor(values, dtype=torch.long)
|
||||
# Downstream stacks per item, so hand over a flat (t, h, w).
|
||||
torch.save(grid.reshape(-1)[:3], buf)
|
||||
flat = [v for item in values for v in (item if isinstance(item, list) else [item])]
|
||||
# Floats stay float64 so timestamp strings format exactly as the
|
||||
# encoder computed them.
|
||||
dtype = torch.float64 if any(isinstance(v, float) for v in flat) else None
|
||||
# Downstream stacks per item, so hand over a flat vector.
|
||||
torch.save(torch.tensor(flat, dtype=dtype), buf)
|
||||
return base64.b64encode(buf.getvalue()).decode()
|
||||
|
||||
|
||||
def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict:
|
||||
"""Replace each image item with a metadata-only reference for the decoder.
|
||||
"""Replace each media item with a metadata-only reference for the decoder.
|
||||
|
||||
The decoder does not need the pixels: the encoder instance already produced
|
||||
the embedding and published it through the EC connector under the same uuid.
|
||||
Sending only the grid lets the decoder size the placeholder range without
|
||||
re-running the image transform.
|
||||
re-running the media transform.
|
||||
|
||||
`item_meta` holds what the encoder reported for each item (its cache key and
|
||||
the grid its processor actually produced), so the grid is never re-derived
|
||||
@@ -146,25 +160,22 @@ def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict:
|
||||
# processor cache); let the decoder process the media itself.
|
||||
new_content.append(item)
|
||||
continue
|
||||
embeds_type = EMBEDS_TYPES[item["type"]]
|
||||
new_content.append(
|
||||
{
|
||||
"type": "image_embeds",
|
||||
"image_embeds": metadata,
|
||||
"uuid": item_uuid,
|
||||
}
|
||||
{"type": embeds_type, embeds_type: metadata, "uuid": item_uuid}
|
||||
)
|
||||
rewritten += 1
|
||||
new_messages.append({**msg, "content": new_content})
|
||||
|
||||
if not rewritten:
|
||||
return req_data
|
||||
logger.info("Rewrote %d image item(s) as metadata references", rewritten)
|
||||
logger.info("Rewrote %d media item(s) as metadata references", rewritten)
|
||||
return {**req_data, "messages": new_messages}
|
||||
|
||||
|
||||
def extract_mm_items(request_data: dict) -> list[dict]:
|
||||
"""
|
||||
Return *all* image/audio items that appear anywhere in `messages`.
|
||||
Return *all* image/audio/video items that appear anywhere in `messages`.
|
||||
|
||||
Each returned dict looks like:
|
||||
{ "type": "image_url", "image_url": {...} }
|
||||
|
||||
@@ -98,6 +98,16 @@ def qwen25omni_model_config_image_embeds():
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def qwen25omni_model_config_video_embeds():
|
||||
return ModelConfig(
|
||||
QWEN25OMNI_MODEL_ID,
|
||||
runner="generate",
|
||||
limit_mm_per_prompt={"video": 2},
|
||||
enable_mm_embeds=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def qwen2_audio_model_config():
|
||||
return ModelConfig(
|
||||
@@ -969,6 +979,133 @@ async def test_parse_chat_messages_audio_embeds_async(
|
||||
_assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None])
|
||||
|
||||
|
||||
def test_parse_chat_messages_empty_video_embeds_with_uuid(
|
||||
qwen25omni_model_config_video_embeds,
|
||||
):
|
||||
"""Test video_embeds with UUID (no actual embeds data)."""
|
||||
uuid = "test-video-uuid-123"
|
||||
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Describe this video"},
|
||||
{"type": "video_embeds", "video_embeds": None, "uuid": uuid},
|
||||
],
|
||||
}
|
||||
],
|
||||
qwen25omni_model_config_video_embeds,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
# Should have video in mm_data as None (UUID provided)
|
||||
assert mm_data is not None
|
||||
assert "video" in mm_data
|
||||
assert isinstance(mm_data["video"], list)
|
||||
assert len(mm_data["video"]) == 1
|
||||
assert mm_data["video"][0] is None
|
||||
|
||||
# UUID should be recorded
|
||||
_assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=[uuid])
|
||||
|
||||
|
||||
def test_parse_chat_messages_video_embeds_with_string(
|
||||
qwen25omni_model_config_video_embeds,
|
||||
):
|
||||
"""Test video_embeds with base64 string embedding data."""
|
||||
hidden_size = qwen25omni_model_config_video_embeds.get_inputs_embeds_size()
|
||||
video_embedding = torch.randn(1, 128, hidden_size)
|
||||
|
||||
base64_video_embedding = tensor2base64(video_embedding)
|
||||
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Describe this video"},
|
||||
{
|
||||
"type": "video_embeds",
|
||||
"video_embeds": base64_video_embedding,
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
qwen25omni_model_config_video_embeds,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
# Should have video embedding in mm_data (single tensor, not a list)
|
||||
assert mm_data is not None
|
||||
assert "video" in mm_data
|
||||
assert isinstance(mm_data["video"], torch.Tensor)
|
||||
assert mm_data["video"].shape == video_embedding.shape
|
||||
# No UUID provided
|
||||
assert mm_uuids is not None
|
||||
assert "video" in mm_uuids
|
||||
_assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=[None])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_chat_messages_video_embeds_async(
|
||||
qwen25omni_model_config_video_embeds,
|
||||
):
|
||||
"""Test video_embeds with async futures."""
|
||||
hidden_size = qwen25omni_model_config_video_embeds.get_inputs_embeds_size()
|
||||
video_embedding = torch.randn(1, 128, hidden_size)
|
||||
|
||||
base64_video_embedding = tensor2base64(video_embedding)
|
||||
|
||||
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Describe this video"},
|
||||
{
|
||||
"type": "video_embeds",
|
||||
"video_embeds": base64_video_embedding,
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
qwen25omni_model_config_video_embeds,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
# Should have video embedding in mm_data (single tensor, not a list)
|
||||
assert mm_data is not None
|
||||
assert "video" in mm_data
|
||||
assert isinstance(mm_data["video"], torch.Tensor)
|
||||
assert mm_data["video"].shape == video_embedding.shape
|
||||
# No UUID provided
|
||||
assert mm_uuids is not None
|
||||
assert "video" in mm_uuids
|
||||
_assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=[None])
|
||||
|
||||
|
||||
def test_parse_chat_messages_video_and_video_embeds_mixed(
|
||||
qwen25omni_model_config_video_embeds,
|
||||
video_url,
|
||||
):
|
||||
"""Test that mixing raw video and video_embeds is rejected."""
|
||||
with pytest.raises(VLLMValidationError, match="Mixing raw video"):
|
||||
parse_chat_messages(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "video_url", "video_url": {"url": video_url}},
|
||||
{"type": "video_embeds", "video_embeds": None, "uuid": "x"},
|
||||
],
|
||||
}
|
||||
],
|
||||
qwen25omni_model_config_video_embeds,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
|
||||
def test_parse_chat_messages_multiple_image_embeds(
|
||||
phi3v_model_config_image_embeds,
|
||||
):
|
||||
|
||||
@@ -11,8 +11,10 @@ from typing import Any
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ...registry import HF_EXAMPLE_MODELS
|
||||
from ...utils import build_model_context
|
||||
|
||||
MODEL_ID = "Qwen/Qwen3-VL-4B-Instruct"
|
||||
@@ -186,6 +188,89 @@ def test_processor_multi_video_list_kwargs(
|
||||
)
|
||||
|
||||
|
||||
def _build_video_embeds_mm_data(
|
||||
hidden_size: int,
|
||||
grid_thw: tuple[int, int, int] = (2, 4, 4),
|
||||
) -> dict[str, Any]:
|
||||
"""Create an embeds-only video item as an EPD consumer receives it:
|
||||
pre-computed embeddings plus the metadata published by the encoder."""
|
||||
import torch
|
||||
|
||||
t, h, w = grid_thw
|
||||
num_tokens = t * h * w // 4 # spatial_merge_size ** 2
|
||||
return {
|
||||
"video": {
|
||||
"video_embeds": torch.zeros(num_tokens, hidden_size),
|
||||
"video_grid_thw": torch.tensor([grid_thw]),
|
||||
"timestamps": torch.tensor([[float(i) for i in range(t)]]),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
def test_processor_video_embeds_with_timestamps(model_id: str) -> None:
|
||||
"""Embeds-only video input must size the placeholder range from the
|
||||
grid and the real timestamps published by the encoder (EC consumer
|
||||
path); synthesized or missing timestamps would change the token count
|
||||
and break embedding merging downstream."""
|
||||
# `build_model_context` forces enable_mm_embeds off, so build the
|
||||
# config directly; keep the same online-availability skip behavior.
|
||||
HF_EXAMPLE_MODELS.find_hf_info(model_id).check_available_online(on_fail="skip")
|
||||
model_config = ModelConfig(
|
||||
model_id,
|
||||
runner="generate",
|
||||
limit_mm_per_prompt={"image": 0, "video": 1},
|
||||
enable_mm_embeds=True,
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
tokenizer = processor.info.get_tokenizer()
|
||||
|
||||
grid_thw = (2, 4, 4)
|
||||
mm_data = _build_video_embeds_mm_data(
|
||||
model_config.get_inputs_embeds_size(), grid_thw
|
||||
)
|
||||
|
||||
prompt = "<|vision_start|><|video_pad|><|vision_end|>"
|
||||
processed = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
video_phs = processed["mm_placeholders"].get("video", [])
|
||||
assert len(video_phs) == 1, (
|
||||
f"Expected exactly 1 video placeholder, got {len(video_phs)}"
|
||||
)
|
||||
|
||||
t, h, w = grid_thw
|
||||
tokens_per_frame = h * w // 4
|
||||
timestamp_tokens = sum(
|
||||
len(tokenizer.encode(f"<{float(i):.1f} seconds>", add_special_tokens=False))
|
||||
for i in range(t)
|
||||
)
|
||||
# per frame: timestamp tokens + vision_start + video tokens + vision_end
|
||||
expected_len = timestamp_tokens + t * (tokens_per_frame + 2)
|
||||
assert video_phs[0].length == expected_len
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
def test_processor_video_embeds_missing_timestamps(model_id: str) -> None:
|
||||
"""Timestamps are required metadata for video embeds: they size the
|
||||
placeholder range, so omitting them must fail loudly at parse time
|
||||
instead of silently producing a wrong prompt."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 0, "video": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
mm_data = _build_video_embeds_mm_data(ctx.model_config.get_inputs_embeds_size())
|
||||
del mm_data["video"]["timestamps"]
|
||||
|
||||
with pytest.raises(ValueError, match="timestamps"):
|
||||
processor.info.parse_mm_data(mm_data)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> None:
|
||||
"""Regression test for memory profiling with ``cap_pixels_per_frame``.
|
||||
|
||||
@@ -14,6 +14,7 @@ from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||
ECConnectorRole,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.collection_utils import is_list_of
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -221,11 +222,15 @@ class ECExampleConnector(ECConnectorBase):
|
||||
# back to processing the media itself.
|
||||
if feature.data is not None:
|
||||
wanted = self._placeholder_metadata_fields(feature.modality)
|
||||
metadata = {
|
||||
key: value.tolist()
|
||||
for key, value in feature.data.get_data().items()
|
||||
if key in wanted and isinstance(value, torch.Tensor)
|
||||
}
|
||||
for key, value in feature.data.get_data().items():
|
||||
if key not in wanted:
|
||||
continue
|
||||
if isinstance(value, torch.Tensor):
|
||||
metadata[key] = value.tolist()
|
||||
elif is_list_of(value, (int, float)):
|
||||
# Some metadata (e.g. Qwen3-VL video timestamps) is
|
||||
# produced as a plain list rather than a tensor.
|
||||
metadata[key] = value
|
||||
items.append({"mm_hash": feature.identifier, **metadata})
|
||||
|
||||
if not items:
|
||||
|
||||
@@ -178,6 +178,22 @@ class ChatCompletionContentPartAudioEmbedsParam(TypedDict, total=False):
|
||||
"""
|
||||
|
||||
|
||||
class ChatCompletionContentPartVideoEmbedsParam(TypedDict, total=False):
|
||||
video_embeds: str | dict[str, str] | None
|
||||
"""
|
||||
The video embeddings. It can be either:
|
||||
- A single base64 string representing a serialized torch tensor.
|
||||
- A dictionary where each value is a base64 string.
|
||||
"""
|
||||
type: Required[Literal["video_embeds"]]
|
||||
"""The type of the content part."""
|
||||
uuid: str | None
|
||||
"""
|
||||
User-provided UUID of a media. User must guarantee that it is properly
|
||||
generated and unique for different medias.
|
||||
"""
|
||||
|
||||
|
||||
class ChatCompletionContentPartPromptEmbedsParam(TypedDict, total=False):
|
||||
data: Required[str]
|
||||
"""
|
||||
@@ -324,6 +340,7 @@ ChatCompletionContentPartParam: TypeAlias = (
|
||||
| CustomChatCompletionContentSimpleImageParam
|
||||
| ChatCompletionContentPartImageEmbedsParam
|
||||
| ChatCompletionContentPartAudioEmbedsParam
|
||||
| ChatCompletionContentPartVideoEmbedsParam
|
||||
| ChatCompletionContentPartPromptEmbedsParam
|
||||
| CustomChatCompletionContentSimpleAudioParam
|
||||
| CustomChatCompletionContentSimpleVideoParam
|
||||
@@ -415,6 +432,7 @@ ModalityStr = Literal[
|
||||
"video",
|
||||
"image_embeds",
|
||||
"audio_embeds",
|
||||
"video_embeds",
|
||||
"vision_chunk",
|
||||
"prompt_embeds",
|
||||
]
|
||||
@@ -755,6 +773,11 @@ def _resolve_items(
|
||||
"Mixing raw audio and embedding inputs is not allowed",
|
||||
parameter="audio_embeds",
|
||||
)
|
||||
if "video" in items_by_modality and "video_embeds" in items_by_modality:
|
||||
raise VLLMValidationError(
|
||||
"Mixing raw video and embedding inputs is not allowed",
|
||||
parameter="video_embeds",
|
||||
)
|
||||
# `prompt_embeds` bypasses HF MM processors. Every other modality requires one.
|
||||
processor_modalities = items_by_modality.keys() - {"prompt_embeds"}
|
||||
if processor_modalities and mm_processor is None:
|
||||
@@ -789,6 +812,14 @@ def _resolve_items(
|
||||
if "video" in items_by_modality:
|
||||
mm_data["video"] = [data for data, uuid in items_by_modality["video"]]
|
||||
mm_uuids["video"] = [uuid for data, uuid in items_by_modality["video"]]
|
||||
if "video_embeds" in items_by_modality:
|
||||
assert mm_processor is not None
|
||||
mm_data["video"] = _get_embeds_data(
|
||||
"video",
|
||||
[data for data, uuid in items_by_modality["video_embeds"]],
|
||||
mm_processor,
|
||||
)
|
||||
mm_uuids["video"] = [uuid for data, uuid in items_by_modality["video_embeds"]]
|
||||
if "vision_chunk" in items_by_modality:
|
||||
assert mm_processor is not None
|
||||
# Process vision_chunk items - extract from (data, modality) tuples
|
||||
@@ -941,6 +972,14 @@ class BaseMultiModalContentParser(ABC):
|
||||
def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def parse_video_embeds(
|
||||
self,
|
||||
video_embeds: str | dict[str, str] | None,
|
||||
uuid: str | None = None,
|
||||
) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class MultiModalContentParser(BaseMultiModalContentParser):
|
||||
def __init__(
|
||||
@@ -1096,6 +1135,32 @@ class MultiModalContentParser(BaseMultiModalContentParser):
|
||||
audio_placeholder = self._tracker.add("audio", (audio, uuid))
|
||||
self._add_placeholder("audio", audio_placeholder)
|
||||
|
||||
def parse_video_embeds(
|
||||
self,
|
||||
video_embeds: str | dict[str, str] | None,
|
||||
uuid: str | None = None,
|
||||
) -> None:
|
||||
mm_config = self.model_config.get_multimodal_config()
|
||||
if not mm_config.enable_mm_embeds:
|
||||
raise VLLMValidationError(
|
||||
"You must set `--enable-mm-embeds` to input `video_embeds`",
|
||||
parameter="video_embeds",
|
||||
)
|
||||
|
||||
if isinstance(video_embeds, dict):
|
||||
embeds = {
|
||||
k: self._connector.fetch_video_embedding(v)
|
||||
for k, v in video_embeds.items()
|
||||
}
|
||||
placeholder = self._tracker.add("video_embeds", (embeds, uuid))
|
||||
elif isinstance(video_embeds, str):
|
||||
embedding = self._connector.fetch_video_embedding(video_embeds)
|
||||
placeholder = self._tracker.add("video_embeds", (embedding, uuid))
|
||||
else:
|
||||
placeholder = self._tracker.add("video_embeds", (None, uuid))
|
||||
|
||||
self._add_placeholder("video", placeholder)
|
||||
|
||||
|
||||
class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
def __init__(
|
||||
@@ -1304,6 +1369,43 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
)
|
||||
self._add_placeholder("audio", audio_placeholder)
|
||||
|
||||
def parse_video_embeds(
|
||||
self,
|
||||
video_embeds: str | dict[str, str] | None,
|
||||
uuid: str | None = None,
|
||||
) -> None:
|
||||
mm_config = self.model_config.get_multimodal_config()
|
||||
if not mm_config.enable_mm_embeds:
|
||||
raise VLLMValidationError(
|
||||
"You must set `--enable-mm-embeds` to input `video_embeds`",
|
||||
parameter="video_embeds",
|
||||
)
|
||||
|
||||
placeholder = self._tracker.add(
|
||||
"video_embeds",
|
||||
partial(self._video_embeds_with_uuid_async, video_embeds, uuid),
|
||||
)
|
||||
self._add_placeholder("video", placeholder)
|
||||
|
||||
async def _video_embeds_with_uuid_async(
|
||||
self,
|
||||
video_embeds: str | dict[str, str] | None,
|
||||
uuid: str | None,
|
||||
):
|
||||
if isinstance(video_embeds, dict):
|
||||
tensors = await asyncio.gather(
|
||||
*(
|
||||
self._connector.fetch_video_embedding_async(v)
|
||||
for v in video_embeds.values()
|
||||
)
|
||||
)
|
||||
embeds = dict(zip(video_embeds, tensors))
|
||||
elif isinstance(video_embeds, str):
|
||||
embeds = await self._connector.fetch_video_embedding_async(video_embeds)
|
||||
else:
|
||||
embeds = None
|
||||
return embeds, uuid
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChatTemplateConfig:
|
||||
@@ -1481,6 +1583,7 @@ def _get_full_multimodal_text_prompt(
|
||||
_TextParser = partial(cast, ChatCompletionContentPartTextParam)
|
||||
_ImageEmbedsParser = partial(cast, ChatCompletionContentPartImageEmbedsParam)
|
||||
_AudioEmbedsParser = partial(cast, ChatCompletionContentPartAudioEmbedsParam)
|
||||
_VideoEmbedsParser = partial(cast, ChatCompletionContentPartVideoEmbedsParam)
|
||||
_PromptEmbedsParser = partial(cast, ChatCompletionContentPartPromptEmbedsParam)
|
||||
_InputAudioParser = partial(cast, ChatCompletionContentPartInputAudioParam)
|
||||
_RefusalParser = partial(cast, ChatCompletionContentPartRefusalParam)
|
||||
@@ -1507,6 +1610,7 @@ MM_PARSER_MAP: dict[
|
||||
"image_url": lambda part: _ImageParser(part).get("image_url", {}).get("url", None),
|
||||
"image_embeds": lambda part: _ImageEmbedsParser(part).get("image_embeds", None),
|
||||
"audio_embeds": lambda part: _AudioEmbedsParser(part).get("audio_embeds", None),
|
||||
"video_embeds": lambda part: _VideoEmbedsParser(part).get("video_embeds", None),
|
||||
"prompt_embeds": lambda part: _PromptEmbedsParser(part).get("data", None),
|
||||
"image_pil": lambda part: _PILImageParser(part).get("image_pil", None),
|
||||
"audio_url": lambda part: _AudioParser(part).get("audio_url", {}).get("url", None),
|
||||
@@ -1605,6 +1709,13 @@ def _parse_chat_message_content_mm_part(
|
||||
)
|
||||
audio_embeds = audio_params.get("audio_embeds", None)
|
||||
return "audio_embeds", audio_embeds
|
||||
if "video_embeds" in part:
|
||||
# "video_embeds" could be None if UUID is provided.
|
||||
video_embeds_params = cast( # type: ignore[assignment]
|
||||
ChatCompletionContentPartVideoEmbedsParam, part
|
||||
)
|
||||
video_embeds = video_embeds_params.get("video_embeds", None)
|
||||
return "video_embeds", video_embeds
|
||||
if "prompt_embeds" in part:
|
||||
prompt_embeds_params = cast( # type: ignore[assignment]
|
||||
ChatCompletionContentPartPromptEmbedsParam, part
|
||||
@@ -1780,6 +1891,10 @@ def _parse_chat_message_content_part(
|
||||
content = cast(str | dict[str, str], content) if content is not None else None
|
||||
mm_parser.parse_audio_embeds(content, uuid)
|
||||
modality = "audio"
|
||||
elif part_type == "video_embeds":
|
||||
content = cast(str | dict[str, str], content) if content is not None else None
|
||||
mm_parser.parse_video_embeds(content, uuid)
|
||||
modality = "video"
|
||||
elif part_type == "prompt_embeds":
|
||||
if not content:
|
||||
raise VLLMValidationError(
|
||||
|
||||
@@ -894,6 +894,19 @@ class Qwen3_VisionTransformer(nn.Module):
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class Qwen3VLMultiModalDataParser(Qwen2VLMultiModalDataParser):
|
||||
# Timestamps are part of the prompt replacement (each frame group's
|
||||
# "<X.X seconds>" text), so they size the placeholder range and must
|
||||
# travel with the grid when embeddings are delivered out of band.
|
||||
embedding_fields = {
|
||||
**Qwen2VLMultiModalDataParser.embedding_fields,
|
||||
"video": {
|
||||
**Qwen2VLMultiModalDataParser.embedding_fields["video"],
|
||||
"timestamps": "metadata",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
|
||||
def get_hf_config(self):
|
||||
return self.ctx.get_hf_config(Qwen3VLConfig)
|
||||
@@ -912,7 +925,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
|
||||
return self.get_hf_processor(**kwargs).video_processor
|
||||
|
||||
def get_data_parser(self):
|
||||
return Qwen2VLMultiModalDataParser(
|
||||
return Qwen3VLMultiModalDataParser(
|
||||
self.get_hf_config().vision_config.spatial_merge_size,
|
||||
video_needs_metadata=True,
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
@@ -1609,7 +1622,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
def get_video_repl(
|
||||
*,
|
||||
tokens_per_frame: list[int],
|
||||
timestamps: list[float | int],
|
||||
timestamps: list[float | int] | torch.Tensor,
|
||||
tokenizer: TokenizerLike,
|
||||
vision_start_token_id: int,
|
||||
vision_end_token_id: int,
|
||||
|
||||
@@ -35,7 +35,7 @@ from vllm.utils.registry import ExtensionManager
|
||||
from .audio import AudioEmbeddingMediaIO, AudioMediaIO
|
||||
from .base import MediaIO, MediaWithBytes
|
||||
from .image import ImageEmbeddingMediaIO, ImageMediaIO
|
||||
from .video import VideoMediaIO
|
||||
from .video import VideoEmbeddingMediaIO, VideoMediaIO
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -653,3 +653,28 @@ class MediaConnector:
|
||||
return await loop.run_in_executor(
|
||||
global_thread_pool, audio_embedding_io.load_base64, "", data
|
||||
)
|
||||
|
||||
def fetch_video_embedding(
|
||||
self,
|
||||
data: str,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Load video embedding from a URL.
|
||||
"""
|
||||
video_embedding_io = VideoEmbeddingMediaIO()
|
||||
|
||||
return video_embedding_io.load_base64("", data)
|
||||
|
||||
async def fetch_video_embedding_async(
|
||||
self,
|
||||
data: str,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Asynchronously load video embedding from a URL.
|
||||
"""
|
||||
video_embedding_io = VideoEmbeddingMediaIO()
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
return await loop.run_in_executor(
|
||||
global_thread_pool, video_embedding_io.load_base64, "", data
|
||||
)
|
||||
|
||||
@@ -1,20 +1,24 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from functools import partial
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
import pybase64
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.serial_utils import tensor2base64
|
||||
from vllm.utils.sparse_utils import check_sparse_tensor_invariants_threadsafe
|
||||
|
||||
from ..video import VIDEO_LOADER_REGISTRY
|
||||
from .base import MediaIO, MediaWithBytes
|
||||
from .image import ImageMediaIO
|
||||
from .image import MAGIC_NUMPY_PREFIX, ImageMediaIO
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -194,3 +198,45 @@ class VideoMediaIO(MediaIO[MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]]):
|
||||
|
||||
msg = "Only JPEG format is supported for now."
|
||||
raise NotImplementedError(msg)
|
||||
|
||||
|
||||
class VideoEmbeddingMediaIO(MediaIO[torch.Tensor]):
|
||||
"""Video embedding MediaIO implementation.
|
||||
|
||||
Configuration values can be user-provided either by --media-io-kwargs or
|
||||
by the runtime API field "media_io_kwargs". Ensure proper validation and
|
||||
error handling.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
|
||||
def _load_pickled_torch(self, data: bytes) -> torch.Tensor:
|
||||
buffer = BytesIO(data)
|
||||
with check_sparse_tensor_invariants_threadsafe():
|
||||
tensor = torch.load(buffer, weights_only=True)
|
||||
return tensor.to_dense()
|
||||
|
||||
def _load_numpy(self, data: bytes) -> torch.Tensor:
|
||||
with BytesIO(data) as buffer:
|
||||
return torch.from_numpy(np.load(buffer))
|
||||
|
||||
def load_bytes(self, data: bytes) -> torch.Tensor:
|
||||
if data[:6] == MAGIC_NUMPY_PREFIX:
|
||||
return self._load_numpy(data)
|
||||
|
||||
return self._load_pickled_torch(data)
|
||||
|
||||
def load_base64(self, media_type: str, data: str) -> torch.Tensor:
|
||||
return self.load_bytes(pybase64.b64decode(data, validate=True))
|
||||
|
||||
def load_file(self, filepath: Path) -> torch.Tensor:
|
||||
if filepath.suffix == ".npy":
|
||||
return torch.from_numpy(np.load(filepath))
|
||||
|
||||
with check_sparse_tensor_invariants_threadsafe():
|
||||
tensor = torch.load(filepath, weights_only=True)
|
||||
return tensor.to_dense()
|
||||
|
||||
def encode_base64(self, media: torch.Tensor) -> str:
|
||||
return tensor2base64(media)
|
||||
|
||||
Reference in New Issue
Block a user