diff --git a/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py b/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py index 267176fd3e..b68f7d2a11 100644 --- a/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py +++ b/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py @@ -10,7 +10,7 @@ clusters: • decode (language-model inference) For MM input we: - 1. Extract *every* image/audio item. + 1. Extract *every* image/audio/video item. 2. Fire N concurrent requests to the encoder cluster (one request per item, with **all text removed**). 3. Wait for all of them to succeed. @@ -61,7 +61,16 @@ encoder_rr_lock = asyncio.Lock() ############################################################################### -MM_TYPES = {"image_url", "audio_url", "input_audio"} +MM_TYPES = {"image_url", "audio_url", "input_audio", "video_url"} + +# The embeds content type each MM item is rewritten to once the encoder has +# published its embedding out of band. +EMBEDS_TYPES = { + "image_url": "image_embeds", + "audio_url": "audio_embeds", + "input_audio": "audio_embeds", + "video_url": "video_embeds", +} def encoder_rr_assignment( @@ -94,7 +103,9 @@ def content_uuid(item: dict) -> str: the unmodified path, which hashes the content, would keep it. That asymmetry silently biases any comparison between the two. """ - url = (item.get("image_url") or item.get("audio_url") or {}).get("url") or "" + url = ( + item.get("image_url") or item.get("audio_url") or item.get("video_url") or {} + ).get("url") or "" payload = url or json.dumps(item, sort_keys=True) return hashlib.sha256(payload.encode()).hexdigest() @@ -103,19 +114,22 @@ def _b64_tensor(values: list) -> str: import torch buf = io.BytesIO() - grid = torch.tensor(values, dtype=torch.long) - # Downstream stacks per item, so hand over a flat (t, h, w). - torch.save(grid.reshape(-1)[:3], buf) + flat = [v for item in values for v in (item if isinstance(item, list) else [item])] + # Floats stay float64 so timestamp strings format exactly as the + # encoder computed them. + dtype = torch.float64 if any(isinstance(v, float) for v in flat) else None + # Downstream stacks per item, so hand over a flat vector. + torch.save(torch.tensor(flat, dtype=dtype), buf) return base64.b64encode(buf.getvalue()).decode() def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict: - """Replace each image item with a metadata-only reference for the decoder. + """Replace each media item with a metadata-only reference for the decoder. The decoder does not need the pixels: the encoder instance already produced the embedding and published it through the EC connector under the same uuid. Sending only the grid lets the decoder size the placeholder range without - re-running the image transform. + re-running the media transform. `item_meta` holds what the encoder reported for each item (its cache key and the grid its processor actually produced), so the grid is never re-derived @@ -146,25 +160,22 @@ def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict: # processor cache); let the decoder process the media itself. new_content.append(item) continue + embeds_type = EMBEDS_TYPES[item["type"]] new_content.append( - { - "type": "image_embeds", - "image_embeds": metadata, - "uuid": item_uuid, - } + {"type": embeds_type, embeds_type: metadata, "uuid": item_uuid} ) rewritten += 1 new_messages.append({**msg, "content": new_content}) if not rewritten: return req_data - logger.info("Rewrote %d image item(s) as metadata references", rewritten) + logger.info("Rewrote %d media item(s) as metadata references", rewritten) return {**req_data, "messages": new_messages} def extract_mm_items(request_data: dict) -> list[dict]: """ - Return *all* image/audio items that appear anywhere in `messages`. + Return *all* image/audio/video items that appear anywhere in `messages`. Each returned dict looks like: { "type": "image_url", "image_url": {...} } diff --git a/tests/entrypoints/unit_tests/test_chat_utils.py b/tests/entrypoints/unit_tests/test_chat_utils.py index 366645bdb4..b1fe4f738c 100644 --- a/tests/entrypoints/unit_tests/test_chat_utils.py +++ b/tests/entrypoints/unit_tests/test_chat_utils.py @@ -98,6 +98,16 @@ def qwen25omni_model_config_image_embeds(): ) +@pytest.fixture(scope="function") +def qwen25omni_model_config_video_embeds(): + return ModelConfig( + QWEN25OMNI_MODEL_ID, + runner="generate", + limit_mm_per_prompt={"video": 2}, + enable_mm_embeds=True, + ) + + @pytest.fixture(scope="function") def qwen2_audio_model_config(): return ModelConfig( @@ -969,6 +979,133 @@ async def test_parse_chat_messages_audio_embeds_async( _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None]) +def test_parse_chat_messages_empty_video_embeds_with_uuid( + qwen25omni_model_config_video_embeds, +): + """Test video_embeds with UUID (no actual embeds data).""" + uuid = "test-video-uuid-123" + + conversation, mm_data, mm_uuids = parse_chat_messages( + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this video"}, + {"type": "video_embeds", "video_embeds": None, "uuid": uuid}, + ], + } + ], + qwen25omni_model_config_video_embeds, + content_format="string", + ) + + # Should have video in mm_data as None (UUID provided) + assert mm_data is not None + assert "video" in mm_data + assert isinstance(mm_data["video"], list) + assert len(mm_data["video"]) == 1 + assert mm_data["video"][0] is None + + # UUID should be recorded + _assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=[uuid]) + + +def test_parse_chat_messages_video_embeds_with_string( + qwen25omni_model_config_video_embeds, +): + """Test video_embeds with base64 string embedding data.""" + hidden_size = qwen25omni_model_config_video_embeds.get_inputs_embeds_size() + video_embedding = torch.randn(1, 128, hidden_size) + + base64_video_embedding = tensor2base64(video_embedding) + + conversation, mm_data, mm_uuids = parse_chat_messages( + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this video"}, + { + "type": "video_embeds", + "video_embeds": base64_video_embedding, + }, + ], + } + ], + qwen25omni_model_config_video_embeds, + content_format="string", + ) + + # Should have video embedding in mm_data (single tensor, not a list) + assert mm_data is not None + assert "video" in mm_data + assert isinstance(mm_data["video"], torch.Tensor) + assert mm_data["video"].shape == video_embedding.shape + # No UUID provided + assert mm_uuids is not None + assert "video" in mm_uuids + _assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=[None]) + + +@pytest.mark.asyncio +async def test_parse_chat_messages_video_embeds_async( + qwen25omni_model_config_video_embeds, +): + """Test video_embeds with async futures.""" + hidden_size = qwen25omni_model_config_video_embeds.get_inputs_embeds_size() + video_embedding = torch.randn(1, 128, hidden_size) + + base64_video_embedding = tensor2base64(video_embedding) + + conversation, mm_data, mm_uuids = await parse_chat_messages_async( + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this video"}, + { + "type": "video_embeds", + "video_embeds": base64_video_embedding, + }, + ], + } + ], + qwen25omni_model_config_video_embeds, + content_format="string", + ) + + # Should have video embedding in mm_data (single tensor, not a list) + assert mm_data is not None + assert "video" in mm_data + assert isinstance(mm_data["video"], torch.Tensor) + assert mm_data["video"].shape == video_embedding.shape + # No UUID provided + assert mm_uuids is not None + assert "video" in mm_uuids + _assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=[None]) + + +def test_parse_chat_messages_video_and_video_embeds_mixed( + qwen25omni_model_config_video_embeds, + video_url, +): + """Test that mixing raw video and video_embeds is rejected.""" + with pytest.raises(VLLMValidationError, match="Mixing raw video"): + parse_chat_messages( + [ + { + "role": "user", + "content": [ + {"type": "video_url", "video_url": {"url": video_url}}, + {"type": "video_embeds", "video_embeds": None, "uuid": "x"}, + ], + } + ], + qwen25omni_model_config_video_embeds, + content_format="string", + ) + + def test_parse_chat_messages_multiple_image_embeds( phi3v_model_config_image_embeds, ): diff --git a/tests/models/multimodal/processing/test_qwen3_vl.py b/tests/models/multimodal/processing/test_qwen3_vl.py index 6f92e71dd4..d17e19bd65 100644 --- a/tests/models/multimodal/processing/test_qwen3_vl.py +++ b/tests/models/multimodal/processing/test_qwen3_vl.py @@ -11,8 +11,10 @@ from typing import Any import numpy as np import pytest +from vllm.config import ModelConfig from vllm.multimodal import MULTIMODAL_REGISTRY +from ...registry import HF_EXAMPLE_MODELS from ...utils import build_model_context MODEL_ID = "Qwen/Qwen3-VL-4B-Instruct" @@ -186,6 +188,89 @@ def test_processor_multi_video_list_kwargs( ) +def _build_video_embeds_mm_data( + hidden_size: int, + grid_thw: tuple[int, int, int] = (2, 4, 4), +) -> dict[str, Any]: + """Create an embeds-only video item as an EPD consumer receives it: + pre-computed embeddings plus the metadata published by the encoder.""" + import torch + + t, h, w = grid_thw + num_tokens = t * h * w // 4 # spatial_merge_size ** 2 + return { + "video": { + "video_embeds": torch.zeros(num_tokens, hidden_size), + "video_grid_thw": torch.tensor([grid_thw]), + "timestamps": torch.tensor([[float(i) for i in range(t)]]), + } + } + + +@pytest.mark.parametrize("model_id", [MODEL_ID]) +def test_processor_video_embeds_with_timestamps(model_id: str) -> None: + """Embeds-only video input must size the placeholder range from the + grid and the real timestamps published by the encoder (EC consumer + path); synthesized or missing timestamps would change the token count + and break embedding merging downstream.""" + # `build_model_context` forces enable_mm_embeds off, so build the + # config directly; keep the same online-availability skip behavior. + HF_EXAMPLE_MODELS.find_hf_info(model_id).check_available_online(on_fail="skip") + model_config = ModelConfig( + model_id, + runner="generate", + limit_mm_per_prompt={"image": 0, "video": 1}, + enable_mm_embeds=True, + ) + processor = MULTIMODAL_REGISTRY.create_processor(model_config) + tokenizer = processor.info.get_tokenizer() + + grid_thw = (2, 4, 4) + mm_data = _build_video_embeds_mm_data( + model_config.get_inputs_embeds_size(), grid_thw + ) + + prompt = "<|vision_start|><|video_pad|><|vision_end|>" + processed = processor( + prompt, + mm_items=processor.info.parse_mm_data(mm_data), + hf_processor_mm_kwargs={}, + ) + + video_phs = processed["mm_placeholders"].get("video", []) + assert len(video_phs) == 1, ( + f"Expected exactly 1 video placeholder, got {len(video_phs)}" + ) + + t, h, w = grid_thw + tokens_per_frame = h * w // 4 + timestamp_tokens = sum( + len(tokenizer.encode(f"<{float(i):.1f} seconds>", add_special_tokens=False)) + for i in range(t) + ) + # per frame: timestamp tokens + vision_start + video tokens + vision_end + expected_len = timestamp_tokens + t * (tokens_per_frame + 2) + assert video_phs[0].length == expected_len + + +@pytest.mark.parametrize("model_id", [MODEL_ID]) +def test_processor_video_embeds_missing_timestamps(model_id: str) -> None: + """Timestamps are required metadata for video embeds: they size the + placeholder range, so omitting them must fail loudly at parse time + instead of silently producing a wrong prompt.""" + ctx = build_model_context( + model_id, + limit_mm_per_prompt={"image": 0, "video": 1}, + ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + + mm_data = _build_video_embeds_mm_data(ctx.model_config.get_inputs_embeds_size()) + del mm_data["video"]["timestamps"] + + with pytest.raises(ValueError, match="timestamps"): + processor.info.parse_mm_data(mm_data) + + @pytest.mark.parametrize("model_id", [MODEL_ID]) def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> None: """Regression test for memory profiling with ``cap_pixels_per_frame``. diff --git a/vllm/distributed/ec_transfer/ec_connector/example_connector.py b/vllm/distributed/ec_transfer/ec_connector/example_connector.py index 8d88934ac1..5b74dc6776 100644 --- a/vllm/distributed/ec_transfer/ec_connector/example_connector.py +++ b/vllm/distributed/ec_transfer/ec_connector/example_connector.py @@ -14,6 +14,7 @@ from vllm.distributed.ec_transfer.ec_connector.base import ( ECConnectorRole, ) from vllm.logger import init_logger +from vllm.utils.collection_utils import is_list_of from vllm.v1.core.sched.output import SchedulerOutput if TYPE_CHECKING: @@ -221,11 +222,15 @@ class ECExampleConnector(ECConnectorBase): # back to processing the media itself. if feature.data is not None: wanted = self._placeholder_metadata_fields(feature.modality) - metadata = { - key: value.tolist() - for key, value in feature.data.get_data().items() - if key in wanted and isinstance(value, torch.Tensor) - } + for key, value in feature.data.get_data().items(): + if key not in wanted: + continue + if isinstance(value, torch.Tensor): + metadata[key] = value.tolist() + elif is_list_of(value, (int, float)): + # Some metadata (e.g. Qwen3-VL video timestamps) is + # produced as a plain list rather than a tensor. + metadata[key] = value items.append({"mm_hash": feature.identifier, **metadata}) if not items: diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index ece9dfd561..515fef140b 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -178,6 +178,22 @@ class ChatCompletionContentPartAudioEmbedsParam(TypedDict, total=False): """ +class ChatCompletionContentPartVideoEmbedsParam(TypedDict, total=False): + video_embeds: str | dict[str, str] | None + """ + The video embeddings. It can be either: + - A single base64 string representing a serialized torch tensor. + - A dictionary where each value is a base64 string. + """ + type: Required[Literal["video_embeds"]] + """The type of the content part.""" + uuid: str | None + """ + User-provided UUID of a media. User must guarantee that it is properly + generated and unique for different medias. + """ + + class ChatCompletionContentPartPromptEmbedsParam(TypedDict, total=False): data: Required[str] """ @@ -324,6 +340,7 @@ ChatCompletionContentPartParam: TypeAlias = ( | CustomChatCompletionContentSimpleImageParam | ChatCompletionContentPartImageEmbedsParam | ChatCompletionContentPartAudioEmbedsParam + | ChatCompletionContentPartVideoEmbedsParam | ChatCompletionContentPartPromptEmbedsParam | CustomChatCompletionContentSimpleAudioParam | CustomChatCompletionContentSimpleVideoParam @@ -415,6 +432,7 @@ ModalityStr = Literal[ "video", "image_embeds", "audio_embeds", + "video_embeds", "vision_chunk", "prompt_embeds", ] @@ -755,6 +773,11 @@ def _resolve_items( "Mixing raw audio and embedding inputs is not allowed", parameter="audio_embeds", ) + if "video" in items_by_modality and "video_embeds" in items_by_modality: + raise VLLMValidationError( + "Mixing raw video and embedding inputs is not allowed", + parameter="video_embeds", + ) # `prompt_embeds` bypasses HF MM processors. Every other modality requires one. processor_modalities = items_by_modality.keys() - {"prompt_embeds"} if processor_modalities and mm_processor is None: @@ -789,6 +812,14 @@ def _resolve_items( if "video" in items_by_modality: mm_data["video"] = [data for data, uuid in items_by_modality["video"]] mm_uuids["video"] = [uuid for data, uuid in items_by_modality["video"]] + if "video_embeds" in items_by_modality: + assert mm_processor is not None + mm_data["video"] = _get_embeds_data( + "video", + [data for data, uuid in items_by_modality["video_embeds"]], + mm_processor, + ) + mm_uuids["video"] = [uuid for data, uuid in items_by_modality["video_embeds"]] if "vision_chunk" in items_by_modality: assert mm_processor is not None # Process vision_chunk items - extract from (data, modality) tuples @@ -941,6 +972,14 @@ class BaseMultiModalContentParser(ABC): def parse_video(self, video_url: str | None, uuid: str | None = None) -> None: raise NotImplementedError + @abstractmethod + def parse_video_embeds( + self, + video_embeds: str | dict[str, str] | None, + uuid: str | None = None, + ) -> None: + raise NotImplementedError + class MultiModalContentParser(BaseMultiModalContentParser): def __init__( @@ -1096,6 +1135,32 @@ class MultiModalContentParser(BaseMultiModalContentParser): audio_placeholder = self._tracker.add("audio", (audio, uuid)) self._add_placeholder("audio", audio_placeholder) + def parse_video_embeds( + self, + video_embeds: str | dict[str, str] | None, + uuid: str | None = None, + ) -> None: + mm_config = self.model_config.get_multimodal_config() + if not mm_config.enable_mm_embeds: + raise VLLMValidationError( + "You must set `--enable-mm-embeds` to input `video_embeds`", + parameter="video_embeds", + ) + + if isinstance(video_embeds, dict): + embeds = { + k: self._connector.fetch_video_embedding(v) + for k, v in video_embeds.items() + } + placeholder = self._tracker.add("video_embeds", (embeds, uuid)) + elif isinstance(video_embeds, str): + embedding = self._connector.fetch_video_embedding(video_embeds) + placeholder = self._tracker.add("video_embeds", (embedding, uuid)) + else: + placeholder = self._tracker.add("video_embeds", (None, uuid)) + + self._add_placeholder("video", placeholder) + class AsyncMultiModalContentParser(BaseMultiModalContentParser): def __init__( @@ -1304,6 +1369,43 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser): ) self._add_placeholder("audio", audio_placeholder) + def parse_video_embeds( + self, + video_embeds: str | dict[str, str] | None, + uuid: str | None = None, + ) -> None: + mm_config = self.model_config.get_multimodal_config() + if not mm_config.enable_mm_embeds: + raise VLLMValidationError( + "You must set `--enable-mm-embeds` to input `video_embeds`", + parameter="video_embeds", + ) + + placeholder = self._tracker.add( + "video_embeds", + partial(self._video_embeds_with_uuid_async, video_embeds, uuid), + ) + self._add_placeholder("video", placeholder) + + async def _video_embeds_with_uuid_async( + self, + video_embeds: str | dict[str, str] | None, + uuid: str | None, + ): + if isinstance(video_embeds, dict): + tensors = await asyncio.gather( + *( + self._connector.fetch_video_embedding_async(v) + for v in video_embeds.values() + ) + ) + embeds = dict(zip(video_embeds, tensors)) + elif isinstance(video_embeds, str): + embeds = await self._connector.fetch_video_embedding_async(video_embeds) + else: + embeds = None + return embeds, uuid + @dataclass class ChatTemplateConfig: @@ -1481,6 +1583,7 @@ def _get_full_multimodal_text_prompt( _TextParser = partial(cast, ChatCompletionContentPartTextParam) _ImageEmbedsParser = partial(cast, ChatCompletionContentPartImageEmbedsParam) _AudioEmbedsParser = partial(cast, ChatCompletionContentPartAudioEmbedsParam) +_VideoEmbedsParser = partial(cast, ChatCompletionContentPartVideoEmbedsParam) _PromptEmbedsParser = partial(cast, ChatCompletionContentPartPromptEmbedsParam) _InputAudioParser = partial(cast, ChatCompletionContentPartInputAudioParam) _RefusalParser = partial(cast, ChatCompletionContentPartRefusalParam) @@ -1507,6 +1610,7 @@ MM_PARSER_MAP: dict[ "image_url": lambda part: _ImageParser(part).get("image_url", {}).get("url", None), "image_embeds": lambda part: _ImageEmbedsParser(part).get("image_embeds", None), "audio_embeds": lambda part: _AudioEmbedsParser(part).get("audio_embeds", None), + "video_embeds": lambda part: _VideoEmbedsParser(part).get("video_embeds", None), "prompt_embeds": lambda part: _PromptEmbedsParser(part).get("data", None), "image_pil": lambda part: _PILImageParser(part).get("image_pil", None), "audio_url": lambda part: _AudioParser(part).get("audio_url", {}).get("url", None), @@ -1605,6 +1709,13 @@ def _parse_chat_message_content_mm_part( ) audio_embeds = audio_params.get("audio_embeds", None) return "audio_embeds", audio_embeds + if "video_embeds" in part: + # "video_embeds" could be None if UUID is provided. + video_embeds_params = cast( # type: ignore[assignment] + ChatCompletionContentPartVideoEmbedsParam, part + ) + video_embeds = video_embeds_params.get("video_embeds", None) + return "video_embeds", video_embeds if "prompt_embeds" in part: prompt_embeds_params = cast( # type: ignore[assignment] ChatCompletionContentPartPromptEmbedsParam, part @@ -1780,6 +1891,10 @@ def _parse_chat_message_content_part( content = cast(str | dict[str, str], content) if content is not None else None mm_parser.parse_audio_embeds(content, uuid) modality = "audio" + elif part_type == "video_embeds": + content = cast(str | dict[str, str], content) if content is not None else None + mm_parser.parse_video_embeds(content, uuid) + modality = "video" elif part_type == "prompt_embeds": if not content: raise VLLMValidationError( diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py index 0aade09076..3d2525035c 100644 --- a/vllm/model_executor/models/qwen3_vl.py +++ b/vllm/model_executor/models/qwen3_vl.py @@ -894,6 +894,19 @@ class Qwen3_VisionTransformer(nn.Module): return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) +class Qwen3VLMultiModalDataParser(Qwen2VLMultiModalDataParser): + # Timestamps are part of the prompt replacement (each frame group's + # "" text), so they size the placeholder range and must + # travel with the grid when embeddings are delivered out of band. + embedding_fields = { + **Qwen2VLMultiModalDataParser.embedding_fields, + "video": { + **Qwen2VLMultiModalDataParser.embedding_fields["video"], + "timestamps": "metadata", + }, + } + + class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo): def get_hf_config(self): return self.ctx.get_hf_config(Qwen3VLConfig) @@ -912,7 +925,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo): return self.get_hf_processor(**kwargs).video_processor def get_data_parser(self): - return Qwen2VLMultiModalDataParser( + return Qwen3VLMultiModalDataParser( self.get_hf_config().vision_config.spatial_merge_size, video_needs_metadata=True, expected_hidden_size=self._get_expected_hidden_size(), @@ -1609,7 +1622,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) def get_video_repl( *, tokens_per_frame: list[int], - timestamps: list[float | int], + timestamps: list[float | int] | torch.Tensor, tokenizer: TokenizerLike, vision_start_token_id: int, vision_end_token_id: int, diff --git a/vllm/multimodal/media/connector.py b/vllm/multimodal/media/connector.py index bf3f9d85f1..8980418b32 100644 --- a/vllm/multimodal/media/connector.py +++ b/vllm/multimodal/media/connector.py @@ -35,7 +35,7 @@ from vllm.utils.registry import ExtensionManager from .audio import AudioEmbeddingMediaIO, AudioMediaIO from .base import MediaIO, MediaWithBytes from .image import ImageEmbeddingMediaIO, ImageMediaIO -from .video import VideoMediaIO +from .video import VideoEmbeddingMediaIO, VideoMediaIO logger = init_logger(__name__) @@ -653,3 +653,28 @@ class MediaConnector: return await loop.run_in_executor( global_thread_pool, audio_embedding_io.load_base64, "", data ) + + def fetch_video_embedding( + self, + data: str, + ) -> torch.Tensor: + """ + Load video embedding from a URL. + """ + video_embedding_io = VideoEmbeddingMediaIO() + + return video_embedding_io.load_base64("", data) + + async def fetch_video_embedding_async( + self, + data: str, + ) -> torch.Tensor: + """ + Asynchronously load video embedding from a URL. + """ + video_embedding_io = VideoEmbeddingMediaIO() + loop = asyncio.get_running_loop() + + return await loop.run_in_executor( + global_thread_pool, video_embedding_io.load_base64, "", data + ) diff --git a/vllm/multimodal/media/video.py b/vllm/multimodal/media/video.py index fd7e23293a..b53f93785a 100644 --- a/vllm/multimodal/media/video.py +++ b/vllm/multimodal/media/video.py @@ -1,20 +1,24 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from functools import partial +from io import BytesIO from pathlib import Path from typing import Any import numpy as np import numpy.typing as npt import pybase64 +import torch from PIL import Image from vllm import envs from vllm.logger import init_logger +from vllm.utils.serial_utils import tensor2base64 +from vllm.utils.sparse_utils import check_sparse_tensor_invariants_threadsafe from ..video import VIDEO_LOADER_REGISTRY from .base import MediaIO, MediaWithBytes -from .image import ImageMediaIO +from .image import MAGIC_NUMPY_PREFIX, ImageMediaIO logger = init_logger(__name__) @@ -194,3 +198,45 @@ class VideoMediaIO(MediaIO[MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]]): msg = "Only JPEG format is supported for now." raise NotImplementedError(msg) + + +class VideoEmbeddingMediaIO(MediaIO[torch.Tensor]): + """Video embedding MediaIO implementation. + + Configuration values can be user-provided either by --media-io-kwargs or + by the runtime API field "media_io_kwargs". Ensure proper validation and + error handling. + """ + + def __init__(self) -> None: + super().__init__() + + def _load_pickled_torch(self, data: bytes) -> torch.Tensor: + buffer = BytesIO(data) + with check_sparse_tensor_invariants_threadsafe(): + tensor = torch.load(buffer, weights_only=True) + return tensor.to_dense() + + def _load_numpy(self, data: bytes) -> torch.Tensor: + with BytesIO(data) as buffer: + return torch.from_numpy(np.load(buffer)) + + def load_bytes(self, data: bytes) -> torch.Tensor: + if data[:6] == MAGIC_NUMPY_PREFIX: + return self._load_numpy(data) + + return self._load_pickled_torch(data) + + def load_base64(self, media_type: str, data: str) -> torch.Tensor: + return self.load_bytes(pybase64.b64decode(data, validate=True)) + + def load_file(self, filepath: Path) -> torch.Tensor: + if filepath.suffix == ".npy": + return torch.from_numpy(np.load(filepath)) + + with check_sparse_tensor_invariants_threadsafe(): + tensor = torch.load(filepath, weights_only=True) + return tensor.to_dense() + + def encode_base64(self, media: torch.Tensor) -> str: + return tensor2base64(media)