From b30291cf0c9a15898589f0347906bad07f3b538b Mon Sep 17 00:00:00 2001 From: Tianyu Guo Date: Thu, 6 Aug 2026 06:59:54 +0800 Subject: [PATCH] [EPD] Remove duplicate image preprocessing in EPD and enable preprocess on GPU (#50390) Signed-off-by: Tianyu Guo --- .../disaggregated_encoder/disagg_epd_proxy.py | 168 ++++++++++++++++- tests/config/test_multimodal_config.py | 174 ++++++++++++++++++ vllm/config/ec_transfer.py | 10 + vllm/config/model.py | 7 + vllm/config/multimodal.py | 136 ++++++++++++++ vllm/config/vllm.py | 99 ++++++++++ .../ec_connector/example_connector.py | 69 ++++++- vllm/engine/arg_utils.py | 27 +++ vllm/model_executor/models/colqwen3.py | 1 + vllm/model_executor/models/colqwen3_5.py | 1 + vllm/model_executor/models/hunyuan_vision.py | 10 +- vllm/model_executor/models/keye.py | 21 ++- vllm/model_executor/models/keye_vl1_5.py | 21 ++- .../model_executor/models/llava_onevision2.py | 10 +- vllm/model_executor/models/minicpmv.py | 25 ++- vllm/model_executor/models/opencua.py | 1 + .../models/qwen2_5_omni_thinker.py | 1 + vllm/model_executor/models/qwen2_vl.py | 15 +- vllm/model_executor/models/qwen3_vl.py | 1 + vllm/multimodal/inputs.py | 14 +- vllm/multimodal/parse.py | 95 +++++++++- vllm/multimodal/processing/context.py | 43 ++++- 22 files changed, 904 insertions(+), 45 deletions(-) diff --git a/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py b/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py index dab9677295..9a1bd51721 100644 --- a/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py +++ b/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py @@ -21,13 +21,18 @@ from __future__ import annotations import argparse import asyncio +import hashlib +import io +import json import logging import os import random +import time import uuid from collections.abc import AsyncIterator import aiohttp +import pybase64 as base64 import uvicorn from fastapi import FastAPI, HTTPException, Request from fastapi.responses import JSONResponse, StreamingResponse @@ -53,6 +58,89 @@ decode_session: aiohttp.ClientSession | None = None MM_TYPES = {"image_url", "audio_url", "input_audio"} +# Diagnostic switch: forward the original request to the decoder so the +# only difference from the rewrite path is the rewrite itself. +NO_REWRITE = False + + +# Grid metadata reported by the encoder instance, keyed by item index. +# Empty when the encoder did not report any (then nothing is rewritten). +def content_uuid(item: dict) -> str: + """Cache key for a multimodal item, derived from its content. + + Must be content-derived, not request-derived: the EC cache is keyed by this + value, so a per-request key (a request id, say) would make every request a + miss and throw away cross-request reuse of already-encoded media -- while + the unmodified path, which hashes the content, would keep it. That asymmetry + silently biases any comparison between the two. + """ + url = (item.get("image_url") or item.get("audio_url") or {}).get("url") or "" + payload = url or json.dumps(item, sort_keys=True) + return hashlib.sha256(payload.encode()).hexdigest() + + +def _b64_tensor(values: list) -> str: + import torch + + buf = io.BytesIO() + grid = torch.tensor(values, dtype=torch.long) + # Downstream stacks per item, so hand over a flat (t, h, w). + torch.save(grid.reshape(-1)[:3], buf) + return base64.b64encode(buf.getvalue()).decode() + + +def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict: + """Replace each image item with a metadata-only reference for the decoder. + + The decoder does not need the pixels: the encoder instance already produced + the embedding and published it through the EC connector under the same uuid. + Sending only the grid lets the decoder size the placeholder range without + re-running the image transform. + + `item_meta` holds what the encoder reported for each item (its cache key and + the grid its processor actually produced), so the grid is never re-derived + here -- a second derivation could disagree with the encoder's. + """ + rewritten = 0 + idx = 0 + new_messages = [] + for msg in req_data.get("messages", []): + content = msg.get("content") + if not isinstance(content, list): + new_messages.append(msg) + continue + new_content = [] + for item in content: + if item.get("type") not in MM_TYPES: + new_content.append(item) + continue + meta = dict(item_meta.get(idx) or {}) + idx += 1 + item_uuid = meta.pop("mm_hash", None) + # Whatever keys the encoder reported are the metadata its model + # declared as needed to size the placeholder range; the proxy does + # not need to know their names. + metadata = {k: _b64_tensor(v) for k, v in meta.items()} + if not metadata or not item_uuid: + # The encoder reported no metadata (e.g. the item came from its + # processor cache); let the decoder process the media itself. + new_content.append(item) + continue + new_content.append( + { + "type": "image_embeds", + "image_embeds": metadata, + "uuid": item_uuid, + } + ) + rewritten += 1 + new_messages.append({**msg, "content": new_content}) + + if not rewritten: + return req_data + logger.info("Rewrote %d image item(s) as metadata references", rewritten) + return {**req_data, "messages": new_messages} + def extract_mm_items(request_data: dict) -> list[dict]: """ @@ -77,22 +165,29 @@ async def fanout_encoder_primer( orig_request: dict, e_urls: list[str], req_id: str, -) -> None: +) -> dict[int, dict]: """ 1. Build one request *per MM item* with all text removed. 2. Send them concurrently to the encode cluster. 3. Raise if any of them fails. + + Returns, per item index, the metadata the encoder reported in + `ec_transfer_params`: its EC cache key and the grid its processor produced. + The proxy still supplies the uuid so both sides key the cache the same way; + the grid can only come from the encoder, which is the side that computed it. """ logger.info("[%s] Processing multimodal items...", req_id) mm_items = extract_mm_items(orig_request) if not mm_items: logger.info("[%s] No multimodal items, skipping encoder", req_id) - return # nothing to do + return {} # nothing to do logger.info("[%s] got %d multimodal items...", req_id, len(mm_items)) tasks = [] + item_uuids: dict[int, str] = {} + item_meta: dict[int, dict] = {} # Round-robin over encode servers to distribute load a bit url_cycle = (e_urls[i % len(e_urls)] for i in range(len(mm_items))) @@ -102,11 +197,24 @@ async def fanout_encoder_primer( child_req_id = f"{req_id}:{idx}:{uuid.uuid4().hex[:6]}" headers = {"x-request-id": child_req_id} + # With --no-rewrite the decoder still receives the raw image and derives + # the cache key by hashing it, so the encoder must do the same -- passing + # a uuid here would make the two disagree and silently defeat the EC + # transfer, leaving the decoder to encode the image itself. + item_uuid = None if NO_REWRITE else content_uuid(item) + if item_uuid is not None: + item_uuids[idx] = item_uuid + encoder_req = { # You *may* need to keep additional fields "model": orig_request.get("model"), "messages": [ - {"role": "user", "content": [item]}, + { + "role": "user", + "content": [ + item if item_uuid is None else {**item, "uuid": item_uuid} + ], + }, ], # No max_tokens cap: the encoder instance never samples, it finishes # once the prompt is encoded and its embeddings are published. @@ -152,9 +260,21 @@ async def fanout_encoder_primer( detail=f"Encoder request failed: {detail}", ) + # The encoder reports each item's cache key and grid here. + try: + params = (await r.json()).get("ec_transfer_params") or {} + reported = params.get("ec_items") or [] + except Exception: + logger.warning("[%s] Could not read encoder metadata #%d", req_id, idx) + reported = [] + if reported and idx in item_uuids: + # One item per encoder request, so the first entry is this item's. + item_meta[idx] = {**reported[0], "mm_hash": item_uuids[idx]} + logger.info( "[%s] All %d encoder requests completed successfully", req_id, len(mm_items) ) + return item_meta async def maybe_prefill( @@ -320,7 +440,11 @@ async def forward_non_stream( ) -> dict: try: # Step 1: Process through Encoder instance (if has MM input) - await fanout_encoder_primer(req_data, e_urls, req_id) + _t0 = time.perf_counter() + item_meta = await fanout_encoder_primer(req_data, e_urls, req_id) + _t1 = time.perf_counter() + req_data = req_data if NO_REWRITE else rewrite_for_decode(req_data, item_meta) + _t2 = time.perf_counter() # Step 2: Process through Prefill instance req_data = await maybe_prefill(req_data, p_url, req_id) @@ -334,7 +458,17 @@ async def forward_non_stream( f"{d_url}/v1/chat/completions", json=req_data, headers=headers ) as resp: resp.raise_for_status() - return await resp.json() + out = await resp.json() + _t3 = time.perf_counter() + logger.info( + "STAGE %s encode=%.1f rewrite=%.1f decode=%.1f total=%.1f", + "no-rewrite" if NO_REWRITE else "rewrite", + (_t1 - _t0) * 1e3, + (_t2 - _t1) * 1e3, + (_t3 - _t2) * 1e3, + (_t3 - _t0) * 1e3, + ) + return out except HTTPException: raise @@ -348,7 +482,11 @@ async def forward_stream( ) -> AsyncIterator[str]: try: # Step 1: Process through Encoder instance (if has MM input) - await fanout_encoder_primer(req_data, e_urls, req_id) + _t0 = time.perf_counter() + item_meta = await fanout_encoder_primer(req_data, e_urls, req_id) + _t1 = time.perf_counter() + req_data = req_data if NO_REWRITE else rewrite_for_decode(req_data, item_meta) + _t2 = time.perf_counter() # Step 2: Process through Prefill instance req_data = await maybe_prefill(req_data, p_url, req_id) @@ -358,6 +496,7 @@ async def forward_stream( headers = {"x-request-id": req_id} # Streaming response + _first = None async with decode_session.post( f"{d_url}/v1/chat/completions", json=req_data, @@ -366,8 +505,19 @@ async def forward_stream( resp.raise_for_status() async for chunk in resp.content.iter_chunked(1024): if chunk: + if _first is None: + _first = time.perf_counter() yield chunk.decode("utf-8", errors="ignore") + _t3 = time.perf_counter() + logger.info( + "STAGE %s encode=%.1f rewrite=%.2f decode_ttfb=%.1f decode_total=%.1f", + "no-rewrite" if NO_REWRITE else "rewrite", + (_t1 - _t0) * 1e3, + (_t2 - _t1) * 1e3, + ((_first or _t3) - _t2) * 1e3, + (_t3 - _t2) * 1e3, + ) logger.info("[%s] Streaming completed", req_id) except HTTPException: @@ -553,6 +703,11 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--host", default="0.0.0.0") parser.add_argument("--port", type=int, default=8000) + parser.add_argument( + "--no-rewrite", + action="store_true", + help="Forward images to the decoder unchanged (for stage-timing A/B).", + ) parser.add_argument( "--encode-servers-urls", required=True, @@ -573,6 +728,7 @@ if __name__ == "__main__": ) args = parser.parse_args() + NO_REWRITE = args.no_rewrite app.state.e_urls = [ u.strip() for u in args.encode_servers_urls.split(",") if u.strip() ] diff --git a/tests/config/test_multimodal_config.py b/tests/config/test_multimodal_config.py index 5d0d755e51..fc731e69bc 100644 --- a/tests/config/test_multimodal_config.py +++ b/tests/config/test_multimodal_config.py @@ -4,10 +4,13 @@ from unittest.mock import MagicMock, patch import pytest +import torch from transformers import PretrainedConfig +from vllm.config.ec_transfer import ECRole, ECTransferConfig from vllm.config.model import ModelConfig from vllm.config.multimodal import MultiModalConfig +from vllm.config.vllm import VllmConfig from vllm.transformers_utils.model_arch_config_convertor import ( ModelArchConfigConvertorBase, ) @@ -173,3 +176,174 @@ def test_sticky_cache_survives_text_subconfig_regeneration(): side_effect=AssertionError("must not re-query registry"), ): assert text_config._supports_multimodal_for_mm_prefix() is False + + +@pytest.mark.parametrize( + ("device", "expected"), + [ + (None, None), + ("cpu", "cpu"), + ("cuda", "cuda"), + # Callers compare against a bare device type, so an indexed device and a + # torch.device must normalise to the same thing as "cuda". + ("cuda:1", "cuda"), + (torch.device("cuda", 1), "cuda"), + ], +) +def test_mm_processor_device_type_normalizes(device: object, expected: str | None): + kwargs = {} if device is None else {"device": device} + config = MultiModalConfig(mm_processor_kwargs=kwargs) + assert config.get_mm_processor_device_type() == expected + + +def _validate_mm_processor_device(*, device: str, ec_role: ECRole | None) -> None: + ec_config = ( + None + if ec_role is None + # `is_ec_producer`/`is_ec_consumer` are False without a connector, so an + # unset one would make every role look like "no EC role at all". + else ECTransferConfig(ec_connector="ECExampleConnector", ec_role=ec_role) + ) + MultiModalConfig( + mm_processor_kwargs={"device": device} + ).validate_mm_processor_device(ec_config) + + +@pytest.mark.parametrize("ec_role", [None, "ec_producer"]) +def test_bad_mm_processor_device_rejected(ec_role: ECRole | None): + """A bad device must fail during startup, not mid-request. + + Rejected for every role, and on a CPU-only platform too, so a typo can never + silently fall through to running the processor somewhere unintended. + """ + with ( + patch("vllm.platforms.current_platform.device_type", "cpu"), + pytest.raises(ValueError, match='Invalid "device" in mm_processor_kwargs'), + ): + _validate_mm_processor_device(device="not-a-device", ec_role=ec_role) + + +@pytest.mark.parametrize("ec_role", [None, "ec_consumer", "ec_both"]) +def test_accelerator_mm_processor_rejected_outside_encoder_instance( + ec_role: ECRole | None, +): + """Only an encode-only EPD instance has the device to itself. + + Every other role runs the language model in the same process, so frontend + accelerator work would contend with the forward pass and allocate outside + the memory profiled for the KV cache. + """ + with ( + patch("vllm.platforms.current_platform.device_type", "cuda"), + pytest.raises(ValueError, match="also runs the language model"), + ): + _validate_mm_processor_device(device="cuda", ec_role=ec_role) + + +def test_accelerator_mm_processor_allowed_on_encoder_instance(): + with patch("vllm.platforms.current_platform.device_type", "cuda"): + _validate_mm_processor_device(device="cuda", ec_role="ec_producer") + + +def test_cpu_mm_processor_needs_no_ec_role(): + """The gate only applies to the accelerator, so CPU stays unrestricted.""" + with patch("vllm.platforms.current_platform.device_type", "cuda"): + _validate_mm_processor_device(device="cpu", ec_role=None) + + +@pytest.mark.parametrize( + ("flag", "kwargs", "expected"), + [ + # "auto" is only settled once the EC role is known, so it must leave no + # device behind for `VllmConfig` to mistake for an explicit request. + ("auto", None, None), + (None, None, None), + ("cpu", None, "cpu"), + ("cuda", None, "cuda"), + # Any explicit value other than "cpu" means "whatever this platform + # calls its accelerator". + ("gpu", None, "cuda"), + # An explicit device in the kwargs wins over the convenience flag. + ("cpu", {"device": "cuda"}, "cuda"), + ("cuda", {"device": "cpu"}, "cpu"), + ], +) +def test_fold_mm_processor_device( + flag: str | None, kwargs: dict | None, expected: str | None +): + with patch("vllm.platforms.current_platform.device_type", "cuda"): + folded = MultiModalConfig.fold_mm_processor_device(kwargs, flag) + + config = MultiModalConfig(mm_processor_kwargs=folded or {}) + assert config.get_mm_processor_device_type() == expected + + +def _resolve_mm_processor_device( + *, + ec_role: ECRole | None, + mm_tensor_ipc: str = "torch_shm", + device: str | None = None, +) -> str | None: + """Run the `auto` resolution and report where the processor ended up.""" + mm_config = MultiModalConfig( + mm_processor_kwargs={} if device is None else {"device": device}, + mm_tensor_ipc=mm_tensor_ipc, # type: ignore[arg-type] + ) + model_config = MagicMock(spec=ModelConfig) + model_config.multimodal_config = mm_config + vllm_config = MagicMock(spec=VllmConfig) + vllm_config.model_config = model_config + vllm_config.ec_transfer_config = ( + None + if ec_role is None + else ECTransferConfig(ec_connector="ECExampleConnector", ec_role=ec_role) + ) + + with patch("vllm.platforms.current_platform.device_type", "cuda"): + VllmConfig._resolve_mm_processor_device(vllm_config) + return mm_config.get_mm_processor_device_type() + + +def test_auto_mm_processor_device_uses_accelerator_on_encoder_instance(): + """The one deployment that has the device to itself and can hand it over.""" + assert _resolve_mm_processor_device(ec_role="ec_producer") == "cuda" + + +@pytest.mark.parametrize("ec_role", [None, "ec_consumer", "ec_both"]) +def test_auto_mm_processor_device_stays_on_cpu_off_encoder_instance( + ec_role: ECRole | None, +): + """Every other role runs the language model in the same process.""" + assert _resolve_mm_processor_device(ec_role=ec_role) is None + + +def test_auto_mm_processor_device_needs_a_device_capable_transport(): + """Other transports serialize host bytes, so the output is copied back.""" + assert ( + _resolve_mm_processor_device(ec_role="ec_producer", mm_tensor_ipc="direct_rpc") + is None + ) + + +def test_auto_mm_processor_device_leaves_an_explicit_request_alone(): + assert _resolve_mm_processor_device(ec_role="ec_producer", device="cpu") == "cpu" + + +def test_vllm_config_runs_the_mm_processor_device_check(): + """Startup must reach the check; the rule itself is covered above. + + Guards the wiring, which no other test would notice going missing. + """ + model_config = MagicMock(spec=ModelConfig) + model_config.multimodal_config = MultiModalConfig( + mm_processor_kwargs={"device": "cuda"} + ) + vllm_config = MagicMock(spec=VllmConfig) + vllm_config.model_config = model_config + vllm_config.ec_transfer_config = None + + with ( + patch("vllm.platforms.current_platform.device_type", "cuda"), + pytest.raises(ValueError, match="also runs the language model"), + ): + VllmConfig._validate_mm_processor_device(vllm_config) diff --git a/vllm/config/ec_transfer.py b/vllm/config/ec_transfer.py index a3a927d51e..c64bf13ec0 100644 --- a/vllm/config/ec_transfer.py +++ b/vllm/config/ec_transfer.py @@ -103,5 +103,15 @@ class ECTransferConfig: def is_ec_consumer(self) -> bool: return self.ec_connector is not None and self.ec_role in get_args(ECConsumer) + @property + def is_encode_only(self) -> bool: + """Whether this instance encodes but does not run the language model. + + It allocates no KV cache either -- `GPUModelRunner.get_kv_cache_spec` + returns {} for it -- so it is the one role that can spend accelerator + time and memory on frontend work. + """ + return self.is_ec_producer and not self.is_ec_consumer + def get_from_extra_config(self, key, default) -> Any: return self.ec_connector_extra_config.get(key, default) diff --git a/vllm/config/model.py b/vllm/config/model.py index 517bb3847c..a3b78744b0 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -18,6 +18,7 @@ from vllm.config.multimodal import ( MMCacheType, MMEncoderTPMode, MMHasherAlgorithm, + MMProcessorDevice, MMTensorIPC, MultiModalConfig, ) @@ -395,6 +396,7 @@ class ModelConfig: video_pruning_method: InitVar[str | None] = None mm_tensor_ipc: InitVar[MMTensorIPC] = None mm_ipc_gpu_memory_gb: InitVar[float | None] = None + mm_processor_device: InitVar[MMProcessorDevice | None] = None def compute_hash(self) -> str: """ @@ -525,6 +527,7 @@ class ModelConfig: video_pruning_method: str | None, mm_tensor_ipc: MMTensorIPC, mm_ipc_gpu_memory_gb: float | None, + mm_processor_device: MMProcessorDevice | None, ) -> None: # Keep set served_model_name before maybe_model_redirect(self.model) self.served_model_name = get_served_model_name( @@ -763,6 +766,10 @@ class ModelConfig: ) mm_encoder_tp_mode = "weights" + mm_processor_kwargs = MultiModalConfig.fold_mm_processor_device( + mm_processor_kwargs, mm_processor_device + ) + mm_config_kwargs = dict( language_model_only=language_model_only, limit_per_prompt=limit_mm_per_prompt, diff --git a/vllm/config/multimodal.py b/vllm/config/multimodal.py index 6c85fc7c93..713a31b3ca 100644 --- a/vllm/config/multimodal.py +++ b/vllm/config/multimodal.py @@ -5,14 +5,19 @@ from collections.abc import Mapping from pathlib import Path from typing import Any, Literal, TypeAlias, TypedDict, cast, final +import torch from pydantic import ConfigDict, Field, field_validator, model_validator from pydantic.dataclasses import dataclass import vllm.envs as envs +from vllm.config.ec_transfer import ECTransferConfig from vllm.config.utils import config, get_from_deprecated_env_if_set +from vllm.logger import init_logger from vllm.utils.hashing import safe_hash from vllm.v1.attention.backends.registry import AttentionBackendEnum +logger = init_logger(__name__) + @dataclass class BaseDummyOptions: @@ -64,6 +69,10 @@ MMCacheType = Literal["shm", "lru"] VideoPruningMethod = Literal["evs", "vidcom2"] MMTensorIPC = Literal["direct_rpc", "torch_shm"] MMHasherAlgorithm = Literal["blake3", "sha256", "sha512"] +MMProcessorDevice: TypeAlias = str +"""`"auto"`, `"cpu"`, or the platform's own accelerator name +(`current_platform.device_type`, e.g. `"cuda"` on CUDA and ROCm, +`"xpu"` on XPU). Validated against that set by the CLI.""" def _get_mm_hasher_algorithm() -> MMHasherAlgorithm: @@ -222,6 +231,20 @@ class MultiModalConfig: - "direct_rpc": Use msgspec serialization via RPC - "torch_shm": Use torch.multiprocessing shared memory for zero-copy IPC Defaults to "direct_rpc". """ + mm_embeds_from_ec_connector: bool = False + """Whether a pre-computed-embedding input may omit the `*_embeds` tensor. + + In an encode/prefill/decode (EPD) deployment the encoder instance publishes + embeddings through the EC connector, so the request that reaches the + prefill/decode instance only needs to carry the grid/size metadata that + sizes the placeholder range — the embeddings themselves come from the + connector, keyed by `mm_hash`. + + Derived, not user-settable: `VllmConfig.__post_init__` sets this to True + exactly on EC consumers. Everywhere else it stays False so that a request + which forgets its embeddings still fails fast in the frontend, with a clear + error, rather than deep inside the model.""" + mm_ipc_gpu_memory_gb: float = Field(default=0, ge=0) """Amount of GPU memory (in GiB) sequestered on the engine's device for GPU-side multimodal work in the API-server (frontend) process, such as @@ -319,6 +342,119 @@ class MultiModalConfig: ) return self + @staticmethod + def fold_mm_processor_device( + mm_processor_kwargs: dict[str, Any] | None, + mm_processor_device: MMProcessorDevice | None, + ) -> dict[str, Any] | None: + """Fold the `mm_processor_device` convenience flag into the kwargs. + + The flag keeps no state of its own: `mm_processor_kwargs["device"]` is + the only representation of where the processor runs, so an explicit + `device` there always wins and `"auto"` stays unresolved for + `VllmConfig`, which is where the EC role needed to resolve it lives. + + Args: + mm_processor_kwargs: The kwargs as given, or None. + mm_processor_device: The flag's value, or None when unset. + + Returns: + The kwargs to build the config with, unchanged unless the flag adds + a `device`. + """ + if mm_processor_device in (None, "auto"): + return mm_processor_kwargs + if (mm_processor_kwargs or {}).get("device") is not None: + return mm_processor_kwargs + + from vllm.platforms import current_platform + + # Any explicit value other than "cpu" means "the accelerator", so a + # programmatically-set "cuda" still works on a platform whose device type + # is named differently ("xpu"), and degrades to CPU where there is none. + device = ( + "cpu" + if mm_processor_device == "cpu" + else (current_platform.device_type or "cpu") + ) + return {**(mm_processor_kwargs or {}), "device": device} + + def get_mm_processor_device_type(self) -> str | None: + """The torch device type `mm_processor_kwargs["device"]` names. + + `mm_processor_kwargs` is untyped, so `device` may be any form torch + accepts -- `"cuda"`, `"cuda:1"`, `torch.device(...)`, or a bare index. + Normalising through torch rather than parsing the string keeps the + non-string forms from slipping past a caller's comparison. + + Returns: + The device type, or None when no device is requested. + + Raises: + ValueError: If `device` is not something `torch.device` accepts. + `validate_mm_processor_device` is what surfaces this during + startup, so the value is only parsed once. + """ + device = (self.mm_processor_kwargs or {}).get("device") + if device is None: + return None + try: + return torch.device(device).type # type: ignore[arg-type] + except (RuntimeError, TypeError, ValueError): + raise ValueError( + f'Invalid "device" in mm_processor_kwargs: {device!r}. Expected a ' + 'torch device such as "cpu", "cuda" or "cuda:0".' + ) from None + + def validate_mm_processor_device(self, ec_config: ECTransferConfig | None) -> None: + """Check `mm_processor_kwargs["device"]` for this deployment. + + The only place the requested device is validated, so it runs even on a + CPU-only platform: the value is parsed before any early return. + + Args: + ec_config: The deployment's EC config, or None when it is not an + encode/prefill/decode deployment. Passed in because it is not + reachable from here, and because a field assigned after + construction would not re-trigger this config's validators. + + Raises: + ValueError: If the requested device is not a torch device, or if it + is the accelerator on an instance that also runs the language + model. + """ + from vllm.platforms import current_platform + + device_type = self.get_mm_processor_device_type() + accelerator = current_platform.device_type + if device_type is None or accelerator in ("", "cpu"): + return + if device_type != accelerator: + return + + if ec_config is None or not ec_config.is_encode_only: + raise ValueError( + f"Cannot run the multi-modal processor on {device_type!r}: this " + "instance also runs the language model. The processor would " + "share the device with the model's forward pass, so its " + "transform kernels contend with that compute, and because it " + "runs in the API-server process its allocations are outside the " + "memory the engine profiled for its KV cache -- risking OOM or " + "a silently shrunken cache.\n" + "Accelerator preprocessing is only supported on an encode-only " + "instance of an encode/prefill/decode deployment (an EC " + "producer that is not also a consumer), which runs no forward " + "pass and allocates no KV cache.\n" + 'Use --mm-processor-device=cpu, or drop "device" from ' + "--mm-processor-kwargs." + ) + + logger.info_once( + "Running the multi-modal processor on %s. Override with " + "--mm-processor-device=cpu.", + device_type, + ) + def compute_hash(self) -> str: """ WARNING: Whenever a new field is added to this config, diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 4cef46020b..427d9dbf5b 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1509,6 +1509,10 @@ class VllmConfig: ) current_platform.check_and_update_config(self) + self._resolve_mm_embeds_from_ec_connector() + self._resolve_mm_processor_device() + self._validate_mm_processor_device() + if self.use_v2_model_runner: self._validate_v2_model_runner() elif self.parallel_config.prefill_context_parallel_size > 1: @@ -2185,6 +2189,101 @@ class VllmConfig: f"kernel_config={self.kernel_config!r}" ) + def _resolve_mm_embeds_from_ec_connector(self) -> None: + """Allow `*_embeds` to be omitted only where the connector supplies them. + + That is exactly an EC consumer: the encoder instance publishes the + embeddings through the EC connector, so the request only has to carry + the grid metadata that sizes the placeholder range. On every other + deployment a missing `*_embeds` is a client error and must keep failing + fast in the frontend. + """ + model_config = self.model_config + if model_config is None: + return + mm_config = model_config.multimodal_config + if mm_config is None: + return + + ec_config = self.ec_transfer_config + # Derived, so overwrite unconditionally rather than honouring a value + # that was set by hand. + mm_config.mm_embeds_from_ec_connector = ( + ec_config is not None and ec_config.is_ec_consumer + ) + if mm_config.mm_embeds_from_ec_connector: + logger.info_once( + "EC consumer: pre-computed-embedding inputs may omit the " + "embedding tensor; embeddings are loaded from the EC connector." + ) + + def _resolve_mm_processor_device(self) -> None: + """Settle `--mm-processor-device=auto` now that the EC role is known. + + "auto" means "the accelerator, but only where the processor has it to + itself and its output can be handed over without a copy back to host": + an encode-only instance whose tensor transport carries device tensors. + Every other deployment keeps the processor on CPU. + + An explicit device -- from `--mm-processor-device` or straight from + `mm_processor_kwargs` -- is already folded in by `MultiModalConfig`, so + it is left alone here and validated by `_validate_mm_processor_device`. + """ + model_config = self.model_config + if model_config is None: + return + mm_config = model_config.multimodal_config + if mm_config is None: + return + if mm_config.get_mm_processor_device_type() is not None: + return + + from vllm.platforms import current_platform + + device_type = current_platform.device_type + if device_type in ("", "cpu"): + return + + ec_config = self.ec_transfer_config + # An EC producer that is not also a consumer runs no forward pass and + # allocates no KV cache, so frontend accelerator work has the device to + # itself. + if ec_config is None or not ec_config.is_encode_only: + return + + if mm_config.mm_tensor_ipc != "torch_shm": + # Any other transport serializes host bytes, so the output would be + # copied back, and that copy costs more than running the transform + # on device saves. + logger.info_once( + "EPD encoder instance: keeping the multi-modal processor on CPU " + "because mm_tensor_ipc=%s cannot carry device tensors. Add " + "--mm-tensor-ipc=torch_shm to run it on the accelerator.", + mm_config.mm_tensor_ipc, + ) + return + + mm_config.mm_processor_kwargs = { + **(mm_config.mm_processor_kwargs or {}), + "device": device_type, + } + logger.info_once( + "EPD encoder instance: running the multi-modal processor on %s. " + "Override with --mm-processor-device=cpu.", + device_type, + ) + + def _validate_mm_processor_device(self) -> None: + """Hand the EC config to `MultiModalConfig`, which owns the rule.""" + model_config = self.model_config + if model_config is None: + return + mm_config = model_config.multimodal_config + if mm_config is None: + return + + mm_config.validate_mm_processor_device(self.ec_transfer_config) + def _get_v2_model_runner_unsupported_features(self) -> list[str]: """Collect features not yet supported by the V2 model runner.""" unsupported: list[str] = [] diff --git a/vllm/distributed/ec_transfer/ec_connector/example_connector.py b/vllm/distributed/ec_transfer/ec_connector/example_connector.py index edcba3a696..8d88934ac1 100644 --- a/vllm/distributed/ec_transfer/ec_connector/example_connector.py +++ b/vllm/distributed/ec_transfer/ec_connector/example_connector.py @@ -2,9 +2,10 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import os from dataclasses import dataclass -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import safetensors +import torch from vllm.config import VllmConfig from vllm.distributed.ec_transfer.ec_connector.base import ( @@ -50,6 +51,8 @@ class ECExampleConnector(ECConnectorBase): super().__init__(vllm_config=vllm_config, role=role) # req_id -> index self._mm_datas_need_loads: dict[str, int] = {} + self._model_config = vllm_config.model_config + self._metadata_fields_cache: dict[str, set[str]] = {} transfer_config = vllm_config.ec_transfer_config if transfer_config is not None: self._storage_path = transfer_config.get_from_extra_config( @@ -165,6 +168,70 @@ class ECExampleConnector(ECConnectorBase): self._mm_datas_need_loads.clear() return meta + def _placeholder_metadata_fields(self, modality: str) -> set[str]: + """Which processed keys this model needs published for `modality`. + + Read from `MultiModalDataParser.embedding_fields`, the same declaration + the consumer's parser requires, so the two cannot drift. An empty set + means the modality cannot be delivered out of band, and the consumer + will process the media itself. + """ + if modality in self._metadata_fields_cache: + return self._metadata_fields_cache[modality] + + fields: set[str] = set() + try: + from vllm.multimodal import MULTIMODAL_REGISTRY + + info = MULTIMODAL_REGISTRY.create_processor(self._model_config).info + fields = info.data_parser.placeholder_metadata_fields(modality) + except Exception: + # Reporting nothing is a safe degradation: the consumer falls back to + # processing the media itself. + logger.warning( + "Could not determine the placeholder metadata fields for " + "modality %s; the consumer will preprocess the media itself.", + modality, + exc_info=True, + ) + + self._metadata_fields_cache[modality] = fields + return fields + + def request_finished( + self, + request: "Request", + ) -> tuple[bool, dict[str, Any] | None]: + """Report each item's cache key and grid so a consumer can skip the + image transform. + + A consumer only needs the grid to size the prompt's placeholder range; + the embedding itself arrives through this connector. Reporting the grid + the producer actually computed keeps the two sides in agreement without + the caller re-deriving it from the raw media. + """ + if not self.is_producer: + return False, None + + items = [] + for feature in request.mm_features: + metadata = {} + # `data` is None for items served from the processor cache, in which + # case the metadata is unavailable here and the consumer has to fall + # back to processing the media itself. + if feature.data is not None: + wanted = self._placeholder_metadata_fields(feature.modality) + metadata = { + key: value.tolist() + for key, value in feature.data.get_data().items() + if key in wanted and isinstance(value, torch.Tensor) + } + items.append({"mm_hash": feature.identifier, **metadata}) + + if not items: + return False, None + return False, {"ec_items": items} + # ============================== # Helper functions # ============================== diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 41b9d3c6e4..d8e46311de 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -90,6 +90,7 @@ from vllm.config.multimodal import ( MMCacheType, MMEncoderTPMode, MMHasherAlgorithm, + MMProcessorDevice, MMTensorIPC, ) from vllm.config.observability import DetailedTraceModules @@ -596,6 +597,7 @@ class EngineArgs: video_pruning_rate: float | None = MultiModalConfig.video_pruning_rate video_pruning_method: str = MultiModalConfig.video_pruning_method mm_tensor_ipc: MMTensorIPC = MultiModalConfig.mm_tensor_ipc + mm_processor_device: MMProcessorDevice = "auto" mm_ipc_gpu_memory_gb: float = MultiModalConfig.mm_ipc_gpu_memory_gb # LoRA fields enable_lora: bool = False @@ -1353,6 +1355,30 @@ class EngineArgs: multimodal_group.add_argument( "--mm-tensor-ipc", **multimodal_kwargs["mm_tensor_ipc"] ) + multimodal_group.add_argument( + "--mm-processor-device", + choices=["auto", "cpu"] + + ( + [current_platform.device_type] + if current_platform.device_type not in ("", "cpu") + else [] + ), + default="auto", + help="Device the HF multi-modal processor runs the image/video " + "transform on. Convenience for `--mm-processor-kwargs " + "'{\"device\": ...}'`: the value is resolved here and stored there, " + "it is not kept as separate state. Only takes effect for HF " + '"fast" (torchvision-backed) processors, which accept a `device` ' + "argument; the others ignore it and stay on CPU.\n\n" + '"auto" uses the accelerator on encoder instances of an ' + "encode/prefill/decode deployment -- an EC producer that is not " + "also a consumer allocates no KV cache, so its accelerator is not " + "contended by the language model -- and then only when " + "`--mm-tensor-ipc=torch_shm` can carry device tensors, since every " + "other transport would copy the result back to the host and that " + 'copy costs more than it saves. "auto" resolves to "cpu" ' + "everywhere else.", + ) multimodal_group.add_argument( "--mm-ipc-gpu-memory-gb", **multimodal_kwargs["mm_ipc_gpu_memory_gb"], @@ -1743,6 +1769,7 @@ class EngineArgs: video_pruning_method=self.video_pruning_method, mm_tensor_ipc=self.mm_tensor_ipc, mm_ipc_gpu_memory_gb=self.mm_ipc_gpu_memory_gb, + mm_processor_device=self.mm_processor_device, io_processor_plugin=self.io_processor_plugin, renderer_num_workers=self.renderer_num_workers, ) diff --git a/vllm/model_executor/models/colqwen3.py b/vllm/model_executor/models/colqwen3.py index 1db5e07420..dee975ae20 100644 --- a/vllm/model_executor/models/colqwen3.py +++ b/vllm/model_executor/models/colqwen3.py @@ -104,6 +104,7 @@ class ColQwen3ProcessingInfo(Qwen3VLProcessingInfo): spatial_merge_size, video_needs_metadata=self._supports_video, expected_hidden_size=self._get_expected_hidden_size(), + embeds_from_ec_connector=self.embeds_from_ec_connector, ) diff --git a/vllm/model_executor/models/colqwen3_5.py b/vllm/model_executor/models/colqwen3_5.py index 1b481ea2f4..309da6d686 100644 --- a/vllm/model_executor/models/colqwen3_5.py +++ b/vllm/model_executor/models/colqwen3_5.py @@ -101,6 +101,7 @@ class ColQwen3_5ProcessingInfo(Qwen3_5ProcessingInfo): spatial_merge_size, video_needs_metadata=self._supports_video, expected_hidden_size=self._get_expected_hidden_size(), + embeds_from_ec_connector=self.embeds_from_ec_connector, ) diff --git a/vllm/model_executor/models/hunyuan_vision.py b/vllm/model_executor/models/hunyuan_vision.py index fc05b17d3b..d1c19f07f5 100644 --- a/vllm/model_executor/models/hunyuan_vision.py +++ b/vllm/model_executor/models/hunyuan_vision.py @@ -551,15 +551,22 @@ def _hunyuan_vl_field_config(hf_inputs: Mapping[str, torch.Tensor]): class HunYuanVLMultiModalDataParser(MultiModalDataParser): + # The patch grid is what sizes the placeholder range. + embedding_fields = { + "image": {"image_embeds": "values", "image_grid_thw": "metadata"}, + } + def _parse_image_data( self, data: dict[str, torch.Tensor] | ModalityData[ImageItem], ) -> ModalityDataItems[Any, Any] | None: if isinstance(data, dict): + required, optional = self.embedding_field_sets("image") return DictEmbeddingItems( data, modality="image", - required_fields={"image_embeds", "image_grid_thw"}, + required_fields=required, + optional_fields=optional, fields_factory=_hunyuan_vl_field_config, ) @@ -592,6 +599,7 @@ class HunYuanVLProcessingInfo(BaseProcessingInfo): def get_data_parser(self): return HunYuanVLMultiModalDataParser( expected_hidden_size=self._get_expected_hidden_size(), + embeds_from_ec_connector=self.embeds_from_ec_connector, ) def get_supported_mm_limits(self) -> Mapping[str, int | None]: diff --git a/vllm/model_executor/models/keye.py b/vllm/model_executor/models/keye.py index c3d69836a7..aa5ca77bff 100644 --- a/vllm/model_executor/models/keye.py +++ b/vllm/model_executor/models/keye.py @@ -881,18 +881,23 @@ def _keye_field_config( class KeyeMultiModalDataParser(MultiModalDataParser): + # The patch grid is what sizes the placeholder range. + embedding_fields = { + "image": {"image_embeds": "values", "image_grid_thw": "metadata"}, + "video": {"video_embeds": "values", "video_grid_thw": "metadata"}, + } + def _parse_image_data( self, data: dict[str, torch.Tensor] | ModalityData[ImageItem], ) -> ModalityDataItems[Any, Any] | None: if isinstance(data, dict): + required, optional = self.embedding_field_sets("image") return DictEmbeddingItems( data, modality="image", - required_fields={ - "image_embeds", - "image_grid_thw", - }, + required_fields=required, + optional_fields=optional, fields_factory=_keye_field_config, ) @@ -903,13 +908,12 @@ class KeyeMultiModalDataParser(MultiModalDataParser): data: dict[str, torch.Tensor] | ModalityData[VideoItem], ) -> ModalityDataItems[Any, Any] | None: if isinstance(data, dict): + required, optional = self.embedding_field_sets("video") return DictEmbeddingItems( data, modality="video", - required_fields={ - "video_embeds", - "video_grid_thw", - }, + required_fields=required, + optional_fields=optional, fields_factory=_keye_field_config, ) @@ -929,6 +933,7 @@ class KeyeProcessingInfo(BaseProcessingInfo): def get_data_parser(self): return KeyeMultiModalDataParser( expected_hidden_size=self._get_expected_hidden_size(), + embeds_from_ec_connector=self.embeds_from_ec_connector, ) def get_supported_mm_limits( diff --git a/vllm/model_executor/models/keye_vl1_5.py b/vllm/model_executor/models/keye_vl1_5.py index a8400b8d17..26870ebc26 100644 --- a/vllm/model_executor/models/keye_vl1_5.py +++ b/vllm/model_executor/models/keye_vl1_5.py @@ -320,18 +320,23 @@ def _keye_field_config( class KeyeVL1_5MultiModalDataParser(MultiModalDataParser): + # The patch grid is what sizes the placeholder range. + embedding_fields = { + "image": {"image_embeds": "values", "image_grid_thw": "metadata"}, + "video": {"video_embeds": "values", "video_grid_thw": "metadata"}, + } + def _parse_image_data( self, data: dict[str, torch.Tensor] | ModalityData[ImageItem], ) -> ModalityDataItems[Any, Any] | None: if isinstance(data, dict): + required, optional = self.embedding_field_sets("image") return DictEmbeddingItems( data, modality="image", - required_fields={ - "image_embeds", - "image_grid_thw", - }, + required_fields=required, + optional_fields=optional, fields_factory=_keye_field_config, ) @@ -342,13 +347,12 @@ class KeyeVL1_5MultiModalDataParser(MultiModalDataParser): data: dict[str, torch.Tensor] | ModalityData[VideoItem], ) -> ModalityDataItems[Any, Any] | None: if isinstance(data, dict): + required, optional = self.embedding_field_sets("video") return DictEmbeddingItems( data, modality="video", - required_fields={ - "video_embeds", - "video_grid_thw", - }, + required_fields=required, + optional_fields=optional, fields_factory=_keye_field_config, ) @@ -359,6 +363,7 @@ class KeyeVL1_5ProcessingInfo(KeyeProcessingInfo): def get_data_parser(self): return KeyeVL1_5MultiModalDataParser( expected_hidden_size=self._get_expected_hidden_size(), + embeds_from_ec_connector=self.embeds_from_ec_connector, ) def get_max_frame_per_video(self) -> int: diff --git a/vllm/model_executor/models/llava_onevision2.py b/vllm/model_executor/models/llava_onevision2.py index 84b6c369df..daae740c38 100644 --- a/vllm/model_executor/models/llava_onevision2.py +++ b/vllm/model_executor/models/llava_onevision2.py @@ -1279,6 +1279,7 @@ class LlavaOnevision2ProcessingInfo(BaseProcessingInfo): return LlavaOnevision2MultiModalDataParser( self.get_hf_config().vision_config.spatial_merge_size, video_needs_metadata=True, + embeds_from_ec_connector=self.embeds_from_ec_connector, ) def get_hf_processor(self, **kwargs: object): @@ -1510,6 +1511,11 @@ class LlavaOnevision2DummyInputsBuilder( class LlavaOnevision2MultiModalDataParser(MultiModalDataParser): + # The patch grid is what sizes the placeholder range. + embedding_fields = { + "image": {"image_embeds": "values", "image_grid_thw": "metadata"}, + } + def __init__(self, spatial_merge_size: int, *args, **kwargs): self._spatial_merge_size = spatial_merge_size super().__init__(*args, **kwargs) @@ -1519,10 +1525,12 @@ class LlavaOnevision2MultiModalDataParser(MultiModalDataParser): data: dict[str, torch.Tensor] | ModalityData[ImageItem], ) -> ModalityDataItems[Any, Any] | None: if isinstance(data, dict): + required, optional = self.embedding_field_sets("image") return DictEmbeddingItems( data, modality="image", - required_fields={"image_embeds", "image_grid_thw"}, + required_fields=required, + optional_fields=optional, fields_factory=_create_field_factory(self._spatial_merge_size), ) return super()._parse_image_data(data) diff --git a/vllm/model_executor/models/minicpmv.py b/vllm/model_executor/models/minicpmv.py index 60b1112982..97485d17ed 100644 --- a/vllm/model_executor/models/minicpmv.py +++ b/vllm/model_executor/models/minicpmv.py @@ -26,7 +26,7 @@ import math from collections import defaultdict -from collections.abc import Callable, Iterable, Mapping, Sequence +from collections.abc import Callable, Iterable, Mapping, Sequence, Set from functools import partial from itertools import chain from typing import Annotated, Any, Literal, TypeAlias @@ -474,11 +474,14 @@ class MiniCPMVImageEmbeddingItems(DictEmbeddingItems): [Mapping[str, torch.Tensor]], Mapping[str, MultiModalFieldConfig], ], + required_fields: Set[str], + optional_fields: Set[str] = frozenset(), ) -> None: super().__init__( data, modality="image", - required_fields={"image_embeds", "image_sizes"}, + required_fields=required_fields, + optional_fields=optional_fields, fields_factory=fields_factory, ) @@ -495,11 +498,14 @@ class MiniCPMVVideoEmbeddingItems(DictEmbeddingItems): [Mapping[str, torch.Tensor]], Mapping[str, MultiModalFieldConfig], ], + required_fields: Set[str], + optional_fields: Set[str] = frozenset(), ) -> None: super().__init__( data, modality="video", - required_fields={"video_embeds", "video_image_sizes"}, + required_fields=required_fields, + optional_fields=optional_fields, fields_factory=fields_factory, ) @@ -512,14 +518,23 @@ class MiniCPMVVideoEmbeddingItems(DictEmbeddingItems): class MiniCPMVMultiModalDataParser(MultiModalDataParser): + # The per-item sizes are what size the placeholder range. + embedding_fields = { + "image": {"image_embeds": "values", "image_sizes": "metadata"}, + "video": {"video_embeds": "values", "video_image_sizes": "metadata"}, + } + def _parse_image_data( self, data: dict[str, torch.Tensor] | ModalityData[ImageItem], ) -> ModalityDataItems[Any, Any] | None: if isinstance(data, dict): + required, optional = self.embedding_field_sets("image") return MiniCPMVImageEmbeddingItems( data, fields_factory=_minicpmv_field_config, + required_fields=required, + optional_fields=optional, ) return super()._parse_image_data(data) @@ -529,9 +544,12 @@ class MiniCPMVMultiModalDataParser(MultiModalDataParser): data: dict[str, torch.Tensor] | ModalityData[VideoItem], ) -> ModalityDataItems[Any, Any] | None: if isinstance(data, dict): + required, optional = self.embedding_field_sets("video") return MiniCPMVVideoEmbeddingItems( data, fields_factory=_minicpmv_field_config, + required_fields=required, + optional_fields=optional, ) return super()._parse_video_data(data) @@ -573,6 +591,7 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo): def get_data_parser(self): return MiniCPMVMultiModalDataParser( expected_hidden_size=self._get_expected_hidden_size(), + embeds_from_ec_connector=self.embeds_from_ec_connector, ) def get_model_version(self): diff --git a/vllm/model_executor/models/opencua.py b/vllm/model_executor/models/opencua.py index abc196e221..9fb0b526b3 100644 --- a/vllm/model_executor/models/opencua.py +++ b/vllm/model_executor/models/opencua.py @@ -57,6 +57,7 @@ class OpenCUAProcessingInfo(Qwen2VLProcessingInfo): return Qwen2VLMultiModalDataParser( self.get_hf_config().vision_config.spatial_merge_size, expected_hidden_size=self._get_expected_hidden_size(), + embeds_from_ec_connector=self.embeds_from_ec_connector, ) def get_hf_config(self): diff --git a/vllm/model_executor/models/qwen2_5_omni_thinker.py b/vllm/model_executor/models/qwen2_5_omni_thinker.py index 470d7f65f0..0715235ebb 100644 --- a/vllm/model_executor/models/qwen2_5_omni_thinker.py +++ b/vllm/model_executor/models/qwen2_5_omni_thinker.py @@ -372,6 +372,7 @@ class Qwen2_5OmniThinkerProcessingInfo( target_sr=feature_extractor.sampling_rate, target_channels=self.get_target_channels(), expected_hidden_size=self._get_expected_hidden_size(), + embeds_from_ec_connector=self.embeds_from_ec_connector, ) def get_target_channels(self) -> int: diff --git a/vllm/model_executor/models/qwen2_vl.py b/vllm/model_executor/models/qwen2_vl.py index e2e9f24524..2772242dd3 100644 --- a/vllm/model_executor/models/qwen2_vl.py +++ b/vllm/model_executor/models/qwen2_vl.py @@ -795,6 +795,12 @@ def _create_qwen2vl_field_factory( class Qwen2VLMultiModalDataParser(MultiModalDataParser): + # The patch grid is what sizes the placeholder range. + embedding_fields = { + "image": {"image_embeds": "values", "image_grid_thw": "metadata"}, + "video": {"video_embeds": "values", "video_grid_thw": "metadata"}, + } + def __init__(self, spatial_merge_size: int, *args, **kwargs): self._spatial_merge_size = spatial_merge_size super().__init__(*args, **kwargs) @@ -804,10 +810,12 @@ class Qwen2VLMultiModalDataParser(MultiModalDataParser): data: dict[str, torch.Tensor] | ModalityData[ImageItem], ) -> ModalityDataItems[Any, Any] | None: if isinstance(data, dict): + required, optional = self.embedding_field_sets("image") return DictEmbeddingItems( data, modality="image", - required_fields={"image_embeds", "image_grid_thw"}, + required_fields=required, + optional_fields=optional, fields_factory=_create_qwen2vl_field_factory(self._spatial_merge_size), ) @@ -818,10 +826,12 @@ class Qwen2VLMultiModalDataParser(MultiModalDataParser): data: dict[str, torch.Tensor] | ModalityData[VideoItem], ) -> ModalityDataItems[Any, Any] | None: if isinstance(data, dict): + required, optional = self.embedding_field_sets("video") return DictEmbeddingItems( data, modality="video", - required_fields={"video_embeds", "video_grid_thw"}, + required_fields=required, + optional_fields=optional, fields_factory=_create_qwen2vl_field_factory(self._spatial_merge_size), ) @@ -846,6 +856,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo): return Qwen2VLMultiModalDataParser( self.get_hf_config().vision_config.spatial_merge_size, expected_hidden_size=self._get_expected_hidden_size(), + embeds_from_ec_connector=self.embeds_from_ec_connector, ) def get_supported_mm_limits(self) -> Mapping[str, int | None]: diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py index 7af3ba76bc..a9b1af286c 100644 --- a/vllm/model_executor/models/qwen3_vl.py +++ b/vllm/model_executor/models/qwen3_vl.py @@ -879,6 +879,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo): self.get_hf_config().vision_config.spatial_merge_size, video_needs_metadata=True, expected_hidden_size=self._get_expected_hidden_size(), + embeds_from_ec_connector=self.embeds_from_ec_connector, ) def _get_vision_info( diff --git a/vllm/multimodal/inputs.py b/vllm/multimodal/inputs.py index ab04dc71e8..c80a78893d 100644 --- a/vllm/multimodal/inputs.py +++ b/vllm/multimodal/inputs.py @@ -23,7 +23,7 @@ from typing_extensions import TypeVar from vllm.utils.collection_utils import is_list_of from vllm.utils.import_utils import LazyLoader -from vllm.utils.jsontree import json_map_leaves +from vllm.utils.jsontree import json_iter_leaves, json_map_leaves from .media import MediaWithBytes @@ -286,6 +286,13 @@ def nested_tensors_equal( return a == b +def _nested_tensors_are_cpu(tensors: NestedTensors) -> bool: + """Whether every tensor in `tensors` lives in host memory.""" + return not any( + isinstance(x, torch.Tensor) and not x.is_cpu for x in json_iter_leaves(tensors) + ) + + def _nested_tensors_h2d( tensors: NestedTensors, device: torch.types.Device, @@ -478,6 +485,11 @@ class BaseMultiModalField(ABC): pin_memory = False batch = [elem.data for elem in elems] + if pin_memory and not _nested_tensors_are_cpu(batch): + # Data already sits on an accelerator (e.g. a device-side HF + # processor handed it over by IPC). Pinning is a host-memory + # concept, and `pin_memory()` rejects device tensors outright. + pin_memory = False out = self._reduce_data(batch, pin_memory=pin_memory) return _nested_tensors_h2d(out, device=device) diff --git a/vllm/multimodal/parse.py b/vllm/multimodal/parse.py index 729a5bad18..3053670a64 100644 --- a/vllm/multimodal/parse.py +++ b/vllm/multimodal/parse.py @@ -40,8 +40,18 @@ from .media import MediaWithBytes _T = TypeVar("_T") _I = TypeVar("_I") +EmbeddingFieldRole: TypeAlias = Literal["values", "metadata"] +"""What a field of a pre-computed-embedding input carries. + +`"values"` is the embedding tensor itself; `"metadata"` is a key that sizes the +prompt's placeholder range. The distinction is what lets one declaration serve +both an ordinary request (everything present) and an EC consumer (embeddings +arrive through the connector, metadata still in the request). +""" + if TYPE_CHECKING: import PIL.Image as PILImage + else: PILImage = LazyLoader("PILImage", globals(), "PIL.Image") @@ -250,34 +260,62 @@ class DictEmbeddingItems( self, data: Mapping[str, torch.Tensor], modality: str, - required_fields: set[str], + required_fields: Set[str], fields_factory: Callable[ [Mapping[str, torch.Tensor]], Mapping[str, MultiModalFieldConfig], ], + optional_fields: Set[str] = frozenset(), ) -> None: + """ + Args: + data: The dictionary of tensors for this modality. + modality: The modality these items belong to. + required_fields: Fields `data` must contain. + fields_factory: Builds the field config from the data. + optional_fields: Fields `data` may omit. Which fields these are is + the caller's decision -- see + `MultiModalDataParser.embedding_field_sets`, where a deployment + that receives embeddings through an EC connector makes the + embeddings optional. They still need a field config, since they + are used whenever they *are* supplied. + """ from transformers.feature_extraction_utils import BatchFeature super().__init__(data, modality) - missing_required_data_keys = required_fields - data.keys() + # Nothing required would leave nothing to size the placeholder range + # from, so the item would parse into zero entries and silently produce a + # wrong prompt instead of failing here. + if not required_fields: + raise ValueError( + f"Cannot parse {modality!r} embeddings: every declared field is " + f"optional ({sorted(optional_fields)}), so nothing is left to " + "size the placeholder range." + ) + + declared_fields = set(required_fields) | set(optional_fields) + + missing_required_data_keys = set(required_fields) - data.keys() if missing_required_data_keys: data_keys = set(data.keys()) msg = ( - f"The data should contain the fields: {required_fields}, " + f"The data should contain the fields: {set(required_fields)}, " f"but only found the following keys: {data_keys}" ) raise ValueError(msg) fields_config = fields_factory(data) - missing_required_fields = required_fields - fields_config.keys() + # Check every declared field, not just the required ones: an optional + # field still needs a config for when it is supplied. + missing_required_fields = declared_fields - fields_config.keys() if missing_required_fields: fields = set(fields_config.keys()) - msg = f"{required_fields=} should be a subset of {fields=}" + msg = f"{declared_fields=} should be a subset of {fields=}" raise ValueError(msg) self.fields_config = fields_config - self.required_fields = required_fields + self.required_fields = set(required_fields) self._kwargs = MultiModalKwargsItems.from_hf_inputs( BatchFeature(dict(data)), @@ -516,8 +554,50 @@ class MultiModalDataParser: embedding inputs. If provided, validates that user-supplied embeddings have the correct hidden size to prevent crashes during model inference. + embeds_from_ec_connector (bool): Whether pre-computed embeddings may be + absent from the request because an encode/prefill/decode encoder + instance publishes them through an EC connector instead. Derived by + `BaseProcessingInfo.embeds_from_ec_connector`. """ + embedding_fields: Mapping[str, Mapping[str, EmbeddingFieldRole]] = {} + """Per-modality field roles for pre-computed-embedding inputs. + + Declared here rather than inside `_parse_*_data` so an EC producer can read + it too: on a producer the request carries real media, so the branch that + builds `DictEmbeddingItems` never runs, yet the producer still has to know + which processed keys to publish. One declaration, both sides, no drift. + + A modality absent from this mapping can only be sent whole in the request. + """ + + @classmethod + def placeholder_metadata_fields(cls, modality: str) -> set[str]: + """The keys that size `modality`'s placeholder range. + + What an EC producer publishes alongside the embedding, and what a + consumer keeps requiring once the embedding itself is gone. + """ + return { + field + for field, role in cls.embedding_fields.get(modality, {}).items() + if role == "metadata" + } + + def embedding_field_sets(self, modality: str) -> tuple[set[str], set[str]]: + """`modality`'s (required, optional) fields for this deployment. + + Resolves the static roles in `embedding_fields` against where the + embeddings actually come from: on an EC consumer they arrive through the + connector, so the request may omit them; anywhere else a request that + claims to carry pre-computed embeddings has to actually carry them. + """ + metadata = self.placeholder_metadata_fields(modality) + values = set(self.embedding_fields.get(modality, {})) - metadata + if self.embeds_from_ec_connector: + return metadata, values + return metadata | values, set() + def __init__( self, *, @@ -526,9 +606,12 @@ class MultiModalDataParser: audio_resample_method: Literal["pyav", "scipy", "soxr"] = "pyav", video_needs_metadata: bool = False, expected_hidden_size: int | None = None, + embeds_from_ec_connector: bool = False, ) -> None: super().__init__() + self.embeds_from_ec_connector = embeds_from_ec_connector + self.audio_resampler = AudioResampler( target_sr=target_sr, method=audio_resample_method, diff --git a/vllm/multimodal/processing/context.py b/vllm/multimodal/processing/context.py index bc893d836d..a1d4958674 100644 --- a/vllm/multimodal/processing/context.py +++ b/vllm/multimodal/processing/context.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import time -from abc import abstractmethod from collections.abc import Callable, Mapping from contextlib import contextmanager from dataclasses import dataclass, field @@ -227,13 +226,30 @@ class InputProcessingContext: self, output: JSONTree, ) -> JSONTree: - def _postprocess_one(x: object): - if isinstance(x, torch.Tensor): # noqa: SIM102 - # This mimics the behavior of transformers.BatchFeature - if x.is_floating_point(): - x = x.to(dtype=self.model_config.dtype) + # "torch_shm" puts tensors on a torch.multiprocessing queue, which + # shares device tensors by CUDA IPC handle, so a device-side processor + # can hand `pixel_values` straight to the worker. Every other transport + # serializes host bytes, so the result has to be copied back first. + keep_on_device = ( + self.model_config.get_multimodal_config().mm_tensor_ipc == "torch_shm" + ) - return x + def _postprocess_one(x: object): + if not isinstance(x, torch.Tensor): + return x + + # Bind to a Tensor-typed local: reassigning the `object`-typed + # parameter would discard the isinstance narrowing. + tensor = x + + # This mimics the behavior of transformers.BatchFeature + if tensor.is_floating_point(): + tensor = tensor.to(dtype=self.model_config.dtype) + + if not tensor.is_cpu and not keep_on_device: + tensor = tensor.cpu() + + return tensor return json_map_leaves(_postprocess_one, output) @@ -350,6 +366,17 @@ class BaseProcessingInfo: return None + @property + def embeds_from_ec_connector(self) -> bool: + """Whether pre-computed embeddings may arrive outside the request. + + True only on an EC consumer, where an encode/prefill/decode encoder + instance publishes them through the connector instead, so the request + carries only the metadata that sizes the placeholder range. + """ + mm_config = self.ctx.model_config.multimodal_config + return mm_config is not None and mm_config.mm_embeds_from_ec_connector + def get_data_parser(self) -> MultiModalDataParser: """ Constructs a parser to preprocess multi-modal data items @@ -362,6 +389,7 @@ class BaseProcessingInfo: """ return MultiModalDataParser( expected_hidden_size=self._get_expected_hidden_size(), + embeds_from_ec_connector=self.embeds_from_ec_connector, ) @cached_property @@ -372,7 +400,6 @@ class BaseProcessingInfo: def skip_prompt_length_check(self) -> bool: return False - @abstractmethod def get_supported_mm_limits(self) -> Mapping[str, int | None]: """ Return the maximum supported number of items for each modality.