[EPD] Remove duplicate image preprocessing in EPD and enable preprocess on GPU (#50390)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
Tianyu Guo
2026-08-05 15:59:54 -07:00
committed by GitHub
parent 38ebd97bca
commit b30291cf0c
22 changed files with 904 additions and 45 deletions
@@ -21,13 +21,18 @@ from __future__ import annotations
import argparse
import asyncio
import hashlib
import io
import json
import logging
import os
import random
import time
import uuid
from collections.abc import AsyncIterator
import aiohttp
import pybase64 as base64
import uvicorn
from fastapi import FastAPI, HTTPException, Request
from fastapi.responses import JSONResponse, StreamingResponse
@@ -53,6 +58,89 @@ decode_session: aiohttp.ClientSession | None = None
MM_TYPES = {"image_url", "audio_url", "input_audio"}
# Diagnostic switch: forward the original request to the decoder so the
# only difference from the rewrite path is the rewrite itself.
NO_REWRITE = False
# Grid metadata reported by the encoder instance, keyed by item index.
# Empty when the encoder did not report any (then nothing is rewritten).
def content_uuid(item: dict) -> str:
"""Cache key for a multimodal item, derived from its content.
Must be content-derived, not request-derived: the EC cache is keyed by this
value, so a per-request key (a request id, say) would make every request a
miss and throw away cross-request reuse of already-encoded media -- while
the unmodified path, which hashes the content, would keep it. That asymmetry
silently biases any comparison between the two.
"""
url = (item.get("image_url") or item.get("audio_url") or {}).get("url") or ""
payload = url or json.dumps(item, sort_keys=True)
return hashlib.sha256(payload.encode()).hexdigest()
def _b64_tensor(values: list) -> str:
import torch
buf = io.BytesIO()
grid = torch.tensor(values, dtype=torch.long)
# Downstream stacks per item, so hand over a flat (t, h, w).
torch.save(grid.reshape(-1)[:3], buf)
return base64.b64encode(buf.getvalue()).decode()
def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict:
"""Replace each image item with a metadata-only reference for the decoder.
The decoder does not need the pixels: the encoder instance already produced
the embedding and published it through the EC connector under the same uuid.
Sending only the grid lets the decoder size the placeholder range without
re-running the image transform.
`item_meta` holds what the encoder reported for each item (its cache key and
the grid its processor actually produced), so the grid is never re-derived
here -- a second derivation could disagree with the encoder's.
"""
rewritten = 0
idx = 0
new_messages = []
for msg in req_data.get("messages", []):
content = msg.get("content")
if not isinstance(content, list):
new_messages.append(msg)
continue
new_content = []
for item in content:
if item.get("type") not in MM_TYPES:
new_content.append(item)
continue
meta = dict(item_meta.get(idx) or {})
idx += 1
item_uuid = meta.pop("mm_hash", None)
# Whatever keys the encoder reported are the metadata its model
# declared as needed to size the placeholder range; the proxy does
# not need to know their names.
metadata = {k: _b64_tensor(v) for k, v in meta.items()}
if not metadata or not item_uuid:
# The encoder reported no metadata (e.g. the item came from its
# processor cache); let the decoder process the media itself.
new_content.append(item)
continue
new_content.append(
{
"type": "image_embeds",
"image_embeds": metadata,
"uuid": item_uuid,
}
)
rewritten += 1
new_messages.append({**msg, "content": new_content})
if not rewritten:
return req_data
logger.info("Rewrote %d image item(s) as metadata references", rewritten)
return {**req_data, "messages": new_messages}
def extract_mm_items(request_data: dict) -> list[dict]:
"""
@@ -77,22 +165,29 @@ async def fanout_encoder_primer(
orig_request: dict,
e_urls: list[str],
req_id: str,
) -> None:
) -> dict[int, dict]:
"""
1. Build one request *per MM item* with all text removed.
2. Send them concurrently to the encode cluster.
3. Raise if any of them fails.
Returns, per item index, the metadata the encoder reported in
`ec_transfer_params`: its EC cache key and the grid its processor produced.
The proxy still supplies the uuid so both sides key the cache the same way;
the grid can only come from the encoder, which is the side that computed it.
"""
logger.info("[%s] Processing multimodal items...", req_id)
mm_items = extract_mm_items(orig_request)
if not mm_items:
logger.info("[%s] No multimodal items, skipping encoder", req_id)
return # nothing to do
return {} # nothing to do
logger.info("[%s] got %d multimodal items...", req_id, len(mm_items))
tasks = []
item_uuids: dict[int, str] = {}
item_meta: dict[int, dict] = {}
# Round-robin over encode servers to distribute load a bit
url_cycle = (e_urls[i % len(e_urls)] for i in range(len(mm_items)))
@@ -102,11 +197,24 @@ async def fanout_encoder_primer(
child_req_id = f"{req_id}:{idx}:{uuid.uuid4().hex[:6]}"
headers = {"x-request-id": child_req_id}
# With --no-rewrite the decoder still receives the raw image and derives
# the cache key by hashing it, so the encoder must do the same -- passing
# a uuid here would make the two disagree and silently defeat the EC
# transfer, leaving the decoder to encode the image itself.
item_uuid = None if NO_REWRITE else content_uuid(item)
if item_uuid is not None:
item_uuids[idx] = item_uuid
encoder_req = {
# You *may* need to keep additional fields
"model": orig_request.get("model"),
"messages": [
{"role": "user", "content": [item]},
{
"role": "user",
"content": [
item if item_uuid is None else {**item, "uuid": item_uuid}
],
},
],
# No max_tokens cap: the encoder instance never samples, it finishes
# once the prompt is encoded and its embeddings are published.
@@ -152,9 +260,21 @@ async def fanout_encoder_primer(
detail=f"Encoder request failed: {detail}",
)
# The encoder reports each item's cache key and grid here.
try:
params = (await r.json()).get("ec_transfer_params") or {}
reported = params.get("ec_items") or []
except Exception:
logger.warning("[%s] Could not read encoder metadata #%d", req_id, idx)
reported = []
if reported and idx in item_uuids:
# One item per encoder request, so the first entry is this item's.
item_meta[idx] = {**reported[0], "mm_hash": item_uuids[idx]}
logger.info(
"[%s] All %d encoder requests completed successfully", req_id, len(mm_items)
)
return item_meta
async def maybe_prefill(
@@ -320,7 +440,11 @@ async def forward_non_stream(
) -> dict:
try:
# Step 1: Process through Encoder instance (if has MM input)
await fanout_encoder_primer(req_data, e_urls, req_id)
_t0 = time.perf_counter()
item_meta = await fanout_encoder_primer(req_data, e_urls, req_id)
_t1 = time.perf_counter()
req_data = req_data if NO_REWRITE else rewrite_for_decode(req_data, item_meta)
_t2 = time.perf_counter()
# Step 2: Process through Prefill instance
req_data = await maybe_prefill(req_data, p_url, req_id)
@@ -334,7 +458,17 @@ async def forward_non_stream(
f"{d_url}/v1/chat/completions", json=req_data, headers=headers
) as resp:
resp.raise_for_status()
return await resp.json()
out = await resp.json()
_t3 = time.perf_counter()
logger.info(
"STAGE %s encode=%.1f rewrite=%.1f decode=%.1f total=%.1f",
"no-rewrite" if NO_REWRITE else "rewrite",
(_t1 - _t0) * 1e3,
(_t2 - _t1) * 1e3,
(_t3 - _t2) * 1e3,
(_t3 - _t0) * 1e3,
)
return out
except HTTPException:
raise
@@ -348,7 +482,11 @@ async def forward_stream(
) -> AsyncIterator[str]:
try:
# Step 1: Process through Encoder instance (if has MM input)
await fanout_encoder_primer(req_data, e_urls, req_id)
_t0 = time.perf_counter()
item_meta = await fanout_encoder_primer(req_data, e_urls, req_id)
_t1 = time.perf_counter()
req_data = req_data if NO_REWRITE else rewrite_for_decode(req_data, item_meta)
_t2 = time.perf_counter()
# Step 2: Process through Prefill instance
req_data = await maybe_prefill(req_data, p_url, req_id)
@@ -358,6 +496,7 @@ async def forward_stream(
headers = {"x-request-id": req_id}
# Streaming response
_first = None
async with decode_session.post(
f"{d_url}/v1/chat/completions",
json=req_data,
@@ -366,8 +505,19 @@ async def forward_stream(
resp.raise_for_status()
async for chunk in resp.content.iter_chunked(1024):
if chunk:
if _first is None:
_first = time.perf_counter()
yield chunk.decode("utf-8", errors="ignore")
_t3 = time.perf_counter()
logger.info(
"STAGE %s encode=%.1f rewrite=%.2f decode_ttfb=%.1f decode_total=%.1f",
"no-rewrite" if NO_REWRITE else "rewrite",
(_t1 - _t0) * 1e3,
(_t2 - _t1) * 1e3,
((_first or _t3) - _t2) * 1e3,
(_t3 - _t2) * 1e3,
)
logger.info("[%s] Streaming completed", req_id)
except HTTPException:
@@ -553,6 +703,11 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--host", default="0.0.0.0")
parser.add_argument("--port", type=int, default=8000)
parser.add_argument(
"--no-rewrite",
action="store_true",
help="Forward images to the decoder unchanged (for stage-timing A/B).",
)
parser.add_argument(
"--encode-servers-urls",
required=True,
@@ -573,6 +728,7 @@ if __name__ == "__main__":
)
args = parser.parse_args()
NO_REWRITE = args.no_rewrite
app.state.e_urls = [
u.strip() for u in args.encode_servers_urls.split(",") if u.strip()
]
+174
View File
@@ -4,10 +4,13 @@
from unittest.mock import MagicMock, patch
import pytest
import torch
from transformers import PretrainedConfig
from vllm.config.ec_transfer import ECRole, ECTransferConfig
from vllm.config.model import ModelConfig
from vllm.config.multimodal import MultiModalConfig
from vllm.config.vllm import VllmConfig
from vllm.transformers_utils.model_arch_config_convertor import (
ModelArchConfigConvertorBase,
)
@@ -173,3 +176,174 @@ def test_sticky_cache_survives_text_subconfig_regeneration():
side_effect=AssertionError("must not re-query registry"),
):
assert text_config._supports_multimodal_for_mm_prefix() is False
@pytest.mark.parametrize(
("device", "expected"),
[
(None, None),
("cpu", "cpu"),
("cuda", "cuda"),
# Callers compare against a bare device type, so an indexed device and a
# torch.device must normalise to the same thing as "cuda".
("cuda:1", "cuda"),
(torch.device("cuda", 1), "cuda"),
],
)
def test_mm_processor_device_type_normalizes(device: object, expected: str | None):
kwargs = {} if device is None else {"device": device}
config = MultiModalConfig(mm_processor_kwargs=kwargs)
assert config.get_mm_processor_device_type() == expected
def _validate_mm_processor_device(*, device: str, ec_role: ECRole | None) -> None:
ec_config = (
None
if ec_role is None
# `is_ec_producer`/`is_ec_consumer` are False without a connector, so an
# unset one would make every role look like "no EC role at all".
else ECTransferConfig(ec_connector="ECExampleConnector", ec_role=ec_role)
)
MultiModalConfig(
mm_processor_kwargs={"device": device}
).validate_mm_processor_device(ec_config)
@pytest.mark.parametrize("ec_role", [None, "ec_producer"])
def test_bad_mm_processor_device_rejected(ec_role: ECRole | None):
"""A bad device must fail during startup, not mid-request.
Rejected for every role, and on a CPU-only platform too, so a typo can never
silently fall through to running the processor somewhere unintended.
"""
with (
patch("vllm.platforms.current_platform.device_type", "cpu"),
pytest.raises(ValueError, match='Invalid "device" in mm_processor_kwargs'),
):
_validate_mm_processor_device(device="not-a-device", ec_role=ec_role)
@pytest.mark.parametrize("ec_role", [None, "ec_consumer", "ec_both"])
def test_accelerator_mm_processor_rejected_outside_encoder_instance(
ec_role: ECRole | None,
):
"""Only an encode-only EPD instance has the device to itself.
Every other role runs the language model in the same process, so frontend
accelerator work would contend with the forward pass and allocate outside
the memory profiled for the KV cache.
"""
with (
patch("vllm.platforms.current_platform.device_type", "cuda"),
pytest.raises(ValueError, match="also runs the language model"),
):
_validate_mm_processor_device(device="cuda", ec_role=ec_role)
def test_accelerator_mm_processor_allowed_on_encoder_instance():
with patch("vllm.platforms.current_platform.device_type", "cuda"):
_validate_mm_processor_device(device="cuda", ec_role="ec_producer")
def test_cpu_mm_processor_needs_no_ec_role():
"""The gate only applies to the accelerator, so CPU stays unrestricted."""
with patch("vllm.platforms.current_platform.device_type", "cuda"):
_validate_mm_processor_device(device="cpu", ec_role=None)
@pytest.mark.parametrize(
("flag", "kwargs", "expected"),
[
# "auto" is only settled once the EC role is known, so it must leave no
# device behind for `VllmConfig` to mistake for an explicit request.
("auto", None, None),
(None, None, None),
("cpu", None, "cpu"),
("cuda", None, "cuda"),
# Any explicit value other than "cpu" means "whatever this platform
# calls its accelerator".
("gpu", None, "cuda"),
# An explicit device in the kwargs wins over the convenience flag.
("cpu", {"device": "cuda"}, "cuda"),
("cuda", {"device": "cpu"}, "cpu"),
],
)
def test_fold_mm_processor_device(
flag: str | None, kwargs: dict | None, expected: str | None
):
with patch("vllm.platforms.current_platform.device_type", "cuda"):
folded = MultiModalConfig.fold_mm_processor_device(kwargs, flag)
config = MultiModalConfig(mm_processor_kwargs=folded or {})
assert config.get_mm_processor_device_type() == expected
def _resolve_mm_processor_device(
*,
ec_role: ECRole | None,
mm_tensor_ipc: str = "torch_shm",
device: str | None = None,
) -> str | None:
"""Run the `auto` resolution and report where the processor ended up."""
mm_config = MultiModalConfig(
mm_processor_kwargs={} if device is None else {"device": device},
mm_tensor_ipc=mm_tensor_ipc, # type: ignore[arg-type]
)
model_config = MagicMock(spec=ModelConfig)
model_config.multimodal_config = mm_config
vllm_config = MagicMock(spec=VllmConfig)
vllm_config.model_config = model_config
vllm_config.ec_transfer_config = (
None
if ec_role is None
else ECTransferConfig(ec_connector="ECExampleConnector", ec_role=ec_role)
)
with patch("vllm.platforms.current_platform.device_type", "cuda"):
VllmConfig._resolve_mm_processor_device(vllm_config)
return mm_config.get_mm_processor_device_type()
def test_auto_mm_processor_device_uses_accelerator_on_encoder_instance():
"""The one deployment that has the device to itself and can hand it over."""
assert _resolve_mm_processor_device(ec_role="ec_producer") == "cuda"
@pytest.mark.parametrize("ec_role", [None, "ec_consumer", "ec_both"])
def test_auto_mm_processor_device_stays_on_cpu_off_encoder_instance(
ec_role: ECRole | None,
):
"""Every other role runs the language model in the same process."""
assert _resolve_mm_processor_device(ec_role=ec_role) is None
def test_auto_mm_processor_device_needs_a_device_capable_transport():
"""Other transports serialize host bytes, so the output is copied back."""
assert (
_resolve_mm_processor_device(ec_role="ec_producer", mm_tensor_ipc="direct_rpc")
is None
)
def test_auto_mm_processor_device_leaves_an_explicit_request_alone():
assert _resolve_mm_processor_device(ec_role="ec_producer", device="cpu") == "cpu"
def test_vllm_config_runs_the_mm_processor_device_check():
"""Startup must reach the check; the rule itself is covered above.
Guards the wiring, which no other test would notice going missing.
"""
model_config = MagicMock(spec=ModelConfig)
model_config.multimodal_config = MultiModalConfig(
mm_processor_kwargs={"device": "cuda"}
)
vllm_config = MagicMock(spec=VllmConfig)
vllm_config.model_config = model_config
vllm_config.ec_transfer_config = None
with (
patch("vllm.platforms.current_platform.device_type", "cuda"),
pytest.raises(ValueError, match="also runs the language model"),
):
VllmConfig._validate_mm_processor_device(vllm_config)
+10
View File
@@ -103,5 +103,15 @@ class ECTransferConfig:
def is_ec_consumer(self) -> bool:
return self.ec_connector is not None and self.ec_role in get_args(ECConsumer)
@property
def is_encode_only(self) -> bool:
"""Whether this instance encodes but does not run the language model.
It allocates no KV cache either -- `GPUModelRunner.get_kv_cache_spec`
returns {} for it -- so it is the one role that can spend accelerator
time and memory on frontend work.
"""
return self.is_ec_producer and not self.is_ec_consumer
def get_from_extra_config(self, key, default) -> Any:
return self.ec_connector_extra_config.get(key, default)
+7
View File
@@ -18,6 +18,7 @@ from vllm.config.multimodal import (
MMCacheType,
MMEncoderTPMode,
MMHasherAlgorithm,
MMProcessorDevice,
MMTensorIPC,
MultiModalConfig,
)
@@ -395,6 +396,7 @@ class ModelConfig:
video_pruning_method: InitVar[str | None] = None
mm_tensor_ipc: InitVar[MMTensorIPC] = None
mm_ipc_gpu_memory_gb: InitVar[float | None] = None
mm_processor_device: InitVar[MMProcessorDevice | None] = None
def compute_hash(self) -> str:
"""
@@ -525,6 +527,7 @@ class ModelConfig:
video_pruning_method: str | None,
mm_tensor_ipc: MMTensorIPC,
mm_ipc_gpu_memory_gb: float | None,
mm_processor_device: MMProcessorDevice | None,
) -> None:
# Keep set served_model_name before maybe_model_redirect(self.model)
self.served_model_name = get_served_model_name(
@@ -763,6 +766,10 @@ class ModelConfig:
)
mm_encoder_tp_mode = "weights"
mm_processor_kwargs = MultiModalConfig.fold_mm_processor_device(
mm_processor_kwargs, mm_processor_device
)
mm_config_kwargs = dict(
language_model_only=language_model_only,
limit_per_prompt=limit_mm_per_prompt,
+136
View File
@@ -5,14 +5,19 @@ from collections.abc import Mapping
from pathlib import Path
from typing import Any, Literal, TypeAlias, TypedDict, cast, final
import torch
from pydantic import ConfigDict, Field, field_validator, model_validator
from pydantic.dataclasses import dataclass
import vllm.envs as envs
from vllm.config.ec_transfer import ECTransferConfig
from vllm.config.utils import config, get_from_deprecated_env_if_set
from vllm.logger import init_logger
from vllm.utils.hashing import safe_hash
from vllm.v1.attention.backends.registry import AttentionBackendEnum
logger = init_logger(__name__)
@dataclass
class BaseDummyOptions:
@@ -64,6 +69,10 @@ MMCacheType = Literal["shm", "lru"]
VideoPruningMethod = Literal["evs", "vidcom2"]
MMTensorIPC = Literal["direct_rpc", "torch_shm"]
MMHasherAlgorithm = Literal["blake3", "sha256", "sha512"]
MMProcessorDevice: TypeAlias = str
"""`"auto"`, `"cpu"`, or the platform's own accelerator name
(`current_platform.device_type`, e.g. `"cuda"` on CUDA and ROCm,
`"xpu"` on XPU). Validated against that set by the CLI."""
def _get_mm_hasher_algorithm() -> MMHasherAlgorithm:
@@ -222,6 +231,20 @@ class MultiModalConfig:
- "direct_rpc": Use msgspec serialization via RPC
- "torch_shm": Use torch.multiprocessing shared memory for zero-copy IPC
Defaults to "direct_rpc". """
mm_embeds_from_ec_connector: bool = False
"""Whether a pre-computed-embedding input may omit the `*_embeds` tensor.
In an encode/prefill/decode (EPD) deployment the encoder instance publishes
embeddings through the EC connector, so the request that reaches the
prefill/decode instance only needs to carry the grid/size metadata that
sizes the placeholder range — the embeddings themselves come from the
connector, keyed by `mm_hash`.
Derived, not user-settable: `VllmConfig.__post_init__` sets this to True
exactly on EC consumers. Everywhere else it stays False so that a request
which forgets its embeddings still fails fast in the frontend, with a clear
error, rather than deep inside the model."""
mm_ipc_gpu_memory_gb: float = Field(default=0, ge=0)
"""Amount of GPU memory (in GiB) sequestered on the engine's device for
GPU-side multimodal work in the API-server (frontend) process, such as
@@ -319,6 +342,119 @@ class MultiModalConfig:
)
return self
@staticmethod
def fold_mm_processor_device(
mm_processor_kwargs: dict[str, Any] | None,
mm_processor_device: MMProcessorDevice | None,
) -> dict[str, Any] | None:
"""Fold the `mm_processor_device` convenience flag into the kwargs.
The flag keeps no state of its own: `mm_processor_kwargs["device"]` is
the only representation of where the processor runs, so an explicit
`device` there always wins and `"auto"` stays unresolved for
`VllmConfig`, which is where the EC role needed to resolve it lives.
Args:
mm_processor_kwargs: The kwargs as given, or None.
mm_processor_device: The flag's value, or None when unset.
Returns:
The kwargs to build the config with, unchanged unless the flag adds
a `device`.
"""
if mm_processor_device in (None, "auto"):
return mm_processor_kwargs
if (mm_processor_kwargs or {}).get("device") is not None:
return mm_processor_kwargs
from vllm.platforms import current_platform
# Any explicit value other than "cpu" means "the accelerator", so a
# programmatically-set "cuda" still works on a platform whose device type
# is named differently ("xpu"), and degrades to CPU where there is none.
device = (
"cpu"
if mm_processor_device == "cpu"
else (current_platform.device_type or "cpu")
)
return {**(mm_processor_kwargs or {}), "device": device}
def get_mm_processor_device_type(self) -> str | None:
"""The torch device type `mm_processor_kwargs["device"]` names.
`mm_processor_kwargs` is untyped, so `device` may be any form torch
accepts -- `"cuda"`, `"cuda:1"`, `torch.device(...)`, or a bare index.
Normalising through torch rather than parsing the string keeps the
non-string forms from slipping past a caller's comparison.
Returns:
The device type, or None when no device is requested.
Raises:
ValueError: If `device` is not something `torch.device` accepts.
`validate_mm_processor_device` is what surfaces this during
startup, so the value is only parsed once.
"""
device = (self.mm_processor_kwargs or {}).get("device")
if device is None:
return None
try:
return torch.device(device).type # type: ignore[arg-type]
except (RuntimeError, TypeError, ValueError):
raise ValueError(
f'Invalid "device" in mm_processor_kwargs: {device!r}. Expected a '
'torch device such as "cpu", "cuda" or "cuda:0".'
) from None
def validate_mm_processor_device(self, ec_config: ECTransferConfig | None) -> None:
"""Check `mm_processor_kwargs["device"]` for this deployment.
The only place the requested device is validated, so it runs even on a
CPU-only platform: the value is parsed before any early return.
Args:
ec_config: The deployment's EC config, or None when it is not an
encode/prefill/decode deployment. Passed in because it is not
reachable from here, and because a field assigned after
construction would not re-trigger this config's validators.
Raises:
ValueError: If the requested device is not a torch device, or if it
is the accelerator on an instance that also runs the language
model.
"""
from vllm.platforms import current_platform
device_type = self.get_mm_processor_device_type()
accelerator = current_platform.device_type
if device_type is None or accelerator in ("", "cpu"):
return
if device_type != accelerator:
return
if ec_config is None or not ec_config.is_encode_only:
raise ValueError(
f"Cannot run the multi-modal processor on {device_type!r}: this "
"instance also runs the language model. The processor would "
"share the device with the model's forward pass, so its "
"transform kernels contend with that compute, and because it "
"runs in the API-server process its allocations are outside the "
"memory the engine profiled for its KV cache -- risking OOM or "
"a silently shrunken cache.\n"
"Accelerator preprocessing is only supported on an encode-only "
"instance of an encode/prefill/decode deployment (an EC "
"producer that is not also a consumer), which runs no forward "
"pass and allocates no KV cache.\n"
'Use --mm-processor-device=cpu, or drop "device" from '
"--mm-processor-kwargs."
)
logger.info_once(
"Running the multi-modal processor on %s. Override with "
"--mm-processor-device=cpu.",
device_type,
)
def compute_hash(self) -> str:
"""
WARNING: Whenever a new field is added to this config,
+99
View File
@@ -1509,6 +1509,10 @@ class VllmConfig:
)
current_platform.check_and_update_config(self)
self._resolve_mm_embeds_from_ec_connector()
self._resolve_mm_processor_device()
self._validate_mm_processor_device()
if self.use_v2_model_runner:
self._validate_v2_model_runner()
elif self.parallel_config.prefill_context_parallel_size > 1:
@@ -2185,6 +2189,101 @@ class VllmConfig:
f"kernel_config={self.kernel_config!r}"
)
def _resolve_mm_embeds_from_ec_connector(self) -> None:
"""Allow `*_embeds` to be omitted only where the connector supplies them.
That is exactly an EC consumer: the encoder instance publishes the
embeddings through the EC connector, so the request only has to carry
the grid metadata that sizes the placeholder range. On every other
deployment a missing `*_embeds` is a client error and must keep failing
fast in the frontend.
"""
model_config = self.model_config
if model_config is None:
return
mm_config = model_config.multimodal_config
if mm_config is None:
return
ec_config = self.ec_transfer_config
# Derived, so overwrite unconditionally rather than honouring a value
# that was set by hand.
mm_config.mm_embeds_from_ec_connector = (
ec_config is not None and ec_config.is_ec_consumer
)
if mm_config.mm_embeds_from_ec_connector:
logger.info_once(
"EC consumer: pre-computed-embedding inputs may omit the "
"embedding tensor; embeddings are loaded from the EC connector."
)
def _resolve_mm_processor_device(self) -> None:
"""Settle `--mm-processor-device=auto` now that the EC role is known.
"auto" means "the accelerator, but only where the processor has it to
itself and its output can be handed over without a copy back to host":
an encode-only instance whose tensor transport carries device tensors.
Every other deployment keeps the processor on CPU.
An explicit device -- from `--mm-processor-device` or straight from
`mm_processor_kwargs` -- is already folded in by `MultiModalConfig`, so
it is left alone here and validated by `_validate_mm_processor_device`.
"""
model_config = self.model_config
if model_config is None:
return
mm_config = model_config.multimodal_config
if mm_config is None:
return
if mm_config.get_mm_processor_device_type() is not None:
return
from vllm.platforms import current_platform
device_type = current_platform.device_type
if device_type in ("", "cpu"):
return
ec_config = self.ec_transfer_config
# An EC producer that is not also a consumer runs no forward pass and
# allocates no KV cache, so frontend accelerator work has the device to
# itself.
if ec_config is None or not ec_config.is_encode_only:
return
if mm_config.mm_tensor_ipc != "torch_shm":
# Any other transport serializes host bytes, so the output would be
# copied back, and that copy costs more than running the transform
# on device saves.
logger.info_once(
"EPD encoder instance: keeping the multi-modal processor on CPU "
"because mm_tensor_ipc=%s cannot carry device tensors. Add "
"--mm-tensor-ipc=torch_shm to run it on the accelerator.",
mm_config.mm_tensor_ipc,
)
return
mm_config.mm_processor_kwargs = {
**(mm_config.mm_processor_kwargs or {}),
"device": device_type,
}
logger.info_once(
"EPD encoder instance: running the multi-modal processor on %s. "
"Override with --mm-processor-device=cpu.",
device_type,
)
def _validate_mm_processor_device(self) -> None:
"""Hand the EC config to `MultiModalConfig`, which owns the rule."""
model_config = self.model_config
if model_config is None:
return
mm_config = model_config.multimodal_config
if mm_config is None:
return
mm_config.validate_mm_processor_device(self.ec_transfer_config)
def _get_v2_model_runner_unsupported_features(self) -> list[str]:
"""Collect features not yet supported by the V2 model runner."""
unsupported: list[str] = []
@@ -2,9 +2,10 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from dataclasses import dataclass
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
import safetensors
import torch
from vllm.config import VllmConfig
from vllm.distributed.ec_transfer.ec_connector.base import (
@@ -50,6 +51,8 @@ class ECExampleConnector(ECConnectorBase):
super().__init__(vllm_config=vllm_config, role=role)
# req_id -> index
self._mm_datas_need_loads: dict[str, int] = {}
self._model_config = vllm_config.model_config
self._metadata_fields_cache: dict[str, set[str]] = {}
transfer_config = vllm_config.ec_transfer_config
if transfer_config is not None:
self._storage_path = transfer_config.get_from_extra_config(
@@ -165,6 +168,70 @@ class ECExampleConnector(ECConnectorBase):
self._mm_datas_need_loads.clear()
return meta
def _placeholder_metadata_fields(self, modality: str) -> set[str]:
"""Which processed keys this model needs published for `modality`.
Read from `MultiModalDataParser.embedding_fields`, the same declaration
the consumer's parser requires, so the two cannot drift. An empty set
means the modality cannot be delivered out of band, and the consumer
will process the media itself.
"""
if modality in self._metadata_fields_cache:
return self._metadata_fields_cache[modality]
fields: set[str] = set()
try:
from vllm.multimodal import MULTIMODAL_REGISTRY
info = MULTIMODAL_REGISTRY.create_processor(self._model_config).info
fields = info.data_parser.placeholder_metadata_fields(modality)
except Exception:
# Reporting nothing is a safe degradation: the consumer falls back to
# processing the media itself.
logger.warning(
"Could not determine the placeholder metadata fields for "
"modality %s; the consumer will preprocess the media itself.",
modality,
exc_info=True,
)
self._metadata_fields_cache[modality] = fields
return fields
def request_finished(
self,
request: "Request",
) -> tuple[bool, dict[str, Any] | None]:
"""Report each item's cache key and grid so a consumer can skip the
image transform.
A consumer only needs the grid to size the prompt's placeholder range;
the embedding itself arrives through this connector. Reporting the grid
the producer actually computed keeps the two sides in agreement without
the caller re-deriving it from the raw media.
"""
if not self.is_producer:
return False, None
items = []
for feature in request.mm_features:
metadata = {}
# `data` is None for items served from the processor cache, in which
# case the metadata is unavailable here and the consumer has to fall
# back to processing the media itself.
if feature.data is not None:
wanted = self._placeholder_metadata_fields(feature.modality)
metadata = {
key: value.tolist()
for key, value in feature.data.get_data().items()
if key in wanted and isinstance(value, torch.Tensor)
}
items.append({"mm_hash": feature.identifier, **metadata})
if not items:
return False, None
return False, {"ec_items": items}
# ==============================
# Helper functions
# ==============================
+27
View File
@@ -90,6 +90,7 @@ from vllm.config.multimodal import (
MMCacheType,
MMEncoderTPMode,
MMHasherAlgorithm,
MMProcessorDevice,
MMTensorIPC,
)
from vllm.config.observability import DetailedTraceModules
@@ -596,6 +597,7 @@ class EngineArgs:
video_pruning_rate: float | None = MultiModalConfig.video_pruning_rate
video_pruning_method: str = MultiModalConfig.video_pruning_method
mm_tensor_ipc: MMTensorIPC = MultiModalConfig.mm_tensor_ipc
mm_processor_device: MMProcessorDevice = "auto"
mm_ipc_gpu_memory_gb: float = MultiModalConfig.mm_ipc_gpu_memory_gb
# LoRA fields
enable_lora: bool = False
@@ -1353,6 +1355,30 @@ class EngineArgs:
multimodal_group.add_argument(
"--mm-tensor-ipc", **multimodal_kwargs["mm_tensor_ipc"]
)
multimodal_group.add_argument(
"--mm-processor-device",
choices=["auto", "cpu"]
+ (
[current_platform.device_type]
if current_platform.device_type not in ("", "cpu")
else []
),
default="auto",
help="Device the HF multi-modal processor runs the image/video "
"transform on. Convenience for `--mm-processor-kwargs "
"'{\"device\": ...}'`: the value is resolved here and stored there, "
"it is not kept as separate state. Only takes effect for HF "
'"fast" (torchvision-backed) processors, which accept a `device` '
"argument; the others ignore it and stay on CPU.\n\n"
'"auto" uses the accelerator on encoder instances of an '
"encode/prefill/decode deployment -- an EC producer that is not "
"also a consumer allocates no KV cache, so its accelerator is not "
"contended by the language model -- and then only when "
"`--mm-tensor-ipc=torch_shm` can carry device tensors, since every "
"other transport would copy the result back to the host and that "
'copy costs more than it saves. "auto" resolves to "cpu" '
"everywhere else.",
)
multimodal_group.add_argument(
"--mm-ipc-gpu-memory-gb",
**multimodal_kwargs["mm_ipc_gpu_memory_gb"],
@@ -1743,6 +1769,7 @@ class EngineArgs:
video_pruning_method=self.video_pruning_method,
mm_tensor_ipc=self.mm_tensor_ipc,
mm_ipc_gpu_memory_gb=self.mm_ipc_gpu_memory_gb,
mm_processor_device=self.mm_processor_device,
io_processor_plugin=self.io_processor_plugin,
renderer_num_workers=self.renderer_num_workers,
)
+1
View File
@@ -104,6 +104,7 @@ class ColQwen3ProcessingInfo(Qwen3VLProcessingInfo):
spatial_merge_size,
video_needs_metadata=self._supports_video,
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
+1
View File
@@ -101,6 +101,7 @@ class ColQwen3_5ProcessingInfo(Qwen3_5ProcessingInfo):
spatial_merge_size,
video_needs_metadata=self._supports_video,
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
+9 -1
View File
@@ -551,15 +551,22 @@ def _hunyuan_vl_field_config(hf_inputs: Mapping[str, torch.Tensor]):
class HunYuanVLMultiModalDataParser(MultiModalDataParser):
# The patch grid is what sizes the placeholder range.
embedding_fields = {
"image": {"image_embeds": "values", "image_grid_thw": "metadata"},
}
def _parse_image_data(
self,
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("image")
return DictEmbeddingItems(
data,
modality="image",
required_fields={"image_embeds", "image_grid_thw"},
required_fields=required,
optional_fields=optional,
fields_factory=_hunyuan_vl_field_config,
)
@@ -592,6 +599,7 @@ class HunYuanVLProcessingInfo(BaseProcessingInfo):
def get_data_parser(self):
return HunYuanVLMultiModalDataParser(
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
+13 -8
View File
@@ -881,18 +881,23 @@ def _keye_field_config(
class KeyeMultiModalDataParser(MultiModalDataParser):
# The patch grid is what sizes the placeholder range.
embedding_fields = {
"image": {"image_embeds": "values", "image_grid_thw": "metadata"},
"video": {"video_embeds": "values", "video_grid_thw": "metadata"},
}
def _parse_image_data(
self,
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("image")
return DictEmbeddingItems(
data,
modality="image",
required_fields={
"image_embeds",
"image_grid_thw",
},
required_fields=required,
optional_fields=optional,
fields_factory=_keye_field_config,
)
@@ -903,13 +908,12 @@ class KeyeMultiModalDataParser(MultiModalDataParser):
data: dict[str, torch.Tensor] | ModalityData[VideoItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("video")
return DictEmbeddingItems(
data,
modality="video",
required_fields={
"video_embeds",
"video_grid_thw",
},
required_fields=required,
optional_fields=optional,
fields_factory=_keye_field_config,
)
@@ -929,6 +933,7 @@ class KeyeProcessingInfo(BaseProcessingInfo):
def get_data_parser(self):
return KeyeMultiModalDataParser(
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
def get_supported_mm_limits(
+13 -8
View File
@@ -320,18 +320,23 @@ def _keye_field_config(
class KeyeVL1_5MultiModalDataParser(MultiModalDataParser):
# The patch grid is what sizes the placeholder range.
embedding_fields = {
"image": {"image_embeds": "values", "image_grid_thw": "metadata"},
"video": {"video_embeds": "values", "video_grid_thw": "metadata"},
}
def _parse_image_data(
self,
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("image")
return DictEmbeddingItems(
data,
modality="image",
required_fields={
"image_embeds",
"image_grid_thw",
},
required_fields=required,
optional_fields=optional,
fields_factory=_keye_field_config,
)
@@ -342,13 +347,12 @@ class KeyeVL1_5MultiModalDataParser(MultiModalDataParser):
data: dict[str, torch.Tensor] | ModalityData[VideoItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("video")
return DictEmbeddingItems(
data,
modality="video",
required_fields={
"video_embeds",
"video_grid_thw",
},
required_fields=required,
optional_fields=optional,
fields_factory=_keye_field_config,
)
@@ -359,6 +363,7 @@ class KeyeVL1_5ProcessingInfo(KeyeProcessingInfo):
def get_data_parser(self):
return KeyeVL1_5MultiModalDataParser(
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
def get_max_frame_per_video(self) -> int:
@@ -1279,6 +1279,7 @@ class LlavaOnevision2ProcessingInfo(BaseProcessingInfo):
return LlavaOnevision2MultiModalDataParser(
self.get_hf_config().vision_config.spatial_merge_size,
video_needs_metadata=True,
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
def get_hf_processor(self, **kwargs: object):
@@ -1510,6 +1511,11 @@ class LlavaOnevision2DummyInputsBuilder(
class LlavaOnevision2MultiModalDataParser(MultiModalDataParser):
# The patch grid is what sizes the placeholder range.
embedding_fields = {
"image": {"image_embeds": "values", "image_grid_thw": "metadata"},
}
def __init__(self, spatial_merge_size: int, *args, **kwargs):
self._spatial_merge_size = spatial_merge_size
super().__init__(*args, **kwargs)
@@ -1519,10 +1525,12 @@ class LlavaOnevision2MultiModalDataParser(MultiModalDataParser):
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("image")
return DictEmbeddingItems(
data,
modality="image",
required_fields={"image_embeds", "image_grid_thw"},
required_fields=required,
optional_fields=optional,
fields_factory=_create_field_factory(self._spatial_merge_size),
)
return super()._parse_image_data(data)
+22 -3
View File
@@ -26,7 +26,7 @@
import math
from collections import defaultdict
from collections.abc import Callable, Iterable, Mapping, Sequence
from collections.abc import Callable, Iterable, Mapping, Sequence, Set
from functools import partial
from itertools import chain
from typing import Annotated, Any, Literal, TypeAlias
@@ -474,11 +474,14 @@ class MiniCPMVImageEmbeddingItems(DictEmbeddingItems):
[Mapping[str, torch.Tensor]],
Mapping[str, MultiModalFieldConfig],
],
required_fields: Set[str],
optional_fields: Set[str] = frozenset(),
) -> None:
super().__init__(
data,
modality="image",
required_fields={"image_embeds", "image_sizes"},
required_fields=required_fields,
optional_fields=optional_fields,
fields_factory=fields_factory,
)
@@ -495,11 +498,14 @@ class MiniCPMVVideoEmbeddingItems(DictEmbeddingItems):
[Mapping[str, torch.Tensor]],
Mapping[str, MultiModalFieldConfig],
],
required_fields: Set[str],
optional_fields: Set[str] = frozenset(),
) -> None:
super().__init__(
data,
modality="video",
required_fields={"video_embeds", "video_image_sizes"},
required_fields=required_fields,
optional_fields=optional_fields,
fields_factory=fields_factory,
)
@@ -512,14 +518,23 @@ class MiniCPMVVideoEmbeddingItems(DictEmbeddingItems):
class MiniCPMVMultiModalDataParser(MultiModalDataParser):
# The per-item sizes are what size the placeholder range.
embedding_fields = {
"image": {"image_embeds": "values", "image_sizes": "metadata"},
"video": {"video_embeds": "values", "video_image_sizes": "metadata"},
}
def _parse_image_data(
self,
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("image")
return MiniCPMVImageEmbeddingItems(
data,
fields_factory=_minicpmv_field_config,
required_fields=required,
optional_fields=optional,
)
return super()._parse_image_data(data)
@@ -529,9 +544,12 @@ class MiniCPMVMultiModalDataParser(MultiModalDataParser):
data: dict[str, torch.Tensor] | ModalityData[VideoItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("video")
return MiniCPMVVideoEmbeddingItems(
data,
fields_factory=_minicpmv_field_config,
required_fields=required,
optional_fields=optional,
)
return super()._parse_video_data(data)
@@ -573,6 +591,7 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
def get_data_parser(self):
return MiniCPMVMultiModalDataParser(
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
def get_model_version(self):
+1
View File
@@ -57,6 +57,7 @@ class OpenCUAProcessingInfo(Qwen2VLProcessingInfo):
return Qwen2VLMultiModalDataParser(
self.get_hf_config().vision_config.spatial_merge_size,
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
def get_hf_config(self):
@@ -372,6 +372,7 @@ class Qwen2_5OmniThinkerProcessingInfo(
target_sr=feature_extractor.sampling_rate,
target_channels=self.get_target_channels(),
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
def get_target_channels(self) -> int:
+13 -2
View File
@@ -795,6 +795,12 @@ def _create_qwen2vl_field_factory(
class Qwen2VLMultiModalDataParser(MultiModalDataParser):
# The patch grid is what sizes the placeholder range.
embedding_fields = {
"image": {"image_embeds": "values", "image_grid_thw": "metadata"},
"video": {"video_embeds": "values", "video_grid_thw": "metadata"},
}
def __init__(self, spatial_merge_size: int, *args, **kwargs):
self._spatial_merge_size = spatial_merge_size
super().__init__(*args, **kwargs)
@@ -804,10 +810,12 @@ class Qwen2VLMultiModalDataParser(MultiModalDataParser):
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("image")
return DictEmbeddingItems(
data,
modality="image",
required_fields={"image_embeds", "image_grid_thw"},
required_fields=required,
optional_fields=optional,
fields_factory=_create_qwen2vl_field_factory(self._spatial_merge_size),
)
@@ -818,10 +826,12 @@ class Qwen2VLMultiModalDataParser(MultiModalDataParser):
data: dict[str, torch.Tensor] | ModalityData[VideoItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("video")
return DictEmbeddingItems(
data,
modality="video",
required_fields={"video_embeds", "video_grid_thw"},
required_fields=required,
optional_fields=optional,
fields_factory=_create_qwen2vl_field_factory(self._spatial_merge_size),
)
@@ -846,6 +856,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
return Qwen2VLMultiModalDataParser(
self.get_hf_config().vision_config.spatial_merge_size,
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
+1
View File
@@ -879,6 +879,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
self.get_hf_config().vision_config.spatial_merge_size,
video_needs_metadata=True,
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
def _get_vision_info(
+13 -1
View File
@@ -23,7 +23,7 @@ from typing_extensions import TypeVar
from vllm.utils.collection_utils import is_list_of
from vllm.utils.import_utils import LazyLoader
from vllm.utils.jsontree import json_map_leaves
from vllm.utils.jsontree import json_iter_leaves, json_map_leaves
from .media import MediaWithBytes
@@ -286,6 +286,13 @@ def nested_tensors_equal(
return a == b
def _nested_tensors_are_cpu(tensors: NestedTensors) -> bool:
"""Whether every tensor in `tensors` lives in host memory."""
return not any(
isinstance(x, torch.Tensor) and not x.is_cpu for x in json_iter_leaves(tensors)
)
def _nested_tensors_h2d(
tensors: NestedTensors,
device: torch.types.Device,
@@ -478,6 +485,11 @@ class BaseMultiModalField(ABC):
pin_memory = False
batch = [elem.data for elem in elems]
if pin_memory and not _nested_tensors_are_cpu(batch):
# Data already sits on an accelerator (e.g. a device-side HF
# processor handed it over by IPC). Pinning is a host-memory
# concept, and `pin_memory()` rejects device tensors outright.
pin_memory = False
out = self._reduce_data(batch, pin_memory=pin_memory)
return _nested_tensors_h2d(out, device=device)
+89 -6
View File
@@ -40,8 +40,18 @@ from .media import MediaWithBytes
_T = TypeVar("_T")
_I = TypeVar("_I")
EmbeddingFieldRole: TypeAlias = Literal["values", "metadata"]
"""What a field of a pre-computed-embedding input carries.
`"values"` is the embedding tensor itself; `"metadata"` is a key that sizes the
prompt's placeholder range. The distinction is what lets one declaration serve
both an ordinary request (everything present) and an EC consumer (embeddings
arrive through the connector, metadata still in the request).
"""
if TYPE_CHECKING:
import PIL.Image as PILImage
else:
PILImage = LazyLoader("PILImage", globals(), "PIL.Image")
@@ -250,34 +260,62 @@ class DictEmbeddingItems(
self,
data: Mapping[str, torch.Tensor],
modality: str,
required_fields: set[str],
required_fields: Set[str],
fields_factory: Callable[
[Mapping[str, torch.Tensor]],
Mapping[str, MultiModalFieldConfig],
],
optional_fields: Set[str] = frozenset(),
) -> None:
"""
Args:
data: The dictionary of tensors for this modality.
modality: The modality these items belong to.
required_fields: Fields `data` must contain.
fields_factory: Builds the field config from the data.
optional_fields: Fields `data` may omit. Which fields these are is
the caller's decision -- see
`MultiModalDataParser.embedding_field_sets`, where a deployment
that receives embeddings through an EC connector makes the
embeddings optional. They still need a field config, since they
are used whenever they *are* supplied.
"""
from transformers.feature_extraction_utils import BatchFeature
super().__init__(data, modality)
missing_required_data_keys = required_fields - data.keys()
# Nothing required would leave nothing to size the placeholder range
# from, so the item would parse into zero entries and silently produce a
# wrong prompt instead of failing here.
if not required_fields:
raise ValueError(
f"Cannot parse {modality!r} embeddings: every declared field is "
f"optional ({sorted(optional_fields)}), so nothing is left to "
"size the placeholder range."
)
declared_fields = set(required_fields) | set(optional_fields)
missing_required_data_keys = set(required_fields) - data.keys()
if missing_required_data_keys:
data_keys = set(data.keys())
msg = (
f"The data should contain the fields: {required_fields}, "
f"The data should contain the fields: {set(required_fields)}, "
f"but only found the following keys: {data_keys}"
)
raise ValueError(msg)
fields_config = fields_factory(data)
missing_required_fields = required_fields - fields_config.keys()
# Check every declared field, not just the required ones: an optional
# field still needs a config for when it is supplied.
missing_required_fields = declared_fields - fields_config.keys()
if missing_required_fields:
fields = set(fields_config.keys())
msg = f"{required_fields=} should be a subset of {fields=}"
msg = f"{declared_fields=} should be a subset of {fields=}"
raise ValueError(msg)
self.fields_config = fields_config
self.required_fields = required_fields
self.required_fields = set(required_fields)
self._kwargs = MultiModalKwargsItems.from_hf_inputs(
BatchFeature(dict(data)),
@@ -516,8 +554,50 @@ class MultiModalDataParser:
embedding inputs. If provided, validates that user-supplied
embeddings have the correct hidden size to prevent crashes
during model inference.
embeds_from_ec_connector (bool): Whether pre-computed embeddings may be
absent from the request because an encode/prefill/decode encoder
instance publishes them through an EC connector instead. Derived by
`BaseProcessingInfo.embeds_from_ec_connector`.
"""
embedding_fields: Mapping[str, Mapping[str, EmbeddingFieldRole]] = {}
"""Per-modality field roles for pre-computed-embedding inputs.
Declared here rather than inside `_parse_*_data` so an EC producer can read
it too: on a producer the request carries real media, so the branch that
builds `DictEmbeddingItems` never runs, yet the producer still has to know
which processed keys to publish. One declaration, both sides, no drift.
A modality absent from this mapping can only be sent whole in the request.
"""
@classmethod
def placeholder_metadata_fields(cls, modality: str) -> set[str]:
"""The keys that size `modality`'s placeholder range.
What an EC producer publishes alongside the embedding, and what a
consumer keeps requiring once the embedding itself is gone.
"""
return {
field
for field, role in cls.embedding_fields.get(modality, {}).items()
if role == "metadata"
}
def embedding_field_sets(self, modality: str) -> tuple[set[str], set[str]]:
"""`modality`'s (required, optional) fields for this deployment.
Resolves the static roles in `embedding_fields` against where the
embeddings actually come from: on an EC consumer they arrive through the
connector, so the request may omit them; anywhere else a request that
claims to carry pre-computed embeddings has to actually carry them.
"""
metadata = self.placeholder_metadata_fields(modality)
values = set(self.embedding_fields.get(modality, {})) - metadata
if self.embeds_from_ec_connector:
return metadata, values
return metadata | values, set()
def __init__(
self,
*,
@@ -526,9 +606,12 @@ class MultiModalDataParser:
audio_resample_method: Literal["pyav", "scipy", "soxr"] = "pyav",
video_needs_metadata: bool = False,
expected_hidden_size: int | None = None,
embeds_from_ec_connector: bool = False,
) -> None:
super().__init__()
self.embeds_from_ec_connector = embeds_from_ec_connector
self.audio_resampler = AudioResampler(
target_sr=target_sr,
method=audio_resample_method,
+35 -8
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from abc import abstractmethod
from collections.abc import Callable, Mapping
from contextlib import contextmanager
from dataclasses import dataclass, field
@@ -227,13 +226,30 @@ class InputProcessingContext:
self,
output: JSONTree,
) -> JSONTree:
def _postprocess_one(x: object):
if isinstance(x, torch.Tensor): # noqa: SIM102
# This mimics the behavior of transformers.BatchFeature
if x.is_floating_point():
x = x.to(dtype=self.model_config.dtype)
# "torch_shm" puts tensors on a torch.multiprocessing queue, which
# shares device tensors by CUDA IPC handle, so a device-side processor
# can hand `pixel_values` straight to the worker. Every other transport
# serializes host bytes, so the result has to be copied back first.
keep_on_device = (
self.model_config.get_multimodal_config().mm_tensor_ipc == "torch_shm"
)
return x
def _postprocess_one(x: object):
if not isinstance(x, torch.Tensor):
return x
# Bind to a Tensor-typed local: reassigning the `object`-typed
# parameter would discard the isinstance narrowing.
tensor = x
# This mimics the behavior of transformers.BatchFeature
if tensor.is_floating_point():
tensor = tensor.to(dtype=self.model_config.dtype)
if not tensor.is_cpu and not keep_on_device:
tensor = tensor.cpu()
return tensor
return json_map_leaves(_postprocess_one, output)
@@ -350,6 +366,17 @@ class BaseProcessingInfo:
return None
@property
def embeds_from_ec_connector(self) -> bool:
"""Whether pre-computed embeddings may arrive outside the request.
True only on an EC consumer, where an encode/prefill/decode encoder
instance publishes them through the connector instead, so the request
carries only the metadata that sizes the placeholder range.
"""
mm_config = self.ctx.model_config.multimodal_config
return mm_config is not None and mm_config.mm_embeds_from_ec_connector
def get_data_parser(self) -> MultiModalDataParser:
"""
Constructs a parser to preprocess multi-modal data items
@@ -362,6 +389,7 @@ class BaseProcessingInfo:
"""
return MultiModalDataParser(
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
@cached_property
@@ -372,7 +400,6 @@ class BaseProcessingInfo:
def skip_prompt_length_check(self) -> bool:
return False
@abstractmethod
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
"""
Return the maximum supported number of items for each modality.