[EPD] Remove duplicate image preprocessing in EPD and enable preprocess on GPU (#50390)
Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
@@ -21,13 +21,18 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
import aiohttp
|
||||
import pybase64 as base64
|
||||
import uvicorn
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
@@ -53,6 +58,89 @@ decode_session: aiohttp.ClientSession | None = None
|
||||
|
||||
MM_TYPES = {"image_url", "audio_url", "input_audio"}
|
||||
|
||||
# Diagnostic switch: forward the original request to the decoder so the
|
||||
# only difference from the rewrite path is the rewrite itself.
|
||||
NO_REWRITE = False
|
||||
|
||||
|
||||
# Grid metadata reported by the encoder instance, keyed by item index.
|
||||
# Empty when the encoder did not report any (then nothing is rewritten).
|
||||
def content_uuid(item: dict) -> str:
|
||||
"""Cache key for a multimodal item, derived from its content.
|
||||
|
||||
Must be content-derived, not request-derived: the EC cache is keyed by this
|
||||
value, so a per-request key (a request id, say) would make every request a
|
||||
miss and throw away cross-request reuse of already-encoded media -- while
|
||||
the unmodified path, which hashes the content, would keep it. That asymmetry
|
||||
silently biases any comparison between the two.
|
||||
"""
|
||||
url = (item.get("image_url") or item.get("audio_url") or {}).get("url") or ""
|
||||
payload = url or json.dumps(item, sort_keys=True)
|
||||
return hashlib.sha256(payload.encode()).hexdigest()
|
||||
|
||||
|
||||
def _b64_tensor(values: list) -> str:
|
||||
import torch
|
||||
|
||||
buf = io.BytesIO()
|
||||
grid = torch.tensor(values, dtype=torch.long)
|
||||
# Downstream stacks per item, so hand over a flat (t, h, w).
|
||||
torch.save(grid.reshape(-1)[:3], buf)
|
||||
return base64.b64encode(buf.getvalue()).decode()
|
||||
|
||||
|
||||
def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict:
|
||||
"""Replace each image item with a metadata-only reference for the decoder.
|
||||
|
||||
The decoder does not need the pixels: the encoder instance already produced
|
||||
the embedding and published it through the EC connector under the same uuid.
|
||||
Sending only the grid lets the decoder size the placeholder range without
|
||||
re-running the image transform.
|
||||
|
||||
`item_meta` holds what the encoder reported for each item (its cache key and
|
||||
the grid its processor actually produced), so the grid is never re-derived
|
||||
here -- a second derivation could disagree with the encoder's.
|
||||
"""
|
||||
rewritten = 0
|
||||
idx = 0
|
||||
new_messages = []
|
||||
for msg in req_data.get("messages", []):
|
||||
content = msg.get("content")
|
||||
if not isinstance(content, list):
|
||||
new_messages.append(msg)
|
||||
continue
|
||||
new_content = []
|
||||
for item in content:
|
||||
if item.get("type") not in MM_TYPES:
|
||||
new_content.append(item)
|
||||
continue
|
||||
meta = dict(item_meta.get(idx) or {})
|
||||
idx += 1
|
||||
item_uuid = meta.pop("mm_hash", None)
|
||||
# Whatever keys the encoder reported are the metadata its model
|
||||
# declared as needed to size the placeholder range; the proxy does
|
||||
# not need to know their names.
|
||||
metadata = {k: _b64_tensor(v) for k, v in meta.items()}
|
||||
if not metadata or not item_uuid:
|
||||
# The encoder reported no metadata (e.g. the item came from its
|
||||
# processor cache); let the decoder process the media itself.
|
||||
new_content.append(item)
|
||||
continue
|
||||
new_content.append(
|
||||
{
|
||||
"type": "image_embeds",
|
||||
"image_embeds": metadata,
|
||||
"uuid": item_uuid,
|
||||
}
|
||||
)
|
||||
rewritten += 1
|
||||
new_messages.append({**msg, "content": new_content})
|
||||
|
||||
if not rewritten:
|
||||
return req_data
|
||||
logger.info("Rewrote %d image item(s) as metadata references", rewritten)
|
||||
return {**req_data, "messages": new_messages}
|
||||
|
||||
|
||||
def extract_mm_items(request_data: dict) -> list[dict]:
|
||||
"""
|
||||
@@ -77,22 +165,29 @@ async def fanout_encoder_primer(
|
||||
orig_request: dict,
|
||||
e_urls: list[str],
|
||||
req_id: str,
|
||||
) -> None:
|
||||
) -> dict[int, dict]:
|
||||
"""
|
||||
1. Build one request *per MM item* with all text removed.
|
||||
2. Send them concurrently to the encode cluster.
|
||||
3. Raise if any of them fails.
|
||||
|
||||
Returns, per item index, the metadata the encoder reported in
|
||||
`ec_transfer_params`: its EC cache key and the grid its processor produced.
|
||||
The proxy still supplies the uuid so both sides key the cache the same way;
|
||||
the grid can only come from the encoder, which is the side that computed it.
|
||||
"""
|
||||
logger.info("[%s] Processing multimodal items...", req_id)
|
||||
|
||||
mm_items = extract_mm_items(orig_request)
|
||||
if not mm_items:
|
||||
logger.info("[%s] No multimodal items, skipping encoder", req_id)
|
||||
return # nothing to do
|
||||
return {} # nothing to do
|
||||
|
||||
logger.info("[%s] got %d multimodal items...", req_id, len(mm_items))
|
||||
|
||||
tasks = []
|
||||
item_uuids: dict[int, str] = {}
|
||||
item_meta: dict[int, dict] = {}
|
||||
|
||||
# Round-robin over encode servers to distribute load a bit
|
||||
url_cycle = (e_urls[i % len(e_urls)] for i in range(len(mm_items)))
|
||||
@@ -102,11 +197,24 @@ async def fanout_encoder_primer(
|
||||
child_req_id = f"{req_id}:{idx}:{uuid.uuid4().hex[:6]}"
|
||||
headers = {"x-request-id": child_req_id}
|
||||
|
||||
# With --no-rewrite the decoder still receives the raw image and derives
|
||||
# the cache key by hashing it, so the encoder must do the same -- passing
|
||||
# a uuid here would make the two disagree and silently defeat the EC
|
||||
# transfer, leaving the decoder to encode the image itself.
|
||||
item_uuid = None if NO_REWRITE else content_uuid(item)
|
||||
if item_uuid is not None:
|
||||
item_uuids[idx] = item_uuid
|
||||
|
||||
encoder_req = {
|
||||
# You *may* need to keep additional fields
|
||||
"model": orig_request.get("model"),
|
||||
"messages": [
|
||||
{"role": "user", "content": [item]},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
item if item_uuid is None else {**item, "uuid": item_uuid}
|
||||
],
|
||||
},
|
||||
],
|
||||
# No max_tokens cap: the encoder instance never samples, it finishes
|
||||
# once the prompt is encoded and its embeddings are published.
|
||||
@@ -152,9 +260,21 @@ async def fanout_encoder_primer(
|
||||
detail=f"Encoder request failed: {detail}",
|
||||
)
|
||||
|
||||
# The encoder reports each item's cache key and grid here.
|
||||
try:
|
||||
params = (await r.json()).get("ec_transfer_params") or {}
|
||||
reported = params.get("ec_items") or []
|
||||
except Exception:
|
||||
logger.warning("[%s] Could not read encoder metadata #%d", req_id, idx)
|
||||
reported = []
|
||||
if reported and idx in item_uuids:
|
||||
# One item per encoder request, so the first entry is this item's.
|
||||
item_meta[idx] = {**reported[0], "mm_hash": item_uuids[idx]}
|
||||
|
||||
logger.info(
|
||||
"[%s] All %d encoder requests completed successfully", req_id, len(mm_items)
|
||||
)
|
||||
return item_meta
|
||||
|
||||
|
||||
async def maybe_prefill(
|
||||
@@ -320,7 +440,11 @@ async def forward_non_stream(
|
||||
) -> dict:
|
||||
try:
|
||||
# Step 1: Process through Encoder instance (if has MM input)
|
||||
await fanout_encoder_primer(req_data, e_urls, req_id)
|
||||
_t0 = time.perf_counter()
|
||||
item_meta = await fanout_encoder_primer(req_data, e_urls, req_id)
|
||||
_t1 = time.perf_counter()
|
||||
req_data = req_data if NO_REWRITE else rewrite_for_decode(req_data, item_meta)
|
||||
_t2 = time.perf_counter()
|
||||
|
||||
# Step 2: Process through Prefill instance
|
||||
req_data = await maybe_prefill(req_data, p_url, req_id)
|
||||
@@ -334,7 +458,17 @@ async def forward_non_stream(
|
||||
f"{d_url}/v1/chat/completions", json=req_data, headers=headers
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
return await resp.json()
|
||||
out = await resp.json()
|
||||
_t3 = time.perf_counter()
|
||||
logger.info(
|
||||
"STAGE %s encode=%.1f rewrite=%.1f decode=%.1f total=%.1f",
|
||||
"no-rewrite" if NO_REWRITE else "rewrite",
|
||||
(_t1 - _t0) * 1e3,
|
||||
(_t2 - _t1) * 1e3,
|
||||
(_t3 - _t2) * 1e3,
|
||||
(_t3 - _t0) * 1e3,
|
||||
)
|
||||
return out
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
@@ -348,7 +482,11 @@ async def forward_stream(
|
||||
) -> AsyncIterator[str]:
|
||||
try:
|
||||
# Step 1: Process through Encoder instance (if has MM input)
|
||||
await fanout_encoder_primer(req_data, e_urls, req_id)
|
||||
_t0 = time.perf_counter()
|
||||
item_meta = await fanout_encoder_primer(req_data, e_urls, req_id)
|
||||
_t1 = time.perf_counter()
|
||||
req_data = req_data if NO_REWRITE else rewrite_for_decode(req_data, item_meta)
|
||||
_t2 = time.perf_counter()
|
||||
|
||||
# Step 2: Process through Prefill instance
|
||||
req_data = await maybe_prefill(req_data, p_url, req_id)
|
||||
@@ -358,6 +496,7 @@ async def forward_stream(
|
||||
headers = {"x-request-id": req_id}
|
||||
|
||||
# Streaming response
|
||||
_first = None
|
||||
async with decode_session.post(
|
||||
f"{d_url}/v1/chat/completions",
|
||||
json=req_data,
|
||||
@@ -366,8 +505,19 @@ async def forward_stream(
|
||||
resp.raise_for_status()
|
||||
async for chunk in resp.content.iter_chunked(1024):
|
||||
if chunk:
|
||||
if _first is None:
|
||||
_first = time.perf_counter()
|
||||
yield chunk.decode("utf-8", errors="ignore")
|
||||
_t3 = time.perf_counter()
|
||||
|
||||
logger.info(
|
||||
"STAGE %s encode=%.1f rewrite=%.2f decode_ttfb=%.1f decode_total=%.1f",
|
||||
"no-rewrite" if NO_REWRITE else "rewrite",
|
||||
(_t1 - _t0) * 1e3,
|
||||
(_t2 - _t1) * 1e3,
|
||||
((_first or _t3) - _t2) * 1e3,
|
||||
(_t3 - _t2) * 1e3,
|
||||
)
|
||||
logger.info("[%s] Streaming completed", req_id)
|
||||
|
||||
except HTTPException:
|
||||
@@ -553,6 +703,11 @@ if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--host", default="0.0.0.0")
|
||||
parser.add_argument("--port", type=int, default=8000)
|
||||
parser.add_argument(
|
||||
"--no-rewrite",
|
||||
action="store_true",
|
||||
help="Forward images to the decoder unchanged (for stage-timing A/B).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--encode-servers-urls",
|
||||
required=True,
|
||||
@@ -573,6 +728,7 @@ if __name__ == "__main__":
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
NO_REWRITE = args.no_rewrite
|
||||
app.state.e_urls = [
|
||||
u.strip() for u in args.encode_servers_urls.split(",") if u.strip()
|
||||
]
|
||||
|
||||
@@ -4,10 +4,13 @@
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config.ec_transfer import ECRole, ECTransferConfig
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
from vllm.config.vllm import VllmConfig
|
||||
from vllm.transformers_utils.model_arch_config_convertor import (
|
||||
ModelArchConfigConvertorBase,
|
||||
)
|
||||
@@ -173,3 +176,174 @@ def test_sticky_cache_survives_text_subconfig_regeneration():
|
||||
side_effect=AssertionError("must not re-query registry"),
|
||||
):
|
||||
assert text_config._supports_multimodal_for_mm_prefix() is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("device", "expected"),
|
||||
[
|
||||
(None, None),
|
||||
("cpu", "cpu"),
|
||||
("cuda", "cuda"),
|
||||
# Callers compare against a bare device type, so an indexed device and a
|
||||
# torch.device must normalise to the same thing as "cuda".
|
||||
("cuda:1", "cuda"),
|
||||
(torch.device("cuda", 1), "cuda"),
|
||||
],
|
||||
)
|
||||
def test_mm_processor_device_type_normalizes(device: object, expected: str | None):
|
||||
kwargs = {} if device is None else {"device": device}
|
||||
config = MultiModalConfig(mm_processor_kwargs=kwargs)
|
||||
assert config.get_mm_processor_device_type() == expected
|
||||
|
||||
|
||||
def _validate_mm_processor_device(*, device: str, ec_role: ECRole | None) -> None:
|
||||
ec_config = (
|
||||
None
|
||||
if ec_role is None
|
||||
# `is_ec_producer`/`is_ec_consumer` are False without a connector, so an
|
||||
# unset one would make every role look like "no EC role at all".
|
||||
else ECTransferConfig(ec_connector="ECExampleConnector", ec_role=ec_role)
|
||||
)
|
||||
MultiModalConfig(
|
||||
mm_processor_kwargs={"device": device}
|
||||
).validate_mm_processor_device(ec_config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ec_role", [None, "ec_producer"])
|
||||
def test_bad_mm_processor_device_rejected(ec_role: ECRole | None):
|
||||
"""A bad device must fail during startup, not mid-request.
|
||||
|
||||
Rejected for every role, and on a CPU-only platform too, so a typo can never
|
||||
silently fall through to running the processor somewhere unintended.
|
||||
"""
|
||||
with (
|
||||
patch("vllm.platforms.current_platform.device_type", "cpu"),
|
||||
pytest.raises(ValueError, match='Invalid "device" in mm_processor_kwargs'),
|
||||
):
|
||||
_validate_mm_processor_device(device="not-a-device", ec_role=ec_role)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ec_role", [None, "ec_consumer", "ec_both"])
|
||||
def test_accelerator_mm_processor_rejected_outside_encoder_instance(
|
||||
ec_role: ECRole | None,
|
||||
):
|
||||
"""Only an encode-only EPD instance has the device to itself.
|
||||
|
||||
Every other role runs the language model in the same process, so frontend
|
||||
accelerator work would contend with the forward pass and allocate outside
|
||||
the memory profiled for the KV cache.
|
||||
"""
|
||||
with (
|
||||
patch("vllm.platforms.current_platform.device_type", "cuda"),
|
||||
pytest.raises(ValueError, match="also runs the language model"),
|
||||
):
|
||||
_validate_mm_processor_device(device="cuda", ec_role=ec_role)
|
||||
|
||||
|
||||
def test_accelerator_mm_processor_allowed_on_encoder_instance():
|
||||
with patch("vllm.platforms.current_platform.device_type", "cuda"):
|
||||
_validate_mm_processor_device(device="cuda", ec_role="ec_producer")
|
||||
|
||||
|
||||
def test_cpu_mm_processor_needs_no_ec_role():
|
||||
"""The gate only applies to the accelerator, so CPU stays unrestricted."""
|
||||
with patch("vllm.platforms.current_platform.device_type", "cuda"):
|
||||
_validate_mm_processor_device(device="cpu", ec_role=None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("flag", "kwargs", "expected"),
|
||||
[
|
||||
# "auto" is only settled once the EC role is known, so it must leave no
|
||||
# device behind for `VllmConfig` to mistake for an explicit request.
|
||||
("auto", None, None),
|
||||
(None, None, None),
|
||||
("cpu", None, "cpu"),
|
||||
("cuda", None, "cuda"),
|
||||
# Any explicit value other than "cpu" means "whatever this platform
|
||||
# calls its accelerator".
|
||||
("gpu", None, "cuda"),
|
||||
# An explicit device in the kwargs wins over the convenience flag.
|
||||
("cpu", {"device": "cuda"}, "cuda"),
|
||||
("cuda", {"device": "cpu"}, "cpu"),
|
||||
],
|
||||
)
|
||||
def test_fold_mm_processor_device(
|
||||
flag: str | None, kwargs: dict | None, expected: str | None
|
||||
):
|
||||
with patch("vllm.platforms.current_platform.device_type", "cuda"):
|
||||
folded = MultiModalConfig.fold_mm_processor_device(kwargs, flag)
|
||||
|
||||
config = MultiModalConfig(mm_processor_kwargs=folded or {})
|
||||
assert config.get_mm_processor_device_type() == expected
|
||||
|
||||
|
||||
def _resolve_mm_processor_device(
|
||||
*,
|
||||
ec_role: ECRole | None,
|
||||
mm_tensor_ipc: str = "torch_shm",
|
||||
device: str | None = None,
|
||||
) -> str | None:
|
||||
"""Run the `auto` resolution and report where the processor ended up."""
|
||||
mm_config = MultiModalConfig(
|
||||
mm_processor_kwargs={} if device is None else {"device": device},
|
||||
mm_tensor_ipc=mm_tensor_ipc, # type: ignore[arg-type]
|
||||
)
|
||||
model_config = MagicMock(spec=ModelConfig)
|
||||
model_config.multimodal_config = mm_config
|
||||
vllm_config = MagicMock(spec=VllmConfig)
|
||||
vllm_config.model_config = model_config
|
||||
vllm_config.ec_transfer_config = (
|
||||
None
|
||||
if ec_role is None
|
||||
else ECTransferConfig(ec_connector="ECExampleConnector", ec_role=ec_role)
|
||||
)
|
||||
|
||||
with patch("vllm.platforms.current_platform.device_type", "cuda"):
|
||||
VllmConfig._resolve_mm_processor_device(vllm_config)
|
||||
return mm_config.get_mm_processor_device_type()
|
||||
|
||||
|
||||
def test_auto_mm_processor_device_uses_accelerator_on_encoder_instance():
|
||||
"""The one deployment that has the device to itself and can hand it over."""
|
||||
assert _resolve_mm_processor_device(ec_role="ec_producer") == "cuda"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ec_role", [None, "ec_consumer", "ec_both"])
|
||||
def test_auto_mm_processor_device_stays_on_cpu_off_encoder_instance(
|
||||
ec_role: ECRole | None,
|
||||
):
|
||||
"""Every other role runs the language model in the same process."""
|
||||
assert _resolve_mm_processor_device(ec_role=ec_role) is None
|
||||
|
||||
|
||||
def test_auto_mm_processor_device_needs_a_device_capable_transport():
|
||||
"""Other transports serialize host bytes, so the output is copied back."""
|
||||
assert (
|
||||
_resolve_mm_processor_device(ec_role="ec_producer", mm_tensor_ipc="direct_rpc")
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_auto_mm_processor_device_leaves_an_explicit_request_alone():
|
||||
assert _resolve_mm_processor_device(ec_role="ec_producer", device="cpu") == "cpu"
|
||||
|
||||
|
||||
def test_vllm_config_runs_the_mm_processor_device_check():
|
||||
"""Startup must reach the check; the rule itself is covered above.
|
||||
|
||||
Guards the wiring, which no other test would notice going missing.
|
||||
"""
|
||||
model_config = MagicMock(spec=ModelConfig)
|
||||
model_config.multimodal_config = MultiModalConfig(
|
||||
mm_processor_kwargs={"device": "cuda"}
|
||||
)
|
||||
vllm_config = MagicMock(spec=VllmConfig)
|
||||
vllm_config.model_config = model_config
|
||||
vllm_config.ec_transfer_config = None
|
||||
|
||||
with (
|
||||
patch("vllm.platforms.current_platform.device_type", "cuda"),
|
||||
pytest.raises(ValueError, match="also runs the language model"),
|
||||
):
|
||||
VllmConfig._validate_mm_processor_device(vllm_config)
|
||||
|
||||
@@ -103,5 +103,15 @@ class ECTransferConfig:
|
||||
def is_ec_consumer(self) -> bool:
|
||||
return self.ec_connector is not None and self.ec_role in get_args(ECConsumer)
|
||||
|
||||
@property
|
||||
def is_encode_only(self) -> bool:
|
||||
"""Whether this instance encodes but does not run the language model.
|
||||
|
||||
It allocates no KV cache either -- `GPUModelRunner.get_kv_cache_spec`
|
||||
returns {} for it -- so it is the one role that can spend accelerator
|
||||
time and memory on frontend work.
|
||||
"""
|
||||
return self.is_ec_producer and not self.is_ec_consumer
|
||||
|
||||
def get_from_extra_config(self, key, default) -> Any:
|
||||
return self.ec_connector_extra_config.get(key, default)
|
||||
|
||||
@@ -18,6 +18,7 @@ from vllm.config.multimodal import (
|
||||
MMCacheType,
|
||||
MMEncoderTPMode,
|
||||
MMHasherAlgorithm,
|
||||
MMProcessorDevice,
|
||||
MMTensorIPC,
|
||||
MultiModalConfig,
|
||||
)
|
||||
@@ -395,6 +396,7 @@ class ModelConfig:
|
||||
video_pruning_method: InitVar[str | None] = None
|
||||
mm_tensor_ipc: InitVar[MMTensorIPC] = None
|
||||
mm_ipc_gpu_memory_gb: InitVar[float | None] = None
|
||||
mm_processor_device: InitVar[MMProcessorDevice | None] = None
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
@@ -525,6 +527,7 @@ class ModelConfig:
|
||||
video_pruning_method: str | None,
|
||||
mm_tensor_ipc: MMTensorIPC,
|
||||
mm_ipc_gpu_memory_gb: float | None,
|
||||
mm_processor_device: MMProcessorDevice | None,
|
||||
) -> None:
|
||||
# Keep set served_model_name before maybe_model_redirect(self.model)
|
||||
self.served_model_name = get_served_model_name(
|
||||
@@ -763,6 +766,10 @@ class ModelConfig:
|
||||
)
|
||||
mm_encoder_tp_mode = "weights"
|
||||
|
||||
mm_processor_kwargs = MultiModalConfig.fold_mm_processor_device(
|
||||
mm_processor_kwargs, mm_processor_device
|
||||
)
|
||||
|
||||
mm_config_kwargs = dict(
|
||||
language_model_only=language_model_only,
|
||||
limit_per_prompt=limit_mm_per_prompt,
|
||||
|
||||
@@ -5,14 +5,19 @@ from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal, TypeAlias, TypedDict, cast, final
|
||||
|
||||
import torch
|
||||
from pydantic import ConfigDict, Field, field_validator, model_validator
|
||||
from pydantic.dataclasses import dataclass
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config.ec_transfer import ECTransferConfig
|
||||
from vllm.config.utils import config, get_from_deprecated_env_if_set
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.hashing import safe_hash
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BaseDummyOptions:
|
||||
@@ -64,6 +69,10 @@ MMCacheType = Literal["shm", "lru"]
|
||||
VideoPruningMethod = Literal["evs", "vidcom2"]
|
||||
MMTensorIPC = Literal["direct_rpc", "torch_shm"]
|
||||
MMHasherAlgorithm = Literal["blake3", "sha256", "sha512"]
|
||||
MMProcessorDevice: TypeAlias = str
|
||||
"""`"auto"`, `"cpu"`, or the platform's own accelerator name
|
||||
(`current_platform.device_type`, e.g. `"cuda"` on CUDA and ROCm,
|
||||
`"xpu"` on XPU). Validated against that set by the CLI."""
|
||||
|
||||
|
||||
def _get_mm_hasher_algorithm() -> MMHasherAlgorithm:
|
||||
@@ -222,6 +231,20 @@ class MultiModalConfig:
|
||||
- "direct_rpc": Use msgspec serialization via RPC
|
||||
- "torch_shm": Use torch.multiprocessing shared memory for zero-copy IPC
|
||||
Defaults to "direct_rpc". """
|
||||
mm_embeds_from_ec_connector: bool = False
|
||||
"""Whether a pre-computed-embedding input may omit the `*_embeds` tensor.
|
||||
|
||||
In an encode/prefill/decode (EPD) deployment the encoder instance publishes
|
||||
embeddings through the EC connector, so the request that reaches the
|
||||
prefill/decode instance only needs to carry the grid/size metadata that
|
||||
sizes the placeholder range — the embeddings themselves come from the
|
||||
connector, keyed by `mm_hash`.
|
||||
|
||||
Derived, not user-settable: `VllmConfig.__post_init__` sets this to True
|
||||
exactly on EC consumers. Everywhere else it stays False so that a request
|
||||
which forgets its embeddings still fails fast in the frontend, with a clear
|
||||
error, rather than deep inside the model."""
|
||||
|
||||
mm_ipc_gpu_memory_gb: float = Field(default=0, ge=0)
|
||||
"""Amount of GPU memory (in GiB) sequestered on the engine's device for
|
||||
GPU-side multimodal work in the API-server (frontend) process, such as
|
||||
@@ -319,6 +342,119 @@ class MultiModalConfig:
|
||||
)
|
||||
return self
|
||||
|
||||
@staticmethod
|
||||
def fold_mm_processor_device(
|
||||
mm_processor_kwargs: dict[str, Any] | None,
|
||||
mm_processor_device: MMProcessorDevice | None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Fold the `mm_processor_device` convenience flag into the kwargs.
|
||||
|
||||
The flag keeps no state of its own: `mm_processor_kwargs["device"]` is
|
||||
the only representation of where the processor runs, so an explicit
|
||||
`device` there always wins and `"auto"` stays unresolved for
|
||||
`VllmConfig`, which is where the EC role needed to resolve it lives.
|
||||
|
||||
Args:
|
||||
mm_processor_kwargs: The kwargs as given, or None.
|
||||
mm_processor_device: The flag's value, or None when unset.
|
||||
|
||||
Returns:
|
||||
The kwargs to build the config with, unchanged unless the flag adds
|
||||
a `device`.
|
||||
"""
|
||||
if mm_processor_device in (None, "auto"):
|
||||
return mm_processor_kwargs
|
||||
if (mm_processor_kwargs or {}).get("device") is not None:
|
||||
return mm_processor_kwargs
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# Any explicit value other than "cpu" means "the accelerator", so a
|
||||
# programmatically-set "cuda" still works on a platform whose device type
|
||||
# is named differently ("xpu"), and degrades to CPU where there is none.
|
||||
device = (
|
||||
"cpu"
|
||||
if mm_processor_device == "cpu"
|
||||
else (current_platform.device_type or "cpu")
|
||||
)
|
||||
return {**(mm_processor_kwargs or {}), "device": device}
|
||||
|
||||
def get_mm_processor_device_type(self) -> str | None:
|
||||
"""The torch device type `mm_processor_kwargs["device"]` names.
|
||||
|
||||
`mm_processor_kwargs` is untyped, so `device` may be any form torch
|
||||
accepts -- `"cuda"`, `"cuda:1"`, `torch.device(...)`, or a bare index.
|
||||
Normalising through torch rather than parsing the string keeps the
|
||||
non-string forms from slipping past a caller's comparison.
|
||||
|
||||
Returns:
|
||||
The device type, or None when no device is requested.
|
||||
|
||||
Raises:
|
||||
ValueError: If `device` is not something `torch.device` accepts.
|
||||
`validate_mm_processor_device` is what surfaces this during
|
||||
startup, so the value is only parsed once.
|
||||
"""
|
||||
device = (self.mm_processor_kwargs or {}).get("device")
|
||||
if device is None:
|
||||
return None
|
||||
try:
|
||||
return torch.device(device).type # type: ignore[arg-type]
|
||||
except (RuntimeError, TypeError, ValueError):
|
||||
raise ValueError(
|
||||
f'Invalid "device" in mm_processor_kwargs: {device!r}. Expected a '
|
||||
'torch device such as "cpu", "cuda" or "cuda:0".'
|
||||
) from None
|
||||
|
||||
def validate_mm_processor_device(self, ec_config: ECTransferConfig | None) -> None:
|
||||
"""Check `mm_processor_kwargs["device"]` for this deployment.
|
||||
|
||||
The only place the requested device is validated, so it runs even on a
|
||||
CPU-only platform: the value is parsed before any early return.
|
||||
|
||||
Args:
|
||||
ec_config: The deployment's EC config, or None when it is not an
|
||||
encode/prefill/decode deployment. Passed in because it is not
|
||||
reachable from here, and because a field assigned after
|
||||
construction would not re-trigger this config's validators.
|
||||
|
||||
Raises:
|
||||
ValueError: If the requested device is not a torch device, or if it
|
||||
is the accelerator on an instance that also runs the language
|
||||
model.
|
||||
"""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
device_type = self.get_mm_processor_device_type()
|
||||
accelerator = current_platform.device_type
|
||||
if device_type is None or accelerator in ("", "cpu"):
|
||||
return
|
||||
if device_type != accelerator:
|
||||
return
|
||||
|
||||
if ec_config is None or not ec_config.is_encode_only:
|
||||
raise ValueError(
|
||||
f"Cannot run the multi-modal processor on {device_type!r}: this "
|
||||
"instance also runs the language model. The processor would "
|
||||
"share the device with the model's forward pass, so its "
|
||||
"transform kernels contend with that compute, and because it "
|
||||
"runs in the API-server process its allocations are outside the "
|
||||
"memory the engine profiled for its KV cache -- risking OOM or "
|
||||
"a silently shrunken cache.\n"
|
||||
"Accelerator preprocessing is only supported on an encode-only "
|
||||
"instance of an encode/prefill/decode deployment (an EC "
|
||||
"producer that is not also a consumer), which runs no forward "
|
||||
"pass and allocates no KV cache.\n"
|
||||
'Use --mm-processor-device=cpu, or drop "device" from '
|
||||
"--mm-processor-kwargs."
|
||||
)
|
||||
|
||||
logger.info_once(
|
||||
"Running the multi-modal processor on %s. Override with "
|
||||
"--mm-processor-device=cpu.",
|
||||
device_type,
|
||||
)
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
|
||||
@@ -1509,6 +1509,10 @@ class VllmConfig:
|
||||
)
|
||||
current_platform.check_and_update_config(self)
|
||||
|
||||
self._resolve_mm_embeds_from_ec_connector()
|
||||
self._resolve_mm_processor_device()
|
||||
self._validate_mm_processor_device()
|
||||
|
||||
if self.use_v2_model_runner:
|
||||
self._validate_v2_model_runner()
|
||||
elif self.parallel_config.prefill_context_parallel_size > 1:
|
||||
@@ -2185,6 +2189,101 @@ class VllmConfig:
|
||||
f"kernel_config={self.kernel_config!r}"
|
||||
)
|
||||
|
||||
def _resolve_mm_embeds_from_ec_connector(self) -> None:
|
||||
"""Allow `*_embeds` to be omitted only where the connector supplies them.
|
||||
|
||||
That is exactly an EC consumer: the encoder instance publishes the
|
||||
embeddings through the EC connector, so the request only has to carry
|
||||
the grid metadata that sizes the placeholder range. On every other
|
||||
deployment a missing `*_embeds` is a client error and must keep failing
|
||||
fast in the frontend.
|
||||
"""
|
||||
model_config = self.model_config
|
||||
if model_config is None:
|
||||
return
|
||||
mm_config = model_config.multimodal_config
|
||||
if mm_config is None:
|
||||
return
|
||||
|
||||
ec_config = self.ec_transfer_config
|
||||
# Derived, so overwrite unconditionally rather than honouring a value
|
||||
# that was set by hand.
|
||||
mm_config.mm_embeds_from_ec_connector = (
|
||||
ec_config is not None and ec_config.is_ec_consumer
|
||||
)
|
||||
if mm_config.mm_embeds_from_ec_connector:
|
||||
logger.info_once(
|
||||
"EC consumer: pre-computed-embedding inputs may omit the "
|
||||
"embedding tensor; embeddings are loaded from the EC connector."
|
||||
)
|
||||
|
||||
def _resolve_mm_processor_device(self) -> None:
|
||||
"""Settle `--mm-processor-device=auto` now that the EC role is known.
|
||||
|
||||
"auto" means "the accelerator, but only where the processor has it to
|
||||
itself and its output can be handed over without a copy back to host":
|
||||
an encode-only instance whose tensor transport carries device tensors.
|
||||
Every other deployment keeps the processor on CPU.
|
||||
|
||||
An explicit device -- from `--mm-processor-device` or straight from
|
||||
`mm_processor_kwargs` -- is already folded in by `MultiModalConfig`, so
|
||||
it is left alone here and validated by `_validate_mm_processor_device`.
|
||||
"""
|
||||
model_config = self.model_config
|
||||
if model_config is None:
|
||||
return
|
||||
mm_config = model_config.multimodal_config
|
||||
if mm_config is None:
|
||||
return
|
||||
if mm_config.get_mm_processor_device_type() is not None:
|
||||
return
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
device_type = current_platform.device_type
|
||||
if device_type in ("", "cpu"):
|
||||
return
|
||||
|
||||
ec_config = self.ec_transfer_config
|
||||
# An EC producer that is not also a consumer runs no forward pass and
|
||||
# allocates no KV cache, so frontend accelerator work has the device to
|
||||
# itself.
|
||||
if ec_config is None or not ec_config.is_encode_only:
|
||||
return
|
||||
|
||||
if mm_config.mm_tensor_ipc != "torch_shm":
|
||||
# Any other transport serializes host bytes, so the output would be
|
||||
# copied back, and that copy costs more than running the transform
|
||||
# on device saves.
|
||||
logger.info_once(
|
||||
"EPD encoder instance: keeping the multi-modal processor on CPU "
|
||||
"because mm_tensor_ipc=%s cannot carry device tensors. Add "
|
||||
"--mm-tensor-ipc=torch_shm to run it on the accelerator.",
|
||||
mm_config.mm_tensor_ipc,
|
||||
)
|
||||
return
|
||||
|
||||
mm_config.mm_processor_kwargs = {
|
||||
**(mm_config.mm_processor_kwargs or {}),
|
||||
"device": device_type,
|
||||
}
|
||||
logger.info_once(
|
||||
"EPD encoder instance: running the multi-modal processor on %s. "
|
||||
"Override with --mm-processor-device=cpu.",
|
||||
device_type,
|
||||
)
|
||||
|
||||
def _validate_mm_processor_device(self) -> None:
|
||||
"""Hand the EC config to `MultiModalConfig`, which owns the rule."""
|
||||
model_config = self.model_config
|
||||
if model_config is None:
|
||||
return
|
||||
mm_config = model_config.multimodal_config
|
||||
if mm_config is None:
|
||||
return
|
||||
|
||||
mm_config.validate_mm_processor_device(self.ec_transfer_config)
|
||||
|
||||
def _get_v2_model_runner_unsupported_features(self) -> list[str]:
|
||||
"""Collect features not yet supported by the V2 model runner."""
|
||||
unsupported: list[str] = []
|
||||
|
||||
@@ -2,9 +2,10 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import safetensors
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||
@@ -50,6 +51,8 @@ class ECExampleConnector(ECConnectorBase):
|
||||
super().__init__(vllm_config=vllm_config, role=role)
|
||||
# req_id -> index
|
||||
self._mm_datas_need_loads: dict[str, int] = {}
|
||||
self._model_config = vllm_config.model_config
|
||||
self._metadata_fields_cache: dict[str, set[str]] = {}
|
||||
transfer_config = vllm_config.ec_transfer_config
|
||||
if transfer_config is not None:
|
||||
self._storage_path = transfer_config.get_from_extra_config(
|
||||
@@ -165,6 +168,70 @@ class ECExampleConnector(ECConnectorBase):
|
||||
self._mm_datas_need_loads.clear()
|
||||
return meta
|
||||
|
||||
def _placeholder_metadata_fields(self, modality: str) -> set[str]:
|
||||
"""Which processed keys this model needs published for `modality`.
|
||||
|
||||
Read from `MultiModalDataParser.embedding_fields`, the same declaration
|
||||
the consumer's parser requires, so the two cannot drift. An empty set
|
||||
means the modality cannot be delivered out of band, and the consumer
|
||||
will process the media itself.
|
||||
"""
|
||||
if modality in self._metadata_fields_cache:
|
||||
return self._metadata_fields_cache[modality]
|
||||
|
||||
fields: set[str] = set()
|
||||
try:
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
info = MULTIMODAL_REGISTRY.create_processor(self._model_config).info
|
||||
fields = info.data_parser.placeholder_metadata_fields(modality)
|
||||
except Exception:
|
||||
# Reporting nothing is a safe degradation: the consumer falls back to
|
||||
# processing the media itself.
|
||||
logger.warning(
|
||||
"Could not determine the placeholder metadata fields for "
|
||||
"modality %s; the consumer will preprocess the media itself.",
|
||||
modality,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
self._metadata_fields_cache[modality] = fields
|
||||
return fields
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
request: "Request",
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""Report each item's cache key and grid so a consumer can skip the
|
||||
image transform.
|
||||
|
||||
A consumer only needs the grid to size the prompt's placeholder range;
|
||||
the embedding itself arrives through this connector. Reporting the grid
|
||||
the producer actually computed keeps the two sides in agreement without
|
||||
the caller re-deriving it from the raw media.
|
||||
"""
|
||||
if not self.is_producer:
|
||||
return False, None
|
||||
|
||||
items = []
|
||||
for feature in request.mm_features:
|
||||
metadata = {}
|
||||
# `data` is None for items served from the processor cache, in which
|
||||
# case the metadata is unavailable here and the consumer has to fall
|
||||
# back to processing the media itself.
|
||||
if feature.data is not None:
|
||||
wanted = self._placeholder_metadata_fields(feature.modality)
|
||||
metadata = {
|
||||
key: value.tolist()
|
||||
for key, value in feature.data.get_data().items()
|
||||
if key in wanted and isinstance(value, torch.Tensor)
|
||||
}
|
||||
items.append({"mm_hash": feature.identifier, **metadata})
|
||||
|
||||
if not items:
|
||||
return False, None
|
||||
return False, {"ec_items": items}
|
||||
|
||||
# ==============================
|
||||
# Helper functions
|
||||
# ==============================
|
||||
|
||||
@@ -90,6 +90,7 @@ from vllm.config.multimodal import (
|
||||
MMCacheType,
|
||||
MMEncoderTPMode,
|
||||
MMHasherAlgorithm,
|
||||
MMProcessorDevice,
|
||||
MMTensorIPC,
|
||||
)
|
||||
from vllm.config.observability import DetailedTraceModules
|
||||
@@ -596,6 +597,7 @@ class EngineArgs:
|
||||
video_pruning_rate: float | None = MultiModalConfig.video_pruning_rate
|
||||
video_pruning_method: str = MultiModalConfig.video_pruning_method
|
||||
mm_tensor_ipc: MMTensorIPC = MultiModalConfig.mm_tensor_ipc
|
||||
mm_processor_device: MMProcessorDevice = "auto"
|
||||
mm_ipc_gpu_memory_gb: float = MultiModalConfig.mm_ipc_gpu_memory_gb
|
||||
# LoRA fields
|
||||
enable_lora: bool = False
|
||||
@@ -1353,6 +1355,30 @@ class EngineArgs:
|
||||
multimodal_group.add_argument(
|
||||
"--mm-tensor-ipc", **multimodal_kwargs["mm_tensor_ipc"]
|
||||
)
|
||||
multimodal_group.add_argument(
|
||||
"--mm-processor-device",
|
||||
choices=["auto", "cpu"]
|
||||
+ (
|
||||
[current_platform.device_type]
|
||||
if current_platform.device_type not in ("", "cpu")
|
||||
else []
|
||||
),
|
||||
default="auto",
|
||||
help="Device the HF multi-modal processor runs the image/video "
|
||||
"transform on. Convenience for `--mm-processor-kwargs "
|
||||
"'{\"device\": ...}'`: the value is resolved here and stored there, "
|
||||
"it is not kept as separate state. Only takes effect for HF "
|
||||
'"fast" (torchvision-backed) processors, which accept a `device` '
|
||||
"argument; the others ignore it and stay on CPU.\n\n"
|
||||
'"auto" uses the accelerator on encoder instances of an '
|
||||
"encode/prefill/decode deployment -- an EC producer that is not "
|
||||
"also a consumer allocates no KV cache, so its accelerator is not "
|
||||
"contended by the language model -- and then only when "
|
||||
"`--mm-tensor-ipc=torch_shm` can carry device tensors, since every "
|
||||
"other transport would copy the result back to the host and that "
|
||||
'copy costs more than it saves. "auto" resolves to "cpu" '
|
||||
"everywhere else.",
|
||||
)
|
||||
multimodal_group.add_argument(
|
||||
"--mm-ipc-gpu-memory-gb",
|
||||
**multimodal_kwargs["mm_ipc_gpu_memory_gb"],
|
||||
@@ -1743,6 +1769,7 @@ class EngineArgs:
|
||||
video_pruning_method=self.video_pruning_method,
|
||||
mm_tensor_ipc=self.mm_tensor_ipc,
|
||||
mm_ipc_gpu_memory_gb=self.mm_ipc_gpu_memory_gb,
|
||||
mm_processor_device=self.mm_processor_device,
|
||||
io_processor_plugin=self.io_processor_plugin,
|
||||
renderer_num_workers=self.renderer_num_workers,
|
||||
)
|
||||
|
||||
@@ -104,6 +104,7 @@ class ColQwen3ProcessingInfo(Qwen3VLProcessingInfo):
|
||||
spatial_merge_size,
|
||||
video_needs_metadata=self._supports_video,
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -101,6 +101,7 @@ class ColQwen3_5ProcessingInfo(Qwen3_5ProcessingInfo):
|
||||
spatial_merge_size,
|
||||
video_needs_metadata=self._supports_video,
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -551,15 +551,22 @@ def _hunyuan_vl_field_config(hf_inputs: Mapping[str, torch.Tensor]):
|
||||
|
||||
|
||||
class HunYuanVLMultiModalDataParser(MultiModalDataParser):
|
||||
# The patch grid is what sizes the placeholder range.
|
||||
embedding_fields = {
|
||||
"image": {"image_embeds": "values", "image_grid_thw": "metadata"},
|
||||
}
|
||||
|
||||
def _parse_image_data(
|
||||
self,
|
||||
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
|
||||
) -> ModalityDataItems[Any, Any] | None:
|
||||
if isinstance(data, dict):
|
||||
required, optional = self.embedding_field_sets("image")
|
||||
return DictEmbeddingItems(
|
||||
data,
|
||||
modality="image",
|
||||
required_fields={"image_embeds", "image_grid_thw"},
|
||||
required_fields=required,
|
||||
optional_fields=optional,
|
||||
fields_factory=_hunyuan_vl_field_config,
|
||||
)
|
||||
|
||||
@@ -592,6 +599,7 @@ class HunYuanVLProcessingInfo(BaseProcessingInfo):
|
||||
def get_data_parser(self):
|
||||
return HunYuanVLMultiModalDataParser(
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
)
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
|
||||
@@ -881,18 +881,23 @@ def _keye_field_config(
|
||||
|
||||
|
||||
class KeyeMultiModalDataParser(MultiModalDataParser):
|
||||
# The patch grid is what sizes the placeholder range.
|
||||
embedding_fields = {
|
||||
"image": {"image_embeds": "values", "image_grid_thw": "metadata"},
|
||||
"video": {"video_embeds": "values", "video_grid_thw": "metadata"},
|
||||
}
|
||||
|
||||
def _parse_image_data(
|
||||
self,
|
||||
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
|
||||
) -> ModalityDataItems[Any, Any] | None:
|
||||
if isinstance(data, dict):
|
||||
required, optional = self.embedding_field_sets("image")
|
||||
return DictEmbeddingItems(
|
||||
data,
|
||||
modality="image",
|
||||
required_fields={
|
||||
"image_embeds",
|
||||
"image_grid_thw",
|
||||
},
|
||||
required_fields=required,
|
||||
optional_fields=optional,
|
||||
fields_factory=_keye_field_config,
|
||||
)
|
||||
|
||||
@@ -903,13 +908,12 @@ class KeyeMultiModalDataParser(MultiModalDataParser):
|
||||
data: dict[str, torch.Tensor] | ModalityData[VideoItem],
|
||||
) -> ModalityDataItems[Any, Any] | None:
|
||||
if isinstance(data, dict):
|
||||
required, optional = self.embedding_field_sets("video")
|
||||
return DictEmbeddingItems(
|
||||
data,
|
||||
modality="video",
|
||||
required_fields={
|
||||
"video_embeds",
|
||||
"video_grid_thw",
|
||||
},
|
||||
required_fields=required,
|
||||
optional_fields=optional,
|
||||
fields_factory=_keye_field_config,
|
||||
)
|
||||
|
||||
@@ -929,6 +933,7 @@ class KeyeProcessingInfo(BaseProcessingInfo):
|
||||
def get_data_parser(self):
|
||||
return KeyeMultiModalDataParser(
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
)
|
||||
|
||||
def get_supported_mm_limits(
|
||||
|
||||
@@ -320,18 +320,23 @@ def _keye_field_config(
|
||||
|
||||
|
||||
class KeyeVL1_5MultiModalDataParser(MultiModalDataParser):
|
||||
# The patch grid is what sizes the placeholder range.
|
||||
embedding_fields = {
|
||||
"image": {"image_embeds": "values", "image_grid_thw": "metadata"},
|
||||
"video": {"video_embeds": "values", "video_grid_thw": "metadata"},
|
||||
}
|
||||
|
||||
def _parse_image_data(
|
||||
self,
|
||||
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
|
||||
) -> ModalityDataItems[Any, Any] | None:
|
||||
if isinstance(data, dict):
|
||||
required, optional = self.embedding_field_sets("image")
|
||||
return DictEmbeddingItems(
|
||||
data,
|
||||
modality="image",
|
||||
required_fields={
|
||||
"image_embeds",
|
||||
"image_grid_thw",
|
||||
},
|
||||
required_fields=required,
|
||||
optional_fields=optional,
|
||||
fields_factory=_keye_field_config,
|
||||
)
|
||||
|
||||
@@ -342,13 +347,12 @@ class KeyeVL1_5MultiModalDataParser(MultiModalDataParser):
|
||||
data: dict[str, torch.Tensor] | ModalityData[VideoItem],
|
||||
) -> ModalityDataItems[Any, Any] | None:
|
||||
if isinstance(data, dict):
|
||||
required, optional = self.embedding_field_sets("video")
|
||||
return DictEmbeddingItems(
|
||||
data,
|
||||
modality="video",
|
||||
required_fields={
|
||||
"video_embeds",
|
||||
"video_grid_thw",
|
||||
},
|
||||
required_fields=required,
|
||||
optional_fields=optional,
|
||||
fields_factory=_keye_field_config,
|
||||
)
|
||||
|
||||
@@ -359,6 +363,7 @@ class KeyeVL1_5ProcessingInfo(KeyeProcessingInfo):
|
||||
def get_data_parser(self):
|
||||
return KeyeVL1_5MultiModalDataParser(
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
)
|
||||
|
||||
def get_max_frame_per_video(self) -> int:
|
||||
|
||||
@@ -1279,6 +1279,7 @@ class LlavaOnevision2ProcessingInfo(BaseProcessingInfo):
|
||||
return LlavaOnevision2MultiModalDataParser(
|
||||
self.get_hf_config().vision_config.spatial_merge_size,
|
||||
video_needs_metadata=True,
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
)
|
||||
|
||||
def get_hf_processor(self, **kwargs: object):
|
||||
@@ -1510,6 +1511,11 @@ class LlavaOnevision2DummyInputsBuilder(
|
||||
|
||||
|
||||
class LlavaOnevision2MultiModalDataParser(MultiModalDataParser):
|
||||
# The patch grid is what sizes the placeholder range.
|
||||
embedding_fields = {
|
||||
"image": {"image_embeds": "values", "image_grid_thw": "metadata"},
|
||||
}
|
||||
|
||||
def __init__(self, spatial_merge_size: int, *args, **kwargs):
|
||||
self._spatial_merge_size = spatial_merge_size
|
||||
super().__init__(*args, **kwargs)
|
||||
@@ -1519,10 +1525,12 @@ class LlavaOnevision2MultiModalDataParser(MultiModalDataParser):
|
||||
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
|
||||
) -> ModalityDataItems[Any, Any] | None:
|
||||
if isinstance(data, dict):
|
||||
required, optional = self.embedding_field_sets("image")
|
||||
return DictEmbeddingItems(
|
||||
data,
|
||||
modality="image",
|
||||
required_fields={"image_embeds", "image_grid_thw"},
|
||||
required_fields=required,
|
||||
optional_fields=optional,
|
||||
fields_factory=_create_field_factory(self._spatial_merge_size),
|
||||
)
|
||||
return super()._parse_image_data(data)
|
||||
|
||||
@@ -26,7 +26,7 @@
|
||||
|
||||
import math
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable, Iterable, Mapping, Sequence
|
||||
from collections.abc import Callable, Iterable, Mapping, Sequence, Set
|
||||
from functools import partial
|
||||
from itertools import chain
|
||||
from typing import Annotated, Any, Literal, TypeAlias
|
||||
@@ -474,11 +474,14 @@ class MiniCPMVImageEmbeddingItems(DictEmbeddingItems):
|
||||
[Mapping[str, torch.Tensor]],
|
||||
Mapping[str, MultiModalFieldConfig],
|
||||
],
|
||||
required_fields: Set[str],
|
||||
optional_fields: Set[str] = frozenset(),
|
||||
) -> None:
|
||||
super().__init__(
|
||||
data,
|
||||
modality="image",
|
||||
required_fields={"image_embeds", "image_sizes"},
|
||||
required_fields=required_fields,
|
||||
optional_fields=optional_fields,
|
||||
fields_factory=fields_factory,
|
||||
)
|
||||
|
||||
@@ -495,11 +498,14 @@ class MiniCPMVVideoEmbeddingItems(DictEmbeddingItems):
|
||||
[Mapping[str, torch.Tensor]],
|
||||
Mapping[str, MultiModalFieldConfig],
|
||||
],
|
||||
required_fields: Set[str],
|
||||
optional_fields: Set[str] = frozenset(),
|
||||
) -> None:
|
||||
super().__init__(
|
||||
data,
|
||||
modality="video",
|
||||
required_fields={"video_embeds", "video_image_sizes"},
|
||||
required_fields=required_fields,
|
||||
optional_fields=optional_fields,
|
||||
fields_factory=fields_factory,
|
||||
)
|
||||
|
||||
@@ -512,14 +518,23 @@ class MiniCPMVVideoEmbeddingItems(DictEmbeddingItems):
|
||||
|
||||
|
||||
class MiniCPMVMultiModalDataParser(MultiModalDataParser):
|
||||
# The per-item sizes are what size the placeholder range.
|
||||
embedding_fields = {
|
||||
"image": {"image_embeds": "values", "image_sizes": "metadata"},
|
||||
"video": {"video_embeds": "values", "video_image_sizes": "metadata"},
|
||||
}
|
||||
|
||||
def _parse_image_data(
|
||||
self,
|
||||
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
|
||||
) -> ModalityDataItems[Any, Any] | None:
|
||||
if isinstance(data, dict):
|
||||
required, optional = self.embedding_field_sets("image")
|
||||
return MiniCPMVImageEmbeddingItems(
|
||||
data,
|
||||
fields_factory=_minicpmv_field_config,
|
||||
required_fields=required,
|
||||
optional_fields=optional,
|
||||
)
|
||||
|
||||
return super()._parse_image_data(data)
|
||||
@@ -529,9 +544,12 @@ class MiniCPMVMultiModalDataParser(MultiModalDataParser):
|
||||
data: dict[str, torch.Tensor] | ModalityData[VideoItem],
|
||||
) -> ModalityDataItems[Any, Any] | None:
|
||||
if isinstance(data, dict):
|
||||
required, optional = self.embedding_field_sets("video")
|
||||
return MiniCPMVVideoEmbeddingItems(
|
||||
data,
|
||||
fields_factory=_minicpmv_field_config,
|
||||
required_fields=required,
|
||||
optional_fields=optional,
|
||||
)
|
||||
|
||||
return super()._parse_video_data(data)
|
||||
@@ -573,6 +591,7 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
|
||||
def get_data_parser(self):
|
||||
return MiniCPMVMultiModalDataParser(
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
)
|
||||
|
||||
def get_model_version(self):
|
||||
|
||||
@@ -57,6 +57,7 @@ class OpenCUAProcessingInfo(Qwen2VLProcessingInfo):
|
||||
return Qwen2VLMultiModalDataParser(
|
||||
self.get_hf_config().vision_config.spatial_merge_size,
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
)
|
||||
|
||||
def get_hf_config(self):
|
||||
|
||||
@@ -372,6 +372,7 @@ class Qwen2_5OmniThinkerProcessingInfo(
|
||||
target_sr=feature_extractor.sampling_rate,
|
||||
target_channels=self.get_target_channels(),
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
)
|
||||
|
||||
def get_target_channels(self) -> int:
|
||||
|
||||
@@ -795,6 +795,12 @@ def _create_qwen2vl_field_factory(
|
||||
|
||||
|
||||
class Qwen2VLMultiModalDataParser(MultiModalDataParser):
|
||||
# The patch grid is what sizes the placeholder range.
|
||||
embedding_fields = {
|
||||
"image": {"image_embeds": "values", "image_grid_thw": "metadata"},
|
||||
"video": {"video_embeds": "values", "video_grid_thw": "metadata"},
|
||||
}
|
||||
|
||||
def __init__(self, spatial_merge_size: int, *args, **kwargs):
|
||||
self._spatial_merge_size = spatial_merge_size
|
||||
super().__init__(*args, **kwargs)
|
||||
@@ -804,10 +810,12 @@ class Qwen2VLMultiModalDataParser(MultiModalDataParser):
|
||||
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
|
||||
) -> ModalityDataItems[Any, Any] | None:
|
||||
if isinstance(data, dict):
|
||||
required, optional = self.embedding_field_sets("image")
|
||||
return DictEmbeddingItems(
|
||||
data,
|
||||
modality="image",
|
||||
required_fields={"image_embeds", "image_grid_thw"},
|
||||
required_fields=required,
|
||||
optional_fields=optional,
|
||||
fields_factory=_create_qwen2vl_field_factory(self._spatial_merge_size),
|
||||
)
|
||||
|
||||
@@ -818,10 +826,12 @@ class Qwen2VLMultiModalDataParser(MultiModalDataParser):
|
||||
data: dict[str, torch.Tensor] | ModalityData[VideoItem],
|
||||
) -> ModalityDataItems[Any, Any] | None:
|
||||
if isinstance(data, dict):
|
||||
required, optional = self.embedding_field_sets("video")
|
||||
return DictEmbeddingItems(
|
||||
data,
|
||||
modality="video",
|
||||
required_fields={"video_embeds", "video_grid_thw"},
|
||||
required_fields=required,
|
||||
optional_fields=optional,
|
||||
fields_factory=_create_qwen2vl_field_factory(self._spatial_merge_size),
|
||||
)
|
||||
|
||||
@@ -846,6 +856,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
|
||||
return Qwen2VLMultiModalDataParser(
|
||||
self.get_hf_config().vision_config.spatial_merge_size,
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
)
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
|
||||
@@ -879,6 +879,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
|
||||
self.get_hf_config().vision_config.spatial_merge_size,
|
||||
video_needs_metadata=True,
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
)
|
||||
|
||||
def _get_vision_info(
|
||||
|
||||
@@ -23,7 +23,7 @@ from typing_extensions import TypeVar
|
||||
|
||||
from vllm.utils.collection_utils import is_list_of
|
||||
from vllm.utils.import_utils import LazyLoader
|
||||
from vllm.utils.jsontree import json_map_leaves
|
||||
from vllm.utils.jsontree import json_iter_leaves, json_map_leaves
|
||||
|
||||
from .media import MediaWithBytes
|
||||
|
||||
@@ -286,6 +286,13 @@ def nested_tensors_equal(
|
||||
return a == b
|
||||
|
||||
|
||||
def _nested_tensors_are_cpu(tensors: NestedTensors) -> bool:
|
||||
"""Whether every tensor in `tensors` lives in host memory."""
|
||||
return not any(
|
||||
isinstance(x, torch.Tensor) and not x.is_cpu for x in json_iter_leaves(tensors)
|
||||
)
|
||||
|
||||
|
||||
def _nested_tensors_h2d(
|
||||
tensors: NestedTensors,
|
||||
device: torch.types.Device,
|
||||
@@ -478,6 +485,11 @@ class BaseMultiModalField(ABC):
|
||||
pin_memory = False
|
||||
|
||||
batch = [elem.data for elem in elems]
|
||||
if pin_memory and not _nested_tensors_are_cpu(batch):
|
||||
# Data already sits on an accelerator (e.g. a device-side HF
|
||||
# processor handed it over by IPC). Pinning is a host-memory
|
||||
# concept, and `pin_memory()` rejects device tensors outright.
|
||||
pin_memory = False
|
||||
out = self._reduce_data(batch, pin_memory=pin_memory)
|
||||
return _nested_tensors_h2d(out, device=device)
|
||||
|
||||
|
||||
@@ -40,8 +40,18 @@ from .media import MediaWithBytes
|
||||
_T = TypeVar("_T")
|
||||
_I = TypeVar("_I")
|
||||
|
||||
EmbeddingFieldRole: TypeAlias = Literal["values", "metadata"]
|
||||
"""What a field of a pre-computed-embedding input carries.
|
||||
|
||||
`"values"` is the embedding tensor itself; `"metadata"` is a key that sizes the
|
||||
prompt's placeholder range. The distinction is what lets one declaration serve
|
||||
both an ordinary request (everything present) and an EC consumer (embeddings
|
||||
arrive through the connector, metadata still in the request).
|
||||
"""
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import PIL.Image as PILImage
|
||||
|
||||
else:
|
||||
PILImage = LazyLoader("PILImage", globals(), "PIL.Image")
|
||||
|
||||
@@ -250,34 +260,62 @@ class DictEmbeddingItems(
|
||||
self,
|
||||
data: Mapping[str, torch.Tensor],
|
||||
modality: str,
|
||||
required_fields: set[str],
|
||||
required_fields: Set[str],
|
||||
fields_factory: Callable[
|
||||
[Mapping[str, torch.Tensor]],
|
||||
Mapping[str, MultiModalFieldConfig],
|
||||
],
|
||||
optional_fields: Set[str] = frozenset(),
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
data: The dictionary of tensors for this modality.
|
||||
modality: The modality these items belong to.
|
||||
required_fields: Fields `data` must contain.
|
||||
fields_factory: Builds the field config from the data.
|
||||
optional_fields: Fields `data` may omit. Which fields these are is
|
||||
the caller's decision -- see
|
||||
`MultiModalDataParser.embedding_field_sets`, where a deployment
|
||||
that receives embeddings through an EC connector makes the
|
||||
embeddings optional. They still need a field config, since they
|
||||
are used whenever they *are* supplied.
|
||||
"""
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
|
||||
super().__init__(data, modality)
|
||||
|
||||
missing_required_data_keys = required_fields - data.keys()
|
||||
# Nothing required would leave nothing to size the placeholder range
|
||||
# from, so the item would parse into zero entries and silently produce a
|
||||
# wrong prompt instead of failing here.
|
||||
if not required_fields:
|
||||
raise ValueError(
|
||||
f"Cannot parse {modality!r} embeddings: every declared field is "
|
||||
f"optional ({sorted(optional_fields)}), so nothing is left to "
|
||||
"size the placeholder range."
|
||||
)
|
||||
|
||||
declared_fields = set(required_fields) | set(optional_fields)
|
||||
|
||||
missing_required_data_keys = set(required_fields) - data.keys()
|
||||
if missing_required_data_keys:
|
||||
data_keys = set(data.keys())
|
||||
msg = (
|
||||
f"The data should contain the fields: {required_fields}, "
|
||||
f"The data should contain the fields: {set(required_fields)}, "
|
||||
f"but only found the following keys: {data_keys}"
|
||||
)
|
||||
raise ValueError(msg)
|
||||
|
||||
fields_config = fields_factory(data)
|
||||
missing_required_fields = required_fields - fields_config.keys()
|
||||
# Check every declared field, not just the required ones: an optional
|
||||
# field still needs a config for when it is supplied.
|
||||
missing_required_fields = declared_fields - fields_config.keys()
|
||||
if missing_required_fields:
|
||||
fields = set(fields_config.keys())
|
||||
msg = f"{required_fields=} should be a subset of {fields=}"
|
||||
msg = f"{declared_fields=} should be a subset of {fields=}"
|
||||
raise ValueError(msg)
|
||||
|
||||
self.fields_config = fields_config
|
||||
self.required_fields = required_fields
|
||||
self.required_fields = set(required_fields)
|
||||
|
||||
self._kwargs = MultiModalKwargsItems.from_hf_inputs(
|
||||
BatchFeature(dict(data)),
|
||||
@@ -516,8 +554,50 @@ class MultiModalDataParser:
|
||||
embedding inputs. If provided, validates that user-supplied
|
||||
embeddings have the correct hidden size to prevent crashes
|
||||
during model inference.
|
||||
embeds_from_ec_connector (bool): Whether pre-computed embeddings may be
|
||||
absent from the request because an encode/prefill/decode encoder
|
||||
instance publishes them through an EC connector instead. Derived by
|
||||
`BaseProcessingInfo.embeds_from_ec_connector`.
|
||||
"""
|
||||
|
||||
embedding_fields: Mapping[str, Mapping[str, EmbeddingFieldRole]] = {}
|
||||
"""Per-modality field roles for pre-computed-embedding inputs.
|
||||
|
||||
Declared here rather than inside `_parse_*_data` so an EC producer can read
|
||||
it too: on a producer the request carries real media, so the branch that
|
||||
builds `DictEmbeddingItems` never runs, yet the producer still has to know
|
||||
which processed keys to publish. One declaration, both sides, no drift.
|
||||
|
||||
A modality absent from this mapping can only be sent whole in the request.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def placeholder_metadata_fields(cls, modality: str) -> set[str]:
|
||||
"""The keys that size `modality`'s placeholder range.
|
||||
|
||||
What an EC producer publishes alongside the embedding, and what a
|
||||
consumer keeps requiring once the embedding itself is gone.
|
||||
"""
|
||||
return {
|
||||
field
|
||||
for field, role in cls.embedding_fields.get(modality, {}).items()
|
||||
if role == "metadata"
|
||||
}
|
||||
|
||||
def embedding_field_sets(self, modality: str) -> tuple[set[str], set[str]]:
|
||||
"""`modality`'s (required, optional) fields for this deployment.
|
||||
|
||||
Resolves the static roles in `embedding_fields` against where the
|
||||
embeddings actually come from: on an EC consumer they arrive through the
|
||||
connector, so the request may omit them; anywhere else a request that
|
||||
claims to carry pre-computed embeddings has to actually carry them.
|
||||
"""
|
||||
metadata = self.placeholder_metadata_fields(modality)
|
||||
values = set(self.embedding_fields.get(modality, {})) - metadata
|
||||
if self.embeds_from_ec_connector:
|
||||
return metadata, values
|
||||
return metadata | values, set()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
@@ -526,9 +606,12 @@ class MultiModalDataParser:
|
||||
audio_resample_method: Literal["pyav", "scipy", "soxr"] = "pyav",
|
||||
video_needs_metadata: bool = False,
|
||||
expected_hidden_size: int | None = None,
|
||||
embeds_from_ec_connector: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.embeds_from_ec_connector = embeds_from_ec_connector
|
||||
|
||||
self.audio_resampler = AudioResampler(
|
||||
target_sr=target_sr,
|
||||
method=audio_resample_method,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import time
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Callable, Mapping
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
@@ -227,13 +226,30 @@ class InputProcessingContext:
|
||||
self,
|
||||
output: JSONTree,
|
||||
) -> JSONTree:
|
||||
def _postprocess_one(x: object):
|
||||
if isinstance(x, torch.Tensor): # noqa: SIM102
|
||||
# This mimics the behavior of transformers.BatchFeature
|
||||
if x.is_floating_point():
|
||||
x = x.to(dtype=self.model_config.dtype)
|
||||
# "torch_shm" puts tensors on a torch.multiprocessing queue, which
|
||||
# shares device tensors by CUDA IPC handle, so a device-side processor
|
||||
# can hand `pixel_values` straight to the worker. Every other transport
|
||||
# serializes host bytes, so the result has to be copied back first.
|
||||
keep_on_device = (
|
||||
self.model_config.get_multimodal_config().mm_tensor_ipc == "torch_shm"
|
||||
)
|
||||
|
||||
return x
|
||||
def _postprocess_one(x: object):
|
||||
if not isinstance(x, torch.Tensor):
|
||||
return x
|
||||
|
||||
# Bind to a Tensor-typed local: reassigning the `object`-typed
|
||||
# parameter would discard the isinstance narrowing.
|
||||
tensor = x
|
||||
|
||||
# This mimics the behavior of transformers.BatchFeature
|
||||
if tensor.is_floating_point():
|
||||
tensor = tensor.to(dtype=self.model_config.dtype)
|
||||
|
||||
if not tensor.is_cpu and not keep_on_device:
|
||||
tensor = tensor.cpu()
|
||||
|
||||
return tensor
|
||||
|
||||
return json_map_leaves(_postprocess_one, output)
|
||||
|
||||
@@ -350,6 +366,17 @@ class BaseProcessingInfo:
|
||||
|
||||
return None
|
||||
|
||||
@property
|
||||
def embeds_from_ec_connector(self) -> bool:
|
||||
"""Whether pre-computed embeddings may arrive outside the request.
|
||||
|
||||
True only on an EC consumer, where an encode/prefill/decode encoder
|
||||
instance publishes them through the connector instead, so the request
|
||||
carries only the metadata that sizes the placeholder range.
|
||||
"""
|
||||
mm_config = self.ctx.model_config.multimodal_config
|
||||
return mm_config is not None and mm_config.mm_embeds_from_ec_connector
|
||||
|
||||
def get_data_parser(self) -> MultiModalDataParser:
|
||||
"""
|
||||
Constructs a parser to preprocess multi-modal data items
|
||||
@@ -362,6 +389,7 @@ class BaseProcessingInfo:
|
||||
"""
|
||||
return MultiModalDataParser(
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
)
|
||||
|
||||
@cached_property
|
||||
@@ -372,7 +400,6 @@ class BaseProcessingInfo:
|
||||
def skip_prompt_length_check(self) -> bool:
|
||||
return False
|
||||
|
||||
@abstractmethod
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
"""
|
||||
Return the maximum supported number of items for each modality.
|
||||
|
||||
Reference in New Issue
Block a user