[Frontend] Add multimodal support to /inference/v1/generate endpoint (#38405)
Signed-off-by: Nithin Chalapathi <nithin.ch10@gmail.com> Signed-off-by: Nithin Chalapathi <nithinc@berkeley.edu> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Cyrus Leung
parent
d0697cc7b6
commit
80b18230e0
@@ -0,0 +1,111 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Roundtrip tests for multimodal serde used by the disagg generate endpoint."""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.entrypoints.serve.disagg.mm_serde import (
|
||||
decode_mm_kwargs_item,
|
||||
encode_mm_kwargs_item,
|
||||
)
|
||||
from vllm.entrypoints.serve.disagg.protocol import (
|
||||
MultiModalFeatures,
|
||||
PlaceholderRangeInfo,
|
||||
)
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalBatchedField,
|
||||
MultiModalFieldElem,
|
||||
MultiModalFlatField,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalSharedField,
|
||||
)
|
||||
|
||||
|
||||
def test_mm_kwargs_item_roundtrip():
|
||||
"""Full roundtrip test with all three field types and multiple dtypes."""
|
||||
e1 = MultiModalFieldElem(
|
||||
data=torch.zeros(1000, dtype=torch.bfloat16),
|
||||
field=MultiModalBatchedField(),
|
||||
)
|
||||
e2 = MultiModalFieldElem(
|
||||
data=torch.ones(100, dtype=torch.int32),
|
||||
field=MultiModalSharedField(batch_size=4),
|
||||
)
|
||||
e3 = MultiModalFieldElem(
|
||||
data=torch.randn(20, dtype=torch.float32),
|
||||
field=MultiModalFlatField(slices=[slice(0, 10), slice(10, 20)], dim=0),
|
||||
)
|
||||
|
||||
item = MultiModalKwargsItem({"pixel_values": e1, "grid_thw": e2, "embeds": e3})
|
||||
encoded = encode_mm_kwargs_item(item)
|
||||
|
||||
# Encoded result is a base64 string
|
||||
assert isinstance(encoded, str)
|
||||
|
||||
decoded = decode_mm_kwargs_item(encoded)
|
||||
|
||||
assert set(decoded.keys()) == {"pixel_values", "grid_thw", "embeds"}
|
||||
assert torch.equal(item["pixel_values"].data, decoded["pixel_values"].data)
|
||||
assert torch.equal(item["grid_thw"].data, decoded["grid_thw"].data)
|
||||
assert torch.equal(item["embeds"].data, decoded["embeds"].data)
|
||||
assert isinstance(decoded["pixel_values"].field, MultiModalBatchedField)
|
||||
assert isinstance(decoded["grid_thw"].field, MultiModalSharedField)
|
||||
assert isinstance(decoded["embeds"].field, MultiModalFlatField)
|
||||
|
||||
|
||||
def test_mm_kwargs_item_none_data():
|
||||
"""Roundtrip with None data field."""
|
||||
elem = MultiModalFieldElem(
|
||||
data=None,
|
||||
field=MultiModalSharedField(batch_size=2),
|
||||
)
|
||||
item = MultiModalKwargsItem({"empty": elem})
|
||||
encoded = encode_mm_kwargs_item(item)
|
||||
decoded = decode_mm_kwargs_item(encoded)
|
||||
|
||||
assert decoded["empty"].data is None
|
||||
assert isinstance(decoded["empty"].field, MultiModalSharedField)
|
||||
|
||||
|
||||
def test_mm_kwargs_item_nested_tensors():
|
||||
"""Roundtrip with nested tensor data."""
|
||||
nested = [torch.randn(3, 4), torch.randn(5, 4)]
|
||||
elem = MultiModalFieldElem(
|
||||
data=nested,
|
||||
field=MultiModalBatchedField(),
|
||||
)
|
||||
item = MultiModalKwargsItem({"nested": elem})
|
||||
encoded = encode_mm_kwargs_item(item)
|
||||
decoded = decode_mm_kwargs_item(encoded)
|
||||
|
||||
decoded_data = decoded["nested"].data
|
||||
assert len(decoded_data) == 2
|
||||
assert torch.equal(nested[0], decoded_data[0])
|
||||
assert torch.equal(nested[1], decoded_data[1])
|
||||
|
||||
|
||||
def test_mm_features_with_kwargs_data():
|
||||
"""Test that MultiModalFeatures can carry serialized tensor data."""
|
||||
elem = MultiModalFieldElem(
|
||||
data=torch.randn(5, 3, dtype=torch.float32),
|
||||
field=MultiModalBatchedField(),
|
||||
)
|
||||
item = MultiModalKwargsItem({"pixel_values": elem})
|
||||
encoded = encode_mm_kwargs_item(item)
|
||||
|
||||
features = MultiModalFeatures(
|
||||
mm_hashes={"image": ["abc123"]},
|
||||
mm_placeholders={"image": [PlaceholderRangeInfo(offset=0, length=10)]},
|
||||
kwargs_data={"image": [encoded]},
|
||||
)
|
||||
|
||||
# JSON roundtrip
|
||||
json_str = features.model_dump_json()
|
||||
features2 = MultiModalFeatures.model_validate_json(json_str)
|
||||
|
||||
assert features2.mm_hashes == {"image": ["abc123"]}
|
||||
assert features2.kwargs_data is not None
|
||||
assert len(features2.kwargs_data["image"]) == 1
|
||||
|
||||
decoded = decode_mm_kwargs_item(features2.kwargs_data["image"][0])
|
||||
assert torch.equal(elem.data, decoded["pixel_values"].data)
|
||||
@@ -0,0 +1,158 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for multimodal features through the /inference/v1/generate endpoint.
|
||||
|
||||
Mirrors test_serving_tokens.py but exercises the multimodal piping
|
||||
using Qwen/Qwen3-VL-2B-Instruct end-to-end via the server's /render ->
|
||||
/generate -> /detokenize path. Intentionally avoids running the HF
|
||||
processor in the pytest parent process to keep os.fork() in sibling
|
||||
tests (e.g. test_weight_transfer_llm.py) deadlock-free.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from PIL import Image
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.multimodal.utils import encode_image_url
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-VL-2B-Instruct"
|
||||
GEN_ENDPOINT = "/inference/v1/generate"
|
||||
RENDER_ENDPOINT = "/v1/chat/completions/render"
|
||||
DETOKENIZE_ENDPOINT = "/detokenize"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def test_image():
|
||||
return Image.new("RGB", (224, 224), color=(255, 0, 0))
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"4096",
|
||||
"--enforce-eager",
|
||||
"--no-enable-prefix-caching",
|
||||
]
|
||||
|
||||
envs = os.environ.copy()
|
||||
envs["VLLM_ROCM_USE_SKINNY_GEMM"] = "0"
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=envs) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server: RemoteOpenAIServer):
|
||||
transport = httpx.AsyncHTTPTransport(uds=server.uds) if server.uds else None
|
||||
headers = {"Authorization": f"Bearer {server.DUMMY_API_KEY}"}
|
||||
async with httpx.AsyncClient(
|
||||
transport=transport,
|
||||
base_url=server.url_root,
|
||||
timeout=600,
|
||||
headers=headers,
|
||||
) as c:
|
||||
yield c
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_render_to_generate_roundtrip(client, test_image):
|
||||
"""End-to-end: render a multimodal chat -> feed into generate -> decode.
|
||||
|
||||
All preprocessing and detokenization happens in the server subprocess;
|
||||
the pytest parent never imports transformers or touches torch tensors.
|
||||
"""
|
||||
data_url = encode_image_url(test_image, format="PNG")
|
||||
|
||||
render_payload = {
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": data_url}},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "What color is this image? Answer in one word.",
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
render_resp = await client.post(RENDER_ENDPOINT, json=render_payload)
|
||||
render_resp.raise_for_status()
|
||||
render_data = render_resp.json()
|
||||
|
||||
# Validate render output structure: keys exist and values are non-empty
|
||||
# and well-typed.
|
||||
assert "token_ids" in render_data
|
||||
assert isinstance(render_data["token_ids"], list)
|
||||
assert len(render_data["token_ids"]) > 0
|
||||
assert all(isinstance(t, int) for t in render_data["token_ids"])
|
||||
|
||||
assert "features" in render_data
|
||||
features = render_data["features"]
|
||||
assert features is not None
|
||||
assert isinstance(features, dict)
|
||||
|
||||
assert "mm_hashes" in features
|
||||
assert "image" in features["mm_hashes"]
|
||||
image_hashes = features["mm_hashes"]["image"]
|
||||
assert isinstance(image_hashes, list)
|
||||
assert len(image_hashes) > 0
|
||||
assert all(isinstance(h, str) and h for h in image_hashes)
|
||||
|
||||
assert "mm_placeholders" in features
|
||||
assert "image" in features["mm_placeholders"]
|
||||
image_placeholders = features["mm_placeholders"]["image"]
|
||||
assert isinstance(image_placeholders, list)
|
||||
assert len(image_placeholders) > 0
|
||||
for p in image_placeholders:
|
||||
assert isinstance(p.get("offset"), int)
|
||||
assert isinstance(p.get("length"), int)
|
||||
assert p["length"] > 0
|
||||
|
||||
assert "kwargs_data" in features
|
||||
assert "image" in features["kwargs_data"]
|
||||
assert len(features["kwargs_data"]["image"]) > 0
|
||||
|
||||
# Build generate request from render output
|
||||
generate_payload = render_data
|
||||
generate_payload["sampling_params"] = {
|
||||
"max_tokens": 10,
|
||||
"temperature": 0.0,
|
||||
}
|
||||
|
||||
gen_resp = await client.post(GEN_ENDPOINT, json=generate_payload)
|
||||
gen_resp.raise_for_status()
|
||||
gen_data = gen_resp.json()
|
||||
|
||||
assert "choices" in gen_data
|
||||
assert isinstance(gen_data["choices"], list)
|
||||
assert len(gen_data["choices"]) >= 1
|
||||
choice = gen_data["choices"][0]
|
||||
assert "token_ids" in choice
|
||||
assert isinstance(choice["token_ids"], list)
|
||||
assert len(choice["token_ids"]) > 0
|
||||
assert all(isinstance(t, int) for t in choice["token_ids"])
|
||||
|
||||
detok_resp = await client.post(
|
||||
DETOKENIZE_ENDPOINT,
|
||||
json={"model": MODEL_NAME, "tokens": choice["token_ids"]},
|
||||
)
|
||||
detok_resp.raise_for_status()
|
||||
detok_data = detok_resp.json()
|
||||
assert "prompt" in detok_data
|
||||
text = detok_data["prompt"]
|
||||
assert isinstance(text, str)
|
||||
assert len(text) > 0
|
||||
assert "red" in text.lower(), (
|
||||
f"Expected model to identify the red image, got: {text!r}"
|
||||
)
|
||||
Reference in New Issue
Block a user