diff --git a/examples/online_serving/disaggregated_serving/example_mm_serve.py b/examples/online_serving/disaggregated_serving/example_mm_serve.py new file mode 100644 index 0000000000..11d81236c5 --- /dev/null +++ b/examples/online_serving/disaggregated_serving/example_mm_serve.py @@ -0,0 +1,117 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Disaggregated multimodal serving: render → generate round-trip. + +Demonstrates the two-phase disaggregated flow: + 1. /v1/chat/completions/render – preprocesses a multimodal chat request + into token IDs and serialized tensor features. + 2. /inference/v1/generate – runs inference on the preprocessed tokens. + +The render response is passed *directly* to generate with only +``sampling_params`` added, showing that the two endpoints compose with +zero client-side transformation. + +Launch the server first: + + vllm serve Qwen/Qwen3-VL-2B-Instruct \ + --dtype bfloat16 --max-model-len 4096 --enforce-eager + +Then run this script: + + python example_mm_serve.py +""" + +import io + +import pybase64 as base64 +import requests +from PIL import Image +from transformers import AutoTokenizer + +BASE_URL = "http://localhost:8000" +MODEL_NAME = "Qwen/Qwen3-VL-2B-Instruct" + + +def make_data_url(image: Image.Image) -> str: + """Encode a PIL image as a base64 data URL.""" + buf = io.BytesIO() + image.save(buf, format="PNG") + b64 = base64.b64encode(buf.getvalue()).decode() + return f"data:image/png;base64,{b64}" + + +def main(): + # -- Step 1: Create a test image (solid red) ------------------------- + image = Image.new("RGB", (224, 224), color=(255, 0, 0)) + data_url = make_data_url(image) + print("Created 224x224 red test image") + + # -- Step 2: Render (preprocess) ------------------------------------- + render_payload = { + "model": MODEL_NAME, + "messages": [ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": data_url}}, + { + "type": "text", + "text": "What color is this image? Answer in one word.", + }, + ], + } + ], + } + + print("\n--- Render ---") + render_resp = requests.post( + f"{BASE_URL}/v1/chat/completions/render", json=render_payload + ) + render_resp.raise_for_status() + render_data = render_resp.json() + + print(f"Response keys: {list(render_data.keys())}") + print(f"Number of token_ids: {len(render_data['token_ids'])}") + + features = render_data.get("features") + if features and features.get("kwargs_data"): + print(f"kwargs_data modalities: {list(features['kwargs_data'].keys())}") + for modality, items in features["kwargs_data"].items(): + print( + f" {modality}: {len(items)} item(s), " + f"first item type: {type(items[0])} length: {len(items[0])}" + if items + else "First item: (empty)" + ) + else: + print("WARNING: no kwargs_data in render response") + + # -- Step 3: Generate (inference) ------------------------------------ + # Pass the render output directly — only add sampling_params. + generate_payload = render_data + generate_payload["sampling_params"] = { + "max_tokens": 20, + "temperature": 0.0, + } + + print("\n--- Generate ---") + gen_resp = requests.post(f"{BASE_URL}/inference/v1/generate", json=generate_payload) + gen_resp.raise_for_status() + gen_data = gen_resp.json() + + # -- Step 4: Decode & print ------------------------------------------ + output_ids = gen_data["choices"][0]["token_ids"] + tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME) + text = tokenizer.decode(output_ids, skip_special_tokens=True) + + print(f"Output token count: {len(output_ids)}") + print(f"Generated text: {text!r}") + + if "red" in text.lower(): + print("\nModel correctly identified the red image.") + else: + print(f"\nWARNING: Expected 'red' in output, got: {text!r}") + + +if __name__ == "__main__": + main() diff --git a/tests/entrypoints/openai/test_mm_serde.py b/tests/entrypoints/openai/test_mm_serde.py new file mode 100644 index 0000000000..c568d822e1 --- /dev/null +++ b/tests/entrypoints/openai/test_mm_serde.py @@ -0,0 +1,111 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Roundtrip tests for multimodal serde used by the disagg generate endpoint.""" + +import torch + +from vllm.entrypoints.serve.disagg.mm_serde import ( + decode_mm_kwargs_item, + encode_mm_kwargs_item, +) +from vllm.entrypoints.serve.disagg.protocol import ( + MultiModalFeatures, + PlaceholderRangeInfo, +) +from vllm.multimodal.inputs import ( + MultiModalBatchedField, + MultiModalFieldElem, + MultiModalFlatField, + MultiModalKwargsItem, + MultiModalSharedField, +) + + +def test_mm_kwargs_item_roundtrip(): + """Full roundtrip test with all three field types and multiple dtypes.""" + e1 = MultiModalFieldElem( + data=torch.zeros(1000, dtype=torch.bfloat16), + field=MultiModalBatchedField(), + ) + e2 = MultiModalFieldElem( + data=torch.ones(100, dtype=torch.int32), + field=MultiModalSharedField(batch_size=4), + ) + e3 = MultiModalFieldElem( + data=torch.randn(20, dtype=torch.float32), + field=MultiModalFlatField(slices=[slice(0, 10), slice(10, 20)], dim=0), + ) + + item = MultiModalKwargsItem({"pixel_values": e1, "grid_thw": e2, "embeds": e3}) + encoded = encode_mm_kwargs_item(item) + + # Encoded result is a base64 string + assert isinstance(encoded, str) + + decoded = decode_mm_kwargs_item(encoded) + + assert set(decoded.keys()) == {"pixel_values", "grid_thw", "embeds"} + assert torch.equal(item["pixel_values"].data, decoded["pixel_values"].data) + assert torch.equal(item["grid_thw"].data, decoded["grid_thw"].data) + assert torch.equal(item["embeds"].data, decoded["embeds"].data) + assert isinstance(decoded["pixel_values"].field, MultiModalBatchedField) + assert isinstance(decoded["grid_thw"].field, MultiModalSharedField) + assert isinstance(decoded["embeds"].field, MultiModalFlatField) + + +def test_mm_kwargs_item_none_data(): + """Roundtrip with None data field.""" + elem = MultiModalFieldElem( + data=None, + field=MultiModalSharedField(batch_size=2), + ) + item = MultiModalKwargsItem({"empty": elem}) + encoded = encode_mm_kwargs_item(item) + decoded = decode_mm_kwargs_item(encoded) + + assert decoded["empty"].data is None + assert isinstance(decoded["empty"].field, MultiModalSharedField) + + +def test_mm_kwargs_item_nested_tensors(): + """Roundtrip with nested tensor data.""" + nested = [torch.randn(3, 4), torch.randn(5, 4)] + elem = MultiModalFieldElem( + data=nested, + field=MultiModalBatchedField(), + ) + item = MultiModalKwargsItem({"nested": elem}) + encoded = encode_mm_kwargs_item(item) + decoded = decode_mm_kwargs_item(encoded) + + decoded_data = decoded["nested"].data + assert len(decoded_data) == 2 + assert torch.equal(nested[0], decoded_data[0]) + assert torch.equal(nested[1], decoded_data[1]) + + +def test_mm_features_with_kwargs_data(): + """Test that MultiModalFeatures can carry serialized tensor data.""" + elem = MultiModalFieldElem( + data=torch.randn(5, 3, dtype=torch.float32), + field=MultiModalBatchedField(), + ) + item = MultiModalKwargsItem({"pixel_values": elem}) + encoded = encode_mm_kwargs_item(item) + + features = MultiModalFeatures( + mm_hashes={"image": ["abc123"]}, + mm_placeholders={"image": [PlaceholderRangeInfo(offset=0, length=10)]}, + kwargs_data={"image": [encoded]}, + ) + + # JSON roundtrip + json_str = features.model_dump_json() + features2 = MultiModalFeatures.model_validate_json(json_str) + + assert features2.mm_hashes == {"image": ["abc123"]} + assert features2.kwargs_data is not None + assert len(features2.kwargs_data["image"]) == 1 + + decoded = decode_mm_kwargs_item(features2.kwargs_data["image"][0]) + assert torch.equal(elem.data, decoded["pixel_values"].data) diff --git a/tests/entrypoints/serve/disagg/test_serving_multimodal_tokens.py b/tests/entrypoints/serve/disagg/test_serving_multimodal_tokens.py new file mode 100644 index 0000000000..e13dd425af --- /dev/null +++ b/tests/entrypoints/serve/disagg/test_serving_multimodal_tokens.py @@ -0,0 +1,158 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for multimodal features through the /inference/v1/generate endpoint. + +Mirrors test_serving_tokens.py but exercises the multimodal piping +using Qwen/Qwen3-VL-2B-Instruct end-to-end via the server's /render -> +/generate -> /detokenize path. Intentionally avoids running the HF +processor in the pytest parent process to keep os.fork() in sibling +tests (e.g. test_weight_transfer_llm.py) deadlock-free. +""" + +import os + +import httpx +import pytest +import pytest_asyncio +from PIL import Image + +from tests.utils import RemoteOpenAIServer +from vllm.multimodal.utils import encode_image_url + +MODEL_NAME = "Qwen/Qwen3-VL-2B-Instruct" +GEN_ENDPOINT = "/inference/v1/generate" +RENDER_ENDPOINT = "/v1/chat/completions/render" +DETOKENIZE_ENDPOINT = "/detokenize" + + +@pytest.fixture(scope="module") +def test_image(): + return Image.new("RGB", (224, 224), color=(255, 0, 0)) + + +@pytest.fixture(scope="module") +def server(): + args = [ + "--dtype", + "bfloat16", + "--max-model-len", + "4096", + "--enforce-eager", + "--no-enable-prefix-caching", + ] + + envs = os.environ.copy() + envs["VLLM_ROCM_USE_SKINNY_GEMM"] = "0" + + with RemoteOpenAIServer(MODEL_NAME, args, env_dict=envs) as remote_server: + yield remote_server + + +@pytest_asyncio.fixture +async def client(server: RemoteOpenAIServer): + transport = httpx.AsyncHTTPTransport(uds=server.uds) if server.uds else None + headers = {"Authorization": f"Bearer {server.DUMMY_API_KEY}"} + async with httpx.AsyncClient( + transport=transport, + base_url=server.url_root, + timeout=600, + headers=headers, + ) as c: + yield c + + +@pytest.mark.asyncio +async def test_render_to_generate_roundtrip(client, test_image): + """End-to-end: render a multimodal chat -> feed into generate -> decode. + + All preprocessing and detokenization happens in the server subprocess; + the pytest parent never imports transformers or touches torch tensors. + """ + data_url = encode_image_url(test_image, format="PNG") + + render_payload = { + "model": MODEL_NAME, + "messages": [ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": data_url}}, + { + "type": "text", + "text": "What color is this image? Answer in one word.", + }, + ], + } + ], + } + + render_resp = await client.post(RENDER_ENDPOINT, json=render_payload) + render_resp.raise_for_status() + render_data = render_resp.json() + + # Validate render output structure: keys exist and values are non-empty + # and well-typed. + assert "token_ids" in render_data + assert isinstance(render_data["token_ids"], list) + assert len(render_data["token_ids"]) > 0 + assert all(isinstance(t, int) for t in render_data["token_ids"]) + + assert "features" in render_data + features = render_data["features"] + assert features is not None + assert isinstance(features, dict) + + assert "mm_hashes" in features + assert "image" in features["mm_hashes"] + image_hashes = features["mm_hashes"]["image"] + assert isinstance(image_hashes, list) + assert len(image_hashes) > 0 + assert all(isinstance(h, str) and h for h in image_hashes) + + assert "mm_placeholders" in features + assert "image" in features["mm_placeholders"] + image_placeholders = features["mm_placeholders"]["image"] + assert isinstance(image_placeholders, list) + assert len(image_placeholders) > 0 + for p in image_placeholders: + assert isinstance(p.get("offset"), int) + assert isinstance(p.get("length"), int) + assert p["length"] > 0 + + assert "kwargs_data" in features + assert "image" in features["kwargs_data"] + assert len(features["kwargs_data"]["image"]) > 0 + + # Build generate request from render output + generate_payload = render_data + generate_payload["sampling_params"] = { + "max_tokens": 10, + "temperature": 0.0, + } + + gen_resp = await client.post(GEN_ENDPOINT, json=generate_payload) + gen_resp.raise_for_status() + gen_data = gen_resp.json() + + assert "choices" in gen_data + assert isinstance(gen_data["choices"], list) + assert len(gen_data["choices"]) >= 1 + choice = gen_data["choices"][0] + assert "token_ids" in choice + assert isinstance(choice["token_ids"], list) + assert len(choice["token_ids"]) > 0 + assert all(isinstance(t, int) for t in choice["token_ids"]) + + detok_resp = await client.post( + DETOKENIZE_ENDPOINT, + json={"model": MODEL_NAME, "tokens": choice["token_ids"]}, + ) + detok_resp.raise_for_status() + detok_data = detok_resp.json() + assert "prompt" in detok_data + text = detok_data["prompt"] + assert isinstance(text, str) + assert len(text) > 0 + assert "red" in text.lower(), ( + f"Expected model to identify the red image, got: {text!r}" + ) diff --git a/tests/utils_/test_serial_utils.py b/tests/utils_/test_serial_utils.py index 42e466709c..85661657d5 100644 --- a/tests/utils_/test_serial_utils.py +++ b/tests/utils_/test_serial_utils.py @@ -7,17 +7,39 @@ from tests.models.utils import check_embeddings_close from vllm.utils.serial_utils import ( EMBED_DTYPES, ENDIANNESS, + MM_METADATA_DTYPES, EmbedDType, Endianness, + MmMetadataDType, binary2tensor, tensor2binary, ) +FLOAT_EMBED_DTYPES = tuple(EMBED_DTYPES.keys()) +INTEGER_EMBED_DTYPES = tuple(MM_METADATA_DTYPES.keys()) + + +def _build_integer_tensor( + embed_dtype: MmMetadataDType, shape: tuple[int, ...] +) -> torch.Tensor: + torch_dtype = MM_METADATA_DTYPES[embed_dtype].torch_dtype + + if torch_dtype is torch.bool: + return torch.randint(0, 2, shape, dtype=torch.int32).to(torch.bool) + if torch_dtype is torch.uint8: + return torch.randint(0, 256, shape, dtype=torch.uint8) + if torch_dtype is torch.int32: + return torch.randint(-(2**20), 2**20, shape, dtype=torch.int32) + if torch_dtype is torch.int64: + return torch.randint(-(2**62), 2**62, shape, dtype=torch.int64) + + raise AssertionError(f"Unsupported non-floating embed dtype: {embed_dtype}") + @pytest.mark.parametrize("endianness", ENDIANNESS) -@pytest.mark.parametrize("embed_dtype", EMBED_DTYPES.keys()) +@pytest.mark.parametrize("embed_dtype", FLOAT_EMBED_DTYPES) @torch.inference_mode() -def test_encode_and_decode(embed_dtype: EmbedDType, endianness: Endianness): +def test_encode_and_decode_floats(embed_dtype: EmbedDType, endianness: Endianness): for i in range(10): tensor = torch.rand(2, 3, 5, 7, 11, 13, device="cpu", dtype=torch.float32) shape = tensor.shape @@ -40,3 +62,20 @@ def test_encode_and_decode(embed_dtype: EmbedDType, endianness: Endianness): name_1="new", tol=1e-2, ) + + +@pytest.mark.parametrize("endianness", ENDIANNESS) +@pytest.mark.parametrize("embed_dtype", INTEGER_EMBED_DTYPES) +@torch.inference_mode() +def test_encode_and_decode_integers( + embed_dtype: MmMetadataDType, endianness: Endianness +): + shape = (2, 3, 5, 7, 11, 13) + + for i in range(10): + tensor = _build_integer_tensor(embed_dtype, shape) + binary = tensor2binary(tensor, embed_dtype, endianness) + new_tensor = binary2tensor(binary, shape, embed_dtype, endianness) + + assert new_tensor.dtype == MM_METADATA_DTYPES[embed_dtype].torch_dtype + torch.testing.assert_close(tensor, new_tensor, atol=0, rtol=0) diff --git a/vllm/entrypoints/serve/disagg/mm_serde.py b/vllm/entrypoints/serve/disagg/mm_serde.py new file mode 100644 index 0000000000..60a3560ad7 --- /dev/null +++ b/vllm/entrypoints/serve/disagg/mm_serde.py @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Encode/decode utilities for multimodal tensors and field metadata +over JSON/HTTP, used by the disaggregated generate endpoint.""" + +from __future__ import annotations + +import pybase64 + +from vllm.multimodal.inputs import MultiModalKwargsItem +from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder + +_encoder = MsgpackEncoder(size_threshold=2**62) # force all tensors inline +_decoder = MsgpackDecoder(t=MultiModalKwargsItem) + + +def encode_mm_kwargs_item(item: MultiModalKwargsItem) -> str: + """Serialize a MultiModalKwargsItem to a base64 string.""" + bufs = _encoder.encode(item) + assert len(bufs) == 1, "All tensors should be inline" + return pybase64.b64encode(bufs[0]).decode("ascii") + + +def decode_mm_kwargs_item(data: str) -> MultiModalKwargsItem: + """Deserialize a base64 string back to a MultiModalKwargsItem.""" + raw = pybase64.b64decode(data) + return _decoder.decode(raw) diff --git a/vllm/entrypoints/serve/disagg/protocol.py b/vllm/entrypoints/serve/disagg/protocol.py index 345992d3b0..6336968246 100644 --- a/vllm/entrypoints/serve/disagg/protocol.py +++ b/vllm/entrypoints/serve/disagg/protocol.py @@ -35,14 +35,6 @@ class MultiModalFeatures(BaseModel): Carries hashes (for cache lookup / identification) and placeholder positions so the downstream `/generate` service knows *where* in the token sequence each multimodal item lives. - - Note: - Phase 1 — metadata only. - Phase 2 should add `mm_kwargs` (processed tensor data) using a - binary transport so the ``/generate` side can skip re-processing. - The `/generate` endpoint must also be updated to inject these - features into `EngineInput` before passing to - `InputProcessor.process_inputs`. """ mm_hashes: dict[str, list[str]] @@ -51,6 +43,15 @@ class MultiModalFeatures(BaseModel): mm_placeholders: dict[str, list[PlaceholderRangeInfo]] """Per-modality placeholder ranges in the token sequence.""" + kwargs_data: dict[str, list[str | None]] | None = None + """Per-modality serialized tensor data. + + Each value is a list parallel to ``mm_hashes[modality]``. A ``str`` + entry is a base64-encoded ``MultiModalKwargsItem``; ``None`` means + the item should be resolved from cache. The entire field is + ``None`` for metadata-only (cache-hit) responses. + """ + class GenerateRequest(BaseModel): request_id: str = Field( diff --git a/vllm/entrypoints/serve/disagg/serving.py b/vllm/entrypoints/serve/disagg/serving.py index 14ba85ecf8..d510125fd0 100644 --- a/vllm/entrypoints/serve/disagg/serving.py +++ b/vllm/entrypoints/serve/disagg/serving.py @@ -25,6 +25,7 @@ from vllm.entrypoints.openai.engine.protocol import ( ) from vllm.entrypoints.openai.engine.serving import OpenAIServing, clamp_prompt_logprobs from vllm.entrypoints.openai.models.serving import OpenAIServingModels +from vllm.entrypoints.serve.disagg.mm_serde import decode_mm_kwargs_item from vllm.entrypoints.serve.disagg.protocol import ( GenerateRequest, GenerateResponse, @@ -34,8 +35,14 @@ from vllm.entrypoints.serve.disagg.protocol import ( ) from vllm.entrypoints.serve.render.serving import OpenAIServingRender from vllm.entrypoints.utils import should_include_usage +from vllm.inputs import EngineInput, mm_input from vllm.logger import init_logger from vllm.logprobs import Logprob +from vllm.multimodal.inputs import ( + MultiModalKwargsItem, + MultiModalKwargsItems, + PlaceholderRange, +) from vllm.outputs import RequestOutput from vllm.sampling_params import RequestOutputKind, SamplingParams from vllm.utils.collection_utils import as_list @@ -103,11 +110,42 @@ class ServingTokens(OpenAIServing): if raw_request: raw_request.state.request_metadata = request_metadata - (engine_input,) = await self.openai_serving_render.preprocess_completion( - request, - prompt_input=request.token_ids, - prompt_embeds=None, - ) + engine_input: EngineInput + if features := request.features: + # Convert PlaceholderRangeInfo → PlaceholderRange per modality. + mm_placeholders: dict[str, list[PlaceholderRange]] = { + modality: [ + PlaceholderRange(offset=p.offset, length=p.length) for p in ranges + ] + for modality, ranges in features.mm_placeholders.items() + } + + # Deserialize tensor data when present; None → cache hit. + mm_kwargs: dict[str, list[MultiModalKwargsItem | None]] = {} + if features.kwargs_data is not None: + for modality, items in features.kwargs_data.items(): + mm_kwargs[modality] = [ + decode_mm_kwargs_item(item) if item is not None else None + for item in items + ] + else: + for modality, hashes in features.mm_hashes.items(): + mm_kwargs[modality] = [None] * len(hashes) + + engine_input = mm_input( + prompt_token_ids=request.token_ids, + mm_kwargs=MultiModalKwargsItems(mm_kwargs), + mm_hashes=features.mm_hashes, + mm_placeholders=mm_placeholders, + cache_salt=request.cache_salt, + ) + else: + (engine_input,) = await self.openai_serving_render.preprocess_completion( + request, + prompt_input=request.token_ids, + prompt_embeds=None, + skip_mm_cache=True, + ) # Schedule the request and get the result generator. result_generator: AsyncGenerator[RequestOutput, None] | None = None diff --git a/vllm/entrypoints/serve/render/serving.py b/vllm/entrypoints/serve/render/serving.py index 99b6cb47cb..3cbf3cc90c 100644 --- a/vllm/entrypoints/serve/render/serving.py +++ b/vllm/entrypoints/serve/render/serving.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from collections.abc import Sequence from http import HTTPStatus -from typing import Any +from typing import Any, cast from openai_harmony import Message as OpenAIMessage @@ -25,6 +25,7 @@ from vllm.entrypoints.openai.parser.harmony_utils import ( render_for_completion, ) from vllm.entrypoints.openai.responses.protocol import ResponsesRequest +from vllm.entrypoints.serve.disagg.mm_serde import encode_mm_kwargs_item from vllm.entrypoints.serve.disagg.protocol import ( GenerateRequest, MultiModalFeatures, @@ -37,6 +38,7 @@ from vllm.entrypoints.utils import ( from vllm.inputs import ( EngineInput, MultiModalHashes, + MultiModalInput, MultiModalPlaceholders, PromptType, SingletonPrompt, @@ -251,6 +253,7 @@ class OpenAIServingRender: default_template_kwargs=self.default_chat_template_kwargs, tool_dicts=tool_dicts, tool_parser=tool_parser, + skip_mm_cache=True, reasoning_parser=self.reasoning_parser, ) else: @@ -342,6 +345,7 @@ class OpenAIServingRender: request, prompt_input=request.prompt, prompt_embeds=request.prompt_embeds, + skip_mm_cache=True, ) return engine_inputs @@ -357,9 +361,10 @@ class OpenAIServingRender: if engine_input.get("type") != "multimodal": return None - # At this point engine_input is a MultiModalInputs TypedDict. - mm_hashes: MultiModalHashes = engine_input["mm_hashes"] # type: ignore[typeddict-item] - raw_placeholders: MultiModalPlaceholders = engine_input["mm_placeholders"] # type: ignore[typeddict-item] + # At this point engine_input is a MultiModalInput TypedDict. + mm_engine_input = cast(MultiModalInput, engine_input) + mm_hashes: MultiModalHashes = mm_engine_input["mm_hashes"] + raw_placeholders: MultiModalPlaceholders = mm_engine_input["mm_placeholders"] mm_placeholders = { modality: [ @@ -368,9 +373,20 @@ class OpenAIServingRender: for modality, ranges in raw_placeholders.items() } + # Serialize tensor data per modality. + kwargs_data: dict[str, list[str | None]] | None = None + if raw_mm_kwargs := mm_engine_input.get("mm_kwargs"): + kwargs_data = {} + for modality, items in raw_mm_kwargs.items(): + kwargs_data[modality] = [ + encode_mm_kwargs_item(item) if item is not None else None + for item in items + ] + return MultiModalFeatures( mm_hashes=mm_hashes, mm_placeholders=mm_placeholders, + kwargs_data=kwargs_data, ) def _make_request_with_harmony( diff --git a/vllm/utils/serial_utils.py b/vllm/utils/serial_utils.py index 596a719351..5fde5ac710 100644 --- a/vllm/utils/serial_utils.py +++ b/vllm/utils/serial_utils.py @@ -27,6 +27,7 @@ class DTypeInfo: EmbedDType = Literal["float32", "float16", "bfloat16", "fp8_e4m3", "fp8_e5m2"] +MmMetadataDType = Literal["int32", "int64", "uint8", "bool"] Endianness = Literal["native", "big", "little"] EncodingFormat = Literal["float", "base64", "bytes", "bytes_only"] @@ -42,6 +43,15 @@ EMBED_DTYPES: Mapping[EmbedDType, DTypeInfo] = { "fp8_e4m3": DTypeInfo(torch.float8_e4m3fn, torch.uint8, np.uint8), "fp8_e5m2": DTypeInfo(torch.float8_e5m2, torch.uint8, np.uint8), } +MM_METADATA_DTYPES: Mapping[MmMetadataDType, DTypeInfo] = { + "int32": DTypeInfo(torch.int32, torch.int32, np.int32), + "int64": DTypeInfo(torch.int64, torch.int64, np.int64), + "uint8": DTypeInfo(torch.uint8, torch.uint8, np.uint8), + "bool": DTypeInfo(torch.bool, torch.uint8, np.uint8), +} +_ALL_SERIAL_DTYPES: Mapping[str, DTypeInfo] = { + k: v for d in (EMBED_DTYPES, MM_METADATA_DTYPES) for k, v in d.items() +} ENDIANNESS: tuple[Endianness, ...] = get_args(Endianness) @@ -56,14 +66,14 @@ def tensor2base64(x: torch.Tensor) -> str: def tensor2binary( tensor: torch.Tensor, - embed_dtype: EmbedDType, + embed_dtype: "EmbedDType | MmMetadataDType", endianness: Endianness, ) -> bytes: assert isinstance(tensor, torch.Tensor) - assert embed_dtype in EMBED_DTYPES + assert embed_dtype in _ALL_SERIAL_DTYPES assert endianness in ENDIANNESS - dtype_info = EMBED_DTYPES[embed_dtype] + dtype_info = _ALL_SERIAL_DTYPES[embed_dtype] np_array = ( tensor.to(dtype_info.torch_dtype) @@ -82,13 +92,13 @@ def tensor2binary( def binary2tensor( binary: bytes, shape: tuple[int, ...], - embed_dtype: EmbedDType, + embed_dtype: "EmbedDType | MmMetadataDType", endianness: Endianness, ) -> torch.Tensor: - assert embed_dtype in EMBED_DTYPES + assert embed_dtype in _ALL_SERIAL_DTYPES assert endianness in ENDIANNESS - dtype_info = EMBED_DTYPES[embed_dtype] + dtype_info = _ALL_SERIAL_DTYPES[embed_dtype] np_array = np.frombuffer(binary, dtype=dtype_info.numpy_view_dtype).reshape(shape)