From 1970f3ed4be7fa8620e4ddc4a12c36a8384cfc27 Mon Sep 17 00:00:00 2001 From: Clinton Thomas <1033162+KernelClint@users.noreply.github.com> Date: Sun, 6 Sep 2026 00:58:04 -0400 Subject: [PATCH] Validate scale-out multimodal data before engine handoff (#51898) Signed-off-by: Clinton Thomas <1033162+KernelClint@users.noreply.github.com> Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com> --- .../token_in_token_out/test_protocol.py | 44 +++++++++++++- .../scale_out/token_in_token_out/protocol.py | 59 ++++++++++++++++++- 2 files changed, 100 insertions(+), 3 deletions(-) diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_protocol.py b/tests/entrypoints/scale_out/token_in_token_out/test_protocol.py index 674ce18b7f..e51530ce45 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_protocol.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_protocol.py @@ -9,7 +9,13 @@ fail loudly if the validator semantics ever drift. import json -from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest +import pytest + +from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( + GenerateRequest, + MultiModalFeatures, + PlaceholderRangeInfo, +) from vllm.sampling_params import SamplingParams @@ -68,3 +74,39 @@ def test_internal_instance_construction_treats_all_as_provided(): assert req.is_sampling_param_provided("temperature") # And keys we never touched should also count as provided in this path. assert req.is_sampling_param_provided("top_p") + + +def test_multimodal_features_reject_mismatched_parallel_fields(): + with pytest.raises(ValueError, match="same length"): + MultiModalFeatures( + mm_hashes={"image": ["a", "b"]}, + mm_placeholders={"image": [PlaceholderRangeInfo(offset=0, length=1)]}, + kwargs_data={"image": ["encoded"]}, + ) + + +def test_multimodal_features_reject_overlapping_placeholders(): + with pytest.raises(ValueError, match="non-overlapping"): + MultiModalFeatures( + mm_hashes={"image": ["a", "b"]}, + mm_placeholders={ + "image": [ + PlaceholderRangeInfo(offset=1, length=2), + PlaceholderRangeInfo(offset=2, length=2), + ] + }, + kwargs_data={"image": ["a", "b"]}, + ) + + +def test_generate_request_rejects_placeholder_outside_prompt(): + with pytest.raises(ValueError, match="within the token_ids sequence"): + GenerateRequest( + token_ids=[1, 2, 3], + sampling_params=SamplingParams(), + features=MultiModalFeatures( + mm_hashes={"image": ["a"]}, + mm_placeholders={"image": [PlaceholderRangeInfo(offset=2, length=2)]}, + kwargs_data={"image": ["encoded"]}, + ), + ) diff --git a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py index f279d8b0dc..ecf75184e7 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py @@ -33,10 +33,10 @@ from vllm.utils import random_uuid class PlaceholderRangeInfo(BaseModel): """Serializable placeholder location for a single multi-modal item.""" - offset: int + offset: int = Field(ge=0) """Start index of the placeholder tokens in the prompt.""" - length: int + length: int = Field(gt=0) """Number of placeholder tokens.""" # TODO: add ``is_embed: list[bool] | None`` once the /generate side @@ -67,6 +67,47 @@ class MultiModalFeatures(BaseModel): ``None`` for metadata-only (cache-hit) responses. """ + @model_validator(mode="after") + def _validate_parallel_fields(self) -> "MultiModalFeatures": + modalities = set(self.mm_hashes) + if set(self.mm_placeholders) != modalities: + raise ValueError( + "mm_hashes and mm_placeholders must use the same modalities" + ) + if self.kwargs_data is not None and set(self.kwargs_data) != modalities: + raise ValueError("kwargs_data must use the same modalities as mm_hashes") + + flattened_ranges: list[tuple[int, int]] = [] + for modality in modalities: + num_hashes = len(self.mm_hashes[modality]) + num_placeholders = len(self.mm_placeholders[modality]) + if num_hashes != num_placeholders: + raise ValueError( + f"{modality} mm_hashes and mm_placeholders must have " + "the same length" + ) + if ( + self.kwargs_data is not None + and len(self.kwargs_data[modality]) != num_hashes + ): + raise ValueError( + f"{modality} kwargs_data and mm_hashes must have the same length" + ) + flattened_ranges.extend( + (placeholder.offset, placeholder.offset + placeholder.length) + for placeholder in self.mm_placeholders[modality] + ) + + flattened_ranges.sort() + for (offset, end), (next_offset, _) in zip( + flattened_ranges, flattened_ranges[1:] + ): + if next_offset < end: + raise ValueError( + "mm_placeholders must be globally non-overlapping and sorted" + ) + return self + class GenerateRequest(BaseModel): request_id: str = Field( @@ -184,6 +225,20 @@ class GenerateRequest(BaseModel): validate_cache_salt(data.get("cache_salt")) return data + @model_validator(mode="after") + def _validate_multimodal_feature_bounds(self) -> "GenerateRequest": + if self.features is None: + return self + + prompt_len = len(self.token_ids) + for ranges in self.features.mm_placeholders.values(): + for placeholder in ranges: + if placeholder.offset + placeholder.length > prompt_len: + raise ValueError( + "mm_placeholders must remain within the token_ids sequence" + ) + return self + def is_sampling_param_provided(self, name: str) -> bool: """Whether the caller explicitly set ``sampling_params.``.