Validate scale-out multimodal data before engine handoff (#51898)
Signed-off-by: Clinton Thomas <1033162+KernelClint@users.noreply.github.com> Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com>
This commit is contained in:
co-authored by
Lucas Bourtoule
parent
f2e2936f91
commit
1970f3ed4b
@@ -9,7 +9,13 @@ fail loudly if the validator semantics ever drift.
|
||||
|
||||
import json
|
||||
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
GenerateRequest,
|
||||
MultiModalFeatures,
|
||||
PlaceholderRangeInfo,
|
||||
)
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
|
||||
@@ -68,3 +74,39 @@ def test_internal_instance_construction_treats_all_as_provided():
|
||||
assert req.is_sampling_param_provided("temperature")
|
||||
# And keys we never touched should also count as provided in this path.
|
||||
assert req.is_sampling_param_provided("top_p")
|
||||
|
||||
|
||||
def test_multimodal_features_reject_mismatched_parallel_fields():
|
||||
with pytest.raises(ValueError, match="same length"):
|
||||
MultiModalFeatures(
|
||||
mm_hashes={"image": ["a", "b"]},
|
||||
mm_placeholders={"image": [PlaceholderRangeInfo(offset=0, length=1)]},
|
||||
kwargs_data={"image": ["encoded"]},
|
||||
)
|
||||
|
||||
|
||||
def test_multimodal_features_reject_overlapping_placeholders():
|
||||
with pytest.raises(ValueError, match="non-overlapping"):
|
||||
MultiModalFeatures(
|
||||
mm_hashes={"image": ["a", "b"]},
|
||||
mm_placeholders={
|
||||
"image": [
|
||||
PlaceholderRangeInfo(offset=1, length=2),
|
||||
PlaceholderRangeInfo(offset=2, length=2),
|
||||
]
|
||||
},
|
||||
kwargs_data={"image": ["a", "b"]},
|
||||
)
|
||||
|
||||
|
||||
def test_generate_request_rejects_placeholder_outside_prompt():
|
||||
with pytest.raises(ValueError, match="within the token_ids sequence"):
|
||||
GenerateRequest(
|
||||
token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(),
|
||||
features=MultiModalFeatures(
|
||||
mm_hashes={"image": ["a"]},
|
||||
mm_placeholders={"image": [PlaceholderRangeInfo(offset=2, length=2)]},
|
||||
kwargs_data={"image": ["encoded"]},
|
||||
),
|
||||
)
|
||||
|
||||
@@ -33,10 +33,10 @@ from vllm.utils import random_uuid
|
||||
class PlaceholderRangeInfo(BaseModel):
|
||||
"""Serializable placeholder location for a single multi-modal item."""
|
||||
|
||||
offset: int
|
||||
offset: int = Field(ge=0)
|
||||
"""Start index of the placeholder tokens in the prompt."""
|
||||
|
||||
length: int
|
||||
length: int = Field(gt=0)
|
||||
"""Number of placeholder tokens."""
|
||||
|
||||
# TODO: add ``is_embed: list[bool] | None`` once the /generate side
|
||||
@@ -67,6 +67,47 @@ class MultiModalFeatures(BaseModel):
|
||||
``None`` for metadata-only (cache-hit) responses.
|
||||
"""
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_parallel_fields(self) -> "MultiModalFeatures":
|
||||
modalities = set(self.mm_hashes)
|
||||
if set(self.mm_placeholders) != modalities:
|
||||
raise ValueError(
|
||||
"mm_hashes and mm_placeholders must use the same modalities"
|
||||
)
|
||||
if self.kwargs_data is not None and set(self.kwargs_data) != modalities:
|
||||
raise ValueError("kwargs_data must use the same modalities as mm_hashes")
|
||||
|
||||
flattened_ranges: list[tuple[int, int]] = []
|
||||
for modality in modalities:
|
||||
num_hashes = len(self.mm_hashes[modality])
|
||||
num_placeholders = len(self.mm_placeholders[modality])
|
||||
if num_hashes != num_placeholders:
|
||||
raise ValueError(
|
||||
f"{modality} mm_hashes and mm_placeholders must have "
|
||||
"the same length"
|
||||
)
|
||||
if (
|
||||
self.kwargs_data is not None
|
||||
and len(self.kwargs_data[modality]) != num_hashes
|
||||
):
|
||||
raise ValueError(
|
||||
f"{modality} kwargs_data and mm_hashes must have the same length"
|
||||
)
|
||||
flattened_ranges.extend(
|
||||
(placeholder.offset, placeholder.offset + placeholder.length)
|
||||
for placeholder in self.mm_placeholders[modality]
|
||||
)
|
||||
|
||||
flattened_ranges.sort()
|
||||
for (offset, end), (next_offset, _) in zip(
|
||||
flattened_ranges, flattened_ranges[1:]
|
||||
):
|
||||
if next_offset < end:
|
||||
raise ValueError(
|
||||
"mm_placeholders must be globally non-overlapping and sorted"
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class GenerateRequest(BaseModel):
|
||||
request_id: str = Field(
|
||||
@@ -184,6 +225,20 @@ class GenerateRequest(BaseModel):
|
||||
validate_cache_salt(data.get("cache_salt"))
|
||||
return data
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_multimodal_feature_bounds(self) -> "GenerateRequest":
|
||||
if self.features is None:
|
||||
return self
|
||||
|
||||
prompt_len = len(self.token_ids)
|
||||
for ranges in self.features.mm_placeholders.values():
|
||||
for placeholder in ranges:
|
||||
if placeholder.offset + placeholder.length > prompt_len:
|
||||
raise ValueError(
|
||||
"mm_placeholders must remain within the token_ids sequence"
|
||||
)
|
||||
return self
|
||||
|
||||
def is_sampling_param_provided(self, name: str) -> bool:
|
||||
"""Whether the caller explicitly set ``sampling_params.<name>``.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user