[Frontend] Expose multimodal metadata for disaggregated prefill (#54659)

Signed-off-by: zhouyou9505 <zhouyou9505@gmail.com>
Co-authored-by: Cursor Grok 4.6 <noreply@cursor.com>
This commit is contained in:
zhouyou
2026-09-06 14:46:52 +00:00
committed by GitHub
co-authored by Cursor Grok 4.6
parent 808f8cd3ac
commit 52358e6e19
7 changed files with 605 additions and 100 deletions
@@ -6,21 +6,30 @@ token_in_token_out generate endpoint.
"""
import torch
from pydantic import ValidationError
from vllm.entrypoints.scale_out.token_in_token_out.mm_features import (
extract_mm_features,
merge_mm_kwargs_items,
mm_kwargs_from_features,
)
from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import (
decode_mm_kwargs_item,
encode_mm_kwargs_item,
)
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
GenerateRequest,
MultiModalFeatures,
PlaceholderRangeInfo,
)
from vllm.inputs import mm_input
from vllm.multimodal.inputs import (
MultiModalBatchedField,
MultiModalFieldElem,
MultiModalFlatField,
MultiModalKwargsItem,
MultiModalSharedField,
PlaceholderRange,
)
@@ -112,3 +121,208 @@ def test_mm_features_with_kwargs_data():
decoded = decode_mm_kwargs_item(features2.kwargs_data["image"][0])
assert torch.equal(elem.data, decoded["pixel_values"].data)
def _image_engine_input(
*,
pixel_keep_on_cpu: bool = False,
grid_keep_on_cpu: bool = True,
) -> tuple[object, MultiModalFieldElem, MultiModalFieldElem]:
pixel_values = MultiModalFieldElem(
data=torch.randn(5, 3, dtype=torch.float32),
field=MultiModalBatchedField(keep_on_cpu=pixel_keep_on_cpu),
)
image_grid_thw = MultiModalFieldElem(
data=torch.tensor([[1, 24, 24]], dtype=torch.int32),
field=MultiModalBatchedField(keep_on_cpu=grid_keep_on_cpu),
)
item = MultiModalKwargsItem(
{"pixel_values": pixel_values, "image_grid_thw": image_grid_thw}
)
engine_input = mm_input(
prompt_token_ids=[1, 2, 3],
mm_kwargs={"image": [item]},
mm_hashes={"image": ["abc123"]},
mm_placeholders={"image": [PlaceholderRange(offset=0, length=1)]},
)
return engine_input, pixel_values, image_grid_thw
def test_render_extracts_metadata_fields_separately():
"""Render metadata contains CPU fields but excludes encoder inputs."""
engine_input, pixel_values, image_grid_thw = _image_engine_input()
features = extract_mm_features(engine_input)
assert features is not None
assert features.mm_metadata is not None
features = MultiModalFeatures.model_validate_json(features.model_dump_json())
assert features.mm_metadata is not None
metadata = decode_mm_kwargs_item(features.mm_metadata["image"][0])
assert set(metadata) == {"image_grid_thw"}
assert torch.equal(metadata["image_grid_thw"].data, image_grid_thw.data)
assert features.kwargs_data is not None
full_data = decode_mm_kwargs_item(features.kwargs_data["image"][0])
assert set(full_data) == {"pixel_values", "image_grid_thw"}
assert torch.equal(full_data["pixel_values"].data, pixel_values.data)
def test_extract_includes_declared_placeholder_metadata_fields():
"""EC placeholder metadata is kept even when keep_on_cpu is unset."""
engine_input, _pixel_values, image_grid_thw = _image_engine_input(
grid_keep_on_cpu=False
)
features = extract_mm_features(
engine_input,
metadata_fields_for=lambda modality: (
{"image_grid_thw"} if modality == "image" else set()
),
)
assert features is not None
assert features.mm_metadata is not None
metadata = decode_mm_kwargs_item(features.mm_metadata["image"][0])
assert set(metadata) == {"image_grid_thw"}
assert torch.equal(metadata["image_grid_thw"].data, image_grid_thw.data)
def test_legacy_kwargs_only_generate_keeps_full_payload():
"""Old clients that omit mm_metadata still reconstruct full kwargs."""
engine_input, pixel_values, image_grid_thw = _image_engine_input()
rendered = extract_mm_features(engine_input)
assert rendered is not None
assert rendered.kwargs_data is not None
features = MultiModalFeatures(
mm_hashes=rendered.mm_hashes,
mm_placeholders=rendered.mm_placeholders,
kwargs_data=rendered.kwargs_data,
)
merged = mm_kwargs_from_features(features)
item = merged["image"][0]
assert item is not None
assert set(item) == {"pixel_values", "image_grid_thw"}
assert torch.equal(item["pixel_values"].data, pixel_values.data)
assert torch.equal(item["image_grid_thw"].data, image_grid_thw.data)
def test_metadata_only_generate_requires_ec_transfer_params():
engine_input, _, _ = _image_engine_input()
rendered = extract_mm_features(engine_input)
assert rendered is not None
payload = {
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 1},
"features": {
"mm_hashes": rendered.mm_hashes,
"mm_placeholders": {
modality: [p.model_dump() for p in ranges]
for modality, ranges in rendered.mm_placeholders.items()
},
"mm_metadata": rendered.mm_metadata,
},
}
try:
GenerateRequest.model_validate(payload)
raise AssertionError("expected metadata-only request without EC to fail")
except ValidationError as exc:
assert "ec_transfer_params" in str(exc)
payload["ec_transfer_params"] = {"image-hash": {"peer_host": "10.0.0.1"}}
request = GenerateRequest.model_validate(payload)
assert request.features is not None
assert request.features.kwargs_data is None
merged = mm_kwargs_from_features(request.features)
item = merged["image"][0]
assert item is not None
assert set(item) == {"image_grid_thw"}
def test_kwargs_and_metadata_generate_does_not_require_ec():
engine_input, pixel_values, image_grid_thw = _image_engine_input()
rendered = extract_mm_features(engine_input)
assert rendered is not None
request = GenerateRequest.model_validate(
{
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 1},
"features": rendered.model_dump(),
}
)
assert request.features is not None
merged = mm_kwargs_from_features(request.features)
item = merged["image"][0]
assert item is not None
assert set(item) == {"pixel_values", "image_grid_thw"}
assert torch.equal(item["pixel_values"].data, pixel_values.data)
assert torch.equal(item["image_grid_thw"].data, image_grid_thw.data)
def test_cache_hit_null_kwargs_keeps_metadata_slot():
pixel_values = MultiModalFieldElem(
data=torch.randn(5, 3, dtype=torch.float32),
field=MultiModalBatchedField(),
)
image_grid_thw = MultiModalFieldElem(
data=torch.tensor([[1, 24, 24]], dtype=torch.int32),
field=MultiModalBatchedField(keep_on_cpu=True),
)
item = MultiModalKwargsItem(
{"pixel_values": pixel_values, "image_grid_thw": image_grid_thw}
)
engine_input = mm_input(
prompt_token_ids=[1, 2, 3, 4],
mm_kwargs={"image": [item, None]},
mm_hashes={"image": ["hit", "cached"]},
mm_placeholders={
"image": [
PlaceholderRange(offset=0, length=1),
PlaceholderRange(offset=2, length=1),
]
},
)
features = extract_mm_features(engine_input)
assert features is not None
assert features.kwargs_data is not None
assert features.mm_metadata is not None
assert features.kwargs_data["image"][1] is None
assert features.mm_metadata["image"][1] is None
assert len(features.mm_metadata["image"]) == 2
metadata = decode_mm_kwargs_item(features.mm_metadata["image"][0])
assert torch.equal(metadata["image_grid_thw"].data, image_grid_thw.data)
def test_features_alignment_rejects_length_mismatch():
try:
MultiModalFeatures(
mm_hashes={"image": ["a", "b"]},
mm_placeholders={"image": [PlaceholderRangeInfo(offset=0, length=1)]},
)
except ValidationError:
return
raise AssertionError("expected placeholder length mismatch to fail")
def test_metadata_can_replace_or_extend_full_mm_data():
"""Metadata-only requests retain the fields required by the model."""
metadata_elem = MultiModalFieldElem(
data=torch.tensor([[1, 24, 24]], dtype=torch.int32),
field=MultiModalBatchedField(keep_on_cpu=True),
)
metadata_item = MultiModalKwargsItem({"image_grid_thw": metadata_elem})
pixel_elem = MultiModalFieldElem(
data=torch.randn(5, 3),
field=MultiModalBatchedField(),
)
full_item = MultiModalKwargsItem(
{"pixel_values": pixel_elem, "image_grid_thw": metadata_elem}
)
assert merge_mm_kwargs_items(None, metadata_item) == metadata_item
merged = merge_mm_kwargs_items(full_item, metadata_item)
assert merged is not None
assert set(merged) == {"pixel_values", "image_grid_thw"}