[Frontend] Return prompt metadata from /inference/v1/generate (#53187)

This commit is contained in:
aoshen02
2026-09-16 15:29:07 -07:00
committed by GitHub
parent 6c3d9816af
commit 0983aef8da
9 changed files with 763 additions and 40 deletions
@@ -12,6 +12,9 @@ from vllm.config.multimodal import MultiModalConfig
from vllm.entrypoints.generate.base.protocol import StreamOptions
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.scale_out.token_in_token_out.mm_features import (
placeholder_ranges_from_engine_input,
)
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
GenerateRequest,
GenerateResponse,
@@ -19,6 +22,7 @@ from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
from vllm.exceptions import GenerationError
from vllm.logprobs import Logprob
from vllm.multimodal.inputs import PlaceholderRange
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.renderers import renderer_from_config
from vllm.renderers.online_renderer import OnlineRenderer
@@ -176,6 +180,29 @@ def _parse_sse_chunks(chunks: list[str]) -> list[Any]:
return parsed
def test_placeholder_ranges_from_engine_input():
engine_input: Any = {
"type": "multimodal",
"mm_placeholders": {
"image": [
PlaceholderRange(offset=1, length=5),
PlaceholderRange(offset=8, length=2),
]
},
}
placeholders = placeholder_ranges_from_engine_input(engine_input)
assert placeholders is not None
assert [item.model_dump() for item in placeholders["image"]] == [
{"offset": 1, "length": 5},
{"offset": 8, "length": 2},
]
text_only: Any = {"type": "token", "prompt_token_ids": [1, 2, 3]}
assert placeholder_ranges_from_engine_input(text_only) is None
@pytest.mark.asyncio
async def test_serve_tokens_skips_mm_cache_for_remote_engine_execution():
engine = _mock_engine()
@@ -294,6 +321,336 @@ async def test_stream_basic():
assert data_chunks[2]["choices"][0]["finish_reason"] == "stop"
@pytest.mark.asyncio
async def test_non_stream_returns_final_prompt_token_ids_when_requested():
engine = _mock_engine()
final_prompt_token_ids = [1, 2, 2, 2, 3]
async def mock_generate(*args, **kwargs):
yield _make_request_output(
"req-1",
token_ids=[10],
finish_reason="stop",
finished=True,
prompt_token_ids=final_prompt_token_ids,
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=1),
model=MODEL_NAME,
return_token_ids=True,
)
response = await serving.serve_tokens(request)
assert isinstance(response, GenerateResponse)
assert response.prompt_token_ids == final_prompt_token_ids
@pytest.mark.asyncio
async def test_stream_returns_prompt_token_ids_only_once_when_requested():
engine = _mock_engine()
final_prompt_token_ids = [1, 2, 2, 2, 3]
async def mock_generate(*args, **kwargs):
yield _make_request_output(
"req-1", token_ids=[10], prompt_token_ids=final_prompt_token_ids
)
yield _make_request_output(
"req-1",
token_ids=[20],
finish_reason="stop",
finished=True,
prompt_token_ids=final_prompt_token_ids,
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=2),
model=MODEL_NAME,
stream=True,
return_token_ids=True,
)
response = await serving.serve_tokens(request)
parsed = _parse_sse_chunks([chunk async for chunk in response])
data_chunks = [c for c in parsed if isinstance(c, dict) and c.get("choices")]
assert data_chunks[0]["prompt_token_ids"] == final_prompt_token_ids
assert "prompt_token_ids" not in data_chunks[1]
MM_ENGINE_INPUT: Any = {
"type": "multimodal",
"prompt_token_ids": [1, 2, 2, 2, 3],
"mm_placeholders": {"image": [PlaceholderRange(offset=1, length=3)]},
}
MM_PLACEHOLDERS = {"image": [{"offset": 1, "length": 3}]}
def _build_mm_serving_tokens(engine: AsyncLLM) -> ServingTokens:
serving = _build_serving_tokens(engine)
async def _fake_preprocess(*args, **kwargs):
return [MM_ENGINE_INPUT]
serving.online_renderer.preprocess_completion = AsyncMock(
side_effect=_fake_preprocess
)
return serving
@pytest.mark.asyncio
@pytest.mark.parametrize("return_token_ids", [False, True])
async def test_non_stream_prompt_metadata_requires_return_token_ids(
return_token_ids: bool,
):
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output(
"req-1", token_ids=[10], finish_reason="stop", finished=True
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_mm_serving_tokens(engine)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=1),
model=MODEL_NAME,
return_token_ids=return_token_ids,
)
response = await serving.serve_tokens(request)
assert isinstance(response, GenerateResponse)
if return_token_ids:
assert response.prompt_token_ids == [1, 2, 3]
assert response.model_dump()["mm_placeholders"] == MM_PLACEHOLDERS
else:
assert response.prompt_token_ids is None
assert response.mm_placeholders is None
assert request._response_mm_placeholders is None
@pytest.mark.asyncio
async def test_non_stream_text_only_prompt_has_no_mm_placeholders():
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output(
"req-1", token_ids=[10], finish_reason="stop", finished=True
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=1),
model=MODEL_NAME,
return_token_ids=True,
)
response = await serving.serve_tokens(request)
assert isinstance(response, GenerateResponse)
assert response.prompt_token_ids == [1, 2, 3]
assert response.mm_placeholders is None
@pytest.mark.asyncio
async def test_non_stream_encoder_decoder_omits_mm_placeholders():
engine = _mock_engine()
engine.model_config.is_encoder_decoder = True
async def mock_generate(*args, **kwargs):
yield _make_request_output(
"req-1", token_ids=[10], finish_reason="stop", finished=True
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_mm_serving_tokens(engine)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=1),
model=MODEL_NAME,
return_token_ids=True,
)
response = await serving.serve_tokens(request)
assert isinstance(response, GenerateResponse)
assert response.prompt_token_ids == [1, 2, 3]
assert response.mm_placeholders is None
@pytest.mark.asyncio
async def test_stream_prompt_metadata_lands_on_first_emitted_chunk():
"""An empty first output is skipped; metadata goes on the first real chunk,
once, and never on the sibling chunk of the same iteration (n=2)."""
engine = _mock_engine()
def _two_outputs(tokens0: list[int], tokens1: list[int], **kwargs):
res = _make_request_output("req-1", token_ids=tokens0, **kwargs)
res.outputs.append(
CompletionOutput(
index=1,
text="",
token_ids=tokens1,
cumulative_logprob=None,
logprobs=None,
finish_reason=kwargs.get("finish_reason"),
)
)
return res
async def mock_generate(*args, **kwargs):
yield _two_outputs([], [])
yield _two_outputs([10], [11])
yield _two_outputs([20], [21], finish_reason="stop", finished=True)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_mm_serving_tokens(engine)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=2, n=2),
model=MODEL_NAME,
stream=True,
return_token_ids=True,
)
response = await serving.serve_tokens(request)
parsed = _parse_sse_chunks([chunk async for chunk in response])
data_chunks = [c for c in parsed if isinstance(c, dict) and c.get("choices")]
assert [c["choices"][0]["token_ids"] for c in data_chunks] == [
[10],
[11],
[20],
[21],
]
assert data_chunks[0]["prompt_token_ids"] == [1, 2, 3]
assert data_chunks[0]["mm_placeholders"] == MM_PLACEHOLDERS
for chunk in data_chunks[1:]:
assert "prompt_token_ids" not in chunk
assert "mm_placeholders" not in chunk
@pytest.mark.asyncio
async def test_stream_omits_prompt_metadata_by_default():
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output(
"req-1", token_ids=[10], finish_reason="stop", finished=True
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_mm_serving_tokens(engine)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=1),
model=MODEL_NAME,
stream=True,
)
response = await serving.serve_tokens(request)
parsed = _parse_sse_chunks([chunk async for chunk in response])
data_chunks = [c for c in parsed if isinstance(c, dict) and c.get("choices")]
assert data_chunks
for chunk in data_chunks:
assert "prompt_token_ids" not in chunk
assert "mm_placeholders" not in chunk
@pytest.mark.asyncio
async def test_stream_text_only_prompt_omits_mm_placeholders_key():
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output(
"req-1", token_ids=[10], finish_reason="stop", finished=True
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=1),
model=MODEL_NAME,
stream=True,
return_token_ids=True,
)
response = await serving.serve_tokens(request)
parsed = _parse_sse_chunks([chunk async for chunk in response])
data_chunks = [c for c in parsed if isinstance(c, dict) and c.get("choices")]
assert data_chunks[0]["prompt_token_ids"] == [1, 2, 3]
assert "mm_placeholders" not in data_chunks[0]
@pytest.mark.asyncio
async def test_stream_zero_token_completion_still_delivers_prompt_metadata():
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output(
"req-1", token_ids=[], finish_reason="stop", finished=True
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_mm_serving_tokens(engine)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=1),
model=MODEL_NAME,
stream=True,
return_token_ids=True,
)
response = await serving.serve_tokens(request)
parsed = _parse_sse_chunks([chunk async for chunk in response])
data_chunks = [c for c in parsed if isinstance(c, dict) and c.get("choices")]
assert len(data_chunks) == 1
assert data_chunks[0]["choices"][0]["token_ids"] == []
assert data_chunks[0]["choices"][0]["finish_reason"] == "stop"
assert data_chunks[0]["prompt_token_ids"] == [1, 2, 3]
assert data_chunks[0]["mm_placeholders"] == MM_PLACEHOLDERS
@pytest.mark.asyncio
async def test_stream_zero_token_completion_emits_no_chunk_by_default():
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output(
"req-1", token_ids=[], finish_reason="stop", finished=True
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=1),
model=MODEL_NAME,
stream=True,
)
response = await serving.serve_tokens(request)
parsed = _parse_sse_chunks([chunk async for chunk in response])
assert not [c for c in parsed if isinstance(c, dict) and c.get("choices")]
assert parsed[-1] == "[DONE]"
@pytest.mark.asyncio
async def test_stream_error_mid_generation():
"""finish_reason='error' mid-stream yields error chunk then [DONE]."""
@@ -154,11 +154,28 @@ async def test_render_to_generate_roundtrip(client, test_image):
)
@pytest.mark.asyncio
async def test_content_parts_generates_tokens(client, test_image):
"""content_parts with raw media should produce output tokens."""
data_url = encode_image_url(test_image, format="PNG")
def _collapse_placeholders(
token_ids: list[int], mm_placeholders: dict[str, list[dict[str, int]]]
) -> list[int]:
"""Undo placeholder expansion: each expanded range becomes one token.
``/generate`` with ``content_parts`` expects unexpanded placeholders and
expands them itself, so the roundtrip must start from a collapsed prompt.
"""
ranges = sorted(
(r["offset"], r["length"]) for rs in mm_placeholders.values() for r in rs
)
collapsed: list[int] = []
pos = 0
for offset, length in ranges:
collapsed.extend(token_ids[pos:offset])
collapsed.append(token_ids[offset])
pos = offset + length
collapsed.extend(token_ids[pos:])
return collapsed
async def _render_image_prompt(client, data_url: str) -> dict:
render_resp = await client.post(
RENDER_ENDPOINT,
json={
@@ -175,14 +192,26 @@ async def test_content_parts_generates_tokens(client, test_image):
},
)
render_resp.raise_for_status()
token_ids = render_resp.json()["token_ids"]
return render_resp.json()
@pytest.mark.asyncio
async def test_content_parts_generates_tokens(client, test_image):
"""Unexpanded prompt + content_parts must expand exactly like /render."""
data_url = encode_image_url(test_image, format="PNG")
render_data = await _render_image_prompt(client, data_url)
expanded = render_data["token_ids"]
mm_placeholders = render_data["features"]["mm_placeholders"]
unexpanded = _collapse_placeholders(expanded, mm_placeholders)
assert len(unexpanded) < len(expanded)
gen_resp = await client.post(
GEN_ENDPOINT,
json={
"token_ids": token_ids,
"token_ids": unexpanded,
"content_parts": [{"type": "image_url", "url": data_url}],
"sampling_params": {"max_tokens": 10, "temperature": 0.0},
"return_token_ids": True,
},
)
gen_resp.raise_for_status()
@@ -192,6 +221,8 @@ async def test_content_parts_generates_tokens(client, test_image):
choice = gen_data["choices"][0]
assert "token_ids" in choice
assert len(choice["token_ids"]) > 0
assert gen_data["prompt_token_ids"] == expanded
assert gen_data["mm_placeholders"] == mm_placeholders
@pytest.mark.asyncio
@@ -200,33 +231,20 @@ async def test_content_parts_streaming(client, test_image):
import json as json_mod
data_url = encode_image_url(test_image, format="PNG")
render_resp = await client.post(
RENDER_ENDPOINT,
json={
"model": MODEL_NAME,
"messages": [
{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": data_url}},
{"type": "text", "text": "Describe this."},
],
}
],
},
)
render_resp.raise_for_status()
token_ids = render_resp.json()["token_ids"]
render_data = await _render_image_prompt(client, data_url)
expanded = render_data["token_ids"]
mm_placeholders = render_data["features"]["mm_placeholders"]
unexpanded = _collapse_placeholders(expanded, mm_placeholders)
async with client.stream(
"POST",
GEN_ENDPOINT,
json={
"token_ids": token_ids,
"token_ids": unexpanded,
"content_parts": [{"type": "image_url", "url": data_url}],
"sampling_params": {"max_tokens": 8, "temperature": 0.0},
"stream": True,
"return_token_ids": True,
},
) as resp:
resp.raise_for_status()
@@ -236,3 +254,9 @@ async def test_content_parts_streaming(client, test_image):
chunks.append(json_mod.loads(line[6:]))
assert len(chunks) > 0
prompt_metadata_chunks = [
chunk for chunk in chunks if chunk.get("prompt_token_ids") is not None
]
assert len(prompt_metadata_chunks) == 1
assert prompt_metadata_chunks[0]["prompt_token_ids"] == expanded
assert prompt_metadata_chunks[0]["mm_placeholders"] == mm_placeholders