[Frontend] Return prompt metadata from /inference/v1/generate (#53187)

This commit is contained in:
aoshen02
2026-09-16 15:29:07 -07:00
committed by GitHub
parent 6c3d9816af
commit 0983aef8da
9 changed files with 763 additions and 40 deletions
@@ -117,6 +117,24 @@ def mm_kwargs_from_features(
return mm_kwargs
def placeholder_ranges_from_engine_input(
engine_input: EngineInput,
) -> dict[str, list[PlaceholderRangeInfo]] | None:
"""Return per-modality placeholder ranges, or ``None`` for text-only prompts."""
if engine_input.get("type") != "multimodal":
return None
raw_placeholders: MultiModalPlaceholders = cast(MultiModalInput, engine_input)[
"mm_placeholders"
]
return {
modality: [
PlaceholderRangeInfo(offset=p.offset, length=p.length) for p in ranges
]
for modality, ranges in raw_placeholders.items()
}
def extract_mm_features(
engine_input: EngineInput,
*,
@@ -134,14 +152,8 @@ def extract_mm_features(
mm_engine_input = cast(MultiModalInput, engine_input)
mm_hashes: MultiModalHashes = mm_engine_input["mm_hashes"]
raw_placeholders: MultiModalPlaceholders = mm_engine_input["mm_placeholders"]
mm_placeholders = {
modality: [
PlaceholderRangeInfo(offset=p.offset, length=p.length) for p in ranges
]
for modality, ranges in raw_placeholders.items()
}
mm_placeholders = placeholder_ranges_from_engine_input(engine_input)
assert mm_placeholders is not None
kwargs_data: dict[str, list[str | None]] | None = None
mm_metadata: dict[str, list[str | None]] | None = None
@@ -211,6 +211,15 @@ class GenerateRequest(BaseModel):
model: str | None = None
return_token_ids: bool | None = Field(
default=None,
description=(
"If true, return the final prompt token IDs after multimodal "
"placeholder expansion, together with multimodal placeholder ranges. "
"In streaming mode, this metadata is included only in the first chunk."
),
)
stream: bool | None = False
stream_options: StreamOptions | None = None
cache_salt: str | None = Field(
@@ -255,6 +264,9 @@ class GenerateRequest(BaseModel):
# ``SamplingParams`` instance (e.g. from internal callers that have
# already resolved values), in which case all fields are considered set.
_sampling_params_provided_keys: set[str] | None = PrivateAttr(default=None)
_response_mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = (
PrivateAttr(default=None)
)
@model_validator(mode="wrap")
@classmethod
@@ -354,6 +366,8 @@ class GenerateStreamResponse(BaseModel):
)
choices: list[GenerateResponseStreamChoice]
usage: UsageInfo | None = Field(default=None)
prompt_token_ids: list[int] | None = None
mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = None
class GenerateResponse(BaseModel):
@@ -370,6 +384,8 @@ class GenerateResponse(BaseModel):
choices: list[GenerateResponseChoice]
usage: UsageInfo | None = Field(default=None)
prompt_logprobs: list[dict[int, Logprob] | None] | None = None
prompt_token_ids: list[int] | None = None
mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = None
kv_transfer_params: dict[str, Any] | None = Field(
default=None,
@@ -44,7 +44,10 @@ from vllm.sampling_params import RequestOutputKind, SamplingParams
from vllm.utils.collection_utils import as_list
from vllm.utils.serial_utils import numpy2base64
from .mm_features import mm_kwargs_from_features
from .mm_features import (
mm_kwargs_from_features,
placeholder_ranges_from_engine_input,
)
from .protocol import (
GenerateRequest,
GenerateResponse,
@@ -195,6 +198,14 @@ class ServingTokens(GenerateBaseServing):
skip_mm_cache=True,
)
# Offsets are relative to the decoder prompt, so they are not
# meaningful for encoder-decoder models.
request._response_mm_placeholders = (
placeholder_ranges_from_engine_input(engine_input)
if request.return_token_ids and not self.model_config.is_encoder_decoder
else None
)
# Schedule the request and get the result generator.
result_generator: AsyncGenerator[RequestOutput, None] | None = None
@@ -359,6 +370,10 @@ class ServingTokens(GenerateBaseServing):
choices=choices,
usage=usage,
prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
prompt_token_ids=(
final_res.prompt_token_ids if request.return_token_ids else None
),
mm_placeholders=request._response_mm_placeholders,
kv_transfer_params=final_res.kv_transfer_params,
ec_transfer_params=final_res.ec_transfer_params,
)
@@ -395,6 +410,7 @@ class ServingTokens(GenerateBaseServing):
num_prompt_tokens = 0
num_generated_tokens: list[int] = []
first_iteration = True
prompt_token_ids: list[int] | None = None
num_cached_tokens = None
sampling_params: SamplingParams = request.sampling_params
@@ -407,6 +423,8 @@ class ServingTokens(GenerateBaseServing):
if first_iteration:
if res.prompt_token_ids is not None:
num_prompt_tokens = len(res.prompt_token_ids)
if request.return_token_ids:
prompt_token_ids = res.prompt_token_ids
if res.encoder_prompt_token_ids is not None:
num_prompt_tokens += len(res.encoder_prompt_token_ids)
num_cached_tokens = res.num_cached_tokens
@@ -421,7 +439,11 @@ class ServingTokens(GenerateBaseServing):
finish_reason = output.finish_reason
self._raise_if_error(finish_reason, request_id)
if not delta_token_ids:
# Still emit a terminal empty chunk while prompt metadata
# is pending, so zero-token completions deliver it.
if not delta_token_ids and (
finish_reason is None or prompt_token_ids is None
):
continue
if sampling_params.logprobs is not None:
@@ -453,6 +475,10 @@ class ServingTokens(GenerateBaseServing):
)
],
)
if prompt_token_ids is not None:
chunk.prompt_token_ids = prompt_token_ids
chunk.mm_placeholders = request._response_mm_placeholders
prompt_token_ids = None
if include_continuous_usage:
chunk.usage = UsageInfo(
prompt_tokens=num_prompt_tokens,
@@ -460,7 +486,13 @@ class ServingTokens(GenerateBaseServing):
total_tokens=(num_prompt_tokens + num_generated_tokens[i]),
)
yield f"data: {chunk.model_dump_json()}\n\n"
# Omit absent prompt metadata (Rust skips None fields).
exclude = {
name
for name in ("prompt_token_ids", "mm_placeholders")
if getattr(chunk, name) is None
}
yield f"data: {chunk.model_dump_json(exclude=exclude)}\n\n"
total_completion_tokens = sum(num_generated_tokens)
final_usage_info = UsageInfo(