[Frontend] Return prompt metadata from /inference/v1/generate (#53187)
This commit is contained in:
@@ -117,6 +117,24 @@ def mm_kwargs_from_features(
|
||||
return mm_kwargs
|
||||
|
||||
|
||||
def placeholder_ranges_from_engine_input(
|
||||
engine_input: EngineInput,
|
||||
) -> dict[str, list[PlaceholderRangeInfo]] | None:
|
||||
"""Return per-modality placeholder ranges, or ``None`` for text-only prompts."""
|
||||
if engine_input.get("type") != "multimodal":
|
||||
return None
|
||||
|
||||
raw_placeholders: MultiModalPlaceholders = cast(MultiModalInput, engine_input)[
|
||||
"mm_placeholders"
|
||||
]
|
||||
return {
|
||||
modality: [
|
||||
PlaceholderRangeInfo(offset=p.offset, length=p.length) for p in ranges
|
||||
]
|
||||
for modality, ranges in raw_placeholders.items()
|
||||
}
|
||||
|
||||
|
||||
def extract_mm_features(
|
||||
engine_input: EngineInput,
|
||||
*,
|
||||
@@ -134,14 +152,8 @@ def extract_mm_features(
|
||||
|
||||
mm_engine_input = cast(MultiModalInput, engine_input)
|
||||
mm_hashes: MultiModalHashes = mm_engine_input["mm_hashes"]
|
||||
raw_placeholders: MultiModalPlaceholders = mm_engine_input["mm_placeholders"]
|
||||
|
||||
mm_placeholders = {
|
||||
modality: [
|
||||
PlaceholderRangeInfo(offset=p.offset, length=p.length) for p in ranges
|
||||
]
|
||||
for modality, ranges in raw_placeholders.items()
|
||||
}
|
||||
mm_placeholders = placeholder_ranges_from_engine_input(engine_input)
|
||||
assert mm_placeholders is not None
|
||||
|
||||
kwargs_data: dict[str, list[str | None]] | None = None
|
||||
mm_metadata: dict[str, list[str | None]] | None = None
|
||||
|
||||
@@ -211,6 +211,15 @@ class GenerateRequest(BaseModel):
|
||||
|
||||
model: str | None = None
|
||||
|
||||
return_token_ids: bool | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"If true, return the final prompt token IDs after multimodal "
|
||||
"placeholder expansion, together with multimodal placeholder ranges. "
|
||||
"In streaming mode, this metadata is included only in the first chunk."
|
||||
),
|
||||
)
|
||||
|
||||
stream: bool | None = False
|
||||
stream_options: StreamOptions | None = None
|
||||
cache_salt: str | None = Field(
|
||||
@@ -255,6 +264,9 @@ class GenerateRequest(BaseModel):
|
||||
# ``SamplingParams`` instance (e.g. from internal callers that have
|
||||
# already resolved values), in which case all fields are considered set.
|
||||
_sampling_params_provided_keys: set[str] | None = PrivateAttr(default=None)
|
||||
_response_mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = (
|
||||
PrivateAttr(default=None)
|
||||
)
|
||||
|
||||
@model_validator(mode="wrap")
|
||||
@classmethod
|
||||
@@ -354,6 +366,8 @@ class GenerateStreamResponse(BaseModel):
|
||||
)
|
||||
choices: list[GenerateResponseStreamChoice]
|
||||
usage: UsageInfo | None = Field(default=None)
|
||||
prompt_token_ids: list[int] | None = None
|
||||
mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = None
|
||||
|
||||
|
||||
class GenerateResponse(BaseModel):
|
||||
@@ -370,6 +384,8 @@ class GenerateResponse(BaseModel):
|
||||
choices: list[GenerateResponseChoice]
|
||||
usage: UsageInfo | None = Field(default=None)
|
||||
prompt_logprobs: list[dict[int, Logprob] | None] | None = None
|
||||
prompt_token_ids: list[int] | None = None
|
||||
mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = None
|
||||
|
||||
kv_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
|
||||
@@ -44,7 +44,10 @@ from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||
from vllm.utils.collection_utils import as_list
|
||||
from vllm.utils.serial_utils import numpy2base64
|
||||
|
||||
from .mm_features import mm_kwargs_from_features
|
||||
from .mm_features import (
|
||||
mm_kwargs_from_features,
|
||||
placeholder_ranges_from_engine_input,
|
||||
)
|
||||
from .protocol import (
|
||||
GenerateRequest,
|
||||
GenerateResponse,
|
||||
@@ -195,6 +198,14 @@ class ServingTokens(GenerateBaseServing):
|
||||
skip_mm_cache=True,
|
||||
)
|
||||
|
||||
# Offsets are relative to the decoder prompt, so they are not
|
||||
# meaningful for encoder-decoder models.
|
||||
request._response_mm_placeholders = (
|
||||
placeholder_ranges_from_engine_input(engine_input)
|
||||
if request.return_token_ids and not self.model_config.is_encoder_decoder
|
||||
else None
|
||||
)
|
||||
|
||||
# Schedule the request and get the result generator.
|
||||
result_generator: AsyncGenerator[RequestOutput, None] | None = None
|
||||
|
||||
@@ -359,6 +370,10 @@ class ServingTokens(GenerateBaseServing):
|
||||
choices=choices,
|
||||
usage=usage,
|
||||
prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
|
||||
prompt_token_ids=(
|
||||
final_res.prompt_token_ids if request.return_token_ids else None
|
||||
),
|
||||
mm_placeholders=request._response_mm_placeholders,
|
||||
kv_transfer_params=final_res.kv_transfer_params,
|
||||
ec_transfer_params=final_res.ec_transfer_params,
|
||||
)
|
||||
@@ -395,6 +410,7 @@ class ServingTokens(GenerateBaseServing):
|
||||
num_prompt_tokens = 0
|
||||
num_generated_tokens: list[int] = []
|
||||
first_iteration = True
|
||||
prompt_token_ids: list[int] | None = None
|
||||
num_cached_tokens = None
|
||||
sampling_params: SamplingParams = request.sampling_params
|
||||
|
||||
@@ -407,6 +423,8 @@ class ServingTokens(GenerateBaseServing):
|
||||
if first_iteration:
|
||||
if res.prompt_token_ids is not None:
|
||||
num_prompt_tokens = len(res.prompt_token_ids)
|
||||
if request.return_token_ids:
|
||||
prompt_token_ids = res.prompt_token_ids
|
||||
if res.encoder_prompt_token_ids is not None:
|
||||
num_prompt_tokens += len(res.encoder_prompt_token_ids)
|
||||
num_cached_tokens = res.num_cached_tokens
|
||||
@@ -421,7 +439,11 @@ class ServingTokens(GenerateBaseServing):
|
||||
finish_reason = output.finish_reason
|
||||
self._raise_if_error(finish_reason, request_id)
|
||||
|
||||
if not delta_token_ids:
|
||||
# Still emit a terminal empty chunk while prompt metadata
|
||||
# is pending, so zero-token completions deliver it.
|
||||
if not delta_token_ids and (
|
||||
finish_reason is None or prompt_token_ids is None
|
||||
):
|
||||
continue
|
||||
|
||||
if sampling_params.logprobs is not None:
|
||||
@@ -453,6 +475,10 @@ class ServingTokens(GenerateBaseServing):
|
||||
)
|
||||
],
|
||||
)
|
||||
if prompt_token_ids is not None:
|
||||
chunk.prompt_token_ids = prompt_token_ids
|
||||
chunk.mm_placeholders = request._response_mm_placeholders
|
||||
prompt_token_ids = None
|
||||
if include_continuous_usage:
|
||||
chunk.usage = UsageInfo(
|
||||
prompt_tokens=num_prompt_tokens,
|
||||
@@ -460,7 +486,13 @@ class ServingTokens(GenerateBaseServing):
|
||||
total_tokens=(num_prompt_tokens + num_generated_tokens[i]),
|
||||
)
|
||||
|
||||
yield f"data: {chunk.model_dump_json()}\n\n"
|
||||
# Omit absent prompt metadata (Rust skips None fields).
|
||||
exclude = {
|
||||
name
|
||||
for name in ("prompt_token_ids", "mm_placeholders")
|
||||
if getattr(chunk, name) is None
|
||||
}
|
||||
yield f"data: {chunk.model_dump_json(exclude=exclude)}\n\n"
|
||||
|
||||
total_completion_tokens = sum(num_generated_tokens)
|
||||
final_usage_info = UsageInfo(
|
||||
|
||||
Reference in New Issue
Block a user