[Frontend] Add output_mode to /inference/v1/generate (RFC #56851 Phase 1) (#58588)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
Martin Hickey
2026-10-03 19:38:25 +08:00
committed by GitHub
parent 5f30fc7031
commit bc21cba967
18 changed files with 1356 additions and 169 deletions
+3 -3
View File
@@ -207,9 +207,9 @@ class BaseFrontendArgs:
enable_scale_out: bool = False
"""
If set to True, register the scale-out endpoints (`/render`, `/derender`,
and `/inference/v1/generate`) on `vllm serve`. Has no effect on
`vllm launch render` or `vllm serve --tokens-only`, which always register
their required endpoints regardless of this flag.
`/inference/v1/generate` and `/inference/v1/abort_requests`) on `vllm serve`.
Has no effect on `vllm launch render` or `vllm serve --tokens-only` which
always register their required endpoints regardless of this flag.
"""
fingerprint_mode: Literal["full", "hash", "custom", "none"] = "full"
"""Controls the ``system_fingerprint`` field on responses.
@@ -29,7 +29,7 @@ from ..token_in_token_out.protocol import (
DerenderCompletionRequest,
DerenderCompletionStreamRequest,
DerenderStreamState,
GenerateResponse,
GenerateTokensResponse,
MultiModalFeatures,
)
@@ -54,7 +54,7 @@ class ServingDerender(BaseServing):
def _validate_derender_bounds(
self,
generate_responses: list[GenerateResponse],
generate_responses: list[GenerateTokensResponse],
) -> ErrorResponse | None:
"""Reject derender payloads that exceed resource bounds.
@@ -21,7 +21,7 @@ from vllm.logger import init_logger
from ...serve.engine.protocol import ErrorResponse
from .protocol import (
GenerateRequest,
GenerateResponse,
GenerateResponseBase,
)
from .serving import ServingTokens
@@ -67,35 +67,51 @@ async def generate(request: GenerateRequest, raw_request: Request):
content=generator.model_dump(), status_code=generator.error.code
)
elif isinstance(generator, GenerateResponse):
elif isinstance(generator, GenerateResponseBase):
return JSONResponse(content=generator.model_dump())
return StreamingResponse(content=generator, media_type="text/event-stream")
abort_router = APIRouter()
async def abort_requests(raw_request: Request):
"""Abort one or more requests. To be used in a
Disaggregated Everything setup.
"""
try:
body = await raw_request.json()
except json.JSONDecodeError as e:
raise HTTPException(
status_code=HTTPStatus.BAD_REQUEST.value,
detail=f"JSON decode error: {e}",
) from e
request_ids = body.get("request_ids")
if request_ids is None:
raise HTTPException(
status_code=HTTPStatus.BAD_REQUEST.value,
detail="Missing 'request_ids' in request body",
)
# Abort requests in background
asyncio.create_task(engine_client(raw_request).abort(request_ids))
return Response(status_code=200)
# Under the `/inference` prefix, so `--api-key` guards it.
router.add_api_route("/inference/v1/abort_requests", abort_requests, methods=["POST"])
# Unauthenticated legacy path, kept for existing `--tokens-only` deployments.
abort_router.add_api_route("/abort_requests", abort_requests, methods=["POST"])
def attach_router(app: FastAPI):
if getattr(app.state.args, "tokens_only", False):
@router.post("/abort_requests")
async def abort_requests(raw_request: Request):
"""Abort one or more requests. To be used in a
Disaggregated Everything setup.
"""
try:
body = await raw_request.json()
except json.JSONDecodeError as e:
raise HTTPException(
status_code=HTTPStatus.BAD_REQUEST.value,
detail=f"JSON decode error: {e}",
) from e
request_ids = body.get("request_ids")
if request_ids is None:
raise HTTPException(
status_code=HTTPStatus.BAD_REQUEST.value,
detail="Missing 'request_ids' in request body",
)
# Abort requests in background
asyncio.create_task(engine_client(raw_request).abort(request_ids))
return Response(status_code=200)
app.include_router(router)
if not getattr(app.state.args, "tokens_only", False):
return
# The RLHF dev router registers its own /abort_requests first. Registering
# this one too would only add a shadowed route with a duplicate operation ID.
has_abort_route = any(
getattr(route, "path", None) == "/abort_requests" for route in app.routes
)
if not has_abort_route:
app.include_router(abort_router)
@@ -4,9 +4,11 @@ from typing import Annotated, Any, Literal, TypeAlias
from pydantic import (
BaseModel,
Discriminator,
Field,
NonNegativeInt,
PrivateAttr,
Tag,
field_validator,
model_validator,
)
@@ -34,6 +36,8 @@ from vllm.utils import random_uuid
####### Tokens IN <> Tokens OUT #######
OutputMode: TypeAlias = Literal["tokens", "text"]
class PlaceholderRangeInfo(BaseModel):
"""Serializable placeholder location for a single multi-modal item."""
@@ -44,7 +48,7 @@ class PlaceholderRangeInfo(BaseModel):
length: int = Field(gt=0)
"""Number of placeholder tokens."""
# TODO: add ``is_embed: list[bool] | None`` once the /generate side
# TODO: add `is_embed: list[bool] | None` once the /generate side
# consumes features — some models (e.g. Qwen-VL) use sparse
# placeholder masks that cannot be recomputed from offset+length alone.
@@ -74,22 +78,22 @@ class MultiModalFeatures(BaseModel):
kwargs_data: dict[str, list[str | None]] | None = None
"""Per-modality serialized tensor data.
Each value is a list parallel to ``mm_hashes[modality]``. A ``str``
entry is a base64-encoded ``MultiModalKwargsItem``; ``None`` means
Each value is a list parallel to `mm_hashes[modality]`. A `str`
entry is a base64-encoded `MultiModalKwargsItem`; `None` means
the item should be resolved from cache. The entire field is
``None`` for metadata-only (cache-hit) responses.
`None` for metadata-only (cache-hit) responses.
"""
mm_metadata: dict[str, list[str | None]] | None = None
"""Per-modality serialized metadata for disaggregated prefill.
Each value is a list parallel to ``mm_hashes[modality]``. A ``str``
entry is a base64-encoded ``MultiModalKwargsItem`` containing only
placeholder-metadata and ``keep_on_cpu`` fields. ``None`` means that
Each value is a list parallel to `mm_hashes[modality]`. A `str`
entry is a base64-encoded `MultiModalKwargsItem` containing only
placeholder-metadata and `keep_on_cpu` fields. `None` means that
the metadata is unavailable for that item. Prefill can use this
instead of ``kwargs_data`` only when ``ec_transfer_params`` is also
instead of `kwargs_data` only when `ec_transfer_params` is also
set, so embeddings arrive through the EC connector rather than from
``pixel_values``.
`pixel_values`.
"""
@model_validator(mode="after")
@@ -174,7 +178,7 @@ class GenerateRequest(BaseModel):
content_parts: list[dict[str, Any]] | None = None
"""Raw multimodal input; server resolves media. Mutually exclusive
with ``features``."""
with `features`."""
@model_validator(mode="after")
def _check_mm_fields_exclusive(self) -> "GenerateRequest":
@@ -217,6 +221,16 @@ class GenerateRequest(BaseModel):
),
)
output_mode: OutputMode = Field(
default="tokens",
description=(
"Response level. 'tokens' returns token IDs only. 'text' also "
"returns the detokenized text and logprobs with decoded token "
"strings. 'text' needs a server that loads a tokenizer and "
"sampling_params.detokenize to be true."
),
)
stream: bool | None = False
stream_options: StreamOptions | None = None
cache_salt: str | None = Field(
@@ -253,12 +267,12 @@ class GenerateRequest(BaseModel):
),
)
# Tracks which keys the caller explicitly set inside ``sampling_params``
# Tracks which keys the caller explicitly set inside `sampling_params`
# when the request was parsed from a JSON body. Lets the server tell
# "client said max_tokens=16" from "client said nothing → dataclass
# default 16" so it can apply server-side defaulting only in the latter
# case. ``None`` means the request was constructed with a pre-built
# ``SamplingParams`` instance (e.g. from internal callers that have
# case. `None` means the request was constructed with a pre-built
# `SamplingParams` instance (e.g. from internal callers that have
# already resolved values), in which case all fields are considered set.
_sampling_params_provided_keys: set[str] | None = PrivateAttr(default=None)
_response_mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = (
@@ -284,6 +298,15 @@ class GenerateRequest(BaseModel):
validate_cache_salt(data.get("cache_salt"))
return data
@model_validator(mode="after")
def _check_output_mode_detokenizes(self) -> "GenerateRequest":
if self.output_mode != "tokens" and not self.sampling_params.detokenize:
raise ValueError(
f"output_mode={self.output_mode!r} requires "
"sampling_params.detokenize to be true"
)
return self
@model_validator(mode="after")
def _validate_multimodal_feature_bounds(self) -> "GenerateRequest":
if self.features is None:
@@ -299,10 +322,10 @@ class GenerateRequest(BaseModel):
return self
def is_sampling_param_provided(self, name: str) -> bool:
"""Whether the caller explicitly set ``sampling_params.<name>``.
"""Whether the caller explicitly set `sampling_params.<name>`.
For requests parsed from a JSON body, this reflects the raw input
dict. For requests constructed with a pre-built ``SamplingParams``
dict. For requests constructed with a pre-built `SamplingParams`
instance, all fields are considered provided so server-side defaults
do not clobber values already resolved upstream.
"""
@@ -317,21 +340,22 @@ class GenerateRequest(BaseModel):
)
class GenerateResponseChoice(BaseModel):
class GenerateChoiceBase(BaseModel):
"""Fields shared by every `output_mode` of a non-streaming choice."""
index: int
logprobs: ChatCompletionLogProbs | None = None
# per OpenAI spec this is the default
finish_reason: str | None = "stop"
token_ids: list[int] | None = None
# Per-token expert routing decisions, base64-encoded ``.npy`` bytes
# Per-token expert routing decisions, base64-encoded `.npy` bytes
# (numpy serialization). Shape after decode:
# (num_tokens - 1, num_layers, num_experts_per_tok) dtype uint8/uint16/int32
# ``num_tokens - 1`` because the last sampled token has not been
# `num_tokens - 1` because the last sampled token has not been
# forwarded yet and therefore has no routing data.
# Decode:
# np.load(io.BytesIO(base64.b64decode(s)))
# ``None`` if (a) the request was aborted before any forward pass,
# or (b) ``enable_return_routed_experts`` is off server-side.
# `None` if (a) the request was aborted before any forward pass,
# or (b) `enable_return_routed_experts` is off server-side.
routed_experts: str | None = None
sampling_mask: list[list[int]] | None = None
@@ -343,16 +367,43 @@ class GenerateResponseChoice(BaseModel):
return v
class GenerateResponseStreamChoice(BaseModel):
index: int
class GenerateTokensChoice(GenerateChoiceBase):
logprobs: ChatCompletionLogProbs | None = None
"""Logprobs whose tokens are `token_id:N` placeholders."""
class GenerateTextChoice(GenerateChoiceBase):
text: str
"""Detokenized output. Excludes a matched stop string unless
`include_stop_str_in_output` is set, while `token_ids` keeps every
generated token."""
logprobs: ChatCompletionLogProbs | None = None
"""Logprobs with decoded token strings and `bytes`."""
class GenerateStreamChoiceBase(BaseModel):
"""Fields shared by every `output_mode` of a streaming choice."""
index: int
finish_reason: str | None = None
token_ids: list[int] | None = None
routed_experts: str | None = None
sampling_mask: list[list[int]] | None = None
class GenerateStreamResponse(BaseModel):
class GenerateTokensStreamChoice(GenerateStreamChoiceBase):
logprobs: ChatCompletionLogProbs | None = None
class GenerateTextStreamChoice(GenerateStreamChoiceBase):
text: str
"""Text delta since the previous chunk of this choice."""
logprobs: ChatCompletionLogProbs | None = None
class GenerateStreamResponseBase(BaseModel):
request_id: str = Field(
default_factory=lambda: f"{random_uuid()}",
description=(
@@ -361,14 +412,23 @@ class GenerateStreamResponse(BaseModel):
"through out the inference process and return in response."
),
)
choices: list[GenerateResponseStreamChoice]
usage: UsageInfo | None = Field(default=None)
prompt_token_ids: list[NonNegativeInt] | None = None
mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = None
metrics: PerRequestMetrics | None = None
class GenerateResponse(BaseModel):
class GenerateTokensStreamResponse(GenerateStreamResponseBase):
output_mode: Literal["tokens"] = "tokens"
choices: list[GenerateTokensStreamChoice]
class GenerateTextStreamResponse(GenerateStreamResponseBase):
output_mode: Literal["text"] = "text"
choices: list[GenerateTextStreamChoice]
class GenerateResponseBase(BaseModel):
request_id: str = Field(
default_factory=lambda: f"{random_uuid()}",
description=(
@@ -379,7 +439,6 @@ class GenerateResponse(BaseModel):
)
model: str | None = None
created: int | None = None
choices: list[GenerateResponseChoice]
usage: UsageInfo | None = Field(default=None)
prompt_logprobs: list[dict[int, Logprob] | None] | None = None
prompt_token_id_logprobs: str | None = None
@@ -399,6 +458,42 @@ class GenerateResponse(BaseModel):
)
class GenerateTokensResponse(GenerateResponseBase):
output_mode: Literal["tokens"] = "tokens"
choices: list[GenerateTokensChoice]
class GenerateTextResponse(GenerateResponseBase):
output_mode: Literal["text"] = "text"
choices: list[GenerateTextChoice]
def output_mode_or_tokens(value: Any) -> Any:
"""Discriminator for `GenerateResponse` and `GenerateStreamResponse`.
A missing `output_mode` means `tokens`. Existing clients and servers
that predate the field never send it.
"""
if isinstance(value, dict):
return value.get("output_mode", "tokens")
return getattr(value, "output_mode", "tokens")
# Type aliases, not classes: use `GenerateResponseBase` or
# `GenerateStreamResponseBase` for `isinstance` checks and the
# `Generate<Level>...` classes to construct.
GenerateResponse: TypeAlias = Annotated[
Annotated[GenerateTokensResponse, Tag("tokens")]
| Annotated[GenerateTextResponse, Tag("text")],
Discriminator(output_mode_or_tokens),
]
GenerateStreamResponse: TypeAlias = Annotated[
Annotated[GenerateTokensStreamResponse, Tag("tokens")]
| Annotated[GenerateTextStreamResponse, Tag("text")],
Discriminator(output_mode_or_tokens),
]
class DerenderChatRequest(BaseModel):
"""Request for the /v1/chat/completions/derender endpoint (non-streaming).
@@ -412,8 +507,13 @@ class DerenderChatRequest(BaseModel):
model: str | None = None
"""Served model name. Defaults to the server's served model name."""
generate_response: GenerateResponse
"""The complete token-in / token-out engine response to derender."""
generate_response: GenerateTokensResponse
"""The complete token-in / token-out engine response to derender.
Only `output_mode="tokens"` responses are accepted. Other modes are
already detokenized and re-decoding their `token_ids` would undo the
engine's stop string truncation.
"""
prompt_tokens: int | None = None
"""Prompt token count for usage; defaults to 0 if omitted.
@@ -455,9 +555,9 @@ class DerenderCompletionRequest(BaseModel):
model: str | None = None
"""Served model name. Defaults to the server's served model name."""
generate_responses: list[GenerateResponse]
"""One response per prompt, parallel to the list[GenerateRequest]
returned by /v1/completions/render."""
generate_responses: list[GenerateTokensResponse]
"""One `output_mode="tokens"` response per prompt, parallel to the
list[GenerateRequest] returned by /v1/completions/render."""
prompt_tokens: list[int] | None = None
"""One prompt token count per response; each defaults to 0 if omitted.
@@ -589,7 +689,7 @@ class DerenderStreamState(BaseModel):
token in here through `parse_delta` (discarding the result) before
processing the current chunk's tokens since parser internal state
cannot be serialized into this stateless model. Unavoidably O(n)
bounded by ``max_model_len`` (enforced server side, not by a field
bounded by `max_model_len` (enforced server side, not by a field
validator here since the bound is model dependent).
"""
@@ -630,24 +730,25 @@ class DerenderChatStreamRequest(BaseModel):
"""One chunk streaming derender request for /v1/chat/completions/derender.
The client sends one request per SSE chunk received from
``/inference/v1/generate``. Each request carries the generate chunk
plus the ``stream_state`` returned by the previous call (``None`` on the
`/inference/v1/generate`. Each request carries the generate chunk
plus the `stream_state` returned by the previous call (`None` on the
first call). The response contains the derendered chunk and the updated
state to be passed to the next call.
This implements stateless no server side session. All mutable state lives in
the client carried ``stream_state``.
the client carried `stream_state`.
"""
# --8<-- [start:derender-chat-stream-request]
stream: Literal[True]
model: str | None = None
generate_chunk: GenerateStreamResponse
"""One SSE chunk from ``/inference/v1/generate`` (``stream=True``)."""
generate_chunk: GenerateTokensStreamResponse
"""One `output_mode="tokens"` SSE chunk from `/inference/v1/generate`
(`stream=True`)."""
stream_state: DerenderStreamState | None = None
"""Client carried detok state from the previous call. ``None`` on first."""
"""Client carried detok state from the previous call. `None` on first."""
prompt_tokens: int | None = None
"""Prompt token count for usage. Forwarded from the render step."""
@@ -655,7 +756,7 @@ class DerenderChatStreamRequest(BaseModel):
prompt_token_ids: list[NonNegativeInt] | None = None
"""Prompt token IDs. Required by the parser path's `parse_delta` to
settle its initial reasoning state (e.g. chat templates that pre-open
``<think>``). `prompt_tokens` is a usage count and cannot serve this
`<think>`). `prompt_tokens` is a usage count and cannot serve this
purpose. Sourced from `GenerateRequest.token_ids` at the render step.
Rejected with a 400 (by `ServingDerender`) when a tool or reasoning
@@ -680,7 +781,7 @@ class DerenderChatStreamRequest(BaseModel):
class DerenderCompletionStreamRequest(BaseModel):
"""One chunk streaming derender request for /v1/completions/derender.
Parallel to ``DerenderChatStreamRequest`` for the completions endpoint.
Parallel to `DerenderChatStreamRequest` for the completions endpoint.
Each call processes one SSE chunk (one output sequence's delta) and
returns the derendered chunk plus updated state.
"""
@@ -689,11 +790,11 @@ class DerenderCompletionStreamRequest(BaseModel):
stream: Literal[True]
model: str | None = None
generate_chunk: GenerateStreamResponse
"""One SSE chunk from ``/inference/v1/generate``."""
generate_chunk: GenerateTokensStreamResponse
"""One `output_mode="tokens"` SSE chunk from `/inference/v1/generate`."""
stream_state: DerenderStreamState | None = None
"""Client-carried detok state. ``None`` on the first call."""
"""Client-carried detok state. `None` on the first call."""
prompt_tokens: int | None = None
"""Prompt token count for usage."""
@@ -723,16 +824,16 @@ class DerenderChatStreamResponse(BaseModel):
class DerenderCompletionStreamResponse(BaseModel):
"""Response for one streaming completions derender chunk.
Parallel to ``DerenderChatStreamResponse`` for the completions endpoint.
Parallel to `DerenderChatStreamResponse` for the completions endpoint.
"""
chunk: CompletionStreamResponse
stream_state: DerenderStreamState
# Determines the type by checking the ``stream`` field's literal value. A body without
# ``stream`` validates as the non-streaming member
# (``stream`` defaults to ``False`` there), so FastAPI can validate and dispatch both
# Determines the type by checking the `stream` field's literal value. A body without
# `stream` validates as the non-streaming member
# (`stream` defaults to `False` there), so FastAPI can validate and dispatch both
# shapes on a single path.
DerenderChatRequestUnion: TypeAlias = DerenderChatRequest | DerenderChatStreamRequest
DerenderCompletionRequestUnion: TypeAlias = (
@@ -6,6 +6,7 @@ import asyncio
import time
from collections.abc import AsyncGenerator
from collections.abc import Sequence as GenericSequence
from typing import Any
import msgspec
from fastapi import Request
@@ -20,6 +21,7 @@ from vllm.entrypoints.generate.base.serving import (
GenerateBaseServing,
build_spec_decoding_metrics,
clamp_prompt_logprobs,
format_token_id_placeholder,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionLogProb,
@@ -46,6 +48,7 @@ from vllm.multimodal.inputs import (
from vllm.outputs import RequestOutput
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import RequestOutputKind, SamplingParams
from vllm.tokenizers import TokenizerLike
from vllm.utils.collection_utils import as_list
from vllm.utils.serial_utils import numpy2base64
@@ -56,14 +59,45 @@ from .mm_features import (
from .protocol import (
GenerateRequest,
GenerateResponse,
GenerateResponseChoice,
GenerateResponseStreamChoice,
GenerateStreamResponse,
GenerateTextChoice,
GenerateTextResponse,
GenerateTextStreamChoice,
GenerateTextStreamResponse,
GenerateTokensChoice,
GenerateTokensResponse,
GenerateTokensStreamChoice,
GenerateTokensStreamResponse,
)
logger = init_logger(__name__)
def _logprob_token(
token_id: int, logprob: Logprob | None, tokenizer: TokenizerLike | None
) -> tuple[str, list[int] | None]:
"""Token string and UTF-8 bytes for one logprob entry.
Without a tokenizer the token is a ``token_id:N`` placeholder with no bytes.
"""
if tokenizer is None:
return format_token_id_placeholder(token_id), None
token = (
tokenizer.decode([token_id])
if logprob is None
else GenerateBaseServing._get_decoded_token(logprob, token_id, tokenizer)
)
return token, list(token.encode("utf-8", errors="replace"))
def _top_logprob(
token_id: int, logprob: Logprob, tokenizer: TokenizerLike | None
) -> ChatCompletionLogProb:
token, token_bytes = _logprob_token(token_id, logprob, tokenizer)
return ChatCompletionLogProb(
token=token, logprob=max(logprob.logprob, -9999.0), bytes=token_bytes
)
class ServingTokens(GenerateBaseServing):
"""Provides Tokens IN <> Tokens OUT functionality to vLLM API."""
@@ -89,6 +123,9 @@ class ServingTokens(GenerateBaseServing):
self.enable_prompt_tokens_details = enable_prompt_tokens_details
self.enable_log_outputs = enable_log_outputs
self.force_no_detokenize = force_no_detokenize
self.has_tokenizer = (
not force_no_detokenize and self.renderer.tokenizer is not None
)
if force_no_detokenize:
logger.info(
"Tokens-only mode is enabled, skipping detokenization "
@@ -171,6 +208,15 @@ class ServingTokens(GenerateBaseServing):
"because detokenization is disabled. Check stop strings on "
"the coordinator, or use stop_token_ids."
)
if request.output_mode != "tokens" and not self.has_tokenizer:
# The no-op detokenizer returns "", so without this check the
# request would succeed with empty text.
return self.create_error_response(
f"output_mode={request.output_mode!r} requires a tokenizer, but "
"this server does not load one (--tokens-only or "
"--skip-tokenizer-init). Send the request to a server that "
"loads a tokenizer, or use output_mode='tokens'."
)
try:
msgspec.msgpack.encode(
(
@@ -331,6 +377,8 @@ class ServingTokens(GenerateBaseServing):
created_time = int(time.time())
final_res: RequestOutput | None = None
sampling_params: SamplingParams = request.sampling_params
text_mode = request.output_mode == "text"
tokenizer = self._logprobs_tokenizer(text_mode)
try:
async for res in result_generator:
@@ -340,7 +388,8 @@ class ServingTokens(GenerateBaseServing):
assert final_res is not None
choices: list[GenerateResponseChoice] = []
tokens_choices: list[GenerateTokensChoice] = []
text_choices: list[GenerateTextChoice] = []
num_generated_tokens = 0
for output in final_res.outputs:
self._raise_if_error(output.finish_reason, request_id)
@@ -355,6 +404,7 @@ class ServingTokens(GenerateBaseServing):
token_ids=token_ids,
top_logprobs=out_logprobs,
num_output_top_logprobs=sampling_params.logprobs,
tokenizer=tokenizer,
)
else:
logprobs = None
@@ -369,7 +419,7 @@ class ServingTokens(GenerateBaseServing):
if output.sampling_mask is not None:
sampling_mask = output.sampling_mask.token_ids
choice_data = GenerateResponseChoice(
choice_fields: dict[str, Any] = dict(
index=output.index,
logprobs=logprobs,
finish_reason=output.finish_reason if output.finish_reason else "stop",
@@ -377,8 +427,12 @@ class ServingTokens(GenerateBaseServing):
routed_experts=routed_experts_b64,
sampling_mask=sampling_mask,
)
choices.append(choice_data)
if text_mode:
text_choices.append(
GenerateTextChoice(text=output.text, **choice_fields)
)
else:
tokens_choices.append(GenerateTokensChoice(**choice_fields))
num_generated_tokens += len(output.token_ids)
assert final_res.prompt_token_ids is not None
@@ -407,11 +461,10 @@ class ServingTokens(GenerateBaseServing):
spec_stats = build_spec_decoding_metrics(final_res)
if spec_stats is not None:
per_request_metrics = PerRequestMetrics(speculative_decoding=spec_stats)
response = GenerateResponse(
response_fields: dict[str, Any] = dict(
request_id=request_id,
created=created_time,
model=model_name,
choices=choices,
usage=usage,
prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
prompt_token_id_logprobs=(
@@ -427,10 +480,15 @@ class ServingTokens(GenerateBaseServing):
kv_transfer_params=final_res.kv_transfer_params,
ec_transfer_params=final_res.ec_transfer_params,
)
response: GenerateTokensResponse | GenerateTextResponse
if text_mode:
response = GenerateTextResponse(choices=text_choices, **response_fields)
else:
response = GenerateTokensResponse(choices=tokens_choices, **response_fields)
# Log complete response if output logging is enabled
if self.enable_log_outputs and self.request_logger:
for choice in choices:
for choice in response.choices:
# Get the corresponding output token IDs
output_token_ids = None
if choice.index < len(final_res.outputs):
@@ -464,6 +522,8 @@ class ServingTokens(GenerateBaseServing):
num_cached_tokens = None
sampling_params: SamplingParams = request.sampling_params
last_res: RequestOutput | None = None
text_mode = request.output_mode == "text"
tokenizer = self._logprobs_tokenizer(text_mode)
include_usage, include_continuous_usage = should_include_usage(
request.stream_options, False
@@ -491,11 +551,21 @@ class ServingTokens(GenerateBaseServing):
finish_reason = output.finish_reason
self._raise_if_error(finish_reason, request_id)
# Still emit a terminal empty chunk while prompt metadata
# is pending, so zero-token completions deliver it.
if not delta_token_ids and (
finish_reason is None or prompt_token_ids is None
):
if text_mode:
# Text held back for stop string matching is flushed
# with the final output, and the abort output has no
# new token IDs, so text or a finish reason is enough.
emit = bool(
delta_token_ids or output.text or finish_reason is not None
)
else:
# Still emit a terminal empty chunk while prompt
# metadata is pending, so zero-token completions
# deliver it.
emit = bool(delta_token_ids) or (
finish_reason is not None and prompt_token_ids is not None
)
if not emit:
continue
if sampling_params.logprobs is not None:
@@ -505,6 +575,7 @@ class ServingTokens(GenerateBaseServing):
token_ids=delta_token_ids,
top_logprobs=out_logprobs,
num_output_top_logprobs=sampling_params.logprobs,
tokenizer=tokenizer,
)
else:
logprobs = None
@@ -518,20 +589,30 @@ class ServingTokens(GenerateBaseServing):
sampling_mask = None
if output.sampling_mask is not None:
sampling_mask = output.sampling_mask.token_ids
chunk = GenerateStreamResponse(
request_id=request_id,
choices=[
GenerateResponseStreamChoice(
index=i,
logprobs=logprobs,
finish_reason=finish_reason,
token_ids=as_list(delta_token_ids),
routed_experts=routed_experts_b64,
sampling_mask=sampling_mask,
)
],
choice_fields: dict[str, Any] = dict(
index=i,
logprobs=logprobs,
finish_reason=finish_reason,
token_ids=as_list(delta_token_ids),
routed_experts=routed_experts_b64,
sampling_mask=sampling_mask,
)
chunk: GenerateTokensStreamResponse | GenerateTextStreamResponse
if text_mode:
chunk = GenerateTextStreamResponse(
request_id=request_id,
choices=[
GenerateTextStreamChoice(
text=output.text, **choice_fields
)
],
)
else:
chunk = GenerateTokensStreamResponse(
request_id=request_id,
choices=[GenerateTokensStreamChoice(**choice_fields)],
)
if prompt_token_ids is not None:
chunk.prompt_token_ids = prompt_token_ids
chunk.mm_placeholders = request._response_mm_placeholders
@@ -571,12 +652,21 @@ class ServingTokens(GenerateBaseServing):
per_request_metrics = PerRequestMetrics(
speculative_decoding=spec_stats
)
final_chunk = GenerateStreamResponse(
request_id=request_id,
choices=[],
usage=final_usage_info,
metrics=per_request_metrics,
)
final_chunk: GenerateTokensStreamResponse | GenerateTextStreamResponse
if text_mode:
final_chunk = GenerateTextStreamResponse(
request_id=request_id,
choices=[],
usage=final_usage_info,
metrics=per_request_metrics,
)
else:
final_chunk = GenerateTokensStreamResponse(
request_id=request_id,
choices=[],
usage=final_usage_info,
metrics=per_request_metrics,
)
yield f"data: {final_chunk.model_dump_json(exclude_none=True)}\n\n"
request_metadata.final_usage_info = final_usage_info
@@ -591,37 +681,49 @@ class ServingTokens(GenerateBaseServing):
yield f"data: {data}\n\n"
yield "data: [DONE]\n\n"
def _logprobs_tokenizer(self, text_mode: bool) -> TokenizerLike | None:
"""Tokenizer that resolves logprob tokens, or None for placeholders.
``--return-tokens-as-token-ids`` keeps placeholders at every level, as
it does on ``/v1/completions``.
"""
if not text_mode or self.return_tokens_as_token_ids:
return None
return self.renderer.tokenizer
def _create_tokens_logprobs(
self,
token_ids: GenericSequence[int],
top_logprobs: GenericSequence[dict[int, Logprob] | None],
num_output_top_logprobs: int | None = None,
tokenizer: TokenizerLike | None = None,
) -> ChatCompletionLogProbs:
"""Create OpenAI-style logprobs."""
"""Create OpenAI-style logprobs.
Tokens are ``token_id:N`` placeholders. With a ``tokenizer`` they are
decoded strings and carry ``bytes``.
"""
logprobs_content: list[ChatCompletionLogProbsContent] = []
for i, token_id in enumerate(token_ids):
token = f"token_id:{token_id}"
step_top_logprobs = top_logprobs[i]
if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
token, token_bytes = _logprob_token(token_id, None, tokenizer)
logprobs_content.append(
ChatCompletionLogProbsContent(
token=token,
)
ChatCompletionLogProbsContent(token=token, bytes=token_bytes)
)
else:
step_token = step_top_logprobs[token_id]
token, token_bytes = _logprob_token(token_id, step_token, tokenizer)
logprobs_content.append(
ChatCompletionLogProbsContent(
token=token,
logprob=max(step_token.logprob, -9999.0),
bytes=token_bytes,
top_logprobs=[
ChatCompletionLogProb(
token=f"token_id:{token_id}",
logprob=max(logprob.logprob, -9999.0),
)
for i, (token_id, logprob) in enumerate(
_top_logprob(top_id, logprob, tokenizer)
for i, (top_id, logprob) in enumerate(
step_top_logprobs.items()
)
if num_output_top_logprobs is not None