Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
@@ -207,9 +207,9 @@ class BaseFrontendArgs:
|
||||
enable_scale_out: bool = False
|
||||
"""
|
||||
If set to True, register the scale-out endpoints (`/render`, `/derender`,
|
||||
and `/inference/v1/generate`) on `vllm serve`. Has no effect on
|
||||
`vllm launch render` or `vllm serve --tokens-only`, which always register
|
||||
their required endpoints regardless of this flag.
|
||||
`/inference/v1/generate` and `/inference/v1/abort_requests`) on `vllm serve`.
|
||||
Has no effect on `vllm launch render` or `vllm serve --tokens-only` which
|
||||
always register their required endpoints regardless of this flag.
|
||||
"""
|
||||
fingerprint_mode: Literal["full", "hash", "custom", "none"] = "full"
|
||||
"""Controls the ``system_fingerprint`` field on responses.
|
||||
|
||||
@@ -29,7 +29,7 @@ from ..token_in_token_out.protocol import (
|
||||
DerenderCompletionRequest,
|
||||
DerenderCompletionStreamRequest,
|
||||
DerenderStreamState,
|
||||
GenerateResponse,
|
||||
GenerateTokensResponse,
|
||||
MultiModalFeatures,
|
||||
)
|
||||
|
||||
@@ -54,7 +54,7 @@ class ServingDerender(BaseServing):
|
||||
|
||||
def _validate_derender_bounds(
|
||||
self,
|
||||
generate_responses: list[GenerateResponse],
|
||||
generate_responses: list[GenerateTokensResponse],
|
||||
) -> ErrorResponse | None:
|
||||
"""Reject derender payloads that exceed resource bounds.
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ from vllm.logger import init_logger
|
||||
from ...serve.engine.protocol import ErrorResponse
|
||||
from .protocol import (
|
||||
GenerateRequest,
|
||||
GenerateResponse,
|
||||
GenerateResponseBase,
|
||||
)
|
||||
from .serving import ServingTokens
|
||||
|
||||
@@ -67,35 +67,51 @@ async def generate(request: GenerateRequest, raw_request: Request):
|
||||
content=generator.model_dump(), status_code=generator.error.code
|
||||
)
|
||||
|
||||
elif isinstance(generator, GenerateResponse):
|
||||
elif isinstance(generator, GenerateResponseBase):
|
||||
return JSONResponse(content=generator.model_dump())
|
||||
|
||||
return StreamingResponse(content=generator, media_type="text/event-stream")
|
||||
|
||||
|
||||
abort_router = APIRouter()
|
||||
|
||||
|
||||
async def abort_requests(raw_request: Request):
|
||||
"""Abort one or more requests. To be used in a
|
||||
Disaggregated Everything setup.
|
||||
"""
|
||||
try:
|
||||
body = await raw_request.json()
|
||||
except json.JSONDecodeError as e:
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.BAD_REQUEST.value,
|
||||
detail=f"JSON decode error: {e}",
|
||||
) from e
|
||||
request_ids = body.get("request_ids")
|
||||
if request_ids is None:
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.BAD_REQUEST.value,
|
||||
detail="Missing 'request_ids' in request body",
|
||||
)
|
||||
# Abort requests in background
|
||||
asyncio.create_task(engine_client(raw_request).abort(request_ids))
|
||||
return Response(status_code=200)
|
||||
|
||||
|
||||
# Under the `/inference` prefix, so `--api-key` guards it.
|
||||
router.add_api_route("/inference/v1/abort_requests", abort_requests, methods=["POST"])
|
||||
# Unauthenticated legacy path, kept for existing `--tokens-only` deployments.
|
||||
abort_router.add_api_route("/abort_requests", abort_requests, methods=["POST"])
|
||||
|
||||
|
||||
def attach_router(app: FastAPI):
|
||||
if getattr(app.state.args, "tokens_only", False):
|
||||
|
||||
@router.post("/abort_requests")
|
||||
async def abort_requests(raw_request: Request):
|
||||
"""Abort one or more requests. To be used in a
|
||||
Disaggregated Everything setup.
|
||||
"""
|
||||
try:
|
||||
body = await raw_request.json()
|
||||
except json.JSONDecodeError as e:
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.BAD_REQUEST.value,
|
||||
detail=f"JSON decode error: {e}",
|
||||
) from e
|
||||
request_ids = body.get("request_ids")
|
||||
if request_ids is None:
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.BAD_REQUEST.value,
|
||||
detail="Missing 'request_ids' in request body",
|
||||
)
|
||||
# Abort requests in background
|
||||
asyncio.create_task(engine_client(raw_request).abort(request_ids))
|
||||
return Response(status_code=200)
|
||||
|
||||
app.include_router(router)
|
||||
if not getattr(app.state.args, "tokens_only", False):
|
||||
return
|
||||
# The RLHF dev router registers its own /abort_requests first. Registering
|
||||
# this one too would only add a shadowed route with a duplicate operation ID.
|
||||
has_abort_route = any(
|
||||
getattr(route, "path", None) == "/abort_requests" for route in app.routes
|
||||
)
|
||||
if not has_abort_route:
|
||||
app.include_router(abort_router)
|
||||
|
||||
@@ -4,9 +4,11 @@ from typing import Annotated, Any, Literal, TypeAlias
|
||||
|
||||
from pydantic import (
|
||||
BaseModel,
|
||||
Discriminator,
|
||||
Field,
|
||||
NonNegativeInt,
|
||||
PrivateAttr,
|
||||
Tag,
|
||||
field_validator,
|
||||
model_validator,
|
||||
)
|
||||
@@ -34,6 +36,8 @@ from vllm.utils import random_uuid
|
||||
|
||||
####### Tokens IN <> Tokens OUT #######
|
||||
|
||||
OutputMode: TypeAlias = Literal["tokens", "text"]
|
||||
|
||||
|
||||
class PlaceholderRangeInfo(BaseModel):
|
||||
"""Serializable placeholder location for a single multi-modal item."""
|
||||
@@ -44,7 +48,7 @@ class PlaceholderRangeInfo(BaseModel):
|
||||
length: int = Field(gt=0)
|
||||
"""Number of placeholder tokens."""
|
||||
|
||||
# TODO: add ``is_embed: list[bool] | None`` once the /generate side
|
||||
# TODO: add `is_embed: list[bool] | None` once the /generate side
|
||||
# consumes features — some models (e.g. Qwen-VL) use sparse
|
||||
# placeholder masks that cannot be recomputed from offset+length alone.
|
||||
|
||||
@@ -74,22 +78,22 @@ class MultiModalFeatures(BaseModel):
|
||||
kwargs_data: dict[str, list[str | None]] | None = None
|
||||
"""Per-modality serialized tensor data.
|
||||
|
||||
Each value is a list parallel to ``mm_hashes[modality]``. A ``str``
|
||||
entry is a base64-encoded ``MultiModalKwargsItem``; ``None`` means
|
||||
Each value is a list parallel to `mm_hashes[modality]`. A `str`
|
||||
entry is a base64-encoded `MultiModalKwargsItem`; `None` means
|
||||
the item should be resolved from cache. The entire field is
|
||||
``None`` for metadata-only (cache-hit) responses.
|
||||
`None` for metadata-only (cache-hit) responses.
|
||||
"""
|
||||
|
||||
mm_metadata: dict[str, list[str | None]] | None = None
|
||||
"""Per-modality serialized metadata for disaggregated prefill.
|
||||
|
||||
Each value is a list parallel to ``mm_hashes[modality]``. A ``str``
|
||||
entry is a base64-encoded ``MultiModalKwargsItem`` containing only
|
||||
placeholder-metadata and ``keep_on_cpu`` fields. ``None`` means that
|
||||
Each value is a list parallel to `mm_hashes[modality]`. A `str`
|
||||
entry is a base64-encoded `MultiModalKwargsItem` containing only
|
||||
placeholder-metadata and `keep_on_cpu` fields. `None` means that
|
||||
the metadata is unavailable for that item. Prefill can use this
|
||||
instead of ``kwargs_data`` only when ``ec_transfer_params`` is also
|
||||
instead of `kwargs_data` only when `ec_transfer_params` is also
|
||||
set, so embeddings arrive through the EC connector rather than from
|
||||
``pixel_values``.
|
||||
`pixel_values`.
|
||||
"""
|
||||
|
||||
@model_validator(mode="after")
|
||||
@@ -174,7 +178,7 @@ class GenerateRequest(BaseModel):
|
||||
|
||||
content_parts: list[dict[str, Any]] | None = None
|
||||
"""Raw multimodal input; server resolves media. Mutually exclusive
|
||||
with ``features``."""
|
||||
with `features`."""
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _check_mm_fields_exclusive(self) -> "GenerateRequest":
|
||||
@@ -217,6 +221,16 @@ class GenerateRequest(BaseModel):
|
||||
),
|
||||
)
|
||||
|
||||
output_mode: OutputMode = Field(
|
||||
default="tokens",
|
||||
description=(
|
||||
"Response level. 'tokens' returns token IDs only. 'text' also "
|
||||
"returns the detokenized text and logprobs with decoded token "
|
||||
"strings. 'text' needs a server that loads a tokenizer and "
|
||||
"sampling_params.detokenize to be true."
|
||||
),
|
||||
)
|
||||
|
||||
stream: bool | None = False
|
||||
stream_options: StreamOptions | None = None
|
||||
cache_salt: str | None = Field(
|
||||
@@ -253,12 +267,12 @@ class GenerateRequest(BaseModel):
|
||||
),
|
||||
)
|
||||
|
||||
# Tracks which keys the caller explicitly set inside ``sampling_params``
|
||||
# Tracks which keys the caller explicitly set inside `sampling_params`
|
||||
# when the request was parsed from a JSON body. Lets the server tell
|
||||
# "client said max_tokens=16" from "client said nothing → dataclass
|
||||
# default 16" so it can apply server-side defaulting only in the latter
|
||||
# case. ``None`` means the request was constructed with a pre-built
|
||||
# ``SamplingParams`` instance (e.g. from internal callers that have
|
||||
# case. `None` means the request was constructed with a pre-built
|
||||
# `SamplingParams` instance (e.g. from internal callers that have
|
||||
# already resolved values), in which case all fields are considered set.
|
||||
_sampling_params_provided_keys: set[str] | None = PrivateAttr(default=None)
|
||||
_response_mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = (
|
||||
@@ -284,6 +298,15 @@ class GenerateRequest(BaseModel):
|
||||
validate_cache_salt(data.get("cache_salt"))
|
||||
return data
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _check_output_mode_detokenizes(self) -> "GenerateRequest":
|
||||
if self.output_mode != "tokens" and not self.sampling_params.detokenize:
|
||||
raise ValueError(
|
||||
f"output_mode={self.output_mode!r} requires "
|
||||
"sampling_params.detokenize to be true"
|
||||
)
|
||||
return self
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_multimodal_feature_bounds(self) -> "GenerateRequest":
|
||||
if self.features is None:
|
||||
@@ -299,10 +322,10 @@ class GenerateRequest(BaseModel):
|
||||
return self
|
||||
|
||||
def is_sampling_param_provided(self, name: str) -> bool:
|
||||
"""Whether the caller explicitly set ``sampling_params.<name>``.
|
||||
"""Whether the caller explicitly set `sampling_params.<name>`.
|
||||
|
||||
For requests parsed from a JSON body, this reflects the raw input
|
||||
dict. For requests constructed with a pre-built ``SamplingParams``
|
||||
dict. For requests constructed with a pre-built `SamplingParams`
|
||||
instance, all fields are considered provided so server-side defaults
|
||||
do not clobber values already resolved upstream.
|
||||
"""
|
||||
@@ -317,21 +340,22 @@ class GenerateRequest(BaseModel):
|
||||
)
|
||||
|
||||
|
||||
class GenerateResponseChoice(BaseModel):
|
||||
class GenerateChoiceBase(BaseModel):
|
||||
"""Fields shared by every `output_mode` of a non-streaming choice."""
|
||||
|
||||
index: int
|
||||
logprobs: ChatCompletionLogProbs | None = None
|
||||
# per OpenAI spec this is the default
|
||||
finish_reason: str | None = "stop"
|
||||
token_ids: list[int] | None = None
|
||||
# Per-token expert routing decisions, base64-encoded ``.npy`` bytes
|
||||
# Per-token expert routing decisions, base64-encoded `.npy` bytes
|
||||
# (numpy serialization). Shape after decode:
|
||||
# (num_tokens - 1, num_layers, num_experts_per_tok) dtype uint8/uint16/int32
|
||||
# ``num_tokens - 1`` because the last sampled token has not been
|
||||
# `num_tokens - 1` because the last sampled token has not been
|
||||
# forwarded yet and therefore has no routing data.
|
||||
# Decode:
|
||||
# np.load(io.BytesIO(base64.b64decode(s)))
|
||||
# ``None`` if (a) the request was aborted before any forward pass,
|
||||
# or (b) ``enable_return_routed_experts`` is off server-side.
|
||||
# `None` if (a) the request was aborted before any forward pass,
|
||||
# or (b) `enable_return_routed_experts` is off server-side.
|
||||
routed_experts: str | None = None
|
||||
sampling_mask: list[list[int]] | None = None
|
||||
|
||||
@@ -343,16 +367,43 @@ class GenerateResponseChoice(BaseModel):
|
||||
return v
|
||||
|
||||
|
||||
class GenerateResponseStreamChoice(BaseModel):
|
||||
index: int
|
||||
class GenerateTokensChoice(GenerateChoiceBase):
|
||||
logprobs: ChatCompletionLogProbs | None = None
|
||||
"""Logprobs whose tokens are `token_id:N` placeholders."""
|
||||
|
||||
|
||||
class GenerateTextChoice(GenerateChoiceBase):
|
||||
text: str
|
||||
"""Detokenized output. Excludes a matched stop string unless
|
||||
`include_stop_str_in_output` is set, while `token_ids` keeps every
|
||||
generated token."""
|
||||
|
||||
logprobs: ChatCompletionLogProbs | None = None
|
||||
"""Logprobs with decoded token strings and `bytes`."""
|
||||
|
||||
|
||||
class GenerateStreamChoiceBase(BaseModel):
|
||||
"""Fields shared by every `output_mode` of a streaming choice."""
|
||||
|
||||
index: int
|
||||
finish_reason: str | None = None
|
||||
token_ids: list[int] | None = None
|
||||
routed_experts: str | None = None
|
||||
sampling_mask: list[list[int]] | None = None
|
||||
|
||||
|
||||
class GenerateStreamResponse(BaseModel):
|
||||
class GenerateTokensStreamChoice(GenerateStreamChoiceBase):
|
||||
logprobs: ChatCompletionLogProbs | None = None
|
||||
|
||||
|
||||
class GenerateTextStreamChoice(GenerateStreamChoiceBase):
|
||||
text: str
|
||||
"""Text delta since the previous chunk of this choice."""
|
||||
|
||||
logprobs: ChatCompletionLogProbs | None = None
|
||||
|
||||
|
||||
class GenerateStreamResponseBase(BaseModel):
|
||||
request_id: str = Field(
|
||||
default_factory=lambda: f"{random_uuid()}",
|
||||
description=(
|
||||
@@ -361,14 +412,23 @@ class GenerateStreamResponse(BaseModel):
|
||||
"through out the inference process and return in response."
|
||||
),
|
||||
)
|
||||
choices: list[GenerateResponseStreamChoice]
|
||||
usage: UsageInfo | None = Field(default=None)
|
||||
prompt_token_ids: list[NonNegativeInt] | None = None
|
||||
mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = None
|
||||
metrics: PerRequestMetrics | None = None
|
||||
|
||||
|
||||
class GenerateResponse(BaseModel):
|
||||
class GenerateTokensStreamResponse(GenerateStreamResponseBase):
|
||||
output_mode: Literal["tokens"] = "tokens"
|
||||
choices: list[GenerateTokensStreamChoice]
|
||||
|
||||
|
||||
class GenerateTextStreamResponse(GenerateStreamResponseBase):
|
||||
output_mode: Literal["text"] = "text"
|
||||
choices: list[GenerateTextStreamChoice]
|
||||
|
||||
|
||||
class GenerateResponseBase(BaseModel):
|
||||
request_id: str = Field(
|
||||
default_factory=lambda: f"{random_uuid()}",
|
||||
description=(
|
||||
@@ -379,7 +439,6 @@ class GenerateResponse(BaseModel):
|
||||
)
|
||||
model: str | None = None
|
||||
created: int | None = None
|
||||
choices: list[GenerateResponseChoice]
|
||||
usage: UsageInfo | None = Field(default=None)
|
||||
prompt_logprobs: list[dict[int, Logprob] | None] | None = None
|
||||
prompt_token_id_logprobs: str | None = None
|
||||
@@ -399,6 +458,42 @@ class GenerateResponse(BaseModel):
|
||||
)
|
||||
|
||||
|
||||
class GenerateTokensResponse(GenerateResponseBase):
|
||||
output_mode: Literal["tokens"] = "tokens"
|
||||
choices: list[GenerateTokensChoice]
|
||||
|
||||
|
||||
class GenerateTextResponse(GenerateResponseBase):
|
||||
output_mode: Literal["text"] = "text"
|
||||
choices: list[GenerateTextChoice]
|
||||
|
||||
|
||||
def output_mode_or_tokens(value: Any) -> Any:
|
||||
"""Discriminator for `GenerateResponse` and `GenerateStreamResponse`.
|
||||
|
||||
A missing `output_mode` means `tokens`. Existing clients and servers
|
||||
that predate the field never send it.
|
||||
"""
|
||||
if isinstance(value, dict):
|
||||
return value.get("output_mode", "tokens")
|
||||
return getattr(value, "output_mode", "tokens")
|
||||
|
||||
|
||||
# Type aliases, not classes: use `GenerateResponseBase` or
|
||||
# `GenerateStreamResponseBase` for `isinstance` checks and the
|
||||
# `Generate<Level>...` classes to construct.
|
||||
GenerateResponse: TypeAlias = Annotated[
|
||||
Annotated[GenerateTokensResponse, Tag("tokens")]
|
||||
| Annotated[GenerateTextResponse, Tag("text")],
|
||||
Discriminator(output_mode_or_tokens),
|
||||
]
|
||||
GenerateStreamResponse: TypeAlias = Annotated[
|
||||
Annotated[GenerateTokensStreamResponse, Tag("tokens")]
|
||||
| Annotated[GenerateTextStreamResponse, Tag("text")],
|
||||
Discriminator(output_mode_or_tokens),
|
||||
]
|
||||
|
||||
|
||||
class DerenderChatRequest(BaseModel):
|
||||
"""Request for the /v1/chat/completions/derender endpoint (non-streaming).
|
||||
|
||||
@@ -412,8 +507,13 @@ class DerenderChatRequest(BaseModel):
|
||||
model: str | None = None
|
||||
"""Served model name. Defaults to the server's served model name."""
|
||||
|
||||
generate_response: GenerateResponse
|
||||
"""The complete token-in / token-out engine response to derender."""
|
||||
generate_response: GenerateTokensResponse
|
||||
"""The complete token-in / token-out engine response to derender.
|
||||
|
||||
Only `output_mode="tokens"` responses are accepted. Other modes are
|
||||
already detokenized and re-decoding their `token_ids` would undo the
|
||||
engine's stop string truncation.
|
||||
"""
|
||||
|
||||
prompt_tokens: int | None = None
|
||||
"""Prompt token count for usage; defaults to 0 if omitted.
|
||||
@@ -455,9 +555,9 @@ class DerenderCompletionRequest(BaseModel):
|
||||
model: str | None = None
|
||||
"""Served model name. Defaults to the server's served model name."""
|
||||
|
||||
generate_responses: list[GenerateResponse]
|
||||
"""One response per prompt, parallel to the list[GenerateRequest]
|
||||
returned by /v1/completions/render."""
|
||||
generate_responses: list[GenerateTokensResponse]
|
||||
"""One `output_mode="tokens"` response per prompt, parallel to the
|
||||
list[GenerateRequest] returned by /v1/completions/render."""
|
||||
|
||||
prompt_tokens: list[int] | None = None
|
||||
"""One prompt token count per response; each defaults to 0 if omitted.
|
||||
@@ -589,7 +689,7 @@ class DerenderStreamState(BaseModel):
|
||||
token in here through `parse_delta` (discarding the result) before
|
||||
processing the current chunk's tokens since parser internal state
|
||||
cannot be serialized into this stateless model. Unavoidably O(n)
|
||||
bounded by ``max_model_len`` (enforced server side, not by a field
|
||||
bounded by `max_model_len` (enforced server side, not by a field
|
||||
validator here since the bound is model dependent).
|
||||
"""
|
||||
|
||||
@@ -630,24 +730,25 @@ class DerenderChatStreamRequest(BaseModel):
|
||||
"""One chunk streaming derender request for /v1/chat/completions/derender.
|
||||
|
||||
The client sends one request per SSE chunk received from
|
||||
``/inference/v1/generate``. Each request carries the generate chunk
|
||||
plus the ``stream_state`` returned by the previous call (``None`` on the
|
||||
`/inference/v1/generate`. Each request carries the generate chunk
|
||||
plus the `stream_state` returned by the previous call (`None` on the
|
||||
first call). The response contains the derendered chunk and the updated
|
||||
state to be passed to the next call.
|
||||
|
||||
This implements stateless no server side session. All mutable state lives in
|
||||
the client carried ``stream_state``.
|
||||
the client carried `stream_state`.
|
||||
"""
|
||||
|
||||
# --8<-- [start:derender-chat-stream-request]
|
||||
stream: Literal[True]
|
||||
|
||||
model: str | None = None
|
||||
generate_chunk: GenerateStreamResponse
|
||||
"""One SSE chunk from ``/inference/v1/generate`` (``stream=True``)."""
|
||||
generate_chunk: GenerateTokensStreamResponse
|
||||
"""One `output_mode="tokens"` SSE chunk from `/inference/v1/generate`
|
||||
(`stream=True`)."""
|
||||
|
||||
stream_state: DerenderStreamState | None = None
|
||||
"""Client carried detok state from the previous call. ``None`` on first."""
|
||||
"""Client carried detok state from the previous call. `None` on first."""
|
||||
|
||||
prompt_tokens: int | None = None
|
||||
"""Prompt token count for usage. Forwarded from the render step."""
|
||||
@@ -655,7 +756,7 @@ class DerenderChatStreamRequest(BaseModel):
|
||||
prompt_token_ids: list[NonNegativeInt] | None = None
|
||||
"""Prompt token IDs. Required by the parser path's `parse_delta` to
|
||||
settle its initial reasoning state (e.g. chat templates that pre-open
|
||||
``<think>``). `prompt_tokens` is a usage count and cannot serve this
|
||||
`<think>`). `prompt_tokens` is a usage count and cannot serve this
|
||||
purpose. Sourced from `GenerateRequest.token_ids` at the render step.
|
||||
|
||||
Rejected with a 400 (by `ServingDerender`) when a tool or reasoning
|
||||
@@ -680,7 +781,7 @@ class DerenderChatStreamRequest(BaseModel):
|
||||
class DerenderCompletionStreamRequest(BaseModel):
|
||||
"""One chunk streaming derender request for /v1/completions/derender.
|
||||
|
||||
Parallel to ``DerenderChatStreamRequest`` for the completions endpoint.
|
||||
Parallel to `DerenderChatStreamRequest` for the completions endpoint.
|
||||
Each call processes one SSE chunk (one output sequence's delta) and
|
||||
returns the derendered chunk plus updated state.
|
||||
"""
|
||||
@@ -689,11 +790,11 @@ class DerenderCompletionStreamRequest(BaseModel):
|
||||
stream: Literal[True]
|
||||
|
||||
model: str | None = None
|
||||
generate_chunk: GenerateStreamResponse
|
||||
"""One SSE chunk from ``/inference/v1/generate``."""
|
||||
generate_chunk: GenerateTokensStreamResponse
|
||||
"""One `output_mode="tokens"` SSE chunk from `/inference/v1/generate`."""
|
||||
|
||||
stream_state: DerenderStreamState | None = None
|
||||
"""Client-carried detok state. ``None`` on the first call."""
|
||||
"""Client-carried detok state. `None` on the first call."""
|
||||
|
||||
prompt_tokens: int | None = None
|
||||
"""Prompt token count for usage."""
|
||||
@@ -723,16 +824,16 @@ class DerenderChatStreamResponse(BaseModel):
|
||||
class DerenderCompletionStreamResponse(BaseModel):
|
||||
"""Response for one streaming completions derender chunk.
|
||||
|
||||
Parallel to ``DerenderChatStreamResponse`` for the completions endpoint.
|
||||
Parallel to `DerenderChatStreamResponse` for the completions endpoint.
|
||||
"""
|
||||
|
||||
chunk: CompletionStreamResponse
|
||||
stream_state: DerenderStreamState
|
||||
|
||||
|
||||
# Determines the type by checking the ``stream`` field's literal value. A body without
|
||||
# ``stream`` validates as the non-streaming member
|
||||
# (``stream`` defaults to ``False`` there), so FastAPI can validate and dispatch both
|
||||
# Determines the type by checking the `stream` field's literal value. A body without
|
||||
# `stream` validates as the non-streaming member
|
||||
# (`stream` defaults to `False` there), so FastAPI can validate and dispatch both
|
||||
# shapes on a single path.
|
||||
DerenderChatRequestUnion: TypeAlias = DerenderChatRequest | DerenderChatStreamRequest
|
||||
DerenderCompletionRequestUnion: TypeAlias = (
|
||||
|
||||
@@ -6,6 +6,7 @@ import asyncio
|
||||
import time
|
||||
from collections.abc import AsyncGenerator
|
||||
from collections.abc import Sequence as GenericSequence
|
||||
from typing import Any
|
||||
|
||||
import msgspec
|
||||
from fastapi import Request
|
||||
@@ -20,6 +21,7 @@ from vllm.entrypoints.generate.base.serving import (
|
||||
GenerateBaseServing,
|
||||
build_spec_decoding_metrics,
|
||||
clamp_prompt_logprobs,
|
||||
format_token_id_placeholder,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionLogProb,
|
||||
@@ -46,6 +48,7 @@ from vllm.multimodal.inputs import (
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils.collection_utils import as_list
|
||||
from vllm.utils.serial_utils import numpy2base64
|
||||
|
||||
@@ -56,14 +59,45 @@ from .mm_features import (
|
||||
from .protocol import (
|
||||
GenerateRequest,
|
||||
GenerateResponse,
|
||||
GenerateResponseChoice,
|
||||
GenerateResponseStreamChoice,
|
||||
GenerateStreamResponse,
|
||||
GenerateTextChoice,
|
||||
GenerateTextResponse,
|
||||
GenerateTextStreamChoice,
|
||||
GenerateTextStreamResponse,
|
||||
GenerateTokensChoice,
|
||||
GenerateTokensResponse,
|
||||
GenerateTokensStreamChoice,
|
||||
GenerateTokensStreamResponse,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _logprob_token(
|
||||
token_id: int, logprob: Logprob | None, tokenizer: TokenizerLike | None
|
||||
) -> tuple[str, list[int] | None]:
|
||||
"""Token string and UTF-8 bytes for one logprob entry.
|
||||
|
||||
Without a tokenizer the token is a ``token_id:N`` placeholder with no bytes.
|
||||
"""
|
||||
if tokenizer is None:
|
||||
return format_token_id_placeholder(token_id), None
|
||||
token = (
|
||||
tokenizer.decode([token_id])
|
||||
if logprob is None
|
||||
else GenerateBaseServing._get_decoded_token(logprob, token_id, tokenizer)
|
||||
)
|
||||
return token, list(token.encode("utf-8", errors="replace"))
|
||||
|
||||
|
||||
def _top_logprob(
|
||||
token_id: int, logprob: Logprob, tokenizer: TokenizerLike | None
|
||||
) -> ChatCompletionLogProb:
|
||||
token, token_bytes = _logprob_token(token_id, logprob, tokenizer)
|
||||
return ChatCompletionLogProb(
|
||||
token=token, logprob=max(logprob.logprob, -9999.0), bytes=token_bytes
|
||||
)
|
||||
|
||||
|
||||
class ServingTokens(GenerateBaseServing):
|
||||
"""Provides Tokens IN <> Tokens OUT functionality to vLLM API."""
|
||||
|
||||
@@ -89,6 +123,9 @@ class ServingTokens(GenerateBaseServing):
|
||||
self.enable_prompt_tokens_details = enable_prompt_tokens_details
|
||||
self.enable_log_outputs = enable_log_outputs
|
||||
self.force_no_detokenize = force_no_detokenize
|
||||
self.has_tokenizer = (
|
||||
not force_no_detokenize and self.renderer.tokenizer is not None
|
||||
)
|
||||
if force_no_detokenize:
|
||||
logger.info(
|
||||
"Tokens-only mode is enabled, skipping detokenization "
|
||||
@@ -171,6 +208,15 @@ class ServingTokens(GenerateBaseServing):
|
||||
"because detokenization is disabled. Check stop strings on "
|
||||
"the coordinator, or use stop_token_ids."
|
||||
)
|
||||
if request.output_mode != "tokens" and not self.has_tokenizer:
|
||||
# The no-op detokenizer returns "", so without this check the
|
||||
# request would succeed with empty text.
|
||||
return self.create_error_response(
|
||||
f"output_mode={request.output_mode!r} requires a tokenizer, but "
|
||||
"this server does not load one (--tokens-only or "
|
||||
"--skip-tokenizer-init). Send the request to a server that "
|
||||
"loads a tokenizer, or use output_mode='tokens'."
|
||||
)
|
||||
try:
|
||||
msgspec.msgpack.encode(
|
||||
(
|
||||
@@ -331,6 +377,8 @@ class ServingTokens(GenerateBaseServing):
|
||||
created_time = int(time.time())
|
||||
final_res: RequestOutput | None = None
|
||||
sampling_params: SamplingParams = request.sampling_params
|
||||
text_mode = request.output_mode == "text"
|
||||
tokenizer = self._logprobs_tokenizer(text_mode)
|
||||
|
||||
try:
|
||||
async for res in result_generator:
|
||||
@@ -340,7 +388,8 @@ class ServingTokens(GenerateBaseServing):
|
||||
|
||||
assert final_res is not None
|
||||
|
||||
choices: list[GenerateResponseChoice] = []
|
||||
tokens_choices: list[GenerateTokensChoice] = []
|
||||
text_choices: list[GenerateTextChoice] = []
|
||||
num_generated_tokens = 0
|
||||
for output in final_res.outputs:
|
||||
self._raise_if_error(output.finish_reason, request_id)
|
||||
@@ -355,6 +404,7 @@ class ServingTokens(GenerateBaseServing):
|
||||
token_ids=token_ids,
|
||||
top_logprobs=out_logprobs,
|
||||
num_output_top_logprobs=sampling_params.logprobs,
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
else:
|
||||
logprobs = None
|
||||
@@ -369,7 +419,7 @@ class ServingTokens(GenerateBaseServing):
|
||||
if output.sampling_mask is not None:
|
||||
sampling_mask = output.sampling_mask.token_ids
|
||||
|
||||
choice_data = GenerateResponseChoice(
|
||||
choice_fields: dict[str, Any] = dict(
|
||||
index=output.index,
|
||||
logprobs=logprobs,
|
||||
finish_reason=output.finish_reason if output.finish_reason else "stop",
|
||||
@@ -377,8 +427,12 @@ class ServingTokens(GenerateBaseServing):
|
||||
routed_experts=routed_experts_b64,
|
||||
sampling_mask=sampling_mask,
|
||||
)
|
||||
|
||||
choices.append(choice_data)
|
||||
if text_mode:
|
||||
text_choices.append(
|
||||
GenerateTextChoice(text=output.text, **choice_fields)
|
||||
)
|
||||
else:
|
||||
tokens_choices.append(GenerateTokensChoice(**choice_fields))
|
||||
num_generated_tokens += len(output.token_ids)
|
||||
|
||||
assert final_res.prompt_token_ids is not None
|
||||
@@ -407,11 +461,10 @@ class ServingTokens(GenerateBaseServing):
|
||||
spec_stats = build_spec_decoding_metrics(final_res)
|
||||
if spec_stats is not None:
|
||||
per_request_metrics = PerRequestMetrics(speculative_decoding=spec_stats)
|
||||
response = GenerateResponse(
|
||||
response_fields: dict[str, Any] = dict(
|
||||
request_id=request_id,
|
||||
created=created_time,
|
||||
model=model_name,
|
||||
choices=choices,
|
||||
usage=usage,
|
||||
prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
|
||||
prompt_token_id_logprobs=(
|
||||
@@ -427,10 +480,15 @@ class ServingTokens(GenerateBaseServing):
|
||||
kv_transfer_params=final_res.kv_transfer_params,
|
||||
ec_transfer_params=final_res.ec_transfer_params,
|
||||
)
|
||||
response: GenerateTokensResponse | GenerateTextResponse
|
||||
if text_mode:
|
||||
response = GenerateTextResponse(choices=text_choices, **response_fields)
|
||||
else:
|
||||
response = GenerateTokensResponse(choices=tokens_choices, **response_fields)
|
||||
|
||||
# Log complete response if output logging is enabled
|
||||
if self.enable_log_outputs and self.request_logger:
|
||||
for choice in choices:
|
||||
for choice in response.choices:
|
||||
# Get the corresponding output token IDs
|
||||
output_token_ids = None
|
||||
if choice.index < len(final_res.outputs):
|
||||
@@ -464,6 +522,8 @@ class ServingTokens(GenerateBaseServing):
|
||||
num_cached_tokens = None
|
||||
sampling_params: SamplingParams = request.sampling_params
|
||||
last_res: RequestOutput | None = None
|
||||
text_mode = request.output_mode == "text"
|
||||
tokenizer = self._logprobs_tokenizer(text_mode)
|
||||
|
||||
include_usage, include_continuous_usage = should_include_usage(
|
||||
request.stream_options, False
|
||||
@@ -491,11 +551,21 @@ class ServingTokens(GenerateBaseServing):
|
||||
finish_reason = output.finish_reason
|
||||
self._raise_if_error(finish_reason, request_id)
|
||||
|
||||
# Still emit a terminal empty chunk while prompt metadata
|
||||
# is pending, so zero-token completions deliver it.
|
||||
if not delta_token_ids and (
|
||||
finish_reason is None or prompt_token_ids is None
|
||||
):
|
||||
if text_mode:
|
||||
# Text held back for stop string matching is flushed
|
||||
# with the final output, and the abort output has no
|
||||
# new token IDs, so text or a finish reason is enough.
|
||||
emit = bool(
|
||||
delta_token_ids or output.text or finish_reason is not None
|
||||
)
|
||||
else:
|
||||
# Still emit a terminal empty chunk while prompt
|
||||
# metadata is pending, so zero-token completions
|
||||
# deliver it.
|
||||
emit = bool(delta_token_ids) or (
|
||||
finish_reason is not None and prompt_token_ids is not None
|
||||
)
|
||||
if not emit:
|
||||
continue
|
||||
|
||||
if sampling_params.logprobs is not None:
|
||||
@@ -505,6 +575,7 @@ class ServingTokens(GenerateBaseServing):
|
||||
token_ids=delta_token_ids,
|
||||
top_logprobs=out_logprobs,
|
||||
num_output_top_logprobs=sampling_params.logprobs,
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
else:
|
||||
logprobs = None
|
||||
@@ -518,20 +589,30 @@ class ServingTokens(GenerateBaseServing):
|
||||
sampling_mask = None
|
||||
if output.sampling_mask is not None:
|
||||
sampling_mask = output.sampling_mask.token_ids
|
||||
|
||||
chunk = GenerateStreamResponse(
|
||||
request_id=request_id,
|
||||
choices=[
|
||||
GenerateResponseStreamChoice(
|
||||
index=i,
|
||||
logprobs=logprobs,
|
||||
finish_reason=finish_reason,
|
||||
token_ids=as_list(delta_token_ids),
|
||||
routed_experts=routed_experts_b64,
|
||||
sampling_mask=sampling_mask,
|
||||
)
|
||||
],
|
||||
choice_fields: dict[str, Any] = dict(
|
||||
index=i,
|
||||
logprobs=logprobs,
|
||||
finish_reason=finish_reason,
|
||||
token_ids=as_list(delta_token_ids),
|
||||
routed_experts=routed_experts_b64,
|
||||
sampling_mask=sampling_mask,
|
||||
)
|
||||
chunk: GenerateTokensStreamResponse | GenerateTextStreamResponse
|
||||
if text_mode:
|
||||
chunk = GenerateTextStreamResponse(
|
||||
request_id=request_id,
|
||||
choices=[
|
||||
GenerateTextStreamChoice(
|
||||
text=output.text, **choice_fields
|
||||
)
|
||||
],
|
||||
)
|
||||
else:
|
||||
chunk = GenerateTokensStreamResponse(
|
||||
request_id=request_id,
|
||||
choices=[GenerateTokensStreamChoice(**choice_fields)],
|
||||
)
|
||||
|
||||
if prompt_token_ids is not None:
|
||||
chunk.prompt_token_ids = prompt_token_ids
|
||||
chunk.mm_placeholders = request._response_mm_placeholders
|
||||
@@ -571,12 +652,21 @@ class ServingTokens(GenerateBaseServing):
|
||||
per_request_metrics = PerRequestMetrics(
|
||||
speculative_decoding=spec_stats
|
||||
)
|
||||
final_chunk = GenerateStreamResponse(
|
||||
request_id=request_id,
|
||||
choices=[],
|
||||
usage=final_usage_info,
|
||||
metrics=per_request_metrics,
|
||||
)
|
||||
final_chunk: GenerateTokensStreamResponse | GenerateTextStreamResponse
|
||||
if text_mode:
|
||||
final_chunk = GenerateTextStreamResponse(
|
||||
request_id=request_id,
|
||||
choices=[],
|
||||
usage=final_usage_info,
|
||||
metrics=per_request_metrics,
|
||||
)
|
||||
else:
|
||||
final_chunk = GenerateTokensStreamResponse(
|
||||
request_id=request_id,
|
||||
choices=[],
|
||||
usage=final_usage_info,
|
||||
metrics=per_request_metrics,
|
||||
)
|
||||
yield f"data: {final_chunk.model_dump_json(exclude_none=True)}\n\n"
|
||||
|
||||
request_metadata.final_usage_info = final_usage_info
|
||||
@@ -591,37 +681,49 @@ class ServingTokens(GenerateBaseServing):
|
||||
yield f"data: {data}\n\n"
|
||||
yield "data: [DONE]\n\n"
|
||||
|
||||
def _logprobs_tokenizer(self, text_mode: bool) -> TokenizerLike | None:
|
||||
"""Tokenizer that resolves logprob tokens, or None for placeholders.
|
||||
|
||||
``--return-tokens-as-token-ids`` keeps placeholders at every level, as
|
||||
it does on ``/v1/completions``.
|
||||
"""
|
||||
if not text_mode or self.return_tokens_as_token_ids:
|
||||
return None
|
||||
return self.renderer.tokenizer
|
||||
|
||||
def _create_tokens_logprobs(
|
||||
self,
|
||||
token_ids: GenericSequence[int],
|
||||
top_logprobs: GenericSequence[dict[int, Logprob] | None],
|
||||
num_output_top_logprobs: int | None = None,
|
||||
tokenizer: TokenizerLike | None = None,
|
||||
) -> ChatCompletionLogProbs:
|
||||
"""Create OpenAI-style logprobs."""
|
||||
"""Create OpenAI-style logprobs.
|
||||
|
||||
Tokens are ``token_id:N`` placeholders. With a ``tokenizer`` they are
|
||||
decoded strings and carry ``bytes``.
|
||||
"""
|
||||
logprobs_content: list[ChatCompletionLogProbsContent] = []
|
||||
|
||||
for i, token_id in enumerate(token_ids):
|
||||
token = f"token_id:{token_id}"
|
||||
step_top_logprobs = top_logprobs[i]
|
||||
if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
|
||||
token, token_bytes = _logprob_token(token_id, None, tokenizer)
|
||||
logprobs_content.append(
|
||||
ChatCompletionLogProbsContent(
|
||||
token=token,
|
||||
)
|
||||
ChatCompletionLogProbsContent(token=token, bytes=token_bytes)
|
||||
)
|
||||
else:
|
||||
step_token = step_top_logprobs[token_id]
|
||||
token, token_bytes = _logprob_token(token_id, step_token, tokenizer)
|
||||
|
||||
logprobs_content.append(
|
||||
ChatCompletionLogProbsContent(
|
||||
token=token,
|
||||
logprob=max(step_token.logprob, -9999.0),
|
||||
bytes=token_bytes,
|
||||
top_logprobs=[
|
||||
ChatCompletionLogProb(
|
||||
token=f"token_id:{token_id}",
|
||||
logprob=max(logprob.logprob, -9999.0),
|
||||
)
|
||||
for i, (token_id, logprob) in enumerate(
|
||||
_top_logprob(top_id, logprob, tokenizer)
|
||||
for i, (top_id, logprob) in enumerate(
|
||||
step_top_logprobs.items()
|
||||
)
|
||||
if num_output_top_logprobs is not None
|
||||
|
||||
Reference in New Issue
Block a user