[Frontend] Add detokenization streaming derender for disaggregated serving (#47301)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
@@ -5,15 +5,17 @@ from http import HTTPStatus
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionResponse
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionResponse
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from ..token_in_token_out.protocol import (
|
||||
DerenderChatRequest,
|
||||
DerenderCompletionRequest,
|
||||
DerenderChatRequestUnion,
|
||||
DerenderChatStreamRequest,
|
||||
DerenderChatStreamResponse,
|
||||
DerenderCompletionRequestUnion,
|
||||
DerenderCompletionStreamRequest,
|
||||
DerenderCompletionStreamResponse,
|
||||
)
|
||||
from .serving import ServingDerender
|
||||
|
||||
@@ -29,46 +31,96 @@ def derender(request: Request) -> ServingDerender | None:
|
||||
@router.post(
|
||||
"/v1/chat/completions/derender",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=ChatCompletionResponse,
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def derender_chat_completion(request: DerenderChatRequest, raw_request: Request):
|
||||
async def derender_chat_completion(
|
||||
request: DerenderChatRequestUnion,
|
||||
raw_request: Request,
|
||||
):
|
||||
"""Derender a generate response into a ChatCompletionResponse.
|
||||
|
||||
Accepts both non-streaming (``stream=false``, default) and streaming
|
||||
(``stream=true``) request bodies on the same path; FastAPI validates and
|
||||
routes on the ``stream`` discriminator.
|
||||
|
||||
Non-streaming: body is ``DerenderChatRequest`` (``generate_response`` with
|
||||
the complete token list). Returns a ``ChatCompletionResponse``.
|
||||
|
||||
Streaming: body is ``DerenderChatStreamRequest`` (one ``generate_chunk``
|
||||
delta + optional ``stream_state``). Returns a ``DerenderChatStreamResponse``
|
||||
(``chunk`` + ``stream_state``). The client carries ``stream_state`` between
|
||||
successive calls, one per SSE chunk from ``/inference/v1/generate``.
|
||||
"""
|
||||
handler = derender(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError(
|
||||
"The model does not support Chat Completions Derender API"
|
||||
)
|
||||
|
||||
result = await handler.derender_chat_response(request)
|
||||
if isinstance(request, DerenderChatStreamRequest):
|
||||
stream_result = await handler.derender_chat_stream_response(request)
|
||||
if isinstance(stream_result, ErrorResponse):
|
||||
return JSONResponse(
|
||||
content=stream_result.model_dump(),
|
||||
status_code=stream_result.error.code,
|
||||
)
|
||||
chunk, stream_state = stream_result
|
||||
response = DerenderChatStreamResponse(chunk=chunk, stream_state=stream_state)
|
||||
return JSONResponse(content=response.model_dump())
|
||||
|
||||
result = await handler.derender_chat_response(request)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/completions/derender",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=CompletionResponse,
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def derender_completion(request: DerenderCompletionRequest, raw_request: Request):
|
||||
async def derender_completion(
|
||||
request: DerenderCompletionRequestUnion,
|
||||
raw_request: Request,
|
||||
):
|
||||
"""Derender a generate response into a CompletionResponse.
|
||||
|
||||
Accepts both non-streaming (``stream=false``, default) and streaming
|
||||
(``stream=true``) request bodies on the same path.
|
||||
|
||||
Non-streaming: body is ``DerenderCompletionRequest``. Returns a
|
||||
``CompletionResponse``.
|
||||
|
||||
Streaming: body is ``DerenderCompletionStreamRequest`` (one
|
||||
``generate_chunk`` + optional ``stream_state``). Returns a
|
||||
``DerenderCompletionStreamResponse`` (``chunk`` + ``stream_state``).
|
||||
"""
|
||||
handler = derender(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Completions Derender API")
|
||||
|
||||
result = await handler.derender_completion_response(request)
|
||||
if isinstance(request, DerenderCompletionStreamRequest):
|
||||
stream_result = await handler.derender_completion_stream_response(request)
|
||||
if isinstance(stream_result, ErrorResponse):
|
||||
return JSONResponse(
|
||||
content=stream_result.model_dump(),
|
||||
status_code=stream_result.error.code,
|
||||
)
|
||||
chunk, stream_state = stream_result
|
||||
response = DerenderCompletionStreamResponse(
|
||||
chunk=chunk, stream_state=stream_state
|
||||
)
|
||||
return JSONResponse(content=response.model_dump())
|
||||
|
||||
result = await handler.derender_completion_response(request)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
@@ -4,8 +4,14 @@ import time
|
||||
from typing import cast
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionResponse
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionResponse
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionResponse,
|
||||
ChatCompletionStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import (
|
||||
CompletionResponse,
|
||||
CompletionStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
UsageInfo,
|
||||
@@ -28,7 +34,10 @@ from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||
from ..token_in_token_out.mm_serde import encode_mm_kwargs_item
|
||||
from ..token_in_token_out.protocol import (
|
||||
DerenderChatRequest,
|
||||
DerenderChatStreamRequest,
|
||||
DerenderCompletionRequest,
|
||||
DerenderCompletionStreamRequest,
|
||||
DerenderStreamState,
|
||||
GenerateResponse,
|
||||
MultiModalFeatures,
|
||||
PlaceholderRangeInfo,
|
||||
@@ -200,7 +209,9 @@ class ServingDerender(BaseServing):
|
||||
total_prompt_tokens,
|
||||
total_completion_tokens,
|
||||
) = await self.online_derenderer.derender_completion(
|
||||
request.generate_responses, request.prompt_tokens
|
||||
request.generate_responses,
|
||||
request.prompt_tokens,
|
||||
completion_request=request.completion_request,
|
||||
)
|
||||
|
||||
first = request.generate_responses[0]
|
||||
@@ -237,6 +248,90 @@ class ServingDerender(BaseServing):
|
||||
kv_transfer_params=kv_params,
|
||||
)
|
||||
|
||||
async def derender_chat_stream_response(
|
||||
self,
|
||||
request: DerenderChatStreamRequest,
|
||||
) -> tuple[ChatCompletionStreamResponse, DerenderStreamState] | ErrorResponse:
|
||||
"""Streaming counterpart to ``derender_chat_response``.
|
||||
|
||||
Processes one ``GenerateStreamResponse`` chunk and returns the
|
||||
derendered chunk together with the updated client carried state.
|
||||
|
||||
``parser is None`` or no ``chat_request`` until reasoning/tool call
|
||||
functionality added in future PR.
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
try:
|
||||
chunk, updated_state = await self.online_derenderer.derender_chat_stream(
|
||||
model=request.model,
|
||||
generate_chunk=request.generate_chunk,
|
||||
state=request.stream_state,
|
||||
chat_request=request.chat_request,
|
||||
prompt_tokens=request.prompt_tokens,
|
||||
)
|
||||
except NotImplementedError as exc:
|
||||
return self.create_error_response(exc)
|
||||
except ValueError as exc:
|
||||
return self.create_error_response(str(exc))
|
||||
except (KeyError, IndexError) as exc:
|
||||
return self.create_error_response(
|
||||
f"invalid stream_state: detokenization failed ({exc!r})"
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"derender_chat_stream request_id=%s model=%s delta_tokens=%d",
|
||||
request.generate_chunk.request_id,
|
||||
request.model,
|
||||
sum(
|
||||
len(c.token_ids) for c in request.generate_chunk.choices if c.token_ids
|
||||
),
|
||||
)
|
||||
return chunk, updated_state
|
||||
|
||||
async def derender_completion_stream_response(
|
||||
self,
|
||||
request: DerenderCompletionStreamRequest,
|
||||
) -> tuple[CompletionStreamResponse, DerenderStreamState] | ErrorResponse:
|
||||
"""Streaming counterpart to ``derender_completion_response``.
|
||||
|
||||
Processes one ``GenerateStreamResponse`` chunk (one output sequence's
|
||||
delta) and returns the derendered chunk and updated state.
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
try:
|
||||
(
|
||||
chunk,
|
||||
updated_state,
|
||||
) = await self.online_derenderer.derender_completion_stream(
|
||||
model=request.model,
|
||||
generate_chunk=request.generate_chunk,
|
||||
state=request.stream_state,
|
||||
prompt_tokens=request.prompt_tokens,
|
||||
completion_request=request.completion_request,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.create_error_response(str(exc))
|
||||
except (KeyError, IndexError) as exc:
|
||||
return self.create_error_response(
|
||||
f"invalid stream_state: detokenization failed ({exc!r})"
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"derender_completion_stream request_id=%s model=%s delta_tokens=%d",
|
||||
request.generate_chunk.request_id,
|
||||
request.model,
|
||||
sum(
|
||||
len(c.token_ids) for c in request.generate_chunk.choices if c.token_ids
|
||||
),
|
||||
)
|
||||
return chunk, updated_state
|
||||
|
||||
@staticmethod
|
||||
def _extract_mm_features(
|
||||
engine_input: EngineInput,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
from typing import Any, Literal, TypeAlias
|
||||
|
||||
from pydantic import (
|
||||
BaseModel,
|
||||
@@ -14,8 +14,12 @@ from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionLogProbs,
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import (
|
||||
CompletionRequest,
|
||||
CompletionStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.engine.protocol import StreamOptions, UsageInfo
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.renderers import TokenizeParams
|
||||
@@ -252,21 +256,16 @@ class GenerateResponse(BaseModel):
|
||||
)
|
||||
|
||||
|
||||
####### Derender (postprocessing) #######
|
||||
|
||||
|
||||
class DerenderChatRequest(BaseModel):
|
||||
"""Request for the /v1/chat/completions/derender endpoint (non-streaming).
|
||||
|
||||
Wraps a complete GenerateResponse and caller-supplied metadata needed to
|
||||
produce a fully-formed ChatCompletionResponse without a GPU.
|
||||
|
||||
Streaming derender would require a separate endpoint design with
|
||||
incremental token delivery, ``OutputProcessor``-based detokenization,
|
||||
and ``parser.parse_delta()`` instead of ``parser.parse()``.
|
||||
Wraps a complete GenerateResponse and caller supplied metadata needed to
|
||||
produce a fully formed ChatCompletionResponse without a GPU.
|
||||
"""
|
||||
|
||||
# --8<-- [start:derender-chat-request]
|
||||
stream: Literal[False] = False
|
||||
|
||||
model: str
|
||||
"""Served model name."""
|
||||
|
||||
@@ -299,6 +298,8 @@ class DerenderCompletionRequest(BaseModel):
|
||||
"""
|
||||
|
||||
# --8<-- [start:derender-completion-request]
|
||||
stream: Literal[False] = False
|
||||
|
||||
model: str
|
||||
"""Served model name."""
|
||||
|
||||
@@ -330,3 +331,156 @@ class DerenderCompletionRequest(BaseModel):
|
||||
f"generate_responses length ({len(self.generate_responses)})"
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class DerenderStreamState(BaseModel):
|
||||
"""Per sequence state for stateless streaming derender.
|
||||
|
||||
The client carries this between successive per chunk HTTP calls to the
|
||||
streaming derender endpoint. All fields are plain JSON serializable data.
|
||||
No opaque tokenizer or parser internals are stored here.
|
||||
|
||||
The detokenization strategy carries the incremental decode offsets
|
||||
directly rather than re-sending the whole token history each chunk.
|
||||
``detokenize_incrementally`` only ever reads the trailing token window
|
||||
``prev_tokens[prefix_offset:]``, so we carry just that tail plus the two
|
||||
offsets. Each chunk resumes exactly where the last one stopped, including
|
||||
any partially processed multi-byte character (tracked by ``read_offset``),
|
||||
then trims and rebases the window so it never grows with generation length.
|
||||
|
||||
Performance:
|
||||
- Compute per chunk is O(delta). One ``detokenize_incrementally`` call per
|
||||
new token, independent of how many tokens preceded it.
|
||||
- Transport per chunk is O(window). The carried tail is bounded by the
|
||||
incremental detokenization offset, so cumulative bytes over the wire are
|
||||
O(n) rather than the O(n^2) a full history round trip would incur.
|
||||
"""
|
||||
|
||||
prev_tokens: list[str] = Field(default_factory=list)
|
||||
"""Trailing decode window. Token strings from ``prefix_offset`` onward.
|
||||
|
||||
Bounded, trimmed and rebased each chunk to the tail
|
||||
``detokenize_incrementally`` still reads, so it does not grow with the
|
||||
number of chunks.
|
||||
"""
|
||||
|
||||
prefix_offset: int = Field(default=0, ge=0)
|
||||
"""Prefix offset into ``prev_tokens`` for incremental detokenization."""
|
||||
|
||||
read_offset: int = Field(default=0, ge=0)
|
||||
"""Read offset into ``prev_tokens`` for incremental detokenization."""
|
||||
|
||||
@field_validator("prev_tokens")
|
||||
@classmethod
|
||||
def _bound_prev_tokens(cls, v: list[str]) -> list[str]:
|
||||
# INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET is small (5) and the trimmed
|
||||
# window is O(offset). A generous limit rejects unusually large or malformed
|
||||
# payloads without restricting legitimate multi-byte sequences.
|
||||
limit = 1024
|
||||
if len(v) > limit:
|
||||
raise ValueError(f"prev_tokens length ({len(v)}) exceeds maximum ({limit})")
|
||||
return v
|
||||
|
||||
role_sent: bool = False
|
||||
"""True once the initial ``role: "assistant"`` delta has been emitted.
|
||||
|
||||
Prevents re-emitting the role on subsequent chunks even when the detok
|
||||
window is transiently empty (e.g. usage only final chunk).
|
||||
"""
|
||||
|
||||
# TODO: Properties used in follow on PR for tool call parsing
|
||||
last_content: str | None = None
|
||||
"""Last emitted cumulative assistant content text."""
|
||||
|
||||
last_reasoning: str | None = None
|
||||
"""Last emitted cumulative reasoning text."""
|
||||
|
||||
last_tool_call_ids: list[str] = Field(default_factory=list)
|
||||
"""Stable tool-call IDs, assigned once when each call first appears.
|
||||
|
||||
Prevents ID regeneration across re-parsing.
|
||||
"""
|
||||
|
||||
|
||||
class DerenderChatStreamRequest(BaseModel):
|
||||
"""One chunk streaming derender request for /v1/chat/completions/derender.
|
||||
|
||||
The client sends one request per SSE chunk received from
|
||||
``/inference/v1/generate``. Each request carries the generate chunk
|
||||
plus the ``stream_state`` returned by the previous call (``None`` on the
|
||||
first call). The response contains the derendered chunk and the updated
|
||||
state to be passed to the next call.
|
||||
|
||||
This implements stateless no server side session. All mutable state lives in
|
||||
the client carried ``stream_state``.
|
||||
"""
|
||||
|
||||
stream: Literal[True]
|
||||
|
||||
model: str
|
||||
generate_chunk: GenerateStreamResponse
|
||||
"""One SSE chunk from ``/inference/v1/generate`` (``stream=True``)."""
|
||||
|
||||
stream_state: DerenderStreamState | None = None
|
||||
"""Client carried detok state from the previous call. ``None`` on first."""
|
||||
|
||||
prompt_tokens: int | None = None
|
||||
"""Prompt token count for usage. Forwarded from the render step."""
|
||||
|
||||
chat_request: ChatCompletionRequest | None = None
|
||||
"""The original (post adjust_request) ChatCompletionRequest from /render."""
|
||||
|
||||
|
||||
class DerenderCompletionStreamRequest(BaseModel):
|
||||
"""One chunk streaming derender request for /v1/completions/derender.
|
||||
|
||||
Parallel to ``DerenderChatStreamRequest`` for the completions endpoint.
|
||||
Each call processes one SSE chunk (one output sequence's delta) and
|
||||
returns the derendered chunk plus updated state.
|
||||
"""
|
||||
|
||||
stream: Literal[True]
|
||||
|
||||
model: str
|
||||
generate_chunk: GenerateStreamResponse
|
||||
"""One SSE chunk from ``/inference/v1/generate``."""
|
||||
|
||||
stream_state: DerenderStreamState | None = None
|
||||
"""Client-carried detok state. ``None`` on the first call."""
|
||||
|
||||
prompt_tokens: int | None = None
|
||||
"""Prompt token count for usage."""
|
||||
|
||||
completion_request: CompletionRequest | None = None
|
||||
"""The original (post adjust_request) CompletionRequest from /render."""
|
||||
|
||||
|
||||
class DerenderChatStreamResponse(BaseModel):
|
||||
"""Response for one streaming chat derender chunk.
|
||||
|
||||
Pairs the derendered SSE chunk with the updated client carried state to
|
||||
pass to the next call.
|
||||
"""
|
||||
|
||||
chunk: ChatCompletionStreamResponse
|
||||
stream_state: DerenderStreamState
|
||||
|
||||
|
||||
class DerenderCompletionStreamResponse(BaseModel):
|
||||
"""Response for one streaming completions derender chunk.
|
||||
|
||||
Parallel to ``DerenderChatStreamResponse`` for the completions endpoint.
|
||||
"""
|
||||
|
||||
chunk: CompletionStreamResponse
|
||||
stream_state: DerenderStreamState
|
||||
|
||||
|
||||
# Determines the type by checking the ``stream`` field's literal value. A body without
|
||||
# ``stream`` validates as the non-streaming member
|
||||
# (``stream`` defaults to ``False`` there), so FastAPI can validate and dispatch both
|
||||
# shapes on a single path.
|
||||
DerenderChatRequestUnion: TypeAlias = DerenderChatRequest | DerenderChatStreamRequest
|
||||
DerenderCompletionRequestUnion: TypeAlias = (
|
||||
DerenderCompletionRequest | DerenderCompletionStreamRequest
|
||||
)
|
||||
|
||||
@@ -17,7 +17,9 @@ from vllm.entrypoints.openai.completion.protocol import (
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
DerenderChatRequest,
|
||||
DerenderChatStreamRequest,
|
||||
DerenderCompletionRequest,
|
||||
DerenderCompletionStreamRequest,
|
||||
GenerateRequest,
|
||||
GenerateResponse,
|
||||
)
|
||||
@@ -54,6 +56,7 @@ CompletionLikeRequest: TypeAlias = (
|
||||
| TokenizeCompletionRequest
|
||||
| DetokenizeRequest
|
||||
| DerenderCompletionRequest
|
||||
| DerenderCompletionStreamRequest
|
||||
)
|
||||
|
||||
ChatLikeRequest: TypeAlias = (
|
||||
@@ -61,6 +64,7 @@ ChatLikeRequest: TypeAlias = (
|
||||
| BatchChatCompletionRequest
|
||||
| TokenizeChatRequest
|
||||
| DerenderChatRequest
|
||||
| DerenderChatStreamRequest
|
||||
)
|
||||
|
||||
SpeechToTextRequest: TypeAlias = TranscriptionRequest | TranslationRequest
|
||||
|
||||
Reference in New Issue
Block a user