[Frontend] Add detokenization streaming derender for disaggregated serving (#47301)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
Martin Hickey
2026-07-29 18:01:20 -07:00
committed by GitHub
parent 451227cb3f
commit 0a31372e5f
6 changed files with 1377 additions and 32 deletions
@@ -5,15 +5,17 @@ from http import HTTPStatus
from fastapi import APIRouter, Depends, Request
from fastapi.responses import JSONResponse
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionResponse
from vllm.entrypoints.openai.completion.protocol import CompletionResponse
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
from vllm.logger import init_logger
from ..token_in_token_out.protocol import (
DerenderChatRequest,
DerenderCompletionRequest,
DerenderChatRequestUnion,
DerenderChatStreamRequest,
DerenderChatStreamResponse,
DerenderCompletionRequestUnion,
DerenderCompletionStreamRequest,
DerenderCompletionStreamResponse,
)
from .serving import ServingDerender
@@ -29,46 +31,96 @@ def derender(request: Request) -> ServingDerender | None:
@router.post(
"/v1/chat/completions/derender",
dependencies=[Depends(validate_json_request)],
response_model=ChatCompletionResponse,
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def derender_chat_completion(request: DerenderChatRequest, raw_request: Request):
async def derender_chat_completion(
request: DerenderChatRequestUnion,
raw_request: Request,
):
"""Derender a generate response into a ChatCompletionResponse.
Accepts both non-streaming (``stream=false``, default) and streaming
(``stream=true``) request bodies on the same path; FastAPI validates and
routes on the ``stream`` discriminator.
Non-streaming: body is ``DerenderChatRequest`` (``generate_response`` with
the complete token list). Returns a ``ChatCompletionResponse``.
Streaming: body is ``DerenderChatStreamRequest`` (one ``generate_chunk``
delta + optional ``stream_state``). Returns a ``DerenderChatStreamResponse``
(``chunk`` + ``stream_state``). The client carries ``stream_state`` between
successive calls, one per SSE chunk from ``/inference/v1/generate``.
"""
handler = derender(raw_request)
if handler is None:
raise NotImplementedError(
"The model does not support Chat Completions Derender API"
)
result = await handler.derender_chat_response(request)
if isinstance(request, DerenderChatStreamRequest):
stream_result = await handler.derender_chat_stream_response(request)
if isinstance(stream_result, ErrorResponse):
return JSONResponse(
content=stream_result.model_dump(),
status_code=stream_result.error.code,
)
chunk, stream_state = stream_result
response = DerenderChatStreamResponse(chunk=chunk, stream_state=stream_state)
return JSONResponse(content=response.model_dump())
result = await handler.derender_chat_response(request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
@router.post(
"/v1/completions/derender",
dependencies=[Depends(validate_json_request)],
response_model=CompletionResponse,
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def derender_completion(request: DerenderCompletionRequest, raw_request: Request):
async def derender_completion(
request: DerenderCompletionRequestUnion,
raw_request: Request,
):
"""Derender a generate response into a CompletionResponse.
Accepts both non-streaming (``stream=false``, default) and streaming
(``stream=true``) request bodies on the same path.
Non-streaming: body is ``DerenderCompletionRequest``. Returns a
``CompletionResponse``.
Streaming: body is ``DerenderCompletionStreamRequest`` (one
``generate_chunk`` + optional ``stream_state``). Returns a
``DerenderCompletionStreamResponse`` (``chunk`` + ``stream_state``).
"""
handler = derender(raw_request)
if handler is None:
raise NotImplementedError("The model does not support Completions Derender API")
result = await handler.derender_completion_response(request)
if isinstance(request, DerenderCompletionStreamRequest):
stream_result = await handler.derender_completion_stream_response(request)
if isinstance(stream_result, ErrorResponse):
return JSONResponse(
content=stream_result.model_dump(),
status_code=stream_result.error.code,
)
chunk, stream_state = stream_result
response = DerenderCompletionStreamResponse(
chunk=chunk, stream_state=stream_state
)
return JSONResponse(content=response.model_dump())
result = await handler.derender_completion_response(request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
+98 -3
View File
@@ -4,8 +4,14 @@ import time
from typing import cast
import vllm.envs as envs
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionResponse
from vllm.entrypoints.openai.completion.protocol import CompletionResponse
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionResponse,
ChatCompletionStreamResponse,
)
from vllm.entrypoints.openai.completion.protocol import (
CompletionResponse,
CompletionStreamResponse,
)
from vllm.entrypoints.openai.engine.protocol import (
ErrorResponse,
UsageInfo,
@@ -28,7 +34,10 @@ from vllm.renderers.online_derenderer import OnlineDerenderer
from ..token_in_token_out.mm_serde import encode_mm_kwargs_item
from ..token_in_token_out.protocol import (
DerenderChatRequest,
DerenderChatStreamRequest,
DerenderCompletionRequest,
DerenderCompletionStreamRequest,
DerenderStreamState,
GenerateResponse,
MultiModalFeatures,
PlaceholderRangeInfo,
@@ -200,7 +209,9 @@ class ServingDerender(BaseServing):
total_prompt_tokens,
total_completion_tokens,
) = await self.online_derenderer.derender_completion(
request.generate_responses, request.prompt_tokens
request.generate_responses,
request.prompt_tokens,
completion_request=request.completion_request,
)
first = request.generate_responses[0]
@@ -237,6 +248,90 @@ class ServingDerender(BaseServing):
kv_transfer_params=kv_params,
)
async def derender_chat_stream_response(
self,
request: DerenderChatStreamRequest,
) -> tuple[ChatCompletionStreamResponse, DerenderStreamState] | ErrorResponse:
"""Streaming counterpart to ``derender_chat_response``.
Processes one ``GenerateStreamResponse`` chunk and returns the
derendered chunk together with the updated client carried state.
``parser is None`` or no ``chat_request`` until reasoning/tool call
functionality added in future PR.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
try:
chunk, updated_state = await self.online_derenderer.derender_chat_stream(
model=request.model,
generate_chunk=request.generate_chunk,
state=request.stream_state,
chat_request=request.chat_request,
prompt_tokens=request.prompt_tokens,
)
except NotImplementedError as exc:
return self.create_error_response(exc)
except ValueError as exc:
return self.create_error_response(str(exc))
except (KeyError, IndexError) as exc:
return self.create_error_response(
f"invalid stream_state: detokenization failed ({exc!r})"
)
logger.debug(
"derender_chat_stream request_id=%s model=%s delta_tokens=%d",
request.generate_chunk.request_id,
request.model,
sum(
len(c.token_ids) for c in request.generate_chunk.choices if c.token_ids
),
)
return chunk, updated_state
async def derender_completion_stream_response(
self,
request: DerenderCompletionStreamRequest,
) -> tuple[CompletionStreamResponse, DerenderStreamState] | ErrorResponse:
"""Streaming counterpart to ``derender_completion_response``.
Processes one ``GenerateStreamResponse`` chunk (one output sequence's
delta) and returns the derendered chunk and updated state.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
try:
(
chunk,
updated_state,
) = await self.online_derenderer.derender_completion_stream(
model=request.model,
generate_chunk=request.generate_chunk,
state=request.stream_state,
prompt_tokens=request.prompt_tokens,
completion_request=request.completion_request,
)
except ValueError as exc:
return self.create_error_response(str(exc))
except (KeyError, IndexError) as exc:
return self.create_error_response(
f"invalid stream_state: detokenization failed ({exc!r})"
)
logger.debug(
"derender_completion_stream request_id=%s model=%s delta_tokens=%d",
request.generate_chunk.request_id,
request.model,
sum(
len(c.token_ids) for c in request.generate_chunk.choices if c.token_ids
),
)
return chunk, updated_state
@staticmethod
def _extract_mm_features(
engine_input: EngineInput,
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
from typing import Any, Literal, TypeAlias
from pydantic import (
BaseModel,
@@ -14,8 +14,12 @@ from vllm.config import ModelConfig
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionLogProbs,
ChatCompletionRequest,
ChatCompletionStreamResponse,
)
from vllm.entrypoints.openai.completion.protocol import (
CompletionRequest,
CompletionStreamResponse,
)
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.engine.protocol import StreamOptions, UsageInfo
from vllm.logprobs import Logprob
from vllm.renderers import TokenizeParams
@@ -252,21 +256,16 @@ class GenerateResponse(BaseModel):
)
####### Derender (postprocessing) #######
class DerenderChatRequest(BaseModel):
"""Request for the /v1/chat/completions/derender endpoint (non-streaming).
Wraps a complete GenerateResponse and caller-supplied metadata needed to
produce a fully-formed ChatCompletionResponse without a GPU.
Streaming derender would require a separate endpoint design with
incremental token delivery, ``OutputProcessor``-based detokenization,
and ``parser.parse_delta()`` instead of ``parser.parse()``.
Wraps a complete GenerateResponse and caller supplied metadata needed to
produce a fully formed ChatCompletionResponse without a GPU.
"""
# --8<-- [start:derender-chat-request]
stream: Literal[False] = False
model: str
"""Served model name."""
@@ -299,6 +298,8 @@ class DerenderCompletionRequest(BaseModel):
"""
# --8<-- [start:derender-completion-request]
stream: Literal[False] = False
model: str
"""Served model name."""
@@ -330,3 +331,156 @@ class DerenderCompletionRequest(BaseModel):
f"generate_responses length ({len(self.generate_responses)})"
)
return self
class DerenderStreamState(BaseModel):
"""Per sequence state for stateless streaming derender.
The client carries this between successive per chunk HTTP calls to the
streaming derender endpoint. All fields are plain JSON serializable data.
No opaque tokenizer or parser internals are stored here.
The detokenization strategy carries the incremental decode offsets
directly rather than re-sending the whole token history each chunk.
``detokenize_incrementally`` only ever reads the trailing token window
``prev_tokens[prefix_offset:]``, so we carry just that tail plus the two
offsets. Each chunk resumes exactly where the last one stopped, including
any partially processed multi-byte character (tracked by ``read_offset``),
then trims and rebases the window so it never grows with generation length.
Performance:
- Compute per chunk is O(delta). One ``detokenize_incrementally`` call per
new token, independent of how many tokens preceded it.
- Transport per chunk is O(window). The carried tail is bounded by the
incremental detokenization offset, so cumulative bytes over the wire are
O(n) rather than the O(n^2) a full history round trip would incur.
"""
prev_tokens: list[str] = Field(default_factory=list)
"""Trailing decode window. Token strings from ``prefix_offset`` onward.
Bounded, trimmed and rebased each chunk to the tail
``detokenize_incrementally`` still reads, so it does not grow with the
number of chunks.
"""
prefix_offset: int = Field(default=0, ge=0)
"""Prefix offset into ``prev_tokens`` for incremental detokenization."""
read_offset: int = Field(default=0, ge=0)
"""Read offset into ``prev_tokens`` for incremental detokenization."""
@field_validator("prev_tokens")
@classmethod
def _bound_prev_tokens(cls, v: list[str]) -> list[str]:
# INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET is small (5) and the trimmed
# window is O(offset). A generous limit rejects unusually large or malformed
# payloads without restricting legitimate multi-byte sequences.
limit = 1024
if len(v) > limit:
raise ValueError(f"prev_tokens length ({len(v)}) exceeds maximum ({limit})")
return v
role_sent: bool = False
"""True once the initial ``role: "assistant"`` delta has been emitted.
Prevents re-emitting the role on subsequent chunks even when the detok
window is transiently empty (e.g. usage only final chunk).
"""
# TODO: Properties used in follow on PR for tool call parsing
last_content: str | None = None
"""Last emitted cumulative assistant content text."""
last_reasoning: str | None = None
"""Last emitted cumulative reasoning text."""
last_tool_call_ids: list[str] = Field(default_factory=list)
"""Stable tool-call IDs, assigned once when each call first appears.
Prevents ID regeneration across re-parsing.
"""
class DerenderChatStreamRequest(BaseModel):
"""One chunk streaming derender request for /v1/chat/completions/derender.
The client sends one request per SSE chunk received from
``/inference/v1/generate``. Each request carries the generate chunk
plus the ``stream_state`` returned by the previous call (``None`` on the
first call). The response contains the derendered chunk and the updated
state to be passed to the next call.
This implements stateless no server side session. All mutable state lives in
the client carried ``stream_state``.
"""
stream: Literal[True]
model: str
generate_chunk: GenerateStreamResponse
"""One SSE chunk from ``/inference/v1/generate`` (``stream=True``)."""
stream_state: DerenderStreamState | None = None
"""Client carried detok state from the previous call. ``None`` on first."""
prompt_tokens: int | None = None
"""Prompt token count for usage. Forwarded from the render step."""
chat_request: ChatCompletionRequest | None = None
"""The original (post adjust_request) ChatCompletionRequest from /render."""
class DerenderCompletionStreamRequest(BaseModel):
"""One chunk streaming derender request for /v1/completions/derender.
Parallel to ``DerenderChatStreamRequest`` for the completions endpoint.
Each call processes one SSE chunk (one output sequence's delta) and
returns the derendered chunk plus updated state.
"""
stream: Literal[True]
model: str
generate_chunk: GenerateStreamResponse
"""One SSE chunk from ``/inference/v1/generate``."""
stream_state: DerenderStreamState | None = None
"""Client-carried detok state. ``None`` on the first call."""
prompt_tokens: int | None = None
"""Prompt token count for usage."""
completion_request: CompletionRequest | None = None
"""The original (post adjust_request) CompletionRequest from /render."""
class DerenderChatStreamResponse(BaseModel):
"""Response for one streaming chat derender chunk.
Pairs the derendered SSE chunk with the updated client carried state to
pass to the next call.
"""
chunk: ChatCompletionStreamResponse
stream_state: DerenderStreamState
class DerenderCompletionStreamResponse(BaseModel):
"""Response for one streaming completions derender chunk.
Parallel to ``DerenderChatStreamResponse`` for the completions endpoint.
"""
chunk: CompletionStreamResponse
stream_state: DerenderStreamState
# Determines the type by checking the ``stream`` field's literal value. A body without
# ``stream`` validates as the non-streaming member
# (``stream`` defaults to ``False`` there), so FastAPI can validate and dispatch both
# shapes on a single path.
DerenderChatRequestUnion: TypeAlias = DerenderChatRequest | DerenderChatStreamRequest
DerenderCompletionRequestUnion: TypeAlias = (
DerenderCompletionRequest | DerenderCompletionStreamRequest
)
+4
View File
@@ -17,7 +17,9 @@ from vllm.entrypoints.openai.completion.protocol import (
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatRequest,
DerenderChatStreamRequest,
DerenderCompletionRequest,
DerenderCompletionStreamRequest,
GenerateRequest,
GenerateResponse,
)
@@ -54,6 +56,7 @@ CompletionLikeRequest: TypeAlias = (
| TokenizeCompletionRequest
| DetokenizeRequest
| DerenderCompletionRequest
| DerenderCompletionStreamRequest
)
ChatLikeRequest: TypeAlias = (
@@ -61,6 +64,7 @@ ChatLikeRequest: TypeAlias = (
| BatchChatCompletionRequest
| TokenizeChatRequest
| DerenderChatRequest
| DerenderChatStreamRequest
)
SpeechToTextRequest: TypeAlias = TranscriptionRequest | TranslationRequest