[Frontend] Add detokenization streaming derender for disaggregated serving (#47301)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
Martin Hickey
2026-07-29 18:01:20 -07:00
committed by GitHub
parent 451227cb3f
commit 0a31372e5f
6 changed files with 1377 additions and 32 deletions
@@ -5,15 +5,17 @@ from http import HTTPStatus
from fastapi import APIRouter, Depends, Request
from fastapi.responses import JSONResponse
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionResponse
from vllm.entrypoints.openai.completion.protocol import CompletionResponse
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
from vllm.logger import init_logger
from ..token_in_token_out.protocol import (
DerenderChatRequest,
DerenderCompletionRequest,
DerenderChatRequestUnion,
DerenderChatStreamRequest,
DerenderChatStreamResponse,
DerenderCompletionRequestUnion,
DerenderCompletionStreamRequest,
DerenderCompletionStreamResponse,
)
from .serving import ServingDerender
@@ -29,46 +31,96 @@ def derender(request: Request) -> ServingDerender | None:
@router.post(
"/v1/chat/completions/derender",
dependencies=[Depends(validate_json_request)],
response_model=ChatCompletionResponse,
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def derender_chat_completion(request: DerenderChatRequest, raw_request: Request):
async def derender_chat_completion(
request: DerenderChatRequestUnion,
raw_request: Request,
):
"""Derender a generate response into a ChatCompletionResponse.
Accepts both non-streaming (``stream=false``, default) and streaming
(``stream=true``) request bodies on the same path; FastAPI validates and
routes on the ``stream`` discriminator.
Non-streaming: body is ``DerenderChatRequest`` (``generate_response`` with
the complete token list). Returns a ``ChatCompletionResponse``.
Streaming: body is ``DerenderChatStreamRequest`` (one ``generate_chunk``
delta + optional ``stream_state``). Returns a ``DerenderChatStreamResponse``
(``chunk`` + ``stream_state``). The client carries ``stream_state`` between
successive calls, one per SSE chunk from ``/inference/v1/generate``.
"""
handler = derender(raw_request)
if handler is None:
raise NotImplementedError(
"The model does not support Chat Completions Derender API"
)
result = await handler.derender_chat_response(request)
if isinstance(request, DerenderChatStreamRequest):
stream_result = await handler.derender_chat_stream_response(request)
if isinstance(stream_result, ErrorResponse):
return JSONResponse(
content=stream_result.model_dump(),
status_code=stream_result.error.code,
)
chunk, stream_state = stream_result
response = DerenderChatStreamResponse(chunk=chunk, stream_state=stream_state)
return JSONResponse(content=response.model_dump())
result = await handler.derender_chat_response(request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
@router.post(
"/v1/completions/derender",
dependencies=[Depends(validate_json_request)],
response_model=CompletionResponse,
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def derender_completion(request: DerenderCompletionRequest, raw_request: Request):
async def derender_completion(
request: DerenderCompletionRequestUnion,
raw_request: Request,
):
"""Derender a generate response into a CompletionResponse.
Accepts both non-streaming (``stream=false``, default) and streaming
(``stream=true``) request bodies on the same path.
Non-streaming: body is ``DerenderCompletionRequest``. Returns a
``CompletionResponse``.
Streaming: body is ``DerenderCompletionStreamRequest`` (one
``generate_chunk`` + optional ``stream_state``). Returns a
``DerenderCompletionStreamResponse`` (``chunk`` + ``stream_state``).
"""
handler = derender(raw_request)
if handler is None:
raise NotImplementedError("The model does not support Completions Derender API")
result = await handler.derender_completion_response(request)
if isinstance(request, DerenderCompletionStreamRequest):
stream_result = await handler.derender_completion_stream_response(request)
if isinstance(stream_result, ErrorResponse):
return JSONResponse(
content=stream_result.model_dump(),
status_code=stream_result.error.code,
)
chunk, stream_state = stream_result
response = DerenderCompletionStreamResponse(
chunk=chunk, stream_state=stream_state
)
return JSONResponse(content=response.model_dump())
result = await handler.derender_completion_response(request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
+98 -3
View File
@@ -4,8 +4,14 @@ import time
from typing import cast
import vllm.envs as envs
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionResponse
from vllm.entrypoints.openai.completion.protocol import CompletionResponse
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionResponse,
ChatCompletionStreamResponse,
)
from vllm.entrypoints.openai.completion.protocol import (
CompletionResponse,
CompletionStreamResponse,
)
from vllm.entrypoints.openai.engine.protocol import (
ErrorResponse,
UsageInfo,
@@ -28,7 +34,10 @@ from vllm.renderers.online_derenderer import OnlineDerenderer
from ..token_in_token_out.mm_serde import encode_mm_kwargs_item
from ..token_in_token_out.protocol import (
DerenderChatRequest,
DerenderChatStreamRequest,
DerenderCompletionRequest,
DerenderCompletionStreamRequest,
DerenderStreamState,
GenerateResponse,
MultiModalFeatures,
PlaceholderRangeInfo,
@@ -200,7 +209,9 @@ class ServingDerender(BaseServing):
total_prompt_tokens,
total_completion_tokens,
) = await self.online_derenderer.derender_completion(
request.generate_responses, request.prompt_tokens
request.generate_responses,
request.prompt_tokens,
completion_request=request.completion_request,
)
first = request.generate_responses[0]
@@ -237,6 +248,90 @@ class ServingDerender(BaseServing):
kv_transfer_params=kv_params,
)
async def derender_chat_stream_response(
self,
request: DerenderChatStreamRequest,
) -> tuple[ChatCompletionStreamResponse, DerenderStreamState] | ErrorResponse:
"""Streaming counterpart to ``derender_chat_response``.
Processes one ``GenerateStreamResponse`` chunk and returns the
derendered chunk together with the updated client carried state.
``parser is None`` or no ``chat_request`` until reasoning/tool call
functionality added in future PR.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
try:
chunk, updated_state = await self.online_derenderer.derender_chat_stream(
model=request.model,
generate_chunk=request.generate_chunk,
state=request.stream_state,
chat_request=request.chat_request,
prompt_tokens=request.prompt_tokens,
)
except NotImplementedError as exc:
return self.create_error_response(exc)
except ValueError as exc:
return self.create_error_response(str(exc))
except (KeyError, IndexError) as exc:
return self.create_error_response(
f"invalid stream_state: detokenization failed ({exc!r})"
)
logger.debug(
"derender_chat_stream request_id=%s model=%s delta_tokens=%d",
request.generate_chunk.request_id,
request.model,
sum(
len(c.token_ids) for c in request.generate_chunk.choices if c.token_ids
),
)
return chunk, updated_state
async def derender_completion_stream_response(
self,
request: DerenderCompletionStreamRequest,
) -> tuple[CompletionStreamResponse, DerenderStreamState] | ErrorResponse:
"""Streaming counterpart to ``derender_completion_response``.
Processes one ``GenerateStreamResponse`` chunk (one output sequence's
delta) and returns the derendered chunk and updated state.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
try:
(
chunk,
updated_state,
) = await self.online_derenderer.derender_completion_stream(
model=request.model,
generate_chunk=request.generate_chunk,
state=request.stream_state,
prompt_tokens=request.prompt_tokens,
completion_request=request.completion_request,
)
except ValueError as exc:
return self.create_error_response(str(exc))
except (KeyError, IndexError) as exc:
return self.create_error_response(
f"invalid stream_state: detokenization failed ({exc!r})"
)
logger.debug(
"derender_completion_stream request_id=%s model=%s delta_tokens=%d",
request.generate_chunk.request_id,
request.model,
sum(
len(c.token_ids) for c in request.generate_chunk.choices if c.token_ids
),
)
return chunk, updated_state
@staticmethod
def _extract_mm_features(
engine_input: EngineInput,
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
from typing import Any, Literal, TypeAlias
from pydantic import (
BaseModel,
@@ -14,8 +14,12 @@ from vllm.config import ModelConfig
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionLogProbs,
ChatCompletionRequest,
ChatCompletionStreamResponse,
)
from vllm.entrypoints.openai.completion.protocol import (
CompletionRequest,
CompletionStreamResponse,
)
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.engine.protocol import StreamOptions, UsageInfo
from vllm.logprobs import Logprob
from vllm.renderers import TokenizeParams
@@ -252,21 +256,16 @@ class GenerateResponse(BaseModel):
)
####### Derender (postprocessing) #######
class DerenderChatRequest(BaseModel):
"""Request for the /v1/chat/completions/derender endpoint (non-streaming).
Wraps a complete GenerateResponse and caller-supplied metadata needed to
produce a fully-formed ChatCompletionResponse without a GPU.
Streaming derender would require a separate endpoint design with
incremental token delivery, ``OutputProcessor``-based detokenization,
and ``parser.parse_delta()`` instead of ``parser.parse()``.
Wraps a complete GenerateResponse and caller supplied metadata needed to
produce a fully formed ChatCompletionResponse without a GPU.
"""
# --8<-- [start:derender-chat-request]
stream: Literal[False] = False
model: str
"""Served model name."""
@@ -299,6 +298,8 @@ class DerenderCompletionRequest(BaseModel):
"""
# --8<-- [start:derender-completion-request]
stream: Literal[False] = False
model: str
"""Served model name."""
@@ -330,3 +331,156 @@ class DerenderCompletionRequest(BaseModel):
f"generate_responses length ({len(self.generate_responses)})"
)
return self
class DerenderStreamState(BaseModel):
"""Per sequence state for stateless streaming derender.
The client carries this between successive per chunk HTTP calls to the
streaming derender endpoint. All fields are plain JSON serializable data.
No opaque tokenizer or parser internals are stored here.
The detokenization strategy carries the incremental decode offsets
directly rather than re-sending the whole token history each chunk.
``detokenize_incrementally`` only ever reads the trailing token window
``prev_tokens[prefix_offset:]``, so we carry just that tail plus the two
offsets. Each chunk resumes exactly where the last one stopped, including
any partially processed multi-byte character (tracked by ``read_offset``),
then trims and rebases the window so it never grows with generation length.
Performance:
- Compute per chunk is O(delta). One ``detokenize_incrementally`` call per
new token, independent of how many tokens preceded it.
- Transport per chunk is O(window). The carried tail is bounded by the
incremental detokenization offset, so cumulative bytes over the wire are
O(n) rather than the O(n^2) a full history round trip would incur.
"""
prev_tokens: list[str] = Field(default_factory=list)
"""Trailing decode window. Token strings from ``prefix_offset`` onward.
Bounded, trimmed and rebased each chunk to the tail
``detokenize_incrementally`` still reads, so it does not grow with the
number of chunks.
"""
prefix_offset: int = Field(default=0, ge=0)
"""Prefix offset into ``prev_tokens`` for incremental detokenization."""
read_offset: int = Field(default=0, ge=0)
"""Read offset into ``prev_tokens`` for incremental detokenization."""
@field_validator("prev_tokens")
@classmethod
def _bound_prev_tokens(cls, v: list[str]) -> list[str]:
# INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET is small (5) and the trimmed
# window is O(offset). A generous limit rejects unusually large or malformed
# payloads without restricting legitimate multi-byte sequences.
limit = 1024
if len(v) > limit:
raise ValueError(f"prev_tokens length ({len(v)}) exceeds maximum ({limit})")
return v
role_sent: bool = False
"""True once the initial ``role: "assistant"`` delta has been emitted.
Prevents re-emitting the role on subsequent chunks even when the detok
window is transiently empty (e.g. usage only final chunk).
"""
# TODO: Properties used in follow on PR for tool call parsing
last_content: str | None = None
"""Last emitted cumulative assistant content text."""
last_reasoning: str | None = None
"""Last emitted cumulative reasoning text."""
last_tool_call_ids: list[str] = Field(default_factory=list)
"""Stable tool-call IDs, assigned once when each call first appears.
Prevents ID regeneration across re-parsing.
"""
class DerenderChatStreamRequest(BaseModel):
"""One chunk streaming derender request for /v1/chat/completions/derender.
The client sends one request per SSE chunk received from
``/inference/v1/generate``. Each request carries the generate chunk
plus the ``stream_state`` returned by the previous call (``None`` on the
first call). The response contains the derendered chunk and the updated
state to be passed to the next call.
This implements stateless no server side session. All mutable state lives in
the client carried ``stream_state``.
"""
stream: Literal[True]
model: str
generate_chunk: GenerateStreamResponse
"""One SSE chunk from ``/inference/v1/generate`` (``stream=True``)."""
stream_state: DerenderStreamState | None = None
"""Client carried detok state from the previous call. ``None`` on first."""
prompt_tokens: int | None = None
"""Prompt token count for usage. Forwarded from the render step."""
chat_request: ChatCompletionRequest | None = None
"""The original (post adjust_request) ChatCompletionRequest from /render."""
class DerenderCompletionStreamRequest(BaseModel):
"""One chunk streaming derender request for /v1/completions/derender.
Parallel to ``DerenderChatStreamRequest`` for the completions endpoint.
Each call processes one SSE chunk (one output sequence's delta) and
returns the derendered chunk plus updated state.
"""
stream: Literal[True]
model: str
generate_chunk: GenerateStreamResponse
"""One SSE chunk from ``/inference/v1/generate``."""
stream_state: DerenderStreamState | None = None
"""Client-carried detok state. ``None`` on the first call."""
prompt_tokens: int | None = None
"""Prompt token count for usage."""
completion_request: CompletionRequest | None = None
"""The original (post adjust_request) CompletionRequest from /render."""
class DerenderChatStreamResponse(BaseModel):
"""Response for one streaming chat derender chunk.
Pairs the derendered SSE chunk with the updated client carried state to
pass to the next call.
"""
chunk: ChatCompletionStreamResponse
stream_state: DerenderStreamState
class DerenderCompletionStreamResponse(BaseModel):
"""Response for one streaming completions derender chunk.
Parallel to ``DerenderChatStreamResponse`` for the completions endpoint.
"""
chunk: CompletionStreamResponse
stream_state: DerenderStreamState
# Determines the type by checking the ``stream`` field's literal value. A body without
# ``stream`` validates as the non-streaming member
# (``stream`` defaults to ``False`` there), so FastAPI can validate and dispatch both
# shapes on a single path.
DerenderChatRequestUnion: TypeAlias = DerenderChatRequest | DerenderChatStreamRequest
DerenderCompletionRequestUnion: TypeAlias = (
DerenderCompletionRequest | DerenderCompletionStreamRequest
)
+4
View File
@@ -17,7 +17,9 @@ from vllm.entrypoints.openai.completion.protocol import (
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatRequest,
DerenderChatStreamRequest,
DerenderCompletionRequest,
DerenderCompletionStreamRequest,
GenerateRequest,
GenerateResponse,
)
@@ -54,6 +56,7 @@ CompletionLikeRequest: TypeAlias = (
| TokenizeCompletionRequest
| DetokenizeRequest
| DerenderCompletionRequest
| DerenderCompletionStreamRequest
)
ChatLikeRequest: TypeAlias = (
@@ -61,6 +64,7 @@ ChatLikeRequest: TypeAlias = (
| BatchChatCompletionRequest
| TokenizeChatRequest
| DerenderChatRequest
| DerenderChatStreamRequest
)
SpeechToTextRequest: TypeAlias = TranscriptionRequest | TranslationRequest
+281 -6
View File
@@ -10,19 +10,29 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionNamedToolChoiceParam,
ChatCompletionRequest,
ChatCompletionResponseChoice,
ChatCompletionResponseStreamChoice,
ChatCompletionStreamResponse,
ChatMessage,
)
from vllm.entrypoints.openai.completion.protocol import (
CompletionLogProbs,
CompletionRequest,
CompletionResponseChoice,
CompletionResponseStreamChoice,
CompletionStreamResponse,
)
from vllm.entrypoints.openai.engine.protocol import DeltaMessage, ToolCall, UsageInfo
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderStreamState,
GenerateResponse,
GenerateStreamResponse,
)
from vllm.entrypoints.openai.engine.protocol import ToolCall
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateResponse
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.logger import init_logger
from vllm.parser import Parser, ParserManager
from vllm.renderers import BaseRenderer
from vllm.tokenizers import TokenizerLike
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
from vllm.utils import random_uuid
from vllm.utils.async_utils import make_async
@@ -168,9 +178,15 @@ class OnlineDerenderer:
tool_calls=tc_items,
)
else:
# No parser: plain detokenization.
# No parser: plain detokenization honouring the request's
# skip_special_tokens (default True when no request was given).
skip_special = (
chat_request.skip_special_tokens
if chat_request is not None
else True
)
decoded_text = tokenizer.decode(
choice.token_ids, skip_special_tokens=True
choice.token_ids, skip_special_tokens=skip_special
)
message = ChatMessage(role="assistant", content=decoded_text)
@@ -185,23 +201,200 @@ class OnlineDerenderer:
return choices
def _detokenize_delta(
self,
tokenizer: TokenizerLike,
delta_token_ids: list[int],
state: DerenderStreamState,
skip_special_tokens: bool = True,
spaces_between_special_tokens: bool = True,
) -> tuple[str, DerenderStreamState]:
"""Incrementally detokenize ``delta_token_ids`` from prior stream state.
Resumes decoding from the offsets carried in ``state`` rather than
replaying token history. ``state.prev_tokens`` holds the trailing decode
window (from ``prefix_offset`` onward) that ``detokenize_incrementally``
still needs to reproduce any partially read multi-byte character
(tracked by ``read_offset``). The delta tokens are fed straight onto it.
The window is bounded. ``detokenize_incrementally`` never reads before
``prefix_offset``, so after processing we trim ``prev_tokens`` to that
tail and rebase the offsets to it. State transport therefore stays
O(window) per chunk instead of re-sending the full token history.
Args:
tokenizer: The tokenizer to decode with.
delta_token_ids: New token IDs from this generate chunk.
state: Client carried detok state from the previous call.
skip_special_tokens: Passed through to the tokenizer.
spaces_between_special_tokens: Passed through to the tokenizer.
Returns:
(new_text, updated_state) — the delta text for this chunk and the
state to pass to the next call.
"""
prev_tokens = list(state.prev_tokens)
prefix_offset = state.prefix_offset
read_offset = state.read_offset
text_parts: list[str] = []
for tok_id in delta_token_ids:
# prev_tokens is a (possibly empty) list, never None, so this
# always takes the non first iter path and only consumes
# all_input_ids[-1].
new_toks, text, prefix_offset, read_offset = detokenize_incrementally(
tokenizer=tokenizer,
all_input_ids=[tok_id],
prev_tokens=prev_tokens,
prefix_offset=prefix_offset,
read_offset=read_offset,
skip_special_tokens=skip_special_tokens,
spaces_between_special_tokens=spaces_between_special_tokens,
)
prev_tokens = prev_tokens + new_toks
text_parts.append(text)
# Trim to the tail still readable by detokenize_incrementally
# (everything before prefix_offset is dead) and rebase the offsets so
# the carried window stays bounded regardless of generation length.
trimmed = prev_tokens[prefix_offset:]
updated_state = state.model_copy(
update={
"prev_tokens": trimmed,
"prefix_offset": 0,
"read_offset": read_offset - prefix_offset,
}
)
return "".join(text_parts), updated_state
async def derender_chat_stream(
self,
model: str,
generate_chunk: GenerateStreamResponse,
state: DerenderStreamState | None = None,
chat_request: ChatCompletionRequest | None = None,
prompt_tokens: int | None = None,
) -> tuple[ChatCompletionStreamResponse, DerenderStreamState]:
"""Process one GenerateStreamResponse chunk for streaming chat derender.
TODO: parse path for reasoning and tool calls is implemented in future PR.
Unlike OpenAI's API, which always emits ``role: "assistant"`` on the
very first chunk, this emits it on the first chunk with a non empty
``choices`` list. A leading usage only chunk therefore defers the
role to the following content chunk instead of sending an empty
role only delta.
Args:
model: Model name for the response object.
generate_chunk: One SSE chunk from ``/inference/v1/generate``.
state: Client carried detok state (``None`` for first call).
chat_request: Original ChatCompletionRequest from ``/render``.
prompt_tokens: Prompt token count for the usage chunk.
Returns:
(chunk, updated_state) — the derendered SSE chunk and the state
the client must pass to the next call.
"""
if state is None:
state = DerenderStreamState()
if self.parser is not None:
# TODO: Follow on PR will implement the parse path. Check on the
# parser alone (fail closed). A parser configured model must never
# fall through to plain detok on the streaming path, even when
# ``chat_request`` is omitted or reasoning/tool markup would leak
# into ``delta.content``.
raise NotImplementedError(
"Streaming chat derender is not yet supported for models with "
"a reasoning or tool parser configured. Use the non-streaming "
"derender endpoint (stream=false) for parsed output."
)
# A single DerenderStreamState is threaded through every choice in
# this chunk. Correct only when there is at most one choice per SSE
# event (n=1, one call per index), as the streaming derender
# protocol assumes. Multiple choices sharing one chunk would corrupt
# each other's detok window.
if len(generate_chunk.choices) > 1:
raise ValueError(
"derender_chat_stream expects at most one choice per chunk"
)
tokenizer = self.renderer.get_tokenizer()
skip_special = (
chat_request.skip_special_tokens if chat_request is not None else True
)
stream_choices: list[ChatCompletionResponseStreamChoice] = []
updated_state = state
for choice in generate_chunk.choices:
delta_tids = choice.token_ids or []
new_text, updated_state = self._detokenize_delta(
tokenizer, delta_tids, updated_state, skip_special_tokens=skip_special
)
include_role = not updated_state.role_sent
if include_role:
updated_state = updated_state.model_copy(update={"role_sent": True})
delta = DeltaMessage(
role="assistant" if include_role else None,
content=new_text if new_text else None,
)
stream_choices.append(
ChatCompletionResponseStreamChoice(
index=choice.index,
delta=delta,
finish_reason=choice.finish_reason,
)
)
usage: UsageInfo | None = None
if generate_chunk.usage is not None:
u = generate_chunk.usage
pt = prompt_tokens if prompt_tokens is not None else (u.prompt_tokens or 0)
ct = u.completion_tokens or 0
usage = UsageInfo(
prompt_tokens=pt,
completion_tokens=ct,
total_tokens=pt + ct,
)
chunk = ChatCompletionStreamResponse(
id=generate_chunk.request_id,
model=model,
choices=stream_choices,
usage=usage,
)
return chunk, updated_state
async def derender_completion(
self,
generate_responses: list[GenerateResponse],
prompt_tokens: list[int] | None = None,
completion_request: CompletionRequest | None = None,
) -> tuple[list[CompletionResponseChoice], int, int]:
return await self._derender_completion_async(generate_responses, prompt_tokens)
return await self._derender_completion_async(
generate_responses, prompt_tokens, completion_request
)
def _derender_completion(
self,
generate_responses: list[GenerateResponse],
prompt_tokens: list[int] | None = None,
completion_request: CompletionRequest | None = None,
) -> tuple[list[CompletionResponseChoice], int, int]:
n = len(generate_responses)
prompt_tokens_list: list[int] = (
prompt_tokens if prompt_tokens is not None else [0] * n
)
skip_special = (
completion_request.skip_special_tokens
if completion_request is not None
else True
)
tokenizer = self.renderer.get_tokenizer()
choices: list[CompletionResponseChoice] = []
total_prompt_tokens = 0
@@ -217,7 +410,7 @@ class OnlineDerenderer:
)
decoded_text = tokenizer.decode(
choice.token_ids, skip_special_tokens=True
choice.token_ids, skip_special_tokens=skip_special
)
completion_logprobs = None
if choice.logprobs is not None:
@@ -239,6 +432,88 @@ class OnlineDerenderer:
return choices, total_prompt_tokens, total_completion_tokens
async def derender_completion_stream(
self,
model: str,
generate_chunk: GenerateStreamResponse,
state: DerenderStreamState | None = None,
prompt_tokens: int | None = None,
completion_request: CompletionRequest | None = None,
) -> tuple[CompletionStreamResponse, DerenderStreamState]:
"""Process one GenerateStreamResponse chunk for streaming completions.
Each call takes one SSE chunk from ``/inference/v1/generate`` plus the
client carried ``stream_state`` and returns a ``CompletionStreamResponse``
chunk and the updated state.
The generate stream emits one choice per SSE event, so this method
processes one output sequence at a time. For ``n > 1`` the client
maintains one ``DerenderStreamState`` per ``choice.index``.
Args:
model: Model name for the response object.
generate_chunk: One SSE chunk from ``/inference/v1/generate``.
state: Client carried detok state (``None`` → first call).
prompt_tokens: Prompt token count for usage (from the render step).
completion_request: Original CompletionRequest from ``/render``;
supplies ``skip_special_tokens``.
Returns:
(chunk, updated_state) — the derendered chunk and updated state.
"""
if state is None:
state = DerenderStreamState()
# See the equivalent check in derender_chat_stream: a single
# DerenderStreamState is threaded through every choice in this
# chunk, so more than one choice per chunk would corrupt the
# detok window across choices.
if len(generate_chunk.choices) > 1:
raise ValueError(
"derender_completion_stream expects at most one choice per chunk"
)
tokenizer = self.renderer.get_tokenizer()
skip_special = (
completion_request.skip_special_tokens
if completion_request is not None
else True
)
stream_choices: list[CompletionResponseStreamChoice] = []
updated_state = state
for choice in generate_chunk.choices:
delta_tids = choice.token_ids or []
new_text, updated_state = self._detokenize_delta(
tokenizer, delta_tids, updated_state, skip_special_tokens=skip_special
)
stream_choices.append(
CompletionResponseStreamChoice(
index=choice.index,
text=new_text,
finish_reason=choice.finish_reason,
)
)
usage: UsageInfo | None = None
if generate_chunk.usage is not None:
u = generate_chunk.usage
pt = prompt_tokens if prompt_tokens is not None else (u.prompt_tokens or 0)
ct = u.completion_tokens or 0
usage = UsageInfo(
prompt_tokens=pt,
completion_tokens=ct,
total_tokens=pt + ct,
)
chunk = CompletionStreamResponse(
id=generate_chunk.request_id,
model=model,
choices=stream_choices,
usage=usage,
)
return chunk, updated_state
def _parse_token_id_placeholder(token: str) -> int | None:
"""Extract token ID from a 'token_id:N' placeholder string."""