[Frontend] Add detokenization streaming derender for disaggregated serving (#47301)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
Martin Hickey
2026-07-29 18:01:20 -07:00
committed by GitHub
parent 451227cb3f
commit 0a31372e5f
6 changed files with 1377 additions and 32 deletions
+281 -6
View File
@@ -10,19 +10,29 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionNamedToolChoiceParam,
ChatCompletionRequest,
ChatCompletionResponseChoice,
ChatCompletionResponseStreamChoice,
ChatCompletionStreamResponse,
ChatMessage,
)
from vllm.entrypoints.openai.completion.protocol import (
CompletionLogProbs,
CompletionRequest,
CompletionResponseChoice,
CompletionResponseStreamChoice,
CompletionStreamResponse,
)
from vllm.entrypoints.openai.engine.protocol import DeltaMessage, ToolCall, UsageInfo
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderStreamState,
GenerateResponse,
GenerateStreamResponse,
)
from vllm.entrypoints.openai.engine.protocol import ToolCall
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateResponse
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.logger import init_logger
from vllm.parser import Parser, ParserManager
from vllm.renderers import BaseRenderer
from vllm.tokenizers import TokenizerLike
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
from vllm.utils import random_uuid
from vllm.utils.async_utils import make_async
@@ -168,9 +178,15 @@ class OnlineDerenderer:
tool_calls=tc_items,
)
else:
# No parser: plain detokenization.
# No parser: plain detokenization honouring the request's
# skip_special_tokens (default True when no request was given).
skip_special = (
chat_request.skip_special_tokens
if chat_request is not None
else True
)
decoded_text = tokenizer.decode(
choice.token_ids, skip_special_tokens=True
choice.token_ids, skip_special_tokens=skip_special
)
message = ChatMessage(role="assistant", content=decoded_text)
@@ -185,23 +201,200 @@ class OnlineDerenderer:
return choices
def _detokenize_delta(
self,
tokenizer: TokenizerLike,
delta_token_ids: list[int],
state: DerenderStreamState,
skip_special_tokens: bool = True,
spaces_between_special_tokens: bool = True,
) -> tuple[str, DerenderStreamState]:
"""Incrementally detokenize ``delta_token_ids`` from prior stream state.
Resumes decoding from the offsets carried in ``state`` rather than
replaying token history. ``state.prev_tokens`` holds the trailing decode
window (from ``prefix_offset`` onward) that ``detokenize_incrementally``
still needs to reproduce any partially read multi-byte character
(tracked by ``read_offset``). The delta tokens are fed straight onto it.
The window is bounded. ``detokenize_incrementally`` never reads before
``prefix_offset``, so after processing we trim ``prev_tokens`` to that
tail and rebase the offsets to it. State transport therefore stays
O(window) per chunk instead of re-sending the full token history.
Args:
tokenizer: The tokenizer to decode with.
delta_token_ids: New token IDs from this generate chunk.
state: Client carried detok state from the previous call.
skip_special_tokens: Passed through to the tokenizer.
spaces_between_special_tokens: Passed through to the tokenizer.
Returns:
(new_text, updated_state) — the delta text for this chunk and the
state to pass to the next call.
"""
prev_tokens = list(state.prev_tokens)
prefix_offset = state.prefix_offset
read_offset = state.read_offset
text_parts: list[str] = []
for tok_id in delta_token_ids:
# prev_tokens is a (possibly empty) list, never None, so this
# always takes the non first iter path and only consumes
# all_input_ids[-1].
new_toks, text, prefix_offset, read_offset = detokenize_incrementally(
tokenizer=tokenizer,
all_input_ids=[tok_id],
prev_tokens=prev_tokens,
prefix_offset=prefix_offset,
read_offset=read_offset,
skip_special_tokens=skip_special_tokens,
spaces_between_special_tokens=spaces_between_special_tokens,
)
prev_tokens = prev_tokens + new_toks
text_parts.append(text)
# Trim to the tail still readable by detokenize_incrementally
# (everything before prefix_offset is dead) and rebase the offsets so
# the carried window stays bounded regardless of generation length.
trimmed = prev_tokens[prefix_offset:]
updated_state = state.model_copy(
update={
"prev_tokens": trimmed,
"prefix_offset": 0,
"read_offset": read_offset - prefix_offset,
}
)
return "".join(text_parts), updated_state
async def derender_chat_stream(
self,
model: str,
generate_chunk: GenerateStreamResponse,
state: DerenderStreamState | None = None,
chat_request: ChatCompletionRequest | None = None,
prompt_tokens: int | None = None,
) -> tuple[ChatCompletionStreamResponse, DerenderStreamState]:
"""Process one GenerateStreamResponse chunk for streaming chat derender.
TODO: parse path for reasoning and tool calls is implemented in future PR.
Unlike OpenAI's API, which always emits ``role: "assistant"`` on the
very first chunk, this emits it on the first chunk with a non empty
``choices`` list. A leading usage only chunk therefore defers the
role to the following content chunk instead of sending an empty
role only delta.
Args:
model: Model name for the response object.
generate_chunk: One SSE chunk from ``/inference/v1/generate``.
state: Client carried detok state (``None`` for first call).
chat_request: Original ChatCompletionRequest from ``/render``.
prompt_tokens: Prompt token count for the usage chunk.
Returns:
(chunk, updated_state) — the derendered SSE chunk and the state
the client must pass to the next call.
"""
if state is None:
state = DerenderStreamState()
if self.parser is not None:
# TODO: Follow on PR will implement the parse path. Check on the
# parser alone (fail closed). A parser configured model must never
# fall through to plain detok on the streaming path, even when
# ``chat_request`` is omitted or reasoning/tool markup would leak
# into ``delta.content``.
raise NotImplementedError(
"Streaming chat derender is not yet supported for models with "
"a reasoning or tool parser configured. Use the non-streaming "
"derender endpoint (stream=false) for parsed output."
)
# A single DerenderStreamState is threaded through every choice in
# this chunk. Correct only when there is at most one choice per SSE
# event (n=1, one call per index), as the streaming derender
# protocol assumes. Multiple choices sharing one chunk would corrupt
# each other's detok window.
if len(generate_chunk.choices) > 1:
raise ValueError(
"derender_chat_stream expects at most one choice per chunk"
)
tokenizer = self.renderer.get_tokenizer()
skip_special = (
chat_request.skip_special_tokens if chat_request is not None else True
)
stream_choices: list[ChatCompletionResponseStreamChoice] = []
updated_state = state
for choice in generate_chunk.choices:
delta_tids = choice.token_ids or []
new_text, updated_state = self._detokenize_delta(
tokenizer, delta_tids, updated_state, skip_special_tokens=skip_special
)
include_role = not updated_state.role_sent
if include_role:
updated_state = updated_state.model_copy(update={"role_sent": True})
delta = DeltaMessage(
role="assistant" if include_role else None,
content=new_text if new_text else None,
)
stream_choices.append(
ChatCompletionResponseStreamChoice(
index=choice.index,
delta=delta,
finish_reason=choice.finish_reason,
)
)
usage: UsageInfo | None = None
if generate_chunk.usage is not None:
u = generate_chunk.usage
pt = prompt_tokens if prompt_tokens is not None else (u.prompt_tokens or 0)
ct = u.completion_tokens or 0
usage = UsageInfo(
prompt_tokens=pt,
completion_tokens=ct,
total_tokens=pt + ct,
)
chunk = ChatCompletionStreamResponse(
id=generate_chunk.request_id,
model=model,
choices=stream_choices,
usage=usage,
)
return chunk, updated_state
async def derender_completion(
self,
generate_responses: list[GenerateResponse],
prompt_tokens: list[int] | None = None,
completion_request: CompletionRequest | None = None,
) -> tuple[list[CompletionResponseChoice], int, int]:
return await self._derender_completion_async(generate_responses, prompt_tokens)
return await self._derender_completion_async(
generate_responses, prompt_tokens, completion_request
)
def _derender_completion(
self,
generate_responses: list[GenerateResponse],
prompt_tokens: list[int] | None = None,
completion_request: CompletionRequest | None = None,
) -> tuple[list[CompletionResponseChoice], int, int]:
n = len(generate_responses)
prompt_tokens_list: list[int] = (
prompt_tokens if prompt_tokens is not None else [0] * n
)
skip_special = (
completion_request.skip_special_tokens
if completion_request is not None
else True
)
tokenizer = self.renderer.get_tokenizer()
choices: list[CompletionResponseChoice] = []
total_prompt_tokens = 0
@@ -217,7 +410,7 @@ class OnlineDerenderer:
)
decoded_text = tokenizer.decode(
choice.token_ids, skip_special_tokens=True
choice.token_ids, skip_special_tokens=skip_special
)
completion_logprobs = None
if choice.logprobs is not None:
@@ -239,6 +432,88 @@ class OnlineDerenderer:
return choices, total_prompt_tokens, total_completion_tokens
async def derender_completion_stream(
self,
model: str,
generate_chunk: GenerateStreamResponse,
state: DerenderStreamState | None = None,
prompt_tokens: int | None = None,
completion_request: CompletionRequest | None = None,
) -> tuple[CompletionStreamResponse, DerenderStreamState]:
"""Process one GenerateStreamResponse chunk for streaming completions.
Each call takes one SSE chunk from ``/inference/v1/generate`` plus the
client carried ``stream_state`` and returns a ``CompletionStreamResponse``
chunk and the updated state.
The generate stream emits one choice per SSE event, so this method
processes one output sequence at a time. For ``n > 1`` the client
maintains one ``DerenderStreamState`` per ``choice.index``.
Args:
model: Model name for the response object.
generate_chunk: One SSE chunk from ``/inference/v1/generate``.
state: Client carried detok state (``None`` → first call).
prompt_tokens: Prompt token count for usage (from the render step).
completion_request: Original CompletionRequest from ``/render``;
supplies ``skip_special_tokens``.
Returns:
(chunk, updated_state) — the derendered chunk and updated state.
"""
if state is None:
state = DerenderStreamState()
# See the equivalent check in derender_chat_stream: a single
# DerenderStreamState is threaded through every choice in this
# chunk, so more than one choice per chunk would corrupt the
# detok window across choices.
if len(generate_chunk.choices) > 1:
raise ValueError(
"derender_completion_stream expects at most one choice per chunk"
)
tokenizer = self.renderer.get_tokenizer()
skip_special = (
completion_request.skip_special_tokens
if completion_request is not None
else True
)
stream_choices: list[CompletionResponseStreamChoice] = []
updated_state = state
for choice in generate_chunk.choices:
delta_tids = choice.token_ids or []
new_text, updated_state = self._detokenize_delta(
tokenizer, delta_tids, updated_state, skip_special_tokens=skip_special
)
stream_choices.append(
CompletionResponseStreamChoice(
index=choice.index,
text=new_text,
finish_reason=choice.finish_reason,
)
)
usage: UsageInfo | None = None
if generate_chunk.usage is not None:
u = generate_chunk.usage
pt = prompt_tokens if prompt_tokens is not None else (u.prompt_tokens or 0)
ct = u.completion_tokens or 0
usage = UsageInfo(
prompt_tokens=pt,
completion_tokens=ct,
total_tokens=pt + ct,
)
chunk = CompletionStreamResponse(
id=generate_chunk.request_id,
model=model,
choices=stream_choices,
usage=usage,
)
return chunk, updated_state
def _parse_token_id_placeholder(token: str) -> int | None:
"""Extract token ID from a 'token_id:N' placeholder string."""