[Render] Add /derender endpoints for disaggregated postprocessing (#43606)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
Martin Hickey
2026-06-13 13:55:33 +08:00
committed by GitHub
co-authored by Isotr0py
parent 1033ffac2e
commit 43f0e024bc
7 changed files with 898 additions and 9 deletions
@@ -46,6 +46,7 @@ from vllm.entrypoints.openai.engine.serving import (
GenerationError,
OpenAIServing,
clamp_prompt_logprobs,
format_token_id_placeholder,
)
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
@@ -1129,7 +1130,7 @@ class OpenAIServingChat(OpenAIServing):
step_top_logprobs = top_logprobs[i]
if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
if should_return_as_token_id:
token = f"token_id:{token_id}"
token = format_token_id_placeholder(token_id)
else:
if tokenizer is None:
raise ValueError(
@@ -31,6 +31,7 @@ from vllm.entrypoints.openai.engine.serving import (
GenerationError,
OpenAIServing,
clamp_prompt_logprobs,
format_token_id_placeholder,
)
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
@@ -628,7 +629,7 @@ class OpenAIServingCompletion(OpenAIServing):
step_top_logprobs = top_logprobs[i]
if step_top_logprobs is None:
if should_return_as_token_id:
token = f"token_id:{token_id}"
token = format_token_id_placeholder(token_id)
else:
if tokenizer is None:
raise VLLMValidationError(
+33 -1
View File
@@ -452,7 +452,7 @@ class OpenAIServing(BeamSearchOnlineMixin):
return_as_token_id: bool = False,
) -> str:
if return_as_token_id:
return f"token_id:{token_id}"
return format_token_id_placeholder(token_id)
if logprob.decoded_token is not None:
return logprob.decoded_token
@@ -472,6 +472,38 @@ class OpenAIServing(BeamSearchOnlineMixin):
return self.models.is_base_model(model_name)
def format_token_id_placeholder(token_id: int) -> str:
return f"token_id:{token_id}"
def resolve_token_id_placeholder(
token: str, tokenizer: TokenizerLike
) -> tuple[str, list[int] | None]:
"""Decode a 'token_id:N' placeholder back to a token string and UTF-8 bytes.
Returns (token, None) unchanged if token is not a placeholder.
This is the inverse of format_token_id_placeholder / _get_decoded_token
when return_as_token_id=True.
"""
suffix = token.removeprefix("token_id:")
if suffix == token:
return token, None
try:
token_id = int(suffix)
except ValueError:
return token, None
token_repr = tokenizer.convert_ids_to_tokens([token_id])[0]
if token_repr is None:
logger.warning_once(
"resolve_token_id_placeholder: token_id %d has no vocab entry; "
"substituting empty string",
token_id,
)
return "", None
token_str = tokenizer.convert_tokens_to_string([token_repr])
return token_str, list(token_str.encode("utf-8", errors="replace"))
def clamp_prompt_logprobs(
prompt_logprobs: PromptLogprobs | None,
) -> PromptLogprobs | None:
+68 -1
View File
@@ -11,7 +11,11 @@ from pydantic import (
)
from vllm.config import ModelConfig
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionLogProbs
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionLogProbs,
ChatCompletionRequest,
)
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.engine.protocol import StreamOptions, UsageInfo
from vllm.logprobs import Logprob
from vllm.renderers import TokenizeParams
@@ -209,3 +213,66 @@ class GenerateResponse(BaseModel):
default=None,
description="KVTransfer parameters used for disaggregated serving.",
)
####### Derender (postprocessing) #######
class DerenderChatRequest(BaseModel):
"""Request for the /v1/chat/completions/derender endpoint.
Wraps a GenerateResponse and caller-supplied metadata needed to produce
a fully-formed ChatCompletionResponse without a GPU.
"""
model: str
generate_response: GenerateResponse
prompt_tokens: int | None = None
"""Prompt token count for usage; defaults to 0 if omitted.
GenerateResponse carries only output tokens; the caller already has
len(GenerateRequest.token_ids) from the render step.
"""
chat_request: ChatCompletionRequest | None = None
"""The original (post-adjust_request) ChatCompletionRequest from /render.
Required by the parsing so that tool/reasoning parsers can receive the full
request context they expect (request.tools, request.tool_choice,
request._grammar_from_tool_parser, etc.).
"""
class DerenderCompletionRequest(BaseModel):
"""Request for the /v1/completions/derender endpoint.
Parallel to DerenderChatRequest but handles the multi-prompt completions
case: one GenerateResponse per prompt, mirroring the list[GenerateRequest]
returned by /v1/completions/render.
"""
model: str
generate_responses: list[GenerateResponse]
prompt_tokens: list[int] | None = None
"""One prompt token count per response; each defaults to 0 if omitted.
If provided, len(prompt_tokens) must equal len(generate_responses).
"""
completion_request: CompletionRequest | None = None
"""The original (post-adjust_request) CompletionRequest from /render.
Mirrors chat_request on DerenderChatRequest. Required by the parsing
so parsers receive the full request context.
"""
@model_validator(mode="after")
def _validate_prompt_tokens_length(self) -> "DerenderCompletionRequest":
if self.prompt_tokens is not None and len(self.prompt_tokens) != len(
self.generate_responses
):
raise ValueError(
f"prompt_tokens length ({len(self.prompt_tokens)}) must equal "
f"generate_responses length ({len(self.generate_responses)})"
)
return self
+61 -3
View File
@@ -5,10 +5,20 @@ from http import HTTPStatus
from fastapi import APIRouter, Depends, FastAPI, Request
from fastapi.responses import JSONResponse
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionResponse,
)
from vllm.entrypoints.openai.completion.protocol import (
CompletionRequest,
CompletionResponse,
)
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.disagg.protocol import GenerateRequest
from vllm.entrypoints.serve.disagg.protocol import (
DerenderChatRequest,
DerenderCompletionRequest,
GenerateRequest,
)
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
from vllm.logger import init_logger
@@ -71,5 +81,53 @@ async def render_completion(request: CompletionRequest, raw_request: Request):
return JSONResponse(content=[item.model_dump() for item in result])
@router.post(
"/v1/chat/completions/derender",
dependencies=[Depends(validate_json_request)],
response_model=ChatCompletionResponse,
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def derender_chat_completion(request: DerenderChatRequest, raw_request: Request):
handler = render(raw_request)
if handler is None:
raise NotImplementedError(
"The model does not support Chat Completions Derender API"
)
result = await handler.derender_chat_response(request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
@router.post(
"/v1/completions/derender",
dependencies=[Depends(validate_json_request)],
response_model=CompletionResponse,
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def derender_completion(request: DerenderCompletionRequest, raw_request: Request):
handler = render(raw_request)
if handler is None:
raise NotImplementedError("The model does not support Completions Derender API")
result = await handler.derender_completion_response(request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
def attach_router(app: FastAPI) -> None:
app.include_router(router)
+244 -2
View File
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from collections.abc import Sequence
from http import HTTPStatus
from typing import Any, cast
@@ -11,11 +12,24 @@ from vllm.entrypoints.chat_utils import (
ChatTemplateContentFormatOption,
ConversationMessage,
)
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionLogProbs,
ChatCompletionRequest,
ChatCompletionResponse,
ChatCompletionResponseChoice,
ChatMessage,
)
from vllm.entrypoints.openai.completion.protocol import (
CompletionLogProbs,
CompletionRequest,
CompletionResponse,
CompletionResponseChoice,
)
from vllm.entrypoints.openai.engine.protocol import (
ErrorResponse,
UsageInfo,
)
from vllm.entrypoints.openai.engine.serving import resolve_token_id_placeholder
from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
from vllm.entrypoints.openai.parser.harmony_utils import (
build_harmony_preamble,
@@ -26,7 +40,10 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.serve.disagg.mm_serde import encode_mm_kwargs_item
from vllm.entrypoints.serve.disagg.protocol import (
DerenderChatRequest,
DerenderCompletionRequest,
GenerateRequest,
GenerateResponseChoice,
MultiModalFeatures,
PlaceholderRangeInfo,
)
@@ -51,6 +68,7 @@ from vllm.renderers.inputs.preprocess import (
parse_model_prompt,
prompt_to_seq,
)
from vllm.tokenizers import TokenizerLike
from vllm.utils import random_uuid
from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser
from vllm.utils.mistral import mt as _mt
@@ -58,6 +76,90 @@ from vllm.utils.mistral import mt as _mt
logger = init_logger(__name__)
def _resolve_logprobs(
logprobs: ChatCompletionLogProbs, tokenizer: TokenizerLike
) -> ChatCompletionLogProbs:
"""Resolve all token_id:N placeholders in a ChatCompletionLogProbs object."""
if logprobs.content is None:
return logprobs
resolved_content = []
for entry in logprobs.content:
token_str, token_bytes = resolve_token_id_placeholder(entry.token, tokenizer)
resolved_top = []
for top in entry.top_logprobs:
top_str, top_bytes = resolve_token_id_placeholder(top.token, tokenizer)
resolved_top.append(
top.model_copy(update={"token": top_str, "bytes": top_bytes})
)
resolved_content.append(
entry.model_copy(
update={
"token": token_str,
"bytes": token_bytes,
"top_logprobs": resolved_top,
}
)
)
return ChatCompletionLogProbs(content=resolved_content)
def _convert_chat_logprobs_to_completion_logprobs(
logprobs: ChatCompletionLogProbs,
) -> CompletionLogProbs:
"""Convert ChatCompletionLogProbs (per-token objects) to CompletionLogProbs
(parallel flat lists) as required by the /v1/completions response schema."""
if logprobs.content is None:
return CompletionLogProbs()
tokens: list[str] = []
token_logprobs: list[float | None] = []
top_logprobs_list: list[dict[str, float] | None] = []
text_offset: list[int] = []
offset = 0
for entry in logprobs.content:
text_offset.append(offset)
tokens.append(entry.token)
token_logprobs.append(entry.logprob)
top_logprobs_list.append(
{t.token: t.logprob for t in entry.top_logprobs}
if entry.top_logprobs
else None
)
offset += len(entry.token)
return CompletionLogProbs(
text_offset=text_offset,
token_logprobs=token_logprobs,
tokens=tokens,
top_logprobs=top_logprobs_list,
)
def _build_chat_choice(
choice: GenerateResponseChoice, tokenizer: TokenizerLike
) -> ChatCompletionResponseChoice:
"""Detokenize and resolve logprobs for a single GenerateResponseChoice.
Raises:
ValueError: if choice.token_ids is empty or None.
"""
if not choice.token_ids:
raise ValueError(f"choice {choice.index} has empty or null token_ids")
decoded_text = tokenizer.decode(choice.token_ids, skip_special_tokens=True)
resolved_logprobs = (
_resolve_logprobs(choice.logprobs, tokenizer)
if choice.logprobs is not None
else None
)
return ChatCompletionResponseChoice(
index=choice.index,
message=ChatMessage(role="assistant", content=decoded_text),
logprobs=resolved_logprobs,
finish_reason=choice.finish_reason,
)
class OpenAIServingRender:
def __init__(
self,
@@ -427,6 +529,146 @@ class OpenAIServingRender:
return messages, [engine_input]
async def derender_chat_response(
self,
request: DerenderChatRequest,
) -> ChatCompletionResponse | ErrorResponse:
"""Postprocess a GenerateResponse into a ChatCompletionResponse.
This is the symmetric inverse of render_chat_request: it detokenizes
output token IDs, resolves token_id:N logprob placeholders, and
formats the result as an OpenAI-compatible chat completion response.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
tokenizer = self.renderer.get_tokenizer()
gen = request.generate_response
choices: list[ChatCompletionResponseChoice] = []
try:
for choice in gen.choices:
choices.append(_build_chat_choice(choice, tokenizer))
except ValueError as exc:
return self.create_error_response(str(exc))
prompt_tokens = (
request.prompt_tokens if request.prompt_tokens is not None else 0
)
completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids)
usage = UsageInfo(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
)
logger.debug(
"derender_chat request_id=%s model=%s choices=%d completion_tokens=%d",
gen.request_id,
request.model,
len(choices),
completion_tokens,
)
return ChatCompletionResponse(
id=gen.request_id,
model=request.model,
created=int(time.time()),
choices=choices,
usage=usage,
prompt_logprobs=gen.prompt_logprobs,
kv_transfer_params=gen.kv_transfer_params,
)
async def derender_completion_response(
self,
request: DerenderCompletionRequest,
) -> CompletionResponse | ErrorResponse:
"""Postprocess a list of GenerateResponses into a CompletionResponse.
Mirrors the multi-prompt completions case: one GenerateResponse per
prompt, parallel to the list[GenerateRequest] from /v1/completions/render.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
n = len(request.generate_responses)
prompt_tokens_list: list[int] = (
request.prompt_tokens if request.prompt_tokens is not None else [0] * n
)
tokenizer = self.renderer.get_tokenizer()
choices: list[CompletionResponseChoice] = []
total_prompt_tokens = 0
total_completion_tokens = 0
index = 0
for gen, pt in zip(request.generate_responses, prompt_tokens_list):
for choice in gen.choices:
if not choice.token_ids:
return self.create_error_response(
f"choice {choice.index} in response {gen.request_id} "
"has empty or null token_ids"
)
decoded_text = tokenizer.decode(
choice.token_ids, skip_special_tokens=True
)
completion_logprobs = None
if choice.logprobs is not None:
resolved = _resolve_logprobs(choice.logprobs, tokenizer)
completion_logprobs = _convert_chat_logprobs_to_completion_logprobs(
resolved
)
choices.append(
CompletionResponseChoice(
index=index,
text=decoded_text,
finish_reason=choice.finish_reason,
logprobs=completion_logprobs,
)
)
total_completion_tokens += len(choice.token_ids)
index += 1
total_prompt_tokens += pt
if not request.generate_responses:
return self.create_error_response("generate_responses must not be empty")
first = request.generate_responses[0]
kv_params = first.kv_transfer_params
if any(
r.kv_transfer_params != kv_params for r in request.generate_responses[1:]
):
logger.warning(
"derender_completion: kv_transfer_params differ across responses; "
"setting to None on the aggregated response"
)
kv_params = None
usage = UsageInfo(
prompt_tokens=total_prompt_tokens,
completion_tokens=total_completion_tokens,
total_tokens=total_prompt_tokens + total_completion_tokens,
)
logger.debug(
"derender_completion request_id=%s model=%s choices=%d"
" completion_tokens=%d",
first.request_id,
request.model,
len(choices),
total_completion_tokens,
)
return CompletionResponse(
id=first.request_id,
model=request.model,
created=int(time.time()),
choices=choices,
usage=usage,
kv_transfer_params=kv_params,
)
def create_error_response(
self,
message: str | Exception,