[Render] Add /derender endpoints for disaggregated postprocessing (#43606)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
co-authored by
Isotr0py
parent
1033ffac2e
commit
43f0e024bc
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import time
|
||||
from collections.abc import Sequence
|
||||
from http import HTTPStatus
|
||||
from typing import Any, cast
|
||||
@@ -11,11 +12,24 @@ from vllm.entrypoints.chat_utils import (
|
||||
ChatTemplateContentFormatOption,
|
||||
ConversationMessage,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionLogProbs,
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponse,
|
||||
ChatCompletionResponseChoice,
|
||||
ChatMessage,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import (
|
||||
CompletionLogProbs,
|
||||
CompletionRequest,
|
||||
CompletionResponse,
|
||||
CompletionResponseChoice,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
UsageInfo,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.serving import resolve_token_id_placeholder
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
|
||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
build_harmony_preamble,
|
||||
@@ -26,7 +40,10 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.serve.disagg.mm_serde import encode_mm_kwargs_item
|
||||
from vllm.entrypoints.serve.disagg.protocol import (
|
||||
DerenderChatRequest,
|
||||
DerenderCompletionRequest,
|
||||
GenerateRequest,
|
||||
GenerateResponseChoice,
|
||||
MultiModalFeatures,
|
||||
PlaceholderRangeInfo,
|
||||
)
|
||||
@@ -51,6 +68,7 @@ from vllm.renderers.inputs.preprocess import (
|
||||
parse_model_prompt,
|
||||
prompt_to_seq,
|
||||
)
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser
|
||||
from vllm.utils.mistral import mt as _mt
|
||||
@@ -58,6 +76,90 @@ from vllm.utils.mistral import mt as _mt
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _resolve_logprobs(
|
||||
logprobs: ChatCompletionLogProbs, tokenizer: TokenizerLike
|
||||
) -> ChatCompletionLogProbs:
|
||||
"""Resolve all token_id:N placeholders in a ChatCompletionLogProbs object."""
|
||||
if logprobs.content is None:
|
||||
return logprobs
|
||||
resolved_content = []
|
||||
for entry in logprobs.content:
|
||||
token_str, token_bytes = resolve_token_id_placeholder(entry.token, tokenizer)
|
||||
resolved_top = []
|
||||
for top in entry.top_logprobs:
|
||||
top_str, top_bytes = resolve_token_id_placeholder(top.token, tokenizer)
|
||||
resolved_top.append(
|
||||
top.model_copy(update={"token": top_str, "bytes": top_bytes})
|
||||
)
|
||||
resolved_content.append(
|
||||
entry.model_copy(
|
||||
update={
|
||||
"token": token_str,
|
||||
"bytes": token_bytes,
|
||||
"top_logprobs": resolved_top,
|
||||
}
|
||||
)
|
||||
)
|
||||
return ChatCompletionLogProbs(content=resolved_content)
|
||||
|
||||
|
||||
def _convert_chat_logprobs_to_completion_logprobs(
|
||||
logprobs: ChatCompletionLogProbs,
|
||||
) -> CompletionLogProbs:
|
||||
"""Convert ChatCompletionLogProbs (per-token objects) to CompletionLogProbs
|
||||
(parallel flat lists) as required by the /v1/completions response schema."""
|
||||
if logprobs.content is None:
|
||||
return CompletionLogProbs()
|
||||
|
||||
tokens: list[str] = []
|
||||
token_logprobs: list[float | None] = []
|
||||
top_logprobs_list: list[dict[str, float] | None] = []
|
||||
text_offset: list[int] = []
|
||||
|
||||
offset = 0
|
||||
for entry in logprobs.content:
|
||||
text_offset.append(offset)
|
||||
tokens.append(entry.token)
|
||||
token_logprobs.append(entry.logprob)
|
||||
top_logprobs_list.append(
|
||||
{t.token: t.logprob for t in entry.top_logprobs}
|
||||
if entry.top_logprobs
|
||||
else None
|
||||
)
|
||||
offset += len(entry.token)
|
||||
|
||||
return CompletionLogProbs(
|
||||
text_offset=text_offset,
|
||||
token_logprobs=token_logprobs,
|
||||
tokens=tokens,
|
||||
top_logprobs=top_logprobs_list,
|
||||
)
|
||||
|
||||
|
||||
def _build_chat_choice(
|
||||
choice: GenerateResponseChoice, tokenizer: TokenizerLike
|
||||
) -> ChatCompletionResponseChoice:
|
||||
"""Detokenize and resolve logprobs for a single GenerateResponseChoice.
|
||||
|
||||
Raises:
|
||||
ValueError: if choice.token_ids is empty or None.
|
||||
"""
|
||||
if not choice.token_ids:
|
||||
raise ValueError(f"choice {choice.index} has empty or null token_ids")
|
||||
decoded_text = tokenizer.decode(choice.token_ids, skip_special_tokens=True)
|
||||
resolved_logprobs = (
|
||||
_resolve_logprobs(choice.logprobs, tokenizer)
|
||||
if choice.logprobs is not None
|
||||
else None
|
||||
)
|
||||
return ChatCompletionResponseChoice(
|
||||
index=choice.index,
|
||||
message=ChatMessage(role="assistant", content=decoded_text),
|
||||
logprobs=resolved_logprobs,
|
||||
finish_reason=choice.finish_reason,
|
||||
)
|
||||
|
||||
|
||||
class OpenAIServingRender:
|
||||
def __init__(
|
||||
self,
|
||||
@@ -427,6 +529,146 @@ class OpenAIServingRender:
|
||||
|
||||
return messages, [engine_input]
|
||||
|
||||
async def derender_chat_response(
|
||||
self,
|
||||
request: DerenderChatRequest,
|
||||
) -> ChatCompletionResponse | ErrorResponse:
|
||||
"""Postprocess a GenerateResponse into a ChatCompletionResponse.
|
||||
|
||||
This is the symmetric inverse of render_chat_request: it detokenizes
|
||||
output token IDs, resolves token_id:N logprob placeholders, and
|
||||
formats the result as an OpenAI-compatible chat completion response.
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
tokenizer = self.renderer.get_tokenizer()
|
||||
gen = request.generate_response
|
||||
choices: list[ChatCompletionResponseChoice] = []
|
||||
|
||||
try:
|
||||
for choice in gen.choices:
|
||||
choices.append(_build_chat_choice(choice, tokenizer))
|
||||
except ValueError as exc:
|
||||
return self.create_error_response(str(exc))
|
||||
|
||||
prompt_tokens = (
|
||||
request.prompt_tokens if request.prompt_tokens is not None else 0
|
||||
)
|
||||
completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids)
|
||||
usage = UsageInfo(
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
total_tokens=prompt_tokens + completion_tokens,
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"derender_chat request_id=%s model=%s choices=%d completion_tokens=%d",
|
||||
gen.request_id,
|
||||
request.model,
|
||||
len(choices),
|
||||
completion_tokens,
|
||||
)
|
||||
return ChatCompletionResponse(
|
||||
id=gen.request_id,
|
||||
model=request.model,
|
||||
created=int(time.time()),
|
||||
choices=choices,
|
||||
usage=usage,
|
||||
prompt_logprobs=gen.prompt_logprobs,
|
||||
kv_transfer_params=gen.kv_transfer_params,
|
||||
)
|
||||
|
||||
async def derender_completion_response(
|
||||
self,
|
||||
request: DerenderCompletionRequest,
|
||||
) -> CompletionResponse | ErrorResponse:
|
||||
"""Postprocess a list of GenerateResponses into a CompletionResponse.
|
||||
|
||||
Mirrors the multi-prompt completions case: one GenerateResponse per
|
||||
prompt, parallel to the list[GenerateRequest] from /v1/completions/render.
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
n = len(request.generate_responses)
|
||||
prompt_tokens_list: list[int] = (
|
||||
request.prompt_tokens if request.prompt_tokens is not None else [0] * n
|
||||
)
|
||||
|
||||
tokenizer = self.renderer.get_tokenizer()
|
||||
choices: list[CompletionResponseChoice] = []
|
||||
total_prompt_tokens = 0
|
||||
total_completion_tokens = 0
|
||||
index = 0
|
||||
|
||||
for gen, pt in zip(request.generate_responses, prompt_tokens_list):
|
||||
for choice in gen.choices:
|
||||
if not choice.token_ids:
|
||||
return self.create_error_response(
|
||||
f"choice {choice.index} in response {gen.request_id} "
|
||||
"has empty or null token_ids"
|
||||
)
|
||||
decoded_text = tokenizer.decode(
|
||||
choice.token_ids, skip_special_tokens=True
|
||||
)
|
||||
completion_logprobs = None
|
||||
if choice.logprobs is not None:
|
||||
resolved = _resolve_logprobs(choice.logprobs, tokenizer)
|
||||
completion_logprobs = _convert_chat_logprobs_to_completion_logprobs(
|
||||
resolved
|
||||
)
|
||||
choices.append(
|
||||
CompletionResponseChoice(
|
||||
index=index,
|
||||
text=decoded_text,
|
||||
finish_reason=choice.finish_reason,
|
||||
logprobs=completion_logprobs,
|
||||
)
|
||||
)
|
||||
total_completion_tokens += len(choice.token_ids)
|
||||
index += 1
|
||||
total_prompt_tokens += pt
|
||||
|
||||
if not request.generate_responses:
|
||||
return self.create_error_response("generate_responses must not be empty")
|
||||
|
||||
first = request.generate_responses[0]
|
||||
kv_params = first.kv_transfer_params
|
||||
if any(
|
||||
r.kv_transfer_params != kv_params for r in request.generate_responses[1:]
|
||||
):
|
||||
logger.warning(
|
||||
"derender_completion: kv_transfer_params differ across responses; "
|
||||
"setting to None on the aggregated response"
|
||||
)
|
||||
kv_params = None
|
||||
|
||||
usage = UsageInfo(
|
||||
prompt_tokens=total_prompt_tokens,
|
||||
completion_tokens=total_completion_tokens,
|
||||
total_tokens=total_prompt_tokens + total_completion_tokens,
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"derender_completion request_id=%s model=%s choices=%d"
|
||||
" completion_tokens=%d",
|
||||
first.request_id,
|
||||
request.model,
|
||||
len(choices),
|
||||
total_completion_tokens,
|
||||
)
|
||||
return CompletionResponse(
|
||||
id=first.request_id,
|
||||
model=request.model,
|
||||
created=int(time.time()),
|
||||
choices=choices,
|
||||
usage=usage,
|
||||
kv_transfer_params=kv_params,
|
||||
)
|
||||
|
||||
def create_error_response(
|
||||
self,
|
||||
message: str | Exception,
|
||||
|
||||
Reference in New Issue
Block a user