diff --git a/tests/entrypoints/serve/render/test_derender.py b/tests/entrypoints/serve/render/test_derender.py new file mode 100644 index 0000000000..a3006595c1 --- /dev/null +++ b/tests/entrypoints/serve/render/test_derender.py @@ -0,0 +1,488 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Tests for the /derender endpoints (postprocessing counterpart to /render).""" + +import httpx +import pytest +import pytest_asyncio + +from tests.utils import RemoteLaunchRenderServer + +MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM" + + +@pytest.fixture(scope="module") +def server(): + with RemoteLaunchRenderServer(MODEL_NAME, []) as remote_server: + yield remote_server + + +@pytest_asyncio.fixture +async def client(server): + async with httpx.AsyncClient( + base_url=server.url_for(""), timeout=30.0 + ) as http_client: + yield http_client + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +async def _render_chat(client: httpx.AsyncClient) -> dict: + """Render a minimal chat request and return the GenerateRequest dict.""" + resp = await client.post( + "/v1/chat/completions/render", + json={ + "model": MODEL_NAME, + "messages": [{"role": "user", "content": "Hello"}], + }, + ) + assert resp.status_code == 200 + return resp.json() + + +def _make_generate_response( + token_ids: list[int] | None, + request_id: str = "chatcmpl-test-id", + finish_reason: str = "stop", + logprobs: dict | None = None, + prompt_logprobs: list | None = None, + kv_transfer_params: dict | None = None, +) -> dict: + choice: dict = { + "index": 0, + "token_ids": token_ids, + "finish_reason": finish_reason, + "logprobs": logprobs, + } + return { + "request_id": request_id, + "choices": [choice], + "prompt_logprobs": prompt_logprobs, + "kv_transfer_params": kv_transfer_params, + } + + +def _make_logprobs_with_placeholders(token_id: int = 1234) -> dict: + entry = { + "token": f"token_id:{token_id}", + "logprob": -1.0, + "bytes": None, + "top_logprobs": [ + {"token": f"token_id:{token_id + 1}", "logprob": -2.0, "bytes": None} + ], + } + return {"content": [entry]} + + +# --------------------------------------------------------------------------- +# Chat derender tests +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_derender_chat_roundtrip(client): + """Render then derender: decoded content should be a non-empty string.""" + gen_req = await _render_chat(client) + # Use the first 5 rendered token IDs as synthetic "generated" tokens. + synthetic_ids = gen_req["token_ids"][:5] + + response = await client.post( + "/v1/chat/completions/derender", + json={ + "model": MODEL_NAME, + "generate_response": _make_generate_response(synthetic_ids), + }, + ) + assert response.status_code == 200 + data = response.json() + assert data["object"] == "chat.completion" + assert len(data["choices"]) == 1 + assert data["choices"][0]["message"]["content"] + assert data["choices"][0]["message"]["role"] == "assistant" + + +@pytest.mark.asyncio +async def test_derender_chat_usage(client): + """Supplied prompt_tokens flows through into usage correctly.""" + gen_req = await _render_chat(client) + synthetic_ids = gen_req["token_ids"][:3] + + response = await client.post( + "/v1/chat/completions/derender", + json={ + "model": MODEL_NAME, + "generate_response": _make_generate_response(synthetic_ids), + "prompt_tokens": 10, + }, + ) + assert response.status_code == 200 + usage = response.json()["usage"] + assert usage["prompt_tokens"] == 10 + assert usage["completion_tokens"] == len(synthetic_ids) + assert usage["total_tokens"] == 10 + len(synthetic_ids) + + +@pytest.mark.asyncio +async def test_derender_chat_usage_default(client): + """Omitting prompt_tokens gives usage.prompt_tokens == 0.""" + gen_req = await _render_chat(client) + synthetic_ids = gen_req["token_ids"][:3] + + response = await client.post( + "/v1/chat/completions/derender", + json={ + "model": MODEL_NAME, + "generate_response": _make_generate_response(synthetic_ids), + }, + ) + assert response.status_code == 200 + usage = response.json()["usage"] + assert usage["prompt_tokens"] == 0 + + +@pytest.mark.asyncio +async def test_derender_chat_logprobs(client): + """token_id:N placeholders in content.token are resolved to real strings.""" + gen_req = await _render_chat(client) + synthetic_ids = gen_req["token_ids"][:3] + token_id = synthetic_ids[0] + + response = await client.post( + "/v1/chat/completions/derender", + json={ + "model": MODEL_NAME, + "generate_response": _make_generate_response( + synthetic_ids, + logprobs=_make_logprobs_with_placeholders(token_id), + ), + }, + ) + assert response.status_code == 200 + data = response.json() + logprobs = data["choices"][0]["logprobs"] + assert logprobs is not None + content = logprobs["content"] + assert content is not None and len(content) == 1 + token_str = content[0]["token"] + assert not token_str.startswith("token_id:"), ( + f"Placeholder was not resolved: {token_str!r}" + ) + + +@pytest.mark.asyncio +async def test_derender_chat_logprobs_bytes(client): + """Resolved logprob entries have bytes populated as list[int].""" + gen_req = await _render_chat(client) + synthetic_ids = gen_req["token_ids"][:3] + token_id = synthetic_ids[0] + + response = await client.post( + "/v1/chat/completions/derender", + json={ + "model": MODEL_NAME, + "generate_response": _make_generate_response( + synthetic_ids, + logprobs=_make_logprobs_with_placeholders(token_id), + ), + }, + ) + assert response.status_code == 200 + content = response.json()["choices"][0]["logprobs"]["content"] + bytes_field = content[0]["bytes"] + assert isinstance(bytes_field, list) + assert len(bytes_field) > 0 + assert all(isinstance(b, int) for b in bytes_field) + + +@pytest.mark.asyncio +async def test_derender_chat_top_logprobs(client): + """top_logprobs entries also have their placeholders resolved.""" + gen_req = await _render_chat(client) + synthetic_ids = gen_req["token_ids"][:3] + token_id = synthetic_ids[0] + + response = await client.post( + "/v1/chat/completions/derender", + json={ + "model": MODEL_NAME, + "generate_response": _make_generate_response( + synthetic_ids, + logprobs=_make_logprobs_with_placeholders(token_id), + ), + }, + ) + assert response.status_code == 200 + content = response.json()["choices"][0]["logprobs"]["content"] + top = content[0]["top_logprobs"] + assert len(top) == 1 + assert not top[0]["token"].startswith("token_id:"), ( + f"top_logprobs placeholder not resolved: {top[0]['token']!r}" + ) + + +@pytest.mark.asyncio +async def test_derender_chat_prompt_logprobs_passthrough(client): + """prompt_logprobs on GenerateResponse passes through unchanged.""" + gen_req = await _render_chat(client) + synthetic_ids = gen_req["token_ids"][:3] + # prompt_logprobs is a list[dict[int, Logprob] | None]; use None entries. + prompt_logprobs = [None, None] + + response = await client.post( + "/v1/chat/completions/derender", + json={ + "model": MODEL_NAME, + "generate_response": _make_generate_response( + synthetic_ids, prompt_logprobs=prompt_logprobs + ), + }, + ) + assert response.status_code == 200 + assert response.json()["prompt_logprobs"] == prompt_logprobs + + +@pytest.mark.asyncio +async def test_derender_chat_kv_transfer_params_passthrough(client): + """kv_transfer_params passes through to the ChatCompletionResponse.""" + gen_req = await _render_chat(client) + synthetic_ids = gen_req["token_ids"][:3] + kv = {"key": "value"} + + response = await client.post( + "/v1/chat/completions/derender", + json={ + "model": MODEL_NAME, + "generate_response": _make_generate_response( + synthetic_ids, kv_transfer_params=kv + ), + }, + ) + assert response.status_code == 200 + assert response.json()["kv_transfer_params"] == kv + + +@pytest.mark.asyncio +async def test_derender_chat_empty_token_ids(client): + """Empty token_ids list returns 400.""" + response = await client.post( + "/v1/chat/completions/derender", + json={ + "model": MODEL_NAME, + "generate_response": _make_generate_response([]), + }, + ) + assert response.status_code == 400 + + +@pytest.mark.asyncio +async def test_derender_chat_null_token_ids(client): + """Null token_ids returns 400.""" + response = await client.post( + "/v1/chat/completions/derender", + json={ + "model": MODEL_NAME, + "generate_response": _make_generate_response(None), + }, + ) + assert response.status_code == 400 + + +@pytest.mark.asyncio +async def test_derender_chat_unknown_model(client): + """Unknown model returns 404.""" + gen_req = await _render_chat(client) + synthetic_ids = gen_req["token_ids"][:3] + + response = await client.post( + "/v1/chat/completions/derender", + json={ + "model": "does-not-exist", + "generate_response": _make_generate_response(synthetic_ids), + }, + ) + assert response.status_code == 404 + + +# --------------------------------------------------------------------------- +# Completion derender tests +# --------------------------------------------------------------------------- + + +async def _render_completion(client: httpx.AsyncClient, prompt: str) -> dict: + """Render a completion prompt and return the first GenerateRequest dict.""" + resp = await client.post( + "/v1/completions/render", + json={"model": MODEL_NAME, "prompt": prompt}, + ) + assert resp.status_code == 200 + data = resp.json() + assert isinstance(data, list) and len(data) >= 1 + return data[0] + + +def _make_completion_generate_response( + token_ids: list[int], + request_id: str, + kv_transfer_params: dict | None = None, + logprobs: dict | None = None, +) -> dict: + return { + "request_id": request_id, + "choices": [ + { + "index": 0, + "token_ids": token_ids, + "finish_reason": "stop", + "logprobs": logprobs, + } + ], + "prompt_logprobs": None, + "kv_transfer_params": kv_transfer_params, + } + + +@pytest.mark.asyncio +async def test_derender_completion_roundtrip(client): + """Two prompts rendered, two GenerateResponses → two choices with indices 0, 1.""" + gr1 = await _render_completion(client, "Hello world") + gr2 = await _render_completion(client, "Goodbye world") + + ids1 = gr1["token_ids"][:4] + ids2 = gr2["token_ids"][:4] + + response = await client.post( + "/v1/completions/derender", + json={ + "model": MODEL_NAME, + "generate_responses": [ + _make_completion_generate_response(ids1, gr1["request_id"]), + _make_completion_generate_response(ids2, gr2["request_id"]), + ], + }, + ) + assert response.status_code == 200 + data = response.json() + assert data["object"] == "text_completion" + choices = data["choices"] + assert len(choices) == 2 + assert choices[0]["index"] == 0 + assert choices[1]["index"] == 1 + assert choices[0]["text"] + assert choices[1]["text"] + + +@pytest.mark.asyncio +async def test_derender_completion_usage_aggregation(client): + """prompt_tokens=[5, 10] is aggregated correctly into usage.""" + gr1 = await _render_completion(client, "Hello") + gr2 = await _render_completion(client, "World") + + ids1 = gr1["token_ids"][:3] + ids2 = gr2["token_ids"][:4] + + response = await client.post( + "/v1/completions/derender", + json={ + "model": MODEL_NAME, + "generate_responses": [ + _make_completion_generate_response(ids1, gr1["request_id"]), + _make_completion_generate_response(ids2, gr2["request_id"]), + ], + "prompt_tokens": [5, 10], + }, + ) + assert response.status_code == 200 + usage = response.json()["usage"] + assert usage["prompt_tokens"] == 15 + assert usage["completion_tokens"] == len(ids1) + len(ids2) + assert usage["total_tokens"] == 15 + len(ids1) + len(ids2) + + +@pytest.mark.asyncio +async def test_derender_completion_prompt_tokens_length_mismatch(client): + """len(prompt_tokens) != len(generate_responses) returns 400.""" + gr1 = await _render_completion(client, "Hello") + ids1 = gr1["token_ids"][:3] + + response = await client.post( + "/v1/completions/derender", + json={ + "model": MODEL_NAME, + "generate_responses": [ + _make_completion_generate_response(ids1, gr1["request_id"]), + ], + "prompt_tokens": [5, 10], + }, + ) + assert response.status_code == 400 + + +@pytest.mark.asyncio +async def test_derender_completion_empty_generate_responses(client): + """Empty generate_responses list returns 400.""" + response = await client.post( + "/v1/completions/derender", + json={"model": MODEL_NAME, "generate_responses": []}, + ) + assert response.status_code == 400 + + +@pytest.mark.asyncio +async def test_derender_completion_logprobs(client): + """token_id:N placeholders in logprobs are resolved; CompletionLogProbs + flat-list structure is returned with non-empty tokens and text_offsets.""" + gr1 = await _render_completion(client, "Hello world") + ids1 = gr1["token_ids"][:3] + token_id = ids1[0] + + response = await client.post( + "/v1/completions/derender", + json={ + "model": MODEL_NAME, + "generate_responses": [ + _make_completion_generate_response( + ids1, + gr1["request_id"], + logprobs=_make_logprobs_with_placeholders(token_id), + ), + ], + }, + ) + assert response.status_code == 200 + logprobs = response.json()["choices"][0]["logprobs"] + assert logprobs is not None + tokens = logprobs["tokens"] + assert len(tokens) == 1 + assert not tokens[0].startswith("token_id:"), ( + f"Placeholder was not resolved: {tokens[0]!r}" + ) + assert len(logprobs["token_logprobs"]) == 1 + assert isinstance(logprobs["token_logprobs"][0], float) + assert len(logprobs["text_offset"]) == 1 + assert logprobs["text_offset"][0] == 0 + + +@pytest.mark.asyncio +async def test_derender_completion_kv_transfer_params_passthrough(client): + """kv_transfer_params passes through to CompletionResponse.""" + gr1 = await _render_completion(client, "Hello") + ids1 = gr1["token_ids"][:3] + kv = {"node": "abc"} + + response = await client.post( + "/v1/completions/derender", + json={ + "model": MODEL_NAME, + "generate_responses": [ + _make_completion_generate_response( + ids1, gr1["request_id"], kv_transfer_params=kv + ), + ], + }, + ) + assert response.status_code == 200 + assert response.json()["kv_transfer_params"] == kv diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index 45b79c6a7e..b570b0c987 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -46,6 +46,7 @@ from vllm.entrypoints.openai.engine.serving import ( GenerationError, OpenAIServing, clamp_prompt_logprobs, + format_token_id_placeholder, ) from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage @@ -1129,7 +1130,7 @@ class OpenAIServingChat(OpenAIServing): step_top_logprobs = top_logprobs[i] if step_top_logprobs is None or step_top_logprobs.get(token_id) is None: if should_return_as_token_id: - token = f"token_id:{token_id}" + token = format_token_id_placeholder(token_id) else: if tokenizer is None: raise ValueError( diff --git a/vllm/entrypoints/openai/completion/serving.py b/vllm/entrypoints/openai/completion/serving.py index bd7e26b2b1..fef1741351 100644 --- a/vllm/entrypoints/openai/completion/serving.py +++ b/vllm/entrypoints/openai/completion/serving.py @@ -31,6 +31,7 @@ from vllm.entrypoints.openai.engine.serving import ( GenerationError, OpenAIServing, clamp_prompt_logprobs, + format_token_id_placeholder, ) from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage @@ -628,7 +629,7 @@ class OpenAIServingCompletion(OpenAIServing): step_top_logprobs = top_logprobs[i] if step_top_logprobs is None: if should_return_as_token_id: - token = f"token_id:{token_id}" + token = format_token_id_placeholder(token_id) else: if tokenizer is None: raise VLLMValidationError( diff --git a/vllm/entrypoints/openai/engine/serving.py b/vllm/entrypoints/openai/engine/serving.py index f3e07336e8..5eb917ef96 100644 --- a/vllm/entrypoints/openai/engine/serving.py +++ b/vllm/entrypoints/openai/engine/serving.py @@ -452,7 +452,7 @@ class OpenAIServing(BeamSearchOnlineMixin): return_as_token_id: bool = False, ) -> str: if return_as_token_id: - return f"token_id:{token_id}" + return format_token_id_placeholder(token_id) if logprob.decoded_token is not None: return logprob.decoded_token @@ -472,6 +472,38 @@ class OpenAIServing(BeamSearchOnlineMixin): return self.models.is_base_model(model_name) +def format_token_id_placeholder(token_id: int) -> str: + return f"token_id:{token_id}" + + +def resolve_token_id_placeholder( + token: str, tokenizer: TokenizerLike +) -> tuple[str, list[int] | None]: + """Decode a 'token_id:N' placeholder back to a token string and UTF-8 bytes. + + Returns (token, None) unchanged if token is not a placeholder. + This is the inverse of format_token_id_placeholder / _get_decoded_token + when return_as_token_id=True. + """ + suffix = token.removeprefix("token_id:") + if suffix == token: + return token, None + try: + token_id = int(suffix) + except ValueError: + return token, None + token_repr = tokenizer.convert_ids_to_tokens([token_id])[0] + if token_repr is None: + logger.warning_once( + "resolve_token_id_placeholder: token_id %d has no vocab entry; " + "substituting empty string", + token_id, + ) + return "", None + token_str = tokenizer.convert_tokens_to_string([token_repr]) + return token_str, list(token_str.encode("utf-8", errors="replace")) + + def clamp_prompt_logprobs( prompt_logprobs: PromptLogprobs | None, ) -> PromptLogprobs | None: diff --git a/vllm/entrypoints/serve/disagg/protocol.py b/vllm/entrypoints/serve/disagg/protocol.py index 60d2a6424a..c13c4c1705 100644 --- a/vllm/entrypoints/serve/disagg/protocol.py +++ b/vllm/entrypoints/serve/disagg/protocol.py @@ -11,7 +11,11 @@ from pydantic import ( ) from vllm.config import ModelConfig -from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionLogProbs +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionLogProbs, + ChatCompletionRequest, +) +from vllm.entrypoints.openai.completion.protocol import CompletionRequest from vllm.entrypoints.openai.engine.protocol import StreamOptions, UsageInfo from vllm.logprobs import Logprob from vllm.renderers import TokenizeParams @@ -209,3 +213,66 @@ class GenerateResponse(BaseModel): default=None, description="KVTransfer parameters used for disaggregated serving.", ) + + +####### Derender (postprocessing) ####### + + +class DerenderChatRequest(BaseModel): + """Request for the /v1/chat/completions/derender endpoint. + + Wraps a GenerateResponse and caller-supplied metadata needed to produce + a fully-formed ChatCompletionResponse without a GPU. + """ + + model: str + generate_response: GenerateResponse + prompt_tokens: int | None = None + """Prompt token count for usage; defaults to 0 if omitted. + + GenerateResponse carries only output tokens; the caller already has + len(GenerateRequest.token_ids) from the render step. + """ + + chat_request: ChatCompletionRequest | None = None + """The original (post-adjust_request) ChatCompletionRequest from /render. + + Required by the parsing so that tool/reasoning parsers can receive the full + request context they expect (request.tools, request.tool_choice, + request._grammar_from_tool_parser, etc.). + """ + + +class DerenderCompletionRequest(BaseModel): + """Request for the /v1/completions/derender endpoint. + + Parallel to DerenderChatRequest but handles the multi-prompt completions + case: one GenerateResponse per prompt, mirroring the list[GenerateRequest] + returned by /v1/completions/render. + """ + + model: str + generate_responses: list[GenerateResponse] + prompt_tokens: list[int] | None = None + """One prompt token count per response; each defaults to 0 if omitted. + + If provided, len(prompt_tokens) must equal len(generate_responses). + """ + + completion_request: CompletionRequest | None = None + """The original (post-adjust_request) CompletionRequest from /render. + + Mirrors chat_request on DerenderChatRequest. Required by the parsing + so parsers receive the full request context. + """ + + @model_validator(mode="after") + def _validate_prompt_tokens_length(self) -> "DerenderCompletionRequest": + if self.prompt_tokens is not None and len(self.prompt_tokens) != len( + self.generate_responses + ): + raise ValueError( + f"prompt_tokens length ({len(self.prompt_tokens)}) must equal " + f"generate_responses length ({len(self.generate_responses)})" + ) + return self diff --git a/vllm/entrypoints/serve/render/api_router.py b/vllm/entrypoints/serve/render/api_router.py index ac0c1ce67d..350260c188 100644 --- a/vllm/entrypoints/serve/render/api_router.py +++ b/vllm/entrypoints/serve/render/api_router.py @@ -5,10 +5,20 @@ from http import HTTPStatus from fastapi import APIRouter, Depends, FastAPI, Request from fastapi.responses import JSONResponse -from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest -from vllm.entrypoints.openai.completion.protocol import CompletionRequest +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest, + ChatCompletionResponse, +) +from vllm.entrypoints.openai.completion.protocol import ( + CompletionRequest, + CompletionResponse, +) from vllm.entrypoints.openai.engine.protocol import ErrorResponse -from vllm.entrypoints.serve.disagg.protocol import GenerateRequest +from vllm.entrypoints.serve.disagg.protocol import ( + DerenderChatRequest, + DerenderCompletionRequest, + GenerateRequest, +) from vllm.entrypoints.serve.render.serving import OpenAIServingRender from vllm.entrypoints.serve.utils.api_utils import validate_json_request from vllm.logger import init_logger @@ -71,5 +81,53 @@ async def render_completion(request: CompletionRequest, raw_request: Request): return JSONResponse(content=[item.model_dump() for item in result]) +@router.post( + "/v1/chat/completions/derender", + dependencies=[Depends(validate_json_request)], + response_model=ChatCompletionResponse, + responses={ + HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse}, + HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse}, + HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse}, + }, +) +async def derender_chat_completion(request: DerenderChatRequest, raw_request: Request): + handler = render(raw_request) + if handler is None: + raise NotImplementedError( + "The model does not support Chat Completions Derender API" + ) + + result = await handler.derender_chat_response(request) + + if isinstance(result, ErrorResponse): + return JSONResponse(content=result.model_dump(), status_code=result.error.code) + + return JSONResponse(content=result.model_dump()) + + +@router.post( + "/v1/completions/derender", + dependencies=[Depends(validate_json_request)], + response_model=CompletionResponse, + responses={ + HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse}, + HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse}, + HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse}, + }, +) +async def derender_completion(request: DerenderCompletionRequest, raw_request: Request): + handler = render(raw_request) + if handler is None: + raise NotImplementedError("The model does not support Completions Derender API") + + result = await handler.derender_completion_response(request) + + if isinstance(result, ErrorResponse): + return JSONResponse(content=result.model_dump(), status_code=result.error.code) + + return JSONResponse(content=result.model_dump()) + + def attach_router(app: FastAPI) -> None: app.include_router(router) diff --git a/vllm/entrypoints/serve/render/serving.py b/vllm/entrypoints/serve/render/serving.py index 6afb26d984..05a2911983 100644 --- a/vllm/entrypoints/serve/render/serving.py +++ b/vllm/entrypoints/serve/render/serving.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import time from collections.abc import Sequence from http import HTTPStatus from typing import Any, cast @@ -11,11 +12,24 @@ from vllm.entrypoints.chat_utils import ( ChatTemplateContentFormatOption, ConversationMessage, ) -from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest -from vllm.entrypoints.openai.completion.protocol import CompletionRequest +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionLogProbs, + ChatCompletionRequest, + ChatCompletionResponse, + ChatCompletionResponseChoice, + ChatMessage, +) +from vllm.entrypoints.openai.completion.protocol import ( + CompletionLogProbs, + CompletionRequest, + CompletionResponse, + CompletionResponseChoice, +) from vllm.entrypoints.openai.engine.protocol import ( ErrorResponse, + UsageInfo, ) +from vllm.entrypoints.openai.engine.serving import resolve_token_id_placeholder from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry from vllm.entrypoints.openai.parser.harmony_utils import ( build_harmony_preamble, @@ -26,7 +40,10 @@ from vllm.entrypoints.openai.parser.harmony_utils import ( from vllm.entrypoints.openai.responses.protocol import ResponsesRequest from vllm.entrypoints.serve.disagg.mm_serde import encode_mm_kwargs_item from vllm.entrypoints.serve.disagg.protocol import ( + DerenderChatRequest, + DerenderCompletionRequest, GenerateRequest, + GenerateResponseChoice, MultiModalFeatures, PlaceholderRangeInfo, ) @@ -51,6 +68,7 @@ from vllm.renderers.inputs.preprocess import ( parse_model_prompt, prompt_to_seq, ) +from vllm.tokenizers import TokenizerLike from vllm.utils import random_uuid from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser from vllm.utils.mistral import mt as _mt @@ -58,6 +76,90 @@ from vllm.utils.mistral import mt as _mt logger = init_logger(__name__) +def _resolve_logprobs( + logprobs: ChatCompletionLogProbs, tokenizer: TokenizerLike +) -> ChatCompletionLogProbs: + """Resolve all token_id:N placeholders in a ChatCompletionLogProbs object.""" + if logprobs.content is None: + return logprobs + resolved_content = [] + for entry in logprobs.content: + token_str, token_bytes = resolve_token_id_placeholder(entry.token, tokenizer) + resolved_top = [] + for top in entry.top_logprobs: + top_str, top_bytes = resolve_token_id_placeholder(top.token, tokenizer) + resolved_top.append( + top.model_copy(update={"token": top_str, "bytes": top_bytes}) + ) + resolved_content.append( + entry.model_copy( + update={ + "token": token_str, + "bytes": token_bytes, + "top_logprobs": resolved_top, + } + ) + ) + return ChatCompletionLogProbs(content=resolved_content) + + +def _convert_chat_logprobs_to_completion_logprobs( + logprobs: ChatCompletionLogProbs, +) -> CompletionLogProbs: + """Convert ChatCompletionLogProbs (per-token objects) to CompletionLogProbs + (parallel flat lists) as required by the /v1/completions response schema.""" + if logprobs.content is None: + return CompletionLogProbs() + + tokens: list[str] = [] + token_logprobs: list[float | None] = [] + top_logprobs_list: list[dict[str, float] | None] = [] + text_offset: list[int] = [] + + offset = 0 + for entry in logprobs.content: + text_offset.append(offset) + tokens.append(entry.token) + token_logprobs.append(entry.logprob) + top_logprobs_list.append( + {t.token: t.logprob for t in entry.top_logprobs} + if entry.top_logprobs + else None + ) + offset += len(entry.token) + + return CompletionLogProbs( + text_offset=text_offset, + token_logprobs=token_logprobs, + tokens=tokens, + top_logprobs=top_logprobs_list, + ) + + +def _build_chat_choice( + choice: GenerateResponseChoice, tokenizer: TokenizerLike +) -> ChatCompletionResponseChoice: + """Detokenize and resolve logprobs for a single GenerateResponseChoice. + + Raises: + ValueError: if choice.token_ids is empty or None. + """ + if not choice.token_ids: + raise ValueError(f"choice {choice.index} has empty or null token_ids") + decoded_text = tokenizer.decode(choice.token_ids, skip_special_tokens=True) + resolved_logprobs = ( + _resolve_logprobs(choice.logprobs, tokenizer) + if choice.logprobs is not None + else None + ) + return ChatCompletionResponseChoice( + index=choice.index, + message=ChatMessage(role="assistant", content=decoded_text), + logprobs=resolved_logprobs, + finish_reason=choice.finish_reason, + ) + + class OpenAIServingRender: def __init__( self, @@ -427,6 +529,146 @@ class OpenAIServingRender: return messages, [engine_input] + async def derender_chat_response( + self, + request: DerenderChatRequest, + ) -> ChatCompletionResponse | ErrorResponse: + """Postprocess a GenerateResponse into a ChatCompletionResponse. + + This is the symmetric inverse of render_chat_request: it detokenizes + output token IDs, resolves token_id:N logprob placeholders, and + formats the result as an OpenAI-compatible chat completion response. + """ + error_check_ret = await self._check_model(request) + if error_check_ret is not None: + return error_check_ret + + tokenizer = self.renderer.get_tokenizer() + gen = request.generate_response + choices: list[ChatCompletionResponseChoice] = [] + + try: + for choice in gen.choices: + choices.append(_build_chat_choice(choice, tokenizer)) + except ValueError as exc: + return self.create_error_response(str(exc)) + + prompt_tokens = ( + request.prompt_tokens if request.prompt_tokens is not None else 0 + ) + completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids) + usage = UsageInfo( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ) + + logger.debug( + "derender_chat request_id=%s model=%s choices=%d completion_tokens=%d", + gen.request_id, + request.model, + len(choices), + completion_tokens, + ) + return ChatCompletionResponse( + id=gen.request_id, + model=request.model, + created=int(time.time()), + choices=choices, + usage=usage, + prompt_logprobs=gen.prompt_logprobs, + kv_transfer_params=gen.kv_transfer_params, + ) + + async def derender_completion_response( + self, + request: DerenderCompletionRequest, + ) -> CompletionResponse | ErrorResponse: + """Postprocess a list of GenerateResponses into a CompletionResponse. + + Mirrors the multi-prompt completions case: one GenerateResponse per + prompt, parallel to the list[GenerateRequest] from /v1/completions/render. + """ + error_check_ret = await self._check_model(request) + if error_check_ret is not None: + return error_check_ret + + n = len(request.generate_responses) + prompt_tokens_list: list[int] = ( + request.prompt_tokens if request.prompt_tokens is not None else [0] * n + ) + + tokenizer = self.renderer.get_tokenizer() + choices: list[CompletionResponseChoice] = [] + total_prompt_tokens = 0 + total_completion_tokens = 0 + index = 0 + + for gen, pt in zip(request.generate_responses, prompt_tokens_list): + for choice in gen.choices: + if not choice.token_ids: + return self.create_error_response( + f"choice {choice.index} in response {gen.request_id} " + "has empty or null token_ids" + ) + decoded_text = tokenizer.decode( + choice.token_ids, skip_special_tokens=True + ) + completion_logprobs = None + if choice.logprobs is not None: + resolved = _resolve_logprobs(choice.logprobs, tokenizer) + completion_logprobs = _convert_chat_logprobs_to_completion_logprobs( + resolved + ) + choices.append( + CompletionResponseChoice( + index=index, + text=decoded_text, + finish_reason=choice.finish_reason, + logprobs=completion_logprobs, + ) + ) + total_completion_tokens += len(choice.token_ids) + index += 1 + total_prompt_tokens += pt + + if not request.generate_responses: + return self.create_error_response("generate_responses must not be empty") + + first = request.generate_responses[0] + kv_params = first.kv_transfer_params + if any( + r.kv_transfer_params != kv_params for r in request.generate_responses[1:] + ): + logger.warning( + "derender_completion: kv_transfer_params differ across responses; " + "setting to None on the aggregated response" + ) + kv_params = None + + usage = UsageInfo( + prompt_tokens=total_prompt_tokens, + completion_tokens=total_completion_tokens, + total_tokens=total_prompt_tokens + total_completion_tokens, + ) + + logger.debug( + "derender_completion request_id=%s model=%s choices=%d" + " completion_tokens=%d", + first.request_id, + request.model, + len(choices), + total_completion_tokens, + ) + return CompletionResponse( + id=first.request_id, + model=request.model, + created=int(time.time()), + choices=choices, + usage=usage, + kv_transfer_params=kv_params, + ) + def create_error_response( self, message: str | Exception,