[Render] Add /derender endpoints for disaggregated postprocessing (#43606)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
co-authored by
Isotr0py
parent
1033ffac2e
commit
43f0e024bc
@@ -0,0 +1,488 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Tests for the /derender endpoints (postprocessing counterpart to /render)."""
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteLaunchRenderServer
|
||||
|
||||
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
with RemoteLaunchRenderServer(MODEL_NAME, []) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with httpx.AsyncClient(
|
||||
base_url=server.url_for(""), timeout=30.0
|
||||
) as http_client:
|
||||
yield http_client
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _render_chat(client: httpx.AsyncClient) -> dict:
|
||||
"""Render a minimal chat request and return the GenerateRequest dict."""
|
||||
resp = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
return resp.json()
|
||||
|
||||
|
||||
def _make_generate_response(
|
||||
token_ids: list[int] | None,
|
||||
request_id: str = "chatcmpl-test-id",
|
||||
finish_reason: str = "stop",
|
||||
logprobs: dict | None = None,
|
||||
prompt_logprobs: list | None = None,
|
||||
kv_transfer_params: dict | None = None,
|
||||
) -> dict:
|
||||
choice: dict = {
|
||||
"index": 0,
|
||||
"token_ids": token_ids,
|
||||
"finish_reason": finish_reason,
|
||||
"logprobs": logprobs,
|
||||
}
|
||||
return {
|
||||
"request_id": request_id,
|
||||
"choices": [choice],
|
||||
"prompt_logprobs": prompt_logprobs,
|
||||
"kv_transfer_params": kv_transfer_params,
|
||||
}
|
||||
|
||||
|
||||
def _make_logprobs_with_placeholders(token_id: int = 1234) -> dict:
|
||||
entry = {
|
||||
"token": f"token_id:{token_id}",
|
||||
"logprob": -1.0,
|
||||
"bytes": None,
|
||||
"top_logprobs": [
|
||||
{"token": f"token_id:{token_id + 1}", "logprob": -2.0, "bytes": None}
|
||||
],
|
||||
}
|
||||
return {"content": [entry]}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Chat derender tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_roundtrip(client):
|
||||
"""Render then derender: decoded content should be a non-empty string."""
|
||||
gen_req = await _render_chat(client)
|
||||
# Use the first 5 rendered token IDs as synthetic "generated" tokens.
|
||||
synthetic_ids = gen_req["token_ids"][:5]
|
||||
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_response": _make_generate_response(synthetic_ids),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["object"] == "chat.completion"
|
||||
assert len(data["choices"]) == 1
|
||||
assert data["choices"][0]["message"]["content"]
|
||||
assert data["choices"][0]["message"]["role"] == "assistant"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_usage(client):
|
||||
"""Supplied prompt_tokens flows through into usage correctly."""
|
||||
gen_req = await _render_chat(client)
|
||||
synthetic_ids = gen_req["token_ids"][:3]
|
||||
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_response": _make_generate_response(synthetic_ids),
|
||||
"prompt_tokens": 10,
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
usage = response.json()["usage"]
|
||||
assert usage["prompt_tokens"] == 10
|
||||
assert usage["completion_tokens"] == len(synthetic_ids)
|
||||
assert usage["total_tokens"] == 10 + len(synthetic_ids)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_usage_default(client):
|
||||
"""Omitting prompt_tokens gives usage.prompt_tokens == 0."""
|
||||
gen_req = await _render_chat(client)
|
||||
synthetic_ids = gen_req["token_ids"][:3]
|
||||
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_response": _make_generate_response(synthetic_ids),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
usage = response.json()["usage"]
|
||||
assert usage["prompt_tokens"] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_logprobs(client):
|
||||
"""token_id:N placeholders in content.token are resolved to real strings."""
|
||||
gen_req = await _render_chat(client)
|
||||
synthetic_ids = gen_req["token_ids"][:3]
|
||||
token_id = synthetic_ids[0]
|
||||
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_response": _make_generate_response(
|
||||
synthetic_ids,
|
||||
logprobs=_make_logprobs_with_placeholders(token_id),
|
||||
),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
logprobs = data["choices"][0]["logprobs"]
|
||||
assert logprobs is not None
|
||||
content = logprobs["content"]
|
||||
assert content is not None and len(content) == 1
|
||||
token_str = content[0]["token"]
|
||||
assert not token_str.startswith("token_id:"), (
|
||||
f"Placeholder was not resolved: {token_str!r}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_logprobs_bytes(client):
|
||||
"""Resolved logprob entries have bytes populated as list[int]."""
|
||||
gen_req = await _render_chat(client)
|
||||
synthetic_ids = gen_req["token_ids"][:3]
|
||||
token_id = synthetic_ids[0]
|
||||
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_response": _make_generate_response(
|
||||
synthetic_ids,
|
||||
logprobs=_make_logprobs_with_placeholders(token_id),
|
||||
),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
content = response.json()["choices"][0]["logprobs"]["content"]
|
||||
bytes_field = content[0]["bytes"]
|
||||
assert isinstance(bytes_field, list)
|
||||
assert len(bytes_field) > 0
|
||||
assert all(isinstance(b, int) for b in bytes_field)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_top_logprobs(client):
|
||||
"""top_logprobs entries also have their placeholders resolved."""
|
||||
gen_req = await _render_chat(client)
|
||||
synthetic_ids = gen_req["token_ids"][:3]
|
||||
token_id = synthetic_ids[0]
|
||||
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_response": _make_generate_response(
|
||||
synthetic_ids,
|
||||
logprobs=_make_logprobs_with_placeholders(token_id),
|
||||
),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
content = response.json()["choices"][0]["logprobs"]["content"]
|
||||
top = content[0]["top_logprobs"]
|
||||
assert len(top) == 1
|
||||
assert not top[0]["token"].startswith("token_id:"), (
|
||||
f"top_logprobs placeholder not resolved: {top[0]['token']!r}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_prompt_logprobs_passthrough(client):
|
||||
"""prompt_logprobs on GenerateResponse passes through unchanged."""
|
||||
gen_req = await _render_chat(client)
|
||||
synthetic_ids = gen_req["token_ids"][:3]
|
||||
# prompt_logprobs is a list[dict[int, Logprob] | None]; use None entries.
|
||||
prompt_logprobs = [None, None]
|
||||
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_response": _make_generate_response(
|
||||
synthetic_ids, prompt_logprobs=prompt_logprobs
|
||||
),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["prompt_logprobs"] == prompt_logprobs
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_kv_transfer_params_passthrough(client):
|
||||
"""kv_transfer_params passes through to the ChatCompletionResponse."""
|
||||
gen_req = await _render_chat(client)
|
||||
synthetic_ids = gen_req["token_ids"][:3]
|
||||
kv = {"key": "value"}
|
||||
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_response": _make_generate_response(
|
||||
synthetic_ids, kv_transfer_params=kv
|
||||
),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["kv_transfer_params"] == kv
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_empty_token_ids(client):
|
||||
"""Empty token_ids list returns 400."""
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_response": _make_generate_response([]),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_null_token_ids(client):
|
||||
"""Null token_ids returns 400."""
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_response": _make_generate_response(None),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_unknown_model(client):
|
||||
"""Unknown model returns 404."""
|
||||
gen_req = await _render_chat(client)
|
||||
synthetic_ids = gen_req["token_ids"][:3]
|
||||
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": "does-not-exist",
|
||||
"generate_response": _make_generate_response(synthetic_ids),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 404
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Completion derender tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _render_completion(client: httpx.AsyncClient, prompt: str) -> dict:
|
||||
"""Render a completion prompt and return the first GenerateRequest dict."""
|
||||
resp = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={"model": MODEL_NAME, "prompt": prompt},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert isinstance(data, list) and len(data) >= 1
|
||||
return data[0]
|
||||
|
||||
|
||||
def _make_completion_generate_response(
|
||||
token_ids: list[int],
|
||||
request_id: str,
|
||||
kv_transfer_params: dict | None = None,
|
||||
logprobs: dict | None = None,
|
||||
) -> dict:
|
||||
return {
|
||||
"request_id": request_id,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"token_ids": token_ids,
|
||||
"finish_reason": "stop",
|
||||
"logprobs": logprobs,
|
||||
}
|
||||
],
|
||||
"prompt_logprobs": None,
|
||||
"kv_transfer_params": kv_transfer_params,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_completion_roundtrip(client):
|
||||
"""Two prompts rendered, two GenerateResponses → two choices with indices 0, 1."""
|
||||
gr1 = await _render_completion(client, "Hello world")
|
||||
gr2 = await _render_completion(client, "Goodbye world")
|
||||
|
||||
ids1 = gr1["token_ids"][:4]
|
||||
ids2 = gr2["token_ids"][:4]
|
||||
|
||||
response = await client.post(
|
||||
"/v1/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_responses": [
|
||||
_make_completion_generate_response(ids1, gr1["request_id"]),
|
||||
_make_completion_generate_response(ids2, gr2["request_id"]),
|
||||
],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["object"] == "text_completion"
|
||||
choices = data["choices"]
|
||||
assert len(choices) == 2
|
||||
assert choices[0]["index"] == 0
|
||||
assert choices[1]["index"] == 1
|
||||
assert choices[0]["text"]
|
||||
assert choices[1]["text"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_completion_usage_aggregation(client):
|
||||
"""prompt_tokens=[5, 10] is aggregated correctly into usage."""
|
||||
gr1 = await _render_completion(client, "Hello")
|
||||
gr2 = await _render_completion(client, "World")
|
||||
|
||||
ids1 = gr1["token_ids"][:3]
|
||||
ids2 = gr2["token_ids"][:4]
|
||||
|
||||
response = await client.post(
|
||||
"/v1/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_responses": [
|
||||
_make_completion_generate_response(ids1, gr1["request_id"]),
|
||||
_make_completion_generate_response(ids2, gr2["request_id"]),
|
||||
],
|
||||
"prompt_tokens": [5, 10],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
usage = response.json()["usage"]
|
||||
assert usage["prompt_tokens"] == 15
|
||||
assert usage["completion_tokens"] == len(ids1) + len(ids2)
|
||||
assert usage["total_tokens"] == 15 + len(ids1) + len(ids2)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_completion_prompt_tokens_length_mismatch(client):
|
||||
"""len(prompt_tokens) != len(generate_responses) returns 400."""
|
||||
gr1 = await _render_completion(client, "Hello")
|
||||
ids1 = gr1["token_ids"][:3]
|
||||
|
||||
response = await client.post(
|
||||
"/v1/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_responses": [
|
||||
_make_completion_generate_response(ids1, gr1["request_id"]),
|
||||
],
|
||||
"prompt_tokens": [5, 10],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_completion_empty_generate_responses(client):
|
||||
"""Empty generate_responses list returns 400."""
|
||||
response = await client.post(
|
||||
"/v1/completions/derender",
|
||||
json={"model": MODEL_NAME, "generate_responses": []},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_completion_logprobs(client):
|
||||
"""token_id:N placeholders in logprobs are resolved; CompletionLogProbs
|
||||
flat-list structure is returned with non-empty tokens and text_offsets."""
|
||||
gr1 = await _render_completion(client, "Hello world")
|
||||
ids1 = gr1["token_ids"][:3]
|
||||
token_id = ids1[0]
|
||||
|
||||
response = await client.post(
|
||||
"/v1/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_responses": [
|
||||
_make_completion_generate_response(
|
||||
ids1,
|
||||
gr1["request_id"],
|
||||
logprobs=_make_logprobs_with_placeholders(token_id),
|
||||
),
|
||||
],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
logprobs = response.json()["choices"][0]["logprobs"]
|
||||
assert logprobs is not None
|
||||
tokens = logprobs["tokens"]
|
||||
assert len(tokens) == 1
|
||||
assert not tokens[0].startswith("token_id:"), (
|
||||
f"Placeholder was not resolved: {tokens[0]!r}"
|
||||
)
|
||||
assert len(logprobs["token_logprobs"]) == 1
|
||||
assert isinstance(logprobs["token_logprobs"][0], float)
|
||||
assert len(logprobs["text_offset"]) == 1
|
||||
assert logprobs["text_offset"][0] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_completion_kv_transfer_params_passthrough(client):
|
||||
"""kv_transfer_params passes through to CompletionResponse."""
|
||||
gr1 = await _render_completion(client, "Hello")
|
||||
ids1 = gr1["token_ids"][:3]
|
||||
kv = {"node": "abc"}
|
||||
|
||||
response = await client.post(
|
||||
"/v1/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_responses": [
|
||||
_make_completion_generate_response(
|
||||
ids1, gr1["request_id"], kv_transfer_params=kv
|
||||
),
|
||||
],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["kv_transfer_params"] == kv
|
||||
@@ -46,6 +46,7 @@ from vllm.entrypoints.openai.engine.serving import (
|
||||
GenerationError,
|
||||
OpenAIServing,
|
||||
clamp_prompt_logprobs,
|
||||
format_token_id_placeholder,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
|
||||
@@ -1129,7 +1130,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
step_top_logprobs = top_logprobs[i]
|
||||
if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
|
||||
if should_return_as_token_id:
|
||||
token = f"token_id:{token_id}"
|
||||
token = format_token_id_placeholder(token_id)
|
||||
else:
|
||||
if tokenizer is None:
|
||||
raise ValueError(
|
||||
|
||||
@@ -31,6 +31,7 @@ from vllm.entrypoints.openai.engine.serving import (
|
||||
GenerationError,
|
||||
OpenAIServing,
|
||||
clamp_prompt_logprobs,
|
||||
format_token_id_placeholder,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
|
||||
@@ -628,7 +629,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
step_top_logprobs = top_logprobs[i]
|
||||
if step_top_logprobs is None:
|
||||
if should_return_as_token_id:
|
||||
token = f"token_id:{token_id}"
|
||||
token = format_token_id_placeholder(token_id)
|
||||
else:
|
||||
if tokenizer is None:
|
||||
raise VLLMValidationError(
|
||||
|
||||
@@ -452,7 +452,7 @@ class OpenAIServing(BeamSearchOnlineMixin):
|
||||
return_as_token_id: bool = False,
|
||||
) -> str:
|
||||
if return_as_token_id:
|
||||
return f"token_id:{token_id}"
|
||||
return format_token_id_placeholder(token_id)
|
||||
|
||||
if logprob.decoded_token is not None:
|
||||
return logprob.decoded_token
|
||||
@@ -472,6 +472,38 @@ class OpenAIServing(BeamSearchOnlineMixin):
|
||||
return self.models.is_base_model(model_name)
|
||||
|
||||
|
||||
def format_token_id_placeholder(token_id: int) -> str:
|
||||
return f"token_id:{token_id}"
|
||||
|
||||
|
||||
def resolve_token_id_placeholder(
|
||||
token: str, tokenizer: TokenizerLike
|
||||
) -> tuple[str, list[int] | None]:
|
||||
"""Decode a 'token_id:N' placeholder back to a token string and UTF-8 bytes.
|
||||
|
||||
Returns (token, None) unchanged if token is not a placeholder.
|
||||
This is the inverse of format_token_id_placeholder / _get_decoded_token
|
||||
when return_as_token_id=True.
|
||||
"""
|
||||
suffix = token.removeprefix("token_id:")
|
||||
if suffix == token:
|
||||
return token, None
|
||||
try:
|
||||
token_id = int(suffix)
|
||||
except ValueError:
|
||||
return token, None
|
||||
token_repr = tokenizer.convert_ids_to_tokens([token_id])[0]
|
||||
if token_repr is None:
|
||||
logger.warning_once(
|
||||
"resolve_token_id_placeholder: token_id %d has no vocab entry; "
|
||||
"substituting empty string",
|
||||
token_id,
|
||||
)
|
||||
return "", None
|
||||
token_str = tokenizer.convert_tokens_to_string([token_repr])
|
||||
return token_str, list(token_str.encode("utf-8", errors="replace"))
|
||||
|
||||
|
||||
def clamp_prompt_logprobs(
|
||||
prompt_logprobs: PromptLogprobs | None,
|
||||
) -> PromptLogprobs | None:
|
||||
|
||||
@@ -11,7 +11,11 @@ from pydantic import (
|
||||
)
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionLogProbs
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionLogProbs,
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.engine.protocol import StreamOptions, UsageInfo
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.renderers import TokenizeParams
|
||||
@@ -209,3 +213,66 @@ class GenerateResponse(BaseModel):
|
||||
default=None,
|
||||
description="KVTransfer parameters used for disaggregated serving.",
|
||||
)
|
||||
|
||||
|
||||
####### Derender (postprocessing) #######
|
||||
|
||||
|
||||
class DerenderChatRequest(BaseModel):
|
||||
"""Request for the /v1/chat/completions/derender endpoint.
|
||||
|
||||
Wraps a GenerateResponse and caller-supplied metadata needed to produce
|
||||
a fully-formed ChatCompletionResponse without a GPU.
|
||||
"""
|
||||
|
||||
model: str
|
||||
generate_response: GenerateResponse
|
||||
prompt_tokens: int | None = None
|
||||
"""Prompt token count for usage; defaults to 0 if omitted.
|
||||
|
||||
GenerateResponse carries only output tokens; the caller already has
|
||||
len(GenerateRequest.token_ids) from the render step.
|
||||
"""
|
||||
|
||||
chat_request: ChatCompletionRequest | None = None
|
||||
"""The original (post-adjust_request) ChatCompletionRequest from /render.
|
||||
|
||||
Required by the parsing so that tool/reasoning parsers can receive the full
|
||||
request context they expect (request.tools, request.tool_choice,
|
||||
request._grammar_from_tool_parser, etc.).
|
||||
"""
|
||||
|
||||
|
||||
class DerenderCompletionRequest(BaseModel):
|
||||
"""Request for the /v1/completions/derender endpoint.
|
||||
|
||||
Parallel to DerenderChatRequest but handles the multi-prompt completions
|
||||
case: one GenerateResponse per prompt, mirroring the list[GenerateRequest]
|
||||
returned by /v1/completions/render.
|
||||
"""
|
||||
|
||||
model: str
|
||||
generate_responses: list[GenerateResponse]
|
||||
prompt_tokens: list[int] | None = None
|
||||
"""One prompt token count per response; each defaults to 0 if omitted.
|
||||
|
||||
If provided, len(prompt_tokens) must equal len(generate_responses).
|
||||
"""
|
||||
|
||||
completion_request: CompletionRequest | None = None
|
||||
"""The original (post-adjust_request) CompletionRequest from /render.
|
||||
|
||||
Mirrors chat_request on DerenderChatRequest. Required by the parsing
|
||||
so parsers receive the full request context.
|
||||
"""
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_prompt_tokens_length(self) -> "DerenderCompletionRequest":
|
||||
if self.prompt_tokens is not None and len(self.prompt_tokens) != len(
|
||||
self.generate_responses
|
||||
):
|
||||
raise ValueError(
|
||||
f"prompt_tokens length ({len(self.prompt_tokens)}) must equal "
|
||||
f"generate_responses length ({len(self.generate_responses)})"
|
||||
)
|
||||
return self
|
||||
|
||||
@@ -5,10 +5,20 @@ from http import HTTPStatus
|
||||
from fastapi import APIRouter, Depends, FastAPI, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import (
|
||||
CompletionRequest,
|
||||
CompletionResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.disagg.protocol import GenerateRequest
|
||||
from vllm.entrypoints.serve.disagg.protocol import (
|
||||
DerenderChatRequest,
|
||||
DerenderCompletionRequest,
|
||||
GenerateRequest,
|
||||
)
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||
from vllm.logger import init_logger
|
||||
@@ -71,5 +81,53 @@ async def render_completion(request: CompletionRequest, raw_request: Request):
|
||||
return JSONResponse(content=[item.model_dump() for item in result])
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/chat/completions/derender",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=ChatCompletionResponse,
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def derender_chat_completion(request: DerenderChatRequest, raw_request: Request):
|
||||
handler = render(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError(
|
||||
"The model does not support Chat Completions Derender API"
|
||||
)
|
||||
|
||||
result = await handler.derender_chat_response(request)
|
||||
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/completions/derender",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=CompletionResponse,
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def derender_completion(request: DerenderCompletionRequest, raw_request: Request):
|
||||
handler = render(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Completions Derender API")
|
||||
|
||||
result = await handler.derender_completion_response(request)
|
||||
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
|
||||
def attach_router(app: FastAPI) -> None:
|
||||
app.include_router(router)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import time
|
||||
from collections.abc import Sequence
|
||||
from http import HTTPStatus
|
||||
from typing import Any, cast
|
||||
@@ -11,11 +12,24 @@ from vllm.entrypoints.chat_utils import (
|
||||
ChatTemplateContentFormatOption,
|
||||
ConversationMessage,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionLogProbs,
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponse,
|
||||
ChatCompletionResponseChoice,
|
||||
ChatMessage,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import (
|
||||
CompletionLogProbs,
|
||||
CompletionRequest,
|
||||
CompletionResponse,
|
||||
CompletionResponseChoice,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
UsageInfo,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.serving import resolve_token_id_placeholder
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
|
||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
build_harmony_preamble,
|
||||
@@ -26,7 +40,10 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.serve.disagg.mm_serde import encode_mm_kwargs_item
|
||||
from vllm.entrypoints.serve.disagg.protocol import (
|
||||
DerenderChatRequest,
|
||||
DerenderCompletionRequest,
|
||||
GenerateRequest,
|
||||
GenerateResponseChoice,
|
||||
MultiModalFeatures,
|
||||
PlaceholderRangeInfo,
|
||||
)
|
||||
@@ -51,6 +68,7 @@ from vllm.renderers.inputs.preprocess import (
|
||||
parse_model_prompt,
|
||||
prompt_to_seq,
|
||||
)
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser
|
||||
from vllm.utils.mistral import mt as _mt
|
||||
@@ -58,6 +76,90 @@ from vllm.utils.mistral import mt as _mt
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _resolve_logprobs(
|
||||
logprobs: ChatCompletionLogProbs, tokenizer: TokenizerLike
|
||||
) -> ChatCompletionLogProbs:
|
||||
"""Resolve all token_id:N placeholders in a ChatCompletionLogProbs object."""
|
||||
if logprobs.content is None:
|
||||
return logprobs
|
||||
resolved_content = []
|
||||
for entry in logprobs.content:
|
||||
token_str, token_bytes = resolve_token_id_placeholder(entry.token, tokenizer)
|
||||
resolved_top = []
|
||||
for top in entry.top_logprobs:
|
||||
top_str, top_bytes = resolve_token_id_placeholder(top.token, tokenizer)
|
||||
resolved_top.append(
|
||||
top.model_copy(update={"token": top_str, "bytes": top_bytes})
|
||||
)
|
||||
resolved_content.append(
|
||||
entry.model_copy(
|
||||
update={
|
||||
"token": token_str,
|
||||
"bytes": token_bytes,
|
||||
"top_logprobs": resolved_top,
|
||||
}
|
||||
)
|
||||
)
|
||||
return ChatCompletionLogProbs(content=resolved_content)
|
||||
|
||||
|
||||
def _convert_chat_logprobs_to_completion_logprobs(
|
||||
logprobs: ChatCompletionLogProbs,
|
||||
) -> CompletionLogProbs:
|
||||
"""Convert ChatCompletionLogProbs (per-token objects) to CompletionLogProbs
|
||||
(parallel flat lists) as required by the /v1/completions response schema."""
|
||||
if logprobs.content is None:
|
||||
return CompletionLogProbs()
|
||||
|
||||
tokens: list[str] = []
|
||||
token_logprobs: list[float | None] = []
|
||||
top_logprobs_list: list[dict[str, float] | None] = []
|
||||
text_offset: list[int] = []
|
||||
|
||||
offset = 0
|
||||
for entry in logprobs.content:
|
||||
text_offset.append(offset)
|
||||
tokens.append(entry.token)
|
||||
token_logprobs.append(entry.logprob)
|
||||
top_logprobs_list.append(
|
||||
{t.token: t.logprob for t in entry.top_logprobs}
|
||||
if entry.top_logprobs
|
||||
else None
|
||||
)
|
||||
offset += len(entry.token)
|
||||
|
||||
return CompletionLogProbs(
|
||||
text_offset=text_offset,
|
||||
token_logprobs=token_logprobs,
|
||||
tokens=tokens,
|
||||
top_logprobs=top_logprobs_list,
|
||||
)
|
||||
|
||||
|
||||
def _build_chat_choice(
|
||||
choice: GenerateResponseChoice, tokenizer: TokenizerLike
|
||||
) -> ChatCompletionResponseChoice:
|
||||
"""Detokenize and resolve logprobs for a single GenerateResponseChoice.
|
||||
|
||||
Raises:
|
||||
ValueError: if choice.token_ids is empty or None.
|
||||
"""
|
||||
if not choice.token_ids:
|
||||
raise ValueError(f"choice {choice.index} has empty or null token_ids")
|
||||
decoded_text = tokenizer.decode(choice.token_ids, skip_special_tokens=True)
|
||||
resolved_logprobs = (
|
||||
_resolve_logprobs(choice.logprobs, tokenizer)
|
||||
if choice.logprobs is not None
|
||||
else None
|
||||
)
|
||||
return ChatCompletionResponseChoice(
|
||||
index=choice.index,
|
||||
message=ChatMessage(role="assistant", content=decoded_text),
|
||||
logprobs=resolved_logprobs,
|
||||
finish_reason=choice.finish_reason,
|
||||
)
|
||||
|
||||
|
||||
class OpenAIServingRender:
|
||||
def __init__(
|
||||
self,
|
||||
@@ -427,6 +529,146 @@ class OpenAIServingRender:
|
||||
|
||||
return messages, [engine_input]
|
||||
|
||||
async def derender_chat_response(
|
||||
self,
|
||||
request: DerenderChatRequest,
|
||||
) -> ChatCompletionResponse | ErrorResponse:
|
||||
"""Postprocess a GenerateResponse into a ChatCompletionResponse.
|
||||
|
||||
This is the symmetric inverse of render_chat_request: it detokenizes
|
||||
output token IDs, resolves token_id:N logprob placeholders, and
|
||||
formats the result as an OpenAI-compatible chat completion response.
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
tokenizer = self.renderer.get_tokenizer()
|
||||
gen = request.generate_response
|
||||
choices: list[ChatCompletionResponseChoice] = []
|
||||
|
||||
try:
|
||||
for choice in gen.choices:
|
||||
choices.append(_build_chat_choice(choice, tokenizer))
|
||||
except ValueError as exc:
|
||||
return self.create_error_response(str(exc))
|
||||
|
||||
prompt_tokens = (
|
||||
request.prompt_tokens if request.prompt_tokens is not None else 0
|
||||
)
|
||||
completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids)
|
||||
usage = UsageInfo(
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
total_tokens=prompt_tokens + completion_tokens,
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"derender_chat request_id=%s model=%s choices=%d completion_tokens=%d",
|
||||
gen.request_id,
|
||||
request.model,
|
||||
len(choices),
|
||||
completion_tokens,
|
||||
)
|
||||
return ChatCompletionResponse(
|
||||
id=gen.request_id,
|
||||
model=request.model,
|
||||
created=int(time.time()),
|
||||
choices=choices,
|
||||
usage=usage,
|
||||
prompt_logprobs=gen.prompt_logprobs,
|
||||
kv_transfer_params=gen.kv_transfer_params,
|
||||
)
|
||||
|
||||
async def derender_completion_response(
|
||||
self,
|
||||
request: DerenderCompletionRequest,
|
||||
) -> CompletionResponse | ErrorResponse:
|
||||
"""Postprocess a list of GenerateResponses into a CompletionResponse.
|
||||
|
||||
Mirrors the multi-prompt completions case: one GenerateResponse per
|
||||
prompt, parallel to the list[GenerateRequest] from /v1/completions/render.
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
n = len(request.generate_responses)
|
||||
prompt_tokens_list: list[int] = (
|
||||
request.prompt_tokens if request.prompt_tokens is not None else [0] * n
|
||||
)
|
||||
|
||||
tokenizer = self.renderer.get_tokenizer()
|
||||
choices: list[CompletionResponseChoice] = []
|
||||
total_prompt_tokens = 0
|
||||
total_completion_tokens = 0
|
||||
index = 0
|
||||
|
||||
for gen, pt in zip(request.generate_responses, prompt_tokens_list):
|
||||
for choice in gen.choices:
|
||||
if not choice.token_ids:
|
||||
return self.create_error_response(
|
||||
f"choice {choice.index} in response {gen.request_id} "
|
||||
"has empty or null token_ids"
|
||||
)
|
||||
decoded_text = tokenizer.decode(
|
||||
choice.token_ids, skip_special_tokens=True
|
||||
)
|
||||
completion_logprobs = None
|
||||
if choice.logprobs is not None:
|
||||
resolved = _resolve_logprobs(choice.logprobs, tokenizer)
|
||||
completion_logprobs = _convert_chat_logprobs_to_completion_logprobs(
|
||||
resolved
|
||||
)
|
||||
choices.append(
|
||||
CompletionResponseChoice(
|
||||
index=index,
|
||||
text=decoded_text,
|
||||
finish_reason=choice.finish_reason,
|
||||
logprobs=completion_logprobs,
|
||||
)
|
||||
)
|
||||
total_completion_tokens += len(choice.token_ids)
|
||||
index += 1
|
||||
total_prompt_tokens += pt
|
||||
|
||||
if not request.generate_responses:
|
||||
return self.create_error_response("generate_responses must not be empty")
|
||||
|
||||
first = request.generate_responses[0]
|
||||
kv_params = first.kv_transfer_params
|
||||
if any(
|
||||
r.kv_transfer_params != kv_params for r in request.generate_responses[1:]
|
||||
):
|
||||
logger.warning(
|
||||
"derender_completion: kv_transfer_params differ across responses; "
|
||||
"setting to None on the aggregated response"
|
||||
)
|
||||
kv_params = None
|
||||
|
||||
usage = UsageInfo(
|
||||
prompt_tokens=total_prompt_tokens,
|
||||
completion_tokens=total_completion_tokens,
|
||||
total_tokens=total_prompt_tokens + total_completion_tokens,
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"derender_completion request_id=%s model=%s choices=%d"
|
||||
" completion_tokens=%d",
|
||||
first.request_id,
|
||||
request.model,
|
||||
len(choices),
|
||||
total_completion_tokens,
|
||||
)
|
||||
return CompletionResponse(
|
||||
id=first.request_id,
|
||||
model=request.model,
|
||||
created=int(time.time()),
|
||||
choices=choices,
|
||||
usage=usage,
|
||||
kv_transfer_params=kv_params,
|
||||
)
|
||||
|
||||
def create_error_response(
|
||||
self,
|
||||
message: str | Exception,
|
||||
|
||||
Reference in New Issue
Block a user