[Bugfix][Frontend] Honor request_id and X-Request-Id in chat and completions render (#61236)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
@@ -367,6 +367,74 @@ async def test_completion_render_multiple_prompts(client):
|
|||||||
assert prompt["request_id"].startswith("cmpl-")
|
assert prompt["request_id"].startswith("cmpl-")
|
||||||
|
|
||||||
|
|
||||||
|
CHAT_BODY = {
|
||||||
|
"model": MODEL_NAME,
|
||||||
|
"messages": [{"role": "user", "content": "Hello"}],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_chat_render_honors_body_request_id(client):
|
||||||
|
response = await client.post(
|
||||||
|
"/v1/chat/completions/render", json={**CHAT_BODY, "request_id": "abc"}
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert response.json()["request_id"] == "chatcmpl-abc"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_chat_render_header_request_id_wins(client):
|
||||||
|
response = await client.post(
|
||||||
|
"/v1/chat/completions/render",
|
||||||
|
json={**CHAT_BODY, "request_id": "abc"},
|
||||||
|
headers={"X-Request-Id": "hdr"},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert response.json()["request_id"] == "chatcmpl-hdr"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_chat_render_generates_unique_request_ids(client):
|
||||||
|
ids = set()
|
||||||
|
for _ in range(2):
|
||||||
|
response = await client.post("/v1/chat/completions/render", json=CHAT_BODY)
|
||||||
|
assert response.status_code == 200
|
||||||
|
request_id = response.json()["request_id"]
|
||||||
|
assert request_id.startswith("chatcmpl-")
|
||||||
|
ids.add(request_id)
|
||||||
|
|
||||||
|
assert len(ids) == 2
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_completion_render_request_id_single_prompt(client):
|
||||||
|
response = await client.post(
|
||||||
|
"/v1/completions/render",
|
||||||
|
json={"model": MODEL_NAME, "prompt": "Hello world", "request_id": "abc"},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert [r["request_id"] for r in response.json()] == ["cmpl-abc"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_completion_render_request_id_multiple_prompts(client):
|
||||||
|
response = await client.post(
|
||||||
|
"/v1/completions/render",
|
||||||
|
json={
|
||||||
|
"model": MODEL_NAME,
|
||||||
|
"prompt": ["Hello world", "Goodbye world"],
|
||||||
|
"request_id": "abc",
|
||||||
|
},
|
||||||
|
headers={"X-Request-Id": "hdr"},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert [r["request_id"] for r in response.json()] == ["cmpl-hdr-0", "cmpl-hdr-1"]
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_chat_completion_render_multi_turn(client):
|
async def test_chat_completion_render_multi_turn(client):
|
||||||
"""Test chat completion render with multi-turn conversation."""
|
"""Test chat completion render with multi-turn conversation."""
|
||||||
|
|||||||
@@ -43,7 +43,7 @@ async def render_chat_completion(request: ChatCompletionRequest, raw_request: Re
|
|||||||
"The model does not support Chat Completions Render API"
|
"The model does not support Chat Completions Render API"
|
||||||
)
|
)
|
||||||
|
|
||||||
result = await handler.render_chat_request(request)
|
result = await handler.render_chat_request(request, raw_request)
|
||||||
|
|
||||||
if isinstance(result, ErrorResponse):
|
if isinstance(result, ErrorResponse):
|
||||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||||
@@ -67,7 +67,7 @@ async def render_messages(request: AnthropicMessagesRequest, raw_request: Reques
|
|||||||
if handler is None:
|
if handler is None:
|
||||||
raise NotImplementedError("The model does not support Messages Render API")
|
raise NotImplementedError("The model does not support Messages Render API")
|
||||||
|
|
||||||
result = await handler.render_messages_request(request)
|
result = await handler.render_messages_request(request, raw_request)
|
||||||
|
|
||||||
if isinstance(result, ErrorResponse):
|
if isinstance(result, ErrorResponse):
|
||||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||||
@@ -90,7 +90,7 @@ async def render_completion(request: CompletionRequest, raw_request: Request):
|
|||||||
if handler is None:
|
if handler is None:
|
||||||
raise NotImplementedError("The model does not support Completions Render API")
|
raise NotImplementedError("The model does not support Completions Render API")
|
||||||
|
|
||||||
result = await handler.render_completion_request(request)
|
result = await handler.render_completion_request(request, raw_request)
|
||||||
|
|
||||||
if isinstance(result, ErrorResponse):
|
if isinstance(result, ErrorResponse):
|
||||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||||
|
|||||||
@@ -2,6 +2,8 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
from fastapi import Request
|
||||||
|
|
||||||
from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
|
from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
|
||||||
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
|
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
@@ -31,7 +33,6 @@ from vllm.renderers.inputs.preprocess import (
|
|||||||
extract_prompt_len,
|
extract_prompt_len,
|
||||||
)
|
)
|
||||||
from vllm.renderers.online_renderer import OnlineRenderer
|
from vllm.renderers.online_renderer import OnlineRenderer
|
||||||
from vllm.utils import random_uuid
|
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
@@ -77,6 +78,7 @@ class ServingRender(BaseServing):
|
|||||||
async def render_chat_request(
|
async def render_chat_request(
|
||||||
self,
|
self,
|
||||||
request: ChatCompletionRequest,
|
request: ChatCompletionRequest,
|
||||||
|
raw_request: Request | None = None,
|
||||||
) -> GenerateRequest | ErrorResponse:
|
) -> GenerateRequest | ErrorResponse:
|
||||||
"""Validate the model and preprocess a chat completion request.
|
"""Validate the model and preprocess a chat completion request.
|
||||||
|
|
||||||
@@ -139,7 +141,9 @@ class ServingRender(BaseServing):
|
|||||||
)
|
)
|
||||||
reasoning_ended = request.resolve_reasoning_ended(parser, token_ids)
|
reasoning_ended = request.resolve_reasoning_ended(parser, token_ids)
|
||||||
|
|
||||||
request_id = f"chatcmpl-{random_uuid()}"
|
request_id = (
|
||||||
|
f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
|
||||||
|
)
|
||||||
|
|
||||||
return GenerateRequest(
|
return GenerateRequest(
|
||||||
request_id=request_id,
|
request_id=request_id,
|
||||||
@@ -161,6 +165,7 @@ class ServingRender(BaseServing):
|
|||||||
async def render_messages_request(
|
async def render_messages_request(
|
||||||
self,
|
self,
|
||||||
request: AnthropicMessagesRequest,
|
request: AnthropicMessagesRequest,
|
||||||
|
raw_request: Request | None = None,
|
||||||
) -> GenerateRequest | ErrorResponse:
|
) -> GenerateRequest | ErrorResponse:
|
||||||
"""Validate the model and preprocess an Anthropic Messages request.
|
"""Validate the model and preprocess an Anthropic Messages request.
|
||||||
|
|
||||||
@@ -172,11 +177,12 @@ class ServingRender(BaseServing):
|
|||||||
request, merge_inline_system=self._merge_inline_system
|
request, merge_inline_system=self._merge_inline_system
|
||||||
)
|
)
|
||||||
chat_req.return_mm_kwargs = request.return_mm_kwargs
|
chat_req.return_mm_kwargs = request.return_mm_kwargs
|
||||||
return await self.render_chat_request(chat_req)
|
return await self.render_chat_request(chat_req, raw_request)
|
||||||
|
|
||||||
async def render_completion_request(
|
async def render_completion_request(
|
||||||
self,
|
self,
|
||||||
request: CompletionRequest,
|
request: CompletionRequest,
|
||||||
|
raw_request: Request | None = None,
|
||||||
) -> list[GenerateRequest] | ErrorResponse:
|
) -> list[GenerateRequest] | ErrorResponse:
|
||||||
"""Validate the model and preprocess a completion request.
|
"""Validate the model and preprocess a completion request.
|
||||||
|
|
||||||
@@ -197,8 +203,9 @@ class ServingRender(BaseServing):
|
|||||||
)
|
)
|
||||||
if isinstance(result, ErrorResponse):
|
if isinstance(result, ErrorResponse):
|
||||||
return result
|
return result
|
||||||
|
base_id = f"cmpl-{self._base_request_id(raw_request, request.request_id)}"
|
||||||
generate_requests: list[GenerateRequest] = []
|
generate_requests: list[GenerateRequest] = []
|
||||||
for engine_input in result:
|
for i, engine_input in enumerate(result):
|
||||||
prompt_components = extract_prompt_components(
|
prompt_components = extract_prompt_components(
|
||||||
self.model_config, engine_input
|
self.model_config, engine_input
|
||||||
)
|
)
|
||||||
@@ -220,7 +227,7 @@ class ServingRender(BaseServing):
|
|||||||
max_tokens, self.default_sampling_params
|
max_tokens, self.default_sampling_params
|
||||||
)
|
)
|
||||||
|
|
||||||
request_id = f"cmpl-{random_uuid()}"
|
request_id = base_id if len(result) == 1 else f"{base_id}-{i}"
|
||||||
|
|
||||||
generate_requests.append(
|
generate_requests.append(
|
||||||
GenerateRequest(
|
GenerateRequest(
|
||||||
|
|||||||
Reference in New Issue
Block a user