[Bugfix][Frontend] Sample batched chat completions from the adjusted requests (#58929)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
@@ -12,6 +12,7 @@ from vllm.entrypoints.chat_utils import ConversationMessage
|
|||||||
from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata
|
from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
BatchChatCompletionRequest,
|
BatchChatCompletionRequest,
|
||||||
|
ChatCompletionRequest,
|
||||||
ChatCompletionResponse,
|
ChatCompletionResponse,
|
||||||
ChatCompletionResponseChoice,
|
ChatCompletionResponseChoice,
|
||||||
ChatMessage,
|
ChatMessage,
|
||||||
@@ -40,7 +41,14 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
|||||||
async def render_batch_chat_request(
|
async def render_batch_chat_request(
|
||||||
self,
|
self,
|
||||||
request: BatchChatCompletionRequest,
|
request: BatchChatCompletionRequest,
|
||||||
) -> tuple[list[list[ConversationMessage]], list[EngineInput]] | ErrorResponse:
|
) -> (
|
||||||
|
tuple[
|
||||||
|
list[ChatCompletionRequest],
|
||||||
|
list[list[ConversationMessage]],
|
||||||
|
list[EngineInput],
|
||||||
|
]
|
||||||
|
| ErrorResponse
|
||||||
|
):
|
||||||
"""Validate the model and preprocess a batched chat completion request.
|
"""Validate the model and preprocess a batched chat completion request.
|
||||||
|
|
||||||
Performs engine-aware checks then delegates per-conversation
|
Performs engine-aware checks then delegates per-conversation
|
||||||
@@ -48,8 +56,10 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
|||||||
once for the whole batch.
|
once for the whole batch.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
A tuple of (all_conversations, engine_prompts) on success — one
|
A tuple of (single_requests, all_conversations, engine_prompts) on
|
||||||
entry per conversation — or an ErrorResponse on failure.
|
success — one entry per conversation — or an ErrorResponse on
|
||||||
|
failure. The renderer adjusts ``single_requests`` in place, so
|
||||||
|
sampling must read them rather than rebuilding from ``request``.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
error_check_ret = await self._check_model(request)
|
error_check_ret = await self._check_model(request)
|
||||||
@@ -74,6 +84,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
|||||||
parser = renderer.parser
|
parser = renderer.parser
|
||||||
tool_dicts: list[dict] | None = None
|
tool_dicts: list[dict] | None = None
|
||||||
|
|
||||||
|
single_requests: list[ChatCompletionRequest] = []
|
||||||
all_conversations: list[list[ConversationMessage]] = []
|
all_conversations: list[list[ConversationMessage]] = []
|
||||||
all_engine_prompts: list[EngineInput] = []
|
all_engine_prompts: list[EngineInput] = []
|
||||||
|
|
||||||
@@ -93,10 +104,11 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
|||||||
tool_dicts=tool_dicts,
|
tool_dicts=tool_dicts,
|
||||||
parser=parser,
|
parser=parser,
|
||||||
)
|
)
|
||||||
|
single_requests.append(single_request)
|
||||||
all_conversations.append(conversation)
|
all_conversations.append(conversation)
|
||||||
all_engine_prompts.append(engine_prompts[0])
|
all_engine_prompts.append(engine_prompts[0])
|
||||||
|
|
||||||
return all_conversations, all_engine_prompts
|
return single_requests, all_conversations, all_engine_prompts
|
||||||
|
|
||||||
async def create_batch_chat_completion(
|
async def create_batch_chat_completion(
|
||||||
self,
|
self,
|
||||||
@@ -111,10 +123,10 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
|||||||
"""
|
"""
|
||||||
tokenizer = self.renderer.tokenizer
|
tokenizer = self.renderer.tokenizer
|
||||||
assert tokenizer is not None
|
assert tokenizer is not None
|
||||||
single_requests = [
|
render_result = await self.render_batch_chat_request(request)
|
||||||
request.to_chat_completion_request(messages)
|
if isinstance(render_result, ErrorResponse):
|
||||||
for messages in request.messages
|
return render_result
|
||||||
]
|
single_requests, all_conversations, engine_prompts = render_result
|
||||||
|
|
||||||
parser: Parser | None = None
|
parser: Parser | None = None
|
||||||
if self.parser_cls is not None:
|
if self.parser_cls is not None:
|
||||||
@@ -127,11 +139,6 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
|||||||
chat_template_kwargs=chat_template_kwargs,
|
chat_template_kwargs=chat_template_kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
render_result = await self.render_batch_chat_request(request)
|
|
||||||
if isinstance(render_result, ErrorResponse):
|
|
||||||
return render_result
|
|
||||||
all_conversations, engine_prompts = render_result
|
|
||||||
|
|
||||||
request_id = (
|
request_id = (
|
||||||
f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
|
f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user