[Refactor] Deprecate ResponsesParser wrapper, inline parsing into ParsableContext (#45431)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
@@ -159,7 +159,7 @@ class HarmonyPythonTool(Tool):
|
||||
|
||||
assert isinstance(context, ParsableContext)
|
||||
|
||||
last_msg = context.parser.response_messages[-1]
|
||||
last_msg = context.response_messages[-1]
|
||||
args = json.loads(last_msg.arguments)
|
||||
|
||||
last_msg_harmony = Message(
|
||||
|
||||
@@ -1,180 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from openai.types.responses import ResponseFunctionToolCall, ResponseOutputItem
|
||||
from openai.types.responses.response_function_tool_call_output_item import (
|
||||
ResponseFunctionToolCallOutputItem,
|
||||
)
|
||||
from openai.types.responses.response_output_item import McpCall
|
||||
from openai.types.responses.response_output_message import ResponseOutputMessage
|
||||
from openai.types.responses.response_output_text import ResponseOutputText
|
||||
|
||||
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
|
||||
from vllm.entrypoints.openai.responses.protocol import (
|
||||
ResponseInputOutputItem,
|
||||
ResponsesRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.utils import build_response_output_items
|
||||
from vllm.entrypoints.serve.utils.constants import MCP_PREFIX
|
||||
from vllm.outputs import CompletionOutput
|
||||
from vllm.parser.abstract_parser import Parser
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ResponsesParser:
|
||||
"""Incremental parser over completion tokens with reasoning support."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
tokenizer: TokenizerLike,
|
||||
parser_cls: type[Parser] | None,
|
||||
response_messages: list[ResponseInputOutputItem],
|
||||
request: ResponsesRequest,
|
||||
chat_template: str | None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
enable_auto_tools: bool = False,
|
||||
tool_call_id_type: str = "random",
|
||||
):
|
||||
self.response_messages: list[ResponseInputOutputItem] = (
|
||||
# TODO: initial messages may not be properly typed
|
||||
response_messages
|
||||
)
|
||||
self.num_init_messages = len(response_messages)
|
||||
self.tokenizer = tokenizer
|
||||
self.request = request
|
||||
|
||||
self.parser_instance: Parser | None = None
|
||||
if parser_cls is not None:
|
||||
chat_template_kwargs = _effective_chat_template_kwargs(
|
||||
request,
|
||||
chat_template=chat_template,
|
||||
chat_template_content_format=chat_template_content_format,
|
||||
)
|
||||
|
||||
self.parser_instance = parser_cls(
|
||||
tokenizer,
|
||||
tools=request.tools,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
|
||||
self.enable_auto_tools = enable_auto_tools
|
||||
self.tool_call_id_type = tool_call_id_type
|
||||
|
||||
# Store the last finish_reason to determine response status
|
||||
self.finish_reason: str | None = None
|
||||
|
||||
def process(self, output: CompletionOutput) -> "ResponsesParser":
|
||||
# Store the finish_reason from the output
|
||||
self.finish_reason = output.finish_reason
|
||||
|
||||
if self.parser_instance is not None:
|
||||
reasoning, content, tool_calls = self.parser_instance.parse(
|
||||
output.text,
|
||||
self.request,
|
||||
enable_auto_tools=self.enable_auto_tools,
|
||||
)
|
||||
output_items = build_response_output_items(
|
||||
reasoning=reasoning,
|
||||
content=content,
|
||||
tool_calls=tool_calls,
|
||||
tool_call_id_type=self.tool_call_id_type,
|
||||
)
|
||||
self.response_messages.extend(output_items)
|
||||
else:
|
||||
# No parser configured, treat entire output as text content
|
||||
if output.text:
|
||||
self.response_messages.append(
|
||||
ResponseOutputMessage(
|
||||
type="message",
|
||||
id=f"msg_{random_uuid()}",
|
||||
status="completed",
|
||||
role="assistant",
|
||||
content=[
|
||||
ResponseOutputText(
|
||||
annotations=[], # TODO
|
||||
type="output_text",
|
||||
text=output.text,
|
||||
logprobs=None, # TODO
|
||||
)
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
return self
|
||||
|
||||
def make_response_output_items_from_parsable_context(
|
||||
self,
|
||||
) -> list[ResponseOutputItem]:
|
||||
"""Given a list of sentences, construct ResponseOutput Items."""
|
||||
response_messages = self.response_messages[self.num_init_messages :]
|
||||
output_messages: list[ResponseOutputItem] = []
|
||||
for message in response_messages:
|
||||
if not isinstance(message, ResponseFunctionToolCallOutputItem):
|
||||
output_messages.append(message)
|
||||
else:
|
||||
if len(output_messages) == 0:
|
||||
raise ValueError(
|
||||
"Cannot have a FunctionToolCallOutput before FunctionToolCall."
|
||||
)
|
||||
if isinstance(output_messages[-1], ResponseFunctionToolCall):
|
||||
mcp_message = McpCall(
|
||||
id=f"{MCP_PREFIX}{random_uuid()}",
|
||||
arguments=output_messages[-1].arguments,
|
||||
name=output_messages[-1].name,
|
||||
server_label=output_messages[
|
||||
-1
|
||||
].name, # TODO: store the server label
|
||||
type="mcp_call",
|
||||
status="completed",
|
||||
output=message.output,
|
||||
# TODO: support error output
|
||||
)
|
||||
output_messages[-1] = mcp_message
|
||||
|
||||
return output_messages
|
||||
|
||||
|
||||
def get_responses_parser_for_simple_context(
|
||||
*,
|
||||
tokenizer: TokenizerLike,
|
||||
parser_cls: type[Parser] | None,
|
||||
response_messages: list[ResponseInputOutputItem],
|
||||
request: ResponsesRequest,
|
||||
chat_template: str | None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
enable_auto_tools: bool = False,
|
||||
tool_call_id_type: str = "random",
|
||||
) -> ResponsesParser:
|
||||
"""Factory function to create a ResponsesParser with
|
||||
optional unified parser.
|
||||
|
||||
Returns:
|
||||
ResponsesParser instance configured with the provided parser
|
||||
"""
|
||||
return ResponsesParser(
|
||||
tokenizer=tokenizer,
|
||||
parser_cls=parser_cls,
|
||||
response_messages=response_messages,
|
||||
request=request,
|
||||
chat_template=chat_template,
|
||||
chat_template_content_format=chat_template_content_format,
|
||||
enable_auto_tools=enable_auto_tools,
|
||||
tool_call_id_type=tool_call_id_type,
|
||||
)
|
||||
|
||||
|
||||
def _effective_chat_template_kwargs(
|
||||
request: ResponsesRequest,
|
||||
chat_template: str | None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
) -> dict[str, Any]:
|
||||
return request.build_chat_params(
|
||||
default_template=chat_template,
|
||||
default_template_content_format=chat_template_content_format,
|
||||
).chat_template_kwargs
|
||||
@@ -10,9 +10,13 @@ from contextlib import AsyncExitStack
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING, Any, Final, Union
|
||||
|
||||
from openai.types.responses import ResponseFunctionToolCall, ResponseOutputItem
|
||||
from openai.types.responses.response_function_tool_call_output_item import (
|
||||
ResponseFunctionToolCallOutputItem,
|
||||
)
|
||||
from openai.types.responses.response_output_item import McpCall
|
||||
from openai.types.responses.response_output_message import ResponseOutputMessage
|
||||
from openai.types.responses.response_output_text import ResponseOutputText
|
||||
from openai.types.responses.tool import Mcp
|
||||
from openai_harmony import Author, Message, Role, StreamState, TextContent
|
||||
|
||||
@@ -30,15 +34,15 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
get_streamable_parser_for_assistant,
|
||||
render_for_completion,
|
||||
)
|
||||
from vllm.entrypoints.openai.parser.responses_parser import (
|
||||
get_responses_parser_for_simple_context,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import (
|
||||
ResponseInputOutputItem,
|
||||
ResponseRawMessageAndToken,
|
||||
ResponsesRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.utils import construct_tool_dicts
|
||||
from vllm.entrypoints.openai.responses.utils import (
|
||||
build_response_output_items,
|
||||
construct_tool_dicts,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.constants import MCP_PREFIX
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.parser.abstract_parser import Parser
|
||||
@@ -286,16 +290,24 @@ class ParsableContext(ConversationContext):
|
||||
# not implemented yet for ParsableContext
|
||||
self.all_turn_metrics: list[TurnMetrics] = []
|
||||
|
||||
self.parser = get_responses_parser_for_simple_context(
|
||||
tokenizer=tokenizer,
|
||||
parser_cls=parser_cls,
|
||||
response_messages=response_messages,
|
||||
request=request,
|
||||
chat_template=chat_template,
|
||||
chat_template_content_format=chat_template_content_format,
|
||||
enable_auto_tools=enable_auto_tools,
|
||||
tool_call_id_type=tool_call_id_type,
|
||||
)
|
||||
self.response_messages: list[ResponseInputOutputItem] = response_messages
|
||||
self.num_init_messages = len(response_messages)
|
||||
self.finish_reason: str | None = None
|
||||
self.enable_auto_tools = enable_auto_tools
|
||||
self.tool_call_id_type = tool_call_id_type
|
||||
|
||||
self.parser_instance: Parser | None = None
|
||||
if parser_cls is not None:
|
||||
chat_template_kwargs = request.build_chat_params(
|
||||
default_template=chat_template,
|
||||
default_template_content_format=chat_template_content_format,
|
||||
).chat_template_kwargs
|
||||
self.parser_instance = parser_cls(
|
||||
tokenizer,
|
||||
tools=request.tools,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
|
||||
self.parser_cls = parser_cls
|
||||
self.request = request
|
||||
|
||||
@@ -318,11 +330,44 @@ class ParsableContext(ConversationContext):
|
||||
self.num_output_tokens += len(output.outputs[0].token_ids or [])
|
||||
if output.kv_transfer_params is not None:
|
||||
self.kv_transfer_params = output.kv_transfer_params
|
||||
self.parser.process(output.outputs[0])
|
||||
output_token_ids = output.outputs[0].token_ids or []
|
||||
self._accumulated_token_ids.extend(output_token_ids)
|
||||
|
||||
# only store if enable_response_messages is True, save memory
|
||||
completion = output.outputs[0]
|
||||
self.finish_reason = completion.finish_reason
|
||||
|
||||
if self.parser_instance is not None:
|
||||
reasoning, content, tool_calls = self.parser_instance.parse(
|
||||
completion.text,
|
||||
self.request,
|
||||
enable_auto_tools=self.enable_auto_tools,
|
||||
)
|
||||
self.response_messages.extend(
|
||||
build_response_output_items(
|
||||
reasoning=reasoning,
|
||||
content=content,
|
||||
tool_calls=tool_calls,
|
||||
tool_call_id_type=self.tool_call_id_type,
|
||||
)
|
||||
)
|
||||
elif completion.text:
|
||||
self.response_messages.append(
|
||||
ResponseOutputMessage(
|
||||
type="message",
|
||||
id=f"msg_{random_uuid()}",
|
||||
status="completed",
|
||||
role="assistant",
|
||||
content=[
|
||||
ResponseOutputText(
|
||||
annotations=[],
|
||||
type="output_text",
|
||||
text=completion.text,
|
||||
logprobs=None,
|
||||
)
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
self._accumulated_token_ids.extend(completion.token_ids or [])
|
||||
|
||||
if self.request.enable_response_messages:
|
||||
output_prompt = output.prompt or ""
|
||||
output_prompt_token_ids = output.prompt_token_ids or []
|
||||
@@ -342,18 +387,18 @@ class ParsableContext(ConversationContext):
|
||||
)
|
||||
self.output_messages.append(
|
||||
ResponseRawMessageAndToken(
|
||||
message=output.outputs[0].text,
|
||||
tokens=output.outputs[0].token_ids,
|
||||
message=completion.text,
|
||||
tokens=completion.token_ids,
|
||||
)
|
||||
)
|
||||
|
||||
def append_tool_output(self, output: list[ResponseInputOutputItem]) -> None:
|
||||
self.parser.response_messages.extend(output)
|
||||
self.response_messages.extend(output)
|
||||
|
||||
def need_builtin_tool_call(self) -> bool:
|
||||
"""Return true if the last message is a builtin tool call
|
||||
that the request has enabled."""
|
||||
last_message = self.parser.response_messages[-1]
|
||||
last_message = self.response_messages[-1]
|
||||
if last_message.type != "function_call":
|
||||
return False
|
||||
if last_message.name in ("code_interpreter", "python"):
|
||||
@@ -457,12 +502,12 @@ class ParsableContext(ConversationContext):
|
||||
return [message]
|
||||
|
||||
async def call_tool(self) -> list[ResponseInputOutputItem]:
|
||||
if not self.parser.response_messages:
|
||||
if not self.response_messages:
|
||||
return []
|
||||
last_msg = self.parser.response_messages[-1]
|
||||
last_msg = self.response_messages[-1]
|
||||
# change this to a mcp_ function call
|
||||
last_msg.id = f"{MCP_PREFIX}{random_uuid()}"
|
||||
self.parser.response_messages[-1] = last_msg
|
||||
self.response_messages[-1] = last_msg
|
||||
if last_msg.name == "code_interpreter":
|
||||
return await self.call_python_tool(self._tool_sessions["python"], last_msg)
|
||||
elif last_msg.name == "web_search_preview":
|
||||
@@ -473,6 +518,29 @@ class ParsableContext(ConversationContext):
|
||||
)
|
||||
return []
|
||||
|
||||
def make_response_output_items(self) -> list[ResponseOutputItem]:
|
||||
response_messages = self.response_messages[self.num_init_messages :]
|
||||
output_messages: list[ResponseOutputItem] = []
|
||||
for message in response_messages:
|
||||
if not isinstance(message, ResponseFunctionToolCallOutputItem):
|
||||
output_messages.append(message)
|
||||
else:
|
||||
if len(output_messages) == 0:
|
||||
raise ValueError(
|
||||
"Cannot have a FunctionToolCallOutput before FunctionToolCall."
|
||||
)
|
||||
if isinstance(output_messages[-1], ResponseFunctionToolCall):
|
||||
output_messages[-1] = McpCall(
|
||||
id=f"{MCP_PREFIX}{random_uuid()}",
|
||||
arguments=output_messages[-1].arguments,
|
||||
name=output_messages[-1].name,
|
||||
server_label=output_messages[-1].name,
|
||||
type="mcp_call",
|
||||
status="completed",
|
||||
output=message.output,
|
||||
)
|
||||
return output_messages
|
||||
|
||||
def render_for_completion(self):
|
||||
raise NotImplementedError("Should not be called.")
|
||||
|
||||
|
||||
@@ -702,7 +702,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
elif isinstance(context, ParsableContext):
|
||||
(engine_input,) = await self._render_next_turn(
|
||||
context.request,
|
||||
context.parser.response_messages,
|
||||
context.response_messages,
|
||||
context.tool_dicts,
|
||||
context.parser_cls,
|
||||
context.chat_template,
|
||||
@@ -805,7 +805,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
else:
|
||||
status = "incomplete"
|
||||
elif isinstance(context, ParsableContext):
|
||||
output = context.parser.make_response_output_items_from_parsable_context()
|
||||
output = context.make_response_output_items()
|
||||
|
||||
if request.enable_response_messages:
|
||||
input_messages = context.input_messages
|
||||
@@ -816,7 +816,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
num_tool_output_tokens = 0
|
||||
|
||||
# Check finish reason from the parser
|
||||
if context.parser.finish_reason == "length":
|
||||
if context.finish_reason == "length":
|
||||
status = "incomplete"
|
||||
else:
|
||||
assert isinstance(context, SimpleContext)
|
||||
|
||||
Reference in New Issue
Block a user