[1/N] Initial Implementation of Parser for ResponsesAPI (#32712)

Signed-off-by: Andrew Xia <axia@fb.com>
Co-authored-by: Andrew Xia <axia@fb.com>
This commit is contained in:
Andrew Xia
2026-02-04 10:59:03 +08:00
committed by GitHub
co-authored by Andrew Xia
parent 02080179a3
commit e1bf04b6c2
10 changed files with 772 additions and 61 deletions
@@ -71,6 +71,7 @@ from vllm.inputs.data import EmbedsPrompt, TokensPrompt
from vllm.logger import init_logger
from vllm.logprobs import Logprob
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.parser import ParserManager
from vllm.sampling_params import BeamSearchParams, SamplingParams
from vllm.tokenizers import TokenizerLike
from vllm.tokenizers.mistral import (
@@ -131,13 +132,15 @@ class OpenAIServingChat(OpenAIServing):
self.logits_processors = self.model_config.logits_processors
# set up reasoning parser
self.reasoning_parser = self._get_reasoning_parser(
self.reasoning_parser = ParserManager.get_reasoning_parser(
reasoning_parser_name=reasoning_parser
)
# set up tool use
self.enable_auto_tools: bool = enable_auto_tools
self.tool_parser = self._get_tool_parser(
tool_parser_name=tool_parser, enable_auto_tools=enable_auto_tools
self.tool_parser = ParserManager.get_tool_parser(
tool_parser_name=tool_parser,
enable_auto_tools=enable_auto_tools,
model_name=self.model_config.model,
)
self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
+1 -42
View File
@@ -107,11 +107,10 @@ from vllm.lora.request import LoRARequest
from vllm.multimodal import MultiModalDataDict
from vllm.outputs import CompletionOutput, PoolingRequestOutput, RequestOutput
from vllm.pooling_params import PoolingParams
from vllm.reasoning import ReasoningParser, ReasoningParserManager
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
from vllm.sampling_params import BeamSearchParams, SamplingParams
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers import ToolParser, ToolParserManager
from vllm.tool_parsers import ToolParser
from vllm.tracing import (
contains_trace_headers,
extract_trace_headers,
@@ -246,46 +245,6 @@ class OpenAIServing:
self.model_config = self.models.model_config
self.max_model_len = self.model_config.max_model_len
def _get_tool_parser(
self, tool_parser_name: str | None = None, enable_auto_tools: bool = False
) -> Callable[[TokenizerLike], ToolParser] | None:
"""Get the tool parser based on the name."""
parser = None
if not enable_auto_tools or tool_parser_name is None:
return parser
logger.info('"auto" tool choice has been enabled.')
try:
if tool_parser_name == "pythonic" and self.model_config.model.startswith(
"meta-llama/Llama-3.2"
):
logger.warning(
"Llama3.2 models may struggle to emit valid pythonic tool calls"
)
parser = ToolParserManager.get_tool_parser(tool_parser_name)
except Exception as e:
raise TypeError(
"Error: --enable-auto-tool-choice requires "
f"tool_parser:'{tool_parser_name}' which has not "
"been registered"
) from e
return parser
def _get_reasoning_parser(
self,
reasoning_parser_name: str,
) -> Callable[[TokenizerLike], ReasoningParser] | None:
"""Get the reasoning parser based on the name."""
parser = None
if not reasoning_parser_name:
return None
try:
parser = ReasoningParserManager.get_reasoning_parser(reasoning_parser_name)
assert parser is not None
except Exception as e:
raise TypeError(f"{reasoning_parser_name=} has not been registered") from e
return parser
async def beam_search(
self,
prompt: PromptType,
+22 -16
View File
@@ -123,6 +123,7 @@ from vllm.logger import init_logger
from vllm.logprobs import Logprob as SampleLogprob
from vllm.logprobs import SampleLogprobs
from vllm.outputs import CompletionOutput
from vllm.parser import ParserManager
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
from vllm.tokenizers import TokenizerLike
from vllm.utils import random_uuid
@@ -217,8 +218,13 @@ class OpenAIServingResponses(OpenAIServing):
self.chat_template_content_format: Final = chat_template_content_format
self.enable_log_outputs = enable_log_outputs
self.reasoning_parser = self._get_reasoning_parser(
reasoning_parser_name=reasoning_parser
# Set up the unified parser - either a unified parser or fall back to
# separate parsers accessed through the parser interface
self.parser = ParserManager.get_parser(
tool_parser_name=tool_parser,
reasoning_parser_name=reasoning_parser,
enable_auto_tools=enable_auto_tools,
model_name=self.model_config.model,
)
self.enable_prompt_tokens_details = enable_prompt_tokens_details
self.enable_force_include_usage = enable_force_include_usage
@@ -263,10 +269,6 @@ class OpenAIServingResponses(OpenAIServing):
self.tool_call_id_type = "random"
self.enable_auto_tools = enable_auto_tools
# set up tool use
self.tool_parser = self._get_tool_parser(
tool_parser_name=tool_parser, enable_auto_tools=enable_auto_tools
)
# HACK(woosuk): This is a hack. We should use a better store.
# FIXME: If enable_store=True, this may cause a memory leak since we
# never remove responses from the store.
@@ -469,9 +471,13 @@ class OpenAIServingResponses(OpenAIServing):
context = ParsableContext(
response_messages=messages,
tokenizer=tokenizer,
reasoning_parser_cls=self.reasoning_parser,
reasoning_parser_cls=self.parser.reasoning_parser_cls
if self.parser
else None,
request=request,
tool_parser_cls=self.tool_parser,
tool_parser_cls=self.parser.tool_parser_cls
if self.parser
else None,
available_tools=available_tools,
chat_template=self.chat_template,
chat_template_content_format=self.chat_template_content_format,
@@ -479,8 +485,8 @@ class OpenAIServingResponses(OpenAIServing):
else:
context = SimpleContext()
if self.reasoning_parser is not None:
reasoning_parser = self.reasoning_parser(tokenizer)
if self.parser and self.parser.reasoning_parser_cls is not None:
reasoning_parser = self.parser.reasoning_parser_cls(tokenizer)
if (
isinstance(
struct_out := sampling_params.structured_outputs,
@@ -617,7 +623,7 @@ class OpenAIServingResponses(OpenAIServing):
default_template_content_format=self.chat_template_content_format,
default_template_kwargs=None,
tool_dicts=tool_dicts,
tool_parser=self.tool_parser,
tool_parser=self.parser.tool_parser_cls if self.parser else None,
)
return messages, engine_prompts
@@ -909,9 +915,9 @@ class OpenAIServingResponses(OpenAIServing):
final_output: CompletionOutput,
tokenizer: TokenizerLike,
) -> list[ResponseOutputItem]:
if self.reasoning_parser:
if self.parser and self.parser.reasoning_parser_cls:
try:
reasoning_parser = self.reasoning_parser(tokenizer)
reasoning_parser = self.parser.reasoning_parser_cls(tokenizer)
except RuntimeError as e:
logger.exception("Error in reasoning parser creation.")
raise e
@@ -958,7 +964,7 @@ class OpenAIServingResponses(OpenAIServing):
tokenizer=tokenizer,
content=content,
enable_auto_tools=self.enable_auto_tools,
tool_parser_cls=self.tool_parser,
tool_parser_cls=self.parser.tool_parser_cls if self.parser else None,
)
if content or (self.use_harmony and tool_calls):
@@ -1339,8 +1345,8 @@ class OpenAIServingResponses(OpenAIServing):
current_output_index = 0
current_item_id = ""
reasoning_parser = None
if self.reasoning_parser:
reasoning_parser = self.reasoning_parser(tokenizer)
if self.parser and self.parser.reasoning_parser_cls:
reasoning_parser = self.parser.reasoning_parser_cls(tokenizer)
previous_text = ""
previous_token_ids: list[int] = []
first_delta_sent = False