[1/N] Initial Implementation of Parser for ResponsesAPI (#32712)
Signed-off-by: Andrew Xia <axia@fb.com> Co-authored-by: Andrew Xia <axia@fb.com>
This commit is contained in:
@@ -71,6 +71,7 @@ from vllm.inputs.data import EmbedsPrompt, TokensPrompt
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.parser import ParserManager
|
||||
from vllm.sampling_params import BeamSearchParams, SamplingParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tokenizers.mistral import (
|
||||
@@ -131,13 +132,15 @@ class OpenAIServingChat(OpenAIServing):
|
||||
self.logits_processors = self.model_config.logits_processors
|
||||
|
||||
# set up reasoning parser
|
||||
self.reasoning_parser = self._get_reasoning_parser(
|
||||
self.reasoning_parser = ParserManager.get_reasoning_parser(
|
||||
reasoning_parser_name=reasoning_parser
|
||||
)
|
||||
# set up tool use
|
||||
self.enable_auto_tools: bool = enable_auto_tools
|
||||
self.tool_parser = self._get_tool_parser(
|
||||
tool_parser_name=tool_parser, enable_auto_tools=enable_auto_tools
|
||||
self.tool_parser = ParserManager.get_tool_parser(
|
||||
tool_parser_name=tool_parser,
|
||||
enable_auto_tools=enable_auto_tools,
|
||||
model_name=self.model_config.model,
|
||||
)
|
||||
self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
|
||||
|
||||
|
||||
@@ -107,11 +107,10 @@ from vllm.lora.request import LoRARequest
|
||||
from vllm.multimodal import MultiModalDataDict
|
||||
from vllm.outputs import CompletionOutput, PoolingRequestOutput, RequestOutput
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.reasoning import ReasoningParser, ReasoningParserManager
|
||||
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
|
||||
from vllm.sampling_params import BeamSearchParams, SamplingParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers import ToolParser, ToolParserManager
|
||||
from vllm.tool_parsers import ToolParser
|
||||
from vllm.tracing import (
|
||||
contains_trace_headers,
|
||||
extract_trace_headers,
|
||||
@@ -246,46 +245,6 @@ class OpenAIServing:
|
||||
self.model_config = self.models.model_config
|
||||
self.max_model_len = self.model_config.max_model_len
|
||||
|
||||
def _get_tool_parser(
|
||||
self, tool_parser_name: str | None = None, enable_auto_tools: bool = False
|
||||
) -> Callable[[TokenizerLike], ToolParser] | None:
|
||||
"""Get the tool parser based on the name."""
|
||||
parser = None
|
||||
if not enable_auto_tools or tool_parser_name is None:
|
||||
return parser
|
||||
logger.info('"auto" tool choice has been enabled.')
|
||||
|
||||
try:
|
||||
if tool_parser_name == "pythonic" and self.model_config.model.startswith(
|
||||
"meta-llama/Llama-3.2"
|
||||
):
|
||||
logger.warning(
|
||||
"Llama3.2 models may struggle to emit valid pythonic tool calls"
|
||||
)
|
||||
parser = ToolParserManager.get_tool_parser(tool_parser_name)
|
||||
except Exception as e:
|
||||
raise TypeError(
|
||||
"Error: --enable-auto-tool-choice requires "
|
||||
f"tool_parser:'{tool_parser_name}' which has not "
|
||||
"been registered"
|
||||
) from e
|
||||
return parser
|
||||
|
||||
def _get_reasoning_parser(
|
||||
self,
|
||||
reasoning_parser_name: str,
|
||||
) -> Callable[[TokenizerLike], ReasoningParser] | None:
|
||||
"""Get the reasoning parser based on the name."""
|
||||
parser = None
|
||||
if not reasoning_parser_name:
|
||||
return None
|
||||
try:
|
||||
parser = ReasoningParserManager.get_reasoning_parser(reasoning_parser_name)
|
||||
assert parser is not None
|
||||
except Exception as e:
|
||||
raise TypeError(f"{reasoning_parser_name=} has not been registered") from e
|
||||
return parser
|
||||
|
||||
async def beam_search(
|
||||
self,
|
||||
prompt: PromptType,
|
||||
|
||||
@@ -123,6 +123,7 @@ from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob as SampleLogprob
|
||||
from vllm.logprobs import SampleLogprobs
|
||||
from vllm.outputs import CompletionOutput
|
||||
from vllm.parser import ParserManager
|
||||
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils import random_uuid
|
||||
@@ -217,8 +218,13 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
self.chat_template_content_format: Final = chat_template_content_format
|
||||
self.enable_log_outputs = enable_log_outputs
|
||||
|
||||
self.reasoning_parser = self._get_reasoning_parser(
|
||||
reasoning_parser_name=reasoning_parser
|
||||
# Set up the unified parser - either a unified parser or fall back to
|
||||
# separate parsers accessed through the parser interface
|
||||
self.parser = ParserManager.get_parser(
|
||||
tool_parser_name=tool_parser,
|
||||
reasoning_parser_name=reasoning_parser,
|
||||
enable_auto_tools=enable_auto_tools,
|
||||
model_name=self.model_config.model,
|
||||
)
|
||||
self.enable_prompt_tokens_details = enable_prompt_tokens_details
|
||||
self.enable_force_include_usage = enable_force_include_usage
|
||||
@@ -263,10 +269,6 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
self.tool_call_id_type = "random"
|
||||
|
||||
self.enable_auto_tools = enable_auto_tools
|
||||
# set up tool use
|
||||
self.tool_parser = self._get_tool_parser(
|
||||
tool_parser_name=tool_parser, enable_auto_tools=enable_auto_tools
|
||||
)
|
||||
# HACK(woosuk): This is a hack. We should use a better store.
|
||||
# FIXME: If enable_store=True, this may cause a memory leak since we
|
||||
# never remove responses from the store.
|
||||
@@ -469,9 +471,13 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
context = ParsableContext(
|
||||
response_messages=messages,
|
||||
tokenizer=tokenizer,
|
||||
reasoning_parser_cls=self.reasoning_parser,
|
||||
reasoning_parser_cls=self.parser.reasoning_parser_cls
|
||||
if self.parser
|
||||
else None,
|
||||
request=request,
|
||||
tool_parser_cls=self.tool_parser,
|
||||
tool_parser_cls=self.parser.tool_parser_cls
|
||||
if self.parser
|
||||
else None,
|
||||
available_tools=available_tools,
|
||||
chat_template=self.chat_template,
|
||||
chat_template_content_format=self.chat_template_content_format,
|
||||
@@ -479,8 +485,8 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
else:
|
||||
context = SimpleContext()
|
||||
|
||||
if self.reasoning_parser is not None:
|
||||
reasoning_parser = self.reasoning_parser(tokenizer)
|
||||
if self.parser and self.parser.reasoning_parser_cls is not None:
|
||||
reasoning_parser = self.parser.reasoning_parser_cls(tokenizer)
|
||||
if (
|
||||
isinstance(
|
||||
struct_out := sampling_params.structured_outputs,
|
||||
@@ -617,7 +623,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
default_template_content_format=self.chat_template_content_format,
|
||||
default_template_kwargs=None,
|
||||
tool_dicts=tool_dicts,
|
||||
tool_parser=self.tool_parser,
|
||||
tool_parser=self.parser.tool_parser_cls if self.parser else None,
|
||||
)
|
||||
return messages, engine_prompts
|
||||
|
||||
@@ -909,9 +915,9 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
final_output: CompletionOutput,
|
||||
tokenizer: TokenizerLike,
|
||||
) -> list[ResponseOutputItem]:
|
||||
if self.reasoning_parser:
|
||||
if self.parser and self.parser.reasoning_parser_cls:
|
||||
try:
|
||||
reasoning_parser = self.reasoning_parser(tokenizer)
|
||||
reasoning_parser = self.parser.reasoning_parser_cls(tokenizer)
|
||||
except RuntimeError as e:
|
||||
logger.exception("Error in reasoning parser creation.")
|
||||
raise e
|
||||
@@ -958,7 +964,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
tokenizer=tokenizer,
|
||||
content=content,
|
||||
enable_auto_tools=self.enable_auto_tools,
|
||||
tool_parser_cls=self.tool_parser,
|
||||
tool_parser_cls=self.parser.tool_parser_cls if self.parser else None,
|
||||
)
|
||||
|
||||
if content or (self.use_harmony and tool_calls):
|
||||
@@ -1339,8 +1345,8 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
current_output_index = 0
|
||||
current_item_id = ""
|
||||
reasoning_parser = None
|
||||
if self.reasoning_parser:
|
||||
reasoning_parser = self.reasoning_parser(tokenizer)
|
||||
if self.parser and self.parser.reasoning_parser_cls:
|
||||
reasoning_parser = self.parser.reasoning_parser_cls(tokenizer)
|
||||
previous_text = ""
|
||||
previous_token_ids: list[int] = []
|
||||
first_delta_sent = False
|
||||
|
||||
Reference in New Issue
Block a user