[Frontend][Metrics] Add vllm:tool_call_parser_invocations_total Prometheus metric (#44448)
Signed-off-by: Yifan Zong <yzong@redhat.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
c9e5bf8135
commit
6ec7dcd641
@@ -35,6 +35,7 @@ from vllm.entrypoints.openai.engine.protocol import (
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.logger import init_logger
|
||||
from vllm.parser.metrics import record_tool_parser_invocation
|
||||
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser
|
||||
@@ -276,7 +277,7 @@ class Parser:
|
||||
def extract_tool_calls(
|
||||
self,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> ExtractedToolCallInformation:
|
||||
"""
|
||||
Extract tool calls from a complete model-generated string.
|
||||
@@ -300,7 +301,7 @@ class Parser:
|
||||
previous_token_ids: Sequence[int],
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> DeltaMessage | None:
|
||||
"""
|
||||
Extract tool calls from a streaming delta message.
|
||||
@@ -514,9 +515,9 @@ class DelegatingParser(Parser):
|
||||
and (request.tool_choice == "auto" or request.tool_choice is None)
|
||||
):
|
||||
# Automatic Tool Call Parsing
|
||||
tool_call_info = self._tool_parser.extract_tool_calls(
|
||||
tool_call_info = self.extract_tool_calls(
|
||||
content if content is not None else "",
|
||||
request=request, # type: ignore
|
||||
request=request,
|
||||
)
|
||||
if tool_call_info is not None and tool_call_info.tools_called:
|
||||
function_calls.extend(
|
||||
@@ -590,9 +591,9 @@ class DelegatingParser(Parser):
|
||||
elif is_auto_tool_choice:
|
||||
# Automatic Tool Call Parsing (also used as fallback for
|
||||
# required/named when supports_required_and_named=False)
|
||||
tool_call_info = tool_parser.extract_tool_calls(
|
||||
tool_call_info = self.extract_tool_calls(
|
||||
content if content is not None else "",
|
||||
request=request, # type: ignore
|
||||
request=request,
|
||||
)
|
||||
if tool_call_info is not None and tool_call_info.tools_called:
|
||||
tool_calls.extend(
|
||||
@@ -644,13 +645,30 @@ class DelegatingParser(Parser):
|
||||
def extract_tool_calls(
|
||||
self,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> ExtractedToolCallInformation:
|
||||
if self._tool_parser is None:
|
||||
return ExtractedToolCallInformation(
|
||||
tools_called=False, tool_calls=[], content=model_output
|
||||
)
|
||||
return self._tool_parser.extract_tool_calls(model_output, request)
|
||||
result = None
|
||||
is_tool_called: bool | Exception = False
|
||||
try:
|
||||
result = self._tool_parser.extract_tool_calls(
|
||||
model_output,
|
||||
request=request, # type: ignore[arg-type]
|
||||
)
|
||||
is_tool_called = bool(result.tools_called)
|
||||
except Exception as e:
|
||||
is_tool_called = e
|
||||
raise
|
||||
finally:
|
||||
record_tool_parser_invocation(
|
||||
is_tool_called=is_tool_called,
|
||||
is_streaming=False,
|
||||
request=request,
|
||||
)
|
||||
return result
|
||||
|
||||
def extract_tool_calls_streaming(
|
||||
self,
|
||||
@@ -660,19 +678,33 @@ class DelegatingParser(Parser):
|
||||
previous_token_ids: Sequence[int],
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> DeltaMessage | None:
|
||||
if self._tool_parser is None:
|
||||
return None
|
||||
return self._tool_parser.extract_tool_calls_streaming(
|
||||
previous_text,
|
||||
current_text,
|
||||
delta_text,
|
||||
previous_token_ids,
|
||||
current_token_ids,
|
||||
delta_token_ids,
|
||||
request,
|
||||
)
|
||||
result = None
|
||||
is_tool_called: bool | Exception = False
|
||||
try:
|
||||
result = self._tool_parser.extract_tool_calls_streaming(
|
||||
previous_text,
|
||||
current_text,
|
||||
delta_text,
|
||||
previous_token_ids,
|
||||
current_token_ids,
|
||||
delta_token_ids,
|
||||
request, # type: ignore[arg-type]
|
||||
)
|
||||
is_tool_called = bool(result and result.tool_calls)
|
||||
except Exception as e:
|
||||
is_tool_called = e
|
||||
raise
|
||||
finally:
|
||||
record_tool_parser_invocation(
|
||||
is_tool_called=is_tool_called,
|
||||
is_streaming=True,
|
||||
request=request,
|
||||
)
|
||||
return result
|
||||
|
||||
def _extract_tool_calls_streaming(
|
||||
self,
|
||||
@@ -731,7 +763,7 @@ class DelegatingParser(Parser):
|
||||
previous_token_ids,
|
||||
current_token_ids,
|
||||
delta_token_ids,
|
||||
request, # type: ignore[arg-type]
|
||||
request,
|
||||
), False
|
||||
|
||||
def is_reasoning_end(self, input_ids: list[int]) -> bool:
|
||||
|
||||
Reference in New Issue
Block a user