diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index c4ed8a5ee9..bd9dfc3931 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -13,7 +13,7 @@ import warnings from argparse import Namespace from collections.abc import AsyncIterator from contextlib import asynccontextmanager -from typing import Any +from typing import Any, cast import uvloop from fastapi import FastAPI, HTTPException @@ -322,6 +322,13 @@ async def init_app_state( vllm_config.structured_outputs_config.enable_in_reasoning ) + if args.tool_call_parser is not None: + from vllm.parser.metrics import init_parser_metrics + + init_parser_metrics( + model_name=cast(str, vllm_config.model_config.served_model_name) + ) + if supported_tasks is None: warnings.warn( "The 'supported_tasks' parameter was not provided to " diff --git a/vllm/parser/abstract_parser.py b/vllm/parser/abstract_parser.py index 4b039f7322..70fb919fce 100644 --- a/vllm/parser/abstract_parser.py +++ b/vllm/parser/abstract_parser.py @@ -35,6 +35,7 @@ from vllm.entrypoints.openai.engine.protocol import ( ) from vllm.entrypoints.openai.responses.protocol import ResponsesRequest from vllm.logger import init_logger +from vllm.parser.metrics import record_tool_parser_invocation from vllm.reasoning.abs_reasoning_parsers import ReasoningParser from vllm.tokenizers import TokenizerLike from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser @@ -276,7 +277,7 @@ class Parser: def extract_tool_calls( self, model_output: str, - request: ChatCompletionRequest, + request: ChatCompletionRequest | ResponsesRequest, ) -> ExtractedToolCallInformation: """ Extract tool calls from a complete model-generated string. @@ -300,7 +301,7 @@ class Parser: previous_token_ids: Sequence[int], current_token_ids: Sequence[int], delta_token_ids: Sequence[int], - request: ChatCompletionRequest, + request: ChatCompletionRequest | ResponsesRequest, ) -> DeltaMessage | None: """ Extract tool calls from a streaming delta message. @@ -514,9 +515,9 @@ class DelegatingParser(Parser): and (request.tool_choice == "auto" or request.tool_choice is None) ): # Automatic Tool Call Parsing - tool_call_info = self._tool_parser.extract_tool_calls( + tool_call_info = self.extract_tool_calls( content if content is not None else "", - request=request, # type: ignore + request=request, ) if tool_call_info is not None and tool_call_info.tools_called: function_calls.extend( @@ -590,9 +591,9 @@ class DelegatingParser(Parser): elif is_auto_tool_choice: # Automatic Tool Call Parsing (also used as fallback for # required/named when supports_required_and_named=False) - tool_call_info = tool_parser.extract_tool_calls( + tool_call_info = self.extract_tool_calls( content if content is not None else "", - request=request, # type: ignore + request=request, ) if tool_call_info is not None and tool_call_info.tools_called: tool_calls.extend( @@ -644,13 +645,30 @@ class DelegatingParser(Parser): def extract_tool_calls( self, model_output: str, - request: ChatCompletionRequest, + request: ChatCompletionRequest | ResponsesRequest, ) -> ExtractedToolCallInformation: if self._tool_parser is None: return ExtractedToolCallInformation( tools_called=False, tool_calls=[], content=model_output ) - return self._tool_parser.extract_tool_calls(model_output, request) + result = None + is_tool_called: bool | Exception = False + try: + result = self._tool_parser.extract_tool_calls( + model_output, + request=request, # type: ignore[arg-type] + ) + is_tool_called = bool(result.tools_called) + except Exception as e: + is_tool_called = e + raise + finally: + record_tool_parser_invocation( + is_tool_called=is_tool_called, + is_streaming=False, + request=request, + ) + return result def extract_tool_calls_streaming( self, @@ -660,19 +678,33 @@ class DelegatingParser(Parser): previous_token_ids: Sequence[int], current_token_ids: Sequence[int], delta_token_ids: Sequence[int], - request: ChatCompletionRequest, + request: ChatCompletionRequest | ResponsesRequest, ) -> DeltaMessage | None: if self._tool_parser is None: return None - return self._tool_parser.extract_tool_calls_streaming( - previous_text, - current_text, - delta_text, - previous_token_ids, - current_token_ids, - delta_token_ids, - request, - ) + result = None + is_tool_called: bool | Exception = False + try: + result = self._tool_parser.extract_tool_calls_streaming( + previous_text, + current_text, + delta_text, + previous_token_ids, + current_token_ids, + delta_token_ids, + request, # type: ignore[arg-type] + ) + is_tool_called = bool(result and result.tool_calls) + except Exception as e: + is_tool_called = e + raise + finally: + record_tool_parser_invocation( + is_tool_called=is_tool_called, + is_streaming=True, + request=request, + ) + return result def _extract_tool_calls_streaming( self, @@ -731,7 +763,7 @@ class DelegatingParser(Parser): previous_token_ids, current_token_ids, delta_token_ids, - request, # type: ignore[arg-type] + request, ), False def is_reasoning_end(self, input_ids: list[int]) -> bool: diff --git a/vllm/parser/metrics.py b/vllm/parser/metrics.py new file mode 100644 index 0000000000..bd700c2483 --- /dev/null +++ b/vllm/parser/metrics.py @@ -0,0 +1,108 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Prometheus metrics for the parsers.""" + +from __future__ import annotations + +from enum import Enum +from itertools import product +from typing import cast + +from prometheus_client import REGISTRY, Counter + +_model_name: str | None = None + +_TOOL_CALL_PARSER_INVOCATIONS_TOTAL = "vllm:tool_call_parser_invocations_total" +_tool_call_parser_invocations: Counter | None = None + + +class ToolCallOutcome(Enum): + TOOL_CALL = "tool_call" + NO_TOOL_CALL = "no_tool_call" + + +class RequestType(Enum): + CHAT_COMPLETIONS = "chat_completions" + RESPONSES = "responses" + OTHER = "other" + + +def init_parser_metrics(*, model_name: str) -> None: + """Lazily register parser metrics and cache the shared model label.""" + global _model_name + _model_name = model_name + + global _tool_call_parser_invocations + try: + _tool_call_parser_invocations = Counter( + name=_TOOL_CALL_PARSER_INVOCATIONS_TOTAL, + documentation=( + "Total number of ToolParser invocations. " + "Non-streaming increments once per choice; " + "streaming increments once per delta." + ), + labelnames=["model_name", "mode", "outcome", "request_type"], + ) + except ValueError: + _tool_call_parser_invocations = cast( + Counter, + REGISTRY._names_to_collectors[_TOOL_CALL_PARSER_INVOCATIONS_TOTAL], + ) + + for mode, outcome, request_type in product( + ("streaming", "non_streaming"), + ToolCallOutcome, + RequestType, + ): + _tool_call_parser_invocations.labels( + model_name=_model_name, + mode=mode, + outcome=outcome.value, + request_type=request_type.value, + ) + + +def record_tool_parser_invocation( + *, + is_tool_called: bool | Exception, + is_streaming: bool, + request: object, +) -> None: + """Increment the tool-call parser invocation counter when registered. + Currently parser failures are treated as no tool calls. + + TODO: To accurately track parser failures, add a new ToolCallOutcome and + more importantly, ensure exceptions are propagated out of the ToolParsers + instead of being caught internally. This would require going through + ToolParser implementation on a case-by-case basis. + """ + if _tool_call_parser_invocations is None: + return + + from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest + from vllm.entrypoints.openai.responses.protocol import ResponsesRequest + + match request: + case ChatCompletionRequest(): + request_type = RequestType.CHAT_COMPLETIONS + case ResponsesRequest(): + request_type = RequestType.RESPONSES + case _: + request_type = RequestType.OTHER + + match is_tool_called: + case bool(): + outcome = ( + ToolCallOutcome.TOOL_CALL + if is_tool_called + else ToolCallOutcome.NO_TOOL_CALL + ) + case _: + outcome = ToolCallOutcome.NO_TOOL_CALL + + _tool_call_parser_invocations.labels( + model_name=_model_name, + mode="streaming" if is_streaming else "non_streaming", + outcome=outcome.value, + request_type=request_type.value, + ).inc()