[Frontend][Metrics] Add vllm:tool_call_parser_invocations_total Prometheus metric (#44448)

Signed-off-by: Yifan Zong <yzong@redhat.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

vllm/entrypoints/openai/api_server.py

@@ -13,7 +13,7 @@ import warnings
 from argparse import Namespace
 from collections.abc import AsyncIterator
 from contextlib import asynccontextmanager
-from typing import Any
+from typing import Any, cast
 
 import uvloop
 from fastapi import FastAPI, HTTPException
@@ -322,6 +322,13 @@ async def init_app_state(
         vllm_config.structured_outputs_config.enable_in_reasoning
     )
 
+    if args.tool_call_parser is not None:
+        from vllm.parser.metrics import init_parser_metrics
+
+        init_parser_metrics(
+            model_name=cast(str, vllm_config.model_config.served_model_name)
+        )
+
     if supported_tasks is None:
         warnings.warn(
             "The 'supported_tasks' parameter was not provided to "

vllm/parser/abstract_parser.py

@@ -35,6 +35,7 @@ from vllm.entrypoints.openai.engine.protocol import (
 )
 from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
 from vllm.logger import init_logger
+from vllm.parser.metrics import record_tool_parser_invocation
 from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
 from vllm.tokenizers import TokenizerLike
 from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser
@@ -276,7 +277,7 @@ class Parser:
     def extract_tool_calls(
         self,
         model_output: str,
-        request: ChatCompletionRequest,
+        request: ChatCompletionRequest | ResponsesRequest,
     ) -> ExtractedToolCallInformation:
         """
         Extract tool calls from a complete model-generated string.
@@ -300,7 +301,7 @@ class Parser:
         previous_token_ids: Sequence[int],
         current_token_ids: Sequence[int],
         delta_token_ids: Sequence[int],
-        request: ChatCompletionRequest,
+        request: ChatCompletionRequest | ResponsesRequest,
     ) -> DeltaMessage | None:
         """
         Extract tool calls from a streaming delta message.
@@ -514,9 +515,9 @@ class DelegatingParser(Parser):
             and (request.tool_choice == "auto" or request.tool_choice is None)
         ):
             # Automatic Tool Call Parsing
-            tool_call_info = self._tool_parser.extract_tool_calls(
+            tool_call_info = self.extract_tool_calls(
                 content if content is not None else "",
-                request=request,  # type: ignore
+                request=request,
             )
             if tool_call_info is not None and tool_call_info.tools_called:
                 function_calls.extend(
@@ -590,9 +591,9 @@ class DelegatingParser(Parser):
         elif is_auto_tool_choice:
             # Automatic Tool Call Parsing (also used as fallback for
             # required/named when supports_required_and_named=False)
-            tool_call_info = tool_parser.extract_tool_calls(
+            tool_call_info = self.extract_tool_calls(
                 content if content is not None else "",
-                request=request,  # type: ignore
+                request=request,
             )
             if tool_call_info is not None and tool_call_info.tools_called:
                 tool_calls.extend(
@@ -644,13 +645,30 @@ class DelegatingParser(Parser):
     def extract_tool_calls(
         self,
         model_output: str,
-        request: ChatCompletionRequest,
+        request: ChatCompletionRequest | ResponsesRequest,
     ) -> ExtractedToolCallInformation:
         if self._tool_parser is None:
             return ExtractedToolCallInformation(
                 tools_called=False, tool_calls=[], content=model_output
             )
-        return self._tool_parser.extract_tool_calls(model_output, request)
+        result = None
+        is_tool_called: bool | Exception = False
+        try:
+            result = self._tool_parser.extract_tool_calls(
+                model_output,
+                request=request,  # type: ignore[arg-type]
+            )
+            is_tool_called = bool(result.tools_called)
+        except Exception as e:
+            is_tool_called = e
+            raise
+        finally:
+            record_tool_parser_invocation(
+                is_tool_called=is_tool_called,
+                is_streaming=False,
+                request=request,
+            )
+        return result
 
     def extract_tool_calls_streaming(
         self,
@@ -660,19 +678,33 @@ class DelegatingParser(Parser):
         previous_token_ids: Sequence[int],
         current_token_ids: Sequence[int],
         delta_token_ids: Sequence[int],
-        request: ChatCompletionRequest,
+        request: ChatCompletionRequest | ResponsesRequest,
     ) -> DeltaMessage | None:
         if self._tool_parser is None:
             return None
-        return self._tool_parser.extract_tool_calls_streaming(
-            previous_text,
-            current_text,
-            delta_text,
-            previous_token_ids,
-            current_token_ids,
-            delta_token_ids,
-            request,
-        )
+        result = None
+        is_tool_called: bool | Exception = False
+        try:
+            result = self._tool_parser.extract_tool_calls_streaming(
+                previous_text,
+                current_text,
+                delta_text,
+                previous_token_ids,
+                current_token_ids,
+                delta_token_ids,
+                request,  # type: ignore[arg-type]
+            )
+            is_tool_called = bool(result and result.tool_calls)
+        except Exception as e:
+            is_tool_called = e
+            raise
+        finally:
+            record_tool_parser_invocation(
+                is_tool_called=is_tool_called,
+                is_streaming=True,
+                request=request,
+            )
+        return result
 
     def _extract_tool_calls_streaming(
         self,
@@ -731,7 +763,7 @@ class DelegatingParser(Parser):
             previous_token_ids,
             current_token_ids,
             delta_token_ids,
-            request,  # type: ignore[arg-type]
+            request,
         ), False
 
     def is_reasoning_end(self, input_ids: list[int]) -> bool:

vllm/parser/metrics.py

@@ -0,0 +1,108 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Prometheus metrics for the parsers."""
+
+from __future__ import annotations
+
+from enum import Enum
+from itertools import product
+from typing import cast
+
+from prometheus_client import REGISTRY, Counter
+
+_model_name: str | None = None
+
+_TOOL_CALL_PARSER_INVOCATIONS_TOTAL = "vllm:tool_call_parser_invocations_total"
+_tool_call_parser_invocations: Counter | None = None
+
+
+class ToolCallOutcome(Enum):
+    TOOL_CALL = "tool_call"
+    NO_TOOL_CALL = "no_tool_call"
+
+
+class RequestType(Enum):
+    CHAT_COMPLETIONS = "chat_completions"
+    RESPONSES = "responses"
+    OTHER = "other"
+
+
+def init_parser_metrics(*, model_name: str) -> None:
+    """Lazily register parser metrics and cache the shared model label."""
+    global _model_name
+    _model_name = model_name
+
+    global _tool_call_parser_invocations
+    try:
+        _tool_call_parser_invocations = Counter(
+            name=_TOOL_CALL_PARSER_INVOCATIONS_TOTAL,
+            documentation=(
+                "Total number of ToolParser invocations. "
+                "Non-streaming increments once per choice; "
+                "streaming increments once per delta."
+            ),
+            labelnames=["model_name", "mode", "outcome", "request_type"],
+        )
+    except ValueError:
+        _tool_call_parser_invocations = cast(
+            Counter,
+            REGISTRY._names_to_collectors[_TOOL_CALL_PARSER_INVOCATIONS_TOTAL],
+        )
+
+    for mode, outcome, request_type in product(
+        ("streaming", "non_streaming"),
+        ToolCallOutcome,
+        RequestType,
+    ):
+        _tool_call_parser_invocations.labels(
+            model_name=_model_name,
+            mode=mode,
+            outcome=outcome.value,
+            request_type=request_type.value,
+        )
+
+
+def record_tool_parser_invocation(
+    *,
+    is_tool_called: bool | Exception,
+    is_streaming: bool,
+    request: object,
+) -> None:
+    """Increment the tool-call parser invocation counter when registered.
+    Currently parser failures are treated as no tool calls.
+
+    TODO: To accurately track parser failures, add a new ToolCallOutcome and
+    more importantly, ensure exceptions are propagated out of the ToolParsers
+    instead of being caught internally. This would require going through
+    ToolParser implementation on a case-by-case basis.
+    """
+    if _tool_call_parser_invocations is None:
+        return
+
+    from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
+    from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
+
+    match request:
+        case ChatCompletionRequest():
+            request_type = RequestType.CHAT_COMPLETIONS
+        case ResponsesRequest():
+            request_type = RequestType.RESPONSES
+        case _:
+            request_type = RequestType.OTHER
+
+    match is_tool_called:
+        case bool():
+            outcome = (
+                ToolCallOutcome.TOOL_CALL
+                if is_tool_called
+                else ToolCallOutcome.NO_TOOL_CALL
+            )
+        case _:
+            outcome = ToolCallOutcome.NO_TOOL_CALL
+
+    _tool_call_parser_invocations.labels(
+        model_name=_model_name,
+        mode="streaming" if is_streaming else "non_streaming",
+        outcome=outcome.value,
+        request_type=request_type.value,
+    ).inc()