[Frontend] Add per-request metrics to Responses API (#55084)

Signed-off-by: wxin <wxin@nvidia.com>
Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
xinnywinne
2026-09-16 22:52:05 +00:00
committed by GitHub
co-authored by Benjamin Chislett
parent 0983aef8da
commit fbf2c5e8be
6 changed files with 184 additions and 14 deletions
@@ -45,6 +45,8 @@ from vllm.utils import random_uuid
if TYPE_CHECKING:
from mcp.client import ClientSession
from vllm.v1.metrics.stats import RequestStateStats
logger = logging.getLogger(__name__)
# This is currently needed as the tool type doesn't 1:1 match the
@@ -102,6 +104,11 @@ class TurnMetrics:
class ConversationContext(ABC):
response_parser: Parser | None = None
request_metrics: "RequestStateStats | None" = None
# Built-in tools can trigger additional model generations. In that case,
# the stored engine timestamps cover only one turn, while token usage is
# accumulated across all turns.
request_metrics_cover_all_generation_turns: bool = True
@abstractmethod
def append_output(self, output: RequestOutput) -> None: