From fbf2c5e8be9754f31c4e0f549189fc9f3bdd213c Mon Sep 17 00:00:00 2001 From: xinnywinne <36653773+xinnywinne@users.noreply.github.com> Date: Wed, 16 Sep 2026 15:52:05 -0700 Subject: [PATCH] [Frontend] Add per-request metrics to Responses API (#55084) Signed-off-by: wxin Co-authored-by: Benjamin Chislett --- docs/features/per_request_metrics.md | 12 ++ .../responses/test_serving_responses.py | 147 ++++++++++++++++-- vllm/entrypoints/generate/api_router.py | 1 + vllm/entrypoints/openai/responses/context.py | 7 + vllm/entrypoints/openai/responses/protocol.py | 13 +- vllm/entrypoints/openai/responses/serving.py | 18 ++- 6 files changed, 184 insertions(+), 14 deletions(-) diff --git a/docs/features/per_request_metrics.md b/docs/features/per_request_metrics.md index 25211ecfc3..da1685f929 100644 --- a/docs/features/per_request_metrics.md +++ b/docs/features/per_request_metrics.md @@ -120,6 +120,18 @@ the same `metrics` response field. As with `n > 1`, metrics are omitted for requests with multiple prompts, because the timing data cannot be attributed to a single prompt's generation. +## Responses API + +Per-request metrics are available on the `/v1/responses` endpoint using the +common `metrics` object described above. Non-streaming responses include it at +the top level. Streaming responses include it in the final response carried by +the `response.completed` event; intermediate events do not include metrics. + +Metrics are omitted for Responses requests that perform multiple +model-generation turns, such as built-in tool-call workflows, because the +response retains timing data for only one generation turn while token usage is +accumulated across all turns. + ## Relationship to Prometheus Metrics The `metrics` response field provides per-request values for a single request. diff --git a/tests/entrypoints/openai/responses/test_serving_responses.py b/tests/entrypoints/openai/responses/test_serving_responses.py index 0f361c1c6a..1a315902e2 100644 --- a/tests/entrypoints/openai/responses/test_serving_responses.py +++ b/tests/entrypoints/openai/responses/test_serving_responses.py @@ -48,6 +48,7 @@ from vllm.entrypoints.openai.responses.context import ( SimpleContext, ) from vllm.entrypoints.openai.responses.protocol import ( + ResponseCompletedEvent, ResponseCreatedEvent, ResponseRawMessageAndToken, ResponsesRequest, @@ -71,6 +72,7 @@ from vllm.renderers.online_renderer import ( _extract_allowed_tools_from_mcp_requests, ) from vllm.sampling_params import SamplingParams +from vllm.v1.metrics.stats import RequestStateStats pytestmark = pytest.mark.skip_global_cleanup @@ -795,6 +797,7 @@ class TestInitializeToolSessions: pass assert serving_responses_instance.engine_client.generate.call_count == 2 + assert context.request_metrics_cover_all_generation_turns is False followup_engine_input = ( serving_responses_instance.engine_client.generate.call_args_list[1].args[0] ) @@ -1248,9 +1251,19 @@ class TestHarmonyPreambleStreaming: assert "response.output_item.done" in type_names -def _make_simple_context_with_output(text, token_ids, response_parser=None): - """Create a SimpleContext with a RequestOutput containing the given text.""" - ctx = SimpleContext(response_parser=response_parser) +_PER_REQUEST_STATS = RequestStateStats( + queued_ts=1.0, + scheduled_ts=1.5, + first_token_ts=2.0, + last_token_ts=3.0, +) + + +def _make_request_output( + text, + token_ids, + metrics: RequestStateStats | None = None, +): completion = CompletionOutput( index=0, text=text, @@ -1260,7 +1273,7 @@ def _make_simple_context_with_output(text, token_ids, response_parser=None): finish_reason=None, stop_reason=None, ) - req_output = RequestOutput( + return RequestOutput( request_id="req", prompt="hi", prompt_token_ids=[7, 8], @@ -1268,13 +1281,29 @@ def _make_simple_context_with_output(text, token_ids, response_parser=None): outputs=[completion], finished=False, num_cached_tokens=0, + metrics=metrics, ) + + +def _make_simple_context_with_output( + text, + token_ids, + response_parser=None, + metrics: RequestStateStats | None = None, +): + """Create a SimpleContext with a RequestOutput containing the given text.""" + ctx = SimpleContext(response_parser=response_parser) + req_output = _make_request_output(text, token_ids, metrics) ctx.append_output(req_output) + ctx.request_metrics = metrics return ctx -def _make_serving_instance_with_reasoning(): - """Create an OpenAIServingResponses with a mocked reasoning parser.""" +def _make_serving_instance( + *, + reasoning_parser: str = "", + enable_per_request_metrics: bool = False, +) -> OpenAIServingResponses: engine_client = MagicMock() model_config = MagicMock() model_config.max_model_len = 100 @@ -1285,18 +1314,112 @@ def _make_serving_instance_with_reasoning(): engine_client.input_processor = MagicMock() engine_client.renderer = MagicMock() - models = MagicMock() - - serving = OpenAIServingResponses( + return OpenAIServingResponses( engine_client=engine_client, - models=models, + models=MagicMock(), online_renderer=MagicMock(), request_logger=None, chat_template=None, chat_template_content_format="auto", - reasoning_parser="qwen3", + reasoning_parser=reasoning_parser, + enable_per_request_metrics=enable_per_request_metrics, ) - return serving + + +async def _empty_context_generator(): + if False: + yield + + +async def _make_full_metrics_response( + enable_per_request_metrics: bool, + request_metrics_cover_all_generation_turns: bool = True, +): + serving = _make_serving_instance( + enable_per_request_metrics=enable_per_request_metrics + ) + request = ResponsesRequest(input="hi", tools=[], stream=False, store=False) + sampling_params = SamplingParams(max_tokens=16) + context = SimpleContext() + context.request_metrics_cover_all_generation_turns = ( + request_metrics_cover_all_generation_turns + ) + + async def generate(*args, **kwargs): + yield _make_request_output("hello", [10, 20], _PER_REQUEST_STATS) + + serving.engine_client.generate.side_effect = generate + result_generator = serving._generate_with_builtin_tools( + request_id=request.request_id, + engine_input=tokens_input([7, 8]), + sampling_params=sampling_params, + context=context, + ) + response = await serving.responses_full_generator( + request=request, + sampling_params=sampling_params, + result_generator=result_generator, + context=context, + model_name="test-model", + tokenizer=MagicMock(), + request_metadata=RequestResponseMetadata(request_id="req"), + ) + assert isinstance(response, ResponsesResponse) + return response + + +@pytest.mark.asyncio +async def test_responses_per_request_metrics_follow_server_flag(): + disabled_response = await _make_full_metrics_response(False) + assert disabled_response.metrics is None + assert "metrics" not in disabled_response.model_dump(mode="json") + + enabled_response = await _make_full_metrics_response(True) + assert enabled_response.metrics is not None + assert enabled_response.metrics.time_to_first_token_ms == pytest.approx(500.0) + + +@pytest.mark.asyncio +async def test_responses_streaming_metrics_only_on_completed_event(): + serving = _make_serving_instance(enable_per_request_metrics=True) + request = ResponsesRequest(input="hi", tools=[], stream=True, store=False) + context = _make_simple_context_with_output( + "hello", [10, 20], metrics=_PER_REQUEST_STATS + ) + + events = [ + event + async for event in serving.responses_stream_generator( + request=request, + sampling_params=SamplingParams(max_tokens=16), + result_generator=_empty_context_generator(), + context=context, + model_name="test-model", + tokenizer=MagicMock(), + request_metadata=RequestResponseMetadata(request_id="req"), + ) + ] + + for event in events[:-1]: + assert "metrics" not in event.response.model_dump(mode="json") + assert isinstance(events[-1], ResponseCompletedEvent) + assert events[-1].response.metrics is not None + assert events[-1].response.metrics.time_to_first_token_ms == pytest.approx(500.0) + + +@pytest.mark.asyncio +async def test_responses_metrics_suppressed_for_multiple_generation_turns(): + response = await _make_full_metrics_response( + True, request_metrics_cover_all_generation_turns=False + ) + + assert response.metrics is None + assert "metrics" not in response.model_dump(mode="json") + + +def _make_serving_instance_with_reasoning(): + """Create an OpenAIServingResponses with a mocked reasoning parser.""" + return _make_serving_instance(reasoning_parser="qwen3") def _identity_increment(event): diff --git a/vllm/entrypoints/generate/api_router.py b/vllm/entrypoints/generate/api_router.py index 12c70e37d5..ac995a4f03 100644 --- a/vllm/entrypoints/generate/api_router.py +++ b/vllm/entrypoints/generate/api_router.py @@ -116,6 +116,7 @@ async def init_generate_state( reasoning_parser=args.structured_outputs_config.reasoning_parser, enable_prompt_tokens_details=args.enable_prompt_tokens_details, enable_force_include_usage=args.enable_force_include_usage, + enable_per_request_metrics=args.enable_per_request_metrics, enable_log_outputs=args.enable_log_outputs, default_chat_template_kwargs=default_chat_template_kwargs, ) diff --git a/vllm/entrypoints/openai/responses/context.py b/vllm/entrypoints/openai/responses/context.py index 3414bd6d82..0db47cf367 100644 --- a/vllm/entrypoints/openai/responses/context.py +++ b/vllm/entrypoints/openai/responses/context.py @@ -45,6 +45,8 @@ from vllm.utils import random_uuid if TYPE_CHECKING: from mcp.client import ClientSession + from vllm.v1.metrics.stats import RequestStateStats + logger = logging.getLogger(__name__) # This is currently needed as the tool type doesn't 1:1 match the @@ -102,6 +104,11 @@ class TurnMetrics: class ConversationContext(ABC): response_parser: Parser | None = None + request_metrics: "RequestStateStats | None" = None + # Built-in tools can trigger additional model generations. In that case, + # the stored engine timestamps cover only one turn, while token usage is + # accumulated across all turns. + request_metrics_cover_all_generation_turns: bool = True @abstractmethod def append_output(self, output: RequestOutput) -> None: diff --git a/vllm/entrypoints/openai/responses/protocol.py b/vllm/entrypoints/openai/responses/protocol.py index ec28d2823b..750d1db84c 100644 --- a/vllm/entrypoints/openai/responses/protocol.py +++ b/vllm/entrypoints/openai/responses/protocol.py @@ -60,7 +60,11 @@ from vllm.entrypoints.chat_utils import ( ChatCompletionMessageParam, ChatTemplateContentFormatOption, ) -from vllm.entrypoints.generate.base.protocol import StopParam, validate_cache_salt +from vllm.entrypoints.generate.base.protocol import ( + PerRequestMetrics, + StopParam, + validate_cache_salt, +) from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel from vllm.exceptions import VLLMValidationError from vllm.logger import init_logger @@ -684,6 +688,11 @@ class ResponsesResponse(OpenAIBaseModel): usage: ResponseUsage | None = None user: str | None = None + # vLLM-specific per-request metrics. Omitted unless enabled server-side. + metrics: PerRequestMetrics | None = Field( + default=None, exclude_if=lambda value: value is None + ) + presence_penalty: float | None = Field( default=None, ge=-2.0, @@ -752,6 +761,7 @@ class ResponsesResponse(OpenAIBaseModel): output: list[ResponseOutputItem], status: ResponseStatus, usage: ResponseUsage | None = None, + metrics: PerRequestMetrics | None = None, input_messages: ResponseInputOutputMessage | None = None, output_messages: ResponseInputOutputMessage | None = None, kv_transfer_params: dict[str, Any] | None = None, @@ -795,6 +805,7 @@ class ResponsesResponse(OpenAIBaseModel): truncation=request.truncation, user=request.user, usage=usage, + metrics=metrics, kv_transfer_params=kv_transfer_params, ec_transfer_params=ec_transfer_params, ) diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index bce54338c4..ee4f7d55ca 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -30,7 +30,10 @@ from vllm.entrypoints.generate.base.protocol import ( DeltaMessage, RequestResponseMetadata, ) -from vllm.entrypoints.generate.base.serving import GenerateBaseServing +from vllm.entrypoints.generate.base.serving import ( + GenerateBaseServing, + build_per_request_timing_metrics, +) from vllm.entrypoints.mcp.tool_server import ToolServer from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.openai.responses.context import ( @@ -109,6 +112,7 @@ class OpenAIServingResponses(GenerateBaseServing): tool_server: ToolServer | None = None, enable_prompt_tokens_details: bool = False, enable_force_include_usage: bool = False, + enable_per_request_metrics: bool = False, enable_log_outputs: bool = False, default_chat_template_kwargs: dict[str, Any] | None = None, ) -> None: @@ -136,6 +140,7 @@ class OpenAIServingResponses(GenerateBaseServing): ) self.enable_prompt_tokens_details = enable_prompt_tokens_details self.enable_force_include_usage = enable_force_include_usage + self.enable_per_request_metrics = enable_per_request_metrics self.default_sampling_params = self.model_config.get_diff_sampling_param() mc = self.model_config @@ -645,6 +650,7 @@ class OpenAIServingResponses(GenerateBaseServing): ) async for res in generator: + context.request_metrics = res.metrics context.append_output(res) # NOTE(woosuk): The stop condition is handled by the engine. yield context @@ -654,6 +660,7 @@ class OpenAIServingResponses(GenerateBaseServing): break # Call the tool and update the context with the result. + context.request_metrics_cover_all_generation_turns = False tool_output = await context.call_tool() context.append_tool_output(tool_output) @@ -857,6 +864,14 @@ class OpenAIServingResponses(GenerateBaseServing): ], ), ) + per_request_metrics = None + if ( + self.enable_per_request_metrics + and context.request_metrics_cover_all_generation_turns + ): + per_request_metrics = build_per_request_timing_metrics( + context.request_metrics, num_generated_tokens + ) response = ResponsesResponse.from_request( request, sampling_params, @@ -867,6 +882,7 @@ class OpenAIServingResponses(GenerateBaseServing): output=output, status=status, usage=usage, + metrics=per_request_metrics, kv_transfer_params=context.kv_transfer_params, ec_transfer_params=context.ec_transfer_params, )