[Frontend] Add per-request metrics to Responses API (#55084)

Signed-off-by: wxin <wxin@nvidia.com>
Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
xinnywinne
2026-09-16 22:52:05 +00:00
committed by GitHub
co-authored by Benjamin Chislett
parent 0983aef8da
commit fbf2c5e8be
6 changed files with 184 additions and 14 deletions
@@ -48,6 +48,7 @@ from vllm.entrypoints.openai.responses.context import (
SimpleContext,
)
from vllm.entrypoints.openai.responses.protocol import (
ResponseCompletedEvent,
ResponseCreatedEvent,
ResponseRawMessageAndToken,
ResponsesRequest,
@@ -71,6 +72,7 @@ from vllm.renderers.online_renderer import (
_extract_allowed_tools_from_mcp_requests,
)
from vllm.sampling_params import SamplingParams
from vllm.v1.metrics.stats import RequestStateStats
pytestmark = pytest.mark.skip_global_cleanup
@@ -795,6 +797,7 @@ class TestInitializeToolSessions:
pass
assert serving_responses_instance.engine_client.generate.call_count == 2
assert context.request_metrics_cover_all_generation_turns is False
followup_engine_input = (
serving_responses_instance.engine_client.generate.call_args_list[1].args[0]
)
@@ -1248,9 +1251,19 @@ class TestHarmonyPreambleStreaming:
assert "response.output_item.done" in type_names
def _make_simple_context_with_output(text, token_ids, response_parser=None):
"""Create a SimpleContext with a RequestOutput containing the given text."""
ctx = SimpleContext(response_parser=response_parser)
_PER_REQUEST_STATS = RequestStateStats(
queued_ts=1.0,
scheduled_ts=1.5,
first_token_ts=2.0,
last_token_ts=3.0,
)
def _make_request_output(
text,
token_ids,
metrics: RequestStateStats | None = None,
):
completion = CompletionOutput(
index=0,
text=text,
@@ -1260,7 +1273,7 @@ def _make_simple_context_with_output(text, token_ids, response_parser=None):
finish_reason=None,
stop_reason=None,
)
req_output = RequestOutput(
return RequestOutput(
request_id="req",
prompt="hi",
prompt_token_ids=[7, 8],
@@ -1268,13 +1281,29 @@ def _make_simple_context_with_output(text, token_ids, response_parser=None):
outputs=[completion],
finished=False,
num_cached_tokens=0,
metrics=metrics,
)
def _make_simple_context_with_output(
text,
token_ids,
response_parser=None,
metrics: RequestStateStats | None = None,
):
"""Create a SimpleContext with a RequestOutput containing the given text."""
ctx = SimpleContext(response_parser=response_parser)
req_output = _make_request_output(text, token_ids, metrics)
ctx.append_output(req_output)
ctx.request_metrics = metrics
return ctx
def _make_serving_instance_with_reasoning():
"""Create an OpenAIServingResponses with a mocked reasoning parser."""
def _make_serving_instance(
*,
reasoning_parser: str = "",
enable_per_request_metrics: bool = False,
) -> OpenAIServingResponses:
engine_client = MagicMock()
model_config = MagicMock()
model_config.max_model_len = 100
@@ -1285,18 +1314,112 @@ def _make_serving_instance_with_reasoning():
engine_client.input_processor = MagicMock()
engine_client.renderer = MagicMock()
models = MagicMock()
serving = OpenAIServingResponses(
return OpenAIServingResponses(
engine_client=engine_client,
models=models,
models=MagicMock(),
online_renderer=MagicMock(),
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
reasoning_parser="qwen3",
reasoning_parser=reasoning_parser,
enable_per_request_metrics=enable_per_request_metrics,
)
return serving
async def _empty_context_generator():
if False:
yield
async def _make_full_metrics_response(
enable_per_request_metrics: bool,
request_metrics_cover_all_generation_turns: bool = True,
):
serving = _make_serving_instance(
enable_per_request_metrics=enable_per_request_metrics
)
request = ResponsesRequest(input="hi", tools=[], stream=False, store=False)
sampling_params = SamplingParams(max_tokens=16)
context = SimpleContext()
context.request_metrics_cover_all_generation_turns = (
request_metrics_cover_all_generation_turns
)
async def generate(*args, **kwargs):
yield _make_request_output("hello", [10, 20], _PER_REQUEST_STATS)
serving.engine_client.generate.side_effect = generate
result_generator = serving._generate_with_builtin_tools(
request_id=request.request_id,
engine_input=tokens_input([7, 8]),
sampling_params=sampling_params,
context=context,
)
response = await serving.responses_full_generator(
request=request,
sampling_params=sampling_params,
result_generator=result_generator,
context=context,
model_name="test-model",
tokenizer=MagicMock(),
request_metadata=RequestResponseMetadata(request_id="req"),
)
assert isinstance(response, ResponsesResponse)
return response
@pytest.mark.asyncio
async def test_responses_per_request_metrics_follow_server_flag():
disabled_response = await _make_full_metrics_response(False)
assert disabled_response.metrics is None
assert "metrics" not in disabled_response.model_dump(mode="json")
enabled_response = await _make_full_metrics_response(True)
assert enabled_response.metrics is not None
assert enabled_response.metrics.time_to_first_token_ms == pytest.approx(500.0)
@pytest.mark.asyncio
async def test_responses_streaming_metrics_only_on_completed_event():
serving = _make_serving_instance(enable_per_request_metrics=True)
request = ResponsesRequest(input="hi", tools=[], stream=True, store=False)
context = _make_simple_context_with_output(
"hello", [10, 20], metrics=_PER_REQUEST_STATS
)
events = [
event
async for event in serving.responses_stream_generator(
request=request,
sampling_params=SamplingParams(max_tokens=16),
result_generator=_empty_context_generator(),
context=context,
model_name="test-model",
tokenizer=MagicMock(),
request_metadata=RequestResponseMetadata(request_id="req"),
)
]
for event in events[:-1]:
assert "metrics" not in event.response.model_dump(mode="json")
assert isinstance(events[-1], ResponseCompletedEvent)
assert events[-1].response.metrics is not None
assert events[-1].response.metrics.time_to_first_token_ms == pytest.approx(500.0)
@pytest.mark.asyncio
async def test_responses_metrics_suppressed_for_multiple_generation_turns():
response = await _make_full_metrics_response(
True, request_metrics_cover_all_generation_turns=False
)
assert response.metrics is None
assert "metrics" not in response.model_dump(mode="json")
def _make_serving_instance_with_reasoning():
"""Create an OpenAIServingResponses with a mocked reasoning parser."""
return _make_serving_instance(reasoning_parser="qwen3")
def _identity_increment(event):