[Frontend] Add per-request metrics to Responses API (#55084)
Signed-off-by: wxin <wxin@nvidia.com> Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
co-authored by
Benjamin Chislett
parent
0983aef8da
commit
fbf2c5e8be
@@ -48,6 +48,7 @@ from vllm.entrypoints.openai.responses.context import (
|
||||
SimpleContext,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import (
|
||||
ResponseCompletedEvent,
|
||||
ResponseCreatedEvent,
|
||||
ResponseRawMessageAndToken,
|
||||
ResponsesRequest,
|
||||
@@ -71,6 +72,7 @@ from vllm.renderers.online_renderer import (
|
||||
_extract_allowed_tools_from_mcp_requests,
|
||||
)
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.metrics.stats import RequestStateStats
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
@@ -795,6 +797,7 @@ class TestInitializeToolSessions:
|
||||
pass
|
||||
|
||||
assert serving_responses_instance.engine_client.generate.call_count == 2
|
||||
assert context.request_metrics_cover_all_generation_turns is False
|
||||
followup_engine_input = (
|
||||
serving_responses_instance.engine_client.generate.call_args_list[1].args[0]
|
||||
)
|
||||
@@ -1248,9 +1251,19 @@ class TestHarmonyPreambleStreaming:
|
||||
assert "response.output_item.done" in type_names
|
||||
|
||||
|
||||
def _make_simple_context_with_output(text, token_ids, response_parser=None):
|
||||
"""Create a SimpleContext with a RequestOutput containing the given text."""
|
||||
ctx = SimpleContext(response_parser=response_parser)
|
||||
_PER_REQUEST_STATS = RequestStateStats(
|
||||
queued_ts=1.0,
|
||||
scheduled_ts=1.5,
|
||||
first_token_ts=2.0,
|
||||
last_token_ts=3.0,
|
||||
)
|
||||
|
||||
|
||||
def _make_request_output(
|
||||
text,
|
||||
token_ids,
|
||||
metrics: RequestStateStats | None = None,
|
||||
):
|
||||
completion = CompletionOutput(
|
||||
index=0,
|
||||
text=text,
|
||||
@@ -1260,7 +1273,7 @@ def _make_simple_context_with_output(text, token_ids, response_parser=None):
|
||||
finish_reason=None,
|
||||
stop_reason=None,
|
||||
)
|
||||
req_output = RequestOutput(
|
||||
return RequestOutput(
|
||||
request_id="req",
|
||||
prompt="hi",
|
||||
prompt_token_ids=[7, 8],
|
||||
@@ -1268,13 +1281,29 @@ def _make_simple_context_with_output(text, token_ids, response_parser=None):
|
||||
outputs=[completion],
|
||||
finished=False,
|
||||
num_cached_tokens=0,
|
||||
metrics=metrics,
|
||||
)
|
||||
|
||||
|
||||
def _make_simple_context_with_output(
|
||||
text,
|
||||
token_ids,
|
||||
response_parser=None,
|
||||
metrics: RequestStateStats | None = None,
|
||||
):
|
||||
"""Create a SimpleContext with a RequestOutput containing the given text."""
|
||||
ctx = SimpleContext(response_parser=response_parser)
|
||||
req_output = _make_request_output(text, token_ids, metrics)
|
||||
ctx.append_output(req_output)
|
||||
ctx.request_metrics = metrics
|
||||
return ctx
|
||||
|
||||
|
||||
def _make_serving_instance_with_reasoning():
|
||||
"""Create an OpenAIServingResponses with a mocked reasoning parser."""
|
||||
def _make_serving_instance(
|
||||
*,
|
||||
reasoning_parser: str = "",
|
||||
enable_per_request_metrics: bool = False,
|
||||
) -> OpenAIServingResponses:
|
||||
engine_client = MagicMock()
|
||||
model_config = MagicMock()
|
||||
model_config.max_model_len = 100
|
||||
@@ -1285,18 +1314,112 @@ def _make_serving_instance_with_reasoning():
|
||||
engine_client.input_processor = MagicMock()
|
||||
engine_client.renderer = MagicMock()
|
||||
|
||||
models = MagicMock()
|
||||
|
||||
serving = OpenAIServingResponses(
|
||||
return OpenAIServingResponses(
|
||||
engine_client=engine_client,
|
||||
models=models,
|
||||
models=MagicMock(),
|
||||
online_renderer=MagicMock(),
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
reasoning_parser="qwen3",
|
||||
reasoning_parser=reasoning_parser,
|
||||
enable_per_request_metrics=enable_per_request_metrics,
|
||||
)
|
||||
return serving
|
||||
|
||||
|
||||
async def _empty_context_generator():
|
||||
if False:
|
||||
yield
|
||||
|
||||
|
||||
async def _make_full_metrics_response(
|
||||
enable_per_request_metrics: bool,
|
||||
request_metrics_cover_all_generation_turns: bool = True,
|
||||
):
|
||||
serving = _make_serving_instance(
|
||||
enable_per_request_metrics=enable_per_request_metrics
|
||||
)
|
||||
request = ResponsesRequest(input="hi", tools=[], stream=False, store=False)
|
||||
sampling_params = SamplingParams(max_tokens=16)
|
||||
context = SimpleContext()
|
||||
context.request_metrics_cover_all_generation_turns = (
|
||||
request_metrics_cover_all_generation_turns
|
||||
)
|
||||
|
||||
async def generate(*args, **kwargs):
|
||||
yield _make_request_output("hello", [10, 20], _PER_REQUEST_STATS)
|
||||
|
||||
serving.engine_client.generate.side_effect = generate
|
||||
result_generator = serving._generate_with_builtin_tools(
|
||||
request_id=request.request_id,
|
||||
engine_input=tokens_input([7, 8]),
|
||||
sampling_params=sampling_params,
|
||||
context=context,
|
||||
)
|
||||
response = await serving.responses_full_generator(
|
||||
request=request,
|
||||
sampling_params=sampling_params,
|
||||
result_generator=result_generator,
|
||||
context=context,
|
||||
model_name="test-model",
|
||||
tokenizer=MagicMock(),
|
||||
request_metadata=RequestResponseMetadata(request_id="req"),
|
||||
)
|
||||
assert isinstance(response, ResponsesResponse)
|
||||
return response
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_per_request_metrics_follow_server_flag():
|
||||
disabled_response = await _make_full_metrics_response(False)
|
||||
assert disabled_response.metrics is None
|
||||
assert "metrics" not in disabled_response.model_dump(mode="json")
|
||||
|
||||
enabled_response = await _make_full_metrics_response(True)
|
||||
assert enabled_response.metrics is not None
|
||||
assert enabled_response.metrics.time_to_first_token_ms == pytest.approx(500.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_streaming_metrics_only_on_completed_event():
|
||||
serving = _make_serving_instance(enable_per_request_metrics=True)
|
||||
request = ResponsesRequest(input="hi", tools=[], stream=True, store=False)
|
||||
context = _make_simple_context_with_output(
|
||||
"hello", [10, 20], metrics=_PER_REQUEST_STATS
|
||||
)
|
||||
|
||||
events = [
|
||||
event
|
||||
async for event in serving.responses_stream_generator(
|
||||
request=request,
|
||||
sampling_params=SamplingParams(max_tokens=16),
|
||||
result_generator=_empty_context_generator(),
|
||||
context=context,
|
||||
model_name="test-model",
|
||||
tokenizer=MagicMock(),
|
||||
request_metadata=RequestResponseMetadata(request_id="req"),
|
||||
)
|
||||
]
|
||||
|
||||
for event in events[:-1]:
|
||||
assert "metrics" not in event.response.model_dump(mode="json")
|
||||
assert isinstance(events[-1], ResponseCompletedEvent)
|
||||
assert events[-1].response.metrics is not None
|
||||
assert events[-1].response.metrics.time_to_first_token_ms == pytest.approx(500.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_metrics_suppressed_for_multiple_generation_turns():
|
||||
response = await _make_full_metrics_response(
|
||||
True, request_metrics_cover_all_generation_turns=False
|
||||
)
|
||||
|
||||
assert response.metrics is None
|
||||
assert "metrics" not in response.model_dump(mode="json")
|
||||
|
||||
|
||||
def _make_serving_instance_with_reasoning():
|
||||
"""Create an OpenAIServingResponses with a mocked reasoning parser."""
|
||||
return _make_serving_instance(reasoning_parser="qwen3")
|
||||
|
||||
|
||||
def _identity_increment(event):
|
||||
|
||||
Reference in New Issue
Block a user