[Frontend] Add per-request metrics to Responses API (#55084)
Signed-off-by: wxin <wxin@nvidia.com> Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
co-authored by
Benjamin Chislett
parent
0983aef8da
commit
fbf2c5e8be
@@ -120,6 +120,18 @@ the same `metrics` response field. As with `n > 1`, metrics are omitted for
|
|||||||
requests with multiple prompts, because the timing data cannot be attributed to
|
requests with multiple prompts, because the timing data cannot be attributed to
|
||||||
a single prompt's generation.
|
a single prompt's generation.
|
||||||
|
|
||||||
|
## Responses API
|
||||||
|
|
||||||
|
Per-request metrics are available on the `/v1/responses` endpoint using the
|
||||||
|
common `metrics` object described above. Non-streaming responses include it at
|
||||||
|
the top level. Streaming responses include it in the final response carried by
|
||||||
|
the `response.completed` event; intermediate events do not include metrics.
|
||||||
|
|
||||||
|
Metrics are omitted for Responses requests that perform multiple
|
||||||
|
model-generation turns, such as built-in tool-call workflows, because the
|
||||||
|
response retains timing data for only one generation turn while token usage is
|
||||||
|
accumulated across all turns.
|
||||||
|
|
||||||
## Relationship to Prometheus Metrics
|
## Relationship to Prometheus Metrics
|
||||||
|
|
||||||
The `metrics` response field provides per-request values for a single request.
|
The `metrics` response field provides per-request values for a single request.
|
||||||
|
|||||||
@@ -48,6 +48,7 @@ from vllm.entrypoints.openai.responses.context import (
|
|||||||
SimpleContext,
|
SimpleContext,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.responses.protocol import (
|
from vllm.entrypoints.openai.responses.protocol import (
|
||||||
|
ResponseCompletedEvent,
|
||||||
ResponseCreatedEvent,
|
ResponseCreatedEvent,
|
||||||
ResponseRawMessageAndToken,
|
ResponseRawMessageAndToken,
|
||||||
ResponsesRequest,
|
ResponsesRequest,
|
||||||
@@ -71,6 +72,7 @@ from vllm.renderers.online_renderer import (
|
|||||||
_extract_allowed_tools_from_mcp_requests,
|
_extract_allowed_tools_from_mcp_requests,
|
||||||
)
|
)
|
||||||
from vllm.sampling_params import SamplingParams
|
from vllm.sampling_params import SamplingParams
|
||||||
|
from vllm.v1.metrics.stats import RequestStateStats
|
||||||
|
|
||||||
pytestmark = pytest.mark.skip_global_cleanup
|
pytestmark = pytest.mark.skip_global_cleanup
|
||||||
|
|
||||||
@@ -795,6 +797,7 @@ class TestInitializeToolSessions:
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
assert serving_responses_instance.engine_client.generate.call_count == 2
|
assert serving_responses_instance.engine_client.generate.call_count == 2
|
||||||
|
assert context.request_metrics_cover_all_generation_turns is False
|
||||||
followup_engine_input = (
|
followup_engine_input = (
|
||||||
serving_responses_instance.engine_client.generate.call_args_list[1].args[0]
|
serving_responses_instance.engine_client.generate.call_args_list[1].args[0]
|
||||||
)
|
)
|
||||||
@@ -1248,9 +1251,19 @@ class TestHarmonyPreambleStreaming:
|
|||||||
assert "response.output_item.done" in type_names
|
assert "response.output_item.done" in type_names
|
||||||
|
|
||||||
|
|
||||||
def _make_simple_context_with_output(text, token_ids, response_parser=None):
|
_PER_REQUEST_STATS = RequestStateStats(
|
||||||
"""Create a SimpleContext with a RequestOutput containing the given text."""
|
queued_ts=1.0,
|
||||||
ctx = SimpleContext(response_parser=response_parser)
|
scheduled_ts=1.5,
|
||||||
|
first_token_ts=2.0,
|
||||||
|
last_token_ts=3.0,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_request_output(
|
||||||
|
text,
|
||||||
|
token_ids,
|
||||||
|
metrics: RequestStateStats | None = None,
|
||||||
|
):
|
||||||
completion = CompletionOutput(
|
completion = CompletionOutput(
|
||||||
index=0,
|
index=0,
|
||||||
text=text,
|
text=text,
|
||||||
@@ -1260,7 +1273,7 @@ def _make_simple_context_with_output(text, token_ids, response_parser=None):
|
|||||||
finish_reason=None,
|
finish_reason=None,
|
||||||
stop_reason=None,
|
stop_reason=None,
|
||||||
)
|
)
|
||||||
req_output = RequestOutput(
|
return RequestOutput(
|
||||||
request_id="req",
|
request_id="req",
|
||||||
prompt="hi",
|
prompt="hi",
|
||||||
prompt_token_ids=[7, 8],
|
prompt_token_ids=[7, 8],
|
||||||
@@ -1268,13 +1281,29 @@ def _make_simple_context_with_output(text, token_ids, response_parser=None):
|
|||||||
outputs=[completion],
|
outputs=[completion],
|
||||||
finished=False,
|
finished=False,
|
||||||
num_cached_tokens=0,
|
num_cached_tokens=0,
|
||||||
|
metrics=metrics,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_simple_context_with_output(
|
||||||
|
text,
|
||||||
|
token_ids,
|
||||||
|
response_parser=None,
|
||||||
|
metrics: RequestStateStats | None = None,
|
||||||
|
):
|
||||||
|
"""Create a SimpleContext with a RequestOutput containing the given text."""
|
||||||
|
ctx = SimpleContext(response_parser=response_parser)
|
||||||
|
req_output = _make_request_output(text, token_ids, metrics)
|
||||||
ctx.append_output(req_output)
|
ctx.append_output(req_output)
|
||||||
|
ctx.request_metrics = metrics
|
||||||
return ctx
|
return ctx
|
||||||
|
|
||||||
|
|
||||||
def _make_serving_instance_with_reasoning():
|
def _make_serving_instance(
|
||||||
"""Create an OpenAIServingResponses with a mocked reasoning parser."""
|
*,
|
||||||
|
reasoning_parser: str = "",
|
||||||
|
enable_per_request_metrics: bool = False,
|
||||||
|
) -> OpenAIServingResponses:
|
||||||
engine_client = MagicMock()
|
engine_client = MagicMock()
|
||||||
model_config = MagicMock()
|
model_config = MagicMock()
|
||||||
model_config.max_model_len = 100
|
model_config.max_model_len = 100
|
||||||
@@ -1285,18 +1314,112 @@ def _make_serving_instance_with_reasoning():
|
|||||||
engine_client.input_processor = MagicMock()
|
engine_client.input_processor = MagicMock()
|
||||||
engine_client.renderer = MagicMock()
|
engine_client.renderer = MagicMock()
|
||||||
|
|
||||||
models = MagicMock()
|
return OpenAIServingResponses(
|
||||||
|
|
||||||
serving = OpenAIServingResponses(
|
|
||||||
engine_client=engine_client,
|
engine_client=engine_client,
|
||||||
models=models,
|
models=MagicMock(),
|
||||||
online_renderer=MagicMock(),
|
online_renderer=MagicMock(),
|
||||||
request_logger=None,
|
request_logger=None,
|
||||||
chat_template=None,
|
chat_template=None,
|
||||||
chat_template_content_format="auto",
|
chat_template_content_format="auto",
|
||||||
reasoning_parser="qwen3",
|
reasoning_parser=reasoning_parser,
|
||||||
|
enable_per_request_metrics=enable_per_request_metrics,
|
||||||
)
|
)
|
||||||
return serving
|
|
||||||
|
|
||||||
|
async def _empty_context_generator():
|
||||||
|
if False:
|
||||||
|
yield
|
||||||
|
|
||||||
|
|
||||||
|
async def _make_full_metrics_response(
|
||||||
|
enable_per_request_metrics: bool,
|
||||||
|
request_metrics_cover_all_generation_turns: bool = True,
|
||||||
|
):
|
||||||
|
serving = _make_serving_instance(
|
||||||
|
enable_per_request_metrics=enable_per_request_metrics
|
||||||
|
)
|
||||||
|
request = ResponsesRequest(input="hi", tools=[], stream=False, store=False)
|
||||||
|
sampling_params = SamplingParams(max_tokens=16)
|
||||||
|
context = SimpleContext()
|
||||||
|
context.request_metrics_cover_all_generation_turns = (
|
||||||
|
request_metrics_cover_all_generation_turns
|
||||||
|
)
|
||||||
|
|
||||||
|
async def generate(*args, **kwargs):
|
||||||
|
yield _make_request_output("hello", [10, 20], _PER_REQUEST_STATS)
|
||||||
|
|
||||||
|
serving.engine_client.generate.side_effect = generate
|
||||||
|
result_generator = serving._generate_with_builtin_tools(
|
||||||
|
request_id=request.request_id,
|
||||||
|
engine_input=tokens_input([7, 8]),
|
||||||
|
sampling_params=sampling_params,
|
||||||
|
context=context,
|
||||||
|
)
|
||||||
|
response = await serving.responses_full_generator(
|
||||||
|
request=request,
|
||||||
|
sampling_params=sampling_params,
|
||||||
|
result_generator=result_generator,
|
||||||
|
context=context,
|
||||||
|
model_name="test-model",
|
||||||
|
tokenizer=MagicMock(),
|
||||||
|
request_metadata=RequestResponseMetadata(request_id="req"),
|
||||||
|
)
|
||||||
|
assert isinstance(response, ResponsesResponse)
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_responses_per_request_metrics_follow_server_flag():
|
||||||
|
disabled_response = await _make_full_metrics_response(False)
|
||||||
|
assert disabled_response.metrics is None
|
||||||
|
assert "metrics" not in disabled_response.model_dump(mode="json")
|
||||||
|
|
||||||
|
enabled_response = await _make_full_metrics_response(True)
|
||||||
|
assert enabled_response.metrics is not None
|
||||||
|
assert enabled_response.metrics.time_to_first_token_ms == pytest.approx(500.0)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_responses_streaming_metrics_only_on_completed_event():
|
||||||
|
serving = _make_serving_instance(enable_per_request_metrics=True)
|
||||||
|
request = ResponsesRequest(input="hi", tools=[], stream=True, store=False)
|
||||||
|
context = _make_simple_context_with_output(
|
||||||
|
"hello", [10, 20], metrics=_PER_REQUEST_STATS
|
||||||
|
)
|
||||||
|
|
||||||
|
events = [
|
||||||
|
event
|
||||||
|
async for event in serving.responses_stream_generator(
|
||||||
|
request=request,
|
||||||
|
sampling_params=SamplingParams(max_tokens=16),
|
||||||
|
result_generator=_empty_context_generator(),
|
||||||
|
context=context,
|
||||||
|
model_name="test-model",
|
||||||
|
tokenizer=MagicMock(),
|
||||||
|
request_metadata=RequestResponseMetadata(request_id="req"),
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
for event in events[:-1]:
|
||||||
|
assert "metrics" not in event.response.model_dump(mode="json")
|
||||||
|
assert isinstance(events[-1], ResponseCompletedEvent)
|
||||||
|
assert events[-1].response.metrics is not None
|
||||||
|
assert events[-1].response.metrics.time_to_first_token_ms == pytest.approx(500.0)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_responses_metrics_suppressed_for_multiple_generation_turns():
|
||||||
|
response = await _make_full_metrics_response(
|
||||||
|
True, request_metrics_cover_all_generation_turns=False
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.metrics is None
|
||||||
|
assert "metrics" not in response.model_dump(mode="json")
|
||||||
|
|
||||||
|
|
||||||
|
def _make_serving_instance_with_reasoning():
|
||||||
|
"""Create an OpenAIServingResponses with a mocked reasoning parser."""
|
||||||
|
return _make_serving_instance(reasoning_parser="qwen3")
|
||||||
|
|
||||||
|
|
||||||
def _identity_increment(event):
|
def _identity_increment(event):
|
||||||
|
|||||||
@@ -116,6 +116,7 @@ async def init_generate_state(
|
|||||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||||
enable_force_include_usage=args.enable_force_include_usage,
|
enable_force_include_usage=args.enable_force_include_usage,
|
||||||
|
enable_per_request_metrics=args.enable_per_request_metrics,
|
||||||
enable_log_outputs=args.enable_log_outputs,
|
enable_log_outputs=args.enable_log_outputs,
|
||||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -45,6 +45,8 @@ from vllm.utils import random_uuid
|
|||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from mcp.client import ClientSession
|
from mcp.client import ClientSession
|
||||||
|
|
||||||
|
from vllm.v1.metrics.stats import RequestStateStats
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
# This is currently needed as the tool type doesn't 1:1 match the
|
# This is currently needed as the tool type doesn't 1:1 match the
|
||||||
@@ -102,6 +104,11 @@ class TurnMetrics:
|
|||||||
|
|
||||||
class ConversationContext(ABC):
|
class ConversationContext(ABC):
|
||||||
response_parser: Parser | None = None
|
response_parser: Parser | None = None
|
||||||
|
request_metrics: "RequestStateStats | None" = None
|
||||||
|
# Built-in tools can trigger additional model generations. In that case,
|
||||||
|
# the stored engine timestamps cover only one turn, while token usage is
|
||||||
|
# accumulated across all turns.
|
||||||
|
request_metrics_cover_all_generation_turns: bool = True
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def append_output(self, output: RequestOutput) -> None:
|
def append_output(self, output: RequestOutput) -> None:
|
||||||
|
|||||||
@@ -60,7 +60,11 @@ from vllm.entrypoints.chat_utils import (
|
|||||||
ChatCompletionMessageParam,
|
ChatCompletionMessageParam,
|
||||||
ChatTemplateContentFormatOption,
|
ChatTemplateContentFormatOption,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.generate.base.protocol import StopParam, validate_cache_salt
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
|
PerRequestMetrics,
|
||||||
|
StopParam,
|
||||||
|
validate_cache_salt,
|
||||||
|
)
|
||||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
|
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
|
||||||
from vllm.exceptions import VLLMValidationError
|
from vllm.exceptions import VLLMValidationError
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
@@ -684,6 +688,11 @@ class ResponsesResponse(OpenAIBaseModel):
|
|||||||
usage: ResponseUsage | None = None
|
usage: ResponseUsage | None = None
|
||||||
user: str | None = None
|
user: str | None = None
|
||||||
|
|
||||||
|
# vLLM-specific per-request metrics. Omitted unless enabled server-side.
|
||||||
|
metrics: PerRequestMetrics | None = Field(
|
||||||
|
default=None, exclude_if=lambda value: value is None
|
||||||
|
)
|
||||||
|
|
||||||
presence_penalty: float | None = Field(
|
presence_penalty: float | None = Field(
|
||||||
default=None,
|
default=None,
|
||||||
ge=-2.0,
|
ge=-2.0,
|
||||||
@@ -752,6 +761,7 @@ class ResponsesResponse(OpenAIBaseModel):
|
|||||||
output: list[ResponseOutputItem],
|
output: list[ResponseOutputItem],
|
||||||
status: ResponseStatus,
|
status: ResponseStatus,
|
||||||
usage: ResponseUsage | None = None,
|
usage: ResponseUsage | None = None,
|
||||||
|
metrics: PerRequestMetrics | None = None,
|
||||||
input_messages: ResponseInputOutputMessage | None = None,
|
input_messages: ResponseInputOutputMessage | None = None,
|
||||||
output_messages: ResponseInputOutputMessage | None = None,
|
output_messages: ResponseInputOutputMessage | None = None,
|
||||||
kv_transfer_params: dict[str, Any] | None = None,
|
kv_transfer_params: dict[str, Any] | None = None,
|
||||||
@@ -795,6 +805,7 @@ class ResponsesResponse(OpenAIBaseModel):
|
|||||||
truncation=request.truncation,
|
truncation=request.truncation,
|
||||||
user=request.user,
|
user=request.user,
|
||||||
usage=usage,
|
usage=usage,
|
||||||
|
metrics=metrics,
|
||||||
kv_transfer_params=kv_transfer_params,
|
kv_transfer_params=kv_transfer_params,
|
||||||
ec_transfer_params=ec_transfer_params,
|
ec_transfer_params=ec_transfer_params,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -30,7 +30,10 @@ from vllm.entrypoints.generate.base.protocol import (
|
|||||||
DeltaMessage,
|
DeltaMessage,
|
||||||
RequestResponseMetadata,
|
RequestResponseMetadata,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.generate.base.serving import GenerateBaseServing
|
from vllm.entrypoints.generate.base.serving import (
|
||||||
|
GenerateBaseServing,
|
||||||
|
build_per_request_timing_metrics,
|
||||||
|
)
|
||||||
from vllm.entrypoints.mcp.tool_server import ToolServer
|
from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
from vllm.entrypoints.openai.responses.context import (
|
from vllm.entrypoints.openai.responses.context import (
|
||||||
@@ -109,6 +112,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
tool_server: ToolServer | None = None,
|
tool_server: ToolServer | None = None,
|
||||||
enable_prompt_tokens_details: bool = False,
|
enable_prompt_tokens_details: bool = False,
|
||||||
enable_force_include_usage: bool = False,
|
enable_force_include_usage: bool = False,
|
||||||
|
enable_per_request_metrics: bool = False,
|
||||||
enable_log_outputs: bool = False,
|
enable_log_outputs: bool = False,
|
||||||
default_chat_template_kwargs: dict[str, Any] | None = None,
|
default_chat_template_kwargs: dict[str, Any] | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
@@ -136,6 +140,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
)
|
)
|
||||||
self.enable_prompt_tokens_details = enable_prompt_tokens_details
|
self.enable_prompt_tokens_details = enable_prompt_tokens_details
|
||||||
self.enable_force_include_usage = enable_force_include_usage
|
self.enable_force_include_usage = enable_force_include_usage
|
||||||
|
self.enable_per_request_metrics = enable_per_request_metrics
|
||||||
|
|
||||||
self.default_sampling_params = self.model_config.get_diff_sampling_param()
|
self.default_sampling_params = self.model_config.get_diff_sampling_param()
|
||||||
mc = self.model_config
|
mc = self.model_config
|
||||||
@@ -645,6 +650,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
)
|
)
|
||||||
|
|
||||||
async for res in generator:
|
async for res in generator:
|
||||||
|
context.request_metrics = res.metrics
|
||||||
context.append_output(res)
|
context.append_output(res)
|
||||||
# NOTE(woosuk): The stop condition is handled by the engine.
|
# NOTE(woosuk): The stop condition is handled by the engine.
|
||||||
yield context
|
yield context
|
||||||
@@ -654,6 +660,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
break
|
break
|
||||||
|
|
||||||
# Call the tool and update the context with the result.
|
# Call the tool and update the context with the result.
|
||||||
|
context.request_metrics_cover_all_generation_turns = False
|
||||||
tool_output = await context.call_tool()
|
tool_output = await context.call_tool()
|
||||||
context.append_tool_output(tool_output)
|
context.append_tool_output(tool_output)
|
||||||
|
|
||||||
@@ -857,6 +864,14 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
],
|
],
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
per_request_metrics = None
|
||||||
|
if (
|
||||||
|
self.enable_per_request_metrics
|
||||||
|
and context.request_metrics_cover_all_generation_turns
|
||||||
|
):
|
||||||
|
per_request_metrics = build_per_request_timing_metrics(
|
||||||
|
context.request_metrics, num_generated_tokens
|
||||||
|
)
|
||||||
response = ResponsesResponse.from_request(
|
response = ResponsesResponse.from_request(
|
||||||
request,
|
request,
|
||||||
sampling_params,
|
sampling_params,
|
||||||
@@ -867,6 +882,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
output=output,
|
output=output,
|
||||||
status=status,
|
status=status,
|
||||||
usage=usage,
|
usage=usage,
|
||||||
|
metrics=per_request_metrics,
|
||||||
kv_transfer_params=context.kv_transfer_params,
|
kv_transfer_params=context.kv_transfer_params,
|
||||||
ec_transfer_params=context.ec_transfer_params,
|
ec_transfer_params=context.ec_transfer_params,
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user