[Frontend] Add per-request metrics to Responses API (#55084)

Signed-off-by: wxin <wxin@nvidia.com>
Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
xinnywinne
2026-09-16 22:52:05 +00:00
committed by GitHub
co-authored by Benjamin Chislett
parent 0983aef8da
commit fbf2c5e8be
6 changed files with 184 additions and 14 deletions
+1
View File
@@ -116,6 +116,7 @@ async def init_generate_state(
reasoning_parser=args.structured_outputs_config.reasoning_parser,
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
enable_force_include_usage=args.enable_force_include_usage,
enable_per_request_metrics=args.enable_per_request_metrics,
enable_log_outputs=args.enable_log_outputs,
default_chat_template_kwargs=default_chat_template_kwargs,
)
@@ -45,6 +45,8 @@ from vllm.utils import random_uuid
if TYPE_CHECKING:
from mcp.client import ClientSession
from vllm.v1.metrics.stats import RequestStateStats
logger = logging.getLogger(__name__)
# This is currently needed as the tool type doesn't 1:1 match the
@@ -102,6 +104,11 @@ class TurnMetrics:
class ConversationContext(ABC):
response_parser: Parser | None = None
request_metrics: "RequestStateStats | None" = None
# Built-in tools can trigger additional model generations. In that case,
# the stored engine timestamps cover only one turn, while token usage is
# accumulated across all turns.
request_metrics_cover_all_generation_turns: bool = True
@abstractmethod
def append_output(self, output: RequestOutput) -> None:
+12 -1
View File
@@ -60,7 +60,11 @@ from vllm.entrypoints.chat_utils import (
ChatCompletionMessageParam,
ChatTemplateContentFormatOption,
)
from vllm.entrypoints.generate.base.protocol import StopParam, validate_cache_salt
from vllm.entrypoints.generate.base.protocol import (
PerRequestMetrics,
StopParam,
validate_cache_salt,
)
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
@@ -684,6 +688,11 @@ class ResponsesResponse(OpenAIBaseModel):
usage: ResponseUsage | None = None
user: str | None = None
# vLLM-specific per-request metrics. Omitted unless enabled server-side.
metrics: PerRequestMetrics | None = Field(
default=None, exclude_if=lambda value: value is None
)
presence_penalty: float | None = Field(
default=None,
ge=-2.0,
@@ -752,6 +761,7 @@ class ResponsesResponse(OpenAIBaseModel):
output: list[ResponseOutputItem],
status: ResponseStatus,
usage: ResponseUsage | None = None,
metrics: PerRequestMetrics | None = None,
input_messages: ResponseInputOutputMessage | None = None,
output_messages: ResponseInputOutputMessage | None = None,
kv_transfer_params: dict[str, Any] | None = None,
@@ -795,6 +805,7 @@ class ResponsesResponse(OpenAIBaseModel):
truncation=request.truncation,
user=request.user,
usage=usage,
metrics=metrics,
kv_transfer_params=kv_transfer_params,
ec_transfer_params=ec_transfer_params,
)
+17 -1
View File
@@ -30,7 +30,10 @@ from vllm.entrypoints.generate.base.protocol import (
DeltaMessage,
RequestResponseMetadata,
)
from vllm.entrypoints.generate.base.serving import GenerateBaseServing
from vllm.entrypoints.generate.base.serving import (
GenerateBaseServing,
build_per_request_timing_metrics,
)
from vllm.entrypoints.mcp.tool_server import ToolServer
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.openai.responses.context import (
@@ -109,6 +112,7 @@ class OpenAIServingResponses(GenerateBaseServing):
tool_server: ToolServer | None = None,
enable_prompt_tokens_details: bool = False,
enable_force_include_usage: bool = False,
enable_per_request_metrics: bool = False,
enable_log_outputs: bool = False,
default_chat_template_kwargs: dict[str, Any] | None = None,
) -> None:
@@ -136,6 +140,7 @@ class OpenAIServingResponses(GenerateBaseServing):
)
self.enable_prompt_tokens_details = enable_prompt_tokens_details
self.enable_force_include_usage = enable_force_include_usage
self.enable_per_request_metrics = enable_per_request_metrics
self.default_sampling_params = self.model_config.get_diff_sampling_param()
mc = self.model_config
@@ -645,6 +650,7 @@ class OpenAIServingResponses(GenerateBaseServing):
)
async for res in generator:
context.request_metrics = res.metrics
context.append_output(res)
# NOTE(woosuk): The stop condition is handled by the engine.
yield context
@@ -654,6 +660,7 @@ class OpenAIServingResponses(GenerateBaseServing):
break
# Call the tool and update the context with the result.
context.request_metrics_cover_all_generation_turns = False
tool_output = await context.call_tool()
context.append_tool_output(tool_output)
@@ -857,6 +864,14 @@ class OpenAIServingResponses(GenerateBaseServing):
],
),
)
per_request_metrics = None
if (
self.enable_per_request_metrics
and context.request_metrics_cover_all_generation_turns
):
per_request_metrics = build_per_request_timing_metrics(
context.request_metrics, num_generated_tokens
)
response = ResponsesResponse.from_request(
request,
sampling_params,
@@ -867,6 +882,7 @@ class OpenAIServingResponses(GenerateBaseServing):
output=output,
status=status,
usage=usage,
metrics=per_request_metrics,
kv_transfer_params=context.kv_transfer_params,
ec_transfer_params=context.ec_transfer_params,
)