[Frontend] Add per-request metrics to Responses API (#55084)
Signed-off-by: wxin <wxin@nvidia.com> Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
co-authored by
Benjamin Chislett
parent
0983aef8da
commit
fbf2c5e8be
@@ -116,6 +116,7 @@ async def init_generate_state(
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
enable_force_include_usage=args.enable_force_include_usage,
|
||||
enable_per_request_metrics=args.enable_per_request_metrics,
|
||||
enable_log_outputs=args.enable_log_outputs,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
)
|
||||
|
||||
@@ -45,6 +45,8 @@ from vllm.utils import random_uuid
|
||||
if TYPE_CHECKING:
|
||||
from mcp.client import ClientSession
|
||||
|
||||
from vllm.v1.metrics.stats import RequestStateStats
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# This is currently needed as the tool type doesn't 1:1 match the
|
||||
@@ -102,6 +104,11 @@ class TurnMetrics:
|
||||
|
||||
class ConversationContext(ABC):
|
||||
response_parser: Parser | None = None
|
||||
request_metrics: "RequestStateStats | None" = None
|
||||
# Built-in tools can trigger additional model generations. In that case,
|
||||
# the stored engine timestamps cover only one turn, while token usage is
|
||||
# accumulated across all turns.
|
||||
request_metrics_cover_all_generation_turns: bool = True
|
||||
|
||||
@abstractmethod
|
||||
def append_output(self, output: RequestOutput) -> None:
|
||||
|
||||
@@ -60,7 +60,11 @@ from vllm.entrypoints.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
ChatTemplateContentFormatOption,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.protocol import StopParam, validate_cache_salt
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
PerRequestMetrics,
|
||||
StopParam,
|
||||
validate_cache_salt,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
@@ -684,6 +688,11 @@ class ResponsesResponse(OpenAIBaseModel):
|
||||
usage: ResponseUsage | None = None
|
||||
user: str | None = None
|
||||
|
||||
# vLLM-specific per-request metrics. Omitted unless enabled server-side.
|
||||
metrics: PerRequestMetrics | None = Field(
|
||||
default=None, exclude_if=lambda value: value is None
|
||||
)
|
||||
|
||||
presence_penalty: float | None = Field(
|
||||
default=None,
|
||||
ge=-2.0,
|
||||
@@ -752,6 +761,7 @@ class ResponsesResponse(OpenAIBaseModel):
|
||||
output: list[ResponseOutputItem],
|
||||
status: ResponseStatus,
|
||||
usage: ResponseUsage | None = None,
|
||||
metrics: PerRequestMetrics | None = None,
|
||||
input_messages: ResponseInputOutputMessage | None = None,
|
||||
output_messages: ResponseInputOutputMessage | None = None,
|
||||
kv_transfer_params: dict[str, Any] | None = None,
|
||||
@@ -795,6 +805,7 @@ class ResponsesResponse(OpenAIBaseModel):
|
||||
truncation=request.truncation,
|
||||
user=request.user,
|
||||
usage=usage,
|
||||
metrics=metrics,
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
ec_transfer_params=ec_transfer_params,
|
||||
)
|
||||
|
||||
@@ -30,7 +30,10 @@ from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaMessage,
|
||||
RequestResponseMetadata,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.serving import GenerateBaseServing
|
||||
from vllm.entrypoints.generate.base.serving import (
|
||||
GenerateBaseServing,
|
||||
build_per_request_timing_metrics,
|
||||
)
|
||||
from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.openai.responses.context import (
|
||||
@@ -109,6 +112,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
tool_server: ToolServer | None = None,
|
||||
enable_prompt_tokens_details: bool = False,
|
||||
enable_force_include_usage: bool = False,
|
||||
enable_per_request_metrics: bool = False,
|
||||
enable_log_outputs: bool = False,
|
||||
default_chat_template_kwargs: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
@@ -136,6 +140,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
)
|
||||
self.enable_prompt_tokens_details = enable_prompt_tokens_details
|
||||
self.enable_force_include_usage = enable_force_include_usage
|
||||
self.enable_per_request_metrics = enable_per_request_metrics
|
||||
|
||||
self.default_sampling_params = self.model_config.get_diff_sampling_param()
|
||||
mc = self.model_config
|
||||
@@ -645,6 +650,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
)
|
||||
|
||||
async for res in generator:
|
||||
context.request_metrics = res.metrics
|
||||
context.append_output(res)
|
||||
# NOTE(woosuk): The stop condition is handled by the engine.
|
||||
yield context
|
||||
@@ -654,6 +660,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
break
|
||||
|
||||
# Call the tool and update the context with the result.
|
||||
context.request_metrics_cover_all_generation_turns = False
|
||||
tool_output = await context.call_tool()
|
||||
context.append_tool_output(tool_output)
|
||||
|
||||
@@ -857,6 +864,14 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
],
|
||||
),
|
||||
)
|
||||
per_request_metrics = None
|
||||
if (
|
||||
self.enable_per_request_metrics
|
||||
and context.request_metrics_cover_all_generation_turns
|
||||
):
|
||||
per_request_metrics = build_per_request_timing_metrics(
|
||||
context.request_metrics, num_generated_tokens
|
||||
)
|
||||
response = ResponsesResponse.from_request(
|
||||
request,
|
||||
sampling_params,
|
||||
@@ -867,6 +882,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
output=output,
|
||||
status=status,
|
||||
usage=usage,
|
||||
metrics=per_request_metrics,
|
||||
kv_transfer_params=context.kv_transfer_params,
|
||||
ec_transfer_params=context.ec_transfer_params,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user