From 86db6c307073217599a993f1a801e3c204f2eb17 Mon Sep 17 00:00:00 2001 From: nv-nedelman-1 <49536618+nv-nedelman-1@users.noreply.github.com> Date: Mon, 6 Jul 2026 19:48:29 -0500 Subject: [PATCH] [Frontend] add per-request timing `metrics` field to response body of Chat/Completions APIs (#46768) Signed-off-by: Nicholas Edelman Signed-off-by: Simon Mo Co-authored-by: Simon Mo Co-authored-by: GPT-5.5 --- docs/features/per_request_metrics.md | 127 +++++++++++++ .../chat_completion/test_serving_chat.py | 173 ++++++++++++++++++ .../completion/test_completion_error.py | 98 +++++++++- tests/entrypoints/openai/test_cli_args.py | 8 + vllm/entrypoints/generate/api_router.py | 2 + vllm/entrypoints/generate/base/serving.py | 57 ++++++ .../openai/chat_completion/protocol.py | 3 + .../openai/chat_completion/serving.py | 35 ++++ vllm/entrypoints/openai/cli_args.py | 13 +- .../entrypoints/openai/completion/protocol.py | 3 + vllm/entrypoints/openai/completion/serving.py | 42 +++++ vllm/entrypoints/openai/engine/protocol.py | 8 + 12 files changed, 565 insertions(+), 4 deletions(-) create mode 100644 docs/features/per_request_metrics.md diff --git a/docs/features/per_request_metrics.md b/docs/features/per_request_metrics.md new file mode 100644 index 0000000000..9bc64d2b86 --- /dev/null +++ b/docs/features/per_request_metrics.md @@ -0,0 +1,127 @@ +# Per-Request Metrics + +vLLM can return per-request timing metrics directly in API responses. +This is useful for billing, SLA monitoring, and latency analysis at the +individual request level, as a complement to the server-aggregated Prometheus +metrics exposed at `/metrics`. + +## Enabling + +Start the server with `--enable-per-request-metrics`: + +```bash +vllm serve meta-llama/Llama-3.1-8B-Instruct --enable-per-request-metrics +``` + +When this flag is set, supported API responses include metrics for each +attributable request. + +!!! note + At high concurrency, enabling per-request metrics computation may introduce + non-negligible CPU overhead. Benchmark your specific workload to evaluate the + impact before enabling in production. + +## Response Format + +When per-request metrics are enabled, the response includes a `metrics` object: + +```json +{ + "id": "chatcmpl-abc123", + "object": "chat.completion", + "model": "meta-llama/Llama-3.1-8B-Instruct", + "choices": [ ... ], + "usage": { + "prompt_tokens": 42, + "completion_tokens": 128, + "total_tokens": 170 + }, + "metrics": { + "time_to_first_token_ms": 85.2, + "generation_time_ms": 1240.5, + "queue_time_ms": 12.3, + "mean_itl_ms": 9.1, + "tokens_per_second": 103.2 + } +} +``` + +| Field | Description | +| --- | --- | +| `time_to_first_token_ms` | Time from when the request was scheduled until the first output token was generated (TTFT). | +| `generation_time_ms` | Decode time: time from the first output token to the last output token. Excludes both queue wait and prefill/TTFT. | +| `queue_time_ms` | Time the request spent waiting in the scheduler queue before processing began. | +| `mean_itl_ms` | Mean inter-token latency (average time between successive output tokens) during the decode phase. `null` for single-token responses. | +| `tokens_per_second` | Overall output token throughput: all generated tokens over the inference interval (scheduling to last output token). Unlike `generation_time_ms`, this includes the prefill phase, so it reflects end-to-end generation speed rather than pure decode speed. | + +All fields are `null` if the underlying timing data is not available for that +request. + +!!! note + Timing metrics describe a single generation stream, so they are only + returned when the request maps to exactly one. They are suppressed (the + `metrics` object is `null`) for requests with `n > 1`, because the + underlying timing data reflects only one of the `n` sequences and cannot be + accurately attributed to the request as a whole. Token usage + (`prompt_tokens`, `completion_tokens`) remains accurate in these cases. + Per-request metrics also require server-side statistics logging, which is + on by default. vLLM rejects `--enable-per-request-metrics` when + `--disable-log-stats` is also set. + +## Example Request + +=== "Non-streaming" + + ```python + from openai import OpenAI + + client = OpenAI(base_url="http://localhost:8000/v1", api_key="token") + + response = client.chat.completions.create( + model="meta-llama/Llama-3.1-8B-Instruct", + messages=[{"role": "user", "content": "What is the capital of France?"}], + ) + + print(response.usage) + print(response.model_extra.get("metrics")) + ``` + +=== "Streaming" + + In streaming responses, metrics are attached to the final usage chunk (the + chunk sent after all content chunks). That chunk is only emitted when usage + reporting is enabled with `stream_options.include_usage: true` or forced + server-side with `--enable-force-include-usage`. Without forced usage, a + streaming client must set `stream_options.include_usage: true` to receive + metrics. + + ```python + from openai import OpenAI + + client = OpenAI(base_url="http://localhost:8000/v1", api_key="token") + + stream = client.chat.completions.create( + model="meta-llama/Llama-3.1-8B-Instruct", + messages=[{"role": "user", "content": "What is the capital of France?"}], + stream=True, + stream_options={"include_usage": True}, + ) + + for chunk in stream: + if chunk.usage: + print("Usage:", chunk.usage) + print("Metrics:", chunk.model_extra.get("metrics")) + ``` + +## Completions API + +Per-request metrics are also available on the `/v1/completions` endpoint using +the same `metrics` response field. As with `n > 1`, metrics are omitted for +requests with multiple prompts, because the timing data cannot be attributed to +a single prompt's generation. + +## Relationship to Prometheus Metrics + +The `metrics` response field provides per-request values for a single request. +The `/metrics` Prometheus endpoint exposes server-level histograms (e.g. +`vllm:time_to_first_token_seconds`) that aggregate across all requests. diff --git a/tests/entrypoints/openai/chat_completion/test_serving_chat.py b/tests/entrypoints/openai/chat_completion/test_serving_chat.py index 7b480bdba6..b43ac31d3d 100644 --- a/tests/entrypoints/openai/chat_completion/test_serving_chat.py +++ b/tests/entrypoints/openai/chat_completion/test_serving_chat.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import asyncio import json +from collections.abc import AsyncIterator from contextlib import suppress from dataclasses import dataclass, field from typing import Any @@ -19,6 +20,7 @@ from tests.entrypoints.openai.utils import ( from tests.utils import RemoteOpenAIServer from vllm._aiter_ops import is_aiter_found_and_supported from vllm.config import MultiModalConfig +from vllm.entrypoints.generate.base.serving import build_per_request_timing_metrics from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionRequest, ChatCompletionResponse, @@ -50,9 +52,17 @@ from vllm.tokenizers import get_tokenizer from vllm.tokenizers.mistral import MistralTokenizer from vllm.tokenizers.registry import cached_tokenizer_from_config from vllm.v1.engine.async_llm import AsyncLLM +from vllm.v1.metrics.stats import RequestStateStats GPT_OSS_MODEL_NAME = "openai/gpt-oss-20b" GPT_OSS_SPECULATOR_NAME = "RedHatAI/gpt-oss-20b-speculator.eagle3" +_PER_REQUEST_STATS = RequestStateStats( + queued_ts=1.0, + scheduled_ts=1.5, + first_token_ts=2.0, + last_token_ts=3.0, + num_generation_tokens=2, +) @pytest.fixture(scope="module") @@ -608,6 +618,169 @@ def _build_serving_chat( return serving_chat +def _build_minimal_metrics_serving_chat( + enable_per_request_metrics: bool, + enable_force_include_usage: bool = False, +) -> OpenAIServingChat: + serving = OpenAIServingChat.__new__(OpenAIServingChat) + serving.response_role = "assistant" + serving.parser_cls = None + serving.enable_auto_tools = False + serving.enable_prompt_tokens_details = False + serving.enable_log_outputs = False + serving.enable_log_deltas = False + serving.enable_force_include_usage = enable_force_include_usage + serving.request_logger = None + serving.system_fingerprint = None + serving.enable_per_request_metrics = enable_per_request_metrics + return serving + + +def _make_metrics_request_output( + metrics: RequestStateStats | None = _PER_REQUEST_STATS, + token_ids: tuple[int, ...] = (100, 101), +) -> RequestOutput: + return RequestOutput( + request_id="test-id", + prompt="Test prompt", + prompt_token_ids=[1, 2, 3], + prompt_logprobs=None, + outputs=[ + CompletionOutput( + index=0, + text="Hello", + token_ids=list(token_ids), + cumulative_logprob=None, + logprobs=None, + finish_reason="stop", + ) + ], + finished=True, + metrics=metrics, + ) + + +async def _single_request_output( + request_output: RequestOutput, +) -> AsyncIterator[RequestOutput]: + yield request_output + + +async def _collect_metrics_stream_chunks( + serving: OpenAIServingChat, + request: ChatCompletionRequest, +) -> list[dict[str, Any]]: + chunks: list[dict[str, Any]] = [] + async for line in serving.chat_completion_stream_generator( + request, + _single_request_output(_make_metrics_request_output()), + "chatcmpl-test-id", + "test-model", + conversation=[{"role": "user", "content": "Test"}], + tokenizer=MagicMock(), + request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"), + ): + line = line.strip() + if not line.startswith("data: "): + continue + payload = line[len("data: ") :] + if payload != "[DONE]": + chunks.append(json.loads(payload)) + return chunks + + +def test_build_per_request_timing_metrics_valid_timestamps(): + metrics = build_per_request_timing_metrics( + _PER_REQUEST_STATS, num_generation_tokens=10 + ) + + assert metrics.time_to_first_token_ms == pytest.approx(500.0) + assert metrics.generation_time_ms == pytest.approx(1000.0) + assert metrics.queue_time_ms == pytest.approx(500.0) + assert metrics.mean_itl_ms == pytest.approx(1000.0 / 9, rel=1e-4) + assert metrics.tokens_per_second == pytest.approx(10.0 / 1.5, rel=1e-4) + + +@pytest.mark.asyncio +async def test_chat_per_request_metrics_follow_server_flag(): + request = ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "Test prompt"}], + max_tokens=10, + stream=False, + ) + request_output = _make_metrics_request_output() + + disabled_serving = _build_minimal_metrics_serving_chat( + enable_per_request_metrics=False + ) + disabled_response = await disabled_serving.chat_completion_full_generator( + request, + _single_request_output(request_output), + "chatcmpl-test-id", + "test-model", + conversation=[{"role": "user", "content": "Test"}], + tokenizer=MagicMock(), + request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"), + ) + assert disabled_response.metrics is None + + enabled_serving = _build_minimal_metrics_serving_chat( + enable_per_request_metrics=True + ) + enabled_response = await enabled_serving.chat_completion_full_generator( + request, + _single_request_output(request_output), + "chatcmpl-test-id", + "test-model", + conversation=[{"role": "user", "content": "Test"}], + tokenizer=MagicMock(), + request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"), + ) + assert enabled_response.metrics is not None + assert enabled_response.metrics.time_to_first_token_ms == pytest.approx(500.0) + + +@pytest.mark.asyncio +async def test_chat_per_request_metrics_suppressed_for_n_greater_than_one(): + serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=True) + response = await serving.chat_completion_full_generator( + ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "Test prompt"}], + max_tokens=10, + stream=False, + n=2, + ), + _single_request_output(_make_metrics_request_output()), + "chatcmpl-test-id", + "test-model", + conversation=[{"role": "user", "content": "Test"}], + tokenizer=MagicMock(), + request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"), + ) + assert response.metrics is None + + +@pytest.mark.asyncio +async def test_chat_streaming_metrics_ride_on_usage_chunk(): + serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=True) + chunks = await _collect_metrics_stream_chunks( + serving, + ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "Test prompt"}], + max_tokens=10, + stream=True, + stream_options={"include_usage": True}, + ), + ) + + usage_chunks = [chunk for chunk in chunks if chunk.get("usage")] + assert usage_chunks + assert usage_chunks[-1]["metrics"]["time_to_first_token_ms"] == pytest.approx(500.0) + + @dataclass class MockEngine: model_config: MockModelConfig = field(default_factory=MockModelConfig) diff --git a/tests/entrypoints/openai/completion/test_completion_error.py b/tests/entrypoints/openai/completion/test_completion_error.py index 062c3e7583..5906063c54 100644 --- a/tests/entrypoints/openai/completion/test_completion_error.py +++ b/tests/entrypoints/openai/completion/test_completion_error.py @@ -11,7 +11,10 @@ from pydantic import ValidationError from vllm.config.multimodal import MultiModalConfig from vllm.entrypoints.openai.completion.protocol import CompletionRequest from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion -from vllm.entrypoints.openai.engine.protocol import GenerationError +from vllm.entrypoints.openai.engine.protocol import ( + GenerationError, + RequestResponseMetadata, +) from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.scale_out.render.serving import ServingRender @@ -20,9 +23,17 @@ from vllm.renderers.hf import HfRenderer from vllm.renderers.online_renderer import OnlineRenderer from vllm.tokenizers.registry import cached_tokenizer_from_config from vllm.v1.engine.async_llm import AsyncLLM +from vllm.v1.metrics.stats import RequestStateStats MODEL_NAME = "openai-community/gpt2" MODEL_NAME_SHORT = "gpt2" +_PER_REQUEST_STATS = RequestStateStats( + queued_ts=1.0, + scheduled_ts=1.5, + first_token_ts=2.0, + last_token_ts=3.0, + num_generation_tokens=2, +) BASE_MODEL_PATHS = [ BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME), BaseModelPath(name=MODEL_NAME_SHORT, model_path=MODEL_NAME_SHORT), @@ -93,6 +104,39 @@ def _build_serving_completion(engine: AsyncLLM) -> OpenAIServingCompletion: ) +def _build_minimal_metrics_serving_completion( + enable_per_request_metrics: bool, +) -> OpenAIServingCompletion: + serving = OpenAIServingCompletion.__new__(OpenAIServingCompletion) + serving.enable_prompt_tokens_details = False + serving.system_fingerprint = None + serving.enable_per_request_metrics = enable_per_request_metrics + return serving + + +def _make_metrics_request_output( + metrics: RequestStateStats | None = _PER_REQUEST_STATS, +) -> RequestOutput: + return RequestOutput( + request_id="test-id", + prompt="Test prompt", + prompt_token_ids=[1, 2, 3], + prompt_logprobs=None, + outputs=[ + CompletionOutput( + index=0, + text="Hello", + token_ids=[100, 101], + cumulative_logprob=None, + logprobs=None, + finish_reason="stop", + ) + ], + finished=True, + metrics=metrics, + ) + + def _build_renderer(model_config: MockModelConfig): return HfRenderer( MockVllmConfig(model_config, parallel_config=MockParallelConfig()), @@ -100,6 +144,58 @@ def _build_renderer(model_config: MockModelConfig): ) +def test_completion_per_request_metrics_follow_server_flag(): + request = CompletionRequest(model=MODEL_NAME, prompt="Test prompt", max_tokens=10) + request_output = _make_metrics_request_output() + + disabled_serving = _build_minimal_metrics_serving_completion( + enable_per_request_metrics=False + ) + disabled_response = disabled_serving.request_output_to_completion_response( + [request_output], + request, + "cmpl-test-id", + 0, + MODEL_NAME, + None, + RequestResponseMetadata(request_id="cmpl-test-id"), + ) + assert disabled_response.metrics is None + + enabled_serving = _build_minimal_metrics_serving_completion( + enable_per_request_metrics=True + ) + enabled_response = enabled_serving.request_output_to_completion_response( + [request_output], + request, + "cmpl-test-id", + 0, + MODEL_NAME, + None, + RequestResponseMetadata(request_id="cmpl-test-id"), + ) + assert enabled_response.metrics is not None + assert enabled_response.metrics.time_to_first_token_ms == pytest.approx(500.0) + + +def test_completion_per_request_metrics_suppressed_for_multiple_prompts(): + serving = _build_minimal_metrics_serving_completion(enable_per_request_metrics=True) + response = serving.request_output_to_completion_response( + [_make_metrics_request_output(), _make_metrics_request_output()], + CompletionRequest( + model=MODEL_NAME, + prompt=["Test prompt", "Another prompt"], + max_tokens=10, + ), + "cmpl-test-id", + 0, + MODEL_NAME, + None, + RequestResponseMetadata(request_id="cmpl-test-id"), + ) + assert response.metrics is None + + @pytest.mark.asyncio async def test_completion_error_non_stream(): """test finish_reason='error' returns 500 InternalServerError (non-streaming)""" diff --git a/tests/entrypoints/openai/test_cli_args.py b/tests/entrypoints/openai/test_cli_args.py index 58dd328b32..1f764202e5 100644 --- a/tests/entrypoints/openai/test_cli_args.py +++ b/tests/entrypoints/openai/test_cli_args.py @@ -206,6 +206,14 @@ def test_chat_template_validation_for_sad_paths(serve_parser): validate_parsed_serve_args(args) +def test_per_request_metrics_requires_log_stats(serve_parser): + args = serve_parser.parse_args( + args=["--enable-per-request-metrics", "--disable-log-stats"] + ) + with pytest.raises(ValueError): + validate_parsed_serve_args(args) + + @pytest.mark.parametrize( "cli_args, expected_middleware", [ diff --git a/vllm/entrypoints/generate/api_router.py b/vllm/entrypoints/generate/api_router.py index 5a26e475b0..062a7947e7 100644 --- a/vllm/entrypoints/generate/api_router.py +++ b/vllm/entrypoints/generate/api_router.py @@ -130,6 +130,7 @@ async def init_generate_state( enable_force_include_usage=args.enable_force_include_usage, enable_log_outputs=args.enable_log_outputs, enable_log_deltas=args.enable_log_deltas, + enable_per_request_metrics=args.enable_per_request_metrics, ) state.openai_serving_chat = ( OpenAIServingChat(**_chat_kwargs) if "generate" in supported_tasks else None @@ -150,6 +151,7 @@ async def init_generate_state( return_tokens_as_token_ids=args.return_tokens_as_token_ids, enable_prompt_tokens_details=args.enable_prompt_tokens_details, enable_force_include_usage=args.enable_force_include_usage, + enable_per_request_metrics=args.enable_per_request_metrics, ) if "generate" in supported_tasks else None diff --git a/vllm/entrypoints/generate/base/serving.py b/vllm/entrypoints/generate/base/serving.py index 196b6e3f87..78f84ca872 100644 --- a/vllm/entrypoints/generate/base/serving.py +++ b/vllm/entrypoints/generate/base/serving.py @@ -18,6 +18,7 @@ from vllm.entrypoints.openai.completion.protocol import CompletionRequest from vllm.entrypoints.openai.engine.protocol import ( ErrorResponse, GenerationError, + PerRequestTimingMetrics, ) from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.openai.responses.protocol import ResponsesRequest @@ -34,6 +35,7 @@ from vllm.tracing import ( extract_trace_headers, log_tracing_disabled_warning, ) +from vllm.v1.metrics.stats import RequestStateStats logger = init_logger(__name__) @@ -41,6 +43,61 @@ RequestT = TypeVar("RequestT", bound=AnyRequest) _T = TypeVar("_T") +def build_per_request_timing_metrics( + metrics: RequestStateStats | None, + num_generation_tokens: int, +) -> PerRequestTimingMetrics: + """Build per-request timing metrics from ``RequestStateStats``. + + ``generation_time_ms`` is the decode interval only (first output token to + last output token); it excludes both queue wait and prefill/TTFT. + ``tokens_per_second`` is overall output throughput: all generated tokens + over the inference interval (scheduling to last output token), so it counts + the prefill/TTFT phase and is not simply the reciprocal of ``mean_itl_ms``. + Each field is left ``None`` when the timestamps it depends on are + unavailable. + """ + if metrics is None: + return PerRequestTimingMetrics() + + queued_ts = metrics.queued_ts + scheduled_ts = metrics.scheduled_ts + first_token_ts = metrics.first_token_ts + last_token_ts = metrics.last_token_ts + + time_to_first_token_ms: float | None = None + generation_time_ms: float | None = None + queue_time_ms: float | None = None + mean_itl_ms: float | None = None + tokens_per_second: float | None = None + + if scheduled_ts > 0 and first_token_ts > 0: + time_to_first_token_ms = (first_token_ts - scheduled_ts) * 1000 + + if first_token_ts > 0 and last_token_ts > 0: + generation_time_ms = (last_token_ts - first_token_ts) * 1000 + + if queued_ts > 0 and scheduled_ts > 0: + queue_time_ms = (scheduled_ts - queued_ts) * 1000 + + if first_token_ts > 0 and last_token_ts > 0 and num_generation_tokens > 1: + decode_time = last_token_ts - first_token_ts + mean_itl_ms = decode_time / (num_generation_tokens - 1) * 1000 + + if scheduled_ts > 0 and last_token_ts > 0: + inference_time_ms = (last_token_ts - scheduled_ts) * 1000 + if inference_time_ms > 0: + tokens_per_second = num_generation_tokens / inference_time_ms * 1000 + + return PerRequestTimingMetrics( + time_to_first_token_ms=time_to_first_token_ms, + generation_time_ms=generation_time_ms, + queue_time_ms=queue_time_ms, + mean_itl_ms=mean_itl_ms, + tokens_per_second=tokens_per_second, + ) + + @dataclass(kw_only=True) class ServeContext(Generic[RequestT]): request: RequestT diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index 22cae80ce8..cce51157f8 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -26,6 +26,7 @@ from vllm.entrypoints.openai.engine.protocol import ( FunctionDefinition, LegacyStructuralTagResponseFormat, OpenAIBaseModel, + PerRequestTimingMetrics, StreamOptions, StructuralTagResponseFormat, ToolCall, @@ -133,6 +134,7 @@ class ChatCompletionResponse(OpenAIBaseModel): kv_transfer_params: dict[str, Any] | None = Field( default=None, description="KVTransfer parameters." ) + metrics: PerRequestTimingMetrics | None = None class ChatCompletionResponseStreamChoice(OpenAIBaseModel): @@ -160,6 +162,7 @@ class ChatCompletionStreamResponse(OpenAIBaseModel): # Rendered prompt text from chat templating (only set when # ``return_prompt_text=True`` on the request); only sent on the first chunk. prompt_text: str | None = None + metrics: PerRequestTimingMetrics | None = None class ChatCompletionToolsParam(OpenAIBaseModel): diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index 9ef144e6a1..caa3f724da 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -22,6 +22,7 @@ from vllm.entrypoints.chat_utils import ( from vllm.entrypoints.generate.base.serving import ( GenerateBaseServing, GenerationError, + build_per_request_timing_metrics, clamp_prompt_logprobs, format_token_id_placeholder, ) @@ -41,6 +42,7 @@ from vllm.entrypoints.openai.engine.protocol import ( DeltaMessage, ErrorResponse, FunctionCall, + PerRequestTimingMetrics, PromptTokenUsageInfo, RequestResponseMetadata, ToolCall, @@ -123,6 +125,7 @@ class OpenAIServingChat(GenerateBaseServing): enable_log_outputs: bool = False, enable_log_deltas: bool = True, default_chat_template_kwargs: dict[str, Any] | None = None, + enable_per_request_metrics: bool = False, ) -> None: super().__init__( engine_client=engine_client, @@ -161,6 +164,7 @@ class OpenAIServingChat(GenerateBaseServing): self.enable_prompt_tokens_details = enable_prompt_tokens_details self.enable_force_include_usage = enable_force_include_usage + self.enable_per_request_metrics = enable_per_request_metrics self.default_sampling_params = self.model_config.get_diff_sampling_param() mc = self.model_config self.override_max_tokens = ( @@ -461,8 +465,10 @@ class OpenAIServingChat(GenerateBaseServing): stream_options, self.enable_force_include_usage ) + last_res: RequestOutput | None = None try: async for res in result_generator: + last_res = res if res.prompt_token_ids is not None: num_prompt_tokens = len(res.prompt_token_ids) if res.encoder_prompt_token_ids is not None: @@ -752,6 +758,21 @@ class OpenAIServingChat(GenerateBaseServing): mm_token_counts, ) + # In streaming, metrics ride on this final usage chunk, which is + # only emitted when usage reporting is enabled (i.e. + # ``stream_options.include_usage=true`` or + # ``--enable-force-include-usage``). + stream_per_request_metrics: PerRequestTimingMetrics | None = None + if ( + self.enable_per_request_metrics + # See note in chat_completion_full_generator: suppress for n>1. + and (request.n or 1) == 1 + ): + last_metrics = last_res.metrics if last_res is not None else None + stream_per_request_metrics = build_per_request_timing_metrics( + last_metrics, completion_tokens + ) + final_usage_chunk = ChatCompletionStreamResponse( id=request_id, object=chunk_object_type, @@ -760,6 +781,7 @@ class OpenAIServingChat(GenerateBaseServing): model=model_name, usage=final_usage, system_fingerprint=self.system_fingerprint, + metrics=stream_per_request_metrics, ) final_usage_data = final_usage_chunk.model_dump_json( exclude_unset=True, exclude_none=True @@ -1003,6 +1025,18 @@ class OpenAIServingChat(GenerateBaseServing): request_metadata.final_usage_info = usage + per_request_metrics: PerRequestTimingMetrics | None = None + if ( + self.enable_per_request_metrics + # Timing metrics describe a single generation stream. For n>1 the + # returned stats belong to only one of the n sequences, so they + # cannot be accurately attributed to the request; suppress instead. + and (request.n or 1) == 1 + ): + per_request_metrics = build_per_request_timing_metrics( + final_res.metrics, num_generated_tokens + ) + # ``final_res.prompt`` is the rendered chat-templated prompt text prompt_text = final_res.prompt if request.return_prompt_text else None @@ -1019,6 +1053,7 @@ class OpenAIServingChat(GenerateBaseServing): ), prompt_text=prompt_text, kv_transfer_params=final_res.kv_transfer_params, + metrics=per_request_metrics, ) # Log complete response if output logging is enabled diff --git a/vllm/entrypoints/openai/cli_args.py b/vllm/entrypoints/openai/cli_args.py index 1533895edc..8dbb699439 100644 --- a/vllm/entrypoints/openai/cli_args.py +++ b/vllm/entrypoints/openai/cli_args.py @@ -25,12 +25,9 @@ from vllm.entrypoints.serve.utils.constants import ( H11_MAX_HEADER_COUNT_DEFAULT, H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT, ) -from vllm.logger import init_logger from vllm.tool_parsers import ToolParserManager from vllm.utils.argparse_utils import FlexibleArgumentParser -logger = init_logger(__name__) - class LoRAParserAction(argparse.Action): def __call__( @@ -134,6 +131,8 @@ class BaseFrontendArgs: log. The default of None means unlimited.""" enable_prompt_tokens_details: bool = False """If set to True, enable prompt_tokens_details in usage.""" + enable_per_request_metrics: bool = False + """If set to True, include per-request timing metrics in API responses.""" enable_server_load_tracking: bool = False """If set to True, enable tracking server_load_metrics in the app state.""" enable_force_include_usage: bool = False @@ -398,6 +397,14 @@ def validate_parsed_serve_args(args: argparse.Namespace): if args.enable_log_outputs and not args.enable_log_requests: raise TypeError("Error: --enable-log-outputs requires --enable-log-requests") + if getattr(args, "enable_per_request_metrics", False) and getattr( + args, "disable_log_stats", False + ): + raise ValueError( + "Error: --enable-per-request-metrics requires engine statistics " + "logging; remove --disable-log-stats to enable per-request metrics." + ) + if args.data_parallel_multi_port_external_lb: from vllm.entrypoints.openai.dp_supervisor import ( validate_multi_port_external_lb_args, diff --git a/vllm/entrypoints/openai/completion/protocol.py b/vllm/entrypoints/openai/completion/protocol.py index b96d4f3c0c..23bf4dd83f 100644 --- a/vllm/entrypoints/openai/completion/protocol.py +++ b/vllm/entrypoints/openai/completion/protocol.py @@ -15,6 +15,7 @@ from vllm.entrypoints.openai.engine.protocol import ( AnyResponseFormat, LegacyStructuralTagResponseFormat, OpenAIBaseModel, + PerRequestTimingMetrics, StreamOptions, StructuralTagResponseFormat, UsageInfo, @@ -558,6 +559,7 @@ class CompletionResponse(OpenAIBaseModel): kv_transfer_params: dict[str, Any] | None = Field( default=None, description="KVTransfer parameters." ) + metrics: PerRequestTimingMetrics | None = None class CompletionResponseStreamChoice(OpenAIBaseModel): @@ -589,3 +591,4 @@ class CompletionStreamResponse(OpenAIBaseModel): # Set only on the final chunk of a stream to mirror non-streaming responses # without the per-chunk serialization overhead. system_fingerprint: str | None = None + metrics: PerRequestTimingMetrics | None = None diff --git a/vllm/entrypoints/openai/completion/serving.py b/vllm/entrypoints/openai/completion/serving.py index aeade30646..d26a455cc8 100644 --- a/vllm/entrypoints/openai/completion/serving.py +++ b/vllm/entrypoints/openai/completion/serving.py @@ -16,6 +16,7 @@ from vllm.engine.protocol import EngineClient from vllm.entrypoints.generate.base.serving import ( GenerateBaseServing, GenerationError, + build_per_request_timing_metrics, clamp_prompt_logprobs, format_token_id_placeholder, ) @@ -29,6 +30,7 @@ from vllm.entrypoints.openai.completion.protocol import ( ) from vllm.entrypoints.openai.engine.protocol import ( ErrorResponse, + PerRequestTimingMetrics, PromptTokenUsageInfo, RequestResponseMetadata, UsageInfo, @@ -61,6 +63,7 @@ class OpenAIServingCompletion(GenerateBaseServing): return_tokens_as_token_ids: bool = False, enable_prompt_tokens_details: bool = False, enable_force_include_usage: bool = False, + enable_per_request_metrics: bool = False, ): super().__init__( engine_client=engine_client, @@ -72,6 +75,7 @@ class OpenAIServingCompletion(GenerateBaseServing): self.online_renderer = online_renderer self.enable_prompt_tokens_details = enable_prompt_tokens_details self.enable_force_include_usage = enable_force_include_usage + self.enable_per_request_metrics = enable_per_request_metrics self.default_sampling_params = self.model_config.get_diff_sampling_param() mc = self.model_config @@ -300,8 +304,10 @@ class OpenAIServingCompletion(GenerateBaseServing): stream_options, self.enable_force_include_usage ) + last_res: RequestOutput | None = None try: async for prompt_idx, res in result_generator: + last_res = res prompt_token_ids = res.prompt_token_ids prompt_logprobs = res.prompt_logprobs @@ -448,6 +454,23 @@ class OpenAIServingCompletion(GenerateBaseServing): ) if include_usage: + # In streaming, metrics ride on this final usage chunk, which is + # only emitted when usage reporting is enabled (i.e. + # ``stream_options.include_usage=true`` or + # ``--enable-force-include-usage``). + stream_per_request_metrics: PerRequestTimingMetrics | None = None + if ( + self.enable_per_request_metrics + # See note in request_output_to_completion_response: suppress + # when not attributable to one stream (multi-prompt or n>1). + and num_prompts == 1 + and (request.n or 1) == 1 + ): + last_metrics = last_res.metrics if last_res is not None else None + stream_per_request_metrics = build_per_request_timing_metrics( + last_metrics, total_completion_tokens + ) + final_usage_chunk = CompletionStreamResponse( id=request_id, created=created_time, @@ -455,6 +478,7 @@ class OpenAIServingCompletion(GenerateBaseServing): choices=[], usage=final_usage_info, system_fingerprint=self.system_fingerprint, + metrics=stream_per_request_metrics, ) final_usage_data = final_usage_chunk.model_dump_json( exclude_unset=False, exclude_none=True @@ -589,6 +613,23 @@ class OpenAIServingCompletion(GenerateBaseServing): ) request_metadata.final_usage_info = usage + + per_request_metrics: PerRequestTimingMetrics | None = None + if ( + self.enable_per_request_metrics + # Metrics describe a single generation stream, so suppress them when + # they cannot be attributed to one: multiple prompts (timestamps + # span prompts) or n>1 (stats belong to one of the n sequences). + and len(final_res_batch) == 1 + and (request.n or 1) == 1 + ): + last_metrics = ( + last_final_res.metrics if last_final_res is not None else None + ) + per_request_metrics = build_per_request_timing_metrics( + last_metrics, num_generated_tokens + ) + if final_res_batch: kv_transfer_params = final_res_batch[0].kv_transfer_params return CompletionResponse( @@ -599,6 +640,7 @@ class OpenAIServingCompletion(GenerateBaseServing): usage=usage, system_fingerprint=self.system_fingerprint, kv_transfer_params=kv_transfer_params, + metrics=per_request_metrics, ) def _create_completion_logprobs( diff --git a/vllm/entrypoints/openai/engine/protocol.py b/vllm/entrypoints/openai/engine/protocol.py index 084d8d429a..2c32fcf20c 100644 --- a/vllm/entrypoints/openai/engine/protocol.py +++ b/vllm/entrypoints/openai/engine/protocol.py @@ -115,6 +115,14 @@ class UsageInfo(OpenAIBaseModel): prompt_tokens_details: PromptTokenUsageInfo | None = None +class PerRequestTimingMetrics(OpenAIBaseModel): + time_to_first_token_ms: float | None = None + generation_time_ms: float | None = None + queue_time_ms: float | None = None + mean_itl_ms: float | None = None + tokens_per_second: float | None = None + + class RequestResponseMetadata(BaseModel): request_id: str final_usage_info: UsageInfo | None = None