[Frontend] add per-request timing metrics field to response body of Chat/Completions APIs (#46768)

Signed-off-by: Nicholas Edelman <nedelman@nvidia.com>
Signed-off-by: Simon Mo <simon@inferact.ai>
Co-authored-by: Simon Mo <simon@inferact.ai>
Co-authored-by: GPT-5.5 <noreply@cursor.com>
This commit is contained in:
nv-nedelman-1
2026-07-06 17:48:29 -07:00
committed by GitHub
co-authored by Simon Mo GPT-5.5
parent 5769a7382c
commit 86db6c3070
12 changed files with 565 additions and 4 deletions
+127
View File
@@ -0,0 +1,127 @@
# Per-Request Metrics
vLLM can return per-request timing metrics directly in API responses.
This is useful for billing, SLA monitoring, and latency analysis at the
individual request level, as a complement to the server-aggregated Prometheus
metrics exposed at `/metrics`.
## Enabling
Start the server with `--enable-per-request-metrics`:
```bash
vllm serve meta-llama/Llama-3.1-8B-Instruct --enable-per-request-metrics
```
When this flag is set, supported API responses include metrics for each
attributable request.
!!! note
At high concurrency, enabling per-request metrics computation may introduce
non-negligible CPU overhead. Benchmark your specific workload to evaluate the
impact before enabling in production.
## Response Format
When per-request metrics are enabled, the response includes a `metrics` object:
```json
{
"id": "chatcmpl-abc123",
"object": "chat.completion",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"choices": [ ... ],
"usage": {
"prompt_tokens": 42,
"completion_tokens": 128,
"total_tokens": 170
},
"metrics": {
"time_to_first_token_ms": 85.2,
"generation_time_ms": 1240.5,
"queue_time_ms": 12.3,
"mean_itl_ms": 9.1,
"tokens_per_second": 103.2
}
}
```
| Field | Description |
| --- | --- |
| `time_to_first_token_ms` | Time from when the request was scheduled until the first output token was generated (TTFT). |
| `generation_time_ms` | Decode time: time from the first output token to the last output token. Excludes both queue wait and prefill/TTFT. |
| `queue_time_ms` | Time the request spent waiting in the scheduler queue before processing began. |
| `mean_itl_ms` | Mean inter-token latency (average time between successive output tokens) during the decode phase. `null` for single-token responses. |
| `tokens_per_second` | Overall output token throughput: all generated tokens over the inference interval (scheduling to last output token). Unlike `generation_time_ms`, this includes the prefill phase, so it reflects end-to-end generation speed rather than pure decode speed. |
All fields are `null` if the underlying timing data is not available for that
request.
!!! note
Timing metrics describe a single generation stream, so they are only
returned when the request maps to exactly one. They are suppressed (the
`metrics` object is `null`) for requests with `n > 1`, because the
underlying timing data reflects only one of the `n` sequences and cannot be
accurately attributed to the request as a whole. Token usage
(`prompt_tokens`, `completion_tokens`) remains accurate in these cases.
Per-request metrics also require server-side statistics logging, which is
on by default. vLLM rejects `--enable-per-request-metrics` when
`--disable-log-stats` is also set.
## Example Request
=== "Non-streaming"
```python
from openai import OpenAI
client = OpenAI(base_url="http://localhost:8000/v1", api_key="token")
response = client.chat.completions.create(
model="meta-llama/Llama-3.1-8B-Instruct",
messages=[{"role": "user", "content": "What is the capital of France?"}],
)
print(response.usage)
print(response.model_extra.get("metrics"))
```
=== "Streaming"
In streaming responses, metrics are attached to the final usage chunk (the
chunk sent after all content chunks). That chunk is only emitted when usage
reporting is enabled with `stream_options.include_usage: true` or forced
server-side with `--enable-force-include-usage`. Without forced usage, a
streaming client must set `stream_options.include_usage: true` to receive
metrics.
```python
from openai import OpenAI
client = OpenAI(base_url="http://localhost:8000/v1", api_key="token")
stream = client.chat.completions.create(
model="meta-llama/Llama-3.1-8B-Instruct",
messages=[{"role": "user", "content": "What is the capital of France?"}],
stream=True,
stream_options={"include_usage": True},
)
for chunk in stream:
if chunk.usage:
print("Usage:", chunk.usage)
print("Metrics:", chunk.model_extra.get("metrics"))
```
## Completions API
Per-request metrics are also available on the `/v1/completions` endpoint using
the same `metrics` response field. As with `n > 1`, metrics are omitted for
requests with multiple prompts, because the timing data cannot be attributed to
a single prompt's generation.
## Relationship to Prometheus Metrics
The `metrics` response field provides per-request values for a single request.
The `/metrics` Prometheus endpoint exposes server-level histograms (e.g.
`vllm:time_to_first_token_seconds`) that aggregate across all requests.
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import json
from collections.abc import AsyncIterator
from contextlib import suppress
from dataclasses import dataclass, field
from typing import Any
@@ -19,6 +20,7 @@ from tests.entrypoints.openai.utils import (
from tests.utils import RemoteOpenAIServer
from vllm._aiter_ops import is_aiter_found_and_supported
from vllm.config import MultiModalConfig
from vllm.entrypoints.generate.base.serving import build_per_request_timing_metrics
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionResponse,
@@ -50,9 +52,17 @@ from vllm.tokenizers import get_tokenizer
from vllm.tokenizers.mistral import MistralTokenizer
from vllm.tokenizers.registry import cached_tokenizer_from_config
from vllm.v1.engine.async_llm import AsyncLLM
from vllm.v1.metrics.stats import RequestStateStats
GPT_OSS_MODEL_NAME = "openai/gpt-oss-20b"
GPT_OSS_SPECULATOR_NAME = "RedHatAI/gpt-oss-20b-speculator.eagle3"
_PER_REQUEST_STATS = RequestStateStats(
queued_ts=1.0,
scheduled_ts=1.5,
first_token_ts=2.0,
last_token_ts=3.0,
num_generation_tokens=2,
)
@pytest.fixture(scope="module")
@@ -608,6 +618,169 @@ def _build_serving_chat(
return serving_chat
def _build_minimal_metrics_serving_chat(
enable_per_request_metrics: bool,
enable_force_include_usage: bool = False,
) -> OpenAIServingChat:
serving = OpenAIServingChat.__new__(OpenAIServingChat)
serving.response_role = "assistant"
serving.parser_cls = None
serving.enable_auto_tools = False
serving.enable_prompt_tokens_details = False
serving.enable_log_outputs = False
serving.enable_log_deltas = False
serving.enable_force_include_usage = enable_force_include_usage
serving.request_logger = None
serving.system_fingerprint = None
serving.enable_per_request_metrics = enable_per_request_metrics
return serving
def _make_metrics_request_output(
metrics: RequestStateStats | None = _PER_REQUEST_STATS,
token_ids: tuple[int, ...] = (100, 101),
) -> RequestOutput:
return RequestOutput(
request_id="test-id",
prompt="Test prompt",
prompt_token_ids=[1, 2, 3],
prompt_logprobs=None,
outputs=[
CompletionOutput(
index=0,
text="Hello",
token_ids=list(token_ids),
cumulative_logprob=None,
logprobs=None,
finish_reason="stop",
)
],
finished=True,
metrics=metrics,
)
async def _single_request_output(
request_output: RequestOutput,
) -> AsyncIterator[RequestOutput]:
yield request_output
async def _collect_metrics_stream_chunks(
serving: OpenAIServingChat,
request: ChatCompletionRequest,
) -> list[dict[str, Any]]:
chunks: list[dict[str, Any]] = []
async for line in serving.chat_completion_stream_generator(
request,
_single_request_output(_make_metrics_request_output()),
"chatcmpl-test-id",
"test-model",
conversation=[{"role": "user", "content": "Test"}],
tokenizer=MagicMock(),
request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"),
):
line = line.strip()
if not line.startswith("data: "):
continue
payload = line[len("data: ") :]
if payload != "[DONE]":
chunks.append(json.loads(payload))
return chunks
def test_build_per_request_timing_metrics_valid_timestamps():
metrics = build_per_request_timing_metrics(
_PER_REQUEST_STATS, num_generation_tokens=10
)
assert metrics.time_to_first_token_ms == pytest.approx(500.0)
assert metrics.generation_time_ms == pytest.approx(1000.0)
assert metrics.queue_time_ms == pytest.approx(500.0)
assert metrics.mean_itl_ms == pytest.approx(1000.0 / 9, rel=1e-4)
assert metrics.tokens_per_second == pytest.approx(10.0 / 1.5, rel=1e-4)
@pytest.mark.asyncio
async def test_chat_per_request_metrics_follow_server_flag():
request = ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "Test prompt"}],
max_tokens=10,
stream=False,
)
request_output = _make_metrics_request_output()
disabled_serving = _build_minimal_metrics_serving_chat(
enable_per_request_metrics=False
)
disabled_response = await disabled_serving.chat_completion_full_generator(
request,
_single_request_output(request_output),
"chatcmpl-test-id",
"test-model",
conversation=[{"role": "user", "content": "Test"}],
tokenizer=MagicMock(),
request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"),
)
assert disabled_response.metrics is None
enabled_serving = _build_minimal_metrics_serving_chat(
enable_per_request_metrics=True
)
enabled_response = await enabled_serving.chat_completion_full_generator(
request,
_single_request_output(request_output),
"chatcmpl-test-id",
"test-model",
conversation=[{"role": "user", "content": "Test"}],
tokenizer=MagicMock(),
request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"),
)
assert enabled_response.metrics is not None
assert enabled_response.metrics.time_to_first_token_ms == pytest.approx(500.0)
@pytest.mark.asyncio
async def test_chat_per_request_metrics_suppressed_for_n_greater_than_one():
serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=True)
response = await serving.chat_completion_full_generator(
ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "Test prompt"}],
max_tokens=10,
stream=False,
n=2,
),
_single_request_output(_make_metrics_request_output()),
"chatcmpl-test-id",
"test-model",
conversation=[{"role": "user", "content": "Test"}],
tokenizer=MagicMock(),
request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"),
)
assert response.metrics is None
@pytest.mark.asyncio
async def test_chat_streaming_metrics_ride_on_usage_chunk():
serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=True)
chunks = await _collect_metrics_stream_chunks(
serving,
ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "Test prompt"}],
max_tokens=10,
stream=True,
stream_options={"include_usage": True},
),
)
usage_chunks = [chunk for chunk in chunks if chunk.get("usage")]
assert usage_chunks
assert usage_chunks[-1]["metrics"]["time_to_first_token_ms"] == pytest.approx(500.0)
@dataclass
class MockEngine:
model_config: MockModelConfig = field(default_factory=MockModelConfig)
@@ -11,7 +11,10 @@ from pydantic import ValidationError
from vllm.config.multimodal import MultiModalConfig
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
from vllm.entrypoints.openai.engine.protocol import GenerationError
from vllm.entrypoints.openai.engine.protocol import (
GenerationError,
RequestResponseMetadata,
)
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.scale_out.render.serving import ServingRender
@@ -20,9 +23,17 @@ from vllm.renderers.hf import HfRenderer
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.tokenizers.registry import cached_tokenizer_from_config
from vllm.v1.engine.async_llm import AsyncLLM
from vllm.v1.metrics.stats import RequestStateStats
MODEL_NAME = "openai-community/gpt2"
MODEL_NAME_SHORT = "gpt2"
_PER_REQUEST_STATS = RequestStateStats(
queued_ts=1.0,
scheduled_ts=1.5,
first_token_ts=2.0,
last_token_ts=3.0,
num_generation_tokens=2,
)
BASE_MODEL_PATHS = [
BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME),
BaseModelPath(name=MODEL_NAME_SHORT, model_path=MODEL_NAME_SHORT),
@@ -93,6 +104,39 @@ def _build_serving_completion(engine: AsyncLLM) -> OpenAIServingCompletion:
)
def _build_minimal_metrics_serving_completion(
enable_per_request_metrics: bool,
) -> OpenAIServingCompletion:
serving = OpenAIServingCompletion.__new__(OpenAIServingCompletion)
serving.enable_prompt_tokens_details = False
serving.system_fingerprint = None
serving.enable_per_request_metrics = enable_per_request_metrics
return serving
def _make_metrics_request_output(
metrics: RequestStateStats | None = _PER_REQUEST_STATS,
) -> RequestOutput:
return RequestOutput(
request_id="test-id",
prompt="Test prompt",
prompt_token_ids=[1, 2, 3],
prompt_logprobs=None,
outputs=[
CompletionOutput(
index=0,
text="Hello",
token_ids=[100, 101],
cumulative_logprob=None,
logprobs=None,
finish_reason="stop",
)
],
finished=True,
metrics=metrics,
)
def _build_renderer(model_config: MockModelConfig):
return HfRenderer(
MockVllmConfig(model_config, parallel_config=MockParallelConfig()),
@@ -100,6 +144,58 @@ def _build_renderer(model_config: MockModelConfig):
)
def test_completion_per_request_metrics_follow_server_flag():
request = CompletionRequest(model=MODEL_NAME, prompt="Test prompt", max_tokens=10)
request_output = _make_metrics_request_output()
disabled_serving = _build_minimal_metrics_serving_completion(
enable_per_request_metrics=False
)
disabled_response = disabled_serving.request_output_to_completion_response(
[request_output],
request,
"cmpl-test-id",
0,
MODEL_NAME,
None,
RequestResponseMetadata(request_id="cmpl-test-id"),
)
assert disabled_response.metrics is None
enabled_serving = _build_minimal_metrics_serving_completion(
enable_per_request_metrics=True
)
enabled_response = enabled_serving.request_output_to_completion_response(
[request_output],
request,
"cmpl-test-id",
0,
MODEL_NAME,
None,
RequestResponseMetadata(request_id="cmpl-test-id"),
)
assert enabled_response.metrics is not None
assert enabled_response.metrics.time_to_first_token_ms == pytest.approx(500.0)
def test_completion_per_request_metrics_suppressed_for_multiple_prompts():
serving = _build_minimal_metrics_serving_completion(enable_per_request_metrics=True)
response = serving.request_output_to_completion_response(
[_make_metrics_request_output(), _make_metrics_request_output()],
CompletionRequest(
model=MODEL_NAME,
prompt=["Test prompt", "Another prompt"],
max_tokens=10,
),
"cmpl-test-id",
0,
MODEL_NAME,
None,
RequestResponseMetadata(request_id="cmpl-test-id"),
)
assert response.metrics is None
@pytest.mark.asyncio
async def test_completion_error_non_stream():
"""test finish_reason='error' returns 500 InternalServerError (non-streaming)"""
@@ -206,6 +206,14 @@ def test_chat_template_validation_for_sad_paths(serve_parser):
validate_parsed_serve_args(args)
def test_per_request_metrics_requires_log_stats(serve_parser):
args = serve_parser.parse_args(
args=["--enable-per-request-metrics", "--disable-log-stats"]
)
with pytest.raises(ValueError):
validate_parsed_serve_args(args)
@pytest.mark.parametrize(
"cli_args, expected_middleware",
[
+2
View File
@@ -130,6 +130,7 @@ async def init_generate_state(
enable_force_include_usage=args.enable_force_include_usage,
enable_log_outputs=args.enable_log_outputs,
enable_log_deltas=args.enable_log_deltas,
enable_per_request_metrics=args.enable_per_request_metrics,
)
state.openai_serving_chat = (
OpenAIServingChat(**_chat_kwargs) if "generate" in supported_tasks else None
@@ -150,6 +151,7 @@ async def init_generate_state(
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
enable_force_include_usage=args.enable_force_include_usage,
enable_per_request_metrics=args.enable_per_request_metrics,
)
if "generate" in supported_tasks
else None
+57
View File
@@ -18,6 +18,7 @@ from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.engine.protocol import (
ErrorResponse,
GenerationError,
PerRequestTimingMetrics,
)
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
@@ -34,6 +35,7 @@ from vllm.tracing import (
extract_trace_headers,
log_tracing_disabled_warning,
)
from vllm.v1.metrics.stats import RequestStateStats
logger = init_logger(__name__)
@@ -41,6 +43,61 @@ RequestT = TypeVar("RequestT", bound=AnyRequest)
_T = TypeVar("_T")
def build_per_request_timing_metrics(
metrics: RequestStateStats | None,
num_generation_tokens: int,
) -> PerRequestTimingMetrics:
"""Build per-request timing metrics from ``RequestStateStats``.
``generation_time_ms`` is the decode interval only (first output token to
last output token); it excludes both queue wait and prefill/TTFT.
``tokens_per_second`` is overall output throughput: all generated tokens
over the inference interval (scheduling to last output token), so it counts
the prefill/TTFT phase and is not simply the reciprocal of ``mean_itl_ms``.
Each field is left ``None`` when the timestamps it depends on are
unavailable.
"""
if metrics is None:
return PerRequestTimingMetrics()
queued_ts = metrics.queued_ts
scheduled_ts = metrics.scheduled_ts
first_token_ts = metrics.first_token_ts
last_token_ts = metrics.last_token_ts
time_to_first_token_ms: float | None = None
generation_time_ms: float | None = None
queue_time_ms: float | None = None
mean_itl_ms: float | None = None
tokens_per_second: float | None = None
if scheduled_ts > 0 and first_token_ts > 0:
time_to_first_token_ms = (first_token_ts - scheduled_ts) * 1000
if first_token_ts > 0 and last_token_ts > 0:
generation_time_ms = (last_token_ts - first_token_ts) * 1000
if queued_ts > 0 and scheduled_ts > 0:
queue_time_ms = (scheduled_ts - queued_ts) * 1000
if first_token_ts > 0 and last_token_ts > 0 and num_generation_tokens > 1:
decode_time = last_token_ts - first_token_ts
mean_itl_ms = decode_time / (num_generation_tokens - 1) * 1000
if scheduled_ts > 0 and last_token_ts > 0:
inference_time_ms = (last_token_ts - scheduled_ts) * 1000
if inference_time_ms > 0:
tokens_per_second = num_generation_tokens / inference_time_ms * 1000
return PerRequestTimingMetrics(
time_to_first_token_ms=time_to_first_token_ms,
generation_time_ms=generation_time_ms,
queue_time_ms=queue_time_ms,
mean_itl_ms=mean_itl_ms,
tokens_per_second=tokens_per_second,
)
@dataclass(kw_only=True)
class ServeContext(Generic[RequestT]):
request: RequestT
@@ -26,6 +26,7 @@ from vllm.entrypoints.openai.engine.protocol import (
FunctionDefinition,
LegacyStructuralTagResponseFormat,
OpenAIBaseModel,
PerRequestTimingMetrics,
StreamOptions,
StructuralTagResponseFormat,
ToolCall,
@@ -133,6 +134,7 @@ class ChatCompletionResponse(OpenAIBaseModel):
kv_transfer_params: dict[str, Any] | None = Field(
default=None, description="KVTransfer parameters."
)
metrics: PerRequestTimingMetrics | None = None
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
@@ -160,6 +162,7 @@ class ChatCompletionStreamResponse(OpenAIBaseModel):
# Rendered prompt text from chat templating (only set when
# ``return_prompt_text=True`` on the request); only sent on the first chunk.
prompt_text: str | None = None
metrics: PerRequestTimingMetrics | None = None
class ChatCompletionToolsParam(OpenAIBaseModel):
@@ -22,6 +22,7 @@ from vllm.entrypoints.chat_utils import (
from vllm.entrypoints.generate.base.serving import (
GenerateBaseServing,
GenerationError,
build_per_request_timing_metrics,
clamp_prompt_logprobs,
format_token_id_placeholder,
)
@@ -41,6 +42,7 @@ from vllm.entrypoints.openai.engine.protocol import (
DeltaMessage,
ErrorResponse,
FunctionCall,
PerRequestTimingMetrics,
PromptTokenUsageInfo,
RequestResponseMetadata,
ToolCall,
@@ -123,6 +125,7 @@ class OpenAIServingChat(GenerateBaseServing):
enable_log_outputs: bool = False,
enable_log_deltas: bool = True,
default_chat_template_kwargs: dict[str, Any] | None = None,
enable_per_request_metrics: bool = False,
) -> None:
super().__init__(
engine_client=engine_client,
@@ -161,6 +164,7 @@ class OpenAIServingChat(GenerateBaseServing):
self.enable_prompt_tokens_details = enable_prompt_tokens_details
self.enable_force_include_usage = enable_force_include_usage
self.enable_per_request_metrics = enable_per_request_metrics
self.default_sampling_params = self.model_config.get_diff_sampling_param()
mc = self.model_config
self.override_max_tokens = (
@@ -461,8 +465,10 @@ class OpenAIServingChat(GenerateBaseServing):
stream_options, self.enable_force_include_usage
)
last_res: RequestOutput | None = None
try:
async for res in result_generator:
last_res = res
if res.prompt_token_ids is not None:
num_prompt_tokens = len(res.prompt_token_ids)
if res.encoder_prompt_token_ids is not None:
@@ -752,6 +758,21 @@ class OpenAIServingChat(GenerateBaseServing):
mm_token_counts,
)
# In streaming, metrics ride on this final usage chunk, which is
# only emitted when usage reporting is enabled (i.e.
# ``stream_options.include_usage=true`` or
# ``--enable-force-include-usage``).
stream_per_request_metrics: PerRequestTimingMetrics | None = None
if (
self.enable_per_request_metrics
# See note in chat_completion_full_generator: suppress for n>1.
and (request.n or 1) == 1
):
last_metrics = last_res.metrics if last_res is not None else None
stream_per_request_metrics = build_per_request_timing_metrics(
last_metrics, completion_tokens
)
final_usage_chunk = ChatCompletionStreamResponse(
id=request_id,
object=chunk_object_type,
@@ -760,6 +781,7 @@ class OpenAIServingChat(GenerateBaseServing):
model=model_name,
usage=final_usage,
system_fingerprint=self.system_fingerprint,
metrics=stream_per_request_metrics,
)
final_usage_data = final_usage_chunk.model_dump_json(
exclude_unset=True, exclude_none=True
@@ -1003,6 +1025,18 @@ class OpenAIServingChat(GenerateBaseServing):
request_metadata.final_usage_info = usage
per_request_metrics: PerRequestTimingMetrics | None = None
if (
self.enable_per_request_metrics
# Timing metrics describe a single generation stream. For n>1 the
# returned stats belong to only one of the n sequences, so they
# cannot be accurately attributed to the request; suppress instead.
and (request.n or 1) == 1
):
per_request_metrics = build_per_request_timing_metrics(
final_res.metrics, num_generated_tokens
)
# ``final_res.prompt`` is the rendered chat-templated prompt text
prompt_text = final_res.prompt if request.return_prompt_text else None
@@ -1019,6 +1053,7 @@ class OpenAIServingChat(GenerateBaseServing):
),
prompt_text=prompt_text,
kv_transfer_params=final_res.kv_transfer_params,
metrics=per_request_metrics,
)
# Log complete response if output logging is enabled
+10 -3
View File
@@ -25,12 +25,9 @@ from vllm.entrypoints.serve.utils.constants import (
H11_MAX_HEADER_COUNT_DEFAULT,
H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT,
)
from vllm.logger import init_logger
from vllm.tool_parsers import ToolParserManager
from vllm.utils.argparse_utils import FlexibleArgumentParser
logger = init_logger(__name__)
class LoRAParserAction(argparse.Action):
def __call__(
@@ -134,6 +131,8 @@ class BaseFrontendArgs:
log. The default of None means unlimited."""
enable_prompt_tokens_details: bool = False
"""If set to True, enable prompt_tokens_details in usage."""
enable_per_request_metrics: bool = False
"""If set to True, include per-request timing metrics in API responses."""
enable_server_load_tracking: bool = False
"""If set to True, enable tracking server_load_metrics in the app state."""
enable_force_include_usage: bool = False
@@ -398,6 +397,14 @@ def validate_parsed_serve_args(args: argparse.Namespace):
if args.enable_log_outputs and not args.enable_log_requests:
raise TypeError("Error: --enable-log-outputs requires --enable-log-requests")
if getattr(args, "enable_per_request_metrics", False) and getattr(
args, "disable_log_stats", False
):
raise ValueError(
"Error: --enable-per-request-metrics requires engine statistics "
"logging; remove --disable-log-stats to enable per-request metrics."
)
if args.data_parallel_multi_port_external_lb:
from vllm.entrypoints.openai.dp_supervisor import (
validate_multi_port_external_lb_args,
@@ -15,6 +15,7 @@ from vllm.entrypoints.openai.engine.protocol import (
AnyResponseFormat,
LegacyStructuralTagResponseFormat,
OpenAIBaseModel,
PerRequestTimingMetrics,
StreamOptions,
StructuralTagResponseFormat,
UsageInfo,
@@ -558,6 +559,7 @@ class CompletionResponse(OpenAIBaseModel):
kv_transfer_params: dict[str, Any] | None = Field(
default=None, description="KVTransfer parameters."
)
metrics: PerRequestTimingMetrics | None = None
class CompletionResponseStreamChoice(OpenAIBaseModel):
@@ -589,3 +591,4 @@ class CompletionStreamResponse(OpenAIBaseModel):
# Set only on the final chunk of a stream to mirror non-streaming responses
# without the per-chunk serialization overhead.
system_fingerprint: str | None = None
metrics: PerRequestTimingMetrics | None = None
@@ -16,6 +16,7 @@ from vllm.engine.protocol import EngineClient
from vllm.entrypoints.generate.base.serving import (
GenerateBaseServing,
GenerationError,
build_per_request_timing_metrics,
clamp_prompt_logprobs,
format_token_id_placeholder,
)
@@ -29,6 +30,7 @@ from vllm.entrypoints.openai.completion.protocol import (
)
from vllm.entrypoints.openai.engine.protocol import (
ErrorResponse,
PerRequestTimingMetrics,
PromptTokenUsageInfo,
RequestResponseMetadata,
UsageInfo,
@@ -61,6 +63,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
return_tokens_as_token_ids: bool = False,
enable_prompt_tokens_details: bool = False,
enable_force_include_usage: bool = False,
enable_per_request_metrics: bool = False,
):
super().__init__(
engine_client=engine_client,
@@ -72,6 +75,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
self.online_renderer = online_renderer
self.enable_prompt_tokens_details = enable_prompt_tokens_details
self.enable_force_include_usage = enable_force_include_usage
self.enable_per_request_metrics = enable_per_request_metrics
self.default_sampling_params = self.model_config.get_diff_sampling_param()
mc = self.model_config
@@ -300,8 +304,10 @@ class OpenAIServingCompletion(GenerateBaseServing):
stream_options, self.enable_force_include_usage
)
last_res: RequestOutput | None = None
try:
async for prompt_idx, res in result_generator:
last_res = res
prompt_token_ids = res.prompt_token_ids
prompt_logprobs = res.prompt_logprobs
@@ -448,6 +454,23 @@ class OpenAIServingCompletion(GenerateBaseServing):
)
if include_usage:
# In streaming, metrics ride on this final usage chunk, which is
# only emitted when usage reporting is enabled (i.e.
# ``stream_options.include_usage=true`` or
# ``--enable-force-include-usage``).
stream_per_request_metrics: PerRequestTimingMetrics | None = None
if (
self.enable_per_request_metrics
# See note in request_output_to_completion_response: suppress
# when not attributable to one stream (multi-prompt or n>1).
and num_prompts == 1
and (request.n or 1) == 1
):
last_metrics = last_res.metrics if last_res is not None else None
stream_per_request_metrics = build_per_request_timing_metrics(
last_metrics, total_completion_tokens
)
final_usage_chunk = CompletionStreamResponse(
id=request_id,
created=created_time,
@@ -455,6 +478,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
choices=[],
usage=final_usage_info,
system_fingerprint=self.system_fingerprint,
metrics=stream_per_request_metrics,
)
final_usage_data = final_usage_chunk.model_dump_json(
exclude_unset=False, exclude_none=True
@@ -589,6 +613,23 @@ class OpenAIServingCompletion(GenerateBaseServing):
)
request_metadata.final_usage_info = usage
per_request_metrics: PerRequestTimingMetrics | None = None
if (
self.enable_per_request_metrics
# Metrics describe a single generation stream, so suppress them when
# they cannot be attributed to one: multiple prompts (timestamps
# span prompts) or n>1 (stats belong to one of the n sequences).
and len(final_res_batch) == 1
and (request.n or 1) == 1
):
last_metrics = (
last_final_res.metrics if last_final_res is not None else None
)
per_request_metrics = build_per_request_timing_metrics(
last_metrics, num_generated_tokens
)
if final_res_batch:
kv_transfer_params = final_res_batch[0].kv_transfer_params
return CompletionResponse(
@@ -599,6 +640,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
usage=usage,
system_fingerprint=self.system_fingerprint,
kv_transfer_params=kv_transfer_params,
metrics=per_request_metrics,
)
def _create_completion_logprobs(
@@ -115,6 +115,14 @@ class UsageInfo(OpenAIBaseModel):
prompt_tokens_details: PromptTokenUsageInfo | None = None
class PerRequestTimingMetrics(OpenAIBaseModel):
time_to_first_token_ms: float | None = None
generation_time_ms: float | None = None
queue_time_ms: float | None = None
mean_itl_ms: float | None = None
tokens_per_second: float | None = None
class RequestResponseMetadata(BaseModel):
request_id: str
final_usage_info: UsageInfo | None = None