[Frontend][Core][Spec Decode] Per-request acceptance stats in OpenAI API responses (#48915)
Signed-off-by: Matthew Kotila <7692737+matthewkotila@users.noreply.github.com> Co-authored-by: Reed Meyerson <31574681+reed-meyerson@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Reed Meyerson
Claude Opus 4.8
parent
2f41c894e3
commit
7cfb97e337
@@ -5,7 +5,7 @@ import time
|
||||
from collections.abc import Awaitable, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from http import HTTPStatus
|
||||
from typing import ClassVar, Generic, TypeVar
|
||||
from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar
|
||||
|
||||
from fastapi import Request
|
||||
from pydantic import ConfigDict
|
||||
@@ -18,7 +18,8 @@ from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
GenerationError,
|
||||
PerRequestTimingMetrics,
|
||||
PerRequestMetrics,
|
||||
SpeculativeDecodingMetrics,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
@@ -37,6 +38,9 @@ from vllm.tracing import (
|
||||
)
|
||||
from vllm.v1.metrics.stats import RequestStateStats
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.outputs import RequestOutput
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
RequestT = TypeVar("RequestT", bound=AnyRequest)
|
||||
@@ -48,7 +52,7 @@ PRIORITY_HEADER = "X-Vllm-Priority"
|
||||
def build_per_request_timing_metrics(
|
||||
metrics: RequestStateStats | None,
|
||||
num_generation_tokens: int,
|
||||
) -> PerRequestTimingMetrics:
|
||||
) -> PerRequestMetrics:
|
||||
"""Build per-request timing metrics from ``RequestStateStats``.
|
||||
|
||||
``generation_time_ms`` is the decode interval only (first output token to
|
||||
@@ -60,7 +64,7 @@ def build_per_request_timing_metrics(
|
||||
unavailable.
|
||||
"""
|
||||
if metrics is None:
|
||||
return PerRequestTimingMetrics()
|
||||
return PerRequestMetrics()
|
||||
|
||||
queued_ts = metrics.queued_ts
|
||||
scheduled_ts = metrics.scheduled_ts
|
||||
@@ -91,7 +95,7 @@ def build_per_request_timing_metrics(
|
||||
if inference_time_ms > 0:
|
||||
tokens_per_second = num_generation_tokens / inference_time_ms * 1000
|
||||
|
||||
return PerRequestTimingMetrics(
|
||||
return PerRequestMetrics(
|
||||
time_to_first_token_ms=time_to_first_token_ms,
|
||||
generation_time_ms=generation_time_ms,
|
||||
queue_time_ms=queue_time_ms,
|
||||
@@ -100,6 +104,23 @@ def build_per_request_timing_metrics(
|
||||
)
|
||||
|
||||
|
||||
def build_spec_decoding_metrics(
|
||||
final_res: "RequestOutput | None",
|
||||
) -> SpeculativeDecodingMetrics | None:
|
||||
"""Build per-request spec-decode acceptance metrics from the single output
|
||||
sequence, or ``None`` when unavailable (metrics disabled, or no sequence
|
||||
yet).
|
||||
|
||||
Only meaningful for single-sequence requests; callers suppress it for n>1.
|
||||
"""
|
||||
if final_res is None or not final_res.outputs:
|
||||
return None
|
||||
metrics = final_res.outputs[0].spec_decode_metrics
|
||||
if metrics is None:
|
||||
return None
|
||||
return SpeculativeDecodingMetrics(**metrics.to_dict())
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class ServeContext(Generic[RequestT]):
|
||||
request: RequestT
|
||||
|
||||
@@ -29,7 +29,7 @@ from vllm.entrypoints.openai.engine.protocol import (
|
||||
FunctionCall,
|
||||
FunctionDefinition,
|
||||
OpenAIBaseModel,
|
||||
PerRequestTimingMetrics,
|
||||
PerRequestMetrics,
|
||||
StopParam,
|
||||
StreamOptions,
|
||||
ToolCall,
|
||||
@@ -146,7 +146,7 @@ class ChatCompletionResponse(OpenAIBaseModel):
|
||||
ec_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None, description="ECTransfer parameters."
|
||||
)
|
||||
metrics: PerRequestTimingMetrics | None = None
|
||||
metrics: PerRequestMetrics | None = None
|
||||
|
||||
|
||||
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
|
||||
@@ -178,7 +178,7 @@ class ChatCompletionStreamResponse(OpenAIBaseModel):
|
||||
# Rendered prompt text from chat templating (only set when
|
||||
# ``return_prompt_text=True`` on the request); only sent on the first chunk.
|
||||
prompt_text: str | None = None
|
||||
metrics: PerRequestTimingMetrics | None = None
|
||||
metrics: PerRequestMetrics | None = None
|
||||
|
||||
|
||||
class ChatCompletionToolsParam(OpenAIBaseModel):
|
||||
|
||||
@@ -20,6 +20,7 @@ from vllm.entrypoints.generate.base.serving import (
|
||||
GenerateBaseServing,
|
||||
GenerationError,
|
||||
build_per_request_timing_metrics,
|
||||
build_spec_decoding_metrics,
|
||||
clamp_prompt_logprobs,
|
||||
format_token_id_placeholder,
|
||||
)
|
||||
@@ -40,7 +41,7 @@ from vllm.entrypoints.openai.engine.protocol import (
|
||||
DeltaMessage,
|
||||
ErrorResponse,
|
||||
FunctionCall,
|
||||
PerRequestTimingMetrics,
|
||||
PerRequestMetrics,
|
||||
PromptTokenUsageInfo,
|
||||
RequestResponseMetadata,
|
||||
ToolCall,
|
||||
@@ -819,16 +820,21 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
# only emitted when usage reporting is enabled (i.e.
|
||||
# ``stream_options.include_usage=true`` or
|
||||
# ``--enable-force-include-usage``).
|
||||
stream_per_request_metrics: PerRequestTimingMetrics | None = None
|
||||
if (
|
||||
self.enable_per_request_metrics
|
||||
# See note in chat_completion_full_generator: suppress for n>1.
|
||||
and (request.n or 1) == 1
|
||||
):
|
||||
last_metrics = last_res.metrics if last_res is not None else None
|
||||
stream_per_request_metrics = build_per_request_timing_metrics(
|
||||
last_metrics, completion_tokens
|
||||
)
|
||||
stream_per_request_metrics: PerRequestMetrics | None = None
|
||||
# See note in chat_completion_full_generator: suppress for n>1.
|
||||
if (request.n or 1) == 1:
|
||||
if self.enable_per_request_metrics:
|
||||
last_metrics = (
|
||||
last_res.metrics if last_res is not None else None
|
||||
)
|
||||
stream_per_request_metrics = build_per_request_timing_metrics(
|
||||
last_metrics, completion_tokens
|
||||
)
|
||||
spec_stats = build_spec_decoding_metrics(last_res)
|
||||
if spec_stats is not None:
|
||||
if stream_per_request_metrics is None:
|
||||
stream_per_request_metrics = PerRequestMetrics()
|
||||
stream_per_request_metrics.speculative_decoding = spec_stats
|
||||
|
||||
final_usage_chunk = ChatCompletionStreamResponse(
|
||||
id=request_id,
|
||||
@@ -1118,17 +1124,20 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
|
||||
request_metadata.final_usage_info = usage
|
||||
|
||||
per_request_metrics: PerRequestTimingMetrics | None = None
|
||||
if (
|
||||
self.enable_per_request_metrics
|
||||
# Timing metrics describe a single generation stream. For n>1 the
|
||||
# returned stats belong to only one of the n sequences, so they
|
||||
# cannot be accurately attributed to the request; suppress instead.
|
||||
and (request.n or 1) == 1
|
||||
):
|
||||
per_request_metrics = build_per_request_timing_metrics(
|
||||
final_res.metrics, num_generated_tokens
|
||||
)
|
||||
per_request_metrics: PerRequestMetrics | None = None
|
||||
# Per-request metrics (timing + spec-decode acceptance) describe a single
|
||||
# generation stream. For n>1 the stats belong to only one of the n
|
||||
# sequences, so they cannot be attributed to the request; suppress.
|
||||
if (request.n or 1) == 1:
|
||||
if self.enable_per_request_metrics:
|
||||
per_request_metrics = build_per_request_timing_metrics(
|
||||
final_res.metrics, num_generated_tokens
|
||||
)
|
||||
spec_stats = build_spec_decoding_metrics(final_res)
|
||||
if spec_stats is not None:
|
||||
if per_request_metrics is None:
|
||||
per_request_metrics = PerRequestMetrics()
|
||||
per_request_metrics.speculative_decoding = spec_stats
|
||||
|
||||
# ``final_res.prompt`` is the rendered chat-templated prompt text
|
||||
prompt_text = final_res.prompt if request.return_prompt_text else None
|
||||
|
||||
@@ -13,7 +13,7 @@ from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
AnyResponseFormat,
|
||||
OpenAIBaseModel,
|
||||
PerRequestTimingMetrics,
|
||||
PerRequestMetrics,
|
||||
StopParam,
|
||||
StreamOptions,
|
||||
UsageInfo,
|
||||
@@ -660,7 +660,7 @@ class CompletionResponse(OpenAIBaseModel):
|
||||
ec_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None, description="ECTransfer parameters."
|
||||
)
|
||||
metrics: PerRequestTimingMetrics | None = None
|
||||
metrics: PerRequestMetrics | None = None
|
||||
|
||||
|
||||
class CompletionResponseStreamChoice(OpenAIBaseModel):
|
||||
@@ -692,4 +692,4 @@ class CompletionStreamResponse(OpenAIBaseModel):
|
||||
# Set only on the final chunk of a stream to mirror non-streaming responses
|
||||
# without the per-chunk serialization overhead.
|
||||
system_fingerprint: str | None = None
|
||||
metrics: PerRequestTimingMetrics | None = None
|
||||
metrics: PerRequestMetrics | None = None
|
||||
|
||||
@@ -14,6 +14,7 @@ from vllm.entrypoints.generate.base.serving import (
|
||||
GenerateBaseServing,
|
||||
GenerationError,
|
||||
build_per_request_timing_metrics,
|
||||
build_spec_decoding_metrics,
|
||||
clamp_prompt_logprobs,
|
||||
format_token_id_placeholder,
|
||||
)
|
||||
@@ -27,7 +28,7 @@ from vllm.entrypoints.openai.completion.protocol import (
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
PerRequestTimingMetrics,
|
||||
PerRequestMetrics,
|
||||
PromptTokenUsageInfo,
|
||||
RequestResponseMetadata,
|
||||
UsageInfo,
|
||||
@@ -460,18 +461,22 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
# only emitted when usage reporting is enabled (i.e.
|
||||
# ``stream_options.include_usage=true`` or
|
||||
# ``--enable-force-include-usage``).
|
||||
stream_per_request_metrics: PerRequestTimingMetrics | None = None
|
||||
if (
|
||||
self.enable_per_request_metrics
|
||||
# See note in request_output_to_completion_response: suppress
|
||||
# when not attributable to one stream (multi-prompt or n>1).
|
||||
and num_prompts == 1
|
||||
and (request.n or 1) == 1
|
||||
):
|
||||
last_metrics = last_res.metrics if last_res is not None else None
|
||||
stream_per_request_metrics = build_per_request_timing_metrics(
|
||||
last_metrics, total_completion_tokens
|
||||
)
|
||||
stream_per_request_metrics: PerRequestMetrics | None = None
|
||||
# See note in request_output_to_completion_response: suppress when
|
||||
# not attributable to one stream (multi-prompt or n>1).
|
||||
if num_prompts == 1 and (request.n or 1) == 1:
|
||||
if self.enable_per_request_metrics:
|
||||
last_metrics = (
|
||||
last_res.metrics if last_res is not None else None
|
||||
)
|
||||
stream_per_request_metrics = build_per_request_timing_metrics(
|
||||
last_metrics, total_completion_tokens
|
||||
)
|
||||
spec_stats = build_spec_decoding_metrics(last_res)
|
||||
if spec_stats is not None:
|
||||
if stream_per_request_metrics is None:
|
||||
stream_per_request_metrics = PerRequestMetrics()
|
||||
stream_per_request_metrics.speculative_decoding = spec_stats
|
||||
|
||||
final_usage_chunk = CompletionStreamResponse(
|
||||
id=request_id,
|
||||
@@ -612,21 +617,24 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
|
||||
request_metadata.final_usage_info = usage
|
||||
|
||||
per_request_metrics: PerRequestTimingMetrics | None = None
|
||||
if (
|
||||
self.enable_per_request_metrics
|
||||
# Metrics describe a single generation stream, so suppress them when
|
||||
# they cannot be attributed to one: multiple prompts (timestamps
|
||||
# span prompts) or n>1 (stats belong to one of the n sequences).
|
||||
and len(final_res_batch) == 1
|
||||
and (request.n or 1) == 1
|
||||
):
|
||||
last_metrics = (
|
||||
last_final_res.metrics if last_final_res is not None else None
|
||||
)
|
||||
per_request_metrics = build_per_request_timing_metrics(
|
||||
last_metrics, num_generated_tokens
|
||||
)
|
||||
per_request_metrics: PerRequestMetrics | None = None
|
||||
# Per-request metrics (timing + spec-decode acceptance) describe a single
|
||||
# generation stream, so suppress them when they cannot be attributed to
|
||||
# one: multiple prompts (timestamps span prompts) or n>1 (stats belong to
|
||||
# one of the n sequences).
|
||||
if len(final_res_batch) == 1 and (request.n or 1) == 1:
|
||||
if self.enable_per_request_metrics:
|
||||
last_metrics = (
|
||||
last_final_res.metrics if last_final_res is not None else None
|
||||
)
|
||||
per_request_metrics = build_per_request_timing_metrics(
|
||||
last_metrics, num_generated_tokens
|
||||
)
|
||||
spec_stats = build_spec_decoding_metrics(last_final_res)
|
||||
if spec_stats is not None:
|
||||
if per_request_metrics is None:
|
||||
per_request_metrics = PerRequestMetrics()
|
||||
per_request_metrics.speculative_decoding = spec_stats
|
||||
|
||||
if final_res_batch:
|
||||
kv_transfer_params = final_res_batch[0].kv_transfer_params
|
||||
|
||||
@@ -129,12 +129,36 @@ class UsageInfo(OpenAIBaseModel):
|
||||
completion_tokens_details: CompletionTokenUsageInfo | None = None
|
||||
|
||||
|
||||
class PerRequestTimingMetrics(OpenAIBaseModel):
|
||||
class SpeculativeDecodingMetrics(OpenAIBaseModel):
|
||||
"""Per-request speculative-decoding acceptance metrics.
|
||||
|
||||
Experimental, subject to change. Only populated for single-sequence requests
|
||||
(`n == 1`); `null` for `n > 1`, mirroring the timing metrics.
|
||||
"""
|
||||
|
||||
mean_acceptance_length: float
|
||||
draft_acceptance_rate: float
|
||||
# Dense histogram: index j holds the number of verify steps that accepted
|
||||
# exactly j draft tokens (length num_spec_tokens + 1). Excludes the
|
||||
# always-accepted bonus token.
|
||||
acceptance_histogram: list[int]
|
||||
num_spec_steps: int
|
||||
num_accepted_draft_tokens: int
|
||||
num_draft_tokens: int
|
||||
num_spec_tokens: int
|
||||
# Ordered per-verify-step arrays; populated only at the `detailed` level.
|
||||
per_step_accepted: list[int] | None = None
|
||||
per_step_drafted: list[int] | None = None
|
||||
|
||||
|
||||
class PerRequestMetrics(OpenAIBaseModel):
|
||||
time_to_first_token_ms: float | None = None
|
||||
generation_time_ms: float | None = None
|
||||
queue_time_ms: float | None = None
|
||||
mean_itl_ms: float | None = None
|
||||
tokens_per_second: float | None = None
|
||||
# Experimental, subject to change.
|
||||
speculative_decoding: SpeculativeDecodingMetrics | None = None
|
||||
|
||||
|
||||
class RequestResponseMetadata(BaseModel):
|
||||
|
||||
Reference in New Issue
Block a user