[Frontend][Core][Spec Decode] Per-request acceptance stats in OpenAI API responses (#48915)

Signed-off-by: Matthew Kotila <7692737+matthewkotila@users.noreply.github.com>
Co-authored-by: Reed Meyerson <31574681+reed-meyerson@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Matthew Kotila
2026-08-20 15:42:49 -07:00
committed by GitHub
co-authored by Reed Meyerson Claude Opus 4.8
parent 2f41c894e3
commit 7cfb97e337
25 changed files with 798 additions and 76 deletions
+25 -1
View File
@@ -129,12 +129,36 @@ class UsageInfo(OpenAIBaseModel):
completion_tokens_details: CompletionTokenUsageInfo | None = None
class PerRequestTimingMetrics(OpenAIBaseModel):
class SpeculativeDecodingMetrics(OpenAIBaseModel):
"""Per-request speculative-decoding acceptance metrics.
Experimental, subject to change. Only populated for single-sequence requests
(`n == 1`); `null` for `n > 1`, mirroring the timing metrics.
"""
mean_acceptance_length: float
draft_acceptance_rate: float
# Dense histogram: index j holds the number of verify steps that accepted
# exactly j draft tokens (length num_spec_tokens + 1). Excludes the
# always-accepted bonus token.
acceptance_histogram: list[int]
num_spec_steps: int
num_accepted_draft_tokens: int
num_draft_tokens: int
num_spec_tokens: int
# Ordered per-verify-step arrays; populated only at the `detailed` level.
per_step_accepted: list[int] | None = None
per_step_drafted: list[int] | None = None
class PerRequestMetrics(OpenAIBaseModel):
time_to_first_token_ms: float | None = None
generation_time_ms: float | None = None
queue_time_ms: float | None = None
mean_itl_ms: float | None = None
tokens_per_second: float | None = None
# Experimental, subject to change.
speculative_decoding: SpeculativeDecodingMetrics | None = None
class RequestResponseMetadata(BaseModel):