[Frontend] Expose per-request spec decode metrics in generate API (#43310)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
Hongxin Xu
2026-09-20 12:21:33 +00:00
committed by GitHub
co-authored by aoshen02 Codex
parent bf01fc4a31
commit 10e6a7f210
12 changed files with 516 additions and 9 deletions
@@ -130,10 +130,36 @@ pub struct EngineCoreOutput {
#[serde(default)]
pub new_sampling_mask: Option<MaybeWireSamplingMask>,
/// Per-request speculative-decoding acceptance metrics, set on the final
/// output when `--per-request-spec-decode-metrics` is enabled. Opaque here;
/// the Rust frontend does not yet surface it in responses.
/// output when `--per-request-spec-decode-metrics` is enabled.
#[serde(default)]
pub spec_decode_metrics: Option<OpaqueValue>,
pub spec_decode_metrics: Option<RequestSpecDecodeMetrics>,
}
/// Raw per-sequence speculative-decoding accumulator.
///
/// Python models this as a plain `@dataclass`, so it is serialized by msgspec
/// as a map (named fields). Derived values such as acceptance rates are
/// computed by the frontend, not carried on the wire.
///
/// Original Python definition: `RequestSpecDecodeMetrics` in
/// `vllm/v1/metrics/stats.py`.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct RequestSpecDecodeMetrics {
/// Configured `num_speculative_tokens`.
#[serde(default)]
pub num_spec_tokens: u64,
/// Verify-step counts indexed by accepted draft-token count.
#[serde(default)]
pub histogram: Vec<u64>,
/// Total proposed draft tokens.
#[serde(default)]
pub num_draft_tokens: u64,
/// Accepted draft count per verify step; empty unless `detailed`.
#[serde(default)]
pub per_step_accepted: Vec<u64>,
/// Proposed draft count per verify step; empty unless `detailed`.
#[serde(default)]
pub per_step_drafted: Vec<u64>,
}
impl EngineCoreOutput {