[Frontend][Core][Spec Decode] Per-request acceptance stats in OpenAI API responses (#48915)

Signed-off-by: Matthew Kotila <7692737+matthewkotila@users.noreply.github.com>
Co-authored-by: Reed Meyerson <31574681+reed-meyerson@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Matthew Kotila
2026-08-20 15:42:49 -07:00
committed by GitHub
co-authored by Reed Meyerson Claude Opus 4.8
parent 2f41c894e3
commit 7cfb97e337
25 changed files with 798 additions and 76 deletions
@@ -125,6 +125,11 @@ pub struct EngineCoreOutput {
pub mm_cache_miss_hashes: Option<Vec<String>>,
#[serde(default)]
pub new_sampling_mask: Option<OpaqueValue>,
/// Per-request speculative-decoding acceptance metrics, set on the final
/// output when `--per-request-spec-decode-metrics` is enabled. Opaque here;
/// the Rust frontend does not yet surface it in responses.
#[serde(default)]
pub spec_decode_metrics: Option<OpaqueValue>,
}
impl EngineCoreOutput {
@@ -443,6 +448,7 @@ mod tests {
num_nans_in_logits: 0,
mm_cache_miss_hashes: None,
new_sampling_mask: None,
spec_decode_metrics: None,
},
],
scheduler_stats: None,
@@ -2743,6 +2743,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
num_nans_in_logits: 0,
mm_cache_miss_hashes: None,
new_sampling_mask: None,
spec_decode_metrics: None,
},
],
scheduler_stats: None,