[Frontend][Core][Spec Decode] Per-request acceptance stats in OpenAI API responses (#48915)
Signed-off-by: Matthew Kotila <7692737+matthewkotila@users.noreply.github.com> Co-authored-by: Reed Meyerson <31574681+reed-meyerson@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Reed Meyerson
Claude Opus 4.8
parent
2f41c894e3
commit
7cfb97e337
@@ -125,6 +125,11 @@ pub struct EngineCoreOutput {
|
||||
pub mm_cache_miss_hashes: Option<Vec<String>>,
|
||||
#[serde(default)]
|
||||
pub new_sampling_mask: Option<OpaqueValue>,
|
||||
/// Per-request speculative-decoding acceptance metrics, set on the final
|
||||
/// output when `--per-request-spec-decode-metrics` is enabled. Opaque here;
|
||||
/// the Rust frontend does not yet surface it in responses.
|
||||
#[serde(default)]
|
||||
pub spec_decode_metrics: Option<OpaqueValue>,
|
||||
}
|
||||
|
||||
impl EngineCoreOutput {
|
||||
@@ -443,6 +448,7 @@ mod tests {
|
||||
num_nans_in_logits: 0,
|
||||
mm_cache_miss_hashes: None,
|
||||
new_sampling_mask: None,
|
||||
spec_decode_metrics: None,
|
||||
},
|
||||
],
|
||||
scheduler_stats: None,
|
||||
|
||||
@@ -2743,6 +2743,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
num_nans_in_logits: 0,
|
||||
mm_cache_miss_hashes: None,
|
||||
new_sampling_mask: None,
|
||||
spec_decode_metrics: None,
|
||||
},
|
||||
],
|
||||
scheduler_stats: None,
|
||||
|
||||
Reference in New Issue
Block a user