[Frontend] Expose per-request spec decode metrics in generate API (#43310)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
co-authored by
aoshen02
Codex
parent
bf01fc4a31
commit
10e6a7f210
@@ -130,10 +130,36 @@ pub struct EngineCoreOutput {
|
||||
#[serde(default)]
|
||||
pub new_sampling_mask: Option<MaybeWireSamplingMask>,
|
||||
/// Per-request speculative-decoding acceptance metrics, set on the final
|
||||
/// output when `--per-request-spec-decode-metrics` is enabled. Opaque here;
|
||||
/// the Rust frontend does not yet surface it in responses.
|
||||
/// output when `--per-request-spec-decode-metrics` is enabled.
|
||||
#[serde(default)]
|
||||
pub spec_decode_metrics: Option<OpaqueValue>,
|
||||
pub spec_decode_metrics: Option<RequestSpecDecodeMetrics>,
|
||||
}
|
||||
|
||||
/// Raw per-sequence speculative-decoding accumulator.
|
||||
///
|
||||
/// Python models this as a plain `@dataclass`, so it is serialized by msgspec
|
||||
/// as a map (named fields). Derived values such as acceptance rates are
|
||||
/// computed by the frontend, not carried on the wire.
|
||||
///
|
||||
/// Original Python definition: `RequestSpecDecodeMetrics` in
|
||||
/// `vllm/v1/metrics/stats.py`.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct RequestSpecDecodeMetrics {
|
||||
/// Configured `num_speculative_tokens`.
|
||||
#[serde(default)]
|
||||
pub num_spec_tokens: u64,
|
||||
/// Verify-step counts indexed by accepted draft-token count.
|
||||
#[serde(default)]
|
||||
pub histogram: Vec<u64>,
|
||||
/// Total proposed draft tokens.
|
||||
#[serde(default)]
|
||||
pub num_draft_tokens: u64,
|
||||
/// Accepted draft count per verify step; empty unless `detailed`.
|
||||
#[serde(default)]
|
||||
pub per_step_accepted: Vec<u64>,
|
||||
/// Proposed draft count per verify step; empty unless `detailed`.
|
||||
#[serde(default)]
|
||||
pub per_step_drafted: Vec<u64>,
|
||||
}
|
||||
|
||||
impl EngineCoreOutput {
|
||||
|
||||
Reference in New Issue
Block a user