[Frontend] Expose per-request spec decode metrics in generate API (#43310)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
co-authored by
aoshen02
Codex
parent
bf01fc4a31
commit
10e6a7f210
@@ -130,10 +130,36 @@ pub struct EngineCoreOutput {
|
||||
#[serde(default)]
|
||||
pub new_sampling_mask: Option<MaybeWireSamplingMask>,
|
||||
/// Per-request speculative-decoding acceptance metrics, set on the final
|
||||
/// output when `--per-request-spec-decode-metrics` is enabled. Opaque here;
|
||||
/// the Rust frontend does not yet surface it in responses.
|
||||
/// output when `--per-request-spec-decode-metrics` is enabled.
|
||||
#[serde(default)]
|
||||
pub spec_decode_metrics: Option<OpaqueValue>,
|
||||
pub spec_decode_metrics: Option<RequestSpecDecodeMetrics>,
|
||||
}
|
||||
|
||||
/// Raw per-sequence speculative-decoding accumulator.
|
||||
///
|
||||
/// Python models this as a plain `@dataclass`, so it is serialized by msgspec
|
||||
/// as a map (named fields). Derived values such as acceptance rates are
|
||||
/// computed by the frontend, not carried on the wire.
|
||||
///
|
||||
/// Original Python definition: `RequestSpecDecodeMetrics` in
|
||||
/// `vllm/v1/metrics/stats.py`.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct RequestSpecDecodeMetrics {
|
||||
/// Configured `num_speculative_tokens`.
|
||||
#[serde(default)]
|
||||
pub num_spec_tokens: u64,
|
||||
/// Verify-step counts indexed by accepted draft-token count.
|
||||
#[serde(default)]
|
||||
pub histogram: Vec<u64>,
|
||||
/// Total proposed draft tokens.
|
||||
#[serde(default)]
|
||||
pub num_draft_tokens: u64,
|
||||
/// Accepted draft count per verify step; empty unless `detailed`.
|
||||
#[serde(default)]
|
||||
pub per_step_accepted: Vec<u64>,
|
||||
/// Proposed draft count per verify step; empty unless `detailed`.
|
||||
#[serde(default)]
|
||||
pub per_step_drafted: Vec<u64>,
|
||||
}
|
||||
|
||||
impl EngineCoreOutput {
|
||||
|
||||
@@ -11,7 +11,9 @@ use futures::stream::FusedStream;
|
||||
use futures::{Stream, StreamExt as _, pin_mut};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use vllm_engine_core_client::protocol::logprobs::Logprobs;
|
||||
use vllm_engine_core_client::protocol::output::{EngineCoreFinishReason, StopReason};
|
||||
use vllm_engine_core_client::protocol::output::{
|
||||
EngineCoreFinishReason, RequestSpecDecodeMetrics, StopReason,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::sampling_mask::SamplingMask;
|
||||
use vllm_engine_core_client::{AbortCause, EngineCoreOutputStream};
|
||||
|
||||
@@ -47,6 +49,8 @@ pub struct CollectedGenerateOutput {
|
||||
pub ec_transfer_params: Option<serde_json::Value>,
|
||||
/// Sampling support sets aligned one-to-one with generated token positions.
|
||||
pub sampling_mask: Option<SamplingMask>,
|
||||
/// Per-request speculative-decoding metrics from the terminal output.
|
||||
pub spec_decode_metrics: Option<RequestSpecDecodeMetrics>,
|
||||
}
|
||||
|
||||
/// Prompt-scoped metadata emitted only once on the first [`GenerateOutput`] for
|
||||
@@ -160,6 +164,8 @@ pub struct GenerateOutput {
|
||||
pub ec_transfer_params: Option<serde_json::Value>,
|
||||
/// Sampling support sets aligned one-to-one with `token_ids`.
|
||||
pub sampling_mask: Option<SamplingMask>,
|
||||
/// Per-request speculative-decoding metrics, present on terminal outputs.
|
||||
pub spec_decode_metrics: Option<RequestSpecDecodeMetrics>,
|
||||
}
|
||||
|
||||
impl GenerateOutput {
|
||||
@@ -208,6 +214,7 @@ impl GenerateOutput {
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
spec_decode_metrics: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -308,6 +315,7 @@ impl Stream for GenerateOutputStream {
|
||||
kv_transfer_params: raw.kv_transfer_params,
|
||||
ec_transfer_params: raw.ec_transfer_params,
|
||||
sampling_mask,
|
||||
spec_decode_metrics: raw.spec_decode_metrics,
|
||||
};
|
||||
|
||||
Poll::Ready(Some(Ok(output)))
|
||||
@@ -396,6 +404,7 @@ impl<T: Stream<Item = Result<GenerateOutput>> + Send> T {
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
sampling_mask,
|
||||
spec_decode_metrics: None,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -409,6 +418,7 @@ impl<T: Stream<Item = Result<GenerateOutput>> + Send> T {
|
||||
};
|
||||
collected.kv_transfer_params = output.kv_transfer_params;
|
||||
collected.ec_transfer_params = output.ec_transfer_params;
|
||||
collected.spec_decode_metrics = output.spec_decode_metrics;
|
||||
if let Some(mask) = collected.sampling_mask.as_ref()
|
||||
&& mask.rows.len() != collected.token_ids.len()
|
||||
{
|
||||
|
||||
@@ -413,6 +413,7 @@ async fn collect_output_rejects_partial_sampling_mask() {
|
||||
sampling_mask: Some(SamplingMask {
|
||||
rows: vec![vec![1, 33, 99]],
|
||||
}),
|
||||
spec_decode_metrics: None,
|
||||
};
|
||||
|
||||
let error = futures::stream::iter([Ok(output)]).collect_output().await.unwrap_err();
|
||||
|
||||
@@ -29,7 +29,8 @@ use vllm_llm::{
|
||||
use self::convert::{ResponseOptions, prepare_generate_request};
|
||||
use self::types::{
|
||||
GenerateLogprob, GenerateResponse, GenerateResponseChoice, GenerateResponseStreamChoice,
|
||||
GenerateStreamResponse, MultiModalPlaceholders, PlaceholderRangeInfo,
|
||||
GenerateStreamResponse, MultiModalPlaceholders, PerRequestMetrics, PlaceholderRangeInfo,
|
||||
SpeculativeDecodingMetrics, StreamingSpeculativeDecodingMetrics,
|
||||
};
|
||||
pub(crate) use self::types::{GenerateRequest, GenerateSamplingParams};
|
||||
pub(crate) use self::validate::validate_request_compat;
|
||||
@@ -160,10 +161,14 @@ async fn generate_chunk_stream(
|
||||
let mut prompt_tokens = None;
|
||||
let mut prompt_token_ids = None;
|
||||
let mut usage = TokenUsage::default();
|
||||
let mut spec_decode_metrics = None;
|
||||
|
||||
while let Some(next) = stream.next().await {
|
||||
match next {
|
||||
Ok(output) => {
|
||||
if let Some(metrics) = output.spec_decode_metrics {
|
||||
spec_decode_metrics = Some(SpeculativeDecodingMetrics::from(metrics));
|
||||
}
|
||||
if prompt_tokens.is_none()
|
||||
&& let Some(info) = output.prompt_info.as_ref()
|
||||
{
|
||||
@@ -223,6 +228,7 @@ async fn generate_chunk_stream(
|
||||
.then(|| Usage::from_token_usage(usage, enable_prompt_tokens_details)),
|
||||
mm_placeholders: prompt_token_ids.as_ref().and_then(|_| mm_placeholders.take()),
|
||||
prompt_token_ids,
|
||||
metrics: None,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
@@ -243,6 +249,11 @@ async fn generate_chunk_stream(
|
||||
usage: Some(Usage::from_token_usage(usage, enable_prompt_tokens_details)),
|
||||
prompt_token_ids: None,
|
||||
mm_placeholders: None,
|
||||
metrics: spec_decode_metrics.map(|speculative_decoding| PerRequestMetrics {
|
||||
speculative_decoding: StreamingSpeculativeDecodingMetrics::from(
|
||||
speculative_decoding,
|
||||
),
|
||||
}),
|
||||
})
|
||||
.await;
|
||||
}
|
||||
@@ -318,6 +329,9 @@ fn collect_generate(
|
||||
mm_placeholders: return_token_ids.then_some(mm_placeholders).flatten(),
|
||||
kv_transfer_params: collected.kv_transfer_params,
|
||||
ec_transfer_params: collected.ec_transfer_params,
|
||||
metrics: collected.spec_decode_metrics.map(|metrics| PerRequestMetrics {
|
||||
speculative_decoding: SpeculativeDecodingMetrics::from(metrics),
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -465,6 +479,7 @@ mod tests {
|
||||
|
||||
use futures::{TryStreamExt as _, stream};
|
||||
use vllm_engine_core_client::protocol::multimodal::{MmModality, PlaceholderRange};
|
||||
use vllm_engine_core_client::protocol::output::RequestSpecDecodeMetrics;
|
||||
use vllm_llm::GeneratePromptInfo;
|
||||
|
||||
use super::*;
|
||||
@@ -482,6 +497,7 @@ mod tests {
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
spec_decode_metrics: None,
|
||||
}),
|
||||
Ok(GenerateOutput {
|
||||
request_id: String::new(),
|
||||
@@ -496,6 +512,7 @@ mod tests {
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
spec_decode_metrics: None,
|
||||
}),
|
||||
]);
|
||||
|
||||
@@ -578,6 +595,7 @@ mod tests {
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
spec_decode_metrics: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -645,6 +663,80 @@ mod tests {
|
||||
assert!(chunks[1].mm_placeholders.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn generate_chunk_stream_returns_terminal_spec_decode_metrics() {
|
||||
let metrics = RequestSpecDecodeMetrics {
|
||||
num_spec_tokens: 3,
|
||||
histogram: vec![0, 1, 1, 0],
|
||||
num_draft_tokens: 6,
|
||||
..Default::default()
|
||||
};
|
||||
let mut output = stream_output(None, Vec::new(), Some(FinishReason::stop_eos()));
|
||||
output.spec_decode_metrics = Some(metrics.clone());
|
||||
|
||||
for include_usage in [false, true] {
|
||||
let chunks: Vec<_> = generate_chunk_stream(
|
||||
stream::iter(vec![Ok(output.clone())]),
|
||||
"raw-stream".to_string(),
|
||||
ApiServerOptions::default(),
|
||||
ResponseOptions {
|
||||
include_usage,
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
)
|
||||
.try_collect()
|
||||
.await
|
||||
.expect("collect chunks");
|
||||
assert_eq!(chunks.len(), 1 + usize::from(include_usage));
|
||||
assert!(chunks[0].metrics.is_none());
|
||||
if include_usage {
|
||||
assert!(chunks[1].choices.is_empty());
|
||||
assert_eq!(
|
||||
chunks[1].metrics,
|
||||
Some(PerRequestMetrics {
|
||||
speculative_decoding: StreamingSpeculativeDecodingMetrics::from(
|
||||
SpeculativeDecodingMetrics::from(metrics.clone())
|
||||
)
|
||||
})
|
||||
);
|
||||
let json = serde_json::to_value(&chunks[1]).expect("serialize chunk");
|
||||
let stats = &json["metrics"]["speculative_decoding"];
|
||||
assert_eq!(stats.as_object().expect("metrics object").len(), 7);
|
||||
assert_eq!(stats["num_draft_tokens"], 6);
|
||||
assert_eq!(stats["num_accepted_draft_tokens"], 3);
|
||||
assert_eq!(stats["mean_acceptance_length"], 2.5);
|
||||
assert_eq!(
|
||||
stats["acceptance_histogram"],
|
||||
serde_json::json!([0, 1, 1, 0])
|
||||
);
|
||||
assert!(stats.get("per_step_accepted").is_none());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn spec_decode_metrics_zero_steps_and_detailed() {
|
||||
let empty = SpeculativeDecodingMetrics::from(RequestSpecDecodeMetrics {
|
||||
num_spec_tokens: 2,
|
||||
histogram: vec![0, 0, 0],
|
||||
..Default::default()
|
||||
});
|
||||
assert_eq!(empty.mean_acceptance_length, 1.0);
|
||||
assert_eq!(empty.draft_acceptance_rate, 0.0);
|
||||
assert_eq!(empty.per_step_accepted, None);
|
||||
|
||||
let detailed = SpeculativeDecodingMetrics::from(RequestSpecDecodeMetrics {
|
||||
num_spec_tokens: 2,
|
||||
histogram: vec![0, 1, 1],
|
||||
num_draft_tokens: 4,
|
||||
per_step_accepted: vec![1, 2],
|
||||
per_step_drafted: vec![2, 2],
|
||||
});
|
||||
assert_eq!(detailed.per_step_accepted, Some(vec![1, 2]));
|
||||
assert_eq!(detailed.per_step_drafted, Some(vec![2, 2]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collect_generate_omits_prompt_metadata_by_default() {
|
||||
let output = CollectedGenerateOutput {
|
||||
@@ -662,6 +754,7 @@ mod tests {
|
||||
ec_transfer_params: None,
|
||||
prompt_token_ids: vec![10, 20],
|
||||
sampling_mask: None,
|
||||
spec_decode_metrics: None,
|
||||
};
|
||||
|
||||
let response = collect_generate(
|
||||
@@ -683,6 +776,50 @@ mod tests {
|
||||
assert!(response.mm_placeholders.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collect_generate_returns_spec_decode_metrics() {
|
||||
let metrics = RequestSpecDecodeMetrics {
|
||||
num_spec_tokens: 3,
|
||||
histogram: vec![0, 1, 1, 0],
|
||||
num_draft_tokens: 6,
|
||||
..Default::default()
|
||||
};
|
||||
let output = CollectedGenerateOutput {
|
||||
request_id: "raw-metrics".to_string(),
|
||||
prompt_logprobs: None,
|
||||
token_ids: vec![30],
|
||||
logprobs: None,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
usage: TokenUsage::default(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
prompt_token_ids: vec![10],
|
||||
sampling_mask: None,
|
||||
spec_decode_metrics: Some(metrics.clone()),
|
||||
};
|
||||
|
||||
let response = collect_generate(
|
||||
output,
|
||||
"raw-metrics".to_string(),
|
||||
ApiServerOptions::default(),
|
||||
ResponseOptions::default(),
|
||||
None,
|
||||
)
|
||||
.expect("response");
|
||||
|
||||
assert_eq!(
|
||||
response.metrics,
|
||||
Some(PerRequestMetrics {
|
||||
speculative_decoding: SpeculativeDecodingMetrics::from(metrics)
|
||||
})
|
||||
);
|
||||
let json = serde_json::to_value(&response).expect("serialize response");
|
||||
let stats = &json["metrics"]["speculative_decoding"];
|
||||
assert_eq!(stats.as_object().expect("metrics object").len(), 9);
|
||||
assert!(stats["per_step_accepted"].is_null());
|
||||
assert!(stats["per_step_drafted"].is_null());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_mm_placeholders_returns_none_without_features() {
|
||||
assert!(extract_mm_placeholders(None).is_none());
|
||||
@@ -749,6 +886,7 @@ mod tests {
|
||||
ec_transfer_params: None,
|
||||
prompt_token_ids: vec![10, 20],
|
||||
sampling_mask: None,
|
||||
spec_decode_metrics: None,
|
||||
};
|
||||
|
||||
let response = collect_generate(
|
||||
@@ -782,6 +920,7 @@ mod tests {
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
spec_decode_metrics: None,
|
||||
prompt_token_ids,
|
||||
};
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ use llm_multimodal::MediaContentPart;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
use validator::Validate;
|
||||
use vllm_engine_core_client::protocol::output::RequestSpecDecodeMetrics;
|
||||
use vllm_text::SamplingParams;
|
||||
|
||||
use crate::routes::openai::utils::types::{ChatLogProbs, Normalizable, StreamOptions, Usage};
|
||||
@@ -84,6 +85,7 @@ pub(super) struct GenerateStreamResponse {
|
||||
pub usage: Option<Usage>,
|
||||
pub prompt_token_ids: Option<Vec<u32>>,
|
||||
pub mm_placeholders: Option<MultiModalPlaceholders>,
|
||||
pub metrics: Option<PerRequestMetrics<StreamingSpeculativeDecodingMetrics>>,
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `GenerateResponse` class.
|
||||
@@ -96,6 +98,91 @@ pub(super) struct GenerateResponse {
|
||||
pub mm_placeholders: Option<MultiModalPlaceholders>,
|
||||
pub kv_transfer_params: Option<Value>,
|
||||
pub ec_transfer_params: Option<Value>,
|
||||
pub metrics: Option<PerRequestMetrics<SpeculativeDecodingMetrics>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize)]
|
||||
pub(super) struct PerRequestMetrics<T> {
|
||||
pub speculative_decoding: T,
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `SpeculativeDecodingMetrics` class.
|
||||
///
|
||||
/// Derived from the raw engine accumulator the same way as Python
|
||||
/// `RequestSpecDecodeMetrics.to_dict`.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize)]
|
||||
pub(super) struct SpeculativeDecodingMetrics {
|
||||
pub mean_acceptance_length: f64,
|
||||
pub draft_acceptance_rate: f64,
|
||||
pub acceptance_histogram: Vec<u64>,
|
||||
pub num_spec_steps: u64,
|
||||
pub num_accepted_draft_tokens: u64,
|
||||
pub num_draft_tokens: u64,
|
||||
pub num_spec_tokens: u64,
|
||||
pub per_step_accepted: Option<Vec<u64>>,
|
||||
pub per_step_drafted: Option<Vec<u64>>,
|
||||
}
|
||||
|
||||
impl From<RequestSpecDecodeMetrics> for SpeculativeDecodingMetrics {
|
||||
fn from(raw: RequestSpecDecodeMetrics) -> Self {
|
||||
let num_spec_steps: u64 = raw.histogram.iter().sum();
|
||||
let num_accepted_draft_tokens: u64 =
|
||||
(0u64..).zip(&raw.histogram).map(|(accepted, count)| accepted * count).sum();
|
||||
let ratio = |num: u64, den: u64| {
|
||||
if den == 0 {
|
||||
0.0
|
||||
} else {
|
||||
num as f64 / den as f64
|
||||
}
|
||||
};
|
||||
let detailed = !raw.per_step_accepted.is_empty();
|
||||
Self {
|
||||
mean_acceptance_length: if num_spec_steps == 0 {
|
||||
1.0
|
||||
} else {
|
||||
1.0 + ratio(num_accepted_draft_tokens, num_spec_steps)
|
||||
},
|
||||
draft_acceptance_rate: ratio(num_accepted_draft_tokens, raw.num_draft_tokens),
|
||||
acceptance_histogram: raw.histogram,
|
||||
num_spec_steps,
|
||||
num_accepted_draft_tokens,
|
||||
num_draft_tokens: raw.num_draft_tokens,
|
||||
num_spec_tokens: raw.num_spec_tokens,
|
||||
per_step_accepted: detailed.then_some(raw.per_step_accepted),
|
||||
per_step_drafted: detailed.then_some(raw.per_step_drafted),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Streaming form omits detailed fields when summary metrics are requested.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, PartialEq, Serialize)]
|
||||
pub(super) struct StreamingSpeculativeDecodingMetrics {
|
||||
pub mean_acceptance_length: f64,
|
||||
pub draft_acceptance_rate: f64,
|
||||
pub acceptance_histogram: Vec<u64>,
|
||||
pub num_spec_steps: u64,
|
||||
pub num_accepted_draft_tokens: u64,
|
||||
pub num_draft_tokens: u64,
|
||||
pub num_spec_tokens: u64,
|
||||
pub per_step_accepted: Option<Vec<u64>>,
|
||||
pub per_step_drafted: Option<Vec<u64>>,
|
||||
}
|
||||
|
||||
impl From<SpeculativeDecodingMetrics> for StreamingSpeculativeDecodingMetrics {
|
||||
fn from(metrics: SpeculativeDecodingMetrics) -> Self {
|
||||
Self {
|
||||
mean_acceptance_length: metrics.mean_acceptance_length,
|
||||
draft_acceptance_rate: metrics.draft_acceptance_rate,
|
||||
acceptance_histogram: metrics.acceptance_histogram,
|
||||
num_spec_steps: metrics.num_spec_steps,
|
||||
num_accepted_draft_tokens: metrics.num_accepted_draft_tokens,
|
||||
num_draft_tokens: metrics.num_draft_tokens,
|
||||
num_spec_tokens: metrics.num_spec_tokens,
|
||||
per_step_accepted: metrics.per_step_accepted,
|
||||
per_step_drafted: metrics.per_step_drafted,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) type MultiModalPlaceholders = HashMap<String, Vec<PlaceholderRangeInfo>>;
|
||||
|
||||
@@ -52,6 +52,7 @@ def _make_generate_response(
|
||||
logprobs: dict | None = None,
|
||||
prompt_logprobs: list | None = None,
|
||||
kv_transfer_params: dict | None = None,
|
||||
metrics: dict | None = None,
|
||||
) -> dict:
|
||||
choice: dict = {
|
||||
"index": 0,
|
||||
@@ -64,6 +65,7 @@ def _make_generate_response(
|
||||
"choices": [choice],
|
||||
"prompt_logprobs": prompt_logprobs,
|
||||
"kv_transfer_params": kv_transfer_params,
|
||||
"metrics": metrics,
|
||||
}
|
||||
|
||||
|
||||
@@ -266,6 +268,49 @@ async def test_derender_chat_kv_transfer_params_passthrough(client):
|
||||
assert response.json()["kv_transfer_params"] == kv
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_metrics_passthrough(client):
|
||||
gen_req = await _render_chat(client)
|
||||
metrics = {
|
||||
"speculative_decoding": {
|
||||
"mean_acceptance_length": 2.0,
|
||||
"draft_acceptance_rate": 0.5,
|
||||
"acceptance_histogram": [0, 1],
|
||||
"num_spec_steps": 1,
|
||||
"num_accepted_draft_tokens": 1,
|
||||
"num_draft_tokens": 2,
|
||||
"num_spec_tokens": 1,
|
||||
}
|
||||
}
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_response": _make_generate_response(
|
||||
gen_req["token_ids"][:3], metrics=metrics
|
||||
),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
actual = response.json()["metrics"]
|
||||
assert set(actual) == {
|
||||
"time_to_first_token_ms",
|
||||
"generation_time_ms",
|
||||
"queue_time_ms",
|
||||
"mean_itl_ms",
|
||||
"tokens_per_second",
|
||||
"speculative_decoding",
|
||||
}
|
||||
assert all(
|
||||
actual[name] is None for name in actual if name != "speculative_decoding"
|
||||
)
|
||||
assert actual["speculative_decoding"] == {
|
||||
**metrics["speculative_decoding"],
|
||||
"per_step_accepted": None,
|
||||
"per_step_drafted": None,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_empty_token_ids(client):
|
||||
"""Empty token_ids list returns 400."""
|
||||
|
||||
@@ -27,6 +27,8 @@ from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaFunctionCall,
|
||||
DeltaMessage,
|
||||
DeltaToolCall,
|
||||
PerRequestMetrics,
|
||||
SpeculativeDecodingMetrics,
|
||||
)
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
DerenderStreamState,
|
||||
@@ -164,6 +166,7 @@ def _make_stream_chunk(
|
||||
finish_reason: str | None = None,
|
||||
request_id: str = "test-req",
|
||||
usage: dict | None = None,
|
||||
metrics: PerRequestMetrics | None = None,
|
||||
) -> GenerateStreamResponse:
|
||||
"""Build a GenerateStreamResponse SSE chunk."""
|
||||
return GenerateStreamResponse(
|
||||
@@ -176,6 +179,7 @@ def _make_stream_chunk(
|
||||
)
|
||||
],
|
||||
usage=UsageInfo(**usage) if usage else None,
|
||||
metrics=metrics,
|
||||
)
|
||||
|
||||
|
||||
@@ -512,6 +516,26 @@ class TestDerenderChatStream:
|
||||
assert chunk1.choices[0].delta.role == "assistant"
|
||||
assert chunk2.choices[0].delta.role is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metrics_passthrough(self, derenderer):
|
||||
metrics = PerRequestMetrics(
|
||||
speculative_decoding=SpeculativeDecodingMetrics(
|
||||
mean_acceptance_length=2.0,
|
||||
draft_acceptance_rate=0.5,
|
||||
acceptance_histogram=[0, 1],
|
||||
num_spec_steps=1,
|
||||
num_accepted_draft_tokens=1,
|
||||
num_draft_tokens=2,
|
||||
num_spec_tokens=1,
|
||||
)
|
||||
)
|
||||
chunk, _ = await derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk([], metrics=metrics),
|
||||
state=DerenderStreamState(),
|
||||
)
|
||||
assert chunk.metrics == metrics
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chunked_equals_oneshot(self, derenderer, tokenizer):
|
||||
"""Sum of streaming content deltas == one shot decode."""
|
||||
|
||||
@@ -28,6 +28,7 @@ from vllm.renderers import renderer_from_config
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
from vllm.v1.metrics.stats import RequestSpecDecodeMetrics
|
||||
|
||||
MODEL_NAME = "openai-community/gpt2"
|
||||
BASE_MODEL_PATHS = [
|
||||
@@ -130,6 +131,7 @@ def _make_request_output(
|
||||
logprobs: list[dict[int, Any] | None] | None = None,
|
||||
num_cached_tokens: int | None = None,
|
||||
index: int = 0,
|
||||
spec_decode_metrics: RequestSpecDecodeMetrics | None = None,
|
||||
) -> RequestOutput:
|
||||
return RequestOutput(
|
||||
request_id=request_id,
|
||||
@@ -144,6 +146,7 @@ def _make_request_output(
|
||||
cumulative_logprob=None,
|
||||
logprobs=logprobs,
|
||||
finish_reason=finish_reason,
|
||||
spec_decode_metrics=spec_decode_metrics,
|
||||
)
|
||||
],
|
||||
finished=finished,
|
||||
@@ -321,6 +324,145 @@ async def test_stream_basic():
|
||||
assert data_chunks[2]["choices"][0]["finish_reason"] == "stop"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
async def test_serve_tokens_returns_spec_decode_metrics(stream: bool):
|
||||
engine = _mock_engine()
|
||||
metrics = RequestSpecDecodeMetrics.new(num_spec_tokens=3)
|
||||
metrics.observe(num_draft_tokens=3, num_accepted=2)
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output(
|
||||
"req-1",
|
||||
token_ids=[10],
|
||||
finish_reason="stop",
|
||||
finished=True,
|
||||
spec_decode_metrics=metrics,
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
request = GenerateRequest(
|
||||
token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
model=MODEL_NAME,
|
||||
stream=stream,
|
||||
stream_options=StreamOptions(include_usage=True) if stream else None,
|
||||
)
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
if stream:
|
||||
chunks = [chunk async for chunk in response]
|
||||
payload = next(
|
||||
chunk
|
||||
for chunk in _parse_sse_chunks(chunks)
|
||||
if isinstance(chunk, dict) and chunk.get("metrics")
|
||||
)["metrics"]["speculative_decoding"]
|
||||
else:
|
||||
assert isinstance(response, GenerateResponse)
|
||||
assert response.metrics is not None
|
||||
assert response.metrics.speculative_decoding is not None
|
||||
payload = response.metrics.speculative_decoding.model_dump()
|
||||
|
||||
assert payload["num_draft_tokens"] == 3
|
||||
assert payload["num_accepted_draft_tokens"] == 2
|
||||
assert payload["num_spec_steps"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("include_usage", [False, True])
|
||||
async def test_stream_returns_spec_decode_metrics_on_empty_terminal_output(
|
||||
include_usage,
|
||||
):
|
||||
engine = _mock_engine()
|
||||
metrics = RequestSpecDecodeMetrics.new(num_spec_tokens=3)
|
||||
metrics.observe(num_draft_tokens=3, num_accepted=2)
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output("req-1", token_ids=[10])
|
||||
yield _make_request_output(
|
||||
"req-1",
|
||||
token_ids=[],
|
||||
finish_reason="stop",
|
||||
finished=True,
|
||||
spec_decode_metrics=metrics,
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
request = GenerateRequest(
|
||||
token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
model=MODEL_NAME,
|
||||
stream=True,
|
||||
stream_options=StreamOptions(include_usage=include_usage),
|
||||
)
|
||||
|
||||
chunks = [chunk async for chunk in await serving.serve_tokens(request)]
|
||||
data_chunks = [chunk for chunk in _parse_sse_chunks(chunks) if chunk != "[DONE]"]
|
||||
|
||||
assert "metrics" not in data_chunks[0]
|
||||
assert len(data_chunks) == 1 + int(include_usage)
|
||||
if include_usage:
|
||||
assert data_chunks[1]["choices"] == []
|
||||
assert (
|
||||
data_chunks[1]["metrics"]["speculative_decoding"]["num_draft_tokens"] == 3
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
async def test_serve_tokens_omits_spec_decode_metrics_for_parallel_sampling(
|
||||
stream: bool,
|
||||
):
|
||||
engine = _mock_engine()
|
||||
metrics = RequestSpecDecodeMetrics.new(num_spec_tokens=3)
|
||||
metrics.observe(num_draft_tokens=3, num_accepted=2)
|
||||
result = _make_request_output(
|
||||
"req-1",
|
||||
token_ids=[10],
|
||||
finish_reason="stop",
|
||||
finished=True,
|
||||
spec_decode_metrics=metrics,
|
||||
)
|
||||
result.outputs.append(
|
||||
CompletionOutput(
|
||||
index=1,
|
||||
text="",
|
||||
token_ids=[20],
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
finish_reason="stop",
|
||||
spec_decode_metrics=metrics,
|
||||
)
|
||||
)
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield result
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
request = GenerateRequest(
|
||||
token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=1, n=2),
|
||||
model=MODEL_NAME,
|
||||
stream=stream,
|
||||
stream_options=StreamOptions(include_usage=True) if stream else None,
|
||||
)
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
if stream:
|
||||
chunks = [chunk async for chunk in response]
|
||||
assert all(
|
||||
"metrics" not in chunk
|
||||
for chunk in _parse_sse_chunks(chunks)
|
||||
if isinstance(chunk, dict)
|
||||
)
|
||||
else:
|
||||
assert isinstance(response, GenerateResponse)
|
||||
assert response.metrics is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_stream_returns_final_prompt_token_ids_when_requested():
|
||||
engine = _mock_engine()
|
||||
|
||||
@@ -179,6 +179,7 @@ class ServingDerender(BaseServing):
|
||||
usage=usage,
|
||||
prompt_logprobs=gen.prompt_logprobs,
|
||||
kv_transfer_params=gen.kv_transfer_params,
|
||||
metrics=gen.metrics,
|
||||
)
|
||||
|
||||
async def derender_completion_response(
|
||||
@@ -245,6 +246,9 @@ class ServingDerender(BaseServing):
|
||||
choices=choices,
|
||||
usage=usage,
|
||||
kv_transfer_params=kv_params,
|
||||
# Metrics describe one prompt. Parallel samples are already
|
||||
# suppressed by /generate; multi-prompt responses cannot be merged.
|
||||
metrics=first.metrics if len(request.generate_responses) == 1 else None,
|
||||
)
|
||||
|
||||
async def derender_chat_stream_response(
|
||||
|
||||
@@ -11,7 +11,11 @@ from pydantic import (
|
||||
)
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.generate.base.protocol import StreamOptions, validate_cache_salt
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
PerRequestMetrics,
|
||||
StreamOptions,
|
||||
validate_cache_salt,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionLogProbs,
|
||||
ChatCompletionRequest,
|
||||
@@ -360,6 +364,7 @@ class GenerateStreamResponse(BaseModel):
|
||||
usage: UsageInfo | None = Field(default=None)
|
||||
prompt_token_ids: list[int] | None = None
|
||||
mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = None
|
||||
metrics: PerRequestMetrics | None = None
|
||||
|
||||
|
||||
class GenerateResponse(BaseModel):
|
||||
@@ -378,6 +383,7 @@ class GenerateResponse(BaseModel):
|
||||
prompt_logprobs: list[dict[int, Logprob] | None] | None = None
|
||||
prompt_token_ids: list[int] | None = None
|
||||
mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = None
|
||||
metrics: PerRequestMetrics | None = None
|
||||
|
||||
kv_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
|
||||
@@ -12,9 +12,13 @@ from fastapi import Request
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.chat_utils import AsyncMultiModalItemTracker
|
||||
from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
PerRequestMetrics,
|
||||
RequestResponseMetadata,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.serving import (
|
||||
GenerateBaseServing,
|
||||
build_spec_decoding_metrics,
|
||||
clamp_prompt_logprobs,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
@@ -373,6 +377,11 @@ class ServingTokens(GenerateBaseServing):
|
||||
|
||||
request_metadata.final_usage_info = usage
|
||||
|
||||
per_request_metrics = None
|
||||
if request.sampling_params.n == 1:
|
||||
spec_stats = build_spec_decoding_metrics(final_res)
|
||||
if spec_stats is not None:
|
||||
per_request_metrics = PerRequestMetrics(speculative_decoding=spec_stats)
|
||||
response = GenerateResponse(
|
||||
request_id=request_id,
|
||||
created=created_time,
|
||||
@@ -384,6 +393,7 @@ class ServingTokens(GenerateBaseServing):
|
||||
final_res.prompt_token_ids if request.return_token_ids else None
|
||||
),
|
||||
mm_placeholders=request._response_mm_placeholders,
|
||||
metrics=per_request_metrics,
|
||||
kv_transfer_params=final_res.kv_transfer_params,
|
||||
ec_transfer_params=final_res.ec_transfer_params,
|
||||
)
|
||||
@@ -423,6 +433,7 @@ class ServingTokens(GenerateBaseServing):
|
||||
prompt_token_ids: list[int] | None = None
|
||||
num_cached_tokens = None
|
||||
sampling_params: SamplingParams = request.sampling_params
|
||||
last_res: RequestOutput | None = None
|
||||
|
||||
include_usage, include_continuous_usage = should_include_usage(
|
||||
request.stream_options, False
|
||||
@@ -430,6 +441,7 @@ class ServingTokens(GenerateBaseServing):
|
||||
|
||||
try:
|
||||
async for res in result_generator:
|
||||
last_res = res
|
||||
if first_iteration:
|
||||
if res.prompt_token_ids is not None:
|
||||
num_prompt_tokens = len(res.prompt_token_ids)
|
||||
@@ -496,10 +508,10 @@ class ServingTokens(GenerateBaseServing):
|
||||
total_tokens=(num_prompt_tokens + num_generated_tokens[i]),
|
||||
)
|
||||
|
||||
# Omit absent prompt metadata (Rust skips None fields).
|
||||
# Omit fields that are absent from token-bearing chunks.
|
||||
exclude = {
|
||||
name
|
||||
for name in ("prompt_token_ids", "mm_placeholders")
|
||||
for name in ("prompt_token_ids", "mm_placeholders", "metrics")
|
||||
if getattr(chunk, name) is None
|
||||
}
|
||||
yield f"data: {chunk.model_dump_json(exclude=exclude)}\n\n"
|
||||
@@ -517,10 +529,18 @@ class ServingTokens(GenerateBaseServing):
|
||||
)
|
||||
|
||||
if include_usage:
|
||||
per_request_metrics = None
|
||||
if sampling_params.n == 1:
|
||||
spec_stats = build_spec_decoding_metrics(last_res)
|
||||
if spec_stats is not None:
|
||||
per_request_metrics = PerRequestMetrics(
|
||||
speculative_decoding=spec_stats
|
||||
)
|
||||
final_chunk = GenerateStreamResponse(
|
||||
request_id=request_id,
|
||||
choices=[],
|
||||
usage=final_usage_info,
|
||||
metrics=per_request_metrics,
|
||||
)
|
||||
yield f"data: {final_chunk.model_dump_json(exclude_none=True)}\n\n"
|
||||
|
||||
|
||||
@@ -398,6 +398,7 @@ class OnlineDerenderer:
|
||||
model=model,
|
||||
choices=stream_choices,
|
||||
usage=usage,
|
||||
metrics=generate_chunk.metrics,
|
||||
)
|
||||
return chunk, updated_state
|
||||
|
||||
@@ -595,6 +596,7 @@ class OnlineDerenderer:
|
||||
model=model,
|
||||
choices=stream_choices,
|
||||
usage=usage,
|
||||
metrics=generate_chunk.metrics,
|
||||
)
|
||||
return chunk, updated_state
|
||||
|
||||
@@ -741,6 +743,7 @@ class OnlineDerenderer:
|
||||
model=model,
|
||||
choices=stream_choices,
|
||||
usage=usage,
|
||||
metrics=generate_chunk.metrics,
|
||||
)
|
||||
return chunk, updated_state
|
||||
|
||||
|
||||
Reference in New Issue
Block a user