From 10e6a7f21094b76c65efb029b39b3f2227516418 Mon Sep 17 00:00:00 2001 From: Hongxin Xu <70438206+xhx1022@users.noreply.github.com> Date: Sun, 20 Sep 2026 20:21:33 +0800 Subject: [PATCH] [Frontend] Expose per-request spec decode metrics in generate API (#43310) Signed-off-by: aoshen02 Co-authored-by: aoshen02 Co-authored-by: Codex --- .../engine-core-client/src/protocol/output.rs | 32 +++- rust/src/llm/src/output.rs | 12 +- rust/src/llm/tests/generate.rs | 1 + .../server/src/routes/inference/generate.rs | 141 ++++++++++++++++- .../src/routes/inference/generate/types.rs | 87 +++++++++++ .../scale_out/derender/test_derender.py | 45 ++++++ .../derender/test_derender_stream.py | 24 +++ .../test_generate_stream.py | 142 ++++++++++++++++++ .../entrypoints/scale_out/derender/serving.py | 4 + .../scale_out/token_in_token_out/protocol.py | 8 +- .../scale_out/token_in_token_out/serving.py | 26 +++- vllm/renderers/online_derenderer.py | 3 + 12 files changed, 516 insertions(+), 9 deletions(-) diff --git a/rust/src/engine-core-client/src/protocol/output.rs b/rust/src/engine-core-client/src/protocol/output.rs index a45cbe3731..0edd0ce788 100644 --- a/rust/src/engine-core-client/src/protocol/output.rs +++ b/rust/src/engine-core-client/src/protocol/output.rs @@ -130,10 +130,36 @@ pub struct EngineCoreOutput { #[serde(default)] pub new_sampling_mask: Option, /// Per-request speculative-decoding acceptance metrics, set on the final - /// output when `--per-request-spec-decode-metrics` is enabled. Opaque here; - /// the Rust frontend does not yet surface it in responses. + /// output when `--per-request-spec-decode-metrics` is enabled. #[serde(default)] - pub spec_decode_metrics: Option, + pub spec_decode_metrics: Option, +} + +/// Raw per-sequence speculative-decoding accumulator. +/// +/// Python models this as a plain `@dataclass`, so it is serialized by msgspec +/// as a map (named fields). Derived values such as acceptance rates are +/// computed by the frontend, not carried on the wire. +/// +/// Original Python definition: `RequestSpecDecodeMetrics` in +/// `vllm/v1/metrics/stats.py`. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct RequestSpecDecodeMetrics { + /// Configured `num_speculative_tokens`. + #[serde(default)] + pub num_spec_tokens: u64, + /// Verify-step counts indexed by accepted draft-token count. + #[serde(default)] + pub histogram: Vec, + /// Total proposed draft tokens. + #[serde(default)] + pub num_draft_tokens: u64, + /// Accepted draft count per verify step; empty unless `detailed`. + #[serde(default)] + pub per_step_accepted: Vec, + /// Proposed draft count per verify step; empty unless `detailed`. + #[serde(default)] + pub per_step_drafted: Vec, } impl EngineCoreOutput { diff --git a/rust/src/llm/src/output.rs b/rust/src/llm/src/output.rs index d0cab4a507..34eae25682 100644 --- a/rust/src/llm/src/output.rs +++ b/rust/src/llm/src/output.rs @@ -11,7 +11,9 @@ use futures::stream::FusedStream; use futures::{Stream, StreamExt as _, pin_mut}; use serde::{Deserialize, Serialize}; use vllm_engine_core_client::protocol::logprobs::Logprobs; -use vllm_engine_core_client::protocol::output::{EngineCoreFinishReason, StopReason}; +use vllm_engine_core_client::protocol::output::{ + EngineCoreFinishReason, RequestSpecDecodeMetrics, StopReason, +}; use vllm_engine_core_client::protocol::sampling_mask::SamplingMask; use vllm_engine_core_client::{AbortCause, EngineCoreOutputStream}; @@ -47,6 +49,8 @@ pub struct CollectedGenerateOutput { pub ec_transfer_params: Option, /// Sampling support sets aligned one-to-one with generated token positions. pub sampling_mask: Option, + /// Per-request speculative-decoding metrics from the terminal output. + pub spec_decode_metrics: Option, } /// Prompt-scoped metadata emitted only once on the first [`GenerateOutput`] for @@ -160,6 +164,8 @@ pub struct GenerateOutput { pub ec_transfer_params: Option, /// Sampling support sets aligned one-to-one with `token_ids`. pub sampling_mask: Option, + /// Per-request speculative-decoding metrics, present on terminal outputs. + pub spec_decode_metrics: Option, } impl GenerateOutput { @@ -208,6 +214,7 @@ impl GenerateOutput { kv_transfer_params: None, ec_transfer_params: None, sampling_mask: None, + spec_decode_metrics: None, } } } @@ -308,6 +315,7 @@ impl Stream for GenerateOutputStream { kv_transfer_params: raw.kv_transfer_params, ec_transfer_params: raw.ec_transfer_params, sampling_mask, + spec_decode_metrics: raw.spec_decode_metrics, }; Poll::Ready(Some(Ok(output))) @@ -396,6 +404,7 @@ impl> + Send> T { kv_transfer_params: None, ec_transfer_params: None, sampling_mask, + spec_decode_metrics: None, }); } @@ -409,6 +418,7 @@ impl> + Send> T { }; collected.kv_transfer_params = output.kv_transfer_params; collected.ec_transfer_params = output.ec_transfer_params; + collected.spec_decode_metrics = output.spec_decode_metrics; if let Some(mask) = collected.sampling_mask.as_ref() && mask.rows.len() != collected.token_ids.len() { diff --git a/rust/src/llm/tests/generate.rs b/rust/src/llm/tests/generate.rs index 09b5199767..c341737a3c 100644 --- a/rust/src/llm/tests/generate.rs +++ b/rust/src/llm/tests/generate.rs @@ -413,6 +413,7 @@ async fn collect_output_rejects_partial_sampling_mask() { sampling_mask: Some(SamplingMask { rows: vec![vec![1, 33, 99]], }), + spec_decode_metrics: None, }; let error = futures::stream::iter([Ok(output)]).collect_output().await.unwrap_err(); diff --git a/rust/src/server/src/routes/inference/generate.rs b/rust/src/server/src/routes/inference/generate.rs index f82059dffd..eb3338a67f 100644 --- a/rust/src/server/src/routes/inference/generate.rs +++ b/rust/src/server/src/routes/inference/generate.rs @@ -29,7 +29,8 @@ use vllm_llm::{ use self::convert::{ResponseOptions, prepare_generate_request}; use self::types::{ GenerateLogprob, GenerateResponse, GenerateResponseChoice, GenerateResponseStreamChoice, - GenerateStreamResponse, MultiModalPlaceholders, PlaceholderRangeInfo, + GenerateStreamResponse, MultiModalPlaceholders, PerRequestMetrics, PlaceholderRangeInfo, + SpeculativeDecodingMetrics, StreamingSpeculativeDecodingMetrics, }; pub(crate) use self::types::{GenerateRequest, GenerateSamplingParams}; pub(crate) use self::validate::validate_request_compat; @@ -160,10 +161,14 @@ async fn generate_chunk_stream( let mut prompt_tokens = None; let mut prompt_token_ids = None; let mut usage = TokenUsage::default(); + let mut spec_decode_metrics = None; while let Some(next) = stream.next().await { match next { Ok(output) => { + if let Some(metrics) = output.spec_decode_metrics { + spec_decode_metrics = Some(SpeculativeDecodingMetrics::from(metrics)); + } if prompt_tokens.is_none() && let Some(info) = output.prompt_info.as_ref() { @@ -223,6 +228,7 @@ async fn generate_chunk_stream( .then(|| Usage::from_token_usage(usage, enable_prompt_tokens_details)), mm_placeholders: prompt_token_ids.as_ref().and_then(|_| mm_placeholders.take()), prompt_token_ids, + metrics: None, }) .await; } @@ -243,6 +249,11 @@ async fn generate_chunk_stream( usage: Some(Usage::from_token_usage(usage, enable_prompt_tokens_details)), prompt_token_ids: None, mm_placeholders: None, + metrics: spec_decode_metrics.map(|speculative_decoding| PerRequestMetrics { + speculative_decoding: StreamingSpeculativeDecodingMetrics::from( + speculative_decoding, + ), + }), }) .await; } @@ -318,6 +329,9 @@ fn collect_generate( mm_placeholders: return_token_ids.then_some(mm_placeholders).flatten(), kv_transfer_params: collected.kv_transfer_params, ec_transfer_params: collected.ec_transfer_params, + metrics: collected.spec_decode_metrics.map(|metrics| PerRequestMetrics { + speculative_decoding: SpeculativeDecodingMetrics::from(metrics), + }), }) } @@ -465,6 +479,7 @@ mod tests { use futures::{TryStreamExt as _, stream}; use vllm_engine_core_client::protocol::multimodal::{MmModality, PlaceholderRange}; + use vllm_engine_core_client::protocol::output::RequestSpecDecodeMetrics; use vllm_llm::GeneratePromptInfo; use super::*; @@ -482,6 +497,7 @@ mod tests { kv_transfer_params: None, ec_transfer_params: None, sampling_mask: None, + spec_decode_metrics: None, }), Ok(GenerateOutput { request_id: String::new(), @@ -496,6 +512,7 @@ mod tests { kv_transfer_params: None, ec_transfer_params: None, sampling_mask: None, + spec_decode_metrics: None, }), ]); @@ -578,6 +595,7 @@ mod tests { kv_transfer_params: None, ec_transfer_params: None, sampling_mask: None, + spec_decode_metrics: None, } } @@ -645,6 +663,80 @@ mod tests { assert!(chunks[1].mm_placeholders.is_none()); } + #[tokio::test] + async fn generate_chunk_stream_returns_terminal_spec_decode_metrics() { + let metrics = RequestSpecDecodeMetrics { + num_spec_tokens: 3, + histogram: vec![0, 1, 1, 0], + num_draft_tokens: 6, + ..Default::default() + }; + let mut output = stream_output(None, Vec::new(), Some(FinishReason::stop_eos())); + output.spec_decode_metrics = Some(metrics.clone()); + + for include_usage in [false, true] { + let chunks: Vec<_> = generate_chunk_stream( + stream::iter(vec![Ok(output.clone())]), + "raw-stream".to_string(), + ApiServerOptions::default(), + ResponseOptions { + include_usage, + ..Default::default() + }, + None, + ) + .try_collect() + .await + .expect("collect chunks"); + assert_eq!(chunks.len(), 1 + usize::from(include_usage)); + assert!(chunks[0].metrics.is_none()); + if include_usage { + assert!(chunks[1].choices.is_empty()); + assert_eq!( + chunks[1].metrics, + Some(PerRequestMetrics { + speculative_decoding: StreamingSpeculativeDecodingMetrics::from( + SpeculativeDecodingMetrics::from(metrics.clone()) + ) + }) + ); + let json = serde_json::to_value(&chunks[1]).expect("serialize chunk"); + let stats = &json["metrics"]["speculative_decoding"]; + assert_eq!(stats.as_object().expect("metrics object").len(), 7); + assert_eq!(stats["num_draft_tokens"], 6); + assert_eq!(stats["num_accepted_draft_tokens"], 3); + assert_eq!(stats["mean_acceptance_length"], 2.5); + assert_eq!( + stats["acceptance_histogram"], + serde_json::json!([0, 1, 1, 0]) + ); + assert!(stats.get("per_step_accepted").is_none()); + } + } + } + + #[test] + fn spec_decode_metrics_zero_steps_and_detailed() { + let empty = SpeculativeDecodingMetrics::from(RequestSpecDecodeMetrics { + num_spec_tokens: 2, + histogram: vec![0, 0, 0], + ..Default::default() + }); + assert_eq!(empty.mean_acceptance_length, 1.0); + assert_eq!(empty.draft_acceptance_rate, 0.0); + assert_eq!(empty.per_step_accepted, None); + + let detailed = SpeculativeDecodingMetrics::from(RequestSpecDecodeMetrics { + num_spec_tokens: 2, + histogram: vec![0, 1, 1], + num_draft_tokens: 4, + per_step_accepted: vec![1, 2], + per_step_drafted: vec![2, 2], + }); + assert_eq!(detailed.per_step_accepted, Some(vec![1, 2])); + assert_eq!(detailed.per_step_drafted, Some(vec![2, 2])); + } + #[test] fn collect_generate_omits_prompt_metadata_by_default() { let output = CollectedGenerateOutput { @@ -662,6 +754,7 @@ mod tests { ec_transfer_params: None, prompt_token_ids: vec![10, 20], sampling_mask: None, + spec_decode_metrics: None, }; let response = collect_generate( @@ -683,6 +776,50 @@ mod tests { assert!(response.mm_placeholders.is_none()); } + #[test] + fn collect_generate_returns_spec_decode_metrics() { + let metrics = RequestSpecDecodeMetrics { + num_spec_tokens: 3, + histogram: vec![0, 1, 1, 0], + num_draft_tokens: 6, + ..Default::default() + }; + let output = CollectedGenerateOutput { + request_id: "raw-metrics".to_string(), + prompt_logprobs: None, + token_ids: vec![30], + logprobs: None, + finish_reason: FinishReason::stop_eos(), + usage: TokenUsage::default(), + kv_transfer_params: None, + ec_transfer_params: None, + prompt_token_ids: vec![10], + sampling_mask: None, + spec_decode_metrics: Some(metrics.clone()), + }; + + let response = collect_generate( + output, + "raw-metrics".to_string(), + ApiServerOptions::default(), + ResponseOptions::default(), + None, + ) + .expect("response"); + + assert_eq!( + response.metrics, + Some(PerRequestMetrics { + speculative_decoding: SpeculativeDecodingMetrics::from(metrics) + }) + ); + let json = serde_json::to_value(&response).expect("serialize response"); + let stats = &json["metrics"]["speculative_decoding"]; + assert_eq!(stats.as_object().expect("metrics object").len(), 9); + assert!(stats["per_step_accepted"].is_null()); + assert!(stats["per_step_drafted"].is_null()); + } + #[test] fn extract_mm_placeholders_returns_none_without_features() { assert!(extract_mm_placeholders(None).is_none()); @@ -749,6 +886,7 @@ mod tests { ec_transfer_params: None, prompt_token_ids: vec![10, 20], sampling_mask: None, + spec_decode_metrics: None, }; let response = collect_generate( @@ -782,6 +920,7 @@ mod tests { kv_transfer_params: None, ec_transfer_params: None, sampling_mask: None, + spec_decode_metrics: None, prompt_token_ids, }; diff --git a/rust/src/server/src/routes/inference/generate/types.rs b/rust/src/server/src/routes/inference/generate/types.rs index 406ec58569..86d8ec00f4 100644 --- a/rust/src/server/src/routes/inference/generate/types.rs +++ b/rust/src/server/src/routes/inference/generate/types.rs @@ -7,6 +7,7 @@ use llm_multimodal::MediaContentPart; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use validator::Validate; +use vllm_engine_core_client::protocol::output::RequestSpecDecodeMetrics; use vllm_text::SamplingParams; use crate::routes::openai::utils::types::{ChatLogProbs, Normalizable, StreamOptions, Usage}; @@ -84,6 +85,7 @@ pub(super) struct GenerateStreamResponse { pub usage: Option, pub prompt_token_ids: Option>, pub mm_placeholders: Option, + pub metrics: Option>, } /// Mirrors the Python vLLM `GenerateResponse` class. @@ -96,6 +98,91 @@ pub(super) struct GenerateResponse { pub mm_placeholders: Option, pub kv_transfer_params: Option, pub ec_transfer_params: Option, + pub metrics: Option>, +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +pub(super) struct PerRequestMetrics { + pub speculative_decoding: T, +} + +/// Mirrors the Python vLLM `SpeculativeDecodingMetrics` class. +/// +/// Derived from the raw engine accumulator the same way as Python +/// `RequestSpecDecodeMetrics.to_dict`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub(super) struct SpeculativeDecodingMetrics { + pub mean_acceptance_length: f64, + pub draft_acceptance_rate: f64, + pub acceptance_histogram: Vec, + pub num_spec_steps: u64, + pub num_accepted_draft_tokens: u64, + pub num_draft_tokens: u64, + pub num_spec_tokens: u64, + pub per_step_accepted: Option>, + pub per_step_drafted: Option>, +} + +impl From for SpeculativeDecodingMetrics { + fn from(raw: RequestSpecDecodeMetrics) -> Self { + let num_spec_steps: u64 = raw.histogram.iter().sum(); + let num_accepted_draft_tokens: u64 = + (0u64..).zip(&raw.histogram).map(|(accepted, count)| accepted * count).sum(); + let ratio = |num: u64, den: u64| { + if den == 0 { + 0.0 + } else { + num as f64 / den as f64 + } + }; + let detailed = !raw.per_step_accepted.is_empty(); + Self { + mean_acceptance_length: if num_spec_steps == 0 { + 1.0 + } else { + 1.0 + ratio(num_accepted_draft_tokens, num_spec_steps) + }, + draft_acceptance_rate: ratio(num_accepted_draft_tokens, raw.num_draft_tokens), + acceptance_histogram: raw.histogram, + num_spec_steps, + num_accepted_draft_tokens, + num_draft_tokens: raw.num_draft_tokens, + num_spec_tokens: raw.num_spec_tokens, + per_step_accepted: detailed.then_some(raw.per_step_accepted), + per_step_drafted: detailed.then_some(raw.per_step_drafted), + } + } +} + +/// Streaming form omits detailed fields when summary metrics are requested. +#[serde_with::skip_serializing_none] +#[derive(Debug, Clone, PartialEq, Serialize)] +pub(super) struct StreamingSpeculativeDecodingMetrics { + pub mean_acceptance_length: f64, + pub draft_acceptance_rate: f64, + pub acceptance_histogram: Vec, + pub num_spec_steps: u64, + pub num_accepted_draft_tokens: u64, + pub num_draft_tokens: u64, + pub num_spec_tokens: u64, + pub per_step_accepted: Option>, + pub per_step_drafted: Option>, +} + +impl From for StreamingSpeculativeDecodingMetrics { + fn from(metrics: SpeculativeDecodingMetrics) -> Self { + Self { + mean_acceptance_length: metrics.mean_acceptance_length, + draft_acceptance_rate: metrics.draft_acceptance_rate, + acceptance_histogram: metrics.acceptance_histogram, + num_spec_steps: metrics.num_spec_steps, + num_accepted_draft_tokens: metrics.num_accepted_draft_tokens, + num_draft_tokens: metrics.num_draft_tokens, + num_spec_tokens: metrics.num_spec_tokens, + per_step_accepted: metrics.per_step_accepted, + per_step_drafted: metrics.per_step_drafted, + } + } } pub(super) type MultiModalPlaceholders = HashMap>; diff --git a/tests/entrypoints/scale_out/derender/test_derender.py b/tests/entrypoints/scale_out/derender/test_derender.py index 9cc6124393..80ecc0f49c 100644 --- a/tests/entrypoints/scale_out/derender/test_derender.py +++ b/tests/entrypoints/scale_out/derender/test_derender.py @@ -52,6 +52,7 @@ def _make_generate_response( logprobs: dict | None = None, prompt_logprobs: list | None = None, kv_transfer_params: dict | None = None, + metrics: dict | None = None, ) -> dict: choice: dict = { "index": 0, @@ -64,6 +65,7 @@ def _make_generate_response( "choices": [choice], "prompt_logprobs": prompt_logprobs, "kv_transfer_params": kv_transfer_params, + "metrics": metrics, } @@ -266,6 +268,49 @@ async def test_derender_chat_kv_transfer_params_passthrough(client): assert response.json()["kv_transfer_params"] == kv +@pytest.mark.asyncio +async def test_derender_chat_metrics_passthrough(client): + gen_req = await _render_chat(client) + metrics = { + "speculative_decoding": { + "mean_acceptance_length": 2.0, + "draft_acceptance_rate": 0.5, + "acceptance_histogram": [0, 1], + "num_spec_steps": 1, + "num_accepted_draft_tokens": 1, + "num_draft_tokens": 2, + "num_spec_tokens": 1, + } + } + response = await client.post( + "/v1/chat/completions/derender", + json={ + "model": MODEL_NAME, + "generate_response": _make_generate_response( + gen_req["token_ids"][:3], metrics=metrics + ), + }, + ) + assert response.status_code == 200 + actual = response.json()["metrics"] + assert set(actual) == { + "time_to_first_token_ms", + "generation_time_ms", + "queue_time_ms", + "mean_itl_ms", + "tokens_per_second", + "speculative_decoding", + } + assert all( + actual[name] is None for name in actual if name != "speculative_decoding" + ) + assert actual["speculative_decoding"] == { + **metrics["speculative_decoding"], + "per_step_accepted": None, + "per_step_drafted": None, + } + + @pytest.mark.asyncio async def test_derender_chat_empty_token_ids(client): """Empty token_ids list returns 400.""" diff --git a/tests/entrypoints/scale_out/derender/test_derender_stream.py b/tests/entrypoints/scale_out/derender/test_derender_stream.py index 4032d9cbf3..e7a2e3b856 100644 --- a/tests/entrypoints/scale_out/derender/test_derender_stream.py +++ b/tests/entrypoints/scale_out/derender/test_derender_stream.py @@ -27,6 +27,8 @@ from vllm.entrypoints.generate.base.protocol import ( DeltaFunctionCall, DeltaMessage, DeltaToolCall, + PerRequestMetrics, + SpeculativeDecodingMetrics, ) from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( DerenderStreamState, @@ -164,6 +166,7 @@ def _make_stream_chunk( finish_reason: str | None = None, request_id: str = "test-req", usage: dict | None = None, + metrics: PerRequestMetrics | None = None, ) -> GenerateStreamResponse: """Build a GenerateStreamResponse SSE chunk.""" return GenerateStreamResponse( @@ -176,6 +179,7 @@ def _make_stream_chunk( ) ], usage=UsageInfo(**usage) if usage else None, + metrics=metrics, ) @@ -512,6 +516,26 @@ class TestDerenderChatStream: assert chunk1.choices[0].delta.role == "assistant" assert chunk2.choices[0].delta.role is None + @pytest.mark.asyncio + async def test_metrics_passthrough(self, derenderer): + metrics = PerRequestMetrics( + speculative_decoding=SpeculativeDecodingMetrics( + mean_acceptance_length=2.0, + draft_acceptance_rate=0.5, + acceptance_histogram=[0, 1], + num_spec_steps=1, + num_accepted_draft_tokens=1, + num_draft_tokens=2, + num_spec_tokens=1, + ) + ) + chunk, _ = await derenderer.derender_chat_stream( + model=MODEL_NAME, + generate_chunk=_make_stream_chunk([], metrics=metrics), + state=DerenderStreamState(), + ) + assert chunk.metrics == metrics + @pytest.mark.asyncio async def test_chunked_equals_oneshot(self, derenderer, tokenizer): """Sum of streaming content deltas == one shot decode.""" diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py b/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py index d9ae424cc8..ba5839990c 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py @@ -28,6 +28,7 @@ from vllm.renderers import renderer_from_config from vllm.renderers.online_renderer import OnlineRenderer from vllm.sampling_params import SamplingParams from vllm.v1.engine.async_llm import AsyncLLM +from vllm.v1.metrics.stats import RequestSpecDecodeMetrics MODEL_NAME = "openai-community/gpt2" BASE_MODEL_PATHS = [ @@ -130,6 +131,7 @@ def _make_request_output( logprobs: list[dict[int, Any] | None] | None = None, num_cached_tokens: int | None = None, index: int = 0, + spec_decode_metrics: RequestSpecDecodeMetrics | None = None, ) -> RequestOutput: return RequestOutput( request_id=request_id, @@ -144,6 +146,7 @@ def _make_request_output( cumulative_logprob=None, logprobs=logprobs, finish_reason=finish_reason, + spec_decode_metrics=spec_decode_metrics, ) ], finished=finished, @@ -321,6 +324,145 @@ async def test_stream_basic(): assert data_chunks[2]["choices"][0]["finish_reason"] == "stop" +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +async def test_serve_tokens_returns_spec_decode_metrics(stream: bool): + engine = _mock_engine() + metrics = RequestSpecDecodeMetrics.new(num_spec_tokens=3) + metrics.observe(num_draft_tokens=3, num_accepted=2) + + async def mock_generate(*args, **kwargs): + yield _make_request_output( + "req-1", + token_ids=[10], + finish_reason="stop", + finished=True, + spec_decode_metrics=metrics, + ) + + engine.generate = MagicMock(side_effect=mock_generate) + serving = _build_serving_tokens(engine) + request = GenerateRequest( + token_ids=[1, 2, 3], + sampling_params=SamplingParams(max_tokens=1), + model=MODEL_NAME, + stream=stream, + stream_options=StreamOptions(include_usage=True) if stream else None, + ) + + response = await serving.serve_tokens(request) + if stream: + chunks = [chunk async for chunk in response] + payload = next( + chunk + for chunk in _parse_sse_chunks(chunks) + if isinstance(chunk, dict) and chunk.get("metrics") + )["metrics"]["speculative_decoding"] + else: + assert isinstance(response, GenerateResponse) + assert response.metrics is not None + assert response.metrics.speculative_decoding is not None + payload = response.metrics.speculative_decoding.model_dump() + + assert payload["num_draft_tokens"] == 3 + assert payload["num_accepted_draft_tokens"] == 2 + assert payload["num_spec_steps"] == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("include_usage", [False, True]) +async def test_stream_returns_spec_decode_metrics_on_empty_terminal_output( + include_usage, +): + engine = _mock_engine() + metrics = RequestSpecDecodeMetrics.new(num_spec_tokens=3) + metrics.observe(num_draft_tokens=3, num_accepted=2) + + async def mock_generate(*args, **kwargs): + yield _make_request_output("req-1", token_ids=[10]) + yield _make_request_output( + "req-1", + token_ids=[], + finish_reason="stop", + finished=True, + spec_decode_metrics=metrics, + ) + + engine.generate = MagicMock(side_effect=mock_generate) + serving = _build_serving_tokens(engine) + request = GenerateRequest( + token_ids=[1, 2, 3], + sampling_params=SamplingParams(max_tokens=1), + model=MODEL_NAME, + stream=True, + stream_options=StreamOptions(include_usage=include_usage), + ) + + chunks = [chunk async for chunk in await serving.serve_tokens(request)] + data_chunks = [chunk for chunk in _parse_sse_chunks(chunks) if chunk != "[DONE]"] + + assert "metrics" not in data_chunks[0] + assert len(data_chunks) == 1 + int(include_usage) + if include_usage: + assert data_chunks[1]["choices"] == [] + assert ( + data_chunks[1]["metrics"]["speculative_decoding"]["num_draft_tokens"] == 3 + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +async def test_serve_tokens_omits_spec_decode_metrics_for_parallel_sampling( + stream: bool, +): + engine = _mock_engine() + metrics = RequestSpecDecodeMetrics.new(num_spec_tokens=3) + metrics.observe(num_draft_tokens=3, num_accepted=2) + result = _make_request_output( + "req-1", + token_ids=[10], + finish_reason="stop", + finished=True, + spec_decode_metrics=metrics, + ) + result.outputs.append( + CompletionOutput( + index=1, + text="", + token_ids=[20], + cumulative_logprob=None, + logprobs=None, + finish_reason="stop", + spec_decode_metrics=metrics, + ) + ) + + async def mock_generate(*args, **kwargs): + yield result + + engine.generate = MagicMock(side_effect=mock_generate) + serving = _build_serving_tokens(engine) + request = GenerateRequest( + token_ids=[1, 2, 3], + sampling_params=SamplingParams(max_tokens=1, n=2), + model=MODEL_NAME, + stream=stream, + stream_options=StreamOptions(include_usage=True) if stream else None, + ) + + response = await serving.serve_tokens(request) + if stream: + chunks = [chunk async for chunk in response] + assert all( + "metrics" not in chunk + for chunk in _parse_sse_chunks(chunks) + if isinstance(chunk, dict) + ) + else: + assert isinstance(response, GenerateResponse) + assert response.metrics is None + + @pytest.mark.asyncio async def test_non_stream_returns_final_prompt_token_ids_when_requested(): engine = _mock_engine() diff --git a/vllm/entrypoints/scale_out/derender/serving.py b/vllm/entrypoints/scale_out/derender/serving.py index d3c5f3aac1..5da7366403 100644 --- a/vllm/entrypoints/scale_out/derender/serving.py +++ b/vllm/entrypoints/scale_out/derender/serving.py @@ -179,6 +179,7 @@ class ServingDerender(BaseServing): usage=usage, prompt_logprobs=gen.prompt_logprobs, kv_transfer_params=gen.kv_transfer_params, + metrics=gen.metrics, ) async def derender_completion_response( @@ -245,6 +246,9 @@ class ServingDerender(BaseServing): choices=choices, usage=usage, kv_transfer_params=kv_params, + # Metrics describe one prompt. Parallel samples are already + # suppressed by /generate; multi-prompt responses cannot be merged. + metrics=first.metrics if len(request.generate_responses) == 1 else None, ) async def derender_chat_stream_response( diff --git a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py index ae0fd62b6e..088ec4d293 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py @@ -11,7 +11,11 @@ from pydantic import ( ) from vllm.config import ModelConfig -from vllm.entrypoints.generate.base.protocol import StreamOptions, validate_cache_salt +from vllm.entrypoints.generate.base.protocol import ( + PerRequestMetrics, + StreamOptions, + validate_cache_salt, +) from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionLogProbs, ChatCompletionRequest, @@ -360,6 +364,7 @@ class GenerateStreamResponse(BaseModel): usage: UsageInfo | None = Field(default=None) prompt_token_ids: list[int] | None = None mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = None + metrics: PerRequestMetrics | None = None class GenerateResponse(BaseModel): @@ -378,6 +383,7 @@ class GenerateResponse(BaseModel): prompt_logprobs: list[dict[int, Logprob] | None] | None = None prompt_token_ids: list[int] | None = None mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = None + metrics: PerRequestMetrics | None = None kv_transfer_params: dict[str, Any] | None = Field( default=None, diff --git a/vllm/entrypoints/scale_out/token_in_token_out/serving.py b/vllm/entrypoints/scale_out/token_in_token_out/serving.py index 7813c7facc..59fda2dbf8 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/serving.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/serving.py @@ -12,9 +12,13 @@ from fastapi import Request from vllm.engine.protocol import EngineClient from vllm.entrypoints.chat_utils import AsyncMultiModalItemTracker -from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata +from vllm.entrypoints.generate.base.protocol import ( + PerRequestMetrics, + RequestResponseMetadata, +) from vllm.entrypoints.generate.base.serving import ( GenerateBaseServing, + build_spec_decoding_metrics, clamp_prompt_logprobs, ) from vllm.entrypoints.openai.chat_completion.protocol import ( @@ -373,6 +377,11 @@ class ServingTokens(GenerateBaseServing): request_metadata.final_usage_info = usage + per_request_metrics = None + if request.sampling_params.n == 1: + spec_stats = build_spec_decoding_metrics(final_res) + if spec_stats is not None: + per_request_metrics = PerRequestMetrics(speculative_decoding=spec_stats) response = GenerateResponse( request_id=request_id, created=created_time, @@ -384,6 +393,7 @@ class ServingTokens(GenerateBaseServing): final_res.prompt_token_ids if request.return_token_ids else None ), mm_placeholders=request._response_mm_placeholders, + metrics=per_request_metrics, kv_transfer_params=final_res.kv_transfer_params, ec_transfer_params=final_res.ec_transfer_params, ) @@ -423,6 +433,7 @@ class ServingTokens(GenerateBaseServing): prompt_token_ids: list[int] | None = None num_cached_tokens = None sampling_params: SamplingParams = request.sampling_params + last_res: RequestOutput | None = None include_usage, include_continuous_usage = should_include_usage( request.stream_options, False @@ -430,6 +441,7 @@ class ServingTokens(GenerateBaseServing): try: async for res in result_generator: + last_res = res if first_iteration: if res.prompt_token_ids is not None: num_prompt_tokens = len(res.prompt_token_ids) @@ -496,10 +508,10 @@ class ServingTokens(GenerateBaseServing): total_tokens=(num_prompt_tokens + num_generated_tokens[i]), ) - # Omit absent prompt metadata (Rust skips None fields). + # Omit fields that are absent from token-bearing chunks. exclude = { name - for name in ("prompt_token_ids", "mm_placeholders") + for name in ("prompt_token_ids", "mm_placeholders", "metrics") if getattr(chunk, name) is None } yield f"data: {chunk.model_dump_json(exclude=exclude)}\n\n" @@ -517,10 +529,18 @@ class ServingTokens(GenerateBaseServing): ) if include_usage: + per_request_metrics = None + if sampling_params.n == 1: + spec_stats = build_spec_decoding_metrics(last_res) + if spec_stats is not None: + per_request_metrics = PerRequestMetrics( + speculative_decoding=spec_stats + ) final_chunk = GenerateStreamResponse( request_id=request_id, choices=[], usage=final_usage_info, + metrics=per_request_metrics, ) yield f"data: {final_chunk.model_dump_json(exclude_none=True)}\n\n" diff --git a/vllm/renderers/online_derenderer.py b/vllm/renderers/online_derenderer.py index a89bc843b8..9dfa0c412f 100644 --- a/vllm/renderers/online_derenderer.py +++ b/vllm/renderers/online_derenderer.py @@ -398,6 +398,7 @@ class OnlineDerenderer: model=model, choices=stream_choices, usage=usage, + metrics=generate_chunk.metrics, ) return chunk, updated_state @@ -595,6 +596,7 @@ class OnlineDerenderer: model=model, choices=stream_choices, usage=usage, + metrics=generate_chunk.metrics, ) return chunk, updated_state @@ -741,6 +743,7 @@ class OnlineDerenderer: model=model, choices=stream_choices, usage=usage, + metrics=generate_chunk.metrics, ) return chunk, updated_state