From 7cfb97e33791a348cd5d7b622cca521d82d8399f Mon Sep 17 00:00:00 2001 From: Matthew Kotila <7692737+matthewkotila@users.noreply.github.com> Date: Thu, 20 Aug 2026 15:42:49 -0700 Subject: [PATCH] [Frontend][Core][Spec Decode] Per-request acceptance stats in OpenAI API responses (#48915) Signed-off-by: Matthew Kotila <7692737+matthewkotila@users.noreply.github.com> Co-authored-by: Reed Meyerson <31574681+reed-meyerson@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) --- docs/features/per_request_metrics.md | 9 ++ docs/features/speculative_decoding/README.md | 1 + .../acceptance_metrics.md | 100 ++++++++++++ .../engine-core-client/src/protocol/output.rs | 6 + .../engine-core-client/src/tests/client.rs | 1 + .../completion/test_completion_error.py | 110 ++++++++++++- tests/test_config.py | 14 ++ tests/v1/core/test_scheduler.py | 122 ++++++++++++-- tests/v1/core/utils.py | 5 + .../v1/spec_decode/test_request_acceptance.py | 150 ++++++++++++++++++ vllm/config/observability.py | 12 ++ vllm/config/vllm.py | 9 ++ vllm/engine/arg_utils.py | 8 + vllm/entrypoints/generate/base/serving.py | 31 +++- .../openai/chat_completion/protocol.py | 6 +- .../openai/chat_completion/serving.py | 53 ++++--- .../entrypoints/openai/completion/protocol.py | 6 +- vllm/entrypoints/openai/completion/serving.py | 64 ++++---- vllm/entrypoints/openai/engine/protocol.py | 26 ++- vllm/outputs.py | 8 +- vllm/v1/core/sched/scheduler.py | 32 +++- vllm/v1/engine/__init__.py | 10 +- vllm/v1/engine/output_processor.py | 8 + vllm/v1/metrics/stats.py | 76 +++++++++ vllm/v1/request.py | 7 +- 25 files changed, 798 insertions(+), 76 deletions(-) create mode 100644 docs/features/speculative_decoding/acceptance_metrics.md create mode 100644 tests/v1/spec_decode/test_request_acceptance.py diff --git a/docs/features/per_request_metrics.md b/docs/features/per_request_metrics.md index 9bc64d2b86..25211ecfc3 100644 --- a/docs/features/per_request_metrics.md +++ b/docs/features/per_request_metrics.md @@ -125,3 +125,12 @@ a single prompt's generation. The `metrics` response field provides per-request values for a single request. The `/metrics` Prometheus endpoint exposes server-level histograms (e.g. `vllm:time_to_first_token_seconds`) that aggregate across all requests. + +## Speculative Decoding Acceptance + +When speculative decoding is enabled, per-request acceptance metrics +(mean acceptance length and the accepted-draft-length distribution) can be +returned via `--per-request-spec-decode-metrics`. They share this `metrics` +object as `metrics.speculative_decoding`, and — like the timing fields — are +reported only for single-sequence (`n == 1`) requests. See +[Per-Request Acceptance Metrics](speculative_decoding/acceptance_metrics.md). diff --git a/docs/features/speculative_decoding/README.md b/docs/features/speculative_decoding/README.md index 7858fff2d4..f4417c7fde 100644 --- a/docs/features/speculative_decoding/README.md +++ b/docs/features/speculative_decoding/README.md @@ -19,6 +19,7 @@ vLLM supports a variety of methods of speculative decoding. Model-based methods - [Custom Proposer Backend (Experimental)](#custom-proposer-backend-experimental) - [Dynamic Speculative Decoding](dynamic_speculative_decoding.md) - [Adaptive Verification](adaptive_verification.md) +- [Per-Request Acceptance Metrics](acceptance_metrics.md) ## Method Selection at a Glance diff --git a/docs/features/speculative_decoding/acceptance_metrics.md b/docs/features/speculative_decoding/acceptance_metrics.md new file mode 100644 index 0000000000..9e31f920ed --- /dev/null +++ b/docs/features/speculative_decoding/acceptance_metrics.md @@ -0,0 +1,100 @@ +# Per-Request Acceptance Metrics + +When speculative decoding is enabled, vLLM can report per-request acceptance +metrics in the response, under `metrics.speculative_decoding`. This lets a +client compute the mean acceptance length and the accepted-draft-length +distribution for an individual request, as a complement to the server-aggregated +spec-decode metrics exposed at `/metrics`. + +!!! warning "Experimental" + `metrics.speculative_decoding` is experimental and its shape may change in a + future release. Pin to a vLLM version if you depend on it. + +## Enabling + +Start the server with `--per-request-spec-decode-metrics` set to `summary` or +`detailed` (default `none`): + +```bash +vllm serve \ + --speculative-config '{"method": "ngram", "num_speculative_tokens": 3, "prompt_lookup_min": 1, "prompt_lookup_max": 3}' \ + --per-request-spec-decode-metrics summary +``` + +| Level | Behavior | +| --- | --- | +| `none` (default) | No collection; responses are unchanged. | +| `summary` | Acceptance metrics per request. | +| `detailed` | `summary` plus ordered per-step arrays. | + +Collection is gated at the source: with `none`, nothing is accumulated. + +## Response Format + +Acceptance metrics share the top-level `metrics` object with the timing +[per-request metrics](../per_request_metrics.md) — `metrics.speculative_decoding` +sits alongside the timing fields. Like timing, they describe a single generation +stream, so they are reported only for single-sequence requests and are `null` +for `n > 1`. + +A `summary` response's `metrics` looks like: + +```json +{ + "choices": [ ... ], + "usage": { ... }, + "metrics": { + "speculative_decoding": { + "mean_acceptance_length": 1.2325581395348837, + "draft_acceptance_rate": 0.07751937984496124, + "acceptance_histogram": [39, 1, 0, 3], + "num_spec_steps": 43, + "num_accepted_draft_tokens": 10, + "num_draft_tokens": 129, + "num_spec_tokens": 3 + } + } +} +``` + +| Field | Description | +| --- | --- | +| `mean_acceptance_length` | Mean tokens emitted per verification step, including the bonus token: `1 + num_accepted_draft_tokens / num_spec_steps`. Ranges from `1.0` (nothing accepted) to `num_spec_tokens + 1`. | +| `draft_acceptance_rate` | Fraction of proposed draft tokens accepted: `num_accepted_draft_tokens / num_draft_tokens`. | +| `acceptance_histogram` | Dense list of length `num_spec_tokens + 1`; index `j` is the number of steps that accepted exactly `j` draft tokens. Excludes the always-accepted bonus token. | +| `num_spec_steps` | Number of verification steps for this request (the sum of the histogram). | +| `num_accepted_draft_tokens` | Total accepted draft tokens, excluding bonus tokens. | +| `num_draft_tokens` | Total proposed draft tokens, after subtracting drafts invalidated by structured-output constraints. | +| `num_spec_tokens` | Configured `num_speculative_tokens` (`k`), i.e. the maximum draft length per step. | + +With `detailed`, two ordered arrays are added, one entry per verification step: + +| Field | Description | +| --- | --- | +| `per_step_accepted` | Accepted draft count at each step. | +| `per_step_drafted` | Proposed draft count at each step. Records the effective proposal length per step, so variable-length drafting (e.g. adaptive speculation) is represented without a schema change. | + +`metrics.speculative_decoding` is present whenever `--per-request-spec-decode-metrics` +is `summary`/`detailed`, speculative decoding is enabled, and `n == 1` (with an +all-zero histogram if the request drafted nothing). It is `null` otherwise. + +## Streaming + +In streaming responses, `metrics` (including `speculative_decoding`) rides the +final usage chunk, which is only emitted when usage reporting is enabled — set +`stream_options.include_usage: true` or start the server with +`--enable-force-include-usage`. + +## Relationship to Prometheus metrics + +The per-request fields are the individual-request counterpart of the +server-aggregated spec-decode counters at `/metrics`. Summed across the +single-sequence requests that report them, they reconcile with the aggregate +counters (which also count `n > 1` requests, so the totals match only for +all-`n == 1` workloads): + +| Per-request field (summed) | Prometheus counter | +| --- | --- | +| `num_spec_steps` | `vllm:spec_decode_num_drafts_total` | +| `num_draft_tokens` | `vllm:spec_decode_num_draft_tokens_total` | +| `num_accepted_draft_tokens` | `vllm:spec_decode_num_accepted_tokens_total` | diff --git a/rust/src/engine-core-client/src/protocol/output.rs b/rust/src/engine-core-client/src/protocol/output.rs index a9695bd1cf..cc7541eae1 100644 --- a/rust/src/engine-core-client/src/protocol/output.rs +++ b/rust/src/engine-core-client/src/protocol/output.rs @@ -125,6 +125,11 @@ pub struct EngineCoreOutput { pub mm_cache_miss_hashes: Option>, #[serde(default)] pub new_sampling_mask: Option, + /// Per-request speculative-decoding acceptance metrics, set on the final + /// output when `--per-request-spec-decode-metrics` is enabled. Opaque here; + /// the Rust frontend does not yet surface it in responses. + #[serde(default)] + pub spec_decode_metrics: Option, } impl EngineCoreOutput { @@ -443,6 +448,7 @@ mod tests { num_nans_in_logits: 0, mm_cache_miss_hashes: None, new_sampling_mask: None, + spec_decode_metrics: None, }, ], scheduler_stats: None, diff --git a/rust/src/engine-core-client/src/tests/client.rs b/rust/src/engine-core-client/src/tests/client.rs index fa5d8e0db8..6f4e53ff1b 100644 --- a/rust/src/engine-core-client/src/tests/client.rs +++ b/rust/src/engine-core-client/src/tests/client.rs @@ -2743,6 +2743,7 @@ fn python_msgpack_fixtures_match_rust_encoding() { num_nans_in_logits: 0, mm_cache_miss_hashes: None, new_sampling_mask: None, + spec_decode_metrics: None, }, ], scheduler_stats: None, diff --git a/tests/entrypoints/openai/completion/test_completion_error.py b/tests/entrypoints/openai/completion/test_completion_error.py index 79fd6206d6..56a2ba66f1 100644 --- a/tests/entrypoints/openai/completion/test_completion_error.py +++ b/tests/entrypoints/openai/completion/test_completion_error.py @@ -23,7 +23,7 @@ from vllm.renderers.hf import HfRenderer from vllm.renderers.online_renderer import OnlineRenderer from vllm.tokenizers.registry import cached_tokenizer_from_config from vllm.v1.engine.async_llm import AsyncLLM -from vllm.v1.metrics.stats import RequestStateStats +from vllm.v1.metrics.stats import RequestSpecDecodeMetrics, RequestStateStats MODEL_NAME = "openai-community/gpt2" MODEL_NAME_SHORT = "gpt2" @@ -196,6 +196,114 @@ def test_completion_per_request_metrics_suppressed_for_multiple_prompts(): assert response.metrics is None +def _spec_decode_metrics() -> RequestSpecDecodeMetrics: + # Two verify steps: accept 3 drafts, then 1 -> histogram [0, 1, 0, 1]. + m = RequestSpecDecodeMetrics.new(num_spec_tokens=3) + m.observe(num_draft_tokens=3, num_accepted=3) + m.observe(num_draft_tokens=3, num_accepted=1) + return m + + +def _make_spec_decode_request_output( + num_seqs: int = 1, with_metrics: bool = True +) -> RequestOutput: + outputs = [ + CompletionOutput( + index=i, + text="Hello", + token_ids=[100, 101], + cumulative_logprob=None, + logprobs=None, + finish_reason="stop", + spec_decode_metrics=_spec_decode_metrics() if with_metrics else None, + ) + for i in range(num_seqs) + ] + return RequestOutput( + request_id="test-id", + prompt="Test prompt", + prompt_token_ids=[1, 2, 3], + prompt_logprobs=None, + outputs=outputs, + finished=True, + metrics=None, + ) + + +def _completion_response(serving, request, request_output): + return serving.request_output_to_completion_response( + [request_output], + request, + "cmpl-test-id", + 0, + MODEL_NAME, + None, + RequestResponseMetadata(request_id="cmpl-test-id"), + ) + + +def test_completion_spec_decode_metrics_present_for_single_sequence(): + # Timing off, but the sequence carries acceptance metrics -> the metrics + # object is created just to hold metrics.speculative_decoding. + serving = _build_minimal_metrics_serving_completion( + enable_per_request_metrics=False + ) + response = _completion_response( + serving, + CompletionRequest(model=MODEL_NAME, prompt="Test prompt", max_tokens=10), + _make_spec_decode_request_output(num_seqs=1), + ) + assert response.metrics is not None + assert response.metrics.time_to_first_token_ms is None # timing not requested + spec = response.metrics.speculative_decoding + assert spec is not None + assert spec.acceptance_histogram == [0, 1, 0, 1] # dense, index j + assert spec.num_spec_steps == 2 + assert spec.num_spec_tokens == 3 + assert spec.mean_acceptance_length == pytest.approx(3.0) # 1 + (3 + 1) / 2 + + +def test_completion_spec_decode_metrics_suppressed_for_n_gt_1(): + # Per-request metrics can't be attributed to one of the n sequences. + serving = _build_minimal_metrics_serving_completion( + enable_per_request_metrics=False + ) + response = _completion_response( + serving, + CompletionRequest(model=MODEL_NAME, prompt="Test prompt", n=2, max_tokens=10), + _make_spec_decode_request_output(num_seqs=2), + ) + assert response.metrics is None + + +def test_completion_spec_decode_metrics_absent_when_not_collected(): + # Flag off -> the sequence carries no acceptance metrics -> no metrics object. + serving = _build_minimal_metrics_serving_completion( + enable_per_request_metrics=False + ) + response = _completion_response( + serving, + CompletionRequest(model=MODEL_NAME, prompt="Test prompt", max_tokens=10), + _make_spec_decode_request_output(num_seqs=1, with_metrics=False), + ) + assert response.metrics is None + + +def test_completion_metrics_carries_both_timing_and_spec_decode(): + serving = _build_minimal_metrics_serving_completion(enable_per_request_metrics=True) + request_output = _make_spec_decode_request_output(num_seqs=1) + request_output.metrics = _PER_REQUEST_STATS # timing source + response = _completion_response( + serving, + CompletionRequest(model=MODEL_NAME, prompt="Test prompt", max_tokens=10), + request_output, + ) + assert response.metrics is not None + assert response.metrics.time_to_first_token_ms == pytest.approx(500.0) + assert response.metrics.speculative_decoding is not None + assert response.metrics.speculative_decoding.num_spec_steps == 2 + + @pytest.mark.asyncio async def test_completion_error_non_stream(): """test finish_reason='error' returns 500 InternalServerError (non-streaming)""" diff --git a/tests/test_config.py b/tests/test_config.py index ee3e0c96b5..0b36258039 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -20,6 +20,7 @@ from vllm.config import ( CompilationConfig, KernelConfig, ModelConfig, + ObservabilityConfig, ParallelConfig, PoolerConfig, SchedulerConfig, @@ -85,6 +86,19 @@ def test_kda_recoverssm_derivation_is_revalidated(): VllmConfig.validate_mamba_cached_kernel(config) +def test_per_request_spec_decode_metrics_requires_spec_decode(): + # The flag only makes sense with speculative decoding configured; enabling + # it without --speculative-config should fail fast rather than silently + # produce no metrics. + for level in ("summary", "detailed"): + with pytest.raises(ValueError, match="speculative"): + VllmConfig( + observability_config=ObservabilityConfig( + per_request_spec_decode_metrics=level + ) + ) + + def test_compile_config_repr_succeeds(): # setup: VllmBackend mutates the config object config = VllmConfig() diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index c0932d975c..920823baeb 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -1306,29 +1306,42 @@ def test_draft_slots_budgeted_per_scheduled_request(tmp_path, monkeypatch): # Note - these test cases mirror some of those in test_rejection_sampler.py @pytest.mark.parametrize( - "spec_tokens,output_tokens,expected", + "spec_tokens,output_tokens,expected,expected_per_req", [ - ([[1, 2, 3]], [[1, 2, 3, 4]], (1, 3, 3, [1, 1, 1])), # perfect match - ([[1, 2, 3]], [[1, 5]], (1, 3, 1, [1, 0, 0])), # early mismatch - ([[1, 2], [3]], [[1, 2, 5], [3, 4]], (2, 3, 3, [2, 1])), # multiple sequences - ([[1]], [[1, 2]], (1, 1, 1, [1])), # single token sequence - ([[]], [[5]], (0, 0, 0, [0])), # empty sequence + ([[1, 2, 3]], [[1, 2, 3, 4]], (1, 3, 3, [1, 1, 1]), [[0, 0, 0, 1]]), # perfect + ([[1, 2, 3]], [[1, 5]], (1, 3, 1, [1, 0, 0]), [[0, 1, 0, 0]]), # early mismatch + ( + [[1, 2], [3]], + [[1, 2, 5], [3, 4]], + (2, 3, 3, [2, 1]), + [[0, 0, 1], [0, 1, 0]], + ), # multiple sequences + ([[1]], [[1, 2]], (1, 1, 1, [1]), [[0, 1]]), # single token sequence + ([[]], [[5]], (0, 0, 0, [0]), [[0, 0]]), # empty sequence -> empty accumulator ( [[1, 2, 3], [4, 5, 6]], [[1, 2, 7], [4, 8]], (2, 6, 3, [2, 1, 0]), + [[0, 0, 1, 0], [0, 1, 0, 0]], ), # multiple mismatches ], ) -def test_schedule_spec_decoding_stats(spec_tokens, output_tokens, expected): +def test_schedule_spec_decoding_stats( + spec_tokens, output_tokens, expected, expected_per_req +): """Test scheduling behavior with speculative decoding. This test verifies that: 1. Speculated tokens get scheduled correctly - 2. Spec decoding stats properly count number of draft and accepted tokens + 2. The aggregate SpecDecodingStats count draft and accepted tokens + 3. The per-request accumulator (enabled via per_request_spec_decode_metrics) + buckets the same acceptance by accepted draft count (j) """ num_spec_tokens = max(1, max(len(t) for t in spec_tokens)) - scheduler = create_scheduler(num_speculative_tokens=num_spec_tokens) + scheduler = create_scheduler( + num_speculative_tokens=num_spec_tokens, + per_request_spec_decode_metrics="summary", + ) requests = create_requests(num_requests=len(spec_tokens), num_tokens=1) req_ids = [] req_to_index = {} @@ -1414,6 +1427,97 @@ def test_schedule_spec_decoding_stats(spec_tokens, output_tokens, expected): assert stats.num_accepted_tokens == expected[2] assert stats.num_accepted_tokens_per_pos == expected[3] + # Per-request accumulator: the same acceptance, bucketed by accepted draft + # count (j) on each request rather than summed across the batch. The + # accumulator is created eagerly on add_request, so every request has one + # (an empty histogram when it drafted nothing). + for i, req_id in enumerate(req_ids): + payload = scheduler.requests[req_id].spec_decode_metrics.to_dict() + assert payload["acceptance_histogram"] == expected_per_req[i] + assert payload["num_draft_tokens"] == len(spec_tokens[i]) + assert "per_step_accepted" not in payload # summary level + + +def _run_spec_verify_steps(scheduler, rounds, num_invalid_per_round=None): + """Drive prefill + one draft/verify step per round for a single request. + + ``rounds`` is a list of ``(spec_token_ids, output_token_ids)`` -- one verify + step each. ``num_invalid_per_round`` optionally injects the grammar-invalid + draft count structured-output decoding would set on the verify output. + Returns the request. + """ + [req] = create_requests(num_requests=1, num_tokens=1) + scheduler.add_request(req) + rid = req.request_id + req_to_index = {rid: 0} + + def _mk_output(sampled): + return ModelRunnerOutput( + req_ids=[rid], + req_id_to_index=req_to_index, + sampled_token_ids=sampled, + logprobs=None, + prompt_logprobs_dict={}, + pooler_output=[], + ) + + # Prefill: sample one token. + scheduler.update_from_output(scheduler.schedule(), _mk_output([[0]])) + for i, (spec, out) in enumerate(rounds): + scheduler.update_draft_token_ids(DraftTokenIds([rid], [spec])) + output = scheduler.schedule() + if num_invalid_per_round is not None and num_invalid_per_round[i]: + output.num_invalid_spec_tokens = {rid: num_invalid_per_round[i]} + scheduler.update_from_output(output, _mk_output([out])) + return req + + +def test_per_request_spec_decode_detailed_records_per_step(): + scheduler = create_scheduler( + num_speculative_tokens=3, + per_request_spec_decode_metrics="detailed", + ) + # Three verify steps for one request; num_accepted = len(output) - 1, so the + # outputs below accept 3, 1, then 0 drafts across the steps. + req = _run_spec_verify_steps( + scheduler, + [ + ([1, 2, 3], [1, 2, 3, 4]), # accept 3 + ([5, 6, 7], [5, 8]), # accept 1 + ([9, 10, 11], [12]), # accept 0 + ], + ) + payload = scheduler.requests[req.request_id].spec_decode_metrics.to_dict() + assert payload["per_step_accepted"] == [3, 1, 0] + assert payload["per_step_drafted"] == [3, 3, 3] + assert payload["num_spec_steps"] == 3 + + +def test_per_request_spec_decode_subtracts_invalid_drafts(): + # Grammar-invalidated drafts (num_invalid_spec_tokens, set by structured + # output) are excluded from the proposed count, mirroring the aggregate. + scheduler = create_scheduler( + num_speculative_tokens=3, + per_request_spec_decode_metrics="summary", + ) + # One verify step: 3 drafted, 1 grammar-invalid, output accepts 2. + req = _run_spec_verify_steps( + scheduler, + [([1, 2, 3], [1, 2, 5])], + num_invalid_per_round=[1], + ) + payload = scheduler.requests[req.request_id].spec_decode_metrics.to_dict() + assert payload["num_draft_tokens"] == 2 # 3 drafted - 1 invalid + assert payload["num_accepted_draft_tokens"] == 2 # len([1,2,5]) - 1 + assert payload["acceptance_histogram"] == [0, 0, 1, 0] + + +def test_per_request_spec_decode_acceptance_disabled_by_default(): + scheduler = create_scheduler(num_speculative_tokens=3) + assert scheduler.spec_decode_metrics_level == "none" + req = _run_spec_verify_steps(scheduler, [([1, 2, 3], [1, 2, 3, 4])]) + assert scheduler.requests[req.request_id].spec_decode_metrics is None + def test_spec_decoding_stats_empty_output(): """Test that spec decoding stats handle empty output tokens gracefully. diff --git a/tests/v1/core/utils.py b/tests/v1/core/utils.py index 0baa68af25..96352324e9 100644 --- a/tests/v1/core/utils.py +++ b/tests/v1/core/utils.py @@ -10,6 +10,7 @@ from vllm.config import ( ECTransferConfig, KVTransferConfig, ModelConfig, + ObservabilityConfig, ParallelConfig, SchedulerConfig, SpeculativeConfig, @@ -71,6 +72,7 @@ def create_scheduler( ec_role: str | None = None, use_v2_model_runner: bool | None = None, kv_cache_spec: KVCacheSpec | None = None, + per_request_spec_decode_metrics: str = "none", ) -> Scheduler | AsyncScheduler: """Create scheduler under test. @@ -175,6 +177,9 @@ def create_scheduler( kv_transfer_config=kv_transfer_config, speculative_config=speculative_config, ec_transfer_config=ec_transfer_config, + observability_config=ObservabilityConfig( + per_request_spec_decode_metrics=per_request_spec_decode_metrics, + ), ) if kv_cache_spec is None: kv_cache_spec = FullAttentionSpec( diff --git a/tests/v1/spec_decode/test_request_acceptance.py b/tests/v1/spec_decode/test_request_acceptance.py new file mode 100644 index 0000000000..1a00308abd --- /dev/null +++ b/tests/v1/spec_decode/test_request_acceptance.py @@ -0,0 +1,150 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for per-request speculative-decoding metrics accumulation. + +These cover the pure histogram/per-step math (no GPU / no model): the engine-core +accumulator ``RequestSpecDecodeMetrics`` and its ``to_dict`` payload surfaced in the +response as ``metrics.speculative_decoding``. +""" + +import msgspec +import pytest + +from vllm.outputs import CompletionOutput +from vllm.v1.engine import EngineCoreOutput +from vllm.v1.metrics.stats import RequestSpecDecodeMetrics + + +def _metrics(pairs, num_spec_tokens=3, detailed=False): + s = RequestSpecDecodeMetrics.new(num_spec_tokens) + for k, j in pairs: + s.observe(num_draft_tokens=k, num_accepted=j, detailed=detailed) + return s + + +def test_new_allocates_dense_histogram_of_k_plus_one(): + s = RequestSpecDecodeMetrics.new(num_spec_tokens=3) + assert s.num_spec_tokens == 3 + assert s.histogram == [0, 0, 0, 0] + assert s.num_draft_tokens == 0 + assert s.per_step_accepted == [] + + +def test_observe_buckets_by_accepted_draft_count(): + s = _metrics([(3, 0), (3, 3), (3, 2), (3, 3), (3, 1)]) + # j=0 ->1, j=1 ->1, j=2 ->1, j=3 ->2 + assert s.histogram == [1, 1, 1, 2] + assert s.num_draft_tokens == 15 + # summary level does not record the ordered per-step arrays + assert s.per_step_accepted == [] + assert s.per_step_drafted == [] + + +def test_observe_detailed_records_ordered_per_step_arrays(): + # Distinct step count (3), max draft length (k=4), and per-step drafted + # counts (4, 2, 4) so no accidental "everything is 3" pattern is implied. + s = _metrics([(4, 3), (2, 2), (4, 0)], num_spec_tokens=4, detailed=True) + assert s.per_step_accepted == [3, 2, 0] + assert s.per_step_drafted == [4, 2, 4] + # histogram (indexed by accepted j, length k+1=5) is still maintained + assert s.histogram == [1, 0, 1, 1, 0] + + +def test_to_dict_summary_omits_per_step_arrays(): + d = _metrics([(3, 0), (3, 3), (3, 2), (3, 3), (3, 1)]).to_dict() + assert d == { + "mean_acceptance_length": pytest.approx(1 + 9 / 5), # j+1 + "draft_acceptance_rate": pytest.approx(9 / 15), + "acceptance_histogram": [1, 1, 1, 2], # dense, index j = step count + "num_spec_steps": 5, + "num_accepted_draft_tokens": 9, + "num_draft_tokens": 15, + "num_spec_tokens": 3, + } + + +def test_to_dict_detailed_appends_per_step_arrays(): + d = _metrics([(3, 3), (3, 2), (3, 0)], detailed=True).to_dict() + assert d["per_step_accepted"] == [3, 2, 0] + assert d["per_step_drafted"] == [3, 3, 3] + # summary fields still present in detailed mode + assert d["num_spec_steps"] == 3 + assert d["num_accepted_draft_tokens"] == 5 + + +def test_to_dict_histogram_is_dense_list_indexed_by_j(): + # length k+1, index j holds the step count that accepted exactly j drafts + d = _metrics([(3, 0), (3, 0), (3, 3)]).to_dict() + assert d["acceptance_histogram"] == [2, 0, 0, 1] + + +def test_all_rejected_gives_mean_one_and_rate_zero(): + d = _metrics([(2, 0), (2, 0), (2, 0), (2, 0)], num_spec_tokens=2).to_dict() + assert d["num_spec_steps"] == 4 + assert d["num_accepted_draft_tokens"] == 0 + assert d["acceptance_histogram"] == [4, 0, 0] + assert d["mean_acceptance_length"] == pytest.approx(1.0) + assert d["draft_acceptance_rate"] == pytest.approx(0.0) + + +def test_empty_metrics_do_not_divide_by_zero(): + d = RequestSpecDecodeMetrics.new(3).to_dict() + assert d["num_spec_steps"] == 0 + assert d["num_draft_tokens"] == 0 + assert d["draft_acceptance_rate"] == 0.0 + assert d["mean_acceptance_length"] == 1.0 + assert "per_step_accepted" not in d + + +def test_observe_records_proposed_and_accepted_independently(): + # observe() takes proposed and accepted as independent inputs: the histogram + # is keyed by accepted, num_draft_tokens sums the proposed as given. (The + # grammar-invalidated-draft subtraction happens in the scheduler before + # observe() -- see test_per_request_spec_decode_subtracts_invalid_drafts.) + s = RequestSpecDecodeMetrics.new(num_spec_tokens=3) + s.observe(num_draft_tokens=2, num_accepted=1) + s.observe(num_draft_tokens=3, num_accepted=1) + d = s.to_dict() + assert d["acceptance_histogram"] == [0, 2, 0, 0] # both steps accepted 1 + assert d["num_draft_tokens"] == 5 # proposed summed independently: 2 + 3 + + +def test_engine_core_output_round_trips_spec_decode_metrics(): + # The accumulator rides EngineCoreOutput (msgspec, array_like) to the + # frontend; verify it serializes (incl. per-step arrays) and is omitted + # when absent. ``new_token_ids`` is EngineCoreOutput's required "tokens + # generated this step" field -- a dummy value here since we only exercise + # spec_decode_metrics. + metrics = _metrics([(3, 0), (3, 3), (3, 2)], detailed=True) + out = EngineCoreOutput( + request_id="r1", new_token_ids=[1, 2], spec_decode_metrics=metrics + ) + decoder = msgspec.msgpack.Decoder(EngineCoreOutput) + decoded = decoder.decode(msgspec.msgpack.encode(out)) + assert decoded.spec_decode_metrics.histogram == [1, 0, 1, 1] + assert decoded.spec_decode_metrics.num_draft_tokens == 9 + assert decoded.spec_decode_metrics.per_step_accepted == [0, 3, 2] + + without = EngineCoreOutput(request_id="r2", new_token_ids=[1]) + decoded_without = decoder.decode(msgspec.msgpack.encode(without)) + assert decoded_without.spec_decode_metrics is None + + +def _completion_output(**kwargs): + return CompletionOutput( + index=0, + text="", + token_ids=[], + cumulative_logprob=None, + logprobs=None, + **kwargs, + ) + + +def test_completion_output_carries_spec_decode_metrics(): + # Metrics are per output sequence, so they ride the CompletionOutput + # (=> choices[i]), not the request-level RequestOutput. + metrics = _metrics([(3, 3), (3, 1)]) + out = _completion_output(spec_decode_metrics=metrics) + assert out.spec_decode_metrics is metrics + assert _completion_output().spec_decode_metrics is None diff --git a/vllm/config/observability.py b/vllm/config/observability.py index 093ed2f684..0394245f3c 100644 --- a/vllm/config/observability.py +++ b/vllm/config/observability.py @@ -45,6 +45,18 @@ class ObservabilityConfig: Note that collecting detailed timing information for each request can be expensive.""" + per_request_spec_decode_metrics: Literal["none", "summary", "detailed"] = "none" + """Include per-request speculative-decoding acceptance metrics in the + response under `metrics.speculative_decoding`. `none` disables; `summary` adds mean + acceptance length, draft acceptance rate, and the step-by-draft-length + histogram; `detailed` additionally records the ordered per-step + accepted/proposed arrays (one entry per verify step). Only reported for + single-sequence requests (`n == 1`), mirroring the timing metrics. No effect + unless speculative decoding is enabled. Independent of `--disable-log-stats`. + This is the per-request response-body counterpart of the aggregate + `vllm:spec_decode_*` Prometheus metrics. The response field is experimental + and its shape may change in a future release.""" + kv_cache_metrics: bool = False """Enable KV cache residency metrics (lifetime, idle time, reuse gaps). Uses sampling to minimize overhead. diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 4fcb3ca17d..5ad162ae5f 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1331,6 +1331,15 @@ class VllmConfig: ) self.model_config.disable_cascade_attn = True + if ( + self.observability_config.per_request_spec_decode_metrics != "none" + and self.speculative_config is None + ): + raise ValueError( + "--per-request-spec-decode-metrics requires speculative decoding " + "to be enabled (via --speculative-config)." + ) + if ( self.model_config is not None and self.model_config.multimodal_config is not None diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 035595d5a8..293e67be5d 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -656,6 +656,9 @@ class EngineArgs: collect_detailed_traces: list[DetailedTraceModules] | None = ( ObservabilityConfig.collect_detailed_traces ) + per_request_spec_decode_metrics: Literal["none", "summary", "detailed"] = ( + ObservabilityConfig.per_request_spec_decode_metrics + ) kv_cache_metrics: bool = ObservabilityConfig.kv_cache_metrics kv_cache_metrics_sample: float = get_field( ObservabilityConfig, "kv_cache_metrics_sample" @@ -1476,6 +1479,10 @@ class EngineArgs: "--collect-detailed-traces", **observability_kwargs["collect_detailed_traces"], ) + observability_group.add_argument( + "--per-request-spec-decode-metrics", + **observability_kwargs["per_request_spec_decode_metrics"], + ) observability_group.add_argument( "--kv-cache-metrics", **observability_kwargs["kv_cache_metrics"] ) @@ -1934,6 +1941,7 @@ class EngineArgs: show_hidden_metrics_for_version=self.show_hidden_metrics_for_version, otlp_traces_endpoint=self.otlp_traces_endpoint, collect_detailed_traces=self.collect_detailed_traces, + per_request_spec_decode_metrics=self.per_request_spec_decode_metrics, kv_cache_metrics=self.kv_cache_metrics, kv_cache_metrics_sample=self.kv_cache_metrics_sample, cudagraph_metrics=self.cudagraph_metrics, diff --git a/vllm/entrypoints/generate/base/serving.py b/vllm/entrypoints/generate/base/serving.py index 4f27b3bc35..d6ae0a2090 100644 --- a/vllm/entrypoints/generate/base/serving.py +++ b/vllm/entrypoints/generate/base/serving.py @@ -5,7 +5,7 @@ import time from collections.abc import Awaitable, Mapping from dataclasses import dataclass, field from http import HTTPStatus -from typing import ClassVar, Generic, TypeVar +from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar from fastapi import Request from pydantic import ConfigDict @@ -18,7 +18,8 @@ from vllm.entrypoints.openai.completion.protocol import CompletionRequest from vllm.entrypoints.openai.engine.protocol import ( ErrorResponse, GenerationError, - PerRequestTimingMetrics, + PerRequestMetrics, + SpeculativeDecodingMetrics, ) from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.openai.responses.protocol import ResponsesRequest @@ -37,6 +38,9 @@ from vllm.tracing import ( ) from vllm.v1.metrics.stats import RequestStateStats +if TYPE_CHECKING: + from vllm.outputs import RequestOutput + logger = init_logger(__name__) RequestT = TypeVar("RequestT", bound=AnyRequest) @@ -48,7 +52,7 @@ PRIORITY_HEADER = "X-Vllm-Priority" def build_per_request_timing_metrics( metrics: RequestStateStats | None, num_generation_tokens: int, -) -> PerRequestTimingMetrics: +) -> PerRequestMetrics: """Build per-request timing metrics from ``RequestStateStats``. ``generation_time_ms`` is the decode interval only (first output token to @@ -60,7 +64,7 @@ def build_per_request_timing_metrics( unavailable. """ if metrics is None: - return PerRequestTimingMetrics() + return PerRequestMetrics() queued_ts = metrics.queued_ts scheduled_ts = metrics.scheduled_ts @@ -91,7 +95,7 @@ def build_per_request_timing_metrics( if inference_time_ms > 0: tokens_per_second = num_generation_tokens / inference_time_ms * 1000 - return PerRequestTimingMetrics( + return PerRequestMetrics( time_to_first_token_ms=time_to_first_token_ms, generation_time_ms=generation_time_ms, queue_time_ms=queue_time_ms, @@ -100,6 +104,23 @@ def build_per_request_timing_metrics( ) +def build_spec_decoding_metrics( + final_res: "RequestOutput | None", +) -> SpeculativeDecodingMetrics | None: + """Build per-request spec-decode acceptance metrics from the single output + sequence, or ``None`` when unavailable (metrics disabled, or no sequence + yet). + + Only meaningful for single-sequence requests; callers suppress it for n>1. + """ + if final_res is None or not final_res.outputs: + return None + metrics = final_res.outputs[0].spec_decode_metrics + if metrics is None: + return None + return SpeculativeDecodingMetrics(**metrics.to_dict()) + + @dataclass(kw_only=True) class ServeContext(Generic[RequestT]): request: RequestT diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index e8bb4e5678..f3a3b398d6 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -29,7 +29,7 @@ from vllm.entrypoints.openai.engine.protocol import ( FunctionCall, FunctionDefinition, OpenAIBaseModel, - PerRequestTimingMetrics, + PerRequestMetrics, StopParam, StreamOptions, ToolCall, @@ -146,7 +146,7 @@ class ChatCompletionResponse(OpenAIBaseModel): ec_transfer_params: dict[str, Any] | None = Field( default=None, description="ECTransfer parameters." ) - metrics: PerRequestTimingMetrics | None = None + metrics: PerRequestMetrics | None = None class ChatCompletionResponseStreamChoice(OpenAIBaseModel): @@ -178,7 +178,7 @@ class ChatCompletionStreamResponse(OpenAIBaseModel): # Rendered prompt text from chat templating (only set when # ``return_prompt_text=True`` on the request); only sent on the first chunk. prompt_text: str | None = None - metrics: PerRequestTimingMetrics | None = None + metrics: PerRequestMetrics | None = None class ChatCompletionToolsParam(OpenAIBaseModel): diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index 28ec1e8a1a..13932aac2d 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -20,6 +20,7 @@ from vllm.entrypoints.generate.base.serving import ( GenerateBaseServing, GenerationError, build_per_request_timing_metrics, + build_spec_decoding_metrics, clamp_prompt_logprobs, format_token_id_placeholder, ) @@ -40,7 +41,7 @@ from vllm.entrypoints.openai.engine.protocol import ( DeltaMessage, ErrorResponse, FunctionCall, - PerRequestTimingMetrics, + PerRequestMetrics, PromptTokenUsageInfo, RequestResponseMetadata, ToolCall, @@ -819,16 +820,21 @@ class OpenAIServingChat(GenerateBaseServing): # only emitted when usage reporting is enabled (i.e. # ``stream_options.include_usage=true`` or # ``--enable-force-include-usage``). - stream_per_request_metrics: PerRequestTimingMetrics | None = None - if ( - self.enable_per_request_metrics - # See note in chat_completion_full_generator: suppress for n>1. - and (request.n or 1) == 1 - ): - last_metrics = last_res.metrics if last_res is not None else None - stream_per_request_metrics = build_per_request_timing_metrics( - last_metrics, completion_tokens - ) + stream_per_request_metrics: PerRequestMetrics | None = None + # See note in chat_completion_full_generator: suppress for n>1. + if (request.n or 1) == 1: + if self.enable_per_request_metrics: + last_metrics = ( + last_res.metrics if last_res is not None else None + ) + stream_per_request_metrics = build_per_request_timing_metrics( + last_metrics, completion_tokens + ) + spec_stats = build_spec_decoding_metrics(last_res) + if spec_stats is not None: + if stream_per_request_metrics is None: + stream_per_request_metrics = PerRequestMetrics() + stream_per_request_metrics.speculative_decoding = spec_stats final_usage_chunk = ChatCompletionStreamResponse( id=request_id, @@ -1118,17 +1124,20 @@ class OpenAIServingChat(GenerateBaseServing): request_metadata.final_usage_info = usage - per_request_metrics: PerRequestTimingMetrics | None = None - if ( - self.enable_per_request_metrics - # Timing metrics describe a single generation stream. For n>1 the - # returned stats belong to only one of the n sequences, so they - # cannot be accurately attributed to the request; suppress instead. - and (request.n or 1) == 1 - ): - per_request_metrics = build_per_request_timing_metrics( - final_res.metrics, num_generated_tokens - ) + per_request_metrics: PerRequestMetrics | None = None + # Per-request metrics (timing + spec-decode acceptance) describe a single + # generation stream. For n>1 the stats belong to only one of the n + # sequences, so they cannot be attributed to the request; suppress. + if (request.n or 1) == 1: + if self.enable_per_request_metrics: + per_request_metrics = build_per_request_timing_metrics( + final_res.metrics, num_generated_tokens + ) + spec_stats = build_spec_decoding_metrics(final_res) + if spec_stats is not None: + if per_request_metrics is None: + per_request_metrics = PerRequestMetrics() + per_request_metrics.speculative_decoding = spec_stats # ``final_res.prompt`` is the rendered chat-templated prompt text prompt_text = final_res.prompt if request.return_prompt_text else None diff --git a/vllm/entrypoints/openai/completion/protocol.py b/vllm/entrypoints/openai/completion/protocol.py index 2fc6ce1cd9..4b9ec1827e 100644 --- a/vllm/entrypoints/openai/completion/protocol.py +++ b/vllm/entrypoints/openai/completion/protocol.py @@ -13,7 +13,7 @@ from vllm.config import ModelConfig from vllm.entrypoints.openai.engine.protocol import ( AnyResponseFormat, OpenAIBaseModel, - PerRequestTimingMetrics, + PerRequestMetrics, StopParam, StreamOptions, UsageInfo, @@ -660,7 +660,7 @@ class CompletionResponse(OpenAIBaseModel): ec_transfer_params: dict[str, Any] | None = Field( default=None, description="ECTransfer parameters." ) - metrics: PerRequestTimingMetrics | None = None + metrics: PerRequestMetrics | None = None class CompletionResponseStreamChoice(OpenAIBaseModel): @@ -692,4 +692,4 @@ class CompletionStreamResponse(OpenAIBaseModel): # Set only on the final chunk of a stream to mirror non-streaming responses # without the per-chunk serialization overhead. system_fingerprint: str | None = None - metrics: PerRequestTimingMetrics | None = None + metrics: PerRequestMetrics | None = None diff --git a/vllm/entrypoints/openai/completion/serving.py b/vllm/entrypoints/openai/completion/serving.py index 37f55f4116..93a2b7d3aa 100644 --- a/vllm/entrypoints/openai/completion/serving.py +++ b/vllm/entrypoints/openai/completion/serving.py @@ -14,6 +14,7 @@ from vllm.entrypoints.generate.base.serving import ( GenerateBaseServing, GenerationError, build_per_request_timing_metrics, + build_spec_decoding_metrics, clamp_prompt_logprobs, format_token_id_placeholder, ) @@ -27,7 +28,7 @@ from vllm.entrypoints.openai.completion.protocol import ( ) from vllm.entrypoints.openai.engine.protocol import ( ErrorResponse, - PerRequestTimingMetrics, + PerRequestMetrics, PromptTokenUsageInfo, RequestResponseMetadata, UsageInfo, @@ -460,18 +461,22 @@ class OpenAIServingCompletion(GenerateBaseServing): # only emitted when usage reporting is enabled (i.e. # ``stream_options.include_usage=true`` or # ``--enable-force-include-usage``). - stream_per_request_metrics: PerRequestTimingMetrics | None = None - if ( - self.enable_per_request_metrics - # See note in request_output_to_completion_response: suppress - # when not attributable to one stream (multi-prompt or n>1). - and num_prompts == 1 - and (request.n or 1) == 1 - ): - last_metrics = last_res.metrics if last_res is not None else None - stream_per_request_metrics = build_per_request_timing_metrics( - last_metrics, total_completion_tokens - ) + stream_per_request_metrics: PerRequestMetrics | None = None + # See note in request_output_to_completion_response: suppress when + # not attributable to one stream (multi-prompt or n>1). + if num_prompts == 1 and (request.n or 1) == 1: + if self.enable_per_request_metrics: + last_metrics = ( + last_res.metrics if last_res is not None else None + ) + stream_per_request_metrics = build_per_request_timing_metrics( + last_metrics, total_completion_tokens + ) + spec_stats = build_spec_decoding_metrics(last_res) + if spec_stats is not None: + if stream_per_request_metrics is None: + stream_per_request_metrics = PerRequestMetrics() + stream_per_request_metrics.speculative_decoding = spec_stats final_usage_chunk = CompletionStreamResponse( id=request_id, @@ -612,21 +617,24 @@ class OpenAIServingCompletion(GenerateBaseServing): request_metadata.final_usage_info = usage - per_request_metrics: PerRequestTimingMetrics | None = None - if ( - self.enable_per_request_metrics - # Metrics describe a single generation stream, so suppress them when - # they cannot be attributed to one: multiple prompts (timestamps - # span prompts) or n>1 (stats belong to one of the n sequences). - and len(final_res_batch) == 1 - and (request.n or 1) == 1 - ): - last_metrics = ( - last_final_res.metrics if last_final_res is not None else None - ) - per_request_metrics = build_per_request_timing_metrics( - last_metrics, num_generated_tokens - ) + per_request_metrics: PerRequestMetrics | None = None + # Per-request metrics (timing + spec-decode acceptance) describe a single + # generation stream, so suppress them when they cannot be attributed to + # one: multiple prompts (timestamps span prompts) or n>1 (stats belong to + # one of the n sequences). + if len(final_res_batch) == 1 and (request.n or 1) == 1: + if self.enable_per_request_metrics: + last_metrics = ( + last_final_res.metrics if last_final_res is not None else None + ) + per_request_metrics = build_per_request_timing_metrics( + last_metrics, num_generated_tokens + ) + spec_stats = build_spec_decoding_metrics(last_final_res) + if spec_stats is not None: + if per_request_metrics is None: + per_request_metrics = PerRequestMetrics() + per_request_metrics.speculative_decoding = spec_stats if final_res_batch: kv_transfer_params = final_res_batch[0].kv_transfer_params diff --git a/vllm/entrypoints/openai/engine/protocol.py b/vllm/entrypoints/openai/engine/protocol.py index c32be4459c..9635ece46b 100644 --- a/vllm/entrypoints/openai/engine/protocol.py +++ b/vllm/entrypoints/openai/engine/protocol.py @@ -129,12 +129,36 @@ class UsageInfo(OpenAIBaseModel): completion_tokens_details: CompletionTokenUsageInfo | None = None -class PerRequestTimingMetrics(OpenAIBaseModel): +class SpeculativeDecodingMetrics(OpenAIBaseModel): + """Per-request speculative-decoding acceptance metrics. + + Experimental, subject to change. Only populated for single-sequence requests + (`n == 1`); `null` for `n > 1`, mirroring the timing metrics. + """ + + mean_acceptance_length: float + draft_acceptance_rate: float + # Dense histogram: index j holds the number of verify steps that accepted + # exactly j draft tokens (length num_spec_tokens + 1). Excludes the + # always-accepted bonus token. + acceptance_histogram: list[int] + num_spec_steps: int + num_accepted_draft_tokens: int + num_draft_tokens: int + num_spec_tokens: int + # Ordered per-verify-step arrays; populated only at the `detailed` level. + per_step_accepted: list[int] | None = None + per_step_drafted: list[int] | None = None + + +class PerRequestMetrics(OpenAIBaseModel): time_to_first_token_ms: float | None = None generation_time_ms: float | None = None queue_time_ms: float | None = None mean_itl_ms: float | None = None tokens_per_second: float | None = None + # Experimental, subject to change. + speculative_decoding: SpeculativeDecodingMetrics | None = None class RequestResponseMetadata(BaseModel): diff --git a/vllm/outputs.py b/vllm/outputs.py index 29584e0e34..84b9fccdc8 100644 --- a/vllm/outputs.py +++ b/vllm/outputs.py @@ -13,7 +13,7 @@ from typing_extensions import TypeVar from vllm.logger import init_logger from vllm.logprobs import PromptLogprobs, SampleLogprobs from vllm.lora.request import LoRARequest -from vllm.v1.metrics.stats import RequestStateStats +from vllm.v1.metrics.stats import RequestSpecDecodeMetrics, RequestStateStats logger = init_logger(__name__) @@ -48,6 +48,11 @@ class CompletionOutput: to stop, None if the completion finished for some other reason including encountering the EOS token. lora_request: The LoRA request that was used to generate the output. + spec_decode_metrics: Per-sequence speculative-decoding acceptance metrics, + populated on finish when speculative decoding ran and + ``--per-request-spec-decode-metrics`` is enabled; None otherwise. + Surfaced in the response as ``metrics.speculative_decoding`` for + single-sequence (``n == 1``) requests. """ index: int @@ -60,6 +65,7 @@ class CompletionOutput: stop_reason: int | str | None = None lora_request: LoRARequest | None = None sampling_mask: SamplingMask | None = None + spec_decode_metrics: RequestSpecDecodeMetrics | None = None def finished(self) -> bool: return self.finish_reason is not None diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 2115ca6734..26674154c6 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -55,7 +55,11 @@ from vllm.v1.core.sched.utils import check_stop, remove_all from vllm.v1.engine import EngineCoreEventType, EngineCoreOutput, EngineCoreOutputs from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.metrics.perf import ModelMetrics, PerfStats -from vllm.v1.metrics.stats import PrefixCacheStats, SchedulerStats +from vllm.v1.metrics.stats import ( + PrefixCacheStats, + RequestSpecDecodeMetrics, + SchedulerStats, +) from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput from vllm.v1.request import Request, RequestStatus, StreamingUpdate from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup @@ -88,6 +92,9 @@ class Scheduler(SchedulerInterface): self.parallel_config = vllm_config.parallel_config self.log_stats = log_stats self.observability_config = vllm_config.observability_config + self.spec_decode_metrics_level = ( + self.observability_config.per_request_spec_decode_metrics + ) self.kv_metrics_collector: KVCacheMetricsCollector | None = None if self.observability_config.kv_cache_metrics: self.kv_metrics_collector = KVCacheMetricsCollector( @@ -1857,6 +1864,20 @@ class Scheduler(SchedulerInterface): num_invalid_spec_tokens=scheduler_output.num_invalid_spec_tokens, request_id=req_id, ) + if request.spec_decode_metrics is not None: + # Exclude grammar-invalidated drafts from the proposed + # count, mirroring make_spec_decoding_stats; the accepted + # bucket (j) is unaffected. + adj_draft_tokens = num_draft_tokens + if scheduler_output.num_invalid_spec_tokens: + adj_draft_tokens -= ( + scheduler_output.num_invalid_spec_tokens.get(req_id, 0) + ) + request.spec_decode_metrics.observe( + num_draft_tokens=adj_draft_tokens, + num_accepted=num_accepted, + detailed=self.spec_decode_metrics_level == "detailed", + ) # Free encoder inputs only after the step has actually executed. if request.has_encoder_inputs: @@ -2020,6 +2041,11 @@ class Scheduler(SchedulerInterface): stop_reason=request.stop_reason, events=request.take_events(), prefill_stats=prefill_stats, + spec_decode_metrics=( + request.spec_decode_metrics + if finish_reason is not None + else None + ), kv_transfer_params=kv_transfer_params, ec_transfer_params=ec_transfer_params, trace_headers=request.trace_headers, @@ -2321,6 +2347,10 @@ class Scheduler(SchedulerInterface): request.streaming_queue = deque() self._enqueue_waiting_request(request) self.requests[request.request_id] = request + if self.spec_decode_metrics_level != "none": + request.spec_decode_metrics = RequestSpecDecodeMetrics.new( + self.num_spec_tokens + ) if self.connector is not None: self.connector.on_new_request(request) if self.log_stats: diff --git a/vllm/v1/engine/__init__.py b/vllm/v1/engine/__init__.py index 0a7f440ef7..5ae9ee0cac 100644 --- a/vllm/v1/engine/__init__.py +++ b/vllm/v1/engine/__init__.py @@ -16,7 +16,11 @@ from vllm.lora.request import LoRARequest from vllm.multimodal.inputs import MultiModalFeatureSpec from vllm.pooling_params import PoolingParams from vllm.sampling_params import SamplingParams -from vllm.v1.metrics.stats import PrefillStats, SchedulerStats +from vllm.v1.metrics.stats import ( + PrefillStats, + RequestSpecDecodeMetrics, + SchedulerStats, +) from vllm.v1.outputs import LogprobsLists, LogprobsTensors, SamplingMaskLists from vllm.v1.serial_utils import UtilityResult @@ -225,6 +229,10 @@ class EngineCoreOutput( new_sampling_mask: SamplingMaskLists | None = None + # Per-request spec-decode acceptance; attached only on the final output. + # Appended last so `array_like` positional serialization stays compatible. + spec_decode_metrics: RequestSpecDecodeMetrics | None = None + @property def finished(self) -> bool: return self.finish_reason is not None diff --git a/vllm/v1/engine/output_processor.py b/vllm/v1/engine/output_processor.py index 99f60d5d5d..055e3af574 100644 --- a/vllm/v1/engine/output_processor.py +++ b/vllm/v1/engine/output_processor.py @@ -35,6 +35,7 @@ from vllm.v1.engine.parallel_sampling import ParentRequest from vllm.v1.metrics.stats import ( IterationStats, LoRARequestStates, + RequestSpecDecodeMetrics, RequestStateStats, SchedulerStats, ) @@ -175,6 +176,9 @@ class RequestState: self.queue = queue self.num_cached_tokens = 0 self.num_cache_creation_tokens = 0 + # Per-sequence spec-decode accumulator; arrives once (on finish) via + # EngineCoreOutput, then attached to this sequence's CompletionOutput. + self.spec_decode_metrics: RequestSpecDecodeMetrics | None = None self.stats = RequestStateStats(arrival_time=arrival_time) if log_stats else None @@ -429,6 +433,7 @@ class RequestState: cumulative_logprob=self.logprobs_processor.cumulative_logprob, finish_reason=str(finish_reason) if finished else None, stop_reason=stop_reason if finished else None, + spec_decode_metrics=self.spec_decode_metrics if finished else None, ) def _new_pooling_output(self, pooling_output: torch.Tensor) -> PoolingOutput: @@ -658,6 +663,9 @@ class OutputProcessor: ) req_state.is_prefilling = False + if engine_core_output.spec_decode_metrics is not None: + req_state.spec_decode_metrics = engine_core_output.spec_decode_metrics + if pooling_output is None: assert req_state.detokenizer is not None assert req_state.logprobs_processor is not None diff --git a/vllm/v1/metrics/stats.py b/vllm/v1/metrics/stats.py index 3956f7e441..3dbc5206ca 100644 --- a/vllm/v1/metrics/stats.py +++ b/vllm/v1/metrics/stats.py @@ -297,6 +297,82 @@ class PrefillStats: ) +@dataclass +class RequestSpecDecodeMetrics: + """Per-output-sequence speculative-decoding statistics accumulator. + + Accumulates, over one sequence's verify steps, a histogram of accepted + draft-token counts (``j``, draft-only) and the total number of proposed + draft tokens. When ``detailed`` is requested it also records the ordered + per-step accepted/proposed sequences (``summary`` omits them). Tracked per + engine ``Request`` (one per sampled sequence, so ``n > 1`` yields one per + child), surfaced via ``EngineCoreOutput`` and the response + ``metrics.speculative_decoding`` for single-sequence requests (see + ``to_dict``). + + Fields: + num_spec_tokens: Configured ``num_speculative_tokens`` (the max ``k``); + also the histogram's upper bound. + histogram: Dense counts indexed by accepted draft tokens ``j`` + (length ``num_spec_tokens + 1``). + num_draft_tokens: Total proposed draft tokens, after the + grammar-invalidated (``num_invalid_spec_tokens``) adjustment. + per_step_accepted: Ordered accepted-draft count per verify step + (``detailed`` only; empty otherwise). + per_step_drafted: Ordered proposed-draft count per verify step + (``detailed`` only; empty otherwise). + """ + + num_spec_tokens: int + histogram: list[int] = field(default_factory=list) + num_draft_tokens: int = 0 + per_step_accepted: list[int] = field(default_factory=list) + per_step_drafted: list[int] = field(default_factory=list) + + @classmethod + def new(cls, num_spec_tokens: int) -> "RequestSpecDecodeMetrics": + return cls( + num_spec_tokens=num_spec_tokens, + histogram=[0] * (num_spec_tokens + 1), + ) + + def observe( + self, num_draft_tokens: int, num_accepted: int, detailed: bool = False + ) -> None: + self.histogram[num_accepted] += 1 + self.num_draft_tokens += num_draft_tokens + if detailed: + self.per_step_accepted.append(num_accepted) + self.per_step_drafted.append(num_draft_tokens) + + def to_dict(self) -> dict[str, Any]: + """Payload matching ``SpeculativeDecodingMetrics`` for the response. + + ``acceptance_histogram`` is a dense list indexed by accepted draft count + ``j`` (length ``num_spec_tokens + 1``). ``mean_acceptance_length`` + includes the bonus token (``j + 1``); ``draft_acceptance_rate`` is + draft-only, full precision. Per-step arrays are included only when + populated (``detailed`` level). + """ + num_spec_steps = sum(self.histogram) + num_accepted = sum(j * count for j, count in enumerate(self.histogram)) + mean_al = 1.0 + num_accepted / num_spec_steps if num_spec_steps else 1.0 + rate = num_accepted / self.num_draft_tokens if self.num_draft_tokens else 0.0 + result: dict[str, Any] = { + "mean_acceptance_length": mean_al, + "draft_acceptance_rate": rate, + "acceptance_histogram": list(self.histogram), + "num_spec_steps": num_spec_steps, + "num_accepted_draft_tokens": num_accepted, + "num_draft_tokens": self.num_draft_tokens, + "num_spec_tokens": self.num_spec_tokens, + } + if self.per_step_accepted: + result["per_step_accepted"] = self.per_step_accepted + result["per_step_drafted"] = self.per_step_drafted + return result + + @dataclass class PromptTokenStats: """Breakdown of prompt tokens by source. diff --git a/vllm/v1/request.py b/vllm/v1/request.py index 44de9ad1f7..8b453a0906 100644 --- a/vllm/v1/request.py +++ b/vllm/v1/request.py @@ -20,7 +20,7 @@ from vllm.v1.engine import ( EngineCoreRequest, FinishReason, ) -from vllm.v1.metrics.stats import PrefillStats +from vllm.v1.metrics.stats import PrefillStats, RequestSpecDecodeMetrics from vllm.v1.structured_output.request import StructuredOutputRequest from vllm.v1.utils import ConstantList @@ -211,6 +211,11 @@ class Request: self.prefill_stats: PrefillStats | None = PrefillStats() + # Per-request speculative-decoding acceptance accumulator. Populated by + # the scheduler when --per-request-spec-decode-metrics is set (eagerly on + # add_request, then observed each verify step); stays None otherwise. + self.spec_decode_metrics: RequestSpecDecodeMetrics | None = None + self.block_hashes: list[BlockHash] = [] # Store the block hasher without binding self to avoid creating a # reference cycle (Request -> partial -> Request) that prevents