[Frontend][Core][Spec Decode] Per-request acceptance stats in OpenAI API responses (#48915)

Signed-off-by: Matthew Kotila <7692737+matthewkotila@users.noreply.github.com>
Co-authored-by: Reed Meyerson <31574681+reed-meyerson@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Matthew Kotila
2026-08-20 15:42:49 -07:00
committed by GitHub
co-authored by Reed Meyerson Claude Opus 4.8
parent 2f41c894e3
commit 7cfb97e337
25 changed files with 798 additions and 76 deletions
@@ -23,7 +23,7 @@ from vllm.renderers.hf import HfRenderer
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.tokenizers.registry import cached_tokenizer_from_config
from vllm.v1.engine.async_llm import AsyncLLM
from vllm.v1.metrics.stats import RequestStateStats
from vllm.v1.metrics.stats import RequestSpecDecodeMetrics, RequestStateStats
MODEL_NAME = "openai-community/gpt2"
MODEL_NAME_SHORT = "gpt2"
@@ -196,6 +196,114 @@ def test_completion_per_request_metrics_suppressed_for_multiple_prompts():
assert response.metrics is None
def _spec_decode_metrics() -> RequestSpecDecodeMetrics:
# Two verify steps: accept 3 drafts, then 1 -> histogram [0, 1, 0, 1].
m = RequestSpecDecodeMetrics.new(num_spec_tokens=3)
m.observe(num_draft_tokens=3, num_accepted=3)
m.observe(num_draft_tokens=3, num_accepted=1)
return m
def _make_spec_decode_request_output(
num_seqs: int = 1, with_metrics: bool = True
) -> RequestOutput:
outputs = [
CompletionOutput(
index=i,
text="Hello",
token_ids=[100, 101],
cumulative_logprob=None,
logprobs=None,
finish_reason="stop",
spec_decode_metrics=_spec_decode_metrics() if with_metrics else None,
)
for i in range(num_seqs)
]
return RequestOutput(
request_id="test-id",
prompt="Test prompt",
prompt_token_ids=[1, 2, 3],
prompt_logprobs=None,
outputs=outputs,
finished=True,
metrics=None,
)
def _completion_response(serving, request, request_output):
return serving.request_output_to_completion_response(
[request_output],
request,
"cmpl-test-id",
0,
MODEL_NAME,
None,
RequestResponseMetadata(request_id="cmpl-test-id"),
)
def test_completion_spec_decode_metrics_present_for_single_sequence():
# Timing off, but the sequence carries acceptance metrics -> the metrics
# object is created just to hold metrics.speculative_decoding.
serving = _build_minimal_metrics_serving_completion(
enable_per_request_metrics=False
)
response = _completion_response(
serving,
CompletionRequest(model=MODEL_NAME, prompt="Test prompt", max_tokens=10),
_make_spec_decode_request_output(num_seqs=1),
)
assert response.metrics is not None
assert response.metrics.time_to_first_token_ms is None # timing not requested
spec = response.metrics.speculative_decoding
assert spec is not None
assert spec.acceptance_histogram == [0, 1, 0, 1] # dense, index j
assert spec.num_spec_steps == 2
assert spec.num_spec_tokens == 3
assert spec.mean_acceptance_length == pytest.approx(3.0) # 1 + (3 + 1) / 2
def test_completion_spec_decode_metrics_suppressed_for_n_gt_1():
# Per-request metrics can't be attributed to one of the n sequences.
serving = _build_minimal_metrics_serving_completion(
enable_per_request_metrics=False
)
response = _completion_response(
serving,
CompletionRequest(model=MODEL_NAME, prompt="Test prompt", n=2, max_tokens=10),
_make_spec_decode_request_output(num_seqs=2),
)
assert response.metrics is None
def test_completion_spec_decode_metrics_absent_when_not_collected():
# Flag off -> the sequence carries no acceptance metrics -> no metrics object.
serving = _build_minimal_metrics_serving_completion(
enable_per_request_metrics=False
)
response = _completion_response(
serving,
CompletionRequest(model=MODEL_NAME, prompt="Test prompt", max_tokens=10),
_make_spec_decode_request_output(num_seqs=1, with_metrics=False),
)
assert response.metrics is None
def test_completion_metrics_carries_both_timing_and_spec_decode():
serving = _build_minimal_metrics_serving_completion(enable_per_request_metrics=True)
request_output = _make_spec_decode_request_output(num_seqs=1)
request_output.metrics = _PER_REQUEST_STATS # timing source
response = _completion_response(
serving,
CompletionRequest(model=MODEL_NAME, prompt="Test prompt", max_tokens=10),
request_output,
)
assert response.metrics is not None
assert response.metrics.time_to_first_token_ms == pytest.approx(500.0)
assert response.metrics.speculative_decoding is not None
assert response.metrics.speculative_decoding.num_spec_steps == 2
@pytest.mark.asyncio
async def test_completion_error_non_stream():
"""test finish_reason='error' returns 500 InternalServerError (non-streaming)"""
+14
View File
@@ -20,6 +20,7 @@ from vllm.config import (
CompilationConfig,
KernelConfig,
ModelConfig,
ObservabilityConfig,
ParallelConfig,
PoolerConfig,
SchedulerConfig,
@@ -85,6 +86,19 @@ def test_kda_recoverssm_derivation_is_revalidated():
VllmConfig.validate_mamba_cached_kernel(config)
def test_per_request_spec_decode_metrics_requires_spec_decode():
# The flag only makes sense with speculative decoding configured; enabling
# it without --speculative-config should fail fast rather than silently
# produce no metrics.
for level in ("summary", "detailed"):
with pytest.raises(ValueError, match="speculative"):
VllmConfig(
observability_config=ObservabilityConfig(
per_request_spec_decode_metrics=level
)
)
def test_compile_config_repr_succeeds():
# setup: VllmBackend mutates the config object
config = VllmConfig()
+113 -9
View File
@@ -1306,29 +1306,42 @@ def test_draft_slots_budgeted_per_scheduled_request(tmp_path, monkeypatch):
# Note - these test cases mirror some of those in test_rejection_sampler.py
@pytest.mark.parametrize(
"spec_tokens,output_tokens,expected",
"spec_tokens,output_tokens,expected,expected_per_req",
[
([[1, 2, 3]], [[1, 2, 3, 4]], (1, 3, 3, [1, 1, 1])), # perfect match
([[1, 2, 3]], [[1, 5]], (1, 3, 1, [1, 0, 0])), # early mismatch
([[1, 2], [3]], [[1, 2, 5], [3, 4]], (2, 3, 3, [2, 1])), # multiple sequences
([[1]], [[1, 2]], (1, 1, 1, [1])), # single token sequence
([[]], [[5]], (0, 0, 0, [0])), # empty sequence
([[1, 2, 3]], [[1, 2, 3, 4]], (1, 3, 3, [1, 1, 1]), [[0, 0, 0, 1]]), # perfect
([[1, 2, 3]], [[1, 5]], (1, 3, 1, [1, 0, 0]), [[0, 1, 0, 0]]), # early mismatch
(
[[1, 2], [3]],
[[1, 2, 5], [3, 4]],
(2, 3, 3, [2, 1]),
[[0, 0, 1], [0, 1, 0]],
), # multiple sequences
([[1]], [[1, 2]], (1, 1, 1, [1]), [[0, 1]]), # single token sequence
([[]], [[5]], (0, 0, 0, [0]), [[0, 0]]), # empty sequence -> empty accumulator
(
[[1, 2, 3], [4, 5, 6]],
[[1, 2, 7], [4, 8]],
(2, 6, 3, [2, 1, 0]),
[[0, 0, 1, 0], [0, 1, 0, 0]],
), # multiple mismatches
],
)
def test_schedule_spec_decoding_stats(spec_tokens, output_tokens, expected):
def test_schedule_spec_decoding_stats(
spec_tokens, output_tokens, expected, expected_per_req
):
"""Test scheduling behavior with speculative decoding.
This test verifies that:
1. Speculated tokens get scheduled correctly
2. Spec decoding stats properly count number of draft and accepted tokens
2. The aggregate SpecDecodingStats count draft and accepted tokens
3. The per-request accumulator (enabled via per_request_spec_decode_metrics)
buckets the same acceptance by accepted draft count (j)
"""
num_spec_tokens = max(1, max(len(t) for t in spec_tokens))
scheduler = create_scheduler(num_speculative_tokens=num_spec_tokens)
scheduler = create_scheduler(
num_speculative_tokens=num_spec_tokens,
per_request_spec_decode_metrics="summary",
)
requests = create_requests(num_requests=len(spec_tokens), num_tokens=1)
req_ids = []
req_to_index = {}
@@ -1414,6 +1427,97 @@ def test_schedule_spec_decoding_stats(spec_tokens, output_tokens, expected):
assert stats.num_accepted_tokens == expected[2]
assert stats.num_accepted_tokens_per_pos == expected[3]
# Per-request accumulator: the same acceptance, bucketed by accepted draft
# count (j) on each request rather than summed across the batch. The
# accumulator is created eagerly on add_request, so every request has one
# (an empty histogram when it drafted nothing).
for i, req_id in enumerate(req_ids):
payload = scheduler.requests[req_id].spec_decode_metrics.to_dict()
assert payload["acceptance_histogram"] == expected_per_req[i]
assert payload["num_draft_tokens"] == len(spec_tokens[i])
assert "per_step_accepted" not in payload # summary level
def _run_spec_verify_steps(scheduler, rounds, num_invalid_per_round=None):
"""Drive prefill + one draft/verify step per round for a single request.
``rounds`` is a list of ``(spec_token_ids, output_token_ids)`` -- one verify
step each. ``num_invalid_per_round`` optionally injects the grammar-invalid
draft count structured-output decoding would set on the verify output.
Returns the request.
"""
[req] = create_requests(num_requests=1, num_tokens=1)
scheduler.add_request(req)
rid = req.request_id
req_to_index = {rid: 0}
def _mk_output(sampled):
return ModelRunnerOutput(
req_ids=[rid],
req_id_to_index=req_to_index,
sampled_token_ids=sampled,
logprobs=None,
prompt_logprobs_dict={},
pooler_output=[],
)
# Prefill: sample one token.
scheduler.update_from_output(scheduler.schedule(), _mk_output([[0]]))
for i, (spec, out) in enumerate(rounds):
scheduler.update_draft_token_ids(DraftTokenIds([rid], [spec]))
output = scheduler.schedule()
if num_invalid_per_round is not None and num_invalid_per_round[i]:
output.num_invalid_spec_tokens = {rid: num_invalid_per_round[i]}
scheduler.update_from_output(output, _mk_output([out]))
return req
def test_per_request_spec_decode_detailed_records_per_step():
scheduler = create_scheduler(
num_speculative_tokens=3,
per_request_spec_decode_metrics="detailed",
)
# Three verify steps for one request; num_accepted = len(output) - 1, so the
# outputs below accept 3, 1, then 0 drafts across the steps.
req = _run_spec_verify_steps(
scheduler,
[
([1, 2, 3], [1, 2, 3, 4]), # accept 3
([5, 6, 7], [5, 8]), # accept 1
([9, 10, 11], [12]), # accept 0
],
)
payload = scheduler.requests[req.request_id].spec_decode_metrics.to_dict()
assert payload["per_step_accepted"] == [3, 1, 0]
assert payload["per_step_drafted"] == [3, 3, 3]
assert payload["num_spec_steps"] == 3
def test_per_request_spec_decode_subtracts_invalid_drafts():
# Grammar-invalidated drafts (num_invalid_spec_tokens, set by structured
# output) are excluded from the proposed count, mirroring the aggregate.
scheduler = create_scheduler(
num_speculative_tokens=3,
per_request_spec_decode_metrics="summary",
)
# One verify step: 3 drafted, 1 grammar-invalid, output accepts 2.
req = _run_spec_verify_steps(
scheduler,
[([1, 2, 3], [1, 2, 5])],
num_invalid_per_round=[1],
)
payload = scheduler.requests[req.request_id].spec_decode_metrics.to_dict()
assert payload["num_draft_tokens"] == 2 # 3 drafted - 1 invalid
assert payload["num_accepted_draft_tokens"] == 2 # len([1,2,5]) - 1
assert payload["acceptance_histogram"] == [0, 0, 1, 0]
def test_per_request_spec_decode_acceptance_disabled_by_default():
scheduler = create_scheduler(num_speculative_tokens=3)
assert scheduler.spec_decode_metrics_level == "none"
req = _run_spec_verify_steps(scheduler, [([1, 2, 3], [1, 2, 3, 4])])
assert scheduler.requests[req.request_id].spec_decode_metrics is None
def test_spec_decoding_stats_empty_output():
"""Test that spec decoding stats handle empty output tokens gracefully.
+5
View File
@@ -10,6 +10,7 @@ from vllm.config import (
ECTransferConfig,
KVTransferConfig,
ModelConfig,
ObservabilityConfig,
ParallelConfig,
SchedulerConfig,
SpeculativeConfig,
@@ -71,6 +72,7 @@ def create_scheduler(
ec_role: str | None = None,
use_v2_model_runner: bool | None = None,
kv_cache_spec: KVCacheSpec | None = None,
per_request_spec_decode_metrics: str = "none",
) -> Scheduler | AsyncScheduler:
"""Create scheduler under test.
@@ -175,6 +177,9 @@ def create_scheduler(
kv_transfer_config=kv_transfer_config,
speculative_config=speculative_config,
ec_transfer_config=ec_transfer_config,
observability_config=ObservabilityConfig(
per_request_spec_decode_metrics=per_request_spec_decode_metrics,
),
)
if kv_cache_spec is None:
kv_cache_spec = FullAttentionSpec(
@@ -0,0 +1,150 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for per-request speculative-decoding metrics accumulation.
These cover the pure histogram/per-step math (no GPU / no model): the engine-core
accumulator ``RequestSpecDecodeMetrics`` and its ``to_dict`` payload surfaced in the
response as ``metrics.speculative_decoding``.
"""
import msgspec
import pytest
from vllm.outputs import CompletionOutput
from vllm.v1.engine import EngineCoreOutput
from vllm.v1.metrics.stats import RequestSpecDecodeMetrics
def _metrics(pairs, num_spec_tokens=3, detailed=False):
s = RequestSpecDecodeMetrics.new(num_spec_tokens)
for k, j in pairs:
s.observe(num_draft_tokens=k, num_accepted=j, detailed=detailed)
return s
def test_new_allocates_dense_histogram_of_k_plus_one():
s = RequestSpecDecodeMetrics.new(num_spec_tokens=3)
assert s.num_spec_tokens == 3
assert s.histogram == [0, 0, 0, 0]
assert s.num_draft_tokens == 0
assert s.per_step_accepted == []
def test_observe_buckets_by_accepted_draft_count():
s = _metrics([(3, 0), (3, 3), (3, 2), (3, 3), (3, 1)])
# j=0 ->1, j=1 ->1, j=2 ->1, j=3 ->2
assert s.histogram == [1, 1, 1, 2]
assert s.num_draft_tokens == 15
# summary level does not record the ordered per-step arrays
assert s.per_step_accepted == []
assert s.per_step_drafted == []
def test_observe_detailed_records_ordered_per_step_arrays():
# Distinct step count (3), max draft length (k=4), and per-step drafted
# counts (4, 2, 4) so no accidental "everything is 3" pattern is implied.
s = _metrics([(4, 3), (2, 2), (4, 0)], num_spec_tokens=4, detailed=True)
assert s.per_step_accepted == [3, 2, 0]
assert s.per_step_drafted == [4, 2, 4]
# histogram (indexed by accepted j, length k+1=5) is still maintained
assert s.histogram == [1, 0, 1, 1, 0]
def test_to_dict_summary_omits_per_step_arrays():
d = _metrics([(3, 0), (3, 3), (3, 2), (3, 3), (3, 1)]).to_dict()
assert d == {
"mean_acceptance_length": pytest.approx(1 + 9 / 5), # j+1
"draft_acceptance_rate": pytest.approx(9 / 15),
"acceptance_histogram": [1, 1, 1, 2], # dense, index j = step count
"num_spec_steps": 5,
"num_accepted_draft_tokens": 9,
"num_draft_tokens": 15,
"num_spec_tokens": 3,
}
def test_to_dict_detailed_appends_per_step_arrays():
d = _metrics([(3, 3), (3, 2), (3, 0)], detailed=True).to_dict()
assert d["per_step_accepted"] == [3, 2, 0]
assert d["per_step_drafted"] == [3, 3, 3]
# summary fields still present in detailed mode
assert d["num_spec_steps"] == 3
assert d["num_accepted_draft_tokens"] == 5
def test_to_dict_histogram_is_dense_list_indexed_by_j():
# length k+1, index j holds the step count that accepted exactly j drafts
d = _metrics([(3, 0), (3, 0), (3, 3)]).to_dict()
assert d["acceptance_histogram"] == [2, 0, 0, 1]
def test_all_rejected_gives_mean_one_and_rate_zero():
d = _metrics([(2, 0), (2, 0), (2, 0), (2, 0)], num_spec_tokens=2).to_dict()
assert d["num_spec_steps"] == 4
assert d["num_accepted_draft_tokens"] == 0
assert d["acceptance_histogram"] == [4, 0, 0]
assert d["mean_acceptance_length"] == pytest.approx(1.0)
assert d["draft_acceptance_rate"] == pytest.approx(0.0)
def test_empty_metrics_do_not_divide_by_zero():
d = RequestSpecDecodeMetrics.new(3).to_dict()
assert d["num_spec_steps"] == 0
assert d["num_draft_tokens"] == 0
assert d["draft_acceptance_rate"] == 0.0
assert d["mean_acceptance_length"] == 1.0
assert "per_step_accepted" not in d
def test_observe_records_proposed_and_accepted_independently():
# observe() takes proposed and accepted as independent inputs: the histogram
# is keyed by accepted, num_draft_tokens sums the proposed as given. (The
# grammar-invalidated-draft subtraction happens in the scheduler before
# observe() -- see test_per_request_spec_decode_subtracts_invalid_drafts.)
s = RequestSpecDecodeMetrics.new(num_spec_tokens=3)
s.observe(num_draft_tokens=2, num_accepted=1)
s.observe(num_draft_tokens=3, num_accepted=1)
d = s.to_dict()
assert d["acceptance_histogram"] == [0, 2, 0, 0] # both steps accepted 1
assert d["num_draft_tokens"] == 5 # proposed summed independently: 2 + 3
def test_engine_core_output_round_trips_spec_decode_metrics():
# The accumulator rides EngineCoreOutput (msgspec, array_like) to the
# frontend; verify it serializes (incl. per-step arrays) and is omitted
# when absent. ``new_token_ids`` is EngineCoreOutput's required "tokens
# generated this step" field -- a dummy value here since we only exercise
# spec_decode_metrics.
metrics = _metrics([(3, 0), (3, 3), (3, 2)], detailed=True)
out = EngineCoreOutput(
request_id="r1", new_token_ids=[1, 2], spec_decode_metrics=metrics
)
decoder = msgspec.msgpack.Decoder(EngineCoreOutput)
decoded = decoder.decode(msgspec.msgpack.encode(out))
assert decoded.spec_decode_metrics.histogram == [1, 0, 1, 1]
assert decoded.spec_decode_metrics.num_draft_tokens == 9
assert decoded.spec_decode_metrics.per_step_accepted == [0, 3, 2]
without = EngineCoreOutput(request_id="r2", new_token_ids=[1])
decoded_without = decoder.decode(msgspec.msgpack.encode(without))
assert decoded_without.spec_decode_metrics is None
def _completion_output(**kwargs):
return CompletionOutput(
index=0,
text="",
token_ids=[],
cumulative_logprob=None,
logprobs=None,
**kwargs,
)
def test_completion_output_carries_spec_decode_metrics():
# Metrics are per output sequence, so they ride the CompletionOutput
# (=> choices[i]), not the request-level RequestOutput.
metrics = _metrics([(3, 3), (3, 1)])
out = _completion_output(spec_decode_metrics=metrics)
assert out.spec_decode_metrics is metrics
assert _completion_output().spec_decode_metrics is None