[Frontend][Core][Spec Decode] Per-request acceptance stats in OpenAI API responses (#48915)
Signed-off-by: Matthew Kotila <7692737+matthewkotila@users.noreply.github.com> Co-authored-by: Reed Meyerson <31574681+reed-meyerson@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Reed Meyerson
Claude Opus 4.8
parent
2f41c894e3
commit
7cfb97e337
@@ -23,7 +23,7 @@ from vllm.renderers.hf import HfRenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
from vllm.v1.metrics.stats import RequestStateStats
|
||||
from vllm.v1.metrics.stats import RequestSpecDecodeMetrics, RequestStateStats
|
||||
|
||||
MODEL_NAME = "openai-community/gpt2"
|
||||
MODEL_NAME_SHORT = "gpt2"
|
||||
@@ -196,6 +196,114 @@ def test_completion_per_request_metrics_suppressed_for_multiple_prompts():
|
||||
assert response.metrics is None
|
||||
|
||||
|
||||
def _spec_decode_metrics() -> RequestSpecDecodeMetrics:
|
||||
# Two verify steps: accept 3 drafts, then 1 -> histogram [0, 1, 0, 1].
|
||||
m = RequestSpecDecodeMetrics.new(num_spec_tokens=3)
|
||||
m.observe(num_draft_tokens=3, num_accepted=3)
|
||||
m.observe(num_draft_tokens=3, num_accepted=1)
|
||||
return m
|
||||
|
||||
|
||||
def _make_spec_decode_request_output(
|
||||
num_seqs: int = 1, with_metrics: bool = True
|
||||
) -> RequestOutput:
|
||||
outputs = [
|
||||
CompletionOutput(
|
||||
index=i,
|
||||
text="Hello",
|
||||
token_ids=[100, 101],
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
finish_reason="stop",
|
||||
spec_decode_metrics=_spec_decode_metrics() if with_metrics else None,
|
||||
)
|
||||
for i in range(num_seqs)
|
||||
]
|
||||
return RequestOutput(
|
||||
request_id="test-id",
|
||||
prompt="Test prompt",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
prompt_logprobs=None,
|
||||
outputs=outputs,
|
||||
finished=True,
|
||||
metrics=None,
|
||||
)
|
||||
|
||||
|
||||
def _completion_response(serving, request, request_output):
|
||||
return serving.request_output_to_completion_response(
|
||||
[request_output],
|
||||
request,
|
||||
"cmpl-test-id",
|
||||
0,
|
||||
MODEL_NAME,
|
||||
None,
|
||||
RequestResponseMetadata(request_id="cmpl-test-id"),
|
||||
)
|
||||
|
||||
|
||||
def test_completion_spec_decode_metrics_present_for_single_sequence():
|
||||
# Timing off, but the sequence carries acceptance metrics -> the metrics
|
||||
# object is created just to hold metrics.speculative_decoding.
|
||||
serving = _build_minimal_metrics_serving_completion(
|
||||
enable_per_request_metrics=False
|
||||
)
|
||||
response = _completion_response(
|
||||
serving,
|
||||
CompletionRequest(model=MODEL_NAME, prompt="Test prompt", max_tokens=10),
|
||||
_make_spec_decode_request_output(num_seqs=1),
|
||||
)
|
||||
assert response.metrics is not None
|
||||
assert response.metrics.time_to_first_token_ms is None # timing not requested
|
||||
spec = response.metrics.speculative_decoding
|
||||
assert spec is not None
|
||||
assert spec.acceptance_histogram == [0, 1, 0, 1] # dense, index j
|
||||
assert spec.num_spec_steps == 2
|
||||
assert spec.num_spec_tokens == 3
|
||||
assert spec.mean_acceptance_length == pytest.approx(3.0) # 1 + (3 + 1) / 2
|
||||
|
||||
|
||||
def test_completion_spec_decode_metrics_suppressed_for_n_gt_1():
|
||||
# Per-request metrics can't be attributed to one of the n sequences.
|
||||
serving = _build_minimal_metrics_serving_completion(
|
||||
enable_per_request_metrics=False
|
||||
)
|
||||
response = _completion_response(
|
||||
serving,
|
||||
CompletionRequest(model=MODEL_NAME, prompt="Test prompt", n=2, max_tokens=10),
|
||||
_make_spec_decode_request_output(num_seqs=2),
|
||||
)
|
||||
assert response.metrics is None
|
||||
|
||||
|
||||
def test_completion_spec_decode_metrics_absent_when_not_collected():
|
||||
# Flag off -> the sequence carries no acceptance metrics -> no metrics object.
|
||||
serving = _build_minimal_metrics_serving_completion(
|
||||
enable_per_request_metrics=False
|
||||
)
|
||||
response = _completion_response(
|
||||
serving,
|
||||
CompletionRequest(model=MODEL_NAME, prompt="Test prompt", max_tokens=10),
|
||||
_make_spec_decode_request_output(num_seqs=1, with_metrics=False),
|
||||
)
|
||||
assert response.metrics is None
|
||||
|
||||
|
||||
def test_completion_metrics_carries_both_timing_and_spec_decode():
|
||||
serving = _build_minimal_metrics_serving_completion(enable_per_request_metrics=True)
|
||||
request_output = _make_spec_decode_request_output(num_seqs=1)
|
||||
request_output.metrics = _PER_REQUEST_STATS # timing source
|
||||
response = _completion_response(
|
||||
serving,
|
||||
CompletionRequest(model=MODEL_NAME, prompt="Test prompt", max_tokens=10),
|
||||
request_output,
|
||||
)
|
||||
assert response.metrics is not None
|
||||
assert response.metrics.time_to_first_token_ms == pytest.approx(500.0)
|
||||
assert response.metrics.speculative_decoding is not None
|
||||
assert response.metrics.speculative_decoding.num_spec_steps == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_error_non_stream():
|
||||
"""test finish_reason='error' returns 500 InternalServerError (non-streaming)"""
|
||||
|
||||
@@ -20,6 +20,7 @@ from vllm.config import (
|
||||
CompilationConfig,
|
||||
KernelConfig,
|
||||
ModelConfig,
|
||||
ObservabilityConfig,
|
||||
ParallelConfig,
|
||||
PoolerConfig,
|
||||
SchedulerConfig,
|
||||
@@ -85,6 +86,19 @@ def test_kda_recoverssm_derivation_is_revalidated():
|
||||
VllmConfig.validate_mamba_cached_kernel(config)
|
||||
|
||||
|
||||
def test_per_request_spec_decode_metrics_requires_spec_decode():
|
||||
# The flag only makes sense with speculative decoding configured; enabling
|
||||
# it without --speculative-config should fail fast rather than silently
|
||||
# produce no metrics.
|
||||
for level in ("summary", "detailed"):
|
||||
with pytest.raises(ValueError, match="speculative"):
|
||||
VllmConfig(
|
||||
observability_config=ObservabilityConfig(
|
||||
per_request_spec_decode_metrics=level
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_compile_config_repr_succeeds():
|
||||
# setup: VllmBackend mutates the config object
|
||||
config = VllmConfig()
|
||||
|
||||
@@ -1306,29 +1306,42 @@ def test_draft_slots_budgeted_per_scheduled_request(tmp_path, monkeypatch):
|
||||
|
||||
# Note - these test cases mirror some of those in test_rejection_sampler.py
|
||||
@pytest.mark.parametrize(
|
||||
"spec_tokens,output_tokens,expected",
|
||||
"spec_tokens,output_tokens,expected,expected_per_req",
|
||||
[
|
||||
([[1, 2, 3]], [[1, 2, 3, 4]], (1, 3, 3, [1, 1, 1])), # perfect match
|
||||
([[1, 2, 3]], [[1, 5]], (1, 3, 1, [1, 0, 0])), # early mismatch
|
||||
([[1, 2], [3]], [[1, 2, 5], [3, 4]], (2, 3, 3, [2, 1])), # multiple sequences
|
||||
([[1]], [[1, 2]], (1, 1, 1, [1])), # single token sequence
|
||||
([[]], [[5]], (0, 0, 0, [0])), # empty sequence
|
||||
([[1, 2, 3]], [[1, 2, 3, 4]], (1, 3, 3, [1, 1, 1]), [[0, 0, 0, 1]]), # perfect
|
||||
([[1, 2, 3]], [[1, 5]], (1, 3, 1, [1, 0, 0]), [[0, 1, 0, 0]]), # early mismatch
|
||||
(
|
||||
[[1, 2], [3]],
|
||||
[[1, 2, 5], [3, 4]],
|
||||
(2, 3, 3, [2, 1]),
|
||||
[[0, 0, 1], [0, 1, 0]],
|
||||
), # multiple sequences
|
||||
([[1]], [[1, 2]], (1, 1, 1, [1]), [[0, 1]]), # single token sequence
|
||||
([[]], [[5]], (0, 0, 0, [0]), [[0, 0]]), # empty sequence -> empty accumulator
|
||||
(
|
||||
[[1, 2, 3], [4, 5, 6]],
|
||||
[[1, 2, 7], [4, 8]],
|
||||
(2, 6, 3, [2, 1, 0]),
|
||||
[[0, 0, 1, 0], [0, 1, 0, 0]],
|
||||
), # multiple mismatches
|
||||
],
|
||||
)
|
||||
def test_schedule_spec_decoding_stats(spec_tokens, output_tokens, expected):
|
||||
def test_schedule_spec_decoding_stats(
|
||||
spec_tokens, output_tokens, expected, expected_per_req
|
||||
):
|
||||
"""Test scheduling behavior with speculative decoding.
|
||||
|
||||
This test verifies that:
|
||||
1. Speculated tokens get scheduled correctly
|
||||
2. Spec decoding stats properly count number of draft and accepted tokens
|
||||
2. The aggregate SpecDecodingStats count draft and accepted tokens
|
||||
3. The per-request accumulator (enabled via per_request_spec_decode_metrics)
|
||||
buckets the same acceptance by accepted draft count (j)
|
||||
"""
|
||||
num_spec_tokens = max(1, max(len(t) for t in spec_tokens))
|
||||
scheduler = create_scheduler(num_speculative_tokens=num_spec_tokens)
|
||||
scheduler = create_scheduler(
|
||||
num_speculative_tokens=num_spec_tokens,
|
||||
per_request_spec_decode_metrics="summary",
|
||||
)
|
||||
requests = create_requests(num_requests=len(spec_tokens), num_tokens=1)
|
||||
req_ids = []
|
||||
req_to_index = {}
|
||||
@@ -1414,6 +1427,97 @@ def test_schedule_spec_decoding_stats(spec_tokens, output_tokens, expected):
|
||||
assert stats.num_accepted_tokens == expected[2]
|
||||
assert stats.num_accepted_tokens_per_pos == expected[3]
|
||||
|
||||
# Per-request accumulator: the same acceptance, bucketed by accepted draft
|
||||
# count (j) on each request rather than summed across the batch. The
|
||||
# accumulator is created eagerly on add_request, so every request has one
|
||||
# (an empty histogram when it drafted nothing).
|
||||
for i, req_id in enumerate(req_ids):
|
||||
payload = scheduler.requests[req_id].spec_decode_metrics.to_dict()
|
||||
assert payload["acceptance_histogram"] == expected_per_req[i]
|
||||
assert payload["num_draft_tokens"] == len(spec_tokens[i])
|
||||
assert "per_step_accepted" not in payload # summary level
|
||||
|
||||
|
||||
def _run_spec_verify_steps(scheduler, rounds, num_invalid_per_round=None):
|
||||
"""Drive prefill + one draft/verify step per round for a single request.
|
||||
|
||||
``rounds`` is a list of ``(spec_token_ids, output_token_ids)`` -- one verify
|
||||
step each. ``num_invalid_per_round`` optionally injects the grammar-invalid
|
||||
draft count structured-output decoding would set on the verify output.
|
||||
Returns the request.
|
||||
"""
|
||||
[req] = create_requests(num_requests=1, num_tokens=1)
|
||||
scheduler.add_request(req)
|
||||
rid = req.request_id
|
||||
req_to_index = {rid: 0}
|
||||
|
||||
def _mk_output(sampled):
|
||||
return ModelRunnerOutput(
|
||||
req_ids=[rid],
|
||||
req_id_to_index=req_to_index,
|
||||
sampled_token_ids=sampled,
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
)
|
||||
|
||||
# Prefill: sample one token.
|
||||
scheduler.update_from_output(scheduler.schedule(), _mk_output([[0]]))
|
||||
for i, (spec, out) in enumerate(rounds):
|
||||
scheduler.update_draft_token_ids(DraftTokenIds([rid], [spec]))
|
||||
output = scheduler.schedule()
|
||||
if num_invalid_per_round is not None and num_invalid_per_round[i]:
|
||||
output.num_invalid_spec_tokens = {rid: num_invalid_per_round[i]}
|
||||
scheduler.update_from_output(output, _mk_output([out]))
|
||||
return req
|
||||
|
||||
|
||||
def test_per_request_spec_decode_detailed_records_per_step():
|
||||
scheduler = create_scheduler(
|
||||
num_speculative_tokens=3,
|
||||
per_request_spec_decode_metrics="detailed",
|
||||
)
|
||||
# Three verify steps for one request; num_accepted = len(output) - 1, so the
|
||||
# outputs below accept 3, 1, then 0 drafts across the steps.
|
||||
req = _run_spec_verify_steps(
|
||||
scheduler,
|
||||
[
|
||||
([1, 2, 3], [1, 2, 3, 4]), # accept 3
|
||||
([5, 6, 7], [5, 8]), # accept 1
|
||||
([9, 10, 11], [12]), # accept 0
|
||||
],
|
||||
)
|
||||
payload = scheduler.requests[req.request_id].spec_decode_metrics.to_dict()
|
||||
assert payload["per_step_accepted"] == [3, 1, 0]
|
||||
assert payload["per_step_drafted"] == [3, 3, 3]
|
||||
assert payload["num_spec_steps"] == 3
|
||||
|
||||
|
||||
def test_per_request_spec_decode_subtracts_invalid_drafts():
|
||||
# Grammar-invalidated drafts (num_invalid_spec_tokens, set by structured
|
||||
# output) are excluded from the proposed count, mirroring the aggregate.
|
||||
scheduler = create_scheduler(
|
||||
num_speculative_tokens=3,
|
||||
per_request_spec_decode_metrics="summary",
|
||||
)
|
||||
# One verify step: 3 drafted, 1 grammar-invalid, output accepts 2.
|
||||
req = _run_spec_verify_steps(
|
||||
scheduler,
|
||||
[([1, 2, 3], [1, 2, 5])],
|
||||
num_invalid_per_round=[1],
|
||||
)
|
||||
payload = scheduler.requests[req.request_id].spec_decode_metrics.to_dict()
|
||||
assert payload["num_draft_tokens"] == 2 # 3 drafted - 1 invalid
|
||||
assert payload["num_accepted_draft_tokens"] == 2 # len([1,2,5]) - 1
|
||||
assert payload["acceptance_histogram"] == [0, 0, 1, 0]
|
||||
|
||||
|
||||
def test_per_request_spec_decode_acceptance_disabled_by_default():
|
||||
scheduler = create_scheduler(num_speculative_tokens=3)
|
||||
assert scheduler.spec_decode_metrics_level == "none"
|
||||
req = _run_spec_verify_steps(scheduler, [([1, 2, 3], [1, 2, 3, 4])])
|
||||
assert scheduler.requests[req.request_id].spec_decode_metrics is None
|
||||
|
||||
|
||||
def test_spec_decoding_stats_empty_output():
|
||||
"""Test that spec decoding stats handle empty output tokens gracefully.
|
||||
|
||||
@@ -10,6 +10,7 @@ from vllm.config import (
|
||||
ECTransferConfig,
|
||||
KVTransferConfig,
|
||||
ModelConfig,
|
||||
ObservabilityConfig,
|
||||
ParallelConfig,
|
||||
SchedulerConfig,
|
||||
SpeculativeConfig,
|
||||
@@ -71,6 +72,7 @@ def create_scheduler(
|
||||
ec_role: str | None = None,
|
||||
use_v2_model_runner: bool | None = None,
|
||||
kv_cache_spec: KVCacheSpec | None = None,
|
||||
per_request_spec_decode_metrics: str = "none",
|
||||
) -> Scheduler | AsyncScheduler:
|
||||
"""Create scheduler under test.
|
||||
|
||||
@@ -175,6 +177,9 @@ def create_scheduler(
|
||||
kv_transfer_config=kv_transfer_config,
|
||||
speculative_config=speculative_config,
|
||||
ec_transfer_config=ec_transfer_config,
|
||||
observability_config=ObservabilityConfig(
|
||||
per_request_spec_decode_metrics=per_request_spec_decode_metrics,
|
||||
),
|
||||
)
|
||||
if kv_cache_spec is None:
|
||||
kv_cache_spec = FullAttentionSpec(
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for per-request speculative-decoding metrics accumulation.
|
||||
|
||||
These cover the pure histogram/per-step math (no GPU / no model): the engine-core
|
||||
accumulator ``RequestSpecDecodeMetrics`` and its ``to_dict`` payload surfaced in the
|
||||
response as ``metrics.speculative_decoding``.
|
||||
"""
|
||||
|
||||
import msgspec
|
||||
import pytest
|
||||
|
||||
from vllm.outputs import CompletionOutput
|
||||
from vllm.v1.engine import EngineCoreOutput
|
||||
from vllm.v1.metrics.stats import RequestSpecDecodeMetrics
|
||||
|
||||
|
||||
def _metrics(pairs, num_spec_tokens=3, detailed=False):
|
||||
s = RequestSpecDecodeMetrics.new(num_spec_tokens)
|
||||
for k, j in pairs:
|
||||
s.observe(num_draft_tokens=k, num_accepted=j, detailed=detailed)
|
||||
return s
|
||||
|
||||
|
||||
def test_new_allocates_dense_histogram_of_k_plus_one():
|
||||
s = RequestSpecDecodeMetrics.new(num_spec_tokens=3)
|
||||
assert s.num_spec_tokens == 3
|
||||
assert s.histogram == [0, 0, 0, 0]
|
||||
assert s.num_draft_tokens == 0
|
||||
assert s.per_step_accepted == []
|
||||
|
||||
|
||||
def test_observe_buckets_by_accepted_draft_count():
|
||||
s = _metrics([(3, 0), (3, 3), (3, 2), (3, 3), (3, 1)])
|
||||
# j=0 ->1, j=1 ->1, j=2 ->1, j=3 ->2
|
||||
assert s.histogram == [1, 1, 1, 2]
|
||||
assert s.num_draft_tokens == 15
|
||||
# summary level does not record the ordered per-step arrays
|
||||
assert s.per_step_accepted == []
|
||||
assert s.per_step_drafted == []
|
||||
|
||||
|
||||
def test_observe_detailed_records_ordered_per_step_arrays():
|
||||
# Distinct step count (3), max draft length (k=4), and per-step drafted
|
||||
# counts (4, 2, 4) so no accidental "everything is 3" pattern is implied.
|
||||
s = _metrics([(4, 3), (2, 2), (4, 0)], num_spec_tokens=4, detailed=True)
|
||||
assert s.per_step_accepted == [3, 2, 0]
|
||||
assert s.per_step_drafted == [4, 2, 4]
|
||||
# histogram (indexed by accepted j, length k+1=5) is still maintained
|
||||
assert s.histogram == [1, 0, 1, 1, 0]
|
||||
|
||||
|
||||
def test_to_dict_summary_omits_per_step_arrays():
|
||||
d = _metrics([(3, 0), (3, 3), (3, 2), (3, 3), (3, 1)]).to_dict()
|
||||
assert d == {
|
||||
"mean_acceptance_length": pytest.approx(1 + 9 / 5), # j+1
|
||||
"draft_acceptance_rate": pytest.approx(9 / 15),
|
||||
"acceptance_histogram": [1, 1, 1, 2], # dense, index j = step count
|
||||
"num_spec_steps": 5,
|
||||
"num_accepted_draft_tokens": 9,
|
||||
"num_draft_tokens": 15,
|
||||
"num_spec_tokens": 3,
|
||||
}
|
||||
|
||||
|
||||
def test_to_dict_detailed_appends_per_step_arrays():
|
||||
d = _metrics([(3, 3), (3, 2), (3, 0)], detailed=True).to_dict()
|
||||
assert d["per_step_accepted"] == [3, 2, 0]
|
||||
assert d["per_step_drafted"] == [3, 3, 3]
|
||||
# summary fields still present in detailed mode
|
||||
assert d["num_spec_steps"] == 3
|
||||
assert d["num_accepted_draft_tokens"] == 5
|
||||
|
||||
|
||||
def test_to_dict_histogram_is_dense_list_indexed_by_j():
|
||||
# length k+1, index j holds the step count that accepted exactly j drafts
|
||||
d = _metrics([(3, 0), (3, 0), (3, 3)]).to_dict()
|
||||
assert d["acceptance_histogram"] == [2, 0, 0, 1]
|
||||
|
||||
|
||||
def test_all_rejected_gives_mean_one_and_rate_zero():
|
||||
d = _metrics([(2, 0), (2, 0), (2, 0), (2, 0)], num_spec_tokens=2).to_dict()
|
||||
assert d["num_spec_steps"] == 4
|
||||
assert d["num_accepted_draft_tokens"] == 0
|
||||
assert d["acceptance_histogram"] == [4, 0, 0]
|
||||
assert d["mean_acceptance_length"] == pytest.approx(1.0)
|
||||
assert d["draft_acceptance_rate"] == pytest.approx(0.0)
|
||||
|
||||
|
||||
def test_empty_metrics_do_not_divide_by_zero():
|
||||
d = RequestSpecDecodeMetrics.new(3).to_dict()
|
||||
assert d["num_spec_steps"] == 0
|
||||
assert d["num_draft_tokens"] == 0
|
||||
assert d["draft_acceptance_rate"] == 0.0
|
||||
assert d["mean_acceptance_length"] == 1.0
|
||||
assert "per_step_accepted" not in d
|
||||
|
||||
|
||||
def test_observe_records_proposed_and_accepted_independently():
|
||||
# observe() takes proposed and accepted as independent inputs: the histogram
|
||||
# is keyed by accepted, num_draft_tokens sums the proposed as given. (The
|
||||
# grammar-invalidated-draft subtraction happens in the scheduler before
|
||||
# observe() -- see test_per_request_spec_decode_subtracts_invalid_drafts.)
|
||||
s = RequestSpecDecodeMetrics.new(num_spec_tokens=3)
|
||||
s.observe(num_draft_tokens=2, num_accepted=1)
|
||||
s.observe(num_draft_tokens=3, num_accepted=1)
|
||||
d = s.to_dict()
|
||||
assert d["acceptance_histogram"] == [0, 2, 0, 0] # both steps accepted 1
|
||||
assert d["num_draft_tokens"] == 5 # proposed summed independently: 2 + 3
|
||||
|
||||
|
||||
def test_engine_core_output_round_trips_spec_decode_metrics():
|
||||
# The accumulator rides EngineCoreOutput (msgspec, array_like) to the
|
||||
# frontend; verify it serializes (incl. per-step arrays) and is omitted
|
||||
# when absent. ``new_token_ids`` is EngineCoreOutput's required "tokens
|
||||
# generated this step" field -- a dummy value here since we only exercise
|
||||
# spec_decode_metrics.
|
||||
metrics = _metrics([(3, 0), (3, 3), (3, 2)], detailed=True)
|
||||
out = EngineCoreOutput(
|
||||
request_id="r1", new_token_ids=[1, 2], spec_decode_metrics=metrics
|
||||
)
|
||||
decoder = msgspec.msgpack.Decoder(EngineCoreOutput)
|
||||
decoded = decoder.decode(msgspec.msgpack.encode(out))
|
||||
assert decoded.spec_decode_metrics.histogram == [1, 0, 1, 1]
|
||||
assert decoded.spec_decode_metrics.num_draft_tokens == 9
|
||||
assert decoded.spec_decode_metrics.per_step_accepted == [0, 3, 2]
|
||||
|
||||
without = EngineCoreOutput(request_id="r2", new_token_ids=[1])
|
||||
decoded_without = decoder.decode(msgspec.msgpack.encode(without))
|
||||
assert decoded_without.spec_decode_metrics is None
|
||||
|
||||
|
||||
def _completion_output(**kwargs):
|
||||
return CompletionOutput(
|
||||
index=0,
|
||||
text="",
|
||||
token_ids=[],
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def test_completion_output_carries_spec_decode_metrics():
|
||||
# Metrics are per output sequence, so they ride the CompletionOutput
|
||||
# (=> choices[i]), not the request-level RequestOutput.
|
||||
metrics = _metrics([(3, 3), (3, 1)])
|
||||
out = _completion_output(spec_decode_metrics=metrics)
|
||||
assert out.spec_decode_metrics is metrics
|
||||
assert _completion_output().spec_decode_metrics is None
|
||||
Reference in New Issue
Block a user