[Frontend][Core][Spec Decode] Per-request acceptance stats in OpenAI API responses (#48915)

Signed-off-by: Matthew Kotila <7692737+matthewkotila@users.noreply.github.com>
Co-authored-by: Reed Meyerson <31574681+reed-meyerson@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Matthew Kotila
2026-08-20 15:42:49 -07:00
committed by GitHub
co-authored by Reed Meyerson Claude Opus 4.8
parent 2f41c894e3
commit 7cfb97e337
25 changed files with 798 additions and 76 deletions
+12
View File
@@ -45,6 +45,18 @@ class ObservabilityConfig:
Note that collecting detailed timing information for each request can be
expensive."""
per_request_spec_decode_metrics: Literal["none", "summary", "detailed"] = "none"
"""Include per-request speculative-decoding acceptance metrics in the
response under `metrics.speculative_decoding`. `none` disables; `summary` adds mean
acceptance length, draft acceptance rate, and the step-by-draft-length
histogram; `detailed` additionally records the ordered per-step
accepted/proposed arrays (one entry per verify step). Only reported for
single-sequence requests (`n == 1`), mirroring the timing metrics. No effect
unless speculative decoding is enabled. Independent of `--disable-log-stats`.
This is the per-request response-body counterpart of the aggregate
`vllm:spec_decode_*` Prometheus metrics. The response field is experimental
and its shape may change in a future release."""
kv_cache_metrics: bool = False
"""Enable KV cache residency metrics (lifetime, idle time, reuse gaps).
Uses sampling to minimize overhead.
+9
View File
@@ -1331,6 +1331,15 @@ class VllmConfig:
)
self.model_config.disable_cascade_attn = True
if (
self.observability_config.per_request_spec_decode_metrics != "none"
and self.speculative_config is None
):
raise ValueError(
"--per-request-spec-decode-metrics requires speculative decoding "
"to be enabled (via --speculative-config)."
)
if (
self.model_config is not None
and self.model_config.multimodal_config is not None
+8
View File
@@ -656,6 +656,9 @@ class EngineArgs:
collect_detailed_traces: list[DetailedTraceModules] | None = (
ObservabilityConfig.collect_detailed_traces
)
per_request_spec_decode_metrics: Literal["none", "summary", "detailed"] = (
ObservabilityConfig.per_request_spec_decode_metrics
)
kv_cache_metrics: bool = ObservabilityConfig.kv_cache_metrics
kv_cache_metrics_sample: float = get_field(
ObservabilityConfig, "kv_cache_metrics_sample"
@@ -1476,6 +1479,10 @@ class EngineArgs:
"--collect-detailed-traces",
**observability_kwargs["collect_detailed_traces"],
)
observability_group.add_argument(
"--per-request-spec-decode-metrics",
**observability_kwargs["per_request_spec_decode_metrics"],
)
observability_group.add_argument(
"--kv-cache-metrics", **observability_kwargs["kv_cache_metrics"]
)
@@ -1934,6 +1941,7 @@ class EngineArgs:
show_hidden_metrics_for_version=self.show_hidden_metrics_for_version,
otlp_traces_endpoint=self.otlp_traces_endpoint,
collect_detailed_traces=self.collect_detailed_traces,
per_request_spec_decode_metrics=self.per_request_spec_decode_metrics,
kv_cache_metrics=self.kv_cache_metrics,
kv_cache_metrics_sample=self.kv_cache_metrics_sample,
cudagraph_metrics=self.cudagraph_metrics,
+26 -5
View File
@@ -5,7 +5,7 @@ import time
from collections.abc import Awaitable, Mapping
from dataclasses import dataclass, field
from http import HTTPStatus
from typing import ClassVar, Generic, TypeVar
from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar
from fastapi import Request
from pydantic import ConfigDict
@@ -18,7 +18,8 @@ from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.engine.protocol import (
ErrorResponse,
GenerationError,
PerRequestTimingMetrics,
PerRequestMetrics,
SpeculativeDecodingMetrics,
)
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
@@ -37,6 +38,9 @@ from vllm.tracing import (
)
from vllm.v1.metrics.stats import RequestStateStats
if TYPE_CHECKING:
from vllm.outputs import RequestOutput
logger = init_logger(__name__)
RequestT = TypeVar("RequestT", bound=AnyRequest)
@@ -48,7 +52,7 @@ PRIORITY_HEADER = "X-Vllm-Priority"
def build_per_request_timing_metrics(
metrics: RequestStateStats | None,
num_generation_tokens: int,
) -> PerRequestTimingMetrics:
) -> PerRequestMetrics:
"""Build per-request timing metrics from ``RequestStateStats``.
``generation_time_ms`` is the decode interval only (first output token to
@@ -60,7 +64,7 @@ def build_per_request_timing_metrics(
unavailable.
"""
if metrics is None:
return PerRequestTimingMetrics()
return PerRequestMetrics()
queued_ts = metrics.queued_ts
scheduled_ts = metrics.scheduled_ts
@@ -91,7 +95,7 @@ def build_per_request_timing_metrics(
if inference_time_ms > 0:
tokens_per_second = num_generation_tokens / inference_time_ms * 1000
return PerRequestTimingMetrics(
return PerRequestMetrics(
time_to_first_token_ms=time_to_first_token_ms,
generation_time_ms=generation_time_ms,
queue_time_ms=queue_time_ms,
@@ -100,6 +104,23 @@ def build_per_request_timing_metrics(
)
def build_spec_decoding_metrics(
final_res: "RequestOutput | None",
) -> SpeculativeDecodingMetrics | None:
"""Build per-request spec-decode acceptance metrics from the single output
sequence, or ``None`` when unavailable (metrics disabled, or no sequence
yet).
Only meaningful for single-sequence requests; callers suppress it for n>1.
"""
if final_res is None or not final_res.outputs:
return None
metrics = final_res.outputs[0].spec_decode_metrics
if metrics is None:
return None
return SpeculativeDecodingMetrics(**metrics.to_dict())
@dataclass(kw_only=True)
class ServeContext(Generic[RequestT]):
request: RequestT
@@ -29,7 +29,7 @@ from vllm.entrypoints.openai.engine.protocol import (
FunctionCall,
FunctionDefinition,
OpenAIBaseModel,
PerRequestTimingMetrics,
PerRequestMetrics,
StopParam,
StreamOptions,
ToolCall,
@@ -146,7 +146,7 @@ class ChatCompletionResponse(OpenAIBaseModel):
ec_transfer_params: dict[str, Any] | None = Field(
default=None, description="ECTransfer parameters."
)
metrics: PerRequestTimingMetrics | None = None
metrics: PerRequestMetrics | None = None
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
@@ -178,7 +178,7 @@ class ChatCompletionStreamResponse(OpenAIBaseModel):
# Rendered prompt text from chat templating (only set when
# ``return_prompt_text=True`` on the request); only sent on the first chunk.
prompt_text: str | None = None
metrics: PerRequestTimingMetrics | None = None
metrics: PerRequestMetrics | None = None
class ChatCompletionToolsParam(OpenAIBaseModel):
@@ -20,6 +20,7 @@ from vllm.entrypoints.generate.base.serving import (
GenerateBaseServing,
GenerationError,
build_per_request_timing_metrics,
build_spec_decoding_metrics,
clamp_prompt_logprobs,
format_token_id_placeholder,
)
@@ -40,7 +41,7 @@ from vllm.entrypoints.openai.engine.protocol import (
DeltaMessage,
ErrorResponse,
FunctionCall,
PerRequestTimingMetrics,
PerRequestMetrics,
PromptTokenUsageInfo,
RequestResponseMetadata,
ToolCall,
@@ -819,16 +820,21 @@ class OpenAIServingChat(GenerateBaseServing):
# only emitted when usage reporting is enabled (i.e.
# ``stream_options.include_usage=true`` or
# ``--enable-force-include-usage``).
stream_per_request_metrics: PerRequestTimingMetrics | None = None
if (
self.enable_per_request_metrics
# See note in chat_completion_full_generator: suppress for n>1.
and (request.n or 1) == 1
):
last_metrics = last_res.metrics if last_res is not None else None
stream_per_request_metrics = build_per_request_timing_metrics(
last_metrics, completion_tokens
)
stream_per_request_metrics: PerRequestMetrics | None = None
# See note in chat_completion_full_generator: suppress for n>1.
if (request.n or 1) == 1:
if self.enable_per_request_metrics:
last_metrics = (
last_res.metrics if last_res is not None else None
)
stream_per_request_metrics = build_per_request_timing_metrics(
last_metrics, completion_tokens
)
spec_stats = build_spec_decoding_metrics(last_res)
if spec_stats is not None:
if stream_per_request_metrics is None:
stream_per_request_metrics = PerRequestMetrics()
stream_per_request_metrics.speculative_decoding = spec_stats
final_usage_chunk = ChatCompletionStreamResponse(
id=request_id,
@@ -1118,17 +1124,20 @@ class OpenAIServingChat(GenerateBaseServing):
request_metadata.final_usage_info = usage
per_request_metrics: PerRequestTimingMetrics | None = None
if (
self.enable_per_request_metrics
# Timing metrics describe a single generation stream. For n>1 the
# returned stats belong to only one of the n sequences, so they
# cannot be accurately attributed to the request; suppress instead.
and (request.n or 1) == 1
):
per_request_metrics = build_per_request_timing_metrics(
final_res.metrics, num_generated_tokens
)
per_request_metrics: PerRequestMetrics | None = None
# Per-request metrics (timing + spec-decode acceptance) describe a single
# generation stream. For n>1 the stats belong to only one of the n
# sequences, so they cannot be attributed to the request; suppress.
if (request.n or 1) == 1:
if self.enable_per_request_metrics:
per_request_metrics = build_per_request_timing_metrics(
final_res.metrics, num_generated_tokens
)
spec_stats = build_spec_decoding_metrics(final_res)
if spec_stats is not None:
if per_request_metrics is None:
per_request_metrics = PerRequestMetrics()
per_request_metrics.speculative_decoding = spec_stats
# ``final_res.prompt`` is the rendered chat-templated prompt text
prompt_text = final_res.prompt if request.return_prompt_text else None
@@ -13,7 +13,7 @@ from vllm.config import ModelConfig
from vllm.entrypoints.openai.engine.protocol import (
AnyResponseFormat,
OpenAIBaseModel,
PerRequestTimingMetrics,
PerRequestMetrics,
StopParam,
StreamOptions,
UsageInfo,
@@ -660,7 +660,7 @@ class CompletionResponse(OpenAIBaseModel):
ec_transfer_params: dict[str, Any] | None = Field(
default=None, description="ECTransfer parameters."
)
metrics: PerRequestTimingMetrics | None = None
metrics: PerRequestMetrics | None = None
class CompletionResponseStreamChoice(OpenAIBaseModel):
@@ -692,4 +692,4 @@ class CompletionStreamResponse(OpenAIBaseModel):
# Set only on the final chunk of a stream to mirror non-streaming responses
# without the per-chunk serialization overhead.
system_fingerprint: str | None = None
metrics: PerRequestTimingMetrics | None = None
metrics: PerRequestMetrics | None = None
+36 -28
View File
@@ -14,6 +14,7 @@ from vllm.entrypoints.generate.base.serving import (
GenerateBaseServing,
GenerationError,
build_per_request_timing_metrics,
build_spec_decoding_metrics,
clamp_prompt_logprobs,
format_token_id_placeholder,
)
@@ -27,7 +28,7 @@ from vllm.entrypoints.openai.completion.protocol import (
)
from vllm.entrypoints.openai.engine.protocol import (
ErrorResponse,
PerRequestTimingMetrics,
PerRequestMetrics,
PromptTokenUsageInfo,
RequestResponseMetadata,
UsageInfo,
@@ -460,18 +461,22 @@ class OpenAIServingCompletion(GenerateBaseServing):
# only emitted when usage reporting is enabled (i.e.
# ``stream_options.include_usage=true`` or
# ``--enable-force-include-usage``).
stream_per_request_metrics: PerRequestTimingMetrics | None = None
if (
self.enable_per_request_metrics
# See note in request_output_to_completion_response: suppress
# when not attributable to one stream (multi-prompt or n>1).
and num_prompts == 1
and (request.n or 1) == 1
):
last_metrics = last_res.metrics if last_res is not None else None
stream_per_request_metrics = build_per_request_timing_metrics(
last_metrics, total_completion_tokens
)
stream_per_request_metrics: PerRequestMetrics | None = None
# See note in request_output_to_completion_response: suppress when
# not attributable to one stream (multi-prompt or n>1).
if num_prompts == 1 and (request.n or 1) == 1:
if self.enable_per_request_metrics:
last_metrics = (
last_res.metrics if last_res is not None else None
)
stream_per_request_metrics = build_per_request_timing_metrics(
last_metrics, total_completion_tokens
)
spec_stats = build_spec_decoding_metrics(last_res)
if spec_stats is not None:
if stream_per_request_metrics is None:
stream_per_request_metrics = PerRequestMetrics()
stream_per_request_metrics.speculative_decoding = spec_stats
final_usage_chunk = CompletionStreamResponse(
id=request_id,
@@ -612,21 +617,24 @@ class OpenAIServingCompletion(GenerateBaseServing):
request_metadata.final_usage_info = usage
per_request_metrics: PerRequestTimingMetrics | None = None
if (
self.enable_per_request_metrics
# Metrics describe a single generation stream, so suppress them when
# they cannot be attributed to one: multiple prompts (timestamps
# span prompts) or n>1 (stats belong to one of the n sequences).
and len(final_res_batch) == 1
and (request.n or 1) == 1
):
last_metrics = (
last_final_res.metrics if last_final_res is not None else None
)
per_request_metrics = build_per_request_timing_metrics(
last_metrics, num_generated_tokens
)
per_request_metrics: PerRequestMetrics | None = None
# Per-request metrics (timing + spec-decode acceptance) describe a single
# generation stream, so suppress them when they cannot be attributed to
# one: multiple prompts (timestamps span prompts) or n>1 (stats belong to
# one of the n sequences).
if len(final_res_batch) == 1 and (request.n or 1) == 1:
if self.enable_per_request_metrics:
last_metrics = (
last_final_res.metrics if last_final_res is not None else None
)
per_request_metrics = build_per_request_timing_metrics(
last_metrics, num_generated_tokens
)
spec_stats = build_spec_decoding_metrics(last_final_res)
if spec_stats is not None:
if per_request_metrics is None:
per_request_metrics = PerRequestMetrics()
per_request_metrics.speculative_decoding = spec_stats
if final_res_batch:
kv_transfer_params = final_res_batch[0].kv_transfer_params
+25 -1
View File
@@ -129,12 +129,36 @@ class UsageInfo(OpenAIBaseModel):
completion_tokens_details: CompletionTokenUsageInfo | None = None
class PerRequestTimingMetrics(OpenAIBaseModel):
class SpeculativeDecodingMetrics(OpenAIBaseModel):
"""Per-request speculative-decoding acceptance metrics.
Experimental, subject to change. Only populated for single-sequence requests
(`n == 1`); `null` for `n > 1`, mirroring the timing metrics.
"""
mean_acceptance_length: float
draft_acceptance_rate: float
# Dense histogram: index j holds the number of verify steps that accepted
# exactly j draft tokens (length num_spec_tokens + 1). Excludes the
# always-accepted bonus token.
acceptance_histogram: list[int]
num_spec_steps: int
num_accepted_draft_tokens: int
num_draft_tokens: int
num_spec_tokens: int
# Ordered per-verify-step arrays; populated only at the `detailed` level.
per_step_accepted: list[int] | None = None
per_step_drafted: list[int] | None = None
class PerRequestMetrics(OpenAIBaseModel):
time_to_first_token_ms: float | None = None
generation_time_ms: float | None = None
queue_time_ms: float | None = None
mean_itl_ms: float | None = None
tokens_per_second: float | None = None
# Experimental, subject to change.
speculative_decoding: SpeculativeDecodingMetrics | None = None
class RequestResponseMetadata(BaseModel):
+7 -1
View File
@@ -13,7 +13,7 @@ from typing_extensions import TypeVar
from vllm.logger import init_logger
from vllm.logprobs import PromptLogprobs, SampleLogprobs
from vllm.lora.request import LoRARequest
from vllm.v1.metrics.stats import RequestStateStats
from vllm.v1.metrics.stats import RequestSpecDecodeMetrics, RequestStateStats
logger = init_logger(__name__)
@@ -48,6 +48,11 @@ class CompletionOutput:
to stop, None if the completion finished for some other reason
including encountering the EOS token.
lora_request: The LoRA request that was used to generate the output.
spec_decode_metrics: Per-sequence speculative-decoding acceptance metrics,
populated on finish when speculative decoding ran and
``--per-request-spec-decode-metrics`` is enabled; None otherwise.
Surfaced in the response as ``metrics.speculative_decoding`` for
single-sequence (``n == 1``) requests.
"""
index: int
@@ -60,6 +65,7 @@ class CompletionOutput:
stop_reason: int | str | None = None
lora_request: LoRARequest | None = None
sampling_mask: SamplingMask | None = None
spec_decode_metrics: RequestSpecDecodeMetrics | None = None
def finished(self) -> bool:
return self.finish_reason is not None
+31 -1
View File
@@ -55,7 +55,11 @@ from vllm.v1.core.sched.utils import check_stop, remove_all
from vllm.v1.engine import EngineCoreEventType, EngineCoreOutput, EngineCoreOutputs
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.metrics.perf import ModelMetrics, PerfStats
from vllm.v1.metrics.stats import PrefixCacheStats, SchedulerStats
from vllm.v1.metrics.stats import (
PrefixCacheStats,
RequestSpecDecodeMetrics,
SchedulerStats,
)
from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput
from vllm.v1.request import Request, RequestStatus, StreamingUpdate
from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup
@@ -88,6 +92,9 @@ class Scheduler(SchedulerInterface):
self.parallel_config = vllm_config.parallel_config
self.log_stats = log_stats
self.observability_config = vllm_config.observability_config
self.spec_decode_metrics_level = (
self.observability_config.per_request_spec_decode_metrics
)
self.kv_metrics_collector: KVCacheMetricsCollector | None = None
if self.observability_config.kv_cache_metrics:
self.kv_metrics_collector = KVCacheMetricsCollector(
@@ -1857,6 +1864,20 @@ class Scheduler(SchedulerInterface):
num_invalid_spec_tokens=scheduler_output.num_invalid_spec_tokens,
request_id=req_id,
)
if request.spec_decode_metrics is not None:
# Exclude grammar-invalidated drafts from the proposed
# count, mirroring make_spec_decoding_stats; the accepted
# bucket (j) is unaffected.
adj_draft_tokens = num_draft_tokens
if scheduler_output.num_invalid_spec_tokens:
adj_draft_tokens -= (
scheduler_output.num_invalid_spec_tokens.get(req_id, 0)
)
request.spec_decode_metrics.observe(
num_draft_tokens=adj_draft_tokens,
num_accepted=num_accepted,
detailed=self.spec_decode_metrics_level == "detailed",
)
# Free encoder inputs only after the step has actually executed.
if request.has_encoder_inputs:
@@ -2020,6 +2041,11 @@ class Scheduler(SchedulerInterface):
stop_reason=request.stop_reason,
events=request.take_events(),
prefill_stats=prefill_stats,
spec_decode_metrics=(
request.spec_decode_metrics
if finish_reason is not None
else None
),
kv_transfer_params=kv_transfer_params,
ec_transfer_params=ec_transfer_params,
trace_headers=request.trace_headers,
@@ -2321,6 +2347,10 @@ class Scheduler(SchedulerInterface):
request.streaming_queue = deque()
self._enqueue_waiting_request(request)
self.requests[request.request_id] = request
if self.spec_decode_metrics_level != "none":
request.spec_decode_metrics = RequestSpecDecodeMetrics.new(
self.num_spec_tokens
)
if self.connector is not None:
self.connector.on_new_request(request)
if self.log_stats:
+9 -1
View File
@@ -16,7 +16,11 @@ from vllm.lora.request import LoRARequest
from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.pooling_params import PoolingParams
from vllm.sampling_params import SamplingParams
from vllm.v1.metrics.stats import PrefillStats, SchedulerStats
from vllm.v1.metrics.stats import (
PrefillStats,
RequestSpecDecodeMetrics,
SchedulerStats,
)
from vllm.v1.outputs import LogprobsLists, LogprobsTensors, SamplingMaskLists
from vllm.v1.serial_utils import UtilityResult
@@ -225,6 +229,10 @@ class EngineCoreOutput(
new_sampling_mask: SamplingMaskLists | None = None
# Per-request spec-decode acceptance; attached only on the final output.
# Appended last so `array_like` positional serialization stays compatible.
spec_decode_metrics: RequestSpecDecodeMetrics | None = None
@property
def finished(self) -> bool:
return self.finish_reason is not None
+8
View File
@@ -35,6 +35,7 @@ from vllm.v1.engine.parallel_sampling import ParentRequest
from vllm.v1.metrics.stats import (
IterationStats,
LoRARequestStates,
RequestSpecDecodeMetrics,
RequestStateStats,
SchedulerStats,
)
@@ -175,6 +176,9 @@ class RequestState:
self.queue = queue
self.num_cached_tokens = 0
self.num_cache_creation_tokens = 0
# Per-sequence spec-decode accumulator; arrives once (on finish) via
# EngineCoreOutput, then attached to this sequence's CompletionOutput.
self.spec_decode_metrics: RequestSpecDecodeMetrics | None = None
self.stats = RequestStateStats(arrival_time=arrival_time) if log_stats else None
@@ -429,6 +433,7 @@ class RequestState:
cumulative_logprob=self.logprobs_processor.cumulative_logprob,
finish_reason=str(finish_reason) if finished else None,
stop_reason=stop_reason if finished else None,
spec_decode_metrics=self.spec_decode_metrics if finished else None,
)
def _new_pooling_output(self, pooling_output: torch.Tensor) -> PoolingOutput:
@@ -658,6 +663,9 @@ class OutputProcessor:
)
req_state.is_prefilling = False
if engine_core_output.spec_decode_metrics is not None:
req_state.spec_decode_metrics = engine_core_output.spec_decode_metrics
if pooling_output is None:
assert req_state.detokenizer is not None
assert req_state.logprobs_processor is not None
+76
View File
@@ -297,6 +297,82 @@ class PrefillStats:
)
@dataclass
class RequestSpecDecodeMetrics:
"""Per-output-sequence speculative-decoding statistics accumulator.
Accumulates, over one sequence's verify steps, a histogram of accepted
draft-token counts (``j``, draft-only) and the total number of proposed
draft tokens. When ``detailed`` is requested it also records the ordered
per-step accepted/proposed sequences (``summary`` omits them). Tracked per
engine ``Request`` (one per sampled sequence, so ``n > 1`` yields one per
child), surfaced via ``EngineCoreOutput`` and the response
``metrics.speculative_decoding`` for single-sequence requests (see
``to_dict``).
Fields:
num_spec_tokens: Configured ``num_speculative_tokens`` (the max ``k``);
also the histogram's upper bound.
histogram: Dense counts indexed by accepted draft tokens ``j``
(length ``num_spec_tokens + 1``).
num_draft_tokens: Total proposed draft tokens, after the
grammar-invalidated (``num_invalid_spec_tokens``) adjustment.
per_step_accepted: Ordered accepted-draft count per verify step
(``detailed`` only; empty otherwise).
per_step_drafted: Ordered proposed-draft count per verify step
(``detailed`` only; empty otherwise).
"""
num_spec_tokens: int
histogram: list[int] = field(default_factory=list)
num_draft_tokens: int = 0
per_step_accepted: list[int] = field(default_factory=list)
per_step_drafted: list[int] = field(default_factory=list)
@classmethod
def new(cls, num_spec_tokens: int) -> "RequestSpecDecodeMetrics":
return cls(
num_spec_tokens=num_spec_tokens,
histogram=[0] * (num_spec_tokens + 1),
)
def observe(
self, num_draft_tokens: int, num_accepted: int, detailed: bool = False
) -> None:
self.histogram[num_accepted] += 1
self.num_draft_tokens += num_draft_tokens
if detailed:
self.per_step_accepted.append(num_accepted)
self.per_step_drafted.append(num_draft_tokens)
def to_dict(self) -> dict[str, Any]:
"""Payload matching ``SpeculativeDecodingMetrics`` for the response.
``acceptance_histogram`` is a dense list indexed by accepted draft count
``j`` (length ``num_spec_tokens + 1``). ``mean_acceptance_length``
includes the bonus token (``j + 1``); ``draft_acceptance_rate`` is
draft-only, full precision. Per-step arrays are included only when
populated (``detailed`` level).
"""
num_spec_steps = sum(self.histogram)
num_accepted = sum(j * count for j, count in enumerate(self.histogram))
mean_al = 1.0 + num_accepted / num_spec_steps if num_spec_steps else 1.0
rate = num_accepted / self.num_draft_tokens if self.num_draft_tokens else 0.0
result: dict[str, Any] = {
"mean_acceptance_length": mean_al,
"draft_acceptance_rate": rate,
"acceptance_histogram": list(self.histogram),
"num_spec_steps": num_spec_steps,
"num_accepted_draft_tokens": num_accepted,
"num_draft_tokens": self.num_draft_tokens,
"num_spec_tokens": self.num_spec_tokens,
}
if self.per_step_accepted:
result["per_step_accepted"] = self.per_step_accepted
result["per_step_drafted"] = self.per_step_drafted
return result
@dataclass
class PromptTokenStats:
"""Breakdown of prompt tokens by source.
+6 -1
View File
@@ -20,7 +20,7 @@ from vllm.v1.engine import (
EngineCoreRequest,
FinishReason,
)
from vllm.v1.metrics.stats import PrefillStats
from vllm.v1.metrics.stats import PrefillStats, RequestSpecDecodeMetrics
from vllm.v1.structured_output.request import StructuredOutputRequest
from vllm.v1.utils import ConstantList
@@ -211,6 +211,11 @@ class Request:
self.prefill_stats: PrefillStats | None = PrefillStats()
# Per-request speculative-decoding acceptance accumulator. Populated by
# the scheduler when --per-request-spec-decode-metrics is set (eagerly on
# add_request, then observed each verify step); stays None otherwise.
self.spec_decode_metrics: RequestSpecDecodeMetrics | None = None
self.block_hashes: list[BlockHash] = []
# Store the block hasher without binding self to avoid creating a
# reference cycle (Request -> partial -> Request) that prevents