[Frontend][Core][Spec Decode] Per-request acceptance stats in OpenAI API responses (#48915)
Signed-off-by: Matthew Kotila <7692737+matthewkotila@users.noreply.github.com> Co-authored-by: Reed Meyerson <31574681+reed-meyerson@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Reed Meyerson
Claude Opus 4.8
parent
2f41c894e3
commit
7cfb97e337
@@ -45,6 +45,18 @@ class ObservabilityConfig:
|
||||
Note that collecting detailed timing information for each request can be
|
||||
expensive."""
|
||||
|
||||
per_request_spec_decode_metrics: Literal["none", "summary", "detailed"] = "none"
|
||||
"""Include per-request speculative-decoding acceptance metrics in the
|
||||
response under `metrics.speculative_decoding`. `none` disables; `summary` adds mean
|
||||
acceptance length, draft acceptance rate, and the step-by-draft-length
|
||||
histogram; `detailed` additionally records the ordered per-step
|
||||
accepted/proposed arrays (one entry per verify step). Only reported for
|
||||
single-sequence requests (`n == 1`), mirroring the timing metrics. No effect
|
||||
unless speculative decoding is enabled. Independent of `--disable-log-stats`.
|
||||
This is the per-request response-body counterpart of the aggregate
|
||||
`vllm:spec_decode_*` Prometheus metrics. The response field is experimental
|
||||
and its shape may change in a future release."""
|
||||
|
||||
kv_cache_metrics: bool = False
|
||||
"""Enable KV cache residency metrics (lifetime, idle time, reuse gaps).
|
||||
Uses sampling to minimize overhead.
|
||||
|
||||
@@ -1331,6 +1331,15 @@ class VllmConfig:
|
||||
)
|
||||
self.model_config.disable_cascade_attn = True
|
||||
|
||||
if (
|
||||
self.observability_config.per_request_spec_decode_metrics != "none"
|
||||
and self.speculative_config is None
|
||||
):
|
||||
raise ValueError(
|
||||
"--per-request-spec-decode-metrics requires speculative decoding "
|
||||
"to be enabled (via --speculative-config)."
|
||||
)
|
||||
|
||||
if (
|
||||
self.model_config is not None
|
||||
and self.model_config.multimodal_config is not None
|
||||
|
||||
@@ -656,6 +656,9 @@ class EngineArgs:
|
||||
collect_detailed_traces: list[DetailedTraceModules] | None = (
|
||||
ObservabilityConfig.collect_detailed_traces
|
||||
)
|
||||
per_request_spec_decode_metrics: Literal["none", "summary", "detailed"] = (
|
||||
ObservabilityConfig.per_request_spec_decode_metrics
|
||||
)
|
||||
kv_cache_metrics: bool = ObservabilityConfig.kv_cache_metrics
|
||||
kv_cache_metrics_sample: float = get_field(
|
||||
ObservabilityConfig, "kv_cache_metrics_sample"
|
||||
@@ -1476,6 +1479,10 @@ class EngineArgs:
|
||||
"--collect-detailed-traces",
|
||||
**observability_kwargs["collect_detailed_traces"],
|
||||
)
|
||||
observability_group.add_argument(
|
||||
"--per-request-spec-decode-metrics",
|
||||
**observability_kwargs["per_request_spec_decode_metrics"],
|
||||
)
|
||||
observability_group.add_argument(
|
||||
"--kv-cache-metrics", **observability_kwargs["kv_cache_metrics"]
|
||||
)
|
||||
@@ -1934,6 +1941,7 @@ class EngineArgs:
|
||||
show_hidden_metrics_for_version=self.show_hidden_metrics_for_version,
|
||||
otlp_traces_endpoint=self.otlp_traces_endpoint,
|
||||
collect_detailed_traces=self.collect_detailed_traces,
|
||||
per_request_spec_decode_metrics=self.per_request_spec_decode_metrics,
|
||||
kv_cache_metrics=self.kv_cache_metrics,
|
||||
kv_cache_metrics_sample=self.kv_cache_metrics_sample,
|
||||
cudagraph_metrics=self.cudagraph_metrics,
|
||||
|
||||
@@ -5,7 +5,7 @@ import time
|
||||
from collections.abc import Awaitable, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from http import HTTPStatus
|
||||
from typing import ClassVar, Generic, TypeVar
|
||||
from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar
|
||||
|
||||
from fastapi import Request
|
||||
from pydantic import ConfigDict
|
||||
@@ -18,7 +18,8 @@ from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
GenerationError,
|
||||
PerRequestTimingMetrics,
|
||||
PerRequestMetrics,
|
||||
SpeculativeDecodingMetrics,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
@@ -37,6 +38,9 @@ from vllm.tracing import (
|
||||
)
|
||||
from vllm.v1.metrics.stats import RequestStateStats
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.outputs import RequestOutput
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
RequestT = TypeVar("RequestT", bound=AnyRequest)
|
||||
@@ -48,7 +52,7 @@ PRIORITY_HEADER = "X-Vllm-Priority"
|
||||
def build_per_request_timing_metrics(
|
||||
metrics: RequestStateStats | None,
|
||||
num_generation_tokens: int,
|
||||
) -> PerRequestTimingMetrics:
|
||||
) -> PerRequestMetrics:
|
||||
"""Build per-request timing metrics from ``RequestStateStats``.
|
||||
|
||||
``generation_time_ms`` is the decode interval only (first output token to
|
||||
@@ -60,7 +64,7 @@ def build_per_request_timing_metrics(
|
||||
unavailable.
|
||||
"""
|
||||
if metrics is None:
|
||||
return PerRequestTimingMetrics()
|
||||
return PerRequestMetrics()
|
||||
|
||||
queued_ts = metrics.queued_ts
|
||||
scheduled_ts = metrics.scheduled_ts
|
||||
@@ -91,7 +95,7 @@ def build_per_request_timing_metrics(
|
||||
if inference_time_ms > 0:
|
||||
tokens_per_second = num_generation_tokens / inference_time_ms * 1000
|
||||
|
||||
return PerRequestTimingMetrics(
|
||||
return PerRequestMetrics(
|
||||
time_to_first_token_ms=time_to_first_token_ms,
|
||||
generation_time_ms=generation_time_ms,
|
||||
queue_time_ms=queue_time_ms,
|
||||
@@ -100,6 +104,23 @@ def build_per_request_timing_metrics(
|
||||
)
|
||||
|
||||
|
||||
def build_spec_decoding_metrics(
|
||||
final_res: "RequestOutput | None",
|
||||
) -> SpeculativeDecodingMetrics | None:
|
||||
"""Build per-request spec-decode acceptance metrics from the single output
|
||||
sequence, or ``None`` when unavailable (metrics disabled, or no sequence
|
||||
yet).
|
||||
|
||||
Only meaningful for single-sequence requests; callers suppress it for n>1.
|
||||
"""
|
||||
if final_res is None or not final_res.outputs:
|
||||
return None
|
||||
metrics = final_res.outputs[0].spec_decode_metrics
|
||||
if metrics is None:
|
||||
return None
|
||||
return SpeculativeDecodingMetrics(**metrics.to_dict())
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class ServeContext(Generic[RequestT]):
|
||||
request: RequestT
|
||||
|
||||
@@ -29,7 +29,7 @@ from vllm.entrypoints.openai.engine.protocol import (
|
||||
FunctionCall,
|
||||
FunctionDefinition,
|
||||
OpenAIBaseModel,
|
||||
PerRequestTimingMetrics,
|
||||
PerRequestMetrics,
|
||||
StopParam,
|
||||
StreamOptions,
|
||||
ToolCall,
|
||||
@@ -146,7 +146,7 @@ class ChatCompletionResponse(OpenAIBaseModel):
|
||||
ec_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None, description="ECTransfer parameters."
|
||||
)
|
||||
metrics: PerRequestTimingMetrics | None = None
|
||||
metrics: PerRequestMetrics | None = None
|
||||
|
||||
|
||||
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
|
||||
@@ -178,7 +178,7 @@ class ChatCompletionStreamResponse(OpenAIBaseModel):
|
||||
# Rendered prompt text from chat templating (only set when
|
||||
# ``return_prompt_text=True`` on the request); only sent on the first chunk.
|
||||
prompt_text: str | None = None
|
||||
metrics: PerRequestTimingMetrics | None = None
|
||||
metrics: PerRequestMetrics | None = None
|
||||
|
||||
|
||||
class ChatCompletionToolsParam(OpenAIBaseModel):
|
||||
|
||||
@@ -20,6 +20,7 @@ from vllm.entrypoints.generate.base.serving import (
|
||||
GenerateBaseServing,
|
||||
GenerationError,
|
||||
build_per_request_timing_metrics,
|
||||
build_spec_decoding_metrics,
|
||||
clamp_prompt_logprobs,
|
||||
format_token_id_placeholder,
|
||||
)
|
||||
@@ -40,7 +41,7 @@ from vllm.entrypoints.openai.engine.protocol import (
|
||||
DeltaMessage,
|
||||
ErrorResponse,
|
||||
FunctionCall,
|
||||
PerRequestTimingMetrics,
|
||||
PerRequestMetrics,
|
||||
PromptTokenUsageInfo,
|
||||
RequestResponseMetadata,
|
||||
ToolCall,
|
||||
@@ -819,16 +820,21 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
# only emitted when usage reporting is enabled (i.e.
|
||||
# ``stream_options.include_usage=true`` or
|
||||
# ``--enable-force-include-usage``).
|
||||
stream_per_request_metrics: PerRequestTimingMetrics | None = None
|
||||
if (
|
||||
self.enable_per_request_metrics
|
||||
# See note in chat_completion_full_generator: suppress for n>1.
|
||||
and (request.n or 1) == 1
|
||||
):
|
||||
last_metrics = last_res.metrics if last_res is not None else None
|
||||
stream_per_request_metrics = build_per_request_timing_metrics(
|
||||
last_metrics, completion_tokens
|
||||
)
|
||||
stream_per_request_metrics: PerRequestMetrics | None = None
|
||||
# See note in chat_completion_full_generator: suppress for n>1.
|
||||
if (request.n or 1) == 1:
|
||||
if self.enable_per_request_metrics:
|
||||
last_metrics = (
|
||||
last_res.metrics if last_res is not None else None
|
||||
)
|
||||
stream_per_request_metrics = build_per_request_timing_metrics(
|
||||
last_metrics, completion_tokens
|
||||
)
|
||||
spec_stats = build_spec_decoding_metrics(last_res)
|
||||
if spec_stats is not None:
|
||||
if stream_per_request_metrics is None:
|
||||
stream_per_request_metrics = PerRequestMetrics()
|
||||
stream_per_request_metrics.speculative_decoding = spec_stats
|
||||
|
||||
final_usage_chunk = ChatCompletionStreamResponse(
|
||||
id=request_id,
|
||||
@@ -1118,17 +1124,20 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
|
||||
request_metadata.final_usage_info = usage
|
||||
|
||||
per_request_metrics: PerRequestTimingMetrics | None = None
|
||||
if (
|
||||
self.enable_per_request_metrics
|
||||
# Timing metrics describe a single generation stream. For n>1 the
|
||||
# returned stats belong to only one of the n sequences, so they
|
||||
# cannot be accurately attributed to the request; suppress instead.
|
||||
and (request.n or 1) == 1
|
||||
):
|
||||
per_request_metrics = build_per_request_timing_metrics(
|
||||
final_res.metrics, num_generated_tokens
|
||||
)
|
||||
per_request_metrics: PerRequestMetrics | None = None
|
||||
# Per-request metrics (timing + spec-decode acceptance) describe a single
|
||||
# generation stream. For n>1 the stats belong to only one of the n
|
||||
# sequences, so they cannot be attributed to the request; suppress.
|
||||
if (request.n or 1) == 1:
|
||||
if self.enable_per_request_metrics:
|
||||
per_request_metrics = build_per_request_timing_metrics(
|
||||
final_res.metrics, num_generated_tokens
|
||||
)
|
||||
spec_stats = build_spec_decoding_metrics(final_res)
|
||||
if spec_stats is not None:
|
||||
if per_request_metrics is None:
|
||||
per_request_metrics = PerRequestMetrics()
|
||||
per_request_metrics.speculative_decoding = spec_stats
|
||||
|
||||
# ``final_res.prompt`` is the rendered chat-templated prompt text
|
||||
prompt_text = final_res.prompt if request.return_prompt_text else None
|
||||
|
||||
@@ -13,7 +13,7 @@ from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
AnyResponseFormat,
|
||||
OpenAIBaseModel,
|
||||
PerRequestTimingMetrics,
|
||||
PerRequestMetrics,
|
||||
StopParam,
|
||||
StreamOptions,
|
||||
UsageInfo,
|
||||
@@ -660,7 +660,7 @@ class CompletionResponse(OpenAIBaseModel):
|
||||
ec_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None, description="ECTransfer parameters."
|
||||
)
|
||||
metrics: PerRequestTimingMetrics | None = None
|
||||
metrics: PerRequestMetrics | None = None
|
||||
|
||||
|
||||
class CompletionResponseStreamChoice(OpenAIBaseModel):
|
||||
@@ -692,4 +692,4 @@ class CompletionStreamResponse(OpenAIBaseModel):
|
||||
# Set only on the final chunk of a stream to mirror non-streaming responses
|
||||
# without the per-chunk serialization overhead.
|
||||
system_fingerprint: str | None = None
|
||||
metrics: PerRequestTimingMetrics | None = None
|
||||
metrics: PerRequestMetrics | None = None
|
||||
|
||||
@@ -14,6 +14,7 @@ from vllm.entrypoints.generate.base.serving import (
|
||||
GenerateBaseServing,
|
||||
GenerationError,
|
||||
build_per_request_timing_metrics,
|
||||
build_spec_decoding_metrics,
|
||||
clamp_prompt_logprobs,
|
||||
format_token_id_placeholder,
|
||||
)
|
||||
@@ -27,7 +28,7 @@ from vllm.entrypoints.openai.completion.protocol import (
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
PerRequestTimingMetrics,
|
||||
PerRequestMetrics,
|
||||
PromptTokenUsageInfo,
|
||||
RequestResponseMetadata,
|
||||
UsageInfo,
|
||||
@@ -460,18 +461,22 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
# only emitted when usage reporting is enabled (i.e.
|
||||
# ``stream_options.include_usage=true`` or
|
||||
# ``--enable-force-include-usage``).
|
||||
stream_per_request_metrics: PerRequestTimingMetrics | None = None
|
||||
if (
|
||||
self.enable_per_request_metrics
|
||||
# See note in request_output_to_completion_response: suppress
|
||||
# when not attributable to one stream (multi-prompt or n>1).
|
||||
and num_prompts == 1
|
||||
and (request.n or 1) == 1
|
||||
):
|
||||
last_metrics = last_res.metrics if last_res is not None else None
|
||||
stream_per_request_metrics = build_per_request_timing_metrics(
|
||||
last_metrics, total_completion_tokens
|
||||
)
|
||||
stream_per_request_metrics: PerRequestMetrics | None = None
|
||||
# See note in request_output_to_completion_response: suppress when
|
||||
# not attributable to one stream (multi-prompt or n>1).
|
||||
if num_prompts == 1 and (request.n or 1) == 1:
|
||||
if self.enable_per_request_metrics:
|
||||
last_metrics = (
|
||||
last_res.metrics if last_res is not None else None
|
||||
)
|
||||
stream_per_request_metrics = build_per_request_timing_metrics(
|
||||
last_metrics, total_completion_tokens
|
||||
)
|
||||
spec_stats = build_spec_decoding_metrics(last_res)
|
||||
if spec_stats is not None:
|
||||
if stream_per_request_metrics is None:
|
||||
stream_per_request_metrics = PerRequestMetrics()
|
||||
stream_per_request_metrics.speculative_decoding = spec_stats
|
||||
|
||||
final_usage_chunk = CompletionStreamResponse(
|
||||
id=request_id,
|
||||
@@ -612,21 +617,24 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
|
||||
request_metadata.final_usage_info = usage
|
||||
|
||||
per_request_metrics: PerRequestTimingMetrics | None = None
|
||||
if (
|
||||
self.enable_per_request_metrics
|
||||
# Metrics describe a single generation stream, so suppress them when
|
||||
# they cannot be attributed to one: multiple prompts (timestamps
|
||||
# span prompts) or n>1 (stats belong to one of the n sequences).
|
||||
and len(final_res_batch) == 1
|
||||
and (request.n or 1) == 1
|
||||
):
|
||||
last_metrics = (
|
||||
last_final_res.metrics if last_final_res is not None else None
|
||||
)
|
||||
per_request_metrics = build_per_request_timing_metrics(
|
||||
last_metrics, num_generated_tokens
|
||||
)
|
||||
per_request_metrics: PerRequestMetrics | None = None
|
||||
# Per-request metrics (timing + spec-decode acceptance) describe a single
|
||||
# generation stream, so suppress them when they cannot be attributed to
|
||||
# one: multiple prompts (timestamps span prompts) or n>1 (stats belong to
|
||||
# one of the n sequences).
|
||||
if len(final_res_batch) == 1 and (request.n or 1) == 1:
|
||||
if self.enable_per_request_metrics:
|
||||
last_metrics = (
|
||||
last_final_res.metrics if last_final_res is not None else None
|
||||
)
|
||||
per_request_metrics = build_per_request_timing_metrics(
|
||||
last_metrics, num_generated_tokens
|
||||
)
|
||||
spec_stats = build_spec_decoding_metrics(last_final_res)
|
||||
if spec_stats is not None:
|
||||
if per_request_metrics is None:
|
||||
per_request_metrics = PerRequestMetrics()
|
||||
per_request_metrics.speculative_decoding = spec_stats
|
||||
|
||||
if final_res_batch:
|
||||
kv_transfer_params = final_res_batch[0].kv_transfer_params
|
||||
|
||||
@@ -129,12 +129,36 @@ class UsageInfo(OpenAIBaseModel):
|
||||
completion_tokens_details: CompletionTokenUsageInfo | None = None
|
||||
|
||||
|
||||
class PerRequestTimingMetrics(OpenAIBaseModel):
|
||||
class SpeculativeDecodingMetrics(OpenAIBaseModel):
|
||||
"""Per-request speculative-decoding acceptance metrics.
|
||||
|
||||
Experimental, subject to change. Only populated for single-sequence requests
|
||||
(`n == 1`); `null` for `n > 1`, mirroring the timing metrics.
|
||||
"""
|
||||
|
||||
mean_acceptance_length: float
|
||||
draft_acceptance_rate: float
|
||||
# Dense histogram: index j holds the number of verify steps that accepted
|
||||
# exactly j draft tokens (length num_spec_tokens + 1). Excludes the
|
||||
# always-accepted bonus token.
|
||||
acceptance_histogram: list[int]
|
||||
num_spec_steps: int
|
||||
num_accepted_draft_tokens: int
|
||||
num_draft_tokens: int
|
||||
num_spec_tokens: int
|
||||
# Ordered per-verify-step arrays; populated only at the `detailed` level.
|
||||
per_step_accepted: list[int] | None = None
|
||||
per_step_drafted: list[int] | None = None
|
||||
|
||||
|
||||
class PerRequestMetrics(OpenAIBaseModel):
|
||||
time_to_first_token_ms: float | None = None
|
||||
generation_time_ms: float | None = None
|
||||
queue_time_ms: float | None = None
|
||||
mean_itl_ms: float | None = None
|
||||
tokens_per_second: float | None = None
|
||||
# Experimental, subject to change.
|
||||
speculative_decoding: SpeculativeDecodingMetrics | None = None
|
||||
|
||||
|
||||
class RequestResponseMetadata(BaseModel):
|
||||
|
||||
+7
-1
@@ -13,7 +13,7 @@ from typing_extensions import TypeVar
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import PromptLogprobs, SampleLogprobs
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.v1.metrics.stats import RequestStateStats
|
||||
from vllm.v1.metrics.stats import RequestSpecDecodeMetrics, RequestStateStats
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -48,6 +48,11 @@ class CompletionOutput:
|
||||
to stop, None if the completion finished for some other reason
|
||||
including encountering the EOS token.
|
||||
lora_request: The LoRA request that was used to generate the output.
|
||||
spec_decode_metrics: Per-sequence speculative-decoding acceptance metrics,
|
||||
populated on finish when speculative decoding ran and
|
||||
``--per-request-spec-decode-metrics`` is enabled; None otherwise.
|
||||
Surfaced in the response as ``metrics.speculative_decoding`` for
|
||||
single-sequence (``n == 1``) requests.
|
||||
"""
|
||||
|
||||
index: int
|
||||
@@ -60,6 +65,7 @@ class CompletionOutput:
|
||||
stop_reason: int | str | None = None
|
||||
lora_request: LoRARequest | None = None
|
||||
sampling_mask: SamplingMask | None = None
|
||||
spec_decode_metrics: RequestSpecDecodeMetrics | None = None
|
||||
|
||||
def finished(self) -> bool:
|
||||
return self.finish_reason is not None
|
||||
|
||||
@@ -55,7 +55,11 @@ from vllm.v1.core.sched.utils import check_stop, remove_all
|
||||
from vllm.v1.engine import EngineCoreEventType, EngineCoreOutput, EngineCoreOutputs
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.metrics.perf import ModelMetrics, PerfStats
|
||||
from vllm.v1.metrics.stats import PrefixCacheStats, SchedulerStats
|
||||
from vllm.v1.metrics.stats import (
|
||||
PrefixCacheStats,
|
||||
RequestSpecDecodeMetrics,
|
||||
SchedulerStats,
|
||||
)
|
||||
from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput
|
||||
from vllm.v1.request import Request, RequestStatus, StreamingUpdate
|
||||
from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup
|
||||
@@ -88,6 +92,9 @@ class Scheduler(SchedulerInterface):
|
||||
self.parallel_config = vllm_config.parallel_config
|
||||
self.log_stats = log_stats
|
||||
self.observability_config = vllm_config.observability_config
|
||||
self.spec_decode_metrics_level = (
|
||||
self.observability_config.per_request_spec_decode_metrics
|
||||
)
|
||||
self.kv_metrics_collector: KVCacheMetricsCollector | None = None
|
||||
if self.observability_config.kv_cache_metrics:
|
||||
self.kv_metrics_collector = KVCacheMetricsCollector(
|
||||
@@ -1857,6 +1864,20 @@ class Scheduler(SchedulerInterface):
|
||||
num_invalid_spec_tokens=scheduler_output.num_invalid_spec_tokens,
|
||||
request_id=req_id,
|
||||
)
|
||||
if request.spec_decode_metrics is not None:
|
||||
# Exclude grammar-invalidated drafts from the proposed
|
||||
# count, mirroring make_spec_decoding_stats; the accepted
|
||||
# bucket (j) is unaffected.
|
||||
adj_draft_tokens = num_draft_tokens
|
||||
if scheduler_output.num_invalid_spec_tokens:
|
||||
adj_draft_tokens -= (
|
||||
scheduler_output.num_invalid_spec_tokens.get(req_id, 0)
|
||||
)
|
||||
request.spec_decode_metrics.observe(
|
||||
num_draft_tokens=adj_draft_tokens,
|
||||
num_accepted=num_accepted,
|
||||
detailed=self.spec_decode_metrics_level == "detailed",
|
||||
)
|
||||
|
||||
# Free encoder inputs only after the step has actually executed.
|
||||
if request.has_encoder_inputs:
|
||||
@@ -2020,6 +2041,11 @@ class Scheduler(SchedulerInterface):
|
||||
stop_reason=request.stop_reason,
|
||||
events=request.take_events(),
|
||||
prefill_stats=prefill_stats,
|
||||
spec_decode_metrics=(
|
||||
request.spec_decode_metrics
|
||||
if finish_reason is not None
|
||||
else None
|
||||
),
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
ec_transfer_params=ec_transfer_params,
|
||||
trace_headers=request.trace_headers,
|
||||
@@ -2321,6 +2347,10 @@ class Scheduler(SchedulerInterface):
|
||||
request.streaming_queue = deque()
|
||||
self._enqueue_waiting_request(request)
|
||||
self.requests[request.request_id] = request
|
||||
if self.spec_decode_metrics_level != "none":
|
||||
request.spec_decode_metrics = RequestSpecDecodeMetrics.new(
|
||||
self.num_spec_tokens
|
||||
)
|
||||
if self.connector is not None:
|
||||
self.connector.on_new_request(request)
|
||||
if self.log_stats:
|
||||
|
||||
@@ -16,7 +16,11 @@ from vllm.lora.request import LoRARequest
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.metrics.stats import PrefillStats, SchedulerStats
|
||||
from vllm.v1.metrics.stats import (
|
||||
PrefillStats,
|
||||
RequestSpecDecodeMetrics,
|
||||
SchedulerStats,
|
||||
)
|
||||
from vllm.v1.outputs import LogprobsLists, LogprobsTensors, SamplingMaskLists
|
||||
from vllm.v1.serial_utils import UtilityResult
|
||||
|
||||
@@ -225,6 +229,10 @@ class EngineCoreOutput(
|
||||
|
||||
new_sampling_mask: SamplingMaskLists | None = None
|
||||
|
||||
# Per-request spec-decode acceptance; attached only on the final output.
|
||||
# Appended last so `array_like` positional serialization stays compatible.
|
||||
spec_decode_metrics: RequestSpecDecodeMetrics | None = None
|
||||
|
||||
@property
|
||||
def finished(self) -> bool:
|
||||
return self.finish_reason is not None
|
||||
|
||||
@@ -35,6 +35,7 @@ from vllm.v1.engine.parallel_sampling import ParentRequest
|
||||
from vllm.v1.metrics.stats import (
|
||||
IterationStats,
|
||||
LoRARequestStates,
|
||||
RequestSpecDecodeMetrics,
|
||||
RequestStateStats,
|
||||
SchedulerStats,
|
||||
)
|
||||
@@ -175,6 +176,9 @@ class RequestState:
|
||||
self.queue = queue
|
||||
self.num_cached_tokens = 0
|
||||
self.num_cache_creation_tokens = 0
|
||||
# Per-sequence spec-decode accumulator; arrives once (on finish) via
|
||||
# EngineCoreOutput, then attached to this sequence's CompletionOutput.
|
||||
self.spec_decode_metrics: RequestSpecDecodeMetrics | None = None
|
||||
|
||||
self.stats = RequestStateStats(arrival_time=arrival_time) if log_stats else None
|
||||
|
||||
@@ -429,6 +433,7 @@ class RequestState:
|
||||
cumulative_logprob=self.logprobs_processor.cumulative_logprob,
|
||||
finish_reason=str(finish_reason) if finished else None,
|
||||
stop_reason=stop_reason if finished else None,
|
||||
spec_decode_metrics=self.spec_decode_metrics if finished else None,
|
||||
)
|
||||
|
||||
def _new_pooling_output(self, pooling_output: torch.Tensor) -> PoolingOutput:
|
||||
@@ -658,6 +663,9 @@ class OutputProcessor:
|
||||
)
|
||||
req_state.is_prefilling = False
|
||||
|
||||
if engine_core_output.spec_decode_metrics is not None:
|
||||
req_state.spec_decode_metrics = engine_core_output.spec_decode_metrics
|
||||
|
||||
if pooling_output is None:
|
||||
assert req_state.detokenizer is not None
|
||||
assert req_state.logprobs_processor is not None
|
||||
|
||||
@@ -297,6 +297,82 @@ class PrefillStats:
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RequestSpecDecodeMetrics:
|
||||
"""Per-output-sequence speculative-decoding statistics accumulator.
|
||||
|
||||
Accumulates, over one sequence's verify steps, a histogram of accepted
|
||||
draft-token counts (``j``, draft-only) and the total number of proposed
|
||||
draft tokens. When ``detailed`` is requested it also records the ordered
|
||||
per-step accepted/proposed sequences (``summary`` omits them). Tracked per
|
||||
engine ``Request`` (one per sampled sequence, so ``n > 1`` yields one per
|
||||
child), surfaced via ``EngineCoreOutput`` and the response
|
||||
``metrics.speculative_decoding`` for single-sequence requests (see
|
||||
``to_dict``).
|
||||
|
||||
Fields:
|
||||
num_spec_tokens: Configured ``num_speculative_tokens`` (the max ``k``);
|
||||
also the histogram's upper bound.
|
||||
histogram: Dense counts indexed by accepted draft tokens ``j``
|
||||
(length ``num_spec_tokens + 1``).
|
||||
num_draft_tokens: Total proposed draft tokens, after the
|
||||
grammar-invalidated (``num_invalid_spec_tokens``) adjustment.
|
||||
per_step_accepted: Ordered accepted-draft count per verify step
|
||||
(``detailed`` only; empty otherwise).
|
||||
per_step_drafted: Ordered proposed-draft count per verify step
|
||||
(``detailed`` only; empty otherwise).
|
||||
"""
|
||||
|
||||
num_spec_tokens: int
|
||||
histogram: list[int] = field(default_factory=list)
|
||||
num_draft_tokens: int = 0
|
||||
per_step_accepted: list[int] = field(default_factory=list)
|
||||
per_step_drafted: list[int] = field(default_factory=list)
|
||||
|
||||
@classmethod
|
||||
def new(cls, num_spec_tokens: int) -> "RequestSpecDecodeMetrics":
|
||||
return cls(
|
||||
num_spec_tokens=num_spec_tokens,
|
||||
histogram=[0] * (num_spec_tokens + 1),
|
||||
)
|
||||
|
||||
def observe(
|
||||
self, num_draft_tokens: int, num_accepted: int, detailed: bool = False
|
||||
) -> None:
|
||||
self.histogram[num_accepted] += 1
|
||||
self.num_draft_tokens += num_draft_tokens
|
||||
if detailed:
|
||||
self.per_step_accepted.append(num_accepted)
|
||||
self.per_step_drafted.append(num_draft_tokens)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""Payload matching ``SpeculativeDecodingMetrics`` for the response.
|
||||
|
||||
``acceptance_histogram`` is a dense list indexed by accepted draft count
|
||||
``j`` (length ``num_spec_tokens + 1``). ``mean_acceptance_length``
|
||||
includes the bonus token (``j + 1``); ``draft_acceptance_rate`` is
|
||||
draft-only, full precision. Per-step arrays are included only when
|
||||
populated (``detailed`` level).
|
||||
"""
|
||||
num_spec_steps = sum(self.histogram)
|
||||
num_accepted = sum(j * count for j, count in enumerate(self.histogram))
|
||||
mean_al = 1.0 + num_accepted / num_spec_steps if num_spec_steps else 1.0
|
||||
rate = num_accepted / self.num_draft_tokens if self.num_draft_tokens else 0.0
|
||||
result: dict[str, Any] = {
|
||||
"mean_acceptance_length": mean_al,
|
||||
"draft_acceptance_rate": rate,
|
||||
"acceptance_histogram": list(self.histogram),
|
||||
"num_spec_steps": num_spec_steps,
|
||||
"num_accepted_draft_tokens": num_accepted,
|
||||
"num_draft_tokens": self.num_draft_tokens,
|
||||
"num_spec_tokens": self.num_spec_tokens,
|
||||
}
|
||||
if self.per_step_accepted:
|
||||
result["per_step_accepted"] = self.per_step_accepted
|
||||
result["per_step_drafted"] = self.per_step_drafted
|
||||
return result
|
||||
|
||||
|
||||
@dataclass
|
||||
class PromptTokenStats:
|
||||
"""Breakdown of prompt tokens by source.
|
||||
|
||||
+6
-1
@@ -20,7 +20,7 @@ from vllm.v1.engine import (
|
||||
EngineCoreRequest,
|
||||
FinishReason,
|
||||
)
|
||||
from vllm.v1.metrics.stats import PrefillStats
|
||||
from vllm.v1.metrics.stats import PrefillStats, RequestSpecDecodeMetrics
|
||||
from vllm.v1.structured_output.request import StructuredOutputRequest
|
||||
from vllm.v1.utils import ConstantList
|
||||
|
||||
@@ -211,6 +211,11 @@ class Request:
|
||||
|
||||
self.prefill_stats: PrefillStats | None = PrefillStats()
|
||||
|
||||
# Per-request speculative-decoding acceptance accumulator. Populated by
|
||||
# the scheduler when --per-request-spec-decode-metrics is set (eagerly on
|
||||
# add_request, then observed each verify step); stays None otherwise.
|
||||
self.spec_decode_metrics: RequestSpecDecodeMetrics | None = None
|
||||
|
||||
self.block_hashes: list[BlockHash] = []
|
||||
# Store the block hasher without binding self to avoid creating a
|
||||
# reference cycle (Request -> partial -> Request) that prevents
|
||||
|
||||
Reference in New Issue
Block a user