[Frontend][Core][Spec Decode] Per-request acceptance stats in OpenAI API responses (#48915)

Signed-off-by: Matthew Kotila <7692737+matthewkotila@users.noreply.github.com>
Co-authored-by: Reed Meyerson <31574681+reed-meyerson@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Matthew Kotila
2026-08-20 15:42:49 -07:00
committed by GitHub
co-authored by Reed Meyerson Claude Opus 4.8
parent 2f41c894e3
commit 7cfb97e337
25 changed files with 798 additions and 76 deletions
+8
View File
@@ -35,6 +35,7 @@ from vllm.v1.engine.parallel_sampling import ParentRequest
from vllm.v1.metrics.stats import (
IterationStats,
LoRARequestStates,
RequestSpecDecodeMetrics,
RequestStateStats,
SchedulerStats,
)
@@ -175,6 +176,9 @@ class RequestState:
self.queue = queue
self.num_cached_tokens = 0
self.num_cache_creation_tokens = 0
# Per-sequence spec-decode accumulator; arrives once (on finish) via
# EngineCoreOutput, then attached to this sequence's CompletionOutput.
self.spec_decode_metrics: RequestSpecDecodeMetrics | None = None
self.stats = RequestStateStats(arrival_time=arrival_time) if log_stats else None
@@ -429,6 +433,7 @@ class RequestState:
cumulative_logprob=self.logprobs_processor.cumulative_logprob,
finish_reason=str(finish_reason) if finished else None,
stop_reason=stop_reason if finished else None,
spec_decode_metrics=self.spec_decode_metrics if finished else None,
)
def _new_pooling_output(self, pooling_output: torch.Tensor) -> PoolingOutput:
@@ -658,6 +663,9 @@ class OutputProcessor:
)
req_state.is_prefilling = False
if engine_core_output.spec_decode_metrics is not None:
req_state.spec_decode_metrics = engine_core_output.spec_decode_metrics
if pooling_output is None:
assert req_state.detokenizer is not None
assert req_state.logprobs_processor is not None