[Frontend][Core][Spec Decode] Per-request acceptance stats in OpenAI API responses (#48915)
Signed-off-by: Matthew Kotila <7692737+matthewkotila@users.noreply.github.com> Co-authored-by: Reed Meyerson <31574681+reed-meyerson@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Reed Meyerson
Claude Opus 4.8
parent
2f41c894e3
commit
7cfb97e337
@@ -35,6 +35,7 @@ from vllm.v1.engine.parallel_sampling import ParentRequest
|
||||
from vllm.v1.metrics.stats import (
|
||||
IterationStats,
|
||||
LoRARequestStates,
|
||||
RequestSpecDecodeMetrics,
|
||||
RequestStateStats,
|
||||
SchedulerStats,
|
||||
)
|
||||
@@ -175,6 +176,9 @@ class RequestState:
|
||||
self.queue = queue
|
||||
self.num_cached_tokens = 0
|
||||
self.num_cache_creation_tokens = 0
|
||||
# Per-sequence spec-decode accumulator; arrives once (on finish) via
|
||||
# EngineCoreOutput, then attached to this sequence's CompletionOutput.
|
||||
self.spec_decode_metrics: RequestSpecDecodeMetrics | None = None
|
||||
|
||||
self.stats = RequestStateStats(arrival_time=arrival_time) if log_stats else None
|
||||
|
||||
@@ -429,6 +433,7 @@ class RequestState:
|
||||
cumulative_logprob=self.logprobs_processor.cumulative_logprob,
|
||||
finish_reason=str(finish_reason) if finished else None,
|
||||
stop_reason=stop_reason if finished else None,
|
||||
spec_decode_metrics=self.spec_decode_metrics if finished else None,
|
||||
)
|
||||
|
||||
def _new_pooling_output(self, pooling_output: torch.Tensor) -> PoolingOutput:
|
||||
@@ -658,6 +663,9 @@ class OutputProcessor:
|
||||
)
|
||||
req_state.is_prefilling = False
|
||||
|
||||
if engine_core_output.spec_decode_metrics is not None:
|
||||
req_state.spec_decode_metrics = engine_core_output.spec_decode_metrics
|
||||
|
||||
if pooling_output is None:
|
||||
assert req_state.detokenizer is not None
|
||||
assert req_state.logprobs_processor is not None
|
||||
|
||||
Reference in New Issue
Block a user