[Frontend][Core][Spec Decode] Per-request acceptance stats in OpenAI API responses (#48915)
Signed-off-by: Matthew Kotila <7692737+matthewkotila@users.noreply.github.com> Co-authored-by: Reed Meyerson <31574681+reed-meyerson@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Reed Meyerson
Claude Opus 4.8
parent
2f41c894e3
commit
7cfb97e337
@@ -14,6 +14,7 @@ from vllm.entrypoints.generate.base.serving import (
|
||||
GenerateBaseServing,
|
||||
GenerationError,
|
||||
build_per_request_timing_metrics,
|
||||
build_spec_decoding_metrics,
|
||||
clamp_prompt_logprobs,
|
||||
format_token_id_placeholder,
|
||||
)
|
||||
@@ -27,7 +28,7 @@ from vllm.entrypoints.openai.completion.protocol import (
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
PerRequestTimingMetrics,
|
||||
PerRequestMetrics,
|
||||
PromptTokenUsageInfo,
|
||||
RequestResponseMetadata,
|
||||
UsageInfo,
|
||||
@@ -460,18 +461,22 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
# only emitted when usage reporting is enabled (i.e.
|
||||
# ``stream_options.include_usage=true`` or
|
||||
# ``--enable-force-include-usage``).
|
||||
stream_per_request_metrics: PerRequestTimingMetrics | None = None
|
||||
if (
|
||||
self.enable_per_request_metrics
|
||||
# See note in request_output_to_completion_response: suppress
|
||||
# when not attributable to one stream (multi-prompt or n>1).
|
||||
and num_prompts == 1
|
||||
and (request.n or 1) == 1
|
||||
):
|
||||
last_metrics = last_res.metrics if last_res is not None else None
|
||||
stream_per_request_metrics = build_per_request_timing_metrics(
|
||||
last_metrics, total_completion_tokens
|
||||
)
|
||||
stream_per_request_metrics: PerRequestMetrics | None = None
|
||||
# See note in request_output_to_completion_response: suppress when
|
||||
# not attributable to one stream (multi-prompt or n>1).
|
||||
if num_prompts == 1 and (request.n or 1) == 1:
|
||||
if self.enable_per_request_metrics:
|
||||
last_metrics = (
|
||||
last_res.metrics if last_res is not None else None
|
||||
)
|
||||
stream_per_request_metrics = build_per_request_timing_metrics(
|
||||
last_metrics, total_completion_tokens
|
||||
)
|
||||
spec_stats = build_spec_decoding_metrics(last_res)
|
||||
if spec_stats is not None:
|
||||
if stream_per_request_metrics is None:
|
||||
stream_per_request_metrics = PerRequestMetrics()
|
||||
stream_per_request_metrics.speculative_decoding = spec_stats
|
||||
|
||||
final_usage_chunk = CompletionStreamResponse(
|
||||
id=request_id,
|
||||
@@ -612,21 +617,24 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
|
||||
request_metadata.final_usage_info = usage
|
||||
|
||||
per_request_metrics: PerRequestTimingMetrics | None = None
|
||||
if (
|
||||
self.enable_per_request_metrics
|
||||
# Metrics describe a single generation stream, so suppress them when
|
||||
# they cannot be attributed to one: multiple prompts (timestamps
|
||||
# span prompts) or n>1 (stats belong to one of the n sequences).
|
||||
and len(final_res_batch) == 1
|
||||
and (request.n or 1) == 1
|
||||
):
|
||||
last_metrics = (
|
||||
last_final_res.metrics if last_final_res is not None else None
|
||||
)
|
||||
per_request_metrics = build_per_request_timing_metrics(
|
||||
last_metrics, num_generated_tokens
|
||||
)
|
||||
per_request_metrics: PerRequestMetrics | None = None
|
||||
# Per-request metrics (timing + spec-decode acceptance) describe a single
|
||||
# generation stream, so suppress them when they cannot be attributed to
|
||||
# one: multiple prompts (timestamps span prompts) or n>1 (stats belong to
|
||||
# one of the n sequences).
|
||||
if len(final_res_batch) == 1 and (request.n or 1) == 1:
|
||||
if self.enable_per_request_metrics:
|
||||
last_metrics = (
|
||||
last_final_res.metrics if last_final_res is not None else None
|
||||
)
|
||||
per_request_metrics = build_per_request_timing_metrics(
|
||||
last_metrics, num_generated_tokens
|
||||
)
|
||||
spec_stats = build_spec_decoding_metrics(last_final_res)
|
||||
if spec_stats is not None:
|
||||
if per_request_metrics is None:
|
||||
per_request_metrics = PerRequestMetrics()
|
||||
per_request_metrics.speculative_decoding = spec_stats
|
||||
|
||||
if final_res_batch:
|
||||
kv_transfer_params = final_res_batch[0].kv_transfer_params
|
||||
|
||||
Reference in New Issue
Block a user