[Frontend][Core][Spec Decode] Per-request acceptance stats in OpenAI API responses (#48915)

Signed-off-by: Matthew Kotila <7692737+matthewkotila@users.noreply.github.com>
Co-authored-by: Reed Meyerson <31574681+reed-meyerson@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Matthew Kotila
2026-08-20 15:42:49 -07:00
committed by GitHub
co-authored by Reed Meyerson Claude Opus 4.8
parent 2f41c894e3
commit 7cfb97e337
25 changed files with 798 additions and 76 deletions
+8
View File
@@ -656,6 +656,9 @@ class EngineArgs:
collect_detailed_traces: list[DetailedTraceModules] | None = (
ObservabilityConfig.collect_detailed_traces
)
per_request_spec_decode_metrics: Literal["none", "summary", "detailed"] = (
ObservabilityConfig.per_request_spec_decode_metrics
)
kv_cache_metrics: bool = ObservabilityConfig.kv_cache_metrics
kv_cache_metrics_sample: float = get_field(
ObservabilityConfig, "kv_cache_metrics_sample"
@@ -1476,6 +1479,10 @@ class EngineArgs:
"--collect-detailed-traces",
**observability_kwargs["collect_detailed_traces"],
)
observability_group.add_argument(
"--per-request-spec-decode-metrics",
**observability_kwargs["per_request_spec_decode_metrics"],
)
observability_group.add_argument(
"--kv-cache-metrics", **observability_kwargs["kv_cache_metrics"]
)
@@ -1934,6 +1941,7 @@ class EngineArgs:
show_hidden_metrics_for_version=self.show_hidden_metrics_for_version,
otlp_traces_endpoint=self.otlp_traces_endpoint,
collect_detailed_traces=self.collect_detailed_traces,
per_request_spec_decode_metrics=self.per_request_spec_decode_metrics,
kv_cache_metrics=self.kv_cache_metrics,
kv_cache_metrics_sample=self.kv_cache_metrics_sample,
cudagraph_metrics=self.cudagraph_metrics,