[Metrics] Add --custom-histogram-buckets to override histogram bucket families (#48867)
The core engine histograms ship with default bucket boundaries tuned for typical serving workloads. The `--custom-histogram-buckets` option replaces the boundaries of one or more bucket families. Use it, for example, to track sub-300ms latency SLOs with the request-phase histograms, whose smallest default boundary is 0.3s. Signed-off-by: Guy Stone <guys@spotify.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
a47e4a5568
commit
37d6174096
@@ -723,6 +723,9 @@ class EngineArgs:
|
||||
kv_cache_metrics_sample: float = get_field(
|
||||
ObservabilityConfig, "kv_cache_metrics_sample"
|
||||
)
|
||||
custom_histogram_buckets: dict[str, list[float]] | None = (
|
||||
ObservabilityConfig.custom_histogram_buckets
|
||||
)
|
||||
cudagraph_metrics: bool = ObservabilityConfig.cudagraph_metrics
|
||||
enable_layerwise_nvtx_tracing: bool = (
|
||||
ObservabilityConfig.enable_layerwise_nvtx_tracing
|
||||
@@ -1633,6 +1636,10 @@ class EngineArgs:
|
||||
"--kv-cache-metrics-sample",
|
||||
**observability_kwargs["kv_cache_metrics_sample"],
|
||||
)
|
||||
observability_group.add_argument(
|
||||
"--custom-histogram-buckets",
|
||||
**observability_kwargs["custom_histogram_buckets"],
|
||||
)
|
||||
observability_group.add_argument(
|
||||
"--cudagraph-metrics",
|
||||
**observability_kwargs["cudagraph_metrics"],
|
||||
@@ -2126,6 +2133,7 @@ class EngineArgs:
|
||||
per_request_spec_decode_metrics=self.per_request_spec_decode_metrics,
|
||||
kv_cache_metrics=self.kv_cache_metrics,
|
||||
kv_cache_metrics_sample=self.kv_cache_metrics_sample,
|
||||
custom_histogram_buckets=self.custom_histogram_buckets,
|
||||
cudagraph_metrics=self.cudagraph_metrics,
|
||||
enable_layerwise_nvtx_tracing=self.enable_layerwise_nvtx_tracing,
|
||||
enable_mfu_metrics=self.enable_mfu_metrics,
|
||||
|
||||
Reference in New Issue
Block a user