[Core] Add PROMOTION_LATENCY histogram metric for tiering offload (#53910)

Signed-off-by: Wen Zhou <wenzhou@redhat.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Wen Zhou
2026-10-15 13:47:41 +03:00
committed by GitHub
co-authored by Claude Opus 4.6 OpenAI Codex Kevin H. Luu Or Ozeri
parent 6fe7401fac
commit e10c10a644
6 changed files with 234 additions and 3 deletions
+1
View File
@@ -65,6 +65,7 @@ class TieringOffloadingMetrics:
)
BACKPRESSURE_STORES_DROPPED = "vllm:kv_offload_tiering_backpressure_stores_dropped"
BACKPRESSURE_BLOCKS_DROPPED = "vllm:kv_offload_tiering_backpressure_blocks_dropped"
PROMOTION_LATENCY = "vllm:kv_offload_tiering_promotion_latency_seconds"
@dataclass
+20
View File
@@ -124,6 +124,8 @@ class TieringMetricsTracker:
self, job_metadata: _JobMetadataLike, result: JobResult
) -> None:
self._observe_finished_job_stats(job_metadata, result)
if job_metadata.transfer_job.is_promotion:
self._observe_promotion_latency(job_metadata, result)
self._decrement_tier_state(job_metadata)
def on_promotion_allocation_failure(self) -> None:
@@ -237,6 +239,24 @@ class TieringMetricsTracker:
time_metric, completed_job.transfer_time, labelvalues
)
def _observe_promotion_latency(
self, job_metadata: _JobMetadataLike, result: JobResult
) -> None:
"""Record promotion time from job creation to reported completion.
This is broader than the tier-reported `JobResult.transfer_time`, which
covers only the active data movement, and so includes tier queueing.
Failed promotions are observed too; PROMOTION_JOB_FAILURES counts them.
"""
self._stats.observe_histogram(
TieringOffloadingMetrics.PROMOTION_LATENCY,
time.monotonic() - job_metadata.transfer_job.submit_time,
(
*self.tier_label(job_metadata.tier_idx),
"success" if result.success else "failure",
),
)
def _observe_active_transfer_stats(self, stats: OffloadingConnectorStats) -> None:
for tier_idx, state in enumerate(self._tier_states):
labelvalues = self.tier_label(tier_idx)
+33 -2
View File
@@ -159,7 +159,10 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
metrics[TieringOffloadingMetrics.READ_TIME] = OffloadingCounterMetadata(
documentation=(
"Total time spent reading from secondary tiers into the primary "
"tier, in seconds, labeled by tier."
"tier, in seconds, labeled by tier. Tiers that do not report "
"per-job transfer time have no series here. Use "
"vllm:kv_offload_tiering_promotion_latency_seconds for "
"tier-independent promotion timing."
),
labelnames=(TIER_LABEL,),
)
@@ -173,7 +176,8 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
metrics[TieringOffloadingMetrics.WRITE_TIME] = OffloadingCounterMetadata(
documentation=(
"Total time spent writing from the primary tier to secondary "
"tiers, in seconds, labeled by tier."
"tiers, in seconds, labeled by tier. Tiers that do not report "
"per-job transfer time have no series here."
),
labelnames=(TIER_LABEL,),
)
@@ -243,6 +247,33 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
),
labelnames=(TIER_LABEL,),
)
metrics[TieringOffloadingMetrics.PROMOTION_LATENCY] = (
OffloadingHistogramMetadata(
documentation=(
"Histogram of per-job latency for secondary-tier promotion "
"attempts, measured from job creation until the tiering "
"manager observes completion so that tier queueing is "
"included, labeled by tier and result, in seconds. Includes "
"failed attempts."
),
labelnames=(TIER_LABEL, "result"),
buckets=(
0.005,
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1,
2.5,
5,
10,
30,
60,
),
)
)
for tier_config, tier_cls in cls._get_secondary_tiers(extra_config):
metrics.update(tier_cls.build_metric_definitions(tier_config))