[Core] Add PROMOTION_LATENCY histogram metric for tiering offload (#53910)
Signed-off-by: Wen Zhou <wenzhou@redhat.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
OpenAI Codex
Kevin H. Luu
Or Ozeri
parent
6fe7401fac
commit
e10c10a644
@@ -65,6 +65,7 @@ class TieringOffloadingMetrics:
|
||||
)
|
||||
BACKPRESSURE_STORES_DROPPED = "vllm:kv_offload_tiering_backpressure_stores_dropped"
|
||||
BACKPRESSURE_BLOCKS_DROPPED = "vllm:kv_offload_tiering_backpressure_blocks_dropped"
|
||||
PROMOTION_LATENCY = "vllm:kv_offload_tiering_promotion_latency_seconds"
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -124,6 +124,8 @@ class TieringMetricsTracker:
|
||||
self, job_metadata: _JobMetadataLike, result: JobResult
|
||||
) -> None:
|
||||
self._observe_finished_job_stats(job_metadata, result)
|
||||
if job_metadata.transfer_job.is_promotion:
|
||||
self._observe_promotion_latency(job_metadata, result)
|
||||
self._decrement_tier_state(job_metadata)
|
||||
|
||||
def on_promotion_allocation_failure(self) -> None:
|
||||
@@ -237,6 +239,24 @@ class TieringMetricsTracker:
|
||||
time_metric, completed_job.transfer_time, labelvalues
|
||||
)
|
||||
|
||||
def _observe_promotion_latency(
|
||||
self, job_metadata: _JobMetadataLike, result: JobResult
|
||||
) -> None:
|
||||
"""Record promotion time from job creation to reported completion.
|
||||
|
||||
This is broader than the tier-reported `JobResult.transfer_time`, which
|
||||
covers only the active data movement, and so includes tier queueing.
|
||||
Failed promotions are observed too; PROMOTION_JOB_FAILURES counts them.
|
||||
"""
|
||||
self._stats.observe_histogram(
|
||||
TieringOffloadingMetrics.PROMOTION_LATENCY,
|
||||
time.monotonic() - job_metadata.transfer_job.submit_time,
|
||||
(
|
||||
*self.tier_label(job_metadata.tier_idx),
|
||||
"success" if result.success else "failure",
|
||||
),
|
||||
)
|
||||
|
||||
def _observe_active_transfer_stats(self, stats: OffloadingConnectorStats) -> None:
|
||||
for tier_idx, state in enumerate(self._tier_states):
|
||||
labelvalues = self.tier_label(tier_idx)
|
||||
|
||||
@@ -159,7 +159,10 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
metrics[TieringOffloadingMetrics.READ_TIME] = OffloadingCounterMetadata(
|
||||
documentation=(
|
||||
"Total time spent reading from secondary tiers into the primary "
|
||||
"tier, in seconds, labeled by tier."
|
||||
"tier, in seconds, labeled by tier. Tiers that do not report "
|
||||
"per-job transfer time have no series here. Use "
|
||||
"vllm:kv_offload_tiering_promotion_latency_seconds for "
|
||||
"tier-independent promotion timing."
|
||||
),
|
||||
labelnames=(TIER_LABEL,),
|
||||
)
|
||||
@@ -173,7 +176,8 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
metrics[TieringOffloadingMetrics.WRITE_TIME] = OffloadingCounterMetadata(
|
||||
documentation=(
|
||||
"Total time spent writing from the primary tier to secondary "
|
||||
"tiers, in seconds, labeled by tier."
|
||||
"tiers, in seconds, labeled by tier. Tiers that do not report "
|
||||
"per-job transfer time have no series here."
|
||||
),
|
||||
labelnames=(TIER_LABEL,),
|
||||
)
|
||||
@@ -243,6 +247,33 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
),
|
||||
labelnames=(TIER_LABEL,),
|
||||
)
|
||||
metrics[TieringOffloadingMetrics.PROMOTION_LATENCY] = (
|
||||
OffloadingHistogramMetadata(
|
||||
documentation=(
|
||||
"Histogram of per-job latency for secondary-tier promotion "
|
||||
"attempts, measured from job creation until the tiering "
|
||||
"manager observes completion so that tier queueing is "
|
||||
"included, labeled by tier and result, in seconds. Includes "
|
||||
"failed attempts."
|
||||
),
|
||||
labelnames=(TIER_LABEL, "result"),
|
||||
buckets=(
|
||||
0.005,
|
||||
0.01,
|
||||
0.025,
|
||||
0.05,
|
||||
0.1,
|
||||
0.25,
|
||||
0.5,
|
||||
1,
|
||||
2.5,
|
||||
5,
|
||||
10,
|
||||
30,
|
||||
60,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
for tier_config, tier_cls in cls._get_secondary_tiers(extra_config):
|
||||
metrics.update(tier_cls.build_metric_definitions(tier_config))
|
||||
|
||||
Reference in New Issue
Block a user