Add tiering offloading metrics (#48798)

Signed-off-by: srinivas_oo7 <sklinkedin0120@gmail.com>
Co-authored-by: srinivas_oo7 <sklinkedin0120@gmail.com>
This commit is contained in:
Srinivas Krovvidi
2026-08-10 05:23:14 +03:00
committed by GitHub
co-authored by srinivas_oo7
parent 11ba93f364
commit a123159f7a
16 changed files with 1031 additions and 237 deletions
@@ -25,6 +25,7 @@ from vllm.v1.kv_offload.base import (
OffloadingHistogramMetadata,
)
from vllm.v1.kv_offload.factory import OffloadingSpecFactory
from vllm.v1.kv_offload.tiering.base import TieringOffloadingMetrics
LOAD_BYTES = _TransferMetricName.LOAD_BYTES
LOAD_TIME = _TransferMetricName.LOAD_TIME
@@ -101,10 +102,15 @@ class _FakeMetric:
class _FakeVllmConfig:
def __init__(self, store_threshold: int = 2):
self.kv_transfer_config = SimpleNamespace(
kv_connector_extra_config={"store_threshold": store_threshold}
)
def __init__(
self,
store_threshold: int = 2,
extra_config: dict[str, Any] | None = None,
):
config = {"store_threshold": store_threshold}
if extra_config is not None:
config.update(extra_config)
self.kv_transfer_config = SimpleNamespace(kv_connector_extra_config=config)
def _spec_cls_with_metric_definitions(
@@ -679,6 +685,30 @@ def test_prom_metrics_uses_configured_manager_metrics():
assert STORES_SKIPPED not in prom_metrics._offloading_metric_metadata
def test_prom_metrics_registers_tiering_metrics_from_spec():
prom_metrics = OffloadPromMetrics(
vllm_config=_FakeVllmConfig(
store_threshold=0,
extra_config={
"spec_name": "TieringOffloadingSpec",
"secondary_tiers": [],
},
), # type: ignore[arg-type]
metric_types={
Gauge: _FakeMetric,
Counter: _FakeMetric,
Histogram: _FakeMetric,
},
labelnames=["model_name", "engine"],
per_engine_labelvalues={0: ["model", "0"]},
)
metric = prom_metrics._offloading_metric_defs[
TieringOffloadingMetrics.BLOCK_QUERIES
]
assert metric.kwargs["labelnames"] == ["model_name", "engine", "tier"]
def test_aggregate_into_empty_stats():
"""Aggregating non-empty stats into a fresh (empty) stats object works."""
empty = OffloadingConnectorStats()
@@ -16,7 +16,7 @@ import numpy as np
import pytest
from vllm.v1.kv_offload.base import LookupResult, ReqContext, ScheduleEndContext
from vllm.v1.kv_offload.tiering.base import JobMetadata, JobResult
from vllm.v1.kv_offload.tiering.base import JobResult, TransferJob
from vllm.v1.kv_offload.tiering.p2p import manager as manager_module
from vllm.v1.kv_offload.tiering.p2p.manager import (
_UNBOUND_STORE_TIMEOUT_S,
@@ -86,12 +86,12 @@ def _job_metadata(
keys: list[bytes] | None = None,
block_ids: list[int] | None = None,
kv_params: dict | None = None,
) -> JobMetadata:
) -> TransferJob:
if keys is None:
keys = [b"key1"]
if block_ids is None:
block_ids = list(range(len(keys)))
return JobMetadata(
return TransferJob(
job_id=job_id,
keys=keys,
block_ids=np.array(block_ids),
@@ -24,7 +24,7 @@ from vllm.v1.kv_offload.base import (
ReqContext,
RequestOffloadingContext,
)
from vllm.v1.kv_offload.tiering.base import JobMetadata
from vllm.v1.kv_offload.tiering.base import TransferJob
from vllm.v1.kv_offload.tiering.p2p.session import (
LoadResult,
P2PSession,
@@ -256,13 +256,13 @@ class FakeParent:
self,
keys: Sequence[OffloadKey],
ctx: ReqContext,
) -> JobMetadata:
) -> TransferJob:
keys_list = list(keys)
self.calls.append(("create_store_job", tuple(keys_list), ctx.req_id))
block_ids = np.array([self.stored[k] for k in keys_list], dtype=np.int32)
job_id = self._next_job_id
self._next_job_id += 1
return JobMetadata(
return TransferJob(
job_id=job_id,
keys=keys_list,
block_ids=block_ids,
+3 -3
View File
@@ -35,7 +35,7 @@ from vllm.v1.kv_offload.config import (
OffloadingModelConfig,
OffloadingParallelConfig,
)
from vllm.v1.kv_offload.tiering.base import JobMetadata
from vllm.v1.kv_offload.tiering.base import TransferJob
from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory
from vllm.v1.kv_offload.tiering.fs.manager import (
FileSystemTierManager,
@@ -105,10 +105,10 @@ def make_job(
keys: list[OffloadKey],
block_ids: list[int] | None = None,
is_promotion: bool = False,
) -> JobMetadata:
) -> TransferJob:
if block_ids is None:
block_ids = list(range(len(keys)))
return JobMetadata(
return TransferJob(
job_id=job_id,
keys=keys,
block_ids=np.array(block_ids, dtype=np.int64),
+226
View File
@@ -0,0 +1,226 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
import numpy as np
import pytest
from vllm.v1.kv_offload.base import LookupResult, ReqContext, make_offload_key
from vllm.v1.kv_offload.tiering.base import (
JobResult,
TieringOffloadingMetrics,
TransferJob,
)
from vllm.v1.kv_offload.tiering.manager import JobMetadata
from vllm.v1.kv_offload.tiering.metrics import TieringMetricsTracker
_CTX = ReqContext(req_id="test")
def to_keys(int_ids: Iterable[int]):
return [make_offload_key(str(i).encode(), 0) for i in int_ids]
def test_tiering_metrics_tracker_records_lookup_metrics():
tracker = TieringMetricsTracker(
tier_types=["fs", "p2p"],
num_primary_blocks=5,
primary_block_size=16,
)
tracker.on_new_request(_CTX)
key = to_keys([1])[0]
tracker.on_lookup(
_CTX,
key,
tracker.primary_tier_label,
LookupResult.MISS,
lookup_duration=0.0,
)
tracker.on_lookup(_CTX, key, tracker.tier_label(0), LookupResult.MISS, 0.0)
tracker.on_lookup(_CTX, key, tracker.tier_label(1), LookupResult.HIT, 0.0)
tracker.on_lookup(_CTX, key, tracker.tier_label(1), LookupResult.HIT, 0.0)
stats = tracker.take_stats()
assert stats is not None
values = stats.data["data"]
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("0:primary",)] == 1
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("1:fs",)] == 1
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("2:p2p",)] == 1
assert values[TieringOffloadingMetrics.BLOCK_HITS][("2:p2p",)] == 1
assert ("1:fs",) not in values[TieringOffloadingMetrics.BLOCK_HITS]
def test_tiering_metrics_tracker_stops_lookup_metrics_after_allocation():
tracker = TieringMetricsTracker(
tier_types=["fs"],
num_primary_blocks=5,
primary_block_size=16,
)
tracker.on_new_request(_CTX)
key = to_keys([1])[0]
tracker.on_lookup(
_CTX,
key,
tracker.primary_tier_label,
LookupResult.MISS,
lookup_duration=0.0,
)
tracker.on_lookup(_CTX, key, tracker.tier_label(0), LookupResult.HIT, 0.0)
stats = tracker.take_stats()
assert stats is not None
values = stats.data["data"]
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("0:primary",)] == 1
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("1:fs",)] == 1
assert values[TieringOffloadingMetrics.BLOCK_HITS][("1:fs",)] == 1
tracker.on_request_allocated(_CTX)
tracker.on_lookup(
_CTX,
key,
tracker.primary_tier_label,
LookupResult.MISS,
lookup_duration=0.0,
)
tracker.on_lookup(_CTX, key, tracker.tier_label(0), LookupResult.HIT, 0.0)
stats = tracker.take_stats()
assert stats is not None
values = stats.data["data"]
assert TieringOffloadingMetrics.BLOCK_QUERIES not in values
assert TieringOffloadingMetrics.BLOCK_HITS not in values
def test_tiering_metrics_tracker_records_finished_job_metrics():
tracker = TieringMetricsTracker(
tier_types=["fs"],
num_primary_blocks=5,
primary_block_size=16,
)
cascade_key, promotion_key, failed_cascade_key, failed_promotion_key = to_keys(
range(4)
)
jobs = [
JobMetadata(
TransferJob(0, [cascade_key], np.array([0]), False, _CTX),
0,
),
JobMetadata(
TransferJob(1, [promotion_key], np.array([1]), True, _CTX),
0,
),
JobMetadata(
TransferJob(2, [failed_cascade_key], np.array([2]), False, _CTX),
0,
),
JobMetadata(
TransferJob(3, [failed_promotion_key], np.array([3]), True, _CTX),
0,
),
]
for job in jobs:
tracker.on_job_registered(job)
tracker.on_job_finished(
jobs[0], JobResult(job_id=0, success=True, transfer_time=0.5)
)
tracker.on_job_finished(
jobs[1], JobResult(job_id=1, success=True, transfer_time=0.25)
)
tracker.on_job_finished(jobs[2], JobResult(job_id=2, success=False))
tracker.on_job_finished(jobs[3], JobResult(job_id=3, success=False))
stats = tracker.take_stats()
assert stats is not None
values = stats.data["data"]
label = ("1:fs",)
assert values[TieringOffloadingMetrics.WRITE_BYTES][label] == 16
assert values[TieringOffloadingMetrics.WRITE_TIME][label] == 0.5
assert values[TieringOffloadingMetrics.READ_BYTES][label] == 16
assert values[TieringOffloadingMetrics.READ_TIME][label] == 0.25
assert values[TieringOffloadingMetrics.CASCADE_JOB_FAILURES][label] == 1
assert values[TieringOffloadingMetrics.PROMOTION_JOB_FAILURES][label] == 1
tracker.assert_idle()
def test_tiering_metrics_tracker_records_partial_promotion_success_bytes():
tracker = TieringMetricsTracker(
tier_types=["fs"],
num_primary_blocks=5,
primary_block_size=16,
)
keys = to_keys(range(3))
job = JobMetadata(
TransferJob(0, keys, np.array([0, 1, 2]), True, _CTX),
0,
)
tracker.on_job_registered(job)
tracker.on_job_finished(
job,
JobResult(
job_id=0,
success=False,
successful_keys=(keys[0], keys[2]),
transfer_time=0.5,
),
)
stats = tracker.take_stats()
assert stats is not None
values = stats.data["data"]
label = ("1:fs",)
assert values[TieringOffloadingMetrics.READ_BYTES][label] == 32
assert values[TieringOffloadingMetrics.READ_TIME][label] == 0.5
assert values[TieringOffloadingMetrics.PROMOTION_JOB_FAILURES][label] == 1
tracker.assert_idle()
def test_tiering_metrics_tracker_reports_active_job_and_primary_usage_gauges():
tracker = TieringMetricsTracker(
tier_types=["fs", "p2p"],
num_primary_blocks=6,
primary_block_size=16,
)
fs_job = JobMetadata(
TransferJob(0, to_keys([0, 1]), np.array([0, 1]), False, _CTX),
0,
)
p2p_job = JobMetadata(
TransferJob(1, to_keys([2, 3, 4]), np.array([2, 3, 4]), True, _CTX),
1,
)
tracker.on_job_registered(fs_job)
tracker.on_job_registered(p2p_job)
stats = tracker.take_stats()
assert stats is not None
values = stats.data["data"]
fs_label = ("1:fs",)
p2p_label = ("2:p2p",)
assert values[TieringOffloadingMetrics.PRIMARY_READ_USAGE_PERC][
fs_label
] == pytest.approx(2 / 6)
assert values[TieringOffloadingMetrics.ACTIVE_CASCADE_JOBS][fs_label] == 1
assert values[TieringOffloadingMetrics.PRIMARY_WRITE_USAGE_PERC][
p2p_label
] == pytest.approx(3 / 6)
assert values[TieringOffloadingMetrics.ACTIVE_PROMOTION_JOBS][p2p_label] == 1
def test_tiering_metrics_tracker_records_promotion_allocation_failures():
tracker = TieringMetricsTracker(
tier_types=["fs"],
num_primary_blocks=1,
primary_block_size=16,
)
tracker.on_promotion_allocation_failure()
stats = tracker.take_stats()
assert stats is not None
values = stats.data["data"]
assert values[TieringOffloadingMetrics.PROMOTION_ALLOCATION_FAILURES][()] == 1
+6 -3
View File
@@ -34,7 +34,7 @@ from vllm.v1.kv_offload.config import (
OffloadingModelConfig,
OffloadingParallelConfig,
)
from vllm.v1.kv_offload.tiering.base import JobMetadata, JobResult
from vllm.v1.kv_offload.tiering.base import JobResult, TransferJob
from vllm.v1.kv_offload.tiering.manager import (
CPUPrimaryTierOffloadingManager,
TieringOffloadingManager,
@@ -105,10 +105,10 @@ def make_job(
job_id: int,
keys: list[OffloadKey],
block_ids: list[int] | None = None,
) -> JobMetadata:
) -> TransferJob:
if block_ids is None:
block_ids = list(range(len(keys)))
return JobMetadata(
return TransferJob(
job_id=job_id,
keys=keys,
block_ids=np.array(block_ids, dtype=np.int64),
@@ -201,6 +201,9 @@ class MockNixlAgent:
def release_dlist_handle(self, handle):
pass
def get_xfer_telemetry(self, handle):
return SimpleNamespace(xferDuration=1000)
def _query_memory(self, queries, mem_type, agent_name):
return [object() if q[3] in self._stored_obj_keys else None for q in queries]
@@ -29,6 +29,7 @@ from vllm.v1.kv_offload.base import (
Medium,
OffloadingCounterMetadata,
OffloadingEvent,
OffloadingGaugeMetadata,
OffloadKey,
OffloadPolicy,
ReqContext,
@@ -39,10 +40,10 @@ from vllm.v1.kv_offload.base import (
make_offload_key,
)
from vllm.v1.kv_offload.tiering.base import (
JobMetadata,
JobResult,
SecondaryTierManager,
TieringOffloadingMetrics,
TransferJob,
)
from vllm.v1.kv_offload.tiering.example.manager import ExampleSecondaryTierManager
from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory
@@ -85,6 +86,16 @@ def count_hits(manager, keys: list[OffloadKey]) -> int | None:
return count
def histogram_count_key(metric_name: str, labelvalues: tuple[str, ...]) -> str:
return f"{metric_name}:{labelvalues}_count"
def has_histogram_count(reduced: dict[str, int | float], metric_name: str) -> bool:
return any(
key.startswith(f"{metric_name}:") and key.endswith("_count") for key in reduced
)
class MetricsSecondaryTierManager(SecondaryTierManager):
"""Test-only secondary tier that declares and emits one labeled metric."""
@@ -102,18 +113,26 @@ class MetricsSecondaryTierManager(SecondaryTierManager):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.stats: OffloadingConnectorStats | None = None
self.lookup_result = LookupResult.MISS
self.finished_jobs: list[JobResult] = []
self.submitted_loads: list[TransferJob] = []
self.submitted_stores: list[TransferJob] = []
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
return False
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
return self.lookup_result
def submit_store(self, job_metadata: JobMetadata) -> None:
def submit_store(self, job_metadata: TransferJob) -> None:
self.submitted_stores.append(job_metadata)
return
def submit_load(self, job_metadata: JobMetadata) -> None:
def submit_load(self, job_metadata: TransferJob) -> None:
self.submitted_loads.append(job_metadata)
return
def get_finished_jobs(self) -> Iterable[JobResult]:
return ()
results = self.finished_jobs
self.finished_jobs = []
return results
def drain_jobs(self) -> None:
return
@@ -141,6 +160,24 @@ def test_tiering_spec_collects_secondary_metric_definitions(monkeypatch):
metadata = metrics[MetricsSecondaryTierManager.MY_TIER_METRIC]
assert metadata.documentation == "Number of bytes served by the test tier."
assert metadata.labelnames == ("tier",)
assert metrics[TieringOffloadingMetrics.BLOCK_QUERIES].labelnames == ("tier",)
assert metrics[TieringOffloadingMetrics.LOOKUP_SYNC_DELAY].labelnames == ("tier",)
assert metrics[TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY].labelnames == ("tier",)
assert metrics[TieringOffloadingMetrics.PROMOTION_JOB_FAILURES].labelnames == (
"tier",
)
assert isinstance(
metrics[TieringOffloadingMetrics.PROMOTION_ALLOCATION_FAILURES],
OffloadingCounterMetadata,
)
for metric_name in (
TieringOffloadingMetrics.PRIMARY_WRITE_USAGE_PERC,
TieringOffloadingMetrics.PRIMARY_READ_USAGE_PERC,
TieringOffloadingMetrics.ACTIVE_PROMOTION_JOBS,
TieringOffloadingMetrics.ACTIVE_CASCADE_JOBS,
):
assert isinstance(metrics[metric_name], OffloadingGaugeMetadata)
assert metrics[metric_name].labelnames == ("tier",)
def test_tiering_manager_aggregates_secondary_stats():
@@ -261,12 +298,15 @@ class TestTieringOffloadingManager:
self._start_request()
# Register an in-flight promotion job for tier1 by hand.
job_id = self.manager._next_job_id()
self.manager._transfer_jobs[job_id] = JobMetadata(
job_id=job_id,
keys=to_keys([1, 2]),
block_ids=[0, 1],
is_promotion=True,
req_context=_CTX,
self.manager._register_job(
TransferJob(
job_id=job_id,
keys=to_keys([1, 2]),
block_ids=[0, 1],
is_promotion=True,
req_context=_CTX,
),
0,
)
failed = JobResult(job_id=job_id, success=False)
with (
@@ -285,12 +325,15 @@ class TestTieringOffloadingManager:
self._start_request()
job_id = self.manager._next_job_id()
self.manager._transfer_jobs[job_id] = JobMetadata(
job_id=job_id,
keys=to_keys([1]),
block_ids=[0],
is_promotion=True,
req_context=_CTX,
self.manager._register_job(
TransferJob(
job_id=job_id,
keys=to_keys([1]),
block_ids=[0],
is_promotion=True,
req_context=_CTX,
),
0,
)
ok = JobResult(job_id=job_id, success=True)
with (
@@ -473,7 +516,7 @@ class TestTieringOffloadingManager:
for block in blocks:
self.secondary_tier1.blocks[block] = True
def submit_partial(job_metadata: JobMetadata) -> None:
def submit_partial(job_metadata: TransferJob) -> None:
successful_keys = (
None
if successful_indices is None
@@ -500,7 +543,7 @@ class TestTieringOffloadingManager:
] == expected_results
def test_lookup_reports_sync_delay_for_resolved_lookups(self, manager_setup):
"""Resolved lookups report one sync delay sample on allocation."""
"""Resolved lookups report one sync delay sample per tier and block."""
self._start_request()
blocks = to_keys(range(2))
@@ -508,22 +551,39 @@ class TestTieringOffloadingManager:
for block in blocks:
assert self.manager.lookup(block, _CTX) is LookupResult.MISS
stats = self.manager.get_stats()
if stats is not None:
assert f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count" not in (
stats.reduce()
)
self._simulate_on_schedule_end(new_req_ids=[_CTX.req_id])
stats = self.manager.get_stats()
assert stats is not None
reduced = stats.reduce()
assert reduced[f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count"] == 1
assert f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count" not in reduced
assert (
reduced[
histogram_count_key(
TieringOffloadingMetrics.LOOKUP_SYNC_DELAY, ("0:primary",)
)
]
== 2
)
assert (
reduced[
histogram_count_key(
TieringOffloadingMetrics.LOOKUP_SYNC_DELAY, ("1:example",)
)
]
== 2
)
assert (
reduced[
histogram_count_key(
TieringOffloadingMetrics.LOOKUP_SYNC_DELAY, ("2:example",)
)
]
== 2
)
assert not has_histogram_count(
reduced, TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY
)
def test_lookup_reports_async_delay_across_promotion(self, manager_setup):
"""A new request reports async delay at schedule end."""
def test_lookup_does_not_report_async_delay_for_promotion(self, manager_setup):
"""Promotion time is not included in lookup async delay."""
self._start_request()
block = to_keys(range(1))[0]
self.secondary_tier1.blocks[block] = True
@@ -531,13 +591,30 @@ class TestTieringOffloadingManager:
# First lookup finds the block in a secondary tier and defers.
assert self.manager.lookup(block, _CTX) is LookupResult.RETRY
# The first scheduler step reports async delay for the new request.
# Promotion is not treated as unresolved secondary lookup time.
self._simulate_on_schedule_end(new_req_ids=[_CTX.req_id])
stats = self.manager.get_stats()
assert stats is not None
reduced = stats.reduce()
assert reduced[f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count"] == 1
assert reduced[f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count"] == 1
assert (
reduced[
histogram_count_key(
TieringOffloadingMetrics.LOOKUP_SYNC_DELAY, ("0:primary",)
)
]
== 1
)
assert (
reduced[
histogram_count_key(
TieringOffloadingMetrics.LOOKUP_SYNC_DELAY, ("1:example",)
)
]
== 1
)
assert not has_histogram_count(
reduced, TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY
)
# Promotion completes on the next scheduler step.
self._simulate_on_schedule_end()
@@ -547,38 +624,51 @@ class TestTieringOffloadingManager:
stats = self.manager.get_stats()
if stats is not None:
assert f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count" not in (
stats.reduce()
assert not has_histogram_count(
stats.reduce(), TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY
)
def test_lookup_reports_async_delay_on_request_finish(self, manager_setup):
"""Never-allocated lookup delays flush at teardown."""
def test_lookup_reports_async_delay_when_deferred_lookup_resolves(
self, manager_setup
):
"""Async delay is observed when a RETRY lookup later resolves."""
ctx = ReqContext(req_id="req_lookup_finish")
self._start_request(ctx)
block = to_keys(range(1))[0]
self.secondary_tier1.blocks[block] = True
self.secondary_tier1.lookup = MagicMock(
side_effect=[LookupResult.RETRY, LookupResult.HIT]
)
# Lookup finds the block in a secondary tier and defers.
# First lookup is deferred by the secondary tier.
assert self.manager.lookup(block, ctx) is LookupResult.RETRY
self._simulate_on_schedule_end()
stats = self.manager.get_stats()
if stats is not None:
assert f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count" not in (
stats.reduce()
)
assert f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count" not in (
stats.reduce()
assert not has_histogram_count(
stats.reduce(), TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY
)
# Request finishes before the deferred lookup is ever resolved.
self.manager.on_request_finished(ctx)
assert self.manager.lookup(block, ctx) is LookupResult.RETRY
stats = self.manager.get_stats()
assert stats is not None
reduced = stats.reduce()
assert reduced[f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count"] == 1
assert reduced[f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count"] == 1
assert (
reduced[
histogram_count_key(
TieringOffloadingMetrics.LOOKUP_SYNC_DELAY, ("1:example",)
)
]
== 1
)
assert (
reduced[
histogram_count_key(
TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY, ("1:example",)
)
]
== 1
)
def test_partial_lookup(self, manager_setup):
"""Test lookup with partial hits."""
@@ -938,9 +1028,7 @@ class TestTieringOffloadingManager:
ctx = ReqContext(req_id="req_policy_lifecycle_2")
result = self.manager.on_new_request(ctx)
assert result.policy == OffloadPolicy.REQUEST_LEVEL
assert self.manager._req_state[ctx.req_id].request_level_tiers == {
self.secondary_tier1
}
assert self.manager._req_state[ctx.req_id].request_level_tiers == {0}
# Cleanup
self.manager.on_request_finished(ctx)
@@ -996,13 +1084,13 @@ class TestTieringOffloadingManager:
primary tier; pending submissions are dropped without being sent
to the secondary tier. Active request state is retained."""
# Cascade — populates primary blocks and leaves cascade jobs
# in _transfer_jobs (the synchronous example tier has already
# in _jobs (the synchronous example tier has already
# queued completions); reset_cache's drain loop will pick them up.
blocks = to_keys(range(3))
self._start_request()
self.manager.prepare_store(blocks, _CTX)
self.manager.complete_store(blocks, _CTX, success=True)
assert self.manager._transfer_jobs
assert self.manager._jobs
# Pending promotion submission (deferred — no on_schedule_end after
# the lookup that staged it).
@@ -1020,9 +1108,7 @@ class TestTieringOffloadingManager:
)
rl_ctx = ReqContext(req_id="rl")
self.manager.on_new_request(rl_ctx)
assert self.manager._req_state[rl_ctx.req_id].request_level_tiers == {
self.secondary_tier1
}
assert self.manager._req_state[rl_ctx.req_id].request_level_tiers == {0}
# Mark this step as already polled (reset_cache must clear it).
self.manager._processed_jobs_this_step = True
@@ -1035,7 +1121,7 @@ class TestTieringOffloadingManager:
self.manager.reset_cache()
# Orchestrator state cleared.
assert self.manager._transfer_jobs == {}
assert self.manager._jobs == {}
assert self.manager._pending_load_submissions == {}
assert set(self.manager._req_state) == {_CTX.req_id, rl_ctx.req_id}
assert self.manager._processed_jobs_this_step is False
@@ -1063,18 +1149,18 @@ class TestTieringOffloadingManager:
wraps=self.secondary_tier2.drain_jobs
)
# Drive a cascade so a job lands in _transfer_jobs.
# Drive a cascade so a job lands in _jobs.
blocks = to_keys(range(3))
self._start_request()
self.manager.prepare_store(blocks, _CTX)
self.manager.complete_store(blocks, _CTX, success=True)
assert self.manager._transfer_jobs
assert self.manager._jobs
self.manager.reset_cache()
self.secondary_tier1.drain_jobs.assert_called_once()
self.secondary_tier2.drain_jobs.assert_called_once()
assert self.manager._transfer_jobs == {}
assert self.manager._jobs == {}
@pytest.mark.parametrize(
"load_tier_filter",