Add tiering offloading metrics (#48798)
Signed-off-by: srinivas_oo7 <sklinkedin0120@gmail.com> Co-authored-by: srinivas_oo7 <sklinkedin0120@gmail.com>
This commit is contained in:
co-authored by
srinivas_oo7
parent
11ba93f364
commit
a123159f7a
@@ -25,6 +25,7 @@ from vllm.v1.kv_offload.base import (
|
||||
OffloadingHistogramMetadata,
|
||||
)
|
||||
from vllm.v1.kv_offload.factory import OffloadingSpecFactory
|
||||
from vllm.v1.kv_offload.tiering.base import TieringOffloadingMetrics
|
||||
|
||||
LOAD_BYTES = _TransferMetricName.LOAD_BYTES
|
||||
LOAD_TIME = _TransferMetricName.LOAD_TIME
|
||||
@@ -101,10 +102,15 @@ class _FakeMetric:
|
||||
|
||||
|
||||
class _FakeVllmConfig:
|
||||
def __init__(self, store_threshold: int = 2):
|
||||
self.kv_transfer_config = SimpleNamespace(
|
||||
kv_connector_extra_config={"store_threshold": store_threshold}
|
||||
)
|
||||
def __init__(
|
||||
self,
|
||||
store_threshold: int = 2,
|
||||
extra_config: dict[str, Any] | None = None,
|
||||
):
|
||||
config = {"store_threshold": store_threshold}
|
||||
if extra_config is not None:
|
||||
config.update(extra_config)
|
||||
self.kv_transfer_config = SimpleNamespace(kv_connector_extra_config=config)
|
||||
|
||||
|
||||
def _spec_cls_with_metric_definitions(
|
||||
@@ -679,6 +685,30 @@ def test_prom_metrics_uses_configured_manager_metrics():
|
||||
assert STORES_SKIPPED not in prom_metrics._offloading_metric_metadata
|
||||
|
||||
|
||||
def test_prom_metrics_registers_tiering_metrics_from_spec():
|
||||
prom_metrics = OffloadPromMetrics(
|
||||
vllm_config=_FakeVllmConfig(
|
||||
store_threshold=0,
|
||||
extra_config={
|
||||
"spec_name": "TieringOffloadingSpec",
|
||||
"secondary_tiers": [],
|
||||
},
|
||||
), # type: ignore[arg-type]
|
||||
metric_types={
|
||||
Gauge: _FakeMetric,
|
||||
Counter: _FakeMetric,
|
||||
Histogram: _FakeMetric,
|
||||
},
|
||||
labelnames=["model_name", "engine"],
|
||||
per_engine_labelvalues={0: ["model", "0"]},
|
||||
)
|
||||
|
||||
metric = prom_metrics._offloading_metric_defs[
|
||||
TieringOffloadingMetrics.BLOCK_QUERIES
|
||||
]
|
||||
assert metric.kwargs["labelnames"] == ["model_name", "engine", "tier"]
|
||||
|
||||
|
||||
def test_aggregate_into_empty_stats():
|
||||
"""Aggregating non-empty stats into a fresh (empty) stats object works."""
|
||||
empty = OffloadingConnectorStats()
|
||||
|
||||
@@ -16,7 +16,7 @@ import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.v1.kv_offload.base import LookupResult, ReqContext, ScheduleEndContext
|
||||
from vllm.v1.kv_offload.tiering.base import JobMetadata, JobResult
|
||||
from vllm.v1.kv_offload.tiering.base import JobResult, TransferJob
|
||||
from vllm.v1.kv_offload.tiering.p2p import manager as manager_module
|
||||
from vllm.v1.kv_offload.tiering.p2p.manager import (
|
||||
_UNBOUND_STORE_TIMEOUT_S,
|
||||
@@ -86,12 +86,12 @@ def _job_metadata(
|
||||
keys: list[bytes] | None = None,
|
||||
block_ids: list[int] | None = None,
|
||||
kv_params: dict | None = None,
|
||||
) -> JobMetadata:
|
||||
) -> TransferJob:
|
||||
if keys is None:
|
||||
keys = [b"key1"]
|
||||
if block_ids is None:
|
||||
block_ids = list(range(len(keys)))
|
||||
return JobMetadata(
|
||||
return TransferJob(
|
||||
job_id=job_id,
|
||||
keys=keys,
|
||||
block_ids=np.array(block_ids),
|
||||
|
||||
@@ -24,7 +24,7 @@ from vllm.v1.kv_offload.base import (
|
||||
ReqContext,
|
||||
RequestOffloadingContext,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.base import JobMetadata
|
||||
from vllm.v1.kv_offload.tiering.base import TransferJob
|
||||
from vllm.v1.kv_offload.tiering.p2p.session import (
|
||||
LoadResult,
|
||||
P2PSession,
|
||||
@@ -256,13 +256,13 @@ class FakeParent:
|
||||
self,
|
||||
keys: Sequence[OffloadKey],
|
||||
ctx: ReqContext,
|
||||
) -> JobMetadata:
|
||||
) -> TransferJob:
|
||||
keys_list = list(keys)
|
||||
self.calls.append(("create_store_job", tuple(keys_list), ctx.req_id))
|
||||
block_ids = np.array([self.stored[k] for k in keys_list], dtype=np.int32)
|
||||
job_id = self._next_job_id
|
||||
self._next_job_id += 1
|
||||
return JobMetadata(
|
||||
return TransferJob(
|
||||
job_id=job_id,
|
||||
keys=keys_list,
|
||||
block_ids=block_ids,
|
||||
|
||||
@@ -35,7 +35,7 @@ from vllm.v1.kv_offload.config import (
|
||||
OffloadingModelConfig,
|
||||
OffloadingParallelConfig,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.base import JobMetadata
|
||||
from vllm.v1.kv_offload.tiering.base import TransferJob
|
||||
from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory
|
||||
from vllm.v1.kv_offload.tiering.fs.manager import (
|
||||
FileSystemTierManager,
|
||||
@@ -105,10 +105,10 @@ def make_job(
|
||||
keys: list[OffloadKey],
|
||||
block_ids: list[int] | None = None,
|
||||
is_promotion: bool = False,
|
||||
) -> JobMetadata:
|
||||
) -> TransferJob:
|
||||
if block_ids is None:
|
||||
block_ids = list(range(len(keys)))
|
||||
return JobMetadata(
|
||||
return TransferJob(
|
||||
job_id=job_id,
|
||||
keys=keys,
|
||||
block_ids=np.array(block_ids, dtype=np.int64),
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.v1.kv_offload.base import LookupResult, ReqContext, make_offload_key
|
||||
from vllm.v1.kv_offload.tiering.base import (
|
||||
JobResult,
|
||||
TieringOffloadingMetrics,
|
||||
TransferJob,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.manager import JobMetadata
|
||||
from vllm.v1.kv_offload.tiering.metrics import TieringMetricsTracker
|
||||
|
||||
_CTX = ReqContext(req_id="test")
|
||||
|
||||
|
||||
def to_keys(int_ids: Iterable[int]):
|
||||
return [make_offload_key(str(i).encode(), 0) for i in int_ids]
|
||||
|
||||
|
||||
def test_tiering_metrics_tracker_records_lookup_metrics():
|
||||
tracker = TieringMetricsTracker(
|
||||
tier_types=["fs", "p2p"],
|
||||
num_primary_blocks=5,
|
||||
primary_block_size=16,
|
||||
)
|
||||
tracker.on_new_request(_CTX)
|
||||
|
||||
key = to_keys([1])[0]
|
||||
tracker.on_lookup(
|
||||
_CTX,
|
||||
key,
|
||||
tracker.primary_tier_label,
|
||||
LookupResult.MISS,
|
||||
lookup_duration=0.0,
|
||||
)
|
||||
tracker.on_lookup(_CTX, key, tracker.tier_label(0), LookupResult.MISS, 0.0)
|
||||
tracker.on_lookup(_CTX, key, tracker.tier_label(1), LookupResult.HIT, 0.0)
|
||||
tracker.on_lookup(_CTX, key, tracker.tier_label(1), LookupResult.HIT, 0.0)
|
||||
|
||||
stats = tracker.take_stats()
|
||||
assert stats is not None
|
||||
values = stats.data["data"]
|
||||
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("0:primary",)] == 1
|
||||
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("1:fs",)] == 1
|
||||
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("2:p2p",)] == 1
|
||||
assert values[TieringOffloadingMetrics.BLOCK_HITS][("2:p2p",)] == 1
|
||||
assert ("1:fs",) not in values[TieringOffloadingMetrics.BLOCK_HITS]
|
||||
|
||||
|
||||
def test_tiering_metrics_tracker_stops_lookup_metrics_after_allocation():
|
||||
tracker = TieringMetricsTracker(
|
||||
tier_types=["fs"],
|
||||
num_primary_blocks=5,
|
||||
primary_block_size=16,
|
||||
)
|
||||
tracker.on_new_request(_CTX)
|
||||
key = to_keys([1])[0]
|
||||
|
||||
tracker.on_lookup(
|
||||
_CTX,
|
||||
key,
|
||||
tracker.primary_tier_label,
|
||||
LookupResult.MISS,
|
||||
lookup_duration=0.0,
|
||||
)
|
||||
tracker.on_lookup(_CTX, key, tracker.tier_label(0), LookupResult.HIT, 0.0)
|
||||
|
||||
stats = tracker.take_stats()
|
||||
assert stats is not None
|
||||
values = stats.data["data"]
|
||||
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("0:primary",)] == 1
|
||||
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("1:fs",)] == 1
|
||||
assert values[TieringOffloadingMetrics.BLOCK_HITS][("1:fs",)] == 1
|
||||
|
||||
tracker.on_request_allocated(_CTX)
|
||||
tracker.on_lookup(
|
||||
_CTX,
|
||||
key,
|
||||
tracker.primary_tier_label,
|
||||
LookupResult.MISS,
|
||||
lookup_duration=0.0,
|
||||
)
|
||||
tracker.on_lookup(_CTX, key, tracker.tier_label(0), LookupResult.HIT, 0.0)
|
||||
|
||||
stats = tracker.take_stats()
|
||||
assert stats is not None
|
||||
values = stats.data["data"]
|
||||
assert TieringOffloadingMetrics.BLOCK_QUERIES not in values
|
||||
assert TieringOffloadingMetrics.BLOCK_HITS not in values
|
||||
|
||||
|
||||
def test_tiering_metrics_tracker_records_finished_job_metrics():
|
||||
tracker = TieringMetricsTracker(
|
||||
tier_types=["fs"],
|
||||
num_primary_blocks=5,
|
||||
primary_block_size=16,
|
||||
)
|
||||
cascade_key, promotion_key, failed_cascade_key, failed_promotion_key = to_keys(
|
||||
range(4)
|
||||
)
|
||||
jobs = [
|
||||
JobMetadata(
|
||||
TransferJob(0, [cascade_key], np.array([0]), False, _CTX),
|
||||
0,
|
||||
),
|
||||
JobMetadata(
|
||||
TransferJob(1, [promotion_key], np.array([1]), True, _CTX),
|
||||
0,
|
||||
),
|
||||
JobMetadata(
|
||||
TransferJob(2, [failed_cascade_key], np.array([2]), False, _CTX),
|
||||
0,
|
||||
),
|
||||
JobMetadata(
|
||||
TransferJob(3, [failed_promotion_key], np.array([3]), True, _CTX),
|
||||
0,
|
||||
),
|
||||
]
|
||||
for job in jobs:
|
||||
tracker.on_job_registered(job)
|
||||
|
||||
tracker.on_job_finished(
|
||||
jobs[0], JobResult(job_id=0, success=True, transfer_time=0.5)
|
||||
)
|
||||
tracker.on_job_finished(
|
||||
jobs[1], JobResult(job_id=1, success=True, transfer_time=0.25)
|
||||
)
|
||||
tracker.on_job_finished(jobs[2], JobResult(job_id=2, success=False))
|
||||
tracker.on_job_finished(jobs[3], JobResult(job_id=3, success=False))
|
||||
|
||||
stats = tracker.take_stats()
|
||||
assert stats is not None
|
||||
values = stats.data["data"]
|
||||
label = ("1:fs",)
|
||||
assert values[TieringOffloadingMetrics.WRITE_BYTES][label] == 16
|
||||
assert values[TieringOffloadingMetrics.WRITE_TIME][label] == 0.5
|
||||
assert values[TieringOffloadingMetrics.READ_BYTES][label] == 16
|
||||
assert values[TieringOffloadingMetrics.READ_TIME][label] == 0.25
|
||||
assert values[TieringOffloadingMetrics.CASCADE_JOB_FAILURES][label] == 1
|
||||
assert values[TieringOffloadingMetrics.PROMOTION_JOB_FAILURES][label] == 1
|
||||
tracker.assert_idle()
|
||||
|
||||
|
||||
def test_tiering_metrics_tracker_records_partial_promotion_success_bytes():
|
||||
tracker = TieringMetricsTracker(
|
||||
tier_types=["fs"],
|
||||
num_primary_blocks=5,
|
||||
primary_block_size=16,
|
||||
)
|
||||
keys = to_keys(range(3))
|
||||
job = JobMetadata(
|
||||
TransferJob(0, keys, np.array([0, 1, 2]), True, _CTX),
|
||||
0,
|
||||
)
|
||||
tracker.on_job_registered(job)
|
||||
|
||||
tracker.on_job_finished(
|
||||
job,
|
||||
JobResult(
|
||||
job_id=0,
|
||||
success=False,
|
||||
successful_keys=(keys[0], keys[2]),
|
||||
transfer_time=0.5,
|
||||
),
|
||||
)
|
||||
|
||||
stats = tracker.take_stats()
|
||||
assert stats is not None
|
||||
values = stats.data["data"]
|
||||
label = ("1:fs",)
|
||||
assert values[TieringOffloadingMetrics.READ_BYTES][label] == 32
|
||||
assert values[TieringOffloadingMetrics.READ_TIME][label] == 0.5
|
||||
assert values[TieringOffloadingMetrics.PROMOTION_JOB_FAILURES][label] == 1
|
||||
tracker.assert_idle()
|
||||
|
||||
|
||||
def test_tiering_metrics_tracker_reports_active_job_and_primary_usage_gauges():
|
||||
tracker = TieringMetricsTracker(
|
||||
tier_types=["fs", "p2p"],
|
||||
num_primary_blocks=6,
|
||||
primary_block_size=16,
|
||||
)
|
||||
fs_job = JobMetadata(
|
||||
TransferJob(0, to_keys([0, 1]), np.array([0, 1]), False, _CTX),
|
||||
0,
|
||||
)
|
||||
p2p_job = JobMetadata(
|
||||
TransferJob(1, to_keys([2, 3, 4]), np.array([2, 3, 4]), True, _CTX),
|
||||
1,
|
||||
)
|
||||
tracker.on_job_registered(fs_job)
|
||||
tracker.on_job_registered(p2p_job)
|
||||
|
||||
stats = tracker.take_stats()
|
||||
assert stats is not None
|
||||
values = stats.data["data"]
|
||||
fs_label = ("1:fs",)
|
||||
p2p_label = ("2:p2p",)
|
||||
assert values[TieringOffloadingMetrics.PRIMARY_READ_USAGE_PERC][
|
||||
fs_label
|
||||
] == pytest.approx(2 / 6)
|
||||
assert values[TieringOffloadingMetrics.ACTIVE_CASCADE_JOBS][fs_label] == 1
|
||||
assert values[TieringOffloadingMetrics.PRIMARY_WRITE_USAGE_PERC][
|
||||
p2p_label
|
||||
] == pytest.approx(3 / 6)
|
||||
assert values[TieringOffloadingMetrics.ACTIVE_PROMOTION_JOBS][p2p_label] == 1
|
||||
|
||||
|
||||
def test_tiering_metrics_tracker_records_promotion_allocation_failures():
|
||||
tracker = TieringMetricsTracker(
|
||||
tier_types=["fs"],
|
||||
num_primary_blocks=1,
|
||||
primary_block_size=16,
|
||||
)
|
||||
|
||||
tracker.on_promotion_allocation_failure()
|
||||
|
||||
stats = tracker.take_stats()
|
||||
assert stats is not None
|
||||
values = stats.data["data"]
|
||||
assert values[TieringOffloadingMetrics.PROMOTION_ALLOCATION_FAILURES][()] == 1
|
||||
@@ -34,7 +34,7 @@ from vllm.v1.kv_offload.config import (
|
||||
OffloadingModelConfig,
|
||||
OffloadingParallelConfig,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.base import JobMetadata, JobResult
|
||||
from vllm.v1.kv_offload.tiering.base import JobResult, TransferJob
|
||||
from vllm.v1.kv_offload.tiering.manager import (
|
||||
CPUPrimaryTierOffloadingManager,
|
||||
TieringOffloadingManager,
|
||||
@@ -105,10 +105,10 @@ def make_job(
|
||||
job_id: int,
|
||||
keys: list[OffloadKey],
|
||||
block_ids: list[int] | None = None,
|
||||
) -> JobMetadata:
|
||||
) -> TransferJob:
|
||||
if block_ids is None:
|
||||
block_ids = list(range(len(keys)))
|
||||
return JobMetadata(
|
||||
return TransferJob(
|
||||
job_id=job_id,
|
||||
keys=keys,
|
||||
block_ids=np.array(block_ids, dtype=np.int64),
|
||||
@@ -201,6 +201,9 @@ class MockNixlAgent:
|
||||
def release_dlist_handle(self, handle):
|
||||
pass
|
||||
|
||||
def get_xfer_telemetry(self, handle):
|
||||
return SimpleNamespace(xferDuration=1000)
|
||||
|
||||
def _query_memory(self, queries, mem_type, agent_name):
|
||||
return [object() if q[3] in self._stored_obj_keys else None for q in queries]
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ from vllm.v1.kv_offload.base import (
|
||||
Medium,
|
||||
OffloadingCounterMetadata,
|
||||
OffloadingEvent,
|
||||
OffloadingGaugeMetadata,
|
||||
OffloadKey,
|
||||
OffloadPolicy,
|
||||
ReqContext,
|
||||
@@ -39,10 +40,10 @@ from vllm.v1.kv_offload.base import (
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.base import (
|
||||
JobMetadata,
|
||||
JobResult,
|
||||
SecondaryTierManager,
|
||||
TieringOffloadingMetrics,
|
||||
TransferJob,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.example.manager import ExampleSecondaryTierManager
|
||||
from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory
|
||||
@@ -85,6 +86,16 @@ def count_hits(manager, keys: list[OffloadKey]) -> int | None:
|
||||
return count
|
||||
|
||||
|
||||
def histogram_count_key(metric_name: str, labelvalues: tuple[str, ...]) -> str:
|
||||
return f"{metric_name}:{labelvalues}_count"
|
||||
|
||||
|
||||
def has_histogram_count(reduced: dict[str, int | float], metric_name: str) -> bool:
|
||||
return any(
|
||||
key.startswith(f"{metric_name}:") and key.endswith("_count") for key in reduced
|
||||
)
|
||||
|
||||
|
||||
class MetricsSecondaryTierManager(SecondaryTierManager):
|
||||
"""Test-only secondary tier that declares and emits one labeled metric."""
|
||||
|
||||
@@ -102,18 +113,26 @@ class MetricsSecondaryTierManager(SecondaryTierManager):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.stats: OffloadingConnectorStats | None = None
|
||||
self.lookup_result = LookupResult.MISS
|
||||
self.finished_jobs: list[JobResult] = []
|
||||
self.submitted_loads: list[TransferJob] = []
|
||||
self.submitted_stores: list[TransferJob] = []
|
||||
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
|
||||
return False
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
||||
return self.lookup_result
|
||||
|
||||
def submit_store(self, job_metadata: JobMetadata) -> None:
|
||||
def submit_store(self, job_metadata: TransferJob) -> None:
|
||||
self.submitted_stores.append(job_metadata)
|
||||
return
|
||||
|
||||
def submit_load(self, job_metadata: JobMetadata) -> None:
|
||||
def submit_load(self, job_metadata: TransferJob) -> None:
|
||||
self.submitted_loads.append(job_metadata)
|
||||
return
|
||||
|
||||
def get_finished_jobs(self) -> Iterable[JobResult]:
|
||||
return ()
|
||||
results = self.finished_jobs
|
||||
self.finished_jobs = []
|
||||
return results
|
||||
|
||||
def drain_jobs(self) -> None:
|
||||
return
|
||||
@@ -141,6 +160,24 @@ def test_tiering_spec_collects_secondary_metric_definitions(monkeypatch):
|
||||
metadata = metrics[MetricsSecondaryTierManager.MY_TIER_METRIC]
|
||||
assert metadata.documentation == "Number of bytes served by the test tier."
|
||||
assert metadata.labelnames == ("tier",)
|
||||
assert metrics[TieringOffloadingMetrics.BLOCK_QUERIES].labelnames == ("tier",)
|
||||
assert metrics[TieringOffloadingMetrics.LOOKUP_SYNC_DELAY].labelnames == ("tier",)
|
||||
assert metrics[TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY].labelnames == ("tier",)
|
||||
assert metrics[TieringOffloadingMetrics.PROMOTION_JOB_FAILURES].labelnames == (
|
||||
"tier",
|
||||
)
|
||||
assert isinstance(
|
||||
metrics[TieringOffloadingMetrics.PROMOTION_ALLOCATION_FAILURES],
|
||||
OffloadingCounterMetadata,
|
||||
)
|
||||
for metric_name in (
|
||||
TieringOffloadingMetrics.PRIMARY_WRITE_USAGE_PERC,
|
||||
TieringOffloadingMetrics.PRIMARY_READ_USAGE_PERC,
|
||||
TieringOffloadingMetrics.ACTIVE_PROMOTION_JOBS,
|
||||
TieringOffloadingMetrics.ACTIVE_CASCADE_JOBS,
|
||||
):
|
||||
assert isinstance(metrics[metric_name], OffloadingGaugeMetadata)
|
||||
assert metrics[metric_name].labelnames == ("tier",)
|
||||
|
||||
|
||||
def test_tiering_manager_aggregates_secondary_stats():
|
||||
@@ -261,12 +298,15 @@ class TestTieringOffloadingManager:
|
||||
self._start_request()
|
||||
# Register an in-flight promotion job for tier1 by hand.
|
||||
job_id = self.manager._next_job_id()
|
||||
self.manager._transfer_jobs[job_id] = JobMetadata(
|
||||
job_id=job_id,
|
||||
keys=to_keys([1, 2]),
|
||||
block_ids=[0, 1],
|
||||
is_promotion=True,
|
||||
req_context=_CTX,
|
||||
self.manager._register_job(
|
||||
TransferJob(
|
||||
job_id=job_id,
|
||||
keys=to_keys([1, 2]),
|
||||
block_ids=[0, 1],
|
||||
is_promotion=True,
|
||||
req_context=_CTX,
|
||||
),
|
||||
0,
|
||||
)
|
||||
failed = JobResult(job_id=job_id, success=False)
|
||||
with (
|
||||
@@ -285,12 +325,15 @@ class TestTieringOffloadingManager:
|
||||
|
||||
self._start_request()
|
||||
job_id = self.manager._next_job_id()
|
||||
self.manager._transfer_jobs[job_id] = JobMetadata(
|
||||
job_id=job_id,
|
||||
keys=to_keys([1]),
|
||||
block_ids=[0],
|
||||
is_promotion=True,
|
||||
req_context=_CTX,
|
||||
self.manager._register_job(
|
||||
TransferJob(
|
||||
job_id=job_id,
|
||||
keys=to_keys([1]),
|
||||
block_ids=[0],
|
||||
is_promotion=True,
|
||||
req_context=_CTX,
|
||||
),
|
||||
0,
|
||||
)
|
||||
ok = JobResult(job_id=job_id, success=True)
|
||||
with (
|
||||
@@ -473,7 +516,7 @@ class TestTieringOffloadingManager:
|
||||
for block in blocks:
|
||||
self.secondary_tier1.blocks[block] = True
|
||||
|
||||
def submit_partial(job_metadata: JobMetadata) -> None:
|
||||
def submit_partial(job_metadata: TransferJob) -> None:
|
||||
successful_keys = (
|
||||
None
|
||||
if successful_indices is None
|
||||
@@ -500,7 +543,7 @@ class TestTieringOffloadingManager:
|
||||
] == expected_results
|
||||
|
||||
def test_lookup_reports_sync_delay_for_resolved_lookups(self, manager_setup):
|
||||
"""Resolved lookups report one sync delay sample on allocation."""
|
||||
"""Resolved lookups report one sync delay sample per tier and block."""
|
||||
self._start_request()
|
||||
blocks = to_keys(range(2))
|
||||
|
||||
@@ -508,22 +551,39 @@ class TestTieringOffloadingManager:
|
||||
for block in blocks:
|
||||
assert self.manager.lookup(block, _CTX) is LookupResult.MISS
|
||||
|
||||
stats = self.manager.get_stats()
|
||||
if stats is not None:
|
||||
assert f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count" not in (
|
||||
stats.reduce()
|
||||
)
|
||||
|
||||
self._simulate_on_schedule_end(new_req_ids=[_CTX.req_id])
|
||||
|
||||
stats = self.manager.get_stats()
|
||||
assert stats is not None
|
||||
reduced = stats.reduce()
|
||||
assert reduced[f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count"] == 1
|
||||
assert f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count" not in reduced
|
||||
assert (
|
||||
reduced[
|
||||
histogram_count_key(
|
||||
TieringOffloadingMetrics.LOOKUP_SYNC_DELAY, ("0:primary",)
|
||||
)
|
||||
]
|
||||
== 2
|
||||
)
|
||||
assert (
|
||||
reduced[
|
||||
histogram_count_key(
|
||||
TieringOffloadingMetrics.LOOKUP_SYNC_DELAY, ("1:example",)
|
||||
)
|
||||
]
|
||||
== 2
|
||||
)
|
||||
assert (
|
||||
reduced[
|
||||
histogram_count_key(
|
||||
TieringOffloadingMetrics.LOOKUP_SYNC_DELAY, ("2:example",)
|
||||
)
|
||||
]
|
||||
== 2
|
||||
)
|
||||
assert not has_histogram_count(
|
||||
reduced, TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY
|
||||
)
|
||||
|
||||
def test_lookup_reports_async_delay_across_promotion(self, manager_setup):
|
||||
"""A new request reports async delay at schedule end."""
|
||||
def test_lookup_does_not_report_async_delay_for_promotion(self, manager_setup):
|
||||
"""Promotion time is not included in lookup async delay."""
|
||||
self._start_request()
|
||||
block = to_keys(range(1))[0]
|
||||
self.secondary_tier1.blocks[block] = True
|
||||
@@ -531,13 +591,30 @@ class TestTieringOffloadingManager:
|
||||
# First lookup finds the block in a secondary tier and defers.
|
||||
assert self.manager.lookup(block, _CTX) is LookupResult.RETRY
|
||||
|
||||
# The first scheduler step reports async delay for the new request.
|
||||
# Promotion is not treated as unresolved secondary lookup time.
|
||||
self._simulate_on_schedule_end(new_req_ids=[_CTX.req_id])
|
||||
stats = self.manager.get_stats()
|
||||
assert stats is not None
|
||||
reduced = stats.reduce()
|
||||
assert reduced[f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count"] == 1
|
||||
assert reduced[f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count"] == 1
|
||||
assert (
|
||||
reduced[
|
||||
histogram_count_key(
|
||||
TieringOffloadingMetrics.LOOKUP_SYNC_DELAY, ("0:primary",)
|
||||
)
|
||||
]
|
||||
== 1
|
||||
)
|
||||
assert (
|
||||
reduced[
|
||||
histogram_count_key(
|
||||
TieringOffloadingMetrics.LOOKUP_SYNC_DELAY, ("1:example",)
|
||||
)
|
||||
]
|
||||
== 1
|
||||
)
|
||||
assert not has_histogram_count(
|
||||
reduced, TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY
|
||||
)
|
||||
|
||||
# Promotion completes on the next scheduler step.
|
||||
self._simulate_on_schedule_end()
|
||||
@@ -547,38 +624,51 @@ class TestTieringOffloadingManager:
|
||||
|
||||
stats = self.manager.get_stats()
|
||||
if stats is not None:
|
||||
assert f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count" not in (
|
||||
stats.reduce()
|
||||
assert not has_histogram_count(
|
||||
stats.reduce(), TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY
|
||||
)
|
||||
|
||||
def test_lookup_reports_async_delay_on_request_finish(self, manager_setup):
|
||||
"""Never-allocated lookup delays flush at teardown."""
|
||||
def test_lookup_reports_async_delay_when_deferred_lookup_resolves(
|
||||
self, manager_setup
|
||||
):
|
||||
"""Async delay is observed when a RETRY lookup later resolves."""
|
||||
ctx = ReqContext(req_id="req_lookup_finish")
|
||||
self._start_request(ctx)
|
||||
block = to_keys(range(1))[0]
|
||||
self.secondary_tier1.blocks[block] = True
|
||||
self.secondary_tier1.lookup = MagicMock(
|
||||
side_effect=[LookupResult.RETRY, LookupResult.HIT]
|
||||
)
|
||||
|
||||
# Lookup finds the block in a secondary tier and defers.
|
||||
# First lookup is deferred by the secondary tier.
|
||||
assert self.manager.lookup(block, ctx) is LookupResult.RETRY
|
||||
|
||||
self._simulate_on_schedule_end()
|
||||
stats = self.manager.get_stats()
|
||||
if stats is not None:
|
||||
assert f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count" not in (
|
||||
stats.reduce()
|
||||
)
|
||||
assert f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count" not in (
|
||||
stats.reduce()
|
||||
assert not has_histogram_count(
|
||||
stats.reduce(), TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY
|
||||
)
|
||||
|
||||
# Request finishes before the deferred lookup is ever resolved.
|
||||
self.manager.on_request_finished(ctx)
|
||||
assert self.manager.lookup(block, ctx) is LookupResult.RETRY
|
||||
|
||||
stats = self.manager.get_stats()
|
||||
assert stats is not None
|
||||
reduced = stats.reduce()
|
||||
assert reduced[f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count"] == 1
|
||||
assert reduced[f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count"] == 1
|
||||
assert (
|
||||
reduced[
|
||||
histogram_count_key(
|
||||
TieringOffloadingMetrics.LOOKUP_SYNC_DELAY, ("1:example",)
|
||||
)
|
||||
]
|
||||
== 1
|
||||
)
|
||||
assert (
|
||||
reduced[
|
||||
histogram_count_key(
|
||||
TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY, ("1:example",)
|
||||
)
|
||||
]
|
||||
== 1
|
||||
)
|
||||
|
||||
def test_partial_lookup(self, manager_setup):
|
||||
"""Test lookup with partial hits."""
|
||||
@@ -938,9 +1028,7 @@ class TestTieringOffloadingManager:
|
||||
ctx = ReqContext(req_id="req_policy_lifecycle_2")
|
||||
result = self.manager.on_new_request(ctx)
|
||||
assert result.policy == OffloadPolicy.REQUEST_LEVEL
|
||||
assert self.manager._req_state[ctx.req_id].request_level_tiers == {
|
||||
self.secondary_tier1
|
||||
}
|
||||
assert self.manager._req_state[ctx.req_id].request_level_tiers == {0}
|
||||
|
||||
# Cleanup
|
||||
self.manager.on_request_finished(ctx)
|
||||
@@ -996,13 +1084,13 @@ class TestTieringOffloadingManager:
|
||||
primary tier; pending submissions are dropped without being sent
|
||||
to the secondary tier. Active request state is retained."""
|
||||
# Cascade — populates primary blocks and leaves cascade jobs
|
||||
# in _transfer_jobs (the synchronous example tier has already
|
||||
# in _jobs (the synchronous example tier has already
|
||||
# queued completions); reset_cache's drain loop will pick them up.
|
||||
blocks = to_keys(range(3))
|
||||
self._start_request()
|
||||
self.manager.prepare_store(blocks, _CTX)
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
assert self.manager._transfer_jobs
|
||||
assert self.manager._jobs
|
||||
|
||||
# Pending promotion submission (deferred — no on_schedule_end after
|
||||
# the lookup that staged it).
|
||||
@@ -1020,9 +1108,7 @@ class TestTieringOffloadingManager:
|
||||
)
|
||||
rl_ctx = ReqContext(req_id="rl")
|
||||
self.manager.on_new_request(rl_ctx)
|
||||
assert self.manager._req_state[rl_ctx.req_id].request_level_tiers == {
|
||||
self.secondary_tier1
|
||||
}
|
||||
assert self.manager._req_state[rl_ctx.req_id].request_level_tiers == {0}
|
||||
|
||||
# Mark this step as already polled (reset_cache must clear it).
|
||||
self.manager._processed_jobs_this_step = True
|
||||
@@ -1035,7 +1121,7 @@ class TestTieringOffloadingManager:
|
||||
self.manager.reset_cache()
|
||||
|
||||
# Orchestrator state cleared.
|
||||
assert self.manager._transfer_jobs == {}
|
||||
assert self.manager._jobs == {}
|
||||
assert self.manager._pending_load_submissions == {}
|
||||
assert set(self.manager._req_state) == {_CTX.req_id, rl_ctx.req_id}
|
||||
assert self.manager._processed_jobs_this_step is False
|
||||
@@ -1063,18 +1149,18 @@ class TestTieringOffloadingManager:
|
||||
wraps=self.secondary_tier2.drain_jobs
|
||||
)
|
||||
|
||||
# Drive a cascade so a job lands in _transfer_jobs.
|
||||
# Drive a cascade so a job lands in _jobs.
|
||||
blocks = to_keys(range(3))
|
||||
self._start_request()
|
||||
self.manager.prepare_store(blocks, _CTX)
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
assert self.manager._transfer_jobs
|
||||
assert self.manager._jobs
|
||||
|
||||
self.manager.reset_cache()
|
||||
|
||||
self.secondary_tier1.drain_jobs.assert_called_once()
|
||||
self.secondary_tier2.drain_jobs.assert_called_once()
|
||||
assert self.manager._transfer_jobs == {}
|
||||
assert self.manager._jobs == {}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"load_tier_filter",
|
||||
|
||||
Reference in New Issue
Block a user