Add tiering offloading metrics (#48798)
Signed-off-by: srinivas_oo7 <sklinkedin0120@gmail.com> Co-authored-by: srinivas_oo7 <sklinkedin0120@gmail.com>
This commit is contained in:
co-authored by
srinivas_oo7
parent
11ba93f364
commit
a123159f7a
@@ -25,6 +25,7 @@ from vllm.v1.kv_offload.base import (
|
|||||||
OffloadingHistogramMetadata,
|
OffloadingHistogramMetadata,
|
||||||
)
|
)
|
||||||
from vllm.v1.kv_offload.factory import OffloadingSpecFactory
|
from vllm.v1.kv_offload.factory import OffloadingSpecFactory
|
||||||
|
from vllm.v1.kv_offload.tiering.base import TieringOffloadingMetrics
|
||||||
|
|
||||||
LOAD_BYTES = _TransferMetricName.LOAD_BYTES
|
LOAD_BYTES = _TransferMetricName.LOAD_BYTES
|
||||||
LOAD_TIME = _TransferMetricName.LOAD_TIME
|
LOAD_TIME = _TransferMetricName.LOAD_TIME
|
||||||
@@ -101,10 +102,15 @@ class _FakeMetric:
|
|||||||
|
|
||||||
|
|
||||||
class _FakeVllmConfig:
|
class _FakeVllmConfig:
|
||||||
def __init__(self, store_threshold: int = 2):
|
def __init__(
|
||||||
self.kv_transfer_config = SimpleNamespace(
|
self,
|
||||||
kv_connector_extra_config={"store_threshold": store_threshold}
|
store_threshold: int = 2,
|
||||||
)
|
extra_config: dict[str, Any] | None = None,
|
||||||
|
):
|
||||||
|
config = {"store_threshold": store_threshold}
|
||||||
|
if extra_config is not None:
|
||||||
|
config.update(extra_config)
|
||||||
|
self.kv_transfer_config = SimpleNamespace(kv_connector_extra_config=config)
|
||||||
|
|
||||||
|
|
||||||
def _spec_cls_with_metric_definitions(
|
def _spec_cls_with_metric_definitions(
|
||||||
@@ -679,6 +685,30 @@ def test_prom_metrics_uses_configured_manager_metrics():
|
|||||||
assert STORES_SKIPPED not in prom_metrics._offloading_metric_metadata
|
assert STORES_SKIPPED not in prom_metrics._offloading_metric_metadata
|
||||||
|
|
||||||
|
|
||||||
|
def test_prom_metrics_registers_tiering_metrics_from_spec():
|
||||||
|
prom_metrics = OffloadPromMetrics(
|
||||||
|
vllm_config=_FakeVllmConfig(
|
||||||
|
store_threshold=0,
|
||||||
|
extra_config={
|
||||||
|
"spec_name": "TieringOffloadingSpec",
|
||||||
|
"secondary_tiers": [],
|
||||||
|
},
|
||||||
|
), # type: ignore[arg-type]
|
||||||
|
metric_types={
|
||||||
|
Gauge: _FakeMetric,
|
||||||
|
Counter: _FakeMetric,
|
||||||
|
Histogram: _FakeMetric,
|
||||||
|
},
|
||||||
|
labelnames=["model_name", "engine"],
|
||||||
|
per_engine_labelvalues={0: ["model", "0"]},
|
||||||
|
)
|
||||||
|
|
||||||
|
metric = prom_metrics._offloading_metric_defs[
|
||||||
|
TieringOffloadingMetrics.BLOCK_QUERIES
|
||||||
|
]
|
||||||
|
assert metric.kwargs["labelnames"] == ["model_name", "engine", "tier"]
|
||||||
|
|
||||||
|
|
||||||
def test_aggregate_into_empty_stats():
|
def test_aggregate_into_empty_stats():
|
||||||
"""Aggregating non-empty stats into a fresh (empty) stats object works."""
|
"""Aggregating non-empty stats into a fresh (empty) stats object works."""
|
||||||
empty = OffloadingConnectorStats()
|
empty = OffloadingConnectorStats()
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ import numpy as np
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.v1.kv_offload.base import LookupResult, ReqContext, ScheduleEndContext
|
from vllm.v1.kv_offload.base import LookupResult, ReqContext, ScheduleEndContext
|
||||||
from vllm.v1.kv_offload.tiering.base import JobMetadata, JobResult
|
from vllm.v1.kv_offload.tiering.base import JobResult, TransferJob
|
||||||
from vllm.v1.kv_offload.tiering.p2p import manager as manager_module
|
from vllm.v1.kv_offload.tiering.p2p import manager as manager_module
|
||||||
from vllm.v1.kv_offload.tiering.p2p.manager import (
|
from vllm.v1.kv_offload.tiering.p2p.manager import (
|
||||||
_UNBOUND_STORE_TIMEOUT_S,
|
_UNBOUND_STORE_TIMEOUT_S,
|
||||||
@@ -86,12 +86,12 @@ def _job_metadata(
|
|||||||
keys: list[bytes] | None = None,
|
keys: list[bytes] | None = None,
|
||||||
block_ids: list[int] | None = None,
|
block_ids: list[int] | None = None,
|
||||||
kv_params: dict | None = None,
|
kv_params: dict | None = None,
|
||||||
) -> JobMetadata:
|
) -> TransferJob:
|
||||||
if keys is None:
|
if keys is None:
|
||||||
keys = [b"key1"]
|
keys = [b"key1"]
|
||||||
if block_ids is None:
|
if block_ids is None:
|
||||||
block_ids = list(range(len(keys)))
|
block_ids = list(range(len(keys)))
|
||||||
return JobMetadata(
|
return TransferJob(
|
||||||
job_id=job_id,
|
job_id=job_id,
|
||||||
keys=keys,
|
keys=keys,
|
||||||
block_ids=np.array(block_ids),
|
block_ids=np.array(block_ids),
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ from vllm.v1.kv_offload.base import (
|
|||||||
ReqContext,
|
ReqContext,
|
||||||
RequestOffloadingContext,
|
RequestOffloadingContext,
|
||||||
)
|
)
|
||||||
from vllm.v1.kv_offload.tiering.base import JobMetadata
|
from vllm.v1.kv_offload.tiering.base import TransferJob
|
||||||
from vllm.v1.kv_offload.tiering.p2p.session import (
|
from vllm.v1.kv_offload.tiering.p2p.session import (
|
||||||
LoadResult,
|
LoadResult,
|
||||||
P2PSession,
|
P2PSession,
|
||||||
@@ -256,13 +256,13 @@ class FakeParent:
|
|||||||
self,
|
self,
|
||||||
keys: Sequence[OffloadKey],
|
keys: Sequence[OffloadKey],
|
||||||
ctx: ReqContext,
|
ctx: ReqContext,
|
||||||
) -> JobMetadata:
|
) -> TransferJob:
|
||||||
keys_list = list(keys)
|
keys_list = list(keys)
|
||||||
self.calls.append(("create_store_job", tuple(keys_list), ctx.req_id))
|
self.calls.append(("create_store_job", tuple(keys_list), ctx.req_id))
|
||||||
block_ids = np.array([self.stored[k] for k in keys_list], dtype=np.int32)
|
block_ids = np.array([self.stored[k] for k in keys_list], dtype=np.int32)
|
||||||
job_id = self._next_job_id
|
job_id = self._next_job_id
|
||||||
self._next_job_id += 1
|
self._next_job_id += 1
|
||||||
return JobMetadata(
|
return TransferJob(
|
||||||
job_id=job_id,
|
job_id=job_id,
|
||||||
keys=keys_list,
|
keys=keys_list,
|
||||||
block_ids=block_ids,
|
block_ids=block_ids,
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ from vllm.v1.kv_offload.config import (
|
|||||||
OffloadingModelConfig,
|
OffloadingModelConfig,
|
||||||
OffloadingParallelConfig,
|
OffloadingParallelConfig,
|
||||||
)
|
)
|
||||||
from vllm.v1.kv_offload.tiering.base import JobMetadata
|
from vllm.v1.kv_offload.tiering.base import TransferJob
|
||||||
from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory
|
from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory
|
||||||
from vllm.v1.kv_offload.tiering.fs.manager import (
|
from vllm.v1.kv_offload.tiering.fs.manager import (
|
||||||
FileSystemTierManager,
|
FileSystemTierManager,
|
||||||
@@ -105,10 +105,10 @@ def make_job(
|
|||||||
keys: list[OffloadKey],
|
keys: list[OffloadKey],
|
||||||
block_ids: list[int] | None = None,
|
block_ids: list[int] | None = None,
|
||||||
is_promotion: bool = False,
|
is_promotion: bool = False,
|
||||||
) -> JobMetadata:
|
) -> TransferJob:
|
||||||
if block_ids is None:
|
if block_ids is None:
|
||||||
block_ids = list(range(len(keys)))
|
block_ids = list(range(len(keys)))
|
||||||
return JobMetadata(
|
return TransferJob(
|
||||||
job_id=job_id,
|
job_id=job_id,
|
||||||
keys=keys,
|
keys=keys,
|
||||||
block_ids=np.array(block_ids, dtype=np.int64),
|
block_ids=np.array(block_ids, dtype=np.int64),
|
||||||
|
|||||||
@@ -0,0 +1,226 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
from collections.abc import Iterable
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from vllm.v1.kv_offload.base import LookupResult, ReqContext, make_offload_key
|
||||||
|
from vllm.v1.kv_offload.tiering.base import (
|
||||||
|
JobResult,
|
||||||
|
TieringOffloadingMetrics,
|
||||||
|
TransferJob,
|
||||||
|
)
|
||||||
|
from vllm.v1.kv_offload.tiering.manager import JobMetadata
|
||||||
|
from vllm.v1.kv_offload.tiering.metrics import TieringMetricsTracker
|
||||||
|
|
||||||
|
_CTX = ReqContext(req_id="test")
|
||||||
|
|
||||||
|
|
||||||
|
def to_keys(int_ids: Iterable[int]):
|
||||||
|
return [make_offload_key(str(i).encode(), 0) for i in int_ids]
|
||||||
|
|
||||||
|
|
||||||
|
def test_tiering_metrics_tracker_records_lookup_metrics():
|
||||||
|
tracker = TieringMetricsTracker(
|
||||||
|
tier_types=["fs", "p2p"],
|
||||||
|
num_primary_blocks=5,
|
||||||
|
primary_block_size=16,
|
||||||
|
)
|
||||||
|
tracker.on_new_request(_CTX)
|
||||||
|
|
||||||
|
key = to_keys([1])[0]
|
||||||
|
tracker.on_lookup(
|
||||||
|
_CTX,
|
||||||
|
key,
|
||||||
|
tracker.primary_tier_label,
|
||||||
|
LookupResult.MISS,
|
||||||
|
lookup_duration=0.0,
|
||||||
|
)
|
||||||
|
tracker.on_lookup(_CTX, key, tracker.tier_label(0), LookupResult.MISS, 0.0)
|
||||||
|
tracker.on_lookup(_CTX, key, tracker.tier_label(1), LookupResult.HIT, 0.0)
|
||||||
|
tracker.on_lookup(_CTX, key, tracker.tier_label(1), LookupResult.HIT, 0.0)
|
||||||
|
|
||||||
|
stats = tracker.take_stats()
|
||||||
|
assert stats is not None
|
||||||
|
values = stats.data["data"]
|
||||||
|
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("0:primary",)] == 1
|
||||||
|
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("1:fs",)] == 1
|
||||||
|
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("2:p2p",)] == 1
|
||||||
|
assert values[TieringOffloadingMetrics.BLOCK_HITS][("2:p2p",)] == 1
|
||||||
|
assert ("1:fs",) not in values[TieringOffloadingMetrics.BLOCK_HITS]
|
||||||
|
|
||||||
|
|
||||||
|
def test_tiering_metrics_tracker_stops_lookup_metrics_after_allocation():
|
||||||
|
tracker = TieringMetricsTracker(
|
||||||
|
tier_types=["fs"],
|
||||||
|
num_primary_blocks=5,
|
||||||
|
primary_block_size=16,
|
||||||
|
)
|
||||||
|
tracker.on_new_request(_CTX)
|
||||||
|
key = to_keys([1])[0]
|
||||||
|
|
||||||
|
tracker.on_lookup(
|
||||||
|
_CTX,
|
||||||
|
key,
|
||||||
|
tracker.primary_tier_label,
|
||||||
|
LookupResult.MISS,
|
||||||
|
lookup_duration=0.0,
|
||||||
|
)
|
||||||
|
tracker.on_lookup(_CTX, key, tracker.tier_label(0), LookupResult.HIT, 0.0)
|
||||||
|
|
||||||
|
stats = tracker.take_stats()
|
||||||
|
assert stats is not None
|
||||||
|
values = stats.data["data"]
|
||||||
|
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("0:primary",)] == 1
|
||||||
|
assert values[TieringOffloadingMetrics.BLOCK_QUERIES][("1:fs",)] == 1
|
||||||
|
assert values[TieringOffloadingMetrics.BLOCK_HITS][("1:fs",)] == 1
|
||||||
|
|
||||||
|
tracker.on_request_allocated(_CTX)
|
||||||
|
tracker.on_lookup(
|
||||||
|
_CTX,
|
||||||
|
key,
|
||||||
|
tracker.primary_tier_label,
|
||||||
|
LookupResult.MISS,
|
||||||
|
lookup_duration=0.0,
|
||||||
|
)
|
||||||
|
tracker.on_lookup(_CTX, key, tracker.tier_label(0), LookupResult.HIT, 0.0)
|
||||||
|
|
||||||
|
stats = tracker.take_stats()
|
||||||
|
assert stats is not None
|
||||||
|
values = stats.data["data"]
|
||||||
|
assert TieringOffloadingMetrics.BLOCK_QUERIES not in values
|
||||||
|
assert TieringOffloadingMetrics.BLOCK_HITS not in values
|
||||||
|
|
||||||
|
|
||||||
|
def test_tiering_metrics_tracker_records_finished_job_metrics():
|
||||||
|
tracker = TieringMetricsTracker(
|
||||||
|
tier_types=["fs"],
|
||||||
|
num_primary_blocks=5,
|
||||||
|
primary_block_size=16,
|
||||||
|
)
|
||||||
|
cascade_key, promotion_key, failed_cascade_key, failed_promotion_key = to_keys(
|
||||||
|
range(4)
|
||||||
|
)
|
||||||
|
jobs = [
|
||||||
|
JobMetadata(
|
||||||
|
TransferJob(0, [cascade_key], np.array([0]), False, _CTX),
|
||||||
|
0,
|
||||||
|
),
|
||||||
|
JobMetadata(
|
||||||
|
TransferJob(1, [promotion_key], np.array([1]), True, _CTX),
|
||||||
|
0,
|
||||||
|
),
|
||||||
|
JobMetadata(
|
||||||
|
TransferJob(2, [failed_cascade_key], np.array([2]), False, _CTX),
|
||||||
|
0,
|
||||||
|
),
|
||||||
|
JobMetadata(
|
||||||
|
TransferJob(3, [failed_promotion_key], np.array([3]), True, _CTX),
|
||||||
|
0,
|
||||||
|
),
|
||||||
|
]
|
||||||
|
for job in jobs:
|
||||||
|
tracker.on_job_registered(job)
|
||||||
|
|
||||||
|
tracker.on_job_finished(
|
||||||
|
jobs[0], JobResult(job_id=0, success=True, transfer_time=0.5)
|
||||||
|
)
|
||||||
|
tracker.on_job_finished(
|
||||||
|
jobs[1], JobResult(job_id=1, success=True, transfer_time=0.25)
|
||||||
|
)
|
||||||
|
tracker.on_job_finished(jobs[2], JobResult(job_id=2, success=False))
|
||||||
|
tracker.on_job_finished(jobs[3], JobResult(job_id=3, success=False))
|
||||||
|
|
||||||
|
stats = tracker.take_stats()
|
||||||
|
assert stats is not None
|
||||||
|
values = stats.data["data"]
|
||||||
|
label = ("1:fs",)
|
||||||
|
assert values[TieringOffloadingMetrics.WRITE_BYTES][label] == 16
|
||||||
|
assert values[TieringOffloadingMetrics.WRITE_TIME][label] == 0.5
|
||||||
|
assert values[TieringOffloadingMetrics.READ_BYTES][label] == 16
|
||||||
|
assert values[TieringOffloadingMetrics.READ_TIME][label] == 0.25
|
||||||
|
assert values[TieringOffloadingMetrics.CASCADE_JOB_FAILURES][label] == 1
|
||||||
|
assert values[TieringOffloadingMetrics.PROMOTION_JOB_FAILURES][label] == 1
|
||||||
|
tracker.assert_idle()
|
||||||
|
|
||||||
|
|
||||||
|
def test_tiering_metrics_tracker_records_partial_promotion_success_bytes():
|
||||||
|
tracker = TieringMetricsTracker(
|
||||||
|
tier_types=["fs"],
|
||||||
|
num_primary_blocks=5,
|
||||||
|
primary_block_size=16,
|
||||||
|
)
|
||||||
|
keys = to_keys(range(3))
|
||||||
|
job = JobMetadata(
|
||||||
|
TransferJob(0, keys, np.array([0, 1, 2]), True, _CTX),
|
||||||
|
0,
|
||||||
|
)
|
||||||
|
tracker.on_job_registered(job)
|
||||||
|
|
||||||
|
tracker.on_job_finished(
|
||||||
|
job,
|
||||||
|
JobResult(
|
||||||
|
job_id=0,
|
||||||
|
success=False,
|
||||||
|
successful_keys=(keys[0], keys[2]),
|
||||||
|
transfer_time=0.5,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
stats = tracker.take_stats()
|
||||||
|
assert stats is not None
|
||||||
|
values = stats.data["data"]
|
||||||
|
label = ("1:fs",)
|
||||||
|
assert values[TieringOffloadingMetrics.READ_BYTES][label] == 32
|
||||||
|
assert values[TieringOffloadingMetrics.READ_TIME][label] == 0.5
|
||||||
|
assert values[TieringOffloadingMetrics.PROMOTION_JOB_FAILURES][label] == 1
|
||||||
|
tracker.assert_idle()
|
||||||
|
|
||||||
|
|
||||||
|
def test_tiering_metrics_tracker_reports_active_job_and_primary_usage_gauges():
|
||||||
|
tracker = TieringMetricsTracker(
|
||||||
|
tier_types=["fs", "p2p"],
|
||||||
|
num_primary_blocks=6,
|
||||||
|
primary_block_size=16,
|
||||||
|
)
|
||||||
|
fs_job = JobMetadata(
|
||||||
|
TransferJob(0, to_keys([0, 1]), np.array([0, 1]), False, _CTX),
|
||||||
|
0,
|
||||||
|
)
|
||||||
|
p2p_job = JobMetadata(
|
||||||
|
TransferJob(1, to_keys([2, 3, 4]), np.array([2, 3, 4]), True, _CTX),
|
||||||
|
1,
|
||||||
|
)
|
||||||
|
tracker.on_job_registered(fs_job)
|
||||||
|
tracker.on_job_registered(p2p_job)
|
||||||
|
|
||||||
|
stats = tracker.take_stats()
|
||||||
|
assert stats is not None
|
||||||
|
values = stats.data["data"]
|
||||||
|
fs_label = ("1:fs",)
|
||||||
|
p2p_label = ("2:p2p",)
|
||||||
|
assert values[TieringOffloadingMetrics.PRIMARY_READ_USAGE_PERC][
|
||||||
|
fs_label
|
||||||
|
] == pytest.approx(2 / 6)
|
||||||
|
assert values[TieringOffloadingMetrics.ACTIVE_CASCADE_JOBS][fs_label] == 1
|
||||||
|
assert values[TieringOffloadingMetrics.PRIMARY_WRITE_USAGE_PERC][
|
||||||
|
p2p_label
|
||||||
|
] == pytest.approx(3 / 6)
|
||||||
|
assert values[TieringOffloadingMetrics.ACTIVE_PROMOTION_JOBS][p2p_label] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_tiering_metrics_tracker_records_promotion_allocation_failures():
|
||||||
|
tracker = TieringMetricsTracker(
|
||||||
|
tier_types=["fs"],
|
||||||
|
num_primary_blocks=1,
|
||||||
|
primary_block_size=16,
|
||||||
|
)
|
||||||
|
|
||||||
|
tracker.on_promotion_allocation_failure()
|
||||||
|
|
||||||
|
stats = tracker.take_stats()
|
||||||
|
assert stats is not None
|
||||||
|
values = stats.data["data"]
|
||||||
|
assert values[TieringOffloadingMetrics.PROMOTION_ALLOCATION_FAILURES][()] == 1
|
||||||
@@ -34,7 +34,7 @@ from vllm.v1.kv_offload.config import (
|
|||||||
OffloadingModelConfig,
|
OffloadingModelConfig,
|
||||||
OffloadingParallelConfig,
|
OffloadingParallelConfig,
|
||||||
)
|
)
|
||||||
from vllm.v1.kv_offload.tiering.base import JobMetadata, JobResult
|
from vllm.v1.kv_offload.tiering.base import JobResult, TransferJob
|
||||||
from vllm.v1.kv_offload.tiering.manager import (
|
from vllm.v1.kv_offload.tiering.manager import (
|
||||||
CPUPrimaryTierOffloadingManager,
|
CPUPrimaryTierOffloadingManager,
|
||||||
TieringOffloadingManager,
|
TieringOffloadingManager,
|
||||||
@@ -105,10 +105,10 @@ def make_job(
|
|||||||
job_id: int,
|
job_id: int,
|
||||||
keys: list[OffloadKey],
|
keys: list[OffloadKey],
|
||||||
block_ids: list[int] | None = None,
|
block_ids: list[int] | None = None,
|
||||||
) -> JobMetadata:
|
) -> TransferJob:
|
||||||
if block_ids is None:
|
if block_ids is None:
|
||||||
block_ids = list(range(len(keys)))
|
block_ids = list(range(len(keys)))
|
||||||
return JobMetadata(
|
return TransferJob(
|
||||||
job_id=job_id,
|
job_id=job_id,
|
||||||
keys=keys,
|
keys=keys,
|
||||||
block_ids=np.array(block_ids, dtype=np.int64),
|
block_ids=np.array(block_ids, dtype=np.int64),
|
||||||
@@ -201,6 +201,9 @@ class MockNixlAgent:
|
|||||||
def release_dlist_handle(self, handle):
|
def release_dlist_handle(self, handle):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
def get_xfer_telemetry(self, handle):
|
||||||
|
return SimpleNamespace(xferDuration=1000)
|
||||||
|
|
||||||
def _query_memory(self, queries, mem_type, agent_name):
|
def _query_memory(self, queries, mem_type, agent_name):
|
||||||
return [object() if q[3] in self._stored_obj_keys else None for q in queries]
|
return [object() if q[3] in self._stored_obj_keys else None for q in queries]
|
||||||
|
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ from vllm.v1.kv_offload.base import (
|
|||||||
Medium,
|
Medium,
|
||||||
OffloadingCounterMetadata,
|
OffloadingCounterMetadata,
|
||||||
OffloadingEvent,
|
OffloadingEvent,
|
||||||
|
OffloadingGaugeMetadata,
|
||||||
OffloadKey,
|
OffloadKey,
|
||||||
OffloadPolicy,
|
OffloadPolicy,
|
||||||
ReqContext,
|
ReqContext,
|
||||||
@@ -39,10 +40,10 @@ from vllm.v1.kv_offload.base import (
|
|||||||
make_offload_key,
|
make_offload_key,
|
||||||
)
|
)
|
||||||
from vllm.v1.kv_offload.tiering.base import (
|
from vllm.v1.kv_offload.tiering.base import (
|
||||||
JobMetadata,
|
|
||||||
JobResult,
|
JobResult,
|
||||||
SecondaryTierManager,
|
SecondaryTierManager,
|
||||||
TieringOffloadingMetrics,
|
TieringOffloadingMetrics,
|
||||||
|
TransferJob,
|
||||||
)
|
)
|
||||||
from vllm.v1.kv_offload.tiering.example.manager import ExampleSecondaryTierManager
|
from vllm.v1.kv_offload.tiering.example.manager import ExampleSecondaryTierManager
|
||||||
from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory
|
from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory
|
||||||
@@ -85,6 +86,16 @@ def count_hits(manager, keys: list[OffloadKey]) -> int | None:
|
|||||||
return count
|
return count
|
||||||
|
|
||||||
|
|
||||||
|
def histogram_count_key(metric_name: str, labelvalues: tuple[str, ...]) -> str:
|
||||||
|
return f"{metric_name}:{labelvalues}_count"
|
||||||
|
|
||||||
|
|
||||||
|
def has_histogram_count(reduced: dict[str, int | float], metric_name: str) -> bool:
|
||||||
|
return any(
|
||||||
|
key.startswith(f"{metric_name}:") and key.endswith("_count") for key in reduced
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class MetricsSecondaryTierManager(SecondaryTierManager):
|
class MetricsSecondaryTierManager(SecondaryTierManager):
|
||||||
"""Test-only secondary tier that declares and emits one labeled metric."""
|
"""Test-only secondary tier that declares and emits one labeled metric."""
|
||||||
|
|
||||||
@@ -102,18 +113,26 @@ class MetricsSecondaryTierManager(SecondaryTierManager):
|
|||||||
def __init__(self, *args, **kwargs):
|
def __init__(self, *args, **kwargs):
|
||||||
super().__init__(*args, **kwargs)
|
super().__init__(*args, **kwargs)
|
||||||
self.stats: OffloadingConnectorStats | None = None
|
self.stats: OffloadingConnectorStats | None = None
|
||||||
|
self.lookup_result = LookupResult.MISS
|
||||||
|
self.finished_jobs: list[JobResult] = []
|
||||||
|
self.submitted_loads: list[TransferJob] = []
|
||||||
|
self.submitted_stores: list[TransferJob] = []
|
||||||
|
|
||||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
|
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
||||||
return False
|
return self.lookup_result
|
||||||
|
|
||||||
def submit_store(self, job_metadata: JobMetadata) -> None:
|
def submit_store(self, job_metadata: TransferJob) -> None:
|
||||||
|
self.submitted_stores.append(job_metadata)
|
||||||
return
|
return
|
||||||
|
|
||||||
def submit_load(self, job_metadata: JobMetadata) -> None:
|
def submit_load(self, job_metadata: TransferJob) -> None:
|
||||||
|
self.submitted_loads.append(job_metadata)
|
||||||
return
|
return
|
||||||
|
|
||||||
def get_finished_jobs(self) -> Iterable[JobResult]:
|
def get_finished_jobs(self) -> Iterable[JobResult]:
|
||||||
return ()
|
results = self.finished_jobs
|
||||||
|
self.finished_jobs = []
|
||||||
|
return results
|
||||||
|
|
||||||
def drain_jobs(self) -> None:
|
def drain_jobs(self) -> None:
|
||||||
return
|
return
|
||||||
@@ -141,6 +160,24 @@ def test_tiering_spec_collects_secondary_metric_definitions(monkeypatch):
|
|||||||
metadata = metrics[MetricsSecondaryTierManager.MY_TIER_METRIC]
|
metadata = metrics[MetricsSecondaryTierManager.MY_TIER_METRIC]
|
||||||
assert metadata.documentation == "Number of bytes served by the test tier."
|
assert metadata.documentation == "Number of bytes served by the test tier."
|
||||||
assert metadata.labelnames == ("tier",)
|
assert metadata.labelnames == ("tier",)
|
||||||
|
assert metrics[TieringOffloadingMetrics.BLOCK_QUERIES].labelnames == ("tier",)
|
||||||
|
assert metrics[TieringOffloadingMetrics.LOOKUP_SYNC_DELAY].labelnames == ("tier",)
|
||||||
|
assert metrics[TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY].labelnames == ("tier",)
|
||||||
|
assert metrics[TieringOffloadingMetrics.PROMOTION_JOB_FAILURES].labelnames == (
|
||||||
|
"tier",
|
||||||
|
)
|
||||||
|
assert isinstance(
|
||||||
|
metrics[TieringOffloadingMetrics.PROMOTION_ALLOCATION_FAILURES],
|
||||||
|
OffloadingCounterMetadata,
|
||||||
|
)
|
||||||
|
for metric_name in (
|
||||||
|
TieringOffloadingMetrics.PRIMARY_WRITE_USAGE_PERC,
|
||||||
|
TieringOffloadingMetrics.PRIMARY_READ_USAGE_PERC,
|
||||||
|
TieringOffloadingMetrics.ACTIVE_PROMOTION_JOBS,
|
||||||
|
TieringOffloadingMetrics.ACTIVE_CASCADE_JOBS,
|
||||||
|
):
|
||||||
|
assert isinstance(metrics[metric_name], OffloadingGaugeMetadata)
|
||||||
|
assert metrics[metric_name].labelnames == ("tier",)
|
||||||
|
|
||||||
|
|
||||||
def test_tiering_manager_aggregates_secondary_stats():
|
def test_tiering_manager_aggregates_secondary_stats():
|
||||||
@@ -261,12 +298,15 @@ class TestTieringOffloadingManager:
|
|||||||
self._start_request()
|
self._start_request()
|
||||||
# Register an in-flight promotion job for tier1 by hand.
|
# Register an in-flight promotion job for tier1 by hand.
|
||||||
job_id = self.manager._next_job_id()
|
job_id = self.manager._next_job_id()
|
||||||
self.manager._transfer_jobs[job_id] = JobMetadata(
|
self.manager._register_job(
|
||||||
job_id=job_id,
|
TransferJob(
|
||||||
keys=to_keys([1, 2]),
|
job_id=job_id,
|
||||||
block_ids=[0, 1],
|
keys=to_keys([1, 2]),
|
||||||
is_promotion=True,
|
block_ids=[0, 1],
|
||||||
req_context=_CTX,
|
is_promotion=True,
|
||||||
|
req_context=_CTX,
|
||||||
|
),
|
||||||
|
0,
|
||||||
)
|
)
|
||||||
failed = JobResult(job_id=job_id, success=False)
|
failed = JobResult(job_id=job_id, success=False)
|
||||||
with (
|
with (
|
||||||
@@ -285,12 +325,15 @@ class TestTieringOffloadingManager:
|
|||||||
|
|
||||||
self._start_request()
|
self._start_request()
|
||||||
job_id = self.manager._next_job_id()
|
job_id = self.manager._next_job_id()
|
||||||
self.manager._transfer_jobs[job_id] = JobMetadata(
|
self.manager._register_job(
|
||||||
job_id=job_id,
|
TransferJob(
|
||||||
keys=to_keys([1]),
|
job_id=job_id,
|
||||||
block_ids=[0],
|
keys=to_keys([1]),
|
||||||
is_promotion=True,
|
block_ids=[0],
|
||||||
req_context=_CTX,
|
is_promotion=True,
|
||||||
|
req_context=_CTX,
|
||||||
|
),
|
||||||
|
0,
|
||||||
)
|
)
|
||||||
ok = JobResult(job_id=job_id, success=True)
|
ok = JobResult(job_id=job_id, success=True)
|
||||||
with (
|
with (
|
||||||
@@ -473,7 +516,7 @@ class TestTieringOffloadingManager:
|
|||||||
for block in blocks:
|
for block in blocks:
|
||||||
self.secondary_tier1.blocks[block] = True
|
self.secondary_tier1.blocks[block] = True
|
||||||
|
|
||||||
def submit_partial(job_metadata: JobMetadata) -> None:
|
def submit_partial(job_metadata: TransferJob) -> None:
|
||||||
successful_keys = (
|
successful_keys = (
|
||||||
None
|
None
|
||||||
if successful_indices is None
|
if successful_indices is None
|
||||||
@@ -500,7 +543,7 @@ class TestTieringOffloadingManager:
|
|||||||
] == expected_results
|
] == expected_results
|
||||||
|
|
||||||
def test_lookup_reports_sync_delay_for_resolved_lookups(self, manager_setup):
|
def test_lookup_reports_sync_delay_for_resolved_lookups(self, manager_setup):
|
||||||
"""Resolved lookups report one sync delay sample on allocation."""
|
"""Resolved lookups report one sync delay sample per tier and block."""
|
||||||
self._start_request()
|
self._start_request()
|
||||||
blocks = to_keys(range(2))
|
blocks = to_keys(range(2))
|
||||||
|
|
||||||
@@ -508,22 +551,39 @@ class TestTieringOffloadingManager:
|
|||||||
for block in blocks:
|
for block in blocks:
|
||||||
assert self.manager.lookup(block, _CTX) is LookupResult.MISS
|
assert self.manager.lookup(block, _CTX) is LookupResult.MISS
|
||||||
|
|
||||||
stats = self.manager.get_stats()
|
|
||||||
if stats is not None:
|
|
||||||
assert f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count" not in (
|
|
||||||
stats.reduce()
|
|
||||||
)
|
|
||||||
|
|
||||||
self._simulate_on_schedule_end(new_req_ids=[_CTX.req_id])
|
|
||||||
|
|
||||||
stats = self.manager.get_stats()
|
stats = self.manager.get_stats()
|
||||||
assert stats is not None
|
assert stats is not None
|
||||||
reduced = stats.reduce()
|
reduced = stats.reduce()
|
||||||
assert reduced[f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count"] == 1
|
assert (
|
||||||
assert f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count" not in reduced
|
reduced[
|
||||||
|
histogram_count_key(
|
||||||
|
TieringOffloadingMetrics.LOOKUP_SYNC_DELAY, ("0:primary",)
|
||||||
|
)
|
||||||
|
]
|
||||||
|
== 2
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
reduced[
|
||||||
|
histogram_count_key(
|
||||||
|
TieringOffloadingMetrics.LOOKUP_SYNC_DELAY, ("1:example",)
|
||||||
|
)
|
||||||
|
]
|
||||||
|
== 2
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
reduced[
|
||||||
|
histogram_count_key(
|
||||||
|
TieringOffloadingMetrics.LOOKUP_SYNC_DELAY, ("2:example",)
|
||||||
|
)
|
||||||
|
]
|
||||||
|
== 2
|
||||||
|
)
|
||||||
|
assert not has_histogram_count(
|
||||||
|
reduced, TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY
|
||||||
|
)
|
||||||
|
|
||||||
def test_lookup_reports_async_delay_across_promotion(self, manager_setup):
|
def test_lookup_does_not_report_async_delay_for_promotion(self, manager_setup):
|
||||||
"""A new request reports async delay at schedule end."""
|
"""Promotion time is not included in lookup async delay."""
|
||||||
self._start_request()
|
self._start_request()
|
||||||
block = to_keys(range(1))[0]
|
block = to_keys(range(1))[0]
|
||||||
self.secondary_tier1.blocks[block] = True
|
self.secondary_tier1.blocks[block] = True
|
||||||
@@ -531,13 +591,30 @@ class TestTieringOffloadingManager:
|
|||||||
# First lookup finds the block in a secondary tier and defers.
|
# First lookup finds the block in a secondary tier and defers.
|
||||||
assert self.manager.lookup(block, _CTX) is LookupResult.RETRY
|
assert self.manager.lookup(block, _CTX) is LookupResult.RETRY
|
||||||
|
|
||||||
# The first scheduler step reports async delay for the new request.
|
# Promotion is not treated as unresolved secondary lookup time.
|
||||||
self._simulate_on_schedule_end(new_req_ids=[_CTX.req_id])
|
self._simulate_on_schedule_end(new_req_ids=[_CTX.req_id])
|
||||||
stats = self.manager.get_stats()
|
stats = self.manager.get_stats()
|
||||||
assert stats is not None
|
assert stats is not None
|
||||||
reduced = stats.reduce()
|
reduced = stats.reduce()
|
||||||
assert reduced[f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count"] == 1
|
assert (
|
||||||
assert reduced[f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count"] == 1
|
reduced[
|
||||||
|
histogram_count_key(
|
||||||
|
TieringOffloadingMetrics.LOOKUP_SYNC_DELAY, ("0:primary",)
|
||||||
|
)
|
||||||
|
]
|
||||||
|
== 1
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
reduced[
|
||||||
|
histogram_count_key(
|
||||||
|
TieringOffloadingMetrics.LOOKUP_SYNC_DELAY, ("1:example",)
|
||||||
|
)
|
||||||
|
]
|
||||||
|
== 1
|
||||||
|
)
|
||||||
|
assert not has_histogram_count(
|
||||||
|
reduced, TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY
|
||||||
|
)
|
||||||
|
|
||||||
# Promotion completes on the next scheduler step.
|
# Promotion completes on the next scheduler step.
|
||||||
self._simulate_on_schedule_end()
|
self._simulate_on_schedule_end()
|
||||||
@@ -547,38 +624,51 @@ class TestTieringOffloadingManager:
|
|||||||
|
|
||||||
stats = self.manager.get_stats()
|
stats = self.manager.get_stats()
|
||||||
if stats is not None:
|
if stats is not None:
|
||||||
assert f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count" not in (
|
assert not has_histogram_count(
|
||||||
stats.reduce()
|
stats.reduce(), TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_lookup_reports_async_delay_on_request_finish(self, manager_setup):
|
def test_lookup_reports_async_delay_when_deferred_lookup_resolves(
|
||||||
"""Never-allocated lookup delays flush at teardown."""
|
self, manager_setup
|
||||||
|
):
|
||||||
|
"""Async delay is observed when a RETRY lookup later resolves."""
|
||||||
ctx = ReqContext(req_id="req_lookup_finish")
|
ctx = ReqContext(req_id="req_lookup_finish")
|
||||||
self._start_request(ctx)
|
self._start_request(ctx)
|
||||||
block = to_keys(range(1))[0]
|
block = to_keys(range(1))[0]
|
||||||
self.secondary_tier1.blocks[block] = True
|
self.secondary_tier1.lookup = MagicMock(
|
||||||
|
side_effect=[LookupResult.RETRY, LookupResult.HIT]
|
||||||
|
)
|
||||||
|
|
||||||
# Lookup finds the block in a secondary tier and defers.
|
# First lookup is deferred by the secondary tier.
|
||||||
assert self.manager.lookup(block, ctx) is LookupResult.RETRY
|
assert self.manager.lookup(block, ctx) is LookupResult.RETRY
|
||||||
|
|
||||||
self._simulate_on_schedule_end()
|
|
||||||
stats = self.manager.get_stats()
|
stats = self.manager.get_stats()
|
||||||
if stats is not None:
|
if stats is not None:
|
||||||
assert f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count" not in (
|
assert not has_histogram_count(
|
||||||
stats.reduce()
|
stats.reduce(), TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY
|
||||||
)
|
|
||||||
assert f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count" not in (
|
|
||||||
stats.reduce()
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# Request finishes before the deferred lookup is ever resolved.
|
assert self.manager.lookup(block, ctx) is LookupResult.RETRY
|
||||||
self.manager.on_request_finished(ctx)
|
|
||||||
|
|
||||||
stats = self.manager.get_stats()
|
stats = self.manager.get_stats()
|
||||||
assert stats is not None
|
assert stats is not None
|
||||||
reduced = stats.reduce()
|
reduced = stats.reduce()
|
||||||
assert reduced[f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count"] == 1
|
assert (
|
||||||
assert reduced[f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count"] == 1
|
reduced[
|
||||||
|
histogram_count_key(
|
||||||
|
TieringOffloadingMetrics.LOOKUP_SYNC_DELAY, ("1:example",)
|
||||||
|
)
|
||||||
|
]
|
||||||
|
== 1
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
reduced[
|
||||||
|
histogram_count_key(
|
||||||
|
TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY, ("1:example",)
|
||||||
|
)
|
||||||
|
]
|
||||||
|
== 1
|
||||||
|
)
|
||||||
|
|
||||||
def test_partial_lookup(self, manager_setup):
|
def test_partial_lookup(self, manager_setup):
|
||||||
"""Test lookup with partial hits."""
|
"""Test lookup with partial hits."""
|
||||||
@@ -938,9 +1028,7 @@ class TestTieringOffloadingManager:
|
|||||||
ctx = ReqContext(req_id="req_policy_lifecycle_2")
|
ctx = ReqContext(req_id="req_policy_lifecycle_2")
|
||||||
result = self.manager.on_new_request(ctx)
|
result = self.manager.on_new_request(ctx)
|
||||||
assert result.policy == OffloadPolicy.REQUEST_LEVEL
|
assert result.policy == OffloadPolicy.REQUEST_LEVEL
|
||||||
assert self.manager._req_state[ctx.req_id].request_level_tiers == {
|
assert self.manager._req_state[ctx.req_id].request_level_tiers == {0}
|
||||||
self.secondary_tier1
|
|
||||||
}
|
|
||||||
|
|
||||||
# Cleanup
|
# Cleanup
|
||||||
self.manager.on_request_finished(ctx)
|
self.manager.on_request_finished(ctx)
|
||||||
@@ -996,13 +1084,13 @@ class TestTieringOffloadingManager:
|
|||||||
primary tier; pending submissions are dropped without being sent
|
primary tier; pending submissions are dropped without being sent
|
||||||
to the secondary tier. Active request state is retained."""
|
to the secondary tier. Active request state is retained."""
|
||||||
# Cascade — populates primary blocks and leaves cascade jobs
|
# Cascade — populates primary blocks and leaves cascade jobs
|
||||||
# in _transfer_jobs (the synchronous example tier has already
|
# in _jobs (the synchronous example tier has already
|
||||||
# queued completions); reset_cache's drain loop will pick them up.
|
# queued completions); reset_cache's drain loop will pick them up.
|
||||||
blocks = to_keys(range(3))
|
blocks = to_keys(range(3))
|
||||||
self._start_request()
|
self._start_request()
|
||||||
self.manager.prepare_store(blocks, _CTX)
|
self.manager.prepare_store(blocks, _CTX)
|
||||||
self.manager.complete_store(blocks, _CTX, success=True)
|
self.manager.complete_store(blocks, _CTX, success=True)
|
||||||
assert self.manager._transfer_jobs
|
assert self.manager._jobs
|
||||||
|
|
||||||
# Pending promotion submission (deferred — no on_schedule_end after
|
# Pending promotion submission (deferred — no on_schedule_end after
|
||||||
# the lookup that staged it).
|
# the lookup that staged it).
|
||||||
@@ -1020,9 +1108,7 @@ class TestTieringOffloadingManager:
|
|||||||
)
|
)
|
||||||
rl_ctx = ReqContext(req_id="rl")
|
rl_ctx = ReqContext(req_id="rl")
|
||||||
self.manager.on_new_request(rl_ctx)
|
self.manager.on_new_request(rl_ctx)
|
||||||
assert self.manager._req_state[rl_ctx.req_id].request_level_tiers == {
|
assert self.manager._req_state[rl_ctx.req_id].request_level_tiers == {0}
|
||||||
self.secondary_tier1
|
|
||||||
}
|
|
||||||
|
|
||||||
# Mark this step as already polled (reset_cache must clear it).
|
# Mark this step as already polled (reset_cache must clear it).
|
||||||
self.manager._processed_jobs_this_step = True
|
self.manager._processed_jobs_this_step = True
|
||||||
@@ -1035,7 +1121,7 @@ class TestTieringOffloadingManager:
|
|||||||
self.manager.reset_cache()
|
self.manager.reset_cache()
|
||||||
|
|
||||||
# Orchestrator state cleared.
|
# Orchestrator state cleared.
|
||||||
assert self.manager._transfer_jobs == {}
|
assert self.manager._jobs == {}
|
||||||
assert self.manager._pending_load_submissions == {}
|
assert self.manager._pending_load_submissions == {}
|
||||||
assert set(self.manager._req_state) == {_CTX.req_id, rl_ctx.req_id}
|
assert set(self.manager._req_state) == {_CTX.req_id, rl_ctx.req_id}
|
||||||
assert self.manager._processed_jobs_this_step is False
|
assert self.manager._processed_jobs_this_step is False
|
||||||
@@ -1063,18 +1149,18 @@ class TestTieringOffloadingManager:
|
|||||||
wraps=self.secondary_tier2.drain_jobs
|
wraps=self.secondary_tier2.drain_jobs
|
||||||
)
|
)
|
||||||
|
|
||||||
# Drive a cascade so a job lands in _transfer_jobs.
|
# Drive a cascade so a job lands in _jobs.
|
||||||
blocks = to_keys(range(3))
|
blocks = to_keys(range(3))
|
||||||
self._start_request()
|
self._start_request()
|
||||||
self.manager.prepare_store(blocks, _CTX)
|
self.manager.prepare_store(blocks, _CTX)
|
||||||
self.manager.complete_store(blocks, _CTX, success=True)
|
self.manager.complete_store(blocks, _CTX, success=True)
|
||||||
assert self.manager._transfer_jobs
|
assert self.manager._jobs
|
||||||
|
|
||||||
self.manager.reset_cache()
|
self.manager.reset_cache()
|
||||||
|
|
||||||
self.secondary_tier1.drain_jobs.assert_called_once()
|
self.secondary_tier1.drain_jobs.assert_called_once()
|
||||||
self.secondary_tier2.drain_jobs.assert_called_once()
|
self.secondary_tier2.drain_jobs.assert_called_once()
|
||||||
assert self.manager._transfer_jobs == {}
|
assert self.manager._jobs == {}
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"load_tier_filter",
|
"load_tier_filter",
|
||||||
|
|||||||
@@ -39,10 +39,25 @@ class TieringOffloadingMetrics:
|
|||||||
|
|
||||||
LOOKUP_SYNC_DELAY = "vllm:kv_offload_tiering_lookup_sync_delay_seconds"
|
LOOKUP_SYNC_DELAY = "vllm:kv_offload_tiering_lookup_sync_delay_seconds"
|
||||||
LOOKUP_ASYNC_DELAY = "vllm:kv_offload_tiering_lookup_async_delay_seconds"
|
LOOKUP_ASYNC_DELAY = "vllm:kv_offload_tiering_lookup_async_delay_seconds"
|
||||||
|
READ_BYTES = "vllm:kv_offload_tiering_read_bytes"
|
||||||
|
READ_TIME = "vllm:kv_offload_tiering_read_time"
|
||||||
|
WRITE_BYTES = "vllm:kv_offload_tiering_write_bytes"
|
||||||
|
WRITE_TIME = "vllm:kv_offload_tiering_write_time"
|
||||||
|
PROMOTION_JOB_FAILURES = "vllm:kv_offload_tiering_promotion_job_failures"
|
||||||
|
CASCADE_JOB_FAILURES = "vllm:kv_offload_tiering_cascade_job_failures"
|
||||||
|
BLOCK_QUERIES = "vllm:kv_offload_tiering_block_queries"
|
||||||
|
BLOCK_HITS = "vllm:kv_offload_tiering_block_hits"
|
||||||
|
PRIMARY_WRITE_USAGE_PERC = "vllm:kv_offload_tiering_primary_write_usage_perc"
|
||||||
|
PRIMARY_READ_USAGE_PERC = "vllm:kv_offload_tiering_primary_read_usage_perc"
|
||||||
|
PROMOTION_ALLOCATION_FAILURES = (
|
||||||
|
"vllm:kv_offload_tiering_promotion_allocation_failures"
|
||||||
|
)
|
||||||
|
ACTIVE_PROMOTION_JOBS = "vllm:kv_offload_tiering_active_promotion_jobs"
|
||||||
|
ACTIVE_CASCADE_JOBS = "vllm:kv_offload_tiering_active_cascade_jobs"
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class JobMetadata:
|
class TransferJob:
|
||||||
"""Metadata for an in-flight async transfer job."""
|
"""Metadata for an in-flight async transfer job."""
|
||||||
|
|
||||||
job_id: JobId
|
job_id: JobId
|
||||||
@@ -63,6 +78,7 @@ class JobResult:
|
|||||||
# keys that were successfully loaded. None means all keys share the fate
|
# keys that were successfully loaded. None means all keys share the fate
|
||||||
# indicated by `success`. Must be a subset of the job's original keys.
|
# indicated by `success`. Must be a subset of the job's original keys.
|
||||||
successful_keys: Collection[OffloadKey] | None = None
|
successful_keys: Collection[OffloadKey] | None = None
|
||||||
|
transfer_time: float | None = None
|
||||||
|
|
||||||
|
|
||||||
class ParentManager(ABC):
|
class ParentManager(ABC):
|
||||||
@@ -96,7 +112,7 @@ class ParentManager(ABC):
|
|||||||
self,
|
self,
|
||||||
keys: Collection[OffloadKey],
|
keys: Collection[OffloadKey],
|
||||||
req_context: ReqContext,
|
req_context: ReqContext,
|
||||||
) -> JobMetadata: ...
|
) -> TransferJob: ...
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def on_request_finished(self, req_context: ReqContext) -> None: ...
|
def on_request_finished(self, req_context: ReqContext) -> None: ...
|
||||||
@@ -153,7 +169,7 @@ class SecondaryTierManager(ABC):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def submit_store(self, job_metadata: JobMetadata) -> None:
|
def submit_store(self, job_metadata: TransferJob) -> None:
|
||||||
"""
|
"""
|
||||||
Submit an async job to store blocks from the primary tier to this
|
Submit an async job to store blocks from the primary tier to this
|
||||||
secondary tier.
|
secondary tier.
|
||||||
@@ -181,7 +197,7 @@ class SecondaryTierManager(ABC):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def submit_load(self, job_metadata: JobMetadata) -> None:
|
def submit_load(self, job_metadata: TransferJob) -> None:
|
||||||
"""
|
"""
|
||||||
Submit an async job to load blocks from this secondary tier to the
|
Submit an async job to load blocks from this secondary tier to the
|
||||||
primary tier.
|
primary tier.
|
||||||
|
|||||||
@@ -23,9 +23,9 @@ from vllm.v1.kv_offload.base import (
|
|||||||
RequestOffloadingContext,
|
RequestOffloadingContext,
|
||||||
)
|
)
|
||||||
from vllm.v1.kv_offload.tiering.base import (
|
from vllm.v1.kv_offload.tiering.base import (
|
||||||
JobMetadata,
|
|
||||||
JobResult,
|
JobResult,
|
||||||
SecondaryTierManager,
|
SecondaryTierManager,
|
||||||
|
TransferJob,
|
||||||
)
|
)
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -73,6 +73,8 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
|
|||||||
|
|
||||||
# Completed jobs waiting to be retrieved by get_finished_jobs()
|
# Completed jobs waiting to be retrieved by get_finished_jobs()
|
||||||
self.completed_jobs: list[JobResult] = []
|
self.completed_jobs: list[JobResult] = []
|
||||||
|
assert primary_kv_view.strides is not None
|
||||||
|
self._block_size = primary_kv_view.strides[0]
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
||||||
@@ -89,7 +91,7 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
|
|||||||
return LookupResult.HIT if key in self.blocks else LookupResult.MISS
|
return LookupResult.HIT if key in self.blocks else LookupResult.MISS
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def submit_store(self, job_metadata: JobMetadata) -> None:
|
def submit_store(self, job_metadata: TransferJob) -> None:
|
||||||
"""
|
"""
|
||||||
Submit a job to store blocks from primary tier to this tier.
|
Submit a job to store blocks from primary tier to this tier.
|
||||||
|
|
||||||
@@ -106,10 +108,16 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
|
|||||||
|
|
||||||
for key in keys:
|
for key in keys:
|
||||||
self.blocks[key] = True
|
self.blocks[key] = True
|
||||||
self.completed_jobs.append(JobResult(job_id=job_metadata.job_id, success=True))
|
self.completed_jobs.append(
|
||||||
|
JobResult(
|
||||||
|
job_id=job_metadata.job_id,
|
||||||
|
success=True,
|
||||||
|
transfer_time=0.0,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def submit_load(self, job_metadata: JobMetadata) -> None:
|
def submit_load(self, job_metadata: TransferJob) -> None:
|
||||||
"""
|
"""
|
||||||
Submit a job to load blocks from this tier to primary tier.
|
Submit a job to load blocks from this tier to primary tier.
|
||||||
|
|
||||||
@@ -131,7 +139,13 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
|
|||||||
)
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
self.completed_jobs.append(JobResult(job_id=job_metadata.job_id, success=True))
|
self.completed_jobs.append(
|
||||||
|
JobResult(
|
||||||
|
job_id=job_metadata.job_id,
|
||||||
|
success=True,
|
||||||
|
transfer_time=0.0,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def get_finished_jobs(self) -> Iterable[JobResult]:
|
def get_finished_jobs(self) -> Iterable[JobResult]:
|
||||||
|
|||||||
@@ -43,11 +43,11 @@ from vllm.v1.kv_offload.file_mapper import FileMapper
|
|||||||
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager
|
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager
|
||||||
from vllm.v1.kv_offload.tiering.base import (
|
from vllm.v1.kv_offload.tiering.base import (
|
||||||
JobId,
|
JobId,
|
||||||
JobMetadata,
|
|
||||||
JobResult,
|
JobResult,
|
||||||
RequestOffloadingContext,
|
RequestOffloadingContext,
|
||||||
ScheduleEndContext,
|
ScheduleEndContext,
|
||||||
SecondaryTierManager,
|
SecondaryTierManager,
|
||||||
|
TransferJob,
|
||||||
)
|
)
|
||||||
from vllm.v1.kv_offload.tiering.fs.io import (
|
from vllm.v1.kv_offload.tiering.fs.io import (
|
||||||
batch_load_block,
|
batch_load_block,
|
||||||
@@ -214,12 +214,13 @@ class FileSystemTierManager(SecondaryTierManager):
|
|||||||
return LookupResult.HIT if result else LookupResult.MISS
|
return LookupResult.HIT if result else LookupResult.MISS
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def submit_store(self, job_metadata: JobMetadata) -> None:
|
def submit_store(self, job_metadata: TransferJob) -> None:
|
||||||
|
keys = list(job_metadata.keys)
|
||||||
if self.events is not None:
|
if self.events is not None:
|
||||||
self._store_job_keys[job_metadata.job_id] = list(job_metadata.keys)
|
self._store_job_keys[job_metadata.job_id] = keys
|
||||||
task = functools.partial(
|
task = functools.partial(
|
||||||
batch_store_block,
|
batch_store_block,
|
||||||
[self.file_mapper.get_file_name(key) for key in job_metadata.keys],
|
[self.file_mapper.get_file_name(key) for key in keys],
|
||||||
self._primary_kv_view,
|
self._primary_kv_view,
|
||||||
[int(bid) * self._block_size for bid in job_metadata.block_ids],
|
[int(bid) * self._block_size for bid in job_metadata.block_ids],
|
||||||
self._block_size,
|
self._block_size,
|
||||||
@@ -228,7 +229,7 @@ class FileSystemTierManager(SecondaryTierManager):
|
|||||||
self._pool.enqueue_store(job_metadata.job_id, 1, [task])
|
self._pool.enqueue_store(job_metadata.job_id, 1, [task])
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def submit_load(self, job_metadata: JobMetadata) -> None:
|
def submit_load(self, job_metadata: TransferJob) -> None:
|
||||||
job_id = job_metadata.job_id
|
job_id = job_metadata.job_id
|
||||||
# Track this load's keys so a failed promotion can mark only its failed
|
# Track this load's keys so a failed promotion can mark only its failed
|
||||||
# keys as a miss (see get_finished_jobs).
|
# keys as a miss (see get_finished_jobs).
|
||||||
@@ -271,7 +272,7 @@ class FileSystemTierManager(SecondaryTierManager):
|
|||||||
"""Collect finished jobs; a failed promotion marks only its failed keys
|
"""Collect finished jobs; a failed promotion marks only its failed keys
|
||||||
as a miss here (scheduler thread)."""
|
as a miss here (scheduler thread)."""
|
||||||
results = []
|
results = []
|
||||||
for job_id, success in self._pool.get_finished():
|
for job_id, success, transfer_time in self._pool.get_finished():
|
||||||
if self.events is not None:
|
if self.events is not None:
|
||||||
keys = self._store_job_keys.pop(job_id, None)
|
keys = self._store_job_keys.pop(job_id, None)
|
||||||
if success and keys:
|
if success and keys:
|
||||||
@@ -298,10 +299,17 @@ class FileSystemTierManager(SecondaryTierManager):
|
|||||||
job_id=job_id,
|
job_id=job_id,
|
||||||
success=False,
|
success=False,
|
||||||
successful_keys=tuple(successful) if successful else None,
|
successful_keys=tuple(successful) if successful else None,
|
||||||
|
transfer_time=transfer_time,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
else:
|
continue
|
||||||
results.append(JobResult(job_id=job_id, success=success))
|
results.append(
|
||||||
|
JobResult(
|
||||||
|
job_id=job_id,
|
||||||
|
success=success,
|
||||||
|
transfer_time=transfer_time,
|
||||||
|
)
|
||||||
|
)
|
||||||
return results
|
return results
|
||||||
|
|
||||||
@override
|
@override
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ Thread pool:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import threading
|
import threading
|
||||||
|
import time
|
||||||
from collections import deque
|
from collections import deque
|
||||||
from collections.abc import Callable, Iterable
|
from collections.abc import Callable, Iterable
|
||||||
|
|
||||||
@@ -25,26 +26,37 @@ class JobState:
|
|||||||
Each task calls task_done(success) when it finishes.
|
Each task calls task_done(success) when it finishes.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
__slots__ = ("_job_id", "_n_tasks", "_completed", "_success", "_lock")
|
__slots__ = (
|
||||||
|
"_job_id",
|
||||||
|
"_n_tasks",
|
||||||
|
"_completed",
|
||||||
|
"_success",
|
||||||
|
"_transfer_time",
|
||||||
|
"_lock",
|
||||||
|
)
|
||||||
|
|
||||||
def __init__(self, job_id: JobId, n_tasks: int) -> None:
|
def __init__(self, job_id: JobId, n_tasks: int) -> None:
|
||||||
self._job_id: JobId = job_id
|
self._job_id: JobId = job_id
|
||||||
self._n_tasks = n_tasks
|
self._n_tasks = n_tasks
|
||||||
self._completed = 0
|
self._completed = 0
|
||||||
self._success = True
|
self._success = True
|
||||||
|
self._transfer_time = 0.0
|
||||||
self._lock = threading.Lock()
|
self._lock = threading.Lock()
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def job_id(self) -> JobId:
|
def job_id(self) -> JobId:
|
||||||
return self._job_id
|
return self._job_id
|
||||||
|
|
||||||
def task_done(self, success: bool) -> tuple[bool, bool]:
|
def task_done(
|
||||||
|
self, success: bool, transfer_time: float
|
||||||
|
) -> tuple[bool, bool, float]:
|
||||||
"""Returns if job completed and success flag"""
|
"""Returns if job completed and success flag"""
|
||||||
with self._lock:
|
with self._lock:
|
||||||
self._completed += 1
|
self._completed += 1
|
||||||
|
self._transfer_time += transfer_time
|
||||||
if not success:
|
if not success:
|
||||||
self._success = False
|
self._success = False
|
||||||
return self._completed == self._n_tasks, self._success
|
return self._completed == self._n_tasks, self._success, self._transfer_time
|
||||||
|
|
||||||
|
|
||||||
class DualQueueThreadPool:
|
class DualQueueThreadPool:
|
||||||
@@ -67,7 +79,7 @@ class DualQueueThreadPool:
|
|||||||
self._condition = threading.Condition(threading.Lock())
|
self._condition = threading.Condition(threading.Lock())
|
||||||
self._stop = False
|
self._stop = False
|
||||||
self._threads: list[threading.Thread] = []
|
self._threads: list[threading.Thread] = []
|
||||||
self._finished_q: deque[tuple[JobId, bool]] = deque()
|
self._finished_q: deque[tuple[JobId, bool, float]] = deque()
|
||||||
self._inflight_jobs = 0 # guarded by _condition
|
self._inflight_jobs = 0 # guarded by _condition
|
||||||
|
|
||||||
for i in range(n_read_threads):
|
for i in range(n_read_threads):
|
||||||
@@ -118,7 +130,7 @@ class DualQueueThreadPool:
|
|||||||
self._store_q.append((fn, state))
|
self._store_q.append((fn, state))
|
||||||
self._condition.notify(n_tasks)
|
self._condition.notify(n_tasks)
|
||||||
|
|
||||||
def get_finished(self) -> list[tuple[JobId, bool]]:
|
def get_finished(self) -> list[tuple[JobId, bool, float]]:
|
||||||
# No lock needed: deque is thread-safe for concurrent append/popleft,
|
# No lock needed: deque is thread-safe for concurrent append/popleft,
|
||||||
# and the manager is the sole popper.
|
# and the manager is the sole popper.
|
||||||
jobs = []
|
jobs = []
|
||||||
@@ -163,18 +175,23 @@ class DualQueueThreadPool:
|
|||||||
secondary = self._store_q if load_priority else self._load_q
|
secondary = self._store_q if load_priority else self._load_q
|
||||||
task, state = primary.popleft() if primary else secondary.popleft()
|
task, state = primary.popleft() if primary else secondary.popleft()
|
||||||
try:
|
try:
|
||||||
|
start_time = time.monotonic()
|
||||||
task()
|
task()
|
||||||
job_finished, success = state.task_done(True)
|
transfer_time = time.monotonic() - start_time
|
||||||
|
job_finished, success, total_time = state.task_done(True, transfer_time)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
|
transfer_time = time.monotonic() - start_time
|
||||||
logger.error(
|
logger.error(
|
||||||
"Job %s block I/O failed: %s",
|
"Job %s block I/O failed: %s",
|
||||||
state.job_id,
|
state.job_id,
|
||||||
exc,
|
exc,
|
||||||
)
|
)
|
||||||
job_finished, success = state.task_done(False)
|
job_finished, success, total_time = state.task_done(
|
||||||
|
False, transfer_time
|
||||||
|
)
|
||||||
|
|
||||||
if job_finished:
|
if job_finished:
|
||||||
with self._condition:
|
with self._condition:
|
||||||
self._finished_q.append((state.job_id, success))
|
self._finished_q.append((state.job_id, success, total_time))
|
||||||
self._inflight_jobs -= 1
|
self._inflight_jobs -= 1
|
||||||
self._condition.notify_all()
|
self._condition.notify_all()
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ Key Design Principles:
|
|||||||
import time
|
import time
|
||||||
from collections.abc import Collection, Iterable, Sequence
|
from collections.abc import Collection, Iterable, Sequence
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
|
from typing import NamedTuple
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
from typing_extensions import override
|
from typing_extensions import override
|
||||||
@@ -48,12 +49,12 @@ from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
|
|||||||
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||||
from vllm.v1.kv_offload.tiering.base import (
|
from vllm.v1.kv_offload.tiering.base import (
|
||||||
JobId,
|
JobId,
|
||||||
JobMetadata,
|
|
||||||
JobResult,
|
JobResult,
|
||||||
ParentManager,
|
ParentManager,
|
||||||
SecondaryTierManager,
|
SecondaryTierManager,
|
||||||
TieringOffloadingMetrics,
|
TransferJob,
|
||||||
)
|
)
|
||||||
|
from vllm.v1.kv_offload.tiering.metrics import TieringMetricsTracker
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
@@ -72,11 +73,12 @@ class RequestState:
|
|||||||
req_context: ReqContext
|
req_context: ReqContext
|
||||||
pending_primary_stores: int = 0
|
pending_primary_stores: int = 0
|
||||||
is_finished: bool = False
|
is_finished: bool = False
|
||||||
request_level_tiers: set[SecondaryTierManager] | None = None
|
request_level_tiers: set[int] | None = None
|
||||||
sync_lookup_delay: float = 0.0
|
|
||||||
# time.monotonic() of this request's first deferred secondary-tier lookup;
|
|
||||||
# None once consumed (observed) or while no secondary lookup is pending.
|
class JobMetadata(NamedTuple):
|
||||||
secondary_lookup_start_time: float | None = None
|
transfer_job: TransferJob
|
||||||
|
tier_idx: int
|
||||||
|
|
||||||
|
|
||||||
class CPUPrimaryTierOffloadingManager(CPUOffloadingManager):
|
class CPUPrimaryTierOffloadingManager(CPUOffloadingManager):
|
||||||
@@ -132,31 +134,33 @@ class CPUPrimaryTierOffloadingManager(CPUOffloadingManager):
|
|||||||
|
|
||||||
class _SecondaryTierFacingParent(ParentManager):
|
class _SecondaryTierFacingParent(ParentManager):
|
||||||
"""Wrapper that implements ParentManager by delegating to the
|
"""Wrapper that implements ParentManager by delegating to the
|
||||||
TieringOffloadingManager with exclude_tier set to the origin tier."""
|
TieringOffloadingManager with exclude_tier_idx set to the origin tier."""
|
||||||
|
|
||||||
__slots__ = ("_m", "_origin")
|
__slots__ = ("_m", "_origin_idx")
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
manager: "TieringOffloadingManager",
|
manager: "TieringOffloadingManager",
|
||||||
tier: SecondaryTierManager,
|
tier_idx: int,
|
||||||
):
|
):
|
||||||
self._m = manager
|
self._m = manager
|
||||||
self._origin = tier
|
self._origin_idx = tier_idx
|
||||||
|
|
||||||
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
|
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
|
||||||
return self._m.on_new_request(req_context, exclude_tier=self._origin)
|
return self._m.on_new_request(req_context, exclude_tier_idx=self._origin_idx)
|
||||||
|
|
||||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
||||||
return self._m.lookup(key, req_context, exclude_tier=self._origin)
|
return self._m.lookup(key, req_context, exclude_tier_idx=self._origin_idx)
|
||||||
|
|
||||||
def create_store_job(
|
def create_store_job(
|
||||||
self, keys: Collection[OffloadKey], req_context: ReqContext
|
self, keys: Collection[OffloadKey], req_context: ReqContext
|
||||||
) -> JobMetadata:
|
) -> TransferJob:
|
||||||
return self._m.create_store_job(keys, req_context)
|
return self._m.create_store_job(keys, req_context, self._origin_idx)
|
||||||
|
|
||||||
def on_request_finished(self, req_context: ReqContext) -> None:
|
def on_request_finished(self, req_context: ReqContext) -> None:
|
||||||
return self._m.on_request_finished(req_context, exclude_tier=self._origin)
|
return self._m.on_request_finished(
|
||||||
|
req_context, exclude_tier_idx=self._origin_idx
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class TieringOffloadingManager(OffloadingManager):
|
class TieringOffloadingManager(OffloadingManager):
|
||||||
@@ -193,18 +197,23 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
|
|
||||||
self._job_id_counter: int = 0
|
self._job_id_counter: int = 0
|
||||||
# Job tracking: maps job_id to metadata for all in-flight transfers.
|
# Job tracking: maps job_id to metadata for all in-flight transfers.
|
||||||
# JobMetadata.is_promotion distinguishes direction:
|
# TransferJob.is_promotion distinguishes direction:
|
||||||
# True: secondary → primary (promotion)
|
# True: secondary → primary (promotion)
|
||||||
# False: primary → secondary (cascade)
|
# False: primary → secondary (cascade)
|
||||||
self._transfer_jobs: dict[JobId, JobMetadata] = {}
|
self._jobs: dict[JobId, JobMetadata] = {}
|
||||||
|
primary_view = self.primary_tier.get_kv_memoryview()
|
||||||
|
assert primary_view.strides is not None
|
||||||
|
self._metrics = TieringMetricsTracker(
|
||||||
|
tier_types=[tier.tier_type for tier in self.secondary_tiers],
|
||||||
|
num_primary_blocks=self.primary_tier._num_blocks,
|
||||||
|
primary_block_size=primary_view.strides[0],
|
||||||
|
)
|
||||||
|
|
||||||
# Pending promotion requests accumulated during lookup() calls; flushed
|
# Pending promotion requests accumulated during lookup() calls; flushed
|
||||||
# as one batched submit_load() per (tier, request) in on_schedule_end().
|
# as one batched submit_load() per (tier, request) in on_schedule_end().
|
||||||
# Outer key: tier. Inner key: req_context.req_id — the same ReqContext
|
# Outer key: tier index. Inner key: req_context.req_id — the same ReqContext
|
||||||
# object is reused for all block lookups of a given request per engine step.
|
# object is reused for all block lookups of a given request per engine step.
|
||||||
self._pending_load_submissions: dict[
|
self._pending_load_submissions: dict[int, dict[str, PendingPromotion]] = {}
|
||||||
SecondaryTierManager, dict[str, PendingPromotion]
|
|
||||||
] = {}
|
|
||||||
|
|
||||||
# Gate for once-per-step execution of _maybe_process_finished_jobs().
|
# Gate for once-per-step execution of _maybe_process_finished_jobs().
|
||||||
# Reset at the end of each step in on_schedule_end().
|
# Reset at the end of each step in on_schedule_end().
|
||||||
@@ -217,13 +226,13 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
|
|
||||||
# Cached ParentManager wrappers for each secondary tier.
|
# Cached ParentManager wrappers for each secondary tier.
|
||||||
self._tier_parents: dict[SecondaryTierManager, _SecondaryTierFacingParent] = {
|
self._tier_parents: dict[SecondaryTierManager, _SecondaryTierFacingParent] = {
|
||||||
tier: _SecondaryTierFacingParent(self, tier)
|
tier: _SecondaryTierFacingParent(self, tier_idx)
|
||||||
for tier in self.secondary_tiers
|
for tier_idx, tier in enumerate(self.secondary_tiers)
|
||||||
}
|
}
|
||||||
|
|
||||||
# Buffers manager-level observations (e.g. lookup delay) between
|
@property
|
||||||
# get_stats() calls; merged in and reset each time get_stats() runs.
|
def _transfer_jobs(self) -> dict[JobId, JobMetadata]:
|
||||||
self._stats = OffloadingConnectorStats()
|
return self._jobs
|
||||||
|
|
||||||
def _next_job_id(self) -> JobId:
|
def _next_job_id(self) -> JobId:
|
||||||
"""Generate a unique job ID for async transfer tracking."""
|
"""Generate a unique job ID for async transfer tracking."""
|
||||||
@@ -231,6 +240,14 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
self._job_id_counter += 1
|
self._job_id_counter += 1
|
||||||
return job_id
|
return job_id
|
||||||
|
|
||||||
|
def _register_job(self, transfer_job: TransferJob, tier_idx: int) -> None:
|
||||||
|
job_metadata = JobMetadata(transfer_job, tier_idx)
|
||||||
|
self._jobs[transfer_job.job_id] = job_metadata
|
||||||
|
self._metrics.on_job_registered(job_metadata)
|
||||||
|
|
||||||
|
def _pop_job(self, job_id: JobId) -> JobMetadata | None:
|
||||||
|
return self._jobs.pop(job_id, None)
|
||||||
|
|
||||||
def _maybe_process_finished_jobs(self):
|
def _maybe_process_finished_jobs(self):
|
||||||
"""
|
"""
|
||||||
Poll secondary tiers for completed jobs (at most once per step).
|
Poll secondary tiers for completed jobs (at most once per step).
|
||||||
@@ -247,13 +264,14 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
def _complete_promotion(
|
def _complete_promotion(
|
||||||
self, job_metadata: JobMetadata, completed_job: JobResult
|
self, job_metadata: JobMetadata, completed_job: JobResult
|
||||||
) -> None:
|
) -> None:
|
||||||
|
transfer_job = job_metadata.transfer_job
|
||||||
successful_keys = completed_job.successful_keys
|
successful_keys = completed_job.successful_keys
|
||||||
failed_keys: Collection[OffloadKey]
|
failed_keys: Collection[OffloadKey]
|
||||||
if completed_job.success:
|
if completed_job.success:
|
||||||
successful_keys = job_metadata.keys
|
successful_keys = transfer_job.keys
|
||||||
failed_keys = ()
|
failed_keys = ()
|
||||||
elif successful_keys:
|
elif successful_keys:
|
||||||
failed_keys_set = set(job_metadata.keys)
|
failed_keys_set = set(transfer_job.keys)
|
||||||
assert failed_keys_set.issuperset(successful_keys), (
|
assert failed_keys_set.issuperset(successful_keys), (
|
||||||
f"Finished promotion job_id {completed_job.job_id} "
|
f"Finished promotion job_id {completed_job.job_id} "
|
||||||
"reported unknown successful keys"
|
"reported unknown successful keys"
|
||||||
@@ -262,18 +280,18 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
failed_keys = failed_keys_set
|
failed_keys = failed_keys_set
|
||||||
else:
|
else:
|
||||||
successful_keys = ()
|
successful_keys = ()
|
||||||
failed_keys = job_metadata.keys
|
failed_keys = transfer_job.keys
|
||||||
|
|
||||||
if successful_keys:
|
if successful_keys:
|
||||||
self.primary_tier.complete_write(
|
self.primary_tier.complete_write(
|
||||||
successful_keys,
|
successful_keys,
|
||||||
job_metadata.req_context,
|
transfer_job.req_context,
|
||||||
True,
|
True,
|
||||||
)
|
)
|
||||||
if failed_keys:
|
if failed_keys:
|
||||||
self.primary_tier.complete_write(
|
self.primary_tier.complete_write(
|
||||||
failed_keys,
|
failed_keys,
|
||||||
job_metadata.req_context,
|
transfer_job.req_context,
|
||||||
False,
|
False,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -291,13 +309,19 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
for i, tier in enumerate(self.secondary_tiers):
|
for i, tier in enumerate(self.secondary_tiers):
|
||||||
for completed_job in tier.get_finished_jobs():
|
for completed_job in tier.get_finished_jobs():
|
||||||
job_id = completed_job.job_id
|
job_id = completed_job.job_id
|
||||||
job_metadata = self._transfer_jobs.pop(job_id, None)
|
job_metadata = self._pop_job(job_id)
|
||||||
assert job_metadata is not None, (
|
assert job_metadata is not None, (
|
||||||
f"Finished job_id {job_id} from tier #{i}"
|
f"Finished job_id {job_id} from tier #{i}"
|
||||||
f" ({tier.tier_type}) not in _transfer_jobs"
|
f" ({tier.tier_type}) not in _jobs"
|
||||||
)
|
)
|
||||||
|
assert job_metadata.tier_idx == i, (
|
||||||
|
f"Finished job_id {job_id} reported by tier #{i}"
|
||||||
|
f" but belongs to tier #{job_metadata.tier_idx}"
|
||||||
|
)
|
||||||
|
transfer_job = job_metadata.transfer_job
|
||||||
|
self._metrics.on_job_finished(job_metadata, completed_job)
|
||||||
|
|
||||||
if job_metadata.is_promotion:
|
if transfer_job.is_promotion:
|
||||||
# secondary→primary transfer (promotion) completed.
|
# secondary→primary transfer (promotion) completed.
|
||||||
# Make blocks available in primary tier.
|
# Make blocks available in primary tier.
|
||||||
self._complete_promotion(job_metadata, completed_job)
|
self._complete_promotion(job_metadata, completed_job)
|
||||||
@@ -305,7 +329,7 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
# primary→secondary transfer completed.
|
# primary→secondary transfer completed.
|
||||||
# Decrement ref_cnt on primary blocks.
|
# Decrement ref_cnt on primary blocks.
|
||||||
self.primary_tier.complete_read(
|
self.primary_tier.complete_read(
|
||||||
job_metadata.keys, job_metadata.req_context
|
transfer_job.keys, transfer_job.req_context
|
||||||
)
|
)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
@@ -314,7 +338,7 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
key: OffloadKey,
|
key: OffloadKey,
|
||||||
req_context: ReqContext,
|
req_context: ReqContext,
|
||||||
*,
|
*,
|
||||||
exclude_tier: SecondaryTierManager | None = None,
|
exclude_tier_idx: int | None = None,
|
||||||
) -> LookupResult:
|
) -> LookupResult:
|
||||||
"""
|
"""
|
||||||
Check whether a single block is offloaded and ready.
|
Check whether a single block is offloaded and ready.
|
||||||
@@ -343,73 +367,58 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
# in time for a promotion this lookup may initiate.
|
# in time for a promotion this lookup may initiate.
|
||||||
self._maybe_process_finished_jobs()
|
self._maybe_process_finished_jobs()
|
||||||
|
|
||||||
req_state = self._req_state.get(req_context.req_id)
|
start_time = time.monotonic()
|
||||||
|
|
||||||
primary_hit = self.primary_tier.lookup(key, req_context)
|
primary_hit = self.primary_tier.lookup(key, req_context)
|
||||||
|
lookup_duration = time.monotonic() - start_time
|
||||||
|
self._metrics.on_lookup(
|
||||||
|
req_context,
|
||||||
|
key,
|
||||||
|
self._metrics.primary_tier_label,
|
||||||
|
primary_hit,
|
||||||
|
lookup_duration,
|
||||||
|
)
|
||||||
if primary_hit is LookupResult.HIT:
|
if primary_hit is LookupResult.HIT:
|
||||||
return LookupResult.HIT
|
return LookupResult.HIT
|
||||||
if primary_hit is LookupResult.HIT_PENDING:
|
if primary_hit is LookupResult.HIT_PENDING:
|
||||||
return LookupResult.HIT_PENDING
|
return LookupResult.HIT_PENDING
|
||||||
|
|
||||||
lookup_start = time.monotonic()
|
|
||||||
any_retry = False
|
any_retry = False
|
||||||
for tier in self.secondary_tiers:
|
for i, tier in enumerate(self.secondary_tiers):
|
||||||
if tier is exclude_tier:
|
if i == exclude_tier_idx:
|
||||||
continue
|
continue
|
||||||
if not req_context.load_tier_filter.allows(tier.medium, tier.locality):
|
if not req_context.load_tier_filter.allows(tier.medium, tier.locality):
|
||||||
continue
|
continue
|
||||||
|
labelvalues = self._metrics.tier_label(i)
|
||||||
|
start_time = time.monotonic()
|
||||||
result = tier.lookup(key, req_context)
|
result = tier.lookup(key, req_context)
|
||||||
|
lookup_duration = time.monotonic() - start_time
|
||||||
if result is LookupResult.HIT:
|
if result is LookupResult.HIT:
|
||||||
promoted = self._initiate_promotion(tier, key, req_context)
|
self._metrics.on_lookup(
|
||||||
self._accumulate_lookup_sync_delay(req_state, lookup_start)
|
req_context,
|
||||||
if (
|
key,
|
||||||
req_state is not None
|
labelvalues,
|
||||||
and promoted
|
result,
|
||||||
and req_state.secondary_lookup_start_time is None
|
lookup_duration,
|
||||||
):
|
)
|
||||||
req_state.secondary_lookup_start_time = lookup_start
|
promoted = self._initiate_promotion(i, key, req_context)
|
||||||
return LookupResult.MISS if not promoted else LookupResult.RETRY
|
return LookupResult.MISS if not promoted else LookupResult.RETRY
|
||||||
if result is LookupResult.RETRY:
|
if result is LookupResult.RETRY:
|
||||||
any_retry = True
|
any_retry = True
|
||||||
|
self._metrics.on_lookup(
|
||||||
|
req_context,
|
||||||
|
key,
|
||||||
|
labelvalues,
|
||||||
|
result,
|
||||||
|
lookup_duration,
|
||||||
|
)
|
||||||
|
|
||||||
self._accumulate_lookup_sync_delay(req_state, lookup_start)
|
|
||||||
if any_retry:
|
if any_retry:
|
||||||
if req_state is not None and req_state.secondary_lookup_start_time is None:
|
|
||||||
req_state.secondary_lookup_start_time = lookup_start
|
|
||||||
return LookupResult.RETRY
|
return LookupResult.RETRY
|
||||||
return LookupResult.MISS
|
return LookupResult.MISS
|
||||||
|
|
||||||
def _accumulate_lookup_sync_delay(
|
|
||||||
self, req_state: RequestState | None, start_time: float
|
|
||||||
) -> None:
|
|
||||||
"""Accumulate secondary-tier lookup time until allocation or finish."""
|
|
||||||
if req_state is not None:
|
|
||||||
req_state.sync_lookup_delay += time.monotonic() - start_time
|
|
||||||
|
|
||||||
def _maybe_observe_lookup_sync_delay(self, req_state: RequestState) -> None:
|
|
||||||
delay = req_state.sync_lookup_delay
|
|
||||||
if delay == 0:
|
|
||||||
return
|
|
||||||
req_state.sync_lookup_delay = 0.0
|
|
||||||
self._stats.observe_histogram(
|
|
||||||
TieringOffloadingMetrics.LOOKUP_SYNC_DELAY,
|
|
||||||
delay,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _maybe_observe_lookup_async_delay(self, req_state: RequestState) -> None:
|
|
||||||
"""Flush a pending deferred secondary-tier lookup timer, if any."""
|
|
||||||
start_time = req_state.secondary_lookup_start_time
|
|
||||||
if start_time is None:
|
|
||||||
return
|
|
||||||
req_state.secondary_lookup_start_time = None
|
|
||||||
self._stats.observe_histogram(
|
|
||||||
TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY,
|
|
||||||
time.monotonic() - start_time,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _initiate_promotion(
|
def _initiate_promotion(
|
||||||
self,
|
self,
|
||||||
tier: SecondaryTierManager,
|
tier_idx: int,
|
||||||
key: OffloadKey,
|
key: OffloadKey,
|
||||||
req_context: ReqContext,
|
req_context: ReqContext,
|
||||||
) -> bool:
|
) -> bool:
|
||||||
@@ -423,7 +432,7 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
batched job.
|
batched job.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
tier: The secondary tier to promote from
|
tier_idx: The secondary tier index to promote from
|
||||||
key: Block to promote
|
key: Block to promote
|
||||||
req_context: Per-request context forwarded to primary.prepare_write().
|
req_context: Per-request context forwarded to primary.prepare_write().
|
||||||
|
|
||||||
@@ -439,13 +448,14 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
if primary_write_result is None:
|
if primary_write_result is None:
|
||||||
# Primary tier is full; caller should treat the block as unavailable
|
# Primary tier is full; caller should treat the block as unavailable
|
||||||
# rather than retrying indefinitely.
|
# rather than retrying indefinitely.
|
||||||
|
self._metrics.on_promotion_allocation_failure()
|
||||||
return False
|
return False
|
||||||
|
|
||||||
store_spec = primary_write_result.store_spec
|
store_spec = primary_write_result.store_spec
|
||||||
assert isinstance(store_spec, CPULoadStoreSpec)
|
assert isinstance(store_spec, CPULoadStoreSpec)
|
||||||
# Defer submit_load to on_schedule_end(). Group by (tier, request) so
|
# Defer submit_load to on_schedule_end(). Group by (tier, request) so
|
||||||
# each request's blocks are submitted as one batched job per tier.
|
# each request's blocks are submitted as one batched job per tier.
|
||||||
tier_pending = self._pending_load_submissions.setdefault(tier, {})
|
tier_pending = self._pending_load_submissions.setdefault(tier_idx, {})
|
||||||
ctx_id = req_context.req_id
|
ctx_id = req_context.req_id
|
||||||
if ctx_id not in tier_pending:
|
if ctx_id not in tier_pending:
|
||||||
tier_pending[ctx_id] = PendingPromotion(
|
tier_pending[ctx_id] = PendingPromotion(
|
||||||
@@ -465,17 +475,18 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
if not self._pending_load_submissions:
|
if not self._pending_load_submissions:
|
||||||
return
|
return
|
||||||
|
|
||||||
for tier, pending_by_ctx in self._pending_load_submissions.items():
|
for tier_idx, pending_by_ctx in self._pending_load_submissions.items():
|
||||||
|
tier = self.secondary_tiers[tier_idx]
|
||||||
for entry in pending_by_ctx.values():
|
for entry in pending_by_ctx.values():
|
||||||
job_id = self._next_job_id()
|
job_id = self._next_job_id()
|
||||||
job_metadata = JobMetadata(
|
job_metadata = TransferJob(
|
||||||
job_id=job_id,
|
job_id=job_id,
|
||||||
keys=entry.keys,
|
keys=entry.keys,
|
||||||
block_ids=np.array(entry.block_ids, dtype=np.int64),
|
block_ids=np.array(entry.block_ids, dtype=np.int64),
|
||||||
is_promotion=True,
|
is_promotion=True,
|
||||||
req_context=entry.req_context,
|
req_context=entry.req_context,
|
||||||
)
|
)
|
||||||
self._transfer_jobs[job_id] = job_metadata
|
self._register_job(job_metadata, tier_idx)
|
||||||
tier.submit_load(job_metadata)
|
tier.submit_load(job_metadata)
|
||||||
|
|
||||||
self._pending_load_submissions.clear()
|
self._pending_load_submissions.clear()
|
||||||
@@ -594,7 +605,7 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
self,
|
self,
|
||||||
keys: Sequence[OffloadKey],
|
keys: Sequence[OffloadKey],
|
||||||
req_context: ReqContext,
|
req_context: ReqContext,
|
||||||
request_level_tiers: set[SecondaryTierManager],
|
request_level_tiers: set[int],
|
||||||
) -> None:
|
) -> None:
|
||||||
"""
|
"""
|
||||||
For tiers that requested request-level policy, submit_store() for
|
For tiers that requested request-level policy, submit_store() for
|
||||||
@@ -609,8 +620,9 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
if not ready_keys:
|
if not ready_keys:
|
||||||
return
|
return
|
||||||
|
|
||||||
for tier in request_level_tiers:
|
for tier_idx in request_level_tiers:
|
||||||
job_metadata = self.create_store_job(ready_keys, req_context)
|
job_metadata = self.create_store_job(ready_keys, req_context, tier_idx)
|
||||||
|
tier = self.secondary_tiers[tier_idx]
|
||||||
tier.submit_store(job_metadata)
|
tier.submit_store(job_metadata)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
@@ -647,8 +659,8 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
# LoadStoreSpec AND to increment ref_cnt (protecting blocks from
|
# LoadStoreSpec AND to increment ref_cnt (protecting blocks from
|
||||||
# eviction during the async transfer). One prepare_read() call per
|
# eviction during the async transfer). One prepare_read() call per
|
||||||
# secondary tier.
|
# secondary tier.
|
||||||
for tier in self.secondary_tiers:
|
for tier_idx, tier in enumerate(self.secondary_tiers):
|
||||||
job_metadata = self.create_store_job(keys, req_context)
|
job_metadata = self.create_store_job(keys, req_context, tier_idx)
|
||||||
tier.submit_store(job_metadata)
|
tier.submit_store(job_metadata)
|
||||||
|
|
||||||
# Note: The async transfers are now in flight. Their completion is
|
# Note: The async transfers are now in flight. Their completion is
|
||||||
@@ -663,12 +675,13 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
self,
|
self,
|
||||||
keys: Collection[OffloadKey],
|
keys: Collection[OffloadKey],
|
||||||
req_context: ReqContext,
|
req_context: ReqContext,
|
||||||
) -> JobMetadata:
|
tier_idx: int = 0,
|
||||||
|
) -> TransferJob:
|
||||||
"""Pin blocks in the primary tier and create a tracked store job.
|
"""Pin blocks in the primary tier and create a tracked store job.
|
||||||
|
|
||||||
Calls prepare_read() to increment ref_cnt (protecting blocks
|
Calls prepare_read() to increment ref_cnt (protecting blocks
|
||||||
from eviction during the async transfer), allocates a job ID,
|
from eviction during the async transfer), allocates a job ID,
|
||||||
and registers the job in _transfer_jobs.
|
and registers the job in _jobs.
|
||||||
|
|
||||||
The caller is responsible for the actual data transfer and
|
The caller is responsible for the actual data transfer and
|
||||||
reporting completion via get_finished_jobs().
|
reporting completion via get_finished_jobs().
|
||||||
@@ -676,14 +689,14 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
primary_blocks_spec = self.primary_tier.prepare_read(keys, req_context)
|
primary_blocks_spec = self.primary_tier.prepare_read(keys, req_context)
|
||||||
assert isinstance(primary_blocks_spec, CPULoadStoreSpec)
|
assert isinstance(primary_blocks_spec, CPULoadStoreSpec)
|
||||||
job_id = self._next_job_id()
|
job_id = self._next_job_id()
|
||||||
job_metadata = JobMetadata(
|
job_metadata = TransferJob(
|
||||||
job_id=job_id,
|
job_id=job_id,
|
||||||
keys=keys,
|
keys=keys,
|
||||||
block_ids=primary_blocks_spec.block_ids,
|
block_ids=primary_blocks_spec.block_ids,
|
||||||
is_promotion=False,
|
is_promotion=False,
|
||||||
req_context=req_context,
|
req_context=req_context,
|
||||||
)
|
)
|
||||||
self._transfer_jobs[job_id] = job_metadata
|
self._register_job(job_metadata, tier_idx)
|
||||||
return job_metadata
|
return job_metadata
|
||||||
|
|
||||||
@override
|
@override
|
||||||
@@ -691,7 +704,7 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
self,
|
self,
|
||||||
req_context: ReqContext,
|
req_context: ReqContext,
|
||||||
*,
|
*,
|
||||||
exclude_tier: SecondaryTierManager | None = None,
|
exclude_tier_idx: int | None = None,
|
||||||
) -> RequestOffloadingContext:
|
) -> RequestOffloadingContext:
|
||||||
"""
|
"""
|
||||||
Query each secondary tier for its offload policy preference.
|
Query each secondary tier for its offload policy preference.
|
||||||
@@ -700,14 +713,15 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
Only stores REQUEST_LEVEL tier decisions for use in prepare_store.
|
Only stores REQUEST_LEVEL tier decisions for use in prepare_store.
|
||||||
"""
|
"""
|
||||||
state = RequestState(req_context=req_context)
|
state = RequestState(req_context=req_context)
|
||||||
for tier in self.secondary_tiers:
|
self._metrics.on_new_request(req_context)
|
||||||
if tier is exclude_tier:
|
for tier_idx, tier in enumerate(self.secondary_tiers):
|
||||||
|
if tier_idx == exclude_tier_idx:
|
||||||
continue
|
continue
|
||||||
tier_ctx = tier.on_new_request(req_context)
|
tier_ctx = tier.on_new_request(req_context)
|
||||||
if tier_ctx.policy == OffloadPolicy.REQUEST_LEVEL:
|
if tier_ctx.policy == OffloadPolicy.REQUEST_LEVEL:
|
||||||
if state.request_level_tiers is None:
|
if state.request_level_tiers is None:
|
||||||
state.request_level_tiers = set()
|
state.request_level_tiers = set()
|
||||||
state.request_level_tiers.add(tier)
|
state.request_level_tiers.add(tier_idx)
|
||||||
self._req_state[req_context.req_id] = state
|
self._req_state[req_context.req_id] = state
|
||||||
|
|
||||||
policy = (
|
policy = (
|
||||||
@@ -722,17 +736,17 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
self,
|
self,
|
||||||
req_context: ReqContext,
|
req_context: ReqContext,
|
||||||
*,
|
*,
|
||||||
exclude_tier: SecondaryTierManager | None = None,
|
exclude_tier_idx: int | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.primary_tier.on_request_finished(req_context)
|
self.primary_tier.on_request_finished(req_context)
|
||||||
state = self._req_state[req_context.req_id]
|
state = self._req_state[req_context.req_id]
|
||||||
state.is_finished = True
|
state.is_finished = True
|
||||||
self._maybe_finalize_request(req_context.req_id, exclude_tier)
|
self._maybe_finalize_request(req_context.req_id, exclude_tier_idx)
|
||||||
|
|
||||||
def _maybe_finalize_request(
|
def _maybe_finalize_request(
|
||||||
self,
|
self,
|
||||||
req_id: str,
|
req_id: str,
|
||||||
exclude_tier: SecondaryTierManager | None = None,
|
exclude_tier_idx: int | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Finalize secondary tiers once no more store cascades can be submitted.
|
"""Finalize secondary tiers once no more store cascades can be submitted.
|
||||||
|
|
||||||
@@ -746,12 +760,11 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
if state.pending_primary_stores != 0:
|
if state.pending_primary_stores != 0:
|
||||||
return
|
return
|
||||||
|
|
||||||
for tier in self.secondary_tiers:
|
for tier_idx, tier in enumerate(self.secondary_tiers):
|
||||||
if tier is exclude_tier:
|
if tier_idx == exclude_tier_idx:
|
||||||
continue
|
continue
|
||||||
tier.on_request_finished(state.req_context)
|
tier.on_request_finished(state.req_context)
|
||||||
self._maybe_observe_lookup_sync_delay(state)
|
self._metrics.on_request_finished(state.req_context)
|
||||||
self._maybe_observe_lookup_async_delay(state)
|
|
||||||
del self._req_state[req_id]
|
del self._req_state[req_id]
|
||||||
|
|
||||||
@override
|
@override
|
||||||
@@ -782,15 +795,14 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
state = self._req_state.get(req_id)
|
state = self._req_state.get(req_id)
|
||||||
if state is None:
|
if state is None:
|
||||||
continue
|
continue
|
||||||
self._maybe_observe_lookup_sync_delay(state)
|
self._metrics.on_request_allocated(state.req_context)
|
||||||
self._maybe_observe_lookup_async_delay(state)
|
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def has_pending_work(self) -> bool:
|
def has_pending_work(self) -> bool:
|
||||||
# In-flight primary<->secondary transfers (pending promotions are
|
# In-flight primary<->secondary transfers (pending promotions are
|
||||||
# translated to transfer jobs in on_schedule_end), plus any work the
|
# translated to transfer jobs in on_schedule_end), plus any work the
|
||||||
# secondary tiers themselves still have outstanding.
|
# secondary tiers themselves still have outstanding.
|
||||||
return bool(self._transfer_jobs) or any(
|
return bool(self._jobs) or any(
|
||||||
tier.has_pending_work() for tier in self.secondary_tiers
|
tier.has_pending_work() for tier in self.secondary_tiers
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -825,11 +837,13 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
# All tier I/O has stopped; consume their completion notifications
|
# All tier I/O has stopped; consume their completion notifications
|
||||||
# so manager bookkeeping is consistent before the primary reset.
|
# so manager bookkeeping is consistent before the primary reset.
|
||||||
self._process_finished_jobs()
|
self._process_finished_jobs()
|
||||||
|
assert not self._jobs
|
||||||
|
|
||||||
# Deferred promotion submissions reserve primary slots that the
|
# Deferred promotion submissions reserve primary slots that the
|
||||||
# reset below invalidates; their submit_load() has not yet been
|
# reset below invalidates; their submit_load() has not yet been
|
||||||
# called so no tier I/O is touching that memory.
|
# called so no tier I/O is touching that memory.
|
||||||
self._pending_load_submissions.clear()
|
self._pending_load_submissions.clear()
|
||||||
|
self._metrics.assert_idle()
|
||||||
|
|
||||||
finished_req_ids = []
|
finished_req_ids = []
|
||||||
for req_id, state in self._req_state.items():
|
for req_id, state in self._req_state.items():
|
||||||
@@ -838,8 +852,7 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
continue
|
continue
|
||||||
for tier in self.secondary_tiers:
|
for tier in self.secondary_tiers:
|
||||||
tier.on_request_finished(state.req_context)
|
tier.on_request_finished(state.req_context)
|
||||||
self._maybe_observe_lookup_sync_delay(state)
|
self._metrics.on_request_finished(state.req_context)
|
||||||
self._maybe_observe_lookup_async_delay(state)
|
|
||||||
finished_req_ids.append(req_id)
|
finished_req_ids.append(req_id)
|
||||||
|
|
||||||
self.primary_tier.reset_cache()
|
self.primary_tier.reset_cache()
|
||||||
@@ -855,6 +868,13 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
if stats is not None and stats.is_empty():
|
if stats is not None and stats.is_empty():
|
||||||
stats = None
|
stats = None
|
||||||
|
|
||||||
|
metrics_stats = self._metrics.take_stats()
|
||||||
|
if metrics_stats is not None:
|
||||||
|
if stats is None:
|
||||||
|
stats = metrics_stats
|
||||||
|
else:
|
||||||
|
stats.aggregate(metrics_stats)
|
||||||
|
|
||||||
for tier in self.secondary_tiers:
|
for tier in self.secondary_tiers:
|
||||||
tier_stats = tier.get_stats()
|
tier_stats = tier.get_stats()
|
||||||
if tier_stats is None or tier_stats.is_empty():
|
if tier_stats is None or tier_stats.is_empty():
|
||||||
@@ -864,13 +884,6 @@ class TieringOffloadingManager(OffloadingManager):
|
|||||||
else:
|
else:
|
||||||
stats.aggregate(tier_stats)
|
stats.aggregate(tier_stats)
|
||||||
|
|
||||||
if not self._stats.is_empty():
|
|
||||||
if stats is None:
|
|
||||||
stats = self._stats
|
|
||||||
else:
|
|
||||||
stats.aggregate(self._stats)
|
|
||||||
self._stats = OffloadingConnectorStats()
|
|
||||||
|
|
||||||
return stats
|
return stats
|
||||||
|
|
||||||
@override
|
@override
|
||||||
|
|||||||
@@ -0,0 +1,269 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
import functools
|
||||||
|
import time
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from typing import Protocol
|
||||||
|
|
||||||
|
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||||
|
OffloadingConnectorStats,
|
||||||
|
)
|
||||||
|
from vllm.v1.kv_offload.base import LookupResult, OffloadKey, ReqContext
|
||||||
|
from vllm.v1.kv_offload.tiering.base import (
|
||||||
|
JobResult,
|
||||||
|
TieringOffloadingMetrics,
|
||||||
|
TransferJob,
|
||||||
|
)
|
||||||
|
|
||||||
|
TierLabel = tuple[str]
|
||||||
|
PRIMARY_TIER_LABEL: TierLabel = ("0:primary",)
|
||||||
|
|
||||||
|
|
||||||
|
class _JobMetadataLike(Protocol):
|
||||||
|
@property
|
||||||
|
def transfer_job(self) -> TransferJob: ...
|
||||||
|
|
||||||
|
@property
|
||||||
|
def tier_idx(self) -> int: ...
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(slots=True)
|
||||||
|
class _RequestMetricsState:
|
||||||
|
observed_lookups: dict[TierLabel, dict[OffloadKey, float | None]] | None = field(
|
||||||
|
default_factory=dict
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class _TierState:
|
||||||
|
active_promotion_count: int = 0
|
||||||
|
active_cascade_count: int = 0
|
||||||
|
primary_write_block_count: int = 0
|
||||||
|
primary_read_block_count: int = 0
|
||||||
|
|
||||||
|
|
||||||
|
class TieringMetricsTracker:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
tier_types: list[str],
|
||||||
|
num_primary_blocks: int,
|
||||||
|
primary_block_size: int,
|
||||||
|
) -> None:
|
||||||
|
self._tier_types = tier_types
|
||||||
|
self._num_primary_blocks = num_primary_blocks
|
||||||
|
self._primary_block_size = primary_block_size
|
||||||
|
self._request_states: dict[str, _RequestMetricsState] = {}
|
||||||
|
self._tier_states = [_TierState() for _ in tier_types]
|
||||||
|
self._stats = OffloadingConnectorStats()
|
||||||
|
|
||||||
|
@functools.cache # noqa: B019
|
||||||
|
def tier_label(self, tier_idx: int) -> TierLabel:
|
||||||
|
return (f"{tier_idx + 1}:{self._tier_types[tier_idx]}",)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def primary_tier_label(self) -> TierLabel:
|
||||||
|
return PRIMARY_TIER_LABEL
|
||||||
|
|
||||||
|
def on_new_request(self, req_context: ReqContext) -> None:
|
||||||
|
self._request_states[req_context.req_id] = _RequestMetricsState()
|
||||||
|
|
||||||
|
def on_request_allocated(self, req_context: ReqContext) -> None:
|
||||||
|
state = self._request_states.get(req_context.req_id)
|
||||||
|
if state is None:
|
||||||
|
return
|
||||||
|
state.observed_lookups = None
|
||||||
|
|
||||||
|
def on_request_finished(self, req_context: ReqContext) -> None:
|
||||||
|
self._request_states.pop(req_context.req_id, None)
|
||||||
|
|
||||||
|
def on_lookup(
|
||||||
|
self,
|
||||||
|
req_context: ReqContext,
|
||||||
|
key: OffloadKey,
|
||||||
|
tier_label: TierLabel,
|
||||||
|
result: LookupResult,
|
||||||
|
lookup_duration: float,
|
||||||
|
) -> None:
|
||||||
|
state = self._request_states.get(req_context.req_id)
|
||||||
|
if state is None:
|
||||||
|
state = _RequestMetricsState()
|
||||||
|
self._request_states[req_context.req_id] = state
|
||||||
|
|
||||||
|
if state.observed_lookups is not None and result in (
|
||||||
|
LookupResult.HIT,
|
||||||
|
LookupResult.MISS,
|
||||||
|
):
|
||||||
|
observed = state.observed_lookups.setdefault(tier_label, {})
|
||||||
|
start_time = observed.get(key)
|
||||||
|
if key not in observed or start_time is not None:
|
||||||
|
observed[key] = None
|
||||||
|
self._observe_resolved_lookup(
|
||||||
|
tier_label,
|
||||||
|
result,
|
||||||
|
lookup_duration,
|
||||||
|
start_time,
|
||||||
|
)
|
||||||
|
elif result is LookupResult.RETRY and state.observed_lookups is not None:
|
||||||
|
observed = state.observed_lookups.setdefault(tier_label, {})
|
||||||
|
observed.setdefault(key, time.monotonic() - lookup_duration)
|
||||||
|
|
||||||
|
def on_job_registered(self, job_metadata: _JobMetadataLike) -> None:
|
||||||
|
transfer_job = job_metadata.transfer_job
|
||||||
|
state = self._tier_states[job_metadata.tier_idx]
|
||||||
|
block_count = len(transfer_job.block_ids)
|
||||||
|
if transfer_job.is_promotion:
|
||||||
|
state.active_promotion_count += 1
|
||||||
|
state.primary_write_block_count += block_count
|
||||||
|
else:
|
||||||
|
state.active_cascade_count += 1
|
||||||
|
state.primary_read_block_count += block_count
|
||||||
|
|
||||||
|
def on_job_finished(
|
||||||
|
self, job_metadata: _JobMetadataLike, result: JobResult
|
||||||
|
) -> None:
|
||||||
|
self._observe_finished_job_stats(job_metadata, result)
|
||||||
|
self._decrement_tier_state(job_metadata)
|
||||||
|
|
||||||
|
def on_promotion_allocation_failure(self) -> None:
|
||||||
|
self._stats.increase_counter(
|
||||||
|
TieringOffloadingMetrics.PROMOTION_ALLOCATION_FAILURES
|
||||||
|
)
|
||||||
|
|
||||||
|
def take_stats(self) -> OffloadingConnectorStats | None:
|
||||||
|
active_transfer_stats = OffloadingConnectorStats()
|
||||||
|
self._observe_active_transfer_stats(active_transfer_stats)
|
||||||
|
|
||||||
|
stats = None
|
||||||
|
if not active_transfer_stats.is_empty():
|
||||||
|
stats = active_transfer_stats
|
||||||
|
if not self._stats.is_empty():
|
||||||
|
if stats is None:
|
||||||
|
stats = self._stats
|
||||||
|
else:
|
||||||
|
stats.aggregate(self._stats)
|
||||||
|
self._stats = OffloadingConnectorStats()
|
||||||
|
return stats
|
||||||
|
|
||||||
|
def assert_idle(self) -> None:
|
||||||
|
assert all(
|
||||||
|
state.active_promotion_count == 0
|
||||||
|
and state.active_cascade_count == 0
|
||||||
|
and state.primary_write_block_count == 0
|
||||||
|
and state.primary_read_block_count == 0
|
||||||
|
for state in self._tier_states
|
||||||
|
)
|
||||||
|
|
||||||
|
def _decrement_tier_state(self, job_metadata: _JobMetadataLike) -> None:
|
||||||
|
transfer_job = job_metadata.transfer_job
|
||||||
|
state = self._tier_states[job_metadata.tier_idx]
|
||||||
|
block_count = len(transfer_job.block_ids)
|
||||||
|
if transfer_job.is_promotion:
|
||||||
|
assert state.active_promotion_count > 0
|
||||||
|
state.active_promotion_count -= 1
|
||||||
|
state.primary_write_block_count -= block_count
|
||||||
|
assert state.primary_write_block_count >= 0
|
||||||
|
else:
|
||||||
|
assert state.active_cascade_count > 0
|
||||||
|
state.active_cascade_count -= 1
|
||||||
|
state.primary_read_block_count -= block_count
|
||||||
|
assert state.primary_read_block_count >= 0
|
||||||
|
|
||||||
|
def _observe_finished_job_stats(
|
||||||
|
self,
|
||||||
|
job_metadata: _JobMetadataLike,
|
||||||
|
completed_job: JobResult,
|
||||||
|
) -> None:
|
||||||
|
transfer_job = job_metadata.transfer_job
|
||||||
|
labelvalues = self.tier_label(job_metadata.tier_idx)
|
||||||
|
completed_key_count = len(transfer_job.keys)
|
||||||
|
if not completed_job.success:
|
||||||
|
failure_metric = (
|
||||||
|
TieringOffloadingMetrics.PROMOTION_JOB_FAILURES
|
||||||
|
if transfer_job.is_promotion
|
||||||
|
else TieringOffloadingMetrics.CASCADE_JOB_FAILURES
|
||||||
|
)
|
||||||
|
self._stats.increase_counter(failure_metric, labelvalues=labelvalues)
|
||||||
|
if transfer_job.is_promotion and completed_job.successful_keys:
|
||||||
|
completed_key_count = len(completed_job.successful_keys)
|
||||||
|
else:
|
||||||
|
return
|
||||||
|
|
||||||
|
bytes_metric = (
|
||||||
|
TieringOffloadingMetrics.READ_BYTES
|
||||||
|
if transfer_job.is_promotion
|
||||||
|
else TieringOffloadingMetrics.WRITE_BYTES
|
||||||
|
)
|
||||||
|
time_metric = (
|
||||||
|
TieringOffloadingMetrics.READ_TIME
|
||||||
|
if transfer_job.is_promotion
|
||||||
|
else TieringOffloadingMetrics.WRITE_TIME
|
||||||
|
)
|
||||||
|
transfer_size = completed_key_count * self._primary_block_size
|
||||||
|
self._stats.increase_counter(bytes_metric, transfer_size, labelvalues)
|
||||||
|
if completed_job.transfer_time is not None:
|
||||||
|
self._stats.increase_counter(
|
||||||
|
time_metric, completed_job.transfer_time, labelvalues
|
||||||
|
)
|
||||||
|
|
||||||
|
def _observe_active_transfer_stats(self, stats: OffloadingConnectorStats) -> None:
|
||||||
|
for tier_idx, state in enumerate(self._tier_states):
|
||||||
|
labelvalues = self.tier_label(tier_idx)
|
||||||
|
write_usage = (
|
||||||
|
state.primary_write_block_count / self._num_primary_blocks
|
||||||
|
if self._num_primary_blocks > 0
|
||||||
|
else 0.0
|
||||||
|
)
|
||||||
|
read_usage = (
|
||||||
|
state.primary_read_block_count / self._num_primary_blocks
|
||||||
|
if self._num_primary_blocks > 0
|
||||||
|
else 0.0
|
||||||
|
)
|
||||||
|
stats.set_gauge(
|
||||||
|
TieringOffloadingMetrics.PRIMARY_WRITE_USAGE_PERC,
|
||||||
|
write_usage,
|
||||||
|
labelvalues,
|
||||||
|
)
|
||||||
|
stats.set_gauge(
|
||||||
|
TieringOffloadingMetrics.PRIMARY_READ_USAGE_PERC,
|
||||||
|
read_usage,
|
||||||
|
labelvalues,
|
||||||
|
)
|
||||||
|
stats.set_gauge(
|
||||||
|
TieringOffloadingMetrics.ACTIVE_PROMOTION_JOBS,
|
||||||
|
state.active_promotion_count,
|
||||||
|
labelvalues,
|
||||||
|
)
|
||||||
|
stats.set_gauge(
|
||||||
|
TieringOffloadingMetrics.ACTIVE_CASCADE_JOBS,
|
||||||
|
state.active_cascade_count,
|
||||||
|
labelvalues,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _observe_resolved_lookup(
|
||||||
|
self,
|
||||||
|
tier_label: TierLabel,
|
||||||
|
result: LookupResult,
|
||||||
|
lookup_duration: float,
|
||||||
|
async_start_time: float | None,
|
||||||
|
) -> None:
|
||||||
|
self._stats.increase_counter(
|
||||||
|
TieringOffloadingMetrics.BLOCK_QUERIES,
|
||||||
|
labelvalues=tier_label,
|
||||||
|
)
|
||||||
|
if result is LookupResult.HIT:
|
||||||
|
self._stats.increase_counter(
|
||||||
|
TieringOffloadingMetrics.BLOCK_HITS,
|
||||||
|
labelvalues=tier_label,
|
||||||
|
)
|
||||||
|
self._stats.observe_histogram(
|
||||||
|
TieringOffloadingMetrics.LOOKUP_SYNC_DELAY,
|
||||||
|
lookup_duration,
|
||||||
|
tier_label,
|
||||||
|
)
|
||||||
|
if async_start_time is not None:
|
||||||
|
self._stats.observe_histogram(
|
||||||
|
TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY,
|
||||||
|
time.monotonic() - async_start_time,
|
||||||
|
tier_label,
|
||||||
|
)
|
||||||
@@ -22,11 +22,11 @@ from vllm.v1.kv_offload.file_mapper import FileMapper
|
|||||||
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager
|
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager
|
||||||
from vllm.v1.kv_offload.tiering.base import (
|
from vllm.v1.kv_offload.tiering.base import (
|
||||||
JobId,
|
JobId,
|
||||||
JobMetadata,
|
|
||||||
JobResult,
|
JobResult,
|
||||||
RequestOffloadingContext,
|
RequestOffloadingContext,
|
||||||
ScheduleEndContext,
|
ScheduleEndContext,
|
||||||
SecondaryTierManager,
|
SecondaryTierManager,
|
||||||
|
TransferJob,
|
||||||
)
|
)
|
||||||
from vllm.v1.kv_offload.tiering.obj.config import ObjStoreConfig
|
from vllm.v1.kv_offload.tiering.obj.config import ObjStoreConfig
|
||||||
|
|
||||||
@@ -267,7 +267,11 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
|
|||||||
self._pending_results.append(JobResult(job_id=job_id, success=False))
|
self._pending_results.append(JobResult(job_id=job_id, success=False))
|
||||||
return
|
return
|
||||||
|
|
||||||
self._transfers[job_id] = TransferEntry(xfer_handle, files_desc, obj_handle)
|
self._transfers[job_id] = TransferEntry(
|
||||||
|
xfer_handle=xfer_handle,
|
||||||
|
files_desc=files_desc,
|
||||||
|
obj_handle=obj_handle,
|
||||||
|
)
|
||||||
|
|
||||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
||||||
result = self._lookup_manager.lookup(key, req_context)
|
result = self._lookup_manager.lookup(key, req_context)
|
||||||
@@ -275,7 +279,7 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
|
|||||||
return LookupResult.RETRY
|
return LookupResult.RETRY
|
||||||
return LookupResult.HIT if result else LookupResult.MISS
|
return LookupResult.HIT if result else LookupResult.MISS
|
||||||
|
|
||||||
def submit_store(self, job_metadata: JobMetadata) -> None:
|
def submit_store(self, job_metadata: TransferJob) -> None:
|
||||||
if self.events is not None:
|
if self.events is not None:
|
||||||
self._store_job_keys[job_metadata.job_id] = list(job_metadata.keys)
|
self._store_job_keys[job_metadata.job_id] = list(job_metadata.keys)
|
||||||
obj_keys = (self._file_mapper.get_file_name(k) for k in job_metadata.keys)
|
obj_keys = (self._file_mapper.get_file_name(k) for k in job_metadata.keys)
|
||||||
@@ -283,7 +287,7 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
|
|||||||
job_metadata.job_id, job_metadata.block_ids, obj_keys, NIXL_WRITE
|
job_metadata.job_id, job_metadata.block_ids, obj_keys, NIXL_WRITE
|
||||||
)
|
)
|
||||||
|
|
||||||
def submit_load(self, job_metadata: JobMetadata) -> None:
|
def submit_load(self, job_metadata: TransferJob) -> None:
|
||||||
self._load_job_keys[job_metadata.job_id] = list(job_metadata.keys)
|
self._load_job_keys[job_metadata.job_id] = list(job_metadata.keys)
|
||||||
obj_keys = (self._file_mapper.get_file_name(k) for k in job_metadata.keys)
|
obj_keys = (self._file_mapper.get_file_name(k) for k in job_metadata.keys)
|
||||||
self._submit_transfer(
|
self._submit_transfer(
|
||||||
@@ -317,6 +321,11 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
|
|||||||
success = False
|
success = False
|
||||||
logger.warning("transfer failed job=%d state=%s", job_id, state)
|
logger.warning("transfer failed job=%d state=%s", job_id, state)
|
||||||
|
|
||||||
|
transfer_time = None
|
||||||
|
if success:
|
||||||
|
telemetry = self._agent.get_xfer_telemetry(entry.xfer_handle)
|
||||||
|
transfer_time = telemetry.xferDuration / 1e6
|
||||||
|
|
||||||
try:
|
try:
|
||||||
self._agent.release_xfer_handle(entry.xfer_handle)
|
self._agent.release_xfer_handle(entry.xfer_handle)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
@@ -341,7 +350,13 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
|
|||||||
logger.warning("deregister_memory failed for job %d: %s", job_id, exc)
|
logger.warning("deregister_memory failed for job %d: %s", job_id, exc)
|
||||||
|
|
||||||
del self._transfers[job_id]
|
del self._transfers[job_id]
|
||||||
self._pending_results.append(JobResult(job_id=job_id, success=success))
|
self._pending_results.append(
|
||||||
|
JobResult(
|
||||||
|
job_id=job_id,
|
||||||
|
success=success,
|
||||||
|
transfer_time=transfer_time,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
def get_finished_jobs(self) -> Iterable[JobResult]:
|
def get_finished_jobs(self) -> Iterable[JobResult]:
|
||||||
"""Poll transfers; a failed promotion marks its cached verdicts False
|
"""Poll transfers; a failed promotion marks its cached verdicts False
|
||||||
|
|||||||
@@ -28,9 +28,9 @@ from vllm.v1.kv_offload.base import (
|
|||||||
)
|
)
|
||||||
from vllm.v1.kv_offload.file_mapper import FileMapper
|
from vllm.v1.kv_offload.file_mapper import FileMapper
|
||||||
from vllm.v1.kv_offload.tiering.base import (
|
from vllm.v1.kv_offload.tiering.base import (
|
||||||
JobMetadata,
|
|
||||||
JobResult,
|
JobResult,
|
||||||
SecondaryTierManager,
|
SecondaryTierManager,
|
||||||
|
TransferJob,
|
||||||
)
|
)
|
||||||
from vllm.v1.kv_offload.tiering.p2p.control import ControlTransport, ZmqTransport
|
from vllm.v1.kv_offload.tiering.p2p.control import ControlTransport, ZmqTransport
|
||||||
from vllm.v1.kv_offload.tiering.p2p.data import DataTransport, NixlTransport
|
from vllm.v1.kv_offload.tiering.p2p.data import DataTransport, NixlTransport
|
||||||
@@ -411,7 +411,7 @@ class P2PSecondaryTierManager(SecondaryTierManager):
|
|||||||
return
|
return
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def submit_store(self, job_metadata: JobMetadata) -> None:
|
def submit_store(self, job_metadata: TransferJob) -> None:
|
||||||
job_id = job_metadata.job_id
|
job_id = job_metadata.job_id
|
||||||
keys = list(job_metadata.keys)
|
keys = list(job_metadata.keys)
|
||||||
block_ids = job_metadata.block_ids
|
block_ids = job_metadata.block_ids
|
||||||
@@ -470,7 +470,7 @@ class P2PSecondaryTierManager(SecondaryTierManager):
|
|||||||
)
|
)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def submit_load(self, job_metadata: JobMetadata) -> None:
|
def submit_load(self, job_metadata: TransferJob) -> None:
|
||||||
job_id = job_metadata.job_id
|
job_id = job_metadata.job_id
|
||||||
keys = list(job_metadata.keys)
|
keys = list(job_metadata.keys)
|
||||||
block_ids = job_metadata.block_ids
|
block_ids = job_metadata.block_ids
|
||||||
|
|||||||
@@ -61,6 +61,8 @@ from typing_extensions import override
|
|||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.v1.kv_offload.base import (
|
from vllm.v1.kv_offload.base import (
|
||||||
CanonicalKVCaches,
|
CanonicalKVCaches,
|
||||||
|
OffloadingCounterMetadata,
|
||||||
|
OffloadingGaugeMetadata,
|
||||||
OffloadingHistogramMetadata,
|
OffloadingHistogramMetadata,
|
||||||
OffloadingManager,
|
OffloadingManager,
|
||||||
OffloadingMetricMetadata,
|
OffloadingMetricMetadata,
|
||||||
@@ -103,10 +105,10 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
|||||||
metrics[TieringOffloadingMetrics.LOOKUP_SYNC_DELAY] = (
|
metrics[TieringOffloadingMetrics.LOOKUP_SYNC_DELAY] = (
|
||||||
OffloadingHistogramMetadata(
|
OffloadingHistogramMetadata(
|
||||||
documentation=(
|
documentation=(
|
||||||
"Histogram of total blocking time spent querying secondary "
|
"Histogram of blocking time spent in a per-block tier lookup "
|
||||||
"tiers for a request, accumulated from first lookup until "
|
"that resolved as a hit or miss, labeled by tier, in seconds."
|
||||||
"the request is allocated or finishes, in seconds."
|
|
||||||
),
|
),
|
||||||
|
labelnames=("tier",),
|
||||||
buckets=(
|
buckets=(
|
||||||
0.00001,
|
0.00001,
|
||||||
0.00005,
|
0.00005,
|
||||||
@@ -125,10 +127,11 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
|||||||
metrics[TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY] = (
|
metrics[TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY] = (
|
||||||
OffloadingHistogramMetadata(
|
OffloadingHistogramMetadata(
|
||||||
documentation=(
|
documentation=(
|
||||||
"Histogram of wall-clock time from a request's first deferred "
|
"Histogram of wall-clock time from a per-block tier lookup "
|
||||||
"secondary-tier lookup until the request is allocated or "
|
"first returning retry until that same tier lookup resolves "
|
||||||
"finishes, in seconds."
|
"as a hit or miss, labeled by tier, in seconds."
|
||||||
),
|
),
|
||||||
|
labelnames=("tier",),
|
||||||
buckets=(
|
buckets=(
|
||||||
0.0001,
|
0.0001,
|
||||||
0.0005,
|
0.0005,
|
||||||
@@ -144,6 +147,100 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
metrics[TieringOffloadingMetrics.READ_BYTES] = OffloadingCounterMetadata(
|
||||||
|
documentation=(
|
||||||
|
"Total bytes read from secondary tiers into the primary tier, "
|
||||||
|
"labeled by tier."
|
||||||
|
),
|
||||||
|
labelnames=("tier",),
|
||||||
|
)
|
||||||
|
metrics[TieringOffloadingMetrics.READ_TIME] = OffloadingCounterMetadata(
|
||||||
|
documentation=(
|
||||||
|
"Total time spent reading from secondary tiers into the primary "
|
||||||
|
"tier, in seconds, labeled by tier."
|
||||||
|
),
|
||||||
|
labelnames=("tier",),
|
||||||
|
)
|
||||||
|
metrics[TieringOffloadingMetrics.WRITE_BYTES] = OffloadingCounterMetadata(
|
||||||
|
documentation=(
|
||||||
|
"Total bytes written from the primary tier to secondary tiers, "
|
||||||
|
"labeled by tier."
|
||||||
|
),
|
||||||
|
labelnames=("tier",),
|
||||||
|
)
|
||||||
|
metrics[TieringOffloadingMetrics.WRITE_TIME] = OffloadingCounterMetadata(
|
||||||
|
documentation=(
|
||||||
|
"Total time spent writing from the primary tier to secondary "
|
||||||
|
"tiers, in seconds, labeled by tier."
|
||||||
|
),
|
||||||
|
labelnames=("tier",),
|
||||||
|
)
|
||||||
|
metrics[TieringOffloadingMetrics.PROMOTION_JOB_FAILURES] = (
|
||||||
|
OffloadingCounterMetadata(
|
||||||
|
documentation=(
|
||||||
|
"Number of failed secondary-tier promotion jobs, labeled by tier."
|
||||||
|
),
|
||||||
|
labelnames=("tier",),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
metrics[TieringOffloadingMetrics.CASCADE_JOB_FAILURES] = (
|
||||||
|
OffloadingCounterMetadata(
|
||||||
|
documentation=(
|
||||||
|
"Number of failed secondary-tier cascade jobs, labeled by tier."
|
||||||
|
),
|
||||||
|
labelnames=("tier",),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
metrics[TieringOffloadingMetrics.BLOCK_QUERIES] = OffloadingCounterMetadata(
|
||||||
|
documentation=(
|
||||||
|
"Number of block lookup queries sent to a tier, labeled by tier."
|
||||||
|
),
|
||||||
|
labelnames=("tier",),
|
||||||
|
)
|
||||||
|
metrics[TieringOffloadingMetrics.BLOCK_HITS] = OffloadingCounterMetadata(
|
||||||
|
documentation="Number of block lookup hits in a tier, labeled by tier.",
|
||||||
|
labelnames=("tier",),
|
||||||
|
)
|
||||||
|
metrics[TieringOffloadingMetrics.PROMOTION_ALLOCATION_FAILURES] = (
|
||||||
|
OffloadingCounterMetadata(
|
||||||
|
documentation=(
|
||||||
|
"Number of promotion attempts that failed because the "
|
||||||
|
"primary tier could not allocate space."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
metrics[TieringOffloadingMetrics.PRIMARY_WRITE_USAGE_PERC] = (
|
||||||
|
OffloadingGaugeMetadata(
|
||||||
|
documentation=(
|
||||||
|
"Current fraction of primary-tier space used by writes from "
|
||||||
|
"secondary tiers, labeled by tier."
|
||||||
|
),
|
||||||
|
labelnames=("tier",),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
metrics[TieringOffloadingMetrics.PRIMARY_READ_USAGE_PERC] = (
|
||||||
|
OffloadingGaugeMetadata(
|
||||||
|
documentation=(
|
||||||
|
"Current fraction of primary-tier space used by reads to "
|
||||||
|
"secondary tiers, labeled by tier."
|
||||||
|
),
|
||||||
|
labelnames=("tier",),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
metrics[TieringOffloadingMetrics.ACTIVE_PROMOTION_JOBS] = (
|
||||||
|
OffloadingGaugeMetadata(
|
||||||
|
documentation=(
|
||||||
|
"Number of active secondary-tier promotion jobs, labeled by tier."
|
||||||
|
),
|
||||||
|
labelnames=("tier",),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
metrics[TieringOffloadingMetrics.ACTIVE_CASCADE_JOBS] = OffloadingGaugeMetadata(
|
||||||
|
documentation=(
|
||||||
|
"Number of active secondary-tier cascade jobs, labeled by tier."
|
||||||
|
),
|
||||||
|
labelnames=("tier",),
|
||||||
|
)
|
||||||
secondary_tier_configs = extra_config.get("secondary_tiers", [])
|
secondary_tier_configs = extra_config.get("secondary_tiers", [])
|
||||||
if not isinstance(secondary_tier_configs, list):
|
if not isinstance(secondary_tier_configs, list):
|
||||||
raise ValueError("secondary_tiers must be a list of tier configurations")
|
raise ValueError("secondary_tiers must be a list of tier configurations")
|
||||||
|
|||||||
Reference in New Issue
Block a user