[KV Offloading] Add tiering metric plumbing (#45959)
Signed-off-by: srinivas_oo7 <sklinkedin0120@gmail.com> Co-authored-by: srinivas_oo7 <sklinkedin0120@gmail.com>
This commit is contained in:
co-authored by
srinivas_oo7
parent
3554ada5d8
commit
091bc1026e
@@ -17,18 +17,29 @@ from unittest.mock import MagicMock
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
OffloadingConnectorStats,
|
||||
)
|
||||
from vllm.v1.kv_offload.base import (
|
||||
OffloadingCounterMetadata,
|
||||
OffloadKey,
|
||||
OffloadPolicy,
|
||||
ReqContext,
|
||||
RequestOffloadingContext,
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.base import (
|
||||
JobMetadata,
|
||||
JobResult,
|
||||
SecondaryTierManager,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.example.manager import ExampleSecondaryTierManager
|
||||
from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory
|
||||
from vllm.v1.kv_offload.tiering.manager import (
|
||||
CPUPrimaryTierOffloadingManager,
|
||||
TieringOffloadingManager,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.spec import TieringOffloadingSpec
|
||||
|
||||
_CTX = ReqContext(req_id="test")
|
||||
_MOCK_OFFLOADING_SPEC = MagicMock()
|
||||
@@ -63,6 +74,102 @@ def count_hits(manager, keys: list[OffloadKey]) -> int | None:
|
||||
return count
|
||||
|
||||
|
||||
class MetricsSecondaryTierManager(SecondaryTierManager):
|
||||
"""Test-only secondary tier that declares and emits one labeled metric."""
|
||||
|
||||
MY_TIER_METRIC = "my_tier_metric"
|
||||
|
||||
@classmethod
|
||||
def build_metric_definitions(cls, extra_config):
|
||||
return {
|
||||
cls.MY_TIER_METRIC: OffloadingCounterMetadata(
|
||||
documentation="Number of bytes served by the test tier.",
|
||||
labelnames=("tier",),
|
||||
)
|
||||
}
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.stats: OffloadingConnectorStats | None = None
|
||||
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
|
||||
return False
|
||||
|
||||
def submit_store(self, job_metadata: JobMetadata) -> None:
|
||||
return
|
||||
|
||||
def submit_load(self, job_metadata: JobMetadata) -> None:
|
||||
return
|
||||
|
||||
def get_finished_jobs(self) -> Iterable[JobResult]:
|
||||
return ()
|
||||
|
||||
def drain_jobs(self) -> None:
|
||||
return
|
||||
|
||||
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
|
||||
return RequestOffloadingContext()
|
||||
|
||||
def get_stats(self) -> OffloadingConnectorStats | None:
|
||||
stats = self.stats
|
||||
self.stats = None
|
||||
return stats
|
||||
|
||||
|
||||
def test_tiering_spec_collects_secondary_metric_definitions(monkeypatch):
|
||||
monkeypatch.setitem(
|
||||
SecondaryTierFactory._registry,
|
||||
"test_metrics",
|
||||
lambda: MetricsSecondaryTierManager,
|
||||
)
|
||||
|
||||
metrics = TieringOffloadingSpec.build_metric_definitions(
|
||||
{"secondary_tiers": [{"type": "test_metrics"}]}
|
||||
)
|
||||
|
||||
metadata = metrics[MetricsSecondaryTierManager.MY_TIER_METRIC]
|
||||
assert metadata.documentation == "Number of bytes served by the test tier."
|
||||
assert metadata.labelnames == ("tier",)
|
||||
|
||||
|
||||
def test_tiering_manager_aggregates_secondary_stats():
|
||||
mock_region = _mock_mmap_region(5)
|
||||
primary_tier = CPUPrimaryTierOffloadingManager(
|
||||
num_blocks=5, mmap_region=mock_region
|
||||
)
|
||||
secondary_tier = MetricsSecondaryTierManager(
|
||||
offloading_spec=_MOCK_OFFLOADING_SPEC,
|
||||
primary_kv_view=mock_region.create_kv_memoryview(),
|
||||
tier_type="test_metrics",
|
||||
)
|
||||
secondary_stats = OffloadingConnectorStats()
|
||||
secondary_stats.increase_counter(
|
||||
MetricsSecondaryTierManager.MY_TIER_METRIC, 7, ("test_metrics",)
|
||||
)
|
||||
secondary_tier.stats = secondary_stats
|
||||
manager = TieringOffloadingManager(
|
||||
primary_tier=primary_tier,
|
||||
secondary_tiers=[secondary_tier],
|
||||
)
|
||||
|
||||
stats = manager.get_stats()
|
||||
|
||||
assert stats is not None
|
||||
assert (
|
||||
stats.data["data"][MetricsSecondaryTierManager.MY_TIER_METRIC][
|
||||
("test_metrics",)
|
||||
]
|
||||
== 7
|
||||
)
|
||||
|
||||
# The primary tier's cache-usage gauge is always reported, so get_stats()
|
||||
# never returns None, but the secondary tier has nothing new to report
|
||||
# once its stats have been consumed.
|
||||
second_stats = manager.get_stats()
|
||||
assert second_stats is not None
|
||||
assert MetricsSecondaryTierManager.MY_TIER_METRIC not in second_stats.data["data"]
|
||||
|
||||
|
||||
class TestExampleSecondaryTierManager:
|
||||
"""Tests for ExampleSecondaryTierManager implementation."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user