[KV Offloading] Add tiering metric plumbing (#45959)
Signed-off-by: srinivas_oo7 <sklinkedin0120@gmail.com> Co-authored-by: srinivas_oo7 <sklinkedin0120@gmail.com>
This commit is contained in:
co-authored by
srinivas_oo7
parent
3554ada5d8
commit
091bc1026e
@@ -7,13 +7,21 @@ Abstract interfaces and data types for the secondary tiering layer.
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Collection, Iterable
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.v1.kv_offload.base import OffloadKey, ReqContext, RequestOffloadingContext
|
||||
from vllm.v1.kv_offload.base import (
|
||||
OffloadingMetricMetadata,
|
||||
OffloadKey,
|
||||
ReqContext,
|
||||
RequestOffloadingContext,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
OffloadingConnectorStats,
|
||||
)
|
||||
from vllm.v1.kv_offload.base import OffloadingSpec
|
||||
|
||||
# Type alias for job IDs used in async transfer tracking
|
||||
@@ -228,3 +236,14 @@ class SecondaryTierManager(ABC):
|
||||
def shutdown(self) -> None:
|
||||
"""Release resources held by this tier (threads, connections, etc.)."""
|
||||
return
|
||||
|
||||
@classmethod
|
||||
def build_metric_definitions(
|
||||
cls, extra_config: dict[str, Any]
|
||||
) -> dict[str, OffloadingMetricMetadata]:
|
||||
"""Return Prometheus metric definitions emitted by this tier."""
|
||||
return {}
|
||||
|
||||
def get_stats(self) -> "OffloadingConnectorStats | None":
|
||||
"""Return and reset metric observations collected by this tier."""
|
||||
return None
|
||||
|
||||
@@ -31,19 +31,9 @@ class SecondaryTierFactory:
|
||||
primary_kv_view: memoryview,
|
||||
offloading_spec: "OffloadingSpec",
|
||||
) -> SecondaryTierManager:
|
||||
tier_cls = cls.get_tier_class(tier_config)
|
||||
config = tier_config.copy()
|
||||
|
||||
tier_type = config.pop("type", None)
|
||||
if not tier_type:
|
||||
raise ValueError("Secondary tier configuration must include 'type'")
|
||||
|
||||
if tier_type not in cls._registry:
|
||||
raise ValueError(
|
||||
f"Unknown secondary tier type: {tier_type!r}. "
|
||||
f"Supported types: {list(cls._registry)}"
|
||||
)
|
||||
|
||||
tier_cls = cls._registry[tier_type]()
|
||||
tier_type = config.pop("type")
|
||||
return tier_cls(
|
||||
offloading_spec=offloading_spec,
|
||||
primary_kv_view=primary_kv_view,
|
||||
@@ -51,6 +41,18 @@ class SecondaryTierFactory:
|
||||
**config,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_tier_class(cls, tier_config: dict) -> type[SecondaryTierManager]:
|
||||
tier_type = tier_config.get("type")
|
||||
if not tier_type:
|
||||
raise ValueError("Secondary tier configuration must include 'type'")
|
||||
if tier_type not in cls._registry:
|
||||
raise ValueError(
|
||||
f"Unknown secondary tier type: {tier_type!r}. "
|
||||
f"Supported types: {list(cls._registry)}"
|
||||
)
|
||||
return cls._registry[tier_type]()
|
||||
|
||||
|
||||
SecondaryTierFactory.register_tier(
|
||||
"example",
|
||||
|
||||
@@ -27,6 +27,9 @@ from dataclasses import dataclass, field
|
||||
import numpy as np
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
OffloadingConnectorStats,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LoadStoreSpec,
|
||||
@@ -628,6 +631,24 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
self._request_level_tiers.clear()
|
||||
self._processed_jobs_this_step = False
|
||||
|
||||
@override
|
||||
def get_stats(self) -> OffloadingConnectorStats | None:
|
||||
stats = self.primary_tier.get_stats()
|
||||
|
||||
if stats is not None and stats.is_empty():
|
||||
stats = None
|
||||
|
||||
for tier in self.secondary_tiers:
|
||||
tier_stats = tier.get_stats()
|
||||
if tier_stats is None or tier_stats.is_empty():
|
||||
continue
|
||||
if stats is None:
|
||||
stats = tier_stats
|
||||
else:
|
||||
stats.aggregate(tier_stats)
|
||||
|
||||
return stats
|
||||
|
||||
@override
|
||||
def shutdown(self) -> None:
|
||||
"""Shutdown all tiers and release resources."""
|
||||
|
||||
@@ -31,13 +31,19 @@ Example configuration:
|
||||
}
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.kv_offload.base import CanonicalKVCaches, OffloadingManager
|
||||
from vllm.v1.kv_offload.base import (
|
||||
CanonicalKVCaches,
|
||||
OffloadingManager,
|
||||
OffloadingMetricMetadata,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.gpu_worker import CpuGpuOffloadingHandlers
|
||||
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||
from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec
|
||||
@@ -65,6 +71,22 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
|
||||
BLOCK_SIZE_ALIGNMENT = SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT
|
||||
|
||||
@classmethod
|
||||
@override
|
||||
def build_metric_definitions(
|
||||
cls, extra_config: dict[str, Any]
|
||||
) -> dict[str, OffloadingMetricMetadata]:
|
||||
metrics = super().build_metric_definitions(extra_config)
|
||||
secondary_tier_configs = extra_config.get("secondary_tiers", [])
|
||||
if not isinstance(secondary_tier_configs, list):
|
||||
raise ValueError("secondary_tiers must be a list of tier configurations")
|
||||
|
||||
for tier_config in secondary_tier_configs:
|
||||
assert isinstance(tier_config, dict)
|
||||
tier_cls = SecondaryTierFactory.get_tier_class(tier_config)
|
||||
metrics.update(tier_cls.build_metric_definitions(tier_config))
|
||||
return metrics
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, kv_cache_config: KVCacheConfig):
|
||||
super().__init__(vllm_config, kv_cache_config)
|
||||
# Redeclare for mypy: parent sets this but `--follow-imports skip` hides it
|
||||
|
||||
Reference in New Issue
Block a user