[Metrics][KV Offload] Add vllm:kv_offload_config_info - generic info metrics for offloading the offloading connector (#56867)
Signed-off-by: Amir Friedman <Amir.Friedman1@ibm.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
Claude Opus 5
Or Ozeri
parent
3a5a6b11a3
commit
775fefa804
@@ -99,8 +99,13 @@ class KVConnectorLogging:
|
||||
# Produce a single cumulative stats object for the last time
|
||||
# interval from the recorded observations.
|
||||
xfer_metrics = self.transfer_stats_accumulator.reduce()
|
||||
xfer_metrics_str = ", ".join(f"{k}={v}" for k, v in xfer_metrics.items())
|
||||
log_fn("KV Transfer metrics: %s", xfer_metrics_str)
|
||||
# A payload can hold static config facts alone (info metric), which reduce()
|
||||
# drops, because they belong to the Prometheus metrics only.
|
||||
if xfer_metrics:
|
||||
xfer_metrics_str = ", ".join(
|
||||
f"{k}={v}" for k, v in xfer_metrics.items()
|
||||
)
|
||||
log_fn("KV Transfer metrics: %s", xfer_metrics_str)
|
||||
|
||||
# Reset metrics for next interval
|
||||
self.reset()
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
@@ -10,14 +11,30 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.base import (
|
||||
ConfigInfoMapping,
|
||||
OffloadingCounterMetadata,
|
||||
OffloadingGaugeMetadata,
|
||||
OffloadingHistogramMetadata,
|
||||
OffloadingMetricMetadata,
|
||||
OffloadingSpec,
|
||||
)
|
||||
from vllm.v1.kv_offload.factory import OffloadingSpecFactory
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
KV_OFFLOAD_CONFIG_INFO = "vllm:kv_offload_config_info"
|
||||
|
||||
_INFO_METRIC_HELP = (
|
||||
"Static configuration of the KV offload managers of this engine instance. "
|
||||
"The configured offloading spec declares the label names, and each manager "
|
||||
"fills the values, so a series appears from the first scheduler step of its "
|
||||
"engine. A manager reports one series for each configuration it holds, and "
|
||||
"a label of its own tells the series apart. Each engine reports its own "
|
||||
"configuration, not the instance total."
|
||||
)
|
||||
|
||||
|
||||
class _TransferMetricName:
|
||||
"""Flat metric names for GPU↔offload-medium transfer operations."""
|
||||
@@ -166,6 +183,9 @@ class _StatsKey:
|
||||
TYPES = "types"
|
||||
# Maps metric name -> {label values tuple -> observed value (number or list)}
|
||||
DATA = "data"
|
||||
# One mapping of info metric label name -> label value for each series.
|
||||
# None until a manager reports it, and one empty mapping for no fact.
|
||||
INFO = "info"
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -177,6 +197,7 @@ class OffloadingConnectorStats(KVConnectorStats):
|
||||
{
|
||||
_StatsKey.TYPES: {name: _MetricType.*, ...},
|
||||
_StatsKey.DATA: {name: {labelvalues: value, ...}, ...},
|
||||
_StatsKey.INFO: [{labelname: labelvalue, ...}, ...] | None,
|
||||
}
|
||||
|
||||
This structure is self-describing: it survives IPC serialization
|
||||
@@ -187,6 +208,12 @@ class OffloadingConnectorStats(KVConnectorStats):
|
||||
use the latest snapshot per-label-tuple, and histogram values are lists of
|
||||
observed samples per-label-tuple. Unlabeled metrics use ``()`` as their
|
||||
labelvalues tuple.
|
||||
|
||||
``INFO`` holds the static config facts of one series each, which the
|
||||
scheduler sends once per process. It stays out of ``DATA``, because its
|
||||
label names are known only when the payload arrives. It holds label names,
|
||||
not metric names, so a manager that accesses its own metrics walks
|
||||
``metric_sections()``.
|
||||
"""
|
||||
|
||||
def __post_init__(self):
|
||||
@@ -197,6 +224,7 @@ class OffloadingConnectorStats(KVConnectorStats):
|
||||
self.data: dict[str, Any] = {
|
||||
_StatsKey.TYPES: {},
|
||||
_StatsKey.DATA: {},
|
||||
_StatsKey.INFO: None,
|
||||
}
|
||||
|
||||
@property
|
||||
@@ -207,10 +235,28 @@ class OffloadingConnectorStats(KVConnectorStats):
|
||||
def _values(self) -> dict[str, Any]:
|
||||
return self.data[_StatsKey.DATA]
|
||||
|
||||
def metric_sections(self) -> tuple[dict[str, Any], ...]:
|
||||
"""Return the type section and the value section, in that order.
|
||||
|
||||
A manager that accesses its own metrics walks these sections. Each one
|
||||
is a mutable reference.
|
||||
"""
|
||||
return (self._types, self._values)
|
||||
|
||||
def aggregate(self, other: "KVConnectorStats") -> "KVConnectorStats":
|
||||
if other.is_empty():
|
||||
return self
|
||||
assert isinstance(other, OffloadingConnectorStats)
|
||||
# The scheduler payload merges into the worker payload
|
||||
# (v1/core/sched/scheduler.py), so the info must survive the merge in
|
||||
# this direction to reach the Prometheus frontend. reduce() drops it, so
|
||||
# it never reaches a log line. The scheduler sends it once, so a later
|
||||
# payload with no info must not clear it.
|
||||
other_info = other.data.get(_StatsKey.INFO)
|
||||
if other_info is not None:
|
||||
self.data[_StatsKey.INFO] = [
|
||||
dict(series_info) for series_info in other_info
|
||||
]
|
||||
other_types = other._types
|
||||
other_values = other._values
|
||||
for key, other_label_values in other_values.items():
|
||||
@@ -268,7 +314,11 @@ class OffloadingConnectorStats(KVConnectorStats):
|
||||
return return_dict
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
return not self.data.get(_StatsKey.DATA)
|
||||
# An info-only payload is not empty. It carries no observation, but the
|
||||
# caller drops an empty payload, and the info must reach the frontend.
|
||||
return (
|
||||
not self.data.get(_StatsKey.DATA) and self.data.get(_StatsKey.INFO) is None
|
||||
)
|
||||
|
||||
def increase_counter(
|
||||
self,
|
||||
@@ -294,6 +344,18 @@ class OffloadingConnectorStats(KVConnectorStats):
|
||||
gauge_values = self._values.setdefault(gauge_name, {})
|
||||
gauge_values[labelvalues] = gauge_value
|
||||
|
||||
def set_info(self, info: Sequence[ConfigInfoMapping]) -> None:
|
||||
"""Put the static config facts of this engine on the stats payload.
|
||||
|
||||
Args:
|
||||
info: One mapping of info metric label name to label value for each
|
||||
series, as OffloadingManager.config_info() returns it. An empty
|
||||
mapping still gives one series, with an empty value on every
|
||||
label the spec declared.
|
||||
|
||||
"""
|
||||
self.data[_StatsKey.INFO] = [dict(series_info) for series_info in info]
|
||||
|
||||
def observe_histogram(
|
||||
self,
|
||||
histogram_name: str,
|
||||
@@ -322,14 +384,22 @@ class OffloadPromMetrics(KVConnectorPromMetrics):
|
||||
kv_transfer_config = vllm_config.kv_transfer_config
|
||||
assert kv_transfer_config is not None
|
||||
extra_config = kv_transfer_config.kv_connector_extra_config
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(extra_config)
|
||||
self._spec_cls: type[OffloadingSpec] = OffloadingSpecFactory.get_spec_cls(
|
||||
extra_config
|
||||
)
|
||||
# The spec declares the info label names here; a manager fills the
|
||||
# values later, and _observe_info() aligns the two.
|
||||
self._info_keys: tuple[str, ...] = self._spec_cls.config_info_keys(extra_config)
|
||||
self._offloading_metric_metadata: dict[str, OffloadingMetricMetadata] = {
|
||||
**spec_cls.build_metric_definitions(extra_config),
|
||||
**self._spec_cls.build_metric_definitions(extra_config),
|
||||
**get_connector_metric_definitions(),
|
||||
KV_OFFLOAD_CONFIG_INFO: OffloadingGaugeMetadata(
|
||||
documentation=_INFO_METRIC_HELP, labelnames=self._info_keys
|
||||
),
|
||||
}
|
||||
from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec
|
||||
|
||||
self._observe_deprecated_metrics = issubclass(spec_cls, CPUOffloadingSpec)
|
||||
self._observe_deprecated_metrics = issubclass(self._spec_cls, CPUOffloadingSpec)
|
||||
self._offloading_metric_defs: dict[str, PromMetricT] = {}
|
||||
# (engine_idx, metric_name, labelvalues) -> metric with bound labels
|
||||
self.offloading_metrics: dict[
|
||||
@@ -393,6 +463,7 @@ class OffloadPromMetrics(KVConnectorPromMetrics):
|
||||
metric_cls = self._counter_cls
|
||||
elif isinstance(metadata, OffloadingGaugeMetadata):
|
||||
metric_cls = self._gauge_cls
|
||||
kwargs["multiprocess_mode"] = metadata.multiprocess_mode
|
||||
elif isinstance(metadata, OffloadingHistogramMetadata):
|
||||
metric_cls = self._histogram_cls
|
||||
if metadata.buckets is not None:
|
||||
@@ -476,8 +547,52 @@ class OffloadPromMetrics(KVConnectorPromMetrics):
|
||||
observation
|
||||
)
|
||||
|
||||
def _observe_info(self, info: Sequence[dict[str, Any]], engine_idx: int) -> None:
|
||||
"""Publish the static config facts of one engine, one series per mapping.
|
||||
|
||||
The spec declares the label names, so a payload only fills them. A
|
||||
declared name that no series fills reads empty on every series. A payload
|
||||
name that the spec did not declare is dropped. Either gap logs once.
|
||||
|
||||
The check covers the payload as a whole, and not one series, because the
|
||||
declaration holds the names of every series. A name that one series owns
|
||||
reads empty on the other series by design. A series that leaves a name of
|
||||
its own unfilled therefore stays hidden here.
|
||||
|
||||
Args:
|
||||
info: One mapping of label name to label value for each series, as
|
||||
OffloadingManager.config_info() returns it.
|
||||
engine_idx: Index of the reporting engine.
|
||||
|
||||
"""
|
||||
filled: dict[str, None] = {}
|
||||
for series_info in info:
|
||||
filled.update(dict.fromkeys(series_info))
|
||||
labelvalues = tuple(
|
||||
str(series_info.get(key, "")) for key in self._info_keys
|
||||
)
|
||||
self._set_gauge(KV_OFFLOAD_CONFIG_INFO, 1, labelvalues, engine_idx)
|
||||
|
||||
never_filled = tuple(key for key in self._info_keys if key not in filled)
|
||||
dropped = tuple(key for key in filled if key not in self._info_keys)
|
||||
if never_filled or dropped:
|
||||
logger.warning_once(
|
||||
"%s: spec %s and the manager of engine %d disagree on the KV "
|
||||
"offload config labels. Declared, and no series filled them, so "
|
||||
"empty on every series: %s. Filled, and the spec did not "
|
||||
"declare them, so dropped: %s.",
|
||||
KV_OFFLOAD_CONFIG_INFO,
|
||||
self._spec_cls.__name__,
|
||||
engine_idx,
|
||||
never_filled,
|
||||
dropped,
|
||||
)
|
||||
|
||||
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
|
||||
"""Observe transfer statistics."""
|
||||
info = transfer_stats_data.get(_StatsKey.INFO)
|
||||
if info is not None:
|
||||
self._observe_info(info, engine_idx)
|
||||
metric_types = transfer_stats_data.get(_StatsKey.TYPES, {})
|
||||
metric_data = transfer_stats_data.get(_StatsKey.DATA, {})
|
||||
for key, label_value_map in metric_data.items():
|
||||
|
||||
@@ -574,6 +574,10 @@ class OffloadingConnectorScheduler:
|
||||
)
|
||||
self.manager: OffloadingManager = spec.get_manager()
|
||||
self._connector_stats = OffloadingConnectorStats()
|
||||
# The static config facts ride on the first stats payload of this
|
||||
# process. A Prometheus child holds the values it received, so one
|
||||
# payload is enough.
|
||||
self._info_sent = False
|
||||
|
||||
full_attention_groups: list[int] = []
|
||||
sliding_window_groups: list[int] = []
|
||||
@@ -2004,6 +2008,14 @@ class OffloadingConnectorScheduler:
|
||||
else:
|
||||
stats.aggregate(manager_stats)
|
||||
|
||||
if not self._info_sent:
|
||||
if stats is None:
|
||||
stats = OffloadingConnectorStats()
|
||||
# Sent even when the manager reports no facts, so that the metric
|
||||
# exists whenever offloading runs.
|
||||
stats.set_info(self.manager.config_info())
|
||||
self._info_sent = True
|
||||
|
||||
return stats
|
||||
|
||||
def request_finished(
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
"""Core abstractions for KV cache offloading in vLLM v1."""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Collection, Iterable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from collections.abc import Collection, Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field, fields
|
||||
from enum import Enum, auto
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, NewType, TypeVar
|
||||
|
||||
@@ -214,7 +214,12 @@ class OffloadingCounterMetadata(OffloadingMetricMetadata):
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OffloadingGaugeMetadata(OffloadingMetricMetadata):
|
||||
pass
|
||||
# Gauge-only in prometheus_client: how MultiProcessCollector merges samples
|
||||
# written by different API-server processes. Offloading stats reach one
|
||||
# frontend per step as complete per-engine snapshots, so the freshest write
|
||||
# is the correct value and summing would multiply it by the number of
|
||||
# participating frontends.
|
||||
multiprocess_mode: str = "mostrecent"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -232,6 +237,36 @@ class OffloadingKVEventsConfig:
|
||||
self_describing_kv_events: bool
|
||||
|
||||
|
||||
# The labels of one info metric series: a label name mapped to its value.
|
||||
ConfigInfoMapping = Mapping[str, str | int | float | bool]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ConfigInfo:
|
||||
"""Static facts that one component publishes on vllm:kv_offload_config_info.
|
||||
|
||||
Each field is one label. The OffloadingSpec declares the names with
|
||||
config_info_keys(), and the OffloadingManager fills the values with
|
||||
as_config_info(). A declared name that the manager does not fill reads
|
||||
empty, and an undeclared name is dropped. The values must stay fixed for
|
||||
the process lifetime. Document each field in
|
||||
docs/features/kv_offloading_usage.md. See CPUOffloadingInfo for an example.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def config_info_keys(cls) -> tuple[str, ...]:
|
||||
"""Return one label name for each field, in field order."""
|
||||
return tuple(info.name for info in fields(cls))
|
||||
|
||||
def as_config_info(self) -> ConfigInfoMapping:
|
||||
"""Return the label values. A None value becomes "None", not empty."""
|
||||
values = (getattr(self, info.name) for info in fields(self))
|
||||
return {
|
||||
key: "None" if value is None else value
|
||||
for key, value in zip(self.config_info_keys(), values)
|
||||
}
|
||||
|
||||
|
||||
class OffloadingManager(ABC):
|
||||
@abstractmethod
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
||||
@@ -416,6 +451,13 @@ class OffloadingManager(ABC):
|
||||
"""Return collected metrics since last call, or None if disabled."""
|
||||
return None
|
||||
|
||||
def config_info(self) -> Sequence[ConfigInfoMapping]:
|
||||
"""Return the info metric labels, one mapping for each series.
|
||||
|
||||
The scheduler reads this once. See ConfigInfo.
|
||||
"""
|
||||
return [{}]
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Shutdown the manager and release any resources."""
|
||||
return
|
||||
@@ -602,6 +644,18 @@ class OffloadingSpec(ABC):
|
||||
"""Return Prometheus metric definitions emitted by this spec."""
|
||||
return {}
|
||||
|
||||
@classmethod
|
||||
def config_info_keys(cls, extra_config: dict[str, Any]) -> tuple[str, ...]:
|
||||
"""Return the info metric label names that the manager fills.
|
||||
|
||||
See ConfigInfo.
|
||||
|
||||
Args:
|
||||
extra_config: kv_connector_extra_config of this instance.
|
||||
|
||||
"""
|
||||
return ()
|
||||
|
||||
def __init__(self, config: OffloadingConfig):
|
||||
self.config = config
|
||||
self.extra_config = config.extra_config
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.v1.kv_offload.base import BlockIDsLoadStoreSpec
|
||||
from vllm.v1.kv_offload.base import BlockIDsLoadStoreSpec, ConfigInfo
|
||||
|
||||
|
||||
class CPUOffloadingMetrics:
|
||||
@@ -24,3 +26,11 @@ class CPULoadStoreSpec(BlockIDsLoadStoreSpec):
|
||||
@property
|
||||
def chunk_ids(self) -> np.ndarray:
|
||||
return self.block_ids
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CPUOffloadingInfo(ConfigInfo):
|
||||
"""Static facts about the CPU offload tier."""
|
||||
|
||||
# Chunk slots in the tier. Chunks, not GPU blocks.
|
||||
cpu_num_chunks: int
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Collection, Iterable
|
||||
from collections.abc import Collection, Iterable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from typing_extensions import override
|
||||
@@ -10,6 +10,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
OffloadingConnectorStats,
|
||||
)
|
||||
from vllm.v1.kv_offload.base import (
|
||||
ConfigInfoMapping,
|
||||
LoadStoreSpec,
|
||||
LookupResult,
|
||||
Medium,
|
||||
@@ -23,6 +24,7 @@ from vllm.v1.kv_offload.base import (
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.common import (
|
||||
CPULoadStoreSpec,
|
||||
CPUOffloadingInfo,
|
||||
CPUOffloadingMetrics,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.policies.base import CachePolicy, ChunkStatus
|
||||
@@ -66,6 +68,9 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
):
|
||||
self.medium: Medium = Medium.CPU
|
||||
self._num_chunks: int = num_chunks
|
||||
self._config_info: ConfigInfoMapping = CPUOffloadingInfo(
|
||||
cpu_num_chunks=num_chunks
|
||||
).as_config_info()
|
||||
self._num_allocated_chunks: int = 0
|
||||
self._free_list: list[int] = []
|
||||
self.events: list[OffloadingEvent] | None = [] if enable_events else None
|
||||
@@ -446,6 +451,11 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
yield from self.events
|
||||
self.events.clear()
|
||||
|
||||
@override
|
||||
def config_info(self) -> Sequence[ConfigInfoMapping]:
|
||||
"""Report the CPU cache facts of the one cache this manager holds."""
|
||||
return [self._config_info]
|
||||
|
||||
def get_stats(self) -> OffloadingConnectorStats | None:
|
||||
stats = OffloadingConnectorStats()
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ from vllm.v1.kv_offload.base import (
|
||||
OffloadingWorker,
|
||||
)
|
||||
from vllm.v1.kv_offload.config import OffloadingConfig
|
||||
from vllm.v1.kv_offload.cpu.common import CPUOffloadingMetrics
|
||||
from vllm.v1.kv_offload.cpu.common import CPUOffloadingInfo, CPUOffloadingMetrics
|
||||
from vllm.v1.kv_offload.cpu.gpu_worker import CPUOffloadingWorker
|
||||
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
|
||||
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||
@@ -136,6 +136,12 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
"cache_policy_module_path"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@override
|
||||
def config_info_keys(cls, extra_config: dict[str, Any]) -> tuple[str, ...]:
|
||||
"""Return the label names CPUOffloadingManager.config_info() fills."""
|
||||
return CPUOffloadingInfo.config_info_keys()
|
||||
|
||||
@override
|
||||
def get_manager(self) -> OffloadingManager:
|
||||
if not self._manager:
|
||||
|
||||
@@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Any, ClassVar
|
||||
import numpy as np
|
||||
|
||||
from vllm.v1.kv_offload.base import (
|
||||
ConfigInfoMapping,
|
||||
Locality,
|
||||
LookupResult,
|
||||
Medium,
|
||||
@@ -36,6 +37,8 @@ if TYPE_CHECKING:
|
||||
# Type alias for job IDs used in async transfer tracking
|
||||
JobId = int
|
||||
|
||||
TIER_LABEL = "tier"
|
||||
|
||||
|
||||
class TieringOffloadingMetrics:
|
||||
"""Metric names for TieringOffloadingManager."""
|
||||
@@ -370,3 +373,23 @@ class SecondaryTierManager(ABC):
|
||||
def get_stats(self) -> "OffloadingConnectorStats | None":
|
||||
"""Return and reset metric observations collected by this tier."""
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def config_info_keys(cls, tier_config: dict[str, Any]) -> tuple[str, ...]:
|
||||
"""Return the info metric label names of this tier.
|
||||
|
||||
TieringOffloadingSpec merges the names of all tiers and adds
|
||||
TIER_LABEL, so a tier must not declare it. See ConfigInfo.
|
||||
|
||||
Args:
|
||||
tier_config: Configuration dict of this tier.
|
||||
|
||||
"""
|
||||
return ()
|
||||
|
||||
def config_info(self) -> ConfigInfoMapping:
|
||||
"""Return the info metric labels of this tier, for its one series.
|
||||
|
||||
TieringOffloadingManager adds TIER_LABEL. See ConfigInfo.
|
||||
"""
|
||||
return {}
|
||||
|
||||
@@ -10,11 +10,14 @@ TieringOffloadingManager without requiring actual storage or network backends.
|
||||
|
||||
import logging
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm.v1.kv_offload.base import (
|
||||
ConfigInfo,
|
||||
ConfigInfoMapping,
|
||||
LookupResult,
|
||||
Medium,
|
||||
OffloadKey,
|
||||
@@ -34,6 +37,17 @@ if TYPE_CHECKING:
|
||||
from vllm.v1.kv_offload.tiering.backpressure import BackpressureDetector
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ExampleTierInfo(ConfigInfo):
|
||||
"""Config facts of one ExampleSecondaryTierManager.
|
||||
|
||||
A field name is the label name. A secondary tier needs no name of its own
|
||||
for the tier, because the tier label tells the series of the tiers apart.
|
||||
"""
|
||||
|
||||
example_info: int
|
||||
|
||||
|
||||
class ExampleSecondaryTierManager(SecondaryTierManager):
|
||||
"""A simple in-memory secondary tier.
|
||||
|
||||
@@ -69,6 +83,8 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
|
||||
backpressure_detector=backpressure_detector,
|
||||
)
|
||||
|
||||
self._config_info = ExampleTierInfo(example_info=custom_param)
|
||||
|
||||
logger.info(
|
||||
"ExampleSecondaryTierManager initialized with custom_param=%d", custom_param
|
||||
)
|
||||
@@ -81,6 +97,15 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
|
||||
assert primary_kv_view.strides is not None
|
||||
self._chunk_size = primary_kv_view.strides[0]
|
||||
|
||||
@classmethod
|
||||
@override
|
||||
def config_info_keys(cls, tier_config: dict[str, Any]) -> tuple[str, ...]:
|
||||
return ExampleTierInfo.config_info_keys()
|
||||
|
||||
@override
|
||||
def config_info(self) -> ConfigInfoMapping:
|
||||
return self._config_info.as_config_info()
|
||||
|
||||
@override
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
||||
"""Check whether a chunk exists in this secondary tier.
|
||||
|
||||
@@ -523,7 +523,7 @@ class KVCRSecondaryTierManager(SecondaryTierManager):
|
||||
stats = cast(OffloadingConnectorStats | None, self._kvcr.get_stats())
|
||||
if stats is not None:
|
||||
# Add the framework namespace only at the vLLM boundary.
|
||||
for metrics in stats.data.values():
|
||||
for metrics in stats.metric_sections():
|
||||
for name in tuple(metrics):
|
||||
metrics[_vllm_metric_name(name)] = metrics.pop(name)
|
||||
return stats
|
||||
|
||||
@@ -32,6 +32,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.base import (
|
||||
ConfigInfoMapping,
|
||||
LoadStoreSpec,
|
||||
LookupResult,
|
||||
OffloadingEvent,
|
||||
@@ -47,6 +48,7 @@ from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
|
||||
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
|
||||
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||
from vllm.v1.kv_offload.tiering.base import (
|
||||
TIER_LABEL,
|
||||
JobId,
|
||||
JobResult,
|
||||
ParentManager,
|
||||
@@ -1007,6 +1009,26 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
|
||||
return stats
|
||||
|
||||
@override
|
||||
def config_info(self) -> Sequence[ConfigInfoMapping]:
|
||||
"""Return one mapping for each tier, primary tier first, with TIER_LABEL.
|
||||
|
||||
Raises:
|
||||
ValueError: If the primary tier returns more than one mapping.
|
||||
|
||||
"""
|
||||
(primary_info,) = self.primary_tier.config_info()
|
||||
primary_tier_info = dict(primary_info)
|
||||
primary_tier_info[TIER_LABEL] = self._metrics.primary_tier_label[0]
|
||||
tier_infos = [primary_tier_info]
|
||||
|
||||
for tier in self.secondary_tiers:
|
||||
tier_info = dict(tier.config_info())
|
||||
tier_info[TIER_LABEL] = self._metrics.tier_label(self._tier_index[tier])[0]
|
||||
tier_infos.append(tier_info)
|
||||
|
||||
return tier_infos
|
||||
|
||||
@override
|
||||
def shutdown(self) -> None:
|
||||
"""Shut down secondary tiers before releasing primary resources.
|
||||
|
||||
@@ -69,7 +69,11 @@ from vllm.v1.kv_offload.config import OffloadingConfig
|
||||
from vllm.v1.kv_offload.cpu.gpu_worker import CPUOffloadingWorker
|
||||
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||
from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec
|
||||
from vllm.v1.kv_offload.tiering.base import TieringOffloadingMetrics
|
||||
from vllm.v1.kv_offload.tiering.base import (
|
||||
TIER_LABEL,
|
||||
SecondaryTierManager,
|
||||
TieringOffloadingMetrics,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory
|
||||
from vllm.v1.kv_offload.tiering.manager import (
|
||||
CPUPrimaryTierOffloadingManager,
|
||||
@@ -105,7 +109,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
"Histogram of blocking time spent in a per-chunk tier lookup "
|
||||
"that resolved as a hit or miss, labeled by tier, in seconds."
|
||||
),
|
||||
labelnames=("tier",),
|
||||
labelnames=(TIER_LABEL,),
|
||||
buckets=(
|
||||
0.00001,
|
||||
0.00005,
|
||||
@@ -128,7 +132,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
"first returning retry until that same tier lookup resolves "
|
||||
"as a hit or miss, labeled by tier, in seconds."
|
||||
),
|
||||
labelnames=("tier",),
|
||||
labelnames=(TIER_LABEL,),
|
||||
buckets=(
|
||||
0.0001,
|
||||
0.0005,
|
||||
@@ -149,35 +153,35 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
"Total bytes read from secondary tiers into the primary tier, "
|
||||
"labeled by tier."
|
||||
),
|
||||
labelnames=("tier",),
|
||||
labelnames=(TIER_LABEL,),
|
||||
)
|
||||
metrics[TieringOffloadingMetrics.READ_TIME] = OffloadingCounterMetadata(
|
||||
documentation=(
|
||||
"Total time spent reading from secondary tiers into the primary "
|
||||
"tier, in seconds, labeled by tier."
|
||||
),
|
||||
labelnames=("tier",),
|
||||
labelnames=(TIER_LABEL,),
|
||||
)
|
||||
metrics[TieringOffloadingMetrics.WRITE_BYTES] = OffloadingCounterMetadata(
|
||||
documentation=(
|
||||
"Total bytes written from the primary tier to secondary tiers, "
|
||||
"labeled by tier."
|
||||
),
|
||||
labelnames=("tier",),
|
||||
labelnames=(TIER_LABEL,),
|
||||
)
|
||||
metrics[TieringOffloadingMetrics.WRITE_TIME] = OffloadingCounterMetadata(
|
||||
documentation=(
|
||||
"Total time spent writing from the primary tier to secondary "
|
||||
"tiers, in seconds, labeled by tier."
|
||||
),
|
||||
labelnames=("tier",),
|
||||
labelnames=(TIER_LABEL,),
|
||||
)
|
||||
metrics[TieringOffloadingMetrics.PROMOTION_JOB_FAILURES] = (
|
||||
OffloadingCounterMetadata(
|
||||
documentation=(
|
||||
"Number of failed secondary-tier promotion jobs, labeled by tier."
|
||||
),
|
||||
labelnames=("tier",),
|
||||
labelnames=(TIER_LABEL,),
|
||||
)
|
||||
)
|
||||
metrics[TieringOffloadingMetrics.CASCADE_JOB_FAILURES] = (
|
||||
@@ -185,18 +189,18 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
documentation=(
|
||||
"Number of failed secondary-tier cascade jobs, labeled by tier."
|
||||
),
|
||||
labelnames=("tier",),
|
||||
labelnames=(TIER_LABEL,),
|
||||
)
|
||||
)
|
||||
metrics[TieringOffloadingMetrics.CHUNK_QUERIES] = OffloadingCounterMetadata(
|
||||
documentation=(
|
||||
"Number of chunk lookup queries sent to a tier, labeled by tier."
|
||||
),
|
||||
labelnames=("tier",),
|
||||
labelnames=(TIER_LABEL,),
|
||||
)
|
||||
metrics[TieringOffloadingMetrics.CHUNK_HITS] = OffloadingCounterMetadata(
|
||||
documentation="Number of chunk lookup hits in a tier, labeled by tier.",
|
||||
labelnames=("tier",),
|
||||
labelnames=(TIER_LABEL,),
|
||||
)
|
||||
metrics[TieringOffloadingMetrics.PROMOTION_ALLOCATION_FAILURES] = (
|
||||
OffloadingCounterMetadata(
|
||||
@@ -212,7 +216,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
"Current fraction of primary-tier space used by writes from "
|
||||
"secondary tiers, labeled by tier."
|
||||
),
|
||||
labelnames=("tier",),
|
||||
labelnames=(TIER_LABEL,),
|
||||
)
|
||||
)
|
||||
metrics[TieringOffloadingMetrics.PRIMARY_READ_USAGE_PERC] = (
|
||||
@@ -221,7 +225,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
"Current fraction of primary-tier space used by reads to "
|
||||
"secondary tiers, labeled by tier."
|
||||
),
|
||||
labelnames=("tier",),
|
||||
labelnames=(TIER_LABEL,),
|
||||
)
|
||||
)
|
||||
metrics[TieringOffloadingMetrics.ACTIVE_PROMOTION_JOBS] = (
|
||||
@@ -229,22 +233,17 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
documentation=(
|
||||
"Number of active secondary-tier promotion jobs, labeled by tier."
|
||||
),
|
||||
labelnames=("tier",),
|
||||
labelnames=(TIER_LABEL,),
|
||||
)
|
||||
)
|
||||
metrics[TieringOffloadingMetrics.ACTIVE_CASCADE_JOBS] = OffloadingGaugeMetadata(
|
||||
documentation=(
|
||||
"Number of active secondary-tier cascade jobs, labeled by tier."
|
||||
),
|
||||
labelnames=("tier",),
|
||||
labelnames=(TIER_LABEL,),
|
||||
)
|
||||
secondary_tier_configs = extra_config.get("secondary_tiers", [])
|
||||
if not isinstance(secondary_tier_configs, list):
|
||||
raise ValueError("secondary_tiers must be a list of tier configurations")
|
||||
|
||||
for tier_config in secondary_tier_configs:
|
||||
assert isinstance(tier_config, dict)
|
||||
tier_cls = SecondaryTierFactory.get_tier_class(tier_config)
|
||||
for tier_config, tier_cls in cls._get_secondary_tiers(extra_config):
|
||||
metrics.update(tier_cls.build_metric_definitions(tier_config))
|
||||
|
||||
metrics[TieringOffloadingMetrics.BACKPRESSURE_STORE_LATENCY_EMA] = (
|
||||
@@ -253,7 +252,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
"Exponential moving average of store latency "
|
||||
"for back-pressure detection, in s/MiB."
|
||||
),
|
||||
labelnames=("tier",),
|
||||
labelnames=(TIER_LABEL,),
|
||||
)
|
||||
)
|
||||
metrics[TieringOffloadingMetrics.BACKPRESSURE_STORES_DROPPED] = (
|
||||
@@ -262,7 +261,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
"Number of store operations dropped due to "
|
||||
"back-pressure on a secondary tier."
|
||||
),
|
||||
labelnames=("tier",),
|
||||
labelnames=(TIER_LABEL,),
|
||||
)
|
||||
)
|
||||
metrics[TieringOffloadingMetrics.BACKPRESSURE_BLOCKS_DROPPED] = (
|
||||
@@ -270,7 +269,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
documentation=(
|
||||
"Number of blocks dropped due to back-pressure on a secondary tier."
|
||||
),
|
||||
labelnames=("tier",),
|
||||
labelnames=(TIER_LABEL,),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -426,6 +425,50 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
|
||||
return self._manager
|
||||
|
||||
@classmethod
|
||||
@override
|
||||
def config_info_keys(cls, extra_config: dict[str, Any]) -> tuple[str, ...]:
|
||||
"""Return TIER_LABEL and the label names of all tiers, with no repeat."""
|
||||
# dict.fromkeys() keeps one copy of a name that two tier types share.
|
||||
keys = dict.fromkeys(super().config_info_keys(extra_config))
|
||||
keys[TIER_LABEL] = None
|
||||
for tier_config, tier_cls in cls._get_secondary_tiers(extra_config):
|
||||
tier_keys = tier_cls.config_info_keys(tier_config)
|
||||
assert TIER_LABEL not in tier_keys, (
|
||||
f"{tier_cls.__name__} must not declare the reserved label "
|
||||
f"{TIER_LABEL!r}"
|
||||
)
|
||||
keys.update(dict.fromkeys(tier_keys))
|
||||
return tuple(keys)
|
||||
|
||||
@classmethod
|
||||
def _get_secondary_tiers(
|
||||
cls, extra_config: dict[str, Any]
|
||||
) -> list[tuple[dict[str, Any], type[SecondaryTierManager]]]:
|
||||
"""Resolve every configured secondary tier to its manager class.
|
||||
|
||||
Args:
|
||||
extra_config: kv_connector_extra_config of this instance.
|
||||
|
||||
Returns:
|
||||
One (tier config, tier class) pair for each secondary tier, in
|
||||
configuration order. That order sets the tier index.
|
||||
|
||||
Raises:
|
||||
ValueError: If secondary_tiers is not a list.
|
||||
|
||||
"""
|
||||
secondary_tier_configs = extra_config.get("secondary_tiers", [])
|
||||
if not isinstance(secondary_tier_configs, list):
|
||||
raise ValueError("secondary_tiers must be a list of tier configurations")
|
||||
|
||||
tiers = []
|
||||
for tier_config in secondary_tier_configs:
|
||||
assert isinstance(tier_config, dict)
|
||||
tier_cls = SecondaryTierFactory.get_tier_class(tier_config)
|
||||
tiers.append((tier_config, tier_cls))
|
||||
return tiers
|
||||
|
||||
@override
|
||||
def _uses_shared_region(self) -> bool:
|
||||
# Tiering always allocates on the shared region (every platform), so the
|
||||
|
||||
Reference in New Issue
Block a user