169 lines
6.2 KiB
Python
169 lines
6.2 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
from dataclasses import dataclass, field
|
|
from typing import Any, TypeAlias, TypeVar
|
|
|
|
from prometheus_client import Counter, Gauge, Histogram
|
|
|
|
from vllm.config import ECTransferConfig, VllmConfig
|
|
from vllm.distributed.ec_transfer.ec_connector.factory import ECConnectorFactory
|
|
from vllm.logger import init_logger
|
|
|
|
PromMetric: TypeAlias = Gauge | Counter | Histogram
|
|
PromMetricT = TypeVar("PromMetricT", bound=PromMetric)
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
@dataclass
|
|
class ECConnectorStats:
|
|
"""Base class for EC Connector Stats, a container for transfer performance
|
|
metrics or otherwise important telemetry from the connector.
|
|
All sub-classes need to be serializable as stats are sent from worker to
|
|
logger process.
|
|
"""
|
|
|
|
data: dict[str, Any] = field(default_factory=dict)
|
|
|
|
def reset(self):
|
|
"""Reset the stats, clear the state."""
|
|
raise NotImplementedError
|
|
|
|
def aggregate(self, other: "ECConnectorStats") -> "ECConnectorStats":
|
|
"""Aggregate stats with another `ECConnectorStats` object."""
|
|
raise NotImplementedError
|
|
|
|
def reduce(self) -> dict[str, int | float]:
|
|
"""Reduce the observations collected during a time interval to one or
|
|
more representative values (eg avg/median/sum of the series).
|
|
This is meant to be called by the logger to produce a summary of the
|
|
stats for the last time interval.
|
|
"""
|
|
raise NotImplementedError
|
|
|
|
def is_empty(self) -> bool:
|
|
"""Return True if the stats are empty."""
|
|
raise NotImplementedError
|
|
|
|
|
|
class ECConnectorLogging:
|
|
def __init__(self, ec_transfer_config: ECTransferConfig | None):
|
|
# Instantiate the connector's stats class.
|
|
if ec_transfer_config and ec_transfer_config.ec_connector:
|
|
self.connector_cls = ECConnectorFactory.get_connector_class(
|
|
ec_transfer_config
|
|
)
|
|
self.reset()
|
|
|
|
def reset(self):
|
|
self.transfer_stats_accumulator: ECConnectorStats | None = None
|
|
|
|
def observe(self, transfer_stats_data: dict[str, Any]):
|
|
# Should not be called when an ECConnector is not configured.
|
|
assert self.connector_cls is not None
|
|
# Called periodically when connector syncs with the scheduler.
|
|
# Note that this is not the same as the logging interval.
|
|
# We expect transfer_stats_data to be aggregated across all workers and
|
|
# consist of observations from a single connector.
|
|
transfer_stats = self.connector_cls.build_ec_connector_stats(
|
|
transfer_stats_data
|
|
)
|
|
if transfer_stats is None:
|
|
logger.warning_once(
|
|
"The connector %s is collecting stats but "
|
|
"does not implement the "
|
|
"`build_ec_connector_stats` method. "
|
|
"Stats will not be logged.",
|
|
self.connector_cls,
|
|
)
|
|
return
|
|
|
|
if self.transfer_stats_accumulator is None:
|
|
self.transfer_stats_accumulator = transfer_stats
|
|
else:
|
|
# Accumulate last interval stats.
|
|
self.transfer_stats_accumulator = self.transfer_stats_accumulator.aggregate(
|
|
transfer_stats
|
|
)
|
|
|
|
def log(self, log_fn=logger.info):
|
|
"""Log transfer metrics periodically, similar to throughput logging"""
|
|
if (
|
|
self.transfer_stats_accumulator
|
|
and not self.transfer_stats_accumulator.is_empty()
|
|
):
|
|
# Produce a single cumulative stats object for the last time
|
|
# interval from the recorded observations.
|
|
xfer_metrics = self.transfer_stats_accumulator.reduce()
|
|
xfer_metrics_str = ", ".join(f"{k}={v}" for k, v in xfer_metrics.items())
|
|
log_fn("EC Transfer metrics: %s", xfer_metrics_str)
|
|
|
|
# Reset metrics for next interval
|
|
self.reset()
|
|
|
|
|
|
class ECConnectorPromMetrics:
|
|
"""A base class for per-connector Prometheus metric registration
|
|
and recording.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
vllm_config: VllmConfig,
|
|
metric_types: dict[type[PromMetric], type[PromMetricT]],
|
|
labelnames: list[str],
|
|
per_engine_labelvalues: dict[int, list[object]],
|
|
):
|
|
self._ec_transfer_config = vllm_config.ec_transfer_config
|
|
self._gauge_cls = metric_types[Gauge]
|
|
self._counter_cls = metric_types[Counter]
|
|
self._histogram_cls = metric_types[Histogram]
|
|
self._labelnames = labelnames
|
|
self.per_engine_labelvalues = per_engine_labelvalues
|
|
|
|
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
|
|
"""Record the supplied transfer statistics to Prometheus metrics. These
|
|
statistics are engine-specific, and should be recorded to a metric
|
|
with the appropriate 'engine' label. These metric instances can be
|
|
created using the create_metric_per_engine() helper method.
|
|
"""
|
|
raise NotImplementedError
|
|
|
|
|
|
class ECConnectorProm:
|
|
"""Support for registering per-connector Prometheus metrics, and
|
|
recording transfer statistics to those metrics. Uses
|
|
ECConnectorBase.build_prom_metrics().
|
|
"""
|
|
|
|
_gauge_cls = Gauge
|
|
_counter_cls = Counter
|
|
_histogram_cls = Histogram
|
|
|
|
def __init__(
|
|
self,
|
|
vllm_config: VllmConfig,
|
|
labelnames: list[str],
|
|
per_engine_labelvalues: dict[int, list[object]],
|
|
):
|
|
self.prom_metrics: ECConnectorPromMetrics | None = None
|
|
ec_transfer_config = vllm_config.ec_transfer_config
|
|
if ec_transfer_config and ec_transfer_config.ec_connector:
|
|
connector_cls = ECConnectorFactory.get_connector_class(ec_transfer_config)
|
|
metric_types = {
|
|
Gauge: self._gauge_cls,
|
|
Counter: self._counter_cls,
|
|
Histogram: self._histogram_cls,
|
|
}
|
|
self.prom_metrics = connector_cls.build_prom_metrics(
|
|
vllm_config,
|
|
metric_types,
|
|
labelnames,
|
|
per_engine_labelvalues,
|
|
)
|
|
|
|
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
|
|
if self.prom_metrics is None:
|
|
return
|
|
self.prom_metrics.observe(transfer_stats_data, engine_idx)
|