[EC Connector] Add Metrics Collection (#54960)

Signed-off-by: omerpaz95 <omerpaz95@gmail.com>
This commit is contained in:
omerpaz95
2026-09-17 10:36:23 +08:00
committed by GitHub
parent 2307a1d16c
commit fa3622a4a1
12 changed files with 472 additions and 3 deletions
@@ -35,6 +35,12 @@ from vllm.v1.outputs import ECConnectorOutput
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.distributed.ec_transfer.ec_connector.metrics import (
ECConnectorPromMetrics,
ECConnectorStats,
PromMetric,
PromMetricT,
)
from vllm.v1.request import Request
logger = init_logger(__name__)
@@ -219,6 +225,17 @@ class ECConnectorBase(ABC):
"""
return None
def get_ec_connector_stats(self) -> "ECConnectorStats | None":
"""
Get the EC connector stats collected during the last interval.
Callable on either a worker-side or scheduler-side connector
instance: the worker-side instance reports stats gathered during
cache save/load, while the scheduler-side instance reports stats
gathered during scheduling decisions.
"""
return None
# ==============================
# Scheduler-side methods
# ==============================
@@ -336,3 +353,29 @@ class ECConnectorBase(ABC):
leave this as False.
"""
return False
@classmethod
def build_ec_connector_stats(
cls, data: dict[str, Any] | None = None
) -> "ECConnectorStats | None":
"""
ECConnectorStats resolution method. This method allows dynamically
registered connectors to return their own ECConnectorStats object,
which can implement custom aggregation logic on the data dict.
"""
return None
@classmethod
def build_prom_metrics(
cls,
vllm_config: "VllmConfig",
metric_types: dict[type["PromMetric"], type["PromMetricT"]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
) -> "ECConnectorPromMetrics | None":
"""
Create an ECConnectorPromMetrics subclass which should register
per-connector Prometheus metrics and implement observe() to
expose connector transfer stats via Prometheus.
"""
return None
@@ -0,0 +1,175 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass, field
from typing import Any, TypeAlias, TypeVar
from prometheus_client import Counter, Gauge, Histogram
from vllm.config import ECTransferConfig, VllmConfig
from vllm.distributed.ec_transfer.ec_connector.factory import ECConnectorFactory
from vllm.logger import init_logger
PromMetric: TypeAlias = Gauge | Counter | Histogram
PromMetricT = TypeVar("PromMetricT", bound=PromMetric)
logger = init_logger(__name__)
@dataclass
class ECConnectorStats:
"""
Base class for EC Connector Stats, a container for transfer performance
metrics or otherwise important telemetry from the connector.
All sub-classes need to be serializable as stats are sent from worker to
logger process.
"""
data: dict[str, Any] = field(default_factory=dict)
def reset(self):
"""Reset the stats, clear the state."""
raise NotImplementedError
def aggregate(self, other: "ECConnectorStats") -> "ECConnectorStats":
"""
Aggregate stats with another `ECConnectorStats` object.
"""
raise NotImplementedError
def reduce(self) -> dict[str, int | float]:
"""
Reduce the observations collected during a time interval to one or
more representative values (eg avg/median/sum of the series).
This is meant to be called by the logger to produce a summary of the
stats for the last time interval.
"""
raise NotImplementedError
def is_empty(self) -> bool:
"""Return True if the stats are empty."""
raise NotImplementedError
class ECConnectorLogging:
def __init__(self, ec_transfer_config: ECTransferConfig | None):
# Instantiate the connector's stats class.
if ec_transfer_config and ec_transfer_config.ec_connector:
self.connector_cls = ECConnectorFactory.get_connector_class(
ec_transfer_config
)
self.reset()
def reset(self):
self.transfer_stats_accumulator: ECConnectorStats | None = None
def observe(self, transfer_stats_data: dict[str, Any]):
# Should not be called when an ECConnector is not configured.
assert self.connector_cls is not None
# Called periodically when connector syncs with the scheduler.
# Note that this is not the same as the logging interval.
# We expect transfer_stats_data to be aggregated across all workers and
# consist of observations from a single connector.
transfer_stats = self.connector_cls.build_ec_connector_stats(
transfer_stats_data
)
if transfer_stats is None:
logger.warning_once(
"The connector %s is collecting stats but "
"does not implement the "
"`build_ec_connector_stats` method. "
"Stats will not be logged.",
self.connector_cls,
)
return
if self.transfer_stats_accumulator is None:
self.transfer_stats_accumulator = transfer_stats
else:
# Accumulate last interval stats.
self.transfer_stats_accumulator = self.transfer_stats_accumulator.aggregate(
transfer_stats
)
def log(self, log_fn=logger.info):
"""Log transfer metrics periodically, similar to throughput logging"""
if (
self.transfer_stats_accumulator
and not self.transfer_stats_accumulator.is_empty()
):
# Produce a single cumulative stats object for the last time
# interval from the recorded observations.
xfer_metrics = self.transfer_stats_accumulator.reduce()
xfer_metrics_str = ", ".join(f"{k}={v}" for k, v in xfer_metrics.items())
log_fn("EC Transfer metrics: %s", xfer_metrics_str)
# Reset metrics for next interval
self.reset()
class ECConnectorPromMetrics:
"""
A base class for per-connector Prometheus metric registration
and recording.
"""
def __init__(
self,
vllm_config: VllmConfig,
metric_types: dict[type[PromMetric], type[PromMetricT]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
):
self._ec_transfer_config = vllm_config.ec_transfer_config
self._gauge_cls = metric_types[Gauge]
self._counter_cls = metric_types[Counter]
self._histogram_cls = metric_types[Histogram]
self._labelnames = labelnames
self.per_engine_labelvalues = per_engine_labelvalues
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
"""
Record the supplied transfer statistics to Prometheus metrics. These
statistics are engine-specific, and should be recorded to a metric
with the appropriate 'engine' label. These metric instances can be
created using the create_metric_per_engine() helper method.
"""
raise NotImplementedError
class ECConnectorProm:
"""
Support for registering per-connector Prometheus metrics, and
recording transfer statistics to those metrics. Uses
ECConnectorBase.build_prom_metrics().
"""
_gauge_cls = Gauge
_counter_cls = Counter
_histogram_cls = Histogram
def __init__(
self,
vllm_config: VllmConfig,
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
):
self.prom_metrics: ECConnectorPromMetrics | None = None
ec_transfer_config = vllm_config.ec_transfer_config
if ec_transfer_config and ec_transfer_config.ec_connector:
connector_cls = ECConnectorFactory.get_connector_class(ec_transfer_config)
metric_types = {
Gauge: self._gauge_cls,
Counter: self._counter_cls,
Histogram: self._histogram_cls,
}
self.prom_metrics = connector_cls.build_prom_metrics(
vllm_config,
metric_types,
labelnames,
per_engine_labelvalues,
)
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
if self.prom_metrics is None:
return
self.prom_metrics.observe(transfer_stats_data, engine_idx)
@@ -103,6 +103,7 @@ class ECOutputAggregator:
finished_sending = set[str]()
finished_recving = set[str]()
worker_meta = None
connector_stats = None
for model_runner_output in outputs:
assert model_runner_output is not None
ec_output = model_runner_output.ec_connector_output
@@ -117,9 +118,17 @@ class ECOutputAggregator:
meta if worker_meta is None else worker_meta.aggregate(meta)
)
if stats := ec_output.ec_connector_stats:
connector_stats = (
stats
if connector_stats is None
else connector_stats.aggregate(stats)
)
aggregated = ECConnectorOutput(
finished_sending=finished_sending or None,
finished_recving=finished_recving or None,
ec_connector_stats=connector_stats,
ec_connector_worker_meta=worker_meta,
)
if aggregated.is_empty():