diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index 3add7beda0..ef818f27c7 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -19,6 +19,7 @@ from vllm.config import ( SpeculativeConfig, VllmConfig, ) +from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import ( HiSparseConnector, HiSparseConnectorScheduler, @@ -5296,6 +5297,87 @@ def test_scheduler_kv_connector_stats(): assert final_stats == expected_data +def test_scheduler_ec_connector_stats(): + """Test worker-side, scheduler-side, and combined EC connector stats.""" + + class GenericECConnectorStats(ECConnectorStats): + def reset(self): + self.data = {} + + def aggregate(self, other: ECConnectorStats) -> ECConnectorStats: + self.data.update(other.data) + return self + + def reduce(self) -> dict[str, int | float]: + return {} + + def is_empty(self) -> bool: + return not self.data + + test_cases = ( + ({"worker": 1}, None, {"worker": 1}), + (None, {"scheduler": 2}, {"scheduler": 2}), + ({"worker": 1}, {"scheduler": 2}, {"worker": 1, "scheduler": 2}), + ) + + for worker_data, scheduler_data, expected_data in test_cases: + scheduler = create_scheduler() + worker_stats = ( + GenericECConnectorStats(data=worker_data) if worker_data else None + ) + scheduler_stats = ( + GenericECConnectorStats(data=scheduler_data) if scheduler_data else None + ) + scheduler.ec_connector = Mock() + scheduler.ec_connector.take_unavailable_requests.return_value = set() + scheduler.ec_connector.get_ec_connector_stats.return_value = ( + scheduler_stats if worker_stats is None else None + ) + + def update_connector_output( + ec_connector_output: ECConnectorOutput, + scheduler=scheduler, + scheduler_stats=scheduler_stats, + ): + scheduler.ec_connector.get_ec_connector_stats.return_value = scheduler_stats + + scheduler.ec_connector.update_connector_output.side_effect = ( + update_connector_output + ) + + model_output = ModelRunnerOutput( + req_ids=["req_0"], + req_id_to_index={"req_0": 0}, + sampled_token_ids=[[123]], + logprobs=None, + prompt_logprobs_dict={}, + pooler_output=[None], + ec_connector_output=ECConnectorOutput(ec_connector_stats=worker_stats) + if worker_stats + else None, + ) + scheduler_output = SchedulerOutput( + scheduled_new_reqs=[], + scheduled_cached_reqs=None, + num_scheduled_tokens={"req_0": 1}, + total_num_scheduled_tokens=1, + scheduled_spec_decode_tokens={}, + scheduled_encoder_inputs={}, + num_common_prefix_blocks=[0], + finished_req_ids=set(), + free_encoder_mm_hashes=[], + ) + + engine_core_outputs = scheduler.update_from_output( + scheduler_output, model_output + ) + + final_stats = next( + iter(engine_core_outputs.values()) + ).scheduler_stats.ec_connector_stats + assert final_stats == expected_data + + def test_ec_connector_update_connector_output_called(): """Test that worker-side EC connector output is forwarded to the EC connector's update_connector_output hook.""" diff --git a/tests/v1/ec_connector/unit/test_ec_connector_metrics.py b/tests/v1/ec_connector/unit/test_ec_connector_metrics.py new file mode 100644 index 0000000000..239f0267f0 --- /dev/null +++ b/tests/v1/ec_connector/unit/test_ec_connector_metrics.py @@ -0,0 +1,85 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for the connector-agnostic EC connector metrics plumbing. + +Covers only the base infra: no concrete connector type implements these +hooks yet, so every assertion here is about the documented no-op defaults +and the container contracts, mirroring the equivalent KV connector tests. +""" + +from unittest.mock import Mock + +import pytest + +from vllm.config import VllmConfig +from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorRole +from vllm.distributed.ec_transfer.ec_connector.example_connector import ( + ECExampleConnector, +) +from vllm.distributed.ec_transfer.ec_connector.metrics import ( + ECConnectorLogging, + ECConnectorProm, + ECConnectorStats, +) +from vllm.v1.outputs import ECConnectorOutput + +pytestmark = pytest.mark.cpu_test + + +@pytest.fixture +def example_connector(tmp_path): + config = Mock(spec=VllmConfig) + config.ec_transfer_config = Mock() + config.ec_transfer_config.get_from_extra_config = Mock(return_value=str(tmp_path)) + config.ec_transfer_config.is_ec_producer = True + config.ec_transfer_config.is_ec_consumer = False + return ECExampleConnector(vllm_config=config, role=ECConnectorRole.SCHEDULER) + + +def test_base_stats_methods_raise_not_implemented(): + """The base container has no aggregation logic of its own.""" + stats = ECConnectorStats() + + with pytest.raises(NotImplementedError): + stats.reset() + with pytest.raises(NotImplementedError): + stats.aggregate(stats) + with pytest.raises(NotImplementedError): + stats.reduce() + with pytest.raises(NotImplementedError): + stats.is_empty() + + +def test_connector_hooks_default_to_none(example_connector): + """No connector type overrides these yet, so every hook is a no-op.""" + assert example_connector.get_ec_connector_stats() is None + assert ECExampleConnector.build_ec_connector_stats() is None + assert ( + ECExampleConnector.build_prom_metrics( + vllm_config=Mock(spec=VllmConfig), + metric_types={}, + labelnames=[], + per_engine_labelvalues={}, + ) + is None + ) + + +def test_ec_connector_output_is_empty_reflects_stats_field(): + output = ECConnectorOutput() + assert output.is_empty() + + output.ec_connector_stats = ECConnectorStats(data={"x": 1}) + assert not output.is_empty() + + +def test_logging_and_prom_are_no_ops_without_ec_transfer_config(): + """A normal run with no EC connector configured must stay silent.""" + logging = ECConnectorLogging(ec_transfer_config=None) + logging.log() # Should not raise, nothing to log. + + vllm_config = Mock(spec=VllmConfig) + vllm_config.ec_transfer_config = None + prom = ECConnectorProm(vllm_config, labelnames=[], per_engine_labelvalues={}) + assert prom.prom_metrics is None + prom.observe({}) # Should not raise. diff --git a/tests/v1/ec_connector/unit/test_ec_output_aggregator.py b/tests/v1/ec_connector/unit/test_ec_output_aggregator.py index 4d5b7dae10..df98ec364f 100644 --- a/tests/v1/ec_connector/unit/test_ec_output_aggregator.py +++ b/tests/v1/ec_connector/unit/test_ec_output_aggregator.py @@ -5,6 +5,7 @@ import pytest from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorWorkerMetadata +from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats from vllm.distributed.ec_transfer.ec_connector.utils import ECOutputAggregator from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator from vllm.v1.outputs import ( @@ -29,6 +30,16 @@ class FakeWorkerMeta(ECConnectorWorkerMetadata): return FakeWorkerMeta(self.saves + other.saves) +class FakeConnectorStats(ECConnectorStats): + """Records merge order, mirroring FakeWorkerMeta above.""" + + def __init__(self, saves: list[str]): + self.saves = saves + + def aggregate(self, other: "FakeConnectorStats") -> "FakeConnectorStats": + return FakeConnectorStats(self.saves + other.saves) + + def _worker_output(ec_output: ECConnectorOutput | None) -> ModelRunnerOutput: return ModelRunnerOutput( req_ids=[], req_id_to_index={}, ec_connector_output=ec_output @@ -38,19 +49,23 @@ def _worker_output(ec_output: ECConnectorOutput | None) -> ModelRunnerOutput: def test_aggregate_folds_every_rank_onto_output_rank(): """EC work done on any rank reaches the scheduler via output_rank's output. - The middle rank reports no worker metadata: it must neither seed nor clobber - the accumulator. + The middle rank reports no worker metadata or stats: it must neither seed + nor clobber either accumulator. """ outputs = [ _worker_output( ECConnectorOutput( finished_sending={"mm0"}, + ec_connector_stats=FakeConnectorStats(["mm0"]), ec_connector_worker_meta=FakeWorkerMeta(["mm0"]), ) ), _worker_output(ECConnectorOutput(finished_recving={"mm1"})), _worker_output( - ECConnectorOutput(ec_connector_worker_meta=FakeWorkerMeta(["mm2"])) + ECConnectorOutput( + ec_connector_stats=FakeConnectorStats(["mm2"]), + ec_connector_worker_meta=FakeWorkerMeta(["mm2"]), + ) ), ] @@ -59,6 +74,7 @@ def test_aggregate_folds_every_rank_onto_output_rank(): assert result is outputs[2] assert result.ec_connector_output.finished_sending == {"mm0"} assert result.ec_connector_output.finished_recving == {"mm1"} + assert result.ec_connector_output.ec_connector_stats.saves == ["mm0", "mm2"] assert result.ec_connector_output.ec_connector_worker_meta.saves == ["mm0", "mm2"] diff --git a/tests/v1/ec_connector/unit/test_worker_ec_connector.py b/tests/v1/ec_connector/unit/test_worker_ec_connector.py index 01dad9d89e..911810b53a 100644 --- a/tests/v1/ec_connector/unit/test_worker_ec_connector.py +++ b/tests/v1/ec_connector/unit/test_worker_ec_connector.py @@ -20,6 +20,7 @@ from vllm.v1.worker.gpu.ec_connector import NO_OP_EC_CONNECTOR, ActiveECConnecto pytestmark = pytest.mark.cpu_test WORKER_META = object() +CONNECTOR_STATS = object() def _scheduler_output() -> SimpleNamespace: @@ -38,6 +39,7 @@ def _connector( fake.is_consumer = is_consumer fake.get_finished.return_value = (None, None) fake.build_connector_worker_meta.return_value = WORKER_META + fake.get_ec_connector_stats.return_value = CONNECTOR_STATS with patch("vllm.v1.worker.gpu.ec_connector.get_ec_transfer", return_value=fake): return ActiveECConnector(SimpleNamespace(), encoder_cache or {}), fake @@ -72,6 +74,16 @@ def test_worker_meta_is_reported_on_context_exit(): assert fake.clear_connector_metadata.called +def test_connector_stats_are_reported_on_context_exit(): + """Reported in the finally block, alongside the worker meta.""" + connector, fake = _connector() + + with connector.maybe_get_output(_scheduler_output()) as output: + assert output.ec_connector_stats is None + + assert output.ec_connector_stats is CONNECTOR_STATS + + def test_v1_producer_receives_existing_encoder_cache(): encoder_cache = {"mm_cached": None} fake = MagicMock(spec=ECConnectorBase) diff --git a/vllm/distributed/ec_transfer/ec_connector/base.py b/vllm/distributed/ec_transfer/ec_connector/base.py index ec3166ce33..716f443af2 100644 --- a/vllm/distributed/ec_transfer/ec_connector/base.py +++ b/vllm/distributed/ec_transfer/ec_connector/base.py @@ -35,6 +35,12 @@ from vllm.v1.outputs import ECConnectorOutput if TYPE_CHECKING: from vllm.config import VllmConfig + from vllm.distributed.ec_transfer.ec_connector.metrics import ( + ECConnectorPromMetrics, + ECConnectorStats, + PromMetric, + PromMetricT, + ) from vllm.v1.request import Request logger = init_logger(__name__) @@ -219,6 +225,17 @@ class ECConnectorBase(ABC): """ return None + def get_ec_connector_stats(self) -> "ECConnectorStats | None": + """ + Get the EC connector stats collected during the last interval. + + Callable on either a worker-side or scheduler-side connector + instance: the worker-side instance reports stats gathered during + cache save/load, while the scheduler-side instance reports stats + gathered during scheduling decisions. + """ + return None + # ============================== # Scheduler-side methods # ============================== @@ -336,3 +353,29 @@ class ECConnectorBase(ABC): leave this as False. """ return False + + @classmethod + def build_ec_connector_stats( + cls, data: dict[str, Any] | None = None + ) -> "ECConnectorStats | None": + """ + ECConnectorStats resolution method. This method allows dynamically + registered connectors to return their own ECConnectorStats object, + which can implement custom aggregation logic on the data dict. + """ + return None + + @classmethod + def build_prom_metrics( + cls, + vllm_config: "VllmConfig", + metric_types: dict[type["PromMetric"], type["PromMetricT"]], + labelnames: list[str], + per_engine_labelvalues: dict[int, list[object]], + ) -> "ECConnectorPromMetrics | None": + """ + Create an ECConnectorPromMetrics subclass which should register + per-connector Prometheus metrics and implement observe() to + expose connector transfer stats via Prometheus. + """ + return None diff --git a/vllm/distributed/ec_transfer/ec_connector/metrics.py b/vllm/distributed/ec_transfer/ec_connector/metrics.py new file mode 100644 index 0000000000..4a329096ce --- /dev/null +++ b/vllm/distributed/ec_transfer/ec_connector/metrics.py @@ -0,0 +1,175 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from dataclasses import dataclass, field +from typing import Any, TypeAlias, TypeVar + +from prometheus_client import Counter, Gauge, Histogram + +from vllm.config import ECTransferConfig, VllmConfig +from vllm.distributed.ec_transfer.ec_connector.factory import ECConnectorFactory +from vllm.logger import init_logger + +PromMetric: TypeAlias = Gauge | Counter | Histogram +PromMetricT = TypeVar("PromMetricT", bound=PromMetric) + +logger = init_logger(__name__) + + +@dataclass +class ECConnectorStats: + """ + Base class for EC Connector Stats, a container for transfer performance + metrics or otherwise important telemetry from the connector. + All sub-classes need to be serializable as stats are sent from worker to + logger process. + """ + + data: dict[str, Any] = field(default_factory=dict) + + def reset(self): + """Reset the stats, clear the state.""" + raise NotImplementedError + + def aggregate(self, other: "ECConnectorStats") -> "ECConnectorStats": + """ + Aggregate stats with another `ECConnectorStats` object. + """ + raise NotImplementedError + + def reduce(self) -> dict[str, int | float]: + """ + Reduce the observations collected during a time interval to one or + more representative values (eg avg/median/sum of the series). + This is meant to be called by the logger to produce a summary of the + stats for the last time interval. + """ + raise NotImplementedError + + def is_empty(self) -> bool: + """Return True if the stats are empty.""" + raise NotImplementedError + + +class ECConnectorLogging: + def __init__(self, ec_transfer_config: ECTransferConfig | None): + # Instantiate the connector's stats class. + if ec_transfer_config and ec_transfer_config.ec_connector: + self.connector_cls = ECConnectorFactory.get_connector_class( + ec_transfer_config + ) + self.reset() + + def reset(self): + self.transfer_stats_accumulator: ECConnectorStats | None = None + + def observe(self, transfer_stats_data: dict[str, Any]): + # Should not be called when an ECConnector is not configured. + assert self.connector_cls is not None + # Called periodically when connector syncs with the scheduler. + # Note that this is not the same as the logging interval. + # We expect transfer_stats_data to be aggregated across all workers and + # consist of observations from a single connector. + transfer_stats = self.connector_cls.build_ec_connector_stats( + transfer_stats_data + ) + if transfer_stats is None: + logger.warning_once( + "The connector %s is collecting stats but " + "does not implement the " + "`build_ec_connector_stats` method. " + "Stats will not be logged.", + self.connector_cls, + ) + return + + if self.transfer_stats_accumulator is None: + self.transfer_stats_accumulator = transfer_stats + else: + # Accumulate last interval stats. + self.transfer_stats_accumulator = self.transfer_stats_accumulator.aggregate( + transfer_stats + ) + + def log(self, log_fn=logger.info): + """Log transfer metrics periodically, similar to throughput logging""" + if ( + self.transfer_stats_accumulator + and not self.transfer_stats_accumulator.is_empty() + ): + # Produce a single cumulative stats object for the last time + # interval from the recorded observations. + xfer_metrics = self.transfer_stats_accumulator.reduce() + xfer_metrics_str = ", ".join(f"{k}={v}" for k, v in xfer_metrics.items()) + log_fn("EC Transfer metrics: %s", xfer_metrics_str) + + # Reset metrics for next interval + self.reset() + + +class ECConnectorPromMetrics: + """ + A base class for per-connector Prometheus metric registration + and recording. + """ + + def __init__( + self, + vllm_config: VllmConfig, + metric_types: dict[type[PromMetric], type[PromMetricT]], + labelnames: list[str], + per_engine_labelvalues: dict[int, list[object]], + ): + self._ec_transfer_config = vllm_config.ec_transfer_config + self._gauge_cls = metric_types[Gauge] + self._counter_cls = metric_types[Counter] + self._histogram_cls = metric_types[Histogram] + self._labelnames = labelnames + self.per_engine_labelvalues = per_engine_labelvalues + + def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0): + """ + Record the supplied transfer statistics to Prometheus metrics. These + statistics are engine-specific, and should be recorded to a metric + with the appropriate 'engine' label. These metric instances can be + created using the create_metric_per_engine() helper method. + """ + raise NotImplementedError + + +class ECConnectorProm: + """ + Support for registering per-connector Prometheus metrics, and + recording transfer statistics to those metrics. Uses + ECConnectorBase.build_prom_metrics(). + """ + + _gauge_cls = Gauge + _counter_cls = Counter + _histogram_cls = Histogram + + def __init__( + self, + vllm_config: VllmConfig, + labelnames: list[str], + per_engine_labelvalues: dict[int, list[object]], + ): + self.prom_metrics: ECConnectorPromMetrics | None = None + ec_transfer_config = vllm_config.ec_transfer_config + if ec_transfer_config and ec_transfer_config.ec_connector: + connector_cls = ECConnectorFactory.get_connector_class(ec_transfer_config) + metric_types = { + Gauge: self._gauge_cls, + Counter: self._counter_cls, + Histogram: self._histogram_cls, + } + self.prom_metrics = connector_cls.build_prom_metrics( + vllm_config, + metric_types, + labelnames, + per_engine_labelvalues, + ) + + def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0): + if self.prom_metrics is None: + return + self.prom_metrics.observe(transfer_stats_data, engine_idx) diff --git a/vllm/distributed/ec_transfer/ec_connector/utils.py b/vllm/distributed/ec_transfer/ec_connector/utils.py index 46963c2b41..f54d6877c1 100644 --- a/vllm/distributed/ec_transfer/ec_connector/utils.py +++ b/vllm/distributed/ec_transfer/ec_connector/utils.py @@ -103,6 +103,7 @@ class ECOutputAggregator: finished_sending = set[str]() finished_recving = set[str]() worker_meta = None + connector_stats = None for model_runner_output in outputs: assert model_runner_output is not None ec_output = model_runner_output.ec_connector_output @@ -117,9 +118,17 @@ class ECOutputAggregator: meta if worker_meta is None else worker_meta.aggregate(meta) ) + if stats := ec_output.ec_connector_stats: + connector_stats = ( + stats + if connector_stats is None + else connector_stats.aggregate(stats) + ) + aggregated = ECConnectorOutput( finished_sending=finished_sending or None, finished_recving=finished_recving or None, + ec_connector_stats=connector_stats, ec_connector_worker_meta=worker_meta, ) if aggregated.is_empty(): diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 22c231cc43..ed80daa8c2 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -15,6 +15,7 @@ from vllm.distributed.ec_transfer.ec_connector.base import ( ECConnectorRole, ) from vllm.distributed.ec_transfer.ec_connector.factory import ECConnectorFactory +from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats from vllm.distributed.kv_events import EventPublisherFactory, KVEventBatch from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory from vllm.distributed.kv_transfer.kv_connector.v1 import ( @@ -2243,6 +2244,23 @@ class Scheduler(SchedulerInterface): else scheduler_kv_connector_stats ) + # Worker-side EC connector stats from the model runner output. + ec_connector_stats: ECConnectorStats | None = ( + ec_connector_output.ec_connector_stats if ec_connector_output else None + ) + if self.ec_connector: + # Scheduler-side EC connector stats collected after connector update. + scheduler_ec_connector_stats = self.ec_connector.get_ec_connector_stats() + if ( + scheduler_ec_connector_stats is not None + and not scheduler_ec_connector_stats.is_empty() + ): + ec_connector_stats = ( + ec_connector_stats.aggregate(scheduler_ec_connector_stats) + if ec_connector_stats is not None + else scheduler_ec_connector_stats + ) + # collect KV cache events from KV cache manager events = self.kv_cache_manager.take_events() @@ -2287,6 +2305,7 @@ class Scheduler(SchedulerInterface): kv_connector_stats, cudagraph_stats, perf_stats, + ec_connector_stats=ec_connector_stats, ) ) is not None: # Return stats to only one of the front-ends. @@ -2772,6 +2791,7 @@ class Scheduler(SchedulerInterface): kv_connector_stats: KVConnectorStats | None = None, cudagraph_stats: CUDAGraphStat | None = None, perf_stats: PerfStats | None = None, + ec_connector_stats: ECConnectorStats | None = None, ) -> SchedulerStats | None: if not self.log_stats: return None @@ -2790,6 +2810,9 @@ class Scheduler(SchedulerInterface): connector_stats_payload = ( kv_connector_stats.to_dict() if kv_connector_stats else None ) + ec_connector_stats_payload = ( + ec_connector_stats.data if ec_connector_stats else None + ) return SchedulerStats( num_running_reqs=len(self.running), num_waiting_reqs=len(self.waiting), @@ -2802,6 +2825,7 @@ class Scheduler(SchedulerInterface): kv_connector_stats=connector_stats_payload, cudagraph_stats=cudagraph_stats, perf_stats=perf_stats, + ec_connector_stats=ec_connector_stats_payload, ) def make_spec_decoding_stats( diff --git a/vllm/v1/metrics/loggers.py b/vllm/v1/metrics/loggers.py index 50e606d3b3..fee6679fe7 100644 --- a/vllm/v1/metrics/loggers.py +++ b/vllm/v1/metrics/loggers.py @@ -11,6 +11,10 @@ from prometheus_client import Counter, Gauge, Histogram import vllm.envs as envs from vllm.compilation.cuda_graph import CUDAGraphLogging from vllm.config import SupportsMetricsInfo, VllmConfig +from vllm.distributed.ec_transfer.ec_connector.metrics import ( + ECConnectorLogging, + ECConnectorProm, +) from vllm.distributed.kv_transfer.kv_connector.v1.metrics import ( KVConnectorLogging, KVConnectorProm, @@ -118,6 +122,8 @@ class LoggingStatLogger(StatLoggerBase): self.spec_decoding_logging = SpecDecodingLogging(is_diffusion=is_diffusion) kv_transfer_config = self.vllm_config.kv_transfer_config self.kv_connector_logging = KVConnectorLogging(kv_transfer_config) + ec_transfer_config = self.vllm_config.ec_transfer_config + self.ec_connector_logging = ECConnectorLogging(ec_transfer_config) self.cudagraph_logging = None if self.vllm_config.observability_config.cudagraph_metrics: self.cudagraph_logging = CUDAGraphLogging( @@ -224,6 +230,8 @@ class LoggingStatLogger(StatLoggerBase): self.spec_decoding_logging.observe(scheduler_stats.spec_decoding_stats) if kv_connector_stats := scheduler_stats.kv_connector_stats: self.kv_connector_logging.observe(kv_connector_stats) + if ec_connector_stats := scheduler_stats.ec_connector_stats: + self.ec_connector_logging.observe(ec_connector_stats) if ( self.cudagraph_logging is not None and scheduler_stats.cudagraph_stats is not None @@ -319,6 +327,7 @@ class LoggingStatLogger(StatLoggerBase): self.spec_decoding_logging.log(log_fn=log_fn) self.kv_connector_logging.log(log_fn=log_fn) + self.ec_connector_logging.log(log_fn=log_fn) if self.cudagraph_logging is not None: self.cudagraph_logging.log(log_fn=log_fn) if self._enable_perf_stats(): @@ -451,6 +460,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase): _histogram_cls = Histogram _spec_decoding_cls = SpecDecodingProm _kv_connector_cls = KVConnectorProm + _ec_connector_cls = ECConnectorProm _perf_metrics_cls = PerfMetricsProm def __init__( @@ -488,6 +498,9 @@ class PrometheusStatLogger(AggregateStatLoggerBase): self.kv_connector_prom = self._kv_connector_cls( vllm_config, labelnames, per_engine_labelvalues ) + self.ec_connector_prom = self._ec_connector_cls( + vllm_config, labelnames, per_engine_labelvalues + ) self.perf_metrics_prom = self._perf_metrics_cls( vllm_config, labelnames, per_engine_labelvalues ) @@ -1064,6 +1077,11 @@ class PrometheusStatLogger(AggregateStatLoggerBase): scheduler_stats.kv_connector_stats, engine_idx ) + if scheduler_stats.ec_connector_stats is not None: + self.ec_connector_prom.observe( + scheduler_stats.ec_connector_stats, engine_idx + ) + if scheduler_stats.perf_stats is not None: self.perf_metrics_prom.observe(scheduler_stats.perf_stats, engine_idx) diff --git a/vllm/v1/metrics/stats.py b/vllm/v1/metrics/stats.py index fe17208ce4..66088c7954 100644 --- a/vllm/v1/metrics/stats.py +++ b/vllm/v1/metrics/stats.py @@ -205,6 +205,7 @@ class SchedulerStats: spec_decoding_stats: SpecDecodingStats | None = None kv_connector_stats: dict[str, Any] | None = None + ec_connector_stats: dict[str, Any] | None = None waiting_lora_adapters: dict[str, int] = field(default_factory=dict) running_lora_adapters: dict[str, int] = field(default_factory=dict) diff --git a/vllm/v1/outputs.py b/vllm/v1/outputs.py index 35780dce9c..c4535f47f2 100644 --- a/vllm/v1/outputs.py +++ b/vllm/v1/outputs.py @@ -16,6 +16,7 @@ from vllm.v1.core.sched.output import SchedulerOutput if TYPE_CHECKING: from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorWorkerMetadata + from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats from vllm.distributed.kv_events import KVConnectorKVEvents from vllm.distributed.kv_transfer.kv_connector.v1.base import ( KVConnectorWorkerMetadata, @@ -307,12 +308,14 @@ class ECConnectorOutput: # [mm_hash] finished_sending: set[str] | None = None finished_recving: set[str] | None = None + ec_connector_stats: "ECConnectorStats | None" = None ec_connector_worker_meta: ECConnectorWorkerMetadata | None = None def is_empty(self): return ( not self.finished_sending and not self.finished_recving + and not self.ec_connector_stats and not self.ec_connector_worker_meta ) diff --git a/vllm/v1/worker/gpu/ec_connector.py b/vllm/v1/worker/gpu/ec_connector.py index 1285d9fd14..c81fd7c11e 100644 --- a/vllm/v1/worker/gpu/ec_connector.py +++ b/vllm/v1/worker/gpu/ec_connector.py @@ -80,6 +80,7 @@ class ActiveECConnector(ECConnector): output.finished_sending, output.finished_recving = ( ec_connector.get_finished(scheduler_output.finished_req_ids) ) + output.ec_connector_stats = ec_connector.get_ec_connector_stats() output.ec_connector_worker_meta = ec_connector.build_connector_worker_meta() ec_connector.clear_connector_metadata()