From 2307a1d16cdd0077cd882bb9fc41b453f6f1999e Mon Sep 17 00:00:00 2001 From: Oxana Korzh Date: Wed, 16 Sep 2026 20:30:59 -0600 Subject: [PATCH 1/4] [Core] Don't issue a blocking collective RPC during engine handshake (#57226) Signed-off-by: okorzh Co-authored-by: Andreas Karatzas --- tests/v1/engine/test_engine_core_client.py | 38 ++++++++++++++++++++++ vllm/v1/engine/core.py | 2 ++ 2 files changed, 40 insertions(+) diff --git a/tests/v1/engine/test_engine_core_client.py b/tests/v1/engine/test_engine_core_client.py index a23ae46b8f..a153b23b10 100644 --- a/tests/v1/engine/test_engine_core_client.py +++ b/tests/v1/engine/test_engine_core_client.py @@ -1452,3 +1452,41 @@ def test_engine_core_proc_instantiation_cuda_empty(monkeypatch: pytest.MonkeyPat ) engine_core_proc.shutdown() + + +def _ready_response_with_weight_transfer(weight_transfer_config): + """Build a ready response against a mock executor.""" + from vllm.v1.engine.core import EngineCoreProc + + executor = MagicMock() + executor.supports_draft_weight_updates.return_value = True + + proc = MagicMock() + proc.model_executor = executor + proc.vllm_config = MagicMock() + proc.vllm_config.weight_transfer_config = weight_transfer_config + proc.vllm_config.lora_config = None + proc.vllm_config.model_config.enable_sleep_mode = False + proc.scheduler = MagicMock() + + response = EngineCoreProc._make_ready_response(proc) + return response, executor + + +def test_ready_response_skips_draft_rpc_without_weight_transfer(): + """No weight-transfer config: the executor must not be consulted.""" + response, executor = _ready_response_with_weight_transfer(None) + + executor.supports_draft_weight_updates.assert_not_called() + assert response.supports_draft_weight_updates is False + + +def test_ready_response_queries_executor_with_weight_transfer(): + """Weight transfer configured: the executor is still consulted.""" + weight_transfer_config = MagicMock() + weight_transfer_config.backend = "nixl" + + response, executor = _ready_response_with_weight_transfer(weight_transfer_config) + + executor.supports_draft_weight_updates.assert_called_once() + assert response.supports_draft_weight_updates is True diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index 9c84827a63..7da7f2e0ec 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -1691,6 +1691,8 @@ class EngineCoreProc(EngineCore): enable_sleep_mode=self.vllm_config.model_config.enable_sleep_mode, supports_draft_weight_updates=( self.model_executor.supports_draft_weight_updates() + if self.vllm_config.weight_transfer_config is not None + else False ), ) From fa3622a4a1cd16294a7843895a3246bf29c452c4 Mon Sep 17 00:00:00 2001 From: omerpaz95 <73347585+omerpaz95@users.noreply.github.com> Date: Thu, 17 Sep 2026 05:36:23 +0300 Subject: [PATCH 2/4] [EC Connector] Add Metrics Collection (#54960) Signed-off-by: omerpaz95 --- tests/v1/core/test_scheduler.py | 82 ++++++++ .../unit/test_ec_connector_metrics.py | 85 +++++++++ .../unit/test_ec_output_aggregator.py | 22 ++- .../unit/test_worker_ec_connector.py | 12 ++ .../ec_transfer/ec_connector/base.py | 43 +++++ .../ec_transfer/ec_connector/metrics.py | 175 ++++++++++++++++++ .../ec_transfer/ec_connector/utils.py | 9 + vllm/v1/core/sched/scheduler.py | 24 +++ vllm/v1/metrics/loggers.py | 18 ++ vllm/v1/metrics/stats.py | 1 + vllm/v1/outputs.py | 3 + vllm/v1/worker/gpu/ec_connector.py | 1 + 12 files changed, 472 insertions(+), 3 deletions(-) create mode 100644 tests/v1/ec_connector/unit/test_ec_connector_metrics.py create mode 100644 vllm/distributed/ec_transfer/ec_connector/metrics.py diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index 3add7beda0..ef818f27c7 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -19,6 +19,7 @@ from vllm.config import ( SpeculativeConfig, VllmConfig, ) +from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import ( HiSparseConnector, HiSparseConnectorScheduler, @@ -5296,6 +5297,87 @@ def test_scheduler_kv_connector_stats(): assert final_stats == expected_data +def test_scheduler_ec_connector_stats(): + """Test worker-side, scheduler-side, and combined EC connector stats.""" + + class GenericECConnectorStats(ECConnectorStats): + def reset(self): + self.data = {} + + def aggregate(self, other: ECConnectorStats) -> ECConnectorStats: + self.data.update(other.data) + return self + + def reduce(self) -> dict[str, int | float]: + return {} + + def is_empty(self) -> bool: + return not self.data + + test_cases = ( + ({"worker": 1}, None, {"worker": 1}), + (None, {"scheduler": 2}, {"scheduler": 2}), + ({"worker": 1}, {"scheduler": 2}, {"worker": 1, "scheduler": 2}), + ) + + for worker_data, scheduler_data, expected_data in test_cases: + scheduler = create_scheduler() + worker_stats = ( + GenericECConnectorStats(data=worker_data) if worker_data else None + ) + scheduler_stats = ( + GenericECConnectorStats(data=scheduler_data) if scheduler_data else None + ) + scheduler.ec_connector = Mock() + scheduler.ec_connector.take_unavailable_requests.return_value = set() + scheduler.ec_connector.get_ec_connector_stats.return_value = ( + scheduler_stats if worker_stats is None else None + ) + + def update_connector_output( + ec_connector_output: ECConnectorOutput, + scheduler=scheduler, + scheduler_stats=scheduler_stats, + ): + scheduler.ec_connector.get_ec_connector_stats.return_value = scheduler_stats + + scheduler.ec_connector.update_connector_output.side_effect = ( + update_connector_output + ) + + model_output = ModelRunnerOutput( + req_ids=["req_0"], + req_id_to_index={"req_0": 0}, + sampled_token_ids=[[123]], + logprobs=None, + prompt_logprobs_dict={}, + pooler_output=[None], + ec_connector_output=ECConnectorOutput(ec_connector_stats=worker_stats) + if worker_stats + else None, + ) + scheduler_output = SchedulerOutput( + scheduled_new_reqs=[], + scheduled_cached_reqs=None, + num_scheduled_tokens={"req_0": 1}, + total_num_scheduled_tokens=1, + scheduled_spec_decode_tokens={}, + scheduled_encoder_inputs={}, + num_common_prefix_blocks=[0], + finished_req_ids=set(), + free_encoder_mm_hashes=[], + ) + + engine_core_outputs = scheduler.update_from_output( + scheduler_output, model_output + ) + + final_stats = next( + iter(engine_core_outputs.values()) + ).scheduler_stats.ec_connector_stats + assert final_stats == expected_data + + def test_ec_connector_update_connector_output_called(): """Test that worker-side EC connector output is forwarded to the EC connector's update_connector_output hook.""" diff --git a/tests/v1/ec_connector/unit/test_ec_connector_metrics.py b/tests/v1/ec_connector/unit/test_ec_connector_metrics.py new file mode 100644 index 0000000000..239f0267f0 --- /dev/null +++ b/tests/v1/ec_connector/unit/test_ec_connector_metrics.py @@ -0,0 +1,85 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for the connector-agnostic EC connector metrics plumbing. + +Covers only the base infra: no concrete connector type implements these +hooks yet, so every assertion here is about the documented no-op defaults +and the container contracts, mirroring the equivalent KV connector tests. +""" + +from unittest.mock import Mock + +import pytest + +from vllm.config import VllmConfig +from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorRole +from vllm.distributed.ec_transfer.ec_connector.example_connector import ( + ECExampleConnector, +) +from vllm.distributed.ec_transfer.ec_connector.metrics import ( + ECConnectorLogging, + ECConnectorProm, + ECConnectorStats, +) +from vllm.v1.outputs import ECConnectorOutput + +pytestmark = pytest.mark.cpu_test + + +@pytest.fixture +def example_connector(tmp_path): + config = Mock(spec=VllmConfig) + config.ec_transfer_config = Mock() + config.ec_transfer_config.get_from_extra_config = Mock(return_value=str(tmp_path)) + config.ec_transfer_config.is_ec_producer = True + config.ec_transfer_config.is_ec_consumer = False + return ECExampleConnector(vllm_config=config, role=ECConnectorRole.SCHEDULER) + + +def test_base_stats_methods_raise_not_implemented(): + """The base container has no aggregation logic of its own.""" + stats = ECConnectorStats() + + with pytest.raises(NotImplementedError): + stats.reset() + with pytest.raises(NotImplementedError): + stats.aggregate(stats) + with pytest.raises(NotImplementedError): + stats.reduce() + with pytest.raises(NotImplementedError): + stats.is_empty() + + +def test_connector_hooks_default_to_none(example_connector): + """No connector type overrides these yet, so every hook is a no-op.""" + assert example_connector.get_ec_connector_stats() is None + assert ECExampleConnector.build_ec_connector_stats() is None + assert ( + ECExampleConnector.build_prom_metrics( + vllm_config=Mock(spec=VllmConfig), + metric_types={}, + labelnames=[], + per_engine_labelvalues={}, + ) + is None + ) + + +def test_ec_connector_output_is_empty_reflects_stats_field(): + output = ECConnectorOutput() + assert output.is_empty() + + output.ec_connector_stats = ECConnectorStats(data={"x": 1}) + assert not output.is_empty() + + +def test_logging_and_prom_are_no_ops_without_ec_transfer_config(): + """A normal run with no EC connector configured must stay silent.""" + logging = ECConnectorLogging(ec_transfer_config=None) + logging.log() # Should not raise, nothing to log. + + vllm_config = Mock(spec=VllmConfig) + vllm_config.ec_transfer_config = None + prom = ECConnectorProm(vllm_config, labelnames=[], per_engine_labelvalues={}) + assert prom.prom_metrics is None + prom.observe({}) # Should not raise. diff --git a/tests/v1/ec_connector/unit/test_ec_output_aggregator.py b/tests/v1/ec_connector/unit/test_ec_output_aggregator.py index 4d5b7dae10..df98ec364f 100644 --- a/tests/v1/ec_connector/unit/test_ec_output_aggregator.py +++ b/tests/v1/ec_connector/unit/test_ec_output_aggregator.py @@ -5,6 +5,7 @@ import pytest from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorWorkerMetadata +from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats from vllm.distributed.ec_transfer.ec_connector.utils import ECOutputAggregator from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator from vllm.v1.outputs import ( @@ -29,6 +30,16 @@ class FakeWorkerMeta(ECConnectorWorkerMetadata): return FakeWorkerMeta(self.saves + other.saves) +class FakeConnectorStats(ECConnectorStats): + """Records merge order, mirroring FakeWorkerMeta above.""" + + def __init__(self, saves: list[str]): + self.saves = saves + + def aggregate(self, other: "FakeConnectorStats") -> "FakeConnectorStats": + return FakeConnectorStats(self.saves + other.saves) + + def _worker_output(ec_output: ECConnectorOutput | None) -> ModelRunnerOutput: return ModelRunnerOutput( req_ids=[], req_id_to_index={}, ec_connector_output=ec_output @@ -38,19 +49,23 @@ def _worker_output(ec_output: ECConnectorOutput | None) -> ModelRunnerOutput: def test_aggregate_folds_every_rank_onto_output_rank(): """EC work done on any rank reaches the scheduler via output_rank's output. - The middle rank reports no worker metadata: it must neither seed nor clobber - the accumulator. + The middle rank reports no worker metadata or stats: it must neither seed + nor clobber either accumulator. """ outputs = [ _worker_output( ECConnectorOutput( finished_sending={"mm0"}, + ec_connector_stats=FakeConnectorStats(["mm0"]), ec_connector_worker_meta=FakeWorkerMeta(["mm0"]), ) ), _worker_output(ECConnectorOutput(finished_recving={"mm1"})), _worker_output( - ECConnectorOutput(ec_connector_worker_meta=FakeWorkerMeta(["mm2"])) + ECConnectorOutput( + ec_connector_stats=FakeConnectorStats(["mm2"]), + ec_connector_worker_meta=FakeWorkerMeta(["mm2"]), + ) ), ] @@ -59,6 +74,7 @@ def test_aggregate_folds_every_rank_onto_output_rank(): assert result is outputs[2] assert result.ec_connector_output.finished_sending == {"mm0"} assert result.ec_connector_output.finished_recving == {"mm1"} + assert result.ec_connector_output.ec_connector_stats.saves == ["mm0", "mm2"] assert result.ec_connector_output.ec_connector_worker_meta.saves == ["mm0", "mm2"] diff --git a/tests/v1/ec_connector/unit/test_worker_ec_connector.py b/tests/v1/ec_connector/unit/test_worker_ec_connector.py index 01dad9d89e..911810b53a 100644 --- a/tests/v1/ec_connector/unit/test_worker_ec_connector.py +++ b/tests/v1/ec_connector/unit/test_worker_ec_connector.py @@ -20,6 +20,7 @@ from vllm.v1.worker.gpu.ec_connector import NO_OP_EC_CONNECTOR, ActiveECConnecto pytestmark = pytest.mark.cpu_test WORKER_META = object() +CONNECTOR_STATS = object() def _scheduler_output() -> SimpleNamespace: @@ -38,6 +39,7 @@ def _connector( fake.is_consumer = is_consumer fake.get_finished.return_value = (None, None) fake.build_connector_worker_meta.return_value = WORKER_META + fake.get_ec_connector_stats.return_value = CONNECTOR_STATS with patch("vllm.v1.worker.gpu.ec_connector.get_ec_transfer", return_value=fake): return ActiveECConnector(SimpleNamespace(), encoder_cache or {}), fake @@ -72,6 +74,16 @@ def test_worker_meta_is_reported_on_context_exit(): assert fake.clear_connector_metadata.called +def test_connector_stats_are_reported_on_context_exit(): + """Reported in the finally block, alongside the worker meta.""" + connector, fake = _connector() + + with connector.maybe_get_output(_scheduler_output()) as output: + assert output.ec_connector_stats is None + + assert output.ec_connector_stats is CONNECTOR_STATS + + def test_v1_producer_receives_existing_encoder_cache(): encoder_cache = {"mm_cached": None} fake = MagicMock(spec=ECConnectorBase) diff --git a/vllm/distributed/ec_transfer/ec_connector/base.py b/vllm/distributed/ec_transfer/ec_connector/base.py index ec3166ce33..716f443af2 100644 --- a/vllm/distributed/ec_transfer/ec_connector/base.py +++ b/vllm/distributed/ec_transfer/ec_connector/base.py @@ -35,6 +35,12 @@ from vllm.v1.outputs import ECConnectorOutput if TYPE_CHECKING: from vllm.config import VllmConfig + from vllm.distributed.ec_transfer.ec_connector.metrics import ( + ECConnectorPromMetrics, + ECConnectorStats, + PromMetric, + PromMetricT, + ) from vllm.v1.request import Request logger = init_logger(__name__) @@ -219,6 +225,17 @@ class ECConnectorBase(ABC): """ return None + def get_ec_connector_stats(self) -> "ECConnectorStats | None": + """ + Get the EC connector stats collected during the last interval. + + Callable on either a worker-side or scheduler-side connector + instance: the worker-side instance reports stats gathered during + cache save/load, while the scheduler-side instance reports stats + gathered during scheduling decisions. + """ + return None + # ============================== # Scheduler-side methods # ============================== @@ -336,3 +353,29 @@ class ECConnectorBase(ABC): leave this as False. """ return False + + @classmethod + def build_ec_connector_stats( + cls, data: dict[str, Any] | None = None + ) -> "ECConnectorStats | None": + """ + ECConnectorStats resolution method. This method allows dynamically + registered connectors to return their own ECConnectorStats object, + which can implement custom aggregation logic on the data dict. + """ + return None + + @classmethod + def build_prom_metrics( + cls, + vllm_config: "VllmConfig", + metric_types: dict[type["PromMetric"], type["PromMetricT"]], + labelnames: list[str], + per_engine_labelvalues: dict[int, list[object]], + ) -> "ECConnectorPromMetrics | None": + """ + Create an ECConnectorPromMetrics subclass which should register + per-connector Prometheus metrics and implement observe() to + expose connector transfer stats via Prometheus. + """ + return None diff --git a/vllm/distributed/ec_transfer/ec_connector/metrics.py b/vllm/distributed/ec_transfer/ec_connector/metrics.py new file mode 100644 index 0000000000..4a329096ce --- /dev/null +++ b/vllm/distributed/ec_transfer/ec_connector/metrics.py @@ -0,0 +1,175 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from dataclasses import dataclass, field +from typing import Any, TypeAlias, TypeVar + +from prometheus_client import Counter, Gauge, Histogram + +from vllm.config import ECTransferConfig, VllmConfig +from vllm.distributed.ec_transfer.ec_connector.factory import ECConnectorFactory +from vllm.logger import init_logger + +PromMetric: TypeAlias = Gauge | Counter | Histogram +PromMetricT = TypeVar("PromMetricT", bound=PromMetric) + +logger = init_logger(__name__) + + +@dataclass +class ECConnectorStats: + """ + Base class for EC Connector Stats, a container for transfer performance + metrics or otherwise important telemetry from the connector. + All sub-classes need to be serializable as stats are sent from worker to + logger process. + """ + + data: dict[str, Any] = field(default_factory=dict) + + def reset(self): + """Reset the stats, clear the state.""" + raise NotImplementedError + + def aggregate(self, other: "ECConnectorStats") -> "ECConnectorStats": + """ + Aggregate stats with another `ECConnectorStats` object. + """ + raise NotImplementedError + + def reduce(self) -> dict[str, int | float]: + """ + Reduce the observations collected during a time interval to one or + more representative values (eg avg/median/sum of the series). + This is meant to be called by the logger to produce a summary of the + stats for the last time interval. + """ + raise NotImplementedError + + def is_empty(self) -> bool: + """Return True if the stats are empty.""" + raise NotImplementedError + + +class ECConnectorLogging: + def __init__(self, ec_transfer_config: ECTransferConfig | None): + # Instantiate the connector's stats class. + if ec_transfer_config and ec_transfer_config.ec_connector: + self.connector_cls = ECConnectorFactory.get_connector_class( + ec_transfer_config + ) + self.reset() + + def reset(self): + self.transfer_stats_accumulator: ECConnectorStats | None = None + + def observe(self, transfer_stats_data: dict[str, Any]): + # Should not be called when an ECConnector is not configured. + assert self.connector_cls is not None + # Called periodically when connector syncs with the scheduler. + # Note that this is not the same as the logging interval. + # We expect transfer_stats_data to be aggregated across all workers and + # consist of observations from a single connector. + transfer_stats = self.connector_cls.build_ec_connector_stats( + transfer_stats_data + ) + if transfer_stats is None: + logger.warning_once( + "The connector %s is collecting stats but " + "does not implement the " + "`build_ec_connector_stats` method. " + "Stats will not be logged.", + self.connector_cls, + ) + return + + if self.transfer_stats_accumulator is None: + self.transfer_stats_accumulator = transfer_stats + else: + # Accumulate last interval stats. + self.transfer_stats_accumulator = self.transfer_stats_accumulator.aggregate( + transfer_stats + ) + + def log(self, log_fn=logger.info): + """Log transfer metrics periodically, similar to throughput logging""" + if ( + self.transfer_stats_accumulator + and not self.transfer_stats_accumulator.is_empty() + ): + # Produce a single cumulative stats object for the last time + # interval from the recorded observations. + xfer_metrics = self.transfer_stats_accumulator.reduce() + xfer_metrics_str = ", ".join(f"{k}={v}" for k, v in xfer_metrics.items()) + log_fn("EC Transfer metrics: %s", xfer_metrics_str) + + # Reset metrics for next interval + self.reset() + + +class ECConnectorPromMetrics: + """ + A base class for per-connector Prometheus metric registration + and recording. + """ + + def __init__( + self, + vllm_config: VllmConfig, + metric_types: dict[type[PromMetric], type[PromMetricT]], + labelnames: list[str], + per_engine_labelvalues: dict[int, list[object]], + ): + self._ec_transfer_config = vllm_config.ec_transfer_config + self._gauge_cls = metric_types[Gauge] + self._counter_cls = metric_types[Counter] + self._histogram_cls = metric_types[Histogram] + self._labelnames = labelnames + self.per_engine_labelvalues = per_engine_labelvalues + + def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0): + """ + Record the supplied transfer statistics to Prometheus metrics. These + statistics are engine-specific, and should be recorded to a metric + with the appropriate 'engine' label. These metric instances can be + created using the create_metric_per_engine() helper method. + """ + raise NotImplementedError + + +class ECConnectorProm: + """ + Support for registering per-connector Prometheus metrics, and + recording transfer statistics to those metrics. Uses + ECConnectorBase.build_prom_metrics(). + """ + + _gauge_cls = Gauge + _counter_cls = Counter + _histogram_cls = Histogram + + def __init__( + self, + vllm_config: VllmConfig, + labelnames: list[str], + per_engine_labelvalues: dict[int, list[object]], + ): + self.prom_metrics: ECConnectorPromMetrics | None = None + ec_transfer_config = vllm_config.ec_transfer_config + if ec_transfer_config and ec_transfer_config.ec_connector: + connector_cls = ECConnectorFactory.get_connector_class(ec_transfer_config) + metric_types = { + Gauge: self._gauge_cls, + Counter: self._counter_cls, + Histogram: self._histogram_cls, + } + self.prom_metrics = connector_cls.build_prom_metrics( + vllm_config, + metric_types, + labelnames, + per_engine_labelvalues, + ) + + def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0): + if self.prom_metrics is None: + return + self.prom_metrics.observe(transfer_stats_data, engine_idx) diff --git a/vllm/distributed/ec_transfer/ec_connector/utils.py b/vllm/distributed/ec_transfer/ec_connector/utils.py index 46963c2b41..f54d6877c1 100644 --- a/vllm/distributed/ec_transfer/ec_connector/utils.py +++ b/vllm/distributed/ec_transfer/ec_connector/utils.py @@ -103,6 +103,7 @@ class ECOutputAggregator: finished_sending = set[str]() finished_recving = set[str]() worker_meta = None + connector_stats = None for model_runner_output in outputs: assert model_runner_output is not None ec_output = model_runner_output.ec_connector_output @@ -117,9 +118,17 @@ class ECOutputAggregator: meta if worker_meta is None else worker_meta.aggregate(meta) ) + if stats := ec_output.ec_connector_stats: + connector_stats = ( + stats + if connector_stats is None + else connector_stats.aggregate(stats) + ) + aggregated = ECConnectorOutput( finished_sending=finished_sending or None, finished_recving=finished_recving or None, + ec_connector_stats=connector_stats, ec_connector_worker_meta=worker_meta, ) if aggregated.is_empty(): diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 22c231cc43..ed80daa8c2 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -15,6 +15,7 @@ from vllm.distributed.ec_transfer.ec_connector.base import ( ECConnectorRole, ) from vllm.distributed.ec_transfer.ec_connector.factory import ECConnectorFactory +from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats from vllm.distributed.kv_events import EventPublisherFactory, KVEventBatch from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory from vllm.distributed.kv_transfer.kv_connector.v1 import ( @@ -2243,6 +2244,23 @@ class Scheduler(SchedulerInterface): else scheduler_kv_connector_stats ) + # Worker-side EC connector stats from the model runner output. + ec_connector_stats: ECConnectorStats | None = ( + ec_connector_output.ec_connector_stats if ec_connector_output else None + ) + if self.ec_connector: + # Scheduler-side EC connector stats collected after connector update. + scheduler_ec_connector_stats = self.ec_connector.get_ec_connector_stats() + if ( + scheduler_ec_connector_stats is not None + and not scheduler_ec_connector_stats.is_empty() + ): + ec_connector_stats = ( + ec_connector_stats.aggregate(scheduler_ec_connector_stats) + if ec_connector_stats is not None + else scheduler_ec_connector_stats + ) + # collect KV cache events from KV cache manager events = self.kv_cache_manager.take_events() @@ -2287,6 +2305,7 @@ class Scheduler(SchedulerInterface): kv_connector_stats, cudagraph_stats, perf_stats, + ec_connector_stats=ec_connector_stats, ) ) is not None: # Return stats to only one of the front-ends. @@ -2772,6 +2791,7 @@ class Scheduler(SchedulerInterface): kv_connector_stats: KVConnectorStats | None = None, cudagraph_stats: CUDAGraphStat | None = None, perf_stats: PerfStats | None = None, + ec_connector_stats: ECConnectorStats | None = None, ) -> SchedulerStats | None: if not self.log_stats: return None @@ -2790,6 +2810,9 @@ class Scheduler(SchedulerInterface): connector_stats_payload = ( kv_connector_stats.to_dict() if kv_connector_stats else None ) + ec_connector_stats_payload = ( + ec_connector_stats.data if ec_connector_stats else None + ) return SchedulerStats( num_running_reqs=len(self.running), num_waiting_reqs=len(self.waiting), @@ -2802,6 +2825,7 @@ class Scheduler(SchedulerInterface): kv_connector_stats=connector_stats_payload, cudagraph_stats=cudagraph_stats, perf_stats=perf_stats, + ec_connector_stats=ec_connector_stats_payload, ) def make_spec_decoding_stats( diff --git a/vllm/v1/metrics/loggers.py b/vllm/v1/metrics/loggers.py index 50e606d3b3..fee6679fe7 100644 --- a/vllm/v1/metrics/loggers.py +++ b/vllm/v1/metrics/loggers.py @@ -11,6 +11,10 @@ from prometheus_client import Counter, Gauge, Histogram import vllm.envs as envs from vllm.compilation.cuda_graph import CUDAGraphLogging from vllm.config import SupportsMetricsInfo, VllmConfig +from vllm.distributed.ec_transfer.ec_connector.metrics import ( + ECConnectorLogging, + ECConnectorProm, +) from vllm.distributed.kv_transfer.kv_connector.v1.metrics import ( KVConnectorLogging, KVConnectorProm, @@ -118,6 +122,8 @@ class LoggingStatLogger(StatLoggerBase): self.spec_decoding_logging = SpecDecodingLogging(is_diffusion=is_diffusion) kv_transfer_config = self.vllm_config.kv_transfer_config self.kv_connector_logging = KVConnectorLogging(kv_transfer_config) + ec_transfer_config = self.vllm_config.ec_transfer_config + self.ec_connector_logging = ECConnectorLogging(ec_transfer_config) self.cudagraph_logging = None if self.vllm_config.observability_config.cudagraph_metrics: self.cudagraph_logging = CUDAGraphLogging( @@ -224,6 +230,8 @@ class LoggingStatLogger(StatLoggerBase): self.spec_decoding_logging.observe(scheduler_stats.spec_decoding_stats) if kv_connector_stats := scheduler_stats.kv_connector_stats: self.kv_connector_logging.observe(kv_connector_stats) + if ec_connector_stats := scheduler_stats.ec_connector_stats: + self.ec_connector_logging.observe(ec_connector_stats) if ( self.cudagraph_logging is not None and scheduler_stats.cudagraph_stats is not None @@ -319,6 +327,7 @@ class LoggingStatLogger(StatLoggerBase): self.spec_decoding_logging.log(log_fn=log_fn) self.kv_connector_logging.log(log_fn=log_fn) + self.ec_connector_logging.log(log_fn=log_fn) if self.cudagraph_logging is not None: self.cudagraph_logging.log(log_fn=log_fn) if self._enable_perf_stats(): @@ -451,6 +460,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase): _histogram_cls = Histogram _spec_decoding_cls = SpecDecodingProm _kv_connector_cls = KVConnectorProm + _ec_connector_cls = ECConnectorProm _perf_metrics_cls = PerfMetricsProm def __init__( @@ -488,6 +498,9 @@ class PrometheusStatLogger(AggregateStatLoggerBase): self.kv_connector_prom = self._kv_connector_cls( vllm_config, labelnames, per_engine_labelvalues ) + self.ec_connector_prom = self._ec_connector_cls( + vllm_config, labelnames, per_engine_labelvalues + ) self.perf_metrics_prom = self._perf_metrics_cls( vllm_config, labelnames, per_engine_labelvalues ) @@ -1064,6 +1077,11 @@ class PrometheusStatLogger(AggregateStatLoggerBase): scheduler_stats.kv_connector_stats, engine_idx ) + if scheduler_stats.ec_connector_stats is not None: + self.ec_connector_prom.observe( + scheduler_stats.ec_connector_stats, engine_idx + ) + if scheduler_stats.perf_stats is not None: self.perf_metrics_prom.observe(scheduler_stats.perf_stats, engine_idx) diff --git a/vllm/v1/metrics/stats.py b/vllm/v1/metrics/stats.py index fe17208ce4..66088c7954 100644 --- a/vllm/v1/metrics/stats.py +++ b/vllm/v1/metrics/stats.py @@ -205,6 +205,7 @@ class SchedulerStats: spec_decoding_stats: SpecDecodingStats | None = None kv_connector_stats: dict[str, Any] | None = None + ec_connector_stats: dict[str, Any] | None = None waiting_lora_adapters: dict[str, int] = field(default_factory=dict) running_lora_adapters: dict[str, int] = field(default_factory=dict) diff --git a/vllm/v1/outputs.py b/vllm/v1/outputs.py index 35780dce9c..c4535f47f2 100644 --- a/vllm/v1/outputs.py +++ b/vllm/v1/outputs.py @@ -16,6 +16,7 @@ from vllm.v1.core.sched.output import SchedulerOutput if TYPE_CHECKING: from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorWorkerMetadata + from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats from vllm.distributed.kv_events import KVConnectorKVEvents from vllm.distributed.kv_transfer.kv_connector.v1.base import ( KVConnectorWorkerMetadata, @@ -307,12 +308,14 @@ class ECConnectorOutput: # [mm_hash] finished_sending: set[str] | None = None finished_recving: set[str] | None = None + ec_connector_stats: "ECConnectorStats | None" = None ec_connector_worker_meta: ECConnectorWorkerMetadata | None = None def is_empty(self): return ( not self.finished_sending and not self.finished_recving + and not self.ec_connector_stats and not self.ec_connector_worker_meta ) diff --git a/vllm/v1/worker/gpu/ec_connector.py b/vllm/v1/worker/gpu/ec_connector.py index 1285d9fd14..c81fd7c11e 100644 --- a/vllm/v1/worker/gpu/ec_connector.py +++ b/vllm/v1/worker/gpu/ec_connector.py @@ -80,6 +80,7 @@ class ActiveECConnector(ECConnector): output.finished_sending, output.finished_recving = ( ec_connector.get_finished(scheduler_output.finished_req_ids) ) + output.ec_connector_stats = ec_connector.get_ec_connector_stats() output.ec_connector_worker_meta = ec_connector.build_connector_worker_meta() ec_connector.clear_connector_metadata() From 5efbf0886a7110ae0801b9540e3072e89a23eb36 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Wed, 16 Sep 2026 19:40:21 -0700 Subject: [PATCH 3/4] [CI] Support `FORCE_COLOR` env var; enable colors in CPU CI output (#56913) Signed-off-by: Nick Hill --- .../scripts/hardware_ci/run-cpu-test-arm.sh | 2 +- .../scripts/hardware_ci/run-cpu-test.sh | 2 +- tests/test_logger.py | 23 +++++++++++++++++++ vllm/envs.py | 4 ++++ vllm/logger.py | 2 +- 5 files changed, 30 insertions(+), 3 deletions(-) diff --git a/.buildkite/scripts/hardware_ci/run-cpu-test-arm.sh b/.buildkite/scripts/hardware_ci/run-cpu-test-arm.sh index 77e0f3e07b..689ca92d6a 100755 --- a/.buildkite/scripts/hardware_ci/run-cpu-test-arm.sh +++ b/.buildkite/scripts/hardware_ci/run-cpu-test-arm.sh @@ -30,7 +30,7 @@ remove_docker_container docker pull "$IMAGE" # Run the image -docker run -itd --cpuset-cpus="$CORE_RANGE" --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=16 --env VLLM_CPU_CI_ENV=1 -e E2E_OMP_THREADS="$OMP_CORE_RANGE" --shm-size=4g --name "$CONTAINER_NAME" "$IMAGE" +docker run -itd --cpuset-cpus="$CORE_RANGE" --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=16 --env VLLM_CPU_CI_ENV=1 -e E2E_OMP_THREADS="$OMP_CORE_RANGE" -e TERM=xterm-256color -e PY_COLORS=1 -e FORCE_COLOR=1 -e CLICOLOR_FORCE=1 --shm-size=4g --name "$CONTAINER_NAME" "$IMAGE" print_packages() { docker exec "$CONTAINER_NAME" bash -c " diff --git a/.buildkite/scripts/hardware_ci/run-cpu-test.sh b/.buildkite/scripts/hardware_ci/run-cpu-test.sh index 43555a19ab..eecf70ea2b 100644 --- a/.buildkite/scripts/hardware_ci/run-cpu-test.sh +++ b/.buildkite/scripts/hardware_ci/run-cpu-test.sh @@ -99,7 +99,7 @@ rm -f "$build_log" OFFLINE_RETRY_PATTERN='huggingface_hub\.errors\.(LocalEntryNotFoundError|OfflineModeIsEnabled)|Invalid repository ID or local directory specified|Cannot find any model weights with' run_test() { local hf_offline=$1 - docker run --rm --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" -v ~/.cache/huggingface:/root/.cache/huggingface -v ~/.cache/vllm:/root/.cache/vllm --privileged=true -e HF_TOKEN -e VLLM_CPU_KVCACHE_SPACE=16 -e VLLM_CPU_CI_ENV=1 -e VLLM_CPU_SIM_MULTI_NUMA=1 -e VLLM_CPU_ATTN_SPLIT_KV=0 -e HF_HUB_OFFLINE="$hf_offline" -e HF_DATASETS_OFFLINE="$hf_offline" --shm-size=4g "$IMAGE_NAME" \ + docker run --rm --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" -v ~/.cache/huggingface:/root/.cache/huggingface -v ~/.cache/vllm:/root/.cache/vllm --privileged=true -e HF_TOKEN -e VLLM_CPU_KVCACHE_SPACE=16 -e VLLM_CPU_CI_ENV=1 -e VLLM_CPU_SIM_MULTI_NUMA=1 -e VLLM_CPU_ATTN_SPLIT_KV=0 -e HF_HUB_OFFLINE="$hf_offline" -e HF_DATASETS_OFFLINE="$hf_offline" -e TERM=xterm-256color -e PY_COLORS=1 -e FORCE_COLOR=1 -e CLICOLOR_FORCE=1 --shm-size=4g "$IMAGE_NAME" \ timeout "$TIMEOUT_VAL" bash -c "set -euox pipefail; echo \"--- Print packages\"; pip list; echo \"--- Running tests\"; ${TEST_COMMAND}" } diff --git a/tests/test_logger.py b/tests/test_logger.py index 2ff100151b..9dcc4a1ea8 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import enum +import io import json import logging import os @@ -19,6 +20,7 @@ from vllm.logger import ( _DATE_FORMAT, _FORMAT, _configure_vllm_root_logger, + _use_color, enable_trace_function_call, init_logger, ) @@ -72,6 +74,27 @@ def test_default_vllm_root_logger_configuration(monkeypatch): assert formatter.datefmt == _DATE_FORMAT +def test_use_color_force_color(monkeypatch): + """FORCE_COLOR forces colored logs without a TTY, while NO_COLOR and an + explicit VLLM_LOGGING_COLOR=0 take precedence over it.""" + monkeypatch.setattr(sys, "stdout", io.StringIO()) + monkeypatch.setattr(sys, "stderr", io.StringIO()) + for var in ("NO_COLOR", "FORCE_COLOR", "VLLM_LOGGING_COLOR"): + monkeypatch.delenv(var, raising=False) + + assert not _use_color() + + monkeypatch.setenv("FORCE_COLOR", "1") + assert _use_color() + + monkeypatch.setenv("VLLM_LOGGING_COLOR", "0") + assert not _use_color() + monkeypatch.delenv("VLLM_LOGGING_COLOR") + + monkeypatch.setenv("NO_COLOR", "1") + assert not _use_color() + + def test_descendent_loggers_depend_on_and_propagate_logs_to_root_logger(monkeypatch): """This test presumes that VLLM_CONFIGURE_LOGGING (default: True) and VLLM_LOGGING_CONFIG_PATH (default: None) are not configured and default diff --git a/vllm/envs.py b/vllm/envs.py index 0a3f6d3c67..c512df457a 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -44,6 +44,7 @@ if TYPE_CHECKING: VLLM_LOGGING_CONFIG_PATH: str | None = None VLLM_LOGGING_COLOR: str = "auto" NO_COLOR: bool = False + FORCE_COLOR: bool = False VLLM_LOG_STATS_INTERVAL: float = 10.0 VLLM_TRACE_FUNCTION: int = 0 VLLM_USE_FLASHINFER_SAMPLER: bool = True @@ -840,6 +841,8 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_LOGGING_COLOR": lambda: os.getenv("VLLM_LOGGING_COLOR", "auto"), # Standard unix flag for disabling ANSI color codes "NO_COLOR": lambda: os.getenv("NO_COLOR", "0") != "0", + # De-facto standard flag for forcing ANSI color codes (e.g. non-tty case) + "FORCE_COLOR": lambda: os.getenv("FORCE_COLOR", "0") != "0", # If set, vllm will log stats at this interval in seconds # If not set, vllm will log stats every 10 seconds. "VLLM_LOG_STATS_INTERVAL": lambda: ( @@ -2335,6 +2338,7 @@ def compile_factors() -> dict[str, object]: "CUDA_VISIBLE_DEVICES", "NO_COLOR", "VLLM_SKIP_VERSION_SUFFIX", + "FORCE_COLOR", } from vllm.config.utils import normalize_value diff --git a/vllm/logger.py b/vllm/logger.py index cdce7748f2..05ee5936d4 100644 --- a/vllm/logger.py +++ b/vllm/logger.py @@ -29,7 +29,7 @@ _DATE_FORMAT = "%m-%d %H:%M:%S" def _use_color() -> bool: if envs.NO_COLOR or envs.VLLM_LOGGING_COLOR == "0": return False - if envs.VLLM_LOGGING_COLOR == "1": + if envs.VLLM_LOGGING_COLOR == "1" or envs.FORCE_COLOR: return True if envs.VLLM_LOGGING_STREAM == "ext://sys.stdout": # stdout return hasattr(sys.stdout, "isatty") and sys.stdout.isatty() From 9854b580dfe518171eed3a6ad76cbd402ca2bdfa Mon Sep 17 00:00:00 2001 From: Itay Etelis <92247226+Etelis@users.noreply.github.com> Date: Thu, 17 Sep 2026 06:24:06 +0300 Subject: [PATCH 4/4] [Bugfix][KV Offload] Compact canonical MLA rows (#56799) Signed-off-by: Itay Etelis <92247226+Etelis@users.noreply.github.com> Co-authored-by: Or Ozeri --- .../unit/offloading_connector/test_config.py | 40 +++++++++++++++++++ .../unit/offloading_connector/test_worker.py | 8 +++- .../kv_connector/v1/offloading/config.py | 14 +++++++ .../kv_connector/v1/offloading/worker.py | 4 +- 4 files changed, 63 insertions(+), 3 deletions(-) diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_config.py b/tests/v1/kv_connector/unit/offloading_connector/test_config.py index f597c8fa27..acf55972df 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_config.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_config.py @@ -17,6 +17,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import ( SchedulerOffloadConfig, ) from vllm.platforms import current_platform +from vllm.v1.core.kv_cache_utils import generate_scheduler_kv_cache_config from vllm.v1.kv_cache_interface import ( FullAttentionSpec, HiddenStateCacheSpec, @@ -32,6 +33,8 @@ from vllm.v1.kv_cache_interface import ( UniformTypeKVCacheSpecs, ) from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry +from vllm.v1.kv_offload.file_mapper import FileMapper +from vllm.v1.kv_offload.tiering.spec import TieringOffloadingSpec def _make_vllm_config( @@ -876,6 +879,43 @@ def test_canonical_layout_certifies_v2_model_runner(): assert _parallelism_agnostic(groups, canonical=True, v2=True) +@pytest.mark.parametrize("tp_size", [1, 2, 4]) +@pytest.mark.parametrize("scheduler", [False, True]) +def test_canonical_mla_dsa_rows_are_tp_independent(tp_size, scheduler): + specs = { + f"l{i}": _mla_spec(block_size=64, head_size=size, dtype=torch.uint8) + for i, size in enumerate([656] * 4 + [132] * 3) + } + uniform = UniformTypeKVCacheSpecs(block_size=64, kv_cache_specs=specs) + kv_cache_config = KVCacheConfig( + num_blocks=4, + kv_cache_tensors=[ + KVCacheTensor( + size=4 * uniform.page_size_bytes, + layers=list(specs), + layer_stride=0, + block_stride=uniform.page_size_bytes, + ) + ], + kv_cache_groups=[KVCacheGroupSpec(list(specs), uniform)], + ) + if scheduler: + kv_cache_config = generate_scheduler_kv_cache_config([kv_cache_config]) + config = _make_vllm_config( + tensor_parallel_size=tp_size, + extra_config={"canonical_layout": True, "cpu_bytes_to_use": 196608 * 8}, + ) + config.cache_config.kv_cache_layout = "LBHNC" + config.parallel_config.distributed_executor_backend = "mp" + offloading_config = build_offloading_config(config, kv_cache_config) + spec = TieringOffloadingSpec(offloading_config) + assert spec.kv_bytes_per_chunk == 196608 + assert spec.num_chunks == 8 + mapper = FileMapper.from_offloading_spec("/cache", spec, parallel_agnostic=True) + assert mapper.get_run_config()["tp_size"] == 1 + assert mapper.get_run_config()["replicated_layout"] is True + + def test_parallelism_agnostic_disabled_on_v2_model_runner(): config = _make_vllm_config() config.use_v2_model_runner = True diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_worker.py b/tests/v1/kv_connector/unit/offloading_connector/test_worker.py index 7a987fef3d..64f481d02e 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_worker.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_worker.py @@ -97,6 +97,7 @@ def _make_worker( kv_cache_config: KVCacheConfig, replicated_layout: bool = False, rank: int = 0, + canonical_layout: bool = False, ): """Create an OffloadingConnectorWorker with mocked dependencies.""" from vllm.distributed.kv_transfer.kv_connector.v1.offloading.worker import ( @@ -106,6 +107,7 @@ def _make_worker( spec = MagicMock(spec=OffloadingSpec) spec.replicated_layout = replicated_layout spec.config = MagicMock() + spec.config.canonical_layout = canonical_layout spec.config.parallel.rank = rank spec.get_worker.return_value = MagicMock() @@ -204,11 +206,13 @@ def test_prepare_store_kv_non_writer_marks_completed_without_submit(): assert meta.completed_jobs == {7: 1} -def test_prepare_store_kv_writer_submits_store(): +@pytest.mark.parametrize("rank,canonical_layout", [(0, False), (0, True), (1, True)]) +def test_prepare_store_kv_writer_submits_store(rank, canonical_layout): worker, _ = _make_worker( KVCacheConfig(num_blocks=0, kv_cache_tensors=[], kv_cache_groups=[]), replicated_layout=True, - rank=0, + rank=rank, + canonical_layout=canonical_layout, ) worker.prepare_store_kv(_store_metadata(8)) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/config.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/config.py index c4e6d0f2a2..ca503d4e67 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/config.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/config.py @@ -233,6 +233,20 @@ def build_offloading_config( and parallel_config.world_size == tp_size ) + if canonical_layout and is_parallelism_agnostic: + replicated_layout = ( + all( + type(spec) is MLAAttentionSpec + for _, group in selected_groups + for spec in iter_layer_specs(group.kv_cache_spec) + ) + and parallel_config.nnodes_within_dp == 1 + and ( + parallel_config.world_size == 1 + or parallel_config.distributed_executor_backend == "mp" + ) + ) + kv_events_config = vllm_config.kv_events_config cache_dtype = ( vllm_config.model_config.dtype diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py index 94e8a8109d..5b14e50dda 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py @@ -53,7 +53,9 @@ class OffloadingConnectorWorker: self.worker: OffloadingWorker | None = None # Non-writers still ack: pending_count waits for world_size per job. self._is_store_writer = ( - not self.spec.replicated_layout or self.spec.config.parallel.rank == 0 + not self.spec.replicated_layout + or self.spec.config.canonical_layout + or self.spec.config.parallel.rank == 0 ) # job_id -> req_id for in-flight loads.