[EC Connector] Add Metrics Collection (#54960)

Signed-off-by: omerpaz95 <omerpaz95@gmail.com>
This commit is contained in:
omerpaz95
2026-09-17 10:36:23 +08:00
committed by GitHub
parent 2307a1d16c
commit fa3622a4a1
12 changed files with 472 additions and 3 deletions
+82
View File
@@ -19,6 +19,7 @@ from vllm.config import (
SpeculativeConfig,
VllmConfig,
)
from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
HiSparseConnector,
HiSparseConnectorScheduler,
@@ -5296,6 +5297,87 @@ def test_scheduler_kv_connector_stats():
assert final_stats == expected_data
def test_scheduler_ec_connector_stats():
"""Test worker-side, scheduler-side, and combined EC connector stats."""
class GenericECConnectorStats(ECConnectorStats):
def reset(self):
self.data = {}
def aggregate(self, other: ECConnectorStats) -> ECConnectorStats:
self.data.update(other.data)
return self
def reduce(self) -> dict[str, int | float]:
return {}
def is_empty(self) -> bool:
return not self.data
test_cases = (
({"worker": 1}, None, {"worker": 1}),
(None, {"scheduler": 2}, {"scheduler": 2}),
({"worker": 1}, {"scheduler": 2}, {"worker": 1, "scheduler": 2}),
)
for worker_data, scheduler_data, expected_data in test_cases:
scheduler = create_scheduler()
worker_stats = (
GenericECConnectorStats(data=worker_data) if worker_data else None
)
scheduler_stats = (
GenericECConnectorStats(data=scheduler_data) if scheduler_data else None
)
scheduler.ec_connector = Mock()
scheduler.ec_connector.take_unavailable_requests.return_value = set()
scheduler.ec_connector.get_ec_connector_stats.return_value = (
scheduler_stats if worker_stats is None else None
)
def update_connector_output(
ec_connector_output: ECConnectorOutput,
scheduler=scheduler,
scheduler_stats=scheduler_stats,
):
scheduler.ec_connector.get_ec_connector_stats.return_value = scheduler_stats
scheduler.ec_connector.update_connector_output.side_effect = (
update_connector_output
)
model_output = ModelRunnerOutput(
req_ids=["req_0"],
req_id_to_index={"req_0": 0},
sampled_token_ids=[[123]],
logprobs=None,
prompt_logprobs_dict={},
pooler_output=[None],
ec_connector_output=ECConnectorOutput(ec_connector_stats=worker_stats)
if worker_stats
else None,
)
scheduler_output = SchedulerOutput(
scheduled_new_reqs=[],
scheduled_cached_reqs=None,
num_scheduled_tokens={"req_0": 1},
total_num_scheduled_tokens=1,
scheduled_spec_decode_tokens={},
scheduled_encoder_inputs={},
num_common_prefix_blocks=[0],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
engine_core_outputs = scheduler.update_from_output(
scheduler_output, model_output
)
final_stats = next(
iter(engine_core_outputs.values())
).scheduler_stats.ec_connector_stats
assert final_stats == expected_data
def test_ec_connector_update_connector_output_called():
"""Test that worker-side EC connector output is forwarded to the
EC connector's update_connector_output hook."""
@@ -0,0 +1,85 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the connector-agnostic EC connector metrics plumbing.
Covers only the base infra: no concrete connector type implements these
hooks yet, so every assertion here is about the documented no-op defaults
and the container contracts, mirroring the equivalent KV connector tests.
"""
from unittest.mock import Mock
import pytest
from vllm.config import VllmConfig
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorRole
from vllm.distributed.ec_transfer.ec_connector.example_connector import (
ECExampleConnector,
)
from vllm.distributed.ec_transfer.ec_connector.metrics import (
ECConnectorLogging,
ECConnectorProm,
ECConnectorStats,
)
from vllm.v1.outputs import ECConnectorOutput
pytestmark = pytest.mark.cpu_test
@pytest.fixture
def example_connector(tmp_path):
config = Mock(spec=VllmConfig)
config.ec_transfer_config = Mock()
config.ec_transfer_config.get_from_extra_config = Mock(return_value=str(tmp_path))
config.ec_transfer_config.is_ec_producer = True
config.ec_transfer_config.is_ec_consumer = False
return ECExampleConnector(vllm_config=config, role=ECConnectorRole.SCHEDULER)
def test_base_stats_methods_raise_not_implemented():
"""The base container has no aggregation logic of its own."""
stats = ECConnectorStats()
with pytest.raises(NotImplementedError):
stats.reset()
with pytest.raises(NotImplementedError):
stats.aggregate(stats)
with pytest.raises(NotImplementedError):
stats.reduce()
with pytest.raises(NotImplementedError):
stats.is_empty()
def test_connector_hooks_default_to_none(example_connector):
"""No connector type overrides these yet, so every hook is a no-op."""
assert example_connector.get_ec_connector_stats() is None
assert ECExampleConnector.build_ec_connector_stats() is None
assert (
ECExampleConnector.build_prom_metrics(
vllm_config=Mock(spec=VllmConfig),
metric_types={},
labelnames=[],
per_engine_labelvalues={},
)
is None
)
def test_ec_connector_output_is_empty_reflects_stats_field():
output = ECConnectorOutput()
assert output.is_empty()
output.ec_connector_stats = ECConnectorStats(data={"x": 1})
assert not output.is_empty()
def test_logging_and_prom_are_no_ops_without_ec_transfer_config():
"""A normal run with no EC connector configured must stay silent."""
logging = ECConnectorLogging(ec_transfer_config=None)
logging.log() # Should not raise, nothing to log.
vllm_config = Mock(spec=VllmConfig)
vllm_config.ec_transfer_config = None
prom = ECConnectorProm(vllm_config, labelnames=[], per_engine_labelvalues={})
assert prom.prom_metrics is None
prom.observe({}) # Should not raise.
@@ -5,6 +5,7 @@
import pytest
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorWorkerMetadata
from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats
from vllm.distributed.ec_transfer.ec_connector.utils import ECOutputAggregator
from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator
from vllm.v1.outputs import (
@@ -29,6 +30,16 @@ class FakeWorkerMeta(ECConnectorWorkerMetadata):
return FakeWorkerMeta(self.saves + other.saves)
class FakeConnectorStats(ECConnectorStats):
"""Records merge order, mirroring FakeWorkerMeta above."""
def __init__(self, saves: list[str]):
self.saves = saves
def aggregate(self, other: "FakeConnectorStats") -> "FakeConnectorStats":
return FakeConnectorStats(self.saves + other.saves)
def _worker_output(ec_output: ECConnectorOutput | None) -> ModelRunnerOutput:
return ModelRunnerOutput(
req_ids=[], req_id_to_index={}, ec_connector_output=ec_output
@@ -38,19 +49,23 @@ def _worker_output(ec_output: ECConnectorOutput | None) -> ModelRunnerOutput:
def test_aggregate_folds_every_rank_onto_output_rank():
"""EC work done on any rank reaches the scheduler via output_rank's output.
The middle rank reports no worker metadata: it must neither seed nor clobber
the accumulator.
The middle rank reports no worker metadata or stats: it must neither seed
nor clobber either accumulator.
"""
outputs = [
_worker_output(
ECConnectorOutput(
finished_sending={"mm0"},
ec_connector_stats=FakeConnectorStats(["mm0"]),
ec_connector_worker_meta=FakeWorkerMeta(["mm0"]),
)
),
_worker_output(ECConnectorOutput(finished_recving={"mm1"})),
_worker_output(
ECConnectorOutput(ec_connector_worker_meta=FakeWorkerMeta(["mm2"]))
ECConnectorOutput(
ec_connector_stats=FakeConnectorStats(["mm2"]),
ec_connector_worker_meta=FakeWorkerMeta(["mm2"]),
)
),
]
@@ -59,6 +74,7 @@ def test_aggregate_folds_every_rank_onto_output_rank():
assert result is outputs[2]
assert result.ec_connector_output.finished_sending == {"mm0"}
assert result.ec_connector_output.finished_recving == {"mm1"}
assert result.ec_connector_output.ec_connector_stats.saves == ["mm0", "mm2"]
assert result.ec_connector_output.ec_connector_worker_meta.saves == ["mm0", "mm2"]
@@ -20,6 +20,7 @@ from vllm.v1.worker.gpu.ec_connector import NO_OP_EC_CONNECTOR, ActiveECConnecto
pytestmark = pytest.mark.cpu_test
WORKER_META = object()
CONNECTOR_STATS = object()
def _scheduler_output() -> SimpleNamespace:
@@ -38,6 +39,7 @@ def _connector(
fake.is_consumer = is_consumer
fake.get_finished.return_value = (None, None)
fake.build_connector_worker_meta.return_value = WORKER_META
fake.get_ec_connector_stats.return_value = CONNECTOR_STATS
with patch("vllm.v1.worker.gpu.ec_connector.get_ec_transfer", return_value=fake):
return ActiveECConnector(SimpleNamespace(), encoder_cache or {}), fake
@@ -72,6 +74,16 @@ def test_worker_meta_is_reported_on_context_exit():
assert fake.clear_connector_metadata.called
def test_connector_stats_are_reported_on_context_exit():
"""Reported in the finally block, alongside the worker meta."""
connector, fake = _connector()
with connector.maybe_get_output(_scheduler_output()) as output:
assert output.ec_connector_stats is None
assert output.ec_connector_stats is CONNECTOR_STATS
def test_v1_producer_receives_existing_encoder_cache():
encoder_cache = {"mm_cached": None}
fake = MagicMock(spec=ECConnectorBase)