[EC Connector] Add Metrics Collection (#54960)
Signed-off-by: omerpaz95 <omerpaz95@gmail.com>
This commit is contained in:
@@ -19,6 +19,7 @@ from vllm.config import (
|
||||
SpeculativeConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
|
||||
HiSparseConnector,
|
||||
HiSparseConnectorScheduler,
|
||||
@@ -5296,6 +5297,87 @@ def test_scheduler_kv_connector_stats():
|
||||
assert final_stats == expected_data
|
||||
|
||||
|
||||
def test_scheduler_ec_connector_stats():
|
||||
"""Test worker-side, scheduler-side, and combined EC connector stats."""
|
||||
|
||||
class GenericECConnectorStats(ECConnectorStats):
|
||||
def reset(self):
|
||||
self.data = {}
|
||||
|
||||
def aggregate(self, other: ECConnectorStats) -> ECConnectorStats:
|
||||
self.data.update(other.data)
|
||||
return self
|
||||
|
||||
def reduce(self) -> dict[str, int | float]:
|
||||
return {}
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
return not self.data
|
||||
|
||||
test_cases = (
|
||||
({"worker": 1}, None, {"worker": 1}),
|
||||
(None, {"scheduler": 2}, {"scheduler": 2}),
|
||||
({"worker": 1}, {"scheduler": 2}, {"worker": 1, "scheduler": 2}),
|
||||
)
|
||||
|
||||
for worker_data, scheduler_data, expected_data in test_cases:
|
||||
scheduler = create_scheduler()
|
||||
worker_stats = (
|
||||
GenericECConnectorStats(data=worker_data) if worker_data else None
|
||||
)
|
||||
scheduler_stats = (
|
||||
GenericECConnectorStats(data=scheduler_data) if scheduler_data else None
|
||||
)
|
||||
scheduler.ec_connector = Mock()
|
||||
scheduler.ec_connector.take_unavailable_requests.return_value = set()
|
||||
scheduler.ec_connector.get_ec_connector_stats.return_value = (
|
||||
scheduler_stats if worker_stats is None else None
|
||||
)
|
||||
|
||||
def update_connector_output(
|
||||
ec_connector_output: ECConnectorOutput,
|
||||
scheduler=scheduler,
|
||||
scheduler_stats=scheduler_stats,
|
||||
):
|
||||
scheduler.ec_connector.get_ec_connector_stats.return_value = scheduler_stats
|
||||
|
||||
scheduler.ec_connector.update_connector_output.side_effect = (
|
||||
update_connector_output
|
||||
)
|
||||
|
||||
model_output = ModelRunnerOutput(
|
||||
req_ids=["req_0"],
|
||||
req_id_to_index={"req_0": 0},
|
||||
sampled_token_ids=[[123]],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[None],
|
||||
ec_connector_output=ECConnectorOutput(ec_connector_stats=worker_stats)
|
||||
if worker_stats
|
||||
else None,
|
||||
)
|
||||
scheduler_output = SchedulerOutput(
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_cached_reqs=None,
|
||||
num_scheduled_tokens={"req_0": 1},
|
||||
total_num_scheduled_tokens=1,
|
||||
scheduled_spec_decode_tokens={},
|
||||
scheduled_encoder_inputs={},
|
||||
num_common_prefix_blocks=[0],
|
||||
finished_req_ids=set(),
|
||||
free_encoder_mm_hashes=[],
|
||||
)
|
||||
|
||||
engine_core_outputs = scheduler.update_from_output(
|
||||
scheduler_output, model_output
|
||||
)
|
||||
|
||||
final_stats = next(
|
||||
iter(engine_core_outputs.values())
|
||||
).scheduler_stats.ec_connector_stats
|
||||
assert final_stats == expected_data
|
||||
|
||||
|
||||
def test_ec_connector_update_connector_output_called():
|
||||
"""Test that worker-side EC connector output is forwarded to the
|
||||
EC connector's update_connector_output hook."""
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the connector-agnostic EC connector metrics plumbing.
|
||||
|
||||
Covers only the base infra: no concrete connector type implements these
|
||||
hooks yet, so every assertion here is about the documented no-op defaults
|
||||
and the container contracts, mirroring the equivalent KV connector tests.
|
||||
"""
|
||||
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorRole
|
||||
from vllm.distributed.ec_transfer.ec_connector.example_connector import (
|
||||
ECExampleConnector,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.metrics import (
|
||||
ECConnectorLogging,
|
||||
ECConnectorProm,
|
||||
ECConnectorStats,
|
||||
)
|
||||
from vllm.v1.outputs import ECConnectorOutput
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def example_connector(tmp_path):
|
||||
config = Mock(spec=VllmConfig)
|
||||
config.ec_transfer_config = Mock()
|
||||
config.ec_transfer_config.get_from_extra_config = Mock(return_value=str(tmp_path))
|
||||
config.ec_transfer_config.is_ec_producer = True
|
||||
config.ec_transfer_config.is_ec_consumer = False
|
||||
return ECExampleConnector(vllm_config=config, role=ECConnectorRole.SCHEDULER)
|
||||
|
||||
|
||||
def test_base_stats_methods_raise_not_implemented():
|
||||
"""The base container has no aggregation logic of its own."""
|
||||
stats = ECConnectorStats()
|
||||
|
||||
with pytest.raises(NotImplementedError):
|
||||
stats.reset()
|
||||
with pytest.raises(NotImplementedError):
|
||||
stats.aggregate(stats)
|
||||
with pytest.raises(NotImplementedError):
|
||||
stats.reduce()
|
||||
with pytest.raises(NotImplementedError):
|
||||
stats.is_empty()
|
||||
|
||||
|
||||
def test_connector_hooks_default_to_none(example_connector):
|
||||
"""No connector type overrides these yet, so every hook is a no-op."""
|
||||
assert example_connector.get_ec_connector_stats() is None
|
||||
assert ECExampleConnector.build_ec_connector_stats() is None
|
||||
assert (
|
||||
ECExampleConnector.build_prom_metrics(
|
||||
vllm_config=Mock(spec=VllmConfig),
|
||||
metric_types={},
|
||||
labelnames=[],
|
||||
per_engine_labelvalues={},
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_ec_connector_output_is_empty_reflects_stats_field():
|
||||
output = ECConnectorOutput()
|
||||
assert output.is_empty()
|
||||
|
||||
output.ec_connector_stats = ECConnectorStats(data={"x": 1})
|
||||
assert not output.is_empty()
|
||||
|
||||
|
||||
def test_logging_and_prom_are_no_ops_without_ec_transfer_config():
|
||||
"""A normal run with no EC connector configured must stay silent."""
|
||||
logging = ECConnectorLogging(ec_transfer_config=None)
|
||||
logging.log() # Should not raise, nothing to log.
|
||||
|
||||
vllm_config = Mock(spec=VllmConfig)
|
||||
vllm_config.ec_transfer_config = None
|
||||
prom = ECConnectorProm(vllm_config, labelnames=[], per_engine_labelvalues={})
|
||||
assert prom.prom_metrics is None
|
||||
prom.observe({}) # Should not raise.
|
||||
@@ -5,6 +5,7 @@
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorWorkerMetadata
|
||||
from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats
|
||||
from vllm.distributed.ec_transfer.ec_connector.utils import ECOutputAggregator
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator
|
||||
from vllm.v1.outputs import (
|
||||
@@ -29,6 +30,16 @@ class FakeWorkerMeta(ECConnectorWorkerMetadata):
|
||||
return FakeWorkerMeta(self.saves + other.saves)
|
||||
|
||||
|
||||
class FakeConnectorStats(ECConnectorStats):
|
||||
"""Records merge order, mirroring FakeWorkerMeta above."""
|
||||
|
||||
def __init__(self, saves: list[str]):
|
||||
self.saves = saves
|
||||
|
||||
def aggregate(self, other: "FakeConnectorStats") -> "FakeConnectorStats":
|
||||
return FakeConnectorStats(self.saves + other.saves)
|
||||
|
||||
|
||||
def _worker_output(ec_output: ECConnectorOutput | None) -> ModelRunnerOutput:
|
||||
return ModelRunnerOutput(
|
||||
req_ids=[], req_id_to_index={}, ec_connector_output=ec_output
|
||||
@@ -38,19 +49,23 @@ def _worker_output(ec_output: ECConnectorOutput | None) -> ModelRunnerOutput:
|
||||
def test_aggregate_folds_every_rank_onto_output_rank():
|
||||
"""EC work done on any rank reaches the scheduler via output_rank's output.
|
||||
|
||||
The middle rank reports no worker metadata: it must neither seed nor clobber
|
||||
the accumulator.
|
||||
The middle rank reports no worker metadata or stats: it must neither seed
|
||||
nor clobber either accumulator.
|
||||
"""
|
||||
outputs = [
|
||||
_worker_output(
|
||||
ECConnectorOutput(
|
||||
finished_sending={"mm0"},
|
||||
ec_connector_stats=FakeConnectorStats(["mm0"]),
|
||||
ec_connector_worker_meta=FakeWorkerMeta(["mm0"]),
|
||||
)
|
||||
),
|
||||
_worker_output(ECConnectorOutput(finished_recving={"mm1"})),
|
||||
_worker_output(
|
||||
ECConnectorOutput(ec_connector_worker_meta=FakeWorkerMeta(["mm2"]))
|
||||
ECConnectorOutput(
|
||||
ec_connector_stats=FakeConnectorStats(["mm2"]),
|
||||
ec_connector_worker_meta=FakeWorkerMeta(["mm2"]),
|
||||
)
|
||||
),
|
||||
]
|
||||
|
||||
@@ -59,6 +74,7 @@ def test_aggregate_folds_every_rank_onto_output_rank():
|
||||
assert result is outputs[2]
|
||||
assert result.ec_connector_output.finished_sending == {"mm0"}
|
||||
assert result.ec_connector_output.finished_recving == {"mm1"}
|
||||
assert result.ec_connector_output.ec_connector_stats.saves == ["mm0", "mm2"]
|
||||
assert result.ec_connector_output.ec_connector_worker_meta.saves == ["mm0", "mm2"]
|
||||
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@ from vllm.v1.worker.gpu.ec_connector import NO_OP_EC_CONNECTOR, ActiveECConnecto
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
WORKER_META = object()
|
||||
CONNECTOR_STATS = object()
|
||||
|
||||
|
||||
def _scheduler_output() -> SimpleNamespace:
|
||||
@@ -38,6 +39,7 @@ def _connector(
|
||||
fake.is_consumer = is_consumer
|
||||
fake.get_finished.return_value = (None, None)
|
||||
fake.build_connector_worker_meta.return_value = WORKER_META
|
||||
fake.get_ec_connector_stats.return_value = CONNECTOR_STATS
|
||||
with patch("vllm.v1.worker.gpu.ec_connector.get_ec_transfer", return_value=fake):
|
||||
return ActiveECConnector(SimpleNamespace(), encoder_cache or {}), fake
|
||||
|
||||
@@ -72,6 +74,16 @@ def test_worker_meta_is_reported_on_context_exit():
|
||||
assert fake.clear_connector_metadata.called
|
||||
|
||||
|
||||
def test_connector_stats_are_reported_on_context_exit():
|
||||
"""Reported in the finally block, alongside the worker meta."""
|
||||
connector, fake = _connector()
|
||||
|
||||
with connector.maybe_get_output(_scheduler_output()) as output:
|
||||
assert output.ec_connector_stats is None
|
||||
|
||||
assert output.ec_connector_stats is CONNECTOR_STATS
|
||||
|
||||
|
||||
def test_v1_producer_receives_existing_encoder_cache():
|
||||
encoder_cache = {"mm_cached": None}
|
||||
fake = MagicMock(spec=ECConnectorBase)
|
||||
|
||||
Reference in New Issue
Block a user