[EC Connector] Add Metrics Collection (#54960)

Signed-off-by: omerpaz95 <omerpaz95@gmail.com>
This commit is contained in:
omerpaz95
2026-09-17 10:36:23 +08:00
committed by GitHub
parent 2307a1d16c
commit fa3622a4a1
12 changed files with 472 additions and 3 deletions
+82
View File
@@ -19,6 +19,7 @@ from vllm.config import (
SpeculativeConfig,
VllmConfig,
)
from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
HiSparseConnector,
HiSparseConnectorScheduler,
@@ -5296,6 +5297,87 @@ def test_scheduler_kv_connector_stats():
assert final_stats == expected_data
def test_scheduler_ec_connector_stats():
"""Test worker-side, scheduler-side, and combined EC connector stats."""
class GenericECConnectorStats(ECConnectorStats):
def reset(self):
self.data = {}
def aggregate(self, other: ECConnectorStats) -> ECConnectorStats:
self.data.update(other.data)
return self
def reduce(self) -> dict[str, int | float]:
return {}
def is_empty(self) -> bool:
return not self.data
test_cases = (
({"worker": 1}, None, {"worker": 1}),
(None, {"scheduler": 2}, {"scheduler": 2}),
({"worker": 1}, {"scheduler": 2}, {"worker": 1, "scheduler": 2}),
)
for worker_data, scheduler_data, expected_data in test_cases:
scheduler = create_scheduler()
worker_stats = (
GenericECConnectorStats(data=worker_data) if worker_data else None
)
scheduler_stats = (
GenericECConnectorStats(data=scheduler_data) if scheduler_data else None
)
scheduler.ec_connector = Mock()
scheduler.ec_connector.take_unavailable_requests.return_value = set()
scheduler.ec_connector.get_ec_connector_stats.return_value = (
scheduler_stats if worker_stats is None else None
)
def update_connector_output(
ec_connector_output: ECConnectorOutput,
scheduler=scheduler,
scheduler_stats=scheduler_stats,
):
scheduler.ec_connector.get_ec_connector_stats.return_value = scheduler_stats
scheduler.ec_connector.update_connector_output.side_effect = (
update_connector_output
)
model_output = ModelRunnerOutput(
req_ids=["req_0"],
req_id_to_index={"req_0": 0},
sampled_token_ids=[[123]],
logprobs=None,
prompt_logprobs_dict={},
pooler_output=[None],
ec_connector_output=ECConnectorOutput(ec_connector_stats=worker_stats)
if worker_stats
else None,
)
scheduler_output = SchedulerOutput(
scheduled_new_reqs=[],
scheduled_cached_reqs=None,
num_scheduled_tokens={"req_0": 1},
total_num_scheduled_tokens=1,
scheduled_spec_decode_tokens={},
scheduled_encoder_inputs={},
num_common_prefix_blocks=[0],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
engine_core_outputs = scheduler.update_from_output(
scheduler_output, model_output
)
final_stats = next(
iter(engine_core_outputs.values())
).scheduler_stats.ec_connector_stats
assert final_stats == expected_data
def test_ec_connector_update_connector_output_called():
"""Test that worker-side EC connector output is forwarded to the
EC connector's update_connector_output hook."""
@@ -0,0 +1,85 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the connector-agnostic EC connector metrics plumbing.
Covers only the base infra: no concrete connector type implements these
hooks yet, so every assertion here is about the documented no-op defaults
and the container contracts, mirroring the equivalent KV connector tests.
"""
from unittest.mock import Mock
import pytest
from vllm.config import VllmConfig
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorRole
from vllm.distributed.ec_transfer.ec_connector.example_connector import (
ECExampleConnector,
)
from vllm.distributed.ec_transfer.ec_connector.metrics import (
ECConnectorLogging,
ECConnectorProm,
ECConnectorStats,
)
from vllm.v1.outputs import ECConnectorOutput
pytestmark = pytest.mark.cpu_test
@pytest.fixture
def example_connector(tmp_path):
config = Mock(spec=VllmConfig)
config.ec_transfer_config = Mock()
config.ec_transfer_config.get_from_extra_config = Mock(return_value=str(tmp_path))
config.ec_transfer_config.is_ec_producer = True
config.ec_transfer_config.is_ec_consumer = False
return ECExampleConnector(vllm_config=config, role=ECConnectorRole.SCHEDULER)
def test_base_stats_methods_raise_not_implemented():
"""The base container has no aggregation logic of its own."""
stats = ECConnectorStats()
with pytest.raises(NotImplementedError):
stats.reset()
with pytest.raises(NotImplementedError):
stats.aggregate(stats)
with pytest.raises(NotImplementedError):
stats.reduce()
with pytest.raises(NotImplementedError):
stats.is_empty()
def test_connector_hooks_default_to_none(example_connector):
"""No connector type overrides these yet, so every hook is a no-op."""
assert example_connector.get_ec_connector_stats() is None
assert ECExampleConnector.build_ec_connector_stats() is None
assert (
ECExampleConnector.build_prom_metrics(
vllm_config=Mock(spec=VllmConfig),
metric_types={},
labelnames=[],
per_engine_labelvalues={},
)
is None
)
def test_ec_connector_output_is_empty_reflects_stats_field():
output = ECConnectorOutput()
assert output.is_empty()
output.ec_connector_stats = ECConnectorStats(data={"x": 1})
assert not output.is_empty()
def test_logging_and_prom_are_no_ops_without_ec_transfer_config():
"""A normal run with no EC connector configured must stay silent."""
logging = ECConnectorLogging(ec_transfer_config=None)
logging.log() # Should not raise, nothing to log.
vllm_config = Mock(spec=VllmConfig)
vllm_config.ec_transfer_config = None
prom = ECConnectorProm(vllm_config, labelnames=[], per_engine_labelvalues={})
assert prom.prom_metrics is None
prom.observe({}) # Should not raise.
@@ -5,6 +5,7 @@
import pytest
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorWorkerMetadata
from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats
from vllm.distributed.ec_transfer.ec_connector.utils import ECOutputAggregator
from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator
from vllm.v1.outputs import (
@@ -29,6 +30,16 @@ class FakeWorkerMeta(ECConnectorWorkerMetadata):
return FakeWorkerMeta(self.saves + other.saves)
class FakeConnectorStats(ECConnectorStats):
"""Records merge order, mirroring FakeWorkerMeta above."""
def __init__(self, saves: list[str]):
self.saves = saves
def aggregate(self, other: "FakeConnectorStats") -> "FakeConnectorStats":
return FakeConnectorStats(self.saves + other.saves)
def _worker_output(ec_output: ECConnectorOutput | None) -> ModelRunnerOutput:
return ModelRunnerOutput(
req_ids=[], req_id_to_index={}, ec_connector_output=ec_output
@@ -38,19 +49,23 @@ def _worker_output(ec_output: ECConnectorOutput | None) -> ModelRunnerOutput:
def test_aggregate_folds_every_rank_onto_output_rank():
"""EC work done on any rank reaches the scheduler via output_rank's output.
The middle rank reports no worker metadata: it must neither seed nor clobber
the accumulator.
The middle rank reports no worker metadata or stats: it must neither seed
nor clobber either accumulator.
"""
outputs = [
_worker_output(
ECConnectorOutput(
finished_sending={"mm0"},
ec_connector_stats=FakeConnectorStats(["mm0"]),
ec_connector_worker_meta=FakeWorkerMeta(["mm0"]),
)
),
_worker_output(ECConnectorOutput(finished_recving={"mm1"})),
_worker_output(
ECConnectorOutput(ec_connector_worker_meta=FakeWorkerMeta(["mm2"]))
ECConnectorOutput(
ec_connector_stats=FakeConnectorStats(["mm2"]),
ec_connector_worker_meta=FakeWorkerMeta(["mm2"]),
)
),
]
@@ -59,6 +74,7 @@ def test_aggregate_folds_every_rank_onto_output_rank():
assert result is outputs[2]
assert result.ec_connector_output.finished_sending == {"mm0"}
assert result.ec_connector_output.finished_recving == {"mm1"}
assert result.ec_connector_output.ec_connector_stats.saves == ["mm0", "mm2"]
assert result.ec_connector_output.ec_connector_worker_meta.saves == ["mm0", "mm2"]
@@ -20,6 +20,7 @@ from vllm.v1.worker.gpu.ec_connector import NO_OP_EC_CONNECTOR, ActiveECConnecto
pytestmark = pytest.mark.cpu_test
WORKER_META = object()
CONNECTOR_STATS = object()
def _scheduler_output() -> SimpleNamespace:
@@ -38,6 +39,7 @@ def _connector(
fake.is_consumer = is_consumer
fake.get_finished.return_value = (None, None)
fake.build_connector_worker_meta.return_value = WORKER_META
fake.get_ec_connector_stats.return_value = CONNECTOR_STATS
with patch("vllm.v1.worker.gpu.ec_connector.get_ec_transfer", return_value=fake):
return ActiveECConnector(SimpleNamespace(), encoder_cache or {}), fake
@@ -72,6 +74,16 @@ def test_worker_meta_is_reported_on_context_exit():
assert fake.clear_connector_metadata.called
def test_connector_stats_are_reported_on_context_exit():
"""Reported in the finally block, alongside the worker meta."""
connector, fake = _connector()
with connector.maybe_get_output(_scheduler_output()) as output:
assert output.ec_connector_stats is None
assert output.ec_connector_stats is CONNECTOR_STATS
def test_v1_producer_receives_existing_encoder_cache():
encoder_cache = {"mm_cached": None}
fake = MagicMock(spec=ECConnectorBase)
@@ -35,6 +35,12 @@ from vllm.v1.outputs import ECConnectorOutput
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.distributed.ec_transfer.ec_connector.metrics import (
ECConnectorPromMetrics,
ECConnectorStats,
PromMetric,
PromMetricT,
)
from vllm.v1.request import Request
logger = init_logger(__name__)
@@ -219,6 +225,17 @@ class ECConnectorBase(ABC):
"""
return None
def get_ec_connector_stats(self) -> "ECConnectorStats | None":
"""
Get the EC connector stats collected during the last interval.
Callable on either a worker-side or scheduler-side connector
instance: the worker-side instance reports stats gathered during
cache save/load, while the scheduler-side instance reports stats
gathered during scheduling decisions.
"""
return None
# ==============================
# Scheduler-side methods
# ==============================
@@ -336,3 +353,29 @@ class ECConnectorBase(ABC):
leave this as False.
"""
return False
@classmethod
def build_ec_connector_stats(
cls, data: dict[str, Any] | None = None
) -> "ECConnectorStats | None":
"""
ECConnectorStats resolution method. This method allows dynamically
registered connectors to return their own ECConnectorStats object,
which can implement custom aggregation logic on the data dict.
"""
return None
@classmethod
def build_prom_metrics(
cls,
vllm_config: "VllmConfig",
metric_types: dict[type["PromMetric"], type["PromMetricT"]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
) -> "ECConnectorPromMetrics | None":
"""
Create an ECConnectorPromMetrics subclass which should register
per-connector Prometheus metrics and implement observe() to
expose connector transfer stats via Prometheus.
"""
return None
@@ -0,0 +1,175 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass, field
from typing import Any, TypeAlias, TypeVar
from prometheus_client import Counter, Gauge, Histogram
from vllm.config import ECTransferConfig, VllmConfig
from vllm.distributed.ec_transfer.ec_connector.factory import ECConnectorFactory
from vllm.logger import init_logger
PromMetric: TypeAlias = Gauge | Counter | Histogram
PromMetricT = TypeVar("PromMetricT", bound=PromMetric)
logger = init_logger(__name__)
@dataclass
class ECConnectorStats:
"""
Base class for EC Connector Stats, a container for transfer performance
metrics or otherwise important telemetry from the connector.
All sub-classes need to be serializable as stats are sent from worker to
logger process.
"""
data: dict[str, Any] = field(default_factory=dict)
def reset(self):
"""Reset the stats, clear the state."""
raise NotImplementedError
def aggregate(self, other: "ECConnectorStats") -> "ECConnectorStats":
"""
Aggregate stats with another `ECConnectorStats` object.
"""
raise NotImplementedError
def reduce(self) -> dict[str, int | float]:
"""
Reduce the observations collected during a time interval to one or
more representative values (eg avg/median/sum of the series).
This is meant to be called by the logger to produce a summary of the
stats for the last time interval.
"""
raise NotImplementedError
def is_empty(self) -> bool:
"""Return True if the stats are empty."""
raise NotImplementedError
class ECConnectorLogging:
def __init__(self, ec_transfer_config: ECTransferConfig | None):
# Instantiate the connector's stats class.
if ec_transfer_config and ec_transfer_config.ec_connector:
self.connector_cls = ECConnectorFactory.get_connector_class(
ec_transfer_config
)
self.reset()
def reset(self):
self.transfer_stats_accumulator: ECConnectorStats | None = None
def observe(self, transfer_stats_data: dict[str, Any]):
# Should not be called when an ECConnector is not configured.
assert self.connector_cls is not None
# Called periodically when connector syncs with the scheduler.
# Note that this is not the same as the logging interval.
# We expect transfer_stats_data to be aggregated across all workers and
# consist of observations from a single connector.
transfer_stats = self.connector_cls.build_ec_connector_stats(
transfer_stats_data
)
if transfer_stats is None:
logger.warning_once(
"The connector %s is collecting stats but "
"does not implement the "
"`build_ec_connector_stats` method. "
"Stats will not be logged.",
self.connector_cls,
)
return
if self.transfer_stats_accumulator is None:
self.transfer_stats_accumulator = transfer_stats
else:
# Accumulate last interval stats.
self.transfer_stats_accumulator = self.transfer_stats_accumulator.aggregate(
transfer_stats
)
def log(self, log_fn=logger.info):
"""Log transfer metrics periodically, similar to throughput logging"""
if (
self.transfer_stats_accumulator
and not self.transfer_stats_accumulator.is_empty()
):
# Produce a single cumulative stats object for the last time
# interval from the recorded observations.
xfer_metrics = self.transfer_stats_accumulator.reduce()
xfer_metrics_str = ", ".join(f"{k}={v}" for k, v in xfer_metrics.items())
log_fn("EC Transfer metrics: %s", xfer_metrics_str)
# Reset metrics for next interval
self.reset()
class ECConnectorPromMetrics:
"""
A base class for per-connector Prometheus metric registration
and recording.
"""
def __init__(
self,
vllm_config: VllmConfig,
metric_types: dict[type[PromMetric], type[PromMetricT]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
):
self._ec_transfer_config = vllm_config.ec_transfer_config
self._gauge_cls = metric_types[Gauge]
self._counter_cls = metric_types[Counter]
self._histogram_cls = metric_types[Histogram]
self._labelnames = labelnames
self.per_engine_labelvalues = per_engine_labelvalues
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
"""
Record the supplied transfer statistics to Prometheus metrics. These
statistics are engine-specific, and should be recorded to a metric
with the appropriate 'engine' label. These metric instances can be
created using the create_metric_per_engine() helper method.
"""
raise NotImplementedError
class ECConnectorProm:
"""
Support for registering per-connector Prometheus metrics, and
recording transfer statistics to those metrics. Uses
ECConnectorBase.build_prom_metrics().
"""
_gauge_cls = Gauge
_counter_cls = Counter
_histogram_cls = Histogram
def __init__(
self,
vllm_config: VllmConfig,
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
):
self.prom_metrics: ECConnectorPromMetrics | None = None
ec_transfer_config = vllm_config.ec_transfer_config
if ec_transfer_config and ec_transfer_config.ec_connector:
connector_cls = ECConnectorFactory.get_connector_class(ec_transfer_config)
metric_types = {
Gauge: self._gauge_cls,
Counter: self._counter_cls,
Histogram: self._histogram_cls,
}
self.prom_metrics = connector_cls.build_prom_metrics(
vllm_config,
metric_types,
labelnames,
per_engine_labelvalues,
)
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
if self.prom_metrics is None:
return
self.prom_metrics.observe(transfer_stats_data, engine_idx)
@@ -103,6 +103,7 @@ class ECOutputAggregator:
finished_sending = set[str]()
finished_recving = set[str]()
worker_meta = None
connector_stats = None
for model_runner_output in outputs:
assert model_runner_output is not None
ec_output = model_runner_output.ec_connector_output
@@ -117,9 +118,17 @@ class ECOutputAggregator:
meta if worker_meta is None else worker_meta.aggregate(meta)
)
if stats := ec_output.ec_connector_stats:
connector_stats = (
stats
if connector_stats is None
else connector_stats.aggregate(stats)
)
aggregated = ECConnectorOutput(
finished_sending=finished_sending or None,
finished_recving=finished_recving or None,
ec_connector_stats=connector_stats,
ec_connector_worker_meta=worker_meta,
)
if aggregated.is_empty():
+24
View File
@@ -15,6 +15,7 @@ from vllm.distributed.ec_transfer.ec_connector.base import (
ECConnectorRole,
)
from vllm.distributed.ec_transfer.ec_connector.factory import ECConnectorFactory
from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats
from vllm.distributed.kv_events import EventPublisherFactory, KVEventBatch
from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory
from vllm.distributed.kv_transfer.kv_connector.v1 import (
@@ -2243,6 +2244,23 @@ class Scheduler(SchedulerInterface):
else scheduler_kv_connector_stats
)
# Worker-side EC connector stats from the model runner output.
ec_connector_stats: ECConnectorStats | None = (
ec_connector_output.ec_connector_stats if ec_connector_output else None
)
if self.ec_connector:
# Scheduler-side EC connector stats collected after connector update.
scheduler_ec_connector_stats = self.ec_connector.get_ec_connector_stats()
if (
scheduler_ec_connector_stats is not None
and not scheduler_ec_connector_stats.is_empty()
):
ec_connector_stats = (
ec_connector_stats.aggregate(scheduler_ec_connector_stats)
if ec_connector_stats is not None
else scheduler_ec_connector_stats
)
# collect KV cache events from KV cache manager
events = self.kv_cache_manager.take_events()
@@ -2287,6 +2305,7 @@ class Scheduler(SchedulerInterface):
kv_connector_stats,
cudagraph_stats,
perf_stats,
ec_connector_stats=ec_connector_stats,
)
) is not None:
# Return stats to only one of the front-ends.
@@ -2772,6 +2791,7 @@ class Scheduler(SchedulerInterface):
kv_connector_stats: KVConnectorStats | None = None,
cudagraph_stats: CUDAGraphStat | None = None,
perf_stats: PerfStats | None = None,
ec_connector_stats: ECConnectorStats | None = None,
) -> SchedulerStats | None:
if not self.log_stats:
return None
@@ -2790,6 +2810,9 @@ class Scheduler(SchedulerInterface):
connector_stats_payload = (
kv_connector_stats.to_dict() if kv_connector_stats else None
)
ec_connector_stats_payload = (
ec_connector_stats.data if ec_connector_stats else None
)
return SchedulerStats(
num_running_reqs=len(self.running),
num_waiting_reqs=len(self.waiting),
@@ -2802,6 +2825,7 @@ class Scheduler(SchedulerInterface):
kv_connector_stats=connector_stats_payload,
cudagraph_stats=cudagraph_stats,
perf_stats=perf_stats,
ec_connector_stats=ec_connector_stats_payload,
)
def make_spec_decoding_stats(
+18
View File
@@ -11,6 +11,10 @@ from prometheus_client import Counter, Gauge, Histogram
import vllm.envs as envs
from vllm.compilation.cuda_graph import CUDAGraphLogging
from vllm.config import SupportsMetricsInfo, VllmConfig
from vllm.distributed.ec_transfer.ec_connector.metrics import (
ECConnectorLogging,
ECConnectorProm,
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorLogging,
KVConnectorProm,
@@ -118,6 +122,8 @@ class LoggingStatLogger(StatLoggerBase):
self.spec_decoding_logging = SpecDecodingLogging(is_diffusion=is_diffusion)
kv_transfer_config = self.vllm_config.kv_transfer_config
self.kv_connector_logging = KVConnectorLogging(kv_transfer_config)
ec_transfer_config = self.vllm_config.ec_transfer_config
self.ec_connector_logging = ECConnectorLogging(ec_transfer_config)
self.cudagraph_logging = None
if self.vllm_config.observability_config.cudagraph_metrics:
self.cudagraph_logging = CUDAGraphLogging(
@@ -224,6 +230,8 @@ class LoggingStatLogger(StatLoggerBase):
self.spec_decoding_logging.observe(scheduler_stats.spec_decoding_stats)
if kv_connector_stats := scheduler_stats.kv_connector_stats:
self.kv_connector_logging.observe(kv_connector_stats)
if ec_connector_stats := scheduler_stats.ec_connector_stats:
self.ec_connector_logging.observe(ec_connector_stats)
if (
self.cudagraph_logging is not None
and scheduler_stats.cudagraph_stats is not None
@@ -319,6 +327,7 @@ class LoggingStatLogger(StatLoggerBase):
self.spec_decoding_logging.log(log_fn=log_fn)
self.kv_connector_logging.log(log_fn=log_fn)
self.ec_connector_logging.log(log_fn=log_fn)
if self.cudagraph_logging is not None:
self.cudagraph_logging.log(log_fn=log_fn)
if self._enable_perf_stats():
@@ -451,6 +460,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
_histogram_cls = Histogram
_spec_decoding_cls = SpecDecodingProm
_kv_connector_cls = KVConnectorProm
_ec_connector_cls = ECConnectorProm
_perf_metrics_cls = PerfMetricsProm
def __init__(
@@ -488,6 +498,9 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
self.kv_connector_prom = self._kv_connector_cls(
vllm_config, labelnames, per_engine_labelvalues
)
self.ec_connector_prom = self._ec_connector_cls(
vllm_config, labelnames, per_engine_labelvalues
)
self.perf_metrics_prom = self._perf_metrics_cls(
vllm_config, labelnames, per_engine_labelvalues
)
@@ -1064,6 +1077,11 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
scheduler_stats.kv_connector_stats, engine_idx
)
if scheduler_stats.ec_connector_stats is not None:
self.ec_connector_prom.observe(
scheduler_stats.ec_connector_stats, engine_idx
)
if scheduler_stats.perf_stats is not None:
self.perf_metrics_prom.observe(scheduler_stats.perf_stats, engine_idx)
+1
View File
@@ -205,6 +205,7 @@ class SchedulerStats:
spec_decoding_stats: SpecDecodingStats | None = None
kv_connector_stats: dict[str, Any] | None = None
ec_connector_stats: dict[str, Any] | None = None
waiting_lora_adapters: dict[str, int] = field(default_factory=dict)
running_lora_adapters: dict[str, int] = field(default_factory=dict)
+3
View File
@@ -16,6 +16,7 @@ from vllm.v1.core.sched.output import SchedulerOutput
if TYPE_CHECKING:
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorWorkerMetadata
from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats
from vllm.distributed.kv_events import KVConnectorKVEvents
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorWorkerMetadata,
@@ -307,12 +308,14 @@ class ECConnectorOutput:
# [mm_hash]
finished_sending: set[str] | None = None
finished_recving: set[str] | None = None
ec_connector_stats: "ECConnectorStats | None" = None
ec_connector_worker_meta: ECConnectorWorkerMetadata | None = None
def is_empty(self):
return (
not self.finished_sending
and not self.finished_recving
and not self.ec_connector_stats
and not self.ec_connector_worker_meta
)
+1
View File
@@ -80,6 +80,7 @@ class ActiveECConnector(ECConnector):
output.finished_sending, output.finished_recving = (
ec_connector.get_finished(scheduler_output.finished_req_ids)
)
output.ec_connector_stats = ec_connector.get_ec_connector_stats()
output.ec_connector_worker_meta = ec_connector.build_connector_worker_meta()
ec_connector.clear_connector_metadata()