Feature/offloading manager stats (#35669)

Signed-off-by: Sriusa4414@gmail.com
Signed-off-by: srinivas_oo7 <Sriusa4414@gmail.com>
Signed-off-by: srinivas_oo7 <sklinkedin0120@gmail.com>
Signed-off-by: Srinivasoo7 <158864704+Srinivasoo7@users.noreply.github.com>
Signed-off-by: Or Ozeri <oro@il.ibm.com>
Co-authored-by: srinivas_oo7 <sklinkedin0120@gmail.com>
Co-authored-by: Srinivasoo7 <158864704+Srinivasoo7@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Srinivas Krovvidi
2026-06-10 12:44:55 +00:00
committed by GitHub
co-authored by srinivas_oo7 Srinivasoo7 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Or Ozeri
parent 9ad08c4d15
commit 9dfc313bdc
14 changed files with 1200 additions and 213 deletions
@@ -11,6 +11,45 @@ from vllm.v1.kv_offload.worker.worker import TransferSpec
ReqId = str
@dataclass(slots=True)
class DirectionalTransferStats:
bytes: int = 0
time: float = 0.0
sizes: list[int | float] = field(default_factory=list)
def aggregate(
self, other: "DirectionalTransferStats"
) -> "DirectionalTransferStats":
return DirectionalTransferStats(
bytes=self.bytes + other.bytes,
time=self.time + other.time,
sizes=[*self.sizes, *other.sizes],
)
def record(self, num_bytes: int, time: float) -> None:
self.bytes += num_bytes
self.time += time
self.sizes.append(num_bytes)
def is_empty(self) -> bool:
return self.bytes == 0 and self.time == 0.0 and not self.sizes
@dataclass(slots=True)
class TransferStats:
load: DirectionalTransferStats = field(default_factory=DirectionalTransferStats)
store: DirectionalTransferStats = field(default_factory=DirectionalTransferStats)
def aggregate(self, other: "TransferStats") -> "TransferStats":
return TransferStats(
load=self.load.aggregate(other.load),
store=self.store.aggregate(other.store),
)
def is_empty(self) -> bool:
return self.load.is_empty() and self.store.is_empty()
@dataclass
class TransferJob:
"""A transfer job bundling request context with transfer spec.
@@ -43,6 +82,7 @@ class OffloadingWorkerMetadata(KVConnectorWorkerMetadata):
"""
completed_jobs: dict[int, int] = field(default_factory=dict)
transfer_stats: TransferStats = field(default_factory=TransferStats)
def mark_completed(self, job_id: int) -> None:
"""Record a transfer job completion from this worker."""
@@ -57,4 +97,7 @@ class OffloadingWorkerMetadata(KVConnectorWorkerMetadata):
for job_id, v in other.completed_jobs.items():
merged[job_id] = merged.get(job_id, 0) + v
return OffloadingWorkerMetadata(completed_jobs=merged)
return OffloadingWorkerMetadata(
completed_jobs=merged,
transfer_stats=self.transfer_stats.aggregate(other.transfer_stats),
)
@@ -10,37 +10,176 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
PromMetric,
PromMetricT,
)
from vllm.logger import init_logger
from vllm.v1.kv_offload.worker.worker import TransferType
logger = init_logger(__name__)
from vllm.v1.kv_offload.base import (
OffloadingCounterMetadata,
OffloadingGaugeMetadata,
OffloadingHistogramMetadata,
OffloadingMetricMetadata,
)
from vllm.v1.kv_offload.factory import OffloadingSpecFactory
@dataclass
class OffloadingOperationMetrics:
op_size: int
op_time: float
class _TransferMetricName:
"""Flat metric names for GPU↔offload-medium transfer operations."""
LOAD_BYTES = "vllm:kv_offload_load_bytes"
LOAD_TIME = "vllm:kv_offload_load_time"
LOAD_SIZE = "vllm:kv_offload_load_size"
STORE_BYTES = "vllm:kv_offload_store_bytes"
STORE_TIME = "vllm:kv_offload_store_time"
STORE_SIZE = "vllm:kv_offload_store_size"
class _TransferType:
"""Transfer direction labels for deprecated CPU offload metrics."""
LOAD = "CPU_to_GPU"
STORE = "GPU_to_CPU"
ALL = (LOAD, STORE)
TRANSFER_SIZE_BUCKETS = (
1e6,
5e6,
10e6,
20e6,
40e6,
60e6,
80e6,
100e6,
150e6,
200e6,
)
def get_connector_metric_definitions() -> dict[str, OffloadingMetricMetadata]:
return {
_TransferMetricName.LOAD_BYTES: OffloadingCounterMetadata(
documentation="Total bytes loaded from offload storage to GPU.",
),
_TransferMetricName.LOAD_TIME: OffloadingCounterMetadata(
documentation="Total load time from offload storage to GPU, in seconds.",
),
_TransferMetricName.LOAD_SIZE: OffloadingHistogramMetadata(
documentation="Histogram of KV offload load operation size, in bytes.",
buckets=TRANSFER_SIZE_BUCKETS,
),
_TransferMetricName.STORE_BYTES: OffloadingCounterMetadata(
documentation="Total bytes stored from GPU to offload storage.",
),
_TransferMetricName.STORE_TIME: OffloadingCounterMetadata(
documentation="Total store time from GPU to offload storage, in seconds.",
),
_TransferMetricName.STORE_SIZE: OffloadingHistogramMetadata(
documentation="Histogram of KV offload store operation size, in bytes.",
buckets=TRANSFER_SIZE_BUCKETS,
),
}
_DEPRECATED_TOTAL_BYTES = "vllm:kv_offload_total_bytes"
_DEPRECATED_TOTAL_TIME = "vllm:kv_offload_total_time"
_DEPRECATED_SIZE = "vllm:kv_offload_size"
# Deprecated legacy transfer metrics, kept during the migration to the flat
# metric names above. These stay in a separate definition block because they
# use a transfer_type label, but are emitted from the same flat stats payload
# for compatibility.
_DEPRECATED_CONNECTOR_METRIC_DEFINITIONS: dict[str, OffloadingMetricMetadata] = {
_DEPRECATED_TOTAL_BYTES: OffloadingCounterMetadata(
documentation="Number of bytes offloaded by KV connector",
),
_DEPRECATED_TOTAL_TIME: OffloadingCounterMetadata(
documentation="Total time measured by all KV offloading operations",
),
_DEPRECATED_SIZE: OffloadingHistogramMetadata(
documentation="Histogram of KV offload transfer size, in bytes.",
buckets=TRANSFER_SIZE_BUCKETS,
),
}
class _MetricType:
"""Type tags embedded in the serialized stats payload."""
COUNTER = "counter"
GAUGE = "gauge"
HISTOGRAM = "histogram"
class _StatsKey:
"""Top-level keys in the serialized stats dict."""
# Maps metric name -> _MetricType value
TYPES = "types"
# Maps metric name -> observed value (number or list)
DATA = "data"
@dataclass
class OffloadingConnectorStats(KVConnectorStats):
"""
Offloading connector stats use flat metric names as keys.
The ``data`` dict is structured using ``_StatsKey`` / ``_MetricType``::
{
_StatsKey.TYPES: {name: _MetricType.*, ...},
_StatsKey.DATA: {name: value, ...},
}
This structure is self-describing: it survives IPC serialization
without needing the full ``OffloadingMetricMetadata`` objects on the
receiving side.
Counter values are aggregated by summing, gauge values use the latest
snapshot, and histogram values are lists of observed samples.
"""
def __post_init__(self):
if not self.data:
# Empty container init, no data is passed in.
if _StatsKey.DATA not in self.data:
self.reset()
def reset(self):
self.data: dict[str, list[OffloadingOperationMetrics]] = {}
self.data: dict[str, Any] = {
_StatsKey.TYPES: {},
_StatsKey.DATA: {},
}
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
if not other.is_empty():
for k, v in other.data.items():
if k not in self.data:
self.data[k] = v
@property
def _types(self) -> dict[str, str]:
return self.data[_StatsKey.TYPES]
@property
def _values(self) -> dict[str, Any]:
return self.data[_StatsKey.DATA]
def aggregate(self, other: "KVConnectorStats") -> "KVConnectorStats":
if other.is_empty():
return self
assert isinstance(other, OffloadingConnectorStats)
other_types = other._types
other_values = other._values
for key, value in other_values.items():
type_str = other_types.get(key)
if type_str is None:
raise AssertionError(f"Unknown offloading stats key: {key}")
self._types.setdefault(key, type_str)
if type_str == _MetricType.HISTOGRAM:
assert isinstance(value, list)
if key not in self._values:
self._values[key] = value
else:
accumulator = self.data[k]
assert isinstance(accumulator, list)
accumulator.extend(v)
assert isinstance(self._values[key], list)
self._values[key].extend(value)
elif type_str == _MetricType.COUNTER:
assert isinstance(value, int | float)
self._values[key] = self._values.get(key, 0) + value
elif type_str == _MetricType.GAUGE:
assert isinstance(value, int | float)
self._values[key] = value
else:
raise AssertionError(f"Unknown metric type '{type_str}' for key: {key}")
return self
def reduce(self) -> dict[str, int | float]:
@@ -51,29 +190,44 @@ class OffloadingConnectorStats(KVConnectorStats):
stats for the last time interval.
"""
return_dict: dict[str, int | float] = {}
for transfer_type, ops_list in self.data.items():
assert isinstance(ops_list, list)
total_bytes = 0
total_time = 0.0
for op in ops_list:
assert isinstance(op, dict)
total_bytes += op["op_size"]
total_time += op["op_time"]
return_dict[f"{transfer_type}_total_bytes"] = total_bytes
return_dict[f"{transfer_type}_total_time"] = total_time
for key, value in self._values.items():
type_str = self._types.get(key)
if type_str is None:
raise AssertionError(f"Unknown offloading stats key: {key}")
if type_str == _MetricType.HISTOGRAM:
assert isinstance(value, list)
return_dict[f"{key}_count"] = len(value)
return_dict[f"{key}_sum"] = sum(value)
elif type_str in (_MetricType.COUNTER, _MetricType.GAUGE):
assert isinstance(value, int | float)
return_dict[key] = value
else:
raise AssertionError(f"Unknown metric type '{type_str}' for key: {key}")
return return_dict
def is_empty(self) -> bool:
return not self.data
return not self.data.get(_StatsKey.DATA)
def record_transfer(self, num_bytes: int, time: float, transfer_type: TransferType):
src, dst = transfer_type
transfer_type_key = src + "_to_" + dst
op = OffloadingOperationMetrics(num_bytes, time)
if transfer_type_key in self.data:
self.data[transfer_type_key].append(op)
else:
self.data[transfer_type_key] = [op]
def increase_counter(
self, counter_name: str, counter_increase_value: int | float
) -> None:
"""Increase a counter on the stats payload."""
self._types.setdefault(counter_name, _MetricType.COUNTER)
self._values[counter_name] = (
self._values.get(counter_name, 0) + counter_increase_value
)
def set_gauge(self, gauge_name: str, gauge_value: int | float) -> None:
"""Set a gauge snapshot on the stats payload."""
self._types.setdefault(gauge_name, _MetricType.GAUGE)
self._values[gauge_name] = gauge_value
def observe_histogram(
self, histogram_name: str, histogram_value: int | float
) -> None:
"""Record a histogram observation on the stats payload."""
self._types.setdefault(histogram_name, _MetricType.HISTOGRAM)
self._values.setdefault(histogram_name, []).append(histogram_value)
class OffloadPromMetrics(KVConnectorPromMetrics):
@@ -89,77 +243,145 @@ class OffloadPromMetrics(KVConnectorPromMetrics):
self.histogram_transfer_size: dict[tuple[int, str], PromMetricT] = {}
self.counter_kv_bytes: dict[tuple[int, str], PromMetricT] = {}
self.counter_kv_transfer_time: dict[tuple[int, str], PromMetricT] = {}
buckets = [ # In bytes
1e6,
5e6,
10e6,
20e6,
40e6,
60e6,
80e6,
100e6,
150e6,
200e6,
]
spec_cls = OffloadingSpecFactory.get_spec_cls(vllm_config)
kv_transfer_config = vllm_config.kv_transfer_config
assert kv_transfer_config is not None
extra_config = kv_transfer_config.kv_connector_extra_config
self._offloading_metric_metadata: dict[str, OffloadingMetricMetadata] = {
**spec_cls.build_metric_definitions(extra_config),
**get_connector_metric_definitions(),
}
from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec
self._observe_deprecated_metrics = issubclass(spec_cls, CPUOffloadingSpec)
self._offloading_metric_defs: dict[str, PromMetricT] = {}
self.offloading_metrics: dict[tuple[int, str], PromMetricT] = {}
self._counter_kv_bytes = self._counter_cls(
name="vllm:kv_offload_total_bytes",
documentation="Number of bytes offloaded by KV connector",
name=_DEPRECATED_TOTAL_BYTES,
documentation=_DEPRECATED_CONNECTOR_METRIC_DEFINITIONS[
_DEPRECATED_TOTAL_BYTES
].documentation,
labelnames=labelnames + ["transfer_type"],
)
self._counter_kv_transfer_time = self._counter_cls(
name="vllm:kv_offload_total_time",
documentation="Total time measured by all KV offloading operations",
name=_DEPRECATED_TOTAL_TIME,
documentation=_DEPRECATED_CONNECTOR_METRIC_DEFINITIONS[
_DEPRECATED_TOTAL_TIME
].documentation,
labelnames=labelnames + ["transfer_type"],
)
deprecated_size_metadata = _DEPRECATED_CONNECTOR_METRIC_DEFINITIONS[
_DEPRECATED_SIZE
]
assert isinstance(deprecated_size_metadata, OffloadingHistogramMetadata)
self._histogram_transfer_size = self._histogram_cls(
name="vllm:kv_offload_size",
documentation="Histogram of KV offload transfer size, in bytes.",
buckets=buckets[:],
name=_DEPRECATED_SIZE,
documentation=deprecated_size_metadata.documentation,
buckets=deprecated_size_metadata.buckets,
labelnames=labelnames + ["transfer_type"],
)
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
"""
Observe transfer statistics from the new data structure.
transfer_stats_data is expected to be a dict where:
- keys are transfer type strings (e.g., "cpu_to_gpu", "gpu_to_cpu")
- values are lists of OffloadingOperationMetrics objects
"""
for transfer_type, ops in transfer_stats_data.items():
# Cache:
if (engine_idx, transfer_type) not in self.histogram_transfer_size:
for engine_idx, labelvalues in per_engine_labelvalues.items():
for transfer_type in _TransferType.ALL:
bounded_labelvalues = labelvalues + [transfer_type]
self.histogram_transfer_size[(engine_idx, transfer_type)] = (
self._histogram_transfer_size.labels(
*(self.per_engine_labelvalues[engine_idx] + [transfer_type])
)
self._histogram_transfer_size.labels(*bounded_labelvalues)
)
self.counter_kv_bytes[(engine_idx, transfer_type)] = (
self._counter_kv_bytes.labels(
*(self.per_engine_labelvalues[engine_idx] + [transfer_type])
)
self._counter_kv_bytes.labels(*bounded_labelvalues)
)
self.counter_kv_transfer_time[(engine_idx, transfer_type)] = (
self._counter_kv_transfer_time.labels(
*(self.per_engine_labelvalues[engine_idx] + [transfer_type])
)
self._counter_kv_transfer_time.labels(*bounded_labelvalues)
)
# Process ops:
assert isinstance(ops, list)
for op in ops: # ops is a list of serialized OffloadingOperationMetrics
assert isinstance(op, dict)
# Observe size histogram
self.histogram_transfer_size[(engine_idx, transfer_type)].observe(
op["op_size"]
for metric_name, metadata in self._offloading_metric_metadata.items():
self._offloading_metric_defs[metric_name] = self._create_metric(
metric_name, metadata
)
for engine_idx, labelvalues in per_engine_labelvalues.items():
self.offloading_metrics[(engine_idx, metric_name)] = (
self._offloading_metric_defs[metric_name].labels(*labelvalues)
)
# Increment byte and time counters
self.counter_kv_bytes[(engine_idx, transfer_type)].inc(op["op_size"])
def _create_metric(
self, metric_name: str, metadata: OffloadingMetricMetadata
) -> Any:
kwargs: dict[str, Any] = {
"name": metric_name,
"documentation": metadata.documentation,
"labelnames": self._labelnames,
}
if isinstance(metadata, OffloadingCounterMetadata):
metric_cls = self._counter_cls
elif isinstance(metadata, OffloadingGaugeMetadata):
metric_cls = self._gauge_cls
elif isinstance(metadata, OffloadingHistogramMetadata):
metric_cls = self._histogram_cls
if metadata.buckets is not None:
kwargs["buckets"] = metadata.buckets
else:
raise AssertionError(f"Unknown offloading metric metadata: {metadata}")
return metric_cls(**kwargs)
self.counter_kv_transfer_time[(engine_idx, transfer_type)].inc(
op["op_time"]
def _increase_counter(
self, metric_name: str, value: int | float, engine_idx: int
) -> None:
self.offloading_metrics[(engine_idx, metric_name)].inc(value)
if not self._observe_deprecated_metrics:
return
# Keep deprecated CPU offload transfer metrics updated during the
# transition to flat metric names.
if metric_name == _TransferMetricName.LOAD_BYTES:
self.counter_kv_bytes[(engine_idx, _TransferType.LOAD)].inc(value)
elif metric_name == _TransferMetricName.LOAD_TIME:
self.counter_kv_transfer_time[(engine_idx, _TransferType.LOAD)].inc(value)
elif metric_name == _TransferMetricName.STORE_BYTES:
self.counter_kv_bytes[(engine_idx, _TransferType.STORE)].inc(value)
elif metric_name == _TransferMetricName.STORE_TIME:
self.counter_kv_transfer_time[(engine_idx, _TransferType.STORE)].inc(value)
def _set_gauge(self, metric_name: str, value: int | float, engine_idx: int) -> None:
self.offloading_metrics[(engine_idx, metric_name)].set(value)
def _observe_histogram(
self, metric_name: str, value: list[int | float], engine_idx: int
) -> None:
for observation in value:
self.offloading_metrics[(engine_idx, metric_name)].observe(observation)
if not self._observe_deprecated_metrics:
continue
# Keep deprecated CPU offload transfer metrics updated during the
# transition to flat metric names.
if metric_name == _TransferMetricName.LOAD_SIZE:
self.histogram_transfer_size[(engine_idx, _TransferType.LOAD)].observe(
observation
)
elif metric_name == _TransferMetricName.STORE_SIZE:
self.histogram_transfer_size[(engine_idx, _TransferType.STORE)].observe(
observation
)
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
"""Observe transfer statistics."""
metric_types = transfer_stats_data.get(_StatsKey.TYPES, {})
metric_data = transfer_stats_data.get(_StatsKey.DATA, {})
for key, value in metric_data.items():
type_str = metric_types.get(key)
if type_str is None:
raise AssertionError(f"Unknown offloading stats key: {key}")
assert key in self._offloading_metric_defs
if type_str == _MetricType.COUNTER:
assert isinstance(value, int | float)
self._increase_counter(key, value, engine_idx)
elif type_str == _MetricType.GAUGE:
assert isinstance(value, int | float)
self._set_gauge(key, value, engine_idx)
elif type_str == _MetricType.HISTOGRAM:
assert isinstance(value, list)
assert all(isinstance(v, int | float) for v in value)
self._observe_histogram(key, value, engine_idx)
else:
raise AssertionError(f"Unknown metric type '{type_str}' for key: {key}")
@@ -14,6 +14,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
ReqId,
TransferJob,
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
OffloadingConnectorStats,
_TransferMetricName,
)
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
@@ -259,9 +263,13 @@ def _create_req_context(req: Request) -> ReqContext:
class OffloadingConnectorScheduler:
"""Implementation of Scheduler side methods"""
def __init__(self, spec: OffloadingSpec):
def __init__(
self,
spec: OffloadingSpec,
):
self.config = SchedulerOffloadConfig.from_spec(spec)
self.manager: OffloadingManager = spec.get_manager()
self._connector_stats: OffloadingConnectorStats | None = None
full_attention_groups: list[int] = []
sliding_window_groups: list[int] = []
@@ -952,6 +960,39 @@ class OffloadingConnectorScheduler:
if not isinstance(meta, OffloadingWorkerMetadata):
assert meta is None
meta = OffloadingWorkerMetadata()
if not meta.transfer_stats.is_empty():
transfer_stats = OffloadingConnectorStats()
if not meta.transfer_stats.load.is_empty():
transfer_stats.increase_counter(
_TransferMetricName.LOAD_BYTES,
meta.transfer_stats.load.bytes,
)
transfer_stats.increase_counter(
_TransferMetricName.LOAD_TIME,
meta.transfer_stats.load.time,
)
for size in meta.transfer_stats.load.sizes:
transfer_stats.observe_histogram(
_TransferMetricName.LOAD_SIZE, size
)
if not meta.transfer_stats.store.is_empty():
transfer_stats.increase_counter(
_TransferMetricName.STORE_BYTES,
meta.transfer_stats.store.bytes,
)
transfer_stats.increase_counter(
_TransferMetricName.STORE_TIME,
meta.transfer_stats.store.time,
)
for size in meta.transfer_stats.store.sizes:
transfer_stats.observe_histogram(
_TransferMetricName.STORE_SIZE, size
)
if self._connector_stats is None:
self._connector_stats = transfer_stats
else:
self._connector_stats.aggregate(transfer_stats)
for job_id, count in meta.completed_jobs.items():
assert count > 0
if job_id < self._stale_job_threshold:
@@ -990,6 +1031,19 @@ class OffloadingConnectorScheduler:
if not req_status.transfer_jobs and req_status.req.is_finished():
del self._req_status[job_status.req_id]
def get_stats(self) -> OffloadingConnectorStats | None:
stats = self._connector_stats
self._connector_stats = None
manager_stats = self.manager.get_stats()
if manager_stats is not None:
if stats is None:
stats = manager_stats
else:
stats.aggregate(manager_stats)
return stats
def request_finished(
self,
request: Request,
@@ -5,17 +5,11 @@ from dataclasses import replace
import torch
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorStats,
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
OffloadingConnectorMetadata,
OffloadingWorkerMetadata,
ReqId,
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
OffloadingConnectorStats,
)
from vllm.logger import init_logger
from vllm.v1.attention.backend import AttentionBackend
from vllm.v1.kv_cache_interface import (
@@ -44,7 +38,6 @@ class OffloadingConnectorWorker:
self.spec = spec
self.worker = OffloadingWorker()
self.kv_connector_stats = OffloadingConnectorStats()
# job_id -> req_id for in-flight loads.
self._load_jobs: dict[int, ReqId] = {}
self._unsubmitted_store_jobs: list[tuple[int, TransferSpec]] = []
@@ -271,15 +264,18 @@ class OffloadingConnectorWorker:
# we currently do not support job failures
job_id = transfer_result.job_id
assert transfer_result.success
is_load = job_id in self._load_jobs
if (
transfer_result.transfer_time
transfer_result.transfer_time is not None
and transfer_result.transfer_size is not None
and transfer_result.transfer_type is not None
):
self.kv_connector_stats.record_transfer(
num_bytes=transfer_result.transfer_size,
time=transfer_result.transfer_time,
transfer_type=transfer_result.transfer_type,
if is_load:
stats = self._connector_worker_meta.transfer_stats.load
else:
stats = self._connector_worker_meta.transfer_stats.store
stats.record(
transfer_result.transfer_size,
transfer_result.transfer_time,
)
self._connector_worker_meta.mark_completed(job_id)
@@ -297,18 +293,6 @@ class OffloadingConnectorWorker:
self._connector_worker_meta = OffloadingWorkerMetadata()
return meta
def get_kv_connector_stats(self) -> KVConnectorStats | None:
"""
Get the KV transfer stats for the connector.
"""
if self.kv_connector_stats.is_empty():
return None
# Clear stats for next iteration
kv_connector_stats = self.kv_connector_stats
self.kv_connector_stats = OffloadingConnectorStats()
return kv_connector_stats
def shutdown(self) -> None:
self._unsubmitted_store_jobs.clear()
self._load_jobs.clear()
@@ -184,9 +184,14 @@ class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
return True
def get_kv_connector_stats(self) -> KVConnectorStats | None:
if self.connector_worker is None:
return None # We only emit stats from the worker-side
return self.connector_worker.get_kv_connector_stats()
if self.connector_scheduler is not None:
return self.connector_scheduler.get_stats()
# TODO(orozery): Remove once PR #43877 lands
if self.connector_worker is not None:
return OffloadingConnectorStats()
return None
@classmethod
def build_kv_connector_stats(