Feature/offloading manager stats (#35669)
Signed-off-by: Sriusa4414@gmail.com Signed-off-by: srinivas_oo7 <Sriusa4414@gmail.com> Signed-off-by: srinivas_oo7 <sklinkedin0120@gmail.com> Signed-off-by: Srinivasoo7 <158864704+Srinivasoo7@users.noreply.github.com> Signed-off-by: Or Ozeri <oro@il.ibm.com> Co-authored-by: srinivas_oo7 <sklinkedin0120@gmail.com> Co-authored-by: Srinivasoo7 <158864704+Srinivasoo7@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
srinivas_oo7
Srinivasoo7
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Or Ozeri
parent
9ad08c4d15
commit
9dfc313bdc
@@ -11,6 +11,45 @@ from vllm.v1.kv_offload.worker.worker import TransferSpec
|
||||
ReqId = str
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class DirectionalTransferStats:
|
||||
bytes: int = 0
|
||||
time: float = 0.0
|
||||
sizes: list[int | float] = field(default_factory=list)
|
||||
|
||||
def aggregate(
|
||||
self, other: "DirectionalTransferStats"
|
||||
) -> "DirectionalTransferStats":
|
||||
return DirectionalTransferStats(
|
||||
bytes=self.bytes + other.bytes,
|
||||
time=self.time + other.time,
|
||||
sizes=[*self.sizes, *other.sizes],
|
||||
)
|
||||
|
||||
def record(self, num_bytes: int, time: float) -> None:
|
||||
self.bytes += num_bytes
|
||||
self.time += time
|
||||
self.sizes.append(num_bytes)
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
return self.bytes == 0 and self.time == 0.0 and not self.sizes
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class TransferStats:
|
||||
load: DirectionalTransferStats = field(default_factory=DirectionalTransferStats)
|
||||
store: DirectionalTransferStats = field(default_factory=DirectionalTransferStats)
|
||||
|
||||
def aggregate(self, other: "TransferStats") -> "TransferStats":
|
||||
return TransferStats(
|
||||
load=self.load.aggregate(other.load),
|
||||
store=self.store.aggregate(other.store),
|
||||
)
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
return self.load.is_empty() and self.store.is_empty()
|
||||
|
||||
|
||||
@dataclass
|
||||
class TransferJob:
|
||||
"""A transfer job bundling request context with transfer spec.
|
||||
@@ -43,6 +82,7 @@ class OffloadingWorkerMetadata(KVConnectorWorkerMetadata):
|
||||
"""
|
||||
|
||||
completed_jobs: dict[int, int] = field(default_factory=dict)
|
||||
transfer_stats: TransferStats = field(default_factory=TransferStats)
|
||||
|
||||
def mark_completed(self, job_id: int) -> None:
|
||||
"""Record a transfer job completion from this worker."""
|
||||
@@ -57,4 +97,7 @@ class OffloadingWorkerMetadata(KVConnectorWorkerMetadata):
|
||||
for job_id, v in other.completed_jobs.items():
|
||||
merged[job_id] = merged.get(job_id, 0) + v
|
||||
|
||||
return OffloadingWorkerMetadata(completed_jobs=merged)
|
||||
return OffloadingWorkerMetadata(
|
||||
completed_jobs=merged,
|
||||
transfer_stats=self.transfer_stats.aggregate(other.transfer_stats),
|
||||
)
|
||||
|
||||
@@ -10,37 +10,176 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.worker.worker import TransferType
|
||||
|
||||
logger = init_logger(__name__)
|
||||
from vllm.v1.kv_offload.base import (
|
||||
OffloadingCounterMetadata,
|
||||
OffloadingGaugeMetadata,
|
||||
OffloadingHistogramMetadata,
|
||||
OffloadingMetricMetadata,
|
||||
)
|
||||
from vllm.v1.kv_offload.factory import OffloadingSpecFactory
|
||||
|
||||
|
||||
@dataclass
|
||||
class OffloadingOperationMetrics:
|
||||
op_size: int
|
||||
op_time: float
|
||||
class _TransferMetricName:
|
||||
"""Flat metric names for GPU↔offload-medium transfer operations."""
|
||||
|
||||
LOAD_BYTES = "vllm:kv_offload_load_bytes"
|
||||
LOAD_TIME = "vllm:kv_offload_load_time"
|
||||
LOAD_SIZE = "vllm:kv_offload_load_size"
|
||||
STORE_BYTES = "vllm:kv_offload_store_bytes"
|
||||
STORE_TIME = "vllm:kv_offload_store_time"
|
||||
STORE_SIZE = "vllm:kv_offload_store_size"
|
||||
|
||||
|
||||
class _TransferType:
|
||||
"""Transfer direction labels for deprecated CPU offload metrics."""
|
||||
|
||||
LOAD = "CPU_to_GPU"
|
||||
STORE = "GPU_to_CPU"
|
||||
ALL = (LOAD, STORE)
|
||||
|
||||
|
||||
TRANSFER_SIZE_BUCKETS = (
|
||||
1e6,
|
||||
5e6,
|
||||
10e6,
|
||||
20e6,
|
||||
40e6,
|
||||
60e6,
|
||||
80e6,
|
||||
100e6,
|
||||
150e6,
|
||||
200e6,
|
||||
)
|
||||
|
||||
|
||||
def get_connector_metric_definitions() -> dict[str, OffloadingMetricMetadata]:
|
||||
return {
|
||||
_TransferMetricName.LOAD_BYTES: OffloadingCounterMetadata(
|
||||
documentation="Total bytes loaded from offload storage to GPU.",
|
||||
),
|
||||
_TransferMetricName.LOAD_TIME: OffloadingCounterMetadata(
|
||||
documentation="Total load time from offload storage to GPU, in seconds.",
|
||||
),
|
||||
_TransferMetricName.LOAD_SIZE: OffloadingHistogramMetadata(
|
||||
documentation="Histogram of KV offload load operation size, in bytes.",
|
||||
buckets=TRANSFER_SIZE_BUCKETS,
|
||||
),
|
||||
_TransferMetricName.STORE_BYTES: OffloadingCounterMetadata(
|
||||
documentation="Total bytes stored from GPU to offload storage.",
|
||||
),
|
||||
_TransferMetricName.STORE_TIME: OffloadingCounterMetadata(
|
||||
documentation="Total store time from GPU to offload storage, in seconds.",
|
||||
),
|
||||
_TransferMetricName.STORE_SIZE: OffloadingHistogramMetadata(
|
||||
documentation="Histogram of KV offload store operation size, in bytes.",
|
||||
buckets=TRANSFER_SIZE_BUCKETS,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
_DEPRECATED_TOTAL_BYTES = "vllm:kv_offload_total_bytes"
|
||||
_DEPRECATED_TOTAL_TIME = "vllm:kv_offload_total_time"
|
||||
_DEPRECATED_SIZE = "vllm:kv_offload_size"
|
||||
|
||||
# Deprecated legacy transfer metrics, kept during the migration to the flat
|
||||
# metric names above. These stay in a separate definition block because they
|
||||
# use a transfer_type label, but are emitted from the same flat stats payload
|
||||
# for compatibility.
|
||||
_DEPRECATED_CONNECTOR_METRIC_DEFINITIONS: dict[str, OffloadingMetricMetadata] = {
|
||||
_DEPRECATED_TOTAL_BYTES: OffloadingCounterMetadata(
|
||||
documentation="Number of bytes offloaded by KV connector",
|
||||
),
|
||||
_DEPRECATED_TOTAL_TIME: OffloadingCounterMetadata(
|
||||
documentation="Total time measured by all KV offloading operations",
|
||||
),
|
||||
_DEPRECATED_SIZE: OffloadingHistogramMetadata(
|
||||
documentation="Histogram of KV offload transfer size, in bytes.",
|
||||
buckets=TRANSFER_SIZE_BUCKETS,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
class _MetricType:
|
||||
"""Type tags embedded in the serialized stats payload."""
|
||||
|
||||
COUNTER = "counter"
|
||||
GAUGE = "gauge"
|
||||
HISTOGRAM = "histogram"
|
||||
|
||||
|
||||
class _StatsKey:
|
||||
"""Top-level keys in the serialized stats dict."""
|
||||
|
||||
# Maps metric name -> _MetricType value
|
||||
TYPES = "types"
|
||||
# Maps metric name -> observed value (number or list)
|
||||
DATA = "data"
|
||||
|
||||
|
||||
@dataclass
|
||||
class OffloadingConnectorStats(KVConnectorStats):
|
||||
"""
|
||||
Offloading connector stats use flat metric names as keys.
|
||||
|
||||
The ``data`` dict is structured using ``_StatsKey`` / ``_MetricType``::
|
||||
|
||||
{
|
||||
_StatsKey.TYPES: {name: _MetricType.*, ...},
|
||||
_StatsKey.DATA: {name: value, ...},
|
||||
}
|
||||
|
||||
This structure is self-describing: it survives IPC serialization
|
||||
without needing the full ``OffloadingMetricMetadata`` objects on the
|
||||
receiving side.
|
||||
|
||||
Counter values are aggregated by summing, gauge values use the latest
|
||||
snapshot, and histogram values are lists of observed samples.
|
||||
"""
|
||||
|
||||
def __post_init__(self):
|
||||
if not self.data:
|
||||
# Empty container init, no data is passed in.
|
||||
if _StatsKey.DATA not in self.data:
|
||||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
self.data: dict[str, list[OffloadingOperationMetrics]] = {}
|
||||
self.data: dict[str, Any] = {
|
||||
_StatsKey.TYPES: {},
|
||||
_StatsKey.DATA: {},
|
||||
}
|
||||
|
||||
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
|
||||
if not other.is_empty():
|
||||
for k, v in other.data.items():
|
||||
if k not in self.data:
|
||||
self.data[k] = v
|
||||
@property
|
||||
def _types(self) -> dict[str, str]:
|
||||
return self.data[_StatsKey.TYPES]
|
||||
|
||||
@property
|
||||
def _values(self) -> dict[str, Any]:
|
||||
return self.data[_StatsKey.DATA]
|
||||
|
||||
def aggregate(self, other: "KVConnectorStats") -> "KVConnectorStats":
|
||||
if other.is_empty():
|
||||
return self
|
||||
assert isinstance(other, OffloadingConnectorStats)
|
||||
other_types = other._types
|
||||
other_values = other._values
|
||||
for key, value in other_values.items():
|
||||
type_str = other_types.get(key)
|
||||
if type_str is None:
|
||||
raise AssertionError(f"Unknown offloading stats key: {key}")
|
||||
self._types.setdefault(key, type_str)
|
||||
if type_str == _MetricType.HISTOGRAM:
|
||||
assert isinstance(value, list)
|
||||
if key not in self._values:
|
||||
self._values[key] = value
|
||||
else:
|
||||
accumulator = self.data[k]
|
||||
assert isinstance(accumulator, list)
|
||||
accumulator.extend(v)
|
||||
assert isinstance(self._values[key], list)
|
||||
self._values[key].extend(value)
|
||||
elif type_str == _MetricType.COUNTER:
|
||||
assert isinstance(value, int | float)
|
||||
self._values[key] = self._values.get(key, 0) + value
|
||||
elif type_str == _MetricType.GAUGE:
|
||||
assert isinstance(value, int | float)
|
||||
self._values[key] = value
|
||||
else:
|
||||
raise AssertionError(f"Unknown metric type '{type_str}' for key: {key}")
|
||||
return self
|
||||
|
||||
def reduce(self) -> dict[str, int | float]:
|
||||
@@ -51,29 +190,44 @@ class OffloadingConnectorStats(KVConnectorStats):
|
||||
stats for the last time interval.
|
||||
"""
|
||||
return_dict: dict[str, int | float] = {}
|
||||
for transfer_type, ops_list in self.data.items():
|
||||
assert isinstance(ops_list, list)
|
||||
total_bytes = 0
|
||||
total_time = 0.0
|
||||
for op in ops_list:
|
||||
assert isinstance(op, dict)
|
||||
total_bytes += op["op_size"]
|
||||
total_time += op["op_time"]
|
||||
return_dict[f"{transfer_type}_total_bytes"] = total_bytes
|
||||
return_dict[f"{transfer_type}_total_time"] = total_time
|
||||
for key, value in self._values.items():
|
||||
type_str = self._types.get(key)
|
||||
if type_str is None:
|
||||
raise AssertionError(f"Unknown offloading stats key: {key}")
|
||||
if type_str == _MetricType.HISTOGRAM:
|
||||
assert isinstance(value, list)
|
||||
return_dict[f"{key}_count"] = len(value)
|
||||
return_dict[f"{key}_sum"] = sum(value)
|
||||
elif type_str in (_MetricType.COUNTER, _MetricType.GAUGE):
|
||||
assert isinstance(value, int | float)
|
||||
return_dict[key] = value
|
||||
else:
|
||||
raise AssertionError(f"Unknown metric type '{type_str}' for key: {key}")
|
||||
return return_dict
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
return not self.data
|
||||
return not self.data.get(_StatsKey.DATA)
|
||||
|
||||
def record_transfer(self, num_bytes: int, time: float, transfer_type: TransferType):
|
||||
src, dst = transfer_type
|
||||
transfer_type_key = src + "_to_" + dst
|
||||
op = OffloadingOperationMetrics(num_bytes, time)
|
||||
if transfer_type_key in self.data:
|
||||
self.data[transfer_type_key].append(op)
|
||||
else:
|
||||
self.data[transfer_type_key] = [op]
|
||||
def increase_counter(
|
||||
self, counter_name: str, counter_increase_value: int | float
|
||||
) -> None:
|
||||
"""Increase a counter on the stats payload."""
|
||||
self._types.setdefault(counter_name, _MetricType.COUNTER)
|
||||
self._values[counter_name] = (
|
||||
self._values.get(counter_name, 0) + counter_increase_value
|
||||
)
|
||||
|
||||
def set_gauge(self, gauge_name: str, gauge_value: int | float) -> None:
|
||||
"""Set a gauge snapshot on the stats payload."""
|
||||
self._types.setdefault(gauge_name, _MetricType.GAUGE)
|
||||
self._values[gauge_name] = gauge_value
|
||||
|
||||
def observe_histogram(
|
||||
self, histogram_name: str, histogram_value: int | float
|
||||
) -> None:
|
||||
"""Record a histogram observation on the stats payload."""
|
||||
self._types.setdefault(histogram_name, _MetricType.HISTOGRAM)
|
||||
self._values.setdefault(histogram_name, []).append(histogram_value)
|
||||
|
||||
|
||||
class OffloadPromMetrics(KVConnectorPromMetrics):
|
||||
@@ -89,77 +243,145 @@ class OffloadPromMetrics(KVConnectorPromMetrics):
|
||||
self.histogram_transfer_size: dict[tuple[int, str], PromMetricT] = {}
|
||||
self.counter_kv_bytes: dict[tuple[int, str], PromMetricT] = {}
|
||||
self.counter_kv_transfer_time: dict[tuple[int, str], PromMetricT] = {}
|
||||
buckets = [ # In bytes
|
||||
1e6,
|
||||
5e6,
|
||||
10e6,
|
||||
20e6,
|
||||
40e6,
|
||||
60e6,
|
||||
80e6,
|
||||
100e6,
|
||||
150e6,
|
||||
200e6,
|
||||
]
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(vllm_config)
|
||||
kv_transfer_config = vllm_config.kv_transfer_config
|
||||
assert kv_transfer_config is not None
|
||||
extra_config = kv_transfer_config.kv_connector_extra_config
|
||||
self._offloading_metric_metadata: dict[str, OffloadingMetricMetadata] = {
|
||||
**spec_cls.build_metric_definitions(extra_config),
|
||||
**get_connector_metric_definitions(),
|
||||
}
|
||||
from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec
|
||||
|
||||
self._observe_deprecated_metrics = issubclass(spec_cls, CPUOffloadingSpec)
|
||||
self._offloading_metric_defs: dict[str, PromMetricT] = {}
|
||||
self.offloading_metrics: dict[tuple[int, str], PromMetricT] = {}
|
||||
|
||||
self._counter_kv_bytes = self._counter_cls(
|
||||
name="vllm:kv_offload_total_bytes",
|
||||
documentation="Number of bytes offloaded by KV connector",
|
||||
name=_DEPRECATED_TOTAL_BYTES,
|
||||
documentation=_DEPRECATED_CONNECTOR_METRIC_DEFINITIONS[
|
||||
_DEPRECATED_TOTAL_BYTES
|
||||
].documentation,
|
||||
labelnames=labelnames + ["transfer_type"],
|
||||
)
|
||||
|
||||
self._counter_kv_transfer_time = self._counter_cls(
|
||||
name="vllm:kv_offload_total_time",
|
||||
documentation="Total time measured by all KV offloading operations",
|
||||
name=_DEPRECATED_TOTAL_TIME,
|
||||
documentation=_DEPRECATED_CONNECTOR_METRIC_DEFINITIONS[
|
||||
_DEPRECATED_TOTAL_TIME
|
||||
].documentation,
|
||||
labelnames=labelnames + ["transfer_type"],
|
||||
)
|
||||
|
||||
deprecated_size_metadata = _DEPRECATED_CONNECTOR_METRIC_DEFINITIONS[
|
||||
_DEPRECATED_SIZE
|
||||
]
|
||||
assert isinstance(deprecated_size_metadata, OffloadingHistogramMetadata)
|
||||
self._histogram_transfer_size = self._histogram_cls(
|
||||
name="vllm:kv_offload_size",
|
||||
documentation="Histogram of KV offload transfer size, in bytes.",
|
||||
buckets=buckets[:],
|
||||
name=_DEPRECATED_SIZE,
|
||||
documentation=deprecated_size_metadata.documentation,
|
||||
buckets=deprecated_size_metadata.buckets,
|
||||
labelnames=labelnames + ["transfer_type"],
|
||||
)
|
||||
|
||||
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
|
||||
"""
|
||||
Observe transfer statistics from the new data structure.
|
||||
transfer_stats_data is expected to be a dict where:
|
||||
- keys are transfer type strings (e.g., "cpu_to_gpu", "gpu_to_cpu")
|
||||
- values are lists of OffloadingOperationMetrics objects
|
||||
"""
|
||||
|
||||
for transfer_type, ops in transfer_stats_data.items():
|
||||
# Cache:
|
||||
if (engine_idx, transfer_type) not in self.histogram_transfer_size:
|
||||
for engine_idx, labelvalues in per_engine_labelvalues.items():
|
||||
for transfer_type in _TransferType.ALL:
|
||||
bounded_labelvalues = labelvalues + [transfer_type]
|
||||
self.histogram_transfer_size[(engine_idx, transfer_type)] = (
|
||||
self._histogram_transfer_size.labels(
|
||||
*(self.per_engine_labelvalues[engine_idx] + [transfer_type])
|
||||
)
|
||||
self._histogram_transfer_size.labels(*bounded_labelvalues)
|
||||
)
|
||||
self.counter_kv_bytes[(engine_idx, transfer_type)] = (
|
||||
self._counter_kv_bytes.labels(
|
||||
*(self.per_engine_labelvalues[engine_idx] + [transfer_type])
|
||||
)
|
||||
self._counter_kv_bytes.labels(*bounded_labelvalues)
|
||||
)
|
||||
self.counter_kv_transfer_time[(engine_idx, transfer_type)] = (
|
||||
self._counter_kv_transfer_time.labels(
|
||||
*(self.per_engine_labelvalues[engine_idx] + [transfer_type])
|
||||
)
|
||||
self._counter_kv_transfer_time.labels(*bounded_labelvalues)
|
||||
)
|
||||
|
||||
# Process ops:
|
||||
assert isinstance(ops, list)
|
||||
for op in ops: # ops is a list of serialized OffloadingOperationMetrics
|
||||
assert isinstance(op, dict)
|
||||
# Observe size histogram
|
||||
self.histogram_transfer_size[(engine_idx, transfer_type)].observe(
|
||||
op["op_size"]
|
||||
for metric_name, metadata in self._offloading_metric_metadata.items():
|
||||
self._offloading_metric_defs[metric_name] = self._create_metric(
|
||||
metric_name, metadata
|
||||
)
|
||||
for engine_idx, labelvalues in per_engine_labelvalues.items():
|
||||
self.offloading_metrics[(engine_idx, metric_name)] = (
|
||||
self._offloading_metric_defs[metric_name].labels(*labelvalues)
|
||||
)
|
||||
|
||||
# Increment byte and time counters
|
||||
self.counter_kv_bytes[(engine_idx, transfer_type)].inc(op["op_size"])
|
||||
def _create_metric(
|
||||
self, metric_name: str, metadata: OffloadingMetricMetadata
|
||||
) -> Any:
|
||||
kwargs: dict[str, Any] = {
|
||||
"name": metric_name,
|
||||
"documentation": metadata.documentation,
|
||||
"labelnames": self._labelnames,
|
||||
}
|
||||
if isinstance(metadata, OffloadingCounterMetadata):
|
||||
metric_cls = self._counter_cls
|
||||
elif isinstance(metadata, OffloadingGaugeMetadata):
|
||||
metric_cls = self._gauge_cls
|
||||
elif isinstance(metadata, OffloadingHistogramMetadata):
|
||||
metric_cls = self._histogram_cls
|
||||
if metadata.buckets is not None:
|
||||
kwargs["buckets"] = metadata.buckets
|
||||
else:
|
||||
raise AssertionError(f"Unknown offloading metric metadata: {metadata}")
|
||||
return metric_cls(**kwargs)
|
||||
|
||||
self.counter_kv_transfer_time[(engine_idx, transfer_type)].inc(
|
||||
op["op_time"]
|
||||
def _increase_counter(
|
||||
self, metric_name: str, value: int | float, engine_idx: int
|
||||
) -> None:
|
||||
self.offloading_metrics[(engine_idx, metric_name)].inc(value)
|
||||
if not self._observe_deprecated_metrics:
|
||||
return
|
||||
# Keep deprecated CPU offload transfer metrics updated during the
|
||||
# transition to flat metric names.
|
||||
if metric_name == _TransferMetricName.LOAD_BYTES:
|
||||
self.counter_kv_bytes[(engine_idx, _TransferType.LOAD)].inc(value)
|
||||
elif metric_name == _TransferMetricName.LOAD_TIME:
|
||||
self.counter_kv_transfer_time[(engine_idx, _TransferType.LOAD)].inc(value)
|
||||
elif metric_name == _TransferMetricName.STORE_BYTES:
|
||||
self.counter_kv_bytes[(engine_idx, _TransferType.STORE)].inc(value)
|
||||
elif metric_name == _TransferMetricName.STORE_TIME:
|
||||
self.counter_kv_transfer_time[(engine_idx, _TransferType.STORE)].inc(value)
|
||||
|
||||
def _set_gauge(self, metric_name: str, value: int | float, engine_idx: int) -> None:
|
||||
self.offloading_metrics[(engine_idx, metric_name)].set(value)
|
||||
|
||||
def _observe_histogram(
|
||||
self, metric_name: str, value: list[int | float], engine_idx: int
|
||||
) -> None:
|
||||
for observation in value:
|
||||
self.offloading_metrics[(engine_idx, metric_name)].observe(observation)
|
||||
if not self._observe_deprecated_metrics:
|
||||
continue
|
||||
# Keep deprecated CPU offload transfer metrics updated during the
|
||||
# transition to flat metric names.
|
||||
if metric_name == _TransferMetricName.LOAD_SIZE:
|
||||
self.histogram_transfer_size[(engine_idx, _TransferType.LOAD)].observe(
|
||||
observation
|
||||
)
|
||||
elif metric_name == _TransferMetricName.STORE_SIZE:
|
||||
self.histogram_transfer_size[(engine_idx, _TransferType.STORE)].observe(
|
||||
observation
|
||||
)
|
||||
|
||||
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
|
||||
"""Observe transfer statistics."""
|
||||
metric_types = transfer_stats_data.get(_StatsKey.TYPES, {})
|
||||
metric_data = transfer_stats_data.get(_StatsKey.DATA, {})
|
||||
for key, value in metric_data.items():
|
||||
type_str = metric_types.get(key)
|
||||
if type_str is None:
|
||||
raise AssertionError(f"Unknown offloading stats key: {key}")
|
||||
assert key in self._offloading_metric_defs
|
||||
if type_str == _MetricType.COUNTER:
|
||||
assert isinstance(value, int | float)
|
||||
self._increase_counter(key, value, engine_idx)
|
||||
elif type_str == _MetricType.GAUGE:
|
||||
assert isinstance(value, int | float)
|
||||
self._set_gauge(key, value, engine_idx)
|
||||
elif type_str == _MetricType.HISTOGRAM:
|
||||
assert isinstance(value, list)
|
||||
assert all(isinstance(v, int | float) for v in value)
|
||||
self._observe_histogram(key, value, engine_idx)
|
||||
else:
|
||||
raise AssertionError(f"Unknown metric type '{type_str}' for key: {key}")
|
||||
|
||||
@@ -14,6 +14,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
|
||||
ReqId,
|
||||
TransferJob,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
OffloadingConnectorStats,
|
||||
_TransferMetricName,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
@@ -259,9 +263,13 @@ def _create_req_context(req: Request) -> ReqContext:
|
||||
class OffloadingConnectorScheduler:
|
||||
"""Implementation of Scheduler side methods"""
|
||||
|
||||
def __init__(self, spec: OffloadingSpec):
|
||||
def __init__(
|
||||
self,
|
||||
spec: OffloadingSpec,
|
||||
):
|
||||
self.config = SchedulerOffloadConfig.from_spec(spec)
|
||||
self.manager: OffloadingManager = spec.get_manager()
|
||||
self._connector_stats: OffloadingConnectorStats | None = None
|
||||
|
||||
full_attention_groups: list[int] = []
|
||||
sliding_window_groups: list[int] = []
|
||||
@@ -952,6 +960,39 @@ class OffloadingConnectorScheduler:
|
||||
if not isinstance(meta, OffloadingWorkerMetadata):
|
||||
assert meta is None
|
||||
meta = OffloadingWorkerMetadata()
|
||||
if not meta.transfer_stats.is_empty():
|
||||
transfer_stats = OffloadingConnectorStats()
|
||||
if not meta.transfer_stats.load.is_empty():
|
||||
transfer_stats.increase_counter(
|
||||
_TransferMetricName.LOAD_BYTES,
|
||||
meta.transfer_stats.load.bytes,
|
||||
)
|
||||
transfer_stats.increase_counter(
|
||||
_TransferMetricName.LOAD_TIME,
|
||||
meta.transfer_stats.load.time,
|
||||
)
|
||||
for size in meta.transfer_stats.load.sizes:
|
||||
transfer_stats.observe_histogram(
|
||||
_TransferMetricName.LOAD_SIZE, size
|
||||
)
|
||||
if not meta.transfer_stats.store.is_empty():
|
||||
transfer_stats.increase_counter(
|
||||
_TransferMetricName.STORE_BYTES,
|
||||
meta.transfer_stats.store.bytes,
|
||||
)
|
||||
transfer_stats.increase_counter(
|
||||
_TransferMetricName.STORE_TIME,
|
||||
meta.transfer_stats.store.time,
|
||||
)
|
||||
for size in meta.transfer_stats.store.sizes:
|
||||
transfer_stats.observe_histogram(
|
||||
_TransferMetricName.STORE_SIZE, size
|
||||
)
|
||||
if self._connector_stats is None:
|
||||
self._connector_stats = transfer_stats
|
||||
else:
|
||||
self._connector_stats.aggregate(transfer_stats)
|
||||
|
||||
for job_id, count in meta.completed_jobs.items():
|
||||
assert count > 0
|
||||
if job_id < self._stale_job_threshold:
|
||||
@@ -990,6 +1031,19 @@ class OffloadingConnectorScheduler:
|
||||
if not req_status.transfer_jobs and req_status.req.is_finished():
|
||||
del self._req_status[job_status.req_id]
|
||||
|
||||
def get_stats(self) -> OffloadingConnectorStats | None:
|
||||
stats = self._connector_stats
|
||||
self._connector_stats = None
|
||||
|
||||
manager_stats = self.manager.get_stats()
|
||||
if manager_stats is not None:
|
||||
if stats is None:
|
||||
stats = manager_stats
|
||||
else:
|
||||
stats.aggregate(manager_stats)
|
||||
|
||||
return stats
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
request: Request,
|
||||
|
||||
@@ -5,17 +5,11 @@ from dataclasses import replace
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorStats,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
|
||||
OffloadingConnectorMetadata,
|
||||
OffloadingWorkerMetadata,
|
||||
ReqId,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
OffloadingConnectorStats,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
@@ -44,7 +38,6 @@ class OffloadingConnectorWorker:
|
||||
self.spec = spec
|
||||
self.worker = OffloadingWorker()
|
||||
|
||||
self.kv_connector_stats = OffloadingConnectorStats()
|
||||
# job_id -> req_id for in-flight loads.
|
||||
self._load_jobs: dict[int, ReqId] = {}
|
||||
self._unsubmitted_store_jobs: list[tuple[int, TransferSpec]] = []
|
||||
@@ -271,15 +264,18 @@ class OffloadingConnectorWorker:
|
||||
# we currently do not support job failures
|
||||
job_id = transfer_result.job_id
|
||||
assert transfer_result.success
|
||||
is_load = job_id in self._load_jobs
|
||||
if (
|
||||
transfer_result.transfer_time
|
||||
transfer_result.transfer_time is not None
|
||||
and transfer_result.transfer_size is not None
|
||||
and transfer_result.transfer_type is not None
|
||||
):
|
||||
self.kv_connector_stats.record_transfer(
|
||||
num_bytes=transfer_result.transfer_size,
|
||||
time=transfer_result.transfer_time,
|
||||
transfer_type=transfer_result.transfer_type,
|
||||
if is_load:
|
||||
stats = self._connector_worker_meta.transfer_stats.load
|
||||
else:
|
||||
stats = self._connector_worker_meta.transfer_stats.store
|
||||
stats.record(
|
||||
transfer_result.transfer_size,
|
||||
transfer_result.transfer_time,
|
||||
)
|
||||
|
||||
self._connector_worker_meta.mark_completed(job_id)
|
||||
@@ -297,18 +293,6 @@ class OffloadingConnectorWorker:
|
||||
self._connector_worker_meta = OffloadingWorkerMetadata()
|
||||
return meta
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
"""
|
||||
Get the KV transfer stats for the connector.
|
||||
"""
|
||||
|
||||
if self.kv_connector_stats.is_empty():
|
||||
return None
|
||||
# Clear stats for next iteration
|
||||
kv_connector_stats = self.kv_connector_stats
|
||||
self.kv_connector_stats = OffloadingConnectorStats()
|
||||
return kv_connector_stats
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._unsubmitted_store_jobs.clear()
|
||||
self._load_jobs.clear()
|
||||
|
||||
@@ -184,9 +184,14 @@ class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
return True
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
if self.connector_worker is None:
|
||||
return None # We only emit stats from the worker-side
|
||||
return self.connector_worker.get_kv_connector_stats()
|
||||
if self.connector_scheduler is not None:
|
||||
return self.connector_scheduler.get_stats()
|
||||
|
||||
# TODO(orozery): Remove once PR #43877 lands
|
||||
if self.connector_worker is not None:
|
||||
return OffloadingConnectorStats()
|
||||
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def build_kv_connector_stats(
|
||||
|
||||
Reference in New Issue
Block a user