[HiSparse] Log steady-state max concurrency and expose host-tier utilization gauges (#58949)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Matthew Bonanni
parent
8542c3eff5
commit
2e3154aa18
@@ -30,10 +30,12 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
|
||||
from vllm.v1.hisparse.layout import get_hisparse_steady_state_concurrency
|
||||
from vllm.v1.hisparse.types import SparseKVOffloadCommand, SparseKVRowMirror
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
|
||||
@@ -44,6 +46,8 @@ if TYPE_CHECKING:
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class HiSparseConnectorMetadata(KVConnectorMetadata):
|
||||
@@ -99,6 +103,17 @@ class HiSparseConnectorScheduler:
|
||||
assert self.coordinator is None
|
||||
self.coordinator = coordinator
|
||||
|
||||
def get_kv_connector_stats(self) -> HiSparseKVConnectorStats | None:
|
||||
if self.coordinator is None:
|
||||
return None
|
||||
host_pool = self.coordinator.get_host_block_pool()
|
||||
assert host_pool is not None
|
||||
stats = HiSparseKVConnectorStats()
|
||||
stats.record_host_usage(
|
||||
host_pool.get_usage(), len(self.coordinator.pending_spills)
|
||||
)
|
||||
return stats
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> HiSparseConnectorMetadata:
|
||||
@@ -223,6 +238,16 @@ class HiSparseConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
),
|
||||
draft_kv_lookahead=vllm_config.num_lookahead_tokens,
|
||||
)
|
||||
max_model_len = vllm_config.model_config.max_model_len
|
||||
steady_concurrency = get_hisparse_steady_state_concurrency(
|
||||
vllm_config, kv_cache_config
|
||||
)
|
||||
logger.info_once(
|
||||
"HiSparse steady-state maximum concurrency for %s tokens per "
|
||||
"request: %.2fx (running requests reading from host).",
|
||||
f"{max_model_len:,}",
|
||||
steady_concurrency,
|
||||
)
|
||||
elif role == KVConnectorRole.WORKER:
|
||||
self.connector_worker = HiSparseConnectorWorker(
|
||||
vllm_config, kv_cache_config
|
||||
@@ -263,9 +288,11 @@ class HiSparseConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
self.connector_worker.reset_hot_state()
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
if self.connector_worker is None:
|
||||
return None
|
||||
return self.connector_worker.get_kv_connector_stats()
|
||||
if self.connector_worker is not None:
|
||||
return self.connector_worker.get_kv_connector_stats()
|
||||
if self.connector_scheduler is not None:
|
||||
return self.connector_scheduler.get_kv_connector_stats()
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def build_kv_connector_stats(
|
||||
|
||||
@@ -14,14 +14,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
)
|
||||
from vllm.v1.metrics.utils import create_metric_per_engine
|
||||
|
||||
_HISPARSE_COUNTERS: tuple[tuple[str, str], ...] = (
|
||||
("cache_hits", "Number of HiSparse device hot-buffer hits."),
|
||||
("cache_misses", "Number of HiSparse device hot-buffer misses."),
|
||||
(
|
||||
"host_to_device_bytes",
|
||||
"Bytes transferred from host KV storage to HiSparse hot buffers.",
|
||||
),
|
||||
)
|
||||
_HISPARSE_LEVEL_KEYS = frozenset({"host_cache_usage_perc", "pending_page_transfers"})
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -30,6 +23,8 @@ class HiSparseKVConnectorStats(KVConnectorStats):
|
||||
|
||||
Each list entry is the delta recorded over one device counter snapshot
|
||||
interval, so a list can hold multiple snapshots per logging interval.
|
||||
The host-tier gauges are level values, not deltas: aggregation keeps
|
||||
only the most recent observation.
|
||||
"""
|
||||
|
||||
def __post_init__(self):
|
||||
@@ -39,10 +34,12 @@ class HiSparseKVConnectorStats(KVConnectorStats):
|
||||
|
||||
def reset(self):
|
||||
# Must be serializable
|
||||
self.data: dict[str, list[int]] = {
|
||||
self.data: dict[str, list[int | float]] = {
|
||||
"cache_hits": [],
|
||||
"cache_misses": [],
|
||||
"host_to_device_bytes": [],
|
||||
"host_cache_usage_perc": [],
|
||||
"pending_page_transfers": [],
|
||||
}
|
||||
|
||||
def record_snapshot(self, hits: int, misses: int, host_to_device_bytes: int):
|
||||
@@ -50,21 +47,32 @@ class HiSparseKVConnectorStats(KVConnectorStats):
|
||||
self.data["cache_misses"].append(misses)
|
||||
self.data["host_to_device_bytes"].append(host_to_device_bytes)
|
||||
|
||||
def record_host_usage(self, usage: float, pending_page_transfers: int) -> None:
|
||||
self.data["host_cache_usage_perc"].append(usage)
|
||||
self.data["pending_page_transfers"].append(pending_page_transfers)
|
||||
|
||||
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
|
||||
if not other.is_empty():
|
||||
for k, v in other.data.items():
|
||||
accumulator = self.data[k]
|
||||
assert isinstance(accumulator, list)
|
||||
accumulator.extend(v)
|
||||
if k in _HISPARSE_LEVEL_KEYS:
|
||||
accumulator.extend(v[-1:])
|
||||
else:
|
||||
accumulator.extend(v)
|
||||
return self
|
||||
|
||||
def reduce(self) -> dict[str, int | float]:
|
||||
# Compute compact representative stats suitable for CLI logging.
|
||||
return {
|
||||
reduced: dict[str, int | float] = {
|
||||
"HiSparse hot-buffer hits": sum(self.data["cache_hits"]),
|
||||
"HiSparse hot-buffer misses": sum(self.data["cache_misses"]),
|
||||
"HiSparse host-to-device bytes": sum(self.data["host_to_device_bytes"]),
|
||||
}
|
||||
if self.data["host_cache_usage_perc"]:
|
||||
usage = self.data["host_cache_usage_perc"][-1]
|
||||
reduced["HiSparse host KV cache usage %"] = round(100 * usage, 1)
|
||||
return reduced
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
return all(len(values) == 0 for values in self.data.values())
|
||||
@@ -80,18 +88,66 @@ class HiSparsePromMetrics(KVConnectorPromMetrics):
|
||||
):
|
||||
super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues)
|
||||
|
||||
self.hisparse_counters: dict[str, Any] = {}
|
||||
for name, documentation in _HISPARSE_COUNTERS:
|
||||
counter = self._counter_cls(
|
||||
name=f"vllm:hisparse_{name}",
|
||||
documentation=documentation,
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.hisparse_counters[name] = create_metric_per_engine(
|
||||
counter, self.per_engine_labelvalues
|
||||
)
|
||||
counter_cache_hits = self._counter_cls(
|
||||
name="vllm:hisparse_cache_hits",
|
||||
documentation="Number of HiSparse device hot-buffer hits.",
|
||||
labelnames=labelnames,
|
||||
)
|
||||
counter_cache_misses = self._counter_cls(
|
||||
name="vllm:hisparse_cache_misses",
|
||||
documentation="Number of HiSparse device hot-buffer misses.",
|
||||
labelnames=labelnames,
|
||||
)
|
||||
counter_host_to_device_bytes = self._counter_cls(
|
||||
name="vllm:hisparse_host_to_device_bytes",
|
||||
documentation=(
|
||||
"Bytes transferred from host KV storage to HiSparse hot buffers."
|
||||
),
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.hisparse_counters: dict[str, Any] = {
|
||||
"cache_hits": create_metric_per_engine(
|
||||
counter_cache_hits, self.per_engine_labelvalues
|
||||
),
|
||||
"cache_misses": create_metric_per_engine(
|
||||
counter_cache_misses, self.per_engine_labelvalues
|
||||
),
|
||||
"host_to_device_bytes": create_metric_per_engine(
|
||||
counter_host_to_device_bytes, self.per_engine_labelvalues
|
||||
),
|
||||
}
|
||||
|
||||
gauge_host_cache_usage = self._gauge_cls(
|
||||
name="vllm:hisparse_host_cache_usage_perc",
|
||||
documentation=(
|
||||
"HiSparse host KV-cache usage. 1 means 100 percent usage. "
|
||||
"Evictable cached blocks count as free."
|
||||
),
|
||||
multiprocess_mode="mostrecent",
|
||||
labelnames=labelnames,
|
||||
)
|
||||
gauge_pending_page_transfers = self._gauge_cls(
|
||||
name="vllm:hisparse_pending_page_transfers",
|
||||
documentation=(
|
||||
"HiSparse page transfers (write-backs and restores) not yet completed."
|
||||
),
|
||||
multiprocess_mode="mostrecent",
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.hisparse_gauges: dict[str, Any] = {
|
||||
"host_cache_usage_perc": create_metric_per_engine(
|
||||
gauge_host_cache_usage, self.per_engine_labelvalues
|
||||
),
|
||||
"pending_page_transfers": create_metric_per_engine(
|
||||
gauge_pending_page_transfers, self.per_engine_labelvalues
|
||||
),
|
||||
}
|
||||
|
||||
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
|
||||
for name, counter in self.hisparse_counters.items():
|
||||
for value in transfer_stats_data.get(name, []):
|
||||
counter[engine_idx].inc(value)
|
||||
for name, gauge in self.hisparse_gauges.items():
|
||||
values = transfer_stats_data.get(name, [])
|
||||
if values:
|
||||
gauge[engine_idx].set(values[-1])
|
||||
|
||||
Reference in New Issue
Block a user