[HiSparse] Log steady-state max concurrency and expose host-tier utilization gauges (#58949)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
Robert Shaw
2026-10-05 14:20:32 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 Matthew Bonanni
parent 8542c3eff5
commit 2e3154aa18
12 changed files with 323 additions and 34 deletions
@@ -30,10 +30,12 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
PromMetric,
PromMetricT,
)
from vllm.logger import init_logger
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
from vllm.v1.hisparse.layout import get_hisparse_steady_state_concurrency
from vllm.v1.hisparse.types import SparseKVOffloadCommand, SparseKVRowMirror
from vllm.v1.outputs import KVConnectorOutput
@@ -44,6 +46,8 @@ if TYPE_CHECKING:
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.request import Request
logger = init_logger(__name__)
@dataclass
class HiSparseConnectorMetadata(KVConnectorMetadata):
@@ -99,6 +103,17 @@ class HiSparseConnectorScheduler:
assert self.coordinator is None
self.coordinator = coordinator
def get_kv_connector_stats(self) -> HiSparseKVConnectorStats | None:
if self.coordinator is None:
return None
host_pool = self.coordinator.get_host_block_pool()
assert host_pool is not None
stats = HiSparseKVConnectorStats()
stats.record_host_usage(
host_pool.get_usage(), len(self.coordinator.pending_spills)
)
return stats
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> HiSparseConnectorMetadata:
@@ -223,6 +238,16 @@ class HiSparseConnector(KVConnectorBase_V1, SupportsHMA):
),
draft_kv_lookahead=vllm_config.num_lookahead_tokens,
)
max_model_len = vllm_config.model_config.max_model_len
steady_concurrency = get_hisparse_steady_state_concurrency(
vllm_config, kv_cache_config
)
logger.info_once(
"HiSparse steady-state maximum concurrency for %s tokens per "
"request: %.2fx (running requests reading from host).",
f"{max_model_len:,}",
steady_concurrency,
)
elif role == KVConnectorRole.WORKER:
self.connector_worker = HiSparseConnectorWorker(
vllm_config, kv_cache_config
@@ -263,9 +288,11 @@ class HiSparseConnector(KVConnectorBase_V1, SupportsHMA):
self.connector_worker.reset_hot_state()
def get_kv_connector_stats(self) -> KVConnectorStats | None:
if self.connector_worker is None:
return None
return self.connector_worker.get_kv_connector_stats()
if self.connector_worker is not None:
return self.connector_worker.get_kv_connector_stats()
if self.connector_scheduler is not None:
return self.connector_scheduler.get_kv_connector_stats()
return None
@classmethod
def build_kv_connector_stats(
@@ -14,14 +14,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
)
from vllm.v1.metrics.utils import create_metric_per_engine
_HISPARSE_COUNTERS: tuple[tuple[str, str], ...] = (
("cache_hits", "Number of HiSparse device hot-buffer hits."),
("cache_misses", "Number of HiSparse device hot-buffer misses."),
(
"host_to_device_bytes",
"Bytes transferred from host KV storage to HiSparse hot buffers.",
),
)
_HISPARSE_LEVEL_KEYS = frozenset({"host_cache_usage_perc", "pending_page_transfers"})
@dataclass
@@ -30,6 +23,8 @@ class HiSparseKVConnectorStats(KVConnectorStats):
Each list entry is the delta recorded over one device counter snapshot
interval, so a list can hold multiple snapshots per logging interval.
The host-tier gauges are level values, not deltas: aggregation keeps
only the most recent observation.
"""
def __post_init__(self):
@@ -39,10 +34,12 @@ class HiSparseKVConnectorStats(KVConnectorStats):
def reset(self):
# Must be serializable
self.data: dict[str, list[int]] = {
self.data: dict[str, list[int | float]] = {
"cache_hits": [],
"cache_misses": [],
"host_to_device_bytes": [],
"host_cache_usage_perc": [],
"pending_page_transfers": [],
}
def record_snapshot(self, hits: int, misses: int, host_to_device_bytes: int):
@@ -50,21 +47,32 @@ class HiSparseKVConnectorStats(KVConnectorStats):
self.data["cache_misses"].append(misses)
self.data["host_to_device_bytes"].append(host_to_device_bytes)
def record_host_usage(self, usage: float, pending_page_transfers: int) -> None:
self.data["host_cache_usage_perc"].append(usage)
self.data["pending_page_transfers"].append(pending_page_transfers)
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
if not other.is_empty():
for k, v in other.data.items():
accumulator = self.data[k]
assert isinstance(accumulator, list)
accumulator.extend(v)
if k in _HISPARSE_LEVEL_KEYS:
accumulator.extend(v[-1:])
else:
accumulator.extend(v)
return self
def reduce(self) -> dict[str, int | float]:
# Compute compact representative stats suitable for CLI logging.
return {
reduced: dict[str, int | float] = {
"HiSparse hot-buffer hits": sum(self.data["cache_hits"]),
"HiSparse hot-buffer misses": sum(self.data["cache_misses"]),
"HiSparse host-to-device bytes": sum(self.data["host_to_device_bytes"]),
}
if self.data["host_cache_usage_perc"]:
usage = self.data["host_cache_usage_perc"][-1]
reduced["HiSparse host KV cache usage %"] = round(100 * usage, 1)
return reduced
def is_empty(self) -> bool:
return all(len(values) == 0 for values in self.data.values())
@@ -80,18 +88,66 @@ class HiSparsePromMetrics(KVConnectorPromMetrics):
):
super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues)
self.hisparse_counters: dict[str, Any] = {}
for name, documentation in _HISPARSE_COUNTERS:
counter = self._counter_cls(
name=f"vllm:hisparse_{name}",
documentation=documentation,
labelnames=labelnames,
)
self.hisparse_counters[name] = create_metric_per_engine(
counter, self.per_engine_labelvalues
)
counter_cache_hits = self._counter_cls(
name="vllm:hisparse_cache_hits",
documentation="Number of HiSparse device hot-buffer hits.",
labelnames=labelnames,
)
counter_cache_misses = self._counter_cls(
name="vllm:hisparse_cache_misses",
documentation="Number of HiSparse device hot-buffer misses.",
labelnames=labelnames,
)
counter_host_to_device_bytes = self._counter_cls(
name="vllm:hisparse_host_to_device_bytes",
documentation=(
"Bytes transferred from host KV storage to HiSparse hot buffers."
),
labelnames=labelnames,
)
self.hisparse_counters: dict[str, Any] = {
"cache_hits": create_metric_per_engine(
counter_cache_hits, self.per_engine_labelvalues
),
"cache_misses": create_metric_per_engine(
counter_cache_misses, self.per_engine_labelvalues
),
"host_to_device_bytes": create_metric_per_engine(
counter_host_to_device_bytes, self.per_engine_labelvalues
),
}
gauge_host_cache_usage = self._gauge_cls(
name="vllm:hisparse_host_cache_usage_perc",
documentation=(
"HiSparse host KV-cache usage. 1 means 100 percent usage. "
"Evictable cached blocks count as free."
),
multiprocess_mode="mostrecent",
labelnames=labelnames,
)
gauge_pending_page_transfers = self._gauge_cls(
name="vllm:hisparse_pending_page_transfers",
documentation=(
"HiSparse page transfers (write-backs and restores) not yet completed."
),
multiprocess_mode="mostrecent",
labelnames=labelnames,
)
self.hisparse_gauges: dict[str, Any] = {
"host_cache_usage_perc": create_metric_per_engine(
gauge_host_cache_usage, self.per_engine_labelvalues
),
"pending_page_transfers": create_metric_per_engine(
gauge_pending_page_transfers, self.per_engine_labelvalues
),
}
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
for name, counter in self.hisparse_counters.items():
for value in transfer_stats_data.get(name, []):
counter[engine_idx].inc(value)
for name, gauge in self.hisparse_gauges.items():
values = transfer_stats_data.get(name, [])
if values:
gauge[engine_idx].set(values[-1])