[HiSparse] Log steady-state max concurrency and expose host-tier utilization gauges (#58949)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
Robert Shaw
2026-10-05 14:20:32 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 Matthew Bonanni
parent 8542c3eff5
commit 2e3154aa18
12 changed files with 323 additions and 34 deletions
+43
View File
@@ -60,6 +60,7 @@ from vllm.v1.core.kv_cache_utils import (
from vllm.v1.hisparse.layout import (
create_hisparse_layout,
get_hisparse_kv_cache_groups,
get_hisparse_steady_state_concurrency,
)
from vllm.v1.kv_cache_interface import (
ChunkedLocalAttentionSpec,
@@ -198,6 +199,48 @@ def test_hisparse_hma_uses_resolved_gpu_block_size(
assert scheduler_block_size == hash_block_size == gpu_block_size
def test_hisparse_steady_state_concurrency_excludes_spilled_resident_pages():
"""Running requests pin only their resident tail; the newest still needs its
full in-flight window to be admitted."""
block_size = 16
config = SimpleNamespace(
model_config=SimpleNamespace(max_model_len=10 * block_size),
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
max_in_flight_tokens=10 * block_size,
)
attn_spec = FullAttentionSpec(
block_size=block_size, num_kv_heads=1, head_size=64, dtype=torch.float16
)
page_size = attn_spec.page_size_bytes
kv_cache_config = KVCacheConfig(
num_blocks=64,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(["source"], attn_spec, host_resident=True),
KVCacheGroupSpec(["indexer"], attn_spec),
KVCacheGroupSpec(
["resident"],
HiSparseResidentSpec(block_size=block_size, page_size=page_size),
),
KVCacheGroupSpec(
["hot"],
HiSparseHotSpec(
block_size=block_size, page_size=page_size, blocks_per_request=4
),
),
],
hisparse_host_num_blocks=100,
)
# Admission takes indexer 10 + resident 10 + hot 4 = 24 blocks and steady
# state 16, so 3 fit (2 x 16 + 24 <= 64) but not 4 (3 x 16 + 24 > 64).
assert get_hisparse_steady_state_concurrency(config, kv_cache_config) == 3.5
# The worst-case bound charges all 10 resident pages.
assert get_max_concurrency_for_kv_cache_config(
config, kv_cache_config
) == pytest.approx(64 / 24)
@pytest.mark.parametrize("extra_blocks,ok", [(1, False), (2, True)])
def test_hisparse_host_pool_must_fit_max_model_len(monkeypatch, extra_blocks, ok):
"""Every page needs a host block, so one max_model_len request must fit."""
@@ -64,6 +64,42 @@ def test_hisparse_requires_block_outermost_device_layout():
assert HiSparseConnector.get_required_kvcache_layout(MagicMock()) == "BLHNC"
def test_scheduler_stats_report_host_pool_usage():
"""The scheduler-side stats sample the coordinator's host pool level."""
from tests.v1.core.test_prefix_caching import (
HISPARSE_BLOCK_SIZE,
make_hisparse_kv_cache_manager,
make_request,
sha256,
)
from vllm.v1.core.kv_cache_utils import init_none_hash
init_none_hash(sha256)
manager = make_hisparse_kv_cache_manager(32, host_num_blocks=8)
scheduler = HiSparseConnectorScheduler(async_speculative=False)
scheduler.bind_coordinator(get_hisparse_coordinator(manager))
coordinator = scheduler.coordinator
stats = scheduler.get_kv_connector_stats()
assert stats.data["host_cache_usage_perc"] == [0.0]
assert stats.data["pending_page_transfers"] == [0]
request = make_request(
"request", list(range(4 * HISPARSE_BLOCK_SIZE)), HISPARSE_BLOCK_SIZE, sha256
)
assert manager.allocate_slots(request, num_new_tokens=64) is not None
host_blocks = coordinator.host_manager.req_to_blocks[request.request_id]
num_used = sum(not block.is_null for block in host_blocks)
# The pool reserves the null block, so 8 configured blocks yield 7 usable.
expected = pytest.approx(num_used / 7)
assert 0 < num_used <= 7
for _ in range(2):
stats = scheduler.get_kv_connector_stats()
assert stats.data["host_cache_usage_perc"] == [expected]
def test_no_forward_enqueues_deferred_hisparse_transfers():
"""A zero-token step must still enqueue deferred post-forward transfers."""
connector = object.__new__(ActiveKVConnector)
@@ -26,6 +26,28 @@ def test_record_snapshot_and_reduce():
assert reduced["HiSparse host-to-device bytes"] == 64
def test_record_host_usage_reports_latest_level():
stats = HiSparseKVConnectorStats()
stats.record_host_usage(usage=0.4, pending_page_transfers=2)
stats.record_host_usage(usage=0.8, pending_page_transfers=0)
reduced = stats.reduce()
assert reduced["HiSparse host KV cache usage %"] == 80.0
def test_aggregate_keeps_latest_host_usage_level():
first = HiSparseKVConnectorStats()
first.record_host_usage(usage=0.4, pending_page_transfers=2)
second = HiSparseKVConnectorStats()
second.record_host_usage(usage=0.8, pending_page_transfers=0)
second.record_host_usage(usage=0.6, pending_page_transfers=1)
first.aggregate(second)
assert first.data["host_cache_usage_perc"] == [0.4, 0.6]
assert first.data["pending_page_transfers"] == [2, 1]
def test_aggregate_extends_snapshot_deltas():
first = HiSparseKVConnectorStats()
first.record_snapshot(hits=7, misses=3, host_to_device_bytes=48)
@@ -34,11 +56,9 @@ def test_aggregate_extends_snapshot_deltas():
first.aggregate(second)
assert first.data == {
"cache_hits": [7, 5],
"cache_misses": [3, 1],
"host_to_device_bytes": [48, 16],
}
assert first.data["cache_hits"] == [7, 5]
assert first.data["cache_misses"] == [3, 1]
assert first.data["host_to_device_bytes"] == [48, 16]
def test_aggregate_skips_empty_stats():
@@ -60,3 +80,54 @@ def test_build_kv_connector_stats_round_trip():
assert rebuilt is not None
assert isinstance(rebuilt, HiSparseKVConnectorStats)
assert rebuilt.reduce() == stats.reduce()
def test_prom_metrics_observe_host_usage_gauges():
from types import SimpleNamespace
from typing import Any
from prometheus_client import Counter, Gauge, Histogram
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import (
HiSparsePromMetrics,
)
class _FakeMetric:
def __init__(self, **kwargs: Any):
self.kwargs = kwargs
self.increments: list[int | float] = []
self.set_values: list[int | float] = []
def labels(self, *labelvalues: object) -> "_FakeMetric":
return self
def inc(self, value: int | float) -> None:
self.increments.append(value)
def set(self, value: int | float) -> None:
self.set_values.append(value)
created: dict[str, _FakeMetric] = {}
class _NamedFake(_FakeMetric):
def __init__(self, **kwargs: Any):
super().__init__(**kwargs)
created[kwargs["name"]] = self
prom = HiSparsePromMetrics(
vllm_config=SimpleNamespace(kv_transfer_config=None),
metric_types={Gauge: _NamedFake, Counter: _NamedFake, Histogram: _NamedFake},
labelnames=["model_name"],
per_engine_labelvalues={0: ["model"]},
)
stats = HiSparseKVConnectorStats()
stats.record_snapshot(hits=3, misses=2, host_to_device_bytes=32)
stats.record_host_usage(usage=0.5, pending_page_transfers=1)
stats.record_host_usage(usage=0.75, pending_page_transfers=3)
prom.observe(stats.to_dict())
assert created["vllm:hisparse_cache_hits"].increments == [3]
assert created["vllm:hisparse_host_to_device_bytes"].increments == [32]
assert created["vllm:hisparse_host_cache_usage_perc"].set_values == [0.75]
assert created["vllm:hisparse_pending_page_transfers"].set_values == [3]
+2
View File
@@ -92,6 +92,8 @@ def test_hisparse_worker_get_kv_connector_stats_reads_completed_snapshot(monkeyp
"cache_hits": [12],
"cache_misses": [4],
"host_to_device_bytes": [64],
"host_cache_usage_perc": [],
"pending_page_transfers": [],
}