[HiSparse] Log steady-state max concurrency and expose host-tier utilization gauges (#58949)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Matthew Bonanni
parent
8542c3eff5
commit
2e3154aa18
@@ -60,6 +60,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
from vllm.v1.hisparse.layout import (
|
||||
create_hisparse_layout,
|
||||
get_hisparse_kv_cache_groups,
|
||||
get_hisparse_steady_state_concurrency,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
@@ -198,6 +199,48 @@ def test_hisparse_hma_uses_resolved_gpu_block_size(
|
||||
assert scheduler_block_size == hash_block_size == gpu_block_size
|
||||
|
||||
|
||||
def test_hisparse_steady_state_concurrency_excludes_spilled_resident_pages():
|
||||
"""Running requests pin only their resident tail; the newest still needs its
|
||||
full in-flight window to be admitted."""
|
||||
block_size = 16
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(max_model_len=10 * block_size),
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
|
||||
max_in_flight_tokens=10 * block_size,
|
||||
)
|
||||
attn_spec = FullAttentionSpec(
|
||||
block_size=block_size, num_kv_heads=1, head_size=64, dtype=torch.float16
|
||||
)
|
||||
page_size = attn_spec.page_size_bytes
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=64,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["source"], attn_spec, host_resident=True),
|
||||
KVCacheGroupSpec(["indexer"], attn_spec),
|
||||
KVCacheGroupSpec(
|
||||
["resident"],
|
||||
HiSparseResidentSpec(block_size=block_size, page_size=page_size),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["hot"],
|
||||
HiSparseHotSpec(
|
||||
block_size=block_size, page_size=page_size, blocks_per_request=4
|
||||
),
|
||||
),
|
||||
],
|
||||
hisparse_host_num_blocks=100,
|
||||
)
|
||||
|
||||
# Admission takes indexer 10 + resident 10 + hot 4 = 24 blocks and steady
|
||||
# state 16, so 3 fit (2 x 16 + 24 <= 64) but not 4 (3 x 16 + 24 > 64).
|
||||
assert get_hisparse_steady_state_concurrency(config, kv_cache_config) == 3.5
|
||||
# The worst-case bound charges all 10 resident pages.
|
||||
assert get_max_concurrency_for_kv_cache_config(
|
||||
config, kv_cache_config
|
||||
) == pytest.approx(64 / 24)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("extra_blocks,ok", [(1, False), (2, True)])
|
||||
def test_hisparse_host_pool_must_fit_max_model_len(monkeypatch, extra_blocks, ok):
|
||||
"""Every page needs a host block, so one max_model_len request must fit."""
|
||||
|
||||
@@ -64,6 +64,42 @@ def test_hisparse_requires_block_outermost_device_layout():
|
||||
assert HiSparseConnector.get_required_kvcache_layout(MagicMock()) == "BLHNC"
|
||||
|
||||
|
||||
def test_scheduler_stats_report_host_pool_usage():
|
||||
"""The scheduler-side stats sample the coordinator's host pool level."""
|
||||
from tests.v1.core.test_prefix_caching import (
|
||||
HISPARSE_BLOCK_SIZE,
|
||||
make_hisparse_kv_cache_manager,
|
||||
make_request,
|
||||
sha256,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_utils import init_none_hash
|
||||
|
||||
init_none_hash(sha256)
|
||||
|
||||
manager = make_hisparse_kv_cache_manager(32, host_num_blocks=8)
|
||||
scheduler = HiSparseConnectorScheduler(async_speculative=False)
|
||||
scheduler.bind_coordinator(get_hisparse_coordinator(manager))
|
||||
coordinator = scheduler.coordinator
|
||||
|
||||
stats = scheduler.get_kv_connector_stats()
|
||||
assert stats.data["host_cache_usage_perc"] == [0.0]
|
||||
assert stats.data["pending_page_transfers"] == [0]
|
||||
|
||||
request = make_request(
|
||||
"request", list(range(4 * HISPARSE_BLOCK_SIZE)), HISPARSE_BLOCK_SIZE, sha256
|
||||
)
|
||||
assert manager.allocate_slots(request, num_new_tokens=64) is not None
|
||||
host_blocks = coordinator.host_manager.req_to_blocks[request.request_id]
|
||||
num_used = sum(not block.is_null for block in host_blocks)
|
||||
|
||||
# The pool reserves the null block, so 8 configured blocks yield 7 usable.
|
||||
expected = pytest.approx(num_used / 7)
|
||||
assert 0 < num_used <= 7
|
||||
for _ in range(2):
|
||||
stats = scheduler.get_kv_connector_stats()
|
||||
assert stats.data["host_cache_usage_perc"] == [expected]
|
||||
|
||||
|
||||
def test_no_forward_enqueues_deferred_hisparse_transfers():
|
||||
"""A zero-token step must still enqueue deferred post-forward transfers."""
|
||||
connector = object.__new__(ActiveKVConnector)
|
||||
|
||||
@@ -26,6 +26,28 @@ def test_record_snapshot_and_reduce():
|
||||
assert reduced["HiSparse host-to-device bytes"] == 64
|
||||
|
||||
|
||||
def test_record_host_usage_reports_latest_level():
|
||||
stats = HiSparseKVConnectorStats()
|
||||
stats.record_host_usage(usage=0.4, pending_page_transfers=2)
|
||||
stats.record_host_usage(usage=0.8, pending_page_transfers=0)
|
||||
|
||||
reduced = stats.reduce()
|
||||
assert reduced["HiSparse host KV cache usage %"] == 80.0
|
||||
|
||||
|
||||
def test_aggregate_keeps_latest_host_usage_level():
|
||||
first = HiSparseKVConnectorStats()
|
||||
first.record_host_usage(usage=0.4, pending_page_transfers=2)
|
||||
second = HiSparseKVConnectorStats()
|
||||
second.record_host_usage(usage=0.8, pending_page_transfers=0)
|
||||
second.record_host_usage(usage=0.6, pending_page_transfers=1)
|
||||
|
||||
first.aggregate(second)
|
||||
|
||||
assert first.data["host_cache_usage_perc"] == [0.4, 0.6]
|
||||
assert first.data["pending_page_transfers"] == [2, 1]
|
||||
|
||||
|
||||
def test_aggregate_extends_snapshot_deltas():
|
||||
first = HiSparseKVConnectorStats()
|
||||
first.record_snapshot(hits=7, misses=3, host_to_device_bytes=48)
|
||||
@@ -34,11 +56,9 @@ def test_aggregate_extends_snapshot_deltas():
|
||||
|
||||
first.aggregate(second)
|
||||
|
||||
assert first.data == {
|
||||
"cache_hits": [7, 5],
|
||||
"cache_misses": [3, 1],
|
||||
"host_to_device_bytes": [48, 16],
|
||||
}
|
||||
assert first.data["cache_hits"] == [7, 5]
|
||||
assert first.data["cache_misses"] == [3, 1]
|
||||
assert first.data["host_to_device_bytes"] == [48, 16]
|
||||
|
||||
|
||||
def test_aggregate_skips_empty_stats():
|
||||
@@ -60,3 +80,54 @@ def test_build_kv_connector_stats_round_trip():
|
||||
assert rebuilt is not None
|
||||
assert isinstance(rebuilt, HiSparseKVConnectorStats)
|
||||
assert rebuilt.reduce() == stats.reduce()
|
||||
|
||||
|
||||
def test_prom_metrics_observe_host_usage_gauges():
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
from prometheus_client import Counter, Gauge, Histogram
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import (
|
||||
HiSparsePromMetrics,
|
||||
)
|
||||
|
||||
class _FakeMetric:
|
||||
def __init__(self, **kwargs: Any):
|
||||
self.kwargs = kwargs
|
||||
self.increments: list[int | float] = []
|
||||
self.set_values: list[int | float] = []
|
||||
|
||||
def labels(self, *labelvalues: object) -> "_FakeMetric":
|
||||
return self
|
||||
|
||||
def inc(self, value: int | float) -> None:
|
||||
self.increments.append(value)
|
||||
|
||||
def set(self, value: int | float) -> None:
|
||||
self.set_values.append(value)
|
||||
|
||||
created: dict[str, _FakeMetric] = {}
|
||||
|
||||
class _NamedFake(_FakeMetric):
|
||||
def __init__(self, **kwargs: Any):
|
||||
super().__init__(**kwargs)
|
||||
created[kwargs["name"]] = self
|
||||
|
||||
prom = HiSparsePromMetrics(
|
||||
vllm_config=SimpleNamespace(kv_transfer_config=None),
|
||||
metric_types={Gauge: _NamedFake, Counter: _NamedFake, Histogram: _NamedFake},
|
||||
labelnames=["model_name"],
|
||||
per_engine_labelvalues={0: ["model"]},
|
||||
)
|
||||
|
||||
stats = HiSparseKVConnectorStats()
|
||||
stats.record_snapshot(hits=3, misses=2, host_to_device_bytes=32)
|
||||
stats.record_host_usage(usage=0.5, pending_page_transfers=1)
|
||||
stats.record_host_usage(usage=0.75, pending_page_transfers=3)
|
||||
prom.observe(stats.to_dict())
|
||||
|
||||
assert created["vllm:hisparse_cache_hits"].increments == [3]
|
||||
assert created["vllm:hisparse_host_to_device_bytes"].increments == [32]
|
||||
assert created["vllm:hisparse_host_cache_usage_perc"].set_values == [0.75]
|
||||
assert created["vllm:hisparse_pending_page_transfers"].set_values == [3]
|
||||
|
||||
@@ -92,6 +92,8 @@ def test_hisparse_worker_get_kv_connector_stats_reads_completed_snapshot(monkeyp
|
||||
"cache_hits": [12],
|
||||
"cache_misses": [4],
|
||||
"host_to_device_bytes": [64],
|
||||
"host_cache_usage_perc": [],
|
||||
"pending_page_transfers": [],
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user