[HiSparse] Log steady-state max concurrency and expose host-tier utilization gauges (#58949)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
Robert Shaw
2026-10-05 14:20:32 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 Matthew Bonanni
parent 8542c3eff5
commit 2e3154aa18
12 changed files with 323 additions and 34 deletions
+43
View File
@@ -60,6 +60,7 @@ from vllm.v1.core.kv_cache_utils import (
from vllm.v1.hisparse.layout import (
create_hisparse_layout,
get_hisparse_kv_cache_groups,
get_hisparse_steady_state_concurrency,
)
from vllm.v1.kv_cache_interface import (
ChunkedLocalAttentionSpec,
@@ -198,6 +199,48 @@ def test_hisparse_hma_uses_resolved_gpu_block_size(
assert scheduler_block_size == hash_block_size == gpu_block_size
def test_hisparse_steady_state_concurrency_excludes_spilled_resident_pages():
"""Running requests pin only their resident tail; the newest still needs its
full in-flight window to be admitted."""
block_size = 16
config = SimpleNamespace(
model_config=SimpleNamespace(max_model_len=10 * block_size),
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
max_in_flight_tokens=10 * block_size,
)
attn_spec = FullAttentionSpec(
block_size=block_size, num_kv_heads=1, head_size=64, dtype=torch.float16
)
page_size = attn_spec.page_size_bytes
kv_cache_config = KVCacheConfig(
num_blocks=64,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(["source"], attn_spec, host_resident=True),
KVCacheGroupSpec(["indexer"], attn_spec),
KVCacheGroupSpec(
["resident"],
HiSparseResidentSpec(block_size=block_size, page_size=page_size),
),
KVCacheGroupSpec(
["hot"],
HiSparseHotSpec(
block_size=block_size, page_size=page_size, blocks_per_request=4
),
),
],
hisparse_host_num_blocks=100,
)
# Admission takes indexer 10 + resident 10 + hot 4 = 24 blocks and steady
# state 16, so 3 fit (2 x 16 + 24 <= 64) but not 4 (3 x 16 + 24 > 64).
assert get_hisparse_steady_state_concurrency(config, kv_cache_config) == 3.5
# The worst-case bound charges all 10 resident pages.
assert get_max_concurrency_for_kv_cache_config(
config, kv_cache_config
) == pytest.approx(64 / 24)
@pytest.mark.parametrize("extra_blocks,ok", [(1, False), (2, True)])
def test_hisparse_host_pool_must_fit_max_model_len(monkeypatch, extra_blocks, ok):
"""Every page needs a host block, so one max_model_len request must fit."""