[HiSparse] Log steady-state max concurrency and expose host-tier utilization gauges (#58949)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
Robert Shaw
2026-10-05 14:20:32 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 Matthew Bonanni
parent 8542c3eff5
commit 2e3154aa18
12 changed files with 323 additions and 34 deletions
+2 -5
View File
@@ -19,6 +19,7 @@ from vllm.v1.core.single_type_kv_cache_manager import (
SingleTypeKVCacheManager,
)
from vllm.v1.hisparse.types import (
ACTIVE_TAIL_PAGES,
SparseKVOffloadCommand,
SparseKVPageTransfer,
SparseKVRowMirror,
@@ -32,10 +33,6 @@ from vllm.v1.request import Request
if TYPE_CHECKING:
from vllm.v1.core.kv_cache_manager import KVCacheManager
# Sealed pages this many positions behind the block-table tail stay pinned so
# a page written by an in-flight step is never handed out under it.
_ACTIVE_TAIL_PAGES = 2
@dataclass
class _PendingPublication:
@@ -343,7 +340,7 @@ class HiSparseCoordinator:
req_blocks = manager.req_to_blocks.get(request_id)
if (
req_blocks is None
or page_idx >= len(req_blocks) - _ACTIVE_TAIL_PAGES
or page_idx >= len(req_blocks) - ACTIVE_TAIL_PAGES
or req_blocks[page_idx].is_null
):
return None
+32
View File
@@ -14,6 +14,7 @@ from vllm.v1.hisparse.runtime import (
get_hisparse_host_block_stride,
use_shared_hisparse_host_pool,
)
from vllm.v1.hisparse.types import ACTIVE_TAIL_PAGES
from vllm.v1.kv_cache_interface import (
HiSparseHotSpec,
HiSparseResidentSpec,
@@ -373,3 +374,34 @@ def get_hisparse_kv_cache_config(
vllm_config.cache_config.prefix_cache_retention_interval
),
)
def get_hisparse_steady_state_concurrency(
vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
) -> float:
"""Max concurrency at max_model_len once running requests read from host.
A request reading from host pins only its active tail of each resident
group, but admitting one still takes its full in-flight window, so the
bound is all-but-one requests at steady state plus one being admitted.
"""
admission_blocks = 0
steady_blocks = 0
host_blocks = 0
for group in kv_cache_config.kv_cache_groups:
spec = group.kv_cache_spec
required = cdiv(spec.max_memory_usage_bytes(vllm_config), spec.page_size_bytes)
if group.host_resident:
host_blocks += required
continue
admission_blocks += required
if isinstance(spec, HiSparseResidentSpec):
required = min(required, ACTIVE_TAIL_PAGES)
steady_blocks += required
assert kv_cache_config.hisparse_host_num_blocks is not None
num_blocks = kv_cache_config.num_blocks
if num_blocks < admission_blocks:
gpu_concurrency = num_blocks / admission_blocks
else:
gpu_concurrency = 1 + (num_blocks - admission_blocks) / steady_blocks
return min(gpu_concurrency, kv_cache_config.hisparse_host_num_blocks / host_blocks)
+4
View File
@@ -3,6 +3,10 @@
from dataclasses import dataclass
# Sealed pages this many positions behind the block-table tail stay pinned so
# a page written by an in-flight step is never handed out under it.
ACTIVE_TAIL_PAGES = 2
@dataclass(frozen=True)
class SparseKVPageTransfer: