[HiSparse] Log steady-state max concurrency and expose host-tier utilization gauges (#58949)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Matthew Bonanni
parent
8542c3eff5
commit
2e3154aa18
@@ -19,6 +19,7 @@ from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
SingleTypeKVCacheManager,
|
||||
)
|
||||
from vllm.v1.hisparse.types import (
|
||||
ACTIVE_TAIL_PAGES,
|
||||
SparseKVOffloadCommand,
|
||||
SparseKVPageTransfer,
|
||||
SparseKVRowMirror,
|
||||
@@ -32,10 +33,6 @@ from vllm.v1.request import Request
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheManager
|
||||
|
||||
# Sealed pages this many positions behind the block-table tail stay pinned so
|
||||
# a page written by an in-flight step is never handed out under it.
|
||||
_ACTIVE_TAIL_PAGES = 2
|
||||
|
||||
|
||||
@dataclass
|
||||
class _PendingPublication:
|
||||
@@ -343,7 +340,7 @@ class HiSparseCoordinator:
|
||||
req_blocks = manager.req_to_blocks.get(request_id)
|
||||
if (
|
||||
req_blocks is None
|
||||
or page_idx >= len(req_blocks) - _ACTIVE_TAIL_PAGES
|
||||
or page_idx >= len(req_blocks) - ACTIVE_TAIL_PAGES
|
||||
or req_blocks[page_idx].is_null
|
||||
):
|
||||
return None
|
||||
|
||||
@@ -14,6 +14,7 @@ from vllm.v1.hisparse.runtime import (
|
||||
get_hisparse_host_block_stride,
|
||||
use_shared_hisparse_host_pool,
|
||||
)
|
||||
from vllm.v1.hisparse.types import ACTIVE_TAIL_PAGES
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
HiSparseHotSpec,
|
||||
HiSparseResidentSpec,
|
||||
@@ -373,3 +374,34 @@ def get_hisparse_kv_cache_config(
|
||||
vllm_config.cache_config.prefix_cache_retention_interval
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def get_hisparse_steady_state_concurrency(
|
||||
vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
|
||||
) -> float:
|
||||
"""Max concurrency at max_model_len once running requests read from host.
|
||||
|
||||
A request reading from host pins only its active tail of each resident
|
||||
group, but admitting one still takes its full in-flight window, so the
|
||||
bound is all-but-one requests at steady state plus one being admitted.
|
||||
"""
|
||||
admission_blocks = 0
|
||||
steady_blocks = 0
|
||||
host_blocks = 0
|
||||
for group in kv_cache_config.kv_cache_groups:
|
||||
spec = group.kv_cache_spec
|
||||
required = cdiv(spec.max_memory_usage_bytes(vllm_config), spec.page_size_bytes)
|
||||
if group.host_resident:
|
||||
host_blocks += required
|
||||
continue
|
||||
admission_blocks += required
|
||||
if isinstance(spec, HiSparseResidentSpec):
|
||||
required = min(required, ACTIVE_TAIL_PAGES)
|
||||
steady_blocks += required
|
||||
assert kv_cache_config.hisparse_host_num_blocks is not None
|
||||
num_blocks = kv_cache_config.num_blocks
|
||||
if num_blocks < admission_blocks:
|
||||
gpu_concurrency = num_blocks / admission_blocks
|
||||
else:
|
||||
gpu_concurrency = 1 + (num_blocks - admission_blocks) / steady_blocks
|
||||
return min(gpu_concurrency, kv_cache_config.hisparse_host_num_blocks / host_blocks)
|
||||
|
||||
@@ -3,6 +3,10 @@
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
# Sealed pages this many positions behind the block-table tail stay pinned so
|
||||
# a page written by an in-flight step is never handed out under it.
|
||||
ACTIVE_TAIL_PAGES = 2
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SparseKVPageTransfer:
|
||||
|
||||
Reference in New Issue
Block a user