[Perf][HiSparse] Reuse pinned host pool after graph profiling (#61668)
Signed-off-by: Itay Etelis <92247226+Etelis@users.noreply.github.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Matthew Bonanni
Claude Opus 5.5
parent
105605b8e9
commit
49f944be3e
@@ -28,6 +28,7 @@ from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
|
||||
from vllm.v1.hisparse.layout import HISPARSE_HOT_SUFFIX
|
||||
from vllm.v1.hisparse.runtime import (
|
||||
HiSparseCacheHandle,
|
||||
drop_hisparse_host_pool,
|
||||
release_pinned_state,
|
||||
update_hisparse_residency,
|
||||
)
|
||||
@@ -254,6 +255,7 @@ class HiSparseConnectorWorker:
|
||||
pinned_host_pools,
|
||||
shared_host_region,
|
||||
)
|
||||
drop_hisparse_host_pool(self.vllm_config)
|
||||
raise
|
||||
|
||||
def initialize(
|
||||
@@ -972,4 +974,5 @@ class HiSparseConnectorWorker:
|
||||
self.pinned_host_pools,
|
||||
self.shared_host_region,
|
||||
)
|
||||
drop_hisparse_host_pool(self.vllm_config)
|
||||
self._initialized = False
|
||||
|
||||
+11
-29
@@ -17,6 +17,8 @@ from vllm.v1.hisparse.layout import (
|
||||
from vllm.v1.hisparse.runtime import (
|
||||
HiSparseCacheHandle,
|
||||
HiSparseHostPool,
|
||||
drop_hisparse_host_pool,
|
||||
get_hisparse_host_pool,
|
||||
initialize_hisparse_runtime_buffers,
|
||||
release_pinned_state,
|
||||
)
|
||||
@@ -126,7 +128,7 @@ def init_hisparse_kv_cache(
|
||||
block_tables: "BlockTables",
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""Allocate and bind HiSparse caches within the caller's allocation context."""
|
||||
host_pool = HiSparseHostPool(vllm_config, kv_cache_config)
|
||||
host_pool = get_hisparse_host_pool(vllm_config, kv_cache_config)
|
||||
try:
|
||||
kv_caches = allocate_hisparse_kv_caches(
|
||||
kv_cache_config,
|
||||
@@ -150,6 +152,9 @@ def init_hisparse_kv_cache(
|
||||
except Exception:
|
||||
if host_pool.shared_region is not None:
|
||||
host_pool.shared_region.cleanup()
|
||||
elif host_pool.registered is not None:
|
||||
release_pinned_state([], [host_pool.registered])
|
||||
drop_hisparse_host_pool(vllm_config)
|
||||
raise
|
||||
|
||||
|
||||
@@ -163,37 +168,14 @@ def _get_hisparse_cache(
|
||||
|
||||
|
||||
def release_hisparse_profiling_cache(forward_context: dict[str, Any]) -> None:
|
||||
cache_handles = [
|
||||
cache
|
||||
for layer in forward_context.values()
|
||||
if (cache := getattr(layer, "hisparse_cache", None)) is not None
|
||||
]
|
||||
"""Detach the profiling caches but keep the host pool for the real KV cache."""
|
||||
runtimes = {
|
||||
id(cache.runtime): cache.runtime
|
||||
for cache in cache_handles
|
||||
if hasattr(cache.runtime, "_host_cache")
|
||||
for layer in forward_context.values()
|
||||
if (cache := getattr(layer, "hisparse_cache", None)) is not None
|
||||
and hasattr(cache.runtime, "_host_cache")
|
||||
}
|
||||
if not runtimes:
|
||||
return
|
||||
|
||||
shared_regions = {
|
||||
id(runtime.shared_host_region): runtime.shared_host_region
|
||||
for runtime in runtimes.values()
|
||||
if runtime.shared_host_region is not None
|
||||
}
|
||||
assert len(shared_regions) <= 1
|
||||
shared_region = next(iter(shared_regions.values()), None)
|
||||
registered_pools = (
|
||||
[]
|
||||
if shared_region is not None
|
||||
else list(
|
||||
{
|
||||
runtime.registered_host_pool.data_ptr(): runtime.registered_host_pool
|
||||
for runtime in runtimes.values()
|
||||
}.values()
|
||||
)
|
||||
)
|
||||
release_pinned_state(list(runtimes.values()), registered_pools, shared_region)
|
||||
release_pinned_state(list(runtimes.values()), [])
|
||||
|
||||
|
||||
def bind_hisparse_kv_caches(
|
||||
|
||||
@@ -223,7 +223,9 @@ class HiSparseHostPool:
|
||||
self.shared_region: SharedOffloadRegion | None = None
|
||||
|
||||
def allocate(self, size: int) -> torch.Tensor:
|
||||
assert self.backing is None, "HiSparse host pool is already allocated."
|
||||
if self.backing is not None:
|
||||
assert self.backing.numel() == size, "HiSparse host pool size changed."
|
||||
return self.backing
|
||||
config = self.kv_cache_config
|
||||
if config.hisparse_shared_host_pool:
|
||||
tensor_sizes = [
|
||||
@@ -253,6 +255,23 @@ class HiSparseHostPool:
|
||||
return self.backing
|
||||
|
||||
|
||||
_host_pools: dict[str, HiSparseHostPool] = {}
|
||||
|
||||
|
||||
def get_hisparse_host_pool(
|
||||
vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
|
||||
) -> HiSparseHostPool:
|
||||
"""Return the engine's host pool, which outlives the profiling KV cache."""
|
||||
instance_id = vllm_config.instance_id
|
||||
if instance_id not in _host_pools:
|
||||
_host_pools[instance_id] = HiSparseHostPool(vllm_config, kv_cache_config)
|
||||
return _host_pools[instance_id]
|
||||
|
||||
|
||||
def drop_hisparse_host_pool(vllm_config: VllmConfig) -> None:
|
||||
_host_pools.pop(vllm_config.instance_id, None)
|
||||
|
||||
|
||||
def use_shared_hisparse_host_pool(vllm_config: VllmConfig) -> bool:
|
||||
"""Whether replicated MLA host KV can share one local mmap."""
|
||||
parallel = vllm_config.parallel_config
|
||||
|
||||
Reference in New Issue
Block a user