[Perf][HiSparse] Reuse pinned host pool after graph profiling (#61668)
Signed-off-by: Itay Etelis <92247226+Etelis@users.noreply.github.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Matthew Bonanni
Claude Opus 5.5
parent
105605b8e9
commit
49f944be3e
@@ -9,12 +9,14 @@ never addressed by the logical view.
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.v1.hisparse.binding as attn_utils_module
|
||||
import vllm.v1.hisparse.runtime as hisparse_runtime_module
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
@@ -403,49 +405,63 @@ class _FakeSharedHostRegion:
|
||||
self.cleanup_calls += 1
|
||||
|
||||
|
||||
def test_profiling_cleanup_releases_tp_shared_region_once(monkeypatch):
|
||||
"""TP-shared profiling pools must use region-aware chunk cleanup."""
|
||||
region = _FakeSharedHostRegion()
|
||||
def test_profiling_cleanup_keeps_host_pool_for_real_kv_cache(monkeypatch):
|
||||
"""The real KV cache must reuse the profiling host pool, not pin it again."""
|
||||
pinned = torch.empty(8, dtype=torch.int8)
|
||||
allocate_pinned = MagicMock(return_value=(pinned, pinned))
|
||||
monkeypatch.setattr(
|
||||
hisparse_runtime_module, "allocate_pinned_host_pool", allocate_pinned
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
hisparse_runtime_module, "check_hisparse_host_memory", lambda size: None
|
||||
)
|
||||
vllm_config = SimpleNamespace(instance_id="profiling-reuse")
|
||||
kv_cache_config = SimpleNamespace(hisparse_shared_host_pool=False)
|
||||
pool = attn_utils_module.get_hisparse_host_pool(vllm_config, kv_cache_config)
|
||||
backing = pool.allocate(8)
|
||||
runtime = SimpleNamespace(
|
||||
_host_cache=object(),
|
||||
registered_host_pool=region.base_tensor,
|
||||
hot_backing=object(),
|
||||
shared_host_region=region,
|
||||
_host_cache=backing, registered_host_pool=pinned, hot_backing=object()
|
||||
)
|
||||
forward_context = {
|
||||
"layer": SimpleNamespace(
|
||||
hisparse_cache=SimpleNamespace(runtime=runtime),
|
||||
)
|
||||
"layer": SimpleNamespace(hisparse_cache=SimpleNamespace(runtime=runtime))
|
||||
}
|
||||
released = []
|
||||
|
||||
def release_pinned_state(runtimes, pinned_host_pools, shared_host_region):
|
||||
released.append((runtimes, pinned_host_pools, shared_host_region))
|
||||
|
||||
monkeypatch.setattr(
|
||||
attn_utils_module,
|
||||
"release_pinned_state",
|
||||
release_pinned_state,
|
||||
)
|
||||
|
||||
attn_utils_module.release_hisparse_profiling_cache(forward_context)
|
||||
|
||||
assert released == [([runtime], [], region)]
|
||||
assert not hasattr(runtime, "_host_cache")
|
||||
reused = attn_utils_module.get_hisparse_host_pool(vllm_config, kv_cache_config)
|
||||
assert reused is pool
|
||||
assert reused.allocate(8) is backing
|
||||
allocate_pinned.assert_called_once_with(8)
|
||||
hisparse_runtime_module.drop_hisparse_host_pool(vllm_config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure_phase", ["allocation", "binding", "buffers"])
|
||||
def test_init_hisparse_rolls_back_shared_region(monkeypatch, failure_phase):
|
||||
"""A failure after mmap allocation must not leak the shared registration."""
|
||||
@pytest.mark.parametrize("shared", [False, True])
|
||||
def test_init_hisparse_rolls_back_host_pool(monkeypatch, failure_phase, shared):
|
||||
"""Failed initialization must release shared or private registrations."""
|
||||
region = _FakeSharedHostRegion()
|
||||
registered = torch.empty(4096, dtype=torch.int8)
|
||||
cudart = MagicMock()
|
||||
cudart.cudaHostUnregister.return_value.value = 0
|
||||
monkeypatch.setattr(torch.accelerator, "synchronize", lambda: None)
|
||||
monkeypatch.setattr(torch.cuda, "cudart", lambda: cudart)
|
||||
vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(
|
||||
get_resolved_kv_cache_layout=lambda: KVCacheLayout.BLHNC
|
||||
),
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=1),
|
||||
instance_id="rollback",
|
||||
)
|
||||
pools = []
|
||||
|
||||
def allocate(*args):
|
||||
args[-1].shared_region = region
|
||||
pools.append(args[-1])
|
||||
if shared:
|
||||
args[-1].shared_region = region
|
||||
else:
|
||||
args[-1].registered = registered
|
||||
args[-1].backing = registered[:8]
|
||||
if failure_phase == "allocation":
|
||||
raise RuntimeError("initialization failed")
|
||||
return {}
|
||||
@@ -472,7 +488,13 @@ def test_init_hisparse_rolls_back_shared_region(monkeypatch, failure_phase):
|
||||
{},
|
||||
SimpleNamespace(),
|
||||
)
|
||||
assert region.cleanup_calls == 1
|
||||
assert region.cleanup_calls == int(shared)
|
||||
if shared:
|
||||
cudart.cudaHostUnregister.assert_not_called()
|
||||
else:
|
||||
cudart.cudaHostUnregister.assert_called_once_with(registered.data_ptr())
|
||||
assert attn_utils_module.get_hisparse_host_pool(vllm_config, None) is not pools[0]
|
||||
hisparse_runtime_module.drop_hisparse_host_pool(vllm_config)
|
||||
|
||||
|
||||
def test_reshape_padded_kv_cache_strides_by_padded_page():
|
||||
|
||||
@@ -62,7 +62,7 @@ def test_mrv2_kv_pool_only_wraps_backing_allocation(monkeypatch, hisparse) -> No
|
||||
|
||||
monkeypatch.setattr(
|
||||
hisparse_binding,
|
||||
"HiSparseHostPool",
|
||||
"get_hisparse_host_pool",
|
||||
lambda vllm_config, kv_cache_config: host_pool,
|
||||
)
|
||||
monkeypatch.setattr(hisparse_binding, "allocate_hisparse_kv_caches", allocate)
|
||||
|
||||
@@ -1257,6 +1257,8 @@ def test_hisparse_worker_shutdown_releases_pinned_state(monkeypatch):
|
||||
worker.cache_handles = []
|
||||
worker.pinned_host_pools = []
|
||||
worker.shared_host_region = object()
|
||||
worker.vllm_config = SimpleNamespace(instance_id="shutdown")
|
||||
pool = hisparse_runtime_module.get_hisparse_host_pool(worker.vllm_config, None)
|
||||
released = False
|
||||
|
||||
def release_pinned_state(runtimes, pinned_host_pools, shared_host_region):
|
||||
@@ -1273,6 +1275,9 @@ def test_hisparse_worker_shutdown_releases_pinned_state(monkeypatch):
|
||||
worker.shutdown()
|
||||
|
||||
assert released
|
||||
new_pool = hisparse_runtime_module.get_hisparse_host_pool(worker.vllm_config, None)
|
||||
assert new_pool is not pool
|
||||
hisparse_runtime_module.drop_hisparse_host_pool(worker.vllm_config)
|
||||
|
||||
|
||||
class _TestReplaySSMMixer(MambaMixer2):
|
||||
|
||||
Reference in New Issue
Block a user