Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Chao Lei <leichao139636@163.com> Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Chao Lei <leichao139636@163.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Chao Lei
Claude Fable 5
Matthew Bonanni
OpenAI Codex
OpenAI Codex
parent
ebe1dec2da
commit
e19a3e172e
@@ -3,6 +3,7 @@
|
||||
import copy
|
||||
import hashlib
|
||||
import importlib
|
||||
import mmap
|
||||
import subprocess
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
@@ -15,6 +16,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
import vllm.v1.core.kv_cache_utils as kv_cache_utils
|
||||
import vllm.v1.hisparse.runtime as hisparse_runtime_module
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
KVTransferConfig,
|
||||
@@ -90,7 +92,13 @@ pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
@pytest.mark.parametrize("gpu_block_size", [32, 64])
|
||||
def test_hisparse_hma_uses_resolved_gpu_block_size(monkeypatch, gpu_block_size):
|
||||
@pytest.mark.parametrize("shared_host_pool", [False, True])
|
||||
def test_hisparse_hma_uses_resolved_gpu_block_size(
|
||||
monkeypatch, gpu_block_size, shared_host_pool
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
hisparse_runtime_module.current_platform, "is_cuda_alike", lambda: True
|
||||
)
|
||||
specs = {
|
||||
"model.layers.0.self_attn": MLAAttentionSpec(
|
||||
block_size=gpu_block_size,
|
||||
@@ -116,7 +124,15 @@ def test_hisparse_hma_uses_resolved_gpu_block_size(monkeypatch, gpu_block_size):
|
||||
hf_config=SimpleNamespace(index_topk=128),
|
||||
max_model_len=gpu_block_size,
|
||||
),
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
|
||||
parallel_config=SimpleNamespace(
|
||||
tensor_parallel_size=2 if shared_host_pool else 1,
|
||||
pipeline_parallel_size=1,
|
||||
prefill_context_parallel_size=1,
|
||||
decode_context_parallel_size=1,
|
||||
world_size=2 if shared_host_pool else 1,
|
||||
distributed_executor_backend="mp",
|
||||
nnodes_within_dp=1,
|
||||
),
|
||||
cache_config=SimpleNamespace(
|
||||
num_gpu_blocks_override=7,
|
||||
prefix_cache_retention_interval=None,
|
||||
@@ -137,6 +153,12 @@ def test_hisparse_hma_uses_resolved_gpu_block_size(monkeypatch, gpu_block_size):
|
||||
assert cache_config.hisparse_host_num_blocks > 7
|
||||
|
||||
host_group, indexer_group, *auxiliary_groups = cache_config.kv_cache_groups
|
||||
assert cache_config.hisparse_shared_host_pool is shared_host_pool
|
||||
host_page = host_group.kv_cache_spec.page_size_bytes
|
||||
alignment = mmap.PAGESIZE if shared_host_pool else 1
|
||||
expected_host_stride = (host_page + alignment - 1) // alignment * alignment
|
||||
assert cache_config.hisparse_host_block_stride == expected_host_stride
|
||||
assert cache_config.hisparse_host_num_blocks == 2**30 // expected_host_stride
|
||||
assert host_group.host_resident
|
||||
assert not any(group.host_resident for group in [indexer_group, *auxiliary_groups])
|
||||
host_layers = set(host_group.layer_names)
|
||||
|
||||
@@ -367,6 +367,42 @@ def test_hisparse_offloads_only_indexer_group():
|
||||
assert [group.group_idx for group in scheduler_config.kv_group_configs] == [1]
|
||||
|
||||
|
||||
def test_hisparse_partial_group_size_survives_scheduler_flattening():
|
||||
"""Worker and scheduler representations must produce identical mmap rows."""
|
||||
layer_specs = {name: _full_attention_spec() for name in ("indexer.0", "indexer.1")}
|
||||
wrapped = UniformTypeKVCacheSpecs.from_specs(layer_specs)
|
||||
assert wrapped is not None
|
||||
source = KVCacheGroupSpec(
|
||||
["source"],
|
||||
_full_attention_spec(),
|
||||
host_resident=True,
|
||||
role=KVCacheGroupRole.HISPARSE_SOURCE,
|
||||
)
|
||||
worker_indexer = KVCacheGroupSpec(
|
||||
list(layer_specs), wrapped, role=KVCacheGroupRole.HISPARSE_INDEXER
|
||||
)
|
||||
scheduler_indexer = KVCacheGroupSpec(
|
||||
list(layer_specs),
|
||||
next(iter(layer_specs.values())),
|
||||
role=KVCacheGroupRole.HISPARSE_INDEXER,
|
||||
)
|
||||
|
||||
def make_config(indexer: KVCacheGroupSpec) -> KVCacheConfig:
|
||||
return KVCacheConfig(
|
||||
num_blocks=4,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[source, indexer],
|
||||
hisparse_host_num_blocks=4,
|
||||
)
|
||||
|
||||
config = _make_vllm_config()
|
||||
worker = build_offloading_config(config, make_config(worker_indexer))
|
||||
scheduler = build_offloading_config(config, make_config(scheduler_indexer))
|
||||
|
||||
assert worker.worker_kv_bytes_per_block == scheduler.worker_kv_bytes_per_block
|
||||
assert worker.worker_kv_bytes_per_block == wrapped.page_size_bytes
|
||||
|
||||
|
||||
def test_zero_blocks_skips_tensor_layout_validation():
|
||||
kv_cache_config = _make_sizing_kv_cache_config(packed=False)
|
||||
kv_cache_config.num_blocks = 0
|
||||
|
||||
@@ -10,11 +10,12 @@ import os
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import MagicMock, call
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.utils.system_utils import get_mp_context
|
||||
from vllm.v1.kv_offload.cpu import shared_offload_region as region_module
|
||||
from vllm.v1.kv_offload.cpu.shared_offload_region import (
|
||||
SharedOffloadRegion,
|
||||
_wait_for_file_size,
|
||||
@@ -192,28 +193,43 @@ def _mp_barrier_construct_and_hold(
|
||||
barrier,
|
||||
fill_value: int,
|
||||
done_queue,
|
||||
replicated: bool = False,
|
||||
) -> None:
|
||||
"""Construct with a real cross-process barrier, write, then hold the
|
||||
mapping (no cleanup) until the parent SIGKILLs this process."""
|
||||
try:
|
||||
populate_calls = 0
|
||||
get_populate_write_fn = region_module._get_populate_write_fn
|
||||
|
||||
def track_population(mmap_obj):
|
||||
nonlocal populate_calls
|
||||
populate_calls += 1
|
||||
return get_populate_write_fn(mmap_obj)
|
||||
|
||||
region_module._get_populate_write_fn = track_population
|
||||
region = SharedOffloadRegion(
|
||||
engine_id=engine_id,
|
||||
num_chunks=2,
|
||||
rank=rank,
|
||||
kv_bytes_per_chunk=num_workers * PAGE_SIZE,
|
||||
rank=0 if replicated else rank,
|
||||
kv_bytes_per_chunk=PAGE_SIZE if replicated else num_workers * PAGE_SIZE,
|
||||
cpu_page_size=PAGE_SIZE,
|
||||
barrier=lambda: barrier.wait(30),
|
||||
populate_only_on_creator=replicated,
|
||||
)
|
||||
# The constructor's barrier precedes the creator's unlink.
|
||||
barrier.wait(30)
|
||||
t = region.create_next_worker_view(PAGE_SIZE)
|
||||
t[:, :] = fill_value
|
||||
if not replicated or rank == 0:
|
||||
t[:, :] = fill_value
|
||||
barrier.wait(30)
|
||||
done_queue.put(
|
||||
{
|
||||
"rank": rank,
|
||||
"error": None,
|
||||
"inode": os.fstat(region.fd).st_ino,
|
||||
"path_exists": os.path.exists(region.mmap_path),
|
||||
"populate_calls": populate_calls,
|
||||
"contents_match": bool((t == (1 if replicated else fill_value)).all()),
|
||||
}
|
||||
)
|
||||
time.sleep(60) # hold the mapping until killed
|
||||
@@ -752,6 +768,26 @@ def test_cleanup_idempotent(iid):
|
||||
r.cleanup() # must be a no-op
|
||||
|
||||
|
||||
def test_cleanup_unregisters_every_pinned_chunk(iid, monkeypatch):
|
||||
"""cleanup() must release every chunk from a split host registration."""
|
||||
r = _make_region(iid)
|
||||
cudart = MagicMock()
|
||||
cudart.cudaHostUnregister.return_value = MagicMock(value=0)
|
||||
monkeypatch.setattr(region_module, "current_platform", MagicMock())
|
||||
region_module.current_platform.is_cuda_alike.return_value = True
|
||||
monkeypatch.setattr(region_module.torch.cuda, "cudart", lambda: cudart)
|
||||
r.pinned_addresses = [0x100000, 0x200000, 0x300000]
|
||||
r.is_pinned = True
|
||||
|
||||
r.cleanup()
|
||||
|
||||
assert cudart.cudaHostUnregister.call_args_list == [
|
||||
call(0x300000),
|
||||
call(0x200000),
|
||||
call(0x100000),
|
||||
]
|
||||
|
||||
|
||||
def test_cleanup_after_create_next_worker_view_releases_mmap(iid):
|
||||
"""cleanup() must close the mmap even after create_next_worker_view was called.
|
||||
create_next_worker_view returns a view that shares storage with _base; both must be
|
||||
@@ -811,11 +847,53 @@ def test_wait_for_file_size_timeout(tmp_path):
|
||||
os.close(fd)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_wait_for_file_size_rejects_unlinked_file(tmp_path):
|
||||
path = tmp_path / "unlinked.mmap"
|
||||
fd = os.open(str(path), os.O_CREAT | os.O_RDWR, 0o600)
|
||||
try:
|
||||
os.unlink(path)
|
||||
with pytest.raises(RuntimeError, match="creator failed"):
|
||||
_wait_for_file_size(fd, PAGE_SIZE, timeout=1.0)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constructor — capacity validation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.skipif(not os.path.isdir("/dev/shm"), reason="requires /dev/shm")
|
||||
def test_creator_memory_check_runs_only_for_creator(iid):
|
||||
checked_sizes: list[int] = []
|
||||
creator = SharedOffloadRegion(
|
||||
engine_id=iid,
|
||||
num_chunks=4,
|
||||
rank=0,
|
||||
kv_bytes_per_chunk=PAGE_SIZE,
|
||||
cpu_page_size=PAGE_SIZE,
|
||||
creator_memory_check=checked_sizes.append,
|
||||
)
|
||||
joiner: SharedOffloadRegion | None = None
|
||||
try:
|
||||
joiner = SharedOffloadRegion(
|
||||
engine_id=iid,
|
||||
num_chunks=4,
|
||||
rank=0,
|
||||
kv_bytes_per_chunk=PAGE_SIZE,
|
||||
cpu_page_size=PAGE_SIZE,
|
||||
creator_memory_check=checked_sizes.append,
|
||||
)
|
||||
assert checked_sizes == [4 * PAGE_SIZE]
|
||||
finally:
|
||||
if joiner is not None:
|
||||
joiner.cleanup()
|
||||
creator.cleanup()
|
||||
_cleanup_file(creator.mmap_path)
|
||||
|
||||
|
||||
def test_insufficient_space_raises_clear_error(monkeypatch):
|
||||
"""A failed creator capacity check must clean up and give a clear error."""
|
||||
import vllm.v1.kv_offload.cpu.shared_offload_region as region
|
||||
@@ -916,7 +994,8 @@ def test_barrier_failure_unlinks_creator_and_raises(iid):
|
||||
assert not os.path.exists(path)
|
||||
|
||||
|
||||
def test_mp_barrier_unlinks_file_and_survives_sigkill(iid):
|
||||
@pytest.mark.parametrize("replicated", [False, True])
|
||||
def test_mp_barrier_unlinks_file_and_survives_sigkill(iid, replicated):
|
||||
"""With a real cross-process barrier every worker maps one shared inode,
|
||||
the name is gone while they run, and SIGKILL leaks nothing."""
|
||||
num_workers = 2
|
||||
@@ -927,7 +1006,7 @@ def test_mp_barrier_unlinks_file_and_survives_sigkill(iid):
|
||||
procs = [
|
||||
ctx.Process(
|
||||
target=_mp_barrier_construct_and_hold,
|
||||
args=(iid, rank, num_workers, barrier, rank + 1, done_queue),
|
||||
args=(iid, rank, num_workers, barrier, rank + 1, done_queue, replicated),
|
||||
)
|
||||
for rank in range(num_workers)
|
||||
]
|
||||
@@ -939,6 +1018,10 @@ def test_mp_barrier_unlinks_file_and_survives_sigkill(iid):
|
||||
assert r["error"] is None, f"rank {r['rank']}: {r['error']}"
|
||||
assert len({r["inode"] for r in results}) == 1, "workers split onto two files"
|
||||
assert not any(r["path_exists"] for r in results)
|
||||
assert all(r["contents_match"] for r in results)
|
||||
assert sum(r["populate_calls"] for r in results) == (
|
||||
1 if replicated else num_workers
|
||||
)
|
||||
assert not os.path.exists(path)
|
||||
finally:
|
||||
for p in procs:
|
||||
@@ -950,23 +1033,36 @@ def test_mp_barrier_unlinks_file_and_survives_sigkill(iid):
|
||||
assert restarted._creator is True, "restart must be able to create anew"
|
||||
|
||||
|
||||
def test_setup_failure_before_barrier_releases_peers(iid, monkeypatch):
|
||||
@pytest.mark.parametrize("failure_phase", ["shm", "memory", "population"])
|
||||
def test_setup_failure_before_barrier_releases_peers(iid, monkeypatch, failure_phase):
|
||||
"""A worker that dies before the rendezvous must still arrive at the
|
||||
barrier, or its peers block in the collective until it times out."""
|
||||
import vllm.v1.kv_offload.cpu.shared_offload_region as region
|
||||
|
||||
monkeypatch.setattr(
|
||||
region,
|
||||
"check_shm_free_space",
|
||||
MagicMock(side_effect=RuntimeError("Insufficient space")),
|
||||
failure = MagicMock(side_effect=ValueError("Insufficient space"))
|
||||
memory_check = failure if failure_phase == "memory" else None
|
||||
if failure_phase == "shm":
|
||||
monkeypatch.setattr(region_module, "check_shm_free_space", failure)
|
||||
elif failure_phase == "population":
|
||||
monkeypatch.setattr(region_module, "_get_populate_write_fn", failure)
|
||||
path = f"/dev/shm/vllm_offload_{iid}.mmap"
|
||||
seen_at_barrier = []
|
||||
barrier = MagicMock(
|
||||
side_effect=lambda: seen_at_barrier.append(os.path.exists(path))
|
||||
)
|
||||
barrier = MagicMock()
|
||||
|
||||
with pytest.raises(RuntimeError, match="Insufficient space"):
|
||||
_make_region(iid, barrier=barrier)
|
||||
with pytest.raises(ValueError, match="Insufficient space"):
|
||||
SharedOffloadRegion(
|
||||
engine_id=iid,
|
||||
num_chunks=1,
|
||||
rank=0,
|
||||
kv_bytes_per_chunk=PAGE_SIZE,
|
||||
cpu_page_size=PAGE_SIZE,
|
||||
barrier=barrier,
|
||||
creator_memory_check=memory_check,
|
||||
populate_only_on_creator=True,
|
||||
)
|
||||
|
||||
barrier.assert_called_once_with()
|
||||
assert not os.path.exists(f"/dev/shm/vllm_offload_{iid}.mmap")
|
||||
assert seen_at_barrier == [False]
|
||||
|
||||
|
||||
def test_mmap_failure_unlinks_creator_before_releasing_peers(iid, monkeypatch):
|
||||
|
||||
@@ -12,6 +12,7 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.v1.hisparse.binding as attn_utils_module
|
||||
from tests.v1.attention.utils import dense_kv_cache_views
|
||||
from vllm.v1.attention.backend import AttentionBackend, AttentionCGSupport, MultipleOf
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
|
||||
@@ -239,6 +240,87 @@ def test_get_kv_sharing_fast_prefill_eligible_layers(monkeypatch: pytest.MonkeyP
|
||||
assert attn_utils.get_kv_sharing_fast_prefill_eligible_layers(vllm_config) == set()
|
||||
|
||||
|
||||
class _FakeSharedHostRegion:
|
||||
def __init__(self) -> None:
|
||||
self.cleanup_calls = 0
|
||||
self.base_tensor = torch.empty(1, dtype=torch.int8)
|
||||
|
||||
def cleanup(self) -> None:
|
||||
self.cleanup_calls += 1
|
||||
|
||||
|
||||
def test_profiling_cleanup_releases_tp_shared_region_once(monkeypatch):
|
||||
"""TP-shared profiling pools must use region-aware chunk cleanup."""
|
||||
region = _FakeSharedHostRegion()
|
||||
runtime = SimpleNamespace(
|
||||
_host_cache=object(),
|
||||
registered_host_pool=region.base_tensor,
|
||||
hot_backing=object(),
|
||||
shared_host_region=region,
|
||||
)
|
||||
forward_context = {
|
||||
"layer": SimpleNamespace(
|
||||
hisparse_cache=SimpleNamespace(runtime=runtime),
|
||||
)
|
||||
}
|
||||
released = []
|
||||
|
||||
def release_pinned_state(runtimes, pinned_host_pools, shared_host_region):
|
||||
released.append((runtimes, pinned_host_pools, shared_host_region))
|
||||
|
||||
monkeypatch.setattr(
|
||||
attn_utils_module,
|
||||
"release_pinned_state",
|
||||
release_pinned_state,
|
||||
)
|
||||
|
||||
attn_utils_module.release_hisparse_profiling_cache(forward_context)
|
||||
|
||||
assert released == [([runtime], [], region)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure_phase", ["allocation", "binding", "buffers"])
|
||||
def test_init_hisparse_rolls_back_shared_region(monkeypatch, failure_phase):
|
||||
"""A failure after mmap allocation must not leak the shared registration."""
|
||||
region = _FakeSharedHostRegion()
|
||||
vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(
|
||||
get_resolved_kv_cache_layout=lambda: KVCacheLayout.BLHNC
|
||||
),
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=1, max_num_batched_tokens=1),
|
||||
)
|
||||
|
||||
def allocate(*args):
|
||||
args[-1].shared_region = region
|
||||
if failure_phase == "allocation":
|
||||
raise RuntimeError("initialization failed")
|
||||
return {}
|
||||
|
||||
def bind(**kwargs):
|
||||
if failure_phase == "binding":
|
||||
raise RuntimeError("initialization failed")
|
||||
return []
|
||||
|
||||
def buffers(*args, **kwargs):
|
||||
raise RuntimeError("initialization failed")
|
||||
|
||||
monkeypatch.setattr(attn_utils_module, "allocate_hisparse_kv_caches", allocate)
|
||||
monkeypatch.setattr(attn_utils_module, "bind_hisparse_kv_caches", bind)
|
||||
monkeypatch.setattr(
|
||||
attn_utils_module, "initialize_hisparse_runtime_buffers", buffers
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="initialization failed"):
|
||||
attn_utils_module.init_hisparse_kv_cache(
|
||||
SimpleNamespace(),
|
||||
torch.device("cpu"),
|
||||
[],
|
||||
vllm_config,
|
||||
{},
|
||||
SimpleNamespace(),
|
||||
)
|
||||
assert region.cleanup_calls == 1
|
||||
|
||||
|
||||
def test_reshape_padded_kv_cache_strides_by_padded_page():
|
||||
num_blocks = 3
|
||||
spec = FullAttentionSpec(
|
||||
|
||||
@@ -60,7 +60,11 @@ def test_mrv2_kv_pool_only_wraps_backing_allocation(monkeypatch, hisparse) -> No
|
||||
)
|
||||
]
|
||||
|
||||
monkeypatch.setattr(hisparse_binding, "HiSparseHostPool", lambda: host_pool)
|
||||
monkeypatch.setattr(
|
||||
hisparse_binding,
|
||||
"HiSparseHostPool",
|
||||
lambda vllm_config, kv_cache_config: host_pool,
|
||||
)
|
||||
monkeypatch.setattr(hisparse_binding, "allocate_hisparse_kv_caches", allocate)
|
||||
monkeypatch.setattr(hisparse_binding, "bind_hisparse_kv_caches", bind_hisparse)
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import mmap
|
||||
from collections import deque
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
@@ -188,6 +189,232 @@ def test_hisparse_submits_layer_mirror_at_replayed_attention_boundary(
|
||||
assert handle.submit_layer_mirror.call_count == expected_calls
|
||||
|
||||
|
||||
def _hisparse_parallel_config(**overrides):
|
||||
values = {
|
||||
"tensor_parallel_size": 2,
|
||||
"pipeline_parallel_size": 1,
|
||||
"prefill_context_parallel_size": 1,
|
||||
"decode_context_parallel_size": 1,
|
||||
"world_size": 2,
|
||||
"distributed_executor_backend": "mp",
|
||||
"nnodes_within_dp": 1,
|
||||
"data_parallel_index": 3,
|
||||
}
|
||||
values.update(overrides)
|
||||
return SimpleNamespace(**values)
|
||||
|
||||
|
||||
def test_hisparse_shares_host_pool_only_for_local_tp(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
hisparse_runtime_module.current_platform, "is_cuda_alike", lambda: True
|
||||
)
|
||||
config = SimpleNamespace(parallel_config=_hisparse_parallel_config())
|
||||
|
||||
assert hisparse_runtime_module.use_shared_hisparse_host_pool(config)
|
||||
|
||||
unsupported = (
|
||||
{"tensor_parallel_size": 1, "world_size": 1},
|
||||
{"pipeline_parallel_size": 2, "world_size": 4},
|
||||
{"prefill_context_parallel_size": 2, "world_size": 4},
|
||||
{"decode_context_parallel_size": 2},
|
||||
{"world_size": 4},
|
||||
{"distributed_executor_backend": "ray"},
|
||||
{"nnodes_within_dp": 2},
|
||||
)
|
||||
for overrides in unsupported:
|
||||
config.parallel_config = _hisparse_parallel_config(**overrides)
|
||||
assert not hisparse_runtime_module.use_shared_hisparse_host_pool(config)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_hisparse_shared_host_pool_uses_one_replicated_mmap(monkeypatch):
|
||||
page = mmap.PAGESIZE
|
||||
tp_group = MagicMock()
|
||||
monkeypatch.setattr(hisparse_runtime_module, "get_tp_group", lambda: tp_group)
|
||||
|
||||
class FakeSharedOffloadRegion:
|
||||
BLOCK_SIZE_ALIGNMENT = page
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.kwargs = kwargs
|
||||
self.total_size_bytes = kwargs["num_chunks"] * kwargs["kv_bytes_per_chunk"]
|
||||
self.base_tensor = torch.empty(self.total_size_bytes, dtype=torch.int8)
|
||||
self.is_pinned = False
|
||||
self.pinned_addresses = []
|
||||
self.view_sizes = []
|
||||
self.offset = 0
|
||||
|
||||
def create_next_canonical_view(self, size):
|
||||
self.view_sizes.append(size)
|
||||
view = torch.as_strided(
|
||||
self.base_tensor,
|
||||
size=(self.kwargs["num_chunks"], size),
|
||||
stride=(self.kwargs["kv_bytes_per_chunk"], 1),
|
||||
storage_offset=self.offset,
|
||||
)
|
||||
self.offset += size
|
||||
return view
|
||||
|
||||
def cleanup(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(
|
||||
hisparse_runtime_module, "SharedOffloadRegion", FakeSharedOffloadRegion
|
||||
)
|
||||
registration_ranges = MagicMock(return_value=((0, page), (page, 4 * page)))
|
||||
monkeypatch.setattr(
|
||||
hisparse_runtime_module,
|
||||
"_hisparse_registration_ranges",
|
||||
registration_ranges,
|
||||
)
|
||||
pinned: list[torch.Tensor] = []
|
||||
|
||||
def pin_tensor(tensor):
|
||||
pinned.append(tensor)
|
||||
|
||||
monkeypatch.setattr(hisparse_runtime_module, "pin_tensor", pin_tensor)
|
||||
config = SimpleNamespace(
|
||||
instance_id="instance",
|
||||
parallel_config=_hisparse_parallel_config(
|
||||
tensor_parallel_size=1,
|
||||
world_size=1,
|
||||
distributed_executor_backend="uni",
|
||||
),
|
||||
)
|
||||
|
||||
pools, private_pools, region = hisparse_runtime_module.allocate_hisparse_host_pools(
|
||||
config,
|
||||
[24, 40],
|
||||
num_blocks=4,
|
||||
host_block_stride=page,
|
||||
use_shared_host_pool=True,
|
||||
)
|
||||
|
||||
assert region is not None
|
||||
assert private_pools == []
|
||||
assert region.kwargs == {
|
||||
"engine_id": "hisparse_instance_dp3",
|
||||
"num_chunks": 1,
|
||||
"rank": 0,
|
||||
"kv_bytes_per_chunk": 4 * page,
|
||||
"cpu_page_size": 64,
|
||||
"barrier": tp_group.barrier,
|
||||
"creator_memory_check": hisparse_runtime_module.check_hisparse_host_memory,
|
||||
"populate_only_on_creator": True,
|
||||
}
|
||||
assert region.view_sizes == [24, 40]
|
||||
assert [pool.shape for pool in pools] == [(24,), (40,)]
|
||||
registration_ranges.assert_called_once_with([24, 40], 4, page)
|
||||
assert [
|
||||
(tensor.data_ptr() - region.base_tensor.data_ptr(), tensor.nbytes)
|
||||
for tensor in pinned
|
||||
] == [(0, page), (page, 3 * page)]
|
||||
assert region.is_pinned
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fail_registration", [None, 1, 2])
|
||||
def test_shared_host_pool_tracks_successful_registrations(
|
||||
monkeypatch, fail_registration
|
||||
):
|
||||
"""Only successful registrations may be unregistered on allocation failure."""
|
||||
page = mmap.PAGESIZE
|
||||
monkeypatch.setattr(hisparse_runtime_module, "get_tp_group", MagicMock())
|
||||
backing = torch.empty(3 * page, dtype=torch.uint8)
|
||||
region = MagicMock(base_tensor=backing, pinned_addresses=[], is_pinned=False)
|
||||
monkeypatch.setattr(
|
||||
hisparse_runtime_module, "SharedOffloadRegion", lambda **kw: region
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
hisparse_runtime_module,
|
||||
"_hisparse_registration_ranges",
|
||||
lambda *args: ((0, page), (page, 3 * page)),
|
||||
)
|
||||
cudart = MagicMock()
|
||||
cudart.cudaHostRegister.side_effect = [
|
||||
SimpleNamespace(value=int(fail_registration == i)) for i in (1, 2)
|
||||
]
|
||||
monkeypatch.setattr(torch.cuda, "cudart", lambda: cudart)
|
||||
config = SimpleNamespace(
|
||||
instance_id="test", parallel_config=SimpleNamespace(data_parallel_index=0)
|
||||
)
|
||||
context = (
|
||||
pytest.raises(RuntimeError, match="cudaHostRegister failed")
|
||||
if fail_registration
|
||||
else nullcontext()
|
||||
)
|
||||
with context:
|
||||
hisparse_runtime_module.allocate_hisparse_host_pools(
|
||||
config, [3 * page], 3, page, use_shared_host_pool=True
|
||||
)
|
||||
successful = 2 if fail_registration is None else fail_registration - 1
|
||||
assert region.pinned_addresses == [
|
||||
backing.data_ptr() + i * page for i in range(successful)
|
||||
]
|
||||
assert region.is_pinned == bool(successful)
|
||||
assert region.cleanup.call_count == int(fail_registration is not None)
|
||||
attempts = 2 if fail_registration is None else fail_registration
|
||||
assert (
|
||||
cudart.cudaHostRegister.call_args_list
|
||||
== [
|
||||
call(backing.data_ptr(), page, 0),
|
||||
call(backing.data_ptr() + page, 2 * page, 0),
|
||||
][:attempts]
|
||||
)
|
||||
|
||||
|
||||
def test_shared_host_pool_registers_layer_spans_in_one_backing(monkeypatch):
|
||||
"""Aliased tensor configs must not count the shared backing once per view."""
|
||||
region = SimpleNamespace(base_tensor=torch.empty(64, dtype=torch.int8))
|
||||
allocate = MagicMock(return_value=([], [], region))
|
||||
monkeypatch.setattr(
|
||||
hisparse_runtime_module, "allocate_hisparse_host_pools", allocate
|
||||
)
|
||||
config = SimpleNamespace(
|
||||
hisparse_shared_host_pool=True,
|
||||
hisparse_host_num_blocks=4,
|
||||
hisparse_host_block_stride=16,
|
||||
kv_cache_tensors=[
|
||||
SimpleNamespace(
|
||||
host_resident=True,
|
||||
layers=["a", "b"],
|
||||
offset=0,
|
||||
layer_stride=12,
|
||||
size=44,
|
||||
),
|
||||
SimpleNamespace(
|
||||
host_resident=True, layers=["c"], offset=24, layer_stride=20, size=44
|
||||
),
|
||||
],
|
||||
)
|
||||
vllm_config = SimpleNamespace()
|
||||
pool = hisparse_runtime_module.HiSparseHostPool(vllm_config, config)
|
||||
backing = pool.allocate(44)
|
||||
allocate.assert_called_once_with(
|
||||
vllm_config, [12, 12, 20], 4, 16, use_shared_host_pool=True
|
||||
)
|
||||
assert backing.numel() == 44
|
||||
assert backing.data_ptr() == region.base_tensor.data_ptr()
|
||||
assert pool.registered is region.base_tensor
|
||||
|
||||
|
||||
def test_hisparse_registration_chunks_end_between_host_blocks():
|
||||
"""Registration seams must not bisect a DMA-addressable host block."""
|
||||
page = mmap.PAGESIZE
|
||||
ranges = hisparse_runtime_module._hisparse_registration_ranges(
|
||||
tensor_sizes=[4 * 3 * page, 4 * 5 * page],
|
||||
num_blocks=4,
|
||||
host_block_stride=8 * page,
|
||||
max_chunk_bytes=10 * page,
|
||||
)
|
||||
|
||||
assert ranges == (
|
||||
(0, 9 * page),
|
||||
(9 * page, 17 * page),
|
||||
(17 * page, 27 * page),
|
||||
(27 * page, 32 * page),
|
||||
)
|
||||
|
||||
|
||||
def test_copy_cpu_kv_cache_logical_blocks_ignores_storage_padding():
|
||||
backing = torch.full((10, 2, 3), -1, dtype=torch.float32)
|
||||
cache = backing[1:9]
|
||||
@@ -931,6 +1158,32 @@ def test_hisparse_cache_handles_join_index_groups_during_construction(monkeypatc
|
||||
assert streams == []
|
||||
|
||||
|
||||
def test_hisparse_worker_shutdown_releases_pinned_state(monkeypatch):
|
||||
worker = object.__new__(HiSparseConnectorWorker)
|
||||
worker._initialized = True
|
||||
worker._slot_mapping_staging = None
|
||||
worker.dma_stream = None
|
||||
worker.cache_handles = []
|
||||
worker.pinned_host_pools = []
|
||||
worker.shared_host_region = object()
|
||||
released = False
|
||||
|
||||
def release_pinned_state(runtimes, pinned_host_pools, shared_host_region):
|
||||
nonlocal released
|
||||
assert runtimes == []
|
||||
assert pinned_host_pools == []
|
||||
assert shared_host_region is worker.shared_host_region
|
||||
released = True
|
||||
|
||||
monkeypatch.setattr(
|
||||
hisparse_worker_module, "release_pinned_state", release_pinned_state
|
||||
)
|
||||
|
||||
worker.shutdown()
|
||||
|
||||
assert released
|
||||
|
||||
|
||||
class _TestReplaySSMMixer(MambaMixer2):
|
||||
def __init__(self):
|
||||
torch.nn.Module.__init__(self)
|
||||
|
||||
Reference in New Issue
Block a user