[Bugfix][HiSparse] Never allocate GPU pages without host backing (#59036)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Lucas Wilkinson
2026-09-30 04:13:29 +00:00
committed by GitHub
co-authored by Matthew Bonanni Claude Opus 5.5
parent e006d761a5
commit 90e13fc757
8 changed files with 252 additions and 85 deletions
+13
View File
@@ -420,6 +420,19 @@ def test_hisparse_rejects_disabled_hybrid_kv_cache_manager(monkeypatch):
)
def test_hisparse_rejects_disabled_full_isl_reservation(monkeypatch):
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
with pytest.raises(ValueError, match="requires --scheduler-reserve-full-isl"):
VllmConfig(
attention_config=AttentionConfig(hisparse_config=HiSparseConfig()),
scheduler_config=SchedulerConfig(
max_model_len=2048,
is_encoder_decoder=False,
scheduler_reserve_full_isl=False,
),
)
def test_hisparse_rejects_non_cuda(monkeypatch):
monkeypatch.setattr(current_platform, "is_cuda", lambda: False)
with pytest.raises(ValueError, match="requires NVIDIA CUDA"):
+44
View File
@@ -205,6 +205,50 @@ def test_hisparse_hma_uses_resolved_gpu_block_size(
assert scheduler_block_size == hash_block_size == gpu_block_size
@pytest.mark.parametrize("extra_blocks,ok", [(1, False), (2, True)])
def test_hisparse_host_pool_must_fit_max_model_len(monkeypatch, extra_blocks, ok):
"""Every page needs a host block, so one max_model_len request must fit."""
monkeypatch.setattr(
hisparse_runtime_module.current_platform, "is_cuda_alike", lambda: True
)
specs = {
"model.layers.0.self_attn": MLAAttentionSpec(
block_size=64,
num_kv_heads=1,
head_size=576,
dtype=torch.bfloat16,
is_index_group_leader=True,
),
"model.layers.0.self_attn.indexer": MLAAttentionSpec(
block_size=64,
num_kv_heads=1,
head_size=128,
dtype=torch.bfloat16,
cache_role=SparseCacheRole.INDEXER,
),
}
group_spec = UniformTypeKVCacheSpecs.from_specs(specs)
assert group_spec is not None
config = SimpleNamespace(
attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()),
model_config=SimpleNamespace(
hf_config=SimpleNamespace(index_topk=128), max_model_len=64 * 4
),
parallel_config=SimpleNamespace(tensor_parallel_size=1, world_size=1),
cache_config=SimpleNamespace(num_gpu_blocks_override=7),
)
host_page = specs["model.layers.0.self_attn"].page_size_bytes
host_budget = (4 + extra_blocks) * host_page
group = KVCacheGroupSpec(list(specs), group_spec)
if ok:
layout = create_hisparse_layout(config, [group], host_budget=host_budget)
assert layout.host_num_blocks == 4 + extra_blocks
else:
with pytest.raises(ValueError, match="increase host_pool_gib"):
create_hisparse_layout(config, [group], host_budget=host_budget)
def test_hisparse_rejects_deepseek_v4():
full_specs = {
"model.layers.0.attn": MLAAttentionSpec(
+80 -44
View File
@@ -782,14 +782,15 @@ def test_hisparse_inflight_host_import_reserves_remaining_gpu_pages():
assert required == 4
@pytest.mark.parametrize(
"full_sequence_must_fit,host_num_blocks,admitted",
[(True, 2, False), (True, 3, True), (False, 2, False), (False, 3, True)],
)
def test_hisparse_async_admission_requires_only_import_destinations(
full_sequence_must_fit, host_num_blocks, admitted, tmp_path
@pytest.mark.parametrize("host_num_blocks,admitted", [(5, False), (6, True)])
def test_hisparse_async_admission_requires_full_prompt_host_pages(
host_num_blocks, admitted, tmp_path
):
"""Imports need host destinations, but do not reserve future prefill pages."""
"""An async load is admitted only if its whole prompt's host pages fit.
The in-flight prefill's host page is not free, so the waiting request's
four prompt pages need six host blocks including the null block.
"""
from .utils import create_scheduler, mock_kv
(tmp_path / "config.json").write_text(
@@ -804,7 +805,7 @@ def test_hisparse_async_admission_requires_only_import_destinations(
manager = make_hisparse_kv_cache_manager(32, host_num_blocks)
scheduler.kv_cache_manager = manager
scheduler.kv_cache_config = manager.kv_cache_config
scheduler.scheduler_reserve_full_isl = full_sequence_must_fit
assert scheduler.scheduler_reserve_full_isl
inflight = make_request(
"inflight", list(range(3 * HISPARSE_BLOCK_SIZE)), HISPARSE_BLOCK_SIZE, sha256
)
@@ -822,6 +823,52 @@ def test_hisparse_async_admission_requires_only_import_destinations(
assert bool(manager.get_blocks(request.request_id).blocks[0]) == admitted
def test_hisparse_admitted_async_loads_can_finish_with_nothing_running(tmp_path):
"""Admitted async loads must not strand each other on host capacity.
Waiting requests keep their blocks and are never preempted, so two imports
that each hold a host page while together needing more than the pool has
left would never be scheduled again. HiSparse relies on full-ISL admission:
a load is admitted only if its whole prompt's host pages fit, so the newest
admitted load can always finish and free host blocks for the others.
"""
from .utils import create_scheduler, mock_kv
(tmp_path / "config.json").write_text(
'{"architectures": ["OPTForCausalLM"], "model_type": "opt"}'
)
scheduler = create_scheduler(
model=str(tmp_path),
skip_tokenizer_init=True,
max_model_len=128,
use_kv_connector=mock_kv(matched_tokens=HISPARSE_BLOCK_SIZE, is_async=True),
)
# One null block plus three usable host blocks.
manager = make_hisparse_kv_cache_manager(32, 4)
scheduler.kv_cache_manager = manager
scheduler.kv_cache_config = manager.kv_cache_config
assert scheduler.scheduler_reserve_full_isl
first = make_request(
"first", list(range(3 * HISPARSE_BLOCK_SIZE)), HISPARSE_BLOCK_SIZE, sha256
)
second = make_request(
"second",
list(range(1000, 1000 + 3 * HISPARSE_BLOCK_SIZE)),
HISPARSE_BLOCK_SIZE,
sha256,
)
for request in (first, second):
scheduler.add_request(request)
scheduler.schedule()
admitted = {r.request_id for r in scheduler._inflight_prefills}
scheduler.finished_recving_kv_req_ids.update(admitted)
num_scheduled_tokens = 0
for _ in range(4):
num_scheduled_tokens += scheduler.schedule().total_num_scheduled_tokens
assert num_scheduled_tokens > 0
@pytest.mark.parametrize(
"evictable,local_tokens,free_blocks,admitted",
[
@@ -845,58 +892,47 @@ def test_hisparse_import_capacity_includes_hits_and_cow(
assert host_pool.get_num_free_blocks() == free_blocks
assert (
coordinator.host_manager.get_num_blocks_to_allocate(
"waiting", 64, [hit], 32, local_tokens, 64
"waiting", 32, [hit], 32, local_tokens, 32
)
<= host_pool.get_num_free_blocks()
) == admitted
@pytest.mark.parametrize("enable_caching", [False, True])
def test_hisparse_host_exhaustion_keeps_gpu_pages_readable(enable_caching):
"""Pages without a host destination must survive GPU cache reclamation."""
manager = make_hisparse_kv_cache_manager(32, 2, enable_caching=enable_caching)
def test_hisparse_host_exhaustion_defers_allocation(enable_caching):
"""Pages are only allocated with host backing, so every one can be reclaimed."""
manager = make_hisparse_kv_cache_manager(32, 5, enable_caching=enable_caching)
donor = make_request("donor", [99] * 16, HISPARSE_BLOCK_SIZE, sha256)
assert manager.allocate_slots(donor, 16) is not None
request = make_request("resident", list(range(64)), HISPARSE_BLOCK_SIZE, sha256)
assert manager.allocate_slots(request, 64) is not None
free_gpu = manager.block_pool.get_num_free_blocks()
assert manager.allocate_slots(request, 64) is None
assert manager.block_pool.get_num_free_blocks() == free_gpu
# The donor's in-flight write-back holds its host block until acked.
manager.free(donor)
coordinator = get_hisparse_coordinator(manager)
assert coordinator.has_pending_reclamation()
assert manager.allocate_slots(request, 64) is None
_publish_hisparse_pages(manager)
assert not coordinator.has_pending_reclamation()
assert manager.allocate_slots(request, 64) is not None
host = coordinator.host_manager
assert host is not None
assert [b.is_null for b in host.req_to_blocks[request.request_id]] == [
False,
True,
True,
True,
]
assert not any(b.is_null for b in host.req_to_blocks[request.request_id])
_publish_hisparse_pages(manager)
for hot in coordinator.hot_managers:
hot.require_hot(request.request_id)
hot.allocate_new_blocks(request.request_id, 64, 64)
coordinator.update_residency(request.request_id)
assert coordinator.request_states[request.request_id].unpinned_pages == {0, 1}
for resident in coordinator.resident_managers:
for block in resident.req_to_blocks[request.request_id][1:]:
assert not block.is_null and block.ref_cnt > 0
assert coordinator.request_states[request.request_id].valid_pages == {0}
pool = coordinator.get_host_block_pool()
assert pool is not None
manager.free(request)
assert pool.get_num_free_blocks() == 1
def test_hisparse_host_cow_takes_priority_over_new_pages():
"""The last host block must hold the private tail, not a fresh page."""
manager = make_hisparse_kv_cache_manager(32, 3)
coordinator = get_hisparse_coordinator(manager)
host = coordinator.host_manager
assert host is not None
source = host.block_pool.get_new_blocks(1)[0]
host.req_to_blocks["cow"] = [source]
host._partial_hit_reqs["cow"] = (0, source)
host.allocate_new_blocks("cow", 32, 32)
private, missing = host.req_to_blocks["cow"]
assert not private.is_null and private is not source
assert missing.is_null
assert host.take_host_cow_copies() == [(source, private)]
assert [b.ref_cnt for b in resident.req_to_blocks[request.request_id]] == [
0,
0,
1,
1,
]
@pytest.mark.parametrize("failed", [False, True])
+78
View File
@@ -223,3 +223,81 @@ def test_hisparse_spill_and_prefix_restore(
assert actual == expected
if with_offloading:
assert _offload_load_bytes() > load_bytes
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="HiSparse requires NVIDIA CUDA"
)
@fork_new_process_for_each_test
def test_hisparse_host_exhaustion_defers_requests(
monkeypatch: pytest.MonkeyPatch,
vllm_runner: type[VllmRunner],
):
"""A full host pool defers requests instead of leaving pages GPU-only."""
capability = current_platform.get_device_capability()
if capability is None or capability.major < 9:
pytest.skip("Sparse MLA requires Hopper or newer")
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
monkeypatch.setenv("VLLM_DEEP_GEMM_WARMUP", "skip")
prompts = [
[1000 + request_idx * 64 + i % 64 for i in range(257)]
for request_idx in range(3)
]
with vllm_runner(
MODEL,
load_format="dummy",
hf_overrides=_shrink_config,
attention_config=AttentionConfig(
hisparse_config=HiSparseConfig(device_buffer_size=512)
),
kv_transfer_config=KVTransferConfig(
kv_connector="HiSparseConnector",
kv_role="kv_both",
kv_connector_extra_config={"host_pool_gib": 1},
),
block_size=64,
max_model_len=320,
max_num_batched_tokens=512,
max_num_seqs=3,
num_gpu_blocks_override=128,
enable_prefix_caching=False,
enable_chunked_prefill=True,
enforce_eager=True,
) as runner:
expected = [runner.generate_greedy([p], max_tokens=16)[0] for p in prompts]
engine = runner.llm.llm_engine.engine_core.engine_core
coordinator = get_hisparse_coordinator(engine.scheduler.kv_cache_manager)
host = coordinator.host_manager
assert host is not None
# Leave room for one full request plus part of another.
spare_blocks = 8
num_pressure = host.block_pool.get_num_free_blocks() - spare_blocks
host.allocate_new_blocks(
"host-pressure",
num_pressure * host.block_size,
num_pressure * host.block_size,
)
original_plan = coordinator.plan_prefix_materialization
original_count = host.get_num_blocks_to_allocate
checked_steps = 0
refusals = 0
def plan(request_id, num_computed_tokens):
nonlocal checked_steps
assert not any(b.is_null for b in host.req_to_blocks[request_id])
checked_steps += 1
original_plan(request_id, num_computed_tokens)
def count(*args, **kwargs):
nonlocal refusals
required = original_count(*args, **kwargs)
refusals += int(required > 0)
return required
monkeypatch.setattr(coordinator, "plan_prefix_materialization", plan)
monkeypatch.setattr(host, "get_num_blocks_to_allocate", count)
actual = runner.generate_greedy(prompts, max_tokens=16)
assert checked_steps > 0
assert refusals > 0
assert actual == expected