[Bugfix][HiSparse] Never allocate GPU pages without host backing (#59036)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Lucas Wilkinson
2026-09-30 04:13:29 +00:00
committed by GitHub
co-authored by Matthew Bonanni Claude Opus 5.5
parent e006d761a5
commit 90e13fc757
8 changed files with 252 additions and 85 deletions
+13
View File
@@ -420,6 +420,19 @@ def test_hisparse_rejects_disabled_hybrid_kv_cache_manager(monkeypatch):
)
def test_hisparse_rejects_disabled_full_isl_reservation(monkeypatch):
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
with pytest.raises(ValueError, match="requires --scheduler-reserve-full-isl"):
VllmConfig(
attention_config=AttentionConfig(hisparse_config=HiSparseConfig()),
scheduler_config=SchedulerConfig(
max_model_len=2048,
is_encoder_decoder=False,
scheduler_reserve_full_isl=False,
),
)
def test_hisparse_rejects_non_cuda(monkeypatch):
monkeypatch.setattr(current_platform, "is_cuda", lambda: False)
with pytest.raises(ValueError, match="requires NVIDIA CUDA"):
+44
View File
@@ -205,6 +205,50 @@ def test_hisparse_hma_uses_resolved_gpu_block_size(
assert scheduler_block_size == hash_block_size == gpu_block_size
@pytest.mark.parametrize("extra_blocks,ok", [(1, False), (2, True)])
def test_hisparse_host_pool_must_fit_max_model_len(monkeypatch, extra_blocks, ok):
"""Every page needs a host block, so one max_model_len request must fit."""
monkeypatch.setattr(
hisparse_runtime_module.current_platform, "is_cuda_alike", lambda: True
)
specs = {
"model.layers.0.self_attn": MLAAttentionSpec(
block_size=64,
num_kv_heads=1,
head_size=576,
dtype=torch.bfloat16,
is_index_group_leader=True,
),
"model.layers.0.self_attn.indexer": MLAAttentionSpec(
block_size=64,
num_kv_heads=1,
head_size=128,
dtype=torch.bfloat16,
cache_role=SparseCacheRole.INDEXER,
),
}
group_spec = UniformTypeKVCacheSpecs.from_specs(specs)
assert group_spec is not None
config = SimpleNamespace(
attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()),
model_config=SimpleNamespace(
hf_config=SimpleNamespace(index_topk=128), max_model_len=64 * 4
),
parallel_config=SimpleNamespace(tensor_parallel_size=1, world_size=1),
cache_config=SimpleNamespace(num_gpu_blocks_override=7),
)
host_page = specs["model.layers.0.self_attn"].page_size_bytes
host_budget = (4 + extra_blocks) * host_page
group = KVCacheGroupSpec(list(specs), group_spec)
if ok:
layout = create_hisparse_layout(config, [group], host_budget=host_budget)
assert layout.host_num_blocks == 4 + extra_blocks
else:
with pytest.raises(ValueError, match="increase host_pool_gib"):
create_hisparse_layout(config, [group], host_budget=host_budget)
def test_hisparse_rejects_deepseek_v4():
full_specs = {
"model.layers.0.attn": MLAAttentionSpec(
+80 -44
View File
@@ -782,14 +782,15 @@ def test_hisparse_inflight_host_import_reserves_remaining_gpu_pages():
assert required == 4
@pytest.mark.parametrize(
"full_sequence_must_fit,host_num_blocks,admitted",
[(True, 2, False), (True, 3, True), (False, 2, False), (False, 3, True)],
)
def test_hisparse_async_admission_requires_only_import_destinations(
full_sequence_must_fit, host_num_blocks, admitted, tmp_path
@pytest.mark.parametrize("host_num_blocks,admitted", [(5, False), (6, True)])
def test_hisparse_async_admission_requires_full_prompt_host_pages(
host_num_blocks, admitted, tmp_path
):
"""Imports need host destinations, but do not reserve future prefill pages."""
"""An async load is admitted only if its whole prompt's host pages fit.
The in-flight prefill's host page is not free, so the waiting request's
four prompt pages need six host blocks including the null block.
"""
from .utils import create_scheduler, mock_kv
(tmp_path / "config.json").write_text(
@@ -804,7 +805,7 @@ def test_hisparse_async_admission_requires_only_import_destinations(
manager = make_hisparse_kv_cache_manager(32, host_num_blocks)
scheduler.kv_cache_manager = manager
scheduler.kv_cache_config = manager.kv_cache_config
scheduler.scheduler_reserve_full_isl = full_sequence_must_fit
assert scheduler.scheduler_reserve_full_isl
inflight = make_request(
"inflight", list(range(3 * HISPARSE_BLOCK_SIZE)), HISPARSE_BLOCK_SIZE, sha256
)
@@ -822,6 +823,52 @@ def test_hisparse_async_admission_requires_only_import_destinations(
assert bool(manager.get_blocks(request.request_id).blocks[0]) == admitted
def test_hisparse_admitted_async_loads_can_finish_with_nothing_running(tmp_path):
"""Admitted async loads must not strand each other on host capacity.
Waiting requests keep their blocks and are never preempted, so two imports
that each hold a host page while together needing more than the pool has
left would never be scheduled again. HiSparse relies on full-ISL admission:
a load is admitted only if its whole prompt's host pages fit, so the newest
admitted load can always finish and free host blocks for the others.
"""
from .utils import create_scheduler, mock_kv
(tmp_path / "config.json").write_text(
'{"architectures": ["OPTForCausalLM"], "model_type": "opt"}'
)
scheduler = create_scheduler(
model=str(tmp_path),
skip_tokenizer_init=True,
max_model_len=128,
use_kv_connector=mock_kv(matched_tokens=HISPARSE_BLOCK_SIZE, is_async=True),
)
# One null block plus three usable host blocks.
manager = make_hisparse_kv_cache_manager(32, 4)
scheduler.kv_cache_manager = manager
scheduler.kv_cache_config = manager.kv_cache_config
assert scheduler.scheduler_reserve_full_isl
first = make_request(
"first", list(range(3 * HISPARSE_BLOCK_SIZE)), HISPARSE_BLOCK_SIZE, sha256
)
second = make_request(
"second",
list(range(1000, 1000 + 3 * HISPARSE_BLOCK_SIZE)),
HISPARSE_BLOCK_SIZE,
sha256,
)
for request in (first, second):
scheduler.add_request(request)
scheduler.schedule()
admitted = {r.request_id for r in scheduler._inflight_prefills}
scheduler.finished_recving_kv_req_ids.update(admitted)
num_scheduled_tokens = 0
for _ in range(4):
num_scheduled_tokens += scheduler.schedule().total_num_scheduled_tokens
assert num_scheduled_tokens > 0
@pytest.mark.parametrize(
"evictable,local_tokens,free_blocks,admitted",
[
@@ -845,58 +892,47 @@ def test_hisparse_import_capacity_includes_hits_and_cow(
assert host_pool.get_num_free_blocks() == free_blocks
assert (
coordinator.host_manager.get_num_blocks_to_allocate(
"waiting", 64, [hit], 32, local_tokens, 64
"waiting", 32, [hit], 32, local_tokens, 32
)
<= host_pool.get_num_free_blocks()
) == admitted
@pytest.mark.parametrize("enable_caching", [False, True])
def test_hisparse_host_exhaustion_keeps_gpu_pages_readable(enable_caching):
"""Pages without a host destination must survive GPU cache reclamation."""
manager = make_hisparse_kv_cache_manager(32, 2, enable_caching=enable_caching)
def test_hisparse_host_exhaustion_defers_allocation(enable_caching):
"""Pages are only allocated with host backing, so every one can be reclaimed."""
manager = make_hisparse_kv_cache_manager(32, 5, enable_caching=enable_caching)
donor = make_request("donor", [99] * 16, HISPARSE_BLOCK_SIZE, sha256)
assert manager.allocate_slots(donor, 16) is not None
request = make_request("resident", list(range(64)), HISPARSE_BLOCK_SIZE, sha256)
assert manager.allocate_slots(request, 64) is not None
free_gpu = manager.block_pool.get_num_free_blocks()
assert manager.allocate_slots(request, 64) is None
assert manager.block_pool.get_num_free_blocks() == free_gpu
# The donor's in-flight write-back holds its host block until acked.
manager.free(donor)
coordinator = get_hisparse_coordinator(manager)
assert coordinator.has_pending_reclamation()
assert manager.allocate_slots(request, 64) is None
_publish_hisparse_pages(manager)
assert not coordinator.has_pending_reclamation()
assert manager.allocate_slots(request, 64) is not None
host = coordinator.host_manager
assert host is not None
assert [b.is_null for b in host.req_to_blocks[request.request_id]] == [
False,
True,
True,
True,
]
assert not any(b.is_null for b in host.req_to_blocks[request.request_id])
_publish_hisparse_pages(manager)
for hot in coordinator.hot_managers:
hot.require_hot(request.request_id)
hot.allocate_new_blocks(request.request_id, 64, 64)
coordinator.update_residency(request.request_id)
assert coordinator.request_states[request.request_id].unpinned_pages == {0, 1}
for resident in coordinator.resident_managers:
for block in resident.req_to_blocks[request.request_id][1:]:
assert not block.is_null and block.ref_cnt > 0
assert coordinator.request_states[request.request_id].valid_pages == {0}
pool = coordinator.get_host_block_pool()
assert pool is not None
manager.free(request)
assert pool.get_num_free_blocks() == 1
def test_hisparse_host_cow_takes_priority_over_new_pages():
"""The last host block must hold the private tail, not a fresh page."""
manager = make_hisparse_kv_cache_manager(32, 3)
coordinator = get_hisparse_coordinator(manager)
host = coordinator.host_manager
assert host is not None
source = host.block_pool.get_new_blocks(1)[0]
host.req_to_blocks["cow"] = [source]
host._partial_hit_reqs["cow"] = (0, source)
host.allocate_new_blocks("cow", 32, 32)
private, missing = host.req_to_blocks["cow"]
assert not private.is_null and private is not source
assert missing.is_null
assert host.take_host_cow_copies() == [(source, private)]
assert [b.ref_cnt for b in resident.req_to_blocks[request.request_id]] == [
0,
0,
1,
1,
]
@pytest.mark.parametrize("failed", [False, True])
+78
View File
@@ -223,3 +223,81 @@ def test_hisparse_spill_and_prefix_restore(
assert actual == expected
if with_offloading:
assert _offload_load_bytes() > load_bytes
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="HiSparse requires NVIDIA CUDA"
)
@fork_new_process_for_each_test
def test_hisparse_host_exhaustion_defers_requests(
monkeypatch: pytest.MonkeyPatch,
vllm_runner: type[VllmRunner],
):
"""A full host pool defers requests instead of leaving pages GPU-only."""
capability = current_platform.get_device_capability()
if capability is None or capability.major < 9:
pytest.skip("Sparse MLA requires Hopper or newer")
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
monkeypatch.setenv("VLLM_DEEP_GEMM_WARMUP", "skip")
prompts = [
[1000 + request_idx * 64 + i % 64 for i in range(257)]
for request_idx in range(3)
]
with vllm_runner(
MODEL,
load_format="dummy",
hf_overrides=_shrink_config,
attention_config=AttentionConfig(
hisparse_config=HiSparseConfig(device_buffer_size=512)
),
kv_transfer_config=KVTransferConfig(
kv_connector="HiSparseConnector",
kv_role="kv_both",
kv_connector_extra_config={"host_pool_gib": 1},
),
block_size=64,
max_model_len=320,
max_num_batched_tokens=512,
max_num_seqs=3,
num_gpu_blocks_override=128,
enable_prefix_caching=False,
enable_chunked_prefill=True,
enforce_eager=True,
) as runner:
expected = [runner.generate_greedy([p], max_tokens=16)[0] for p in prompts]
engine = runner.llm.llm_engine.engine_core.engine_core
coordinator = get_hisparse_coordinator(engine.scheduler.kv_cache_manager)
host = coordinator.host_manager
assert host is not None
# Leave room for one full request plus part of another.
spare_blocks = 8
num_pressure = host.block_pool.get_num_free_blocks() - spare_blocks
host.allocate_new_blocks(
"host-pressure",
num_pressure * host.block_size,
num_pressure * host.block_size,
)
original_plan = coordinator.plan_prefix_materialization
original_count = host.get_num_blocks_to_allocate
checked_steps = 0
refusals = 0
def plan(request_id, num_computed_tokens):
nonlocal checked_steps
assert not any(b.is_null for b in host.req_to_blocks[request_id])
checked_steps += 1
original_plan(request_id, num_computed_tokens)
def count(*args, **kwargs):
nonlocal refusals
required = original_count(*args, **kwargs)
refusals += int(required > 0)
return required
monkeypatch.setattr(coordinator, "plan_prefix_materialization", plan)
monkeypatch.setattr(host, "get_num_blocks_to_allocate", count)
actual = runner.generate_greedy(prompts, max_tokens=16)
assert checked_steps > 0
assert refusals > 0
assert actual == expected
+8
View File
@@ -1822,6 +1822,14 @@ class VllmConfig:
raise ValueError(
"HiSparse does not support decode context parallelism."
)
if not self.scheduler_config.scheduler_reserve_full_isl:
# Without it, async loads admitted against free host blocks can
# each wait on host pages the others hold, and waiting requests
# are never preempted to free them.
raise ValueError(
"HiSparse requires --scheduler-reserve-full-isl; remove "
"--no-scheduler-reserve-full-isl."
)
if self.model_config is not None and not hasattr(
self.model_config.hf_config, "index_topk"
):
+15 -38
View File
@@ -2223,8 +2223,8 @@ class SinkFullAttentionManager(FullAttentionManager):
class HiSparseSourceManager(FullAttentionManager):
"""Host-tier manager with a private pool; publishes hashes once durable.
Host capacity is best effort: a page that cannot get a host block keeps a
null host entry, so its GPU copy is never written back and stays pinned.
Every page gets a host block: allocation fails when the host pool cannot
back it, so each GPU-resident page can be written back and later reclaimed.
"""
coordinator: "HiSparseCoordinator | None" = None
@@ -2273,44 +2273,21 @@ class HiSparseSourceManager(FullAttentionManager):
num_tokens_main_model: int,
apply_admission_cap: bool = False,
) -> int:
if (
total_computed_tokens > num_local_computed_tokens
or self._has_partial_local_hit(
new_computed_blocks, num_local_computed_tokens
)
):
# External loads need real destinations; future GPU-computed pages
# remain best effort. Use the same admission sentinel as Mamba.
required = super().get_num_blocks_to_allocate(
request_id,
total_computed_tokens,
new_computed_blocks,
total_computed_tokens,
num_local_computed_tokens,
total_computed_tokens,
)
if required > self.block_pool.get_num_free_blocks():
assert self.coordinator is not None
assert self.coordinator.gpu_pool is not None
return self.coordinator.gpu_pool.num_gpu_blocks + 1
return 0
def allocate_new_blocks(
self, request_id: str, num_tokens: int, num_tokens_main_model: int
) -> list[KVCacheBlock]:
req_blocks = self.req_to_blocks[request_id]
num_required = cdiv(num_tokens, self.block_size)
num_free = self.block_pool.get_num_free_blocks()
if request_id in self._partial_hit_reqs:
num_free -= 1
fit_blocks = min(num_required, len(req_blocks) + max(0, num_free))
new_blocks = super().allocate_new_blocks(
required = super().get_num_blocks_to_allocate(
request_id,
fit_blocks * self.block_size,
min(num_tokens_main_model, fit_blocks * self.block_size),
num_tokens,
new_computed_blocks,
total_computed_tokens,
num_local_computed_tokens,
num_tokens_main_model,
)
req_blocks.extend([self._null_block] * (num_required - len(req_blocks)))
return new_blocks
# Host blocks come from a private pool, so they never count against
# the GPU pool. Use the same admission sentinel as Mamba instead.
if required > self.block_pool.get_num_free_blocks():
assert self.coordinator is not None
assert self.coordinator.gpu_pool is not None
return self.coordinator.gpu_pool.num_gpu_blocks + 1
return 0
def allocate_external_computed_blocks(
self,
+6 -3
View File
@@ -660,10 +660,13 @@ class HiSparseCoordinator:
return command
def has_pending_reclamation(self) -> bool:
"""Whether an in-flight spill will free GPU blocks on completion."""
"""Whether an in-flight spill will free GPU or host blocks on completion."""
return any(
pending.page[0] in self.request_states
and self._can_read_from_host(pending.page[0])
pending.host_block.ref_cnt == 1
or (
pending.page[0] in self.request_states
and self._can_read_from_host(pending.page[0])
)
for pending in self.pending_spills.values()
)
+8
View File
@@ -235,6 +235,14 @@ def create_hisparse_layout(
host_num_blocks = host_budget // host_block_stride
if host_num_blocks <= 0:
raise ValueError("HiSparse has no allocatable host blocks.")
# Every computed page needs a host block, so one request at max_model_len
# must fit alongside the pool's null block and a copy-on-write tail.
min_host_blocks = cdiv(vllm_config.model_config.max_model_len, gpu_block_size) + 2
if host_num_blocks < min_host_blocks:
raise ValueError(
f"HiSparse host pool has {host_num_blocks} blocks but max_model_len "
f"needs {min_host_blocks}; increase host_pool_gib."
)
return HiSparseLayout(
source_group=source_group,