[Bugfix][HiSparse] Never allocate GPU pages without host backing (#59036)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Matthew Bonanni
Claude Opus 5.5
parent
e006d761a5
commit
90e13fc757
@@ -420,6 +420,19 @@ def test_hisparse_rejects_disabled_hybrid_kv_cache_manager(monkeypatch):
|
||||
)
|
||||
|
||||
|
||||
def test_hisparse_rejects_disabled_full_isl_reservation(monkeypatch):
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||
with pytest.raises(ValueError, match="requires --scheduler-reserve-full-isl"):
|
||||
VllmConfig(
|
||||
attention_config=AttentionConfig(hisparse_config=HiSparseConfig()),
|
||||
scheduler_config=SchedulerConfig(
|
||||
max_model_len=2048,
|
||||
is_encoder_decoder=False,
|
||||
scheduler_reserve_full_isl=False,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_hisparse_rejects_non_cuda(monkeypatch):
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: False)
|
||||
with pytest.raises(ValueError, match="requires NVIDIA CUDA"):
|
||||
|
||||
@@ -205,6 +205,50 @@ def test_hisparse_hma_uses_resolved_gpu_block_size(
|
||||
assert scheduler_block_size == hash_block_size == gpu_block_size
|
||||
|
||||
|
||||
@pytest.mark.parametrize("extra_blocks,ok", [(1, False), (2, True)])
|
||||
def test_hisparse_host_pool_must_fit_max_model_len(monkeypatch, extra_blocks, ok):
|
||||
"""Every page needs a host block, so one max_model_len request must fit."""
|
||||
monkeypatch.setattr(
|
||||
hisparse_runtime_module.current_platform, "is_cuda_alike", lambda: True
|
||||
)
|
||||
specs = {
|
||||
"model.layers.0.self_attn": MLAAttentionSpec(
|
||||
block_size=64,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
is_index_group_leader=True,
|
||||
),
|
||||
"model.layers.0.self_attn.indexer": MLAAttentionSpec(
|
||||
block_size=64,
|
||||
num_kv_heads=1,
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
cache_role=SparseCacheRole.INDEXER,
|
||||
),
|
||||
}
|
||||
group_spec = UniformTypeKVCacheSpecs.from_specs(specs)
|
||||
assert group_spec is not None
|
||||
config = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()),
|
||||
model_config=SimpleNamespace(
|
||||
hf_config=SimpleNamespace(index_topk=128), max_model_len=64 * 4
|
||||
),
|
||||
parallel_config=SimpleNamespace(tensor_parallel_size=1, world_size=1),
|
||||
cache_config=SimpleNamespace(num_gpu_blocks_override=7),
|
||||
)
|
||||
host_page = specs["model.layers.0.self_attn"].page_size_bytes
|
||||
host_budget = (4 + extra_blocks) * host_page
|
||||
group = KVCacheGroupSpec(list(specs), group_spec)
|
||||
|
||||
if ok:
|
||||
layout = create_hisparse_layout(config, [group], host_budget=host_budget)
|
||||
assert layout.host_num_blocks == 4 + extra_blocks
|
||||
else:
|
||||
with pytest.raises(ValueError, match="increase host_pool_gib"):
|
||||
create_hisparse_layout(config, [group], host_budget=host_budget)
|
||||
|
||||
|
||||
def test_hisparse_rejects_deepseek_v4():
|
||||
full_specs = {
|
||||
"model.layers.0.attn": MLAAttentionSpec(
|
||||
|
||||
@@ -782,14 +782,15 @@ def test_hisparse_inflight_host_import_reserves_remaining_gpu_pages():
|
||||
assert required == 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"full_sequence_must_fit,host_num_blocks,admitted",
|
||||
[(True, 2, False), (True, 3, True), (False, 2, False), (False, 3, True)],
|
||||
)
|
||||
def test_hisparse_async_admission_requires_only_import_destinations(
|
||||
full_sequence_must_fit, host_num_blocks, admitted, tmp_path
|
||||
@pytest.mark.parametrize("host_num_blocks,admitted", [(5, False), (6, True)])
|
||||
def test_hisparse_async_admission_requires_full_prompt_host_pages(
|
||||
host_num_blocks, admitted, tmp_path
|
||||
):
|
||||
"""Imports need host destinations, but do not reserve future prefill pages."""
|
||||
"""An async load is admitted only if its whole prompt's host pages fit.
|
||||
|
||||
The in-flight prefill's host page is not free, so the waiting request's
|
||||
four prompt pages need six host blocks including the null block.
|
||||
"""
|
||||
from .utils import create_scheduler, mock_kv
|
||||
|
||||
(tmp_path / "config.json").write_text(
|
||||
@@ -804,7 +805,7 @@ def test_hisparse_async_admission_requires_only_import_destinations(
|
||||
manager = make_hisparse_kv_cache_manager(32, host_num_blocks)
|
||||
scheduler.kv_cache_manager = manager
|
||||
scheduler.kv_cache_config = manager.kv_cache_config
|
||||
scheduler.scheduler_reserve_full_isl = full_sequence_must_fit
|
||||
assert scheduler.scheduler_reserve_full_isl
|
||||
inflight = make_request(
|
||||
"inflight", list(range(3 * HISPARSE_BLOCK_SIZE)), HISPARSE_BLOCK_SIZE, sha256
|
||||
)
|
||||
@@ -822,6 +823,52 @@ def test_hisparse_async_admission_requires_only_import_destinations(
|
||||
assert bool(manager.get_blocks(request.request_id).blocks[0]) == admitted
|
||||
|
||||
|
||||
def test_hisparse_admitted_async_loads_can_finish_with_nothing_running(tmp_path):
|
||||
"""Admitted async loads must not strand each other on host capacity.
|
||||
|
||||
Waiting requests keep their blocks and are never preempted, so two imports
|
||||
that each hold a host page while together needing more than the pool has
|
||||
left would never be scheduled again. HiSparse relies on full-ISL admission:
|
||||
a load is admitted only if its whole prompt's host pages fit, so the newest
|
||||
admitted load can always finish and free host blocks for the others.
|
||||
"""
|
||||
from .utils import create_scheduler, mock_kv
|
||||
|
||||
(tmp_path / "config.json").write_text(
|
||||
'{"architectures": ["OPTForCausalLM"], "model_type": "opt"}'
|
||||
)
|
||||
scheduler = create_scheduler(
|
||||
model=str(tmp_path),
|
||||
skip_tokenizer_init=True,
|
||||
max_model_len=128,
|
||||
use_kv_connector=mock_kv(matched_tokens=HISPARSE_BLOCK_SIZE, is_async=True),
|
||||
)
|
||||
# One null block plus three usable host blocks.
|
||||
manager = make_hisparse_kv_cache_manager(32, 4)
|
||||
scheduler.kv_cache_manager = manager
|
||||
scheduler.kv_cache_config = manager.kv_cache_config
|
||||
assert scheduler.scheduler_reserve_full_isl
|
||||
first = make_request(
|
||||
"first", list(range(3 * HISPARSE_BLOCK_SIZE)), HISPARSE_BLOCK_SIZE, sha256
|
||||
)
|
||||
second = make_request(
|
||||
"second",
|
||||
list(range(1000, 1000 + 3 * HISPARSE_BLOCK_SIZE)),
|
||||
HISPARSE_BLOCK_SIZE,
|
||||
sha256,
|
||||
)
|
||||
for request in (first, second):
|
||||
scheduler.add_request(request)
|
||||
scheduler.schedule()
|
||||
admitted = {r.request_id for r in scheduler._inflight_prefills}
|
||||
|
||||
scheduler.finished_recving_kv_req_ids.update(admitted)
|
||||
num_scheduled_tokens = 0
|
||||
for _ in range(4):
|
||||
num_scheduled_tokens += scheduler.schedule().total_num_scheduled_tokens
|
||||
assert num_scheduled_tokens > 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"evictable,local_tokens,free_blocks,admitted",
|
||||
[
|
||||
@@ -845,58 +892,47 @@ def test_hisparse_import_capacity_includes_hits_and_cow(
|
||||
assert host_pool.get_num_free_blocks() == free_blocks
|
||||
assert (
|
||||
coordinator.host_manager.get_num_blocks_to_allocate(
|
||||
"waiting", 64, [hit], 32, local_tokens, 64
|
||||
"waiting", 32, [hit], 32, local_tokens, 32
|
||||
)
|
||||
<= host_pool.get_num_free_blocks()
|
||||
) == admitted
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enable_caching", [False, True])
|
||||
def test_hisparse_host_exhaustion_keeps_gpu_pages_readable(enable_caching):
|
||||
"""Pages without a host destination must survive GPU cache reclamation."""
|
||||
manager = make_hisparse_kv_cache_manager(32, 2, enable_caching=enable_caching)
|
||||
def test_hisparse_host_exhaustion_defers_allocation(enable_caching):
|
||||
"""Pages are only allocated with host backing, so every one can be reclaimed."""
|
||||
manager = make_hisparse_kv_cache_manager(32, 5, enable_caching=enable_caching)
|
||||
donor = make_request("donor", [99] * 16, HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert manager.allocate_slots(donor, 16) is not None
|
||||
request = make_request("resident", list(range(64)), HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert manager.allocate_slots(request, 64) is not None
|
||||
free_gpu = manager.block_pool.get_num_free_blocks()
|
||||
assert manager.allocate_slots(request, 64) is None
|
||||
assert manager.block_pool.get_num_free_blocks() == free_gpu
|
||||
|
||||
# The donor's in-flight write-back holds its host block until acked.
|
||||
manager.free(donor)
|
||||
coordinator = get_hisparse_coordinator(manager)
|
||||
assert coordinator.has_pending_reclamation()
|
||||
assert manager.allocate_slots(request, 64) is None
|
||||
_publish_hisparse_pages(manager)
|
||||
assert not coordinator.has_pending_reclamation()
|
||||
assert manager.allocate_slots(request, 64) is not None
|
||||
host = coordinator.host_manager
|
||||
assert host is not None
|
||||
assert [b.is_null for b in host.req_to_blocks[request.request_id]] == [
|
||||
False,
|
||||
True,
|
||||
True,
|
||||
True,
|
||||
]
|
||||
assert not any(b.is_null for b in host.req_to_blocks[request.request_id])
|
||||
_publish_hisparse_pages(manager)
|
||||
for hot in coordinator.hot_managers:
|
||||
hot.require_hot(request.request_id)
|
||||
hot.allocate_new_blocks(request.request_id, 64, 64)
|
||||
coordinator.update_residency(request.request_id)
|
||||
assert coordinator.request_states[request.request_id].unpinned_pages == {0, 1}
|
||||
for resident in coordinator.resident_managers:
|
||||
for block in resident.req_to_blocks[request.request_id][1:]:
|
||||
assert not block.is_null and block.ref_cnt > 0
|
||||
assert coordinator.request_states[request.request_id].valid_pages == {0}
|
||||
pool = coordinator.get_host_block_pool()
|
||||
assert pool is not None
|
||||
manager.free(request)
|
||||
assert pool.get_num_free_blocks() == 1
|
||||
|
||||
|
||||
def test_hisparse_host_cow_takes_priority_over_new_pages():
|
||||
"""The last host block must hold the private tail, not a fresh page."""
|
||||
manager = make_hisparse_kv_cache_manager(32, 3)
|
||||
coordinator = get_hisparse_coordinator(manager)
|
||||
host = coordinator.host_manager
|
||||
assert host is not None
|
||||
source = host.block_pool.get_new_blocks(1)[0]
|
||||
host.req_to_blocks["cow"] = [source]
|
||||
host._partial_hit_reqs["cow"] = (0, source)
|
||||
|
||||
host.allocate_new_blocks("cow", 32, 32)
|
||||
|
||||
private, missing = host.req_to_blocks["cow"]
|
||||
assert not private.is_null and private is not source
|
||||
assert missing.is_null
|
||||
assert host.take_host_cow_copies() == [(source, private)]
|
||||
assert [b.ref_cnt for b in resident.req_to_blocks[request.request_id]] == [
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failed", [False, True])
|
||||
|
||||
@@ -223,3 +223,81 @@ def test_hisparse_spill_and_prefix_restore(
|
||||
assert actual == expected
|
||||
if with_offloading:
|
||||
assert _offload_load_bytes() > load_bytes
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="HiSparse requires NVIDIA CUDA"
|
||||
)
|
||||
@fork_new_process_for_each_test
|
||||
def test_hisparse_host_exhaustion_defers_requests(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
vllm_runner: type[VllmRunner],
|
||||
):
|
||||
"""A full host pool defers requests instead of leaving pages GPU-only."""
|
||||
capability = current_platform.get_device_capability()
|
||||
if capability is None or capability.major < 9:
|
||||
pytest.skip("Sparse MLA requires Hopper or newer")
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
monkeypatch.setenv("VLLM_DEEP_GEMM_WARMUP", "skip")
|
||||
prompts = [
|
||||
[1000 + request_idx * 64 + i % 64 for i in range(257)]
|
||||
for request_idx in range(3)
|
||||
]
|
||||
with vllm_runner(
|
||||
MODEL,
|
||||
load_format="dummy",
|
||||
hf_overrides=_shrink_config,
|
||||
attention_config=AttentionConfig(
|
||||
hisparse_config=HiSparseConfig(device_buffer_size=512)
|
||||
),
|
||||
kv_transfer_config=KVTransferConfig(
|
||||
kv_connector="HiSparseConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config={"host_pool_gib": 1},
|
||||
),
|
||||
block_size=64,
|
||||
max_model_len=320,
|
||||
max_num_batched_tokens=512,
|
||||
max_num_seqs=3,
|
||||
num_gpu_blocks_override=128,
|
||||
enable_prefix_caching=False,
|
||||
enable_chunked_prefill=True,
|
||||
enforce_eager=True,
|
||||
) as runner:
|
||||
expected = [runner.generate_greedy([p], max_tokens=16)[0] for p in prompts]
|
||||
engine = runner.llm.llm_engine.engine_core.engine_core
|
||||
coordinator = get_hisparse_coordinator(engine.scheduler.kv_cache_manager)
|
||||
host = coordinator.host_manager
|
||||
assert host is not None
|
||||
# Leave room for one full request plus part of another.
|
||||
spare_blocks = 8
|
||||
num_pressure = host.block_pool.get_num_free_blocks() - spare_blocks
|
||||
host.allocate_new_blocks(
|
||||
"host-pressure",
|
||||
num_pressure * host.block_size,
|
||||
num_pressure * host.block_size,
|
||||
)
|
||||
original_plan = coordinator.plan_prefix_materialization
|
||||
original_count = host.get_num_blocks_to_allocate
|
||||
checked_steps = 0
|
||||
refusals = 0
|
||||
|
||||
def plan(request_id, num_computed_tokens):
|
||||
nonlocal checked_steps
|
||||
assert not any(b.is_null for b in host.req_to_blocks[request_id])
|
||||
checked_steps += 1
|
||||
original_plan(request_id, num_computed_tokens)
|
||||
|
||||
def count(*args, **kwargs):
|
||||
nonlocal refusals
|
||||
required = original_count(*args, **kwargs)
|
||||
refusals += int(required > 0)
|
||||
return required
|
||||
|
||||
monkeypatch.setattr(coordinator, "plan_prefix_materialization", plan)
|
||||
monkeypatch.setattr(host, "get_num_blocks_to_allocate", count)
|
||||
actual = runner.generate_greedy(prompts, max_tokens=16)
|
||||
|
||||
assert checked_steps > 0
|
||||
assert refusals > 0
|
||||
assert actual == expected
|
||||
|
||||
@@ -1822,6 +1822,14 @@ class VllmConfig:
|
||||
raise ValueError(
|
||||
"HiSparse does not support decode context parallelism."
|
||||
)
|
||||
if not self.scheduler_config.scheduler_reserve_full_isl:
|
||||
# Without it, async loads admitted against free host blocks can
|
||||
# each wait on host pages the others hold, and waiting requests
|
||||
# are never preempted to free them.
|
||||
raise ValueError(
|
||||
"HiSparse requires --scheduler-reserve-full-isl; remove "
|
||||
"--no-scheduler-reserve-full-isl."
|
||||
)
|
||||
if self.model_config is not None and not hasattr(
|
||||
self.model_config.hf_config, "index_topk"
|
||||
):
|
||||
|
||||
@@ -2223,8 +2223,8 @@ class SinkFullAttentionManager(FullAttentionManager):
|
||||
class HiSparseSourceManager(FullAttentionManager):
|
||||
"""Host-tier manager with a private pool; publishes hashes once durable.
|
||||
|
||||
Host capacity is best effort: a page that cannot get a host block keeps a
|
||||
null host entry, so its GPU copy is never written back and stays pinned.
|
||||
Every page gets a host block: allocation fails when the host pool cannot
|
||||
back it, so each GPU-resident page can be written back and later reclaimed.
|
||||
"""
|
||||
|
||||
coordinator: "HiSparseCoordinator | None" = None
|
||||
@@ -2273,44 +2273,21 @@ class HiSparseSourceManager(FullAttentionManager):
|
||||
num_tokens_main_model: int,
|
||||
apply_admission_cap: bool = False,
|
||||
) -> int:
|
||||
if (
|
||||
total_computed_tokens > num_local_computed_tokens
|
||||
or self._has_partial_local_hit(
|
||||
new_computed_blocks, num_local_computed_tokens
|
||||
)
|
||||
):
|
||||
# External loads need real destinations; future GPU-computed pages
|
||||
# remain best effort. Use the same admission sentinel as Mamba.
|
||||
required = super().get_num_blocks_to_allocate(
|
||||
request_id,
|
||||
total_computed_tokens,
|
||||
new_computed_blocks,
|
||||
total_computed_tokens,
|
||||
num_local_computed_tokens,
|
||||
total_computed_tokens,
|
||||
)
|
||||
if required > self.block_pool.get_num_free_blocks():
|
||||
assert self.coordinator is not None
|
||||
assert self.coordinator.gpu_pool is not None
|
||||
return self.coordinator.gpu_pool.num_gpu_blocks + 1
|
||||
return 0
|
||||
|
||||
def allocate_new_blocks(
|
||||
self, request_id: str, num_tokens: int, num_tokens_main_model: int
|
||||
) -> list[KVCacheBlock]:
|
||||
req_blocks = self.req_to_blocks[request_id]
|
||||
num_required = cdiv(num_tokens, self.block_size)
|
||||
num_free = self.block_pool.get_num_free_blocks()
|
||||
if request_id in self._partial_hit_reqs:
|
||||
num_free -= 1
|
||||
fit_blocks = min(num_required, len(req_blocks) + max(0, num_free))
|
||||
new_blocks = super().allocate_new_blocks(
|
||||
required = super().get_num_blocks_to_allocate(
|
||||
request_id,
|
||||
fit_blocks * self.block_size,
|
||||
min(num_tokens_main_model, fit_blocks * self.block_size),
|
||||
num_tokens,
|
||||
new_computed_blocks,
|
||||
total_computed_tokens,
|
||||
num_local_computed_tokens,
|
||||
num_tokens_main_model,
|
||||
)
|
||||
req_blocks.extend([self._null_block] * (num_required - len(req_blocks)))
|
||||
return new_blocks
|
||||
# Host blocks come from a private pool, so they never count against
|
||||
# the GPU pool. Use the same admission sentinel as Mamba instead.
|
||||
if required > self.block_pool.get_num_free_blocks():
|
||||
assert self.coordinator is not None
|
||||
assert self.coordinator.gpu_pool is not None
|
||||
return self.coordinator.gpu_pool.num_gpu_blocks + 1
|
||||
return 0
|
||||
|
||||
def allocate_external_computed_blocks(
|
||||
self,
|
||||
|
||||
@@ -660,10 +660,13 @@ class HiSparseCoordinator:
|
||||
return command
|
||||
|
||||
def has_pending_reclamation(self) -> bool:
|
||||
"""Whether an in-flight spill will free GPU blocks on completion."""
|
||||
"""Whether an in-flight spill will free GPU or host blocks on completion."""
|
||||
return any(
|
||||
pending.page[0] in self.request_states
|
||||
and self._can_read_from_host(pending.page[0])
|
||||
pending.host_block.ref_cnt == 1
|
||||
or (
|
||||
pending.page[0] in self.request_states
|
||||
and self._can_read_from_host(pending.page[0])
|
||||
)
|
||||
for pending in self.pending_spills.values()
|
||||
)
|
||||
|
||||
|
||||
@@ -235,6 +235,14 @@ def create_hisparse_layout(
|
||||
host_num_blocks = host_budget // host_block_stride
|
||||
if host_num_blocks <= 0:
|
||||
raise ValueError("HiSparse has no allocatable host blocks.")
|
||||
# Every computed page needs a host block, so one request at max_model_len
|
||||
# must fit alongside the pool's null block and a copy-on-write tail.
|
||||
min_host_blocks = cdiv(vllm_config.model_config.max_model_len, gpu_block_size) + 2
|
||||
if host_num_blocks < min_host_blocks:
|
||||
raise ValueError(
|
||||
f"HiSparse host pool has {host_num_blocks} blocks but max_model_len "
|
||||
f"needs {min_host_blocks}; increase host_pool_gib."
|
||||
)
|
||||
|
||||
return HiSparseLayout(
|
||||
source_group=source_group,
|
||||
|
||||
Reference in New Issue
Block a user