[Bugfix][HiSparse] Never allocate GPU pages without host backing (#59036)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Matthew Bonanni
Claude Opus 5.5
parent
e006d761a5
commit
90e13fc757
@@ -223,3 +223,81 @@ def test_hisparse_spill_and_prefix_restore(
|
||||
assert actual == expected
|
||||
if with_offloading:
|
||||
assert _offload_load_bytes() > load_bytes
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="HiSparse requires NVIDIA CUDA"
|
||||
)
|
||||
@fork_new_process_for_each_test
|
||||
def test_hisparse_host_exhaustion_defers_requests(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
vllm_runner: type[VllmRunner],
|
||||
):
|
||||
"""A full host pool defers requests instead of leaving pages GPU-only."""
|
||||
capability = current_platform.get_device_capability()
|
||||
if capability is None or capability.major < 9:
|
||||
pytest.skip("Sparse MLA requires Hopper or newer")
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
monkeypatch.setenv("VLLM_DEEP_GEMM_WARMUP", "skip")
|
||||
prompts = [
|
||||
[1000 + request_idx * 64 + i % 64 for i in range(257)]
|
||||
for request_idx in range(3)
|
||||
]
|
||||
with vllm_runner(
|
||||
MODEL,
|
||||
load_format="dummy",
|
||||
hf_overrides=_shrink_config,
|
||||
attention_config=AttentionConfig(
|
||||
hisparse_config=HiSparseConfig(device_buffer_size=512)
|
||||
),
|
||||
kv_transfer_config=KVTransferConfig(
|
||||
kv_connector="HiSparseConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config={"host_pool_gib": 1},
|
||||
),
|
||||
block_size=64,
|
||||
max_model_len=320,
|
||||
max_num_batched_tokens=512,
|
||||
max_num_seqs=3,
|
||||
num_gpu_blocks_override=128,
|
||||
enable_prefix_caching=False,
|
||||
enable_chunked_prefill=True,
|
||||
enforce_eager=True,
|
||||
) as runner:
|
||||
expected = [runner.generate_greedy([p], max_tokens=16)[0] for p in prompts]
|
||||
engine = runner.llm.llm_engine.engine_core.engine_core
|
||||
coordinator = get_hisparse_coordinator(engine.scheduler.kv_cache_manager)
|
||||
host = coordinator.host_manager
|
||||
assert host is not None
|
||||
# Leave room for one full request plus part of another.
|
||||
spare_blocks = 8
|
||||
num_pressure = host.block_pool.get_num_free_blocks() - spare_blocks
|
||||
host.allocate_new_blocks(
|
||||
"host-pressure",
|
||||
num_pressure * host.block_size,
|
||||
num_pressure * host.block_size,
|
||||
)
|
||||
original_plan = coordinator.plan_prefix_materialization
|
||||
original_count = host.get_num_blocks_to_allocate
|
||||
checked_steps = 0
|
||||
refusals = 0
|
||||
|
||||
def plan(request_id, num_computed_tokens):
|
||||
nonlocal checked_steps
|
||||
assert not any(b.is_null for b in host.req_to_blocks[request_id])
|
||||
checked_steps += 1
|
||||
original_plan(request_id, num_computed_tokens)
|
||||
|
||||
def count(*args, **kwargs):
|
||||
nonlocal refusals
|
||||
required = original_count(*args, **kwargs)
|
||||
refusals += int(required > 0)
|
||||
return required
|
||||
|
||||
monkeypatch.setattr(coordinator, "plan_prefix_materialization", plan)
|
||||
monkeypatch.setattr(host, "get_num_blocks_to_allocate", count)
|
||||
actual = runner.generate_greedy(prompts, max_tokens=16)
|
||||
|
||||
assert checked_steps > 0
|
||||
assert refusals > 0
|
||||
assert actual == expected
|
||||
|
||||
Reference in New Issue
Block a user