diff --git a/tests/test_config.py b/tests/test_config.py index 39798acb05..30161e89fb 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -420,6 +420,19 @@ def test_hisparse_rejects_disabled_hybrid_kv_cache_manager(monkeypatch): ) +def test_hisparse_rejects_disabled_full_isl_reservation(monkeypatch): + monkeypatch.setattr(current_platform, "is_cuda", lambda: True) + with pytest.raises(ValueError, match="requires --scheduler-reserve-full-isl"): + VllmConfig( + attention_config=AttentionConfig(hisparse_config=HiSparseConfig()), + scheduler_config=SchedulerConfig( + max_model_len=2048, + is_encoder_decoder=False, + scheduler_reserve_full_isl=False, + ), + ) + + def test_hisparse_rejects_non_cuda(monkeypatch): monkeypatch.setattr(current_platform, "is_cuda", lambda: False) with pytest.raises(ValueError, match="requires NVIDIA CUDA"): diff --git a/tests/v1/core/test_kv_cache_utils.py b/tests/v1/core/test_kv_cache_utils.py index 478707af9d..fb82d28c36 100644 --- a/tests/v1/core/test_kv_cache_utils.py +++ b/tests/v1/core/test_kv_cache_utils.py @@ -205,6 +205,50 @@ def test_hisparse_hma_uses_resolved_gpu_block_size( assert scheduler_block_size == hash_block_size == gpu_block_size +@pytest.mark.parametrize("extra_blocks,ok", [(1, False), (2, True)]) +def test_hisparse_host_pool_must_fit_max_model_len(monkeypatch, extra_blocks, ok): + """Every page needs a host block, so one max_model_len request must fit.""" + monkeypatch.setattr( + hisparse_runtime_module.current_platform, "is_cuda_alike", lambda: True + ) + specs = { + "model.layers.0.self_attn": MLAAttentionSpec( + block_size=64, + num_kv_heads=1, + head_size=576, + dtype=torch.bfloat16, + is_index_group_leader=True, + ), + "model.layers.0.self_attn.indexer": MLAAttentionSpec( + block_size=64, + num_kv_heads=1, + head_size=128, + dtype=torch.bfloat16, + cache_role=SparseCacheRole.INDEXER, + ), + } + group_spec = UniformTypeKVCacheSpecs.from_specs(specs) + assert group_spec is not None + config = SimpleNamespace( + attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()), + model_config=SimpleNamespace( + hf_config=SimpleNamespace(index_topk=128), max_model_len=64 * 4 + ), + parallel_config=SimpleNamespace(tensor_parallel_size=1, world_size=1), + cache_config=SimpleNamespace(num_gpu_blocks_override=7), + ) + host_page = specs["model.layers.0.self_attn"].page_size_bytes + host_budget = (4 + extra_blocks) * host_page + group = KVCacheGroupSpec(list(specs), group_spec) + + if ok: + layout = create_hisparse_layout(config, [group], host_budget=host_budget) + assert layout.host_num_blocks == 4 + extra_blocks + else: + with pytest.raises(ValueError, match="increase host_pool_gib"): + create_hisparse_layout(config, [group], host_budget=host_budget) + + def test_hisparse_rejects_deepseek_v4(): full_specs = { "model.layers.0.attn": MLAAttentionSpec( diff --git a/tests/v1/core/test_prefix_caching.py b/tests/v1/core/test_prefix_caching.py index c46c179260..c8d42665f0 100644 --- a/tests/v1/core/test_prefix_caching.py +++ b/tests/v1/core/test_prefix_caching.py @@ -782,14 +782,15 @@ def test_hisparse_inflight_host_import_reserves_remaining_gpu_pages(): assert required == 4 -@pytest.mark.parametrize( - "full_sequence_must_fit,host_num_blocks,admitted", - [(True, 2, False), (True, 3, True), (False, 2, False), (False, 3, True)], -) -def test_hisparse_async_admission_requires_only_import_destinations( - full_sequence_must_fit, host_num_blocks, admitted, tmp_path +@pytest.mark.parametrize("host_num_blocks,admitted", [(5, False), (6, True)]) +def test_hisparse_async_admission_requires_full_prompt_host_pages( + host_num_blocks, admitted, tmp_path ): - """Imports need host destinations, but do not reserve future prefill pages.""" + """An async load is admitted only if its whole prompt's host pages fit. + + The in-flight prefill's host page is not free, so the waiting request's + four prompt pages need six host blocks including the null block. + """ from .utils import create_scheduler, mock_kv (tmp_path / "config.json").write_text( @@ -804,7 +805,7 @@ def test_hisparse_async_admission_requires_only_import_destinations( manager = make_hisparse_kv_cache_manager(32, host_num_blocks) scheduler.kv_cache_manager = manager scheduler.kv_cache_config = manager.kv_cache_config - scheduler.scheduler_reserve_full_isl = full_sequence_must_fit + assert scheduler.scheduler_reserve_full_isl inflight = make_request( "inflight", list(range(3 * HISPARSE_BLOCK_SIZE)), HISPARSE_BLOCK_SIZE, sha256 ) @@ -822,6 +823,52 @@ def test_hisparse_async_admission_requires_only_import_destinations( assert bool(manager.get_blocks(request.request_id).blocks[0]) == admitted +def test_hisparse_admitted_async_loads_can_finish_with_nothing_running(tmp_path): + """Admitted async loads must not strand each other on host capacity. + + Waiting requests keep their blocks and are never preempted, so two imports + that each hold a host page while together needing more than the pool has + left would never be scheduled again. HiSparse relies on full-ISL admission: + a load is admitted only if its whole prompt's host pages fit, so the newest + admitted load can always finish and free host blocks for the others. + """ + from .utils import create_scheduler, mock_kv + + (tmp_path / "config.json").write_text( + '{"architectures": ["OPTForCausalLM"], "model_type": "opt"}' + ) + scheduler = create_scheduler( + model=str(tmp_path), + skip_tokenizer_init=True, + max_model_len=128, + use_kv_connector=mock_kv(matched_tokens=HISPARSE_BLOCK_SIZE, is_async=True), + ) + # One null block plus three usable host blocks. + manager = make_hisparse_kv_cache_manager(32, 4) + scheduler.kv_cache_manager = manager + scheduler.kv_cache_config = manager.kv_cache_config + assert scheduler.scheduler_reserve_full_isl + first = make_request( + "first", list(range(3 * HISPARSE_BLOCK_SIZE)), HISPARSE_BLOCK_SIZE, sha256 + ) + second = make_request( + "second", + list(range(1000, 1000 + 3 * HISPARSE_BLOCK_SIZE)), + HISPARSE_BLOCK_SIZE, + sha256, + ) + for request in (first, second): + scheduler.add_request(request) + scheduler.schedule() + admitted = {r.request_id for r in scheduler._inflight_prefills} + + scheduler.finished_recving_kv_req_ids.update(admitted) + num_scheduled_tokens = 0 + for _ in range(4): + num_scheduled_tokens += scheduler.schedule().total_num_scheduled_tokens + assert num_scheduled_tokens > 0 + + @pytest.mark.parametrize( "evictable,local_tokens,free_blocks,admitted", [ @@ -845,58 +892,47 @@ def test_hisparse_import_capacity_includes_hits_and_cow( assert host_pool.get_num_free_blocks() == free_blocks assert ( coordinator.host_manager.get_num_blocks_to_allocate( - "waiting", 64, [hit], 32, local_tokens, 64 + "waiting", 32, [hit], 32, local_tokens, 32 ) <= host_pool.get_num_free_blocks() ) == admitted @pytest.mark.parametrize("enable_caching", [False, True]) -def test_hisparse_host_exhaustion_keeps_gpu_pages_readable(enable_caching): - """Pages without a host destination must survive GPU cache reclamation.""" - manager = make_hisparse_kv_cache_manager(32, 2, enable_caching=enable_caching) +def test_hisparse_host_exhaustion_defers_allocation(enable_caching): + """Pages are only allocated with host backing, so every one can be reclaimed.""" + manager = make_hisparse_kv_cache_manager(32, 5, enable_caching=enable_caching) + donor = make_request("donor", [99] * 16, HISPARSE_BLOCK_SIZE, sha256) + assert manager.allocate_slots(donor, 16) is not None request = make_request("resident", list(range(64)), HISPARSE_BLOCK_SIZE, sha256) - assert manager.allocate_slots(request, 64) is not None + free_gpu = manager.block_pool.get_num_free_blocks() + assert manager.allocate_slots(request, 64) is None + assert manager.block_pool.get_num_free_blocks() == free_gpu + + # The donor's in-flight write-back holds its host block until acked. + manager.free(donor) coordinator = get_hisparse_coordinator(manager) + assert coordinator.has_pending_reclamation() + assert manager.allocate_slots(request, 64) is None + _publish_hisparse_pages(manager) + assert not coordinator.has_pending_reclamation() + assert manager.allocate_slots(request, 64) is not None host = coordinator.host_manager assert host is not None - assert [b.is_null for b in host.req_to_blocks[request.request_id]] == [ - False, - True, - True, - True, - ] + assert not any(b.is_null for b in host.req_to_blocks[request.request_id]) _publish_hisparse_pages(manager) for hot in coordinator.hot_managers: hot.require_hot(request.request_id) hot.allocate_new_blocks(request.request_id, 64, 64) coordinator.update_residency(request.request_id) + assert coordinator.request_states[request.request_id].unpinned_pages == {0, 1} for resident in coordinator.resident_managers: - for block in resident.req_to_blocks[request.request_id][1:]: - assert not block.is_null and block.ref_cnt > 0 - assert coordinator.request_states[request.request_id].valid_pages == {0} - pool = coordinator.get_host_block_pool() - assert pool is not None - manager.free(request) - assert pool.get_num_free_blocks() == 1 - - -def test_hisparse_host_cow_takes_priority_over_new_pages(): - """The last host block must hold the private tail, not a fresh page.""" - manager = make_hisparse_kv_cache_manager(32, 3) - coordinator = get_hisparse_coordinator(manager) - host = coordinator.host_manager - assert host is not None - source = host.block_pool.get_new_blocks(1)[0] - host.req_to_blocks["cow"] = [source] - host._partial_hit_reqs["cow"] = (0, source) - - host.allocate_new_blocks("cow", 32, 32) - - private, missing = host.req_to_blocks["cow"] - assert not private.is_null and private is not source - assert missing.is_null - assert host.take_host_cow_copies() == [(source, private)] + assert [b.ref_cnt for b in resident.req_to_blocks[request.request_id]] == [ + 0, + 0, + 1, + 1, + ] @pytest.mark.parametrize("failed", [False, True]) diff --git a/tests/v1/e2e/general/test_hisparse.py b/tests/v1/e2e/general/test_hisparse.py index dd4c588657..21f28d0b6b 100644 --- a/tests/v1/e2e/general/test_hisparse.py +++ b/tests/v1/e2e/general/test_hisparse.py @@ -223,3 +223,81 @@ def test_hisparse_spill_and_prefix_restore( assert actual == expected if with_offloading: assert _offload_load_bytes() > load_bytes + + +@pytest.mark.skipif( + not current_platform.is_cuda(), reason="HiSparse requires NVIDIA CUDA" +) +@fork_new_process_for_each_test +def test_hisparse_host_exhaustion_defers_requests( + monkeypatch: pytest.MonkeyPatch, + vllm_runner: type[VllmRunner], +): + """A full host pool defers requests instead of leaving pages GPU-only.""" + capability = current_platform.get_device_capability() + if capability is None or capability.major < 9: + pytest.skip("Sparse MLA requires Hopper or newer") + monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") + monkeypatch.setenv("VLLM_DEEP_GEMM_WARMUP", "skip") + prompts = [ + [1000 + request_idx * 64 + i % 64 for i in range(257)] + for request_idx in range(3) + ] + with vllm_runner( + MODEL, + load_format="dummy", + hf_overrides=_shrink_config, + attention_config=AttentionConfig( + hisparse_config=HiSparseConfig(device_buffer_size=512) + ), + kv_transfer_config=KVTransferConfig( + kv_connector="HiSparseConnector", + kv_role="kv_both", + kv_connector_extra_config={"host_pool_gib": 1}, + ), + block_size=64, + max_model_len=320, + max_num_batched_tokens=512, + max_num_seqs=3, + num_gpu_blocks_override=128, + enable_prefix_caching=False, + enable_chunked_prefill=True, + enforce_eager=True, + ) as runner: + expected = [runner.generate_greedy([p], max_tokens=16)[0] for p in prompts] + engine = runner.llm.llm_engine.engine_core.engine_core + coordinator = get_hisparse_coordinator(engine.scheduler.kv_cache_manager) + host = coordinator.host_manager + assert host is not None + # Leave room for one full request plus part of another. + spare_blocks = 8 + num_pressure = host.block_pool.get_num_free_blocks() - spare_blocks + host.allocate_new_blocks( + "host-pressure", + num_pressure * host.block_size, + num_pressure * host.block_size, + ) + original_plan = coordinator.plan_prefix_materialization + original_count = host.get_num_blocks_to_allocate + checked_steps = 0 + refusals = 0 + + def plan(request_id, num_computed_tokens): + nonlocal checked_steps + assert not any(b.is_null for b in host.req_to_blocks[request_id]) + checked_steps += 1 + original_plan(request_id, num_computed_tokens) + + def count(*args, **kwargs): + nonlocal refusals + required = original_count(*args, **kwargs) + refusals += int(required > 0) + return required + + monkeypatch.setattr(coordinator, "plan_prefix_materialization", plan) + monkeypatch.setattr(host, "get_num_blocks_to_allocate", count) + actual = runner.generate_greedy(prompts, max_tokens=16) + + assert checked_steps > 0 + assert refusals > 0 + assert actual == expected diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index a01023e34b..824c49efa0 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1822,6 +1822,14 @@ class VllmConfig: raise ValueError( "HiSparse does not support decode context parallelism." ) + if not self.scheduler_config.scheduler_reserve_full_isl: + # Without it, async loads admitted against free host blocks can + # each wait on host pages the others hold, and waiting requests + # are never preempted to free them. + raise ValueError( + "HiSparse requires --scheduler-reserve-full-isl; remove " + "--no-scheduler-reserve-full-isl." + ) if self.model_config is not None and not hasattr( self.model_config.hf_config, "index_topk" ): diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index 707786f17f..31e610b578 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -2223,8 +2223,8 @@ class SinkFullAttentionManager(FullAttentionManager): class HiSparseSourceManager(FullAttentionManager): """Host-tier manager with a private pool; publishes hashes once durable. - Host capacity is best effort: a page that cannot get a host block keeps a - null host entry, so its GPU copy is never written back and stays pinned. + Every page gets a host block: allocation fails when the host pool cannot + back it, so each GPU-resident page can be written back and later reclaimed. """ coordinator: "HiSparseCoordinator | None" = None @@ -2273,44 +2273,21 @@ class HiSparseSourceManager(FullAttentionManager): num_tokens_main_model: int, apply_admission_cap: bool = False, ) -> int: - if ( - total_computed_tokens > num_local_computed_tokens - or self._has_partial_local_hit( - new_computed_blocks, num_local_computed_tokens - ) - ): - # External loads need real destinations; future GPU-computed pages - # remain best effort. Use the same admission sentinel as Mamba. - required = super().get_num_blocks_to_allocate( - request_id, - total_computed_tokens, - new_computed_blocks, - total_computed_tokens, - num_local_computed_tokens, - total_computed_tokens, - ) - if required > self.block_pool.get_num_free_blocks(): - assert self.coordinator is not None - assert self.coordinator.gpu_pool is not None - return self.coordinator.gpu_pool.num_gpu_blocks + 1 - return 0 - - def allocate_new_blocks( - self, request_id: str, num_tokens: int, num_tokens_main_model: int - ) -> list[KVCacheBlock]: - req_blocks = self.req_to_blocks[request_id] - num_required = cdiv(num_tokens, self.block_size) - num_free = self.block_pool.get_num_free_blocks() - if request_id in self._partial_hit_reqs: - num_free -= 1 - fit_blocks = min(num_required, len(req_blocks) + max(0, num_free)) - new_blocks = super().allocate_new_blocks( + required = super().get_num_blocks_to_allocate( request_id, - fit_blocks * self.block_size, - min(num_tokens_main_model, fit_blocks * self.block_size), + num_tokens, + new_computed_blocks, + total_computed_tokens, + num_local_computed_tokens, + num_tokens_main_model, ) - req_blocks.extend([self._null_block] * (num_required - len(req_blocks))) - return new_blocks + # Host blocks come from a private pool, so they never count against + # the GPU pool. Use the same admission sentinel as Mamba instead. + if required > self.block_pool.get_num_free_blocks(): + assert self.coordinator is not None + assert self.coordinator.gpu_pool is not None + return self.coordinator.gpu_pool.num_gpu_blocks + 1 + return 0 def allocate_external_computed_blocks( self, diff --git a/vllm/v1/hisparse/coordinator.py b/vllm/v1/hisparse/coordinator.py index 056e19479a..0ca0d40c88 100644 --- a/vllm/v1/hisparse/coordinator.py +++ b/vllm/v1/hisparse/coordinator.py @@ -660,10 +660,13 @@ class HiSparseCoordinator: return command def has_pending_reclamation(self) -> bool: - """Whether an in-flight spill will free GPU blocks on completion.""" + """Whether an in-flight spill will free GPU or host blocks on completion.""" return any( - pending.page[0] in self.request_states - and self._can_read_from_host(pending.page[0]) + pending.host_block.ref_cnt == 1 + or ( + pending.page[0] in self.request_states + and self._can_read_from_host(pending.page[0]) + ) for pending in self.pending_spills.values() ) diff --git a/vllm/v1/hisparse/layout.py b/vllm/v1/hisparse/layout.py index b3b45d3334..e177768fc9 100644 --- a/vllm/v1/hisparse/layout.py +++ b/vllm/v1/hisparse/layout.py @@ -235,6 +235,14 @@ def create_hisparse_layout( host_num_blocks = host_budget // host_block_stride if host_num_blocks <= 0: raise ValueError("HiSparse has no allocatable host blocks.") + # Every computed page needs a host block, so one request at max_model_len + # must fit alongside the pool's null block and a copy-on-write tail. + min_host_blocks = cdiv(vllm_config.model_config.max_model_len, gpu_block_size) + 2 + if host_num_blocks < min_host_blocks: + raise ValueError( + f"HiSparse host pool has {host_num_blocks} blocks but max_model_len " + f"needs {min_host_blocks}; increase host_pool_gib." + ) return HiSparseLayout( source_group=source_group,