diff --git a/tests/v1/simple_kv_offload/test_scheduler.py b/tests/v1/simple_kv_offload/test_scheduler.py index 5e299d2788..dee5446c61 100644 --- a/tests/v1/simple_kv_offload/test_scheduler.py +++ b/tests/v1/simple_kv_offload/test_scheduler.py @@ -6,6 +6,7 @@ from __future__ import annotations import logging from dataclasses import dataclass +from unittest.mock import Mock import pytest import torch @@ -17,6 +18,7 @@ from vllm.config import ( KVTransferConfig, ModelConfig, SchedulerConfig, + SpeculativeConfig, VllmConfig, ) from vllm.config.cache import MambaCacheMode @@ -26,7 +28,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector i ) from vllm.utils.hashing import sha256 from vllm.v1.core.block_pool import BlockPool -from vllm.v1.core.kv_cache_manager import KVCacheBlocks +from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager from vllm.v1.core.kv_cache_utils import ( get_request_block_hasher, init_none_hash, @@ -3089,3 +3091,184 @@ def test_lazy_target_blocks_ignore_non_prefix_cacheable_groups() -> None: without_ring, max_batched ) assert target_with == target_without + + +@pytest.mark.parametrize("retention_interval", [None, 0]) +@pytest.mark.parametrize("chunk_size,prefill_lookahead", [(64, 1), (1024, 1), (1, 3)]) +@pytest.mark.parametrize( + "draft_sliding_window,draft_sharded", + [(True, False), (False, False), (False, True)], + ids=["swa", "replicated_full", "sharded_full"], +) +def test_eager_store_matches_gpu_cacheable_prefix( + retention_interval: int | None, + chunk_size: int, + prefill_lookahead: int, + draft_sliding_window: bool, + draft_sharded: bool, +) -> None: + """Preserve GPU hits for replicated SWA and DCP-sharded full-attention drafts.""" + block_size = 64 + scheduler_block_size = 8 * block_size + draft_block_size = scheduler_block_size if draft_sharded else block_size + prompt_len = 16 * scheduler_block_size + draft_block_size + prefill_lookahead + kv_cache_config = _make_kv_cache_config(256, num_groups=2) + for group_id, group in enumerate(kv_cache_config.kv_cache_groups): + group.kv_cache_spec = FullAttentionSpec( + block_size=block_size, + num_kv_heads=NUM_KV_HEADS, + head_size=HEAD_SIZE, + dtype=DTYPE, + dcp_sharded=group_id == 0 or draft_sharded, + ) + for tensor in kv_cache_config.kv_cache_tensors: + tensor.size *= block_size // BLOCK_SIZE + tensor.layer_stride *= block_size // BLOCK_SIZE + tensor.block_stride *= block_size // BLOCK_SIZE + if draft_sliding_window: + kv_cache_config.kv_cache_groups[1].kv_cache_spec = SlidingWindowSpec( + block_size=block_size, + num_kv_heads=NUM_KV_HEADS, + head_size=HEAD_SIZE, + dtype=DTYPE, + sliding_window=scheduler_block_size, + extra_retained_tokens=prefill_lookahead - 1, + dcp_sharded=False, + ) + kv_cache_config.kv_cache_groups[1].is_eagle_group = True + kv_cache_config.prefix_cache_retention_interval = retention_interval + vllm_config = _make_cp_vllm_config(dcp_world_size=8) + vllm_config.model_config.max_model_len = 16384 + speculative_config = Mock(spec=SpeculativeConfig) + speculative_config.num_speculative_tokens = 3 + speculative_config.use_eagle.return_value = True + speculative_config.use_eagle_block_drop.return_value = True + speculative_config.use_multi_module_mtp.return_value = prefill_lookahead > 1 + vllm_config.speculative_config = speculative_config + scheduler = SimpleCPUOffloadScheduler( + vllm_config=vllm_config, + kv_cache_config=kv_cache_config, + cpu_capacity_bytes=kv_cache_config.kv_cache_tensors[0].size, + scheduler_block_size=scheduler_block_size, + hash_block_size=block_size, + ) + gpu_manager = KVCacheManager( + kv_cache_config, + max_model_len=16384, + max_in_flight_tokens=chunk_size, + scheduler_block_size=scheduler_block_size, + hash_block_size=block_size, + dcp_world_size=8, + use_eagle=True, + num_prefill_lookahead=prefill_lookahead, + ) + scheduler.bind_gpu_block_pool(gpu_manager.block_pool) + request = Request( + request_id="dcp-draft-source", + prompt_token_ids=list(range(prompt_len)), + sampling_params=SamplingParams(max_tokens=2), + pooling_params=None, + mm_features=None, + block_hasher=get_request_block_hasher(block_size, sha256), + ) + # Follow allocate -> connector metadata -> compute completion, including + # one decode step so the final prefill blocks can be offloaded. + while request.num_computed_tokens <= prompt_len: + if request.num_computed_tokens == prompt_len: + request.append_output_token_ids([prompt_len]) + num_new_tokens = min( + chunk_size, request.num_tokens - request.num_computed_tokens + ) + new_blocks = gpu_manager.allocate_slots( + request, num_new_tokens, num_lookahead_tokens=3 + ) + assert new_blocks is not None + if request.num_computed_tokens == 0: + scheduler.update_state_after_alloc( + request, gpu_manager.get_blocks(request.request_id), 0 + ) + output = make_scheduler_output( + {request.request_id: num_new_tokens}, + new_reqs={request.request_id: new_blocks.get_block_ids()}, + ) + else: + output = make_scheduler_output( + {request.request_id: num_new_tokens}, + cached_req_new_blocks={request.request_id: new_blocks.get_block_ids()}, + ) + store_meta = scheduler.build_connector_meta(output) + request.num_computed_tokens += num_new_tokens + if store_meta.store_event >= 0: + simulate_store_completion(scheduler, store_meta.store_event) + + matching_request = Request( + request_id="dcp-draft-load", + prompt_token_ids=request.prompt_token_ids, + sampling_params=request.sampling_params, + pooling_params=None, + mm_features=None, + block_hasher=request._block_hasher, + ) + expected_hit = ( + 15 if draft_sliding_window and retention_interval is not None else 16 + ) * scheduler_block_size + assert gpu_manager.get_computed_blocks(matching_request)[1] == expected_hit + scheduler.request_finished(request, []) + gpu_manager.free(request) + assert gpu_manager.block_pool.reset_prefix_cache() + + hit_tokens, is_async = scheduler.get_num_new_matched_tokens( + matching_request, num_computed_tokens=0 + ) + assert hit_tokens is not None + assert (hit_tokens, is_async) == (expected_hit, True) + + assert ( + gpu_manager.allocate_slots( + matching_request, + num_new_tokens=0, + num_external_computed_tokens=hit_tokens, + delay_cache_blocks=True, + ) + is not None + ) + load_blocks = gpu_manager.get_blocks(matching_request.request_id) + scheduler.update_state_after_alloc(matching_request, load_blocks, hit_tokens) + load_meta = scheduler.build_connector_meta( + make_scheduler_output( + {matching_request.request_id: 0}, + new_reqs={matching_request.request_id: load_blocks.get_block_ids()}, + ) + ) + # SWA loads only the eight-block tail after null padding; a sharded full + # draft must load every prefix block, with the same DCP geometry as the target. + num_draft_blocks = 8 if draft_sliding_window else hit_tokens // draft_block_size + assert load_meta.load_gpu_blocks == [ + block.block_id + for block in ( + *load_blocks.blocks[0], + *load_blocks.blocks[1][-num_draft_blocks:], + ) + ] + assert ( + len(load_meta.load_gpu_blocks) + == hit_tokens // scheduler_block_size + num_draft_blocks + ) + target_hashes = [ + make_block_hash_with_group_id(matching_request.block_hashes[(i + 1) * 8 - 1], 0) + for i in range(hit_tokens // scheduler_block_size) + ] + draft_hashes = [ + make_block_hash_with_group_id( + matching_request.block_hashes[(i + 1) * draft_block_size // block_size - 1], + 1, + ) + for i in range( + hit_tokens // draft_block_size - num_draft_blocks, + hit_tokens // draft_block_size, + ) + ] + assert [ + scheduler.cpu_block_pool.blocks[bid].block_hash + for bid in load_meta.load_cpu_blocks + ] == target_hashes + draft_hashes diff --git a/vllm/v1/core/kv_cache_coordinator.py b/vllm/v1/core/kv_cache_coordinator.py index 9b91bb1920..deb7f89967 100644 --- a/vllm/v1/core/kv_cache_coordinator.py +++ b/vllm/v1/core/kv_cache_coordinator.py @@ -341,6 +341,14 @@ class KVCacheCoordinator(ABC): extension = request.num_prompt_tokens // block * block return tuple(sorted({max(resend - block, 0), max(extension - block, 0)})) + def get_num_cacheable_tokens( + self, num_computed_tokens: int, kv_cache_group_id: int + ) -> int: + """Return the prefix eligible for hashing in this group.""" + # Only cache tokens with finalized KV. The last num_reprefillable_tokens + # tokens can be re-prefilled during multi-module MTP. + return max(0, num_computed_tokens - self.num_reprefillable_tokens) + def cache_blocks(self, request: Request, num_computed_tokens: int) -> None: """Cache the blocks for the request. @@ -352,17 +360,12 @@ class KVCacheCoordinator(ABC): """ boundaries = self.get_replay_boundaries(request) - for manager in self.single_type_managers: + for group_id, manager in enumerate(self.single_type_managers): if not manager.enable_caching: continue - # Only cache tokens with finalized KV. The last num_reprefillable_tokens - # tokens can be re-prefilled during multi-module MTP. - num_tokens_to_cache = max( - 0, num_computed_tokens - self.num_reprefillable_tokens - ) manager.cache_blocks( request, - num_tokens_to_cache, + self.get_num_cacheable_tokens(num_computed_tokens, group_id), retention_interval=self.retention_interval, replay_boundaries=boundaries, ) @@ -809,39 +812,22 @@ class HybridKVCacheCoordinator(KVCacheCoordinator): return num_tokens return round_down(num_tokens, self.scheduler_block_size) - def cache_blocks(self, request: Request, num_computed_tokens: int) -> None: + def get_num_cacheable_tokens( + self, num_computed_tokens: int, kv_cache_group_id: int + ) -> int: cached_num_computed_tokens = self._align_cacheable(num_computed_tokens) - boundaries = self.get_replay_boundaries(request) - for manager in self.single_type_managers: - if not manager.enable_caching: - continue - num_tokens_to_cache = cached_num_computed_tokens - # EAGLE groups match one block past each aligned boundary and drop - # it, so make that lookahead block eligible to be cached. - if manager.use_eagle and cached_num_computed_tokens > 0: - # Only cache tokens with finalized KV. The last - # num_reprefillable_tokens tokens can be re-prefilled during - # multi-module MTP. - num_finalized_computed_tokens = max( - 0, num_computed_tokens - self.num_reprefillable_tokens - ) - cached_num_finalized_computed_tokens = self._align_cacheable( - num_finalized_computed_tokens - ) - num_tokens_to_cache = min( - num_finalized_computed_tokens, - cached_num_finalized_computed_tokens + manager.block_size, - ) - # The manager already knows the fine hit granularity - # (``scheduler_block_size``); retention is passed separately so it - # can keep both the coarse segment tails and the fine replay - # boundary (which needs the fine value). - manager.cache_blocks( - request, - num_tokens_to_cache, - retention_interval=self.retention_interval, - replay_boundaries=boundaries, + manager = self.single_type_managers[kv_cache_group_id] + # EAGLE matches one block past the aligned boundary, then drops it. + if manager.use_eagle and cached_num_computed_tokens > 0: + num_finalized_computed_tokens = max( + 0, num_computed_tokens - self.num_reprefillable_tokens ) + return min( + num_finalized_computed_tokens, + self._align_cacheable(num_finalized_computed_tokens) + + manager.block_size, + ) + return cached_num_computed_tokens def find_longest_cache_hit( self, diff --git a/vllm/v1/simple_kv_offload/manager.py b/vllm/v1/simple_kv_offload/manager.py index 9d95940a11..dfc24e420a 100644 --- a/vllm/v1/simple_kv_offload/manager.py +++ b/vllm/v1/simple_kv_offload/manager.py @@ -216,6 +216,7 @@ class SimpleCPUOffloadScheduler: scheduler_block_size=self.block_size, hash_block_size=self.hash_block_size, allow_partial_hash_hits=not lazy_offload, + num_prefill_lookahead=vllm_config.num_prefill_lookahead_tokens, ) self.group_block_sizes = self.cpu_coordinator.group_block_sizes # FA group's own resolved block_size; divides scheduler_block_size (the @@ -837,19 +838,6 @@ class SimpleCPUOffloadScheduler: ) request = state.request confirmed_tokens = request.num_computed_tokens - request.num_output_placeholders - # Truncate to the granularity a lookup can actually land on. With - # fine-grained hits that is hash_block_size; otherwise hits only land on - # the scheduler block (the group LCM). Using the LCM unconditionally - # would drop whole blocks that sit between the last LCM boundary and a - # reachable fine-grained boundary, so the other groups would hold that - # boundary and this one would not, and the joint hybrid lookup would - # reconcile to zero. - store_alignment = ( - self.hash_block_size - if self.cpu_coordinator.enable_partial_hash_hits - else self.block_size - ) - aligned_tokens = confirmed_tokens // store_alignment * store_alignment num_free = self.cpu_block_pool.get_num_free_blocks() for g, group_gpu_ids in enumerate(block_ids_by_group): @@ -864,7 +852,11 @@ class SimpleCPUOffloadScheduler: # num_stored_blocks can be stale and omit evicted blocks in # the middle of the request. group_size = self.group_block_sizes[g] - ready = min(len(group_gpu_ids), aligned_tokens // group_size) + # Mimic the GPU KV cache manager's cacheable prefix. + cacheable_tokens = self.cpu_coordinator.get_num_cacheable_tokens( + confirmed_tokens, g + ) + ready = min(len(group_gpu_ids), cacheable_tokens // group_size) resolved_hashes = resolve_block_hashes( request.block_hashes, self.hash_block_size, group_size )