[Bugfix][KV Offload] Honor speculative cacheability in SimpleCPU (#60071)

Signed-off-by: Sherif Waly <sherif.waly@mistral.ai>
This commit is contained in:
Sherif Waly
2026-10-06 14:25:08 +02:00
committed by GitHub
parent 3e182185aa
commit 25385102fe
3 changed files with 214 additions and 53 deletions
+184 -1
View File
@@ -6,6 +6,7 @@ from __future__ import annotations
import logging
from dataclasses import dataclass
from unittest.mock import Mock
import pytest
import torch
@@ -17,6 +18,7 @@ from vllm.config import (
KVTransferConfig,
ModelConfig,
SchedulerConfig,
SpeculativeConfig,
VllmConfig,
)
from vllm.config.cache import MambaCacheMode
@@ -26,7 +28,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector i
)
from vllm.utils.hashing import sha256
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager
from vllm.v1.core.kv_cache_utils import (
get_request_block_hasher,
init_none_hash,
@@ -3089,3 +3091,184 @@ def test_lazy_target_blocks_ignore_non_prefix_cacheable_groups() -> None:
without_ring, max_batched
)
assert target_with == target_without
@pytest.mark.parametrize("retention_interval", [None, 0])
@pytest.mark.parametrize("chunk_size,prefill_lookahead", [(64, 1), (1024, 1), (1, 3)])
@pytest.mark.parametrize(
"draft_sliding_window,draft_sharded",
[(True, False), (False, False), (False, True)],
ids=["swa", "replicated_full", "sharded_full"],
)
def test_eager_store_matches_gpu_cacheable_prefix(
retention_interval: int | None,
chunk_size: int,
prefill_lookahead: int,
draft_sliding_window: bool,
draft_sharded: bool,
) -> None:
"""Preserve GPU hits for replicated SWA and DCP-sharded full-attention drafts."""
block_size = 64
scheduler_block_size = 8 * block_size
draft_block_size = scheduler_block_size if draft_sharded else block_size
prompt_len = 16 * scheduler_block_size + draft_block_size + prefill_lookahead
kv_cache_config = _make_kv_cache_config(256, num_groups=2)
for group_id, group in enumerate(kv_cache_config.kv_cache_groups):
group.kv_cache_spec = FullAttentionSpec(
block_size=block_size,
num_kv_heads=NUM_KV_HEADS,
head_size=HEAD_SIZE,
dtype=DTYPE,
dcp_sharded=group_id == 0 or draft_sharded,
)
for tensor in kv_cache_config.kv_cache_tensors:
tensor.size *= block_size // BLOCK_SIZE
tensor.layer_stride *= block_size // BLOCK_SIZE
tensor.block_stride *= block_size // BLOCK_SIZE
if draft_sliding_window:
kv_cache_config.kv_cache_groups[1].kv_cache_spec = SlidingWindowSpec(
block_size=block_size,
num_kv_heads=NUM_KV_HEADS,
head_size=HEAD_SIZE,
dtype=DTYPE,
sliding_window=scheduler_block_size,
extra_retained_tokens=prefill_lookahead - 1,
dcp_sharded=False,
)
kv_cache_config.kv_cache_groups[1].is_eagle_group = True
kv_cache_config.prefix_cache_retention_interval = retention_interval
vllm_config = _make_cp_vllm_config(dcp_world_size=8)
vllm_config.model_config.max_model_len = 16384
speculative_config = Mock(spec=SpeculativeConfig)
speculative_config.num_speculative_tokens = 3
speculative_config.use_eagle.return_value = True
speculative_config.use_eagle_block_drop.return_value = True
speculative_config.use_multi_module_mtp.return_value = prefill_lookahead > 1
vllm_config.speculative_config = speculative_config
scheduler = SimpleCPUOffloadScheduler(
vllm_config=vllm_config,
kv_cache_config=kv_cache_config,
cpu_capacity_bytes=kv_cache_config.kv_cache_tensors[0].size,
scheduler_block_size=scheduler_block_size,
hash_block_size=block_size,
)
gpu_manager = KVCacheManager(
kv_cache_config,
max_model_len=16384,
max_in_flight_tokens=chunk_size,
scheduler_block_size=scheduler_block_size,
hash_block_size=block_size,
dcp_world_size=8,
use_eagle=True,
num_prefill_lookahead=prefill_lookahead,
)
scheduler.bind_gpu_block_pool(gpu_manager.block_pool)
request = Request(
request_id="dcp-draft-source",
prompt_token_ids=list(range(prompt_len)),
sampling_params=SamplingParams(max_tokens=2),
pooling_params=None,
mm_features=None,
block_hasher=get_request_block_hasher(block_size, sha256),
)
# Follow allocate -> connector metadata -> compute completion, including
# one decode step so the final prefill blocks can be offloaded.
while request.num_computed_tokens <= prompt_len:
if request.num_computed_tokens == prompt_len:
request.append_output_token_ids([prompt_len])
num_new_tokens = min(
chunk_size, request.num_tokens - request.num_computed_tokens
)
new_blocks = gpu_manager.allocate_slots(
request, num_new_tokens, num_lookahead_tokens=3
)
assert new_blocks is not None
if request.num_computed_tokens == 0:
scheduler.update_state_after_alloc(
request, gpu_manager.get_blocks(request.request_id), 0
)
output = make_scheduler_output(
{request.request_id: num_new_tokens},
new_reqs={request.request_id: new_blocks.get_block_ids()},
)
else:
output = make_scheduler_output(
{request.request_id: num_new_tokens},
cached_req_new_blocks={request.request_id: new_blocks.get_block_ids()},
)
store_meta = scheduler.build_connector_meta(output)
request.num_computed_tokens += num_new_tokens
if store_meta.store_event >= 0:
simulate_store_completion(scheduler, store_meta.store_event)
matching_request = Request(
request_id="dcp-draft-load",
prompt_token_ids=request.prompt_token_ids,
sampling_params=request.sampling_params,
pooling_params=None,
mm_features=None,
block_hasher=request._block_hasher,
)
expected_hit = (
15 if draft_sliding_window and retention_interval is not None else 16
) * scheduler_block_size
assert gpu_manager.get_computed_blocks(matching_request)[1] == expected_hit
scheduler.request_finished(request, [])
gpu_manager.free(request)
assert gpu_manager.block_pool.reset_prefix_cache()
hit_tokens, is_async = scheduler.get_num_new_matched_tokens(
matching_request, num_computed_tokens=0
)
assert hit_tokens is not None
assert (hit_tokens, is_async) == (expected_hit, True)
assert (
gpu_manager.allocate_slots(
matching_request,
num_new_tokens=0,
num_external_computed_tokens=hit_tokens,
delay_cache_blocks=True,
)
is not None
)
load_blocks = gpu_manager.get_blocks(matching_request.request_id)
scheduler.update_state_after_alloc(matching_request, load_blocks, hit_tokens)
load_meta = scheduler.build_connector_meta(
make_scheduler_output(
{matching_request.request_id: 0},
new_reqs={matching_request.request_id: load_blocks.get_block_ids()},
)
)
# SWA loads only the eight-block tail after null padding; a sharded full
# draft must load every prefix block, with the same DCP geometry as the target.
num_draft_blocks = 8 if draft_sliding_window else hit_tokens // draft_block_size
assert load_meta.load_gpu_blocks == [
block.block_id
for block in (
*load_blocks.blocks[0],
*load_blocks.blocks[1][-num_draft_blocks:],
)
]
assert (
len(load_meta.load_gpu_blocks)
== hit_tokens // scheduler_block_size + num_draft_blocks
)
target_hashes = [
make_block_hash_with_group_id(matching_request.block_hashes[(i + 1) * 8 - 1], 0)
for i in range(hit_tokens // scheduler_block_size)
]
draft_hashes = [
make_block_hash_with_group_id(
matching_request.block_hashes[(i + 1) * draft_block_size // block_size - 1],
1,
)
for i in range(
hit_tokens // draft_block_size - num_draft_blocks,
hit_tokens // draft_block_size,
)
]
assert [
scheduler.cpu_block_pool.blocks[bid].block_hash
for bid in load_meta.load_cpu_blocks
] == target_hashes + draft_hashes
+24 -38
View File
@@ -341,6 +341,14 @@ class KVCacheCoordinator(ABC):
extension = request.num_prompt_tokens // block * block
return tuple(sorted({max(resend - block, 0), max(extension - block, 0)}))
def get_num_cacheable_tokens(
self, num_computed_tokens: int, kv_cache_group_id: int
) -> int:
"""Return the prefix eligible for hashing in this group."""
# Only cache tokens with finalized KV. The last num_reprefillable_tokens
# tokens can be re-prefilled during multi-module MTP.
return max(0, num_computed_tokens - self.num_reprefillable_tokens)
def cache_blocks(self, request: Request, num_computed_tokens: int) -> None:
"""Cache the blocks for the request.
@@ -352,17 +360,12 @@ class KVCacheCoordinator(ABC):
"""
boundaries = self.get_replay_boundaries(request)
for manager in self.single_type_managers:
for group_id, manager in enumerate(self.single_type_managers):
if not manager.enable_caching:
continue
# Only cache tokens with finalized KV. The last num_reprefillable_tokens
# tokens can be re-prefilled during multi-module MTP.
num_tokens_to_cache = max(
0, num_computed_tokens - self.num_reprefillable_tokens
)
manager.cache_blocks(
request,
num_tokens_to_cache,
self.get_num_cacheable_tokens(num_computed_tokens, group_id),
retention_interval=self.retention_interval,
replay_boundaries=boundaries,
)
@@ -809,39 +812,22 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
return num_tokens
return round_down(num_tokens, self.scheduler_block_size)
def cache_blocks(self, request: Request, num_computed_tokens: int) -> None:
def get_num_cacheable_tokens(
self, num_computed_tokens: int, kv_cache_group_id: int
) -> int:
cached_num_computed_tokens = self._align_cacheable(num_computed_tokens)
boundaries = self.get_replay_boundaries(request)
for manager in self.single_type_managers:
if not manager.enable_caching:
continue
num_tokens_to_cache = cached_num_computed_tokens
# EAGLE groups match one block past each aligned boundary and drop
# it, so make that lookahead block eligible to be cached.
if manager.use_eagle and cached_num_computed_tokens > 0:
# Only cache tokens with finalized KV. The last
# num_reprefillable_tokens tokens can be re-prefilled during
# multi-module MTP.
num_finalized_computed_tokens = max(
0, num_computed_tokens - self.num_reprefillable_tokens
)
cached_num_finalized_computed_tokens = self._align_cacheable(
num_finalized_computed_tokens
)
num_tokens_to_cache = min(
num_finalized_computed_tokens,
cached_num_finalized_computed_tokens + manager.block_size,
)
# The manager already knows the fine hit granularity
# (``scheduler_block_size``); retention is passed separately so it
# can keep both the coarse segment tails and the fine replay
# boundary (which needs the fine value).
manager.cache_blocks(
request,
num_tokens_to_cache,
retention_interval=self.retention_interval,
replay_boundaries=boundaries,
manager = self.single_type_managers[kv_cache_group_id]
# EAGLE matches one block past the aligned boundary, then drops it.
if manager.use_eagle and cached_num_computed_tokens > 0:
num_finalized_computed_tokens = max(
0, num_computed_tokens - self.num_reprefillable_tokens
)
return min(
num_finalized_computed_tokens,
self._align_cacheable(num_finalized_computed_tokens)
+ manager.block_size,
)
return cached_num_computed_tokens
def find_longest_cache_hit(
self,
+6 -14
View File
@@ -216,6 +216,7 @@ class SimpleCPUOffloadScheduler:
scheduler_block_size=self.block_size,
hash_block_size=self.hash_block_size,
allow_partial_hash_hits=not lazy_offload,
num_prefill_lookahead=vllm_config.num_prefill_lookahead_tokens,
)
self.group_block_sizes = self.cpu_coordinator.group_block_sizes
# FA group's own resolved block_size; divides scheduler_block_size (the
@@ -837,19 +838,6 @@ class SimpleCPUOffloadScheduler:
)
request = state.request
confirmed_tokens = request.num_computed_tokens - request.num_output_placeholders
# Truncate to the granularity a lookup can actually land on. With
# fine-grained hits that is hash_block_size; otherwise hits only land on
# the scheduler block (the group LCM). Using the LCM unconditionally
# would drop whole blocks that sit between the last LCM boundary and a
# reachable fine-grained boundary, so the other groups would hold that
# boundary and this one would not, and the joint hybrid lookup would
# reconcile to zero.
store_alignment = (
self.hash_block_size
if self.cpu_coordinator.enable_partial_hash_hits
else self.block_size
)
aligned_tokens = confirmed_tokens // store_alignment * store_alignment
num_free = self.cpu_block_pool.get_num_free_blocks()
for g, group_gpu_ids in enumerate(block_ids_by_group):
@@ -864,7 +852,11 @@ class SimpleCPUOffloadScheduler:
# num_stored_blocks can be stale and omit evicted blocks in
# the middle of the request.
group_size = self.group_block_sizes[g]
ready = min(len(group_gpu_ids), aligned_tokens // group_size)
# Mimic the GPU KV cache manager's cacheable prefix.
cacheable_tokens = self.cpu_coordinator.get_num_cacheable_tokens(
confirmed_tokens, g
)
ready = min(len(group_gpu_ids), cacheable_tokens // group_size)
resolved_hashes = resolve_block_hashes(
request.block_hashes, self.hash_block_size, group_size
)