[Bugfix][KV Offload] Honor speculative cacheability in SimpleCPU (#60071)
Signed-off-by: Sherif Waly <sherif.waly@mistral.ai>
This commit is contained in:
@@ -6,6 +6,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -17,6 +18,7 @@ from vllm.config import (
|
||||
KVTransferConfig,
|
||||
ModelConfig,
|
||||
SchedulerConfig,
|
||||
SpeculativeConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.config.cache import MambaCacheMode
|
||||
@@ -26,7 +28,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector i
|
||||
)
|
||||
from vllm.utils.hashing import sha256
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
get_request_block_hasher,
|
||||
init_none_hash,
|
||||
@@ -3089,3 +3091,184 @@ def test_lazy_target_blocks_ignore_non_prefix_cacheable_groups() -> None:
|
||||
without_ring, max_batched
|
||||
)
|
||||
assert target_with == target_without
|
||||
|
||||
|
||||
@pytest.mark.parametrize("retention_interval", [None, 0])
|
||||
@pytest.mark.parametrize("chunk_size,prefill_lookahead", [(64, 1), (1024, 1), (1, 3)])
|
||||
@pytest.mark.parametrize(
|
||||
"draft_sliding_window,draft_sharded",
|
||||
[(True, False), (False, False), (False, True)],
|
||||
ids=["swa", "replicated_full", "sharded_full"],
|
||||
)
|
||||
def test_eager_store_matches_gpu_cacheable_prefix(
|
||||
retention_interval: int | None,
|
||||
chunk_size: int,
|
||||
prefill_lookahead: int,
|
||||
draft_sliding_window: bool,
|
||||
draft_sharded: bool,
|
||||
) -> None:
|
||||
"""Preserve GPU hits for replicated SWA and DCP-sharded full-attention drafts."""
|
||||
block_size = 64
|
||||
scheduler_block_size = 8 * block_size
|
||||
draft_block_size = scheduler_block_size if draft_sharded else block_size
|
||||
prompt_len = 16 * scheduler_block_size + draft_block_size + prefill_lookahead
|
||||
kv_cache_config = _make_kv_cache_config(256, num_groups=2)
|
||||
for group_id, group in enumerate(kv_cache_config.kv_cache_groups):
|
||||
group.kv_cache_spec = FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=NUM_KV_HEADS,
|
||||
head_size=HEAD_SIZE,
|
||||
dtype=DTYPE,
|
||||
dcp_sharded=group_id == 0 or draft_sharded,
|
||||
)
|
||||
for tensor in kv_cache_config.kv_cache_tensors:
|
||||
tensor.size *= block_size // BLOCK_SIZE
|
||||
tensor.layer_stride *= block_size // BLOCK_SIZE
|
||||
tensor.block_stride *= block_size // BLOCK_SIZE
|
||||
if draft_sliding_window:
|
||||
kv_cache_config.kv_cache_groups[1].kv_cache_spec = SlidingWindowSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=NUM_KV_HEADS,
|
||||
head_size=HEAD_SIZE,
|
||||
dtype=DTYPE,
|
||||
sliding_window=scheduler_block_size,
|
||||
extra_retained_tokens=prefill_lookahead - 1,
|
||||
dcp_sharded=False,
|
||||
)
|
||||
kv_cache_config.kv_cache_groups[1].is_eagle_group = True
|
||||
kv_cache_config.prefix_cache_retention_interval = retention_interval
|
||||
vllm_config = _make_cp_vllm_config(dcp_world_size=8)
|
||||
vllm_config.model_config.max_model_len = 16384
|
||||
speculative_config = Mock(spec=SpeculativeConfig)
|
||||
speculative_config.num_speculative_tokens = 3
|
||||
speculative_config.use_eagle.return_value = True
|
||||
speculative_config.use_eagle_block_drop.return_value = True
|
||||
speculative_config.use_multi_module_mtp.return_value = prefill_lookahead > 1
|
||||
vllm_config.speculative_config = speculative_config
|
||||
scheduler = SimpleCPUOffloadScheduler(
|
||||
vllm_config=vllm_config,
|
||||
kv_cache_config=kv_cache_config,
|
||||
cpu_capacity_bytes=kv_cache_config.kv_cache_tensors[0].size,
|
||||
scheduler_block_size=scheduler_block_size,
|
||||
hash_block_size=block_size,
|
||||
)
|
||||
gpu_manager = KVCacheManager(
|
||||
kv_cache_config,
|
||||
max_model_len=16384,
|
||||
max_in_flight_tokens=chunk_size,
|
||||
scheduler_block_size=scheduler_block_size,
|
||||
hash_block_size=block_size,
|
||||
dcp_world_size=8,
|
||||
use_eagle=True,
|
||||
num_prefill_lookahead=prefill_lookahead,
|
||||
)
|
||||
scheduler.bind_gpu_block_pool(gpu_manager.block_pool)
|
||||
request = Request(
|
||||
request_id="dcp-draft-source",
|
||||
prompt_token_ids=list(range(prompt_len)),
|
||||
sampling_params=SamplingParams(max_tokens=2),
|
||||
pooling_params=None,
|
||||
mm_features=None,
|
||||
block_hasher=get_request_block_hasher(block_size, sha256),
|
||||
)
|
||||
# Follow allocate -> connector metadata -> compute completion, including
|
||||
# one decode step so the final prefill blocks can be offloaded.
|
||||
while request.num_computed_tokens <= prompt_len:
|
||||
if request.num_computed_tokens == prompt_len:
|
||||
request.append_output_token_ids([prompt_len])
|
||||
num_new_tokens = min(
|
||||
chunk_size, request.num_tokens - request.num_computed_tokens
|
||||
)
|
||||
new_blocks = gpu_manager.allocate_slots(
|
||||
request, num_new_tokens, num_lookahead_tokens=3
|
||||
)
|
||||
assert new_blocks is not None
|
||||
if request.num_computed_tokens == 0:
|
||||
scheduler.update_state_after_alloc(
|
||||
request, gpu_manager.get_blocks(request.request_id), 0
|
||||
)
|
||||
output = make_scheduler_output(
|
||||
{request.request_id: num_new_tokens},
|
||||
new_reqs={request.request_id: new_blocks.get_block_ids()},
|
||||
)
|
||||
else:
|
||||
output = make_scheduler_output(
|
||||
{request.request_id: num_new_tokens},
|
||||
cached_req_new_blocks={request.request_id: new_blocks.get_block_ids()},
|
||||
)
|
||||
store_meta = scheduler.build_connector_meta(output)
|
||||
request.num_computed_tokens += num_new_tokens
|
||||
if store_meta.store_event >= 0:
|
||||
simulate_store_completion(scheduler, store_meta.store_event)
|
||||
|
||||
matching_request = Request(
|
||||
request_id="dcp-draft-load",
|
||||
prompt_token_ids=request.prompt_token_ids,
|
||||
sampling_params=request.sampling_params,
|
||||
pooling_params=None,
|
||||
mm_features=None,
|
||||
block_hasher=request._block_hasher,
|
||||
)
|
||||
expected_hit = (
|
||||
15 if draft_sliding_window and retention_interval is not None else 16
|
||||
) * scheduler_block_size
|
||||
assert gpu_manager.get_computed_blocks(matching_request)[1] == expected_hit
|
||||
scheduler.request_finished(request, [])
|
||||
gpu_manager.free(request)
|
||||
assert gpu_manager.block_pool.reset_prefix_cache()
|
||||
|
||||
hit_tokens, is_async = scheduler.get_num_new_matched_tokens(
|
||||
matching_request, num_computed_tokens=0
|
||||
)
|
||||
assert hit_tokens is not None
|
||||
assert (hit_tokens, is_async) == (expected_hit, True)
|
||||
|
||||
assert (
|
||||
gpu_manager.allocate_slots(
|
||||
matching_request,
|
||||
num_new_tokens=0,
|
||||
num_external_computed_tokens=hit_tokens,
|
||||
delay_cache_blocks=True,
|
||||
)
|
||||
is not None
|
||||
)
|
||||
load_blocks = gpu_manager.get_blocks(matching_request.request_id)
|
||||
scheduler.update_state_after_alloc(matching_request, load_blocks, hit_tokens)
|
||||
load_meta = scheduler.build_connector_meta(
|
||||
make_scheduler_output(
|
||||
{matching_request.request_id: 0},
|
||||
new_reqs={matching_request.request_id: load_blocks.get_block_ids()},
|
||||
)
|
||||
)
|
||||
# SWA loads only the eight-block tail after null padding; a sharded full
|
||||
# draft must load every prefix block, with the same DCP geometry as the target.
|
||||
num_draft_blocks = 8 if draft_sliding_window else hit_tokens // draft_block_size
|
||||
assert load_meta.load_gpu_blocks == [
|
||||
block.block_id
|
||||
for block in (
|
||||
*load_blocks.blocks[0],
|
||||
*load_blocks.blocks[1][-num_draft_blocks:],
|
||||
)
|
||||
]
|
||||
assert (
|
||||
len(load_meta.load_gpu_blocks)
|
||||
== hit_tokens // scheduler_block_size + num_draft_blocks
|
||||
)
|
||||
target_hashes = [
|
||||
make_block_hash_with_group_id(matching_request.block_hashes[(i + 1) * 8 - 1], 0)
|
||||
for i in range(hit_tokens // scheduler_block_size)
|
||||
]
|
||||
draft_hashes = [
|
||||
make_block_hash_with_group_id(
|
||||
matching_request.block_hashes[(i + 1) * draft_block_size // block_size - 1],
|
||||
1,
|
||||
)
|
||||
for i in range(
|
||||
hit_tokens // draft_block_size - num_draft_blocks,
|
||||
hit_tokens // draft_block_size,
|
||||
)
|
||||
]
|
||||
assert [
|
||||
scheduler.cpu_block_pool.blocks[bid].block_hash
|
||||
for bid in load_meta.load_cpu_blocks
|
||||
] == target_hashes + draft_hashes
|
||||
|
||||
@@ -341,6 +341,14 @@ class KVCacheCoordinator(ABC):
|
||||
extension = request.num_prompt_tokens // block * block
|
||||
return tuple(sorted({max(resend - block, 0), max(extension - block, 0)}))
|
||||
|
||||
def get_num_cacheable_tokens(
|
||||
self, num_computed_tokens: int, kv_cache_group_id: int
|
||||
) -> int:
|
||||
"""Return the prefix eligible for hashing in this group."""
|
||||
# Only cache tokens with finalized KV. The last num_reprefillable_tokens
|
||||
# tokens can be re-prefilled during multi-module MTP.
|
||||
return max(0, num_computed_tokens - self.num_reprefillable_tokens)
|
||||
|
||||
def cache_blocks(self, request: Request, num_computed_tokens: int) -> None:
|
||||
"""Cache the blocks for the request.
|
||||
|
||||
@@ -352,17 +360,12 @@ class KVCacheCoordinator(ABC):
|
||||
|
||||
"""
|
||||
boundaries = self.get_replay_boundaries(request)
|
||||
for manager in self.single_type_managers:
|
||||
for group_id, manager in enumerate(self.single_type_managers):
|
||||
if not manager.enable_caching:
|
||||
continue
|
||||
# Only cache tokens with finalized KV. The last num_reprefillable_tokens
|
||||
# tokens can be re-prefilled during multi-module MTP.
|
||||
num_tokens_to_cache = max(
|
||||
0, num_computed_tokens - self.num_reprefillable_tokens
|
||||
)
|
||||
manager.cache_blocks(
|
||||
request,
|
||||
num_tokens_to_cache,
|
||||
self.get_num_cacheable_tokens(num_computed_tokens, group_id),
|
||||
retention_interval=self.retention_interval,
|
||||
replay_boundaries=boundaries,
|
||||
)
|
||||
@@ -809,39 +812,22 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
return num_tokens
|
||||
return round_down(num_tokens, self.scheduler_block_size)
|
||||
|
||||
def cache_blocks(self, request: Request, num_computed_tokens: int) -> None:
|
||||
def get_num_cacheable_tokens(
|
||||
self, num_computed_tokens: int, kv_cache_group_id: int
|
||||
) -> int:
|
||||
cached_num_computed_tokens = self._align_cacheable(num_computed_tokens)
|
||||
boundaries = self.get_replay_boundaries(request)
|
||||
for manager in self.single_type_managers:
|
||||
if not manager.enable_caching:
|
||||
continue
|
||||
num_tokens_to_cache = cached_num_computed_tokens
|
||||
# EAGLE groups match one block past each aligned boundary and drop
|
||||
# it, so make that lookahead block eligible to be cached.
|
||||
if manager.use_eagle and cached_num_computed_tokens > 0:
|
||||
# Only cache tokens with finalized KV. The last
|
||||
# num_reprefillable_tokens tokens can be re-prefilled during
|
||||
# multi-module MTP.
|
||||
num_finalized_computed_tokens = max(
|
||||
0, num_computed_tokens - self.num_reprefillable_tokens
|
||||
)
|
||||
cached_num_finalized_computed_tokens = self._align_cacheable(
|
||||
num_finalized_computed_tokens
|
||||
)
|
||||
num_tokens_to_cache = min(
|
||||
num_finalized_computed_tokens,
|
||||
cached_num_finalized_computed_tokens + manager.block_size,
|
||||
)
|
||||
# The manager already knows the fine hit granularity
|
||||
# (``scheduler_block_size``); retention is passed separately so it
|
||||
# can keep both the coarse segment tails and the fine replay
|
||||
# boundary (which needs the fine value).
|
||||
manager.cache_blocks(
|
||||
request,
|
||||
num_tokens_to_cache,
|
||||
retention_interval=self.retention_interval,
|
||||
replay_boundaries=boundaries,
|
||||
manager = self.single_type_managers[kv_cache_group_id]
|
||||
# EAGLE matches one block past the aligned boundary, then drops it.
|
||||
if manager.use_eagle and cached_num_computed_tokens > 0:
|
||||
num_finalized_computed_tokens = max(
|
||||
0, num_computed_tokens - self.num_reprefillable_tokens
|
||||
)
|
||||
return min(
|
||||
num_finalized_computed_tokens,
|
||||
self._align_cacheable(num_finalized_computed_tokens)
|
||||
+ manager.block_size,
|
||||
)
|
||||
return cached_num_computed_tokens
|
||||
|
||||
def find_longest_cache_hit(
|
||||
self,
|
||||
|
||||
@@ -216,6 +216,7 @@ class SimpleCPUOffloadScheduler:
|
||||
scheduler_block_size=self.block_size,
|
||||
hash_block_size=self.hash_block_size,
|
||||
allow_partial_hash_hits=not lazy_offload,
|
||||
num_prefill_lookahead=vllm_config.num_prefill_lookahead_tokens,
|
||||
)
|
||||
self.group_block_sizes = self.cpu_coordinator.group_block_sizes
|
||||
# FA group's own resolved block_size; divides scheduler_block_size (the
|
||||
@@ -837,19 +838,6 @@ class SimpleCPUOffloadScheduler:
|
||||
)
|
||||
request = state.request
|
||||
confirmed_tokens = request.num_computed_tokens - request.num_output_placeholders
|
||||
# Truncate to the granularity a lookup can actually land on. With
|
||||
# fine-grained hits that is hash_block_size; otherwise hits only land on
|
||||
# the scheduler block (the group LCM). Using the LCM unconditionally
|
||||
# would drop whole blocks that sit between the last LCM boundary and a
|
||||
# reachable fine-grained boundary, so the other groups would hold that
|
||||
# boundary and this one would not, and the joint hybrid lookup would
|
||||
# reconcile to zero.
|
||||
store_alignment = (
|
||||
self.hash_block_size
|
||||
if self.cpu_coordinator.enable_partial_hash_hits
|
||||
else self.block_size
|
||||
)
|
||||
aligned_tokens = confirmed_tokens // store_alignment * store_alignment
|
||||
num_free = self.cpu_block_pool.get_num_free_blocks()
|
||||
|
||||
for g, group_gpu_ids in enumerate(block_ids_by_group):
|
||||
@@ -864,7 +852,11 @@ class SimpleCPUOffloadScheduler:
|
||||
# num_stored_blocks can be stale and omit evicted blocks in
|
||||
# the middle of the request.
|
||||
group_size = self.group_block_sizes[g]
|
||||
ready = min(len(group_gpu_ids), aligned_tokens // group_size)
|
||||
# Mimic the GPU KV cache manager's cacheable prefix.
|
||||
cacheable_tokens = self.cpu_coordinator.get_num_cacheable_tokens(
|
||||
confirmed_tokens, g
|
||||
)
|
||||
ready = min(len(group_gpu_ids), cacheable_tokens // group_size)
|
||||
resolved_hashes = resolve_block_hashes(
|
||||
request.block_hashes, self.hash_block_size, group_size
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user