[Bugfix][KV Offload] Honor speculative cacheability in SimpleCPU (#60071)
Signed-off-by: Sherif Waly <sherif.waly@mistral.ai>
This commit is contained in:
@@ -6,6 +6,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -17,6 +18,7 @@ from vllm.config import (
|
||||
KVTransferConfig,
|
||||
ModelConfig,
|
||||
SchedulerConfig,
|
||||
SpeculativeConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.config.cache import MambaCacheMode
|
||||
@@ -26,7 +28,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector i
|
||||
)
|
||||
from vllm.utils.hashing import sha256
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
get_request_block_hasher,
|
||||
init_none_hash,
|
||||
@@ -3089,3 +3091,184 @@ def test_lazy_target_blocks_ignore_non_prefix_cacheable_groups() -> None:
|
||||
without_ring, max_batched
|
||||
)
|
||||
assert target_with == target_without
|
||||
|
||||
|
||||
@pytest.mark.parametrize("retention_interval", [None, 0])
|
||||
@pytest.mark.parametrize("chunk_size,prefill_lookahead", [(64, 1), (1024, 1), (1, 3)])
|
||||
@pytest.mark.parametrize(
|
||||
"draft_sliding_window,draft_sharded",
|
||||
[(True, False), (False, False), (False, True)],
|
||||
ids=["swa", "replicated_full", "sharded_full"],
|
||||
)
|
||||
def test_eager_store_matches_gpu_cacheable_prefix(
|
||||
retention_interval: int | None,
|
||||
chunk_size: int,
|
||||
prefill_lookahead: int,
|
||||
draft_sliding_window: bool,
|
||||
draft_sharded: bool,
|
||||
) -> None:
|
||||
"""Preserve GPU hits for replicated SWA and DCP-sharded full-attention drafts."""
|
||||
block_size = 64
|
||||
scheduler_block_size = 8 * block_size
|
||||
draft_block_size = scheduler_block_size if draft_sharded else block_size
|
||||
prompt_len = 16 * scheduler_block_size + draft_block_size + prefill_lookahead
|
||||
kv_cache_config = _make_kv_cache_config(256, num_groups=2)
|
||||
for group_id, group in enumerate(kv_cache_config.kv_cache_groups):
|
||||
group.kv_cache_spec = FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=NUM_KV_HEADS,
|
||||
head_size=HEAD_SIZE,
|
||||
dtype=DTYPE,
|
||||
dcp_sharded=group_id == 0 or draft_sharded,
|
||||
)
|
||||
for tensor in kv_cache_config.kv_cache_tensors:
|
||||
tensor.size *= block_size // BLOCK_SIZE
|
||||
tensor.layer_stride *= block_size // BLOCK_SIZE
|
||||
tensor.block_stride *= block_size // BLOCK_SIZE
|
||||
if draft_sliding_window:
|
||||
kv_cache_config.kv_cache_groups[1].kv_cache_spec = SlidingWindowSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=NUM_KV_HEADS,
|
||||
head_size=HEAD_SIZE,
|
||||
dtype=DTYPE,
|
||||
sliding_window=scheduler_block_size,
|
||||
extra_retained_tokens=prefill_lookahead - 1,
|
||||
dcp_sharded=False,
|
||||
)
|
||||
kv_cache_config.kv_cache_groups[1].is_eagle_group = True
|
||||
kv_cache_config.prefix_cache_retention_interval = retention_interval
|
||||
vllm_config = _make_cp_vllm_config(dcp_world_size=8)
|
||||
vllm_config.model_config.max_model_len = 16384
|
||||
speculative_config = Mock(spec=SpeculativeConfig)
|
||||
speculative_config.num_speculative_tokens = 3
|
||||
speculative_config.use_eagle.return_value = True
|
||||
speculative_config.use_eagle_block_drop.return_value = True
|
||||
speculative_config.use_multi_module_mtp.return_value = prefill_lookahead > 1
|
||||
vllm_config.speculative_config = speculative_config
|
||||
scheduler = SimpleCPUOffloadScheduler(
|
||||
vllm_config=vllm_config,
|
||||
kv_cache_config=kv_cache_config,
|
||||
cpu_capacity_bytes=kv_cache_config.kv_cache_tensors[0].size,
|
||||
scheduler_block_size=scheduler_block_size,
|
||||
hash_block_size=block_size,
|
||||
)
|
||||
gpu_manager = KVCacheManager(
|
||||
kv_cache_config,
|
||||
max_model_len=16384,
|
||||
max_in_flight_tokens=chunk_size,
|
||||
scheduler_block_size=scheduler_block_size,
|
||||
hash_block_size=block_size,
|
||||
dcp_world_size=8,
|
||||
use_eagle=True,
|
||||
num_prefill_lookahead=prefill_lookahead,
|
||||
)
|
||||
scheduler.bind_gpu_block_pool(gpu_manager.block_pool)
|
||||
request = Request(
|
||||
request_id="dcp-draft-source",
|
||||
prompt_token_ids=list(range(prompt_len)),
|
||||
sampling_params=SamplingParams(max_tokens=2),
|
||||
pooling_params=None,
|
||||
mm_features=None,
|
||||
block_hasher=get_request_block_hasher(block_size, sha256),
|
||||
)
|
||||
# Follow allocate -> connector metadata -> compute completion, including
|
||||
# one decode step so the final prefill blocks can be offloaded.
|
||||
while request.num_computed_tokens <= prompt_len:
|
||||
if request.num_computed_tokens == prompt_len:
|
||||
request.append_output_token_ids([prompt_len])
|
||||
num_new_tokens = min(
|
||||
chunk_size, request.num_tokens - request.num_computed_tokens
|
||||
)
|
||||
new_blocks = gpu_manager.allocate_slots(
|
||||
request, num_new_tokens, num_lookahead_tokens=3
|
||||
)
|
||||
assert new_blocks is not None
|
||||
if request.num_computed_tokens == 0:
|
||||
scheduler.update_state_after_alloc(
|
||||
request, gpu_manager.get_blocks(request.request_id), 0
|
||||
)
|
||||
output = make_scheduler_output(
|
||||
{request.request_id: num_new_tokens},
|
||||
new_reqs={request.request_id: new_blocks.get_block_ids()},
|
||||
)
|
||||
else:
|
||||
output = make_scheduler_output(
|
||||
{request.request_id: num_new_tokens},
|
||||
cached_req_new_blocks={request.request_id: new_blocks.get_block_ids()},
|
||||
)
|
||||
store_meta = scheduler.build_connector_meta(output)
|
||||
request.num_computed_tokens += num_new_tokens
|
||||
if store_meta.store_event >= 0:
|
||||
simulate_store_completion(scheduler, store_meta.store_event)
|
||||
|
||||
matching_request = Request(
|
||||
request_id="dcp-draft-load",
|
||||
prompt_token_ids=request.prompt_token_ids,
|
||||
sampling_params=request.sampling_params,
|
||||
pooling_params=None,
|
||||
mm_features=None,
|
||||
block_hasher=request._block_hasher,
|
||||
)
|
||||
expected_hit = (
|
||||
15 if draft_sliding_window and retention_interval is not None else 16
|
||||
) * scheduler_block_size
|
||||
assert gpu_manager.get_computed_blocks(matching_request)[1] == expected_hit
|
||||
scheduler.request_finished(request, [])
|
||||
gpu_manager.free(request)
|
||||
assert gpu_manager.block_pool.reset_prefix_cache()
|
||||
|
||||
hit_tokens, is_async = scheduler.get_num_new_matched_tokens(
|
||||
matching_request, num_computed_tokens=0
|
||||
)
|
||||
assert hit_tokens is not None
|
||||
assert (hit_tokens, is_async) == (expected_hit, True)
|
||||
|
||||
assert (
|
||||
gpu_manager.allocate_slots(
|
||||
matching_request,
|
||||
num_new_tokens=0,
|
||||
num_external_computed_tokens=hit_tokens,
|
||||
delay_cache_blocks=True,
|
||||
)
|
||||
is not None
|
||||
)
|
||||
load_blocks = gpu_manager.get_blocks(matching_request.request_id)
|
||||
scheduler.update_state_after_alloc(matching_request, load_blocks, hit_tokens)
|
||||
load_meta = scheduler.build_connector_meta(
|
||||
make_scheduler_output(
|
||||
{matching_request.request_id: 0},
|
||||
new_reqs={matching_request.request_id: load_blocks.get_block_ids()},
|
||||
)
|
||||
)
|
||||
# SWA loads only the eight-block tail after null padding; a sharded full
|
||||
# draft must load every prefix block, with the same DCP geometry as the target.
|
||||
num_draft_blocks = 8 if draft_sliding_window else hit_tokens // draft_block_size
|
||||
assert load_meta.load_gpu_blocks == [
|
||||
block.block_id
|
||||
for block in (
|
||||
*load_blocks.blocks[0],
|
||||
*load_blocks.blocks[1][-num_draft_blocks:],
|
||||
)
|
||||
]
|
||||
assert (
|
||||
len(load_meta.load_gpu_blocks)
|
||||
== hit_tokens // scheduler_block_size + num_draft_blocks
|
||||
)
|
||||
target_hashes = [
|
||||
make_block_hash_with_group_id(matching_request.block_hashes[(i + 1) * 8 - 1], 0)
|
||||
for i in range(hit_tokens // scheduler_block_size)
|
||||
]
|
||||
draft_hashes = [
|
||||
make_block_hash_with_group_id(
|
||||
matching_request.block_hashes[(i + 1) * draft_block_size // block_size - 1],
|
||||
1,
|
||||
)
|
||||
for i in range(
|
||||
hit_tokens // draft_block_size - num_draft_blocks,
|
||||
hit_tokens // draft_block_size,
|
||||
)
|
||||
]
|
||||
assert [
|
||||
scheduler.cpu_block_pool.blocks[bid].block_hash
|
||||
for bid in load_meta.load_cpu_blocks
|
||||
] == target_hashes + draft_hashes
|
||||
|
||||
Reference in New Issue
Block a user