[Bugfix][Core] Retire stale Mamba spec blocks in prefill checkpoint steps (#59759)
Signed-off-by: Simon Veitner <sveitner@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Robert Shaw
parent
2b32d9ac6a
commit
6fa67afed9
@@ -6,10 +6,14 @@ import random
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils.hashing import sha256
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
KVCacheBlock,
|
||||
get_request_block_hasher,
|
||||
init_none_hash,
|
||||
make_block_hash_with_group_id,
|
||||
)
|
||||
from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
@@ -30,6 +34,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
RSWASpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.request import Request
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
@@ -238,6 +243,71 @@ def test_mamba_checkpoint_admission_matches_allocation(
|
||||
assert admission_estimate == allocation_estimate == allocated
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_speculative_blocks", [1, 2, 3])
|
||||
@pytest.mark.parametrize("first_chunk_blocks", [2, 4])
|
||||
@pytest.mark.parametrize("retention_interval", [0, None])
|
||||
def test_mamba_checkpoint_hash_maps_only_to_checkpoint_block(
|
||||
num_speculative_blocks, first_chunk_blocks, retention_interval
|
||||
):
|
||||
"""A checkpoint step must not leave a never-written speculative block
|
||||
cached."""
|
||||
block_size = 128
|
||||
spec = MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
num_speculative_blocks=num_speculative_blocks,
|
||||
num_prefill_checkpoint_blocks=1,
|
||||
prefill_checkpoint_alignment=16,
|
||||
)
|
||||
pool = BlockPool(num_gpu_blocks=64, enable_caching=True, hash_block_size=128)
|
||||
manager = MambaManager(
|
||||
spec,
|
||||
block_pool=pool,
|
||||
enable_caching=True,
|
||||
kv_cache_group_id=0,
|
||||
scheduler_block_size=block_size,
|
||||
)
|
||||
# MTP/EAGLE moves the checkpoint one block below the last full block.
|
||||
manager.drop_eagle_checkpoint_block = True
|
||||
init_none_hash(sha256)
|
||||
prompt_tokens = 6 * block_size + 50
|
||||
request = Request(
|
||||
request_id="r",
|
||||
prompt_token_ids=list(range(prompt_tokens)),
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
pooling_params=None,
|
||||
block_hasher=get_request_block_hasher(block_size, sha256),
|
||||
)
|
||||
|
||||
def step(start, end):
|
||||
estimate = manager.get_num_blocks_to_allocate("r", end, [], start, start, end)
|
||||
free_before = pool.get_num_free_blocks()
|
||||
manager.allocate_new_blocks("r", end, end)
|
||||
assert free_before - pool.get_num_free_blocks() == estimate
|
||||
manager.cache_blocks(
|
||||
request,
|
||||
end,
|
||||
retention_interval=retention_interval,
|
||||
replay_boundaries=[5 * block_size],
|
||||
)
|
||||
request.num_computed_tokens = end
|
||||
|
||||
step(0, first_chunk_blocks * block_size)
|
||||
speculative_blocks = manager.req_to_blocks["r"][-num_speculative_blocks:]
|
||||
manager.new_step_starts()
|
||||
step(first_chunk_blocks * block_size, prompt_tokens)
|
||||
|
||||
checkpoint_position, checkpoint_idx = manager._checkpoints["r"]
|
||||
checkpoint_block = manager.req_to_blocks["r"][checkpoint_idx]
|
||||
assert all(
|
||||
b.block_hash is None or b is checkpoint_block for b in speculative_blocks
|
||||
)
|
||||
block_hash = request.block_hashes[checkpoint_position // block_size - 1]
|
||||
assert pool.get_cached_block(block_hash, [0]) == [checkpoint_block]
|
||||
|
||||
|
||||
def get_sliding_window_manager(
|
||||
sliding_window_spec,
|
||||
block_pool,
|
||||
|
||||
@@ -1812,7 +1812,7 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
if num_new_blocks > 0:
|
||||
blocks_allocated = request_id in self._allocated_block_reqs
|
||||
physical_block_cap = 1 + int(has_partial_hit) + checkpoint_block
|
||||
if not blocks_allocated or checkpoint_block:
|
||||
if not blocks_allocated:
|
||||
physical_block_cap += self.num_speculative_blocks
|
||||
num_new_blocks = min(num_new_blocks, physical_block_cap)
|
||||
|
||||
@@ -1885,12 +1885,14 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
[self._null_block for _ in range(prev_block_len, null_end)]
|
||||
)
|
||||
|
||||
if blocks_allocated and not checkpoint_block:
|
||||
# Relocate exclusively owned speculative scratch blocks.
|
||||
if blocks_allocated:
|
||||
# Relocate exclusively owned scratch blocks this step leaves
|
||||
# behind. In a checkpoint step, one at the checkpoint index
|
||||
# stays: the worker exports the checkpoint into that column.
|
||||
for block_idx in range(
|
||||
prev_block_len - self.num_speculative_blocks, prev_block_len
|
||||
):
|
||||
if block_idx < num_skipped_blocks:
|
||||
if block_idx < num_skipped_blocks - checkpoint_block:
|
||||
self._relocate_speculative_block(req_blocks, block_idx)
|
||||
else:
|
||||
break
|
||||
@@ -1898,7 +1900,7 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
if has_partial_hit:
|
||||
num_new_blocks = max(num_new_blocks, 0) + 1
|
||||
max_new_blocks = 1 + int(has_partial_hit) + checkpoint_block
|
||||
if not blocks_allocated or checkpoint_block:
|
||||
if not blocks_allocated:
|
||||
max_new_blocks += self.num_speculative_blocks
|
||||
assert num_new_blocks <= max_new_blocks
|
||||
new_blocks = self.block_pool.get_new_blocks(num_new_blocks)
|
||||
|
||||
Reference in New Issue
Block a user