[Bugfix][Prefix caching] Fix inconsistent hybrid Mamba prefix cache boundaries (#60533)

Signed-off-by: Wei Zhao <weizha@aws-cmh-slurm-1-login-01.cm.cluster>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Wei Zhao <weizha@aws-cmh-slurm-1-login-01.cm.cluster>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Wei Zhao
2026-10-08 18:33:48 +00:00
committed by GitHub
co-authored by Wei Zhao Codex zjy0516 Claude Opus 5.5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 1664c4f444
commit d8fa463e2b
17 changed files with 546 additions and 200 deletions
@@ -22,6 +22,7 @@ from vllm.v1.core.kv_cache_utils import (
get_block_hash,
get_group_id,
init_none_hash,
resolve_cache_hit_alignment_tokens,
)
from vllm.v1.core.sched.scheduler import Scheduler
from vllm.v1.kv_cache_interface import (
@@ -148,29 +149,84 @@ def make_full_mamba_manager(
)
def test_dcp_fine_hit_retention_uses_hash_alignment_without_eagle():
"""DCP must not discard a reusable Mamba state at hash alignment."""
hash_block_size = 2
@pytest.mark.parametrize(
"dcp_world_size,hash_block_size,block_size,prompt_tokens,checkpoint_blocks",
[
(4, 2, 2, 7, 0),
(8, 16, 16, 512, 1),
(8, 16, 16, 513, 1),
*[(8, 128, 896, p, 1) for p in (127, 128, 129, 7040, 7168, 7296, 7449)],
],
)
@pytest.mark.parametrize("use_eagle", [False, True])
@pytest.mark.parametrize("extension_tokens", [0, 32])
def test_dcp_fine_hit_retention_uses_hash_alignment(
dcp_world_size,
hash_block_size,
block_size,
prompt_tokens,
checkpoint_blocks,
use_eagle,
extension_tokens,
):
"""Resends and extensions reuse the retained hash-aligned checkpoint."""
manager = make_full_mamba_manager(
dcp_world_size=4,
dcp_world_size=dcp_world_size,
hash_block_size=hash_block_size,
full_block_size=hash_block_size,
mamba_block_size=hash_block_size,
num_blocks=64,
full_block_size=block_size,
mamba_block_size=block_size,
num_blocks=128,
use_eagle=use_eagle,
num_speculative_blocks=4 if use_eagle else 0,
num_prefill_checkpoint_blocks=checkpoint_blocks,
)
manager.coordinator.retention_interval = 0
token_ids = list(range(7))
producer = make_request("producer", token_ids, hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(producer)
assert num_computed == 0
assert manager.allocate_slots(producer, 6, 0, computed_blocks) is not None
assert manager.coordinator.enable_partial_hash_hits
scheduler = SimpleNamespace(
block_size=block_size * dcp_world_size,
cache_config=SimpleNamespace(block_size=block_size),
hash_block_size=hash_block_size,
max_num_scheduled_tokens=8192,
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
use_eagle_block_drop=use_eagle,
mamba_partial_cache_hit=True,
mamba_shared_prefix_checkpoint=False,
mamba_has_prefill_checkpoint_blocks=bool(checkpoint_blocks),
mamba_prefill_checkpoint_alignment=16,
)
producer = make_request(
"producer", list(range(prompt_tokens)), hash_block_size, sha256
)
while producer.num_computed_tokens < prompt_tokens:
scheduled = Scheduler._mamba_block_aligned_split(
scheduler, producer, prompt_tokens - producer.num_computed_tokens
)
assert scheduled > 0
manager.new_step_starts()
assert manager.allocate_slots(producer, scheduled) is not None
producer.num_computed_tokens += scheduled
manager.free(producer)
manager.new_step_starts()
consumer = make_request("consumer", token_ids, hash_block_size, sha256)
_, num_computed, _ = manager.get_computed_blocks(consumer)
assert num_computed == 6
replay = make_request(
"replay", list(range(prompt_tokens + extension_tokens)), hash_block_size, sha256
)
_, hit, _ = manager.get_computed_blocks(replay)
expected = (
max((prompt_tokens // hash_block_size - 1) * hash_block_size, 0)
if use_eagle
else ((prompt_tokens - 1) // hash_block_size * hash_block_size)
)
assert hit == expected
cached_states = {
end
for end in range(hash_block_size, prompt_tokens + 1, hash_block_size)
if manager.block_pool.get_cached_block(
producer.block_hashes[end // hash_block_size - 1], [1]
)
is not None
}
assert cached_states == ({expected} if expected else set())
@pytest.mark.parametrize("dcp_world_size", [1, 4])
@@ -339,7 +395,7 @@ def test_hybrid_mamba_align_partial_hash_hit():
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert num_computed == 0
blocks = manager.allocate_slots(req0, 6, num_computed, computed_blocks)
@@ -488,7 +544,7 @@ def test_hybrid_mamba_partial_tail_owner_uses_cow_on_continue():
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert num_computed == 0
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
@@ -538,7 +594,7 @@ def test_partial_hit_then_internal_checkpoint_uses_distinct_mamba_blocks():
num_prefill_checkpoint_blocks=1,
)
owner = make_request("owner", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
owner = make_request("owner", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(owner)
assert manager.allocate_slots(owner, 6, num_computed, computed_blocks) is not None
manager.free(owner)
@@ -619,44 +675,70 @@ def test_internal_checkpoint_uses_partial_hash_lifecycle():
assert num_computed == 112
@pytest.mark.parametrize(
"retention_interval,transient_published",
[(None, True), (0, False)],
)
@pytest.mark.parametrize("retention_interval", [None, 0, 32, 64, 256])
@pytest.mark.parametrize("hash_block_size", [16, 32])
@pytest.mark.parametrize("use_eagle", [False, True])
def test_internal_checkpoint_publication_respects_retention(
retention_interval, transient_published
retention_interval, hash_block_size, use_eagle
):
"""With retention_interval=0, a mid-prompt internal checkpoint is
request-local and must not enter the prefix cache; the prompt-end
checkpoint keeps the existing publication behavior. Dense retention
(None) is unchanged.
"""
hash_block_size = 16
"""Publish only retained block boundaries and the final prompt checkpoint."""
manager = make_full_mamba_manager(
dcp_world_size=1,
hash_block_size=hash_block_size,
full_block_size=hash_block_size,
mamba_block_size=32,
num_prefill_checkpoint_blocks=1,
use_eagle=use_eagle,
)
manager.coordinator.retention_interval = retention_interval
request = make_request("producer", list(range(240)), hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(request)
# Mid-prompt chunk: exports a transient checkpoint at 112.
# The EAGLE adjustment can make a chunk checkpoint block-aligned or partial.
assert manager.allocate_slots(request, 128, num_computed, computed_blocks)
transient_hash = request.block_hashes[112 // hash_block_size - 1]
checkpoint = (128 if use_eagle else 127) // hash_block_size * hash_block_size - (
hash_block_size if use_eagle else 0
)
retained_checkpoint = checkpoint % 32 == 0 and (
retention_interval is None
or (retention_interval > 0 and checkpoint % retention_interval == 0)
)
transient_hash = request.block_hashes[checkpoint // hash_block_size - 1]
transient_hit = manager.block_pool.get_cached_block(transient_hash, [1])
assert (transient_hit is not None) == transient_published
assert (transient_hit is not None) == retained_checkpoint
offloads = drain_boundary_state_offloads(manager).get(request.request_id, [])
assert any(position == checkpoint for _, _, position in offloads) == (
retained_checkpoint
)
assert all(
position % 32 == 0
and (
retention_interval is None
or (retention_interval > 0 and position % retention_interval == 0)
)
for _, _, position in offloads
)
if retention_interval in (None, 32, 64):
retained_hash = request.block_hashes[128 // hash_block_size - 1]
assert manager.block_pool.get_cached_block(retained_hash, [1]) is not None
request.num_computed_tokens = 128
manager.new_step_starts()
# Final chunk: the prompt-end checkpoint at 224 stays published.
# The final checkpoint is reusable regardless of the retention interval.
assert manager.allocate_slots(request, 112) is not None
end_hash = request.block_hashes[224 // hash_block_size - 1]
prompt_checkpoint = (
240 if use_eagle else 239
) // hash_block_size * hash_block_size - (hash_block_size if use_eagle else 0)
end_hash = request.block_hashes[prompt_checkpoint // hash_block_size - 1]
assert manager.block_pool.get_cached_block(end_hash, [1]) is not None
assert any(
position == prompt_checkpoint
for _, _, position in drain_boundary_state_offloads(manager).get(
request.request_id, []
)
)
def test_transient_checkpoint_evicts_retained_boundary_hash():
@@ -802,9 +884,9 @@ def test_hash_aligned_query_end_uses_regular_partial_tail():
mamba_block_size=mamba_block_size,
num_prefill_checkpoint_blocks=1,
)
request = make_request("producer", list(range(14)), hash_block_size, sha256)
request = make_request("producer", list(range(15)), hash_block_size, sha256)
new_blocks = manager.allocate_slots(request, request.num_tokens)
new_blocks = manager.allocate_slots(request, 14)
assert new_blocks is not None
mamba_manager = manager.coordinator.single_type_managers[1]
@@ -928,7 +1010,7 @@ def test_boundary_state_offloads_returns_cow_target():
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
@@ -1004,7 +1086,7 @@ def test_finished_partial_tail_uses_table_source_once():
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
source_block_id = manager.get_blocks("0").get_block_ids()[1][1]
@@ -1024,7 +1106,7 @@ def test_finished_partial_tail_uses_table_source_once():
enable_caching=True,
hash_block_size=hash_block_size,
)
req1 = make_request("1", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = advanced_manager.get_computed_blocks(req1)
assert (
advanced_manager.allocate_slots(req1, 6, num_computed, computed_blocks)
@@ -1212,7 +1294,7 @@ def test_truncate_computed_blocks_preserves_sparse_prefix_positions():
enable_caching=True,
hash_block_size=hash_block_size,
)
producer = make_request("producer", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
producer = make_request("producer", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
blocks, num_computed, _ = manager.get_computed_blocks(producer)
assert manager.allocate_slots(producer, 6, num_computed, blocks) is not None
manager.free(producer)
@@ -1267,7 +1349,7 @@ def test_truncate_computed_blocks_allows_short_mamba_group_only():
enable_caching=True,
hash_block_size=hash_block_size,
)
producer = make_request("producer", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
producer = make_request("producer", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
blocks, num_computed, _ = manager.get_computed_blocks(producer)
assert manager.allocate_slots(producer, 6, num_computed, blocks) is not None
manager.free(producer)
@@ -1331,7 +1413,7 @@ def test_hybrid_mamba_partial_tail_owner_continue_preserves_later_hit():
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert num_computed == 0
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
@@ -1411,7 +1493,7 @@ def test_hybrid_mamba_moved_partial_entry_defers_same_step_hit():
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert num_computed == 0
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
@@ -1470,7 +1552,7 @@ def test_hybrid_full_attention_partial_hash_hit_uses_cow():
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert num_computed == 0
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
@@ -1544,7 +1626,7 @@ def test_hybrid_partial_hit_cow_target_starts_uncached():
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert num_computed == 0
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
@@ -1700,7 +1782,7 @@ def test_cow_retained_blocks_returned_for_release():
enable_caching=True,
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
@@ -2133,6 +2215,72 @@ def test_hybrid_sliding_window_group_disables_partial_hash_hits():
assert len(computed_blocks.blocks[0]) * hash_block_size == num_computed
@pytest.mark.parametrize("with_sliding_window", [False, True])
def test_resolved_cache_hit_alignment_matches_coordinator(with_sliding_window):
"""Workers place Mamba checkpoints on the engine-resolved alignment, so it
must be where the coordinator lands hits, including when a group without
fine-grained lookups disables partial hits."""
hash_block_size = 2
mamba_block_size = 4 * hash_block_size
kv_cache_groups = [
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=hash_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=mamba_block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
]
if with_sliding_window:
kv_cache_groups.append(
KVCacheGroupSpec(
["swa"],
SlidingWindowSpec(
block_size=2 * hash_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=2 * hash_block_size,
),
)
)
kv_cache_config = KVCacheConfig(
num_blocks=64, kv_cache_tensors=[], kv_cache_groups=kv_cache_groups
)
coordinator = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
).coordinator
vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(enable_prefix_caching=True),
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
)
assert coordinator.enable_partial_hash_hits != with_sliding_window
assert (
resolve_cache_hit_alignment_tokens(
kv_cache_config,
vllm_config,
coordinator.scheduler_block_size,
hash_block_size,
)
== coordinator._cache_hit_alignment_tokens
)
def test_opted_out_scratch_group_keeps_partial_hash_hits():
hash_block_size = 2
mamba_block_size = 2 * hash_block_size
@@ -2179,7 +2327,7 @@ def test_opted_out_scratch_group_keeps_partial_hash_hits():
assert manager.coordinator.enable_partial_hash_hits
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
manager.free(req0)
@@ -2240,7 +2388,7 @@ def test_kpool_tail_supports_128_token_partial_hash_hits():
assert manager.coordinator.enable_partial_hash_hits
shared_prefix = [10] * (12 * hash_block_size)
req0 = make_request("0", shared_prefix, hash_block_size, sha256)
req0 = make_request("0", shared_prefix + [11], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert (
manager.allocate_slots(req0, len(shared_prefix), num_computed, computed_blocks)
@@ -2266,7 +2414,7 @@ def test_hybrid_partial_hash_hit_uses_cow_under_dcp(dcp_world_size: int):
)
assert manager.coordinator.enable_partial_hash_hits
req0 = make_request("dcp-owner", [0, 0, 1, 1, 2, 2], 2, sha256)
req0 = make_request("dcp-owner", [0, 0, 1, 1, 2, 2, 3], 2, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
manager.free(req0)
@@ -2318,7 +2466,7 @@ def test_dcp_partial_hit_resumes_on_replicated_mamba_snapshot(
assert manager.coordinator.single_type_managers[1].block_size == block_size
prefix = list(range(12))
req0 = make_request("snapshot-owner", prefix, block_size, sha256)
req0 = make_request("snapshot-owner", prefix + [12], block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 12, num_computed, computed_blocks) is not None
manager.free(req0)
@@ -2355,7 +2503,7 @@ def test_dcp_joint_hit_is_bounded_by_replicated_mamba_snapshots():
mamba_block_size=block_size,
)
prefix = list(range(12))
req0 = make_request("joint-owner", prefix, block_size, sha256)
req0 = make_request("joint-owner", prefix + [12], block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 8, num_computed, computed_blocks) is not None
manager.new_step_starts()
@@ -2702,6 +2850,7 @@ def test_checkpoint_reservation_matches_worker(
mamba_block_size=mamba_block_size,
checkpoint_alignment=16,
drop_eagle_block=use_eagle,
cache_hit_alignment_tokens=mamba_manager.cache_hit_alignment_tokens,
)
reserved = mamba_manager._checkpoints.get(request.request_id)
if reserved is not None: