[Bugfix][Prefix caching] Fix inconsistent hybrid Mamba prefix cache boundaries (#60533)

Signed-off-by: Wei Zhao <weizha@aws-cmh-slurm-1-login-01.cm.cluster>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Wei Zhao <weizha@aws-cmh-slurm-1-login-01.cm.cluster>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Wei Zhao
2026-10-08 18:33:48 +00:00
committed by GitHub
co-authored by Wei Zhao Codex zjy0516 Claude Opus 5.5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 1664c4f444
commit d8fa463e2b
17 changed files with 546 additions and 200 deletions
@@ -75,7 +75,7 @@ def _builder(
vllm_config.cache_config.prefix_match_unit = prefix_match_unit
if num_spec:
vllm_config.speculative_config = _StubSpeculativeConfig(num_spec)
return KimiK3ROCmKDAMetadataBuilder(
builder = KimiK3ROCmKDAMetadataBuilder(
kv_cache_spec=MambaSpec(
block_size=block_size,
shapes=((16, 64),),
@@ -91,6 +91,11 @@ def _builder(
vllm_config=vllm_config,
device=torch.device("cuda"),
)
vllm_config.cache_config.hash_block_size = prefix_match_unit or block_size
vllm_config.cache_config.cache_hit_alignment_tokens = (
prefix_match_unit or block_size
)
return builder
def _build(batch: BatchSpec, block_size: int = BLOCK_SIZE, **kw):
+52 -8
View File
@@ -100,6 +100,8 @@ def _make_builder(
prefix_match_unit: int | None = None,
use_eagle: bool = False,
disable_eagle_block_drop: bool = False,
cache_hit_alignment_tokens: int | None = None,
hash_block_size: int | None = None,
) -> AttentionMetadataBuilder:
vllm_config = create_vllm_config(
model_name="Qwen/Qwen3.5-0.8B",
@@ -122,6 +124,11 @@ def _make_builder(
vllm_config.cache_config.use_replayssm = use_recoverssm
vllm_config.cache_config.use_kda_recoverssm = use_recoverssm
vllm_config.cache_config.prefix_match_unit = prefix_match_unit
hash_block_size = hash_block_size or prefix_match_unit or mamba_block_size
vllm_config.cache_config.hash_block_size = hash_block_size
vllm_config.cache_config.cache_hit_alignment_tokens = (
cache_hit_alignment_tokens or hash_block_size
)
builder = builder_cls(
kv_cache_spec=MambaSpec(
block_size=mamba_block_size,
@@ -220,6 +227,8 @@ def test_kda_recoverssm_startup_metadata_flow_without_model(monkeypatch):
vllm_config=builder_config,
device=DEVICE,
)
builder_config.cache_config.hash_block_size = BLOCK_SIZE
builder_config.cache_config.cache_hit_alignment_tokens = BLOCK_SIZE
# An all-prefill speculative batch used to leave an all-false spec mask
# alive, then access active_non_spec_mask_cpu before it was initialized.
@@ -287,18 +296,45 @@ def test_kda_recoverssm_startup_metadata_flow_without_model(monkeypatch):
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_internal_checkpoint_metadata_targets_last_aligned_boundary():
@pytest.mark.parametrize(
"batch,prefix_match_unit,hit_alignment,expected_offsets,expected_states",
[
(
BatchSpec(seq_lens=[50, 32], query_lens=[50, 16]),
None,
16,
[48, 0],
[2, NULL_BLOCK_ID],
),
(
BatchSpec(seq_lens=[50, 32], query_lens=[50, 16]),
16,
16,
[48, 0],
[2, NULL_BLOCK_ID],
),
(BatchSpec(seq_lens=[530], query_lens=[530]), None, 128, [512], [32]),
(BatchSpec(seq_lens=[530], query_lens=[50]), None, 128, [32], [32]),
(BatchSpec(seq_lens=[530], query_lens=[530]), 16, 16, [528], [32]),
(BatchSpec(seq_lens=[530], query_lens=[50]), 16, 16, [48], [32]),
],
)
def test_internal_checkpoint_metadata_targets_last_aligned_boundary(
batch, prefix_match_unit, hit_alignment, expected_offsets, expected_states
):
device = torch.device("cuda")
batch = BatchSpec(seq_lens=[50, 32], query_lens=[50, 16])
common_attn_metadata = create_common_attn_metadata(
batch, BLOCK_SIZE, device, arange_block_indices=True
).replace(is_prefilling=torch.tensor([True, True]))
).replace(is_prefilling=torch.tensor([True] * len(batch.seq_lens)))
builder = _make_builder(
KimiK3KDAMetadataBuilder,
num_speculative_tokens=0,
full_cuda_graph=False,
mamba_cache_mode="align",
num_prefill_checkpoint_blocks=1,
prefix_match_unit=prefix_match_unit,
cache_hit_alignment_tokens=hit_alignment,
hash_block_size=16,
device=device,
)
assert isinstance(builder, KimiK3KDAMetadataBuilder)
@@ -313,26 +349,33 @@ def test_internal_checkpoint_metadata_targets_last_aligned_boundary():
assert actual.checkpoint is not None
torch.testing.assert_close(
actual.checkpoint.state_indices,
torch.tensor([2, NULL_BLOCK_ID], dtype=torch.int32, device=device),
torch.tensor(expected_states, dtype=torch.int32, device=device),
)
torch.testing.assert_close(
actual.checkpoint.checkpoint_offsets,
torch.tensor([48, 0], dtype=torch.int32, device=device),
torch.tensor(expected_offsets, dtype=torch.int32, device=device),
)
@pytest.mark.parametrize(
("disable_eagle_block_drop", "prefix_match_unit", "expected_offset"),
[(False, 16, 80), (True, 16, 96), (False, 8, None)],
("seq_len", "disable_eagle_block_drop", "prefix_match_unit", "expected_offset"),
[
(100, False, 16, 80),
(100, True, 16, 96),
(100, False, 8, None),
(96, False, 16, 80),
(96, True, 16, 80),
],
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_spec_internal_checkpoint_metadata_targets_replay_boundary(
seq_len: int,
disable_eagle_block_drop: bool,
prefix_match_unit: int,
expected_offset: int | None,
) -> None:
device = torch.device("cuda")
batch = BatchSpec(seq_lens=[100], query_lens=[100])
batch = BatchSpec(seq_lens=[seq_len], query_lens=[seq_len])
common_attn_metadata = create_common_attn_metadata(
batch, BLOCK_SIZE, device, arange_block_indices=True
)
@@ -390,6 +433,7 @@ def test_internal_checkpoint_metadata_skips_unaligned_offset():
full_cuda_graph=False,
mamba_cache_mode="align",
num_prefill_checkpoint_blocks=1,
prefix_match_unit=16,
device=device,
)
assert isinstance(builder, KimiK3KDAMetadataBuilder)
+40
View File
@@ -8,6 +8,7 @@ import random
from collections.abc import Iterator
from dataclasses import dataclass
from math import lcm
from uuid import uuid4
import pytest
import requests
@@ -353,6 +354,45 @@ def _format(turns: list[Turn]) -> str:
return "\n".join(rows)
@pytest.mark.parametrize(
"prompt_len",
[
# Blocks=6144, PMU=128. P=7040 is PMU-aligned but not block-aligned:
# ensure lookup reads the cached attention proof at 7040 despite
# the final hit limit of 7039, then applies EAGLE's 128-token drop.
pytest.param(7040, id="eagle-proof-scan"),
# Blocks=6144, PMU=128. P=7296 is an exact PMU multiple:
# ensure the Mamba checkpoint is 7168, not 7040 from applying both
# the P - 1 cap and EAGLE's 128-token drop.
pytest.param(7296, id="eagle-double-cap"),
],
)
def test_dspark_exact_resend_reuses_prompt_checkpoint(prompt_len: int) -> None:
"""Resends reuse the Mamba checkpoint and its companion EAGLE attention proof."""
mode = Mode(128, True)
salt = str(uuid4())
rng = random.Random(0)
prompt = [rng.randint(1000, 150000) for _ in range(prompt_len)]
deployment = DEPLOYMENTS["plain"]
with _serve(deployment, mode) as servers:
url = servers[0].url_root
cold = _complete(url, prompt, salt, max_tokens=1)
resend = _complete(url, prompt, salt, max_tokens=1)
recompute = _complete(url, prompt, f"{salt}-recompute", max_tokens=1)
# EAGLE's dropped hash unit already leaves tokens to recompute on resend.
expected = (prompt_len // 128 - 1) * 128
actual = _cached(resend)
assert _cached(cold) == _cached(recompute) == 0
assert actual == expected, f"{prompt_len=}: {actual=}, {expected=}"
check_logprobs_close(
outputs_0_lst=[_tokens_text_logprobs(recompute)],
outputs_1_lst=[_tokens_text_logprobs(resend)],
name_0="recompute",
name_1="resend",
)
@pytest.mark.parametrize(
"name, mode", [_case(name, mode) for name in DEPLOYMENTS for mode in MODES]
)
@@ -385,7 +385,7 @@ def test_cudagraph_capture_batch_stays_decode_only():
def _create_checkpoint_builder_and_batch(
num_spec: int, prefix_match_unit: int = 16
num_spec: int, prefix_match_unit: int | None = 16
) -> tuple[GDNAttentionMetadataBuilder, CommonAttentionMetadata, dict]:
vllm_config = create_vllm_config(
model_name="Qwen/Qwen3.5-0.8B",
@@ -413,6 +413,8 @@ def _create_checkpoint_builder_and_batch(
vllm_config=vllm_config,
device=DEVICE,
)
vllm_config.cache_config.hash_block_size = prefix_match_unit or 64
vllm_config.cache_config.cache_hit_alignment_tokens = prefix_match_unit or 64
batch = BatchSpec(
seq_lens=[65, 100, 100], query_lens=[3 if num_spec else 1, 100, 99]
)
@@ -431,7 +433,9 @@ def _create_checkpoint_builder_and_batch(
@pytest.mark.parametrize("num_spec", [0, 3])
@pytest.mark.parametrize("prefix_match_unit, expected_offset", [(16, 96), (8, 96)])
@pytest.mark.parametrize(
"prefix_match_unit, expected_offset", [(None, 64), (16, 96), (8, 96)]
)
def test_checkpoint_metadata_preserves_non_spec_order(
num_spec, prefix_match_unit, expected_offset
):
@@ -77,7 +77,9 @@ def _create_mamba2_builder(
num_prefill_checkpoint_blocks: int = 1,
prefill_checkpoint_alignment: int | None = 1,
) -> Mamba2AttentionMetadataBuilder:
vllm_config = create_vllm_config(block_size=MAMBA_BLOCK_SIZE)
vllm_config = create_vllm_config(
model_name="Qwen/Qwen3.5-0.8B", block_size=MAMBA_BLOCK_SIZE
)
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
# get_mamba_chunk_size() reads `mamba_chunk_size` then `chunk_size`.
vllm_config.model_config.hf_text_config.mamba_chunk_size = CHUNK_SIZE
@@ -89,12 +91,15 @@ def _create_mamba2_builder(
num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks,
prefill_checkpoint_alignment=prefill_checkpoint_alignment,
)
return Mamba2AttentionMetadataBuilder(
builder = Mamba2AttentionMetadataBuilder(
kv_cache_spec=spec,
layer_names=["layer.0"],
vllm_config=vllm_config,
device=DEVICE,
)
vllm_config.cache_config.hash_block_size = MAMBA_BLOCK_SIZE
vllm_config.cache_config.cache_hit_alignment_tokens = MAMBA_BLOCK_SIZE
return builder
def _build(builder, seq_lens, query_lens):
@@ -22,6 +22,7 @@ from vllm.v1.core.kv_cache_utils import (
get_block_hash,
get_group_id,
init_none_hash,
resolve_cache_hit_alignment_tokens,
)
from vllm.v1.core.sched.scheduler import Scheduler
from vllm.v1.kv_cache_interface import (
@@ -148,29 +149,84 @@ def make_full_mamba_manager(
)
def test_dcp_fine_hit_retention_uses_hash_alignment_without_eagle():
"""DCP must not discard a reusable Mamba state at hash alignment."""
hash_block_size = 2
@pytest.mark.parametrize(
"dcp_world_size,hash_block_size,block_size,prompt_tokens,checkpoint_blocks",
[
(4, 2, 2, 7, 0),
(8, 16, 16, 512, 1),
(8, 16, 16, 513, 1),
*[(8, 128, 896, p, 1) for p in (127, 128, 129, 7040, 7168, 7296, 7449)],
],
)
@pytest.mark.parametrize("use_eagle", [False, True])
@pytest.mark.parametrize("extension_tokens", [0, 32])
def test_dcp_fine_hit_retention_uses_hash_alignment(
dcp_world_size,
hash_block_size,
block_size,
prompt_tokens,
checkpoint_blocks,
use_eagle,
extension_tokens,
):
"""Resends and extensions reuse the retained hash-aligned checkpoint."""
manager = make_full_mamba_manager(
dcp_world_size=4,
dcp_world_size=dcp_world_size,
hash_block_size=hash_block_size,
full_block_size=hash_block_size,
mamba_block_size=hash_block_size,
num_blocks=64,
full_block_size=block_size,
mamba_block_size=block_size,
num_blocks=128,
use_eagle=use_eagle,
num_speculative_blocks=4 if use_eagle else 0,
num_prefill_checkpoint_blocks=checkpoint_blocks,
)
manager.coordinator.retention_interval = 0
token_ids = list(range(7))
producer = make_request("producer", token_ids, hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(producer)
assert num_computed == 0
assert manager.allocate_slots(producer, 6, 0, computed_blocks) is not None
assert manager.coordinator.enable_partial_hash_hits
scheduler = SimpleNamespace(
block_size=block_size * dcp_world_size,
cache_config=SimpleNamespace(block_size=block_size),
hash_block_size=hash_block_size,
max_num_scheduled_tokens=8192,
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
use_eagle_block_drop=use_eagle,
mamba_partial_cache_hit=True,
mamba_shared_prefix_checkpoint=False,
mamba_has_prefill_checkpoint_blocks=bool(checkpoint_blocks),
mamba_prefill_checkpoint_alignment=16,
)
producer = make_request(
"producer", list(range(prompt_tokens)), hash_block_size, sha256
)
while producer.num_computed_tokens < prompt_tokens:
scheduled = Scheduler._mamba_block_aligned_split(
scheduler, producer, prompt_tokens - producer.num_computed_tokens
)
assert scheduled > 0
manager.new_step_starts()
assert manager.allocate_slots(producer, scheduled) is not None
producer.num_computed_tokens += scheduled
manager.free(producer)
manager.new_step_starts()
consumer = make_request("consumer", token_ids, hash_block_size, sha256)
_, num_computed, _ = manager.get_computed_blocks(consumer)
assert num_computed == 6
replay = make_request(
"replay", list(range(prompt_tokens + extension_tokens)), hash_block_size, sha256
)
_, hit, _ = manager.get_computed_blocks(replay)
expected = (
max((prompt_tokens // hash_block_size - 1) * hash_block_size, 0)
if use_eagle
else ((prompt_tokens - 1) // hash_block_size * hash_block_size)
)
assert hit == expected
cached_states = {
end
for end in range(hash_block_size, prompt_tokens + 1, hash_block_size)
if manager.block_pool.get_cached_block(
producer.block_hashes[end // hash_block_size - 1], [1]
)
is not None
}
assert cached_states == ({expected} if expected else set())
@pytest.mark.parametrize("dcp_world_size", [1, 4])
@@ -339,7 +395,7 @@ def test_hybrid_mamba_align_partial_hash_hit():
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert num_computed == 0
blocks = manager.allocate_slots(req0, 6, num_computed, computed_blocks)
@@ -488,7 +544,7 @@ def test_hybrid_mamba_partial_tail_owner_uses_cow_on_continue():
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert num_computed == 0
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
@@ -538,7 +594,7 @@ def test_partial_hit_then_internal_checkpoint_uses_distinct_mamba_blocks():
num_prefill_checkpoint_blocks=1,
)
owner = make_request("owner", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
owner = make_request("owner", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(owner)
assert manager.allocate_slots(owner, 6, num_computed, computed_blocks) is not None
manager.free(owner)
@@ -619,44 +675,70 @@ def test_internal_checkpoint_uses_partial_hash_lifecycle():
assert num_computed == 112
@pytest.mark.parametrize(
"retention_interval,transient_published",
[(None, True), (0, False)],
)
@pytest.mark.parametrize("retention_interval", [None, 0, 32, 64, 256])
@pytest.mark.parametrize("hash_block_size", [16, 32])
@pytest.mark.parametrize("use_eagle", [False, True])
def test_internal_checkpoint_publication_respects_retention(
retention_interval, transient_published
retention_interval, hash_block_size, use_eagle
):
"""With retention_interval=0, a mid-prompt internal checkpoint is
request-local and must not enter the prefix cache; the prompt-end
checkpoint keeps the existing publication behavior. Dense retention
(None) is unchanged.
"""
hash_block_size = 16
"""Publish only retained block boundaries and the final prompt checkpoint."""
manager = make_full_mamba_manager(
dcp_world_size=1,
hash_block_size=hash_block_size,
full_block_size=hash_block_size,
mamba_block_size=32,
num_prefill_checkpoint_blocks=1,
use_eagle=use_eagle,
)
manager.coordinator.retention_interval = retention_interval
request = make_request("producer", list(range(240)), hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(request)
# Mid-prompt chunk: exports a transient checkpoint at 112.
# The EAGLE adjustment can make a chunk checkpoint block-aligned or partial.
assert manager.allocate_slots(request, 128, num_computed, computed_blocks)
transient_hash = request.block_hashes[112 // hash_block_size - 1]
checkpoint = (128 if use_eagle else 127) // hash_block_size * hash_block_size - (
hash_block_size if use_eagle else 0
)
retained_checkpoint = checkpoint % 32 == 0 and (
retention_interval is None
or (retention_interval > 0 and checkpoint % retention_interval == 0)
)
transient_hash = request.block_hashes[checkpoint // hash_block_size - 1]
transient_hit = manager.block_pool.get_cached_block(transient_hash, [1])
assert (transient_hit is not None) == transient_published
assert (transient_hit is not None) == retained_checkpoint
offloads = drain_boundary_state_offloads(manager).get(request.request_id, [])
assert any(position == checkpoint for _, _, position in offloads) == (
retained_checkpoint
)
assert all(
position % 32 == 0
and (
retention_interval is None
or (retention_interval > 0 and position % retention_interval == 0)
)
for _, _, position in offloads
)
if retention_interval in (None, 32, 64):
retained_hash = request.block_hashes[128 // hash_block_size - 1]
assert manager.block_pool.get_cached_block(retained_hash, [1]) is not None
request.num_computed_tokens = 128
manager.new_step_starts()
# Final chunk: the prompt-end checkpoint at 224 stays published.
# The final checkpoint is reusable regardless of the retention interval.
assert manager.allocate_slots(request, 112) is not None
end_hash = request.block_hashes[224 // hash_block_size - 1]
prompt_checkpoint = (
240 if use_eagle else 239
) // hash_block_size * hash_block_size - (hash_block_size if use_eagle else 0)
end_hash = request.block_hashes[prompt_checkpoint // hash_block_size - 1]
assert manager.block_pool.get_cached_block(end_hash, [1]) is not None
assert any(
position == prompt_checkpoint
for _, _, position in drain_boundary_state_offloads(manager).get(
request.request_id, []
)
)
def test_transient_checkpoint_evicts_retained_boundary_hash():
@@ -802,9 +884,9 @@ def test_hash_aligned_query_end_uses_regular_partial_tail():
mamba_block_size=mamba_block_size,
num_prefill_checkpoint_blocks=1,
)
request = make_request("producer", list(range(14)), hash_block_size, sha256)
request = make_request("producer", list(range(15)), hash_block_size, sha256)
new_blocks = manager.allocate_slots(request, request.num_tokens)
new_blocks = manager.allocate_slots(request, 14)
assert new_blocks is not None
mamba_manager = manager.coordinator.single_type_managers[1]
@@ -928,7 +1010,7 @@ def test_boundary_state_offloads_returns_cow_target():
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
@@ -1004,7 +1086,7 @@ def test_finished_partial_tail_uses_table_source_once():
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
source_block_id = manager.get_blocks("0").get_block_ids()[1][1]
@@ -1024,7 +1106,7 @@ def test_finished_partial_tail_uses_table_source_once():
enable_caching=True,
hash_block_size=hash_block_size,
)
req1 = make_request("1", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = advanced_manager.get_computed_blocks(req1)
assert (
advanced_manager.allocate_slots(req1, 6, num_computed, computed_blocks)
@@ -1212,7 +1294,7 @@ def test_truncate_computed_blocks_preserves_sparse_prefix_positions():
enable_caching=True,
hash_block_size=hash_block_size,
)
producer = make_request("producer", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
producer = make_request("producer", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
blocks, num_computed, _ = manager.get_computed_blocks(producer)
assert manager.allocate_slots(producer, 6, num_computed, blocks) is not None
manager.free(producer)
@@ -1267,7 +1349,7 @@ def test_truncate_computed_blocks_allows_short_mamba_group_only():
enable_caching=True,
hash_block_size=hash_block_size,
)
producer = make_request("producer", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
producer = make_request("producer", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
blocks, num_computed, _ = manager.get_computed_blocks(producer)
assert manager.allocate_slots(producer, 6, num_computed, blocks) is not None
manager.free(producer)
@@ -1331,7 +1413,7 @@ def test_hybrid_mamba_partial_tail_owner_continue_preserves_later_hit():
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert num_computed == 0
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
@@ -1411,7 +1493,7 @@ def test_hybrid_mamba_moved_partial_entry_defers_same_step_hit():
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert num_computed == 0
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
@@ -1470,7 +1552,7 @@ def test_hybrid_full_attention_partial_hash_hit_uses_cow():
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert num_computed == 0
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
@@ -1544,7 +1626,7 @@ def test_hybrid_partial_hit_cow_target_starts_uncached():
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert num_computed == 0
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
@@ -1700,7 +1782,7 @@ def test_cow_retained_blocks_returned_for_release():
enable_caching=True,
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
@@ -2133,6 +2215,72 @@ def test_hybrid_sliding_window_group_disables_partial_hash_hits():
assert len(computed_blocks.blocks[0]) * hash_block_size == num_computed
@pytest.mark.parametrize("with_sliding_window", [False, True])
def test_resolved_cache_hit_alignment_matches_coordinator(with_sliding_window):
"""Workers place Mamba checkpoints on the engine-resolved alignment, so it
must be where the coordinator lands hits, including when a group without
fine-grained lookups disables partial hits."""
hash_block_size = 2
mamba_block_size = 4 * hash_block_size
kv_cache_groups = [
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=hash_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=mamba_block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
]
if with_sliding_window:
kv_cache_groups.append(
KVCacheGroupSpec(
["swa"],
SlidingWindowSpec(
block_size=2 * hash_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=2 * hash_block_size,
),
)
)
kv_cache_config = KVCacheConfig(
num_blocks=64, kv_cache_tensors=[], kv_cache_groups=kv_cache_groups
)
coordinator = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
).coordinator
vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(enable_prefix_caching=True),
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
)
assert coordinator.enable_partial_hash_hits != with_sliding_window
assert (
resolve_cache_hit_alignment_tokens(
kv_cache_config,
vllm_config,
coordinator.scheduler_block_size,
hash_block_size,
)
== coordinator._cache_hit_alignment_tokens
)
def test_opted_out_scratch_group_keeps_partial_hash_hits():
hash_block_size = 2
mamba_block_size = 2 * hash_block_size
@@ -2179,7 +2327,7 @@ def test_opted_out_scratch_group_keeps_partial_hash_hits():
assert manager.coordinator.enable_partial_hash_hits
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
manager.free(req0)
@@ -2240,7 +2388,7 @@ def test_kpool_tail_supports_128_token_partial_hash_hits():
assert manager.coordinator.enable_partial_hash_hits
shared_prefix = [10] * (12 * hash_block_size)
req0 = make_request("0", shared_prefix, hash_block_size, sha256)
req0 = make_request("0", shared_prefix + [11], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert (
manager.allocate_slots(req0, len(shared_prefix), num_computed, computed_blocks)
@@ -2266,7 +2414,7 @@ def test_hybrid_partial_hash_hit_uses_cow_under_dcp(dcp_world_size: int):
)
assert manager.coordinator.enable_partial_hash_hits
req0 = make_request("dcp-owner", [0, 0, 1, 1, 2, 2], 2, sha256)
req0 = make_request("dcp-owner", [0, 0, 1, 1, 2, 2, 3], 2, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
manager.free(req0)
@@ -2318,7 +2466,7 @@ def test_dcp_partial_hit_resumes_on_replicated_mamba_snapshot(
assert manager.coordinator.single_type_managers[1].block_size == block_size
prefix = list(range(12))
req0 = make_request("snapshot-owner", prefix, block_size, sha256)
req0 = make_request("snapshot-owner", prefix + [12], block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 12, num_computed, computed_blocks) is not None
manager.free(req0)
@@ -2355,7 +2503,7 @@ def test_dcp_joint_hit_is_bounded_by_replicated_mamba_snapshots():
mamba_block_size=block_size,
)
prefix = list(range(12))
req0 = make_request("joint-owner", prefix, block_size, sha256)
req0 = make_request("joint-owner", prefix + [12], block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 8, num_computed, computed_blocks) is not None
manager.new_step_starts()
@@ -2702,6 +2850,7 @@ def test_checkpoint_reservation_matches_worker(
mamba_block_size=mamba_block_size,
checkpoint_alignment=16,
drop_eagle_block=use_eagle,
cache_hit_alignment_tokens=mamba_manager.cache_hit_alignment_tokens,
)
reserved = mamba_manager._checkpoints.get(request.request_id)
if reserved is not None:
+10 -26
View File
@@ -5047,19 +5047,10 @@ def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary():
def test_hybrid_mamba_retention_mtp_resend_of_aligned_prompt():
"""An identical resend and a longer sibling resume at DIFFERENT positions.
"""EAGLE resend and extension reuse the same retained checkpoint.
How far a lookup matches depends on who is asking. A resend of the same
prompt caps its lookup at ``num_tokens - 1`` (the last token is recomputed
for logits), while a sibling whose prompt merely starts with this one caps
above the prompt. The two coincide unless the prompt length is an exact
multiple of the alignment -- there they differ by one alignment unit, and
under the EAGLE drop BOTH are reachable.
Retaining only the higher one leaves the resend with every retained state
above every candidate its lookup can produce, and the reconciled hit
collapses to 0 -- the same zero-hit failure sparse retention already fixes
at unaligned prompt lengths.
Attention may inspect the prompt's final block before dropping it, so an
aligned resend must not lose another block to the P - 1 lookup limit.
"""
block_size = 32
num_spec = 3
@@ -5097,16 +5088,12 @@ def test_hybrid_mamba_retention_mtp_resend_of_aligned_prompt():
use_eagle=True,
)
# 128 tokens, an exact multiple of the 32-token alignment. A longer sibling
# matches 128 and drops to 96; this prompt's own resend caps at 127, matches
# 96 and drops to 64. Both states must survive retention.
# Both requests prove 128 attention tokens, drop one block, and reuse 96.
token_ids = [i for i in range(4) for _ in range(block_size)]
req0 = make_request("0", token_ids, block_size, sha256)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
assert num_computed_tokens == 0
# Prefill in block-aligned chunks the way the align-mode scheduler does: a
# state only materializes as a chunk's running-state block, so a
# single-shot prefill could not retain the lower one.
# Materialize the checkpoint through normal block-aligned prefill chunks.
for chunk_end in (32, 64, 96, 128):
blocks = manager.allocate_slots(
req0,
@@ -5118,8 +5105,7 @@ def test_hybrid_mamba_retention_mtp_resend_of_aligned_prompt():
assert blocks is not None
req0.num_computed_tokens = chunk_end
# Block ``i`` ends at token ``(i + 1) * 32``, so positions 64 and 96 are
# mamba blocks 1 and 2.
# Both materialized replay checkpoints survive; resend and extension use 96.
pool = manager.block_pool
expected_mamba_cached = {1, 2}
for i in range(4):
@@ -5130,15 +5116,13 @@ def test_hybrid_mamba_retention_mtp_resend_of_aligned_prompt():
assert cached is None, f"mamba hash {i} should not be cached"
manager.free(req0)
# The identical resend: full attention matches blocks 0-2 (96 tokens, capped
# by num_tokens - 1) and the EAGLE drop caps the candidate at 64. Without
# the lower state retained the reconciled hit would be 0.
# The identical resend must reuse the retained checkpoint, not miss to zero.
req1 = make_request("1", token_ids, block_size, sha256)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
assert num_computed_tokens == 2 * block_size
assert [len(blocks) for blocks in computed_blocks.blocks] == [2, 2]
assert num_computed_tokens == 3 * block_size
assert [len(blocks) for blocks in computed_blocks.blocks] == [3, 3]
# The longer sibling resumes one alignment unit higher, off the same prompt.
# The longer sibling reuses that same checkpoint.
longer = make_request("2", token_ids + [9] * block_size, block_size, sha256)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(longer)
assert num_computed_tokens == 3 * block_size