[Bugfix][Prefix caching] Fix inconsistent hybrid Mamba prefix cache boundaries (#60533)

Signed-off-by: Wei Zhao <weizha@aws-cmh-slurm-1-login-01.cm.cluster>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Wei Zhao <weizha@aws-cmh-slurm-1-login-01.cm.cluster>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Wei Zhao
2026-10-08 18:33:48 +00:00
committed by GitHub
co-authored by Wei Zhao Codex zjy0516 Claude Opus 5.5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 1664c4f444
commit d8fa463e2b
17 changed files with 546 additions and 200 deletions
@@ -75,7 +75,7 @@ def _builder(
vllm_config.cache_config.prefix_match_unit = prefix_match_unit
if num_spec:
vllm_config.speculative_config = _StubSpeculativeConfig(num_spec)
return KimiK3ROCmKDAMetadataBuilder(
builder = KimiK3ROCmKDAMetadataBuilder(
kv_cache_spec=MambaSpec(
block_size=block_size,
shapes=((16, 64),),
@@ -91,6 +91,11 @@ def _builder(
vllm_config=vllm_config,
device=torch.device("cuda"),
)
vllm_config.cache_config.hash_block_size = prefix_match_unit or block_size
vllm_config.cache_config.cache_hit_alignment_tokens = (
prefix_match_unit or block_size
)
return builder
def _build(batch: BatchSpec, block_size: int = BLOCK_SIZE, **kw):
+52 -8
View File
@@ -100,6 +100,8 @@ def _make_builder(
prefix_match_unit: int | None = None,
use_eagle: bool = False,
disable_eagle_block_drop: bool = False,
cache_hit_alignment_tokens: int | None = None,
hash_block_size: int | None = None,
) -> AttentionMetadataBuilder:
vllm_config = create_vllm_config(
model_name="Qwen/Qwen3.5-0.8B",
@@ -122,6 +124,11 @@ def _make_builder(
vllm_config.cache_config.use_replayssm = use_recoverssm
vllm_config.cache_config.use_kda_recoverssm = use_recoverssm
vllm_config.cache_config.prefix_match_unit = prefix_match_unit
hash_block_size = hash_block_size or prefix_match_unit or mamba_block_size
vllm_config.cache_config.hash_block_size = hash_block_size
vllm_config.cache_config.cache_hit_alignment_tokens = (
cache_hit_alignment_tokens or hash_block_size
)
builder = builder_cls(
kv_cache_spec=MambaSpec(
block_size=mamba_block_size,
@@ -220,6 +227,8 @@ def test_kda_recoverssm_startup_metadata_flow_without_model(monkeypatch):
vllm_config=builder_config,
device=DEVICE,
)
builder_config.cache_config.hash_block_size = BLOCK_SIZE
builder_config.cache_config.cache_hit_alignment_tokens = BLOCK_SIZE
# An all-prefill speculative batch used to leave an all-false spec mask
# alive, then access active_non_spec_mask_cpu before it was initialized.
@@ -287,18 +296,45 @@ def test_kda_recoverssm_startup_metadata_flow_without_model(monkeypatch):
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_internal_checkpoint_metadata_targets_last_aligned_boundary():
@pytest.mark.parametrize(
"batch,prefix_match_unit,hit_alignment,expected_offsets,expected_states",
[
(
BatchSpec(seq_lens=[50, 32], query_lens=[50, 16]),
None,
16,
[48, 0],
[2, NULL_BLOCK_ID],
),
(
BatchSpec(seq_lens=[50, 32], query_lens=[50, 16]),
16,
16,
[48, 0],
[2, NULL_BLOCK_ID],
),
(BatchSpec(seq_lens=[530], query_lens=[530]), None, 128, [512], [32]),
(BatchSpec(seq_lens=[530], query_lens=[50]), None, 128, [32], [32]),
(BatchSpec(seq_lens=[530], query_lens=[530]), 16, 16, [528], [32]),
(BatchSpec(seq_lens=[530], query_lens=[50]), 16, 16, [48], [32]),
],
)
def test_internal_checkpoint_metadata_targets_last_aligned_boundary(
batch, prefix_match_unit, hit_alignment, expected_offsets, expected_states
):
device = torch.device("cuda")
batch = BatchSpec(seq_lens=[50, 32], query_lens=[50, 16])
common_attn_metadata = create_common_attn_metadata(
batch, BLOCK_SIZE, device, arange_block_indices=True
).replace(is_prefilling=torch.tensor([True, True]))
).replace(is_prefilling=torch.tensor([True] * len(batch.seq_lens)))
builder = _make_builder(
KimiK3KDAMetadataBuilder,
num_speculative_tokens=0,
full_cuda_graph=False,
mamba_cache_mode="align",
num_prefill_checkpoint_blocks=1,
prefix_match_unit=prefix_match_unit,
cache_hit_alignment_tokens=hit_alignment,
hash_block_size=16,
device=device,
)
assert isinstance(builder, KimiK3KDAMetadataBuilder)
@@ -313,26 +349,33 @@ def test_internal_checkpoint_metadata_targets_last_aligned_boundary():
assert actual.checkpoint is not None
torch.testing.assert_close(
actual.checkpoint.state_indices,
torch.tensor([2, NULL_BLOCK_ID], dtype=torch.int32, device=device),
torch.tensor(expected_states, dtype=torch.int32, device=device),
)
torch.testing.assert_close(
actual.checkpoint.checkpoint_offsets,
torch.tensor([48, 0], dtype=torch.int32, device=device),
torch.tensor(expected_offsets, dtype=torch.int32, device=device),
)
@pytest.mark.parametrize(
("disable_eagle_block_drop", "prefix_match_unit", "expected_offset"),
[(False, 16, 80), (True, 16, 96), (False, 8, None)],
("seq_len", "disable_eagle_block_drop", "prefix_match_unit", "expected_offset"),
[
(100, False, 16, 80),
(100, True, 16, 96),
(100, False, 8, None),
(96, False, 16, 80),
(96, True, 16, 80),
],
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_spec_internal_checkpoint_metadata_targets_replay_boundary(
seq_len: int,
disable_eagle_block_drop: bool,
prefix_match_unit: int,
expected_offset: int | None,
) -> None:
device = torch.device("cuda")
batch = BatchSpec(seq_lens=[100], query_lens=[100])
batch = BatchSpec(seq_lens=[seq_len], query_lens=[seq_len])
common_attn_metadata = create_common_attn_metadata(
batch, BLOCK_SIZE, device, arange_block_indices=True
)
@@ -390,6 +433,7 @@ def test_internal_checkpoint_metadata_skips_unaligned_offset():
full_cuda_graph=False,
mamba_cache_mode="align",
num_prefill_checkpoint_blocks=1,
prefix_match_unit=16,
device=device,
)
assert isinstance(builder, KimiK3KDAMetadataBuilder)
+40
View File
@@ -8,6 +8,7 @@ import random
from collections.abc import Iterator
from dataclasses import dataclass
from math import lcm
from uuid import uuid4
import pytest
import requests
@@ -353,6 +354,45 @@ def _format(turns: list[Turn]) -> str:
return "\n".join(rows)
@pytest.mark.parametrize(
"prompt_len",
[
# Blocks=6144, PMU=128. P=7040 is PMU-aligned but not block-aligned:
# ensure lookup reads the cached attention proof at 7040 despite
# the final hit limit of 7039, then applies EAGLE's 128-token drop.
pytest.param(7040, id="eagle-proof-scan"),
# Blocks=6144, PMU=128. P=7296 is an exact PMU multiple:
# ensure the Mamba checkpoint is 7168, not 7040 from applying both
# the P - 1 cap and EAGLE's 128-token drop.
pytest.param(7296, id="eagle-double-cap"),
],
)
def test_dspark_exact_resend_reuses_prompt_checkpoint(prompt_len: int) -> None:
"""Resends reuse the Mamba checkpoint and its companion EAGLE attention proof."""
mode = Mode(128, True)
salt = str(uuid4())
rng = random.Random(0)
prompt = [rng.randint(1000, 150000) for _ in range(prompt_len)]
deployment = DEPLOYMENTS["plain"]
with _serve(deployment, mode) as servers:
url = servers[0].url_root
cold = _complete(url, prompt, salt, max_tokens=1)
resend = _complete(url, prompt, salt, max_tokens=1)
recompute = _complete(url, prompt, f"{salt}-recompute", max_tokens=1)
# EAGLE's dropped hash unit already leaves tokens to recompute on resend.
expected = (prompt_len // 128 - 1) * 128
actual = _cached(resend)
assert _cached(cold) == _cached(recompute) == 0
assert actual == expected, f"{prompt_len=}: {actual=}, {expected=}"
check_logprobs_close(
outputs_0_lst=[_tokens_text_logprobs(recompute)],
outputs_1_lst=[_tokens_text_logprobs(resend)],
name_0="recompute",
name_1="resend",
)
@pytest.mark.parametrize(
"name, mode", [_case(name, mode) for name in DEPLOYMENTS for mode in MODES]
)