[Bugfix][Prefix caching] Fix inconsistent hybrid Mamba prefix cache boundaries (#60533)

Signed-off-by: Wei Zhao <weizha@aws-cmh-slurm-1-login-01.cm.cluster>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Wei Zhao <weizha@aws-cmh-slurm-1-login-01.cm.cluster>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Wei Zhao
2026-10-08 18:33:48 +00:00
committed by GitHub
co-authored by Wei Zhao Codex zjy0516 Claude Opus 5.5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 1664c4f444
commit d8fa463e2b
17 changed files with 546 additions and 200 deletions
@@ -385,7 +385,7 @@ def test_cudagraph_capture_batch_stays_decode_only():
def _create_checkpoint_builder_and_batch(
num_spec: int, prefix_match_unit: int = 16
num_spec: int, prefix_match_unit: int | None = 16
) -> tuple[GDNAttentionMetadataBuilder, CommonAttentionMetadata, dict]:
vllm_config = create_vllm_config(
model_name="Qwen/Qwen3.5-0.8B",
@@ -413,6 +413,8 @@ def _create_checkpoint_builder_and_batch(
vllm_config=vllm_config,
device=DEVICE,
)
vllm_config.cache_config.hash_block_size = prefix_match_unit or 64
vllm_config.cache_config.cache_hit_alignment_tokens = prefix_match_unit or 64
batch = BatchSpec(
seq_lens=[65, 100, 100], query_lens=[3 if num_spec else 1, 100, 99]
)
@@ -431,7 +433,9 @@ def _create_checkpoint_builder_and_batch(
@pytest.mark.parametrize("num_spec", [0, 3])
@pytest.mark.parametrize("prefix_match_unit, expected_offset", [(16, 96), (8, 96)])
@pytest.mark.parametrize(
"prefix_match_unit, expected_offset", [(None, 64), (16, 96), (8, 96)]
)
def test_checkpoint_metadata_preserves_non_spec_order(
num_spec, prefix_match_unit, expected_offset
):
@@ -77,7 +77,9 @@ def _create_mamba2_builder(
num_prefill_checkpoint_blocks: int = 1,
prefill_checkpoint_alignment: int | None = 1,
) -> Mamba2AttentionMetadataBuilder:
vllm_config = create_vllm_config(block_size=MAMBA_BLOCK_SIZE)
vllm_config = create_vllm_config(
model_name="Qwen/Qwen3.5-0.8B", block_size=MAMBA_BLOCK_SIZE
)
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
# get_mamba_chunk_size() reads `mamba_chunk_size` then `chunk_size`.
vllm_config.model_config.hf_text_config.mamba_chunk_size = CHUNK_SIZE
@@ -89,12 +91,15 @@ def _create_mamba2_builder(
num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks,
prefill_checkpoint_alignment=prefill_checkpoint_alignment,
)
return Mamba2AttentionMetadataBuilder(
builder = Mamba2AttentionMetadataBuilder(
kv_cache_spec=spec,
layer_names=["layer.0"],
vllm_config=vllm_config,
device=DEVICE,
)
vllm_config.cache_config.hash_block_size = MAMBA_BLOCK_SIZE
vllm_config.cache_config.cache_hit_alignment_tokens = MAMBA_BLOCK_SIZE
return builder
def _build(builder, seq_lens, query_lens):