[Bugfix][Prefix caching] Fix inconsistent hybrid Mamba prefix cache boundaries (#60533)
Signed-off-by: Wei Zhao <weizha@aws-cmh-slurm-1-login-01.cm.cluster> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Wei Zhao <weizha@aws-cmh-slurm-1-login-01.cm.cluster> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Wei Zhao
Codex
zjy0516
Claude Opus 5.5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
1664c4f444
commit
d8fa463e2b
@@ -385,7 +385,7 @@ def test_cudagraph_capture_batch_stays_decode_only():
|
||||
|
||||
|
||||
def _create_checkpoint_builder_and_batch(
|
||||
num_spec: int, prefix_match_unit: int = 16
|
||||
num_spec: int, prefix_match_unit: int | None = 16
|
||||
) -> tuple[GDNAttentionMetadataBuilder, CommonAttentionMetadata, dict]:
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="Qwen/Qwen3.5-0.8B",
|
||||
@@ -413,6 +413,8 @@ def _create_checkpoint_builder_and_batch(
|
||||
vllm_config=vllm_config,
|
||||
device=DEVICE,
|
||||
)
|
||||
vllm_config.cache_config.hash_block_size = prefix_match_unit or 64
|
||||
vllm_config.cache_config.cache_hit_alignment_tokens = prefix_match_unit or 64
|
||||
batch = BatchSpec(
|
||||
seq_lens=[65, 100, 100], query_lens=[3 if num_spec else 1, 100, 99]
|
||||
)
|
||||
@@ -431,7 +433,9 @@ def _create_checkpoint_builder_and_batch(
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_spec", [0, 3])
|
||||
@pytest.mark.parametrize("prefix_match_unit, expected_offset", [(16, 96), (8, 96)])
|
||||
@pytest.mark.parametrize(
|
||||
"prefix_match_unit, expected_offset", [(None, 64), (16, 96), (8, 96)]
|
||||
)
|
||||
def test_checkpoint_metadata_preserves_non_spec_order(
|
||||
num_spec, prefix_match_unit, expected_offset
|
||||
):
|
||||
|
||||
@@ -77,7 +77,9 @@ def _create_mamba2_builder(
|
||||
num_prefill_checkpoint_blocks: int = 1,
|
||||
prefill_checkpoint_alignment: int | None = 1,
|
||||
) -> Mamba2AttentionMetadataBuilder:
|
||||
vllm_config = create_vllm_config(block_size=MAMBA_BLOCK_SIZE)
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="Qwen/Qwen3.5-0.8B", block_size=MAMBA_BLOCK_SIZE
|
||||
)
|
||||
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
|
||||
# get_mamba_chunk_size() reads `mamba_chunk_size` then `chunk_size`.
|
||||
vllm_config.model_config.hf_text_config.mamba_chunk_size = CHUNK_SIZE
|
||||
@@ -89,12 +91,15 @@ def _create_mamba2_builder(
|
||||
num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks,
|
||||
prefill_checkpoint_alignment=prefill_checkpoint_alignment,
|
||||
)
|
||||
return Mamba2AttentionMetadataBuilder(
|
||||
builder = Mamba2AttentionMetadataBuilder(
|
||||
kv_cache_spec=spec,
|
||||
layer_names=["layer.0"],
|
||||
vllm_config=vllm_config,
|
||||
device=DEVICE,
|
||||
)
|
||||
vllm_config.cache_config.hash_block_size = MAMBA_BLOCK_SIZE
|
||||
vllm_config.cache_config.cache_hit_alignment_tokens = MAMBA_BLOCK_SIZE
|
||||
return builder
|
||||
|
||||
|
||||
def _build(builder, seq_lens, query_lens):
|
||||
|
||||
Reference in New Issue
Block a user