[Bugfix][Prefix caching] Fix inconsistent hybrid Mamba prefix cache boundaries (#60533)
Signed-off-by: Wei Zhao <weizha@aws-cmh-slurm-1-login-01.cm.cluster> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Wei Zhao <weizha@aws-cmh-slurm-1-login-01.cm.cluster> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Wei Zhao
Codex
zjy0516
Claude Opus 5.5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
1664c4f444
commit
d8fa463e2b
@@ -25,6 +25,7 @@ def compute_mamba_prefill_checkpoints(
|
||||
mamba_block_size: int,
|
||||
checkpoint_alignment: int | None,
|
||||
drop_eagle_block: bool,
|
||||
cache_hit_alignment_tokens: int,
|
||||
) -> tuple[list[int], list[int]]:
|
||||
"""Per-row internal prefill checkpoint offsets and cache block columns.
|
||||
|
||||
@@ -43,7 +44,7 @@ def compute_mamba_prefill_checkpoints(
|
||||
for seq_len, query_len in zip(seq_lens, query_lens):
|
||||
query_start = seq_len - query_len
|
||||
position = get_mamba_prefill_checkpoint_position(
|
||||
seq_len, hash_block_size, drop_eagle_block=drop_eagle_block
|
||||
seq_len, cache_hit_alignment_tokens, drop_eagle_block=drop_eagle_block
|
||||
)
|
||||
valid = is_mamba_prefill_checkpoint_valid(
|
||||
query_start=query_start,
|
||||
@@ -106,12 +107,17 @@ class MambaPrefillCheckpointBuilder:
|
||||
return None
|
||||
if self.kv_cache_spec.num_prefill_checkpoint_blocks == 0:
|
||||
return None
|
||||
cache_config = self.vllm_config.cache_config
|
||||
hash_block_size = cache_config.hash_block_size
|
||||
cache_hit_alignment_tokens = cache_config.cache_hit_alignment_tokens
|
||||
if hash_block_size is None or cache_hit_alignment_tokens is None:
|
||||
# Unresolved only for the minimal KV cache built for profiling.
|
||||
return None
|
||||
assert m.seq_lens_cpu_upper_bound is not None
|
||||
all_query_lens = m.query_start_loc_cpu.diff().tolist()
|
||||
query_lens = [all_query_lens[row] for row in request_rows]
|
||||
seq_lens = m.seq_lens_cpu_upper_bound.tolist()
|
||||
block_size = self.kv_cache_spec.block_size
|
||||
hash_block_size = self.vllm_config.cache_config.prefix_match_unit or block_size
|
||||
speculative_config = self.vllm_config.speculative_config
|
||||
drop_eagle_block = (
|
||||
speculative_config is not None and speculative_config.use_eagle_block_drop()
|
||||
@@ -123,6 +129,7 @@ class MambaPrefillCheckpointBuilder:
|
||||
mamba_block_size=block_size,
|
||||
checkpoint_alignment=self.kv_cache_spec.prefill_checkpoint_alignment,
|
||||
drop_eagle_block=drop_eagle_block,
|
||||
cache_hit_alignment_tokens=cache_hit_alignment_tokens,
|
||||
)
|
||||
if not any(checkpoint_offsets):
|
||||
return None
|
||||
|
||||
Reference in New Issue
Block a user