[Bugfix][Prefix caching] Fix inconsistent hybrid Mamba prefix cache boundaries (#60533)
Signed-off-by: Wei Zhao <weizha@aws-cmh-slurm-1-login-01.cm.cluster> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Wei Zhao <weizha@aws-cmh-slurm-1-login-01.cm.cluster> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Wei Zhao
Codex
zjy0516
Claude Opus 5.5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
1664c4f444
commit
d8fa463e2b
@@ -1127,7 +1127,10 @@ def get_mamba_prefill_checkpoint_position(
|
||||
drop_eagle_block: bool,
|
||||
) -> int:
|
||||
"""Return the reusable Mamba checkpoint boundary for a prefill."""
|
||||
checkpoint_position = (num_tokens - 1) // hash_block_size * hash_block_size
|
||||
# Without EAGLE, leave one token to recompute on resend.
|
||||
# EAGLE's block drop already leaves tokens to recompute.
|
||||
proof_limit = num_tokens if drop_eagle_block else num_tokens - 1
|
||||
checkpoint_position = proof_limit // hash_block_size * hash_block_size
|
||||
if drop_eagle_block:
|
||||
checkpoint_position -= hash_block_size
|
||||
return max(checkpoint_position, 0)
|
||||
@@ -1487,6 +1490,10 @@ class KVCacheConfig:
|
||||
"""Resolved retention policy for local prefix-cache checkpoints."""
|
||||
kv_cache_layout: str | None = None
|
||||
"""The KV cache layout resolved by the engine core, adopted by all workers."""
|
||||
hash_block_size: int | None = None
|
||||
"""Tokens per prefix-cache block hash, resolved by the engine core."""
|
||||
cache_hit_alignment_tokens: int | None = None
|
||||
"""Token granularity of prefix-cache hits, resolved by the engine core."""
|
||||
hisparse_host_num_blocks: int | None = None
|
||||
"""Capacity of the dedicated HiSparse host-block manager, when enabled."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user