[Bugfix][Prefix caching] Fix inconsistent hybrid Mamba prefix cache boundaries (#60533)
Signed-off-by: Wei Zhao <weizha@aws-cmh-slurm-1-login-01.cm.cluster> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Wei Zhao <weizha@aws-cmh-slurm-1-login-01.cm.cluster> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Wei Zhao
Codex
zjy0516
Claude Opus 5.5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
1664c4f444
commit
d8fa463e2b
@@ -230,6 +230,15 @@ class CacheConfig:
|
||||
for hybrid models where requests occupy multiple KV cache groups."""
|
||||
kv_cache_max_concurrency: float | None = field(default=None, init=False)
|
||||
"""Per-DP-engine maximum concurrency at max_model_len tokens."""
|
||||
# The two differ when hashes are finer than the scheduler block but partial
|
||||
# hash hits are off (e.g. a sliding-window group cannot look up sub-block
|
||||
# hits), so hits only land on the scheduler block.
|
||||
# TODO(zjy0516): Merge into one value
|
||||
hash_block_size: int | None = field(default=None, init=False)
|
||||
"""Tokens per prefix-cache block hash."""
|
||||
cache_hit_alignment_tokens: int | None = field(default=None, init=False)
|
||||
"""Token granularity at which prefix-cache hits land: `hash_block_size`
|
||||
with partial hash hits, otherwise the scheduler block size."""
|
||||
|
||||
kv_sharing_fast_prefill: bool = False
|
||||
"""In some KV sharing setups, e.g. YOCO (https://arxiv.org/abs/2405.05254),
|
||||
@@ -298,6 +307,8 @@ class CacheConfig:
|
||||
"effective_attention_block_size",
|
||||
"kv_cache_size_tokens",
|
||||
"kv_cache_max_concurrency",
|
||||
"hash_block_size",
|
||||
"cache_hit_alignment_tokens",
|
||||
# Feature toggles not impacting compiled graph shape
|
||||
"kv_sharing_fast_prefill",
|
||||
"swa_bounded_replay",
|
||||
|
||||
Reference in New Issue
Block a user