[Kimi-K3] support DCP partial prefix cache hit (#50493)
Signed-off-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Yifan Qiao
parent
0e8989b416
commit
0db502c8d8
@@ -619,15 +619,22 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
"full-attention and Mamba groups, got: "
|
||||
f"{type(g.kv_cache_spec).__name__}."
|
||||
)
|
||||
# Fine-grained hash hits require Mamba "align", no context
|
||||
# parallelism, and compatible cache managers in every group.
|
||||
# Fine-grained hash hits require Mamba "align" and compatible cache
|
||||
# managers in every group. TP needs hashing finer than the Mamba block;
|
||||
# DCP accepts equality because it scales the effective full-attention
|
||||
# block instead.
|
||||
has_partial_mamba_group = any(
|
||||
isinstance(g.kv_cache_spec, MambaSpec)
|
||||
and g.kv_cache_spec.mamba_cache_mode == "align"
|
||||
and g.kv_cache_spec.block_size > hash_block_size
|
||||
and (
|
||||
(dcp_world_size == 1 and g.kv_cache_spec.block_size > hash_block_size)
|
||||
or (
|
||||
dcp_world_size > 1 and g.kv_cache_spec.block_size >= hash_block_size
|
||||
)
|
||||
)
|
||||
for g in kv_cache_config.kv_cache_groups
|
||||
)
|
||||
self.enable_partial_hash_hits = dcp_world_size == 1 and has_partial_mamba_group
|
||||
self.enable_partial_hash_hits = has_partial_mamba_group
|
||||
if self.enable_partial_hash_hits:
|
||||
unsupported_partial_hit_managers = {
|
||||
type(manager).__name__
|
||||
|
||||
Reference in New Issue
Block a user