[ROCm][DSpark][DCP] Serve prefix cache hits under DCP for Kimi-K3 (#53598)

Signed-off-by: Yichao Zhu <Yichao.Zhu@amd.com>
Co-authored-by: andyluo7 <andy.luo@amd.com>
This commit is contained in:
YukioZzz
2026-08-31 11:18:10 -07:00
committed by GitHub
co-authored by andyluo7
parent dbb7fffddb
commit 2ba984a5d0
6 changed files with 161 additions and 11 deletions
+18 -10
View File
@@ -11,6 +11,7 @@ from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
from vllm.v1.core.kv_cache_utils import (
BlockHash,
KVCacheBlock,
dcp_world_size_for_kv_cache_spec,
)
from vllm.v1.core.single_type_kv_cache_manager import (
CrossAttentionManager,
@@ -141,7 +142,9 @@ class KVCacheCoordinator(ABC):
block_pool=self.block_pool,
enable_caching=enable_caching,
kv_cache_group_id=i,
dcp_world_size=dcp_world_size,
dcp_world_size=dcp_world_size_for_kv_cache_spec(
kv_cache_group.kv_cache_spec, dcp_world_size
),
pcp_world_size=pcp_world_size,
scheduler_block_size=self.scheduler_block_size,
needs_kv_cache_zeroing=self.kv_cache_config.needs_kv_cache_zeroing,
@@ -506,11 +509,9 @@ class UnitaryKVCacheCoordinator(KVCacheCoordinator):
num_prefill_lookahead=num_prefill_lookahead,
)
self.kv_cache_spec = self.kv_cache_config.kv_cache_groups[0].kv_cache_spec
self.block_size = self.kv_cache_spec.block_size
self.dcp_world_size = dcp_world_size
self.dcp_world_size = self.single_type_managers[0].dcp_world_size
self.pcp_world_size = pcp_world_size
if dcp_world_size > 1:
self.block_size *= dcp_world_size
self.block_size = self.single_type_managers[0].block_size
# For models using only Mamba, block_size is set to max_model_len when
# prefix caching is disabled, and hash_block_size validation is skipped.
assert not enable_caching or (hash_block_size == self.block_size), (
@@ -656,6 +657,9 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
"cache managers require block-aligned lookups: %s.",
", ".join(sorted(unsupported_partial_hit_managers)),
)
cache_hit_alignment_tokens = self._cache_hit_alignment_tokens
for manager in self.single_type_managers:
manager.cache_hit_alignment_tokens = cache_hit_alignment_tokens
self.verify_and_split_kv_cache_groups()
@property
@@ -852,11 +856,12 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
kv_cache_spec=spec,
drop_eagle_block=drop_eagle_block,
alignment_tokens=self._cache_hit_alignment_tokens,
dcp_world_size=(
self.dcp_world_size
if isinstance(spec, FullAttentionSpec)
else 1
),
dcp_world_size=self.single_type_managers[
first_group_id
].dcp_world_size,
pcp_world_size=self.single_type_managers[
first_group_id
].pcp_world_size,
)
if drop_eagle_block:
eagle_verified.add(idx)
@@ -913,6 +918,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
hit_lengths: list[int] = [0] * num_groups
for spec, group_ids, manager_cls, use_eagle in self.attention_groups:
manager = self.single_type_managers[group_ids[0]]
blocks, group_hit = manager_cls.find_longest_cache_hit(
block_hashes=block_hashes,
max_length=max_cache_hit_length,
@@ -921,6 +927,8 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
kv_cache_spec=spec,
drop_eagle_block=use_eagle,
alignment_tokens=self._cache_hit_alignment_tokens,
dcp_world_size=manager.dcp_world_size,
pcp_world_size=manager.pcp_world_size,
)
for gid, blks in zip(group_ids, blocks):
hit_blocks[gid] = blks
+22
View File
@@ -675,6 +675,28 @@ def resolve_dcp_kv_cache_spec(spec: KVCacheSpec, dcp_world_size: int) -> KVCache
return replace(spec, block_size=block_size)
def dcp_world_size_for_kv_cache_spec(spec: KVCacheSpec, dcp_world_size: int) -> int:
"""Return the DCP size that owns this group's block geometry.
Full-attention KV (including MLA) is sharded across DCP ranks, so prefix
hashing and manager ``block_size`` use the process DCP size. Other specs
keep replicated per-rank state (Mamba, sliding window, chunked-local) and
must keep ``dcp_world_size=1`` even when the process runs with DCP > 1.
Draft MLA groups on the sharded DSpark path are ``FullAttentionSpec`` /
``MLAAttentionSpec`` and therefore keep the process DCP size. A replicated
draft group would need a different spec, not this helper.
"""
if dcp_world_size <= 1:
return 1
inner = spec
if isinstance(spec, UniformTypeKVCacheSpecs):
inner = next(iter(spec.kv_cache_specs.values()))
if isinstance(inner, FullAttentionSpec):
return dcp_world_size
return 1
def resolve_kv_cache_block_sizes(
kv_cache_config: KVCacheConfig,
vllm_config: VllmConfig,
+4 -1
View File
@@ -72,6 +72,9 @@ class SingleTypeKVCacheManager(ABC):
block until the request finishes.
"""
self.scheduler_block_size = scheduler_block_size
# Hybrid fine-grained lookup may lower this after all participating
# managers have been validated by the coordinator.
self.cache_hit_alignment_tokens = scheduler_block_size
# The block size for this manager; used for actual block allocation.
self.block_size = kv_cache_spec.block_size
self.dcp_world_size = dcp_world_size
@@ -460,7 +463,7 @@ class SingleTypeKVCacheManager(ABC):
block_mask = self.reachable_block_mask(
start_block=num_cached_blocks,
end_block=num_full_blocks,
alignment_tokens=self.scheduler_block_size,
alignment_tokens=self.cache_hit_alignment_tokens,
kv_cache_spec=self.kv_cache_spec,
use_eagle=self.use_eagle,
retention_interval=retention_interval,