From 2ba984a5d06db414f3b2474fe9338faf6cd80a1c Mon Sep 17 00:00:00 2001 From: YukioZzz <424114284@qq.com> Date: Tue, 1 Sep 2026 02:18:10 +0800 Subject: [PATCH] [ROCm][DSpark][DCP] Serve prefix cache hits under DCP for Kimi-K3 (#53598) Signed-off-by: Yichao Zhu Co-authored-by: andyluo7 --- .../test_partial_prefix_cache_hits.py | 25 ++++++ tests/v1/core/test_kv_cache_utils.py | 15 ++++ tests/v1/core/test_prefix_caching.py | 77 +++++++++++++++++++ vllm/v1/core/kv_cache_coordinator.py | 28 ++++--- vllm/v1/core/kv_cache_utils.py | 22 ++++++ vllm/v1/core/single_type_kv_cache_manager.py | 5 +- 6 files changed, 161 insertions(+), 11 deletions(-) diff --git a/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py b/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py index 11f3445225..1d5a139eb8 100644 --- a/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py +++ b/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py @@ -132,6 +132,31 @@ def make_full_mamba_manager( ) +def test_dcp_fine_hit_retention_uses_hash_alignment_without_eagle(): + """DCP must not discard a reusable Mamba state at hash alignment.""" + hash_block_size = 2 + manager = make_full_mamba_manager( + dcp_world_size=4, + hash_block_size=hash_block_size, + full_block_size=hash_block_size, + mamba_block_size=hash_block_size, + num_blocks=64, + ) + manager.coordinator.retention_interval = 0 + + token_ids = list(range(7)) + producer = make_request("producer", token_ids, hash_block_size, sha256) + computed_blocks, num_computed, _ = manager.get_computed_blocks(producer) + assert num_computed == 0 + assert manager.allocate_slots(producer, 6, 0, computed_blocks) is not None + manager.free(producer) + manager.new_step_starts() + + consumer = make_request("consumer", token_ids, hash_block_size, sha256) + _, num_computed, _ = manager.get_computed_blocks(consumer) + assert num_computed == 6 + + @pytest.mark.parametrize("dcp_world_size", [1, 4]) def test_mamba_align_split_partial_tail_schedule(dcp_world_size: int): """Chunk ends with partial hits on: block-aligned chunks, one extra stop diff --git a/tests/v1/core/test_kv_cache_utils.py b/tests/v1/core/test_kv_cache_utils.py index 9ca04c5c94..69ccfafc68 100644 --- a/tests/v1/core/test_kv_cache_utils.py +++ b/tests/v1/core/test_kv_cache_utils.py @@ -1318,6 +1318,21 @@ def test_project_kv_cache_groups_to_worker(): assert set(proj_spec.kv_cache_specs.keys()) == {"layer1", "layer3"} +def test_dcp_world_size_for_kv_cache_spec_shards_full_attention_only(): + dcp = 8 + full = FullAttentionSpec( + block_size=16, num_kv_heads=1, head_size=1, dtype=torch.float32 + ) + mla = new_mla_spec() + mamba = new_mamba_spec() + uniform_mla = UniformTypeKVCacheSpecs(block_size=16, kv_cache_specs={"layer": mla}) + assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(full, dcp) == dcp + assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(mla, dcp) == dcp + assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(uniform_mla, dcp) == dcp + assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(mamba, dcp) == 1 + assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(full, 1) == 1 + + @pytest.mark.parametrize( "layer_type,dcp_size,expected_width", [ diff --git a/tests/v1/core/test_prefix_caching.py b/tests/v1/core/test_prefix_caching.py index 6fc2131b6e..c42ec12738 100644 --- a/tests/v1/core/test_prefix_caching.py +++ b/tests/v1/core/test_prefix_caching.py @@ -232,6 +232,83 @@ def make_kv_cache_config_three_types( ) +def test_prefix_cache_hit_uses_per_group_dcp_geometry(): + """Prefix lookup must use each group's DCP size, not the process-wide one. + + Target and draft MLA are both sharded (DCP=8); Mamba stays replicated + (DCP=1). Hits then align to the sharded full-attention block, not the + unsharded page size. + """ + block_size = 16 + dcp = 8 + sharded_block = block_size * dcp + kv_cache_config = KVCacheConfig( + num_blocks=64, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec( + ["target_mla"], + FullAttentionSpec( + block_size=block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + ), + KVCacheGroupSpec( + ["draft_mla"], + MLAAttentionSpec( + block_size=block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + ), + KVCacheGroupSpec( + ["mamba"], + MambaSpec( + block_size=block_size, + shapes=(1, 1), + dtypes=(torch.float32,), + ), + ), + ], + ) + manager = KVCacheManager( + kv_cache_config, + max_model_len=8192, + enable_caching=True, + hash_block_size=block_size, + scheduler_block_size=sharded_block, + dcp_world_size=dcp, + ) + target_mgr, draft_mgr, mamba_mgr = manager.coordinator.single_type_managers + assert target_mgr.dcp_world_size == dcp + assert draft_mgr.dcp_world_size == dcp + assert mamba_mgr.dcp_world_size == 1 + assert target_mgr.block_size == sharded_block + assert draft_mgr.block_size == sharded_block + assert mamba_mgr.block_size == block_size + + hash_fn = sha256 + common_token_ids = [i for i in range(2) for _ in range(sharded_block)] + req0 = make_request("0", common_token_ids + [99] * 7, block_size, hash_fn) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0) + assert num_computed_tokens == 0 + blocks = manager.allocate_slots( + req0, len(req0.prompt_token_ids), 0, computed_blocks + ) + assert blocks is not None + + req1 = make_request("1", common_token_ids + [100] * 5, block_size, hash_fn) + computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1) + assert num_computed_tokens == 2 * sharded_block + assert [len(group) for group in computed_blocks.blocks] == [2, 2, 16] + + manager.free(req0) + manager.free(req1) + + @pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor]) def test_prefill(hash_fn): block_size = 16 diff --git a/vllm/v1/core/kv_cache_coordinator.py b/vllm/v1/core/kv_cache_coordinator.py index 3d5adb9014..b3a9b4c032 100644 --- a/vllm/v1/core/kv_cache_coordinator.py +++ b/vllm/v1/core/kv_cache_coordinator.py @@ -11,6 +11,7 @@ from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector from vllm.v1.core.kv_cache_utils import ( BlockHash, KVCacheBlock, + dcp_world_size_for_kv_cache_spec, ) from vllm.v1.core.single_type_kv_cache_manager import ( CrossAttentionManager, @@ -141,7 +142,9 @@ class KVCacheCoordinator(ABC): block_pool=self.block_pool, enable_caching=enable_caching, kv_cache_group_id=i, - dcp_world_size=dcp_world_size, + dcp_world_size=dcp_world_size_for_kv_cache_spec( + kv_cache_group.kv_cache_spec, dcp_world_size + ), pcp_world_size=pcp_world_size, scheduler_block_size=self.scheduler_block_size, needs_kv_cache_zeroing=self.kv_cache_config.needs_kv_cache_zeroing, @@ -506,11 +509,9 @@ class UnitaryKVCacheCoordinator(KVCacheCoordinator): num_prefill_lookahead=num_prefill_lookahead, ) self.kv_cache_spec = self.kv_cache_config.kv_cache_groups[0].kv_cache_spec - self.block_size = self.kv_cache_spec.block_size - self.dcp_world_size = dcp_world_size + self.dcp_world_size = self.single_type_managers[0].dcp_world_size self.pcp_world_size = pcp_world_size - if dcp_world_size > 1: - self.block_size *= dcp_world_size + self.block_size = self.single_type_managers[0].block_size # For models using only Mamba, block_size is set to max_model_len when # prefix caching is disabled, and hash_block_size validation is skipped. assert not enable_caching or (hash_block_size == self.block_size), ( @@ -656,6 +657,9 @@ class HybridKVCacheCoordinator(KVCacheCoordinator): "cache managers require block-aligned lookups: %s.", ", ".join(sorted(unsupported_partial_hit_managers)), ) + cache_hit_alignment_tokens = self._cache_hit_alignment_tokens + for manager in self.single_type_managers: + manager.cache_hit_alignment_tokens = cache_hit_alignment_tokens self.verify_and_split_kv_cache_groups() @property @@ -852,11 +856,12 @@ class HybridKVCacheCoordinator(KVCacheCoordinator): kv_cache_spec=spec, drop_eagle_block=drop_eagle_block, alignment_tokens=self._cache_hit_alignment_tokens, - dcp_world_size=( - self.dcp_world_size - if isinstance(spec, FullAttentionSpec) - else 1 - ), + dcp_world_size=self.single_type_managers[ + first_group_id + ].dcp_world_size, + pcp_world_size=self.single_type_managers[ + first_group_id + ].pcp_world_size, ) if drop_eagle_block: eagle_verified.add(idx) @@ -913,6 +918,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator): hit_lengths: list[int] = [0] * num_groups for spec, group_ids, manager_cls, use_eagle in self.attention_groups: + manager = self.single_type_managers[group_ids[0]] blocks, group_hit = manager_cls.find_longest_cache_hit( block_hashes=block_hashes, max_length=max_cache_hit_length, @@ -921,6 +927,8 @@ class HybridKVCacheCoordinator(KVCacheCoordinator): kv_cache_spec=spec, drop_eagle_block=use_eagle, alignment_tokens=self._cache_hit_alignment_tokens, + dcp_world_size=manager.dcp_world_size, + pcp_world_size=manager.pcp_world_size, ) for gid, blks in zip(group_ids, blocks): hit_blocks[gid] = blks diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py index 45d8e8532e..0b57ce7e96 100644 --- a/vllm/v1/core/kv_cache_utils.py +++ b/vllm/v1/core/kv_cache_utils.py @@ -675,6 +675,28 @@ def resolve_dcp_kv_cache_spec(spec: KVCacheSpec, dcp_world_size: int) -> KVCache return replace(spec, block_size=block_size) +def dcp_world_size_for_kv_cache_spec(spec: KVCacheSpec, dcp_world_size: int) -> int: + """Return the DCP size that owns this group's block geometry. + + Full-attention KV (including MLA) is sharded across DCP ranks, so prefix + hashing and manager ``block_size`` use the process DCP size. Other specs + keep replicated per-rank state (Mamba, sliding window, chunked-local) and + must keep ``dcp_world_size=1`` even when the process runs with DCP > 1. + + Draft MLA groups on the sharded DSpark path are ``FullAttentionSpec`` / + ``MLAAttentionSpec`` and therefore keep the process DCP size. A replicated + draft group would need a different spec, not this helper. + """ + if dcp_world_size <= 1: + return 1 + inner = spec + if isinstance(spec, UniformTypeKVCacheSpecs): + inner = next(iter(spec.kv_cache_specs.values())) + if isinstance(inner, FullAttentionSpec): + return dcp_world_size + return 1 + + def resolve_kv_cache_block_sizes( kv_cache_config: KVCacheConfig, vllm_config: VllmConfig, diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index f3036eb051..443017012f 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -72,6 +72,9 @@ class SingleTypeKVCacheManager(ABC): block until the request finishes. """ self.scheduler_block_size = scheduler_block_size + # Hybrid fine-grained lookup may lower this after all participating + # managers have been validated by the coordinator. + self.cache_hit_alignment_tokens = scheduler_block_size # The block size for this manager; used for actual block allocation. self.block_size = kv_cache_spec.block_size self.dcp_world_size = dcp_world_size @@ -460,7 +463,7 @@ class SingleTypeKVCacheManager(ABC): block_mask = self.reachable_block_mask( start_block=num_cached_blocks, end_block=num_full_blocks, - alignment_tokens=self.scheduler_block_size, + alignment_tokens=self.cache_hit_alignment_tokens, kv_cache_spec=self.kv_cache_spec, use_eagle=self.use_eagle, retention_interval=retention_interval,