[ROCm][DSpark][DCP] Serve prefix cache hits under DCP for Kimi-K3 (#53598)
Signed-off-by: Yichao Zhu <Yichao.Zhu@amd.com> Co-authored-by: andyluo7 <andy.luo@amd.com>
This commit is contained in:
@@ -132,6 +132,31 @@ def make_full_mamba_manager(
|
||||
)
|
||||
|
||||
|
||||
def test_dcp_fine_hit_retention_uses_hash_alignment_without_eagle():
|
||||
"""DCP must not discard a reusable Mamba state at hash alignment."""
|
||||
hash_block_size = 2
|
||||
manager = make_full_mamba_manager(
|
||||
dcp_world_size=4,
|
||||
hash_block_size=hash_block_size,
|
||||
full_block_size=hash_block_size,
|
||||
mamba_block_size=hash_block_size,
|
||||
num_blocks=64,
|
||||
)
|
||||
manager.coordinator.retention_interval = 0
|
||||
|
||||
token_ids = list(range(7))
|
||||
producer = make_request("producer", token_ids, hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(producer)
|
||||
assert num_computed == 0
|
||||
assert manager.allocate_slots(producer, 6, 0, computed_blocks) is not None
|
||||
manager.free(producer)
|
||||
manager.new_step_starts()
|
||||
|
||||
consumer = make_request("consumer", token_ids, hash_block_size, sha256)
|
||||
_, num_computed, _ = manager.get_computed_blocks(consumer)
|
||||
assert num_computed == 6
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dcp_world_size", [1, 4])
|
||||
def test_mamba_align_split_partial_tail_schedule(dcp_world_size: int):
|
||||
"""Chunk ends with partial hits on: block-aligned chunks, one extra stop
|
||||
|
||||
@@ -1318,6 +1318,21 @@ def test_project_kv_cache_groups_to_worker():
|
||||
assert set(proj_spec.kv_cache_specs.keys()) == {"layer1", "layer3"}
|
||||
|
||||
|
||||
def test_dcp_world_size_for_kv_cache_spec_shards_full_attention_only():
|
||||
dcp = 8
|
||||
full = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=1, dtype=torch.float32
|
||||
)
|
||||
mla = new_mla_spec()
|
||||
mamba = new_mamba_spec()
|
||||
uniform_mla = UniformTypeKVCacheSpecs(block_size=16, kv_cache_specs={"layer": mla})
|
||||
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(full, dcp) == dcp
|
||||
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(mla, dcp) == dcp
|
||||
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(uniform_mla, dcp) == dcp
|
||||
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(mamba, dcp) == 1
|
||||
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(full, 1) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"layer_type,dcp_size,expected_width",
|
||||
[
|
||||
|
||||
@@ -232,6 +232,83 @@ def make_kv_cache_config_three_types(
|
||||
)
|
||||
|
||||
|
||||
def test_prefix_cache_hit_uses_per_group_dcp_geometry():
|
||||
"""Prefix lookup must use each group's DCP size, not the process-wide one.
|
||||
|
||||
Target and draft MLA are both sharded (DCP=8); Mamba stays replicated
|
||||
(DCP=1). Hits then align to the sharded full-attention block, not the
|
||||
unsharded page size.
|
||||
"""
|
||||
block_size = 16
|
||||
dcp = 8
|
||||
sharded_block = block_size * dcp
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=64,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["target_mla"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["draft_mla"],
|
||||
MLAAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = KVCacheManager(
|
||||
kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
scheduler_block_size=sharded_block,
|
||||
dcp_world_size=dcp,
|
||||
)
|
||||
target_mgr, draft_mgr, mamba_mgr = manager.coordinator.single_type_managers
|
||||
assert target_mgr.dcp_world_size == dcp
|
||||
assert draft_mgr.dcp_world_size == dcp
|
||||
assert mamba_mgr.dcp_world_size == 1
|
||||
assert target_mgr.block_size == sharded_block
|
||||
assert draft_mgr.block_size == sharded_block
|
||||
assert mamba_mgr.block_size == block_size
|
||||
|
||||
hash_fn = sha256
|
||||
common_token_ids = [i for i in range(2) for _ in range(sharded_block)]
|
||||
req0 = make_request("0", common_token_ids + [99] * 7, block_size, hash_fn)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
|
||||
assert num_computed_tokens == 0
|
||||
blocks = manager.allocate_slots(
|
||||
req0, len(req0.prompt_token_ids), 0, computed_blocks
|
||||
)
|
||||
assert blocks is not None
|
||||
|
||||
req1 = make_request("1", common_token_ids + [100] * 5, block_size, hash_fn)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
|
||||
assert num_computed_tokens == 2 * sharded_block
|
||||
assert [len(group) for group in computed_blocks.blocks] == [2, 2, 16]
|
||||
|
||||
manager.free(req0)
|
||||
manager.free(req1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor])
|
||||
def test_prefill(hash_fn):
|
||||
block_size = 16
|
||||
|
||||
@@ -11,6 +11,7 @@ from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
KVCacheBlock,
|
||||
dcp_world_size_for_kv_cache_spec,
|
||||
)
|
||||
from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
CrossAttentionManager,
|
||||
@@ -141,7 +142,9 @@ class KVCacheCoordinator(ABC):
|
||||
block_pool=self.block_pool,
|
||||
enable_caching=enable_caching,
|
||||
kv_cache_group_id=i,
|
||||
dcp_world_size=dcp_world_size,
|
||||
dcp_world_size=dcp_world_size_for_kv_cache_spec(
|
||||
kv_cache_group.kv_cache_spec, dcp_world_size
|
||||
),
|
||||
pcp_world_size=pcp_world_size,
|
||||
scheduler_block_size=self.scheduler_block_size,
|
||||
needs_kv_cache_zeroing=self.kv_cache_config.needs_kv_cache_zeroing,
|
||||
@@ -506,11 +509,9 @@ class UnitaryKVCacheCoordinator(KVCacheCoordinator):
|
||||
num_prefill_lookahead=num_prefill_lookahead,
|
||||
)
|
||||
self.kv_cache_spec = self.kv_cache_config.kv_cache_groups[0].kv_cache_spec
|
||||
self.block_size = self.kv_cache_spec.block_size
|
||||
self.dcp_world_size = dcp_world_size
|
||||
self.dcp_world_size = self.single_type_managers[0].dcp_world_size
|
||||
self.pcp_world_size = pcp_world_size
|
||||
if dcp_world_size > 1:
|
||||
self.block_size *= dcp_world_size
|
||||
self.block_size = self.single_type_managers[0].block_size
|
||||
# For models using only Mamba, block_size is set to max_model_len when
|
||||
# prefix caching is disabled, and hash_block_size validation is skipped.
|
||||
assert not enable_caching or (hash_block_size == self.block_size), (
|
||||
@@ -656,6 +657,9 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
"cache managers require block-aligned lookups: %s.",
|
||||
", ".join(sorted(unsupported_partial_hit_managers)),
|
||||
)
|
||||
cache_hit_alignment_tokens = self._cache_hit_alignment_tokens
|
||||
for manager in self.single_type_managers:
|
||||
manager.cache_hit_alignment_tokens = cache_hit_alignment_tokens
|
||||
self.verify_and_split_kv_cache_groups()
|
||||
|
||||
@property
|
||||
@@ -852,11 +856,12 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
kv_cache_spec=spec,
|
||||
drop_eagle_block=drop_eagle_block,
|
||||
alignment_tokens=self._cache_hit_alignment_tokens,
|
||||
dcp_world_size=(
|
||||
self.dcp_world_size
|
||||
if isinstance(spec, FullAttentionSpec)
|
||||
else 1
|
||||
),
|
||||
dcp_world_size=self.single_type_managers[
|
||||
first_group_id
|
||||
].dcp_world_size,
|
||||
pcp_world_size=self.single_type_managers[
|
||||
first_group_id
|
||||
].pcp_world_size,
|
||||
)
|
||||
if drop_eagle_block:
|
||||
eagle_verified.add(idx)
|
||||
@@ -913,6 +918,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
hit_lengths: list[int] = [0] * num_groups
|
||||
|
||||
for spec, group_ids, manager_cls, use_eagle in self.attention_groups:
|
||||
manager = self.single_type_managers[group_ids[0]]
|
||||
blocks, group_hit = manager_cls.find_longest_cache_hit(
|
||||
block_hashes=block_hashes,
|
||||
max_length=max_cache_hit_length,
|
||||
@@ -921,6 +927,8 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
kv_cache_spec=spec,
|
||||
drop_eagle_block=use_eagle,
|
||||
alignment_tokens=self._cache_hit_alignment_tokens,
|
||||
dcp_world_size=manager.dcp_world_size,
|
||||
pcp_world_size=manager.pcp_world_size,
|
||||
)
|
||||
for gid, blks in zip(group_ids, blocks):
|
||||
hit_blocks[gid] = blks
|
||||
|
||||
@@ -675,6 +675,28 @@ def resolve_dcp_kv_cache_spec(spec: KVCacheSpec, dcp_world_size: int) -> KVCache
|
||||
return replace(spec, block_size=block_size)
|
||||
|
||||
|
||||
def dcp_world_size_for_kv_cache_spec(spec: KVCacheSpec, dcp_world_size: int) -> int:
|
||||
"""Return the DCP size that owns this group's block geometry.
|
||||
|
||||
Full-attention KV (including MLA) is sharded across DCP ranks, so prefix
|
||||
hashing and manager ``block_size`` use the process DCP size. Other specs
|
||||
keep replicated per-rank state (Mamba, sliding window, chunked-local) and
|
||||
must keep ``dcp_world_size=1`` even when the process runs with DCP > 1.
|
||||
|
||||
Draft MLA groups on the sharded DSpark path are ``FullAttentionSpec`` /
|
||||
``MLAAttentionSpec`` and therefore keep the process DCP size. A replicated
|
||||
draft group would need a different spec, not this helper.
|
||||
"""
|
||||
if dcp_world_size <= 1:
|
||||
return 1
|
||||
inner = spec
|
||||
if isinstance(spec, UniformTypeKVCacheSpecs):
|
||||
inner = next(iter(spec.kv_cache_specs.values()))
|
||||
if isinstance(inner, FullAttentionSpec):
|
||||
return dcp_world_size
|
||||
return 1
|
||||
|
||||
|
||||
def resolve_kv_cache_block_sizes(
|
||||
kv_cache_config: KVCacheConfig,
|
||||
vllm_config: VllmConfig,
|
||||
|
||||
@@ -72,6 +72,9 @@ class SingleTypeKVCacheManager(ABC):
|
||||
block until the request finishes.
|
||||
"""
|
||||
self.scheduler_block_size = scheduler_block_size
|
||||
# Hybrid fine-grained lookup may lower this after all participating
|
||||
# managers have been validated by the coordinator.
|
||||
self.cache_hit_alignment_tokens = scheduler_block_size
|
||||
# The block size for this manager; used for actual block allocation.
|
||||
self.block_size = kv_cache_spec.block_size
|
||||
self.dcp_world_size = dcp_world_size
|
||||
@@ -460,7 +463,7 @@ class SingleTypeKVCacheManager(ABC):
|
||||
block_mask = self.reachable_block_mask(
|
||||
start_block=num_cached_blocks,
|
||||
end_block=num_full_blocks,
|
||||
alignment_tokens=self.scheduler_block_size,
|
||||
alignment_tokens=self.cache_hit_alignment_tokens,
|
||||
kv_cache_spec=self.kv_cache_spec,
|
||||
use_eagle=self.use_eagle,
|
||||
retention_interval=retention_interval,
|
||||
|
||||
Reference in New Issue
Block a user