[ROCm][DSpark][DCP] Serve prefix cache hits under DCP for Kimi-K3 (#53598)

Signed-off-by: Yichao Zhu <Yichao.Zhu@amd.com>
Co-authored-by: andyluo7 <andy.luo@amd.com>
This commit is contained in:
YukioZzz
2026-08-31 11:18:10 -07:00
committed by GitHub
co-authored by andyluo7
parent dbb7fffddb
commit 2ba984a5d0
6 changed files with 161 additions and 11 deletions
@@ -132,6 +132,31 @@ def make_full_mamba_manager(
)
def test_dcp_fine_hit_retention_uses_hash_alignment_without_eagle():
"""DCP must not discard a reusable Mamba state at hash alignment."""
hash_block_size = 2
manager = make_full_mamba_manager(
dcp_world_size=4,
hash_block_size=hash_block_size,
full_block_size=hash_block_size,
mamba_block_size=hash_block_size,
num_blocks=64,
)
manager.coordinator.retention_interval = 0
token_ids = list(range(7))
producer = make_request("producer", token_ids, hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(producer)
assert num_computed == 0
assert manager.allocate_slots(producer, 6, 0, computed_blocks) is not None
manager.free(producer)
manager.new_step_starts()
consumer = make_request("consumer", token_ids, hash_block_size, sha256)
_, num_computed, _ = manager.get_computed_blocks(consumer)
assert num_computed == 6
@pytest.mark.parametrize("dcp_world_size", [1, 4])
def test_mamba_align_split_partial_tail_schedule(dcp_world_size: int):
"""Chunk ends with partial hits on: block-aligned chunks, one extra stop
+15
View File
@@ -1318,6 +1318,21 @@ def test_project_kv_cache_groups_to_worker():
assert set(proj_spec.kv_cache_specs.keys()) == {"layer1", "layer3"}
def test_dcp_world_size_for_kv_cache_spec_shards_full_attention_only():
dcp = 8
full = FullAttentionSpec(
block_size=16, num_kv_heads=1, head_size=1, dtype=torch.float32
)
mla = new_mla_spec()
mamba = new_mamba_spec()
uniform_mla = UniformTypeKVCacheSpecs(block_size=16, kv_cache_specs={"layer": mla})
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(full, dcp) == dcp
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(mla, dcp) == dcp
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(uniform_mla, dcp) == dcp
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(mamba, dcp) == 1
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(full, 1) == 1
@pytest.mark.parametrize(
"layer_type,dcp_size,expected_width",
[
+77
View File
@@ -232,6 +232,83 @@ def make_kv_cache_config_three_types(
)
def test_prefix_cache_hit_uses_per_group_dcp_geometry():
"""Prefix lookup must use each group's DCP size, not the process-wide one.
Target and draft MLA are both sharded (DCP=8); Mamba stays replicated
(DCP=1). Hits then align to the sharded full-attention block, not the
unsharded page size.
"""
block_size = 16
dcp = 8
sharded_block = block_size * dcp
kv_cache_config = KVCacheConfig(
num_blocks=64,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["target_mla"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["draft_mla"],
MLAAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
),
),
],
)
manager = KVCacheManager(
kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
scheduler_block_size=sharded_block,
dcp_world_size=dcp,
)
target_mgr, draft_mgr, mamba_mgr = manager.coordinator.single_type_managers
assert target_mgr.dcp_world_size == dcp
assert draft_mgr.dcp_world_size == dcp
assert mamba_mgr.dcp_world_size == 1
assert target_mgr.block_size == sharded_block
assert draft_mgr.block_size == sharded_block
assert mamba_mgr.block_size == block_size
hash_fn = sha256
common_token_ids = [i for i in range(2) for _ in range(sharded_block)]
req0 = make_request("0", common_token_ids + [99] * 7, block_size, hash_fn)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
assert num_computed_tokens == 0
blocks = manager.allocate_slots(
req0, len(req0.prompt_token_ids), 0, computed_blocks
)
assert blocks is not None
req1 = make_request("1", common_token_ids + [100] * 5, block_size, hash_fn)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
assert num_computed_tokens == 2 * sharded_block
assert [len(group) for group in computed_blocks.blocks] == [2, 2, 16]
manager.free(req0)
manager.free(req1)
@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor])
def test_prefill(hash_fn):
block_size = 16