[Kimi-K3] support DCP partial prefix cache hit (#50493)

Signed-off-by: Summer Yang <girasoleyang@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Summer Yang
2026-08-17 17:05:51 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Yifan Qiao
parent 0e8989b416
commit 0db502c8d8
8 changed files with 523 additions and 37 deletions
@@ -4,6 +4,7 @@
"align") models: scheduler chunk splitting, partial tail registration, CoW
on partial hits, and same-step deferral."""
from math import lcm
from types import SimpleNamespace
from unittest.mock import MagicMock
@@ -68,12 +69,62 @@ def test_capable_connector_uses_divergent_partial_hit_lookup():
manager.get_computed_blocks.assert_not_called()
def test_mamba_align_split_partial_tail_schedule():
def make_full_mamba_manager(
*,
dcp_world_size: int,
hash_block_size: int = 2,
full_block_size: int = 4,
mamba_block_size: int = 4,
num_blocks: int = 32,
use_eagle: bool = False,
):
kv_cache_config = KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=full_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=mamba_block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
)
scheduler_block_size = lcm(
full_block_size * dcp_world_size,
mamba_block_size,
)
return make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
dcp_world_size=dcp_world_size,
scheduler_block_size=scheduler_block_size,
hash_block_size=hash_block_size,
use_eagle=use_eagle,
)
@pytest.mark.parametrize("dcp_world_size", [1, 4])
def test_mamba_align_split_partial_tail_schedule(dcp_world_size: int):
"""Chunk ends with partial hits on: block-aligned chunks, one extra stop
at the prompt's last hash boundary (registering the partial tail), then
the remaining tokens. block=512, hash=32, prompt=10000, budget=8192:
0 -> 8192 -> 9728 -> 9984 -> 10000."""
block_size = 512
scheduler_block_size = block_size * dcp_world_size
hash_block_size = 32
mock = SimpleNamespace(
cache_config=SimpleNamespace(block_size=block_size),
@@ -81,6 +132,8 @@ def test_mamba_align_split_partial_tail_schedule():
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
use_eagle=False,
hash_block_size=hash_block_size,
dcp_world_size=dcp_world_size,
scheduler_block_size=scheduler_block_size,
mamba_partial_cache_hit=True,
)
split = Scheduler._mamba_block_aligned_split
@@ -109,6 +162,7 @@ def test_mamba_align_split_partial_tail_schedule():
# stops at the next block boundary (10240), later chunk ends re-align.
req2 = make_request("1", [0] * 12000, hash_block_size, sha256)
req2.num_computed_tokens = 9984
assert req2.num_computed_tokens % scheduler_block_size != 0
assert split(self=mock, request=req2, num_new_tokens=2016) == 256
req2.num_computed_tokens = 10240
assert split(self=mock, request=req2, num_new_tokens=1000) == 512
@@ -1499,3 +1553,151 @@ def test_hybrid_sliding_window_group_disables_partial_hash_hits():
assert num_computed == mamba_block_size
assert len(computed_blocks.blocks[0]) * hash_block_size == num_computed
@pytest.mark.parametrize("dcp_world_size", [1, 2, 4])
def test_hybrid_partial_hash_hit_uses_cow_under_dcp(dcp_world_size: int):
hash_block_size = 2
physical_block_size = 4
manager = make_full_mamba_manager(
dcp_world_size=dcp_world_size,
hash_block_size=hash_block_size,
full_block_size=physical_block_size,
mamba_block_size=physical_block_size,
)
assert manager.coordinator.enable_partial_hash_hits
req0 = make_request("dcp-owner", [0, 0, 1, 1, 2, 2], 2, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
manager.free(req0)
manager.new_step_starts()
partial_hash = req0.block_hashes[2]
partial_full_block = manager.block_pool.get_cached_block(partial_hash, [0])
partial_mamba_block = manager.block_pool.get_cached_block(partial_hash, [1])
assert partial_full_block is not None
assert partial_mamba_block is not None
req1 = make_request("dcp-replay", [0, 0, 1, 1, 2, 2, 3, 3], 2, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req1)
assert num_computed == 6
full_block_size = physical_block_size * dcp_world_size
assert [len(group) for group in computed_blocks.blocks] == [
(6 + full_block_size - 1) // full_block_size,
2,
]
new_blocks = manager.allocate_slots(req1, 2, num_computed, computed_blocks)
assert new_blocks is not None
full_new_block_id = new_blocks.get_block_ids()[0][0]
mamba_new_block_id = new_blocks.get_block_ids()[1][0]
copies, retained = manager.take_kv_cache_block_copies()
assert KVCacheBlockCopy(partial_full_block[0].block_id, full_new_block_id) in copies
assert (
KVCacheBlockCopy(partial_mamba_block[0].block_id, mamba_new_block_id) in copies
)
manager.block_pool.free_blocks(retained)
@pytest.mark.parametrize("dcp_world_size", [2, 4])
def test_dcp_partial_hit_resumes_on_replicated_mamba_snapshot(
dcp_world_size: int,
):
block_size = 4
manager = make_full_mamba_manager(
dcp_world_size=dcp_world_size,
hash_block_size=block_size,
full_block_size=block_size,
mamba_block_size=block_size,
)
assert manager.coordinator.enable_partial_hash_hits
assert manager.coordinator._cache_hit_alignment_tokens == block_size
assert manager.coordinator.single_type_managers[0].block_size == (
block_size * dcp_world_size
)
assert manager.coordinator.single_type_managers[1].block_size == block_size
prefix = list(range(12))
req0 = make_request("snapshot-owner", prefix, block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 12, num_computed, computed_blocks) is not None
manager.free(req0)
manager.new_step_starts()
req1 = make_request(
"snapshot-replay", prefix + list(range(12, 16)), block_size, sha256
)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req1)
assert num_computed == 12
assert [len(group) for group in computed_blocks.blocks] == [
(12 + block_size * dcp_world_size - 1) // (block_size * dcp_world_size),
3,
]
partial_full_block = computed_blocks.blocks[0][-1]
new_blocks = manager.allocate_slots(req1, 4, num_computed, computed_blocks)
assert new_blocks is not None
full_new_block_id = new_blocks.get_block_ids()[0][0]
copies, retained = manager.take_kv_cache_block_copies()
assert KVCacheBlockCopy(partial_full_block.block_id, full_new_block_id) in copies
assert all(
copy.src_block_id != computed_blocks.blocks[1][-1].block_id for copy in copies
)
manager.block_pool.free_blocks(retained)
def test_dcp_joint_hit_is_bounded_by_replicated_mamba_snapshots():
block_size = 4
manager = make_full_mamba_manager(
dcp_world_size=2,
hash_block_size=block_size,
full_block_size=block_size,
mamba_block_size=block_size,
)
prefix = list(range(12))
req0 = make_request("joint-owner", prefix, block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 8, num_computed, computed_blocks) is not None
manager.new_step_starts()
replay_tokens = prefix + list(range(12, 16))
req1 = make_request("joint-before", replay_tokens, block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req1)
assert num_computed == 8
assert [len(group) for group in computed_blocks.blocks] == [1, 2]
req0.num_computed_tokens = 8
assert manager.allocate_slots(req0, 4) is not None
manager.new_step_starts()
req2 = make_request("joint-after", replay_tokens, block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req2)
assert num_computed == 12
assert [len(group) for group in computed_blocks.blocks] == [2, 3]
def test_dcp_partial_hit_with_eagle_rewinds_one_hash_unit():
hash_block_size = 2
manager = make_full_mamba_manager(
dcp_world_size=2,
hash_block_size=hash_block_size,
full_block_size=4,
mamba_block_size=4,
use_eagle=True,
)
req0 = make_request("eagle-owner", [7] * 6, hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 4, num_computed, computed_blocks) is not None
req0.num_computed_tokens = 4
manager.new_step_starts()
assert manager.allocate_slots(req0, 2) is not None
req0.num_computed_tokens = 6
manager.new_step_starts()
req1 = make_request("eagle-replay", [7] * 6 + [9] * 2, 2, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req1)
assert num_computed == 4
assert [len(group) for group in computed_blocks.blocks] == [1, 1]
assert manager.allocate_slots(req1, 4, num_computed, computed_blocks) is not None
@@ -55,9 +55,9 @@ def cache_full_block_and_partial_tail(
token_ids: list[int],
*,
enable_kv_cache_events: bool = False,
block_size: int = 6,
) -> tuple[BlockPool, Request, list[KVCacheBlock], BlockHash]:
hash_block_size = 2
block_size = 6
kv_cache_group_id = 0
req = make_request("0", token_ids, hash_block_size, sha256)
pool = BlockPool(
@@ -389,27 +389,34 @@ def test_reset_prefix_cache_clears_partial_entry_metadata():
assert pool.cached_block_hashes_by_block == {}
def test_evict_cached_block_removes_full_hash_and_partial_entry():
pool, req, blocks, partial_hash_10 = cache_full_block_and_partial_tail(
[0, 0, 1, 1, 2, 2, 3, 3, 4, 4]
@pytest.mark.parametrize("dcp_world_size", [1, 2, 4])
def test_evict_cached_block_removes_full_hash_and_partial_entry(
dcp_world_size: int,
):
block_size = 6 * dcp_world_size
partial_num_tokens = 2 * block_size - 2
pool, req, blocks, partial_hash = cache_full_block_and_partial_tail(
list(range(partial_num_tokens)), block_size=block_size
)
full_hash = BlockHashListWithBlockSize(req.block_hashes, 2, 6)[0]
full_hash = BlockHashListWithBlockSize(req.block_hashes, 2, block_size)[0]
assert pool.get_cached_block(full_hash, [0]) == [blocks[0]]
assert pool.get_cached_block(partial_hash_10, [0]) == [blocks[1]]
assert pool.get_cached_block(partial_hash, [0]) == [blocks[1]]
pool.evict_blocks({blocks[0].block_id, blocks[1].block_id})
assert pool.get_cached_block(full_hash, [0]) is None
assert pool.get_cached_block(partial_hash_10, [0]) is None
assert pool.get_cached_block(partial_hash, [0]) is None
assert pool.cached_block_hashes_by_block == {}
def test_partial_block_promotes_to_direct_full_block_hash():
@pytest.mark.parametrize("dcp_world_size", [1, 2, 4])
def test_partial_block_promotes_to_direct_full_block_hash(dcp_world_size: int):
hash_block_size = 2
block_size = 6
block_size = 6 * dcp_world_size
kv_cache_group_id = 0
token_ids = [0, 0, 1, 1, 2, 2, 3, 3, 4, 4]
partial_num_tokens = 2 * block_size - hash_block_size
token_ids = list(range(partial_num_tokens))
req = make_request("0", token_ids, hash_block_size, sha256)
pool = BlockPool(
num_gpu_blocks=3,
@@ -426,27 +433,22 @@ def test_partial_block_promotes_to_direct_full_block_hash():
block_size=block_size,
kv_cache_group_id=kv_cache_group_id,
)
partial_hash_10 = boundary_hash(req, hash_block_size, 10)
partial_hash = boundary_hash(req, hash_block_size, partial_num_tokens)
assert pool.cache_partial_block(
request=req,
block=blocks[1],
num_tokens=10,
num_tokens=partial_num_tokens,
kv_cache_group_id=kv_cache_group_id,
block_size=block_size,
)
assert pool.get_cached_block(partial_hash_10, [kv_cache_group_id]) == [blocks[1]]
assert pool.get_cached_block(partial_hash, [kv_cache_group_id]) == [blocks[1]]
req.append_output_token_ids([5, 5])
req.append_output_token_ids(list(range(partial_num_tokens, 2 * block_size)))
full_hashes = BlockHashListWithBlockSize(
req.block_hashes, hash_block_size, block_size
)
promoted_full_hash = full_hashes[1]
# The promoted full-block hash is the fine hash at the 12-token boundary,
# not a concatenation of the fine hashes inside the block.
assert promoted_full_hash == req.block_hashes[12 // hash_block_size - 1]
assert promoted_full_hash != BlockHash(
req.block_hashes[3] + req.block_hashes[4] + req.block_hashes[5]
)
assert promoted_full_hash == req.block_hashes[2 * block_size // hash_block_size - 1]
pool.cache_full_blocks(
request=req,
@@ -457,4 +459,4 @@ def test_partial_block_promotes_to_direct_full_block_hash():
kv_cache_group_id=kv_cache_group_id,
)
assert pool.get_cached_block(promoted_full_hash, [kv_cache_group_id]) == [blocks[1]]
assert pool.get_cached_block(partial_hash_10, [kv_cache_group_id]) is None
assert pool.get_cached_block(partial_hash, [kv_cache_group_id]) is None