[Kimi-K3] support DCP partial prefix cache hit (#50493)
Signed-off-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Yifan Qiao
parent
0e8989b416
commit
0db502c8d8
@@ -4,6 +4,7 @@
|
||||
"align") models: scheduler chunk splitting, partial tail registration, CoW
|
||||
on partial hits, and same-step deferral."""
|
||||
|
||||
from math import lcm
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
@@ -68,12 +69,62 @@ def test_capable_connector_uses_divergent_partial_hit_lookup():
|
||||
manager.get_computed_blocks.assert_not_called()
|
||||
|
||||
|
||||
def test_mamba_align_split_partial_tail_schedule():
|
||||
def make_full_mamba_manager(
|
||||
*,
|
||||
dcp_world_size: int,
|
||||
hash_block_size: int = 2,
|
||||
full_block_size: int = 4,
|
||||
mamba_block_size: int = 4,
|
||||
num_blocks: int = 32,
|
||||
use_eagle: bool = False,
|
||||
):
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=full_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=mamba_block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
scheduler_block_size = lcm(
|
||||
full_block_size * dcp_world_size,
|
||||
mamba_block_size,
|
||||
)
|
||||
return make_kv_cache_manager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
dcp_world_size=dcp_world_size,
|
||||
scheduler_block_size=scheduler_block_size,
|
||||
hash_block_size=hash_block_size,
|
||||
use_eagle=use_eagle,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dcp_world_size", [1, 4])
|
||||
def test_mamba_align_split_partial_tail_schedule(dcp_world_size: int):
|
||||
"""Chunk ends with partial hits on: block-aligned chunks, one extra stop
|
||||
at the prompt's last hash boundary (registering the partial tail), then
|
||||
the remaining tokens. block=512, hash=32, prompt=10000, budget=8192:
|
||||
0 -> 8192 -> 9728 -> 9984 -> 10000."""
|
||||
block_size = 512
|
||||
scheduler_block_size = block_size * dcp_world_size
|
||||
hash_block_size = 32
|
||||
mock = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=block_size),
|
||||
@@ -81,6 +132,8 @@ def test_mamba_align_split_partial_tail_schedule():
|
||||
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
||||
use_eagle=False,
|
||||
hash_block_size=hash_block_size,
|
||||
dcp_world_size=dcp_world_size,
|
||||
scheduler_block_size=scheduler_block_size,
|
||||
mamba_partial_cache_hit=True,
|
||||
)
|
||||
split = Scheduler._mamba_block_aligned_split
|
||||
@@ -109,6 +162,7 @@ def test_mamba_align_split_partial_tail_schedule():
|
||||
# stops at the next block boundary (10240), later chunk ends re-align.
|
||||
req2 = make_request("1", [0] * 12000, hash_block_size, sha256)
|
||||
req2.num_computed_tokens = 9984
|
||||
assert req2.num_computed_tokens % scheduler_block_size != 0
|
||||
assert split(self=mock, request=req2, num_new_tokens=2016) == 256
|
||||
req2.num_computed_tokens = 10240
|
||||
assert split(self=mock, request=req2, num_new_tokens=1000) == 512
|
||||
@@ -1499,3 +1553,151 @@ def test_hybrid_sliding_window_group_disables_partial_hash_hits():
|
||||
|
||||
assert num_computed == mamba_block_size
|
||||
assert len(computed_blocks.blocks[0]) * hash_block_size == num_computed
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dcp_world_size", [1, 2, 4])
|
||||
def test_hybrid_partial_hash_hit_uses_cow_under_dcp(dcp_world_size: int):
|
||||
hash_block_size = 2
|
||||
physical_block_size = 4
|
||||
manager = make_full_mamba_manager(
|
||||
dcp_world_size=dcp_world_size,
|
||||
hash_block_size=hash_block_size,
|
||||
full_block_size=physical_block_size,
|
||||
mamba_block_size=physical_block_size,
|
||||
)
|
||||
assert manager.coordinator.enable_partial_hash_hits
|
||||
|
||||
req0 = make_request("dcp-owner", [0, 0, 1, 1, 2, 2], 2, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
manager.free(req0)
|
||||
manager.new_step_starts()
|
||||
|
||||
partial_hash = req0.block_hashes[2]
|
||||
partial_full_block = manager.block_pool.get_cached_block(partial_hash, [0])
|
||||
partial_mamba_block = manager.block_pool.get_cached_block(partial_hash, [1])
|
||||
assert partial_full_block is not None
|
||||
assert partial_mamba_block is not None
|
||||
|
||||
req1 = make_request("dcp-replay", [0, 0, 1, 1, 2, 2, 3, 3], 2, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req1)
|
||||
assert num_computed == 6
|
||||
full_block_size = physical_block_size * dcp_world_size
|
||||
assert [len(group) for group in computed_blocks.blocks] == [
|
||||
(6 + full_block_size - 1) // full_block_size,
|
||||
2,
|
||||
]
|
||||
|
||||
new_blocks = manager.allocate_slots(req1, 2, num_computed, computed_blocks)
|
||||
assert new_blocks is not None
|
||||
full_new_block_id = new_blocks.get_block_ids()[0][0]
|
||||
mamba_new_block_id = new_blocks.get_block_ids()[1][0]
|
||||
copies, retained = manager.take_kv_cache_block_copies()
|
||||
assert KVCacheBlockCopy(partial_full_block[0].block_id, full_new_block_id) in copies
|
||||
assert (
|
||||
KVCacheBlockCopy(partial_mamba_block[0].block_id, mamba_new_block_id) in copies
|
||||
)
|
||||
manager.block_pool.free_blocks(retained)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dcp_world_size", [2, 4])
|
||||
def test_dcp_partial_hit_resumes_on_replicated_mamba_snapshot(
|
||||
dcp_world_size: int,
|
||||
):
|
||||
block_size = 4
|
||||
manager = make_full_mamba_manager(
|
||||
dcp_world_size=dcp_world_size,
|
||||
hash_block_size=block_size,
|
||||
full_block_size=block_size,
|
||||
mamba_block_size=block_size,
|
||||
)
|
||||
assert manager.coordinator.enable_partial_hash_hits
|
||||
assert manager.coordinator._cache_hit_alignment_tokens == block_size
|
||||
assert manager.coordinator.single_type_managers[0].block_size == (
|
||||
block_size * dcp_world_size
|
||||
)
|
||||
assert manager.coordinator.single_type_managers[1].block_size == block_size
|
||||
|
||||
prefix = list(range(12))
|
||||
req0 = make_request("snapshot-owner", prefix, block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 12, num_computed, computed_blocks) is not None
|
||||
manager.free(req0)
|
||||
manager.new_step_starts()
|
||||
|
||||
req1 = make_request(
|
||||
"snapshot-replay", prefix + list(range(12, 16)), block_size, sha256
|
||||
)
|
||||
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req1)
|
||||
assert num_computed == 12
|
||||
assert [len(group) for group in computed_blocks.blocks] == [
|
||||
(12 + block_size * dcp_world_size - 1) // (block_size * dcp_world_size),
|
||||
3,
|
||||
]
|
||||
partial_full_block = computed_blocks.blocks[0][-1]
|
||||
new_blocks = manager.allocate_slots(req1, 4, num_computed, computed_blocks)
|
||||
assert new_blocks is not None
|
||||
full_new_block_id = new_blocks.get_block_ids()[0][0]
|
||||
copies, retained = manager.take_kv_cache_block_copies()
|
||||
assert KVCacheBlockCopy(partial_full_block.block_id, full_new_block_id) in copies
|
||||
assert all(
|
||||
copy.src_block_id != computed_blocks.blocks[1][-1].block_id for copy in copies
|
||||
)
|
||||
manager.block_pool.free_blocks(retained)
|
||||
|
||||
|
||||
def test_dcp_joint_hit_is_bounded_by_replicated_mamba_snapshots():
|
||||
block_size = 4
|
||||
manager = make_full_mamba_manager(
|
||||
dcp_world_size=2,
|
||||
hash_block_size=block_size,
|
||||
full_block_size=block_size,
|
||||
mamba_block_size=block_size,
|
||||
)
|
||||
prefix = list(range(12))
|
||||
req0 = make_request("joint-owner", prefix, block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 8, num_computed, computed_blocks) is not None
|
||||
manager.new_step_starts()
|
||||
|
||||
replay_tokens = prefix + list(range(12, 16))
|
||||
req1 = make_request("joint-before", replay_tokens, block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req1)
|
||||
assert num_computed == 8
|
||||
assert [len(group) for group in computed_blocks.blocks] == [1, 2]
|
||||
|
||||
req0.num_computed_tokens = 8
|
||||
assert manager.allocate_slots(req0, 4) is not None
|
||||
manager.new_step_starts()
|
||||
|
||||
req2 = make_request("joint-after", replay_tokens, block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req2)
|
||||
assert num_computed == 12
|
||||
assert [len(group) for group in computed_blocks.blocks] == [2, 3]
|
||||
|
||||
|
||||
def test_dcp_partial_hit_with_eagle_rewinds_one_hash_unit():
|
||||
hash_block_size = 2
|
||||
manager = make_full_mamba_manager(
|
||||
dcp_world_size=2,
|
||||
hash_block_size=hash_block_size,
|
||||
full_block_size=4,
|
||||
mamba_block_size=4,
|
||||
use_eagle=True,
|
||||
)
|
||||
|
||||
req0 = make_request("eagle-owner", [7] * 6, hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 4, num_computed, computed_blocks) is not None
|
||||
req0.num_computed_tokens = 4
|
||||
manager.new_step_starts()
|
||||
assert manager.allocate_slots(req0, 2) is not None
|
||||
req0.num_computed_tokens = 6
|
||||
manager.new_step_starts()
|
||||
|
||||
req1 = make_request("eagle-replay", [7] * 6 + [9] * 2, 2, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req1)
|
||||
assert num_computed == 4
|
||||
assert [len(group) for group in computed_blocks.blocks] == [1, 1]
|
||||
assert manager.allocate_slots(req1, 4, num_computed, computed_blocks) is not None
|
||||
|
||||
@@ -55,9 +55,9 @@ def cache_full_block_and_partial_tail(
|
||||
token_ids: list[int],
|
||||
*,
|
||||
enable_kv_cache_events: bool = False,
|
||||
block_size: int = 6,
|
||||
) -> tuple[BlockPool, Request, list[KVCacheBlock], BlockHash]:
|
||||
hash_block_size = 2
|
||||
block_size = 6
|
||||
kv_cache_group_id = 0
|
||||
req = make_request("0", token_ids, hash_block_size, sha256)
|
||||
pool = BlockPool(
|
||||
@@ -389,27 +389,34 @@ def test_reset_prefix_cache_clears_partial_entry_metadata():
|
||||
assert pool.cached_block_hashes_by_block == {}
|
||||
|
||||
|
||||
def test_evict_cached_block_removes_full_hash_and_partial_entry():
|
||||
pool, req, blocks, partial_hash_10 = cache_full_block_and_partial_tail(
|
||||
[0, 0, 1, 1, 2, 2, 3, 3, 4, 4]
|
||||
@pytest.mark.parametrize("dcp_world_size", [1, 2, 4])
|
||||
def test_evict_cached_block_removes_full_hash_and_partial_entry(
|
||||
dcp_world_size: int,
|
||||
):
|
||||
block_size = 6 * dcp_world_size
|
||||
partial_num_tokens = 2 * block_size - 2
|
||||
pool, req, blocks, partial_hash = cache_full_block_and_partial_tail(
|
||||
list(range(partial_num_tokens)), block_size=block_size
|
||||
)
|
||||
full_hash = BlockHashListWithBlockSize(req.block_hashes, 2, 6)[0]
|
||||
full_hash = BlockHashListWithBlockSize(req.block_hashes, 2, block_size)[0]
|
||||
|
||||
assert pool.get_cached_block(full_hash, [0]) == [blocks[0]]
|
||||
assert pool.get_cached_block(partial_hash_10, [0]) == [blocks[1]]
|
||||
assert pool.get_cached_block(partial_hash, [0]) == [blocks[1]]
|
||||
|
||||
pool.evict_blocks({blocks[0].block_id, blocks[1].block_id})
|
||||
|
||||
assert pool.get_cached_block(full_hash, [0]) is None
|
||||
assert pool.get_cached_block(partial_hash_10, [0]) is None
|
||||
assert pool.get_cached_block(partial_hash, [0]) is None
|
||||
assert pool.cached_block_hashes_by_block == {}
|
||||
|
||||
|
||||
def test_partial_block_promotes_to_direct_full_block_hash():
|
||||
@pytest.mark.parametrize("dcp_world_size", [1, 2, 4])
|
||||
def test_partial_block_promotes_to_direct_full_block_hash(dcp_world_size: int):
|
||||
hash_block_size = 2
|
||||
block_size = 6
|
||||
block_size = 6 * dcp_world_size
|
||||
kv_cache_group_id = 0
|
||||
token_ids = [0, 0, 1, 1, 2, 2, 3, 3, 4, 4]
|
||||
partial_num_tokens = 2 * block_size - hash_block_size
|
||||
token_ids = list(range(partial_num_tokens))
|
||||
req = make_request("0", token_ids, hash_block_size, sha256)
|
||||
pool = BlockPool(
|
||||
num_gpu_blocks=3,
|
||||
@@ -426,27 +433,22 @@ def test_partial_block_promotes_to_direct_full_block_hash():
|
||||
block_size=block_size,
|
||||
kv_cache_group_id=kv_cache_group_id,
|
||||
)
|
||||
partial_hash_10 = boundary_hash(req, hash_block_size, 10)
|
||||
partial_hash = boundary_hash(req, hash_block_size, partial_num_tokens)
|
||||
assert pool.cache_partial_block(
|
||||
request=req,
|
||||
block=blocks[1],
|
||||
num_tokens=10,
|
||||
num_tokens=partial_num_tokens,
|
||||
kv_cache_group_id=kv_cache_group_id,
|
||||
block_size=block_size,
|
||||
)
|
||||
assert pool.get_cached_block(partial_hash_10, [kv_cache_group_id]) == [blocks[1]]
|
||||
assert pool.get_cached_block(partial_hash, [kv_cache_group_id]) == [blocks[1]]
|
||||
|
||||
req.append_output_token_ids([5, 5])
|
||||
req.append_output_token_ids(list(range(partial_num_tokens, 2 * block_size)))
|
||||
full_hashes = BlockHashListWithBlockSize(
|
||||
req.block_hashes, hash_block_size, block_size
|
||||
)
|
||||
promoted_full_hash = full_hashes[1]
|
||||
# The promoted full-block hash is the fine hash at the 12-token boundary,
|
||||
# not a concatenation of the fine hashes inside the block.
|
||||
assert promoted_full_hash == req.block_hashes[12 // hash_block_size - 1]
|
||||
assert promoted_full_hash != BlockHash(
|
||||
req.block_hashes[3] + req.block_hashes[4] + req.block_hashes[5]
|
||||
)
|
||||
assert promoted_full_hash == req.block_hashes[2 * block_size // hash_block_size - 1]
|
||||
|
||||
pool.cache_full_blocks(
|
||||
request=req,
|
||||
@@ -457,4 +459,4 @@ def test_partial_block_promotes_to_direct_full_block_hash():
|
||||
kv_cache_group_id=kv_cache_group_id,
|
||||
)
|
||||
assert pool.get_cached_block(promoted_full_hash, [kv_cache_group_id]) == [blocks[1]]
|
||||
assert pool.get_cached_block(partial_hash_10, [kv_cache_group_id]) is None
|
||||
assert pool.get_cached_block(partial_hash, [kv_cache_group_id]) is None
|
||||
|
||||
Reference in New Issue
Block a user