[Kimi K3] Support internal prefix checkpoints with partial prefix caching and spec-decoding (#53614)
Signed-off-by: Ziming Huang <zelda.huanghuang@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
1970f3ed4b
commit
144e79c810
@@ -92,8 +92,23 @@ def make_full_mamba_manager(
|
||||
mamba_block_size: int = 4,
|
||||
num_blocks: int = 32,
|
||||
use_eagle: bool = False,
|
||||
num_speculative_blocks: int = 0,
|
||||
num_prefill_checkpoint_blocks: int = 0,
|
||||
):
|
||||
mamba_group = KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=mamba_block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
num_speculative_blocks=num_speculative_blocks,
|
||||
num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks,
|
||||
prefill_checkpoint_alignment=(
|
||||
16 if num_prefill_checkpoint_blocks > 0 else None
|
||||
),
|
||||
),
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[],
|
||||
@@ -107,16 +122,7 @@ def make_full_mamba_manager(
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=mamba_block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks,
|
||||
),
|
||||
),
|
||||
mamba_group,
|
||||
],
|
||||
)
|
||||
scheduler_block_size = lcm(
|
||||
@@ -169,6 +175,7 @@ def test_mamba_align_split_partial_tail_schedule(dcp_world_size: int):
|
||||
scheduler_block_size = block_size * dcp_world_size
|
||||
hash_block_size = 32
|
||||
mock = SimpleNamespace(
|
||||
block_size=scheduler_block_size,
|
||||
cache_config=SimpleNamespace(block_size=block_size),
|
||||
max_num_scheduled_tokens=8192,
|
||||
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
||||
@@ -217,6 +224,7 @@ def test_mamba_align_split_when_block_exceeds_scheduling_budget():
|
||||
token_budget = 8192
|
||||
prompt_length = 30000
|
||||
mock = SimpleNamespace(
|
||||
block_size=block_size,
|
||||
cache_config=SimpleNamespace(block_size=block_size),
|
||||
max_num_scheduled_tokens=token_budget,
|
||||
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
||||
@@ -254,6 +262,7 @@ def test_mamba_align_split_when_block_exceeds_long_prefill_threshold():
|
||||
long_prefill_threshold = 384
|
||||
prompt_length = 1300
|
||||
mock = SimpleNamespace(
|
||||
block_size=block_size,
|
||||
cache_config=SimpleNamespace(block_size=block_size),
|
||||
max_num_scheduled_tokens=token_budget,
|
||||
scheduler_config=SimpleNamespace(
|
||||
@@ -557,9 +566,9 @@ def test_partial_hit_then_internal_checkpoint_uses_distinct_mamba_blocks():
|
||||
assert mamba_blocks[3].block_id == running_block_id
|
||||
|
||||
|
||||
def test_internal_checkpoint_requires_block_aligned_start():
|
||||
hash_block_size = 2
|
||||
mamba_block_size = 16
|
||||
def test_internal_checkpoint_uses_partial_hash_lifecycle():
|
||||
hash_block_size = 16
|
||||
mamba_block_size = 32
|
||||
manager = make_full_mamba_manager(
|
||||
dcp_world_size=1,
|
||||
hash_block_size=hash_block_size,
|
||||
@@ -567,23 +576,90 @@ def test_internal_checkpoint_requires_block_aligned_start():
|
||||
mamba_block_size=mamba_block_size,
|
||||
num_prefill_checkpoint_blocks=1,
|
||||
)
|
||||
request = make_request("producer", list(range(50)), hash_block_size, sha256)
|
||||
request = make_request("producer", list(range(120)), hash_block_size, sha256)
|
||||
|
||||
# Compute token 0 first, so the next query starts at token 1, which is not
|
||||
# aligned to the Mamba block size.
|
||||
assert manager.allocate_slots(request, 1) is not None
|
||||
request.num_computed_tokens = 1
|
||||
manager.new_step_starts()
|
||||
|
||||
new_blocks = manager.allocate_slots(request, 49)
|
||||
new_blocks = manager.allocate_slots(request, request.num_tokens)
|
||||
|
||||
assert new_blocks is not None
|
||||
mamba_blocks = manager.get_blocks(request.request_id).blocks[1]
|
||||
checkpoint_block_idx = 48 // mamba_block_size - 1
|
||||
assert mamba_blocks[checkpoint_block_idx].is_null
|
||||
assert not mamba_blocks[-1].is_null
|
||||
checkpoint_hash = request.block_hashes[48 // hash_block_size - 1]
|
||||
assert manager.block_pool.get_cached_block(checkpoint_hash, [1]) is None
|
||||
checkpoint_idx = 2
|
||||
checkpoint_block = mamba_blocks[checkpoint_idx]
|
||||
running_block = mamba_blocks[checkpoint_idx + 1]
|
||||
assert not checkpoint_block.is_null
|
||||
assert not running_block.is_null
|
||||
assert checkpoint_block is not running_block
|
||||
|
||||
# The internal block is exported at state@112, replacing its temporary
|
||||
# full-block state@96 key while retaining #52789's req_to_blocks ownership.
|
||||
partial_hash = request.block_hashes[112 // hash_block_size - 1]
|
||||
partial_hit = manager.block_pool.get_cached_block(partial_hash, [1])
|
||||
assert partial_hit is not None
|
||||
assert partial_hit[0] is checkpoint_block
|
||||
assert checkpoint_block.block_hash_num_tokens == 112
|
||||
full_hash = request.block_hashes[96 // hash_block_size - 1]
|
||||
assert manager.block_pool.get_cached_block(full_hash, [1]) is None
|
||||
assert drain_boundary_state_offloads(manager) == {
|
||||
request.request_id: [(1, checkpoint_block.block_id, 112)]
|
||||
}
|
||||
|
||||
manager.free(request)
|
||||
replay = make_request("replay", list(range(120)), hash_block_size, sha256)
|
||||
_, num_computed, _ = manager.get_computed_blocks(replay)
|
||||
assert num_computed == 112
|
||||
|
||||
|
||||
def test_eagle_block_aligned_checkpoint_replaces_newer_hash():
|
||||
hash_block_size = mamba_block_size = 32
|
||||
manager = make_full_mamba_manager(
|
||||
dcp_world_size=1,
|
||||
hash_block_size=hash_block_size,
|
||||
full_block_size=hash_block_size,
|
||||
mamba_block_size=mamba_block_size,
|
||||
use_eagle=True,
|
||||
num_speculative_blocks=3,
|
||||
num_prefill_checkpoint_blocks=1,
|
||||
)
|
||||
request = make_request("producer", list(range(104)), hash_block_size, sha256)
|
||||
|
||||
assert manager.allocate_slots(request, request.num_tokens) is not None
|
||||
|
||||
mamba_blocks = manager.get_blocks(request.request_id).blocks[1]
|
||||
checkpoint_block = mamba_blocks[2]
|
||||
checkpoint_hash = request.block_hashes[64 // hash_block_size - 1]
|
||||
checkpoint_hit = manager.block_pool.get_cached_block(checkpoint_hash, [1])
|
||||
assert checkpoint_hit is not None
|
||||
assert checkpoint_hit[0] is checkpoint_block
|
||||
assert checkpoint_block.block_hash_num_tokens == 64
|
||||
|
||||
overwritten_full_hash = request.block_hashes[96 // hash_block_size - 1]
|
||||
assert manager.block_pool.get_cached_block(overwritten_full_hash, [1]) is None
|
||||
|
||||
|
||||
def test_hash_aligned_query_end_uses_regular_partial_tail():
|
||||
hash_block_size = 2
|
||||
mamba_block_size = 4
|
||||
manager = make_full_mamba_manager(
|
||||
dcp_world_size=1,
|
||||
hash_block_size=hash_block_size,
|
||||
full_block_size=hash_block_size,
|
||||
mamba_block_size=mamba_block_size,
|
||||
num_prefill_checkpoint_blocks=1,
|
||||
)
|
||||
request = make_request("producer", list(range(14)), hash_block_size, sha256)
|
||||
|
||||
new_blocks = manager.allocate_slots(request, request.num_tokens)
|
||||
|
||||
assert new_blocks is not None
|
||||
mamba_manager = manager.coordinator.single_type_managers[1]
|
||||
assert mamba_manager._checkpoint_positions[request.request_id] == 0
|
||||
mamba_blocks = manager.get_blocks(request.request_id).blocks[1]
|
||||
assert sum(not block.is_null for block in mamba_blocks) == 1
|
||||
|
||||
partial_hash = request.block_hashes[14 // hash_block_size - 1]
|
||||
cached = manager.block_pool.get_cached_block(partial_hash, [1])
|
||||
assert cached is not None
|
||||
assert cached[0] is mamba_blocks[14 // mamba_block_size]
|
||||
assert cached[0].block_hash_num_tokens == 14
|
||||
|
||||
|
||||
def test_external_mamba_hit_same_block_uses_running_cow_on_continue():
|
||||
@@ -699,18 +775,9 @@ def test_boundary_state_offloads_returns_cow_target():
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
|
||||
# Step A offers the materialized aligned checkpoint immediately.
|
||||
((group_id, block_id, boundary_tokens),) = drain_boundary_state_offloads(manager)[
|
||||
"0"
|
||||
]
|
||||
assert group_id == 1
|
||||
assert boundary_tokens == 4
|
||||
aligned_hash = req0.block_hashes[4 // hash_block_size - 1]
|
||||
aligned_block = manager.block_pool.get_cached_block(
|
||||
aligned_hash, kv_cache_group_ids=[1]
|
||||
)
|
||||
assert aligned_block is not None
|
||||
assert block_id == aligned_block[0].block_id
|
||||
# The final hash-aligned boundary uses the regular partial-tail path rather
|
||||
# than allocating an internal checkpoint at the query end.
|
||||
assert drain_boundary_state_offloads(manager) == {}
|
||||
|
||||
partial_mamba_hash = req0.block_hashes[6 // hash_block_size - 1]
|
||||
source_block = manager.block_pool.get_cached_block(
|
||||
|
||||
Reference in New Issue
Block a user