[Kimi K3] Support internal prefix checkpoints with partial prefix caching and spec-decoding (#53614)

Signed-off-by: Ziming Huang <zelda.huanghuang@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Ziming Huang
2026-09-06 13:18:15 +08:00
committed by GitHub
co-authored by OpenAI Codex
parent 1970f3ed4b
commit 144e79c810
11 changed files with 482 additions and 107 deletions
@@ -92,8 +92,23 @@ def make_full_mamba_manager(
mamba_block_size: int = 4,
num_blocks: int = 32,
use_eagle: bool = False,
num_speculative_blocks: int = 0,
num_prefill_checkpoint_blocks: int = 0,
):
mamba_group = KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=mamba_block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
num_speculative_blocks=num_speculative_blocks,
num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks,
prefill_checkpoint_alignment=(
16 if num_prefill_checkpoint_blocks > 0 else None
),
),
)
kv_cache_config = KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=[],
@@ -107,16 +122,7 @@ def make_full_mamba_manager(
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=mamba_block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks,
),
),
mamba_group,
],
)
scheduler_block_size = lcm(
@@ -169,6 +175,7 @@ def test_mamba_align_split_partial_tail_schedule(dcp_world_size: int):
scheduler_block_size = block_size * dcp_world_size
hash_block_size = 32
mock = SimpleNamespace(
block_size=scheduler_block_size,
cache_config=SimpleNamespace(block_size=block_size),
max_num_scheduled_tokens=8192,
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
@@ -217,6 +224,7 @@ def test_mamba_align_split_when_block_exceeds_scheduling_budget():
token_budget = 8192
prompt_length = 30000
mock = SimpleNamespace(
block_size=block_size,
cache_config=SimpleNamespace(block_size=block_size),
max_num_scheduled_tokens=token_budget,
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
@@ -254,6 +262,7 @@ def test_mamba_align_split_when_block_exceeds_long_prefill_threshold():
long_prefill_threshold = 384
prompt_length = 1300
mock = SimpleNamespace(
block_size=block_size,
cache_config=SimpleNamespace(block_size=block_size),
max_num_scheduled_tokens=token_budget,
scheduler_config=SimpleNamespace(
@@ -557,9 +566,9 @@ def test_partial_hit_then_internal_checkpoint_uses_distinct_mamba_blocks():
assert mamba_blocks[3].block_id == running_block_id
def test_internal_checkpoint_requires_block_aligned_start():
hash_block_size = 2
mamba_block_size = 16
def test_internal_checkpoint_uses_partial_hash_lifecycle():
hash_block_size = 16
mamba_block_size = 32
manager = make_full_mamba_manager(
dcp_world_size=1,
hash_block_size=hash_block_size,
@@ -567,23 +576,90 @@ def test_internal_checkpoint_requires_block_aligned_start():
mamba_block_size=mamba_block_size,
num_prefill_checkpoint_blocks=1,
)
request = make_request("producer", list(range(50)), hash_block_size, sha256)
request = make_request("producer", list(range(120)), hash_block_size, sha256)
# Compute token 0 first, so the next query starts at token 1, which is not
# aligned to the Mamba block size.
assert manager.allocate_slots(request, 1) is not None
request.num_computed_tokens = 1
manager.new_step_starts()
new_blocks = manager.allocate_slots(request, 49)
new_blocks = manager.allocate_slots(request, request.num_tokens)
assert new_blocks is not None
mamba_blocks = manager.get_blocks(request.request_id).blocks[1]
checkpoint_block_idx = 48 // mamba_block_size - 1
assert mamba_blocks[checkpoint_block_idx].is_null
assert not mamba_blocks[-1].is_null
checkpoint_hash = request.block_hashes[48 // hash_block_size - 1]
assert manager.block_pool.get_cached_block(checkpoint_hash, [1]) is None
checkpoint_idx = 2
checkpoint_block = mamba_blocks[checkpoint_idx]
running_block = mamba_blocks[checkpoint_idx + 1]
assert not checkpoint_block.is_null
assert not running_block.is_null
assert checkpoint_block is not running_block
# The internal block is exported at state@112, replacing its temporary
# full-block state@96 key while retaining #52789's req_to_blocks ownership.
partial_hash = request.block_hashes[112 // hash_block_size - 1]
partial_hit = manager.block_pool.get_cached_block(partial_hash, [1])
assert partial_hit is not None
assert partial_hit[0] is checkpoint_block
assert checkpoint_block.block_hash_num_tokens == 112
full_hash = request.block_hashes[96 // hash_block_size - 1]
assert manager.block_pool.get_cached_block(full_hash, [1]) is None
assert drain_boundary_state_offloads(manager) == {
request.request_id: [(1, checkpoint_block.block_id, 112)]
}
manager.free(request)
replay = make_request("replay", list(range(120)), hash_block_size, sha256)
_, num_computed, _ = manager.get_computed_blocks(replay)
assert num_computed == 112
def test_eagle_block_aligned_checkpoint_replaces_newer_hash():
hash_block_size = mamba_block_size = 32
manager = make_full_mamba_manager(
dcp_world_size=1,
hash_block_size=hash_block_size,
full_block_size=hash_block_size,
mamba_block_size=mamba_block_size,
use_eagle=True,
num_speculative_blocks=3,
num_prefill_checkpoint_blocks=1,
)
request = make_request("producer", list(range(104)), hash_block_size, sha256)
assert manager.allocate_slots(request, request.num_tokens) is not None
mamba_blocks = manager.get_blocks(request.request_id).blocks[1]
checkpoint_block = mamba_blocks[2]
checkpoint_hash = request.block_hashes[64 // hash_block_size - 1]
checkpoint_hit = manager.block_pool.get_cached_block(checkpoint_hash, [1])
assert checkpoint_hit is not None
assert checkpoint_hit[0] is checkpoint_block
assert checkpoint_block.block_hash_num_tokens == 64
overwritten_full_hash = request.block_hashes[96 // hash_block_size - 1]
assert manager.block_pool.get_cached_block(overwritten_full_hash, [1]) is None
def test_hash_aligned_query_end_uses_regular_partial_tail():
hash_block_size = 2
mamba_block_size = 4
manager = make_full_mamba_manager(
dcp_world_size=1,
hash_block_size=hash_block_size,
full_block_size=hash_block_size,
mamba_block_size=mamba_block_size,
num_prefill_checkpoint_blocks=1,
)
request = make_request("producer", list(range(14)), hash_block_size, sha256)
new_blocks = manager.allocate_slots(request, request.num_tokens)
assert new_blocks is not None
mamba_manager = manager.coordinator.single_type_managers[1]
assert mamba_manager._checkpoint_positions[request.request_id] == 0
mamba_blocks = manager.get_blocks(request.request_id).blocks[1]
assert sum(not block.is_null for block in mamba_blocks) == 1
partial_hash = request.block_hashes[14 // hash_block_size - 1]
cached = manager.block_pool.get_cached_block(partial_hash, [1])
assert cached is not None
assert cached[0] is mamba_blocks[14 // mamba_block_size]
assert cached[0].block_hash_num_tokens == 14
def test_external_mamba_hit_same_block_uses_running_cow_on_continue():
@@ -699,18 +775,9 @@ def test_boundary_state_offloads_returns_cow_target():
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
# Step A offers the materialized aligned checkpoint immediately.
((group_id, block_id, boundary_tokens),) = drain_boundary_state_offloads(manager)[
"0"
]
assert group_id == 1
assert boundary_tokens == 4
aligned_hash = req0.block_hashes[4 // hash_block_size - 1]
aligned_block = manager.block_pool.get_cached_block(
aligned_hash, kv_cache_group_ids=[1]
)
assert aligned_block is not None
assert block_id == aligned_block[0].block_id
# The final hash-aligned boundary uses the regular partial-tail path rather
# than allocating an internal checkpoint at the query end.
assert drain_boundary_state_offloads(manager) == {}
partial_mamba_hash = req0.block_hashes[6 // hash_block_size - 1]
source_block = manager.block_pool.get_cached_block(