[Kimi K3] Support internal prefix checkpoints with partial prefix caching and spec-decoding (#53614)

Signed-off-by: Ziming Huang <zelda.huanghuang@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Ziming Huang
2026-09-06 13:18:15 +08:00
committed by GitHub
co-authored by OpenAI Codex
parent 1970f3ed4b
commit 144e79c810
11 changed files with 482 additions and 107 deletions
+71 -1
View File
@@ -92,6 +92,10 @@ def _make_builder(
mamba_cache_mode: str = "none",
use_recoverssm: bool = False,
num_prefill_checkpoint_blocks: int = 0,
mamba_block_size: int = BLOCK_SIZE,
prefix_match_unit: int | None = None,
use_eagle: bool = False,
disable_eagle_block_drop: bool = False,
) -> AttentionMetadataBuilder:
vllm_config = create_vllm_config(
model_name="Qwen/Qwen3.5-0.8B",
@@ -102,20 +106,29 @@ def _make_builder(
method="ngram",
num_speculative_tokens=num_speculative_tokens,
)
if use_eagle:
vllm_config.speculative_config.method = "eagle3"
vllm_config.speculative_config.disable_eagle_block_drop = (
disable_eagle_block_drop
)
vllm_config.compilation_config.cudagraph_mode = (
CUDAGraphMode.FULL_AND_PIECEWISE if full_cuda_graph else CUDAGraphMode.NONE
)
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
vllm_config.cache_config.use_replayssm = use_recoverssm
vllm_config.cache_config.use_kda_recoverssm = use_recoverssm
vllm_config.cache_config.prefix_match_unit = prefix_match_unit
builder = builder_cls(
kv_cache_spec=MambaSpec(
block_size=BLOCK_SIZE,
block_size=mamba_block_size,
shapes=((16, 64),),
dtypes=(torch.float16,),
mamba_cache_mode=mamba_cache_mode,
num_speculative_blocks=(0 if use_recoverssm else num_speculative_tokens),
num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks,
prefill_checkpoint_alignment=(
16 if num_prefill_checkpoint_blocks > 0 else None
),
),
layer_names=["layer.0"],
vllm_config=vllm_config,
@@ -156,6 +169,7 @@ def test_kda_recoverssm_startup_metadata_flow_without_model(monkeypatch):
mamba_type=MambaAttentionBackendEnum.GDN_ATTN,
mamba_cache_mode="align",
num_prefill_checkpoint_blocks=1,
prefill_checkpoint_alignment=16,
)
# This is the same compatibility check performed while initializing the
@@ -177,11 +191,13 @@ def test_kda_recoverssm_startup_metadata_flow_without_model(monkeypatch):
cache_config=SimpleNamespace(
mamba_cache_mode="align",
use_kda_recoverssm=True,
prefix_match_unit=None,
),
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
speculative_config=SimpleNamespace(
num_speculative_tokens=2,
parallel_drafting=False,
use_eagle_block_drop=Mock(return_value=False),
),
compilation_config=SimpleNamespace(
cudagraph_mode=CUDAGraphMode.NONE,
@@ -300,6 +316,60 @@ def test_internal_checkpoint_metadata_targets_last_aligned_boundary():
)
@pytest.mark.parametrize(
("disable_eagle_block_drop", "prefix_match_unit", "expected_offset"),
[(False, 16, 80), (True, 16, 96), (False, 8, None)],
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_spec_internal_checkpoint_metadata_targets_replay_boundary(
disable_eagle_block_drop: bool,
prefix_match_unit: int,
expected_offset: int | None,
) -> None:
device = torch.device("cuda")
batch = BatchSpec(seq_lens=[100], query_lens=[100])
common_attn_metadata = create_common_attn_metadata(
batch, BLOCK_SIZE, device, arange_block_indices=True
)
common_attn_metadata = common_attn_metadata.replace(
is_prefilling=torch.tensor([True]),
block_table_tensor=common_attn_metadata.block_table_tensor + 1,
)
builder = _make_builder(
KimiK3KDAMetadataBuilder,
num_speculative_tokens=3,
full_cuda_graph=False,
mamba_cache_mode="align",
num_prefill_checkpoint_blocks=1,
mamba_block_size=64,
prefix_match_unit=prefix_match_unit,
use_eagle=True,
disable_eagle_block_drop=disable_eagle_block_drop,
device=device,
)
assert isinstance(builder, KimiK3KDAMetadataBuilder)
builder.mamba_aligned_state_indices = mamba_get_block_table_tensor(
common_attn_metadata.block_table_tensor,
common_attn_metadata.seq_lens,
builder.kv_cache_spec,
"align",
)
actual = builder.build(0, common_attn_metadata)
if expected_offset is None:
assert actual.checkpoint is None
return
assert actual.checkpoint is not None
torch.testing.assert_close(
actual.checkpoint.state_indices,
torch.tensor([1], dtype=torch.int32, device=device),
)
torch.testing.assert_close(
actual.checkpoint.checkpoint_offsets,
torch.tensor([expected_offset], dtype=torch.int32, device=device),
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_internal_checkpoint_metadata_skips_unaligned_offset():
device = torch.device("cuda")
@@ -92,8 +92,23 @@ def make_full_mamba_manager(
mamba_block_size: int = 4,
num_blocks: int = 32,
use_eagle: bool = False,
num_speculative_blocks: int = 0,
num_prefill_checkpoint_blocks: int = 0,
):
mamba_group = KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=mamba_block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
num_speculative_blocks=num_speculative_blocks,
num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks,
prefill_checkpoint_alignment=(
16 if num_prefill_checkpoint_blocks > 0 else None
),
),
)
kv_cache_config = KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=[],
@@ -107,16 +122,7 @@ def make_full_mamba_manager(
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=mamba_block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks,
),
),
mamba_group,
],
)
scheduler_block_size = lcm(
@@ -169,6 +175,7 @@ def test_mamba_align_split_partial_tail_schedule(dcp_world_size: int):
scheduler_block_size = block_size * dcp_world_size
hash_block_size = 32
mock = SimpleNamespace(
block_size=scheduler_block_size,
cache_config=SimpleNamespace(block_size=block_size),
max_num_scheduled_tokens=8192,
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
@@ -217,6 +224,7 @@ def test_mamba_align_split_when_block_exceeds_scheduling_budget():
token_budget = 8192
prompt_length = 30000
mock = SimpleNamespace(
block_size=block_size,
cache_config=SimpleNamespace(block_size=block_size),
max_num_scheduled_tokens=token_budget,
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
@@ -254,6 +262,7 @@ def test_mamba_align_split_when_block_exceeds_long_prefill_threshold():
long_prefill_threshold = 384
prompt_length = 1300
mock = SimpleNamespace(
block_size=block_size,
cache_config=SimpleNamespace(block_size=block_size),
max_num_scheduled_tokens=token_budget,
scheduler_config=SimpleNamespace(
@@ -557,9 +566,9 @@ def test_partial_hit_then_internal_checkpoint_uses_distinct_mamba_blocks():
assert mamba_blocks[3].block_id == running_block_id
def test_internal_checkpoint_requires_block_aligned_start():
hash_block_size = 2
mamba_block_size = 16
def test_internal_checkpoint_uses_partial_hash_lifecycle():
hash_block_size = 16
mamba_block_size = 32
manager = make_full_mamba_manager(
dcp_world_size=1,
hash_block_size=hash_block_size,
@@ -567,23 +576,90 @@ def test_internal_checkpoint_requires_block_aligned_start():
mamba_block_size=mamba_block_size,
num_prefill_checkpoint_blocks=1,
)
request = make_request("producer", list(range(50)), hash_block_size, sha256)
request = make_request("producer", list(range(120)), hash_block_size, sha256)
# Compute token 0 first, so the next query starts at token 1, which is not
# aligned to the Mamba block size.
assert manager.allocate_slots(request, 1) is not None
request.num_computed_tokens = 1
manager.new_step_starts()
new_blocks = manager.allocate_slots(request, 49)
new_blocks = manager.allocate_slots(request, request.num_tokens)
assert new_blocks is not None
mamba_blocks = manager.get_blocks(request.request_id).blocks[1]
checkpoint_block_idx = 48 // mamba_block_size - 1
assert mamba_blocks[checkpoint_block_idx].is_null
assert not mamba_blocks[-1].is_null
checkpoint_hash = request.block_hashes[48 // hash_block_size - 1]
assert manager.block_pool.get_cached_block(checkpoint_hash, [1]) is None
checkpoint_idx = 2
checkpoint_block = mamba_blocks[checkpoint_idx]
running_block = mamba_blocks[checkpoint_idx + 1]
assert not checkpoint_block.is_null
assert not running_block.is_null
assert checkpoint_block is not running_block
# The internal block is exported at state@112, replacing its temporary
# full-block state@96 key while retaining #52789's req_to_blocks ownership.
partial_hash = request.block_hashes[112 // hash_block_size - 1]
partial_hit = manager.block_pool.get_cached_block(partial_hash, [1])
assert partial_hit is not None
assert partial_hit[0] is checkpoint_block
assert checkpoint_block.block_hash_num_tokens == 112
full_hash = request.block_hashes[96 // hash_block_size - 1]
assert manager.block_pool.get_cached_block(full_hash, [1]) is None
assert drain_boundary_state_offloads(manager) == {
request.request_id: [(1, checkpoint_block.block_id, 112)]
}
manager.free(request)
replay = make_request("replay", list(range(120)), hash_block_size, sha256)
_, num_computed, _ = manager.get_computed_blocks(replay)
assert num_computed == 112
def test_eagle_block_aligned_checkpoint_replaces_newer_hash():
hash_block_size = mamba_block_size = 32
manager = make_full_mamba_manager(
dcp_world_size=1,
hash_block_size=hash_block_size,
full_block_size=hash_block_size,
mamba_block_size=mamba_block_size,
use_eagle=True,
num_speculative_blocks=3,
num_prefill_checkpoint_blocks=1,
)
request = make_request("producer", list(range(104)), hash_block_size, sha256)
assert manager.allocate_slots(request, request.num_tokens) is not None
mamba_blocks = manager.get_blocks(request.request_id).blocks[1]
checkpoint_block = mamba_blocks[2]
checkpoint_hash = request.block_hashes[64 // hash_block_size - 1]
checkpoint_hit = manager.block_pool.get_cached_block(checkpoint_hash, [1])
assert checkpoint_hit is not None
assert checkpoint_hit[0] is checkpoint_block
assert checkpoint_block.block_hash_num_tokens == 64
overwritten_full_hash = request.block_hashes[96 // hash_block_size - 1]
assert manager.block_pool.get_cached_block(overwritten_full_hash, [1]) is None
def test_hash_aligned_query_end_uses_regular_partial_tail():
hash_block_size = 2
mamba_block_size = 4
manager = make_full_mamba_manager(
dcp_world_size=1,
hash_block_size=hash_block_size,
full_block_size=hash_block_size,
mamba_block_size=mamba_block_size,
num_prefill_checkpoint_blocks=1,
)
request = make_request("producer", list(range(14)), hash_block_size, sha256)
new_blocks = manager.allocate_slots(request, request.num_tokens)
assert new_blocks is not None
mamba_manager = manager.coordinator.single_type_managers[1]
assert mamba_manager._checkpoint_positions[request.request_id] == 0
mamba_blocks = manager.get_blocks(request.request_id).blocks[1]
assert sum(not block.is_null for block in mamba_blocks) == 1
partial_hash = request.block_hashes[14 // hash_block_size - 1]
cached = manager.block_pool.get_cached_block(partial_hash, [1])
assert cached is not None
assert cached[0] is mamba_blocks[14 // mamba_block_size]
assert cached[0].block_hash_num_tokens == 14
def test_external_mamba_hit_same_block_uses_running_cow_on_continue():
@@ -699,18 +775,9 @@ def test_boundary_state_offloads_returns_cow_target():
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
# Step A offers the materialized aligned checkpoint immediately.
((group_id, block_id, boundary_tokens),) = drain_boundary_state_offloads(manager)[
"0"
]
assert group_id == 1
assert boundary_tokens == 4
aligned_hash = req0.block_hashes[4 // hash_block_size - 1]
aligned_block = manager.block_pool.get_cached_block(
aligned_hash, kv_cache_group_ids=[1]
)
assert aligned_block is not None
assert block_id == aligned_block[0].block_id
# The final hash-aligned boundary uses the regular partial-tail path rather
# than allocating an internal checkpoint at the query end.
assert drain_boundary_state_offloads(manager) == {}
partial_mamba_hash = req0.block_hashes[6 // hash_block_size - 1]
source_block = manager.block_pool.get_cached_block(
+92 -13
View File
@@ -8,6 +8,7 @@ the wrong offset, and a later chunk crossing that boundary publishes it anyway.
Requests resuming from it then restore a truncated state (#43559).
"""
import sys
from types import SimpleNamespace
import pytest
@@ -21,6 +22,7 @@ from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
MambaSpec,
is_mamba_prefill_checkpoint_valid,
)
from vllm.v1.request import Request
@@ -37,6 +39,49 @@ PROMPT_LEN = 2002
MAMBA_GROUP_ID = 1
@pytest.mark.parametrize(
(
"query_start",
"query_end",
"checkpoint_position",
"hash_size",
"block_size",
"alignment",
"expected",
),
[
(0, 100, 88, 8, 64, 16, False),
(0, 100, 96, 8, 64, 16, True),
(0, 100, 96, 8, 64, None, False),
(8, 100, 96, 8, 64, 16, False),
(15360, 16896, 16768, 128, 1536, 16, False),
(15360, 16960, 16896, 128, 1536, 16, True),
(15360, 16960, 16768, 128, 1536, 16, True),
(16064, 18433, 18432, 128, 1536, 16, False),
],
)
def test_mamba_prefill_checkpoint_valid(
query_start: int,
query_end: int,
checkpoint_position: int,
hash_size: int,
block_size: int,
alignment: int | None,
expected: bool,
) -> None:
assert (
is_mamba_prefill_checkpoint_valid(
query_start=query_start,
query_end=query_end,
checkpoint_position=checkpoint_position,
hash_block_size=hash_size,
mamba_block_size=block_size,
checkpoint_alignment=alignment,
)
is expected
)
def _make_hybrid_kv_cache_manager(
num_prefill_checkpoint_blocks: int = 0,
) -> KVCacheManager:
@@ -62,6 +107,9 @@ def _make_hybrid_kv_cache_manager(
mamba_cache_mode="align",
num_speculative_blocks=NUM_SPEC,
num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks,
prefill_checkpoint_alignment=(
16 if num_prefill_checkpoint_blocks > 0 else None
),
),
),
],
@@ -83,20 +131,23 @@ def _split(
use_eagle_block_drop: bool | None = None,
partial_hit: bool = False,
num_prefill_checkpoint_blocks: int = 0,
max_num_scheduled_tokens: int = 16384,
) -> int:
"""Call the real `Scheduler._mamba_block_aligned_split` on a stub self."""
if use_eagle_block_drop is None:
use_eagle_block_drop = use_eagle
stub = SimpleNamespace(
block_size=MAMBA_BLOCK_SIZE,
cache_config=SimpleNamespace(block_size=MAMBA_BLOCK_SIZE),
use_eagle_block_drop=use_eagle_block_drop,
max_num_scheduled_tokens=16384,
max_num_scheduled_tokens=max_num_scheduled_tokens,
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
# `prefix_match_unit` finer than the block size (#46384).
mamba_partial_cache_hit=partial_hit,
hash_block_size=ATTN_BLOCK_SIZE,
mamba_has_prefill_checkpoint_blocks=(
num_prefill_checkpoint_blocks > 0 and not use_eagle
mamba_has_prefill_checkpoint_blocks=(num_prefill_checkpoint_blocks > 0),
mamba_prefill_checkpoint_alignment=(
16 if num_prefill_checkpoint_blocks > 0 else None
),
)
return Scheduler._mamba_block_aligned_split(stub, request, num_new_tokens)
@@ -106,8 +157,11 @@ def _split(
("prompt_len", "num_new_tokens", "use_eagle", "expected"),
[
(2002, 2002, False, 2002),
(3602, 2000, False, 2000),
(3602, 3602, True, MAMBA_BLOCK_SIZE),
(3602, 2000, False, MAMBA_BLOCK_SIZE),
(3602, 3602, True, 3602),
(2002, 2002, True, 2002),
(3602, 512, True, 0),
(3602, 1700, True, MAMBA_BLOCK_SIZE),
],
)
def test_internal_checkpoint_split(
@@ -123,12 +177,39 @@ def test_internal_checkpoint_split(
)
== expected
)
if not use_eagle:
manager = _make_hybrid_kv_cache_manager(num_prefill_checkpoint_blocks=1)
assert manager.allocate_slots(request, expected) is not None
mamba_manager = manager.coordinator.single_type_managers[MAMBA_GROUP_ID]
blocks = mamba_manager.req_to_blocks[request.request_id]
assert all(not block.is_null for block in blocks) # checkpoint + running state
def test_partial_checkpoint_resume_stops_at_mamba_block_boundary() -> None:
prompt_len = 3602
resume_at = 1984
(request,) = create_requests(1, num_tokens=prompt_len, block_size=ATTN_BLOCK_SIZE)
request.num_computed_tokens = resume_at
# Without an internal checkpoint export, the forward must stop at the
# physical Mamba boundary so it does not publish a stale running state.
assert (
_split(
request,
prompt_len - resume_at,
use_eagle=False,
partial_hit=True,
)
== MAMBA_BLOCK_SIZE - resume_at % MAMBA_BLOCK_SIZE
)
# A hash-aligned resume inside a physical Mamba block cannot use the fixed
# internal-checkpoint column: it aliases the initial-state column. Enabling
# checkpoints must therefore preserve the normal aligned split.
assert (
_split(
request,
prompt_len - resume_at,
use_eagle=False,
partial_hit=True,
num_prefill_checkpoint_blocks=1,
)
== MAMBA_BLOCK_SIZE - resume_at % MAMBA_BLOCK_SIZE
)
def test_disabling_eagle_block_drop_keeps_the_trailing_cache_boundary() -> None:
@@ -252,8 +333,6 @@ def test_poisoning_is_block_size_independent(
budgets: list[int],
) -> None:
"""The invariant is per-block, so large mamba blocks are not safer."""
import sys
monkeypatch.setattr(sys.modules[__name__], "MAMBA_BLOCK_SIZE", block_size)
assert _prefill(prompt_len, budgets=budgets) > 0
+1
View File
@@ -1162,6 +1162,7 @@ def test_hybrid_cache_mamba_align_shared_prefix_detection():
# junction (absolute) and stops the chunk there (req_2 has num_computed 0).
# Create minimal mock with just the needed attributes
mock = SimpleNamespace(
block_size=block_size,
cache_config=SimpleNamespace(block_size=block_size),
max_num_scheduled_tokens=3 * block_size,
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),