[Kimi K3] Support internal prefix checkpoints with partial prefix caching and spec-decoding (#53614)

Signed-off-by: Ziming Huang <zelda.huanghuang@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Ziming Huang
2026-09-06 13:18:15 +08:00
committed by GitHub
co-authored by OpenAI Codex
parent 1970f3ed4b
commit 144e79c810
11 changed files with 482 additions and 107 deletions
+38
View File
@@ -890,6 +890,7 @@ class MambaSpec(KVCacheSpec):
mamba_cache_mode: str = "none"
num_speculative_blocks: int = 0
num_prefill_checkpoint_blocks: int = 0
prefill_checkpoint_alignment: int | None = None
num_heads: int = 1
tokens_per_state: int = -1
# False: the state is sharded across TP ranks (e.g. GDN). True: every TP
@@ -950,12 +951,49 @@ class MambaSpec(KVCacheSpec):
isinstance(spec, MambaSpec)
and spec.num_speculative_blocks == self.num_speculative_blocks
and spec.num_prefill_checkpoint_blocks == self.num_prefill_checkpoint_blocks
and spec.prefill_checkpoint_alignment == self.prefill_checkpoint_alignment
and spec.page_size_bytes == self.page_size_bytes
and spec.tp_replicated == self.tp_replicated
for spec in kv_cache_specs.values()
)
def get_mamba_prefill_checkpoint_position(
num_tokens: int,
hash_block_size: int,
drop_eagle_block: bool,
) -> int:
"""Return the reusable Mamba checkpoint boundary for a prefill."""
checkpoint_position = (num_tokens - 1) // hash_block_size * hash_block_size
if drop_eagle_block:
checkpoint_position -= hash_block_size
return max(checkpoint_position, 0)
def is_mamba_prefill_checkpoint_valid(
query_start: int,
query_end: int,
checkpoint_position: int,
hash_block_size: int,
mamba_block_size: int,
checkpoint_alignment: int | None,
) -> bool:
"""Whether a backend can export the checkpoint in this query."""
if checkpoint_alignment is None:
return False
assert checkpoint_alignment > 0
initial_state_col = (query_start - 1) // mamba_block_size
checkpoint_col = cdiv(query_end, mamba_block_size) - 2
return (
query_start % hash_block_size == 0
and checkpoint_col > initial_state_col
and query_start + hash_block_size <= checkpoint_position
and query_start < checkpoint_position < query_end
and (checkpoint_position - query_start) % checkpoint_alignment == 0
)
@dataclass(frozen=True)
class EncoderOnlyAttentionSpec(AttentionSpec):
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int: