[Kimi K3] Support internal prefix checkpoints with partial prefix caching and spec-decoding (#53614)
Signed-off-by: Ziming Huang <zelda.huanghuang@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
1970f3ed4b
commit
144e79c810
@@ -890,6 +890,7 @@ class MambaSpec(KVCacheSpec):
|
||||
mamba_cache_mode: str = "none"
|
||||
num_speculative_blocks: int = 0
|
||||
num_prefill_checkpoint_blocks: int = 0
|
||||
prefill_checkpoint_alignment: int | None = None
|
||||
num_heads: int = 1
|
||||
tokens_per_state: int = -1
|
||||
# False: the state is sharded across TP ranks (e.g. GDN). True: every TP
|
||||
@@ -950,12 +951,49 @@ class MambaSpec(KVCacheSpec):
|
||||
isinstance(spec, MambaSpec)
|
||||
and spec.num_speculative_blocks == self.num_speculative_blocks
|
||||
and spec.num_prefill_checkpoint_blocks == self.num_prefill_checkpoint_blocks
|
||||
and spec.prefill_checkpoint_alignment == self.prefill_checkpoint_alignment
|
||||
and spec.page_size_bytes == self.page_size_bytes
|
||||
and spec.tp_replicated == self.tp_replicated
|
||||
for spec in kv_cache_specs.values()
|
||||
)
|
||||
|
||||
|
||||
def get_mamba_prefill_checkpoint_position(
|
||||
num_tokens: int,
|
||||
hash_block_size: int,
|
||||
drop_eagle_block: bool,
|
||||
) -> int:
|
||||
"""Return the reusable Mamba checkpoint boundary for a prefill."""
|
||||
checkpoint_position = (num_tokens - 1) // hash_block_size * hash_block_size
|
||||
if drop_eagle_block:
|
||||
checkpoint_position -= hash_block_size
|
||||
return max(checkpoint_position, 0)
|
||||
|
||||
|
||||
def is_mamba_prefill_checkpoint_valid(
|
||||
query_start: int,
|
||||
query_end: int,
|
||||
checkpoint_position: int,
|
||||
hash_block_size: int,
|
||||
mamba_block_size: int,
|
||||
checkpoint_alignment: int | None,
|
||||
) -> bool:
|
||||
"""Whether a backend can export the checkpoint in this query."""
|
||||
if checkpoint_alignment is None:
|
||||
return False
|
||||
assert checkpoint_alignment > 0
|
||||
|
||||
initial_state_col = (query_start - 1) // mamba_block_size
|
||||
checkpoint_col = cdiv(query_end, mamba_block_size) - 2
|
||||
return (
|
||||
query_start % hash_block_size == 0
|
||||
and checkpoint_col > initial_state_col
|
||||
and query_start + hash_block_size <= checkpoint_position
|
||||
and query_start < checkpoint_position < query_end
|
||||
and (checkpoint_position - query_start) % checkpoint_alignment == 0
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EncoderOnlyAttentionSpec(AttentionSpec):
|
||||
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
|
||||
|
||||
Reference in New Issue
Block a user