[Kimi K3] Support internal prefix checkpoints with partial prefix caching and spec-decoding (#53614)

Signed-off-by: Ziming Huang <zelda.huanghuang@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Ziming Huang
2026-09-06 13:18:15 +08:00
committed by GitHub
co-authored by OpenAI Codex
parent 1970f3ed4b
commit 144e79c810
11 changed files with 482 additions and 107 deletions
+71 -1
View File
@@ -92,6 +92,10 @@ def _make_builder(
mamba_cache_mode: str = "none",
use_recoverssm: bool = False,
num_prefill_checkpoint_blocks: int = 0,
mamba_block_size: int = BLOCK_SIZE,
prefix_match_unit: int | None = None,
use_eagle: bool = False,
disable_eagle_block_drop: bool = False,
) -> AttentionMetadataBuilder:
vllm_config = create_vllm_config(
model_name="Qwen/Qwen3.5-0.8B",
@@ -102,20 +106,29 @@ def _make_builder(
method="ngram",
num_speculative_tokens=num_speculative_tokens,
)
if use_eagle:
vllm_config.speculative_config.method = "eagle3"
vllm_config.speculative_config.disable_eagle_block_drop = (
disable_eagle_block_drop
)
vllm_config.compilation_config.cudagraph_mode = (
CUDAGraphMode.FULL_AND_PIECEWISE if full_cuda_graph else CUDAGraphMode.NONE
)
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
vllm_config.cache_config.use_replayssm = use_recoverssm
vllm_config.cache_config.use_kda_recoverssm = use_recoverssm
vllm_config.cache_config.prefix_match_unit = prefix_match_unit
builder = builder_cls(
kv_cache_spec=MambaSpec(
block_size=BLOCK_SIZE,
block_size=mamba_block_size,
shapes=((16, 64),),
dtypes=(torch.float16,),
mamba_cache_mode=mamba_cache_mode,
num_speculative_blocks=(0 if use_recoverssm else num_speculative_tokens),
num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks,
prefill_checkpoint_alignment=(
16 if num_prefill_checkpoint_blocks > 0 else None
),
),
layer_names=["layer.0"],
vllm_config=vllm_config,
@@ -156,6 +169,7 @@ def test_kda_recoverssm_startup_metadata_flow_without_model(monkeypatch):
mamba_type=MambaAttentionBackendEnum.GDN_ATTN,
mamba_cache_mode="align",
num_prefill_checkpoint_blocks=1,
prefill_checkpoint_alignment=16,
)
# This is the same compatibility check performed while initializing the
@@ -177,11 +191,13 @@ def test_kda_recoverssm_startup_metadata_flow_without_model(monkeypatch):
cache_config=SimpleNamespace(
mamba_cache_mode="align",
use_kda_recoverssm=True,
prefix_match_unit=None,
),
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
speculative_config=SimpleNamespace(
num_speculative_tokens=2,
parallel_drafting=False,
use_eagle_block_drop=Mock(return_value=False),
),
compilation_config=SimpleNamespace(
cudagraph_mode=CUDAGraphMode.NONE,
@@ -300,6 +316,60 @@ def test_internal_checkpoint_metadata_targets_last_aligned_boundary():
)
@pytest.mark.parametrize(
("disable_eagle_block_drop", "prefix_match_unit", "expected_offset"),
[(False, 16, 80), (True, 16, 96), (False, 8, None)],
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_spec_internal_checkpoint_metadata_targets_replay_boundary(
disable_eagle_block_drop: bool,
prefix_match_unit: int,
expected_offset: int | None,
) -> None:
device = torch.device("cuda")
batch = BatchSpec(seq_lens=[100], query_lens=[100])
common_attn_metadata = create_common_attn_metadata(
batch, BLOCK_SIZE, device, arange_block_indices=True
)
common_attn_metadata = common_attn_metadata.replace(
is_prefilling=torch.tensor([True]),
block_table_tensor=common_attn_metadata.block_table_tensor + 1,
)
builder = _make_builder(
KimiK3KDAMetadataBuilder,
num_speculative_tokens=3,
full_cuda_graph=False,
mamba_cache_mode="align",
num_prefill_checkpoint_blocks=1,
mamba_block_size=64,
prefix_match_unit=prefix_match_unit,
use_eagle=True,
disable_eagle_block_drop=disable_eagle_block_drop,
device=device,
)
assert isinstance(builder, KimiK3KDAMetadataBuilder)
builder.mamba_aligned_state_indices = mamba_get_block_table_tensor(
common_attn_metadata.block_table_tensor,
common_attn_metadata.seq_lens,
builder.kv_cache_spec,
"align",
)
actual = builder.build(0, common_attn_metadata)
if expected_offset is None:
assert actual.checkpoint is None
return
assert actual.checkpoint is not None
torch.testing.assert_close(
actual.checkpoint.state_indices,
torch.tensor([1], dtype=torch.int32, device=device),
)
torch.testing.assert_close(
actual.checkpoint.checkpoint_offsets,
torch.tensor([expected_offset], dtype=torch.int32, device=device),
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_internal_checkpoint_metadata_skips_unaligned_offset():
device = torch.device("cuda")
@@ -92,8 +92,23 @@ def make_full_mamba_manager(
mamba_block_size: int = 4,
num_blocks: int = 32,
use_eagle: bool = False,
num_speculative_blocks: int = 0,
num_prefill_checkpoint_blocks: int = 0,
):
mamba_group = KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=mamba_block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
num_speculative_blocks=num_speculative_blocks,
num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks,
prefill_checkpoint_alignment=(
16 if num_prefill_checkpoint_blocks > 0 else None
),
),
)
kv_cache_config = KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=[],
@@ -107,16 +122,7 @@ def make_full_mamba_manager(
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=mamba_block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks,
),
),
mamba_group,
],
)
scheduler_block_size = lcm(
@@ -169,6 +175,7 @@ def test_mamba_align_split_partial_tail_schedule(dcp_world_size: int):
scheduler_block_size = block_size * dcp_world_size
hash_block_size = 32
mock = SimpleNamespace(
block_size=scheduler_block_size,
cache_config=SimpleNamespace(block_size=block_size),
max_num_scheduled_tokens=8192,
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
@@ -217,6 +224,7 @@ def test_mamba_align_split_when_block_exceeds_scheduling_budget():
token_budget = 8192
prompt_length = 30000
mock = SimpleNamespace(
block_size=block_size,
cache_config=SimpleNamespace(block_size=block_size),
max_num_scheduled_tokens=token_budget,
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
@@ -254,6 +262,7 @@ def test_mamba_align_split_when_block_exceeds_long_prefill_threshold():
long_prefill_threshold = 384
prompt_length = 1300
mock = SimpleNamespace(
block_size=block_size,
cache_config=SimpleNamespace(block_size=block_size),
max_num_scheduled_tokens=token_budget,
scheduler_config=SimpleNamespace(
@@ -557,9 +566,9 @@ def test_partial_hit_then_internal_checkpoint_uses_distinct_mamba_blocks():
assert mamba_blocks[3].block_id == running_block_id
def test_internal_checkpoint_requires_block_aligned_start():
hash_block_size = 2
mamba_block_size = 16
def test_internal_checkpoint_uses_partial_hash_lifecycle():
hash_block_size = 16
mamba_block_size = 32
manager = make_full_mamba_manager(
dcp_world_size=1,
hash_block_size=hash_block_size,
@@ -567,23 +576,90 @@ def test_internal_checkpoint_requires_block_aligned_start():
mamba_block_size=mamba_block_size,
num_prefill_checkpoint_blocks=1,
)
request = make_request("producer", list(range(50)), hash_block_size, sha256)
request = make_request("producer", list(range(120)), hash_block_size, sha256)
# Compute token 0 first, so the next query starts at token 1, which is not
# aligned to the Mamba block size.
assert manager.allocate_slots(request, 1) is not None
request.num_computed_tokens = 1
manager.new_step_starts()
new_blocks = manager.allocate_slots(request, 49)
new_blocks = manager.allocate_slots(request, request.num_tokens)
assert new_blocks is not None
mamba_blocks = manager.get_blocks(request.request_id).blocks[1]
checkpoint_block_idx = 48 // mamba_block_size - 1
assert mamba_blocks[checkpoint_block_idx].is_null
assert not mamba_blocks[-1].is_null
checkpoint_hash = request.block_hashes[48 // hash_block_size - 1]
assert manager.block_pool.get_cached_block(checkpoint_hash, [1]) is None
checkpoint_idx = 2
checkpoint_block = mamba_blocks[checkpoint_idx]
running_block = mamba_blocks[checkpoint_idx + 1]
assert not checkpoint_block.is_null
assert not running_block.is_null
assert checkpoint_block is not running_block
# The internal block is exported at state@112, replacing its temporary
# full-block state@96 key while retaining #52789's req_to_blocks ownership.
partial_hash = request.block_hashes[112 // hash_block_size - 1]
partial_hit = manager.block_pool.get_cached_block(partial_hash, [1])
assert partial_hit is not None
assert partial_hit[0] is checkpoint_block
assert checkpoint_block.block_hash_num_tokens == 112
full_hash = request.block_hashes[96 // hash_block_size - 1]
assert manager.block_pool.get_cached_block(full_hash, [1]) is None
assert drain_boundary_state_offloads(manager) == {
request.request_id: [(1, checkpoint_block.block_id, 112)]
}
manager.free(request)
replay = make_request("replay", list(range(120)), hash_block_size, sha256)
_, num_computed, _ = manager.get_computed_blocks(replay)
assert num_computed == 112
def test_eagle_block_aligned_checkpoint_replaces_newer_hash():
hash_block_size = mamba_block_size = 32
manager = make_full_mamba_manager(
dcp_world_size=1,
hash_block_size=hash_block_size,
full_block_size=hash_block_size,
mamba_block_size=mamba_block_size,
use_eagle=True,
num_speculative_blocks=3,
num_prefill_checkpoint_blocks=1,
)
request = make_request("producer", list(range(104)), hash_block_size, sha256)
assert manager.allocate_slots(request, request.num_tokens) is not None
mamba_blocks = manager.get_blocks(request.request_id).blocks[1]
checkpoint_block = mamba_blocks[2]
checkpoint_hash = request.block_hashes[64 // hash_block_size - 1]
checkpoint_hit = manager.block_pool.get_cached_block(checkpoint_hash, [1])
assert checkpoint_hit is not None
assert checkpoint_hit[0] is checkpoint_block
assert checkpoint_block.block_hash_num_tokens == 64
overwritten_full_hash = request.block_hashes[96 // hash_block_size - 1]
assert manager.block_pool.get_cached_block(overwritten_full_hash, [1]) is None
def test_hash_aligned_query_end_uses_regular_partial_tail():
hash_block_size = 2
mamba_block_size = 4
manager = make_full_mamba_manager(
dcp_world_size=1,
hash_block_size=hash_block_size,
full_block_size=hash_block_size,
mamba_block_size=mamba_block_size,
num_prefill_checkpoint_blocks=1,
)
request = make_request("producer", list(range(14)), hash_block_size, sha256)
new_blocks = manager.allocate_slots(request, request.num_tokens)
assert new_blocks is not None
mamba_manager = manager.coordinator.single_type_managers[1]
assert mamba_manager._checkpoint_positions[request.request_id] == 0
mamba_blocks = manager.get_blocks(request.request_id).blocks[1]
assert sum(not block.is_null for block in mamba_blocks) == 1
partial_hash = request.block_hashes[14 // hash_block_size - 1]
cached = manager.block_pool.get_cached_block(partial_hash, [1])
assert cached is not None
assert cached[0] is mamba_blocks[14 // mamba_block_size]
assert cached[0].block_hash_num_tokens == 14
def test_external_mamba_hit_same_block_uses_running_cow_on_continue():
@@ -699,18 +775,9 @@ def test_boundary_state_offloads_returns_cow_target():
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
# Step A offers the materialized aligned checkpoint immediately.
((group_id, block_id, boundary_tokens),) = drain_boundary_state_offloads(manager)[
"0"
]
assert group_id == 1
assert boundary_tokens == 4
aligned_hash = req0.block_hashes[4 // hash_block_size - 1]
aligned_block = manager.block_pool.get_cached_block(
aligned_hash, kv_cache_group_ids=[1]
)
assert aligned_block is not None
assert block_id == aligned_block[0].block_id
# The final hash-aligned boundary uses the regular partial-tail path rather
# than allocating an internal checkpoint at the query end.
assert drain_boundary_state_offloads(manager) == {}
partial_mamba_hash = req0.block_hashes[6 // hash_block_size - 1]
source_block = manager.block_pool.get_cached_block(
+92 -13
View File
@@ -8,6 +8,7 @@ the wrong offset, and a later chunk crossing that boundary publishes it anyway.
Requests resuming from it then restore a truncated state (#43559).
"""
import sys
from types import SimpleNamespace
import pytest
@@ -21,6 +22,7 @@ from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
MambaSpec,
is_mamba_prefill_checkpoint_valid,
)
from vllm.v1.request import Request
@@ -37,6 +39,49 @@ PROMPT_LEN = 2002
MAMBA_GROUP_ID = 1
@pytest.mark.parametrize(
(
"query_start",
"query_end",
"checkpoint_position",
"hash_size",
"block_size",
"alignment",
"expected",
),
[
(0, 100, 88, 8, 64, 16, False),
(0, 100, 96, 8, 64, 16, True),
(0, 100, 96, 8, 64, None, False),
(8, 100, 96, 8, 64, 16, False),
(15360, 16896, 16768, 128, 1536, 16, False),
(15360, 16960, 16896, 128, 1536, 16, True),
(15360, 16960, 16768, 128, 1536, 16, True),
(16064, 18433, 18432, 128, 1536, 16, False),
],
)
def test_mamba_prefill_checkpoint_valid(
query_start: int,
query_end: int,
checkpoint_position: int,
hash_size: int,
block_size: int,
alignment: int | None,
expected: bool,
) -> None:
assert (
is_mamba_prefill_checkpoint_valid(
query_start=query_start,
query_end=query_end,
checkpoint_position=checkpoint_position,
hash_block_size=hash_size,
mamba_block_size=block_size,
checkpoint_alignment=alignment,
)
is expected
)
def _make_hybrid_kv_cache_manager(
num_prefill_checkpoint_blocks: int = 0,
) -> KVCacheManager:
@@ -62,6 +107,9 @@ def _make_hybrid_kv_cache_manager(
mamba_cache_mode="align",
num_speculative_blocks=NUM_SPEC,
num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks,
prefill_checkpoint_alignment=(
16 if num_prefill_checkpoint_blocks > 0 else None
),
),
),
],
@@ -83,20 +131,23 @@ def _split(
use_eagle_block_drop: bool | None = None,
partial_hit: bool = False,
num_prefill_checkpoint_blocks: int = 0,
max_num_scheduled_tokens: int = 16384,
) -> int:
"""Call the real `Scheduler._mamba_block_aligned_split` on a stub self."""
if use_eagle_block_drop is None:
use_eagle_block_drop = use_eagle
stub = SimpleNamespace(
block_size=MAMBA_BLOCK_SIZE,
cache_config=SimpleNamespace(block_size=MAMBA_BLOCK_SIZE),
use_eagle_block_drop=use_eagle_block_drop,
max_num_scheduled_tokens=16384,
max_num_scheduled_tokens=max_num_scheduled_tokens,
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
# `prefix_match_unit` finer than the block size (#46384).
mamba_partial_cache_hit=partial_hit,
hash_block_size=ATTN_BLOCK_SIZE,
mamba_has_prefill_checkpoint_blocks=(
num_prefill_checkpoint_blocks > 0 and not use_eagle
mamba_has_prefill_checkpoint_blocks=(num_prefill_checkpoint_blocks > 0),
mamba_prefill_checkpoint_alignment=(
16 if num_prefill_checkpoint_blocks > 0 else None
),
)
return Scheduler._mamba_block_aligned_split(stub, request, num_new_tokens)
@@ -106,8 +157,11 @@ def _split(
("prompt_len", "num_new_tokens", "use_eagle", "expected"),
[
(2002, 2002, False, 2002),
(3602, 2000, False, 2000),
(3602, 3602, True, MAMBA_BLOCK_SIZE),
(3602, 2000, False, MAMBA_BLOCK_SIZE),
(3602, 3602, True, 3602),
(2002, 2002, True, 2002),
(3602, 512, True, 0),
(3602, 1700, True, MAMBA_BLOCK_SIZE),
],
)
def test_internal_checkpoint_split(
@@ -123,12 +177,39 @@ def test_internal_checkpoint_split(
)
== expected
)
if not use_eagle:
manager = _make_hybrid_kv_cache_manager(num_prefill_checkpoint_blocks=1)
assert manager.allocate_slots(request, expected) is not None
mamba_manager = manager.coordinator.single_type_managers[MAMBA_GROUP_ID]
blocks = mamba_manager.req_to_blocks[request.request_id]
assert all(not block.is_null for block in blocks) # checkpoint + running state
def test_partial_checkpoint_resume_stops_at_mamba_block_boundary() -> None:
prompt_len = 3602
resume_at = 1984
(request,) = create_requests(1, num_tokens=prompt_len, block_size=ATTN_BLOCK_SIZE)
request.num_computed_tokens = resume_at
# Without an internal checkpoint export, the forward must stop at the
# physical Mamba boundary so it does not publish a stale running state.
assert (
_split(
request,
prompt_len - resume_at,
use_eagle=False,
partial_hit=True,
)
== MAMBA_BLOCK_SIZE - resume_at % MAMBA_BLOCK_SIZE
)
# A hash-aligned resume inside a physical Mamba block cannot use the fixed
# internal-checkpoint column: it aliases the initial-state column. Enabling
# checkpoints must therefore preserve the normal aligned split.
assert (
_split(
request,
prompt_len - resume_at,
use_eagle=False,
partial_hit=True,
num_prefill_checkpoint_blocks=1,
)
== MAMBA_BLOCK_SIZE - resume_at % MAMBA_BLOCK_SIZE
)
def test_disabling_eagle_block_drop_keeps_the_trailing_cache_boundary() -> None:
@@ -252,8 +333,6 @@ def test_poisoning_is_block_size_independent(
budgets: list[int],
) -> None:
"""The invariant is per-block, so large mamba blocks are not safer."""
import sys
monkeypatch.setattr(sys.modules[__name__], "MAMBA_BLOCK_SIZE", block_size)
assert _prefill(prompt_len, budgets=budgets) > 0
+1
View File
@@ -1162,6 +1162,7 @@ def test_hybrid_cache_mamba_align_shared_prefix_detection():
# junction (absolute) and stops the chunk there (req_2 has num_computed 0).
# Create minimal mock with just the needed attributes
mock = SimpleNamespace(
block_size=block_size,
cache_config=SimpleNamespace(block_size=block_size),
max_num_scheduled_tokens=3 * block_size,
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
+3
View File
@@ -609,6 +609,9 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
return replace(
spec,
num_prefill_checkpoint_blocks=int(self.kda_prefill_backend == "flashkda"),
prefill_checkpoint_alignment=(
16 if self.kda_prefill_backend == "flashkda" else None
),
)
def forward(
+35 -15
View File
@@ -20,6 +20,7 @@ import torch
from vllm.config import VllmConfig
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backends.gdn_attn import (
@@ -36,7 +37,11 @@ from vllm.v1.attention.backends.utils import (
compute_causal_conv1d_metadata,
split_decodes_and_prefills,
)
from vllm.v1.kv_cache_interface import MambaSpec
from vllm.v1.kv_cache_interface import (
MambaSpec,
get_mamba_prefill_checkpoint_position,
is_mamba_prefill_checkpoint_valid,
)
if TYPE_CHECKING:
from vllm.models.kimi_k3.nvidia.ops.recoverssm import (
@@ -44,9 +49,6 @@ if TYPE_CHECKING:
)
FLASHKDA_CHUNK_SIZE = 16
@cache
def _metadata_launch_pdl() -> bool:
return current_platform.is_arch_support_pdl()
@@ -582,11 +584,11 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
has_initial_state = None
checkpoint = None
if (
num_prefills > 0
checkpoint_enabled = (
self.vllm_config.cache_config.mamba_cache_mode == "align"
and self.kv_cache_spec.num_prefill_checkpoint_blocks > 0
and self.vllm_config.cache_config.mamba_cache_mode == "align"
):
)
if num_prefills > 0 and checkpoint_enabled:
# prepare checkpoint metadata
assert m.seq_lens_cpu_upper_bound is not None
request_rows = list(range(m.num_reqs))
@@ -597,21 +599,39 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
query_lens = [all_query_lens[row] for row in request_rows]
seq_lens = m.seq_lens_cpu_upper_bound.tolist()
block_size = self.kv_cache_spec.block_size
hash_block_size = (
self.vllm_config.cache_config.prefix_match_unit or block_size
)
speculative_config = self.vllm_config.speculative_config
drop_eagle_block = (
speculative_config is not None
and speculative_config.use_eagle_block_drop()
)
checkpoint_splits = []
checkpoint_cols = []
for row, query_len in zip(request_rows, query_lens):
seq_len = seq_lens[row]
offset = seq_len // block_size * block_size - (seq_len - query_len)
# offset should be less than query_len
valid = (
seq_len % block_size != 0
and 0 < offset < query_len
and offset % FLASHKDA_CHUNK_SIZE == 0
query_start = seq_len - query_len
checkpoint_position = get_mamba_prefill_checkpoint_position(
seq_len,
hash_block_size,
drop_eagle_block=drop_eagle_block,
)
offset = checkpoint_position - query_start
valid = is_mamba_prefill_checkpoint_valid(
query_start=query_start,
query_end=seq_len,
checkpoint_position=checkpoint_position,
hash_block_size=hash_block_size,
mamba_block_size=block_size,
checkpoint_alignment=(
self.kv_cache_spec.prefill_checkpoint_alignment
),
)
offset = offset if valid else 0
first_len = offset or query_len
checkpoint_splits.append((first_len, query_len - first_len))
checkpoint_cols.append(seq_len // block_size - 1 if valid else -1)
checkpoint_cols.append(cdiv(seq_len, block_size) - 2 if valid else -1)
if any(tail for _, tail in checkpoint_splits):
checkpoint_offsets_tensor = async_tensor_h2d(
[first if tail else 0 for first, tail in checkpoint_splits],
+12 -3
View File
@@ -450,6 +450,7 @@ class BlockPool:
num_tokens: int,
kv_cache_group_id: int,
block_size: int,
replace_existing_hashes: bool = False,
) -> BlockHashWithGroupId | None:
"""Register a partial prefix-cache entry for an existing block.
@@ -477,6 +478,9 @@ class BlockPool:
entry hash itself is always the prefix-chain hash at
``num_tokens``; ``block_size`` is used to assert that the
entry is partial within the owning cache block.
replace_existing_hashes: Whether the block contents were replaced
and all existing cache entries must be removed before the new
entry is registered.
Returns:
The hash key with group ID if a partial entry can be registered;
@@ -485,9 +489,10 @@ class BlockPool:
if block.is_null:
return None
assert block_size > self.hash_block_size
assert block_size % self.hash_block_size == 0
assert num_tokens % block_size != 0
assert replace_existing_hashes or (
block_size > self.hash_block_size and num_tokens % block_size != 0
)
block_hash = self._get_partial_block_hash(request, num_tokens)
num_hash_blocks = num_tokens // self.hash_block_size
block_hash_with_group_id = make_block_hash_with_group_id(
@@ -498,7 +503,11 @@ class BlockPool:
block_hash_with_group_id, block.block_id
)
)
if (
if replace_existing_hashes:
removed_hashes = self._remove_cached_block_hashes(block)
self._emit_block_removed_events(removed_hashes)
already_cached = False
elif (
not already_cached
and block.block_hash is not None
and block.block_hash_num_tokens is not None
+6
View File
@@ -15,6 +15,7 @@ from vllm.v1.core.kv_cache_utils import (
)
from vllm.v1.core.single_type_kv_cache_manager import (
CrossAttentionManager,
MambaManager,
SingleTypeKVCacheManager,
get_manager_for_kv_cache_spec,
)
@@ -151,6 +152,11 @@ class KVCacheCoordinator(ABC):
)
for i, kv_cache_group in enumerate(self.kv_cache_config.kv_cache_groups)
)
# Match Mamba checkpoints to Eagle's attention replay boundary.
if use_eagle:
for manager in self.single_type_managers:
if isinstance(manager, MambaManager):
manager.drop_eagle_checkpoint_block = True
# A positive retention interval must be a multiple of the base hit granularity
# (``scheduler_block_size``) to land on real cache-hit boundaries.
+39 -13
View File
@@ -54,7 +54,12 @@ from vllm.v1.core.sched.request_queue import (
)
from vllm.v1.core.sched.utils import check_stop, remove_all
from vllm.v1.engine import EngineCoreEventType, EngineCoreOutput, EngineCoreOutputs
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
MambaSpec,
get_mamba_prefill_checkpoint_position,
is_mamba_prefill_checkpoint_valid,
)
from vllm.v1.metrics.perf import ModelMetrics, PerfStats
from vllm.v1.metrics.stats import (
PrefixCacheStats,
@@ -334,15 +339,20 @@ class Scheduler(SchedulerInterface):
self.need_mamba_block_aligned_split = (
self.has_mamba_layers and self.cache_config.mamba_cache_mode == "align"
)
self.mamba_has_prefill_checkpoint_blocks = (
self.has_mamba_layers
# TODO: support spec decoding
and not self.use_eagle
and all(
not isinstance(group.kv_cache_spec, MambaSpec)
or group.kv_cache_spec.num_prefill_checkpoint_blocks > 0
# TODO: Support models with multiple Mamba specs that require different
# prefill checkpoint alignments instead of selecting the first one.
self.mamba_prefill_checkpoint_alignment = next(
(
group.kv_cache_spec.prefill_checkpoint_alignment
for group in kv_cache_config.kv_cache_groups
)
if isinstance(group.kv_cache_spec, MambaSpec)
),
None,
)
self.mamba_has_prefill_checkpoint_blocks = self.has_mamba_layers and all(
not isinstance(group.kv_cache_spec, MambaSpec)
or group.kv_cache_spec.num_prefill_checkpoint_blocks > 0
for group in kv_cache_config.kv_cache_groups
)
# A finer prefix_match_unit is configured: a mamba partial tail entry
# can only be registered by a step ending exactly at the prompt's last
@@ -426,8 +436,22 @@ class Scheduler(SchedulerInterface):
last_cache_position = max(last_cache_position - block_size, 0)
end = start + num_new_tokens
checkpoint_position = get_mamba_prefill_checkpoint_position(
prefill_end,
self.hash_block_size,
drop_eagle_block=self.use_eagle_block_drop,
)
use_internal_checkpoint = (
self.mamba_has_prefill_checkpoint_blocks and start % block_size == 0
self.mamba_has_prefill_checkpoint_blocks
and end >= prefill_end
and is_mamba_prefill_checkpoint_valid(
query_start=start,
query_end=end,
checkpoint_position=checkpoint_position,
hash_block_size=self.hash_block_size,
mamba_block_size=block_size,
checkpoint_alignment=self.mamba_prefill_checkpoint_alignment,
)
)
if use_internal_checkpoint:
last_cache_position = 0
@@ -436,7 +460,7 @@ class Scheduler(SchedulerInterface):
# aligned. Exempt: the prompt's last chunk, whose slot decode advances
# to the boundary. A block too wide for one chunk advances sub-block
# and re-aligns at the next boundary.
if end < prefill_end and not use_internal_checkpoint:
if end < prefill_end:
max_prefill_tokens = self.max_num_scheduled_tokens
long_prefill_threshold = self.scheduler_config.long_prefill_token_threshold
if long_prefill_threshold > 0:
@@ -448,13 +472,15 @@ class Scheduler(SchedulerInterface):
next_block_boundary = (start // block_size + 1) * block_size
tail_boundary = (
request.num_prompt_tokens // self.hash_block_size * self.hash_block_size
if self.mamba_partial_cache_hit
if self.mamba_partial_cache_hit and not use_internal_checkpoint
else 0
)
stops = (
# Same invariant: a chunk starting mid-block stops at the boundary
# rather than running past it.
next_block_boundary if start % block_size != 0 else 0,
next_block_boundary
if start % block_size != 0 and not use_internal_checkpoint
else 0,
# Never run past the last cacheable block boundary mid-chunk.
last_cache_position,
# Fine-grained hits: the prompt's partial-tail entry can only be
+80 -24
View File
@@ -31,6 +31,8 @@ from vllm.v1.kv_cache_interface import (
SinkFullAttentionSpec,
SlidingWindowMLASpec,
SlidingWindowSpec,
get_mamba_prefill_checkpoint_position,
is_mamba_prefill_checkpoint_valid,
)
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
from vllm.v1.request import Request
@@ -1399,6 +1401,12 @@ class MambaManager(SingleTypeKVCacheManager):
self.block_size = kv_cache_spec.block_size
self.mamba_cache_mode = kv_cache_spec.mamba_cache_mode
self.num_speculative_blocks: int = kv_cache_spec.num_speculative_blocks
self.has_prefill_checkpoint_blocks = (
self.mamba_cache_mode == "align"
and kv_cache_spec.num_prefill_checkpoint_blocks > 0
)
# Mamba checkpoints follow Eagle's global replay boundary.
self.drop_eagle_checkpoint_block = False
self.cached_blocks_this_step: set[BlockHashWithGroupId] = set()
if self.mamba_cache_mode == "align":
# Mapping from request ID to the index of the block
@@ -1406,9 +1414,9 @@ class MambaManager(SingleTypeKVCacheManager):
self.last_state_block_idx: dict[str, int] = {}
# The set of the requests that have been allocated blocks
self._allocated_block_reqs: set[str] = set()
# Number of internal checkpoint blocks required by each request's
# Absolute checkpoint position selected by the scheduler for the
# current allocation.
self._num_checkpoint_blocks: dict[str, int] = {}
self._checkpoint_positions: dict[str, int] = {}
# Requests that registered their own last-prompt-boundary partial
# tail (producers). A later CoW hands its private copy to the
# connector; a request that finishes first hands off this table
@@ -1592,18 +1600,32 @@ class MambaManager(SingleTypeKVCacheManager):
def _needs_internal_checkpoint(
self,
request_id: str,
num_tokens: int,
num_computed_tokens: int,
query_start: int,
query_end: int,
checkpoint_position: int,
) -> bool:
assert isinstance(self.kv_cache_spec, MambaSpec)
checkpoint_idx = cdiv(num_tokens, self.block_size) - 2
checkpoint_idx = cdiv(query_end, self.block_size) - 2
blocks = self.req_to_blocks[request_id]
return (
self.kv_cache_spec.num_prefill_checkpoint_blocks > 0
and num_tokens % self.block_size != 0
and num_computed_tokens % self.block_size == 0
self.has_prefill_checkpoint_blocks
and is_mamba_prefill_checkpoint_valid(
query_start=query_start,
query_end=query_end,
checkpoint_position=checkpoint_position,
hash_block_size=self.block_pool.hash_block_size,
mamba_block_size=self.block_size,
checkpoint_alignment=(self.kv_cache_spec.prefill_checkpoint_alignment),
)
and checkpoint_idx >= 0
and (checkpoint_idx >= len(blocks) or blocks[checkpoint_idx].is_null)
and (
checkpoint_idx >= len(blocks)
or blocks[checkpoint_idx].is_null
or (
request_id in self._allocated_block_reqs
and checkpoint_idx >= len(blocks) - self.num_speculative_blocks
)
)
)
def get_num_blocks_to_allocate(
@@ -1668,17 +1690,27 @@ class MambaManager(SingleTypeKVCacheManager):
)
if has_partial_hit:
num_new_blocks = max(num_new_blocks, 0) + 1
checkpoint_block = int(
self._needs_internal_checkpoint(
request_id, num_tokens, total_computed_tokens
)
checkpoint_position = get_mamba_prefill_checkpoint_position(
num_tokens,
self.block_pool.hash_block_size,
self.drop_eagle_checkpoint_block,
)
if not self._needs_internal_checkpoint(
request_id,
total_computed_tokens,
num_tokens,
checkpoint_position,
):
checkpoint_position = 0
checkpoint_block = int(checkpoint_position > 0)
if not apply_admission_cap:
self._num_checkpoint_blocks[request_id] = checkpoint_block
self._checkpoint_positions[request_id] = checkpoint_position
if num_new_blocks > 0:
num_new_blocks = 1 + int(has_partial_hit) + checkpoint_block
if request_id not in self._allocated_block_reqs:
num_new_blocks += self.num_speculative_blocks
blocks_allocated = request_id in self._allocated_block_reqs
if not (checkpoint_block and blocks_allocated):
num_new_blocks = 1 + int(has_partial_hit) + checkpoint_block
if not blocks_allocated:
num_new_blocks += self.num_speculative_blocks
num_evictable_computed_blocks = self._get_num_evictable_blocks(
new_computed_blocks
@@ -1710,12 +1742,17 @@ class MambaManager(SingleTypeKVCacheManager):
num_required_blocks = (
cdiv(num_tokens, self.block_size) + self.num_speculative_blocks
)
checkpoint_block = self._num_checkpoint_blocks.get(request_id, 0)
checkpoint_position = self._checkpoint_positions.get(request_id, 0)
checkpoint_block = int(checkpoint_position > 0)
partial_hit = self._partial_hit_reqs.get(request_id)
has_partial_hit = partial_hit is not None
# `num_required_blocks` might be less than `len(req_blocks)` if blocks are
# over-allocated at last round.
if num_required_blocks <= len(req_blocks) and not has_partial_hit:
if (
num_required_blocks <= len(req_blocks)
and not has_partial_hit
and not checkpoint_block
):
self._allocated_block_reqs.add(request_id)
return []
else:
@@ -1745,7 +1782,7 @@ class MambaManager(SingleTypeKVCacheManager):
[self._null_block for _ in range(prev_block_len, null_end)]
)
if blocks_allocated:
if blocks_allocated and not checkpoint_block:
# Relocate exclusively owned speculative scratch blocks.
for block_idx in range(
prev_block_len - self.num_speculative_blocks, prev_block_len
@@ -1754,11 +1791,11 @@ class MambaManager(SingleTypeKVCacheManager):
self._relocate_speculative_block(req_blocks, block_idx)
else:
break
num_new_blocks = num_required_blocks - len(req_blocks)
num_new_blocks = max(num_required_blocks - len(req_blocks), 0)
if has_partial_hit:
num_new_blocks = max(num_new_blocks, 0) + 1
max_new_blocks = 1 + int(has_partial_hit) + checkpoint_block
if not blocks_allocated:
if not blocks_allocated or checkpoint_block:
max_new_blocks += self.num_speculative_blocks
assert num_new_blocks <= max_new_blocks
new_blocks = self.block_pool.get_new_blocks(num_new_blocks)
@@ -1838,7 +1875,7 @@ class MambaManager(SingleTypeKVCacheManager):
if self.mamba_cache_mode == "align":
self._allocated_block_reqs.discard(request_id)
self.last_state_block_idx.pop(request_id, None)
self._num_checkpoint_blocks.pop(request_id, None)
self._checkpoint_positions.pop(request_id, None)
self._producer_partial_tail_reqs.pop(request_id, None)
# An offer is only guaranteed to hold committed bytes until the end
# of the pass that made it. This request's blocks are going back to
@@ -1885,6 +1922,7 @@ class MambaManager(SingleTypeKVCacheManager):
continue
self.cached_blocks_this_step.add(block.block_hash)
if self.mamba_cache_mode == "align":
assert block.block_hash_num_tokens is not None
# Offer every retained boundary with its exact block.
# The connector filters against its save window, which may
# extend past the original prompt during resumed prefill.
@@ -1893,7 +1931,7 @@ class MambaManager(SingleTypeKVCacheManager):
request.request_id,
self.kv_cache_group_id,
block,
(idx + 1) * self.block_size,
block.block_hash_num_tokens,
)
)
@@ -1906,6 +1944,24 @@ class MambaManager(SingleTypeKVCacheManager):
num_tokens: int,
) -> BlockHashWithGroupId | None:
hash_block_size = self.block_pool.hash_block_size
# Re-key the reserved block at its exported checkpoint boundary.
checkpoint_position = self._checkpoint_positions.get(request.request_id, 0)
if checkpoint_position > 0:
# TODO: Store the reserved slot explicitly for multi-module MTP.
checkpoint_idx = cdiv(num_tokens, self.block_size) - 2
blocks = self.req_to_blocks[request.request_id]
assert 0 <= checkpoint_idx < len(blocks)
checkpoint_block = blocks[checkpoint_idx]
if checkpoint_block.block_hash_num_tokens == checkpoint_position:
return None
return self.block_pool.cache_partial_block(
request=request,
block=checkpoint_block,
num_tokens=checkpoint_position,
kv_cache_group_id=self.kv_cache_group_id,
block_size=self.block_size,
replace_existing_hashes=True,
)
if self.block_size == hash_block_size:
return None
if num_tokens % self.block_size == 0:
+38
View File
@@ -890,6 +890,7 @@ class MambaSpec(KVCacheSpec):
mamba_cache_mode: str = "none"
num_speculative_blocks: int = 0
num_prefill_checkpoint_blocks: int = 0
prefill_checkpoint_alignment: int | None = None
num_heads: int = 1
tokens_per_state: int = -1
# False: the state is sharded across TP ranks (e.g. GDN). True: every TP
@@ -950,12 +951,49 @@ class MambaSpec(KVCacheSpec):
isinstance(spec, MambaSpec)
and spec.num_speculative_blocks == self.num_speculative_blocks
and spec.num_prefill_checkpoint_blocks == self.num_prefill_checkpoint_blocks
and spec.prefill_checkpoint_alignment == self.prefill_checkpoint_alignment
and spec.page_size_bytes == self.page_size_bytes
and spec.tp_replicated == self.tp_replicated
for spec in kv_cache_specs.values()
)
def get_mamba_prefill_checkpoint_position(
num_tokens: int,
hash_block_size: int,
drop_eagle_block: bool,
) -> int:
"""Return the reusable Mamba checkpoint boundary for a prefill."""
checkpoint_position = (num_tokens - 1) // hash_block_size * hash_block_size
if drop_eagle_block:
checkpoint_position -= hash_block_size
return max(checkpoint_position, 0)
def is_mamba_prefill_checkpoint_valid(
query_start: int,
query_end: int,
checkpoint_position: int,
hash_block_size: int,
mamba_block_size: int,
checkpoint_alignment: int | None,
) -> bool:
"""Whether a backend can export the checkpoint in this query."""
if checkpoint_alignment is None:
return False
assert checkpoint_alignment > 0
initial_state_col = (query_start - 1) // mamba_block_size
checkpoint_col = cdiv(query_end, mamba_block_size) - 2
return (
query_start % hash_block_size == 0
and checkpoint_col > initial_state_col
and query_start + hash_block_size <= checkpoint_position
and query_start < checkpoint_position < query_end
and (checkpoint_position - query_start) % checkpoint_alignment == 0
)
@dataclass(frozen=True)
class EncoderOnlyAttentionSpec(AttentionSpec):
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int: