From 144e79c8106da23141ac010394b782f730cc7fe8 Mon Sep 17 00:00:00 2001 From: Ziming Huang Date: Sun, 6 Sep 2026 13:18:15 +0800 Subject: [PATCH] [Kimi K3] Support internal prefix checkpoints with partial prefix caching and spec-decoding (#53614) Signed-off-by: Ziming Huang Co-authored-by: OpenAI Codex --- tests/models/kimi_k3/test_kda_metadata.py | 72 ++++++++- .../test_partial_prefix_cache_hits.py | 143 +++++++++++++----- tests/v1/core/test_mamba_align_chunk_split.py | 105 +++++++++++-- tests/v1/core/test_prefix_caching.py | 1 + vllm/models/kimi_k3/nvidia/kda.py | 3 + vllm/models/kimi_k3/nvidia/kda_metadata.py | 50 ++++-- vllm/v1/core/block_pool.py | 15 +- vllm/v1/core/kv_cache_coordinator.py | 6 + vllm/v1/core/sched/scheduler.py | 52 +++++-- vllm/v1/core/single_type_kv_cache_manager.py | 104 ++++++++++--- vllm/v1/kv_cache_interface.py | 38 +++++ 11 files changed, 482 insertions(+), 107 deletions(-) diff --git a/tests/models/kimi_k3/test_kda_metadata.py b/tests/models/kimi_k3/test_kda_metadata.py index ecf8d8e7c3..25ba2d6b17 100644 --- a/tests/models/kimi_k3/test_kda_metadata.py +++ b/tests/models/kimi_k3/test_kda_metadata.py @@ -92,6 +92,10 @@ def _make_builder( mamba_cache_mode: str = "none", use_recoverssm: bool = False, num_prefill_checkpoint_blocks: int = 0, + mamba_block_size: int = BLOCK_SIZE, + prefix_match_unit: int | None = None, + use_eagle: bool = False, + disable_eagle_block_drop: bool = False, ) -> AttentionMetadataBuilder: vllm_config = create_vllm_config( model_name="Qwen/Qwen3.5-0.8B", @@ -102,20 +106,29 @@ def _make_builder( method="ngram", num_speculative_tokens=num_speculative_tokens, ) + if use_eagle: + vllm_config.speculative_config.method = "eagle3" + vllm_config.speculative_config.disable_eagle_block_drop = ( + disable_eagle_block_drop + ) vllm_config.compilation_config.cudagraph_mode = ( CUDAGraphMode.FULL_AND_PIECEWISE if full_cuda_graph else CUDAGraphMode.NONE ) vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode vllm_config.cache_config.use_replayssm = use_recoverssm vllm_config.cache_config.use_kda_recoverssm = use_recoverssm + vllm_config.cache_config.prefix_match_unit = prefix_match_unit builder = builder_cls( kv_cache_spec=MambaSpec( - block_size=BLOCK_SIZE, + block_size=mamba_block_size, shapes=((16, 64),), dtypes=(torch.float16,), mamba_cache_mode=mamba_cache_mode, num_speculative_blocks=(0 if use_recoverssm else num_speculative_tokens), num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks, + prefill_checkpoint_alignment=( + 16 if num_prefill_checkpoint_blocks > 0 else None + ), ), layer_names=["layer.0"], vllm_config=vllm_config, @@ -156,6 +169,7 @@ def test_kda_recoverssm_startup_metadata_flow_without_model(monkeypatch): mamba_type=MambaAttentionBackendEnum.GDN_ATTN, mamba_cache_mode="align", num_prefill_checkpoint_blocks=1, + prefill_checkpoint_alignment=16, ) # This is the same compatibility check performed while initializing the @@ -177,11 +191,13 @@ def test_kda_recoverssm_startup_metadata_flow_without_model(monkeypatch): cache_config=SimpleNamespace( mamba_cache_mode="align", use_kda_recoverssm=True, + prefix_match_unit=None, ), parallel_config=SimpleNamespace(decode_context_parallel_size=1), speculative_config=SimpleNamespace( num_speculative_tokens=2, parallel_drafting=False, + use_eagle_block_drop=Mock(return_value=False), ), compilation_config=SimpleNamespace( cudagraph_mode=CUDAGraphMode.NONE, @@ -300,6 +316,60 @@ def test_internal_checkpoint_metadata_targets_last_aligned_boundary(): ) +@pytest.mark.parametrize( + ("disable_eagle_block_drop", "prefix_match_unit", "expected_offset"), + [(False, 16, 80), (True, 16, 96), (False, 8, None)], +) +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_spec_internal_checkpoint_metadata_targets_replay_boundary( + disable_eagle_block_drop: bool, + prefix_match_unit: int, + expected_offset: int | None, +) -> None: + device = torch.device("cuda") + batch = BatchSpec(seq_lens=[100], query_lens=[100]) + common_attn_metadata = create_common_attn_metadata( + batch, BLOCK_SIZE, device, arange_block_indices=True + ) + common_attn_metadata = common_attn_metadata.replace( + is_prefilling=torch.tensor([True]), + block_table_tensor=common_attn_metadata.block_table_tensor + 1, + ) + builder = _make_builder( + KimiK3KDAMetadataBuilder, + num_speculative_tokens=3, + full_cuda_graph=False, + mamba_cache_mode="align", + num_prefill_checkpoint_blocks=1, + mamba_block_size=64, + prefix_match_unit=prefix_match_unit, + use_eagle=True, + disable_eagle_block_drop=disable_eagle_block_drop, + device=device, + ) + assert isinstance(builder, KimiK3KDAMetadataBuilder) + builder.mamba_aligned_state_indices = mamba_get_block_table_tensor( + common_attn_metadata.block_table_tensor, + common_attn_metadata.seq_lens, + builder.kv_cache_spec, + "align", + ) + actual = builder.build(0, common_attn_metadata) + + if expected_offset is None: + assert actual.checkpoint is None + return + assert actual.checkpoint is not None + torch.testing.assert_close( + actual.checkpoint.state_indices, + torch.tensor([1], dtype=torch.int32, device=device), + ) + torch.testing.assert_close( + actual.checkpoint.checkpoint_offsets, + torch.tensor([expected_offset], dtype=torch.int32, device=device), + ) + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") def test_internal_checkpoint_metadata_skips_unaligned_offset(): device = torch.device("cuda") diff --git a/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py b/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py index 864d88831a..aeffc96453 100644 --- a/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py +++ b/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py @@ -92,8 +92,23 @@ def make_full_mamba_manager( mamba_block_size: int = 4, num_blocks: int = 32, use_eagle: bool = False, + num_speculative_blocks: int = 0, num_prefill_checkpoint_blocks: int = 0, ): + mamba_group = KVCacheGroupSpec( + ["mamba"], + MambaSpec( + block_size=mamba_block_size, + shapes=(1, 1), + dtypes=(torch.float32,), + mamba_cache_mode="align", + num_speculative_blocks=num_speculative_blocks, + num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks, + prefill_checkpoint_alignment=( + 16 if num_prefill_checkpoint_blocks > 0 else None + ), + ), + ) kv_cache_config = KVCacheConfig( num_blocks=num_blocks, kv_cache_tensors=[], @@ -107,16 +122,7 @@ def make_full_mamba_manager( dtype=torch.float32, ), ), - KVCacheGroupSpec( - ["mamba"], - MambaSpec( - block_size=mamba_block_size, - shapes=(1, 1), - dtypes=(torch.float32,), - mamba_cache_mode="align", - num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks, - ), - ), + mamba_group, ], ) scheduler_block_size = lcm( @@ -169,6 +175,7 @@ def test_mamba_align_split_partial_tail_schedule(dcp_world_size: int): scheduler_block_size = block_size * dcp_world_size hash_block_size = 32 mock = SimpleNamespace( + block_size=scheduler_block_size, cache_config=SimpleNamespace(block_size=block_size), max_num_scheduled_tokens=8192, scheduler_config=SimpleNamespace(long_prefill_token_threshold=0), @@ -217,6 +224,7 @@ def test_mamba_align_split_when_block_exceeds_scheduling_budget(): token_budget = 8192 prompt_length = 30000 mock = SimpleNamespace( + block_size=block_size, cache_config=SimpleNamespace(block_size=block_size), max_num_scheduled_tokens=token_budget, scheduler_config=SimpleNamespace(long_prefill_token_threshold=0), @@ -254,6 +262,7 @@ def test_mamba_align_split_when_block_exceeds_long_prefill_threshold(): long_prefill_threshold = 384 prompt_length = 1300 mock = SimpleNamespace( + block_size=block_size, cache_config=SimpleNamespace(block_size=block_size), max_num_scheduled_tokens=token_budget, scheduler_config=SimpleNamespace( @@ -557,9 +566,9 @@ def test_partial_hit_then_internal_checkpoint_uses_distinct_mamba_blocks(): assert mamba_blocks[3].block_id == running_block_id -def test_internal_checkpoint_requires_block_aligned_start(): - hash_block_size = 2 - mamba_block_size = 16 +def test_internal_checkpoint_uses_partial_hash_lifecycle(): + hash_block_size = 16 + mamba_block_size = 32 manager = make_full_mamba_manager( dcp_world_size=1, hash_block_size=hash_block_size, @@ -567,23 +576,90 @@ def test_internal_checkpoint_requires_block_aligned_start(): mamba_block_size=mamba_block_size, num_prefill_checkpoint_blocks=1, ) - request = make_request("producer", list(range(50)), hash_block_size, sha256) + request = make_request("producer", list(range(120)), hash_block_size, sha256) - # Compute token 0 first, so the next query starts at token 1, which is not - # aligned to the Mamba block size. - assert manager.allocate_slots(request, 1) is not None - request.num_computed_tokens = 1 - manager.new_step_starts() - - new_blocks = manager.allocate_slots(request, 49) + new_blocks = manager.allocate_slots(request, request.num_tokens) assert new_blocks is not None mamba_blocks = manager.get_blocks(request.request_id).blocks[1] - checkpoint_block_idx = 48 // mamba_block_size - 1 - assert mamba_blocks[checkpoint_block_idx].is_null - assert not mamba_blocks[-1].is_null - checkpoint_hash = request.block_hashes[48 // hash_block_size - 1] - assert manager.block_pool.get_cached_block(checkpoint_hash, [1]) is None + checkpoint_idx = 2 + checkpoint_block = mamba_blocks[checkpoint_idx] + running_block = mamba_blocks[checkpoint_idx + 1] + assert not checkpoint_block.is_null + assert not running_block.is_null + assert checkpoint_block is not running_block + + # The internal block is exported at state@112, replacing its temporary + # full-block state@96 key while retaining #52789's req_to_blocks ownership. + partial_hash = request.block_hashes[112 // hash_block_size - 1] + partial_hit = manager.block_pool.get_cached_block(partial_hash, [1]) + assert partial_hit is not None + assert partial_hit[0] is checkpoint_block + assert checkpoint_block.block_hash_num_tokens == 112 + full_hash = request.block_hashes[96 // hash_block_size - 1] + assert manager.block_pool.get_cached_block(full_hash, [1]) is None + assert drain_boundary_state_offloads(manager) == { + request.request_id: [(1, checkpoint_block.block_id, 112)] + } + + manager.free(request) + replay = make_request("replay", list(range(120)), hash_block_size, sha256) + _, num_computed, _ = manager.get_computed_blocks(replay) + assert num_computed == 112 + + +def test_eagle_block_aligned_checkpoint_replaces_newer_hash(): + hash_block_size = mamba_block_size = 32 + manager = make_full_mamba_manager( + dcp_world_size=1, + hash_block_size=hash_block_size, + full_block_size=hash_block_size, + mamba_block_size=mamba_block_size, + use_eagle=True, + num_speculative_blocks=3, + num_prefill_checkpoint_blocks=1, + ) + request = make_request("producer", list(range(104)), hash_block_size, sha256) + + assert manager.allocate_slots(request, request.num_tokens) is not None + + mamba_blocks = manager.get_blocks(request.request_id).blocks[1] + checkpoint_block = mamba_blocks[2] + checkpoint_hash = request.block_hashes[64 // hash_block_size - 1] + checkpoint_hit = manager.block_pool.get_cached_block(checkpoint_hash, [1]) + assert checkpoint_hit is not None + assert checkpoint_hit[0] is checkpoint_block + assert checkpoint_block.block_hash_num_tokens == 64 + + overwritten_full_hash = request.block_hashes[96 // hash_block_size - 1] + assert manager.block_pool.get_cached_block(overwritten_full_hash, [1]) is None + + +def test_hash_aligned_query_end_uses_regular_partial_tail(): + hash_block_size = 2 + mamba_block_size = 4 + manager = make_full_mamba_manager( + dcp_world_size=1, + hash_block_size=hash_block_size, + full_block_size=hash_block_size, + mamba_block_size=mamba_block_size, + num_prefill_checkpoint_blocks=1, + ) + request = make_request("producer", list(range(14)), hash_block_size, sha256) + + new_blocks = manager.allocate_slots(request, request.num_tokens) + + assert new_blocks is not None + mamba_manager = manager.coordinator.single_type_managers[1] + assert mamba_manager._checkpoint_positions[request.request_id] == 0 + mamba_blocks = manager.get_blocks(request.request_id).blocks[1] + assert sum(not block.is_null for block in mamba_blocks) == 1 + + partial_hash = request.block_hashes[14 // hash_block_size - 1] + cached = manager.block_pool.get_cached_block(partial_hash, [1]) + assert cached is not None + assert cached[0] is mamba_blocks[14 // mamba_block_size] + assert cached[0].block_hash_num_tokens == 14 def test_external_mamba_hit_same_block_uses_running_cow_on_continue(): @@ -699,18 +775,9 @@ def test_boundary_state_offloads_returns_cow_target(): computed_blocks, num_computed, _ = manager.get_computed_blocks(req0) assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None - # Step A offers the materialized aligned checkpoint immediately. - ((group_id, block_id, boundary_tokens),) = drain_boundary_state_offloads(manager)[ - "0" - ] - assert group_id == 1 - assert boundary_tokens == 4 - aligned_hash = req0.block_hashes[4 // hash_block_size - 1] - aligned_block = manager.block_pool.get_cached_block( - aligned_hash, kv_cache_group_ids=[1] - ) - assert aligned_block is not None - assert block_id == aligned_block[0].block_id + # The final hash-aligned boundary uses the regular partial-tail path rather + # than allocating an internal checkpoint at the query end. + assert drain_boundary_state_offloads(manager) == {} partial_mamba_hash = req0.block_hashes[6 // hash_block_size - 1] source_block = manager.block_pool.get_cached_block( diff --git a/tests/v1/core/test_mamba_align_chunk_split.py b/tests/v1/core/test_mamba_align_chunk_split.py index 2d7d77f1ce..cab832cbee 100644 --- a/tests/v1/core/test_mamba_align_chunk_split.py +++ b/tests/v1/core/test_mamba_align_chunk_split.py @@ -8,6 +8,7 @@ the wrong offset, and a later chunk crossing that boundary publishes it anyway. Requests resuming from it then restore a truncated state (#43559). """ +import sys from types import SimpleNamespace import pytest @@ -21,6 +22,7 @@ from vllm.v1.kv_cache_interface import ( KVCacheConfig, KVCacheGroupSpec, MambaSpec, + is_mamba_prefill_checkpoint_valid, ) from vllm.v1.request import Request @@ -37,6 +39,49 @@ PROMPT_LEN = 2002 MAMBA_GROUP_ID = 1 +@pytest.mark.parametrize( + ( + "query_start", + "query_end", + "checkpoint_position", + "hash_size", + "block_size", + "alignment", + "expected", + ), + [ + (0, 100, 88, 8, 64, 16, False), + (0, 100, 96, 8, 64, 16, True), + (0, 100, 96, 8, 64, None, False), + (8, 100, 96, 8, 64, 16, False), + (15360, 16896, 16768, 128, 1536, 16, False), + (15360, 16960, 16896, 128, 1536, 16, True), + (15360, 16960, 16768, 128, 1536, 16, True), + (16064, 18433, 18432, 128, 1536, 16, False), + ], +) +def test_mamba_prefill_checkpoint_valid( + query_start: int, + query_end: int, + checkpoint_position: int, + hash_size: int, + block_size: int, + alignment: int | None, + expected: bool, +) -> None: + assert ( + is_mamba_prefill_checkpoint_valid( + query_start=query_start, + query_end=query_end, + checkpoint_position=checkpoint_position, + hash_block_size=hash_size, + mamba_block_size=block_size, + checkpoint_alignment=alignment, + ) + is expected + ) + + def _make_hybrid_kv_cache_manager( num_prefill_checkpoint_blocks: int = 0, ) -> KVCacheManager: @@ -62,6 +107,9 @@ def _make_hybrid_kv_cache_manager( mamba_cache_mode="align", num_speculative_blocks=NUM_SPEC, num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks, + prefill_checkpoint_alignment=( + 16 if num_prefill_checkpoint_blocks > 0 else None + ), ), ), ], @@ -83,20 +131,23 @@ def _split( use_eagle_block_drop: bool | None = None, partial_hit: bool = False, num_prefill_checkpoint_blocks: int = 0, + max_num_scheduled_tokens: int = 16384, ) -> int: """Call the real `Scheduler._mamba_block_aligned_split` on a stub self.""" if use_eagle_block_drop is None: use_eagle_block_drop = use_eagle stub = SimpleNamespace( + block_size=MAMBA_BLOCK_SIZE, cache_config=SimpleNamespace(block_size=MAMBA_BLOCK_SIZE), use_eagle_block_drop=use_eagle_block_drop, - max_num_scheduled_tokens=16384, + max_num_scheduled_tokens=max_num_scheduled_tokens, scheduler_config=SimpleNamespace(long_prefill_token_threshold=0), # `prefix_match_unit` finer than the block size (#46384). mamba_partial_cache_hit=partial_hit, hash_block_size=ATTN_BLOCK_SIZE, - mamba_has_prefill_checkpoint_blocks=( - num_prefill_checkpoint_blocks > 0 and not use_eagle + mamba_has_prefill_checkpoint_blocks=(num_prefill_checkpoint_blocks > 0), + mamba_prefill_checkpoint_alignment=( + 16 if num_prefill_checkpoint_blocks > 0 else None ), ) return Scheduler._mamba_block_aligned_split(stub, request, num_new_tokens) @@ -106,8 +157,11 @@ def _split( ("prompt_len", "num_new_tokens", "use_eagle", "expected"), [ (2002, 2002, False, 2002), - (3602, 2000, False, 2000), - (3602, 3602, True, MAMBA_BLOCK_SIZE), + (3602, 2000, False, MAMBA_BLOCK_SIZE), + (3602, 3602, True, 3602), + (2002, 2002, True, 2002), + (3602, 512, True, 0), + (3602, 1700, True, MAMBA_BLOCK_SIZE), ], ) def test_internal_checkpoint_split( @@ -123,12 +177,39 @@ def test_internal_checkpoint_split( ) == expected ) - if not use_eagle: - manager = _make_hybrid_kv_cache_manager(num_prefill_checkpoint_blocks=1) - assert manager.allocate_slots(request, expected) is not None - mamba_manager = manager.coordinator.single_type_managers[MAMBA_GROUP_ID] - blocks = mamba_manager.req_to_blocks[request.request_id] - assert all(not block.is_null for block in blocks) # checkpoint + running state + + +def test_partial_checkpoint_resume_stops_at_mamba_block_boundary() -> None: + prompt_len = 3602 + resume_at = 1984 + (request,) = create_requests(1, num_tokens=prompt_len, block_size=ATTN_BLOCK_SIZE) + request.num_computed_tokens = resume_at + + # Without an internal checkpoint export, the forward must stop at the + # physical Mamba boundary so it does not publish a stale running state. + assert ( + _split( + request, + prompt_len - resume_at, + use_eagle=False, + partial_hit=True, + ) + == MAMBA_BLOCK_SIZE - resume_at % MAMBA_BLOCK_SIZE + ) + + # A hash-aligned resume inside a physical Mamba block cannot use the fixed + # internal-checkpoint column: it aliases the initial-state column. Enabling + # checkpoints must therefore preserve the normal aligned split. + assert ( + _split( + request, + prompt_len - resume_at, + use_eagle=False, + partial_hit=True, + num_prefill_checkpoint_blocks=1, + ) + == MAMBA_BLOCK_SIZE - resume_at % MAMBA_BLOCK_SIZE + ) def test_disabling_eagle_block_drop_keeps_the_trailing_cache_boundary() -> None: @@ -252,8 +333,6 @@ def test_poisoning_is_block_size_independent( budgets: list[int], ) -> None: """The invariant is per-block, so large mamba blocks are not safer.""" - import sys - monkeypatch.setattr(sys.modules[__name__], "MAMBA_BLOCK_SIZE", block_size) assert _prefill(prompt_len, budgets=budgets) > 0 diff --git a/tests/v1/core/test_prefix_caching.py b/tests/v1/core/test_prefix_caching.py index 168450a7f2..bcb6457d8c 100644 --- a/tests/v1/core/test_prefix_caching.py +++ b/tests/v1/core/test_prefix_caching.py @@ -1162,6 +1162,7 @@ def test_hybrid_cache_mamba_align_shared_prefix_detection(): # junction (absolute) and stops the chunk there (req_2 has num_computed 0). # Create minimal mock with just the needed attributes mock = SimpleNamespace( + block_size=block_size, cache_config=SimpleNamespace(block_size=block_size), max_num_scheduled_tokens=3 * block_size, scheduler_config=SimpleNamespace(long_prefill_token_threshold=0), diff --git a/vllm/models/kimi_k3/nvidia/kda.py b/vllm/models/kimi_k3/nvidia/kda.py index bfc3572a18..6e33be668e 100644 --- a/vllm/models/kimi_k3/nvidia/kda.py +++ b/vllm/models/kimi_k3/nvidia/kda.py @@ -609,6 +609,9 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): return replace( spec, num_prefill_checkpoint_blocks=int(self.kda_prefill_backend == "flashkda"), + prefill_checkpoint_alignment=( + 16 if self.kda_prefill_backend == "flashkda" else None + ), ) def forward( diff --git a/vllm/models/kimi_k3/nvidia/kda_metadata.py b/vllm/models/kimi_k3/nvidia/kda_metadata.py index 2920bf62a6..1035f45948 100644 --- a/vllm/models/kimi_k3/nvidia/kda_metadata.py +++ b/vllm/models/kimi_k3/nvidia/kda_metadata.py @@ -20,6 +20,7 @@ import torch from vllm.config import VllmConfig from vllm.platforms import current_platform from vllm.triton_utils import tl, triton +from vllm.utils.math_utils import cdiv from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.attention.backend import CommonAttentionMetadata from vllm.v1.attention.backends.gdn_attn import ( @@ -36,7 +37,11 @@ from vllm.v1.attention.backends.utils import ( compute_causal_conv1d_metadata, split_decodes_and_prefills, ) -from vllm.v1.kv_cache_interface import MambaSpec +from vllm.v1.kv_cache_interface import ( + MambaSpec, + get_mamba_prefill_checkpoint_position, + is_mamba_prefill_checkpoint_valid, +) if TYPE_CHECKING: from vllm.models.kimi_k3.nvidia.ops.recoverssm import ( @@ -44,9 +49,6 @@ if TYPE_CHECKING: ) -FLASHKDA_CHUNK_SIZE = 16 - - @cache def _metadata_launch_pdl() -> bool: return current_platform.is_arch_support_pdl() @@ -582,11 +584,11 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder): has_initial_state = None checkpoint = None - if ( - num_prefills > 0 + checkpoint_enabled = ( + self.vllm_config.cache_config.mamba_cache_mode == "align" and self.kv_cache_spec.num_prefill_checkpoint_blocks > 0 - and self.vllm_config.cache_config.mamba_cache_mode == "align" - ): + ) + if num_prefills > 0 and checkpoint_enabled: # prepare checkpoint metadata assert m.seq_lens_cpu_upper_bound is not None request_rows = list(range(m.num_reqs)) @@ -597,21 +599,39 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder): query_lens = [all_query_lens[row] for row in request_rows] seq_lens = m.seq_lens_cpu_upper_bound.tolist() block_size = self.kv_cache_spec.block_size + hash_block_size = ( + self.vllm_config.cache_config.prefix_match_unit or block_size + ) + speculative_config = self.vllm_config.speculative_config + drop_eagle_block = ( + speculative_config is not None + and speculative_config.use_eagle_block_drop() + ) checkpoint_splits = [] checkpoint_cols = [] for row, query_len in zip(request_rows, query_lens): seq_len = seq_lens[row] - offset = seq_len // block_size * block_size - (seq_len - query_len) - # offset should be less than query_len - valid = ( - seq_len % block_size != 0 - and 0 < offset < query_len - and offset % FLASHKDA_CHUNK_SIZE == 0 + query_start = seq_len - query_len + checkpoint_position = get_mamba_prefill_checkpoint_position( + seq_len, + hash_block_size, + drop_eagle_block=drop_eagle_block, + ) + offset = checkpoint_position - query_start + valid = is_mamba_prefill_checkpoint_valid( + query_start=query_start, + query_end=seq_len, + checkpoint_position=checkpoint_position, + hash_block_size=hash_block_size, + mamba_block_size=block_size, + checkpoint_alignment=( + self.kv_cache_spec.prefill_checkpoint_alignment + ), ) offset = offset if valid else 0 first_len = offset or query_len checkpoint_splits.append((first_len, query_len - first_len)) - checkpoint_cols.append(seq_len // block_size - 1 if valid else -1) + checkpoint_cols.append(cdiv(seq_len, block_size) - 2 if valid else -1) if any(tail for _, tail in checkpoint_splits): checkpoint_offsets_tensor = async_tensor_h2d( [first if tail else 0 for first, tail in checkpoint_splits], diff --git a/vllm/v1/core/block_pool.py b/vllm/v1/core/block_pool.py index a53d1b323b..1efa0e0cea 100644 --- a/vllm/v1/core/block_pool.py +++ b/vllm/v1/core/block_pool.py @@ -450,6 +450,7 @@ class BlockPool: num_tokens: int, kv_cache_group_id: int, block_size: int, + replace_existing_hashes: bool = False, ) -> BlockHashWithGroupId | None: """Register a partial prefix-cache entry for an existing block. @@ -477,6 +478,9 @@ class BlockPool: entry hash itself is always the prefix-chain hash at ``num_tokens``; ``block_size`` is used to assert that the entry is partial within the owning cache block. + replace_existing_hashes: Whether the block contents were replaced + and all existing cache entries must be removed before the new + entry is registered. Returns: The hash key with group ID if a partial entry can be registered; @@ -485,9 +489,10 @@ class BlockPool: if block.is_null: return None - assert block_size > self.hash_block_size assert block_size % self.hash_block_size == 0 - assert num_tokens % block_size != 0 + assert replace_existing_hashes or ( + block_size > self.hash_block_size and num_tokens % block_size != 0 + ) block_hash = self._get_partial_block_hash(request, num_tokens) num_hash_blocks = num_tokens // self.hash_block_size block_hash_with_group_id = make_block_hash_with_group_id( @@ -498,7 +503,11 @@ class BlockPool: block_hash_with_group_id, block.block_id ) ) - if ( + if replace_existing_hashes: + removed_hashes = self._remove_cached_block_hashes(block) + self._emit_block_removed_events(removed_hashes) + already_cached = False + elif ( not already_cached and block.block_hash is not None and block.block_hash_num_tokens is not None diff --git a/vllm/v1/core/kv_cache_coordinator.py b/vllm/v1/core/kv_cache_coordinator.py index 34711d13cc..12f2df1f16 100644 --- a/vllm/v1/core/kv_cache_coordinator.py +++ b/vllm/v1/core/kv_cache_coordinator.py @@ -15,6 +15,7 @@ from vllm.v1.core.kv_cache_utils import ( ) from vllm.v1.core.single_type_kv_cache_manager import ( CrossAttentionManager, + MambaManager, SingleTypeKVCacheManager, get_manager_for_kv_cache_spec, ) @@ -151,6 +152,11 @@ class KVCacheCoordinator(ABC): ) for i, kv_cache_group in enumerate(self.kv_cache_config.kv_cache_groups) ) + # Match Mamba checkpoints to Eagle's attention replay boundary. + if use_eagle: + for manager in self.single_type_managers: + if isinstance(manager, MambaManager): + manager.drop_eagle_checkpoint_block = True # A positive retention interval must be a multiple of the base hit granularity # (``scheduler_block_size``) to land on real cache-hit boundaries. diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index dbc697a798..08cd21c099 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -54,7 +54,12 @@ from vllm.v1.core.sched.request_queue import ( ) from vllm.v1.core.sched.utils import check_stop, remove_all from vllm.v1.engine import EngineCoreEventType, EngineCoreOutput, EngineCoreOutputs -from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec +from vllm.v1.kv_cache_interface import ( + KVCacheConfig, + MambaSpec, + get_mamba_prefill_checkpoint_position, + is_mamba_prefill_checkpoint_valid, +) from vllm.v1.metrics.perf import ModelMetrics, PerfStats from vllm.v1.metrics.stats import ( PrefixCacheStats, @@ -334,15 +339,20 @@ class Scheduler(SchedulerInterface): self.need_mamba_block_aligned_split = ( self.has_mamba_layers and self.cache_config.mamba_cache_mode == "align" ) - self.mamba_has_prefill_checkpoint_blocks = ( - self.has_mamba_layers - # TODO: support spec decoding - and not self.use_eagle - and all( - not isinstance(group.kv_cache_spec, MambaSpec) - or group.kv_cache_spec.num_prefill_checkpoint_blocks > 0 + # TODO: Support models with multiple Mamba specs that require different + # prefill checkpoint alignments instead of selecting the first one. + self.mamba_prefill_checkpoint_alignment = next( + ( + group.kv_cache_spec.prefill_checkpoint_alignment for group in kv_cache_config.kv_cache_groups - ) + if isinstance(group.kv_cache_spec, MambaSpec) + ), + None, + ) + self.mamba_has_prefill_checkpoint_blocks = self.has_mamba_layers and all( + not isinstance(group.kv_cache_spec, MambaSpec) + or group.kv_cache_spec.num_prefill_checkpoint_blocks > 0 + for group in kv_cache_config.kv_cache_groups ) # A finer prefix_match_unit is configured: a mamba partial tail entry # can only be registered by a step ending exactly at the prompt's last @@ -426,8 +436,22 @@ class Scheduler(SchedulerInterface): last_cache_position = max(last_cache_position - block_size, 0) end = start + num_new_tokens + checkpoint_position = get_mamba_prefill_checkpoint_position( + prefill_end, + self.hash_block_size, + drop_eagle_block=self.use_eagle_block_drop, + ) use_internal_checkpoint = ( - self.mamba_has_prefill_checkpoint_blocks and start % block_size == 0 + self.mamba_has_prefill_checkpoint_blocks + and end >= prefill_end + and is_mamba_prefill_checkpoint_valid( + query_start=start, + query_end=end, + checkpoint_position=checkpoint_position, + hash_block_size=self.hash_block_size, + mamba_block_size=block_size, + checkpoint_alignment=self.mamba_prefill_checkpoint_alignment, + ) ) if use_internal_checkpoint: last_cache_position = 0 @@ -436,7 +460,7 @@ class Scheduler(SchedulerInterface): # aligned. Exempt: the prompt's last chunk, whose slot decode advances # to the boundary. A block too wide for one chunk advances sub-block # and re-aligns at the next boundary. - if end < prefill_end and not use_internal_checkpoint: + if end < prefill_end: max_prefill_tokens = self.max_num_scheduled_tokens long_prefill_threshold = self.scheduler_config.long_prefill_token_threshold if long_prefill_threshold > 0: @@ -448,13 +472,15 @@ class Scheduler(SchedulerInterface): next_block_boundary = (start // block_size + 1) * block_size tail_boundary = ( request.num_prompt_tokens // self.hash_block_size * self.hash_block_size - if self.mamba_partial_cache_hit + if self.mamba_partial_cache_hit and not use_internal_checkpoint else 0 ) stops = ( # Same invariant: a chunk starting mid-block stops at the boundary # rather than running past it. - next_block_boundary if start % block_size != 0 else 0, + next_block_boundary + if start % block_size != 0 and not use_internal_checkpoint + else 0, # Never run past the last cacheable block boundary mid-chunk. last_cache_position, # Fine-grained hits: the prompt's partial-tail entry can only be diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index 4fb17bc79c..45ef988059 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -31,6 +31,8 @@ from vllm.v1.kv_cache_interface import ( SinkFullAttentionSpec, SlidingWindowMLASpec, SlidingWindowSpec, + get_mamba_prefill_checkpoint_position, + is_mamba_prefill_checkpoint_valid, ) from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry from vllm.v1.request import Request @@ -1399,6 +1401,12 @@ class MambaManager(SingleTypeKVCacheManager): self.block_size = kv_cache_spec.block_size self.mamba_cache_mode = kv_cache_spec.mamba_cache_mode self.num_speculative_blocks: int = kv_cache_spec.num_speculative_blocks + self.has_prefill_checkpoint_blocks = ( + self.mamba_cache_mode == "align" + and kv_cache_spec.num_prefill_checkpoint_blocks > 0 + ) + # Mamba checkpoints follow Eagle's global replay boundary. + self.drop_eagle_checkpoint_block = False self.cached_blocks_this_step: set[BlockHashWithGroupId] = set() if self.mamba_cache_mode == "align": # Mapping from request ID to the index of the block @@ -1406,9 +1414,9 @@ class MambaManager(SingleTypeKVCacheManager): self.last_state_block_idx: dict[str, int] = {} # The set of the requests that have been allocated blocks self._allocated_block_reqs: set[str] = set() - # Number of internal checkpoint blocks required by each request's + # Absolute checkpoint position selected by the scheduler for the # current allocation. - self._num_checkpoint_blocks: dict[str, int] = {} + self._checkpoint_positions: dict[str, int] = {} # Requests that registered their own last-prompt-boundary partial # tail (producers). A later CoW hands its private copy to the # connector; a request that finishes first hands off this table @@ -1592,18 +1600,32 @@ class MambaManager(SingleTypeKVCacheManager): def _needs_internal_checkpoint( self, request_id: str, - num_tokens: int, - num_computed_tokens: int, + query_start: int, + query_end: int, + checkpoint_position: int, ) -> bool: assert isinstance(self.kv_cache_spec, MambaSpec) - checkpoint_idx = cdiv(num_tokens, self.block_size) - 2 + checkpoint_idx = cdiv(query_end, self.block_size) - 2 blocks = self.req_to_blocks[request_id] return ( - self.kv_cache_spec.num_prefill_checkpoint_blocks > 0 - and num_tokens % self.block_size != 0 - and num_computed_tokens % self.block_size == 0 + self.has_prefill_checkpoint_blocks + and is_mamba_prefill_checkpoint_valid( + query_start=query_start, + query_end=query_end, + checkpoint_position=checkpoint_position, + hash_block_size=self.block_pool.hash_block_size, + mamba_block_size=self.block_size, + checkpoint_alignment=(self.kv_cache_spec.prefill_checkpoint_alignment), + ) and checkpoint_idx >= 0 - and (checkpoint_idx >= len(blocks) or blocks[checkpoint_idx].is_null) + and ( + checkpoint_idx >= len(blocks) + or blocks[checkpoint_idx].is_null + or ( + request_id in self._allocated_block_reqs + and checkpoint_idx >= len(blocks) - self.num_speculative_blocks + ) + ) ) def get_num_blocks_to_allocate( @@ -1668,17 +1690,27 @@ class MambaManager(SingleTypeKVCacheManager): ) if has_partial_hit: num_new_blocks = max(num_new_blocks, 0) + 1 - checkpoint_block = int( - self._needs_internal_checkpoint( - request_id, num_tokens, total_computed_tokens - ) + checkpoint_position = get_mamba_prefill_checkpoint_position( + num_tokens, + self.block_pool.hash_block_size, + self.drop_eagle_checkpoint_block, ) + if not self._needs_internal_checkpoint( + request_id, + total_computed_tokens, + num_tokens, + checkpoint_position, + ): + checkpoint_position = 0 + checkpoint_block = int(checkpoint_position > 0) if not apply_admission_cap: - self._num_checkpoint_blocks[request_id] = checkpoint_block + self._checkpoint_positions[request_id] = checkpoint_position if num_new_blocks > 0: - num_new_blocks = 1 + int(has_partial_hit) + checkpoint_block - if request_id not in self._allocated_block_reqs: - num_new_blocks += self.num_speculative_blocks + blocks_allocated = request_id in self._allocated_block_reqs + if not (checkpoint_block and blocks_allocated): + num_new_blocks = 1 + int(has_partial_hit) + checkpoint_block + if not blocks_allocated: + num_new_blocks += self.num_speculative_blocks num_evictable_computed_blocks = self._get_num_evictable_blocks( new_computed_blocks @@ -1710,12 +1742,17 @@ class MambaManager(SingleTypeKVCacheManager): num_required_blocks = ( cdiv(num_tokens, self.block_size) + self.num_speculative_blocks ) - checkpoint_block = self._num_checkpoint_blocks.get(request_id, 0) + checkpoint_position = self._checkpoint_positions.get(request_id, 0) + checkpoint_block = int(checkpoint_position > 0) partial_hit = self._partial_hit_reqs.get(request_id) has_partial_hit = partial_hit is not None # `num_required_blocks` might be less than `len(req_blocks)` if blocks are # over-allocated at last round. - if num_required_blocks <= len(req_blocks) and not has_partial_hit: + if ( + num_required_blocks <= len(req_blocks) + and not has_partial_hit + and not checkpoint_block + ): self._allocated_block_reqs.add(request_id) return [] else: @@ -1745,7 +1782,7 @@ class MambaManager(SingleTypeKVCacheManager): [self._null_block for _ in range(prev_block_len, null_end)] ) - if blocks_allocated: + if blocks_allocated and not checkpoint_block: # Relocate exclusively owned speculative scratch blocks. for block_idx in range( prev_block_len - self.num_speculative_blocks, prev_block_len @@ -1754,11 +1791,11 @@ class MambaManager(SingleTypeKVCacheManager): self._relocate_speculative_block(req_blocks, block_idx) else: break - num_new_blocks = num_required_blocks - len(req_blocks) + num_new_blocks = max(num_required_blocks - len(req_blocks), 0) if has_partial_hit: num_new_blocks = max(num_new_blocks, 0) + 1 max_new_blocks = 1 + int(has_partial_hit) + checkpoint_block - if not blocks_allocated: + if not blocks_allocated or checkpoint_block: max_new_blocks += self.num_speculative_blocks assert num_new_blocks <= max_new_blocks new_blocks = self.block_pool.get_new_blocks(num_new_blocks) @@ -1838,7 +1875,7 @@ class MambaManager(SingleTypeKVCacheManager): if self.mamba_cache_mode == "align": self._allocated_block_reqs.discard(request_id) self.last_state_block_idx.pop(request_id, None) - self._num_checkpoint_blocks.pop(request_id, None) + self._checkpoint_positions.pop(request_id, None) self._producer_partial_tail_reqs.pop(request_id, None) # An offer is only guaranteed to hold committed bytes until the end # of the pass that made it. This request's blocks are going back to @@ -1885,6 +1922,7 @@ class MambaManager(SingleTypeKVCacheManager): continue self.cached_blocks_this_step.add(block.block_hash) if self.mamba_cache_mode == "align": + assert block.block_hash_num_tokens is not None # Offer every retained boundary with its exact block. # The connector filters against its save window, which may # extend past the original prompt during resumed prefill. @@ -1893,7 +1931,7 @@ class MambaManager(SingleTypeKVCacheManager): request.request_id, self.kv_cache_group_id, block, - (idx + 1) * self.block_size, + block.block_hash_num_tokens, ) ) @@ -1906,6 +1944,24 @@ class MambaManager(SingleTypeKVCacheManager): num_tokens: int, ) -> BlockHashWithGroupId | None: hash_block_size = self.block_pool.hash_block_size + # Re-key the reserved block at its exported checkpoint boundary. + checkpoint_position = self._checkpoint_positions.get(request.request_id, 0) + if checkpoint_position > 0: + # TODO: Store the reserved slot explicitly for multi-module MTP. + checkpoint_idx = cdiv(num_tokens, self.block_size) - 2 + blocks = self.req_to_blocks[request.request_id] + assert 0 <= checkpoint_idx < len(blocks) + checkpoint_block = blocks[checkpoint_idx] + if checkpoint_block.block_hash_num_tokens == checkpoint_position: + return None + return self.block_pool.cache_partial_block( + request=request, + block=checkpoint_block, + num_tokens=checkpoint_position, + kv_cache_group_id=self.kv_cache_group_id, + block_size=self.block_size, + replace_existing_hashes=True, + ) if self.block_size == hash_block_size: return None if num_tokens % self.block_size == 0: diff --git a/vllm/v1/kv_cache_interface.py b/vllm/v1/kv_cache_interface.py index bb9494d15c..1888404a57 100644 --- a/vllm/v1/kv_cache_interface.py +++ b/vllm/v1/kv_cache_interface.py @@ -890,6 +890,7 @@ class MambaSpec(KVCacheSpec): mamba_cache_mode: str = "none" num_speculative_blocks: int = 0 num_prefill_checkpoint_blocks: int = 0 + prefill_checkpoint_alignment: int | None = None num_heads: int = 1 tokens_per_state: int = -1 # False: the state is sharded across TP ranks (e.g. GDN). True: every TP @@ -950,12 +951,49 @@ class MambaSpec(KVCacheSpec): isinstance(spec, MambaSpec) and spec.num_speculative_blocks == self.num_speculative_blocks and spec.num_prefill_checkpoint_blocks == self.num_prefill_checkpoint_blocks + and spec.prefill_checkpoint_alignment == self.prefill_checkpoint_alignment and spec.page_size_bytes == self.page_size_bytes and spec.tp_replicated == self.tp_replicated for spec in kv_cache_specs.values() ) +def get_mamba_prefill_checkpoint_position( + num_tokens: int, + hash_block_size: int, + drop_eagle_block: bool, +) -> int: + """Return the reusable Mamba checkpoint boundary for a prefill.""" + checkpoint_position = (num_tokens - 1) // hash_block_size * hash_block_size + if drop_eagle_block: + checkpoint_position -= hash_block_size + return max(checkpoint_position, 0) + + +def is_mamba_prefill_checkpoint_valid( + query_start: int, + query_end: int, + checkpoint_position: int, + hash_block_size: int, + mamba_block_size: int, + checkpoint_alignment: int | None, +) -> bool: + """Whether a backend can export the checkpoint in this query.""" + if checkpoint_alignment is None: + return False + assert checkpoint_alignment > 0 + + initial_state_col = (query_start - 1) // mamba_block_size + checkpoint_col = cdiv(query_end, mamba_block_size) - 2 + return ( + query_start % hash_block_size == 0 + and checkpoint_col > initial_state_col + and query_start + hash_block_size <= checkpoint_position + and query_start < checkpoint_position < query_end + and (checkpoint_position - query_start) % checkpoint_alignment == 0 + ) + + @dataclass(frozen=True) class EncoderOnlyAttentionSpec(AttentionSpec): def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int: