[Bugfix][Core] Fix mamba prefill checkpoint block reservation and prompt-end eviction in align mode (#59175)

Signed-off-by: kliuae <kuanfu.liu@embeddedllm.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
kliuae
2026-09-30 21:26:54 +00:00
committed by GitHub
co-authored by Nick Hill
parent 42f0c17ea7
commit 7e583e615c
5 changed files with 182 additions and 0 deletions
+9
View File
@@ -174,6 +174,7 @@ class KVCacheCoordinator(ABC):
num_local_computed_tokens: int,
num_tokens_main_model: int,
apply_admission_cap: bool = False,
prefill_end: int = 0,
) -> int:
"""Get the number of device blocks needed to be allocated for the request.
@@ -195,12 +196,19 @@ class KVCacheCoordinator(ABC):
per-request admission cap (SWA / chunked-local). Set only by
the full-sequence admission gate; per-step allocation must
leave it False so the predictor matches `allocate_new_blocks`.
prefill_end: The token index the request's prefill ends at, the
same value the scheduler splits chunks against. Under sparse
retention Mamba reserves a prefill checkpoint only on the chunk
reaching it; under dense retention every chunk publishes a
state, so it is ignored (0).
Returns:
The number of blocks to allocate.
"""
num_blocks_to_allocate = 0
if self.retention_interval != 0:
prefill_end = 0
for i, manager in enumerate(self.single_type_managers):
if isinstance(manager, CrossAttentionManager):
# For cross-attention, we issue a single static allocation
@@ -223,6 +231,7 @@ class KVCacheCoordinator(ABC):
num_local_computed_tokens,
num_tokens_main_model,
apply_admission_cap=apply_admission_cap,
prefill_end=prefill_end,
)
return num_blocks_to_allocate
+6
View File
@@ -512,6 +512,10 @@ class KVCacheManager:
):
watermark_blocks = self.watermark_blocks
# Matches the scheduler's own prefill boundary: `num_tokens - 1`
# extends it to resumed requests replaying their output tokens.
prefill_end = max(request.num_prompt_tokens, request.num_tokens - 1)
if full_sequence_must_fit:
# First check and fail if the full request sequence won't fit.
full_num_tokens = min(request.num_tokens, self.max_model_len)
@@ -525,6 +529,7 @@ class KVCacheManager:
num_local_computed_tokens=num_local_computed_tokens,
num_tokens_main_model=full_num_tokens,
apply_admission_cap=True,
prefill_end=prefill_end,
)
required_blocks = num_blocks_to_allocate + watermark_blocks
if required_blocks > self.block_pool.get_num_free_blocks():
@@ -559,6 +564,7 @@ class KVCacheManager:
+ num_external_computed_tokens,
num_local_computed_tokens=num_local_computed_tokens,
num_tokens_main_model=num_tokens_main_model,
prefill_end=prefill_end,
)
# Keep `reserved_blocks` free for other in-flight sequences, and an
+1
View File
@@ -3003,6 +3003,7 @@ class Scheduler(SchedulerInterface):
num_local_computed_tokens=request.num_computed_tokens,
num_tokens_main_model=full_num_tokens,
apply_admission_cap=True,
prefill_end=max(request.num_prompt_tokens, request.num_tokens - 1),
)
return num_blocks + self._spec_decode_step_blocks()
@@ -184,6 +184,7 @@ class SingleTypeKVCacheManager(ABC):
num_local_computed_tokens: int,
num_tokens_main_model: int,
apply_admission_cap: bool = False,
prefill_end: int = 0,
) -> int:
"""Get the number of blocks needed to be allocated for the request.
@@ -203,6 +204,10 @@ class SingleTypeKVCacheManager(ABC):
apply_admission_cap: If True, clamp by `num_required_blocks` by
`_max_admission_blocks_per_request`for recycling-aware specs
(SWA, chunked-local).
prefill_end: The token index the request's prefill ends at, the
same value the scheduler splits chunks against. Mamba reserves
a prefill checkpoint only on the chunk reaching it; 0 (dense
retention) reserves one on every chunk.
Returns:
The number of blocks to allocate.
@@ -1215,6 +1220,7 @@ class CircularBufferManager(FullAttentionManager):
num_local_computed_tokens: int,
num_tokens_main_model: int,
apply_admission_cap: bool = False,
prefill_end: int = 0,
) -> int:
return 0 if self.req_to_blocks.get(request_id) else 1
@@ -1678,12 +1684,14 @@ class MambaManager(SingleTypeKVCacheManager):
query_start: int,
query_end: int,
checkpoint_position: int,
prefill_end: int,
) -> bool:
assert isinstance(self.kv_cache_spec, MambaSpec)
checkpoint_idx = cdiv(query_end, self.block_size) - 2
blocks = self.req_to_blocks[request_id]
return (
self.has_prefill_checkpoint_blocks
and query_end >= prefill_end
and is_mamba_prefill_checkpoint_valid(
query_start=query_start,
query_end=query_end,
@@ -1712,6 +1720,7 @@ class MambaManager(SingleTypeKVCacheManager):
num_local_computed_tokens: int,
num_tokens_main_model: int,
apply_admission_cap: bool = False,
prefill_end: int = 0,
) -> int:
assert isinstance(self.kv_cache_spec, MambaSpec)
if (
@@ -1738,6 +1747,7 @@ class MambaManager(SingleTypeKVCacheManager):
num_local_computed_tokens,
num_tokens_main_model,
apply_admission_cap=apply_admission_cap,
prefill_end=prefill_end,
)
else:
# We don't allocate blocks for lookahead tokens in align mode, because if
@@ -1765,6 +1775,11 @@ class MambaManager(SingleTypeKVCacheManager):
)
if has_partial_hit:
num_new_blocks = max(num_new_blocks, 0) + 1
# Keyed on the chunk end like the worker's
# `compute_mamba_prefill_checkpoints`, which writes the state there.
# The helper returns a boundary strictly below its input, so every
# chunk has one; under sparse retention only the prefill-end chunk's
# is kept, so `prefill_end` skips reserving blocks for the others.
checkpoint_position = get_mamba_prefill_checkpoint_position(
num_tokens,
self.block_pool.hash_block_size,
@@ -1775,6 +1790,7 @@ class MambaManager(SingleTypeKVCacheManager):
total_computed_tokens,
num_tokens,
checkpoint_position,
prefill_end,
):
checkpoint_position = 0
checkpoint_block = int(checkpoint_position > 0)
@@ -2272,6 +2288,7 @@ class HiSparseSourceManager(FullAttentionManager):
num_local_computed_tokens: int,
num_tokens_main_model: int,
apply_admission_cap: bool = False,
prefill_end: int = 0,
) -> int:
required = super().get_num_blocks_to_allocate(
request_id,
@@ -2407,6 +2424,7 @@ class HiSparseHotManager(_HiSparseAuxiliaryManager):
num_local_computed_tokens: int,
num_tokens_main_model: int,
apply_admission_cap: bool = False,
prefill_end: int = 0,
) -> int:
assert not new_computed_blocks
# A hot region is needed to read host-backed history: an external
@@ -2479,6 +2497,7 @@ class HiSparseResidentManager(_HiSparseAuxiliaryManager):
num_local_computed_tokens: int,
num_tokens_main_model: int,
apply_admission_cap: bool = False,
prefill_end: int = 0,
) -> int:
del num_tokens_main_model
assert not new_computed_blocks