[Bugfix][Core] Fix mamba prefill checkpoint block reservation and prompt-end eviction in align mode (#59175)
Signed-off-by: kliuae <kuanfu.liu@embeddedllm.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -174,6 +174,7 @@ class KVCacheCoordinator(ABC):
|
||||
num_local_computed_tokens: int,
|
||||
num_tokens_main_model: int,
|
||||
apply_admission_cap: bool = False,
|
||||
prefill_end: int = 0,
|
||||
) -> int:
|
||||
"""Get the number of device blocks needed to be allocated for the request.
|
||||
|
||||
@@ -195,12 +196,19 @@ class KVCacheCoordinator(ABC):
|
||||
per-request admission cap (SWA / chunked-local). Set only by
|
||||
the full-sequence admission gate; per-step allocation must
|
||||
leave it False so the predictor matches `allocate_new_blocks`.
|
||||
prefill_end: The token index the request's prefill ends at, the
|
||||
same value the scheduler splits chunks against. Under sparse
|
||||
retention Mamba reserves a prefill checkpoint only on the chunk
|
||||
reaching it; under dense retention every chunk publishes a
|
||||
state, so it is ignored (0).
|
||||
|
||||
Returns:
|
||||
The number of blocks to allocate.
|
||||
|
||||
"""
|
||||
num_blocks_to_allocate = 0
|
||||
if self.retention_interval != 0:
|
||||
prefill_end = 0
|
||||
for i, manager in enumerate(self.single_type_managers):
|
||||
if isinstance(manager, CrossAttentionManager):
|
||||
# For cross-attention, we issue a single static allocation
|
||||
@@ -223,6 +231,7 @@ class KVCacheCoordinator(ABC):
|
||||
num_local_computed_tokens,
|
||||
num_tokens_main_model,
|
||||
apply_admission_cap=apply_admission_cap,
|
||||
prefill_end=prefill_end,
|
||||
)
|
||||
return num_blocks_to_allocate
|
||||
|
||||
|
||||
@@ -512,6 +512,10 @@ class KVCacheManager:
|
||||
):
|
||||
watermark_blocks = self.watermark_blocks
|
||||
|
||||
# Matches the scheduler's own prefill boundary: `num_tokens - 1`
|
||||
# extends it to resumed requests replaying their output tokens.
|
||||
prefill_end = max(request.num_prompt_tokens, request.num_tokens - 1)
|
||||
|
||||
if full_sequence_must_fit:
|
||||
# First check and fail if the full request sequence won't fit.
|
||||
full_num_tokens = min(request.num_tokens, self.max_model_len)
|
||||
@@ -525,6 +529,7 @@ class KVCacheManager:
|
||||
num_local_computed_tokens=num_local_computed_tokens,
|
||||
num_tokens_main_model=full_num_tokens,
|
||||
apply_admission_cap=True,
|
||||
prefill_end=prefill_end,
|
||||
)
|
||||
required_blocks = num_blocks_to_allocate + watermark_blocks
|
||||
if required_blocks > self.block_pool.get_num_free_blocks():
|
||||
@@ -559,6 +564,7 @@ class KVCacheManager:
|
||||
+ num_external_computed_tokens,
|
||||
num_local_computed_tokens=num_local_computed_tokens,
|
||||
num_tokens_main_model=num_tokens_main_model,
|
||||
prefill_end=prefill_end,
|
||||
)
|
||||
|
||||
# Keep `reserved_blocks` free for other in-flight sequences, and an
|
||||
|
||||
@@ -3003,6 +3003,7 @@ class Scheduler(SchedulerInterface):
|
||||
num_local_computed_tokens=request.num_computed_tokens,
|
||||
num_tokens_main_model=full_num_tokens,
|
||||
apply_admission_cap=True,
|
||||
prefill_end=max(request.num_prompt_tokens, request.num_tokens - 1),
|
||||
)
|
||||
return num_blocks + self._spec_decode_step_blocks()
|
||||
|
||||
|
||||
@@ -184,6 +184,7 @@ class SingleTypeKVCacheManager(ABC):
|
||||
num_local_computed_tokens: int,
|
||||
num_tokens_main_model: int,
|
||||
apply_admission_cap: bool = False,
|
||||
prefill_end: int = 0,
|
||||
) -> int:
|
||||
"""Get the number of blocks needed to be allocated for the request.
|
||||
|
||||
@@ -203,6 +204,10 @@ class SingleTypeKVCacheManager(ABC):
|
||||
apply_admission_cap: If True, clamp by `num_required_blocks` by
|
||||
`_max_admission_blocks_per_request`for recycling-aware specs
|
||||
(SWA, chunked-local).
|
||||
prefill_end: The token index the request's prefill ends at, the
|
||||
same value the scheduler splits chunks against. Mamba reserves
|
||||
a prefill checkpoint only on the chunk reaching it; 0 (dense
|
||||
retention) reserves one on every chunk.
|
||||
|
||||
Returns:
|
||||
The number of blocks to allocate.
|
||||
@@ -1215,6 +1220,7 @@ class CircularBufferManager(FullAttentionManager):
|
||||
num_local_computed_tokens: int,
|
||||
num_tokens_main_model: int,
|
||||
apply_admission_cap: bool = False,
|
||||
prefill_end: int = 0,
|
||||
) -> int:
|
||||
return 0 if self.req_to_blocks.get(request_id) else 1
|
||||
|
||||
@@ -1678,12 +1684,14 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
query_start: int,
|
||||
query_end: int,
|
||||
checkpoint_position: int,
|
||||
prefill_end: int,
|
||||
) -> bool:
|
||||
assert isinstance(self.kv_cache_spec, MambaSpec)
|
||||
checkpoint_idx = cdiv(query_end, self.block_size) - 2
|
||||
blocks = self.req_to_blocks[request_id]
|
||||
return (
|
||||
self.has_prefill_checkpoint_blocks
|
||||
and query_end >= prefill_end
|
||||
and is_mamba_prefill_checkpoint_valid(
|
||||
query_start=query_start,
|
||||
query_end=query_end,
|
||||
@@ -1712,6 +1720,7 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
num_local_computed_tokens: int,
|
||||
num_tokens_main_model: int,
|
||||
apply_admission_cap: bool = False,
|
||||
prefill_end: int = 0,
|
||||
) -> int:
|
||||
assert isinstance(self.kv_cache_spec, MambaSpec)
|
||||
if (
|
||||
@@ -1738,6 +1747,7 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
num_local_computed_tokens,
|
||||
num_tokens_main_model,
|
||||
apply_admission_cap=apply_admission_cap,
|
||||
prefill_end=prefill_end,
|
||||
)
|
||||
else:
|
||||
# We don't allocate blocks for lookahead tokens in align mode, because if
|
||||
@@ -1765,6 +1775,11 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
)
|
||||
if has_partial_hit:
|
||||
num_new_blocks = max(num_new_blocks, 0) + 1
|
||||
# Keyed on the chunk end like the worker's
|
||||
# `compute_mamba_prefill_checkpoints`, which writes the state there.
|
||||
# The helper returns a boundary strictly below its input, so every
|
||||
# chunk has one; under sparse retention only the prefill-end chunk's
|
||||
# is kept, so `prefill_end` skips reserving blocks for the others.
|
||||
checkpoint_position = get_mamba_prefill_checkpoint_position(
|
||||
num_tokens,
|
||||
self.block_pool.hash_block_size,
|
||||
@@ -1775,6 +1790,7 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
total_computed_tokens,
|
||||
num_tokens,
|
||||
checkpoint_position,
|
||||
prefill_end,
|
||||
):
|
||||
checkpoint_position = 0
|
||||
checkpoint_block = int(checkpoint_position > 0)
|
||||
@@ -2272,6 +2288,7 @@ class HiSparseSourceManager(FullAttentionManager):
|
||||
num_local_computed_tokens: int,
|
||||
num_tokens_main_model: int,
|
||||
apply_admission_cap: bool = False,
|
||||
prefill_end: int = 0,
|
||||
) -> int:
|
||||
required = super().get_num_blocks_to_allocate(
|
||||
request_id,
|
||||
@@ -2407,6 +2424,7 @@ class HiSparseHotManager(_HiSparseAuxiliaryManager):
|
||||
num_local_computed_tokens: int,
|
||||
num_tokens_main_model: int,
|
||||
apply_admission_cap: bool = False,
|
||||
prefill_end: int = 0,
|
||||
) -> int:
|
||||
assert not new_computed_blocks
|
||||
# A hot region is needed to read host-backed history: an external
|
||||
@@ -2479,6 +2497,7 @@ class HiSparseResidentManager(_HiSparseAuxiliaryManager):
|
||||
num_local_computed_tokens: int,
|
||||
num_tokens_main_model: int,
|
||||
apply_admission_cap: bool = False,
|
||||
prefill_end: int = 0,
|
||||
) -> int:
|
||||
del num_tokens_main_model
|
||||
assert not new_computed_blocks
|
||||
|
||||
Reference in New Issue
Block a user