[Bugfix] Fix failure to allocate KV blocks error (#41282)

Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
This commit is contained in:
Wei Zhao
2026-04-29 18:44:07 -07:00
committed by GitHub
parent 0ab67c0222
commit 0ff1bf9bb1
4 changed files with 69 additions and 2 deletions
+12 -1
View File
@@ -85,6 +85,7 @@ class KVCacheCoordinator(ABC):
num_encoder_tokens: int,
total_computed_tokens: int,
num_tokens_main_model: int,
apply_admission_cap: bool = False,
) -> int:
"""
Get the number of blocks needed to be allocated for the request.
@@ -101,6 +102,10 @@ class KVCacheCoordinator(ABC):
num_tokens_main_model: The number of tokens for the main model (aka target
model in spec decode). w/o spec decode, it is num_tokens;
with spec decode, it is num_tokens - num_lookahead_tokens.
apply_admission_cap: If True, apply the recycling-aware
per-request admission cap (SWA / chunked-local). Set only by
the full-sequence admission gate; per-step allocation must
leave it False so the predictor matches `allocate_new_blocks`.
Returns:
The number of blocks to allocate.
@@ -111,7 +116,12 @@ class KVCacheCoordinator(ABC):
# For cross-attention, we issue a single static allocation
# of blocks based on the number of encoder input tokens.
num_blocks_to_allocate += manager.get_num_blocks_to_allocate(
request_id, num_encoder_tokens, [], 0, num_encoder_tokens
request_id,
num_encoder_tokens,
[],
0,
num_encoder_tokens,
apply_admission_cap=apply_admission_cap,
)
else:
num_blocks_to_allocate += manager.get_num_blocks_to_allocate(
@@ -120,6 +130,7 @@ class KVCacheCoordinator(ABC):
new_computed_blocks[i],
total_computed_tokens,
num_tokens_main_model,
apply_admission_cap=apply_admission_cap,
)
return num_blocks_to_allocate
+1
View File
@@ -257,6 +257,7 @@ class KVCacheManager:
num_encoder_tokens=num_encoder_tokens,
total_computed_tokens=total_computed_tokens,
num_tokens_main_model=full_num_tokens,
apply_admission_cap=True,
)
return num_blocks_to_allocate <= self.block_pool.get_num_free_blocks()
+7 -1
View File
@@ -92,6 +92,7 @@ class SingleTypeKVCacheManager(ABC):
new_computed_blocks: Sequence[KVCacheBlock],
total_computed_tokens: int,
num_tokens_main_model: int,
apply_admission_cap: bool = False,
) -> int:
"""
Get the number of blocks needed to be allocated for the request.
@@ -107,13 +108,16 @@ class SingleTypeKVCacheManager(ABC):
num_tokens_main_model: The number of tokens for the main model (aka target
model in spec decode). w/o spec decode, it is num_tokens;
with spec decode, it is num_tokens - num_lookahead_tokens.
apply_admission_cap: If True, clamp by `num_required_blocks` by
`_max_admission_blocks_per_request`for recycling-aware specs
(SWA, chunked-local).
Returns:
The number of blocks to allocate.
"""
num_required_blocks = cdiv(num_tokens, self.block_size)
if self._max_admission_blocks_per_request is not None:
if apply_admission_cap and self._max_admission_blocks_per_request is not None:
# Recycling-aware specs (SWA, chunked-local) cap the per-request
# reservation here so admission matches the startup pool sizer
# (`SlidingWindowSpec.max_admission_blocks_per_request` / its
@@ -893,6 +897,7 @@ class MambaManager(SingleTypeKVCacheManager):
new_computed_blocks: Sequence[KVCacheBlock],
total_computed_tokens: int,
num_tokens_main_model: int,
apply_admission_cap: bool = False,
) -> int:
assert isinstance(self.kv_cache_spec, MambaSpec)
if (
@@ -917,6 +922,7 @@ class MambaManager(SingleTypeKVCacheManager):
new_computed_blocks,
total_computed_tokens,
num_tokens_main_model,
apply_admission_cap=apply_admission_cap,
)
else:
# We don't allocate blocks for lookahead tokens in align mode, because if