[Bugfix][HiSparse] Fix a chunked-prefill preemption livelock (#59494)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
26bfdfb6bc
commit
3a69636645
@@ -112,7 +112,7 @@ class SingleTypeKVCacheManager(ABC):
|
||||
self.kv_cache_spec = kv_cache_spec
|
||||
self.block_pool = block_pool
|
||||
self.enable_caching = enable_caching
|
||||
self._max_admission_blocks_per_request = max_admission_blocks_per_request
|
||||
self.max_admission_blocks_per_request = max_admission_blocks_per_request
|
||||
# Record newly allocated block ids only when worker-side zeroing will
|
||||
# consume them and this manager holds a spec type that gets zeroed.
|
||||
self._record_new_block_ids = (
|
||||
@@ -202,7 +202,7 @@ class SingleTypeKVCacheManager(ABC):
|
||||
model in spec decode). w/o spec decode, it is num_tokens;
|
||||
with spec decode, it is num_tokens - num_lookahead_tokens.
|
||||
apply_admission_cap: If True, clamp by `num_required_blocks` by
|
||||
`_max_admission_blocks_per_request`for recycling-aware specs
|
||||
`max_admission_blocks_per_request`for recycling-aware specs
|
||||
(SWA, chunked-local).
|
||||
prefill_end: The token index the request's prefill ends at, the
|
||||
same value the scheduler splits chunks against. Mamba reserves
|
||||
@@ -214,7 +214,7 @@ class SingleTypeKVCacheManager(ABC):
|
||||
|
||||
"""
|
||||
num_required_blocks = cdiv(num_tokens, self.block_size)
|
||||
if apply_admission_cap and self._max_admission_blocks_per_request is not None:
|
||||
if apply_admission_cap and self.max_admission_blocks_per_request is not None:
|
||||
# Recycling-aware specs (SWA, chunked-local) cap the per-request
|
||||
# reservation here so admission matches the startup pool sizer
|
||||
# (`SlidingWindowSpec.max_admission_blocks_per_request` / its
|
||||
@@ -225,7 +225,7 @@ class SingleTypeKVCacheManager(ABC):
|
||||
# Drift between the two would re-introduce the deadlock from
|
||||
# issue #39734 or, worse, mid-prefill OOM.
|
||||
num_required_blocks = min(
|
||||
num_required_blocks, self._max_admission_blocks_per_request
|
||||
num_required_blocks, self.max_admission_blocks_per_request
|
||||
)
|
||||
num_req_blocks = len(self.req_to_blocks.get(request_id, ()))
|
||||
|
||||
@@ -2488,6 +2488,8 @@ class HiSparseHotManager(_HiSparseAuxiliaryManager):
|
||||
class HiSparseResidentManager(_HiSparseAuxiliaryManager):
|
||||
"""Track GPU-resident pages for otherwise host-backed KV."""
|
||||
|
||||
max_admission_blocks_per_request: int
|
||||
|
||||
def get_num_blocks_to_allocate(
|
||||
self,
|
||||
request_id: str,
|
||||
@@ -2514,8 +2516,7 @@ class HiSparseResidentManager(_HiSparseAuxiliaryManager):
|
||||
host_pages = cdiv(num_local_computed_tokens, self.block_size)
|
||||
required = cdiv(num_tokens, self.block_size)
|
||||
if apply_admission_cap:
|
||||
assert self._max_admission_blocks_per_request is not None
|
||||
required = min(required, self._max_admission_blocks_per_request)
|
||||
required = min(required, self.max_admission_blocks_per_request)
|
||||
return max(required - max(existing, host_pages), 0)
|
||||
|
||||
def allocate_external_computed_blocks(
|
||||
|
||||
Reference in New Issue
Block a user