[Bugfix][Core] Exempt exactly the blocks an async KV load writes from zeroing (#59504)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Codex
parent
1a001d5842
commit
ca0df1f122
@@ -241,6 +241,7 @@ class KVCacheCoordinator(ABC):
|
||||
new_computed_blocks: tuple[Sequence[KVCacheBlock], ...],
|
||||
num_local_computed_tokens: int,
|
||||
num_external_computed_tokens: int,
|
||||
skip_zeroing_group_ids: tuple[int, ...] = (),
|
||||
) -> None:
|
||||
"""Add the new computed blocks to the request. Optionally allocate new
|
||||
blocks for external computed tokens (if any).
|
||||
@@ -251,6 +252,8 @@ class KVCacheCoordinator(ABC):
|
||||
prefix cache.
|
||||
num_local_computed_tokens: The number of local computed tokens.
|
||||
num_external_computed_tokens: The number of external computed tokens.
|
||||
skip_zeroing_group_ids: Groups whose external-token blocks are
|
||||
written by an async load and must not be zeroed.
|
||||
|
||||
"""
|
||||
# A running request is already tracked in num_cached_block and won't
|
||||
@@ -274,11 +277,12 @@ class KVCacheCoordinator(ABC):
|
||||
num_external_computed_tokens,
|
||||
)
|
||||
if num_external_computed_tokens > 0:
|
||||
for manager in self.single_type_managers:
|
||||
for i, manager in enumerate(self.single_type_managers):
|
||||
manager.allocate_external_computed_blocks(
|
||||
request_id,
|
||||
num_local_computed_tokens,
|
||||
num_external_computed_tokens,
|
||||
record_for_zeroing=i not in skip_zeroing_group_ids,
|
||||
)
|
||||
|
||||
def allocate_new_blocks(
|
||||
|
||||
@@ -381,6 +381,7 @@ class KVCacheManager:
|
||||
full_sequence_must_fit: bool = False,
|
||||
reserved_blocks: int = 0,
|
||||
has_scheduled_reqs: bool = True,
|
||||
skip_zeroing_group_ids: tuple[int, ...] = (),
|
||||
) -> KVCacheBlocks | None:
|
||||
"""Add slots for a request with new tokens to append.
|
||||
|
||||
@@ -413,6 +414,8 @@ class KVCacheManager:
|
||||
blocks an already in-flight (prefilling) sequence is relying on.
|
||||
has_scheduled_reqs: Whether any requests are already scheduled to run
|
||||
this step, controls whether watermark is applied.
|
||||
skip_zeroing_group_ids: Groups whose external-token blocks will be
|
||||
written by an async load and must not be zeroed concurrently.
|
||||
|
||||
Blocks layout:
|
||||
```
|
||||
@@ -586,6 +589,7 @@ class KVCacheManager:
|
||||
new_computed_blocks=new_computed_block_list,
|
||||
num_local_computed_tokens=num_local_computed_tokens,
|
||||
num_external_computed_tokens=num_external_computed_tokens,
|
||||
skip_zeroing_group_ids=skip_zeroing_group_ids,
|
||||
)
|
||||
|
||||
new_blocks = self.coordinator.allocate_new_blocks(
|
||||
@@ -863,20 +867,6 @@ class KVCacheManager:
|
||||
ids.extend(mgr.take_new_block_ids())
|
||||
return ids
|
||||
|
||||
def get_zeroing_block_ids_in_range(
|
||||
self, request_id: str, start_token: int, end_token: int
|
||||
) -> list[int]:
|
||||
"""The request's block ids covering [start_token, end_token), from
|
||||
the groups whose new blocks are zeroed by the worker."""
|
||||
ids: list[int] = []
|
||||
for mgr in self.coordinator.single_type_managers:
|
||||
if mgr.records_new_block_ids:
|
||||
start_idx = start_token // mgr.block_size
|
||||
end_idx = cdiv(end_token, mgr.block_size)
|
||||
blocks = mgr.req_to_blocks[request_id]
|
||||
ids.extend(blk.block_id for blk in blocks[start_idx:end_idx])
|
||||
return ids
|
||||
|
||||
def record_blocks_for_zeroing(self, request_id: str, start_token: int) -> None:
|
||||
"""Re-record the request's blocks from start_token onwards for
|
||||
zeroing, e.g. blocks a failed async KV load left unwritten.
|
||||
|
||||
@@ -353,9 +353,6 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
self.has_mamba_layers = kv_cache_config.has_mamba_layers
|
||||
self.needs_kv_cache_zeroing = kv_cache_config.needs_kv_cache_zeroing
|
||||
# Blocks that async KV loads will overwrite this step, skipped from
|
||||
# zeroing since the zeroing could race the out-of-band write.
|
||||
self._skip_zero_block_ids: set[int] = set()
|
||||
self.need_mamba_block_aligned_split = (
|
||||
self.has_mamba_layers and self.cache_config.mamba_cache_mode == "align"
|
||||
)
|
||||
@@ -1230,6 +1227,11 @@ class Scheduler(SchedulerInterface):
|
||||
num_lookahead_tokens=effective_lookahead_tokens,
|
||||
num_external_computed_tokens=num_external_computed_tokens,
|
||||
delay_cache_blocks=load_kv_async,
|
||||
skip_zeroing_group_ids=(
|
||||
self.connector.get_loaded_kv_cache_group_ids(request)
|
||||
if load_kv_async and self.connector is not None
|
||||
else ()
|
||||
),
|
||||
num_encoder_tokens=num_encoder_tokens,
|
||||
full_sequence_must_fit=self.scheduler_reserve_full_isl,
|
||||
reserved_blocks=reserved_blocks,
|
||||
@@ -1291,16 +1293,6 @@ class Scheduler(SchedulerInterface):
|
||||
request.num_computed_tokens = num_computed_tokens
|
||||
self._inflight_prefills.add(request)
|
||||
self._set_kv_fetch_stage(request, KV_FETCH_IN_PROGRESS)
|
||||
if self.needs_kv_cache_zeroing:
|
||||
# Skip zeroing of the blocks the async load will
|
||||
# overwrite; the zeroing could race the write.
|
||||
self._skip_zero_block_ids.update(
|
||||
self.kv_cache_manager.get_zeroing_block_ids_in_range(
|
||||
request_id,
|
||||
num_new_local_computed_tokens,
|
||||
num_computed_tokens,
|
||||
)
|
||||
)
|
||||
skip_request(request_queue)
|
||||
continue
|
||||
|
||||
@@ -1545,12 +1537,6 @@ class Scheduler(SchedulerInterface):
|
||||
new_block_ids_to_zero = self.kv_cache_manager.take_new_block_ids()
|
||||
if not self.needs_kv_cache_zeroing:
|
||||
return None
|
||||
|
||||
if self._skip_zero_block_ids:
|
||||
skip = self._skip_zero_block_ids
|
||||
new_block_ids_to_zero = [b for b in new_block_ids_to_zero if b not in skip]
|
||||
skip.clear()
|
||||
|
||||
return new_block_ids_to_zero or None
|
||||
|
||||
def _preempt_request(
|
||||
|
||||
@@ -331,6 +331,7 @@ class SingleTypeKVCacheManager(ABC):
|
||||
request_id: str,
|
||||
num_local_computed_tokens: int,
|
||||
num_external_computed_tokens: int,
|
||||
record_for_zeroing: bool = True,
|
||||
) -> None:
|
||||
"""Allocate new blocks for external (KV-connector) computed tokens.
|
||||
|
||||
@@ -342,6 +343,8 @@ class SingleTypeKVCacheManager(ABC):
|
||||
request_id: The request ID.
|
||||
num_local_computed_tokens: The number of local computed tokens.
|
||||
num_external_computed_tokens: The number of external computed tokens.
|
||||
record_for_zeroing: Whether the new blocks need zeroing. False when
|
||||
the load writes them after this step, which zeroing would race.
|
||||
|
||||
"""
|
||||
num_total_computed_tokens = (
|
||||
@@ -363,7 +366,7 @@ class SingleTypeKVCacheManager(ABC):
|
||||
)
|
||||
allocated_blocks = self.block_pool.get_new_blocks(num_new_blocks)
|
||||
req_blocks.extend(allocated_blocks)
|
||||
if self._record_new_block_ids:
|
||||
if self._record_new_block_ids and record_for_zeroing:
|
||||
self.new_block_ids.extend(b.block_id for b in allocated_blocks)
|
||||
|
||||
def allocate_new_blocks(
|
||||
@@ -1201,13 +1204,15 @@ class CircularBufferManager(FullAttentionManager):
|
||||
|
||||
supports_fine_grained_hash_lookup: ClassVar[bool] = False
|
||||
|
||||
def _claim_ring_block(self, request_id: str) -> list[KVCacheBlock]:
|
||||
def _claim_ring_block(
|
||||
self, request_id: str, record_for_zeroing: bool = True
|
||||
) -> list[KVCacheBlock]:
|
||||
req_blocks = self.req_to_blocks[request_id]
|
||||
if req_blocks:
|
||||
return []
|
||||
new_blocks = self.block_pool.get_new_blocks(1)
|
||||
req_blocks.extend(new_blocks)
|
||||
if self._record_new_block_ids:
|
||||
if self._record_new_block_ids and record_for_zeroing:
|
||||
self.new_block_ids.extend(block.block_id for block in new_blocks)
|
||||
return new_blocks
|
||||
|
||||
@@ -1234,8 +1239,9 @@ class CircularBufferManager(FullAttentionManager):
|
||||
request_id: str,
|
||||
num_local_computed_tokens: int,
|
||||
num_external_computed_tokens: int,
|
||||
record_for_zeroing: bool = True,
|
||||
) -> None:
|
||||
self._claim_ring_block(request_id)
|
||||
self._claim_ring_block(request_id, record_for_zeroing)
|
||||
|
||||
@classmethod
|
||||
def find_longest_cache_hit(
|
||||
@@ -2166,6 +2172,7 @@ class CrossAttentionManager(SingleTypeKVCacheManager):
|
||||
request_id: str,
|
||||
num_local_computed_tokens: int,
|
||||
num_external_computed_tokens: int,
|
||||
record_for_zeroing: bool = True,
|
||||
) -> None:
|
||||
# Cross-attention does not use prefix caching / external KV loads.
|
||||
return
|
||||
@@ -2311,6 +2318,7 @@ class HiSparseSourceManager(FullAttentionManager):
|
||||
request_id: str,
|
||||
num_local_computed_tokens: int,
|
||||
num_external_computed_tokens: int,
|
||||
record_for_zeroing: bool = True,
|
||||
) -> None:
|
||||
if num_external_computed_tokens <= 0:
|
||||
return
|
||||
@@ -2321,7 +2329,10 @@ class HiSparseSourceManager(FullAttentionManager):
|
||||
request_id, num_local_computed_tokens + num_external_computed_tokens
|
||||
)
|
||||
super().allocate_external_computed_blocks(
|
||||
request_id, num_local_computed_tokens, num_external_computed_tokens
|
||||
request_id,
|
||||
num_local_computed_tokens,
|
||||
num_external_computed_tokens,
|
||||
record_for_zeroing,
|
||||
)
|
||||
|
||||
def cache_blocks(
|
||||
@@ -2456,6 +2467,7 @@ class HiSparseHotManager(_HiSparseAuxiliaryManager):
|
||||
request_id: str,
|
||||
num_local_computed_tokens: int,
|
||||
num_external_computed_tokens: int,
|
||||
record_for_zeroing: bool = True,
|
||||
) -> None:
|
||||
self.require_hot(request_id)
|
||||
|
||||
@@ -2518,6 +2530,7 @@ class HiSparseResidentManager(_HiSparseAuxiliaryManager):
|
||||
request_id: str,
|
||||
num_local_computed_tokens: int,
|
||||
num_external_computed_tokens: int,
|
||||
record_for_zeroing: bool = True,
|
||||
) -> None:
|
||||
"""Reserve a writable final page; earlier imported pages stay on host."""
|
||||
assert num_external_computed_tokens > 0
|
||||
|
||||
Reference in New Issue
Block a user