[Bugfix][Core] Exempt exactly the blocks an async KV load writes from zeroing (#59504)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Yifan Qiao
2026-10-03 06:26:12 +00:00
committed by GitHub
co-authored by Claude Opus 5.5 Codex
parent 1a001d5842
commit ca0df1f122
17 changed files with 162 additions and 72 deletions
+5 -1
View File
@@ -241,6 +241,7 @@ class KVCacheCoordinator(ABC):
new_computed_blocks: tuple[Sequence[KVCacheBlock], ...],
num_local_computed_tokens: int,
num_external_computed_tokens: int,
skip_zeroing_group_ids: tuple[int, ...] = (),
) -> None:
"""Add the new computed blocks to the request. Optionally allocate new
blocks for external computed tokens (if any).
@@ -251,6 +252,8 @@ class KVCacheCoordinator(ABC):
prefix cache.
num_local_computed_tokens: The number of local computed tokens.
num_external_computed_tokens: The number of external computed tokens.
skip_zeroing_group_ids: Groups whose external-token blocks are
written by an async load and must not be zeroed.
"""
# A running request is already tracked in num_cached_block and won't
@@ -274,11 +277,12 @@ class KVCacheCoordinator(ABC):
num_external_computed_tokens,
)
if num_external_computed_tokens > 0:
for manager in self.single_type_managers:
for i, manager in enumerate(self.single_type_managers):
manager.allocate_external_computed_blocks(
request_id,
num_local_computed_tokens,
num_external_computed_tokens,
record_for_zeroing=i not in skip_zeroing_group_ids,
)
def allocate_new_blocks(
+4 -14
View File
@@ -381,6 +381,7 @@ class KVCacheManager:
full_sequence_must_fit: bool = False,
reserved_blocks: int = 0,
has_scheduled_reqs: bool = True,
skip_zeroing_group_ids: tuple[int, ...] = (),
) -> KVCacheBlocks | None:
"""Add slots for a request with new tokens to append.
@@ -413,6 +414,8 @@ class KVCacheManager:
blocks an already in-flight (prefilling) sequence is relying on.
has_scheduled_reqs: Whether any requests are already scheduled to run
this step, controls whether watermark is applied.
skip_zeroing_group_ids: Groups whose external-token blocks will be
written by an async load and must not be zeroed concurrently.
Blocks layout:
```
@@ -586,6 +589,7 @@ class KVCacheManager:
new_computed_blocks=new_computed_block_list,
num_local_computed_tokens=num_local_computed_tokens,
num_external_computed_tokens=num_external_computed_tokens,
skip_zeroing_group_ids=skip_zeroing_group_ids,
)
new_blocks = self.coordinator.allocate_new_blocks(
@@ -863,20 +867,6 @@ class KVCacheManager:
ids.extend(mgr.take_new_block_ids())
return ids
def get_zeroing_block_ids_in_range(
self, request_id: str, start_token: int, end_token: int
) -> list[int]:
"""The request's block ids covering [start_token, end_token), from
the groups whose new blocks are zeroed by the worker."""
ids: list[int] = []
for mgr in self.coordinator.single_type_managers:
if mgr.records_new_block_ids:
start_idx = start_token // mgr.block_size
end_idx = cdiv(end_token, mgr.block_size)
blocks = mgr.req_to_blocks[request_id]
ids.extend(blk.block_id for blk in blocks[start_idx:end_idx])
return ids
def record_blocks_for_zeroing(self, request_id: str, start_token: int) -> None:
"""Re-record the request's blocks from start_token onwards for
zeroing, e.g. blocks a failed async KV load left unwritten.
+5 -19
View File
@@ -353,9 +353,6 @@ class Scheduler(SchedulerInterface):
self.has_mamba_layers = kv_cache_config.has_mamba_layers
self.needs_kv_cache_zeroing = kv_cache_config.needs_kv_cache_zeroing
# Blocks that async KV loads will overwrite this step, skipped from
# zeroing since the zeroing could race the out-of-band write.
self._skip_zero_block_ids: set[int] = set()
self.need_mamba_block_aligned_split = (
self.has_mamba_layers and self.cache_config.mamba_cache_mode == "align"
)
@@ -1230,6 +1227,11 @@ class Scheduler(SchedulerInterface):
num_lookahead_tokens=effective_lookahead_tokens,
num_external_computed_tokens=num_external_computed_tokens,
delay_cache_blocks=load_kv_async,
skip_zeroing_group_ids=(
self.connector.get_loaded_kv_cache_group_ids(request)
if load_kv_async and self.connector is not None
else ()
),
num_encoder_tokens=num_encoder_tokens,
full_sequence_must_fit=self.scheduler_reserve_full_isl,
reserved_blocks=reserved_blocks,
@@ -1291,16 +1293,6 @@ class Scheduler(SchedulerInterface):
request.num_computed_tokens = num_computed_tokens
self._inflight_prefills.add(request)
self._set_kv_fetch_stage(request, KV_FETCH_IN_PROGRESS)
if self.needs_kv_cache_zeroing:
# Skip zeroing of the blocks the async load will
# overwrite; the zeroing could race the write.
self._skip_zero_block_ids.update(
self.kv_cache_manager.get_zeroing_block_ids_in_range(
request_id,
num_new_local_computed_tokens,
num_computed_tokens,
)
)
skip_request(request_queue)
continue
@@ -1545,12 +1537,6 @@ class Scheduler(SchedulerInterface):
new_block_ids_to_zero = self.kv_cache_manager.take_new_block_ids()
if not self.needs_kv_cache_zeroing:
return None
if self._skip_zero_block_ids:
skip = self._skip_zero_block_ids
new_block_ids_to_zero = [b for b in new_block_ids_to_zero if b not in skip]
skip.clear()
return new_block_ids_to_zero or None
def _preempt_request(
+18 -5
View File
@@ -331,6 +331,7 @@ class SingleTypeKVCacheManager(ABC):
request_id: str,
num_local_computed_tokens: int,
num_external_computed_tokens: int,
record_for_zeroing: bool = True,
) -> None:
"""Allocate new blocks for external (KV-connector) computed tokens.
@@ -342,6 +343,8 @@ class SingleTypeKVCacheManager(ABC):
request_id: The request ID.
num_local_computed_tokens: The number of local computed tokens.
num_external_computed_tokens: The number of external computed tokens.
record_for_zeroing: Whether the new blocks need zeroing. False when
the load writes them after this step, which zeroing would race.
"""
num_total_computed_tokens = (
@@ -363,7 +366,7 @@ class SingleTypeKVCacheManager(ABC):
)
allocated_blocks = self.block_pool.get_new_blocks(num_new_blocks)
req_blocks.extend(allocated_blocks)
if self._record_new_block_ids:
if self._record_new_block_ids and record_for_zeroing:
self.new_block_ids.extend(b.block_id for b in allocated_blocks)
def allocate_new_blocks(
@@ -1201,13 +1204,15 @@ class CircularBufferManager(FullAttentionManager):
supports_fine_grained_hash_lookup: ClassVar[bool] = False
def _claim_ring_block(self, request_id: str) -> list[KVCacheBlock]:
def _claim_ring_block(
self, request_id: str, record_for_zeroing: bool = True
) -> list[KVCacheBlock]:
req_blocks = self.req_to_blocks[request_id]
if req_blocks:
return []
new_blocks = self.block_pool.get_new_blocks(1)
req_blocks.extend(new_blocks)
if self._record_new_block_ids:
if self._record_new_block_ids and record_for_zeroing:
self.new_block_ids.extend(block.block_id for block in new_blocks)
return new_blocks
@@ -1234,8 +1239,9 @@ class CircularBufferManager(FullAttentionManager):
request_id: str,
num_local_computed_tokens: int,
num_external_computed_tokens: int,
record_for_zeroing: bool = True,
) -> None:
self._claim_ring_block(request_id)
self._claim_ring_block(request_id, record_for_zeroing)
@classmethod
def find_longest_cache_hit(
@@ -2166,6 +2172,7 @@ class CrossAttentionManager(SingleTypeKVCacheManager):
request_id: str,
num_local_computed_tokens: int,
num_external_computed_tokens: int,
record_for_zeroing: bool = True,
) -> None:
# Cross-attention does not use prefix caching / external KV loads.
return
@@ -2311,6 +2318,7 @@ class HiSparseSourceManager(FullAttentionManager):
request_id: str,
num_local_computed_tokens: int,
num_external_computed_tokens: int,
record_for_zeroing: bool = True,
) -> None:
if num_external_computed_tokens <= 0:
return
@@ -2321,7 +2329,10 @@ class HiSparseSourceManager(FullAttentionManager):
request_id, num_local_computed_tokens + num_external_computed_tokens
)
super().allocate_external_computed_blocks(
request_id, num_local_computed_tokens, num_external_computed_tokens
request_id,
num_local_computed_tokens,
num_external_computed_tokens,
record_for_zeroing,
)
def cache_blocks(
@@ -2456,6 +2467,7 @@ class HiSparseHotManager(_HiSparseAuxiliaryManager):
request_id: str,
num_local_computed_tokens: int,
num_external_computed_tokens: int,
record_for_zeroing: bool = True,
) -> None:
self.require_hot(request_id)
@@ -2518,6 +2530,7 @@ class HiSparseResidentManager(_HiSparseAuxiliaryManager):
request_id: str,
num_local_computed_tokens: int,
num_external_computed_tokens: int,
record_for_zeroing: bool = True,
) -> None:
"""Reserve a writable final page; earlier imported pages stay on host."""
assert num_external_computed_tokens > 0