[Bugfix][Core] Exempt exactly the blocks an async KV load writes from zeroing (#59504)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Yifan Qiao
2026-10-03 06:26:12 +00:00
committed by GitHub
co-authored by Claude Opus 5.5 Codex
parent 1a001d5842
commit ca0df1f122
17 changed files with 162 additions and 72 deletions
+5 -19
View File
@@ -353,9 +353,6 @@ class Scheduler(SchedulerInterface):
self.has_mamba_layers = kv_cache_config.has_mamba_layers
self.needs_kv_cache_zeroing = kv_cache_config.needs_kv_cache_zeroing
# Blocks that async KV loads will overwrite this step, skipped from
# zeroing since the zeroing could race the out-of-band write.
self._skip_zero_block_ids: set[int] = set()
self.need_mamba_block_aligned_split = (
self.has_mamba_layers and self.cache_config.mamba_cache_mode == "align"
)
@@ -1230,6 +1227,11 @@ class Scheduler(SchedulerInterface):
num_lookahead_tokens=effective_lookahead_tokens,
num_external_computed_tokens=num_external_computed_tokens,
delay_cache_blocks=load_kv_async,
skip_zeroing_group_ids=(
self.connector.get_loaded_kv_cache_group_ids(request)
if load_kv_async and self.connector is not None
else ()
),
num_encoder_tokens=num_encoder_tokens,
full_sequence_must_fit=self.scheduler_reserve_full_isl,
reserved_blocks=reserved_blocks,
@@ -1291,16 +1293,6 @@ class Scheduler(SchedulerInterface):
request.num_computed_tokens = num_computed_tokens
self._inflight_prefills.add(request)
self._set_kv_fetch_stage(request, KV_FETCH_IN_PROGRESS)
if self.needs_kv_cache_zeroing:
# Skip zeroing of the blocks the async load will
# overwrite; the zeroing could race the write.
self._skip_zero_block_ids.update(
self.kv_cache_manager.get_zeroing_block_ids_in_range(
request_id,
num_new_local_computed_tokens,
num_computed_tokens,
)
)
skip_request(request_queue)
continue
@@ -1545,12 +1537,6 @@ class Scheduler(SchedulerInterface):
new_block_ids_to_zero = self.kv_cache_manager.take_new_block_ids()
if not self.needs_kv_cache_zeroing:
return None
if self._skip_zero_block_ids:
skip = self._skip_zero_block_ids
new_block_ids_to_zero = [b for b in new_block_ids_to_zero if b not in skip]
skip.clear()
return new_block_ids_to_zero or None
def _preempt_request(