[Bugfix][Core] Exempt exactly the blocks an async KV load writes from zeroing (#59504)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Codex
parent
1a001d5842
commit
ca0df1f122
@@ -353,9 +353,6 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
self.has_mamba_layers = kv_cache_config.has_mamba_layers
|
||||
self.needs_kv_cache_zeroing = kv_cache_config.needs_kv_cache_zeroing
|
||||
# Blocks that async KV loads will overwrite this step, skipped from
|
||||
# zeroing since the zeroing could race the out-of-band write.
|
||||
self._skip_zero_block_ids: set[int] = set()
|
||||
self.need_mamba_block_aligned_split = (
|
||||
self.has_mamba_layers and self.cache_config.mamba_cache_mode == "align"
|
||||
)
|
||||
@@ -1230,6 +1227,11 @@ class Scheduler(SchedulerInterface):
|
||||
num_lookahead_tokens=effective_lookahead_tokens,
|
||||
num_external_computed_tokens=num_external_computed_tokens,
|
||||
delay_cache_blocks=load_kv_async,
|
||||
skip_zeroing_group_ids=(
|
||||
self.connector.get_loaded_kv_cache_group_ids(request)
|
||||
if load_kv_async and self.connector is not None
|
||||
else ()
|
||||
),
|
||||
num_encoder_tokens=num_encoder_tokens,
|
||||
full_sequence_must_fit=self.scheduler_reserve_full_isl,
|
||||
reserved_blocks=reserved_blocks,
|
||||
@@ -1291,16 +1293,6 @@ class Scheduler(SchedulerInterface):
|
||||
request.num_computed_tokens = num_computed_tokens
|
||||
self._inflight_prefills.add(request)
|
||||
self._set_kv_fetch_stage(request, KV_FETCH_IN_PROGRESS)
|
||||
if self.needs_kv_cache_zeroing:
|
||||
# Skip zeroing of the blocks the async load will
|
||||
# overwrite; the zeroing could race the write.
|
||||
self._skip_zero_block_ids.update(
|
||||
self.kv_cache_manager.get_zeroing_block_ids_in_range(
|
||||
request_id,
|
||||
num_new_local_computed_tokens,
|
||||
num_computed_tokens,
|
||||
)
|
||||
)
|
||||
skip_request(request_queue)
|
||||
continue
|
||||
|
||||
@@ -1545,12 +1537,6 @@ class Scheduler(SchedulerInterface):
|
||||
new_block_ids_to_zero = self.kv_cache_manager.take_new_block_ids()
|
||||
if not self.needs_kv_cache_zeroing:
|
||||
return None
|
||||
|
||||
if self._skip_zero_block_ids:
|
||||
skip = self._skip_zero_block_ids
|
||||
new_block_ids_to_zero = [b for b in new_block_ids_to_zero if b not in skip]
|
||||
skip.clear()
|
||||
|
||||
return new_block_ids_to_zero or None
|
||||
|
||||
def _preempt_request(
|
||||
|
||||
Reference in New Issue
Block a user