From 961c5b6b7ea98b765a6d436434d18ef945596f32 Mon Sep 17 00:00:00 2001 From: Liqian Lin Date: Wed, 16 Sep 2026 13:14:09 +0800 Subject: [PATCH] [Bugfix][KV Offload] Restore MTP-retained sliding-window history (#56709) Signed-off-by: jacklin78911-collab Co-authored-by: Codex Co-authored-by: Or Ozeri --- .../offloading_connector/test_scheduler.py | 92 +++++++++---------- .../kv_connector/v1/offloading/scheduler.py | 30 +++++- 2 files changed, 70 insertions(+), 52 deletions(-) diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py index 968474dd74..89b77ae43f 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py @@ -79,20 +79,31 @@ from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput from vllm.v1.request import RequestStatus -@pytest.mark.parametrize("boundary", [3840, 3904]) +@pytest.mark.parametrize( + "window,extra_retained", [(128, 0), (128, 3), (256, 1), (256, 3)] +) +@pytest.mark.parametrize("boundary", [3840, 3904, 3968]) @pytest.mark.parametrize("eagle", [False, True]) @pytest.mark.parametrize("left_state", ["present", "missing", "pending", "loading"]) -def test_swa_offload_window_covers_unaligned_hit(boundary, eagle, left_state): +def test_swa_offload_window_covers_unaligned_hit( + boundary, eagle, left_state, window, extra_retained +): """A smaller SWA group can move the hit inside another group's CPU chunk.""" groups = [] - for i, (block_size, window) in enumerate([(256, None), (64, 128), (8, 128)]): + for i, (block_size, group_window) in enumerate( + [(256, None), (64, window), (8, 128)] + ): kwargs = dict( block_size=block_size, num_kv_heads=1, head_size=1, dtype=torch.float32 ) kv_spec = ( FullAttentionSpec(**kwargs) - if window is None - else SlidingWindowSpec(**kwargs, sliding_window=window) + if group_window is None + else SlidingWindowSpec( + **kwargs, + sliding_window=group_window, + extra_retained_tokens=extra_retained if i == 1 else 0, + ) ) groups.append( KVCacheGroupSpec([f"layer{i}"], kv_spec, is_eagle_group=eagle and i == 1) @@ -153,7 +164,9 @@ def test_swa_offload_window_covers_unaligned_hit(boundary, eagle, left_state): sched._chunks_being_loaded.add(left_key) external, load_async = sched.get_num_new_matched_tokens(request, 0) - if boundary == 3904 and left_state != "present": + first_retained_block = (boundary - window + 1 - extra_retained) // 64 + needs_left = first_retained_block // 4 < end - 1 + if needs_left and left_state != "present": assert external == (0 if left_state == "missing" else None) assert not load_async return @@ -177,8 +190,8 @@ def test_swa_offload_window_covers_unaligned_hit(boundary, eagle, left_state): sched.update_state_after_alloc(request, blocks, external) assert len(sched._jobs) == 1 job = next(iter(sched._jobs.values())) - if boundary == 3904: - assert {swa[14], swa[15]} <= job.keys + if needs_left: + assert {left_key, swa[end - 1]} <= job.keys manager.complete_load(job.keys, state.req_context) @@ -2629,27 +2642,14 @@ def test_async_preempt_readmit_before_transfer_output_is_deferred(request_runner assert req_status.transfer_jobs == pending_store_jobs +@pytest.mark.parametrize("extra_retained", [0, 1, 3]) @pytest.mark.parametrize("async_scheduling", [True, False]) -def test_swa_alignment_skip(request_runner, async_scheduling: bool): - """SWA blocks unreachable by the load path are skipped during store. +def test_swa_alignment_skip(request_runner, async_scheduling: bool, extra_retained): + """Store the window a cold consumer must restore at each FA boundary. - Simulates a DeepSeek V4-like hybrid architecture where SWA groups have - much smaller block sizes than the full-attention (MLA) group, causing - most SWA blocks to be unreachable by the alignment-based load path. - - Setup: - - Group 0: full attention (MLA-like), block_size=16 - - Group 1: SWA, block_size=4, sliding_window=8 - - alignment_tokens = 16, so 16 / 4 = 4 SWA blocks per alignment segment. - sliding_window_size_in_chunks = ceil(8 / 4) = 2. - Within each segment of 4 SWA blocks, only the trailing 2 are stored. - - With 32 tokens (2 full-attn blocks, 8 SWA blocks): - - Group 0 stores: blocks 0, 1 (all full-attn blocks) - - Group 1 stores: blocks 2, 3, 6, 7 (skip 0,1,4,5) - - For real DeepSeek V4 (100K tokens), this reduces SWA stores by ~78%. + With FA blocks of 16 tokens and SWA blocks of 4 tokens, a window of 8 + needs the last two blocks per segment. Retaining three extra tokens for + MTP requires one more block; retaining just one token does not. """ full_attn_block_size = 16 swa_block_size = 4 @@ -2674,6 +2674,7 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool): head_size=1, dtype=torch.float32, sliding_window=sliding_window, + extra_retained_tokens=extra_retained, ), ), ] @@ -2691,16 +2692,17 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool): # Group 0: full attention assert kv_group_configs[0].sliding_window_size_in_chunks is None assert kv_group_configs[0].tokens_per_chunk == full_attn_block_size - # Group 1: SWA with sliding_window_size of 2 chunks - assert kv_group_configs[1].sliding_window_size_in_chunks == 2 + # Group 1: SWA, including any additional retained blocks + assert kv_group_configs[1].sliding_window_size_in_chunks == ( + 3 if extra_retained > 1 else 2 + ) assert kv_group_configs[1].tokens_per_chunk == swa_block_size # alignment_tokens = full_attn_block_size = 16 assert runner.connector_scheduler.config.alignment_tokens == full_attn_block_size - # Send 32 tokens = 2 full-attn blocks (block_size=16) = 8 SWA blocks - # (block_size=4). Decode 1 token to kick off processing (stores are - # deferred to next step). - num_tokens = 32 + # Keep a partial last block so decoding does not reuse the producer's + # freed blocks before the harness records its deferred stores. + num_tokens = 33 runner.new_request(token_ids=[0] * num_tokens) runner.manager.prepare_store.side_effect = lambda keys, req_context: ( generate_store_output(keys) @@ -2713,12 +2715,7 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool): ) runner.run( decoded_tokens=[EOS_TOKEN_ID], - # Group 0 (full attn, block_size=16): 2 offloaded chunks - # -> GPU blocks (0, 0) and (0, 1) - # Group 1 (SWA, block_size=4): 8 offloaded chunks, skip first 2 - # per segment of 4: - # Segment 0 (blocks 0-3): skip 0,1 -> store (1, 2), (1, 3) - # Segment 1 (blocks 4-7): skip 4,5 -> store (1, 6), (1, 7) + # Retain each segment's tail, including MTP history when needed. expected_stored=( (0, 0), (0, 1), @@ -2726,27 +2723,26 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool): (1, 3), (1, 6), (1, 7), - ), + ) + + (((1, 1), (1, 5)) if extra_retained > 1 else ()), ) # Verify that loads still work correctly for the stored SWA blocks. runner.scheduler.reset_prefix_cache() runner.new_request(token_ids=[0] * num_tokens + [1]) - runner.manager.lookup.return_value = LookupResult.HIT - runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 2 + runner.manager.lookup.side_effect = lambda key, ctx: ( + LookupResult.HIT if (key, 0) in runner.offloaded else LookupResult.MISS + ) runner.run( decoded_tokens=[EOS_TOKEN_ID], - # Group 0: full prefix lookup hits 2 offloaded chunks - # -> loads GPU blocks (0, 0), (0, 1) - # Group 1: sliding window lookup finds trailing 2 from last segment - # (blocks 6, 7 which were stored) - # -> loads GPU blocks (1, 6), (1, 7) + # The consumer restores the full FA prefix and the final SWA tail. expected_loaded=( (0, 0), (0, 1), (1, 6), (1, 7), - ), + ) + + (((1, 5),) if extra_retained > 1 else ()), ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index 8cd5e90b6f..5cc66e4bf0 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import time from collections.abc import Iterable, Sequence -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from itertools import chain, islice from typing import Any, NamedTuple @@ -114,7 +114,10 @@ class GroupOffloadConfig(NamedTuple): window = max( window, cdiv( - self.kv_cache_spec.sliding_window - 1 + right_padding, + self.kv_cache_spec.sliding_window + - 1 + + self.kv_cache_spec.extra_retained_tokens + + right_padding, self.tokens_per_chunk, ), ) @@ -126,7 +129,13 @@ def get_sliding_window_size_in_chunks( ) -> int | None: if isinstance(kv_cache_spec, SlidingWindowSpec): assert kv_cache_spec.sliding_window > 0 - return cdiv(kv_cache_spec.sliding_window, tokens_per_chunk) + return max( + cdiv(kv_cache_spec.sliding_window, tokens_per_chunk), + cdiv( + kv_cache_spec.sliding_window - 1 + kv_cache_spec.extra_retained_tokens, + tokens_per_chunk, + ), + ) if isinstance(kv_cache_spec, ChunkedLocalAttentionSpec): # Attention never reaches back past one chunk @@ -1461,11 +1470,24 @@ class OffloadingConnectorScheduler: ) -> list[bool] | None: """Build the block mask for a range of candidate offload chunks.""" blocks_per_chunk = self.config.blocks_per_chunk + kv_cache_spec = group_config.kv_cache_spec + if isinstance(kv_cache_spec, SlidingWindowSpec) and ( + kv_cache_spec.extra_retained_tokens + ): + # Offload restores all allocated history, including MTP re-prefill + # tokens. Widen only the store mask, not the model's attention window. + kv_cache_spec = replace( + kv_cache_spec, + sliding_window=( + kv_cache_spec.sliding_window + kv_cache_spec.extra_retained_tokens + ), + extra_retained_tokens=0, + ) return group_config.manager_cls.reachable_block_mask( start_block=start_chunk_idx * blocks_per_chunk, end_block=end_chunk_idx * blocks_per_chunk, alignment_tokens=self.config.alignment_tokens, - kv_cache_spec=group_config.kv_cache_spec, + kv_cache_spec=kv_cache_spec, use_eagle=group_config.is_eagle_group, retention_interval=self.config.retention_interval, reachable_boundaries=reachable_boundaries,