[Bugfix][KV Offload] Restore MTP-retained sliding-window history (#56709)

Signed-off-by: jacklin78911-collab <jacklin78911@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Liqian Lin
2026-09-16 08:14:09 +03:00
committed by GitHub
co-authored by Codex Or Ozeri
parent 711768fc17
commit 961c5b6b7e
2 changed files with 70 additions and 52 deletions
@@ -79,20 +79,31 @@ from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput
from vllm.v1.request import RequestStatus
@pytest.mark.parametrize("boundary", [3840, 3904])
@pytest.mark.parametrize(
"window,extra_retained", [(128, 0), (128, 3), (256, 1), (256, 3)]
)
@pytest.mark.parametrize("boundary", [3840, 3904, 3968])
@pytest.mark.parametrize("eagle", [False, True])
@pytest.mark.parametrize("left_state", ["present", "missing", "pending", "loading"])
def test_swa_offload_window_covers_unaligned_hit(boundary, eagle, left_state):
def test_swa_offload_window_covers_unaligned_hit(
boundary, eagle, left_state, window, extra_retained
):
"""A smaller SWA group can move the hit inside another group's CPU chunk."""
groups = []
for i, (block_size, window) in enumerate([(256, None), (64, 128), (8, 128)]):
for i, (block_size, group_window) in enumerate(
[(256, None), (64, window), (8, 128)]
):
kwargs = dict(
block_size=block_size, num_kv_heads=1, head_size=1, dtype=torch.float32
)
kv_spec = (
FullAttentionSpec(**kwargs)
if window is None
else SlidingWindowSpec(**kwargs, sliding_window=window)
if group_window is None
else SlidingWindowSpec(
**kwargs,
sliding_window=group_window,
extra_retained_tokens=extra_retained if i == 1 else 0,
)
)
groups.append(
KVCacheGroupSpec([f"layer{i}"], kv_spec, is_eagle_group=eagle and i == 1)
@@ -153,7 +164,9 @@ def test_swa_offload_window_covers_unaligned_hit(boundary, eagle, left_state):
sched._chunks_being_loaded.add(left_key)
external, load_async = sched.get_num_new_matched_tokens(request, 0)
if boundary == 3904 and left_state != "present":
first_retained_block = (boundary - window + 1 - extra_retained) // 64
needs_left = first_retained_block // 4 < end - 1
if needs_left and left_state != "present":
assert external == (0 if left_state == "missing" else None)
assert not load_async
return
@@ -177,8 +190,8 @@ def test_swa_offload_window_covers_unaligned_hit(boundary, eagle, left_state):
sched.update_state_after_alloc(request, blocks, external)
assert len(sched._jobs) == 1
job = next(iter(sched._jobs.values()))
if boundary == 3904:
assert {swa[14], swa[15]} <= job.keys
if needs_left:
assert {left_key, swa[end - 1]} <= job.keys
manager.complete_load(job.keys, state.req_context)
@@ -2629,27 +2642,14 @@ def test_async_preempt_readmit_before_transfer_output_is_deferred(request_runner
assert req_status.transfer_jobs == pending_store_jobs
@pytest.mark.parametrize("extra_retained", [0, 1, 3])
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_swa_alignment_skip(request_runner, async_scheduling: bool):
"""SWA blocks unreachable by the load path are skipped during store.
def test_swa_alignment_skip(request_runner, async_scheduling: bool, extra_retained):
"""Store the window a cold consumer must restore at each FA boundary.
Simulates a DeepSeek V4-like hybrid architecture where SWA groups have
much smaller block sizes than the full-attention (MLA) group, causing
most SWA blocks to be unreachable by the alignment-based load path.
Setup:
- Group 0: full attention (MLA-like), block_size=16
- Group 1: SWA, block_size=4, sliding_window=8
alignment_tokens = 16, so 16 / 4 = 4 SWA blocks per alignment segment.
sliding_window_size_in_chunks = ceil(8 / 4) = 2.
Within each segment of 4 SWA blocks, only the trailing 2 are stored.
With 32 tokens (2 full-attn blocks, 8 SWA blocks):
- Group 0 stores: blocks 0, 1 (all full-attn blocks)
- Group 1 stores: blocks 2, 3, 6, 7 (skip 0,1,4,5)
For real DeepSeek V4 (100K tokens), this reduces SWA stores by ~78%.
With FA blocks of 16 tokens and SWA blocks of 4 tokens, a window of 8
needs the last two blocks per segment. Retaining three extra tokens for
MTP requires one more block; retaining just one token does not.
"""
full_attn_block_size = 16
swa_block_size = 4
@@ -2674,6 +2674,7 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
head_size=1,
dtype=torch.float32,
sliding_window=sliding_window,
extra_retained_tokens=extra_retained,
),
),
]
@@ -2691,16 +2692,17 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
# Group 0: full attention
assert kv_group_configs[0].sliding_window_size_in_chunks is None
assert kv_group_configs[0].tokens_per_chunk == full_attn_block_size
# Group 1: SWA with sliding_window_size of 2 chunks
assert kv_group_configs[1].sliding_window_size_in_chunks == 2
# Group 1: SWA, including any additional retained blocks
assert kv_group_configs[1].sliding_window_size_in_chunks == (
3 if extra_retained > 1 else 2
)
assert kv_group_configs[1].tokens_per_chunk == swa_block_size
# alignment_tokens = full_attn_block_size = 16
assert runner.connector_scheduler.config.alignment_tokens == full_attn_block_size
# Send 32 tokens = 2 full-attn blocks (block_size=16) = 8 SWA blocks
# (block_size=4). Decode 1 token to kick off processing (stores are
# deferred to next step).
num_tokens = 32
# Keep a partial last block so decoding does not reuse the producer's
# freed blocks before the harness records its deferred stores.
num_tokens = 33
runner.new_request(token_ids=[0] * num_tokens)
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
@@ -2713,12 +2715,7 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
)
runner.run(
decoded_tokens=[EOS_TOKEN_ID],
# Group 0 (full attn, block_size=16): 2 offloaded chunks
# -> GPU blocks (0, 0) and (0, 1)
# Group 1 (SWA, block_size=4): 8 offloaded chunks, skip first 2
# per segment of 4:
# Segment 0 (blocks 0-3): skip 0,1 -> store (1, 2), (1, 3)
# Segment 1 (blocks 4-7): skip 4,5 -> store (1, 6), (1, 7)
# Retain each segment's tail, including MTP history when needed.
expected_stored=(
(0, 0),
(0, 1),
@@ -2726,27 +2723,26 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
(1, 3),
(1, 6),
(1, 7),
),
)
+ (((1, 1), (1, 5)) if extra_retained > 1 else ()),
)
# Verify that loads still work correctly for the stored SWA blocks.
runner.scheduler.reset_prefix_cache()
runner.new_request(token_ids=[0] * num_tokens + [1])
runner.manager.lookup.return_value = LookupResult.HIT
runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 2
runner.manager.lookup.side_effect = lambda key, ctx: (
LookupResult.HIT if (key, 0) in runner.offloaded else LookupResult.MISS
)
runner.run(
decoded_tokens=[EOS_TOKEN_ID],
# Group 0: full prefix lookup hits 2 offloaded chunks
# -> loads GPU blocks (0, 0), (0, 1)
# Group 1: sliding window lookup finds trailing 2 from last segment
# (blocks 6, 7 which were stored)
# -> loads GPU blocks (1, 6), (1, 7)
# The consumer restores the full FA prefix and the final SWA tail.
expected_loaded=(
(0, 0),
(0, 1),
(1, 6),
(1, 7),
),
)
+ (((1, 5),) if extra_retained > 1 else ()),
)
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from collections.abc import Iterable, Sequence
from dataclasses import dataclass, field
from dataclasses import dataclass, field, replace
from itertools import chain, islice
from typing import Any, NamedTuple
@@ -114,7 +114,10 @@ class GroupOffloadConfig(NamedTuple):
window = max(
window,
cdiv(
self.kv_cache_spec.sliding_window - 1 + right_padding,
self.kv_cache_spec.sliding_window
- 1
+ self.kv_cache_spec.extra_retained_tokens
+ right_padding,
self.tokens_per_chunk,
),
)
@@ -126,7 +129,13 @@ def get_sliding_window_size_in_chunks(
) -> int | None:
if isinstance(kv_cache_spec, SlidingWindowSpec):
assert kv_cache_spec.sliding_window > 0
return cdiv(kv_cache_spec.sliding_window, tokens_per_chunk)
return max(
cdiv(kv_cache_spec.sliding_window, tokens_per_chunk),
cdiv(
kv_cache_spec.sliding_window - 1 + kv_cache_spec.extra_retained_tokens,
tokens_per_chunk,
),
)
if isinstance(kv_cache_spec, ChunkedLocalAttentionSpec):
# Attention never reaches back past one chunk
@@ -1461,11 +1470,24 @@ class OffloadingConnectorScheduler:
) -> list[bool] | None:
"""Build the block mask for a range of candidate offload chunks."""
blocks_per_chunk = self.config.blocks_per_chunk
kv_cache_spec = group_config.kv_cache_spec
if isinstance(kv_cache_spec, SlidingWindowSpec) and (
kv_cache_spec.extra_retained_tokens
):
# Offload restores all allocated history, including MTP re-prefill
# tokens. Widen only the store mask, not the model's attention window.
kv_cache_spec = replace(
kv_cache_spec,
sliding_window=(
kv_cache_spec.sliding_window + kv_cache_spec.extra_retained_tokens
),
extra_retained_tokens=0,
)
return group_config.manager_cls.reachable_block_mask(
start_block=start_chunk_idx * blocks_per_chunk,
end_block=end_chunk_idx * blocks_per_chunk,
alignment_tokens=self.config.alignment_tokens,
kv_cache_spec=group_config.kv_cache_spec,
kv_cache_spec=kv_cache_spec,
use_eagle=group_config.is_eagle_group,
retention_interval=self.config.retention_interval,
reachable_boundaries=reachable_boundaries,