[Bugfix][KV Offload] Restore MTP-retained sliding-window history (#56709)

Signed-off-by: jacklin78911-collab <jacklin78911@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Liqian Lin
2026-09-16 08:14:09 +03:00
committed by GitHub
co-authored by Codex Or Ozeri
parent 711768fc17
commit 961c5b6b7e
2 changed files with 70 additions and 52 deletions
@@ -79,20 +79,31 @@ from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput
from vllm.v1.request import RequestStatus
@pytest.mark.parametrize("boundary", [3840, 3904])
@pytest.mark.parametrize(
"window,extra_retained", [(128, 0), (128, 3), (256, 1), (256, 3)]
)
@pytest.mark.parametrize("boundary", [3840, 3904, 3968])
@pytest.mark.parametrize("eagle", [False, True])
@pytest.mark.parametrize("left_state", ["present", "missing", "pending", "loading"])
def test_swa_offload_window_covers_unaligned_hit(boundary, eagle, left_state):
def test_swa_offload_window_covers_unaligned_hit(
boundary, eagle, left_state, window, extra_retained
):
"""A smaller SWA group can move the hit inside another group's CPU chunk."""
groups = []
for i, (block_size, window) in enumerate([(256, None), (64, 128), (8, 128)]):
for i, (block_size, group_window) in enumerate(
[(256, None), (64, window), (8, 128)]
):
kwargs = dict(
block_size=block_size, num_kv_heads=1, head_size=1, dtype=torch.float32
)
kv_spec = (
FullAttentionSpec(**kwargs)
if window is None
else SlidingWindowSpec(**kwargs, sliding_window=window)
if group_window is None
else SlidingWindowSpec(
**kwargs,
sliding_window=group_window,
extra_retained_tokens=extra_retained if i == 1 else 0,
)
)
groups.append(
KVCacheGroupSpec([f"layer{i}"], kv_spec, is_eagle_group=eagle and i == 1)
@@ -153,7 +164,9 @@ def test_swa_offload_window_covers_unaligned_hit(boundary, eagle, left_state):
sched._chunks_being_loaded.add(left_key)
external, load_async = sched.get_num_new_matched_tokens(request, 0)
if boundary == 3904 and left_state != "present":
first_retained_block = (boundary - window + 1 - extra_retained) // 64
needs_left = first_retained_block // 4 < end - 1
if needs_left and left_state != "present":
assert external == (0 if left_state == "missing" else None)
assert not load_async
return
@@ -177,8 +190,8 @@ def test_swa_offload_window_covers_unaligned_hit(boundary, eagle, left_state):
sched.update_state_after_alloc(request, blocks, external)
assert len(sched._jobs) == 1
job = next(iter(sched._jobs.values()))
if boundary == 3904:
assert {swa[14], swa[15]} <= job.keys
if needs_left:
assert {left_key, swa[end - 1]} <= job.keys
manager.complete_load(job.keys, state.req_context)
@@ -2629,27 +2642,14 @@ def test_async_preempt_readmit_before_transfer_output_is_deferred(request_runner
assert req_status.transfer_jobs == pending_store_jobs
@pytest.mark.parametrize("extra_retained", [0, 1, 3])
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_swa_alignment_skip(request_runner, async_scheduling: bool):
"""SWA blocks unreachable by the load path are skipped during store.
def test_swa_alignment_skip(request_runner, async_scheduling: bool, extra_retained):
"""Store the window a cold consumer must restore at each FA boundary.
Simulates a DeepSeek V4-like hybrid architecture where SWA groups have
much smaller block sizes than the full-attention (MLA) group, causing
most SWA blocks to be unreachable by the alignment-based load path.
Setup:
- Group 0: full attention (MLA-like), block_size=16
- Group 1: SWA, block_size=4, sliding_window=8
alignment_tokens = 16, so 16 / 4 = 4 SWA blocks per alignment segment.
sliding_window_size_in_chunks = ceil(8 / 4) = 2.
Within each segment of 4 SWA blocks, only the trailing 2 are stored.
With 32 tokens (2 full-attn blocks, 8 SWA blocks):
- Group 0 stores: blocks 0, 1 (all full-attn blocks)
- Group 1 stores: blocks 2, 3, 6, 7 (skip 0,1,4,5)
For real DeepSeek V4 (100K tokens), this reduces SWA stores by ~78%.
With FA blocks of 16 tokens and SWA blocks of 4 tokens, a window of 8
needs the last two blocks per segment. Retaining three extra tokens for
MTP requires one more block; retaining just one token does not.
"""
full_attn_block_size = 16
swa_block_size = 4
@@ -2674,6 +2674,7 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
head_size=1,
dtype=torch.float32,
sliding_window=sliding_window,
extra_retained_tokens=extra_retained,
),
),
]
@@ -2691,16 +2692,17 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
# Group 0: full attention
assert kv_group_configs[0].sliding_window_size_in_chunks is None
assert kv_group_configs[0].tokens_per_chunk == full_attn_block_size
# Group 1: SWA with sliding_window_size of 2 chunks
assert kv_group_configs[1].sliding_window_size_in_chunks == 2
# Group 1: SWA, including any additional retained blocks
assert kv_group_configs[1].sliding_window_size_in_chunks == (
3 if extra_retained > 1 else 2
)
assert kv_group_configs[1].tokens_per_chunk == swa_block_size
# alignment_tokens = full_attn_block_size = 16
assert runner.connector_scheduler.config.alignment_tokens == full_attn_block_size
# Send 32 tokens = 2 full-attn blocks (block_size=16) = 8 SWA blocks
# (block_size=4). Decode 1 token to kick off processing (stores are
# deferred to next step).
num_tokens = 32
# Keep a partial last block so decoding does not reuse the producer's
# freed blocks before the harness records its deferred stores.
num_tokens = 33
runner.new_request(token_ids=[0] * num_tokens)
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
@@ -2713,12 +2715,7 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
)
runner.run(
decoded_tokens=[EOS_TOKEN_ID],
# Group 0 (full attn, block_size=16): 2 offloaded chunks
# -> GPU blocks (0, 0) and (0, 1)
# Group 1 (SWA, block_size=4): 8 offloaded chunks, skip first 2
# per segment of 4:
# Segment 0 (blocks 0-3): skip 0,1 -> store (1, 2), (1, 3)
# Segment 1 (blocks 4-7): skip 4,5 -> store (1, 6), (1, 7)
# Retain each segment's tail, including MTP history when needed.
expected_stored=(
(0, 0),
(0, 1),
@@ -2726,27 +2723,26 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
(1, 3),
(1, 6),
(1, 7),
),
)
+ (((1, 1), (1, 5)) if extra_retained > 1 else ()),
)
# Verify that loads still work correctly for the stored SWA blocks.
runner.scheduler.reset_prefix_cache()
runner.new_request(token_ids=[0] * num_tokens + [1])
runner.manager.lookup.return_value = LookupResult.HIT
runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 2
runner.manager.lookup.side_effect = lambda key, ctx: (
LookupResult.HIT if (key, 0) in runner.offloaded else LookupResult.MISS
)
runner.run(
decoded_tokens=[EOS_TOKEN_ID],
# Group 0: full prefix lookup hits 2 offloaded chunks
# -> loads GPU blocks (0, 0), (0, 1)
# Group 1: sliding window lookup finds trailing 2 from last segment
# (blocks 6, 7 which were stored)
# -> loads GPU blocks (1, 6), (1, 7)
# The consumer restores the full FA prefix and the final SWA tail.
expected_loaded=(
(0, 0),
(0, 1),
(1, 6),
(1, 7),
),
)
+ (((1, 5),) if extra_retained > 1 else ()),
)