[Bugfix][KV Offload] Restore MTP-retained sliding-window history (#56709)
Signed-off-by: jacklin78911-collab <jacklin78911@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
Codex
Or Ozeri
parent
711768fc17
commit
961c5b6b7e
@@ -79,20 +79,31 @@ from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput
|
||||
from vllm.v1.request import RequestStatus
|
||||
|
||||
|
||||
@pytest.mark.parametrize("boundary", [3840, 3904])
|
||||
@pytest.mark.parametrize(
|
||||
"window,extra_retained", [(128, 0), (128, 3), (256, 1), (256, 3)]
|
||||
)
|
||||
@pytest.mark.parametrize("boundary", [3840, 3904, 3968])
|
||||
@pytest.mark.parametrize("eagle", [False, True])
|
||||
@pytest.mark.parametrize("left_state", ["present", "missing", "pending", "loading"])
|
||||
def test_swa_offload_window_covers_unaligned_hit(boundary, eagle, left_state):
|
||||
def test_swa_offload_window_covers_unaligned_hit(
|
||||
boundary, eagle, left_state, window, extra_retained
|
||||
):
|
||||
"""A smaller SWA group can move the hit inside another group's CPU chunk."""
|
||||
groups = []
|
||||
for i, (block_size, window) in enumerate([(256, None), (64, 128), (8, 128)]):
|
||||
for i, (block_size, group_window) in enumerate(
|
||||
[(256, None), (64, window), (8, 128)]
|
||||
):
|
||||
kwargs = dict(
|
||||
block_size=block_size, num_kv_heads=1, head_size=1, dtype=torch.float32
|
||||
)
|
||||
kv_spec = (
|
||||
FullAttentionSpec(**kwargs)
|
||||
if window is None
|
||||
else SlidingWindowSpec(**kwargs, sliding_window=window)
|
||||
if group_window is None
|
||||
else SlidingWindowSpec(
|
||||
**kwargs,
|
||||
sliding_window=group_window,
|
||||
extra_retained_tokens=extra_retained if i == 1 else 0,
|
||||
)
|
||||
)
|
||||
groups.append(
|
||||
KVCacheGroupSpec([f"layer{i}"], kv_spec, is_eagle_group=eagle and i == 1)
|
||||
@@ -153,7 +164,9 @@ def test_swa_offload_window_covers_unaligned_hit(boundary, eagle, left_state):
|
||||
sched._chunks_being_loaded.add(left_key)
|
||||
|
||||
external, load_async = sched.get_num_new_matched_tokens(request, 0)
|
||||
if boundary == 3904 and left_state != "present":
|
||||
first_retained_block = (boundary - window + 1 - extra_retained) // 64
|
||||
needs_left = first_retained_block // 4 < end - 1
|
||||
if needs_left and left_state != "present":
|
||||
assert external == (0 if left_state == "missing" else None)
|
||||
assert not load_async
|
||||
return
|
||||
@@ -177,8 +190,8 @@ def test_swa_offload_window_covers_unaligned_hit(boundary, eagle, left_state):
|
||||
sched.update_state_after_alloc(request, blocks, external)
|
||||
assert len(sched._jobs) == 1
|
||||
job = next(iter(sched._jobs.values()))
|
||||
if boundary == 3904:
|
||||
assert {swa[14], swa[15]} <= job.keys
|
||||
if needs_left:
|
||||
assert {left_key, swa[end - 1]} <= job.keys
|
||||
manager.complete_load(job.keys, state.req_context)
|
||||
|
||||
|
||||
@@ -2629,27 +2642,14 @@ def test_async_preempt_readmit_before_transfer_output_is_deferred(request_runner
|
||||
assert req_status.transfer_jobs == pending_store_jobs
|
||||
|
||||
|
||||
@pytest.mark.parametrize("extra_retained", [0, 1, 3])
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_swa_alignment_skip(request_runner, async_scheduling: bool):
|
||||
"""SWA blocks unreachable by the load path are skipped during store.
|
||||
def test_swa_alignment_skip(request_runner, async_scheduling: bool, extra_retained):
|
||||
"""Store the window a cold consumer must restore at each FA boundary.
|
||||
|
||||
Simulates a DeepSeek V4-like hybrid architecture where SWA groups have
|
||||
much smaller block sizes than the full-attention (MLA) group, causing
|
||||
most SWA blocks to be unreachable by the alignment-based load path.
|
||||
|
||||
Setup:
|
||||
- Group 0: full attention (MLA-like), block_size=16
|
||||
- Group 1: SWA, block_size=4, sliding_window=8
|
||||
|
||||
alignment_tokens = 16, so 16 / 4 = 4 SWA blocks per alignment segment.
|
||||
sliding_window_size_in_chunks = ceil(8 / 4) = 2.
|
||||
Within each segment of 4 SWA blocks, only the trailing 2 are stored.
|
||||
|
||||
With 32 tokens (2 full-attn blocks, 8 SWA blocks):
|
||||
- Group 0 stores: blocks 0, 1 (all full-attn blocks)
|
||||
- Group 1 stores: blocks 2, 3, 6, 7 (skip 0,1,4,5)
|
||||
|
||||
For real DeepSeek V4 (100K tokens), this reduces SWA stores by ~78%.
|
||||
With FA blocks of 16 tokens and SWA blocks of 4 tokens, a window of 8
|
||||
needs the last two blocks per segment. Retaining three extra tokens for
|
||||
MTP requires one more block; retaining just one token does not.
|
||||
"""
|
||||
full_attn_block_size = 16
|
||||
swa_block_size = 4
|
||||
@@ -2674,6 +2674,7 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=sliding_window,
|
||||
extra_retained_tokens=extra_retained,
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -2691,16 +2692,17 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
|
||||
# Group 0: full attention
|
||||
assert kv_group_configs[0].sliding_window_size_in_chunks is None
|
||||
assert kv_group_configs[0].tokens_per_chunk == full_attn_block_size
|
||||
# Group 1: SWA with sliding_window_size of 2 chunks
|
||||
assert kv_group_configs[1].sliding_window_size_in_chunks == 2
|
||||
# Group 1: SWA, including any additional retained blocks
|
||||
assert kv_group_configs[1].sliding_window_size_in_chunks == (
|
||||
3 if extra_retained > 1 else 2
|
||||
)
|
||||
assert kv_group_configs[1].tokens_per_chunk == swa_block_size
|
||||
# alignment_tokens = full_attn_block_size = 16
|
||||
assert runner.connector_scheduler.config.alignment_tokens == full_attn_block_size
|
||||
|
||||
# Send 32 tokens = 2 full-attn blocks (block_size=16) = 8 SWA blocks
|
||||
# (block_size=4). Decode 1 token to kick off processing (stores are
|
||||
# deferred to next step).
|
||||
num_tokens = 32
|
||||
# Keep a partial last block so decoding does not reuse the producer's
|
||||
# freed blocks before the harness records its deferred stores.
|
||||
num_tokens = 33
|
||||
runner.new_request(token_ids=[0] * num_tokens)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
@@ -2713,12 +2715,7 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
|
||||
)
|
||||
runner.run(
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
# Group 0 (full attn, block_size=16): 2 offloaded chunks
|
||||
# -> GPU blocks (0, 0) and (0, 1)
|
||||
# Group 1 (SWA, block_size=4): 8 offloaded chunks, skip first 2
|
||||
# per segment of 4:
|
||||
# Segment 0 (blocks 0-3): skip 0,1 -> store (1, 2), (1, 3)
|
||||
# Segment 1 (blocks 4-7): skip 4,5 -> store (1, 6), (1, 7)
|
||||
# Retain each segment's tail, including MTP history when needed.
|
||||
expected_stored=(
|
||||
(0, 0),
|
||||
(0, 1),
|
||||
@@ -2726,27 +2723,26 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
|
||||
(1, 3),
|
||||
(1, 6),
|
||||
(1, 7),
|
||||
),
|
||||
)
|
||||
+ (((1, 1), (1, 5)) if extra_retained > 1 else ()),
|
||||
)
|
||||
|
||||
# Verify that loads still work correctly for the stored SWA blocks.
|
||||
runner.scheduler.reset_prefix_cache()
|
||||
runner.new_request(token_ids=[0] * num_tokens + [1])
|
||||
runner.manager.lookup.return_value = LookupResult.HIT
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 2
|
||||
runner.manager.lookup.side_effect = lambda key, ctx: (
|
||||
LookupResult.HIT if (key, 0) in runner.offloaded else LookupResult.MISS
|
||||
)
|
||||
runner.run(
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
# Group 0: full prefix lookup hits 2 offloaded chunks
|
||||
# -> loads GPU blocks (0, 0), (0, 1)
|
||||
# Group 1: sliding window lookup finds trailing 2 from last segment
|
||||
# (blocks 6, 7 which were stored)
|
||||
# -> loads GPU blocks (1, 6), (1, 7)
|
||||
# The consumer restores the full FA prefix and the final SWA tail.
|
||||
expected_loaded=(
|
||||
(0, 0),
|
||||
(0, 1),
|
||||
(1, 6),
|
||||
(1, 7),
|
||||
),
|
||||
)
|
||||
+ (((1, 5),) if extra_retained > 1 else ()),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import time
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass, field, replace
|
||||
from itertools import chain, islice
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
@@ -114,7 +114,10 @@ class GroupOffloadConfig(NamedTuple):
|
||||
window = max(
|
||||
window,
|
||||
cdiv(
|
||||
self.kv_cache_spec.sliding_window - 1 + right_padding,
|
||||
self.kv_cache_spec.sliding_window
|
||||
- 1
|
||||
+ self.kv_cache_spec.extra_retained_tokens
|
||||
+ right_padding,
|
||||
self.tokens_per_chunk,
|
||||
),
|
||||
)
|
||||
@@ -126,7 +129,13 @@ def get_sliding_window_size_in_chunks(
|
||||
) -> int | None:
|
||||
if isinstance(kv_cache_spec, SlidingWindowSpec):
|
||||
assert kv_cache_spec.sliding_window > 0
|
||||
return cdiv(kv_cache_spec.sliding_window, tokens_per_chunk)
|
||||
return max(
|
||||
cdiv(kv_cache_spec.sliding_window, tokens_per_chunk),
|
||||
cdiv(
|
||||
kv_cache_spec.sliding_window - 1 + kv_cache_spec.extra_retained_tokens,
|
||||
tokens_per_chunk,
|
||||
),
|
||||
)
|
||||
|
||||
if isinstance(kv_cache_spec, ChunkedLocalAttentionSpec):
|
||||
# Attention never reaches back past one chunk
|
||||
@@ -1461,11 +1470,24 @@ class OffloadingConnectorScheduler:
|
||||
) -> list[bool] | None:
|
||||
"""Build the block mask for a range of candidate offload chunks."""
|
||||
blocks_per_chunk = self.config.blocks_per_chunk
|
||||
kv_cache_spec = group_config.kv_cache_spec
|
||||
if isinstance(kv_cache_spec, SlidingWindowSpec) and (
|
||||
kv_cache_spec.extra_retained_tokens
|
||||
):
|
||||
# Offload restores all allocated history, including MTP re-prefill
|
||||
# tokens. Widen only the store mask, not the model's attention window.
|
||||
kv_cache_spec = replace(
|
||||
kv_cache_spec,
|
||||
sliding_window=(
|
||||
kv_cache_spec.sliding_window + kv_cache_spec.extra_retained_tokens
|
||||
),
|
||||
extra_retained_tokens=0,
|
||||
)
|
||||
return group_config.manager_cls.reachable_block_mask(
|
||||
start_block=start_chunk_idx * blocks_per_chunk,
|
||||
end_block=end_chunk_idx * blocks_per_chunk,
|
||||
alignment_tokens=self.config.alignment_tokens,
|
||||
kv_cache_spec=group_config.kv_cache_spec,
|
||||
kv_cache_spec=kv_cache_spec,
|
||||
use_eagle=group_config.is_eagle_group,
|
||||
retention_interval=self.config.retention_interval,
|
||||
reachable_boundaries=reachable_boundaries,
|
||||
|
||||
Reference in New Issue
Block a user