[Bugfix][KV Offload] Fix SWA store reachability during chunked prefill (#54362)

Signed-off-by: Will Hampson <whamp@ggl.slmail.me>
Co-authored-by: Will Hampson <whamp@ggl.slmail.me>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Will Hampson
2026-09-06 13:50:39 +03:00
committed by GitHub
co-authored by Will Hampson Or Ozeri
parent dd0760165c
commit 569adb5a97
4 changed files with 385 additions and 15 deletions
+35
View File
@@ -4389,6 +4389,41 @@ def test_mamba_shared_prefix_reuse_under_zero_retention():
assert last_req_hit(retention=0, pin=True) == 2 * block_size
@pytest.mark.skip_global_cleanup
def test_swa_reachable_block_mask_final_partial_segment():
"""Only a final SWA horizon makes its incomplete segment tail reachable."""
from vllm.v1.core.single_type_kv_cache_manager import SlidingWindowManager
spec = SlidingWindowSpec(
block_size=4,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=16,
)
start_block = 243
end_block = 250
def reachable_blocks(
final_segment_end_block: int, *, use_eagle: bool = False
) -> set[int]:
mask = SlidingWindowManager.reachable_block_mask(
start_block=start_block,
end_block=end_block,
alignment_tokens=36,
kv_cache_spec=spec,
use_eagle=use_eagle,
final_segment_end_block=final_segment_end_block,
)
assert mask is not None
return {start_block + i for i, reachable in enumerate(mask) if reachable}
assert reachable_blocks(300) == {248, 249}
assert reachable_blocks(250) == {246, 247, 248, 249}
assert reachable_blocks(300, use_eagle=True) == {243, 248, 249}
assert reachable_blocks(250, use_eagle=True) == {243, 248, 249}
def test_swa_reachable_block_mask_pins_shared_prefix():
"""SWA analog of the Mamba pin: the shared-prefix junction must keep the
``need``-block sliding-window tail ending on that boundary (not a single
@@ -48,6 +48,7 @@ from vllm.v1.core.sched.output import (
from vllm.v1.kv_cache_interface import (
ChunkedLocalAttentionSpec,
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
SlidingWindowSpec,
)
@@ -3390,6 +3391,220 @@ class TestEagle:
f"interior hole in stored blocks: {offsets}"
)
@pytest.mark.skip_global_cleanup
@pytest.mark.parametrize(
"abort_after_first_step,async_scheduling,store_horizon_tokens",
[
pytest.param(False, False, 1200, id="completed-prompt"),
pytest.param(True, False, 1000, id="aborted-sync"),
pytest.param(True, True, 1000, id="aborted-async"),
],
)
def test_chunked_prefill_uses_final_swa_store_horizon(
self,
request_runner,
abort_after_first_step: bool,
async_scheduling: bool,
store_horizon_tokens: int,
):
"""SWA stores must use the completed or aborted request horizon."""
full_attn_block_size = 36
swa_block_size = 4
sliding_window = 16
num_tokens = 1200
kv_cache_groups = [
KVCacheGroupSpec(
["layer0"],
FullAttentionSpec(
block_size=full_attn_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["layer1"],
SlidingWindowSpec(
block_size=swa_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=sliding_window,
),
),
]
runner = request_runner(
block_size=swa_block_size,
num_gpu_blocks=1000,
async_scheduling=async_scheduling,
kv_cache_groups=kv_cache_groups,
)
runner.new_request(token_ids=[0] * num_tokens)
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
def abort_after_prefill_step() -> None:
if runner.scheduler.running:
runner.scheduler.finish_requests(
(str(runner.req_id),), RequestStatus.FINISHED_ABORTED
)
runner._run(
[1] if abort_after_first_step else [1, 1, EOS_TOKEN_ID],
complete_transfers=True,
post_step_fn=(abort_after_prefill_step if abort_after_first_step else None),
)
stored_swa_chunks = sorted(
block.request_block_offset
for transfer in runner.completed_stores
for block in transfer.gpu_blocks
if block.group_idx == 1
)
store_horizon_chunks = store_horizon_tokens // swa_block_size
alignment_chunks = full_attn_block_size // swa_block_size
sliding_window_chunks = sliding_window // swa_block_size
expected_swa_chunks = []
for chunk in range(store_horizon_chunks):
segment_start = chunk - chunk % alignment_chunks
segment_length = min(alignment_chunks, store_horizon_chunks - segment_start)
if chunk % alignment_chunks >= segment_length - sliding_window_chunks:
expected_swa_chunks.append(chunk)
assert stored_swa_chunks == expected_swa_chunks
@pytest.mark.skip_global_cleanup
def test_active_decode_does_not_advance_swa_final_horizon(self, request_runner):
"""Active decode chunks are not final partial SWA segments."""
full_attn_block_size = 36
swa_block_size = 4
prompt_tokens = 1200
kv_cache_groups = [
KVCacheGroupSpec(
["layer0"],
FullAttentionSpec(
block_size=full_attn_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["layer1"],
SlidingWindowSpec(
block_size=swa_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=16,
),
),
]
runner = request_runner(
block_size=swa_block_size,
num_gpu_blocks=1000,
async_scheduling=False,
kv_cache_groups=kv_cache_groups,
)
runner.new_request(token_ids=[0] * prompt_tokens)
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
runner._run([1] * 12, complete_transfers=True)
assert runner.scheduler.running
stored_swa_chunks = {
block.request_block_offset
for transfer in runner.completed_stores
for block in transfer.gpu_blocks
if block.group_idx == 1
}
first_decode_chunk = prompt_tokens // swa_block_size
assert first_decode_chunk not in stored_swa_chunks
def test_dcp_abort_reconsiders_final_swa_tail(self):
"""DCP-expanded SWA blocks are not scaled twice during abort."""
vllm_config = _make_vllm_config(
extra_config={"offload_prompt_only": False},
tensor_parallel_size=2,
decode_context_parallel_size=2,
)
vllm_config.speculative_config = None
vllm_config.kv_events_config = KVEventsConfig(
enable_kv_cache_events=True, publisher="null"
)
kv_cache_config = KVCacheConfig(
num_blocks=0,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=36,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["swa"],
SlidingWindowSpec(
block_size=4,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=16,
),
),
],
)
spec = MockOffloadingSpec(build_offloading_config(vllm_config, kv_cache_config))
scheduler = OffloadingConnectorScheduler(spec, vllm_config, kv_cache_config)
scheduler.manager.prepare_store.side_effect = (
lambda keys, req_context: generate_store_output(keys)
)
request = MagicMock()
request.request_id = "req"
request.kv_transfer_params = None
request.num_prompt_tokens = 1200
request.num_tokens = 1200
request.num_computed_tokens = 0
request.block_hashes = [BlockHash(f"h{i}".encode()) for i in range(150)]
request.all_token_ids = list(range(1200))
request.lora_request = None
request.shared_prefix_boundary = 0
request.status = RequestStatus.RUNNING
request.is_finished.return_value = False
scheduler.on_new_request(request)
full_block_ids = list(range(1, 15))
swa_block_ids = list(range(1001, 1126))
first_output = SchedulerOutput.make_empty()
first_output.scheduled_new_reqs = [
SimpleNamespace(req_id="req", block_ids=(full_block_ids, swa_block_ids))
]
first_output.num_scheduled_tokens = {"req": 1000}
first_output.total_num_scheduled_tokens = 1000
scheduler.build_connector_meta(first_output)
request.num_computed_tokens = 1000
request.status = RequestStatus.FINISHED_ABORTED
request.is_finished.return_value = True
abort_output = SchedulerOutput.make_empty()
abort_output.finished_req_ids = {"req"}
abort_meta = scheduler.build_connector_meta(abort_output)
assert len(abort_meta.store_jobs) == 1
[store_job] = abort_meta.store_jobs.values()
assert isinstance(store_job.src_spec, GPULoadStoreSpec)
assert store_job.src_spec.block_ids.tolist() == [swa_block_ids[123]]
assert store_job.src_spec.group_sizes == [0, 1]
assert store_job.src_spec.block_indices == [0, 123]
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_full_attn_store_then_load(self, request_runner, async_scheduling: bool):
"""Eagle group constrains load: convergence tightens both groups.
@@ -1326,6 +1326,46 @@ class OffloadingConnectorScheduler:
return store_jobs
def _reachable_store_block_mask(
self,
group_config: GroupOffloadConfig,
start_chunk_idx: int,
end_chunk_idx: int,
final_segment_end_chunk_idx: int | None,
reachable_boundaries: tuple[int, ...],
) -> list[bool] | None:
"""Build the block mask for a range of candidate offload chunks."""
blocks_per_chunk = self.config.blocks_per_chunk
return group_config.manager_cls.reachable_block_mask(
start_block=start_chunk_idx * blocks_per_chunk,
end_block=end_chunk_idx * blocks_per_chunk,
alignment_tokens=self.config.alignment_tokens,
kv_cache_spec=group_config.kv_cache_spec,
use_eagle=group_config.is_eagle_group,
retention_interval=self.config.retention_interval,
reachable_boundaries=reachable_boundaries,
dcp_world_size=self.config.dcp_world_size,
final_segment_end_block=(
final_segment_end_chunk_idx * blocks_per_chunk
if final_segment_end_chunk_idx is not None
else None
),
)
def _final_swa_alignment_blocks(
self, group_config: GroupOffloadConfig
) -> int | None:
"""Return representable final-tail alignment for a non-EAGLE SWA group."""
alignment_tokens = self.config.alignment_tokens
if (
alignment_tokens is None
or group_config.is_eagle_group
or not isinstance(group_config.kv_cache_spec, SlidingWindowSpec)
or alignment_tokens % group_config.tokens_per_block != 0
):
return None
return alignment_tokens // group_config.tokens_per_block
def _build_store_jobs(
self,
scheduler_output: SchedulerOutput,
@@ -1352,10 +1392,14 @@ class OffloadingConnectorScheduler:
num_offloadable_tokens = self._calc_num_offloadable_tokens(
req_status, num_tokens_after_batch
)
prompt_offloadable_tokens = self._calc_num_offloadable_tokens(
req_status, req.num_prompt_tokens
)
# Filter out chunks skipped due to sliding window attention / SSM
# or unreachable by the load path's alignment constraints.
new_offload_keys: list[OffloadKey] = []
group_store_ranges: list[tuple[int, int]] = []
reachable_boundaries: tuple[int, ...] = ()
if self.config.retention_interval is not None:
@@ -1370,10 +1414,47 @@ class OffloadingConnectorScheduler:
group_config, group_state, num_offloadable_tokens
)
start_chunk_idx = group_state.next_stored_chunk_idx
prompt_horizon_chunks = (
prompt_offloadable_tokens // group_config.tokens_per_chunk
)
final_swa_alignment_blocks = self._final_swa_alignment_blocks(
group_config
)
reconsider_final_swa_tail = (
req.is_finished()
and num_chunks != prompt_horizon_chunks
and self.config.retention_interval is None
and final_swa_alignment_blocks is not None
)
if (
self.config.retention_interval is not None
or group_config.is_eagle_group
):
store_horizon_chunks = None
elif req.is_finished():
store_horizon_chunks = num_chunks
elif num_chunks <= prompt_horizon_chunks:
store_horizon_chunks = prompt_horizon_chunks
else:
# An active decode frontier is not a final request
# boundary. Only fixed alignment tails are reachable.
store_horizon_chunks = None
if reconsider_final_swa_tail:
assert final_swa_alignment_blocks is not None
horizon_blocks = num_chunks * blocks_per_chunk
partial_segment_start_block = (
horizon_blocks - horizon_blocks % final_swa_alignment_blocks
)
partial_segment_start_chunk = (
partial_segment_start_block // blocks_per_chunk
)
start_chunk_idx = min(start_chunk_idx, partial_segment_start_chunk)
group_store_ranges.append((start_chunk_idx, num_chunks))
if group_config.requires_cow_source:
continue
start_chunk_idx = group_state.next_stored_chunk_idx
if num_chunks <= start_chunk_idx:
continue
offload_keys = group_state.offload_keys[start_chunk_idx:num_chunks]
@@ -1393,22 +1474,44 @@ class OffloadingConnectorScheduler:
# (SWA/Mamba sparsity + retention interval).
# reachable_block_mask operates in KV-block coordinates,
# so convert chunk indices to block indices.
block_mask = group_config.manager_cls.reachable_block_mask(
start_block=start_chunk_idx * blocks_per_chunk,
end_block=num_chunks * blocks_per_chunk,
alignment_tokens=self.config.alignment_tokens,
kv_cache_spec=group_config.kv_cache_spec,
use_eagle=group_config.is_eagle_group,
retention_interval=self.config.retention_interval,
block_mask = self._reachable_store_block_mask(
group_config=group_config,
start_chunk_idx=start_chunk_idx,
end_chunk_idx=num_chunks,
final_segment_end_chunk_idx=store_horizon_chunks,
reachable_boundaries=reachable_boundaries,
dcp_world_size=self.config.dcp_world_size,
)
prompt_horizon_block_mask: list[bool] | None = None
if reconsider_final_swa_tail:
prompt_horizon_block_mask = self._reachable_store_block_mask(
group_config=group_config,
start_chunk_idx=start_chunk_idx,
end_chunk_idx=num_chunks,
final_segment_end_chunk_idx=prompt_horizon_chunks,
reachable_boundaries=reachable_boundaries,
)
for key_idx, (offload_key, block_id) in enumerate(
zip(offload_keys, offload_block_ids)
):
if block_id == 0:
continue
abs_chunk_idx = start_chunk_idx + key_idx
if (
reconsider_final_swa_tail
and abs_chunk_idx < group_state.next_stored_chunk_idx
and (
prompt_horizon_block_mask is None
or any(
prompt_horizon_block_mask[
key_idx * blocks_per_chunk + block_idx
]
for block_idx in range(blocks_per_chunk)
)
)
):
continue
# A chunk is reachable if any of its constituent
# blocks is reachable.
if block_mask is not None and not any(
@@ -1445,16 +1548,15 @@ class OffloadingConnectorScheduler:
src_block_ids: list[int] = []
fenced_block_ids: list[int] = []
deferred_fence_block_ids: list[int] = []
for group_config, group_state in zip(
self.config.kv_group_configs, req_status.group_states
for group_config, group_state, store_range in zip(
self.config.kv_group_configs,
req_status.group_states,
group_store_ranges,
):
is_sliding_window = (
group_config.sliding_window_size_in_chunks is not None
)
num_chunks = req_status.storable_chunks(
group_config, group_state, num_offloadable_tokens
)
start_chunk_idx = group_state.next_stored_chunk_idx
start_chunk_idx, num_chunks = store_range
block_ids = group_state.block_ids
num_group_blocks = 0
start_gpu_block_idx: int | None = None
@@ -508,6 +508,7 @@ class SingleTypeKVCacheManager(ABC):
retention_interval: int | None = None,
reachable_boundaries: Sequence[int] = (),
dcp_world_size: int = 1,
final_segment_end_block: int | None = None,
) -> list[bool] | None:
"""Per-block mask for ``cache_full_blocks``. ``None`` means cache
every (non-null) block — the default for full attention.
@@ -516,6 +517,10 @@ class SingleTypeKVCacheManager(ABC):
blocks that can never serve a hit at any alignment-aligned prefix length.
``reachable_boundaries`` are token positions whose reachable tail must be
retained; the base (dense) policy ignores them.
``final_segment_end_block`` is the exclusive end of the request's final
segment. It may be later than ``end_block`` while prefill is still in
progress. Non-EAGLE sparse managers may keep its reachable tail. The
base policy already keeps every block, so it ignores this value.
"""
return None
@@ -1033,6 +1038,7 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
retention_interval: int | None = None,
reachable_boundaries: Sequence[int] = (),
dcp_world_size: int = 1,
final_segment_end_block: int | None = None,
) -> list[bool] | None:
assert isinstance(kv_cache_spec, SlidingWindowSpec)
if alignment_tokens is None:
@@ -1079,6 +1085,17 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
if i >= shift and (i - shift) % per_segment >= per_segment - need:
mask[i - start_block] = True
if final_segment_end_block is not None and not use_eagle:
final_segment_size = final_segment_end_block % per_segment
if final_segment_size:
final_segment_start = final_segment_end_block - final_segment_size
final_tail_start = max(
final_segment_start, final_segment_end_block - need
)
final_tail_end = min(end_block, final_segment_end_block)
for i in range(max(start_block, final_tail_start), final_tail_end):
mask[i - start_block] = True
# (2) Reachable-boundary tails: the replay boundary (``num_prompt - 1``,
# capped by ``get_computed_blocks``) and any shared-prefix junction. Both
# land before segments would cover them under sparse retention, so keep
@@ -1513,6 +1530,7 @@ class MambaManager(SingleTypeKVCacheManager):
retention_interval: int | None = None,
reachable_boundaries: Sequence[int] = (),
dcp_world_size: int = 1,
final_segment_end_block: int | None = None,
) -> list[bool] | None:
"""Sparse Mamba state-snapshot retention.