[Bugfix][KV Offload] Fix SWA store reachability during chunked prefill (#54362)
Signed-off-by: Will Hampson <whamp@ggl.slmail.me> Co-authored-by: Will Hampson <whamp@ggl.slmail.me> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
Will Hampson
Or Ozeri
parent
dd0760165c
commit
569adb5a97
@@ -4389,6 +4389,41 @@ def test_mamba_shared_prefix_reuse_under_zero_retention():
|
||||
assert last_req_hit(retention=0, pin=True) == 2 * block_size
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_swa_reachable_block_mask_final_partial_segment():
|
||||
"""Only a final SWA horizon makes its incomplete segment tail reachable."""
|
||||
from vllm.v1.core.single_type_kv_cache_manager import SlidingWindowManager
|
||||
|
||||
spec = SlidingWindowSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=16,
|
||||
)
|
||||
start_block = 243
|
||||
end_block = 250
|
||||
|
||||
def reachable_blocks(
|
||||
final_segment_end_block: int, *, use_eagle: bool = False
|
||||
) -> set[int]:
|
||||
mask = SlidingWindowManager.reachable_block_mask(
|
||||
start_block=start_block,
|
||||
end_block=end_block,
|
||||
alignment_tokens=36,
|
||||
kv_cache_spec=spec,
|
||||
use_eagle=use_eagle,
|
||||
final_segment_end_block=final_segment_end_block,
|
||||
)
|
||||
assert mask is not None
|
||||
return {start_block + i for i, reachable in enumerate(mask) if reachable}
|
||||
|
||||
assert reachable_blocks(300) == {248, 249}
|
||||
assert reachable_blocks(250) == {246, 247, 248, 249}
|
||||
assert reachable_blocks(300, use_eagle=True) == {243, 248, 249}
|
||||
assert reachable_blocks(250, use_eagle=True) == {243, 248, 249}
|
||||
|
||||
|
||||
def test_swa_reachable_block_mask_pins_shared_prefix():
|
||||
"""SWA analog of the Mamba pin: the shared-prefix junction must keep the
|
||||
``need``-block sliding-window tail ending on that boundary (not a single
|
||||
|
||||
@@ -48,6 +48,7 @@ from vllm.v1.core.sched.output import (
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
@@ -3390,6 +3391,220 @@ class TestEagle:
|
||||
f"interior hole in stored blocks: {offsets}"
|
||||
)
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.parametrize(
|
||||
"abort_after_first_step,async_scheduling,store_horizon_tokens",
|
||||
[
|
||||
pytest.param(False, False, 1200, id="completed-prompt"),
|
||||
pytest.param(True, False, 1000, id="aborted-sync"),
|
||||
pytest.param(True, True, 1000, id="aborted-async"),
|
||||
],
|
||||
)
|
||||
def test_chunked_prefill_uses_final_swa_store_horizon(
|
||||
self,
|
||||
request_runner,
|
||||
abort_after_first_step: bool,
|
||||
async_scheduling: bool,
|
||||
store_horizon_tokens: int,
|
||||
):
|
||||
"""SWA stores must use the completed or aborted request horizon."""
|
||||
full_attn_block_size = 36
|
||||
swa_block_size = 4
|
||||
sliding_window = 16
|
||||
num_tokens = 1200
|
||||
|
||||
kv_cache_groups = [
|
||||
KVCacheGroupSpec(
|
||||
["layer0"],
|
||||
FullAttentionSpec(
|
||||
block_size=full_attn_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["layer1"],
|
||||
SlidingWindowSpec(
|
||||
block_size=swa_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=sliding_window,
|
||||
),
|
||||
),
|
||||
]
|
||||
runner = request_runner(
|
||||
block_size=swa_block_size,
|
||||
num_gpu_blocks=1000,
|
||||
async_scheduling=async_scheduling,
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
)
|
||||
|
||||
runner.new_request(token_ids=[0] * num_tokens)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
|
||||
def abort_after_prefill_step() -> None:
|
||||
if runner.scheduler.running:
|
||||
runner.scheduler.finish_requests(
|
||||
(str(runner.req_id),), RequestStatus.FINISHED_ABORTED
|
||||
)
|
||||
|
||||
runner._run(
|
||||
[1] if abort_after_first_step else [1, 1, EOS_TOKEN_ID],
|
||||
complete_transfers=True,
|
||||
post_step_fn=(abort_after_prefill_step if abort_after_first_step else None),
|
||||
)
|
||||
|
||||
stored_swa_chunks = sorted(
|
||||
block.request_block_offset
|
||||
for transfer in runner.completed_stores
|
||||
for block in transfer.gpu_blocks
|
||||
if block.group_idx == 1
|
||||
)
|
||||
store_horizon_chunks = store_horizon_tokens // swa_block_size
|
||||
alignment_chunks = full_attn_block_size // swa_block_size
|
||||
sliding_window_chunks = sliding_window // swa_block_size
|
||||
expected_swa_chunks = []
|
||||
for chunk in range(store_horizon_chunks):
|
||||
segment_start = chunk - chunk % alignment_chunks
|
||||
segment_length = min(alignment_chunks, store_horizon_chunks - segment_start)
|
||||
if chunk % alignment_chunks >= segment_length - sliding_window_chunks:
|
||||
expected_swa_chunks.append(chunk)
|
||||
assert stored_swa_chunks == expected_swa_chunks
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_active_decode_does_not_advance_swa_final_horizon(self, request_runner):
|
||||
"""Active decode chunks are not final partial SWA segments."""
|
||||
full_attn_block_size = 36
|
||||
swa_block_size = 4
|
||||
prompt_tokens = 1200
|
||||
kv_cache_groups = [
|
||||
KVCacheGroupSpec(
|
||||
["layer0"],
|
||||
FullAttentionSpec(
|
||||
block_size=full_attn_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["layer1"],
|
||||
SlidingWindowSpec(
|
||||
block_size=swa_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=16,
|
||||
),
|
||||
),
|
||||
]
|
||||
runner = request_runner(
|
||||
block_size=swa_block_size,
|
||||
num_gpu_blocks=1000,
|
||||
async_scheduling=False,
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
)
|
||||
runner.new_request(token_ids=[0] * prompt_tokens)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
|
||||
runner._run([1] * 12, complete_transfers=True)
|
||||
|
||||
assert runner.scheduler.running
|
||||
stored_swa_chunks = {
|
||||
block.request_block_offset
|
||||
for transfer in runner.completed_stores
|
||||
for block in transfer.gpu_blocks
|
||||
if block.group_idx == 1
|
||||
}
|
||||
first_decode_chunk = prompt_tokens // swa_block_size
|
||||
assert first_decode_chunk not in stored_swa_chunks
|
||||
|
||||
def test_dcp_abort_reconsiders_final_swa_tail(self):
|
||||
"""DCP-expanded SWA blocks are not scaled twice during abort."""
|
||||
vllm_config = _make_vllm_config(
|
||||
extra_config={"offload_prompt_only": False},
|
||||
tensor_parallel_size=2,
|
||||
decode_context_parallel_size=2,
|
||||
)
|
||||
vllm_config.speculative_config = None
|
||||
vllm_config.kv_events_config = KVEventsConfig(
|
||||
enable_kv_cache_events=True, publisher="null"
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=0,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=36,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["swa"],
|
||||
SlidingWindowSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=16,
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
spec = MockOffloadingSpec(build_offloading_config(vllm_config, kv_cache_config))
|
||||
scheduler = OffloadingConnectorScheduler(spec, vllm_config, kv_cache_config)
|
||||
scheduler.manager.prepare_store.side_effect = (
|
||||
lambda keys, req_context: generate_store_output(keys)
|
||||
)
|
||||
|
||||
request = MagicMock()
|
||||
request.request_id = "req"
|
||||
request.kv_transfer_params = None
|
||||
request.num_prompt_tokens = 1200
|
||||
request.num_tokens = 1200
|
||||
request.num_computed_tokens = 0
|
||||
request.block_hashes = [BlockHash(f"h{i}".encode()) for i in range(150)]
|
||||
request.all_token_ids = list(range(1200))
|
||||
request.lora_request = None
|
||||
request.shared_prefix_boundary = 0
|
||||
request.status = RequestStatus.RUNNING
|
||||
request.is_finished.return_value = False
|
||||
scheduler.on_new_request(request)
|
||||
|
||||
full_block_ids = list(range(1, 15))
|
||||
swa_block_ids = list(range(1001, 1126))
|
||||
first_output = SchedulerOutput.make_empty()
|
||||
first_output.scheduled_new_reqs = [
|
||||
SimpleNamespace(req_id="req", block_ids=(full_block_ids, swa_block_ids))
|
||||
]
|
||||
first_output.num_scheduled_tokens = {"req": 1000}
|
||||
first_output.total_num_scheduled_tokens = 1000
|
||||
scheduler.build_connector_meta(first_output)
|
||||
|
||||
request.num_computed_tokens = 1000
|
||||
request.status = RequestStatus.FINISHED_ABORTED
|
||||
request.is_finished.return_value = True
|
||||
abort_output = SchedulerOutput.make_empty()
|
||||
abort_output.finished_req_ids = {"req"}
|
||||
abort_meta = scheduler.build_connector_meta(abort_output)
|
||||
|
||||
assert len(abort_meta.store_jobs) == 1
|
||||
[store_job] = abort_meta.store_jobs.values()
|
||||
assert isinstance(store_job.src_spec, GPULoadStoreSpec)
|
||||
assert store_job.src_spec.block_ids.tolist() == [swa_block_ids[123]]
|
||||
assert store_job.src_spec.group_sizes == [0, 1]
|
||||
assert store_job.src_spec.block_indices == [0, 123]
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_full_attn_store_then_load(self, request_runner, async_scheduling: bool):
|
||||
"""Eagle group constrains load: convergence tightens both groups.
|
||||
|
||||
@@ -1326,6 +1326,46 @@ class OffloadingConnectorScheduler:
|
||||
|
||||
return store_jobs
|
||||
|
||||
def _reachable_store_block_mask(
|
||||
self,
|
||||
group_config: GroupOffloadConfig,
|
||||
start_chunk_idx: int,
|
||||
end_chunk_idx: int,
|
||||
final_segment_end_chunk_idx: int | None,
|
||||
reachable_boundaries: tuple[int, ...],
|
||||
) -> list[bool] | None:
|
||||
"""Build the block mask for a range of candidate offload chunks."""
|
||||
blocks_per_chunk = self.config.blocks_per_chunk
|
||||
return group_config.manager_cls.reachable_block_mask(
|
||||
start_block=start_chunk_idx * blocks_per_chunk,
|
||||
end_block=end_chunk_idx * blocks_per_chunk,
|
||||
alignment_tokens=self.config.alignment_tokens,
|
||||
kv_cache_spec=group_config.kv_cache_spec,
|
||||
use_eagle=group_config.is_eagle_group,
|
||||
retention_interval=self.config.retention_interval,
|
||||
reachable_boundaries=reachable_boundaries,
|
||||
dcp_world_size=self.config.dcp_world_size,
|
||||
final_segment_end_block=(
|
||||
final_segment_end_chunk_idx * blocks_per_chunk
|
||||
if final_segment_end_chunk_idx is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
def _final_swa_alignment_blocks(
|
||||
self, group_config: GroupOffloadConfig
|
||||
) -> int | None:
|
||||
"""Return representable final-tail alignment for a non-EAGLE SWA group."""
|
||||
alignment_tokens = self.config.alignment_tokens
|
||||
if (
|
||||
alignment_tokens is None
|
||||
or group_config.is_eagle_group
|
||||
or not isinstance(group_config.kv_cache_spec, SlidingWindowSpec)
|
||||
or alignment_tokens % group_config.tokens_per_block != 0
|
||||
):
|
||||
return None
|
||||
return alignment_tokens // group_config.tokens_per_block
|
||||
|
||||
def _build_store_jobs(
|
||||
self,
|
||||
scheduler_output: SchedulerOutput,
|
||||
@@ -1352,10 +1392,14 @@ class OffloadingConnectorScheduler:
|
||||
num_offloadable_tokens = self._calc_num_offloadable_tokens(
|
||||
req_status, num_tokens_after_batch
|
||||
)
|
||||
prompt_offloadable_tokens = self._calc_num_offloadable_tokens(
|
||||
req_status, req.num_prompt_tokens
|
||||
)
|
||||
|
||||
# Filter out chunks skipped due to sliding window attention / SSM
|
||||
# or unreachable by the load path's alignment constraints.
|
||||
new_offload_keys: list[OffloadKey] = []
|
||||
group_store_ranges: list[tuple[int, int]] = []
|
||||
|
||||
reachable_boundaries: tuple[int, ...] = ()
|
||||
if self.config.retention_interval is not None:
|
||||
@@ -1370,10 +1414,47 @@ class OffloadingConnectorScheduler:
|
||||
group_config, group_state, num_offloadable_tokens
|
||||
)
|
||||
|
||||
start_chunk_idx = group_state.next_stored_chunk_idx
|
||||
prompt_horizon_chunks = (
|
||||
prompt_offloadable_tokens // group_config.tokens_per_chunk
|
||||
)
|
||||
final_swa_alignment_blocks = self._final_swa_alignment_blocks(
|
||||
group_config
|
||||
)
|
||||
reconsider_final_swa_tail = (
|
||||
req.is_finished()
|
||||
and num_chunks != prompt_horizon_chunks
|
||||
and self.config.retention_interval is None
|
||||
and final_swa_alignment_blocks is not None
|
||||
)
|
||||
if (
|
||||
self.config.retention_interval is not None
|
||||
or group_config.is_eagle_group
|
||||
):
|
||||
store_horizon_chunks = None
|
||||
elif req.is_finished():
|
||||
store_horizon_chunks = num_chunks
|
||||
elif num_chunks <= prompt_horizon_chunks:
|
||||
store_horizon_chunks = prompt_horizon_chunks
|
||||
else:
|
||||
# An active decode frontier is not a final request
|
||||
# boundary. Only fixed alignment tails are reachable.
|
||||
store_horizon_chunks = None
|
||||
if reconsider_final_swa_tail:
|
||||
assert final_swa_alignment_blocks is not None
|
||||
horizon_blocks = num_chunks * blocks_per_chunk
|
||||
partial_segment_start_block = (
|
||||
horizon_blocks - horizon_blocks % final_swa_alignment_blocks
|
||||
)
|
||||
partial_segment_start_chunk = (
|
||||
partial_segment_start_block // blocks_per_chunk
|
||||
)
|
||||
start_chunk_idx = min(start_chunk_idx, partial_segment_start_chunk)
|
||||
group_store_ranges.append((start_chunk_idx, num_chunks))
|
||||
|
||||
if group_config.requires_cow_source:
|
||||
continue
|
||||
|
||||
start_chunk_idx = group_state.next_stored_chunk_idx
|
||||
if num_chunks <= start_chunk_idx:
|
||||
continue
|
||||
offload_keys = group_state.offload_keys[start_chunk_idx:num_chunks]
|
||||
@@ -1393,22 +1474,44 @@ class OffloadingConnectorScheduler:
|
||||
# (SWA/Mamba sparsity + retention interval).
|
||||
# reachable_block_mask operates in KV-block coordinates,
|
||||
# so convert chunk indices to block indices.
|
||||
block_mask = group_config.manager_cls.reachable_block_mask(
|
||||
start_block=start_chunk_idx * blocks_per_chunk,
|
||||
end_block=num_chunks * blocks_per_chunk,
|
||||
alignment_tokens=self.config.alignment_tokens,
|
||||
kv_cache_spec=group_config.kv_cache_spec,
|
||||
use_eagle=group_config.is_eagle_group,
|
||||
retention_interval=self.config.retention_interval,
|
||||
block_mask = self._reachable_store_block_mask(
|
||||
group_config=group_config,
|
||||
start_chunk_idx=start_chunk_idx,
|
||||
end_chunk_idx=num_chunks,
|
||||
final_segment_end_chunk_idx=store_horizon_chunks,
|
||||
reachable_boundaries=reachable_boundaries,
|
||||
dcp_world_size=self.config.dcp_world_size,
|
||||
)
|
||||
|
||||
prompt_horizon_block_mask: list[bool] | None = None
|
||||
if reconsider_final_swa_tail:
|
||||
prompt_horizon_block_mask = self._reachable_store_block_mask(
|
||||
group_config=group_config,
|
||||
start_chunk_idx=start_chunk_idx,
|
||||
end_chunk_idx=num_chunks,
|
||||
final_segment_end_chunk_idx=prompt_horizon_chunks,
|
||||
reachable_boundaries=reachable_boundaries,
|
||||
)
|
||||
|
||||
for key_idx, (offload_key, block_id) in enumerate(
|
||||
zip(offload_keys, offload_block_ids)
|
||||
):
|
||||
if block_id == 0:
|
||||
continue
|
||||
abs_chunk_idx = start_chunk_idx + key_idx
|
||||
if (
|
||||
reconsider_final_swa_tail
|
||||
and abs_chunk_idx < group_state.next_stored_chunk_idx
|
||||
and (
|
||||
prompt_horizon_block_mask is None
|
||||
or any(
|
||||
prompt_horizon_block_mask[
|
||||
key_idx * blocks_per_chunk + block_idx
|
||||
]
|
||||
for block_idx in range(blocks_per_chunk)
|
||||
)
|
||||
)
|
||||
):
|
||||
continue
|
||||
# A chunk is reachable if any of its constituent
|
||||
# blocks is reachable.
|
||||
if block_mask is not None and not any(
|
||||
@@ -1445,16 +1548,15 @@ class OffloadingConnectorScheduler:
|
||||
src_block_ids: list[int] = []
|
||||
fenced_block_ids: list[int] = []
|
||||
deferred_fence_block_ids: list[int] = []
|
||||
for group_config, group_state in zip(
|
||||
self.config.kv_group_configs, req_status.group_states
|
||||
for group_config, group_state, store_range in zip(
|
||||
self.config.kv_group_configs,
|
||||
req_status.group_states,
|
||||
group_store_ranges,
|
||||
):
|
||||
is_sliding_window = (
|
||||
group_config.sliding_window_size_in_chunks is not None
|
||||
)
|
||||
num_chunks = req_status.storable_chunks(
|
||||
group_config, group_state, num_offloadable_tokens
|
||||
)
|
||||
start_chunk_idx = group_state.next_stored_chunk_idx
|
||||
start_chunk_idx, num_chunks = store_range
|
||||
block_ids = group_state.block_ids
|
||||
num_group_blocks = 0
|
||||
start_gpu_block_idx: int | None = None
|
||||
|
||||
@@ -508,6 +508,7 @@ class SingleTypeKVCacheManager(ABC):
|
||||
retention_interval: int | None = None,
|
||||
reachable_boundaries: Sequence[int] = (),
|
||||
dcp_world_size: int = 1,
|
||||
final_segment_end_block: int | None = None,
|
||||
) -> list[bool] | None:
|
||||
"""Per-block mask for ``cache_full_blocks``. ``None`` means cache
|
||||
every (non-null) block — the default for full attention.
|
||||
@@ -516,6 +517,10 @@ class SingleTypeKVCacheManager(ABC):
|
||||
blocks that can never serve a hit at any alignment-aligned prefix length.
|
||||
``reachable_boundaries`` are token positions whose reachable tail must be
|
||||
retained; the base (dense) policy ignores them.
|
||||
``final_segment_end_block`` is the exclusive end of the request's final
|
||||
segment. It may be later than ``end_block`` while prefill is still in
|
||||
progress. Non-EAGLE sparse managers may keep its reachable tail. The
|
||||
base policy already keeps every block, so it ignores this value.
|
||||
"""
|
||||
return None
|
||||
|
||||
@@ -1033,6 +1038,7 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
|
||||
retention_interval: int | None = None,
|
||||
reachable_boundaries: Sequence[int] = (),
|
||||
dcp_world_size: int = 1,
|
||||
final_segment_end_block: int | None = None,
|
||||
) -> list[bool] | None:
|
||||
assert isinstance(kv_cache_spec, SlidingWindowSpec)
|
||||
if alignment_tokens is None:
|
||||
@@ -1079,6 +1085,17 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
|
||||
if i >= shift and (i - shift) % per_segment >= per_segment - need:
|
||||
mask[i - start_block] = True
|
||||
|
||||
if final_segment_end_block is not None and not use_eagle:
|
||||
final_segment_size = final_segment_end_block % per_segment
|
||||
if final_segment_size:
|
||||
final_segment_start = final_segment_end_block - final_segment_size
|
||||
final_tail_start = max(
|
||||
final_segment_start, final_segment_end_block - need
|
||||
)
|
||||
final_tail_end = min(end_block, final_segment_end_block)
|
||||
for i in range(max(start_block, final_tail_start), final_tail_end):
|
||||
mask[i - start_block] = True
|
||||
|
||||
# (2) Reachable-boundary tails: the replay boundary (``num_prompt - 1``,
|
||||
# capped by ``get_computed_blocks``) and any shared-prefix junction. Both
|
||||
# land before segments would cover them under sparse retention, so keep
|
||||
@@ -1513,6 +1530,7 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
retention_interval: int | None = None,
|
||||
reachable_boundaries: Sequence[int] = (),
|
||||
dcp_world_size: int = 1,
|
||||
final_segment_end_block: int | None = None,
|
||||
) -> list[bool] | None:
|
||||
"""Sparse Mamba state-snapshot retention.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user