From 569adb5a9780f9c02d22a6b29826acf711512356 Mon Sep 17 00:00:00 2001 From: Will Hampson Date: Sun, 6 Sep 2026 03:50:39 -0700 Subject: [PATCH] [Bugfix][KV Offload] Fix SWA store reachability during chunked prefill (#54362) Signed-off-by: Will Hampson Co-authored-by: Will Hampson Co-authored-by: Or Ozeri --- tests/v1/core/test_prefix_caching.py | 35 +++ .../offloading_connector/test_scheduler.py | 215 ++++++++++++++++++ .../kv_connector/v1/offloading/scheduler.py | 132 +++++++++-- vllm/v1/core/single_type_kv_cache_manager.py | 18 ++ 4 files changed, 385 insertions(+), 15 deletions(-) diff --git a/tests/v1/core/test_prefix_caching.py b/tests/v1/core/test_prefix_caching.py index bcb6457d8c..e2c172b79c 100644 --- a/tests/v1/core/test_prefix_caching.py +++ b/tests/v1/core/test_prefix_caching.py @@ -4389,6 +4389,41 @@ def test_mamba_shared_prefix_reuse_under_zero_retention(): assert last_req_hit(retention=0, pin=True) == 2 * block_size +@pytest.mark.skip_global_cleanup +def test_swa_reachable_block_mask_final_partial_segment(): + """Only a final SWA horizon makes its incomplete segment tail reachable.""" + from vllm.v1.core.single_type_kv_cache_manager import SlidingWindowManager + + spec = SlidingWindowSpec( + block_size=4, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + sliding_window=16, + ) + start_block = 243 + end_block = 250 + + def reachable_blocks( + final_segment_end_block: int, *, use_eagle: bool = False + ) -> set[int]: + mask = SlidingWindowManager.reachable_block_mask( + start_block=start_block, + end_block=end_block, + alignment_tokens=36, + kv_cache_spec=spec, + use_eagle=use_eagle, + final_segment_end_block=final_segment_end_block, + ) + assert mask is not None + return {start_block + i for i, reachable in enumerate(mask) if reachable} + + assert reachable_blocks(300) == {248, 249} + assert reachable_blocks(250) == {246, 247, 248, 249} + assert reachable_blocks(300, use_eagle=True) == {243, 248, 249} + assert reachable_blocks(250, use_eagle=True) == {243, 248, 249} + + def test_swa_reachable_block_mask_pins_shared_prefix(): """SWA analog of the Mamba pin: the shared-prefix junction must keep the ``need``-block sliding-window tail ending on that boundary (not a single diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py index 69c39efcc3..eb3b0f4175 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py @@ -48,6 +48,7 @@ from vllm.v1.core.sched.output import ( from vllm.v1.kv_cache_interface import ( ChunkedLocalAttentionSpec, FullAttentionSpec, + KVCacheConfig, KVCacheGroupSpec, SlidingWindowSpec, ) @@ -3390,6 +3391,220 @@ class TestEagle: f"interior hole in stored blocks: {offsets}" ) + @pytest.mark.skip_global_cleanup + @pytest.mark.parametrize( + "abort_after_first_step,async_scheduling,store_horizon_tokens", + [ + pytest.param(False, False, 1200, id="completed-prompt"), + pytest.param(True, False, 1000, id="aborted-sync"), + pytest.param(True, True, 1000, id="aborted-async"), + ], + ) + def test_chunked_prefill_uses_final_swa_store_horizon( + self, + request_runner, + abort_after_first_step: bool, + async_scheduling: bool, + store_horizon_tokens: int, + ): + """SWA stores must use the completed or aborted request horizon.""" + full_attn_block_size = 36 + swa_block_size = 4 + sliding_window = 16 + num_tokens = 1200 + + kv_cache_groups = [ + KVCacheGroupSpec( + ["layer0"], + FullAttentionSpec( + block_size=full_attn_block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + ), + KVCacheGroupSpec( + ["layer1"], + SlidingWindowSpec( + block_size=swa_block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + sliding_window=sliding_window, + ), + ), + ] + runner = request_runner( + block_size=swa_block_size, + num_gpu_blocks=1000, + async_scheduling=async_scheduling, + kv_cache_groups=kv_cache_groups, + ) + + runner.new_request(token_ids=[0] * num_tokens) + runner.manager.prepare_store.side_effect = lambda keys, req_context: ( + generate_store_output(keys) + ) + + def abort_after_prefill_step() -> None: + if runner.scheduler.running: + runner.scheduler.finish_requests( + (str(runner.req_id),), RequestStatus.FINISHED_ABORTED + ) + + runner._run( + [1] if abort_after_first_step else [1, 1, EOS_TOKEN_ID], + complete_transfers=True, + post_step_fn=(abort_after_prefill_step if abort_after_first_step else None), + ) + + stored_swa_chunks = sorted( + block.request_block_offset + for transfer in runner.completed_stores + for block in transfer.gpu_blocks + if block.group_idx == 1 + ) + store_horizon_chunks = store_horizon_tokens // swa_block_size + alignment_chunks = full_attn_block_size // swa_block_size + sliding_window_chunks = sliding_window // swa_block_size + expected_swa_chunks = [] + for chunk in range(store_horizon_chunks): + segment_start = chunk - chunk % alignment_chunks + segment_length = min(alignment_chunks, store_horizon_chunks - segment_start) + if chunk % alignment_chunks >= segment_length - sliding_window_chunks: + expected_swa_chunks.append(chunk) + assert stored_swa_chunks == expected_swa_chunks + + @pytest.mark.skip_global_cleanup + def test_active_decode_does_not_advance_swa_final_horizon(self, request_runner): + """Active decode chunks are not final partial SWA segments.""" + full_attn_block_size = 36 + swa_block_size = 4 + prompt_tokens = 1200 + kv_cache_groups = [ + KVCacheGroupSpec( + ["layer0"], + FullAttentionSpec( + block_size=full_attn_block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + ), + KVCacheGroupSpec( + ["layer1"], + SlidingWindowSpec( + block_size=swa_block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + sliding_window=16, + ), + ), + ] + runner = request_runner( + block_size=swa_block_size, + num_gpu_blocks=1000, + async_scheduling=False, + kv_cache_groups=kv_cache_groups, + ) + runner.new_request(token_ids=[0] * prompt_tokens) + runner.manager.prepare_store.side_effect = lambda keys, req_context: ( + generate_store_output(keys) + ) + + runner._run([1] * 12, complete_transfers=True) + + assert runner.scheduler.running + stored_swa_chunks = { + block.request_block_offset + for transfer in runner.completed_stores + for block in transfer.gpu_blocks + if block.group_idx == 1 + } + first_decode_chunk = prompt_tokens // swa_block_size + assert first_decode_chunk not in stored_swa_chunks + + def test_dcp_abort_reconsiders_final_swa_tail(self): + """DCP-expanded SWA blocks are not scaled twice during abort.""" + vllm_config = _make_vllm_config( + extra_config={"offload_prompt_only": False}, + tensor_parallel_size=2, + decode_context_parallel_size=2, + ) + vllm_config.speculative_config = None + vllm_config.kv_events_config = KVEventsConfig( + enable_kv_cache_events=True, publisher="null" + ) + kv_cache_config = KVCacheConfig( + num_blocks=0, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec( + ["full"], + FullAttentionSpec( + block_size=36, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + ), + KVCacheGroupSpec( + ["swa"], + SlidingWindowSpec( + block_size=4, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + sliding_window=16, + ), + ), + ], + ) + spec = MockOffloadingSpec(build_offloading_config(vllm_config, kv_cache_config)) + scheduler = OffloadingConnectorScheduler(spec, vllm_config, kv_cache_config) + scheduler.manager.prepare_store.side_effect = ( + lambda keys, req_context: generate_store_output(keys) + ) + + request = MagicMock() + request.request_id = "req" + request.kv_transfer_params = None + request.num_prompt_tokens = 1200 + request.num_tokens = 1200 + request.num_computed_tokens = 0 + request.block_hashes = [BlockHash(f"h{i}".encode()) for i in range(150)] + request.all_token_ids = list(range(1200)) + request.lora_request = None + request.shared_prefix_boundary = 0 + request.status = RequestStatus.RUNNING + request.is_finished.return_value = False + scheduler.on_new_request(request) + + full_block_ids = list(range(1, 15)) + swa_block_ids = list(range(1001, 1126)) + first_output = SchedulerOutput.make_empty() + first_output.scheduled_new_reqs = [ + SimpleNamespace(req_id="req", block_ids=(full_block_ids, swa_block_ids)) + ] + first_output.num_scheduled_tokens = {"req": 1000} + first_output.total_num_scheduled_tokens = 1000 + scheduler.build_connector_meta(first_output) + + request.num_computed_tokens = 1000 + request.status = RequestStatus.FINISHED_ABORTED + request.is_finished.return_value = True + abort_output = SchedulerOutput.make_empty() + abort_output.finished_req_ids = {"req"} + abort_meta = scheduler.build_connector_meta(abort_output) + + assert len(abort_meta.store_jobs) == 1 + [store_job] = abort_meta.store_jobs.values() + assert isinstance(store_job.src_spec, GPULoadStoreSpec) + assert store_job.src_spec.block_ids.tolist() == [swa_block_ids[123]] + assert store_job.src_spec.group_sizes == [0, 1] + assert store_job.src_spec.block_indices == [0, 123] + @pytest.mark.parametrize("async_scheduling", [True, False]) def test_full_attn_store_then_load(self, request_runner, async_scheduling: bool): """Eagle group constrains load: convergence tightens both groups. diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index da0aea06de..fcc4ba9e91 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -1326,6 +1326,46 @@ class OffloadingConnectorScheduler: return store_jobs + def _reachable_store_block_mask( + self, + group_config: GroupOffloadConfig, + start_chunk_idx: int, + end_chunk_idx: int, + final_segment_end_chunk_idx: int | None, + reachable_boundaries: tuple[int, ...], + ) -> list[bool] | None: + """Build the block mask for a range of candidate offload chunks.""" + blocks_per_chunk = self.config.blocks_per_chunk + return group_config.manager_cls.reachable_block_mask( + start_block=start_chunk_idx * blocks_per_chunk, + end_block=end_chunk_idx * blocks_per_chunk, + alignment_tokens=self.config.alignment_tokens, + kv_cache_spec=group_config.kv_cache_spec, + use_eagle=group_config.is_eagle_group, + retention_interval=self.config.retention_interval, + reachable_boundaries=reachable_boundaries, + dcp_world_size=self.config.dcp_world_size, + final_segment_end_block=( + final_segment_end_chunk_idx * blocks_per_chunk + if final_segment_end_chunk_idx is not None + else None + ), + ) + + def _final_swa_alignment_blocks( + self, group_config: GroupOffloadConfig + ) -> int | None: + """Return representable final-tail alignment for a non-EAGLE SWA group.""" + alignment_tokens = self.config.alignment_tokens + if ( + alignment_tokens is None + or group_config.is_eagle_group + or not isinstance(group_config.kv_cache_spec, SlidingWindowSpec) + or alignment_tokens % group_config.tokens_per_block != 0 + ): + return None + return alignment_tokens // group_config.tokens_per_block + def _build_store_jobs( self, scheduler_output: SchedulerOutput, @@ -1352,10 +1392,14 @@ class OffloadingConnectorScheduler: num_offloadable_tokens = self._calc_num_offloadable_tokens( req_status, num_tokens_after_batch ) + prompt_offloadable_tokens = self._calc_num_offloadable_tokens( + req_status, req.num_prompt_tokens + ) # Filter out chunks skipped due to sliding window attention / SSM # or unreachable by the load path's alignment constraints. new_offload_keys: list[OffloadKey] = [] + group_store_ranges: list[tuple[int, int]] = [] reachable_boundaries: tuple[int, ...] = () if self.config.retention_interval is not None: @@ -1370,10 +1414,47 @@ class OffloadingConnectorScheduler: group_config, group_state, num_offloadable_tokens ) + start_chunk_idx = group_state.next_stored_chunk_idx + prompt_horizon_chunks = ( + prompt_offloadable_tokens // group_config.tokens_per_chunk + ) + final_swa_alignment_blocks = self._final_swa_alignment_blocks( + group_config + ) + reconsider_final_swa_tail = ( + req.is_finished() + and num_chunks != prompt_horizon_chunks + and self.config.retention_interval is None + and final_swa_alignment_blocks is not None + ) + if ( + self.config.retention_interval is not None + or group_config.is_eagle_group + ): + store_horizon_chunks = None + elif req.is_finished(): + store_horizon_chunks = num_chunks + elif num_chunks <= prompt_horizon_chunks: + store_horizon_chunks = prompt_horizon_chunks + else: + # An active decode frontier is not a final request + # boundary. Only fixed alignment tails are reachable. + store_horizon_chunks = None + if reconsider_final_swa_tail: + assert final_swa_alignment_blocks is not None + horizon_blocks = num_chunks * blocks_per_chunk + partial_segment_start_block = ( + horizon_blocks - horizon_blocks % final_swa_alignment_blocks + ) + partial_segment_start_chunk = ( + partial_segment_start_block // blocks_per_chunk + ) + start_chunk_idx = min(start_chunk_idx, partial_segment_start_chunk) + group_store_ranges.append((start_chunk_idx, num_chunks)) + if group_config.requires_cow_source: continue - start_chunk_idx = group_state.next_stored_chunk_idx if num_chunks <= start_chunk_idx: continue offload_keys = group_state.offload_keys[start_chunk_idx:num_chunks] @@ -1393,22 +1474,44 @@ class OffloadingConnectorScheduler: # (SWA/Mamba sparsity + retention interval). # reachable_block_mask operates in KV-block coordinates, # so convert chunk indices to block indices. - block_mask = group_config.manager_cls.reachable_block_mask( - start_block=start_chunk_idx * blocks_per_chunk, - end_block=num_chunks * blocks_per_chunk, - alignment_tokens=self.config.alignment_tokens, - kv_cache_spec=group_config.kv_cache_spec, - use_eagle=group_config.is_eagle_group, - retention_interval=self.config.retention_interval, + block_mask = self._reachable_store_block_mask( + group_config=group_config, + start_chunk_idx=start_chunk_idx, + end_chunk_idx=num_chunks, + final_segment_end_chunk_idx=store_horizon_chunks, reachable_boundaries=reachable_boundaries, - dcp_world_size=self.config.dcp_world_size, ) + prompt_horizon_block_mask: list[bool] | None = None + if reconsider_final_swa_tail: + prompt_horizon_block_mask = self._reachable_store_block_mask( + group_config=group_config, + start_chunk_idx=start_chunk_idx, + end_chunk_idx=num_chunks, + final_segment_end_chunk_idx=prompt_horizon_chunks, + reachable_boundaries=reachable_boundaries, + ) + for key_idx, (offload_key, block_id) in enumerate( zip(offload_keys, offload_block_ids) ): if block_id == 0: continue + abs_chunk_idx = start_chunk_idx + key_idx + if ( + reconsider_final_swa_tail + and abs_chunk_idx < group_state.next_stored_chunk_idx + and ( + prompt_horizon_block_mask is None + or any( + prompt_horizon_block_mask[ + key_idx * blocks_per_chunk + block_idx + ] + for block_idx in range(blocks_per_chunk) + ) + ) + ): + continue # A chunk is reachable if any of its constituent # blocks is reachable. if block_mask is not None and not any( @@ -1445,16 +1548,15 @@ class OffloadingConnectorScheduler: src_block_ids: list[int] = [] fenced_block_ids: list[int] = [] deferred_fence_block_ids: list[int] = [] - for group_config, group_state in zip( - self.config.kv_group_configs, req_status.group_states + for group_config, group_state, store_range in zip( + self.config.kv_group_configs, + req_status.group_states, + group_store_ranges, ): is_sliding_window = ( group_config.sliding_window_size_in_chunks is not None ) - num_chunks = req_status.storable_chunks( - group_config, group_state, num_offloadable_tokens - ) - start_chunk_idx = group_state.next_stored_chunk_idx + start_chunk_idx, num_chunks = store_range block_ids = group_state.block_ids num_group_blocks = 0 start_gpu_block_idx: int | None = None diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index 45ef988059..0403ba416a 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -508,6 +508,7 @@ class SingleTypeKVCacheManager(ABC): retention_interval: int | None = None, reachable_boundaries: Sequence[int] = (), dcp_world_size: int = 1, + final_segment_end_block: int | None = None, ) -> list[bool] | None: """Per-block mask for ``cache_full_blocks``. ``None`` means cache every (non-null) block — the default for full attention. @@ -516,6 +517,10 @@ class SingleTypeKVCacheManager(ABC): blocks that can never serve a hit at any alignment-aligned prefix length. ``reachable_boundaries`` are token positions whose reachable tail must be retained; the base (dense) policy ignores them. + ``final_segment_end_block`` is the exclusive end of the request's final + segment. It may be later than ``end_block`` while prefill is still in + progress. Non-EAGLE sparse managers may keep its reachable tail. The + base policy already keeps every block, so it ignores this value. """ return None @@ -1033,6 +1038,7 @@ class SlidingWindowManager(SingleTypeKVCacheManager): retention_interval: int | None = None, reachable_boundaries: Sequence[int] = (), dcp_world_size: int = 1, + final_segment_end_block: int | None = None, ) -> list[bool] | None: assert isinstance(kv_cache_spec, SlidingWindowSpec) if alignment_tokens is None: @@ -1079,6 +1085,17 @@ class SlidingWindowManager(SingleTypeKVCacheManager): if i >= shift and (i - shift) % per_segment >= per_segment - need: mask[i - start_block] = True + if final_segment_end_block is not None and not use_eagle: + final_segment_size = final_segment_end_block % per_segment + if final_segment_size: + final_segment_start = final_segment_end_block - final_segment_size + final_tail_start = max( + final_segment_start, final_segment_end_block - need + ) + final_tail_end = min(end_block, final_segment_end_block) + for i in range(max(start_block, final_tail_start), final_tail_end): + mask[i - start_block] = True + # (2) Reachable-boundary tails: the replay boundary (``num_prompt - 1``, # capped by ``get_computed_blocks``) and any shared-prefix junction. Both # land before segments would cover them under sparse retention, so keep @@ -1513,6 +1530,7 @@ class MambaManager(SingleTypeKVCacheManager): retention_interval: int | None = None, reachable_boundaries: Sequence[int] = (), dcp_world_size: int = 1, + final_segment_end_block: int | None = None, ) -> list[bool] | None: """Sparse Mamba state-snapshot retention.