diff --git a/tests/models/test_deepseek_v41_replay_start.py b/tests/models/test_deepseek_v41_replay_start.py new file mode 100644 index 0000000000..7d4e45fe81 --- /dev/null +++ b/tests/models/test_deepseek_v41_replay_start.py @@ -0,0 +1,112 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""SWA bounded replay in DeepseekV41ModelState: the batch's replay starts reach +the sliding-window builders, and the replayed tokens' slots are padded in the +prefix-cacheable groups only.""" + +from dataclasses import replace +from types import SimpleNamespace +from unittest.mock import MagicMock + +import numpy as np +import pytest +import torch + +from vllm.config import CUDAGraphMode +from vllm.models.deepseek_v41.nvidia.model_state import DeepseekV41ModelState +from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadataBuilder +from vllm.v1.attention.backends.utils import PAD_SLOT_ID +from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers +from vllm.v1.worker.gpu.model_states.default import DefaultModelState + +pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") + +DEVICE = torch.device("cuda") +WINDOW = 4 +# Group 0 is prefix-cacheable (compressed KV), group 1 is the replayed window. +KV_CACHE_CONFIG = SimpleNamespace( + kv_cache_groups=[ + SimpleNamespace( + kv_cache_spec=SimpleNamespace(prefix_cacheable=True, prefix_replay_tokens=0) + ), + SimpleNamespace( + kv_cache_spec=SimpleNamespace( + prefix_cacheable=False, prefix_replay_tokens=WINDOW + ) + ), + ] +) + + +@pytest.fixture +def state(monkeypatch): + cfg = MagicMock() + cfg.model_config.enable_prompt_embeds = False + cfg.model_config.uses_mrope = False + cfg.model_config.is_multimodal_model = False + cfg.scheduler_config.max_num_seqs = 8 + cfg.scheduler_config.max_num_batched_tokens = 64 + seen: dict = {} + + def prepare_attn(self, input_batch, cg_mode, block_tables, slot_mappings, *a, **kw): + seen["slot_mappings"] = slot_mappings.clone() + seen["extra"] = kw["model_specific_attn_metadata"] + return {} + + monkeypatch.setattr(DefaultModelState, "prepare_attn", prepare_attn) + state = DeepseekV41ModelState( + cfg, SimpleNamespace(token_lookback_depth=0), None, DEVICE + ) + state.seen = seen # type: ignore[attr-defined] + return state + + +def _batch(query_lens, seq_lens, idx_mapping, is_prefilling): + num_reqs, num_tokens = len(query_lens), sum(query_lens) + batch = InputBatch.make_dummy(num_reqs, num_tokens, InputBuffers(8, 64, DEVICE)) + return replace( + batch, + query_start_loc=torch.tensor( + [0, *np.cumsum(query_lens)], dtype=torch.int32, device=DEVICE + ), + idx_mapping_np=np.array(idx_mapping, dtype=np.intp), + is_prefilling_np=np.array(is_prefilling, dtype=np.bool_), + positions=torch.cat( + [torch.arange(s - q, s) for q, s in zip(query_lens, seq_lens)] + ).to(DEVICE), + ) + + +def _prepare(state, batch): + slot_mappings = torch.arange(2 * batch.num_tokens, device=DEVICE).view(2, -1) + state.prepare_attn( + batch, CUDAGraphMode.NONE, (), slot_mappings, [], KV_CACHE_CONFIG + ) + swa_builder = MagicMock(spec=DeepseekSparseSWAMetadataBuilder) + extra = state.seen["extra"] + replay_start = extra.get_extra_attn_kwargs(swa_builder, batch.num_reqs)[ + "replay_start" + ] + assert extra.get_extra_attn_kwargs(MagicMock(), batch.num_reqs) == {} + return replay_start.tolist(), state.seen["slot_mappings"] + + +def test_replayed_tokens_write_only_the_window_group(state): + # Request state 3 resumes a hit at 16: positions 16..19 are replayed. + state.add_request(3, SimpleNamespace(replay_start=16)) + state.add_request(1, SimpleNamespace(replay_start=0)) + batch = _batch([6, 2], [22, 2], idx_mapping=[3, 1], is_prefilling=[True, True]) + replay_start, slots = _prepare(state, batch) + assert replay_start == [16, 0] + padded = [PAD_SLOT_ID] * WINDOW + [4, 5, 6, 7] + assert slots[0].tolist() == padded # compressed KV keeps its cached rows + assert slots[1].tolist() == list(range(8, 16)) # the window is rebuilt + + +def test_only_prefills_carry_a_replay_start(state): + state.add_request(0, SimpleNamespace(replay_start=16)) + # The same request, decoding: its rows sit above the hit. + batch = _batch([1], [40], idx_mapping=[0], is_prefilling=[False]) + replay_start, slots = _prepare(state, batch) + assert replay_start == [0] + assert slots.tolist() == [[0], [1]] diff --git a/tests/v1/attention/test_deepseek_v4_swa_visible.py b/tests/v1/attention/test_deepseek_v4_swa_visible.py index 1b5097de8e..7a74ce8c81 100644 --- a/tests/v1/attention/test_deepseek_v4_swa_visible.py +++ b/tests/v1/attention/test_deepseek_v4_swa_visible.py @@ -11,6 +11,8 @@ at ``max(pos - (window - 1) - max(left - (window - 1), 0), 0)`` and ends at ``pos + right`` (inclusive). """ +import inspect + import pytest import torch from typing_extensions import TypedDict @@ -57,10 +59,13 @@ def ref_left_right( return lefts, rights -def ref_swa_bounds(pos: int, window: int, left: int, right: int) -> tuple[int, int]: - """Reference [start, end) window bounds for one query token.""" +def ref_swa_bounds( + pos: int, window: int, left: int, right: int, replay_start: int = 0 +) -> tuple[int, int]: + """Reference [start, end) window bounds for one query token. Under SWA + bounded replay no window KV exists below ``replay_start``.""" left_add = max(left - (window - 1), 0) - start = max(pos - (window - 1) - left_add, 0) + start = max(pos - (window - 1) - left_add, 0, replay_start) return start, pos + right + 1 @@ -97,12 +102,14 @@ def ref_swa_slot_rows( window: int, max_image_tokens: int, width: int, + replay_starts: list[int] | None = None, ) -> tuple[list[list[int]], list[int]]: """Reference paged slot-id rows and lens for every token in the batch.""" block_table_cpu = block_table.cpu() lefts, rights = ref_left_right( seq_lens, query_lens, spans_per_req, max_image_tokens ) + replay_starts = replay_starts or [0] * len(seq_lens) rows: list[list[int]] = [] lens: list[int] = [] token = 0 @@ -110,7 +117,9 @@ def ref_swa_slot_rows( prefix_len = seq_len - query_len for i in range(query_len): pos = prefix_len + i - start, end = ref_swa_bounds(pos, window, lefts[token], rights[token]) + start, end = ref_swa_bounds( + pos, window, lefts[token], rights[token], replay_starts[req] + ) row = [] for p in range(start, end): blk = int(block_table_cpu[req, p // BLOCK_SIZE]) @@ -139,6 +148,7 @@ def run_swa_kernel( swa_indices = torch.zeros(num_tokens, 1, width, dtype=torch.int32, device=device) swa_lens = torch.zeros(num_tokens, dtype=torch.int32, device=device) is_valid = slot_mapping >= 0 + replay_start_t = torch.zeros(len(seq_lens), dtype=torch.int32, device=device) if with_image: lefts, rights = ref_left_right( @@ -164,6 +174,7 @@ def run_swa_kernel( block_table, block_table.stride(0), BLOCK_SIZE, + replay_start_t, token_offset=0, HAS_IMAGE=with_image, TRITON_BLOCK_SIZE=1024, @@ -289,18 +300,25 @@ def combine_case( query_lens: list[int], spans: list[list[tuple[int, int]]], with_image: bool, + replay_starts: list[int] | None = None, + combine_fn=combine_topk_swa_indices, ): """Run combine_topk_swa_indices and return (indices, lens, expected).""" device = torch.device("cuda") num_reqs = len(seq_lens) + replay_starts = replay_starts or [0] * num_reqs query_start_loc = torch.zeros(num_reqs + 1, dtype=torch.int32, device=device) query_start_loc[1:] = torch.tensor( query_lens, dtype=torch.int32, device=device ).cumsum(0) num_tokens = int(query_start_loc[-1]) seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device=device) + # The builder's gather covers only the context above the replay start. gather_lens = torch.tensor( - [q + min(s - q, WINDOW - 1) for s, q in zip(seq_lens, query_lens)], + [ + q + min(max(s - q - r, 0), WINDOW - 1) + for s, q, r in zip(seq_lens, query_lens, replay_starts) + ], dtype=torch.int32, device=device, ) @@ -319,7 +337,14 @@ def combine_case( else: left_t = right_t = None - combined_indices, combined_lens = combine_topk_swa_indices( + # V4.1's combine has no image windows (causal SWA only). + has_image = "left_visible" in inspect.signature(combine_fn).parameters + image_kwargs = ( + dict(left_visible=left_t, right_visible=right_t, max_image_tokens=MAX_IMG) + if has_image + else {} + ) + combined_indices, combined_lens = combine_fn( topk_indices, query_start_loc, seq_lens_t, @@ -329,9 +354,7 @@ def combine_case( topk, M, N, - left_visible=left_t, - right_visible=right_t, - max_image_tokens=MAX_IMG, + **image_kwargs, ) # Reference rows. @@ -342,7 +365,7 @@ def combine_case( MAX_IMG, ) topk_cpu = topk_indices.cpu() - width = WINDOW + MAX_IMG + width = WINDOW + (MAX_IMG if has_image else 0) combined_topk = (topk + width + 127) // 128 * 128 rows = [] lens = [] @@ -354,6 +377,8 @@ def combine_case( pos = prefix_len + i topk_len = min((pos + 1) // compress_ratio, topk) start, end = ref_swa_bounds(pos, WINDOW, lefts[token], rights[token]) + # The window never reaches below the gathered buffer. + start = max(start, gather_start) swa_len = end - start row = [-1] * combined_topk for j in range(topk_len): @@ -388,6 +413,31 @@ def test_combine_topk_swa_with_image_spans(cfg): assert indices.cpu().tolist() == rows +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +@pytest.mark.parametrize("cfg", COMBINE_CASES) +def test_v41_combine_topk_swa_stops_at_replay_start(cfg): + """SWA bounded replay: the gathered buffer starts at replay_start, so the + window never indexes below it.""" + from vllm.models.deepseek_v41.common.ops.cache_utils import ( + combine_topk_swa_indices as combine_v41, + ) + + # Request 0 replays [16, 40): the windows of its first rows would + # otherwise reach below 16. + indices, lens, rows, exp_lens = combine_case( + cfg["compress_ratio"], + cfg["topk"], + seq_lens=[40, 12], + query_lens=[24, 12], + spans=[[], []], + with_image=False, + replay_starts=[16, 0], + combine_fn=combine_v41, + ) + assert lens.cpu().tolist() == exp_lens + assert indices.cpu().tolist() == rows + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") @pytest.mark.parametrize("cfg", COMBINE_CASES) def test_combine_topk_swa_without_image_unchanged(cfg): @@ -472,6 +522,7 @@ def make_builder( DeepseekSparseSWAMetadataBuilder ] = DeepseekSparseSWAMetadataBuilder, mm_prefix_clamp_sliding_window: bool = True, + bounded_replay: bool = False, ) -> DeepseekSparseSWAMetadataBuilder: overrides: dict = {"sliding_window": window} if vision: @@ -498,6 +549,7 @@ def make_builder( sliding_window=window, cache_dtype_str="auto", model_version="deepseek_v4", + bounded_replay=bounded_replay, ) return builder_cls( kv_cache_spec=spec, @@ -512,11 +564,17 @@ def build_metadata( seq_lens: list[int], query_lens: list[int], mm_req_doc_ranges: dict[int, list[tuple[int, int]]] | None, + replay_starts: list[int] | None = None, ): device = torch.device("cuda") query_start_loc, seq_lens_t, _, slot_mapping, block_table = make_batch( seq_lens, query_lens, device ) + replay_start = ( + None + if replay_starts is None + else torch.tensor(replay_starts, dtype=torch.int32, device=device) + ) return builder.build( 0, CommonAttentionMetadata( @@ -533,6 +591,7 @@ def build_metadata( causal=True, mm_req_doc_ranges=mm_req_doc_ranges, ), + replay_start=replay_start, ) @@ -653,3 +712,108 @@ def test_v41_image_prefill_uses_causal_swa(compress_ratio, query_len): assert absolute_keys.tolist() == list(range(lo, hi)) assert paged_lens[token] == hi - lo assert paged_indices[token, 0, : hi - lo].tolist() == list(range(lo, hi)) + + +# SWA bounded replay: every prefill index path clamps the window at replay_start. + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_builder_replay_start_bounds_prefill_window_and_gather(): + """Paged-direct prefill indices stop at replay_start and the FlashMLA + gather only covers the context the request may see.""" + seq_lens = [40, 12, 30] + query_lens = [24, 12, 10] + replay_starts = [12, 0, 0] + builder = make_builder(vision=False) + md = build_metadata(builder, seq_lens, query_lens, None, replay_starts) + assert md.replay_start is not None + assert md.replay_start.cpu().tolist() == replay_starts + + _, _, _, _, block_table = make_batch(seq_lens, query_lens, torch.device("cuda")) + rows, lens = ref_swa_slot_rows( + seq_lens, + query_lens, + [[], [], []], + block_table, + WINDOW, + MAX_IMG, + WINDOW, + replay_starts=replay_starts, + ) + assert md.prefill_swa_lens.cpu().tolist() == lens + assert md.prefill_swa_indices[:, 0].cpu().tolist() == rows + # Request 0: the first replayed row (pos 16) sees [12, 16]; its last row + # (pos 39) is a full window above the replay start and sees all of it. + assert md.prefill_swa_lens[0].item() == 16 - 12 + 1 + assert md.prefill_swa_lens[23].item() == WINDOW + # gather_len = query_len + min(prefix_len - replay_start, WINDOW - 1). + assert md.prefill_gather_lens.cpu().tolist() == [ + 24 + min(16 - 12, WINDOW - 1), + 12, + 10 + min(20, WINDOW - 1), + ] + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_builder_replaying_group_builds_without_replay_start(): + """Graph captures build without model-specific kwargs: a replaying group's + metadata then bounds nothing.""" + builder = make_builder(vision=False, bounded_replay=True) + md = build_metadata(builder, [40, 12], [24, 12], None) + assert not md.replay_start.any() + assert md.prefill_swa_lens[0].item() == WINDOW + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_v41_flashinfer_mixed_sparse_indices_respect_replay_start(): + from vllm.models.deepseek_v41.common.ops.cache_utils import ( + build_flashinfer_mixed_sparse_indices as build_v41, + ) + + device = torch.device("cuda") + # 1 decode token (req 0) + two prefill requests; req 1 resumes at 16. + seq_lens = [20, 40, 9] + query_lens = [1, 24, 9] + replay_starts = [0, 16, 0] + query_start_loc = torch.tensor([0, 1, 25, 34], dtype=torch.int32, device=device) + seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device=device) + token_to_req = torch.tensor( + [0] + [1] * 24 + [2] * 9, dtype=torch.int32, device=device + ) + block_table = torch.arange(3, dtype=torch.int32, device=device).view(3, 1) + kwargs = dict( + decode_swa_indices=torch.zeros((1, WINDOW), dtype=torch.int32, device=device), + decode_compressed_indices=None, + decode_compressed_topk_lens=None, + prefill_topk_indices=torch.empty((33, 0), dtype=torch.int32, device=device), + query_start_loc=query_start_loc, + seq_lens=seq_lens_t, + token_to_req_indices=token_to_req, + swa_block_table=block_table, + swa_block_size=BLOCK_SIZE, + compressed_block_table=None, + compressed_block_size=BLOCK_SIZE, + window_size=WINDOW, + compress_ratio=1, + topk=0, + ) + plain, _ = build_v41( + replay_start=torch.zeros(3, dtype=torch.int32, device=device), **kwargs + ) + bounded, _ = build_v41( + replay_start=torch.tensor(replay_starts, dtype=torch.int32, device=device), + **kwargs, + ) + rows, _ = ref_swa_slot_rows( + seq_lens, + query_lens, + [[], [], []], + block_table, + WINDOW, + MAX_IMG, + WINDOW, + replay_starts=replay_starts, + ) + # Decode row is copied through; prefill rows follow the bounded window. + assert bounded[0].cpu().tolist() == plain[0].cpu().tolist() + assert bounded[1:].cpu().tolist() == rows[1:] diff --git a/tests/v1/attention/test_dspark_noncausal_sparse_mla.py b/tests/v1/attention/test_dspark_noncausal_sparse_mla.py index 30c35a730e..3d4af17413 100644 --- a/tests/v1/attention/test_dspark_noncausal_sparse_mla.py +++ b/tests/v1/attention/test_dspark_noncausal_sparse_mla.py @@ -669,6 +669,7 @@ def test_dsv41_flashinfer_dspark_window_matches_reference( ), block_table=torch.arange(48, device=device, dtype=torch.int32).view(3, -1), block_size=128, + replay_start=torch.zeros(3, device=device, dtype=torch.int32), flashinfer_sparse_index_cache={}, max_decode_query_len=5, ) diff --git a/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py b/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py index f3b4b7f5ba..83fceef018 100644 --- a/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py +++ b/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py @@ -14,6 +14,7 @@ from vllm.models.deepseek_v41.sparse_mla import ( from vllm.v1.attention.backend import CommonAttentionMetadata from vllm.v1.attention.backends.mla.compressor_utils import ( CompressedSlotMappingKernel, + get_compressed_slot_mapping, ) from vllm.v1.attention.backends.mla.indexer import ( BuildPrefillChunkMetadataKernel, @@ -235,6 +236,28 @@ def test_compressed_slot_mapping_warmup_includes_index_kpool(): assert {(key.compress_ratio, key.block_size) for key in keys} == {(32, 2)} +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_compressed_slot_mapping_inherits_padded_token_slots(): + """A token whose own slot is padded (SWA bounded replay) closes no + compressed state either.""" + device = torch.device("cuda") + query_start_loc = torch.tensor([0, 8], dtype=torch.int32, device=device) + seq_lens = torch.tensor([8], dtype=torch.int32, device=device) + block_table = torch.tensor([[3]], dtype=torch.int32, device=device) + slot_mapping = torch.arange(8, dtype=torch.int64, device=device) + slot_mapping[:4] = -1 + compressed = get_compressed_slot_mapping( + 8, + slot_mapping, + query_start_loc, + seq_lens, + block_table, + block_size=4, + compress_ratio=2, + ) + assert compressed.tolist() == [-1, -1, -1, -1, -1, 3 * 4 + 2, -1, 3 * 4 + 3] + + def test_index_conversion_warmup_uses_physical_block_stride(): config = SimpleNamespace( cache_config=SimpleNamespace(block_size=64), diff --git a/tests/v1/core/test_contiguous_kv_packing.py b/tests/v1/core/test_contiguous_kv_packing.py index 2fcf70c6ef..58d4e20bf7 100644 --- a/tests/v1/core/test_contiguous_kv_packing.py +++ b/tests/v1/core/test_contiguous_kv_packing.py @@ -651,3 +651,67 @@ class TestCompressorRingGroup: scheduler_block_size=128, ) assert len(manager.coordinator.single_type_managers) == len(groups) + + +class TestSWABoundedReplayGrouping: + def test_non_cacheable_swa_leaves_hits_to_the_paged_group(self): + """Under SWA bounded replay only the paged MLA group hashes (hash + block = its block) and the coordinator probes one spec group.""" + config = _shared_layout_config() + config.cache_config.enable_prefix_caching = True + specs: dict[str, KVCacheSpec] = { + "layers.2.attn": MLAAttentionSpec( + block_size=128, + num_kv_heads=1, + head_size=584, + dtype=torch.uint8, + tokens_per_state=2, + alignment=576, + ), + } + swa = SlidingWindowMLASpec( + block_size=32, + num_kv_heads=1, + head_size=584, + dtype=torch.uint8, + sliding_window=128, + alignment=576, + bounded_replay=True, + ) + for layer in range(4): + specs[f"layers.{layer}.attn.swa_cache"] = swa + + groups = get_kv_cache_groups(config, specs) + # The worker reads the replay window off the (packed) group specs to + # arm the window clamp; the wrapper must forward it like cacheability. + assert sorted( + (g.kv_cache_spec.prefix_cacheable, g.kv_cache_spec.prefix_replay_tokens) + for g in groups + ) == [(False, 128)] * 4 + [(True, 0)] + kv_cache_config = get_kv_cache_config_from_groups( + config, groups, available_memory=64 * _get_kv_cache_bytes_per_block(groups) + ) + manager = KVCacheManager( + generate_scheduler_kv_cache_config([kv_cache_config]), + max_model_len=8192, + enable_caching=True, + hash_block_size=128, + scheduler_block_size=128, + ) + assert len(manager.coordinator.attention_groups) == 1 + assert isinstance( + manager.coordinator.attention_groups[0].spec, MLAAttentionSpec + ) + + # Allocating and caching a request goes through every group's manager; + # the 32-token SWA group must skip hashing rather than assert on the + # 128-token hash block. + from tests.v1.core.utils import create_requests + + request = create_requests(num_requests=1, num_tokens=256, block_size=128)[0] + assert manager.allocate_slots(request, 256) is not None + request.num_computed_tokens = 256 + manager.cache_blocks(request, 256) + for single in manager.coordinator.single_type_managers: + cached = single.num_cached_block.get(request.request_id, 0) + assert cached == (2 if single.kv_cache_spec.prefix_cacheable else 0) diff --git a/tests/v1/core/test_prefix_caching.py b/tests/v1/core/test_prefix_caching.py index 229f9c17f9..40e810c580 100644 --- a/tests/v1/core/test_prefix_caching.py +++ b/tests/v1/core/test_prefix_caching.py @@ -6,7 +6,7 @@ import copy from collections.abc import Callable from dataclasses import replace from math import lcm -from types import SimpleNamespace +from types import MethodType, SimpleNamespace from unittest.mock import MagicMock import pytest @@ -533,7 +533,9 @@ def test_nixl_hisparse_full_block_import_keeps_a_writable_tail(num_tokens): kv_cache_manager=manager, failed_recving_kv_req_ids=set(), finished_recving_kv_req_ids={request.request_id}, + prefix_replay_tokens=0, ) + scheduler._mark_prefix_replay = MethodType(Scheduler._mark_prefix_replay, scheduler) Scheduler._update_waiting_for_remote_kv(scheduler, request) assert request.num_tokens - request.num_computed_tokens == 1 assert manager.allocate_slots(request, num_new_tokens=1) is not None diff --git a/tests/v1/core/test_prefix_replay.py b/tests/v1/core/test_prefix_replay.py new file mode 100644 index 0000000000..b2e78e1f92 --- /dev/null +++ b/tests/v1/core/test_prefix_replay.py @@ -0,0 +1,279 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""SWA bounded replay: after a prefix hit the scheduler recomputes the tail of +the hit to rebuild the non-cacheable sliding-window group, keeps the hit's +blocks, and hands the worker the replay start.""" + +import pytest +import torch + +from tests.v1.kv_connector.unit.utils import MockKVConfig, create_model_runner_output +from vllm.v1.core.sched.scheduler import Scheduler +from vllm.v1.kv_cache_interface import ( + FullAttentionSpec, + KVCacheConfig, + KVCacheGroupSpec, + SlidingWindowMLASpec, +) +from vllm.v1.outputs import ModelRunnerOutput +from vllm.v1.request import RequestStatus +from vllm.v1.structured_output import StructuredOutputManager + +from .utils import create_requests, create_scheduler + +BLOCK_SIZE = 16 +WINDOW = 32 +NUM_PROMPT_TOKENS = 100 +# 100 tokens -> 6 full blocks cached -> 96-token hit. +HIT_TOKENS = NUM_PROMPT_TOKENS // BLOCK_SIZE * BLOCK_SIZE +FULL, SWA = 0, 1 + + +def _replay_scheduler( + *, + long_prefill_token_threshold: int = 0, + use_kv_connector: MockKVConfig | None = None, +) -> Scheduler: + """A hybrid layout: one prefix-cacheable full-attention group and one + replayed sliding-window group.""" + base = create_scheduler( + block_size=BLOCK_SIZE, + enable_prefix_caching=True, + long_prefill_token_threshold=long_prefill_token_threshold, + use_kv_connector=use_kv_connector, + ) + vllm_config = base.vllm_config + kv_cache_config = KVCacheConfig( + num_blocks=10000, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec( + ["full"], + FullAttentionSpec( + block_size=BLOCK_SIZE, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + ), + KVCacheGroupSpec( + ["swa"], + SlidingWindowMLASpec( + block_size=BLOCK_SIZE, + num_kv_heads=1, + head_size=64, + dtype=torch.bfloat16, + sliding_window=WINDOW, + bounded_replay=True, + ), + ), + ], + ) + scheduler = Scheduler( + vllm_config=vllm_config, + kv_cache_config=kv_cache_config, + block_size=BLOCK_SIZE, + log_stats=True, + structured_output_manager=StructuredOutputManager(vllm_config), + ) + scheduler.use_v2_model_runner = True + assert scheduler.prefix_replay_tokens == WINDOW + return scheduler + + +def _prefill(scheduler: Scheduler, request) -> None: + scheduler.add_request(request) + out = scheduler.schedule() + assert out.num_scheduled_tokens[request.request_id] == request.num_prompt_tokens + scheduler.update_from_output(out, _step_output(out, [request])) + + +def _step_output(out, requests) -> ModelRunnerOutput: + """Model output for every scheduled request: a token once its prefill is + done (non-zero, so outputs never extend the all-zero shared prompt in the + cache), nothing while it is still prefilling. schedule() has already + advanced num_computed_tokens past this step's chunk.""" + scheduled = [r for r in requests if r.request_id in out.num_scheduled_tokens] + return ModelRunnerOutput( + req_ids=[r.request_id for r in scheduled], + req_id_to_index={r.request_id: i for i, r in enumerate(scheduled)}, + sampled_token_ids=[ + [1000] if r.num_computed_tokens >= r.num_prompt_tokens else [] + for r in scheduled + ], + logprobs=None, + prompt_logprobs_dict={}, + pooler_output=None, + ) + + +def _new_req_data(out, request): + return next(r for r in out.scheduled_new_reqs if r.req_id == request.request_id) + + +def test_hit_replays_window_without_reallocating(): + scheduler = _replay_scheduler() + expected_replay = WINDOW + first, second = create_requests( + num_requests=2, + num_tokens=NUM_PROMPT_TOKENS, + same_prompt=True, + block_size=BLOCK_SIZE, + ) + _prefill(scheduler, first) + manager = scheduler.kv_cache_manager + + scheduler.add_request(second) + out = scheduler.schedule() + + replay_start = HIT_TOKENS - expected_replay + new_req = _new_req_data(out, second) + assert new_req.num_computed_tokens == replay_start + assert new_req.replay_start == replay_start + assert ( + out.num_scheduled_tokens[second.request_id] == NUM_PROMPT_TOKENS - replay_start + ) + + first_blocks = manager.get_blocks(first.request_id).blocks + second_blocks = manager.get_blocks(second.request_id).blocks + # The full-attention hit is shared (replayed tokens included); only the + # tail block is new. + num_hit_blocks = HIT_TOKENS // BLOCK_SIZE + assert second_blocks[FULL][:num_hit_blocks] == first_blocks[FULL][:num_hit_blocks] + assert all(b.ref_cnt == 2 for b in second_blocks[FULL][:num_hit_blocks]) + assert len(second_blocks[FULL]) == num_hit_blocks + 1 + # The window group never hits: blocks below the window are null, the + # window (replayed and new tokens) gets fresh blocks. + num_skipped_blocks = (HIT_TOKENS - WINDOW + 1) // BLOCK_SIZE + assert all(b.is_null for b in second_blocks[SWA][:num_skipped_blocks]) + num_window_blocks = len(second_blocks[SWA]) - num_skipped_blocks + assert num_window_blocks == -(-NUM_PROMPT_TOKENS // BLOCK_SIZE) - num_skipped_blocks + assert not any(b.is_null for b in second_blocks[SWA][num_skipped_blocks:]) + + scheduler.update_from_output(out, _step_output(out, [first, second])) + assert second.num_computed_tokens == NUM_PROMPT_TOKENS + + +def test_replay_only_chunks_make_progress(): + """A chunk cap smaller than the replay window schedules replay-only + chunks; they advance the request instead of stalling it.""" + scheduler = _replay_scheduler(long_prefill_token_threshold=BLOCK_SIZE) + first, second = create_requests( + num_requests=2, + num_tokens=NUM_PROMPT_TOKENS, + same_prompt=True, + block_size=BLOCK_SIZE, + ) + scheduler.add_request(first) + while first.num_computed_tokens < NUM_PROMPT_TOKENS: + out = scheduler.schedule() + scheduler.update_from_output(out, _step_output(out, [first])) + + scheduler.add_request(second) + + def full_blocks(): + return scheduler.kv_cache_manager.get_blocks(second.request_id).blocks[FULL] + + scheduled: list[int] = [] + while second.num_computed_tokens < NUM_PROMPT_TOKENS: + out = scheduler.schedule() + if not scheduled: + new_req = _new_req_data(out, second) + assert new_req.num_computed_tokens == HIT_TOKENS - WINDOW + assert new_req.replay_start == HIT_TOKENS - WINDOW + # A replay-only chunk adopts the hit and allocates nothing new. + assert len(full_blocks()) == HIT_TOKENS // BLOCK_SIZE + scheduled.append(out.num_scheduled_tokens[second.request_id]) + scheduler.update_from_output(out, _step_output(out, [first, second])) + # 32 replayed + 4 new tokens in 16-token chunks: two replay-only chunks. + assert scheduled == [BLOCK_SIZE, BLOCK_SIZE, NUM_PROMPT_TOKENS - HIT_TOKENS] + assert len(full_blocks()) == -(-NUM_PROMPT_TOKENS // BLOCK_SIZE) + assert second.status == RequestStatus.RUNNING + + +def test_async_remote_kv_hit_replays_after_load(): + """A KV-connector hit loaded asynchronously carries no window state + either; the replay is applied when the load completes.""" + matched = 64 + scheduler = _replay_scheduler( + use_kv_connector=MockKVConfig(matched_tokens=matched, is_async=True) + ) + request = create_requests( + num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE + )[0] + scheduler.add_request(request) + out = scheduler.schedule() + assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS + assert request.request_id not in out.num_scheduled_tokens + scheduler.update_from_output( + out, create_model_runner_output([], finished_recving={request.request_id}) + ) + + out = scheduler.schedule() + new_req = _new_req_data(out, request) + assert new_req.num_computed_tokens == matched - WINDOW + assert new_req.replay_start == matched - WINDOW + assert ( + out.num_scheduled_tokens[request.request_id] + == NUM_PROMPT_TOKENS - matched + WINDOW + ) + + +def test_remote_kv_hit_is_taken_in_whole_blocks(): + """A hit ending one token short of a block boundary would put the replay + window's first token in a block the sliding-window group retires, so a + connector hit is cut back to whole blocks.""" + matched = 4 * BLOCK_SIZE - 1 + scheduler = _replay_scheduler( + use_kv_connector=MockKVConfig(matched_tokens=matched, is_async=True) + ) + request = create_requests( + num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE + )[0] + scheduler.add_request(request) + out = scheduler.schedule() + scheduler.update_from_output( + out, create_model_runner_output([], finished_recving={request.request_id}) + ) + out = scheduler.schedule() + new_req = _new_req_data(out, request) + hit = 3 * BLOCK_SIZE + assert new_req.replay_start == hit - WINDOW + assert new_req.num_computed_tokens == hit - WINDOW + swa_manager = scheduler.kv_cache_manager.coordinator.single_type_managers[SWA] + swa_blocks = scheduler.kv_cache_manager.get_blocks(request.request_id).blocks[SWA] + assert swa_blocks[new_req.replay_start // BLOCK_SIZE] is not swa_manager._null_block + + +@pytest.mark.parametrize("via_connector", [False, True]) +def test_hit_no_longer_than_window_is_ignored(via_connector): + """Such a hit would be recomputed in full anyway. It is not adopted (an + async load of it is not even started), so a request has computed tokens + iff it replays.""" + if via_connector: + scheduler = _replay_scheduler( + use_kv_connector=MockKVConfig(matched_tokens=WINDOW, is_async=True) + ) + request = create_requests( + num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE + )[0] + else: + scheduler = _replay_scheduler() + first = create_requests( + num_requests=1, num_tokens=WINDOW, same_prompt=True, block_size=BLOCK_SIZE + )[0] + _prefill(scheduler, first) + request = create_requests( + num_requests=1, + num_tokens=NUM_PROMPT_TOKENS, + same_prompt=True, + block_size=BLOCK_SIZE, + req_ids=["long"], + )[0] + scheduler.add_request(request) + out = scheduler.schedule() + new_req = _new_req_data(out, request) + assert request.status == RequestStatus.RUNNING + assert new_req.num_computed_tokens == 0 + assert new_req.replay_start == 0 + assert out.num_scheduled_tokens[request.request_id] == NUM_PROMPT_TOKENS diff --git a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py index f9cacabdd4..a4dfc168f3 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py @@ -1505,6 +1505,7 @@ def test_failed_load_rezeroes_unwritten_skipped_blocks(): scheduler.kv_cache_manager.cache_blocks = MagicMock() scheduler.failed_recving_kv_req_ids = {"req-1"} scheduler.finished_recving_kv_req_ids = {"req-1"} + scheduler.prefix_replay_tokens = 0 request = MagicMock() request.request_id = "req-1" diff --git a/vllm/config/cache.py b/vllm/config/cache.py index 37ad5b2b34..c12433a367 100644 --- a/vllm/config/cache.py +++ b/vllm/config/cache.py @@ -227,6 +227,11 @@ class CacheConfig: necessary for implementing this optimization in some models (e.g. Gemma3n) """ + swa_bounded_replay: bool = True + """Keep the sliding-window KV of models that support it (DeepSeek-V4.1) + out of prefix caching and rebuild it after a prefix hit by recomputing the + hit's last window. Requires model runner V2.""" + kv_cache_memory_bytes: int | None = None """Size of KV Cache per GPU in bytes. By default, this is set to None and vllm can automatically infer the kv cache size based on @@ -281,8 +286,9 @@ class CacheConfig: "effective_attention_block_size", "kv_cache_size_tokens", "kv_cache_max_concurrency", - # WIP feature toggle not impacting compiled graph shape + # Feature toggles not impacting compiled graph shape "kv_sharing_fast_prefill", + "swa_bounded_replay", } from vllm.config.utils import get_hash_factors, hash_factors diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 2dd4b04f2b..67d8a2f2a0 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -781,6 +781,7 @@ class EngineArgs: stream_interval: int = SchedulerConfig.stream_interval kv_sharing_fast_prefill: bool = CacheConfig.kv_sharing_fast_prefill + swa_bounded_replay: bool = CacheConfig.swa_bounded_replay optimization_level: OptimizationLevel = VllmConfig.optimization_level performance_mode: PerformanceMode = VllmConfig.performance_mode @@ -1310,6 +1311,9 @@ class EngineArgs: cache_group.add_argument( "--kv-sharing-fast-prefill", **cache_kwargs["kv_sharing_fast_prefill"] ) + cache_group.add_argument( + "--swa-bounded-replay", **cache_kwargs["swa_bounded_replay"] + ) cache_group.add_argument( "--mamba-cache-dtype", **cache_kwargs["mamba_cache_dtype"] ) @@ -2118,6 +2122,7 @@ class EngineArgs: prefix_cache_retention_interval=self.prefix_cache_retention_interval, kv_cache_dtype_skip_layers=self.kv_cache_dtype_skip_layers, kv_sharing_fast_prefill=self.kv_sharing_fast_prefill, + swa_bounded_replay=self.swa_bounded_replay, mamba_cache_dtype=self.mamba_cache_dtype, mamba_ssm_cache_dtype=self.mamba_ssm_cache_dtype, mamba_block_size=self.mamba_block_size, diff --git a/vllm/models/deepseek_v4/amd/rocm.py b/vllm/models/deepseek_v4/amd/rocm.py index 545f29bd35..3cf26f8767 100644 --- a/vllm/models/deepseek_v4/amd/rocm.py +++ b/vllm/models/deepseek_v4/amd/rocm.py @@ -495,11 +495,13 @@ class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuild common_prefix_len: int, common_attn_metadata: CommonAttentionMetadata, fast_build: bool = False, + replay_start: torch.Tensor | None = None, ) -> DeepseekV4ROCMAiterSparseSWAMetadata: base = super().build( common_prefix_len=common_prefix_len, common_attn_metadata=common_attn_metadata, fast_build=fast_build, + replay_start=replay_start, ) ragged_indices = None diff --git a/vllm/models/deepseek_v4/sparse_mla.py b/vllm/models/deepseek_v4/sparse_mla.py index 4953bdc925..a4000183a0 100644 --- a/vllm/models/deepseek_v4/sparse_mla.py +++ b/vllm/models/deepseek_v4/sparse_mla.py @@ -195,6 +195,7 @@ class DeepseekV4SparseMLAMetadataBuilder( if self.compress_ratio > 1: slot_mapping = get_compressed_slot_mapping( cm.num_actual_tokens, + cm.slot_mapping, cm.query_start_loc, cm.seq_lens, cm.block_table_tensor.clamp_(min=0), diff --git a/vllm/models/deepseek_v41/amd/rocm.py b/vllm/models/deepseek_v41/amd/rocm.py index b3d80f8d25..4d838e121e 100644 --- a/vllm/models/deepseek_v41/amd/rocm.py +++ b/vllm/models/deepseek_v41/amd/rocm.py @@ -425,11 +425,13 @@ class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekV41SparseSWAMetadataBu common_prefix_len: int, common_attn_metadata: CommonAttentionMetadata, fast_build: bool = False, + replay_start: torch.Tensor | None = None, ) -> DeepseekV4ROCMAiterSparseSWAMetadata: base = super().build( common_prefix_len=common_prefix_len, common_attn_metadata=common_attn_metadata, fast_build=fast_build, + replay_start=replay_start, ) ragged_indices = None diff --git a/vllm/models/deepseek_v41/attention.py b/vllm/models/deepseek_v41/attention.py index 5b26a98c1c..bf645a8f15 100644 --- a/vllm/models/deepseek_v41/attention.py +++ b/vllm/models/deepseek_v41/attention.py @@ -512,6 +512,25 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): "decodes only on SM100." ) + swa_bounded_replay = cache_config.swa_bounded_replay + if swa_bounded_replay and not vllm_config.use_v2_model_runner: + logger.warning_once( + "SWA bounded replay needs model runner V2 (only it skips the " + "paged-KV writes of replayed tokens); the sliding-window cache " + "takes part in prefix caching instead." + ) + swa_bounded_replay = False + if ( + swa_bounded_replay + and vllm_config.parallel_config.prefill_context_parallel_size > 1 + ): + logger.warning_once( + "SWA bounded replay is off under prefill context parallelism " + "(the replayed tokens' slot padding knows the rank-local batch " + "only); the sliding-window cache takes part in prefix caching " + "instead." + ) + swa_bounded_replay = False self.swa_cache_layer = DeepseekV4SWACache( head_dim=self.head_dim, window_size=self.window_size, @@ -522,6 +541,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): block_size=32, packed_bytes_per_token=self.swa_bytes_per_token, packed_page_alignment=self.kv_page_alignment, + bounded_replay=swa_bounded_replay, ) # The attention layer itself was already registered with the diff --git a/vllm/models/deepseek_v41/common/ops/cache_utils.py b/vllm/models/deepseek_v41/common/ops/cache_utils.py index 04b2a9bac1..02fb0d5539 100644 --- a/vllm/models/deepseek_v41/common/ops/cache_utils.py +++ b/vllm/models/deepseek_v41/common/ops/cache_utils.py @@ -982,6 +982,10 @@ class CombineTopkSwaIndicesKernel( # turns the store index below negative). topk_len = 0 swa_start = tl.maximum(pos - (WINDOW_SIZE - 1), 0) + # gather_len already excludes context below the request's replay + # start (SWA bounded replay), so the window cannot start before + # the gathered buffer does. + swa_start = tl.maximum(swa_start, gather_start) swa_len = pos - swa_start + 1 offset = tl.arange(0, PADDED_TOP_K) @@ -1128,6 +1132,8 @@ def build_flashinfer_mixed_sparse_indices( decode_is_valid_token: torch.Tensor | None = None, swa_block_span: int | None = None, compressed_block_span: int | None = None, + *, + replay_start: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: """Build the FlashInfer DSV4 sparse-index matrix for decode-first batches. @@ -1137,6 +1143,9 @@ def build_flashinfer_mixed_sparse_indices( per token). Decode tokens read precomputed SWA/compressed indices; prefill tokens derive their SWA window from the position and translate local compressed indices to global slots via the block tables. + + ``replay_start`` ([num_reqs], SWA bounded replay) lower-bounds every + prefill token's window: positions below it hold no window KV. """ assert decode_swa_indices.dtype == torch.int32 assert decode_swa_indices.dim() == 2 @@ -1227,6 +1236,7 @@ def build_flashinfer_mixed_sparse_indices( query_start_loc, seq_lens, token_to_req_indices, + replay_start, swa_block_table, swa_block_table.stride(0), swa_block_size, @@ -1295,6 +1305,7 @@ def _build_flashinfer_mixed_sparse_indices_kernel( query_start_loc_ptr, seq_lens_ptr, token_to_req_indices_ptr, + replay_start_ptr, swa_block_table_ptr, swa_block_table_stride, swa_block_size, @@ -1394,6 +1405,8 @@ def _build_flashinfer_mixed_sparse_indices_kernel( token_idx_in_query = token_idx - query_start pos = start_pos + token_idx_in_query swa_start_pos = tl.maximum(pos - (WINDOW_SIZE - 1), 0) + # SWA bounded replay: no window KV exists below the request's replay start. + swa_start_pos = tl.maximum(swa_start_pos, tl.load(replay_start_ptr + req_idx)) swa_len = pos - swa_start_pos + 1 if COMPRESS_RATIO > 0: topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, TOP_K) diff --git a/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py b/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py index a247ae7d74..4dcc972e2c 100644 --- a/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py +++ b/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py @@ -200,8 +200,11 @@ class DeepseekSparseSWAFlashInferMetadataBuilder(DeepseekV41SparseSWAMetadataBui common_prefix_len: int, common_attn_metadata: CommonAttentionMetadata, fast_build: bool = False, + replay_start: torch.Tensor | None = None, ) -> "DeepseekSparseSWAMetadata": - metadata = super().build(common_prefix_len, common_attn_metadata, fast_build) + metadata = super().build( + common_prefix_len, common_attn_metadata, fast_build, replay_start + ) num_tokens = metadata.num_decode_tokens if not common_attn_metadata.causal and num_tokens > 0: assert metadata.decode_swa_lens is not None @@ -375,6 +378,7 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention): assert swa_metadata.token_to_req_indices is not None assert swa_metadata.decode_swa_indices is not None assert swa_metadata.block_table is not None + assert swa_metadata.replay_start is not None decode_swa_indices = swa_metadata.decode_swa_indices.reshape( num_decode_tokens, swa_metadata.decode_swa_width @@ -454,6 +458,7 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention): decode_is_valid_token=decode_is_valid_token, swa_block_span=swa_block_span, compressed_block_span=compressed_block_span, + replay_start=swa_metadata.replay_start[:num_reqs], ) if swa_only: swa_metadata.flashinfer_sparse_index_cache["swa_only"] = ( diff --git a/vllm/models/deepseek_v41/nvidia/model_state.py b/vllm/models/deepseek_v41/nvidia/model_state.py index 1ba0f41603..af20976ded 100644 --- a/vllm/models/deepseek_v41/nvidia/model_state.py +++ b/vllm/models/deepseek_v41/nvidia/model_state.py @@ -2,15 +2,23 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from typing import Any +import numpy as np import torch import torch.nn as nn -from vllm.config import VllmConfig +from vllm.config import CUDAGraphMode, VllmConfig from vllm.triton_utils import tl, triton +from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadataBuilder +from vllm.v1.attention.backends.utils import PAD_SLOT_ID +from vllm.v1.core.sched.output import NewRequestData +from vllm.v1.kv_cache_interface import KVCacheConfig +from vllm.v1.worker.gpu.buffer_utils import UvaBufferPool from vllm.v1.worker.gpu.input_batch import InputBatch from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache from vllm.v1.worker.gpu.model_states.default import DefaultModelState +from vllm.v1.worker.gpu.model_states.interface import ModelSpecificAttnMetadata from vllm.v1.worker.gpu.states import RequestState +from vllm.v1.worker.utils import AttentionGroup @triton.jit @@ -41,14 +49,69 @@ def _gather_lookback_kernel( tl.store(lookback_ptr + batch_idx * DEPTH + offs, ids, mask=offs < DEPTH) +@triton.jit +def _pad_replayed_slots_kernel( + slot_mappings_ptr, # [num_groups, num_tokens_padded] + group_stride, + cacheable_groups_ptr, # [NUM_GROUPS] indices of the prefix-cacheable groups + query_start_loc_ptr, # [num_reqs + 1] + positions_ptr, # [num_tokens] + replay_start_ptr, # [num_reqs] + window, + pad_slot_id, + NUM_GROUPS: tl.constexpr, + BLOCK: tl.constexpr, +): + req = tl.program_id(0) + start = tl.load(replay_start_ptr + req) + if start == 0: + return + begin = tl.load(query_start_loc_ptr + req) + end = tl.load(query_start_loc_ptr + req + 1) + for tok in range(begin, end, BLOCK): + offs = tok + tl.arange(0, BLOCK) + pos = tl.load(positions_ptr + offs, mask=offs < end, other=0) + replayed = (offs < end) & (pos >= start) & (pos < start + window) + for i in tl.static_range(NUM_GROUPS): + group = tl.load(cacheable_groups_ptr + i) + tl.store( + slot_mappings_ptr + group * group_stride + offs, + pad_slot_id, + mask=replayed, + ) + + +class ReplayAttnMetadata(ModelSpecificAttnMetadata): + """Hands the batch's replay starts to the sliding-window builders.""" + + def __init__(self, replay_start: torch.Tensor) -> None: + self.replay_start = replay_start + + def get_extra_attn_kwargs( + self, attn_metadata_builder: Any, num_reqs: int + ) -> dict[str, Any]: + if isinstance(attn_metadata_builder, DeepseekSparseSWAMetadataBuilder): + return {"replay_start": self.replay_start[:num_reqs]} + return {} + + class DeepseekV41ModelState(DefaultModelState): - """DefaultModelState plus the engram lookback window. + """DefaultModelState plus the engram lookback window and SWA bounded replay. The engram n-gram hash needs the ids of the ``depth`` tokens preceding each request's chunk start (see ``common/engram.py``). The runner keeps the full token history on device, so the window is gathered there every step: exact for prompt and generated tokens alike, whatever instance produced their KV. + + SWA bounded replay (``CacheConfig.swa_bounded_replay``) keeps the + sliding-window KV out of prefix caching and rebuilds it after a prefix hit + by recomputing the hit's last window; the scheduler tells each request the + position from which it holds window KV (``NewRequestData.replay_start``). + ``prepare_attn`` gathers those per batch, pads the replayed tokens' slots + in the prefix-cacheable groups so the cached KV stays as is, and hands the + starts to the sliding-window metadata builders, whose kernels read no + window KV below them. """ def __init__( @@ -67,6 +130,20 @@ class DeepseekV41ModelState(DefaultModelState): (self.max_num_reqs, depth), -1, dtype=torch.int32, device=device ) + # Per request state index; batches gather from it (see prepare_attn). + self._replay_start_np = np.zeros(self.max_num_reqs, dtype=np.int32) + self._replay_start = torch.zeros( + self.max_num_reqs, dtype=torch.int32, device=device + ) + self._replay_start_staging = UvaBufferPool(self.max_num_reqs, torch.int32) + # The replay window and the indices of the prefix-cacheable groups, from + # the KV cache config on first use; None until then. + self._replay: tuple[int, torch.Tensor] | None = None + + def add_request(self, req_index: int, new_req_data: NewRequestData) -> None: + super().add_request(req_index, new_req_data) + self._replay_start_np[req_index] = new_req_data.replay_start + def prepare_inputs( self, input_batch: InputBatch, req_states: RequestState ) -> dict[str, torch.Tensor | None]: @@ -96,3 +173,67 @@ class DeepseekV41ModelState(DefaultModelState): self.lookback_token_ids.fill_(-1) model_inputs["lookback_token_ids"] = self.lookback_token_ids return model_inputs + + def prepare_attn( + self, + input_batch: InputBatch, + cudagraph_mode: CUDAGraphMode, + block_tables: tuple[torch.Tensor, ...], + slot_mappings: torch.Tensor, + attn_groups: list[list[AttentionGroup]], + kv_cache_config: KVCacheConfig, + for_capture: bool = False, + ubatch_idx: int = 0, + model_specific_attn_metadata: ModelSpecificAttnMetadata | None = None, + ) -> dict[str, Any]: + if self._replay is None: + specs = [group.kv_cache_spec for group in kv_cache_config.kv_cache_groups] + self._replay = ( + max(spec.prefix_replay_tokens for spec in specs), + torch.tensor( + [i for i, spec in enumerate(specs) if spec.prefix_cacheable], + dtype=torch.int32, + device=self.device, + ), + ) + window, cacheable_groups = self._replay + if window: + num_reqs = input_batch.num_reqs + # Decode rows sit above the hit, so only prefills carry a replay + # start; dummy batches (captures, profiling) carry none. + replay_start_np = np.where( + input_batch.is_prefilling_np[:num_reqs], + self._replay_start_np[input_batch.idx_mapping_np[:num_reqs]], + 0, + ).astype(np.int32) + replay_start = self._replay_start_staging.copy_to_gpu( + replay_start_np, out=self._replay_start[:num_reqs] + ) + if replay_start_np.any(): + # The replayed tokens rebuild window KV only: their slots in the + # prefix-cacheable groups are padded so the cached KV stays as is. + _pad_replayed_slots_kernel[(num_reqs,)]( + slot_mappings, + slot_mappings.stride(0), + cacheable_groups, + input_batch.query_start_loc, + input_batch.positions, + replay_start, + window, + PAD_SLOT_ID, + NUM_GROUPS=cacheable_groups.numel(), + BLOCK=1024, + ) + assert model_specific_attn_metadata is None + model_specific_attn_metadata = ReplayAttnMetadata(replay_start) + return super().prepare_attn( + input_batch, + cudagraph_mode, + block_tables, + slot_mappings, + attn_groups, + kv_cache_config, + for_capture=for_capture, + ubatch_idx=ubatch_idx, + model_specific_attn_metadata=model_specific_attn_metadata, + ) diff --git a/vllm/models/deepseek_v41/sparse_mla.py b/vllm/models/deepseek_v41/sparse_mla.py index 3d69499d2a..8db6d4ed2b 100644 --- a/vllm/models/deepseek_v41/sparse_mla.py +++ b/vllm/models/deepseek_v41/sparse_mla.py @@ -195,6 +195,7 @@ class DeepseekV4SparseMLAMetadataBuilder( if self.compress_ratio > 1: slot_mapping = get_compressed_slot_mapping( cm.num_actual_tokens, + cm.slot_mapping, cm.query_start_loc, cm.seq_lens, cm.block_table_tensor.clamp_(min=0), diff --git a/vllm/v1/attention/backends/mla/compressor_utils.py b/vllm/v1/attention/backends/mla/compressor_utils.py index b32af7234b..25e24f347c 100644 --- a/vllm/v1/attention/backends/mla/compressor_utils.py +++ b/vllm/v1/attention/backends/mla/compressor_utils.py @@ -42,6 +42,8 @@ class CompressedSlotMappingKernel( @triton.jit(do_not_specialize=["block_table_stride"]) def kernel( # [num_tokens] + compressed_slot_mapping_ptr, + # [num_tokens] the tokens' own slots slot_mapping_ptr, # [num_reqs + 1] query_start_loc_ptr, @@ -69,7 +71,12 @@ class CompressedSlotMappingKernel( mask = offset < query_len pos = start_pos + i + tl.arange(0, TRITON_BLOCK_SIZE) - is_valid = (pos + 1) % COMPRESS_RATIO == 0 + # A replayed token has a PAD slot (SWA bounded replay: its KV is + # cached already); its compressed KV is cached too, so PAD here. + slot = tl.load( + slot_mapping_ptr + query_start + offset, mask=mask, other=PAD_ID + ) + is_valid = ((pos + 1) % COMPRESS_RATIO == 0) & (slot != PAD_ID) pos_after_compress = pos // COMPRESS_RATIO block_ids = pos_after_compress // block_size @@ -81,7 +88,9 @@ class CompressedSlotMappingKernel( # NOTE slot_ids = tl.where(is_valid, slot_ids, PAD_ID) - tl.store(slot_mapping_ptr + query_start + offset, slot_ids, mask=mask) + tl.store( + compressed_slot_mapping_ptr + query_start + offset, slot_ids, mask=mask + ) def dispatch( # type: ignore[override] self, @@ -121,6 +130,7 @@ class CompressedSlotMappingKernel( def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]: int32_ptr = TritonWarmupTensor(torch.int32) return dict( + compressed_slot_mapping=TritonWarmupTensor(torch.int64), slot_mapping=TritonWarmupTensor(torch.int64), query_start_loc=int32_ptr, seq_lens=int32_ptr, @@ -132,6 +142,7 @@ class CompressedSlotMappingKernel( @kernel_launcher def __call__( self, + compressed_slot_mapping: torch.Tensor, slot_mapping: torch.Tensor, query_start_loc: torch.Tensor, seq_lens: torch.Tensor, @@ -149,6 +160,7 @@ class CompressedSlotMappingKernel( def get_compressed_slot_mapping( num_tokens: int, + slot_mapping: torch.Tensor, query_start_loc: torch.Tensor, seq_lens: torch.Tensor, block_table: torch.Tensor, @@ -156,19 +168,28 @@ def get_compressed_slot_mapping( compress_ratio: int, out: torch.Tensor | None = None, ) -> torch.Tensor: + """Slot mapping for writing the compressed states of ``num_tokens`` tokens. + + Every ``compress_ratio`` tokens share one compressed state, written by the + last of them: that token maps to the state's slot, the others to PAD. A + token whose own ``slot_mapping`` entry is PAD maps to PAD too: SWA bounded + replay recomputes tokens whose KV is cached already, and their compressed + states must not be rewritten either. + """ if out is not None: # Guard: for padded / invalid sequences. # Negative positions produce bogus block indices that lead to illegal memory # accesses inside the block_table load. # NOTE: Fill -1 to the whole tensor, not just the first `num_tokens`. out.fill_(-1) - slot_mapping = out[:num_tokens] + compressed_slot_mapping = out[:num_tokens] else: - slot_mapping = torch.full( + compressed_slot_mapping = torch.full( (num_tokens,), -1, dtype=torch.int64, device=query_start_loc.device ) _COMPRESSED_SLOT_MAPPING_KERNEL( + compressed_slot_mapping, slot_mapping, query_start_loc, seq_lens, @@ -176,7 +197,7 @@ def get_compressed_slot_mapping( block_size, compress_ratio, ) - return slot_mapping + return compressed_slot_mapping _COMPRESSED_SLOT_MAPPING_KERNEL = CompressedSlotMappingKernel() diff --git a/vllm/v1/attention/backends/mla/indexer.py b/vllm/v1/attention/backends/mla/indexer.py index 53206afd93..430bd10c99 100644 --- a/vllm/v1/attention/backends/mla/indexer.py +++ b/vllm/v1/attention/backends/mla/indexer.py @@ -795,6 +795,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0 self.pcp_world_size = parallel_config.prefill_context_parallel_size self.use_pcp = self.pcp_world_size > 1 + self.pcp_rank = get_pcp_group().rank_in_group if self.use_pcp else 0 self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size # NOTE(Chen):an estimated max size of flattened_kv. Need to double check. self.max_prefill_buffer_size = get_max_prefill_buffer_size(self.vllm_config) @@ -1217,10 +1218,18 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): factor = self.kv_cache_spec.block_size // kernel_block_size indexer_block_table = (block_table[:, ::factor] // factor).contiguous() padded_num_tokens = num_tokens - if self.pcp_world_size > 1: + local_slot_mapping = slot_mapping + if self.use_pcp: + # The gathered layout holds each rank's local tokens, padded, in + # rank order, so this rank's segment lines up with query_start_loc. padded_num_tokens = slot_mapping.shape[0] // self.pcp_world_size + local_slot_mapping = slot_mapping[ + self.pcp_rank * padded_num_tokens : (self.pcp_rank + 1) + * padded_num_tokens + ] compressed_slot_mapping = get_compressed_slot_mapping( num_tokens, + local_slot_mapping, query_start_loc, seq_lens, indexer_block_table, diff --git a/vllm/v1/attention/backends/mla/sparse_swa.py b/vllm/v1/attention/backends/mla/sparse_swa.py index 0fa5ae414c..6fe20e0e69 100644 --- a/vllm/v1/attention/backends/mla/sparse_swa.py +++ b/vllm/v1/attention/backends/mla/sparse_swa.py @@ -81,9 +81,12 @@ class DeepseekV4SWACache(torch.nn.Module, AttentionLayerBase): block_size: int = 64, packed_bytes_per_token: int = 584, packed_page_alignment: int = 576, + bounded_replay: bool = False, ): super().__init__() self.backend_cls = backend_cls or DeepseekSparseSWABackend + # DeepseekV4.1 SWA bounded replay. + self.bounded_replay = bounded_replay self.kv_cache = torch.tensor([]) self.head_dim = head_dim self.window_size = window_size @@ -117,6 +120,7 @@ class DeepseekV4SWACache(torch.nn.Module, AttentionLayerBase): "nvfp4_ds_mla", ) return SlidingWindowMLASpec( + bounded_replay=self.bounded_replay, block_size=self.block_size, num_kv_heads=1, head_size=self.head_dim, @@ -198,6 +202,10 @@ class DeepseekSparseSWAMetadata: # None when the model is text-only or the batch has no image spans. prefill_left_visible: torch.Tensor | None = None prefill_right_visible: torch.Tensor | None = None + # SWA bounded replay: [num_reqs] lower bound of the positions a request's + # window attention may read (SWA bounded replay); + # zeros when nothing replays. + replay_start: torch.Tensor | None = None # Number of decode/prefill requests/tokens (batch is reordered: decodes first) num_decodes: int = 0 @@ -256,10 +264,12 @@ class DeepseekSparseSWAMetadata: has_compressed = compress_ratio > 1 # query_len <= max_num_batched_tokens and - # gather_len = query_len + min(prefix_len, window_size - 1), so the - # worst-case gathered width is bounded by - # max_num_batched_tokens + window_size - 1. The compressed prefix pool - # is bounded by ceil(max_model_len / compress_ratio). + # gather_len <= query_len + min(prefix_len, window_size - 1) (a replay + # boundary only shrinks it), so the worst-case gathered width is + # bounded by max_num_batched_tokens + window_size - 1. The compressed + # prefix pool is bounded by ceil(max_model_len / compress_ratio). The + # CPU gather lengths below are this upper bound, for chunk planning + # only; the kernels use the exact per-request values. max_workspace_area = prefill_chunk_size * ( (cdiv(self.prefill_max_model_len, compress_ratio) if has_compressed else 0) + self.prefill_window_size @@ -331,6 +341,7 @@ class ComputePrefillMetadataKernel( # Inputs seq_lens_ptr, query_start_loc_ptr, + replay_start_ptr, num_prefills, num_decodes, window_size, @@ -352,7 +363,10 @@ class ComputePrefillMetadataKernel( query_len = qsl_end - qsl_start prefix_len = seq_len - query_len - gather_len = query_len + tl.minimum(prefix_len, window_size - 1) + # Context below replay_start has no window KV (SWA bounded replay). + replay_start = tl.load(replay_start_ptr + num_decodes + safe_offset, mask=mask) + visible_prefix = tl.maximum(prefix_len - replay_start, 0) + gather_len = query_len + tl.minimum(visible_prefix, window_size - 1) tl.store(prefill_gather_lens_ptr + offset, gather_len, mask=mask) @@ -384,6 +398,7 @@ class ComputePrefillMetadataKernel( prefill_gather_lens=int32_ptr, seq_lens=int32_ptr, query_start_loc=int32_ptr, + replay_start=int32_ptr, num_prefills=compile_key.block_size, num_decodes=0, window_size=1, @@ -395,6 +410,7 @@ class ComputePrefillMetadataKernel( prefill_gather_lens: torch.Tensor, seq_lens: torch.Tensor, query_start_loc: torch.Tensor, + replay_start: torch.Tensor, num_prefills: int, num_decodes: int, window_size: int, @@ -550,12 +566,19 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder): ) self.decode_swa_indices_noncausal: torch.Tensor | None = None self._max_tokens = max_tokens + # replay_start when no group replays: no lower bound. + self.no_replay_start = torch.zeros( + self.vllm_config.scheduler_config.max_num_seqs + 1, + dtype=torch.int32, + device=self.device, + ) def build( self, common_prefix_len: int, common_attn_metadata: CommonAttentionMetadata, fast_build: bool = False, + replay_start: torch.Tensor | None = None, ) -> DeepseekSparseSWAMetadata: """Build SWA metadata for mixed decode/prefill batches. @@ -564,6 +587,10 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder): separate window_topk_idxs for each portion. For prefill, we use chunked prefill to align with the indexer's chunking. + + ``replay_start`` ([num_reqs], SWA bounded replay): the position from + which each request holds window KV; the model state passes it for every + batch of a replaying group, graph captures pass nothing. """ seq_lens = common_attn_metadata.seq_lens seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound @@ -587,6 +614,9 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder): is_valid_token = self.is_valid_token[: slot_mapping.shape[0]] is_valid_token.copy_(slot_mapping >= 0) + if replay_start is None: + # Graph captures build without it; their dummy batches replay nothing. + replay_start = self.no_replay_start non_causal = not common_attn_metadata.causal decode_swa_width = ( @@ -609,6 +639,8 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder): device=self.device, ) decode_swa_indices = self.decode_swa_indices_noncausal + # No replay_start: decode rows sit above the hit, so a + # request's replayed window never reaches them. _COMPUTE_DSPARK_NONCAUSAL_SWA_INDICES_KERNEL( decode_swa_indices, self.decode_swa_lens, @@ -637,6 +669,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder): is_valid_token, block_table, self.block_size, + replay_start, num_tokens=num_decode_tokens, token_offset=0, ) @@ -683,6 +716,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder): is_valid_token, block_table, self.block_size, + replay_start, num_tokens=num_prefill_tokens, token_offset=num_decode_tokens, has_image=has_image, @@ -696,6 +730,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder): seq_lens_cpu, query_start_loc, query_start_loc_cpu, + replay_start, ) # Per-layer-type tile-scheduler plan holders. Empty FlashMLASchedMeta @@ -727,6 +762,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder): ), prefill_left_visible=prefill_left_visible, prefill_right_visible=prefill_right_visible, + replay_start=replay_start, block_size=self.block_size, num_decodes=num_decodes, num_prefills=num_prefills, @@ -803,6 +839,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder): assert metadata.is_valid_token is not None assert metadata.decode_swa_indices is not None assert metadata.decode_swa_lens is not None + assert metadata.replay_start is not None _COMPUTE_SWA_INDICES_AND_LENS_KERNEL( metadata.decode_swa_indices, @@ -817,6 +854,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder): metadata.is_valid_token, metadata.block_table, self.block_size, + metadata.replay_start, num_tokens=metadata.num_decode_tokens, token_offset=0, ) @@ -872,6 +910,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder): seq_lens_cpu: torch.Tensor | None, query_start_loc: torch.Tensor, query_start_loc_cpu: torch.Tensor, + replay_start: torch.Tensor, ) -> dict[str, torch.Tensor | int | None]: """Pre-compute DeepseekV4 prefill metadata during the metadata build phase. @@ -893,6 +932,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder): pfx_gather_lens, seq_lens, query_start_loc, + replay_start, num_prefills, num_decodes, self.window_size, @@ -982,6 +1022,7 @@ def _compute_swa_indices_and_lens_kernel( block_table_ptr, block_table_stride, block_size, + replay_start_ptr, token_offset, HAS_IMAGE: tl.constexpr, TRITON_BLOCK_SIZE: tl.constexpr, @@ -1022,6 +1063,8 @@ def _compute_swa_indices_and_lens_kernel( right = 0 left_add = tl.maximum(left - (window_size - 1), 0) start_pos = tl.maximum(pos - (window_size - 1) - left_add, 0) + # SWA bounded replay: no window KV exists below the request's replay start. + start_pos = tl.maximum(start_pos, tl.load(replay_start_ptr + req_idx)) end_pos = pos + right + 1 swa_len = end_pos - start_pos @@ -1111,6 +1154,7 @@ class ComputeSWAIndicesAndLensKernel( is_valid_token=TritonWarmupTensor(torch.bool), block_table=TritonWarmupTensor(torch.int32, shape=(1, 1), strides=(1, 1)), block_size=compile_key.block_size, + replay_start=int32_ptr, num_tokens=1, token_offset=0, has_image=bool(compile_key.has_image), @@ -1131,6 +1175,7 @@ class ComputeSWAIndicesAndLensKernel( is_valid_token: torch.Tensor, block_table: torch.Tensor, block_size: int, + replay_start: torch.Tensor, *, num_tokens: int, token_offset: int, diff --git a/vllm/v1/core/kv_cache_manager.py b/vllm/v1/core/kv_cache_manager.py index 49f62a2670..1adf386cab 100644 --- a/vllm/v1/core/kv_cache_manager.py +++ b/vllm/v1/core/kv_cache_manager.py @@ -465,12 +465,17 @@ class KVCacheManager: A list of new allocated blocks. """ - # When loading KV data asynchronously, we may have zero new tokens to - # compute while still allocating slots for externally computed tokens. - if num_new_tokens == 0 and num_external_computed_tokens == 0: + # A step may need no slots of its own while still adopting computed + # tokens: an async KV load, or a chunk that ends inside the replayed + # range of a hit (SWA bounded replay). + if ( + num_new_tokens == 0 + and num_external_computed_tokens == 0 + and num_new_computed_tokens == 0 + ): raise ValueError( "num_new_tokens must be greater than 0 when there are no " - "external computed tokens" + "computed tokens to adopt" ) if new_computed_blocks is not None: diff --git a/vllm/v1/core/sched/output.py b/vllm/v1/core/sched/output.py index 3a46fa40a7..7501c51b2c 100644 --- a/vllm/v1/core/sched/output.py +++ b/vllm/v1/core/sched/output.py @@ -48,6 +48,8 @@ class NewRequestData: # Only used for v2 model runner. prefill_token_ids: list[int] | None = None + # DeepSeek-V4.1 only: SWA bounded replay; see Request.replay_start. + replay_start: int = 0 @classmethod def from_request( @@ -73,6 +75,7 @@ class NewRequestData: prompt_embeds=request.prompt_embeds, prompt_is_token_ids=request.prompt_is_token_ids, prefill_token_ids=prefill_token_ids, + replay_start=request.replay_start, ) @property diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index e0c4997b63..19ac16284c 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -271,6 +271,18 @@ class Scheduler(SchedulerInterface): self.use_eagle_block_drop = False self.num_spec_tokens = vllm_config.num_speculative_tokens self.num_lookahead_tokens = vllm_config.num_lookahead_tokens + # DSV41 SWA bounded replay: groups that declare a replay window are rebuilt + # after a prefix hit by recomputing its trailing tokens. One window + # for all such groups, so the rewind matches every group's allocation. + replay_windows = { + group.kv_cache_spec.prefix_replay_tokens + for group in kv_cache_config.kv_cache_groups + if group.kv_cache_spec.prefix_replay_tokens > 0 + } + assert len(replay_windows) <= 1, ( + f"Prefix replay windows should agree: {sorted(replay_windows)}" + ) + self.prefix_replay_tokens = replay_windows.pop() if replay_windows else 0 self.num_prefill_lookahead = vllm_config.num_prefill_lookahead_tokens self.dynamic_sd_lookup: list[int] | None = None if speculative_config is not None: @@ -946,6 +958,17 @@ class Scheduler(SchedulerInterface): step_skipped_waiting.prepend_request(request) continue + if self.prefix_replay_tokens: + # SWA bounded replay recomputes the hit's last + # window, from the block holding its first + # token; the sliding-window groups retire + # whole blocks below the window of the hit's + # next token. Hits end on a block boundary, or + # a hit ending one token short of one would + # retire the block the replay starts in. + ext_tokens -= ext_tokens % self.block_size + load_kv_async = load_kv_async and ext_tokens > 0 + if partial_tail and ext_tokens > partial_tail: # Remote strictly exceeds the full local hit: drop the # sub-block tail so no CoW is needed, and let the load @@ -988,6 +1011,17 @@ class Scheduler(SchedulerInterface): num_new_local_computed_tokens + num_external_computed_tokens ) assert num_computed_tokens <= request.num_tokens + if 0 < num_computed_tokens <= self.prefix_replay_tokens: + # SWA bounded replay: a hit no longer than the replayed + # window would be recomputed in full and save nothing. + new_computed_blocks = ( + self.kv_cache_manager.empty_kv_cache_blocks + ) + num_new_local_computed_tokens = 0 + num_external_computed_tokens = 0 + connector_prefix_cache_hits = 0 + num_computed_tokens = 0 + load_kv_async = False # Skip request with pending mm encoding prefetches if self._ec_transfer_pending(request, num_computed_tokens): @@ -1022,6 +1056,11 @@ class Scheduler(SchedulerInterface): new_encoder_compute_budget = encoder_compute_budget pad_spec_decode = False + # SWA bounded replay: recompute the tail of the hit without + # rewriting its cached KV. An async load replays once the KV + # has arrived (_update_waiting_for_remote_kv). + num_replay_tokens = 0 + if load_kv_async: # KVTransfer: loading remote KV, do not allocate for new work. assert num_external_computed_tokens > 0 @@ -1032,6 +1071,12 @@ class Scheduler(SchedulerInterface): break else: request_token_budget = min(token_budget, input_budget - draft_slots) + if did_prefix_cache_lookup: + # Fresh admission: local and/or sync external hit. + num_replay_tokens = self._mark_prefix_replay( + request, num_computed_tokens + ) + num_computed_tokens -= num_replay_tokens # Number of tokens to be scheduled. # We use `request.num_tokens` instead of # `request.num_prompt_tokens` to consider the resumed @@ -1159,9 +1204,12 @@ class Scheduler(SchedulerInterface): + self._spec_decode_step_blocks() ) + # Replayed tokens are already counted in the adopted hit; a + # chunk that ends inside the replayed range needs no new slots. + num_tokens_past_hit = max(num_new_tokens - num_replay_tokens, 0) new_blocks = self.kv_cache_manager.allocate_slots( request, - num_new_tokens, + num_tokens_past_hit, num_new_computed_tokens=num_new_local_computed_tokens, new_computed_blocks=new_computed_blocks, num_lookahead_tokens=effective_lookahead_tokens, @@ -2963,6 +3011,20 @@ class Scheduler(SchedulerInterface): self._request_remaining_blocks(req) for req in self._inflight_prefills ) + def _mark_prefix_replay(self, request: Request, num_hit_tokens: int) -> int: + """Record where a prefix hit's replay starts and return the number of + hit tokens to recompute; the caller rewinds the computed count by it. + + The replayed tokens are the hit's last window (see + ``Request.replay_start``): the worker rebuilds their sliding-window KV + and leaves their cached KV alone. + """ + if not self.prefix_replay_tokens: + return 0 + num_replay_tokens = min(self.prefix_replay_tokens, num_hit_tokens) + request.replay_start = num_hit_tokens - num_replay_tokens + return num_replay_tokens + def _update_waiting_for_remote_kv(self, request: Request) -> None: """KV Connector: update request state after async recv is finished. @@ -2998,11 +3060,16 @@ class Scheduler(SchedulerInterface): # This will cache the blocks iff caching is enabled. self.kv_cache_manager.cache_blocks(request, request.num_computed_tokens) - # on a full prompt hit, we need to re-compute the last token - # in order to be able to sample the next token - if request.num_computed_tokens == request.num_tokens: - request.num_computed_tokens = request.num_tokens - 1 - + # SWA bounded replay recomputes the tail of the hit, which covers the + # last token; otherwise a full prompt hit re-computes that token so the + # next one can be sampled. + num_replay_tokens = self._mark_prefix_replay( + request, request.num_computed_tokens + ) + if num_replay_tokens > 0: + request.num_computed_tokens -= num_replay_tokens + elif request.num_computed_tokens == request.num_tokens: + request.num_computed_tokens = request.num_tokens - 1 self.finished_recving_kv_req_ids.remove(request.request_id) def _try_promote_blocked_waiting_request(self, request: Request) -> bool: diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index 835a434c1e..9fe59dfdbe 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -492,6 +492,8 @@ class SingleTypeKVCacheManager(ABC): can resume at, from ``get_replay_boundaries``. """ + if not self.kv_cache_spec.prefix_cacheable: + return num_cached_blocks = self.num_cached_block.get(request.request_id, 0) num_full_blocks = num_tokens // self.block_size diff --git a/vllm/v1/kv_cache_interface.py b/vllm/v1/kv_cache_interface.py index 6595be14d2..fc8b88921a 100644 --- a/vllm/v1/kv_cache_interface.py +++ b/vllm/v1/kv_cache_interface.py @@ -164,6 +164,11 @@ class KVCacheSpec: """Whether this spec's group participates in prefix caching.""" return True + @property + def prefix_replay_tokens(self) -> int: + """DeepSeek-V4.1 only: bounded replay. Currently only for DSV41 SWA.""" + return 0 + @property def num_heads(self) -> int: raise NotImplementedError @@ -908,6 +913,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec): # DeepseekV4-only: see MLAAttentionSpec.model_version. alignment: int | None = None # Default to None for no padding. model_version: str | None = None + bounded_replay: bool = False # MLA stores a single latent vector per state; there is no separate V. head_size_v: int = 0 @@ -919,6 +925,14 @@ class SlidingWindowMLASpec(SlidingWindowSpec): super().__post_init__() _apply_alignment_padding(self) + @property + def prefix_cacheable(self) -> bool: + return not self.bounded_replay + + @property + def prefix_replay_tokens(self) -> int: + return self.sliding_window if self.bounded_replay else 0 + @classmethod def merge(cls, specs: list[Self]) -> Self: assert all(isinstance(spec, SlidingWindowMLASpec) for spec in specs), ( @@ -930,16 +944,18 @@ class SlidingWindowMLASpec(SlidingWindowSpec): model_version_set = set(spec.model_version for spec in specs) sliding_window_set = set(spec.sliding_window for spec in specs) extra_retained_set = set(spec.extra_retained_tokens for spec in specs) + bounded_replay_set = set(spec.bounded_replay for spec in specs) assert ( len(cache_dtype_str_set) == 1 and len(tokens_per_state_set) == 1 and len(model_version_set) == 1 and len(sliding_window_set) == 1 and len(extra_retained_set) == 1 + and len(bounded_replay_set) == 1 ), ( "All attention layers in the same KV cache group must use the same " "quantization method, tokens per state, model version, sliding " - "window size, and retained token count." + "window size, retained token count, and replay policy." ) return cls( block_size=specs[0].block_size, @@ -954,6 +970,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec): cache_dtype_str=cache_dtype_str_set.pop(), tokens_per_state=tokens_per_state_set.pop(), model_version=model_version_set.pop(), + bounded_replay=bounded_replay_set.pop(), ) def is_uniform_with_collection( @@ -962,6 +979,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec): return all( isinstance(spec, SlidingWindowMLASpec) and spec.sliding_window == self.sliding_window + and spec.bounded_replay == self.bounded_replay for spec in kv_cache_specs.values() ) @@ -1195,6 +1213,10 @@ class UniformTypeKVCacheSpecs(KVCacheSpec): def prefix_cacheable(self) -> bool: return all(spec.prefix_cacheable for spec in self.kv_cache_specs.values()) + @property + def prefix_replay_tokens(self) -> int: + return max(spec.prefix_replay_tokens for spec in self.kv_cache_specs.values()) + @property def first_spec(self) -> KVCacheSpec: """Return the first spec in the group.""" diff --git a/vllm/v1/request.py b/vllm/v1/request.py index ad875264af..dcf679093b 100644 --- a/vllm/v1/request.py +++ b/vllm/v1/request.py @@ -201,6 +201,9 @@ class Request: # in the (sparse) prefix cache; 0 means none. Set at admission for # hybrid/Mamba models when a shared prefix is detected (Marconi-style). self.shared_prefix_boundary = 0 + # DeepSeek-V4.1 only: SWA bounded replay. The request holds no + # sliding-window KV below this position; 0 when nothing replays. + self.replay_start = 0 # The number of NaNs in logits. A value greater than 0 # indicates that the output is corrupted diff --git a/vllm/v1/worker/gpu/model_states/default.py b/vllm/v1/worker/gpu/model_states/default.py index 27a9591e4b..35ca9ccba1 100644 --- a/vllm/v1/worker/gpu/model_states/default.py +++ b/vllm/v1/worker/gpu/model_states/default.py @@ -16,7 +16,10 @@ from vllm.v1.worker.gpu.attn_utils import ( from vllm.v1.worker.gpu.input_batch import InputBatch from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache from vllm.v1.worker.gpu.mm.rope import get_rope_state -from vllm.v1.worker.gpu.model_states.interface import ModelState +from vllm.v1.worker.gpu.model_states.interface import ( + ModelSpecificAttnMetadata, + ModelState, +) from vllm.v1.worker.gpu.model_states.mm_pruning import maybe_create_mm_pruner from vllm.v1.worker.gpu.model_states.prompt_embeds import PromptEmbedsState from vllm.v1.worker.gpu.states import RequestState @@ -173,6 +176,7 @@ class DefaultModelState(ModelState): kv_cache_config: KVCacheConfig, for_capture: bool = False, ubatch_idx: int = 0, + model_specific_attn_metadata: ModelSpecificAttnMetadata | None = None, ) -> dict[str, Any]: if cudagraph_mode == CUDAGraphMode.FULL: # Use padded sizes - padding is handled by model_runner.prepare_attn. @@ -229,5 +233,6 @@ class DefaultModelState(ModelState): req_idx=input_batch.idx_mapping_np, ubatch_idx=ubatch_idx, fast_prefill=input_batch.fast_prefill, + model_specific_attn_metadata=model_specific_attn_metadata, ) return attn_metadata diff --git a/vllm/v1/worker/gpu/model_states/encoder_only.py b/vllm/v1/worker/gpu/model_states/encoder_only.py index 32a70e7331..19c97e2658 100644 --- a/vllm/v1/worker/gpu/model_states/encoder_only.py +++ b/vllm/v1/worker/gpu/model_states/encoder_only.py @@ -19,6 +19,7 @@ from vllm.v1.kv_cache_interface import EncoderOnlyAttentionSpec, KVCacheConfig from vllm.v1.worker.gpu.input_batch import InputBatch from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache from vllm.v1.worker.gpu.model_states.default import DefaultModelState +from vllm.v1.worker.gpu.model_states.interface import ModelSpecificAttnMetadata from vllm.v1.worker.gpu.states import RequestState from vllm.v1.worker.utils import AttentionGroup @@ -168,6 +169,7 @@ class EncoderOnlyModelState(DefaultModelState): kv_cache_config: KVCacheConfig, for_capture: bool = False, ubatch_idx: int = 0, + model_specific_attn_metadata: ModelSpecificAttnMetadata | None = None, ) -> dict[str, Any]: assert ubatch_idx == 0, "DBO is not supported" attn_metadata = super().prepare_attn( @@ -178,6 +180,7 @@ class EncoderOnlyModelState(DefaultModelState): attn_groups, kv_cache_config, for_capture, + model_specific_attn_metadata=model_specific_attn_metadata, ) attn_metadata.update( self._build_encoder_attn_metadata(input_batch, cudagraph_mode, for_capture) diff --git a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py index 0ac3f2f63b..bfad8a1b73 100644 --- a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py +++ b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py @@ -239,8 +239,10 @@ class MambaHybridModelState(DefaultModelState): kv_cache_config: KVCacheConfig, for_capture: bool = False, ubatch_idx: int = 0, + model_specific_attn_metadata: ModelSpecificAttnMetadata | None = None, ) -> dict[str, Any]: assert ubatch_idx == 0, "DBO is not supported" + assert model_specific_attn_metadata is None if cudagraph_mode == CUDAGraphMode.FULL: num_reqs = input_batch.num_reqs_after_padding num_tokens = input_batch.num_tokens_after_padding