[Feat][Model] Support encoder-side SWA-bounded replay for DeepSeek-V4.1-Flash (#56227)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
Yifan Qiao
2026-09-18 15:01:24 -07:00
committed by GitHub
co-authored by Claude Fable 5.1 Woosuk Kwon
parent 6aab78f477
commit 1dc2d854c1
30 changed files with 1067 additions and 38 deletions
@@ -0,0 +1,112 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""SWA bounded replay in DeepseekV41ModelState: the batch's replay starts reach
the sliding-window builders, and the replayed tokens' slots are padded in the
prefix-cacheable groups only."""
from dataclasses import replace
from types import SimpleNamespace
from unittest.mock import MagicMock
import numpy as np
import pytest
import torch
from vllm.config import CUDAGraphMode
from vllm.models.deepseek_v41.nvidia.model_state import DeepseekV41ModelState
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadataBuilder
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
DEVICE = torch.device("cuda")
WINDOW = 4
# Group 0 is prefix-cacheable (compressed KV), group 1 is the replayed window.
KV_CACHE_CONFIG = SimpleNamespace(
kv_cache_groups=[
SimpleNamespace(
kv_cache_spec=SimpleNamespace(prefix_cacheable=True, prefix_replay_tokens=0)
),
SimpleNamespace(
kv_cache_spec=SimpleNamespace(
prefix_cacheable=False, prefix_replay_tokens=WINDOW
)
),
]
)
@pytest.fixture
def state(monkeypatch):
cfg = MagicMock()
cfg.model_config.enable_prompt_embeds = False
cfg.model_config.uses_mrope = False
cfg.model_config.is_multimodal_model = False
cfg.scheduler_config.max_num_seqs = 8
cfg.scheduler_config.max_num_batched_tokens = 64
seen: dict = {}
def prepare_attn(self, input_batch, cg_mode, block_tables, slot_mappings, *a, **kw):
seen["slot_mappings"] = slot_mappings.clone()
seen["extra"] = kw["model_specific_attn_metadata"]
return {}
monkeypatch.setattr(DefaultModelState, "prepare_attn", prepare_attn)
state = DeepseekV41ModelState(
cfg, SimpleNamespace(token_lookback_depth=0), None, DEVICE
)
state.seen = seen # type: ignore[attr-defined]
return state
def _batch(query_lens, seq_lens, idx_mapping, is_prefilling):
num_reqs, num_tokens = len(query_lens), sum(query_lens)
batch = InputBatch.make_dummy(num_reqs, num_tokens, InputBuffers(8, 64, DEVICE))
return replace(
batch,
query_start_loc=torch.tensor(
[0, *np.cumsum(query_lens)], dtype=torch.int32, device=DEVICE
),
idx_mapping_np=np.array(idx_mapping, dtype=np.intp),
is_prefilling_np=np.array(is_prefilling, dtype=np.bool_),
positions=torch.cat(
[torch.arange(s - q, s) for q, s in zip(query_lens, seq_lens)]
).to(DEVICE),
)
def _prepare(state, batch):
slot_mappings = torch.arange(2 * batch.num_tokens, device=DEVICE).view(2, -1)
state.prepare_attn(
batch, CUDAGraphMode.NONE, (), slot_mappings, [], KV_CACHE_CONFIG
)
swa_builder = MagicMock(spec=DeepseekSparseSWAMetadataBuilder)
extra = state.seen["extra"]
replay_start = extra.get_extra_attn_kwargs(swa_builder, batch.num_reqs)[
"replay_start"
]
assert extra.get_extra_attn_kwargs(MagicMock(), batch.num_reqs) == {}
return replay_start.tolist(), state.seen["slot_mappings"]
def test_replayed_tokens_write_only_the_window_group(state):
# Request state 3 resumes a hit at 16: positions 16..19 are replayed.
state.add_request(3, SimpleNamespace(replay_start=16))
state.add_request(1, SimpleNamespace(replay_start=0))
batch = _batch([6, 2], [22, 2], idx_mapping=[3, 1], is_prefilling=[True, True])
replay_start, slots = _prepare(state, batch)
assert replay_start == [16, 0]
padded = [PAD_SLOT_ID] * WINDOW + [4, 5, 6, 7]
assert slots[0].tolist() == padded # compressed KV keeps its cached rows
assert slots[1].tolist() == list(range(8, 16)) # the window is rebuilt
def test_only_prefills_carry_a_replay_start(state):
state.add_request(0, SimpleNamespace(replay_start=16))
# The same request, decoding: its rows sit above the hit.
batch = _batch([1], [40], idx_mapping=[0], is_prefilling=[False])
replay_start, slots = _prepare(state, batch)
assert replay_start == [0]
assert slots.tolist() == [[0], [1]]
@@ -11,6 +11,8 @@ at ``max(pos - (window - 1) - max(left - (window - 1), 0), 0)`` and ends at
``pos + right`` (inclusive).
"""
import inspect
import pytest
import torch
from typing_extensions import TypedDict
@@ -57,10 +59,13 @@ def ref_left_right(
return lefts, rights
def ref_swa_bounds(pos: int, window: int, left: int, right: int) -> tuple[int, int]:
"""Reference [start, end) window bounds for one query token."""
def ref_swa_bounds(
pos: int, window: int, left: int, right: int, replay_start: int = 0
) -> tuple[int, int]:
"""Reference [start, end) window bounds for one query token. Under SWA
bounded replay no window KV exists below ``replay_start``."""
left_add = max(left - (window - 1), 0)
start = max(pos - (window - 1) - left_add, 0)
start = max(pos - (window - 1) - left_add, 0, replay_start)
return start, pos + right + 1
@@ -97,12 +102,14 @@ def ref_swa_slot_rows(
window: int,
max_image_tokens: int,
width: int,
replay_starts: list[int] | None = None,
) -> tuple[list[list[int]], list[int]]:
"""Reference paged slot-id rows and lens for every token in the batch."""
block_table_cpu = block_table.cpu()
lefts, rights = ref_left_right(
seq_lens, query_lens, spans_per_req, max_image_tokens
)
replay_starts = replay_starts or [0] * len(seq_lens)
rows: list[list[int]] = []
lens: list[int] = []
token = 0
@@ -110,7 +117,9 @@ def ref_swa_slot_rows(
prefix_len = seq_len - query_len
for i in range(query_len):
pos = prefix_len + i
start, end = ref_swa_bounds(pos, window, lefts[token], rights[token])
start, end = ref_swa_bounds(
pos, window, lefts[token], rights[token], replay_starts[req]
)
row = []
for p in range(start, end):
blk = int(block_table_cpu[req, p // BLOCK_SIZE])
@@ -139,6 +148,7 @@ def run_swa_kernel(
swa_indices = torch.zeros(num_tokens, 1, width, dtype=torch.int32, device=device)
swa_lens = torch.zeros(num_tokens, dtype=torch.int32, device=device)
is_valid = slot_mapping >= 0
replay_start_t = torch.zeros(len(seq_lens), dtype=torch.int32, device=device)
if with_image:
lefts, rights = ref_left_right(
@@ -164,6 +174,7 @@ def run_swa_kernel(
block_table,
block_table.stride(0),
BLOCK_SIZE,
replay_start_t,
token_offset=0,
HAS_IMAGE=with_image,
TRITON_BLOCK_SIZE=1024,
@@ -289,18 +300,25 @@ def combine_case(
query_lens: list[int],
spans: list[list[tuple[int, int]]],
with_image: bool,
replay_starts: list[int] | None = None,
combine_fn=combine_topk_swa_indices,
):
"""Run combine_topk_swa_indices and return (indices, lens, expected)."""
device = torch.device("cuda")
num_reqs = len(seq_lens)
replay_starts = replay_starts or [0] * num_reqs
query_start_loc = torch.zeros(num_reqs + 1, dtype=torch.int32, device=device)
query_start_loc[1:] = torch.tensor(
query_lens, dtype=torch.int32, device=device
).cumsum(0)
num_tokens = int(query_start_loc[-1])
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device=device)
# The builder's gather covers only the context above the replay start.
gather_lens = torch.tensor(
[q + min(s - q, WINDOW - 1) for s, q in zip(seq_lens, query_lens)],
[
q + min(max(s - q - r, 0), WINDOW - 1)
for s, q, r in zip(seq_lens, query_lens, replay_starts)
],
dtype=torch.int32,
device=device,
)
@@ -319,7 +337,14 @@ def combine_case(
else:
left_t = right_t = None
combined_indices, combined_lens = combine_topk_swa_indices(
# V4.1's combine has no image windows (causal SWA only).
has_image = "left_visible" in inspect.signature(combine_fn).parameters
image_kwargs = (
dict(left_visible=left_t, right_visible=right_t, max_image_tokens=MAX_IMG)
if has_image
else {}
)
combined_indices, combined_lens = combine_fn(
topk_indices,
query_start_loc,
seq_lens_t,
@@ -329,9 +354,7 @@ def combine_case(
topk,
M,
N,
left_visible=left_t,
right_visible=right_t,
max_image_tokens=MAX_IMG,
**image_kwargs,
)
# Reference rows.
@@ -342,7 +365,7 @@ def combine_case(
MAX_IMG,
)
topk_cpu = topk_indices.cpu()
width = WINDOW + MAX_IMG
width = WINDOW + (MAX_IMG if has_image else 0)
combined_topk = (topk + width + 127) // 128 * 128
rows = []
lens = []
@@ -354,6 +377,8 @@ def combine_case(
pos = prefix_len + i
topk_len = min((pos + 1) // compress_ratio, topk)
start, end = ref_swa_bounds(pos, WINDOW, lefts[token], rights[token])
# The window never reaches below the gathered buffer.
start = max(start, gather_start)
swa_len = end - start
row = [-1] * combined_topk
for j in range(topk_len):
@@ -388,6 +413,31 @@ def test_combine_topk_swa_with_image_spans(cfg):
assert indices.cpu().tolist() == rows
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize("cfg", COMBINE_CASES)
def test_v41_combine_topk_swa_stops_at_replay_start(cfg):
"""SWA bounded replay: the gathered buffer starts at replay_start, so the
window never indexes below it."""
from vllm.models.deepseek_v41.common.ops.cache_utils import (
combine_topk_swa_indices as combine_v41,
)
# Request 0 replays [16, 40): the windows of its first rows would
# otherwise reach below 16.
indices, lens, rows, exp_lens = combine_case(
cfg["compress_ratio"],
cfg["topk"],
seq_lens=[40, 12],
query_lens=[24, 12],
spans=[[], []],
with_image=False,
replay_starts=[16, 0],
combine_fn=combine_v41,
)
assert lens.cpu().tolist() == exp_lens
assert indices.cpu().tolist() == rows
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize("cfg", COMBINE_CASES)
def test_combine_topk_swa_without_image_unchanged(cfg):
@@ -472,6 +522,7 @@ def make_builder(
DeepseekSparseSWAMetadataBuilder
] = DeepseekSparseSWAMetadataBuilder,
mm_prefix_clamp_sliding_window: bool = True,
bounded_replay: bool = False,
) -> DeepseekSparseSWAMetadataBuilder:
overrides: dict = {"sliding_window": window}
if vision:
@@ -498,6 +549,7 @@ def make_builder(
sliding_window=window,
cache_dtype_str="auto",
model_version="deepseek_v4",
bounded_replay=bounded_replay,
)
return builder_cls(
kv_cache_spec=spec,
@@ -512,11 +564,17 @@ def build_metadata(
seq_lens: list[int],
query_lens: list[int],
mm_req_doc_ranges: dict[int, list[tuple[int, int]]] | None,
replay_starts: list[int] | None = None,
):
device = torch.device("cuda")
query_start_loc, seq_lens_t, _, slot_mapping, block_table = make_batch(
seq_lens, query_lens, device
)
replay_start = (
None
if replay_starts is None
else torch.tensor(replay_starts, dtype=torch.int32, device=device)
)
return builder.build(
0,
CommonAttentionMetadata(
@@ -533,6 +591,7 @@ def build_metadata(
causal=True,
mm_req_doc_ranges=mm_req_doc_ranges,
),
replay_start=replay_start,
)
@@ -653,3 +712,108 @@ def test_v41_image_prefill_uses_causal_swa(compress_ratio, query_len):
assert absolute_keys.tolist() == list(range(lo, hi))
assert paged_lens[token] == hi - lo
assert paged_indices[token, 0, : hi - lo].tolist() == list(range(lo, hi))
# SWA bounded replay: every prefill index path clamps the window at replay_start.
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_builder_replay_start_bounds_prefill_window_and_gather():
"""Paged-direct prefill indices stop at replay_start and the FlashMLA
gather only covers the context the request may see."""
seq_lens = [40, 12, 30]
query_lens = [24, 12, 10]
replay_starts = [12, 0, 0]
builder = make_builder(vision=False)
md = build_metadata(builder, seq_lens, query_lens, None, replay_starts)
assert md.replay_start is not None
assert md.replay_start.cpu().tolist() == replay_starts
_, _, _, _, block_table = make_batch(seq_lens, query_lens, torch.device("cuda"))
rows, lens = ref_swa_slot_rows(
seq_lens,
query_lens,
[[], [], []],
block_table,
WINDOW,
MAX_IMG,
WINDOW,
replay_starts=replay_starts,
)
assert md.prefill_swa_lens.cpu().tolist() == lens
assert md.prefill_swa_indices[:, 0].cpu().tolist() == rows
# Request 0: the first replayed row (pos 16) sees [12, 16]; its last row
# (pos 39) is a full window above the replay start and sees all of it.
assert md.prefill_swa_lens[0].item() == 16 - 12 + 1
assert md.prefill_swa_lens[23].item() == WINDOW
# gather_len = query_len + min(prefix_len - replay_start, WINDOW - 1).
assert md.prefill_gather_lens.cpu().tolist() == [
24 + min(16 - 12, WINDOW - 1),
12,
10 + min(20, WINDOW - 1),
]
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_builder_replaying_group_builds_without_replay_start():
"""Graph captures build without model-specific kwargs: a replaying group's
metadata then bounds nothing."""
builder = make_builder(vision=False, bounded_replay=True)
md = build_metadata(builder, [40, 12], [24, 12], None)
assert not md.replay_start.any()
assert md.prefill_swa_lens[0].item() == WINDOW
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_v41_flashinfer_mixed_sparse_indices_respect_replay_start():
from vllm.models.deepseek_v41.common.ops.cache_utils import (
build_flashinfer_mixed_sparse_indices as build_v41,
)
device = torch.device("cuda")
# 1 decode token (req 0) + two prefill requests; req 1 resumes at 16.
seq_lens = [20, 40, 9]
query_lens = [1, 24, 9]
replay_starts = [0, 16, 0]
query_start_loc = torch.tensor([0, 1, 25, 34], dtype=torch.int32, device=device)
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device=device)
token_to_req = torch.tensor(
[0] + [1] * 24 + [2] * 9, dtype=torch.int32, device=device
)
block_table = torch.arange(3, dtype=torch.int32, device=device).view(3, 1)
kwargs = dict(
decode_swa_indices=torch.zeros((1, WINDOW), dtype=torch.int32, device=device),
decode_compressed_indices=None,
decode_compressed_topk_lens=None,
prefill_topk_indices=torch.empty((33, 0), dtype=torch.int32, device=device),
query_start_loc=query_start_loc,
seq_lens=seq_lens_t,
token_to_req_indices=token_to_req,
swa_block_table=block_table,
swa_block_size=BLOCK_SIZE,
compressed_block_table=None,
compressed_block_size=BLOCK_SIZE,
window_size=WINDOW,
compress_ratio=1,
topk=0,
)
plain, _ = build_v41(
replay_start=torch.zeros(3, dtype=torch.int32, device=device), **kwargs
)
bounded, _ = build_v41(
replay_start=torch.tensor(replay_starts, dtype=torch.int32, device=device),
**kwargs,
)
rows, _ = ref_swa_slot_rows(
seq_lens,
query_lens,
[[], [], []],
block_table,
WINDOW,
MAX_IMG,
WINDOW,
replay_starts=replay_starts,
)
# Decode row is copied through; prefill rows follow the bounded window.
assert bounded[0].cpu().tolist() == plain[0].cpu().tolist()
assert bounded[1:].cpu().tolist() == rows[1:]
@@ -669,6 +669,7 @@ def test_dsv41_flashinfer_dspark_window_matches_reference(
),
block_table=torch.arange(48, device=device, dtype=torch.int32).view(3, -1),
block_size=128,
replay_start=torch.zeros(3, device=device, dtype=torch.int32),
flashinfer_sparse_index_cache={},
max_decode_query_len=5,
)
@@ -14,6 +14,7 @@ from vllm.models.deepseek_v41.sparse_mla import (
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backends.mla.compressor_utils import (
CompressedSlotMappingKernel,
get_compressed_slot_mapping,
)
from vllm.v1.attention.backends.mla.indexer import (
BuildPrefillChunkMetadataKernel,
@@ -235,6 +236,28 @@ def test_compressed_slot_mapping_warmup_includes_index_kpool():
assert {(key.compress_ratio, key.block_size) for key in keys} == {(32, 2)}
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_compressed_slot_mapping_inherits_padded_token_slots():
"""A token whose own slot is padded (SWA bounded replay) closes no
compressed state either."""
device = torch.device("cuda")
query_start_loc = torch.tensor([0, 8], dtype=torch.int32, device=device)
seq_lens = torch.tensor([8], dtype=torch.int32, device=device)
block_table = torch.tensor([[3]], dtype=torch.int32, device=device)
slot_mapping = torch.arange(8, dtype=torch.int64, device=device)
slot_mapping[:4] = -1
compressed = get_compressed_slot_mapping(
8,
slot_mapping,
query_start_loc,
seq_lens,
block_table,
block_size=4,
compress_ratio=2,
)
assert compressed.tolist() == [-1, -1, -1, -1, -1, 3 * 4 + 2, -1, 3 * 4 + 3]
def test_index_conversion_warmup_uses_physical_block_stride():
config = SimpleNamespace(
cache_config=SimpleNamespace(block_size=64),
@@ -651,3 +651,67 @@ class TestCompressorRingGroup:
scheduler_block_size=128,
)
assert len(manager.coordinator.single_type_managers) == len(groups)
class TestSWABoundedReplayGrouping:
def test_non_cacheable_swa_leaves_hits_to_the_paged_group(self):
"""Under SWA bounded replay only the paged MLA group hashes (hash
block = its block) and the coordinator probes one spec group."""
config = _shared_layout_config()
config.cache_config.enable_prefix_caching = True
specs: dict[str, KVCacheSpec] = {
"layers.2.attn": MLAAttentionSpec(
block_size=128,
num_kv_heads=1,
head_size=584,
dtype=torch.uint8,
tokens_per_state=2,
alignment=576,
),
}
swa = SlidingWindowMLASpec(
block_size=32,
num_kv_heads=1,
head_size=584,
dtype=torch.uint8,
sliding_window=128,
alignment=576,
bounded_replay=True,
)
for layer in range(4):
specs[f"layers.{layer}.attn.swa_cache"] = swa
groups = get_kv_cache_groups(config, specs)
# The worker reads the replay window off the (packed) group specs to
# arm the window clamp; the wrapper must forward it like cacheability.
assert sorted(
(g.kv_cache_spec.prefix_cacheable, g.kv_cache_spec.prefix_replay_tokens)
for g in groups
) == [(False, 128)] * 4 + [(True, 0)]
kv_cache_config = get_kv_cache_config_from_groups(
config, groups, available_memory=64 * _get_kv_cache_bytes_per_block(groups)
)
manager = KVCacheManager(
generate_scheduler_kv_cache_config([kv_cache_config]),
max_model_len=8192,
enable_caching=True,
hash_block_size=128,
scheduler_block_size=128,
)
assert len(manager.coordinator.attention_groups) == 1
assert isinstance(
manager.coordinator.attention_groups[0].spec, MLAAttentionSpec
)
# Allocating and caching a request goes through every group's manager;
# the 32-token SWA group must skip hashing rather than assert on the
# 128-token hash block.
from tests.v1.core.utils import create_requests
request = create_requests(num_requests=1, num_tokens=256, block_size=128)[0]
assert manager.allocate_slots(request, 256) is not None
request.num_computed_tokens = 256
manager.cache_blocks(request, 256)
for single in manager.coordinator.single_type_managers:
cached = single.num_cached_block.get(request.request_id, 0)
assert cached == (2 if single.kv_cache_spec.prefix_cacheable else 0)
+3 -1
View File
@@ -6,7 +6,7 @@ import copy
from collections.abc import Callable
from dataclasses import replace
from math import lcm
from types import SimpleNamespace
from types import MethodType, SimpleNamespace
from unittest.mock import MagicMock
import pytest
@@ -533,7 +533,9 @@ def test_nixl_hisparse_full_block_import_keeps_a_writable_tail(num_tokens):
kv_cache_manager=manager,
failed_recving_kv_req_ids=set(),
finished_recving_kv_req_ids={request.request_id},
prefix_replay_tokens=0,
)
scheduler._mark_prefix_replay = MethodType(Scheduler._mark_prefix_replay, scheduler)
Scheduler._update_waiting_for_remote_kv(scheduler, request)
assert request.num_tokens - request.num_computed_tokens == 1
assert manager.allocate_slots(request, num_new_tokens=1) is not None
+279
View File
@@ -0,0 +1,279 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""SWA bounded replay: after a prefix hit the scheduler recomputes the tail of
the hit to rebuild the non-cacheable sliding-window group, keeps the hit's
blocks, and hands the worker the replay start."""
import pytest
import torch
from tests.v1.kv_connector.unit.utils import MockKVConfig, create_model_runner_output
from vllm.v1.core.sched.scheduler import Scheduler
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
SlidingWindowMLASpec,
)
from vllm.v1.outputs import ModelRunnerOutput
from vllm.v1.request import RequestStatus
from vllm.v1.structured_output import StructuredOutputManager
from .utils import create_requests, create_scheduler
BLOCK_SIZE = 16
WINDOW = 32
NUM_PROMPT_TOKENS = 100
# 100 tokens -> 6 full blocks cached -> 96-token hit.
HIT_TOKENS = NUM_PROMPT_TOKENS // BLOCK_SIZE * BLOCK_SIZE
FULL, SWA = 0, 1
def _replay_scheduler(
*,
long_prefill_token_threshold: int = 0,
use_kv_connector: MockKVConfig | None = None,
) -> Scheduler:
"""A hybrid layout: one prefix-cacheable full-attention group and one
replayed sliding-window group."""
base = create_scheduler(
block_size=BLOCK_SIZE,
enable_prefix_caching=True,
long_prefill_token_threshold=long_prefill_token_threshold,
use_kv_connector=use_kv_connector,
)
vllm_config = base.vllm_config
kv_cache_config = KVCacheConfig(
num_blocks=10000,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=BLOCK_SIZE,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["swa"],
SlidingWindowMLASpec(
block_size=BLOCK_SIZE,
num_kv_heads=1,
head_size=64,
dtype=torch.bfloat16,
sliding_window=WINDOW,
bounded_replay=True,
),
),
],
)
scheduler = Scheduler(
vllm_config=vllm_config,
kv_cache_config=kv_cache_config,
block_size=BLOCK_SIZE,
log_stats=True,
structured_output_manager=StructuredOutputManager(vllm_config),
)
scheduler.use_v2_model_runner = True
assert scheduler.prefix_replay_tokens == WINDOW
return scheduler
def _prefill(scheduler: Scheduler, request) -> None:
scheduler.add_request(request)
out = scheduler.schedule()
assert out.num_scheduled_tokens[request.request_id] == request.num_prompt_tokens
scheduler.update_from_output(out, _step_output(out, [request]))
def _step_output(out, requests) -> ModelRunnerOutput:
"""Model output for every scheduled request: a token once its prefill is
done (non-zero, so outputs never extend the all-zero shared prompt in the
cache), nothing while it is still prefilling. schedule() has already
advanced num_computed_tokens past this step's chunk."""
scheduled = [r for r in requests if r.request_id in out.num_scheduled_tokens]
return ModelRunnerOutput(
req_ids=[r.request_id for r in scheduled],
req_id_to_index={r.request_id: i for i, r in enumerate(scheduled)},
sampled_token_ids=[
[1000] if r.num_computed_tokens >= r.num_prompt_tokens else []
for r in scheduled
],
logprobs=None,
prompt_logprobs_dict={},
pooler_output=None,
)
def _new_req_data(out, request):
return next(r for r in out.scheduled_new_reqs if r.req_id == request.request_id)
def test_hit_replays_window_without_reallocating():
scheduler = _replay_scheduler()
expected_replay = WINDOW
first, second = create_requests(
num_requests=2,
num_tokens=NUM_PROMPT_TOKENS,
same_prompt=True,
block_size=BLOCK_SIZE,
)
_prefill(scheduler, first)
manager = scheduler.kv_cache_manager
scheduler.add_request(second)
out = scheduler.schedule()
replay_start = HIT_TOKENS - expected_replay
new_req = _new_req_data(out, second)
assert new_req.num_computed_tokens == replay_start
assert new_req.replay_start == replay_start
assert (
out.num_scheduled_tokens[second.request_id] == NUM_PROMPT_TOKENS - replay_start
)
first_blocks = manager.get_blocks(first.request_id).blocks
second_blocks = manager.get_blocks(second.request_id).blocks
# The full-attention hit is shared (replayed tokens included); only the
# tail block is new.
num_hit_blocks = HIT_TOKENS // BLOCK_SIZE
assert second_blocks[FULL][:num_hit_blocks] == first_blocks[FULL][:num_hit_blocks]
assert all(b.ref_cnt == 2 for b in second_blocks[FULL][:num_hit_blocks])
assert len(second_blocks[FULL]) == num_hit_blocks + 1
# The window group never hits: blocks below the window are null, the
# window (replayed and new tokens) gets fresh blocks.
num_skipped_blocks = (HIT_TOKENS - WINDOW + 1) // BLOCK_SIZE
assert all(b.is_null for b in second_blocks[SWA][:num_skipped_blocks])
num_window_blocks = len(second_blocks[SWA]) - num_skipped_blocks
assert num_window_blocks == -(-NUM_PROMPT_TOKENS // BLOCK_SIZE) - num_skipped_blocks
assert not any(b.is_null for b in second_blocks[SWA][num_skipped_blocks:])
scheduler.update_from_output(out, _step_output(out, [first, second]))
assert second.num_computed_tokens == NUM_PROMPT_TOKENS
def test_replay_only_chunks_make_progress():
"""A chunk cap smaller than the replay window schedules replay-only
chunks; they advance the request instead of stalling it."""
scheduler = _replay_scheduler(long_prefill_token_threshold=BLOCK_SIZE)
first, second = create_requests(
num_requests=2,
num_tokens=NUM_PROMPT_TOKENS,
same_prompt=True,
block_size=BLOCK_SIZE,
)
scheduler.add_request(first)
while first.num_computed_tokens < NUM_PROMPT_TOKENS:
out = scheduler.schedule()
scheduler.update_from_output(out, _step_output(out, [first]))
scheduler.add_request(second)
def full_blocks():
return scheduler.kv_cache_manager.get_blocks(second.request_id).blocks[FULL]
scheduled: list[int] = []
while second.num_computed_tokens < NUM_PROMPT_TOKENS:
out = scheduler.schedule()
if not scheduled:
new_req = _new_req_data(out, second)
assert new_req.num_computed_tokens == HIT_TOKENS - WINDOW
assert new_req.replay_start == HIT_TOKENS - WINDOW
# A replay-only chunk adopts the hit and allocates nothing new.
assert len(full_blocks()) == HIT_TOKENS // BLOCK_SIZE
scheduled.append(out.num_scheduled_tokens[second.request_id])
scheduler.update_from_output(out, _step_output(out, [first, second]))
# 32 replayed + 4 new tokens in 16-token chunks: two replay-only chunks.
assert scheduled == [BLOCK_SIZE, BLOCK_SIZE, NUM_PROMPT_TOKENS - HIT_TOKENS]
assert len(full_blocks()) == -(-NUM_PROMPT_TOKENS // BLOCK_SIZE)
assert second.status == RequestStatus.RUNNING
def test_async_remote_kv_hit_replays_after_load():
"""A KV-connector hit loaded asynchronously carries no window state
either; the replay is applied when the load completes."""
matched = 64
scheduler = _replay_scheduler(
use_kv_connector=MockKVConfig(matched_tokens=matched, is_async=True)
)
request = create_requests(
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
)[0]
scheduler.add_request(request)
out = scheduler.schedule()
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
assert request.request_id not in out.num_scheduled_tokens
scheduler.update_from_output(
out, create_model_runner_output([], finished_recving={request.request_id})
)
out = scheduler.schedule()
new_req = _new_req_data(out, request)
assert new_req.num_computed_tokens == matched - WINDOW
assert new_req.replay_start == matched - WINDOW
assert (
out.num_scheduled_tokens[request.request_id]
== NUM_PROMPT_TOKENS - matched + WINDOW
)
def test_remote_kv_hit_is_taken_in_whole_blocks():
"""A hit ending one token short of a block boundary would put the replay
window's first token in a block the sliding-window group retires, so a
connector hit is cut back to whole blocks."""
matched = 4 * BLOCK_SIZE - 1
scheduler = _replay_scheduler(
use_kv_connector=MockKVConfig(matched_tokens=matched, is_async=True)
)
request = create_requests(
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
)[0]
scheduler.add_request(request)
out = scheduler.schedule()
scheduler.update_from_output(
out, create_model_runner_output([], finished_recving={request.request_id})
)
out = scheduler.schedule()
new_req = _new_req_data(out, request)
hit = 3 * BLOCK_SIZE
assert new_req.replay_start == hit - WINDOW
assert new_req.num_computed_tokens == hit - WINDOW
swa_manager = scheduler.kv_cache_manager.coordinator.single_type_managers[SWA]
swa_blocks = scheduler.kv_cache_manager.get_blocks(request.request_id).blocks[SWA]
assert swa_blocks[new_req.replay_start // BLOCK_SIZE] is not swa_manager._null_block
@pytest.mark.parametrize("via_connector", [False, True])
def test_hit_no_longer_than_window_is_ignored(via_connector):
"""Such a hit would be recomputed in full anyway. It is not adopted (an
async load of it is not even started), so a request has computed tokens
iff it replays."""
if via_connector:
scheduler = _replay_scheduler(
use_kv_connector=MockKVConfig(matched_tokens=WINDOW, is_async=True)
)
request = create_requests(
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
)[0]
else:
scheduler = _replay_scheduler()
first = create_requests(
num_requests=1, num_tokens=WINDOW, same_prompt=True, block_size=BLOCK_SIZE
)[0]
_prefill(scheduler, first)
request = create_requests(
num_requests=1,
num_tokens=NUM_PROMPT_TOKENS,
same_prompt=True,
block_size=BLOCK_SIZE,
req_ids=["long"],
)[0]
scheduler.add_request(request)
out = scheduler.schedule()
new_req = _new_req_data(out, request)
assert request.status == RequestStatus.RUNNING
assert new_req.num_computed_tokens == 0
assert new_req.replay_start == 0
assert out.num_scheduled_tokens[request.request_id] == NUM_PROMPT_TOKENS
@@ -1505,6 +1505,7 @@ def test_failed_load_rezeroes_unwritten_skipped_blocks():
scheduler.kv_cache_manager.cache_blocks = MagicMock()
scheduler.failed_recving_kv_req_ids = {"req-1"}
scheduler.finished_recving_kv_req_ids = {"req-1"}
scheduler.prefix_replay_tokens = 0
request = MagicMock()
request.request_id = "req-1"
+7 -1
View File
@@ -227,6 +227,11 @@ class CacheConfig:
necessary for implementing this optimization in some models (e.g. Gemma3n)
"""
swa_bounded_replay: bool = True
"""Keep the sliding-window KV of models that support it (DeepSeek-V4.1)
out of prefix caching and rebuild it after a prefix hit by recomputing the
hit's last window. Requires model runner V2."""
kv_cache_memory_bytes: int | None = None
"""Size of KV Cache per GPU in bytes. By default, this is set to None
and vllm can automatically infer the kv cache size based on
@@ -281,8 +286,9 @@ class CacheConfig:
"effective_attention_block_size",
"kv_cache_size_tokens",
"kv_cache_max_concurrency",
# WIP feature toggle not impacting compiled graph shape
# Feature toggles not impacting compiled graph shape
"kv_sharing_fast_prefill",
"swa_bounded_replay",
}
from vllm.config.utils import get_hash_factors, hash_factors
+5
View File
@@ -781,6 +781,7 @@ class EngineArgs:
stream_interval: int = SchedulerConfig.stream_interval
kv_sharing_fast_prefill: bool = CacheConfig.kv_sharing_fast_prefill
swa_bounded_replay: bool = CacheConfig.swa_bounded_replay
optimization_level: OptimizationLevel = VllmConfig.optimization_level
performance_mode: PerformanceMode = VllmConfig.performance_mode
@@ -1310,6 +1311,9 @@ class EngineArgs:
cache_group.add_argument(
"--kv-sharing-fast-prefill", **cache_kwargs["kv_sharing_fast_prefill"]
)
cache_group.add_argument(
"--swa-bounded-replay", **cache_kwargs["swa_bounded_replay"]
)
cache_group.add_argument(
"--mamba-cache-dtype", **cache_kwargs["mamba_cache_dtype"]
)
@@ -2118,6 +2122,7 @@ class EngineArgs:
prefix_cache_retention_interval=self.prefix_cache_retention_interval,
kv_cache_dtype_skip_layers=self.kv_cache_dtype_skip_layers,
kv_sharing_fast_prefill=self.kv_sharing_fast_prefill,
swa_bounded_replay=self.swa_bounded_replay,
mamba_cache_dtype=self.mamba_cache_dtype,
mamba_ssm_cache_dtype=self.mamba_ssm_cache_dtype,
mamba_block_size=self.mamba_block_size,
+2
View File
@@ -495,11 +495,13 @@ class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuild
common_prefix_len: int,
common_attn_metadata: CommonAttentionMetadata,
fast_build: bool = False,
replay_start: torch.Tensor | None = None,
) -> DeepseekV4ROCMAiterSparseSWAMetadata:
base = super().build(
common_prefix_len=common_prefix_len,
common_attn_metadata=common_attn_metadata,
fast_build=fast_build,
replay_start=replay_start,
)
ragged_indices = None
+1
View File
@@ -195,6 +195,7 @@ class DeepseekV4SparseMLAMetadataBuilder(
if self.compress_ratio > 1:
slot_mapping = get_compressed_slot_mapping(
cm.num_actual_tokens,
cm.slot_mapping,
cm.query_start_loc,
cm.seq_lens,
cm.block_table_tensor.clamp_(min=0),
+2
View File
@@ -425,11 +425,13 @@ class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekV41SparseSWAMetadataBu
common_prefix_len: int,
common_attn_metadata: CommonAttentionMetadata,
fast_build: bool = False,
replay_start: torch.Tensor | None = None,
) -> DeepseekV4ROCMAiterSparseSWAMetadata:
base = super().build(
common_prefix_len=common_prefix_len,
common_attn_metadata=common_attn_metadata,
fast_build=fast_build,
replay_start=replay_start,
)
ragged_indices = None
+20
View File
@@ -512,6 +512,25 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
"decodes only on SM100."
)
swa_bounded_replay = cache_config.swa_bounded_replay
if swa_bounded_replay and not vllm_config.use_v2_model_runner:
logger.warning_once(
"SWA bounded replay needs model runner V2 (only it skips the "
"paged-KV writes of replayed tokens); the sliding-window cache "
"takes part in prefix caching instead."
)
swa_bounded_replay = False
if (
swa_bounded_replay
and vllm_config.parallel_config.prefill_context_parallel_size > 1
):
logger.warning_once(
"SWA bounded replay is off under prefill context parallelism "
"(the replayed tokens' slot padding knows the rank-local batch "
"only); the sliding-window cache takes part in prefix caching "
"instead."
)
swa_bounded_replay = False
self.swa_cache_layer = DeepseekV4SWACache(
head_dim=self.head_dim,
window_size=self.window_size,
@@ -522,6 +541,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
block_size=32,
packed_bytes_per_token=self.swa_bytes_per_token,
packed_page_alignment=self.kv_page_alignment,
bounded_replay=swa_bounded_replay,
)
# The attention layer itself was already registered with the
@@ -982,6 +982,10 @@ class CombineTopkSwaIndicesKernel(
# turns the store index below negative).
topk_len = 0
swa_start = tl.maximum(pos - (WINDOW_SIZE - 1), 0)
# gather_len already excludes context below the request's replay
# start (SWA bounded replay), so the window cannot start before
# the gathered buffer does.
swa_start = tl.maximum(swa_start, gather_start)
swa_len = pos - swa_start + 1
offset = tl.arange(0, PADDED_TOP_K)
@@ -1128,6 +1132,8 @@ def build_flashinfer_mixed_sparse_indices(
decode_is_valid_token: torch.Tensor | None = None,
swa_block_span: int | None = None,
compressed_block_span: int | None = None,
*,
replay_start: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Build the FlashInfer DSV4 sparse-index matrix for decode-first batches.
@@ -1137,6 +1143,9 @@ def build_flashinfer_mixed_sparse_indices(
per token). Decode tokens read precomputed SWA/compressed indices; prefill
tokens derive their SWA window from the position and translate local
compressed indices to global slots via the block tables.
``replay_start`` ([num_reqs], SWA bounded replay) lower-bounds every
prefill token's window: positions below it hold no window KV.
"""
assert decode_swa_indices.dtype == torch.int32
assert decode_swa_indices.dim() == 2
@@ -1227,6 +1236,7 @@ def build_flashinfer_mixed_sparse_indices(
query_start_loc,
seq_lens,
token_to_req_indices,
replay_start,
swa_block_table,
swa_block_table.stride(0),
swa_block_size,
@@ -1295,6 +1305,7 @@ def _build_flashinfer_mixed_sparse_indices_kernel(
query_start_loc_ptr,
seq_lens_ptr,
token_to_req_indices_ptr,
replay_start_ptr,
swa_block_table_ptr,
swa_block_table_stride,
swa_block_size,
@@ -1394,6 +1405,8 @@ def _build_flashinfer_mixed_sparse_indices_kernel(
token_idx_in_query = token_idx - query_start
pos = start_pos + token_idx_in_query
swa_start_pos = tl.maximum(pos - (WINDOW_SIZE - 1), 0)
# SWA bounded replay: no window KV exists below the request's replay start.
swa_start_pos = tl.maximum(swa_start_pos, tl.load(replay_start_ptr + req_idx))
swa_len = pos - swa_start_pos + 1
if COMPRESS_RATIO > 0:
topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, TOP_K)
@@ -200,8 +200,11 @@ class DeepseekSparseSWAFlashInferMetadataBuilder(DeepseekV41SparseSWAMetadataBui
common_prefix_len: int,
common_attn_metadata: CommonAttentionMetadata,
fast_build: bool = False,
replay_start: torch.Tensor | None = None,
) -> "DeepseekSparseSWAMetadata":
metadata = super().build(common_prefix_len, common_attn_metadata, fast_build)
metadata = super().build(
common_prefix_len, common_attn_metadata, fast_build, replay_start
)
num_tokens = metadata.num_decode_tokens
if not common_attn_metadata.causal and num_tokens > 0:
assert metadata.decode_swa_lens is not None
@@ -375,6 +378,7 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
assert swa_metadata.token_to_req_indices is not None
assert swa_metadata.decode_swa_indices is not None
assert swa_metadata.block_table is not None
assert swa_metadata.replay_start is not None
decode_swa_indices = swa_metadata.decode_swa_indices.reshape(
num_decode_tokens, swa_metadata.decode_swa_width
@@ -454,6 +458,7 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
decode_is_valid_token=decode_is_valid_token,
swa_block_span=swa_block_span,
compressed_block_span=compressed_block_span,
replay_start=swa_metadata.replay_start[:num_reqs],
)
if swa_only:
swa_metadata.flashinfer_sparse_index_cache["swa_only"] = (
+143 -2
View File
@@ -2,15 +2,23 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import numpy as np
import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.config import CUDAGraphMode, VllmConfig
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadataBuilder
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
from vllm.v1.core.sched.output import NewRequestData
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu.buffer_utils import UvaBufferPool
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
from vllm.v1.worker.gpu.model_states.interface import ModelSpecificAttnMetadata
from vllm.v1.worker.gpu.states import RequestState
from vllm.v1.worker.utils import AttentionGroup
@triton.jit
@@ -41,14 +49,69 @@ def _gather_lookback_kernel(
tl.store(lookback_ptr + batch_idx * DEPTH + offs, ids, mask=offs < DEPTH)
@triton.jit
def _pad_replayed_slots_kernel(
slot_mappings_ptr, # [num_groups, num_tokens_padded]
group_stride,
cacheable_groups_ptr, # [NUM_GROUPS] indices of the prefix-cacheable groups
query_start_loc_ptr, # [num_reqs + 1]
positions_ptr, # [num_tokens]
replay_start_ptr, # [num_reqs]
window,
pad_slot_id,
NUM_GROUPS: tl.constexpr,
BLOCK: tl.constexpr,
):
req = tl.program_id(0)
start = tl.load(replay_start_ptr + req)
if start == 0:
return
begin = tl.load(query_start_loc_ptr + req)
end = tl.load(query_start_loc_ptr + req + 1)
for tok in range(begin, end, BLOCK):
offs = tok + tl.arange(0, BLOCK)
pos = tl.load(positions_ptr + offs, mask=offs < end, other=0)
replayed = (offs < end) & (pos >= start) & (pos < start + window)
for i in tl.static_range(NUM_GROUPS):
group = tl.load(cacheable_groups_ptr + i)
tl.store(
slot_mappings_ptr + group * group_stride + offs,
pad_slot_id,
mask=replayed,
)
class ReplayAttnMetadata(ModelSpecificAttnMetadata):
"""Hands the batch's replay starts to the sliding-window builders."""
def __init__(self, replay_start: torch.Tensor) -> None:
self.replay_start = replay_start
def get_extra_attn_kwargs(
self, attn_metadata_builder: Any, num_reqs: int
) -> dict[str, Any]:
if isinstance(attn_metadata_builder, DeepseekSparseSWAMetadataBuilder):
return {"replay_start": self.replay_start[:num_reqs]}
return {}
class DeepseekV41ModelState(DefaultModelState):
"""DefaultModelState plus the engram lookback window.
"""DefaultModelState plus the engram lookback window and SWA bounded replay.
The engram n-gram hash needs the ids of the ``depth`` tokens preceding
each request's chunk start (see ``common/engram.py``). The runner keeps
the full token history on device, so the window is gathered there every
step: exact for prompt and generated tokens alike, whatever instance
produced their KV.
SWA bounded replay (``CacheConfig.swa_bounded_replay``) keeps the
sliding-window KV out of prefix caching and rebuilds it after a prefix hit
by recomputing the hit's last window; the scheduler tells each request the
position from which it holds window KV (``NewRequestData.replay_start``).
``prepare_attn`` gathers those per batch, pads the replayed tokens' slots
in the prefix-cacheable groups so the cached KV stays as is, and hands the
starts to the sliding-window metadata builders, whose kernels read no
window KV below them.
"""
def __init__(
@@ -67,6 +130,20 @@ class DeepseekV41ModelState(DefaultModelState):
(self.max_num_reqs, depth), -1, dtype=torch.int32, device=device
)
# Per request state index; batches gather from it (see prepare_attn).
self._replay_start_np = np.zeros(self.max_num_reqs, dtype=np.int32)
self._replay_start = torch.zeros(
self.max_num_reqs, dtype=torch.int32, device=device
)
self._replay_start_staging = UvaBufferPool(self.max_num_reqs, torch.int32)
# The replay window and the indices of the prefix-cacheable groups, from
# the KV cache config on first use; None until then.
self._replay: tuple[int, torch.Tensor] | None = None
def add_request(self, req_index: int, new_req_data: NewRequestData) -> None:
super().add_request(req_index, new_req_data)
self._replay_start_np[req_index] = new_req_data.replay_start
def prepare_inputs(
self, input_batch: InputBatch, req_states: RequestState
) -> dict[str, torch.Tensor | None]:
@@ -96,3 +173,67 @@ class DeepseekV41ModelState(DefaultModelState):
self.lookback_token_ids.fill_(-1)
model_inputs["lookback_token_ids"] = self.lookback_token_ids
return model_inputs
def prepare_attn(
self,
input_batch: InputBatch,
cudagraph_mode: CUDAGraphMode,
block_tables: tuple[torch.Tensor, ...],
slot_mappings: torch.Tensor,
attn_groups: list[list[AttentionGroup]],
kv_cache_config: KVCacheConfig,
for_capture: bool = False,
ubatch_idx: int = 0,
model_specific_attn_metadata: ModelSpecificAttnMetadata | None = None,
) -> dict[str, Any]:
if self._replay is None:
specs = [group.kv_cache_spec for group in kv_cache_config.kv_cache_groups]
self._replay = (
max(spec.prefix_replay_tokens for spec in specs),
torch.tensor(
[i for i, spec in enumerate(specs) if spec.prefix_cacheable],
dtype=torch.int32,
device=self.device,
),
)
window, cacheable_groups = self._replay
if window:
num_reqs = input_batch.num_reqs
# Decode rows sit above the hit, so only prefills carry a replay
# start; dummy batches (captures, profiling) carry none.
replay_start_np = np.where(
input_batch.is_prefilling_np[:num_reqs],
self._replay_start_np[input_batch.idx_mapping_np[:num_reqs]],
0,
).astype(np.int32)
replay_start = self._replay_start_staging.copy_to_gpu(
replay_start_np, out=self._replay_start[:num_reqs]
)
if replay_start_np.any():
# The replayed tokens rebuild window KV only: their slots in the
# prefix-cacheable groups are padded so the cached KV stays as is.
_pad_replayed_slots_kernel[(num_reqs,)](
slot_mappings,
slot_mappings.stride(0),
cacheable_groups,
input_batch.query_start_loc,
input_batch.positions,
replay_start,
window,
PAD_SLOT_ID,
NUM_GROUPS=cacheable_groups.numel(),
BLOCK=1024,
)
assert model_specific_attn_metadata is None
model_specific_attn_metadata = ReplayAttnMetadata(replay_start)
return super().prepare_attn(
input_batch,
cudagraph_mode,
block_tables,
slot_mappings,
attn_groups,
kv_cache_config,
for_capture=for_capture,
ubatch_idx=ubatch_idx,
model_specific_attn_metadata=model_specific_attn_metadata,
)
+1
View File
@@ -195,6 +195,7 @@ class DeepseekV4SparseMLAMetadataBuilder(
if self.compress_ratio > 1:
slot_mapping = get_compressed_slot_mapping(
cm.num_actual_tokens,
cm.slot_mapping,
cm.query_start_loc,
cm.seq_lens,
cm.block_table_tensor.clamp_(min=0),
@@ -42,6 +42,8 @@ class CompressedSlotMappingKernel(
@triton.jit(do_not_specialize=["block_table_stride"])
def kernel(
# [num_tokens]
compressed_slot_mapping_ptr,
# [num_tokens] the tokens' own slots
slot_mapping_ptr,
# [num_reqs + 1]
query_start_loc_ptr,
@@ -69,7 +71,12 @@ class CompressedSlotMappingKernel(
mask = offset < query_len
pos = start_pos + i + tl.arange(0, TRITON_BLOCK_SIZE)
is_valid = (pos + 1) % COMPRESS_RATIO == 0
# A replayed token has a PAD slot (SWA bounded replay: its KV is
# cached already); its compressed KV is cached too, so PAD here.
slot = tl.load(
slot_mapping_ptr + query_start + offset, mask=mask, other=PAD_ID
)
is_valid = ((pos + 1) % COMPRESS_RATIO == 0) & (slot != PAD_ID)
pos_after_compress = pos // COMPRESS_RATIO
block_ids = pos_after_compress // block_size
@@ -81,7 +88,9 @@ class CompressedSlotMappingKernel(
# NOTE
slot_ids = tl.where(is_valid, slot_ids, PAD_ID)
tl.store(slot_mapping_ptr + query_start + offset, slot_ids, mask=mask)
tl.store(
compressed_slot_mapping_ptr + query_start + offset, slot_ids, mask=mask
)
def dispatch( # type: ignore[override]
self,
@@ -121,6 +130,7 @@ class CompressedSlotMappingKernel(
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
int32_ptr = TritonWarmupTensor(torch.int32)
return dict(
compressed_slot_mapping=TritonWarmupTensor(torch.int64),
slot_mapping=TritonWarmupTensor(torch.int64),
query_start_loc=int32_ptr,
seq_lens=int32_ptr,
@@ -132,6 +142,7 @@ class CompressedSlotMappingKernel(
@kernel_launcher
def __call__(
self,
compressed_slot_mapping: torch.Tensor,
slot_mapping: torch.Tensor,
query_start_loc: torch.Tensor,
seq_lens: torch.Tensor,
@@ -149,6 +160,7 @@ class CompressedSlotMappingKernel(
def get_compressed_slot_mapping(
num_tokens: int,
slot_mapping: torch.Tensor,
query_start_loc: torch.Tensor,
seq_lens: torch.Tensor,
block_table: torch.Tensor,
@@ -156,19 +168,28 @@ def get_compressed_slot_mapping(
compress_ratio: int,
out: torch.Tensor | None = None,
) -> torch.Tensor:
"""Slot mapping for writing the compressed states of ``num_tokens`` tokens.
Every ``compress_ratio`` tokens share one compressed state, written by the
last of them: that token maps to the state's slot, the others to PAD. A
token whose own ``slot_mapping`` entry is PAD maps to PAD too: SWA bounded
replay recomputes tokens whose KV is cached already, and their compressed
states must not be rewritten either.
"""
if out is not None:
# Guard: for padded / invalid sequences.
# Negative positions produce bogus block indices that lead to illegal memory
# accesses inside the block_table load.
# NOTE: Fill -1 to the whole tensor, not just the first `num_tokens`.
out.fill_(-1)
slot_mapping = out[:num_tokens]
compressed_slot_mapping = out[:num_tokens]
else:
slot_mapping = torch.full(
compressed_slot_mapping = torch.full(
(num_tokens,), -1, dtype=torch.int64, device=query_start_loc.device
)
_COMPRESSED_SLOT_MAPPING_KERNEL(
compressed_slot_mapping,
slot_mapping,
query_start_loc,
seq_lens,
@@ -176,7 +197,7 @@ def get_compressed_slot_mapping(
block_size,
compress_ratio,
)
return slot_mapping
return compressed_slot_mapping
_COMPRESSED_SLOT_MAPPING_KERNEL = CompressedSlotMappingKernel()
+10 -1
View File
@@ -795,6 +795,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0
self.pcp_world_size = parallel_config.prefill_context_parallel_size
self.use_pcp = self.pcp_world_size > 1
self.pcp_rank = get_pcp_group().rank_in_group if self.use_pcp else 0
self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size
# NOTE(Chen):an estimated max size of flattened_kv. Need to double check.
self.max_prefill_buffer_size = get_max_prefill_buffer_size(self.vllm_config)
@@ -1217,10 +1218,18 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
factor = self.kv_cache_spec.block_size // kernel_block_size
indexer_block_table = (block_table[:, ::factor] // factor).contiguous()
padded_num_tokens = num_tokens
if self.pcp_world_size > 1:
local_slot_mapping = slot_mapping
if self.use_pcp:
# The gathered layout holds each rank's local tokens, padded, in
# rank order, so this rank's segment lines up with query_start_loc.
padded_num_tokens = slot_mapping.shape[0] // self.pcp_world_size
local_slot_mapping = slot_mapping[
self.pcp_rank * padded_num_tokens : (self.pcp_rank + 1)
* padded_num_tokens
]
compressed_slot_mapping = get_compressed_slot_mapping(
num_tokens,
local_slot_mapping,
query_start_loc,
seq_lens,
indexer_block_table,
+50 -5
View File
@@ -81,9 +81,12 @@ class DeepseekV4SWACache(torch.nn.Module, AttentionLayerBase):
block_size: int = 64,
packed_bytes_per_token: int = 584,
packed_page_alignment: int = 576,
bounded_replay: bool = False,
):
super().__init__()
self.backend_cls = backend_cls or DeepseekSparseSWABackend
# DeepseekV4.1 SWA bounded replay.
self.bounded_replay = bounded_replay
self.kv_cache = torch.tensor([])
self.head_dim = head_dim
self.window_size = window_size
@@ -117,6 +120,7 @@ class DeepseekV4SWACache(torch.nn.Module, AttentionLayerBase):
"nvfp4_ds_mla",
)
return SlidingWindowMLASpec(
bounded_replay=self.bounded_replay,
block_size=self.block_size,
num_kv_heads=1,
head_size=self.head_dim,
@@ -198,6 +202,10 @@ class DeepseekSparseSWAMetadata:
# None when the model is text-only or the batch has no image spans.
prefill_left_visible: torch.Tensor | None = None
prefill_right_visible: torch.Tensor | None = None
# SWA bounded replay: [num_reqs] lower bound of the positions a request's
# window attention may read (SWA bounded replay);
# zeros when nothing replays.
replay_start: torch.Tensor | None = None
# Number of decode/prefill requests/tokens (batch is reordered: decodes first)
num_decodes: int = 0
@@ -256,10 +264,12 @@ class DeepseekSparseSWAMetadata:
has_compressed = compress_ratio > 1
# query_len <= max_num_batched_tokens and
# gather_len = query_len + min(prefix_len, window_size - 1), so the
# worst-case gathered width is bounded by
# max_num_batched_tokens + window_size - 1. The compressed prefix pool
# is bounded by ceil(max_model_len / compress_ratio).
# gather_len <= query_len + min(prefix_len, window_size - 1) (a replay
# boundary only shrinks it), so the worst-case gathered width is
# bounded by max_num_batched_tokens + window_size - 1. The compressed
# prefix pool is bounded by ceil(max_model_len / compress_ratio). The
# CPU gather lengths below are this upper bound, for chunk planning
# only; the kernels use the exact per-request values.
max_workspace_area = prefill_chunk_size * (
(cdiv(self.prefill_max_model_len, compress_ratio) if has_compressed else 0)
+ self.prefill_window_size
@@ -331,6 +341,7 @@ class ComputePrefillMetadataKernel(
# Inputs
seq_lens_ptr,
query_start_loc_ptr,
replay_start_ptr,
num_prefills,
num_decodes,
window_size,
@@ -352,7 +363,10 @@ class ComputePrefillMetadataKernel(
query_len = qsl_end - qsl_start
prefix_len = seq_len - query_len
gather_len = query_len + tl.minimum(prefix_len, window_size - 1)
# Context below replay_start has no window KV (SWA bounded replay).
replay_start = tl.load(replay_start_ptr + num_decodes + safe_offset, mask=mask)
visible_prefix = tl.maximum(prefix_len - replay_start, 0)
gather_len = query_len + tl.minimum(visible_prefix, window_size - 1)
tl.store(prefill_gather_lens_ptr + offset, gather_len, mask=mask)
@@ -384,6 +398,7 @@ class ComputePrefillMetadataKernel(
prefill_gather_lens=int32_ptr,
seq_lens=int32_ptr,
query_start_loc=int32_ptr,
replay_start=int32_ptr,
num_prefills=compile_key.block_size,
num_decodes=0,
window_size=1,
@@ -395,6 +410,7 @@ class ComputePrefillMetadataKernel(
prefill_gather_lens: torch.Tensor,
seq_lens: torch.Tensor,
query_start_loc: torch.Tensor,
replay_start: torch.Tensor,
num_prefills: int,
num_decodes: int,
window_size: int,
@@ -550,12 +566,19 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
)
self.decode_swa_indices_noncausal: torch.Tensor | None = None
self._max_tokens = max_tokens
# replay_start when no group replays: no lower bound.
self.no_replay_start = torch.zeros(
self.vllm_config.scheduler_config.max_num_seqs + 1,
dtype=torch.int32,
device=self.device,
)
def build(
self,
common_prefix_len: int,
common_attn_metadata: CommonAttentionMetadata,
fast_build: bool = False,
replay_start: torch.Tensor | None = None,
) -> DeepseekSparseSWAMetadata:
"""Build SWA metadata for mixed decode/prefill batches.
@@ -564,6 +587,10 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
separate window_topk_idxs for each portion.
For prefill, we use chunked prefill to align with the indexer's chunking.
``replay_start`` ([num_reqs], SWA bounded replay): the position from
which each request holds window KV; the model state passes it for every
batch of a replaying group, graph captures pass nothing.
"""
seq_lens = common_attn_metadata.seq_lens
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
@@ -587,6 +614,9 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
is_valid_token = self.is_valid_token[: slot_mapping.shape[0]]
is_valid_token.copy_(slot_mapping >= 0)
if replay_start is None:
# Graph captures build without it; their dummy batches replay nothing.
replay_start = self.no_replay_start
non_causal = not common_attn_metadata.causal
decode_swa_width = (
@@ -609,6 +639,8 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
device=self.device,
)
decode_swa_indices = self.decode_swa_indices_noncausal
# No replay_start: decode rows sit above the hit, so a
# request's replayed window never reaches them.
_COMPUTE_DSPARK_NONCAUSAL_SWA_INDICES_KERNEL(
decode_swa_indices,
self.decode_swa_lens,
@@ -637,6 +669,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
is_valid_token,
block_table,
self.block_size,
replay_start,
num_tokens=num_decode_tokens,
token_offset=0,
)
@@ -683,6 +716,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
is_valid_token,
block_table,
self.block_size,
replay_start,
num_tokens=num_prefill_tokens,
token_offset=num_decode_tokens,
has_image=has_image,
@@ -696,6 +730,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
seq_lens_cpu,
query_start_loc,
query_start_loc_cpu,
replay_start,
)
# Per-layer-type tile-scheduler plan holders. Empty FlashMLASchedMeta
@@ -727,6 +762,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
),
prefill_left_visible=prefill_left_visible,
prefill_right_visible=prefill_right_visible,
replay_start=replay_start,
block_size=self.block_size,
num_decodes=num_decodes,
num_prefills=num_prefills,
@@ -803,6 +839,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
assert metadata.is_valid_token is not None
assert metadata.decode_swa_indices is not None
assert metadata.decode_swa_lens is not None
assert metadata.replay_start is not None
_COMPUTE_SWA_INDICES_AND_LENS_KERNEL(
metadata.decode_swa_indices,
@@ -817,6 +854,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
metadata.is_valid_token,
metadata.block_table,
self.block_size,
metadata.replay_start,
num_tokens=metadata.num_decode_tokens,
token_offset=0,
)
@@ -872,6 +910,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
seq_lens_cpu: torch.Tensor | None,
query_start_loc: torch.Tensor,
query_start_loc_cpu: torch.Tensor,
replay_start: torch.Tensor,
) -> dict[str, torch.Tensor | int | None]:
"""Pre-compute DeepseekV4 prefill metadata during the metadata build phase.
@@ -893,6 +932,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
pfx_gather_lens,
seq_lens,
query_start_loc,
replay_start,
num_prefills,
num_decodes,
self.window_size,
@@ -982,6 +1022,7 @@ def _compute_swa_indices_and_lens_kernel(
block_table_ptr,
block_table_stride,
block_size,
replay_start_ptr,
token_offset,
HAS_IMAGE: tl.constexpr,
TRITON_BLOCK_SIZE: tl.constexpr,
@@ -1022,6 +1063,8 @@ def _compute_swa_indices_and_lens_kernel(
right = 0
left_add = tl.maximum(left - (window_size - 1), 0)
start_pos = tl.maximum(pos - (window_size - 1) - left_add, 0)
# SWA bounded replay: no window KV exists below the request's replay start.
start_pos = tl.maximum(start_pos, tl.load(replay_start_ptr + req_idx))
end_pos = pos + right + 1
swa_len = end_pos - start_pos
@@ -1111,6 +1154,7 @@ class ComputeSWAIndicesAndLensKernel(
is_valid_token=TritonWarmupTensor(torch.bool),
block_table=TritonWarmupTensor(torch.int32, shape=(1, 1), strides=(1, 1)),
block_size=compile_key.block_size,
replay_start=int32_ptr,
num_tokens=1,
token_offset=0,
has_image=bool(compile_key.has_image),
@@ -1131,6 +1175,7 @@ class ComputeSWAIndicesAndLensKernel(
is_valid_token: torch.Tensor,
block_table: torch.Tensor,
block_size: int,
replay_start: torch.Tensor,
*,
num_tokens: int,
token_offset: int,
+9 -4
View File
@@ -465,12 +465,17 @@ class KVCacheManager:
A list of new allocated blocks.
"""
# When loading KV data asynchronously, we may have zero new tokens to
# compute while still allocating slots for externally computed tokens.
if num_new_tokens == 0 and num_external_computed_tokens == 0:
# A step may need no slots of its own while still adopting computed
# tokens: an async KV load, or a chunk that ends inside the replayed
# range of a hit (SWA bounded replay).
if (
num_new_tokens == 0
and num_external_computed_tokens == 0
and num_new_computed_tokens == 0
):
raise ValueError(
"num_new_tokens must be greater than 0 when there are no "
"external computed tokens"
"computed tokens to adopt"
)
if new_computed_blocks is not None:
+3
View File
@@ -48,6 +48,8 @@ class NewRequestData:
# Only used for v2 model runner.
prefill_token_ids: list[int] | None = None
# DeepSeek-V4.1 only: SWA bounded replay; see Request.replay_start.
replay_start: int = 0
@classmethod
def from_request(
@@ -73,6 +75,7 @@ class NewRequestData:
prompt_embeds=request.prompt_embeds,
prompt_is_token_ids=request.prompt_is_token_ids,
prefill_token_ids=prefill_token_ids,
replay_start=request.replay_start,
)
@property
+73 -6
View File
@@ -271,6 +271,18 @@ class Scheduler(SchedulerInterface):
self.use_eagle_block_drop = False
self.num_spec_tokens = vllm_config.num_speculative_tokens
self.num_lookahead_tokens = vllm_config.num_lookahead_tokens
# DSV41 SWA bounded replay: groups that declare a replay window are rebuilt
# after a prefix hit by recomputing its trailing tokens. One window
# for all such groups, so the rewind matches every group's allocation.
replay_windows = {
group.kv_cache_spec.prefix_replay_tokens
for group in kv_cache_config.kv_cache_groups
if group.kv_cache_spec.prefix_replay_tokens > 0
}
assert len(replay_windows) <= 1, (
f"Prefix replay windows should agree: {sorted(replay_windows)}"
)
self.prefix_replay_tokens = replay_windows.pop() if replay_windows else 0
self.num_prefill_lookahead = vllm_config.num_prefill_lookahead_tokens
self.dynamic_sd_lookup: list[int] | None = None
if speculative_config is not None:
@@ -946,6 +958,17 @@ class Scheduler(SchedulerInterface):
step_skipped_waiting.prepend_request(request)
continue
if self.prefix_replay_tokens:
# SWA bounded replay recomputes the hit's last
# window, from the block holding its first
# token; the sliding-window groups retire
# whole blocks below the window of the hit's
# next token. Hits end on a block boundary, or
# a hit ending one token short of one would
# retire the block the replay starts in.
ext_tokens -= ext_tokens % self.block_size
load_kv_async = load_kv_async and ext_tokens > 0
if partial_tail and ext_tokens > partial_tail:
# Remote strictly exceeds the full local hit: drop the
# sub-block tail so no CoW is needed, and let the load
@@ -988,6 +1011,17 @@ class Scheduler(SchedulerInterface):
num_new_local_computed_tokens + num_external_computed_tokens
)
assert num_computed_tokens <= request.num_tokens
if 0 < num_computed_tokens <= self.prefix_replay_tokens:
# SWA bounded replay: a hit no longer than the replayed
# window would be recomputed in full and save nothing.
new_computed_blocks = (
self.kv_cache_manager.empty_kv_cache_blocks
)
num_new_local_computed_tokens = 0
num_external_computed_tokens = 0
connector_prefix_cache_hits = 0
num_computed_tokens = 0
load_kv_async = False
# Skip request with pending mm encoding prefetches
if self._ec_transfer_pending(request, num_computed_tokens):
@@ -1022,6 +1056,11 @@ class Scheduler(SchedulerInterface):
new_encoder_compute_budget = encoder_compute_budget
pad_spec_decode = False
# SWA bounded replay: recompute the tail of the hit without
# rewriting its cached KV. An async load replays once the KV
# has arrived (_update_waiting_for_remote_kv).
num_replay_tokens = 0
if load_kv_async:
# KVTransfer: loading remote KV, do not allocate for new work.
assert num_external_computed_tokens > 0
@@ -1032,6 +1071,12 @@ class Scheduler(SchedulerInterface):
break
else:
request_token_budget = min(token_budget, input_budget - draft_slots)
if did_prefix_cache_lookup:
# Fresh admission: local and/or sync external hit.
num_replay_tokens = self._mark_prefix_replay(
request, num_computed_tokens
)
num_computed_tokens -= num_replay_tokens
# Number of tokens to be scheduled.
# We use `request.num_tokens` instead of
# `request.num_prompt_tokens` to consider the resumed
@@ -1159,9 +1204,12 @@ class Scheduler(SchedulerInterface):
+ self._spec_decode_step_blocks()
)
# Replayed tokens are already counted in the adopted hit; a
# chunk that ends inside the replayed range needs no new slots.
num_tokens_past_hit = max(num_new_tokens - num_replay_tokens, 0)
new_blocks = self.kv_cache_manager.allocate_slots(
request,
num_new_tokens,
num_tokens_past_hit,
num_new_computed_tokens=num_new_local_computed_tokens,
new_computed_blocks=new_computed_blocks,
num_lookahead_tokens=effective_lookahead_tokens,
@@ -2963,6 +3011,20 @@ class Scheduler(SchedulerInterface):
self._request_remaining_blocks(req) for req in self._inflight_prefills
)
def _mark_prefix_replay(self, request: Request, num_hit_tokens: int) -> int:
"""Record where a prefix hit's replay starts and return the number of
hit tokens to recompute; the caller rewinds the computed count by it.
The replayed tokens are the hit's last window (see
``Request.replay_start``): the worker rebuilds their sliding-window KV
and leaves their cached KV alone.
"""
if not self.prefix_replay_tokens:
return 0
num_replay_tokens = min(self.prefix_replay_tokens, num_hit_tokens)
request.replay_start = num_hit_tokens - num_replay_tokens
return num_replay_tokens
def _update_waiting_for_remote_kv(self, request: Request) -> None:
"""KV Connector: update request state after async recv is finished.
@@ -2998,11 +3060,16 @@ class Scheduler(SchedulerInterface):
# This will cache the blocks iff caching is enabled.
self.kv_cache_manager.cache_blocks(request, request.num_computed_tokens)
# on a full prompt hit, we need to re-compute the last token
# in order to be able to sample the next token
if request.num_computed_tokens == request.num_tokens:
request.num_computed_tokens = request.num_tokens - 1
# SWA bounded replay recomputes the tail of the hit, which covers the
# last token; otherwise a full prompt hit re-computes that token so the
# next one can be sampled.
num_replay_tokens = self._mark_prefix_replay(
request, request.num_computed_tokens
)
if num_replay_tokens > 0:
request.num_computed_tokens -= num_replay_tokens
elif request.num_computed_tokens == request.num_tokens:
request.num_computed_tokens = request.num_tokens - 1
self.finished_recving_kv_req_ids.remove(request.request_id)
def _try_promote_blocked_waiting_request(self, request: Request) -> bool:
@@ -492,6 +492,8 @@ class SingleTypeKVCacheManager(ABC):
can resume at, from ``get_replay_boundaries``.
"""
if not self.kv_cache_spec.prefix_cacheable:
return
num_cached_blocks = self.num_cached_block.get(request.request_id, 0)
num_full_blocks = num_tokens // self.block_size
+23 -1
View File
@@ -164,6 +164,11 @@ class KVCacheSpec:
"""Whether this spec's group participates in prefix caching."""
return True
@property
def prefix_replay_tokens(self) -> int:
"""DeepSeek-V4.1 only: bounded replay. Currently only for DSV41 SWA."""
return 0
@property
def num_heads(self) -> int:
raise NotImplementedError
@@ -908,6 +913,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
# DeepseekV4-only: see MLAAttentionSpec.model_version.
alignment: int | None = None # Default to None for no padding.
model_version: str | None = None
bounded_replay: bool = False
# MLA stores a single latent vector per state; there is no separate V.
head_size_v: int = 0
@@ -919,6 +925,14 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
super().__post_init__()
_apply_alignment_padding(self)
@property
def prefix_cacheable(self) -> bool:
return not self.bounded_replay
@property
def prefix_replay_tokens(self) -> int:
return self.sliding_window if self.bounded_replay else 0
@classmethod
def merge(cls, specs: list[Self]) -> Self:
assert all(isinstance(spec, SlidingWindowMLASpec) for spec in specs), (
@@ -930,16 +944,18 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
model_version_set = set(spec.model_version for spec in specs)
sliding_window_set = set(spec.sliding_window for spec in specs)
extra_retained_set = set(spec.extra_retained_tokens for spec in specs)
bounded_replay_set = set(spec.bounded_replay for spec in specs)
assert (
len(cache_dtype_str_set) == 1
and len(tokens_per_state_set) == 1
and len(model_version_set) == 1
and len(sliding_window_set) == 1
and len(extra_retained_set) == 1
and len(bounded_replay_set) == 1
), (
"All attention layers in the same KV cache group must use the same "
"quantization method, tokens per state, model version, sliding "
"window size, and retained token count."
"window size, retained token count, and replay policy."
)
return cls(
block_size=specs[0].block_size,
@@ -954,6 +970,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
cache_dtype_str=cache_dtype_str_set.pop(),
tokens_per_state=tokens_per_state_set.pop(),
model_version=model_version_set.pop(),
bounded_replay=bounded_replay_set.pop(),
)
def is_uniform_with_collection(
@@ -962,6 +979,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
return all(
isinstance(spec, SlidingWindowMLASpec)
and spec.sliding_window == self.sliding_window
and spec.bounded_replay == self.bounded_replay
for spec in kv_cache_specs.values()
)
@@ -1195,6 +1213,10 @@ class UniformTypeKVCacheSpecs(KVCacheSpec):
def prefix_cacheable(self) -> bool:
return all(spec.prefix_cacheable for spec in self.kv_cache_specs.values())
@property
def prefix_replay_tokens(self) -> int:
return max(spec.prefix_replay_tokens for spec in self.kv_cache_specs.values())
@property
def first_spec(self) -> KVCacheSpec:
"""Return the first spec in the group."""
+3
View File
@@ -201,6 +201,9 @@ class Request:
# in the (sparse) prefix cache; 0 means none. Set at admission for
# hybrid/Mamba models when a shared prefix is detected (Marconi-style).
self.shared_prefix_boundary = 0
# DeepSeek-V4.1 only: SWA bounded replay. The request holds no
# sliding-window KV below this position; 0 when nothing replays.
self.replay_start = 0
# The number of NaNs in logits. A value greater than 0
# indicates that the output is corrupted
+6 -1
View File
@@ -16,7 +16,10 @@ from vllm.v1.worker.gpu.attn_utils import (
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
from vllm.v1.worker.gpu.mm.rope import get_rope_state
from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.gpu.model_states.interface import (
ModelSpecificAttnMetadata,
ModelState,
)
from vllm.v1.worker.gpu.model_states.mm_pruning import maybe_create_mm_pruner
from vllm.v1.worker.gpu.model_states.prompt_embeds import PromptEmbedsState
from vllm.v1.worker.gpu.states import RequestState
@@ -173,6 +176,7 @@ class DefaultModelState(ModelState):
kv_cache_config: KVCacheConfig,
for_capture: bool = False,
ubatch_idx: int = 0,
model_specific_attn_metadata: ModelSpecificAttnMetadata | None = None,
) -> dict[str, Any]:
if cudagraph_mode == CUDAGraphMode.FULL:
# Use padded sizes - padding is handled by model_runner.prepare_attn.
@@ -229,5 +233,6 @@ class DefaultModelState(ModelState):
req_idx=input_batch.idx_mapping_np,
ubatch_idx=ubatch_idx,
fast_prefill=input_batch.fast_prefill,
model_specific_attn_metadata=model_specific_attn_metadata,
)
return attn_metadata
@@ -19,6 +19,7 @@ from vllm.v1.kv_cache_interface import EncoderOnlyAttentionSpec, KVCacheConfig
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
from vllm.v1.worker.gpu.model_states.interface import ModelSpecificAttnMetadata
from vllm.v1.worker.gpu.states import RequestState
from vllm.v1.worker.utils import AttentionGroup
@@ -168,6 +169,7 @@ class EncoderOnlyModelState(DefaultModelState):
kv_cache_config: KVCacheConfig,
for_capture: bool = False,
ubatch_idx: int = 0,
model_specific_attn_metadata: ModelSpecificAttnMetadata | None = None,
) -> dict[str, Any]:
assert ubatch_idx == 0, "DBO is not supported"
attn_metadata = super().prepare_attn(
@@ -178,6 +180,7 @@ class EncoderOnlyModelState(DefaultModelState):
attn_groups,
kv_cache_config,
for_capture,
model_specific_attn_metadata=model_specific_attn_metadata,
)
attn_metadata.update(
self._build_encoder_attn_metadata(input_batch, cudagraph_mode, for_capture)
@@ -239,8 +239,10 @@ class MambaHybridModelState(DefaultModelState):
kv_cache_config: KVCacheConfig,
for_capture: bool = False,
ubatch_idx: int = 0,
model_specific_attn_metadata: ModelSpecificAttnMetadata | None = None,
) -> dict[str, Any]:
assert ubatch_idx == 0, "DBO is not supported"
assert model_specific_attn_metadata is None
if cudagraph_mode == CUDAGraphMode.FULL:
num_reqs = input_batch.num_reqs_after_padding
num_tokens = input_batch.num_tokens_after_padding