[Feat][Model] Support encoder-side SWA-bounded replay for DeepSeek-V4.1-Flash (#56227)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
co-authored by
Claude Fable 5.1
Woosuk Kwon
parent
6aab78f477
commit
1dc2d854c1
@@ -0,0 +1,112 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""SWA bounded replay in DeepseekV41ModelState: the batch's replay starts reach
|
||||
the sliding-window builders, and the replayed tokens' slots are padded in the
|
||||
prefix-cacheable groups only."""
|
||||
|
||||
from dataclasses import replace
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.models.deepseek_v41.nvidia.model_state import DeepseekV41ModelState
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadataBuilder
|
||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
|
||||
|
||||
pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
|
||||
|
||||
DEVICE = torch.device("cuda")
|
||||
WINDOW = 4
|
||||
# Group 0 is prefix-cacheable (compressed KV), group 1 is the replayed window.
|
||||
KV_CACHE_CONFIG = SimpleNamespace(
|
||||
kv_cache_groups=[
|
||||
SimpleNamespace(
|
||||
kv_cache_spec=SimpleNamespace(prefix_cacheable=True, prefix_replay_tokens=0)
|
||||
),
|
||||
SimpleNamespace(
|
||||
kv_cache_spec=SimpleNamespace(
|
||||
prefix_cacheable=False, prefix_replay_tokens=WINDOW
|
||||
)
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def state(monkeypatch):
|
||||
cfg = MagicMock()
|
||||
cfg.model_config.enable_prompt_embeds = False
|
||||
cfg.model_config.uses_mrope = False
|
||||
cfg.model_config.is_multimodal_model = False
|
||||
cfg.scheduler_config.max_num_seqs = 8
|
||||
cfg.scheduler_config.max_num_batched_tokens = 64
|
||||
seen: dict = {}
|
||||
|
||||
def prepare_attn(self, input_batch, cg_mode, block_tables, slot_mappings, *a, **kw):
|
||||
seen["slot_mappings"] = slot_mappings.clone()
|
||||
seen["extra"] = kw["model_specific_attn_metadata"]
|
||||
return {}
|
||||
|
||||
monkeypatch.setattr(DefaultModelState, "prepare_attn", prepare_attn)
|
||||
state = DeepseekV41ModelState(
|
||||
cfg, SimpleNamespace(token_lookback_depth=0), None, DEVICE
|
||||
)
|
||||
state.seen = seen # type: ignore[attr-defined]
|
||||
return state
|
||||
|
||||
|
||||
def _batch(query_lens, seq_lens, idx_mapping, is_prefilling):
|
||||
num_reqs, num_tokens = len(query_lens), sum(query_lens)
|
||||
batch = InputBatch.make_dummy(num_reqs, num_tokens, InputBuffers(8, 64, DEVICE))
|
||||
return replace(
|
||||
batch,
|
||||
query_start_loc=torch.tensor(
|
||||
[0, *np.cumsum(query_lens)], dtype=torch.int32, device=DEVICE
|
||||
),
|
||||
idx_mapping_np=np.array(idx_mapping, dtype=np.intp),
|
||||
is_prefilling_np=np.array(is_prefilling, dtype=np.bool_),
|
||||
positions=torch.cat(
|
||||
[torch.arange(s - q, s) for q, s in zip(query_lens, seq_lens)]
|
||||
).to(DEVICE),
|
||||
)
|
||||
|
||||
|
||||
def _prepare(state, batch):
|
||||
slot_mappings = torch.arange(2 * batch.num_tokens, device=DEVICE).view(2, -1)
|
||||
state.prepare_attn(
|
||||
batch, CUDAGraphMode.NONE, (), slot_mappings, [], KV_CACHE_CONFIG
|
||||
)
|
||||
swa_builder = MagicMock(spec=DeepseekSparseSWAMetadataBuilder)
|
||||
extra = state.seen["extra"]
|
||||
replay_start = extra.get_extra_attn_kwargs(swa_builder, batch.num_reqs)[
|
||||
"replay_start"
|
||||
]
|
||||
assert extra.get_extra_attn_kwargs(MagicMock(), batch.num_reqs) == {}
|
||||
return replay_start.tolist(), state.seen["slot_mappings"]
|
||||
|
||||
|
||||
def test_replayed_tokens_write_only_the_window_group(state):
|
||||
# Request state 3 resumes a hit at 16: positions 16..19 are replayed.
|
||||
state.add_request(3, SimpleNamespace(replay_start=16))
|
||||
state.add_request(1, SimpleNamespace(replay_start=0))
|
||||
batch = _batch([6, 2], [22, 2], idx_mapping=[3, 1], is_prefilling=[True, True])
|
||||
replay_start, slots = _prepare(state, batch)
|
||||
assert replay_start == [16, 0]
|
||||
padded = [PAD_SLOT_ID] * WINDOW + [4, 5, 6, 7]
|
||||
assert slots[0].tolist() == padded # compressed KV keeps its cached rows
|
||||
assert slots[1].tolist() == list(range(8, 16)) # the window is rebuilt
|
||||
|
||||
|
||||
def test_only_prefills_carry_a_replay_start(state):
|
||||
state.add_request(0, SimpleNamespace(replay_start=16))
|
||||
# The same request, decoding: its rows sit above the hit.
|
||||
batch = _batch([1], [40], idx_mapping=[0], is_prefilling=[False])
|
||||
replay_start, slots = _prepare(state, batch)
|
||||
assert replay_start == [0]
|
||||
assert slots.tolist() == [[0], [1]]
|
||||
@@ -11,6 +11,8 @@ at ``max(pos - (window - 1) - max(left - (window - 1), 0), 0)`` and ends at
|
||||
``pos + right`` (inclusive).
|
||||
"""
|
||||
|
||||
import inspect
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from typing_extensions import TypedDict
|
||||
@@ -57,10 +59,13 @@ def ref_left_right(
|
||||
return lefts, rights
|
||||
|
||||
|
||||
def ref_swa_bounds(pos: int, window: int, left: int, right: int) -> tuple[int, int]:
|
||||
"""Reference [start, end) window bounds for one query token."""
|
||||
def ref_swa_bounds(
|
||||
pos: int, window: int, left: int, right: int, replay_start: int = 0
|
||||
) -> tuple[int, int]:
|
||||
"""Reference [start, end) window bounds for one query token. Under SWA
|
||||
bounded replay no window KV exists below ``replay_start``."""
|
||||
left_add = max(left - (window - 1), 0)
|
||||
start = max(pos - (window - 1) - left_add, 0)
|
||||
start = max(pos - (window - 1) - left_add, 0, replay_start)
|
||||
return start, pos + right + 1
|
||||
|
||||
|
||||
@@ -97,12 +102,14 @@ def ref_swa_slot_rows(
|
||||
window: int,
|
||||
max_image_tokens: int,
|
||||
width: int,
|
||||
replay_starts: list[int] | None = None,
|
||||
) -> tuple[list[list[int]], list[int]]:
|
||||
"""Reference paged slot-id rows and lens for every token in the batch."""
|
||||
block_table_cpu = block_table.cpu()
|
||||
lefts, rights = ref_left_right(
|
||||
seq_lens, query_lens, spans_per_req, max_image_tokens
|
||||
)
|
||||
replay_starts = replay_starts or [0] * len(seq_lens)
|
||||
rows: list[list[int]] = []
|
||||
lens: list[int] = []
|
||||
token = 0
|
||||
@@ -110,7 +117,9 @@ def ref_swa_slot_rows(
|
||||
prefix_len = seq_len - query_len
|
||||
for i in range(query_len):
|
||||
pos = prefix_len + i
|
||||
start, end = ref_swa_bounds(pos, window, lefts[token], rights[token])
|
||||
start, end = ref_swa_bounds(
|
||||
pos, window, lefts[token], rights[token], replay_starts[req]
|
||||
)
|
||||
row = []
|
||||
for p in range(start, end):
|
||||
blk = int(block_table_cpu[req, p // BLOCK_SIZE])
|
||||
@@ -139,6 +148,7 @@ def run_swa_kernel(
|
||||
swa_indices = torch.zeros(num_tokens, 1, width, dtype=torch.int32, device=device)
|
||||
swa_lens = torch.zeros(num_tokens, dtype=torch.int32, device=device)
|
||||
is_valid = slot_mapping >= 0
|
||||
replay_start_t = torch.zeros(len(seq_lens), dtype=torch.int32, device=device)
|
||||
|
||||
if with_image:
|
||||
lefts, rights = ref_left_right(
|
||||
@@ -164,6 +174,7 @@ def run_swa_kernel(
|
||||
block_table,
|
||||
block_table.stride(0),
|
||||
BLOCK_SIZE,
|
||||
replay_start_t,
|
||||
token_offset=0,
|
||||
HAS_IMAGE=with_image,
|
||||
TRITON_BLOCK_SIZE=1024,
|
||||
@@ -289,18 +300,25 @@ def combine_case(
|
||||
query_lens: list[int],
|
||||
spans: list[list[tuple[int, int]]],
|
||||
with_image: bool,
|
||||
replay_starts: list[int] | None = None,
|
||||
combine_fn=combine_topk_swa_indices,
|
||||
):
|
||||
"""Run combine_topk_swa_indices and return (indices, lens, expected)."""
|
||||
device = torch.device("cuda")
|
||||
num_reqs = len(seq_lens)
|
||||
replay_starts = replay_starts or [0] * num_reqs
|
||||
query_start_loc = torch.zeros(num_reqs + 1, dtype=torch.int32, device=device)
|
||||
query_start_loc[1:] = torch.tensor(
|
||||
query_lens, dtype=torch.int32, device=device
|
||||
).cumsum(0)
|
||||
num_tokens = int(query_start_loc[-1])
|
||||
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device=device)
|
||||
# The builder's gather covers only the context above the replay start.
|
||||
gather_lens = torch.tensor(
|
||||
[q + min(s - q, WINDOW - 1) for s, q in zip(seq_lens, query_lens)],
|
||||
[
|
||||
q + min(max(s - q - r, 0), WINDOW - 1)
|
||||
for s, q, r in zip(seq_lens, query_lens, replay_starts)
|
||||
],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
@@ -319,7 +337,14 @@ def combine_case(
|
||||
else:
|
||||
left_t = right_t = None
|
||||
|
||||
combined_indices, combined_lens = combine_topk_swa_indices(
|
||||
# V4.1's combine has no image windows (causal SWA only).
|
||||
has_image = "left_visible" in inspect.signature(combine_fn).parameters
|
||||
image_kwargs = (
|
||||
dict(left_visible=left_t, right_visible=right_t, max_image_tokens=MAX_IMG)
|
||||
if has_image
|
||||
else {}
|
||||
)
|
||||
combined_indices, combined_lens = combine_fn(
|
||||
topk_indices,
|
||||
query_start_loc,
|
||||
seq_lens_t,
|
||||
@@ -329,9 +354,7 @@ def combine_case(
|
||||
topk,
|
||||
M,
|
||||
N,
|
||||
left_visible=left_t,
|
||||
right_visible=right_t,
|
||||
max_image_tokens=MAX_IMG,
|
||||
**image_kwargs,
|
||||
)
|
||||
|
||||
# Reference rows.
|
||||
@@ -342,7 +365,7 @@ def combine_case(
|
||||
MAX_IMG,
|
||||
)
|
||||
topk_cpu = topk_indices.cpu()
|
||||
width = WINDOW + MAX_IMG
|
||||
width = WINDOW + (MAX_IMG if has_image else 0)
|
||||
combined_topk = (topk + width + 127) // 128 * 128
|
||||
rows = []
|
||||
lens = []
|
||||
@@ -354,6 +377,8 @@ def combine_case(
|
||||
pos = prefix_len + i
|
||||
topk_len = min((pos + 1) // compress_ratio, topk)
|
||||
start, end = ref_swa_bounds(pos, WINDOW, lefts[token], rights[token])
|
||||
# The window never reaches below the gathered buffer.
|
||||
start = max(start, gather_start)
|
||||
swa_len = end - start
|
||||
row = [-1] * combined_topk
|
||||
for j in range(topk_len):
|
||||
@@ -388,6 +413,31 @@ def test_combine_topk_swa_with_image_spans(cfg):
|
||||
assert indices.cpu().tolist() == rows
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize("cfg", COMBINE_CASES)
|
||||
def test_v41_combine_topk_swa_stops_at_replay_start(cfg):
|
||||
"""SWA bounded replay: the gathered buffer starts at replay_start, so the
|
||||
window never indexes below it."""
|
||||
from vllm.models.deepseek_v41.common.ops.cache_utils import (
|
||||
combine_topk_swa_indices as combine_v41,
|
||||
)
|
||||
|
||||
# Request 0 replays [16, 40): the windows of its first rows would
|
||||
# otherwise reach below 16.
|
||||
indices, lens, rows, exp_lens = combine_case(
|
||||
cfg["compress_ratio"],
|
||||
cfg["topk"],
|
||||
seq_lens=[40, 12],
|
||||
query_lens=[24, 12],
|
||||
spans=[[], []],
|
||||
with_image=False,
|
||||
replay_starts=[16, 0],
|
||||
combine_fn=combine_v41,
|
||||
)
|
||||
assert lens.cpu().tolist() == exp_lens
|
||||
assert indices.cpu().tolist() == rows
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize("cfg", COMBINE_CASES)
|
||||
def test_combine_topk_swa_without_image_unchanged(cfg):
|
||||
@@ -472,6 +522,7 @@ def make_builder(
|
||||
DeepseekSparseSWAMetadataBuilder
|
||||
] = DeepseekSparseSWAMetadataBuilder,
|
||||
mm_prefix_clamp_sliding_window: bool = True,
|
||||
bounded_replay: bool = False,
|
||||
) -> DeepseekSparseSWAMetadataBuilder:
|
||||
overrides: dict = {"sliding_window": window}
|
||||
if vision:
|
||||
@@ -498,6 +549,7 @@ def make_builder(
|
||||
sliding_window=window,
|
||||
cache_dtype_str="auto",
|
||||
model_version="deepseek_v4",
|
||||
bounded_replay=bounded_replay,
|
||||
)
|
||||
return builder_cls(
|
||||
kv_cache_spec=spec,
|
||||
@@ -512,11 +564,17 @@ def build_metadata(
|
||||
seq_lens: list[int],
|
||||
query_lens: list[int],
|
||||
mm_req_doc_ranges: dict[int, list[tuple[int, int]]] | None,
|
||||
replay_starts: list[int] | None = None,
|
||||
):
|
||||
device = torch.device("cuda")
|
||||
query_start_loc, seq_lens_t, _, slot_mapping, block_table = make_batch(
|
||||
seq_lens, query_lens, device
|
||||
)
|
||||
replay_start = (
|
||||
None
|
||||
if replay_starts is None
|
||||
else torch.tensor(replay_starts, dtype=torch.int32, device=device)
|
||||
)
|
||||
return builder.build(
|
||||
0,
|
||||
CommonAttentionMetadata(
|
||||
@@ -533,6 +591,7 @@ def build_metadata(
|
||||
causal=True,
|
||||
mm_req_doc_ranges=mm_req_doc_ranges,
|
||||
),
|
||||
replay_start=replay_start,
|
||||
)
|
||||
|
||||
|
||||
@@ -653,3 +712,108 @@ def test_v41_image_prefill_uses_causal_swa(compress_ratio, query_len):
|
||||
assert absolute_keys.tolist() == list(range(lo, hi))
|
||||
assert paged_lens[token] == hi - lo
|
||||
assert paged_indices[token, 0, : hi - lo].tolist() == list(range(lo, hi))
|
||||
|
||||
|
||||
# SWA bounded replay: every prefill index path clamps the window at replay_start.
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_builder_replay_start_bounds_prefill_window_and_gather():
|
||||
"""Paged-direct prefill indices stop at replay_start and the FlashMLA
|
||||
gather only covers the context the request may see."""
|
||||
seq_lens = [40, 12, 30]
|
||||
query_lens = [24, 12, 10]
|
||||
replay_starts = [12, 0, 0]
|
||||
builder = make_builder(vision=False)
|
||||
md = build_metadata(builder, seq_lens, query_lens, None, replay_starts)
|
||||
assert md.replay_start is not None
|
||||
assert md.replay_start.cpu().tolist() == replay_starts
|
||||
|
||||
_, _, _, _, block_table = make_batch(seq_lens, query_lens, torch.device("cuda"))
|
||||
rows, lens = ref_swa_slot_rows(
|
||||
seq_lens,
|
||||
query_lens,
|
||||
[[], [], []],
|
||||
block_table,
|
||||
WINDOW,
|
||||
MAX_IMG,
|
||||
WINDOW,
|
||||
replay_starts=replay_starts,
|
||||
)
|
||||
assert md.prefill_swa_lens.cpu().tolist() == lens
|
||||
assert md.prefill_swa_indices[:, 0].cpu().tolist() == rows
|
||||
# Request 0: the first replayed row (pos 16) sees [12, 16]; its last row
|
||||
# (pos 39) is a full window above the replay start and sees all of it.
|
||||
assert md.prefill_swa_lens[0].item() == 16 - 12 + 1
|
||||
assert md.prefill_swa_lens[23].item() == WINDOW
|
||||
# gather_len = query_len + min(prefix_len - replay_start, WINDOW - 1).
|
||||
assert md.prefill_gather_lens.cpu().tolist() == [
|
||||
24 + min(16 - 12, WINDOW - 1),
|
||||
12,
|
||||
10 + min(20, WINDOW - 1),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_builder_replaying_group_builds_without_replay_start():
|
||||
"""Graph captures build without model-specific kwargs: a replaying group's
|
||||
metadata then bounds nothing."""
|
||||
builder = make_builder(vision=False, bounded_replay=True)
|
||||
md = build_metadata(builder, [40, 12], [24, 12], None)
|
||||
assert not md.replay_start.any()
|
||||
assert md.prefill_swa_lens[0].item() == WINDOW
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_v41_flashinfer_mixed_sparse_indices_respect_replay_start():
|
||||
from vllm.models.deepseek_v41.common.ops.cache_utils import (
|
||||
build_flashinfer_mixed_sparse_indices as build_v41,
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
# 1 decode token (req 0) + two prefill requests; req 1 resumes at 16.
|
||||
seq_lens = [20, 40, 9]
|
||||
query_lens = [1, 24, 9]
|
||||
replay_starts = [0, 16, 0]
|
||||
query_start_loc = torch.tensor([0, 1, 25, 34], dtype=torch.int32, device=device)
|
||||
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device=device)
|
||||
token_to_req = torch.tensor(
|
||||
[0] + [1] * 24 + [2] * 9, dtype=torch.int32, device=device
|
||||
)
|
||||
block_table = torch.arange(3, dtype=torch.int32, device=device).view(3, 1)
|
||||
kwargs = dict(
|
||||
decode_swa_indices=torch.zeros((1, WINDOW), dtype=torch.int32, device=device),
|
||||
decode_compressed_indices=None,
|
||||
decode_compressed_topk_lens=None,
|
||||
prefill_topk_indices=torch.empty((33, 0), dtype=torch.int32, device=device),
|
||||
query_start_loc=query_start_loc,
|
||||
seq_lens=seq_lens_t,
|
||||
token_to_req_indices=token_to_req,
|
||||
swa_block_table=block_table,
|
||||
swa_block_size=BLOCK_SIZE,
|
||||
compressed_block_table=None,
|
||||
compressed_block_size=BLOCK_SIZE,
|
||||
window_size=WINDOW,
|
||||
compress_ratio=1,
|
||||
topk=0,
|
||||
)
|
||||
plain, _ = build_v41(
|
||||
replay_start=torch.zeros(3, dtype=torch.int32, device=device), **kwargs
|
||||
)
|
||||
bounded, _ = build_v41(
|
||||
replay_start=torch.tensor(replay_starts, dtype=torch.int32, device=device),
|
||||
**kwargs,
|
||||
)
|
||||
rows, _ = ref_swa_slot_rows(
|
||||
seq_lens,
|
||||
query_lens,
|
||||
[[], [], []],
|
||||
block_table,
|
||||
WINDOW,
|
||||
MAX_IMG,
|
||||
WINDOW,
|
||||
replay_starts=replay_starts,
|
||||
)
|
||||
# Decode row is copied through; prefill rows follow the bounded window.
|
||||
assert bounded[0].cpu().tolist() == plain[0].cpu().tolist()
|
||||
assert bounded[1:].cpu().tolist() == rows[1:]
|
||||
|
||||
@@ -669,6 +669,7 @@ def test_dsv41_flashinfer_dspark_window_matches_reference(
|
||||
),
|
||||
block_table=torch.arange(48, device=device, dtype=torch.int32).view(3, -1),
|
||||
block_size=128,
|
||||
replay_start=torch.zeros(3, device=device, dtype=torch.int32),
|
||||
flashinfer_sparse_index_cache={},
|
||||
max_decode_query_len=5,
|
||||
)
|
||||
|
||||
@@ -14,6 +14,7 @@ from vllm.models.deepseek_v41.sparse_mla import (
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.mla.compressor_utils import (
|
||||
CompressedSlotMappingKernel,
|
||||
get_compressed_slot_mapping,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
BuildPrefillChunkMetadataKernel,
|
||||
@@ -235,6 +236,28 @@ def test_compressed_slot_mapping_warmup_includes_index_kpool():
|
||||
assert {(key.compress_ratio, key.block_size) for key in keys} == {(32, 2)}
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_compressed_slot_mapping_inherits_padded_token_slots():
|
||||
"""A token whose own slot is padded (SWA bounded replay) closes no
|
||||
compressed state either."""
|
||||
device = torch.device("cuda")
|
||||
query_start_loc = torch.tensor([0, 8], dtype=torch.int32, device=device)
|
||||
seq_lens = torch.tensor([8], dtype=torch.int32, device=device)
|
||||
block_table = torch.tensor([[3]], dtype=torch.int32, device=device)
|
||||
slot_mapping = torch.arange(8, dtype=torch.int64, device=device)
|
||||
slot_mapping[:4] = -1
|
||||
compressed = get_compressed_slot_mapping(
|
||||
8,
|
||||
slot_mapping,
|
||||
query_start_loc,
|
||||
seq_lens,
|
||||
block_table,
|
||||
block_size=4,
|
||||
compress_ratio=2,
|
||||
)
|
||||
assert compressed.tolist() == [-1, -1, -1, -1, -1, 3 * 4 + 2, -1, 3 * 4 + 3]
|
||||
|
||||
|
||||
def test_index_conversion_warmup_uses_physical_block_stride():
|
||||
config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=64),
|
||||
|
||||
@@ -651,3 +651,67 @@ class TestCompressorRingGroup:
|
||||
scheduler_block_size=128,
|
||||
)
|
||||
assert len(manager.coordinator.single_type_managers) == len(groups)
|
||||
|
||||
|
||||
class TestSWABoundedReplayGrouping:
|
||||
def test_non_cacheable_swa_leaves_hits_to_the_paged_group(self):
|
||||
"""Under SWA bounded replay only the paged MLA group hashes (hash
|
||||
block = its block) and the coordinator probes one spec group."""
|
||||
config = _shared_layout_config()
|
||||
config.cache_config.enable_prefix_caching = True
|
||||
specs: dict[str, KVCacheSpec] = {
|
||||
"layers.2.attn": MLAAttentionSpec(
|
||||
block_size=128,
|
||||
num_kv_heads=1,
|
||||
head_size=584,
|
||||
dtype=torch.uint8,
|
||||
tokens_per_state=2,
|
||||
alignment=576,
|
||||
),
|
||||
}
|
||||
swa = SlidingWindowMLASpec(
|
||||
block_size=32,
|
||||
num_kv_heads=1,
|
||||
head_size=584,
|
||||
dtype=torch.uint8,
|
||||
sliding_window=128,
|
||||
alignment=576,
|
||||
bounded_replay=True,
|
||||
)
|
||||
for layer in range(4):
|
||||
specs[f"layers.{layer}.attn.swa_cache"] = swa
|
||||
|
||||
groups = get_kv_cache_groups(config, specs)
|
||||
# The worker reads the replay window off the (packed) group specs to
|
||||
# arm the window clamp; the wrapper must forward it like cacheability.
|
||||
assert sorted(
|
||||
(g.kv_cache_spec.prefix_cacheable, g.kv_cache_spec.prefix_replay_tokens)
|
||||
for g in groups
|
||||
) == [(False, 128)] * 4 + [(True, 0)]
|
||||
kv_cache_config = get_kv_cache_config_from_groups(
|
||||
config, groups, available_memory=64 * _get_kv_cache_bytes_per_block(groups)
|
||||
)
|
||||
manager = KVCacheManager(
|
||||
generate_scheduler_kv_cache_config([kv_cache_config]),
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=128,
|
||||
scheduler_block_size=128,
|
||||
)
|
||||
assert len(manager.coordinator.attention_groups) == 1
|
||||
assert isinstance(
|
||||
manager.coordinator.attention_groups[0].spec, MLAAttentionSpec
|
||||
)
|
||||
|
||||
# Allocating and caching a request goes through every group's manager;
|
||||
# the 32-token SWA group must skip hashing rather than assert on the
|
||||
# 128-token hash block.
|
||||
from tests.v1.core.utils import create_requests
|
||||
|
||||
request = create_requests(num_requests=1, num_tokens=256, block_size=128)[0]
|
||||
assert manager.allocate_slots(request, 256) is not None
|
||||
request.num_computed_tokens = 256
|
||||
manager.cache_blocks(request, 256)
|
||||
for single in manager.coordinator.single_type_managers:
|
||||
cached = single.num_cached_block.get(request.request_id, 0)
|
||||
assert cached == (2 if single.kv_cache_spec.prefix_cacheable else 0)
|
||||
|
||||
@@ -6,7 +6,7 @@ import copy
|
||||
from collections.abc import Callable
|
||||
from dataclasses import replace
|
||||
from math import lcm
|
||||
from types import SimpleNamespace
|
||||
from types import MethodType, SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
@@ -533,7 +533,9 @@ def test_nixl_hisparse_full_block_import_keeps_a_writable_tail(num_tokens):
|
||||
kv_cache_manager=manager,
|
||||
failed_recving_kv_req_ids=set(),
|
||||
finished_recving_kv_req_ids={request.request_id},
|
||||
prefix_replay_tokens=0,
|
||||
)
|
||||
scheduler._mark_prefix_replay = MethodType(Scheduler._mark_prefix_replay, scheduler)
|
||||
Scheduler._update_waiting_for_remote_kv(scheduler, request)
|
||||
assert request.num_tokens - request.num_computed_tokens == 1
|
||||
assert manager.allocate_slots(request, num_new_tokens=1) is not None
|
||||
|
||||
@@ -0,0 +1,279 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""SWA bounded replay: after a prefix hit the scheduler recomputes the tail of
|
||||
the hit to rebuild the non-cacheable sliding-window group, keeps the hit's
|
||||
blocks, and hands the worker the replay start."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.v1.kv_connector.unit.utils import MockKVConfig, create_model_runner_output
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
SlidingWindowMLASpec,
|
||||
)
|
||||
from vllm.v1.outputs import ModelRunnerOutput
|
||||
from vllm.v1.request import RequestStatus
|
||||
from vllm.v1.structured_output import StructuredOutputManager
|
||||
|
||||
from .utils import create_requests, create_scheduler
|
||||
|
||||
BLOCK_SIZE = 16
|
||||
WINDOW = 32
|
||||
NUM_PROMPT_TOKENS = 100
|
||||
# 100 tokens -> 6 full blocks cached -> 96-token hit.
|
||||
HIT_TOKENS = NUM_PROMPT_TOKENS // BLOCK_SIZE * BLOCK_SIZE
|
||||
FULL, SWA = 0, 1
|
||||
|
||||
|
||||
def _replay_scheduler(
|
||||
*,
|
||||
long_prefill_token_threshold: int = 0,
|
||||
use_kv_connector: MockKVConfig | None = None,
|
||||
) -> Scheduler:
|
||||
"""A hybrid layout: one prefix-cacheable full-attention group and one
|
||||
replayed sliding-window group."""
|
||||
base = create_scheduler(
|
||||
block_size=BLOCK_SIZE,
|
||||
enable_prefix_caching=True,
|
||||
long_prefill_token_threshold=long_prefill_token_threshold,
|
||||
use_kv_connector=use_kv_connector,
|
||||
)
|
||||
vllm_config = base.vllm_config
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=10000,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["swa"],
|
||||
SlidingWindowMLASpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=1,
|
||||
head_size=64,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=WINDOW,
|
||||
bounded_replay=True,
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
scheduler = Scheduler(
|
||||
vllm_config=vllm_config,
|
||||
kv_cache_config=kv_cache_config,
|
||||
block_size=BLOCK_SIZE,
|
||||
log_stats=True,
|
||||
structured_output_manager=StructuredOutputManager(vllm_config),
|
||||
)
|
||||
scheduler.use_v2_model_runner = True
|
||||
assert scheduler.prefix_replay_tokens == WINDOW
|
||||
return scheduler
|
||||
|
||||
|
||||
def _prefill(scheduler: Scheduler, request) -> None:
|
||||
scheduler.add_request(request)
|
||||
out = scheduler.schedule()
|
||||
assert out.num_scheduled_tokens[request.request_id] == request.num_prompt_tokens
|
||||
scheduler.update_from_output(out, _step_output(out, [request]))
|
||||
|
||||
|
||||
def _step_output(out, requests) -> ModelRunnerOutput:
|
||||
"""Model output for every scheduled request: a token once its prefill is
|
||||
done (non-zero, so outputs never extend the all-zero shared prompt in the
|
||||
cache), nothing while it is still prefilling. schedule() has already
|
||||
advanced num_computed_tokens past this step's chunk."""
|
||||
scheduled = [r for r in requests if r.request_id in out.num_scheduled_tokens]
|
||||
return ModelRunnerOutput(
|
||||
req_ids=[r.request_id for r in scheduled],
|
||||
req_id_to_index={r.request_id: i for i, r in enumerate(scheduled)},
|
||||
sampled_token_ids=[
|
||||
[1000] if r.num_computed_tokens >= r.num_prompt_tokens else []
|
||||
for r in scheduled
|
||||
],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=None,
|
||||
)
|
||||
|
||||
|
||||
def _new_req_data(out, request):
|
||||
return next(r for r in out.scheduled_new_reqs if r.req_id == request.request_id)
|
||||
|
||||
|
||||
def test_hit_replays_window_without_reallocating():
|
||||
scheduler = _replay_scheduler()
|
||||
expected_replay = WINDOW
|
||||
first, second = create_requests(
|
||||
num_requests=2,
|
||||
num_tokens=NUM_PROMPT_TOKENS,
|
||||
same_prompt=True,
|
||||
block_size=BLOCK_SIZE,
|
||||
)
|
||||
_prefill(scheduler, first)
|
||||
manager = scheduler.kv_cache_manager
|
||||
|
||||
scheduler.add_request(second)
|
||||
out = scheduler.schedule()
|
||||
|
||||
replay_start = HIT_TOKENS - expected_replay
|
||||
new_req = _new_req_data(out, second)
|
||||
assert new_req.num_computed_tokens == replay_start
|
||||
assert new_req.replay_start == replay_start
|
||||
assert (
|
||||
out.num_scheduled_tokens[second.request_id] == NUM_PROMPT_TOKENS - replay_start
|
||||
)
|
||||
|
||||
first_blocks = manager.get_blocks(first.request_id).blocks
|
||||
second_blocks = manager.get_blocks(second.request_id).blocks
|
||||
# The full-attention hit is shared (replayed tokens included); only the
|
||||
# tail block is new.
|
||||
num_hit_blocks = HIT_TOKENS // BLOCK_SIZE
|
||||
assert second_blocks[FULL][:num_hit_blocks] == first_blocks[FULL][:num_hit_blocks]
|
||||
assert all(b.ref_cnt == 2 for b in second_blocks[FULL][:num_hit_blocks])
|
||||
assert len(second_blocks[FULL]) == num_hit_blocks + 1
|
||||
# The window group never hits: blocks below the window are null, the
|
||||
# window (replayed and new tokens) gets fresh blocks.
|
||||
num_skipped_blocks = (HIT_TOKENS - WINDOW + 1) // BLOCK_SIZE
|
||||
assert all(b.is_null for b in second_blocks[SWA][:num_skipped_blocks])
|
||||
num_window_blocks = len(second_blocks[SWA]) - num_skipped_blocks
|
||||
assert num_window_blocks == -(-NUM_PROMPT_TOKENS // BLOCK_SIZE) - num_skipped_blocks
|
||||
assert not any(b.is_null for b in second_blocks[SWA][num_skipped_blocks:])
|
||||
|
||||
scheduler.update_from_output(out, _step_output(out, [first, second]))
|
||||
assert second.num_computed_tokens == NUM_PROMPT_TOKENS
|
||||
|
||||
|
||||
def test_replay_only_chunks_make_progress():
|
||||
"""A chunk cap smaller than the replay window schedules replay-only
|
||||
chunks; they advance the request instead of stalling it."""
|
||||
scheduler = _replay_scheduler(long_prefill_token_threshold=BLOCK_SIZE)
|
||||
first, second = create_requests(
|
||||
num_requests=2,
|
||||
num_tokens=NUM_PROMPT_TOKENS,
|
||||
same_prompt=True,
|
||||
block_size=BLOCK_SIZE,
|
||||
)
|
||||
scheduler.add_request(first)
|
||||
while first.num_computed_tokens < NUM_PROMPT_TOKENS:
|
||||
out = scheduler.schedule()
|
||||
scheduler.update_from_output(out, _step_output(out, [first]))
|
||||
|
||||
scheduler.add_request(second)
|
||||
|
||||
def full_blocks():
|
||||
return scheduler.kv_cache_manager.get_blocks(second.request_id).blocks[FULL]
|
||||
|
||||
scheduled: list[int] = []
|
||||
while second.num_computed_tokens < NUM_PROMPT_TOKENS:
|
||||
out = scheduler.schedule()
|
||||
if not scheduled:
|
||||
new_req = _new_req_data(out, second)
|
||||
assert new_req.num_computed_tokens == HIT_TOKENS - WINDOW
|
||||
assert new_req.replay_start == HIT_TOKENS - WINDOW
|
||||
# A replay-only chunk adopts the hit and allocates nothing new.
|
||||
assert len(full_blocks()) == HIT_TOKENS // BLOCK_SIZE
|
||||
scheduled.append(out.num_scheduled_tokens[second.request_id])
|
||||
scheduler.update_from_output(out, _step_output(out, [first, second]))
|
||||
# 32 replayed + 4 new tokens in 16-token chunks: two replay-only chunks.
|
||||
assert scheduled == [BLOCK_SIZE, BLOCK_SIZE, NUM_PROMPT_TOKENS - HIT_TOKENS]
|
||||
assert len(full_blocks()) == -(-NUM_PROMPT_TOKENS // BLOCK_SIZE)
|
||||
assert second.status == RequestStatus.RUNNING
|
||||
|
||||
|
||||
def test_async_remote_kv_hit_replays_after_load():
|
||||
"""A KV-connector hit loaded asynchronously carries no window state
|
||||
either; the replay is applied when the load completes."""
|
||||
matched = 64
|
||||
scheduler = _replay_scheduler(
|
||||
use_kv_connector=MockKVConfig(matched_tokens=matched, is_async=True)
|
||||
)
|
||||
request = create_requests(
|
||||
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
|
||||
)[0]
|
||||
scheduler.add_request(request)
|
||||
out = scheduler.schedule()
|
||||
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
assert request.request_id not in out.num_scheduled_tokens
|
||||
scheduler.update_from_output(
|
||||
out, create_model_runner_output([], finished_recving={request.request_id})
|
||||
)
|
||||
|
||||
out = scheduler.schedule()
|
||||
new_req = _new_req_data(out, request)
|
||||
assert new_req.num_computed_tokens == matched - WINDOW
|
||||
assert new_req.replay_start == matched - WINDOW
|
||||
assert (
|
||||
out.num_scheduled_tokens[request.request_id]
|
||||
== NUM_PROMPT_TOKENS - matched + WINDOW
|
||||
)
|
||||
|
||||
|
||||
def test_remote_kv_hit_is_taken_in_whole_blocks():
|
||||
"""A hit ending one token short of a block boundary would put the replay
|
||||
window's first token in a block the sliding-window group retires, so a
|
||||
connector hit is cut back to whole blocks."""
|
||||
matched = 4 * BLOCK_SIZE - 1
|
||||
scheduler = _replay_scheduler(
|
||||
use_kv_connector=MockKVConfig(matched_tokens=matched, is_async=True)
|
||||
)
|
||||
request = create_requests(
|
||||
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
|
||||
)[0]
|
||||
scheduler.add_request(request)
|
||||
out = scheduler.schedule()
|
||||
scheduler.update_from_output(
|
||||
out, create_model_runner_output([], finished_recving={request.request_id})
|
||||
)
|
||||
out = scheduler.schedule()
|
||||
new_req = _new_req_data(out, request)
|
||||
hit = 3 * BLOCK_SIZE
|
||||
assert new_req.replay_start == hit - WINDOW
|
||||
assert new_req.num_computed_tokens == hit - WINDOW
|
||||
swa_manager = scheduler.kv_cache_manager.coordinator.single_type_managers[SWA]
|
||||
swa_blocks = scheduler.kv_cache_manager.get_blocks(request.request_id).blocks[SWA]
|
||||
assert swa_blocks[new_req.replay_start // BLOCK_SIZE] is not swa_manager._null_block
|
||||
|
||||
|
||||
@pytest.mark.parametrize("via_connector", [False, True])
|
||||
def test_hit_no_longer_than_window_is_ignored(via_connector):
|
||||
"""Such a hit would be recomputed in full anyway. It is not adopted (an
|
||||
async load of it is not even started), so a request has computed tokens
|
||||
iff it replays."""
|
||||
if via_connector:
|
||||
scheduler = _replay_scheduler(
|
||||
use_kv_connector=MockKVConfig(matched_tokens=WINDOW, is_async=True)
|
||||
)
|
||||
request = create_requests(
|
||||
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
|
||||
)[0]
|
||||
else:
|
||||
scheduler = _replay_scheduler()
|
||||
first = create_requests(
|
||||
num_requests=1, num_tokens=WINDOW, same_prompt=True, block_size=BLOCK_SIZE
|
||||
)[0]
|
||||
_prefill(scheduler, first)
|
||||
request = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=NUM_PROMPT_TOKENS,
|
||||
same_prompt=True,
|
||||
block_size=BLOCK_SIZE,
|
||||
req_ids=["long"],
|
||||
)[0]
|
||||
scheduler.add_request(request)
|
||||
out = scheduler.schedule()
|
||||
new_req = _new_req_data(out, request)
|
||||
assert request.status == RequestStatus.RUNNING
|
||||
assert new_req.num_computed_tokens == 0
|
||||
assert new_req.replay_start == 0
|
||||
assert out.num_scheduled_tokens[request.request_id] == NUM_PROMPT_TOKENS
|
||||
@@ -1505,6 +1505,7 @@ def test_failed_load_rezeroes_unwritten_skipped_blocks():
|
||||
scheduler.kv_cache_manager.cache_blocks = MagicMock()
|
||||
scheduler.failed_recving_kv_req_ids = {"req-1"}
|
||||
scheduler.finished_recving_kv_req_ids = {"req-1"}
|
||||
scheduler.prefix_replay_tokens = 0
|
||||
|
||||
request = MagicMock()
|
||||
request.request_id = "req-1"
|
||||
|
||||
@@ -227,6 +227,11 @@ class CacheConfig:
|
||||
necessary for implementing this optimization in some models (e.g. Gemma3n)
|
||||
"""
|
||||
|
||||
swa_bounded_replay: bool = True
|
||||
"""Keep the sliding-window KV of models that support it (DeepSeek-V4.1)
|
||||
out of prefix caching and rebuild it after a prefix hit by recomputing the
|
||||
hit's last window. Requires model runner V2."""
|
||||
|
||||
kv_cache_memory_bytes: int | None = None
|
||||
"""Size of KV Cache per GPU in bytes. By default, this is set to None
|
||||
and vllm can automatically infer the kv cache size based on
|
||||
@@ -281,8 +286,9 @@ class CacheConfig:
|
||||
"effective_attention_block_size",
|
||||
"kv_cache_size_tokens",
|
||||
"kv_cache_max_concurrency",
|
||||
# WIP feature toggle not impacting compiled graph shape
|
||||
# Feature toggles not impacting compiled graph shape
|
||||
"kv_sharing_fast_prefill",
|
||||
"swa_bounded_replay",
|
||||
}
|
||||
|
||||
from vllm.config.utils import get_hash_factors, hash_factors
|
||||
|
||||
@@ -781,6 +781,7 @@ class EngineArgs:
|
||||
stream_interval: int = SchedulerConfig.stream_interval
|
||||
|
||||
kv_sharing_fast_prefill: bool = CacheConfig.kv_sharing_fast_prefill
|
||||
swa_bounded_replay: bool = CacheConfig.swa_bounded_replay
|
||||
optimization_level: OptimizationLevel = VllmConfig.optimization_level
|
||||
performance_mode: PerformanceMode = VllmConfig.performance_mode
|
||||
|
||||
@@ -1310,6 +1311,9 @@ class EngineArgs:
|
||||
cache_group.add_argument(
|
||||
"--kv-sharing-fast-prefill", **cache_kwargs["kv_sharing_fast_prefill"]
|
||||
)
|
||||
cache_group.add_argument(
|
||||
"--swa-bounded-replay", **cache_kwargs["swa_bounded_replay"]
|
||||
)
|
||||
cache_group.add_argument(
|
||||
"--mamba-cache-dtype", **cache_kwargs["mamba_cache_dtype"]
|
||||
)
|
||||
@@ -2118,6 +2122,7 @@ class EngineArgs:
|
||||
prefix_cache_retention_interval=self.prefix_cache_retention_interval,
|
||||
kv_cache_dtype_skip_layers=self.kv_cache_dtype_skip_layers,
|
||||
kv_sharing_fast_prefill=self.kv_sharing_fast_prefill,
|
||||
swa_bounded_replay=self.swa_bounded_replay,
|
||||
mamba_cache_dtype=self.mamba_cache_dtype,
|
||||
mamba_ssm_cache_dtype=self.mamba_ssm_cache_dtype,
|
||||
mamba_block_size=self.mamba_block_size,
|
||||
|
||||
@@ -495,11 +495,13 @@ class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuild
|
||||
common_prefix_len: int,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
fast_build: bool = False,
|
||||
replay_start: torch.Tensor | None = None,
|
||||
) -> DeepseekV4ROCMAiterSparseSWAMetadata:
|
||||
base = super().build(
|
||||
common_prefix_len=common_prefix_len,
|
||||
common_attn_metadata=common_attn_metadata,
|
||||
fast_build=fast_build,
|
||||
replay_start=replay_start,
|
||||
)
|
||||
|
||||
ragged_indices = None
|
||||
|
||||
@@ -195,6 +195,7 @@ class DeepseekV4SparseMLAMetadataBuilder(
|
||||
if self.compress_ratio > 1:
|
||||
slot_mapping = get_compressed_slot_mapping(
|
||||
cm.num_actual_tokens,
|
||||
cm.slot_mapping,
|
||||
cm.query_start_loc,
|
||||
cm.seq_lens,
|
||||
cm.block_table_tensor.clamp_(min=0),
|
||||
|
||||
@@ -425,11 +425,13 @@ class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekV41SparseSWAMetadataBu
|
||||
common_prefix_len: int,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
fast_build: bool = False,
|
||||
replay_start: torch.Tensor | None = None,
|
||||
) -> DeepseekV4ROCMAiterSparseSWAMetadata:
|
||||
base = super().build(
|
||||
common_prefix_len=common_prefix_len,
|
||||
common_attn_metadata=common_attn_metadata,
|
||||
fast_build=fast_build,
|
||||
replay_start=replay_start,
|
||||
)
|
||||
|
||||
ragged_indices = None
|
||||
|
||||
@@ -512,6 +512,25 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
"decodes only on SM100."
|
||||
)
|
||||
|
||||
swa_bounded_replay = cache_config.swa_bounded_replay
|
||||
if swa_bounded_replay and not vllm_config.use_v2_model_runner:
|
||||
logger.warning_once(
|
||||
"SWA bounded replay needs model runner V2 (only it skips the "
|
||||
"paged-KV writes of replayed tokens); the sliding-window cache "
|
||||
"takes part in prefix caching instead."
|
||||
)
|
||||
swa_bounded_replay = False
|
||||
if (
|
||||
swa_bounded_replay
|
||||
and vllm_config.parallel_config.prefill_context_parallel_size > 1
|
||||
):
|
||||
logger.warning_once(
|
||||
"SWA bounded replay is off under prefill context parallelism "
|
||||
"(the replayed tokens' slot padding knows the rank-local batch "
|
||||
"only); the sliding-window cache takes part in prefix caching "
|
||||
"instead."
|
||||
)
|
||||
swa_bounded_replay = False
|
||||
self.swa_cache_layer = DeepseekV4SWACache(
|
||||
head_dim=self.head_dim,
|
||||
window_size=self.window_size,
|
||||
@@ -522,6 +541,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
block_size=32,
|
||||
packed_bytes_per_token=self.swa_bytes_per_token,
|
||||
packed_page_alignment=self.kv_page_alignment,
|
||||
bounded_replay=swa_bounded_replay,
|
||||
)
|
||||
|
||||
# The attention layer itself was already registered with the
|
||||
|
||||
@@ -982,6 +982,10 @@ class CombineTopkSwaIndicesKernel(
|
||||
# turns the store index below negative).
|
||||
topk_len = 0
|
||||
swa_start = tl.maximum(pos - (WINDOW_SIZE - 1), 0)
|
||||
# gather_len already excludes context below the request's replay
|
||||
# start (SWA bounded replay), so the window cannot start before
|
||||
# the gathered buffer does.
|
||||
swa_start = tl.maximum(swa_start, gather_start)
|
||||
swa_len = pos - swa_start + 1
|
||||
|
||||
offset = tl.arange(0, PADDED_TOP_K)
|
||||
@@ -1128,6 +1132,8 @@ def build_flashinfer_mixed_sparse_indices(
|
||||
decode_is_valid_token: torch.Tensor | None = None,
|
||||
swa_block_span: int | None = None,
|
||||
compressed_block_span: int | None = None,
|
||||
*,
|
||||
replay_start: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Build the FlashInfer DSV4 sparse-index matrix for decode-first batches.
|
||||
|
||||
@@ -1137,6 +1143,9 @@ def build_flashinfer_mixed_sparse_indices(
|
||||
per token). Decode tokens read precomputed SWA/compressed indices; prefill
|
||||
tokens derive their SWA window from the position and translate local
|
||||
compressed indices to global slots via the block tables.
|
||||
|
||||
``replay_start`` ([num_reqs], SWA bounded replay) lower-bounds every
|
||||
prefill token's window: positions below it hold no window KV.
|
||||
"""
|
||||
assert decode_swa_indices.dtype == torch.int32
|
||||
assert decode_swa_indices.dim() == 2
|
||||
@@ -1227,6 +1236,7 @@ def build_flashinfer_mixed_sparse_indices(
|
||||
query_start_loc,
|
||||
seq_lens,
|
||||
token_to_req_indices,
|
||||
replay_start,
|
||||
swa_block_table,
|
||||
swa_block_table.stride(0),
|
||||
swa_block_size,
|
||||
@@ -1295,6 +1305,7 @@ def _build_flashinfer_mixed_sparse_indices_kernel(
|
||||
query_start_loc_ptr,
|
||||
seq_lens_ptr,
|
||||
token_to_req_indices_ptr,
|
||||
replay_start_ptr,
|
||||
swa_block_table_ptr,
|
||||
swa_block_table_stride,
|
||||
swa_block_size,
|
||||
@@ -1394,6 +1405,8 @@ def _build_flashinfer_mixed_sparse_indices_kernel(
|
||||
token_idx_in_query = token_idx - query_start
|
||||
pos = start_pos + token_idx_in_query
|
||||
swa_start_pos = tl.maximum(pos - (WINDOW_SIZE - 1), 0)
|
||||
# SWA bounded replay: no window KV exists below the request's replay start.
|
||||
swa_start_pos = tl.maximum(swa_start_pos, tl.load(replay_start_ptr + req_idx))
|
||||
swa_len = pos - swa_start_pos + 1
|
||||
if COMPRESS_RATIO > 0:
|
||||
topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, TOP_K)
|
||||
|
||||
@@ -200,8 +200,11 @@ class DeepseekSparseSWAFlashInferMetadataBuilder(DeepseekV41SparseSWAMetadataBui
|
||||
common_prefix_len: int,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
fast_build: bool = False,
|
||||
replay_start: torch.Tensor | None = None,
|
||||
) -> "DeepseekSparseSWAMetadata":
|
||||
metadata = super().build(common_prefix_len, common_attn_metadata, fast_build)
|
||||
metadata = super().build(
|
||||
common_prefix_len, common_attn_metadata, fast_build, replay_start
|
||||
)
|
||||
num_tokens = metadata.num_decode_tokens
|
||||
if not common_attn_metadata.causal and num_tokens > 0:
|
||||
assert metadata.decode_swa_lens is not None
|
||||
@@ -375,6 +378,7 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
|
||||
assert swa_metadata.token_to_req_indices is not None
|
||||
assert swa_metadata.decode_swa_indices is not None
|
||||
assert swa_metadata.block_table is not None
|
||||
assert swa_metadata.replay_start is not None
|
||||
|
||||
decode_swa_indices = swa_metadata.decode_swa_indices.reshape(
|
||||
num_decode_tokens, swa_metadata.decode_swa_width
|
||||
@@ -454,6 +458,7 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
|
||||
decode_is_valid_token=decode_is_valid_token,
|
||||
swa_block_span=swa_block_span,
|
||||
compressed_block_span=compressed_block_span,
|
||||
replay_start=swa_metadata.replay_start[:num_reqs],
|
||||
)
|
||||
if swa_only:
|
||||
swa_metadata.flashinfer_sparse_index_cache["swa_only"] = (
|
||||
|
||||
@@ -2,15 +2,23 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import CUDAGraphMode, VllmConfig
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadataBuilder
|
||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||
from vllm.v1.core.sched.output import NewRequestData
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.gpu.buffer_utils import UvaBufferPool
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelSpecificAttnMetadata
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -41,14 +49,69 @@ def _gather_lookback_kernel(
|
||||
tl.store(lookback_ptr + batch_idx * DEPTH + offs, ids, mask=offs < DEPTH)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _pad_replayed_slots_kernel(
|
||||
slot_mappings_ptr, # [num_groups, num_tokens_padded]
|
||||
group_stride,
|
||||
cacheable_groups_ptr, # [NUM_GROUPS] indices of the prefix-cacheable groups
|
||||
query_start_loc_ptr, # [num_reqs + 1]
|
||||
positions_ptr, # [num_tokens]
|
||||
replay_start_ptr, # [num_reqs]
|
||||
window,
|
||||
pad_slot_id,
|
||||
NUM_GROUPS: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
req = tl.program_id(0)
|
||||
start = tl.load(replay_start_ptr + req)
|
||||
if start == 0:
|
||||
return
|
||||
begin = tl.load(query_start_loc_ptr + req)
|
||||
end = tl.load(query_start_loc_ptr + req + 1)
|
||||
for tok in range(begin, end, BLOCK):
|
||||
offs = tok + tl.arange(0, BLOCK)
|
||||
pos = tl.load(positions_ptr + offs, mask=offs < end, other=0)
|
||||
replayed = (offs < end) & (pos >= start) & (pos < start + window)
|
||||
for i in tl.static_range(NUM_GROUPS):
|
||||
group = tl.load(cacheable_groups_ptr + i)
|
||||
tl.store(
|
||||
slot_mappings_ptr + group * group_stride + offs,
|
||||
pad_slot_id,
|
||||
mask=replayed,
|
||||
)
|
||||
|
||||
|
||||
class ReplayAttnMetadata(ModelSpecificAttnMetadata):
|
||||
"""Hands the batch's replay starts to the sliding-window builders."""
|
||||
|
||||
def __init__(self, replay_start: torch.Tensor) -> None:
|
||||
self.replay_start = replay_start
|
||||
|
||||
def get_extra_attn_kwargs(
|
||||
self, attn_metadata_builder: Any, num_reqs: int
|
||||
) -> dict[str, Any]:
|
||||
if isinstance(attn_metadata_builder, DeepseekSparseSWAMetadataBuilder):
|
||||
return {"replay_start": self.replay_start[:num_reqs]}
|
||||
return {}
|
||||
|
||||
|
||||
class DeepseekV41ModelState(DefaultModelState):
|
||||
"""DefaultModelState plus the engram lookback window.
|
||||
"""DefaultModelState plus the engram lookback window and SWA bounded replay.
|
||||
|
||||
The engram n-gram hash needs the ids of the ``depth`` tokens preceding
|
||||
each request's chunk start (see ``common/engram.py``). The runner keeps
|
||||
the full token history on device, so the window is gathered there every
|
||||
step: exact for prompt and generated tokens alike, whatever instance
|
||||
produced their KV.
|
||||
|
||||
SWA bounded replay (``CacheConfig.swa_bounded_replay``) keeps the
|
||||
sliding-window KV out of prefix caching and rebuilds it after a prefix hit
|
||||
by recomputing the hit's last window; the scheduler tells each request the
|
||||
position from which it holds window KV (``NewRequestData.replay_start``).
|
||||
``prepare_attn`` gathers those per batch, pads the replayed tokens' slots
|
||||
in the prefix-cacheable groups so the cached KV stays as is, and hands the
|
||||
starts to the sliding-window metadata builders, whose kernels read no
|
||||
window KV below them.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -67,6 +130,20 @@ class DeepseekV41ModelState(DefaultModelState):
|
||||
(self.max_num_reqs, depth), -1, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
# Per request state index; batches gather from it (see prepare_attn).
|
||||
self._replay_start_np = np.zeros(self.max_num_reqs, dtype=np.int32)
|
||||
self._replay_start = torch.zeros(
|
||||
self.max_num_reqs, dtype=torch.int32, device=device
|
||||
)
|
||||
self._replay_start_staging = UvaBufferPool(self.max_num_reqs, torch.int32)
|
||||
# The replay window and the indices of the prefix-cacheable groups, from
|
||||
# the KV cache config on first use; None until then.
|
||||
self._replay: tuple[int, torch.Tensor] | None = None
|
||||
|
||||
def add_request(self, req_index: int, new_req_data: NewRequestData) -> None:
|
||||
super().add_request(req_index, new_req_data)
|
||||
self._replay_start_np[req_index] = new_req_data.replay_start
|
||||
|
||||
def prepare_inputs(
|
||||
self, input_batch: InputBatch, req_states: RequestState
|
||||
) -> dict[str, torch.Tensor | None]:
|
||||
@@ -96,3 +173,67 @@ class DeepseekV41ModelState(DefaultModelState):
|
||||
self.lookback_token_ids.fill_(-1)
|
||||
model_inputs["lookback_token_ids"] = self.lookback_token_ids
|
||||
return model_inputs
|
||||
|
||||
def prepare_attn(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
cudagraph_mode: CUDAGraphMode,
|
||||
block_tables: tuple[torch.Tensor, ...],
|
||||
slot_mappings: torch.Tensor,
|
||||
attn_groups: list[list[AttentionGroup]],
|
||||
kv_cache_config: KVCacheConfig,
|
||||
for_capture: bool = False,
|
||||
ubatch_idx: int = 0,
|
||||
model_specific_attn_metadata: ModelSpecificAttnMetadata | None = None,
|
||||
) -> dict[str, Any]:
|
||||
if self._replay is None:
|
||||
specs = [group.kv_cache_spec for group in kv_cache_config.kv_cache_groups]
|
||||
self._replay = (
|
||||
max(spec.prefix_replay_tokens for spec in specs),
|
||||
torch.tensor(
|
||||
[i for i, spec in enumerate(specs) if spec.prefix_cacheable],
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
),
|
||||
)
|
||||
window, cacheable_groups = self._replay
|
||||
if window:
|
||||
num_reqs = input_batch.num_reqs
|
||||
# Decode rows sit above the hit, so only prefills carry a replay
|
||||
# start; dummy batches (captures, profiling) carry none.
|
||||
replay_start_np = np.where(
|
||||
input_batch.is_prefilling_np[:num_reqs],
|
||||
self._replay_start_np[input_batch.idx_mapping_np[:num_reqs]],
|
||||
0,
|
||||
).astype(np.int32)
|
||||
replay_start = self._replay_start_staging.copy_to_gpu(
|
||||
replay_start_np, out=self._replay_start[:num_reqs]
|
||||
)
|
||||
if replay_start_np.any():
|
||||
# The replayed tokens rebuild window KV only: their slots in the
|
||||
# prefix-cacheable groups are padded so the cached KV stays as is.
|
||||
_pad_replayed_slots_kernel[(num_reqs,)](
|
||||
slot_mappings,
|
||||
slot_mappings.stride(0),
|
||||
cacheable_groups,
|
||||
input_batch.query_start_loc,
|
||||
input_batch.positions,
|
||||
replay_start,
|
||||
window,
|
||||
PAD_SLOT_ID,
|
||||
NUM_GROUPS=cacheable_groups.numel(),
|
||||
BLOCK=1024,
|
||||
)
|
||||
assert model_specific_attn_metadata is None
|
||||
model_specific_attn_metadata = ReplayAttnMetadata(replay_start)
|
||||
return super().prepare_attn(
|
||||
input_batch,
|
||||
cudagraph_mode,
|
||||
block_tables,
|
||||
slot_mappings,
|
||||
attn_groups,
|
||||
kv_cache_config,
|
||||
for_capture=for_capture,
|
||||
ubatch_idx=ubatch_idx,
|
||||
model_specific_attn_metadata=model_specific_attn_metadata,
|
||||
)
|
||||
|
||||
@@ -195,6 +195,7 @@ class DeepseekV4SparseMLAMetadataBuilder(
|
||||
if self.compress_ratio > 1:
|
||||
slot_mapping = get_compressed_slot_mapping(
|
||||
cm.num_actual_tokens,
|
||||
cm.slot_mapping,
|
||||
cm.query_start_loc,
|
||||
cm.seq_lens,
|
||||
cm.block_table_tensor.clamp_(min=0),
|
||||
|
||||
@@ -42,6 +42,8 @@ class CompressedSlotMappingKernel(
|
||||
@triton.jit(do_not_specialize=["block_table_stride"])
|
||||
def kernel(
|
||||
# [num_tokens]
|
||||
compressed_slot_mapping_ptr,
|
||||
# [num_tokens] the tokens' own slots
|
||||
slot_mapping_ptr,
|
||||
# [num_reqs + 1]
|
||||
query_start_loc_ptr,
|
||||
@@ -69,7 +71,12 @@ class CompressedSlotMappingKernel(
|
||||
mask = offset < query_len
|
||||
|
||||
pos = start_pos + i + tl.arange(0, TRITON_BLOCK_SIZE)
|
||||
is_valid = (pos + 1) % COMPRESS_RATIO == 0
|
||||
# A replayed token has a PAD slot (SWA bounded replay: its KV is
|
||||
# cached already); its compressed KV is cached too, so PAD here.
|
||||
slot = tl.load(
|
||||
slot_mapping_ptr + query_start + offset, mask=mask, other=PAD_ID
|
||||
)
|
||||
is_valid = ((pos + 1) % COMPRESS_RATIO == 0) & (slot != PAD_ID)
|
||||
pos_after_compress = pos // COMPRESS_RATIO
|
||||
|
||||
block_ids = pos_after_compress // block_size
|
||||
@@ -81,7 +88,9 @@ class CompressedSlotMappingKernel(
|
||||
|
||||
# NOTE
|
||||
slot_ids = tl.where(is_valid, slot_ids, PAD_ID)
|
||||
tl.store(slot_mapping_ptr + query_start + offset, slot_ids, mask=mask)
|
||||
tl.store(
|
||||
compressed_slot_mapping_ptr + query_start + offset, slot_ids, mask=mask
|
||||
)
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
@@ -121,6 +130,7 @@ class CompressedSlotMappingKernel(
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
int32_ptr = TritonWarmupTensor(torch.int32)
|
||||
return dict(
|
||||
compressed_slot_mapping=TritonWarmupTensor(torch.int64),
|
||||
slot_mapping=TritonWarmupTensor(torch.int64),
|
||||
query_start_loc=int32_ptr,
|
||||
seq_lens=int32_ptr,
|
||||
@@ -132,6 +142,7 @@ class CompressedSlotMappingKernel(
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
compressed_slot_mapping: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
@@ -149,6 +160,7 @@ class CompressedSlotMappingKernel(
|
||||
|
||||
def get_compressed_slot_mapping(
|
||||
num_tokens: int,
|
||||
slot_mapping: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
@@ -156,19 +168,28 @@ def get_compressed_slot_mapping(
|
||||
compress_ratio: int,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Slot mapping for writing the compressed states of ``num_tokens`` tokens.
|
||||
|
||||
Every ``compress_ratio`` tokens share one compressed state, written by the
|
||||
last of them: that token maps to the state's slot, the others to PAD. A
|
||||
token whose own ``slot_mapping`` entry is PAD maps to PAD too: SWA bounded
|
||||
replay recomputes tokens whose KV is cached already, and their compressed
|
||||
states must not be rewritten either.
|
||||
"""
|
||||
if out is not None:
|
||||
# Guard: for padded / invalid sequences.
|
||||
# Negative positions produce bogus block indices that lead to illegal memory
|
||||
# accesses inside the block_table load.
|
||||
# NOTE: Fill -1 to the whole tensor, not just the first `num_tokens`.
|
||||
out.fill_(-1)
|
||||
slot_mapping = out[:num_tokens]
|
||||
compressed_slot_mapping = out[:num_tokens]
|
||||
else:
|
||||
slot_mapping = torch.full(
|
||||
compressed_slot_mapping = torch.full(
|
||||
(num_tokens,), -1, dtype=torch.int64, device=query_start_loc.device
|
||||
)
|
||||
|
||||
_COMPRESSED_SLOT_MAPPING_KERNEL(
|
||||
compressed_slot_mapping,
|
||||
slot_mapping,
|
||||
query_start_loc,
|
||||
seq_lens,
|
||||
@@ -176,7 +197,7 @@ def get_compressed_slot_mapping(
|
||||
block_size,
|
||||
compress_ratio,
|
||||
)
|
||||
return slot_mapping
|
||||
return compressed_slot_mapping
|
||||
|
||||
|
||||
_COMPRESSED_SLOT_MAPPING_KERNEL = CompressedSlotMappingKernel()
|
||||
|
||||
@@ -795,6 +795,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0
|
||||
self.pcp_world_size = parallel_config.prefill_context_parallel_size
|
||||
self.use_pcp = self.pcp_world_size > 1
|
||||
self.pcp_rank = get_pcp_group().rank_in_group if self.use_pcp else 0
|
||||
self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size
|
||||
# NOTE(Chen):an estimated max size of flattened_kv. Need to double check.
|
||||
self.max_prefill_buffer_size = get_max_prefill_buffer_size(self.vllm_config)
|
||||
@@ -1217,10 +1218,18 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
factor = self.kv_cache_spec.block_size // kernel_block_size
|
||||
indexer_block_table = (block_table[:, ::factor] // factor).contiguous()
|
||||
padded_num_tokens = num_tokens
|
||||
if self.pcp_world_size > 1:
|
||||
local_slot_mapping = slot_mapping
|
||||
if self.use_pcp:
|
||||
# The gathered layout holds each rank's local tokens, padded, in
|
||||
# rank order, so this rank's segment lines up with query_start_loc.
|
||||
padded_num_tokens = slot_mapping.shape[0] // self.pcp_world_size
|
||||
local_slot_mapping = slot_mapping[
|
||||
self.pcp_rank * padded_num_tokens : (self.pcp_rank + 1)
|
||||
* padded_num_tokens
|
||||
]
|
||||
compressed_slot_mapping = get_compressed_slot_mapping(
|
||||
num_tokens,
|
||||
local_slot_mapping,
|
||||
query_start_loc,
|
||||
seq_lens,
|
||||
indexer_block_table,
|
||||
|
||||
@@ -81,9 +81,12 @@ class DeepseekV4SWACache(torch.nn.Module, AttentionLayerBase):
|
||||
block_size: int = 64,
|
||||
packed_bytes_per_token: int = 584,
|
||||
packed_page_alignment: int = 576,
|
||||
bounded_replay: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
self.backend_cls = backend_cls or DeepseekSparseSWABackend
|
||||
# DeepseekV4.1 SWA bounded replay.
|
||||
self.bounded_replay = bounded_replay
|
||||
self.kv_cache = torch.tensor([])
|
||||
self.head_dim = head_dim
|
||||
self.window_size = window_size
|
||||
@@ -117,6 +120,7 @@ class DeepseekV4SWACache(torch.nn.Module, AttentionLayerBase):
|
||||
"nvfp4_ds_mla",
|
||||
)
|
||||
return SlidingWindowMLASpec(
|
||||
bounded_replay=self.bounded_replay,
|
||||
block_size=self.block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=self.head_dim,
|
||||
@@ -198,6 +202,10 @@ class DeepseekSparseSWAMetadata:
|
||||
# None when the model is text-only or the batch has no image spans.
|
||||
prefill_left_visible: torch.Tensor | None = None
|
||||
prefill_right_visible: torch.Tensor | None = None
|
||||
# SWA bounded replay: [num_reqs] lower bound of the positions a request's
|
||||
# window attention may read (SWA bounded replay);
|
||||
# zeros when nothing replays.
|
||||
replay_start: torch.Tensor | None = None
|
||||
|
||||
# Number of decode/prefill requests/tokens (batch is reordered: decodes first)
|
||||
num_decodes: int = 0
|
||||
@@ -256,10 +264,12 @@ class DeepseekSparseSWAMetadata:
|
||||
has_compressed = compress_ratio > 1
|
||||
|
||||
# query_len <= max_num_batched_tokens and
|
||||
# gather_len = query_len + min(prefix_len, window_size - 1), so the
|
||||
# worst-case gathered width is bounded by
|
||||
# max_num_batched_tokens + window_size - 1. The compressed prefix pool
|
||||
# is bounded by ceil(max_model_len / compress_ratio).
|
||||
# gather_len <= query_len + min(prefix_len, window_size - 1) (a replay
|
||||
# boundary only shrinks it), so the worst-case gathered width is
|
||||
# bounded by max_num_batched_tokens + window_size - 1. The compressed
|
||||
# prefix pool is bounded by ceil(max_model_len / compress_ratio). The
|
||||
# CPU gather lengths below are this upper bound, for chunk planning
|
||||
# only; the kernels use the exact per-request values.
|
||||
max_workspace_area = prefill_chunk_size * (
|
||||
(cdiv(self.prefill_max_model_len, compress_ratio) if has_compressed else 0)
|
||||
+ self.prefill_window_size
|
||||
@@ -331,6 +341,7 @@ class ComputePrefillMetadataKernel(
|
||||
# Inputs
|
||||
seq_lens_ptr,
|
||||
query_start_loc_ptr,
|
||||
replay_start_ptr,
|
||||
num_prefills,
|
||||
num_decodes,
|
||||
window_size,
|
||||
@@ -352,7 +363,10 @@ class ComputePrefillMetadataKernel(
|
||||
|
||||
query_len = qsl_end - qsl_start
|
||||
prefix_len = seq_len - query_len
|
||||
gather_len = query_len + tl.minimum(prefix_len, window_size - 1)
|
||||
# Context below replay_start has no window KV (SWA bounded replay).
|
||||
replay_start = tl.load(replay_start_ptr + num_decodes + safe_offset, mask=mask)
|
||||
visible_prefix = tl.maximum(prefix_len - replay_start, 0)
|
||||
gather_len = query_len + tl.minimum(visible_prefix, window_size - 1)
|
||||
|
||||
tl.store(prefill_gather_lens_ptr + offset, gather_len, mask=mask)
|
||||
|
||||
@@ -384,6 +398,7 @@ class ComputePrefillMetadataKernel(
|
||||
prefill_gather_lens=int32_ptr,
|
||||
seq_lens=int32_ptr,
|
||||
query_start_loc=int32_ptr,
|
||||
replay_start=int32_ptr,
|
||||
num_prefills=compile_key.block_size,
|
||||
num_decodes=0,
|
||||
window_size=1,
|
||||
@@ -395,6 +410,7 @@ class ComputePrefillMetadataKernel(
|
||||
prefill_gather_lens: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
replay_start: torch.Tensor,
|
||||
num_prefills: int,
|
||||
num_decodes: int,
|
||||
window_size: int,
|
||||
@@ -550,12 +566,19 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
|
||||
)
|
||||
self.decode_swa_indices_noncausal: torch.Tensor | None = None
|
||||
self._max_tokens = max_tokens
|
||||
# replay_start when no group replays: no lower bound.
|
||||
self.no_replay_start = torch.zeros(
|
||||
self.vllm_config.scheduler_config.max_num_seqs + 1,
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
def build(
|
||||
self,
|
||||
common_prefix_len: int,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
fast_build: bool = False,
|
||||
replay_start: torch.Tensor | None = None,
|
||||
) -> DeepseekSparseSWAMetadata:
|
||||
"""Build SWA metadata for mixed decode/prefill batches.
|
||||
|
||||
@@ -564,6 +587,10 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
|
||||
separate window_topk_idxs for each portion.
|
||||
|
||||
For prefill, we use chunked prefill to align with the indexer's chunking.
|
||||
|
||||
``replay_start`` ([num_reqs], SWA bounded replay): the position from
|
||||
which each request holds window KV; the model state passes it for every
|
||||
batch of a replaying group, graph captures pass nothing.
|
||||
"""
|
||||
seq_lens = common_attn_metadata.seq_lens
|
||||
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
|
||||
@@ -587,6 +614,9 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
|
||||
|
||||
is_valid_token = self.is_valid_token[: slot_mapping.shape[0]]
|
||||
is_valid_token.copy_(slot_mapping >= 0)
|
||||
if replay_start is None:
|
||||
# Graph captures build without it; their dummy batches replay nothing.
|
||||
replay_start = self.no_replay_start
|
||||
|
||||
non_causal = not common_attn_metadata.causal
|
||||
decode_swa_width = (
|
||||
@@ -609,6 +639,8 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
|
||||
device=self.device,
|
||||
)
|
||||
decode_swa_indices = self.decode_swa_indices_noncausal
|
||||
# No replay_start: decode rows sit above the hit, so a
|
||||
# request's replayed window never reaches them.
|
||||
_COMPUTE_DSPARK_NONCAUSAL_SWA_INDICES_KERNEL(
|
||||
decode_swa_indices,
|
||||
self.decode_swa_lens,
|
||||
@@ -637,6 +669,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
|
||||
is_valid_token,
|
||||
block_table,
|
||||
self.block_size,
|
||||
replay_start,
|
||||
num_tokens=num_decode_tokens,
|
||||
token_offset=0,
|
||||
)
|
||||
@@ -683,6 +716,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
|
||||
is_valid_token,
|
||||
block_table,
|
||||
self.block_size,
|
||||
replay_start,
|
||||
num_tokens=num_prefill_tokens,
|
||||
token_offset=num_decode_tokens,
|
||||
has_image=has_image,
|
||||
@@ -696,6 +730,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
|
||||
seq_lens_cpu,
|
||||
query_start_loc,
|
||||
query_start_loc_cpu,
|
||||
replay_start,
|
||||
)
|
||||
|
||||
# Per-layer-type tile-scheduler plan holders. Empty FlashMLASchedMeta
|
||||
@@ -727,6 +762,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
|
||||
),
|
||||
prefill_left_visible=prefill_left_visible,
|
||||
prefill_right_visible=prefill_right_visible,
|
||||
replay_start=replay_start,
|
||||
block_size=self.block_size,
|
||||
num_decodes=num_decodes,
|
||||
num_prefills=num_prefills,
|
||||
@@ -803,6 +839,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
|
||||
assert metadata.is_valid_token is not None
|
||||
assert metadata.decode_swa_indices is not None
|
||||
assert metadata.decode_swa_lens is not None
|
||||
assert metadata.replay_start is not None
|
||||
|
||||
_COMPUTE_SWA_INDICES_AND_LENS_KERNEL(
|
||||
metadata.decode_swa_indices,
|
||||
@@ -817,6 +854,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
|
||||
metadata.is_valid_token,
|
||||
metadata.block_table,
|
||||
self.block_size,
|
||||
metadata.replay_start,
|
||||
num_tokens=metadata.num_decode_tokens,
|
||||
token_offset=0,
|
||||
)
|
||||
@@ -872,6 +910,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
|
||||
seq_lens_cpu: torch.Tensor | None,
|
||||
query_start_loc: torch.Tensor,
|
||||
query_start_loc_cpu: torch.Tensor,
|
||||
replay_start: torch.Tensor,
|
||||
) -> dict[str, torch.Tensor | int | None]:
|
||||
"""Pre-compute DeepseekV4 prefill metadata during the metadata build phase.
|
||||
|
||||
@@ -893,6 +932,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
|
||||
pfx_gather_lens,
|
||||
seq_lens,
|
||||
query_start_loc,
|
||||
replay_start,
|
||||
num_prefills,
|
||||
num_decodes,
|
||||
self.window_size,
|
||||
@@ -982,6 +1022,7 @@ def _compute_swa_indices_and_lens_kernel(
|
||||
block_table_ptr,
|
||||
block_table_stride,
|
||||
block_size,
|
||||
replay_start_ptr,
|
||||
token_offset,
|
||||
HAS_IMAGE: tl.constexpr,
|
||||
TRITON_BLOCK_SIZE: tl.constexpr,
|
||||
@@ -1022,6 +1063,8 @@ def _compute_swa_indices_and_lens_kernel(
|
||||
right = 0
|
||||
left_add = tl.maximum(left - (window_size - 1), 0)
|
||||
start_pos = tl.maximum(pos - (window_size - 1) - left_add, 0)
|
||||
# SWA bounded replay: no window KV exists below the request's replay start.
|
||||
start_pos = tl.maximum(start_pos, tl.load(replay_start_ptr + req_idx))
|
||||
end_pos = pos + right + 1
|
||||
|
||||
swa_len = end_pos - start_pos
|
||||
@@ -1111,6 +1154,7 @@ class ComputeSWAIndicesAndLensKernel(
|
||||
is_valid_token=TritonWarmupTensor(torch.bool),
|
||||
block_table=TritonWarmupTensor(torch.int32, shape=(1, 1), strides=(1, 1)),
|
||||
block_size=compile_key.block_size,
|
||||
replay_start=int32_ptr,
|
||||
num_tokens=1,
|
||||
token_offset=0,
|
||||
has_image=bool(compile_key.has_image),
|
||||
@@ -1131,6 +1175,7 @@ class ComputeSWAIndicesAndLensKernel(
|
||||
is_valid_token: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
block_size: int,
|
||||
replay_start: torch.Tensor,
|
||||
*,
|
||||
num_tokens: int,
|
||||
token_offset: int,
|
||||
|
||||
@@ -465,12 +465,17 @@ class KVCacheManager:
|
||||
A list of new allocated blocks.
|
||||
|
||||
"""
|
||||
# When loading KV data asynchronously, we may have zero new tokens to
|
||||
# compute while still allocating slots for externally computed tokens.
|
||||
if num_new_tokens == 0 and num_external_computed_tokens == 0:
|
||||
# A step may need no slots of its own while still adopting computed
|
||||
# tokens: an async KV load, or a chunk that ends inside the replayed
|
||||
# range of a hit (SWA bounded replay).
|
||||
if (
|
||||
num_new_tokens == 0
|
||||
and num_external_computed_tokens == 0
|
||||
and num_new_computed_tokens == 0
|
||||
):
|
||||
raise ValueError(
|
||||
"num_new_tokens must be greater than 0 when there are no "
|
||||
"external computed tokens"
|
||||
"computed tokens to adopt"
|
||||
)
|
||||
|
||||
if new_computed_blocks is not None:
|
||||
|
||||
@@ -48,6 +48,8 @@ class NewRequestData:
|
||||
|
||||
# Only used for v2 model runner.
|
||||
prefill_token_ids: list[int] | None = None
|
||||
# DeepSeek-V4.1 only: SWA bounded replay; see Request.replay_start.
|
||||
replay_start: int = 0
|
||||
|
||||
@classmethod
|
||||
def from_request(
|
||||
@@ -73,6 +75,7 @@ class NewRequestData:
|
||||
prompt_embeds=request.prompt_embeds,
|
||||
prompt_is_token_ids=request.prompt_is_token_ids,
|
||||
prefill_token_ids=prefill_token_ids,
|
||||
replay_start=request.replay_start,
|
||||
)
|
||||
|
||||
@property
|
||||
|
||||
@@ -271,6 +271,18 @@ class Scheduler(SchedulerInterface):
|
||||
self.use_eagle_block_drop = False
|
||||
self.num_spec_tokens = vllm_config.num_speculative_tokens
|
||||
self.num_lookahead_tokens = vllm_config.num_lookahead_tokens
|
||||
# DSV41 SWA bounded replay: groups that declare a replay window are rebuilt
|
||||
# after a prefix hit by recomputing its trailing tokens. One window
|
||||
# for all such groups, so the rewind matches every group's allocation.
|
||||
replay_windows = {
|
||||
group.kv_cache_spec.prefix_replay_tokens
|
||||
for group in kv_cache_config.kv_cache_groups
|
||||
if group.kv_cache_spec.prefix_replay_tokens > 0
|
||||
}
|
||||
assert len(replay_windows) <= 1, (
|
||||
f"Prefix replay windows should agree: {sorted(replay_windows)}"
|
||||
)
|
||||
self.prefix_replay_tokens = replay_windows.pop() if replay_windows else 0
|
||||
self.num_prefill_lookahead = vllm_config.num_prefill_lookahead_tokens
|
||||
self.dynamic_sd_lookup: list[int] | None = None
|
||||
if speculative_config is not None:
|
||||
@@ -946,6 +958,17 @@ class Scheduler(SchedulerInterface):
|
||||
step_skipped_waiting.prepend_request(request)
|
||||
continue
|
||||
|
||||
if self.prefix_replay_tokens:
|
||||
# SWA bounded replay recomputes the hit's last
|
||||
# window, from the block holding its first
|
||||
# token; the sliding-window groups retire
|
||||
# whole blocks below the window of the hit's
|
||||
# next token. Hits end on a block boundary, or
|
||||
# a hit ending one token short of one would
|
||||
# retire the block the replay starts in.
|
||||
ext_tokens -= ext_tokens % self.block_size
|
||||
load_kv_async = load_kv_async and ext_tokens > 0
|
||||
|
||||
if partial_tail and ext_tokens > partial_tail:
|
||||
# Remote strictly exceeds the full local hit: drop the
|
||||
# sub-block tail so no CoW is needed, and let the load
|
||||
@@ -988,6 +1011,17 @@ class Scheduler(SchedulerInterface):
|
||||
num_new_local_computed_tokens + num_external_computed_tokens
|
||||
)
|
||||
assert num_computed_tokens <= request.num_tokens
|
||||
if 0 < num_computed_tokens <= self.prefix_replay_tokens:
|
||||
# SWA bounded replay: a hit no longer than the replayed
|
||||
# window would be recomputed in full and save nothing.
|
||||
new_computed_blocks = (
|
||||
self.kv_cache_manager.empty_kv_cache_blocks
|
||||
)
|
||||
num_new_local_computed_tokens = 0
|
||||
num_external_computed_tokens = 0
|
||||
connector_prefix_cache_hits = 0
|
||||
num_computed_tokens = 0
|
||||
load_kv_async = False
|
||||
|
||||
# Skip request with pending mm encoding prefetches
|
||||
if self._ec_transfer_pending(request, num_computed_tokens):
|
||||
@@ -1022,6 +1056,11 @@ class Scheduler(SchedulerInterface):
|
||||
new_encoder_compute_budget = encoder_compute_budget
|
||||
pad_spec_decode = False
|
||||
|
||||
# SWA bounded replay: recompute the tail of the hit without
|
||||
# rewriting its cached KV. An async load replays once the KV
|
||||
# has arrived (_update_waiting_for_remote_kv).
|
||||
num_replay_tokens = 0
|
||||
|
||||
if load_kv_async:
|
||||
# KVTransfer: loading remote KV, do not allocate for new work.
|
||||
assert num_external_computed_tokens > 0
|
||||
@@ -1032,6 +1071,12 @@ class Scheduler(SchedulerInterface):
|
||||
break
|
||||
else:
|
||||
request_token_budget = min(token_budget, input_budget - draft_slots)
|
||||
if did_prefix_cache_lookup:
|
||||
# Fresh admission: local and/or sync external hit.
|
||||
num_replay_tokens = self._mark_prefix_replay(
|
||||
request, num_computed_tokens
|
||||
)
|
||||
num_computed_tokens -= num_replay_tokens
|
||||
# Number of tokens to be scheduled.
|
||||
# We use `request.num_tokens` instead of
|
||||
# `request.num_prompt_tokens` to consider the resumed
|
||||
@@ -1159,9 +1204,12 @@ class Scheduler(SchedulerInterface):
|
||||
+ self._spec_decode_step_blocks()
|
||||
)
|
||||
|
||||
# Replayed tokens are already counted in the adopted hit; a
|
||||
# chunk that ends inside the replayed range needs no new slots.
|
||||
num_tokens_past_hit = max(num_new_tokens - num_replay_tokens, 0)
|
||||
new_blocks = self.kv_cache_manager.allocate_slots(
|
||||
request,
|
||||
num_new_tokens,
|
||||
num_tokens_past_hit,
|
||||
num_new_computed_tokens=num_new_local_computed_tokens,
|
||||
new_computed_blocks=new_computed_blocks,
|
||||
num_lookahead_tokens=effective_lookahead_tokens,
|
||||
@@ -2963,6 +3011,20 @@ class Scheduler(SchedulerInterface):
|
||||
self._request_remaining_blocks(req) for req in self._inflight_prefills
|
||||
)
|
||||
|
||||
def _mark_prefix_replay(self, request: Request, num_hit_tokens: int) -> int:
|
||||
"""Record where a prefix hit's replay starts and return the number of
|
||||
hit tokens to recompute; the caller rewinds the computed count by it.
|
||||
|
||||
The replayed tokens are the hit's last window (see
|
||||
``Request.replay_start``): the worker rebuilds their sliding-window KV
|
||||
and leaves their cached KV alone.
|
||||
"""
|
||||
if not self.prefix_replay_tokens:
|
||||
return 0
|
||||
num_replay_tokens = min(self.prefix_replay_tokens, num_hit_tokens)
|
||||
request.replay_start = num_hit_tokens - num_replay_tokens
|
||||
return num_replay_tokens
|
||||
|
||||
def _update_waiting_for_remote_kv(self, request: Request) -> None:
|
||||
"""KV Connector: update request state after async recv is finished.
|
||||
|
||||
@@ -2998,11 +3060,16 @@ class Scheduler(SchedulerInterface):
|
||||
# This will cache the blocks iff caching is enabled.
|
||||
self.kv_cache_manager.cache_blocks(request, request.num_computed_tokens)
|
||||
|
||||
# on a full prompt hit, we need to re-compute the last token
|
||||
# in order to be able to sample the next token
|
||||
if request.num_computed_tokens == request.num_tokens:
|
||||
request.num_computed_tokens = request.num_tokens - 1
|
||||
|
||||
# SWA bounded replay recomputes the tail of the hit, which covers the
|
||||
# last token; otherwise a full prompt hit re-computes that token so the
|
||||
# next one can be sampled.
|
||||
num_replay_tokens = self._mark_prefix_replay(
|
||||
request, request.num_computed_tokens
|
||||
)
|
||||
if num_replay_tokens > 0:
|
||||
request.num_computed_tokens -= num_replay_tokens
|
||||
elif request.num_computed_tokens == request.num_tokens:
|
||||
request.num_computed_tokens = request.num_tokens - 1
|
||||
self.finished_recving_kv_req_ids.remove(request.request_id)
|
||||
|
||||
def _try_promote_blocked_waiting_request(self, request: Request) -> bool:
|
||||
|
||||
@@ -492,6 +492,8 @@ class SingleTypeKVCacheManager(ABC):
|
||||
can resume at, from ``get_replay_boundaries``.
|
||||
|
||||
"""
|
||||
if not self.kv_cache_spec.prefix_cacheable:
|
||||
return
|
||||
num_cached_blocks = self.num_cached_block.get(request.request_id, 0)
|
||||
num_full_blocks = num_tokens // self.block_size
|
||||
|
||||
|
||||
@@ -164,6 +164,11 @@ class KVCacheSpec:
|
||||
"""Whether this spec's group participates in prefix caching."""
|
||||
return True
|
||||
|
||||
@property
|
||||
def prefix_replay_tokens(self) -> int:
|
||||
"""DeepSeek-V4.1 only: bounded replay. Currently only for DSV41 SWA."""
|
||||
return 0
|
||||
|
||||
@property
|
||||
def num_heads(self) -> int:
|
||||
raise NotImplementedError
|
||||
@@ -908,6 +913,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
# DeepseekV4-only: see MLAAttentionSpec.model_version.
|
||||
alignment: int | None = None # Default to None for no padding.
|
||||
model_version: str | None = None
|
||||
bounded_replay: bool = False
|
||||
|
||||
# MLA stores a single latent vector per state; there is no separate V.
|
||||
head_size_v: int = 0
|
||||
@@ -919,6 +925,14 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
super().__post_init__()
|
||||
_apply_alignment_padding(self)
|
||||
|
||||
@property
|
||||
def prefix_cacheable(self) -> bool:
|
||||
return not self.bounded_replay
|
||||
|
||||
@property
|
||||
def prefix_replay_tokens(self) -> int:
|
||||
return self.sliding_window if self.bounded_replay else 0
|
||||
|
||||
@classmethod
|
||||
def merge(cls, specs: list[Self]) -> Self:
|
||||
assert all(isinstance(spec, SlidingWindowMLASpec) for spec in specs), (
|
||||
@@ -930,16 +944,18 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
model_version_set = set(spec.model_version for spec in specs)
|
||||
sliding_window_set = set(spec.sliding_window for spec in specs)
|
||||
extra_retained_set = set(spec.extra_retained_tokens for spec in specs)
|
||||
bounded_replay_set = set(spec.bounded_replay for spec in specs)
|
||||
assert (
|
||||
len(cache_dtype_str_set) == 1
|
||||
and len(tokens_per_state_set) == 1
|
||||
and len(model_version_set) == 1
|
||||
and len(sliding_window_set) == 1
|
||||
and len(extra_retained_set) == 1
|
||||
and len(bounded_replay_set) == 1
|
||||
), (
|
||||
"All attention layers in the same KV cache group must use the same "
|
||||
"quantization method, tokens per state, model version, sliding "
|
||||
"window size, and retained token count."
|
||||
"window size, retained token count, and replay policy."
|
||||
)
|
||||
return cls(
|
||||
block_size=specs[0].block_size,
|
||||
@@ -954,6 +970,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
cache_dtype_str=cache_dtype_str_set.pop(),
|
||||
tokens_per_state=tokens_per_state_set.pop(),
|
||||
model_version=model_version_set.pop(),
|
||||
bounded_replay=bounded_replay_set.pop(),
|
||||
)
|
||||
|
||||
def is_uniform_with_collection(
|
||||
@@ -962,6 +979,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
return all(
|
||||
isinstance(spec, SlidingWindowMLASpec)
|
||||
and spec.sliding_window == self.sliding_window
|
||||
and spec.bounded_replay == self.bounded_replay
|
||||
for spec in kv_cache_specs.values()
|
||||
)
|
||||
|
||||
@@ -1195,6 +1213,10 @@ class UniformTypeKVCacheSpecs(KVCacheSpec):
|
||||
def prefix_cacheable(self) -> bool:
|
||||
return all(spec.prefix_cacheable for spec in self.kv_cache_specs.values())
|
||||
|
||||
@property
|
||||
def prefix_replay_tokens(self) -> int:
|
||||
return max(spec.prefix_replay_tokens for spec in self.kv_cache_specs.values())
|
||||
|
||||
@property
|
||||
def first_spec(self) -> KVCacheSpec:
|
||||
"""Return the first spec in the group."""
|
||||
|
||||
@@ -201,6 +201,9 @@ class Request:
|
||||
# in the (sparse) prefix cache; 0 means none. Set at admission for
|
||||
# hybrid/Mamba models when a shared prefix is detected (Marconi-style).
|
||||
self.shared_prefix_boundary = 0
|
||||
# DeepSeek-V4.1 only: SWA bounded replay. The request holds no
|
||||
# sliding-window KV below this position; 0 when nothing replays.
|
||||
self.replay_start = 0
|
||||
|
||||
# The number of NaNs in logits. A value greater than 0
|
||||
# indicates that the output is corrupted
|
||||
|
||||
@@ -16,7 +16,10 @@ from vllm.v1.worker.gpu.attn_utils import (
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
from vllm.v1.worker.gpu.mm.rope import get_rope_state
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.gpu.model_states.interface import (
|
||||
ModelSpecificAttnMetadata,
|
||||
ModelState,
|
||||
)
|
||||
from vllm.v1.worker.gpu.model_states.mm_pruning import maybe_create_mm_pruner
|
||||
from vllm.v1.worker.gpu.model_states.prompt_embeds import PromptEmbedsState
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
@@ -173,6 +176,7 @@ class DefaultModelState(ModelState):
|
||||
kv_cache_config: KVCacheConfig,
|
||||
for_capture: bool = False,
|
||||
ubatch_idx: int = 0,
|
||||
model_specific_attn_metadata: ModelSpecificAttnMetadata | None = None,
|
||||
) -> dict[str, Any]:
|
||||
if cudagraph_mode == CUDAGraphMode.FULL:
|
||||
# Use padded sizes - padding is handled by model_runner.prepare_attn.
|
||||
@@ -229,5 +233,6 @@ class DefaultModelState(ModelState):
|
||||
req_idx=input_batch.idx_mapping_np,
|
||||
ubatch_idx=ubatch_idx,
|
||||
fast_prefill=input_batch.fast_prefill,
|
||||
model_specific_attn_metadata=model_specific_attn_metadata,
|
||||
)
|
||||
return attn_metadata
|
||||
|
||||
@@ -19,6 +19,7 @@ from vllm.v1.kv_cache_interface import EncoderOnlyAttentionSpec, KVCacheConfig
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelSpecificAttnMetadata
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
@@ -168,6 +169,7 @@ class EncoderOnlyModelState(DefaultModelState):
|
||||
kv_cache_config: KVCacheConfig,
|
||||
for_capture: bool = False,
|
||||
ubatch_idx: int = 0,
|
||||
model_specific_attn_metadata: ModelSpecificAttnMetadata | None = None,
|
||||
) -> dict[str, Any]:
|
||||
assert ubatch_idx == 0, "DBO is not supported"
|
||||
attn_metadata = super().prepare_attn(
|
||||
@@ -178,6 +180,7 @@ class EncoderOnlyModelState(DefaultModelState):
|
||||
attn_groups,
|
||||
kv_cache_config,
|
||||
for_capture,
|
||||
model_specific_attn_metadata=model_specific_attn_metadata,
|
||||
)
|
||||
attn_metadata.update(
|
||||
self._build_encoder_attn_metadata(input_batch, cudagraph_mode, for_capture)
|
||||
|
||||
@@ -239,8 +239,10 @@ class MambaHybridModelState(DefaultModelState):
|
||||
kv_cache_config: KVCacheConfig,
|
||||
for_capture: bool = False,
|
||||
ubatch_idx: int = 0,
|
||||
model_specific_attn_metadata: ModelSpecificAttnMetadata | None = None,
|
||||
) -> dict[str, Any]:
|
||||
assert ubatch_idx == 0, "DBO is not supported"
|
||||
assert model_specific_attn_metadata is None
|
||||
if cudagraph_mode == CUDAGraphMode.FULL:
|
||||
num_reqs = input_batch.num_reqs_after_padding
|
||||
num_tokens = input_batch.num_tokens_after_padding
|
||||
|
||||
Reference in New Issue
Block a user