[Model] Decoder-side SWA bounded replay for DeepSeek-V4.1 (#58132)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Yifan Qiao
2026-09-29 08:25:46 +00:00
committed by GitHub
co-authored by Claude Fable 5.1
parent 9af952c555
commit 4b2e1cfa7b
9 changed files with 1000 additions and 59 deletions
@@ -0,0 +1,104 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DecoderReplayLayers runs the replay layers on the forward's replay batch."""
import pytest
import torch
from vllm.forward_context import (
ForwardContext,
get_forward_context,
override_forward_context,
)
from vllm.models.deepseek_v41.decoder_replay_layers import DecoderReplayLayers
pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
DEVICE = torch.device("cuda")
NUM_TOKENS = 401
HC, HIDDEN = 2, 3
# decode (1 token), trimmed prefill (300 of 300), untrimmed prefill (100 of 300)
WINDOW = 128
REPLAY_ROWS = [0, *range(301 - WINDOW, 301), *range(301, 401)]
def _context(metadata):
return ForwardContext(
no_compile_layers={},
attn_metadata=metadata,
slot_mapping={},
is_padding=torch.zeros(NUM_TOKENS, dtype=torch.bool, device=DEVICE),
)
def _set_replay_batch(layers, rows, metadata):
layers.rows = torch.tensor(rows, device=DEVICE)
layers.forward_context = ForwardContext(
no_compile_layers={}, attn_metadata=metadata, slot_mapping={}
)
def _states(num_tokens: int):
"""(hidden_states, positions, input_ids, pre_mix, post_mix, res_mix, residual),
shaped as the model's: hidden states [T, H], mixes [T, HC], residual [T, HC, H]."""
hidden = torch.arange(num_tokens, device=DEVICE, dtype=torch.float32)
hidden = hidden[:, None].expand(num_tokens, HIDDEN).contiguous()
mix = hidden[:, :HC].contiguous()
residual = hidden[:, None, :].expand(num_tokens, HC, HIDDEN).contiguous()
return (
hidden,
torch.arange(num_tokens, device=DEVICE),
None,
mix + 1,
mix + 2,
mix + 3,
residual,
)
def test_run_gathers_states_and_realigns_shared_indexer_buffers():
topk = torch.arange(NUM_TOKENS * 4, device=DEVICE).view(NUM_TOKENS, 4).int()
candidates = torch.arange(NUM_TOKENS * 3, device=DEVICE).view(NUM_TOKENS, 3).int()
topk_before, candidates_before = topk.clone(), candidates.clone()
seen = {}
def run_layers(hidden_states, positions, input_ids, pre_mix, post, res, residual):
seen["hidden_states"] = hidden_states.clone()
replay_context = get_forward_context()
seen["attn_metadata"] = replay_context.attn_metadata
seen["is_padding"] = replay_context.is_padding
return residual, pre_mix
layers = DecoderReplayLayers(WINDOW, run_layers, [topk, candidates])
states = _states(NUM_TOKENS)
hidden, residual = states[0], states[-1]
full, sub = object(), object()
_set_replay_batch(layers, REPLAY_ROWS, sub)
context = _context(full)
with override_forward_context(context):
outputs = layers(*states)
assert get_forward_context() is context
rows = torch.tensor(REPLAY_ROWS, device=DEVICE)
assert torch.equal(seen["hidden_states"], hidden[rows])
assert seen["attn_metadata"] is sub and seen["is_padding"] is None
assert torch.equal(topk[: len(REPLAY_ROWS)], topk_before[rows])
assert torch.equal(candidates[: len(REPLAY_ROWS)], candidates_before[rows])
assert len(outputs) == 2 and outputs[0].shape == residual.shape
assert torch.equal(outputs[0][rows], residual[rows])
assert outputs[0][1:173].abs().sum() == 0
def test_no_replay_batch_runs_the_whole_batch():
seen = {}
def run_layers(hidden_states, *rest):
seen["attn_metadata"] = get_forward_context().attn_metadata
return (hidden_states,)
layers = DecoderReplayLayers(WINDOW, run_layers, [])
states = _states(NUM_TOKENS)
full = object()
with override_forward_context(_context(full)):
(output,) = layers(*states)
assert output is states[0] and seen["attn_metadata"] is full
@@ -0,0 +1,259 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The decoder-side SWA bounded replay batch DeepseekV41ModelState prepares."""
from dataclasses import replace
from types import SimpleNamespace
from unittest.mock import MagicMock
import numpy as np
import pytest
import torch
from vllm.config import CUDAGraphMode
from vllm.models.deepseek_v41.nvidia.model_state import DeepseekV41ModelState
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
pytestmark = pytest.mark.skipif(
not torch.cuda.is_available(), reason="the replay buffers live on the GPU"
)
WINDOW = 128
DEVICE = torch.device("cuda")
GROUPS = SimpleNamespace(
kv_cache_groups=[
SimpleNamespace(
layer_names=["swa"],
kv_cache_spec=SimpleNamespace(
prefix_cacheable=False, prefix_replay_tokens=WINDOW
),
),
SimpleNamespace(
layer_names=["mla"],
kv_cache_spec=SimpleNamespace(
prefix_cacheable=True, prefix_replay_tokens=0
),
),
]
)
BLOCK_TABLES = (torch.zeros(4, 4, device=DEVICE),) * 2
# decode (1 token), trimmed prefill (300 of 300), untrimmed prefill (100 of 300)
QUERY_LENS = [1, 300, 100]
SEQ_LENS = [500, 300, 300]
PREFILLING = [False, True, True]
REPLAY_ROWS = [0, *range(301 - WINDOW, 301), *range(301, 401)]
@pytest.fixture
def state(monkeypatch):
"""A DeepseekV41ModelState whose attention builds are recorded, not run."""
cfg = MagicMock()
cfg.model_config.enable_prompt_embeds = False
cfg.model_config.uses_mrope = False
cfg.model_config.is_multimodal_model = False
cfg.scheduler_config.max_num_seqs = 16
cfg.scheduler_config.max_num_batched_tokens = 1024
cfg.parallel_config.data_parallel_size = 1
cfg.compilation_config.fast_moe_cold_start = False
layers = SimpleNamespace(window=WINDOW, rows=None, forward_context=None)
model = SimpleNamespace(token_lookback_depth=0, decoder_replay_layers=layers)
builds: list = []
def prepare_attn(self, input_batch, cg_mode, block_tables, slot_mappings, *a, **kw):
build = SimpleNamespace(
batch=input_batch,
cg_mode=cg_mode,
slot_mappings=slot_mappings.clone(),
replay_start=kw["model_specific_attn_metadata"].replay_start,
attn_metadata=object(),
)
builds.append(build)
return {"swa": build.attn_metadata}
monkeypatch.setattr(DefaultModelState, "prepare_attn", prepare_attn)
state = DeepseekV41ModelState(cfg, model, None, DEVICE)
state.builds = builds # type: ignore[attr-defined]
return state
def _input_batch(
query_lens: list[int],
seq_lens: list[int],
is_prefilling: list[bool],
device_query_lens: list[int] | None = None,
num_tokens_after_padding: int | None = None,
max_query_len: int | None = None,
) -> InputBatch:
num_reqs, num_tokens = len(query_lens), sum(query_lens)
num_padded = num_tokens_after_padding or num_tokens
batch = InputBatch.make_dummy(num_reqs, num_tokens, InputBuffers(16, 1024, DEVICE))
return replace(
batch,
num_tokens=num_tokens,
num_tokens_after_padding=num_padded,
query_start_loc=torch.tensor(
[0, *np.cumsum(device_query_lens or query_lens)],
dtype=torch.int32,
device=DEVICE,
),
query_start_loc_np=np.array([0, *np.cumsum(query_lens)], dtype=np.int32),
seq_lens=torch.tensor(seq_lens, dtype=torch.int32, device=DEVICE),
seq_lens_cpu_upper_bound=torch.tensor(seq_lens, dtype=torch.int32),
is_prefilling_np=np.array(is_prefilling, dtype=np.bool_),
positions=torch.cat(
[torch.arange(s - q, s) for q, s in zip(query_lens, seq_lens)]
+ [torch.zeros(num_padded - num_tokens, dtype=torch.int64)]
).to(DEVICE),
is_padding=torch.arange(num_padded, device=DEVICE) >= num_tokens,
max_query_len=max_query_len,
)
def _slot_mappings(num_tokens: int) -> torch.Tensor:
return torch.stack([torch.arange(num_tokens), torch.arange(num_tokens) * 10]).to(
DEVICE
)
def _prepare(state, batch, cg_mode):
"""Runs prepare_attn; returns the replay batch and the replay build's inputs."""
state.prepare_attn(
batch,
cg_mode,
BLOCK_TABLES,
_slot_mappings(batch.num_tokens_after_padding),
[],
GROUPS,
)
layers = state.decoder_replay_layers
if layers.rows is None:
assert layers.forward_context is None
return None, None
return SimpleNamespace(
rows=layers.rows, forward_context=layers.forward_context
), state.builds[-1]
def test_replay_batch_keeps_each_request_window(state):
state._replay_start_np[2] = 50 # the encoder-side replay start of request 2
batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING)
replay, build = _prepare(state, batch, CUDAGraphMode.NONE)
assert replay is not None
rows = torch.tensor(REPLAY_ROWS, device=DEVICE)
assert torch.equal(replay.rows, rows)
sub = build.batch
assert build.cg_mode == CUDAGraphMode.NONE
assert sub.query_start_loc.tolist() == [0, 1, 129, 229]
assert sub.query_start_loc_np.tolist() == [0, 1, 129, 229]
assert sub.num_tokens == sub.num_tokens_after_padding == 229
assert sub.max_query_len == WINDOW
# The trimmed request's window starts at its replay window; a higher
# encoder-side replay start stands.
assert build.replay_start.tolist() == [0, 300 - WINDOW, 50]
assert torch.equal(sub.positions, batch.positions[rows])
assert torch.equal(build.slot_mappings, _slot_mappings(401)[:, rows])
context = replay.forward_context
assert context.attn_metadata == {"swa": build.attn_metadata}
assert torch.equal(context.slot_mapping["mla"], _slot_mappings(401)[1, rows])
assert context.dp_metadata is None and context.is_padding is None
def test_replay_batch_keeps_device_decode_boundaries(state):
"""Adaptive verification resizes decodes on the GPU (CPU [2, 2], device
[1, 3]): their rows stay and the boundaries shift only by trimmed rows."""
batch = _input_batch(
[2, 2, 300], [10, 10, 300], PREFILLING, device_query_lens=[1, 3, 300]
)
replay, build = _prepare(state, batch, CUDAGraphMode.NONE)
assert replay is not None
assert replay.rows.tolist() == [0, 1, 2, 3, *range(304 - WINDOW, 304)]
assert build.batch.query_start_loc.tolist() == [0, 1, 4, 4 + WINDOW]
def test_prompt_logprobs_requests_keep_their_rows(state):
"""Every prompt row of a prompt-logprobs request is read, so it never
trims; the other prefills still do."""
state._keeps_rows_np[1] = True
batch = _input_batch([1, 300, 300], [500, 300, 300], PREFILLING)
replay, build = _prepare(state, batch, CUDAGraphMode.NONE)
assert replay is not None
assert replay.rows.tolist() == [0, *range(1, 301), *range(601 - WINDOW, 601)]
assert build.batch.max_query_len == 300
def test_replay_batch_keeps_adaptive_verification_query_bound(state):
"""Adaptive verification bounds the decodes' device-side query lengths
above their CPU lengths; the replay batch keeps that bound."""
batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING, max_query_len=200)
replay, build = _prepare(state, batch, CUDAGraphMode.NONE)
assert replay is not None and build.batch.max_query_len == 200
def test_only_eager_steps_trim(state):
"""A CUDA graph keeps the layers on its whole batch; an eager step trims a
padded batch to its real rows."""
batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING, num_tokens_after_padding=512)
for cg_mode in (CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL):
assert _prepare(state, batch, cg_mode) == (None, None)
replay, build = _prepare(state, batch, CUDAGraphMode.NONE)
assert replay is not None
assert replay.rows.tolist() == REPLAY_ROWS
assert build.cg_mode == CUDAGraphMode.NONE
assert build.batch.num_tokens_after_padding == len(REPLAY_ROWS)
def test_whole_batch_forwards_get_no_replay_batch(state):
short = _input_batch([100, 100], [100, 100], [True, True])
for cg_mode in (CUDAGraphMode.NONE, CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL):
assert _prepare(state, short, cg_mode) == (None, None)
assert len(state.builds) == 3 # the batch's own metadata only
# Dummy batches, captures included (prepared as eager), are not prefills and
# keep their rows.
dummy = _input_batch(
QUERY_LENS, SEQ_LENS, [False] * 3, num_tokens_after_padding=512
)
assert _prepare(state, dummy, CUDAGraphMode.NONE) == (None, None)
@pytest.fixture
def dp_state(state, monkeypatch):
"""`state` on DP rank 0 of 2; `state.other` sets what rank 1 reports
(trims, replay tokens)."""
import vllm.models.deepseek_v41.nvidia.model_state as module
state.vllm_config.parallel_config.data_parallel_size = 2
state.vllm_config.parallel_config.data_parallel_rank = 0
state.vllm_config.parallel_config.is_moe_model = True
state.other = (False, 0) # type: ignore[attr-defined]
def all_reduce(tensor, group):
tensor[:, 1] = torch.tensor(state.other, dtype=torch.int32)
monkeypatch.setattr(module.dist, "all_reduce", all_reduce)
monkeypatch.setattr(module, "get_dp_group", lambda: SimpleNamespace(cpu_group=None))
return state
def test_dp_ranks_replay_together(dp_state):
"""A trimming rank makes the others replay too, on every rank's count."""
short = _input_batch([100, 100], [100, 100], [True, True])
dp_state.other = (False, 200)
assert _prepare(dp_state, short, CUDAGraphMode.NONE) == (None, None)
dp_state.other = (True, 300) # rank 1 trims to 300 rows
replay, build = _prepare(dp_state, short, CUDAGraphMode.NONE)
assert replay is not None
assert replay.rows.tolist() == list(range(200)) # this rank keeps every row
dp_metadata = replay.forward_context.dp_metadata
assert dp_metadata.num_tokens_across_dp_cpu.tolist() == [200, 300]
def test_idle_dp_rank_dummy_trims_with_its_peers(dp_state):
dummy = _input_batch([512], [512], [False])
dp_state.other = (True, 129)
replay, _ = _prepare(dp_state, dummy, CUDAGraphMode.NONE)
assert replay is not None and replay.rows.shape[0] == WINDOW
dp_metadata = replay.forward_context.dp_metadata
assert dp_metadata.num_tokens_across_dp_cpu.tolist() == [WINDOW, 129]