[Perf][DSv4.1] Run decoder replay layers in CUDA graphs and trim in PIECEWISE graphs (#59532)

Signed-off-by: Juntian Liu <Juntianl777@gmail.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Juntian Liu
2026-10-05 22:11:45 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 21d93d0d8c
commit eb2901b735
10 changed files with 445 additions and 71 deletions
@@ -2,15 +2,26 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DecoderReplayLayers runs the replay layers on the forward's replay batch."""
from unittest.mock import MagicMock
import pytest
import torch
from vllm.config import CompilationConfig, CUDAGraphMode
from vllm.forward_context import (
BatchDescriptor,
ForwardContext,
get_forward_context,
override_forward_context,
)
from vllm.models.deepseek_v41.decoder_replay_layers import DecoderReplayLayers
from vllm.models.deepseek_v41.decoder_replay_layers import (
DecoderReplayLayers,
ReplayBatch,
)
from vllm.models.deepseek_v41.nvidia.decoder_replay_cudagraph import (
DecoderReplayCudaGraphManager,
)
from vllm.v1.worker.gpu import cudagraph_utils
pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
@@ -32,9 +43,9 @@ def _context(metadata):
def _set_replay_batch(layers, rows, metadata):
layers.rows = torch.tensor(rows, device=DEVICE)
layers.forward_context = ForwardContext(
no_compile_layers={}, attn_metadata=metadata, slot_mapping={}
layers.replay_batch = ReplayBatch(
torch.tensor(rows, device=DEVICE),
ForwardContext(no_compile_layers={}, attn_metadata=metadata, slot_mapping={}),
)
@@ -69,7 +80,7 @@ def test_run_gathers_states_and_realigns_shared_indexer_buffers():
seen["is_padding"] = replay_context.is_padding
return residual, pre_mix
layers = DecoderReplayLayers(WINDOW, run_layers, [topk, candidates])
layers = DecoderReplayLayers(WINDOW, run_layers, [topk, candidates], set())
states = _states(NUM_TOKENS)
hidden, residual = states[0], states[-1]
full, sub = object(), object()
@@ -96,9 +107,54 @@ def test_no_replay_batch_runs_the_whole_batch():
seen["attn_metadata"] = get_forward_context().attn_metadata
return (hidden_states,)
layers = DecoderReplayLayers(WINDOW, run_layers, [])
layers = DecoderReplayLayers(WINDOW, run_layers, [], set())
states = _states(NUM_TOKENS)
full = object()
with override_forward_context(_context(full)):
(output,) = layers(*states)
assert output is states[0] and seen["attn_metadata"] is full
def _graph_context(num_tokens):
return ForwardContext(
no_compile_layers={},
attn_metadata={},
slot_mapping={},
cudagraph_runtime_mode=CUDAGraphMode.PIECEWISE,
batch_descriptor=BatchDescriptor(num_tokens=num_tokens),
)
def test_replay_graph_matches_eager(monkeypatch):
"""The replay graph of the next captured size pads and runs the rows."""
monkeypatch.setattr(cudagraph_utils, "get_pp_group", MagicMock)
capture = torch.cuda.stream(torch.cuda.Stream())
monkeypatch.setattr(cudagraph_utils, "graph_capture", lambda device: capture)
compilation = CompilationConfig(decoder_replay_cudagraph_capture_sizes=[4])
cfg = MagicMock(compilation_config=compilation, speculative_config=None)
cfg.scheduler_config = MagicMock(max_num_seqs=2, max_num_batched_tokens=64)
cfg.cache_config.use_kda_recoverssm = False
cfg.model_config.hf_config = MagicMock(hc_mult=HC, hidden_size=HIDDEN)
cfg.model_config.dtype = torch.float32
def run_layers(hidden, positions, ids, pre, post, mix, residual):
return residual + hidden[:, None] + positions[:, None, None], pre + ids[:, None]
layers = DecoderReplayLayers(4, run_layers, [], set())
manager = DecoderReplayCudaGraphManager(cfg, DEVICE, layers)
def prepare(desc):
rows = torch.arange(desc.num_tokens)
layers.replay_batch = ReplayBatch(rows, _graph_context(desc.num_tokens))
manager.capture_replay_graphs(prepare)
states = [
torch.randn(12, *b.shape[1:], device=DEVICE).to(b.dtype)
for b in manager.input_buffers
]
rows = torch.tensor([11, 2, 7], device=DEVICE)
desc = manager.dispatch(2, 3, None, 0)
with override_forward_context(_graph_context(desc.num_tokens)):
actual = manager.run(rows, states)
for a, e in zip(actual, run_layers(*(t[rows] for t in states))):
torch.testing.assert_close(a, e, rtol=0, atol=0)
+31 -14
View File
@@ -12,6 +12,7 @@ import torch
from vllm.config import CUDAGraphMode
from vllm.models.deepseek_v41.nvidia.model_state import DeepseekV41ModelState
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
@@ -57,7 +58,8 @@ def state(monkeypatch):
cfg.scheduler_config.max_num_batched_tokens = 1024
cfg.parallel_config.data_parallel_size = 1
cfg.compilation_config.fast_moe_cold_start = False
layers = SimpleNamespace(window=WINDOW, rows=None, forward_context=None)
cfg.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
layers = SimpleNamespace(window=WINDOW, replay_batch=None, metadata_prefixes=set())
model = SimpleNamespace(token_lookback_depth=0, decoder_replay_layers=layers)
builds: list = []
@@ -127,13 +129,8 @@ def _prepare(state, batch, cg_mode):
[],
GROUPS,
)
layers = state.decoder_replay_layers
if layers.rows is None:
assert layers.forward_context is None
return None, None
return SimpleNamespace(
rows=layers.rows, forward_context=layers.forward_context
), state.builds[-1]
replay = state.decoder_replay_layers.replay_batch
return (None, None) if replay is None else (replay, state.builds[-1])
def test_replay_batch_keeps_each_request_window(state):
@@ -191,9 +188,9 @@ def test_replay_batch_keeps_adaptive_verification_query_bound(state):
assert replay is not None and build.batch.max_query_len == 200
def test_only_eager_steps_trim(state):
"""A CUDA graph keeps the layers on its whole batch; an eager step trims a
padded batch to its real rows."""
def test_graph_steps_trim_only_piecewise_at_threshold(state):
"""A CUDA graph keeps the layers on its whole batch, unless it is a PIECEWISE
one at the trim threshold or above; an eager step trims a batch to its rows."""
batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING, num_tokens_after_padding=512)
for cg_mode in (CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL):
assert _prepare(state, batch, cg_mode) == (None, None)
@@ -202,6 +199,10 @@ def test_only_eager_steps_trim(state):
assert replay.rows.tolist() == REPLAY_ROWS
assert build.cg_mode == CUDAGraphMode.NONE
assert build.batch.num_tokens_after_padding == len(REPLAY_ROWS)
state._trim_threshold = 256
replay, _ = _prepare(state, batch, CUDAGraphMode.PIECEWISE)
assert replay is not None and replay.rows.tolist() == REPLAY_ROWS
assert _prepare(state, batch, CUDAGraphMode.FULL) == (None, None)
def test_whole_batch_forwards_get_no_replay_batch(state):
@@ -251,9 +252,25 @@ def test_dp_ranks_replay_together(dp_state):
def test_idle_dp_rank_dummy_trims_with_its_peers(dp_state):
dummy = _input_batch([512], [512], [False])
"""An idle rank's dummy keeps one row, so peers do not wait on its replay."""
dummy = _input_batch([256, 256], [256, 256], [False, False])
dp_state.other = (True, 129)
replay, _ = _prepare(dp_state, dummy, CUDAGraphMode.NONE)
assert replay is not None and replay.rows.shape[0] == WINDOW
assert replay is not None and replay.rows.shape[0] == 1
dp_metadata = replay.forward_context.dp_metadata
assert dp_metadata.num_tokens_across_dp_cpu.tolist() == [WINDOW, 129]
assert dp_metadata.num_tokens_across_dp_cpu.tolist() == [1, 129]
def test_dp_ranks_pad_to_one_replay_graph(dp_state):
"""Every rank pads its replay batch to the graph fitting the largest rank's."""
dp_state.replay_cudagraphs = MagicMock()
dp_state.replay_cudagraphs.dispatch.return_value = BatchExecutionDescriptor(
CUDAGraphMode.PIECEWISE, 512, None
)
dp_state.other = (True, 300)
batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING)
replay, _ = _prepare(dp_state, batch, CUDAGraphMode.NONE)
assert dp_state.replay_cudagraphs.dispatch.call_args.args[:2] == (3, 300)
assert replay.forward_context.batch_descriptor.num_tokens == 512
dp_metadata = replay.forward_context.dp_metadata
assert dp_metadata.num_tokens_across_dp_cpu.tolist() == [512, 512]