[Perf][DSv4.1] Run decoder replay layers in CUDA graphs and trim in PIECEWISE graphs (#59532)
Signed-off-by: Juntian Liu <Juntianl777@gmail.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
21d93d0d8c
commit
eb2901b735
@@ -2,15 +2,26 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DecoderReplayLayers runs the replay layers on the forward's replay batch."""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import CompilationConfig, CUDAGraphMode
|
||||
from vllm.forward_context import (
|
||||
BatchDescriptor,
|
||||
ForwardContext,
|
||||
get_forward_context,
|
||||
override_forward_context,
|
||||
)
|
||||
from vllm.models.deepseek_v41.decoder_replay_layers import DecoderReplayLayers
|
||||
from vllm.models.deepseek_v41.decoder_replay_layers import (
|
||||
DecoderReplayLayers,
|
||||
ReplayBatch,
|
||||
)
|
||||
from vllm.models.deepseek_v41.nvidia.decoder_replay_cudagraph import (
|
||||
DecoderReplayCudaGraphManager,
|
||||
)
|
||||
from vllm.v1.worker.gpu import cudagraph_utils
|
||||
|
||||
pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
|
||||
|
||||
@@ -32,9 +43,9 @@ def _context(metadata):
|
||||
|
||||
|
||||
def _set_replay_batch(layers, rows, metadata):
|
||||
layers.rows = torch.tensor(rows, device=DEVICE)
|
||||
layers.forward_context = ForwardContext(
|
||||
no_compile_layers={}, attn_metadata=metadata, slot_mapping={}
|
||||
layers.replay_batch = ReplayBatch(
|
||||
torch.tensor(rows, device=DEVICE),
|
||||
ForwardContext(no_compile_layers={}, attn_metadata=metadata, slot_mapping={}),
|
||||
)
|
||||
|
||||
|
||||
@@ -69,7 +80,7 @@ def test_run_gathers_states_and_realigns_shared_indexer_buffers():
|
||||
seen["is_padding"] = replay_context.is_padding
|
||||
return residual, pre_mix
|
||||
|
||||
layers = DecoderReplayLayers(WINDOW, run_layers, [topk, candidates])
|
||||
layers = DecoderReplayLayers(WINDOW, run_layers, [topk, candidates], set())
|
||||
states = _states(NUM_TOKENS)
|
||||
hidden, residual = states[0], states[-1]
|
||||
full, sub = object(), object()
|
||||
@@ -96,9 +107,54 @@ def test_no_replay_batch_runs_the_whole_batch():
|
||||
seen["attn_metadata"] = get_forward_context().attn_metadata
|
||||
return (hidden_states,)
|
||||
|
||||
layers = DecoderReplayLayers(WINDOW, run_layers, [])
|
||||
layers = DecoderReplayLayers(WINDOW, run_layers, [], set())
|
||||
states = _states(NUM_TOKENS)
|
||||
full = object()
|
||||
with override_forward_context(_context(full)):
|
||||
(output,) = layers(*states)
|
||||
assert output is states[0] and seen["attn_metadata"] is full
|
||||
|
||||
|
||||
def _graph_context(num_tokens):
|
||||
return ForwardContext(
|
||||
no_compile_layers={},
|
||||
attn_metadata={},
|
||||
slot_mapping={},
|
||||
cudagraph_runtime_mode=CUDAGraphMode.PIECEWISE,
|
||||
batch_descriptor=BatchDescriptor(num_tokens=num_tokens),
|
||||
)
|
||||
|
||||
|
||||
def test_replay_graph_matches_eager(monkeypatch):
|
||||
"""The replay graph of the next captured size pads and runs the rows."""
|
||||
monkeypatch.setattr(cudagraph_utils, "get_pp_group", MagicMock)
|
||||
capture = torch.cuda.stream(torch.cuda.Stream())
|
||||
monkeypatch.setattr(cudagraph_utils, "graph_capture", lambda device: capture)
|
||||
compilation = CompilationConfig(decoder_replay_cudagraph_capture_sizes=[4])
|
||||
cfg = MagicMock(compilation_config=compilation, speculative_config=None)
|
||||
cfg.scheduler_config = MagicMock(max_num_seqs=2, max_num_batched_tokens=64)
|
||||
cfg.cache_config.use_kda_recoverssm = False
|
||||
cfg.model_config.hf_config = MagicMock(hc_mult=HC, hidden_size=HIDDEN)
|
||||
cfg.model_config.dtype = torch.float32
|
||||
|
||||
def run_layers(hidden, positions, ids, pre, post, mix, residual):
|
||||
return residual + hidden[:, None] + positions[:, None, None], pre + ids[:, None]
|
||||
|
||||
layers = DecoderReplayLayers(4, run_layers, [], set())
|
||||
manager = DecoderReplayCudaGraphManager(cfg, DEVICE, layers)
|
||||
|
||||
def prepare(desc):
|
||||
rows = torch.arange(desc.num_tokens)
|
||||
layers.replay_batch = ReplayBatch(rows, _graph_context(desc.num_tokens))
|
||||
|
||||
manager.capture_replay_graphs(prepare)
|
||||
states = [
|
||||
torch.randn(12, *b.shape[1:], device=DEVICE).to(b.dtype)
|
||||
for b in manager.input_buffers
|
||||
]
|
||||
rows = torch.tensor([11, 2, 7], device=DEVICE)
|
||||
desc = manager.dispatch(2, 3, None, 0)
|
||||
with override_forward_context(_graph_context(desc.num_tokens)):
|
||||
actual = manager.run(rows, states)
|
||||
for a, e in zip(actual, run_layers(*(t[rows] for t in states))):
|
||||
torch.testing.assert_close(a, e, rtol=0, atol=0)
|
||||
|
||||
@@ -12,6 +12,7 @@ import torch
|
||||
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.models.deepseek_v41.nvidia.model_state import DeepseekV41ModelState
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
|
||||
|
||||
@@ -57,7 +58,8 @@ def state(monkeypatch):
|
||||
cfg.scheduler_config.max_num_batched_tokens = 1024
|
||||
cfg.parallel_config.data_parallel_size = 1
|
||||
cfg.compilation_config.fast_moe_cold_start = False
|
||||
layers = SimpleNamespace(window=WINDOW, rows=None, forward_context=None)
|
||||
cfg.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
|
||||
layers = SimpleNamespace(window=WINDOW, replay_batch=None, metadata_prefixes=set())
|
||||
model = SimpleNamespace(token_lookback_depth=0, decoder_replay_layers=layers)
|
||||
builds: list = []
|
||||
|
||||
@@ -127,13 +129,8 @@ def _prepare(state, batch, cg_mode):
|
||||
[],
|
||||
GROUPS,
|
||||
)
|
||||
layers = state.decoder_replay_layers
|
||||
if layers.rows is None:
|
||||
assert layers.forward_context is None
|
||||
return None, None
|
||||
return SimpleNamespace(
|
||||
rows=layers.rows, forward_context=layers.forward_context
|
||||
), state.builds[-1]
|
||||
replay = state.decoder_replay_layers.replay_batch
|
||||
return (None, None) if replay is None else (replay, state.builds[-1])
|
||||
|
||||
|
||||
def test_replay_batch_keeps_each_request_window(state):
|
||||
@@ -191,9 +188,9 @@ def test_replay_batch_keeps_adaptive_verification_query_bound(state):
|
||||
assert replay is not None and build.batch.max_query_len == 200
|
||||
|
||||
|
||||
def test_only_eager_steps_trim(state):
|
||||
"""A CUDA graph keeps the layers on its whole batch; an eager step trims a
|
||||
padded batch to its real rows."""
|
||||
def test_graph_steps_trim_only_piecewise_at_threshold(state):
|
||||
"""A CUDA graph keeps the layers on its whole batch, unless it is a PIECEWISE
|
||||
one at the trim threshold or above; an eager step trims a batch to its rows."""
|
||||
batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING, num_tokens_after_padding=512)
|
||||
for cg_mode in (CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL):
|
||||
assert _prepare(state, batch, cg_mode) == (None, None)
|
||||
@@ -202,6 +199,10 @@ def test_only_eager_steps_trim(state):
|
||||
assert replay.rows.tolist() == REPLAY_ROWS
|
||||
assert build.cg_mode == CUDAGraphMode.NONE
|
||||
assert build.batch.num_tokens_after_padding == len(REPLAY_ROWS)
|
||||
state._trim_threshold = 256
|
||||
replay, _ = _prepare(state, batch, CUDAGraphMode.PIECEWISE)
|
||||
assert replay is not None and replay.rows.tolist() == REPLAY_ROWS
|
||||
assert _prepare(state, batch, CUDAGraphMode.FULL) == (None, None)
|
||||
|
||||
|
||||
def test_whole_batch_forwards_get_no_replay_batch(state):
|
||||
@@ -251,9 +252,25 @@ def test_dp_ranks_replay_together(dp_state):
|
||||
|
||||
|
||||
def test_idle_dp_rank_dummy_trims_with_its_peers(dp_state):
|
||||
dummy = _input_batch([512], [512], [False])
|
||||
"""An idle rank's dummy keeps one row, so peers do not wait on its replay."""
|
||||
dummy = _input_batch([256, 256], [256, 256], [False, False])
|
||||
dp_state.other = (True, 129)
|
||||
replay, _ = _prepare(dp_state, dummy, CUDAGraphMode.NONE)
|
||||
assert replay is not None and replay.rows.shape[0] == WINDOW
|
||||
assert replay is not None and replay.rows.shape[0] == 1
|
||||
dp_metadata = replay.forward_context.dp_metadata
|
||||
assert dp_metadata.num_tokens_across_dp_cpu.tolist() == [WINDOW, 129]
|
||||
assert dp_metadata.num_tokens_across_dp_cpu.tolist() == [1, 129]
|
||||
|
||||
|
||||
def test_dp_ranks_pad_to_one_replay_graph(dp_state):
|
||||
"""Every rank pads its replay batch to the graph fitting the largest rank's."""
|
||||
dp_state.replay_cudagraphs = MagicMock()
|
||||
dp_state.replay_cudagraphs.dispatch.return_value = BatchExecutionDescriptor(
|
||||
CUDAGraphMode.PIECEWISE, 512, None
|
||||
)
|
||||
dp_state.other = (True, 300)
|
||||
batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING)
|
||||
replay, _ = _prepare(dp_state, batch, CUDAGraphMode.NONE)
|
||||
assert dp_state.replay_cudagraphs.dispatch.call_args.args[:2] == (3, 300)
|
||||
assert replay.forward_context.batch_descriptor.num_tokens == 512
|
||||
dp_metadata = replay.forward_context.dp_metadata
|
||||
assert dp_metadata.num_tokens_across_dp_cpu.tolist() == [512, 512]
|
||||
|
||||
@@ -259,7 +259,9 @@ def _make_capture_runner(captured: bool) -> GPUModelRunner:
|
||||
"""Minimal V2 runner for capture_model: fakes everything except the
|
||||
cudagraph_manager's needs_capture decision."""
|
||||
runner = GPUModelRunner.__new__(GPUModelRunner)
|
||||
runner.model_state = SimpleNamespace(supports_mm_inputs=False)
|
||||
runner.model_state = SimpleNamespace(
|
||||
supports_mm_inputs=False, capture_inner_cudagraphs=lambda *args: None
|
||||
)
|
||||
runner.cudagraph_manager = SimpleNamespace(
|
||||
needs_capture=lambda: captured,
|
||||
capture=lambda *args, **kwargs: None,
|
||||
|
||||
Reference in New Issue
Block a user