[Perf][DSv4.1] Run decoder replay layers in CUDA graphs and trim in PIECEWISE graphs (#59532)

Signed-off-by: Juntian Liu <Juntianl777@gmail.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Juntian Liu
2026-10-05 22:11:45 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 21d93d0d8c
commit eb2901b735
10 changed files with 445 additions and 71 deletions
@@ -0,0 +1,102 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CUDA graphs of the decoder replay layers on a step's replay batch."""
from collections.abc import Callable
from copy import copy
from dataclasses import replace
import torch
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphWrapper
from vllm.config import CUDAGraphMode, VllmConfig
from vllm.config.kernel import MEGA_MOE_BACKENDS
from vllm.forward_context import get_forward_context, override_forward_context
from vllm.models.deepseek_v41.decoder_replay_layers import DecoderReplayLayers
from vllm.v1.worker.gpu.cudagraph_utils import (
BatchExecutionDescriptor,
CudaGraphManager,
)
class DecoderReplayCudaGraphManager(CudaGraphManager):
"""Breakable PIECEWISE graphs of the replay layers keyed by the replay batch's
padded size. They read the layer inputs from static buffers."""
def __init__(self, vllm_config: VllmConfig, device, layers: DecoderReplayLayers):
compilation = copy(vllm_config.compilation_config)
scheduler = vllm_config.scheduler_config
sizes = compilation.decoder_replay_cudagraph_capture_sizes
if not sizes:
bound = min(
compilation.max_cudagraph_capture_size,
scheduler.max_num_seqs * layers.window,
)
# Replay batches hold at least `window` rows: step by window, or by 1/8
# of the power-of-two interval once wider (256 in [2048, 4096)).
sizes, s = [bound], layers.window
while s < bound:
sizes.append(s)
s += max(layers.window, 1 << max(s.bit_length() - 4, 0))
sizes = sorted(set(sizes))
if max(sizes) > scheduler.max_num_batched_tokens:
raise ValueError(
"decoder_replay_cudagraph_capture_sizes must not exceed "
"max_num_batched_tokens"
)
compilation.cudagraph_capture_sizes = sizes
compilation.max_cudagraph_capture_size = max(sizes)
replay_config = copy(vllm_config)
replay_config.compilation_config = compilation
super().__init__(replay_config, device, CUDAGraphMode.PIECEWISE, 1)
self.layers = layers
self.breakable_cg_runner: BreakableCUDAGraphWrapper = BreakableCUDAGraphWrapper(
layers.run_layers, replay_config
)
config, act = vllm_config.model_config.hf_config, vllm_config.model_config.dtype
assert isinstance(act, torch.dtype)
hc, h, f32 = config.hc_mult, config.hidden_size, torch.float32
mega_moe = vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS
ids = torch.int64 if mega_moe else torch.int32
# hidden_states, positions, input_ids, pre_mix, post_mix, res_mix, residual
shapes = [(h,), (), (), (hc,), (hc, 1), (hc, hc), (hc, h)]
dtypes = [act, torch.int64, ids, f32, f32, f32, act]
n = compilation.max_cudagraph_capture_size
self.input_buffers = [
torch.zeros(n, *s, dtype=d, device=device) for s, d in zip(shapes, dtypes)
]
def capture_replay_graphs(
self, prepare: Callable[[BatchExecutionDescriptor], None]
) -> None:
"""Capture each size on the replay batch ``prepare`` sets on the layers."""
def create_forward_fn(desc: BatchExecutionDescriptor, warmup: bool):
prepare(desc)
assert self.layers.replay_batch is not None
context = self.layers.replay_batch.forward_context
inputs = [buf[: desc.num_tokens] for buf in self.input_buffers]
def forward_fn(cg_mode: CUDAGraphMode) -> None:
with override_forward_context(
replace(context, cudagraph_runtime_mode=cg_mode)
):
self.breakable_cg_runner(*inputs)
return forward_fn
self.capture(create_forward_fn, "Capturing decoder replay CUDA graphs")
self.layers.replay_batch = None
def run(self, rows: torch.Tensor, inputs: tuple) -> tuple[torch.Tensor, ...]:
"""Replay the forward context's graph on ``rows`` of ``inputs``."""
num_rows = rows.shape[0]
for buf, t in zip(self.input_buffers, inputs):
torch.index_select(t, 0, rows, out=buf[:num_rows])
batch_descriptor = get_forward_context().batch_descriptor
assert batch_descriptor is not None
num_tokens = batch_descriptor.num_tokens
outputs = self.breakable_cg_runner(
*(buf[:num_tokens] for buf in self.input_buffers)
)
return tuple(out[:num_rows] for out in outputs)
+10
View File
@@ -766,6 +766,15 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
and self.layers[cut].attn.swa_cache_layer.bounded_replay
):
self.decoder_replay_start = cut + 1
# The attention metadata keys the replay layers read.
metadata_prefixes = set()
for layer in islice(self.layers, cut + 1, self.end_layer):
attn = typing.cast(DeepseekV4DecoderLayer, layer).attn
metadata_prefixes.add(attn.swa_cache_layer.prefix)
if attn.compressed_cache_prefix is not None:
metadata_prefixes.add(attn.compressed_cache_prefix)
if attn.indexer is not None:
metadata_prefixes.add(attn.indexer.k_cache.prefix)
self.decoder_replay_layers = DecoderReplayLayers(
config.sliding_window,
self._run_replay_layers,
@@ -774,6 +783,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
for buf in (self.topk_indices_buffer, self.candidate_block_buffer)
if buf is not None
],
metadata_prefixes,
)
logger.info_once(
"Decoder SWA bounded replay: in eager prefill steps, layers "
+136 -32
View File
@@ -8,10 +8,17 @@ import torch
import torch.distributed as dist
import torch.nn as nn
from vllm.compilation.breakable_cudagraph import is_breakable_cudagraph_enabled
from vllm.config import CUDAGraphMode, VllmConfig
from vllm.distributed.parallel_state import get_dp_group
from vllm.forward_context import DPMetadata, create_forward_context
from vllm.models.deepseek_v41.decoder_replay_layers import DecoderReplayLayers
from vllm.distributed.parallel_state import get_dp_group, get_pp_group
from vllm.forward_context import BatchDescriptor, DPMetadata, create_forward_context
from vllm.models.deepseek_v41.decoder_replay_layers import (
DecoderReplayLayers,
ReplayBatch,
)
from vllm.models.deepseek_v41.nvidia.decoder_replay_cudagraph import (
DecoderReplayCudaGraphManager,
)
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadataBuilder
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
@@ -19,8 +26,10 @@ from vllm.v1.core.sched.output import NewRequestData
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.dp_utils import should_skip_dp_coordination
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.buffer_utils import UvaBufferPool
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
from vllm.v1.worker.gpu.model_states.interface import ModelSpecificAttnMetadata
@@ -230,6 +239,31 @@ class DeepseekV41ModelState(DefaultModelState):
self.max_num_reqs, dtype=torch.int32, device=device
)
self._replay_attn_groups: list[list[AttentionGroup]] | None = None
layers = self.decoder_replay_layers
compilation_config = vllm_config.compilation_config
self.replay_cudagraphs: DecoderReplayCudaGraphManager | None = None
# PIECEWISE graphs of this many tokens or more break out to the replay batch.
self._trim_threshold: int | None = None
if (
layers is not None
and compilation_config.cudagraph_decoder_replay
and compilation_config.cudagraph_mode.has_piecewise_cudagraphs()
and is_breakable_cudagraph_enabled()
and get_pp_group().is_first_rank
# The graphs' fixed inputs hold no LoRA mapping and no None input ids.
and vllm_config.lora_config is None
and not vllm_config.model_config.enable_prompt_embeds
):
threshold = compilation_config.decoder_replay_trim_threshold
if threshold <= layers.window:
raise ValueError(
"decoder_replay_trim_threshold must exceed the replay window "
f"of {layers.window} tokens: smaller graphs have nothing to trim"
)
self.replay_cudagraphs = DecoderReplayCudaGraphManager(
vllm_config, device, layers
)
self._trim_threshold = threshold
def add_request(self, req_index: int, new_req_data: NewRequestData) -> None:
super().add_request(req_index, new_req_data)
@@ -409,15 +443,21 @@ class DeepseekV41ModelState(DefaultModelState):
attn_groups: list[list[AttentionGroup]],
kv_cache_config: KVCacheConfig,
replay_start: torch.Tensor,
capture_desc: BatchExecutionDescriptor | None = None,
) -> None:
"""Set the replay layers' batch for this forward, or none when nothing
trims. Only eager steps trim: a CUDA graph keeps the layers on the whole
batch, and ranks share the graph mode. Under data parallelism every rank
replays if any does."""
"""Set the replay layers' batch for this forward, or none. Eager steps trim;
captures and PIECEWISE graphs at the trim threshold or above break out to a
replay batch, trimmed or not; FULL and smaller PIECEWISE graphs keep the
whole batch. Under DP every rank replays if any does."""
layers = self.decoder_replay_layers
assert layers is not None
layers.rows = layers.forward_context = None
if cudagraph_mode != CUDAGraphMode.NONE:
layers.replay_batch = None
force_replay_batch = capture_desc is not None or (
cudagraph_mode == CUDAGraphMode.PIECEWISE
and self._trim_threshold is not None
and input_batch.num_tokens_after_padding >= self._trim_threshold
)
if cudagraph_mode != CUDAGraphMode.NONE and not force_replay_batch:
return
num_reqs = input_batch.num_reqs
@@ -427,20 +467,47 @@ class DeepseekV41ModelState(DefaultModelState):
query_lens,
np.minimum(query_lens, layers.window),
)
# A batch of context-free non-prefill rows is a dummy (an idle DP rank's):
# nothing reads it and its idx_mapping hits stale slots, so keep one row.
seq_lens = input_batch.seq_lens_cpu_upper_bound[:num_reqs].numpy()
is_dummy = not input_batch.is_prefilling_np[:num_reqs].any() and bool(
(seq_lens == query_lens).all()
)
if is_dummy and capture_desc is None:
kept_lens = np.zeros_like(kept_lens)
kept_lens[0] = 1
# Dummy batches (captures, an idle DP rank's) are not prefills: they
# keep their rows unless a rank trims.
trims = bool(
needs_replay_batch = force_replay_batch or bool(
(kept_lens < query_lens)[input_batch.is_prefilling_np[:num_reqs]].any()
)
num_tokens = int(kept_lens.sum())
dp_metadata = None
dp_tokens = None
if self.vllm_config.parallel_config.data_parallel_size > 1:
trims, dp_metadata = self._agree_across_dp(trims, num_tokens)
if not trims:
needs_replay_batch, dp_tokens = self._agree_across_dp(
needs_replay_batch, num_tokens
)
if not needs_replay_batch:
return
cudagraphs, run_graph = self.replay_cudagraphs, None
num_padded, cg_mode = num_tokens, CUDAGraphMode.NONE
if cudagraphs is not None:
# Every rank runs the graph that fits the largest rank's batch.
dp_max_tokens = num_tokens if dp_tokens is None else int(dp_tokens.max())
desc = capture_desc or cudagraphs.dispatch(num_reqs, dp_max_tokens, None, 0)
if desc.cg_mode == CUDAGraphMode.PIECEWISE:
num_padded, cg_mode = desc.num_tokens, desc.cg_mode
run_graph = cudagraphs.run
if dp_tokens is not None:
dp_tokens.fill_(num_padded)
dp_metadata = None
if dp_tokens is not None:
dp_metadata = DPMetadata.make(
self.vllm_config.parallel_config, num_padded, dp_tokens
)
kept_batch, kept_slot_mappings = self._kept_input_batch(
input_batch, slot_mappings, replay_start, kept_lens
input_batch, slot_mappings, replay_start, kept_lens, num_padded
)
attn_metadata = super().prepare_attn(
kept_batch,
@@ -453,22 +520,55 @@ class DeepseekV41ModelState(DefaultModelState):
self._kept_kv_start[:num_reqs]
),
)
layers.rows = self._kept_rows[:num_tokens]
layers.forward_context = create_forward_context(
attn_metadata,
self.vllm_config,
dp_metadata=dp_metadata,
slot_mapping=build_slot_mappings_by_layer(
kept_slot_mappings, kv_cache_config
layers.replay_batch = ReplayBatch(
self._kept_rows[:num_tokens],
create_forward_context(
attn_metadata,
self.vllm_config,
dp_metadata=dp_metadata,
cudagraph_runtime_mode=cg_mode,
batch_descriptor=BatchDescriptor(num_tokens=num_padded),
slot_mapping=build_slot_mappings_by_layer(
kept_slot_mappings, kv_cache_config
),
),
run_graph,
)
def capture_inner_cudagraphs(
self,
input_buffers: InputBuffers,
block_tables: BlockTables,
attn_groups: list[list[AttentionGroup]],
kv_cache_config: KVCacheConfig,
) -> None:
def prepare(desc: BatchExecutionDescriptor) -> None:
# Capture reuses the runtime path on a dummy batch, so the graph reads
# the buffers a real step fills.
num_reqs = min(desc.num_tokens, self.max_num_reqs)
self._prepare_replay_batch(
InputBatch.make_dummy(num_reqs, desc.num_tokens, input_buffers),
CUDAGraphMode.NONE,
block_tables.get_dummy_block_tables(num_reqs),
block_tables.get_dummy_slot_mappings(desc.num_tokens),
attn_groups,
kv_cache_config,
self._replay_start[:num_reqs].zero_(),
capture_desc=desc,
)
if self.replay_cudagraphs is not None:
# profile_only captures use pre-KV-cache groups: drop the cache.
self._replay_attn_groups = None
self.replay_cudagraphs.capture_replay_graphs(prepare)
def _kept_input_batch(
self,
input_batch: InputBatch,
slot_mappings: torch.Tensor,
replay_start: torch.Tensor,
kept_lens: np.ndarray,
num_padded: int,
) -> tuple[InputBatch, torch.Tensor]:
"""Build the sub-``InputBatch`` of each request's last ``kept_lens``
rows, and its slot mappings.
@@ -486,7 +586,8 @@ class DeepseekV41ModelState(DefaultModelState):
input_batch.query_start_loc_np[: num_reqs + 1] - kept_query_start_loc_np
)
assert self._kept_slot_mappings is not None
kept_slot_mappings = self._kept_slot_mappings[:, :num_tokens]
kept_slot_mappings = self._kept_slot_mappings[:, :num_padded]
kept_slot_mappings[:, num_tokens:].fill_(PAD_SLOT_ID)
_gather_replay_batch_kernel[(num_reqs,)](
input_batch.query_start_loc,
dropped_before,
@@ -511,42 +612,45 @@ class DeepseekV41ModelState(DefaultModelState):
kept_batch = replace(
input_batch,
num_tokens=num_tokens,
num_tokens_after_padding=num_tokens,
num_tokens_after_padding=num_padded,
query_start_loc=self._kept_query_start_loc[: num_reqs + 1],
query_start_loc_np=kept_query_start_loc_np,
max_query_len=max_query_len,
positions=self._kept_positions[:num_tokens],
positions=self._kept_positions[:num_padded],
fast_prefill=None,
)
return kept_batch, kept_slot_mappings
def _agree_across_dp(
self, trims: bool, num_tokens: int
) -> tuple[bool, DPMetadata | None]:
"""Whether any rank trims and, then, the replay layers' DP metadata from
every rank's replay token count."""
self, needs_replay_batch: bool, num_tokens: int
) -> tuple[bool, torch.Tensor | None]:
"""Whether any rank needs a replay batch and, if so, each rank's token count."""
parallel_config = self.vllm_config.parallel_config
agreed = torch.zeros(2, parallel_config.data_parallel_size, dtype=torch.int32)
agreed[:, parallel_config.data_parallel_rank] = torch.tensor(
[trims, num_tokens]
[needs_replay_batch, num_tokens]
)
if not should_skip_dp_coordination():
dist.all_reduce(agreed, group=get_dp_group().cpu_group)
if not agreed[0].any():
return False, None
return True, DPMetadata.make(parallel_config, num_tokens, agreed[1])
return True, agreed[1]
def _replay_groups(
self, attn_groups: list[list[AttentionGroup]]
) -> list[list[AttentionGroup]]:
"""The attention groups with metadata builders of their own, like each
microbatch's: a builder keeps the metadata it built, and the runner's
hold the batch's."""
hold the batch's. Only the groups the replay layers read are built."""
if self._replay_attn_groups is None:
assert self.decoder_replay_layers is not None
prefixes = self.decoder_replay_layers.metadata_prefixes
self._replay_attn_groups = []
for groups in attn_groups:
replay_groups = []
for group in groups:
if prefixes.isdisjoint(group.layer_names):
continue
replay_group = replace(group, metadata_builders=[])
replay_group.create_metadata_builders(
self.vllm_config,