[Perf][DSv4.1] Run decoder replay layers in CUDA graphs and trim in PIECEWISE graphs (#59532)

Signed-off-by: Juntian Liu <Juntianl777@gmail.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Juntian Liu
2026-10-05 22:11:45 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 21d93d0d8c
commit eb2901b735
10 changed files with 445 additions and 71 deletions
+136 -32
View File
@@ -8,10 +8,17 @@ import torch
import torch.distributed as dist
import torch.nn as nn
from vllm.compilation.breakable_cudagraph import is_breakable_cudagraph_enabled
from vllm.config import CUDAGraphMode, VllmConfig
from vllm.distributed.parallel_state import get_dp_group
from vllm.forward_context import DPMetadata, create_forward_context
from vllm.models.deepseek_v41.decoder_replay_layers import DecoderReplayLayers
from vllm.distributed.parallel_state import get_dp_group, get_pp_group
from vllm.forward_context import BatchDescriptor, DPMetadata, create_forward_context
from vllm.models.deepseek_v41.decoder_replay_layers import (
DecoderReplayLayers,
ReplayBatch,
)
from vllm.models.deepseek_v41.nvidia.decoder_replay_cudagraph import (
DecoderReplayCudaGraphManager,
)
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadataBuilder
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
@@ -19,8 +26,10 @@ from vllm.v1.core.sched.output import NewRequestData
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.dp_utils import should_skip_dp_coordination
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.buffer_utils import UvaBufferPool
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
from vllm.v1.worker.gpu.model_states.interface import ModelSpecificAttnMetadata
@@ -230,6 +239,31 @@ class DeepseekV41ModelState(DefaultModelState):
self.max_num_reqs, dtype=torch.int32, device=device
)
self._replay_attn_groups: list[list[AttentionGroup]] | None = None
layers = self.decoder_replay_layers
compilation_config = vllm_config.compilation_config
self.replay_cudagraphs: DecoderReplayCudaGraphManager | None = None
# PIECEWISE graphs of this many tokens or more break out to the replay batch.
self._trim_threshold: int | None = None
if (
layers is not None
and compilation_config.cudagraph_decoder_replay
and compilation_config.cudagraph_mode.has_piecewise_cudagraphs()
and is_breakable_cudagraph_enabled()
and get_pp_group().is_first_rank
# The graphs' fixed inputs hold no LoRA mapping and no None input ids.
and vllm_config.lora_config is None
and not vllm_config.model_config.enable_prompt_embeds
):
threshold = compilation_config.decoder_replay_trim_threshold
if threshold <= layers.window:
raise ValueError(
"decoder_replay_trim_threshold must exceed the replay window "
f"of {layers.window} tokens: smaller graphs have nothing to trim"
)
self.replay_cudagraphs = DecoderReplayCudaGraphManager(
vllm_config, device, layers
)
self._trim_threshold = threshold
def add_request(self, req_index: int, new_req_data: NewRequestData) -> None:
super().add_request(req_index, new_req_data)
@@ -409,15 +443,21 @@ class DeepseekV41ModelState(DefaultModelState):
attn_groups: list[list[AttentionGroup]],
kv_cache_config: KVCacheConfig,
replay_start: torch.Tensor,
capture_desc: BatchExecutionDescriptor | None = None,
) -> None:
"""Set the replay layers' batch for this forward, or none when nothing
trims. Only eager steps trim: a CUDA graph keeps the layers on the whole
batch, and ranks share the graph mode. Under data parallelism every rank
replays if any does."""
"""Set the replay layers' batch for this forward, or none. Eager steps trim;
captures and PIECEWISE graphs at the trim threshold or above break out to a
replay batch, trimmed or not; FULL and smaller PIECEWISE graphs keep the
whole batch. Under DP every rank replays if any does."""
layers = self.decoder_replay_layers
assert layers is not None
layers.rows = layers.forward_context = None
if cudagraph_mode != CUDAGraphMode.NONE:
layers.replay_batch = None
force_replay_batch = capture_desc is not None or (
cudagraph_mode == CUDAGraphMode.PIECEWISE
and self._trim_threshold is not None
and input_batch.num_tokens_after_padding >= self._trim_threshold
)
if cudagraph_mode != CUDAGraphMode.NONE and not force_replay_batch:
return
num_reqs = input_batch.num_reqs
@@ -427,20 +467,47 @@ class DeepseekV41ModelState(DefaultModelState):
query_lens,
np.minimum(query_lens, layers.window),
)
# A batch of context-free non-prefill rows is a dummy (an idle DP rank's):
# nothing reads it and its idx_mapping hits stale slots, so keep one row.
seq_lens = input_batch.seq_lens_cpu_upper_bound[:num_reqs].numpy()
is_dummy = not input_batch.is_prefilling_np[:num_reqs].any() and bool(
(seq_lens == query_lens).all()
)
if is_dummy and capture_desc is None:
kept_lens = np.zeros_like(kept_lens)
kept_lens[0] = 1
# Dummy batches (captures, an idle DP rank's) are not prefills: they
# keep their rows unless a rank trims.
trims = bool(
needs_replay_batch = force_replay_batch or bool(
(kept_lens < query_lens)[input_batch.is_prefilling_np[:num_reqs]].any()
)
num_tokens = int(kept_lens.sum())
dp_metadata = None
dp_tokens = None
if self.vllm_config.parallel_config.data_parallel_size > 1:
trims, dp_metadata = self._agree_across_dp(trims, num_tokens)
if not trims:
needs_replay_batch, dp_tokens = self._agree_across_dp(
needs_replay_batch, num_tokens
)
if not needs_replay_batch:
return
cudagraphs, run_graph = self.replay_cudagraphs, None
num_padded, cg_mode = num_tokens, CUDAGraphMode.NONE
if cudagraphs is not None:
# Every rank runs the graph that fits the largest rank's batch.
dp_max_tokens = num_tokens if dp_tokens is None else int(dp_tokens.max())
desc = capture_desc or cudagraphs.dispatch(num_reqs, dp_max_tokens, None, 0)
if desc.cg_mode == CUDAGraphMode.PIECEWISE:
num_padded, cg_mode = desc.num_tokens, desc.cg_mode
run_graph = cudagraphs.run
if dp_tokens is not None:
dp_tokens.fill_(num_padded)
dp_metadata = None
if dp_tokens is not None:
dp_metadata = DPMetadata.make(
self.vllm_config.parallel_config, num_padded, dp_tokens
)
kept_batch, kept_slot_mappings = self._kept_input_batch(
input_batch, slot_mappings, replay_start, kept_lens
input_batch, slot_mappings, replay_start, kept_lens, num_padded
)
attn_metadata = super().prepare_attn(
kept_batch,
@@ -453,22 +520,55 @@ class DeepseekV41ModelState(DefaultModelState):
self._kept_kv_start[:num_reqs]
),
)
layers.rows = self._kept_rows[:num_tokens]
layers.forward_context = create_forward_context(
attn_metadata,
self.vllm_config,
dp_metadata=dp_metadata,
slot_mapping=build_slot_mappings_by_layer(
kept_slot_mappings, kv_cache_config
layers.replay_batch = ReplayBatch(
self._kept_rows[:num_tokens],
create_forward_context(
attn_metadata,
self.vllm_config,
dp_metadata=dp_metadata,
cudagraph_runtime_mode=cg_mode,
batch_descriptor=BatchDescriptor(num_tokens=num_padded),
slot_mapping=build_slot_mappings_by_layer(
kept_slot_mappings, kv_cache_config
),
),
run_graph,
)
def capture_inner_cudagraphs(
self,
input_buffers: InputBuffers,
block_tables: BlockTables,
attn_groups: list[list[AttentionGroup]],
kv_cache_config: KVCacheConfig,
) -> None:
def prepare(desc: BatchExecutionDescriptor) -> None:
# Capture reuses the runtime path on a dummy batch, so the graph reads
# the buffers a real step fills.
num_reqs = min(desc.num_tokens, self.max_num_reqs)
self._prepare_replay_batch(
InputBatch.make_dummy(num_reqs, desc.num_tokens, input_buffers),
CUDAGraphMode.NONE,
block_tables.get_dummy_block_tables(num_reqs),
block_tables.get_dummy_slot_mappings(desc.num_tokens),
attn_groups,
kv_cache_config,
self._replay_start[:num_reqs].zero_(),
capture_desc=desc,
)
if self.replay_cudagraphs is not None:
# profile_only captures use pre-KV-cache groups: drop the cache.
self._replay_attn_groups = None
self.replay_cudagraphs.capture_replay_graphs(prepare)
def _kept_input_batch(
self,
input_batch: InputBatch,
slot_mappings: torch.Tensor,
replay_start: torch.Tensor,
kept_lens: np.ndarray,
num_padded: int,
) -> tuple[InputBatch, torch.Tensor]:
"""Build the sub-``InputBatch`` of each request's last ``kept_lens``
rows, and its slot mappings.
@@ -486,7 +586,8 @@ class DeepseekV41ModelState(DefaultModelState):
input_batch.query_start_loc_np[: num_reqs + 1] - kept_query_start_loc_np
)
assert self._kept_slot_mappings is not None
kept_slot_mappings = self._kept_slot_mappings[:, :num_tokens]
kept_slot_mappings = self._kept_slot_mappings[:, :num_padded]
kept_slot_mappings[:, num_tokens:].fill_(PAD_SLOT_ID)
_gather_replay_batch_kernel[(num_reqs,)](
input_batch.query_start_loc,
dropped_before,
@@ -511,42 +612,45 @@ class DeepseekV41ModelState(DefaultModelState):
kept_batch = replace(
input_batch,
num_tokens=num_tokens,
num_tokens_after_padding=num_tokens,
num_tokens_after_padding=num_padded,
query_start_loc=self._kept_query_start_loc[: num_reqs + 1],
query_start_loc_np=kept_query_start_loc_np,
max_query_len=max_query_len,
positions=self._kept_positions[:num_tokens],
positions=self._kept_positions[:num_padded],
fast_prefill=None,
)
return kept_batch, kept_slot_mappings
def _agree_across_dp(
self, trims: bool, num_tokens: int
) -> tuple[bool, DPMetadata | None]:
"""Whether any rank trims and, then, the replay layers' DP metadata from
every rank's replay token count."""
self, needs_replay_batch: bool, num_tokens: int
) -> tuple[bool, torch.Tensor | None]:
"""Whether any rank needs a replay batch and, if so, each rank's token count."""
parallel_config = self.vllm_config.parallel_config
agreed = torch.zeros(2, parallel_config.data_parallel_size, dtype=torch.int32)
agreed[:, parallel_config.data_parallel_rank] = torch.tensor(
[trims, num_tokens]
[needs_replay_batch, num_tokens]
)
if not should_skip_dp_coordination():
dist.all_reduce(agreed, group=get_dp_group().cpu_group)
if not agreed[0].any():
return False, None
return True, DPMetadata.make(parallel_config, num_tokens, agreed[1])
return True, agreed[1]
def _replay_groups(
self, attn_groups: list[list[AttentionGroup]]
) -> list[list[AttentionGroup]]:
"""The attention groups with metadata builders of their own, like each
microbatch's: a builder keeps the metadata it built, and the runner's
hold the batch's."""
hold the batch's. Only the groups the replay layers read are built."""
if self._replay_attn_groups is None:
assert self.decoder_replay_layers is not None
prefixes = self.decoder_replay_layers.metadata_prefixes
self._replay_attn_groups = []
for groups in attn_groups:
replay_groups = []
for group in groups:
if prefixes.isdisjoint(group.layer_names):
continue
replay_group = replace(group, metadata_builders=[])
replay_group.create_metadata_builders(
self.vllm_config,