[Perf][DSv4.1] Run decoder replay layers in CUDA graphs and trim in PIECEWISE graphs (#59532)
Signed-off-by: Juntian Liu <Juntianl777@gmail.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
21d93d0d8c
commit
eb2901b735
@@ -7,25 +7,43 @@ in eager prefill steps they run on each request's last ``window`` rows only.
|
||||
``DeepseekV41ModelState`` prepares those rows as a sub-batch with attention
|
||||
metadata and a forward context of its own, like a microbatch;
|
||||
``DecoderReplayLayers`` gathers the layer inputs by its rows, runs the layers
|
||||
under that context and scatters the outputs back to batch rows. Steps that run
|
||||
in a CUDA graph keep the layers on the whole batch, inside the graph.
|
||||
under that context and scatters the outputs back to batch rows. FULL graphs and
|
||||
small PIECEWISE ones keep the layers on the whole batch; larger PIECEWISE graphs
|
||||
break out to the replay batch, which may run in a graph of its own
|
||||
(decoder_replay_cudagraph.py).
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.forward_context import ForwardContext, override_forward_context
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.forward_context import (
|
||||
ForwardContext,
|
||||
in_piecewise_cudagraph,
|
||||
override_forward_context,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import MoEOutput
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReplayBatch:
|
||||
"""One forward's replay batch; ``run_graph`` is set when it runs in a graph."""
|
||||
|
||||
rows: torch.Tensor
|
||||
forward_context: ForwardContext
|
||||
run_graph: Callable[..., tuple[torch.Tensor, ...]] | None = None
|
||||
|
||||
|
||||
class DecoderReplayLayers:
|
||||
"""Runs the replay layers on the step's replay batch.
|
||||
|
||||
``run_layers`` takes a batch's layer inputs and returns its per-row
|
||||
outputs. ``row_buffers`` hold per-row results the source layer's indexer
|
||||
publishes for the layers after it; they are compacted to the replay rows
|
||||
in place.
|
||||
in place. ``metadata_prefixes`` are the attention metadata keys the layers
|
||||
read, so the replay batch builds only those.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -33,32 +51,57 @@ class DecoderReplayLayers:
|
||||
window: int,
|
||||
run_layers: Callable[..., tuple[torch.Tensor, ...]],
|
||||
row_buffers: list[torch.Tensor],
|
||||
metadata_prefixes: set[str],
|
||||
) -> None:
|
||||
self.window = window
|
||||
self.run_layers = run_layers
|
||||
self.row_buffers = row_buffers
|
||||
# The replay batch, set by the model state every step: its rows of the
|
||||
# batch and its forward context. None runs the layers on the batch.
|
||||
self.rows: torch.Tensor | None = None
|
||||
self.forward_context: ForwardContext | None = None
|
||||
self.metadata_prefixes = metadata_prefixes
|
||||
# Set by the model state every step; None runs the layers on the batch.
|
||||
self.replay_batch: ReplayBatch | None = None
|
||||
self._break_outputs: list[torch.Tensor] | None = None
|
||||
|
||||
def __call__(
|
||||
self, hidden_states: torch.Tensor | MoEOutput, *states: torch.Tensor | None
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
rows = self.rows
|
||||
if rows is None:
|
||||
if self.replay_batch is not None and in_piecewise_cudagraph():
|
||||
return self._run_in_graph_break(hidden_states, *states)
|
||||
return self._run(hidden_states, *states)
|
||||
|
||||
@eager_break_during_capture
|
||||
def _run_in_graph_break(
|
||||
self, hidden_states: torch.Tensor | MoEOutput, *states: torch.Tensor | None
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
"""A graph break: the outputs land in fixed buffers the next segment reads."""
|
||||
outputs = self._run(hidden_states, *states)
|
||||
if self._break_outputs is None:
|
||||
# Sized by the first call: the largest model graph is captured first.
|
||||
self._break_outputs = [torch.empty_like(out) for out in outputs]
|
||||
return tuple(
|
||||
buf[: out.shape[0]].copy_(out)
|
||||
for buf, out in zip(self._break_outputs, outputs)
|
||||
)
|
||||
|
||||
def _run(
|
||||
self, hidden_states: torch.Tensor | MoEOutput, *states: torch.Tensor | None
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
replay_batch = self.replay_batch
|
||||
if replay_batch is None:
|
||||
return self.run_layers(hidden_states, *states)
|
||||
# A trimming step holds a prefill longer than the window, more tokens
|
||||
# than any step whose MoE leaves its finalize to the next layer.
|
||||
# Replay batch steps hold more than `window` tokens: no deferred MoE finalize.
|
||||
assert isinstance(hidden_states, torch.Tensor)
|
||||
rows = replay_batch.rows
|
||||
num_rows = rows.shape[0]
|
||||
for buf in self.row_buffers:
|
||||
buf[:num_rows].copy_(buf.index_select(0, rows))
|
||||
with override_forward_context(self.forward_context):
|
||||
row_outputs = self.run_layers(
|
||||
hidden_states.index_select(0, rows),
|
||||
*(None if t is None else t.index_select(0, rows) for t in states),
|
||||
)
|
||||
inputs = (hidden_states, *states)
|
||||
with override_forward_context(replay_batch.forward_context):
|
||||
if replay_batch.run_graph is not None:
|
||||
row_outputs = replay_batch.run_graph(rows, inputs)
|
||||
else:
|
||||
row_outputs = self.run_layers(
|
||||
*(None if t is None else t.index_select(0, rows) for t in inputs)
|
||||
)
|
||||
# The trimmed rows' outputs stay zero; nothing reads them.
|
||||
num_tokens = hidden_states.shape[0]
|
||||
return tuple(
|
||||
|
||||
Reference in New Issue
Block a user