[Perf][DSv4.1] Run decoder replay layers in CUDA graphs and trim in PIECEWISE graphs (#59532)

Signed-off-by: Juntian Liu <Juntianl777@gmail.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Juntian Liu
2026-10-05 22:11:45 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 21d93d0d8c
commit eb2901b735
10 changed files with 445 additions and 71 deletions
+10
View File
@@ -766,6 +766,15 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
and self.layers[cut].attn.swa_cache_layer.bounded_replay
):
self.decoder_replay_start = cut + 1
# The attention metadata keys the replay layers read.
metadata_prefixes = set()
for layer in islice(self.layers, cut + 1, self.end_layer):
attn = typing.cast(DeepseekV4DecoderLayer, layer).attn
metadata_prefixes.add(attn.swa_cache_layer.prefix)
if attn.compressed_cache_prefix is not None:
metadata_prefixes.add(attn.compressed_cache_prefix)
if attn.indexer is not None:
metadata_prefixes.add(attn.indexer.k_cache.prefix)
self.decoder_replay_layers = DecoderReplayLayers(
config.sliding_window,
self._run_replay_layers,
@@ -774,6 +783,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
for buf in (self.topk_indices_buffer, self.candidate_block_buffer)
if buf is not None
],
metadata_prefixes,
)
logger.info_once(
"Decoder SWA bounded replay: in eager prefill steps, layers "