[Perf][DSv4.1] Run decoder replay layers in CUDA graphs and trim in PIECEWISE graphs (#59532)
Signed-off-by: Juntian Liu <Juntianl777@gmail.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
21d93d0d8c
commit
eb2901b735
@@ -766,6 +766,15 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
and self.layers[cut].attn.swa_cache_layer.bounded_replay
|
||||
):
|
||||
self.decoder_replay_start = cut + 1
|
||||
# The attention metadata keys the replay layers read.
|
||||
metadata_prefixes = set()
|
||||
for layer in islice(self.layers, cut + 1, self.end_layer):
|
||||
attn = typing.cast(DeepseekV4DecoderLayer, layer).attn
|
||||
metadata_prefixes.add(attn.swa_cache_layer.prefix)
|
||||
if attn.compressed_cache_prefix is not None:
|
||||
metadata_prefixes.add(attn.compressed_cache_prefix)
|
||||
if attn.indexer is not None:
|
||||
metadata_prefixes.add(attn.indexer.k_cache.prefix)
|
||||
self.decoder_replay_layers = DecoderReplayLayers(
|
||||
config.sliding_window,
|
||||
self._run_replay_layers,
|
||||
@@ -774,6 +783,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
for buf in (self.topk_indices_buffer, self.candidate_block_buffer)
|
||||
if buf is not None
|
||||
],
|
||||
metadata_prefixes,
|
||||
)
|
||||
logger.info_once(
|
||||
"Decoder SWA bounded replay: in eager prefill steps, layers "
|
||||
|
||||
Reference in New Issue
Block a user