[Bugfix] Pick the DeepSeek V4 eager cudagraph region per model runner (#52401)

This commit is contained in:
Nick Hill
2026-08-15 21:04:08 -07:00
committed by GitHub
parent 6593754e61
commit 8efa13b700
3 changed files with 95 additions and 65 deletions
+65 -4
View File
@@ -298,6 +298,13 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
eager_scratch_pool=eager_scratch_pool,
)
self._prepare_and_attn_fn = self._prepare_and_attn
if not vllm_config.use_v2_model_runner:
# MRV1's piecewise capture only tolerates the wide eager region: with
# the narrow one the attention input preparation stays in the captured
# graph and MRV1 produces garbage (#51430).
self._prepare_and_attn_fn = self._prepare_and_attn_eager
# Will be None on ROCm for now.
self.aux_stream_list = aux_stream_list
# [0]: GEMM start / post-GEMM event0. [1..3]: GEMM done events;
@@ -379,6 +386,64 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
self.eps,
)
self._prepare_and_attn_fn(
hidden_states,
qr,
kv,
kv_score,
indexer_kv_score,
indexer_weights,
positions,
o_padded,
)
o = o_padded[:, : self.n_local_heads, :]
# Inverse-RoPE + wo_a + wo_b output projection (platform-specific).
return self._o_proj(o, positions)
@eager_break_during_capture
def _prepare_and_attn_eager(
self,
hidden_states: torch.Tensor,
qr: torch.Tensor,
kv: torch.Tensor,
kv_score: torch.Tensor,
indexer_kv_score: torch.Tensor,
indexer_weights: torch.Tensor,
positions: torch.Tensor,
o_padded: torch.Tensor,
) -> None:
"""Wide eager region: the whole of ``_prepare_and_attn`` runs eagerly.
The nested ``_sparse_indexer_and_attn`` break runs inline, since
``add_eager`` clears ``_capturing`` before invoking this.
"""
self._prepare_and_attn(
hidden_states,
qr,
kv,
kv_score,
indexer_kv_score,
indexer_weights,
positions,
o_padded,
)
def _prepare_and_attn(
self,
hidden_states: torch.Tensor,
qr: torch.Tensor,
kv: torch.Tensor,
kv_score: torch.Tensor,
indexer_kv_score: torch.Tensor,
indexer_weights: torch.Tensor,
positions: torch.Tensor,
o_padded: torch.Tensor,
) -> None:
"""Attention input preparation followed by the sparse indexer and MLA.
Only the latter runs in the eager break.
"""
attn_metadata = get_forward_context().attn_metadata
indexer = self.indexer
compressor = self.compressor
@@ -438,10 +503,6 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
positions,
o_padded,
)
o = o_padded[:, : self.n_local_heads, :]
# Inverse-RoPE + wo_a + wo_b output projection (platform-specific).
return self._o_proj(o, positions)
def _fused_wqa_wkv_gemm(self, hidden_states: torch.Tensor) -> torch.Tensor:
# Override point: the ROCm layer preshuffles this weight in place, so