[Bugfix] Pick the DeepSeek V4 eager cudagraph region per model runner (#52401)
This commit is contained in:
@@ -298,6 +298,13 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
eager_scratch_pool=eager_scratch_pool,
|
||||
)
|
||||
|
||||
self._prepare_and_attn_fn = self._prepare_and_attn
|
||||
if not vllm_config.use_v2_model_runner:
|
||||
# MRV1's piecewise capture only tolerates the wide eager region: with
|
||||
# the narrow one the attention input preparation stays in the captured
|
||||
# graph and MRV1 produces garbage (#51430).
|
||||
self._prepare_and_attn_fn = self._prepare_and_attn_eager
|
||||
|
||||
# Will be None on ROCm for now.
|
||||
self.aux_stream_list = aux_stream_list
|
||||
# [0]: GEMM start / post-GEMM event0. [1..3]: GEMM done events;
|
||||
@@ -379,6 +386,64 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
self.eps,
|
||||
)
|
||||
|
||||
self._prepare_and_attn_fn(
|
||||
hidden_states,
|
||||
qr,
|
||||
kv,
|
||||
kv_score,
|
||||
indexer_kv_score,
|
||||
indexer_weights,
|
||||
positions,
|
||||
o_padded,
|
||||
)
|
||||
o = o_padded[:, : self.n_local_heads, :]
|
||||
|
||||
# Inverse-RoPE + wo_a + wo_b output projection (platform-specific).
|
||||
return self._o_proj(o, positions)
|
||||
|
||||
@eager_break_during_capture
|
||||
def _prepare_and_attn_eager(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
qr: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
kv_score: torch.Tensor,
|
||||
indexer_kv_score: torch.Tensor,
|
||||
indexer_weights: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
o_padded: torch.Tensor,
|
||||
) -> None:
|
||||
"""Wide eager region: the whole of ``_prepare_and_attn`` runs eagerly.
|
||||
|
||||
The nested ``_sparse_indexer_and_attn`` break runs inline, since
|
||||
``add_eager`` clears ``_capturing`` before invoking this.
|
||||
"""
|
||||
self._prepare_and_attn(
|
||||
hidden_states,
|
||||
qr,
|
||||
kv,
|
||||
kv_score,
|
||||
indexer_kv_score,
|
||||
indexer_weights,
|
||||
positions,
|
||||
o_padded,
|
||||
)
|
||||
|
||||
def _prepare_and_attn(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
qr: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
kv_score: torch.Tensor,
|
||||
indexer_kv_score: torch.Tensor,
|
||||
indexer_weights: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
o_padded: torch.Tensor,
|
||||
) -> None:
|
||||
"""Attention input preparation followed by the sparse indexer and MLA.
|
||||
|
||||
Only the latter runs in the eager break.
|
||||
"""
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
indexer = self.indexer
|
||||
compressor = self.compressor
|
||||
@@ -438,10 +503,6 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
positions,
|
||||
o_padded,
|
||||
)
|
||||
o = o_padded[:, : self.n_local_heads, :]
|
||||
|
||||
# Inverse-RoPE + wo_a + wo_b output projection (platform-specific).
|
||||
return self._o_proj(o, positions)
|
||||
|
||||
def _fused_wqa_wkv_gemm(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
# Override point: the ROCm layer preshuffles this weight in place, so
|
||||
|
||||
Reference in New Issue
Block a user