From 46b58640541274334e2ee502a9c744841fb310b3 Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Fri, 7 Aug 2026 14:37:43 -0700 Subject: [PATCH] [Perf] Narrow DeepSeek V3.2 eager CUDA graph region (#51425) Signed-off-by: Woosuk Kwon --- vllm/models/deepseek_v32/amd/rocm.py | 29 ++++++++++++++++ vllm/models/deepseek_v32/attention.py | 48 ++++++++++++++++++--------- 2 files changed, 61 insertions(+), 16 deletions(-) diff --git a/vllm/models/deepseek_v32/amd/rocm.py b/vllm/models/deepseek_v32/amd/rocm.py index e1abe93571..0297fc3538 100644 --- a/vllm/models/deepseek_v32/amd/rocm.py +++ b/vllm/models/deepseek_v32/amd/rocm.py @@ -66,6 +66,35 @@ class DeepseekV32MLAAttention(DeepseekV32Attention): self._fp8_kv = is_quantized_kv_cache(self.kv_cache_dtype) self._fp8_kv_needs_view = self._fp8_kv and self.kv_cache_dtype != "fp8_ds_mla" + def forward( # type: ignore[override] + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + qkv_lora = self.fused_qkv_a_proj(hidden_states)[0] + q_c, kv_c, k_pe = qkv_lora.split( + [self.q_lora_rank, self.kv_lora_rank, self.qk_rope_head_dim], dim=-1 + ) + + if self.indexer is not None and not self.skip_topk: + kw = self.indexer.wk_weights_proj(hidden_states)[0] + index_k = kw[:, : self.indexer.head_dim] + index_weights = kw[:, self.indexer.head_dim :] + else: + index_k = None + index_weights = None + + num_tokens = hidden_states.shape[0] + output = torch.empty( + (num_tokens, self.num_local_heads * self.v_head_dim), + dtype=hidden_states.dtype, + device=hidden_states.device, + ) + self._fused_attention( + positions, q_c, kv_c, k_pe, index_k, index_weights, output + ) + return self.o_proj(output)[0] + def _compute_ql_nope(self, q_c: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim) q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1) diff --git a/vllm/models/deepseek_v32/attention.py b/vllm/models/deepseek_v32/attention.py index 6ad6ccba5d..87458316c0 100644 --- a/vllm/models/deepseek_v32/attention.py +++ b/vllm/models/deepseek_v32/attention.py @@ -353,22 +353,6 @@ class DeepseekV32Attention(MLAAttention): dtype=hidden_states.dtype, device=hidden_states.device, ) - self._fused_attention( - positions, q_c, kv_c, k_pe, index_k, index_weights, output - ) - return self.o_proj(output)[0] - - @eager_break_during_capture - def _fused_attention( - self, - positions: torch.Tensor, - q_c: torch.Tensor, - kv_c: torch.Tensor, - k_pe: torch.Tensor, - index_k: torch.Tensor | None, - index_weights: torch.Tensor | None, - output: torch.Tensor, - ) -> None: forward_context = get_forward_context() attn_metadata_raw = forward_context.attn_metadata if isinstance(attn_metadata_raw, dict): @@ -462,6 +446,26 @@ class DeepseekV32Attention(MLAAttention): quantize_mqa=self._fp8_query, ) + self._sparse_indexer_and_attn( + q_c, + index_q_fp8, + index_weights_out, + ql_nope, + mqa_q, + output, + ) + return self.o_proj(output)[0] + + @eager_break_during_capture + def _sparse_indexer_and_attn( + self, + q_c: torch.Tensor, + index_q_fp8: torch.Tensor | None, + index_weights_out: torch.Tensor | None, + ql_nope: torch.Tensor, + mqa_q: torch.Tensor, + output: torch.Tensor, + ) -> None: if self.indexer is not None and not self.skip_topk: sparse_attn_indexer( q_c, @@ -486,6 +490,14 @@ class DeepseekV32Attention(MLAAttention): skip_topk_buffer_clear=True, ) + attn_metadata_raw = get_forward_context().attn_metadata + if isinstance(attn_metadata_raw, dict): + attn_metadata = attn_metadata_raw.get(self.layer_name) + elif isinstance(attn_metadata_raw, list): + attn_metadata = attn_metadata_raw[0].get(self.layer_name) + else: + attn_metadata = attn_metadata_raw + if attn_metadata is None: output.zero_() return @@ -504,6 +516,10 @@ class DeepseekV32Attention(MLAAttention): attn_out, _ = self.impl.forward_mqa( # type: ignore[attr-defined] mqa_q_arg, kv_cache, attn_metadata, self ) + + # NOTE(woosuk): While the below does not need to be in the eager region, + # we put it here to avoid copying the attention output. Move this back to the + # captured region once forward_mqa supports `out` argument. x = attn_out.view( num_actual, self.num_local_heads, self.kv_lora_rank ).transpose(0, 1)