[Perf] Narrow DeepSeek V3.2 eager CUDA graph region (#51425)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
Woosuk Kwon
2026-08-07 14:37:43 -07:00
committed by GitHub
parent c8fe1d5715
commit 46b5864054
2 changed files with 61 additions and 16 deletions
+29
View File
@@ -66,6 +66,35 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
self._fp8_kv = is_quantized_kv_cache(self.kv_cache_dtype)
self._fp8_kv_needs_view = self._fp8_kv and self.kv_cache_dtype != "fp8_ds_mla"
def forward( # type: ignore[override]
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
) -> torch.Tensor:
qkv_lora = self.fused_qkv_a_proj(hidden_states)[0]
q_c, kv_c, k_pe = qkv_lora.split(
[self.q_lora_rank, self.kv_lora_rank, self.qk_rope_head_dim], dim=-1
)
if self.indexer is not None and not self.skip_topk:
kw = self.indexer.wk_weights_proj(hidden_states)[0]
index_k = kw[:, : self.indexer.head_dim]
index_weights = kw[:, self.indexer.head_dim :]
else:
index_k = None
index_weights = None
num_tokens = hidden_states.shape[0]
output = torch.empty(
(num_tokens, self.num_local_heads * self.v_head_dim),
dtype=hidden_states.dtype,
device=hidden_states.device,
)
self._fused_attention(
positions, q_c, kv_c, k_pe, index_k, index_weights, output
)
return self.o_proj(output)[0]
def _compute_ql_nope(self, q_c: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim)
q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1)
+32 -16
View File
@@ -353,22 +353,6 @@ class DeepseekV32Attention(MLAAttention):
dtype=hidden_states.dtype,
device=hidden_states.device,
)
self._fused_attention(
positions, q_c, kv_c, k_pe, index_k, index_weights, output
)
return self.o_proj(output)[0]
@eager_break_during_capture
def _fused_attention(
self,
positions: torch.Tensor,
q_c: torch.Tensor,
kv_c: torch.Tensor,
k_pe: torch.Tensor,
index_k: torch.Tensor | None,
index_weights: torch.Tensor | None,
output: torch.Tensor,
) -> None:
forward_context = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata
if isinstance(attn_metadata_raw, dict):
@@ -462,6 +446,26 @@ class DeepseekV32Attention(MLAAttention):
quantize_mqa=self._fp8_query,
)
self._sparse_indexer_and_attn(
q_c,
index_q_fp8,
index_weights_out,
ql_nope,
mqa_q,
output,
)
return self.o_proj(output)[0]
@eager_break_during_capture
def _sparse_indexer_and_attn(
self,
q_c: torch.Tensor,
index_q_fp8: torch.Tensor | None,
index_weights_out: torch.Tensor | None,
ql_nope: torch.Tensor,
mqa_q: torch.Tensor,
output: torch.Tensor,
) -> None:
if self.indexer is not None and not self.skip_topk:
sparse_attn_indexer(
q_c,
@@ -486,6 +490,14 @@ class DeepseekV32Attention(MLAAttention):
skip_topk_buffer_clear=True,
)
attn_metadata_raw = get_forward_context().attn_metadata
if isinstance(attn_metadata_raw, dict):
attn_metadata = attn_metadata_raw.get(self.layer_name)
elif isinstance(attn_metadata_raw, list):
attn_metadata = attn_metadata_raw[0].get(self.layer_name)
else:
attn_metadata = attn_metadata_raw
if attn_metadata is None:
output.zero_()
return
@@ -504,6 +516,10 @@ class DeepseekV32Attention(MLAAttention):
attn_out, _ = self.impl.forward_mqa( # type: ignore[attr-defined]
mqa_q_arg, kv_cache, attn_metadata, self
)
# NOTE(woosuk): While the below does not need to be in the eager region,
# we put it here to avoid copying the attention output. Move this back to the
# captured region once forward_mqa supports `out` argument.
x = attn_out.view(
num_actual, self.num_local_heads, self.kv_lora_rank
).transpose(0, 1)