[Bugfix][DCP] Fix NVIDIA DeepSeek-V3.2 / GLM-5.2 fused attention (#50005)

Signed-off-by: 云挚 <ningyunxiao.nyx@antgroup.com>
Signed-off-by: Mikhail Kostryukov <mike@triptrack.net>
Co-authored-by: Mikhail Kostryukov <mike@triptrack.net>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Yunxiao Ning
2026-08-31 00:30:45 -07:00
committed by GitHub
co-authored by Mikhail Kostryukov Lucas Wilkinson
parent f9d666f917
commit 2cf82bcdd1
3 changed files with 102 additions and 22 deletions
+29 -13
View File
@@ -521,29 +521,45 @@ class DeepseekV32Attention(MLAAttention):
if isinstance(mqa_q_arg, tuple):
mqa_q_arg = torch.cat(mqa_q_arg, dim=-1)
mqa_q_arg = get_tp_group().all_gather(mqa_q_arg, dim=1)
elif not self.use_pcp and self.impl.dcp_world_size > 1:
assert self.dcp_manager is not None
if isinstance(mqa_q_arg, tuple):
mqa_q_arg = torch.cat(mqa_q_arg, dim=-1)
assert self.dcp_manager.query_gather is not None
mqa_q_arg = self.dcp_manager.query_gather(mqa_q_arg)
attn_out, lse = self.impl.forward_mqa( # type: ignore[attr-defined]
mqa_q_arg, kv_cache, attn_metadata, self
)
if self.use_pcp and self.impl.dcp_world_size > 1:
if self.impl.dcp_world_size > 1:
assert lse is not None and self.dcp_manager is not None
seq_lens = (
attn_metadata.decode.seq_lens
if attn_metadata.decode is not None
else cast(torch.Tensor, attn_metadata.seq_lens)[ # type: ignore[attr-defined]
: attn_metadata.num_decodes
seq_lens: torch.Tensor | None
query_start_loc: torch.Tensor | None
if self.use_pcp:
if attn_metadata.decode is not None:
seq_lens = attn_metadata.decode.seq_lens
else:
all_seq_lens = cast(
torch.Tensor,
attn_metadata.seq_lens, # type: ignore[attr-defined]
)
seq_lens = all_seq_lens[: attn_metadata.num_decodes]
query_start_loc = attn_metadata.query_start_loc[
: attn_metadata.num_decodes + 1
]
)
query_start_loc = attn_metadata.query_start_loc[
: attn_metadata.num_decodes + 1
]
else:
# The backend emits (0, -inf) for empty local shards, so no
# PCP-only empty-shard metadata is needed.
seq_lens = None
query_start_loc = None
attn_out = self.dcp_manager.combine(
attn_out,
lse,
seq_lens=seq_lens,
query_start_loc=query_start_loc,
seq_lens=seq_lens, # type: ignore[arg-type]
query_start_loc=query_start_loc, # type: ignore[arg-type]
)
attn_out = finalize_mla_pcp_decode(attn_out, self.num_heads)
if self.use_pcp:
attn_out = finalize_mla_pcp_decode(attn_out, self.num_heads)
# NOTE(woosuk): While the below does not need to be in the eager region,
# we put it here to avoid copying the attention output. Move this back to the
+16 -9
View File
@@ -181,22 +181,29 @@ def _fused_norm_rope_kernel(
)
return
if slot_mapping_ptr is None:
if kv_out_ptr is None and kpe_out_ptr is None and index_k_out_ptr is None:
return
elif tl.load(slot_mapping_ptr + tok_idx) < 0:
# Padding
return
if pid == 2:
# Q RMS norm
# Q RMS norm. Runs for every row: under DCP a negative slot only
# means another rank owns this token's KV slot, but the query is
# still needed on every rank (queries are not sharded), so the
# slot-based skip below must not gate it. Padding rows do harmless
# row-local extra work (no position load, no cache write).
q_block = tl.arange(0, Q_BLOCK_SIZE)
q_mask = q_block < Q_DIM
q_c = tl.load(q_c_ptr + tok_idx * q_c_stride + q_block, mask=q_mask, other=0.0)
q_c_rms_w = tl.load(q_rms_norm_w_ptr + q_block, mask=q_mask)
q_c = _rms_norm(q_c, q_c_rms_w, q_rms_eps, Q_DIM)
tl.store(q_c_out_ptr + tok_idx * q_c_out_stride + q_block, q_c, mask=q_mask)
elif pid == 1:
return
if slot_mapping_ptr is None:
if kv_out_ptr is None and kpe_out_ptr is None and index_k_out_ptr is None:
return
elif tl.load(slot_mapping_ptr + tok_idx) < 0:
# Padding, or (under DCP) a token whose KV slot another rank owns:
# skip the K-side norms and cache writes.
return
if pid == 1:
# KV RMS Norm + KV RoPE + MLA concat_and_cache.
# Merged so the normed kv_c and RoPE'd k_pe can be written
# to the MLA KV cache directly without a separate kernel.