[Bugfix][DCP] Fix NVIDIA DeepSeek-V3.2 / GLM-5.2 fused attention (#50005)
Signed-off-by: 云挚 <ningyunxiao.nyx@antgroup.com> Signed-off-by: Mikhail Kostryukov <mike@triptrack.net> Co-authored-by: Mikhail Kostryukov <mike@triptrack.net> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Mikhail Kostryukov
Lucas Wilkinson
parent
f9d666f917
commit
2cf82bcdd1
@@ -521,29 +521,45 @@ class DeepseekV32Attention(MLAAttention):
|
||||
if isinstance(mqa_q_arg, tuple):
|
||||
mqa_q_arg = torch.cat(mqa_q_arg, dim=-1)
|
||||
mqa_q_arg = get_tp_group().all_gather(mqa_q_arg, dim=1)
|
||||
elif not self.use_pcp and self.impl.dcp_world_size > 1:
|
||||
assert self.dcp_manager is not None
|
||||
if isinstance(mqa_q_arg, tuple):
|
||||
mqa_q_arg = torch.cat(mqa_q_arg, dim=-1)
|
||||
assert self.dcp_manager.query_gather is not None
|
||||
mqa_q_arg = self.dcp_manager.query_gather(mqa_q_arg)
|
||||
attn_out, lse = self.impl.forward_mqa( # type: ignore[attr-defined]
|
||||
mqa_q_arg, kv_cache, attn_metadata, self
|
||||
)
|
||||
|
||||
if self.use_pcp and self.impl.dcp_world_size > 1:
|
||||
if self.impl.dcp_world_size > 1:
|
||||
assert lse is not None and self.dcp_manager is not None
|
||||
seq_lens = (
|
||||
attn_metadata.decode.seq_lens
|
||||
if attn_metadata.decode is not None
|
||||
else cast(torch.Tensor, attn_metadata.seq_lens)[ # type: ignore[attr-defined]
|
||||
: attn_metadata.num_decodes
|
||||
seq_lens: torch.Tensor | None
|
||||
query_start_loc: torch.Tensor | None
|
||||
if self.use_pcp:
|
||||
if attn_metadata.decode is not None:
|
||||
seq_lens = attn_metadata.decode.seq_lens
|
||||
else:
|
||||
all_seq_lens = cast(
|
||||
torch.Tensor,
|
||||
attn_metadata.seq_lens, # type: ignore[attr-defined]
|
||||
)
|
||||
seq_lens = all_seq_lens[: attn_metadata.num_decodes]
|
||||
query_start_loc = attn_metadata.query_start_loc[
|
||||
: attn_metadata.num_decodes + 1
|
||||
]
|
||||
)
|
||||
query_start_loc = attn_metadata.query_start_loc[
|
||||
: attn_metadata.num_decodes + 1
|
||||
]
|
||||
else:
|
||||
# The backend emits (0, -inf) for empty local shards, so no
|
||||
# PCP-only empty-shard metadata is needed.
|
||||
seq_lens = None
|
||||
query_start_loc = None
|
||||
attn_out = self.dcp_manager.combine(
|
||||
attn_out,
|
||||
lse,
|
||||
seq_lens=seq_lens,
|
||||
query_start_loc=query_start_loc,
|
||||
seq_lens=seq_lens, # type: ignore[arg-type]
|
||||
query_start_loc=query_start_loc, # type: ignore[arg-type]
|
||||
)
|
||||
attn_out = finalize_mla_pcp_decode(attn_out, self.num_heads)
|
||||
if self.use_pcp:
|
||||
attn_out = finalize_mla_pcp_decode(attn_out, self.num_heads)
|
||||
|
||||
# NOTE(woosuk): While the below does not need to be in the eager region,
|
||||
# we put it here to avoid copying the attention output. Move this back to the
|
||||
|
||||
@@ -181,22 +181,29 @@ def _fused_norm_rope_kernel(
|
||||
)
|
||||
return
|
||||
|
||||
if slot_mapping_ptr is None:
|
||||
if kv_out_ptr is None and kpe_out_ptr is None and index_k_out_ptr is None:
|
||||
return
|
||||
elif tl.load(slot_mapping_ptr + tok_idx) < 0:
|
||||
# Padding
|
||||
return
|
||||
|
||||
if pid == 2:
|
||||
# Q RMS norm
|
||||
# Q RMS norm. Runs for every row: under DCP a negative slot only
|
||||
# means another rank owns this token's KV slot, but the query is
|
||||
# still needed on every rank (queries are not sharded), so the
|
||||
# slot-based skip below must not gate it. Padding rows do harmless
|
||||
# row-local extra work (no position load, no cache write).
|
||||
q_block = tl.arange(0, Q_BLOCK_SIZE)
|
||||
q_mask = q_block < Q_DIM
|
||||
q_c = tl.load(q_c_ptr + tok_idx * q_c_stride + q_block, mask=q_mask, other=0.0)
|
||||
q_c_rms_w = tl.load(q_rms_norm_w_ptr + q_block, mask=q_mask)
|
||||
q_c = _rms_norm(q_c, q_c_rms_w, q_rms_eps, Q_DIM)
|
||||
tl.store(q_c_out_ptr + tok_idx * q_c_out_stride + q_block, q_c, mask=q_mask)
|
||||
elif pid == 1:
|
||||
return
|
||||
|
||||
if slot_mapping_ptr is None:
|
||||
if kv_out_ptr is None and kpe_out_ptr is None and index_k_out_ptr is None:
|
||||
return
|
||||
elif tl.load(slot_mapping_ptr + tok_idx) < 0:
|
||||
# Padding, or (under DCP) a token whose KV slot another rank owns:
|
||||
# skip the K-side norms and cache writes.
|
||||
return
|
||||
|
||||
if pid == 1:
|
||||
# KV RMS Norm + KV RoPE + MLA concat_and_cache.
|
||||
# Merged so the normed kv_c and RoPE'd k_pe can be written
|
||||
# to the MLA KV cache directly without a separate kernel.
|
||||
|
||||
Reference in New Issue
Block a user