[Attention][MLA][DCP] Query replication for MLA decode (DeepSeek-V2/R1 + Kimi-K2.5) (#45964)

Signed-off-by: Sungsoo Ha <sungsooh@nvidia.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
sungsoo ha
2026-07-20 23:51:27 +00:00
committed by GitHub
co-authored by Claude Opus 4.8 Lucas Wilkinson Matthew Bonanni
parent 97a668152b
commit 2396a61108
6 changed files with 157 additions and 15 deletions
+20 -4
View File
@@ -93,6 +93,10 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
# the topk_tokens buffer written by a previous layer in the same pass.
# Refer: https://arxiv.org/abs/2603.12201 for more details.
self.skip_topk = skip_topk
# qrep is active when the query projection is a DCP-group-sharded layer
# that materializes the full group head set locally.
q_proj_layer = self.q_b_proj if self.q_lora_rank is not None else self.q_proj
self.dcp_q_replicate = getattr(q_proj_layer, "qrep_active", False)
if self.indexer is not None:
assert hasattr(self.indexer, "topk_tokens")
self.topk_tokens = self.indexer.topk_tokens
@@ -110,6 +114,7 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
quant_config=quant_config,
prefix=f"{prefix}.attn",
kv_b_proj=self.kv_b_proj,
dcp_q_replicate=self.dcp_q_replicate,
use_sparse=self.is_sparse,
indexer=self.indexer,
topk_indices_buffer=mla_modules.topk_indices_buffer,
@@ -143,7 +148,8 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
dim=-1,
)
q_c = self.q_a_layernorm(q_c)
q = self.q_b_proj(q_c)[0]
q_proj_layer = self.q_b_proj
q_proj_input = q_c
else:
assert self.kv_a_proj_with_mqa is not None, (
"kv_a_proj_with_mqa is required when q_lora_rank is None"
@@ -152,15 +158,20 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
"q_proj is required when q_lora_rank is None"
)
kv_lora = self.kv_a_proj_with_mqa(hidden_states)[0]
q = self.q_proj(hidden_states)[0]
q_proj_layer = self.q_proj
q_proj_input = hidden_states
kv_c, k_pe = kv_lora.split([self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
kv_c_normed = self.kv_a_layernorm(kv_c)
q = q.view(-1, self.num_heads, self.qk_head_dim)
# Add head dim of 1 to k_pe
k_pe = k_pe.unsqueeze(1)
q = q_proj_layer(q_proj_input)[0]
heads = self.num_heads
if self.dcp_q_replicate:
heads *= q_proj_layer.group_size
q = q.view(-1, heads, self.qk_head_dim)
if self.rotary_emb is not None:
q[..., self.qk_nope_head_dim :], k_pe = self.rotary_emb(
positions, q[..., self.qk_nope_head_dim :], k_pe
@@ -172,11 +183,16 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
if llama_4_scaling is not None:
q *= llama_4_scaling
q_dcp_replicated = None
if self.dcp_q_replicate:
q_dcp_replicated, q = q, q_proj_layer._local_view(q)
attn_out = self.mla_attn(
q,
kv_c_normed,
k_pe,
output_shape=(hidden_states.shape[0], self.num_heads * self.v_head_dim),
q_dcp_replicated=q_dcp_replicated,
)
return self.o_proj(attn_out)[0]