[Attention][MLA][DCP] Query replication for MLA decode (DeepSeek-V2/R1 + Kimi-K2.5) (#45964)
Signed-off-by: Sungsoo Ha <sungsooh@nvidia.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Lucas Wilkinson
Matthew Bonanni
parent
97a668152b
commit
2396a61108
@@ -93,6 +93,10 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
# the topk_tokens buffer written by a previous layer in the same pass.
|
||||
# Refer: https://arxiv.org/abs/2603.12201 for more details.
|
||||
self.skip_topk = skip_topk
|
||||
# qrep is active when the query projection is a DCP-group-sharded layer
|
||||
# that materializes the full group head set locally.
|
||||
q_proj_layer = self.q_b_proj if self.q_lora_rank is not None else self.q_proj
|
||||
self.dcp_q_replicate = getattr(q_proj_layer, "qrep_active", False)
|
||||
if self.indexer is not None:
|
||||
assert hasattr(self.indexer, "topk_tokens")
|
||||
self.topk_tokens = self.indexer.topk_tokens
|
||||
@@ -110,6 +114,7 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.attn",
|
||||
kv_b_proj=self.kv_b_proj,
|
||||
dcp_q_replicate=self.dcp_q_replicate,
|
||||
use_sparse=self.is_sparse,
|
||||
indexer=self.indexer,
|
||||
topk_indices_buffer=mla_modules.topk_indices_buffer,
|
||||
@@ -143,7 +148,8 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
dim=-1,
|
||||
)
|
||||
q_c = self.q_a_layernorm(q_c)
|
||||
q = self.q_b_proj(q_c)[0]
|
||||
q_proj_layer = self.q_b_proj
|
||||
q_proj_input = q_c
|
||||
else:
|
||||
assert self.kv_a_proj_with_mqa is not None, (
|
||||
"kv_a_proj_with_mqa is required when q_lora_rank is None"
|
||||
@@ -152,15 +158,20 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
"q_proj is required when q_lora_rank is None"
|
||||
)
|
||||
kv_lora = self.kv_a_proj_with_mqa(hidden_states)[0]
|
||||
q = self.q_proj(hidden_states)[0]
|
||||
q_proj_layer = self.q_proj
|
||||
q_proj_input = hidden_states
|
||||
|
||||
kv_c, k_pe = kv_lora.split([self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
|
||||
kv_c_normed = self.kv_a_layernorm(kv_c)
|
||||
|
||||
q = q.view(-1, self.num_heads, self.qk_head_dim)
|
||||
# Add head dim of 1 to k_pe
|
||||
k_pe = k_pe.unsqueeze(1)
|
||||
|
||||
q = q_proj_layer(q_proj_input)[0]
|
||||
heads = self.num_heads
|
||||
if self.dcp_q_replicate:
|
||||
heads *= q_proj_layer.group_size
|
||||
q = q.view(-1, heads, self.qk_head_dim)
|
||||
|
||||
if self.rotary_emb is not None:
|
||||
q[..., self.qk_nope_head_dim :], k_pe = self.rotary_emb(
|
||||
positions, q[..., self.qk_nope_head_dim :], k_pe
|
||||
@@ -172,11 +183,16 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
if llama_4_scaling is not None:
|
||||
q *= llama_4_scaling
|
||||
|
||||
q_dcp_replicated = None
|
||||
if self.dcp_q_replicate:
|
||||
q_dcp_replicated, q = q, q_proj_layer._local_view(q)
|
||||
|
||||
attn_out = self.mla_attn(
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
output_shape=(hidden_states.shape[0], self.num_heads * self.v_head_dim),
|
||||
q_dcp_replicated=q_dcp_replicated,
|
||||
)
|
||||
|
||||
return self.o_proj(attn_out)[0]
|
||||
|
||||
Reference in New Issue
Block a user