[Attention][MLA][DCP] Query replication for MLA decode (DeepSeek-V2/R1 + Kimi-K2.5) (#45964)

Signed-off-by: Sungsoo Ha <sungsooh@nvidia.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
sungsoo ha
2026-07-20 23:51:27 +00:00
committed by GitHub
co-authored by Claude Opus 4.8 Lucas Wilkinson Matthew Bonanni
parent 97a668152b
commit 2396a61108
6 changed files with 157 additions and 15 deletions
@@ -366,6 +366,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
q_lora_rank: int | None,
kv_lora_rank: int,
kv_b_proj: ColumnParallelLinear,
dcp_q_replicate: bool = False,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
@@ -384,10 +385,11 @@ class MLAAttention(nn.Module, AttentionLayerBase):
self.q_lora_rank = q_lora_rank
self.kv_lora_rank = kv_lora_rank
self.kv_b_proj = kv_b_proj
self.dcp_q_replicate = dcp_q_replicate
self.W_UK_T_dcp_qrep: torch.Tensor | None = None
self.head_size = kv_lora_rank + qk_rope_head_dim
self.layer_name = prefix
self.indexer = indexer
self.num_kv_heads = 1
self.qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim
@@ -591,6 +593,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
kv_c_normed: torch.Tensor,
k_pe: torch.Tensor,
output_shape: torch.Size | None = None,
q_dcp_replicated: torch.Tensor | None = None,
) -> torch.Tensor:
if self.calculate_kv_scales:
torch.ops.vllm.maybe_calc_kv_scales(
@@ -646,6 +649,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
self_kv_cache,
attn_metadata,
output=output,
q_dcp_replicated=q_dcp_replicated,
)
return output
else:
@@ -665,6 +669,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
output,
encoded,
kv_cache_dummy_dep=kv_cache_dummy_dep,
q_dcp_replicated=q_dcp_replicated,
)
return output
@@ -682,6 +687,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
quant_scale_ue8m0: bool | None = None,
quant_col_major: bool | None = None,
quant_tma_aligned: bool | None = None,
q_dcp_replicated: torch.Tensor | None = None,
) -> torch.Tensor:
assert output is not None, "Output tensor must be provided."
@@ -738,6 +744,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
output_padded = output
output = output[:num_actual_toks, ...]
q = q[:num_actual_toks, ...]
if q_dcp_replicated is not None:
q_dcp_replicated = q_dcp_replicated[:num_actual_toks, ...]
k_c_normed = k_c_normed[:num_actual_toks, ...]
k_pe = k_pe[:num_actual_toks, ...]
@@ -793,7 +801,12 @@ class MLAAttention(nn.Module, AttentionLayerBase):
)
if num_mqa_tokens > 0:
mqa_q = q[:num_mqa_tokens]
if q_dcp_replicated is not None:
mqa_q = q_dcp_replicated[:num_mqa_tokens]
qrep_decode = True
else:
mqa_q = q[:num_mqa_tokens]
qrep_decode = False
mqa_output_slice = output[:num_mqa_tokens]
mqa_q_nope, mqa_q_pe = mqa_q.split(
@@ -833,7 +846,9 @@ class MLAAttention(nn.Module, AttentionLayerBase):
else:
# Pads the head_dim if necessary (for the underlying kernel)
N, B, P = mqa_q_nope.shape
_, _, L = self.W_UK_T.shape
W_UK_T = self.W_UK_T_dcp_qrep if qrep_decode else self.W_UK_T
assert W_UK_T is not None
_, _, L = W_UK_T.shape
if self.q_pad_num_heads is not None:
mqa_ql_nope = mqa_q_nope.new_empty((self.q_pad_num_heads, B, L))
@@ -842,7 +857,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
mqa_ql_nope = mqa_q_nope.new_empty((N, B, L))
# Multiply (N, B, P) x (N, P, L) -> (N, B, L)
torch.bmm(mqa_q_nope, self.W_UK_T, out=mqa_ql_nope)
torch.bmm(mqa_q_nope, W_UK_T, out=mqa_ql_nope)
# Convert from (N, B, L) to (B, N, L)
mqa_ql_nope = mqa_ql_nope.transpose(0, 1)
@@ -855,6 +870,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
)
else:
mqa_q = (mqa_ql_nope, mqa_q_pe)
# concatenate nope + pe -> (B, N, L + P) (fp8 op above may have fused)
if self.impl.dcp_world_size > 1:
if self.use_pcp:
if self.impl.dcp_world_size > self.impl.pcp_world_size:
@@ -863,8 +879,11 @@ class MLAAttention(nn.Module, AttentionLayerBase):
mqa_q = get_tp_group().all_gather(mqa_q, dim=1)
else:
if isinstance(mqa_q, tuple):
# concatenate mqa_ql_nope and mqa_q_pe -> (B, N, L + P)
mqa_q = torch.cat(mqa_q, dim=-1)
mqa_q = get_dcp_group().all_gather(mqa_q, dim=1)
if not qrep_decode:
# mqa_q do allgather in head dim.
mqa_q = get_dcp_group().all_gather(mqa_q, dim=1)
# call decode attn
if not self.impl.is_sparse:
@@ -952,6 +971,21 @@ class MLAAttention(nn.Module, AttentionLayerBase):
self.kv_b_proj, out_dtype=act_dtype
).T
if self.dcp_q_replicate:
# qrep wired here: validate unsupported decode backends once.
assert self.q_pad_num_heads in (None, self.num_heads), (
"DCP query replication is unsupported on head-padding MLA "
"backends (q_pad_num_heads)."
)
if (
self.is_aiter_triton_fp4_bmm_enabled
or self.is_aiter_triton_fp8_bmm_enabled
):
raise NotImplementedError(
"DCP query replication is not implemented for the aiter "
"FP4/FP8 MLA BMM paths."
)
assert kv_b_proj_weight.shape == (
self.kv_lora_rank,
self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
@@ -1032,6 +1066,10 @@ class MLAAttention(nn.Module, AttentionLayerBase):
replace_parameter(self, "W_UV", W_UV.transpose(0, 1), prefer_copy=True)
# Convert from (L, N, P) to (N, P, L)
replace_parameter(self, "W_UK_T", W_UK.permute(1, 2, 0), prefer_copy=True)
if self.dcp_q_replicate:
self.W_UK_T_dcp_qrep = get_dcp_group().all_gather(
self.W_UK_T.contiguous(), dim=0
)
# If we should not load quant weights, we initialize the scales to 1.0
# as the default value. See [Note: Register q/k/v/prob scales in state dict]
@@ -1176,6 +1214,7 @@ def unified_mla_attention_with_output(
quant_scale_ue8m0: bool | None = None,
quant_col_major: bool | None = None,
quant_tma_aligned: bool | None = None,
q_dcp_replicated: torch.Tensor | None = None,
) -> None:
# kv_cache_dummy_dep is not used but accepting it creates a data dependency
# that ensures torch.compile preserves ordering between KV cache update and
@@ -1196,6 +1235,7 @@ def unified_mla_attention_with_output(
quant_scale_ue8m0=quant_scale_ue8m0,
quant_col_major=quant_col_major,
quant_tma_aligned=quant_tma_aligned,
q_dcp_replicated=q_dcp_replicated,
)
@@ -1212,6 +1252,7 @@ def unified_mla_attention_with_output_fake(
quant_scale_ue8m0: bool | None = None,
quant_col_major: bool | None = None,
quant_tma_aligned: bool | None = None,
q_dcp_replicated: torch.Tensor | None = None,
) -> None:
return