[Attention][MLA][DCP] Query replication for MLA decode (DeepSeek-V2/R1 + Kimi-K2.5) (#45964)
Signed-off-by: Sungsoo Ha <sungsooh@nvidia.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Lucas Wilkinson
Matthew Bonanni
parent
97a668152b
commit
2396a61108
@@ -33,6 +33,7 @@ from torch import nn
|
||||
from transformers import DeepseekV2Config, DeepseekV3Config
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
import vllm.envs as envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, ParallelConfig, VllmConfig, get_current_vllm_config
|
||||
@@ -56,6 +57,7 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
from vllm.model_executor.layers.layernorm import LayerNorm, RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
DCPGroupColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
ReplicatedLinear,
|
||||
@@ -1015,9 +1017,17 @@ class DeepseekV2MLAAttention(nn.Module):
|
||||
prefix=f"{prefix}.kv_a_proj_with_mqa",
|
||||
)
|
||||
|
||||
qrep_enabled = (
|
||||
envs.VLLM_DCP_Q_REPLICATE
|
||||
and vllm_config.parallel_config.decode_context_parallel_size > 1
|
||||
and vllm_config.parallel_config.prefill_context_parallel_size <= 1
|
||||
)
|
||||
q_proj_cls = (
|
||||
DCPGroupColumnParallelLinear if qrep_enabled else ColumnParallelLinear
|
||||
)
|
||||
if self.q_lora_rank is not None:
|
||||
self.q_a_layernorm = RMSNorm(self.q_lora_rank, eps=config.rms_norm_eps)
|
||||
self.q_b_proj = ColumnParallelLinear(
|
||||
self.q_b_proj = q_proj_cls(
|
||||
self.q_lora_rank,
|
||||
self.num_heads * self.qk_head_dim,
|
||||
bias=False,
|
||||
@@ -1025,7 +1035,7 @@ class DeepseekV2MLAAttention(nn.Module):
|
||||
prefix=f"{prefix}.q_b_proj",
|
||||
)
|
||||
else:
|
||||
self.q_proj = ColumnParallelLinear(
|
||||
self.q_proj = q_proj_cls(
|
||||
proj_input_size,
|
||||
self.num_heads * self.qk_head_dim,
|
||||
bias=False,
|
||||
|
||||
Reference in New Issue
Block a user