From f8ac0c7cf0e3d4ac8894346005bdffe3bd7bd378 Mon Sep 17 00:00:00 2001 From: Xiaoshuang Wang <1790571317@qq.com> Date: Mon, 27 Apr 2026 20:57:13 +0800 Subject: [PATCH] [Bugfix] Fix k_norm weight sharding in MiniMaxM2Attention when total_num_kv_heads < tp_size (#38191) Signed-off-by: wxsIcey <1790571317@qq.com> Signed-off-by: Icey <1790571317@qq.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../layers/mamba/linear_attn.py | 37 ++++++++++++++----- vllm/model_executor/models/minimax_m2.py | 19 ++++++++-- 2 files changed, 44 insertions(+), 12 deletions(-) diff --git a/vllm/model_executor/layers/mamba/linear_attn.py b/vllm/model_executor/layers/mamba/linear_attn.py index 8e8527aed8..b277c9ed6b 100644 --- a/vllm/model_executor/layers/mamba/linear_attn.py +++ b/vllm/model_executor/layers/mamba/linear_attn.py @@ -3,6 +3,7 @@ import math from collections.abc import Callable +from functools import partial import torch import torch.nn.functional as F @@ -33,28 +34,46 @@ from vllm.v1.attention.backend import AttentionMetadata from vllm.v1.attention.backends.linear_attn import LinearAttentionMetadata +@CustomOp.register("minimax_text01_rmsnorm_tp") class MiniMaxText01RMSNormTP(CustomOp): - name = "MiniMaxText01RMSNormTP" - - def __init__(self, hidden_size: int, eps: float = 1e-6) -> None: + def __init__( + self, + hidden_size: int, + eps: float = 1e-6, + *, + weight_shard_world_size: int | None = None, + weight_shard_rank: int | None = None, + ) -> None: super().__init__() self.tp_world = get_tensor_model_parallel_world_size() self.tp_rank = get_tensor_model_parallel_rank() - self.weight = nn.Parameter(torch.ones(int(hidden_size / self.tp_world))) + self.weight_shard_world = weight_shard_world_size or self.tp_world + self.weight_shard_rank = ( + self.tp_rank if weight_shard_rank is None else weight_shard_rank + ) - self.weight.weight_loader = self.weight_loader + self.weight = nn.Parameter(torch.ones(hidden_size // self.weight_shard_world)) + self.weight.weight_loader = partial( + self.weight_loader, + shard_world_size=self.weight_shard_world, + shard_rank=self.weight_shard_rank, + ) self.variance_epsilon = eps @staticmethod def weight_loader( param: nn.Parameter, loaded_weight: torch.Tensor, + shard_world_size: int | None = None, + shard_rank: int | None = None, ) -> None: - tp_world = get_tensor_model_parallel_world_size() - tp_rank = get_tensor_model_parallel_rank() + if shard_world_size is None: + shard_world_size = get_tensor_model_parallel_world_size() + if shard_rank is None: + shard_rank = get_tensor_model_parallel_rank() - shard_size = loaded_weight.shape[0] // tp_world - shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size) + shard_size = loaded_weight.shape[0] // shard_world_size + shard = slice(shard_rank * shard_size, (shard_rank + 1) * shard_size) param.data.copy_(loaded_weight[shard]) def _forward( diff --git a/vllm/model_executor/models/minimax_m2.py b/vllm/model_executor/models/minimax_m2.py index c4a00f4101..818b7e6811 100644 --- a/vllm/model_executor/models/minimax_m2.py +++ b/vllm/model_executor/models/minimax_m2.py @@ -35,6 +35,7 @@ from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, ModelConfig, VllmConfig from vllm.distributed import ( get_pp_group, + get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size, ) from vllm.model_executor.layers.attention import Attention @@ -220,9 +221,21 @@ class MiniMaxM2Attention(nn.Module): self.q_norm = MiniMaxText01RMSNormTP( self.head_dim * self.total_num_heads, eps=rms_norm_eps ) - self.k_norm = MiniMaxText01RMSNormTP( - self.head_dim * self.total_num_kv_heads, eps=rms_norm_eps - ) + if self.total_num_kv_heads >= tp_size: + self.k_norm = MiniMaxText01RMSNormTP( + self.head_dim * self.total_num_kv_heads, eps=rms_norm_eps + ) + else: + # KV heads are replicated across TP ranks; shard k_norm weight by + # total_num_kv_heads rather than tp_size to avoid incorrect sharding. + num_kv_head_replicas = tp_size // self.total_num_kv_heads + self.k_norm = MiniMaxText01RMSNormTP( + self.head_dim * self.total_num_kv_heads, + eps=rms_norm_eps, + weight_shard_world_size=self.total_num_kv_heads, + weight_shard_rank=get_tensor_model_parallel_rank() + // num_kv_head_replicas, + ) def forward( self,