[Bugfix] Fix k_norm weight sharding in MiniMaxM2Attention when total_num_kv_heads < tp_size (#38191)

Signed-off-by: wxsIcey <1790571317@qq.com>
Signed-off-by: Icey <1790571317@qq.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Xiaoshuang Wang
2026-04-27 05:57:13 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent ebf862c351
commit f8ac0c7cf0
2 changed files with 44 additions and 12 deletions
+16 -3
View File
@@ -35,6 +35,7 @@ from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, ModelConfig, VllmConfig
from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.model_executor.layers.attention import Attention
@@ -220,9 +221,21 @@ class MiniMaxM2Attention(nn.Module):
self.q_norm = MiniMaxText01RMSNormTP(
self.head_dim * self.total_num_heads, eps=rms_norm_eps
)
self.k_norm = MiniMaxText01RMSNormTP(
self.head_dim * self.total_num_kv_heads, eps=rms_norm_eps
)
if self.total_num_kv_heads >= tp_size:
self.k_norm = MiniMaxText01RMSNormTP(
self.head_dim * self.total_num_kv_heads, eps=rms_norm_eps
)
else:
# KV heads are replicated across TP ranks; shard k_norm weight by
# total_num_kv_heads rather than tp_size to avoid incorrect sharding.
num_kv_head_replicas = tp_size // self.total_num_kv_heads
self.k_norm = MiniMaxText01RMSNormTP(
self.head_dim * self.total_num_kv_heads,
eps=rms_norm_eps,
weight_shard_world_size=self.total_num_kv_heads,
weight_shard_rank=get_tensor_model_parallel_rank()
// num_kv_head_replicas,
)
def forward(
self,