[Bugfix] Fix k_norm weight sharding in MiniMaxM2Attention when total_num_kv_heads < tp_size (#38191)
Signed-off-by: wxsIcey <1790571317@qq.com> Signed-off-by: Icey <1790571317@qq.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
ebf862c351
commit
f8ac0c7cf0
@@ -35,6 +35,7 @@ from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
@@ -220,9 +221,21 @@ class MiniMaxM2Attention(nn.Module):
|
||||
self.q_norm = MiniMaxText01RMSNormTP(
|
||||
self.head_dim * self.total_num_heads, eps=rms_norm_eps
|
||||
)
|
||||
self.k_norm = MiniMaxText01RMSNormTP(
|
||||
self.head_dim * self.total_num_kv_heads, eps=rms_norm_eps
|
||||
)
|
||||
if self.total_num_kv_heads >= tp_size:
|
||||
self.k_norm = MiniMaxText01RMSNormTP(
|
||||
self.head_dim * self.total_num_kv_heads, eps=rms_norm_eps
|
||||
)
|
||||
else:
|
||||
# KV heads are replicated across TP ranks; shard k_norm weight by
|
||||
# total_num_kv_heads rather than tp_size to avoid incorrect sharding.
|
||||
num_kv_head_replicas = tp_size // self.total_num_kv_heads
|
||||
self.k_norm = MiniMaxText01RMSNormTP(
|
||||
self.head_dim * self.total_num_kv_heads,
|
||||
eps=rms_norm_eps,
|
||||
weight_shard_world_size=self.total_num_kv_heads,
|
||||
weight_shard_rank=get_tensor_model_parallel_rank()
|
||||
// num_kv_head_replicas,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user