[BugFix] 1D query fix for MoE models (#3597)

This commit is contained in:
Nick Hill
2024-03-24 16:00:16 -07:00
committed by GitHub
parent af9e53496f
commit 41deac4a3d
4 changed files with 15 additions and 15 deletions
+3 -3
View File
@@ -132,9 +132,9 @@ class MixtralMoE(nn.Module):
linear_method=None)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
batch_size, sequence_length, hidden_dim = hidden_states.shape
num_tokens, hidden_dim = hidden_states.shape
hidden_states = hidden_states.view(-1, hidden_dim)
# router_logits: (batch * sequence_length, n_experts)
# router_logits: (num_tokens, n_experts)
router_logits, _ = self.gate(hidden_states)
routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float)
@@ -158,7 +158,7 @@ class MixtralMoE(nn.Module):
final_hidden_states.add_(current_hidden_states)
return tensor_model_parallel_all_reduce(final_hidden_states).view(
batch_size, sequence_length, hidden_dim)
num_tokens, hidden_dim)
class MixtralAttention(nn.Module):