[Bugfix] Keep the Moondream3 MoE all-reduce out of the fused-path try (#54152)
Signed-off-by: Oxana Korzh <okorzh@amd.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5
Douglas Lehr
parent
2aac565cae
commit
c274d36103
@@ -497,7 +497,13 @@ class Moondream3TextMoE(nn.Module):
|
||||
# Softmax over selected experts
|
||||
topk_weights = F.softmax(topk_logits, dim=-1, dtype=torch.float32).to(x.dtype)
|
||||
|
||||
out = None
|
||||
if self._use_fused_moe and x.is_cuda:
|
||||
# Only the expert computation may fall back. The all-reduce below
|
||||
# is deliberately outside this `try`: if it were inside and raised
|
||||
# on a single rank, that rank would run the fallback loop and issue
|
||||
# a *second* all-reduce while its peers issued one, desyncing the
|
||||
# TP group and hanging it for good.
|
||||
try:
|
||||
out = fused_experts(
|
||||
hidden_states=x.contiguous(),
|
||||
@@ -510,8 +516,6 @@ class Moondream3TextMoE(nn.Module):
|
||||
expert_map=self._expert_map,
|
||||
quant_config=biased_moe_quant_config(self._fused_w1_bias, None),
|
||||
)
|
||||
out = tensor_model_parallel_all_reduce(out)
|
||||
return out
|
||||
except (NotImplementedError, RuntimeError) as exc:
|
||||
self._use_fused_moe = False
|
||||
logger.warning_once(
|
||||
@@ -520,11 +524,27 @@ class Moondream3TextMoE(nn.Module):
|
||||
str(exc),
|
||||
)
|
||||
|
||||
if out is None:
|
||||
out = self._expert_loop(x, topk_weights, topk_ids)
|
||||
|
||||
# Combine each rank's partial expert outputs. Reached exactly once per
|
||||
# forward, on every rank, whichever expert path ran above.
|
||||
return tensor_model_parallel_all_reduce(out)
|
||||
|
||||
def _expert_loop(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Fallback for environments where fused kernels are unavailable.
|
||||
|
||||
Returns this rank's partial output; the caller performs the all-reduce.
|
||||
"""
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
# Compute local expert range
|
||||
local_expert_start = tp_rank * self.experts_per_rank
|
||||
|
||||
# Fallback path for environments where fused kernels are unavailable.
|
||||
out = x.new_zeros(x.shape)
|
||||
|
||||
for local_expert_idx in range(self.num_local_experts):
|
||||
@@ -558,9 +578,6 @@ class Moondream3TextMoE(nn.Module):
|
||||
# Accumulate output
|
||||
out.index_add_(0, token_pos, y)
|
||||
|
||||
# All-reduce to combine results from all experts across GPUs
|
||||
out = tensor_model_parallel_all_reduce(out)
|
||||
|
||||
return out
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user