[Bugfix] Keep the Moondream3 MoE all-reduce out of the fused-path try (#54152)

Signed-off-by: Oxana Korzh <okorzh@amd.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
Oxana Korzh
2026-08-28 15:06:42 -07:00
committed by GitHub
co-authored by Claude Opus 5 Douglas Lehr
parent 2aac565cae
commit c274d36103
2 changed files with 205 additions and 6 deletions
+23 -6
View File
@@ -497,7 +497,13 @@ class Moondream3TextMoE(nn.Module):
# Softmax over selected experts
topk_weights = F.softmax(topk_logits, dim=-1, dtype=torch.float32).to(x.dtype)
out = None
if self._use_fused_moe and x.is_cuda:
# Only the expert computation may fall back. The all-reduce below
# is deliberately outside this `try`: if it were inside and raised
# on a single rank, that rank would run the fallback loop and issue
# a *second* all-reduce while its peers issued one, desyncing the
# TP group and hanging it for good.
try:
out = fused_experts(
hidden_states=x.contiguous(),
@@ -510,8 +516,6 @@ class Moondream3TextMoE(nn.Module):
expert_map=self._expert_map,
quant_config=biased_moe_quant_config(self._fused_w1_bias, None),
)
out = tensor_model_parallel_all_reduce(out)
return out
except (NotImplementedError, RuntimeError) as exc:
self._use_fused_moe = False
logger.warning_once(
@@ -520,11 +524,27 @@ class Moondream3TextMoE(nn.Module):
str(exc),
)
if out is None:
out = self._expert_loop(x, topk_weights, topk_ids)
# Combine each rank's partial expert outputs. Reached exactly once per
# forward, on every rank, whichever expert path ran above.
return tensor_model_parallel_all_reduce(out)
def _expert_loop(
self,
x: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
) -> torch.Tensor:
"""Fallback for environments where fused kernels are unavailable.
Returns this rank's partial output; the caller performs the all-reduce.
"""
tp_rank = get_tensor_model_parallel_rank()
# Compute local expert range
local_expert_start = tp_rank * self.experts_per_rank
# Fallback path for environments where fused kernels are unavailable.
out = x.new_zeros(x.shape)
for local_expert_idx in range(self.num_local_experts):
@@ -558,9 +578,6 @@ class Moondream3TextMoE(nn.Module):
# Accumulate output
out.index_add_(0, token_pos, y)
# All-reduce to combine results from all experts across GPUs
out = tensor_model_parallel_all_reduce(out)
return out