[Bugfix] Keep the Moondream3 MoE all-reduce out of the fused-path try (#54152)
Signed-off-by: Oxana Korzh <okorzh@amd.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5
Douglas Lehr
parent
2aac565cae
commit
c274d36103
@@ -0,0 +1,182 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Collective-safety tests for the Moondream3 MoE layer.
|
||||
|
||||
The MoE forward may fall back from the fused kernel to a Python expert loop.
|
||||
That decision is made per rank from a rank-local exception, so every path has
|
||||
to issue exactly one all-reduce. A path that issues two (or none) desyncs the
|
||||
tensor-parallel group and wedges it for the rest of the run.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.utils import ensure_current_vllm_config
|
||||
from vllm.distributed.parallel_state import (
|
||||
destroy_distributed_environment,
|
||||
destroy_model_parallel,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.model_executor.models import moondream3
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.utils.system_utils import update_environment_variables
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
HIDDEN_SIZE = 8
|
||||
EXPERT_INNER_DIM = 16
|
||||
NUM_EXPERTS = 4
|
||||
EXPERTS_PER_TOKEN = 2
|
||||
NUM_TOKENS = 3
|
||||
|
||||
|
||||
class _StubGate(torch.nn.Module):
|
||||
"""Router stand-in, so the test does not depend on platform-specific
|
||||
``ReplicatedLinear`` weight post-processing."""
|
||||
|
||||
def __init__(self, hidden_size: int, num_experts: int):
|
||||
super().__init__()
|
||||
self.linear = torch.nn.Linear(hidden_size, num_experts)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, None]:
|
||||
return self.linear(x), None
|
||||
|
||||
|
||||
class _CountingAllReduce:
|
||||
"""Stand-in for tensor_model_parallel_all_reduce that counts calls."""
|
||||
|
||||
def __init__(self):
|
||||
self.calls = 0
|
||||
|
||||
def __call__(self, x: torch.Tensor) -> torch.Tensor:
|
||||
self.calls += 1
|
||||
return x
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tp1_cpu():
|
||||
"""A single-rank CPU tensor-parallel group, enough to build the layer."""
|
||||
update_environment_variables(
|
||||
{
|
||||
"RANK": "0",
|
||||
"LOCAL_RANK": "0",
|
||||
"WORLD_SIZE": "1",
|
||||
"MASTER_ADDR": "localhost",
|
||||
"MASTER_PORT": str(get_open_port()),
|
||||
}
|
||||
)
|
||||
init_distributed_environment(backend="gloo")
|
||||
with ensure_current_vllm_config():
|
||||
initialize_model_parallel(tensor_model_parallel_size=1)
|
||||
yield
|
||||
destroy_model_parallel()
|
||||
destroy_distributed_environment()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def moe(tp1_cpu) -> moondream3.Moondream3TextMoE:
|
||||
"""A single-rank MoE layer with deterministic weights."""
|
||||
layer = moondream3.Moondream3TextMoE(
|
||||
hidden_size=HIDDEN_SIZE,
|
||||
expert_inner_dim=EXPERT_INNER_DIM,
|
||||
num_experts=NUM_EXPERTS,
|
||||
experts_per_token=EXPERTS_PER_TOKEN,
|
||||
)
|
||||
layer.gate = _StubGate(HIDDEN_SIZE, NUM_EXPERTS)
|
||||
torch.nn.init.normal_(layer.fc1_weight, std=0.02)
|
||||
torch.nn.init.normal_(layer.fc2_weight, std=0.02)
|
||||
return layer
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def all_reduce(monkeypatch: pytest.MonkeyPatch) -> _CountingAllReduce:
|
||||
counter = _CountingAllReduce()
|
||||
monkeypatch.setattr(moondream3, "tensor_model_parallel_all_reduce", counter)
|
||||
return counter
|
||||
|
||||
|
||||
def test_expert_loop_all_reduces_once(moe, all_reduce):
|
||||
"""The Python expert loop issues exactly one all-reduce."""
|
||||
moe._use_fused_moe = False
|
||||
x = torch.randn(NUM_TOKENS, HIDDEN_SIZE)
|
||||
|
||||
out = moe(x)
|
||||
|
||||
assert out.shape == x.shape
|
||||
assert all_reduce.calls == 1
|
||||
|
||||
|
||||
def test_fused_path_all_reduces_once(moe, all_reduce, monkeypatch):
|
||||
"""The fused kernel path issues exactly one all-reduce."""
|
||||
monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
|
||||
monkeypatch.setattr(
|
||||
moondream3,
|
||||
"fused_experts",
|
||||
lambda **kwargs: torch.zeros_like(kwargs["hidden_states"]),
|
||||
)
|
||||
x = torch.randn(NUM_TOKENS, HIDDEN_SIZE)
|
||||
|
||||
out = moe(x)
|
||||
|
||||
assert out.shape == x.shape
|
||||
assert all_reduce.calls == 1
|
||||
|
||||
|
||||
def test_all_reduce_failure_is_not_retried(moe, monkeypatch):
|
||||
"""A failing all-reduce must propagate, not trigger a second collective.
|
||||
|
||||
Regression test. The all-reduce used to sit inside the ``try`` guarding
|
||||
the fused kernel, so a ``RuntimeError`` raised by the collective itself
|
||||
was caught, and this rank went on to run the fallback loop and all-reduce
|
||||
a *second* time -- two collectives on one rank against one on each of its
|
||||
peers, which desyncs the tensor-parallel group and wedges it for good.
|
||||
"""
|
||||
monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
|
||||
monkeypatch.setattr(
|
||||
moondream3,
|
||||
"fused_experts",
|
||||
lambda **kwargs: torch.zeros_like(kwargs["hidden_states"]),
|
||||
)
|
||||
calls = []
|
||||
|
||||
def _failing_all_reduce(x: torch.Tensor) -> torch.Tensor:
|
||||
calls.append(x)
|
||||
raise RuntimeError("collective failed")
|
||||
|
||||
monkeypatch.setattr(
|
||||
moondream3, "tensor_model_parallel_all_reduce", _failing_all_reduce
|
||||
)
|
||||
x = torch.randn(NUM_TOKENS, HIDDEN_SIZE)
|
||||
|
||||
with pytest.raises(RuntimeError, match="collective failed"):
|
||||
moe(x)
|
||||
|
||||
assert len(calls) == 1, (
|
||||
"the failed all-reduce was retried on the fallback path; peers issued "
|
||||
"one collective and this rank issued two"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("exc", [RuntimeError("boom"), NotImplementedError()])
|
||||
def test_fallback_after_fused_failure_all_reduces_once(
|
||||
moe, all_reduce, monkeypatch, exc
|
||||
):
|
||||
"""A rank whose fused kernel fails still all-reduces exactly once."""
|
||||
monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
|
||||
|
||||
def _raise(**kwargs):
|
||||
raise exc
|
||||
|
||||
monkeypatch.setattr(moondream3, "fused_experts", _raise)
|
||||
x = torch.randn(NUM_TOKENS, HIDDEN_SIZE)
|
||||
|
||||
out = moe(x)
|
||||
|
||||
assert out.shape == x.shape
|
||||
assert all_reduce.calls == 1
|
||||
# The fused path is disabled for subsequent forwards...
|
||||
assert moe._use_fused_moe is False
|
||||
# ...and those still all-reduce exactly once each.
|
||||
moe(x)
|
||||
assert all_reduce.calls == 2
|
||||
@@ -497,7 +497,13 @@ class Moondream3TextMoE(nn.Module):
|
||||
# Softmax over selected experts
|
||||
topk_weights = F.softmax(topk_logits, dim=-1, dtype=torch.float32).to(x.dtype)
|
||||
|
||||
out = None
|
||||
if self._use_fused_moe and x.is_cuda:
|
||||
# Only the expert computation may fall back. The all-reduce below
|
||||
# is deliberately outside this `try`: if it were inside and raised
|
||||
# on a single rank, that rank would run the fallback loop and issue
|
||||
# a *second* all-reduce while its peers issued one, desyncing the
|
||||
# TP group and hanging it for good.
|
||||
try:
|
||||
out = fused_experts(
|
||||
hidden_states=x.contiguous(),
|
||||
@@ -510,8 +516,6 @@ class Moondream3TextMoE(nn.Module):
|
||||
expert_map=self._expert_map,
|
||||
quant_config=biased_moe_quant_config(self._fused_w1_bias, None),
|
||||
)
|
||||
out = tensor_model_parallel_all_reduce(out)
|
||||
return out
|
||||
except (NotImplementedError, RuntimeError) as exc:
|
||||
self._use_fused_moe = False
|
||||
logger.warning_once(
|
||||
@@ -520,11 +524,27 @@ class Moondream3TextMoE(nn.Module):
|
||||
str(exc),
|
||||
)
|
||||
|
||||
if out is None:
|
||||
out = self._expert_loop(x, topk_weights, topk_ids)
|
||||
|
||||
# Combine each rank's partial expert outputs. Reached exactly once per
|
||||
# forward, on every rank, whichever expert path ran above.
|
||||
return tensor_model_parallel_all_reduce(out)
|
||||
|
||||
def _expert_loop(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Fallback for environments where fused kernels are unavailable.
|
||||
|
||||
Returns this rank's partial output; the caller performs the all-reduce.
|
||||
"""
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
# Compute local expert range
|
||||
local_expert_start = tp_rank * self.experts_per_rank
|
||||
|
||||
# Fallback path for environments where fused kernels are unavailable.
|
||||
out = x.new_zeros(x.shape)
|
||||
|
||||
for local_expert_idx in range(self.num_local_experts):
|
||||
@@ -558,9 +578,6 @@ class Moondream3TextMoE(nn.Module):
|
||||
# Accumulate output
|
||||
out.index_add_(0, token_pos, y)
|
||||
|
||||
# All-reduce to combine results from all experts across GPUs
|
||||
out = tensor_model_parallel_all_reduce(out)
|
||||
|
||||
return out
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user