[MoE Refactor] Move the shared/fused expert output sum into MoERunnerBase (#35949)

Signed-off-by: Bill Nell <bnell@redhat.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
bnellnm
2026-04-20 12:28:46 -04:00
committed by GitHub
co-authored by Robert Shaw
parent 595562651a
commit 726efe177b
52 changed files with 325 additions and 700 deletions
+17 -76
View File
@@ -236,7 +236,6 @@ class MoETestConfig:
use_gate: bool
use_routed_input_transform: bool
enable_eplb: bool = False
reduce_results: bool = False
backend: str | None = None
ep_size: int = 1
dp_size: int = 1
@@ -295,7 +294,6 @@ def generate_valid_test_configs(
use_shared_experts,
use_gate,
use_routed_input_transform,
reduce_results,
) in product(
SHAPE_COMBOS,
NUM_EXPERTS,
@@ -304,7 +302,6 @@ def generate_valid_test_configs(
[False, True], # shared
[False, True], # gate
[False, True], # routed input exform
[False, True], # reduce results
):
config = MoETestConfig(
shape[0], # m
@@ -318,7 +315,6 @@ def generate_valid_test_configs(
use_gate,
use_routed_input_transform,
enable_eplb,
reduce_results,
backend,
ep_size,
dp_size,
@@ -395,18 +391,7 @@ def is_valid_config(config: MoETestConfig) -> tuple[bool, str | None]:
and config.backend.startswith("flashinfer_nvlink")
and not current_platform.has_device_capability(90)
):
return False, "flashinfer_nvlink needs an H100+ GPUs"
# reduce_results incompatibilities
if config.reduce_results and config.use_shared_experts:
return False, "reduce_results=True is not compatible with shared_experts=True"
if config.reduce_results and config.quantization is not None:
return (
False,
"reduce_results=True only tested with unquantized data types in "
"order to limit number of tests run",
)
return False, "flashinfer_nvlink needs H100+ GPUs"
# Backend-specific checks
if config.backend is not None:
@@ -448,10 +433,6 @@ def is_valid_config(config: MoETestConfig) -> tuple[bool, str | None]:
if config.enable_eplb and config.backend not in EPLB_SUPPORTED_BACKENDS:
return False, f"EPLB not supported with {config.backend}."
world_size = config.tp_size * config.dp_size
if config.reduce_results and world_size == 1:
return False, "reduce_results=True only makes sense for multi-GPU tests"
if (
config.backend is not None
and config.backend.startswith("flashinfer_nvlink")
@@ -846,7 +827,6 @@ def make_fused_moe_layer(
tp_size: int,
ep_size: int,
dp_size: int,
reduce_results: bool,
w1: torch.Tensor,
w2: torch.Tensor,
top_k: int,
@@ -874,7 +854,7 @@ def make_fused_moe_layer(
routed_input_transform: torch.nn.Module | None = None,
routed_output_transform: torch.nn.Module | None = None,
pcp_size: int | None = 1,
) -> tuple[Callable, FusedMoE]:
) -> FusedMoE:
quant_config, qw = make_quant_config(quantization, w1, w2, global_num_experts)
kwargs = dict()
@@ -887,8 +867,10 @@ def make_fused_moe_layer(
# Add gate and routed_input_transform if provided
if gate is not None:
kwargs["gate"] = gate
if routed_input_transform is not None:
kwargs["routed_input_transform"] = routed_input_transform
kwargs["routed_output_transform"] = routed_output_transform
layer = builder(
num_experts=global_num_experts,
@@ -896,7 +878,6 @@ def make_fused_moe_layer(
hidden_size=hidden_size,
intermediate_size=intermediate_size,
params_dtype=in_dtype,
reduce_results=reduce_results,
renormalize=renormalize,
use_grouped_topk=use_grouped_topk,
num_expert_group=num_expert_group,
@@ -936,36 +917,7 @@ def make_fused_moe_layer(
layer.quant_method.process_weights_after_loading(layer)
def _moe(
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
) -> torch.Tensor:
if shared_experts is None:
final_shared_states = None
final_hidden_states = layer(hidden_states, router_logits)
else:
final_shared_states, final_hidden_states = layer(
hidden_states, router_logits
)
# Apply routed output transform if provided
# (e.g., latent space -> original space)
if routed_output_transform is not None:
final_hidden_states = routed_output_transform(final_hidden_states)
if shared_experts is not None:
assert not reduce_results
assert final_shared_states is not None
final_hidden_states += final_shared_states
if not reduce_results and layer.tp_size > 1:
final_hidden_states = layer.maybe_all_reduce_tensor_model_parallel(
final_hidden_states
)
return final_hidden_states
return _moe, layer
return layer
def make_fake_moe_layer(
@@ -999,7 +951,6 @@ def make_fake_moe_layer(
tp_size: int = 1,
dp_size: int = 1,
ep_size: int = 1,
reduce_results: bool = False,
) -> Callable:
activation = MoEActivation.from_str(activation)
@@ -1101,7 +1052,7 @@ def make_fake_moe_layer(
def _test_body_regular(
moe_fn: Callable,
moe_layer: Callable,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
vllm_config: VllmConfig,
@@ -1118,13 +1069,12 @@ def _test_body_regular(
num_tokens=num_tokens,
num_tokens_across_dp=num_tokens_across_dp,
):
output = moe_fn(hidden_states, router_logits)
output = moe_layer(hidden_states, router_logits)
return baseline_output, output
def _test_body_eplb(
moe_fn: Callable,
moe_layer: FusedMoE,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
@@ -1145,7 +1095,6 @@ def _test_body_eplb(
n: int,
top_k: int,
shared_experts,
reduce_results: bool,
gate: torch.nn.Module | None,
routed_input_transform: torch.nn.Module | None,
routed_output_transform: torch.nn.Module | None,
@@ -1161,7 +1110,7 @@ def _test_body_eplb(
num_tokens=num_tokens,
num_tokens_across_dp=num_tokens_across_dp,
):
output_before = moe_fn(hidden_states, router_logits)
output_before = moe_layer(hidden_states, router_logits)
# Create a fresh FusedMoE layer with enable_eplb=True
# Delete the original layer's registration so the constructor can
@@ -1174,7 +1123,7 @@ def _test_body_eplb(
# When using routed_input_transform, experts operate in latent space
hidden_size_for_layer = k // 2 if routed_input_transform is not None else k
moe_fn, moe_layer = make_fused_moe_layer(
eplb_moe_layer = make_fused_moe_layer(
quantization=quantization,
use_ep=use_ep,
hidden_size=hidden_size_for_layer,
@@ -1183,7 +1132,6 @@ def _test_body_eplb(
tp_size=tp_size,
ep_size=ep_size,
dp_size=dp_size,
reduce_results=reduce_results,
w1=w1,
w2=w2,
top_k=top_k,
@@ -1196,14 +1144,14 @@ def _test_body_eplb(
)
# Necessary?
if moe_layer._expert_map is not None:
moe_layer._expert_map = moe_layer._expert_map.to(device)
if eplb_moe_layer._expert_map is not None:
eplb_moe_layer._expert_map = eplb_moe_layer._expert_map.to(device)
# All ranks must generate the same permutation
initial_indices = torch.arange(num_experts, dtype=torch.long)
shuffled_indices = initial_indices[torch.randperm(num_experts)]
expert_weights = [list(moe_layer.get_expert_weights())]
expert_weights = [list(eplb_moe_layer.get_expert_weights())]
communicator = create_eplb_communicator(
group_coordinator=get_eplb_group(),
@@ -1227,7 +1175,7 @@ def _test_body_eplb(
num_experts, dtype=torch.int32, device=device
)
moe_layer.set_eplb_state(
eplb_moe_layer.set_eplb_state(
moe_layer_idx=0,
expert_load_view=torch.zeros(
(1, num_experts),
@@ -1244,7 +1192,7 @@ def _test_body_eplb(
),
)
moe_layer.eplb_state.should_record_tensor = torch.ones(
eplb_moe_layer.eplb_state.should_record_tensor = torch.ones(
(), dtype=torch.bool, device=device
)
@@ -1255,7 +1203,7 @@ def _test_body_eplb(
num_tokens=num_tokens,
num_tokens_across_dp=num_tokens_across_dp,
):
output_after = moe_fn(hidden_states, router_logits)
output_after = eplb_moe_layer(hidden_states, router_logits)
return output_before, output_after
@@ -1274,7 +1222,6 @@ def _run_one_config(
num_experts: int,
top_k: int,
quantization: str | None,
reduce_results: bool,
backend: str | None,
test_body_fn: Callable,
use_shared_experts: bool,
@@ -1341,7 +1288,6 @@ def _run_one_config(
tp_size=tp_size,
ep_size=ep_size,
dp_size=dp_size,
reduce_results=reduce_results,
)
baseline_output = baseline_layer(hidden_states, router_logits)
@@ -1369,7 +1315,7 @@ def _run_one_config(
hidden_size_for_layer = k // 2 if routed_input_transform is not None else k
# Create initial MoE layer
moe_fn, moe_layer = make_fused_moe_layer(
moe_layer = make_fused_moe_layer(
quantization=quantization,
use_ep=use_ep,
hidden_size=hidden_size_for_layer,
@@ -1378,7 +1324,6 @@ def _run_one_config(
tp_size=tp_size,
ep_size=ep_size,
dp_size=dp_size,
reduce_results=reduce_results,
w1=w1,
w2=w2,
top_k=top_k,
@@ -1402,7 +1347,6 @@ def _run_one_config(
# Call the test body function with all necessary context
expected, actual = test_body_fn(
moe_fn=moe_fn,
moe_layer=moe_layer,
hidden_states=hidden_states,
router_logits=router_logits,
@@ -1423,7 +1367,6 @@ def _run_one_config(
m=m,
top_k=top_k,
shared_experts=shared_experts,
reduce_results=reduce_results,
gate=gate,
routed_input_transform=routed_input_transform,
routed_output_transform=routed_output_transform,
@@ -1520,7 +1463,6 @@ def test_moe_layer_no_parallel(
test_config.num_experts,
test_config.top_k,
test_config.quantization,
test_config.reduce_results,
test_config.backend,
_test_body_regular,
use_shared_experts=test_config.use_shared_experts,
@@ -1578,7 +1520,6 @@ def _parallel_worker(
test_config.num_experts,
test_config.top_k,
test_config.quantization,
test_config.reduce_results,
test_config.backend,
functools.partial(
_test_body_config, test_config=test_config, cpu_group=cpu_group
@@ -1597,7 +1538,7 @@ def _parallel_worker(
failed = failed + 1
if verbosity > 0:
traceback.print_exc()
print(f"\n{str(ex)}\nFAILED {ex.__class__}")
print(f"\n{str(ex)}\nFAILED")
else:
print("F", end="")
finally:
@@ -165,7 +165,6 @@ def test_routed_input_transform_inside_vs_outside(
top_k=top_k,
hidden_size=latent_size,
intermediate_size=intermediate_size,
reduce_results=False,
renormalize=True,
params_dtype=dtype,
tp_size=1,
@@ -183,7 +182,6 @@ def test_routed_input_transform_inside_vs_outside(
top_k=top_k,
hidden_size=latent_size,
intermediate_size=intermediate_size,
reduce_results=False,
renormalize=True,
params_dtype=dtype,
tp_size=1,
@@ -212,34 +210,20 @@ def test_routed_input_transform_inside_vs_outside(
hidden_states = torch.randn(num_tokens, hidden_size, device="cuda", dtype=dtype)
router_logits = torch.randn(num_tokens, num_experts, device="cuda", dtype=dtype)
# Clone inputs so any in-place modification by Method A
# cannot affect Method B's computation.
hidden_states_A = hidden_states.clone()
router_logits_A = router_logits.clone()
with set_forward_context(None, vllm_config, num_tokens=num_tokens):
shared_out_A, routed_out_A = moe_with_transform(
hidden_states_A, router_logits_A
)
# Method A: combined output (shared + routed)
combined_A = moe_with_transform(hidden_states, router_logits)
# Method B: manually transform, get routed output, add shared
transformed_hidden = routed_transform(hidden_states)
shared_out_B, routed_out_B = moe_without_transform(
transformed_hidden, router_logits
)
routed_out_B = moe_without_transform(transformed_hidden, router_logits)
shared_out_B = shared_experts(hidden_states)
combined_B = shared_out_B + routed_out_B
expected_shared_out = shared_experts(hidden_states)
_assert_close(
routed_out_A,
routed_out_B,
torch.testing.assert_close(
combined_A,
combined_B,
atol=1e-3,
rtol=1e-3,
label="Routed output: transform inside vs outside",
)
_assert_close(
shared_out_A,
expected_shared_out,
atol=1e-3,
rtol=1e-3,
label="Shared expert output",
msg="Combined output should match: transform inside vs outside",
)