[MoE Refactor] FusedMoE/MoERunner inversion refactor (#41184)

Signed-off-by: Bill Nell <bnell@redhat.com>
This commit is contained in:
bnellnm
2026-06-08 10:42:58 -04:00
committed by GitHub
parent 753e9d55e6
commit dc68bd8c41
90 changed files with 2738 additions and 2031 deletions
@@ -77,9 +77,9 @@ def make_fused_moe_layer(
intermediate_size=test_config.intermediate_size,
prefix=f"dummy_layer_{layer_idx}",
activation="silu",
is_act_and_mul=True,
params_dtype=test_config.weight_dtype,
)
re = fml.routed_experts
device = torch.device(f"cuda:{rank}")
@@ -92,12 +92,12 @@ def make_fused_moe_layer(
tensor_device=device,
)
assert isinstance(fml.w13_weight.data, torch.Tensor)
assert isinstance(fml.w2_weight.data, torch.Tensor)
fml.w13_weight.data = fml.w13_weight.data.to(device=device)
fml.w2_weight.data = fml.w2_weight.data.to(device=device)
w13_weight = fml.w13_weight.data
w2_weight = fml.w2_weight.data
assert isinstance(re.w13_weight.data, torch.Tensor)
assert isinstance(re.w2_weight.data, torch.Tensor)
re.w13_weight.data = re.w13_weight.data.to(device=device)
re.w2_weight.data = re.w2_weight.data.to(device=device)
w13_weight = re.w13_weight.data
w2_weight = re.w2_weight.data
assert w13_weight.size(0) == test_config.num_local_experts
for i in range(test_config.num_local_experts):
g_i = rank * test_config.num_local_experts + i
@@ -172,10 +172,10 @@ def make_fused_moe_layer(
assert not w2_weight_scale_inv.is_contiguous()
# Add scales to the parameter list
fml.w13_weight_scale_inv = torch.nn.Parameter(
re.w13_weight_scale_inv = torch.nn.Parameter(
w13_weight_scale_inv, requires_grad=False
)
fml.w2_weight_scale_inv = torch.nn.Parameter(
re.w2_weight_scale_inv = torch.nn.Parameter(
w2_weight_scale_inv, requires_grad=False
)
@@ -61,7 +61,6 @@ def make_fused_moe_layer(
intermediate_size=test_config.intermediate_size,
prefix=f"dummy_layer_{layer_idx}",
activation="silu",
is_act_and_mul=True,
params_dtype=torch.bfloat16,
quant_config=quant_config,
)
@@ -77,6 +76,7 @@ def make_fused_moe_layer(
)
fml = fml.to(device)
re = fml.routed_experts
w1_q, w2_q, quant_config = make_test_quant_config(
test_config.num_local_experts,
test_config.intermediate_size,
@@ -87,21 +87,21 @@ def make_fused_moe_layer(
per_act_token_quant=False,
)
fml.w13_weight.data = w1_q
fml.w2_weight.data = w2_q
re.w13_weight.data = w1_q
re.w2_weight.data = w2_q
fml.w2_input_scale.data = torch.randn_like(fml.w2_input_scale.data) / 5
fml.w13_input_scale.data = torch.randn_like(fml.w13_input_scale.data) / 5
fml.w2_weight_scale_2.data = torch.randn_like(fml.w2_weight_scale_2.data) / 5
fml.w13_weight_scale_2.data = torch.randn_like(fml.w13_weight_scale_2.data) / 5
fml.w2_weight_scale.data = (
torch.randn(fml.w2_weight_scale.data.shape, device=device) / 5
).to(fml.w2_weight_scale.data.dtype)
fml.w13_weight_scale.data = (
torch.randn(fml.w13_weight_scale.data.shape, device=device) / 5
).to(fml.w13_weight_scale.data.dtype)
re.w2_input_scale.data = torch.randn_like(re.w2_input_scale.data) / 5
re.w13_input_scale.data = torch.randn_like(re.w13_input_scale.data) / 5
re.w2_weight_scale_2.data = torch.randn_like(re.w2_weight_scale_2.data) / 5
re.w13_weight_scale_2.data = torch.randn_like(re.w13_weight_scale_2.data) / 5
re.w2_weight_scale.data = (
torch.randn(re.w2_weight_scale.data.shape, device=device) / 5
).to(re.w2_weight_scale.data.dtype)
re.w13_weight_scale.data = (
torch.randn(re.w13_weight_scale.data.shape, device=device) / 5
).to(re.w13_weight_scale.data.dtype)
nvfp4_fused_moe.process_weights_after_loading(fml)
nvfp4_fused_moe.process_weights_after_loading(re)
fml.maybe_init_modular_kernel()