[MoE Refactor] FusedMoE/MoERunner inversion refactor (#41184)
Signed-off-by: Bill Nell <bnell@redhat.com>
This commit is contained in:
@@ -77,9 +77,9 @@ def make_fused_moe_layer(
|
||||
intermediate_size=test_config.intermediate_size,
|
||||
prefix=f"dummy_layer_{layer_idx}",
|
||||
activation="silu",
|
||||
is_act_and_mul=True,
|
||||
params_dtype=test_config.weight_dtype,
|
||||
)
|
||||
re = fml.routed_experts
|
||||
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
|
||||
@@ -92,12 +92,12 @@ def make_fused_moe_layer(
|
||||
tensor_device=device,
|
||||
)
|
||||
|
||||
assert isinstance(fml.w13_weight.data, torch.Tensor)
|
||||
assert isinstance(fml.w2_weight.data, torch.Tensor)
|
||||
fml.w13_weight.data = fml.w13_weight.data.to(device=device)
|
||||
fml.w2_weight.data = fml.w2_weight.data.to(device=device)
|
||||
w13_weight = fml.w13_weight.data
|
||||
w2_weight = fml.w2_weight.data
|
||||
assert isinstance(re.w13_weight.data, torch.Tensor)
|
||||
assert isinstance(re.w2_weight.data, torch.Tensor)
|
||||
re.w13_weight.data = re.w13_weight.data.to(device=device)
|
||||
re.w2_weight.data = re.w2_weight.data.to(device=device)
|
||||
w13_weight = re.w13_weight.data
|
||||
w2_weight = re.w2_weight.data
|
||||
assert w13_weight.size(0) == test_config.num_local_experts
|
||||
for i in range(test_config.num_local_experts):
|
||||
g_i = rank * test_config.num_local_experts + i
|
||||
@@ -172,10 +172,10 @@ def make_fused_moe_layer(
|
||||
assert not w2_weight_scale_inv.is_contiguous()
|
||||
|
||||
# Add scales to the parameter list
|
||||
fml.w13_weight_scale_inv = torch.nn.Parameter(
|
||||
re.w13_weight_scale_inv = torch.nn.Parameter(
|
||||
w13_weight_scale_inv, requires_grad=False
|
||||
)
|
||||
fml.w2_weight_scale_inv = torch.nn.Parameter(
|
||||
re.w2_weight_scale_inv = torch.nn.Parameter(
|
||||
w2_weight_scale_inv, requires_grad=False
|
||||
)
|
||||
|
||||
|
||||
@@ -61,7 +61,6 @@ def make_fused_moe_layer(
|
||||
intermediate_size=test_config.intermediate_size,
|
||||
prefix=f"dummy_layer_{layer_idx}",
|
||||
activation="silu",
|
||||
is_act_and_mul=True,
|
||||
params_dtype=torch.bfloat16,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
@@ -77,6 +76,7 @@ def make_fused_moe_layer(
|
||||
)
|
||||
|
||||
fml = fml.to(device)
|
||||
re = fml.routed_experts
|
||||
w1_q, w2_q, quant_config = make_test_quant_config(
|
||||
test_config.num_local_experts,
|
||||
test_config.intermediate_size,
|
||||
@@ -87,21 +87,21 @@ def make_fused_moe_layer(
|
||||
per_act_token_quant=False,
|
||||
)
|
||||
|
||||
fml.w13_weight.data = w1_q
|
||||
fml.w2_weight.data = w2_q
|
||||
re.w13_weight.data = w1_q
|
||||
re.w2_weight.data = w2_q
|
||||
|
||||
fml.w2_input_scale.data = torch.randn_like(fml.w2_input_scale.data) / 5
|
||||
fml.w13_input_scale.data = torch.randn_like(fml.w13_input_scale.data) / 5
|
||||
fml.w2_weight_scale_2.data = torch.randn_like(fml.w2_weight_scale_2.data) / 5
|
||||
fml.w13_weight_scale_2.data = torch.randn_like(fml.w13_weight_scale_2.data) / 5
|
||||
fml.w2_weight_scale.data = (
|
||||
torch.randn(fml.w2_weight_scale.data.shape, device=device) / 5
|
||||
).to(fml.w2_weight_scale.data.dtype)
|
||||
fml.w13_weight_scale.data = (
|
||||
torch.randn(fml.w13_weight_scale.data.shape, device=device) / 5
|
||||
).to(fml.w13_weight_scale.data.dtype)
|
||||
re.w2_input_scale.data = torch.randn_like(re.w2_input_scale.data) / 5
|
||||
re.w13_input_scale.data = torch.randn_like(re.w13_input_scale.data) / 5
|
||||
re.w2_weight_scale_2.data = torch.randn_like(re.w2_weight_scale_2.data) / 5
|
||||
re.w13_weight_scale_2.data = torch.randn_like(re.w13_weight_scale_2.data) / 5
|
||||
re.w2_weight_scale.data = (
|
||||
torch.randn(re.w2_weight_scale.data.shape, device=device) / 5
|
||||
).to(re.w2_weight_scale.data.dtype)
|
||||
re.w13_weight_scale.data = (
|
||||
torch.randn(re.w13_weight_scale.data.shape, device=device) / 5
|
||||
).to(re.w13_weight_scale.data.dtype)
|
||||
|
||||
nvfp4_fused_moe.process_weights_after_loading(fml)
|
||||
nvfp4_fused_moe.process_weights_after_loading(re)
|
||||
|
||||
fml.maybe_init_modular_kernel()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user