[Bugfix] Size and iterate w13 by shard count for non-gated MoE (#51125)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Matej Sirovatka <matej.sirovatka@gmail.com> Co-authored-by: Rishi Puri <riship@nvidia.com> Co-authored-by: Hsiao-Yuan Chen <littlecircle0730@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Matej Sirovatka
Rishi Puri
Hsiao-Yuan Chen
Claude Opus 5
parent
811622c410
commit
7c77868cdf
@@ -215,7 +215,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size // 2,
|
||||
dtype=weight_dtype,
|
||||
),
|
||||
@@ -227,7 +227,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size // mxfp4_block,
|
||||
dtype=scale_dtype,
|
||||
),
|
||||
@@ -267,7 +267,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
w13_bias = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=torch.bfloat16,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -305,13 +305,13 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
assert (
|
||||
w13.dim() == 3
|
||||
and w13.shape[0] == num_experts
|
||||
and w13.shape[1] == intermediate_size * 2
|
||||
and w13.shape[1] == intermediate_size * self.moe.w13_num_shards
|
||||
and w13.shape[2] == hidden_size // 2
|
||||
)
|
||||
assert (
|
||||
w13_scale.dim() == 3
|
||||
and w13_scale.shape[0] == num_experts
|
||||
and w13_scale.shape[1] == intermediate_size * 2
|
||||
and w13_scale.shape[1] == intermediate_size * self.moe.w13_num_shards
|
||||
and w13_scale.shape[2] == hidden_size // sf_block_size
|
||||
)
|
||||
assert (
|
||||
@@ -329,7 +329,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
assert (
|
||||
w13_bias.dim() == 2
|
||||
and w13_bias.shape[0] == num_experts
|
||||
and w13_bias.shape[1] == intermediate_size * 2
|
||||
and w13_bias.shape[1] == intermediate_size * self.moe.w13_num_shards
|
||||
)
|
||||
if w2_bias is not None:
|
||||
assert (
|
||||
@@ -599,7 +599,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size // 2,
|
||||
dtype=weight_dtype,
|
||||
),
|
||||
@@ -611,7 +611,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size // mxfp4_block,
|
||||
dtype=scale_dtype,
|
||||
),
|
||||
@@ -651,7 +651,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
w13_bias = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=torch.bfloat16,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -689,13 +689,13 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
assert (
|
||||
w13.dim() == 3
|
||||
and w13.shape[0] == num_experts
|
||||
and w13.shape[1] == intermediate_size * 2
|
||||
and w13.shape[1] == intermediate_size * self.moe.w13_num_shards
|
||||
and w13.shape[2] == hidden_size // 2
|
||||
)
|
||||
assert (
|
||||
w13_scale.dim() == 3
|
||||
and w13_scale.shape[0] == num_experts
|
||||
and w13_scale.shape[1] == intermediate_size * 2
|
||||
and w13_scale.shape[1] == intermediate_size * self.moe.w13_num_shards
|
||||
and w13_scale.shape[2] == hidden_size // sf_block_size
|
||||
)
|
||||
assert (
|
||||
@@ -713,7 +713,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
assert (
|
||||
w13_bias.dim() == 2
|
||||
and w13_bias.shape[0] == num_experts
|
||||
and w13_bias.shape[1] == intermediate_size * 2
|
||||
and w13_bias.shape[1] == intermediate_size * self.moe.w13_num_shards
|
||||
)
|
||||
if w2_bias is not None:
|
||||
assert (
|
||||
|
||||
Reference in New Issue
Block a user