[Bugfix] Size and iterate w13 by shard count for non-gated MoE (#51125)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Matej Sirovatka <matej.sirovatka@gmail.com> Co-authored-by: Rishi Puri <riship@nvidia.com> Co-authored-by: Hsiao-Yuan Chen <littlecircle0730@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Matej Sirovatka
Rishi Puri
Hsiao-Yuan Chen
Claude Opus 5
parent
811622c410
commit
7c77868cdf
@@ -582,12 +582,11 @@ class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
hidden_size = layer.moe_config.hidden_dim_unpadded
|
||||
intermediate_size = layer.moe_config.intermediate_size_per_partition_unpadded
|
||||
|
||||
w13_half_size = layer.w13_weight.shape[1] // 2
|
||||
if w13_half_size > intermediate_size:
|
||||
layer.w13_weight[:, intermediate_size:w13_half_size, :] = 0
|
||||
layer.w13_weight[
|
||||
:, w13_half_size + intermediate_size : 2 * w13_half_size, :
|
||||
] = 0
|
||||
w13_shard = layer.w13_weight.shape[1] // self.moe.w13_num_shards
|
||||
if w13_shard > intermediate_size:
|
||||
for shard in range(self.moe.w13_num_shards):
|
||||
start = shard * w13_shard + intermediate_size
|
||||
layer.w13_weight[:, start : (shard + 1) * w13_shard, :] = 0
|
||||
if layer.w13_weight.shape[2] > hidden_size:
|
||||
layer.w13_weight[:, :, hidden_size:] = 0
|
||||
|
||||
@@ -597,12 +596,11 @@ class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
layer.w2_weight[:, :, intermediate_size:] = 0
|
||||
|
||||
if getattr(layer, "w13_bias", None) is not None:
|
||||
w13_bias_half_size = layer.w13_bias.shape[1] // 2
|
||||
if w13_bias_half_size > intermediate_size:
|
||||
layer.w13_bias[:, intermediate_size:w13_bias_half_size] = 0
|
||||
layer.w13_bias[
|
||||
:, w13_bias_half_size + intermediate_size : 2 * w13_bias_half_size
|
||||
] = 0
|
||||
w13_bias_shard = layer.w13_bias.shape[1] // self.moe.w13_num_shards
|
||||
if w13_bias_shard > intermediate_size:
|
||||
for shard in range(self.moe.w13_num_shards):
|
||||
start = shard * w13_bias_shard + intermediate_size
|
||||
layer.w13_bias[:, start : (shard + 1) * w13_bias_shard] = 0
|
||||
|
||||
if (
|
||||
getattr(layer, "w2_bias", None) is not None
|
||||
|
||||
@@ -40,7 +40,7 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size,
|
||||
device="meta",
|
||||
dtype=params_dtype,
|
||||
@@ -69,7 +69,7 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
|
||||
w13_bias = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
device="meta",
|
||||
dtype=layer.orig_dtype,
|
||||
),
|
||||
|
||||
Reference in New Issue
Block a user