[Bugfix] Size and iterate w13 by shard count for non-gated MoE (#51125)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Matej Sirovatka <matej.sirovatka@gmail.com> Co-authored-by: Rishi Puri <riship@nvidia.com> Co-authored-by: Hsiao-Yuan Chen <littlecircle0730@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Matej Sirovatka
Rishi Puri
Hsiao-Yuan Chen
Claude Opus 5
parent
811622c410
commit
7c77868cdf
@@ -545,7 +545,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
@@ -571,7 +571,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
w13_bias = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=layer.orig_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -588,13 +588,16 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
# WEIGHT_SCALES
|
||||
if not self.block_quant:
|
||||
# For per-tensor quant, the scales are per expert and weight.
|
||||
w13_scale_data = torch.ones(num_experts, 2, dtype=torch.float32)
|
||||
w13_scale_data = torch.ones(
|
||||
num_experts, self.moe.w13_num_shards, dtype=torch.float32
|
||||
)
|
||||
w2_scale_data = torch.ones(num_experts, dtype=torch.float32)
|
||||
else:
|
||||
# For block quant, the scales are per block (typically 128x128).
|
||||
w13_scale_data = torch.ones(
|
||||
num_experts,
|
||||
2 * ((intermediate_size_per_partition + block_n - 1) // block_n),
|
||||
self.moe.w13_num_shards
|
||||
* ((intermediate_size_per_partition + block_n - 1) // block_n),
|
||||
(hidden_size + block_k - 1) // block_k,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
@@ -722,7 +725,11 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
if not self.block_quant:
|
||||
shard_size = layer.intermediate_size_per_partition
|
||||
w13, w13_scale = process_fp8_weight_tensor_strategy_moe(
|
||||
w13, w13_scale, shard_size, layer.local_num_experts
|
||||
w13,
|
||||
w13_scale,
|
||||
shard_size,
|
||||
layer.local_num_experts,
|
||||
is_act_and_mul=self.moe.is_act_and_mul,
|
||||
)
|
||||
|
||||
# Shuffle weights to runtime format and setup kernel.
|
||||
|
||||
Reference in New Issue
Block a user