[Bugfix] Size and iterate w13 by shard count for non-gated MoE (#51125)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Matej Sirovatka <matej.sirovatka@gmail.com>
Co-authored-by: Rishi Puri <riship@nvidia.com>
Co-authored-by: Hsiao-Yuan Chen <littlecircle0730@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-08-05 17:24:47 -07:00
committed by GitHub
co-authored by Matej Sirovatka Rishi Puri Hsiao-Yuan Chen Claude Opus 5
parent 811622c410
commit 7c77868cdf
16 changed files with 104 additions and 80 deletions
@@ -216,7 +216,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
w13_weight = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
hidden_size,
dtype=params_dtype,
),
@@ -243,7 +243,10 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
# They are combined to a single scale after weight loading.
if self.model_type != "gpt_oss":
w13_weight_scale = torch.nn.Parameter(
torch.ones(num_experts, 2, dtype=torch.float32), requires_grad=False
torch.ones(
num_experts, self.moe.w13_num_shards, dtype=torch.float32
),
requires_grad=False,
)
else:
# For gpt_oss, the w1(gate) & w3(up) are fused as one.
@@ -267,7 +270,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
w13_weight_scale = torch.nn.Parameter(
torch.ones(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
dtype=torch.float32,
),
requires_grad=False,
@@ -306,7 +309,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
w13_bias = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
dtype=torch.float32,
),
requires_grad=False,
@@ -400,7 +403,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
# For non-gpt_oss, process w1 and w3 shards separately
for expert_id in range(layer.local_num_experts):
start = 0
for shard_id in range(2):
for shard_id in range(self.moe.w13_num_shards):
dq_weight = per_tensor_dequantize(
layer.w13_weight[expert_id][start : start + shard_size, :],
layer.w13_weight_scale[expert_id][shard_id],
@@ -560,7 +563,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
w13_weight = torch.nn.Parameter(
torch.empty(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
hidden_size,
dtype=params_dtype,
),
@@ -586,7 +589,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
w13_weight_scale = torch.nn.Parameter(
torch.ones(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
dtype=torch.float32,
),
requires_grad=False,
@@ -605,7 +608,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
else:
# per-tensor: one scalar per expert (two for the fused w1/w3)
w13_weight_scale = torch.nn.Parameter(
torch.ones(num_experts, 2, dtype=torch.float32),
torch.ones(num_experts, self.moe.w13_num_shards, dtype=torch.float32),
requires_grad=False,
)
layer.register_parameter("w13_weight_scale", w13_weight_scale)
@@ -644,7 +647,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
# ZERO POINTS (loaded but discarded after loading; kernel uses symmetric)
w13_input_zero_point = torch.nn.Parameter(
torch.zeros(num_experts, 2, dtype=torch.int8),
torch.zeros(num_experts, self.moe.w13_num_shards, dtype=torch.int8),
requires_grad=False,
)
layer.register_parameter("w13_input_zero_point", w13_input_zero_point)
@@ -661,7 +664,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
w13_weight_zero_point = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
dtype=torch.int8,
),
requires_grad=False,
@@ -672,7 +675,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
)
else:
w13_weight_zero_point = torch.nn.Parameter(
torch.zeros(num_experts, 2, dtype=torch.int8),
torch.zeros(num_experts, self.moe.w13_num_shards, dtype=torch.int8),
requires_grad=False,
)
w2_weight_zero_point = torch.nn.Parameter(
@@ -689,7 +692,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
w13_bias = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
dtype=torch.float32,
),
requires_grad=False,
@@ -762,7 +765,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
for expert_id in range(layer.local_num_experts):
start = 0
for shard_id in range(2):
for shard_id in range(self.moe.w13_num_shards):
dq_weight = per_tensor_dequantize(
layer.w13_weight[expert_id][start : start + shard_size, :],
layer.w13_weight_scale[expert_id][shard_id],
@@ -905,7 +908,7 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod):
w13_weight = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
hidden_size // 8, # INT32 packing for W4
dtype=params_dtype,
),
@@ -927,7 +930,8 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod):
# Per-tensor fp8 weight scales
w13_weight_scale = torch.nn.Parameter(
torch.ones(num_experts, 2, dtype=torch.float32), requires_grad=False
torch.ones(num_experts, self.moe.w13_num_shards, dtype=torch.float32),
requires_grad=False,
)
w2_weight_scale = torch.nn.Parameter(
torch.ones(num_experts, dtype=torch.float32), requires_grad=False
@@ -944,7 +948,7 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod):
w13_weight_scale_2 = torch.nn.Parameter(
torch.ones(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
dtype=torch.float32,
),
requires_grad=False,
@@ -978,7 +982,7 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod):
for expert_id in range(layer.local_num_experts):
start = 0
max_w13_scale_fp8 = max_w13_scales[expert_id]
for shard_id in range(2):
for shard_id in range(self.moe.w13_num_shards):
if layer.w13_weight_scale[expert_id][shard_id] != max_w13_scale_fp8:
int4_rescale = (
layer.w13_weight_scale[expert_id][shard_id] / max_w13_scale_fp8
@@ -1193,7 +1197,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
w13_weight = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
self.get_packed_dim(hidden_size, self.weight_dtype),
dtype=params_dtype,
),
@@ -1220,7 +1224,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
w13_weight_scale = torch.nn.Parameter(
torch.ones(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
hidden_size // OCP_MX_BLOCK_SIZE,
dtype=params_dtype,
),
@@ -1245,7 +1249,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
w13_bias = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
dtype=torch.float32,
),
requires_grad=False,
@@ -1499,13 +1503,12 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
layer.quant_config = self.quant_config
weight_dtype = torch.uint8
weight_scale_dtype = torch.float8_e4m3fn
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
# GEMM 1 - w13 weight
w13_weight = torch.nn.Parameter(
torch.empty(
num_experts,
w13_num_shards * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
# 2 fp4 items are packed in the input dimension
hidden_size // 2,
dtype=weight_dtype,
@@ -1533,7 +1536,7 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
w13_weight_scale = torch.nn.Parameter(
torch.empty(
num_experts,
w13_num_shards * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
hidden_size // self.group_size,
dtype=weight_scale_dtype,
),
@@ -1563,7 +1566,7 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
)
w13_weight_scale_2 = torch.nn.Parameter(
torch.empty(num_experts, w13_num_shards, dtype=torch.float32),
torch.empty(num_experts, self.moe.w13_num_shards, dtype=torch.float32),
requires_grad=False,
)
layer.register_parameter("w13_weight_scale_2", w13_weight_scale_2)
@@ -1578,7 +1581,7 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
# Input global scales (per-tensor FP32 scales)
w13_input_scale_2 = torch.nn.Parameter(
torch.empty(num_experts, w13_num_shards, dtype=torch.float32),
torch.empty(num_experts, self.moe.w13_num_shards, dtype=torch.float32),
requires_grad=False,
)
layer.register_parameter("w13_input_scale_2", w13_input_scale_2)