[Bugfix] Size and iterate w13 by shard count for non-gated MoE (#51125)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Matej Sirovatka <matej.sirovatka@gmail.com> Co-authored-by: Rishi Puri <riship@nvidia.com> Co-authored-by: Hsiao-Yuan Chen <littlecircle0730@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Matej Sirovatka
Rishi Puri
Hsiao-Yuan Chen
Claude Opus 5
parent
811622c410
commit
7c77868cdf
@@ -216,7 +216,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
@@ -243,7 +243,10 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
# They are combined to a single scale after weight loading.
|
||||
if self.model_type != "gpt_oss":
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.ones(num_experts, 2, dtype=torch.float32), requires_grad=False
|
||||
torch.ones(
|
||||
num_experts, self.moe.w13_num_shards, dtype=torch.float32
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
else:
|
||||
# For gpt_oss, the w1(gate) & w3(up) are fused as one.
|
||||
@@ -267,7 +270,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.ones(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -306,7 +309,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
w13_bias = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -400,7 +403,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
# For non-gpt_oss, process w1 and w3 shards separately
|
||||
for expert_id in range(layer.local_num_experts):
|
||||
start = 0
|
||||
for shard_id in range(2):
|
||||
for shard_id in range(self.moe.w13_num_shards):
|
||||
dq_weight = per_tensor_dequantize(
|
||||
layer.w13_weight[expert_id][start : start + shard_size, :],
|
||||
layer.w13_weight_scale[expert_id][shard_id],
|
||||
@@ -560,7 +563,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
@@ -586,7 +589,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.ones(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -605,7 +608,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
|
||||
else:
|
||||
# per-tensor: one scalar per expert (two for the fused w1/w3)
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.ones(num_experts, 2, dtype=torch.float32),
|
||||
torch.ones(num_experts, self.moe.w13_num_shards, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight_scale", w13_weight_scale)
|
||||
@@ -644,7 +647,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
|
||||
|
||||
# ZERO POINTS (loaded but discarded after loading; kernel uses symmetric)
|
||||
w13_input_zero_point = torch.nn.Parameter(
|
||||
torch.zeros(num_experts, 2, dtype=torch.int8),
|
||||
torch.zeros(num_experts, self.moe.w13_num_shards, dtype=torch.int8),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_input_zero_point", w13_input_zero_point)
|
||||
@@ -661,7 +664,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
|
||||
w13_weight_zero_point = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=torch.int8,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -672,7 +675,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
|
||||
)
|
||||
else:
|
||||
w13_weight_zero_point = torch.nn.Parameter(
|
||||
torch.zeros(num_experts, 2, dtype=torch.int8),
|
||||
torch.zeros(num_experts, self.moe.w13_num_shards, dtype=torch.int8),
|
||||
requires_grad=False,
|
||||
)
|
||||
w2_weight_zero_point = torch.nn.Parameter(
|
||||
@@ -689,7 +692,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
|
||||
w13_bias = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -762,7 +765,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
|
||||
|
||||
for expert_id in range(layer.local_num_experts):
|
||||
start = 0
|
||||
for shard_id in range(2):
|
||||
for shard_id in range(self.moe.w13_num_shards):
|
||||
dq_weight = per_tensor_dequantize(
|
||||
layer.w13_weight[expert_id][start : start + shard_size, :],
|
||||
layer.w13_weight_scale[expert_id][shard_id],
|
||||
@@ -905,7 +908,7 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size // 8, # INT32 packing for W4
|
||||
dtype=params_dtype,
|
||||
),
|
||||
@@ -927,7 +930,8 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
|
||||
# Per-tensor fp8 weight scales
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.ones(num_experts, 2, dtype=torch.float32), requires_grad=False
|
||||
torch.ones(num_experts, self.moe.w13_num_shards, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
w2_weight_scale = torch.nn.Parameter(
|
||||
torch.ones(num_experts, dtype=torch.float32), requires_grad=False
|
||||
@@ -944,7 +948,7 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
w13_weight_scale_2 = torch.nn.Parameter(
|
||||
torch.ones(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -978,7 +982,7 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
for expert_id in range(layer.local_num_experts):
|
||||
start = 0
|
||||
max_w13_scale_fp8 = max_w13_scales[expert_id]
|
||||
for shard_id in range(2):
|
||||
for shard_id in range(self.moe.w13_num_shards):
|
||||
if layer.w13_weight_scale[expert_id][shard_id] != max_w13_scale_fp8:
|
||||
int4_rescale = (
|
||||
layer.w13_weight_scale[expert_id][shard_id] / max_w13_scale_fp8
|
||||
@@ -1193,7 +1197,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
self.get_packed_dim(hidden_size, self.weight_dtype),
|
||||
dtype=params_dtype,
|
||||
),
|
||||
@@ -1220,7 +1224,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.ones(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size // OCP_MX_BLOCK_SIZE,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
@@ -1245,7 +1249,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
w13_bias = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -1499,13 +1503,12 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
|
||||
layer.quant_config = self.quant_config
|
||||
weight_dtype = torch.uint8
|
||||
weight_scale_dtype = torch.float8_e4m3fn
|
||||
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
|
||||
|
||||
# GEMM 1 - w13 weight
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
# 2 fp4 items are packed in the input dimension
|
||||
hidden_size // 2,
|
||||
dtype=weight_dtype,
|
||||
@@ -1533,7 +1536,7 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size // self.group_size,
|
||||
dtype=weight_scale_dtype,
|
||||
),
|
||||
@@ -1563,7 +1566,7 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
|
||||
)
|
||||
|
||||
w13_weight_scale_2 = torch.nn.Parameter(
|
||||
torch.empty(num_experts, w13_num_shards, dtype=torch.float32),
|
||||
torch.empty(num_experts, self.moe.w13_num_shards, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight_scale_2", w13_weight_scale_2)
|
||||
@@ -1578,7 +1581,7 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
|
||||
|
||||
# Input global scales (per-tensor FP32 scales)
|
||||
w13_input_scale_2 = torch.nn.Parameter(
|
||||
torch.empty(num_experts, w13_num_shards, dtype=torch.float32),
|
||||
torch.empty(num_experts, self.moe.w13_num_shards, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_input_scale_2", w13_input_scale_2)
|
||||
|
||||
Reference in New Issue
Block a user