[Bugfix] Size and iterate w13 by shard count for non-gated MoE (#51125)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Matej Sirovatka <matej.sirovatka@gmail.com>
Co-authored-by: Rishi Puri <riship@nvidia.com>
Co-authored-by: Hsiao-Yuan Chen <littlecircle0730@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-08-05 17:24:47 -07:00
committed by GitHub
co-authored by Matej Sirovatka Rishi Puri Hsiao-Yuan Chen Claude Opus 5
parent 811622c410
commit 7c77868cdf
16 changed files with 104 additions and 80 deletions
@@ -592,7 +592,9 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
torch.empty(
num_experts,
hidden_size,
2 * intermediate_size_per_partition // self.quant_config.pack_factor,
self.moe.w13_num_shards
* intermediate_size_per_partition
// self.quant_config.pack_factor,
dtype=torch.int32,
),
requires_grad=False,
@@ -623,7 +625,7 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
torch.empty(
num_experts,
num_groups_w13,
intermediate_size_per_partition * 2,
intermediate_size_per_partition * self.moe.w13_num_shards,
dtype=params_dtype,
),
requires_grad=False,
@@ -644,7 +646,9 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
torch.empty(
num_experts,
num_groups_w13,
2 * intermediate_size_per_partition // self.quant_config.pack_factor,
self.moe.w13_num_shards
* intermediate_size_per_partition
// self.quant_config.pack_factor,
dtype=torch.int32,
),
requires_grad=False,
@@ -541,7 +541,7 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
torch.empty(
num_experts,
hidden_size // self.quant_config.pack_factor,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
dtype=torch.int32,
),
requires_grad=False,
@@ -565,7 +565,7 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
torch.empty(
num_experts,
scales_size13,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
dtype=params_dtype,
),
requires_grad=False,
@@ -586,7 +586,9 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
torch.empty(
num_experts,
scales_size13,
2 * intermediate_size_per_partition // self.quant_config.pack_factor,
self.moe.w13_num_shards
* intermediate_size_per_partition
// self.quant_config.pack_factor,
dtype=torch.int32,
),
requires_grad=False,
@@ -654,7 +656,7 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
w13_bias = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
dtype=params_dtype,
),
requires_grad=False,
@@ -523,7 +523,7 @@ class BitsAndBytesMoEMethod(FusedMoEMethodBase):
quant_ratio = calculate_quant_ratio(params_dtype)
# Fused gate_up_proj (column parallel)
w13_total_size = (
hidden_size * 2 * intermediate_size_per_partition
hidden_size * self.moe.w13_num_shards * intermediate_size_per_partition
) // quant_ratio
w13_qweight = torch.nn.Parameter(
torch.empty(
@@ -541,10 +541,10 @@ class BitsAndBytesMoEMethod(FusedMoEMethodBase):
{
"num_experts": num_experts,
"input_dim": hidden_size,
"output_dim": 2 * intermediate_size_per_partition,
"output_dim": self.moe.w13_num_shards * intermediate_size_per_partition,
"experts_shape": (
num_experts,
intermediate_size_per_partition * 2,
intermediate_size_per_partition * self.moe.w13_num_shards,
hidden_size,
),
"pack_factor": quant_ratio,
@@ -75,7 +75,7 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
w13_weight = torch.nn.Parameter(
torch.empty(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
# 2 fp4 items are packed in the input dimension
hidden_size // 2,
requires_grad=False,
@@ -102,7 +102,7 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
w13_weight_scale = torch.nn.Parameter(
torch.empty(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
# 2 fp4 items are packed in the input dimension
hidden_size // self.group_size,
dtype=torch.uint8,
@@ -79,7 +79,7 @@ class CompressedTensorsW4A8Fp8MoEMethod(CompressedTensorsMoEMethod):
w13_weight_packed = torch.nn.Parameter(
torch.empty(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
hidden_size // self.packed_factor,
dtype=params_dtype,
),
@@ -107,7 +107,7 @@ class CompressedTensorsW4A8Fp8MoEMethod(CompressedTensorsMoEMethod):
w13_weight_scale = torch.nn.Parameter(
torch.ones(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
hidden_size // self.group_size,
dtype=layer.orig_dtype,
),
@@ -147,7 +147,7 @@ class CompressedTensorsW4A8Fp8MoEMethod(CompressedTensorsMoEMethod):
w13_weight_chan_scale = torch.nn.Parameter(
torch.ones(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
dtype=torch.float32,
),
requires_grad=False,
+12 -5
View File
@@ -545,7 +545,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
w13_weight = torch.nn.Parameter(
torch.empty(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
hidden_size,
dtype=params_dtype,
),
@@ -571,7 +571,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
w13_bias = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
dtype=layer.orig_dtype,
),
requires_grad=False,
@@ -588,13 +588,16 @@ class Fp8MoEMethod(FusedMoEMethodBase):
# WEIGHT_SCALES
if not self.block_quant:
# For per-tensor quant, the scales are per expert and weight.
w13_scale_data = torch.ones(num_experts, 2, dtype=torch.float32)
w13_scale_data = torch.ones(
num_experts, self.moe.w13_num_shards, dtype=torch.float32
)
w2_scale_data = torch.ones(num_experts, dtype=torch.float32)
else:
# For block quant, the scales are per block (typically 128x128).
w13_scale_data = torch.ones(
num_experts,
2 * ((intermediate_size_per_partition + block_n - 1) // block_n),
self.moe.w13_num_shards
* ((intermediate_size_per_partition + block_n - 1) // block_n),
(hidden_size + block_k - 1) // block_k,
dtype=torch.float32,
)
@@ -722,7 +725,11 @@ class Fp8MoEMethod(FusedMoEMethodBase):
if not self.block_quant:
shard_size = layer.intermediate_size_per_partition
w13, w13_scale = process_fp8_weight_tensor_strategy_moe(
w13, w13_scale, shard_size, layer.local_num_experts
w13,
w13_scale,
shard_size,
layer.local_num_experts,
is_act_and_mul=self.moe.is_act_and_mul,
)
# Shuffle weights to runtime format and setup kernel.
@@ -728,10 +728,10 @@ class HummingMoEMethod(FusedMoEMethodBase):
# The weight names of sublayer start with the prefix "{sublayer_name}_"
layer.sublayer_configs = {
"w13": {
"shape_n": intermediate_size_per_partition * 2,
"shape_n": intermediate_size_per_partition * self.moe.w13_num_shards,
"shape_k": hidden_size,
"tensors_attrs": self.weight_schema.get_padded_tensors_attrs(
shape_n=intermediate_size_per_partition * 2,
shape_n=intermediate_size_per_partition * self.moe.w13_num_shards,
shape_k=hidden_size,
num_experts=num_experts,
param_dtype=params_dtype,
@@ -88,7 +88,7 @@ class INCMxfp4MoEMethod(FusedMoEMethodBase):
w13_weight = torch.nn.Parameter(
torch.empty(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
hidden_size // 2,
dtype=torch.uint8,
),
@@ -113,7 +113,7 @@ class INCMxfp4MoEMethod(FusedMoEMethodBase):
w13_weight_scale = torch.nn.Parameter(
torch.empty(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
hidden_size // self.group_size,
dtype=torch.uint8,
),
@@ -276,7 +276,7 @@ class MoeWNA16Method(FusedMoEMethodBase):
w13_qweight = torch.nn.Parameter(
torch.empty(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
hidden_size // bit8_pack_factor,
dtype=torch.uint8,
),
@@ -301,7 +301,7 @@ class MoeWNA16Method(FusedMoEMethodBase):
w13_scales = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
hidden_size // group_size,
dtype=params_dtype,
),
@@ -326,7 +326,9 @@ class MoeWNA16Method(FusedMoEMethodBase):
w13_qzeros = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition // bit8_pack_factor,
self.moe.w13_num_shards
* intermediate_size_per_partition
// bit8_pack_factor,
hidden_size // group_size,
dtype=torch.uint8,
),
@@ -215,7 +215,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
w13_weight = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
hidden_size // 2,
dtype=weight_dtype,
),
@@ -227,7 +227,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
w13_weight_scale = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
hidden_size // mxfp4_block,
dtype=scale_dtype,
),
@@ -267,7 +267,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
w13_bias = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
dtype=torch.bfloat16,
),
requires_grad=False,
@@ -305,13 +305,13 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
assert (
w13.dim() == 3
and w13.shape[0] == num_experts
and w13.shape[1] == intermediate_size * 2
and w13.shape[1] == intermediate_size * self.moe.w13_num_shards
and w13.shape[2] == hidden_size // 2
)
assert (
w13_scale.dim() == 3
and w13_scale.shape[0] == num_experts
and w13_scale.shape[1] == intermediate_size * 2
and w13_scale.shape[1] == intermediate_size * self.moe.w13_num_shards
and w13_scale.shape[2] == hidden_size // sf_block_size
)
assert (
@@ -329,7 +329,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
assert (
w13_bias.dim() == 2
and w13_bias.shape[0] == num_experts
and w13_bias.shape[1] == intermediate_size * 2
and w13_bias.shape[1] == intermediate_size * self.moe.w13_num_shards
)
if w2_bias is not None:
assert (
@@ -599,7 +599,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
w13_weight = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
hidden_size // 2,
dtype=weight_dtype,
),
@@ -611,7 +611,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
w13_weight_scale = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
hidden_size // mxfp4_block,
dtype=scale_dtype,
),
@@ -651,7 +651,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
w13_bias = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
dtype=torch.bfloat16,
),
requires_grad=False,
@@ -689,13 +689,13 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
assert (
w13.dim() == 3
and w13.shape[0] == num_experts
and w13.shape[1] == intermediate_size * 2
and w13.shape[1] == intermediate_size * self.moe.w13_num_shards
and w13.shape[2] == hidden_size // 2
)
assert (
w13_scale.dim() == 3
and w13_scale.shape[0] == num_experts
and w13_scale.shape[1] == intermediate_size * 2
and w13_scale.shape[1] == intermediate_size * self.moe.w13_num_shards
and w13_scale.shape[2] == hidden_size // sf_block_size
)
assert (
@@ -713,7 +713,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
assert (
w13_bias.dim() == 2
and w13_bias.shape[0] == num_experts
and w13_bias.shape[1] == intermediate_size * 2
and w13_bias.shape[1] == intermediate_size * self.moe.w13_num_shards
)
if w2_bias is not None:
assert (
@@ -582,12 +582,11 @@ class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
hidden_size = layer.moe_config.hidden_dim_unpadded
intermediate_size = layer.moe_config.intermediate_size_per_partition_unpadded
w13_half_size = layer.w13_weight.shape[1] // 2
if w13_half_size > intermediate_size:
layer.w13_weight[:, intermediate_size:w13_half_size, :] = 0
layer.w13_weight[
:, w13_half_size + intermediate_size : 2 * w13_half_size, :
] = 0
w13_shard = layer.w13_weight.shape[1] // self.moe.w13_num_shards
if w13_shard > intermediate_size:
for shard in range(self.moe.w13_num_shards):
start = shard * w13_shard + intermediate_size
layer.w13_weight[:, start : (shard + 1) * w13_shard, :] = 0
if layer.w13_weight.shape[2] > hidden_size:
layer.w13_weight[:, :, hidden_size:] = 0
@@ -597,12 +596,11 @@ class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
layer.w2_weight[:, :, intermediate_size:] = 0
if getattr(layer, "w13_bias", None) is not None:
w13_bias_half_size = layer.w13_bias.shape[1] // 2
if w13_bias_half_size > intermediate_size:
layer.w13_bias[:, intermediate_size:w13_bias_half_size] = 0
layer.w13_bias[
:, w13_bias_half_size + intermediate_size : 2 * w13_bias_half_size
] = 0
w13_bias_shard = layer.w13_bias.shape[1] // self.moe.w13_num_shards
if w13_bias_shard > intermediate_size:
for shard in range(self.moe.w13_num_shards):
start = shard * w13_bias_shard + intermediate_size
layer.w13_bias[:, start : (shard + 1) * w13_bias_shard] = 0
if (
getattr(layer, "w2_bias", None) is not None
@@ -40,7 +40,7 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
w13_weight = torch.nn.Parameter(
torch.empty(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
hidden_size,
device="meta",
dtype=params_dtype,
@@ -69,7 +69,7 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
w13_bias = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
device="meta",
dtype=layer.orig_dtype,
),
@@ -216,7 +216,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
w13_weight = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
hidden_size,
dtype=params_dtype,
),
@@ -243,7 +243,10 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
# They are combined to a single scale after weight loading.
if self.model_type != "gpt_oss":
w13_weight_scale = torch.nn.Parameter(
torch.ones(num_experts, 2, dtype=torch.float32), requires_grad=False
torch.ones(
num_experts, self.moe.w13_num_shards, dtype=torch.float32
),
requires_grad=False,
)
else:
# For gpt_oss, the w1(gate) & w3(up) are fused as one.
@@ -267,7 +270,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
w13_weight_scale = torch.nn.Parameter(
torch.ones(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
dtype=torch.float32,
),
requires_grad=False,
@@ -306,7 +309,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
w13_bias = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
dtype=torch.float32,
),
requires_grad=False,
@@ -400,7 +403,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
# For non-gpt_oss, process w1 and w3 shards separately
for expert_id in range(layer.local_num_experts):
start = 0
for shard_id in range(2):
for shard_id in range(self.moe.w13_num_shards):
dq_weight = per_tensor_dequantize(
layer.w13_weight[expert_id][start : start + shard_size, :],
layer.w13_weight_scale[expert_id][shard_id],
@@ -560,7 +563,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
w13_weight = torch.nn.Parameter(
torch.empty(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
hidden_size,
dtype=params_dtype,
),
@@ -586,7 +589,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
w13_weight_scale = torch.nn.Parameter(
torch.ones(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
dtype=torch.float32,
),
requires_grad=False,
@@ -605,7 +608,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
else:
# per-tensor: one scalar per expert (two for the fused w1/w3)
w13_weight_scale = torch.nn.Parameter(
torch.ones(num_experts, 2, dtype=torch.float32),
torch.ones(num_experts, self.moe.w13_num_shards, dtype=torch.float32),
requires_grad=False,
)
layer.register_parameter("w13_weight_scale", w13_weight_scale)
@@ -644,7 +647,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
# ZERO POINTS (loaded but discarded after loading; kernel uses symmetric)
w13_input_zero_point = torch.nn.Parameter(
torch.zeros(num_experts, 2, dtype=torch.int8),
torch.zeros(num_experts, self.moe.w13_num_shards, dtype=torch.int8),
requires_grad=False,
)
layer.register_parameter("w13_input_zero_point", w13_input_zero_point)
@@ -661,7 +664,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
w13_weight_zero_point = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
dtype=torch.int8,
),
requires_grad=False,
@@ -672,7 +675,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
)
else:
w13_weight_zero_point = torch.nn.Parameter(
torch.zeros(num_experts, 2, dtype=torch.int8),
torch.zeros(num_experts, self.moe.w13_num_shards, dtype=torch.int8),
requires_grad=False,
)
w2_weight_zero_point = torch.nn.Parameter(
@@ -689,7 +692,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
w13_bias = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
dtype=torch.float32,
),
requires_grad=False,
@@ -762,7 +765,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
for expert_id in range(layer.local_num_experts):
start = 0
for shard_id in range(2):
for shard_id in range(self.moe.w13_num_shards):
dq_weight = per_tensor_dequantize(
layer.w13_weight[expert_id][start : start + shard_size, :],
layer.w13_weight_scale[expert_id][shard_id],
@@ -905,7 +908,7 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod):
w13_weight = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
hidden_size // 8, # INT32 packing for W4
dtype=params_dtype,
),
@@ -927,7 +930,8 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod):
# Per-tensor fp8 weight scales
w13_weight_scale = torch.nn.Parameter(
torch.ones(num_experts, 2, dtype=torch.float32), requires_grad=False
torch.ones(num_experts, self.moe.w13_num_shards, dtype=torch.float32),
requires_grad=False,
)
w2_weight_scale = torch.nn.Parameter(
torch.ones(num_experts, dtype=torch.float32), requires_grad=False
@@ -944,7 +948,7 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod):
w13_weight_scale_2 = torch.nn.Parameter(
torch.ones(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
dtype=torch.float32,
),
requires_grad=False,
@@ -978,7 +982,7 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod):
for expert_id in range(layer.local_num_experts):
start = 0
max_w13_scale_fp8 = max_w13_scales[expert_id]
for shard_id in range(2):
for shard_id in range(self.moe.w13_num_shards):
if layer.w13_weight_scale[expert_id][shard_id] != max_w13_scale_fp8:
int4_rescale = (
layer.w13_weight_scale[expert_id][shard_id] / max_w13_scale_fp8
@@ -1193,7 +1197,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
w13_weight = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
self.get_packed_dim(hidden_size, self.weight_dtype),
dtype=params_dtype,
),
@@ -1220,7 +1224,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
w13_weight_scale = torch.nn.Parameter(
torch.ones(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
hidden_size // OCP_MX_BLOCK_SIZE,
dtype=params_dtype,
),
@@ -1245,7 +1249,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
w13_bias = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
dtype=torch.float32,
),
requires_grad=False,
@@ -1499,13 +1503,12 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
layer.quant_config = self.quant_config
weight_dtype = torch.uint8
weight_scale_dtype = torch.float8_e4m3fn
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
# GEMM 1 - w13 weight
w13_weight = torch.nn.Parameter(
torch.empty(
num_experts,
w13_num_shards * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
# 2 fp4 items are packed in the input dimension
hidden_size // 2,
dtype=weight_dtype,
@@ -1533,7 +1536,7 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
w13_weight_scale = torch.nn.Parameter(
torch.empty(
num_experts,
w13_num_shards * intermediate_size_per_partition,
self.moe.w13_num_shards * intermediate_size_per_partition,
hidden_size // self.group_size,
dtype=weight_scale_dtype,
),
@@ -1563,7 +1566,7 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
)
w13_weight_scale_2 = torch.nn.Parameter(
torch.empty(num_experts, w13_num_shards, dtype=torch.float32),
torch.empty(num_experts, self.moe.w13_num_shards, dtype=torch.float32),
requires_grad=False,
)
layer.register_parameter("w13_weight_scale_2", w13_weight_scale_2)
@@ -1578,7 +1581,7 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
# Input global scales (per-tensor FP32 scales)
w13_input_scale_2 = torch.nn.Parameter(
torch.empty(num_experts, w13_num_shards, dtype=torch.float32),
torch.empty(num_experts, self.moe.w13_num_shards, dtype=torch.float32),
requires_grad=False,
)
layer.register_parameter("w13_input_scale_2", w13_input_scale_2)