[Bugfix] Size and iterate w13 by shard count for non-gated MoE (#51125)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Matej Sirovatka <matej.sirovatka@gmail.com> Co-authored-by: Rishi Puri <riship@nvidia.com> Co-authored-by: Hsiao-Yuan Chen <littlecircle0730@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Matej Sirovatka
Rishi Puri
Hsiao-Yuan Chen
Claude Opus 5
parent
811622c410
commit
7c77868cdf
@@ -592,7 +592,9 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
|
||||
torch.empty(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
2 * intermediate_size_per_partition // self.quant_config.pack_factor,
|
||||
self.moe.w13_num_shards
|
||||
* intermediate_size_per_partition
|
||||
// self.quant_config.pack_factor,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -623,7 +625,7 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
|
||||
torch.empty(
|
||||
num_experts,
|
||||
num_groups_w13,
|
||||
intermediate_size_per_partition * 2,
|
||||
intermediate_size_per_partition * self.moe.w13_num_shards,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -644,7 +646,9 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
|
||||
torch.empty(
|
||||
num_experts,
|
||||
num_groups_w13,
|
||||
2 * intermediate_size_per_partition // self.quant_config.pack_factor,
|
||||
self.moe.w13_num_shards
|
||||
* intermediate_size_per_partition
|
||||
// self.quant_config.pack_factor,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
|
||||
@@ -541,7 +541,7 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
torch.empty(
|
||||
num_experts,
|
||||
hidden_size // self.quant_config.pack_factor,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -565,7 +565,7 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
torch.empty(
|
||||
num_experts,
|
||||
scales_size13,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -586,7 +586,9 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
torch.empty(
|
||||
num_experts,
|
||||
scales_size13,
|
||||
2 * intermediate_size_per_partition // self.quant_config.pack_factor,
|
||||
self.moe.w13_num_shards
|
||||
* intermediate_size_per_partition
|
||||
// self.quant_config.pack_factor,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -654,7 +656,7 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
w13_bias = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
|
||||
@@ -523,7 +523,7 @@ class BitsAndBytesMoEMethod(FusedMoEMethodBase):
|
||||
quant_ratio = calculate_quant_ratio(params_dtype)
|
||||
# Fused gate_up_proj (column parallel)
|
||||
w13_total_size = (
|
||||
hidden_size * 2 * intermediate_size_per_partition
|
||||
hidden_size * self.moe.w13_num_shards * intermediate_size_per_partition
|
||||
) // quant_ratio
|
||||
w13_qweight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
@@ -541,10 +541,10 @@ class BitsAndBytesMoEMethod(FusedMoEMethodBase):
|
||||
{
|
||||
"num_experts": num_experts,
|
||||
"input_dim": hidden_size,
|
||||
"output_dim": 2 * intermediate_size_per_partition,
|
||||
"output_dim": self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
"experts_shape": (
|
||||
num_experts,
|
||||
intermediate_size_per_partition * 2,
|
||||
intermediate_size_per_partition * self.moe.w13_num_shards,
|
||||
hidden_size,
|
||||
),
|
||||
"pack_factor": quant_ratio,
|
||||
|
||||
+2
-2
@@ -75,7 +75,7 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
# 2 fp4 items are packed in the input dimension
|
||||
hidden_size // 2,
|
||||
requires_grad=False,
|
||||
@@ -102,7 +102,7 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
# 2 fp4 items are packed in the input dimension
|
||||
hidden_size // self.group_size,
|
||||
dtype=torch.uint8,
|
||||
|
||||
+3
-3
@@ -79,7 +79,7 @@ class CompressedTensorsW4A8Fp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
w13_weight_packed = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size // self.packed_factor,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
@@ -107,7 +107,7 @@ class CompressedTensorsW4A8Fp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.ones(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size // self.group_size,
|
||||
dtype=layer.orig_dtype,
|
||||
),
|
||||
@@ -147,7 +147,7 @@ class CompressedTensorsW4A8Fp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
w13_weight_chan_scale = torch.nn.Parameter(
|
||||
torch.ones(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
|
||||
@@ -545,7 +545,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
@@ -571,7 +571,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
w13_bias = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=layer.orig_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -588,13 +588,16 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
# WEIGHT_SCALES
|
||||
if not self.block_quant:
|
||||
# For per-tensor quant, the scales are per expert and weight.
|
||||
w13_scale_data = torch.ones(num_experts, 2, dtype=torch.float32)
|
||||
w13_scale_data = torch.ones(
|
||||
num_experts, self.moe.w13_num_shards, dtype=torch.float32
|
||||
)
|
||||
w2_scale_data = torch.ones(num_experts, dtype=torch.float32)
|
||||
else:
|
||||
# For block quant, the scales are per block (typically 128x128).
|
||||
w13_scale_data = torch.ones(
|
||||
num_experts,
|
||||
2 * ((intermediate_size_per_partition + block_n - 1) // block_n),
|
||||
self.moe.w13_num_shards
|
||||
* ((intermediate_size_per_partition + block_n - 1) // block_n),
|
||||
(hidden_size + block_k - 1) // block_k,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
@@ -722,7 +725,11 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
if not self.block_quant:
|
||||
shard_size = layer.intermediate_size_per_partition
|
||||
w13, w13_scale = process_fp8_weight_tensor_strategy_moe(
|
||||
w13, w13_scale, shard_size, layer.local_num_experts
|
||||
w13,
|
||||
w13_scale,
|
||||
shard_size,
|
||||
layer.local_num_experts,
|
||||
is_act_and_mul=self.moe.is_act_and_mul,
|
||||
)
|
||||
|
||||
# Shuffle weights to runtime format and setup kernel.
|
||||
|
||||
@@ -728,10 +728,10 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
# The weight names of sublayer start with the prefix "{sublayer_name}_"
|
||||
layer.sublayer_configs = {
|
||||
"w13": {
|
||||
"shape_n": intermediate_size_per_partition * 2,
|
||||
"shape_n": intermediate_size_per_partition * self.moe.w13_num_shards,
|
||||
"shape_k": hidden_size,
|
||||
"tensors_attrs": self.weight_schema.get_padded_tensors_attrs(
|
||||
shape_n=intermediate_size_per_partition * 2,
|
||||
shape_n=intermediate_size_per_partition * self.moe.w13_num_shards,
|
||||
shape_k=hidden_size,
|
||||
num_experts=num_experts,
|
||||
param_dtype=params_dtype,
|
||||
|
||||
@@ -88,7 +88,7 @@ class INCMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size // 2,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
@@ -113,7 +113,7 @@ class INCMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size // self.group_size,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
|
||||
@@ -276,7 +276,7 @@ class MoeWNA16Method(FusedMoEMethodBase):
|
||||
w13_qweight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size // bit8_pack_factor,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
@@ -301,7 +301,7 @@ class MoeWNA16Method(FusedMoEMethodBase):
|
||||
w13_scales = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size // group_size,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
@@ -326,7 +326,9 @@ class MoeWNA16Method(FusedMoEMethodBase):
|
||||
w13_qzeros = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition // bit8_pack_factor,
|
||||
self.moe.w13_num_shards
|
||||
* intermediate_size_per_partition
|
||||
// bit8_pack_factor,
|
||||
hidden_size // group_size,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
|
||||
@@ -215,7 +215,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size // 2,
|
||||
dtype=weight_dtype,
|
||||
),
|
||||
@@ -227,7 +227,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size // mxfp4_block,
|
||||
dtype=scale_dtype,
|
||||
),
|
||||
@@ -267,7 +267,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
w13_bias = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=torch.bfloat16,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -305,13 +305,13 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
assert (
|
||||
w13.dim() == 3
|
||||
and w13.shape[0] == num_experts
|
||||
and w13.shape[1] == intermediate_size * 2
|
||||
and w13.shape[1] == intermediate_size * self.moe.w13_num_shards
|
||||
and w13.shape[2] == hidden_size // 2
|
||||
)
|
||||
assert (
|
||||
w13_scale.dim() == 3
|
||||
and w13_scale.shape[0] == num_experts
|
||||
and w13_scale.shape[1] == intermediate_size * 2
|
||||
and w13_scale.shape[1] == intermediate_size * self.moe.w13_num_shards
|
||||
and w13_scale.shape[2] == hidden_size // sf_block_size
|
||||
)
|
||||
assert (
|
||||
@@ -329,7 +329,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
assert (
|
||||
w13_bias.dim() == 2
|
||||
and w13_bias.shape[0] == num_experts
|
||||
and w13_bias.shape[1] == intermediate_size * 2
|
||||
and w13_bias.shape[1] == intermediate_size * self.moe.w13_num_shards
|
||||
)
|
||||
if w2_bias is not None:
|
||||
assert (
|
||||
@@ -599,7 +599,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size // 2,
|
||||
dtype=weight_dtype,
|
||||
),
|
||||
@@ -611,7 +611,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size // mxfp4_block,
|
||||
dtype=scale_dtype,
|
||||
),
|
||||
@@ -651,7 +651,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
w13_bias = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=torch.bfloat16,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -689,13 +689,13 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
assert (
|
||||
w13.dim() == 3
|
||||
and w13.shape[0] == num_experts
|
||||
and w13.shape[1] == intermediate_size * 2
|
||||
and w13.shape[1] == intermediate_size * self.moe.w13_num_shards
|
||||
and w13.shape[2] == hidden_size // 2
|
||||
)
|
||||
assert (
|
||||
w13_scale.dim() == 3
|
||||
and w13_scale.shape[0] == num_experts
|
||||
and w13_scale.shape[1] == intermediate_size * 2
|
||||
and w13_scale.shape[1] == intermediate_size * self.moe.w13_num_shards
|
||||
and w13_scale.shape[2] == hidden_size // sf_block_size
|
||||
)
|
||||
assert (
|
||||
@@ -713,7 +713,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
assert (
|
||||
w13_bias.dim() == 2
|
||||
and w13_bias.shape[0] == num_experts
|
||||
and w13_bias.shape[1] == intermediate_size * 2
|
||||
and w13_bias.shape[1] == intermediate_size * self.moe.w13_num_shards
|
||||
)
|
||||
if w2_bias is not None:
|
||||
assert (
|
||||
|
||||
@@ -582,12 +582,11 @@ class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
hidden_size = layer.moe_config.hidden_dim_unpadded
|
||||
intermediate_size = layer.moe_config.intermediate_size_per_partition_unpadded
|
||||
|
||||
w13_half_size = layer.w13_weight.shape[1] // 2
|
||||
if w13_half_size > intermediate_size:
|
||||
layer.w13_weight[:, intermediate_size:w13_half_size, :] = 0
|
||||
layer.w13_weight[
|
||||
:, w13_half_size + intermediate_size : 2 * w13_half_size, :
|
||||
] = 0
|
||||
w13_shard = layer.w13_weight.shape[1] // self.moe.w13_num_shards
|
||||
if w13_shard > intermediate_size:
|
||||
for shard in range(self.moe.w13_num_shards):
|
||||
start = shard * w13_shard + intermediate_size
|
||||
layer.w13_weight[:, start : (shard + 1) * w13_shard, :] = 0
|
||||
if layer.w13_weight.shape[2] > hidden_size:
|
||||
layer.w13_weight[:, :, hidden_size:] = 0
|
||||
|
||||
@@ -597,12 +596,11 @@ class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
layer.w2_weight[:, :, intermediate_size:] = 0
|
||||
|
||||
if getattr(layer, "w13_bias", None) is not None:
|
||||
w13_bias_half_size = layer.w13_bias.shape[1] // 2
|
||||
if w13_bias_half_size > intermediate_size:
|
||||
layer.w13_bias[:, intermediate_size:w13_bias_half_size] = 0
|
||||
layer.w13_bias[
|
||||
:, w13_bias_half_size + intermediate_size : 2 * w13_bias_half_size
|
||||
] = 0
|
||||
w13_bias_shard = layer.w13_bias.shape[1] // self.moe.w13_num_shards
|
||||
if w13_bias_shard > intermediate_size:
|
||||
for shard in range(self.moe.w13_num_shards):
|
||||
start = shard * w13_bias_shard + intermediate_size
|
||||
layer.w13_bias[:, start : (shard + 1) * w13_bias_shard] = 0
|
||||
|
||||
if (
|
||||
getattr(layer, "w2_bias", None) is not None
|
||||
|
||||
@@ -40,7 +40,7 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size,
|
||||
device="meta",
|
||||
dtype=params_dtype,
|
||||
@@ -69,7 +69,7 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
|
||||
w13_bias = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
device="meta",
|
||||
dtype=layer.orig_dtype,
|
||||
),
|
||||
|
||||
@@ -216,7 +216,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
@@ -243,7 +243,10 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
# They are combined to a single scale after weight loading.
|
||||
if self.model_type != "gpt_oss":
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.ones(num_experts, 2, dtype=torch.float32), requires_grad=False
|
||||
torch.ones(
|
||||
num_experts, self.moe.w13_num_shards, dtype=torch.float32
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
else:
|
||||
# For gpt_oss, the w1(gate) & w3(up) are fused as one.
|
||||
@@ -267,7 +270,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.ones(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -306,7 +309,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
w13_bias = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -400,7 +403,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
# For non-gpt_oss, process w1 and w3 shards separately
|
||||
for expert_id in range(layer.local_num_experts):
|
||||
start = 0
|
||||
for shard_id in range(2):
|
||||
for shard_id in range(self.moe.w13_num_shards):
|
||||
dq_weight = per_tensor_dequantize(
|
||||
layer.w13_weight[expert_id][start : start + shard_size, :],
|
||||
layer.w13_weight_scale[expert_id][shard_id],
|
||||
@@ -560,7 +563,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
@@ -586,7 +589,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.ones(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -605,7 +608,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
|
||||
else:
|
||||
# per-tensor: one scalar per expert (two for the fused w1/w3)
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.ones(num_experts, 2, dtype=torch.float32),
|
||||
torch.ones(num_experts, self.moe.w13_num_shards, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight_scale", w13_weight_scale)
|
||||
@@ -644,7 +647,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
|
||||
|
||||
# ZERO POINTS (loaded but discarded after loading; kernel uses symmetric)
|
||||
w13_input_zero_point = torch.nn.Parameter(
|
||||
torch.zeros(num_experts, 2, dtype=torch.int8),
|
||||
torch.zeros(num_experts, self.moe.w13_num_shards, dtype=torch.int8),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_input_zero_point", w13_input_zero_point)
|
||||
@@ -661,7 +664,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
|
||||
w13_weight_zero_point = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=torch.int8,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -672,7 +675,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
|
||||
)
|
||||
else:
|
||||
w13_weight_zero_point = torch.nn.Parameter(
|
||||
torch.zeros(num_experts, 2, dtype=torch.int8),
|
||||
torch.zeros(num_experts, self.moe.w13_num_shards, dtype=torch.int8),
|
||||
requires_grad=False,
|
||||
)
|
||||
w2_weight_zero_point = torch.nn.Parameter(
|
||||
@@ -689,7 +692,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
|
||||
w13_bias = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -762,7 +765,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
|
||||
|
||||
for expert_id in range(layer.local_num_experts):
|
||||
start = 0
|
||||
for shard_id in range(2):
|
||||
for shard_id in range(self.moe.w13_num_shards):
|
||||
dq_weight = per_tensor_dequantize(
|
||||
layer.w13_weight[expert_id][start : start + shard_size, :],
|
||||
layer.w13_weight_scale[expert_id][shard_id],
|
||||
@@ -905,7 +908,7 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size // 8, # INT32 packing for W4
|
||||
dtype=params_dtype,
|
||||
),
|
||||
@@ -927,7 +930,8 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
|
||||
# Per-tensor fp8 weight scales
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.ones(num_experts, 2, dtype=torch.float32), requires_grad=False
|
||||
torch.ones(num_experts, self.moe.w13_num_shards, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
w2_weight_scale = torch.nn.Parameter(
|
||||
torch.ones(num_experts, dtype=torch.float32), requires_grad=False
|
||||
@@ -944,7 +948,7 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
w13_weight_scale_2 = torch.nn.Parameter(
|
||||
torch.ones(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -978,7 +982,7 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
for expert_id in range(layer.local_num_experts):
|
||||
start = 0
|
||||
max_w13_scale_fp8 = max_w13_scales[expert_id]
|
||||
for shard_id in range(2):
|
||||
for shard_id in range(self.moe.w13_num_shards):
|
||||
if layer.w13_weight_scale[expert_id][shard_id] != max_w13_scale_fp8:
|
||||
int4_rescale = (
|
||||
layer.w13_weight_scale[expert_id][shard_id] / max_w13_scale_fp8
|
||||
@@ -1193,7 +1197,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
self.get_packed_dim(hidden_size, self.weight_dtype),
|
||||
dtype=params_dtype,
|
||||
),
|
||||
@@ -1220,7 +1224,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.ones(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size // OCP_MX_BLOCK_SIZE,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
@@ -1245,7 +1249,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
w13_bias = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -1499,13 +1503,12 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
|
||||
layer.quant_config = self.quant_config
|
||||
weight_dtype = torch.uint8
|
||||
weight_scale_dtype = torch.float8_e4m3fn
|
||||
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
|
||||
|
||||
# GEMM 1 - w13 weight
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
# 2 fp4 items are packed in the input dimension
|
||||
hidden_size // 2,
|
||||
dtype=weight_dtype,
|
||||
@@ -1533,7 +1536,7 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size // self.group_size,
|
||||
dtype=weight_scale_dtype,
|
||||
),
|
||||
@@ -1563,7 +1566,7 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
|
||||
)
|
||||
|
||||
w13_weight_scale_2 = torch.nn.Parameter(
|
||||
torch.empty(num_experts, w13_num_shards, dtype=torch.float32),
|
||||
torch.empty(num_experts, self.moe.w13_num_shards, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight_scale_2", w13_weight_scale_2)
|
||||
@@ -1578,7 +1581,7 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
|
||||
|
||||
# Input global scales (per-tensor FP32 scales)
|
||||
w13_input_scale_2 = torch.nn.Parameter(
|
||||
torch.empty(num_experts, w13_num_shards, dtype=torch.float32),
|
||||
torch.empty(num_experts, self.moe.w13_num_shards, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_input_scale_2", w13_input_scale_2)
|
||||
|
||||
Reference in New Issue
Block a user