[Bugfix][Quantization] Add Humming to the W4A8 (INT4xFP8) MoE oracle (#58427)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -511,6 +511,13 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
def _is_dynamic_token_w4a8_int(
|
||||
weight_quant: QuantizationArgs, input_quant: QuantizationArgs
|
||||
) -> bool:
|
||||
if not weight_quant or not input_quant:
|
||||
return False
|
||||
if (
|
||||
weight_quant.type != QuantizationType.INT
|
||||
or input_quant.type != QuantizationType.INT
|
||||
):
|
||||
return False
|
||||
is_weight_4_bits = weight_quant.num_bits == 4
|
||||
is_activation_8_bits = input_quant.num_bits == 8
|
||||
weight_strategy = (
|
||||
@@ -575,6 +582,11 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
) -> bool:
|
||||
if not weight_quant or not input_quant:
|
||||
return False
|
||||
if (
|
||||
weight_quant.type != QuantizationType.INT
|
||||
or input_quant.type != QuantizationType.FLOAT
|
||||
):
|
||||
return False
|
||||
is_weight_4_bits = weight_quant.num_bits == 4
|
||||
is_activation_8_bits = input_quant.num_bits == 8
|
||||
weight_strategy = weight_quant.strategy == QuantizationStrategy.GROUP.value
|
||||
|
||||
+1
-1
@@ -164,7 +164,7 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
|
||||
return CompressedTensorsW8A8Int8MoEMethod(
|
||||
weight_quant, input_quant, layer.moe_config
|
||||
)
|
||||
elif quant_config._is_fp8_w4a8_sm90(weight_quant, input_quant):
|
||||
elif quant_config._is_fp8_w4a8(weight_quant, input_quant):
|
||||
from .compressed_tensors_moe_w4a8_fp8 import (
|
||||
CompressedTensorsW4A8Fp8MoEMethod,
|
||||
)
|
||||
|
||||
+27
-33
@@ -143,57 +143,49 @@ class CompressedTensorsW4A8Fp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
layer.register_parameter("w13_weight_shape", w13_weight_shape)
|
||||
set_weight_attrs(w13_weight_shape, extra_weight_attrs)
|
||||
|
||||
w13_weight_chan_scale = torch.nn.Parameter(
|
||||
torch.ones(
|
||||
num_experts,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight_chan_scale", w13_weight_chan_scale)
|
||||
|
||||
w2_weight_chan_scale = torch.nn.Parameter(
|
||||
torch.ones(num_experts, hidden_size, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight_chan_scale", w2_weight_chan_scale)
|
||||
|
||||
# don't use input scales
|
||||
layer.w13_input_scale = None
|
||||
layer.w2_input_scale = None
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
for name in ("w13", "w2"):
|
||||
setattr(layer, f"{name}_weight", getattr(layer, f"{name}_weight_packed"))
|
||||
delattr(layer, f"{name}_weight_packed")
|
||||
|
||||
(
|
||||
w13_weight_packed,
|
||||
w2_weight_packed,
|
||||
w13_weight,
|
||||
w2_weight,
|
||||
w13_weight_scale,
|
||||
w2_weight_scale,
|
||||
w13_weight_chan_scale,
|
||||
w2_weight_chan_scale,
|
||||
b_strides1,
|
||||
b_strides2,
|
||||
self.b_strides1,
|
||||
self.b_strides2,
|
||||
) = convert_to_w4a8_moe_kernel_format(
|
||||
w13_weight_packed=layer.w13_weight_packed,
|
||||
w2_weight_packed=layer.w2_weight_packed,
|
||||
backend=self.w4a8_backend,
|
||||
layer=layer,
|
||||
group_size=self.group_size,
|
||||
w13_weight=layer.w13_weight,
|
||||
w2_weight=layer.w2_weight,
|
||||
w13_weight_scale=layer.w13_weight_scale,
|
||||
w2_weight_scale=layer.w2_weight_scale,
|
||||
)
|
||||
|
||||
replace_parameter(layer, "w13_weight_packed", w13_weight_packed)
|
||||
replace_parameter(layer, "w2_weight_packed", w2_weight_packed)
|
||||
replace_parameter(layer, "w13_weight_scale", w13_weight_scale)
|
||||
replace_parameter(layer, "w2_weight_scale", w2_weight_scale)
|
||||
replace_parameter(layer, "w13_weight_chan_scale", w13_weight_chan_scale)
|
||||
replace_parameter(layer, "w2_weight_chan_scale", w2_weight_chan_scale)
|
||||
|
||||
self.b_strides1 = b_strides1
|
||||
self.b_strides2 = b_strides2
|
||||
for name, tensor in (
|
||||
("w13_weight", w13_weight),
|
||||
("w2_weight", w2_weight),
|
||||
("w13_weight_scale", w13_weight_scale),
|
||||
("w2_weight_scale", w2_weight_scale),
|
||||
("w13_weight_chan_scale", w13_weight_chan_scale),
|
||||
("w2_weight_chan_scale", w2_weight_chan_scale),
|
||||
):
|
||||
replace_parameter(layer, name, tensor)
|
||||
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
if self.moe_quant_config is not None:
|
||||
assert self.experts_cls is not None
|
||||
self.moe_kernel = make_w4a8_moe_kernel(
|
||||
backend=self.w4a8_backend,
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
@@ -205,10 +197,12 @@ class CompressedTensorsW4A8Fp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
|
||||
def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig:
|
||||
return make_w4a8_moe_quant_config(
|
||||
backend=self.w4a8_backend,
|
||||
w1_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
g1_alphas=layer.w13_weight_chan_scale,
|
||||
g2_alphas=layer.w2_weight_chan_scale,
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def apply(
|
||||
@@ -224,8 +218,8 @@ class CompressedTensorsW4A8Fp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
hidden_states=x,
|
||||
w1=layer.w13_weight_packed,
|
||||
w2=layer.w2_weight_packed,
|
||||
w1=layer.w13_weight,
|
||||
w2=layer.w2_weight,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=layer.activation,
|
||||
|
||||
Reference in New Issue
Block a user