[Bugfix] Fix activation quantization dispatch for WNA4Int/WNA8Int (#48785)
Signed-off-by: HDCharles <charlesdavidhernandez@gmail.com>
This commit is contained in:
+2
-2
@@ -96,7 +96,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
|
||||
return 75
|
||||
|
||||
def _build_input_quant_config(self) -> dict | None:
|
||||
"""Build the config dict that HummingInputSchema.from_config expects."""
|
||||
"""Build the config dict that BaseInputSchema.from_config expects."""
|
||||
if self.input_quant is None:
|
||||
return None
|
||||
iq = self.input_quant
|
||||
@@ -112,7 +112,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
|
||||
"dynamic": iq.dynamic,
|
||||
"group_size": iq.group_size or 0,
|
||||
"quant_method": "compressed-tensors",
|
||||
"format": self.quant_format,
|
||||
"format": "int-quantized",
|
||||
}
|
||||
|
||||
def create_weights(
|
||||
|
||||
+2
-2
@@ -96,7 +96,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
|
||||
return 75
|
||||
|
||||
def _build_input_quant_config(self) -> dict | None:
|
||||
"""Build the config dict that HummingInputSchema.from_config expects."""
|
||||
"""Build the config dict that BaseInputSchema.from_config expects."""
|
||||
if self.input_quant is None:
|
||||
return None
|
||||
iq = self.input_quant
|
||||
@@ -112,7 +112,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
|
||||
"dynamic": iq.dynamic,
|
||||
"group_size": iq.group_size or 0,
|
||||
"quant_method": "compressed-tensors",
|
||||
"format": self.quant_format,
|
||||
"format": "int-quantized",
|
||||
}
|
||||
|
||||
def create_weights(
|
||||
|
||||
@@ -441,6 +441,7 @@ def prepare_humming_layer(
|
||||
input_quant_config: dict | None = None,
|
||||
):
|
||||
from vllm.utils.humming import (
|
||||
BaseInputSchema,
|
||||
BaseWeightSchema,
|
||||
HummingInputSchema,
|
||||
HummingMethod,
|
||||
@@ -448,7 +449,7 @@ def prepare_humming_layer(
|
||||
|
||||
weight_schema = BaseWeightSchema.from_config(quant_config)
|
||||
if input_quant_config is not None:
|
||||
input_schema = HummingInputSchema.from_config(input_quant_config)
|
||||
input_schema = BaseInputSchema.from_config(input_quant_config)
|
||||
else:
|
||||
input_schema = HummingInputSchema()
|
||||
|
||||
@@ -463,13 +464,19 @@ def prepare_humming_layer(
|
||||
shape_k_stacks = [input_size_per_partition]
|
||||
shape_n_stacks = layer.output_partition_sizes
|
||||
|
||||
# Step 1: convert weight to humming standard format
|
||||
# Step 1: convert weight and input schemas to humming standard format
|
||||
weight_schema, tensors = weight_schema.convert_humming(
|
||||
tensors=dict(layer.named_parameters()),
|
||||
shape_n_stacks=shape_n_stacks,
|
||||
shape_k_stacks=shape_k_stacks,
|
||||
param_dtype=layer.params_dtype,
|
||||
)
|
||||
input_schema, _ = input_schema.convert_humming(
|
||||
tensors={},
|
||||
shape_n_stacks=shape_n_stacks,
|
||||
shape_k_stacks=shape_k_stacks,
|
||||
param_dtype=layer.params_dtype,
|
||||
)
|
||||
|
||||
layer.weight_schema = weight_schema
|
||||
|
||||
|
||||
Reference in New Issue
Block a user