[Bugfix] Fix activation quantization dispatch for WNA4Int/WNA8Int (#48785)
Signed-off-by: HDCharles <charlesdavidhernandez@gmail.com>
This commit is contained in:
@@ -441,6 +441,7 @@ def prepare_humming_layer(
|
||||
input_quant_config: dict | None = None,
|
||||
):
|
||||
from vllm.utils.humming import (
|
||||
BaseInputSchema,
|
||||
BaseWeightSchema,
|
||||
HummingInputSchema,
|
||||
HummingMethod,
|
||||
@@ -448,7 +449,7 @@ def prepare_humming_layer(
|
||||
|
||||
weight_schema = BaseWeightSchema.from_config(quant_config)
|
||||
if input_quant_config is not None:
|
||||
input_schema = HummingInputSchema.from_config(input_quant_config)
|
||||
input_schema = BaseInputSchema.from_config(input_quant_config)
|
||||
else:
|
||||
input_schema = HummingInputSchema()
|
||||
|
||||
@@ -463,13 +464,19 @@ def prepare_humming_layer(
|
||||
shape_k_stacks = [input_size_per_partition]
|
||||
shape_n_stacks = layer.output_partition_sizes
|
||||
|
||||
# Step 1: convert weight to humming standard format
|
||||
# Step 1: convert weight and input schemas to humming standard format
|
||||
weight_schema, tensors = weight_schema.convert_humming(
|
||||
tensors=dict(layer.named_parameters()),
|
||||
shape_n_stacks=shape_n_stacks,
|
||||
shape_k_stacks=shape_k_stacks,
|
||||
param_dtype=layer.params_dtype,
|
||||
)
|
||||
input_schema, _ = input_schema.convert_humming(
|
||||
tensors={},
|
||||
shape_n_stacks=shape_n_stacks,
|
||||
shape_k_stacks=shape_k_stacks,
|
||||
param_dtype=layer.params_dtype,
|
||||
)
|
||||
|
||||
layer.weight_schema = weight_schema
|
||||
|
||||
|
||||
Reference in New Issue
Block a user