[Quant] Enable humming w[2-7]a[4,8] inference with compressed-tensors (#46390)
Signed-off-by: HDCharles <charlesdavidhernandez@gmail.com>
This commit is contained in:
@@ -435,7 +435,11 @@ def convert_linear_layer_to_humming_standard(
|
||||
setattr(layer, name, param)
|
||||
|
||||
|
||||
def prepare_humming_layer(layer: LinearBase, quant_config: dict):
|
||||
def prepare_humming_layer(
|
||||
layer: LinearBase,
|
||||
quant_config: dict,
|
||||
input_quant_config: dict | None = None,
|
||||
):
|
||||
from vllm.utils.humming import (
|
||||
BaseWeightSchema,
|
||||
HummingInputSchema,
|
||||
@@ -443,7 +447,10 @@ def prepare_humming_layer(layer: LinearBase, quant_config: dict):
|
||||
)
|
||||
|
||||
weight_schema = BaseWeightSchema.from_config(quant_config)
|
||||
input_schema = HummingInputSchema()
|
||||
if input_quant_config is not None:
|
||||
input_schema = HummingInputSchema.from_config(input_quant_config)
|
||||
else:
|
||||
input_schema = HummingInputSchema()
|
||||
|
||||
# ReplicatedLinear has no TP partitioning and so does not set
|
||||
# input_size_per_partition; for it that is just input_size. Use hasattr
|
||||
|
||||
Reference in New Issue
Block a user