[Quant] Enable humming w[2-7]a[4,8] inference with compressed-tensors (#46390)
Signed-off-by: HDCharles <charlesdavidhernandez@gmail.com>
This commit is contained in:
@@ -0,0 +1,8 @@
|
||||
model_name: "nm-testing/Qwen3-4B-mixed-quant-RTN-wNaM"
|
||||
accuracy_threshold: 0.85
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 4096
|
||||
--linear-backend humming
|
||||
@@ -2,3 +2,4 @@ Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming-act-int8.yaml
|
||||
Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml
|
||||
Qwen3-30B-A3B-AWQ-humming.yaml
|
||||
Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml
|
||||
Qwen3-4B-mixed-quant-RTN-humming.yaml
|
||||
|
||||
@@ -45,7 +45,10 @@ class HummingLinearKernel(MPLinearKernel):
|
||||
quant_config["has_zero_point"] = True
|
||||
|
||||
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
|
||||
prepare_humming_layer(layer, quant_config)
|
||||
input_quant_config = getattr(layer, "_humming_input_quant_config", None)
|
||||
prepare_humming_layer(
|
||||
layer, quant_config, input_quant_config=input_quant_config
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
|
||||
@@ -47,6 +47,8 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
|
||||
CompressedTensorsW8A8Int8,
|
||||
CompressedTensorsW8A8Mxfp8,
|
||||
CompressedTensorsW8A16Fp8,
|
||||
CompressedTensorsWNA4Int,
|
||||
CompressedTensorsWNA8Int,
|
||||
CompressedTensorsWNA8O8Int,
|
||||
CompressedTensorsWNA16,
|
||||
)
|
||||
@@ -682,6 +684,41 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
)
|
||||
return is_intN_weight and (is_static_int8_in and is_static_int8_out)
|
||||
|
||||
@staticmethod
|
||||
def _is_wNaM_int(
|
||||
weight_quant: QuantizationArgs,
|
||||
input_quant: QuantizationArgs | None,
|
||||
format: str | None,
|
||||
) -> bool:
|
||||
"""Weight N-bit INT with symmetric dynamic INT activation quant
|
||||
via Humming kernel."""
|
||||
if input_quant is None:
|
||||
return False
|
||||
is_pack_format = format == CompressionFormat.pack_quantized.value
|
||||
is_channel_group = weight_quant.strategy in (
|
||||
QuantizationStrategy.CHANNEL.value,
|
||||
QuantizationStrategy.GROUP.value,
|
||||
)
|
||||
is_int_N_weight = (
|
||||
weight_quant.type == QuantizationType.INT and not weight_quant.dynamic
|
||||
)
|
||||
is_int_input = input_quant.type == QuantizationType.INT
|
||||
is_symmetric_input = input_quant.symmetric
|
||||
is_dynamic_input = input_quant.dynamic
|
||||
is_per_token_or_group_input = input_quant.strategy in (
|
||||
QuantizationStrategy.TOKEN.value,
|
||||
QuantizationStrategy.GROUP.value,
|
||||
)
|
||||
return (
|
||||
is_int_N_weight
|
||||
and is_channel_group
|
||||
and is_pack_format
|
||||
and is_int_input
|
||||
and is_symmetric_input
|
||||
and is_dynamic_input
|
||||
and is_per_token_or_group_input
|
||||
)
|
||||
|
||||
def _get_scheme_from_parts(
|
||||
self,
|
||||
weight_quant: QuantizationArgs,
|
||||
@@ -733,6 +770,31 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
quant_format=format,
|
||||
)
|
||||
|
||||
if (
|
||||
self._is_wNaM_int(weight_quant, input_quant, format)
|
||||
and input_quant.num_bits == 8
|
||||
):
|
||||
return CompressedTensorsWNA8Int(
|
||||
num_bits=weight_quant.num_bits,
|
||||
strategy=weight_quant.strategy,
|
||||
group_size=weight_quant.group_size,
|
||||
input_quant=input_quant,
|
||||
layer_name=layer_name,
|
||||
quant_format=format,
|
||||
)
|
||||
if (
|
||||
self._is_wNaM_int(weight_quant, input_quant, format)
|
||||
and input_quant.num_bits == 4
|
||||
):
|
||||
return CompressedTensorsWNA4Int(
|
||||
num_bits=weight_quant.num_bits,
|
||||
strategy=weight_quant.strategy,
|
||||
group_size=weight_quant.group_size,
|
||||
input_quant=input_quant,
|
||||
layer_name=layer_name,
|
||||
quant_format=format,
|
||||
)
|
||||
|
||||
if self._is_wNa16_group_channel(weight_quant, input_quant) and (
|
||||
format == CompressionFormat.pack_quantized.value
|
||||
):
|
||||
|
||||
@@ -10,6 +10,8 @@ from .compressed_tensors_w8a8_fp8 import CompressedTensorsW8A8Fp8
|
||||
from .compressed_tensors_w8a8_int8 import CompressedTensorsW8A8Int8
|
||||
from .compressed_tensors_w8a8_mxfp8 import CompressedTensorsW8A8Mxfp8
|
||||
from .compressed_tensors_w8a16_fp8 import CompressedTensorsW8A16Fp8
|
||||
from .compressed_tensors_wNa4 import CompressedTensorsWNA4Int
|
||||
from .compressed_tensors_wNa8 import CompressedTensorsWNA8Int
|
||||
from .compressed_tensors_wNa8o8 import CompressedTensorsWNA8O8Int
|
||||
from .compressed_tensors_wNa16 import CompressedTensorsWNA16
|
||||
|
||||
@@ -25,4 +27,6 @@ __all__ = [
|
||||
"CompressedTensorsW4A8Int",
|
||||
"CompressedTensorsW4A8Fp8",
|
||||
"CompressedTensorsW8A8Mxfp8",
|
||||
"CompressedTensorsWNA4Int",
|
||||
"CompressedTensorsWNA8Int",
|
||||
]
|
||||
|
||||
+228
@@ -0,0 +1,228 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Weight N-bit INT scheme with symmetric INT4 activation quant via Humming.
|
||||
|
||||
Handles compressed-tensors pack-quantized INT weight checkpoints (2-8 bit)
|
||||
with INT4 symmetric dynamic per-token/per-group input activation
|
||||
quantization. Static, per-tensor, and asymmetric activation quantization
|
||||
are not supported.
|
||||
"""
|
||||
|
||||
import math
|
||||
from collections.abc import Callable
|
||||
from fractions import Fraction
|
||||
|
||||
import torch
|
||||
from compressed_tensors.quantization import (
|
||||
QuantizationArgs,
|
||||
QuantizationStrategy,
|
||||
)
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
MPLinearLayerConfig,
|
||||
choose_mp_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
|
||||
CompressedTensorsScheme,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa: E501
|
||||
WNA16_SUPPORTED_TYPES_MAP,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
marlin_repeat_scales_on_all_ranks,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
BasevLLMParameter,
|
||||
ChannelQuantScaleParameter,
|
||||
GroupQuantScaleParameter,
|
||||
PackedvLLMParameter,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
__all__ = ["CompressedTensorsWNA4Int"]
|
||||
|
||||
|
||||
class CompressedTensorsWNA4Int(CompressedTensorsScheme):
|
||||
_kernel_backends_being_used: set[str] = set()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_bits: int,
|
||||
strategy: str,
|
||||
group_size: int | None = None,
|
||||
input_quant: QuantizationArgs | None = None,
|
||||
layer_name: str | None = None,
|
||||
quant_format: str = "pack-quantized",
|
||||
):
|
||||
self.num_bits = num_bits
|
||||
self.pack_factor = Fraction(32, num_bits)
|
||||
self.strategy = strategy
|
||||
self.group_size = -1 if group_size is None else group_size
|
||||
self.input_quant = input_quant
|
||||
self.layer_name = layer_name
|
||||
self.quant_format = quant_format
|
||||
|
||||
if num_bits not in WNA16_SUPPORTED_TYPES_MAP:
|
||||
raise ValueError(
|
||||
f"Unsupported num_bits = {num_bits} for WNA4Int; "
|
||||
f"supported = {sorted(WNA16_SUPPORTED_TYPES_MAP)}"
|
||||
)
|
||||
self.quant_type = WNA16_SUPPORTED_TYPES_MAP[num_bits]
|
||||
|
||||
if input_quant is not None:
|
||||
if not input_quant.symmetric:
|
||||
raise ValueError(
|
||||
"WNA4Int requires symmetric activation quantization, "
|
||||
f"got symmetric={input_quant.symmetric}"
|
||||
)
|
||||
if not input_quant.dynamic:
|
||||
raise ValueError(
|
||||
"WNA4Int requires dynamic activation quantization, "
|
||||
f"got dynamic={input_quant.dynamic}"
|
||||
)
|
||||
if input_quant.strategy not in (
|
||||
QuantizationStrategy.TOKEN.value,
|
||||
QuantizationStrategy.GROUP.value,
|
||||
):
|
||||
raise ValueError(
|
||||
"WNA4Int requires per-token or per-group activation "
|
||||
f"quantization, got strategy={input_quant.strategy}"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return 75
|
||||
|
||||
def _build_input_quant_config(self) -> dict | None:
|
||||
"""Build the config dict that HummingInputSchema.from_config expects."""
|
||||
if self.input_quant is None:
|
||||
return None
|
||||
iq = self.input_quant
|
||||
type_val = iq.type.value if hasattr(iq.type, "value") else iq.type
|
||||
strategy_val = (
|
||||
iq.strategy.value if hasattr(iq.strategy, "value") else iq.strategy
|
||||
)
|
||||
return {
|
||||
"num_bits": iq.num_bits,
|
||||
"type": type_val,
|
||||
"strategy": strategy_val,
|
||||
"symmetric": iq.symmetric,
|
||||
"dynamic": iq.dynamic,
|
||||
"group_size": iq.group_size or 0,
|
||||
"quant_method": "compressed-tensors",
|
||||
"format": self.quant_format,
|
||||
}
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
output_size: int,
|
||||
input_size: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size_per_partition: int,
|
||||
params_dtype: torch.dtype,
|
||||
weight_loader: Callable,
|
||||
**kwargs,
|
||||
):
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
layer.input_size_per_partition = input_size_per_partition
|
||||
layer.output_size_per_partition = output_size_per_partition
|
||||
layer.output_partition_sizes = output_partition_sizes
|
||||
layer.params_dtype = params_dtype
|
||||
if not hasattr(layer, "has_bias"):
|
||||
layer.has_bias = False
|
||||
|
||||
mp_config = MPLinearLayerConfig(
|
||||
full_weight_shape=(input_size, output_size),
|
||||
partition_weight_shape=(
|
||||
input_size_per_partition,
|
||||
output_size_per_partition,
|
||||
),
|
||||
weight_type=self.quant_type,
|
||||
act_type=params_dtype,
|
||||
group_size=self.group_size,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
|
||||
kernel_type = choose_mp_linear_kernel(mp_config)
|
||||
if kernel_type.__name__ not in self._kernel_backends_being_used:
|
||||
logger.info("Using %s for CompressedTensorsWNA4Int", kernel_type.__name__)
|
||||
self._kernel_backends_being_used.add(kernel_type.__name__)
|
||||
|
||||
self.kernel = kernel_type(
|
||||
mp_config,
|
||||
w_q_param_name="weight_packed",
|
||||
w_s_param_name="weight_scale",
|
||||
)
|
||||
|
||||
input_quant_config = self._build_input_quant_config()
|
||||
if input_quant_config is not None:
|
||||
layer._humming_input_quant_config = input_quant_config
|
||||
|
||||
# --- weight parameters ---
|
||||
group_size = self.group_size if self.group_size != -1 else input_size
|
||||
row_parallel = input_size != input_size_per_partition
|
||||
partition_scales = not marlin_repeat_scales_on_all_ranks(
|
||||
False, self.group_size, row_parallel
|
||||
)
|
||||
scales_size = input_size // group_size
|
||||
if partition_scales:
|
||||
assert input_size_per_partition % group_size == 0
|
||||
scales_size = input_size_per_partition // group_size
|
||||
|
||||
packed_input_dim = math.ceil(input_size_per_partition * self.num_bits / 32)
|
||||
layer.register_parameter(
|
||||
"weight_packed",
|
||||
PackedvLLMParameter(
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
packed_factor=self.pack_factor,
|
||||
packed_dim=1,
|
||||
weight_loader=weight_loader,
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
packed_input_dim,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
scale_data = torch.empty(
|
||||
output_size_per_partition, scales_size, dtype=params_dtype
|
||||
)
|
||||
if partition_scales:
|
||||
weight_scale = GroupQuantScaleParameter(
|
||||
data=scale_data,
|
||||
output_dim=0,
|
||||
input_dim=1,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
else:
|
||||
weight_scale = ChannelQuantScaleParameter(
|
||||
data=scale_data,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
layer.register_parameter(
|
||||
"weight_shape",
|
||||
BasevLLMParameter(
|
||||
data=torch.empty(2, dtype=torch.int64),
|
||||
weight_loader=weight_loader,
|
||||
),
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
return self.kernel.apply_weights(layer, x, bias)
|
||||
+228
@@ -0,0 +1,228 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Weight N-bit INT scheme with symmetric INT8 activation quant via Humming.
|
||||
|
||||
Handles compressed-tensors pack-quantized INT weight checkpoints (2-8 bit)
|
||||
with INT8 symmetric dynamic per-token/per-group input activation
|
||||
quantization. Static, per-tensor, and asymmetric activation quantization
|
||||
are not supported.
|
||||
"""
|
||||
|
||||
import math
|
||||
from collections.abc import Callable
|
||||
from fractions import Fraction
|
||||
|
||||
import torch
|
||||
from compressed_tensors.quantization import (
|
||||
QuantizationArgs,
|
||||
QuantizationStrategy,
|
||||
)
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
MPLinearLayerConfig,
|
||||
choose_mp_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
|
||||
CompressedTensorsScheme,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa: E501
|
||||
WNA16_SUPPORTED_TYPES_MAP,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
marlin_repeat_scales_on_all_ranks,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
BasevLLMParameter,
|
||||
ChannelQuantScaleParameter,
|
||||
GroupQuantScaleParameter,
|
||||
PackedvLLMParameter,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
__all__ = ["CompressedTensorsWNA8Int"]
|
||||
|
||||
|
||||
class CompressedTensorsWNA8Int(CompressedTensorsScheme):
|
||||
_kernel_backends_being_used: set[str] = set()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_bits: int,
|
||||
strategy: str,
|
||||
group_size: int | None = None,
|
||||
input_quant: QuantizationArgs | None = None,
|
||||
layer_name: str | None = None,
|
||||
quant_format: str = "pack-quantized",
|
||||
):
|
||||
self.num_bits = num_bits
|
||||
self.pack_factor = Fraction(32, num_bits)
|
||||
self.strategy = strategy
|
||||
self.group_size = -1 if group_size is None else group_size
|
||||
self.input_quant = input_quant
|
||||
self.layer_name = layer_name
|
||||
self.quant_format = quant_format
|
||||
|
||||
if num_bits not in WNA16_SUPPORTED_TYPES_MAP:
|
||||
raise ValueError(
|
||||
f"Unsupported num_bits = {num_bits} for WNA8Int; "
|
||||
f"supported = {sorted(WNA16_SUPPORTED_TYPES_MAP)}"
|
||||
)
|
||||
self.quant_type = WNA16_SUPPORTED_TYPES_MAP[num_bits]
|
||||
|
||||
if input_quant is not None:
|
||||
if not input_quant.symmetric:
|
||||
raise ValueError(
|
||||
"WNA8Int requires symmetric activation quantization, "
|
||||
f"got symmetric={input_quant.symmetric}"
|
||||
)
|
||||
if not input_quant.dynamic:
|
||||
raise ValueError(
|
||||
"WNA8Int requires dynamic activation quantization, "
|
||||
f"got dynamic={input_quant.dynamic}"
|
||||
)
|
||||
if input_quant.strategy not in (
|
||||
QuantizationStrategy.TOKEN.value,
|
||||
QuantizationStrategy.GROUP.value,
|
||||
):
|
||||
raise ValueError(
|
||||
"WNA8Int requires per-token or per-group activation "
|
||||
f"quantization, got strategy={input_quant.strategy}"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return 75
|
||||
|
||||
def _build_input_quant_config(self) -> dict | None:
|
||||
"""Build the config dict that HummingInputSchema.from_config expects."""
|
||||
if self.input_quant is None:
|
||||
return None
|
||||
iq = self.input_quant
|
||||
type_val = iq.type.value if hasattr(iq.type, "value") else iq.type
|
||||
strategy_val = (
|
||||
iq.strategy.value if hasattr(iq.strategy, "value") else iq.strategy
|
||||
)
|
||||
return {
|
||||
"num_bits": iq.num_bits,
|
||||
"type": type_val,
|
||||
"strategy": strategy_val,
|
||||
"symmetric": iq.symmetric,
|
||||
"dynamic": iq.dynamic,
|
||||
"group_size": iq.group_size or 0,
|
||||
"quant_method": "compressed-tensors",
|
||||
"format": self.quant_format,
|
||||
}
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
output_size: int,
|
||||
input_size: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size_per_partition: int,
|
||||
params_dtype: torch.dtype,
|
||||
weight_loader: Callable,
|
||||
**kwargs,
|
||||
):
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
layer.input_size_per_partition = input_size_per_partition
|
||||
layer.output_size_per_partition = output_size_per_partition
|
||||
layer.output_partition_sizes = output_partition_sizes
|
||||
layer.params_dtype = params_dtype
|
||||
if not hasattr(layer, "has_bias"):
|
||||
layer.has_bias = False
|
||||
|
||||
mp_config = MPLinearLayerConfig(
|
||||
full_weight_shape=(input_size, output_size),
|
||||
partition_weight_shape=(
|
||||
input_size_per_partition,
|
||||
output_size_per_partition,
|
||||
),
|
||||
weight_type=self.quant_type,
|
||||
act_type=params_dtype,
|
||||
group_size=self.group_size,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
|
||||
kernel_type = choose_mp_linear_kernel(mp_config)
|
||||
if kernel_type.__name__ not in self._kernel_backends_being_used:
|
||||
logger.info("Using %s for CompressedTensorsWNA8Int", kernel_type.__name__)
|
||||
self._kernel_backends_being_used.add(kernel_type.__name__)
|
||||
|
||||
self.kernel = kernel_type(
|
||||
mp_config,
|
||||
w_q_param_name="weight_packed",
|
||||
w_s_param_name="weight_scale",
|
||||
)
|
||||
|
||||
input_quant_config = self._build_input_quant_config()
|
||||
if input_quant_config is not None:
|
||||
layer._humming_input_quant_config = input_quant_config
|
||||
|
||||
# --- weight parameters ---
|
||||
group_size = self.group_size if self.group_size != -1 else input_size
|
||||
row_parallel = input_size != input_size_per_partition
|
||||
partition_scales = not marlin_repeat_scales_on_all_ranks(
|
||||
False, self.group_size, row_parallel
|
||||
)
|
||||
scales_size = input_size // group_size
|
||||
if partition_scales:
|
||||
assert input_size_per_partition % group_size == 0
|
||||
scales_size = input_size_per_partition // group_size
|
||||
|
||||
packed_input_dim = math.ceil(input_size_per_partition * self.num_bits / 32)
|
||||
layer.register_parameter(
|
||||
"weight_packed",
|
||||
PackedvLLMParameter(
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
packed_factor=self.pack_factor,
|
||||
packed_dim=1,
|
||||
weight_loader=weight_loader,
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
packed_input_dim,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
scale_data = torch.empty(
|
||||
output_size_per_partition, scales_size, dtype=params_dtype
|
||||
)
|
||||
if partition_scales:
|
||||
weight_scale = GroupQuantScaleParameter(
|
||||
data=scale_data,
|
||||
output_dim=0,
|
||||
input_dim=1,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
else:
|
||||
weight_scale = ChannelQuantScaleParameter(
|
||||
data=scale_data,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
layer.register_parameter(
|
||||
"weight_shape",
|
||||
BasevLLMParameter(
|
||||
data=torch.empty(2, dtype=torch.int64),
|
||||
weight_loader=weight_loader,
|
||||
),
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
return self.kernel.apply_weights(layer, x, bias)
|
||||
@@ -435,7 +435,11 @@ def convert_linear_layer_to_humming_standard(
|
||||
setattr(layer, name, param)
|
||||
|
||||
|
||||
def prepare_humming_layer(layer: LinearBase, quant_config: dict):
|
||||
def prepare_humming_layer(
|
||||
layer: LinearBase,
|
||||
quant_config: dict,
|
||||
input_quant_config: dict | None = None,
|
||||
):
|
||||
from vllm.utils.humming import (
|
||||
BaseWeightSchema,
|
||||
HummingInputSchema,
|
||||
@@ -443,7 +447,10 @@ def prepare_humming_layer(layer: LinearBase, quant_config: dict):
|
||||
)
|
||||
|
||||
weight_schema = BaseWeightSchema.from_config(quant_config)
|
||||
input_schema = HummingInputSchema()
|
||||
if input_quant_config is not None:
|
||||
input_schema = HummingInputSchema.from_config(input_quant_config)
|
||||
else:
|
||||
input_schema = HummingInputSchema()
|
||||
|
||||
# ReplicatedLinear has no TP partitioning and so does not set
|
||||
# input_size_per_partition; for it that is just input_size. Use hasattr
|
||||
|
||||
Reference in New Issue
Block a user