From 9b2be4e9a55033a410a6f9e7c18eda5ce9105162 Mon Sep 17 00:00:00 2001 From: HDCharles <39544797+HDCharles@users.noreply.github.com> Date: Tue, 14 Jul 2026 22:22:31 -0400 Subject: [PATCH] [Quant] Enable humming w[2-7]a[4,8] inference with compressed-tensors (#46390) Signed-off-by: HDCharles --- .../Qwen3-4B-mixed-quant-RTN-humming.yaml | 8 + .../gsm8k/configs/humming/config-act-int8.txt | 1 + .../kernels/linear/mixed_precision/humming.py | 5 +- .../compressed_tensors/compressed_tensors.py | 62 +++++ .../compressed_tensors/schemes/__init__.py | 4 + .../schemes/compressed_tensors_wNa4.py | 228 ++++++++++++++++++ .../schemes/compressed_tensors_wNa8.py | 228 ++++++++++++++++++ .../quantization/utils/humming_utils.py | 11 +- 8 files changed, 544 insertions(+), 3 deletions(-) create mode 100644 tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-humming.yaml create mode 100644 vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa4.py create mode 100644 vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa8.py diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-humming.yaml new file mode 100644 index 0000000000..af768d214f --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-humming.yaml @@ -0,0 +1,8 @@ +model_name: "nm-testing/Qwen3-4B-mixed-quant-RTN-wNaM" +accuracy_threshold: 0.85 +num_questions: 1319 +num_fewshot: 5 +server_args: >- + --enforce-eager + --max-model-len 4096 + --linear-backend humming diff --git a/tests/evals/gsm8k/configs/humming/config-act-int8.txt b/tests/evals/gsm8k/configs/humming/config-act-int8.txt index b018b35cbf..38d8fc726d 100644 --- a/tests/evals/gsm8k/configs/humming/config-act-int8.txt +++ b/tests/evals/gsm8k/configs/humming/config-act-int8.txt @@ -2,3 +2,4 @@ Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming-act-int8.yaml Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml Qwen3-30B-A3B-AWQ-humming.yaml Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml +Qwen3-4B-mixed-quant-RTN-humming.yaml diff --git a/vllm/model_executor/kernels/linear/mixed_precision/humming.py b/vllm/model_executor/kernels/linear/mixed_precision/humming.py index 7f1a9024ee..ec93e935d8 100644 --- a/vllm/model_executor/kernels/linear/mixed_precision/humming.py +++ b/vllm/model_executor/kernels/linear/mixed_precision/humming.py @@ -45,7 +45,10 @@ class HummingLinearKernel(MPLinearKernel): quant_config["has_zero_point"] = True convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map) - prepare_humming_layer(layer, quant_config) + input_quant_config = getattr(layer, "_humming_input_quant_config", None) + prepare_humming_layer( + layer, quant_config, input_quant_config=input_quant_config + ) def apply_weights( self, diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py index 3789cd172f..6f1d20bd7f 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py @@ -47,6 +47,8 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import ( CompressedTensorsW8A8Int8, CompressedTensorsW8A8Mxfp8, CompressedTensorsW8A16Fp8, + CompressedTensorsWNA4Int, + CompressedTensorsWNA8Int, CompressedTensorsWNA8O8Int, CompressedTensorsWNA16, ) @@ -682,6 +684,41 @@ class CompressedTensorsConfig(QuantizationConfig): ) return is_intN_weight and (is_static_int8_in and is_static_int8_out) + @staticmethod + def _is_wNaM_int( + weight_quant: QuantizationArgs, + input_quant: QuantizationArgs | None, + format: str | None, + ) -> bool: + """Weight N-bit INT with symmetric dynamic INT activation quant + via Humming kernel.""" + if input_quant is None: + return False + is_pack_format = format == CompressionFormat.pack_quantized.value + is_channel_group = weight_quant.strategy in ( + QuantizationStrategy.CHANNEL.value, + QuantizationStrategy.GROUP.value, + ) + is_int_N_weight = ( + weight_quant.type == QuantizationType.INT and not weight_quant.dynamic + ) + is_int_input = input_quant.type == QuantizationType.INT + is_symmetric_input = input_quant.symmetric + is_dynamic_input = input_quant.dynamic + is_per_token_or_group_input = input_quant.strategy in ( + QuantizationStrategy.TOKEN.value, + QuantizationStrategy.GROUP.value, + ) + return ( + is_int_N_weight + and is_channel_group + and is_pack_format + and is_int_input + and is_symmetric_input + and is_dynamic_input + and is_per_token_or_group_input + ) + def _get_scheme_from_parts( self, weight_quant: QuantizationArgs, @@ -733,6 +770,31 @@ class CompressedTensorsConfig(QuantizationConfig): quant_format=format, ) + if ( + self._is_wNaM_int(weight_quant, input_quant, format) + and input_quant.num_bits == 8 + ): + return CompressedTensorsWNA8Int( + num_bits=weight_quant.num_bits, + strategy=weight_quant.strategy, + group_size=weight_quant.group_size, + input_quant=input_quant, + layer_name=layer_name, + quant_format=format, + ) + if ( + self._is_wNaM_int(weight_quant, input_quant, format) + and input_quant.num_bits == 4 + ): + return CompressedTensorsWNA4Int( + num_bits=weight_quant.num_bits, + strategy=weight_quant.strategy, + group_size=weight_quant.group_size, + input_quant=input_quant, + layer_name=layer_name, + quant_format=format, + ) + if self._is_wNa16_group_channel(weight_quant, input_quant) and ( format == CompressionFormat.pack_quantized.value ): diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py index 2826bf7b47..0f46d581f0 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py @@ -10,6 +10,8 @@ from .compressed_tensors_w8a8_fp8 import CompressedTensorsW8A8Fp8 from .compressed_tensors_w8a8_int8 import CompressedTensorsW8A8Int8 from .compressed_tensors_w8a8_mxfp8 import CompressedTensorsW8A8Mxfp8 from .compressed_tensors_w8a16_fp8 import CompressedTensorsW8A16Fp8 +from .compressed_tensors_wNa4 import CompressedTensorsWNA4Int +from .compressed_tensors_wNa8 import CompressedTensorsWNA8Int from .compressed_tensors_wNa8o8 import CompressedTensorsWNA8O8Int from .compressed_tensors_wNa16 import CompressedTensorsWNA16 @@ -25,4 +27,6 @@ __all__ = [ "CompressedTensorsW4A8Int", "CompressedTensorsW4A8Fp8", "CompressedTensorsW8A8Mxfp8", + "CompressedTensorsWNA4Int", + "CompressedTensorsWNA8Int", ] diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa4.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa4.py new file mode 100644 index 0000000000..aee482a145 --- /dev/null +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa4.py @@ -0,0 +1,228 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Weight N-bit INT scheme with symmetric INT4 activation quant via Humming. + +Handles compressed-tensors pack-quantized INT weight checkpoints (2-8 bit) +with INT4 symmetric dynamic per-token/per-group input activation +quantization. Static, per-tensor, and asymmetric activation quantization +are not supported. +""" + +import math +from collections.abc import Callable +from fractions import Fraction + +import torch +from compressed_tensors.quantization import ( + QuantizationArgs, + QuantizationStrategy, +) + +from vllm.logger import init_logger +from vllm.model_executor.kernels.linear import ( + MPLinearLayerConfig, + choose_mp_linear_kernel, +) +from vllm.model_executor.layers.quantization.compressed_tensors.schemes import ( + CompressedTensorsScheme, +) +from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa: E501 + WNA16_SUPPORTED_TYPES_MAP, +) +from vllm.model_executor.layers.quantization.utils.marlin_utils import ( + marlin_repeat_scales_on_all_ranks, +) +from vllm.model_executor.parameter import ( + BasevLLMParameter, + ChannelQuantScaleParameter, + GroupQuantScaleParameter, + PackedvLLMParameter, +) + +logger = init_logger(__name__) + +__all__ = ["CompressedTensorsWNA4Int"] + + +class CompressedTensorsWNA4Int(CompressedTensorsScheme): + _kernel_backends_being_used: set[str] = set() + + def __init__( + self, + num_bits: int, + strategy: str, + group_size: int | None = None, + input_quant: QuantizationArgs | None = None, + layer_name: str | None = None, + quant_format: str = "pack-quantized", + ): + self.num_bits = num_bits + self.pack_factor = Fraction(32, num_bits) + self.strategy = strategy + self.group_size = -1 if group_size is None else group_size + self.input_quant = input_quant + self.layer_name = layer_name + self.quant_format = quant_format + + if num_bits not in WNA16_SUPPORTED_TYPES_MAP: + raise ValueError( + f"Unsupported num_bits = {num_bits} for WNA4Int; " + f"supported = {sorted(WNA16_SUPPORTED_TYPES_MAP)}" + ) + self.quant_type = WNA16_SUPPORTED_TYPES_MAP[num_bits] + + if input_quant is not None: + if not input_quant.symmetric: + raise ValueError( + "WNA4Int requires symmetric activation quantization, " + f"got symmetric={input_quant.symmetric}" + ) + if not input_quant.dynamic: + raise ValueError( + "WNA4Int requires dynamic activation quantization, " + f"got dynamic={input_quant.dynamic}" + ) + if input_quant.strategy not in ( + QuantizationStrategy.TOKEN.value, + QuantizationStrategy.GROUP.value, + ): + raise ValueError( + "WNA4Int requires per-token or per-group activation " + f"quantization, got strategy={input_quant.strategy}" + ) + + @classmethod + def get_min_capability(cls) -> int: + return 75 + + def _build_input_quant_config(self) -> dict | None: + """Build the config dict that HummingInputSchema.from_config expects.""" + if self.input_quant is None: + return None + iq = self.input_quant + type_val = iq.type.value if hasattr(iq.type, "value") else iq.type + strategy_val = ( + iq.strategy.value if hasattr(iq.strategy, "value") else iq.strategy + ) + return { + "num_bits": iq.num_bits, + "type": type_val, + "strategy": strategy_val, + "symmetric": iq.symmetric, + "dynamic": iq.dynamic, + "group_size": iq.group_size or 0, + "quant_method": "compressed-tensors", + "format": self.quant_format, + } + + def create_weights( + self, + layer: torch.nn.Module, + output_size: int, + input_size: int, + output_partition_sizes: list[int], + input_size_per_partition: int, + params_dtype: torch.dtype, + weight_loader: Callable, + **kwargs, + ): + output_size_per_partition = sum(output_partition_sizes) + layer.input_size_per_partition = input_size_per_partition + layer.output_size_per_partition = output_size_per_partition + layer.output_partition_sizes = output_partition_sizes + layer.params_dtype = params_dtype + if not hasattr(layer, "has_bias"): + layer.has_bias = False + + mp_config = MPLinearLayerConfig( + full_weight_shape=(input_size, output_size), + partition_weight_shape=( + input_size_per_partition, + output_size_per_partition, + ), + weight_type=self.quant_type, + act_type=params_dtype, + group_size=self.group_size, + zero_points=False, + has_g_idx=False, + ) + + kernel_type = choose_mp_linear_kernel(mp_config) + if kernel_type.__name__ not in self._kernel_backends_being_used: + logger.info("Using %s for CompressedTensorsWNA4Int", kernel_type.__name__) + self._kernel_backends_being_used.add(kernel_type.__name__) + + self.kernel = kernel_type( + mp_config, + w_q_param_name="weight_packed", + w_s_param_name="weight_scale", + ) + + input_quant_config = self._build_input_quant_config() + if input_quant_config is not None: + layer._humming_input_quant_config = input_quant_config + + # --- weight parameters --- + group_size = self.group_size if self.group_size != -1 else input_size + row_parallel = input_size != input_size_per_partition + partition_scales = not marlin_repeat_scales_on_all_ranks( + False, self.group_size, row_parallel + ) + scales_size = input_size // group_size + if partition_scales: + assert input_size_per_partition % group_size == 0 + scales_size = input_size_per_partition // group_size + + packed_input_dim = math.ceil(input_size_per_partition * self.num_bits / 32) + layer.register_parameter( + "weight_packed", + PackedvLLMParameter( + input_dim=1, + output_dim=0, + packed_factor=self.pack_factor, + packed_dim=1, + weight_loader=weight_loader, + data=torch.empty( + output_size_per_partition, + packed_input_dim, + dtype=torch.int32, + ), + ), + ) + + scale_data = torch.empty( + output_size_per_partition, scales_size, dtype=params_dtype + ) + if partition_scales: + weight_scale = GroupQuantScaleParameter( + data=scale_data, + output_dim=0, + input_dim=1, + weight_loader=weight_loader, + ) + else: + weight_scale = ChannelQuantScaleParameter( + data=scale_data, + output_dim=0, + weight_loader=weight_loader, + ) + layer.register_parameter("weight_scale", weight_scale) + + layer.register_parameter( + "weight_shape", + BasevLLMParameter( + data=torch.empty(2, dtype=torch.int64), + weight_loader=weight_loader, + ), + ) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + self.kernel.process_weights_after_loading(layer) + + def apply_weights( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None, + ) -> torch.Tensor: + return self.kernel.apply_weights(layer, x, bias) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa8.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa8.py new file mode 100644 index 0000000000..57988f66f7 --- /dev/null +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa8.py @@ -0,0 +1,228 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Weight N-bit INT scheme with symmetric INT8 activation quant via Humming. + +Handles compressed-tensors pack-quantized INT weight checkpoints (2-8 bit) +with INT8 symmetric dynamic per-token/per-group input activation +quantization. Static, per-tensor, and asymmetric activation quantization +are not supported. +""" + +import math +from collections.abc import Callable +from fractions import Fraction + +import torch +from compressed_tensors.quantization import ( + QuantizationArgs, + QuantizationStrategy, +) + +from vllm.logger import init_logger +from vllm.model_executor.kernels.linear import ( + MPLinearLayerConfig, + choose_mp_linear_kernel, +) +from vllm.model_executor.layers.quantization.compressed_tensors.schemes import ( + CompressedTensorsScheme, +) +from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa: E501 + WNA16_SUPPORTED_TYPES_MAP, +) +from vllm.model_executor.layers.quantization.utils.marlin_utils import ( + marlin_repeat_scales_on_all_ranks, +) +from vllm.model_executor.parameter import ( + BasevLLMParameter, + ChannelQuantScaleParameter, + GroupQuantScaleParameter, + PackedvLLMParameter, +) + +logger = init_logger(__name__) + +__all__ = ["CompressedTensorsWNA8Int"] + + +class CompressedTensorsWNA8Int(CompressedTensorsScheme): + _kernel_backends_being_used: set[str] = set() + + def __init__( + self, + num_bits: int, + strategy: str, + group_size: int | None = None, + input_quant: QuantizationArgs | None = None, + layer_name: str | None = None, + quant_format: str = "pack-quantized", + ): + self.num_bits = num_bits + self.pack_factor = Fraction(32, num_bits) + self.strategy = strategy + self.group_size = -1 if group_size is None else group_size + self.input_quant = input_quant + self.layer_name = layer_name + self.quant_format = quant_format + + if num_bits not in WNA16_SUPPORTED_TYPES_MAP: + raise ValueError( + f"Unsupported num_bits = {num_bits} for WNA8Int; " + f"supported = {sorted(WNA16_SUPPORTED_TYPES_MAP)}" + ) + self.quant_type = WNA16_SUPPORTED_TYPES_MAP[num_bits] + + if input_quant is not None: + if not input_quant.symmetric: + raise ValueError( + "WNA8Int requires symmetric activation quantization, " + f"got symmetric={input_quant.symmetric}" + ) + if not input_quant.dynamic: + raise ValueError( + "WNA8Int requires dynamic activation quantization, " + f"got dynamic={input_quant.dynamic}" + ) + if input_quant.strategy not in ( + QuantizationStrategy.TOKEN.value, + QuantizationStrategy.GROUP.value, + ): + raise ValueError( + "WNA8Int requires per-token or per-group activation " + f"quantization, got strategy={input_quant.strategy}" + ) + + @classmethod + def get_min_capability(cls) -> int: + return 75 + + def _build_input_quant_config(self) -> dict | None: + """Build the config dict that HummingInputSchema.from_config expects.""" + if self.input_quant is None: + return None + iq = self.input_quant + type_val = iq.type.value if hasattr(iq.type, "value") else iq.type + strategy_val = ( + iq.strategy.value if hasattr(iq.strategy, "value") else iq.strategy + ) + return { + "num_bits": iq.num_bits, + "type": type_val, + "strategy": strategy_val, + "symmetric": iq.symmetric, + "dynamic": iq.dynamic, + "group_size": iq.group_size or 0, + "quant_method": "compressed-tensors", + "format": self.quant_format, + } + + def create_weights( + self, + layer: torch.nn.Module, + output_size: int, + input_size: int, + output_partition_sizes: list[int], + input_size_per_partition: int, + params_dtype: torch.dtype, + weight_loader: Callable, + **kwargs, + ): + output_size_per_partition = sum(output_partition_sizes) + layer.input_size_per_partition = input_size_per_partition + layer.output_size_per_partition = output_size_per_partition + layer.output_partition_sizes = output_partition_sizes + layer.params_dtype = params_dtype + if not hasattr(layer, "has_bias"): + layer.has_bias = False + + mp_config = MPLinearLayerConfig( + full_weight_shape=(input_size, output_size), + partition_weight_shape=( + input_size_per_partition, + output_size_per_partition, + ), + weight_type=self.quant_type, + act_type=params_dtype, + group_size=self.group_size, + zero_points=False, + has_g_idx=False, + ) + + kernel_type = choose_mp_linear_kernel(mp_config) + if kernel_type.__name__ not in self._kernel_backends_being_used: + logger.info("Using %s for CompressedTensorsWNA8Int", kernel_type.__name__) + self._kernel_backends_being_used.add(kernel_type.__name__) + + self.kernel = kernel_type( + mp_config, + w_q_param_name="weight_packed", + w_s_param_name="weight_scale", + ) + + input_quant_config = self._build_input_quant_config() + if input_quant_config is not None: + layer._humming_input_quant_config = input_quant_config + + # --- weight parameters --- + group_size = self.group_size if self.group_size != -1 else input_size + row_parallel = input_size != input_size_per_partition + partition_scales = not marlin_repeat_scales_on_all_ranks( + False, self.group_size, row_parallel + ) + scales_size = input_size // group_size + if partition_scales: + assert input_size_per_partition % group_size == 0 + scales_size = input_size_per_partition // group_size + + packed_input_dim = math.ceil(input_size_per_partition * self.num_bits / 32) + layer.register_parameter( + "weight_packed", + PackedvLLMParameter( + input_dim=1, + output_dim=0, + packed_factor=self.pack_factor, + packed_dim=1, + weight_loader=weight_loader, + data=torch.empty( + output_size_per_partition, + packed_input_dim, + dtype=torch.int32, + ), + ), + ) + + scale_data = torch.empty( + output_size_per_partition, scales_size, dtype=params_dtype + ) + if partition_scales: + weight_scale = GroupQuantScaleParameter( + data=scale_data, + output_dim=0, + input_dim=1, + weight_loader=weight_loader, + ) + else: + weight_scale = ChannelQuantScaleParameter( + data=scale_data, + output_dim=0, + weight_loader=weight_loader, + ) + layer.register_parameter("weight_scale", weight_scale) + + layer.register_parameter( + "weight_shape", + BasevLLMParameter( + data=torch.empty(2, dtype=torch.int64), + weight_loader=weight_loader, + ), + ) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + self.kernel.process_weights_after_loading(layer) + + def apply_weights( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None, + ) -> torch.Tensor: + return self.kernel.apply_weights(layer, x, bias) diff --git a/vllm/model_executor/layers/quantization/utils/humming_utils.py b/vllm/model_executor/layers/quantization/utils/humming_utils.py index 2d9e7aca76..4ed7adcf43 100644 --- a/vllm/model_executor/layers/quantization/utils/humming_utils.py +++ b/vllm/model_executor/layers/quantization/utils/humming_utils.py @@ -435,7 +435,11 @@ def convert_linear_layer_to_humming_standard( setattr(layer, name, param) -def prepare_humming_layer(layer: LinearBase, quant_config: dict): +def prepare_humming_layer( + layer: LinearBase, + quant_config: dict, + input_quant_config: dict | None = None, +): from vllm.utils.humming import ( BaseWeightSchema, HummingInputSchema, @@ -443,7 +447,10 @@ def prepare_humming_layer(layer: LinearBase, quant_config: dict): ) weight_schema = BaseWeightSchema.from_config(quant_config) - input_schema = HummingInputSchema() + if input_quant_config is not None: + input_schema = HummingInputSchema.from_config(input_quant_config) + else: + input_schema = HummingInputSchema() # ReplicatedLinear has no TP partitioning and so does not set # input_size_per_partition; for it that is just input_size. Use hasattr