diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index f1ef63b4ed..9d27b2e83a 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -571,7 +571,7 @@ steps: - label: ":nvidia: (H100) LM Eval Humming Activation FP8/INT8 TEMPORARY" key: lm-eval-humming-h100-act optional: true - timeout_in_minutes: 70 + timeout_in_minutes: 90 device: h100 num_devices: 1 source_file_dependencies: @@ -582,7 +582,7 @@ steps: - vllm/model_executor/kernels/linear/ commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt - # - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt - label: ":nvidia: (H100) LM Eval Humming DeepEP TEMPORARY" key: lm-eval-humming-h100-deepep diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-humming.yaml index 502ec79522..e41aa130ae 100644 --- a/tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-humming.yaml +++ b/tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-humming.yaml @@ -1,5 +1,5 @@ -model_name: "nm-testing/Qwen3-4B-mixed-quant-RTN-wNaM" -accuracy_threshold: 0.78 +model_name: "nm-testing/Qwen3-4B-mixed-quant-RTN-wNaM-a4" +accuracy_threshold: 0.70 num_questions: 1319 num_fewshot: 5 server_args: >- diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-no-w4a4-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-no-w4a4-humming.yaml new file mode 100644 index 0000000000..965a0fe54d --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-no-w4a4-humming.yaml @@ -0,0 +1,8 @@ +model_name: "nm-testing/Qwen3-4B-mixed-quant-RTN-wNaM-asym" +accuracy_threshold: 0.85 +num_questions: 1319 +num_fewshot: 5 +server_args: >- + --enforce-eager + --max-model-len 4096 + --linear-backend humming diff --git a/tests/evals/gsm8k/configs/humming/config-a100-act-int8-shard-0.txt b/tests/evals/gsm8k/configs/humming/config-a100-act-int8-shard-0.txt index 0537ed974a..0852d14633 100644 --- a/tests/evals/gsm8k/configs/humming/config-a100-act-int8-shard-0.txt +++ b/tests/evals/gsm8k/configs/humming/config-a100-act-int8-shard-0.txt @@ -1,3 +1,4 @@ gpt-oss-20b-humming.yaml Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml Qwen3.5-35B-A3B-int6wc-hadamard-humming-act-int8.yaml +Qwen3-4B-mixed-quant-RTN-no-w4a4-humming.yaml diff --git a/tests/evals/gsm8k/configs/humming/config-act-int8.txt b/tests/evals/gsm8k/configs/humming/config-act-int8.txt index 68f5cd1c15..7a608c7c5a 100644 --- a/tests/evals/gsm8k/configs/humming/config-act-int8.txt +++ b/tests/evals/gsm8k/configs/humming/config-act-int8.txt @@ -4,4 +4,4 @@ Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml Qwen3-30B-A3B-AWQ-humming.yaml Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml Qwen3.5-35B-A3B-int6wc-hadamard-humming-act-int8.yaml -Qwen3-4B-mixed-quant-RTN-humming.yaml +Qwen3-4B-mixed-quant-RTN-no-w4a4-humming.yaml diff --git a/vllm/model_executor/kernels/linear/mixed_precision/humming.py b/vllm/model_executor/kernels/linear/mixed_precision/humming.py index b98dfee18b..4655181f78 100644 --- a/vllm/model_executor/kernels/linear/mixed_precision/humming.py +++ b/vllm/model_executor/kernels/linear/mixed_precision/humming.py @@ -31,11 +31,14 @@ class HummingLinearKernel(MPLinearKernel): ) name_map = {"weight": self.w_q_name, "weight_scale": self.w_s_name} + if self.w_zp_name is not None and hasattr(layer, self.w_zp_name): + name_map["zero_point"] = self.w_zp_name group_size = self.config.group_size quant_config = { "quant_method": "humming", "dtype": "int" + str(self.config.weight_type.size_bits), "group_size": 0 if group_size == -1 else group_size, + "has_zero_point": self.config.zero_points, } if self.config.zero_points: diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py index 3603fd8bde..a4cd5d4d13 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py @@ -787,6 +787,7 @@ class CompressedTensorsConfig(QuantizationConfig): num_bits=weight_quant.num_bits, strategy=weight_quant.strategy, group_size=weight_quant.group_size, + symmetric=weight_quant.symmetric, input_quant=input_quant, layer_name=layer_name, quant_format=format, @@ -799,6 +800,7 @@ class CompressedTensorsConfig(QuantizationConfig): num_bits=weight_quant.num_bits, strategy=weight_quant.strategy, group_size=weight_quant.group_size, + symmetric=weight_quant.symmetric, input_quant=input_quant, layer_name=layer_name, quant_format=format, diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py index 42688ab834..8cfce07544 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py @@ -42,7 +42,15 @@ WNA16_SUPPORTED_TYPES_MAP = { 7: scalar_types.uint7b64, 8: scalar_types.uint8b128, } -WNA16_ZP_SUPPORTED_TYPES_MAP = {4: scalar_types.uint4, 8: scalar_types.uint8} +WNA16_ZP_SUPPORTED_TYPES_MAP = { + 2: scalar_types.uint2, + 3: scalar_types.uint3, + 4: scalar_types.uint4, + 5: scalar_types.uint5, + 6: scalar_types.uint6, + 7: scalar_types.uint7, + 8: scalar_types.uint8, +} WNA16_SUPPORTED_BITS = list(WNA16_SUPPORTED_TYPES_MAP.keys()) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa4.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa4.py index 427c92310f..acd4a96701 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa4.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa4.py @@ -1,11 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Weight N-bit INT scheme with symmetric INT4 activation quant via Humming. +"""Weight N-bit INT scheme with INT4 activation quant via Humming. Handles compressed-tensors pack-quantized INT weight checkpoints (2-8 bit) with INT4 symmetric dynamic per-token/per-group input activation -quantization. Static, per-tensor, and asymmetric activation quantization -are not supported. +quantization. Supports both symmetric and asymmetric weight quantization. +Static, per-tensor, and asymmetric activation quantization are not supported. """ import math @@ -28,6 +28,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import ( ) from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa: E501 WNA16_SUPPORTED_TYPES_MAP, + WNA16_ZP_SUPPORTED_TYPES_MAP, ) from vllm.model_executor.layers.quantization.utils.marlin_utils import ( marlin_repeat_scales_on_all_ranks, @@ -36,6 +37,7 @@ from vllm.model_executor.parameter import ( BasevLLMParameter, ChannelQuantScaleParameter, GroupQuantScaleParameter, + PackedColumnParameter, PackedvLLMParameter, ) @@ -52,6 +54,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme): num_bits: int, strategy: str, group_size: int | None = None, + symmetric: bool = True, input_quant: QuantizationArgs | None = None, layer_name: str | None = None, quant_format: str = "pack-quantized", @@ -60,6 +63,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme): self.pack_factor = Fraction(32, num_bits) self.strategy = strategy self.group_size = -1 if group_size is None else group_size + self.symmetric = symmetric self.input_quant = input_quant self.layer_name = layer_name self.quant_format = quant_format @@ -69,7 +73,19 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme): f"Unsupported num_bits = {num_bits} for WNA4Int; " f"supported = {sorted(WNA16_SUPPORTED_TYPES_MAP)}" ) - self.quant_type = WNA16_SUPPORTED_TYPES_MAP[num_bits] + + if not self.symmetric and num_bits not in WNA16_ZP_SUPPORTED_TYPES_MAP: + raise ValueError( + f"Asymmetric quantization not supported for " + f"num_bits = {num_bits}. Supported: " + f"{list(WNA16_ZP_SUPPORTED_TYPES_MAP)}" + ) + + self.quant_type = ( + WNA16_ZP_SUPPORTED_TYPES_MAP[num_bits] + if not self.symmetric + else WNA16_SUPPORTED_TYPES_MAP[num_bits] + ) if input_quant is not None: if not input_quant.symmetric: @@ -143,7 +159,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme): weight_type=self.quant_type, act_type=params_dtype, group_size=self.group_size, - zero_points=False, + zero_points=not self.symmetric, ) kernel_type = choose_mp_linear_kernel(mp_config) @@ -151,12 +167,6 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme): logger.info("Using %s for CompressedTensorsWNA4Int", kernel_type.__name__) self._kernel_backends_being_used.add(kernel_type.__name__) - self.kernel = kernel_type( - mp_config, - w_q_param_name="weight_packed", - w_s_param_name="weight_scale", - ) - input_quant_config = self._build_input_quant_config() if input_quant_config is not None: layer._humming_input_quant_config = input_quant_config @@ -167,10 +177,10 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme): partition_scales = not marlin_repeat_scales_on_all_ranks( self.group_size, row_parallel ) - scales_size = input_size // group_size + scales_and_zp_size = input_size // group_size if partition_scales: assert input_size_per_partition % group_size == 0 - scales_size = input_size_per_partition // group_size + scales_and_zp_size = input_size_per_partition // group_size packed_input_dim = math.ceil(input_size_per_partition * self.num_bits / 32) layer.register_parameter( @@ -190,7 +200,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme): ) scale_data = torch.empty( - output_size_per_partition, scales_size, dtype=params_dtype + output_size_per_partition, scales_and_zp_size, dtype=params_dtype ) if partition_scales: weight_scale = GroupQuantScaleParameter( @@ -207,6 +217,34 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme): ) layer.register_parameter("weight_scale", weight_scale) + if not self.symmetric: + packed_output_dim = math.ceil( + output_size_per_partition * self.num_bits / 32 + ) + zeros_data = torch.zeros( + packed_output_dim, + scales_and_zp_size, + dtype=torch.int32, + ) + if not partition_scales: + qzeros = PackedColumnParameter( + output_dim=0, + packed_dim=0, + packed_factor=self.pack_factor, + weight_loader=weight_loader, + data=zeros_data, + ) + else: + qzeros = PackedvLLMParameter( + input_dim=1, + output_dim=0, + packed_dim=0, + packed_factor=self.pack_factor, + weight_loader=weight_loader, + data=zeros_data, + ) + layer.register_parameter("weight_zero_point", qzeros) + layer.register_parameter( "weight_shape", BasevLLMParameter( @@ -215,6 +253,13 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme): ), ) + self.kernel = kernel_type( + mp_config, + w_q_param_name="weight_packed", + w_s_param_name="weight_scale", + w_zp_param_name="weight_zero_point", + ) + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: self.kernel.process_weights_after_loading(layer) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa8.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa8.py index a14077d899..156f29908d 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa8.py @@ -1,11 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Weight N-bit INT scheme with symmetric INT8 activation quant via Humming. +"""Weight N-bit INT scheme with INT8 activation quant via Humming. Handles compressed-tensors pack-quantized INT weight checkpoints (2-8 bit) with INT8 symmetric dynamic per-token/per-group input activation -quantization. Static, per-tensor, and asymmetric activation quantization -are not supported. +quantization. Supports both symmetric and asymmetric weight quantization. +Static, per-tensor, and asymmetric activation quantization are not supported. """ import math @@ -28,6 +28,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import ( ) from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa: E501 WNA16_SUPPORTED_TYPES_MAP, + WNA16_ZP_SUPPORTED_TYPES_MAP, ) from vllm.model_executor.layers.quantization.utils.marlin_utils import ( marlin_repeat_scales_on_all_ranks, @@ -36,6 +37,7 @@ from vllm.model_executor.parameter import ( BasevLLMParameter, ChannelQuantScaleParameter, GroupQuantScaleParameter, + PackedColumnParameter, PackedvLLMParameter, ) @@ -52,6 +54,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme): num_bits: int, strategy: str, group_size: int | None = None, + symmetric: bool = True, input_quant: QuantizationArgs | None = None, layer_name: str | None = None, quant_format: str = "pack-quantized", @@ -60,6 +63,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme): self.pack_factor = Fraction(32, num_bits) self.strategy = strategy self.group_size = -1 if group_size is None else group_size + self.symmetric = symmetric self.input_quant = input_quant self.layer_name = layer_name self.quant_format = quant_format @@ -69,7 +73,19 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme): f"Unsupported num_bits = {num_bits} for WNA8Int; " f"supported = {sorted(WNA16_SUPPORTED_TYPES_MAP)}" ) - self.quant_type = WNA16_SUPPORTED_TYPES_MAP[num_bits] + + if not self.symmetric and num_bits not in WNA16_ZP_SUPPORTED_TYPES_MAP: + raise ValueError( + f"Asymmetric quantization not supported for " + f"num_bits = {num_bits}. Supported: " + f"{list(WNA16_ZP_SUPPORTED_TYPES_MAP)}" + ) + + self.quant_type = ( + WNA16_ZP_SUPPORTED_TYPES_MAP[num_bits] + if not self.symmetric + else WNA16_SUPPORTED_TYPES_MAP[num_bits] + ) if input_quant is not None: if not input_quant.symmetric: @@ -143,7 +159,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme): weight_type=self.quant_type, act_type=params_dtype, group_size=self.group_size, - zero_points=False, + zero_points=not self.symmetric, ) kernel_type = choose_mp_linear_kernel(mp_config) @@ -151,12 +167,6 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme): logger.info("Using %s for CompressedTensorsWNA8Int", kernel_type.__name__) self._kernel_backends_being_used.add(kernel_type.__name__) - self.kernel = kernel_type( - mp_config, - w_q_param_name="weight_packed", - w_s_param_name="weight_scale", - ) - input_quant_config = self._build_input_quant_config() if input_quant_config is not None: layer._humming_input_quant_config = input_quant_config @@ -167,10 +177,10 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme): partition_scales = not marlin_repeat_scales_on_all_ranks( self.group_size, row_parallel ) - scales_size = input_size // group_size + scales_and_zp_size = input_size // group_size if partition_scales: assert input_size_per_partition % group_size == 0 - scales_size = input_size_per_partition // group_size + scales_and_zp_size = input_size_per_partition // group_size packed_input_dim = math.ceil(input_size_per_partition * self.num_bits / 32) layer.register_parameter( @@ -190,7 +200,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme): ) scale_data = torch.empty( - output_size_per_partition, scales_size, dtype=params_dtype + output_size_per_partition, scales_and_zp_size, dtype=params_dtype ) if partition_scales: weight_scale = GroupQuantScaleParameter( @@ -207,6 +217,34 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme): ) layer.register_parameter("weight_scale", weight_scale) + if not self.symmetric: + packed_output_dim = math.ceil( + output_size_per_partition * self.num_bits / 32 + ) + zeros_data = torch.zeros( + packed_output_dim, + scales_and_zp_size, + dtype=torch.int32, + ) + if not partition_scales: + qzeros = PackedColumnParameter( + output_dim=0, + packed_dim=0, + packed_factor=self.pack_factor, + weight_loader=weight_loader, + data=zeros_data, + ) + else: + qzeros = PackedvLLMParameter( + input_dim=1, + output_dim=0, + packed_dim=0, + packed_factor=self.pack_factor, + weight_loader=weight_loader, + data=zeros_data, + ) + layer.register_parameter("weight_zero_point", qzeros) + layer.register_parameter( "weight_shape", BasevLLMParameter( @@ -215,6 +253,13 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme): ), ) + self.kernel = kernel_type( + mp_config, + w_q_param_name="weight_packed", + w_s_param_name="weight_scale", + w_zp_param_name="weight_zero_point", + ) + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: self.kernel.process_weights_after_loading(layer) diff --git a/vllm/model_executor/layers/quantization/utils/humming/linear.py b/vllm/model_executor/layers/quantization/utils/humming/linear.py index 339ff640b0..0c64645c43 100644 --- a/vllm/model_executor/layers/quantization/utils/humming/linear.py +++ b/vllm/model_executor/layers/quantization/utils/humming/linear.py @@ -101,6 +101,19 @@ def prepare_humming_linear_layer_config( param_dtype=layer.params_dtype, ) + if ( + weight_schema.has_zero_point + and input_schema.a_dtype is not None + and weight_schema.b_dtype.num_bits == input_schema.a_dtype.num_bits + ): + raise ValueError( + "Humming does not support asymmetric (zero_point) weights " + "when weight and activation bit widths are equal " + f"(W{weight_schema.b_dtype.num_bits}" + f"A{input_schema.a_dtype.num_bits}). " + "Use symmetric weights or a different activation bit width." + ) + tensors.update(input_tensors) layer.weight_schema = weight_schema input_schema = check_and_fallback_input_schema( diff --git a/vllm/scalar_type.py b/vllm/scalar_type.py index d73827f499..311b328c40 100644 --- a/vllm/scalar_type.py +++ b/vllm/scalar_type.py @@ -316,8 +316,13 @@ class ScalarType: class scalar_types: + uint2 = ScalarType.uint(2, None) + uint3 = ScalarType.uint(3, None) int4 = ScalarType.int_(4, None) uint4 = ScalarType.uint(4, None) + uint5 = ScalarType.uint(5, None) + uint6 = ScalarType.uint(6, None) + uint7 = ScalarType.uint(7, None) int8 = ScalarType.int_(8, None) uint8 = ScalarType.uint(8, None) float8_e4m3fn = ScalarType.float_(4, 3, True, NanRepr.EXTD_RANGE_MAX_MIN)