iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Quantization] Enable humming wNaM asymmetric quant (zero_point) with compressed-tensors (#46528)

Browse Source
This commit is contained in:
HDCharles
2026-09-23 10:23:11 -04:00
committed by GitHub
parent 88afb9dcac
commit 9f07d023d0
12 changed files with 164 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.buildkite/test_areas/lm_eval.yaml
+2 -2
View File
@@ -571,7 +571,7 @@ steps:
- label: ":nvidia: (H100) LM Eval Humming Activation FP8/INT8 TEMPORARY"
key: lm-eval-humming-h100-act
optional: true
timeout_in_minutes: 70
timeout_in_minutes: 90
device: h100
num_devices: 1
source_file_dependencies:
@@ -582,7 +582,7 @@ steps:
- vllm/model_executor/kernels/linear/
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt
# - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt
- label: ":nvidia: (H100) LM Eval Humming DeepEP TEMPORARY"
key: lm-eval-humming-h100-deepep
tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-humming.yaml
+2 -2
View File
@@ -1,5 +1,5 @@
model_name: "nm-testing/Qwen3-4B-mixed-quant-RTN-wNaM"
accuracy_threshold: 0.78
model_name: "nm-testing/Qwen3-4B-mixed-quant-RTN-wNaM-a4"
accuracy_threshold: 0.70
num_questions: 1319
num_fewshot: 5
server_args: >-
tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-no-w4a4-humming.yaml
+8
View File
@@ -0,0 +1,8 @@
model_name: "nm-testing/Qwen3-4B-mixed-quant-RTN-wNaM-asym"
accuracy_threshold: 0.85
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 4096
--linear-backend humming
tests/evals/gsm8k/configs/humming/config-a100-act-int8-shard-0.txt
+1
View File
@@ -1,3 +1,4 @@
gpt-oss-20b-humming.yaml
Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml
Qwen3.5-35B-A3B-int6wc-hadamard-humming-act-int8.yaml
Qwen3-4B-mixed-quant-RTN-no-w4a4-humming.yaml
tests/evals/gsm8k/configs/humming/config-act-int8.txt
+1 -1
View File
@@ -4,4 +4,4 @@ Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml
Qwen3-30B-A3B-AWQ-humming.yaml
Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml
Qwen3.5-35B-A3B-int6wc-hadamard-humming-act-int8.yaml
Qwen3-4B-mixed-quant-RTN-humming.yaml
Qwen3-4B-mixed-quant-RTN-no-w4a4-humming.yaml
vllm/model_executor/kernels/linear/mixed_precision/humming.py
+3
View File
@@ -31,11 +31,14 @@ class HummingLinearKernel(MPLinearKernel):
)
name_map = {"weight": self.w_q_name, "weight_scale": self.w_s_name}
if self.w_zp_name is not None and hasattr(layer, self.w_zp_name):
name_map["zero_point"] = self.w_zp_name
group_size = self.config.group_size
quant_config = {
"quant_method": "humming",
"dtype": "int" + str(self.config.weight_type.size_bits),
"group_size": 0 if group_size == -1 else group_size,
"has_zero_point": self.config.zero_points,
}
if self.config.zero_points:
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+2
View File
@@ -787,6 +787,7 @@ class CompressedTensorsConfig(QuantizationConfig):
num_bits=weight_quant.num_bits,
strategy=weight_quant.strategy,
group_size=weight_quant.group_size,
symmetric=weight_quant.symmetric,
input_quant=input_quant,
layer_name=layer_name,
quant_format=format,
@@ -799,6 +800,7 @@ class CompressedTensorsConfig(QuantizationConfig):
num_bits=weight_quant.num_bits,
strategy=weight_quant.strategy,
group_size=weight_quant.group_size,
symmetric=weight_quant.symmetric,
input_quant=input_quant,
layer_name=layer_name,
quant_format=format,
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+9 -1
View File
@@ -42,7 +42,15 @@ WNA16_SUPPORTED_TYPES_MAP = {
7: scalar_types.uint7b64,
8: scalar_types.uint8b128,
}
WNA16_ZP_SUPPORTED_TYPES_MAP = {4: scalar_types.uint4, 8: scalar_types.uint8}
WNA16_ZP_SUPPORTED_TYPES_MAP = {
2: scalar_types.uint2,
3: scalar_types.uint3,
4: scalar_types.uint4,
5: scalar_types.uint5,
6: scalar_types.uint6,
7: scalar_types.uint7,
8: scalar_types.uint8,
}
WNA16_SUPPORTED_BITS = list(WNA16_SUPPORTED_TYPES_MAP.keys())
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa4.py
+59 -14
View File
@@ -1,11 +1,11 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Weight N-bit INT scheme with symmetric INT4 activation quant via Humming.
"""Weight N-bit INT scheme with INT4 activation quant via Humming.
Handles compressed-tensors pack-quantized INT weight checkpoints (2-8 bit)
with INT4 symmetric dynamic per-token/per-group input activation
quantization. Static, per-tensor, and asymmetric activation quantization
are not supported.
quantization. Supports both symmetric and asymmetric weight quantization.
Static, per-tensor, and asymmetric activation quantization are not supported.
"""
import math
@@ -28,6 +28,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
)
from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa: E501
WNA16_SUPPORTED_TYPES_MAP,
WNA16_ZP_SUPPORTED_TYPES_MAP,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
marlin_repeat_scales_on_all_ranks,
@@ -36,6 +37,7 @@ from vllm.model_executor.parameter import (
BasevLLMParameter,
ChannelQuantScaleParameter,
GroupQuantScaleParameter,
PackedColumnParameter,
PackedvLLMParameter,
)
@@ -52,6 +54,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
num_bits: int,
strategy: str,
group_size: int | None = None,
symmetric: bool = True,
input_quant: QuantizationArgs | None = None,
layer_name: str | None = None,
quant_format: str = "pack-quantized",
@@ -60,6 +63,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
self.pack_factor = Fraction(32, num_bits)
self.strategy = strategy
self.group_size = -1 if group_size is None else group_size
self.symmetric = symmetric
self.input_quant = input_quant
self.layer_name = layer_name
self.quant_format = quant_format
@@ -69,7 +73,19 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
f"Unsupported num_bits = {num_bits} for WNA4Int; "
f"supported = {sorted(WNA16_SUPPORTED_TYPES_MAP)}"
)
self.quant_type = WNA16_SUPPORTED_TYPES_MAP[num_bits]
if not self.symmetric and num_bits not in WNA16_ZP_SUPPORTED_TYPES_MAP:
raise ValueError(
f"Asymmetric quantization not supported for "
f"num_bits = {num_bits}. Supported: "
f"{list(WNA16_ZP_SUPPORTED_TYPES_MAP)}"
)
self.quant_type = (
WNA16_ZP_SUPPORTED_TYPES_MAP[num_bits]
if not self.symmetric
else WNA16_SUPPORTED_TYPES_MAP[num_bits]
)
if input_quant is not None:
if not input_quant.symmetric:
@@ -143,7 +159,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
weight_type=self.quant_type,
act_type=params_dtype,
group_size=self.group_size,
zero_points=False,
zero_points=not self.symmetric,
)
kernel_type = choose_mp_linear_kernel(mp_config)
@@ -151,12 +167,6 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
logger.info("Using %s for CompressedTensorsWNA4Int", kernel_type.__name__)
self._kernel_backends_being_used.add(kernel_type.__name__)
self.kernel = kernel_type(
mp_config,
w_q_param_name="weight_packed",
w_s_param_name="weight_scale",
)
input_quant_config = self._build_input_quant_config()
if input_quant_config is not None:
layer._humming_input_quant_config = input_quant_config
@@ -167,10 +177,10 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
partition_scales = not marlin_repeat_scales_on_all_ranks(
self.group_size, row_parallel
)
scales_size = input_size // group_size
scales_and_zp_size = input_size // group_size
if partition_scales:
assert input_size_per_partition % group_size == 0
scales_size = input_size_per_partition // group_size
scales_and_zp_size = input_size_per_partition // group_size
packed_input_dim = math.ceil(input_size_per_partition * self.num_bits / 32)
layer.register_parameter(
@@ -190,7 +200,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
)
scale_data = torch.empty(
output_size_per_partition, scales_size, dtype=params_dtype
output_size_per_partition, scales_and_zp_size, dtype=params_dtype
)
if partition_scales:
weight_scale = GroupQuantScaleParameter(
@@ -207,6 +217,34 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
)
layer.register_parameter("weight_scale", weight_scale)
if not self.symmetric:
packed_output_dim = math.ceil(
output_size_per_partition * self.num_bits / 32
)
zeros_data = torch.zeros(
packed_output_dim,
scales_and_zp_size,
dtype=torch.int32,
)
if not partition_scales:
qzeros = PackedColumnParameter(
output_dim=0,
packed_dim=0,
packed_factor=self.pack_factor,
weight_loader=weight_loader,
data=zeros_data,
)
else:
qzeros = PackedvLLMParameter(
input_dim=1,
output_dim=0,
packed_dim=0,
packed_factor=self.pack_factor,
weight_loader=weight_loader,
data=zeros_data,
)
layer.register_parameter("weight_zero_point", qzeros)
layer.register_parameter(
"weight_shape",
BasevLLMParameter(
@@ -215,6 +253,13 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
),
)
self.kernel = kernel_type(
mp_config,
w_q_param_name="weight_packed",
w_s_param_name="weight_scale",
w_zp_param_name="weight_zero_point",
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
self.kernel.process_weights_after_loading(layer)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa8.py
+59 -14
View File
@@ -1,11 +1,11 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Weight N-bit INT scheme with symmetric INT8 activation quant via Humming.
"""Weight N-bit INT scheme with INT8 activation quant via Humming.
Handles compressed-tensors pack-quantized INT weight checkpoints (2-8 bit)
with INT8 symmetric dynamic per-token/per-group input activation
quantization. Static, per-tensor, and asymmetric activation quantization
are not supported.
quantization. Supports both symmetric and asymmetric weight quantization.
Static, per-tensor, and asymmetric activation quantization are not supported.
"""
import math
@@ -28,6 +28,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
)
from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa: E501
WNA16_SUPPORTED_TYPES_MAP,
WNA16_ZP_SUPPORTED_TYPES_MAP,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
marlin_repeat_scales_on_all_ranks,
@@ -36,6 +37,7 @@ from vllm.model_executor.parameter import (
BasevLLMParameter,
ChannelQuantScaleParameter,
GroupQuantScaleParameter,
PackedColumnParameter,
PackedvLLMParameter,
)
@@ -52,6 +54,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
num_bits: int,
strategy: str,
group_size: int | None = None,
symmetric: bool = True,
input_quant: QuantizationArgs | None = None,
layer_name: str | None = None,
quant_format: str = "pack-quantized",
@@ -60,6 +63,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
self.pack_factor = Fraction(32, num_bits)
self.strategy = strategy
self.group_size = -1 if group_size is None else group_size
self.symmetric = symmetric
self.input_quant = input_quant
self.layer_name = layer_name
self.quant_format = quant_format
@@ -69,7 +73,19 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
f"Unsupported num_bits = {num_bits} for WNA8Int; "
f"supported = {sorted(WNA16_SUPPORTED_TYPES_MAP)}"
)
self.quant_type = WNA16_SUPPORTED_TYPES_MAP[num_bits]
if not self.symmetric and num_bits not in WNA16_ZP_SUPPORTED_TYPES_MAP:
raise ValueError(
f"Asymmetric quantization not supported for "
f"num_bits = {num_bits}. Supported: "
f"{list(WNA16_ZP_SUPPORTED_TYPES_MAP)}"
)
self.quant_type = (
WNA16_ZP_SUPPORTED_TYPES_MAP[num_bits]
if not self.symmetric
else WNA16_SUPPORTED_TYPES_MAP[num_bits]
)
if input_quant is not None:
if not input_quant.symmetric:
@@ -143,7 +159,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
weight_type=self.quant_type,
act_type=params_dtype,
group_size=self.group_size,
zero_points=False,
zero_points=not self.symmetric,
)
kernel_type = choose_mp_linear_kernel(mp_config)
@@ -151,12 +167,6 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
logger.info("Using %s for CompressedTensorsWNA8Int", kernel_type.__name__)
self._kernel_backends_being_used.add(kernel_type.__name__)
self.kernel = kernel_type(
mp_config,
w_q_param_name="weight_packed",
w_s_param_name="weight_scale",
)
input_quant_config = self._build_input_quant_config()
if input_quant_config is not None:
layer._humming_input_quant_config = input_quant_config
@@ -167,10 +177,10 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
partition_scales = not marlin_repeat_scales_on_all_ranks(
self.group_size, row_parallel
)
scales_size = input_size // group_size
scales_and_zp_size = input_size // group_size
if partition_scales:
assert input_size_per_partition % group_size == 0
scales_size = input_size_per_partition // group_size
scales_and_zp_size = input_size_per_partition // group_size
packed_input_dim = math.ceil(input_size_per_partition * self.num_bits / 32)
layer.register_parameter(
@@ -190,7 +200,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
)
scale_data = torch.empty(
output_size_per_partition, scales_size, dtype=params_dtype
output_size_per_partition, scales_and_zp_size, dtype=params_dtype
)
if partition_scales:
weight_scale = GroupQuantScaleParameter(
@@ -207,6 +217,34 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
)
layer.register_parameter("weight_scale", weight_scale)
if not self.symmetric:
packed_output_dim = math.ceil(
output_size_per_partition * self.num_bits / 32
)
zeros_data = torch.zeros(
packed_output_dim,
scales_and_zp_size,
dtype=torch.int32,
)
if not partition_scales:
qzeros = PackedColumnParameter(
output_dim=0,
packed_dim=0,
packed_factor=self.pack_factor,
weight_loader=weight_loader,
data=zeros_data,
)
else:
qzeros = PackedvLLMParameter(
input_dim=1,
output_dim=0,
packed_dim=0,
packed_factor=self.pack_factor,
weight_loader=weight_loader,
data=zeros_data,
)
layer.register_parameter("weight_zero_point", qzeros)
layer.register_parameter(
"weight_shape",
BasevLLMParameter(
@@ -215,6 +253,13 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
),
)
self.kernel = kernel_type(
mp_config,
w_q_param_name="weight_packed",
w_s_param_name="weight_scale",
w_zp_param_name="weight_zero_point",
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
self.kernel.process_weights_after_loading(layer)
vllm/model_executor/layers/quantization/utils/humming/linear.py
+13
View File
@@ -101,6 +101,19 @@ def prepare_humming_linear_layer_config(
param_dtype=layer.params_dtype,
)
if (
weight_schema.has_zero_point
and input_schema.a_dtype is not None
and weight_schema.b_dtype.num_bits == input_schema.a_dtype.num_bits
):
raise ValueError(
"Humming does not support asymmetric (zero_point) weights "
"when weight and activation bit widths are equal "
f"(W{weight_schema.b_dtype.num_bits}"
f"A{input_schema.a_dtype.num_bits}). "
"Use symmetric weights or a different activation bit width."
)
tensors.update(input_tensors)
layer.weight_schema = weight_schema
input_schema = check_and_fallback_input_schema(
vllm/scalar_type.py
+5
View File
@@ -316,8 +316,13 @@ class ScalarType:
class scalar_types:
uint2 = ScalarType.uint(2, None)
uint3 = ScalarType.uint(3, None)
int4 = ScalarType.int_(4, None)
uint4 = ScalarType.uint(4, None)
uint5 = ScalarType.uint(5, None)
uint6 = ScalarType.uint(6, None)
uint7 = ScalarType.uint(7, None)
int8 = ScalarType.int_(8, None)
uint8 = ScalarType.uint(8, None)
float8_e4m3fn = ScalarType.float_(4, 3, True, NanRepr.EXTD_RANGE_MAX_MIN)