Humming support for 2/3/5/6/7-bit pack-quantized weight-only inference (#46389)

Signed-off-by: HDCharles <charlesdavidhernandez@gmail.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
HDCharles
2026-06-24 13:53:54 -04:00
committed by GitHub
co-authored by Claude Opus 4.6
parent cf57311187
commit 56ca5997ea
4 changed files with 40 additions and 22 deletions
@@ -37,7 +37,6 @@ from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tenso
CompressedTensorsMoEMethod,
)
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
WNA16_SUPPORTED_BITS,
CompressedTensorsScheme,
CompressedTensorsW4A4Fp4,
CompressedTensorsW4A4Mxfp4,
@@ -680,14 +679,7 @@ class CompressedTensorsConfig(QuantizationConfig):
and output_quant.num_bits == 8
and not output_quant.dynamic
)
# Static int8-activation layers, plus sub-byte weight-only layers (e.g.
# 2-bit lm_head) that marlin-backed WNA16 cannot serve. Standard 4/8-bit
# weight-only (no activations) falls through to WNA16.
is_subbyte_weight_only = weight_quant.num_bits not in WNA16_SUPPORTED_BITS
needs_wNa8o8 = is_intN_weight and (
(is_static_int8_in and is_static_int8_out) or is_subbyte_weight_only
)
return needs_wNa8o8
return is_intN_weight and (is_static_int8_in or is_static_int8_out)
def _get_scheme_from_parts(
self,
@@ -740,10 +732,8 @@ class CompressedTensorsConfig(QuantizationConfig):
quant_format=format,
)
if (
self._is_wNa16_group_channel(weight_quant, input_quant)
and (format == CompressionFormat.pack_quantized.value)
and (weight_quant.num_bits in WNA16_SUPPORTED_BITS)
if self._is_wNa16_group_channel(weight_quant, input_quant) and (
format == CompressionFormat.pack_quantized.value
):
return CompressedTensorsWNA16(
num_bits=weight_quant.num_bits,
@@ -11,7 +11,7 @@ from .compressed_tensors_w8a8_int8 import CompressedTensorsW8A8Int8
from .compressed_tensors_w8a8_mxfp8 import CompressedTensorsW8A8Mxfp8
from .compressed_tensors_w8a16_fp8 import CompressedTensorsW8A16Fp8
from .compressed_tensors_wNa8o8 import CompressedTensorsWNA8O8Int
from .compressed_tensors_wNa16 import WNA16_SUPPORTED_BITS, CompressedTensorsWNA16
from .compressed_tensors_wNa16 import CompressedTensorsWNA16
__all__ = [
"CompressedTensorsScheme",
@@ -20,7 +20,6 @@ __all__ = [
"CompressedTensorsW8A16Fp8",
"CompressedTensorsW8A8Int8",
"CompressedTensorsW8A8Fp8",
"WNA16_SUPPORTED_BITS",
"CompressedTensorsW4A4Mxfp4",
"CompressedTensorsW4A4Fp4",
"CompressedTensorsW4A8Int",
@@ -1,7 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from collections.abc import Callable
from fractions import Fraction
import torch
from compressed_tensors.quantization import ActivationOrdering
@@ -32,7 +34,15 @@ from vllm.scalar_type import scalar_types
logger = init_logger(__name__)
__all__ = ["CompressedTensorsWNA16"]
WNA16_SUPPORTED_TYPES_MAP = {4: scalar_types.uint4b8, 8: scalar_types.uint8b128}
WNA16_SUPPORTED_TYPES_MAP = {
2: scalar_types.uint2b2,
3: scalar_types.uint3b4,
4: scalar_types.uint4b8,
5: scalar_types.uint5b16,
6: scalar_types.uint6b32,
7: scalar_types.uint7b64,
8: scalar_types.uint8b128,
}
WNA16_ZP_SUPPORTED_TYPES_MAP = {4: scalar_types.uint4, 8: scalar_types.uint8}
WNA16_SUPPORTED_BITS = list(WNA16_SUPPORTED_TYPES_MAP.keys())
@@ -49,7 +59,8 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
actorder: ActivationOrdering | None = None,
layer_name: str | None = None,
):
self.pack_factor = 32 // num_bits
self.num_bits = num_bits
self.pack_factor = Fraction(32, num_bits)
self.strategy = strategy
self.symmetric = symmetric
self.group_size = -1 if group_size is None else group_size
@@ -58,15 +69,22 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
if self.group_size == -1 and self.strategy != "channel":
raise ValueError(
"Marlin kernels require group quantization or "
"channelwise quantization, but found no group "
"Pack-quantized format requires group quantization "
"or channelwise quantization, but found no group "
"size and strategy is not channelwise."
)
if num_bits not in WNA16_SUPPORTED_TYPES_MAP:
raise ValueError(
f"Unsupported num_bits = {num_bits}. "
f"Supported num_bits = {WNA16_SUPPORTED_TYPES_MAP.keys()}"
f"Supported num_bits = {list(WNA16_SUPPORTED_TYPES_MAP)}"
)
if not self.symmetric and num_bits not in WNA16_ZP_SUPPORTED_TYPES_MAP:
raise ValueError(
f"Asymmetric quantization not supported for "
f"num_bits = {num_bits}. Supported: "
f"{list(WNA16_ZP_SUPPORTED_TYPES_MAP)}"
)
self.quant_type = (
@@ -92,6 +110,12 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
**kwargs,
):
output_size_per_partition = sum(output_partition_sizes)
layer.input_size_per_partition = input_size_per_partition
layer.output_size_per_partition = output_size_per_partition
layer.output_partition_sizes = output_partition_sizes
layer.params_dtype = params_dtype
if not hasattr(layer, "has_bias"):
layer.has_bias = False
mp_linear_kernel_config = MPLinearLayerConfig(
full_weight_shape=(input_size, output_size),
@@ -130,6 +154,7 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
assert input_size_per_partition % group_size == 0
scales_and_zp_size = input_size_per_partition // group_size
packed_input_dim = math.ceil(input_size_per_partition * self.num_bits / 32)
weight = PackedvLLMParameter(
input_dim=1,
output_dim=0,
@@ -138,7 +163,7 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
packed_dim=1,
data=torch.empty(
output_size_per_partition,
input_size_per_partition // self.pack_factor,
packed_input_dim,
dtype=torch.int32,
),
)
@@ -152,10 +177,11 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
),
}
packed_output_dim = math.ceil(output_size_per_partition * self.num_bits / 32)
zeros_args = {
"weight_loader": weight_loader,
"data": torch.zeros(
output_size_per_partition // self.pack_factor,
packed_output_dim,
scales_and_zp_size,
dtype=torch.int32,
),