diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py index 229112739a..d52386d5d1 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py @@ -37,7 +37,6 @@ from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tenso CompressedTensorsMoEMethod, ) from vllm.model_executor.layers.quantization.compressed_tensors.schemes import ( - WNA16_SUPPORTED_BITS, CompressedTensorsScheme, CompressedTensorsW4A4Fp4, CompressedTensorsW4A4Mxfp4, @@ -680,14 +679,7 @@ class CompressedTensorsConfig(QuantizationConfig): and output_quant.num_bits == 8 and not output_quant.dynamic ) - # Static int8-activation layers, plus sub-byte weight-only layers (e.g. - # 2-bit lm_head) that marlin-backed WNA16 cannot serve. Standard 4/8-bit - # weight-only (no activations) falls through to WNA16. - is_subbyte_weight_only = weight_quant.num_bits not in WNA16_SUPPORTED_BITS - needs_wNa8o8 = is_intN_weight and ( - (is_static_int8_in and is_static_int8_out) or is_subbyte_weight_only - ) - return needs_wNa8o8 + return is_intN_weight and (is_static_int8_in or is_static_int8_out) def _get_scheme_from_parts( self, @@ -740,10 +732,8 @@ class CompressedTensorsConfig(QuantizationConfig): quant_format=format, ) - if ( - self._is_wNa16_group_channel(weight_quant, input_quant) - and (format == CompressionFormat.pack_quantized.value) - and (weight_quant.num_bits in WNA16_SUPPORTED_BITS) + if self._is_wNa16_group_channel(weight_quant, input_quant) and ( + format == CompressionFormat.pack_quantized.value ): return CompressedTensorsWNA16( num_bits=weight_quant.num_bits, diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py index d81db4a052..2826bf7b47 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py @@ -11,7 +11,7 @@ from .compressed_tensors_w8a8_int8 import CompressedTensorsW8A8Int8 from .compressed_tensors_w8a8_mxfp8 import CompressedTensorsW8A8Mxfp8 from .compressed_tensors_w8a16_fp8 import CompressedTensorsW8A16Fp8 from .compressed_tensors_wNa8o8 import CompressedTensorsWNA8O8Int -from .compressed_tensors_wNa16 import WNA16_SUPPORTED_BITS, CompressedTensorsWNA16 +from .compressed_tensors_wNa16 import CompressedTensorsWNA16 __all__ = [ "CompressedTensorsScheme", @@ -20,7 +20,6 @@ __all__ = [ "CompressedTensorsW8A16Fp8", "CompressedTensorsW8A8Int8", "CompressedTensorsW8A8Fp8", - "WNA16_SUPPORTED_BITS", "CompressedTensorsW4A4Mxfp4", "CompressedTensorsW4A4Fp4", "CompressedTensorsW4A8Int", diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py index 1883d4ae32..f69c11f3d5 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py @@ -1,7 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import math from collections.abc import Callable +from fractions import Fraction import torch from compressed_tensors.quantization import ActivationOrdering @@ -32,7 +34,15 @@ from vllm.scalar_type import scalar_types logger = init_logger(__name__) __all__ = ["CompressedTensorsWNA16"] -WNA16_SUPPORTED_TYPES_MAP = {4: scalar_types.uint4b8, 8: scalar_types.uint8b128} +WNA16_SUPPORTED_TYPES_MAP = { + 2: scalar_types.uint2b2, + 3: scalar_types.uint3b4, + 4: scalar_types.uint4b8, + 5: scalar_types.uint5b16, + 6: scalar_types.uint6b32, + 7: scalar_types.uint7b64, + 8: scalar_types.uint8b128, +} WNA16_ZP_SUPPORTED_TYPES_MAP = {4: scalar_types.uint4, 8: scalar_types.uint8} WNA16_SUPPORTED_BITS = list(WNA16_SUPPORTED_TYPES_MAP.keys()) @@ -49,7 +59,8 @@ class CompressedTensorsWNA16(CompressedTensorsScheme): actorder: ActivationOrdering | None = None, layer_name: str | None = None, ): - self.pack_factor = 32 // num_bits + self.num_bits = num_bits + self.pack_factor = Fraction(32, num_bits) self.strategy = strategy self.symmetric = symmetric self.group_size = -1 if group_size is None else group_size @@ -58,15 +69,22 @@ class CompressedTensorsWNA16(CompressedTensorsScheme): if self.group_size == -1 and self.strategy != "channel": raise ValueError( - "Marlin kernels require group quantization or " - "channelwise quantization, but found no group " + "Pack-quantized format requires group quantization " + "or channelwise quantization, but found no group " "size and strategy is not channelwise." ) if num_bits not in WNA16_SUPPORTED_TYPES_MAP: raise ValueError( f"Unsupported num_bits = {num_bits}. " - f"Supported num_bits = {WNA16_SUPPORTED_TYPES_MAP.keys()}" + f"Supported num_bits = {list(WNA16_SUPPORTED_TYPES_MAP)}" + ) + + if not self.symmetric and num_bits not in WNA16_ZP_SUPPORTED_TYPES_MAP: + raise ValueError( + f"Asymmetric quantization not supported for " + f"num_bits = {num_bits}. Supported: " + f"{list(WNA16_ZP_SUPPORTED_TYPES_MAP)}" ) self.quant_type = ( @@ -92,6 +110,12 @@ class CompressedTensorsWNA16(CompressedTensorsScheme): **kwargs, ): output_size_per_partition = sum(output_partition_sizes) + layer.input_size_per_partition = input_size_per_partition + layer.output_size_per_partition = output_size_per_partition + layer.output_partition_sizes = output_partition_sizes + layer.params_dtype = params_dtype + if not hasattr(layer, "has_bias"): + layer.has_bias = False mp_linear_kernel_config = MPLinearLayerConfig( full_weight_shape=(input_size, output_size), @@ -130,6 +154,7 @@ class CompressedTensorsWNA16(CompressedTensorsScheme): assert input_size_per_partition % group_size == 0 scales_and_zp_size = input_size_per_partition // group_size + packed_input_dim = math.ceil(input_size_per_partition * self.num_bits / 32) weight = PackedvLLMParameter( input_dim=1, output_dim=0, @@ -138,7 +163,7 @@ class CompressedTensorsWNA16(CompressedTensorsScheme): packed_dim=1, data=torch.empty( output_size_per_partition, - input_size_per_partition // self.pack_factor, + packed_input_dim, dtype=torch.int32, ), ) @@ -152,10 +177,11 @@ class CompressedTensorsWNA16(CompressedTensorsScheme): ), } + packed_output_dim = math.ceil(output_size_per_partition * self.num_bits / 32) zeros_args = { "weight_loader": weight_loader, "data": torch.zeros( - output_size_per_partition // self.pack_factor, + packed_output_dim, scales_and_zp_size, dtype=torch.int32, ), diff --git a/vllm/scalar_type.py b/vllm/scalar_type.py index 05760f3f82..db52e93465 100644 --- a/vllm/scalar_type.py +++ b/vllm/scalar_type.py @@ -348,6 +348,9 @@ class scalar_types: uint2b2 = ScalarType.uint(2, 2) uint3b4 = ScalarType.uint(3, 4) uint4b8 = ScalarType.uint(4, 8) + uint5b16 = ScalarType.uint(5, 16) + uint6b32 = ScalarType.uint(6, 32) + uint7b64 = ScalarType.uint(7, 64) uint8b128 = ScalarType.uint(8, 128) # colloquial names