[Misc] Update w4a16 compressed-tensors support to include w8a16 (#5794)
This commit is contained in:
@@ -7,9 +7,10 @@ from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
|
||||
from vllm.model_executor.layers.quantization.base_config import ( # noqa: E501
|
||||
QuantizationConfig)
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
|
||||
CompressedTensorsScheme, CompressedTensorsW4A16,
|
||||
CompressedTensorsW4A16Sparse24, CompressedTensorsW8A8DynamicToken,
|
||||
CompressedTensorsW8A8StaticTensor)
|
||||
W4A16SPARSE24_SUPPORTED_BITS, WNA16_SUPPORTED_BITS,
|
||||
CompressedTensorsScheme, CompressedTensorsW4A16Sparse24,
|
||||
CompressedTensorsW8A8DynamicToken, CompressedTensorsW8A8StaticTensor,
|
||||
CompressedTensorsWNA16)
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
|
||||
CompressionFormat, QuantizationArgs, QuantizationStrategy,
|
||||
find_first_name_or_class_match)
|
||||
@@ -108,26 +109,31 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
|
||||
return is_8_bits and is_token and is_symmetric and is_dynamic
|
||||
|
||||
def _is_w4a16(self, weight_quant: BaseModel,
|
||||
input_quant: BaseModel) -> bool:
|
||||
def _is_wNa16_group_channel(self, weight_quant: BaseModel,
|
||||
input_quant: BaseModel) -> bool:
|
||||
input_quant_none = input_quant is None
|
||||
is_4_bits = weight_quant.num_bits == 4
|
||||
is_symmetric = weight_quant.symmetric
|
||||
is_channel_group = (
|
||||
weight_quant.strategy == QuantizationStrategy.CHANNEL.value
|
||||
or weight_quant.strategy == QuantizationStrategy.GROUP.value)
|
||||
is_static = not weight_quant.dynamic
|
||||
|
||||
return is_4_bits and input_quant_none and is_symmetric and is_static
|
||||
return (is_channel_group and input_quant_none and is_symmetric
|
||||
and is_static)
|
||||
|
||||
def _get_schema(self, weight_quant: BaseModel,
|
||||
input_quant: BaseModel) -> "CompressedTensorsScheme":
|
||||
|
||||
if self._is_w4a16(weight_quant, input_quant):
|
||||
if self.quant_format == CompressionFormat.marlin_24.value:
|
||||
if self._is_wNa16_group_channel(weight_quant, input_quant):
|
||||
if (self.quant_format == CompressionFormat.marlin_24.value
|
||||
and weight_quant.num_bits in W4A16SPARSE24_SUPPORTED_BITS):
|
||||
return CompressedTensorsW4A16Sparse24(
|
||||
strategy=weight_quant.strategy,
|
||||
num_bits=weight_quant.num_bits,
|
||||
group_size=weight_quant.group_size)
|
||||
if self.quant_format == CompressionFormat.pack_quantized.value:
|
||||
return CompressedTensorsW4A16(
|
||||
if (self.quant_format == CompressionFormat.pack_quantized.value
|
||||
and weight_quant.num_bits in WNA16_SUPPORTED_BITS):
|
||||
return CompressedTensorsWNA16(
|
||||
num_bits=weight_quant.num_bits,
|
||||
strategy=weight_quant.strategy,
|
||||
group_size=weight_quant.group_size)
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
from .compressed_tensors_scheme import CompressedTensorsScheme # noqa: F401
|
||||
from .compressed_tensors_unquantized import ( # noqa: F401
|
||||
CompressedTensorsUnquantized)
|
||||
from .compressed_tensors_w4a16 import CompressedTensorsW4A16 # noqa: F401
|
||||
from .compressed_tensors_w4a16_24 import ( # noqa: F401
|
||||
CompressedTensorsW4A16Sparse24)
|
||||
W4A16SPARSE24_SUPPORTED_BITS, CompressedTensorsW4A16Sparse24)
|
||||
from .compressed_tensors_w8a8_dynamictoken import ( # noqa: F401, E501
|
||||
CompressedTensorsW8A8DynamicToken)
|
||||
from .compressed_tensors_w8a8_statictensor import ( # noqa: F401, E501
|
||||
CompressedTensorsW8A8StaticTensor)
|
||||
from .compressed_tensors_wNa16 import WNA16_SUPPORTED_BITS # noqa: F401
|
||||
from .compressed_tensors_wNa16 import CompressedTensorsWNA16 # noqa: F401
|
||||
|
||||
+1
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
|
||||
__all__ = ["CompressedTensorsW4A16Sparse24"]
|
||||
W4A16SPARSE24_SUPPORTED_BITS = [4]
|
||||
|
||||
|
||||
class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme):
|
||||
|
||||
+3
-2
@@ -11,10 +11,11 @@ from vllm.model_executor.layers.quantization.gptq_marlin import (
|
||||
marlin_permute_scales)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
|
||||
__all__ = ["CompressedTensorsW4A16"]
|
||||
__all__ = ["CompressedTensorsWNA16"]
|
||||
WNA16_SUPPORTED_BITS = [4, 8]
|
||||
|
||||
|
||||
class CompressedTensorsW4A16(CompressedTensorsScheme):
|
||||
class CompressedTensorsWNA16(CompressedTensorsScheme):
|
||||
|
||||
def __init__(self,
|
||||
strategy: str,
|
||||
Reference in New Issue
Block a user