Convert formatting to use ruff instead of yapf + isort (#26247)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-10-05 07:06:22 -07:00
committed by GitHub
parent 17edd8a807
commit d6953beb91
1508 changed files with 115205 additions and 94107 deletions
@@ -7,16 +7,23 @@ from packaging import version
from vllm.logger import init_logger
from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
from vllm.model_executor.layers.quantization import (QuantizationConfig,
QuantizationMethods)
from vllm.model_executor.layers.quantization import (
QuantizationConfig,
QuantizationMethods,
)
from vllm.model_executor.layers.quantization.utils.bitblas_utils import (
BITBLAS_OPTIMIZE_FEATURES, BITBLAS_SUPPORTED_NUM_BITS,
BITBLAS_SUPPORTED_SYM, MINIMUM_BITBLAS_VERSION)
BITBLAS_OPTIMIZE_FEATURES,
BITBLAS_SUPPORTED_NUM_BITS,
BITBLAS_SUPPORTED_SYM,
MINIMUM_BITBLAS_VERSION,
)
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.parameter import (BasevLLMParameter,
ChannelQuantScaleParameter,
GroupQuantScaleParameter,
PackedvLLMParameter)
from vllm.model_executor.parameter import (
BasevLLMParameter,
ChannelQuantScaleParameter,
GroupQuantScaleParameter,
PackedvLLMParameter,
)
from vllm.model_executor.utils import set_weight_attrs
logger = init_logger(__name__)
@@ -27,6 +34,7 @@ class BitBLASConfig(QuantizationConfig):
Reference: https://github.com/Microsoft/BitBLAS
"""
TORCH_DTYPE = torch.float16
STORAGE_DTYPE = "int8" # assume int8 storage
TORCH_STORAGE_DTYPE = getattr(torch, STORAGE_DTYPE)
@@ -45,11 +53,14 @@ class BitBLASConfig(QuantizationConfig):
) -> None:
try:
import bitblas
if version.parse(bitblas.__version__) < version.parse(
MINIMUM_BITBLAS_VERSION):
MINIMUM_BITBLAS_VERSION
):
raise ImportError(
"bitblas version is wrong. Please "
f"install bitblas>={MINIMUM_BITBLAS_VERSION}")
f"install bitblas>={MINIMUM_BITBLAS_VERSION}"
)
except ImportError as e:
bitblas_import_exception = e
raise ValueError(
@@ -77,12 +88,14 @@ class BitBLASConfig(QuantizationConfig):
raise ValueError(
f"BitBLAS does not support weight_bits = {self.weight_bits}. "
f"Only weight_bits = {BITBLAS_SUPPORTED_NUM_BITS} "
"are supported.")
"are supported."
)
if self.is_sym not in BITBLAS_SUPPORTED_SYM:
raise ValueError(
f"BitBLAS does not support is_sym = {self.is_sym}. "
f"Only sym = {BITBLAS_SUPPORTED_SYM} are supported.")
f"Only sym = {BITBLAS_SUPPORTED_SYM} are supported."
)
storage_dtype = self.STORAGE_DTYPE
storage_nbit = int("".join(c for c in storage_dtype if c.isdigit()))
@@ -97,11 +110,13 @@ class BitBLASConfig(QuantizationConfig):
self.zeros_mode = self.ZEROS_MODE
def __repr__(self) -> str:
return (f"BitBLASConfig(weight_bits={self.weight_bits}, "
f"group_size={self.group_size}, "
f"desc_act={self.desc_act}, "
f"is_sym={self.is_sym}, "
f"quant_method={self.quant_method})")
return (
f"BitBLASConfig(weight_bits={self.weight_bits}, "
f"group_size={self.group_size}, "
f"desc_act={self.desc_act}, "
f"is_sym={self.is_sym}, "
f"quant_method={self.quant_method})"
)
@classmethod
def get_name(cls) -> QuantizationMethods:
@@ -121,9 +136,9 @@ class BitBLASConfig(QuantizationConfig):
return ["quantize_config.json"]
@staticmethod
def get_from_keys(config: dict[str, Any],
keys: list[str],
default: Any = None) -> Any:
def get_from_keys(
config: dict[str, Any], keys: list[str], default: Any = None
) -> Any:
"""Get a value from the model's quantization config."""
for key in keys:
if key in config:
@@ -137,34 +152,40 @@ class BitBLASConfig(QuantizationConfig):
desc_act = cls.get_from_keys(config, ["desc_act"], False)
is_sym = cls.get_from_keys(config, ["sym"], False)
quant_method = cls.get_from_keys(config, ["quant_method"])
lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"],
default=False)
return cls(weight_bits, group_size, desc_act, is_sym, quant_method,
lm_head_quantized)
lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"], default=False)
return cls(
weight_bits, group_size, desc_act, is_sym, quant_method, lm_head_quantized
)
@classmethod
def override_quantization_method(
cls, hf_quant_cfg, user_quant) -> Optional[QuantizationMethods]:
cls, hf_quant_cfg, user_quant
) -> Optional[QuantizationMethods]:
# compat: autogptq >=0.8.0 use checkpoint_format: str
# compat: autogptq <=0.7.1 is_bitblas_format: bool
is_bitblas_format = (hf_quant_cfg.get("checkpoint_format") == "bitblas"
or hf_quant_cfg.get("is_bitblas_format", False))
is_bitblas_format = hf_quant_cfg.get(
"checkpoint_format"
) == "bitblas" or hf_quant_cfg.get("is_bitblas_format", False)
is_valid_user_quant = (user_quant is None or user_quant == "gptq"
or user_quant == "bitblas")
is_valid_user_quant = (
user_quant is None or user_quant == "gptq" or user_quant == "bitblas"
)
if is_bitblas_format and is_valid_user_quant:
msg = ("The model is serialized in {} format. Using {} kernel.".
format(cls.get_name(), cls.get_name()))
msg = "The model is serialized in {} format. Using {} kernel.".format(
cls.get_name(), cls.get_name()
)
logger.info(msg)
return cls.get_name()
return None
def get_quant_method(self, layer: torch.nn.Module,
prefix: str) -> Optional["BitBLASLinearMethod"]:
if isinstance(layer, LinearBase) or (isinstance(layer, ParallelLMHead)
and self.lm_head_quantized):
def get_quant_method(
self, layer: torch.nn.Module, prefix: str
) -> Optional["BitBLASLinearMethod"]:
if isinstance(layer, LinearBase) or (
isinstance(layer, ParallelLMHead) and self.lm_head_quantized
):
return BitBLASLinearMethod(self)
return None
@@ -175,6 +196,7 @@ class BitBLASLinearMethod(LinearMethodBase):
Args:
quant_config: The BitBLAS quantization config.
"""
# USE BITBLAS_OPTIMIZE_FEATURES_CONTIGUOUS
# Instead of BITBLAS_OPTIMIZE_FEATURES
# If you want to high contiguous batching
@@ -204,7 +226,7 @@ class BitBLASLinearMethod(LinearMethodBase):
) -> None:
"""Creates quantized weights for use in linear operations.
The function initializes and returns a dictionary containing quantized
The function initializes and returns a dictionary containing quantized
weights, scales, and zeros
for performing quantized matrix multiplication operations.
@@ -213,11 +235,11 @@ class BitBLASLinearMethod(LinearMethodBase):
output_partition_sizes: List of output partition sizes.
input_size: The total size of the input (unused).
output_size: The total size of the output (unused).
params_dtype:
params_dtype:
The data type of the parameters (expected to be torch.float16).
Returns:
A dictionary containing the quantized weights ('qweight'),
A dictionary containing the quantized weights ('qweight'),
scales ('scales'), and zeros ('zeros').
Raises:
@@ -229,17 +251,19 @@ class BitBLASLinearMethod(LinearMethodBase):
weight_loader = extra_weight_attrs["weight_loader"]
if params_dtype not in self.quant_config.get_supported_act_dtypes():
raise ValueError("Parameter data type must be torch.float16, "
f"but got {params_dtype}")
raise ValueError(
f"Parameter data type must be torch.float16, but got {params_dtype}"
)
group_size = self.quant_config.group_size
if group_size is None:
group_size = -1
# Validate output_size_per_partition
output_size_per_partition = sum(output_partition_sizes)
if (group_size != -1 and input_size_per_partition % group_size != 0):
if group_size != -1 and input_size_per_partition % group_size != 0:
raise ValueError(
f"Input size per partition ({input_size_per_partition}) must "
f"be divisible by group size ({group_size}).")
f"be divisible by group size ({group_size})."
)
# Initialize or retrieve the BitBLAS matrix multiplication operator.
self._configure_bitblas_matmul(
@@ -265,34 +289,33 @@ class BitBLASLinearMethod(LinearMethodBase):
output_dim=0,
packed_dim=1,
packed_factor=self.quant_config.pack_factor,
bitblas_tile_size=(self.bitblas_matmul.retrieve_weight_shape()[-2]
if self.bitblas_matmul.propagate_b else None),
bitblas_tile_size=(
self.bitblas_matmul.retrieve_weight_shape()[-2]
if self.bitblas_matmul.propagate_b
else None
),
weight_loader=weight_loader,
)
# Compute the number of input groups for channel-wise quantization.
input_groups = (1 if group_size == -1 else input_size_per_partition //
group_size)
input_groups = 1 if group_size == -1 else input_size_per_partition // group_size
# Initialize scales and zeros for the quantized weights.
weight_scale_args = {
"data":
torch.empty(
"data": torch.empty(
output_size_per_partition,
input_groups,
device="cuda",
dtype=params_dtype,
),
"weight_loader":
weight_loader
"weight_loader": weight_loader,
}
if input_groups == 1:
scales = ChannelQuantScaleParameter(output_dim=0,
**weight_scale_args)
scales = ChannelQuantScaleParameter(output_dim=0, **weight_scale_args)
else:
scales = GroupQuantScaleParameter(output_dim=0,
input_dim=1,
**weight_scale_args)
scales = GroupQuantScaleParameter(
output_dim=0, input_dim=1, **weight_scale_args
)
if self.quant_config.zeros_mode == "quantized":
zeros = PackedvLLMParameter(
@@ -312,17 +335,22 @@ class BitBLASLinearMethod(LinearMethodBase):
else:
zeros = BasevLLMParameter(
torch.empty(output_size_per_partition,
input_groups,
device="cuda",
dtype=params_dtype),
torch.empty(
output_size_per_partition,
input_groups,
device="cuda",
dtype=params_dtype,
),
weight_loader=weight_loader,
)
# Set attributes to indicate how scales and zeros are applied.
set_weight_attrs(zeros, {
"input_dim": None if input_groups == 1 else 1,
"output_dim": 0,
})
set_weight_attrs(
zeros,
{
"input_dim": None if input_groups == 1 else 1,
"output_dim": 0,
},
)
layer.register_parameter("qweight", qweight)
layer.register_parameter("scales", scales)
@@ -339,13 +367,19 @@ class BitBLASLinearMethod(LinearMethodBase):
**extra_weight_attrs,
):
if self.quant_config.quant_method == "gptq":
return self.create_weights_gptq(layer, input_size_per_partition,
output_partition_sizes, input_size,
output_size, params_dtype,
**extra_weight_attrs)
return self.create_weights_gptq(
layer,
input_size_per_partition,
output_partition_sizes,
input_size,
output_size,
params_dtype,
**extra_weight_attrs,
)
else:
raise ValueError(
f"Unsupported quant_method {self.quant_config.quant_method}")
f"Unsupported quant_method {self.quant_config.quant_method}"
)
def _configure_bitblas_matmul(
self,
@@ -359,6 +393,7 @@ class BitBLASLinearMethod(LinearMethodBase):
out_dtype="float16",
):
from bitblas import MatmulConfig
bitblas_dtype = self.BITBLAS_DTYPES[params_dtype]
with_scaling = False
@@ -374,7 +409,8 @@ class BitBLASLinearMethod(LinearMethodBase):
W_dtype = f"int{bits}"
else:
raise ValueError(
f"Unsupported quant_method {self.quant_config.quant_method}")
f"Unsupported quant_method {self.quant_config.quant_method}"
)
matmul_config = MatmulConfig(
N=outfeatures,
@@ -392,38 +428,40 @@ class BitBLASLinearMethod(LinearMethodBase):
zeros_mode=zeros_mode,
)
self.bitblas_matmul = self._get_or_create_bitblas_operator(
matmul_config, enable_tuning)
matmul_config, enable_tuning
)
def _get_or_create_bitblas_operator(self, config, enable_tuning):
from bitblas import Matmul, auto_detect_nvidia_target
from bitblas.cache import get_database_path, global_operator_cache
BITBLAS_DATABASE_PATH = get_database_path()
BITBLAS_TARGET = auto_detect_nvidia_target()
if global_operator_cache.size() == 0:
global_operator_cache.load_from_database(BITBLAS_DATABASE_PATH,
BITBLAS_TARGET)
global_operator_cache.load_from_database(
BITBLAS_DATABASE_PATH, BITBLAS_TARGET
)
bitblas_matmul = global_operator_cache.get(config)
if bitblas_matmul is None:
bitblas_matmul = Matmul(config,
target=BITBLAS_TARGET,
enable_tuning=False)
bitblas_matmul = Matmul(config, target=BITBLAS_TARGET, enable_tuning=False)
if enable_tuning:
TUNING_MESSAGE = (f"BitBLAS Operator {config} is tuning ...")
TUNING_MESSAGE = f"BitBLAS Operator {config} is tuning ..."
logger.info(TUNING_MESSAGE)
bitblas_matmul.hardware_aware_finetune(topk=20)
global_operator_cache.add(config, bitblas_matmul)
global_operator_cache.save_into_database(
BITBLAS_DATABASE_PATH, BITBLAS_TARGET)
BITBLAS_DATABASE_PATH, BITBLAS_TARGET
)
TUNED_MESSAGE = (
f"BitBLAS Operator {config} tuned and saved to database.")
f"BitBLAS Operator {config} tuned and saved to database."
)
logger.info(TUNED_MESSAGE)
else:
_message = f"BitBLAS Operator {config} created."
logger.info(_message)
else:
_message = (
f"BitBLAS Operator {config} found in global_operator_cache.")
_message = f"BitBLAS Operator {config} found in global_operator_cache."
logger.info(_message)
return bitblas_matmul
@@ -444,7 +482,7 @@ class BitBLASLinearMethod(LinearMethodBase):
else:
output_2d = self.bitblas_matmul(x_2d, qweight, scales, qzeros)
output = output_2d.view(x.shape[:-1] + (output_2d.shape[1], ))
output = output_2d.view(x.shape[:-1] + (output_2d.shape[1],))
if bias is not None:
output.add_(bias) # In-place add
@@ -460,4 +498,5 @@ class BitBLASLinearMethod(LinearMethodBase):
return self.apply_gptq(*args, **kwargs)
else:
raise ValueError(
f"Unsupported quant_method {self.quant_config.quant_method}")
f"Unsupported quant_method {self.quant_config.quant_method}"
)