diff --git a/vllm/distributed/utils.py b/vllm/distributed/utils.py index 1e38794603..ef3c11ff64 100644 --- a/vllm/distributed/utils.py +++ b/vllm/distributed/utils.py @@ -64,6 +64,24 @@ def divide(numerator, denominator): return numerator // denominator +def verify_group_size_divides_partition( + input_size_per_partition: int, + group_size: int, + layer_name: str | None = None, + extra_suggestion: str = "", +) -> None: + """Validate that a TP-sharded layer holds a whole number of quant groups.""" + if input_size_per_partition % group_size == 0: + return + location = f" for layer '{layer_name}'" if layer_name else "" + raise ValueError( + f"Weight {input_size_per_partition=}{location} is not divisible by " + f"{group_size=}. This happens when tensor_parallel_size splits the layer input " + "into shards that are not a whole number of quant groups. Consider reducing " + f"tensor_parallel_size{extra_suggestion}." + ) + + def is_weak_contiguous(inp: torch.Tensor) -> bool: """Check that *inp* occupies a single contiguous block of memory. diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a8_fp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a8_fp8.py index cf64cc180d..22c3539e9a 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a8_fp8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a8_fp8.py @@ -6,6 +6,7 @@ from collections.abc import Callable import torch from compressed_tensors.quantization import ActivationOrdering +from vllm.distributed.utils import verify_group_size_divides_partition from vllm.logger import init_logger from vllm.model_executor.kernels.linear import ( MPLinearLayerConfig, @@ -112,7 +113,7 @@ class CompressedTensorsW4A8Fp8(CompressedTensorsScheme): scales_and_zp_size = input_size // group_size if partition_scales: - assert input_size_per_partition % group_size == 0 + verify_group_size_divides_partition(input_size_per_partition, group_size) scales_and_zp_size = input_size_per_partition // group_size weight = PackedvLLMParameter( diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a8_int.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a8_int.py index 1822df5697..77933ea2c7 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a8_int.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a8_int.py @@ -5,6 +5,7 @@ from collections.abc import Callable import torch +from vllm.distributed.utils import verify_group_size_divides_partition from vllm.logger import init_logger from vllm.model_executor.kernels.linear import ( MPLinearLayerConfig, @@ -79,9 +80,8 @@ class CompressedTensorsW4A8Int(CompressedTensorsScheme): effective_group_size = self.group_size # Ensure group_size divides input_size_per_partition - assert input_size_per_partition % effective_group_size == 0, ( - f"input_size_per_partition {input_size_per_partition}" - f" not divisible by group_size {effective_group_size}" + verify_group_size_divides_partition( + input_size_per_partition, effective_group_size ) # Determine scale partitioning diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py index f69c11f3d5..3aa8f85188 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py @@ -8,6 +8,7 @@ from fractions import Fraction import torch from compressed_tensors.quantization import ActivationOrdering +from vllm.distributed.utils import verify_group_size_divides_partition from vllm.logger import init_logger from vllm.model_executor.kernels.linear import ( MarlinLinearKernel, @@ -151,7 +152,9 @@ class CompressedTensorsWNA16(CompressedTensorsScheme): scales_and_zp_size = input_size // group_size if partition_scales: - assert input_size_per_partition % group_size == 0 + verify_group_size_divides_partition( + input_size_per_partition, group_size, self.layer_name + ) scales_and_zp_size = input_size_per_partition // group_size packed_input_dim = math.ceil(input_size_per_partition * self.num_bits / 32) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa8o8.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa8o8.py index 52d9cfeb05..3ab02a895a 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa8o8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa8o8.py @@ -13,6 +13,7 @@ from collections.abc import Callable import torch from compressed_tensors.compressors.pack_quantized.helpers import pack_to_int32 +from vllm.distributed.utils import verify_group_size_divides_partition from vllm.model_executor.kernels.linear import ( MPLinearLayerConfig, choose_mp_linear_kernel, @@ -171,7 +172,9 @@ class CompressedTensorsWNA8O8Int(CompressedTensorsScheme): scales = (input_size_per_partition if partitioned else input_size) // group_size scale_data = torch.empty(out, scales, dtype=params_dtype) if partitioned: - assert input_size_per_partition % group_size == 0 + verify_group_size_divides_partition( + input_size_per_partition, group_size, self.layer_name + ) weight_scale = GroupQuantScaleParameter( data=scale_data, output_dim=0, input_dim=1, weight_loader=weight_loader ) diff --git a/vllm/model_executor/layers/quantization/utils/marlin_utils.py b/vllm/model_executor/layers/quantization/utils/marlin_utils.py index cd6fae8cf2..ea47ed06cb 100644 --- a/vllm/model_executor/layers/quantization/utils/marlin_utils.py +++ b/vllm/model_executor/layers/quantization/utils/marlin_utils.py @@ -9,6 +9,7 @@ import torch import vllm.envs as envs from vllm import _custom_ops as ops +from vllm.distributed.utils import verify_group_size_divides_partition from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe import RoutedExperts from vllm.model_executor.layers.linear import LinearBase @@ -193,12 +194,11 @@ def verify_marlin_supports_shape( "with --quantization gptq." ) - if group_size < input_size and input_size_per_partition % group_size != 0: - raise ValueError( - f"Weight input_size_per_partition = {input_size_per_partition}" - f" is not divisible by group_size = {group_size}. " - "Consider reducing tensor_parallel_size or running " - "with --quantization gptq." + if group_size < input_size: + verify_group_size_divides_partition( + input_size_per_partition, + group_size, + extra_suggestion=" or running with --quantization gptq", )