[Bugfix][Quant] Raise actionable error instead of bare assert for group-size/TP mismatch (#46230) (#46236)
Signed-off-by: Arsalan Shakil <shakil.arsalan@yahoo.com>
This commit is contained in:
+2
-1
@@ -6,6 +6,7 @@ from collections.abc import Callable
|
||||
import torch
|
||||
from compressed_tensors.quantization import ActivationOrdering
|
||||
|
||||
from vllm.distributed.utils import verify_group_size_divides_partition
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
MPLinearLayerConfig,
|
||||
@@ -112,7 +113,7 @@ class CompressedTensorsW4A8Fp8(CompressedTensorsScheme):
|
||||
scales_and_zp_size = input_size // group_size
|
||||
|
||||
if partition_scales:
|
||||
assert input_size_per_partition % group_size == 0
|
||||
verify_group_size_divides_partition(input_size_per_partition, group_size)
|
||||
scales_and_zp_size = input_size_per_partition // group_size
|
||||
|
||||
weight = PackedvLLMParameter(
|
||||
|
||||
+3
-3
@@ -5,6 +5,7 @@ from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.utils import verify_group_size_divides_partition
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
MPLinearLayerConfig,
|
||||
@@ -79,9 +80,8 @@ class CompressedTensorsW4A8Int(CompressedTensorsScheme):
|
||||
effective_group_size = self.group_size
|
||||
|
||||
# Ensure group_size divides input_size_per_partition
|
||||
assert input_size_per_partition % effective_group_size == 0, (
|
||||
f"input_size_per_partition {input_size_per_partition}"
|
||||
f" not divisible by group_size {effective_group_size}"
|
||||
verify_group_size_divides_partition(
|
||||
input_size_per_partition, effective_group_size
|
||||
)
|
||||
|
||||
# Determine scale partitioning
|
||||
|
||||
+4
-1
@@ -8,6 +8,7 @@ from fractions import Fraction
|
||||
import torch
|
||||
from compressed_tensors.quantization import ActivationOrdering
|
||||
|
||||
from vllm.distributed.utils import verify_group_size_divides_partition
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
MarlinLinearKernel,
|
||||
@@ -151,7 +152,9 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
|
||||
scales_and_zp_size = input_size // group_size
|
||||
|
||||
if partition_scales:
|
||||
assert input_size_per_partition % group_size == 0
|
||||
verify_group_size_divides_partition(
|
||||
input_size_per_partition, group_size, self.layer_name
|
||||
)
|
||||
scales_and_zp_size = input_size_per_partition // group_size
|
||||
|
||||
packed_input_dim = math.ceil(input_size_per_partition * self.num_bits / 32)
|
||||
|
||||
+4
-1
@@ -13,6 +13,7 @@ from collections.abc import Callable
|
||||
import torch
|
||||
from compressed_tensors.compressors.pack_quantized.helpers import pack_to_int32
|
||||
|
||||
from vllm.distributed.utils import verify_group_size_divides_partition
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
MPLinearLayerConfig,
|
||||
choose_mp_linear_kernel,
|
||||
@@ -171,7 +172,9 @@ class CompressedTensorsWNA8O8Int(CompressedTensorsScheme):
|
||||
scales = (input_size_per_partition if partitioned else input_size) // group_size
|
||||
scale_data = torch.empty(out, scales, dtype=params_dtype)
|
||||
if partitioned:
|
||||
assert input_size_per_partition % group_size == 0
|
||||
verify_group_size_divides_partition(
|
||||
input_size_per_partition, group_size, self.layer_name
|
||||
)
|
||||
weight_scale = GroupQuantScaleParameter(
|
||||
data=scale_data, output_dim=0, input_dim=1, weight_loader=weight_loader
|
||||
)
|
||||
|
||||
@@ -9,6 +9,7 @@ import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.distributed.utils import verify_group_size_divides_partition
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
@@ -193,12 +194,11 @@ def verify_marlin_supports_shape(
|
||||
"with --quantization gptq."
|
||||
)
|
||||
|
||||
if group_size < input_size and input_size_per_partition % group_size != 0:
|
||||
raise ValueError(
|
||||
f"Weight input_size_per_partition = {input_size_per_partition}"
|
||||
f" is not divisible by group_size = {group_size}. "
|
||||
"Consider reducing tensor_parallel_size or running "
|
||||
"with --quantization gptq."
|
||||
if group_size < input_size:
|
||||
verify_group_size_divides_partition(
|
||||
input_size_per_partition,
|
||||
group_size,
|
||||
extra_suggestion=" or running with --quantization gptq",
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user