[ROCM] [Communication] Add INT3 quantization method for quickreduce (#45666)
Signed-off-by: Haoyang Li <lihaoyang0109@gmail.com>
This commit is contained in:
@@ -28,11 +28,13 @@ from vllm.distributed.utils import is_weak_contiguous # noqa: E402, F401
|
||||
|
||||
|
||||
class QuickReduceRegime(Enum):
|
||||
# Keep integer ids aligned with csrc/quickreduce/quick_reduce.h
|
||||
FP = 0
|
||||
INT8 = 1
|
||||
INT6 = 2
|
||||
INT4 = 3
|
||||
NONE = 4
|
||||
INT3 = 4
|
||||
NONE = 5
|
||||
|
||||
|
||||
KB = 1024
|
||||
@@ -43,14 +45,20 @@ class QuickAllReduce:
|
||||
_SUPPORTED_WORLD_SIZES = [2, 4, 8]
|
||||
_SUPPORTED_DTYPES = [torch.float16, torch.bfloat16]
|
||||
# The following data is based on kernel tests.
|
||||
# In this order [FP, INT8, INT6, INT4].
|
||||
# In this order [FP, INT8, INT6, INT4, INT3].
|
||||
_QR_MIN_SIZE = {
|
||||
(torch.float16, 2): [1 * MB, 2 * MB, 2 * MB, 1 * MB],
|
||||
(torch.float16, 4): [1 * MB, 16 * MB, 4 * MB, 2 * MB],
|
||||
(torch.float16, 8): [16 * MB, 4 * MB, 4 * MB, 2 * MB],
|
||||
(torch.bfloat16, 2): [2 * MB, 8 * MB, 8 * MB, 8 * MB],
|
||||
(torch.bfloat16, 4): [8 * MB, 64 * MB, 64 * MB, 16 * MB],
|
||||
(torch.bfloat16, 8): [16 * MB, 2048 * MB, 2048 * MB, 2048 * MB],
|
||||
(torch.float16, 2): [1 * MB, 2 * MB, 2 * MB, 1 * MB, 1 * MB],
|
||||
(torch.float16, 4): [1 * MB, 16 * MB, 4 * MB, 2 * MB, 2 * MB],
|
||||
(torch.float16, 8): [16 * MB, 4 * MB, 4 * MB, 2 * MB, 2 * MB],
|
||||
(torch.bfloat16, 2): [2 * MB, 8 * MB, 8 * MB, 8 * MB, 8 * MB],
|
||||
(torch.bfloat16, 4): [8 * MB, 64 * MB, 64 * MB, 16 * MB, 16 * MB],
|
||||
(torch.bfloat16, 8): [
|
||||
16 * MB,
|
||||
2048 * MB,
|
||||
2048 * MB,
|
||||
2048 * MB,
|
||||
2048 * MB,
|
||||
],
|
||||
}
|
||||
|
||||
def __init__(self, group: ProcessGroup, device: int | str | torch.device) -> None:
|
||||
@@ -59,8 +67,10 @@ class QuickAllReduce:
|
||||
available for CUDA and ROCm MI300 series.
|
||||
|
||||
Custom quick allreduce leverages quantization for further
|
||||
acceleration on ROCm. It currently supports Q8, Q6, and Q4
|
||||
quantization formats and FP(float16, bfloat16).
|
||||
acceleration on ROCm. It currently supports Q8, Q6, Q4, and Q3
|
||||
quantization formats and FP(float16, bfloat16). Q3 (INT3) is
|
||||
restricted to TP2 (world_size == 2) due to poor performance on
|
||||
larger world sizes.
|
||||
|
||||
Quick allreduce is designed as a complement to custom allreduce.
|
||||
Its initialization requires even stricter conditions.
|
||||
@@ -178,6 +188,23 @@ class QuickAllReduce:
|
||||
)
|
||||
return
|
||||
self.qr_quant_level = QuickReduceRegime[regime_str]
|
||||
|
||||
# INT3 is only enabled for TP2 (world_size == 2).
|
||||
# Kernel benchmarks show INT3 all-reduce on TP4/TP8 has poor
|
||||
# performance (the extra ranks make the 3-bit codec's pack/unpack
|
||||
# overhead outweigh the reduced communication volume), so INT3 is
|
||||
# restricted to 2-GPU tensor parallelism. For TP4/TP8 use a wider
|
||||
# codec (e.g. INT4) or NONE instead.
|
||||
if self.qr_quant_level == QuickReduceRegime.INT3 and self.world_size != 2:
|
||||
logger.warning(
|
||||
"Custom quick allreduce is disabled: INT3 quantization is "
|
||||
"only supported for TP2 (world_size == 2), but world_size "
|
||||
"is %d. INT3 on TP4/TP8 is disabled due to poor kernel "
|
||||
"performance. Use INT4/NONE for this world size.",
|
||||
self.world_size,
|
||||
)
|
||||
return
|
||||
|
||||
self.qr_quantization_min_size = self._get_qr_quantization_min_size()
|
||||
vllm_config = get_current_vllm_config_or_none()
|
||||
if (
|
||||
|
||||
Reference in New Issue
Block a user