[ROCM] [Communication] Add INT3 quantization method for quickreduce (#45666)

Signed-off-by: Haoyang Li <lihaoyang0109@gmail.com>
This commit is contained in:
haoyangli0109
2026-06-25 15:14:15 +00:00
committed by GitHub
parent cdfa2fd7e9
commit 1744adc256
7 changed files with 264 additions and 20 deletions
@@ -28,11 +28,13 @@ from vllm.distributed.utils import is_weak_contiguous # noqa: E402, F401
class QuickReduceRegime(Enum):
# Keep integer ids aligned with csrc/quickreduce/quick_reduce.h
FP = 0
INT8 = 1
INT6 = 2
INT4 = 3
NONE = 4
INT3 = 4
NONE = 5
KB = 1024
@@ -43,14 +45,20 @@ class QuickAllReduce:
_SUPPORTED_WORLD_SIZES = [2, 4, 8]
_SUPPORTED_DTYPES = [torch.float16, torch.bfloat16]
# The following data is based on kernel tests.
# In this order [FP, INT8, INT6, INT4].
# In this order [FP, INT8, INT6, INT4, INT3].
_QR_MIN_SIZE = {
(torch.float16, 2): [1 * MB, 2 * MB, 2 * MB, 1 * MB],
(torch.float16, 4): [1 * MB, 16 * MB, 4 * MB, 2 * MB],
(torch.float16, 8): [16 * MB, 4 * MB, 4 * MB, 2 * MB],
(torch.bfloat16, 2): [2 * MB, 8 * MB, 8 * MB, 8 * MB],
(torch.bfloat16, 4): [8 * MB, 64 * MB, 64 * MB, 16 * MB],
(torch.bfloat16, 8): [16 * MB, 2048 * MB, 2048 * MB, 2048 * MB],
(torch.float16, 2): [1 * MB, 2 * MB, 2 * MB, 1 * MB, 1 * MB],
(torch.float16, 4): [1 * MB, 16 * MB, 4 * MB, 2 * MB, 2 * MB],
(torch.float16, 8): [16 * MB, 4 * MB, 4 * MB, 2 * MB, 2 * MB],
(torch.bfloat16, 2): [2 * MB, 8 * MB, 8 * MB, 8 * MB, 8 * MB],
(torch.bfloat16, 4): [8 * MB, 64 * MB, 64 * MB, 16 * MB, 16 * MB],
(torch.bfloat16, 8): [
16 * MB,
2048 * MB,
2048 * MB,
2048 * MB,
2048 * MB,
],
}
def __init__(self, group: ProcessGroup, device: int | str | torch.device) -> None:
@@ -59,8 +67,10 @@ class QuickAllReduce:
available for CUDA and ROCm MI300 series.
Custom quick allreduce leverages quantization for further
acceleration on ROCm. It currently supports Q8, Q6, and Q4
quantization formats and FP(float16, bfloat16).
acceleration on ROCm. It currently supports Q8, Q6, Q4, and Q3
quantization formats and FP(float16, bfloat16). Q3 (INT3) is
restricted to TP2 (world_size == 2) due to poor performance on
larger world sizes.
Quick allreduce is designed as a complement to custom allreduce.
Its initialization requires even stricter conditions.
@@ -178,6 +188,23 @@ class QuickAllReduce:
)
return
self.qr_quant_level = QuickReduceRegime[regime_str]
# INT3 is only enabled for TP2 (world_size == 2).
# Kernel benchmarks show INT3 all-reduce on TP4/TP8 has poor
# performance (the extra ranks make the 3-bit codec's pack/unpack
# overhead outweigh the reduced communication volume), so INT3 is
# restricted to 2-GPU tensor parallelism. For TP4/TP8 use a wider
# codec (e.g. INT4) or NONE instead.
if self.qr_quant_level == QuickReduceRegime.INT3 and self.world_size != 2:
logger.warning(
"Custom quick allreduce is disabled: INT3 quantization is "
"only supported for TP2 (world_size == 2), but world_size "
"is %d. INT3 on TP4/TP8 is disabled due to poor kernel "
"performance. Use INT4/NONE for this world size.",
self.world_size,
)
return
self.qr_quantization_min_size = self._get_qr_quantization_min_size()
vllm_config = get_current_vllm_config_or_none()
if (