[Quantization] Share online weight scales across TP (#49764)

Signed-off-by: S1ro1 <matej.sirovatka@gmail.com>
Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Roger Wang <hey@rogerw.io>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
Matej Sirovatka
2026-08-06 16:34:33 -07:00
committed by GitHub
co-authored by OpenAI Codex aoshen02 Claude Opus 5 Roger Wang
parent 27930df9c2
commit 8170c23c4f
5 changed files with 314 additions and 23 deletions
@@ -11,6 +11,7 @@ import numpy
import torch
from torch import fx
from vllm.distributed.parallel_state import get_ep_group, get_tp_group
from vllm.platforms import current_platform
from vllm.scalar_type import ScalarType, scalar_types
@@ -24,6 +25,43 @@ INT4_DTYPE = scalar_types.uint4b8
INT8_DTYPE = scalar_types.uint8b128
def weight_amax(
weight: torch.Tensor, *, dim: int | None = None, keepdim: bool = False
) -> torch.Tensor:
"""``max(|weight|)``, without materializing a full-size ``abs()``."""
lo, hi = weight.aminmax(dim=dim, keepdim=keepdim)
return torch.maximum(lo.abs(), hi.abs())
def amax_for_tp_weight_quant(amax: torch.Tensor, is_sharded: bool) -> torch.Tensor:
"""Reduce a weight ``amax`` over the TP group when the weight is sharded
along a dim the ``amax`` reduces over, so each shard derives the same scale
it would as part of the whole weight.
"""
if is_sharded:
torch.distributed.all_reduce(
amax,
op=torch.distributed.ReduceOp.MAX,
group=get_tp_group().device_group,
)
return amax
def amax_for_moe_weight_quant(amax: torch.Tensor, moe_tp_size: int) -> torch.Tensor:
"""Reduce a per-expert weight ``amax`` over the ranks that tensor-shard the
MoE weights. That sharding is flattened over DP x PCP x TP, exactly the EP
group's span. Under EP ``moe_tp_size`` is 1 and each rank owns whole
experts, so no reduction is needed.
"""
if moe_tp_size > 1:
torch.distributed.all_reduce(
amax,
op=torch.distributed.ReduceOp.MAX,
group=get_ep_group().device_group,
)
return amax
def get_fp8_min_max() -> tuple[float, float]:
"""Get the min and max values for FP8 quantization."""
# Using the default value (240.0) from pytorch will cause accuracy