[Quantization] Share online weight scales across TP (#49764)

Signed-off-by: S1ro1 <matej.sirovatka@gmail.com>
Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Roger Wang <hey@rogerw.io>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
Matej Sirovatka
2026-08-06 16:34:33 -07:00
committed by GitHub
co-authored by OpenAI Codex aoshen02 Claude Opus 5 Roger Wang
parent 27930df9c2
commit 8170c23c4f
5 changed files with 314 additions and 23 deletions
@@ -34,13 +34,17 @@ from vllm.model_executor.layers.quantization.online.moe_base import (
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
amax_for_moe_weight_quant,
amax_for_tp_weight_quant,
create_fp8_quant_key,
get_fp8_min_max,
kFp8Dynamic128Sym,
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8Static128BlockSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
weight_amax,
)
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
cutlass_fp8_supported,
@@ -59,6 +63,54 @@ from vllm.utils.math_utils import round_up
# ---------------------------------------------------------------------------
def _fp8_max(device: torch.device) -> torch.Tensor:
# A 0-d tensor divisor keeps the true division the quant kernels do;
# dividing by a Python float lowers to multiply-by-reciprocal instead.
return torch.tensor(get_fp8_min_max()[1], device=device, dtype=torch.float32)
def _fp8_scale(amax: torch.Tensor) -> torch.Tensor:
amax = amax.to(torch.float32)
return amax / _fp8_max(amax.device)
def _fp8_channel_scale(amax: torch.Tensor) -> torch.Tensor:
fp8_max = _fp8_max(amax.device)
return _fp8_scale(amax).clamp_min(1.0 / (fp8_max * 512))
# Chunk so the fp32 intermediate from the divide stays under ~64 MB, regardless
# of how wide the weight is.
_QUANT_CHUNK_ELEMS = 16 * 1024 * 1024
def _fp8_quant_per_channel(weight: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
# True division, matching the dynamic per-token kernel; the static kernel
# multiplies by the reciprocal instead and can round a value that sits on a
# representable midpoint the other way.
fp8_min, fp8_max = get_fp8_min_max()
out = torch.empty_like(weight, dtype=current_platform.fp8_dtype())
chunk_rows = max(1, _QUANT_CHUNK_ELEMS // weight.shape[-1])
for start in range(0, weight.shape[0], chunk_rows):
rows = slice(start, start + chunk_rows)
out[rows] = (weight[rows] / scale[rows]).clamp_(fp8_min, fp8_max).to(out.dtype)
return out
def _is_tp_sharded(layer: Module, *, reduces_output_dim: bool = True) -> bool:
"""Whether the weight is sharded along a dim its ``amax`` reduces over.
Row parallel shards the input dim, column parallel the output dim.
Per-output-channel scales only reduce the input dim, so they are already
unsharded-equivalent on column-parallel layers.
"""
if layer.tp_size == 1:
return False
is_row_parallel = layer.input_size != layer.input_size_per_partition
is_column_parallel = layer.output_size != layer.output_size_per_partition
return is_row_parallel or (reduces_output_dim and is_column_parallel)
class _Fp8OnlineLinearBase(LinearMethodBase):
"""Shared base for online FP8 linear methods. Loads fp16/bf16 checkpoint
weights onto meta device and materializes them just-in-time."""
@@ -156,7 +208,10 @@ class Fp8PerTensorOnlineLinearMethod(_Fp8OnlineLinearBase):
return
layer.input_scale = None
qweight, weight_scale = ops.scaled_fp8_quant(layer.weight, scale=None)
amax = weight_amax(layer.weight).reshape(1)
amax = amax_for_tp_weight_quant(amax, _is_tp_sharded(layer))
weight_scale = _fp8_scale(amax)
qweight, _ = ops.scaled_fp8_quant(layer.weight, scale=weight_scale)
# Update layer with new values.
replace_parameter(layer, "weight", qweight.t().data)
@@ -335,9 +390,12 @@ class Fp8PtpcOnlineLinearMethod(_Fp8OnlineLinearBase):
return
layer.input_scale = None
qweight, weight_scale = ops.scaled_fp8_quant(
layer.weight, scale=None, use_per_token_if_dynamic=True
amax = weight_amax(layer.weight, dim=-1, keepdim=True)
amax = amax_for_tp_weight_quant(
amax, _is_tp_sharded(layer, reduces_output_dim=False)
)
weight_scale = _fp8_channel_scale(amax)
qweight = _fp8_quant_per_channel(layer.weight, weight_scale)
replace_parameter(layer, "weight", qweight.t())
replace_parameter(layer, "weight_scale", weight_scale)
@@ -514,19 +572,23 @@ class Fp8PerTensorOnlineMoEMethod(_Fp8OnlineMoEBase):
fp8_dtype = current_platform.fp8_dtype()
w13 = torch.empty_like(layer.w13_weight, dtype=fp8_dtype)
w2 = torch.empty_like(layer.w2_weight, dtype=fp8_dtype)
w13_scale = torch.ones(
layer.num_experts, device=w13.device, dtype=torch.float32
)
w2_scale = torch.ones(layer.num_experts, device=w2.device, dtype=torch.float32)
layer.w13_input_scale = None
layer.w2_input_scale = None
moe_tp_size = self.moe.tp_size
w13_amax = weight_amax(layer.w13_weight.flatten(1), dim=-1)
w13_amax = amax_for_moe_weight_quant(w13_amax, moe_tp_size)
w13_scale = _fp8_scale(w13_amax)
w2_amax = weight_amax(layer.w2_weight.flatten(1), dim=-1)
w2_amax = amax_for_moe_weight_quant(w2_amax, moe_tp_size)
w2_scale = _fp8_scale(w2_amax)
for expert in range(layer.local_num_experts):
w13[expert, :, :], w13_scale[expert] = ops.scaled_fp8_quant(
layer.w13_weight[expert, :, :]
w13[expert, :, :], _ = ops.scaled_fp8_quant(
layer.w13_weight[expert, :, :], scale=w13_scale[expert]
)
w2[expert, :, :], w2_scale[expert] = ops.scaled_fp8_quant(
layer.w2_weight[expert, :, :]
w2[expert, :, :], _ = ops.scaled_fp8_quant(
layer.w2_weight[expert, :, :], scale=w2_scale[expert]
)
# Shuffle weights to runtime format and setup kernel.
@@ -723,26 +785,24 @@ class Fp8PtpcOnlineMoEMethod(_Fp8OnlineMoEBase):
# construction, so it cannot drift from the weight's expert count
# under EP / padded MoE.
n_w13 = layer.w13_weight.shape[1]
n_w2 = layer.w2_weight.shape[1]
w13_scale = torch.ones(
w13.shape[0], n_w13, 1, device=w13.device, dtype=torch.float32
)
w2_scale = torch.ones(
w2.shape[0], n_w2, 1, device=w2.device, dtype=torch.float32
)
layer.w13_input_scale = None
layer.w2_input_scale = None
w2_amax = weight_amax(layer.w2_weight, dim=-1, keepdim=True)
w2_amax = amax_for_moe_weight_quant(w2_amax, self.moe.tp_size)
w2_scale = _fp8_channel_scale(w2_amax)
for expert in range(layer.local_num_experts):
w13[expert], w13_scale[expert] = ops.scaled_fp8_quant(
layer.w13_weight[expert],
scale=None,
use_per_token_if_dynamic=True,
)
w2[expert], w2_scale[expert] = ops.scaled_fp8_quant(
layer.w2_weight[expert],
scale=None,
use_per_token_if_dynamic=True,
w2[expert] = _fp8_quant_per_channel(
layer.w2_weight[expert], w2_scale[expert]
)
self._setup_kernel(
@@ -22,8 +22,10 @@ from vllm.model_executor.layers.quantization.online.moe_base import (
OnlineMoEMethodBase,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
amax_for_moe_weight_quant,
kInt8DynamicTokenSym,
kInt8StaticChannelSym,
weight_amax,
)
from vllm.model_executor.utils import replace_parameter
@@ -72,6 +74,9 @@ class Int8OnlineMoEMethod(OnlineMoEMethodBase):
dtype=torch.float32,
)
w2_amax = weight_amax(layer.w2_weight, dim=-1)
w2_amax = amax_for_moe_weight_quant(w2_amax, self.moe.tp_size)
for expert in range(layer.local_num_experts):
# w13: per-row quantization over hidden_size dim
w = layer.w13_weight[expert, :, :]
@@ -82,7 +87,7 @@ class Int8OnlineMoEMethod(OnlineMoEMethodBase):
# w2: per-row quantization over intermediate_size dim
w = layer.w2_weight[expert, :, :]
scales = w.abs().amax(dim=1) / vmax
scales = w2_amax[expert] / vmax
q = w.div(scales.unsqueeze(1)).round().clamp(-vmax, vmax)
w2[expert, :, :] = q.to(torch.int8)
w2_scale[expert, :] = scales
@@ -20,8 +20,10 @@ from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import
FLOAT4_E2M1_MAX,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
amax_for_moe_weight_quant,
kNvfp4Dynamic,
kNvfp4Static,
weight_amax,
)
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
@@ -31,6 +33,7 @@ FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max
def _quantize_moe_weight_to_nvfp4(
weight: torch.Tensor,
moe_tp_size: int = 1,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Quantize stacked MoE expert weights ``(E, N, K)`` to NVFP4.
@@ -43,7 +46,8 @@ def _quantize_moe_weight_to_nvfp4(
k = weight.shape[-1]
assert k % 16 == 0, f"last dim must be a multiple of 16, got {k}"
amax = weight.abs().amax(dim=(1, 2)).to(torch.float32).clamp_min(1e-8)
amax = weight_amax(weight.flatten(1), dim=-1).to(torch.float32)
amax = amax_for_moe_weight_quant(amax, moe_tp_size).clamp_min(1e-8)
global_scale = (FLOAT4_E2M1_MAX * FLOAT8_E4M3_MAX) / amax
weight_scale_2 = (1.0 / global_scale).to(torch.float32)
@@ -106,8 +110,13 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
layer._already_called_process_weights_after_loading = True
def _quantize_weights(self, layer: Module) -> None:
w13, w13_scale, w13_scale_2 = _quantize_moe_weight_to_nvfp4(layer.w13_weight)
w2, w2_scale, w2_scale_2 = _quantize_moe_weight_to_nvfp4(layer.w2_weight)
moe_tp_size = self.moe.tp_size
w13, w13_scale, w13_scale_2 = _quantize_moe_weight_to_nvfp4(
layer.w13_weight, moe_tp_size
)
w2, w2_scale, w2_scale_2 = _quantize_moe_weight_to_nvfp4(
layer.w2_weight, moe_tp_size
)
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w13_weight_scale", w13_scale)
@@ -11,6 +11,7 @@ import numpy
import torch
from torch import fx
from vllm.distributed.parallel_state import get_ep_group, get_tp_group
from vllm.platforms import current_platform
from vllm.scalar_type import ScalarType, scalar_types
@@ -24,6 +25,43 @@ INT4_DTYPE = scalar_types.uint4b8
INT8_DTYPE = scalar_types.uint8b128
def weight_amax(
weight: torch.Tensor, *, dim: int | None = None, keepdim: bool = False
) -> torch.Tensor:
"""``max(|weight|)``, without materializing a full-size ``abs()``."""
lo, hi = weight.aminmax(dim=dim, keepdim=keepdim)
return torch.maximum(lo.abs(), hi.abs())
def amax_for_tp_weight_quant(amax: torch.Tensor, is_sharded: bool) -> torch.Tensor:
"""Reduce a weight ``amax`` over the TP group when the weight is sharded
along a dim the ``amax`` reduces over, so each shard derives the same scale
it would as part of the whole weight.
"""
if is_sharded:
torch.distributed.all_reduce(
amax,
op=torch.distributed.ReduceOp.MAX,
group=get_tp_group().device_group,
)
return amax
def amax_for_moe_weight_quant(amax: torch.Tensor, moe_tp_size: int) -> torch.Tensor:
"""Reduce a per-expert weight ``amax`` over the ranks that tensor-shard the
MoE weights. That sharding is flattened over DP x PCP x TP, exactly the EP
group's span. Under EP ``moe_tp_size`` is 1 and each rank owns whole
experts, so no reduction is needed.
"""
if moe_tp_size > 1:
torch.distributed.all_reduce(
amax,
op=torch.distributed.ReduceOp.MAX,
group=get_ep_group().device_group,
)
return amax
def get_fp8_min_max() -> tuple[float, float]:
"""Get the min and max values for FP8 quantization."""
# Using the default value (240.0) from pytorch will cause accuracy