[Quantization] Share online weight scales across TP (#49764)

Signed-off-by: S1ro1 <matej.sirovatka@gmail.com>
Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Roger Wang <hey@rogerw.io>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
Matej Sirovatka
2026-08-06 16:34:33 -07:00
committed by GitHub
co-authored by OpenAI Codex aoshen02 Claude Opus 5 Roger Wang
parent 27930df9c2
commit 8170c23c4f
5 changed files with 314 additions and 23 deletions
@@ -20,8 +20,10 @@ from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import
FLOAT4_E2M1_MAX,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
amax_for_moe_weight_quant,
kNvfp4Dynamic,
kNvfp4Static,
weight_amax,
)
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
@@ -31,6 +33,7 @@ FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max
def _quantize_moe_weight_to_nvfp4(
weight: torch.Tensor,
moe_tp_size: int = 1,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Quantize stacked MoE expert weights ``(E, N, K)`` to NVFP4.
@@ -43,7 +46,8 @@ def _quantize_moe_weight_to_nvfp4(
k = weight.shape[-1]
assert k % 16 == 0, f"last dim must be a multiple of 16, got {k}"
amax = weight.abs().amax(dim=(1, 2)).to(torch.float32).clamp_min(1e-8)
amax = weight_amax(weight.flatten(1), dim=-1).to(torch.float32)
amax = amax_for_moe_weight_quant(amax, moe_tp_size).clamp_min(1e-8)
global_scale = (FLOAT4_E2M1_MAX * FLOAT8_E4M3_MAX) / amax
weight_scale_2 = (1.0 / global_scale).to(torch.float32)
@@ -106,8 +110,13 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
layer._already_called_process_weights_after_loading = True
def _quantize_weights(self, layer: Module) -> None:
w13, w13_scale, w13_scale_2 = _quantize_moe_weight_to_nvfp4(layer.w13_weight)
w2, w2_scale, w2_scale_2 = _quantize_moe_weight_to_nvfp4(layer.w2_weight)
moe_tp_size = self.moe.tp_size
w13, w13_scale, w13_scale_2 = _quantize_moe_weight_to_nvfp4(
layer.w13_weight, moe_tp_size
)
w2, w2_scale, w2_scale_2 = _quantize_moe_weight_to_nvfp4(
layer.w2_weight, moe_tp_size
)
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w13_weight_scale", w13_scale)