[Quantization] Share online weight scales across TP (#49764)
Signed-off-by: S1ro1 <matej.sirovatka@gmail.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: Roger Wang <hey@rogerw.io> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
co-authored by
OpenAI Codex
aoshen02
Claude Opus 5
Roger Wang
parent
27930df9c2
commit
8170c23c4f
@@ -20,8 +20,10 @@ from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import
|
||||
FLOAT4_E2M1_MAX,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
amax_for_moe_weight_quant,
|
||||
kNvfp4Dynamic,
|
||||
kNvfp4Static,
|
||||
weight_amax,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
@@ -31,6 +33,7 @@ FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max
|
||||
|
||||
def _quantize_moe_weight_to_nvfp4(
|
||||
weight: torch.Tensor,
|
||||
moe_tp_size: int = 1,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Quantize stacked MoE expert weights ``(E, N, K)`` to NVFP4.
|
||||
|
||||
@@ -43,7 +46,8 @@ def _quantize_moe_weight_to_nvfp4(
|
||||
k = weight.shape[-1]
|
||||
assert k % 16 == 0, f"last dim must be a multiple of 16, got {k}"
|
||||
|
||||
amax = weight.abs().amax(dim=(1, 2)).to(torch.float32).clamp_min(1e-8)
|
||||
amax = weight_amax(weight.flatten(1), dim=-1).to(torch.float32)
|
||||
amax = amax_for_moe_weight_quant(amax, moe_tp_size).clamp_min(1e-8)
|
||||
global_scale = (FLOAT4_E2M1_MAX * FLOAT8_E4M3_MAX) / amax
|
||||
weight_scale_2 = (1.0 / global_scale).to(torch.float32)
|
||||
|
||||
@@ -106,8 +110,13 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
layer._already_called_process_weights_after_loading = True
|
||||
|
||||
def _quantize_weights(self, layer: Module) -> None:
|
||||
w13, w13_scale, w13_scale_2 = _quantize_moe_weight_to_nvfp4(layer.w13_weight)
|
||||
w2, w2_scale, w2_scale_2 = _quantize_moe_weight_to_nvfp4(layer.w2_weight)
|
||||
moe_tp_size = self.moe.tp_size
|
||||
w13, w13_scale, w13_scale_2 = _quantize_moe_weight_to_nvfp4(
|
||||
layer.w13_weight, moe_tp_size
|
||||
)
|
||||
w2, w2_scale, w2_scale_2 = _quantize_moe_weight_to_nvfp4(
|
||||
layer.w2_weight, moe_tp_size
|
||||
)
|
||||
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
replace_parameter(layer, "w13_weight_scale", w13_scale)
|
||||
|
||||
Reference in New Issue
Block a user