[Bugfix][Quantization] Refresh online NVFP4 scales before reload post-processing (#57954)

Signed-off-by: S1ro1 <matej.sirovatka@gmail.com>
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
Matej Sirovatka
2026-09-24 18:36:08 -07:00
committed by GitHub
co-authored by aoshen02 Claude Opus 5 aoshen02
parent 16070ed7e9
commit 3652f35e7c
4 changed files with 104 additions and 20 deletions
@@ -125,6 +125,10 @@ class TrtLlmNvFp4ExpertsBase:
self.gemm1_beta = _per_expert(situ_linear_beta)
self.gemm1_clamp_limit = None
# Unfolded values: process_weights_after_loading reruns on weight reload.
self._gemm1_clamp_limit_unfolded = self.gemm1_clamp_limit
self._gemm1_beta_unfolded = self.gemm1_beta
logger.debug_once(
"activation=%s, gemm1_alpha=%s, gemm1_beta=%s, gemm1_clamp_limit=%s",
moe_config.activation,
@@ -164,15 +168,15 @@ class TrtLlmNvFp4ExpertsBase:
# Pre-fold the per-expert g1_alphas (= output1_scale_gate_scalar)
# division so the TRTLLM kernel receives the raw-GEMM-space clamp
# directly. g1_alphas is set once here in process_weights_after_loading
# (via the in-place mul above) and never changes again, so this is a
# static, per-expert constant. Register on the layer so EPLB
# rearranges it alongside the other expert tensors.
# directly. Fold from the unfolded value, since g1_alphas changes when
# weights are reloaded. Register on the layer so EPLB rearranges it
# alongside the other expert tensors.
# SITU alpha/beta act on the dequantized gate/up (tanh clamps), not the
# raw GEMM1 accumulator, so they are registered as-is without the
# g1_alphas fold used by the SwiGLU-OAI clamp/beta below.
if self.gemm1_clamp_limit is not None and not self.is_situ:
gemm1_clamp_limit = self.gemm1_clamp_limit / self.quant_config.g1_alphas
clamp_limit = self._gemm1_clamp_limit_unfolded
if clamp_limit is not None and not self.is_situ:
gemm1_clamp_limit = clamp_limit / self.quant_config.g1_alphas
layer.register_parameter(
"gemm1_clamp_limit",
torch.nn.Parameter(gemm1_clamp_limit, requires_grad=False),
@@ -183,12 +187,9 @@ class TrtLlmNvFp4ExpertsBase:
# clamp limit. alpha is applied to the dequantized gate, so it stays
# raw. Register both on the layer so EPLB rearranges them with the
# other per-expert tensors.
if self.gemm1_beta is not None:
gemm1_beta = (
self.gemm1_beta
if self.is_situ
else self.gemm1_beta / self.quant_config.g1_alphas
)
beta = self._gemm1_beta_unfolded
if beta is not None:
gemm1_beta = beta if self.is_situ else beta / self.quant_config.g1_alphas
layer.register_parameter(
"gemm1_beta",
torch.nn.Parameter(gemm1_beta, requires_grad=False),
@@ -178,6 +178,14 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
routing_tables=layer._expert_routing_tables(),
per_token_activation=True,
)
else:
# Reload creates new scale tensors; derived kernel scales must use
# their new values before layerwise reload restores captured storage.
assert self.moe_quant_config is not None
assert self.moe_quant_config.g1_alphas is not None
assert self.moe_quant_config.g2_alphas is not None
self.moe_quant_config.g1_alphas.copy_(w13_scale_2)
self.moe_quant_config.g2_alphas.copy_(w2_scale_2)
self.moe_kernel.fused_experts.process_weights_after_loading(layer)