[Bugfix][Quantization] Refresh online NVFP4 scales before reload post-processing (#57954)
Signed-off-by: S1ro1 <matej.sirovatka@gmail.com> Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
co-authored by
aoshen02
Claude Opus 5
aoshen02
parent
16070ed7e9
commit
3652f35e7c
@@ -125,6 +125,10 @@ class TrtLlmNvFp4ExpertsBase:
|
||||
self.gemm1_beta = _per_expert(situ_linear_beta)
|
||||
self.gemm1_clamp_limit = None
|
||||
|
||||
# Unfolded values: process_weights_after_loading reruns on weight reload.
|
||||
self._gemm1_clamp_limit_unfolded = self.gemm1_clamp_limit
|
||||
self._gemm1_beta_unfolded = self.gemm1_beta
|
||||
|
||||
logger.debug_once(
|
||||
"activation=%s, gemm1_alpha=%s, gemm1_beta=%s, gemm1_clamp_limit=%s",
|
||||
moe_config.activation,
|
||||
@@ -164,15 +168,15 @@ class TrtLlmNvFp4ExpertsBase:
|
||||
|
||||
# Pre-fold the per-expert g1_alphas (= output1_scale_gate_scalar)
|
||||
# division so the TRTLLM kernel receives the raw-GEMM-space clamp
|
||||
# directly. g1_alphas is set once here in process_weights_after_loading
|
||||
# (via the in-place mul above) and never changes again, so this is a
|
||||
# static, per-expert constant. Register on the layer so EPLB
|
||||
# rearranges it alongside the other expert tensors.
|
||||
# directly. Fold from the unfolded value, since g1_alphas changes when
|
||||
# weights are reloaded. Register on the layer so EPLB rearranges it
|
||||
# alongside the other expert tensors.
|
||||
# SITU alpha/beta act on the dequantized gate/up (tanh clamps), not the
|
||||
# raw GEMM1 accumulator, so they are registered as-is without the
|
||||
# g1_alphas fold used by the SwiGLU-OAI clamp/beta below.
|
||||
if self.gemm1_clamp_limit is not None and not self.is_situ:
|
||||
gemm1_clamp_limit = self.gemm1_clamp_limit / self.quant_config.g1_alphas
|
||||
clamp_limit = self._gemm1_clamp_limit_unfolded
|
||||
if clamp_limit is not None and not self.is_situ:
|
||||
gemm1_clamp_limit = clamp_limit / self.quant_config.g1_alphas
|
||||
layer.register_parameter(
|
||||
"gemm1_clamp_limit",
|
||||
torch.nn.Parameter(gemm1_clamp_limit, requires_grad=False),
|
||||
@@ -183,12 +187,9 @@ class TrtLlmNvFp4ExpertsBase:
|
||||
# clamp limit. alpha is applied to the dequantized gate, so it stays
|
||||
# raw. Register both on the layer so EPLB rearranges them with the
|
||||
# other per-expert tensors.
|
||||
if self.gemm1_beta is not None:
|
||||
gemm1_beta = (
|
||||
self.gemm1_beta
|
||||
if self.is_situ
|
||||
else self.gemm1_beta / self.quant_config.g1_alphas
|
||||
)
|
||||
beta = self._gemm1_beta_unfolded
|
||||
if beta is not None:
|
||||
gemm1_beta = beta if self.is_situ else beta / self.quant_config.g1_alphas
|
||||
layer.register_parameter(
|
||||
"gemm1_beta",
|
||||
torch.nn.Parameter(gemm1_beta, requires_grad=False),
|
||||
|
||||
@@ -178,6 +178,14 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
per_token_activation=True,
|
||||
)
|
||||
else:
|
||||
# Reload creates new scale tensors; derived kernel scales must use
|
||||
# their new values before layerwise reload restores captured storage.
|
||||
assert self.moe_quant_config is not None
|
||||
assert self.moe_quant_config.g1_alphas is not None
|
||||
assert self.moe_quant_config.g2_alphas is not None
|
||||
self.moe_quant_config.g1_alphas.copy_(w13_scale_2)
|
||||
self.moe_quant_config.g2_alphas.copy_(w2_scale_2)
|
||||
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user