[Bugfix] Fix FP8 Marlin MoE and enable for compressed-tensors models (#18026)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Michael Goin
2025-05-13 19:48:33 -07:00
committed by GitHub
parent 6266c57bae
commit 9a2a6357de
3 changed files with 53 additions and 6 deletions
@@ -268,6 +268,7 @@ def prepare_moe_fp8_layer_for_marlin(layer: torch.nn.Module,
tensor_list.append(marlin_scales)
scales = torch.cat([x.unsqueeze(0) for x in tensor_list], 0)
scales = fp8_fused_exponent_bias_into_scales(scales)
scales = torch.nn.Parameter(scales, requires_grad=False)
setattr(layer, name + "_weight_scale", scales)