[ModelOpt] Load w13/w2_input_scale for all experts, nvfp4 (#26135)
Signed-off-by: Shu Wang <shuw@nvidia.com> Signed-off-by: Shu Wang. <shuw@nvidia.com> Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
@@ -49,6 +49,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
build_flashinfer_fp8_cutlass_moe_prepare_finalize,
|
||||
flashinfer_cutlass_moe_fp8,
|
||||
get_flashinfer_moe_backend,
|
||||
is_flashinfer_supporting_global_sf,
|
||||
register_moe_scaling_factors,
|
||||
rotate_flashinfer_fp8_moe_weights,
|
||||
select_cutlass_fp8_gemm_impl,
|
||||
@@ -1217,6 +1218,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
weight_dtype = torch.uint8
|
||||
weight_scale_dtype = torch.float8_e4m3fn
|
||||
weight_loader = extra_weight_attrs.get("weight_loader")
|
||||
global_num_experts = extra_weight_attrs.get("global_num_experts")
|
||||
# GEMM 1
|
||||
w13_weight = ModelWeightParameter(
|
||||
data=torch.empty(
|
||||
@@ -1295,14 +1297,19 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
{"quant_method": FusedMoeWeightScaleSupported.TENSOR.value}
|
||||
)
|
||||
|
||||
use_global_sf = self.allow_flashinfer and is_flashinfer_supporting_global_sf(
|
||||
self.flashinfer_moe_backend
|
||||
)
|
||||
global_scale_num_experts = global_num_experts if use_global_sf else num_experts
|
||||
|
||||
w13_input_scale = PerTensorScaleParameter(
|
||||
data=torch.empty(num_experts, 2, dtype=torch.float32),
|
||||
data=torch.empty(global_scale_num_experts, 2, dtype=torch.float32),
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("w13_input_scale", w13_input_scale)
|
||||
|
||||
w2_input_scale = PerTensorScaleParameter(
|
||||
data=torch.empty(num_experts, dtype=torch.float32),
|
||||
data=torch.empty(global_scale_num_experts, dtype=torch.float32),
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("w2_input_scale", w2_input_scale)
|
||||
@@ -1457,7 +1464,17 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
layer.w13_weight_scale_2 = Parameter(w13_weight_scale_2, requires_grad=False)
|
||||
|
||||
# Common processing for input scales and alphas
|
||||
w13_input_scale = layer.w13_input_scale.max(dim=1).values.to(torch.float32)
|
||||
use_global_sf = self.allow_flashinfer and is_flashinfer_supporting_global_sf(
|
||||
self.flashinfer_moe_backend
|
||||
)
|
||||
if use_global_sf:
|
||||
# For backends provide by Flashinfer, the input global scales are
|
||||
# shared across all experts.
|
||||
w13_input_scale = (
|
||||
layer.w13_input_scale.max().to(torch.float32).expand(layer.num_experts)
|
||||
)
|
||||
else:
|
||||
w13_input_scale = layer.w13_input_scale.max(dim=1).values.to(torch.float32)
|
||||
layer.g1_alphas = Parameter(
|
||||
(w13_input_scale * w13_weight_scale_2).to(torch.float32),
|
||||
requires_grad=False,
|
||||
@@ -1469,14 +1486,22 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
)
|
||||
|
||||
# GEMM 2 processing
|
||||
if use_global_sf:
|
||||
# For backends provide by Flashinfer, the input global scales are
|
||||
# shared across all experts.
|
||||
w2_input_scale = (
|
||||
layer.w2_input_scale.max().to(torch.float32).expand(layer.num_experts)
|
||||
)
|
||||
else:
|
||||
w2_input_scale = layer.w2_input_scale
|
||||
layer.g2_alphas = Parameter(
|
||||
(layer.w2_input_scale * layer.w2_weight_scale_2).to(torch.float32),
|
||||
(w2_input_scale * layer.w2_weight_scale_2).to(torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
# This is for quantization, so we need to invert it.
|
||||
layer.w2_input_scale_quant = Parameter(
|
||||
(1 / layer.w2_input_scale).to(torch.float32), requires_grad=False
|
||||
(1 / w2_input_scale).to(torch.float32), requires_grad=False
|
||||
)
|
||||
|
||||
# TensorRT-LLM specific processing
|
||||
|
||||
Reference in New Issue
Block a user