[Quantization][Autoround][XPU] Support AutoRound MXFP8 MoE models (#51248)
Signed-off-by: jl9876 <jia.li@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
@@ -0,0 +1,196 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEQuantConfig,
|
||||
FusedMoeWeightScaleSupported,
|
||||
RoutedExperts,
|
||||
SharedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
|
||||
FusedMoEMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
make_fp8_moe_kernel,
|
||||
make_fp8_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import (
|
||||
select_mxfp8_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
MXFP8_BLOCK_SIZE,
|
||||
MXFP8_SCALE_DTYPE,
|
||||
MXFP8_VALUE_DTYPE,
|
||||
)
|
||||
from vllm.model_executor.parameter import ModelWeightParameter
|
||||
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
|
||||
|
||||
|
||||
class INCMxfp8MoEMethod(FusedMoEMethodBase):
|
||||
"""W8A8 MXFP8 MoE method for serialized AutoRound checkpoints."""
|
||||
|
||||
def __init__(self, moe) -> None:
|
||||
super().__init__(moe)
|
||||
self.weight_block_size = [1, MXFP8_BLOCK_SIZE]
|
||||
self.mxfp8_backend, self.experts_cls = select_mxfp8_moe_backend(config=self.moe)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
assert layer.intermediate_size_per_partition == intermediate_size_per_partition
|
||||
assert layer.hidden_size == hidden_size
|
||||
layer.orig_dtype = params_dtype
|
||||
|
||||
if hidden_size % MXFP8_BLOCK_SIZE != 0:
|
||||
raise ValueError(
|
||||
f"MXFP8 MoE requires hidden_size divisible by {MXFP8_BLOCK_SIZE}, "
|
||||
f"got {hidden_size}."
|
||||
)
|
||||
if intermediate_size_per_partition % MXFP8_BLOCK_SIZE != 0:
|
||||
raise ValueError(
|
||||
"MXFP8 MoE requires intermediate_size_per_partition divisible by "
|
||||
f"{MXFP8_BLOCK_SIZE}, got {intermediate_size_per_partition}."
|
||||
)
|
||||
|
||||
layer.num_experts = num_experts
|
||||
weight_loader = extra_weight_attrs.get("weight_loader")
|
||||
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
|
||||
|
||||
w13_weight = ModelWeightParameter(
|
||||
data=torch.empty(
|
||||
num_experts,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size,
|
||||
dtype=MXFP8_VALUE_DTYPE,
|
||||
),
|
||||
input_dim=2,
|
||||
output_dim=1,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("w13_weight", w13_weight)
|
||||
|
||||
w2_weight = ModelWeightParameter(
|
||||
data=torch.empty(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition,
|
||||
dtype=MXFP8_VALUE_DTYPE,
|
||||
),
|
||||
input_dim=2,
|
||||
output_dim=1,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("w2_weight", w2_weight)
|
||||
|
||||
w13_weight_scale = ModelWeightParameter(
|
||||
data=torch.empty(
|
||||
num_experts,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size // MXFP8_BLOCK_SIZE,
|
||||
dtype=MXFP8_SCALE_DTYPE,
|
||||
),
|
||||
input_dim=2,
|
||||
output_dim=1,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("w13_weight_scale", w13_weight_scale)
|
||||
|
||||
w2_weight_scale = ModelWeightParameter(
|
||||
data=torch.empty(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition // MXFP8_BLOCK_SIZE,
|
||||
dtype=MXFP8_SCALE_DTYPE,
|
||||
),
|
||||
input_dim=2,
|
||||
output_dim=1,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("w2_weight_scale", w2_weight_scale)
|
||||
|
||||
set_weight_attrs(
|
||||
layer.w13_weight_scale,
|
||||
{"quant_method": FusedMoeWeightScaleSupported.BLOCK.value},
|
||||
)
|
||||
set_weight_attrs(
|
||||
layer.w2_weight_scale,
|
||||
{"quant_method": FusedMoeWeightScaleSupported.BLOCK.value},
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
layer.weight_block_size = self.weight_block_size
|
||||
|
||||
w13, w2, w13_scale, w2_scale = convert_to_fp8_moe_kernel_format(
|
||||
fp8_backend=self.mxfp8_backend,
|
||||
layer=layer,
|
||||
w13=layer.w13_weight,
|
||||
w2=layer.w2_weight,
|
||||
w13_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
w13_input_scale=None,
|
||||
w2_input_scale=None,
|
||||
)
|
||||
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
replace_parameter(layer, "w2_weight", w2)
|
||||
replace_parameter(layer, "w13_weight_scale", w13_scale)
|
||||
replace_parameter(layer, "w2_weight_scale", w2_scale)
|
||||
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
assert self.moe_quant_config is not None
|
||||
self.moe_kernel = make_fp8_moe_kernel(
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
fp8_backend=self.mxfp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: RoutedExperts
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
return make_fp8_moe_quant_config(
|
||||
fp8_backend=self.mxfp8_backend,
|
||||
w1_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
a1_scale=None,
|
||||
a2_scale=None,
|
||||
block_shape=self.weight_block_size,
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
x: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
x,
|
||||
layer.w13_weight,
|
||||
layer.w2_weight,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
activation=layer.activation,
|
||||
global_num_experts=layer.global_num_experts,
|
||||
expert_map=layer.expert_map,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
shared_experts=shared_experts,
|
||||
shared_experts_input=shared_experts_input,
|
||||
)
|
||||
@@ -29,3 +29,15 @@ class INCMxfp8Scheme(INCScheme):
|
||||
from .inc_mxfp8_linear import INCMxfp8LinearScheme
|
||||
|
||||
return INCLinearMethod(INCMxfp8LinearScheme())
|
||||
|
||||
def get_moe_method(
|
||||
self,
|
||||
config: "INCConfig",
|
||||
layer: "torch.nn.Module",
|
||||
prefix: str,
|
||||
layer_config: "INCLayerConfig",
|
||||
):
|
||||
del config, prefix, layer_config
|
||||
from .inc_mxfp8_moe import INCMxfp8MoEMethod
|
||||
|
||||
return INCMxfp8MoEMethod(layer.moe_config)
|
||||
|
||||
Reference in New Issue
Block a user