[Quantization] add humming mxfp4 moe backend (#41083)
Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com>
This commit is contained in:
@@ -9,11 +9,9 @@ import regex as re
|
||||
import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEQuantConfig,
|
||||
FusedMoEQuantDesc,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.layer import (
|
||||
FusedMoE,
|
||||
@@ -32,7 +30,6 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.parameter import (
|
||||
BasevLLMParameter,
|
||||
@@ -215,6 +212,15 @@ class HummingConfig(QuantizationConfig):
|
||||
def override_quantization_method(
|
||||
cls, hf_quant_cfg, user_quant, hf_config=None
|
||||
) -> QuantizationMethods | None:
|
||||
if user_quant == "humming" and hf_config is not None:
|
||||
model_type = hf_config.model_type
|
||||
quant_method = hf_quant_cfg.get("quant_method", None)
|
||||
if model_type == "gpt_oss" and quant_method == "mxfp4":
|
||||
msg = (
|
||||
"For gpt-oss model, use '--moe-backend humming' "
|
||||
"instead of '--quantization humming'."
|
||||
)
|
||||
raise ValueError(msg)
|
||||
return "humming" if user_quant == "humming" else None
|
||||
|
||||
def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):
|
||||
@@ -299,8 +305,6 @@ class HummingConfig(QuantizationConfig):
|
||||
force_weight_schema = schema
|
||||
|
||||
if weight_schema is not None:
|
||||
if weight_schema.quant_method == "gpt_oss_mxfp4" and layer_type != "moe":
|
||||
return None
|
||||
input_schema = None
|
||||
force_input_schema = None
|
||||
|
||||
@@ -335,12 +339,6 @@ class HummingConfig(QuantizationConfig):
|
||||
elif isinstance(layer, LinearBase):
|
||||
layer_type = "linear"
|
||||
|
||||
# TODO: remove this after humming moe backend is ready
|
||||
quant_method = self.full_config.get("quant_method", None)
|
||||
moe_activation = getattr(layer, "activation", None)
|
||||
if quant_method == "mxfp4" and moe_activation == MoEActivation.SWIGLUOAI:
|
||||
self.full_config["quan_method"] = "gpt_oss_mxfp4"
|
||||
|
||||
quant_config = self.get_quant_config_for_layer(prefix, layer_type)
|
||||
if quant_config is None:
|
||||
if isinstance(layer, FusedMoE):
|
||||
@@ -760,62 +758,18 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
layer.register_buffer("locks", locks)
|
||||
|
||||
def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig:
|
||||
self.process_weights_after_loading(layer)
|
||||
|
||||
input_schema = self.input_schemas["w13"]
|
||||
weight_schema = self.weight_schemas["w13"]
|
||||
|
||||
a_dtype = input_schema.a_dtype
|
||||
if a_dtype is None or a_dtype.num_bits == 16:
|
||||
a_quant_desc = FusedMoEQuantDesc(dtype=None)
|
||||
else:
|
||||
shape = GroupShape(row=1, col=-1)
|
||||
a_quant_desc = FusedMoEQuantDesc(dtype=str(a_dtype), shape=shape)
|
||||
|
||||
weight_scale_group_size = weight_schema.weight_scale_group_size
|
||||
weight_scale_group_size_n = weight_schema.weight_scale_group_size_n
|
||||
weight_group_shape: tuple[int, ...] = ()
|
||||
if weight_scale_group_size_n > 1:
|
||||
weight_group_shape = GroupShape(
|
||||
row=weight_scale_group_size,
|
||||
col=weight_scale_group_size_n,
|
||||
)
|
||||
elif weight_scale_group_size == 0:
|
||||
weight_group_shape = GroupShape(row=-1, col=1)
|
||||
else:
|
||||
weight_group_shape = GroupShape(row=weight_scale_group_size, col=1)
|
||||
|
||||
w1_quant_desc = FusedMoEQuantDesc(
|
||||
dtype=str(weight_schema.b_dtype),
|
||||
shape=weight_group_shape,
|
||||
scale=getattr(layer, "w13_weight_scale", None),
|
||||
alpha_or_gscale=getattr(layer, "w13_global_scale", None),
|
||||
zp=getattr(layer, "w13_zero_point", None),
|
||||
bias=getattr(layer, "w13_bias", None),
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
get_humming_moe_quant_config,
|
||||
)
|
||||
|
||||
w2_quant_desc = FusedMoEQuantDesc(
|
||||
dtype=str(weight_schema.b_dtype),
|
||||
shape=weight_group_shape,
|
||||
scale=getattr(layer, "w2_weight_scale", None),
|
||||
alpha_or_gscale=getattr(layer, "w2_global_scale", None),
|
||||
zp=getattr(layer, "w2_zero_point", None),
|
||||
bias=getattr(layer, "w2_bias", None),
|
||||
)
|
||||
|
||||
return FusedMoEQuantConfig(
|
||||
_a1=a_quant_desc,
|
||||
_a2=a_quant_desc,
|
||||
_w1=w1_quant_desc,
|
||||
_w2=w2_quant_desc,
|
||||
)
|
||||
return get_humming_moe_quant_config(layer)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
if getattr(self, "processed", False):
|
||||
return
|
||||
self.processed = True
|
||||
self.weight_schemas = {}
|
||||
self.input_schemas = {}
|
||||
layer.weight_schemas = {}
|
||||
layer.input_schemas = {}
|
||||
for sublayer_name, configs in layer.sublayer_configs.items():
|
||||
input_schema = self.input_schema
|
||||
weight_schema = self.weight_schema
|
||||
@@ -858,8 +812,8 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
param = torch.nn.Parameter(tensor, requires_grad=False)
|
||||
setattr(layer, name, param)
|
||||
|
||||
self.weight_schemas[sublayer_name] = weight_schema
|
||||
self.input_schemas[sublayer_name] = input_schema
|
||||
layer.weight_schemas[sublayer_name] = weight_schema
|
||||
layer.input_schemas[sublayer_name] = input_schema
|
||||
|
||||
# force requant (origin quant setting -> fp16/bf16 -> new_quant setting)
|
||||
assert isinstance(weight_schema, HummingWeightSchema)
|
||||
@@ -913,10 +867,11 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
|
||||
# use moe modular
|
||||
experts: HummingIndexedExperts | HummingGroupedExperts
|
||||
assert self.moe_quant_config is not None
|
||||
if get_humming_moe_gemm_type() == "indexed":
|
||||
experts = HummingIndexedExperts(layer, self)
|
||||
experts = HummingIndexedExperts(layer, self.moe, self.moe_quant_config)
|
||||
else:
|
||||
experts = HummingGroupedExperts(layer, self)
|
||||
experts = HummingGroupedExperts(layer, self.moe, self.moe_quant_config)
|
||||
self.experts = experts
|
||||
|
||||
def select_gemm_impl(
|
||||
@@ -927,12 +882,19 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
from vllm.model_executor.layers.fused_moe import modular_kernel as mk
|
||||
|
||||
activation_format = prepare_finalize.activation_format
|
||||
assert self.moe_quant_config is not None
|
||||
if activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
|
||||
return BatchedHummingGroupedExperts(layer, self, prepare_finalize)
|
||||
return BatchedHummingGroupedExperts(
|
||||
layer=layer,
|
||||
moe_config=self.moe,
|
||||
quant_config=self.moe_quant_config,
|
||||
max_num_tokens=prepare_finalize.max_num_tokens_per_rank(),
|
||||
num_dispatchers=prepare_finalize.num_dispatchers(),
|
||||
)
|
||||
elif get_humming_moe_gemm_type() == "indexed":
|
||||
return HummingIndexedExperts(layer, self, prepare_finalize)
|
||||
return HummingIndexedExperts(layer, self.moe, self.moe_quant_config)
|
||||
else:
|
||||
return HummingGroupedExperts(layer, self, prepare_finalize)
|
||||
return HummingGroupedExperts(layer, self.moe, self.moe_quant_config)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
|
||||
@@ -366,6 +366,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._maybe_init_expert_routing_tables(),
|
||||
shared_experts=layer.shared_experts,
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer):
|
||||
@@ -404,6 +405,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
gemm1_alpha=1.702,
|
||||
gemm1_beta=1.0,
|
||||
swiglu_limit=7.0,
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def select_gemm_impl(
|
||||
@@ -692,6 +694,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._maybe_init_expert_routing_tables(),
|
||||
shared_experts=layer.shared_experts,
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer):
|
||||
@@ -729,6 +732,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
w1_bias=w1_bias,
|
||||
w2_bias=w2_bias,
|
||||
swiglu_limit=swiglu_limit,
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def select_gemm_impl(
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
|
||||
moe_align_block_size,
|
||||
)
|
||||
|
||||
|
||||
def humming_moe_align(
|
||||
configs: list[int],
|
||||
topk_ids: torch.Tensor,
|
||||
num_experts: int,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
assert len(configs) > 0 and len(configs) % 3 == 0
|
||||
# NOTE: we choose moe_block_size based on
|
||||
# num_tokens * top_k (= topk_ids.nelement())
|
||||
shape_m = topk_ids.nelement()
|
||||
|
||||
for i in range(len(configs) // 3):
|
||||
if shape_m > configs[i * 3] and shape_m <= configs[i * 3 + 1]:
|
||||
block_size = configs[i * 3 + 2]
|
||||
break
|
||||
else:
|
||||
raise ValueError(f"Could not find a matching block_size for shape_m={shape_m}")
|
||||
|
||||
return moe_align_block_size(
|
||||
topk_ids=topk_ids,
|
||||
block_size=block_size,
|
||||
num_experts=num_experts,
|
||||
expert_map=expert_map,
|
||||
pad_sorted_ids=False,
|
||||
ignore_invalid_experts=True,
|
||||
)
|
||||
@@ -0,0 +1,214 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
from humming.layer import HummingInputSchema, HummingMethod
|
||||
from humming.schema import BaseWeightSchema
|
||||
|
||||
from vllm import envs
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
FusedMoEQuantDesc,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||
|
||||
|
||||
def humming_is_layer_skipped(config: dict[str, Any], prefix: str):
|
||||
if not config:
|
||||
return True
|
||||
|
||||
keys = ["ignored_layers", "ignore", "modules_to_not_convert"]
|
||||
ignored_layers: list[str] = []
|
||||
for key in keys:
|
||||
ignored_layers = config.get(key, []) or []
|
||||
if not ignored_layers:
|
||||
break
|
||||
|
||||
if any(module_name in prefix for module_name in ignored_layers):
|
||||
return True
|
||||
if "lm_head" in prefix:
|
||||
return True
|
||||
|
||||
for regex in config.get("dynamic", {}):
|
||||
if regex[:1] != "-":
|
||||
continue
|
||||
if re.match(regex[2:], prefix):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def prepare_humming_layer(layer: LinearBase, quant_config: dict):
|
||||
weight_schema = BaseWeightSchema.from_config(quant_config)
|
||||
input_schema = HummingInputSchema()
|
||||
|
||||
shape_k_stacks = [layer.input_size_per_partition]
|
||||
shape_n_stacks = layer.output_partition_sizes
|
||||
|
||||
# Step 1: convert weight to humming standard format
|
||||
weight_schema, tensors = weight_schema.convert_humming(
|
||||
tensors=layer.named_parameters(),
|
||||
shape_n_stacks=shape_n_stacks,
|
||||
shape_k_stacks=shape_k_stacks,
|
||||
param_dtype=layer.params_dtype,
|
||||
)
|
||||
|
||||
layer.weight_schema = weight_schema
|
||||
|
||||
for name, _ in list(layer.named_parameters()):
|
||||
delattr(layer, name)
|
||||
|
||||
for name, tensor in tensors.items():
|
||||
param = torch.nn.Parameter(tensor, requires_grad=False)
|
||||
setattr(layer, name, param)
|
||||
|
||||
# Step 2: transform weight (humming standard format) for forwarding
|
||||
HummingMethod.prepare_layer_meta(
|
||||
layer=layer,
|
||||
shape_n=layer.output_partition_sizes_sum,
|
||||
shape_k=layer.input_size_per_partition,
|
||||
weight_schema=weight_schema,
|
||||
input_schema=input_schema,
|
||||
pad_n_to_multiple=256,
|
||||
pad_k_to_multiple=128,
|
||||
has_bias=layer.has_bias,
|
||||
torch_dtype=layer.param_dtype,
|
||||
)
|
||||
|
||||
HummingMethod.transform_humming_layer(layer)
|
||||
|
||||
|
||||
def prepare_humming_moe_layer(layer: FusedMoE, quant_config: dict):
|
||||
weight_schema = BaseWeightSchema.from_config(quant_config)
|
||||
input_quant_config = envs.VLLM_HUMMING_INPUT_QUANT_CONFIG or {}
|
||||
if humming_is_layer_skipped(input_quant_config, layer.layer_name):
|
||||
input_schema = HummingInputSchema()
|
||||
else:
|
||||
# TODO: read input_quant_config from quant_config
|
||||
input_schema = HummingInputSchema.from_config(input_quant_config)
|
||||
|
||||
is_gated = layer.activation.is_gated
|
||||
shape_config = {
|
||||
"w13": (
|
||||
layer.moe_config.intermediate_size_per_partition * 2,
|
||||
layer.moe_config.hidden_dim,
|
||||
),
|
||||
"w2": (
|
||||
layer.moe_config.hidden_dim,
|
||||
layer.moe_config.intermediate_size_per_partition * (1 if is_gated else 2),
|
||||
),
|
||||
}
|
||||
|
||||
layer.weight_schemas = {}
|
||||
layer.input_schemas = {}
|
||||
|
||||
for sublayer_name in shape_config:
|
||||
# Step 1: convert weight to humming standard format
|
||||
tensors: dict[str, torch.Tensor] = dict(
|
||||
(key.removeprefix(sublayer_name + "_"), value)
|
||||
for key, value in layer.state_dict().items()
|
||||
if key.startswith(sublayer_name + "_")
|
||||
)
|
||||
|
||||
shape_n, shape_k = shape_config[sublayer_name]
|
||||
shape_n_stacks = [shape_n]
|
||||
shape_k_stacks = [shape_k]
|
||||
if sublayer_name == "w13":
|
||||
shape_n_stacks = [shape_n // 2] * 2
|
||||
|
||||
weight_schema_new, tensors = weight_schema.convert_humming(
|
||||
tensors=tensors,
|
||||
shape_n_stacks=shape_n_stacks,
|
||||
shape_k_stacks=shape_k_stacks,
|
||||
num_experts=layer.local_num_experts,
|
||||
param_dtype=layer.params_dtype,
|
||||
)
|
||||
|
||||
layer.weight_schemas[sublayer_name] = weight_schema_new
|
||||
layer.input_schemas[sublayer_name] = input_schema
|
||||
|
||||
for name, _ in list(layer.named_parameters()):
|
||||
if not name.startswith(sublayer_name + "_"):
|
||||
continue
|
||||
delattr(layer, name)
|
||||
|
||||
for name, tensor in tensors.items():
|
||||
name = f"{sublayer_name}_{name}"
|
||||
param = torch.nn.Parameter(tensor, requires_grad=False)
|
||||
setattr(layer, name, param)
|
||||
|
||||
# Step 2: transform weight (humming standard format) for forwarding
|
||||
HummingMethod.prepare_layer_meta(
|
||||
layer=layer,
|
||||
shape_n=shape_n,
|
||||
shape_k=shape_k,
|
||||
pad_n_to_multiple=256,
|
||||
pad_k_to_multiple=128,
|
||||
input_schema=input_schema,
|
||||
weight_schema=weight_schema_new,
|
||||
has_bias=layer.moe_config.has_bias,
|
||||
num_experts=layer.num_experts,
|
||||
torch_dtype=layer.params_dtype,
|
||||
sublayer_name=sublayer_name,
|
||||
)
|
||||
|
||||
HummingMethod.transform_humming_layer(layer, sublayer_name=sublayer_name)
|
||||
|
||||
if not hasattr(layer, "locks"):
|
||||
device = layer.w13_weight.device
|
||||
locks = torch.zeros(1024, dtype=torch.int32, device=device)
|
||||
layer.register_buffer("locks", locks)
|
||||
|
||||
|
||||
def get_humming_moe_quant_config(layer: FusedMoE):
|
||||
input_schema = layer.input_schemas["w13"]
|
||||
weight_schema = layer.weight_schemas["w13"]
|
||||
|
||||
a_dtype = input_schema.a_dtype
|
||||
if a_dtype is None or a_dtype.num_bits == 16:
|
||||
a_quant_desc = FusedMoEQuantDesc(dtype=None)
|
||||
else:
|
||||
shape = GroupShape(row=1, col=-1)
|
||||
a_quant_desc = FusedMoEQuantDesc(dtype=str(a_dtype), shape=shape)
|
||||
|
||||
weight_scale_group_size = weight_schema.weight_scale_group_size
|
||||
weight_scale_group_size_n = weight_schema.weight_scale_group_size_n
|
||||
weight_group_shape: tuple[int, ...] = ()
|
||||
if weight_scale_group_size_n > 1:
|
||||
weight_group_shape = GroupShape(
|
||||
row=weight_scale_group_size,
|
||||
col=weight_scale_group_size_n,
|
||||
)
|
||||
elif weight_scale_group_size == 0:
|
||||
weight_group_shape = GroupShape(row=-1, col=1)
|
||||
else:
|
||||
weight_group_shape = GroupShape(row=weight_scale_group_size, col=1)
|
||||
|
||||
w1_quant_desc = FusedMoEQuantDesc(
|
||||
dtype=str(weight_schema.b_dtype),
|
||||
shape=weight_group_shape,
|
||||
scale=getattr(layer, "w13_weight_scale", None),
|
||||
alpha_or_gscale=getattr(layer, "w13_global_scale", None),
|
||||
zp=getattr(layer, "w13_zero_point", None),
|
||||
bias=getattr(layer, "w13_bias", None),
|
||||
)
|
||||
|
||||
w2_quant_desc = FusedMoEQuantDesc(
|
||||
dtype=str(weight_schema.b_dtype),
|
||||
shape=weight_group_shape,
|
||||
scale=getattr(layer, "w2_weight_scale", None),
|
||||
alpha_or_gscale=getattr(layer, "w2_global_scale", None),
|
||||
zp=getattr(layer, "w2_zero_point", None),
|
||||
bias=getattr(layer, "w2_bias", None),
|
||||
)
|
||||
|
||||
return FusedMoEQuantConfig(
|
||||
_a1=a_quant_desc,
|
||||
_a2=a_quant_desc,
|
||||
_w1=w1_quant_desc,
|
||||
_w2=w2_quant_desc,
|
||||
)
|
||||
Reference in New Issue
Block a user