[Quantization] add humming mxfp4 moe backend (#41083)

Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com>
This commit is contained in:
Jinzhen Lin
2026-05-02 18:36:03 -07:00
committed by GitHub
parent 856ec4804a
commit 08834cc3ce
9 changed files with 383 additions and 156 deletions
@@ -9,11 +9,9 @@ import regex as re
import torch
from vllm import envs
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEQuantConfig,
FusedMoEQuantDesc,
)
from vllm.model_executor.layers.fused_moe.layer import (
FusedMoE,
@@ -32,7 +30,6 @@ from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
QuantizeMethodBase,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.parameter import (
BasevLLMParameter,
@@ -215,6 +212,15 @@ class HummingConfig(QuantizationConfig):
def override_quantization_method(
cls, hf_quant_cfg, user_quant, hf_config=None
) -> QuantizationMethods | None:
if user_quant == "humming" and hf_config is not None:
model_type = hf_config.model_type
quant_method = hf_quant_cfg.get("quant_method", None)
if model_type == "gpt_oss" and quant_method == "mxfp4":
msg = (
"For gpt-oss model, use '--moe-backend humming' "
"instead of '--quantization humming'."
)
raise ValueError(msg)
return "humming" if user_quant == "humming" else None
def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):
@@ -299,8 +305,6 @@ class HummingConfig(QuantizationConfig):
force_weight_schema = schema
if weight_schema is not None:
if weight_schema.quant_method == "gpt_oss_mxfp4" and layer_type != "moe":
return None
input_schema = None
force_input_schema = None
@@ -335,12 +339,6 @@ class HummingConfig(QuantizationConfig):
elif isinstance(layer, LinearBase):
layer_type = "linear"
# TODO: remove this after humming moe backend is ready
quant_method = self.full_config.get("quant_method", None)
moe_activation = getattr(layer, "activation", None)
if quant_method == "mxfp4" and moe_activation == MoEActivation.SWIGLUOAI:
self.full_config["quan_method"] = "gpt_oss_mxfp4"
quant_config = self.get_quant_config_for_layer(prefix, layer_type)
if quant_config is None:
if isinstance(layer, FusedMoE):
@@ -760,62 +758,18 @@ class HummingMoEMethod(FusedMoEMethodBase):
layer.register_buffer("locks", locks)
def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig:
self.process_weights_after_loading(layer)
input_schema = self.input_schemas["w13"]
weight_schema = self.weight_schemas["w13"]
a_dtype = input_schema.a_dtype
if a_dtype is None or a_dtype.num_bits == 16:
a_quant_desc = FusedMoEQuantDesc(dtype=None)
else:
shape = GroupShape(row=1, col=-1)
a_quant_desc = FusedMoEQuantDesc(dtype=str(a_dtype), shape=shape)
weight_scale_group_size = weight_schema.weight_scale_group_size
weight_scale_group_size_n = weight_schema.weight_scale_group_size_n
weight_group_shape: tuple[int, ...] = ()
if weight_scale_group_size_n > 1:
weight_group_shape = GroupShape(
row=weight_scale_group_size,
col=weight_scale_group_size_n,
)
elif weight_scale_group_size == 0:
weight_group_shape = GroupShape(row=-1, col=1)
else:
weight_group_shape = GroupShape(row=weight_scale_group_size, col=1)
w1_quant_desc = FusedMoEQuantDesc(
dtype=str(weight_schema.b_dtype),
shape=weight_group_shape,
scale=getattr(layer, "w13_weight_scale", None),
alpha_or_gscale=getattr(layer, "w13_global_scale", None),
zp=getattr(layer, "w13_zero_point", None),
bias=getattr(layer, "w13_bias", None),
from vllm.model_executor.layers.quantization.utils.humming_utils import (
get_humming_moe_quant_config,
)
w2_quant_desc = FusedMoEQuantDesc(
dtype=str(weight_schema.b_dtype),
shape=weight_group_shape,
scale=getattr(layer, "w2_weight_scale", None),
alpha_or_gscale=getattr(layer, "w2_global_scale", None),
zp=getattr(layer, "w2_zero_point", None),
bias=getattr(layer, "w2_bias", None),
)
return FusedMoEQuantConfig(
_a1=a_quant_desc,
_a2=a_quant_desc,
_w1=w1_quant_desc,
_w2=w2_quant_desc,
)
return get_humming_moe_quant_config(layer)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
if getattr(self, "processed", False):
return
self.processed = True
self.weight_schemas = {}
self.input_schemas = {}
layer.weight_schemas = {}
layer.input_schemas = {}
for sublayer_name, configs in layer.sublayer_configs.items():
input_schema = self.input_schema
weight_schema = self.weight_schema
@@ -858,8 +812,8 @@ class HummingMoEMethod(FusedMoEMethodBase):
param = torch.nn.Parameter(tensor, requires_grad=False)
setattr(layer, name, param)
self.weight_schemas[sublayer_name] = weight_schema
self.input_schemas[sublayer_name] = input_schema
layer.weight_schemas[sublayer_name] = weight_schema
layer.input_schemas[sublayer_name] = input_schema
# force requant (origin quant setting -> fp16/bf16 -> new_quant setting)
assert isinstance(weight_schema, HummingWeightSchema)
@@ -913,10 +867,11 @@ class HummingMoEMethod(FusedMoEMethodBase):
# use moe modular
experts: HummingIndexedExperts | HummingGroupedExperts
assert self.moe_quant_config is not None
if get_humming_moe_gemm_type() == "indexed":
experts = HummingIndexedExperts(layer, self)
experts = HummingIndexedExperts(layer, self.moe, self.moe_quant_config)
else:
experts = HummingGroupedExperts(layer, self)
experts = HummingGroupedExperts(layer, self.moe, self.moe_quant_config)
self.experts = experts
def select_gemm_impl(
@@ -927,12 +882,19 @@ class HummingMoEMethod(FusedMoEMethodBase):
from vllm.model_executor.layers.fused_moe import modular_kernel as mk
activation_format = prepare_finalize.activation_format
assert self.moe_quant_config is not None
if activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
return BatchedHummingGroupedExperts(layer, self, prepare_finalize)
return BatchedHummingGroupedExperts(
layer=layer,
moe_config=self.moe,
quant_config=self.moe_quant_config,
max_num_tokens=prepare_finalize.max_num_tokens_per_rank(),
num_dispatchers=prepare_finalize.num_dispatchers(),
)
elif get_humming_moe_gemm_type() == "indexed":
return HummingIndexedExperts(layer, self, prepare_finalize)
return HummingIndexedExperts(layer, self.moe, self.moe_quant_config)
else:
return HummingGroupedExperts(layer, self, prepare_finalize)
return HummingGroupedExperts(layer, self.moe, self.moe_quant_config)
def apply(
self,
@@ -366,6 +366,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
experts_cls=self.experts_cls,
routing_tables=layer._maybe_init_expert_routing_tables(),
shared_experts=layer.shared_experts,
layer=layer,
)
def process_weights_after_loading(self, layer):
@@ -404,6 +405,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
gemm1_alpha=1.702,
gemm1_beta=1.0,
swiglu_limit=7.0,
layer=layer,
)
def select_gemm_impl(
@@ -692,6 +694,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
experts_cls=self.experts_cls,
routing_tables=layer._maybe_init_expert_routing_tables(),
shared_experts=layer.shared_experts,
layer=layer,
)
def process_weights_after_loading(self, layer):
@@ -729,6 +732,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
w1_bias=w1_bias,
w2_bias=w2_bias,
swiglu_limit=swiglu_limit,
layer=layer,
)
def select_gemm_impl(
@@ -1,35 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
moe_align_block_size,
)
def humming_moe_align(
configs: list[int],
topk_ids: torch.Tensor,
num_experts: int,
expert_map: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
assert len(configs) > 0 and len(configs) % 3 == 0
# NOTE: we choose moe_block_size based on
# num_tokens * top_k (= topk_ids.nelement())
shape_m = topk_ids.nelement()
for i in range(len(configs) // 3):
if shape_m > configs[i * 3] and shape_m <= configs[i * 3 + 1]:
block_size = configs[i * 3 + 2]
break
else:
raise ValueError(f"Could not find a matching block_size for shape_m={shape_m}")
return moe_align_block_size(
topk_ids=topk_ids,
block_size=block_size,
num_experts=num_experts,
expert_map=expert_map,
pad_sorted_ids=False,
ignore_invalid_experts=True,
)
@@ -0,0 +1,214 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import regex as re
import torch
from humming.layer import HummingInputSchema, HummingMethod
from humming.schema import BaseWeightSchema
from vllm import envs
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
FusedMoEQuantDesc,
)
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
from vllm.model_executor.layers.linear import LinearBase
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
def humming_is_layer_skipped(config: dict[str, Any], prefix: str):
if not config:
return True
keys = ["ignored_layers", "ignore", "modules_to_not_convert"]
ignored_layers: list[str] = []
for key in keys:
ignored_layers = config.get(key, []) or []
if not ignored_layers:
break
if any(module_name in prefix for module_name in ignored_layers):
return True
if "lm_head" in prefix:
return True
for regex in config.get("dynamic", {}):
if regex[:1] != "-":
continue
if re.match(regex[2:], prefix):
return True
return False
def prepare_humming_layer(layer: LinearBase, quant_config: dict):
weight_schema = BaseWeightSchema.from_config(quant_config)
input_schema = HummingInputSchema()
shape_k_stacks = [layer.input_size_per_partition]
shape_n_stacks = layer.output_partition_sizes
# Step 1: convert weight to humming standard format
weight_schema, tensors = weight_schema.convert_humming(
tensors=layer.named_parameters(),
shape_n_stacks=shape_n_stacks,
shape_k_stacks=shape_k_stacks,
param_dtype=layer.params_dtype,
)
layer.weight_schema = weight_schema
for name, _ in list(layer.named_parameters()):
delattr(layer, name)
for name, tensor in tensors.items():
param = torch.nn.Parameter(tensor, requires_grad=False)
setattr(layer, name, param)
# Step 2: transform weight (humming standard format) for forwarding
HummingMethod.prepare_layer_meta(
layer=layer,
shape_n=layer.output_partition_sizes_sum,
shape_k=layer.input_size_per_partition,
weight_schema=weight_schema,
input_schema=input_schema,
pad_n_to_multiple=256,
pad_k_to_multiple=128,
has_bias=layer.has_bias,
torch_dtype=layer.param_dtype,
)
HummingMethod.transform_humming_layer(layer)
def prepare_humming_moe_layer(layer: FusedMoE, quant_config: dict):
weight_schema = BaseWeightSchema.from_config(quant_config)
input_quant_config = envs.VLLM_HUMMING_INPUT_QUANT_CONFIG or {}
if humming_is_layer_skipped(input_quant_config, layer.layer_name):
input_schema = HummingInputSchema()
else:
# TODO: read input_quant_config from quant_config
input_schema = HummingInputSchema.from_config(input_quant_config)
is_gated = layer.activation.is_gated
shape_config = {
"w13": (
layer.moe_config.intermediate_size_per_partition * 2,
layer.moe_config.hidden_dim,
),
"w2": (
layer.moe_config.hidden_dim,
layer.moe_config.intermediate_size_per_partition * (1 if is_gated else 2),
),
}
layer.weight_schemas = {}
layer.input_schemas = {}
for sublayer_name in shape_config:
# Step 1: convert weight to humming standard format
tensors: dict[str, torch.Tensor] = dict(
(key.removeprefix(sublayer_name + "_"), value)
for key, value in layer.state_dict().items()
if key.startswith(sublayer_name + "_")
)
shape_n, shape_k = shape_config[sublayer_name]
shape_n_stacks = [shape_n]
shape_k_stacks = [shape_k]
if sublayer_name == "w13":
shape_n_stacks = [shape_n // 2] * 2
weight_schema_new, tensors = weight_schema.convert_humming(
tensors=tensors,
shape_n_stacks=shape_n_stacks,
shape_k_stacks=shape_k_stacks,
num_experts=layer.local_num_experts,
param_dtype=layer.params_dtype,
)
layer.weight_schemas[sublayer_name] = weight_schema_new
layer.input_schemas[sublayer_name] = input_schema
for name, _ in list(layer.named_parameters()):
if not name.startswith(sublayer_name + "_"):
continue
delattr(layer, name)
for name, tensor in tensors.items():
name = f"{sublayer_name}_{name}"
param = torch.nn.Parameter(tensor, requires_grad=False)
setattr(layer, name, param)
# Step 2: transform weight (humming standard format) for forwarding
HummingMethod.prepare_layer_meta(
layer=layer,
shape_n=shape_n,
shape_k=shape_k,
pad_n_to_multiple=256,
pad_k_to_multiple=128,
input_schema=input_schema,
weight_schema=weight_schema_new,
has_bias=layer.moe_config.has_bias,
num_experts=layer.num_experts,
torch_dtype=layer.params_dtype,
sublayer_name=sublayer_name,
)
HummingMethod.transform_humming_layer(layer, sublayer_name=sublayer_name)
if not hasattr(layer, "locks"):
device = layer.w13_weight.device
locks = torch.zeros(1024, dtype=torch.int32, device=device)
layer.register_buffer("locks", locks)
def get_humming_moe_quant_config(layer: FusedMoE):
input_schema = layer.input_schemas["w13"]
weight_schema = layer.weight_schemas["w13"]
a_dtype = input_schema.a_dtype
if a_dtype is None or a_dtype.num_bits == 16:
a_quant_desc = FusedMoEQuantDesc(dtype=None)
else:
shape = GroupShape(row=1, col=-1)
a_quant_desc = FusedMoEQuantDesc(dtype=str(a_dtype), shape=shape)
weight_scale_group_size = weight_schema.weight_scale_group_size
weight_scale_group_size_n = weight_schema.weight_scale_group_size_n
weight_group_shape: tuple[int, ...] = ()
if weight_scale_group_size_n > 1:
weight_group_shape = GroupShape(
row=weight_scale_group_size,
col=weight_scale_group_size_n,
)
elif weight_scale_group_size == 0:
weight_group_shape = GroupShape(row=-1, col=1)
else:
weight_group_shape = GroupShape(row=weight_scale_group_size, col=1)
w1_quant_desc = FusedMoEQuantDesc(
dtype=str(weight_schema.b_dtype),
shape=weight_group_shape,
scale=getattr(layer, "w13_weight_scale", None),
alpha_or_gscale=getattr(layer, "w13_global_scale", None),
zp=getattr(layer, "w13_zero_point", None),
bias=getattr(layer, "w13_bias", None),
)
w2_quant_desc = FusedMoEQuantDesc(
dtype=str(weight_schema.b_dtype),
shape=weight_group_shape,
scale=getattr(layer, "w2_weight_scale", None),
alpha_or_gscale=getattr(layer, "w2_global_scale", None),
zp=getattr(layer, "w2_zero_point", None),
bias=getattr(layer, "w2_bias", None),
)
return FusedMoEQuantConfig(
_a1=a_quant_desc,
_a2=a_quant_desc,
_w1=w1_quant_desc,
_w2=w2_quant_desc,
)