[Quantization] add humming mxfp4 moe backend (#41083)

Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com>
This commit is contained in:
Jinzhen Lin
2026-05-02 18:36:03 -07:00
committed by GitHub
parent 856ec4804a
commit 08834cc3ce
9 changed files with 383 additions and 156 deletions
+2
View File
@@ -118,6 +118,7 @@ MoEBackend = Literal[
"flashinfer_cutlass",
"flashinfer_cutedsl",
"marlin",
"humming",
"aiter",
"emulation",
]
@@ -148,6 +149,7 @@ class KernelConfig:
- "flashinfer_cutlass": Use FlashInfer with CUTLASS kernels
- "flashinfer_cutedsl": Use FlashInfer with CuteDSL kernels (FP4 only)
- "marlin": Use Marlin kernels (weight-only quantization)
- "humming": Use Humming Mixed Precision kernels
- "aiter": Use AMD AITer kernels (ROCm only)
- "emulation": use BF16/FP16 GEMM, dequantizing weights and
running QDQ on activations.
+2 -2
View File
@@ -1231,8 +1231,8 @@ environment_variables: dict[str, Callable[[], Any]] = {
# if 1, force use indexed gemm
# if 0, force use grouped gemm
# if None, choose better gemm type automatically
"VLLM_HUMMING_MOE_GEMM_TYPE": lambda: maybe_convert_bool(
os.environ.get("VLLM_HUMMING_MOE_GEMM_TYPE", None)
"VLLM_HUMMING_MOE_GEMM_TYPE": lambda: os.environ.get(
"VLLM_HUMMING_MOE_GEMM_TYPE", None
),
# Whether to use DeepEPLL kernels for NVFP4 quantization and dispatch method
# only supported on Blackwell GPUs and with
@@ -4,7 +4,7 @@
import json
import math
from typing import TYPE_CHECKING, Any
from typing import Any
import torch
from humming import dtypes
@@ -16,7 +16,11 @@ from vllm import envs
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
moe_align_block_size,
)
@@ -34,21 +38,16 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
from vllm.platforms import current_platform
from vllm.v1.worker.workspace import current_workspace_manager
if TYPE_CHECKING:
from vllm.model_executor.layers.quantization.humming import HummingMoEMethod
logger = init_logger(__name__)
def get_humming_moe_gemm_type() -> str:
env_gemm_type: str = envs.VLLM_HUMMING_MOE_GEMM_TYPE or ""
env_gemm_type = env_gemm_type.lower()
if env_gemm_type in ["indexed", "grouped"]:
if env_gemm_type == "indexed":
gemm_type = env_gemm_type
elif current_platform.has_device_capability(90):
# for device that supports TMA, use grouped gemm
gemm_type = "grouped"
elif env_gemm_type in ["grouped_contiguous", "grouped"]:
gemm_type = "grouped_contiguous"
else:
gemm_type = "indexed"
@@ -60,49 +59,44 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
def __init__(
self,
layer: torch.nn.Module,
quant_method: "HummingMoEMethod",
prepare_finalize: mk.FusedMoEPrepareAndFinalizeModular | None = None,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
max_num_tokens: int | None = None,
num_dispatchers: int | None = None,
):
self.layer = layer
self.num_experts = self.layer.num_experts
self.global_num_experts = self.layer.global_num_experts
self.init_humming_moe()
if prepare_finalize is not None:
max_num_tokens: int | None = None
num_dispatchers: int | None = None
if self.is_batched:
max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
num_dispatchers = prepare_finalize.num_dispatchers()
if self.is_batched():
assert max_num_tokens is not None and num_dispatchers is not None
assert quant_method.moe_quant_config is not None
super().__init__(
moe_config=quant_method.moe,
quant_config=quant_method.moe_quant_config,
max_num_tokens=max_num_tokens,
num_dispatchers=num_dispatchers,
)
else:
assert not self.is_batched
super().__init__(
moe_config=moe_config,
quant_config=quant_config,
max_num_tokens=max_num_tokens,
num_dispatchers=num_dispatchers,
)
def init_humming_moe(self):
self.compute_config = {
"use_batch_invariant": envs.VLLM_BATCH_INVARIANT,
"use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,
"gemm_type": self.humming_gemm_type.value,
"gemm_type": self.humming_gemm_type().value,
}
self.w13_tuning_config = HummingMethod.get_default_tuning_configs(
layer=self.layer,
use_f16_accum=envs.VLLM_HUMMING_USE_F16_ACCUM,
use_batch_invariant=envs.VLLM_BATCH_INVARIANT,
gemm_type=self.humming_gemm_type,
gemm_type=self.humming_gemm_type(),
sublayer_name="w13",
)
self.w2_tuning_config = HummingMethod.get_default_tuning_configs(
layer=self.layer,
use_f16_accum=envs.VLLM_HUMMING_USE_F16_ACCUM,
use_batch_invariant=envs.VLLM_BATCH_INVARIANT,
gemm_type=self.humming_gemm_type,
gemm_type=self.humming_gemm_type(),
sublayer_name="w2",
)
self.compute_config_str = json.dumps(self.compute_config)
@@ -124,13 +118,13 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
global_num_experts = self.global_num_experts
return math.ceil(global_valid_shape_m * num_experts / global_num_experts)
@property
def humming_gemm_type(self) -> HummingGemmType:
@staticmethod
def humming_gemm_type() -> HummingGemmType:
raise NotImplementedError
@property
def is_batched(self) -> bool:
return self.activation_format() == mk.FusedMoEActivationFormat.BatchedExperts
@classmethod
def is_batched(cls) -> bool:
return cls.activation_format() == mk.FusedMoEActivationFormat.BatchedExperts
@staticmethod
def _supports_quant_scheme(
@@ -189,7 +183,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
assert w1.size(0) == num_experts
assert w2.size(0) == num_experts
if not self.is_batched:
if not self.is_batched():
num_tokens = a1.size(0)
assert topk_ids.size(0) == num_tokens
else:
@@ -201,7 +195,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
def get_buffer_metas(self, M: int, topk: int, activation: MoEActivation):
num_experts = self.num_experts
N = self.layer.intermediate_size
N = self.layer.intermediate_size_per_partition
K = self.layer.hidden_size
assert isinstance(num_experts, int)
assert isinstance(N, int)
@@ -218,7 +212,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
# The output must be derived from workspace1.
output_shape: tuple[int, ...]
if self.is_batched:
if self.is_batched():
max_num_tokens = self.max_num_tokens
num_dispatchers = self.num_dispatchers
assert max_num_tokens is not None and num_dispatchers is not None
@@ -227,7 +221,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
output_shape = (num_experts, max_num_tokens * num_dispatchers, K)
else:
input_shape_m = M
if self.humming_gemm_type != HummingGemmType.INDEXED:
if self.humming_gemm_type() != HummingGemmType.INDEXED:
input_shape_m = M * topk
real_shape_m = M * topk
output_shape = (M, K)
@@ -262,7 +256,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
"dtype": torch_dtype_map[a_dtype],
},
"down_output": {
"shape": output_shape if self.is_batched else (real_shape_m, K),
"shape": output_shape if self.is_batched() else (real_shape_m, K),
"dtype": torch_dtype_map[c_dtype],
},
"output": {
@@ -288,7 +282,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
]
# batched moe use down_output as output
if not self.is_batched:
if not self.is_batched():
required_buffers.append("output")
return buffer_metas, required_buffers
@@ -308,7 +302,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
else:
workspace2_nbytes = max(workspace2_nbytes, nbytes)
output_key = "down_output" if self.is_batched else "output"
output_key = "down_output" if self.is_batched() else "output"
output_shape = buffer_metas[output_key]["shape"]
return (workspace1_nbytes // 2,), (workspace2_nbytes // 2,), output_shape
@@ -395,6 +389,33 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
):
raise NotImplementedError
@staticmethod
def is_supported_config(
cls: type[mk.FusedMoEExperts],
moe_config: FusedMoEConfig,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
activation_format: mk.FusedMoEActivationFormat,
) -> tuple[bool, str | None]:
if activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
supported = cls.activation_format() == activation_format
reason = "activation_format mismatched"
elif activation_format == mk.FusedMoEActivationFormat.Standard:
if cls.activation_format() != mk.FusedMoEActivationFormat.Standard:
supported = False
reason = "activation_format mismatched"
else:
assert hasattr(cls, "humming_gemm_type")
gemm_type = cls.humming_gemm_type().value.lower()
preferred_gemm_type = get_humming_moe_gemm_type().lower()
supported = preferred_gemm_type == gemm_type
reason = "preferred gemm type mismatched"
else:
supported = False
reason = "unsupported activation_format"
return supported, None if supported else reason
class HummingIndexedExperts(HummingExpertsBase):
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
@@ -404,8 +425,8 @@ class HummingIndexedExperts(HummingExpertsBase):
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@property
def humming_gemm_type(self) -> HummingGemmType:
@staticmethod
def humming_gemm_type() -> HummingGemmType:
return HummingGemmType.INDEXED
def prepare_humming_moe_kwargs(
@@ -526,8 +547,8 @@ class HummingGroupedExperts(HummingExpertsBase):
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@property
def humming_gemm_type(self) -> HummingGemmType:
@staticmethod
def humming_gemm_type() -> HummingGemmType:
return HummingGemmType.GROUPED_CONTIGUOUS
def main_apply(
@@ -619,8 +640,8 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.BatchedExperts
@property
def humming_gemm_type(self) -> HummingGemmType:
@staticmethod
def humming_gemm_type() -> HummingGemmType:
return HummingGemmType.GROUPED_MASKED
def main_apply(
@@ -1103,9 +1103,6 @@ class FusedMoE(PluggableLayer):
return_success: bool = False,
) -> bool | None:
quant_config_name = self.quant_config and self.quant_config.get_name()
if quant_config_name == "humming":
assert hasattr(self.quant_method, "weight_schema")
quant_config_name = self.quant_method.weight_schema.quant_method
if quant_config_name == "gpt_oss_mxfp4":
# (FIXME) for gpt-oss all experts are combined
if "bias" in weight_name:
@@ -68,6 +68,8 @@ class Mxfp4MoeBackend(Enum):
XPU = "XPU"
# Emulation
EMULATION = "EMULATION"
# Humming
HUMMING = "HUMMING"
# Backends that share the same TRTLLM weight format
@@ -130,6 +132,19 @@ def backend_to_kernel_cls(
return [UnfusedOAITritonExperts]
elif backend == Mxfp4MoeBackend.HUMMING:
from vllm.model_executor.layers.fused_moe.fused_humming_moe import (
BatchedHummingGroupedExperts,
HummingGroupedExperts,
HummingIndexedExperts,
)
return [
BatchedHummingGroupedExperts,
HummingGroupedExperts,
HummingIndexedExperts,
]
elif backend == Mxfp4MoeBackend.MARLIN:
from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
MarlinExperts,
@@ -177,6 +192,7 @@ def map_mxfp4_backend(runner_backend: MoEBackend) -> Mxfp4MoeBackend:
"flashinfer_cutlass_afp8": Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
"triton": Mxfp4MoeBackend.TRITON,
"triton_unfused": Mxfp4MoeBackend.TRITON_UNFUSED,
"humming": Mxfp4MoeBackend.HUMMING,
"marlin": Mxfp4MoeBackend.MARLIN,
"aiter": Mxfp4MoeBackend.AITER,
"xpu": Mxfp4MoeBackend.XPU,
@@ -573,7 +589,21 @@ def convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
sf_block_size = 32 # mxfp4 block size
if mxfp4_backend in (
if mxfp4_backend == Mxfp4MoeBackend.HUMMING:
from vllm.model_executor.layers.quantization.utils.humming_utils import (
prepare_humming_moe_layer,
)
prepare_humming_moe_layer(layer, {"quant_method": "gpt_oss_mxfp4"})
return (
layer.w13_weight,
layer.w2_weight,
layer.w13_weight_scale,
layer.w2_weight_scale,
getattr(layer, "w13_bias", None),
getattr(layer, "w2_bias", None),
)
elif mxfp4_backend in (
Mxfp4MoeBackend.MARLIN,
Mxfp4MoeBackend.BATCHED_MARLIN,
):
@@ -970,6 +1000,21 @@ def convert_weight_to_mxfp4_moe_kernel_format(
w2_bias,
)
if mxfp4_backend == Mxfp4MoeBackend.HUMMING:
from vllm.model_executor.layers.quantization.utils.humming_utils import (
prepare_humming_moe_layer,
)
prepare_humming_moe_layer(layer, {"quant_method": "mxfp4"})
return (
layer.w13_weight,
layer.w2_weight,
layer.w13_weight_scale,
layer.w2_weight_scale,
getattr(layer, "w13_bias", None),
getattr(layer, "w2_bias", None),
)
if mxfp4_backend in (Mxfp4MoeBackend.MARLIN, Mxfp4MoeBackend.BATCHED_MARLIN):
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
prepare_moe_mxfp4_layer_for_marlin,
@@ -1175,6 +1220,7 @@ def make_mxfp4_moe_quant_config(
swiglu_limit: float | None = None,
w1_bias: torch.Tensor | None = None,
w2_bias: torch.Tensor | None = None,
layer: torch.nn.Module | None = None,
) -> FusedMoEQuantConfig | None:
"""Create a FusedMoEQuantConfig for the given MXFP4 backend."""
if mxfp4_backend == Mxfp4MoeBackend.DEEPGEMM_MXFP4:
@@ -1234,6 +1280,14 @@ def make_mxfp4_moe_quant_config(
gemm1_beta=gemm1_beta,
gemm1_clamp_limit=swiglu_limit,
)
elif mxfp4_backend == Mxfp4MoeBackend.HUMMING:
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
from vllm.model_executor.layers.quantization.utils.humming_utils import (
get_humming_moe_quant_config,
)
assert isinstance(layer, FusedMoE)
return get_humming_moe_quant_config(layer)
else:
return ocp_mx_moe_quant_config(
quant_dtype="mxfp4",
@@ -1254,6 +1308,7 @@ def make_mxfp4_moe_kernel(
mxfp4_backend: Mxfp4MoeBackend,
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
shared_experts: torch.nn.Module | None = None,
layer: torch.nn.Module | None = None,
) -> mk.FusedMoEKernel:
"""Create a FusedMoEKernel for the given MXFP4 backend."""
is_monolithic = issubclass(experts_cls, mk.FusedMoEExpertsMonolithic)
@@ -1269,6 +1324,11 @@ def make_mxfp4_moe_kernel(
logger.info_once("Using %s", prepare_finalize.__class__.__name__)
extra_kwargs = {}
if mxfp4_backend == Mxfp4MoeBackend.HUMMING:
assert layer is not None
extra_kwargs["layer"] = layer
# Create Experts.
if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
@@ -1278,11 +1338,13 @@ def make_mxfp4_moe_kernel(
quant_config=moe_quant_config,
max_num_tokens=max_num_tokens,
num_dispatchers=prepare_finalize.num_dispatchers(),
**extra_kwargs,
)
else:
experts = experts_cls(
moe_config=moe_config,
quant_config=moe_quant_config,
**extra_kwargs,
)
kernel = mk.FusedMoEKernel(
@@ -9,11 +9,9 @@ import regex as re
import torch
from vllm import envs
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEQuantConfig,
FusedMoEQuantDesc,
)
from vllm.model_executor.layers.fused_moe.layer import (
FusedMoE,
@@ -32,7 +30,6 @@ from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
QuantizeMethodBase,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.parameter import (
BasevLLMParameter,
@@ -215,6 +212,15 @@ class HummingConfig(QuantizationConfig):
def override_quantization_method(
cls, hf_quant_cfg, user_quant, hf_config=None
) -> QuantizationMethods | None:
if user_quant == "humming" and hf_config is not None:
model_type = hf_config.model_type
quant_method = hf_quant_cfg.get("quant_method", None)
if model_type == "gpt_oss" and quant_method == "mxfp4":
msg = (
"For gpt-oss model, use '--moe-backend humming' "
"instead of '--quantization humming'."
)
raise ValueError(msg)
return "humming" if user_quant == "humming" else None
def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):
@@ -299,8 +305,6 @@ class HummingConfig(QuantizationConfig):
force_weight_schema = schema
if weight_schema is not None:
if weight_schema.quant_method == "gpt_oss_mxfp4" and layer_type != "moe":
return None
input_schema = None
force_input_schema = None
@@ -335,12 +339,6 @@ class HummingConfig(QuantizationConfig):
elif isinstance(layer, LinearBase):
layer_type = "linear"
# TODO: remove this after humming moe backend is ready
quant_method = self.full_config.get("quant_method", None)
moe_activation = getattr(layer, "activation", None)
if quant_method == "mxfp4" and moe_activation == MoEActivation.SWIGLUOAI:
self.full_config["quan_method"] = "gpt_oss_mxfp4"
quant_config = self.get_quant_config_for_layer(prefix, layer_type)
if quant_config is None:
if isinstance(layer, FusedMoE):
@@ -760,62 +758,18 @@ class HummingMoEMethod(FusedMoEMethodBase):
layer.register_buffer("locks", locks)
def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig:
self.process_weights_after_loading(layer)
input_schema = self.input_schemas["w13"]
weight_schema = self.weight_schemas["w13"]
a_dtype = input_schema.a_dtype
if a_dtype is None or a_dtype.num_bits == 16:
a_quant_desc = FusedMoEQuantDesc(dtype=None)
else:
shape = GroupShape(row=1, col=-1)
a_quant_desc = FusedMoEQuantDesc(dtype=str(a_dtype), shape=shape)
weight_scale_group_size = weight_schema.weight_scale_group_size
weight_scale_group_size_n = weight_schema.weight_scale_group_size_n
weight_group_shape: tuple[int, ...] = ()
if weight_scale_group_size_n > 1:
weight_group_shape = GroupShape(
row=weight_scale_group_size,
col=weight_scale_group_size_n,
)
elif weight_scale_group_size == 0:
weight_group_shape = GroupShape(row=-1, col=1)
else:
weight_group_shape = GroupShape(row=weight_scale_group_size, col=1)
w1_quant_desc = FusedMoEQuantDesc(
dtype=str(weight_schema.b_dtype),
shape=weight_group_shape,
scale=getattr(layer, "w13_weight_scale", None),
alpha_or_gscale=getattr(layer, "w13_global_scale", None),
zp=getattr(layer, "w13_zero_point", None),
bias=getattr(layer, "w13_bias", None),
from vllm.model_executor.layers.quantization.utils.humming_utils import (
get_humming_moe_quant_config,
)
w2_quant_desc = FusedMoEQuantDesc(
dtype=str(weight_schema.b_dtype),
shape=weight_group_shape,
scale=getattr(layer, "w2_weight_scale", None),
alpha_or_gscale=getattr(layer, "w2_global_scale", None),
zp=getattr(layer, "w2_zero_point", None),
bias=getattr(layer, "w2_bias", None),
)
return FusedMoEQuantConfig(
_a1=a_quant_desc,
_a2=a_quant_desc,
_w1=w1_quant_desc,
_w2=w2_quant_desc,
)
return get_humming_moe_quant_config(layer)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
if getattr(self, "processed", False):
return
self.processed = True
self.weight_schemas = {}
self.input_schemas = {}
layer.weight_schemas = {}
layer.input_schemas = {}
for sublayer_name, configs in layer.sublayer_configs.items():
input_schema = self.input_schema
weight_schema = self.weight_schema
@@ -858,8 +812,8 @@ class HummingMoEMethod(FusedMoEMethodBase):
param = torch.nn.Parameter(tensor, requires_grad=False)
setattr(layer, name, param)
self.weight_schemas[sublayer_name] = weight_schema
self.input_schemas[sublayer_name] = input_schema
layer.weight_schemas[sublayer_name] = weight_schema
layer.input_schemas[sublayer_name] = input_schema
# force requant (origin quant setting -> fp16/bf16 -> new_quant setting)
assert isinstance(weight_schema, HummingWeightSchema)
@@ -913,10 +867,11 @@ class HummingMoEMethod(FusedMoEMethodBase):
# use moe modular
experts: HummingIndexedExperts | HummingGroupedExperts
assert self.moe_quant_config is not None
if get_humming_moe_gemm_type() == "indexed":
experts = HummingIndexedExperts(layer, self)
experts = HummingIndexedExperts(layer, self.moe, self.moe_quant_config)
else:
experts = HummingGroupedExperts(layer, self)
experts = HummingGroupedExperts(layer, self.moe, self.moe_quant_config)
self.experts = experts
def select_gemm_impl(
@@ -927,12 +882,19 @@ class HummingMoEMethod(FusedMoEMethodBase):
from vllm.model_executor.layers.fused_moe import modular_kernel as mk
activation_format = prepare_finalize.activation_format
assert self.moe_quant_config is not None
if activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
return BatchedHummingGroupedExperts(layer, self, prepare_finalize)
return BatchedHummingGroupedExperts(
layer=layer,
moe_config=self.moe,
quant_config=self.moe_quant_config,
max_num_tokens=prepare_finalize.max_num_tokens_per_rank(),
num_dispatchers=prepare_finalize.num_dispatchers(),
)
elif get_humming_moe_gemm_type() == "indexed":
return HummingIndexedExperts(layer, self, prepare_finalize)
return HummingIndexedExperts(layer, self.moe, self.moe_quant_config)
else:
return HummingGroupedExperts(layer, self, prepare_finalize)
return HummingGroupedExperts(layer, self.moe, self.moe_quant_config)
def apply(
self,
@@ -366,6 +366,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
experts_cls=self.experts_cls,
routing_tables=layer._maybe_init_expert_routing_tables(),
shared_experts=layer.shared_experts,
layer=layer,
)
def process_weights_after_loading(self, layer):
@@ -404,6 +405,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
gemm1_alpha=1.702,
gemm1_beta=1.0,
swiglu_limit=7.0,
layer=layer,
)
def select_gemm_impl(
@@ -692,6 +694,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
experts_cls=self.experts_cls,
routing_tables=layer._maybe_init_expert_routing_tables(),
shared_experts=layer.shared_experts,
layer=layer,
)
def process_weights_after_loading(self, layer):
@@ -729,6 +732,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
w1_bias=w1_bias,
w2_bias=w2_bias,
swiglu_limit=swiglu_limit,
layer=layer,
)
def select_gemm_impl(
@@ -1,35 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
moe_align_block_size,
)
def humming_moe_align(
configs: list[int],
topk_ids: torch.Tensor,
num_experts: int,
expert_map: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
assert len(configs) > 0 and len(configs) % 3 == 0
# NOTE: we choose moe_block_size based on
# num_tokens * top_k (= topk_ids.nelement())
shape_m = topk_ids.nelement()
for i in range(len(configs) // 3):
if shape_m > configs[i * 3] and shape_m <= configs[i * 3 + 1]:
block_size = configs[i * 3 + 2]
break
else:
raise ValueError(f"Could not find a matching block_size for shape_m={shape_m}")
return moe_align_block_size(
topk_ids=topk_ids,
block_size=block_size,
num_experts=num_experts,
expert_map=expert_map,
pad_sorted_ids=False,
ignore_invalid_experts=True,
)
@@ -0,0 +1,214 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import regex as re
import torch
from humming.layer import HummingInputSchema, HummingMethod
from humming.schema import BaseWeightSchema
from vllm import envs
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
FusedMoEQuantDesc,
)
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
from vllm.model_executor.layers.linear import LinearBase
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
def humming_is_layer_skipped(config: dict[str, Any], prefix: str):
if not config:
return True
keys = ["ignored_layers", "ignore", "modules_to_not_convert"]
ignored_layers: list[str] = []
for key in keys:
ignored_layers = config.get(key, []) or []
if not ignored_layers:
break
if any(module_name in prefix for module_name in ignored_layers):
return True
if "lm_head" in prefix:
return True
for regex in config.get("dynamic", {}):
if regex[:1] != "-":
continue
if re.match(regex[2:], prefix):
return True
return False
def prepare_humming_layer(layer: LinearBase, quant_config: dict):
weight_schema = BaseWeightSchema.from_config(quant_config)
input_schema = HummingInputSchema()
shape_k_stacks = [layer.input_size_per_partition]
shape_n_stacks = layer.output_partition_sizes
# Step 1: convert weight to humming standard format
weight_schema, tensors = weight_schema.convert_humming(
tensors=layer.named_parameters(),
shape_n_stacks=shape_n_stacks,
shape_k_stacks=shape_k_stacks,
param_dtype=layer.params_dtype,
)
layer.weight_schema = weight_schema
for name, _ in list(layer.named_parameters()):
delattr(layer, name)
for name, tensor in tensors.items():
param = torch.nn.Parameter(tensor, requires_grad=False)
setattr(layer, name, param)
# Step 2: transform weight (humming standard format) for forwarding
HummingMethod.prepare_layer_meta(
layer=layer,
shape_n=layer.output_partition_sizes_sum,
shape_k=layer.input_size_per_partition,
weight_schema=weight_schema,
input_schema=input_schema,
pad_n_to_multiple=256,
pad_k_to_multiple=128,
has_bias=layer.has_bias,
torch_dtype=layer.param_dtype,
)
HummingMethod.transform_humming_layer(layer)
def prepare_humming_moe_layer(layer: FusedMoE, quant_config: dict):
weight_schema = BaseWeightSchema.from_config(quant_config)
input_quant_config = envs.VLLM_HUMMING_INPUT_QUANT_CONFIG or {}
if humming_is_layer_skipped(input_quant_config, layer.layer_name):
input_schema = HummingInputSchema()
else:
# TODO: read input_quant_config from quant_config
input_schema = HummingInputSchema.from_config(input_quant_config)
is_gated = layer.activation.is_gated
shape_config = {
"w13": (
layer.moe_config.intermediate_size_per_partition * 2,
layer.moe_config.hidden_dim,
),
"w2": (
layer.moe_config.hidden_dim,
layer.moe_config.intermediate_size_per_partition * (1 if is_gated else 2),
),
}
layer.weight_schemas = {}
layer.input_schemas = {}
for sublayer_name in shape_config:
# Step 1: convert weight to humming standard format
tensors: dict[str, torch.Tensor] = dict(
(key.removeprefix(sublayer_name + "_"), value)
for key, value in layer.state_dict().items()
if key.startswith(sublayer_name + "_")
)
shape_n, shape_k = shape_config[sublayer_name]
shape_n_stacks = [shape_n]
shape_k_stacks = [shape_k]
if sublayer_name == "w13":
shape_n_stacks = [shape_n // 2] * 2
weight_schema_new, tensors = weight_schema.convert_humming(
tensors=tensors,
shape_n_stacks=shape_n_stacks,
shape_k_stacks=shape_k_stacks,
num_experts=layer.local_num_experts,
param_dtype=layer.params_dtype,
)
layer.weight_schemas[sublayer_name] = weight_schema_new
layer.input_schemas[sublayer_name] = input_schema
for name, _ in list(layer.named_parameters()):
if not name.startswith(sublayer_name + "_"):
continue
delattr(layer, name)
for name, tensor in tensors.items():
name = f"{sublayer_name}_{name}"
param = torch.nn.Parameter(tensor, requires_grad=False)
setattr(layer, name, param)
# Step 2: transform weight (humming standard format) for forwarding
HummingMethod.prepare_layer_meta(
layer=layer,
shape_n=shape_n,
shape_k=shape_k,
pad_n_to_multiple=256,
pad_k_to_multiple=128,
input_schema=input_schema,
weight_schema=weight_schema_new,
has_bias=layer.moe_config.has_bias,
num_experts=layer.num_experts,
torch_dtype=layer.params_dtype,
sublayer_name=sublayer_name,
)
HummingMethod.transform_humming_layer(layer, sublayer_name=sublayer_name)
if not hasattr(layer, "locks"):
device = layer.w13_weight.device
locks = torch.zeros(1024, dtype=torch.int32, device=device)
layer.register_buffer("locks", locks)
def get_humming_moe_quant_config(layer: FusedMoE):
input_schema = layer.input_schemas["w13"]
weight_schema = layer.weight_schemas["w13"]
a_dtype = input_schema.a_dtype
if a_dtype is None or a_dtype.num_bits == 16:
a_quant_desc = FusedMoEQuantDesc(dtype=None)
else:
shape = GroupShape(row=1, col=-1)
a_quant_desc = FusedMoEQuantDesc(dtype=str(a_dtype), shape=shape)
weight_scale_group_size = weight_schema.weight_scale_group_size
weight_scale_group_size_n = weight_schema.weight_scale_group_size_n
weight_group_shape: tuple[int, ...] = ()
if weight_scale_group_size_n > 1:
weight_group_shape = GroupShape(
row=weight_scale_group_size,
col=weight_scale_group_size_n,
)
elif weight_scale_group_size == 0:
weight_group_shape = GroupShape(row=-1, col=1)
else:
weight_group_shape = GroupShape(row=weight_scale_group_size, col=1)
w1_quant_desc = FusedMoEQuantDesc(
dtype=str(weight_schema.b_dtype),
shape=weight_group_shape,
scale=getattr(layer, "w13_weight_scale", None),
alpha_or_gscale=getattr(layer, "w13_global_scale", None),
zp=getattr(layer, "w13_zero_point", None),
bias=getattr(layer, "w13_bias", None),
)
w2_quant_desc = FusedMoEQuantDesc(
dtype=str(weight_schema.b_dtype),
shape=weight_group_shape,
scale=getattr(layer, "w2_weight_scale", None),
alpha_or_gscale=getattr(layer, "w2_global_scale", None),
zp=getattr(layer, "w2_zero_point", None),
bias=getattr(layer, "w2_bias", None),
)
return FusedMoEQuantConfig(
_a1=a_quant_desc,
_a2=a_quant_desc,
_w1=w1_quant_desc,
_w2=w2_quant_desc,
)