From 08834cc3ceb86d77231666a5e07847f741637355 Mon Sep 17 00:00:00 2001 From: Jinzhen Lin Date: Sun, 3 May 2026 09:36:03 +0800 Subject: [PATCH] [Quantization] add humming mxfp4 moe backend (#41083) Signed-off-by: Jinzhen Lin --- vllm/config/kernel.py | 2 + vllm/envs.py | 4 +- .../layers/fused_moe/fused_humming_moe.py | 117 ++++++---- vllm/model_executor/layers/fused_moe/layer.py | 3 - .../layers/fused_moe/oracle/mxfp4.py | 64 +++++- .../layers/quantization/humming.py | 96 +++----- .../layers/quantization/mxfp4.py | 4 + .../quantization/utils/humming_moe_utils.py | 35 --- .../quantization/utils/humming_utils.py | 214 ++++++++++++++++++ 9 files changed, 383 insertions(+), 156 deletions(-) delete mode 100644 vllm/model_executor/layers/quantization/utils/humming_moe_utils.py create mode 100644 vllm/model_executor/layers/quantization/utils/humming_utils.py diff --git a/vllm/config/kernel.py b/vllm/config/kernel.py index e5dcc20b50..f7d5f19e23 100644 --- a/vllm/config/kernel.py +++ b/vllm/config/kernel.py @@ -118,6 +118,7 @@ MoEBackend = Literal[ "flashinfer_cutlass", "flashinfer_cutedsl", "marlin", + "humming", "aiter", "emulation", ] @@ -148,6 +149,7 @@ class KernelConfig: - "flashinfer_cutlass": Use FlashInfer with CUTLASS kernels - "flashinfer_cutedsl": Use FlashInfer with CuteDSL kernels (FP4 only) - "marlin": Use Marlin kernels (weight-only quantization) + - "humming": Use Humming Mixed Precision kernels - "aiter": Use AMD AITer kernels (ROCm only) - "emulation": use BF16/FP16 GEMM, dequantizing weights and running QDQ on activations. diff --git a/vllm/envs.py b/vllm/envs.py index 0955894754..b2db5a8112 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -1231,8 +1231,8 @@ environment_variables: dict[str, Callable[[], Any]] = { # if 1, force use indexed gemm # if 0, force use grouped gemm # if None, choose better gemm type automatically - "VLLM_HUMMING_MOE_GEMM_TYPE": lambda: maybe_convert_bool( - os.environ.get("VLLM_HUMMING_MOE_GEMM_TYPE", None) + "VLLM_HUMMING_MOE_GEMM_TYPE": lambda: os.environ.get( + "VLLM_HUMMING_MOE_GEMM_TYPE", None ), # Whether to use DeepEPLL kernels for NVFP4 quantization and dispatch method # only supported on Blackwell GPUs and with diff --git a/vllm/model_executor/layers/fused_moe/fused_humming_moe.py b/vllm/model_executor/layers/fused_moe/fused_humming_moe.py index 6a2417cd4d..64d12f9558 100644 --- a/vllm/model_executor/layers/fused_moe/fused_humming_moe.py +++ b/vllm/model_executor/layers/fused_moe/fused_humming_moe.py @@ -4,7 +4,7 @@ import json import math -from typing import TYPE_CHECKING, Any +from typing import Any import torch from humming import dtypes @@ -16,7 +16,11 @@ from vllm import envs from vllm.forward_context import get_forward_context from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe.activation import MoEActivation -from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig +from vllm.model_executor.layers.fused_moe.config import ( + FusedMoEConfig, + FusedMoEParallelConfig, + FusedMoEQuantConfig, +) from vllm.model_executor.layers.fused_moe.moe_align_block_size import ( moe_align_block_size, ) @@ -34,21 +38,16 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey from vllm.platforms import current_platform from vllm.v1.worker.workspace import current_workspace_manager -if TYPE_CHECKING: - from vllm.model_executor.layers.quantization.humming import HummingMoEMethod - - logger = init_logger(__name__) def get_humming_moe_gemm_type() -> str: env_gemm_type: str = envs.VLLM_HUMMING_MOE_GEMM_TYPE or "" env_gemm_type = env_gemm_type.lower() - if env_gemm_type in ["indexed", "grouped"]: + if env_gemm_type == "indexed": gemm_type = env_gemm_type - elif current_platform.has_device_capability(90): - # for device that supports TMA, use grouped gemm - gemm_type = "grouped" + elif env_gemm_type in ["grouped_contiguous", "grouped"]: + gemm_type = "grouped_contiguous" else: gemm_type = "indexed" @@ -60,49 +59,44 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): def __init__( self, layer: torch.nn.Module, - quant_method: "HummingMoEMethod", - prepare_finalize: mk.FusedMoEPrepareAndFinalizeModular | None = None, + moe_config: FusedMoEConfig, + quant_config: FusedMoEQuantConfig, + max_num_tokens: int | None = None, + num_dispatchers: int | None = None, ): self.layer = layer self.num_experts = self.layer.num_experts self.global_num_experts = self.layer.global_num_experts self.init_humming_moe() - if prepare_finalize is not None: - max_num_tokens: int | None = None - num_dispatchers: int | None = None - if self.is_batched: - max_num_tokens = prepare_finalize.max_num_tokens_per_rank() - num_dispatchers = prepare_finalize.num_dispatchers() + if self.is_batched(): + assert max_num_tokens is not None and num_dispatchers is not None - assert quant_method.moe_quant_config is not None - super().__init__( - moe_config=quant_method.moe, - quant_config=quant_method.moe_quant_config, - max_num_tokens=max_num_tokens, - num_dispatchers=num_dispatchers, - ) - else: - assert not self.is_batched + super().__init__( + moe_config=moe_config, + quant_config=quant_config, + max_num_tokens=max_num_tokens, + num_dispatchers=num_dispatchers, + ) def init_humming_moe(self): self.compute_config = { "use_batch_invariant": envs.VLLM_BATCH_INVARIANT, "use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM, - "gemm_type": self.humming_gemm_type.value, + "gemm_type": self.humming_gemm_type().value, } self.w13_tuning_config = HummingMethod.get_default_tuning_configs( layer=self.layer, use_f16_accum=envs.VLLM_HUMMING_USE_F16_ACCUM, use_batch_invariant=envs.VLLM_BATCH_INVARIANT, - gemm_type=self.humming_gemm_type, + gemm_type=self.humming_gemm_type(), sublayer_name="w13", ) self.w2_tuning_config = HummingMethod.get_default_tuning_configs( layer=self.layer, use_f16_accum=envs.VLLM_HUMMING_USE_F16_ACCUM, use_batch_invariant=envs.VLLM_BATCH_INVARIANT, - gemm_type=self.humming_gemm_type, + gemm_type=self.humming_gemm_type(), sublayer_name="w2", ) self.compute_config_str = json.dumps(self.compute_config) @@ -124,13 +118,13 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): global_num_experts = self.global_num_experts return math.ceil(global_valid_shape_m * num_experts / global_num_experts) - @property - def humming_gemm_type(self) -> HummingGemmType: + @staticmethod + def humming_gemm_type() -> HummingGemmType: raise NotImplementedError - @property - def is_batched(self) -> bool: - return self.activation_format() == mk.FusedMoEActivationFormat.BatchedExperts + @classmethod + def is_batched(cls) -> bool: + return cls.activation_format() == mk.FusedMoEActivationFormat.BatchedExperts @staticmethod def _supports_quant_scheme( @@ -189,7 +183,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): assert w1.size(0) == num_experts assert w2.size(0) == num_experts - if not self.is_batched: + if not self.is_batched(): num_tokens = a1.size(0) assert topk_ids.size(0) == num_tokens else: @@ -201,7 +195,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): def get_buffer_metas(self, M: int, topk: int, activation: MoEActivation): num_experts = self.num_experts - N = self.layer.intermediate_size + N = self.layer.intermediate_size_per_partition K = self.layer.hidden_size assert isinstance(num_experts, int) assert isinstance(N, int) @@ -218,7 +212,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): # The output must be derived from workspace1. output_shape: tuple[int, ...] - if self.is_batched: + if self.is_batched(): max_num_tokens = self.max_num_tokens num_dispatchers = self.num_dispatchers assert max_num_tokens is not None and num_dispatchers is not None @@ -227,7 +221,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): output_shape = (num_experts, max_num_tokens * num_dispatchers, K) else: input_shape_m = M - if self.humming_gemm_type != HummingGemmType.INDEXED: + if self.humming_gemm_type() != HummingGemmType.INDEXED: input_shape_m = M * topk real_shape_m = M * topk output_shape = (M, K) @@ -262,7 +256,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): "dtype": torch_dtype_map[a_dtype], }, "down_output": { - "shape": output_shape if self.is_batched else (real_shape_m, K), + "shape": output_shape if self.is_batched() else (real_shape_m, K), "dtype": torch_dtype_map[c_dtype], }, "output": { @@ -288,7 +282,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): ] # batched moe use down_output as output - if not self.is_batched: + if not self.is_batched(): required_buffers.append("output") return buffer_metas, required_buffers @@ -308,7 +302,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): else: workspace2_nbytes = max(workspace2_nbytes, nbytes) - output_key = "down_output" if self.is_batched else "output" + output_key = "down_output" if self.is_batched() else "output" output_shape = buffer_metas[output_key]["shape"] return (workspace1_nbytes // 2,), (workspace2_nbytes // 2,), output_shape @@ -395,6 +389,33 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): ): raise NotImplementedError + @staticmethod + def is_supported_config( + cls: type[mk.FusedMoEExperts], + moe_config: FusedMoEConfig, + weight_key: QuantKey | None, + activation_key: QuantKey | None, + activation_format: mk.FusedMoEActivationFormat, + ) -> tuple[bool, str | None]: + if activation_format == mk.FusedMoEActivationFormat.BatchedExperts: + supported = cls.activation_format() == activation_format + reason = "activation_format mismatched" + elif activation_format == mk.FusedMoEActivationFormat.Standard: + if cls.activation_format() != mk.FusedMoEActivationFormat.Standard: + supported = False + reason = "activation_format mismatched" + else: + assert hasattr(cls, "humming_gemm_type") + gemm_type = cls.humming_gemm_type().value.lower() + preferred_gemm_type = get_humming_moe_gemm_type().lower() + supported = preferred_gemm_type == gemm_type + reason = "preferred gemm type mismatched" + else: + supported = False + reason = "unsupported activation_format" + + return supported, None if supported else reason + class HummingIndexedExperts(HummingExpertsBase): def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce: @@ -404,8 +425,8 @@ class HummingIndexedExperts(HummingExpertsBase): def activation_format() -> mk.FusedMoEActivationFormat: return mk.FusedMoEActivationFormat.Standard - @property - def humming_gemm_type(self) -> HummingGemmType: + @staticmethod + def humming_gemm_type() -> HummingGemmType: return HummingGemmType.INDEXED def prepare_humming_moe_kwargs( @@ -526,8 +547,8 @@ class HummingGroupedExperts(HummingExpertsBase): def activation_format() -> mk.FusedMoEActivationFormat: return mk.FusedMoEActivationFormat.Standard - @property - def humming_gemm_type(self) -> HummingGemmType: + @staticmethod + def humming_gemm_type() -> HummingGemmType: return HummingGemmType.GROUPED_CONTIGUOUS def main_apply( @@ -619,8 +640,8 @@ class BatchedHummingGroupedExperts(HummingExpertsBase): def activation_format() -> mk.FusedMoEActivationFormat: return mk.FusedMoEActivationFormat.BatchedExperts - @property - def humming_gemm_type(self) -> HummingGemmType: + @staticmethod + def humming_gemm_type() -> HummingGemmType: return HummingGemmType.GROUPED_MASKED def main_apply( diff --git a/vllm/model_executor/layers/fused_moe/layer.py b/vllm/model_executor/layers/fused_moe/layer.py index 577f9a9867..3de05cd93d 100644 --- a/vllm/model_executor/layers/fused_moe/layer.py +++ b/vllm/model_executor/layers/fused_moe/layer.py @@ -1103,9 +1103,6 @@ class FusedMoE(PluggableLayer): return_success: bool = False, ) -> bool | None: quant_config_name = self.quant_config and self.quant_config.get_name() - if quant_config_name == "humming": - assert hasattr(self.quant_method, "weight_schema") - quant_config_name = self.quant_method.weight_schema.quant_method if quant_config_name == "gpt_oss_mxfp4": # (FIXME) for gpt-oss all experts are combined if "bias" in weight_name: diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index c1423362d7..3f2aca2771 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -68,6 +68,8 @@ class Mxfp4MoeBackend(Enum): XPU = "XPU" # Emulation EMULATION = "EMULATION" + # Humming + HUMMING = "HUMMING" # Backends that share the same TRTLLM weight format @@ -130,6 +132,19 @@ def backend_to_kernel_cls( return [UnfusedOAITritonExperts] + elif backend == Mxfp4MoeBackend.HUMMING: + from vllm.model_executor.layers.fused_moe.fused_humming_moe import ( + BatchedHummingGroupedExperts, + HummingGroupedExperts, + HummingIndexedExperts, + ) + + return [ + BatchedHummingGroupedExperts, + HummingGroupedExperts, + HummingIndexedExperts, + ] + elif backend == Mxfp4MoeBackend.MARLIN: from vllm.model_executor.layers.fused_moe.fused_marlin_moe import ( MarlinExperts, @@ -177,6 +192,7 @@ def map_mxfp4_backend(runner_backend: MoEBackend) -> Mxfp4MoeBackend: "flashinfer_cutlass_afp8": Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8, "triton": Mxfp4MoeBackend.TRITON, "triton_unfused": Mxfp4MoeBackend.TRITON_UNFUSED, + "humming": Mxfp4MoeBackend.HUMMING, "marlin": Mxfp4MoeBackend.MARLIN, "aiter": Mxfp4MoeBackend.AITER, "xpu": Mxfp4MoeBackend.XPU, @@ -573,7 +589,21 @@ def convert_gpt_oss_weight_to_mxfp4_moe_kernel_format( sf_block_size = 32 # mxfp4 block size - if mxfp4_backend in ( + if mxfp4_backend == Mxfp4MoeBackend.HUMMING: + from vllm.model_executor.layers.quantization.utils.humming_utils import ( + prepare_humming_moe_layer, + ) + + prepare_humming_moe_layer(layer, {"quant_method": "gpt_oss_mxfp4"}) + return ( + layer.w13_weight, + layer.w2_weight, + layer.w13_weight_scale, + layer.w2_weight_scale, + getattr(layer, "w13_bias", None), + getattr(layer, "w2_bias", None), + ) + elif mxfp4_backend in ( Mxfp4MoeBackend.MARLIN, Mxfp4MoeBackend.BATCHED_MARLIN, ): @@ -970,6 +1000,21 @@ def convert_weight_to_mxfp4_moe_kernel_format( w2_bias, ) + if mxfp4_backend == Mxfp4MoeBackend.HUMMING: + from vllm.model_executor.layers.quantization.utils.humming_utils import ( + prepare_humming_moe_layer, + ) + + prepare_humming_moe_layer(layer, {"quant_method": "mxfp4"}) + return ( + layer.w13_weight, + layer.w2_weight, + layer.w13_weight_scale, + layer.w2_weight_scale, + getattr(layer, "w13_bias", None), + getattr(layer, "w2_bias", None), + ) + if mxfp4_backend in (Mxfp4MoeBackend.MARLIN, Mxfp4MoeBackend.BATCHED_MARLIN): from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import ( prepare_moe_mxfp4_layer_for_marlin, @@ -1175,6 +1220,7 @@ def make_mxfp4_moe_quant_config( swiglu_limit: float | None = None, w1_bias: torch.Tensor | None = None, w2_bias: torch.Tensor | None = None, + layer: torch.nn.Module | None = None, ) -> FusedMoEQuantConfig | None: """Create a FusedMoEQuantConfig for the given MXFP4 backend.""" if mxfp4_backend == Mxfp4MoeBackend.DEEPGEMM_MXFP4: @@ -1234,6 +1280,14 @@ def make_mxfp4_moe_quant_config( gemm1_beta=gemm1_beta, gemm1_clamp_limit=swiglu_limit, ) + elif mxfp4_backend == Mxfp4MoeBackend.HUMMING: + from vllm.model_executor.layers.fused_moe.layer import FusedMoE + from vllm.model_executor.layers.quantization.utils.humming_utils import ( + get_humming_moe_quant_config, + ) + + assert isinstance(layer, FusedMoE) + return get_humming_moe_quant_config(layer) else: return ocp_mx_moe_quant_config( quant_dtype="mxfp4", @@ -1254,6 +1308,7 @@ def make_mxfp4_moe_kernel( mxfp4_backend: Mxfp4MoeBackend, routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None, shared_experts: torch.nn.Module | None = None, + layer: torch.nn.Module | None = None, ) -> mk.FusedMoEKernel: """Create a FusedMoEKernel for the given MXFP4 backend.""" is_monolithic = issubclass(experts_cls, mk.FusedMoEExpertsMonolithic) @@ -1269,6 +1324,11 @@ def make_mxfp4_moe_kernel( logger.info_once("Using %s", prepare_finalize.__class__.__name__) + extra_kwargs = {} + if mxfp4_backend == Mxfp4MoeBackend.HUMMING: + assert layer is not None + extra_kwargs["layer"] = layer + # Create Experts. if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts: max_num_tokens = prepare_finalize.max_num_tokens_per_rank() @@ -1278,11 +1338,13 @@ def make_mxfp4_moe_kernel( quant_config=moe_quant_config, max_num_tokens=max_num_tokens, num_dispatchers=prepare_finalize.num_dispatchers(), + **extra_kwargs, ) else: experts = experts_cls( moe_config=moe_config, quant_config=moe_quant_config, + **extra_kwargs, ) kernel = mk.FusedMoEKernel( diff --git a/vllm/model_executor/layers/quantization/humming.py b/vllm/model_executor/layers/quantization/humming.py index 59f9c2ee9b..79a1057c60 100644 --- a/vllm/model_executor/layers/quantization/humming.py +++ b/vllm/model_executor/layers/quantization/humming.py @@ -9,11 +9,9 @@ import regex as re import torch from vllm import envs -from vllm.model_executor.layers.fused_moe.activation import MoEActivation from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, FusedMoEQuantConfig, - FusedMoEQuantDesc, ) from vllm.model_executor.layers.fused_moe.layer import ( FusedMoE, @@ -32,7 +30,6 @@ from vllm.model_executor.layers.quantization.base_config import ( QuantizationConfig, QuantizeMethodBase, ) -from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.parameter import ( BasevLLMParameter, @@ -215,6 +212,15 @@ class HummingConfig(QuantizationConfig): def override_quantization_method( cls, hf_quant_cfg, user_quant, hf_config=None ) -> QuantizationMethods | None: + if user_quant == "humming" and hf_config is not None: + model_type = hf_config.model_type + quant_method = hf_quant_cfg.get("quant_method", None) + if model_type == "gpt_oss" and quant_method == "mxfp4": + msg = ( + "For gpt-oss model, use '--moe-backend humming' " + "instead of '--quantization humming'." + ) + raise ValueError(msg) return "humming" if user_quant == "humming" else None def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"): @@ -299,8 +305,6 @@ class HummingConfig(QuantizationConfig): force_weight_schema = schema if weight_schema is not None: - if weight_schema.quant_method == "gpt_oss_mxfp4" and layer_type != "moe": - return None input_schema = None force_input_schema = None @@ -335,12 +339,6 @@ class HummingConfig(QuantizationConfig): elif isinstance(layer, LinearBase): layer_type = "linear" - # TODO: remove this after humming moe backend is ready - quant_method = self.full_config.get("quant_method", None) - moe_activation = getattr(layer, "activation", None) - if quant_method == "mxfp4" and moe_activation == MoEActivation.SWIGLUOAI: - self.full_config["quan_method"] = "gpt_oss_mxfp4" - quant_config = self.get_quant_config_for_layer(prefix, layer_type) if quant_config is None: if isinstance(layer, FusedMoE): @@ -760,62 +758,18 @@ class HummingMoEMethod(FusedMoEMethodBase): layer.register_buffer("locks", locks) def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig: - self.process_weights_after_loading(layer) - - input_schema = self.input_schemas["w13"] - weight_schema = self.weight_schemas["w13"] - - a_dtype = input_schema.a_dtype - if a_dtype is None or a_dtype.num_bits == 16: - a_quant_desc = FusedMoEQuantDesc(dtype=None) - else: - shape = GroupShape(row=1, col=-1) - a_quant_desc = FusedMoEQuantDesc(dtype=str(a_dtype), shape=shape) - - weight_scale_group_size = weight_schema.weight_scale_group_size - weight_scale_group_size_n = weight_schema.weight_scale_group_size_n - weight_group_shape: tuple[int, ...] = () - if weight_scale_group_size_n > 1: - weight_group_shape = GroupShape( - row=weight_scale_group_size, - col=weight_scale_group_size_n, - ) - elif weight_scale_group_size == 0: - weight_group_shape = GroupShape(row=-1, col=1) - else: - weight_group_shape = GroupShape(row=weight_scale_group_size, col=1) - - w1_quant_desc = FusedMoEQuantDesc( - dtype=str(weight_schema.b_dtype), - shape=weight_group_shape, - scale=getattr(layer, "w13_weight_scale", None), - alpha_or_gscale=getattr(layer, "w13_global_scale", None), - zp=getattr(layer, "w13_zero_point", None), - bias=getattr(layer, "w13_bias", None), + from vllm.model_executor.layers.quantization.utils.humming_utils import ( + get_humming_moe_quant_config, ) - w2_quant_desc = FusedMoEQuantDesc( - dtype=str(weight_schema.b_dtype), - shape=weight_group_shape, - scale=getattr(layer, "w2_weight_scale", None), - alpha_or_gscale=getattr(layer, "w2_global_scale", None), - zp=getattr(layer, "w2_zero_point", None), - bias=getattr(layer, "w2_bias", None), - ) - - return FusedMoEQuantConfig( - _a1=a_quant_desc, - _a2=a_quant_desc, - _w1=w1_quant_desc, - _w2=w2_quant_desc, - ) + return get_humming_moe_quant_config(layer) def process_weights_after_loading(self, layer: torch.nn.Module) -> None: if getattr(self, "processed", False): return self.processed = True - self.weight_schemas = {} - self.input_schemas = {} + layer.weight_schemas = {} + layer.input_schemas = {} for sublayer_name, configs in layer.sublayer_configs.items(): input_schema = self.input_schema weight_schema = self.weight_schema @@ -858,8 +812,8 @@ class HummingMoEMethod(FusedMoEMethodBase): param = torch.nn.Parameter(tensor, requires_grad=False) setattr(layer, name, param) - self.weight_schemas[sublayer_name] = weight_schema - self.input_schemas[sublayer_name] = input_schema + layer.weight_schemas[sublayer_name] = weight_schema + layer.input_schemas[sublayer_name] = input_schema # force requant (origin quant setting -> fp16/bf16 -> new_quant setting) assert isinstance(weight_schema, HummingWeightSchema) @@ -913,10 +867,11 @@ class HummingMoEMethod(FusedMoEMethodBase): # use moe modular experts: HummingIndexedExperts | HummingGroupedExperts + assert self.moe_quant_config is not None if get_humming_moe_gemm_type() == "indexed": - experts = HummingIndexedExperts(layer, self) + experts = HummingIndexedExperts(layer, self.moe, self.moe_quant_config) else: - experts = HummingGroupedExperts(layer, self) + experts = HummingGroupedExperts(layer, self.moe, self.moe_quant_config) self.experts = experts def select_gemm_impl( @@ -927,12 +882,19 @@ class HummingMoEMethod(FusedMoEMethodBase): from vllm.model_executor.layers.fused_moe import modular_kernel as mk activation_format = prepare_finalize.activation_format + assert self.moe_quant_config is not None if activation_format == mk.FusedMoEActivationFormat.BatchedExperts: - return BatchedHummingGroupedExperts(layer, self, prepare_finalize) + return BatchedHummingGroupedExperts( + layer=layer, + moe_config=self.moe, + quant_config=self.moe_quant_config, + max_num_tokens=prepare_finalize.max_num_tokens_per_rank(), + num_dispatchers=prepare_finalize.num_dispatchers(), + ) elif get_humming_moe_gemm_type() == "indexed": - return HummingIndexedExperts(layer, self, prepare_finalize) + return HummingIndexedExperts(layer, self.moe, self.moe_quant_config) else: - return HummingGroupedExperts(layer, self, prepare_finalize) + return HummingGroupedExperts(layer, self.moe, self.moe_quant_config) def apply( self, diff --git a/vllm/model_executor/layers/quantization/mxfp4.py b/vllm/model_executor/layers/quantization/mxfp4.py index 0a516831c4..2be77f2b8b 100644 --- a/vllm/model_executor/layers/quantization/mxfp4.py +++ b/vllm/model_executor/layers/quantization/mxfp4.py @@ -366,6 +366,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase): experts_cls=self.experts_cls, routing_tables=layer._maybe_init_expert_routing_tables(), shared_experts=layer.shared_experts, + layer=layer, ) def process_weights_after_loading(self, layer): @@ -404,6 +405,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase): gemm1_alpha=1.702, gemm1_beta=1.0, swiglu_limit=7.0, + layer=layer, ) def select_gemm_impl( @@ -692,6 +694,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): experts_cls=self.experts_cls, routing_tables=layer._maybe_init_expert_routing_tables(), shared_experts=layer.shared_experts, + layer=layer, ) def process_weights_after_loading(self, layer): @@ -729,6 +732,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): w1_bias=w1_bias, w2_bias=w2_bias, swiglu_limit=swiglu_limit, + layer=layer, ) def select_gemm_impl( diff --git a/vllm/model_executor/layers/quantization/utils/humming_moe_utils.py b/vllm/model_executor/layers/quantization/utils/humming_moe_utils.py deleted file mode 100644 index 82788a0e76..0000000000 --- a/vllm/model_executor/layers/quantization/utils/humming_moe_utils.py +++ /dev/null @@ -1,35 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import torch - -from vllm.model_executor.layers.fused_moe.moe_align_block_size import ( - moe_align_block_size, -) - - -def humming_moe_align( - configs: list[int], - topk_ids: torch.Tensor, - num_experts: int, - expert_map: torch.Tensor | None = None, -) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - assert len(configs) > 0 and len(configs) % 3 == 0 - # NOTE: we choose moe_block_size based on - # num_tokens * top_k (= topk_ids.nelement()) - shape_m = topk_ids.nelement() - - for i in range(len(configs) // 3): - if shape_m > configs[i * 3] and shape_m <= configs[i * 3 + 1]: - block_size = configs[i * 3 + 2] - break - else: - raise ValueError(f"Could not find a matching block_size for shape_m={shape_m}") - - return moe_align_block_size( - topk_ids=topk_ids, - block_size=block_size, - num_experts=num_experts, - expert_map=expert_map, - pad_sorted_ids=False, - ignore_invalid_experts=True, - ) diff --git a/vllm/model_executor/layers/quantization/utils/humming_utils.py b/vllm/model_executor/layers/quantization/utils/humming_utils.py new file mode 100644 index 0000000000..f8c10bdcae --- /dev/null +++ b/vllm/model_executor/layers/quantization/utils/humming_utils.py @@ -0,0 +1,214 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from typing import Any + +import regex as re +import torch +from humming.layer import HummingInputSchema, HummingMethod +from humming.schema import BaseWeightSchema + +from vllm import envs +from vllm.model_executor.layers.fused_moe.config import ( + FusedMoEQuantConfig, + FusedMoEQuantDesc, +) +from vllm.model_executor.layers.fused_moe.layer import FusedMoE +from vllm.model_executor.layers.linear import LinearBase +from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape + + +def humming_is_layer_skipped(config: dict[str, Any], prefix: str): + if not config: + return True + + keys = ["ignored_layers", "ignore", "modules_to_not_convert"] + ignored_layers: list[str] = [] + for key in keys: + ignored_layers = config.get(key, []) or [] + if not ignored_layers: + break + + if any(module_name in prefix for module_name in ignored_layers): + return True + if "lm_head" in prefix: + return True + + for regex in config.get("dynamic", {}): + if regex[:1] != "-": + continue + if re.match(regex[2:], prefix): + return True + + return False + + +def prepare_humming_layer(layer: LinearBase, quant_config: dict): + weight_schema = BaseWeightSchema.from_config(quant_config) + input_schema = HummingInputSchema() + + shape_k_stacks = [layer.input_size_per_partition] + shape_n_stacks = layer.output_partition_sizes + + # Step 1: convert weight to humming standard format + weight_schema, tensors = weight_schema.convert_humming( + tensors=layer.named_parameters(), + shape_n_stacks=shape_n_stacks, + shape_k_stacks=shape_k_stacks, + param_dtype=layer.params_dtype, + ) + + layer.weight_schema = weight_schema + + for name, _ in list(layer.named_parameters()): + delattr(layer, name) + + for name, tensor in tensors.items(): + param = torch.nn.Parameter(tensor, requires_grad=False) + setattr(layer, name, param) + + # Step 2: transform weight (humming standard format) for forwarding + HummingMethod.prepare_layer_meta( + layer=layer, + shape_n=layer.output_partition_sizes_sum, + shape_k=layer.input_size_per_partition, + weight_schema=weight_schema, + input_schema=input_schema, + pad_n_to_multiple=256, + pad_k_to_multiple=128, + has_bias=layer.has_bias, + torch_dtype=layer.param_dtype, + ) + + HummingMethod.transform_humming_layer(layer) + + +def prepare_humming_moe_layer(layer: FusedMoE, quant_config: dict): + weight_schema = BaseWeightSchema.from_config(quant_config) + input_quant_config = envs.VLLM_HUMMING_INPUT_QUANT_CONFIG or {} + if humming_is_layer_skipped(input_quant_config, layer.layer_name): + input_schema = HummingInputSchema() + else: + # TODO: read input_quant_config from quant_config + input_schema = HummingInputSchema.from_config(input_quant_config) + + is_gated = layer.activation.is_gated + shape_config = { + "w13": ( + layer.moe_config.intermediate_size_per_partition * 2, + layer.moe_config.hidden_dim, + ), + "w2": ( + layer.moe_config.hidden_dim, + layer.moe_config.intermediate_size_per_partition * (1 if is_gated else 2), + ), + } + + layer.weight_schemas = {} + layer.input_schemas = {} + + for sublayer_name in shape_config: + # Step 1: convert weight to humming standard format + tensors: dict[str, torch.Tensor] = dict( + (key.removeprefix(sublayer_name + "_"), value) + for key, value in layer.state_dict().items() + if key.startswith(sublayer_name + "_") + ) + + shape_n, shape_k = shape_config[sublayer_name] + shape_n_stacks = [shape_n] + shape_k_stacks = [shape_k] + if sublayer_name == "w13": + shape_n_stacks = [shape_n // 2] * 2 + + weight_schema_new, tensors = weight_schema.convert_humming( + tensors=tensors, + shape_n_stacks=shape_n_stacks, + shape_k_stacks=shape_k_stacks, + num_experts=layer.local_num_experts, + param_dtype=layer.params_dtype, + ) + + layer.weight_schemas[sublayer_name] = weight_schema_new + layer.input_schemas[sublayer_name] = input_schema + + for name, _ in list(layer.named_parameters()): + if not name.startswith(sublayer_name + "_"): + continue + delattr(layer, name) + + for name, tensor in tensors.items(): + name = f"{sublayer_name}_{name}" + param = torch.nn.Parameter(tensor, requires_grad=False) + setattr(layer, name, param) + + # Step 2: transform weight (humming standard format) for forwarding + HummingMethod.prepare_layer_meta( + layer=layer, + shape_n=shape_n, + shape_k=shape_k, + pad_n_to_multiple=256, + pad_k_to_multiple=128, + input_schema=input_schema, + weight_schema=weight_schema_new, + has_bias=layer.moe_config.has_bias, + num_experts=layer.num_experts, + torch_dtype=layer.params_dtype, + sublayer_name=sublayer_name, + ) + + HummingMethod.transform_humming_layer(layer, sublayer_name=sublayer_name) + + if not hasattr(layer, "locks"): + device = layer.w13_weight.device + locks = torch.zeros(1024, dtype=torch.int32, device=device) + layer.register_buffer("locks", locks) + + +def get_humming_moe_quant_config(layer: FusedMoE): + input_schema = layer.input_schemas["w13"] + weight_schema = layer.weight_schemas["w13"] + + a_dtype = input_schema.a_dtype + if a_dtype is None or a_dtype.num_bits == 16: + a_quant_desc = FusedMoEQuantDesc(dtype=None) + else: + shape = GroupShape(row=1, col=-1) + a_quant_desc = FusedMoEQuantDesc(dtype=str(a_dtype), shape=shape) + + weight_scale_group_size = weight_schema.weight_scale_group_size + weight_scale_group_size_n = weight_schema.weight_scale_group_size_n + weight_group_shape: tuple[int, ...] = () + if weight_scale_group_size_n > 1: + weight_group_shape = GroupShape( + row=weight_scale_group_size, + col=weight_scale_group_size_n, + ) + elif weight_scale_group_size == 0: + weight_group_shape = GroupShape(row=-1, col=1) + else: + weight_group_shape = GroupShape(row=weight_scale_group_size, col=1) + + w1_quant_desc = FusedMoEQuantDesc( + dtype=str(weight_schema.b_dtype), + shape=weight_group_shape, + scale=getattr(layer, "w13_weight_scale", None), + alpha_or_gscale=getattr(layer, "w13_global_scale", None), + zp=getattr(layer, "w13_zero_point", None), + bias=getattr(layer, "w13_bias", None), + ) + + w2_quant_desc = FusedMoEQuantDesc( + dtype=str(weight_schema.b_dtype), + shape=weight_group_shape, + scale=getattr(layer, "w2_weight_scale", None), + alpha_or_gscale=getattr(layer, "w2_global_scale", None), + zp=getattr(layer, "w2_zero_point", None), + bias=getattr(layer, "w2_bias", None), + ) + + return FusedMoEQuantConfig( + _a1=a_quant_desc, + _a2=a_quant_desc, + _w1=w1_quant_desc, + _w2=w2_quant_desc, + )