diff --git a/requirements/cuda.txt b/requirements/cuda.txt index 1776723560..462155f188 100644 --- a/requirements/cuda.txt +++ b/requirements/cuda.txt @@ -32,4 +32,4 @@ quack-kernels>=0.6.1 # Required for CUTLASS DSL 4.6 by MSA tokenspeed-mla==0.1.8; platform_system == "Linux" # Humming kernels for quantization gemm -humming-kernels[cu13]==0.1.10 +humming-kernels[cu13]==0.1.12 diff --git a/tests/evals/gsm8k/configs/humming/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4-humming.yaml b/tests/evals/gsm8k/configs/humming/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4-humming.yaml new file mode 100644 index 0000000000..21ab4bf3ec --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4-humming.yaml @@ -0,0 +1,9 @@ +model_name: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4" +accuracy_threshold: 0.29 +num_questions: 1319 +num_fewshot: 5 +server_args: >- + --enforce-eager + --max-model-len 8192 + --moe-backend humming + --linear-backend humming diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-humming.yaml index af768d214f..502ec79522 100644 --- a/tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-humming.yaml +++ b/tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-humming.yaml @@ -1,5 +1,5 @@ model_name: "nm-testing/Qwen3-4B-mixed-quant-RTN-wNaM" -accuracy_threshold: 0.85 +accuracy_threshold: 0.78 num_questions: 1319 num_fewshot: 5 server_args: >- diff --git a/tests/evals/gsm8k/configs/humming/config-act-int8.txt b/tests/evals/gsm8k/configs/humming/config-act-int8.txt index 38d8fc726d..a7efbac3d0 100644 --- a/tests/evals/gsm8k/configs/humming/config-act-int8.txt +++ b/tests/evals/gsm8k/configs/humming/config-act-int8.txt @@ -1,3 +1,4 @@ +gpt-oss-20b-humming.yaml Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming-act-int8.yaml Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml Qwen3-30B-A3B-AWQ-humming.yaml diff --git a/tests/evals/gsm8k/configs/humming/config.txt b/tests/evals/gsm8k/configs/humming/config.txt index 144ed95993..1b57191ddc 100644 --- a/tests/evals/gsm8k/configs/humming/config.txt +++ b/tests/evals/gsm8k/configs/humming/config.txt @@ -11,3 +11,4 @@ Qwen3-30B-A3B-NVFP4-humming.yaml Qwen3-30B-A3B-AWQ-humming.yaml Qwen3.5-35B-A3B-FP8-humming.yaml Qwen3.5-35B-A3B-experts-int8-humming.yaml +NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4-humming.yaml diff --git a/tests/kernels/moe/test_moe.py b/tests/kernels/moe/test_moe.py index c70ce0b469..058f9124b9 100644 --- a/tests/kernels/moe/test_moe.py +++ b/tests/kernels/moe/test_moe.py @@ -1265,11 +1265,10 @@ def _make_humming_indexed_experts(activation: MoEActivation): layer.intermediate_size_per_partition = intermediate_size quant_config = humming_utils.get_humming_moe_quant_config(layer) experts = HummingIndexedExperts( - layer, - moe_config, - quant_config, + moe_config=moe_config, + quant_config=quant_config, ) - return experts + return experts, layer @pytest.mark.parametrize("activation", list(MoEActivation)) @@ -1328,8 +1327,7 @@ def test_humming_delegates_to_instance_activation(): def test_humming_gated_non_gated_shape_contract(activation: MoEActivation): from vllm.utils import humming - experts = _make_humming_indexed_experts(activation) - layer = experts.layer + experts, layer = _make_humming_indexed_experts(activation) moe_config = experts.moe_config top_k = moe_config.experts_per_token num_experts = moe_config.num_experts @@ -1337,7 +1335,7 @@ def test_humming_gated_non_gated_shape_contract(activation: MoEActivation): intermediate_size = moe_config.intermediate_size gate_up_size = intermediate_size * 2 if activation.is_gated else intermediate_size - w13_meta, w2_meta = (layer.humming_metas[name] for name in ("w13", "w2")) + w13_meta, w2_meta = (experts.humming_configs[name] for name in ("w13", "w2")) for meta in (w13_meta, w2_meta): assert meta.a_dtype == humming.dtypes.bfloat16 assert meta.b_dtype == humming.dtypes.float4e2m1 @@ -1364,7 +1362,7 @@ def test_humming_indexed_writes_supplied_output_buffer(): from vllm.forward_context import set_forward_context activation = MoEActivation.SILU - experts = _make_humming_indexed_experts(activation) + experts, layer = _make_humming_indexed_experts(activation) moe_config = experts.moe_config num_tokens = 1 top_k = moe_config.experts_per_token @@ -1382,7 +1380,7 @@ def test_humming_indexed_writes_supplied_output_buffer(): ) device = torch.device("cuda") - dtype = experts.layer.params_dtype + dtype = layer.params_dtype workspace13 = torch.empty(workspace13_shape, dtype=dtype, device=device) workspace2 = torch.empty(workspace2_shape, dtype=dtype, device=device) hidden_states = torch.ones((num_tokens, hidden_size), dtype=dtype, device=device) @@ -1394,14 +1392,12 @@ def test_humming_indexed_writes_supplied_output_buffer(): device=device, ) topk_ids = torch.arange(top_k, dtype=torch.int32, device=device).unsqueeze(0) - unused = torch.empty((num_experts, 0), device=device) - with set_forward_context(None, vllm_config, num_tokens=num_tokens): experts.apply( output=output, hidden_states=hidden_states, - w1=unused, - w2=unused, + w1=layer.w13_weight, + w2=layer.w2_weight, topk_weights=topk_weights, topk_ids=topk_ids, activation=activation, diff --git a/tests/kernels/moe/test_ocp_mx_moe.py b/tests/kernels/moe/test_ocp_mx_moe.py index 18c70093a1..4d96011251 100644 --- a/tests/kernels/moe/test_ocp_mx_moe.py +++ b/tests/kernels/moe/test_ocp_mx_moe.py @@ -1489,7 +1489,6 @@ def test_rocm_mxfp4_moe_oracle( mxfp4_backend=backend, experts_cls=experts_cls, routing_tables=None, - layer=None, ) # Create inputs diff --git a/tests/quantization/test_moe_wna16.py b/tests/quantization/test_moe_wna16.py index 58faaf4923..8624832e7b 100644 --- a/tests/quantization/test_moe_wna16.py +++ b/tests/quantization/test_moe_wna16.py @@ -161,7 +161,6 @@ def test_moe_wna16_setup_forwards_selected_backend(monkeypatch): assert method.moe_kernel is kernel assert captured["backend"] == WNA16MoEBackend.HUMMING - assert captured["layer"] is layer def test_moe_wna16_humming_adapter_repacks_uint8_tensors(): @@ -198,7 +197,9 @@ def test_moe_wna16_uses_humming_quant_config(monkeypatch): monkeypatch.setattr( humming_utils, "get_humming_moe_quant_config", - lambda actual_layer: quant_config if actual_layer is layer else None, + lambda actual_layer, *args, **kwargs: ( + quant_config if actual_layer is layer else None + ), ) assert method.get_fused_moe_quant_config(layer) is quant_config diff --git a/vllm/model_executor/kernels/linear/mixed_precision/humming.py b/vllm/model_executor/kernels/linear/mixed_precision/humming.py index ec93e935d8..7d57fa680f 100644 --- a/vllm/model_executor/kernels/linear/mixed_precision/humming.py +++ b/vllm/model_executor/kernels/linear/mixed_precision/humming.py @@ -28,7 +28,8 @@ class HummingLinearKernel(MPLinearKernel): def process_weights_after_loading(self, layer: torch.nn.Module) -> None: from vllm.model_executor.layers.quantization.utils.humming_utils import ( convert_linear_layer_to_humming_standard, - prepare_humming_layer, + get_humming_linear_compute_config, + prepare_humming_linear_layer_config, ) name_map = {"weight": self.w_q_name, "weight_scale": self.w_s_name} @@ -46,9 +47,11 @@ class HummingLinearKernel(MPLinearKernel): convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map) input_quant_config = getattr(layer, "_humming_input_quant_config", None) - prepare_humming_layer( + self.layer_config = prepare_humming_linear_layer_config( layer, quant_config, input_quant_config=input_quant_config ) + self.compute_config = get_humming_linear_compute_config() + self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device) def apply_weights( self, @@ -56,12 +59,14 @@ class HummingLinearKernel(MPLinearKernel): x: torch.Tensor, bias: torch.Tensor | None = None, ) -> torch.Tensor: - from vllm.utils.humming import HummingMethod + from vllm.model_executor.layers.quantization.utils.humming_utils import ( + apply_humming_linear, + ) - flatten_inputs = x.view(-1, x.size(-1)) - output = HummingMethod.forward_layer( - layer=layer, - inputs=flatten_inputs, - compute_config=layer.compute_config, + return apply_humming_linear( + layer, + x, + layer_config=self.layer_config, + compute_config=self.compute_config, + locks=self.locks, ) - return output.view(*x.shape[:-1], output.size(-1)) diff --git a/vllm/model_executor/kernels/linear/mxfp4/humming.py b/vllm/model_executor/kernels/linear/mxfp4/humming.py index f38b8ead63..660e31dcf4 100644 --- a/vllm/model_executor/kernels/linear/mxfp4/humming.py +++ b/vllm/model_executor/kernels/linear/mxfp4/humming.py @@ -5,11 +5,14 @@ import torch from vllm.logger import init_logger from vllm.model_executor.layers.quantization.utils.humming_utils import ( + apply_humming_linear, convert_linear_layer_to_humming_standard, - prepare_humming_layer, + get_humming_linear_compute_config, + prepare_humming_linear_layer_config, ) from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Dynamic from vllm.platforms import current_platform +from vllm.utils.import_utils import has_humming from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig @@ -26,6 +29,9 @@ class HummingMxFp4LinearKernel(MxFp4LinearKernel): if not current_platform.is_cuda(): return False, "Humming only supported on CUDA" + if not has_humming(): + return False, "Humming is not installed" + if not current_platform.has_device_capability(75): return False, "Humming only supported on SM75+" @@ -56,7 +62,9 @@ class HummingMxFp4LinearKernel(MxFp4LinearKernel): } convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map) - prepare_humming_layer(layer, quant_config) + self.layer_config = prepare_humming_linear_layer_config(layer, quant_config) + self.compute_config = get_humming_linear_compute_config() + self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device) def apply_weights( self, @@ -64,12 +72,10 @@ class HummingMxFp4LinearKernel(MxFp4LinearKernel): x: torch.Tensor, bias: torch.Tensor | None = None, ) -> torch.Tensor: - from vllm.utils.humming import HummingMethod - - flatten_inputs = x.view(-1, x.size(-1)) - output = HummingMethod.forward_layer( - layer=layer, - inputs=flatten_inputs, - compute_config=layer.compute_config, + return apply_humming_linear( + layer, + x, + layer_config=self.layer_config, + compute_config=self.compute_config, + locks=self.locks, ) - return output.view(*x.shape[:-1], output.size(-1)) diff --git a/vllm/model_executor/kernels/linear/mxfp8/humming.py b/vllm/model_executor/kernels/linear/mxfp8/humming.py index ed1cd39cbd..bde1565d2a 100644 --- a/vllm/model_executor/kernels/linear/mxfp8/humming.py +++ b/vllm/model_executor/kernels/linear/mxfp8/humming.py @@ -4,10 +4,13 @@ import torch from vllm.model_executor.layers.quantization.utils.humming_utils import ( + apply_humming_linear, convert_linear_layer_to_humming_standard, - prepare_humming_layer, + get_humming_linear_compute_config, + prepare_humming_linear_layer_config, ) from vllm.platforms import current_platform +from vllm.utils.import_utils import has_humming from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig @@ -22,6 +25,9 @@ class HummingMxfp8LinearKernel(Mxfp8LinearKernel): if not current_platform.is_cuda(): return False, "Humming only supported on CUDA" + if not has_humming(): + return False, "Humming is not installed" + if not current_platform.has_device_capability(75): return False, "Humming only supported on SM75+" @@ -44,7 +50,9 @@ class HummingMxfp8LinearKernel(Mxfp8LinearKernel): } convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map) - prepare_humming_layer(layer, quant_config) + self.layer_config = prepare_humming_linear_layer_config(layer, quant_config) + self.compute_config = get_humming_linear_compute_config() + self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device) def apply_weights( self, @@ -52,12 +60,10 @@ class HummingMxfp8LinearKernel(Mxfp8LinearKernel): x: torch.Tensor, bias: torch.Tensor | None = None, ) -> torch.Tensor: - from vllm.utils.humming import HummingMethod - - flatten_inputs = x.view(-1, x.size(-1)) - output = HummingMethod.forward_layer( - layer=layer, - inputs=flatten_inputs, - compute_config=layer.compute_config, + return apply_humming_linear( + layer, + x, + layer_config=self.layer_config, + compute_config=self.compute_config, + locks=self.locks, ) - return output.view(*x.shape[:-1], output.size(-1)) diff --git a/vllm/model_executor/kernels/linear/nvfp4/humming.py b/vllm/model_executor/kernels/linear/nvfp4/humming.py index 7e39034385..54a0cf6c1d 100644 --- a/vllm/model_executor/kernels/linear/nvfp4/humming.py +++ b/vllm/model_executor/kernels/linear/nvfp4/humming.py @@ -5,9 +5,12 @@ import torch from vllm.logger import init_logger from vllm.model_executor.layers.quantization.utils.humming_utils import ( - prepare_humming_layer, + apply_humming_linear, + get_humming_linear_compute_config, + prepare_humming_linear_layer_config, ) from vllm.platforms import current_platform +from vllm.utils.import_utils import has_humming from .base import NvFp4LinearKernel, NvFp4LinearLayerConfig @@ -24,6 +27,9 @@ class HummingNvFp4LinearKernel(NvFp4LinearKernel): if not current_platform.is_cuda(): return False, "Humming only supported on CUDA" + if not has_humming(): + return False, "Humming is not installed" + if not current_platform.has_device_capability(75): return False, "Humming only supported on SM75+" @@ -54,7 +60,9 @@ class HummingNvFp4LinearKernel(NvFp4LinearKernel): layer.weight_global_scale = torch.nn.Parameter( 1.0 / layer.weight_global_scale, requires_grad=False ) - prepare_humming_layer(layer, quant_config) + self.layer_config = prepare_humming_linear_layer_config(layer, quant_config) + self.compute_config = get_humming_linear_compute_config() + self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device) def apply_weights( self, @@ -62,12 +70,10 @@ class HummingNvFp4LinearKernel(NvFp4LinearKernel): x: torch.Tensor, bias: torch.Tensor | None = None, ) -> torch.Tensor: - from vllm.utils.humming import HummingMethod - - flatten_inputs = x.view(-1, x.size(-1)) - output = HummingMethod.forward_layer( - layer=layer, - inputs=flatten_inputs, - compute_config=layer.compute_config, + return apply_humming_linear( + layer, + x, + layer_config=self.layer_config, + compute_config=self.compute_config, + locks=self.locks, ) - return output.view(*x.shape[:-1], output.size(-1)) diff --git a/vllm/model_executor/kernels/linear/scaled_mm/humming.py b/vllm/model_executor/kernels/linear/scaled_mm/humming.py index 7b8ed21fbd..b52e73b5a4 100644 --- a/vllm/model_executor/kernels/linear/scaled_mm/humming.py +++ b/vllm/model_executor/kernels/linear/scaled_mm/humming.py @@ -5,10 +5,13 @@ import torch from vllm.logger import init_logger from vllm.model_executor.layers.quantization.utils.humming_utils import ( + apply_humming_linear, convert_linear_layer_to_humming_standard, - prepare_humming_layer, + get_humming_linear_compute_config, + prepare_humming_linear_layer_config, ) from vllm.platforms import current_platform +from vllm.utils.import_utils import has_humming from .ScaledMMLinearKernel import ( FP8ScaledMMLinearKernel, @@ -30,6 +33,9 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel): if not current_platform.is_cuda(): return False, "Humming only supported on CUDA" + if not has_humming(): + return False, "Humming is not installed" + if not current_platform.has_device_capability(75): return False, "Humming only supported on SM75+" @@ -58,9 +64,6 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel): scale_group_shape = self.config.weight_quant_key.scale.group_shape if scale_group_shape.is_per_tensor(): quant_config["weight_scale_type"] = "tensor" - if not hasattr(layer, "global_scale") and hasattr(layer, "weight_scale"): - del name_map["weight_scale"] - name_map["global_scale"] = "weight_scale" elif scale_group_shape.is_per_channel(): quant_config["weight_scale_type"] = "channel" elif scale_group_shape.is_per_group(): @@ -76,7 +79,9 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel): name_map["weight_scale"] = "weight_scale_inv" convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map) - prepare_humming_layer(layer, quant_config) + self.layer_config = prepare_humming_linear_layer_config(layer, quant_config) + self.compute_config = get_humming_linear_compute_config() + self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device) def apply_weights( self, @@ -84,15 +89,13 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel): x: torch.Tensor, bias: torch.Tensor | None = None, ) -> torch.Tensor: - from vllm.utils.humming import HummingMethod - - flatten_inputs = x.view(-1, x.size(-1)) - output = HummingMethod.forward_layer( - layer=layer, - inputs=flatten_inputs, - compute_config=layer.compute_config, + return apply_humming_linear( + layer, + x, + layer_config=self.layer_config, + compute_config=self.compute_config, + locks=self.locks, ) - return output.view(*x.shape[:-1], output.size(-1)) def apply_scaled_mm( self, @@ -105,7 +108,9 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel): bias: torch.Tensor | None, output_shape: list, ) -> torch.Tensor: - pass + raise NotImplementedError( + "HummingFP8ScaledMMLinearKernel uses apply_weights directly" + ) class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel): @@ -118,6 +123,9 @@ class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel): if not current_platform.is_cuda(): return False, "Humming only supported on CUDA" + if not has_humming(): + return False, "Humming is not installed" + if not current_platform.has_device_capability(75): return False, "Humming only supported on SM75+" @@ -137,7 +145,9 @@ class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel): weight.data = weight.data + 128 convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map) - prepare_humming_layer(layer, quant_config) + self.layer_config = prepare_humming_linear_layer_config(layer, quant_config) + self.compute_config = get_humming_linear_compute_config() + self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device) def apply_weights( self, @@ -145,12 +155,10 @@ class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel): x: torch.Tensor, bias: torch.Tensor | None = None, ) -> torch.Tensor: - from vllm.utils.humming import HummingMethod - - flatten_inputs = x.view(-1, x.size(-1)) - output = HummingMethod.forward_layer( - layer=layer, - inputs=flatten_inputs, - compute_config=layer.compute_config, + return apply_humming_linear( + layer, + x, + layer_config=self.layer_config, + compute_config=self.compute_config, + locks=self.locks, ) - return output.view(*x.shape[:-1], output.size(-1)) diff --git a/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py b/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py index e0ae629d1b..96c42967e9 100644 --- a/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py @@ -4,7 +4,7 @@ import json import math -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast import torch @@ -62,8 +62,15 @@ from vllm.utils.import_utils import has_humming from vllm.v1.worker.workspace import current_workspace_manager if TYPE_CHECKING: - from vllm.model_executor.layers.fused_moe import RoutedExperts - from vllm.utils.humming import GemmType as HummingGemmType + from vllm.model_executor.layers.quantization.utils.humming_utils import ( + HummingMoEQuantConfig, + ) + from vllm.utils.humming import ( + GemmType as HummingGemmType, + ) + from vllm.utils.humming import ( + LayerConfig as HummingLayerConfig, + ) logger = init_logger(__name__) @@ -88,15 +95,20 @@ def get_humming_moe_gemm_type() -> str: class HummingExpertsBase(mk.FusedMoEExpertsModular): def __init__( self, - layer: "RoutedExperts", moe_config: FusedMoEConfig, quant_config: FusedMoEQuantConfig, max_num_tokens: int | None = None, num_dispatchers: int | None = None, ): - self.layer = layer - self.num_experts = self.layer.local_num_experts - self.global_num_experts = self.layer.global_num_experts + humming_quant_config = cast("HummingMoEQuantConfig", quant_config) + self.humming_configs: dict[str, HummingLayerConfig] = { + "w13": humming_quant_config.w1_humming_config, + "w2": humming_quant_config.w2_humming_config, + } + self.locks = torch.zeros(1024, dtype=torch.int32, device=moe_config.device) + self.num_experts = moe_config.num_local_experts + self.global_num_experts = moe_config.num_experts + self.quant_config = quant_config self.init_humming_moe() if self.is_batched(): @@ -111,31 +123,73 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): self._permute_scratch: MoEPermuteScratch | None = None def init_humming_moe(self): - from vllm.utils.humming import HummingMethod + from vllm.utils.humming import get_heuristics_config self.compute_config = { "use_batch_invariant": envs.VLLM_BATCH_INVARIANT, "use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM, "gemm_type": self.humming_gemm_type().value, } - self.w13_tuning_config = HummingMethod.get_default_tuning_configs( - layer=self.layer, + self.w13_tuning_config = get_heuristics_config( + layer_config=self.humming_configs["w13"], use_f16_accum=envs.VLLM_HUMMING_USE_F16_ACCUM, use_batch_invariant=envs.VLLM_BATCH_INVARIANT, gemm_type=self.humming_gemm_type(), - sublayer_name="w13", ) - self.w2_tuning_config = HummingMethod.get_default_tuning_configs( - layer=self.layer, + self.w2_tuning_config = get_heuristics_config( + layer_config=self.humming_configs["w2"], use_f16_accum=envs.VLLM_HUMMING_USE_F16_ACCUM, use_batch_invariant=envs.VLLM_BATCH_INVARIANT, gemm_type=self.humming_gemm_type(), - sublayer_name="w2", ) self.compute_config_str = json.dumps(self.compute_config) self.w13_tuning_config_str = json.dumps(self.w13_tuning_config) self.w2_tuning_config_str = json.dumps(self.w2_tuning_config) + def quantize_input( + self, + sublayer_name: str, + inputs: torch.Tensor, + quanted_input: torch.Tensor | None, + ) -> tuple[torch.Tensor, torch.Tensor | None]: + from vllm.utils.humming import may_quant_input + + return may_quant_input( + self.humming_configs[sublayer_name], + inputs=inputs, + quanted_input=quanted_input, + ) + + def humming_forward( + self, + sublayer_name: str, + inputs: torch.Tensor, + weight: torch.Tensor, + input_scale: torch.Tensor | None, + outputs: torch.Tensor, + **kwargs: Any, + ) -> torch.Tensor: + from vllm.utils.humming import humming_forward + + is_w13 = sublayer_name == "w13" + return humming_forward( + self.humming_configs[sublayer_name], + inputs=inputs, + weight=weight, + weight_scale=( + self.quant_config.w1_scale if is_w13 else self.quant_config.w2_scale + ), + zero_point=(self.quant_config.w1_zp if is_w13 else self.quant_config.w2_zp), + bias=(self.quant_config.w1_bias if is_w13 else self.quant_config.w2_bias), + weight_scale_2=( + self.quant_config.g1_alphas if is_w13 else self.quant_config.g2_alphas + ), + input_scale=input_scale, + outputs=outputs, + locks=self.locks, + **kwargs, + ) + def _get_permute_scratch(self) -> MoEPermuteScratch | None: if self._permute_scratch is None and moe_permute_unpermute_supported(): self._permute_scratch = MoEPermuteScratch( @@ -218,8 +272,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): @property def expects_unquantized_inputs(self) -> bool: """ - Humming kernels handle input quantization internally via - HummingMethod.may_quant_input() in the apply() method. + Humming kernels handle input quantization internally in apply(). This property tells the prepare/finalize step to skip input quantization (by setting defer_input_quant=True) and pass @@ -255,6 +308,10 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool: return True + @staticmethod + def _supports_batch_invariance() -> bool: + return True + def moe_problem_size( self, a1: torch.Tensor, @@ -262,10 +319,8 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): w2: torch.Tensor, topk_ids: torch.Tensor, ) -> tuple[int, int, int, int, int]: - from vllm.utils.humming import HummingLayerMeta - - meta1: HummingLayerMeta = self.layer.humming_metas["w13"] - meta2: HummingLayerMeta = self.layer.humming_metas["w2"] + meta1 = self.humming_configs["w13"] + meta2 = self.humming_configs["w2"] assert meta1.num_experts == meta2.num_experts @@ -282,25 +337,25 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): assert a1.size(0) == num_experts num_tokens = a1.size(1) - return ( - meta1.num_experts, - num_tokens, - # Logical intermediate width for both gated and non-gated activations - self.layer.intermediate_size_per_partition, - meta1.shape_k, - top_k, - ) + intermediate_dim = meta2.shape_k - meta2.pad_shape_k + hidden_dim = meta1.shape_k - meta1.pad_shape_k + return meta1.num_experts, num_tokens, intermediate_dim, hidden_dim, top_k def get_buffer_metas(self, M: int, topk: int, activation: MoEActivation): from vllm.utils.humming import GemmType as HummingGemmType from vllm.utils.humming import dtypes num_experts = self.num_experts - N = self.layer.intermediate_size_per_partition - K = self.layer.hidden_size + w13_config = self.humming_configs["w13"] + w2_config = self.humming_configs["w2"] + gate_up_dim = w13_config.shape_n - w13_config.pad_shape_n + intermediate_dim = w2_config.shape_k - w2_config.pad_shape_k + K = w13_config.shape_k - w13_config.pad_shape_k assert isinstance(num_experts, int) - assert isinstance(N, int) + assert isinstance(gate_up_dim, int) + assert isinstance(intermediate_dim, int) assert isinstance(K, int) + assert intermediate_dim == self.adjust_N_for_activation(gate_up_dim, activation) # hidden_states # (-> quanted_gate_up_input) (if not BF16/FP16 activation) @@ -310,7 +365,8 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): # -> down_output # (-> output) (if not is_batched) # Neighboring nodes are required to utilize distinct workspaces. - # The output must be derived from workspace1. + # The final output buffer is supplied by the modular kernel and may + # alias workspace1. output_shape: tuple[int, ...] if self.is_batched(): @@ -327,10 +383,8 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): real_shape_m = M * topk output_shape = (M, K) - gate_up_size = N * (2 if activation.is_gated else 1) - down_input_size = N - a_dtype = self.layer.humming_metas["w13"].a_dtype - c_dtype = self.layer.humming_metas["w13"].c_dtype + a_dtype = self.humming_configs["w13"].a_dtype + c_dtype = self.humming_configs["w13"].c_dtype num_bits = a_dtype.num_bits torch_dtype_map = { dtypes.float16: torch.float16, @@ -348,15 +402,15 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): "dtype": torch_dtype_map[a_dtype], }, "gate_up_output": { - "shape": (real_shape_m, gate_up_size), + "shape": (real_shape_m, gate_up_dim), "dtype": torch_dtype_map[c_dtype], }, "activation_output": { - "shape": (real_shape_m, down_input_size), + "shape": (real_shape_m, intermediate_dim), "dtype": torch_dtype_map[c_dtype], }, "quanted_down_input": { - "shape": (real_shape_m, down_input_size), + "shape": (real_shape_m, intermediate_dim), "dtype": torch_dtype_map[a_dtype], }, "down_output": { @@ -414,7 +468,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): output_key = "down_output" if self.is_batched() else "output" output_shape = buffer_metas[output_key]["shape"] - elem_size = self.layer.params_dtype.itemsize + elem_size = self.moe_config.in_dtype.itemsize return ( (workspace1_nbytes // elem_size,), @@ -438,7 +492,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): def make_workspaces(self, M: int, topk: int, activation: MoEActivation): shapes = self._workspace_shapes(M, topk, activation) workspace1_shape, workspace2_shape, output_shape = shapes - torch_dtype = self.layer.params_dtype + torch_dtype = self.moe_config.in_dtype workspace1, workspace2 = current_workspace_manager().get_simultaneous( (workspace1_shape, torch_dtype), (workspace2_shape, torch_dtype), @@ -600,12 +654,11 @@ class HummingIndexedExperts(HummingExpertsBase): """ Standard apply implementation for Humming indexed experts. - Note: Humming kernels handle weights and quantization internally through - the layer object, so w1, w2, a1q_scale, a2_scale parameters are not used. - The output is written into workspace13 via the buffer management. + Humming performs activation quantization internally and consumes the + weights supplied by the modular kernel interface. + Results are written to the output buffer supplied by the modular + kernel, which may alias workspace13. """ - from vllm.utils.humming import HummingMethod - assert not apply_router_weight_on_input hidden_states = hidden_states.view(-1, hidden_states.size(-1)) @@ -616,6 +669,7 @@ class HummingIndexedExperts(HummingExpertsBase): topk_ids.size(1), activation, ) + buffers["output"] = output moe_kwargs1, moe_kwargs2 = self.prepare_humming_moe_kwargs( topk_ids=topk_ids, @@ -623,19 +677,18 @@ class HummingIndexedExperts(HummingExpertsBase): expert_tokens_meta=expert_tokens_meta, ) - inputs, input_scale = HummingMethod.may_quant_input( - layer=self.layer, + inputs, input_scale = self.quantize_input( + "w13", inputs=hidden_states, quanted_input=buffers.get("quanted_gate_up_input", None), - sublayer_name="w13", ) - HummingMethod.forward_layer( - layer=self.layer, + self.humming_forward( + "w13", inputs=inputs, + weight=w1, input_scale=input_scale, outputs=buffers["gate_up_output"], - sublayer_name="w13", **moe_kwargs1, ) @@ -645,19 +698,18 @@ class HummingIndexedExperts(HummingExpertsBase): output=buffers["activation_output"], ) - inputs, input_scale = HummingMethod.may_quant_input( - layer=self.layer, + inputs, input_scale = self.quantize_input( + "w2", inputs=buffers["activation_output"], quanted_input=buffers.get("quanted_down_input", None), - sublayer_name="w2", ) - HummingMethod.forward_layer( - layer=self.layer, + self.humming_forward( + "w2", inputs=inputs, + weight=w2, input_scale=input_scale, outputs=buffers["down_output"].view(-1, hidden_states.size(-1)), - sublayer_name="w2", **moe_kwargs2, ) @@ -705,12 +757,11 @@ class HummingGroupedExperts(HummingExpertsBase): """ Standard apply implementation for Humming grouped experts. - Note: Humming kernels handle weights and quantization internally through - the layer object, so w1, w2, a1q_scale, a2_scale parameters are not used. - The output is written into workspace13 via the buffer management. + Humming performs activation quantization internally and consumes the + weights supplied by the modular kernel interface. + Results are written to the output buffer supplied by the modular + kernel, which may alias workspace13. """ - from vllm.utils.humming import HummingMethod - assert not apply_router_weight_on_input valid_shape_m = self.estimate_local_valid_shape_m(topk_ids) @@ -722,6 +773,7 @@ class HummingGroupedExperts(HummingExpertsBase): topk_ids.size(1), activation, ) + buffers["output"] = output hidden_states, _, expert_first_token_offset, inv_perm, _ = moe_permute( hidden_states=hidden_states, @@ -733,23 +785,22 @@ class HummingGroupedExperts(HummingExpertsBase): scratch=self._get_permute_scratch(), ) - inputs, input_scale = HummingMethod.may_quant_input( - layer=self.layer, + inputs, input_scale = self.quantize_input( + "w13", inputs=hidden_states, quanted_input=buffers.get("quanted_gate_up_input", None), - sublayer_name="w13", ) - HummingMethod.forward_layer( - layer=self.layer, + self.humming_forward( + "w13", inputs=inputs, + weight=w1, input_scale=input_scale, outputs=buffers["gate_up_output"], valid_shape_m=valid_shape_m, expert_layout=expert_first_token_offset, compute_config=self.compute_config_str, tuning_config=self.w13_tuning_config_str, - sublayer_name="w13", ) self.apply_activation( @@ -758,23 +809,22 @@ class HummingGroupedExperts(HummingExpertsBase): output=buffers["activation_output"], ) - inputs, input_scale = HummingMethod.may_quant_input( - layer=self.layer, + inputs, input_scale = self.quantize_input( + "w2", inputs=buffers["activation_output"], quanted_input=buffers.get("quanted_down_input", None), - sublayer_name="w2", ) - HummingMethod.forward_layer( - layer=self.layer, + self.humming_forward( + "w2", inputs=inputs, + weight=w2, input_scale=input_scale, outputs=buffers["down_output"], valid_shape_m=valid_shape_m, expert_layout=expert_first_token_offset, compute_config=self.compute_config_str, tuning_config=self.w2_tuning_config_str, - sublayer_name="w2", ) moe_unpermute( @@ -821,12 +871,11 @@ class BatchedHummingGroupedExperts(HummingExpertsBase): """ Standard apply implementation for Humming batched grouped experts. - Note: Humming kernels handle weights and quantization internally through - the layer object, so w1, w2, a1q_scale, a2_scale parameters are not used. - The output is written into workspace13 via the buffer management. + Humming performs activation quantization internally and consumes the + weights supplied by the modular kernel interface. + Results are written to the output buffer supplied by the modular + kernel, which may alias workspace13. """ - from vllm.utils.humming import HummingMethod - assert not apply_router_weight_on_input assert expert_tokens_meta is not None @@ -841,24 +890,24 @@ class BatchedHummingGroupedExperts(HummingExpertsBase): topk_ids.size(1), activation, ) + buffers["down_output"] = output - inputs, input_scale = HummingMethod.may_quant_input( - layer=self.layer, + inputs, input_scale = self.quantize_input( + "w13", inputs=hidden_states, quanted_input=buffers.get("quanted_gate_up_input", None), - sublayer_name="w13", ) - HummingMethod.forward_layer( - layer=self.layer, + self.humming_forward( + "w13", inputs=inputs, + weight=w1, input_scale=input_scale, outputs=buffers["gate_up_output"], valid_shape_m=valid_shape_m, expert_layout=expert_num_tokens, compute_config=self.compute_config_str, tuning_config=self.w13_tuning_config_str, - sublayer_name="w13", ) self.apply_activation( @@ -867,21 +916,20 @@ class BatchedHummingGroupedExperts(HummingExpertsBase): output=buffers["activation_output"], ) - inputs, input_scale = HummingMethod.may_quant_input( - layer=self.layer, + inputs, input_scale = self.quantize_input( + "w2", inputs=buffers["activation_output"], quanted_input=buffers.get("quanted_down_input", None), - sublayer_name="w2", ) - HummingMethod.forward_layer( - layer=self.layer, + self.humming_forward( + "w2", inputs=inputs, + weight=w2, input_scale=input_scale, outputs=output.view(-1, hidden_states.size(-1)), valid_shape_m=valid_shape_m, expert_layout=expert_num_tokens, compute_config=self.compute_config_str, tuning_config=self.w2_tuning_config_str, - sublayer_name="w2", ) diff --git a/vllm/model_executor/layers/fused_moe/oracle/fp8.py b/vllm/model_executor/layers/fused_moe/oracle/fp8.py index 7501943668..6153ed28c1 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/fp8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/fp8.py @@ -619,7 +619,12 @@ def make_fp8_moe_quant_config( ) assert isinstance(layer, RoutedExperts) - return get_humming_moe_quant_config(layer) + return get_humming_moe_quant_config( + layer, + gemm1_alpha=gemm1_alpha, + gemm1_beta=gemm1_beta, + gemm1_clamp_limit=swiglu_limit, + ) # Flashinfer CUTLASS or HPC per-tensor uses single dq scale # (alpha = w_scale * a_scale) and inverse a2 scale. @@ -694,7 +699,6 @@ def make_fp8_moe_kernel( experts_cls: type[mk.FusedMoEExperts], fp8_backend: Fp8MoeBackend, routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None, - layer: torch.nn.Module | None = None, ) -> mk.FusedMoEKernel: # Create Prepare/Finalize. prepare_finalize = maybe_make_prepare_finalize( @@ -708,11 +712,6 @@ def make_fp8_moe_kernel( logger.info_once("Using %s", prepare_finalize.__class__.__name__) - extra_kwargs = {} - if fp8_backend == Fp8MoeBackend.HUMMING: - assert layer is not None - extra_kwargs = {"layer": layer} - # Create Experts. if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts: max_num_tokens = prepare_finalize.max_num_tokens_per_rank() @@ -722,13 +721,11 @@ def make_fp8_moe_kernel( quant_config=moe_quant_config, max_num_tokens=max_num_tokens, num_dispatchers=prepare_finalize.num_dispatchers(), - **extra_kwargs, ) else: experts = experts_cls( moe_config=moe_config, quant_config=moe_quant_config, - **extra_kwargs, ) kernel = mk.FusedMoEKernel( diff --git a/vllm/model_executor/layers/fused_moe/oracle/int8.py b/vllm/model_executor/layers/fused_moe/oracle/int8.py index be99d0b09a..2d6e5b1369 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/int8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/int8.py @@ -195,7 +195,12 @@ def make_int8_moe_quant_config( ) assert isinstance(layer, RoutedExperts) - return get_humming_moe_quant_config(layer) + return get_humming_moe_quant_config( + layer, + gemm1_alpha=getattr(layer, "swiglu_alpha", None), + gemm1_beta=getattr(layer, "swiglu_beta", None), + gemm1_clamp_limit=getattr(layer, "swiglu_limit", None), + ) if scales_absent and not per_act_token_quant: return int8_w8a16_moe_quant_config( @@ -251,9 +256,9 @@ def convert_to_int8_moe_kernel_format( ) assert layer is not None - # Humming reads canonical CT scales (w*_weight_scale) from the layer. - # Online int8 produces per-channel (E, N) w*_scale; expose them as the - # (E, N, 1) w*_weight_scale humming's loader expects. + # The Humming conversion expects canonical CT scale parameter names. + # Online int8 produces per-channel (E, N) w*_scale; expose them as + # (E, N, 1) w*_weight_scale tensors before conversion. for sub in ("w13", "w2"): if hasattr(layer, f"{sub}_weight_scale"): continue @@ -279,7 +284,6 @@ def make_int8_moe_kernel( moe_config: FusedMoEConfig, experts_cls: type[mk.FusedMoEExperts], routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None, - layer: torch.nn.Module | None = None, ) -> mk.FusedMoEKernel: # Create Prepare/Finalize. prepare_finalize = maybe_make_prepare_finalize( @@ -293,11 +297,6 @@ def make_int8_moe_kernel( logger.info_once("Using %s", prepare_finalize.__class__.__name__) - extra_kwargs = {} - if int8_backend == Int8MoeBackend.HUMMING: - assert layer is not None - extra_kwargs = {"layer": layer} - # Create Experts. if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts: max_num_tokens = prepare_finalize.max_num_tokens_per_rank() @@ -307,13 +306,11 @@ def make_int8_moe_kernel( quant_config=moe_quant_config, max_num_tokens=max_num_tokens, num_dispatchers=prepare_finalize.num_dispatchers(), - **extra_kwargs, ) else: experts = experts_cls( moe_config=moe_config, quant_config=moe_quant_config, - **extra_kwargs, ) kernel = mk.FusedMoEKernel( diff --git a/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py b/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py index 2ac53525c9..5ab0277afc 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py +++ b/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py @@ -358,7 +358,6 @@ def make_wna16_moe_kernel( moe_config: FusedMoEConfig, experts_cls: type[mk.FusedMoEExperts], backend: WNA16MoEBackend = WNA16MoEBackend.MARLIN, - layer: torch.nn.Module | None = None, is_k_full: bool = False, w13_g_idx: torch.Tensor | None = None, w2_g_idx: torch.Tensor | None = None, @@ -410,10 +409,7 @@ def make_wna16_moe_kernel( logger.info_once("Using %s", experts_cls.__name__, scope="local") extra_args: dict[str, Any] = {} - if backend == WNA16MoEBackend.HUMMING: - assert layer is not None - extra_args = {"layer": layer} - elif issubclass(experts_cls, MarlinExpertsBase): + if issubclass(experts_cls, MarlinExpertsBase): extra_args = { "w13_g_idx": w13_g_idx, "w2_g_idx": w2_g_idx, diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index 7612e7f795..cdb9c335d0 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -1788,7 +1788,6 @@ def make_mxfp4_moe_kernel( experts_cls: type[mk.FusedMoEExperts], mxfp4_backend: Mxfp4MoeBackend, routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None, - layer: RoutedExperts | None = None, ) -> mk.FusedMoEKernel: """Create a FusedMoEKernel for the given MXFP4 backend.""" is_monolithic = issubclass(experts_cls, mk.FusedMoEExpertsMonolithic) @@ -1805,11 +1804,6 @@ def make_mxfp4_moe_kernel( logger.info_once("Using %s", prepare_finalize.__class__.__name__) logger.info_once("Using %s", experts_cls.__name__) - extra_kwargs = {} - if mxfp4_backend == Mxfp4MoeBackend.HUMMING: - assert layer is not None - extra_kwargs["layer"] = layer - # Create Experts. if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts: max_num_tokens = prepare_finalize.max_num_tokens_per_rank() @@ -1819,13 +1813,11 @@ def make_mxfp4_moe_kernel( quant_config=moe_quant_config, max_num_tokens=max_num_tokens, num_dispatchers=prepare_finalize.num_dispatchers(), - **extra_kwargs, ) else: experts = experts_cls( moe_config=moe_config, quant_config=moe_quant_config, - **extra_kwargs, ) kernel = mk.FusedMoEKernel( diff --git a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py index de3fb6fa58..f48e79e13d 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py @@ -398,10 +398,10 @@ def convert_to_nvfp4_moe_kernel_format( a2_scale = None w13 = layer.w13_weight w13_scale = layer.w13_weight_scale - w13_scale_2 = getattr(layer, "w13_global_scale", None) + w13_scale_2 = getattr(layer, "w13_weight_scale_2", None) w2 = layer.w2_weight w2_scale = layer.w2_weight_scale - w2_scale_2 = getattr(layer, "w2_global_scale", None) + w2_scale_2 = getattr(layer, "w2_weight_scale_2", None) elif nvfp4_backend == NvFp4MoeBackend.MARLIN: a13_scale = None a2_scale = None @@ -485,7 +485,12 @@ def make_nvfp4_moe_quant_config( ) assert isinstance(layer, RoutedExperts) - return get_humming_moe_quant_config(layer) + return get_humming_moe_quant_config( + layer, + gemm1_alpha=getattr(layer, "swiglu_alpha", None), + gemm1_beta=getattr(layer, "swiglu_beta", None), + gemm1_clamp_limit=swiglu_limit, + ) elif backend == NvFp4MoeBackend.MARLIN: return nvfp4_w4a16_moe_quant_config( g1_alphas=w13_scale_2, @@ -544,7 +549,6 @@ def make_nvfp4_moe_kernel( experts_cls: type[mk.FusedMoEExperts], backend: NvFp4MoeBackend, routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None, - layer: torch.nn.Module | None = None, per_token_activation: bool = False, ) -> mk.FusedMoEKernel: # Create Prepare/Finalize. @@ -560,9 +564,6 @@ def make_nvfp4_moe_kernel( logger.info_once("Using %s", prepare_finalize.__class__.__name__) extra_kwargs = {} - if backend == NvFp4MoeBackend.HUMMING: - assert layer is not None - extra_kwargs = {"layer": layer} if backend == NvFp4MoeBackend.FLASHINFER_TRTLLM and per_token_activation: extra_kwargs["per_token_activation"] = True diff --git a/vllm/model_executor/layers/quantization/auto_awq.py b/vllm/model_executor/layers/quantization/auto_awq.py index 789d3a1d1b..83ef60b41f 100644 --- a/vllm/model_executor/layers/quantization/auto_awq.py +++ b/vllm/model_executor/layers/quantization/auto_awq.py @@ -751,7 +751,6 @@ class AutoAWQMoEMethod(FusedMoEMethodBase): moe_config=self.moe, experts_cls=self.experts_cls, backend=self.wna16_moe_backend, - layer=layer, is_k_full=self.is_k_full, w13_g_idx=getattr(layer, "w13_g_idx", None), w2_g_idx=getattr(layer, "w2_g_idx", None), @@ -766,7 +765,12 @@ class AutoAWQMoEMethod(FusedMoEMethodBase): get_humming_moe_quant_config, ) - return get_humming_moe_quant_config(layer) + return get_humming_moe_quant_config( + layer, + gemm1_alpha=getattr(layer, "swiglu_alpha", None), + gemm1_beta=getattr(layer, "swiglu_beta", None), + gemm1_clamp_limit=getattr(layer, "swiglu_limit", None), + ) return make_wna16_moe_quant_config( w1_scale=layer.w13_scales, w2_scale=layer.w2_scales, diff --git a/vllm/model_executor/layers/quantization/auto_gptq.py b/vllm/model_executor/layers/quantization/auto_gptq.py index 14b9aa94c8..1e6570abee 100644 --- a/vllm/model_executor/layers/quantization/auto_gptq.py +++ b/vllm/model_executor/layers/quantization/auto_gptq.py @@ -772,7 +772,6 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase): moe_config=self.moe, experts_cls=self.experts_cls, backend=self.wna16_moe_backend, - layer=layer, is_k_full=self.is_k_full, w13_g_idx=getattr(layer, "w13_g_idx", None), w2_g_idx=getattr(layer, "w2_g_idx", None), @@ -787,7 +786,12 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase): get_humming_moe_quant_config, ) - return get_humming_moe_quant_config(layer) + return get_humming_moe_quant_config( + layer, + gemm1_alpha=getattr(layer, "swiglu_alpha", None), + gemm1_beta=getattr(layer, "swiglu_beta", None), + gemm1_clamp_limit=getattr(layer, "swiglu_limit", None), + ) from vllm.model_executor.layers.fused_moe.config import ( gptq_marlin_moe_quant_config, diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_nvfp4.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_nvfp4.py index 4980a1a511..40f3897166 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_nvfp4.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_nvfp4.py @@ -251,7 +251,6 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod): experts_cls=self.experts_cls, backend=self.nvfp4_backend, routing_tables=layer._expert_routing_tables(), - layer=layer, ) self.moe_kernel.fused_experts.process_weights_after_loading(layer) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_fp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_fp8.py index 8cdc928b5e..704bc9a87c 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_fp8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_fp8.py @@ -339,7 +339,6 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod): fp8_backend=self.fp8_backend, experts_cls=self.experts_cls, routing_tables=layer._expert_routing_tables(), - layer=layer, ) def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig: diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py index fd265fa858..f27c54c8f4 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py @@ -159,7 +159,6 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod): moe_config=self.moe, experts_cls=self.experts_cls, routing_tables=layer._expert_routing_tables(), - layer=layer, ) self.moe_kernel.fused_experts.process_weights_after_loading(layer) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_mxfp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_mxfp8.py index ee4a42786b..ee3b321363 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_mxfp8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_mxfp8.py @@ -139,7 +139,6 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod): fp8_backend=self.fp8_backend, experts_cls=self.experts_cls, routing_tables=layer._expert_routing_tables(), - layer=layer, ) def get_fused_moe_quant_config( diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py index ea77d8c0e8..7c04a654fb 100644 --- a/vllm/model_executor/layers/quantization/fp8.py +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -679,7 +679,6 @@ class Fp8MoEMethod(FusedMoEMethodBase): fp8_backend=self.fp8_backend, experts_cls=self.experts_cls, routing_tables=layer._expert_routing_tables(), - layer=layer, ) def process_weights_after_loading(self, layer: RoutedExperts) -> None: diff --git a/vllm/model_executor/layers/quantization/humming.py b/vllm/model_executor/layers/quantization/humming.py index 4e1ae0d0e4..b814205922 100644 --- a/vllm/model_executor/layers/quantization/humming.py +++ b/vllm/model_executor/layers/quantization/humming.py @@ -1,6 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import json import math from collections.abc import Callable from typing import TYPE_CHECKING, Any @@ -32,6 +31,7 @@ from vllm.model_executor.layers.quantization.base_config import ( ) from vllm.model_executor.layers.quantization.utils.humming_utils import ( convert_to_humming_moe_kernel_format, + get_humming_linear_compute_config, get_humming_moe_quant_config, input_schema_to_quant_key, make_humming_moe_kernel, @@ -425,7 +425,6 @@ class HummingLinearMethod(LinearMethodBase): for name, _ in list(layer.named_parameters()): if name != "bias": delattr(layer, name) - delattr(layer, "locks") self.__class__ = UnquantizedLinearMethod # type: ignore tensor = torch.empty( ( @@ -513,9 +512,6 @@ class HummingLinearMethod(LinearMethodBase): param = prepare_param(tensor, name, extra_attrs) setattr(layer, name, param) - locks = torch.zeros(1024, dtype=torch.int32) - layer.register_buffer("locks", locks) - if self.force_input_schema is not None: self.input_schema = self.force_input_schema @@ -574,9 +570,7 @@ class HummingLinearMethod(LinearMethodBase): del tensors - # prepare layer config from humming kernel - _hm.HummingMethod.prepare_layer_meta( - layer=layer, + self.layer_config = _hm.prepare_layer_config( shape_n=layer.output_partition_sizes_sum, shape_k=layer.input_size_per_partition, weight_schema=self.weight_schema, @@ -586,19 +580,17 @@ class HummingLinearMethod(LinearMethodBase): has_bias=layer.has_bias, torch_dtype=layer.param_dtype, ) + tensors = _hm.transform_humming_tensors( + self.layer_config, dict(layer.named_parameters()) + ) + for name, _ in list(layer.named_parameters()): + delattr(layer, name) + for name, tensor in tensors.items(): + param = torch.nn.Parameter(tensor, requires_grad=False) + setattr(layer, name, param) - # preprocess weight for inference - _hm.HummingMethod.transform_humming_layer(layer) - - # compute_config: kernel configs that do not directly affect weights - # but significantly impact kernel behavior or computation precision. - # see https://github.com/inclusionAI/humming/blob/main/docs/config.md - compute_config = { - "use_batch_invariant": envs.VLLM_BATCH_INVARIANT, - "use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM, - "gemm_type": "dense", - } - self.compute_config = json.dumps(compute_config) + self.compute_config = get_humming_linear_compute_config() + self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device) def apply( self, @@ -606,10 +598,16 @@ class HummingLinearMethod(LinearMethodBase): x: torch.Tensor, bias: torch.Tensor | None = None, ) -> torch.Tensor: - flatten_inputs = x.view(-1, x.size(-1)) - output = _hm.HummingMethod.forward_layer( - layer=layer, + flatten_inputs = x.reshape(-1, x.size(-1)) + output = _hm.humming_forward( + self.layer_config, inputs=flatten_inputs, + weight=layer.weight, + weight_scale=getattr(layer, "weight_scale", None), + zero_point=getattr(layer, "zero_point", None), + bias=getattr(layer, "bias", None), + weight_scale_2=getattr(layer, "weight_scale_2", None), + locks=self.locks, compute_config=self.compute_config, ) output = output.view(*x.shape[:-1], output.size(-1)) @@ -726,12 +724,13 @@ class HummingMoEMethod(FusedMoEMethodBase): # sublayer: a layer contains multiple sets of weights for quantized GEMM # (e.g., weight, weight_scale, etc.). # The weight names of sublayer start with the prefix "{sublayer_name}_" + w13_output_size = intermediate_size_per_partition * self.moe.w13_num_shards layer.sublayer_configs = { "w13": { - "shape_n": intermediate_size_per_partition * self.moe.w13_num_shards, + "shape_n": w13_output_size, "shape_k": hidden_size, "tensors_attrs": self.weight_schema.get_padded_tensors_attrs( - shape_n=intermediate_size_per_partition * self.moe.w13_num_shards, + shape_n=w13_output_size, shape_k=hidden_size, num_experts=num_experts, param_dtype=params_dtype, @@ -763,11 +762,14 @@ class HummingMoEMethod(FusedMoEMethodBase): if self.force_input_schema is not None: self.input_schema = self.force_input_schema - locks = torch.zeros(1024, dtype=torch.int32) - layer.register_buffer("locks", locks) - def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig: - return get_humming_moe_quant_config(layer) + return get_humming_moe_quant_config( + layer, + self.humming_configs, + gemm1_alpha=getattr(layer, "swiglu_alpha", None), + gemm1_beta=getattr(layer, "swiglu_beta", None), + gemm1_clamp_limit=getattr(layer, "swiglu_limit", None), + ) def process_weights_after_loading(self, layer: RoutedExperts) -> None: if getattr(self, "processed", False): @@ -775,7 +777,7 @@ class HummingMoEMethod(FusedMoEMethodBase): self.processed = True # Convert weights to Humming kernel format - convert_to_humming_moe_kernel_format( + self.humming_configs = convert_to_humming_moe_kernel_format( layer=layer, sublayer_configs=layer.sublayer_configs, weight_schema=self.weight_schema, @@ -791,7 +793,6 @@ class HummingMoEMethod(FusedMoEMethodBase): self.moe_quant_config, self.moe, self.experts_cls, - layer=layer, routing_tables=layer._expert_routing_tables(), ) @@ -814,13 +815,11 @@ class HummingMoEMethod(FusedMoEMethodBase): 3. Finalization (weight application & reduction - no-op for Humming since it's already done internally) - Humming handles all quantization, weight application, and reduction - internally in the experts.apply() method via HummingMethod calls. + Humming handles quantization, weight application, and reduction + internally in the experts implementation. - Note: Although w1/w2 weights are passed to the kernel for interface - consistency, Humming's experts.apply() reads weights directly from - the layer object via HummingMethod.forward_layer() and ignores the - w1/w2 parameters. + Humming experts consume the weights and quantization tensors passed + through the standard modular kernel interface. """ assert self.moe_kernel is not None return self.moe_kernel.apply( diff --git a/vllm/model_executor/layers/quantization/modelopt.py b/vllm/model_executor/layers/quantization/modelopt.py index 7bf77cf069..db98b76b3f 100644 --- a/vllm/model_executor/layers/quantization/modelopt.py +++ b/vllm/model_executor/layers/quantization/modelopt.py @@ -886,7 +886,6 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase): fp8_backend=self.fp8_backend, experts_cls=self.experts_cls, routing_tables=layer._expert_routing_tables(), - layer=layer, ) def process_weights_after_loading(self, layer: RoutedExperts) -> None: @@ -1579,7 +1578,6 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase): experts_cls=self.experts_cls, backend=self.nvfp4_backend, routing_tables=layer._expert_routing_tables(), - layer=layer, ) self.moe_kernel.fused_experts.process_weights_after_loading(layer) @@ -2058,7 +2056,6 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase): fp8_backend=self.mxfp8_backend, experts_cls=self.experts_cls, routing_tables=layer._expert_routing_tables(), - layer=layer, ) # No native MXFP8 MoE kernel on this device (e.g. gfx942): the emulation diff --git a/vllm/model_executor/layers/quantization/moe_wna16.py b/vllm/model_executor/layers/quantization/moe_wna16.py index 1e54a627b6..02207abf3d 100644 --- a/vllm/model_executor/layers/quantization/moe_wna16.py +++ b/vllm/model_executor/layers/quantization/moe_wna16.py @@ -370,7 +370,12 @@ class MoeWNA16Method(FusedMoEMethodBase): get_humming_moe_quant_config, ) - return get_humming_moe_quant_config(layer) + return get_humming_moe_quant_config( + layer, + gemm1_alpha=getattr(layer, "swiglu_alpha", None), + gemm1_beta=getattr(layer, "swiglu_beta", None), + gemm1_clamp_limit=getattr(layer, "swiglu_limit", None), + ) has_zp = self.quant_config.has_zp return make_wna16_moe_quant_config( @@ -391,7 +396,6 @@ class MoeWNA16Method(FusedMoEMethodBase): moe_config=self.moe, experts_cls=self.experts_cls, backend=self.wna16_backend, - layer=layer, routing_tables=layer._expert_routing_tables(), ) diff --git a/vllm/model_executor/layers/quantization/mxfp4.py b/vllm/model_executor/layers/quantization/mxfp4.py index d85713e150..26a396f23a 100644 --- a/vllm/model_executor/layers/quantization/mxfp4.py +++ b/vllm/model_executor/layers/quantization/mxfp4.py @@ -382,7 +382,6 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase): mxfp4_backend=self.mxfp4_backend, experts_cls=self.experts_cls, routing_tables=layer._expert_routing_tables(), - layer=layer, ) def process_weights_after_loading(self, layer: RoutedExperts) -> None: @@ -773,7 +772,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): mxfp4_backend=self.mxfp4_backend, experts_cls=self.experts_cls, routing_tables=layer._expert_routing_tables(), - layer=layer, ) def _convert_k3_situ_weight_to_kernel_format( diff --git a/vllm/model_executor/layers/quantization/online/fp8.py b/vllm/model_executor/layers/quantization/online/fp8.py index 3e98cfef5c..38b6267968 100644 --- a/vllm/model_executor/layers/quantization/online/fp8.py +++ b/vllm/model_executor/layers/quantization/online/fp8.py @@ -516,7 +516,6 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase): fp8_backend=self.fp8_backend, experts_cls=self.experts_cls, routing_tables=layer._expert_routing_tables(), - layer=layer, ) def get_fused_moe_quant_config( diff --git a/vllm/model_executor/layers/quantization/online/int8.py b/vllm/model_executor/layers/quantization/online/int8.py index 661da7f997..952229eced 100644 --- a/vllm/model_executor/layers/quantization/online/int8.py +++ b/vllm/model_executor/layers/quantization/online/int8.py @@ -117,7 +117,6 @@ class Int8OnlineMoEMethod(OnlineMoEMethodBase): moe_config=self.moe, experts_cls=self.experts_cls, routing_tables=layer._expert_routing_tables(), - layer=layer, ) self.moe_kernel.fused_experts.process_weights_after_loading(layer) diff --git a/vllm/model_executor/layers/quantization/online/mxfp8.py b/vllm/model_executor/layers/quantization/online/mxfp8.py index 84a81bd906..5eb54f9a09 100644 --- a/vllm/model_executor/layers/quantization/online/mxfp8.py +++ b/vllm/model_executor/layers/quantization/online/mxfp8.py @@ -200,7 +200,6 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase): fp8_backend=self.fp8_backend, experts_cls=self.experts_cls, routing_tables=layer._expert_routing_tables(), - layer=layer, ) def get_fused_moe_quant_config( diff --git a/vllm/model_executor/layers/quantization/online/nvfp4.py b/vllm/model_executor/layers/quantization/online/nvfp4.py index 0a8a60a73a..48de2a08fc 100644 --- a/vllm/model_executor/layers/quantization/online/nvfp4.py +++ b/vllm/model_executor/layers/quantization/online/nvfp4.py @@ -172,7 +172,6 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase): experts_cls=self.experts_cls, backend=self.nvfp4_backend, routing_tables=layer._expert_routing_tables(), - layer=layer, per_token_activation=True, ) self.moe_kernel.fused_experts.process_weights_after_loading(layer) diff --git a/vllm/model_executor/layers/quantization/quark/quark_moe.py b/vllm/model_executor/layers/quantization/quark/quark_moe.py index 54e10fa76e..a7b4d28480 100644 --- a/vllm/model_executor/layers/quantization/quark/quark_moe.py +++ b/vllm/model_executor/layers/quantization/quark/quark_moe.py @@ -804,7 +804,6 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod): moe_config=self.moe, experts_cls=self.experts_cls, routing_tables=layer._expert_routing_tables(), - layer=layer, ) def get_fused_moe_quant_config( @@ -1646,7 +1645,6 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod): experts_cls=self.experts_cls, backend=self.nvfp4_backend, routing_tables=layer._expert_routing_tables(), - layer=layer, ) def get_fused_moe_quant_config( diff --git a/vllm/model_executor/layers/quantization/utils/humming_utils.py b/vllm/model_executor/layers/quantization/utils/humming_utils.py index 15c6a178b0..7447cbef12 100644 --- a/vllm/model_executor/layers/quantization/utils/humming_utils.py +++ b/vllm/model_executor/layers/quantization/utils/humming_utils.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import json +from dataclasses import dataclass from typing import TYPE_CHECKING, Any import regex as re @@ -47,11 +48,19 @@ if TYPE_CHECKING: GPTQWeightSchema, HummingInputSchema, HummingWeightSchema, + LayerConfig, ) from vllm.utils.humming import dtypes as humming_dtypes logger = init_logger(__name__) + +@dataclass(kw_only=True) +class HummingMoEQuantConfig(FusedMoEQuantConfig): + w1_humming_config: "LayerConfig" + w2_humming_config: "LayerConfig" + + if has_humming(): from vllm.utils.humming import dtypes as humming_dtypes @@ -435,16 +444,17 @@ def convert_linear_layer_to_humming_standard( setattr(layer, name, param) -def prepare_humming_layer( +def prepare_humming_linear_layer_config( layer: LinearBase, quant_config: dict, input_quant_config: dict | None = None, -): +) -> "LayerConfig": from vllm.utils.humming import ( BaseInputSchema, BaseWeightSchema, HummingInputSchema, - HummingMethod, + prepare_layer_config, + transform_humming_tensors, ) weight_schema = BaseWeightSchema.from_config(quant_config) @@ -480,18 +490,8 @@ def prepare_humming_layer( layer.weight_schema = weight_schema - for name, _ in list(layer.named_parameters()): - delattr(layer, name) - - for name, tensor in tensors.items(): - if isinstance(tensor, torch.nn.Parameter): - tensor = tensor.data - param = torch.nn.Parameter(tensor, requires_grad=False) - setattr(layer, name, param) - - # Step 2: transform weight (humming standard format) for forwarding - HummingMethod.prepare_layer_meta( - layer=layer, + # Step 2: transform weight (humming standard format) for forwarding. + config = prepare_layer_config( shape_n=sum(layer.output_partition_sizes), shape_k=input_size_per_partition, weight_schema=weight_schema, @@ -501,20 +501,49 @@ def prepare_humming_layer( has_bias=layer.has_bias, torch_dtype=layer.params_dtype, ) + tensors = transform_humming_tensors(config, tensors) + for name, _ in list(layer.named_parameters()): + delattr(layer, name) + for name, tensor in tensors.items(): + param = torch.nn.Parameter(tensor, requires_grad=False) + setattr(layer, name, param) - HummingMethod.transform_humming_layer(layer) - if not hasattr(layer, "locks"): - device = layer.weight.device - locks = torch.zeros(1024, dtype=torch.int32, device=device) - layer.register_buffer("locks", locks) + return config - compute_config = { - "use_batch_invariant": envs.VLLM_BATCH_INVARIANT, - "use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM, - "gemm_type": "dense", - } - layer.compute_config = json.dumps(compute_config) +def get_humming_linear_compute_config() -> str: + return json.dumps( + { + "use_batch_invariant": envs.VLLM_BATCH_INVARIANT, + "use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM, + "gemm_type": "dense", + } + ) + + +def apply_humming_linear( + layer: LinearBase, + x: torch.Tensor, + *, + layer_config: "LayerConfig", + compute_config: str, + locks: torch.Tensor, +) -> torch.Tensor: + from vllm.utils.humming import humming_forward + + flatten_inputs = x.reshape(-1, x.size(-1)) + output = humming_forward( + layer_config, + inputs=flatten_inputs, + weight=layer.weight, + weight_scale=getattr(layer, "weight_scale", None), + zero_point=getattr(layer, "zero_point", None), + bias=getattr(layer, "bias", None), + weight_scale_2=getattr(layer, "weight_scale_2", None), + locks=locks, + compute_config=compute_config, + ) + return output.view(*x.shape[:-1], output.size(-1)) def make_humming_moe_quant_config( @@ -532,7 +561,9 @@ def make_humming_moe_quant_config( gemm1_alpha: float | None = None, gemm1_beta: float | None = None, gemm1_clamp_limit: float | None = None, -) -> FusedMoEQuantConfig: + humming_configs: dict[str, "LayerConfig"] | None = None, +) -> HummingMoEQuantConfig: + assert humming_configs is not None if quant_dtype is None: a_quant_desc = FusedMoEQuantDesc(dtype=None) else: @@ -557,7 +588,7 @@ def make_humming_moe_quant_config( bias=w2_bias, ) - return FusedMoEQuantConfig( + return HummingMoEQuantConfig( _a1=a_quant_desc, _a2=a_quant_desc, _w1=w1_quant_desc, @@ -565,15 +596,20 @@ def make_humming_moe_quant_config( gemm1_alpha=gemm1_alpha, gemm1_beta=gemm1_beta, gemm1_clamp_limit=gemm1_clamp_limit, + w1_humming_config=humming_configs["w13"], + w2_humming_config=humming_configs["w2"], ) def get_humming_moe_quant_config( layer: "RoutedExperts", + humming_configs: dict[str, "LayerConfig"] | None = None, gemm1_alpha: float | None = None, gemm1_beta: float | None = None, gemm1_clamp_limit: float | None = None, ): + if humming_configs is None: + humming_configs = layer.humming_configs input_schema = layer.input_schemas["w13"] weight_schema = layer.weight_schemas["w13"] @@ -600,13 +636,17 @@ def get_humming_moe_quant_config( weight_dtype=str(weight_schema.b_dtype), weight_group_shape=weight_group_shape, w1_scale=getattr(layer, "w13_weight_scale", None), - w1_gscale=getattr(layer, "w13_global_scale", None), + w1_gscale=getattr(layer, "w13_weight_scale_2", None), w1_zp=getattr(layer, "w13_zero_point", None), w1_bias=getattr(layer, "w13_bias", None), w2_scale=getattr(layer, "w2_weight_scale", None), - w2_gscale=getattr(layer, "w2_global_scale", None), + w2_gscale=getattr(layer, "w2_weight_scale_2", None), w2_zp=getattr(layer, "w2_zero_point", None), w2_bias=getattr(layer, "w2_bias", None), + gemm1_alpha=gemm1_alpha, + gemm1_beta=gemm1_beta, + gemm1_clamp_limit=gemm1_clamp_limit, + humming_configs=humming_configs, ) @@ -677,7 +717,6 @@ def make_humming_moe_kernel( moe_quant_config: FusedMoEQuantConfig, moe_config: FusedMoEConfig, experts_cls: type[mk.FusedMoEExperts], - layer: "RoutedExperts", routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None, ) -> mk.FusedMoEKernel: # Create Prepare/Finalize. @@ -692,8 +731,6 @@ def make_humming_moe_kernel( logger.info_once("Using %s", prepare_finalize.__class__.__name__) - extra_args: dict[str, Any] = {"layer": layer} - # Create Experts. if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts: max_num_tokens = prepare_finalize.max_num_tokens_per_rank() @@ -703,13 +740,11 @@ def make_humming_moe_kernel( quant_config=moe_quant_config, max_num_tokens=max_num_tokens, num_dispatchers=prepare_finalize.num_dispatchers(), - **extra_args, ) else: experts = experts_cls( moe_config=moe_config, quant_config=moe_quant_config, - **extra_args, ) kernel = mk.FusedMoEKernel( @@ -822,16 +857,14 @@ def _prepare_and_transform_sublayer( has_bias: bool, num_experts: int, param_dtype: torch.dtype, -) -> None: - """ - Prepare layer metadata and transform weights for a sublayer. +) -> "LayerConfig": + """Prepare Humming configuration and transform one sublayer's tensors.""" + from vllm.utils.humming import ( + prepare_layer_config, + transform_humming_tensors, + ) - This calls Humming's prepare_layer_meta and transform_humming_layer. - """ - from vllm.utils.humming import HummingMethod - - HummingMethod.prepare_layer_meta( - layer=layer, + config = prepare_layer_config( shape_n=shape_n, shape_k=shape_k, pad_n_to_multiple=256, @@ -841,10 +874,13 @@ def _prepare_and_transform_sublayer( has_bias=has_bias, num_experts=num_experts, torch_dtype=param_dtype, - sublayer_name=sublayer_name, ) - - HummingMethod.transform_humming_layer(layer, sublayer_name=sublayer_name) + tensors = transform_humming_tensors( + config, + _extract_sublayer_tensors(layer, sublayer_name), + ) + _replace_layer_parameters(layer, sublayer_name, tensors) + return config def _process_single_sublayer( @@ -858,7 +894,7 @@ def _process_single_sublayer( num_experts: int, param_dtype: torch.dtype, force_weight_schema: Any | None = None, -) -> tuple[Any, Any]: +) -> tuple[Any, Any, "LayerConfig"]: """ Process a single sublayer: convert, optionally requant, prepare, and transform. @@ -878,7 +914,7 @@ def _process_single_sublayer( force_weight_schema: Optional schema to force requantization to Returns: - Tuple of (final_weight_schema, final_input_schema) + Tuple of the final weight schema, input schema, and Humming layer config. """ from vllm.utils.humming import HummingWeightSchema @@ -910,7 +946,7 @@ def _process_single_sublayer( del tensors # Step 3: Prepare layer metadata and transform weights - _prepare_and_transform_sublayer( + config = _prepare_and_transform_sublayer( layer=layer, sublayer_name=sublayer_name, shape_n=shape_n, @@ -922,7 +958,7 @@ def _process_single_sublayer( param_dtype=param_dtype, ) - return current_weight_schema, current_input_schema + return current_weight_schema, current_input_schema, config def convert_to_humming_moe_kernel_format( @@ -932,7 +968,7 @@ def convert_to_humming_moe_kernel_format( weight_schema: Any | None = None, input_schema: Any | None = None, force_weight_schema: Any | None = None, -) -> None: +) -> dict[str, "LayerConfig"]: """ Convert MoE weights from checkpoint format to Humming kernel format. @@ -959,6 +995,7 @@ def convert_to_humming_moe_kernel_format( Side effects: - Modifies layer parameters in place - Sets layer.weight_schemas and layer.input_schemas + - Sets layer.humming_configs for quant config construction """ # Build schemas from quant_config if not provided @@ -1005,25 +1042,27 @@ def convert_to_humming_moe_kernel_format( layer.weight_schemas = {} layer.input_schemas = {} + humming_configs = {} for sublayer_name, configs in sublayer_configs.items(): - final_weight_schema, final_input_schema = _process_single_sublayer( - layer=layer, - sublayer_name=sublayer_name, - shape_n=configs["shape_n"], - shape_k=configs["shape_k"], - weight_schema=weight_schema, - input_schema=input_schema, - has_bias=has_bias, - num_experts=num_experts, - param_dtype=param_dtype, - force_weight_schema=force_weight_schema, + final_weight_schema, final_input_schema, humming_config = ( + _process_single_sublayer( + layer=layer, + sublayer_name=sublayer_name, + shape_n=configs["shape_n"], + shape_k=configs["shape_k"], + weight_schema=weight_schema, + input_schema=input_schema, + has_bias=has_bias, + num_experts=num_experts, + param_dtype=param_dtype, + force_weight_schema=force_weight_schema, + ) ) layer.weight_schemas[sublayer_name] = final_weight_schema layer.input_schemas[sublayer_name] = final_input_schema + humming_configs[sublayer_name] = humming_config - if not hasattr(layer, "locks"): - device = layer.w13_weight.device - locks = torch.zeros(1024, dtype=torch.int32, device=device) - layer.register_buffer("locks", locks) + layer.humming_configs = humming_configs + return humming_configs diff --git a/vllm/utils/humming.py b/vllm/utils/humming.py index bdd519bd8c..014603ff39 100644 --- a/vllm/utils/humming.py +++ b/vllm/utils/humming.py @@ -15,9 +15,13 @@ _EXPORTS: dict[str, str] = { "dtypes": "humming.dtypes", "DataType": "humming.dtypes:DataType", "GemmType": "humming.config:GemmType", + "LayerConfig": "humming.config:LayerConfig", "WeightScaleType": "humming.config:WeightScaleType", - "HummingMethod": "humming.layer:HummingMethod", - "HummingLayerMeta": "humming.layer:HummingLayerMeta", + "humming_forward": "humming.forward:humming_forward", + "may_quant_input": "humming.forward:may_quant_input", + "prepare_layer_config": "humming.transform:prepare_layer_config", + "transform_humming_tensors": "humming.transform:transform_humming_tensors", + "get_heuristics_config": "humming.tune:get_heuristics_config", "BaseInputSchema": "humming.schema:BaseInputSchema", "BaseWeightSchema": "humming.schema:BaseWeightSchema", "HummingInputSchema": "humming.schema:HummingInputSchema",