From 87c15d46e3a3807258eb5f859875714c78ba173d Mon Sep 17 00:00:00 2001 From: Michael Goin Date: Wed, 10 Jun 2026 09:06:17 -0400 Subject: [PATCH] [Bugfix] Lazily import the humming quantization backend (#44921) Signed-off-by: mgoin Co-authored-by: Claude --- .../kernels/linear/mixed_precision/humming.py | 6 +- .../fused_moe/experts/fused_humming_moe.py | 5 +- .../layers/quantization/humming.py | 82 +++++++++---------- .../quantization/utils/humming_utils.py | 3 +- vllm/utils/humming.py | 42 ++++++++++ 5 files changed, 86 insertions(+), 52 deletions(-) create mode 100644 vllm/utils/humming.py diff --git a/vllm/model_executor/kernels/linear/mixed_precision/humming.py b/vllm/model_executor/kernels/linear/mixed_precision/humming.py index cb02d66129..764c0f4227 100644 --- a/vllm/model_executor/kernels/linear/mixed_precision/humming.py +++ b/vllm/model_executor/kernels/linear/mixed_precision/humming.py @@ -5,7 +5,7 @@ import torch from vllm.platforms import current_platform -from vllm.utils.import_utils import _has_module +from vllm.utils.import_utils import has_humming from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig @@ -19,7 +19,7 @@ class HummingLinearKernel(MPLinearKernel): def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]: if not current_platform.is_cuda(): return False, "Humming is only supported on CUDA" - if not _has_module("humming"): + if not has_humming(): return False, "Humming is not installed" if c.has_g_idx: return False, "Humming does not support act-order (g_idx)" @@ -50,7 +50,7 @@ class HummingLinearKernel(MPLinearKernel): x: torch.Tensor, bias: torch.Tensor | None = None, ) -> torch.Tensor: - from humming.layer import HummingMethod + from vllm.utils.humming import HummingMethod flatten_inputs = x.view(-1, x.size(-1)) output = HummingMethod.forward_layer( diff --git a/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py b/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py index 53623f1325..5177fa0cde 100644 --- a/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py @@ -7,9 +7,6 @@ import math from typing import TYPE_CHECKING, Any import torch -from humming import dtypes -from humming.config import GemmType as HummingGemmType -from humming.layer import HummingLayerMeta, HummingMethod import vllm.model_executor.layers.fused_moe.modular_kernel as mk from vllm import envs @@ -41,6 +38,8 @@ from vllm.model_executor.layers.fused_moe.utils import ( ) from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey from vllm.platforms import current_platform +from vllm.utils.humming import GemmType as HummingGemmType +from vllm.utils.humming import HummingLayerMeta, HummingMethod, dtypes from vllm.v1.worker.workspace import current_workspace_manager if TYPE_CHECKING: diff --git a/vllm/model_executor/layers/quantization/humming.py b/vllm/model_executor/layers/quantization/humming.py index 91730b0639..49e2f18ef6 100644 --- a/vllm/model_executor/layers/quantization/humming.py +++ b/vllm/model_executor/layers/quantization/humming.py @@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any import regex as re import torch +import vllm.utils.humming as _hm from vllm import envs from vllm.model_executor.layers.fused_moe import ( FusedMoEConfig, @@ -41,37 +42,16 @@ from vllm.model_executor.parameter import ( RowvLLMParameter, ) from vllm.model_executor.utils import set_weight_attrs -from vllm.platforms import current_platform -from vllm.utils.import_utils import has_humming - -if has_humming() and current_platform.is_cuda(): - from humming.dtypes import DataType - from humming.layer import HummingMethod - from humming.schema import ( - BaseInputSchema, - BaseWeightSchema, - HummingInputSchema, - HummingWeightSchema, - ) - from humming.utils.weight import quantize_weight - - from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import ( - BatchedHummingGroupedExperts, - HummingGroupedExperts, - HummingIndexedExperts, - get_humming_moe_gemm_type, - ) if TYPE_CHECKING: - from humming.schema import ( + from vllm.model_executor.models.utils import WeightsMapper + from vllm.utils.humming import ( BaseInputSchema, BaseWeightSchema, HummingInputSchema, HummingWeightSchema, ) - from vllm.model_executor.models.utils import WeightsMapper - def prepare_padded_shape(shape, x): padded_shape = math.ceil(shape / x) * x @@ -265,7 +245,7 @@ class HummingConfig(QuantizationConfig): break if "quant_method" in layer_config: - return BaseWeightSchema.from_config(layer_config) + return _hm.BaseWeightSchema.from_config(layer_config) return None def get_layer_input_schema(self, config: dict[str, Any], prefix: str): @@ -277,8 +257,8 @@ class HummingConfig(QuantizationConfig): return None config = group_config - if config.get("quant_method", None) in BaseInputSchema.INPUT_SCHEMA_MAP: - return BaseInputSchema.from_config(config) + if config.get("quant_method", None) in _hm.BaseInputSchema.INPUT_SCHEMA_MAP: + return _hm.BaseInputSchema.from_config(config) return None def get_quant_config_for_layer( @@ -316,7 +296,7 @@ class HummingConfig(QuantizationConfig): input_schema = force_input_schema if force_weight_schema is not None and force_input_schema is None: - force_input_schema = HummingInputSchema() + force_input_schema = _hm.HummingInputSchema() return HummingLayerQuantizationConfig( weight_schema=weight_schema, @@ -360,7 +340,7 @@ class HummingLayerQuantizationConfig(HummingConfig): ): self.weight_schema = weight_schema if input_schema is None: - input_schema = HummingInputSchema() + input_schema = _hm.HummingInputSchema() self.input_schema = input_schema self.force_weight_schema = force_weight_schema self.force_input_schema = force_input_schema @@ -368,7 +348,7 @@ class HummingLayerQuantizationConfig(HummingConfig): @classmethod def from_config(cls, config): - weight_schema = BaseWeightSchema.from_config(config) + weight_schema = _hm.BaseWeightSchema.from_config(config) return cls(weight_schema) def get_quant_method( @@ -397,10 +377,10 @@ class HummingLinearMethod(LinearMethodBase): is_unquantized = name == "weight" and loaded_weight.dtype in float_dtypes if is_unquantized and self.is_online_quant: # online quant (fp16/bf16 -> quant_type) - assert isinstance(self.weight_schema, HummingWeightSchema) - f16_dtype = DataType.from_torch_dtype(layer.param_dtype) + assert isinstance(self.weight_schema, _hm.HummingWeightSchema) + f16_dtype = _hm.DataType.from_torch_dtype(layer.param_dtype) has_global_scale = "TENSOR" in str(self.weight_schema.weight_scale_type) - tensor_list = quantize_weight( + tensor_list = _hm.quantize_weight( weight=loaded_weight, dtype=self.weight_schema.b_dtype, scale_dtype=self.weight_schema.bs_dtype or f16_dtype, @@ -531,7 +511,7 @@ class HummingLinearMethod(LinearMethodBase): return None # convert from checkpoint format to humming format - if not isinstance(self.weight_schema, HummingWeightSchema): + if not isinstance(self.weight_schema, _hm.HummingWeightSchema): self.weight_schema, tensors = self.weight_schema.convert_humming( tensors=layer.state_dict(), shape_n_stacks=layer.output_partition_sizes, @@ -556,7 +536,7 @@ class HummingLinearMethod(LinearMethodBase): del tensors # force requant (origin quant setting -> fp16/bf16 -> new_quant setting) - assert isinstance(self.weight_schema, HummingWeightSchema) + assert isinstance(self.weight_schema, _hm.HummingWeightSchema) force_requant = self.force_weight_schema is not None if force_requant and self.weight_schema != self.force_weight_schema: tensors = self.weight_schema.requant_tensors( @@ -578,7 +558,7 @@ class HummingLinearMethod(LinearMethodBase): del tensors # prepare layer config from humming kernel - HummingMethod.prepare_layer_meta( + _hm.HummingMethod.prepare_layer_meta( layer=layer, shape_n=layer.output_partition_sizes_sum, shape_k=layer.input_size_per_partition, @@ -591,7 +571,7 @@ class HummingLinearMethod(LinearMethodBase): ) # preprocess weight for inference - HummingMethod.transform_humming_layer(layer) + _hm.HummingMethod.transform_humming_layer(layer) # compute_config: kernel configs that do not directly affect weights # but significantly impact kernel behavior or computation precision. @@ -610,7 +590,7 @@ class HummingLinearMethod(LinearMethodBase): bias: torch.Tensor | None = None, ) -> torch.Tensor: flatten_inputs = x.view(-1, x.size(-1)) - output = HummingMethod.forward_layer( + output = _hm.HummingMethod.forward_layer( layer=layer, inputs=flatten_inputs, compute_config=self.compute_config, @@ -645,10 +625,10 @@ class HummingMoEMethod(FusedMoEMethodBase): is_unquantized = name == "weight" and loaded_weight.dtype in float_dtypes # online quant (fp16/bf16 -> quant_type) if is_unquantized: - assert isinstance(self.weight_schema, HummingWeightSchema) - f16_dtype = DataType.from_torch_dtype(layer.param_dtype) + assert isinstance(self.weight_schema, _hm.HummingWeightSchema) + f16_dtype = _hm.DataType.from_torch_dtype(layer.param_dtype) has_global_scale = "TENSOR" in str(self.weight_schema.weight_scale_type) - tensor_list = quantize_weight( + tensor_list = _hm.quantize_weight( weight=loaded_weight, dtype=self.weight_schema.b_dtype, scale_dtype=self.weight_schema.bs_dtype or f16_dtype, @@ -771,7 +751,7 @@ class HummingMoEMethod(FusedMoEMethodBase): input_schema = self.input_schema weight_schema = self.weight_schema # convert from checkpoint format to humming format - if not isinstance(weight_schema, HummingWeightSchema): + if not isinstance(weight_schema, _hm.HummingWeightSchema): tensors: dict[str, torch.Tensor] = dict( (key.removeprefix(sublayer_name + "_"), value) for key, value in layer.state_dict().items() @@ -813,7 +793,7 @@ class HummingMoEMethod(FusedMoEMethodBase): layer.input_schemas[sublayer_name] = input_schema # force requant (origin quant setting -> fp16/bf16 -> new_quant setting) - assert isinstance(weight_schema, HummingWeightSchema) + assert isinstance(weight_schema, _hm.HummingWeightSchema) force_requant = self.force_weight_schema is not None if force_requant and weight_schema != self.force_weight_schema: tensors = dict( @@ -845,7 +825,7 @@ class HummingMoEMethod(FusedMoEMethodBase): del tensors # prepare layer config from humming kernel - HummingMethod.prepare_layer_meta( + _hm.HummingMethod.prepare_layer_meta( layer=layer, shape_n=configs["shape_n"], shape_k=configs["shape_k"], @@ -860,7 +840,15 @@ class HummingMoEMethod(FusedMoEMethodBase): ) # preprocess weight for inference - HummingMethod.transform_humming_layer(layer, sublayer_name=sublayer_name) + _hm.HummingMethod.transform_humming_layer( + layer, sublayer_name=sublayer_name + ) + + from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import ( + HummingGroupedExperts, + HummingIndexedExperts, + get_humming_moe_gemm_type, + ) # use moe modular experts: HummingIndexedExperts | HummingGroupedExperts @@ -878,6 +866,12 @@ class HummingMoEMethod(FusedMoEMethodBase): layer: torch.nn.Module, ): from vllm.model_executor.layers.fused_moe import modular_kernel as mk + from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import ( + BatchedHummingGroupedExperts, + HummingGroupedExperts, + HummingIndexedExperts, + get_humming_moe_gemm_type, + ) activation_format = prepare_finalize.activation_format assert self.moe_quant_config is not None diff --git a/vllm/model_executor/layers/quantization/utils/humming_utils.py b/vllm/model_executor/layers/quantization/utils/humming_utils.py index d9e02542c6..9169e376e7 100644 --- a/vllm/model_executor/layers/quantization/utils/humming_utils.py +++ b/vllm/model_executor/layers/quantization/utils/humming_utils.py @@ -5,8 +5,6 @@ from typing import Any import regex as re import torch -from humming.layer import HummingInputSchema, HummingMethod -from humming.schema import BaseWeightSchema from vllm import envs from vllm.model_executor.layers.fused_moe import RoutedExperts @@ -16,6 +14,7 @@ from vllm.model_executor.layers.fused_moe.config import ( ) from vllm.model_executor.layers.linear import LinearBase from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape +from vllm.utils.humming import BaseWeightSchema, HummingInputSchema, HummingMethod def humming_is_layer_skipped(config: dict[str, Any], prefix: str): diff --git a/vllm/utils/humming.py b/vllm/utils/humming.py new file mode 100644 index 0000000000..b8d9445c3f --- /dev/null +++ b/vllm/utils/humming.py @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Lazy facade for the optional ``humming`` package. + +vLLM code should import humming symbols from here so that ``import humming`` +(which has import-time side effects) is deferred until first use. Add new +symbols by appending one entry to ``_EXPORTS`` as ``"module.path:attr"``, +or ``"module.path"`` for a whole-module re-export. +""" + +import importlib +from typing import Any + +_EXPORTS: dict[str, str] = { + "dtypes": "humming.dtypes", + "DataType": "humming.dtypes:DataType", + "GemmType": "humming.config:GemmType", + "HummingMethod": "humming.layer:HummingMethod", + "HummingLayerMeta": "humming.layer:HummingLayerMeta", + "BaseInputSchema": "humming.schema:BaseInputSchema", + "BaseWeightSchema": "humming.schema:BaseWeightSchema", + "HummingInputSchema": "humming.schema:HummingInputSchema", + "HummingWeightSchema": "humming.schema:HummingWeightSchema", + "quantize_weight": "humming.utils.weight:quantize_weight", +} + + +def __getattr__(name: str) -> Any: + spec = _EXPORTS.get(name) + if spec is None: + raise AttributeError(f"module 'vllm.utils.humming' has no attribute {name!r}") + if ":" in spec: + mod_path, attr = spec.split(":", 1) + obj = getattr(importlib.import_module(mod_path), attr) + else: + obj = importlib.import_module(spec) + globals()[name] = obj + return obj + + +def __dir__() -> list[str]: + return sorted({*globals(), *_EXPORTS})