[Bugfix] Lazily import the humming quantization backend (#44921)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Michael Goin
2026-06-10 06:06:17 -07:00
committed by GitHub
co-authored by Claude
parent 4882fd7632
commit 87c15d46e3
5 changed files with 86 additions and 52 deletions
@@ -5,7 +5,7 @@
import torch
from vllm.platforms import current_platform
from vllm.utils.import_utils import _has_module
from vllm.utils.import_utils import has_humming
from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
@@ -19,7 +19,7 @@ class HummingLinearKernel(MPLinearKernel):
def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]:
if not current_platform.is_cuda():
return False, "Humming is only supported on CUDA"
if not _has_module("humming"):
if not has_humming():
return False, "Humming is not installed"
if c.has_g_idx:
return False, "Humming does not support act-order (g_idx)"
@@ -50,7 +50,7 @@ class HummingLinearKernel(MPLinearKernel):
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
from humming.layer import HummingMethod
from vllm.utils.humming import HummingMethod
flatten_inputs = x.view(-1, x.size(-1))
output = HummingMethod.forward_layer(
@@ -7,9 +7,6 @@ import math
from typing import TYPE_CHECKING, Any
import torch
from humming import dtypes
from humming.config import GemmType as HummingGemmType
from humming.layer import HummingLayerMeta, HummingMethod
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm import envs
@@ -41,6 +38,8 @@ from vllm.model_executor.layers.fused_moe.utils import (
)
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
from vllm.platforms import current_platform
from vllm.utils.humming import GemmType as HummingGemmType
from vllm.utils.humming import HummingLayerMeta, HummingMethod, dtypes
from vllm.v1.worker.workspace import current_workspace_manager
if TYPE_CHECKING:
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any
import regex as re
import torch
import vllm.utils.humming as _hm
from vllm import envs
from vllm.model_executor.layers.fused_moe import (
FusedMoEConfig,
@@ -41,37 +42,16 @@ from vllm.model_executor.parameter import (
RowvLLMParameter,
)
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_humming
if has_humming() and current_platform.is_cuda():
from humming.dtypes import DataType
from humming.layer import HummingMethod
from humming.schema import (
BaseInputSchema,
BaseWeightSchema,
HummingInputSchema,
HummingWeightSchema,
)
from humming.utils.weight import quantize_weight
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
BatchedHummingGroupedExperts,
HummingGroupedExperts,
HummingIndexedExperts,
get_humming_moe_gemm_type,
)
if TYPE_CHECKING:
from humming.schema import (
from vllm.model_executor.models.utils import WeightsMapper
from vllm.utils.humming import (
BaseInputSchema,
BaseWeightSchema,
HummingInputSchema,
HummingWeightSchema,
)
from vllm.model_executor.models.utils import WeightsMapper
def prepare_padded_shape(shape, x):
padded_shape = math.ceil(shape / x) * x
@@ -265,7 +245,7 @@ class HummingConfig(QuantizationConfig):
break
if "quant_method" in layer_config:
return BaseWeightSchema.from_config(layer_config)
return _hm.BaseWeightSchema.from_config(layer_config)
return None
def get_layer_input_schema(self, config: dict[str, Any], prefix: str):
@@ -277,8 +257,8 @@ class HummingConfig(QuantizationConfig):
return None
config = group_config
if config.get("quant_method", None) in BaseInputSchema.INPUT_SCHEMA_MAP:
return BaseInputSchema.from_config(config)
if config.get("quant_method", None) in _hm.BaseInputSchema.INPUT_SCHEMA_MAP:
return _hm.BaseInputSchema.from_config(config)
return None
def get_quant_config_for_layer(
@@ -316,7 +296,7 @@ class HummingConfig(QuantizationConfig):
input_schema = force_input_schema
if force_weight_schema is not None and force_input_schema is None:
force_input_schema = HummingInputSchema()
force_input_schema = _hm.HummingInputSchema()
return HummingLayerQuantizationConfig(
weight_schema=weight_schema,
@@ -360,7 +340,7 @@ class HummingLayerQuantizationConfig(HummingConfig):
):
self.weight_schema = weight_schema
if input_schema is None:
input_schema = HummingInputSchema()
input_schema = _hm.HummingInputSchema()
self.input_schema = input_schema
self.force_weight_schema = force_weight_schema
self.force_input_schema = force_input_schema
@@ -368,7 +348,7 @@ class HummingLayerQuantizationConfig(HummingConfig):
@classmethod
def from_config(cls, config):
weight_schema = BaseWeightSchema.from_config(config)
weight_schema = _hm.BaseWeightSchema.from_config(config)
return cls(weight_schema)
def get_quant_method(
@@ -397,10 +377,10 @@ class HummingLinearMethod(LinearMethodBase):
is_unquantized = name == "weight" and loaded_weight.dtype in float_dtypes
if is_unquantized and self.is_online_quant:
# online quant (fp16/bf16 -> quant_type)
assert isinstance(self.weight_schema, HummingWeightSchema)
f16_dtype = DataType.from_torch_dtype(layer.param_dtype)
assert isinstance(self.weight_schema, _hm.HummingWeightSchema)
f16_dtype = _hm.DataType.from_torch_dtype(layer.param_dtype)
has_global_scale = "TENSOR" in str(self.weight_schema.weight_scale_type)
tensor_list = quantize_weight(
tensor_list = _hm.quantize_weight(
weight=loaded_weight,
dtype=self.weight_schema.b_dtype,
scale_dtype=self.weight_schema.bs_dtype or f16_dtype,
@@ -531,7 +511,7 @@ class HummingLinearMethod(LinearMethodBase):
return None
# convert from checkpoint format to humming format
if not isinstance(self.weight_schema, HummingWeightSchema):
if not isinstance(self.weight_schema, _hm.HummingWeightSchema):
self.weight_schema, tensors = self.weight_schema.convert_humming(
tensors=layer.state_dict(),
shape_n_stacks=layer.output_partition_sizes,
@@ -556,7 +536,7 @@ class HummingLinearMethod(LinearMethodBase):
del tensors
# force requant (origin quant setting -> fp16/bf16 -> new_quant setting)
assert isinstance(self.weight_schema, HummingWeightSchema)
assert isinstance(self.weight_schema, _hm.HummingWeightSchema)
force_requant = self.force_weight_schema is not None
if force_requant and self.weight_schema != self.force_weight_schema:
tensors = self.weight_schema.requant_tensors(
@@ -578,7 +558,7 @@ class HummingLinearMethod(LinearMethodBase):
del tensors
# prepare layer config from humming kernel
HummingMethod.prepare_layer_meta(
_hm.HummingMethod.prepare_layer_meta(
layer=layer,
shape_n=layer.output_partition_sizes_sum,
shape_k=layer.input_size_per_partition,
@@ -591,7 +571,7 @@ class HummingLinearMethod(LinearMethodBase):
)
# preprocess weight for inference
HummingMethod.transform_humming_layer(layer)
_hm.HummingMethod.transform_humming_layer(layer)
# compute_config: kernel configs that do not directly affect weights
# but significantly impact kernel behavior or computation precision.
@@ -610,7 +590,7 @@ class HummingLinearMethod(LinearMethodBase):
bias: torch.Tensor | None = None,
) -> torch.Tensor:
flatten_inputs = x.view(-1, x.size(-1))
output = HummingMethod.forward_layer(
output = _hm.HummingMethod.forward_layer(
layer=layer,
inputs=flatten_inputs,
compute_config=self.compute_config,
@@ -645,10 +625,10 @@ class HummingMoEMethod(FusedMoEMethodBase):
is_unquantized = name == "weight" and loaded_weight.dtype in float_dtypes
# online quant (fp16/bf16 -> quant_type)
if is_unquantized:
assert isinstance(self.weight_schema, HummingWeightSchema)
f16_dtype = DataType.from_torch_dtype(layer.param_dtype)
assert isinstance(self.weight_schema, _hm.HummingWeightSchema)
f16_dtype = _hm.DataType.from_torch_dtype(layer.param_dtype)
has_global_scale = "TENSOR" in str(self.weight_schema.weight_scale_type)
tensor_list = quantize_weight(
tensor_list = _hm.quantize_weight(
weight=loaded_weight,
dtype=self.weight_schema.b_dtype,
scale_dtype=self.weight_schema.bs_dtype or f16_dtype,
@@ -771,7 +751,7 @@ class HummingMoEMethod(FusedMoEMethodBase):
input_schema = self.input_schema
weight_schema = self.weight_schema
# convert from checkpoint format to humming format
if not isinstance(weight_schema, HummingWeightSchema):
if not isinstance(weight_schema, _hm.HummingWeightSchema):
tensors: dict[str, torch.Tensor] = dict(
(key.removeprefix(sublayer_name + "_"), value)
for key, value in layer.state_dict().items()
@@ -813,7 +793,7 @@ class HummingMoEMethod(FusedMoEMethodBase):
layer.input_schemas[sublayer_name] = input_schema
# force requant (origin quant setting -> fp16/bf16 -> new_quant setting)
assert isinstance(weight_schema, HummingWeightSchema)
assert isinstance(weight_schema, _hm.HummingWeightSchema)
force_requant = self.force_weight_schema is not None
if force_requant and weight_schema != self.force_weight_schema:
tensors = dict(
@@ -845,7 +825,7 @@ class HummingMoEMethod(FusedMoEMethodBase):
del tensors
# prepare layer config from humming kernel
HummingMethod.prepare_layer_meta(
_hm.HummingMethod.prepare_layer_meta(
layer=layer,
shape_n=configs["shape_n"],
shape_k=configs["shape_k"],
@@ -860,7 +840,15 @@ class HummingMoEMethod(FusedMoEMethodBase):
)
# preprocess weight for inference
HummingMethod.transform_humming_layer(layer, sublayer_name=sublayer_name)
_hm.HummingMethod.transform_humming_layer(
layer, sublayer_name=sublayer_name
)
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
HummingGroupedExperts,
HummingIndexedExperts,
get_humming_moe_gemm_type,
)
# use moe modular
experts: HummingIndexedExperts | HummingGroupedExperts
@@ -878,6 +866,12 @@ class HummingMoEMethod(FusedMoEMethodBase):
layer: torch.nn.Module,
):
from vllm.model_executor.layers.fused_moe import modular_kernel as mk
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
BatchedHummingGroupedExperts,
HummingGroupedExperts,
HummingIndexedExperts,
get_humming_moe_gemm_type,
)
activation_format = prepare_finalize.activation_format
assert self.moe_quant_config is not None
@@ -5,8 +5,6 @@ from typing import Any
import regex as re
import torch
from humming.layer import HummingInputSchema, HummingMethod
from humming.schema import BaseWeightSchema
from vllm import envs
from vllm.model_executor.layers.fused_moe import RoutedExperts
@@ -16,6 +14,7 @@ from vllm.model_executor.layers.fused_moe.config import (
)
from vllm.model_executor.layers.linear import LinearBase
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
from vllm.utils.humming import BaseWeightSchema, HummingInputSchema, HummingMethod
def humming_is_layer_skipped(config: dict[str, Any], prefix: str):
+42
View File
@@ -0,0 +1,42 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Lazy facade for the optional ``humming`` package.
vLLM code should import humming symbols from here so that ``import humming``
(which has import-time side effects) is deferred until first use. Add new
symbols by appending one entry to ``_EXPORTS`` as ``"module.path:attr"``,
or ``"module.path"`` for a whole-module re-export.
"""
import importlib
from typing import Any
_EXPORTS: dict[str, str] = {
"dtypes": "humming.dtypes",
"DataType": "humming.dtypes:DataType",
"GemmType": "humming.config:GemmType",
"HummingMethod": "humming.layer:HummingMethod",
"HummingLayerMeta": "humming.layer:HummingLayerMeta",
"BaseInputSchema": "humming.schema:BaseInputSchema",
"BaseWeightSchema": "humming.schema:BaseWeightSchema",
"HummingInputSchema": "humming.schema:HummingInputSchema",
"HummingWeightSchema": "humming.schema:HummingWeightSchema",
"quantize_weight": "humming.utils.weight:quantize_weight",
}
def __getattr__(name: str) -> Any:
spec = _EXPORTS.get(name)
if spec is None:
raise AttributeError(f"module 'vllm.utils.humming' has no attribute {name!r}")
if ":" in spec:
mod_path, attr = spec.split(":", 1)
obj = getattr(importlib.import_module(mod_path), attr)
else:
obj = importlib.import_module(spec)
globals()[name] = obj
return obj
def __dir__() -> list[str]:
return sorted({*globals(), *_EXPORTS})