[Bugfix] Lazily import the humming quantization backend (#44921)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Michael Goin
2026-06-10 06:06:17 -07:00
committed by GitHub
co-authored by Claude
parent 4882fd7632
commit 87c15d46e3
5 changed files with 86 additions and 52 deletions
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any
import regex as re
import torch
import vllm.utils.humming as _hm
from vllm import envs
from vllm.model_executor.layers.fused_moe import (
FusedMoEConfig,
@@ -41,37 +42,16 @@ from vllm.model_executor.parameter import (
RowvLLMParameter,
)
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_humming
if has_humming() and current_platform.is_cuda():
from humming.dtypes import DataType
from humming.layer import HummingMethod
from humming.schema import (
BaseInputSchema,
BaseWeightSchema,
HummingInputSchema,
HummingWeightSchema,
)
from humming.utils.weight import quantize_weight
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
BatchedHummingGroupedExperts,
HummingGroupedExperts,
HummingIndexedExperts,
get_humming_moe_gemm_type,
)
if TYPE_CHECKING:
from humming.schema import (
from vllm.model_executor.models.utils import WeightsMapper
from vllm.utils.humming import (
BaseInputSchema,
BaseWeightSchema,
HummingInputSchema,
HummingWeightSchema,
)
from vllm.model_executor.models.utils import WeightsMapper
def prepare_padded_shape(shape, x):
padded_shape = math.ceil(shape / x) * x
@@ -265,7 +245,7 @@ class HummingConfig(QuantizationConfig):
break
if "quant_method" in layer_config:
return BaseWeightSchema.from_config(layer_config)
return _hm.BaseWeightSchema.from_config(layer_config)
return None
def get_layer_input_schema(self, config: dict[str, Any], prefix: str):
@@ -277,8 +257,8 @@ class HummingConfig(QuantizationConfig):
return None
config = group_config
if config.get("quant_method", None) in BaseInputSchema.INPUT_SCHEMA_MAP:
return BaseInputSchema.from_config(config)
if config.get("quant_method", None) in _hm.BaseInputSchema.INPUT_SCHEMA_MAP:
return _hm.BaseInputSchema.from_config(config)
return None
def get_quant_config_for_layer(
@@ -316,7 +296,7 @@ class HummingConfig(QuantizationConfig):
input_schema = force_input_schema
if force_weight_schema is not None and force_input_schema is None:
force_input_schema = HummingInputSchema()
force_input_schema = _hm.HummingInputSchema()
return HummingLayerQuantizationConfig(
weight_schema=weight_schema,
@@ -360,7 +340,7 @@ class HummingLayerQuantizationConfig(HummingConfig):
):
self.weight_schema = weight_schema
if input_schema is None:
input_schema = HummingInputSchema()
input_schema = _hm.HummingInputSchema()
self.input_schema = input_schema
self.force_weight_schema = force_weight_schema
self.force_input_schema = force_input_schema
@@ -368,7 +348,7 @@ class HummingLayerQuantizationConfig(HummingConfig):
@classmethod
def from_config(cls, config):
weight_schema = BaseWeightSchema.from_config(config)
weight_schema = _hm.BaseWeightSchema.from_config(config)
return cls(weight_schema)
def get_quant_method(
@@ -397,10 +377,10 @@ class HummingLinearMethod(LinearMethodBase):
is_unquantized = name == "weight" and loaded_weight.dtype in float_dtypes
if is_unquantized and self.is_online_quant:
# online quant (fp16/bf16 -> quant_type)
assert isinstance(self.weight_schema, HummingWeightSchema)
f16_dtype = DataType.from_torch_dtype(layer.param_dtype)
assert isinstance(self.weight_schema, _hm.HummingWeightSchema)
f16_dtype = _hm.DataType.from_torch_dtype(layer.param_dtype)
has_global_scale = "TENSOR" in str(self.weight_schema.weight_scale_type)
tensor_list = quantize_weight(
tensor_list = _hm.quantize_weight(
weight=loaded_weight,
dtype=self.weight_schema.b_dtype,
scale_dtype=self.weight_schema.bs_dtype or f16_dtype,
@@ -531,7 +511,7 @@ class HummingLinearMethod(LinearMethodBase):
return None
# convert from checkpoint format to humming format
if not isinstance(self.weight_schema, HummingWeightSchema):
if not isinstance(self.weight_schema, _hm.HummingWeightSchema):
self.weight_schema, tensors = self.weight_schema.convert_humming(
tensors=layer.state_dict(),
shape_n_stacks=layer.output_partition_sizes,
@@ -556,7 +536,7 @@ class HummingLinearMethod(LinearMethodBase):
del tensors
# force requant (origin quant setting -> fp16/bf16 -> new_quant setting)
assert isinstance(self.weight_schema, HummingWeightSchema)
assert isinstance(self.weight_schema, _hm.HummingWeightSchema)
force_requant = self.force_weight_schema is not None
if force_requant and self.weight_schema != self.force_weight_schema:
tensors = self.weight_schema.requant_tensors(
@@ -578,7 +558,7 @@ class HummingLinearMethod(LinearMethodBase):
del tensors
# prepare layer config from humming kernel
HummingMethod.prepare_layer_meta(
_hm.HummingMethod.prepare_layer_meta(
layer=layer,
shape_n=layer.output_partition_sizes_sum,
shape_k=layer.input_size_per_partition,
@@ -591,7 +571,7 @@ class HummingLinearMethod(LinearMethodBase):
)
# preprocess weight for inference
HummingMethod.transform_humming_layer(layer)
_hm.HummingMethod.transform_humming_layer(layer)
# compute_config: kernel configs that do not directly affect weights
# but significantly impact kernel behavior or computation precision.
@@ -610,7 +590,7 @@ class HummingLinearMethod(LinearMethodBase):
bias: torch.Tensor | None = None,
) -> torch.Tensor:
flatten_inputs = x.view(-1, x.size(-1))
output = HummingMethod.forward_layer(
output = _hm.HummingMethod.forward_layer(
layer=layer,
inputs=flatten_inputs,
compute_config=self.compute_config,
@@ -645,10 +625,10 @@ class HummingMoEMethod(FusedMoEMethodBase):
is_unquantized = name == "weight" and loaded_weight.dtype in float_dtypes
# online quant (fp16/bf16 -> quant_type)
if is_unquantized:
assert isinstance(self.weight_schema, HummingWeightSchema)
f16_dtype = DataType.from_torch_dtype(layer.param_dtype)
assert isinstance(self.weight_schema, _hm.HummingWeightSchema)
f16_dtype = _hm.DataType.from_torch_dtype(layer.param_dtype)
has_global_scale = "TENSOR" in str(self.weight_schema.weight_scale_type)
tensor_list = quantize_weight(
tensor_list = _hm.quantize_weight(
weight=loaded_weight,
dtype=self.weight_schema.b_dtype,
scale_dtype=self.weight_schema.bs_dtype or f16_dtype,
@@ -771,7 +751,7 @@ class HummingMoEMethod(FusedMoEMethodBase):
input_schema = self.input_schema
weight_schema = self.weight_schema
# convert from checkpoint format to humming format
if not isinstance(weight_schema, HummingWeightSchema):
if not isinstance(weight_schema, _hm.HummingWeightSchema):
tensors: dict[str, torch.Tensor] = dict(
(key.removeprefix(sublayer_name + "_"), value)
for key, value in layer.state_dict().items()
@@ -813,7 +793,7 @@ class HummingMoEMethod(FusedMoEMethodBase):
layer.input_schemas[sublayer_name] = input_schema
# force requant (origin quant setting -> fp16/bf16 -> new_quant setting)
assert isinstance(weight_schema, HummingWeightSchema)
assert isinstance(weight_schema, _hm.HummingWeightSchema)
force_requant = self.force_weight_schema is not None
if force_requant and weight_schema != self.force_weight_schema:
tensors = dict(
@@ -845,7 +825,7 @@ class HummingMoEMethod(FusedMoEMethodBase):
del tensors
# prepare layer config from humming kernel
HummingMethod.prepare_layer_meta(
_hm.HummingMethod.prepare_layer_meta(
layer=layer,
shape_n=configs["shape_n"],
shape_k=configs["shape_k"],
@@ -860,7 +840,15 @@ class HummingMoEMethod(FusedMoEMethodBase):
)
# preprocess weight for inference
HummingMethod.transform_humming_layer(layer, sublayer_name=sublayer_name)
_hm.HummingMethod.transform_humming_layer(
layer, sublayer_name=sublayer_name
)
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
HummingGroupedExperts,
HummingIndexedExperts,
get_humming_moe_gemm_type,
)
# use moe modular
experts: HummingIndexedExperts | HummingGroupedExperts
@@ -878,6 +866,12 @@ class HummingMoEMethod(FusedMoEMethodBase):
layer: torch.nn.Module,
):
from vllm.model_executor.layers.fused_moe import modular_kernel as mk
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
BatchedHummingGroupedExperts,
HummingGroupedExperts,
HummingIndexedExperts,
get_humming_moe_gemm_type,
)
activation_format = prepare_finalize.activation_format
assert self.moe_quant_config is not None
@@ -5,8 +5,6 @@ from typing import Any
import regex as re
import torch
from humming.layer import HummingInputSchema, HummingMethod
from humming.schema import BaseWeightSchema
from vllm import envs
from vllm.model_executor.layers.fused_moe import RoutedExperts
@@ -16,6 +14,7 @@ from vllm.model_executor.layers.fused_moe.config import (
)
from vllm.model_executor.layers.linear import LinearBase
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
from vllm.utils.humming import BaseWeightSchema, HummingInputSchema, HummingMethod
def humming_is_layer_skipped(config: dict[str, Any], prefix: str):