[Bugfix] Lazily import the humming quantization backend (#44921)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -5,7 +5,7 @@
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import _has_module
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
|
||||
|
||||
@@ -19,7 +19,7 @@ class HummingLinearKernel(MPLinearKernel):
|
||||
def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_cuda():
|
||||
return False, "Humming is only supported on CUDA"
|
||||
if not _has_module("humming"):
|
||||
if not has_humming():
|
||||
return False, "Humming is not installed"
|
||||
if c.has_g_idx:
|
||||
return False, "Humming does not support act-order (g_idx)"
|
||||
@@ -50,7 +50,7 @@ class HummingLinearKernel(MPLinearKernel):
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
from humming.layer import HummingMethod
|
||||
from vllm.utils.humming import HummingMethod
|
||||
|
||||
flatten_inputs = x.view(-1, x.size(-1))
|
||||
output = HummingMethod.forward_layer(
|
||||
|
||||
@@ -7,9 +7,6 @@ import math
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
from humming import dtypes
|
||||
from humming.config import GemmType as HummingGemmType
|
||||
from humming.layer import HummingLayerMeta, HummingMethod
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm import envs
|
||||
@@ -41,6 +38,8 @@ from vllm.model_executor.layers.fused_moe.utils import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.humming import GemmType as HummingGemmType
|
||||
from vllm.utils.humming import HummingLayerMeta, HummingMethod, dtypes
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
||||
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any
|
||||
import regex as re
|
||||
import torch
|
||||
|
||||
import vllm.utils.humming as _hm
|
||||
from vllm import envs
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEConfig,
|
||||
@@ -41,37 +42,16 @@ from vllm.model_executor.parameter import (
|
||||
RowvLLMParameter,
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
if has_humming() and current_platform.is_cuda():
|
||||
from humming.dtypes import DataType
|
||||
from humming.layer import HummingMethod
|
||||
from humming.schema import (
|
||||
BaseInputSchema,
|
||||
BaseWeightSchema,
|
||||
HummingInputSchema,
|
||||
HummingWeightSchema,
|
||||
)
|
||||
from humming.utils.weight import quantize_weight
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
BatchedHummingGroupedExperts,
|
||||
HummingGroupedExperts,
|
||||
HummingIndexedExperts,
|
||||
get_humming_moe_gemm_type,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from humming.schema import (
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
from vllm.utils.humming import (
|
||||
BaseInputSchema,
|
||||
BaseWeightSchema,
|
||||
HummingInputSchema,
|
||||
HummingWeightSchema,
|
||||
)
|
||||
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
|
||||
|
||||
def prepare_padded_shape(shape, x):
|
||||
padded_shape = math.ceil(shape / x) * x
|
||||
@@ -265,7 +245,7 @@ class HummingConfig(QuantizationConfig):
|
||||
break
|
||||
|
||||
if "quant_method" in layer_config:
|
||||
return BaseWeightSchema.from_config(layer_config)
|
||||
return _hm.BaseWeightSchema.from_config(layer_config)
|
||||
return None
|
||||
|
||||
def get_layer_input_schema(self, config: dict[str, Any], prefix: str):
|
||||
@@ -277,8 +257,8 @@ class HummingConfig(QuantizationConfig):
|
||||
return None
|
||||
config = group_config
|
||||
|
||||
if config.get("quant_method", None) in BaseInputSchema.INPUT_SCHEMA_MAP:
|
||||
return BaseInputSchema.from_config(config)
|
||||
if config.get("quant_method", None) in _hm.BaseInputSchema.INPUT_SCHEMA_MAP:
|
||||
return _hm.BaseInputSchema.from_config(config)
|
||||
return None
|
||||
|
||||
def get_quant_config_for_layer(
|
||||
@@ -316,7 +296,7 @@ class HummingConfig(QuantizationConfig):
|
||||
input_schema = force_input_schema
|
||||
|
||||
if force_weight_schema is not None and force_input_schema is None:
|
||||
force_input_schema = HummingInputSchema()
|
||||
force_input_schema = _hm.HummingInputSchema()
|
||||
|
||||
return HummingLayerQuantizationConfig(
|
||||
weight_schema=weight_schema,
|
||||
@@ -360,7 +340,7 @@ class HummingLayerQuantizationConfig(HummingConfig):
|
||||
):
|
||||
self.weight_schema = weight_schema
|
||||
if input_schema is None:
|
||||
input_schema = HummingInputSchema()
|
||||
input_schema = _hm.HummingInputSchema()
|
||||
self.input_schema = input_schema
|
||||
self.force_weight_schema = force_weight_schema
|
||||
self.force_input_schema = force_input_schema
|
||||
@@ -368,7 +348,7 @@ class HummingLayerQuantizationConfig(HummingConfig):
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config):
|
||||
weight_schema = BaseWeightSchema.from_config(config)
|
||||
weight_schema = _hm.BaseWeightSchema.from_config(config)
|
||||
return cls(weight_schema)
|
||||
|
||||
def get_quant_method(
|
||||
@@ -397,10 +377,10 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
is_unquantized = name == "weight" and loaded_weight.dtype in float_dtypes
|
||||
if is_unquantized and self.is_online_quant:
|
||||
# online quant (fp16/bf16 -> quant_type)
|
||||
assert isinstance(self.weight_schema, HummingWeightSchema)
|
||||
f16_dtype = DataType.from_torch_dtype(layer.param_dtype)
|
||||
assert isinstance(self.weight_schema, _hm.HummingWeightSchema)
|
||||
f16_dtype = _hm.DataType.from_torch_dtype(layer.param_dtype)
|
||||
has_global_scale = "TENSOR" in str(self.weight_schema.weight_scale_type)
|
||||
tensor_list = quantize_weight(
|
||||
tensor_list = _hm.quantize_weight(
|
||||
weight=loaded_weight,
|
||||
dtype=self.weight_schema.b_dtype,
|
||||
scale_dtype=self.weight_schema.bs_dtype or f16_dtype,
|
||||
@@ -531,7 +511,7 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
return None
|
||||
|
||||
# convert from checkpoint format to humming format
|
||||
if not isinstance(self.weight_schema, HummingWeightSchema):
|
||||
if not isinstance(self.weight_schema, _hm.HummingWeightSchema):
|
||||
self.weight_schema, tensors = self.weight_schema.convert_humming(
|
||||
tensors=layer.state_dict(),
|
||||
shape_n_stacks=layer.output_partition_sizes,
|
||||
@@ -556,7 +536,7 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
del tensors
|
||||
|
||||
# force requant (origin quant setting -> fp16/bf16 -> new_quant setting)
|
||||
assert isinstance(self.weight_schema, HummingWeightSchema)
|
||||
assert isinstance(self.weight_schema, _hm.HummingWeightSchema)
|
||||
force_requant = self.force_weight_schema is not None
|
||||
if force_requant and self.weight_schema != self.force_weight_schema:
|
||||
tensors = self.weight_schema.requant_tensors(
|
||||
@@ -578,7 +558,7 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
del tensors
|
||||
|
||||
# prepare layer config from humming kernel
|
||||
HummingMethod.prepare_layer_meta(
|
||||
_hm.HummingMethod.prepare_layer_meta(
|
||||
layer=layer,
|
||||
shape_n=layer.output_partition_sizes_sum,
|
||||
shape_k=layer.input_size_per_partition,
|
||||
@@ -591,7 +571,7 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
)
|
||||
|
||||
# preprocess weight for inference
|
||||
HummingMethod.transform_humming_layer(layer)
|
||||
_hm.HummingMethod.transform_humming_layer(layer)
|
||||
|
||||
# compute_config: kernel configs that do not directly affect weights
|
||||
# but significantly impact kernel behavior or computation precision.
|
||||
@@ -610,7 +590,7 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
flatten_inputs = x.view(-1, x.size(-1))
|
||||
output = HummingMethod.forward_layer(
|
||||
output = _hm.HummingMethod.forward_layer(
|
||||
layer=layer,
|
||||
inputs=flatten_inputs,
|
||||
compute_config=self.compute_config,
|
||||
@@ -645,10 +625,10 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
is_unquantized = name == "weight" and loaded_weight.dtype in float_dtypes
|
||||
# online quant (fp16/bf16 -> quant_type)
|
||||
if is_unquantized:
|
||||
assert isinstance(self.weight_schema, HummingWeightSchema)
|
||||
f16_dtype = DataType.from_torch_dtype(layer.param_dtype)
|
||||
assert isinstance(self.weight_schema, _hm.HummingWeightSchema)
|
||||
f16_dtype = _hm.DataType.from_torch_dtype(layer.param_dtype)
|
||||
has_global_scale = "TENSOR" in str(self.weight_schema.weight_scale_type)
|
||||
tensor_list = quantize_weight(
|
||||
tensor_list = _hm.quantize_weight(
|
||||
weight=loaded_weight,
|
||||
dtype=self.weight_schema.b_dtype,
|
||||
scale_dtype=self.weight_schema.bs_dtype or f16_dtype,
|
||||
@@ -771,7 +751,7 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
input_schema = self.input_schema
|
||||
weight_schema = self.weight_schema
|
||||
# convert from checkpoint format to humming format
|
||||
if not isinstance(weight_schema, HummingWeightSchema):
|
||||
if not isinstance(weight_schema, _hm.HummingWeightSchema):
|
||||
tensors: dict[str, torch.Tensor] = dict(
|
||||
(key.removeprefix(sublayer_name + "_"), value)
|
||||
for key, value in layer.state_dict().items()
|
||||
@@ -813,7 +793,7 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
layer.input_schemas[sublayer_name] = input_schema
|
||||
|
||||
# force requant (origin quant setting -> fp16/bf16 -> new_quant setting)
|
||||
assert isinstance(weight_schema, HummingWeightSchema)
|
||||
assert isinstance(weight_schema, _hm.HummingWeightSchema)
|
||||
force_requant = self.force_weight_schema is not None
|
||||
if force_requant and weight_schema != self.force_weight_schema:
|
||||
tensors = dict(
|
||||
@@ -845,7 +825,7 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
del tensors
|
||||
|
||||
# prepare layer config from humming kernel
|
||||
HummingMethod.prepare_layer_meta(
|
||||
_hm.HummingMethod.prepare_layer_meta(
|
||||
layer=layer,
|
||||
shape_n=configs["shape_n"],
|
||||
shape_k=configs["shape_k"],
|
||||
@@ -860,7 +840,15 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
|
||||
# preprocess weight for inference
|
||||
HummingMethod.transform_humming_layer(layer, sublayer_name=sublayer_name)
|
||||
_hm.HummingMethod.transform_humming_layer(
|
||||
layer, sublayer_name=sublayer_name
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
HummingGroupedExperts,
|
||||
HummingIndexedExperts,
|
||||
get_humming_moe_gemm_type,
|
||||
)
|
||||
|
||||
# use moe modular
|
||||
experts: HummingIndexedExperts | HummingGroupedExperts
|
||||
@@ -878,6 +866,12 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
layer: torch.nn.Module,
|
||||
):
|
||||
from vllm.model_executor.layers.fused_moe import modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
BatchedHummingGroupedExperts,
|
||||
HummingGroupedExperts,
|
||||
HummingIndexedExperts,
|
||||
get_humming_moe_gemm_type,
|
||||
)
|
||||
|
||||
activation_format = prepare_finalize.activation_format
|
||||
assert self.moe_quant_config is not None
|
||||
|
||||
@@ -5,8 +5,6 @@ from typing import Any
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
from humming.layer import HummingInputSchema, HummingMethod
|
||||
from humming.schema import BaseWeightSchema
|
||||
|
||||
from vllm import envs
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
@@ -16,6 +14,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
)
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||
from vllm.utils.humming import BaseWeightSchema, HummingInputSchema, HummingMethod
|
||||
|
||||
|
||||
def humming_is_layer_skipped(config: dict[str, Any], prefix: str):
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Lazy facade for the optional ``humming`` package.
|
||||
|
||||
vLLM code should import humming symbols from here so that ``import humming``
|
||||
(which has import-time side effects) is deferred until first use. Add new
|
||||
symbols by appending one entry to ``_EXPORTS`` as ``"module.path:attr"``,
|
||||
or ``"module.path"`` for a whole-module re-export.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
from typing import Any
|
||||
|
||||
_EXPORTS: dict[str, str] = {
|
||||
"dtypes": "humming.dtypes",
|
||||
"DataType": "humming.dtypes:DataType",
|
||||
"GemmType": "humming.config:GemmType",
|
||||
"HummingMethod": "humming.layer:HummingMethod",
|
||||
"HummingLayerMeta": "humming.layer:HummingLayerMeta",
|
||||
"BaseInputSchema": "humming.schema:BaseInputSchema",
|
||||
"BaseWeightSchema": "humming.schema:BaseWeightSchema",
|
||||
"HummingInputSchema": "humming.schema:HummingInputSchema",
|
||||
"HummingWeightSchema": "humming.schema:HummingWeightSchema",
|
||||
"quantize_weight": "humming.utils.weight:quantize_weight",
|
||||
}
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
spec = _EXPORTS.get(name)
|
||||
if spec is None:
|
||||
raise AttributeError(f"module 'vllm.utils.humming' has no attribute {name!r}")
|
||||
if ":" in spec:
|
||||
mod_path, attr = spec.split(":", 1)
|
||||
obj = getattr(importlib.import_module(mod_path), attr)
|
||||
else:
|
||||
obj = importlib.import_module(spec)
|
||||
globals()[name] = obj
|
||||
return obj
|
||||
|
||||
|
||||
def __dir__() -> list[str]:
|
||||
return sorted({*globals(), *_EXPORTS})
|
||||
Reference in New Issue
Block a user