[Bugfix] Lazily import the humming quantization backend (#44921)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any
|
||||
import regex as re
|
||||
import torch
|
||||
|
||||
import vllm.utils.humming as _hm
|
||||
from vllm import envs
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEConfig,
|
||||
@@ -41,37 +42,16 @@ from vllm.model_executor.parameter import (
|
||||
RowvLLMParameter,
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
if has_humming() and current_platform.is_cuda():
|
||||
from humming.dtypes import DataType
|
||||
from humming.layer import HummingMethod
|
||||
from humming.schema import (
|
||||
BaseInputSchema,
|
||||
BaseWeightSchema,
|
||||
HummingInputSchema,
|
||||
HummingWeightSchema,
|
||||
)
|
||||
from humming.utils.weight import quantize_weight
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
BatchedHummingGroupedExperts,
|
||||
HummingGroupedExperts,
|
||||
HummingIndexedExperts,
|
||||
get_humming_moe_gemm_type,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from humming.schema import (
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
from vllm.utils.humming import (
|
||||
BaseInputSchema,
|
||||
BaseWeightSchema,
|
||||
HummingInputSchema,
|
||||
HummingWeightSchema,
|
||||
)
|
||||
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
|
||||
|
||||
def prepare_padded_shape(shape, x):
|
||||
padded_shape = math.ceil(shape / x) * x
|
||||
@@ -265,7 +245,7 @@ class HummingConfig(QuantizationConfig):
|
||||
break
|
||||
|
||||
if "quant_method" in layer_config:
|
||||
return BaseWeightSchema.from_config(layer_config)
|
||||
return _hm.BaseWeightSchema.from_config(layer_config)
|
||||
return None
|
||||
|
||||
def get_layer_input_schema(self, config: dict[str, Any], prefix: str):
|
||||
@@ -277,8 +257,8 @@ class HummingConfig(QuantizationConfig):
|
||||
return None
|
||||
config = group_config
|
||||
|
||||
if config.get("quant_method", None) in BaseInputSchema.INPUT_SCHEMA_MAP:
|
||||
return BaseInputSchema.from_config(config)
|
||||
if config.get("quant_method", None) in _hm.BaseInputSchema.INPUT_SCHEMA_MAP:
|
||||
return _hm.BaseInputSchema.from_config(config)
|
||||
return None
|
||||
|
||||
def get_quant_config_for_layer(
|
||||
@@ -316,7 +296,7 @@ class HummingConfig(QuantizationConfig):
|
||||
input_schema = force_input_schema
|
||||
|
||||
if force_weight_schema is not None and force_input_schema is None:
|
||||
force_input_schema = HummingInputSchema()
|
||||
force_input_schema = _hm.HummingInputSchema()
|
||||
|
||||
return HummingLayerQuantizationConfig(
|
||||
weight_schema=weight_schema,
|
||||
@@ -360,7 +340,7 @@ class HummingLayerQuantizationConfig(HummingConfig):
|
||||
):
|
||||
self.weight_schema = weight_schema
|
||||
if input_schema is None:
|
||||
input_schema = HummingInputSchema()
|
||||
input_schema = _hm.HummingInputSchema()
|
||||
self.input_schema = input_schema
|
||||
self.force_weight_schema = force_weight_schema
|
||||
self.force_input_schema = force_input_schema
|
||||
@@ -368,7 +348,7 @@ class HummingLayerQuantizationConfig(HummingConfig):
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config):
|
||||
weight_schema = BaseWeightSchema.from_config(config)
|
||||
weight_schema = _hm.BaseWeightSchema.from_config(config)
|
||||
return cls(weight_schema)
|
||||
|
||||
def get_quant_method(
|
||||
@@ -397,10 +377,10 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
is_unquantized = name == "weight" and loaded_weight.dtype in float_dtypes
|
||||
if is_unquantized and self.is_online_quant:
|
||||
# online quant (fp16/bf16 -> quant_type)
|
||||
assert isinstance(self.weight_schema, HummingWeightSchema)
|
||||
f16_dtype = DataType.from_torch_dtype(layer.param_dtype)
|
||||
assert isinstance(self.weight_schema, _hm.HummingWeightSchema)
|
||||
f16_dtype = _hm.DataType.from_torch_dtype(layer.param_dtype)
|
||||
has_global_scale = "TENSOR" in str(self.weight_schema.weight_scale_type)
|
||||
tensor_list = quantize_weight(
|
||||
tensor_list = _hm.quantize_weight(
|
||||
weight=loaded_weight,
|
||||
dtype=self.weight_schema.b_dtype,
|
||||
scale_dtype=self.weight_schema.bs_dtype or f16_dtype,
|
||||
@@ -531,7 +511,7 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
return None
|
||||
|
||||
# convert from checkpoint format to humming format
|
||||
if not isinstance(self.weight_schema, HummingWeightSchema):
|
||||
if not isinstance(self.weight_schema, _hm.HummingWeightSchema):
|
||||
self.weight_schema, tensors = self.weight_schema.convert_humming(
|
||||
tensors=layer.state_dict(),
|
||||
shape_n_stacks=layer.output_partition_sizes,
|
||||
@@ -556,7 +536,7 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
del tensors
|
||||
|
||||
# force requant (origin quant setting -> fp16/bf16 -> new_quant setting)
|
||||
assert isinstance(self.weight_schema, HummingWeightSchema)
|
||||
assert isinstance(self.weight_schema, _hm.HummingWeightSchema)
|
||||
force_requant = self.force_weight_schema is not None
|
||||
if force_requant and self.weight_schema != self.force_weight_schema:
|
||||
tensors = self.weight_schema.requant_tensors(
|
||||
@@ -578,7 +558,7 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
del tensors
|
||||
|
||||
# prepare layer config from humming kernel
|
||||
HummingMethod.prepare_layer_meta(
|
||||
_hm.HummingMethod.prepare_layer_meta(
|
||||
layer=layer,
|
||||
shape_n=layer.output_partition_sizes_sum,
|
||||
shape_k=layer.input_size_per_partition,
|
||||
@@ -591,7 +571,7 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
)
|
||||
|
||||
# preprocess weight for inference
|
||||
HummingMethod.transform_humming_layer(layer)
|
||||
_hm.HummingMethod.transform_humming_layer(layer)
|
||||
|
||||
# compute_config: kernel configs that do not directly affect weights
|
||||
# but significantly impact kernel behavior or computation precision.
|
||||
@@ -610,7 +590,7 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
flatten_inputs = x.view(-1, x.size(-1))
|
||||
output = HummingMethod.forward_layer(
|
||||
output = _hm.HummingMethod.forward_layer(
|
||||
layer=layer,
|
||||
inputs=flatten_inputs,
|
||||
compute_config=self.compute_config,
|
||||
@@ -645,10 +625,10 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
is_unquantized = name == "weight" and loaded_weight.dtype in float_dtypes
|
||||
# online quant (fp16/bf16 -> quant_type)
|
||||
if is_unquantized:
|
||||
assert isinstance(self.weight_schema, HummingWeightSchema)
|
||||
f16_dtype = DataType.from_torch_dtype(layer.param_dtype)
|
||||
assert isinstance(self.weight_schema, _hm.HummingWeightSchema)
|
||||
f16_dtype = _hm.DataType.from_torch_dtype(layer.param_dtype)
|
||||
has_global_scale = "TENSOR" in str(self.weight_schema.weight_scale_type)
|
||||
tensor_list = quantize_weight(
|
||||
tensor_list = _hm.quantize_weight(
|
||||
weight=loaded_weight,
|
||||
dtype=self.weight_schema.b_dtype,
|
||||
scale_dtype=self.weight_schema.bs_dtype or f16_dtype,
|
||||
@@ -771,7 +751,7 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
input_schema = self.input_schema
|
||||
weight_schema = self.weight_schema
|
||||
# convert from checkpoint format to humming format
|
||||
if not isinstance(weight_schema, HummingWeightSchema):
|
||||
if not isinstance(weight_schema, _hm.HummingWeightSchema):
|
||||
tensors: dict[str, torch.Tensor] = dict(
|
||||
(key.removeprefix(sublayer_name + "_"), value)
|
||||
for key, value in layer.state_dict().items()
|
||||
@@ -813,7 +793,7 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
layer.input_schemas[sublayer_name] = input_schema
|
||||
|
||||
# force requant (origin quant setting -> fp16/bf16 -> new_quant setting)
|
||||
assert isinstance(weight_schema, HummingWeightSchema)
|
||||
assert isinstance(weight_schema, _hm.HummingWeightSchema)
|
||||
force_requant = self.force_weight_schema is not None
|
||||
if force_requant and weight_schema != self.force_weight_schema:
|
||||
tensors = dict(
|
||||
@@ -845,7 +825,7 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
del tensors
|
||||
|
||||
# prepare layer config from humming kernel
|
||||
HummingMethod.prepare_layer_meta(
|
||||
_hm.HummingMethod.prepare_layer_meta(
|
||||
layer=layer,
|
||||
shape_n=configs["shape_n"],
|
||||
shape_k=configs["shape_k"],
|
||||
@@ -860,7 +840,15 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
|
||||
# preprocess weight for inference
|
||||
HummingMethod.transform_humming_layer(layer, sublayer_name=sublayer_name)
|
||||
_hm.HummingMethod.transform_humming_layer(
|
||||
layer, sublayer_name=sublayer_name
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
HummingGroupedExperts,
|
||||
HummingIndexedExperts,
|
||||
get_humming_moe_gemm_type,
|
||||
)
|
||||
|
||||
# use moe modular
|
||||
experts: HummingIndexedExperts | HummingGroupedExperts
|
||||
@@ -878,6 +866,12 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
layer: torch.nn.Module,
|
||||
):
|
||||
from vllm.model_executor.layers.fused_moe import modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
BatchedHummingGroupedExperts,
|
||||
HummingGroupedExperts,
|
||||
HummingIndexedExperts,
|
||||
get_humming_moe_gemm_type,
|
||||
)
|
||||
|
||||
activation_format = prepare_finalize.activation_format
|
||||
assert self.moe_quant_config is not None
|
||||
|
||||
@@ -5,8 +5,6 @@ from typing import Any
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
from humming.layer import HummingInputSchema, HummingMethod
|
||||
from humming.schema import BaseWeightSchema
|
||||
|
||||
from vllm import envs
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
@@ -16,6 +14,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
)
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||
from vllm.utils.humming import BaseWeightSchema, HummingInputSchema, HummingMethod
|
||||
|
||||
|
||||
def humming_is_layer_skipped(config: dict[str, Any], prefix: str):
|
||||
|
||||
Reference in New Issue
Block a user