[Feature][Humming] Humming feature integration (#56685)
Signed-off-by: jinzhen.ljz <jinzhen.ljz@antgroup.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Codex
mgoin
parent
27757dde02
commit
7d99c2c4fd
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
from math import gcd
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import regex as re
|
||||
@@ -28,12 +29,15 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
from vllm.model_executor.layers.quantization.utils.humming import (
|
||||
check_and_fallback_input_schema,
|
||||
convert_to_humming_moe_kernel_format,
|
||||
get_humming_linear_compute_config,
|
||||
get_humming_moe_quant_config,
|
||||
humming_update_schema_hadamard_block_size,
|
||||
input_schema_to_quant_key,
|
||||
make_humming_moe_kernel,
|
||||
resolve_humming_layer_config,
|
||||
select_humming_moe_experts,
|
||||
weight_schema_to_quant_key,
|
||||
)
|
||||
@@ -227,23 +231,13 @@ class HummingConfig(QuantizationConfig):
|
||||
return None
|
||||
config = group_config
|
||||
|
||||
layer_config = config
|
||||
layer_dynamic = config.get("dynamic", {})
|
||||
if not isinstance(layer_dynamic, dict):
|
||||
layer_dynamic = {}
|
||||
for regex, override_config in layer_dynamic.items():
|
||||
if regex[:1] != "+":
|
||||
continue
|
||||
if re.match(regex[2:], prefix):
|
||||
layer_config = config.copy()
|
||||
layer_config.update(override_config)
|
||||
break
|
||||
layer_config = resolve_humming_layer_config(config, prefix)
|
||||
|
||||
if "quant_method" in layer_config:
|
||||
return _hm.BaseWeightSchema.from_config(layer_config)
|
||||
return None
|
||||
|
||||
def get_layer_input_schema(self, config: dict[str, Any], prefix: str):
|
||||
def get_layer_input_config(self, config: dict[str, Any], prefix: str):
|
||||
if self.is_layer_skipped(config, prefix):
|
||||
return None
|
||||
if config["quant_method"] in ["compressed-tensors", "modelopt"]:
|
||||
@@ -252,10 +246,15 @@ class HummingConfig(QuantizationConfig):
|
||||
return None
|
||||
config = group_config
|
||||
|
||||
config = resolve_humming_layer_config(config, prefix)
|
||||
if config.get("quant_method", None) in _hm.BaseInputSchema.INPUT_SCHEMA_MAP:
|
||||
return _hm.BaseInputSchema.from_config(config)
|
||||
return config
|
||||
return None
|
||||
|
||||
def get_layer_input_schema(self, config: dict[str, Any], prefix: str):
|
||||
config = self.get_layer_input_config(config, prefix)
|
||||
return None if config is None else _hm.BaseInputSchema.from_config(config)
|
||||
|
||||
def get_quant_config_for_layer(
|
||||
self, prefix: str, layer_type: str
|
||||
) -> "HummingLayerQuantizationConfig | None":
|
||||
@@ -279,14 +278,25 @@ class HummingConfig(QuantizationConfig):
|
||||
if weight_schema is not None:
|
||||
input_schema = None
|
||||
force_input_schema = None
|
||||
allow_input_schema_fallback = True
|
||||
|
||||
if self.full_config:
|
||||
input_schema = self.get_layer_input_schema(self.full_config, prefix)
|
||||
input_config = self.get_layer_input_config(self.full_config, prefix)
|
||||
if input_config is not None:
|
||||
allow_input_schema_fallback = input_config.pop(
|
||||
"allow_fallback", True
|
||||
)
|
||||
input_schema = _hm.BaseInputSchema.from_config(input_config)
|
||||
|
||||
if envs.VLLM_HUMMING_INPUT_QUANT_CONFIG:
|
||||
quant_config = envs.VLLM_HUMMING_INPUT_QUANT_CONFIG.copy()
|
||||
quant_config["quant_method"] = "humming"
|
||||
force_input_schema = self.get_layer_input_schema(quant_config, prefix)
|
||||
input_config = self.get_layer_input_config(quant_config, prefix)
|
||||
if input_config is not None:
|
||||
allow_input_schema_fallback = input_config.pop(
|
||||
"allow_fallback", False
|
||||
)
|
||||
force_input_schema = _hm.BaseInputSchema.from_config(input_config)
|
||||
if input_schema is None:
|
||||
input_schema = force_input_schema
|
||||
|
||||
@@ -299,6 +309,7 @@ class HummingConfig(QuantizationConfig):
|
||||
force_weight_schema=force_weight_schema,
|
||||
force_input_schema=force_input_schema,
|
||||
is_online_quant=is_online_quant,
|
||||
allow_input_schema_fallback=allow_input_schema_fallback,
|
||||
)
|
||||
return None
|
||||
|
||||
@@ -318,6 +329,11 @@ class HummingConfig(QuantizationConfig):
|
||||
elif isinstance(layer, LinearBase):
|
||||
return UnquantizedLinearMethod()
|
||||
elif isinstance(layer, LinearBase):
|
||||
input_size = layer.input_size
|
||||
partition_size = getattr(layer, "input_size_per_partition", input_size)
|
||||
is_online_quant = quant_config.is_online_quant
|
||||
if is_online_quant and (input_size % 32 != 0 or partition_size % 32 != 0):
|
||||
return UnquantizedLinearMethod()
|
||||
return HummingLinearMethod(quant_config)
|
||||
elif isinstance(layer, RoutedExperts):
|
||||
return HummingMoEMethod(quant_config, layer.moe_config)
|
||||
@@ -332,6 +348,7 @@ class HummingLayerQuantizationConfig(HummingConfig):
|
||||
force_weight_schema: "HummingWeightSchema | None" = None,
|
||||
force_input_schema: "HummingInputSchema | None" = None,
|
||||
is_online_quant: bool = False,
|
||||
allow_input_schema_fallback: bool = True,
|
||||
):
|
||||
self.weight_schema = weight_schema
|
||||
if input_schema is None:
|
||||
@@ -340,6 +357,7 @@ class HummingLayerQuantizationConfig(HummingConfig):
|
||||
self.force_weight_schema = force_weight_schema
|
||||
self.force_input_schema = force_input_schema
|
||||
self.is_online_quant = is_online_quant
|
||||
self.allow_input_schema_fallback = allow_input_schema_fallback
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config):
|
||||
@@ -373,21 +391,13 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
if is_unquantized and self.is_online_quant:
|
||||
# online quant (fp16/bf16 -> quant_type)
|
||||
assert isinstance(self.weight_schema, _hm.HummingWeightSchema)
|
||||
f16_dtype = _hm.DataType.from_torch_dtype(layer.param_dtype)
|
||||
has_global_scale = "TENSOR" in str(self.weight_schema.weight_scale_type)
|
||||
tensor_list = _hm.quantize_weight(
|
||||
weight=loaded_weight,
|
||||
dtype=self.weight_schema.b_dtype,
|
||||
scale_dtype=self.weight_schema.bs_dtype or f16_dtype,
|
||||
group_size=self.weight_schema.weight_scale_group_size,
|
||||
has_zero_point=self.weight_schema.has_zero_point,
|
||||
has_global_scale=has_global_scale,
|
||||
is_fp_zero_point=self.weight_schema.is_fp_zero_point,
|
||||
pack=True,
|
||||
tensors = self.weight_schema.quant_tensor(
|
||||
loaded_weight.to(param.device),
|
||||
self.weight_schema,
|
||||
layer.param_dtype,
|
||||
)
|
||||
|
||||
key_list = ["weight", "weight_scale", "zero_point", "global_scale"]
|
||||
for key, tensor in zip(key_list, tensor_list):
|
||||
for key, tensor in tensors.items():
|
||||
if tensor is None or tensor.nelement() == 0:
|
||||
continue
|
||||
param = getattr(layer, key)
|
||||
@@ -459,6 +469,18 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
layer.output_partition_sizes = output_partition_sizes
|
||||
layer.extra_weight_attrs = extra_weight_attrs.copy()
|
||||
|
||||
input_schema = self.force_input_schema or self.input_schema
|
||||
input_schema = input_schema.to_humming_schema(params_dtype)
|
||||
for name in ("weight_schema", "force_weight_schema"):
|
||||
schema = getattr(self, name)
|
||||
if getattr(schema, "hadamard_block_size", None) == -1:
|
||||
schema = humming_update_schema_hadamard_block_size(
|
||||
weight_schema=schema,
|
||||
input_schema=input_schema,
|
||||
shape_k=input_size_per_partition,
|
||||
)
|
||||
setattr(self, name, schema)
|
||||
|
||||
weight_loader = extra_weight_attrs.get("weight_loader", default_weight_loader)
|
||||
new_weight_loader = self.prepare_weight_loader(layer, weight_loader)
|
||||
extra_weight_attrs["weight_loader"] = new_weight_loader
|
||||
@@ -502,7 +524,10 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
return None
|
||||
|
||||
# convert from checkpoint format to humming format
|
||||
if not isinstance(self.weight_schema, _hm.HummingWeightSchema):
|
||||
is_humming_weight = isinstance(self.weight_schema, _hm.HummingWeightSchema)
|
||||
is_humming_input = isinstance(self.input_schema, _hm.HummingInputSchema)
|
||||
|
||||
if not is_humming_weight or not is_humming_input:
|
||||
self.weight_schema, tensors = self.weight_schema.convert_humming(
|
||||
tensors=layer.state_dict(),
|
||||
shape_n_stacks=layer.output_partition_sizes,
|
||||
@@ -510,12 +535,13 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
param_dtype=layer.param_dtype,
|
||||
)
|
||||
|
||||
self.input_schema, _ = self.input_schema.convert_humming(
|
||||
self.input_schema, input_tensors = self.input_schema.convert_humming(
|
||||
tensors=layer.state_dict(),
|
||||
shape_n_stacks=layer.output_partition_sizes,
|
||||
shape_k_stacks=[layer.input_size_per_partition],
|
||||
param_dtype=layer.param_dtype,
|
||||
)
|
||||
tensors.update(input_tensors)
|
||||
|
||||
for name, _ in list(layer.named_parameters()):
|
||||
delattr(layer, name)
|
||||
@@ -530,11 +556,15 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
assert isinstance(self.weight_schema, _hm.HummingWeightSchema)
|
||||
force_requant = self.force_weight_schema is not None
|
||||
if force_requant and self.weight_schema != self.force_weight_schema:
|
||||
source_tensors = layer.state_dict()
|
||||
tensors = self.weight_schema.requant_tensors(
|
||||
tensors=layer.state_dict(),
|
||||
tensors=source_tensors,
|
||||
target_weight_schema=self.force_weight_schema,
|
||||
param_dtype=layer.param_dtype,
|
||||
)
|
||||
name = self.input_schema.static_tensor_scale_name
|
||||
if name is not None:
|
||||
tensors[name] = source_tensors[name]
|
||||
|
||||
self.weight_schema = self.force_weight_schema
|
||||
|
||||
@@ -548,6 +578,12 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
|
||||
del tensors
|
||||
|
||||
self.input_schema = check_and_fallback_input_schema(
|
||||
weight_schema=self.weight_schema,
|
||||
input_schema=self.input_schema,
|
||||
param_dtype=layer.param_dtype,
|
||||
allow_fallback=self.quant_config.allow_input_schema_fallback,
|
||||
)
|
||||
self.layer_config = _hm.prepare_layer_config(
|
||||
shape_n=layer.output_partition_sizes_sum,
|
||||
shape_k=layer.input_size_per_partition,
|
||||
@@ -557,10 +593,16 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
pad_k_to_multiple=128,
|
||||
has_bias=layer.has_bias,
|
||||
torch_dtype=layer.param_dtype,
|
||||
device=layer.weight.device,
|
||||
)
|
||||
tensors = _hm.transform_humming_tensors(
|
||||
self.layer_config, dict(layer.named_parameters())
|
||||
)
|
||||
source_tensors = dict(layer.named_parameters())
|
||||
tensors = _hm.transform_humming_tensors(self.layer_config, source_tensors)
|
||||
name = self.input_schema.static_tensor_scale_name
|
||||
if name is not None:
|
||||
tensors[name] = source_tensors[name]
|
||||
if "bias" in tensors:
|
||||
zero_bias = torch.zeros_like(tensors["bias"])
|
||||
layer.register_buffer("zero_bias", zero_bias)
|
||||
for name, _ in list(layer.named_parameters()):
|
||||
delattr(layer, name)
|
||||
for name, tensor in tensors.items():
|
||||
@@ -583,8 +625,11 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
weight=layer.weight,
|
||||
weight_scale=getattr(layer, "weight_scale", None),
|
||||
zero_point=getattr(layer, "zero_point", None),
|
||||
bias=getattr(layer, "bias", None),
|
||||
bias=getattr(layer, "bias" if bias is not None else "zero_bias", None),
|
||||
weight_scale_2=getattr(layer, "weight_scale_2", None),
|
||||
input_scale=getattr(layer, "input_scale", None),
|
||||
input_scale_2=getattr(layer, "input_scale_2", None),
|
||||
hadamard_block_size=self.weight_schema.hadamard_block_size,
|
||||
locks=self.locks,
|
||||
compute_config=self.compute_config,
|
||||
)
|
||||
@@ -606,11 +651,15 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
# Derive QuantKeys from humming schemas.
|
||||
# Prefer force schemas (the final format after requant) over base.
|
||||
weight_key = weight_schema_to_quant_key(
|
||||
self.force_weight_schema or self.weight_schema
|
||||
self.force_weight_schema or self.weight_schema, moe.in_dtype
|
||||
)
|
||||
activation_key = input_schema_to_quant_key(
|
||||
self.force_input_schema or self.input_schema
|
||||
runtime_input_schema = check_and_fallback_input_schema(
|
||||
weight_schema=self.force_weight_schema or self.weight_schema,
|
||||
input_schema=self.force_input_schema or self.input_schema,
|
||||
param_dtype=moe.in_dtype,
|
||||
allow_fallback=quant_config.allow_input_schema_fallback,
|
||||
)
|
||||
activation_key = input_schema_to_quant_key(runtime_input_schema, moe.in_dtype)
|
||||
|
||||
# Select Humming MoE experts
|
||||
self.experts_cls = select_humming_moe_experts(
|
||||
@@ -634,22 +683,14 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
# online quant (fp16/bf16 -> quant_type)
|
||||
if is_unquantized:
|
||||
assert isinstance(self.weight_schema, _hm.HummingWeightSchema)
|
||||
f16_dtype = _hm.DataType.from_torch_dtype(layer.param_dtype)
|
||||
has_global_scale = "TENSOR" in str(self.weight_schema.weight_scale_type)
|
||||
tensor_list = _hm.quantize_weight(
|
||||
weight=loaded_weight,
|
||||
dtype=self.weight_schema.b_dtype,
|
||||
scale_dtype=self.weight_schema.bs_dtype or f16_dtype,
|
||||
group_size=self.weight_schema.weight_scale_group_size,
|
||||
has_zero_point=self.weight_schema.has_zero_point,
|
||||
has_global_scale=has_global_scale,
|
||||
is_fp_zero_point=self.weight_schema.is_fp_zero_point,
|
||||
pack=True,
|
||||
tensors = self.weight_schema.quant_tensor(
|
||||
loaded_weight.to(param.device),
|
||||
self.weight_schema,
|
||||
layer.param_dtype,
|
||||
)
|
||||
|
||||
key_list = ["weight", "weight_scale", "zero_point", "global_scale"]
|
||||
success = True
|
||||
for key, tensor in zip(key_list, tensor_list):
|
||||
for key, tensor in tensors.items():
|
||||
if tensor is None or tensor.nelement() == 0:
|
||||
continue
|
||||
sublayer_name = "w2" if shard_id == "w2" else "w13"
|
||||
@@ -695,6 +736,18 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
layer.num_experts = num_experts
|
||||
layer.param_dtype = params_dtype
|
||||
layer.intermediate_size = intermediate_size_per_partition
|
||||
input_schema = self.force_input_schema or self.input_schema
|
||||
input_schema = input_schema.to_humming_schema(params_dtype)
|
||||
shape_k = gcd(hidden_size, intermediate_size_per_partition)
|
||||
for name in ("weight_schema", "force_weight_schema"):
|
||||
schema = getattr(self, name)
|
||||
if getattr(schema, "hadamard_block_size", None) == -1:
|
||||
schema = humming_update_schema_hadamard_block_size(
|
||||
weight_schema=schema,
|
||||
input_schema=input_schema,
|
||||
shape_k=shape_k,
|
||||
)
|
||||
setattr(self, name, schema)
|
||||
weight_loader = extra_weight_attrs.get("weight_loader", default_weight_loader)
|
||||
weight_loader = self.prepare_weight_loader(layer, weight_loader)
|
||||
extra_weight_attrs["weight_loader"] = weight_loader
|
||||
@@ -761,6 +814,7 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
weight_schema=self.weight_schema,
|
||||
input_schema=self.input_schema,
|
||||
force_weight_schema=self.force_weight_schema,
|
||||
allow_input_schema_fallback=self.quant_config.allow_input_schema_fallback,
|
||||
)
|
||||
|
||||
# Build the MoE kernel
|
||||
|
||||
Reference in New Issue
Block a user