[Feature][Humming] Humming feature integration (#56685)
Signed-off-by: jinzhen.ljz <jinzhen.ljz@antgroup.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Codex
mgoin
parent
27757dde02
commit
7d99c2c4fd
@@ -713,7 +713,7 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
|
||||
|
||||
def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
|
||||
if self.wna16_moe_backend == WNA16MoEBackend.HUMMING:
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
from vllm.model_executor.layers.quantization.utils.humming import (
|
||||
get_humming_moe_quant_config,
|
||||
)
|
||||
|
||||
|
||||
@@ -689,7 +689,7 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
|
||||
def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
|
||||
if self.wna16_moe_backend == WNA16MoEBackend.HUMMING:
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
from vllm.model_executor.layers.quantization.utils.humming import (
|
||||
get_humming_moe_quant_config,
|
||||
)
|
||||
|
||||
|
||||
+1
-1
@@ -474,7 +474,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
self, layer: torch.nn.Module
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
if self.wna16_backend == WNA16MoEBackend.HUMMING:
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
from vllm.model_executor.layers.quantization.utils.humming import (
|
||||
get_humming_moe_quant_config,
|
||||
)
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
from math import gcd
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import regex as re
|
||||
@@ -28,12 +29,15 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
from vllm.model_executor.layers.quantization.utils.humming import (
|
||||
check_and_fallback_input_schema,
|
||||
convert_to_humming_moe_kernel_format,
|
||||
get_humming_linear_compute_config,
|
||||
get_humming_moe_quant_config,
|
||||
humming_update_schema_hadamard_block_size,
|
||||
input_schema_to_quant_key,
|
||||
make_humming_moe_kernel,
|
||||
resolve_humming_layer_config,
|
||||
select_humming_moe_experts,
|
||||
weight_schema_to_quant_key,
|
||||
)
|
||||
@@ -227,23 +231,13 @@ class HummingConfig(QuantizationConfig):
|
||||
return None
|
||||
config = group_config
|
||||
|
||||
layer_config = config
|
||||
layer_dynamic = config.get("dynamic", {})
|
||||
if not isinstance(layer_dynamic, dict):
|
||||
layer_dynamic = {}
|
||||
for regex, override_config in layer_dynamic.items():
|
||||
if regex[:1] != "+":
|
||||
continue
|
||||
if re.match(regex[2:], prefix):
|
||||
layer_config = config.copy()
|
||||
layer_config.update(override_config)
|
||||
break
|
||||
layer_config = resolve_humming_layer_config(config, prefix)
|
||||
|
||||
if "quant_method" in layer_config:
|
||||
return _hm.BaseWeightSchema.from_config(layer_config)
|
||||
return None
|
||||
|
||||
def get_layer_input_schema(self, config: dict[str, Any], prefix: str):
|
||||
def get_layer_input_config(self, config: dict[str, Any], prefix: str):
|
||||
if self.is_layer_skipped(config, prefix):
|
||||
return None
|
||||
if config["quant_method"] in ["compressed-tensors", "modelopt"]:
|
||||
@@ -252,10 +246,15 @@ class HummingConfig(QuantizationConfig):
|
||||
return None
|
||||
config = group_config
|
||||
|
||||
config = resolve_humming_layer_config(config, prefix)
|
||||
if config.get("quant_method", None) in _hm.BaseInputSchema.INPUT_SCHEMA_MAP:
|
||||
return _hm.BaseInputSchema.from_config(config)
|
||||
return config
|
||||
return None
|
||||
|
||||
def get_layer_input_schema(self, config: dict[str, Any], prefix: str):
|
||||
config = self.get_layer_input_config(config, prefix)
|
||||
return None if config is None else _hm.BaseInputSchema.from_config(config)
|
||||
|
||||
def get_quant_config_for_layer(
|
||||
self, prefix: str, layer_type: str
|
||||
) -> "HummingLayerQuantizationConfig | None":
|
||||
@@ -279,14 +278,25 @@ class HummingConfig(QuantizationConfig):
|
||||
if weight_schema is not None:
|
||||
input_schema = None
|
||||
force_input_schema = None
|
||||
allow_input_schema_fallback = True
|
||||
|
||||
if self.full_config:
|
||||
input_schema = self.get_layer_input_schema(self.full_config, prefix)
|
||||
input_config = self.get_layer_input_config(self.full_config, prefix)
|
||||
if input_config is not None:
|
||||
allow_input_schema_fallback = input_config.pop(
|
||||
"allow_fallback", True
|
||||
)
|
||||
input_schema = _hm.BaseInputSchema.from_config(input_config)
|
||||
|
||||
if envs.VLLM_HUMMING_INPUT_QUANT_CONFIG:
|
||||
quant_config = envs.VLLM_HUMMING_INPUT_QUANT_CONFIG.copy()
|
||||
quant_config["quant_method"] = "humming"
|
||||
force_input_schema = self.get_layer_input_schema(quant_config, prefix)
|
||||
input_config = self.get_layer_input_config(quant_config, prefix)
|
||||
if input_config is not None:
|
||||
allow_input_schema_fallback = input_config.pop(
|
||||
"allow_fallback", False
|
||||
)
|
||||
force_input_schema = _hm.BaseInputSchema.from_config(input_config)
|
||||
if input_schema is None:
|
||||
input_schema = force_input_schema
|
||||
|
||||
@@ -299,6 +309,7 @@ class HummingConfig(QuantizationConfig):
|
||||
force_weight_schema=force_weight_schema,
|
||||
force_input_schema=force_input_schema,
|
||||
is_online_quant=is_online_quant,
|
||||
allow_input_schema_fallback=allow_input_schema_fallback,
|
||||
)
|
||||
return None
|
||||
|
||||
@@ -318,6 +329,11 @@ class HummingConfig(QuantizationConfig):
|
||||
elif isinstance(layer, LinearBase):
|
||||
return UnquantizedLinearMethod()
|
||||
elif isinstance(layer, LinearBase):
|
||||
input_size = layer.input_size
|
||||
partition_size = getattr(layer, "input_size_per_partition", input_size)
|
||||
is_online_quant = quant_config.is_online_quant
|
||||
if is_online_quant and (input_size % 32 != 0 or partition_size % 32 != 0):
|
||||
return UnquantizedLinearMethod()
|
||||
return HummingLinearMethod(quant_config)
|
||||
elif isinstance(layer, RoutedExperts):
|
||||
return HummingMoEMethod(quant_config, layer.moe_config)
|
||||
@@ -332,6 +348,7 @@ class HummingLayerQuantizationConfig(HummingConfig):
|
||||
force_weight_schema: "HummingWeightSchema | None" = None,
|
||||
force_input_schema: "HummingInputSchema | None" = None,
|
||||
is_online_quant: bool = False,
|
||||
allow_input_schema_fallback: bool = True,
|
||||
):
|
||||
self.weight_schema = weight_schema
|
||||
if input_schema is None:
|
||||
@@ -340,6 +357,7 @@ class HummingLayerQuantizationConfig(HummingConfig):
|
||||
self.force_weight_schema = force_weight_schema
|
||||
self.force_input_schema = force_input_schema
|
||||
self.is_online_quant = is_online_quant
|
||||
self.allow_input_schema_fallback = allow_input_schema_fallback
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config):
|
||||
@@ -373,21 +391,13 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
if is_unquantized and self.is_online_quant:
|
||||
# online quant (fp16/bf16 -> quant_type)
|
||||
assert isinstance(self.weight_schema, _hm.HummingWeightSchema)
|
||||
f16_dtype = _hm.DataType.from_torch_dtype(layer.param_dtype)
|
||||
has_global_scale = "TENSOR" in str(self.weight_schema.weight_scale_type)
|
||||
tensor_list = _hm.quantize_weight(
|
||||
weight=loaded_weight,
|
||||
dtype=self.weight_schema.b_dtype,
|
||||
scale_dtype=self.weight_schema.bs_dtype or f16_dtype,
|
||||
group_size=self.weight_schema.weight_scale_group_size,
|
||||
has_zero_point=self.weight_schema.has_zero_point,
|
||||
has_global_scale=has_global_scale,
|
||||
is_fp_zero_point=self.weight_schema.is_fp_zero_point,
|
||||
pack=True,
|
||||
tensors = self.weight_schema.quant_tensor(
|
||||
loaded_weight.to(param.device),
|
||||
self.weight_schema,
|
||||
layer.param_dtype,
|
||||
)
|
||||
|
||||
key_list = ["weight", "weight_scale", "zero_point", "global_scale"]
|
||||
for key, tensor in zip(key_list, tensor_list):
|
||||
for key, tensor in tensors.items():
|
||||
if tensor is None or tensor.nelement() == 0:
|
||||
continue
|
||||
param = getattr(layer, key)
|
||||
@@ -459,6 +469,18 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
layer.output_partition_sizes = output_partition_sizes
|
||||
layer.extra_weight_attrs = extra_weight_attrs.copy()
|
||||
|
||||
input_schema = self.force_input_schema or self.input_schema
|
||||
input_schema = input_schema.to_humming_schema(params_dtype)
|
||||
for name in ("weight_schema", "force_weight_schema"):
|
||||
schema = getattr(self, name)
|
||||
if getattr(schema, "hadamard_block_size", None) == -1:
|
||||
schema = humming_update_schema_hadamard_block_size(
|
||||
weight_schema=schema,
|
||||
input_schema=input_schema,
|
||||
shape_k=input_size_per_partition,
|
||||
)
|
||||
setattr(self, name, schema)
|
||||
|
||||
weight_loader = extra_weight_attrs.get("weight_loader", default_weight_loader)
|
||||
new_weight_loader = self.prepare_weight_loader(layer, weight_loader)
|
||||
extra_weight_attrs["weight_loader"] = new_weight_loader
|
||||
@@ -502,7 +524,10 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
return None
|
||||
|
||||
# convert from checkpoint format to humming format
|
||||
if not isinstance(self.weight_schema, _hm.HummingWeightSchema):
|
||||
is_humming_weight = isinstance(self.weight_schema, _hm.HummingWeightSchema)
|
||||
is_humming_input = isinstance(self.input_schema, _hm.HummingInputSchema)
|
||||
|
||||
if not is_humming_weight or not is_humming_input:
|
||||
self.weight_schema, tensors = self.weight_schema.convert_humming(
|
||||
tensors=layer.state_dict(),
|
||||
shape_n_stacks=layer.output_partition_sizes,
|
||||
@@ -510,12 +535,13 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
param_dtype=layer.param_dtype,
|
||||
)
|
||||
|
||||
self.input_schema, _ = self.input_schema.convert_humming(
|
||||
self.input_schema, input_tensors = self.input_schema.convert_humming(
|
||||
tensors=layer.state_dict(),
|
||||
shape_n_stacks=layer.output_partition_sizes,
|
||||
shape_k_stacks=[layer.input_size_per_partition],
|
||||
param_dtype=layer.param_dtype,
|
||||
)
|
||||
tensors.update(input_tensors)
|
||||
|
||||
for name, _ in list(layer.named_parameters()):
|
||||
delattr(layer, name)
|
||||
@@ -530,11 +556,15 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
assert isinstance(self.weight_schema, _hm.HummingWeightSchema)
|
||||
force_requant = self.force_weight_schema is not None
|
||||
if force_requant and self.weight_schema != self.force_weight_schema:
|
||||
source_tensors = layer.state_dict()
|
||||
tensors = self.weight_schema.requant_tensors(
|
||||
tensors=layer.state_dict(),
|
||||
tensors=source_tensors,
|
||||
target_weight_schema=self.force_weight_schema,
|
||||
param_dtype=layer.param_dtype,
|
||||
)
|
||||
name = self.input_schema.static_tensor_scale_name
|
||||
if name is not None:
|
||||
tensors[name] = source_tensors[name]
|
||||
|
||||
self.weight_schema = self.force_weight_schema
|
||||
|
||||
@@ -548,6 +578,12 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
|
||||
del tensors
|
||||
|
||||
self.input_schema = check_and_fallback_input_schema(
|
||||
weight_schema=self.weight_schema,
|
||||
input_schema=self.input_schema,
|
||||
param_dtype=layer.param_dtype,
|
||||
allow_fallback=self.quant_config.allow_input_schema_fallback,
|
||||
)
|
||||
self.layer_config = _hm.prepare_layer_config(
|
||||
shape_n=layer.output_partition_sizes_sum,
|
||||
shape_k=layer.input_size_per_partition,
|
||||
@@ -557,10 +593,16 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
pad_k_to_multiple=128,
|
||||
has_bias=layer.has_bias,
|
||||
torch_dtype=layer.param_dtype,
|
||||
device=layer.weight.device,
|
||||
)
|
||||
tensors = _hm.transform_humming_tensors(
|
||||
self.layer_config, dict(layer.named_parameters())
|
||||
)
|
||||
source_tensors = dict(layer.named_parameters())
|
||||
tensors = _hm.transform_humming_tensors(self.layer_config, source_tensors)
|
||||
name = self.input_schema.static_tensor_scale_name
|
||||
if name is not None:
|
||||
tensors[name] = source_tensors[name]
|
||||
if "bias" in tensors:
|
||||
zero_bias = torch.zeros_like(tensors["bias"])
|
||||
layer.register_buffer("zero_bias", zero_bias)
|
||||
for name, _ in list(layer.named_parameters()):
|
||||
delattr(layer, name)
|
||||
for name, tensor in tensors.items():
|
||||
@@ -583,8 +625,11 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
weight=layer.weight,
|
||||
weight_scale=getattr(layer, "weight_scale", None),
|
||||
zero_point=getattr(layer, "zero_point", None),
|
||||
bias=getattr(layer, "bias", None),
|
||||
bias=getattr(layer, "bias" if bias is not None else "zero_bias", None),
|
||||
weight_scale_2=getattr(layer, "weight_scale_2", None),
|
||||
input_scale=getattr(layer, "input_scale", None),
|
||||
input_scale_2=getattr(layer, "input_scale_2", None),
|
||||
hadamard_block_size=self.weight_schema.hadamard_block_size,
|
||||
locks=self.locks,
|
||||
compute_config=self.compute_config,
|
||||
)
|
||||
@@ -606,11 +651,15 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
# Derive QuantKeys from humming schemas.
|
||||
# Prefer force schemas (the final format after requant) over base.
|
||||
weight_key = weight_schema_to_quant_key(
|
||||
self.force_weight_schema or self.weight_schema
|
||||
self.force_weight_schema or self.weight_schema, moe.in_dtype
|
||||
)
|
||||
activation_key = input_schema_to_quant_key(
|
||||
self.force_input_schema or self.input_schema
|
||||
runtime_input_schema = check_and_fallback_input_schema(
|
||||
weight_schema=self.force_weight_schema or self.weight_schema,
|
||||
input_schema=self.force_input_schema or self.input_schema,
|
||||
param_dtype=moe.in_dtype,
|
||||
allow_fallback=quant_config.allow_input_schema_fallback,
|
||||
)
|
||||
activation_key = input_schema_to_quant_key(runtime_input_schema, moe.in_dtype)
|
||||
|
||||
# Select Humming MoE experts
|
||||
self.experts_cls = select_humming_moe_experts(
|
||||
@@ -634,22 +683,14 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
# online quant (fp16/bf16 -> quant_type)
|
||||
if is_unquantized:
|
||||
assert isinstance(self.weight_schema, _hm.HummingWeightSchema)
|
||||
f16_dtype = _hm.DataType.from_torch_dtype(layer.param_dtype)
|
||||
has_global_scale = "TENSOR" in str(self.weight_schema.weight_scale_type)
|
||||
tensor_list = _hm.quantize_weight(
|
||||
weight=loaded_weight,
|
||||
dtype=self.weight_schema.b_dtype,
|
||||
scale_dtype=self.weight_schema.bs_dtype or f16_dtype,
|
||||
group_size=self.weight_schema.weight_scale_group_size,
|
||||
has_zero_point=self.weight_schema.has_zero_point,
|
||||
has_global_scale=has_global_scale,
|
||||
is_fp_zero_point=self.weight_schema.is_fp_zero_point,
|
||||
pack=True,
|
||||
tensors = self.weight_schema.quant_tensor(
|
||||
loaded_weight.to(param.device),
|
||||
self.weight_schema,
|
||||
layer.param_dtype,
|
||||
)
|
||||
|
||||
key_list = ["weight", "weight_scale", "zero_point", "global_scale"]
|
||||
success = True
|
||||
for key, tensor in zip(key_list, tensor_list):
|
||||
for key, tensor in tensors.items():
|
||||
if tensor is None or tensor.nelement() == 0:
|
||||
continue
|
||||
sublayer_name = "w2" if shard_id == "w2" else "w13"
|
||||
@@ -695,6 +736,18 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
layer.num_experts = num_experts
|
||||
layer.param_dtype = params_dtype
|
||||
layer.intermediate_size = intermediate_size_per_partition
|
||||
input_schema = self.force_input_schema or self.input_schema
|
||||
input_schema = input_schema.to_humming_schema(params_dtype)
|
||||
shape_k = gcd(hidden_size, intermediate_size_per_partition)
|
||||
for name in ("weight_schema", "force_weight_schema"):
|
||||
schema = getattr(self, name)
|
||||
if getattr(schema, "hadamard_block_size", None) == -1:
|
||||
schema = humming_update_schema_hadamard_block_size(
|
||||
weight_schema=schema,
|
||||
input_schema=input_schema,
|
||||
shape_k=shape_k,
|
||||
)
|
||||
setattr(self, name, schema)
|
||||
weight_loader = extra_weight_attrs.get("weight_loader", default_weight_loader)
|
||||
weight_loader = self.prepare_weight_loader(layer, weight_loader)
|
||||
extra_weight_attrs["weight_loader"] = weight_loader
|
||||
@@ -761,6 +814,7 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
weight_schema=self.weight_schema,
|
||||
input_schema=self.input_schema,
|
||||
force_weight_schema=self.force_weight_schema,
|
||||
allow_input_schema_fallback=self.quant_config.allow_input_schema_fallback,
|
||||
)
|
||||
|
||||
# Build the MoE kernel
|
||||
|
||||
@@ -389,7 +389,7 @@ class MoeWNA16Method(FusedMoEMethodBase):
|
||||
self, layer: RoutedExperts
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
if self.wna16_backend == WNA16MoEBackend.HUMMING:
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
from vllm.model_executor.layers.quantization.utils.humming import (
|
||||
get_humming_moe_quant_config,
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Humming quantization integration."""
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.humming.activation import (
|
||||
get_humming_activation,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.humming.linear import (
|
||||
apply_humming_linear,
|
||||
convert_linear_layer_to_humming_standard,
|
||||
get_humming_linear_compute_config,
|
||||
prepare_humming_linear_layer_config,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.humming.moe import (
|
||||
HummingMoEQuantConfig,
|
||||
convert_to_humming_moe_kernel_format,
|
||||
get_humming_moe_quant_config,
|
||||
make_humming_moe_kernel,
|
||||
make_humming_moe_quant_config,
|
||||
select_humming_moe_experts,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.humming.schema import (
|
||||
check_and_fallback_input_schema,
|
||||
humming_is_layer_skipped,
|
||||
humming_update_schema_hadamard_block_size,
|
||||
input_schema_to_quant_key,
|
||||
quant_key_to_input_schema,
|
||||
resolve_humming_layer_config,
|
||||
weight_schema_to_quant_key,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"check_and_fallback_input_schema",
|
||||
"get_humming_activation",
|
||||
"HummingMoEQuantConfig",
|
||||
"get_humming_moe_quant_config",
|
||||
"humming_is_layer_skipped",
|
||||
"humming_update_schema_hadamard_block_size",
|
||||
"make_humming_moe_quant_config",
|
||||
"apply_humming_linear",
|
||||
"convert_linear_layer_to_humming_standard",
|
||||
"get_humming_linear_compute_config",
|
||||
"prepare_humming_linear_layer_config",
|
||||
"make_humming_moe_kernel",
|
||||
"select_humming_moe_experts",
|
||||
"input_schema_to_quant_key",
|
||||
"quant_key_to_input_schema",
|
||||
"resolve_humming_layer_config",
|
||||
"weight_schema_to_quant_key",
|
||||
"convert_to_humming_moe_kernel_format",
|
||||
]
|
||||
@@ -0,0 +1,126 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MoE activation expressions for Humming input processing."""
|
||||
|
||||
from dataclasses import replace
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.activation import (
|
||||
ApplyMoEActivationConfig,
|
||||
MoEActivation,
|
||||
)
|
||||
|
||||
_SILU_IMPL = "a / (1.f + expf(-a))"
|
||||
_GELU_IMPL = "0.5f * a * (1.f + erff(a * 0.7071067811865476f))"
|
||||
_RELU2_IMPL = "fmaxf(a, 0.f) * fmaxf(a, 0.f)"
|
||||
_GELU_TANH_IMPL = """[](float a) {
|
||||
const float cubic = 0.044715f * a * a * a;
|
||||
const float inner = 0.7978845608028654f * (a + cubic);
|
||||
return 0.5f * a * (1.f + tanhf(inner));
|
||||
}(a)"""
|
||||
_CLAMPED_SILU_IMPL = """[](float a, float b) {{
|
||||
const float gate = fminf(a, {limit});
|
||||
const float up = fmaxf(fminf(b, {limit}), -{limit});
|
||||
const float activated_gate = gate / (1.f + expf(-{alpha} * gate));
|
||||
return activated_gate * (up + {beta});
|
||||
}}(a, b)"""
|
||||
|
||||
# Parameterized expressions are resolved by get_humming_activation().
|
||||
_HUMMING_ACTIVATIONS: dict[MoEActivation, dict[str, str]] = {
|
||||
MoEActivation.SILU: {
|
||||
"activation_type": "binary_split",
|
||||
"activation_impl": f"({_SILU_IMPL}) * b",
|
||||
},
|
||||
MoEActivation.GELU: {
|
||||
"activation_type": "binary_split",
|
||||
"activation_impl": f"({_GELU_IMPL}) * b",
|
||||
},
|
||||
MoEActivation.GELU_TANH: {
|
||||
"activation_type": "binary_split",
|
||||
"activation_impl": f"({_GELU_TANH_IMPL}) * b",
|
||||
},
|
||||
MoEActivation.RELU2: {
|
||||
"activation_type": "binary_split",
|
||||
"activation_impl": f"({_RELU2_IMPL}) * b",
|
||||
},
|
||||
MoEActivation.SWIGLUOAI: {
|
||||
"activation_type": "binary_interleaved",
|
||||
"activation_impl": _CLAMPED_SILU_IMPL,
|
||||
},
|
||||
MoEActivation.SITU: {
|
||||
"activation_type": "binary_split",
|
||||
"activation_impl": """[](float a, float b) {{
|
||||
const float gate = {situ_beta} * tanhf(a / {situ_beta});
|
||||
const float activated_gate = gate / (1.f + expf(-a));
|
||||
const float up = {situ_up};
|
||||
return activated_gate * up;
|
||||
}}(a, b)""",
|
||||
},
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE: {
|
||||
"activation_type": "binary_split",
|
||||
"activation_impl": _CLAMPED_SILU_IMPL,
|
||||
},
|
||||
MoEActivation.SWIGLUSTEP: {
|
||||
"activation_type": "binary_split",
|
||||
"activation_impl": """[](float a, float b) {
|
||||
const float activated_gate = a / (1.f + expf(-a));
|
||||
const float gate = fminf(activated_gate, 7.f);
|
||||
const float up = fmaxf(fminf(b, 7.f), -7.f);
|
||||
return gate * up;
|
||||
}(a, b)""",
|
||||
},
|
||||
MoEActivation.SILU_NO_MUL: {
|
||||
"activation_type": "unary",
|
||||
"activation_impl": _SILU_IMPL,
|
||||
},
|
||||
MoEActivation.GELU_NO_MUL: {
|
||||
"activation_type": "unary",
|
||||
"activation_impl": _GELU_IMPL,
|
||||
},
|
||||
MoEActivation.GELU_TANH_NO_MUL: {
|
||||
"activation_type": "unary",
|
||||
"activation_impl": _GELU_TANH_IMPL,
|
||||
},
|
||||
MoEActivation.RELU2_NO_MUL: {
|
||||
"activation_type": "unary",
|
||||
"activation_impl": _RELU2_IMPL,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def get_humming_activation(
|
||||
activation: MoEActivation,
|
||||
config: ApplyMoEActivationConfig | None = None,
|
||||
) -> dict[str, str]:
|
||||
config = config or ApplyMoEActivationConfig()
|
||||
use_clamped_silu = False
|
||||
if activation == MoEActivation.SWIGLUOAI:
|
||||
config = ApplyMoEActivationConfig(clamp_limit=7.0, alpha=1.702, beta=1.0)
|
||||
use_clamped_silu = True
|
||||
elif activation == MoEActivation.SILU and config.clamp_limit is not None:
|
||||
config = replace(config, alpha=1.0, beta=0.0)
|
||||
use_clamped_silu = True
|
||||
elif activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
|
||||
if config.clamp_limit is None:
|
||||
raise ValueError("SWIGLUOAI_UNINTERLEAVE requires clamp_limit")
|
||||
use_clamped_silu = True
|
||||
|
||||
definition = _HUMMING_ACTIVATIONS[activation].copy()
|
||||
if activation == MoEActivation.SITU:
|
||||
if config.activation_situ_beta is None:
|
||||
raise ValueError("SITU requires activation_situ_beta")
|
||||
linear_beta = config.activation_situ_linear_beta
|
||||
situ_up = "b"
|
||||
if linear_beta is not None and linear_beta > 0:
|
||||
situ_up = f"({float(linear_beta)}f) * tanhf(b / ({float(linear_beta)}f))"
|
||||
situ_beta = f"({float(config.activation_situ_beta)}f)"
|
||||
activation_impl = definition["activation_impl"]
|
||||
activation_impl = activation_impl.format(situ_beta=situ_beta, situ_up=situ_up)
|
||||
definition["activation_impl"] = activation_impl
|
||||
elif use_clamped_silu:
|
||||
assert config.clamp_limit is not None
|
||||
limit = f"({float(config.clamp_limit)}f)"
|
||||
alpha = f"({float(config.alpha)}f)"
|
||||
beta = f"({float(config.beta)}f)"
|
||||
activation_impl = _CLAMPED_SILU_IMPL.format(limit=limit, alpha=alpha, beta=beta)
|
||||
definition["activation_impl"] = activation_impl
|
||||
return definition
|
||||
@@ -0,0 +1,183 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Prepare and execute Humming linear layers."""
|
||||
|
||||
import json
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
from vllm.model_executor.layers.quantization.utils.humming.schema import (
|
||||
check_and_fallback_input_schema,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.utils.humming import BaseInputSchema, LayerConfig
|
||||
|
||||
|
||||
def convert_linear_layer_to_humming_standard(
|
||||
layer: LinearBase, name_map: dict[str, str]
|
||||
):
|
||||
"""Rename/reshape a linear layer's quantized params (the canonical MPLinear
|
||||
layout: ``weight_packed`` int32 + ``weight_scale``) into the parameter names
|
||||
and layout humming's weight schema expects (``weight`` / ``weight_scale``)."""
|
||||
for name, checkpoint_name in name_map.items():
|
||||
tensor = getattr(layer, checkpoint_name)
|
||||
delattr(layer, checkpoint_name)
|
||||
|
||||
if name == "weight":
|
||||
input_dim = getattr(tensor, "input_dim", 1)
|
||||
output_dim = getattr(tensor, "output_dim", 0)
|
||||
|
||||
if input_dim == 0 and output_dim == 1:
|
||||
tensor = tensor.transpose(1, 0).contiguous()
|
||||
else:
|
||||
assert output_dim == 0 and input_dim == 1
|
||||
|
||||
tensor = tensor.view(tensor.size(0), -1).view(torch.int32)
|
||||
elif name in ["weight_scale", "zero_point"]:
|
||||
if getattr(tensor, "output_dim", 0) == 1:
|
||||
tensor = tensor.transpose(0, 1).contiguous()
|
||||
if tensor.ndim == 1:
|
||||
tensor = tensor.unsqueeze(1)
|
||||
|
||||
tensor = tensor.view(torch.int32) if name == "zero_point" else tensor
|
||||
|
||||
if isinstance(tensor, torch.nn.Parameter):
|
||||
param = tensor
|
||||
else:
|
||||
param = torch.nn.Parameter(tensor, requires_grad=False)
|
||||
|
||||
setattr(layer, name, param)
|
||||
|
||||
|
||||
def prepare_humming_linear_layer_config(
|
||||
layer: LinearBase,
|
||||
quant_config: dict,
|
||||
input_quant_config: dict | None = None,
|
||||
*,
|
||||
input_schema: "BaseInputSchema | None" = None,
|
||||
) -> "LayerConfig":
|
||||
from vllm.utils.humming import (
|
||||
BaseInputSchema,
|
||||
BaseWeightSchema,
|
||||
HummingInputSchema,
|
||||
prepare_layer_config,
|
||||
transform_humming_tensors,
|
||||
)
|
||||
|
||||
weight_schema = BaseWeightSchema.from_config(quant_config)
|
||||
if input_schema is None:
|
||||
if input_quant_config is not None:
|
||||
input_schema = BaseInputSchema.from_config(input_quant_config)
|
||||
else:
|
||||
input_schema = HummingInputSchema()
|
||||
|
||||
# ReplicatedLinear has no TP partitioning and so does not set
|
||||
# input_size_per_partition; for it that is just input_size. Use hasattr
|
||||
# rather than getattr's default arg, which is evaluated eagerly and would
|
||||
# raise on layers lacking input_size (e.g. ParallelLMHead).
|
||||
if hasattr(layer, "input_size_per_partition"):
|
||||
input_size_per_partition = layer.input_size_per_partition
|
||||
else:
|
||||
input_size_per_partition = layer.input_size
|
||||
shape_k_stacks = [input_size_per_partition]
|
||||
shape_n_stacks = layer.output_partition_sizes
|
||||
|
||||
# Step 1: convert weight and input schemas to humming standard format
|
||||
source_tensors = dict(layer.named_parameters())
|
||||
weight_schema, tensors = weight_schema.convert_humming(
|
||||
tensors=source_tensors,
|
||||
shape_n_stacks=shape_n_stacks,
|
||||
shape_k_stacks=shape_k_stacks,
|
||||
param_dtype=layer.params_dtype,
|
||||
)
|
||||
input_schema, input_tensors = input_schema.convert_humming(
|
||||
tensors=source_tensors,
|
||||
shape_n_stacks=shape_n_stacks,
|
||||
shape_k_stacks=shape_k_stacks,
|
||||
param_dtype=layer.params_dtype,
|
||||
)
|
||||
|
||||
tensors.update(input_tensors)
|
||||
layer.weight_schema = weight_schema
|
||||
input_schema = check_and_fallback_input_schema(
|
||||
weight_schema=weight_schema,
|
||||
input_schema=input_schema,
|
||||
param_dtype=layer.params_dtype,
|
||||
)
|
||||
for name in ("input_scale", "input_scale_2"):
|
||||
if name != input_schema.static_tensor_scale_name:
|
||||
tensors.pop(name, None)
|
||||
input_tensors.pop(name, None)
|
||||
|
||||
# Step 2: transform weight (humming standard format) for forwarding.
|
||||
config = prepare_layer_config(
|
||||
shape_n=sum(layer.output_partition_sizes),
|
||||
shape_k=input_size_per_partition,
|
||||
weight_schema=weight_schema,
|
||||
input_schema=input_schema,
|
||||
pad_n_to_multiple=256,
|
||||
pad_k_to_multiple=128,
|
||||
has_bias=layer.has_bias,
|
||||
torch_dtype=layer.params_dtype,
|
||||
device=tensors["weight"].device,
|
||||
)
|
||||
tensors = transform_humming_tensors(config, tensors)
|
||||
tensors.update(input_tensors)
|
||||
if "bias" in tensors:
|
||||
zero_bias = torch.zeros_like(tensors["bias"])
|
||||
layer.register_buffer("zero_bias", zero_bias)
|
||||
for name, _ in list(layer.named_parameters()):
|
||||
delattr(layer, name)
|
||||
for name, tensor in tensors.items():
|
||||
param = torch.nn.Parameter(tensor, requires_grad=False)
|
||||
setattr(layer, name, param)
|
||||
|
||||
return config
|
||||
|
||||
|
||||
def get_humming_linear_compute_config() -> str:
|
||||
return json.dumps(
|
||||
{
|
||||
"use_batch_invariant": envs.VLLM_BATCH_INVARIANT,
|
||||
"use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,
|
||||
"gemm_type": "dense",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def apply_humming_linear(
|
||||
layer: LinearBase,
|
||||
x: torch.Tensor,
|
||||
*,
|
||||
skip_bias_add: bool = False,
|
||||
layer_config: "LayerConfig",
|
||||
compute_config: str,
|
||||
locks: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
from vllm.utils.humming import humming_forward
|
||||
|
||||
mode = layer_config.input_quant_mode
|
||||
input_scale = getattr(layer, "input_scale", None) if mode.should_quantize else None
|
||||
input_scale_2 = (
|
||||
getattr(layer, "input_scale_2", None) if mode.has_secondary_scale else None
|
||||
)
|
||||
flatten_inputs = x.reshape(-1, x.size(-1))
|
||||
output = humming_forward(
|
||||
layer_config,
|
||||
inputs=flatten_inputs,
|
||||
weight=layer.weight,
|
||||
weight_scale=getattr(layer, "weight_scale", None),
|
||||
zero_point=getattr(layer, "zero_point", None),
|
||||
bias=getattr(layer, "zero_bias" if skip_bias_add else "bias", None),
|
||||
weight_scale_2=getattr(layer, "weight_scale_2", None),
|
||||
input_scale=input_scale,
|
||||
input_scale_2=input_scale_2,
|
||||
hadamard_block_size=layer.weight_schema.hadamard_block_size,
|
||||
locks=locks,
|
||||
compute_config=compute_config,
|
||||
)
|
||||
return output.view(*x.shape[:-1], output.size(-1))
|
||||
+65
-539
@@ -1,13 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
"""Configure, prepare weights for, and assemble Humming MoE kernels."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
import vllm.model_executor.layers.quantization.utils.humming.schema as humming_schema
|
||||
from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
@@ -23,34 +24,19 @@ from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
HummingGroupedExperts,
|
||||
HummingIndexedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
from vllm.model_executor.layers.quantization.utils.humming.schema import (
|
||||
humming_is_layer_skipped,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
FP4_DTYPE,
|
||||
FP8_DTYPE,
|
||||
INT4_DTYPE,
|
||||
INT8_DTYPE,
|
||||
MXFP_SCALE_DTYPE,
|
||||
GroupShape,
|
||||
QuantKey,
|
||||
ScaleDesc,
|
||||
)
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||
from vllm.utils.humming import (
|
||||
AWQWeightSchema,
|
||||
BaseInputSchema,
|
||||
BaseWeightSchema,
|
||||
CompressedTensorsInputSchema,
|
||||
CompressedTensorsWeightSchema,
|
||||
Fp8WeightSchema,
|
||||
GPTQWeightSchema,
|
||||
HummingInputSchema,
|
||||
HummingWeightSchema,
|
||||
LayerConfig,
|
||||
)
|
||||
from vllm.utils.humming import dtypes as humming_dtypes
|
||||
from vllm.utils.humming import LayerConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -59,504 +45,12 @@ logger = init_logger(__name__)
|
||||
class HummingMoEQuantConfig(FusedMoEQuantConfig):
|
||||
w1_humming_config: "LayerConfig"
|
||||
w2_humming_config: "LayerConfig"
|
||||
|
||||
|
||||
if has_humming():
|
||||
from vllm.utils.humming import dtypes as humming_dtypes
|
||||
|
||||
_HUMMING_TO_QUANT_DTYPE: dict[humming_dtypes.DataType, Any] = {
|
||||
humming_dtypes.float4e2m1: FP4_DTYPE,
|
||||
humming_dtypes.float8e4m3: FP8_DTYPE,
|
||||
humming_dtypes.float8e5m2: torch.float8_e5m2,
|
||||
humming_dtypes.int8: torch.int8,
|
||||
humming_dtypes.uint4: INT4_DTYPE,
|
||||
humming_dtypes.uint8: INT8_DTYPE,
|
||||
humming_dtypes.uint2: torch.uint8,
|
||||
humming_dtypes.uint3: torch.uint8,
|
||||
humming_dtypes.uint5: torch.uint8,
|
||||
humming_dtypes.uint6: torch.uint8,
|
||||
humming_dtypes.uint7: torch.uint8,
|
||||
}
|
||||
|
||||
_HUMMING_TO_SCALE_DTYPE: dict[humming_dtypes.DataType, torch.dtype] = {
|
||||
humming_dtypes.float8e8m0: MXFP_SCALE_DTYPE,
|
||||
humming_dtypes.float8e4m3: FP8_DTYPE,
|
||||
humming_dtypes.float16: torch.float16,
|
||||
humming_dtypes.bfloat16: torch.bfloat16,
|
||||
humming_dtypes.float32: torch.float32,
|
||||
}
|
||||
|
||||
|
||||
def _group_shape(group_size: int, group_size_n: int = 0) -> GroupShape:
|
||||
"""Map humming group sizes to QuantKey GroupShape.
|
||||
|
||||
group_size: elements per group along K (col); 0 means full dimension.
|
||||
group_size_n: elements per group along N (row); 0 means 1 (per-row).
|
||||
|
||||
GroupShape convention: row = N dim, col = K dim.
|
||||
"""
|
||||
if group_size == 0 and group_size_n == 0:
|
||||
return GroupShape.PER_CHANNEL
|
||||
|
||||
row = group_size_n if group_size_n > 0 else 1
|
||||
col = group_size if group_size > 0 else -1
|
||||
return GroupShape(row=row, col=col)
|
||||
|
||||
|
||||
# ---- HummingWeightSchema (post-conversion) --------------------------------
|
||||
|
||||
|
||||
def _humming_weight_schema_to_quant_key(
|
||||
schema: "HummingWeightSchema",
|
||||
) -> QuantKey:
|
||||
from vllm.utils.humming import WeightScaleType
|
||||
|
||||
"""Convert a HummingWeightSchema to a QuantKey."""
|
||||
dtype = _HUMMING_TO_QUANT_DTYPE[schema.b_dtype]
|
||||
|
||||
if schema.bs_dtype is not None:
|
||||
scale_dtype = _HUMMING_TO_SCALE_DTYPE[schema.bs_dtype]
|
||||
else:
|
||||
scale_dtype = torch.float32
|
||||
|
||||
group_shape = _group_shape(
|
||||
schema.weight_scale_group_size,
|
||||
schema.weight_scale_group_size_n,
|
||||
)
|
||||
|
||||
scale = ScaleDesc(dtype=scale_dtype, static=True, group_shape=group_shape)
|
||||
|
||||
scale2 = None
|
||||
if schema.weight_scale_type == WeightScaleType.GROUP_TENSOR:
|
||||
scale2 = ScaleDesc(
|
||||
dtype=torch.float32,
|
||||
static=True,
|
||||
group_shape=GroupShape.PER_TENSOR,
|
||||
)
|
||||
|
||||
return QuantKey(
|
||||
dtype=dtype,
|
||||
scale=scale,
|
||||
scale2=scale2,
|
||||
symmetric=not schema.has_zero_point,
|
||||
)
|
||||
|
||||
|
||||
# ---- Checkpoint-format weight schemas (pre-conversion) --------------------
|
||||
|
||||
|
||||
def _fp8_weight_schema_to_quant_key(schema: "Fp8WeightSchema") -> QuantKey:
|
||||
if schema.weight_block_size is not None:
|
||||
gs_n, gs_k = schema.weight_block_size
|
||||
group_shape = GroupShape(row=gs_n, col=gs_k)
|
||||
else:
|
||||
group_shape = GroupShape.PER_CHANNEL
|
||||
|
||||
scale = ScaleDesc(dtype=torch.float32, static=True, group_shape=group_shape)
|
||||
return QuantKey(dtype=FP8_DTYPE, scale=scale, symmetric=True)
|
||||
|
||||
|
||||
def _awq_weight_schema_to_quant_key(schema: "AWQWeightSchema") -> QuantKey:
|
||||
group_shape = _group_shape(schema.group_size)
|
||||
scale = ScaleDesc(
|
||||
dtype=torch.float16,
|
||||
static=True,
|
||||
group_shape=group_shape,
|
||||
)
|
||||
return QuantKey(
|
||||
dtype=INT4_DTYPE,
|
||||
scale=scale,
|
||||
symmetric=not schema.zero_point,
|
||||
)
|
||||
|
||||
|
||||
def _gptq_weight_schema_to_quant_key(schema: "GPTQWeightSchema") -> QuantKey:
|
||||
group_shape = _group_shape(schema.group_size)
|
||||
scale = ScaleDesc(
|
||||
dtype=torch.float16,
|
||||
static=True,
|
||||
group_shape=group_shape,
|
||||
)
|
||||
return QuantKey(dtype=INT4_DTYPE, scale=scale, symmetric=schema.sym)
|
||||
|
||||
|
||||
def _compressed_tensors_weight_schema_to_quant_key(
|
||||
schema: "CompressedTensorsWeightSchema",
|
||||
) -> QuantKey:
|
||||
# Determine dtype from format/type/num_bits
|
||||
fmt = schema.format
|
||||
if fmt in ("int-quantized", "float-quantized", "naive-quantized"):
|
||||
dtype = INT8_DTYPE if schema.type == "int" else FP8_DTYPE
|
||||
elif "nvfp4" in fmt or "mxfp4" in fmt:
|
||||
dtype = FP4_DTYPE
|
||||
else:
|
||||
dtype = _HUMMING_TO_QUANT_DTYPE[
|
||||
humming_dtypes.DataType.from_str(f"uint{schema.num_bits}")
|
||||
]
|
||||
|
||||
# Determine group shape from strategy
|
||||
if schema.strategy in ("group", "tensor_group"):
|
||||
group_shape = _group_shape(schema.group_size or 0)
|
||||
elif schema.strategy == "block" and schema.block_structure is not None:
|
||||
group_shape = GroupShape(
|
||||
row=schema.block_structure[0],
|
||||
col=schema.block_structure[1],
|
||||
)
|
||||
else:
|
||||
group_shape = GroupShape.PER_CHANNEL
|
||||
|
||||
# Determine scale dtype
|
||||
if "mxfp" in fmt:
|
||||
scale_dtype = MXFP_SCALE_DTYPE
|
||||
elif "nvfp4" in fmt:
|
||||
scale_dtype = FP8_DTYPE
|
||||
else:
|
||||
scale_dtype = torch.float32
|
||||
|
||||
scale = ScaleDesc(dtype=scale_dtype, static=True, group_shape=group_shape)
|
||||
|
||||
scale2 = None
|
||||
if "nvfp4" in fmt or schema.strategy == "tensor_group":
|
||||
scale2 = ScaleDesc(
|
||||
dtype=torch.float32,
|
||||
static=True,
|
||||
group_shape=GroupShape.PER_TENSOR,
|
||||
)
|
||||
|
||||
return QuantKey(
|
||||
dtype=dtype,
|
||||
scale=scale,
|
||||
scale2=scale2,
|
||||
symmetric=schema.symmetric,
|
||||
)
|
||||
|
||||
|
||||
# ---- Dispatch for any BaseWeightSchema ------------------------------------
|
||||
|
||||
|
||||
def weight_schema_to_quant_key(
|
||||
schema: "BaseWeightSchema",
|
||||
) -> QuantKey:
|
||||
from vllm.utils.humming import (
|
||||
AWQWeightSchema,
|
||||
BitnetWeightSchema,
|
||||
CompressedTensorsWeightSchema,
|
||||
Fp8WeightSchema,
|
||||
GptOssMxfp4WeightSchema,
|
||||
GPTQWeightSchema,
|
||||
HummingWeightSchema,
|
||||
ModeloptMxfp8WeightSchema,
|
||||
ModeloptNvfp4WeightSchema,
|
||||
Mxfp4WeightSchema,
|
||||
)
|
||||
|
||||
"""Convert any BaseWeightSchema to a QuantKey."""
|
||||
if isinstance(schema, HummingWeightSchema):
|
||||
return _humming_weight_schema_to_quant_key(schema)
|
||||
|
||||
# Schemas with fixed QuantKeys
|
||||
if isinstance(schema, (Mxfp4WeightSchema, GptOssMxfp4WeightSchema)):
|
||||
return QuantKey(
|
||||
dtype=FP4_DTYPE,
|
||||
scale=ScaleDesc(MXFP_SCALE_DTYPE, True, GroupShape(1, 32)),
|
||||
)
|
||||
if isinstance(schema, ModeloptMxfp8WeightSchema):
|
||||
return QuantKey(
|
||||
dtype=FP8_DTYPE,
|
||||
scale=ScaleDesc(MXFP_SCALE_DTYPE, True, GroupShape(1, 32)),
|
||||
)
|
||||
if isinstance(schema, ModeloptNvfp4WeightSchema):
|
||||
return QuantKey(
|
||||
dtype=FP4_DTYPE,
|
||||
scale=ScaleDesc(FP8_DTYPE, True, GroupShape(1, 16)),
|
||||
scale2=ScaleDesc(torch.float32, True, GroupShape.PER_TENSOR),
|
||||
)
|
||||
if isinstance(schema, BitnetWeightSchema):
|
||||
return QuantKey(
|
||||
dtype=torch.uint8,
|
||||
scale=ScaleDesc(torch.float32, True, GroupShape.PER_CHANNEL),
|
||||
)
|
||||
|
||||
# Schemas requiring config inspection
|
||||
if isinstance(schema, Fp8WeightSchema):
|
||||
return _fp8_weight_schema_to_quant_key(schema)
|
||||
if isinstance(schema, AWQWeightSchema):
|
||||
return _awq_weight_schema_to_quant_key(schema)
|
||||
if isinstance(schema, GPTQWeightSchema):
|
||||
return _gptq_weight_schema_to_quant_key(schema)
|
||||
if isinstance(schema, CompressedTensorsWeightSchema):
|
||||
return _compressed_tensors_weight_schema_to_quant_key(schema)
|
||||
|
||||
raise TypeError(f"Unsupported weight schema type: {type(schema)}")
|
||||
|
||||
|
||||
# ---- HummingInputSchema (post-conversion) ----------------------------------
|
||||
|
||||
|
||||
def _humming_input_schema_to_quant_key(
|
||||
schema: "HummingInputSchema",
|
||||
) -> QuantKey | None:
|
||||
"""Convert a HummingInputSchema to a QuantKey. Returns None if
|
||||
the schema represents unquantized (bf16/fp16) inputs."""
|
||||
if schema.a_dtype is None or schema.a_dtype.num_bits >= 16:
|
||||
return None
|
||||
|
||||
dtype = _HUMMING_TO_QUANT_DTYPE[schema.a_dtype]
|
||||
|
||||
gs = schema.input_scale_group_size
|
||||
group_shape = GroupShape(row=1, col=gs) if gs > 0 else GroupShape.PER_TOKEN
|
||||
|
||||
# Pick the scale dtype the Humming kernel actually consumes. An explicit
|
||||
# input_scale_dtype always wins. Otherwise infer from the grouping: MX
|
||||
# microscale activations (group size 32, e.g. MXFP8) carry an e8m0 (uint8)
|
||||
# scale, while block-FP8 (group size 128) and per-token FP8/int8 carry a
|
||||
# float32 scale. Getting this right lets a grouped FP8 activation match
|
||||
# kFp8Dynamic128Sym instead of an unmatchable uint8-scaled key.
|
||||
if schema.input_scale_dtype is not None:
|
||||
scale_dtype = _HUMMING_TO_SCALE_DTYPE[schema.input_scale_dtype]
|
||||
elif gs == 32:
|
||||
scale_dtype = MXFP_SCALE_DTYPE
|
||||
else:
|
||||
scale_dtype = torch.float32
|
||||
|
||||
scale = ScaleDesc(dtype=scale_dtype, static=False, group_shape=group_shape)
|
||||
|
||||
return QuantKey(dtype=dtype, scale=scale, symmetric=True)
|
||||
|
||||
|
||||
# ---- Checkpoint-format input schemas (pre-conversion) ----------------------
|
||||
|
||||
|
||||
def _resolve_input_quant_key(
|
||||
origin_a_dtype: "humming_dtypes.DataType",
|
||||
group_size: int,
|
||||
) -> QuantKey | None:
|
||||
from vllm.utils.humming import HummingInputSchema
|
||||
|
||||
"""Resolve the actual activation QuantKey after platform fallback."""
|
||||
a_dtype = HummingInputSchema().get_fallback_input_dtype(origin_a_dtype)
|
||||
if a_dtype is None or a_dtype.num_bits >= 16:
|
||||
return None
|
||||
|
||||
dtype = _HUMMING_TO_QUANT_DTYPE[a_dtype]
|
||||
gs = group_size if a_dtype == humming_dtypes.float4e2m1 else 0
|
||||
group_shape = GroupShape(row=1, col=gs) if gs > 0 else GroupShape.PER_TOKEN
|
||||
scale_dtype = MXFP_SCALE_DTYPE if gs > 0 else torch.float32
|
||||
|
||||
scale = ScaleDesc(dtype=scale_dtype, static=False, group_shape=group_shape)
|
||||
return QuantKey(dtype=dtype, scale=scale, symmetric=True)
|
||||
|
||||
|
||||
def _compressed_tensors_input_schema_to_quant_key(
|
||||
schema: "CompressedTensorsInputSchema",
|
||||
) -> QuantKey | None:
|
||||
type_bits_to_dtype = {
|
||||
("float", 8): humming_dtypes.float8e4m3,
|
||||
("float", 4): humming_dtypes.float4e2m1,
|
||||
("int", 8): humming_dtypes.int8,
|
||||
("int", 4): humming_dtypes.int4,
|
||||
}
|
||||
origin = type_bits_to_dtype.get((schema.type, schema.num_bits))
|
||||
if origin is None:
|
||||
return None
|
||||
return _resolve_input_quant_key(origin, schema.group_size)
|
||||
|
||||
|
||||
# ---- Dispatch for any BaseInputSchema -------------------------------------
|
||||
|
||||
|
||||
def input_schema_to_quant_key(
|
||||
schema: "BaseInputSchema",
|
||||
) -> QuantKey | None:
|
||||
from vllm.utils.humming import (
|
||||
CompressedTensorsInputSchema,
|
||||
Fp8InputSchema,
|
||||
HummingInputSchema,
|
||||
ModeloptNvfp4InputSchema,
|
||||
)
|
||||
|
||||
"""Convert any BaseInputSchema to a QuantKey. Returns None if
|
||||
the schema represents unquantized (bf16/fp16) inputs."""
|
||||
if isinstance(schema, HummingInputSchema):
|
||||
return _humming_input_schema_to_quant_key(schema)
|
||||
|
||||
if isinstance(schema, Fp8InputSchema):
|
||||
return _resolve_input_quant_key(humming_dtypes.float8e4m3, 0)
|
||||
|
||||
if isinstance(schema, ModeloptNvfp4InputSchema):
|
||||
return _resolve_input_quant_key(
|
||||
humming_dtypes.float8e4m3,
|
||||
schema.group_size,
|
||||
)
|
||||
|
||||
if isinstance(schema, CompressedTensorsInputSchema):
|
||||
return _compressed_tensors_input_schema_to_quant_key(schema)
|
||||
|
||||
raise TypeError(f"Unsupported input schema type: {type(schema)}")
|
||||
|
||||
|
||||
def humming_is_layer_skipped(config: dict[str, Any], prefix: str):
|
||||
if not config:
|
||||
return True
|
||||
|
||||
keys = ["ignored_layers", "ignore", "modules_to_not_convert"]
|
||||
ignored_layers: list[str] = []
|
||||
for key in keys:
|
||||
candidate = config.get(key, []) or []
|
||||
if candidate:
|
||||
ignored_layers = candidate
|
||||
break
|
||||
|
||||
if any(module_name in prefix for module_name in ignored_layers):
|
||||
return True
|
||||
if "lm_head" in prefix:
|
||||
return True
|
||||
|
||||
for regex in config.get("dynamic", {}):
|
||||
if regex[:1] != "-":
|
||||
continue
|
||||
if re.match(regex[2:], prefix):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def convert_linear_layer_to_humming_standard(
|
||||
layer: LinearBase, name_map: dict[str, str]
|
||||
):
|
||||
"""Rename/reshape a linear layer's quantized params (the canonical MPLinear
|
||||
layout: ``weight_packed`` int32 + ``weight_scale``) into the parameter names
|
||||
and layout humming's weight schema expects (``weight`` / ``weight_scale``)."""
|
||||
for name, checkpoint_name in name_map.items():
|
||||
tensor = getattr(layer, checkpoint_name)
|
||||
delattr(layer, checkpoint_name)
|
||||
|
||||
if name == "weight":
|
||||
input_dim = getattr(tensor, "input_dim", 1)
|
||||
output_dim = getattr(tensor, "output_dim", 0)
|
||||
|
||||
if input_dim == 0 and output_dim == 1:
|
||||
tensor = tensor.transpose(1, 0).contiguous()
|
||||
else:
|
||||
assert output_dim == 0 and input_dim == 1
|
||||
|
||||
tensor = tensor.view(tensor.size(0), -1).view(torch.int32)
|
||||
elif name in ["weight_scale", "zero_point"]:
|
||||
if getattr(tensor, "output_dim", 0) == 1:
|
||||
tensor = tensor.transpose(0, 1).contiguous()
|
||||
if tensor.ndim == 1:
|
||||
tensor = tensor.unsqueeze(1)
|
||||
|
||||
tensor = tensor.view(torch.int32) if name == "zero_point" else tensor
|
||||
|
||||
if isinstance(tensor, torch.nn.Parameter):
|
||||
param = tensor
|
||||
else:
|
||||
param = torch.nn.Parameter(tensor, requires_grad=False)
|
||||
|
||||
setattr(layer, name, param)
|
||||
|
||||
|
||||
def prepare_humming_linear_layer_config(
|
||||
layer: LinearBase,
|
||||
quant_config: dict,
|
||||
input_quant_config: dict | None = None,
|
||||
) -> "LayerConfig":
|
||||
from vllm.utils.humming import (
|
||||
BaseInputSchema,
|
||||
BaseWeightSchema,
|
||||
HummingInputSchema,
|
||||
prepare_layer_config,
|
||||
transform_humming_tensors,
|
||||
)
|
||||
|
||||
weight_schema = BaseWeightSchema.from_config(quant_config)
|
||||
if input_quant_config is not None:
|
||||
input_schema = BaseInputSchema.from_config(input_quant_config)
|
||||
else:
|
||||
input_schema = HummingInputSchema()
|
||||
|
||||
# ReplicatedLinear has no TP partitioning and so does not set
|
||||
# input_size_per_partition; for it that is just input_size. Use hasattr
|
||||
# rather than getattr's default arg, which is evaluated eagerly and would
|
||||
# raise on layers lacking input_size (e.g. ParallelLMHead).
|
||||
if hasattr(layer, "input_size_per_partition"):
|
||||
input_size_per_partition = layer.input_size_per_partition
|
||||
else:
|
||||
input_size_per_partition = layer.input_size
|
||||
shape_k_stacks = [input_size_per_partition]
|
||||
shape_n_stacks = layer.output_partition_sizes
|
||||
|
||||
# Step 1: convert weight and input schemas to humming standard format
|
||||
weight_schema, tensors = weight_schema.convert_humming(
|
||||
tensors=dict(layer.named_parameters()),
|
||||
shape_n_stacks=shape_n_stacks,
|
||||
shape_k_stacks=shape_k_stacks,
|
||||
param_dtype=layer.params_dtype,
|
||||
)
|
||||
input_schema, _ = input_schema.convert_humming(
|
||||
tensors={},
|
||||
shape_n_stacks=shape_n_stacks,
|
||||
shape_k_stacks=shape_k_stacks,
|
||||
param_dtype=layer.params_dtype,
|
||||
)
|
||||
|
||||
layer.weight_schema = weight_schema
|
||||
|
||||
# Step 2: transform weight (humming standard format) for forwarding.
|
||||
config = prepare_layer_config(
|
||||
shape_n=sum(layer.output_partition_sizes),
|
||||
shape_k=input_size_per_partition,
|
||||
weight_schema=weight_schema,
|
||||
input_schema=input_schema,
|
||||
pad_n_to_multiple=256,
|
||||
pad_k_to_multiple=128,
|
||||
has_bias=layer.has_bias,
|
||||
torch_dtype=layer.params_dtype,
|
||||
)
|
||||
tensors = transform_humming_tensors(config, tensors)
|
||||
for name, _ in list(layer.named_parameters()):
|
||||
delattr(layer, name)
|
||||
for name, tensor in tensors.items():
|
||||
param = torch.nn.Parameter(tensor, requires_grad=False)
|
||||
setattr(layer, name, param)
|
||||
|
||||
return config
|
||||
|
||||
|
||||
def get_humming_linear_compute_config() -> str:
|
||||
return json.dumps(
|
||||
{
|
||||
"use_batch_invariant": envs.VLLM_BATCH_INVARIANT,
|
||||
"use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,
|
||||
"gemm_type": "dense",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def apply_humming_linear(
|
||||
layer: LinearBase,
|
||||
x: torch.Tensor,
|
||||
*,
|
||||
layer_config: "LayerConfig",
|
||||
compute_config: str,
|
||||
locks: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
from vllm.utils.humming import humming_forward
|
||||
|
||||
flatten_inputs = x.reshape(-1, x.size(-1))
|
||||
output = humming_forward(
|
||||
layer_config,
|
||||
inputs=flatten_inputs,
|
||||
weight=layer.weight,
|
||||
weight_scale=getattr(layer, "weight_scale", None),
|
||||
zero_point=getattr(layer, "zero_point", None),
|
||||
bias=getattr(layer, "bias", None),
|
||||
weight_scale_2=getattr(layer, "weight_scale_2", None),
|
||||
locks=locks,
|
||||
compute_config=compute_config,
|
||||
)
|
||||
return output.view(*x.shape[:-1], output.size(-1))
|
||||
w1_hadamard_block_size: int = 1
|
||||
w2_hadamard_block_size: int = 1
|
||||
w1_input_scale: torch.Tensor | None = None
|
||||
w2_input_scale: torch.Tensor | None = None
|
||||
w1_input_scale_2: torch.Tensor | None = None
|
||||
w2_input_scale_2: torch.Tensor | None = None
|
||||
|
||||
|
||||
def make_humming_moe_quant_config(
|
||||
@@ -648,8 +142,12 @@ def get_humming_moe_quant_config(
|
||||
q_dtype is not None
|
||||
and q_dtype.startswith("float8")
|
||||
and input_scale_group_size == 128
|
||||
and weight_schema.hadamard_block_size <= 1
|
||||
and input_schema.input_quant_mode in (None, "dynamic_group")
|
||||
):
|
||||
q_dtype = _HUMMING_TO_QUANT_DTYPE.get(input_schema.a_dtype, FP8_DTYPE)
|
||||
q_dtype = humming_schema._HUMMING_TO_QUANT_DTYPE.get(
|
||||
input_schema.a_dtype, FP8_DTYPE
|
||||
)
|
||||
activation_group_shape = GroupShape(row=1, col=input_scale_group_size)
|
||||
|
||||
weight_scale_group_size = weight_schema.weight_scale_group_size
|
||||
@@ -665,7 +163,7 @@ def get_humming_moe_quant_config(
|
||||
else:
|
||||
weight_group_shape = GroupShape(row=weight_scale_group_size, col=1)
|
||||
|
||||
return make_humming_moe_quant_config(
|
||||
config = make_humming_moe_quant_config(
|
||||
quant_dtype=q_dtype,
|
||||
weight_dtype=str(weight_schema.b_dtype),
|
||||
weight_group_shape=weight_group_shape,
|
||||
@@ -683,6 +181,17 @@ def get_humming_moe_quant_config(
|
||||
gemm1_clamp_limit=gemm1_clamp_limit,
|
||||
humming_configs=humming_configs,
|
||||
)
|
||||
for prefix, sublayer in (("w1", "w13"), ("w2", "w2")):
|
||||
setattr(
|
||||
config,
|
||||
f"{prefix}_hadamard_block_size",
|
||||
layer.weight_schemas[sublayer].hadamard_block_size,
|
||||
)
|
||||
for name in ("input_scale", "input_scale_2"):
|
||||
key = f"{prefix}_{name}"
|
||||
value = getattr(layer, f"{sublayer}_{name}", None)
|
||||
setattr(config, key, value)
|
||||
return config
|
||||
|
||||
|
||||
def select_humming_moe_experts(
|
||||
@@ -846,12 +355,6 @@ def _convert_sublayer_to_humming(
|
||||
Tuple of (converted_weight_schema, converted_input_schema)
|
||||
|
||||
"""
|
||||
from vllm.utils.humming import HummingWeightSchema
|
||||
|
||||
if isinstance(weight_schema, HummingWeightSchema):
|
||||
# Already in Humming format
|
||||
return weight_schema, input_schema
|
||||
|
||||
tensors = _extract_sublayer_tensors(layer, sublayer_name)
|
||||
|
||||
shape_k_stacks = [shape_k]
|
||||
@@ -867,13 +370,14 @@ def _convert_sublayer_to_humming(
|
||||
num_experts=num_experts,
|
||||
)
|
||||
|
||||
converted_input_schema, _ = input_schema.convert_humming(
|
||||
tensors=converted_tensors,
|
||||
converted_input_schema, input_tensors = input_schema.convert_humming(
|
||||
tensors=tensors,
|
||||
shape_n_stacks=shape_n_stacks,
|
||||
shape_k_stacks=shape_k_stacks,
|
||||
param_dtype=param_dtype,
|
||||
num_experts=num_experts,
|
||||
)
|
||||
converted_tensors.update(input_tensors)
|
||||
|
||||
_replace_layer_parameters(layer, sublayer_name, converted_tensors)
|
||||
|
||||
@@ -907,11 +411,16 @@ def _prepare_and_transform_sublayer(
|
||||
has_bias=has_bias,
|
||||
num_experts=num_experts,
|
||||
torch_dtype=param_dtype,
|
||||
device=getattr(layer, f"{sublayer_name}_weight").device,
|
||||
)
|
||||
tensors = transform_humming_tensors(
|
||||
config,
|
||||
_extract_sublayer_tensors(layer, sublayer_name),
|
||||
)
|
||||
source_tensors = _extract_sublayer_tensors(layer, sublayer_name)
|
||||
for name in ("input_scale", "input_scale_2"):
|
||||
if name != input_schema.static_tensor_scale_name:
|
||||
source_tensors.pop(name, None)
|
||||
tensors = transform_humming_tensors(config, source_tensors)
|
||||
for name in ("input_scale", "input_scale_2"):
|
||||
if name in source_tensors:
|
||||
tensors[name] = source_tensors[name]
|
||||
_replace_layer_parameters(layer, sublayer_name, tensors)
|
||||
return config
|
||||
|
||||
@@ -927,6 +436,7 @@ def _process_single_sublayer(
|
||||
num_experts: int,
|
||||
param_dtype: torch.dtype,
|
||||
force_weight_schema: Any | None = None,
|
||||
allow_input_schema_fallback: bool = True,
|
||||
) -> tuple[Any, Any, "LayerConfig"]:
|
||||
"""Process a single sublayer: convert, optionally requant, prepare, and transform.
|
||||
|
||||
@@ -944,6 +454,7 @@ def _process_single_sublayer(
|
||||
num_experts: Number of experts
|
||||
param_dtype: Parameter data type
|
||||
force_weight_schema: Optional schema to force requantization to
|
||||
allow_input_schema_fallback: Whether incompatible input schemas may be replaced.
|
||||
|
||||
Returns:
|
||||
Tuple of the final weight schema, input schema, and Humming layer config.
|
||||
@@ -966,19 +477,28 @@ def _process_single_sublayer(
|
||||
# Step 2: Force requant if needed
|
||||
assert isinstance(current_weight_schema, HummingWeightSchema)
|
||||
if force_weight_schema is not None and current_weight_schema != force_weight_schema:
|
||||
tensors = _extract_sublayer_tensors(layer, sublayer_name)
|
||||
source_tensors = _extract_sublayer_tensors(layer, sublayer_name)
|
||||
|
||||
tensors = current_weight_schema.requant_tensors(
|
||||
tensors=tensors,
|
||||
tensors=source_tensors,
|
||||
target_weight_schema=force_weight_schema,
|
||||
param_dtype=param_dtype,
|
||||
)
|
||||
name = current_input_schema.static_tensor_scale_name
|
||||
if name is not None:
|
||||
tensors[name] = source_tensors[name]
|
||||
|
||||
current_weight_schema = force_weight_schema
|
||||
_replace_layer_parameters(layer, sublayer_name, tensors, preserve_bias=True)
|
||||
del tensors
|
||||
|
||||
# Step 3: Prepare layer metadata and transform weights
|
||||
current_input_schema = humming_schema.check_and_fallback_input_schema(
|
||||
weight_schema=current_weight_schema,
|
||||
input_schema=current_input_schema,
|
||||
param_dtype=param_dtype,
|
||||
allow_fallback=allow_input_schema_fallback,
|
||||
)
|
||||
config = _prepare_and_transform_sublayer(
|
||||
layer=layer,
|
||||
sublayer_name=sublayer_name,
|
||||
@@ -1001,6 +521,7 @@ def convert_to_humming_moe_kernel_format(
|
||||
weight_schema: Any | None = None,
|
||||
input_schema: Any | None = None,
|
||||
force_weight_schema: Any | None = None,
|
||||
allow_input_schema_fallback: bool = True,
|
||||
) -> dict[str, "LayerConfig"]:
|
||||
"""Convert MoE weights from checkpoint format to Humming kernel format.
|
||||
|
||||
@@ -1023,6 +544,7 @@ def convert_to_humming_moe_kernel_format(
|
||||
input_schema: Optional initial input quantization schema.
|
||||
If None, built from quant_config or env vars.
|
||||
force_weight_schema: Optional schema to force requantization to
|
||||
allow_input_schema_fallback: Whether incompatible input schemas may be replaced.
|
||||
|
||||
Side effects:
|
||||
- Modifies layer parameters in place
|
||||
@@ -1041,20 +563,23 @@ def convert_to_humming_moe_kernel_format(
|
||||
"Must provide either weight_schema/input_schema or quant_config"
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
humming_is_layer_skipped,
|
||||
)
|
||||
from vllm.utils.humming import BaseWeightSchema, HummingInputSchema
|
||||
|
||||
if weight_schema is None:
|
||||
weight_schema = BaseWeightSchema.from_config(quant_config)
|
||||
|
||||
if input_schema is None:
|
||||
input_quant_config = envs.VLLM_HUMMING_INPUT_QUANT_CONFIG or {}
|
||||
input_quant_config = (envs.VLLM_HUMMING_INPUT_QUANT_CONFIG or {}).copy()
|
||||
if humming_is_layer_skipped(input_quant_config, layer.layer_name):
|
||||
input_schema = HummingInputSchema()
|
||||
else:
|
||||
# TODO: read input_quant_config from quant_config
|
||||
input_quant_config = humming_schema.resolve_humming_layer_config(
|
||||
input_quant_config, layer.layer_name
|
||||
)
|
||||
allow_input_schema_fallback = input_quant_config.pop(
|
||||
"allow_fallback", False
|
||||
)
|
||||
input_schema = HummingInputSchema.from_config(input_quant_config)
|
||||
|
||||
# Build sublayer configs from layer properties if not provided
|
||||
@@ -1089,6 +614,7 @@ def convert_to_humming_moe_kernel_format(
|
||||
num_experts=num_experts,
|
||||
param_dtype=param_dtype,
|
||||
force_weight_schema=force_weight_schema,
|
||||
allow_input_schema_fallback=allow_input_schema_fallback,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -0,0 +1,437 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Map Humming schemas and handle shared checkpoint quantization settings."""
|
||||
|
||||
import dataclasses
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
FP4_DTYPE,
|
||||
FP8_DTYPE,
|
||||
INT4_DTYPE,
|
||||
INT8_DTYPE,
|
||||
MXFP_SCALE_DTYPE,
|
||||
GroupShape,
|
||||
QuantKey,
|
||||
ScaleDesc,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.utils.humming import (
|
||||
BaseInputSchema,
|
||||
BaseWeightSchema,
|
||||
HummingInputSchema,
|
||||
HummingWeightSchema,
|
||||
)
|
||||
from vllm.utils.humming import dtypes as humming_dtypes
|
||||
|
||||
|
||||
def humming_update_schema_hadamard_block_size(
|
||||
weight_schema: "HummingWeightSchema",
|
||||
input_schema: "HummingInputSchema",
|
||||
shape_k: int,
|
||||
) -> "HummingWeightSchema":
|
||||
assert shape_k > 0
|
||||
block_size = 32
|
||||
weight_group_size = weight_schema.weight_scale_group_size
|
||||
input_group_size = input_schema.input_scale_group_size
|
||||
for group_size in (weight_group_size, input_group_size):
|
||||
if group_size is not None and group_size > 0:
|
||||
block_size = min(block_size, group_size)
|
||||
block_size = 1 << (block_size.bit_length() - 1)
|
||||
while shape_k % block_size:
|
||||
block_size //= 2
|
||||
return dataclasses.replace(weight_schema, hadamard_block_size=block_size)
|
||||
|
||||
|
||||
if has_humming():
|
||||
from vllm.utils.humming import dtypes as humming_dtypes
|
||||
|
||||
_HUMMING_TO_QUANT_DTYPE: dict[humming_dtypes.DataType, Any] = {
|
||||
humming_dtypes.float4e0m3: FP4_DTYPE,
|
||||
humming_dtypes.float4e2m1: FP4_DTYPE,
|
||||
humming_dtypes.float6e2m3: scalar_types.float6_e2m3f,
|
||||
humming_dtypes.float6e3m2: scalar_types.float6_e3m2f,
|
||||
humming_dtypes.float8e3m4: FP8_DTYPE,
|
||||
humming_dtypes.float8e4m3: FP8_DTYPE,
|
||||
humming_dtypes.float8e5m2: torch.float8_e5m2,
|
||||
humming_dtypes.int8: torch.int8,
|
||||
humming_dtypes.uint4: INT4_DTYPE,
|
||||
humming_dtypes.uint8: INT8_DTYPE,
|
||||
humming_dtypes.uint2: torch.uint8,
|
||||
humming_dtypes.uint3: torch.uint8,
|
||||
humming_dtypes.uint5: torch.uint8,
|
||||
humming_dtypes.uint6: torch.uint8,
|
||||
humming_dtypes.uint7: torch.uint8,
|
||||
}
|
||||
|
||||
_HUMMING_TO_SCALE_DTYPE: dict[humming_dtypes.DataType, torch.dtype] = {
|
||||
humming_dtypes.float8e8m0: MXFP_SCALE_DTYPE,
|
||||
humming_dtypes.float8e4m3: FP8_DTYPE,
|
||||
humming_dtypes.float16: torch.float16,
|
||||
humming_dtypes.bfloat16: torch.bfloat16,
|
||||
humming_dtypes.float32: torch.float32,
|
||||
}
|
||||
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _group_shape(group_size: int, group_size_n: int = 0) -> GroupShape:
|
||||
"""Map humming group sizes to QuantKey GroupShape.
|
||||
|
||||
group_size: elements per group along K (col); 0 means full dimension.
|
||||
group_size_n: elements per group along N (row); 0 means 1 (per-row).
|
||||
|
||||
GroupShape convention: row = N dim, col = K dim.
|
||||
"""
|
||||
if group_size == 0 and group_size_n == 0:
|
||||
return GroupShape.PER_CHANNEL
|
||||
|
||||
row = group_size_n if group_size_n > 0 else 1
|
||||
col = group_size if group_size > 0 else -1
|
||||
return GroupShape(row=row, col=col)
|
||||
|
||||
|
||||
def weight_schema_to_quant_key(
|
||||
schema: "BaseWeightSchema",
|
||||
param_dtype: torch.dtype | None = None,
|
||||
) -> QuantKey:
|
||||
from vllm.utils.humming import WeightScale2Type, WeightScaleType
|
||||
|
||||
if param_dtype is None:
|
||||
param_dtype = torch.get_default_dtype()
|
||||
if param_dtype not in [torch.float16, torch.bfloat16]:
|
||||
param_dtype = torch.bfloat16
|
||||
if not current_platform.has_device_capability(80):
|
||||
param_dtype = torch.float16
|
||||
|
||||
schema = schema.to_humming_schema(param_dtype)
|
||||
dtype = _HUMMING_TO_QUANT_DTYPE[schema.b_dtype]
|
||||
|
||||
if schema.bs_dtype is not None:
|
||||
scale_dtype = _HUMMING_TO_SCALE_DTYPE[schema.bs_dtype]
|
||||
else:
|
||||
scale_dtype = torch.float32
|
||||
|
||||
group_shape = _group_shape(
|
||||
schema.weight_scale_group_size,
|
||||
schema.weight_scale_group_size_n,
|
||||
)
|
||||
if schema.weight_scale_type == WeightScaleType.TENSOR:
|
||||
group_shape = GroupShape.PER_TENSOR
|
||||
|
||||
scale = ScaleDesc(dtype=scale_dtype, static=True, group_shape=group_shape)
|
||||
|
||||
scale2 = None
|
||||
if schema.weight_scale_2_type == WeightScale2Type.TENSOR:
|
||||
scale2 = ScaleDesc(
|
||||
dtype=torch.float32,
|
||||
static=True,
|
||||
group_shape=GroupShape.PER_TENSOR,
|
||||
)
|
||||
|
||||
return QuantKey(
|
||||
dtype=dtype,
|
||||
scale=scale,
|
||||
scale2=scale2,
|
||||
symmetric=not schema.has_zero_point,
|
||||
)
|
||||
|
||||
|
||||
def input_schema_to_quant_key(
|
||||
schema: "BaseInputSchema",
|
||||
param_dtype: torch.dtype | None = None,
|
||||
) -> QuantKey | None:
|
||||
from vllm.utils.humming import InputQuantizationMode
|
||||
|
||||
if param_dtype is None:
|
||||
param_dtype = torch.get_default_dtype()
|
||||
if param_dtype not in [torch.float16, torch.bfloat16]:
|
||||
param_dtype = torch.bfloat16
|
||||
if not current_platform.has_device_capability(80):
|
||||
param_dtype = torch.float16
|
||||
|
||||
schema = schema.to_humming_schema(param_dtype)
|
||||
if schema.a_dtype is None or schema.a_dtype.num_bits >= 16:
|
||||
return None
|
||||
|
||||
mode = schema.input_quant_mode
|
||||
if mode == InputQuantizationMode.Disabled:
|
||||
return None
|
||||
dtype = _HUMMING_TO_QUANT_DTYPE[schema.a_dtype]
|
||||
|
||||
gs = schema.input_scale_group_size
|
||||
group_shape = GroupShape(row=1, col=gs) if gs > 0 else GroupShape.PER_TOKEN
|
||||
|
||||
# Pick the scale dtype the Humming kernel actually consumes. An explicit
|
||||
# input_scale_dtype always wins. Otherwise infer from the grouping: MX
|
||||
# microscale activations (group size 32, e.g. MXFP8) carry an e8m0 (uint8)
|
||||
# scale, while block-FP8 (group size 128) and per-token FP8/int8 carry a
|
||||
# float32 scale. Getting this right lets a grouped FP8 activation match
|
||||
# kFp8Dynamic128Sym instead of an unmatchable uint8-scaled key.
|
||||
if schema.input_scale_dtype is not None:
|
||||
scale_dtype = _HUMMING_TO_SCALE_DTYPE[schema.input_scale_dtype]
|
||||
elif gs == 16:
|
||||
scale_dtype = FP8_DTYPE
|
||||
elif gs == 32:
|
||||
scale_dtype = MXFP_SCALE_DTYPE
|
||||
else:
|
||||
scale_dtype = torch.float32
|
||||
|
||||
if mode == InputQuantizationMode.StaticTensor:
|
||||
group_shape = GroupShape.PER_TENSOR
|
||||
scale = ScaleDesc(
|
||||
dtype=scale_dtype,
|
||||
static=mode == InputQuantizationMode.StaticTensor,
|
||||
group_shape=group_shape,
|
||||
)
|
||||
scale2 = None
|
||||
has_scale2 = (
|
||||
InputQuantizationMode.StaticTensorDynamicGroup,
|
||||
InputQuantizationMode.DynamicGroupToken,
|
||||
)
|
||||
if mode in has_scale2:
|
||||
static = mode == InputQuantizationMode.StaticTensorDynamicGroup
|
||||
scale2 = ScaleDesc(
|
||||
torch.float32,
|
||||
static,
|
||||
GroupShape.PER_TENSOR if static else GroupShape.PER_TOKEN,
|
||||
)
|
||||
|
||||
return QuantKey(dtype=dtype, scale=scale, scale2=scale2, symmetric=True)
|
||||
|
||||
|
||||
def quant_key_to_input_schema(key: QuantKey | None) -> "HummingInputSchema":
|
||||
from vllm.utils.humming import HummingInputSchema, InputQuantizationMode
|
||||
|
||||
if key is None:
|
||||
return HummingInputSchema(input_quant_mode=InputQuantizationMode.Disabled)
|
||||
|
||||
if not key.symmetric:
|
||||
raise ValueError("Humming input quantization must be symmetric")
|
||||
|
||||
quant_dtypes = {value: dtype for dtype, value in _HUMMING_TO_QUANT_DTYPE.items()}
|
||||
quant_dtypes[FP4_DTYPE] = humming_dtypes.float4e2m1
|
||||
quant_dtypes[FP8_DTYPE] = humming_dtypes.float8e4m3
|
||||
scale_dtypes = {value: dtype for dtype, value in _HUMMING_TO_SCALE_DTYPE.items()}
|
||||
if key.dtype not in quant_dtypes or key.scale.dtype not in scale_dtypes:
|
||||
raise ValueError(f"Unsupported Humming input or scale dtype: {key}")
|
||||
|
||||
scale = key.scale
|
||||
group_size = 0
|
||||
if scale.static and scale.group_shape.is_per_tensor() and key.scale2 is None:
|
||||
mode = InputQuantizationMode.StaticTensor
|
||||
elif not scale.static and scale.group_shape.is_per_token() and key.scale2 is None:
|
||||
mode = InputQuantizationMode.DynamicToken
|
||||
elif not scale.static and scale.group_shape.is_per_group():
|
||||
group_size = scale.group_shape.col
|
||||
scale2 = key.scale2
|
||||
if scale2 is None:
|
||||
mode = InputQuantizationMode.DynamicGroup
|
||||
elif scale2.dtype != torch.float32:
|
||||
raise ValueError("Humming secondary input scales must be float32")
|
||||
elif scale2.static and scale2.group_shape.is_per_tensor():
|
||||
mode = InputQuantizationMode.StaticTensorDynamicGroup
|
||||
elif not scale2.static and scale2.group_shape.is_per_token():
|
||||
mode = InputQuantizationMode.DynamicGroupToken
|
||||
else:
|
||||
raise ValueError(f"Unsupported Humming secondary input scale: {scale2}")
|
||||
else:
|
||||
raise ValueError(f"Unsupported Humming input scale: {key}")
|
||||
|
||||
if group_size == 0 and scale.dtype != torch.float32:
|
||||
raise ValueError("Humming tensor and token input scales must be float32")
|
||||
|
||||
return HummingInputSchema(
|
||||
a_dtype=quant_dtypes[key.dtype],
|
||||
input_scale_group_size=group_size,
|
||||
input_scale_dtype=scale_dtypes[scale.dtype],
|
||||
input_quant_mode=mode,
|
||||
)
|
||||
|
||||
|
||||
def check_and_fallback_input_schema(
|
||||
weight_schema: "BaseWeightSchema",
|
||||
input_schema: "BaseInputSchema",
|
||||
param_dtype: torch.dtype | None = None,
|
||||
allow_fallback: bool = True,
|
||||
) -> "HummingInputSchema":
|
||||
from vllm.utils.humming import HummingInputSchema, InputQuantizationMode
|
||||
|
||||
capability = current_platform.get_device_capability()
|
||||
assert capability is not None
|
||||
sm_version = capability.to_int()
|
||||
if param_dtype is None:
|
||||
param_dtype = torch.get_default_dtype()
|
||||
if param_dtype not in (torch.float16, torch.bfloat16):
|
||||
param_dtype = torch.bfloat16 if sm_version >= 80 else torch.float16
|
||||
assert param_dtype in (torch.float16, torch.bfloat16)
|
||||
|
||||
dtype_fallback_order_map = {
|
||||
humming_dtypes.float8e5m2: (
|
||||
humming_dtypes.float8e4m3,
|
||||
humming_dtypes.float8e3m4,
|
||||
humming_dtypes.int8,
|
||||
),
|
||||
humming_dtypes.float8e4m3: (
|
||||
humming_dtypes.float8e3m4,
|
||||
humming_dtypes.int8,
|
||||
),
|
||||
humming_dtypes.float8e3m4: (
|
||||
humming_dtypes.float8e4m3,
|
||||
humming_dtypes.int8,
|
||||
),
|
||||
humming_dtypes.float4e2m1: (
|
||||
humming_dtypes.float8e4m3,
|
||||
humming_dtypes.float8e3m4,
|
||||
humming_dtypes.int8,
|
||||
),
|
||||
humming_dtypes.float4e0m3: (
|
||||
humming_dtypes.int4,
|
||||
humming_dtypes.float8e4m3,
|
||||
humming_dtypes.float8e3m4,
|
||||
humming_dtypes.int8,
|
||||
),
|
||||
humming_dtypes.int8: (
|
||||
humming_dtypes.float8e4m3,
|
||||
humming_dtypes.float8e3m4,
|
||||
),
|
||||
humming_dtypes.int4: (
|
||||
humming_dtypes.int8,
|
||||
humming_dtypes.float8e4m3,
|
||||
humming_dtypes.float8e3m4,
|
||||
),
|
||||
}
|
||||
|
||||
weight_schema = weight_schema.to_humming_schema(param_dtype)
|
||||
input_schema = input_schema.to_humming_schema(param_dtype)
|
||||
fp6_dtypes = (humming_dtypes.float6e2m3, humming_dtypes.float6e3m2)
|
||||
if input_schema.a_dtype in fp6_dtypes:
|
||||
input_schema.a_dtype = humming_dtypes.float8e4m3
|
||||
a_dtype = input_schema.a_dtype
|
||||
input_bits = a_dtype.num_bits if a_dtype is not None else 16
|
||||
weight_group_size = weight_schema.weight_scale_group_size
|
||||
input_group_size = input_schema.input_scale_group_size
|
||||
input_scale_dtype = input_schema.input_scale_dtype
|
||||
|
||||
def is_deprecated(dtype: "humming_dtypes.DataType | None") -> bool:
|
||||
if dtype is None or dtype.num_bits == 16:
|
||||
return False
|
||||
is_int4_deprecated = dtype == humming_dtypes.int4 and sm_version >= 90
|
||||
is_int8_deprecated = dtype == humming_dtypes.int8 and 103 <= sm_version < 110
|
||||
# Humming has not yet implemented high-performance FP8 computation
|
||||
# on SM10X and SM11X. Therefore, it is temporarily considered deprecated.
|
||||
# TODO: removed this after Humming has high-performance FP8 computation
|
||||
is_fp8_deprecated = dtype.num_bits < 16 and 100 <= sm_version < 120
|
||||
return is_int4_deprecated or is_int8_deprecated or is_fp8_deprecated
|
||||
|
||||
if input_schema.is_compatible_with(weight_schema, param_dtype):
|
||||
if not allow_fallback:
|
||||
if is_deprecated(a_dtype):
|
||||
msg = f"Humming {a_dtype} activation is slow on SM{sm_version} now."
|
||||
logger.warning_once(msg)
|
||||
return input_schema
|
||||
|
||||
if a_dtype is None or a_dtype.num_bits == 16:
|
||||
return input_schema
|
||||
|
||||
if not is_deprecated(a_dtype):
|
||||
is_mxfp8 = (
|
||||
a_dtype.is_floating_point_type
|
||||
and a_dtype.num_bits == 8
|
||||
and input_group_size == 32
|
||||
and input_scale_dtype == humming_dtypes.float8e8m0
|
||||
and sm_version >= 120
|
||||
)
|
||||
is_groupwise = input_group_size > 0 or weight_group_size > 0
|
||||
if input_bits == 8 and is_groupwise and not is_mxfp8:
|
||||
# prefer wna16 for groupwise weight or groupwise input
|
||||
return HummingInputSchema()
|
||||
|
||||
return input_schema
|
||||
|
||||
if allow_fallback:
|
||||
fallback_dtypes = (
|
||||
a_dtype,
|
||||
*dtype_fallback_order_map.get(a_dtype, ()),
|
||||
humming_dtypes.DataType.from_any(param_dtype),
|
||||
)
|
||||
for dtype in fallback_dtypes:
|
||||
if dtype is None or is_deprecated(dtype):
|
||||
continue
|
||||
|
||||
group_size = 0
|
||||
scale_dtype = humming_dtypes.float32
|
||||
quant_mode = InputQuantizationMode.DynamicToken
|
||||
if dtype.num_bits == 16:
|
||||
scale_dtype = None
|
||||
quant_mode = InputQuantizationMode.Disabled
|
||||
elif dtype in (humming_dtypes.float8e4m3, humming_dtypes.float8e3m4):
|
||||
is_channelwise_weight = weight_schema.weight_scale_group_size == 0
|
||||
weight_group_size = weight_schema.weight_scale_group_size
|
||||
is_e8m0_scale = weight_schema.bs_dtype == humming_dtypes.float8e8m0
|
||||
is_mx_weight = weight_group_size == 32 and is_e8m0_scale
|
||||
if sm_version >= 120 and (is_channelwise_weight or is_mx_weight):
|
||||
group_size = 32
|
||||
scale_dtype = humming_dtypes.float8e8m0
|
||||
quant_mode = InputQuantizationMode.DynamicGroup
|
||||
|
||||
candidate = HummingInputSchema(
|
||||
a_dtype=dtype,
|
||||
input_scale_group_size=group_size,
|
||||
input_scale_dtype=scale_dtype,
|
||||
input_quant_mode=quant_mode,
|
||||
)
|
||||
if candidate.is_compatible_with(weight_schema, param_dtype):
|
||||
return candidate
|
||||
|
||||
raise ValueError(
|
||||
f"No compatible Humming input schema for {input_schema} with "
|
||||
f"weight schema {weight_schema} on SM{sm_version} "
|
||||
f"({allow_fallback=})"
|
||||
)
|
||||
|
||||
|
||||
def resolve_humming_layer_config(config: dict[str, Any], prefix: str) -> dict[str, Any]:
|
||||
layer_config = config.copy()
|
||||
dynamic = config.get("dynamic", {})
|
||||
if isinstance(dynamic, dict):
|
||||
for pattern, overrides in dynamic.items():
|
||||
if pattern.startswith("+") and re.match(pattern[2:], prefix):
|
||||
layer_config.update(overrides)
|
||||
break
|
||||
return layer_config
|
||||
|
||||
|
||||
def humming_is_layer_skipped(config: dict[str, Any], prefix: str):
|
||||
if not config:
|
||||
return True
|
||||
|
||||
keys = ["ignored_layers", "ignore", "modules_to_not_convert"]
|
||||
ignored_layers: list[str] = []
|
||||
for key in keys:
|
||||
candidate = config.get(key, []) or []
|
||||
if candidate:
|
||||
ignored_layers = candidate
|
||||
break
|
||||
|
||||
if any(module_name in prefix for module_name in ignored_layers):
|
||||
return True
|
||||
if "lm_head" in prefix:
|
||||
return True
|
||||
|
||||
for regex in config.get("dynamic", {}):
|
||||
if regex[:1] != "-":
|
||||
continue
|
||||
if re.match(regex[2:], prefix):
|
||||
return True
|
||||
|
||||
return False
|
||||
Reference in New Issue
Block a user