[Feature][Humming] Humming feature integration (#56685)

Signed-off-by: jinzhen.ljz <jinzhen.ljz@antgroup.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Jinzhen Lin
2026-09-20 08:11:01 -07:00
committed by GitHub
co-authored by Codex mgoin
parent 27757dde02
commit 7d99c2c4fd
62 changed files with 2040 additions and 1086 deletions
@@ -0,0 +1,51 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Humming quantization integration."""
from vllm.model_executor.layers.quantization.utils.humming.activation import (
get_humming_activation,
)
from vllm.model_executor.layers.quantization.utils.humming.linear import (
apply_humming_linear,
convert_linear_layer_to_humming_standard,
get_humming_linear_compute_config,
prepare_humming_linear_layer_config,
)
from vllm.model_executor.layers.quantization.utils.humming.moe import (
HummingMoEQuantConfig,
convert_to_humming_moe_kernel_format,
get_humming_moe_quant_config,
make_humming_moe_kernel,
make_humming_moe_quant_config,
select_humming_moe_experts,
)
from vllm.model_executor.layers.quantization.utils.humming.schema import (
check_and_fallback_input_schema,
humming_is_layer_skipped,
humming_update_schema_hadamard_block_size,
input_schema_to_quant_key,
quant_key_to_input_schema,
resolve_humming_layer_config,
weight_schema_to_quant_key,
)
__all__ = [
"check_and_fallback_input_schema",
"get_humming_activation",
"HummingMoEQuantConfig",
"get_humming_moe_quant_config",
"humming_is_layer_skipped",
"humming_update_schema_hadamard_block_size",
"make_humming_moe_quant_config",
"apply_humming_linear",
"convert_linear_layer_to_humming_standard",
"get_humming_linear_compute_config",
"prepare_humming_linear_layer_config",
"make_humming_moe_kernel",
"select_humming_moe_experts",
"input_schema_to_quant_key",
"quant_key_to_input_schema",
"resolve_humming_layer_config",
"weight_schema_to_quant_key",
"convert_to_humming_moe_kernel_format",
]
@@ -0,0 +1,126 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MoE activation expressions for Humming input processing."""
from dataclasses import replace
from vllm.model_executor.layers.fused_moe.activation import (
ApplyMoEActivationConfig,
MoEActivation,
)
_SILU_IMPL = "a / (1.f + expf(-a))"
_GELU_IMPL = "0.5f * a * (1.f + erff(a * 0.7071067811865476f))"
_RELU2_IMPL = "fmaxf(a, 0.f) * fmaxf(a, 0.f)"
_GELU_TANH_IMPL = """[](float a) {
const float cubic = 0.044715f * a * a * a;
const float inner = 0.7978845608028654f * (a + cubic);
return 0.5f * a * (1.f + tanhf(inner));
}(a)"""
_CLAMPED_SILU_IMPL = """[](float a, float b) {{
const float gate = fminf(a, {limit});
const float up = fmaxf(fminf(b, {limit}), -{limit});
const float activated_gate = gate / (1.f + expf(-{alpha} * gate));
return activated_gate * (up + {beta});
}}(a, b)"""
# Parameterized expressions are resolved by get_humming_activation().
_HUMMING_ACTIVATIONS: dict[MoEActivation, dict[str, str]] = {
MoEActivation.SILU: {
"activation_type": "binary_split",
"activation_impl": f"({_SILU_IMPL}) * b",
},
MoEActivation.GELU: {
"activation_type": "binary_split",
"activation_impl": f"({_GELU_IMPL}) * b",
},
MoEActivation.GELU_TANH: {
"activation_type": "binary_split",
"activation_impl": f"({_GELU_TANH_IMPL}) * b",
},
MoEActivation.RELU2: {
"activation_type": "binary_split",
"activation_impl": f"({_RELU2_IMPL}) * b",
},
MoEActivation.SWIGLUOAI: {
"activation_type": "binary_interleaved",
"activation_impl": _CLAMPED_SILU_IMPL,
},
MoEActivation.SITU: {
"activation_type": "binary_split",
"activation_impl": """[](float a, float b) {{
const float gate = {situ_beta} * tanhf(a / {situ_beta});
const float activated_gate = gate / (1.f + expf(-a));
const float up = {situ_up};
return activated_gate * up;
}}(a, b)""",
},
MoEActivation.SWIGLUOAI_UNINTERLEAVE: {
"activation_type": "binary_split",
"activation_impl": _CLAMPED_SILU_IMPL,
},
MoEActivation.SWIGLUSTEP: {
"activation_type": "binary_split",
"activation_impl": """[](float a, float b) {
const float activated_gate = a / (1.f + expf(-a));
const float gate = fminf(activated_gate, 7.f);
const float up = fmaxf(fminf(b, 7.f), -7.f);
return gate * up;
}(a, b)""",
},
MoEActivation.SILU_NO_MUL: {
"activation_type": "unary",
"activation_impl": _SILU_IMPL,
},
MoEActivation.GELU_NO_MUL: {
"activation_type": "unary",
"activation_impl": _GELU_IMPL,
},
MoEActivation.GELU_TANH_NO_MUL: {
"activation_type": "unary",
"activation_impl": _GELU_TANH_IMPL,
},
MoEActivation.RELU2_NO_MUL: {
"activation_type": "unary",
"activation_impl": _RELU2_IMPL,
},
}
def get_humming_activation(
activation: MoEActivation,
config: ApplyMoEActivationConfig | None = None,
) -> dict[str, str]:
config = config or ApplyMoEActivationConfig()
use_clamped_silu = False
if activation == MoEActivation.SWIGLUOAI:
config = ApplyMoEActivationConfig(clamp_limit=7.0, alpha=1.702, beta=1.0)
use_clamped_silu = True
elif activation == MoEActivation.SILU and config.clamp_limit is not None:
config = replace(config, alpha=1.0, beta=0.0)
use_clamped_silu = True
elif activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
if config.clamp_limit is None:
raise ValueError("SWIGLUOAI_UNINTERLEAVE requires clamp_limit")
use_clamped_silu = True
definition = _HUMMING_ACTIVATIONS[activation].copy()
if activation == MoEActivation.SITU:
if config.activation_situ_beta is None:
raise ValueError("SITU requires activation_situ_beta")
linear_beta = config.activation_situ_linear_beta
situ_up = "b"
if linear_beta is not None and linear_beta > 0:
situ_up = f"({float(linear_beta)}f) * tanhf(b / ({float(linear_beta)}f))"
situ_beta = f"({float(config.activation_situ_beta)}f)"
activation_impl = definition["activation_impl"]
activation_impl = activation_impl.format(situ_beta=situ_beta, situ_up=situ_up)
definition["activation_impl"] = activation_impl
elif use_clamped_silu:
assert config.clamp_limit is not None
limit = f"({float(config.clamp_limit)}f)"
alpha = f"({float(config.alpha)}f)"
beta = f"({float(config.beta)}f)"
activation_impl = _CLAMPED_SILU_IMPL.format(limit=limit, alpha=alpha, beta=beta)
definition["activation_impl"] = activation_impl
return definition
@@ -0,0 +1,183 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Prepare and execute Humming linear layers."""
import json
from typing import TYPE_CHECKING
import torch
from vllm import envs
from vllm.model_executor.layers.linear import LinearBase
from vllm.model_executor.layers.quantization.utils.humming.schema import (
check_and_fallback_input_schema,
)
if TYPE_CHECKING:
from vllm.utils.humming import BaseInputSchema, LayerConfig
def convert_linear_layer_to_humming_standard(
layer: LinearBase, name_map: dict[str, str]
):
"""Rename/reshape a linear layer's quantized params (the canonical MPLinear
layout: ``weight_packed`` int32 + ``weight_scale``) into the parameter names
and layout humming's weight schema expects (``weight`` / ``weight_scale``)."""
for name, checkpoint_name in name_map.items():
tensor = getattr(layer, checkpoint_name)
delattr(layer, checkpoint_name)
if name == "weight":
input_dim = getattr(tensor, "input_dim", 1)
output_dim = getattr(tensor, "output_dim", 0)
if input_dim == 0 and output_dim == 1:
tensor = tensor.transpose(1, 0).contiguous()
else:
assert output_dim == 0 and input_dim == 1
tensor = tensor.view(tensor.size(0), -1).view(torch.int32)
elif name in ["weight_scale", "zero_point"]:
if getattr(tensor, "output_dim", 0) == 1:
tensor = tensor.transpose(0, 1).contiguous()
if tensor.ndim == 1:
tensor = tensor.unsqueeze(1)
tensor = tensor.view(torch.int32) if name == "zero_point" else tensor
if isinstance(tensor, torch.nn.Parameter):
param = tensor
else:
param = torch.nn.Parameter(tensor, requires_grad=False)
setattr(layer, name, param)
def prepare_humming_linear_layer_config(
layer: LinearBase,
quant_config: dict,
input_quant_config: dict | None = None,
*,
input_schema: "BaseInputSchema | None" = None,
) -> "LayerConfig":
from vllm.utils.humming import (
BaseInputSchema,
BaseWeightSchema,
HummingInputSchema,
prepare_layer_config,
transform_humming_tensors,
)
weight_schema = BaseWeightSchema.from_config(quant_config)
if input_schema is None:
if input_quant_config is not None:
input_schema = BaseInputSchema.from_config(input_quant_config)
else:
input_schema = HummingInputSchema()
# ReplicatedLinear has no TP partitioning and so does not set
# input_size_per_partition; for it that is just input_size. Use hasattr
# rather than getattr's default arg, which is evaluated eagerly and would
# raise on layers lacking input_size (e.g. ParallelLMHead).
if hasattr(layer, "input_size_per_partition"):
input_size_per_partition = layer.input_size_per_partition
else:
input_size_per_partition = layer.input_size
shape_k_stacks = [input_size_per_partition]
shape_n_stacks = layer.output_partition_sizes
# Step 1: convert weight and input schemas to humming standard format
source_tensors = dict(layer.named_parameters())
weight_schema, tensors = weight_schema.convert_humming(
tensors=source_tensors,
shape_n_stacks=shape_n_stacks,
shape_k_stacks=shape_k_stacks,
param_dtype=layer.params_dtype,
)
input_schema, input_tensors = input_schema.convert_humming(
tensors=source_tensors,
shape_n_stacks=shape_n_stacks,
shape_k_stacks=shape_k_stacks,
param_dtype=layer.params_dtype,
)
tensors.update(input_tensors)
layer.weight_schema = weight_schema
input_schema = check_and_fallback_input_schema(
weight_schema=weight_schema,
input_schema=input_schema,
param_dtype=layer.params_dtype,
)
for name in ("input_scale", "input_scale_2"):
if name != input_schema.static_tensor_scale_name:
tensors.pop(name, None)
input_tensors.pop(name, None)
# Step 2: transform weight (humming standard format) for forwarding.
config = prepare_layer_config(
shape_n=sum(layer.output_partition_sizes),
shape_k=input_size_per_partition,
weight_schema=weight_schema,
input_schema=input_schema,
pad_n_to_multiple=256,
pad_k_to_multiple=128,
has_bias=layer.has_bias,
torch_dtype=layer.params_dtype,
device=tensors["weight"].device,
)
tensors = transform_humming_tensors(config, tensors)
tensors.update(input_tensors)
if "bias" in tensors:
zero_bias = torch.zeros_like(tensors["bias"])
layer.register_buffer("zero_bias", zero_bias)
for name, _ in list(layer.named_parameters()):
delattr(layer, name)
for name, tensor in tensors.items():
param = torch.nn.Parameter(tensor, requires_grad=False)
setattr(layer, name, param)
return config
def get_humming_linear_compute_config() -> str:
return json.dumps(
{
"use_batch_invariant": envs.VLLM_BATCH_INVARIANT,
"use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,
"gemm_type": "dense",
}
)
def apply_humming_linear(
layer: LinearBase,
x: torch.Tensor,
*,
skip_bias_add: bool = False,
layer_config: "LayerConfig",
compute_config: str,
locks: torch.Tensor,
) -> torch.Tensor:
from vllm.utils.humming import humming_forward
mode = layer_config.input_quant_mode
input_scale = getattr(layer, "input_scale", None) if mode.should_quantize else None
input_scale_2 = (
getattr(layer, "input_scale_2", None) if mode.has_secondary_scale else None
)
flatten_inputs = x.reshape(-1, x.size(-1))
output = humming_forward(
layer_config,
inputs=flatten_inputs,
weight=layer.weight,
weight_scale=getattr(layer, "weight_scale", None),
zero_point=getattr(layer, "zero_point", None),
bias=getattr(layer, "zero_bias" if skip_bias_add else "bias", None),
weight_scale_2=getattr(layer, "weight_scale_2", None),
input_scale=input_scale,
input_scale_2=input_scale_2,
hadamard_block_size=layer.weight_schema.hadamard_block_size,
locks=locks,
compute_config=compute_config,
)
return output.view(*x.shape[:-1], output.size(-1))
@@ -0,0 +1,626 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Configure, prepare weights for, and assemble Humming MoE kernels."""
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
import vllm.model_executor.layers.quantization.utils.humming.schema as humming_schema
from vllm import envs
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEQuantConfig,
FusedMoEQuantDesc,
)
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
BatchedHummingGroupedExperts,
HummingGroupedExperts,
HummingIndexedExperts,
)
from vllm.model_executor.layers.quantization.utils.humming.schema import (
humming_is_layer_skipped,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
FP8_DTYPE,
GroupShape,
QuantKey,
)
from vllm.utils.import_utils import has_humming
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
from vllm.utils.humming import LayerConfig
logger = init_logger(__name__)
@dataclass(kw_only=True)
class HummingMoEQuantConfig(FusedMoEQuantConfig):
w1_humming_config: "LayerConfig"
w2_humming_config: "LayerConfig"
w1_hadamard_block_size: int = 1
w2_hadamard_block_size: int = 1
w1_input_scale: torch.Tensor | None = None
w2_input_scale: torch.Tensor | None = None
w1_input_scale_2: torch.Tensor | None = None
w2_input_scale_2: torch.Tensor | None = None
def make_humming_moe_quant_config(
quant_dtype: torch.dtype | str | None,
weight_dtype: torch.dtype | str | None,
weight_group_shape: GroupShape | None = None,
activation_group_shape: GroupShape | None = None,
w1_scale: torch.Tensor | None = None,
w2_scale: torch.Tensor | None = None,
w1_zp: torch.Tensor | None = None,
w2_zp: torch.Tensor | None = None,
w1_bias: torch.Tensor | None = None,
w2_bias: torch.Tensor | None = None,
w1_gscale: torch.Tensor | None = None,
w2_gscale: torch.Tensor | None = None,
gemm1_alpha: float | None = None,
gemm1_beta: float | None = None,
gemm1_clamp_limit: float | None = None,
humming_configs: dict[str, "LayerConfig"] | None = None,
) -> HummingMoEQuantConfig:
assert humming_configs is not None
if quant_dtype is None:
a_quant_desc = FusedMoEQuantDesc(dtype=None)
elif activation_group_shape is not None:
# Pre-dispatch quantization.
a_quant_desc = FusedMoEQuantDesc(
dtype=quant_dtype, shape=activation_group_shape
)
else:
# Deferred path: Humming quantizes the activation internally, so the
# descriptor only needs a non-None dtype to mark it as quantized.
shape = GroupShape(row=1, col=-1)
a_quant_desc = FusedMoEQuantDesc(dtype=quant_dtype, shape=shape)
w1_quant_desc = FusedMoEQuantDesc(
dtype=weight_dtype,
shape=weight_group_shape,
scale=w1_scale,
alpha_or_gscale=w1_gscale,
zp=w1_zp,
bias=w1_bias,
)
w2_quant_desc = FusedMoEQuantDesc(
dtype=weight_dtype,
shape=weight_group_shape,
scale=w2_scale,
alpha_or_gscale=w2_gscale,
zp=w2_zp,
bias=w2_bias,
)
return HummingMoEQuantConfig(
_a1=a_quant_desc,
_a2=a_quant_desc,
_w1=w1_quant_desc,
_w2=w2_quant_desc,
gemm1_alpha=gemm1_alpha,
gemm1_beta=gemm1_beta,
gemm1_clamp_limit=gemm1_clamp_limit,
w1_humming_config=humming_configs["w13"],
w2_humming_config=humming_configs["w2"],
)
def get_humming_moe_quant_config(
layer: "RoutedExperts",
humming_configs: dict[str, "LayerConfig"] | None = None,
gemm1_alpha: float | None = None,
gemm1_beta: float | None = None,
gemm1_clamp_limit: float | None = None,
):
if humming_configs is None:
humming_configs = layer.humming_configs
input_schema = layer.input_schemas["w13"]
weight_schema = layer.weight_schemas["w13"]
if input_schema.a_dtype is None or input_schema.a_dtype.num_bits == 16:
q_dtype = None
else:
q_dtype = str(input_schema.a_dtype)
# Block-FP8 (group-128) activations are quantized *before* the EP all-to-all
# dispatch (so FP8 rather than BF16 crosses the interconnect) and consumed by
# Humming as-is.
activation_group_shape: GroupShape | None = None
input_scale_group_size = getattr(input_schema, "input_scale_group_size", 0) or 0
if (
q_dtype is not None
and q_dtype.startswith("float8")
and input_scale_group_size == 128
and weight_schema.hadamard_block_size <= 1
and input_schema.input_quant_mode in (None, "dynamic_group")
):
q_dtype = humming_schema._HUMMING_TO_QUANT_DTYPE.get(
input_schema.a_dtype, FP8_DTYPE
)
activation_group_shape = GroupShape(row=1, col=input_scale_group_size)
weight_scale_group_size = weight_schema.weight_scale_group_size
weight_scale_group_size_n = weight_schema.weight_scale_group_size_n
weight_group_shape: tuple[int, ...] = ()
if weight_scale_group_size_n > 1:
weight_group_shape = GroupShape(
row=weight_scale_group_size,
col=weight_scale_group_size_n,
)
elif weight_scale_group_size == 0:
weight_group_shape = GroupShape(row=-1, col=1)
else:
weight_group_shape = GroupShape(row=weight_scale_group_size, col=1)
config = make_humming_moe_quant_config(
quant_dtype=q_dtype,
weight_dtype=str(weight_schema.b_dtype),
weight_group_shape=weight_group_shape,
activation_group_shape=activation_group_shape,
w1_scale=getattr(layer, "w13_weight_scale", None),
w1_gscale=getattr(layer, "w13_weight_scale_2", None),
w1_zp=getattr(layer, "w13_zero_point", None),
w1_bias=getattr(layer, "w13_bias", None),
w2_scale=getattr(layer, "w2_weight_scale", None),
w2_gscale=getattr(layer, "w2_weight_scale_2", None),
w2_zp=getattr(layer, "w2_zero_point", None),
w2_bias=getattr(layer, "w2_bias", None),
gemm1_alpha=gemm1_alpha,
gemm1_beta=gemm1_beta,
gemm1_clamp_limit=gemm1_clamp_limit,
humming_configs=humming_configs,
)
for prefix, sublayer in (("w1", "w13"), ("w2", "w2")):
setattr(
config,
f"{prefix}_hadamard_block_size",
layer.weight_schemas[sublayer].hadamard_block_size,
)
for name in ("input_scale", "input_scale_2"):
key = f"{prefix}_{name}"
value = getattr(layer, f"{sublayer}_{name}", None)
setattr(config, key, value)
return config
def select_humming_moe_experts(
config: FusedMoEConfig,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> type[mk.FusedMoEExperts] | None:
"""Select the primary Humming MoE Experts class
Note: Shape-specific fallbacks may still occur at runtime.
"""
if not has_humming():
return None
# NOTE: the kernels are selected in the following order.
AVAILABLE_EXPERTS: list[type[mk.FusedMoEExperts]] = [
BatchedHummingGroupedExperts,
HummingGroupedExperts,
HummingIndexedExperts,
]
# NOTE(rob): We need to peak into the P/F selection to determine
# if we are using the batched or standard expert format, which
# if not ideal. Once we unify TP + DP/EP, we can select P/F first.
activation_format = (
mk.FusedMoEActivationFormat.BatchedExperts
if config.moe_parallel_config.use_batched_activation_format
else mk.FusedMoEActivationFormat.Standard
)
def _make_log_backend(experts_cls: type[mk.FusedMoEExperts]):
return f"Using {experts_cls.__name__} Humming MoE backend."
def _make_log_unsupported(
experts_cls: type[mk.FusedMoEExperts], reason: str | None
) -> str:
if reason:
return (
f"Humming MoE experts {experts_cls.__name__} does not support the "
f"deployment configuration since {reason}."
)
else:
return (
f"Humming MoE experts '{experts_cls.__name__}' does not support the "
"deployment configuration."
)
for k_cls in AVAILABLE_EXPERTS:
supported, reason = k_cls.is_supported_config(
k_cls,
config,
weight_key,
activation_key,
activation_format,
)
if supported:
logger.info_once(_make_log_backend(k_cls))
return k_cls
else:
logger.debug_once(_make_log_unsupported(k_cls, reason))
return None
def make_humming_moe_kernel(
moe_quant_config: FusedMoEQuantConfig,
moe_config: FusedMoEConfig,
experts_cls: type[mk.FusedMoEExperts],
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
) -> mk.FusedMoEKernel:
# Create Prepare/Finalize.
prepare_finalize = maybe_make_prepare_finalize(
moe=moe_config,
quant_config=moe_quant_config,
routing_tables=routing_tables,
allow_new_interface=True,
use_monolithic=issubclass(experts_cls, mk.FusedMoEExpertsMonolithic),
)
assert prepare_finalize is not None
logger.info_once("Using %s", prepare_finalize.__class__.__name__)
# Create Experts.
if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
assert max_num_tokens is not None
experts = experts_cls(
moe_config=moe_config,
quant_config=moe_quant_config,
max_num_tokens=max_num_tokens,
num_dispatchers=prepare_finalize.num_dispatchers(),
)
else:
experts = experts_cls(
moe_config=moe_config,
quant_config=moe_quant_config,
)
kernel = mk.FusedMoEKernel(
prepare_finalize,
experts,
)
return kernel
def _extract_sublayer_tensors(
layer: "RoutedExperts",
sublayer_name: str,
) -> dict[str, torch.Tensor]:
"""Extract tensors for a specific sublayer from the layer's state dict."""
return dict(
(key.removeprefix(sublayer_name + "_"), value)
for key, value in layer.state_dict().items()
if key.startswith(sublayer_name + "_")
)
def _replace_layer_parameters(
layer: "RoutedExperts",
sublayer_name: str,
tensors: dict[str, torch.Tensor],
preserve_bias: bool = False,
) -> None:
"""Replace layer parameters for a sublayer with new tensors.
Args:
layer: The RoutedExperts layer
sublayer_name: Name of the sublayer (e.g., "w13", "w2")
tensors: Dict of parameter name to tensor
preserve_bias: If True, don't delete bias parameters
"""
# Delete old parameters
for name, _ in list(layer.named_parameters()):
if not name.startswith(sublayer_name + "_"):
continue
if preserve_bias and name == sublayer_name + "_bias":
continue
delattr(layer, name)
# Set new parameters
for name, tensor in tensors.items():
param_name = f"{sublayer_name}_{name}"
param = torch.nn.Parameter(tensor, requires_grad=False)
setattr(layer, param_name, param)
def _convert_sublayer_to_humming(
layer: "RoutedExperts",
sublayer_name: str,
shape_n: int,
shape_k: int,
weight_schema: Any,
input_schema: Any,
num_experts: int,
param_dtype: torch.dtype,
) -> tuple[Any, Any]:
"""Convert a sublayer's weights from checkpoint format to Humming format.
Returns:
Tuple of (converted_weight_schema, converted_input_schema)
"""
tensors = _extract_sublayer_tensors(layer, sublayer_name)
shape_k_stacks = [shape_k]
shape_n_stacks = [shape_n]
if sublayer_name == "w13" and layer.moe_config.activation.is_gated:
shape_n_stacks = [shape_n // 2] * 2
converted_weight_schema, converted_tensors = weight_schema.convert_humming(
tensors=tensors,
shape_n_stacks=shape_n_stacks,
shape_k_stacks=shape_k_stacks,
param_dtype=param_dtype,
num_experts=num_experts,
)
converted_input_schema, input_tensors = input_schema.convert_humming(
tensors=tensors,
shape_n_stacks=shape_n_stacks,
shape_k_stacks=shape_k_stacks,
param_dtype=param_dtype,
num_experts=num_experts,
)
converted_tensors.update(input_tensors)
_replace_layer_parameters(layer, sublayer_name, converted_tensors)
return converted_weight_schema, converted_input_schema
def _prepare_and_transform_sublayer(
layer: "RoutedExperts",
sublayer_name: str,
shape_n: int,
shape_k: int,
weight_schema: Any,
input_schema: Any,
has_bias: bool,
num_experts: int,
param_dtype: torch.dtype,
) -> "LayerConfig":
"""Prepare Humming configuration and transform one sublayer's tensors."""
from vllm.utils.humming import (
prepare_layer_config,
transform_humming_tensors,
)
config = prepare_layer_config(
shape_n=shape_n,
shape_k=shape_k,
pad_n_to_multiple=256,
pad_k_to_multiple=128,
input_schema=input_schema,
weight_schema=weight_schema,
has_bias=has_bias,
num_experts=num_experts,
torch_dtype=param_dtype,
device=getattr(layer, f"{sublayer_name}_weight").device,
)
source_tensors = _extract_sublayer_tensors(layer, sublayer_name)
for name in ("input_scale", "input_scale_2"):
if name != input_schema.static_tensor_scale_name:
source_tensors.pop(name, None)
tensors = transform_humming_tensors(config, source_tensors)
for name in ("input_scale", "input_scale_2"):
if name in source_tensors:
tensors[name] = source_tensors[name]
_replace_layer_parameters(layer, sublayer_name, tensors)
return config
def _process_single_sublayer(
layer: "RoutedExperts",
sublayer_name: str,
shape_n: int,
shape_k: int,
weight_schema: Any,
input_schema: Any,
has_bias: bool,
num_experts: int,
param_dtype: torch.dtype,
force_weight_schema: Any | None = None,
allow_input_schema_fallback: bool = True,
) -> tuple[Any, Any, "LayerConfig"]:
"""Process a single sublayer: convert, optionally requant, prepare, and transform.
This combines the common logic from convert_to_humming_moe_kernel_format
for processing a single sublayer.
Args:
layer: The RoutedExperts layer
sublayer_name: Name of the sublayer (e.g., "w13", "w2")
shape_n: Output dimension size
shape_k: Input dimension size
weight_schema: Initial weight quantization schema
input_schema: Initial input quantization schema
has_bias: Whether the layer has bias terms
num_experts: Number of experts
param_dtype: Parameter data type
force_weight_schema: Optional schema to force requantization to
allow_input_schema_fallback: Whether incompatible input schemas may be replaced.
Returns:
Tuple of the final weight schema, input schema, and Humming layer config.
"""
from vllm.utils.humming import HummingWeightSchema
# Step 1: Convert from checkpoint format to humming format if needed
current_weight_schema, current_input_schema = _convert_sublayer_to_humming(
layer=layer,
sublayer_name=sublayer_name,
shape_n=shape_n,
shape_k=shape_k,
weight_schema=weight_schema,
input_schema=input_schema,
num_experts=num_experts,
param_dtype=param_dtype,
)
# Step 2: Force requant if needed
assert isinstance(current_weight_schema, HummingWeightSchema)
if force_weight_schema is not None and current_weight_schema != force_weight_schema:
source_tensors = _extract_sublayer_tensors(layer, sublayer_name)
tensors = current_weight_schema.requant_tensors(
tensors=source_tensors,
target_weight_schema=force_weight_schema,
param_dtype=param_dtype,
)
name = current_input_schema.static_tensor_scale_name
if name is not None:
tensors[name] = source_tensors[name]
current_weight_schema = force_weight_schema
_replace_layer_parameters(layer, sublayer_name, tensors, preserve_bias=True)
del tensors
# Step 3: Prepare layer metadata and transform weights
current_input_schema = humming_schema.check_and_fallback_input_schema(
weight_schema=current_weight_schema,
input_schema=current_input_schema,
param_dtype=param_dtype,
allow_fallback=allow_input_schema_fallback,
)
config = _prepare_and_transform_sublayer(
layer=layer,
sublayer_name=sublayer_name,
shape_n=shape_n,
shape_k=shape_k,
weight_schema=current_weight_schema,
input_schema=current_input_schema,
has_bias=has_bias,
num_experts=num_experts,
param_dtype=param_dtype,
)
return current_weight_schema, current_input_schema, config
def convert_to_humming_moe_kernel_format(
layer: "RoutedExperts",
quant_config: dict | None = None,
sublayer_configs: dict[str, Any] | None = None,
weight_schema: Any | None = None,
input_schema: Any | None = None,
force_weight_schema: Any | None = None,
allow_input_schema_fallback: bool = True,
) -> dict[str, "LayerConfig"]:
"""Convert MoE weights from checkpoint format to Humming kernel format.
This function processes weights for each sublayer (w13, w2) by:
1. Converting from checkpoint format to humming format if needed
2. Force requanting if a different quantization schema is specified
3. Preparing layer metadata for the Humming kernel
4. Transforming weights for inference
Args:
layer: The RoutedExperts layer containing weights to process
quant_config: Optional quantization config dict. Required if weight_schema
or input_schema are None. Used to build schemas via
BaseWeightSchema.from_config().
sublayer_configs: Optional configuration dict for each sublayer (w13, w2).
Each config must have "shape_n" and "shape_k" keys.
If None, configs are built from layer.moe_config properties.
weight_schema: Optional initial weight quantization schema.
If None, built from quant_config.
input_schema: Optional initial input quantization schema.
If None, built from quant_config or env vars.
force_weight_schema: Optional schema to force requantization to
allow_input_schema_fallback: Whether incompatible input schemas may be replaced.
Side effects:
- Modifies layer parameters in place
- Sets layer.weight_schemas and layer.input_schemas
- Sets layer.humming_configs for quant config construction
"""
# Build schemas from quant_config if not provided
has_bias = layer.moe_config.has_bias
num_experts = layer.moe_config.num_local_experts
param_dtype = layer.params_dtype
if weight_schema is None or input_schema is None:
if quant_config is None:
raise ValueError(
"Must provide either weight_schema/input_schema or quant_config"
)
from vllm.utils.humming import BaseWeightSchema, HummingInputSchema
if weight_schema is None:
weight_schema = BaseWeightSchema.from_config(quant_config)
if input_schema is None:
input_quant_config = (envs.VLLM_HUMMING_INPUT_QUANT_CONFIG or {}).copy()
if humming_is_layer_skipped(input_quant_config, layer.layer_name):
input_schema = HummingInputSchema()
else:
# TODO: read input_quant_config from quant_config
input_quant_config = humming_schema.resolve_humming_layer_config(
input_quant_config, layer.layer_name
)
allow_input_schema_fallback = input_quant_config.pop(
"allow_fallback", False
)
input_schema = HummingInputSchema.from_config(input_quant_config)
# Build sublayer configs from layer properties if not provided
if sublayer_configs is None:
is_gated = layer.moe_config.activation.is_gated
intermediate_size = layer.moe_config.intermediate_size_per_partition
sublayer_configs = {
"w13": {
"shape_n": intermediate_size * (2 if is_gated else 1),
"shape_k": layer.moe_config.hidden_dim,
},
"w2": {
"shape_n": layer.moe_config.hidden_dim,
"shape_k": intermediate_size,
},
}
layer.weight_schemas = {}
layer.input_schemas = {}
humming_configs = {}
for sublayer_name, configs in sublayer_configs.items():
final_weight_schema, final_input_schema, humming_config = (
_process_single_sublayer(
layer=layer,
sublayer_name=sublayer_name,
shape_n=configs["shape_n"],
shape_k=configs["shape_k"],
weight_schema=weight_schema,
input_schema=input_schema,
has_bias=has_bias,
num_experts=num_experts,
param_dtype=param_dtype,
force_weight_schema=force_weight_schema,
allow_input_schema_fallback=allow_input_schema_fallback,
)
)
layer.weight_schemas[sublayer_name] = final_weight_schema
layer.input_schemas[sublayer_name] = final_input_schema
humming_configs[sublayer_name] = humming_config
layer.humming_configs = humming_configs
return humming_configs
@@ -0,0 +1,437 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Map Humming schemas and handle shared checkpoint quantization settings."""
import dataclasses
from typing import TYPE_CHECKING, Any
import regex as re
import torch
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.quant_utils import (
FP4_DTYPE,
FP8_DTYPE,
INT4_DTYPE,
INT8_DTYPE,
MXFP_SCALE_DTYPE,
GroupShape,
QuantKey,
ScaleDesc,
)
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
from vllm.utils.import_utils import has_humming
if TYPE_CHECKING:
from vllm.utils.humming import (
BaseInputSchema,
BaseWeightSchema,
HummingInputSchema,
HummingWeightSchema,
)
from vllm.utils.humming import dtypes as humming_dtypes
def humming_update_schema_hadamard_block_size(
weight_schema: "HummingWeightSchema",
input_schema: "HummingInputSchema",
shape_k: int,
) -> "HummingWeightSchema":
assert shape_k > 0
block_size = 32
weight_group_size = weight_schema.weight_scale_group_size
input_group_size = input_schema.input_scale_group_size
for group_size in (weight_group_size, input_group_size):
if group_size is not None and group_size > 0:
block_size = min(block_size, group_size)
block_size = 1 << (block_size.bit_length() - 1)
while shape_k % block_size:
block_size //= 2
return dataclasses.replace(weight_schema, hadamard_block_size=block_size)
if has_humming():
from vllm.utils.humming import dtypes as humming_dtypes
_HUMMING_TO_QUANT_DTYPE: dict[humming_dtypes.DataType, Any] = {
humming_dtypes.float4e0m3: FP4_DTYPE,
humming_dtypes.float4e2m1: FP4_DTYPE,
humming_dtypes.float6e2m3: scalar_types.float6_e2m3f,
humming_dtypes.float6e3m2: scalar_types.float6_e3m2f,
humming_dtypes.float8e3m4: FP8_DTYPE,
humming_dtypes.float8e4m3: FP8_DTYPE,
humming_dtypes.float8e5m2: torch.float8_e5m2,
humming_dtypes.int8: torch.int8,
humming_dtypes.uint4: INT4_DTYPE,
humming_dtypes.uint8: INT8_DTYPE,
humming_dtypes.uint2: torch.uint8,
humming_dtypes.uint3: torch.uint8,
humming_dtypes.uint5: torch.uint8,
humming_dtypes.uint6: torch.uint8,
humming_dtypes.uint7: torch.uint8,
}
_HUMMING_TO_SCALE_DTYPE: dict[humming_dtypes.DataType, torch.dtype] = {
humming_dtypes.float8e8m0: MXFP_SCALE_DTYPE,
humming_dtypes.float8e4m3: FP8_DTYPE,
humming_dtypes.float16: torch.float16,
humming_dtypes.bfloat16: torch.bfloat16,
humming_dtypes.float32: torch.float32,
}
logger = init_logger(__name__)
def _group_shape(group_size: int, group_size_n: int = 0) -> GroupShape:
"""Map humming group sizes to QuantKey GroupShape.
group_size: elements per group along K (col); 0 means full dimension.
group_size_n: elements per group along N (row); 0 means 1 (per-row).
GroupShape convention: row = N dim, col = K dim.
"""
if group_size == 0 and group_size_n == 0:
return GroupShape.PER_CHANNEL
row = group_size_n if group_size_n > 0 else 1
col = group_size if group_size > 0 else -1
return GroupShape(row=row, col=col)
def weight_schema_to_quant_key(
schema: "BaseWeightSchema",
param_dtype: torch.dtype | None = None,
) -> QuantKey:
from vllm.utils.humming import WeightScale2Type, WeightScaleType
if param_dtype is None:
param_dtype = torch.get_default_dtype()
if param_dtype not in [torch.float16, torch.bfloat16]:
param_dtype = torch.bfloat16
if not current_platform.has_device_capability(80):
param_dtype = torch.float16
schema = schema.to_humming_schema(param_dtype)
dtype = _HUMMING_TO_QUANT_DTYPE[schema.b_dtype]
if schema.bs_dtype is not None:
scale_dtype = _HUMMING_TO_SCALE_DTYPE[schema.bs_dtype]
else:
scale_dtype = torch.float32
group_shape = _group_shape(
schema.weight_scale_group_size,
schema.weight_scale_group_size_n,
)
if schema.weight_scale_type == WeightScaleType.TENSOR:
group_shape = GroupShape.PER_TENSOR
scale = ScaleDesc(dtype=scale_dtype, static=True, group_shape=group_shape)
scale2 = None
if schema.weight_scale_2_type == WeightScale2Type.TENSOR:
scale2 = ScaleDesc(
dtype=torch.float32,
static=True,
group_shape=GroupShape.PER_TENSOR,
)
return QuantKey(
dtype=dtype,
scale=scale,
scale2=scale2,
symmetric=not schema.has_zero_point,
)
def input_schema_to_quant_key(
schema: "BaseInputSchema",
param_dtype: torch.dtype | None = None,
) -> QuantKey | None:
from vllm.utils.humming import InputQuantizationMode
if param_dtype is None:
param_dtype = torch.get_default_dtype()
if param_dtype not in [torch.float16, torch.bfloat16]:
param_dtype = torch.bfloat16
if not current_platform.has_device_capability(80):
param_dtype = torch.float16
schema = schema.to_humming_schema(param_dtype)
if schema.a_dtype is None or schema.a_dtype.num_bits >= 16:
return None
mode = schema.input_quant_mode
if mode == InputQuantizationMode.Disabled:
return None
dtype = _HUMMING_TO_QUANT_DTYPE[schema.a_dtype]
gs = schema.input_scale_group_size
group_shape = GroupShape(row=1, col=gs) if gs > 0 else GroupShape.PER_TOKEN
# Pick the scale dtype the Humming kernel actually consumes. An explicit
# input_scale_dtype always wins. Otherwise infer from the grouping: MX
# microscale activations (group size 32, e.g. MXFP8) carry an e8m0 (uint8)
# scale, while block-FP8 (group size 128) and per-token FP8/int8 carry a
# float32 scale. Getting this right lets a grouped FP8 activation match
# kFp8Dynamic128Sym instead of an unmatchable uint8-scaled key.
if schema.input_scale_dtype is not None:
scale_dtype = _HUMMING_TO_SCALE_DTYPE[schema.input_scale_dtype]
elif gs == 16:
scale_dtype = FP8_DTYPE
elif gs == 32:
scale_dtype = MXFP_SCALE_DTYPE
else:
scale_dtype = torch.float32
if mode == InputQuantizationMode.StaticTensor:
group_shape = GroupShape.PER_TENSOR
scale = ScaleDesc(
dtype=scale_dtype,
static=mode == InputQuantizationMode.StaticTensor,
group_shape=group_shape,
)
scale2 = None
has_scale2 = (
InputQuantizationMode.StaticTensorDynamicGroup,
InputQuantizationMode.DynamicGroupToken,
)
if mode in has_scale2:
static = mode == InputQuantizationMode.StaticTensorDynamicGroup
scale2 = ScaleDesc(
torch.float32,
static,
GroupShape.PER_TENSOR if static else GroupShape.PER_TOKEN,
)
return QuantKey(dtype=dtype, scale=scale, scale2=scale2, symmetric=True)
def quant_key_to_input_schema(key: QuantKey | None) -> "HummingInputSchema":
from vllm.utils.humming import HummingInputSchema, InputQuantizationMode
if key is None:
return HummingInputSchema(input_quant_mode=InputQuantizationMode.Disabled)
if not key.symmetric:
raise ValueError("Humming input quantization must be symmetric")
quant_dtypes = {value: dtype for dtype, value in _HUMMING_TO_QUANT_DTYPE.items()}
quant_dtypes[FP4_DTYPE] = humming_dtypes.float4e2m1
quant_dtypes[FP8_DTYPE] = humming_dtypes.float8e4m3
scale_dtypes = {value: dtype for dtype, value in _HUMMING_TO_SCALE_DTYPE.items()}
if key.dtype not in quant_dtypes or key.scale.dtype not in scale_dtypes:
raise ValueError(f"Unsupported Humming input or scale dtype: {key}")
scale = key.scale
group_size = 0
if scale.static and scale.group_shape.is_per_tensor() and key.scale2 is None:
mode = InputQuantizationMode.StaticTensor
elif not scale.static and scale.group_shape.is_per_token() and key.scale2 is None:
mode = InputQuantizationMode.DynamicToken
elif not scale.static and scale.group_shape.is_per_group():
group_size = scale.group_shape.col
scale2 = key.scale2
if scale2 is None:
mode = InputQuantizationMode.DynamicGroup
elif scale2.dtype != torch.float32:
raise ValueError("Humming secondary input scales must be float32")
elif scale2.static and scale2.group_shape.is_per_tensor():
mode = InputQuantizationMode.StaticTensorDynamicGroup
elif not scale2.static and scale2.group_shape.is_per_token():
mode = InputQuantizationMode.DynamicGroupToken
else:
raise ValueError(f"Unsupported Humming secondary input scale: {scale2}")
else:
raise ValueError(f"Unsupported Humming input scale: {key}")
if group_size == 0 and scale.dtype != torch.float32:
raise ValueError("Humming tensor and token input scales must be float32")
return HummingInputSchema(
a_dtype=quant_dtypes[key.dtype],
input_scale_group_size=group_size,
input_scale_dtype=scale_dtypes[scale.dtype],
input_quant_mode=mode,
)
def check_and_fallback_input_schema(
weight_schema: "BaseWeightSchema",
input_schema: "BaseInputSchema",
param_dtype: torch.dtype | None = None,
allow_fallback: bool = True,
) -> "HummingInputSchema":
from vllm.utils.humming import HummingInputSchema, InputQuantizationMode
capability = current_platform.get_device_capability()
assert capability is not None
sm_version = capability.to_int()
if param_dtype is None:
param_dtype = torch.get_default_dtype()
if param_dtype not in (torch.float16, torch.bfloat16):
param_dtype = torch.bfloat16 if sm_version >= 80 else torch.float16
assert param_dtype in (torch.float16, torch.bfloat16)
dtype_fallback_order_map = {
humming_dtypes.float8e5m2: (
humming_dtypes.float8e4m3,
humming_dtypes.float8e3m4,
humming_dtypes.int8,
),
humming_dtypes.float8e4m3: (
humming_dtypes.float8e3m4,
humming_dtypes.int8,
),
humming_dtypes.float8e3m4: (
humming_dtypes.float8e4m3,
humming_dtypes.int8,
),
humming_dtypes.float4e2m1: (
humming_dtypes.float8e4m3,
humming_dtypes.float8e3m4,
humming_dtypes.int8,
),
humming_dtypes.float4e0m3: (
humming_dtypes.int4,
humming_dtypes.float8e4m3,
humming_dtypes.float8e3m4,
humming_dtypes.int8,
),
humming_dtypes.int8: (
humming_dtypes.float8e4m3,
humming_dtypes.float8e3m4,
),
humming_dtypes.int4: (
humming_dtypes.int8,
humming_dtypes.float8e4m3,
humming_dtypes.float8e3m4,
),
}
weight_schema = weight_schema.to_humming_schema(param_dtype)
input_schema = input_schema.to_humming_schema(param_dtype)
fp6_dtypes = (humming_dtypes.float6e2m3, humming_dtypes.float6e3m2)
if input_schema.a_dtype in fp6_dtypes:
input_schema.a_dtype = humming_dtypes.float8e4m3
a_dtype = input_schema.a_dtype
input_bits = a_dtype.num_bits if a_dtype is not None else 16
weight_group_size = weight_schema.weight_scale_group_size
input_group_size = input_schema.input_scale_group_size
input_scale_dtype = input_schema.input_scale_dtype
def is_deprecated(dtype: "humming_dtypes.DataType | None") -> bool:
if dtype is None or dtype.num_bits == 16:
return False
is_int4_deprecated = dtype == humming_dtypes.int4 and sm_version >= 90
is_int8_deprecated = dtype == humming_dtypes.int8 and 103 <= sm_version < 110
# Humming has not yet implemented high-performance FP8 computation
# on SM10X and SM11X. Therefore, it is temporarily considered deprecated.
# TODO: removed this after Humming has high-performance FP8 computation
is_fp8_deprecated = dtype.num_bits < 16 and 100 <= sm_version < 120
return is_int4_deprecated or is_int8_deprecated or is_fp8_deprecated
if input_schema.is_compatible_with(weight_schema, param_dtype):
if not allow_fallback:
if is_deprecated(a_dtype):
msg = f"Humming {a_dtype} activation is slow on SM{sm_version} now."
logger.warning_once(msg)
return input_schema
if a_dtype is None or a_dtype.num_bits == 16:
return input_schema
if not is_deprecated(a_dtype):
is_mxfp8 = (
a_dtype.is_floating_point_type
and a_dtype.num_bits == 8
and input_group_size == 32
and input_scale_dtype == humming_dtypes.float8e8m0
and sm_version >= 120
)
is_groupwise = input_group_size > 0 or weight_group_size > 0
if input_bits == 8 and is_groupwise and not is_mxfp8:
# prefer wna16 for groupwise weight or groupwise input
return HummingInputSchema()
return input_schema
if allow_fallback:
fallback_dtypes = (
a_dtype,
*dtype_fallback_order_map.get(a_dtype, ()),
humming_dtypes.DataType.from_any(param_dtype),
)
for dtype in fallback_dtypes:
if dtype is None or is_deprecated(dtype):
continue
group_size = 0
scale_dtype = humming_dtypes.float32
quant_mode = InputQuantizationMode.DynamicToken
if dtype.num_bits == 16:
scale_dtype = None
quant_mode = InputQuantizationMode.Disabled
elif dtype in (humming_dtypes.float8e4m3, humming_dtypes.float8e3m4):
is_channelwise_weight = weight_schema.weight_scale_group_size == 0
weight_group_size = weight_schema.weight_scale_group_size
is_e8m0_scale = weight_schema.bs_dtype == humming_dtypes.float8e8m0
is_mx_weight = weight_group_size == 32 and is_e8m0_scale
if sm_version >= 120 and (is_channelwise_weight or is_mx_weight):
group_size = 32
scale_dtype = humming_dtypes.float8e8m0
quant_mode = InputQuantizationMode.DynamicGroup
candidate = HummingInputSchema(
a_dtype=dtype,
input_scale_group_size=group_size,
input_scale_dtype=scale_dtype,
input_quant_mode=quant_mode,
)
if candidate.is_compatible_with(weight_schema, param_dtype):
return candidate
raise ValueError(
f"No compatible Humming input schema for {input_schema} with "
f"weight schema {weight_schema} on SM{sm_version} "
f"({allow_fallback=})"
)
def resolve_humming_layer_config(config: dict[str, Any], prefix: str) -> dict[str, Any]:
layer_config = config.copy()
dynamic = config.get("dynamic", {})
if isinstance(dynamic, dict):
for pattern, overrides in dynamic.items():
if pattern.startswith("+") and re.match(pattern[2:], prefix):
layer_config.update(overrides)
break
return layer_config
def humming_is_layer_skipped(config: dict[str, Any], prefix: str):
if not config:
return True
keys = ["ignored_layers", "ignore", "modules_to_not_convert"]
ignored_layers: list[str] = []
for key in keys:
candidate = config.get(key, []) or []
if candidate:
ignored_layers = candidate
break
if any(module_name in prefix for module_name in ignored_layers):
return True
if "lm_head" in prefix:
return True
for regex in config.get("dynamic", {}):
if regex[:1] != "-":
continue
if re.match(regex[2:], prefix):
return True
return False