[Refactor] refactor humming linear and moe backends to use explicit layer configs (#49610)

Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Jinzhen Lin
2026-08-07 10:03:19 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent e229fdbc87
commit fcde8e1460
37 changed files with 448 additions and 341 deletions
@@ -751,7 +751,6 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
moe_config=self.moe,
experts_cls=self.experts_cls,
backend=self.wna16_moe_backend,
layer=layer,
is_k_full=self.is_k_full,
w13_g_idx=getattr(layer, "w13_g_idx", None),
w2_g_idx=getattr(layer, "w2_g_idx", None),
@@ -766,7 +765,12 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
get_humming_moe_quant_config,
)
return get_humming_moe_quant_config(layer)
return get_humming_moe_quant_config(
layer,
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
)
return make_wna16_moe_quant_config(
w1_scale=layer.w13_scales,
w2_scale=layer.w2_scales,
@@ -772,7 +772,6 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
moe_config=self.moe,
experts_cls=self.experts_cls,
backend=self.wna16_moe_backend,
layer=layer,
is_k_full=self.is_k_full,
w13_g_idx=getattr(layer, "w13_g_idx", None),
w2_g_idx=getattr(layer, "w2_g_idx", None),
@@ -787,7 +786,12 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
get_humming_moe_quant_config,
)
return get_humming_moe_quant_config(layer)
return get_humming_moe_quant_config(
layer,
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
)
from vllm.model_executor.layers.fused_moe.config import (
gptq_marlin_moe_quant_config,
@@ -251,7 +251,6 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
experts_cls=self.experts_cls,
backend=self.nvfp4_backend,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
@@ -339,7 +339,6 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig:
@@ -159,7 +159,6 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
moe_config=self.moe,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
@@ -139,7 +139,6 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod):
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def get_fused_moe_quant_config(
@@ -679,7 +679,6 @@ class Fp8MoEMethod(FusedMoEMethodBase):
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import math
from collections.abc import Callable
from typing import TYPE_CHECKING, Any
@@ -32,6 +31,7 @@ from vllm.model_executor.layers.quantization.base_config import (
)
from vllm.model_executor.layers.quantization.utils.humming_utils import (
convert_to_humming_moe_kernel_format,
get_humming_linear_compute_config,
get_humming_moe_quant_config,
input_schema_to_quant_key,
make_humming_moe_kernel,
@@ -425,7 +425,6 @@ class HummingLinearMethod(LinearMethodBase):
for name, _ in list(layer.named_parameters()):
if name != "bias":
delattr(layer, name)
delattr(layer, "locks")
self.__class__ = UnquantizedLinearMethod # type: ignore
tensor = torch.empty(
(
@@ -513,9 +512,6 @@ class HummingLinearMethod(LinearMethodBase):
param = prepare_param(tensor, name, extra_attrs)
setattr(layer, name, param)
locks = torch.zeros(1024, dtype=torch.int32)
layer.register_buffer("locks", locks)
if self.force_input_schema is not None:
self.input_schema = self.force_input_schema
@@ -574,9 +570,7 @@ class HummingLinearMethod(LinearMethodBase):
del tensors
# prepare layer config from humming kernel
_hm.HummingMethod.prepare_layer_meta(
layer=layer,
self.layer_config = _hm.prepare_layer_config(
shape_n=layer.output_partition_sizes_sum,
shape_k=layer.input_size_per_partition,
weight_schema=self.weight_schema,
@@ -586,19 +580,17 @@ class HummingLinearMethod(LinearMethodBase):
has_bias=layer.has_bias,
torch_dtype=layer.param_dtype,
)
tensors = _hm.transform_humming_tensors(
self.layer_config, dict(layer.named_parameters())
)
for name, _ in list(layer.named_parameters()):
delattr(layer, name)
for name, tensor in tensors.items():
param = torch.nn.Parameter(tensor, requires_grad=False)
setattr(layer, name, param)
# preprocess weight for inference
_hm.HummingMethod.transform_humming_layer(layer)
# compute_config: kernel configs that do not directly affect weights
# but significantly impact kernel behavior or computation precision.
# see https://github.com/inclusionAI/humming/blob/main/docs/config.md
compute_config = {
"use_batch_invariant": envs.VLLM_BATCH_INVARIANT,
"use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,
"gemm_type": "dense",
}
self.compute_config = json.dumps(compute_config)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
def apply(
self,
@@ -606,10 +598,16 @@ class HummingLinearMethod(LinearMethodBase):
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
flatten_inputs = x.view(-1, x.size(-1))
output = _hm.HummingMethod.forward_layer(
layer=layer,
flatten_inputs = x.reshape(-1, x.size(-1))
output = _hm.humming_forward(
self.layer_config,
inputs=flatten_inputs,
weight=layer.weight,
weight_scale=getattr(layer, "weight_scale", None),
zero_point=getattr(layer, "zero_point", None),
bias=getattr(layer, "bias", None),
weight_scale_2=getattr(layer, "weight_scale_2", None),
locks=self.locks,
compute_config=self.compute_config,
)
output = output.view(*x.shape[:-1], output.size(-1))
@@ -726,12 +724,13 @@ class HummingMoEMethod(FusedMoEMethodBase):
# sublayer: a layer contains multiple sets of weights for quantized GEMM
# (e.g., weight, weight_scale, etc.).
# The weight names of sublayer start with the prefix "{sublayer_name}_"
w13_output_size = intermediate_size_per_partition * self.moe.w13_num_shards
layer.sublayer_configs = {
"w13": {
"shape_n": intermediate_size_per_partition * self.moe.w13_num_shards,
"shape_n": w13_output_size,
"shape_k": hidden_size,
"tensors_attrs": self.weight_schema.get_padded_tensors_attrs(
shape_n=intermediate_size_per_partition * self.moe.w13_num_shards,
shape_n=w13_output_size,
shape_k=hidden_size,
num_experts=num_experts,
param_dtype=params_dtype,
@@ -763,11 +762,14 @@ class HummingMoEMethod(FusedMoEMethodBase):
if self.force_input_schema is not None:
self.input_schema = self.force_input_schema
locks = torch.zeros(1024, dtype=torch.int32)
layer.register_buffer("locks", locks)
def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
return get_humming_moe_quant_config(layer)
return get_humming_moe_quant_config(
layer,
self.humming_configs,
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
if getattr(self, "processed", False):
@@ -775,7 +777,7 @@ class HummingMoEMethod(FusedMoEMethodBase):
self.processed = True
# Convert weights to Humming kernel format
convert_to_humming_moe_kernel_format(
self.humming_configs = convert_to_humming_moe_kernel_format(
layer=layer,
sublayer_configs=layer.sublayer_configs,
weight_schema=self.weight_schema,
@@ -791,7 +793,6 @@ class HummingMoEMethod(FusedMoEMethodBase):
self.moe_quant_config,
self.moe,
self.experts_cls,
layer=layer,
routing_tables=layer._expert_routing_tables(),
)
@@ -814,13 +815,11 @@ class HummingMoEMethod(FusedMoEMethodBase):
3. Finalization (weight application & reduction - no-op for Humming
since it's already done internally)
Humming handles all quantization, weight application, and reduction
internally in the experts.apply() method via HummingMethod calls.
Humming handles quantization, weight application, and reduction
internally in the experts implementation.
Note: Although w1/w2 weights are passed to the kernel for interface
consistency, Humming's experts.apply() reads weights directly from
the layer object via HummingMethod.forward_layer() and ignores the
w1/w2 parameters.
Humming experts consume the weights and quantization tensors passed
through the standard modular kernel interface.
"""
assert self.moe_kernel is not None
return self.moe_kernel.apply(
@@ -886,7 +886,6 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
@@ -1579,7 +1578,6 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
experts_cls=self.experts_cls,
backend=self.nvfp4_backend,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
@@ -2058,7 +2056,6 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
fp8_backend=self.mxfp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
# No native MXFP8 MoE kernel on this device (e.g. gfx942): the emulation
@@ -370,7 +370,12 @@ class MoeWNA16Method(FusedMoEMethodBase):
get_humming_moe_quant_config,
)
return get_humming_moe_quant_config(layer)
return get_humming_moe_quant_config(
layer,
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
)
has_zp = self.quant_config.has_zp
return make_wna16_moe_quant_config(
@@ -391,7 +396,6 @@ class MoeWNA16Method(FusedMoEMethodBase):
moe_config=self.moe,
experts_cls=self.experts_cls,
backend=self.wna16_backend,
layer=layer,
routing_tables=layer._expert_routing_tables(),
)
@@ -382,7 +382,6 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
mxfp4_backend=self.mxfp4_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
@@ -773,7 +772,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
mxfp4_backend=self.mxfp4_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def _convert_k3_situ_weight_to_kernel_format(
@@ -516,7 +516,6 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def get_fused_moe_quant_config(
@@ -117,7 +117,6 @@ class Int8OnlineMoEMethod(OnlineMoEMethodBase):
moe_config=self.moe,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
@@ -200,7 +200,6 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def get_fused_moe_quant_config(
@@ -172,7 +172,6 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
experts_cls=self.experts_cls,
backend=self.nvfp4_backend,
routing_tables=layer._expert_routing_tables(),
layer=layer,
per_token_activation=True,
)
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
@@ -804,7 +804,6 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
moe_config=self.moe,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def get_fused_moe_quant_config(
@@ -1646,7 +1645,6 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
experts_cls=self.experts_cls,
backend=self.nvfp4_backend,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def get_fused_moe_quant_config(
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
import regex as re
@@ -47,11 +48,19 @@ if TYPE_CHECKING:
GPTQWeightSchema,
HummingInputSchema,
HummingWeightSchema,
LayerConfig,
)
from vllm.utils.humming import dtypes as humming_dtypes
logger = init_logger(__name__)
@dataclass(kw_only=True)
class HummingMoEQuantConfig(FusedMoEQuantConfig):
w1_humming_config: "LayerConfig"
w2_humming_config: "LayerConfig"
if has_humming():
from vllm.utils.humming import dtypes as humming_dtypes
@@ -435,16 +444,17 @@ def convert_linear_layer_to_humming_standard(
setattr(layer, name, param)
def prepare_humming_layer(
def prepare_humming_linear_layer_config(
layer: LinearBase,
quant_config: dict,
input_quant_config: dict | None = None,
):
) -> "LayerConfig":
from vllm.utils.humming import (
BaseInputSchema,
BaseWeightSchema,
HummingInputSchema,
HummingMethod,
prepare_layer_config,
transform_humming_tensors,
)
weight_schema = BaseWeightSchema.from_config(quant_config)
@@ -480,18 +490,8 @@ def prepare_humming_layer(
layer.weight_schema = weight_schema
for name, _ in list(layer.named_parameters()):
delattr(layer, name)
for name, tensor in tensors.items():
if isinstance(tensor, torch.nn.Parameter):
tensor = tensor.data
param = torch.nn.Parameter(tensor, requires_grad=False)
setattr(layer, name, param)
# Step 2: transform weight (humming standard format) for forwarding
HummingMethod.prepare_layer_meta(
layer=layer,
# Step 2: transform weight (humming standard format) for forwarding.
config = prepare_layer_config(
shape_n=sum(layer.output_partition_sizes),
shape_k=input_size_per_partition,
weight_schema=weight_schema,
@@ -501,20 +501,49 @@ def prepare_humming_layer(
has_bias=layer.has_bias,
torch_dtype=layer.params_dtype,
)
tensors = transform_humming_tensors(config, tensors)
for name, _ in list(layer.named_parameters()):
delattr(layer, name)
for name, tensor in tensors.items():
param = torch.nn.Parameter(tensor, requires_grad=False)
setattr(layer, name, param)
HummingMethod.transform_humming_layer(layer)
if not hasattr(layer, "locks"):
device = layer.weight.device
locks = torch.zeros(1024, dtype=torch.int32, device=device)
layer.register_buffer("locks", locks)
return config
compute_config = {
"use_batch_invariant": envs.VLLM_BATCH_INVARIANT,
"use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,
"gemm_type": "dense",
}
layer.compute_config = json.dumps(compute_config)
def get_humming_linear_compute_config() -> str:
return json.dumps(
{
"use_batch_invariant": envs.VLLM_BATCH_INVARIANT,
"use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,
"gemm_type": "dense",
}
)
def apply_humming_linear(
layer: LinearBase,
x: torch.Tensor,
*,
layer_config: "LayerConfig",
compute_config: str,
locks: torch.Tensor,
) -> torch.Tensor:
from vllm.utils.humming import humming_forward
flatten_inputs = x.reshape(-1, x.size(-1))
output = humming_forward(
layer_config,
inputs=flatten_inputs,
weight=layer.weight,
weight_scale=getattr(layer, "weight_scale", None),
zero_point=getattr(layer, "zero_point", None),
bias=getattr(layer, "bias", None),
weight_scale_2=getattr(layer, "weight_scale_2", None),
locks=locks,
compute_config=compute_config,
)
return output.view(*x.shape[:-1], output.size(-1))
def make_humming_moe_quant_config(
@@ -532,7 +561,9 @@ def make_humming_moe_quant_config(
gemm1_alpha: float | None = None,
gemm1_beta: float | None = None,
gemm1_clamp_limit: float | None = None,
) -> FusedMoEQuantConfig:
humming_configs: dict[str, "LayerConfig"] | None = None,
) -> HummingMoEQuantConfig:
assert humming_configs is not None
if quant_dtype is None:
a_quant_desc = FusedMoEQuantDesc(dtype=None)
else:
@@ -557,7 +588,7 @@ def make_humming_moe_quant_config(
bias=w2_bias,
)
return FusedMoEQuantConfig(
return HummingMoEQuantConfig(
_a1=a_quant_desc,
_a2=a_quant_desc,
_w1=w1_quant_desc,
@@ -565,15 +596,20 @@ def make_humming_moe_quant_config(
gemm1_alpha=gemm1_alpha,
gemm1_beta=gemm1_beta,
gemm1_clamp_limit=gemm1_clamp_limit,
w1_humming_config=humming_configs["w13"],
w2_humming_config=humming_configs["w2"],
)
def get_humming_moe_quant_config(
layer: "RoutedExperts",
humming_configs: dict[str, "LayerConfig"] | None = None,
gemm1_alpha: float | None = None,
gemm1_beta: float | None = None,
gemm1_clamp_limit: float | None = None,
):
if humming_configs is None:
humming_configs = layer.humming_configs
input_schema = layer.input_schemas["w13"]
weight_schema = layer.weight_schemas["w13"]
@@ -600,13 +636,17 @@ def get_humming_moe_quant_config(
weight_dtype=str(weight_schema.b_dtype),
weight_group_shape=weight_group_shape,
w1_scale=getattr(layer, "w13_weight_scale", None),
w1_gscale=getattr(layer, "w13_global_scale", None),
w1_gscale=getattr(layer, "w13_weight_scale_2", None),
w1_zp=getattr(layer, "w13_zero_point", None),
w1_bias=getattr(layer, "w13_bias", None),
w2_scale=getattr(layer, "w2_weight_scale", None),
w2_gscale=getattr(layer, "w2_global_scale", None),
w2_gscale=getattr(layer, "w2_weight_scale_2", None),
w2_zp=getattr(layer, "w2_zero_point", None),
w2_bias=getattr(layer, "w2_bias", None),
gemm1_alpha=gemm1_alpha,
gemm1_beta=gemm1_beta,
gemm1_clamp_limit=gemm1_clamp_limit,
humming_configs=humming_configs,
)
@@ -677,7 +717,6 @@ def make_humming_moe_kernel(
moe_quant_config: FusedMoEQuantConfig,
moe_config: FusedMoEConfig,
experts_cls: type[mk.FusedMoEExperts],
layer: "RoutedExperts",
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
) -> mk.FusedMoEKernel:
# Create Prepare/Finalize.
@@ -692,8 +731,6 @@ def make_humming_moe_kernel(
logger.info_once("Using %s", prepare_finalize.__class__.__name__)
extra_args: dict[str, Any] = {"layer": layer}
# Create Experts.
if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
@@ -703,13 +740,11 @@ def make_humming_moe_kernel(
quant_config=moe_quant_config,
max_num_tokens=max_num_tokens,
num_dispatchers=prepare_finalize.num_dispatchers(),
**extra_args,
)
else:
experts = experts_cls(
moe_config=moe_config,
quant_config=moe_quant_config,
**extra_args,
)
kernel = mk.FusedMoEKernel(
@@ -822,16 +857,14 @@ def _prepare_and_transform_sublayer(
has_bias: bool,
num_experts: int,
param_dtype: torch.dtype,
) -> None:
"""
Prepare layer metadata and transform weights for a sublayer.
) -> "LayerConfig":
"""Prepare Humming configuration and transform one sublayer's tensors."""
from vllm.utils.humming import (
prepare_layer_config,
transform_humming_tensors,
)
This calls Humming's prepare_layer_meta and transform_humming_layer.
"""
from vllm.utils.humming import HummingMethod
HummingMethod.prepare_layer_meta(
layer=layer,
config = prepare_layer_config(
shape_n=shape_n,
shape_k=shape_k,
pad_n_to_multiple=256,
@@ -841,10 +874,13 @@ def _prepare_and_transform_sublayer(
has_bias=has_bias,
num_experts=num_experts,
torch_dtype=param_dtype,
sublayer_name=sublayer_name,
)
HummingMethod.transform_humming_layer(layer, sublayer_name=sublayer_name)
tensors = transform_humming_tensors(
config,
_extract_sublayer_tensors(layer, sublayer_name),
)
_replace_layer_parameters(layer, sublayer_name, tensors)
return config
def _process_single_sublayer(
@@ -858,7 +894,7 @@ def _process_single_sublayer(
num_experts: int,
param_dtype: torch.dtype,
force_weight_schema: Any | None = None,
) -> tuple[Any, Any]:
) -> tuple[Any, Any, "LayerConfig"]:
"""
Process a single sublayer: convert, optionally requant, prepare, and transform.
@@ -878,7 +914,7 @@ def _process_single_sublayer(
force_weight_schema: Optional schema to force requantization to
Returns:
Tuple of (final_weight_schema, final_input_schema)
Tuple of the final weight schema, input schema, and Humming layer config.
"""
from vllm.utils.humming import HummingWeightSchema
@@ -910,7 +946,7 @@ def _process_single_sublayer(
del tensors
# Step 3: Prepare layer metadata and transform weights
_prepare_and_transform_sublayer(
config = _prepare_and_transform_sublayer(
layer=layer,
sublayer_name=sublayer_name,
shape_n=shape_n,
@@ -922,7 +958,7 @@ def _process_single_sublayer(
param_dtype=param_dtype,
)
return current_weight_schema, current_input_schema
return current_weight_schema, current_input_schema, config
def convert_to_humming_moe_kernel_format(
@@ -932,7 +968,7 @@ def convert_to_humming_moe_kernel_format(
weight_schema: Any | None = None,
input_schema: Any | None = None,
force_weight_schema: Any | None = None,
) -> None:
) -> dict[str, "LayerConfig"]:
"""
Convert MoE weights from checkpoint format to Humming kernel format.
@@ -959,6 +995,7 @@ def convert_to_humming_moe_kernel_format(
Side effects:
- Modifies layer parameters in place
- Sets layer.weight_schemas and layer.input_schemas
- Sets layer.humming_configs for quant config construction
"""
# Build schemas from quant_config if not provided
@@ -1005,25 +1042,27 @@ def convert_to_humming_moe_kernel_format(
layer.weight_schemas = {}
layer.input_schemas = {}
humming_configs = {}
for sublayer_name, configs in sublayer_configs.items():
final_weight_schema, final_input_schema = _process_single_sublayer(
layer=layer,
sublayer_name=sublayer_name,
shape_n=configs["shape_n"],
shape_k=configs["shape_k"],
weight_schema=weight_schema,
input_schema=input_schema,
has_bias=has_bias,
num_experts=num_experts,
param_dtype=param_dtype,
force_weight_schema=force_weight_schema,
final_weight_schema, final_input_schema, humming_config = (
_process_single_sublayer(
layer=layer,
sublayer_name=sublayer_name,
shape_n=configs["shape_n"],
shape_k=configs["shape_k"],
weight_schema=weight_schema,
input_schema=input_schema,
has_bias=has_bias,
num_experts=num_experts,
param_dtype=param_dtype,
force_weight_schema=force_weight_schema,
)
)
layer.weight_schemas[sublayer_name] = final_weight_schema
layer.input_schemas[sublayer_name] = final_input_schema
humming_configs[sublayer_name] = humming_config
if not hasattr(layer, "locks"):
device = layer.w13_weight.device
locks = torch.zeros(1024, dtype=torch.int32, device=device)
layer.register_buffer("locks", locks)
layer.humming_configs = humming_configs
return humming_configs