[Refactor] refactor humming linear and moe backends to use explicit layer configs (#49610)

Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Jinzhen Lin
2026-08-07 10:03:19 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent e229fdbc87
commit fcde8e1460
37 changed files with 448 additions and 341 deletions
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import math
from collections.abc import Callable
from typing import TYPE_CHECKING, Any
@@ -32,6 +31,7 @@ from vllm.model_executor.layers.quantization.base_config import (
)
from vllm.model_executor.layers.quantization.utils.humming_utils import (
convert_to_humming_moe_kernel_format,
get_humming_linear_compute_config,
get_humming_moe_quant_config,
input_schema_to_quant_key,
make_humming_moe_kernel,
@@ -425,7 +425,6 @@ class HummingLinearMethod(LinearMethodBase):
for name, _ in list(layer.named_parameters()):
if name != "bias":
delattr(layer, name)
delattr(layer, "locks")
self.__class__ = UnquantizedLinearMethod # type: ignore
tensor = torch.empty(
(
@@ -513,9 +512,6 @@ class HummingLinearMethod(LinearMethodBase):
param = prepare_param(tensor, name, extra_attrs)
setattr(layer, name, param)
locks = torch.zeros(1024, dtype=torch.int32)
layer.register_buffer("locks", locks)
if self.force_input_schema is not None:
self.input_schema = self.force_input_schema
@@ -574,9 +570,7 @@ class HummingLinearMethod(LinearMethodBase):
del tensors
# prepare layer config from humming kernel
_hm.HummingMethod.prepare_layer_meta(
layer=layer,
self.layer_config = _hm.prepare_layer_config(
shape_n=layer.output_partition_sizes_sum,
shape_k=layer.input_size_per_partition,
weight_schema=self.weight_schema,
@@ -586,19 +580,17 @@ class HummingLinearMethod(LinearMethodBase):
has_bias=layer.has_bias,
torch_dtype=layer.param_dtype,
)
tensors = _hm.transform_humming_tensors(
self.layer_config, dict(layer.named_parameters())
)
for name, _ in list(layer.named_parameters()):
delattr(layer, name)
for name, tensor in tensors.items():
param = torch.nn.Parameter(tensor, requires_grad=False)
setattr(layer, name, param)
# preprocess weight for inference
_hm.HummingMethod.transform_humming_layer(layer)
# compute_config: kernel configs that do not directly affect weights
# but significantly impact kernel behavior or computation precision.
# see https://github.com/inclusionAI/humming/blob/main/docs/config.md
compute_config = {
"use_batch_invariant": envs.VLLM_BATCH_INVARIANT,
"use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,
"gemm_type": "dense",
}
self.compute_config = json.dumps(compute_config)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
def apply(
self,
@@ -606,10 +598,16 @@ class HummingLinearMethod(LinearMethodBase):
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
flatten_inputs = x.view(-1, x.size(-1))
output = _hm.HummingMethod.forward_layer(
layer=layer,
flatten_inputs = x.reshape(-1, x.size(-1))
output = _hm.humming_forward(
self.layer_config,
inputs=flatten_inputs,
weight=layer.weight,
weight_scale=getattr(layer, "weight_scale", None),
zero_point=getattr(layer, "zero_point", None),
bias=getattr(layer, "bias", None),
weight_scale_2=getattr(layer, "weight_scale_2", None),
locks=self.locks,
compute_config=self.compute_config,
)
output = output.view(*x.shape[:-1], output.size(-1))
@@ -726,12 +724,13 @@ class HummingMoEMethod(FusedMoEMethodBase):
# sublayer: a layer contains multiple sets of weights for quantized GEMM
# (e.g., weight, weight_scale, etc.).
# The weight names of sublayer start with the prefix "{sublayer_name}_"
w13_output_size = intermediate_size_per_partition * self.moe.w13_num_shards
layer.sublayer_configs = {
"w13": {
"shape_n": intermediate_size_per_partition * self.moe.w13_num_shards,
"shape_n": w13_output_size,
"shape_k": hidden_size,
"tensors_attrs": self.weight_schema.get_padded_tensors_attrs(
shape_n=intermediate_size_per_partition * self.moe.w13_num_shards,
shape_n=w13_output_size,
shape_k=hidden_size,
num_experts=num_experts,
param_dtype=params_dtype,
@@ -763,11 +762,14 @@ class HummingMoEMethod(FusedMoEMethodBase):
if self.force_input_schema is not None:
self.input_schema = self.force_input_schema
locks = torch.zeros(1024, dtype=torch.int32)
layer.register_buffer("locks", locks)
def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
return get_humming_moe_quant_config(layer)
return get_humming_moe_quant_config(
layer,
self.humming_configs,
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
if getattr(self, "processed", False):
@@ -775,7 +777,7 @@ class HummingMoEMethod(FusedMoEMethodBase):
self.processed = True
# Convert weights to Humming kernel format
convert_to_humming_moe_kernel_format(
self.humming_configs = convert_to_humming_moe_kernel_format(
layer=layer,
sublayer_configs=layer.sublayer_configs,
weight_schema=self.weight_schema,
@@ -791,7 +793,6 @@ class HummingMoEMethod(FusedMoEMethodBase):
self.moe_quant_config,
self.moe,
self.experts_cls,
layer=layer,
routing_tables=layer._expert_routing_tables(),
)
@@ -814,13 +815,11 @@ class HummingMoEMethod(FusedMoEMethodBase):
3. Finalization (weight application & reduction - no-op for Humming
since it's already done internally)
Humming handles all quantization, weight application, and reduction
internally in the experts.apply() method via HummingMethod calls.
Humming handles quantization, weight application, and reduction
internally in the experts implementation.
Note: Although w1/w2 weights are passed to the kernel for interface
consistency, Humming's experts.apply() reads weights directly from
the layer object via HummingMethod.forward_layer() and ignores the
w1/w2 parameters.
Humming experts consume the weights and quantization tensors passed
through the standard modular kernel interface.
"""
assert self.moe_kernel is not None
return self.moe_kernel.apply(