[Refactor] refactor humming linear and moe backends to use explicit layer configs (#49610)

Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Jinzhen Lin
2026-08-07 10:03:19 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent e229fdbc87
commit fcde8e1460
37 changed files with 448 additions and 341 deletions
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
import regex as re
@@ -47,11 +48,19 @@ if TYPE_CHECKING:
GPTQWeightSchema,
HummingInputSchema,
HummingWeightSchema,
LayerConfig,
)
from vllm.utils.humming import dtypes as humming_dtypes
logger = init_logger(__name__)
@dataclass(kw_only=True)
class HummingMoEQuantConfig(FusedMoEQuantConfig):
w1_humming_config: "LayerConfig"
w2_humming_config: "LayerConfig"
if has_humming():
from vllm.utils.humming import dtypes as humming_dtypes
@@ -435,16 +444,17 @@ def convert_linear_layer_to_humming_standard(
setattr(layer, name, param)
def prepare_humming_layer(
def prepare_humming_linear_layer_config(
layer: LinearBase,
quant_config: dict,
input_quant_config: dict | None = None,
):
) -> "LayerConfig":
from vllm.utils.humming import (
BaseInputSchema,
BaseWeightSchema,
HummingInputSchema,
HummingMethod,
prepare_layer_config,
transform_humming_tensors,
)
weight_schema = BaseWeightSchema.from_config(quant_config)
@@ -480,18 +490,8 @@ def prepare_humming_layer(
layer.weight_schema = weight_schema
for name, _ in list(layer.named_parameters()):
delattr(layer, name)
for name, tensor in tensors.items():
if isinstance(tensor, torch.nn.Parameter):
tensor = tensor.data
param = torch.nn.Parameter(tensor, requires_grad=False)
setattr(layer, name, param)
# Step 2: transform weight (humming standard format) for forwarding
HummingMethod.prepare_layer_meta(
layer=layer,
# Step 2: transform weight (humming standard format) for forwarding.
config = prepare_layer_config(
shape_n=sum(layer.output_partition_sizes),
shape_k=input_size_per_partition,
weight_schema=weight_schema,
@@ -501,20 +501,49 @@ def prepare_humming_layer(
has_bias=layer.has_bias,
torch_dtype=layer.params_dtype,
)
tensors = transform_humming_tensors(config, tensors)
for name, _ in list(layer.named_parameters()):
delattr(layer, name)
for name, tensor in tensors.items():
param = torch.nn.Parameter(tensor, requires_grad=False)
setattr(layer, name, param)
HummingMethod.transform_humming_layer(layer)
if not hasattr(layer, "locks"):
device = layer.weight.device
locks = torch.zeros(1024, dtype=torch.int32, device=device)
layer.register_buffer("locks", locks)
return config
compute_config = {
"use_batch_invariant": envs.VLLM_BATCH_INVARIANT,
"use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,
"gemm_type": "dense",
}
layer.compute_config = json.dumps(compute_config)
def get_humming_linear_compute_config() -> str:
return json.dumps(
{
"use_batch_invariant": envs.VLLM_BATCH_INVARIANT,
"use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,
"gemm_type": "dense",
}
)
def apply_humming_linear(
layer: LinearBase,
x: torch.Tensor,
*,
layer_config: "LayerConfig",
compute_config: str,
locks: torch.Tensor,
) -> torch.Tensor:
from vllm.utils.humming import humming_forward
flatten_inputs = x.reshape(-1, x.size(-1))
output = humming_forward(
layer_config,
inputs=flatten_inputs,
weight=layer.weight,
weight_scale=getattr(layer, "weight_scale", None),
zero_point=getattr(layer, "zero_point", None),
bias=getattr(layer, "bias", None),
weight_scale_2=getattr(layer, "weight_scale_2", None),
locks=locks,
compute_config=compute_config,
)
return output.view(*x.shape[:-1], output.size(-1))
def make_humming_moe_quant_config(
@@ -532,7 +561,9 @@ def make_humming_moe_quant_config(
gemm1_alpha: float | None = None,
gemm1_beta: float | None = None,
gemm1_clamp_limit: float | None = None,
) -> FusedMoEQuantConfig:
humming_configs: dict[str, "LayerConfig"] | None = None,
) -> HummingMoEQuantConfig:
assert humming_configs is not None
if quant_dtype is None:
a_quant_desc = FusedMoEQuantDesc(dtype=None)
else:
@@ -557,7 +588,7 @@ def make_humming_moe_quant_config(
bias=w2_bias,
)
return FusedMoEQuantConfig(
return HummingMoEQuantConfig(
_a1=a_quant_desc,
_a2=a_quant_desc,
_w1=w1_quant_desc,
@@ -565,15 +596,20 @@ def make_humming_moe_quant_config(
gemm1_alpha=gemm1_alpha,
gemm1_beta=gemm1_beta,
gemm1_clamp_limit=gemm1_clamp_limit,
w1_humming_config=humming_configs["w13"],
w2_humming_config=humming_configs["w2"],
)
def get_humming_moe_quant_config(
layer: "RoutedExperts",
humming_configs: dict[str, "LayerConfig"] | None = None,
gemm1_alpha: float | None = None,
gemm1_beta: float | None = None,
gemm1_clamp_limit: float | None = None,
):
if humming_configs is None:
humming_configs = layer.humming_configs
input_schema = layer.input_schemas["w13"]
weight_schema = layer.weight_schemas["w13"]
@@ -600,13 +636,17 @@ def get_humming_moe_quant_config(
weight_dtype=str(weight_schema.b_dtype),
weight_group_shape=weight_group_shape,
w1_scale=getattr(layer, "w13_weight_scale", None),
w1_gscale=getattr(layer, "w13_global_scale", None),
w1_gscale=getattr(layer, "w13_weight_scale_2", None),
w1_zp=getattr(layer, "w13_zero_point", None),
w1_bias=getattr(layer, "w13_bias", None),
w2_scale=getattr(layer, "w2_weight_scale", None),
w2_gscale=getattr(layer, "w2_global_scale", None),
w2_gscale=getattr(layer, "w2_weight_scale_2", None),
w2_zp=getattr(layer, "w2_zero_point", None),
w2_bias=getattr(layer, "w2_bias", None),
gemm1_alpha=gemm1_alpha,
gemm1_beta=gemm1_beta,
gemm1_clamp_limit=gemm1_clamp_limit,
humming_configs=humming_configs,
)
@@ -677,7 +717,6 @@ def make_humming_moe_kernel(
moe_quant_config: FusedMoEQuantConfig,
moe_config: FusedMoEConfig,
experts_cls: type[mk.FusedMoEExperts],
layer: "RoutedExperts",
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
) -> mk.FusedMoEKernel:
# Create Prepare/Finalize.
@@ -692,8 +731,6 @@ def make_humming_moe_kernel(
logger.info_once("Using %s", prepare_finalize.__class__.__name__)
extra_args: dict[str, Any] = {"layer": layer}
# Create Experts.
if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
@@ -703,13 +740,11 @@ def make_humming_moe_kernel(
quant_config=moe_quant_config,
max_num_tokens=max_num_tokens,
num_dispatchers=prepare_finalize.num_dispatchers(),
**extra_args,
)
else:
experts = experts_cls(
moe_config=moe_config,
quant_config=moe_quant_config,
**extra_args,
)
kernel = mk.FusedMoEKernel(
@@ -822,16 +857,14 @@ def _prepare_and_transform_sublayer(
has_bias: bool,
num_experts: int,
param_dtype: torch.dtype,
) -> None:
"""
Prepare layer metadata and transform weights for a sublayer.
) -> "LayerConfig":
"""Prepare Humming configuration and transform one sublayer's tensors."""
from vllm.utils.humming import (
prepare_layer_config,
transform_humming_tensors,
)
This calls Humming's prepare_layer_meta and transform_humming_layer.
"""
from vllm.utils.humming import HummingMethod
HummingMethod.prepare_layer_meta(
layer=layer,
config = prepare_layer_config(
shape_n=shape_n,
shape_k=shape_k,
pad_n_to_multiple=256,
@@ -841,10 +874,13 @@ def _prepare_and_transform_sublayer(
has_bias=has_bias,
num_experts=num_experts,
torch_dtype=param_dtype,
sublayer_name=sublayer_name,
)
HummingMethod.transform_humming_layer(layer, sublayer_name=sublayer_name)
tensors = transform_humming_tensors(
config,
_extract_sublayer_tensors(layer, sublayer_name),
)
_replace_layer_parameters(layer, sublayer_name, tensors)
return config
def _process_single_sublayer(
@@ -858,7 +894,7 @@ def _process_single_sublayer(
num_experts: int,
param_dtype: torch.dtype,
force_weight_schema: Any | None = None,
) -> tuple[Any, Any]:
) -> tuple[Any, Any, "LayerConfig"]:
"""
Process a single sublayer: convert, optionally requant, prepare, and transform.
@@ -878,7 +914,7 @@ def _process_single_sublayer(
force_weight_schema: Optional schema to force requantization to
Returns:
Tuple of (final_weight_schema, final_input_schema)
Tuple of the final weight schema, input schema, and Humming layer config.
"""
from vllm.utils.humming import HummingWeightSchema
@@ -910,7 +946,7 @@ def _process_single_sublayer(
del tensors
# Step 3: Prepare layer metadata and transform weights
_prepare_and_transform_sublayer(
config = _prepare_and_transform_sublayer(
layer=layer,
sublayer_name=sublayer_name,
shape_n=shape_n,
@@ -922,7 +958,7 @@ def _process_single_sublayer(
param_dtype=param_dtype,
)
return current_weight_schema, current_input_schema
return current_weight_schema, current_input_schema, config
def convert_to_humming_moe_kernel_format(
@@ -932,7 +968,7 @@ def convert_to_humming_moe_kernel_format(
weight_schema: Any | None = None,
input_schema: Any | None = None,
force_weight_schema: Any | None = None,
) -> None:
) -> dict[str, "LayerConfig"]:
"""
Convert MoE weights from checkpoint format to Humming kernel format.
@@ -959,6 +995,7 @@ def convert_to_humming_moe_kernel_format(
Side effects:
- Modifies layer parameters in place
- Sets layer.weight_schemas and layer.input_schemas
- Sets layer.humming_configs for quant config construction
"""
# Build schemas from quant_config if not provided
@@ -1005,25 +1042,27 @@ def convert_to_humming_moe_kernel_format(
layer.weight_schemas = {}
layer.input_schemas = {}
humming_configs = {}
for sublayer_name, configs in sublayer_configs.items():
final_weight_schema, final_input_schema = _process_single_sublayer(
layer=layer,
sublayer_name=sublayer_name,
shape_n=configs["shape_n"],
shape_k=configs["shape_k"],
weight_schema=weight_schema,
input_schema=input_schema,
has_bias=has_bias,
num_experts=num_experts,
param_dtype=param_dtype,
force_weight_schema=force_weight_schema,
final_weight_schema, final_input_schema, humming_config = (
_process_single_sublayer(
layer=layer,
sublayer_name=sublayer_name,
shape_n=configs["shape_n"],
shape_k=configs["shape_k"],
weight_schema=weight_schema,
input_schema=input_schema,
has_bias=has_bias,
num_experts=num_experts,
param_dtype=param_dtype,
force_weight_schema=force_weight_schema,
)
)
layer.weight_schemas[sublayer_name] = final_weight_schema
layer.input_schemas[sublayer_name] = final_input_schema
humming_configs[sublayer_name] = humming_config
if not hasattr(layer, "locks"):
device = layer.w13_weight.device
locks = torch.zeros(1024, dtype=torch.int32, device=device)
layer.register_buffer("locks", locks)
layer.humming_configs = humming_configs
return humming_configs