[Refactor] refactor humming linear and moe backends to use explicit layer configs (#49610)

Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Jinzhen Lin
2026-08-07 10:03:19 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent e229fdbc87
commit fcde8e1460
37 changed files with 448 additions and 341 deletions
+1 -1
View File
@@ -32,4 +32,4 @@ quack-kernels>=0.6.1 # Required for CUTLASS DSL 4.6 by MSA
tokenspeed-mla==0.1.8; platform_system == "Linux"
# Humming kernels for quantization gemm
humming-kernels[cu13]==0.1.10
humming-kernels[cu13]==0.1.12
@@ -0,0 +1,9 @@
model_name: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4"
accuracy_threshold: 0.29
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
@@ -1,5 +1,5 @@
model_name: "nm-testing/Qwen3-4B-mixed-quant-RTN-wNaM"
accuracy_threshold: 0.85
accuracy_threshold: 0.78
num_questions: 1319
num_fewshot: 5
server_args: >-
@@ -1,3 +1,4 @@
gpt-oss-20b-humming.yaml
Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming-act-int8.yaml
Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml
Qwen3-30B-A3B-AWQ-humming.yaml
@@ -11,3 +11,4 @@ Qwen3-30B-A3B-NVFP4-humming.yaml
Qwen3-30B-A3B-AWQ-humming.yaml
Qwen3.5-35B-A3B-FP8-humming.yaml
Qwen3.5-35B-A3B-experts-int8-humming.yaml
NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4-humming.yaml
+9 -13
View File
@@ -1265,11 +1265,10 @@ def _make_humming_indexed_experts(activation: MoEActivation):
layer.intermediate_size_per_partition = intermediate_size
quant_config = humming_utils.get_humming_moe_quant_config(layer)
experts = HummingIndexedExperts(
layer,
moe_config,
quant_config,
moe_config=moe_config,
quant_config=quant_config,
)
return experts
return experts, layer
@pytest.mark.parametrize("activation", list(MoEActivation))
@@ -1328,8 +1327,7 @@ def test_humming_delegates_to_instance_activation():
def test_humming_gated_non_gated_shape_contract(activation: MoEActivation):
from vllm.utils import humming
experts = _make_humming_indexed_experts(activation)
layer = experts.layer
experts, layer = _make_humming_indexed_experts(activation)
moe_config = experts.moe_config
top_k = moe_config.experts_per_token
num_experts = moe_config.num_experts
@@ -1337,7 +1335,7 @@ def test_humming_gated_non_gated_shape_contract(activation: MoEActivation):
intermediate_size = moe_config.intermediate_size
gate_up_size = intermediate_size * 2 if activation.is_gated else intermediate_size
w13_meta, w2_meta = (layer.humming_metas[name] for name in ("w13", "w2"))
w13_meta, w2_meta = (experts.humming_configs[name] for name in ("w13", "w2"))
for meta in (w13_meta, w2_meta):
assert meta.a_dtype == humming.dtypes.bfloat16
assert meta.b_dtype == humming.dtypes.float4e2m1
@@ -1364,7 +1362,7 @@ def test_humming_indexed_writes_supplied_output_buffer():
from vllm.forward_context import set_forward_context
activation = MoEActivation.SILU
experts = _make_humming_indexed_experts(activation)
experts, layer = _make_humming_indexed_experts(activation)
moe_config = experts.moe_config
num_tokens = 1
top_k = moe_config.experts_per_token
@@ -1382,7 +1380,7 @@ def test_humming_indexed_writes_supplied_output_buffer():
)
device = torch.device("cuda")
dtype = experts.layer.params_dtype
dtype = layer.params_dtype
workspace13 = torch.empty(workspace13_shape, dtype=dtype, device=device)
workspace2 = torch.empty(workspace2_shape, dtype=dtype, device=device)
hidden_states = torch.ones((num_tokens, hidden_size), dtype=dtype, device=device)
@@ -1394,14 +1392,12 @@ def test_humming_indexed_writes_supplied_output_buffer():
device=device,
)
topk_ids = torch.arange(top_k, dtype=torch.int32, device=device).unsqueeze(0)
unused = torch.empty((num_experts, 0), device=device)
with set_forward_context(None, vllm_config, num_tokens=num_tokens):
experts.apply(
output=output,
hidden_states=hidden_states,
w1=unused,
w2=unused,
w1=layer.w13_weight,
w2=layer.w2_weight,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=activation,
-1
View File
@@ -1489,7 +1489,6 @@ def test_rocm_mxfp4_moe_oracle(
mxfp4_backend=backend,
experts_cls=experts_cls,
routing_tables=None,
layer=None,
)
# Create inputs
+3 -2
View File
@@ -161,7 +161,6 @@ def test_moe_wna16_setup_forwards_selected_backend(monkeypatch):
assert method.moe_kernel is kernel
assert captured["backend"] == WNA16MoEBackend.HUMMING
assert captured["layer"] is layer
def test_moe_wna16_humming_adapter_repacks_uint8_tensors():
@@ -198,7 +197,9 @@ def test_moe_wna16_uses_humming_quant_config(monkeypatch):
monkeypatch.setattr(
humming_utils,
"get_humming_moe_quant_config",
lambda actual_layer: quant_config if actual_layer is layer else None,
lambda actual_layer, *args, **kwargs: (
quant_config if actual_layer is layer else None
),
)
assert method.get_fused_moe_quant_config(layer) is quant_config
@@ -28,7 +28,8 @@ class HummingLinearKernel(MPLinearKernel):
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
from vllm.model_executor.layers.quantization.utils.humming_utils import (
convert_linear_layer_to_humming_standard,
prepare_humming_layer,
get_humming_linear_compute_config,
prepare_humming_linear_layer_config,
)
name_map = {"weight": self.w_q_name, "weight_scale": self.w_s_name}
@@ -46,9 +47,11 @@ class HummingLinearKernel(MPLinearKernel):
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
input_quant_config = getattr(layer, "_humming_input_quant_config", None)
prepare_humming_layer(
self.layer_config = prepare_humming_linear_layer_config(
layer, quant_config, input_quant_config=input_quant_config
)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
def apply_weights(
self,
@@ -56,12 +59,14 @@ class HummingLinearKernel(MPLinearKernel):
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
from vllm.utils.humming import HummingMethod
from vllm.model_executor.layers.quantization.utils.humming_utils import (
apply_humming_linear,
)
flatten_inputs = x.view(-1, x.size(-1))
output = HummingMethod.forward_layer(
layer=layer,
inputs=flatten_inputs,
compute_config=layer.compute_config,
return apply_humming_linear(
layer,
x,
layer_config=self.layer_config,
compute_config=self.compute_config,
locks=self.locks,
)
return output.view(*x.shape[:-1], output.size(-1))
@@ -5,11 +5,14 @@ import torch
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.humming_utils import (
apply_humming_linear,
convert_linear_layer_to_humming_standard,
prepare_humming_layer,
get_humming_linear_compute_config,
prepare_humming_linear_layer_config,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Dynamic
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_humming
from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
@@ -26,6 +29,9 @@ class HummingMxFp4LinearKernel(MxFp4LinearKernel):
if not current_platform.is_cuda():
return False, "Humming only supported on CUDA"
if not has_humming():
return False, "Humming is not installed"
if not current_platform.has_device_capability(75):
return False, "Humming only supported on SM75+"
@@ -56,7 +62,9 @@ class HummingMxFp4LinearKernel(MxFp4LinearKernel):
}
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
prepare_humming_layer(layer, quant_config)
self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
def apply_weights(
self,
@@ -64,12 +72,10 @@ class HummingMxFp4LinearKernel(MxFp4LinearKernel):
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
from vllm.utils.humming import HummingMethod
flatten_inputs = x.view(-1, x.size(-1))
output = HummingMethod.forward_layer(
layer=layer,
inputs=flatten_inputs,
compute_config=layer.compute_config,
return apply_humming_linear(
layer,
x,
layer_config=self.layer_config,
compute_config=self.compute_config,
locks=self.locks,
)
return output.view(*x.shape[:-1], output.size(-1))
@@ -4,10 +4,13 @@
import torch
from vllm.model_executor.layers.quantization.utils.humming_utils import (
apply_humming_linear,
convert_linear_layer_to_humming_standard,
prepare_humming_layer,
get_humming_linear_compute_config,
prepare_humming_linear_layer_config,
)
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_humming
from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
@@ -22,6 +25,9 @@ class HummingMxfp8LinearKernel(Mxfp8LinearKernel):
if not current_platform.is_cuda():
return False, "Humming only supported on CUDA"
if not has_humming():
return False, "Humming is not installed"
if not current_platform.has_device_capability(75):
return False, "Humming only supported on SM75+"
@@ -44,7 +50,9 @@ class HummingMxfp8LinearKernel(Mxfp8LinearKernel):
}
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
prepare_humming_layer(layer, quant_config)
self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
def apply_weights(
self,
@@ -52,12 +60,10 @@ class HummingMxfp8LinearKernel(Mxfp8LinearKernel):
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
from vllm.utils.humming import HummingMethod
flatten_inputs = x.view(-1, x.size(-1))
output = HummingMethod.forward_layer(
layer=layer,
inputs=flatten_inputs,
compute_config=layer.compute_config,
return apply_humming_linear(
layer,
x,
layer_config=self.layer_config,
compute_config=self.compute_config,
locks=self.locks,
)
return output.view(*x.shape[:-1], output.size(-1))
@@ -5,9 +5,12 @@ import torch
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.humming_utils import (
prepare_humming_layer,
apply_humming_linear,
get_humming_linear_compute_config,
prepare_humming_linear_layer_config,
)
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_humming
from .base import NvFp4LinearKernel, NvFp4LinearLayerConfig
@@ -24,6 +27,9 @@ class HummingNvFp4LinearKernel(NvFp4LinearKernel):
if not current_platform.is_cuda():
return False, "Humming only supported on CUDA"
if not has_humming():
return False, "Humming is not installed"
if not current_platform.has_device_capability(75):
return False, "Humming only supported on SM75+"
@@ -54,7 +60,9 @@ class HummingNvFp4LinearKernel(NvFp4LinearKernel):
layer.weight_global_scale = torch.nn.Parameter(
1.0 / layer.weight_global_scale, requires_grad=False
)
prepare_humming_layer(layer, quant_config)
self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
def apply_weights(
self,
@@ -62,12 +70,10 @@ class HummingNvFp4LinearKernel(NvFp4LinearKernel):
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
from vllm.utils.humming import HummingMethod
flatten_inputs = x.view(-1, x.size(-1))
output = HummingMethod.forward_layer(
layer=layer,
inputs=flatten_inputs,
compute_config=layer.compute_config,
return apply_humming_linear(
layer,
x,
layer_config=self.layer_config,
compute_config=self.compute_config,
locks=self.locks,
)
return output.view(*x.shape[:-1], output.size(-1))
@@ -5,10 +5,13 @@ import torch
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.humming_utils import (
apply_humming_linear,
convert_linear_layer_to_humming_standard,
prepare_humming_layer,
get_humming_linear_compute_config,
prepare_humming_linear_layer_config,
)
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_humming
from .ScaledMMLinearKernel import (
FP8ScaledMMLinearKernel,
@@ -30,6 +33,9 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
if not current_platform.is_cuda():
return False, "Humming only supported on CUDA"
if not has_humming():
return False, "Humming is not installed"
if not current_platform.has_device_capability(75):
return False, "Humming only supported on SM75+"
@@ -58,9 +64,6 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
scale_group_shape = self.config.weight_quant_key.scale.group_shape
if scale_group_shape.is_per_tensor():
quant_config["weight_scale_type"] = "tensor"
if not hasattr(layer, "global_scale") and hasattr(layer, "weight_scale"):
del name_map["weight_scale"]
name_map["global_scale"] = "weight_scale"
elif scale_group_shape.is_per_channel():
quant_config["weight_scale_type"] = "channel"
elif scale_group_shape.is_per_group():
@@ -76,7 +79,9 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
name_map["weight_scale"] = "weight_scale_inv"
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
prepare_humming_layer(layer, quant_config)
self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
def apply_weights(
self,
@@ -84,15 +89,13 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
from vllm.utils.humming import HummingMethod
flatten_inputs = x.view(-1, x.size(-1))
output = HummingMethod.forward_layer(
layer=layer,
inputs=flatten_inputs,
compute_config=layer.compute_config,
return apply_humming_linear(
layer,
x,
layer_config=self.layer_config,
compute_config=self.compute_config,
locks=self.locks,
)
return output.view(*x.shape[:-1], output.size(-1))
def apply_scaled_mm(
self,
@@ -105,7 +108,9 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
bias: torch.Tensor | None,
output_shape: list,
) -> torch.Tensor:
pass
raise NotImplementedError(
"HummingFP8ScaledMMLinearKernel uses apply_weights directly"
)
class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):
@@ -118,6 +123,9 @@ class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):
if not current_platform.is_cuda():
return False, "Humming only supported on CUDA"
if not has_humming():
return False, "Humming is not installed"
if not current_platform.has_device_capability(75):
return False, "Humming only supported on SM75+"
@@ -137,7 +145,9 @@ class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):
weight.data = weight.data + 128
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
prepare_humming_layer(layer, quant_config)
self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
def apply_weights(
self,
@@ -145,12 +155,10 @@ class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
from vllm.utils.humming import HummingMethod
flatten_inputs = x.view(-1, x.size(-1))
output = HummingMethod.forward_layer(
layer=layer,
inputs=flatten_inputs,
compute_config=layer.compute_config,
return apply_humming_linear(
layer,
x,
layer_config=self.layer_config,
compute_config=self.compute_config,
locks=self.locks,
)
return output.view(*x.shape[:-1], output.size(-1))
@@ -4,7 +4,7 @@
import json
import math
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, cast
import torch
@@ -62,8 +62,15 @@ from vllm.utils.import_utils import has_humming
from vllm.v1.worker.workspace import current_workspace_manager
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.utils.humming import GemmType as HummingGemmType
from vllm.model_executor.layers.quantization.utils.humming_utils import (
HummingMoEQuantConfig,
)
from vllm.utils.humming import (
GemmType as HummingGemmType,
)
from vllm.utils.humming import (
LayerConfig as HummingLayerConfig,
)
logger = init_logger(__name__)
@@ -88,15 +95,20 @@ def get_humming_moe_gemm_type() -> str:
class HummingExpertsBase(mk.FusedMoEExpertsModular):
def __init__(
self,
layer: "RoutedExperts",
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
max_num_tokens: int | None = None,
num_dispatchers: int | None = None,
):
self.layer = layer
self.num_experts = self.layer.local_num_experts
self.global_num_experts = self.layer.global_num_experts
humming_quant_config = cast("HummingMoEQuantConfig", quant_config)
self.humming_configs: dict[str, HummingLayerConfig] = {
"w13": humming_quant_config.w1_humming_config,
"w2": humming_quant_config.w2_humming_config,
}
self.locks = torch.zeros(1024, dtype=torch.int32, device=moe_config.device)
self.num_experts = moe_config.num_local_experts
self.global_num_experts = moe_config.num_experts
self.quant_config = quant_config
self.init_humming_moe()
if self.is_batched():
@@ -111,31 +123,73 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
self._permute_scratch: MoEPermuteScratch | None = None
def init_humming_moe(self):
from vllm.utils.humming import HummingMethod
from vllm.utils.humming import get_heuristics_config
self.compute_config = {
"use_batch_invariant": envs.VLLM_BATCH_INVARIANT,
"use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,
"gemm_type": self.humming_gemm_type().value,
}
self.w13_tuning_config = HummingMethod.get_default_tuning_configs(
layer=self.layer,
self.w13_tuning_config = get_heuristics_config(
layer_config=self.humming_configs["w13"],
use_f16_accum=envs.VLLM_HUMMING_USE_F16_ACCUM,
use_batch_invariant=envs.VLLM_BATCH_INVARIANT,
gemm_type=self.humming_gemm_type(),
sublayer_name="w13",
)
self.w2_tuning_config = HummingMethod.get_default_tuning_configs(
layer=self.layer,
self.w2_tuning_config = get_heuristics_config(
layer_config=self.humming_configs["w2"],
use_f16_accum=envs.VLLM_HUMMING_USE_F16_ACCUM,
use_batch_invariant=envs.VLLM_BATCH_INVARIANT,
gemm_type=self.humming_gemm_type(),
sublayer_name="w2",
)
self.compute_config_str = json.dumps(self.compute_config)
self.w13_tuning_config_str = json.dumps(self.w13_tuning_config)
self.w2_tuning_config_str = json.dumps(self.w2_tuning_config)
def quantize_input(
self,
sublayer_name: str,
inputs: torch.Tensor,
quanted_input: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
from vllm.utils.humming import may_quant_input
return may_quant_input(
self.humming_configs[sublayer_name],
inputs=inputs,
quanted_input=quanted_input,
)
def humming_forward(
self,
sublayer_name: str,
inputs: torch.Tensor,
weight: torch.Tensor,
input_scale: torch.Tensor | None,
outputs: torch.Tensor,
**kwargs: Any,
) -> torch.Tensor:
from vllm.utils.humming import humming_forward
is_w13 = sublayer_name == "w13"
return humming_forward(
self.humming_configs[sublayer_name],
inputs=inputs,
weight=weight,
weight_scale=(
self.quant_config.w1_scale if is_w13 else self.quant_config.w2_scale
),
zero_point=(self.quant_config.w1_zp if is_w13 else self.quant_config.w2_zp),
bias=(self.quant_config.w1_bias if is_w13 else self.quant_config.w2_bias),
weight_scale_2=(
self.quant_config.g1_alphas if is_w13 else self.quant_config.g2_alphas
),
input_scale=input_scale,
outputs=outputs,
locks=self.locks,
**kwargs,
)
def _get_permute_scratch(self) -> MoEPermuteScratch | None:
if self._permute_scratch is None and moe_permute_unpermute_supported():
self._permute_scratch = MoEPermuteScratch(
@@ -218,8 +272,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
@property
def expects_unquantized_inputs(self) -> bool:
"""
Humming kernels handle input quantization internally via
HummingMethod.may_quant_input() in the apply() method.
Humming kernels handle input quantization internally in apply().
This property tells the prepare/finalize step to skip input
quantization (by setting defer_input_quant=True) and pass
@@ -255,6 +308,10 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
return True
@staticmethod
def _supports_batch_invariance() -> bool:
return True
def moe_problem_size(
self,
a1: torch.Tensor,
@@ -262,10 +319,8 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
w2: torch.Tensor,
topk_ids: torch.Tensor,
) -> tuple[int, int, int, int, int]:
from vllm.utils.humming import HummingLayerMeta
meta1: HummingLayerMeta = self.layer.humming_metas["w13"]
meta2: HummingLayerMeta = self.layer.humming_metas["w2"]
meta1 = self.humming_configs["w13"]
meta2 = self.humming_configs["w2"]
assert meta1.num_experts == meta2.num_experts
@@ -282,25 +337,25 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
assert a1.size(0) == num_experts
num_tokens = a1.size(1)
return (
meta1.num_experts,
num_tokens,
# Logical intermediate width for both gated and non-gated activations
self.layer.intermediate_size_per_partition,
meta1.shape_k,
top_k,
)
intermediate_dim = meta2.shape_k - meta2.pad_shape_k
hidden_dim = meta1.shape_k - meta1.pad_shape_k
return meta1.num_experts, num_tokens, intermediate_dim, hidden_dim, top_k
def get_buffer_metas(self, M: int, topk: int, activation: MoEActivation):
from vllm.utils.humming import GemmType as HummingGemmType
from vllm.utils.humming import dtypes
num_experts = self.num_experts
N = self.layer.intermediate_size_per_partition
K = self.layer.hidden_size
w13_config = self.humming_configs["w13"]
w2_config = self.humming_configs["w2"]
gate_up_dim = w13_config.shape_n - w13_config.pad_shape_n
intermediate_dim = w2_config.shape_k - w2_config.pad_shape_k
K = w13_config.shape_k - w13_config.pad_shape_k
assert isinstance(num_experts, int)
assert isinstance(N, int)
assert isinstance(gate_up_dim, int)
assert isinstance(intermediate_dim, int)
assert isinstance(K, int)
assert intermediate_dim == self.adjust_N_for_activation(gate_up_dim, activation)
# hidden_states
# (-> quanted_gate_up_input) (if not BF16/FP16 activation)
@@ -310,7 +365,8 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
# -> down_output
# (-> output) (if not is_batched)
# Neighboring nodes are required to utilize distinct workspaces.
# The output must be derived from workspace1.
# The final output buffer is supplied by the modular kernel and may
# alias workspace1.
output_shape: tuple[int, ...]
if self.is_batched():
@@ -327,10 +383,8 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
real_shape_m = M * topk
output_shape = (M, K)
gate_up_size = N * (2 if activation.is_gated else 1)
down_input_size = N
a_dtype = self.layer.humming_metas["w13"].a_dtype
c_dtype = self.layer.humming_metas["w13"].c_dtype
a_dtype = self.humming_configs["w13"].a_dtype
c_dtype = self.humming_configs["w13"].c_dtype
num_bits = a_dtype.num_bits
torch_dtype_map = {
dtypes.float16: torch.float16,
@@ -348,15 +402,15 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
"dtype": torch_dtype_map[a_dtype],
},
"gate_up_output": {
"shape": (real_shape_m, gate_up_size),
"shape": (real_shape_m, gate_up_dim),
"dtype": torch_dtype_map[c_dtype],
},
"activation_output": {
"shape": (real_shape_m, down_input_size),
"shape": (real_shape_m, intermediate_dim),
"dtype": torch_dtype_map[c_dtype],
},
"quanted_down_input": {
"shape": (real_shape_m, down_input_size),
"shape": (real_shape_m, intermediate_dim),
"dtype": torch_dtype_map[a_dtype],
},
"down_output": {
@@ -414,7 +468,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
output_key = "down_output" if self.is_batched() else "output"
output_shape = buffer_metas[output_key]["shape"]
elem_size = self.layer.params_dtype.itemsize
elem_size = self.moe_config.in_dtype.itemsize
return (
(workspace1_nbytes // elem_size,),
@@ -438,7 +492,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
def make_workspaces(self, M: int, topk: int, activation: MoEActivation):
shapes = self._workspace_shapes(M, topk, activation)
workspace1_shape, workspace2_shape, output_shape = shapes
torch_dtype = self.layer.params_dtype
torch_dtype = self.moe_config.in_dtype
workspace1, workspace2 = current_workspace_manager().get_simultaneous(
(workspace1_shape, torch_dtype),
(workspace2_shape, torch_dtype),
@@ -600,12 +654,11 @@ class HummingIndexedExperts(HummingExpertsBase):
"""
Standard apply implementation for Humming indexed experts.
Note: Humming kernels handle weights and quantization internally through
the layer object, so w1, w2, a1q_scale, a2_scale parameters are not used.
The output is written into workspace13 via the buffer management.
Humming performs activation quantization internally and consumes the
weights supplied by the modular kernel interface.
Results are written to the output buffer supplied by the modular
kernel, which may alias workspace13.
"""
from vllm.utils.humming import HummingMethod
assert not apply_router_weight_on_input
hidden_states = hidden_states.view(-1, hidden_states.size(-1))
@@ -616,6 +669,7 @@ class HummingIndexedExperts(HummingExpertsBase):
topk_ids.size(1),
activation,
)
buffers["output"] = output
moe_kwargs1, moe_kwargs2 = self.prepare_humming_moe_kwargs(
topk_ids=topk_ids,
@@ -623,19 +677,18 @@ class HummingIndexedExperts(HummingExpertsBase):
expert_tokens_meta=expert_tokens_meta,
)
inputs, input_scale = HummingMethod.may_quant_input(
layer=self.layer,
inputs, input_scale = self.quantize_input(
"w13",
inputs=hidden_states,
quanted_input=buffers.get("quanted_gate_up_input", None),
sublayer_name="w13",
)
HummingMethod.forward_layer(
layer=self.layer,
self.humming_forward(
"w13",
inputs=inputs,
weight=w1,
input_scale=input_scale,
outputs=buffers["gate_up_output"],
sublayer_name="w13",
**moe_kwargs1,
)
@@ -645,19 +698,18 @@ class HummingIndexedExperts(HummingExpertsBase):
output=buffers["activation_output"],
)
inputs, input_scale = HummingMethod.may_quant_input(
layer=self.layer,
inputs, input_scale = self.quantize_input(
"w2",
inputs=buffers["activation_output"],
quanted_input=buffers.get("quanted_down_input", None),
sublayer_name="w2",
)
HummingMethod.forward_layer(
layer=self.layer,
self.humming_forward(
"w2",
inputs=inputs,
weight=w2,
input_scale=input_scale,
outputs=buffers["down_output"].view(-1, hidden_states.size(-1)),
sublayer_name="w2",
**moe_kwargs2,
)
@@ -705,12 +757,11 @@ class HummingGroupedExperts(HummingExpertsBase):
"""
Standard apply implementation for Humming grouped experts.
Note: Humming kernels handle weights and quantization internally through
the layer object, so w1, w2, a1q_scale, a2_scale parameters are not used.
The output is written into workspace13 via the buffer management.
Humming performs activation quantization internally and consumes the
weights supplied by the modular kernel interface.
Results are written to the output buffer supplied by the modular
kernel, which may alias workspace13.
"""
from vllm.utils.humming import HummingMethod
assert not apply_router_weight_on_input
valid_shape_m = self.estimate_local_valid_shape_m(topk_ids)
@@ -722,6 +773,7 @@ class HummingGroupedExperts(HummingExpertsBase):
topk_ids.size(1),
activation,
)
buffers["output"] = output
hidden_states, _, expert_first_token_offset, inv_perm, _ = moe_permute(
hidden_states=hidden_states,
@@ -733,23 +785,22 @@ class HummingGroupedExperts(HummingExpertsBase):
scratch=self._get_permute_scratch(),
)
inputs, input_scale = HummingMethod.may_quant_input(
layer=self.layer,
inputs, input_scale = self.quantize_input(
"w13",
inputs=hidden_states,
quanted_input=buffers.get("quanted_gate_up_input", None),
sublayer_name="w13",
)
HummingMethod.forward_layer(
layer=self.layer,
self.humming_forward(
"w13",
inputs=inputs,
weight=w1,
input_scale=input_scale,
outputs=buffers["gate_up_output"],
valid_shape_m=valid_shape_m,
expert_layout=expert_first_token_offset,
compute_config=self.compute_config_str,
tuning_config=self.w13_tuning_config_str,
sublayer_name="w13",
)
self.apply_activation(
@@ -758,23 +809,22 @@ class HummingGroupedExperts(HummingExpertsBase):
output=buffers["activation_output"],
)
inputs, input_scale = HummingMethod.may_quant_input(
layer=self.layer,
inputs, input_scale = self.quantize_input(
"w2",
inputs=buffers["activation_output"],
quanted_input=buffers.get("quanted_down_input", None),
sublayer_name="w2",
)
HummingMethod.forward_layer(
layer=self.layer,
self.humming_forward(
"w2",
inputs=inputs,
weight=w2,
input_scale=input_scale,
outputs=buffers["down_output"],
valid_shape_m=valid_shape_m,
expert_layout=expert_first_token_offset,
compute_config=self.compute_config_str,
tuning_config=self.w2_tuning_config_str,
sublayer_name="w2",
)
moe_unpermute(
@@ -821,12 +871,11 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):
"""
Standard apply implementation for Humming batched grouped experts.
Note: Humming kernels handle weights and quantization internally through
the layer object, so w1, w2, a1q_scale, a2_scale parameters are not used.
The output is written into workspace13 via the buffer management.
Humming performs activation quantization internally and consumes the
weights supplied by the modular kernel interface.
Results are written to the output buffer supplied by the modular
kernel, which may alias workspace13.
"""
from vllm.utils.humming import HummingMethod
assert not apply_router_weight_on_input
assert expert_tokens_meta is not None
@@ -841,24 +890,24 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):
topk_ids.size(1),
activation,
)
buffers["down_output"] = output
inputs, input_scale = HummingMethod.may_quant_input(
layer=self.layer,
inputs, input_scale = self.quantize_input(
"w13",
inputs=hidden_states,
quanted_input=buffers.get("quanted_gate_up_input", None),
sublayer_name="w13",
)
HummingMethod.forward_layer(
layer=self.layer,
self.humming_forward(
"w13",
inputs=inputs,
weight=w1,
input_scale=input_scale,
outputs=buffers["gate_up_output"],
valid_shape_m=valid_shape_m,
expert_layout=expert_num_tokens,
compute_config=self.compute_config_str,
tuning_config=self.w13_tuning_config_str,
sublayer_name="w13",
)
self.apply_activation(
@@ -867,21 +916,20 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):
output=buffers["activation_output"],
)
inputs, input_scale = HummingMethod.may_quant_input(
layer=self.layer,
inputs, input_scale = self.quantize_input(
"w2",
inputs=buffers["activation_output"],
quanted_input=buffers.get("quanted_down_input", None),
sublayer_name="w2",
)
HummingMethod.forward_layer(
layer=self.layer,
self.humming_forward(
"w2",
inputs=inputs,
weight=w2,
input_scale=input_scale,
outputs=output.view(-1, hidden_states.size(-1)),
valid_shape_m=valid_shape_m,
expert_layout=expert_num_tokens,
compute_config=self.compute_config_str,
tuning_config=self.w2_tuning_config_str,
sublayer_name="w2",
)
@@ -619,7 +619,12 @@ def make_fp8_moe_quant_config(
)
assert isinstance(layer, RoutedExperts)
return get_humming_moe_quant_config(layer)
return get_humming_moe_quant_config(
layer,
gemm1_alpha=gemm1_alpha,
gemm1_beta=gemm1_beta,
gemm1_clamp_limit=swiglu_limit,
)
# Flashinfer CUTLASS or HPC per-tensor uses single dq scale
# (alpha = w_scale * a_scale) and inverse a2 scale.
@@ -694,7 +699,6 @@ def make_fp8_moe_kernel(
experts_cls: type[mk.FusedMoEExperts],
fp8_backend: Fp8MoeBackend,
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
layer: torch.nn.Module | None = None,
) -> mk.FusedMoEKernel:
# Create Prepare/Finalize.
prepare_finalize = maybe_make_prepare_finalize(
@@ -708,11 +712,6 @@ def make_fp8_moe_kernel(
logger.info_once("Using %s", prepare_finalize.__class__.__name__)
extra_kwargs = {}
if fp8_backend == Fp8MoeBackend.HUMMING:
assert layer is not None
extra_kwargs = {"layer": layer}
# Create Experts.
if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
@@ -722,13 +721,11 @@ def make_fp8_moe_kernel(
quant_config=moe_quant_config,
max_num_tokens=max_num_tokens,
num_dispatchers=prepare_finalize.num_dispatchers(),
**extra_kwargs,
)
else:
experts = experts_cls(
moe_config=moe_config,
quant_config=moe_quant_config,
**extra_kwargs,
)
kernel = mk.FusedMoEKernel(
@@ -195,7 +195,12 @@ def make_int8_moe_quant_config(
)
assert isinstance(layer, RoutedExperts)
return get_humming_moe_quant_config(layer)
return get_humming_moe_quant_config(
layer,
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
)
if scales_absent and not per_act_token_quant:
return int8_w8a16_moe_quant_config(
@@ -251,9 +256,9 @@ def convert_to_int8_moe_kernel_format(
)
assert layer is not None
# Humming reads canonical CT scales (w*_weight_scale) from the layer.
# Online int8 produces per-channel (E, N) w*_scale; expose them as the
# (E, N, 1) w*_weight_scale humming's loader expects.
# The Humming conversion expects canonical CT scale parameter names.
# Online int8 produces per-channel (E, N) w*_scale; expose them as
# (E, N, 1) w*_weight_scale tensors before conversion.
for sub in ("w13", "w2"):
if hasattr(layer, f"{sub}_weight_scale"):
continue
@@ -279,7 +284,6 @@ def make_int8_moe_kernel(
moe_config: FusedMoEConfig,
experts_cls: type[mk.FusedMoEExperts],
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
layer: torch.nn.Module | None = None,
) -> mk.FusedMoEKernel:
# Create Prepare/Finalize.
prepare_finalize = maybe_make_prepare_finalize(
@@ -293,11 +297,6 @@ def make_int8_moe_kernel(
logger.info_once("Using %s", prepare_finalize.__class__.__name__)
extra_kwargs = {}
if int8_backend == Int8MoeBackend.HUMMING:
assert layer is not None
extra_kwargs = {"layer": layer}
# Create Experts.
if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
@@ -307,13 +306,11 @@ def make_int8_moe_kernel(
quant_config=moe_quant_config,
max_num_tokens=max_num_tokens,
num_dispatchers=prepare_finalize.num_dispatchers(),
**extra_kwargs,
)
else:
experts = experts_cls(
moe_config=moe_config,
quant_config=moe_quant_config,
**extra_kwargs,
)
kernel = mk.FusedMoEKernel(
@@ -358,7 +358,6 @@ def make_wna16_moe_kernel(
moe_config: FusedMoEConfig,
experts_cls: type[mk.FusedMoEExperts],
backend: WNA16MoEBackend = WNA16MoEBackend.MARLIN,
layer: torch.nn.Module | None = None,
is_k_full: bool = False,
w13_g_idx: torch.Tensor | None = None,
w2_g_idx: torch.Tensor | None = None,
@@ -410,10 +409,7 @@ def make_wna16_moe_kernel(
logger.info_once("Using %s", experts_cls.__name__, scope="local")
extra_args: dict[str, Any] = {}
if backend == WNA16MoEBackend.HUMMING:
assert layer is not None
extra_args = {"layer": layer}
elif issubclass(experts_cls, MarlinExpertsBase):
if issubclass(experts_cls, MarlinExpertsBase):
extra_args = {
"w13_g_idx": w13_g_idx,
"w2_g_idx": w2_g_idx,
@@ -1788,7 +1788,6 @@ def make_mxfp4_moe_kernel(
experts_cls: type[mk.FusedMoEExperts],
mxfp4_backend: Mxfp4MoeBackend,
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
layer: RoutedExperts | None = None,
) -> mk.FusedMoEKernel:
"""Create a FusedMoEKernel for the given MXFP4 backend."""
is_monolithic = issubclass(experts_cls, mk.FusedMoEExpertsMonolithic)
@@ -1805,11 +1804,6 @@ def make_mxfp4_moe_kernel(
logger.info_once("Using %s", prepare_finalize.__class__.__name__)
logger.info_once("Using %s", experts_cls.__name__)
extra_kwargs = {}
if mxfp4_backend == Mxfp4MoeBackend.HUMMING:
assert layer is not None
extra_kwargs["layer"] = layer
# Create Experts.
if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
@@ -1819,13 +1813,11 @@ def make_mxfp4_moe_kernel(
quant_config=moe_quant_config,
max_num_tokens=max_num_tokens,
num_dispatchers=prepare_finalize.num_dispatchers(),
**extra_kwargs,
)
else:
experts = experts_cls(
moe_config=moe_config,
quant_config=moe_quant_config,
**extra_kwargs,
)
kernel = mk.FusedMoEKernel(
@@ -398,10 +398,10 @@ def convert_to_nvfp4_moe_kernel_format(
a2_scale = None
w13 = layer.w13_weight
w13_scale = layer.w13_weight_scale
w13_scale_2 = getattr(layer, "w13_global_scale", None)
w13_scale_2 = getattr(layer, "w13_weight_scale_2", None)
w2 = layer.w2_weight
w2_scale = layer.w2_weight_scale
w2_scale_2 = getattr(layer, "w2_global_scale", None)
w2_scale_2 = getattr(layer, "w2_weight_scale_2", None)
elif nvfp4_backend == NvFp4MoeBackend.MARLIN:
a13_scale = None
a2_scale = None
@@ -485,7 +485,12 @@ def make_nvfp4_moe_quant_config(
)
assert isinstance(layer, RoutedExperts)
return get_humming_moe_quant_config(layer)
return get_humming_moe_quant_config(
layer,
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
gemm1_clamp_limit=swiglu_limit,
)
elif backend == NvFp4MoeBackend.MARLIN:
return nvfp4_w4a16_moe_quant_config(
g1_alphas=w13_scale_2,
@@ -544,7 +549,6 @@ def make_nvfp4_moe_kernel(
experts_cls: type[mk.FusedMoEExperts],
backend: NvFp4MoeBackend,
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
layer: torch.nn.Module | None = None,
per_token_activation: bool = False,
) -> mk.FusedMoEKernel:
# Create Prepare/Finalize.
@@ -560,9 +564,6 @@ def make_nvfp4_moe_kernel(
logger.info_once("Using %s", prepare_finalize.__class__.__name__)
extra_kwargs = {}
if backend == NvFp4MoeBackend.HUMMING:
assert layer is not None
extra_kwargs = {"layer": layer}
if backend == NvFp4MoeBackend.FLASHINFER_TRTLLM and per_token_activation:
extra_kwargs["per_token_activation"] = True
@@ -751,7 +751,6 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
moe_config=self.moe,
experts_cls=self.experts_cls,
backend=self.wna16_moe_backend,
layer=layer,
is_k_full=self.is_k_full,
w13_g_idx=getattr(layer, "w13_g_idx", None),
w2_g_idx=getattr(layer, "w2_g_idx", None),
@@ -766,7 +765,12 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
get_humming_moe_quant_config,
)
return get_humming_moe_quant_config(layer)
return get_humming_moe_quant_config(
layer,
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
)
return make_wna16_moe_quant_config(
w1_scale=layer.w13_scales,
w2_scale=layer.w2_scales,
@@ -772,7 +772,6 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
moe_config=self.moe,
experts_cls=self.experts_cls,
backend=self.wna16_moe_backend,
layer=layer,
is_k_full=self.is_k_full,
w13_g_idx=getattr(layer, "w13_g_idx", None),
w2_g_idx=getattr(layer, "w2_g_idx", None),
@@ -787,7 +786,12 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
get_humming_moe_quant_config,
)
return get_humming_moe_quant_config(layer)
return get_humming_moe_quant_config(
layer,
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
)
from vllm.model_executor.layers.fused_moe.config import (
gptq_marlin_moe_quant_config,
@@ -251,7 +251,6 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
experts_cls=self.experts_cls,
backend=self.nvfp4_backend,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
@@ -339,7 +339,6 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig:
@@ -159,7 +159,6 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
moe_config=self.moe,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
@@ -139,7 +139,6 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod):
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def get_fused_moe_quant_config(
@@ -679,7 +679,6 @@ class Fp8MoEMethod(FusedMoEMethodBase):
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import math
from collections.abc import Callable
from typing import TYPE_CHECKING, Any
@@ -32,6 +31,7 @@ from vllm.model_executor.layers.quantization.base_config import (
)
from vllm.model_executor.layers.quantization.utils.humming_utils import (
convert_to_humming_moe_kernel_format,
get_humming_linear_compute_config,
get_humming_moe_quant_config,
input_schema_to_quant_key,
make_humming_moe_kernel,
@@ -425,7 +425,6 @@ class HummingLinearMethod(LinearMethodBase):
for name, _ in list(layer.named_parameters()):
if name != "bias":
delattr(layer, name)
delattr(layer, "locks")
self.__class__ = UnquantizedLinearMethod # type: ignore
tensor = torch.empty(
(
@@ -513,9 +512,6 @@ class HummingLinearMethod(LinearMethodBase):
param = prepare_param(tensor, name, extra_attrs)
setattr(layer, name, param)
locks = torch.zeros(1024, dtype=torch.int32)
layer.register_buffer("locks", locks)
if self.force_input_schema is not None:
self.input_schema = self.force_input_schema
@@ -574,9 +570,7 @@ class HummingLinearMethod(LinearMethodBase):
del tensors
# prepare layer config from humming kernel
_hm.HummingMethod.prepare_layer_meta(
layer=layer,
self.layer_config = _hm.prepare_layer_config(
shape_n=layer.output_partition_sizes_sum,
shape_k=layer.input_size_per_partition,
weight_schema=self.weight_schema,
@@ -586,19 +580,17 @@ class HummingLinearMethod(LinearMethodBase):
has_bias=layer.has_bias,
torch_dtype=layer.param_dtype,
)
tensors = _hm.transform_humming_tensors(
self.layer_config, dict(layer.named_parameters())
)
for name, _ in list(layer.named_parameters()):
delattr(layer, name)
for name, tensor in tensors.items():
param = torch.nn.Parameter(tensor, requires_grad=False)
setattr(layer, name, param)
# preprocess weight for inference
_hm.HummingMethod.transform_humming_layer(layer)
# compute_config: kernel configs that do not directly affect weights
# but significantly impact kernel behavior or computation precision.
# see https://github.com/inclusionAI/humming/blob/main/docs/config.md
compute_config = {
"use_batch_invariant": envs.VLLM_BATCH_INVARIANT,
"use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,
"gemm_type": "dense",
}
self.compute_config = json.dumps(compute_config)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
def apply(
self,
@@ -606,10 +598,16 @@ class HummingLinearMethod(LinearMethodBase):
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
flatten_inputs = x.view(-1, x.size(-1))
output = _hm.HummingMethod.forward_layer(
layer=layer,
flatten_inputs = x.reshape(-1, x.size(-1))
output = _hm.humming_forward(
self.layer_config,
inputs=flatten_inputs,
weight=layer.weight,
weight_scale=getattr(layer, "weight_scale", None),
zero_point=getattr(layer, "zero_point", None),
bias=getattr(layer, "bias", None),
weight_scale_2=getattr(layer, "weight_scale_2", None),
locks=self.locks,
compute_config=self.compute_config,
)
output = output.view(*x.shape[:-1], output.size(-1))
@@ -726,12 +724,13 @@ class HummingMoEMethod(FusedMoEMethodBase):
# sublayer: a layer contains multiple sets of weights for quantized GEMM
# (e.g., weight, weight_scale, etc.).
# The weight names of sublayer start with the prefix "{sublayer_name}_"
w13_output_size = intermediate_size_per_partition * self.moe.w13_num_shards
layer.sublayer_configs = {
"w13": {
"shape_n": intermediate_size_per_partition * self.moe.w13_num_shards,
"shape_n": w13_output_size,
"shape_k": hidden_size,
"tensors_attrs": self.weight_schema.get_padded_tensors_attrs(
shape_n=intermediate_size_per_partition * self.moe.w13_num_shards,
shape_n=w13_output_size,
shape_k=hidden_size,
num_experts=num_experts,
param_dtype=params_dtype,
@@ -763,11 +762,14 @@ class HummingMoEMethod(FusedMoEMethodBase):
if self.force_input_schema is not None:
self.input_schema = self.force_input_schema
locks = torch.zeros(1024, dtype=torch.int32)
layer.register_buffer("locks", locks)
def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
return get_humming_moe_quant_config(layer)
return get_humming_moe_quant_config(
layer,
self.humming_configs,
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
if getattr(self, "processed", False):
@@ -775,7 +777,7 @@ class HummingMoEMethod(FusedMoEMethodBase):
self.processed = True
# Convert weights to Humming kernel format
convert_to_humming_moe_kernel_format(
self.humming_configs = convert_to_humming_moe_kernel_format(
layer=layer,
sublayer_configs=layer.sublayer_configs,
weight_schema=self.weight_schema,
@@ -791,7 +793,6 @@ class HummingMoEMethod(FusedMoEMethodBase):
self.moe_quant_config,
self.moe,
self.experts_cls,
layer=layer,
routing_tables=layer._expert_routing_tables(),
)
@@ -814,13 +815,11 @@ class HummingMoEMethod(FusedMoEMethodBase):
3. Finalization (weight application & reduction - no-op for Humming
since it's already done internally)
Humming handles all quantization, weight application, and reduction
internally in the experts.apply() method via HummingMethod calls.
Humming handles quantization, weight application, and reduction
internally in the experts implementation.
Note: Although w1/w2 weights are passed to the kernel for interface
consistency, Humming's experts.apply() reads weights directly from
the layer object via HummingMethod.forward_layer() and ignores the
w1/w2 parameters.
Humming experts consume the weights and quantization tensors passed
through the standard modular kernel interface.
"""
assert self.moe_kernel is not None
return self.moe_kernel.apply(
@@ -886,7 +886,6 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
@@ -1579,7 +1578,6 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
experts_cls=self.experts_cls,
backend=self.nvfp4_backend,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
@@ -2058,7 +2056,6 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
fp8_backend=self.mxfp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
# No native MXFP8 MoE kernel on this device (e.g. gfx942): the emulation
@@ -370,7 +370,12 @@ class MoeWNA16Method(FusedMoEMethodBase):
get_humming_moe_quant_config,
)
return get_humming_moe_quant_config(layer)
return get_humming_moe_quant_config(
layer,
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
)
has_zp = self.quant_config.has_zp
return make_wna16_moe_quant_config(
@@ -391,7 +396,6 @@ class MoeWNA16Method(FusedMoEMethodBase):
moe_config=self.moe,
experts_cls=self.experts_cls,
backend=self.wna16_backend,
layer=layer,
routing_tables=layer._expert_routing_tables(),
)
@@ -382,7 +382,6 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
mxfp4_backend=self.mxfp4_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
@@ -773,7 +772,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
mxfp4_backend=self.mxfp4_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def _convert_k3_situ_weight_to_kernel_format(
@@ -516,7 +516,6 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def get_fused_moe_quant_config(
@@ -117,7 +117,6 @@ class Int8OnlineMoEMethod(OnlineMoEMethodBase):
moe_config=self.moe,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
@@ -200,7 +200,6 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def get_fused_moe_quant_config(
@@ -172,7 +172,6 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
experts_cls=self.experts_cls,
backend=self.nvfp4_backend,
routing_tables=layer._expert_routing_tables(),
layer=layer,
per_token_activation=True,
)
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
@@ -804,7 +804,6 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
moe_config=self.moe,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def get_fused_moe_quant_config(
@@ -1646,7 +1645,6 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
experts_cls=self.experts_cls,
backend=self.nvfp4_backend,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def get_fused_moe_quant_config(
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
import regex as re
@@ -47,11 +48,19 @@ if TYPE_CHECKING:
GPTQWeightSchema,
HummingInputSchema,
HummingWeightSchema,
LayerConfig,
)
from vllm.utils.humming import dtypes as humming_dtypes
logger = init_logger(__name__)
@dataclass(kw_only=True)
class HummingMoEQuantConfig(FusedMoEQuantConfig):
w1_humming_config: "LayerConfig"
w2_humming_config: "LayerConfig"
if has_humming():
from vllm.utils.humming import dtypes as humming_dtypes
@@ -435,16 +444,17 @@ def convert_linear_layer_to_humming_standard(
setattr(layer, name, param)
def prepare_humming_layer(
def prepare_humming_linear_layer_config(
layer: LinearBase,
quant_config: dict,
input_quant_config: dict | None = None,
):
) -> "LayerConfig":
from vllm.utils.humming import (
BaseInputSchema,
BaseWeightSchema,
HummingInputSchema,
HummingMethod,
prepare_layer_config,
transform_humming_tensors,
)
weight_schema = BaseWeightSchema.from_config(quant_config)
@@ -480,18 +490,8 @@ def prepare_humming_layer(
layer.weight_schema = weight_schema
for name, _ in list(layer.named_parameters()):
delattr(layer, name)
for name, tensor in tensors.items():
if isinstance(tensor, torch.nn.Parameter):
tensor = tensor.data
param = torch.nn.Parameter(tensor, requires_grad=False)
setattr(layer, name, param)
# Step 2: transform weight (humming standard format) for forwarding
HummingMethod.prepare_layer_meta(
layer=layer,
# Step 2: transform weight (humming standard format) for forwarding.
config = prepare_layer_config(
shape_n=sum(layer.output_partition_sizes),
shape_k=input_size_per_partition,
weight_schema=weight_schema,
@@ -501,20 +501,49 @@ def prepare_humming_layer(
has_bias=layer.has_bias,
torch_dtype=layer.params_dtype,
)
tensors = transform_humming_tensors(config, tensors)
for name, _ in list(layer.named_parameters()):
delattr(layer, name)
for name, tensor in tensors.items():
param = torch.nn.Parameter(tensor, requires_grad=False)
setattr(layer, name, param)
HummingMethod.transform_humming_layer(layer)
if not hasattr(layer, "locks"):
device = layer.weight.device
locks = torch.zeros(1024, dtype=torch.int32, device=device)
layer.register_buffer("locks", locks)
return config
compute_config = {
"use_batch_invariant": envs.VLLM_BATCH_INVARIANT,
"use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,
"gemm_type": "dense",
}
layer.compute_config = json.dumps(compute_config)
def get_humming_linear_compute_config() -> str:
return json.dumps(
{
"use_batch_invariant": envs.VLLM_BATCH_INVARIANT,
"use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,
"gemm_type": "dense",
}
)
def apply_humming_linear(
layer: LinearBase,
x: torch.Tensor,
*,
layer_config: "LayerConfig",
compute_config: str,
locks: torch.Tensor,
) -> torch.Tensor:
from vllm.utils.humming import humming_forward
flatten_inputs = x.reshape(-1, x.size(-1))
output = humming_forward(
layer_config,
inputs=flatten_inputs,
weight=layer.weight,
weight_scale=getattr(layer, "weight_scale", None),
zero_point=getattr(layer, "zero_point", None),
bias=getattr(layer, "bias", None),
weight_scale_2=getattr(layer, "weight_scale_2", None),
locks=locks,
compute_config=compute_config,
)
return output.view(*x.shape[:-1], output.size(-1))
def make_humming_moe_quant_config(
@@ -532,7 +561,9 @@ def make_humming_moe_quant_config(
gemm1_alpha: float | None = None,
gemm1_beta: float | None = None,
gemm1_clamp_limit: float | None = None,
) -> FusedMoEQuantConfig:
humming_configs: dict[str, "LayerConfig"] | None = None,
) -> HummingMoEQuantConfig:
assert humming_configs is not None
if quant_dtype is None:
a_quant_desc = FusedMoEQuantDesc(dtype=None)
else:
@@ -557,7 +588,7 @@ def make_humming_moe_quant_config(
bias=w2_bias,
)
return FusedMoEQuantConfig(
return HummingMoEQuantConfig(
_a1=a_quant_desc,
_a2=a_quant_desc,
_w1=w1_quant_desc,
@@ -565,15 +596,20 @@ def make_humming_moe_quant_config(
gemm1_alpha=gemm1_alpha,
gemm1_beta=gemm1_beta,
gemm1_clamp_limit=gemm1_clamp_limit,
w1_humming_config=humming_configs["w13"],
w2_humming_config=humming_configs["w2"],
)
def get_humming_moe_quant_config(
layer: "RoutedExperts",
humming_configs: dict[str, "LayerConfig"] | None = None,
gemm1_alpha: float | None = None,
gemm1_beta: float | None = None,
gemm1_clamp_limit: float | None = None,
):
if humming_configs is None:
humming_configs = layer.humming_configs
input_schema = layer.input_schemas["w13"]
weight_schema = layer.weight_schemas["w13"]
@@ -600,13 +636,17 @@ def get_humming_moe_quant_config(
weight_dtype=str(weight_schema.b_dtype),
weight_group_shape=weight_group_shape,
w1_scale=getattr(layer, "w13_weight_scale", None),
w1_gscale=getattr(layer, "w13_global_scale", None),
w1_gscale=getattr(layer, "w13_weight_scale_2", None),
w1_zp=getattr(layer, "w13_zero_point", None),
w1_bias=getattr(layer, "w13_bias", None),
w2_scale=getattr(layer, "w2_weight_scale", None),
w2_gscale=getattr(layer, "w2_global_scale", None),
w2_gscale=getattr(layer, "w2_weight_scale_2", None),
w2_zp=getattr(layer, "w2_zero_point", None),
w2_bias=getattr(layer, "w2_bias", None),
gemm1_alpha=gemm1_alpha,
gemm1_beta=gemm1_beta,
gemm1_clamp_limit=gemm1_clamp_limit,
humming_configs=humming_configs,
)
@@ -677,7 +717,6 @@ def make_humming_moe_kernel(
moe_quant_config: FusedMoEQuantConfig,
moe_config: FusedMoEConfig,
experts_cls: type[mk.FusedMoEExperts],
layer: "RoutedExperts",
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
) -> mk.FusedMoEKernel:
# Create Prepare/Finalize.
@@ -692,8 +731,6 @@ def make_humming_moe_kernel(
logger.info_once("Using %s", prepare_finalize.__class__.__name__)
extra_args: dict[str, Any] = {"layer": layer}
# Create Experts.
if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
@@ -703,13 +740,11 @@ def make_humming_moe_kernel(
quant_config=moe_quant_config,
max_num_tokens=max_num_tokens,
num_dispatchers=prepare_finalize.num_dispatchers(),
**extra_args,
)
else:
experts = experts_cls(
moe_config=moe_config,
quant_config=moe_quant_config,
**extra_args,
)
kernel = mk.FusedMoEKernel(
@@ -822,16 +857,14 @@ def _prepare_and_transform_sublayer(
has_bias: bool,
num_experts: int,
param_dtype: torch.dtype,
) -> None:
"""
Prepare layer metadata and transform weights for a sublayer.
) -> "LayerConfig":
"""Prepare Humming configuration and transform one sublayer's tensors."""
from vllm.utils.humming import (
prepare_layer_config,
transform_humming_tensors,
)
This calls Humming's prepare_layer_meta and transform_humming_layer.
"""
from vllm.utils.humming import HummingMethod
HummingMethod.prepare_layer_meta(
layer=layer,
config = prepare_layer_config(
shape_n=shape_n,
shape_k=shape_k,
pad_n_to_multiple=256,
@@ -841,10 +874,13 @@ def _prepare_and_transform_sublayer(
has_bias=has_bias,
num_experts=num_experts,
torch_dtype=param_dtype,
sublayer_name=sublayer_name,
)
HummingMethod.transform_humming_layer(layer, sublayer_name=sublayer_name)
tensors = transform_humming_tensors(
config,
_extract_sublayer_tensors(layer, sublayer_name),
)
_replace_layer_parameters(layer, sublayer_name, tensors)
return config
def _process_single_sublayer(
@@ -858,7 +894,7 @@ def _process_single_sublayer(
num_experts: int,
param_dtype: torch.dtype,
force_weight_schema: Any | None = None,
) -> tuple[Any, Any]:
) -> tuple[Any, Any, "LayerConfig"]:
"""
Process a single sublayer: convert, optionally requant, prepare, and transform.
@@ -878,7 +914,7 @@ def _process_single_sublayer(
force_weight_schema: Optional schema to force requantization to
Returns:
Tuple of (final_weight_schema, final_input_schema)
Tuple of the final weight schema, input schema, and Humming layer config.
"""
from vllm.utils.humming import HummingWeightSchema
@@ -910,7 +946,7 @@ def _process_single_sublayer(
del tensors
# Step 3: Prepare layer metadata and transform weights
_prepare_and_transform_sublayer(
config = _prepare_and_transform_sublayer(
layer=layer,
sublayer_name=sublayer_name,
shape_n=shape_n,
@@ -922,7 +958,7 @@ def _process_single_sublayer(
param_dtype=param_dtype,
)
return current_weight_schema, current_input_schema
return current_weight_schema, current_input_schema, config
def convert_to_humming_moe_kernel_format(
@@ -932,7 +968,7 @@ def convert_to_humming_moe_kernel_format(
weight_schema: Any | None = None,
input_schema: Any | None = None,
force_weight_schema: Any | None = None,
) -> None:
) -> dict[str, "LayerConfig"]:
"""
Convert MoE weights from checkpoint format to Humming kernel format.
@@ -959,6 +995,7 @@ def convert_to_humming_moe_kernel_format(
Side effects:
- Modifies layer parameters in place
- Sets layer.weight_schemas and layer.input_schemas
- Sets layer.humming_configs for quant config construction
"""
# Build schemas from quant_config if not provided
@@ -1005,25 +1042,27 @@ def convert_to_humming_moe_kernel_format(
layer.weight_schemas = {}
layer.input_schemas = {}
humming_configs = {}
for sublayer_name, configs in sublayer_configs.items():
final_weight_schema, final_input_schema = _process_single_sublayer(
layer=layer,
sublayer_name=sublayer_name,
shape_n=configs["shape_n"],
shape_k=configs["shape_k"],
weight_schema=weight_schema,
input_schema=input_schema,
has_bias=has_bias,
num_experts=num_experts,
param_dtype=param_dtype,
force_weight_schema=force_weight_schema,
final_weight_schema, final_input_schema, humming_config = (
_process_single_sublayer(
layer=layer,
sublayer_name=sublayer_name,
shape_n=configs["shape_n"],
shape_k=configs["shape_k"],
weight_schema=weight_schema,
input_schema=input_schema,
has_bias=has_bias,
num_experts=num_experts,
param_dtype=param_dtype,
force_weight_schema=force_weight_schema,
)
)
layer.weight_schemas[sublayer_name] = final_weight_schema
layer.input_schemas[sublayer_name] = final_input_schema
humming_configs[sublayer_name] = humming_config
if not hasattr(layer, "locks"):
device = layer.w13_weight.device
locks = torch.zeros(1024, dtype=torch.int32, device=device)
layer.register_buffer("locks", locks)
layer.humming_configs = humming_configs
return humming_configs
+6 -2
View File
@@ -15,9 +15,13 @@ _EXPORTS: dict[str, str] = {
"dtypes": "humming.dtypes",
"DataType": "humming.dtypes:DataType",
"GemmType": "humming.config:GemmType",
"LayerConfig": "humming.config:LayerConfig",
"WeightScaleType": "humming.config:WeightScaleType",
"HummingMethod": "humming.layer:HummingMethod",
"HummingLayerMeta": "humming.layer:HummingLayerMeta",
"humming_forward": "humming.forward:humming_forward",
"may_quant_input": "humming.forward:may_quant_input",
"prepare_layer_config": "humming.transform:prepare_layer_config",
"transform_humming_tensors": "humming.transform:transform_humming_tensors",
"get_heuristics_config": "humming.tune:get_heuristics_config",
"BaseInputSchema": "humming.schema:BaseInputSchema",
"BaseWeightSchema": "humming.schema:BaseWeightSchema",
"HummingInputSchema": "humming.schema:HummingInputSchema",