[Quantization] add humming moe backend to all dense/moe oracles (#41652)

Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Jinzhen Lin
2026-07-06 13:36:07 -07:00
committed by GitHub
co-authored by mgoin
parent 04adc8843b
commit d891b9bd51
73 changed files with 1337 additions and 123 deletions
+29 -1
View File
@@ -74,6 +74,9 @@ from vllm.model_executor.kernels.linear.mxfp4 import (
from vllm.model_executor.kernels.linear.mxfp4.flashinfer import (
FlashInferMxFp4LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp4.humming import (
HummingMxFp4LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp4.marlin import (
MarlinMxFp4LinearKernel,
)
@@ -91,6 +94,9 @@ from vllm.model_executor.kernels.linear.mxfp8.flashinfer import (
FlashInferCutedslMxfp8LinearKernel,
FlashInferCutlassMxfp8LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp8.humming import (
HummingMxfp8LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp8.marlin import (
MarlinMxfp8LinearKernel,
)
@@ -120,6 +126,9 @@ from vllm.model_executor.kernels.linear.nvfp4.flashinfer import (
FlashInferCutlassNvFp4LinearKernel,
FlashInferTrtllmNvFp4LinearKernel,
)
from vllm.model_executor.kernels.linear.nvfp4.humming import (
HummingNvFp4LinearKernel,
)
from vllm.model_executor.kernels.linear.nvfp4.marlin import (
MarlinNvFp4LinearKernel,
)
@@ -154,6 +163,10 @@ from vllm.model_executor.kernels.linear.scaled_mm.flashinfer import (
FlashInferFp8DeepGEMMDynamicBlockScaledKernel,
FlashInferFP8ScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.humming import (
HummingFP8ScaledMMLinearKernel,
HummingInt8ScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.marlin import (
MarlinFP8ScaledMMLinearKernel,
)
@@ -225,6 +238,14 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
"flashinfer_b12x": {
FlashInferB12xNvFp4LinearKernel,
},
"humming": {
HummingFP8ScaledMMLinearKernel,
HummingInt8ScaledMMLinearKernel,
HummingLinearKernel,
HummingMxfp8LinearKernel,
HummingMxFp4LinearKernel,
HummingNvFp4LinearKernel,
},
"marlin": {
MarlinFP8ScaledMMLinearKernel,
MarlinLinearKernel,
@@ -292,6 +313,7 @@ _POSSIBLE_INT8_KERNELS: dict[PlatformEnum, list[type[Int8ScaledMMLinearKernel]]]
PlatformEnum.CUDA: [
CutlassInt8ScaledMMLinearKernel,
TritonInt8ScaledMMLinearKernel,
HummingInt8ScaledMMLinearKernel,
],
PlatformEnum.ROCM: [AiterInt8ScaledMMLinearKernel, TritonInt8ScaledMMLinearKernel],
}
@@ -304,6 +326,7 @@ _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] =
CutlassFP8ScaledMMLinearKernel,
PerTensorTorchFP8ScaledMMLinearKernel,
ChannelWiseTorchFP8ScaledMMLinearKernel,
HummingFP8ScaledMMLinearKernel,
],
PlatformEnum.ROCM: [
AiterHipbMMPerTokenFp8ScaledMMLinearKernel,
@@ -335,6 +358,7 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[
CutlassFp8BlockScaledMMKernel,
MarlinFP8ScaledMMLinearKernel,
TritonFp8BlockScaledMMKernel,
HummingFP8ScaledMMLinearKernel,
],
PlatformEnum.ROCM: [
AiterFp8BlockScaledMMKernel,
@@ -351,6 +375,7 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[
_POSSIBLE_WFP8A16_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] = {
PlatformEnum.CUDA: [
HummingFP8ScaledMMLinearKernel,
MarlinFP8ScaledMMLinearKernel,
],
PlatformEnum.ROCM: [
@@ -371,10 +396,10 @@ _POSSIBLE_KERNELS: dict[PlatformEnum, list[type[MPLinearKernel]]] = {
MacheteLinearKernel,
AllSparkLinearKernel,
MarlinLinearKernel,
HummingLinearKernel,
ConchLinearKernel,
ExllamaLinearKernel,
TritonW4A16LinearKernel,
HummingLinearKernel,
],
PlatformEnum.ROCM: [
RDNA3W4A16LinearKernel,
@@ -400,6 +425,7 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = {
FlashInferCutlassMxfp8LinearKernel,
MarlinMxfp8LinearKernel,
EmulationMxfp8LinearKernel,
HummingMxfp8LinearKernel,
],
PlatformEnum.ROCM: [
# Native CDNA4 (gfx950) MX linear; is_supported() gates to gfx95x and
@@ -426,6 +452,7 @@ _POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = {
FlashInferCudnnNvFp4LinearKernel,
FbgemmNvFp4LinearKernel,
EmulationNvFp4LinearKernel,
HummingNvFp4LinearKernel,
],
PlatformEnum.ROCM: [
EmulationNvFp4LinearKernel,
@@ -436,6 +463,7 @@ _POSSIBLE_MXFP4_KERNELS: dict[PlatformEnum, list[type[MxFp4LinearKernel]]] = {
PlatformEnum.CUDA: [
FlashInferMxFp4LinearKernel,
MarlinMxFp4LinearKernel,
HummingMxFp4LinearKernel,
],
PlatformEnum.XPU: [
XPUMxFp4LinearKernel,
@@ -19,6 +19,9 @@ from vllm.model_executor.kernels.linear.mixed_precision.dynamic_4bit import (
from vllm.model_executor.kernels.linear.mixed_precision.exllama import (
ExllamaLinearKernel,
)
from vllm.model_executor.kernels.linear.mixed_precision.humming import (
HummingLinearKernel,
)
from vllm.model_executor.kernels.linear.mixed_precision.machete import (
MacheteLinearKernel,
)
@@ -52,6 +55,7 @@ __all__ = [
"CutlassW4A8LinearKernel",
"Dynamic4bitLinearKernel",
"ExllamaLinearKernel",
"HummingLinearKernel",
"MacheteLinearKernel",
"MarlinLinearKernel",
"RDNA3W4A16LinearKernel",
@@ -23,8 +23,6 @@ class HummingLinearKernel(MPLinearKernel):
return False, "Humming is not installed"
if c.has_g_idx:
return False, "Humming does not support act-order (g_idx)"
if c.zero_points:
return False, "Humming linear kernel only supports symmetric weights"
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
@@ -41,6 +39,11 @@ class HummingLinearKernel(MPLinearKernel):
"group_size": 0 if group_size == -1 else group_size,
}
if self.config.zero_points:
assert self.w_zp_name is not None
name_map["zero_point"] = self.w_zp_name
quant_config["has_zero_point"] = True
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
prepare_humming_layer(layer, quant_config)
@@ -0,0 +1,63 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.quantization.utils.humming_utils import (
convert_linear_layer_to_humming_standard,
prepare_humming_layer,
)
from vllm.platforms import current_platform
from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
class HummingMxFp4LinearKernel(MxFp4LinearKernel):
"""Humming GEMM Kernel for MXFP4."""
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_cuda():
return False, "Humming only supported on CUDA"
if not current_platform.has_device_capability(75):
return False, "Humming only supported on SM75+"
return True, None
@classmethod
def can_implement(cls, c: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.weight_scale.data = layer.weight_scale.data.view(torch.float8_e8m0fnu)
name_map = {"weight": "weight", "weight_scale": "weight_scale"}
quant_config = {
"quant_method": "humming",
"dtype": "float4e2m1",
"scale_dtype": "float8e8m0",
"group_size": 32,
"weight_scale_type": "group",
}
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
prepare_humming_layer(layer, quant_config)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
from vllm.utils.humming import HummingMethod
flatten_inputs = x.view(-1, x.size(-1))
output = HummingMethod.forward_layer(
layer=layer,
inputs=flatten_inputs,
compute_config=layer.compute_config,
)
return output.view(*x.shape[:-1], output.size(-1))
@@ -0,0 +1,63 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.quantization.utils.humming_utils import (
convert_linear_layer_to_humming_standard,
prepare_humming_layer,
)
from vllm.platforms import current_platform
from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
class HummingMxfp8LinearKernel(Mxfp8LinearKernel):
"""Humming GEMM Kernel for MXFP8."""
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_cuda():
return False, "Humming only supported on CUDA"
if not current_platform.has_device_capability(75):
return False, "Humming only supported on SM75+"
return True, None
@classmethod
def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]:
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.weight_scale.data = layer.weight_scale.data.view(torch.float8_e8m0fnu)
name_map = {"weight": "weight", "weight_scale": "weight_scale"}
quant_config = {
"quant_method": "humming",
"dtype": "float8e4m3",
"scale_dtype": "float8e8m0",
"group_size": 32,
"weight_scale_type": "group",
}
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
prepare_humming_layer(layer, quant_config)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
from vllm.utils.humming import HummingMethod
flatten_inputs = x.view(-1, x.size(-1))
output = HummingMethod.forward_layer(
layer=layer,
inputs=flatten_inputs,
compute_config=layer.compute_config,
)
return output.view(*x.shape[:-1], output.size(-1))
@@ -0,0 +1,73 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.humming_utils import (
prepare_humming_layer,
)
from vllm.platforms import current_platform
from .base import NvFp4LinearKernel, NvFp4LinearLayerConfig
logger = init_logger(__name__)
class HummingNvFp4LinearKernel(NvFp4LinearKernel):
"""Humming GEMM Kernel for NVFP4."""
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_cuda():
return False, "Humming only supported on CUDA"
if not current_platform.has_device_capability(75):
return False, "Humming only supported on SM75+"
return True, None
@classmethod
def can_implement(cls, config: NvFp4LinearLayerConfig) -> tuple[bool, str | None]:
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# Route through humming's compressed-tensors nvfp4 loader (same path as
# the MoE oracle); the native group_tensor schema mishandles a scalar
# global scale.
quant_config = {
"quant_method": "compressed-tensors",
"format": "nvfp4-pack-quantized",
"type": "float",
"num_bits": 4,
"strategy": "group",
"group_size": 16,
}
# CT pack-quantized reads `weight_packed`; the scheme renamed it to `weight`.
if not hasattr(layer, "weight_packed"):
layer.weight_packed = layer.weight
del layer.weight
# The CT linear scheme inverts the global scale (1/scale) for
# marlin/cutlass; humming wants the original.
layer.weight_global_scale = torch.nn.Parameter(
1.0 / layer.weight_global_scale, requires_grad=False
)
prepare_humming_layer(layer, quant_config)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
from vllm.utils.humming import HummingMethod
flatten_inputs = x.view(-1, x.size(-1))
output = HummingMethod.forward_layer(
layer=layer,
inputs=flatten_inputs,
compute_config=layer.compute_config,
)
return output.view(*x.shape[:-1], output.size(-1))
@@ -0,0 +1,156 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.humming_utils import (
convert_linear_layer_to_humming_standard,
prepare_humming_layer,
)
from vllm.platforms import current_platform
from .ScaledMMLinearKernel import (
FP8ScaledMMLinearKernel,
FP8ScaledMMLinearLayerConfig,
Int8ScaledMMLinearKernel,
Int8ScaledMMLinearLayerConfig,
)
logger = init_logger(__name__)
class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
"""Humming GEMM Kernel for FP8."""
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_cuda():
return False, "Humming only supported on CUDA"
if not current_platform.has_device_capability(75):
return False, "Humming only supported on SM75+"
return True, None
@classmethod
def can_implement(
cls, config: FP8ScaledMMLinearLayerConfig
) -> tuple[bool, str | None]:
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
from vllm.utils.humming import dtypes
name_map = {"weight": "weight", "weight_scale": "weight_scale"}
scale_torch_dtype = self.config.weight_quant_key.scale.dtype
scale_dtype = dtypes.DataType.from_torch_dtype(scale_torch_dtype)
quant_config = {
"quant_method": "humming",
"dtype": "float8e4m3",
"scale_dtype": scale_dtype,
}
assert self.config.weight_quant_key.scale2 is None
scale_group_shape = self.config.weight_quant_key.scale.group_shape
if scale_group_shape.is_per_tensor():
quant_config["weight_scale_type"] = "tensor"
if not hasattr(layer, "global_scale") and hasattr(layer, "weight_scale"):
del name_map["weight_scale"]
name_map["global_scale"] = "weight_scale"
elif scale_group_shape.is_per_channel():
quant_config["weight_scale_type"] = "channel"
elif scale_group_shape.is_per_group():
quant_config["weight_scale_type"] = "group"
quant_config["group_size"] = scale_group_shape.col
else:
assert scale_group_shape.row > 0 and scale_group_shape.col > 0
quant_config["weight_scale_type"] = "block"
quant_config["weight_scale_group_size_n"] = scale_group_shape.row
quant_config["weight_scale_group_size"] = scale_group_shape.col
if hasattr(layer, "weight_scale_inv"):
name_map["weight_scale"] = "weight_scale_inv"
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
prepare_humming_layer(layer, quant_config)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
from vllm.utils.humming import HummingMethod
flatten_inputs = x.view(-1, x.size(-1))
output = HummingMethod.forward_layer(
layer=layer,
inputs=flatten_inputs,
compute_config=layer.compute_config,
)
return output.view(*x.shape[:-1], output.size(-1))
def apply_scaled_mm(
self,
*,
A: torch.Tensor,
B: torch.Tensor,
out_dtype: torch.dtype,
As: torch.Tensor,
Bs: torch.Tensor,
bias: torch.Tensor | None,
output_shape: list,
) -> torch.Tensor:
pass
class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):
"""Humming GEMM Kernel for INT8."""
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_cuda():
return False, "Humming only supported on CUDA"
if not current_platform.has_device_capability(75):
return False, "Humming only supported on SM75+"
return True, None
@classmethod
def can_implement(
cls, config: Int8ScaledMMLinearLayerConfig
) -> tuple[bool, str | None]:
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
weight_name, weight_scale_name, *_ = self.layer_param_names
name_map = {"weight": weight_name, "weight_scale": weight_scale_name}
quant_config = {"quant_method": "humming", "dtype": "int8"}
weight = getattr(layer, weight_name)
weight.data = weight.data + 128
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
prepare_humming_layer(layer, quant_config)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
from vllm.utils.humming import HummingMethod
flatten_inputs = x.view(-1, x.size(-1))
output = HummingMethod.forward_layer(
layer=layer,
inputs=flatten_inputs,
compute_config=layer.compute_config,
)
return output.view(*x.shape[:-1], output.size(-1))