[Feature][Humming] Humming feature integration (#56685)

Signed-off-by: jinzhen.ljz <jinzhen.ljz@antgroup.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Jinzhen Lin
2026-09-20 08:11:01 -07:00
committed by GitHub
co-authored by Codex mgoin
parent 27757dde02
commit 7d99c2c4fd
62 changed files with 2040 additions and 1086 deletions
+2
View File
@@ -2272,6 +2272,7 @@ def moe_align_block_size(
experts_ids: torch.Tensor,
num_tokens_post_pad: torch.Tensor,
expert_map: torch.Tensor | None = None,
scatter_idx: torch.Tensor | None = None,
) -> None:
torch.ops._moe_C.moe_align_block_size(
topk_ids,
@@ -2281,6 +2282,7 @@ def moe_align_block_size(
experts_ids,
num_tokens_post_pad,
expert_map,
scatter_idx,
)
+2 -1
View File
@@ -1524,7 +1524,8 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_HUMMING_ONLINE_QUANT_CONFIG": lambda: maybe_convert_json_str_or_file(
os.environ.get("VLLM_HUMMING_ONLINE_QUANT_CONFIG", None)
),
# The activation dtype config for humming kernel
# The activation dtype config for humming kernel. Explicit schemas disable
# fallback unless the config includes "allow_fallback": true.
"VLLM_HUMMING_INPUT_QUANT_CONFIG": lambda: maybe_convert_json_str_or_file(
os.environ.get("VLLM_HUMMING_INPUT_QUANT_CONFIG", None)
),
@@ -101,6 +101,7 @@ from vllm.model_executor.kernels.linear.mxfp6 import (
from vllm.model_executor.kernels.linear.mxfp6.emulation import (
EmulationMxfp6LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp6.humming import HummingMxFp6LinearKernel
from vllm.model_executor.kernels.linear.mxfp8 import (
Mxfp8LinearKernel,
Mxfp8LinearLayerConfig,
@@ -295,6 +296,7 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
HummingLinearKernel,
HummingMxfp8LinearKernel,
HummingMxFp4LinearKernel,
HummingMxFp6LinearKernel,
HummingNvFp4LinearKernel,
},
"marlin": {
@@ -574,6 +576,7 @@ _POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = {
_POSSIBLE_MXFP6_KERNELS: dict[PlatformEnum, list[type[MxFp6LinearKernel]]] = {
PlatformEnum.CUDA: [
HummingMxFp6LinearKernel,
EmulationMxfp6LinearKernel,
],
PlatformEnum.ROCM: [
@@ -1294,6 +1297,7 @@ __all__ = [
"MxFp6LinearLayerConfig",
"init_mxfp6_linear_kernel",
"EmulationMxfp6LinearKernel",
"HummingMxFp6LinearKernel",
"AiterMxfp4LinearKernel",
"EmulationMxfp4LinearKernel",
"FlashInferMxFp4LinearKernel",
@@ -24,7 +24,7 @@ class HummingLinearKernel(MPLinearKernel):
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
from vllm.model_executor.layers.quantization.utils.humming_utils import (
from vllm.model_executor.layers.quantization.utils.humming import (
convert_linear_layer_to_humming_standard,
get_humming_linear_compute_config,
prepare_humming_linear_layer_config,
@@ -57,13 +57,14 @@ class HummingLinearKernel(MPLinearKernel):
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
from vllm.model_executor.layers.quantization.utils.humming_utils import (
from vllm.model_executor.layers.quantization.utils.humming import (
apply_humming_linear,
)
return apply_humming_linear(
layer,
x,
skip_bias_add=bias is None,
layer_config=self.layer_config,
compute_config=self.compute_config,
locks=self.locks,
@@ -3,21 +3,18 @@
import torch
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.humming_utils import (
from vllm.model_executor.layers.quantization.utils.humming import (
apply_humming_linear,
convert_linear_layer_to_humming_standard,
get_humming_linear_compute_config,
prepare_humming_linear_layer_config,
quant_key_to_input_schema,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Dynamic
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_humming
from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
logger = init_logger(__name__)
class HummingMxFp4LinearKernel(MxFp4LinearKernel):
"""Humming GEMM Kernel for MXFP4."""
@@ -39,14 +36,10 @@ class HummingMxFp4LinearKernel(MxFp4LinearKernel):
@classmethod
def can_implement(cls, config: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
if config.activation_quant_key not in (None, kMxfp4Dynamic):
return False, "only supports MXFP4 dynamic or unquantized activations"
if config.activation_quant_key is not None:
logger.warning_once(
"HummingMxFp4LinearKernel is a weight-only (A16) kernel; "
"the requested activation quantization (%s) is ignored.",
config.activation_quant_key,
)
try:
quant_key_to_input_schema(config.activation_quant_key)
except ValueError as error:
return False, str(error)
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
@@ -62,7 +55,10 @@ class HummingMxFp4LinearKernel(MxFp4LinearKernel):
}
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)
input_schema = quant_key_to_input_schema(self.config.activation_quant_key)
self.layer_config = prepare_humming_linear_layer_config(
layer, quant_config, input_schema=input_schema
)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
@@ -75,6 +71,7 @@ class HummingMxFp4LinearKernel(MxFp4LinearKernel):
return apply_humming_linear(
layer,
x,
skip_bias_add=bias is None,
layer_config=self.layer_config,
compute_config=self.compute_config,
locks=self.locks,
@@ -13,8 +13,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
class MxFp6LinearLayerConfig:
"""Configuration for an MXFP6 linear layer.
All MXFP6 layers share the same structure: packed uint8 weights (2 FP4 values per
byte) and per-block weight scales (group size 32).
All MXFP6 layers share the same structure: packed uint8 weights (4 FP6 values
per 3 bytes) and per-block weight scales (group size 32).
Attributes:
weight_quant_key: Identifies the weight quantization format. Can be
@@ -0,0 +1,84 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.quantization.utils.humming import (
apply_humming_linear,
convert_linear_layer_to_humming_standard,
get_humming_linear_compute_config,
prepare_humming_linear_layer_config,
quant_key_to_input_schema,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp6E2M3Static
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_humming
from .base import MxFp6LinearKernel, MxFp6LinearLayerConfig
class HummingMxFp6LinearKernel(MxFp6LinearKernel):
"""Humming GEMM for packed MXFP6 E2M3 and E3M2 weights."""
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_cuda():
return False, "Humming only supported on CUDA"
if not has_humming():
return False, "Humming is not installed"
if not current_platform.has_device_capability(75):
return False, "Humming only supported on SM75+"
return True, None
@classmethod
def can_implement(cls, config: MxFp6LinearLayerConfig) -> tuple[bool, str | None]:
try:
quant_key_to_input_schema(config.activation_quant_key)
except ValueError as error:
return False, str(error)
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
weight_dtype = "float6e3m2"
if self.config.weight_quant_key == kMxfp6E2M3Static:
weight_dtype = "float6e2m3"
quant_config = {
"quant_method": "humming",
"dtype": weight_dtype,
"scale_dtype": "float8e8m0",
"group_size": 32,
"weight_scale_type": "group",
}
layer.weight_scale.data = layer.weight_scale.data.view(torch.float8_e8m0fnu)
convert_linear_layer_to_humming_standard(
layer=layer,
name_map={"weight": "weight", "weight_scale": "weight_scale"},
)
input_schema = quant_key_to_input_schema(self.config.activation_quant_key)
self.layer_config = prepare_humming_linear_layer_config(
layer, quant_config, input_schema=input_schema
)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return apply_humming_linear(
layer,
x,
skip_bias_add=bias is None,
layer_config=self.layer_config,
compute_config=self.compute_config,
locks=self.locks,
)
@@ -3,12 +3,14 @@
import torch
from vllm.model_executor.layers.quantization.utils.humming_utils import (
from vllm.model_executor.layers.quantization.utils.humming import (
apply_humming_linear,
convert_linear_layer_to_humming_standard,
get_humming_linear_compute_config,
prepare_humming_linear_layer_config,
quant_key_to_input_schema,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Dynamic
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_humming
@@ -50,7 +52,10 @@ class HummingMxfp8LinearKernel(Mxfp8LinearKernel):
}
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)
input_schema = quant_key_to_input_schema(kMxfp8Dynamic)
self.layer_config = prepare_humming_linear_layer_config(
layer, quant_config, input_schema=input_schema
)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
@@ -63,6 +68,7 @@ class HummingMxfp8LinearKernel(Mxfp8LinearKernel):
return apply_humming_linear(
layer,
x,
skip_bias_add=bias is None,
layer_config=self.layer_config,
compute_config=self.compute_config,
locks=self.locks,
@@ -4,11 +4,13 @@
import torch
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.humming_utils import (
from vllm.model_executor.layers.quantization.utils.humming import (
apply_humming_linear,
get_humming_linear_compute_config,
prepare_humming_linear_layer_config,
quant_key_to_input_schema,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import kNvfp4Dynamic
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_humming
@@ -40,9 +42,7 @@ class HummingNvFp4LinearKernel(NvFp4LinearKernel):
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# Route through humming's compressed-tensors nvfp4 loader (same path as
# the MoE oracle); the native group_tensor schema mishandles a scalar
# global scale.
# Reuse the CT loader for packed FP4 weights and tensor scales.
quant_config = {
"quant_method": "compressed-tensors",
"format": "nvfp4-pack-quantized",
@@ -60,7 +60,16 @@ class HummingNvFp4LinearKernel(NvFp4LinearKernel):
layer.weight_global_scale = torch.nn.Parameter(
1.0 / layer.weight_global_scale, requires_grad=False
)
self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)
input_quant_key = None
if hasattr(layer, "input_global_scale"):
layer.input_scale_2 = torch.nn.Parameter(
layer.input_global_scale.reshape(-1), requires_grad=False
)
input_quant_key = kNvfp4Dynamic
input_schema = quant_key_to_input_schema(input_quant_key)
self.layer_config = prepare_humming_linear_layer_config(
layer, quant_config, input_schema=input_schema
)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
@@ -73,6 +82,7 @@ class HummingNvFp4LinearKernel(NvFp4LinearKernel):
return apply_humming_linear(
layer,
x,
skip_bias_add=bias is None,
layer_config=self.layer_config,
compute_config=self.compute_config,
locks=self.locks,
@@ -4,7 +4,7 @@
import torch
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.humming_utils import (
from vllm.model_executor.layers.quantization.utils.humming import (
apply_humming_linear,
convert_linear_layer_to_humming_standard,
get_humming_linear_compute_config,
@@ -92,6 +92,7 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
return apply_humming_linear(
layer,
x,
skip_bias_add=bias is None,
layer_config=self.layer_config,
compute_config=self.compute_config,
locks=self.locks,
@@ -158,6 +159,7 @@ class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):
return apply_humming_linear(
layer,
x,
skip_bias_add=bias is None,
layer_config=self.layer_config,
compute_config=self.compute_config,
locks=self.locks,
@@ -4,6 +4,7 @@
import json
import math
from dataclasses import replace
from typing import TYPE_CHECKING, Any, cast
import torch
@@ -30,6 +31,7 @@ from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import (
get_moe_permute_scratch,
moe_permute,
moe_permute_unpermute_supported,
moe_prepare_scatter,
moe_unpermute,
)
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
@@ -57,14 +59,16 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kMxfp8Static,
kNvfp4Dynamic,
kNvfp4Static,
kStaticTensorScale,
)
from vllm.platforms import current_platform
from vllm.scalar_type import ScalarType
from vllm.utils.import_utils import has_humming
from vllm.utils.platform_utils import num_compute_units
from vllm.v1.worker.workspace import current_workspace_manager
if TYPE_CHECKING:
from vllm.model_executor.layers.quantization.utils.humming_utils import (
from vllm.model_executor.layers.quantization.utils.humming import (
HummingMoEQuantConfig,
)
from vllm.utils.humming import (
@@ -118,38 +122,9 @@ def get_humming_moe_gemm_type(moe_config: FusedMoEConfig | None = None) -> str:
return gemm_type
def _fixup_moe_tuning_config(tuning_config: list, max_k_block: int = 128) -> None:
"""Fix up each MoE tile in place: cap the K-block and widen warp-N.
- block_shape[2] (K-block) > ``max_k_block``: the driver rejects the TMA
descriptor at launch (CUDA_ERROR_MISALIGNED_ADDRESS). Cap at 128, which
Humming already uses for larger M.
- warp_shape[1] (warp-N) < 32: block-FP8 (group-128) activations route the
w13 (gate/up) GEMM to a tuning table that pins warp-N to 16, under-filling
the Hopper WGMMA N dimension and corrupting the GEMM output (gsm8k
0.94 -> 0.89). Widen to 32 whenever block_n % 32 == 0 -- on every tile,
not just the K-capped ones. w2 (down) already uses warp_n=32.
"""
logger.info_once("Attempting to override humming GEMM config")
for entry in tuning_config:
config = entry[2]
block_shape = config.get("block_shape")
if not (block_shape and len(block_shape) == 3):
continue
block_m, block_n, block_k = block_shape
logger.info_once(f"Overriding humming GEMM config. Previous config\n: {config}")
if block_k > max_k_block:
config["block_shape"] = [block_m, block_n, max_k_block]
warp_shape = config.get("warp_shape")
if warp_shape and warp_shape[1] < 32 and block_n % 32 == 0:
config["warp_shape"] = [warp_shape[0], 32, warp_shape[2]]
logger.info_once(f"Overridden humming GEMM config. Current config\n: {config}")
class HummingExpertsBase(mk.FusedMoEExpertsModular):
quant_config: "HummingMoEQuantConfig"
def __init__(
self,
moe_config: FusedMoEConfig,
@@ -165,7 +140,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
self.locks = torch.zeros(1024, dtype=torch.int32, device=moe_config.device)
self.num_experts = moe_config.num_local_experts
self.global_num_experts = moe_config.num_experts
self.quant_config = quant_config
self.quant_config = humming_quant_config
self.init_humming_moe()
if self.is_batched():
@@ -199,31 +174,61 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
gemm_type=self.humming_gemm_type(),
)
self.compute_config_str = json.dumps(self.compute_config)
# Fix up the heuristic-chosen tiles before freezing to JSON -- see
# _fixup_moe_tuning_config.
_fixup_moe_tuning_config(self.w13_tuning_config)
_fixup_moe_tuning_config(self.w2_tuning_config)
self.w13_tuning_config_str = json.dumps(self.w13_tuning_config)
self.w2_tuning_config_str = json.dumps(self.w2_tuning_config)
def quantize_input(
def process_input(
self,
sublayer_name: str,
inputs: torch.Tensor,
quanted_input: torch.Tensor | None,
input_scale: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
from vllm.utils.humming import may_quant_input
# input_scale is set for block-FP8 (group-128) activations that were
# quantized before the EP dispatch: may_quant_input then skips the
# redundant w13 quantization and forwards the pre-computed scale.
return may_quant_input(
self.humming_configs[sublayer_name],
inputs=inputs,
input_scale=input_scale,
quanted_input=quanted_input,
expert_tokens: torch.Tensor | None = None,
activation: MoEActivation | None = None,
scatter_idx: torch.Tensor | None = None,
num_valid_tokens: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None]:
from vllm.model_executor.layers.quantization.utils.humming.activation import (
get_humming_activation,
)
from vllm.utils.humming import may_process_input
quant_config = self.quant_config
config = self.humming_configs[sublayer_name]
if input_scale is not None:
assert scatter_idx is None
if activation is not None:
raise ValueError("Cannot apply activation to prequantized input")
return inputs, input_scale, None
activation_kwargs = {}
if activation is not None:
activation_config = self.activation_config
activation_kwargs = get_humming_activation(activation, activation_config)
prefix = "w1" if sublayer_name == "w13" else "w2"
scale = getattr(quant_config, f"{prefix}_input_scale")
scale2 = getattr(quant_config, f"{prefix}_input_scale_2")
mode = config.input_quant_mode
layout = "grouped_mask" if self.is_batched() else "normal"
if scatter_idx is not None:
layout = "scatter"
inputs, group_scales, token_scales = may_process_input(
config,
inputs=inputs,
outputs=quanted_input,
token_scales=scale2 if mode.has_secondary_scale else scale,
hadamard_block_size=getattr(quant_config, f"{prefix}_hadamard_block_size"),
layout=layout,
expert_tokens=expert_tokens if layout == "grouped_mask" else None,
scatter_idx=scatter_idx,
num_valid_tokens=num_valid_tokens,
**activation_kwargs,
)
input_scale = group_scales if mode.has_group_scale else token_scales
input_scale_2 = token_scales if mode.has_secondary_scale else None
return inputs, input_scale, input_scale_2
def humming_forward(
self,
@@ -232,30 +237,30 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
weight: torch.Tensor,
input_scale: torch.Tensor | None,
outputs: torch.Tensor,
input_scale_2: torch.Tensor | None = None,
**kwargs: Any,
) -> torch.Tensor:
from vllm.utils.humming import humming_forward
is_w13 = sublayer_name == "w13"
index = 1 if sublayer_name == "w13" else 2
return humming_forward(
self.humming_configs[sublayer_name],
inputs=inputs,
weight=weight,
weight_scale=(
self.quant_config.w1_scale if is_w13 else self.quant_config.w2_scale
),
zero_point=(self.quant_config.w1_zp if is_w13 else self.quant_config.w2_zp),
bias=(self.quant_config.w1_bias if is_w13 else self.quant_config.w2_bias),
weight_scale_2=(
self.quant_config.g1_alphas if is_w13 else self.quant_config.g2_alphas
),
weight_scale=getattr(self.quant_config, f"w{index}_scale"),
zero_point=getattr(self.quant_config, f"w{index}_zp"),
bias=getattr(self.quant_config, f"w{index}_bias"),
weight_scale_2=getattr(self.quant_config, f"g{index}_alphas"),
input_scale=input_scale,
input_scale_2=input_scale_2,
outputs=outputs,
locks=self.locks,
**kwargs,
)
def _get_permute_scratch(self, topk: int) -> MoEPermuteScratch | None:
def _get_permute_scratch(
self, topk: int, indices_only: bool = False
) -> MoEPermuteScratch | None:
if not moe_permute_unpermute_supported():
return None
@@ -270,8 +275,8 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
num_experts=self.moe_config.num_experts,
num_local_experts=self.moe_config.num_local_experts,
device=torch.device(self.moe_config.device),
hidden_size=self.moe_config.hidden_dim,
hidden_dtype=self.moe_config.in_dtype,
hidden_size=None if indices_only else self.moe_config.hidden_dim,
hidden_dtype=None if indices_only else self.moe_config.in_dtype,
)
def get_global_valid_shape_m(self, topk_ids: torch.Tensor):
@@ -302,6 +307,16 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
if (
activation_key is not None
and activation_key.scale == kNvfp4Dynamic.scale
and activation_key.dtype == kNvfp4Dynamic.dtype
and activation_key.scale2 is not None
and not activation_key.scale2.static
and activation_key.scale2.group_shape == GroupShape.PER_TOKEN
and activation_key.scale2.dtype == torch.float32
):
activation_key = replace(activation_key, scale2=kStaticTensorScale)
SUPPORTED_W_A = [
(kMxfp4Static, None),
(kMxfp4Static, kMxfp4Dynamic),
@@ -343,7 +358,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
(kMxfp8Static, kMxfp8Dynamic),
]
return (weight_key, activation_key) in SUPPORTED_W_A or (
activation_key in (None, kFp8DynamicTokenSym)
activation_key in (None, kFp8DynamicTokenSym, kInt8DynamicTokenSym)
and _is_supported_wna16_weight_key(weight_key)
)
@@ -355,7 +370,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
to FP8 before dispatch sends FP8 rather than BF16 over the interconnect,
and Humming then consumes the pre-quantized FP8 + scale directly (see
the apply() methods, which forward the dispatch scale into
HummingMethod.may_quant_input, and may_quant_input itself, which is a
HummingExpertsBase.process_input, which is a
no-op when an input scale is already supplied). The scale layout
produced by vLLM's block-FP8 quantization ([M, K // 128] float32,
row-major) matches what the Humming WGMMA grouped GEMM expects.
@@ -373,7 +388,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
inputs).
Humming normally quantizes inputs internally via
HummingMethod.may_quant_input() in apply(), so we defer quantization
HummingExpertsBase.process_input() in apply(), so we defer quantization
(return True) to avoid quantizing twice -- once in prepare and once in
Humming's apply().
@@ -398,8 +413,6 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
# Humming uses apply_moe_activation() callback for activation,
# so any activation supported there can be used here.
return apply_moe_activation_supported(activation)
@staticmethod
@@ -456,10 +469,9 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
assert intermediate_dim == self.adjust_N_for_activation(gate_up_dim, activation)
# hidden_states
# (-> quanted_gate_up_input) (if not BF16/FP16 activation)
# -> quanted_gate_up_input (Hadamard + optional quantization)
# -> gate_up_output
# -> activation_output
# (-> quanted_down_input) (if not BF16/FP16 activation)
# -> quanted_down_input (activation + Hadamard + optional quantization)
# -> down_output
# (-> output) (if not is_batched)
# Neighboring nodes are required to utilize distinct workspaces.
@@ -471,8 +483,8 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
max_num_tokens = self.max_num_tokens
num_dispatchers = self.num_dispatchers
assert max_num_tokens is not None and num_dispatchers is not None
input_shape_m = num_experts * max_num_tokens
real_shape_m = num_experts * max_num_tokens * num_dispatchers
input_shape_m = real_shape_m
output_shape = (num_experts, max_num_tokens * num_dispatchers, K)
else:
input_shape_m = M
@@ -483,15 +495,18 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
a_dtype = self.humming_configs["w13"].a_dtype
c_dtype = self.humming_configs["w13"].c_dtype
num_bits = a_dtype.num_bits
down_a_dtype = w2_config.a_dtype
torch_dtype_map = {
dtypes.float16: torch.float16,
dtypes.bfloat16: torch.bfloat16,
dtypes.float32: torch.float32,
dtypes.float8e3m4: torch.uint8,
dtypes.float8e4m3: torch.float8_e4m3fn,
dtypes.float8e5m2: torch.float8_e5m2,
dtypes.int8: torch.int8,
dtypes.int4: torch.uint8,
dtypes.float4e0m3: torch.uint8,
dtypes.float4e2m1: torch.uint8,
}
buffer_metas = {
@@ -503,13 +518,9 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
"shape": (real_shape_m, gate_up_dim),
"dtype": torch_dtype_map[c_dtype],
},
"activation_output": {
"shape": (real_shape_m, intermediate_dim),
"dtype": torch_dtype_map[c_dtype],
},
"quanted_down_input": {
"shape": (real_shape_m, intermediate_dim),
"dtype": torch_dtype_map[a_dtype],
"dtype": torch_dtype_map[down_a_dtype],
},
"down_output": {
"shape": output_shape if self.is_batched() else (real_shape_m, K),
@@ -523,7 +534,8 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
for key in buffer_metas:
meta = buffer_metas[key]
if "quanted" in key and a_dtype.num_bits == 4:
input_dtype = down_a_dtype if key == "quanted_down_input" else a_dtype
if "quanted" in key and input_dtype.num_bits == 4:
last_dim = meta["shape"][-1]
if last_dim % 2 != 0:
raise ValueError(
@@ -532,25 +544,12 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
)
meta["shape"] = meta["shape"][:-1] + (last_dim // 2,)
if num_bits == 16:
required_buffers = ["gate_up_output", "activation_output", "down_output"]
else:
required_buffers = [
"quanted_gate_up_input",
"gate_up_output",
"activation_output",
"quanted_down_input",
"down_output",
]
# Note: The fused SITU+FP8 quant goes gate_up_output ->
# quanted_down_input directly, never materializing
# activation_output. Dropping activation_output from the chain
# flips the even/odd workspace 2-coloring below so
# gate_up_output and quanted_down_input land
# on DIFFERENT workspaces.
if self.fused_situ_quant_enabled(activation):
required_buffers.remove("activation_output")
required_buffers = [
"quanted_gate_up_input",
"gate_up_output",
"quanted_down_input",
"down_output",
]
# batched moe use down_output as output
if not self.is_batched():
@@ -673,81 +672,6 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
activation_kwargs["valid_token_counts"] = valid_token_counts
self.activation(**activation_kwargs)
def fused_situ_quant_enabled(self, activation: MoEActivation) -> bool:
"""Whether the SITU activation + w2 quant can be fused into one kernel.
Fused only for k-major block-FP8 group-128 e4m3 with float32 scales --
the sole layout situ_and_mul_quant supports. A float32 as_dtype rules out
MXMMA (which uses e8m0 m-major scales), so a group-128 scale is
guaranteed k-major. Everything else (m-major MXFP8, other group sizes,
16-bit passthrough, non-SITU) falls back to the separate
situ_and_mul + quantize_input path.
"""
if activation != MoEActivation.SITU:
return False
w2cfg = self.humming_configs["w2"]
# Report the specific blocker once so a silent fallback to the unfused
# situ_and_mul + quant pair is never a mystery in a trace.
reason: str | None = None
if not (w2cfg.a_dtype.num_bits == 8 and str(w2cfg.a_dtype) == "float8e4m3"):
reason = f"w2 a_dtype is {w2cfg.a_dtype} (need float8e4m3)"
elif w2cfg.input_scale_group_size != 128:
reason = (
f"w2 input_scale_group_size is {w2cfg.input_scale_group_size} "
"(need 128)"
)
elif not (w2cfg.as_dtype is not None and str(w2cfg.as_dtype) == "float32"):
reason = f"w2 as_dtype is {w2cfg.as_dtype} (need float32, k-major)"
if reason is not None:
logger.warning_once(
"Humming fused SITU+FP8 quant disabled, using unfused "
"situ_and_mul + quant: %s",
reason,
)
return False
return True
def fused_situ_quant(
self,
gate_up_output: torch.Tensor,
quanted_down_input: torch.Tensor,
num_valid_tokens: torch.Tensor | None,
topk: int = 1,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Run the fused SITU activation + FP8 quant for the w2 input.
Returns ``(quanted_down_input, input_scale)`` in the same layout the
unfused ``apply_activation`` + ``quantize_input("w2")`` pair produces: an
fp8 [M, d] tensor plus a k-major block-FP8 float32 [M, d // 128] group
scale. Only reached when ``fused_situ_quant_enabled`` (group_size 128).
"""
from vllm.model_executor.layers.fused_moe.activation import (
situ_and_mul_quant,
)
cfg = self.activation_config
assert cfg.activation_situ_beta is not None, (
"SITU requires activation_situ_beta from FusedMoEConfig"
)
group_size = self.humming_configs["w2"].input_scale_group_size
m, d = quanted_down_input.size(0), quanted_down_input.size(1)
input_scale = torch.empty(
(m, d // group_size),
dtype=torch.float32,
device=quanted_down_input.device,
)
situ_and_mul_quant(
quanted_down_input,
input_scale,
gate_up_output,
beta=cfg.activation_situ_beta,
linear_beta=cfg.activation_situ_linear_beta,
group_size=group_size,
num_valid_tokens=num_valid_tokens,
topk=topk,
)
return quanted_down_input, input_scale
class HummingIndexedExperts(HummingExpertsBase):
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
@@ -768,7 +692,7 @@ class HummingIndexedExperts(HummingExpertsBase):
topk_ids: torch.Tensor,
expert_map: torch.Tensor | None,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
) -> tuple[dict[str, Any], dict[str, Any]]:
) -> tuple[dict[str, Any], dict[str, Any], torch.Tensor | None]:
valid_shape_m = self.estimate_local_valid_shape_m(topk_ids)
moe_block_size = None
@@ -784,13 +708,21 @@ class HummingIndexedExperts(HummingExpertsBase):
)
moe_block_size = 64
sorted_ids, expert_ids, num_tokens_padded = moe_align_block_size(
use_scatter = False
if expert_map is not None:
num_sms = num_compute_units(topk_ids.get_device())
use_scatter = topk_ids.numel() > 2 * num_sms
alignment = moe_align_block_size(
topk_ids=topk_ids,
block_size=moe_block_size,
num_experts=self.global_num_experts,
expert_map=expert_map,
ignore_invalid_experts=True,
return_scatter_idx=use_scatter,
)
sorted_ids, expert_ids, num_tokens_padded = alignment[:3]
scatter_idx = alignment[3] if len(alignment) == 4 else None
moe_common_kwargs = {
"sorted_ids": sorted_ids,
@@ -806,7 +738,26 @@ class HummingIndexedExperts(HummingExpertsBase):
moe_kwargs1.update(moe_common_kwargs)
moe_kwargs2.update(moe_common_kwargs)
return moe_kwargs1, moe_kwargs2
w2_block_size = 64
for lower, upper, config in self.w2_tuning_config:
if lower < valid_shape_m <= upper:
w2_block_size = config["block_shape"][0]
break
if w2_block_size != moe_block_size:
sorted_ids2, expert_ids2, num_tokens_padded2 = moe_align_block_size(
topk_ids=topk_ids,
block_size=w2_block_size,
num_experts=self.global_num_experts,
expert_map=expert_map,
ignore_invalid_experts=True,
)
moe_kwargs2.update(
sorted_ids=sorted_ids2,
expert_ids=expert_ids2,
num_tokens_padded=num_tokens_padded2,
)
return moe_kwargs1, moe_kwargs2, scatter_idx
def apply(
self,
@@ -832,7 +783,7 @@ class HummingIndexedExperts(HummingExpertsBase):
object, so w1, w2, a2_scale are unused. a1q_scale is None on the usual
path (Humming quantizes the w13 input itself); for block-FP8 activations
it carries the scale computed before dispatch, which is forwarded to
may_quant_input so Humming skips the redundant w13 quantization. The
process_input so Humming skips the redundant w13 quantization. The
output is written into workspace13 via the buffer management.
"""
assert not apply_router_weight_on_input
@@ -847,13 +798,13 @@ class HummingIndexedExperts(HummingExpertsBase):
)
buffers["output"] = output
moe_kwargs1, moe_kwargs2 = self.prepare_humming_moe_kwargs(
moe_kwargs1, moe_kwargs2, scatter_idx = self.prepare_humming_moe_kwargs(
topk_ids=topk_ids,
expert_map=expert_map,
expert_tokens_meta=expert_tokens_meta,
)
inputs, input_scale = self.quantize_input(
inputs, input_scale, input_scale_2 = self.process_input(
"w13",
inputs=hidden_states,
input_scale=a1q_scale,
@@ -865,52 +816,25 @@ class HummingIndexedExperts(HummingExpertsBase):
inputs=inputs,
weight=w1,
input_scale=input_scale,
input_scale_2=input_scale_2,
outputs=buffers["gate_up_output"],
**moe_kwargs1,
)
# psum[-1:] is the DeepEP valid *token* count as a zero-cost int32 view.
# The fused path and mul_sum consume tokens directly. The unfused
# activation consumes the expanded row count (tokens * topk).
valid_tokens = None
topk = topk_ids.size(1)
if (
expert_tokens_meta is not None
and expert_tokens_meta.psum_recv_per_rank is not None
):
valid_tokens = expert_tokens_meta.psum_recv_per_rank[-1:]
if self.fused_situ_quant_enabled(activation):
# Fused SITU + FP8 quant (per-token or block-FP8 group-128) straight
# into the w2 input, skipping the bf16 activation_output round-trip.
inputs, input_scale = self.fused_situ_quant(
gate_up_output=buffers["gate_up_output"],
quanted_down_input=buffers["quanted_down_input"],
num_valid_tokens=valid_tokens,
topk=topk,
)
else:
valid_token_counts = (
valid_tokens * topk if valid_tokens is not None else None
)
self.apply_activation(
activation=activation,
input=buffers["gate_up_output"],
output=buffers["activation_output"],
valid_token_counts=valid_token_counts,
)
inputs, input_scale = self.quantize_input(
"w2",
inputs=buffers["activation_output"],
quanted_input=buffers.get("quanted_down_input", None),
)
inputs, input_scale, input_scale_2 = self.process_input(
"w2",
inputs=buffers["gate_up_output"],
quanted_input=buffers["quanted_down_input"],
activation=activation,
scatter_idx=scatter_idx,
)
self.humming_forward(
"w2",
inputs=inputs,
weight=w2,
input_scale=input_scale,
input_scale_2=input_scale_2,
outputs=buffers["down_output"].view(-1, hidden_states.size(-1)),
**moe_kwargs2,
)
@@ -918,6 +842,10 @@ class HummingIndexedExperts(HummingExpertsBase):
# expert_map masks any non-local id; num_valid_tokens bounds the
# persistent kernel to the real token rows [0, num_recv) so the padding
# tail is never iterated (CUDA-graph-safe device scalar).
valid_tokens = None
if expert_tokens_meta and expert_tokens_meta.psum_recv_per_rank is not None:
valid_tokens = expert_tokens_meta.psum_recv_per_rank[-1:]
moe_fused_mul_sum(
inputs=buffers["down_output"].view(*topk_ids.shape, -1),
topk_weights=topk_weights,
@@ -983,21 +911,33 @@ class HummingGroupedExperts(HummingExpertsBase):
)
buffers["output"] = output
hidden_states, a1q_scale, expert_first_token_offset, inv_perm, _ = moe_permute(
hidden_states=hidden_states,
a1q_scale=a1q_scale,
topk_ids=topk_ids,
n_expert=global_num_experts,
n_local_expert=self.num_experts,
expert_map=expert_map,
scratch=self._get_permute_scratch(topk_ids.size(1)),
)
if a1q_scale is None:
scratch = self._get_permute_scratch(topk_ids.size(1), indices_only=True)
assert scratch is not None
scatter_metadata = moe_prepare_scatter(topk_ids, expert_map, scratch)
expert_offsets, scatter_idx = scatter_metadata
inv_perm = scatter_idx.flatten()
num_valid_tokens = expert_offsets[-1:] if expert_map is not None else None
else:
hidden_states, a1q_scale, expert_offsets, inv_perm, _ = moe_permute(
hidden_states=hidden_states,
a1q_scale=a1q_scale,
topk_ids=topk_ids,
n_expert=global_num_experts,
n_local_expert=self.num_experts,
expert_map=expert_map,
scratch=self._get_permute_scratch(topk_ids.size(1)),
)
scatter_idx = None
num_valid_tokens = None
inputs, input_scale = self.quantize_input(
inputs, input_scale, input_scale_2 = self.process_input(
"w13",
inputs=hidden_states,
input_scale=a1q_scale,
quanted_input=buffers.get("quanted_gate_up_input", None),
scatter_idx=scatter_idx,
num_valid_tokens=num_valid_tokens,
)
self.humming_forward(
@@ -1005,28 +945,20 @@ class HummingGroupedExperts(HummingExpertsBase):
inputs=inputs,
weight=w1,
input_scale=input_scale,
input_scale_2=input_scale_2,
outputs=buffers["gate_up_output"],
valid_shape_m=valid_shape_m,
expert_layout=expert_first_token_offset,
expert_layout=expert_offsets,
compute_config=self.compute_config_str,
tuning_config=self.w13_tuning_config_str,
)
self.apply_activation(
activation=activation,
input=buffers["gate_up_output"],
output=buffers["activation_output"],
valid_token_counts=(
expert_first_token_offset[-1:].to(torch.int32)
if expert_tokens_meta is None
else None
),
)
inputs, input_scale = self.quantize_input(
inputs, input_scale, input_scale_2 = self.process_input(
"w2",
inputs=buffers["activation_output"],
quanted_input=buffers.get("quanted_down_input", None),
inputs=buffers["gate_up_output"],
quanted_input=buffers["quanted_down_input"],
activation=activation,
num_valid_tokens=expert_offsets[-1:],
)
self.humming_forward(
@@ -1034,9 +966,10 @@ class HummingGroupedExperts(HummingExpertsBase):
inputs=inputs,
weight=w2,
input_scale=input_scale,
input_scale_2=input_scale_2,
outputs=buffers["down_output"],
valid_shape_m=valid_shape_m,
expert_layout=expert_first_token_offset,
expert_layout=expert_offsets,
compute_config=self.compute_config_str,
tuning_config=self.w2_tuning_config_str,
)
@@ -1046,7 +979,7 @@ class HummingGroupedExperts(HummingExpertsBase):
permuted_hidden_states=buffers["down_output"].view(*topk_ids.shape, -1),
topk_weights=topk_weights,
inv_permuted_idx=inv_perm,
expert_first_token_offset=expert_first_token_offset,
expert_first_token_offset=expert_offsets,
)
@@ -1088,7 +1021,7 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):
object, so w1, w2, a2_scale are unused. a1q_scale is None on the usual
path (Humming quantizes the w13 input itself); for block-FP8 activations
it carries the scale computed before dispatch, which is forwarded to
may_quant_input so Humming skips the redundant w13 quantization. The
process_input so Humming skips the redundant w13 quantization. The
output is written into workspace13 via the buffer management.
"""
assert not apply_router_weight_on_input
@@ -1111,11 +1044,12 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):
)
buffers["down_output"] = output
inputs, input_scale = self.quantize_input(
inputs, input_scale, input_scale_2 = self.process_input(
"w13",
inputs=hidden_states,
input_scale=a1q_scale,
quanted_input=buffers.get("quanted_gate_up_input", None),
expert_tokens=expert_num_tokens,
)
self.humming_forward(
@@ -1123,6 +1057,7 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):
inputs=inputs,
weight=w1,
input_scale=input_scale,
input_scale_2=input_scale_2,
outputs=buffers["gate_up_output"],
valid_shape_m=valid_shape_m,
expert_layout=expert_num_tokens,
@@ -1130,16 +1065,12 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):
tuning_config=self.w13_tuning_config_str,
)
self.apply_activation(
activation=activation,
input=buffers["gate_up_output"],
output=buffers["activation_output"],
)
inputs, input_scale = self.quantize_input(
inputs, input_scale, input_scale_2 = self.process_input(
"w2",
inputs=buffers["activation_output"],
quanted_input=buffers.get("quanted_down_input", None),
inputs=buffers["gate_up_output"],
quanted_input=buffers["quanted_down_input"],
expert_tokens=expert_num_tokens,
activation=activation,
)
self.humming_forward(
@@ -1147,6 +1078,7 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):
inputs=inputs,
weight=w2,
input_scale=input_scale,
input_scale_2=input_scale_2,
outputs=output.view(-1, hidden_states.size(-1)),
valid_shape_m=valid_shape_m,
expert_layout=expert_num_tokens,
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Literal, overload
import torch
from vllm import _custom_ops as ops
@@ -8,6 +10,48 @@ from vllm.triton_utils import triton
from vllm.utils.math_utils import round_up
@overload
def moe_align_block_size(
topk_ids: torch.Tensor,
block_size: int,
num_experts: int,
expert_map: torch.Tensor | None = None,
pad_sorted_ids: bool = False,
ignore_invalid_experts: bool = False,
*,
return_scatter_idx: Literal[False] = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: ...
@overload
def moe_align_block_size(
topk_ids: torch.Tensor,
block_size: int,
num_experts: int,
expert_map: torch.Tensor | None = None,
pad_sorted_ids: bool = False,
ignore_invalid_experts: bool = False,
*,
return_scatter_idx: Literal[True],
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: ...
@overload
def moe_align_block_size(
topk_ids: torch.Tensor,
block_size: int,
num_experts: int,
expert_map: torch.Tensor | None = None,
pad_sorted_ids: bool = False,
ignore_invalid_experts: bool = False,
*,
return_scatter_idx: bool,
) -> (
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]
): ...
def moe_align_block_size(
topk_ids: torch.Tensor,
block_size: int,
@@ -15,7 +59,11 @@ def moe_align_block_size(
expert_map: torch.Tensor | None = None,
pad_sorted_ids: bool = False,
ignore_invalid_experts: bool = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
return_scatter_idx: bool = False,
) -> (
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]
):
"""Aligns the token distribution across experts to be compatible with block
size for matrix multiplication.
@@ -42,6 +90,8 @@ def moe_align_block_size(
as -1. When True, all invalid expert_ids in topk_ids will be ignored
and will not participate in counting or ranking, and there will be no
-1 in expert_ids.
return_scatter_idx: Whether to additionally return a masked identity
mapping for the original routed rows.
Returns:
- sorted_token_ids: A tensor containing the sorted token indices according
@@ -49,6 +99,10 @@ def moe_align_block_size(
- expert_ids: A tensor indicating the assigned expert index for each block.
- num_tokens_post_padded: The total number of tokens after padding,
ensuring divisibility by block_size.
- scatter_idx: Only returned when return_scatter_idx=True. An int32 tensor
shaped [topk_ids.numel(), 1], containing the original routed row index
for valid routes and -1 otherwise. With expert_map, this requires
ignore_invalid_experts=True.
This function pads the number of tokens that each expert needs to process
so that it is divisible by block_size.
@@ -71,6 +125,15 @@ def moe_align_block_size(
by block_size for proper block matrix operations.
"""
if return_scatter_idx and expert_map is not None and not ignore_invalid_experts:
raise ValueError("scatter_idx with expert_map requires ignore_invalid_experts")
scatter_idx = None
if return_scatter_idx:
scatter_idx = torch.empty(
(topk_ids.numel(), 1),
dtype=torch.int32,
device=topk_ids.device,
)
max_num_tokens_padded = topk_ids.numel() + num_experts * (block_size - 1)
if pad_sorted_ids:
max_num_tokens_padded = round_up(max_num_tokens_padded, block_size)
@@ -95,11 +158,14 @@ def moe_align_block_size(
expert_ids,
num_tokens_post_pad,
expert_map if ignore_invalid_experts else None,
scatter_idx,
)
if expert_map is not None and not ignore_invalid_experts:
expert_ids = expert_map[expert_ids]
if scatter_idx is not None:
return sorted_ids, expert_ids, num_tokens_post_pad, scatter_idx
return sorted_ids, expert_ids, num_tokens_post_pad
@@ -107,6 +107,37 @@ class MoEPermuteScratch:
return topk_ids_int32
def moe_prepare_scatter(
topk_ids: torch.Tensor,
expert_map: torch.Tensor | None,
scratch: MoEPermuteScratch,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Generate expert offsets and shared scatter/unpermute destination indices."""
assert topk_ids.device == scratch.device
assert topk_ids.size(0) <= scratch.max_num_tokens
assert topk_ids.size(1) == scratch.topk
n_token, topk = topk_ids.shape
expanded_rows = topk_ids.numel()
inverse = scratch.inv_permuted_idx[:expanded_rows].view(n_token, topk)
if expanded_rows == 0:
scratch.expert_first_token_offset.zero_()
return scratch.expert_first_token_offset, inverse
torch.ops._moe_C.moe_prepare_scatter(
scratch.prepare_topk_ids(topk_ids),
scratch.token_expert_indices_view(n_token),
expert_map,
scratch.num_experts,
scratch.num_local_experts,
scratch.expert_first_token_offset,
inverse,
scratch.sort_workspace,
scratch.permuted_experts_id[:expanded_rows].view(n_token, topk),
scratch.sorted_row_idx[:expanded_rows].view(n_token, topk),
scratch.topk_ids_for_sort[:expanded_rows].view(n_token, topk),
)
return scratch.expert_first_token_offset, inverse
def get_moe_permute_scratch(
*,
max_num_tokens: int,
@@ -603,7 +603,7 @@ def convert_to_fp8_moe_kernel_format(
w13.is_shuffled = True
w2.is_shuffled = True
elif fp8_backend == Fp8MoeBackend.HUMMING:
from vllm.model_executor.layers.quantization.utils.humming_utils import (
from vllm.model_executor.layers.quantization.utils.humming import (
convert_to_humming_moe_kernel_format,
)
@@ -739,7 +739,7 @@ def make_fp8_moe_quant_config(
)
elif fp8_backend == Fp8MoeBackend.HUMMING:
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.quantization.utils.humming_utils import (
from vllm.model_executor.layers.quantization.utils.humming import (
get_humming_moe_quant_config,
)
@@ -193,7 +193,7 @@ def make_int8_moe_quant_config(
if int8_backend == Int8MoeBackend.HUMMING:
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.quantization.utils.humming_utils import (
from vllm.model_executor.layers.quantization.utils.humming import (
get_humming_moe_quant_config,
)
@@ -254,7 +254,7 @@ def convert_to_int8_moe_kernel_format(
) -> tuple[torch.Tensor, torch.Tensor]:
"""Convert INT8 MoE weights to backend-specific kernel format."""
if int8_backend == Int8MoeBackend.HUMMING:
from vllm.model_executor.layers.quantization.utils.humming_utils import (
from vllm.model_executor.layers.quantization.utils.humming import (
convert_to_humming_moe_kernel_format,
)
@@ -1456,7 +1456,7 @@ def convert_to_wna16_moe_kernel_format(
"""
if backend == WNA16MoEBackend.HUMMING:
from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
from vllm.model_executor.layers.quantization.utils.humming_utils import (
from vllm.model_executor.layers.quantization.utils.humming import (
convert_to_humming_moe_kernel_format,
)
@@ -844,7 +844,7 @@ def convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
sf_block_size = 32 # mxfp4 block size
if mxfp4_backend == Mxfp4MoeBackend.HUMMING:
from vllm.model_executor.layers.quantization.utils.humming_utils import (
from vllm.model_executor.layers.quantization.utils.humming import (
convert_to_humming_moe_kernel_format,
)
@@ -1428,7 +1428,7 @@ def convert_weight_to_mxfp4_moe_kernel_format(
)
if mxfp4_backend == Mxfp4MoeBackend.HUMMING:
from vllm.model_executor.layers.quantization.utils.humming_utils import (
from vllm.model_executor.layers.quantization.utils.humming import (
convert_to_humming_moe_kernel_format,
)
@@ -1966,7 +1966,7 @@ def make_mxfp4_moe_quant_config(
gemm1_clamp_limit=swiglu_limit,
)
elif mxfp4_backend == Mxfp4MoeBackend.HUMMING:
from vllm.model_executor.layers.quantization.utils.humming_utils import (
from vllm.model_executor.layers.quantization.utils.humming import (
get_humming_moe_quant_config,
)
@@ -409,7 +409,7 @@ def convert_to_nvfp4_moe_kernel_format(
is_act_and_mul=is_act_and_mul,
)
elif nvfp4_backend == NvFp4MoeBackend.HUMMING:
from vllm.model_executor.layers.quantization.utils.humming_utils import (
from vllm.model_executor.layers.quantization.utils.humming import (
convert_to_humming_moe_kernel_format,
)
@@ -523,7 +523,7 @@ def make_nvfp4_moe_quant_config(
use_a16 = _use_a16(backend, use_a16)
if backend == NvFp4MoeBackend.HUMMING:
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.quantization.utils.humming_utils import (
from vllm.model_executor.layers.quantization.utils.humming import (
get_humming_moe_quant_config,
)
@@ -713,7 +713,7 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
if self.wna16_moe_backend == WNA16MoEBackend.HUMMING:
from vllm.model_executor.layers.quantization.utils.humming_utils import (
from vllm.model_executor.layers.quantization.utils.humming import (
get_humming_moe_quant_config,
)
@@ -689,7 +689,7 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
if self.wna16_moe_backend == WNA16MoEBackend.HUMMING:
from vllm.model_executor.layers.quantization.utils.humming_utils import (
from vllm.model_executor.layers.quantization.utils.humming import (
get_humming_moe_quant_config,
)
@@ -474,7 +474,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
self, layer: torch.nn.Module
) -> FusedMoEQuantConfig | None:
if self.wna16_backend == WNA16MoEBackend.HUMMING:
from vllm.model_executor.layers.quantization.utils.humming_utils import (
from vllm.model_executor.layers.quantization.utils.humming import (
get_humming_moe_quant_config,
)
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from math import gcd
from typing import TYPE_CHECKING, Any
import regex as re
@@ -28,12 +29,15 @@ from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
QuantizeMethodBase,
)
from vllm.model_executor.layers.quantization.utils.humming_utils import (
from vllm.model_executor.layers.quantization.utils.humming import (
check_and_fallback_input_schema,
convert_to_humming_moe_kernel_format,
get_humming_linear_compute_config,
get_humming_moe_quant_config,
humming_update_schema_hadamard_block_size,
input_schema_to_quant_key,
make_humming_moe_kernel,
resolve_humming_layer_config,
select_humming_moe_experts,
weight_schema_to_quant_key,
)
@@ -227,23 +231,13 @@ class HummingConfig(QuantizationConfig):
return None
config = group_config
layer_config = config
layer_dynamic = config.get("dynamic", {})
if not isinstance(layer_dynamic, dict):
layer_dynamic = {}
for regex, override_config in layer_dynamic.items():
if regex[:1] != "+":
continue
if re.match(regex[2:], prefix):
layer_config = config.copy()
layer_config.update(override_config)
break
layer_config = resolve_humming_layer_config(config, prefix)
if "quant_method" in layer_config:
return _hm.BaseWeightSchema.from_config(layer_config)
return None
def get_layer_input_schema(self, config: dict[str, Any], prefix: str):
def get_layer_input_config(self, config: dict[str, Any], prefix: str):
if self.is_layer_skipped(config, prefix):
return None
if config["quant_method"] in ["compressed-tensors", "modelopt"]:
@@ -252,10 +246,15 @@ class HummingConfig(QuantizationConfig):
return None
config = group_config
config = resolve_humming_layer_config(config, prefix)
if config.get("quant_method", None) in _hm.BaseInputSchema.INPUT_SCHEMA_MAP:
return _hm.BaseInputSchema.from_config(config)
return config
return None
def get_layer_input_schema(self, config: dict[str, Any], prefix: str):
config = self.get_layer_input_config(config, prefix)
return None if config is None else _hm.BaseInputSchema.from_config(config)
def get_quant_config_for_layer(
self, prefix: str, layer_type: str
) -> "HummingLayerQuantizationConfig | None":
@@ -279,14 +278,25 @@ class HummingConfig(QuantizationConfig):
if weight_schema is not None:
input_schema = None
force_input_schema = None
allow_input_schema_fallback = True
if self.full_config:
input_schema = self.get_layer_input_schema(self.full_config, prefix)
input_config = self.get_layer_input_config(self.full_config, prefix)
if input_config is not None:
allow_input_schema_fallback = input_config.pop(
"allow_fallback", True
)
input_schema = _hm.BaseInputSchema.from_config(input_config)
if envs.VLLM_HUMMING_INPUT_QUANT_CONFIG:
quant_config = envs.VLLM_HUMMING_INPUT_QUANT_CONFIG.copy()
quant_config["quant_method"] = "humming"
force_input_schema = self.get_layer_input_schema(quant_config, prefix)
input_config = self.get_layer_input_config(quant_config, prefix)
if input_config is not None:
allow_input_schema_fallback = input_config.pop(
"allow_fallback", False
)
force_input_schema = _hm.BaseInputSchema.from_config(input_config)
if input_schema is None:
input_schema = force_input_schema
@@ -299,6 +309,7 @@ class HummingConfig(QuantizationConfig):
force_weight_schema=force_weight_schema,
force_input_schema=force_input_schema,
is_online_quant=is_online_quant,
allow_input_schema_fallback=allow_input_schema_fallback,
)
return None
@@ -318,6 +329,11 @@ class HummingConfig(QuantizationConfig):
elif isinstance(layer, LinearBase):
return UnquantizedLinearMethod()
elif isinstance(layer, LinearBase):
input_size = layer.input_size
partition_size = getattr(layer, "input_size_per_partition", input_size)
is_online_quant = quant_config.is_online_quant
if is_online_quant and (input_size % 32 != 0 or partition_size % 32 != 0):
return UnquantizedLinearMethod()
return HummingLinearMethod(quant_config)
elif isinstance(layer, RoutedExperts):
return HummingMoEMethod(quant_config, layer.moe_config)
@@ -332,6 +348,7 @@ class HummingLayerQuantizationConfig(HummingConfig):
force_weight_schema: "HummingWeightSchema | None" = None,
force_input_schema: "HummingInputSchema | None" = None,
is_online_quant: bool = False,
allow_input_schema_fallback: bool = True,
):
self.weight_schema = weight_schema
if input_schema is None:
@@ -340,6 +357,7 @@ class HummingLayerQuantizationConfig(HummingConfig):
self.force_weight_schema = force_weight_schema
self.force_input_schema = force_input_schema
self.is_online_quant = is_online_quant
self.allow_input_schema_fallback = allow_input_schema_fallback
@classmethod
def from_config(cls, config):
@@ -373,21 +391,13 @@ class HummingLinearMethod(LinearMethodBase):
if is_unquantized and self.is_online_quant:
# online quant (fp16/bf16 -> quant_type)
assert isinstance(self.weight_schema, _hm.HummingWeightSchema)
f16_dtype = _hm.DataType.from_torch_dtype(layer.param_dtype)
has_global_scale = "TENSOR" in str(self.weight_schema.weight_scale_type)
tensor_list = _hm.quantize_weight(
weight=loaded_weight,
dtype=self.weight_schema.b_dtype,
scale_dtype=self.weight_schema.bs_dtype or f16_dtype,
group_size=self.weight_schema.weight_scale_group_size,
has_zero_point=self.weight_schema.has_zero_point,
has_global_scale=has_global_scale,
is_fp_zero_point=self.weight_schema.is_fp_zero_point,
pack=True,
tensors = self.weight_schema.quant_tensor(
loaded_weight.to(param.device),
self.weight_schema,
layer.param_dtype,
)
key_list = ["weight", "weight_scale", "zero_point", "global_scale"]
for key, tensor in zip(key_list, tensor_list):
for key, tensor in tensors.items():
if tensor is None or tensor.nelement() == 0:
continue
param = getattr(layer, key)
@@ -459,6 +469,18 @@ class HummingLinearMethod(LinearMethodBase):
layer.output_partition_sizes = output_partition_sizes
layer.extra_weight_attrs = extra_weight_attrs.copy()
input_schema = self.force_input_schema or self.input_schema
input_schema = input_schema.to_humming_schema(params_dtype)
for name in ("weight_schema", "force_weight_schema"):
schema = getattr(self, name)
if getattr(schema, "hadamard_block_size", None) == -1:
schema = humming_update_schema_hadamard_block_size(
weight_schema=schema,
input_schema=input_schema,
shape_k=input_size_per_partition,
)
setattr(self, name, schema)
weight_loader = extra_weight_attrs.get("weight_loader", default_weight_loader)
new_weight_loader = self.prepare_weight_loader(layer, weight_loader)
extra_weight_attrs["weight_loader"] = new_weight_loader
@@ -502,7 +524,10 @@ class HummingLinearMethod(LinearMethodBase):
return None
# convert from checkpoint format to humming format
if not isinstance(self.weight_schema, _hm.HummingWeightSchema):
is_humming_weight = isinstance(self.weight_schema, _hm.HummingWeightSchema)
is_humming_input = isinstance(self.input_schema, _hm.HummingInputSchema)
if not is_humming_weight or not is_humming_input:
self.weight_schema, tensors = self.weight_schema.convert_humming(
tensors=layer.state_dict(),
shape_n_stacks=layer.output_partition_sizes,
@@ -510,12 +535,13 @@ class HummingLinearMethod(LinearMethodBase):
param_dtype=layer.param_dtype,
)
self.input_schema, _ = self.input_schema.convert_humming(
self.input_schema, input_tensors = self.input_schema.convert_humming(
tensors=layer.state_dict(),
shape_n_stacks=layer.output_partition_sizes,
shape_k_stacks=[layer.input_size_per_partition],
param_dtype=layer.param_dtype,
)
tensors.update(input_tensors)
for name, _ in list(layer.named_parameters()):
delattr(layer, name)
@@ -530,11 +556,15 @@ class HummingLinearMethod(LinearMethodBase):
assert isinstance(self.weight_schema, _hm.HummingWeightSchema)
force_requant = self.force_weight_schema is not None
if force_requant and self.weight_schema != self.force_weight_schema:
source_tensors = layer.state_dict()
tensors = self.weight_schema.requant_tensors(
tensors=layer.state_dict(),
tensors=source_tensors,
target_weight_schema=self.force_weight_schema,
param_dtype=layer.param_dtype,
)
name = self.input_schema.static_tensor_scale_name
if name is not None:
tensors[name] = source_tensors[name]
self.weight_schema = self.force_weight_schema
@@ -548,6 +578,12 @@ class HummingLinearMethod(LinearMethodBase):
del tensors
self.input_schema = check_and_fallback_input_schema(
weight_schema=self.weight_schema,
input_schema=self.input_schema,
param_dtype=layer.param_dtype,
allow_fallback=self.quant_config.allow_input_schema_fallback,
)
self.layer_config = _hm.prepare_layer_config(
shape_n=layer.output_partition_sizes_sum,
shape_k=layer.input_size_per_partition,
@@ -557,10 +593,16 @@ class HummingLinearMethod(LinearMethodBase):
pad_k_to_multiple=128,
has_bias=layer.has_bias,
torch_dtype=layer.param_dtype,
device=layer.weight.device,
)
tensors = _hm.transform_humming_tensors(
self.layer_config, dict(layer.named_parameters())
)
source_tensors = dict(layer.named_parameters())
tensors = _hm.transform_humming_tensors(self.layer_config, source_tensors)
name = self.input_schema.static_tensor_scale_name
if name is not None:
tensors[name] = source_tensors[name]
if "bias" in tensors:
zero_bias = torch.zeros_like(tensors["bias"])
layer.register_buffer("zero_bias", zero_bias)
for name, _ in list(layer.named_parameters()):
delattr(layer, name)
for name, tensor in tensors.items():
@@ -583,8 +625,11 @@ class HummingLinearMethod(LinearMethodBase):
weight=layer.weight,
weight_scale=getattr(layer, "weight_scale", None),
zero_point=getattr(layer, "zero_point", None),
bias=getattr(layer, "bias", None),
bias=getattr(layer, "bias" if bias is not None else "zero_bias", None),
weight_scale_2=getattr(layer, "weight_scale_2", None),
input_scale=getattr(layer, "input_scale", None),
input_scale_2=getattr(layer, "input_scale_2", None),
hadamard_block_size=self.weight_schema.hadamard_block_size,
locks=self.locks,
compute_config=self.compute_config,
)
@@ -606,11 +651,15 @@ class HummingMoEMethod(FusedMoEMethodBase):
# Derive QuantKeys from humming schemas.
# Prefer force schemas (the final format after requant) over base.
weight_key = weight_schema_to_quant_key(
self.force_weight_schema or self.weight_schema
self.force_weight_schema or self.weight_schema, moe.in_dtype
)
activation_key = input_schema_to_quant_key(
self.force_input_schema or self.input_schema
runtime_input_schema = check_and_fallback_input_schema(
weight_schema=self.force_weight_schema or self.weight_schema,
input_schema=self.force_input_schema or self.input_schema,
param_dtype=moe.in_dtype,
allow_fallback=quant_config.allow_input_schema_fallback,
)
activation_key = input_schema_to_quant_key(runtime_input_schema, moe.in_dtype)
# Select Humming MoE experts
self.experts_cls = select_humming_moe_experts(
@@ -634,22 +683,14 @@ class HummingMoEMethod(FusedMoEMethodBase):
# online quant (fp16/bf16 -> quant_type)
if is_unquantized:
assert isinstance(self.weight_schema, _hm.HummingWeightSchema)
f16_dtype = _hm.DataType.from_torch_dtype(layer.param_dtype)
has_global_scale = "TENSOR" in str(self.weight_schema.weight_scale_type)
tensor_list = _hm.quantize_weight(
weight=loaded_weight,
dtype=self.weight_schema.b_dtype,
scale_dtype=self.weight_schema.bs_dtype or f16_dtype,
group_size=self.weight_schema.weight_scale_group_size,
has_zero_point=self.weight_schema.has_zero_point,
has_global_scale=has_global_scale,
is_fp_zero_point=self.weight_schema.is_fp_zero_point,
pack=True,
tensors = self.weight_schema.quant_tensor(
loaded_weight.to(param.device),
self.weight_schema,
layer.param_dtype,
)
key_list = ["weight", "weight_scale", "zero_point", "global_scale"]
success = True
for key, tensor in zip(key_list, tensor_list):
for key, tensor in tensors.items():
if tensor is None or tensor.nelement() == 0:
continue
sublayer_name = "w2" if shard_id == "w2" else "w13"
@@ -695,6 +736,18 @@ class HummingMoEMethod(FusedMoEMethodBase):
layer.num_experts = num_experts
layer.param_dtype = params_dtype
layer.intermediate_size = intermediate_size_per_partition
input_schema = self.force_input_schema or self.input_schema
input_schema = input_schema.to_humming_schema(params_dtype)
shape_k = gcd(hidden_size, intermediate_size_per_partition)
for name in ("weight_schema", "force_weight_schema"):
schema = getattr(self, name)
if getattr(schema, "hadamard_block_size", None) == -1:
schema = humming_update_schema_hadamard_block_size(
weight_schema=schema,
input_schema=input_schema,
shape_k=shape_k,
)
setattr(self, name, schema)
weight_loader = extra_weight_attrs.get("weight_loader", default_weight_loader)
weight_loader = self.prepare_weight_loader(layer, weight_loader)
extra_weight_attrs["weight_loader"] = weight_loader
@@ -761,6 +814,7 @@ class HummingMoEMethod(FusedMoEMethodBase):
weight_schema=self.weight_schema,
input_schema=self.input_schema,
force_weight_schema=self.force_weight_schema,
allow_input_schema_fallback=self.quant_config.allow_input_schema_fallback,
)
# Build the MoE kernel
@@ -389,7 +389,7 @@ class MoeWNA16Method(FusedMoEMethodBase):
self, layer: RoutedExperts
) -> FusedMoEQuantConfig | None:
if self.wna16_backend == WNA16MoEBackend.HUMMING:
from vllm.model_executor.layers.quantization.utils.humming_utils import (
from vllm.model_executor.layers.quantization.utils.humming import (
get_humming_moe_quant_config,
)
@@ -0,0 +1,51 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Humming quantization integration."""
from vllm.model_executor.layers.quantization.utils.humming.activation import (
get_humming_activation,
)
from vllm.model_executor.layers.quantization.utils.humming.linear import (
apply_humming_linear,
convert_linear_layer_to_humming_standard,
get_humming_linear_compute_config,
prepare_humming_linear_layer_config,
)
from vllm.model_executor.layers.quantization.utils.humming.moe import (
HummingMoEQuantConfig,
convert_to_humming_moe_kernel_format,
get_humming_moe_quant_config,
make_humming_moe_kernel,
make_humming_moe_quant_config,
select_humming_moe_experts,
)
from vllm.model_executor.layers.quantization.utils.humming.schema import (
check_and_fallback_input_schema,
humming_is_layer_skipped,
humming_update_schema_hadamard_block_size,
input_schema_to_quant_key,
quant_key_to_input_schema,
resolve_humming_layer_config,
weight_schema_to_quant_key,
)
__all__ = [
"check_and_fallback_input_schema",
"get_humming_activation",
"HummingMoEQuantConfig",
"get_humming_moe_quant_config",
"humming_is_layer_skipped",
"humming_update_schema_hadamard_block_size",
"make_humming_moe_quant_config",
"apply_humming_linear",
"convert_linear_layer_to_humming_standard",
"get_humming_linear_compute_config",
"prepare_humming_linear_layer_config",
"make_humming_moe_kernel",
"select_humming_moe_experts",
"input_schema_to_quant_key",
"quant_key_to_input_schema",
"resolve_humming_layer_config",
"weight_schema_to_quant_key",
"convert_to_humming_moe_kernel_format",
]
@@ -0,0 +1,126 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MoE activation expressions for Humming input processing."""
from dataclasses import replace
from vllm.model_executor.layers.fused_moe.activation import (
ApplyMoEActivationConfig,
MoEActivation,
)
_SILU_IMPL = "a / (1.f + expf(-a))"
_GELU_IMPL = "0.5f * a * (1.f + erff(a * 0.7071067811865476f))"
_RELU2_IMPL = "fmaxf(a, 0.f) * fmaxf(a, 0.f)"
_GELU_TANH_IMPL = """[](float a) {
const float cubic = 0.044715f * a * a * a;
const float inner = 0.7978845608028654f * (a + cubic);
return 0.5f * a * (1.f + tanhf(inner));
}(a)"""
_CLAMPED_SILU_IMPL = """[](float a, float b) {{
const float gate = fminf(a, {limit});
const float up = fmaxf(fminf(b, {limit}), -{limit});
const float activated_gate = gate / (1.f + expf(-{alpha} * gate));
return activated_gate * (up + {beta});
}}(a, b)"""
# Parameterized expressions are resolved by get_humming_activation().
_HUMMING_ACTIVATIONS: dict[MoEActivation, dict[str, str]] = {
MoEActivation.SILU: {
"activation_type": "binary_split",
"activation_impl": f"({_SILU_IMPL}) * b",
},
MoEActivation.GELU: {
"activation_type": "binary_split",
"activation_impl": f"({_GELU_IMPL}) * b",
},
MoEActivation.GELU_TANH: {
"activation_type": "binary_split",
"activation_impl": f"({_GELU_TANH_IMPL}) * b",
},
MoEActivation.RELU2: {
"activation_type": "binary_split",
"activation_impl": f"({_RELU2_IMPL}) * b",
},
MoEActivation.SWIGLUOAI: {
"activation_type": "binary_interleaved",
"activation_impl": _CLAMPED_SILU_IMPL,
},
MoEActivation.SITU: {
"activation_type": "binary_split",
"activation_impl": """[](float a, float b) {{
const float gate = {situ_beta} * tanhf(a / {situ_beta});
const float activated_gate = gate / (1.f + expf(-a));
const float up = {situ_up};
return activated_gate * up;
}}(a, b)""",
},
MoEActivation.SWIGLUOAI_UNINTERLEAVE: {
"activation_type": "binary_split",
"activation_impl": _CLAMPED_SILU_IMPL,
},
MoEActivation.SWIGLUSTEP: {
"activation_type": "binary_split",
"activation_impl": """[](float a, float b) {
const float activated_gate = a / (1.f + expf(-a));
const float gate = fminf(activated_gate, 7.f);
const float up = fmaxf(fminf(b, 7.f), -7.f);
return gate * up;
}(a, b)""",
},
MoEActivation.SILU_NO_MUL: {
"activation_type": "unary",
"activation_impl": _SILU_IMPL,
},
MoEActivation.GELU_NO_MUL: {
"activation_type": "unary",
"activation_impl": _GELU_IMPL,
},
MoEActivation.GELU_TANH_NO_MUL: {
"activation_type": "unary",
"activation_impl": _GELU_TANH_IMPL,
},
MoEActivation.RELU2_NO_MUL: {
"activation_type": "unary",
"activation_impl": _RELU2_IMPL,
},
}
def get_humming_activation(
activation: MoEActivation,
config: ApplyMoEActivationConfig | None = None,
) -> dict[str, str]:
config = config or ApplyMoEActivationConfig()
use_clamped_silu = False
if activation == MoEActivation.SWIGLUOAI:
config = ApplyMoEActivationConfig(clamp_limit=7.0, alpha=1.702, beta=1.0)
use_clamped_silu = True
elif activation == MoEActivation.SILU and config.clamp_limit is not None:
config = replace(config, alpha=1.0, beta=0.0)
use_clamped_silu = True
elif activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
if config.clamp_limit is None:
raise ValueError("SWIGLUOAI_UNINTERLEAVE requires clamp_limit")
use_clamped_silu = True
definition = _HUMMING_ACTIVATIONS[activation].copy()
if activation == MoEActivation.SITU:
if config.activation_situ_beta is None:
raise ValueError("SITU requires activation_situ_beta")
linear_beta = config.activation_situ_linear_beta
situ_up = "b"
if linear_beta is not None and linear_beta > 0:
situ_up = f"({float(linear_beta)}f) * tanhf(b / ({float(linear_beta)}f))"
situ_beta = f"({float(config.activation_situ_beta)}f)"
activation_impl = definition["activation_impl"]
activation_impl = activation_impl.format(situ_beta=situ_beta, situ_up=situ_up)
definition["activation_impl"] = activation_impl
elif use_clamped_silu:
assert config.clamp_limit is not None
limit = f"({float(config.clamp_limit)}f)"
alpha = f"({float(config.alpha)}f)"
beta = f"({float(config.beta)}f)"
activation_impl = _CLAMPED_SILU_IMPL.format(limit=limit, alpha=alpha, beta=beta)
definition["activation_impl"] = activation_impl
return definition
@@ -0,0 +1,183 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Prepare and execute Humming linear layers."""
import json
from typing import TYPE_CHECKING
import torch
from vllm import envs
from vllm.model_executor.layers.linear import LinearBase
from vllm.model_executor.layers.quantization.utils.humming.schema import (
check_and_fallback_input_schema,
)
if TYPE_CHECKING:
from vllm.utils.humming import BaseInputSchema, LayerConfig
def convert_linear_layer_to_humming_standard(
layer: LinearBase, name_map: dict[str, str]
):
"""Rename/reshape a linear layer's quantized params (the canonical MPLinear
layout: ``weight_packed`` int32 + ``weight_scale``) into the parameter names
and layout humming's weight schema expects (``weight`` / ``weight_scale``)."""
for name, checkpoint_name in name_map.items():
tensor = getattr(layer, checkpoint_name)
delattr(layer, checkpoint_name)
if name == "weight":
input_dim = getattr(tensor, "input_dim", 1)
output_dim = getattr(tensor, "output_dim", 0)
if input_dim == 0 and output_dim == 1:
tensor = tensor.transpose(1, 0).contiguous()
else:
assert output_dim == 0 and input_dim == 1
tensor = tensor.view(tensor.size(0), -1).view(torch.int32)
elif name in ["weight_scale", "zero_point"]:
if getattr(tensor, "output_dim", 0) == 1:
tensor = tensor.transpose(0, 1).contiguous()
if tensor.ndim == 1:
tensor = tensor.unsqueeze(1)
tensor = tensor.view(torch.int32) if name == "zero_point" else tensor
if isinstance(tensor, torch.nn.Parameter):
param = tensor
else:
param = torch.nn.Parameter(tensor, requires_grad=False)
setattr(layer, name, param)
def prepare_humming_linear_layer_config(
layer: LinearBase,
quant_config: dict,
input_quant_config: dict | None = None,
*,
input_schema: "BaseInputSchema | None" = None,
) -> "LayerConfig":
from vllm.utils.humming import (
BaseInputSchema,
BaseWeightSchema,
HummingInputSchema,
prepare_layer_config,
transform_humming_tensors,
)
weight_schema = BaseWeightSchema.from_config(quant_config)
if input_schema is None:
if input_quant_config is not None:
input_schema = BaseInputSchema.from_config(input_quant_config)
else:
input_schema = HummingInputSchema()
# ReplicatedLinear has no TP partitioning and so does not set
# input_size_per_partition; for it that is just input_size. Use hasattr
# rather than getattr's default arg, which is evaluated eagerly and would
# raise on layers lacking input_size (e.g. ParallelLMHead).
if hasattr(layer, "input_size_per_partition"):
input_size_per_partition = layer.input_size_per_partition
else:
input_size_per_partition = layer.input_size
shape_k_stacks = [input_size_per_partition]
shape_n_stacks = layer.output_partition_sizes
# Step 1: convert weight and input schemas to humming standard format
source_tensors = dict(layer.named_parameters())
weight_schema, tensors = weight_schema.convert_humming(
tensors=source_tensors,
shape_n_stacks=shape_n_stacks,
shape_k_stacks=shape_k_stacks,
param_dtype=layer.params_dtype,
)
input_schema, input_tensors = input_schema.convert_humming(
tensors=source_tensors,
shape_n_stacks=shape_n_stacks,
shape_k_stacks=shape_k_stacks,
param_dtype=layer.params_dtype,
)
tensors.update(input_tensors)
layer.weight_schema = weight_schema
input_schema = check_and_fallback_input_schema(
weight_schema=weight_schema,
input_schema=input_schema,
param_dtype=layer.params_dtype,
)
for name in ("input_scale", "input_scale_2"):
if name != input_schema.static_tensor_scale_name:
tensors.pop(name, None)
input_tensors.pop(name, None)
# Step 2: transform weight (humming standard format) for forwarding.
config = prepare_layer_config(
shape_n=sum(layer.output_partition_sizes),
shape_k=input_size_per_partition,
weight_schema=weight_schema,
input_schema=input_schema,
pad_n_to_multiple=256,
pad_k_to_multiple=128,
has_bias=layer.has_bias,
torch_dtype=layer.params_dtype,
device=tensors["weight"].device,
)
tensors = transform_humming_tensors(config, tensors)
tensors.update(input_tensors)
if "bias" in tensors:
zero_bias = torch.zeros_like(tensors["bias"])
layer.register_buffer("zero_bias", zero_bias)
for name, _ in list(layer.named_parameters()):
delattr(layer, name)
for name, tensor in tensors.items():
param = torch.nn.Parameter(tensor, requires_grad=False)
setattr(layer, name, param)
return config
def get_humming_linear_compute_config() -> str:
return json.dumps(
{
"use_batch_invariant": envs.VLLM_BATCH_INVARIANT,
"use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,
"gemm_type": "dense",
}
)
def apply_humming_linear(
layer: LinearBase,
x: torch.Tensor,
*,
skip_bias_add: bool = False,
layer_config: "LayerConfig",
compute_config: str,
locks: torch.Tensor,
) -> torch.Tensor:
from vllm.utils.humming import humming_forward
mode = layer_config.input_quant_mode
input_scale = getattr(layer, "input_scale", None) if mode.should_quantize else None
input_scale_2 = (
getattr(layer, "input_scale_2", None) if mode.has_secondary_scale else None
)
flatten_inputs = x.reshape(-1, x.size(-1))
output = humming_forward(
layer_config,
inputs=flatten_inputs,
weight=layer.weight,
weight_scale=getattr(layer, "weight_scale", None),
zero_point=getattr(layer, "zero_point", None),
bias=getattr(layer, "zero_bias" if skip_bias_add else "bias", None),
weight_scale_2=getattr(layer, "weight_scale_2", None),
input_scale=input_scale,
input_scale_2=input_scale_2,
hadamard_block_size=layer.weight_schema.hadamard_block_size,
locks=locks,
compute_config=compute_config,
)
return output.view(*x.shape[:-1], output.size(-1))
@@ -1,13 +1,14 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
"""Configure, prepare weights for, and assemble Humming MoE kernels."""
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
import regex as re
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
import vllm.model_executor.layers.quantization.utils.humming.schema as humming_schema
from vllm import envs
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.all2all_utils import (
@@ -23,34 +24,19 @@ from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
HummingGroupedExperts,
HummingIndexedExperts,
)
from vllm.model_executor.layers.linear import LinearBase
from vllm.model_executor.layers.quantization.utils.humming.schema import (
humming_is_layer_skipped,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
FP4_DTYPE,
FP8_DTYPE,
INT4_DTYPE,
INT8_DTYPE,
MXFP_SCALE_DTYPE,
GroupShape,
QuantKey,
ScaleDesc,
)
from vllm.utils.import_utils import has_humming
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
from vllm.utils.humming import (
AWQWeightSchema,
BaseInputSchema,
BaseWeightSchema,
CompressedTensorsInputSchema,
CompressedTensorsWeightSchema,
Fp8WeightSchema,
GPTQWeightSchema,
HummingInputSchema,
HummingWeightSchema,
LayerConfig,
)
from vllm.utils.humming import dtypes as humming_dtypes
from vllm.utils.humming import LayerConfig
logger = init_logger(__name__)
@@ -59,504 +45,12 @@ logger = init_logger(__name__)
class HummingMoEQuantConfig(FusedMoEQuantConfig):
w1_humming_config: "LayerConfig"
w2_humming_config: "LayerConfig"
if has_humming():
from vllm.utils.humming import dtypes as humming_dtypes
_HUMMING_TO_QUANT_DTYPE: dict[humming_dtypes.DataType, Any] = {
humming_dtypes.float4e2m1: FP4_DTYPE,
humming_dtypes.float8e4m3: FP8_DTYPE,
humming_dtypes.float8e5m2: torch.float8_e5m2,
humming_dtypes.int8: torch.int8,
humming_dtypes.uint4: INT4_DTYPE,
humming_dtypes.uint8: INT8_DTYPE,
humming_dtypes.uint2: torch.uint8,
humming_dtypes.uint3: torch.uint8,
humming_dtypes.uint5: torch.uint8,
humming_dtypes.uint6: torch.uint8,
humming_dtypes.uint7: torch.uint8,
}
_HUMMING_TO_SCALE_DTYPE: dict[humming_dtypes.DataType, torch.dtype] = {
humming_dtypes.float8e8m0: MXFP_SCALE_DTYPE,
humming_dtypes.float8e4m3: FP8_DTYPE,
humming_dtypes.float16: torch.float16,
humming_dtypes.bfloat16: torch.bfloat16,
humming_dtypes.float32: torch.float32,
}
def _group_shape(group_size: int, group_size_n: int = 0) -> GroupShape:
"""Map humming group sizes to QuantKey GroupShape.
group_size: elements per group along K (col); 0 means full dimension.
group_size_n: elements per group along N (row); 0 means 1 (per-row).
GroupShape convention: row = N dim, col = K dim.
"""
if group_size == 0 and group_size_n == 0:
return GroupShape.PER_CHANNEL
row = group_size_n if group_size_n > 0 else 1
col = group_size if group_size > 0 else -1
return GroupShape(row=row, col=col)
# ---- HummingWeightSchema (post-conversion) --------------------------------
def _humming_weight_schema_to_quant_key(
schema: "HummingWeightSchema",
) -> QuantKey:
from vllm.utils.humming import WeightScaleType
"""Convert a HummingWeightSchema to a QuantKey."""
dtype = _HUMMING_TO_QUANT_DTYPE[schema.b_dtype]
if schema.bs_dtype is not None:
scale_dtype = _HUMMING_TO_SCALE_DTYPE[schema.bs_dtype]
else:
scale_dtype = torch.float32
group_shape = _group_shape(
schema.weight_scale_group_size,
schema.weight_scale_group_size_n,
)
scale = ScaleDesc(dtype=scale_dtype, static=True, group_shape=group_shape)
scale2 = None
if schema.weight_scale_type == WeightScaleType.GROUP_TENSOR:
scale2 = ScaleDesc(
dtype=torch.float32,
static=True,
group_shape=GroupShape.PER_TENSOR,
)
return QuantKey(
dtype=dtype,
scale=scale,
scale2=scale2,
symmetric=not schema.has_zero_point,
)
# ---- Checkpoint-format weight schemas (pre-conversion) --------------------
def _fp8_weight_schema_to_quant_key(schema: "Fp8WeightSchema") -> QuantKey:
if schema.weight_block_size is not None:
gs_n, gs_k = schema.weight_block_size
group_shape = GroupShape(row=gs_n, col=gs_k)
else:
group_shape = GroupShape.PER_CHANNEL
scale = ScaleDesc(dtype=torch.float32, static=True, group_shape=group_shape)
return QuantKey(dtype=FP8_DTYPE, scale=scale, symmetric=True)
def _awq_weight_schema_to_quant_key(schema: "AWQWeightSchema") -> QuantKey:
group_shape = _group_shape(schema.group_size)
scale = ScaleDesc(
dtype=torch.float16,
static=True,
group_shape=group_shape,
)
return QuantKey(
dtype=INT4_DTYPE,
scale=scale,
symmetric=not schema.zero_point,
)
def _gptq_weight_schema_to_quant_key(schema: "GPTQWeightSchema") -> QuantKey:
group_shape = _group_shape(schema.group_size)
scale = ScaleDesc(
dtype=torch.float16,
static=True,
group_shape=group_shape,
)
return QuantKey(dtype=INT4_DTYPE, scale=scale, symmetric=schema.sym)
def _compressed_tensors_weight_schema_to_quant_key(
schema: "CompressedTensorsWeightSchema",
) -> QuantKey:
# Determine dtype from format/type/num_bits
fmt = schema.format
if fmt in ("int-quantized", "float-quantized", "naive-quantized"):
dtype = INT8_DTYPE if schema.type == "int" else FP8_DTYPE
elif "nvfp4" in fmt or "mxfp4" in fmt:
dtype = FP4_DTYPE
else:
dtype = _HUMMING_TO_QUANT_DTYPE[
humming_dtypes.DataType.from_str(f"uint{schema.num_bits}")
]
# Determine group shape from strategy
if schema.strategy in ("group", "tensor_group"):
group_shape = _group_shape(schema.group_size or 0)
elif schema.strategy == "block" and schema.block_structure is not None:
group_shape = GroupShape(
row=schema.block_structure[0],
col=schema.block_structure[1],
)
else:
group_shape = GroupShape.PER_CHANNEL
# Determine scale dtype
if "mxfp" in fmt:
scale_dtype = MXFP_SCALE_DTYPE
elif "nvfp4" in fmt:
scale_dtype = FP8_DTYPE
else:
scale_dtype = torch.float32
scale = ScaleDesc(dtype=scale_dtype, static=True, group_shape=group_shape)
scale2 = None
if "nvfp4" in fmt or schema.strategy == "tensor_group":
scale2 = ScaleDesc(
dtype=torch.float32,
static=True,
group_shape=GroupShape.PER_TENSOR,
)
return QuantKey(
dtype=dtype,
scale=scale,
scale2=scale2,
symmetric=schema.symmetric,
)
# ---- Dispatch for any BaseWeightSchema ------------------------------------
def weight_schema_to_quant_key(
schema: "BaseWeightSchema",
) -> QuantKey:
from vllm.utils.humming import (
AWQWeightSchema,
BitnetWeightSchema,
CompressedTensorsWeightSchema,
Fp8WeightSchema,
GptOssMxfp4WeightSchema,
GPTQWeightSchema,
HummingWeightSchema,
ModeloptMxfp8WeightSchema,
ModeloptNvfp4WeightSchema,
Mxfp4WeightSchema,
)
"""Convert any BaseWeightSchema to a QuantKey."""
if isinstance(schema, HummingWeightSchema):
return _humming_weight_schema_to_quant_key(schema)
# Schemas with fixed QuantKeys
if isinstance(schema, (Mxfp4WeightSchema, GptOssMxfp4WeightSchema)):
return QuantKey(
dtype=FP4_DTYPE,
scale=ScaleDesc(MXFP_SCALE_DTYPE, True, GroupShape(1, 32)),
)
if isinstance(schema, ModeloptMxfp8WeightSchema):
return QuantKey(
dtype=FP8_DTYPE,
scale=ScaleDesc(MXFP_SCALE_DTYPE, True, GroupShape(1, 32)),
)
if isinstance(schema, ModeloptNvfp4WeightSchema):
return QuantKey(
dtype=FP4_DTYPE,
scale=ScaleDesc(FP8_DTYPE, True, GroupShape(1, 16)),
scale2=ScaleDesc(torch.float32, True, GroupShape.PER_TENSOR),
)
if isinstance(schema, BitnetWeightSchema):
return QuantKey(
dtype=torch.uint8,
scale=ScaleDesc(torch.float32, True, GroupShape.PER_CHANNEL),
)
# Schemas requiring config inspection
if isinstance(schema, Fp8WeightSchema):
return _fp8_weight_schema_to_quant_key(schema)
if isinstance(schema, AWQWeightSchema):
return _awq_weight_schema_to_quant_key(schema)
if isinstance(schema, GPTQWeightSchema):
return _gptq_weight_schema_to_quant_key(schema)
if isinstance(schema, CompressedTensorsWeightSchema):
return _compressed_tensors_weight_schema_to_quant_key(schema)
raise TypeError(f"Unsupported weight schema type: {type(schema)}")
# ---- HummingInputSchema (post-conversion) ----------------------------------
def _humming_input_schema_to_quant_key(
schema: "HummingInputSchema",
) -> QuantKey | None:
"""Convert a HummingInputSchema to a QuantKey. Returns None if
the schema represents unquantized (bf16/fp16) inputs."""
if schema.a_dtype is None or schema.a_dtype.num_bits >= 16:
return None
dtype = _HUMMING_TO_QUANT_DTYPE[schema.a_dtype]
gs = schema.input_scale_group_size
group_shape = GroupShape(row=1, col=gs) if gs > 0 else GroupShape.PER_TOKEN
# Pick the scale dtype the Humming kernel actually consumes. An explicit
# input_scale_dtype always wins. Otherwise infer from the grouping: MX
# microscale activations (group size 32, e.g. MXFP8) carry an e8m0 (uint8)
# scale, while block-FP8 (group size 128) and per-token FP8/int8 carry a
# float32 scale. Getting this right lets a grouped FP8 activation match
# kFp8Dynamic128Sym instead of an unmatchable uint8-scaled key.
if schema.input_scale_dtype is not None:
scale_dtype = _HUMMING_TO_SCALE_DTYPE[schema.input_scale_dtype]
elif gs == 32:
scale_dtype = MXFP_SCALE_DTYPE
else:
scale_dtype = torch.float32
scale = ScaleDesc(dtype=scale_dtype, static=False, group_shape=group_shape)
return QuantKey(dtype=dtype, scale=scale, symmetric=True)
# ---- Checkpoint-format input schemas (pre-conversion) ----------------------
def _resolve_input_quant_key(
origin_a_dtype: "humming_dtypes.DataType",
group_size: int,
) -> QuantKey | None:
from vllm.utils.humming import HummingInputSchema
"""Resolve the actual activation QuantKey after platform fallback."""
a_dtype = HummingInputSchema().get_fallback_input_dtype(origin_a_dtype)
if a_dtype is None or a_dtype.num_bits >= 16:
return None
dtype = _HUMMING_TO_QUANT_DTYPE[a_dtype]
gs = group_size if a_dtype == humming_dtypes.float4e2m1 else 0
group_shape = GroupShape(row=1, col=gs) if gs > 0 else GroupShape.PER_TOKEN
scale_dtype = MXFP_SCALE_DTYPE if gs > 0 else torch.float32
scale = ScaleDesc(dtype=scale_dtype, static=False, group_shape=group_shape)
return QuantKey(dtype=dtype, scale=scale, symmetric=True)
def _compressed_tensors_input_schema_to_quant_key(
schema: "CompressedTensorsInputSchema",
) -> QuantKey | None:
type_bits_to_dtype = {
("float", 8): humming_dtypes.float8e4m3,
("float", 4): humming_dtypes.float4e2m1,
("int", 8): humming_dtypes.int8,
("int", 4): humming_dtypes.int4,
}
origin = type_bits_to_dtype.get((schema.type, schema.num_bits))
if origin is None:
return None
return _resolve_input_quant_key(origin, schema.group_size)
# ---- Dispatch for any BaseInputSchema -------------------------------------
def input_schema_to_quant_key(
schema: "BaseInputSchema",
) -> QuantKey | None:
from vllm.utils.humming import (
CompressedTensorsInputSchema,
Fp8InputSchema,
HummingInputSchema,
ModeloptNvfp4InputSchema,
)
"""Convert any BaseInputSchema to a QuantKey. Returns None if
the schema represents unquantized (bf16/fp16) inputs."""
if isinstance(schema, HummingInputSchema):
return _humming_input_schema_to_quant_key(schema)
if isinstance(schema, Fp8InputSchema):
return _resolve_input_quant_key(humming_dtypes.float8e4m3, 0)
if isinstance(schema, ModeloptNvfp4InputSchema):
return _resolve_input_quant_key(
humming_dtypes.float8e4m3,
schema.group_size,
)
if isinstance(schema, CompressedTensorsInputSchema):
return _compressed_tensors_input_schema_to_quant_key(schema)
raise TypeError(f"Unsupported input schema type: {type(schema)}")
def humming_is_layer_skipped(config: dict[str, Any], prefix: str):
if not config:
return True
keys = ["ignored_layers", "ignore", "modules_to_not_convert"]
ignored_layers: list[str] = []
for key in keys:
candidate = config.get(key, []) or []
if candidate:
ignored_layers = candidate
break
if any(module_name in prefix for module_name in ignored_layers):
return True
if "lm_head" in prefix:
return True
for regex in config.get("dynamic", {}):
if regex[:1] != "-":
continue
if re.match(regex[2:], prefix):
return True
return False
def convert_linear_layer_to_humming_standard(
layer: LinearBase, name_map: dict[str, str]
):
"""Rename/reshape a linear layer's quantized params (the canonical MPLinear
layout: ``weight_packed`` int32 + ``weight_scale``) into the parameter names
and layout humming's weight schema expects (``weight`` / ``weight_scale``)."""
for name, checkpoint_name in name_map.items():
tensor = getattr(layer, checkpoint_name)
delattr(layer, checkpoint_name)
if name == "weight":
input_dim = getattr(tensor, "input_dim", 1)
output_dim = getattr(tensor, "output_dim", 0)
if input_dim == 0 and output_dim == 1:
tensor = tensor.transpose(1, 0).contiguous()
else:
assert output_dim == 0 and input_dim == 1
tensor = tensor.view(tensor.size(0), -1).view(torch.int32)
elif name in ["weight_scale", "zero_point"]:
if getattr(tensor, "output_dim", 0) == 1:
tensor = tensor.transpose(0, 1).contiguous()
if tensor.ndim == 1:
tensor = tensor.unsqueeze(1)
tensor = tensor.view(torch.int32) if name == "zero_point" else tensor
if isinstance(tensor, torch.nn.Parameter):
param = tensor
else:
param = torch.nn.Parameter(tensor, requires_grad=False)
setattr(layer, name, param)
def prepare_humming_linear_layer_config(
layer: LinearBase,
quant_config: dict,
input_quant_config: dict | None = None,
) -> "LayerConfig":
from vllm.utils.humming import (
BaseInputSchema,
BaseWeightSchema,
HummingInputSchema,
prepare_layer_config,
transform_humming_tensors,
)
weight_schema = BaseWeightSchema.from_config(quant_config)
if input_quant_config is not None:
input_schema = BaseInputSchema.from_config(input_quant_config)
else:
input_schema = HummingInputSchema()
# ReplicatedLinear has no TP partitioning and so does not set
# input_size_per_partition; for it that is just input_size. Use hasattr
# rather than getattr's default arg, which is evaluated eagerly and would
# raise on layers lacking input_size (e.g. ParallelLMHead).
if hasattr(layer, "input_size_per_partition"):
input_size_per_partition = layer.input_size_per_partition
else:
input_size_per_partition = layer.input_size
shape_k_stacks = [input_size_per_partition]
shape_n_stacks = layer.output_partition_sizes
# Step 1: convert weight and input schemas to humming standard format
weight_schema, tensors = weight_schema.convert_humming(
tensors=dict(layer.named_parameters()),
shape_n_stacks=shape_n_stacks,
shape_k_stacks=shape_k_stacks,
param_dtype=layer.params_dtype,
)
input_schema, _ = input_schema.convert_humming(
tensors={},
shape_n_stacks=shape_n_stacks,
shape_k_stacks=shape_k_stacks,
param_dtype=layer.params_dtype,
)
layer.weight_schema = weight_schema
# Step 2: transform weight (humming standard format) for forwarding.
config = prepare_layer_config(
shape_n=sum(layer.output_partition_sizes),
shape_k=input_size_per_partition,
weight_schema=weight_schema,
input_schema=input_schema,
pad_n_to_multiple=256,
pad_k_to_multiple=128,
has_bias=layer.has_bias,
torch_dtype=layer.params_dtype,
)
tensors = transform_humming_tensors(config, tensors)
for name, _ in list(layer.named_parameters()):
delattr(layer, name)
for name, tensor in tensors.items():
param = torch.nn.Parameter(tensor, requires_grad=False)
setattr(layer, name, param)
return config
def get_humming_linear_compute_config() -> str:
return json.dumps(
{
"use_batch_invariant": envs.VLLM_BATCH_INVARIANT,
"use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,
"gemm_type": "dense",
}
)
def apply_humming_linear(
layer: LinearBase,
x: torch.Tensor,
*,
layer_config: "LayerConfig",
compute_config: str,
locks: torch.Tensor,
) -> torch.Tensor:
from vllm.utils.humming import humming_forward
flatten_inputs = x.reshape(-1, x.size(-1))
output = humming_forward(
layer_config,
inputs=flatten_inputs,
weight=layer.weight,
weight_scale=getattr(layer, "weight_scale", None),
zero_point=getattr(layer, "zero_point", None),
bias=getattr(layer, "bias", None),
weight_scale_2=getattr(layer, "weight_scale_2", None),
locks=locks,
compute_config=compute_config,
)
return output.view(*x.shape[:-1], output.size(-1))
w1_hadamard_block_size: int = 1
w2_hadamard_block_size: int = 1
w1_input_scale: torch.Tensor | None = None
w2_input_scale: torch.Tensor | None = None
w1_input_scale_2: torch.Tensor | None = None
w2_input_scale_2: torch.Tensor | None = None
def make_humming_moe_quant_config(
@@ -648,8 +142,12 @@ def get_humming_moe_quant_config(
q_dtype is not None
and q_dtype.startswith("float8")
and input_scale_group_size == 128
and weight_schema.hadamard_block_size <= 1
and input_schema.input_quant_mode in (None, "dynamic_group")
):
q_dtype = _HUMMING_TO_QUANT_DTYPE.get(input_schema.a_dtype, FP8_DTYPE)
q_dtype = humming_schema._HUMMING_TO_QUANT_DTYPE.get(
input_schema.a_dtype, FP8_DTYPE
)
activation_group_shape = GroupShape(row=1, col=input_scale_group_size)
weight_scale_group_size = weight_schema.weight_scale_group_size
@@ -665,7 +163,7 @@ def get_humming_moe_quant_config(
else:
weight_group_shape = GroupShape(row=weight_scale_group_size, col=1)
return make_humming_moe_quant_config(
config = make_humming_moe_quant_config(
quant_dtype=q_dtype,
weight_dtype=str(weight_schema.b_dtype),
weight_group_shape=weight_group_shape,
@@ -683,6 +181,17 @@ def get_humming_moe_quant_config(
gemm1_clamp_limit=gemm1_clamp_limit,
humming_configs=humming_configs,
)
for prefix, sublayer in (("w1", "w13"), ("w2", "w2")):
setattr(
config,
f"{prefix}_hadamard_block_size",
layer.weight_schemas[sublayer].hadamard_block_size,
)
for name in ("input_scale", "input_scale_2"):
key = f"{prefix}_{name}"
value = getattr(layer, f"{sublayer}_{name}", None)
setattr(config, key, value)
return config
def select_humming_moe_experts(
@@ -846,12 +355,6 @@ def _convert_sublayer_to_humming(
Tuple of (converted_weight_schema, converted_input_schema)
"""
from vllm.utils.humming import HummingWeightSchema
if isinstance(weight_schema, HummingWeightSchema):
# Already in Humming format
return weight_schema, input_schema
tensors = _extract_sublayer_tensors(layer, sublayer_name)
shape_k_stacks = [shape_k]
@@ -867,13 +370,14 @@ def _convert_sublayer_to_humming(
num_experts=num_experts,
)
converted_input_schema, _ = input_schema.convert_humming(
tensors=converted_tensors,
converted_input_schema, input_tensors = input_schema.convert_humming(
tensors=tensors,
shape_n_stacks=shape_n_stacks,
shape_k_stacks=shape_k_stacks,
param_dtype=param_dtype,
num_experts=num_experts,
)
converted_tensors.update(input_tensors)
_replace_layer_parameters(layer, sublayer_name, converted_tensors)
@@ -907,11 +411,16 @@ def _prepare_and_transform_sublayer(
has_bias=has_bias,
num_experts=num_experts,
torch_dtype=param_dtype,
device=getattr(layer, f"{sublayer_name}_weight").device,
)
tensors = transform_humming_tensors(
config,
_extract_sublayer_tensors(layer, sublayer_name),
)
source_tensors = _extract_sublayer_tensors(layer, sublayer_name)
for name in ("input_scale", "input_scale_2"):
if name != input_schema.static_tensor_scale_name:
source_tensors.pop(name, None)
tensors = transform_humming_tensors(config, source_tensors)
for name in ("input_scale", "input_scale_2"):
if name in source_tensors:
tensors[name] = source_tensors[name]
_replace_layer_parameters(layer, sublayer_name, tensors)
return config
@@ -927,6 +436,7 @@ def _process_single_sublayer(
num_experts: int,
param_dtype: torch.dtype,
force_weight_schema: Any | None = None,
allow_input_schema_fallback: bool = True,
) -> tuple[Any, Any, "LayerConfig"]:
"""Process a single sublayer: convert, optionally requant, prepare, and transform.
@@ -944,6 +454,7 @@ def _process_single_sublayer(
num_experts: Number of experts
param_dtype: Parameter data type
force_weight_schema: Optional schema to force requantization to
allow_input_schema_fallback: Whether incompatible input schemas may be replaced.
Returns:
Tuple of the final weight schema, input schema, and Humming layer config.
@@ -966,19 +477,28 @@ def _process_single_sublayer(
# Step 2: Force requant if needed
assert isinstance(current_weight_schema, HummingWeightSchema)
if force_weight_schema is not None and current_weight_schema != force_weight_schema:
tensors = _extract_sublayer_tensors(layer, sublayer_name)
source_tensors = _extract_sublayer_tensors(layer, sublayer_name)
tensors = current_weight_schema.requant_tensors(
tensors=tensors,
tensors=source_tensors,
target_weight_schema=force_weight_schema,
param_dtype=param_dtype,
)
name = current_input_schema.static_tensor_scale_name
if name is not None:
tensors[name] = source_tensors[name]
current_weight_schema = force_weight_schema
_replace_layer_parameters(layer, sublayer_name, tensors, preserve_bias=True)
del tensors
# Step 3: Prepare layer metadata and transform weights
current_input_schema = humming_schema.check_and_fallback_input_schema(
weight_schema=current_weight_schema,
input_schema=current_input_schema,
param_dtype=param_dtype,
allow_fallback=allow_input_schema_fallback,
)
config = _prepare_and_transform_sublayer(
layer=layer,
sublayer_name=sublayer_name,
@@ -1001,6 +521,7 @@ def convert_to_humming_moe_kernel_format(
weight_schema: Any | None = None,
input_schema: Any | None = None,
force_weight_schema: Any | None = None,
allow_input_schema_fallback: bool = True,
) -> dict[str, "LayerConfig"]:
"""Convert MoE weights from checkpoint format to Humming kernel format.
@@ -1023,6 +544,7 @@ def convert_to_humming_moe_kernel_format(
input_schema: Optional initial input quantization schema.
If None, built from quant_config or env vars.
force_weight_schema: Optional schema to force requantization to
allow_input_schema_fallback: Whether incompatible input schemas may be replaced.
Side effects:
- Modifies layer parameters in place
@@ -1041,20 +563,23 @@ def convert_to_humming_moe_kernel_format(
"Must provide either weight_schema/input_schema or quant_config"
)
from vllm.model_executor.layers.quantization.utils.humming_utils import (
humming_is_layer_skipped,
)
from vllm.utils.humming import BaseWeightSchema, HummingInputSchema
if weight_schema is None:
weight_schema = BaseWeightSchema.from_config(quant_config)
if input_schema is None:
input_quant_config = envs.VLLM_HUMMING_INPUT_QUANT_CONFIG or {}
input_quant_config = (envs.VLLM_HUMMING_INPUT_QUANT_CONFIG or {}).copy()
if humming_is_layer_skipped(input_quant_config, layer.layer_name):
input_schema = HummingInputSchema()
else:
# TODO: read input_quant_config from quant_config
input_quant_config = humming_schema.resolve_humming_layer_config(
input_quant_config, layer.layer_name
)
allow_input_schema_fallback = input_quant_config.pop(
"allow_fallback", False
)
input_schema = HummingInputSchema.from_config(input_quant_config)
# Build sublayer configs from layer properties if not provided
@@ -1089,6 +614,7 @@ def convert_to_humming_moe_kernel_format(
num_experts=num_experts,
param_dtype=param_dtype,
force_weight_schema=force_weight_schema,
allow_input_schema_fallback=allow_input_schema_fallback,
)
)
@@ -0,0 +1,437 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Map Humming schemas and handle shared checkpoint quantization settings."""
import dataclasses
from typing import TYPE_CHECKING, Any
import regex as re
import torch
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.quant_utils import (
FP4_DTYPE,
FP8_DTYPE,
INT4_DTYPE,
INT8_DTYPE,
MXFP_SCALE_DTYPE,
GroupShape,
QuantKey,
ScaleDesc,
)
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
from vllm.utils.import_utils import has_humming
if TYPE_CHECKING:
from vllm.utils.humming import (
BaseInputSchema,
BaseWeightSchema,
HummingInputSchema,
HummingWeightSchema,
)
from vllm.utils.humming import dtypes as humming_dtypes
def humming_update_schema_hadamard_block_size(
weight_schema: "HummingWeightSchema",
input_schema: "HummingInputSchema",
shape_k: int,
) -> "HummingWeightSchema":
assert shape_k > 0
block_size = 32
weight_group_size = weight_schema.weight_scale_group_size
input_group_size = input_schema.input_scale_group_size
for group_size in (weight_group_size, input_group_size):
if group_size is not None and group_size > 0:
block_size = min(block_size, group_size)
block_size = 1 << (block_size.bit_length() - 1)
while shape_k % block_size:
block_size //= 2
return dataclasses.replace(weight_schema, hadamard_block_size=block_size)
if has_humming():
from vllm.utils.humming import dtypes as humming_dtypes
_HUMMING_TO_QUANT_DTYPE: dict[humming_dtypes.DataType, Any] = {
humming_dtypes.float4e0m3: FP4_DTYPE,
humming_dtypes.float4e2m1: FP4_DTYPE,
humming_dtypes.float6e2m3: scalar_types.float6_e2m3f,
humming_dtypes.float6e3m2: scalar_types.float6_e3m2f,
humming_dtypes.float8e3m4: FP8_DTYPE,
humming_dtypes.float8e4m3: FP8_DTYPE,
humming_dtypes.float8e5m2: torch.float8_e5m2,
humming_dtypes.int8: torch.int8,
humming_dtypes.uint4: INT4_DTYPE,
humming_dtypes.uint8: INT8_DTYPE,
humming_dtypes.uint2: torch.uint8,
humming_dtypes.uint3: torch.uint8,
humming_dtypes.uint5: torch.uint8,
humming_dtypes.uint6: torch.uint8,
humming_dtypes.uint7: torch.uint8,
}
_HUMMING_TO_SCALE_DTYPE: dict[humming_dtypes.DataType, torch.dtype] = {
humming_dtypes.float8e8m0: MXFP_SCALE_DTYPE,
humming_dtypes.float8e4m3: FP8_DTYPE,
humming_dtypes.float16: torch.float16,
humming_dtypes.bfloat16: torch.bfloat16,
humming_dtypes.float32: torch.float32,
}
logger = init_logger(__name__)
def _group_shape(group_size: int, group_size_n: int = 0) -> GroupShape:
"""Map humming group sizes to QuantKey GroupShape.
group_size: elements per group along K (col); 0 means full dimension.
group_size_n: elements per group along N (row); 0 means 1 (per-row).
GroupShape convention: row = N dim, col = K dim.
"""
if group_size == 0 and group_size_n == 0:
return GroupShape.PER_CHANNEL
row = group_size_n if group_size_n > 0 else 1
col = group_size if group_size > 0 else -1
return GroupShape(row=row, col=col)
def weight_schema_to_quant_key(
schema: "BaseWeightSchema",
param_dtype: torch.dtype | None = None,
) -> QuantKey:
from vllm.utils.humming import WeightScale2Type, WeightScaleType
if param_dtype is None:
param_dtype = torch.get_default_dtype()
if param_dtype not in [torch.float16, torch.bfloat16]:
param_dtype = torch.bfloat16
if not current_platform.has_device_capability(80):
param_dtype = torch.float16
schema = schema.to_humming_schema(param_dtype)
dtype = _HUMMING_TO_QUANT_DTYPE[schema.b_dtype]
if schema.bs_dtype is not None:
scale_dtype = _HUMMING_TO_SCALE_DTYPE[schema.bs_dtype]
else:
scale_dtype = torch.float32
group_shape = _group_shape(
schema.weight_scale_group_size,
schema.weight_scale_group_size_n,
)
if schema.weight_scale_type == WeightScaleType.TENSOR:
group_shape = GroupShape.PER_TENSOR
scale = ScaleDesc(dtype=scale_dtype, static=True, group_shape=group_shape)
scale2 = None
if schema.weight_scale_2_type == WeightScale2Type.TENSOR:
scale2 = ScaleDesc(
dtype=torch.float32,
static=True,
group_shape=GroupShape.PER_TENSOR,
)
return QuantKey(
dtype=dtype,
scale=scale,
scale2=scale2,
symmetric=not schema.has_zero_point,
)
def input_schema_to_quant_key(
schema: "BaseInputSchema",
param_dtype: torch.dtype | None = None,
) -> QuantKey | None:
from vllm.utils.humming import InputQuantizationMode
if param_dtype is None:
param_dtype = torch.get_default_dtype()
if param_dtype not in [torch.float16, torch.bfloat16]:
param_dtype = torch.bfloat16
if not current_platform.has_device_capability(80):
param_dtype = torch.float16
schema = schema.to_humming_schema(param_dtype)
if schema.a_dtype is None or schema.a_dtype.num_bits >= 16:
return None
mode = schema.input_quant_mode
if mode == InputQuantizationMode.Disabled:
return None
dtype = _HUMMING_TO_QUANT_DTYPE[schema.a_dtype]
gs = schema.input_scale_group_size
group_shape = GroupShape(row=1, col=gs) if gs > 0 else GroupShape.PER_TOKEN
# Pick the scale dtype the Humming kernel actually consumes. An explicit
# input_scale_dtype always wins. Otherwise infer from the grouping: MX
# microscale activations (group size 32, e.g. MXFP8) carry an e8m0 (uint8)
# scale, while block-FP8 (group size 128) and per-token FP8/int8 carry a
# float32 scale. Getting this right lets a grouped FP8 activation match
# kFp8Dynamic128Sym instead of an unmatchable uint8-scaled key.
if schema.input_scale_dtype is not None:
scale_dtype = _HUMMING_TO_SCALE_DTYPE[schema.input_scale_dtype]
elif gs == 16:
scale_dtype = FP8_DTYPE
elif gs == 32:
scale_dtype = MXFP_SCALE_DTYPE
else:
scale_dtype = torch.float32
if mode == InputQuantizationMode.StaticTensor:
group_shape = GroupShape.PER_TENSOR
scale = ScaleDesc(
dtype=scale_dtype,
static=mode == InputQuantizationMode.StaticTensor,
group_shape=group_shape,
)
scale2 = None
has_scale2 = (
InputQuantizationMode.StaticTensorDynamicGroup,
InputQuantizationMode.DynamicGroupToken,
)
if mode in has_scale2:
static = mode == InputQuantizationMode.StaticTensorDynamicGroup
scale2 = ScaleDesc(
torch.float32,
static,
GroupShape.PER_TENSOR if static else GroupShape.PER_TOKEN,
)
return QuantKey(dtype=dtype, scale=scale, scale2=scale2, symmetric=True)
def quant_key_to_input_schema(key: QuantKey | None) -> "HummingInputSchema":
from vllm.utils.humming import HummingInputSchema, InputQuantizationMode
if key is None:
return HummingInputSchema(input_quant_mode=InputQuantizationMode.Disabled)
if not key.symmetric:
raise ValueError("Humming input quantization must be symmetric")
quant_dtypes = {value: dtype for dtype, value in _HUMMING_TO_QUANT_DTYPE.items()}
quant_dtypes[FP4_DTYPE] = humming_dtypes.float4e2m1
quant_dtypes[FP8_DTYPE] = humming_dtypes.float8e4m3
scale_dtypes = {value: dtype for dtype, value in _HUMMING_TO_SCALE_DTYPE.items()}
if key.dtype not in quant_dtypes or key.scale.dtype not in scale_dtypes:
raise ValueError(f"Unsupported Humming input or scale dtype: {key}")
scale = key.scale
group_size = 0
if scale.static and scale.group_shape.is_per_tensor() and key.scale2 is None:
mode = InputQuantizationMode.StaticTensor
elif not scale.static and scale.group_shape.is_per_token() and key.scale2 is None:
mode = InputQuantizationMode.DynamicToken
elif not scale.static and scale.group_shape.is_per_group():
group_size = scale.group_shape.col
scale2 = key.scale2
if scale2 is None:
mode = InputQuantizationMode.DynamicGroup
elif scale2.dtype != torch.float32:
raise ValueError("Humming secondary input scales must be float32")
elif scale2.static and scale2.group_shape.is_per_tensor():
mode = InputQuantizationMode.StaticTensorDynamicGroup
elif not scale2.static and scale2.group_shape.is_per_token():
mode = InputQuantizationMode.DynamicGroupToken
else:
raise ValueError(f"Unsupported Humming secondary input scale: {scale2}")
else:
raise ValueError(f"Unsupported Humming input scale: {key}")
if group_size == 0 and scale.dtype != torch.float32:
raise ValueError("Humming tensor and token input scales must be float32")
return HummingInputSchema(
a_dtype=quant_dtypes[key.dtype],
input_scale_group_size=group_size,
input_scale_dtype=scale_dtypes[scale.dtype],
input_quant_mode=mode,
)
def check_and_fallback_input_schema(
weight_schema: "BaseWeightSchema",
input_schema: "BaseInputSchema",
param_dtype: torch.dtype | None = None,
allow_fallback: bool = True,
) -> "HummingInputSchema":
from vllm.utils.humming import HummingInputSchema, InputQuantizationMode
capability = current_platform.get_device_capability()
assert capability is not None
sm_version = capability.to_int()
if param_dtype is None:
param_dtype = torch.get_default_dtype()
if param_dtype not in (torch.float16, torch.bfloat16):
param_dtype = torch.bfloat16 if sm_version >= 80 else torch.float16
assert param_dtype in (torch.float16, torch.bfloat16)
dtype_fallback_order_map = {
humming_dtypes.float8e5m2: (
humming_dtypes.float8e4m3,
humming_dtypes.float8e3m4,
humming_dtypes.int8,
),
humming_dtypes.float8e4m3: (
humming_dtypes.float8e3m4,
humming_dtypes.int8,
),
humming_dtypes.float8e3m4: (
humming_dtypes.float8e4m3,
humming_dtypes.int8,
),
humming_dtypes.float4e2m1: (
humming_dtypes.float8e4m3,
humming_dtypes.float8e3m4,
humming_dtypes.int8,
),
humming_dtypes.float4e0m3: (
humming_dtypes.int4,
humming_dtypes.float8e4m3,
humming_dtypes.float8e3m4,
humming_dtypes.int8,
),
humming_dtypes.int8: (
humming_dtypes.float8e4m3,
humming_dtypes.float8e3m4,
),
humming_dtypes.int4: (
humming_dtypes.int8,
humming_dtypes.float8e4m3,
humming_dtypes.float8e3m4,
),
}
weight_schema = weight_schema.to_humming_schema(param_dtype)
input_schema = input_schema.to_humming_schema(param_dtype)
fp6_dtypes = (humming_dtypes.float6e2m3, humming_dtypes.float6e3m2)
if input_schema.a_dtype in fp6_dtypes:
input_schema.a_dtype = humming_dtypes.float8e4m3
a_dtype = input_schema.a_dtype
input_bits = a_dtype.num_bits if a_dtype is not None else 16
weight_group_size = weight_schema.weight_scale_group_size
input_group_size = input_schema.input_scale_group_size
input_scale_dtype = input_schema.input_scale_dtype
def is_deprecated(dtype: "humming_dtypes.DataType | None") -> bool:
if dtype is None or dtype.num_bits == 16:
return False
is_int4_deprecated = dtype == humming_dtypes.int4 and sm_version >= 90
is_int8_deprecated = dtype == humming_dtypes.int8 and 103 <= sm_version < 110
# Humming has not yet implemented high-performance FP8 computation
# on SM10X and SM11X. Therefore, it is temporarily considered deprecated.
# TODO: removed this after Humming has high-performance FP8 computation
is_fp8_deprecated = dtype.num_bits < 16 and 100 <= sm_version < 120
return is_int4_deprecated or is_int8_deprecated or is_fp8_deprecated
if input_schema.is_compatible_with(weight_schema, param_dtype):
if not allow_fallback:
if is_deprecated(a_dtype):
msg = f"Humming {a_dtype} activation is slow on SM{sm_version} now."
logger.warning_once(msg)
return input_schema
if a_dtype is None or a_dtype.num_bits == 16:
return input_schema
if not is_deprecated(a_dtype):
is_mxfp8 = (
a_dtype.is_floating_point_type
and a_dtype.num_bits == 8
and input_group_size == 32
and input_scale_dtype == humming_dtypes.float8e8m0
and sm_version >= 120
)
is_groupwise = input_group_size > 0 or weight_group_size > 0
if input_bits == 8 and is_groupwise and not is_mxfp8:
# prefer wna16 for groupwise weight or groupwise input
return HummingInputSchema()
return input_schema
if allow_fallback:
fallback_dtypes = (
a_dtype,
*dtype_fallback_order_map.get(a_dtype, ()),
humming_dtypes.DataType.from_any(param_dtype),
)
for dtype in fallback_dtypes:
if dtype is None or is_deprecated(dtype):
continue
group_size = 0
scale_dtype = humming_dtypes.float32
quant_mode = InputQuantizationMode.DynamicToken
if dtype.num_bits == 16:
scale_dtype = None
quant_mode = InputQuantizationMode.Disabled
elif dtype in (humming_dtypes.float8e4m3, humming_dtypes.float8e3m4):
is_channelwise_weight = weight_schema.weight_scale_group_size == 0
weight_group_size = weight_schema.weight_scale_group_size
is_e8m0_scale = weight_schema.bs_dtype == humming_dtypes.float8e8m0
is_mx_weight = weight_group_size == 32 and is_e8m0_scale
if sm_version >= 120 and (is_channelwise_weight or is_mx_weight):
group_size = 32
scale_dtype = humming_dtypes.float8e8m0
quant_mode = InputQuantizationMode.DynamicGroup
candidate = HummingInputSchema(
a_dtype=dtype,
input_scale_group_size=group_size,
input_scale_dtype=scale_dtype,
input_quant_mode=quant_mode,
)
if candidate.is_compatible_with(weight_schema, param_dtype):
return candidate
raise ValueError(
f"No compatible Humming input schema for {input_schema} with "
f"weight schema {weight_schema} on SM{sm_version} "
f"({allow_fallback=})"
)
def resolve_humming_layer_config(config: dict[str, Any], prefix: str) -> dict[str, Any]:
layer_config = config.copy()
dynamic = config.get("dynamic", {})
if isinstance(dynamic, dict):
for pattern, overrides in dynamic.items():
if pattern.startswith("+") and re.match(pattern[2:], prefix):
layer_config.update(overrides)
break
return layer_config
def humming_is_layer_skipped(config: dict[str, Any], prefix: str):
if not config:
return True
keys = ["ignored_layers", "ignore", "modules_to_not_convert"]
ignored_layers: list[str] = []
for key in keys:
candidate = config.get(key, []) or []
if candidate:
ignored_layers = candidate
break
if any(module_name in prefix for module_name in ignored_layers):
return True
if "lm_head" in prefix:
return True
for regex in config.get("dynamic", {}):
if regex[:1] != "-":
continue
if re.match(regex[2:], prefix):
return True
return False
+52 -3
View File
@@ -4,12 +4,56 @@
vLLM code should import humming symbols from here so that ``import humming``
(which has import-time side effects) is deferred until first use. Add new
symbols by appending one entry to ``_EXPORTS`` as ``"module.path:attr"``,
or ``"module.path"`` for a whole-module re-export.
symbols to both the ``TYPE_CHECKING`` imports and ``_EXPORTS``, using
``"module.path:attr"`` or ``"module.path"`` for a whole-module re-export.
"""
import importlib
from typing import Any
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from humming import dtypes as dtypes
from humming.config import GemmType as GemmType
from humming.config import InputQuantizationMode as InputQuantizationMode
from humming.config import LayerConfig as LayerConfig
from humming.config import MmaType as MmaType
from humming.config import WeightScale2Type as WeightScale2Type
from humming.config import WeightScaleType as WeightScaleType
from humming.dtypes import DataType as DataType
from humming.forward import humming_forward as humming_forward
from humming.forward import may_process_input as may_process_input
from humming.forward import may_quant_input as may_quant_input
from humming.ops import process_input as process_input
from humming.schema import AWQWeightSchema as AWQWeightSchema
from humming.schema import BaseInputSchema as BaseInputSchema
from humming.schema import BaseWeightSchema as BaseWeightSchema
from humming.schema import BitnetWeightSchema as BitnetWeightSchema
from humming.schema import (
CompressedTensorsInputSchema as CompressedTensorsInputSchema,
)
from humming.schema import (
CompressedTensorsWeightSchema as CompressedTensorsWeightSchema,
)
from humming.schema import Fp8InputSchema as Fp8InputSchema
from humming.schema import GptOssMxfp4WeightSchema as GptOssMxfp4WeightSchema
from humming.schema import GPTQWeightSchema as GPTQWeightSchema
from humming.schema import HummingInputSchema as HummingInputSchema
from humming.schema import HummingWeightSchema as HummingWeightSchema
from humming.schema import Mxfp4WeightSchema as Mxfp4WeightSchema
from humming.schema.fp8 import Fp8WeightSchema as Fp8WeightSchema
from humming.schema.modelopt import (
ModeloptMxfp8WeightSchema as ModeloptMxfp8WeightSchema,
)
from humming.schema.modelopt import (
ModeloptNvfp4InputSchema as ModeloptNvfp4InputSchema,
)
from humming.schema.modelopt import (
ModeloptNvfp4WeightSchema as ModeloptNvfp4WeightSchema,
)
from humming.transform import prepare_layer_config as prepare_layer_config
from humming.transform import transform_humming_tensors as transform_humming_tensors
from humming.tune import get_heuristics_config as get_heuristics_config
from humming.utils.weight import quantize_weight as quantize_weight
_EXPORTS: dict[str, str] = {
"dtypes": "humming.dtypes",
@@ -17,7 +61,12 @@ _EXPORTS: dict[str, str] = {
"GemmType": "humming.config:GemmType",
"LayerConfig": "humming.config:LayerConfig",
"WeightScaleType": "humming.config:WeightScaleType",
"WeightScale2Type": "humming.config:WeightScale2Type",
"InputQuantizationMode": "humming.config:InputQuantizationMode",
"MmaType": "humming.config:MmaType",
"humming_forward": "humming.forward:humming_forward",
"may_process_input": "humming.forward:may_process_input",
"process_input": "humming.ops:process_input",
"may_quant_input": "humming.forward:may_quant_input",
"prepare_layer_config": "humming.transform:prepare_layer_config",
"transform_humming_tensors": "humming.transform:transform_humming_tensors",