[Quantization] add humming mxfp4 moe backend (#41083)
Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com>
This commit is contained in:
@@ -118,6 +118,7 @@ MoEBackend = Literal[
|
||||
"flashinfer_cutlass",
|
||||
"flashinfer_cutedsl",
|
||||
"marlin",
|
||||
"humming",
|
||||
"aiter",
|
||||
"emulation",
|
||||
]
|
||||
@@ -148,6 +149,7 @@ class KernelConfig:
|
||||
- "flashinfer_cutlass": Use FlashInfer with CUTLASS kernels
|
||||
- "flashinfer_cutedsl": Use FlashInfer with CuteDSL kernels (FP4 only)
|
||||
- "marlin": Use Marlin kernels (weight-only quantization)
|
||||
- "humming": Use Humming Mixed Precision kernels
|
||||
- "aiter": Use AMD AITer kernels (ROCm only)
|
||||
- "emulation": use BF16/FP16 GEMM, dequantizing weights and
|
||||
running QDQ on activations.
|
||||
|
||||
+2
-2
@@ -1231,8 +1231,8 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# if 1, force use indexed gemm
|
||||
# if 0, force use grouped gemm
|
||||
# if None, choose better gemm type automatically
|
||||
"VLLM_HUMMING_MOE_GEMM_TYPE": lambda: maybe_convert_bool(
|
||||
os.environ.get("VLLM_HUMMING_MOE_GEMM_TYPE", None)
|
||||
"VLLM_HUMMING_MOE_GEMM_TYPE": lambda: os.environ.get(
|
||||
"VLLM_HUMMING_MOE_GEMM_TYPE", None
|
||||
),
|
||||
# Whether to use DeepEPLL kernels for NVFP4 quantization and dispatch method
|
||||
# only supported on Blackwell GPUs and with
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
import json
|
||||
import math
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from humming import dtypes
|
||||
@@ -16,7 +16,11 @@ from vllm import envs
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
|
||||
moe_align_block_size,
|
||||
)
|
||||
@@ -34,21 +38,16 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization.humming import HummingMoEMethod
|
||||
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def get_humming_moe_gemm_type() -> str:
|
||||
env_gemm_type: str = envs.VLLM_HUMMING_MOE_GEMM_TYPE or ""
|
||||
env_gemm_type = env_gemm_type.lower()
|
||||
if env_gemm_type in ["indexed", "grouped"]:
|
||||
if env_gemm_type == "indexed":
|
||||
gemm_type = env_gemm_type
|
||||
elif current_platform.has_device_capability(90):
|
||||
# for device that supports TMA, use grouped gemm
|
||||
gemm_type = "grouped"
|
||||
elif env_gemm_type in ["grouped_contiguous", "grouped"]:
|
||||
gemm_type = "grouped_contiguous"
|
||||
else:
|
||||
gemm_type = "indexed"
|
||||
|
||||
@@ -60,49 +59,44 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
def __init__(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
quant_method: "HummingMoEMethod",
|
||||
prepare_finalize: mk.FusedMoEPrepareAndFinalizeModular | None = None,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
max_num_tokens: int | None = None,
|
||||
num_dispatchers: int | None = None,
|
||||
):
|
||||
self.layer = layer
|
||||
self.num_experts = self.layer.num_experts
|
||||
self.global_num_experts = self.layer.global_num_experts
|
||||
self.init_humming_moe()
|
||||
|
||||
if prepare_finalize is not None:
|
||||
max_num_tokens: int | None = None
|
||||
num_dispatchers: int | None = None
|
||||
if self.is_batched:
|
||||
max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
|
||||
num_dispatchers = prepare_finalize.num_dispatchers()
|
||||
if self.is_batched():
|
||||
assert max_num_tokens is not None and num_dispatchers is not None
|
||||
|
||||
assert quant_method.moe_quant_config is not None
|
||||
super().__init__(
|
||||
moe_config=quant_method.moe,
|
||||
quant_config=quant_method.moe_quant_config,
|
||||
max_num_tokens=max_num_tokens,
|
||||
num_dispatchers=num_dispatchers,
|
||||
)
|
||||
else:
|
||||
assert not self.is_batched
|
||||
super().__init__(
|
||||
moe_config=moe_config,
|
||||
quant_config=quant_config,
|
||||
max_num_tokens=max_num_tokens,
|
||||
num_dispatchers=num_dispatchers,
|
||||
)
|
||||
|
||||
def init_humming_moe(self):
|
||||
self.compute_config = {
|
||||
"use_batch_invariant": envs.VLLM_BATCH_INVARIANT,
|
||||
"use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,
|
||||
"gemm_type": self.humming_gemm_type.value,
|
||||
"gemm_type": self.humming_gemm_type().value,
|
||||
}
|
||||
self.w13_tuning_config = HummingMethod.get_default_tuning_configs(
|
||||
layer=self.layer,
|
||||
use_f16_accum=envs.VLLM_HUMMING_USE_F16_ACCUM,
|
||||
use_batch_invariant=envs.VLLM_BATCH_INVARIANT,
|
||||
gemm_type=self.humming_gemm_type,
|
||||
gemm_type=self.humming_gemm_type(),
|
||||
sublayer_name="w13",
|
||||
)
|
||||
self.w2_tuning_config = HummingMethod.get_default_tuning_configs(
|
||||
layer=self.layer,
|
||||
use_f16_accum=envs.VLLM_HUMMING_USE_F16_ACCUM,
|
||||
use_batch_invariant=envs.VLLM_BATCH_INVARIANT,
|
||||
gemm_type=self.humming_gemm_type,
|
||||
gemm_type=self.humming_gemm_type(),
|
||||
sublayer_name="w2",
|
||||
)
|
||||
self.compute_config_str = json.dumps(self.compute_config)
|
||||
@@ -124,13 +118,13 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
global_num_experts = self.global_num_experts
|
||||
return math.ceil(global_valid_shape_m * num_experts / global_num_experts)
|
||||
|
||||
@property
|
||||
def humming_gemm_type(self) -> HummingGemmType:
|
||||
@staticmethod
|
||||
def humming_gemm_type() -> HummingGemmType:
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
def is_batched(self) -> bool:
|
||||
return self.activation_format() == mk.FusedMoEActivationFormat.BatchedExperts
|
||||
@classmethod
|
||||
def is_batched(cls) -> bool:
|
||||
return cls.activation_format() == mk.FusedMoEActivationFormat.BatchedExperts
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
@@ -189,7 +183,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
assert w1.size(0) == num_experts
|
||||
assert w2.size(0) == num_experts
|
||||
|
||||
if not self.is_batched:
|
||||
if not self.is_batched():
|
||||
num_tokens = a1.size(0)
|
||||
assert topk_ids.size(0) == num_tokens
|
||||
else:
|
||||
@@ -201,7 +195,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
|
||||
def get_buffer_metas(self, M: int, topk: int, activation: MoEActivation):
|
||||
num_experts = self.num_experts
|
||||
N = self.layer.intermediate_size
|
||||
N = self.layer.intermediate_size_per_partition
|
||||
K = self.layer.hidden_size
|
||||
assert isinstance(num_experts, int)
|
||||
assert isinstance(N, int)
|
||||
@@ -218,7 +212,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
# The output must be derived from workspace1.
|
||||
|
||||
output_shape: tuple[int, ...]
|
||||
if self.is_batched:
|
||||
if self.is_batched():
|
||||
max_num_tokens = self.max_num_tokens
|
||||
num_dispatchers = self.num_dispatchers
|
||||
assert max_num_tokens is not None and num_dispatchers is not None
|
||||
@@ -227,7 +221,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
output_shape = (num_experts, max_num_tokens * num_dispatchers, K)
|
||||
else:
|
||||
input_shape_m = M
|
||||
if self.humming_gemm_type != HummingGemmType.INDEXED:
|
||||
if self.humming_gemm_type() != HummingGemmType.INDEXED:
|
||||
input_shape_m = M * topk
|
||||
real_shape_m = M * topk
|
||||
output_shape = (M, K)
|
||||
@@ -262,7 +256,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
"dtype": torch_dtype_map[a_dtype],
|
||||
},
|
||||
"down_output": {
|
||||
"shape": output_shape if self.is_batched else (real_shape_m, K),
|
||||
"shape": output_shape if self.is_batched() else (real_shape_m, K),
|
||||
"dtype": torch_dtype_map[c_dtype],
|
||||
},
|
||||
"output": {
|
||||
@@ -288,7 +282,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
]
|
||||
|
||||
# batched moe use down_output as output
|
||||
if not self.is_batched:
|
||||
if not self.is_batched():
|
||||
required_buffers.append("output")
|
||||
|
||||
return buffer_metas, required_buffers
|
||||
@@ -308,7 +302,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
else:
|
||||
workspace2_nbytes = max(workspace2_nbytes, nbytes)
|
||||
|
||||
output_key = "down_output" if self.is_batched else "output"
|
||||
output_key = "down_output" if self.is_batched() else "output"
|
||||
output_shape = buffer_metas[output_key]["shape"]
|
||||
|
||||
return (workspace1_nbytes // 2,), (workspace2_nbytes // 2,), output_shape
|
||||
@@ -395,6 +389,33 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
):
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
def is_supported_config(
|
||||
cls: type[mk.FusedMoEExperts],
|
||||
moe_config: FusedMoEConfig,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
activation_format: mk.FusedMoEActivationFormat,
|
||||
) -> tuple[bool, str | None]:
|
||||
if activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
|
||||
supported = cls.activation_format() == activation_format
|
||||
reason = "activation_format mismatched"
|
||||
elif activation_format == mk.FusedMoEActivationFormat.Standard:
|
||||
if cls.activation_format() != mk.FusedMoEActivationFormat.Standard:
|
||||
supported = False
|
||||
reason = "activation_format mismatched"
|
||||
else:
|
||||
assert hasattr(cls, "humming_gemm_type")
|
||||
gemm_type = cls.humming_gemm_type().value.lower()
|
||||
preferred_gemm_type = get_humming_moe_gemm_type().lower()
|
||||
supported = preferred_gemm_type == gemm_type
|
||||
reason = "preferred gemm type mismatched"
|
||||
else:
|
||||
supported = False
|
||||
reason = "unsupported activation_format"
|
||||
|
||||
return supported, None if supported else reason
|
||||
|
||||
|
||||
class HummingIndexedExperts(HummingExpertsBase):
|
||||
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
|
||||
@@ -404,8 +425,8 @@ class HummingIndexedExperts(HummingExpertsBase):
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
|
||||
@property
|
||||
def humming_gemm_type(self) -> HummingGemmType:
|
||||
@staticmethod
|
||||
def humming_gemm_type() -> HummingGemmType:
|
||||
return HummingGemmType.INDEXED
|
||||
|
||||
def prepare_humming_moe_kwargs(
|
||||
@@ -526,8 +547,8 @@ class HummingGroupedExperts(HummingExpertsBase):
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
|
||||
@property
|
||||
def humming_gemm_type(self) -> HummingGemmType:
|
||||
@staticmethod
|
||||
def humming_gemm_type() -> HummingGemmType:
|
||||
return HummingGemmType.GROUPED_CONTIGUOUS
|
||||
|
||||
def main_apply(
|
||||
@@ -619,8 +640,8 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.BatchedExperts
|
||||
|
||||
@property
|
||||
def humming_gemm_type(self) -> HummingGemmType:
|
||||
@staticmethod
|
||||
def humming_gemm_type() -> HummingGemmType:
|
||||
return HummingGemmType.GROUPED_MASKED
|
||||
|
||||
def main_apply(
|
||||
|
||||
@@ -1103,9 +1103,6 @@ class FusedMoE(PluggableLayer):
|
||||
return_success: bool = False,
|
||||
) -> bool | None:
|
||||
quant_config_name = self.quant_config and self.quant_config.get_name()
|
||||
if quant_config_name == "humming":
|
||||
assert hasattr(self.quant_method, "weight_schema")
|
||||
quant_config_name = self.quant_method.weight_schema.quant_method
|
||||
if quant_config_name == "gpt_oss_mxfp4":
|
||||
# (FIXME) for gpt-oss all experts are combined
|
||||
if "bias" in weight_name:
|
||||
|
||||
@@ -68,6 +68,8 @@ class Mxfp4MoeBackend(Enum):
|
||||
XPU = "XPU"
|
||||
# Emulation
|
||||
EMULATION = "EMULATION"
|
||||
# Humming
|
||||
HUMMING = "HUMMING"
|
||||
|
||||
|
||||
# Backends that share the same TRTLLM weight format
|
||||
@@ -130,6 +132,19 @@ def backend_to_kernel_cls(
|
||||
|
||||
return [UnfusedOAITritonExperts]
|
||||
|
||||
elif backend == Mxfp4MoeBackend.HUMMING:
|
||||
from vllm.model_executor.layers.fused_moe.fused_humming_moe import (
|
||||
BatchedHummingGroupedExperts,
|
||||
HummingGroupedExperts,
|
||||
HummingIndexedExperts,
|
||||
)
|
||||
|
||||
return [
|
||||
BatchedHummingGroupedExperts,
|
||||
HummingGroupedExperts,
|
||||
HummingIndexedExperts,
|
||||
]
|
||||
|
||||
elif backend == Mxfp4MoeBackend.MARLIN:
|
||||
from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
|
||||
MarlinExperts,
|
||||
@@ -177,6 +192,7 @@ def map_mxfp4_backend(runner_backend: MoEBackend) -> Mxfp4MoeBackend:
|
||||
"flashinfer_cutlass_afp8": Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
|
||||
"triton": Mxfp4MoeBackend.TRITON,
|
||||
"triton_unfused": Mxfp4MoeBackend.TRITON_UNFUSED,
|
||||
"humming": Mxfp4MoeBackend.HUMMING,
|
||||
"marlin": Mxfp4MoeBackend.MARLIN,
|
||||
"aiter": Mxfp4MoeBackend.AITER,
|
||||
"xpu": Mxfp4MoeBackend.XPU,
|
||||
@@ -573,7 +589,21 @@ def convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
|
||||
|
||||
sf_block_size = 32 # mxfp4 block size
|
||||
|
||||
if mxfp4_backend in (
|
||||
if mxfp4_backend == Mxfp4MoeBackend.HUMMING:
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
prepare_humming_moe_layer,
|
||||
)
|
||||
|
||||
prepare_humming_moe_layer(layer, {"quant_method": "gpt_oss_mxfp4"})
|
||||
return (
|
||||
layer.w13_weight,
|
||||
layer.w2_weight,
|
||||
layer.w13_weight_scale,
|
||||
layer.w2_weight_scale,
|
||||
getattr(layer, "w13_bias", None),
|
||||
getattr(layer, "w2_bias", None),
|
||||
)
|
||||
elif mxfp4_backend in (
|
||||
Mxfp4MoeBackend.MARLIN,
|
||||
Mxfp4MoeBackend.BATCHED_MARLIN,
|
||||
):
|
||||
@@ -970,6 +1000,21 @@ def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
w2_bias,
|
||||
)
|
||||
|
||||
if mxfp4_backend == Mxfp4MoeBackend.HUMMING:
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
prepare_humming_moe_layer,
|
||||
)
|
||||
|
||||
prepare_humming_moe_layer(layer, {"quant_method": "mxfp4"})
|
||||
return (
|
||||
layer.w13_weight,
|
||||
layer.w2_weight,
|
||||
layer.w13_weight_scale,
|
||||
layer.w2_weight_scale,
|
||||
getattr(layer, "w13_bias", None),
|
||||
getattr(layer, "w2_bias", None),
|
||||
)
|
||||
|
||||
if mxfp4_backend in (Mxfp4MoeBackend.MARLIN, Mxfp4MoeBackend.BATCHED_MARLIN):
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
|
||||
prepare_moe_mxfp4_layer_for_marlin,
|
||||
@@ -1175,6 +1220,7 @@ def make_mxfp4_moe_quant_config(
|
||||
swiglu_limit: float | None = None,
|
||||
w1_bias: torch.Tensor | None = None,
|
||||
w2_bias: torch.Tensor | None = None,
|
||||
layer: torch.nn.Module | None = None,
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
"""Create a FusedMoEQuantConfig for the given MXFP4 backend."""
|
||||
if mxfp4_backend == Mxfp4MoeBackend.DEEPGEMM_MXFP4:
|
||||
@@ -1234,6 +1280,14 @@ def make_mxfp4_moe_quant_config(
|
||||
gemm1_beta=gemm1_beta,
|
||||
gemm1_clamp_limit=swiglu_limit,
|
||||
)
|
||||
elif mxfp4_backend == Mxfp4MoeBackend.HUMMING:
|
||||
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
get_humming_moe_quant_config,
|
||||
)
|
||||
|
||||
assert isinstance(layer, FusedMoE)
|
||||
return get_humming_moe_quant_config(layer)
|
||||
else:
|
||||
return ocp_mx_moe_quant_config(
|
||||
quant_dtype="mxfp4",
|
||||
@@ -1254,6 +1308,7 @@ def make_mxfp4_moe_kernel(
|
||||
mxfp4_backend: Mxfp4MoeBackend,
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
shared_experts: torch.nn.Module | None = None,
|
||||
layer: torch.nn.Module | None = None,
|
||||
) -> mk.FusedMoEKernel:
|
||||
"""Create a FusedMoEKernel for the given MXFP4 backend."""
|
||||
is_monolithic = issubclass(experts_cls, mk.FusedMoEExpertsMonolithic)
|
||||
@@ -1269,6 +1324,11 @@ def make_mxfp4_moe_kernel(
|
||||
|
||||
logger.info_once("Using %s", prepare_finalize.__class__.__name__)
|
||||
|
||||
extra_kwargs = {}
|
||||
if mxfp4_backend == Mxfp4MoeBackend.HUMMING:
|
||||
assert layer is not None
|
||||
extra_kwargs["layer"] = layer
|
||||
|
||||
# Create Experts.
|
||||
if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
|
||||
max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
|
||||
@@ -1278,11 +1338,13 @@ def make_mxfp4_moe_kernel(
|
||||
quant_config=moe_quant_config,
|
||||
max_num_tokens=max_num_tokens,
|
||||
num_dispatchers=prepare_finalize.num_dispatchers(),
|
||||
**extra_kwargs,
|
||||
)
|
||||
else:
|
||||
experts = experts_cls(
|
||||
moe_config=moe_config,
|
||||
quant_config=moe_quant_config,
|
||||
**extra_kwargs,
|
||||
)
|
||||
|
||||
kernel = mk.FusedMoEKernel(
|
||||
|
||||
@@ -9,11 +9,9 @@ import regex as re
|
||||
import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEQuantConfig,
|
||||
FusedMoEQuantDesc,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.layer import (
|
||||
FusedMoE,
|
||||
@@ -32,7 +30,6 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.parameter import (
|
||||
BasevLLMParameter,
|
||||
@@ -215,6 +212,15 @@ class HummingConfig(QuantizationConfig):
|
||||
def override_quantization_method(
|
||||
cls, hf_quant_cfg, user_quant, hf_config=None
|
||||
) -> QuantizationMethods | None:
|
||||
if user_quant == "humming" and hf_config is not None:
|
||||
model_type = hf_config.model_type
|
||||
quant_method = hf_quant_cfg.get("quant_method", None)
|
||||
if model_type == "gpt_oss" and quant_method == "mxfp4":
|
||||
msg = (
|
||||
"For gpt-oss model, use '--moe-backend humming' "
|
||||
"instead of '--quantization humming'."
|
||||
)
|
||||
raise ValueError(msg)
|
||||
return "humming" if user_quant == "humming" else None
|
||||
|
||||
def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):
|
||||
@@ -299,8 +305,6 @@ class HummingConfig(QuantizationConfig):
|
||||
force_weight_schema = schema
|
||||
|
||||
if weight_schema is not None:
|
||||
if weight_schema.quant_method == "gpt_oss_mxfp4" and layer_type != "moe":
|
||||
return None
|
||||
input_schema = None
|
||||
force_input_schema = None
|
||||
|
||||
@@ -335,12 +339,6 @@ class HummingConfig(QuantizationConfig):
|
||||
elif isinstance(layer, LinearBase):
|
||||
layer_type = "linear"
|
||||
|
||||
# TODO: remove this after humming moe backend is ready
|
||||
quant_method = self.full_config.get("quant_method", None)
|
||||
moe_activation = getattr(layer, "activation", None)
|
||||
if quant_method == "mxfp4" and moe_activation == MoEActivation.SWIGLUOAI:
|
||||
self.full_config["quan_method"] = "gpt_oss_mxfp4"
|
||||
|
||||
quant_config = self.get_quant_config_for_layer(prefix, layer_type)
|
||||
if quant_config is None:
|
||||
if isinstance(layer, FusedMoE):
|
||||
@@ -760,62 +758,18 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
layer.register_buffer("locks", locks)
|
||||
|
||||
def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig:
|
||||
self.process_weights_after_loading(layer)
|
||||
|
||||
input_schema = self.input_schemas["w13"]
|
||||
weight_schema = self.weight_schemas["w13"]
|
||||
|
||||
a_dtype = input_schema.a_dtype
|
||||
if a_dtype is None or a_dtype.num_bits == 16:
|
||||
a_quant_desc = FusedMoEQuantDesc(dtype=None)
|
||||
else:
|
||||
shape = GroupShape(row=1, col=-1)
|
||||
a_quant_desc = FusedMoEQuantDesc(dtype=str(a_dtype), shape=shape)
|
||||
|
||||
weight_scale_group_size = weight_schema.weight_scale_group_size
|
||||
weight_scale_group_size_n = weight_schema.weight_scale_group_size_n
|
||||
weight_group_shape: tuple[int, ...] = ()
|
||||
if weight_scale_group_size_n > 1:
|
||||
weight_group_shape = GroupShape(
|
||||
row=weight_scale_group_size,
|
||||
col=weight_scale_group_size_n,
|
||||
)
|
||||
elif weight_scale_group_size == 0:
|
||||
weight_group_shape = GroupShape(row=-1, col=1)
|
||||
else:
|
||||
weight_group_shape = GroupShape(row=weight_scale_group_size, col=1)
|
||||
|
||||
w1_quant_desc = FusedMoEQuantDesc(
|
||||
dtype=str(weight_schema.b_dtype),
|
||||
shape=weight_group_shape,
|
||||
scale=getattr(layer, "w13_weight_scale", None),
|
||||
alpha_or_gscale=getattr(layer, "w13_global_scale", None),
|
||||
zp=getattr(layer, "w13_zero_point", None),
|
||||
bias=getattr(layer, "w13_bias", None),
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
get_humming_moe_quant_config,
|
||||
)
|
||||
|
||||
w2_quant_desc = FusedMoEQuantDesc(
|
||||
dtype=str(weight_schema.b_dtype),
|
||||
shape=weight_group_shape,
|
||||
scale=getattr(layer, "w2_weight_scale", None),
|
||||
alpha_or_gscale=getattr(layer, "w2_global_scale", None),
|
||||
zp=getattr(layer, "w2_zero_point", None),
|
||||
bias=getattr(layer, "w2_bias", None),
|
||||
)
|
||||
|
||||
return FusedMoEQuantConfig(
|
||||
_a1=a_quant_desc,
|
||||
_a2=a_quant_desc,
|
||||
_w1=w1_quant_desc,
|
||||
_w2=w2_quant_desc,
|
||||
)
|
||||
return get_humming_moe_quant_config(layer)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
if getattr(self, "processed", False):
|
||||
return
|
||||
self.processed = True
|
||||
self.weight_schemas = {}
|
||||
self.input_schemas = {}
|
||||
layer.weight_schemas = {}
|
||||
layer.input_schemas = {}
|
||||
for sublayer_name, configs in layer.sublayer_configs.items():
|
||||
input_schema = self.input_schema
|
||||
weight_schema = self.weight_schema
|
||||
@@ -858,8 +812,8 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
param = torch.nn.Parameter(tensor, requires_grad=False)
|
||||
setattr(layer, name, param)
|
||||
|
||||
self.weight_schemas[sublayer_name] = weight_schema
|
||||
self.input_schemas[sublayer_name] = input_schema
|
||||
layer.weight_schemas[sublayer_name] = weight_schema
|
||||
layer.input_schemas[sublayer_name] = input_schema
|
||||
|
||||
# force requant (origin quant setting -> fp16/bf16 -> new_quant setting)
|
||||
assert isinstance(weight_schema, HummingWeightSchema)
|
||||
@@ -913,10 +867,11 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
|
||||
# use moe modular
|
||||
experts: HummingIndexedExperts | HummingGroupedExperts
|
||||
assert self.moe_quant_config is not None
|
||||
if get_humming_moe_gemm_type() == "indexed":
|
||||
experts = HummingIndexedExperts(layer, self)
|
||||
experts = HummingIndexedExperts(layer, self.moe, self.moe_quant_config)
|
||||
else:
|
||||
experts = HummingGroupedExperts(layer, self)
|
||||
experts = HummingGroupedExperts(layer, self.moe, self.moe_quant_config)
|
||||
self.experts = experts
|
||||
|
||||
def select_gemm_impl(
|
||||
@@ -927,12 +882,19 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
from vllm.model_executor.layers.fused_moe import modular_kernel as mk
|
||||
|
||||
activation_format = prepare_finalize.activation_format
|
||||
assert self.moe_quant_config is not None
|
||||
if activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
|
||||
return BatchedHummingGroupedExperts(layer, self, prepare_finalize)
|
||||
return BatchedHummingGroupedExperts(
|
||||
layer=layer,
|
||||
moe_config=self.moe,
|
||||
quant_config=self.moe_quant_config,
|
||||
max_num_tokens=prepare_finalize.max_num_tokens_per_rank(),
|
||||
num_dispatchers=prepare_finalize.num_dispatchers(),
|
||||
)
|
||||
elif get_humming_moe_gemm_type() == "indexed":
|
||||
return HummingIndexedExperts(layer, self, prepare_finalize)
|
||||
return HummingIndexedExperts(layer, self.moe, self.moe_quant_config)
|
||||
else:
|
||||
return HummingGroupedExperts(layer, self, prepare_finalize)
|
||||
return HummingGroupedExperts(layer, self.moe, self.moe_quant_config)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
|
||||
@@ -366,6 +366,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._maybe_init_expert_routing_tables(),
|
||||
shared_experts=layer.shared_experts,
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer):
|
||||
@@ -404,6 +405,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
gemm1_alpha=1.702,
|
||||
gemm1_beta=1.0,
|
||||
swiglu_limit=7.0,
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def select_gemm_impl(
|
||||
@@ -692,6 +694,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._maybe_init_expert_routing_tables(),
|
||||
shared_experts=layer.shared_experts,
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer):
|
||||
@@ -729,6 +732,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
w1_bias=w1_bias,
|
||||
w2_bias=w2_bias,
|
||||
swiglu_limit=swiglu_limit,
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def select_gemm_impl(
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
|
||||
moe_align_block_size,
|
||||
)
|
||||
|
||||
|
||||
def humming_moe_align(
|
||||
configs: list[int],
|
||||
topk_ids: torch.Tensor,
|
||||
num_experts: int,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
assert len(configs) > 0 and len(configs) % 3 == 0
|
||||
# NOTE: we choose moe_block_size based on
|
||||
# num_tokens * top_k (= topk_ids.nelement())
|
||||
shape_m = topk_ids.nelement()
|
||||
|
||||
for i in range(len(configs) // 3):
|
||||
if shape_m > configs[i * 3] and shape_m <= configs[i * 3 + 1]:
|
||||
block_size = configs[i * 3 + 2]
|
||||
break
|
||||
else:
|
||||
raise ValueError(f"Could not find a matching block_size for shape_m={shape_m}")
|
||||
|
||||
return moe_align_block_size(
|
||||
topk_ids=topk_ids,
|
||||
block_size=block_size,
|
||||
num_experts=num_experts,
|
||||
expert_map=expert_map,
|
||||
pad_sorted_ids=False,
|
||||
ignore_invalid_experts=True,
|
||||
)
|
||||
@@ -0,0 +1,214 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
from humming.layer import HummingInputSchema, HummingMethod
|
||||
from humming.schema import BaseWeightSchema
|
||||
|
||||
from vllm import envs
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
FusedMoEQuantDesc,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||
|
||||
|
||||
def humming_is_layer_skipped(config: dict[str, Any], prefix: str):
|
||||
if not config:
|
||||
return True
|
||||
|
||||
keys = ["ignored_layers", "ignore", "modules_to_not_convert"]
|
||||
ignored_layers: list[str] = []
|
||||
for key in keys:
|
||||
ignored_layers = config.get(key, []) or []
|
||||
if not ignored_layers:
|
||||
break
|
||||
|
||||
if any(module_name in prefix for module_name in ignored_layers):
|
||||
return True
|
||||
if "lm_head" in prefix:
|
||||
return True
|
||||
|
||||
for regex in config.get("dynamic", {}):
|
||||
if regex[:1] != "-":
|
||||
continue
|
||||
if re.match(regex[2:], prefix):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def prepare_humming_layer(layer: LinearBase, quant_config: dict):
|
||||
weight_schema = BaseWeightSchema.from_config(quant_config)
|
||||
input_schema = HummingInputSchema()
|
||||
|
||||
shape_k_stacks = [layer.input_size_per_partition]
|
||||
shape_n_stacks = layer.output_partition_sizes
|
||||
|
||||
# Step 1: convert weight to humming standard format
|
||||
weight_schema, tensors = weight_schema.convert_humming(
|
||||
tensors=layer.named_parameters(),
|
||||
shape_n_stacks=shape_n_stacks,
|
||||
shape_k_stacks=shape_k_stacks,
|
||||
param_dtype=layer.params_dtype,
|
||||
)
|
||||
|
||||
layer.weight_schema = weight_schema
|
||||
|
||||
for name, _ in list(layer.named_parameters()):
|
||||
delattr(layer, name)
|
||||
|
||||
for name, tensor in tensors.items():
|
||||
param = torch.nn.Parameter(tensor, requires_grad=False)
|
||||
setattr(layer, name, param)
|
||||
|
||||
# Step 2: transform weight (humming standard format) for forwarding
|
||||
HummingMethod.prepare_layer_meta(
|
||||
layer=layer,
|
||||
shape_n=layer.output_partition_sizes_sum,
|
||||
shape_k=layer.input_size_per_partition,
|
||||
weight_schema=weight_schema,
|
||||
input_schema=input_schema,
|
||||
pad_n_to_multiple=256,
|
||||
pad_k_to_multiple=128,
|
||||
has_bias=layer.has_bias,
|
||||
torch_dtype=layer.param_dtype,
|
||||
)
|
||||
|
||||
HummingMethod.transform_humming_layer(layer)
|
||||
|
||||
|
||||
def prepare_humming_moe_layer(layer: FusedMoE, quant_config: dict):
|
||||
weight_schema = BaseWeightSchema.from_config(quant_config)
|
||||
input_quant_config = envs.VLLM_HUMMING_INPUT_QUANT_CONFIG or {}
|
||||
if humming_is_layer_skipped(input_quant_config, layer.layer_name):
|
||||
input_schema = HummingInputSchema()
|
||||
else:
|
||||
# TODO: read input_quant_config from quant_config
|
||||
input_schema = HummingInputSchema.from_config(input_quant_config)
|
||||
|
||||
is_gated = layer.activation.is_gated
|
||||
shape_config = {
|
||||
"w13": (
|
||||
layer.moe_config.intermediate_size_per_partition * 2,
|
||||
layer.moe_config.hidden_dim,
|
||||
),
|
||||
"w2": (
|
||||
layer.moe_config.hidden_dim,
|
||||
layer.moe_config.intermediate_size_per_partition * (1 if is_gated else 2),
|
||||
),
|
||||
}
|
||||
|
||||
layer.weight_schemas = {}
|
||||
layer.input_schemas = {}
|
||||
|
||||
for sublayer_name in shape_config:
|
||||
# Step 1: convert weight to humming standard format
|
||||
tensors: dict[str, torch.Tensor] = dict(
|
||||
(key.removeprefix(sublayer_name + "_"), value)
|
||||
for key, value in layer.state_dict().items()
|
||||
if key.startswith(sublayer_name + "_")
|
||||
)
|
||||
|
||||
shape_n, shape_k = shape_config[sublayer_name]
|
||||
shape_n_stacks = [shape_n]
|
||||
shape_k_stacks = [shape_k]
|
||||
if sublayer_name == "w13":
|
||||
shape_n_stacks = [shape_n // 2] * 2
|
||||
|
||||
weight_schema_new, tensors = weight_schema.convert_humming(
|
||||
tensors=tensors,
|
||||
shape_n_stacks=shape_n_stacks,
|
||||
shape_k_stacks=shape_k_stacks,
|
||||
num_experts=layer.local_num_experts,
|
||||
param_dtype=layer.params_dtype,
|
||||
)
|
||||
|
||||
layer.weight_schemas[sublayer_name] = weight_schema_new
|
||||
layer.input_schemas[sublayer_name] = input_schema
|
||||
|
||||
for name, _ in list(layer.named_parameters()):
|
||||
if not name.startswith(sublayer_name + "_"):
|
||||
continue
|
||||
delattr(layer, name)
|
||||
|
||||
for name, tensor in tensors.items():
|
||||
name = f"{sublayer_name}_{name}"
|
||||
param = torch.nn.Parameter(tensor, requires_grad=False)
|
||||
setattr(layer, name, param)
|
||||
|
||||
# Step 2: transform weight (humming standard format) for forwarding
|
||||
HummingMethod.prepare_layer_meta(
|
||||
layer=layer,
|
||||
shape_n=shape_n,
|
||||
shape_k=shape_k,
|
||||
pad_n_to_multiple=256,
|
||||
pad_k_to_multiple=128,
|
||||
input_schema=input_schema,
|
||||
weight_schema=weight_schema_new,
|
||||
has_bias=layer.moe_config.has_bias,
|
||||
num_experts=layer.num_experts,
|
||||
torch_dtype=layer.params_dtype,
|
||||
sublayer_name=sublayer_name,
|
||||
)
|
||||
|
||||
HummingMethod.transform_humming_layer(layer, sublayer_name=sublayer_name)
|
||||
|
||||
if not hasattr(layer, "locks"):
|
||||
device = layer.w13_weight.device
|
||||
locks = torch.zeros(1024, dtype=torch.int32, device=device)
|
||||
layer.register_buffer("locks", locks)
|
||||
|
||||
|
||||
def get_humming_moe_quant_config(layer: FusedMoE):
|
||||
input_schema = layer.input_schemas["w13"]
|
||||
weight_schema = layer.weight_schemas["w13"]
|
||||
|
||||
a_dtype = input_schema.a_dtype
|
||||
if a_dtype is None or a_dtype.num_bits == 16:
|
||||
a_quant_desc = FusedMoEQuantDesc(dtype=None)
|
||||
else:
|
||||
shape = GroupShape(row=1, col=-1)
|
||||
a_quant_desc = FusedMoEQuantDesc(dtype=str(a_dtype), shape=shape)
|
||||
|
||||
weight_scale_group_size = weight_schema.weight_scale_group_size
|
||||
weight_scale_group_size_n = weight_schema.weight_scale_group_size_n
|
||||
weight_group_shape: tuple[int, ...] = ()
|
||||
if weight_scale_group_size_n > 1:
|
||||
weight_group_shape = GroupShape(
|
||||
row=weight_scale_group_size,
|
||||
col=weight_scale_group_size_n,
|
||||
)
|
||||
elif weight_scale_group_size == 0:
|
||||
weight_group_shape = GroupShape(row=-1, col=1)
|
||||
else:
|
||||
weight_group_shape = GroupShape(row=weight_scale_group_size, col=1)
|
||||
|
||||
w1_quant_desc = FusedMoEQuantDesc(
|
||||
dtype=str(weight_schema.b_dtype),
|
||||
shape=weight_group_shape,
|
||||
scale=getattr(layer, "w13_weight_scale", None),
|
||||
alpha_or_gscale=getattr(layer, "w13_global_scale", None),
|
||||
zp=getattr(layer, "w13_zero_point", None),
|
||||
bias=getattr(layer, "w13_bias", None),
|
||||
)
|
||||
|
||||
w2_quant_desc = FusedMoEQuantDesc(
|
||||
dtype=str(weight_schema.b_dtype),
|
||||
shape=weight_group_shape,
|
||||
scale=getattr(layer, "w2_weight_scale", None),
|
||||
alpha_or_gscale=getattr(layer, "w2_global_scale", None),
|
||||
zp=getattr(layer, "w2_zero_point", None),
|
||||
bias=getattr(layer, "w2_bias", None),
|
||||
)
|
||||
|
||||
return FusedMoEQuantConfig(
|
||||
_a1=a_quant_desc,
|
||||
_a2=a_quant_desc,
|
||||
_w1=w1_quant_desc,
|
||||
_w2=w2_quant_desc,
|
||||
)
|
||||
Reference in New Issue
Block a user