[Quantization] add humming moe backend to all dense/moe oracles (#41652)
Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -38,15 +38,20 @@ from vllm.model_executor.layers.fused_moe.utils import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
kInt4Static,
|
||||
kInt8DynamicTokenSym,
|
||||
kInt8Static,
|
||||
kInt8StaticChannelSym,
|
||||
kMxfp4Dynamic,
|
||||
kMxfp4Static,
|
||||
kMxfp8Dynamic,
|
||||
kMxfp8Static,
|
||||
kNvfp4Dynamic,
|
||||
kNvfp4Static,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
@@ -61,9 +66,9 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def get_humming_moe_gemm_type() -> str | None:
|
||||
def get_humming_moe_gemm_type() -> str:
|
||||
env_gemm_type: str | None = envs.VLLM_HUMMING_MOE_GEMM_TYPE
|
||||
gemm_type = None
|
||||
gemm_type = "indexed"
|
||||
if env_gemm_type is not None:
|
||||
env_gemm_type = env_gemm_type.lower()
|
||||
if env_gemm_type == "indexed":
|
||||
@@ -87,7 +92,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
num_dispatchers: int | None = None,
|
||||
):
|
||||
self.layer = layer
|
||||
self.num_experts = self.layer.num_experts
|
||||
self.num_experts = self.layer.local_num_experts
|
||||
self.global_num_experts = self.layer.global_num_experts
|
||||
self.init_humming_moe()
|
||||
|
||||
@@ -186,6 +191,24 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
(kInt4Static, kFp8DynamicTokenSym),
|
||||
(kInt8Static, None),
|
||||
(kInt8Static, kFp8DynamicTokenSym),
|
||||
# Checkpoint-driven (weight, activation) pairs the dense/MoE oracles
|
||||
# pass. Humming defers input quant (see expects_unquantized_inputs),
|
||||
# so the activation key does not constrain support.
|
||||
# fp8 (compressed-tensors / native / modelopt)
|
||||
(kFp8StaticChannelSym, kFp8StaticTensorSym),
|
||||
(kFp8StaticChannelSym, kFp8Dynamic128Sym),
|
||||
(kFp8StaticTensorSym, None),
|
||||
(kFp8StaticTensorSym, kFp8DynamicTokenSym),
|
||||
(kFp8StaticTensorSym, kFp8StaticTensorSym),
|
||||
(kFp8StaticTensorSym, kFp8Dynamic128Sym),
|
||||
(kFp8Static128BlockSym, kFp8Dynamic128Sym),
|
||||
# int8 (compressed-tensors w8a8 / experts_int8)
|
||||
(kInt8StaticChannelSym, None),
|
||||
(kInt8StaticChannelSym, kInt8DynamicTokenSym),
|
||||
# nvfp4 (compressed-tensors / modelopt / quark)
|
||||
(kNvfp4Static, kNvfp4Dynamic),
|
||||
# mxfp8 (compressed-tensors / modelopt / online)
|
||||
(kMxfp8Static, kMxfp8Dynamic),
|
||||
]
|
||||
return (weight_key, activation_key) in SUPPORTED_W_A
|
||||
|
||||
@@ -463,13 +486,12 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
assert hasattr(cls, "humming_gemm_type")
|
||||
gemm_type = cls.humming_gemm_type().value.lower()
|
||||
preferred_gemm_type = get_humming_moe_gemm_type()
|
||||
if preferred_gemm_type is not None:
|
||||
supported = preferred_gemm_type.lower() == gemm_type
|
||||
if not supported:
|
||||
reason = (
|
||||
f"preferred gemm type {preferred_gemm_type} != "
|
||||
f"supported gemm type {gemm_type}"
|
||||
)
|
||||
supported = preferred_gemm_type.lower() == gemm_type
|
||||
if not supported:
|
||||
reason = (
|
||||
f"preferred gemm type {preferred_gemm_type} != "
|
||||
f"supported gemm type {gemm_type}"
|
||||
)
|
||||
|
||||
return supported, reason
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
@@ -45,6 +46,7 @@ class Fp8MoeBackend(Enum):
|
||||
DEEPGEMM = "DEEPGEMM"
|
||||
BATCHED_DEEPGEMM = "BATCHED_DEEPGEMM"
|
||||
MARLIN = "MARLIN"
|
||||
HUMMING = "HUMMING"
|
||||
TRITON = "TRITON"
|
||||
BATCHED_TRITON = "BATCHED_TRITON"
|
||||
AITER = "AITER"
|
||||
@@ -83,6 +85,7 @@ def _get_priority_backends(
|
||||
Fp8MoeBackend.VLLM_CUTLASS,
|
||||
Fp8MoeBackend.TRITON,
|
||||
Fp8MoeBackend.MARLIN,
|
||||
Fp8MoeBackend.HUMMING,
|
||||
Fp8MoeBackend.BATCHED_DEEPGEMM,
|
||||
Fp8MoeBackend.BATCHED_VLLM_CUTLASS,
|
||||
Fp8MoeBackend.BATCHED_TRITON,
|
||||
@@ -162,6 +165,19 @@ def backend_to_kernel_cls(
|
||||
|
||||
return [BatchedDeepGemmExperts]
|
||||
|
||||
elif backend == Fp8MoeBackend.HUMMING:
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
BatchedHummingGroupedExperts,
|
||||
HummingGroupedExperts,
|
||||
HummingIndexedExperts,
|
||||
)
|
||||
|
||||
return [
|
||||
BatchedHummingGroupedExperts,
|
||||
HummingGroupedExperts,
|
||||
HummingIndexedExperts,
|
||||
]
|
||||
|
||||
elif backend == Fp8MoeBackend.MARLIN:
|
||||
from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
|
||||
MarlinExperts,
|
||||
@@ -240,6 +256,7 @@ def map_fp8_backend(runner_backend: MoEBackend) -> Fp8MoeBackend:
|
||||
"flashinfer_trtllm": Fp8MoeBackend.FLASHINFER_TRTLLM,
|
||||
"flashinfer_cutlass": Fp8MoeBackend.FLASHINFER_CUTLASS,
|
||||
"marlin": Fp8MoeBackend.MARLIN,
|
||||
"humming": Fp8MoeBackend.HUMMING,
|
||||
"aiter": Fp8MoeBackend.AITER,
|
||||
"hpc": Fp8MoeBackend.HPC,
|
||||
}
|
||||
@@ -402,6 +419,41 @@ def select_fp8_moe_backend(
|
||||
return Fp8MoeBackend.NONE, None
|
||||
|
||||
|
||||
def _humming_fp8_weight_schema(
|
||||
layer: RoutedExperts, weight: torch.Tensor, weight_scale: torch.Tensor
|
||||
) -> dict[str, Any]:
|
||||
"""Build the humming weight schema from the canonical on-device fp8/mxfp8
|
||||
tensors (scale dtype/shape, block size), not the producing quant method."""
|
||||
# mxfp8: e8m0 group-32 scales (stored as uint8 bytes or e8m0). humming has
|
||||
# no compressed-tensors mxfp8 loader; its modelopt schema fits both sources.
|
||||
if weight_scale.dtype in (torch.uint8, torch.float8_e8m0fnu):
|
||||
return {"quant_method": "modelopt", "quant_algo": "mxfp8"}
|
||||
|
||||
if hasattr(layer, "w13_weight_scale_inv"):
|
||||
assert hasattr(layer, "weight_block_size")
|
||||
return {"quant_method": "fp8", "weight_block_size": layer.weight_block_size}
|
||||
|
||||
# fp8 (e4m3): recover the strategy from the scale layout (block from
|
||||
# weight_block_size, else channel vs tensor by per-expert scale count).
|
||||
config: dict[str, Any] = {
|
||||
"quant_method": "compressed-tensors",
|
||||
"format": "float-quantized",
|
||||
"type": "float",
|
||||
"num_bits": 8,
|
||||
"symmetric": True,
|
||||
}
|
||||
weight_block_size = getattr(layer, "weight_block_size", None)
|
||||
num_experts, num_output = weight.shape[0], weight.shape[-2]
|
||||
if weight_block_size is not None:
|
||||
config["strategy"] = "block"
|
||||
config["block_structure"] = list(weight_block_size)
|
||||
elif weight_scale.numel() >= num_experts * num_output:
|
||||
config["strategy"] = "channel"
|
||||
else:
|
||||
config["strategy"] = "tensor"
|
||||
return config
|
||||
|
||||
|
||||
def convert_to_fp8_moe_kernel_format(
|
||||
fp8_backend: Fp8MoeBackend,
|
||||
# TODO(bnell): replace layer with weight_block_size
|
||||
@@ -429,6 +481,18 @@ def convert_to_fp8_moe_kernel_format(
|
||||
w13, w2, w13_scale, w2_scale = rocm_aiter_ops.shuffle_mxfp8_moe_weights(
|
||||
w13, w2, w13_scale, w2_scale
|
||||
)
|
||||
elif fp8_backend == Fp8MoeBackend.HUMMING:
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
convert_to_humming_moe_kernel_format,
|
||||
)
|
||||
|
||||
convert_to_humming_moe_kernel_format(
|
||||
layer, quant_config=_humming_fp8_weight_schema(layer, w13, w13_scale)
|
||||
)
|
||||
w13 = layer.w13_weight
|
||||
w2 = layer.w2_weight
|
||||
w13_scale = layer.w13_weight_scale
|
||||
w2_scale = layer.w2_weight_scale
|
||||
elif fp8_backend == Fp8MoeBackend.MARLIN:
|
||||
weight_block_size = getattr(layer, "weight_block_size", None)
|
||||
if weight_block_size == [1, 32]:
|
||||
@@ -511,6 +575,7 @@ def make_fp8_moe_quant_config(
|
||||
swiglu_limit: float | None = None,
|
||||
gemm1_alpha: float | None = None,
|
||||
gemm1_beta: float | None = None,
|
||||
layer: torch.nn.Module | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
Create FusedMoEQuantConfig for the specified FP8 Backend.
|
||||
@@ -537,6 +602,14 @@ def make_fp8_moe_quant_config(
|
||||
gemm1_beta=gemm1_beta,
|
||||
gemm1_clamp_limit=swiglu_limit,
|
||||
)
|
||||
elif fp8_backend == Fp8MoeBackend.HUMMING:
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
get_humming_moe_quant_config,
|
||||
)
|
||||
|
||||
assert isinstance(layer, RoutedExperts)
|
||||
return get_humming_moe_quant_config(layer)
|
||||
|
||||
# Flashinfer CUTLASS or HPC per-tensor uses single dq scale
|
||||
# (alpha = w_scale * a_scale) and inverse a2 scale.
|
||||
@@ -600,6 +673,7 @@ def make_fp8_moe_kernel(
|
||||
experts_cls: type[mk.FusedMoEExperts],
|
||||
fp8_backend: Fp8MoeBackend,
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
layer: torch.nn.Module | None = None,
|
||||
) -> mk.FusedMoEKernel:
|
||||
# Create Prepare/Finalize.
|
||||
prepare_finalize = maybe_make_prepare_finalize(
|
||||
@@ -613,6 +687,11 @@ def make_fp8_moe_kernel(
|
||||
|
||||
logger.info_once("Using %s", prepare_finalize.__class__.__name__)
|
||||
|
||||
extra_kwargs = {}
|
||||
if fp8_backend == Fp8MoeBackend.HUMMING:
|
||||
assert layer is not None
|
||||
extra_kwargs = {"layer": layer}
|
||||
|
||||
# Create Experts.
|
||||
if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
|
||||
max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
|
||||
@@ -622,11 +701,13 @@ def make_fp8_moe_kernel(
|
||||
quant_config=moe_quant_config,
|
||||
max_num_tokens=max_num_tokens,
|
||||
num_dispatchers=prepare_finalize.num_dispatchers(),
|
||||
**extra_kwargs,
|
||||
)
|
||||
else:
|
||||
experts = experts_cls(
|
||||
moe_config=moe_config,
|
||||
quant_config=moe_quant_config,
|
||||
**extra_kwargs,
|
||||
)
|
||||
|
||||
kernel = mk.FusedMoEKernel(
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
@@ -22,6 +23,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kInt8DynamicTokenSym,
|
||||
kInt8StaticChannelSym,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -29,6 +31,7 @@ logger = init_logger(__name__)
|
||||
|
||||
class Int8MoeBackend(Enum):
|
||||
TRITON = "TRITON"
|
||||
HUMMING = "HUMMING"
|
||||
CPU = "CPU"
|
||||
|
||||
|
||||
@@ -40,6 +43,7 @@ def _get_priority_backends(
|
||||
"""
|
||||
_AVAILABLE_BACKENDS = [
|
||||
Int8MoeBackend.TRITON,
|
||||
Int8MoeBackend.HUMMING,
|
||||
Int8MoeBackend.CPU,
|
||||
]
|
||||
|
||||
@@ -62,6 +66,19 @@ def backend_to_kernel_cls(
|
||||
|
||||
return [TritonExperts]
|
||||
|
||||
elif backend == Int8MoeBackend.HUMMING:
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
BatchedHummingGroupedExperts,
|
||||
HummingGroupedExperts,
|
||||
HummingIndexedExperts,
|
||||
)
|
||||
|
||||
return [
|
||||
BatchedHummingGroupedExperts,
|
||||
HummingGroupedExperts,
|
||||
HummingIndexedExperts,
|
||||
]
|
||||
|
||||
elif backend == Int8MoeBackend.CPU:
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
CPUExpertsInt8,
|
||||
@@ -77,6 +94,7 @@ def map_int8_backend(runner_backend: MoEBackend) -> Int8MoeBackend:
|
||||
"""Map user's MoEBackend to Int8MoeBackend."""
|
||||
mapping = {
|
||||
"triton": Int8MoeBackend.TRITON,
|
||||
"humming": Int8MoeBackend.HUMMING,
|
||||
}
|
||||
if backend := mapping.get(runner_backend):
|
||||
return backend
|
||||
@@ -163,6 +181,7 @@ def select_int8_moe_backend(
|
||||
|
||||
|
||||
def make_int8_moe_quant_config(
|
||||
int8_backend: Int8MoeBackend,
|
||||
w1_scale: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
a1_scale: torch.Tensor | None = None,
|
||||
@@ -170,11 +189,21 @@ def make_int8_moe_quant_config(
|
||||
w1_bias: torch.Tensor | None = None,
|
||||
w2_bias: torch.Tensor | None = None,
|
||||
per_act_token_quant: bool = False,
|
||||
layer: torch.nn.Module | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
assert (a1_scale is None and a2_scale is None) or (
|
||||
a1_scale is not None and a2_scale is not None
|
||||
), "a1_scale and a2_scale must both be provided or both be None"
|
||||
|
||||
if int8_backend == Int8MoeBackend.HUMMING:
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
get_humming_moe_quant_config,
|
||||
)
|
||||
|
||||
assert isinstance(layer, RoutedExperts)
|
||||
return get_humming_moe_quant_config(layer)
|
||||
|
||||
if a1_scale is None or a2_scale is None:
|
||||
return int8_w8a16_moe_quant_config(
|
||||
w1_scale=w1_scale,
|
||||
@@ -196,13 +225,56 @@ def make_int8_moe_quant_config(
|
||||
)
|
||||
|
||||
|
||||
def _humming_int8_weight_schema(
|
||||
weight: torch.Tensor, weight_scale: torch.Tensor
|
||||
) -> dict[str, Any]:
|
||||
"""Build the humming compressed-tensors int8 schema from the canonical
|
||||
on-device tensors; humming does the signed-int8 -> native conversion."""
|
||||
config: dict[str, Any] = {
|
||||
"quant_method": "compressed-tensors",
|
||||
"format": "int-quantized",
|
||||
"type": "int",
|
||||
"num_bits": 8,
|
||||
"symmetric": True,
|
||||
"strategy": "channel",
|
||||
}
|
||||
num_experts, num_output = weight.shape[0], weight.shape[-2]
|
||||
if weight_scale.numel() < num_experts * num_output:
|
||||
config["strategy"] = "tensor"
|
||||
return config
|
||||
|
||||
|
||||
def convert_to_int8_moe_kernel_format(
|
||||
int8_backend: Int8MoeBackend,
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
layer: torch.nn.Module | None = None,
|
||||
w13_scale: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Convert INT8 MoE weights to backend-specific kernel format."""
|
||||
if int8_backend == Int8MoeBackend.CPU:
|
||||
if int8_backend == Int8MoeBackend.HUMMING:
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
convert_to_humming_moe_kernel_format,
|
||||
)
|
||||
|
||||
assert layer is not None
|
||||
# Humming reads canonical CT scales (w*_weight_scale) from the layer.
|
||||
# Online int8 produces per-channel (E, N) w*_scale; expose them as the
|
||||
# (E, N, 1) w*_weight_scale humming's loader expects.
|
||||
for sub in ("w13", "w2"):
|
||||
if hasattr(layer, f"{sub}_weight_scale"):
|
||||
continue
|
||||
scale = getattr(layer, f"{sub}_scale").data
|
||||
if scale.dim() < 3:
|
||||
scale = scale.unsqueeze(-1)
|
||||
replace_parameter(layer, f"{sub}_weight_scale", scale)
|
||||
delattr(layer, f"{sub}_scale")
|
||||
convert_to_humming_moe_kernel_format(
|
||||
layer,
|
||||
quant_config=_humming_int8_weight_schema(w13, layer.w13_weight_scale),
|
||||
)
|
||||
return layer.w13_weight, layer.w2_weight
|
||||
elif int8_backend == Int8MoeBackend.CPU:
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
prepare_int8_moe_layer_for_cpu,
|
||||
)
|
||||
@@ -215,10 +287,12 @@ def convert_to_int8_moe_kernel_format(
|
||||
|
||||
|
||||
def make_int8_moe_kernel(
|
||||
int8_backend: Int8MoeBackend,
|
||||
moe_quant_config: FusedMoEQuantConfig,
|
||||
moe_config: FusedMoEConfig,
|
||||
experts_cls: type[mk.FusedMoEExperts],
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
layer: torch.nn.Module | None = None,
|
||||
) -> mk.FusedMoEKernel:
|
||||
# Create Prepare/Finalize.
|
||||
prepare_finalize = maybe_make_prepare_finalize(
|
||||
@@ -232,6 +306,11 @@ def make_int8_moe_kernel(
|
||||
|
||||
logger.info_once("Using %s", prepare_finalize.__class__.__name__)
|
||||
|
||||
extra_kwargs = {}
|
||||
if int8_backend == Int8MoeBackend.HUMMING:
|
||||
assert layer is not None
|
||||
extra_kwargs = {"layer": layer}
|
||||
|
||||
# Create Experts.
|
||||
if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
|
||||
max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
|
||||
@@ -241,11 +320,13 @@ def make_int8_moe_kernel(
|
||||
quant_config=moe_quant_config,
|
||||
max_num_tokens=max_num_tokens,
|
||||
num_dispatchers=prepare_finalize.num_dispatchers(),
|
||||
**extra_kwargs,
|
||||
)
|
||||
else:
|
||||
experts = experts_cls(
|
||||
moe_config=moe_config,
|
||||
quant_config=moe_quant_config,
|
||||
**extra_kwargs,
|
||||
)
|
||||
|
||||
kernel = mk.FusedMoEKernel(
|
||||
|
||||
@@ -11,6 +11,7 @@ from compressed_tensors.quantization import (
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.config.kernel import MoEBackend
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
@@ -46,6 +47,7 @@ logger = init_logger(__name__)
|
||||
class WNA16MoEBackend(Enum):
|
||||
MARLIN = "MARLIN"
|
||||
BATCHED_MARLIN = "BATCHED_MARLIN"
|
||||
HUMMING = "HUMMING"
|
||||
CPU = "CPU"
|
||||
FLASHINFER_TRTLLM = "FLASHINFER_TRTLLM"
|
||||
XPU = "XPU"
|
||||
@@ -55,7 +57,19 @@ def backend_to_kernel_cls(
|
||||
backend: WNA16MoEBackend,
|
||||
) -> list[type[mk.FusedMoEExperts]]:
|
||||
"""Return the experts class for the given backend, or None for NONE."""
|
||||
if backend == WNA16MoEBackend.MARLIN:
|
||||
if backend == WNA16MoEBackend.HUMMING:
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
BatchedHummingGroupedExperts,
|
||||
HummingGroupedExperts,
|
||||
HummingIndexedExperts,
|
||||
)
|
||||
|
||||
return [
|
||||
BatchedHummingGroupedExperts,
|
||||
HummingGroupedExperts,
|
||||
HummingIndexedExperts,
|
||||
]
|
||||
elif backend == WNA16MoEBackend.MARLIN:
|
||||
return [MarlinExperts]
|
||||
elif backend == WNA16MoEBackend.BATCHED_MARLIN:
|
||||
return [BatchedMarlinExperts]
|
||||
@@ -90,10 +104,26 @@ def _get_priority_backends() -> list[WNA16MoEBackend]:
|
||||
WNA16MoEBackend.FLASHINFER_TRTLLM,
|
||||
WNA16MoEBackend.MARLIN,
|
||||
WNA16MoEBackend.BATCHED_MARLIN,
|
||||
WNA16MoEBackend.HUMMING,
|
||||
]
|
||||
return _AVAILABLE_BACKENDS
|
||||
|
||||
|
||||
def map_wna16_backend(runner_backend: MoEBackend) -> WNA16MoEBackend:
|
||||
"""Map user's MoEBackend to WNA16MoEBackend."""
|
||||
mapping = {
|
||||
"marlin": WNA16MoEBackend.MARLIN,
|
||||
"humming": WNA16MoEBackend.HUMMING,
|
||||
"flashinfer_trtllm": WNA16MoEBackend.FLASHINFER_TRTLLM,
|
||||
}
|
||||
if backend := mapping.get(runner_backend):
|
||||
return backend
|
||||
raise ValueError(
|
||||
f"moe_backend='{runner_backend}' is not supported for WNA16 MoE. "
|
||||
f"Expected one of {list(mapping.keys())}."
|
||||
)
|
||||
|
||||
|
||||
def select_wna16_moe_backend(
|
||||
config: FusedMoEConfig,
|
||||
weight_key: QuantKey,
|
||||
@@ -146,6 +176,14 @@ def select_wna16_moe_backend(
|
||||
return backend, k_cls
|
||||
raise ValueError(_make_log_unsupported(backend, reason))
|
||||
|
||||
# Handle explicit moe_backend from user.
|
||||
runner_backend = config.moe_backend
|
||||
if runner_backend != "auto":
|
||||
requested_backend = map_wna16_backend(runner_backend)
|
||||
return _return_or_raise(
|
||||
requested_backend, config, weight_key, None, activation_format
|
||||
)
|
||||
|
||||
# Select kernels in order of backend.
|
||||
AVAILABLE_BACKENDS = _get_priority_backends()
|
||||
|
||||
@@ -210,6 +248,8 @@ def make_wna16_moe_kernel(
|
||||
moe_quant_config: FusedMoEQuantConfig,
|
||||
moe_config: FusedMoEConfig,
|
||||
experts_cls: type[mk.FusedMoEExperts],
|
||||
backend: WNA16MoEBackend = WNA16MoEBackend.MARLIN,
|
||||
layer: torch.nn.Module | None = None,
|
||||
is_k_full: bool = False,
|
||||
w13_g_idx: torch.Tensor | None = None,
|
||||
w2_g_idx: torch.Tensor | None = None,
|
||||
@@ -228,14 +268,18 @@ def make_wna16_moe_kernel(
|
||||
)
|
||||
|
||||
# Currently, we only support TrtLlmMxint4ExpertsMonolithic, MarlinExperts,
|
||||
# BatchedMarlinExperts, XPUExpertsWNA16, and CPUExpertsInt4
|
||||
assert experts_cls in (
|
||||
# BatchedMarlinExperts, XPUExpertsWNA16, CPUExpertsInt4, and the Humming
|
||||
# grouped/indexed experts.
|
||||
allowed_experts: tuple[type[mk.FusedMoEExperts], ...] = (
|
||||
MarlinExperts,
|
||||
BatchedMarlinExperts,
|
||||
TrtLlmMxint4ExpertsMonolithic,
|
||||
XPUExpertsWNA16,
|
||||
CPUExpertsInt4,
|
||||
)
|
||||
if backend == WNA16MoEBackend.HUMMING:
|
||||
allowed_experts += tuple(backend_to_kernel_cls(WNA16MoEBackend.HUMMING))
|
||||
assert experts_cls in allowed_experts
|
||||
|
||||
is_monolithic = experts_cls.is_monolithic()
|
||||
|
||||
@@ -251,7 +295,10 @@ def make_wna16_moe_kernel(
|
||||
logger.info_once("Using %s", prepare_finalize.__class__.__name__, scope="local")
|
||||
|
||||
extra_args: dict[str, Any] = {}
|
||||
if issubclass(experts_cls, MarlinExpertsBase):
|
||||
if backend == WNA16MoEBackend.HUMMING:
|
||||
assert layer is not None
|
||||
extra_args = {"layer": layer}
|
||||
elif issubclass(experts_cls, MarlinExpertsBase):
|
||||
extra_args = {
|
||||
"w13_g_idx": w13_g_idx,
|
||||
"w2_g_idx": w2_g_idx,
|
||||
@@ -941,6 +988,35 @@ def _process_weights_xpu(
|
||||
)
|
||||
|
||||
|
||||
def _humming_wna16_weight_schema(
|
||||
quant_config: QuantizationConfig | QuantizationArgs | None,
|
||||
) -> dict[str, Any]:
|
||||
"""Humming weight schema for a WNA16 checkpoint, derived from the quant
|
||||
config rather than the running kernel."""
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
|
||||
|
||||
if isinstance(quant_config, AutoAWQConfig):
|
||||
return {
|
||||
"quant_method": "awq",
|
||||
"bits": quant_config.weight_bits,
|
||||
"group_size": quant_config.group_size,
|
||||
"zero_point": quant_config.zero_point,
|
||||
}
|
||||
if isinstance(quant_config, AutoGPTQConfig):
|
||||
return {
|
||||
"quant_method": "gptq",
|
||||
"bits": quant_config.weight_bits,
|
||||
"group_size": quant_config.group_size,
|
||||
"desc_act": quant_config.desc_act,
|
||||
"sym": quant_config.is_sym,
|
||||
}
|
||||
raise TypeError(
|
||||
"Humming WNA16 MoE requires AutoAWQConfig or AutoGPTQConfig, "
|
||||
f"got {type(quant_config).__name__}."
|
||||
)
|
||||
|
||||
|
||||
def convert_to_wna16_moe_kernel_format(
|
||||
backend: WNA16MoEBackend,
|
||||
layer: torch.nn.Module,
|
||||
@@ -956,26 +1032,30 @@ def convert_to_wna16_moe_kernel_format(
|
||||
w2_qzeros: torch.Tensor | None = None,
|
||||
w13_bias: torch.Tensor | None = None,
|
||||
w2_bias: torch.Tensor | None = None,
|
||||
) -> tuple[
|
||||
torch.Tensor, # w13_qweight
|
||||
torch.Tensor, # w2_qweight
|
||||
torch.Tensor, # w13_scales
|
||||
torch.Tensor, # w2_scales
|
||||
torch.Tensor | None, # w13_g_idx
|
||||
torch.Tensor | None, # w2_g_idx
|
||||
torch.Tensor | None, # w13_g_idx_sort_indices
|
||||
torch.Tensor | None, # w2_g_idx_sort_indices
|
||||
torch.Tensor | None, # w13_qzeros
|
||||
torch.Tensor | None, # w2_qzeros
|
||||
torch.Tensor | None, # w13_input_global_scale
|
||||
torch.Tensor | None, # w2_input_global_scale
|
||||
torch.Tensor | None, # w13_bias
|
||||
torch.Tensor | None, # w2_bias
|
||||
]:
|
||||
) -> (
|
||||
tuple[
|
||||
torch.Tensor, # w13_qweight
|
||||
torch.Tensor, # w2_qweight
|
||||
torch.Tensor, # w13_scales
|
||||
torch.Tensor, # w2_scales
|
||||
torch.Tensor | None, # w13_g_idx
|
||||
torch.Tensor | None, # w2_g_idx
|
||||
torch.Tensor | None, # w13_g_idx_sort_indices
|
||||
torch.Tensor | None, # w2_g_idx_sort_indices
|
||||
torch.Tensor | None, # w13_qzeros
|
||||
torch.Tensor | None, # w2_qzeros
|
||||
torch.Tensor | None, # w13_input_global_scale
|
||||
torch.Tensor | None, # w2_input_global_scale
|
||||
torch.Tensor | None, # w13_bias
|
||||
torch.Tensor | None, # w2_bias
|
||||
]
|
||||
| None
|
||||
):
|
||||
"""Dispatch weight post-processing to the appropriate per-backend handler.
|
||||
|
||||
To add a new backend, implement a ``_process_weights_<name>`` helper and
|
||||
add a branch here.
|
||||
add a branch here. Backends that rewrite the layer's parameters in place
|
||||
(e.g. Humming) return ``None``; the caller then skips the param scatter.
|
||||
|
||||
Args:
|
||||
backend: the selected ``WNA16MoEBackend``.
|
||||
@@ -983,6 +1063,16 @@ def convert_to_wna16_moe_kernel_format(
|
||||
quant_config: the ``QuantizationConfig`` for this layer.
|
||||
input_dtype: optional activation dtype, usually should be 16 bit.
|
||||
"""
|
||||
if backend == WNA16MoEBackend.HUMMING:
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
convert_to_humming_moe_kernel_format,
|
||||
)
|
||||
|
||||
convert_to_humming_moe_kernel_format(
|
||||
layer, quant_config=_humming_wna16_weight_schema(quant_config)
|
||||
)
|
||||
return None
|
||||
|
||||
if backend in (
|
||||
WNA16MoEBackend.MARLIN,
|
||||
WNA16MoEBackend.BATCHED_MARLIN,
|
||||
|
||||
@@ -25,6 +25,7 @@ _SUPPORTED_BACKENDS = (
|
||||
# is_supported_config passes (gfx950 + flydsl installed + not EP). On other
|
||||
# devices / no flydsl / EP it is skipped and native is used.
|
||||
Fp8MoeBackend.AITER_MXFP8,
|
||||
Fp8MoeBackend.HUMMING,
|
||||
)
|
||||
|
||||
_BACKEND_NAME_MAP: dict[str, Fp8MoeBackend] = {
|
||||
@@ -34,6 +35,7 @@ _BACKEND_NAME_MAP: dict[str, Fp8MoeBackend] = {
|
||||
"xpu": Fp8MoeBackend.XPU,
|
||||
"aiter": Fp8MoeBackend.AITER_MXFP8,
|
||||
"triton": Fp8MoeBackend.TRITON_MXFP8,
|
||||
"humming": Fp8MoeBackend.HUMMING,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -43,6 +43,7 @@ class NvFp4MoeBackend(Enum):
|
||||
FLASHINFER_B12X = "FLASHINFER_B12X"
|
||||
VLLM_CUTLASS = "VLLM_CUTLASS"
|
||||
MARLIN = "MARLIN"
|
||||
HUMMING = "HUMMING"
|
||||
EMULATION = "EMULATION"
|
||||
|
||||
|
||||
@@ -119,6 +120,18 @@ def backend_to_kernel_cls(
|
||||
)
|
||||
|
||||
return [MarlinExperts]
|
||||
elif backend == NvFp4MoeBackend.HUMMING:
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
BatchedHummingGroupedExperts,
|
||||
HummingGroupedExperts,
|
||||
HummingIndexedExperts,
|
||||
)
|
||||
|
||||
return [
|
||||
BatchedHummingGroupedExperts,
|
||||
HummingGroupedExperts,
|
||||
HummingIndexedExperts,
|
||||
]
|
||||
elif backend == NvFp4MoeBackend.EMULATION:
|
||||
from vllm.model_executor.layers.fused_moe.experts.nvfp4_emulation_moe import (
|
||||
Nvfp4QuantizationEmulationTritonExperts,
|
||||
@@ -138,6 +151,7 @@ def map_nvfp4_backend(runner_backend: MoEBackend) -> NvFp4MoeBackend:
|
||||
"flashinfer_cutedsl": NvFp4MoeBackend.FLASHINFER_CUTEDSL,
|
||||
"flashinfer_b12x": NvFp4MoeBackend.FLASHINFER_B12X,
|
||||
"marlin": NvFp4MoeBackend.MARLIN,
|
||||
"humming": NvFp4MoeBackend.HUMMING,
|
||||
"emulation": NvFp4MoeBackend.EMULATION,
|
||||
}
|
||||
if backend := mapping.get(runner_backend):
|
||||
@@ -169,6 +183,7 @@ def select_nvfp4_moe_backend(
|
||||
NvFp4MoeBackend.FLASHINFER_CUTLASS,
|
||||
NvFp4MoeBackend.VLLM_CUTLASS,
|
||||
NvFp4MoeBackend.MARLIN,
|
||||
NvFp4MoeBackend.HUMMING,
|
||||
NvFp4MoeBackend.EMULATION,
|
||||
]
|
||||
|
||||
@@ -346,6 +361,41 @@ def convert_to_nvfp4_moe_kernel_format(
|
||||
a2_scale=a2_scale,
|
||||
is_act_and_mul=is_act_and_mul,
|
||||
)
|
||||
elif nvfp4_backend == NvFp4MoeBackend.HUMMING:
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
convert_to_humming_moe_kernel_format,
|
||||
)
|
||||
|
||||
# Discriminate the source checkpoint layout by its global-scale param:
|
||||
# compressed-tensors uses *_weight_global_scale, modelopt *_weight_scale_2.
|
||||
# The logical schema is identical (nvfp4 group-16); only the on-layer
|
||||
# param names differ. TODO: normalize both methods to a single canonical
|
||||
# layout upstream so the oracle needs neither the probe nor the re-alias.
|
||||
if hasattr(layer, "w13_weight_global_scale"):
|
||||
quant_config = {
|
||||
"quant_method": "compressed-tensors",
|
||||
"format": "nvfp4-pack-quantized",
|
||||
"type": "float",
|
||||
"num_bits": 4,
|
||||
"strategy": "group",
|
||||
"group_size": 16,
|
||||
}
|
||||
# CT pack-quantized reads `weight_packed`; the method renamed it to
|
||||
# `weight`. Re-alias (convert replaces all params anyway).
|
||||
layer.w13_weight_packed = layer.w13_weight
|
||||
layer.w2_weight_packed = layer.w2_weight
|
||||
else:
|
||||
quant_config = {"quant_method": "modelopt", "quant_algo": "nvfp4"}
|
||||
|
||||
convert_to_humming_moe_kernel_format(layer, quant_config=quant_config)
|
||||
a13_scale = None
|
||||
a2_scale = None
|
||||
w13 = layer.w13_weight
|
||||
w13_scale = layer.w13_weight_scale
|
||||
w13_scale_2 = getattr(layer, "w13_global_scale", None)
|
||||
w2 = layer.w2_weight
|
||||
w2_scale = layer.w2_weight_scale
|
||||
w2_scale_2 = getattr(layer, "w2_global_scale", None)
|
||||
elif nvfp4_backend == NvFp4MoeBackend.MARLIN:
|
||||
a13_scale = None
|
||||
a2_scale = None
|
||||
@@ -418,8 +468,17 @@ def make_nvfp4_moe_quant_config(
|
||||
a13_scale: torch.Tensor,
|
||||
a2_scale: torch.Tensor,
|
||||
swiglu_limit: float | None = None,
|
||||
layer: torch.nn.Module | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
if backend == NvFp4MoeBackend.MARLIN:
|
||||
if backend == NvFp4MoeBackend.HUMMING:
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
get_humming_moe_quant_config,
|
||||
)
|
||||
|
||||
assert isinstance(layer, RoutedExperts)
|
||||
return get_humming_moe_quant_config(layer)
|
||||
elif backend == NvFp4MoeBackend.MARLIN:
|
||||
return nvfp4_w4a16_moe_quant_config(
|
||||
g1_alphas=w13_scale_2,
|
||||
g2_alphas=w2_scale_2,
|
||||
@@ -467,7 +526,9 @@ def make_nvfp4_moe_kernel(
|
||||
moe_quant_config: FusedMoEQuantConfig,
|
||||
moe_config: FusedMoEConfig,
|
||||
experts_cls: type[mk.FusedMoEExperts],
|
||||
backend: NvFp4MoeBackend,
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
layer: torch.nn.Module | None = None,
|
||||
) -> mk.FusedMoEKernel:
|
||||
# Create Prepare/Finalize.
|
||||
prepare_finalize = maybe_make_prepare_finalize(
|
||||
@@ -481,6 +542,11 @@ def make_nvfp4_moe_kernel(
|
||||
|
||||
logger.info_once("Using %s", prepare_finalize.__class__.__name__)
|
||||
|
||||
extra_kwargs = {}
|
||||
if backend == NvFp4MoeBackend.HUMMING:
|
||||
assert layer is not None
|
||||
extra_kwargs = {"layer": layer}
|
||||
|
||||
# Create Experts.
|
||||
if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
|
||||
max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
|
||||
@@ -490,11 +556,13 @@ def make_nvfp4_moe_kernel(
|
||||
quant_config=moe_quant_config,
|
||||
max_num_tokens=max_num_tokens,
|
||||
num_dispatchers=prepare_finalize.num_dispatchers(),
|
||||
**extra_kwargs,
|
||||
)
|
||||
else:
|
||||
experts = experts_cls(
|
||||
moe_config=moe_config,
|
||||
quant_config=moe_quant_config,
|
||||
**extra_kwargs,
|
||||
)
|
||||
|
||||
kernel = mk.FusedMoEKernel(
|
||||
|
||||
@@ -232,7 +232,9 @@ def select_unquantized_moe_backend(
|
||||
raise ValueError(_make_log_unsupported(backend, reason))
|
||||
|
||||
runner_backend = moe_config.moe_backend
|
||||
if runner_backend != "auto":
|
||||
# 'humming' is quantization-only; an unquantized layer (e.g. excluded via
|
||||
# modules_to_not_convert) falls through to auto instead of erroring.
|
||||
if runner_backend not in ["auto", "humming"]:
|
||||
requested_backend = map_unquantized_backend(runner_backend)
|
||||
if (
|
||||
activation_format == mk.FusedMoEActivationFormat.BatchedExperts
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
|
||||
WNA16MoEBackend,
|
||||
convert_to_wna16_moe_kernel_format,
|
||||
make_wna16_moe_kernel,
|
||||
make_wna16_moe_quant_config,
|
||||
@@ -663,6 +664,26 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
|
||||
layer.workspace = marlin_make_workspace_new(device, 4)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
converted = convert_to_wna16_moe_kernel_format(
|
||||
backend=self.wna16_moe_backend,
|
||||
layer=layer,
|
||||
quant_config=self.quant_config,
|
||||
input_dtype=self.input_dtype,
|
||||
w13=layer.w13_qweight,
|
||||
w2=layer.w2_qweight,
|
||||
w13_scale=layer.w13_scales,
|
||||
w2_scale=layer.w2_scales,
|
||||
w13_qzeros=layer.w13_qzeros,
|
||||
w2_qzeros=layer.w2_qzeros,
|
||||
w13_bias=getattr(layer, "w13_bias", None),
|
||||
w2_bias=getattr(layer, "w2_bias", None),
|
||||
)
|
||||
|
||||
if converted is None:
|
||||
# Backend rewrote the layer's params in place (e.g. Humming).
|
||||
self._setup_kernel(layer)
|
||||
return
|
||||
|
||||
(
|
||||
w13,
|
||||
w2,
|
||||
@@ -678,20 +699,7 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
|
||||
w2_input_global_scale,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
) = convert_to_wna16_moe_kernel_format(
|
||||
backend=self.wna16_moe_backend,
|
||||
layer=layer,
|
||||
quant_config=self.quant_config,
|
||||
input_dtype=self.input_dtype,
|
||||
w13=layer.w13_qweight,
|
||||
w2=layer.w2_qweight,
|
||||
w13_scale=layer.w13_scales,
|
||||
w2_scale=layer.w2_scales,
|
||||
w13_qzeros=layer.w13_qzeros,
|
||||
w2_qzeros=layer.w2_qzeros,
|
||||
w13_bias=getattr(layer, "w13_bias", None),
|
||||
w2_bias=getattr(layer, "w2_bias", None),
|
||||
)
|
||||
) = converted
|
||||
|
||||
replace_parameter(layer, "w13_qweight", w13)
|
||||
replace_parameter(layer, "w2_qweight", w2)
|
||||
@@ -734,6 +742,8 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.wna16_moe_backend,
|
||||
layer=layer,
|
||||
is_k_full=self.is_k_full,
|
||||
w13_g_idx=getattr(layer, "w13_g_idx", None),
|
||||
w2_g_idx=getattr(layer, "w2_g_idx", None),
|
||||
@@ -743,6 +753,12 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
|
||||
if self.wna16_moe_backend == WNA16MoEBackend.HUMMING:
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
get_humming_moe_quant_config,
|
||||
)
|
||||
|
||||
return get_humming_moe_quant_config(layer)
|
||||
return make_wna16_moe_quant_config(
|
||||
w1_scale=layer.w13_scales,
|
||||
w2_scale=layer.w2_scales,
|
||||
@@ -783,8 +799,8 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
hidden_states=x,
|
||||
w1=layer.w13_qweight,
|
||||
w2=layer.w2_qweight,
|
||||
w1=layer.w13_weight,
|
||||
w2=layer.w2_weight,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=layer.activation,
|
||||
@@ -806,8 +822,8 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply_monolithic(
|
||||
hidden_states=x,
|
||||
w1=layer.w13_qweight,
|
||||
w2=layer.w2_qweight,
|
||||
w1=layer.w13_weight,
|
||||
w2=layer.w2_weight,
|
||||
router_logits=router_logits,
|
||||
activation=layer.activation,
|
||||
global_num_experts=layer.global_num_experts,
|
||||
|
||||
@@ -658,6 +658,26 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
"W8A8-INT8 is not supported by marlin kernel."
|
||||
)
|
||||
|
||||
converted = convert_to_wna16_moe_kernel_format(
|
||||
backend=self.wna16_moe_backend,
|
||||
layer=layer,
|
||||
quant_config=self.quant_config,
|
||||
input_dtype=self.input_dtype,
|
||||
w13=layer.w13_qweight,
|
||||
w2=layer.w2_qweight,
|
||||
w13_scale=layer.w13_scales,
|
||||
w2_scale=layer.w2_scales,
|
||||
w13_g_idx=layer.w13_g_idx,
|
||||
w2_g_idx=layer.w2_g_idx,
|
||||
w13_bias=getattr(layer, "w13_bias", None),
|
||||
w2_bias=getattr(layer, "w2_bias", None),
|
||||
)
|
||||
|
||||
if converted is None:
|
||||
# Backend rewrote the layer's params in place (e.g. Humming).
|
||||
self._setup_kernel(layer)
|
||||
return
|
||||
|
||||
(
|
||||
w13,
|
||||
w2,
|
||||
@@ -673,20 +693,7 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
w2_input_global_scale,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
) = convert_to_wna16_moe_kernel_format(
|
||||
backend=self.wna16_moe_backend,
|
||||
layer=layer,
|
||||
quant_config=self.quant_config,
|
||||
input_dtype=self.input_dtype,
|
||||
w13=layer.w13_qweight,
|
||||
w2=layer.w2_qweight,
|
||||
w13_scale=layer.w13_scales,
|
||||
w2_scale=layer.w2_scales,
|
||||
w13_g_idx=layer.w13_g_idx,
|
||||
w2_g_idx=layer.w2_g_idx,
|
||||
w13_bias=getattr(layer, "w13_bias", None),
|
||||
w2_bias=getattr(layer, "w2_bias", None),
|
||||
)
|
||||
) = converted
|
||||
|
||||
replace_parameter(layer, "w13_qweight", w13)
|
||||
replace_parameter(layer, "w2_qweight", w2)
|
||||
@@ -733,6 +740,10 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
"w2_bias", torch.nn.Parameter(w2_bias, requires_grad=False)
|
||||
)
|
||||
|
||||
# The modular kernel reads w13_weight/w2_weight; marlin keeps *_qweight.
|
||||
layer.w13_weight = layer.w13_qweight
|
||||
layer.w2_weight = layer.w2_qweight
|
||||
|
||||
self._setup_kernel(layer)
|
||||
|
||||
def _setup_kernel(self, layer: RoutedExperts) -> None:
|
||||
@@ -743,15 +754,24 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.wna16_moe_backend,
|
||||
layer=layer,
|
||||
is_k_full=self.is_k_full,
|
||||
w13_g_idx=layer.w13_g_idx,
|
||||
w2_g_idx=layer.w2_g_idx,
|
||||
w13_g_idx=getattr(layer, "w13_g_idx", None),
|
||||
w2_g_idx=getattr(layer, "w2_g_idx", None),
|
||||
w13_g_idx_sort_indices=getattr(layer, "w13_g_idx_sort_indices", None),
|
||||
w2_g_idx_sort_indices=getattr(layer, "w2_g_idx_sort_indices", None),
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
|
||||
if self.wna16_moe_backend == WNA16MoEBackend.HUMMING:
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
get_humming_moe_quant_config,
|
||||
)
|
||||
|
||||
return get_humming_moe_quant_config(layer)
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
gptq_marlin_moe_quant_config,
|
||||
)
|
||||
@@ -795,8 +815,8 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
hidden_states=x,
|
||||
w1=layer.w13_qweight,
|
||||
w2=layer.w2_qweight,
|
||||
w1=layer.w13_weight,
|
||||
w2=layer.w2_weight,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=layer.activation,
|
||||
@@ -818,8 +838,8 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply_monolithic(
|
||||
hidden_states=x,
|
||||
w1=layer.w13_qweight,
|
||||
w2=layer.w2_qweight,
|
||||
w1=layer.w13_weight,
|
||||
w2=layer.w2_weight,
|
||||
router_logits=router_logits,
|
||||
activation=layer.activation,
|
||||
global_num_experts=layer.global_num_experts,
|
||||
|
||||
+3
@@ -236,7 +236,9 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.nvfp4_backend,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
@@ -259,6 +261,7 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
a13_scale=layer.w13_input_scale,
|
||||
a2_scale=layer.w2_input_scale,
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def apply_monolithic(
|
||||
|
||||
+4
-2
@@ -338,6 +338,7 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
fp8_backend=self.fp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def maybe_make_prepare_finalize(
|
||||
@@ -355,12 +356,13 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
fp8_backend=self.fp8_backend,
|
||||
w1_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
a1_scale=layer.w13_input_scale,
|
||||
a2_scale=layer.w2_input_scale,
|
||||
a1_scale=getattr(layer, "w13_input_scale", None),
|
||||
a2_scale=getattr(layer, "w2_input_scale", None),
|
||||
per_act_token_quant=is_per_token,
|
||||
per_out_ch_quant=is_per_token,
|
||||
block_shape=self.weight_block_size,
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def apply_monolithic(
|
||||
|
||||
+6
@@ -146,6 +146,8 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
|
||||
int8_backend=self.int8_backend,
|
||||
w13=layer.w13_weight,
|
||||
w2=layer.w2_weight,
|
||||
layer=layer,
|
||||
w13_scale=layer.w13_weight_scale,
|
||||
)
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
replace_parameter(layer, "w2_weight", w2)
|
||||
@@ -153,10 +155,12 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
assert self.experts_cls is not None
|
||||
self.moe_kernel = make_int8_moe_kernel(
|
||||
int8_backend=self.int8_backend,
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def maybe_make_prepare_finalize(
|
||||
@@ -170,11 +174,13 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
|
||||
|
||||
def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig:
|
||||
return make_int8_moe_quant_config(
|
||||
int8_backend=self.int8_backend,
|
||||
w1_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
a1_scale=layer.w13_input_scale,
|
||||
a2_scale=layer.w2_input_scale,
|
||||
per_act_token_quant=True,
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
+2
@@ -140,6 +140,7 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
fp8_backend=self.fp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
@@ -155,6 +156,7 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def maybe_make_prepare_finalize(
|
||||
|
||||
+22
-14
@@ -416,6 +416,27 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
# Process weights using the shared oracle infrastructure
|
||||
is_flashinfer = self.wna16_backend == WNA16MoEBackend.FLASHINFER_TRTLLM
|
||||
converted = convert_to_wna16_moe_kernel_format(
|
||||
backend=self.wna16_backend,
|
||||
layer=layer,
|
||||
quant_config=self.weight_quant,
|
||||
input_dtype=self.marlin_input_dtype,
|
||||
w13=layer.w13_weight_packed,
|
||||
w2=layer.w2_weight_packed,
|
||||
w13_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
w13_g_idx=layer.w13_weight_g_idx,
|
||||
w2_g_idx=layer.w2_weight_g_idx,
|
||||
w13_qzeros=getattr(layer, "w13_weight_zero_point", None),
|
||||
w2_qzeros=getattr(layer, "w2_weight_zero_point", None),
|
||||
)
|
||||
if converted is None:
|
||||
# In-place backends (e.g. Humming) are not wired through this
|
||||
# marlin-only method; fail clearly rather than unpacking None.
|
||||
raise NotImplementedError(
|
||||
f"{type(self).__name__} does not support the "
|
||||
f"{self.wna16_backend.value} MoE backend."
|
||||
)
|
||||
(
|
||||
w13_qweight,
|
||||
w2_qweight,
|
||||
@@ -431,20 +452,7 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
|
||||
w2_input_global_scale,
|
||||
_, # w13_bias
|
||||
_, # w2_bias
|
||||
) = convert_to_wna16_moe_kernel_format(
|
||||
backend=self.wna16_backend,
|
||||
layer=layer,
|
||||
quant_config=self.weight_quant,
|
||||
input_dtype=self.marlin_input_dtype,
|
||||
w13=layer.w13_weight_packed,
|
||||
w2=layer.w2_weight_packed,
|
||||
w13_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
w13_g_idx=layer.w13_weight_g_idx,
|
||||
w2_g_idx=layer.w2_weight_g_idx,
|
||||
w13_qzeros=getattr(layer, "w13_weight_zero_point", None),
|
||||
w2_qzeros=getattr(layer, "w2_weight_zero_point", None),
|
||||
)
|
||||
) = converted
|
||||
|
||||
# Replace common parameters
|
||||
replace_parameter(layer, "w13_weight_packed", w13_qweight)
|
||||
|
||||
+8
@@ -137,8 +137,16 @@ class CompressedTensorsW8A16Fp8(CompressedTensorsScheme):
|
||||
"weight_scale",
|
||||
convert_to_channelwise(layer.weight_scale, layer.logical_widths),
|
||||
)
|
||||
self.strategy = QuantizationStrategy.CHANNEL
|
||||
self.weight_quant_key = STRATEGY_TO_WEIGHT_QUANT_KEY[self.strategy]
|
||||
self.linear_kernel.config.weight_quant_key = self.weight_quant_key
|
||||
|
||||
# Canonicalize to (K, N) for the kernel.
|
||||
replace_parameter(layer, "weight", layer.weight.t())
|
||||
# Preserve the dim tags dropped by the transpose so layout-aware
|
||||
# kernels see (K, N).
|
||||
layer.weight.input_dim = 0
|
||||
layer.weight.output_dim = 1
|
||||
|
||||
self.linear_kernel.process_weights_after_loading(layer)
|
||||
|
||||
|
||||
+4
@@ -181,6 +181,10 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
|
||||
|
||||
# required by torch.compile to be torch.nn.Parameter
|
||||
layer.weight = Parameter(weight.data, requires_grad=False)
|
||||
# Preserve the dim tags dropped by the transpose so layout-aware
|
||||
# kernels (humming) see (K, N) instead of assuming (N, K).
|
||||
layer.weight.input_dim = 0
|
||||
layer.weight.output_dim = 1
|
||||
layer.weight_scale = Parameter(weight_scale.data, requires_grad=False)
|
||||
if input_scale is not None:
|
||||
layer.input_scale = Parameter(input_scale.data, requires_grad=False)
|
||||
|
||||
@@ -714,6 +714,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
fp8_backend=self.fp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
@@ -786,6 +787,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
# Inject biases into the quant config if the model has them
|
||||
|
||||
@@ -905,6 +905,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
fp8_backend=self.fp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
@@ -951,6 +952,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
a1_scale=a1_scale,
|
||||
a2_scale=a2_scale,
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def apply_monolithic(
|
||||
@@ -1602,7 +1604,9 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.nvfp4_backend,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
@@ -1616,6 +1620,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
a13_scale=layer.w13_input_scale,
|
||||
a2_scale=layer.w2_input_scale,
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
@property
|
||||
@@ -2162,6 +2167,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
|
||||
fp8_backend=self.mxfp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
# No native MXFP8 MoE kernel on this device (e.g. gfx942): the emulation
|
||||
@@ -2207,6 +2213,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def apply_monolithic(
|
||||
|
||||
@@ -458,6 +458,7 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
|
||||
fp8_backend=self.fp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
@@ -486,6 +487,7 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ if TYPE_CHECKING:
|
||||
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.fused_moe.oracle.int8 import (
|
||||
convert_to_int8_moe_kernel_format,
|
||||
make_int8_moe_kernel,
|
||||
make_int8_moe_quant_config,
|
||||
select_int8_moe_backend,
|
||||
@@ -92,22 +93,36 @@ class Int8OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
replace_parameter(layer, "w2_scale", w2_scale)
|
||||
|
||||
def _setup_kernel(self, layer: RoutedExperts) -> None:
|
||||
w13, w2 = convert_to_int8_moe_kernel_format(
|
||||
int8_backend=self.int8_backend,
|
||||
w13=layer.w13_weight,
|
||||
w2=layer.w2_weight,
|
||||
layer=layer,
|
||||
w13_scale=layer.w13_scale,
|
||||
)
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
replace_parameter(layer, "w2_weight", w2)
|
||||
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
assert self.moe_quant_config is not None
|
||||
assert self.experts_cls is not None
|
||||
self.moe_kernel = make_int8_moe_kernel(
|
||||
int8_backend=self.int8_backend,
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: torch.nn.Module
|
||||
) -> "FusedMoEQuantConfig | None":
|
||||
return make_int8_moe_quant_config(
|
||||
w1_scale=layer.w13_scale,
|
||||
w2_scale=layer.w2_scale,
|
||||
int8_backend=self.int8_backend,
|
||||
w1_scale=getattr(layer, "w13_scale", None),
|
||||
w2_scale=getattr(layer, "w2_scale", None),
|
||||
w1_bias=getattr(layer, "w13_bias", None),
|
||||
w2_bias=getattr(layer, "w2_bias", None),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
@@ -200,6 +200,7 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
fp8_backend=self.fp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
@@ -226,6 +227,7 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: Module) -> None:
|
||||
|
||||
@@ -1562,7 +1562,9 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.nvfp4_backend,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
@@ -1576,6 +1578,7 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
|
||||
w2_scale_2=layer.w2_weight_scale_2,
|
||||
a13_scale=layer.w13_input_scale_2,
|
||||
a2_scale=layer.w2_input_scale_2,
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
@@ -764,7 +764,7 @@ def _convert_sublayer_to_humming(
|
||||
Returns:
|
||||
Tuple of (converted_weight_schema, converted_input_schema)
|
||||
"""
|
||||
from humming.schema import HummingWeightSchema
|
||||
from vllm.utils.humming import HummingWeightSchema
|
||||
|
||||
if isinstance(weight_schema, HummingWeightSchema):
|
||||
# Already in Humming format
|
||||
@@ -814,7 +814,7 @@ def _prepare_and_transform_sublayer(
|
||||
|
||||
This calls Humming's prepare_layer_meta and transform_humming_layer.
|
||||
"""
|
||||
from humming.layer import HummingMethod
|
||||
from vllm.utils.humming import HummingMethod
|
||||
|
||||
HummingMethod.prepare_layer_meta(
|
||||
layer=layer,
|
||||
@@ -866,7 +866,7 @@ def _process_single_sublayer(
|
||||
Returns:
|
||||
Tuple of (final_weight_schema, final_input_schema)
|
||||
"""
|
||||
from humming.schema import HummingWeightSchema
|
||||
from vllm.utils.humming import HummingWeightSchema
|
||||
|
||||
# Step 1: Convert from checkpoint format to humming format if needed
|
||||
current_weight_schema, current_input_schema = _convert_sublayer_to_humming(
|
||||
@@ -958,12 +958,10 @@ def convert_to_humming_moe_kernel_format(
|
||||
"Must provide either weight_schema/input_schema or quant_config"
|
||||
)
|
||||
|
||||
from humming.layer import HummingInputSchema
|
||||
from humming.schema import BaseWeightSchema
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
humming_is_layer_skipped,
|
||||
)
|
||||
from vllm.utils.humming import BaseWeightSchema, HummingInputSchema
|
||||
|
||||
if weight_schema is None:
|
||||
weight_schema = BaseWeightSchema.from_config(quant_config)
|
||||
|
||||
Reference in New Issue
Block a user