[MoE Refactor] Combine CompressedTensorsWNA16MarlinMoEMethod with CompressedTensorsWNA16MoEMethod (#44570)

Signed-off-by: Bill Nell <bnell@redhat.com>
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
Signed-off-by: Dobrzyniewicz, Agata <agata.dobrzyniewicz@intel.com>
Signed-off-by: Tushar Jain <tushar00jain@users.noreply.github.com>
Signed-off-by: yewentao256 <zhyanwentao@126.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
Co-authored-by: Agata Dobrzyniewicz <160237065+adobrzyn@users.noreply.github.com>
Co-authored-by: Tushar Jain <8455015+tushar00jain@users.noreply.github.com>
Co-authored-by: Tushar Jain <tushar00jain@users.noreply.github.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
bnellnm
2026-07-31 14:13:03 -07:00
committed by GitHub
co-authored by Claude Taneem Ibrahim Agata Dobrzyniewicz Tushar Jain Tushar Jain Wentao Ye Jiangyun Zhu
parent 0c4dc7c488
commit 454ea5b526
14 changed files with 545 additions and 855 deletions
@@ -678,7 +678,7 @@ def test_check_moe_marlin_supports_layer_padding():
layer.hidden_size = hidden
layer.apply_router_weight_on_input = False
layer.moe_config = SimpleNamespace(
intermediate_size_per_partition_unpadded=intermediate
hidden_dim=hidden, intermediate_size_per_partition_unpadded=intermediate
)
return layer
@@ -928,12 +928,12 @@ def test_compressed_tensors_mxfp8_moe_setup(vllm_runner):
(None, 32, 64, 128, (False, 64, True)),
],
)
def test_wna16_marlin_moe_w2_scale_sharding(actorder, group_size, part, full, expected):
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe.compressed_tensors_moe_wna16_marlin import ( # noqa: E501
CompressedTensorsWNA16MarlinMoEMethod,
def test_wna16_moe_w2_scale_sharding(actorder, group_size, part, full, expected):
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe.compressed_tensors_moe_wna16 import ( # noqa: E501
CompressedTensorsWNA16MoEMethod,
)
result = CompressedTensorsWNA16MarlinMoEMethod._w2_scale_sharding(
result = CompressedTensorsWNA16MoEMethod._w2_scale_sharding(
actorder, group_size, part, full
)
assert result == expected
+6
View File
@@ -91,11 +91,17 @@ def test_map_wna16_backend_supports_triton():
def test_wna16_oracle_rejects_incompatible_quant_structures(
backend, quant_config, may_have_zp, may_have_bias, expected
):
from tests.kernels.moe.utils import make_dummy_moe_config
moe_config = make_dummy_moe_config()
reason = _backend_incompatibility_reason(
backend=backend,
moe_config=moe_config,
quant_config=quant_config,
may_have_zp=may_have_zp,
may_have_bias=may_have_bias,
allow_tile_padding=True,
)
assert reason is not None
@@ -593,7 +593,6 @@ class TritonWNA16Experts(TritonExperts):
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
# Why?
return not (
moe_parallel_config.use_fi_nvl_two_sided_kernels
or moe_parallel_config.use_fi_nvl_one_sided_kernels
@@ -32,6 +32,7 @@ from vllm.model_executor.layers.fused_moe.experts.trtllm_mxint4_moe import (
)
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
check_moe_marlin_supports_config,
marlin_act_int8_process_scales,
marlin_moe_padded_intermediate,
marlin_moe_permute_scales,
@@ -125,9 +126,11 @@ def _get_priority_backends() -> list[WNA16MoEBackend]:
def _backend_incompatibility_reason(
backend: WNA16MoEBackend,
moe_config: FusedMoEConfig,
quant_config: QuantizationConfig | QuantizationArgs,
may_have_zp: bool,
may_have_bias: bool,
allow_tile_padding: bool,
) -> str | None:
if backend == WNA16MoEBackend.FLASHINFER_TRTLLM and (may_have_zp or may_have_bias):
return "zero points and bias are not supported"
@@ -149,7 +152,24 @@ def _backend_incompatibility_reason(
):
return "group activation ordering is not supported"
if isinstance(quant_config, MoeWNA16Config) and backend in (
# Marlin only supports certain problem/group sizes.
allow_marlin = not isinstance(quant_config, MoeWNA16Config)
if allow_marlin and backend in (
WNA16MoEBackend.MARLIN,
WNA16MoEBackend.BATCHED_MARLIN,
):
if isinstance(quant_config, (AutoAWQConfig, AutoGPTQConfig, QuantizationArgs)):
group_size = quant_config.group_size
else:
return "Marlin not supported for this layer"
if not check_moe_marlin_supports_config(
moe_config, group_size, allow_tile_padding
):
return "Marlin not supported for this layer"
if not allow_marlin and backend in (
WNA16MoEBackend.MARLIN,
WNA16MoEBackend.BATCHED_MARLIN,
WNA16MoEBackend.EMULATION,
@@ -182,6 +202,7 @@ def select_wna16_moe_backend(
quant_config: QuantizationConfig | QuantizationArgs,
may_have_zp: bool,
may_have_bias: bool,
allow_tile_padding: bool = False,
) -> tuple[WNA16MoEBackend, type[mk.FusedMoEExperts]]:
"""Select the WNA16 MoE backend.
@@ -239,7 +260,12 @@ def select_wna16_moe_backend(
if runner_backend != "auto":
requested_backend = map_wna16_backend(runner_backend)
reason = _backend_incompatibility_reason(
requested_backend, quant_config, may_have_zp, may_have_bias
requested_backend,
config,
quant_config,
may_have_zp,
may_have_bias,
allow_tile_padding,
)
if reason is not None:
raise ValueError(_make_log_unsupported(requested_backend, reason))
@@ -252,7 +278,12 @@ def select_wna16_moe_backend(
for backend in AVAILABLE_BACKENDS:
reason = _backend_incompatibility_reason(
backend, quant_config, may_have_zp, may_have_bias
backend,
config,
quant_config,
may_have_zp,
may_have_bias,
allow_tile_padding,
)
if reason is not None:
logger.debug_once(_make_log_unsupported(backend, reason), scope="local")
@@ -206,7 +206,6 @@ class RoutedExperts(PluggableLayer):
def _needs_intermediate_size_param(self, quant_method: FusedMoEMethodBase) -> bool:
return quant_method.__class__.__name__ in (
"AutoGPTQMoEMethod",
"CompressedTensorsWNA16MarlinMoEMethod",
"CompressedTensorsWNA16MoEMethod",
"CompressedTensorsW4A16FlydslMoEMethod",
)
@@ -659,7 +658,6 @@ class RoutedExperts(PluggableLayer):
# TODO (mgoin): check self.quant_method.quant_config.quant_format
# against known CompressionFormat enum values that have this quality
if quant_method_name in (
"CompressedTensorsWNA16MarlinMoEMethod",
"CompressedTensorsWNA16MoEMethod",
"CompressedTensorsWNA16RDNA3MoEMethod",
"CompressedTensorsW4A16FlydslMoEMethod",
@@ -563,6 +563,7 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
quant_config=self.quant_config,
may_have_zp=self.quant_config.zero_point,
may_have_bias=True,
allow_tile_padding=True,
)
def create_weights(
@@ -782,16 +783,6 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
gemm1_beta=getattr(layer, "swiglu_beta", None),
)
def select_gemm_impl(
self,
prepare_finalize,
layer: RoutedExperts,
):
raise ValueError(
f"{self.__class__.__name__} uses the new modular kernel "
"initialization logic. This function should not be called."
)
def apply(
self,
layer: RoutedExperts,
@@ -490,8 +490,9 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
moe,
weight_key,
quant_config=self.quant_config,
may_have_zp=True,
may_have_zp=not self.quant_config.is_sym,
may_have_bias=True,
allow_tile_padding=not self.quant_config.desc_act,
)
def create_weights(
@@ -806,16 +807,6 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
w2_bias=getattr(layer, "w2_bias", None),
)
def select_gemm_impl(
self,
prepare_finalize,
layer: RoutedExperts,
):
raise ValueError(
f"{self.__class__.__name__} uses the new modular kernel "
"initialization logic. This function should not be called."
)
def apply(
self,
layer: RoutedExperts,
@@ -5,7 +5,6 @@
import torch
from compressed_tensors import CompressionFormat
from compressed_tensors.quantization import (
ActivationOrdering,
QuantizationStrategy,
QuantizationType,
)
@@ -19,9 +18,6 @@ from vllm.model_executor.layers.fused_moe import (
from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa
WNA16_SUPPORTED_BITS,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
check_moe_marlin_supports_layer,
)
from vllm.platforms import current_platform
logger = init_logger(__name__)
@@ -78,9 +74,6 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
return CompressedTensorsW8A8Mxfp8MoEMethod(layer.moe_config)
if quant_config._is_wNa16_group_channel(weight_quant, input_quant):
# group_size=None means channelwise
group_size = weight_quant.group_size or -1
valid_format_and_bits = (
weight_quant.num_bits in WNA16_SUPPORTED_BITS
and format == CompressionFormat.pack_quantized.value
@@ -95,87 +88,53 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
f" and bits: {weight_quant.num_bits}",
)
# Prefer to use the MarlinMoE kernel when it is supported.
is_actorder = (
weight_quant.strategy == QuantizationStrategy.GROUP
and weight_quant.actorder
in (ActivationOrdering.GROUP, ActivationOrdering.DYNAMIC)
)
if (
not check_moe_marlin_supports_layer(
layer, group_size, allow_tile_padding=not is_actorder
)
or current_platform.is_rocm()
):
if is_actorder:
raise ValueError(
"WNA16MoE is not supported with actorder=group/dynamic."
# Native ROCm HIP kernels (RDNA3, etc.)
if current_platform.is_rocm():
from . import rocm_moe_rdna
if rocm_moe_rdna.is_supported(weight_quant):
return rocm_moe_rdna.make_method(
weight_quant, input_quant, layer.moe_config
)
from vllm.platforms.rocm import on_gfx950
vllm_config = get_current_vllm_config()
is_lora_disabled = vllm_config.lora_config is None
moe_backend = vllm_config.kernel_config.moe_backend
group_size = weight_quant.group_size or -1
if (
weight_quant.strategy == QuantizationStrategy.GROUP
and weight_quant.type == QuantizationType.INT
and group_size == 32
and weight_quant.num_bits == 4
and is_lora_disabled
and on_gfx950()
and moe_backend == "flydsl"
):
from .compressed_tensors_moe_w4a16_flydsl import (
CompressedTensorsW4A16FlydslMoEMethod,
)
# Native ROCm HIP kernels (RDNA3, etc.)
if current_platform.is_rocm():
from . import rocm_moe_rdna
logger.info_once("Using CompressedTensorsW4A16FlydslMoEMethod")
return CompressedTensorsW4A16FlydslMoEMethod(
weight_quant, input_quant, layer.moe_config
)
elif moe_backend == "emulation":
logger.info_once(
"Using CompressedTensorsWNA16MoEMethod "
"(emulation backend requested)"
)
if rocm_moe_rdna.is_supported(weight_quant):
return rocm_moe_rdna.make_method(
weight_quant, input_quant, layer.moe_config
)
from vllm.platforms.rocm import on_gfx950
from .compressed_tensors_moe_wna16 import (
CompressedTensorsWNA16MoEMethod,
)
vllm_config = get_current_vllm_config()
is_lora_disabled = vllm_config.lora_config is None
moe_backend = vllm_config.kernel_config.moe_backend
if (
weight_quant.strategy == QuantizationStrategy.GROUP
and weight_quant.type == QuantizationType.INT
and group_size == 32
and weight_quant.num_bits == 4
and is_lora_disabled
and on_gfx950()
and moe_backend == "flydsl"
):
from .compressed_tensors_moe_w4a16_flydsl import (
CompressedTensorsW4A16FlydslMoEMethod,
)
logger.info_once("Using CompressedTensorsW4A16FlydslMoEMethod")
return CompressedTensorsW4A16FlydslMoEMethod(
weight_quant, input_quant, layer.moe_config
)
elif moe_backend == "emulation":
# Although this is called 'Marlin', actually it selects
# emulation backend by calling select_wna16_moe_backend.
# TODO: we need to update CompressedTensorsWNA16MoeMethod
# to honor "--moe-backend" option
from .compressed_tensors_moe_wna16_marlin import (
CompressedTensorsWNA16MarlinMoEMethod,
)
logger.info_once(
"Using CompressedTensorsWNA16MarlinMoEMethod "
"(emulation backend requested)"
)
return CompressedTensorsWNA16MarlinMoEMethod(
weight_quant, input_quant, layer.moe_config, layer_name
)
from .compressed_tensors_moe_wna16 import (
CompressedTensorsWNA16MoEMethod,
)
logger.info_once("Using CompressedTensorsWNA16MoEMethod")
return CompressedTensorsWNA16MoEMethod(
weight_quant, input_quant, layer.moe_config
)
else:
from .compressed_tensors_moe_wna16_marlin import (
CompressedTensorsWNA16MarlinMoEMethod,
)
logger.info_once("Using CompressedTensorsWNA16MarlinMoEMethod")
return CompressedTensorsWNA16MarlinMoEMethod(
weight_quant, input_quant, layer.moe_config
)
logger.info_once("Using CompressedTensorsWNA16MoEMethod")
return CompressedTensorsWNA16MoEMethod(
weight_quant,
input_quant,
layer.moe_config,
)
elif quant_config._is_nvfp4_format(weight_quant):
from .compressed_tensors_moe_w4a4_nvfp4 import (
CompressedTensorsW4A4Nvfp4MoEMethod,
@@ -1,28 +1,51 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import torch
from compressed_tensors.quantization import (
ActivationOrdering,
QuantizationArgs,
QuantizationStrategy,
)
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
FusedMoEExpertsModular,
RoutedExperts,
SharedExperts,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEQuantConfig,
int4_w4a16_moe_quant_config,
int8_w8a16_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
WNA16MoEBackend,
convert_to_wna16_moe_kernel_format,
make_wna16_moe_kernel,
make_wna16_moe_quant_config,
select_wna16_moe_backend,
)
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa E501
CompressedTensorsMoEMethod,
)
from vllm.model_executor.utils import set_weight_attrs
from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa
WNA16_SUPPORTED_TYPES_MAP,
WNA16_ZP_SUPPORTED_TYPES_MAP,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
check_moe_marlin_supports_config,
get_marlin_input_dtype,
marlin_make_workspace_new,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kInt4Static32GroupScale,
kInt4StaticGroupScale,
kInt8StaticGroupScale,
)
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
logger = init_logger(__name__)
@@ -39,19 +62,180 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
self.weight_quant = weight_quant
self.input_quant = input_quant
# Extract properties from weight_quant
self.symmetric = weight_quant.symmetric
self.num_bits = weight_quant.num_bits
self.packed_factor = 32 // weight_quant.num_bits
self.strategy = weight_quant.strategy
# channelwise is not supported by this kernel
assert weight_quant.strategy == "group"
self.group_size = weight_quant.group_size
# grouped actorder isn't supported by this kernel
assert weight_quant.actorder != "group"
assert weight_quant.symmetric, (
"Only symmetric quantization is supported for MoE. "
"Try --moe-backend emulation."
self.actorder = weight_quant.actorder
# Extract quant_type and create weight key for oracle selection
self.quant_type = (
WNA16_SUPPORTED_TYPES_MAP[self.num_bits]
if self.symmetric
else WNA16_ZP_SUPPORTED_TYPES_MAP[self.num_bits]
)
if self.num_bits == 4:
if self.group_size == 32:
scale = kInt4Static32GroupScale
else:
scale = kInt4StaticGroupScale
elif self.num_bits == 8:
assert self.group_size == -1
scale = kInt8StaticGroupScale
else:
raise ValueError(
"CompressedTensorsWNA16MoEMethod only supports int4 and int8 now."
)
weight_key = QuantKey(self.quant_type, scale, symmetric=self.symmetric)
is_actorder = self.strategy == QuantizationStrategy.GROUP and self.actorder in (
ActivationOrdering.GROUP,
ActivationOrdering.DYNAMIC,
)
# Select WNA16 MoE backend via oracle.
self.wna16_backend, self.experts_cls = select_wna16_moe_backend(
config=self.moe,
weight_key=weight_key,
quant_config=self.weight_quant,
may_have_zp=not self.symmetric,
may_have_bias=False,
allow_tile_padding=not is_actorder,
)
self.is_marlin = self.wna16_backend in [
WNA16MoEBackend.MARLIN,
WNA16MoEBackend.BATCHED_MARLIN,
]
self.is_transposed = self.wna16_backend != WNA16MoEBackend.FLASHINFER_TRTLLM
if self.is_marlin:
assert check_moe_marlin_supports_config(
self.moe, self.group_size, allow_tile_padding=not is_actorder
)
self.input_dtype = get_marlin_input_dtype(layer_name)
else:
# channelwise is not supported by this kernel
assert weight_quant.strategy == "group"
# grouped actorder isn't supported by this kernel
assert weight_quant.actorder != "group"
assert self.symmetric, "Only symmetric quantization is supported for MoE"
# Non-Marlin WNA16 always uses bf16/fp16 inputs
self.input_dtype = torch.bfloat16
def get_weight_shape(
self,
weight_name: str,
num_experts: int,
hidden_size: int,
intermediate_size_per_partition: int,
num_groups_w2: int | None = None,
num_groups_w13: int | None = None,
) -> tuple[int, int, int]:
"""
Get the shape of the weight based on the weight name, number of experts
hidden size, intermediate size per partition, number of groups for w2,
and number of groups for w13. Pass in num_groups_w2 and num_groups_w13
for weight scales/zero_points.
"""
if weight_name in ("w13_scale", "w13_zp"):
assert num_groups_w13 is not None, (
"num_groups_w13 must be provided for weight scales/zero_points"
)
if weight_name in ("w2_scale", "w2_zp"):
assert num_groups_w2 is not None, (
"num_groups_w2 must be provided for weight scales/zero_points"
)
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
shape_map = {
"w13_weight": {
"Flashinfer": (
num_experts,
w13_num_shards * intermediate_size_per_partition,
hidden_size // self.packed_factor,
),
"Marlin": (
num_experts,
hidden_size // self.packed_factor,
w13_num_shards * intermediate_size_per_partition,
),
},
"w13_scale": {
"Flashinfer": (
num_experts,
w13_num_shards * intermediate_size_per_partition,
num_groups_w13,
),
"Marlin": (
num_experts,
num_groups_w13,
w13_num_shards * intermediate_size_per_partition,
),
},
"w13_zp": {
"Marlin": (
num_experts,
num_groups_w13,
w13_num_shards
* intermediate_size_per_partition
// self.packed_factor,
),
},
"w2_weight": {
"Flashinfer": (
num_experts,
hidden_size,
intermediate_size_per_partition // self.packed_factor,
),
"Marlin": (
num_experts,
intermediate_size_per_partition // self.packed_factor,
hidden_size,
),
},
"w2_scale": {
"Flashinfer": (num_experts, hidden_size, num_groups_w2),
"Marlin": (num_experts, num_groups_w2, hidden_size),
},
"w2_zp": {
"Marlin": (
num_experts,
num_groups_w2,
hidden_size // self.packed_factor,
),
},
}
backend_key = "Marlin" if self.is_transposed else "Flashinfer"
return shape_map[weight_name][backend_key]
@staticmethod
def _w2_scale_sharding(
actorder,
group_size: int,
intermediate_size_per_partition: int,
intermediate_size_full: int,
) -> tuple[bool, int, bool]:
"""Decide how to shard w2 group scales across TP for WNA16 Marlin MoE.
Only ``actorder="group"`` permutes activations by ``g_idx`` at runtime
and therefore needs the full-K (unsharded) w2 scales plus ``is_k_full``.
``actorder="weight"``/``"static"`` (and ``None``) reorder weights at
quantization time, so scales shard normally per TP rank.
"""
load_full_w2 = (actorder == "group") and group_size != -1
w2_scales_size = (
intermediate_size_full if load_full_w2 else intermediate_size_per_partition
)
is_k_full = (actorder != "group") or (
intermediate_size_per_partition == intermediate_size_full
)
return load_full_w2, w2_scales_size, is_k_full
def create_weights(
self,
layer: torch.nn.Module,
@@ -61,18 +245,23 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
params_dtype: torch.dtype,
**extra_weight_attrs,
):
intermediate_size_full = extra_weight_attrs.pop("intermediate_size_full")
# Will transpose the loaded weight along the
# intermediate and hidden dim sizes. Will
# shard for TP along the transposed dims
extra_weight_attrs.update(
{"is_transposed": True, "quant_method": self.strategy}
{"is_transposed": self.is_transposed, "quant_method": self.strategy}
)
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
w13_weight = torch.nn.Parameter(
torch.empty(
num_experts,
hidden_size // self.packed_factor,
w13_num_shards * intermediate_size_per_partition,
*self.get_weight_shape(
"w13_weight",
num_experts,
hidden_size,
intermediate_size_per_partition,
),
dtype=torch.int32,
),
requires_grad=False,
@@ -82,9 +271,12 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
w2_weight = torch.nn.Parameter(
torch.empty(
num_experts,
intermediate_size_per_partition // self.packed_factor,
hidden_size,
*self.get_weight_shape(
"w2_weight",
num_experts,
hidden_size,
intermediate_size_per_partition,
),
dtype=torch.int32,
),
requires_grad=False,
@@ -92,7 +284,12 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
layer.register_parameter("w2_weight_packed", w2_weight)
set_weight_attrs(w2_weight, extra_weight_attrs)
w2_scales_size = intermediate_size_per_partition
load_full_w2, w2_scales_size, self.is_k_full = self._w2_scale_sharding(
self.actorder,
self.group_size,
intermediate_size_per_partition,
intermediate_size_full,
)
if self.strategy == "channel":
num_groups_w2 = num_groups_w13 = 1
@@ -104,23 +301,34 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
f"({hidden_size}) to be divisible by group_size "
f"({self.group_size})."
)
if intermediate_size_per_partition % self.group_size != 0:
if (
not load_full_w2
and intermediate_size_per_partition % self.group_size != 0
):
raise ValueError(
"CompressedTensors WNA16 MoE with static group scales "
"requires the MoE intermediate size per tensor-parallel "
f"partition ({intermediate_size_per_partition}) to be "
f"divisible by group_size ({self.group_size}). Scale "
"groups would otherwise cross TP shard boundaries; use a "
"compatible TP size or enable expert parallelism."
"CompressedTensors WNA16 MoE with static group "
"scales requires the MoE intermediate size per "
"tensor-parallel partition "
f"({intermediate_size_per_partition}) to be divisible by "
f"group_size ({self.group_size}). Scale groups would "
"otherwise cross TP shard boundaries; use a compatible TP "
"size or enable expert parallelism."
)
num_groups_w2 = w2_scales_size // self.group_size
num_groups_w13 = hidden_size // self.group_size
layer.num_groups_w13 = num_groups_w13
layer.num_groups_w2 = num_groups_w2
w13_scale = torch.nn.Parameter(
torch.ones(
num_experts,
num_groups_w13,
w13_num_shards * intermediate_size_per_partition,
*self.get_weight_shape(
"w13_scale",
num_experts,
hidden_size,
intermediate_size_per_partition,
num_groups_w13=num_groups_w13,
),
dtype=params_dtype,
),
requires_grad=False,
@@ -129,12 +337,54 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
set_weight_attrs(w13_scale, extra_weight_attrs)
w2_scale = torch.nn.Parameter(
torch.ones(num_experts, num_groups_w2, hidden_size, dtype=params_dtype),
torch.ones(
*self.get_weight_shape(
"w2_scale",
num_experts,
hidden_size,
intermediate_size_per_partition,
num_groups_w2=num_groups_w2,
),
dtype=params_dtype,
),
requires_grad=False,
)
layer.register_parameter("w2_weight_scale", w2_scale)
set_weight_attrs(w2_scale, extra_weight_attrs)
set_weight_attrs(w2_scale, {"load_full_w2": False})
set_weight_attrs(w2_scale, {"load_full_w2": load_full_w2})
if not self.symmetric:
w13_zp = torch.nn.Parameter(
torch.zeros(
*self.get_weight_shape(
"w13_zp",
num_experts,
hidden_size,
intermediate_size_per_partition,
num_groups_w13=num_groups_w13,
),
dtype=torch.int32,
),
requires_grad=False,
)
layer.register_parameter("w13_weight_zero_point", w13_zp)
set_weight_attrs(w13_zp, extra_weight_attrs)
w2_zp = torch.nn.Parameter(
torch.zeros(
*self.get_weight_shape(
"w2_zp",
num_experts,
hidden_size,
intermediate_size_per_partition,
num_groups_w2=num_groups_w2,
),
dtype=torch.int32,
),
requires_grad=False,
)
layer.register_parameter("w2_weight_zero_point", w2_zp)
set_weight_attrs(w2_zp, extra_weight_attrs)
w2_weight_shape = torch.nn.Parameter(
torch.empty(num_experts, 2), requires_grad=False
@@ -195,65 +445,161 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
layer.a13_scale = None
layer.a2_scale = None
def _setup_kernel(self, layer: RoutedExperts):
assert self.experts_cls is not None
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
assert self.moe_quant_config is not None
# Add Marlin-specific arguments
marlin_args: dict[str, Any] = {}
if self.is_marlin:
marlin_args = {
"w13_g_idx": layer.w13_weight_g_idx,
"w2_g_idx": layer.w2_weight_g_idx,
"w13_g_idx_sort_indices": layer.w13_g_idx_sort_indices,
"w2_g_idx_sort_indices": layer.w2_g_idx_sort_indices,
"is_k_full": self.is_k_full,
}
self.moe_kernel = make_wna16_moe_kernel(
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
**marlin_args,
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# Reconfigure packed weights and scales to match moe_wna16 format
layer.w13_weight_packed = torch.nn.Parameter(
layer.w13_weight_packed.transpose(1, 2).contiguous().view(torch.uint8),
requires_grad=False,
)
layer.w2_weight_packed = torch.nn.Parameter(
layer.w2_weight_packed.transpose(1, 2).contiguous().view(torch.uint8),
requires_grad=False,
)
layer.w13_weight_scale = torch.nn.Parameter(
layer.w13_weight_scale.transpose(1, 2).contiguous(), requires_grad=False
)
layer.w2_weight_scale = torch.nn.Parameter(
layer.w2_weight_scale.transpose(1, 2).contiguous(), requires_grad=False
# Process weights using the shared oracle infrastructure
converted = convert_to_wna16_moe_kernel_format(
backend=self.wna16_backend,
layer=layer,
quant_config=self.weight_quant,
input_dtype=self.input_dtype,
w13=layer.w13_weight_packed,
w2=layer.w2_weight_packed,
w13_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
w13_g_idx=layer.w13_weight_g_idx,
w2_g_idx=layer.w2_weight_g_idx,
w13_qzeros=getattr(layer, "w13_weight_zero_point", None),
w2_qzeros=getattr(layer, "w2_weight_zero_point", None),
)
if converted is None:
self._setup_kernel(layer)
return
(
w13_qweight,
w2_qweight,
w13_scales,
w2_scales,
w13_g_idx_processed,
w2_g_idx_processed,
w13_g_idx_sort_indices,
w2_g_idx_sort_indices,
w13_qzeros,
w2_qzeros,
w13_input_global_scale,
w2_input_global_scale,
_, # w13_bias
_, # w2_bias
) = converted
# Replace common parameters
replace_parameter(layer, "w13_weight_packed", w13_qweight)
replace_parameter(layer, "w2_weight_packed", w2_qweight)
replace_parameter(layer, "w13_weight_scale", w13_scales)
replace_parameter(layer, "w2_weight_scale", w2_scales)
# CPU fused_experts_cpu requires zero points even for symmetric quant
if not self.symmetric or self.wna16_backend == WNA16MoEBackend.CPU:
assert w13_qzeros is not None and w2_qzeros is not None
replace_parameter(layer, "w13_weight_zero_point", w13_qzeros)
replace_parameter(layer, "w2_weight_zero_point", w2_qzeros)
# Marlin-specific parameters (not needed for Flashinfer)
if self.is_marlin:
if w13_g_idx_processed is not None:
replace_parameter(layer, "w13_weight_g_idx", w13_g_idx_processed)
if w2_g_idx_processed is not None:
replace_parameter(layer, "w2_weight_g_idx", w2_g_idx_processed)
if w13_g_idx_sort_indices is not None:
replace_parameter(
layer, "w13_g_idx_sort_indices", w13_g_idx_sort_indices
)
if w2_g_idx_sort_indices is not None:
replace_parameter(layer, "w2_g_idx_sort_indices", w2_g_idx_sort_indices)
# Register input global scales if present
if w13_input_global_scale is not None:
layer.register_parameter(
"w13_input_global_scale",
torch.nn.Parameter(w13_input_global_scale, requires_grad=False),
)
if w2_input_global_scale is not None:
layer.register_parameter(
"w2_input_global_scale",
torch.nn.Parameter(w2_input_global_scale, requires_grad=False),
)
# Marlin workspace — only needed for Marlin-family backends, not emulation.
if (
self.experts_cls is not None
and issubclass(self.experts_cls, FusedMoEExpertsModular)
and self.wna16_backend != WNA16MoEBackend.EMULATION
):
layer.workspace = marlin_make_workspace_new(
layer.w13_weight_g_idx.device,
4,
existing=getattr(layer, "workspace", None),
)
# Alias packed weights to w13_weight/w2_weight for the modular kernel interface
layer.w13_weight = layer.w13_weight_packed
layer.w2_weight = layer.w2_weight_packed
self._setup_kernel(layer)
def get_fused_moe_quant_config(
self, layer: torch.nn.Module
) -> FusedMoEQuantConfig | None:
assert self.num_bits == 4 or self.num_bits == 8
config_builder = (
int4_w4a16_moe_quant_config
if self.num_bits == 4
else int8_w8a16_moe_quant_config
)
return config_builder(
return make_wna16_moe_quant_config(
w1_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
w1_zp=None,
w2_zp=None,
block_shape=[0, self.group_size],
group_size=self.group_size,
num_bits=self.num_bits,
w1_zp=getattr(layer, "w13_weight_zero_point", None),
w2_zp=getattr(layer, "w2_weight_zero_point", None),
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
)
def select_gemm_impl(
def apply_monolithic(
self,
prepare_finalize: mk.FusedMoEPrepareAndFinalizeModular,
layer: torch.nn.Module,
) -> mk.FusedMoEExpertsModular:
if self.moe.is_lora_enabled:
assert self.moe_quant_config is not None
from vllm.triton_utils import HAS_TRITON
if HAS_TRITON:
from vllm.model_executor.layers.fused_moe import TritonWNA16Experts
layer.w13_weight = layer.w13_weight_packed
layer.w2_weight = layer.w2_weight_packed
return TritonWNA16Experts(
moe_config=self.moe, quant_config=self.moe_quant_config
)
else:
raise NotImplementedError(
"TritonExperts requires Triton. "
"Install triton or disable LoRA for MoE."
)
raise NotImplementedError
layer: RoutedExperts,
x: torch.Tensor,
router_logits: torch.Tensor,
input_ids: torch.Tensor | None = None,
) -> torch.Tensor:
assert self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply_monolithic(
x,
layer.w13_weight,
layer.w2_weight,
router_logits,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
num_expert_group=layer.num_expert_group,
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
)
def apply(
self,
@@ -264,21 +610,22 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
from vllm.model_executor.layers.fused_moe import fused_experts
return fused_experts(
assert not self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply(
x,
layer.w13_weight_packed,
layer.w2_weight_packed,
layer.w13_weight,
layer.w2_weight,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=layer.activation,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
quant_config=self.moe_quant_config,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
shared_experts=shared_experts,
shared_experts_input=shared_experts_input,
)
@property
def supports_eplb(self) -> bool:
return True
return self.wna16_backend == WNA16MoEBackend.TRITON
@@ -1,601 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import torch
from compressed_tensors.quantization import (
QuantizationArgs,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
FusedMoEExpertsModular,
RoutedExperts,
SharedExperts,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
WNA16MoEBackend,
convert_to_wna16_moe_kernel_format,
make_wna16_moe_kernel,
make_wna16_moe_quant_config,
select_wna16_moe_backend,
)
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa E501
CompressedTensorsMoEMethod,
)
from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa
WNA16_SUPPORTED_TYPES_MAP,
WNA16_ZP_SUPPORTED_TYPES_MAP,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
get_marlin_input_dtype,
marlin_make_workspace_new,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kInt4Static32GroupScale,
kInt4StaticGroupScale,
kInt8StaticGroupScale,
)
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
logger = init_logger(__name__)
class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
def __init__(
self,
weight_quant: QuantizationArgs,
input_quant: QuantizationArgs | None,
moe: FusedMoEConfig,
layer_name: str | None = None,
):
super().__init__(moe)
self.weight_quant = weight_quant
self.input_quant = input_quant
self.symmetric = weight_quant.symmetric
# Extract properties from weight_quant
self.num_bits = weight_quant.num_bits
self.packed_factor = 32 // weight_quant.num_bits
self.strategy = weight_quant.strategy
self.group_size = weight_quant.group_size
self.actorder = weight_quant.actorder
self.quant_type = (
WNA16_SUPPORTED_TYPES_MAP[self.num_bits]
if self.symmetric
else WNA16_ZP_SUPPORTED_TYPES_MAP[self.num_bits]
)
self.marlin_input_dtype = get_marlin_input_dtype(layer_name)
if self.num_bits == 4:
if self.group_size == 32:
scale = kInt4Static32GroupScale
else:
scale = kInt4StaticGroupScale
elif self.num_bits == 8:
scale = kInt8StaticGroupScale
else:
raise ValueError(
"CompressedTensorsWNA16MarlinMoEMethod only supports int4 and int8 now."
)
weight_key = QuantKey(self.quant_type, scale, symmetric=self.symmetric)
# Select WNA16 MoE backend via oracle.
self.wna16_backend, self.experts_cls = select_wna16_moe_backend(
config=self.moe,
weight_key=weight_key,
quant_config=self.weight_quant,
may_have_zp=not self.symmetric,
may_have_bias=False,
)
self.is_marlin = self.wna16_backend in [
WNA16MoEBackend.MARLIN,
WNA16MoEBackend.BATCHED_MARLIN,
]
self.is_transposed = self.wna16_backend != WNA16MoEBackend.FLASHINFER_TRTLLM
def get_weight_shape(
self,
weight_name: str,
num_experts: int,
hidden_size: int,
intermediate_size_per_partition: int,
num_groups_w2: int | None = None,
num_groups_w13: int | None = None,
) -> tuple[int, int, int]:
"""
Get the shape of the weight based on the weight name, number of experts
hidden size, intermediate size per partition, number of groups for w2,
and number of groups for w13. Pass in num_groups_w2 and num_groups_w13
for weight scales/zero_points.
"""
if weight_name in ("w13_scale", "w13_zp"):
assert num_groups_w13 is not None, (
"num_groups_w13 must be provided for weight scales/zero_points"
)
if weight_name in ("w2_scale", "w2_zp"):
assert num_groups_w2 is not None, (
"num_groups_w2 must be provided for weight scales/zero_points"
)
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
shape_map = {
"w13_weight": {
"Flashinfer": (
num_experts,
w13_num_shards * intermediate_size_per_partition,
hidden_size // self.packed_factor,
),
"Marlin": (
num_experts,
hidden_size // self.packed_factor,
w13_num_shards * intermediate_size_per_partition,
),
},
"w13_scale": {
"Flashinfer": (
num_experts,
w13_num_shards * intermediate_size_per_partition,
num_groups_w13,
),
"Marlin": (
num_experts,
num_groups_w13,
w13_num_shards * intermediate_size_per_partition,
),
},
"w13_zp": {
"Marlin": (
num_experts,
num_groups_w13,
w13_num_shards
* intermediate_size_per_partition
// self.packed_factor,
),
},
"w2_weight": {
"Flashinfer": (
num_experts,
hidden_size,
intermediate_size_per_partition // self.packed_factor,
),
"Marlin": (
num_experts,
intermediate_size_per_partition // self.packed_factor,
hidden_size,
),
},
"w2_scale": {
"Flashinfer": (num_experts, hidden_size, num_groups_w2),
"Marlin": (num_experts, num_groups_w2, hidden_size),
},
"w2_zp": {
"Marlin": (
num_experts,
num_groups_w2,
hidden_size // self.packed_factor,
),
},
}
backend_key = "Marlin" if self.is_transposed else "Flashinfer"
return shape_map[weight_name][backend_key]
@staticmethod
def _w2_scale_sharding(
actorder,
group_size: int,
intermediate_size_per_partition: int,
intermediate_size_full: int,
) -> tuple[bool, int, bool]:
"""Decide how to shard w2 group scales across TP for WNA16 Marlin MoE.
Only ``actorder="group"`` permutes activations by ``g_idx`` at runtime
and therefore needs the full-K (unsharded) w2 scales plus ``is_k_full``.
``actorder="weight"``/``"static"`` (and ``None``) reorder weights at
quantization time, so scales shard normally per TP rank.
"""
load_full_w2 = (actorder == "group") and group_size != -1
w2_scales_size = (
intermediate_size_full if load_full_w2 else intermediate_size_per_partition
)
is_k_full = (actorder != "group") or (
intermediate_size_per_partition == intermediate_size_full
)
return load_full_w2, w2_scales_size, is_k_full
def create_weights(
self,
layer: torch.nn.Module,
num_experts: int,
hidden_size: int,
intermediate_size_per_partition: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
):
intermediate_size_full = extra_weight_attrs.pop("intermediate_size_full")
# Will transpose the loaded weight along the
# intermediate and hidden dim sizes. Will
# shard for TP along the transposed dims
extra_weight_attrs.update(
{"is_transposed": self.is_transposed, "quant_method": self.strategy}
)
w13_weight = torch.nn.Parameter(
torch.empty(
*self.get_weight_shape(
"w13_weight",
num_experts,
hidden_size,
intermediate_size_per_partition,
),
dtype=torch.int32,
),
requires_grad=False,
)
layer.register_parameter("w13_weight_packed", w13_weight)
set_weight_attrs(w13_weight, extra_weight_attrs)
w2_weight = torch.nn.Parameter(
torch.empty(
*self.get_weight_shape(
"w2_weight",
num_experts,
hidden_size,
intermediate_size_per_partition,
),
dtype=torch.int32,
),
requires_grad=False,
)
layer.register_parameter("w2_weight_packed", w2_weight)
set_weight_attrs(w2_weight, extra_weight_attrs)
load_full_w2, w2_scales_size, self.is_k_full = self._w2_scale_sharding(
self.actorder,
self.group_size,
intermediate_size_per_partition,
intermediate_size_full,
)
if self.strategy == "channel":
num_groups_w2 = num_groups_w13 = 1
self.group_size = -1
else:
if hidden_size % self.group_size != 0:
raise ValueError(
"CompressedTensors WNA16 Marlin MoE requires hidden_size "
f"({hidden_size}) to be divisible by group_size "
f"({self.group_size})."
)
if (
not load_full_w2
and intermediate_size_per_partition % self.group_size != 0
):
raise ValueError(
"CompressedTensors WNA16 Marlin MoE with static group "
"scales requires the MoE intermediate size per "
"tensor-parallel partition "
f"({intermediate_size_per_partition}) to be divisible by "
f"group_size ({self.group_size}). Scale groups would "
"otherwise cross TP shard boundaries; use a compatible TP "
"size or enable expert parallelism."
)
num_groups_w2 = w2_scales_size // self.group_size
num_groups_w13 = hidden_size // self.group_size
layer.num_groups_w13 = num_groups_w13
layer.num_groups_w2 = num_groups_w2
w13_scale = torch.nn.Parameter(
torch.ones(
*self.get_weight_shape(
"w13_scale",
num_experts,
hidden_size,
intermediate_size_per_partition,
num_groups_w13=num_groups_w13,
),
dtype=params_dtype,
),
requires_grad=False,
)
layer.register_parameter("w13_weight_scale", w13_scale)
set_weight_attrs(w13_scale, extra_weight_attrs)
w2_scale = torch.nn.Parameter(
torch.ones(
*self.get_weight_shape(
"w2_scale",
num_experts,
hidden_size,
intermediate_size_per_partition,
num_groups_w2=num_groups_w2,
),
dtype=params_dtype,
),
requires_grad=False,
)
layer.register_parameter("w2_weight_scale", w2_scale)
set_weight_attrs(w2_scale, extra_weight_attrs)
set_weight_attrs(w2_scale, {"load_full_w2": load_full_w2})
if not self.symmetric:
w13_zp = torch.nn.Parameter(
torch.zeros(
*self.get_weight_shape(
"w13_zp",
num_experts,
hidden_size,
intermediate_size_per_partition,
num_groups_w13=num_groups_w13,
),
dtype=torch.int32,
),
requires_grad=False,
)
layer.register_parameter("w13_weight_zero_point", w13_zp)
set_weight_attrs(w13_zp, extra_weight_attrs)
w2_zp = torch.nn.Parameter(
torch.zeros(
*self.get_weight_shape(
"w2_zp",
num_experts,
hidden_size,
intermediate_size_per_partition,
num_groups_w2=num_groups_w2,
),
dtype=torch.int32,
),
requires_grad=False,
)
layer.register_parameter("w2_weight_zero_point", w2_zp)
set_weight_attrs(w2_zp, extra_weight_attrs)
w2_weight_shape = torch.nn.Parameter(
torch.empty(num_experts, 2), requires_grad=False
)
layer.register_parameter("w2_weight_shape", w2_weight_shape)
set_weight_attrs(w2_weight_shape, extra_weight_attrs)
w13_weight_shape = torch.nn.Parameter(
torch.empty(num_experts, 2), requires_grad=False
)
layer.register_parameter("w13_weight_shape", w13_weight_shape)
set_weight_attrs(w13_weight_shape, extra_weight_attrs)
w13_g_idx = torch.nn.Parameter(
torch.empty(
num_experts,
hidden_size,
dtype=torch.int32,
),
requires_grad=False,
)
layer.register_parameter("w13_weight_g_idx", w13_g_idx)
set_weight_attrs(w13_g_idx, extra_weight_attrs)
w2_g_idx = torch.nn.Parameter(
torch.empty(
num_experts,
intermediate_size_per_partition,
dtype=torch.int32,
),
requires_grad=False,
)
layer.register_parameter("w2_weight_g_idx", w2_g_idx)
set_weight_attrs(w2_g_idx, extra_weight_attrs)
w13_g_idx_sort_indices = torch.nn.Parameter(
torch.empty(
num_experts,
hidden_size,
dtype=torch.int32,
),
requires_grad=False,
)
layer.register_parameter("w13_g_idx_sort_indices", w13_g_idx_sort_indices)
set_weight_attrs(w13_g_idx_sort_indices, extra_weight_attrs)
w2_g_idx_sort_indices = torch.nn.Parameter(
torch.empty(
num_experts,
intermediate_size_per_partition,
dtype=torch.int32,
),
requires_grad=False,
)
layer.register_parameter("w2_g_idx_sort_indices", w2_g_idx_sort_indices)
set_weight_attrs(w2_g_idx_sort_indices, extra_weight_attrs)
layer.a13_scale = None
layer.a2_scale = None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# Process weights using the shared oracle infrastructure
converted = convert_to_wna16_moe_kernel_format(
backend=self.wna16_backend,
layer=layer,
quant_config=self.weight_quant,
input_dtype=self.marlin_input_dtype,
w13=layer.w13_weight_packed,
w2=layer.w2_weight_packed,
w13_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
w13_g_idx=layer.w13_weight_g_idx,
w2_g_idx=layer.w2_weight_g_idx,
w13_qzeros=getattr(layer, "w13_weight_zero_point", None),
w2_qzeros=getattr(layer, "w2_weight_zero_point", None),
)
if converted is None:
# In-place backends (e.g. Humming) are not wired through this
# marlin-only method; fail clearly rather than unpacking None.
raise NotImplementedError(
f"{type(self).__name__} does not support the "
f"{self.wna16_backend.value} MoE backend."
)
(
w13_qweight,
w2_qweight,
w13_scales,
w2_scales,
w13_g_idx_processed,
w2_g_idx_processed,
w13_g_idx_sort_indices,
w2_g_idx_sort_indices,
w13_qzeros,
w2_qzeros,
w13_input_global_scale,
w2_input_global_scale,
_, # w13_bias
_, # w2_bias
) = converted
# Replace common parameters
replace_parameter(layer, "w13_weight_packed", w13_qweight)
replace_parameter(layer, "w2_weight_packed", w2_qweight)
replace_parameter(layer, "w13_weight_scale", w13_scales)
replace_parameter(layer, "w2_weight_scale", w2_scales)
# CPU fused_experts_cpu requires zero points even for symmetric quant
if not self.symmetric or self.wna16_backend == WNA16MoEBackend.CPU:
replace_parameter(layer, "w13_weight_zero_point", w13_qzeros)
replace_parameter(layer, "w2_weight_zero_point", w2_qzeros)
# Marlin-specific parameters (not needed for Flashinfer)
if self.is_marlin:
if w13_g_idx_processed is not None:
replace_parameter(layer, "w13_weight_g_idx", w13_g_idx_processed)
if w2_g_idx_processed is not None:
replace_parameter(layer, "w2_weight_g_idx", w2_g_idx_processed)
if w13_g_idx_sort_indices is not None:
replace_parameter(
layer, "w13_g_idx_sort_indices", w13_g_idx_sort_indices
)
if w2_g_idx_sort_indices is not None:
replace_parameter(layer, "w2_g_idx_sort_indices", w2_g_idx_sort_indices)
# Register input global scales if present
if w13_input_global_scale is not None:
layer.register_parameter(
"w13_input_global_scale",
torch.nn.Parameter(w13_input_global_scale, requires_grad=False),
)
if w2_input_global_scale is not None:
layer.register_parameter(
"w2_input_global_scale",
torch.nn.Parameter(w2_input_global_scale, requires_grad=False),
)
# Marlin workspace — only needed for Marlin-family backends, not emulation.
if (
self.experts_cls is not None
and issubclass(self.experts_cls, FusedMoEExpertsModular)
and self.wna16_backend != WNA16MoEBackend.EMULATION
):
layer.workspace = marlin_make_workspace_new(
layer.w13_weight_g_idx.device,
4,
existing=getattr(layer, "workspace", None),
)
# Alias packed weights to w13_weight/w2_weight for the modular kernel interface
layer.w13_weight = layer.w13_weight_packed
layer.w2_weight = layer.w2_weight_packed
assert self.experts_cls is not None
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
assert self.moe_quant_config is not None
# Add Marlin-specific arguments
marlin_args: dict[str, Any] = {}
if self.is_marlin:
marlin_args = {
"w13_g_idx": layer.w13_weight_g_idx,
"w2_g_idx": layer.w2_weight_g_idx,
"w13_g_idx_sort_indices": layer.w13_g_idx_sort_indices,
"w2_g_idx_sort_indices": layer.w2_g_idx_sort_indices,
"is_k_full": self.is_k_full,
}
self.moe_kernel = make_wna16_moe_kernel(
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
**marlin_args,
)
def get_fused_moe_quant_config(
self, layer: torch.nn.Module
) -> FusedMoEQuantConfig | None:
return make_wna16_moe_quant_config(
w1_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
group_size=self.group_size,
num_bits=self.num_bits,
w1_zp=getattr(layer, "w13_weight_zero_point", None),
w2_zp=getattr(layer, "w2_weight_zero_point", None),
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
)
def apply_monolithic(
self,
layer: RoutedExperts,
x: torch.Tensor,
router_logits: torch.Tensor,
input_ids: torch.Tensor | None = None,
) -> torch.Tensor:
assert self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply_monolithic(
x,
layer.w13_weight,
layer.w2_weight,
router_logits,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
num_expert_group=layer.num_expert_group,
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
)
def apply(
self,
layer: RoutedExperts,
x: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
assert not self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply(
x,
layer.w13_weight,
layer.w2_weight,
topk_weights,
topk_ids,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
shared_experts=shared_experts,
shared_experts_input=shared_experts_input,
)
@@ -778,16 +778,6 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
"logic. This function should not be called."
)
def select_gemm_impl(
self,
prepare_finalize: mk.FusedMoEPrepareAndFinalizeModular,
layer: RoutedExperts,
) -> mk.FusedMoEExpertsModular:
raise ValueError(
f"{self.__class__.__name__} uses the new modular kernel initialization "
"logic. This function should not be called."
)
def create_weights(
self,
layer: RoutedExperts,
@@ -2104,16 +2094,6 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
"logic. This function should not be called."
)
def select_gemm_impl(
self,
prepare_finalize: mk.FusedMoEPrepareAndFinalizeModular,
layer: RoutedExperts,
) -> mk.FusedMoEExpertsModular:
raise ValueError(
f"{self.__class__.__name__} uses the new modular kernel initialization "
"logic. This function should not be called."
)
def get_fused_moe_quant_config(
self, layer: RoutedExperts
) -> FusedMoEQuantConfig | None:
@@ -430,16 +430,6 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
layer=layer,
)
def select_gemm_impl(
self,
prepare_finalize: mk.FusedMoEPrepareAndFinalize,
layer: RoutedExperts,
) -> mk.FusedMoEExpertsModular:
raise ValueError(
f"{self.__class__.__name__} uses the new modular kernel "
"initialization logic. This function should not be called."
)
def apply(
self,
layer: RoutedExperts,
@@ -893,16 +883,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
layer=layer,
)
def select_gemm_impl(
self,
prepare_finalize: mk.FusedMoEPrepareAndFinalize,
layer: RoutedExperts,
) -> mk.FusedMoEExpertsModular:
raise ValueError(
f"{self.__class__.__name__} uses the new modular kernel "
"initialization logic. This function should not be called."
)
def apply(
self,
layer: RoutedExperts,
@@ -12,6 +12,7 @@ from vllm import _custom_ops as ops
from vllm.distributed.utils import verify_group_size_divides_partition
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.model_executor.layers.linear import LinearBase
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
from vllm.model_executor.layers.quantization.utils.int8_utils import (
@@ -351,10 +352,12 @@ def marlin_moe_padded_intermediate(intermediate_size: int, group_size: int = -1)
return padded
def check_moe_marlin_supports_layer(
layer: RoutedExperts, group_size: int, allow_tile_padding: bool = False
def check_moe_marlin_supports_config(
config: FusedMoEConfig,
group_size: int,
allow_tile_padding: bool = False,
) -> bool:
"""Whether the fused MoE Marlin kernel supports ``layer``.
"""Whether the fused MoE Marlin kernel supports ``config``.
Callers without act-order may pass ``allow_tile_padding=True``: a
tile-misaligned intermediate size is then zero-padded to a valid thread
@@ -364,15 +367,11 @@ def check_moe_marlin_supports_layer(
"""
if current_platform.is_rocm():
return False
hidden_size = layer.hidden_size
hidden_size = config.hidden_dim
# The layer has not rounded intermediate_size yet; use the stable unpadded
# size. gate-up needs n=2*intermediate % 128, down needs k=intermediate % 64.
intermediate_size_per_partition = (
layer.moe_config.intermediate_size_per_partition_unpadded
)
intermediate_size_per_partition = config.intermediate_size_per_partition_unpadded
assert intermediate_size_per_partition is not None
# apply_router_weight_on_input is not supported for moe marlin
supports_router_weight = not layer.apply_router_weight_on_input
if allow_tile_padding:
supports_shape = hidden_size % 128 == 0 and (
@@ -384,7 +383,17 @@ def check_moe_marlin_supports_layer(
and intermediate_size_per_partition % max(64, group_size) == 0
)
supports_group_size = group_size in [-1, 32, 64, 128]
return supports_shape and supports_group_size and supports_router_weight
return supports_shape and supports_group_size
def check_moe_marlin_supports_layer(
layer: RoutedExperts,
group_size: int,
allow_tile_padding: bool = False,
) -> bool:
return check_moe_marlin_supports_config(
layer.moe_config, group_size, allow_tile_padding
)
def marlin_moe_intermediate_size(w1_packed: torch.Tensor, w2_packed: torch.Tensor):