diff --git a/tests/kernels/quantization/test_marlin_tile_padding.py b/tests/kernels/quantization/test_marlin_tile_padding.py index be987fda6d..981046fa53 100644 --- a/tests/kernels/quantization/test_marlin_tile_padding.py +++ b/tests/kernels/quantization/test_marlin_tile_padding.py @@ -678,7 +678,7 @@ def test_check_moe_marlin_supports_layer_padding(): layer.hidden_size = hidden layer.apply_router_weight_on_input = False layer.moe_config = SimpleNamespace( - intermediate_size_per_partition_unpadded=intermediate + hidden_dim=hidden, intermediate_size_per_partition_unpadded=intermediate ) return layer diff --git a/tests/quantization/test_compressed_tensors.py b/tests/quantization/test_compressed_tensors.py index 8c50988870..19f63f2286 100644 --- a/tests/quantization/test_compressed_tensors.py +++ b/tests/quantization/test_compressed_tensors.py @@ -928,12 +928,12 @@ def test_compressed_tensors_mxfp8_moe_setup(vllm_runner): (None, 32, 64, 128, (False, 64, True)), ], ) -def test_wna16_marlin_moe_w2_scale_sharding(actorder, group_size, part, full, expected): - from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe.compressed_tensors_moe_wna16_marlin import ( # noqa: E501 - CompressedTensorsWNA16MarlinMoEMethod, +def test_wna16_moe_w2_scale_sharding(actorder, group_size, part, full, expected): + from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe.compressed_tensors_moe_wna16 import ( # noqa: E501 + CompressedTensorsWNA16MoEMethod, ) - result = CompressedTensorsWNA16MarlinMoEMethod._w2_scale_sharding( + result = CompressedTensorsWNA16MoEMethod._w2_scale_sharding( actorder, group_size, part, full ) assert result == expected diff --git a/tests/quantization/test_moe_wna16.py b/tests/quantization/test_moe_wna16.py index 16c68fd1c7..58faaf4923 100644 --- a/tests/quantization/test_moe_wna16.py +++ b/tests/quantization/test_moe_wna16.py @@ -91,11 +91,17 @@ def test_map_wna16_backend_supports_triton(): def test_wna16_oracle_rejects_incompatible_quant_structures( backend, quant_config, may_have_zp, may_have_bias, expected ): + from tests.kernels.moe.utils import make_dummy_moe_config + + moe_config = make_dummy_moe_config() + reason = _backend_incompatibility_reason( backend=backend, + moe_config=moe_config, quant_config=quant_config, may_have_zp=may_have_zp, may_have_bias=may_have_bias, + allow_tile_padding=True, ) assert reason is not None diff --git a/vllm/model_executor/layers/fused_moe/experts/triton_moe.py b/vllm/model_executor/layers/fused_moe/experts/triton_moe.py index 6d5733743b..d1bab54dda 100644 --- a/vllm/model_executor/layers/fused_moe/experts/triton_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/triton_moe.py @@ -593,7 +593,6 @@ class TritonWNA16Experts(TritonExperts): @staticmethod def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool: - # Why? return not ( moe_parallel_config.use_fi_nvl_two_sided_kernels or moe_parallel_config.use_fi_nvl_one_sided_kernels diff --git a/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py b/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py index 1cd046ab26..2ac53525c9 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py +++ b/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py @@ -32,6 +32,7 @@ from vllm.model_executor.layers.fused_moe.experts.trtllm_mxint4_moe import ( ) from vllm.model_executor.layers.quantization.base_config import QuantizationConfig from vllm.model_executor.layers.quantization.utils.marlin_utils import ( + check_moe_marlin_supports_config, marlin_act_int8_process_scales, marlin_moe_padded_intermediate, marlin_moe_permute_scales, @@ -125,9 +126,11 @@ def _get_priority_backends() -> list[WNA16MoEBackend]: def _backend_incompatibility_reason( backend: WNA16MoEBackend, + moe_config: FusedMoEConfig, quant_config: QuantizationConfig | QuantizationArgs, may_have_zp: bool, may_have_bias: bool, + allow_tile_padding: bool, ) -> str | None: if backend == WNA16MoEBackend.FLASHINFER_TRTLLM and (may_have_zp or may_have_bias): return "zero points and bias are not supported" @@ -149,7 +152,24 @@ def _backend_incompatibility_reason( ): return "group activation ordering is not supported" - if isinstance(quant_config, MoeWNA16Config) and backend in ( + # Marlin only supports certain problem/group sizes. + allow_marlin = not isinstance(quant_config, MoeWNA16Config) + + if allow_marlin and backend in ( + WNA16MoEBackend.MARLIN, + WNA16MoEBackend.BATCHED_MARLIN, + ): + if isinstance(quant_config, (AutoAWQConfig, AutoGPTQConfig, QuantizationArgs)): + group_size = quant_config.group_size + else: + return "Marlin not supported for this layer" + + if not check_moe_marlin_supports_config( + moe_config, group_size, allow_tile_padding + ): + return "Marlin not supported for this layer" + + if not allow_marlin and backend in ( WNA16MoEBackend.MARLIN, WNA16MoEBackend.BATCHED_MARLIN, WNA16MoEBackend.EMULATION, @@ -182,6 +202,7 @@ def select_wna16_moe_backend( quant_config: QuantizationConfig | QuantizationArgs, may_have_zp: bool, may_have_bias: bool, + allow_tile_padding: bool = False, ) -> tuple[WNA16MoEBackend, type[mk.FusedMoEExperts]]: """Select the WNA16 MoE backend. @@ -239,7 +260,12 @@ def select_wna16_moe_backend( if runner_backend != "auto": requested_backend = map_wna16_backend(runner_backend) reason = _backend_incompatibility_reason( - requested_backend, quant_config, may_have_zp, may_have_bias + requested_backend, + config, + quant_config, + may_have_zp, + may_have_bias, + allow_tile_padding, ) if reason is not None: raise ValueError(_make_log_unsupported(requested_backend, reason)) @@ -252,7 +278,12 @@ def select_wna16_moe_backend( for backend in AVAILABLE_BACKENDS: reason = _backend_incompatibility_reason( - backend, quant_config, may_have_zp, may_have_bias + backend, + config, + quant_config, + may_have_zp, + may_have_bias, + allow_tile_padding, ) if reason is not None: logger.debug_once(_make_log_unsupported(backend, reason), scope="local") diff --git a/vllm/model_executor/layers/fused_moe/routed_experts.py b/vllm/model_executor/layers/fused_moe/routed_experts.py index f9fbe80b2c..0cd0ad9950 100644 --- a/vllm/model_executor/layers/fused_moe/routed_experts.py +++ b/vllm/model_executor/layers/fused_moe/routed_experts.py @@ -206,7 +206,6 @@ class RoutedExperts(PluggableLayer): def _needs_intermediate_size_param(self, quant_method: FusedMoEMethodBase) -> bool: return quant_method.__class__.__name__ in ( "AutoGPTQMoEMethod", - "CompressedTensorsWNA16MarlinMoEMethod", "CompressedTensorsWNA16MoEMethod", "CompressedTensorsW4A16FlydslMoEMethod", ) @@ -659,7 +658,6 @@ class RoutedExperts(PluggableLayer): # TODO (mgoin): check self.quant_method.quant_config.quant_format # against known CompressionFormat enum values that have this quality if quant_method_name in ( - "CompressedTensorsWNA16MarlinMoEMethod", "CompressedTensorsWNA16MoEMethod", "CompressedTensorsWNA16RDNA3MoEMethod", "CompressedTensorsW4A16FlydslMoEMethod", diff --git a/vllm/model_executor/layers/quantization/auto_awq.py b/vllm/model_executor/layers/quantization/auto_awq.py index a77fef0525..5e9946ff35 100644 --- a/vllm/model_executor/layers/quantization/auto_awq.py +++ b/vllm/model_executor/layers/quantization/auto_awq.py @@ -563,6 +563,7 @@ class AutoAWQMoEMethod(FusedMoEMethodBase): quant_config=self.quant_config, may_have_zp=self.quant_config.zero_point, may_have_bias=True, + allow_tile_padding=True, ) def create_weights( @@ -782,16 +783,6 @@ class AutoAWQMoEMethod(FusedMoEMethodBase): gemm1_beta=getattr(layer, "swiglu_beta", None), ) - def select_gemm_impl( - self, - prepare_finalize, - layer: RoutedExperts, - ): - raise ValueError( - f"{self.__class__.__name__} uses the new modular kernel " - "initialization logic. This function should not be called." - ) - def apply( self, layer: RoutedExperts, diff --git a/vllm/model_executor/layers/quantization/auto_gptq.py b/vllm/model_executor/layers/quantization/auto_gptq.py index f94bbb25b3..710e947f0b 100644 --- a/vllm/model_executor/layers/quantization/auto_gptq.py +++ b/vllm/model_executor/layers/quantization/auto_gptq.py @@ -490,8 +490,9 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase): moe, weight_key, quant_config=self.quant_config, - may_have_zp=True, + may_have_zp=not self.quant_config.is_sym, may_have_bias=True, + allow_tile_padding=not self.quant_config.desc_act, ) def create_weights( @@ -806,16 +807,6 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase): w2_bias=getattr(layer, "w2_bias", None), ) - def select_gemm_impl( - self, - prepare_finalize, - layer: RoutedExperts, - ): - raise ValueError( - f"{self.__class__.__name__} uses the new modular kernel " - "initialization logic. This function should not be called." - ) - def apply( self, layer: RoutedExperts, diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe.py index 7c03baf1e0..78d70d492d 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe.py @@ -5,7 +5,6 @@ import torch from compressed_tensors import CompressionFormat from compressed_tensors.quantization import ( - ActivationOrdering, QuantizationStrategy, QuantizationType, ) @@ -19,9 +18,6 @@ from vllm.model_executor.layers.fused_moe import ( from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa WNA16_SUPPORTED_BITS, ) -from vllm.model_executor.layers.quantization.utils.marlin_utils import ( - check_moe_marlin_supports_layer, -) from vllm.platforms import current_platform logger = init_logger(__name__) @@ -78,9 +74,6 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase): return CompressedTensorsW8A8Mxfp8MoEMethod(layer.moe_config) if quant_config._is_wNa16_group_channel(weight_quant, input_quant): - # group_size=None means channelwise - group_size = weight_quant.group_size or -1 - valid_format_and_bits = ( weight_quant.num_bits in WNA16_SUPPORTED_BITS and format == CompressionFormat.pack_quantized.value @@ -95,87 +88,53 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase): f" and bits: {weight_quant.num_bits}", ) - # Prefer to use the MarlinMoE kernel when it is supported. - is_actorder = ( - weight_quant.strategy == QuantizationStrategy.GROUP - and weight_quant.actorder - in (ActivationOrdering.GROUP, ActivationOrdering.DYNAMIC) - ) - if ( - not check_moe_marlin_supports_layer( - layer, group_size, allow_tile_padding=not is_actorder - ) - or current_platform.is_rocm() - ): - if is_actorder: - raise ValueError( - "WNA16MoE is not supported with actorder=group/dynamic." + # Native ROCm HIP kernels (RDNA3, etc.) + if current_platform.is_rocm(): + from . import rocm_moe_rdna + + if rocm_moe_rdna.is_supported(weight_quant): + return rocm_moe_rdna.make_method( + weight_quant, input_quant, layer.moe_config + ) + from vllm.platforms.rocm import on_gfx950 + + vllm_config = get_current_vllm_config() + is_lora_disabled = vllm_config.lora_config is None + moe_backend = vllm_config.kernel_config.moe_backend + group_size = weight_quant.group_size or -1 + if ( + weight_quant.strategy == QuantizationStrategy.GROUP + and weight_quant.type == QuantizationType.INT + and group_size == 32 + and weight_quant.num_bits == 4 + and is_lora_disabled + and on_gfx950() + and moe_backend == "flydsl" + ): + from .compressed_tensors_moe_w4a16_flydsl import ( + CompressedTensorsW4A16FlydslMoEMethod, ) - # Native ROCm HIP kernels (RDNA3, etc.) - if current_platform.is_rocm(): - from . import rocm_moe_rdna + logger.info_once("Using CompressedTensorsW4A16FlydslMoEMethod") + return CompressedTensorsW4A16FlydslMoEMethod( + weight_quant, input_quant, layer.moe_config + ) + elif moe_backend == "emulation": + logger.info_once( + "Using CompressedTensorsWNA16MoEMethod " + "(emulation backend requested)" + ) - if rocm_moe_rdna.is_supported(weight_quant): - return rocm_moe_rdna.make_method( - weight_quant, input_quant, layer.moe_config - ) - from vllm.platforms.rocm import on_gfx950 + from .compressed_tensors_moe_wna16 import ( + CompressedTensorsWNA16MoEMethod, + ) - vllm_config = get_current_vllm_config() - is_lora_disabled = vllm_config.lora_config is None - moe_backend = vllm_config.kernel_config.moe_backend - if ( - weight_quant.strategy == QuantizationStrategy.GROUP - and weight_quant.type == QuantizationType.INT - and group_size == 32 - and weight_quant.num_bits == 4 - and is_lora_disabled - and on_gfx950() - and moe_backend == "flydsl" - ): - from .compressed_tensors_moe_w4a16_flydsl import ( - CompressedTensorsW4A16FlydslMoEMethod, - ) - - logger.info_once("Using CompressedTensorsW4A16FlydslMoEMethod") - return CompressedTensorsW4A16FlydslMoEMethod( - weight_quant, input_quant, layer.moe_config - ) - elif moe_backend == "emulation": - # Although this is called 'Marlin', actually it selects - # emulation backend by calling select_wna16_moe_backend. - # TODO: we need to update CompressedTensorsWNA16MoeMethod - # to honor "--moe-backend" option - from .compressed_tensors_moe_wna16_marlin import ( - CompressedTensorsWNA16MarlinMoEMethod, - ) - - logger.info_once( - "Using CompressedTensorsWNA16MarlinMoEMethod " - "(emulation backend requested)" - ) - return CompressedTensorsWNA16MarlinMoEMethod( - weight_quant, input_quant, layer.moe_config, layer_name - ) - - from .compressed_tensors_moe_wna16 import ( - CompressedTensorsWNA16MoEMethod, - ) - - logger.info_once("Using CompressedTensorsWNA16MoEMethod") - return CompressedTensorsWNA16MoEMethod( - weight_quant, input_quant, layer.moe_config - ) - else: - from .compressed_tensors_moe_wna16_marlin import ( - CompressedTensorsWNA16MarlinMoEMethod, - ) - - logger.info_once("Using CompressedTensorsWNA16MarlinMoEMethod") - return CompressedTensorsWNA16MarlinMoEMethod( - weight_quant, input_quant, layer.moe_config - ) + logger.info_once("Using CompressedTensorsWNA16MoEMethod") + return CompressedTensorsWNA16MoEMethod( + weight_quant, + input_quant, + layer.moe_config, + ) elif quant_config._is_nvfp4_format(weight_quant): from .compressed_tensors_moe_w4a4_nvfp4 import ( CompressedTensorsW4A4Nvfp4MoEMethod, diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16.py index 2dabf7a515..eaec0fec45 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16.py @@ -1,28 +1,51 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from typing import Any import torch from compressed_tensors.quantization import ( + ActivationOrdering, QuantizationArgs, + QuantizationStrategy, ) -import vllm.model_executor.layers.fused_moe.modular_kernel as mk from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe import ( + FusedMoEExpertsModular, RoutedExperts, SharedExperts, ) from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, FusedMoEQuantConfig, - int4_w4a16_moe_quant_config, - int8_w8a16_moe_quant_config, +) +from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import ( + WNA16MoEBackend, + convert_to_wna16_moe_kernel_format, + make_wna16_moe_kernel, + make_wna16_moe_quant_config, + select_wna16_moe_backend, ) from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa E501 CompressedTensorsMoEMethod, ) -from vllm.model_executor.utils import set_weight_attrs +from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa + WNA16_SUPPORTED_TYPES_MAP, + WNA16_ZP_SUPPORTED_TYPES_MAP, +) +from vllm.model_executor.layers.quantization.utils.marlin_utils import ( + check_moe_marlin_supports_config, + get_marlin_input_dtype, + marlin_make_workspace_new, +) +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + QuantKey, + kInt4Static32GroupScale, + kInt4StaticGroupScale, + kInt8StaticGroupScale, +) +from vllm.model_executor.utils import replace_parameter, set_weight_attrs logger = init_logger(__name__) @@ -39,19 +62,180 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): self.weight_quant = weight_quant self.input_quant = input_quant # Extract properties from weight_quant + self.symmetric = weight_quant.symmetric self.num_bits = weight_quant.num_bits self.packed_factor = 32 // weight_quant.num_bits self.strategy = weight_quant.strategy - # channelwise is not supported by this kernel - assert weight_quant.strategy == "group" self.group_size = weight_quant.group_size - # grouped actorder isn't supported by this kernel - assert weight_quant.actorder != "group" - assert weight_quant.symmetric, ( - "Only symmetric quantization is supported for MoE. " - "Try --moe-backend emulation." + self.actorder = weight_quant.actorder + + # Extract quant_type and create weight key for oracle selection + self.quant_type = ( + WNA16_SUPPORTED_TYPES_MAP[self.num_bits] + if self.symmetric + else WNA16_ZP_SUPPORTED_TYPES_MAP[self.num_bits] ) + if self.num_bits == 4: + if self.group_size == 32: + scale = kInt4Static32GroupScale + else: + scale = kInt4StaticGroupScale + elif self.num_bits == 8: + assert self.group_size == -1 + scale = kInt8StaticGroupScale + else: + raise ValueError( + "CompressedTensorsWNA16MoEMethod only supports int4 and int8 now." + ) + + weight_key = QuantKey(self.quant_type, scale, symmetric=self.symmetric) + + is_actorder = self.strategy == QuantizationStrategy.GROUP and self.actorder in ( + ActivationOrdering.GROUP, + ActivationOrdering.DYNAMIC, + ) + + # Select WNA16 MoE backend via oracle. + self.wna16_backend, self.experts_cls = select_wna16_moe_backend( + config=self.moe, + weight_key=weight_key, + quant_config=self.weight_quant, + may_have_zp=not self.symmetric, + may_have_bias=False, + allow_tile_padding=not is_actorder, + ) + + self.is_marlin = self.wna16_backend in [ + WNA16MoEBackend.MARLIN, + WNA16MoEBackend.BATCHED_MARLIN, + ] + self.is_transposed = self.wna16_backend != WNA16MoEBackend.FLASHINFER_TRTLLM + + if self.is_marlin: + assert check_moe_marlin_supports_config( + self.moe, self.group_size, allow_tile_padding=not is_actorder + ) + self.input_dtype = get_marlin_input_dtype(layer_name) + else: + # channelwise is not supported by this kernel + assert weight_quant.strategy == "group" + # grouped actorder isn't supported by this kernel + assert weight_quant.actorder != "group" + + assert self.symmetric, "Only symmetric quantization is supported for MoE" + + # Non-Marlin WNA16 always uses bf16/fp16 inputs + self.input_dtype = torch.bfloat16 + + def get_weight_shape( + self, + weight_name: str, + num_experts: int, + hidden_size: int, + intermediate_size_per_partition: int, + num_groups_w2: int | None = None, + num_groups_w13: int | None = None, + ) -> tuple[int, int, int]: + """ + Get the shape of the weight based on the weight name, number of experts + hidden size, intermediate size per partition, number of groups for w2, + and number of groups for w13. Pass in num_groups_w2 and num_groups_w13 + for weight scales/zero_points. + """ + if weight_name in ("w13_scale", "w13_zp"): + assert num_groups_w13 is not None, ( + "num_groups_w13 must be provided for weight scales/zero_points" + ) + if weight_name in ("w2_scale", "w2_zp"): + assert num_groups_w2 is not None, ( + "num_groups_w2 must be provided for weight scales/zero_points" + ) + w13_num_shards = 2 if self.moe.is_act_and_mul else 1 + shape_map = { + "w13_weight": { + "Flashinfer": ( + num_experts, + w13_num_shards * intermediate_size_per_partition, + hidden_size // self.packed_factor, + ), + "Marlin": ( + num_experts, + hidden_size // self.packed_factor, + w13_num_shards * intermediate_size_per_partition, + ), + }, + "w13_scale": { + "Flashinfer": ( + num_experts, + w13_num_shards * intermediate_size_per_partition, + num_groups_w13, + ), + "Marlin": ( + num_experts, + num_groups_w13, + w13_num_shards * intermediate_size_per_partition, + ), + }, + "w13_zp": { + "Marlin": ( + num_experts, + num_groups_w13, + w13_num_shards + * intermediate_size_per_partition + // self.packed_factor, + ), + }, + "w2_weight": { + "Flashinfer": ( + num_experts, + hidden_size, + intermediate_size_per_partition // self.packed_factor, + ), + "Marlin": ( + num_experts, + intermediate_size_per_partition // self.packed_factor, + hidden_size, + ), + }, + "w2_scale": { + "Flashinfer": (num_experts, hidden_size, num_groups_w2), + "Marlin": (num_experts, num_groups_w2, hidden_size), + }, + "w2_zp": { + "Marlin": ( + num_experts, + num_groups_w2, + hidden_size // self.packed_factor, + ), + }, + } + backend_key = "Marlin" if self.is_transposed else "Flashinfer" + return shape_map[weight_name][backend_key] + + @staticmethod + def _w2_scale_sharding( + actorder, + group_size: int, + intermediate_size_per_partition: int, + intermediate_size_full: int, + ) -> tuple[bool, int, bool]: + """Decide how to shard w2 group scales across TP for WNA16 Marlin MoE. + + Only ``actorder="group"`` permutes activations by ``g_idx`` at runtime + and therefore needs the full-K (unsharded) w2 scales plus ``is_k_full``. + ``actorder="weight"``/``"static"`` (and ``None``) reorder weights at + quantization time, so scales shard normally per TP rank. + """ + load_full_w2 = (actorder == "group") and group_size != -1 + w2_scales_size = ( + intermediate_size_full if load_full_w2 else intermediate_size_per_partition + ) + is_k_full = (actorder != "group") or ( + intermediate_size_per_partition == intermediate_size_full + ) + return load_full_w2, w2_scales_size, is_k_full + def create_weights( self, layer: torch.nn.Module, @@ -61,18 +245,23 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): params_dtype: torch.dtype, **extra_weight_attrs, ): + intermediate_size_full = extra_weight_attrs.pop("intermediate_size_full") + # Will transpose the loaded weight along the # intermediate and hidden dim sizes. Will # shard for TP along the transposed dims extra_weight_attrs.update( - {"is_transposed": True, "quant_method": self.strategy} + {"is_transposed": self.is_transposed, "quant_method": self.strategy} ) - w13_num_shards = 2 if self.moe.is_act_and_mul else 1 + w13_weight = torch.nn.Parameter( torch.empty( - num_experts, - hidden_size // self.packed_factor, - w13_num_shards * intermediate_size_per_partition, + *self.get_weight_shape( + "w13_weight", + num_experts, + hidden_size, + intermediate_size_per_partition, + ), dtype=torch.int32, ), requires_grad=False, @@ -82,9 +271,12 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): w2_weight = torch.nn.Parameter( torch.empty( - num_experts, - intermediate_size_per_partition // self.packed_factor, - hidden_size, + *self.get_weight_shape( + "w2_weight", + num_experts, + hidden_size, + intermediate_size_per_partition, + ), dtype=torch.int32, ), requires_grad=False, @@ -92,7 +284,12 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): layer.register_parameter("w2_weight_packed", w2_weight) set_weight_attrs(w2_weight, extra_weight_attrs) - w2_scales_size = intermediate_size_per_partition + load_full_w2, w2_scales_size, self.is_k_full = self._w2_scale_sharding( + self.actorder, + self.group_size, + intermediate_size_per_partition, + intermediate_size_full, + ) if self.strategy == "channel": num_groups_w2 = num_groups_w13 = 1 @@ -104,23 +301,34 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): f"({hidden_size}) to be divisible by group_size " f"({self.group_size})." ) - if intermediate_size_per_partition % self.group_size != 0: + if ( + not load_full_w2 + and intermediate_size_per_partition % self.group_size != 0 + ): raise ValueError( - "CompressedTensors WNA16 MoE with static group scales " - "requires the MoE intermediate size per tensor-parallel " - f"partition ({intermediate_size_per_partition}) to be " - f"divisible by group_size ({self.group_size}). Scale " - "groups would otherwise cross TP shard boundaries; use a " - "compatible TP size or enable expert parallelism." + "CompressedTensors WNA16 MoE with static group " + "scales requires the MoE intermediate size per " + "tensor-parallel partition " + f"({intermediate_size_per_partition}) to be divisible by " + f"group_size ({self.group_size}). Scale groups would " + "otherwise cross TP shard boundaries; use a compatible TP " + "size or enable expert parallelism." ) num_groups_w2 = w2_scales_size // self.group_size num_groups_w13 = hidden_size // self.group_size + layer.num_groups_w13 = num_groups_w13 + layer.num_groups_w2 = num_groups_w2 + w13_scale = torch.nn.Parameter( torch.ones( - num_experts, - num_groups_w13, - w13_num_shards * intermediate_size_per_partition, + *self.get_weight_shape( + "w13_scale", + num_experts, + hidden_size, + intermediate_size_per_partition, + num_groups_w13=num_groups_w13, + ), dtype=params_dtype, ), requires_grad=False, @@ -129,12 +337,54 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): set_weight_attrs(w13_scale, extra_weight_attrs) w2_scale = torch.nn.Parameter( - torch.ones(num_experts, num_groups_w2, hidden_size, dtype=params_dtype), + torch.ones( + *self.get_weight_shape( + "w2_scale", + num_experts, + hidden_size, + intermediate_size_per_partition, + num_groups_w2=num_groups_w2, + ), + dtype=params_dtype, + ), requires_grad=False, ) layer.register_parameter("w2_weight_scale", w2_scale) set_weight_attrs(w2_scale, extra_weight_attrs) - set_weight_attrs(w2_scale, {"load_full_w2": False}) + set_weight_attrs(w2_scale, {"load_full_w2": load_full_w2}) + + if not self.symmetric: + w13_zp = torch.nn.Parameter( + torch.zeros( + *self.get_weight_shape( + "w13_zp", + num_experts, + hidden_size, + intermediate_size_per_partition, + num_groups_w13=num_groups_w13, + ), + dtype=torch.int32, + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight_zero_point", w13_zp) + set_weight_attrs(w13_zp, extra_weight_attrs) + + w2_zp = torch.nn.Parameter( + torch.zeros( + *self.get_weight_shape( + "w2_zp", + num_experts, + hidden_size, + intermediate_size_per_partition, + num_groups_w2=num_groups_w2, + ), + dtype=torch.int32, + ), + requires_grad=False, + ) + layer.register_parameter("w2_weight_zero_point", w2_zp) + set_weight_attrs(w2_zp, extra_weight_attrs) w2_weight_shape = torch.nn.Parameter( torch.empty(num_experts, 2), requires_grad=False @@ -195,65 +445,161 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): layer.a13_scale = None layer.a2_scale = None + def _setup_kernel(self, layer: RoutedExperts): + assert self.experts_cls is not None + self.moe_quant_config = self.get_fused_moe_quant_config(layer) + assert self.moe_quant_config is not None + + # Add Marlin-specific arguments + marlin_args: dict[str, Any] = {} + if self.is_marlin: + marlin_args = { + "w13_g_idx": layer.w13_weight_g_idx, + "w2_g_idx": layer.w2_weight_g_idx, + "w13_g_idx_sort_indices": layer.w13_g_idx_sort_indices, + "w2_g_idx_sort_indices": layer.w2_g_idx_sort_indices, + "is_k_full": self.is_k_full, + } + + self.moe_kernel = make_wna16_moe_kernel( + moe_quant_config=self.moe_quant_config, + moe_config=self.moe, + experts_cls=self.experts_cls, + routing_tables=layer._expert_routing_tables(), + **marlin_args, + ) + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: - # Reconfigure packed weights and scales to match moe_wna16 format - layer.w13_weight_packed = torch.nn.Parameter( - layer.w13_weight_packed.transpose(1, 2).contiguous().view(torch.uint8), - requires_grad=False, - ) - layer.w2_weight_packed = torch.nn.Parameter( - layer.w2_weight_packed.transpose(1, 2).contiguous().view(torch.uint8), - requires_grad=False, - ) - layer.w13_weight_scale = torch.nn.Parameter( - layer.w13_weight_scale.transpose(1, 2).contiguous(), requires_grad=False - ) - layer.w2_weight_scale = torch.nn.Parameter( - layer.w2_weight_scale.transpose(1, 2).contiguous(), requires_grad=False + # Process weights using the shared oracle infrastructure + converted = convert_to_wna16_moe_kernel_format( + backend=self.wna16_backend, + layer=layer, + quant_config=self.weight_quant, + input_dtype=self.input_dtype, + w13=layer.w13_weight_packed, + w2=layer.w2_weight_packed, + w13_scale=layer.w13_weight_scale, + w2_scale=layer.w2_weight_scale, + w13_g_idx=layer.w13_weight_g_idx, + w2_g_idx=layer.w2_weight_g_idx, + w13_qzeros=getattr(layer, "w13_weight_zero_point", None), + w2_qzeros=getattr(layer, "w2_weight_zero_point", None), ) + if converted is None: + self._setup_kernel(layer) + return + + ( + w13_qweight, + w2_qweight, + w13_scales, + w2_scales, + w13_g_idx_processed, + w2_g_idx_processed, + w13_g_idx_sort_indices, + w2_g_idx_sort_indices, + w13_qzeros, + w2_qzeros, + w13_input_global_scale, + w2_input_global_scale, + _, # w13_bias + _, # w2_bias + ) = converted + + # Replace common parameters + replace_parameter(layer, "w13_weight_packed", w13_qweight) + replace_parameter(layer, "w2_weight_packed", w2_qweight) + replace_parameter(layer, "w13_weight_scale", w13_scales) + replace_parameter(layer, "w2_weight_scale", w2_scales) + + # CPU fused_experts_cpu requires zero points even for symmetric quant + if not self.symmetric or self.wna16_backend == WNA16MoEBackend.CPU: + assert w13_qzeros is not None and w2_qzeros is not None + replace_parameter(layer, "w13_weight_zero_point", w13_qzeros) + replace_parameter(layer, "w2_weight_zero_point", w2_qzeros) + + # Marlin-specific parameters (not needed for Flashinfer) + if self.is_marlin: + if w13_g_idx_processed is not None: + replace_parameter(layer, "w13_weight_g_idx", w13_g_idx_processed) + if w2_g_idx_processed is not None: + replace_parameter(layer, "w2_weight_g_idx", w2_g_idx_processed) + if w13_g_idx_sort_indices is not None: + replace_parameter( + layer, "w13_g_idx_sort_indices", w13_g_idx_sort_indices + ) + if w2_g_idx_sort_indices is not None: + replace_parameter(layer, "w2_g_idx_sort_indices", w2_g_idx_sort_indices) + + # Register input global scales if present + if w13_input_global_scale is not None: + layer.register_parameter( + "w13_input_global_scale", + torch.nn.Parameter(w13_input_global_scale, requires_grad=False), + ) + if w2_input_global_scale is not None: + layer.register_parameter( + "w2_input_global_scale", + torch.nn.Parameter(w2_input_global_scale, requires_grad=False), + ) + + # Marlin workspace — only needed for Marlin-family backends, not emulation. + if ( + self.experts_cls is not None + and issubclass(self.experts_cls, FusedMoEExpertsModular) + and self.wna16_backend != WNA16MoEBackend.EMULATION + ): + layer.workspace = marlin_make_workspace_new( + layer.w13_weight_g_idx.device, + 4, + existing=getattr(layer, "workspace", None), + ) + + # Alias packed weights to w13_weight/w2_weight for the modular kernel interface + layer.w13_weight = layer.w13_weight_packed + layer.w2_weight = layer.w2_weight_packed + + self._setup_kernel(layer) + def get_fused_moe_quant_config( self, layer: torch.nn.Module ) -> FusedMoEQuantConfig | None: - assert self.num_bits == 4 or self.num_bits == 8 - config_builder = ( - int4_w4a16_moe_quant_config - if self.num_bits == 4 - else int8_w8a16_moe_quant_config - ) - - return config_builder( + return make_wna16_moe_quant_config( w1_scale=layer.w13_weight_scale, w2_scale=layer.w2_weight_scale, - w1_zp=None, - w2_zp=None, - block_shape=[0, self.group_size], + group_size=self.group_size, + num_bits=self.num_bits, + w1_zp=getattr(layer, "w13_weight_zero_point", None), + w2_zp=getattr(layer, "w2_weight_zero_point", None), + gemm1_clamp_limit=getattr(layer, "swiglu_limit", None), + gemm1_alpha=getattr(layer, "swiglu_alpha", None), + gemm1_beta=getattr(layer, "swiglu_beta", None), ) - def select_gemm_impl( + def apply_monolithic( self, - prepare_finalize: mk.FusedMoEPrepareAndFinalizeModular, - layer: torch.nn.Module, - ) -> mk.FusedMoEExpertsModular: - if self.moe.is_lora_enabled: - assert self.moe_quant_config is not None - from vllm.triton_utils import HAS_TRITON - - if HAS_TRITON: - from vllm.model_executor.layers.fused_moe import TritonWNA16Experts - - layer.w13_weight = layer.w13_weight_packed - layer.w2_weight = layer.w2_weight_packed - return TritonWNA16Experts( - moe_config=self.moe, quant_config=self.moe_quant_config - ) - else: - raise NotImplementedError( - "TritonExperts requires Triton. " - "Install triton or disable LoRA for MoE." - ) - - raise NotImplementedError + layer: RoutedExperts, + x: torch.Tensor, + router_logits: torch.Tensor, + input_ids: torch.Tensor | None = None, + ) -> torch.Tensor: + assert self.is_monolithic + assert self.moe_kernel is not None + return self.moe_kernel.apply_monolithic( + x, + layer.w13_weight, + layer.w2_weight, + router_logits, + activation=layer.activation, + global_num_experts=layer.global_num_experts, + expert_map=layer.expert_map, + apply_router_weight_on_input=layer.apply_router_weight_on_input, + num_expert_group=layer.num_expert_group, + topk_group=layer.topk_group, + e_score_correction_bias=layer.e_score_correction_bias, + routed_scaling_factor=layer.routed_scaling_factor, + ) def apply( self, @@ -264,21 +610,22 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): shared_experts: SharedExperts | None, shared_experts_input: torch.Tensor | None, ) -> torch.Tensor: - from vllm.model_executor.layers.fused_moe import fused_experts - - return fused_experts( + assert not self.is_monolithic + assert self.moe_kernel is not None + return self.moe_kernel.apply( x, - layer.w13_weight_packed, - layer.w2_weight_packed, + layer.w13_weight, + layer.w2_weight, topk_weights=topk_weights, topk_ids=topk_ids, activation=layer.activation, - apply_router_weight_on_input=layer.apply_router_weight_on_input, global_num_experts=layer.global_num_experts, expert_map=layer.expert_map, - quant_config=self.moe_quant_config, + apply_router_weight_on_input=layer.apply_router_weight_on_input, + shared_experts=shared_experts, + shared_experts_input=shared_experts_input, ) @property def supports_eplb(self) -> bool: - return True + return self.wna16_backend == WNA16MoEBackend.TRITON diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16_marlin.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16_marlin.py deleted file mode 100644 index f29b229249..0000000000 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16_marlin.py +++ /dev/null @@ -1,601 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from typing import Any - -import torch -from compressed_tensors.quantization import ( - QuantizationArgs, -) - -from vllm.logger import init_logger -from vllm.model_executor.layers.fused_moe import ( - FusedMoEExpertsModular, - RoutedExperts, - SharedExperts, -) -from vllm.model_executor.layers.fused_moe.config import ( - FusedMoEConfig, - FusedMoEQuantConfig, -) -from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import ( - WNA16MoEBackend, - convert_to_wna16_moe_kernel_format, - make_wna16_moe_kernel, - make_wna16_moe_quant_config, - select_wna16_moe_backend, -) -from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa E501 - CompressedTensorsMoEMethod, -) -from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa - WNA16_SUPPORTED_TYPES_MAP, - WNA16_ZP_SUPPORTED_TYPES_MAP, -) -from vllm.model_executor.layers.quantization.utils.marlin_utils import ( - get_marlin_input_dtype, - marlin_make_workspace_new, -) -from vllm.model_executor.layers.quantization.utils.quant_utils import ( - QuantKey, - kInt4Static32GroupScale, - kInt4StaticGroupScale, - kInt8StaticGroupScale, -) -from vllm.model_executor.utils import replace_parameter, set_weight_attrs - -logger = init_logger(__name__) - - -class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod): - def __init__( - self, - weight_quant: QuantizationArgs, - input_quant: QuantizationArgs | None, - moe: FusedMoEConfig, - layer_name: str | None = None, - ): - super().__init__(moe) - self.weight_quant = weight_quant - self.input_quant = input_quant - self.symmetric = weight_quant.symmetric - # Extract properties from weight_quant - self.num_bits = weight_quant.num_bits - self.packed_factor = 32 // weight_quant.num_bits - self.strategy = weight_quant.strategy - self.group_size = weight_quant.group_size - self.actorder = weight_quant.actorder - - self.quant_type = ( - WNA16_SUPPORTED_TYPES_MAP[self.num_bits] - if self.symmetric - else WNA16_ZP_SUPPORTED_TYPES_MAP[self.num_bits] - ) - - self.marlin_input_dtype = get_marlin_input_dtype(layer_name) - - if self.num_bits == 4: - if self.group_size == 32: - scale = kInt4Static32GroupScale - else: - scale = kInt4StaticGroupScale - elif self.num_bits == 8: - scale = kInt8StaticGroupScale - else: - raise ValueError( - "CompressedTensorsWNA16MarlinMoEMethod only supports int4 and int8 now." - ) - - weight_key = QuantKey(self.quant_type, scale, symmetric=self.symmetric) - - # Select WNA16 MoE backend via oracle. - self.wna16_backend, self.experts_cls = select_wna16_moe_backend( - config=self.moe, - weight_key=weight_key, - quant_config=self.weight_quant, - may_have_zp=not self.symmetric, - may_have_bias=False, - ) - self.is_marlin = self.wna16_backend in [ - WNA16MoEBackend.MARLIN, - WNA16MoEBackend.BATCHED_MARLIN, - ] - self.is_transposed = self.wna16_backend != WNA16MoEBackend.FLASHINFER_TRTLLM - - def get_weight_shape( - self, - weight_name: str, - num_experts: int, - hidden_size: int, - intermediate_size_per_partition: int, - num_groups_w2: int | None = None, - num_groups_w13: int | None = None, - ) -> tuple[int, int, int]: - """ - Get the shape of the weight based on the weight name, number of experts - hidden size, intermediate size per partition, number of groups for w2, - and number of groups for w13. Pass in num_groups_w2 and num_groups_w13 - for weight scales/zero_points. - """ - if weight_name in ("w13_scale", "w13_zp"): - assert num_groups_w13 is not None, ( - "num_groups_w13 must be provided for weight scales/zero_points" - ) - if weight_name in ("w2_scale", "w2_zp"): - assert num_groups_w2 is not None, ( - "num_groups_w2 must be provided for weight scales/zero_points" - ) - w13_num_shards = 2 if self.moe.is_act_and_mul else 1 - shape_map = { - "w13_weight": { - "Flashinfer": ( - num_experts, - w13_num_shards * intermediate_size_per_partition, - hidden_size // self.packed_factor, - ), - "Marlin": ( - num_experts, - hidden_size // self.packed_factor, - w13_num_shards * intermediate_size_per_partition, - ), - }, - "w13_scale": { - "Flashinfer": ( - num_experts, - w13_num_shards * intermediate_size_per_partition, - num_groups_w13, - ), - "Marlin": ( - num_experts, - num_groups_w13, - w13_num_shards * intermediate_size_per_partition, - ), - }, - "w13_zp": { - "Marlin": ( - num_experts, - num_groups_w13, - w13_num_shards - * intermediate_size_per_partition - // self.packed_factor, - ), - }, - "w2_weight": { - "Flashinfer": ( - num_experts, - hidden_size, - intermediate_size_per_partition // self.packed_factor, - ), - "Marlin": ( - num_experts, - intermediate_size_per_partition // self.packed_factor, - hidden_size, - ), - }, - "w2_scale": { - "Flashinfer": (num_experts, hidden_size, num_groups_w2), - "Marlin": (num_experts, num_groups_w2, hidden_size), - }, - "w2_zp": { - "Marlin": ( - num_experts, - num_groups_w2, - hidden_size // self.packed_factor, - ), - }, - } - backend_key = "Marlin" if self.is_transposed else "Flashinfer" - return shape_map[weight_name][backend_key] - - @staticmethod - def _w2_scale_sharding( - actorder, - group_size: int, - intermediate_size_per_partition: int, - intermediate_size_full: int, - ) -> tuple[bool, int, bool]: - """Decide how to shard w2 group scales across TP for WNA16 Marlin MoE. - - Only ``actorder="group"`` permutes activations by ``g_idx`` at runtime - and therefore needs the full-K (unsharded) w2 scales plus ``is_k_full``. - ``actorder="weight"``/``"static"`` (and ``None``) reorder weights at - quantization time, so scales shard normally per TP rank. - """ - load_full_w2 = (actorder == "group") and group_size != -1 - w2_scales_size = ( - intermediate_size_full if load_full_w2 else intermediate_size_per_partition - ) - is_k_full = (actorder != "group") or ( - intermediate_size_per_partition == intermediate_size_full - ) - return load_full_w2, w2_scales_size, is_k_full - - def create_weights( - self, - layer: torch.nn.Module, - num_experts: int, - hidden_size: int, - intermediate_size_per_partition: int, - params_dtype: torch.dtype, - **extra_weight_attrs, - ): - intermediate_size_full = extra_weight_attrs.pop("intermediate_size_full") - - # Will transpose the loaded weight along the - # intermediate and hidden dim sizes. Will - # shard for TP along the transposed dims - extra_weight_attrs.update( - {"is_transposed": self.is_transposed, "quant_method": self.strategy} - ) - - w13_weight = torch.nn.Parameter( - torch.empty( - *self.get_weight_shape( - "w13_weight", - num_experts, - hidden_size, - intermediate_size_per_partition, - ), - dtype=torch.int32, - ), - requires_grad=False, - ) - layer.register_parameter("w13_weight_packed", w13_weight) - set_weight_attrs(w13_weight, extra_weight_attrs) - - w2_weight = torch.nn.Parameter( - torch.empty( - *self.get_weight_shape( - "w2_weight", - num_experts, - hidden_size, - intermediate_size_per_partition, - ), - dtype=torch.int32, - ), - requires_grad=False, - ) - layer.register_parameter("w2_weight_packed", w2_weight) - set_weight_attrs(w2_weight, extra_weight_attrs) - - load_full_w2, w2_scales_size, self.is_k_full = self._w2_scale_sharding( - self.actorder, - self.group_size, - intermediate_size_per_partition, - intermediate_size_full, - ) - - if self.strategy == "channel": - num_groups_w2 = num_groups_w13 = 1 - self.group_size = -1 - else: - if hidden_size % self.group_size != 0: - raise ValueError( - "CompressedTensors WNA16 Marlin MoE requires hidden_size " - f"({hidden_size}) to be divisible by group_size " - f"({self.group_size})." - ) - if ( - not load_full_w2 - and intermediate_size_per_partition % self.group_size != 0 - ): - raise ValueError( - "CompressedTensors WNA16 Marlin MoE with static group " - "scales requires the MoE intermediate size per " - "tensor-parallel partition " - f"({intermediate_size_per_partition}) to be divisible by " - f"group_size ({self.group_size}). Scale groups would " - "otherwise cross TP shard boundaries; use a compatible TP " - "size or enable expert parallelism." - ) - num_groups_w2 = w2_scales_size // self.group_size - num_groups_w13 = hidden_size // self.group_size - - layer.num_groups_w13 = num_groups_w13 - layer.num_groups_w2 = num_groups_w2 - - w13_scale = torch.nn.Parameter( - torch.ones( - *self.get_weight_shape( - "w13_scale", - num_experts, - hidden_size, - intermediate_size_per_partition, - num_groups_w13=num_groups_w13, - ), - dtype=params_dtype, - ), - requires_grad=False, - ) - layer.register_parameter("w13_weight_scale", w13_scale) - set_weight_attrs(w13_scale, extra_weight_attrs) - - w2_scale = torch.nn.Parameter( - torch.ones( - *self.get_weight_shape( - "w2_scale", - num_experts, - hidden_size, - intermediate_size_per_partition, - num_groups_w2=num_groups_w2, - ), - dtype=params_dtype, - ), - requires_grad=False, - ) - layer.register_parameter("w2_weight_scale", w2_scale) - set_weight_attrs(w2_scale, extra_weight_attrs) - set_weight_attrs(w2_scale, {"load_full_w2": load_full_w2}) - - if not self.symmetric: - w13_zp = torch.nn.Parameter( - torch.zeros( - *self.get_weight_shape( - "w13_zp", - num_experts, - hidden_size, - intermediate_size_per_partition, - num_groups_w13=num_groups_w13, - ), - dtype=torch.int32, - ), - requires_grad=False, - ) - layer.register_parameter("w13_weight_zero_point", w13_zp) - set_weight_attrs(w13_zp, extra_weight_attrs) - - w2_zp = torch.nn.Parameter( - torch.zeros( - *self.get_weight_shape( - "w2_zp", - num_experts, - hidden_size, - intermediate_size_per_partition, - num_groups_w2=num_groups_w2, - ), - dtype=torch.int32, - ), - requires_grad=False, - ) - layer.register_parameter("w2_weight_zero_point", w2_zp) - set_weight_attrs(w2_zp, extra_weight_attrs) - - w2_weight_shape = torch.nn.Parameter( - torch.empty(num_experts, 2), requires_grad=False - ) - layer.register_parameter("w2_weight_shape", w2_weight_shape) - set_weight_attrs(w2_weight_shape, extra_weight_attrs) - w13_weight_shape = torch.nn.Parameter( - torch.empty(num_experts, 2), requires_grad=False - ) - - layer.register_parameter("w13_weight_shape", w13_weight_shape) - set_weight_attrs(w13_weight_shape, extra_weight_attrs) - - w13_g_idx = torch.nn.Parameter( - torch.empty( - num_experts, - hidden_size, - dtype=torch.int32, - ), - requires_grad=False, - ) - layer.register_parameter("w13_weight_g_idx", w13_g_idx) - set_weight_attrs(w13_g_idx, extra_weight_attrs) - - w2_g_idx = torch.nn.Parameter( - torch.empty( - num_experts, - intermediate_size_per_partition, - dtype=torch.int32, - ), - requires_grad=False, - ) - layer.register_parameter("w2_weight_g_idx", w2_g_idx) - set_weight_attrs(w2_g_idx, extra_weight_attrs) - - w13_g_idx_sort_indices = torch.nn.Parameter( - torch.empty( - num_experts, - hidden_size, - dtype=torch.int32, - ), - requires_grad=False, - ) - layer.register_parameter("w13_g_idx_sort_indices", w13_g_idx_sort_indices) - set_weight_attrs(w13_g_idx_sort_indices, extra_weight_attrs) - - w2_g_idx_sort_indices = torch.nn.Parameter( - torch.empty( - num_experts, - intermediate_size_per_partition, - dtype=torch.int32, - ), - requires_grad=False, - ) - layer.register_parameter("w2_g_idx_sort_indices", w2_g_idx_sort_indices) - set_weight_attrs(w2_g_idx_sort_indices, extra_weight_attrs) - - layer.a13_scale = None - layer.a2_scale = None - - def process_weights_after_loading(self, layer: torch.nn.Module) -> None: - # Process weights using the shared oracle infrastructure - converted = convert_to_wna16_moe_kernel_format( - backend=self.wna16_backend, - layer=layer, - quant_config=self.weight_quant, - input_dtype=self.marlin_input_dtype, - w13=layer.w13_weight_packed, - w2=layer.w2_weight_packed, - w13_scale=layer.w13_weight_scale, - w2_scale=layer.w2_weight_scale, - w13_g_idx=layer.w13_weight_g_idx, - w2_g_idx=layer.w2_weight_g_idx, - w13_qzeros=getattr(layer, "w13_weight_zero_point", None), - w2_qzeros=getattr(layer, "w2_weight_zero_point", None), - ) - - if converted is None: - # In-place backends (e.g. Humming) are not wired through this - # marlin-only method; fail clearly rather than unpacking None. - raise NotImplementedError( - f"{type(self).__name__} does not support the " - f"{self.wna16_backend.value} MoE backend." - ) - - ( - w13_qweight, - w2_qweight, - w13_scales, - w2_scales, - w13_g_idx_processed, - w2_g_idx_processed, - w13_g_idx_sort_indices, - w2_g_idx_sort_indices, - w13_qzeros, - w2_qzeros, - w13_input_global_scale, - w2_input_global_scale, - _, # w13_bias - _, # w2_bias - ) = converted - - # Replace common parameters - replace_parameter(layer, "w13_weight_packed", w13_qweight) - replace_parameter(layer, "w2_weight_packed", w2_qweight) - replace_parameter(layer, "w13_weight_scale", w13_scales) - replace_parameter(layer, "w2_weight_scale", w2_scales) - - # CPU fused_experts_cpu requires zero points even for symmetric quant - if not self.symmetric or self.wna16_backend == WNA16MoEBackend.CPU: - replace_parameter(layer, "w13_weight_zero_point", w13_qzeros) - replace_parameter(layer, "w2_weight_zero_point", w2_qzeros) - - # Marlin-specific parameters (not needed for Flashinfer) - if self.is_marlin: - if w13_g_idx_processed is not None: - replace_parameter(layer, "w13_weight_g_idx", w13_g_idx_processed) - if w2_g_idx_processed is not None: - replace_parameter(layer, "w2_weight_g_idx", w2_g_idx_processed) - if w13_g_idx_sort_indices is not None: - replace_parameter( - layer, "w13_g_idx_sort_indices", w13_g_idx_sort_indices - ) - if w2_g_idx_sort_indices is not None: - replace_parameter(layer, "w2_g_idx_sort_indices", w2_g_idx_sort_indices) - - # Register input global scales if present - if w13_input_global_scale is not None: - layer.register_parameter( - "w13_input_global_scale", - torch.nn.Parameter(w13_input_global_scale, requires_grad=False), - ) - if w2_input_global_scale is not None: - layer.register_parameter( - "w2_input_global_scale", - torch.nn.Parameter(w2_input_global_scale, requires_grad=False), - ) - - # Marlin workspace — only needed for Marlin-family backends, not emulation. - if ( - self.experts_cls is not None - and issubclass(self.experts_cls, FusedMoEExpertsModular) - and self.wna16_backend != WNA16MoEBackend.EMULATION - ): - layer.workspace = marlin_make_workspace_new( - layer.w13_weight_g_idx.device, - 4, - existing=getattr(layer, "workspace", None), - ) - - # Alias packed weights to w13_weight/w2_weight for the modular kernel interface - layer.w13_weight = layer.w13_weight_packed - layer.w2_weight = layer.w2_weight_packed - - assert self.experts_cls is not None - self.moe_quant_config = self.get_fused_moe_quant_config(layer) - assert self.moe_quant_config is not None - - # Add Marlin-specific arguments - marlin_args: dict[str, Any] = {} - if self.is_marlin: - marlin_args = { - "w13_g_idx": layer.w13_weight_g_idx, - "w2_g_idx": layer.w2_weight_g_idx, - "w13_g_idx_sort_indices": layer.w13_g_idx_sort_indices, - "w2_g_idx_sort_indices": layer.w2_g_idx_sort_indices, - "is_k_full": self.is_k_full, - } - - self.moe_kernel = make_wna16_moe_kernel( - moe_quant_config=self.moe_quant_config, - moe_config=self.moe, - experts_cls=self.experts_cls, - routing_tables=layer._expert_routing_tables(), - **marlin_args, - ) - - def get_fused_moe_quant_config( - self, layer: torch.nn.Module - ) -> FusedMoEQuantConfig | None: - return make_wna16_moe_quant_config( - w1_scale=layer.w13_weight_scale, - w2_scale=layer.w2_weight_scale, - group_size=self.group_size, - num_bits=self.num_bits, - w1_zp=getattr(layer, "w13_weight_zero_point", None), - w2_zp=getattr(layer, "w2_weight_zero_point", None), - gemm1_clamp_limit=getattr(layer, "swiglu_limit", None), - gemm1_alpha=getattr(layer, "swiglu_alpha", None), - gemm1_beta=getattr(layer, "swiglu_beta", None), - ) - - def apply_monolithic( - self, - layer: RoutedExperts, - x: torch.Tensor, - router_logits: torch.Tensor, - input_ids: torch.Tensor | None = None, - ) -> torch.Tensor: - assert self.is_monolithic - assert self.moe_kernel is not None - return self.moe_kernel.apply_monolithic( - x, - layer.w13_weight, - layer.w2_weight, - router_logits, - activation=layer.activation, - global_num_experts=layer.global_num_experts, - expert_map=layer.expert_map, - apply_router_weight_on_input=layer.apply_router_weight_on_input, - num_expert_group=layer.num_expert_group, - topk_group=layer.topk_group, - e_score_correction_bias=layer.e_score_correction_bias, - routed_scaling_factor=layer.routed_scaling_factor, - ) - - def apply( - self, - layer: RoutedExperts, - x: torch.Tensor, - topk_weights: torch.Tensor, - topk_ids: torch.Tensor, - shared_experts: SharedExperts | None, - shared_experts_input: torch.Tensor | None, - ) -> torch.Tensor: - assert not self.is_monolithic - assert self.moe_kernel is not None - return self.moe_kernel.apply( - x, - layer.w13_weight, - layer.w2_weight, - topk_weights, - topk_ids, - activation=layer.activation, - global_num_experts=layer.global_num_experts, - expert_map=layer.expert_map, - apply_router_weight_on_input=layer.apply_router_weight_on_input, - shared_experts=shared_experts, - shared_experts_input=shared_experts_input, - ) diff --git a/vllm/model_executor/layers/quantization/modelopt.py b/vllm/model_executor/layers/quantization/modelopt.py index 1c94db5a2d..70922b5e83 100644 --- a/vllm/model_executor/layers/quantization/modelopt.py +++ b/vllm/model_executor/layers/quantization/modelopt.py @@ -778,16 +778,6 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase): "logic. This function should not be called." ) - def select_gemm_impl( - self, - prepare_finalize: mk.FusedMoEPrepareAndFinalizeModular, - layer: RoutedExperts, - ) -> mk.FusedMoEExpertsModular: - raise ValueError( - f"{self.__class__.__name__} uses the new modular kernel initialization " - "logic. This function should not be called." - ) - def create_weights( self, layer: RoutedExperts, @@ -2104,16 +2094,6 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase): "logic. This function should not be called." ) - def select_gemm_impl( - self, - prepare_finalize: mk.FusedMoEPrepareAndFinalizeModular, - layer: RoutedExperts, - ) -> mk.FusedMoEExpertsModular: - raise ValueError( - f"{self.__class__.__name__} uses the new modular kernel initialization " - "logic. This function should not be called." - ) - def get_fused_moe_quant_config( self, layer: RoutedExperts ) -> FusedMoEQuantConfig | None: diff --git a/vllm/model_executor/layers/quantization/mxfp4.py b/vllm/model_executor/layers/quantization/mxfp4.py index 6e178e5632..1ee1ad75fd 100644 --- a/vllm/model_executor/layers/quantization/mxfp4.py +++ b/vllm/model_executor/layers/quantization/mxfp4.py @@ -430,16 +430,6 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase): layer=layer, ) - def select_gemm_impl( - self, - prepare_finalize: mk.FusedMoEPrepareAndFinalize, - layer: RoutedExperts, - ) -> mk.FusedMoEExpertsModular: - raise ValueError( - f"{self.__class__.__name__} uses the new modular kernel " - "initialization logic. This function should not be called." - ) - def apply( self, layer: RoutedExperts, @@ -893,16 +883,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): layer=layer, ) - def select_gemm_impl( - self, - prepare_finalize: mk.FusedMoEPrepareAndFinalize, - layer: RoutedExperts, - ) -> mk.FusedMoEExpertsModular: - raise ValueError( - f"{self.__class__.__name__} uses the new modular kernel " - "initialization logic. This function should not be called." - ) - def apply( self, layer: RoutedExperts, diff --git a/vllm/model_executor/layers/quantization/utils/marlin_utils.py b/vllm/model_executor/layers/quantization/utils/marlin_utils.py index 742e791faf..3658761d6f 100644 --- a/vllm/model_executor/layers/quantization/utils/marlin_utils.py +++ b/vllm/model_executor/layers/quantization/utils/marlin_utils.py @@ -12,6 +12,7 @@ from vllm import _custom_ops as ops from vllm.distributed.utils import verify_group_size_divides_partition from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe import RoutedExperts +from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig from vllm.model_executor.layers.linear import LinearBase from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8 from vllm.model_executor.layers.quantization.utils.int8_utils import ( @@ -351,10 +352,12 @@ def marlin_moe_padded_intermediate(intermediate_size: int, group_size: int = -1) return padded -def check_moe_marlin_supports_layer( - layer: RoutedExperts, group_size: int, allow_tile_padding: bool = False +def check_moe_marlin_supports_config( + config: FusedMoEConfig, + group_size: int, + allow_tile_padding: bool = False, ) -> bool: - """Whether the fused MoE Marlin kernel supports ``layer``. + """Whether the fused MoE Marlin kernel supports ``config``. Callers without act-order may pass ``allow_tile_padding=True``: a tile-misaligned intermediate size is then zero-padded to a valid thread @@ -364,15 +367,11 @@ def check_moe_marlin_supports_layer( """ if current_platform.is_rocm(): return False - hidden_size = layer.hidden_size + hidden_size = config.hidden_dim # The layer has not rounded intermediate_size yet; use the stable unpadded # size. gate-up needs n=2*intermediate % 128, down needs k=intermediate % 64. - intermediate_size_per_partition = ( - layer.moe_config.intermediate_size_per_partition_unpadded - ) + intermediate_size_per_partition = config.intermediate_size_per_partition_unpadded assert intermediate_size_per_partition is not None - # apply_router_weight_on_input is not supported for moe marlin - supports_router_weight = not layer.apply_router_weight_on_input if allow_tile_padding: supports_shape = hidden_size % 128 == 0 and ( @@ -384,7 +383,17 @@ def check_moe_marlin_supports_layer( and intermediate_size_per_partition % max(64, group_size) == 0 ) supports_group_size = group_size in [-1, 32, 64, 128] - return supports_shape and supports_group_size and supports_router_weight + return supports_shape and supports_group_size + + +def check_moe_marlin_supports_layer( + layer: RoutedExperts, + group_size: int, + allow_tile_padding: bool = False, +) -> bool: + return check_moe_marlin_supports_config( + layer.moe_config, group_size, allow_tile_padding + ) def marlin_moe_intermediate_size(w1_packed: torch.Tensor, w2_packed: torch.Tensor):