[MoE Refactor] Combine CompressedTensorsWNA16MarlinMoEMethod with CompressedTensorsWNA16MoEMethod (#44570)
Signed-off-by: Bill Nell <bnell@redhat.com> Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Signed-off-by: Dobrzyniewicz, Agata <agata.dobrzyniewicz@intel.com> Signed-off-by: Tushar Jain <tushar00jain@users.noreply.github.com> Signed-off-by: yewentao256 <zhyanwentao@126.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Co-authored-by: Agata Dobrzyniewicz <160237065+adobrzyn@users.noreply.github.com> Co-authored-by: Tushar Jain <8455015+tushar00jain@users.noreply.github.com> Co-authored-by: Tushar Jain <tushar00jain@users.noreply.github.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
co-authored by
Claude
Taneem Ibrahim
Agata Dobrzyniewicz
Tushar Jain
Tushar Jain
Wentao Ye
Jiangyun Zhu
parent
0c4dc7c488
commit
454ea5b526
@@ -563,6 +563,7 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
|
||||
quant_config=self.quant_config,
|
||||
may_have_zp=self.quant_config.zero_point,
|
||||
may_have_bias=True,
|
||||
allow_tile_padding=True,
|
||||
)
|
||||
|
||||
def create_weights(
|
||||
@@ -782,16 +783,6 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
)
|
||||
|
||||
def select_gemm_impl(
|
||||
self,
|
||||
prepare_finalize,
|
||||
layer: RoutedExperts,
|
||||
):
|
||||
raise ValueError(
|
||||
f"{self.__class__.__name__} uses the new modular kernel "
|
||||
"initialization logic. This function should not be called."
|
||||
)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
|
||||
@@ -490,8 +490,9 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
moe,
|
||||
weight_key,
|
||||
quant_config=self.quant_config,
|
||||
may_have_zp=True,
|
||||
may_have_zp=not self.quant_config.is_sym,
|
||||
may_have_bias=True,
|
||||
allow_tile_padding=not self.quant_config.desc_act,
|
||||
)
|
||||
|
||||
def create_weights(
|
||||
@@ -806,16 +807,6 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
w2_bias=getattr(layer, "w2_bias", None),
|
||||
)
|
||||
|
||||
def select_gemm_impl(
|
||||
self,
|
||||
prepare_finalize,
|
||||
layer: RoutedExperts,
|
||||
):
|
||||
raise ValueError(
|
||||
f"{self.__class__.__name__} uses the new modular kernel "
|
||||
"initialization logic. This function should not be called."
|
||||
)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
|
||||
+43
-84
@@ -5,7 +5,6 @@
|
||||
import torch
|
||||
from compressed_tensors import CompressionFormat
|
||||
from compressed_tensors.quantization import (
|
||||
ActivationOrdering,
|
||||
QuantizationStrategy,
|
||||
QuantizationType,
|
||||
)
|
||||
@@ -19,9 +18,6 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa
|
||||
WNA16_SUPPORTED_BITS,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
check_moe_marlin_supports_layer,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -78,9 +74,6 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
|
||||
return CompressedTensorsW8A8Mxfp8MoEMethod(layer.moe_config)
|
||||
|
||||
if quant_config._is_wNa16_group_channel(weight_quant, input_quant):
|
||||
# group_size=None means channelwise
|
||||
group_size = weight_quant.group_size or -1
|
||||
|
||||
valid_format_and_bits = (
|
||||
weight_quant.num_bits in WNA16_SUPPORTED_BITS
|
||||
and format == CompressionFormat.pack_quantized.value
|
||||
@@ -95,87 +88,53 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
|
||||
f" and bits: {weight_quant.num_bits}",
|
||||
)
|
||||
|
||||
# Prefer to use the MarlinMoE kernel when it is supported.
|
||||
is_actorder = (
|
||||
weight_quant.strategy == QuantizationStrategy.GROUP
|
||||
and weight_quant.actorder
|
||||
in (ActivationOrdering.GROUP, ActivationOrdering.DYNAMIC)
|
||||
)
|
||||
if (
|
||||
not check_moe_marlin_supports_layer(
|
||||
layer, group_size, allow_tile_padding=not is_actorder
|
||||
)
|
||||
or current_platform.is_rocm()
|
||||
):
|
||||
if is_actorder:
|
||||
raise ValueError(
|
||||
"WNA16MoE is not supported with actorder=group/dynamic."
|
||||
# Native ROCm HIP kernels (RDNA3, etc.)
|
||||
if current_platform.is_rocm():
|
||||
from . import rocm_moe_rdna
|
||||
|
||||
if rocm_moe_rdna.is_supported(weight_quant):
|
||||
return rocm_moe_rdna.make_method(
|
||||
weight_quant, input_quant, layer.moe_config
|
||||
)
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
is_lora_disabled = vllm_config.lora_config is None
|
||||
moe_backend = vllm_config.kernel_config.moe_backend
|
||||
group_size = weight_quant.group_size or -1
|
||||
if (
|
||||
weight_quant.strategy == QuantizationStrategy.GROUP
|
||||
and weight_quant.type == QuantizationType.INT
|
||||
and group_size == 32
|
||||
and weight_quant.num_bits == 4
|
||||
and is_lora_disabled
|
||||
and on_gfx950()
|
||||
and moe_backend == "flydsl"
|
||||
):
|
||||
from .compressed_tensors_moe_w4a16_flydsl import (
|
||||
CompressedTensorsW4A16FlydslMoEMethod,
|
||||
)
|
||||
|
||||
# Native ROCm HIP kernels (RDNA3, etc.)
|
||||
if current_platform.is_rocm():
|
||||
from . import rocm_moe_rdna
|
||||
logger.info_once("Using CompressedTensorsW4A16FlydslMoEMethod")
|
||||
return CompressedTensorsW4A16FlydslMoEMethod(
|
||||
weight_quant, input_quant, layer.moe_config
|
||||
)
|
||||
elif moe_backend == "emulation":
|
||||
logger.info_once(
|
||||
"Using CompressedTensorsWNA16MoEMethod "
|
||||
"(emulation backend requested)"
|
||||
)
|
||||
|
||||
if rocm_moe_rdna.is_supported(weight_quant):
|
||||
return rocm_moe_rdna.make_method(
|
||||
weight_quant, input_quant, layer.moe_config
|
||||
)
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
from .compressed_tensors_moe_wna16 import (
|
||||
CompressedTensorsWNA16MoEMethod,
|
||||
)
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
is_lora_disabled = vllm_config.lora_config is None
|
||||
moe_backend = vllm_config.kernel_config.moe_backend
|
||||
if (
|
||||
weight_quant.strategy == QuantizationStrategy.GROUP
|
||||
and weight_quant.type == QuantizationType.INT
|
||||
and group_size == 32
|
||||
and weight_quant.num_bits == 4
|
||||
and is_lora_disabled
|
||||
and on_gfx950()
|
||||
and moe_backend == "flydsl"
|
||||
):
|
||||
from .compressed_tensors_moe_w4a16_flydsl import (
|
||||
CompressedTensorsW4A16FlydslMoEMethod,
|
||||
)
|
||||
|
||||
logger.info_once("Using CompressedTensorsW4A16FlydslMoEMethod")
|
||||
return CompressedTensorsW4A16FlydslMoEMethod(
|
||||
weight_quant, input_quant, layer.moe_config
|
||||
)
|
||||
elif moe_backend == "emulation":
|
||||
# Although this is called 'Marlin', actually it selects
|
||||
# emulation backend by calling select_wna16_moe_backend.
|
||||
# TODO: we need to update CompressedTensorsWNA16MoeMethod
|
||||
# to honor "--moe-backend" option
|
||||
from .compressed_tensors_moe_wna16_marlin import (
|
||||
CompressedTensorsWNA16MarlinMoEMethod,
|
||||
)
|
||||
|
||||
logger.info_once(
|
||||
"Using CompressedTensorsWNA16MarlinMoEMethod "
|
||||
"(emulation backend requested)"
|
||||
)
|
||||
return CompressedTensorsWNA16MarlinMoEMethod(
|
||||
weight_quant, input_quant, layer.moe_config, layer_name
|
||||
)
|
||||
|
||||
from .compressed_tensors_moe_wna16 import (
|
||||
CompressedTensorsWNA16MoEMethod,
|
||||
)
|
||||
|
||||
logger.info_once("Using CompressedTensorsWNA16MoEMethod")
|
||||
return CompressedTensorsWNA16MoEMethod(
|
||||
weight_quant, input_quant, layer.moe_config
|
||||
)
|
||||
else:
|
||||
from .compressed_tensors_moe_wna16_marlin import (
|
||||
CompressedTensorsWNA16MarlinMoEMethod,
|
||||
)
|
||||
|
||||
logger.info_once("Using CompressedTensorsWNA16MarlinMoEMethod")
|
||||
return CompressedTensorsWNA16MarlinMoEMethod(
|
||||
weight_quant, input_quant, layer.moe_config
|
||||
)
|
||||
logger.info_once("Using CompressedTensorsWNA16MoEMethod")
|
||||
return CompressedTensorsWNA16MoEMethod(
|
||||
weight_quant,
|
||||
input_quant,
|
||||
layer.moe_config,
|
||||
)
|
||||
elif quant_config._is_nvfp4_format(weight_quant):
|
||||
from .compressed_tensors_moe_w4a4_nvfp4 import (
|
||||
CompressedTensorsW4A4Nvfp4MoEMethod,
|
||||
|
||||
+435
-88
@@ -1,28 +1,51 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from compressed_tensors.quantization import (
|
||||
ActivationOrdering,
|
||||
QuantizationArgs,
|
||||
QuantizationStrategy,
|
||||
)
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEExpertsModular,
|
||||
RoutedExperts,
|
||||
SharedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEQuantConfig,
|
||||
int4_w4a16_moe_quant_config,
|
||||
int8_w8a16_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
|
||||
WNA16MoEBackend,
|
||||
convert_to_wna16_moe_kernel_format,
|
||||
make_wna16_moe_kernel,
|
||||
make_wna16_moe_quant_config,
|
||||
select_wna16_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa E501
|
||||
CompressedTensorsMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa
|
||||
WNA16_SUPPORTED_TYPES_MAP,
|
||||
WNA16_ZP_SUPPORTED_TYPES_MAP,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
check_moe_marlin_supports_config,
|
||||
get_marlin_input_dtype,
|
||||
marlin_make_workspace_new,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kInt4Static32GroupScale,
|
||||
kInt4StaticGroupScale,
|
||||
kInt8StaticGroupScale,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -39,19 +62,180 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
self.weight_quant = weight_quant
|
||||
self.input_quant = input_quant
|
||||
# Extract properties from weight_quant
|
||||
self.symmetric = weight_quant.symmetric
|
||||
self.num_bits = weight_quant.num_bits
|
||||
self.packed_factor = 32 // weight_quant.num_bits
|
||||
self.strategy = weight_quant.strategy
|
||||
# channelwise is not supported by this kernel
|
||||
assert weight_quant.strategy == "group"
|
||||
self.group_size = weight_quant.group_size
|
||||
# grouped actorder isn't supported by this kernel
|
||||
assert weight_quant.actorder != "group"
|
||||
assert weight_quant.symmetric, (
|
||||
"Only symmetric quantization is supported for MoE. "
|
||||
"Try --moe-backend emulation."
|
||||
self.actorder = weight_quant.actorder
|
||||
|
||||
# Extract quant_type and create weight key for oracle selection
|
||||
self.quant_type = (
|
||||
WNA16_SUPPORTED_TYPES_MAP[self.num_bits]
|
||||
if self.symmetric
|
||||
else WNA16_ZP_SUPPORTED_TYPES_MAP[self.num_bits]
|
||||
)
|
||||
|
||||
if self.num_bits == 4:
|
||||
if self.group_size == 32:
|
||||
scale = kInt4Static32GroupScale
|
||||
else:
|
||||
scale = kInt4StaticGroupScale
|
||||
elif self.num_bits == 8:
|
||||
assert self.group_size == -1
|
||||
scale = kInt8StaticGroupScale
|
||||
else:
|
||||
raise ValueError(
|
||||
"CompressedTensorsWNA16MoEMethod only supports int4 and int8 now."
|
||||
)
|
||||
|
||||
weight_key = QuantKey(self.quant_type, scale, symmetric=self.symmetric)
|
||||
|
||||
is_actorder = self.strategy == QuantizationStrategy.GROUP and self.actorder in (
|
||||
ActivationOrdering.GROUP,
|
||||
ActivationOrdering.DYNAMIC,
|
||||
)
|
||||
|
||||
# Select WNA16 MoE backend via oracle.
|
||||
self.wna16_backend, self.experts_cls = select_wna16_moe_backend(
|
||||
config=self.moe,
|
||||
weight_key=weight_key,
|
||||
quant_config=self.weight_quant,
|
||||
may_have_zp=not self.symmetric,
|
||||
may_have_bias=False,
|
||||
allow_tile_padding=not is_actorder,
|
||||
)
|
||||
|
||||
self.is_marlin = self.wna16_backend in [
|
||||
WNA16MoEBackend.MARLIN,
|
||||
WNA16MoEBackend.BATCHED_MARLIN,
|
||||
]
|
||||
self.is_transposed = self.wna16_backend != WNA16MoEBackend.FLASHINFER_TRTLLM
|
||||
|
||||
if self.is_marlin:
|
||||
assert check_moe_marlin_supports_config(
|
||||
self.moe, self.group_size, allow_tile_padding=not is_actorder
|
||||
)
|
||||
self.input_dtype = get_marlin_input_dtype(layer_name)
|
||||
else:
|
||||
# channelwise is not supported by this kernel
|
||||
assert weight_quant.strategy == "group"
|
||||
# grouped actorder isn't supported by this kernel
|
||||
assert weight_quant.actorder != "group"
|
||||
|
||||
assert self.symmetric, "Only symmetric quantization is supported for MoE"
|
||||
|
||||
# Non-Marlin WNA16 always uses bf16/fp16 inputs
|
||||
self.input_dtype = torch.bfloat16
|
||||
|
||||
def get_weight_shape(
|
||||
self,
|
||||
weight_name: str,
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
num_groups_w2: int | None = None,
|
||||
num_groups_w13: int | None = None,
|
||||
) -> tuple[int, int, int]:
|
||||
"""
|
||||
Get the shape of the weight based on the weight name, number of experts
|
||||
hidden size, intermediate size per partition, number of groups for w2,
|
||||
and number of groups for w13. Pass in num_groups_w2 and num_groups_w13
|
||||
for weight scales/zero_points.
|
||||
"""
|
||||
if weight_name in ("w13_scale", "w13_zp"):
|
||||
assert num_groups_w13 is not None, (
|
||||
"num_groups_w13 must be provided for weight scales/zero_points"
|
||||
)
|
||||
if weight_name in ("w2_scale", "w2_zp"):
|
||||
assert num_groups_w2 is not None, (
|
||||
"num_groups_w2 must be provided for weight scales/zero_points"
|
||||
)
|
||||
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
|
||||
shape_map = {
|
||||
"w13_weight": {
|
||||
"Flashinfer": (
|
||||
num_experts,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size // self.packed_factor,
|
||||
),
|
||||
"Marlin": (
|
||||
num_experts,
|
||||
hidden_size // self.packed_factor,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
),
|
||||
},
|
||||
"w13_scale": {
|
||||
"Flashinfer": (
|
||||
num_experts,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
num_groups_w13,
|
||||
),
|
||||
"Marlin": (
|
||||
num_experts,
|
||||
num_groups_w13,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
),
|
||||
},
|
||||
"w13_zp": {
|
||||
"Marlin": (
|
||||
num_experts,
|
||||
num_groups_w13,
|
||||
w13_num_shards
|
||||
* intermediate_size_per_partition
|
||||
// self.packed_factor,
|
||||
),
|
||||
},
|
||||
"w2_weight": {
|
||||
"Flashinfer": (
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition // self.packed_factor,
|
||||
),
|
||||
"Marlin": (
|
||||
num_experts,
|
||||
intermediate_size_per_partition // self.packed_factor,
|
||||
hidden_size,
|
||||
),
|
||||
},
|
||||
"w2_scale": {
|
||||
"Flashinfer": (num_experts, hidden_size, num_groups_w2),
|
||||
"Marlin": (num_experts, num_groups_w2, hidden_size),
|
||||
},
|
||||
"w2_zp": {
|
||||
"Marlin": (
|
||||
num_experts,
|
||||
num_groups_w2,
|
||||
hidden_size // self.packed_factor,
|
||||
),
|
||||
},
|
||||
}
|
||||
backend_key = "Marlin" if self.is_transposed else "Flashinfer"
|
||||
return shape_map[weight_name][backend_key]
|
||||
|
||||
@staticmethod
|
||||
def _w2_scale_sharding(
|
||||
actorder,
|
||||
group_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
intermediate_size_full: int,
|
||||
) -> tuple[bool, int, bool]:
|
||||
"""Decide how to shard w2 group scales across TP for WNA16 Marlin MoE.
|
||||
|
||||
Only ``actorder="group"`` permutes activations by ``g_idx`` at runtime
|
||||
and therefore needs the full-K (unsharded) w2 scales plus ``is_k_full``.
|
||||
``actorder="weight"``/``"static"`` (and ``None``) reorder weights at
|
||||
quantization time, so scales shard normally per TP rank.
|
||||
"""
|
||||
load_full_w2 = (actorder == "group") and group_size != -1
|
||||
w2_scales_size = (
|
||||
intermediate_size_full if load_full_w2 else intermediate_size_per_partition
|
||||
)
|
||||
is_k_full = (actorder != "group") or (
|
||||
intermediate_size_per_partition == intermediate_size_full
|
||||
)
|
||||
return load_full_w2, w2_scales_size, is_k_full
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
@@ -61,18 +245,23 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
intermediate_size_full = extra_weight_attrs.pop("intermediate_size_full")
|
||||
|
||||
# Will transpose the loaded weight along the
|
||||
# intermediate and hidden dim sizes. Will
|
||||
# shard for TP along the transposed dims
|
||||
extra_weight_attrs.update(
|
||||
{"is_transposed": True, "quant_method": self.strategy}
|
||||
{"is_transposed": self.is_transposed, "quant_method": self.strategy}
|
||||
)
|
||||
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
|
||||
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
hidden_size // self.packed_factor,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
*self.get_weight_shape(
|
||||
"w13_weight",
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition,
|
||||
),
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -82,9 +271,12 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
|
||||
w2_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
intermediate_size_per_partition // self.packed_factor,
|
||||
hidden_size,
|
||||
*self.get_weight_shape(
|
||||
"w2_weight",
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition,
|
||||
),
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -92,7 +284,12 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
layer.register_parameter("w2_weight_packed", w2_weight)
|
||||
set_weight_attrs(w2_weight, extra_weight_attrs)
|
||||
|
||||
w2_scales_size = intermediate_size_per_partition
|
||||
load_full_w2, w2_scales_size, self.is_k_full = self._w2_scale_sharding(
|
||||
self.actorder,
|
||||
self.group_size,
|
||||
intermediate_size_per_partition,
|
||||
intermediate_size_full,
|
||||
)
|
||||
|
||||
if self.strategy == "channel":
|
||||
num_groups_w2 = num_groups_w13 = 1
|
||||
@@ -104,23 +301,34 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
f"({hidden_size}) to be divisible by group_size "
|
||||
f"({self.group_size})."
|
||||
)
|
||||
if intermediate_size_per_partition % self.group_size != 0:
|
||||
if (
|
||||
not load_full_w2
|
||||
and intermediate_size_per_partition % self.group_size != 0
|
||||
):
|
||||
raise ValueError(
|
||||
"CompressedTensors WNA16 MoE with static group scales "
|
||||
"requires the MoE intermediate size per tensor-parallel "
|
||||
f"partition ({intermediate_size_per_partition}) to be "
|
||||
f"divisible by group_size ({self.group_size}). Scale "
|
||||
"groups would otherwise cross TP shard boundaries; use a "
|
||||
"compatible TP size or enable expert parallelism."
|
||||
"CompressedTensors WNA16 MoE with static group "
|
||||
"scales requires the MoE intermediate size per "
|
||||
"tensor-parallel partition "
|
||||
f"({intermediate_size_per_partition}) to be divisible by "
|
||||
f"group_size ({self.group_size}). Scale groups would "
|
||||
"otherwise cross TP shard boundaries; use a compatible TP "
|
||||
"size or enable expert parallelism."
|
||||
)
|
||||
num_groups_w2 = w2_scales_size // self.group_size
|
||||
num_groups_w13 = hidden_size // self.group_size
|
||||
|
||||
layer.num_groups_w13 = num_groups_w13
|
||||
layer.num_groups_w2 = num_groups_w2
|
||||
|
||||
w13_scale = torch.nn.Parameter(
|
||||
torch.ones(
|
||||
num_experts,
|
||||
num_groups_w13,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
*self.get_weight_shape(
|
||||
"w13_scale",
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition,
|
||||
num_groups_w13=num_groups_w13,
|
||||
),
|
||||
dtype=params_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -129,12 +337,54 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
set_weight_attrs(w13_scale, extra_weight_attrs)
|
||||
|
||||
w2_scale = torch.nn.Parameter(
|
||||
torch.ones(num_experts, num_groups_w2, hidden_size, dtype=params_dtype),
|
||||
torch.ones(
|
||||
*self.get_weight_shape(
|
||||
"w2_scale",
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition,
|
||||
num_groups_w2=num_groups_w2,
|
||||
),
|
||||
dtype=params_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight_scale", w2_scale)
|
||||
set_weight_attrs(w2_scale, extra_weight_attrs)
|
||||
set_weight_attrs(w2_scale, {"load_full_w2": False})
|
||||
set_weight_attrs(w2_scale, {"load_full_w2": load_full_w2})
|
||||
|
||||
if not self.symmetric:
|
||||
w13_zp = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
*self.get_weight_shape(
|
||||
"w13_zp",
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition,
|
||||
num_groups_w13=num_groups_w13,
|
||||
),
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight_zero_point", w13_zp)
|
||||
set_weight_attrs(w13_zp, extra_weight_attrs)
|
||||
|
||||
w2_zp = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
*self.get_weight_shape(
|
||||
"w2_zp",
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition,
|
||||
num_groups_w2=num_groups_w2,
|
||||
),
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight_zero_point", w2_zp)
|
||||
set_weight_attrs(w2_zp, extra_weight_attrs)
|
||||
|
||||
w2_weight_shape = torch.nn.Parameter(
|
||||
torch.empty(num_experts, 2), requires_grad=False
|
||||
@@ -195,65 +445,161 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
layer.a13_scale = None
|
||||
layer.a2_scale = None
|
||||
|
||||
def _setup_kernel(self, layer: RoutedExperts):
|
||||
assert self.experts_cls is not None
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
assert self.moe_quant_config is not None
|
||||
|
||||
# Add Marlin-specific arguments
|
||||
marlin_args: dict[str, Any] = {}
|
||||
if self.is_marlin:
|
||||
marlin_args = {
|
||||
"w13_g_idx": layer.w13_weight_g_idx,
|
||||
"w2_g_idx": layer.w2_weight_g_idx,
|
||||
"w13_g_idx_sort_indices": layer.w13_g_idx_sort_indices,
|
||||
"w2_g_idx_sort_indices": layer.w2_g_idx_sort_indices,
|
||||
"is_k_full": self.is_k_full,
|
||||
}
|
||||
|
||||
self.moe_kernel = make_wna16_moe_kernel(
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
**marlin_args,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
# Reconfigure packed weights and scales to match moe_wna16 format
|
||||
layer.w13_weight_packed = torch.nn.Parameter(
|
||||
layer.w13_weight_packed.transpose(1, 2).contiguous().view(torch.uint8),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.w2_weight_packed = torch.nn.Parameter(
|
||||
layer.w2_weight_packed.transpose(1, 2).contiguous().view(torch.uint8),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.w13_weight_scale = torch.nn.Parameter(
|
||||
layer.w13_weight_scale.transpose(1, 2).contiguous(), requires_grad=False
|
||||
)
|
||||
layer.w2_weight_scale = torch.nn.Parameter(
|
||||
layer.w2_weight_scale.transpose(1, 2).contiguous(), requires_grad=False
|
||||
# Process weights using the shared oracle infrastructure
|
||||
converted = convert_to_wna16_moe_kernel_format(
|
||||
backend=self.wna16_backend,
|
||||
layer=layer,
|
||||
quant_config=self.weight_quant,
|
||||
input_dtype=self.input_dtype,
|
||||
w13=layer.w13_weight_packed,
|
||||
w2=layer.w2_weight_packed,
|
||||
w13_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
w13_g_idx=layer.w13_weight_g_idx,
|
||||
w2_g_idx=layer.w2_weight_g_idx,
|
||||
w13_qzeros=getattr(layer, "w13_weight_zero_point", None),
|
||||
w2_qzeros=getattr(layer, "w2_weight_zero_point", None),
|
||||
)
|
||||
|
||||
if converted is None:
|
||||
self._setup_kernel(layer)
|
||||
return
|
||||
|
||||
(
|
||||
w13_qweight,
|
||||
w2_qweight,
|
||||
w13_scales,
|
||||
w2_scales,
|
||||
w13_g_idx_processed,
|
||||
w2_g_idx_processed,
|
||||
w13_g_idx_sort_indices,
|
||||
w2_g_idx_sort_indices,
|
||||
w13_qzeros,
|
||||
w2_qzeros,
|
||||
w13_input_global_scale,
|
||||
w2_input_global_scale,
|
||||
_, # w13_bias
|
||||
_, # w2_bias
|
||||
) = converted
|
||||
|
||||
# Replace common parameters
|
||||
replace_parameter(layer, "w13_weight_packed", w13_qweight)
|
||||
replace_parameter(layer, "w2_weight_packed", w2_qweight)
|
||||
replace_parameter(layer, "w13_weight_scale", w13_scales)
|
||||
replace_parameter(layer, "w2_weight_scale", w2_scales)
|
||||
|
||||
# CPU fused_experts_cpu requires zero points even for symmetric quant
|
||||
if not self.symmetric or self.wna16_backend == WNA16MoEBackend.CPU:
|
||||
assert w13_qzeros is not None and w2_qzeros is not None
|
||||
replace_parameter(layer, "w13_weight_zero_point", w13_qzeros)
|
||||
replace_parameter(layer, "w2_weight_zero_point", w2_qzeros)
|
||||
|
||||
# Marlin-specific parameters (not needed for Flashinfer)
|
||||
if self.is_marlin:
|
||||
if w13_g_idx_processed is not None:
|
||||
replace_parameter(layer, "w13_weight_g_idx", w13_g_idx_processed)
|
||||
if w2_g_idx_processed is not None:
|
||||
replace_parameter(layer, "w2_weight_g_idx", w2_g_idx_processed)
|
||||
if w13_g_idx_sort_indices is not None:
|
||||
replace_parameter(
|
||||
layer, "w13_g_idx_sort_indices", w13_g_idx_sort_indices
|
||||
)
|
||||
if w2_g_idx_sort_indices is not None:
|
||||
replace_parameter(layer, "w2_g_idx_sort_indices", w2_g_idx_sort_indices)
|
||||
|
||||
# Register input global scales if present
|
||||
if w13_input_global_scale is not None:
|
||||
layer.register_parameter(
|
||||
"w13_input_global_scale",
|
||||
torch.nn.Parameter(w13_input_global_scale, requires_grad=False),
|
||||
)
|
||||
if w2_input_global_scale is not None:
|
||||
layer.register_parameter(
|
||||
"w2_input_global_scale",
|
||||
torch.nn.Parameter(w2_input_global_scale, requires_grad=False),
|
||||
)
|
||||
|
||||
# Marlin workspace — only needed for Marlin-family backends, not emulation.
|
||||
if (
|
||||
self.experts_cls is not None
|
||||
and issubclass(self.experts_cls, FusedMoEExpertsModular)
|
||||
and self.wna16_backend != WNA16MoEBackend.EMULATION
|
||||
):
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
layer.w13_weight_g_idx.device,
|
||||
4,
|
||||
existing=getattr(layer, "workspace", None),
|
||||
)
|
||||
|
||||
# Alias packed weights to w13_weight/w2_weight for the modular kernel interface
|
||||
layer.w13_weight = layer.w13_weight_packed
|
||||
layer.w2_weight = layer.w2_weight_packed
|
||||
|
||||
self._setup_kernel(layer)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: torch.nn.Module
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
assert self.num_bits == 4 or self.num_bits == 8
|
||||
config_builder = (
|
||||
int4_w4a16_moe_quant_config
|
||||
if self.num_bits == 4
|
||||
else int8_w8a16_moe_quant_config
|
||||
)
|
||||
|
||||
return config_builder(
|
||||
return make_wna16_moe_quant_config(
|
||||
w1_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
w1_zp=None,
|
||||
w2_zp=None,
|
||||
block_shape=[0, self.group_size],
|
||||
group_size=self.group_size,
|
||||
num_bits=self.num_bits,
|
||||
w1_zp=getattr(layer, "w13_weight_zero_point", None),
|
||||
w2_zp=getattr(layer, "w2_weight_zero_point", None),
|
||||
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
)
|
||||
|
||||
def select_gemm_impl(
|
||||
def apply_monolithic(
|
||||
self,
|
||||
prepare_finalize: mk.FusedMoEPrepareAndFinalizeModular,
|
||||
layer: torch.nn.Module,
|
||||
) -> mk.FusedMoEExpertsModular:
|
||||
if self.moe.is_lora_enabled:
|
||||
assert self.moe_quant_config is not None
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
if HAS_TRITON:
|
||||
from vllm.model_executor.layers.fused_moe import TritonWNA16Experts
|
||||
|
||||
layer.w13_weight = layer.w13_weight_packed
|
||||
layer.w2_weight = layer.w2_weight_packed
|
||||
return TritonWNA16Experts(
|
||||
moe_config=self.moe, quant_config=self.moe_quant_config
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
"TritonExperts requires Triton. "
|
||||
"Install triton or disable LoRA for MoE."
|
||||
)
|
||||
|
||||
raise NotImplementedError
|
||||
layer: RoutedExperts,
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply_monolithic(
|
||||
x,
|
||||
layer.w13_weight,
|
||||
layer.w2_weight,
|
||||
router_logits,
|
||||
activation=layer.activation,
|
||||
global_num_experts=layer.global_num_experts,
|
||||
expert_map=layer.expert_map,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
num_expert_group=layer.num_expert_group,
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
@@ -264,21 +610,22 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.model_executor.layers.fused_moe import fused_experts
|
||||
|
||||
return fused_experts(
|
||||
assert not self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
x,
|
||||
layer.w13_weight_packed,
|
||||
layer.w2_weight_packed,
|
||||
layer.w13_weight,
|
||||
layer.w2_weight,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=layer.activation,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
global_num_experts=layer.global_num_experts,
|
||||
expert_map=layer.expert_map,
|
||||
quant_config=self.moe_quant_config,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
shared_experts=shared_experts,
|
||||
shared_experts_input=shared_experts_input,
|
||||
)
|
||||
|
||||
@property
|
||||
def supports_eplb(self) -> bool:
|
||||
return True
|
||||
return self.wna16_backend == WNA16MoEBackend.TRITON
|
||||
|
||||
-601
@@ -1,601 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from compressed_tensors.quantization import (
|
||||
QuantizationArgs,
|
||||
)
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEExpertsModular,
|
||||
RoutedExperts,
|
||||
SharedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
|
||||
WNA16MoEBackend,
|
||||
convert_to_wna16_moe_kernel_format,
|
||||
make_wna16_moe_kernel,
|
||||
make_wna16_moe_quant_config,
|
||||
select_wna16_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa E501
|
||||
CompressedTensorsMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa
|
||||
WNA16_SUPPORTED_TYPES_MAP,
|
||||
WNA16_ZP_SUPPORTED_TYPES_MAP,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
get_marlin_input_dtype,
|
||||
marlin_make_workspace_new,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kInt4Static32GroupScale,
|
||||
kInt4StaticGroupScale,
|
||||
kInt8StaticGroupScale,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
|
||||
def __init__(
|
||||
self,
|
||||
weight_quant: QuantizationArgs,
|
||||
input_quant: QuantizationArgs | None,
|
||||
moe: FusedMoEConfig,
|
||||
layer_name: str | None = None,
|
||||
):
|
||||
super().__init__(moe)
|
||||
self.weight_quant = weight_quant
|
||||
self.input_quant = input_quant
|
||||
self.symmetric = weight_quant.symmetric
|
||||
# Extract properties from weight_quant
|
||||
self.num_bits = weight_quant.num_bits
|
||||
self.packed_factor = 32 // weight_quant.num_bits
|
||||
self.strategy = weight_quant.strategy
|
||||
self.group_size = weight_quant.group_size
|
||||
self.actorder = weight_quant.actorder
|
||||
|
||||
self.quant_type = (
|
||||
WNA16_SUPPORTED_TYPES_MAP[self.num_bits]
|
||||
if self.symmetric
|
||||
else WNA16_ZP_SUPPORTED_TYPES_MAP[self.num_bits]
|
||||
)
|
||||
|
||||
self.marlin_input_dtype = get_marlin_input_dtype(layer_name)
|
||||
|
||||
if self.num_bits == 4:
|
||||
if self.group_size == 32:
|
||||
scale = kInt4Static32GroupScale
|
||||
else:
|
||||
scale = kInt4StaticGroupScale
|
||||
elif self.num_bits == 8:
|
||||
scale = kInt8StaticGroupScale
|
||||
else:
|
||||
raise ValueError(
|
||||
"CompressedTensorsWNA16MarlinMoEMethod only supports int4 and int8 now."
|
||||
)
|
||||
|
||||
weight_key = QuantKey(self.quant_type, scale, symmetric=self.symmetric)
|
||||
|
||||
# Select WNA16 MoE backend via oracle.
|
||||
self.wna16_backend, self.experts_cls = select_wna16_moe_backend(
|
||||
config=self.moe,
|
||||
weight_key=weight_key,
|
||||
quant_config=self.weight_quant,
|
||||
may_have_zp=not self.symmetric,
|
||||
may_have_bias=False,
|
||||
)
|
||||
self.is_marlin = self.wna16_backend in [
|
||||
WNA16MoEBackend.MARLIN,
|
||||
WNA16MoEBackend.BATCHED_MARLIN,
|
||||
]
|
||||
self.is_transposed = self.wna16_backend != WNA16MoEBackend.FLASHINFER_TRTLLM
|
||||
|
||||
def get_weight_shape(
|
||||
self,
|
||||
weight_name: str,
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
num_groups_w2: int | None = None,
|
||||
num_groups_w13: int | None = None,
|
||||
) -> tuple[int, int, int]:
|
||||
"""
|
||||
Get the shape of the weight based on the weight name, number of experts
|
||||
hidden size, intermediate size per partition, number of groups for w2,
|
||||
and number of groups for w13. Pass in num_groups_w2 and num_groups_w13
|
||||
for weight scales/zero_points.
|
||||
"""
|
||||
if weight_name in ("w13_scale", "w13_zp"):
|
||||
assert num_groups_w13 is not None, (
|
||||
"num_groups_w13 must be provided for weight scales/zero_points"
|
||||
)
|
||||
if weight_name in ("w2_scale", "w2_zp"):
|
||||
assert num_groups_w2 is not None, (
|
||||
"num_groups_w2 must be provided for weight scales/zero_points"
|
||||
)
|
||||
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
|
||||
shape_map = {
|
||||
"w13_weight": {
|
||||
"Flashinfer": (
|
||||
num_experts,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size // self.packed_factor,
|
||||
),
|
||||
"Marlin": (
|
||||
num_experts,
|
||||
hidden_size // self.packed_factor,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
),
|
||||
},
|
||||
"w13_scale": {
|
||||
"Flashinfer": (
|
||||
num_experts,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
num_groups_w13,
|
||||
),
|
||||
"Marlin": (
|
||||
num_experts,
|
||||
num_groups_w13,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
),
|
||||
},
|
||||
"w13_zp": {
|
||||
"Marlin": (
|
||||
num_experts,
|
||||
num_groups_w13,
|
||||
w13_num_shards
|
||||
* intermediate_size_per_partition
|
||||
// self.packed_factor,
|
||||
),
|
||||
},
|
||||
"w2_weight": {
|
||||
"Flashinfer": (
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition // self.packed_factor,
|
||||
),
|
||||
"Marlin": (
|
||||
num_experts,
|
||||
intermediate_size_per_partition // self.packed_factor,
|
||||
hidden_size,
|
||||
),
|
||||
},
|
||||
"w2_scale": {
|
||||
"Flashinfer": (num_experts, hidden_size, num_groups_w2),
|
||||
"Marlin": (num_experts, num_groups_w2, hidden_size),
|
||||
},
|
||||
"w2_zp": {
|
||||
"Marlin": (
|
||||
num_experts,
|
||||
num_groups_w2,
|
||||
hidden_size // self.packed_factor,
|
||||
),
|
||||
},
|
||||
}
|
||||
backend_key = "Marlin" if self.is_transposed else "Flashinfer"
|
||||
return shape_map[weight_name][backend_key]
|
||||
|
||||
@staticmethod
|
||||
def _w2_scale_sharding(
|
||||
actorder,
|
||||
group_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
intermediate_size_full: int,
|
||||
) -> tuple[bool, int, bool]:
|
||||
"""Decide how to shard w2 group scales across TP for WNA16 Marlin MoE.
|
||||
|
||||
Only ``actorder="group"`` permutes activations by ``g_idx`` at runtime
|
||||
and therefore needs the full-K (unsharded) w2 scales plus ``is_k_full``.
|
||||
``actorder="weight"``/``"static"`` (and ``None``) reorder weights at
|
||||
quantization time, so scales shard normally per TP rank.
|
||||
"""
|
||||
load_full_w2 = (actorder == "group") and group_size != -1
|
||||
w2_scales_size = (
|
||||
intermediate_size_full if load_full_w2 else intermediate_size_per_partition
|
||||
)
|
||||
is_k_full = (actorder != "group") or (
|
||||
intermediate_size_per_partition == intermediate_size_full
|
||||
)
|
||||
return load_full_w2, w2_scales_size, is_k_full
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
intermediate_size_full = extra_weight_attrs.pop("intermediate_size_full")
|
||||
|
||||
# Will transpose the loaded weight along the
|
||||
# intermediate and hidden dim sizes. Will
|
||||
# shard for TP along the transposed dims
|
||||
extra_weight_attrs.update(
|
||||
{"is_transposed": self.is_transposed, "quant_method": self.strategy}
|
||||
)
|
||||
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
*self.get_weight_shape(
|
||||
"w13_weight",
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition,
|
||||
),
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight_packed", w13_weight)
|
||||
set_weight_attrs(w13_weight, extra_weight_attrs)
|
||||
|
||||
w2_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
*self.get_weight_shape(
|
||||
"w2_weight",
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition,
|
||||
),
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight_packed", w2_weight)
|
||||
set_weight_attrs(w2_weight, extra_weight_attrs)
|
||||
|
||||
load_full_w2, w2_scales_size, self.is_k_full = self._w2_scale_sharding(
|
||||
self.actorder,
|
||||
self.group_size,
|
||||
intermediate_size_per_partition,
|
||||
intermediate_size_full,
|
||||
)
|
||||
|
||||
if self.strategy == "channel":
|
||||
num_groups_w2 = num_groups_w13 = 1
|
||||
self.group_size = -1
|
||||
else:
|
||||
if hidden_size % self.group_size != 0:
|
||||
raise ValueError(
|
||||
"CompressedTensors WNA16 Marlin MoE requires hidden_size "
|
||||
f"({hidden_size}) to be divisible by group_size "
|
||||
f"({self.group_size})."
|
||||
)
|
||||
if (
|
||||
not load_full_w2
|
||||
and intermediate_size_per_partition % self.group_size != 0
|
||||
):
|
||||
raise ValueError(
|
||||
"CompressedTensors WNA16 Marlin MoE with static group "
|
||||
"scales requires the MoE intermediate size per "
|
||||
"tensor-parallel partition "
|
||||
f"({intermediate_size_per_partition}) to be divisible by "
|
||||
f"group_size ({self.group_size}). Scale groups would "
|
||||
"otherwise cross TP shard boundaries; use a compatible TP "
|
||||
"size or enable expert parallelism."
|
||||
)
|
||||
num_groups_w2 = w2_scales_size // self.group_size
|
||||
num_groups_w13 = hidden_size // self.group_size
|
||||
|
||||
layer.num_groups_w13 = num_groups_w13
|
||||
layer.num_groups_w2 = num_groups_w2
|
||||
|
||||
w13_scale = torch.nn.Parameter(
|
||||
torch.ones(
|
||||
*self.get_weight_shape(
|
||||
"w13_scale",
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition,
|
||||
num_groups_w13=num_groups_w13,
|
||||
),
|
||||
dtype=params_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight_scale", w13_scale)
|
||||
set_weight_attrs(w13_scale, extra_weight_attrs)
|
||||
|
||||
w2_scale = torch.nn.Parameter(
|
||||
torch.ones(
|
||||
*self.get_weight_shape(
|
||||
"w2_scale",
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition,
|
||||
num_groups_w2=num_groups_w2,
|
||||
),
|
||||
dtype=params_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight_scale", w2_scale)
|
||||
set_weight_attrs(w2_scale, extra_weight_attrs)
|
||||
set_weight_attrs(w2_scale, {"load_full_w2": load_full_w2})
|
||||
|
||||
if not self.symmetric:
|
||||
w13_zp = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
*self.get_weight_shape(
|
||||
"w13_zp",
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition,
|
||||
num_groups_w13=num_groups_w13,
|
||||
),
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight_zero_point", w13_zp)
|
||||
set_weight_attrs(w13_zp, extra_weight_attrs)
|
||||
|
||||
w2_zp = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
*self.get_weight_shape(
|
||||
"w2_zp",
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition,
|
||||
num_groups_w2=num_groups_w2,
|
||||
),
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight_zero_point", w2_zp)
|
||||
set_weight_attrs(w2_zp, extra_weight_attrs)
|
||||
|
||||
w2_weight_shape = torch.nn.Parameter(
|
||||
torch.empty(num_experts, 2), requires_grad=False
|
||||
)
|
||||
layer.register_parameter("w2_weight_shape", w2_weight_shape)
|
||||
set_weight_attrs(w2_weight_shape, extra_weight_attrs)
|
||||
w13_weight_shape = torch.nn.Parameter(
|
||||
torch.empty(num_experts, 2), requires_grad=False
|
||||
)
|
||||
|
||||
layer.register_parameter("w13_weight_shape", w13_weight_shape)
|
||||
set_weight_attrs(w13_weight_shape, extra_weight_attrs)
|
||||
|
||||
w13_g_idx = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight_g_idx", w13_g_idx)
|
||||
set_weight_attrs(w13_g_idx, extra_weight_attrs)
|
||||
|
||||
w2_g_idx = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
intermediate_size_per_partition,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight_g_idx", w2_g_idx)
|
||||
set_weight_attrs(w2_g_idx, extra_weight_attrs)
|
||||
|
||||
w13_g_idx_sort_indices = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_g_idx_sort_indices", w13_g_idx_sort_indices)
|
||||
set_weight_attrs(w13_g_idx_sort_indices, extra_weight_attrs)
|
||||
|
||||
w2_g_idx_sort_indices = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
intermediate_size_per_partition,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_g_idx_sort_indices", w2_g_idx_sort_indices)
|
||||
set_weight_attrs(w2_g_idx_sort_indices, extra_weight_attrs)
|
||||
|
||||
layer.a13_scale = None
|
||||
layer.a2_scale = None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
# Process weights using the shared oracle infrastructure
|
||||
converted = convert_to_wna16_moe_kernel_format(
|
||||
backend=self.wna16_backend,
|
||||
layer=layer,
|
||||
quant_config=self.weight_quant,
|
||||
input_dtype=self.marlin_input_dtype,
|
||||
w13=layer.w13_weight_packed,
|
||||
w2=layer.w2_weight_packed,
|
||||
w13_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
w13_g_idx=layer.w13_weight_g_idx,
|
||||
w2_g_idx=layer.w2_weight_g_idx,
|
||||
w13_qzeros=getattr(layer, "w13_weight_zero_point", None),
|
||||
w2_qzeros=getattr(layer, "w2_weight_zero_point", None),
|
||||
)
|
||||
|
||||
if converted is None:
|
||||
# In-place backends (e.g. Humming) are not wired through this
|
||||
# marlin-only method; fail clearly rather than unpacking None.
|
||||
raise NotImplementedError(
|
||||
f"{type(self).__name__} does not support the "
|
||||
f"{self.wna16_backend.value} MoE backend."
|
||||
)
|
||||
|
||||
(
|
||||
w13_qweight,
|
||||
w2_qweight,
|
||||
w13_scales,
|
||||
w2_scales,
|
||||
w13_g_idx_processed,
|
||||
w2_g_idx_processed,
|
||||
w13_g_idx_sort_indices,
|
||||
w2_g_idx_sort_indices,
|
||||
w13_qzeros,
|
||||
w2_qzeros,
|
||||
w13_input_global_scale,
|
||||
w2_input_global_scale,
|
||||
_, # w13_bias
|
||||
_, # w2_bias
|
||||
) = converted
|
||||
|
||||
# Replace common parameters
|
||||
replace_parameter(layer, "w13_weight_packed", w13_qweight)
|
||||
replace_parameter(layer, "w2_weight_packed", w2_qweight)
|
||||
replace_parameter(layer, "w13_weight_scale", w13_scales)
|
||||
replace_parameter(layer, "w2_weight_scale", w2_scales)
|
||||
|
||||
# CPU fused_experts_cpu requires zero points even for symmetric quant
|
||||
if not self.symmetric or self.wna16_backend == WNA16MoEBackend.CPU:
|
||||
replace_parameter(layer, "w13_weight_zero_point", w13_qzeros)
|
||||
replace_parameter(layer, "w2_weight_zero_point", w2_qzeros)
|
||||
|
||||
# Marlin-specific parameters (not needed for Flashinfer)
|
||||
if self.is_marlin:
|
||||
if w13_g_idx_processed is not None:
|
||||
replace_parameter(layer, "w13_weight_g_idx", w13_g_idx_processed)
|
||||
if w2_g_idx_processed is not None:
|
||||
replace_parameter(layer, "w2_weight_g_idx", w2_g_idx_processed)
|
||||
if w13_g_idx_sort_indices is not None:
|
||||
replace_parameter(
|
||||
layer, "w13_g_idx_sort_indices", w13_g_idx_sort_indices
|
||||
)
|
||||
if w2_g_idx_sort_indices is not None:
|
||||
replace_parameter(layer, "w2_g_idx_sort_indices", w2_g_idx_sort_indices)
|
||||
|
||||
# Register input global scales if present
|
||||
if w13_input_global_scale is not None:
|
||||
layer.register_parameter(
|
||||
"w13_input_global_scale",
|
||||
torch.nn.Parameter(w13_input_global_scale, requires_grad=False),
|
||||
)
|
||||
if w2_input_global_scale is not None:
|
||||
layer.register_parameter(
|
||||
"w2_input_global_scale",
|
||||
torch.nn.Parameter(w2_input_global_scale, requires_grad=False),
|
||||
)
|
||||
|
||||
# Marlin workspace — only needed for Marlin-family backends, not emulation.
|
||||
if (
|
||||
self.experts_cls is not None
|
||||
and issubclass(self.experts_cls, FusedMoEExpertsModular)
|
||||
and self.wna16_backend != WNA16MoEBackend.EMULATION
|
||||
):
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
layer.w13_weight_g_idx.device,
|
||||
4,
|
||||
existing=getattr(layer, "workspace", None),
|
||||
)
|
||||
|
||||
# Alias packed weights to w13_weight/w2_weight for the modular kernel interface
|
||||
layer.w13_weight = layer.w13_weight_packed
|
||||
layer.w2_weight = layer.w2_weight_packed
|
||||
|
||||
assert self.experts_cls is not None
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
assert self.moe_quant_config is not None
|
||||
|
||||
# Add Marlin-specific arguments
|
||||
marlin_args: dict[str, Any] = {}
|
||||
if self.is_marlin:
|
||||
marlin_args = {
|
||||
"w13_g_idx": layer.w13_weight_g_idx,
|
||||
"w2_g_idx": layer.w2_weight_g_idx,
|
||||
"w13_g_idx_sort_indices": layer.w13_g_idx_sort_indices,
|
||||
"w2_g_idx_sort_indices": layer.w2_g_idx_sort_indices,
|
||||
"is_k_full": self.is_k_full,
|
||||
}
|
||||
|
||||
self.moe_kernel = make_wna16_moe_kernel(
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
**marlin_args,
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: torch.nn.Module
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
return make_wna16_moe_quant_config(
|
||||
w1_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
group_size=self.group_size,
|
||||
num_bits=self.num_bits,
|
||||
w1_zp=getattr(layer, "w13_weight_zero_point", None),
|
||||
w2_zp=getattr(layer, "w2_weight_zero_point", None),
|
||||
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
)
|
||||
|
||||
def apply_monolithic(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply_monolithic(
|
||||
x,
|
||||
layer.w13_weight,
|
||||
layer.w2_weight,
|
||||
router_logits,
|
||||
activation=layer.activation,
|
||||
global_num_experts=layer.global_num_experts,
|
||||
expert_map=layer.expert_map,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
num_expert_group=layer.num_expert_group,
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
x: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
assert not self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
x,
|
||||
layer.w13_weight,
|
||||
layer.w2_weight,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
activation=layer.activation,
|
||||
global_num_experts=layer.global_num_experts,
|
||||
expert_map=layer.expert_map,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
shared_experts=shared_experts,
|
||||
shared_experts_input=shared_experts_input,
|
||||
)
|
||||
@@ -778,16 +778,6 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
"logic. This function should not be called."
|
||||
)
|
||||
|
||||
def select_gemm_impl(
|
||||
self,
|
||||
prepare_finalize: mk.FusedMoEPrepareAndFinalizeModular,
|
||||
layer: RoutedExperts,
|
||||
) -> mk.FusedMoEExpertsModular:
|
||||
raise ValueError(
|
||||
f"{self.__class__.__name__} uses the new modular kernel initialization "
|
||||
"logic. This function should not be called."
|
||||
)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
@@ -2104,16 +2094,6 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
|
||||
"logic. This function should not be called."
|
||||
)
|
||||
|
||||
def select_gemm_impl(
|
||||
self,
|
||||
prepare_finalize: mk.FusedMoEPrepareAndFinalizeModular,
|
||||
layer: RoutedExperts,
|
||||
) -> mk.FusedMoEExpertsModular:
|
||||
raise ValueError(
|
||||
f"{self.__class__.__name__} uses the new modular kernel initialization "
|
||||
"logic. This function should not be called."
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: RoutedExperts
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
|
||||
@@ -430,16 +430,6 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def select_gemm_impl(
|
||||
self,
|
||||
prepare_finalize: mk.FusedMoEPrepareAndFinalize,
|
||||
layer: RoutedExperts,
|
||||
) -> mk.FusedMoEExpertsModular:
|
||||
raise ValueError(
|
||||
f"{self.__class__.__name__} uses the new modular kernel "
|
||||
"initialization logic. This function should not be called."
|
||||
)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
@@ -893,16 +883,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def select_gemm_impl(
|
||||
self,
|
||||
prepare_finalize: mk.FusedMoEPrepareAndFinalize,
|
||||
layer: RoutedExperts,
|
||||
) -> mk.FusedMoEExpertsModular:
|
||||
raise ValueError(
|
||||
f"{self.__class__.__name__} uses the new modular kernel "
|
||||
"initialization logic. This function should not be called."
|
||||
)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm import _custom_ops as ops
|
||||
from vllm.distributed.utils import verify_group_size_divides_partition
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
|
||||
from vllm.model_executor.layers.quantization.utils.int8_utils import (
|
||||
@@ -351,10 +352,12 @@ def marlin_moe_padded_intermediate(intermediate_size: int, group_size: int = -1)
|
||||
return padded
|
||||
|
||||
|
||||
def check_moe_marlin_supports_layer(
|
||||
layer: RoutedExperts, group_size: int, allow_tile_padding: bool = False
|
||||
def check_moe_marlin_supports_config(
|
||||
config: FusedMoEConfig,
|
||||
group_size: int,
|
||||
allow_tile_padding: bool = False,
|
||||
) -> bool:
|
||||
"""Whether the fused MoE Marlin kernel supports ``layer``.
|
||||
"""Whether the fused MoE Marlin kernel supports ``config``.
|
||||
|
||||
Callers without act-order may pass ``allow_tile_padding=True``: a
|
||||
tile-misaligned intermediate size is then zero-padded to a valid thread
|
||||
@@ -364,15 +367,11 @@ def check_moe_marlin_supports_layer(
|
||||
"""
|
||||
if current_platform.is_rocm():
|
||||
return False
|
||||
hidden_size = layer.hidden_size
|
||||
hidden_size = config.hidden_dim
|
||||
# The layer has not rounded intermediate_size yet; use the stable unpadded
|
||||
# size. gate-up needs n=2*intermediate % 128, down needs k=intermediate % 64.
|
||||
intermediate_size_per_partition = (
|
||||
layer.moe_config.intermediate_size_per_partition_unpadded
|
||||
)
|
||||
intermediate_size_per_partition = config.intermediate_size_per_partition_unpadded
|
||||
assert intermediate_size_per_partition is not None
|
||||
# apply_router_weight_on_input is not supported for moe marlin
|
||||
supports_router_weight = not layer.apply_router_weight_on_input
|
||||
|
||||
if allow_tile_padding:
|
||||
supports_shape = hidden_size % 128 == 0 and (
|
||||
@@ -384,7 +383,17 @@ def check_moe_marlin_supports_layer(
|
||||
and intermediate_size_per_partition % max(64, group_size) == 0
|
||||
)
|
||||
supports_group_size = group_size in [-1, 32, 64, 128]
|
||||
return supports_shape and supports_group_size and supports_router_weight
|
||||
return supports_shape and supports_group_size
|
||||
|
||||
|
||||
def check_moe_marlin_supports_layer(
|
||||
layer: RoutedExperts,
|
||||
group_size: int,
|
||||
allow_tile_padding: bool = False,
|
||||
) -> bool:
|
||||
return check_moe_marlin_supports_config(
|
||||
layer.moe_config, group_size, allow_tile_padding
|
||||
)
|
||||
|
||||
|
||||
def marlin_moe_intermediate_size(w1_packed: torch.Tensor, w2_packed: torch.Tensor):
|
||||
|
||||
Reference in New Issue
Block a user