[feature]Add int4 quantization support for emulation moe backend (#48451)

Signed-off-by: Qiang Li <qiang.li2@amd.com>
This commit is contained in:
qli88
2026-07-16 02:00:12 +00:00
committed by GitHub
parent 915dffaa5f
commit 5de1add806
9 changed files with 1653 additions and 10 deletions
File diff suppressed because it is too large Load Diff
@@ -887,6 +887,9 @@ def int4_w4a16_moe_quant_config(
block_shape: list[int] | None = None,
a1_gscale: torch.Tensor | None = None,
a2_gscale: torch.Tensor | None = None,
gemm1_clamp_limit: float | None = None,
gemm1_alpha: float | None = None,
gemm1_beta: float | None = None,
) -> FusedMoEQuantConfig:
"""
Construct a quant config for 16-bit float activations and int4 weights.
@@ -897,6 +900,9 @@ def int4_w4a16_moe_quant_config(
_a2=FusedMoEQuantDesc(shape=group_shape, alpha_or_gscale=a2_gscale),
_w1=FusedMoEQuantDesc("int4", group_shape, w1_scale, None, w1_zp, w1_bias),
_w2=FusedMoEQuantDesc("int4", group_shape, w2_scale, None, w2_zp, w2_bias),
gemm1_clamp_limit=gemm1_clamp_limit,
gemm1_alpha=gemm1_alpha,
gemm1_beta=gemm1_beta,
)
@@ -950,6 +956,9 @@ def int8_w8a16_moe_quant_config(
block_shape: list[int] | None = None,
a1_gscale: torch.Tensor | None = None,
a2_gscale: torch.Tensor | None = None,
gemm1_clamp_limit: float | None = None,
gemm1_alpha: float | None = None,
gemm1_beta: float | None = None,
) -> FusedMoEQuantConfig:
"""
Construct a quant config for 16-bit float activations and int8 weights.
@@ -960,6 +969,9 @@ def int8_w8a16_moe_quant_config(
_a2=FusedMoEQuantDesc(shape=group_shape, alpha_or_gscale=a2_gscale),
_w1=FusedMoEQuantDesc(torch.int8, group_shape, w1_scale, None, w1_zp, w1_bias),
_w2=FusedMoEQuantDesc(torch.int8, group_shape, w2_scale, None, w2_zp, w2_bias),
gemm1_clamp_limit=gemm1_clamp_limit,
gemm1_alpha=gemm1_alpha,
gemm1_beta=gemm1_beta,
)
@@ -0,0 +1,129 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Int4 weight-only quantization emulation for MoE.
Weights are dequantized from packed int4 to BF16 once at load time;
the forward pass then runs plain TritonExperts in BF16.
"""
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.experts.triton_moe import TritonExperts
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kInt4Static,
kInt4Static32,
kInt4Static32Asym,
kInt4StaticAsym,
)
from vllm.platforms import current_platform
logger = init_logger(__name__)
class Int4EmulationTritonExperts(TritonExperts):
"""Int4 W-only MoE that dequantizes weights to BF16 at load time.
Weights arrive already dequantized (convert_to_wna16_moe_kernel_format
does the unpacking); apply() simply forwards to TritonExperts.
"""
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
):
super().__init__(moe_config, quant_config)
logger.warning_once(
"Using Int4EmulationTritonExperts MoE backend. Int4 weights are "
"dequantized to BF16 at load time "
)
# Weights are dequantized to BF16 before apply() is called, so
# TritonExperts must see them as plain float — clear the int4 dtype
# and scales so the hidden-size assertion and kernel dispatch treat
# them as unquantized.
self.quant_config._w1.dtype = None
self.quant_config._w2.dtype = None
self.quant_config._w1.scale = None
self.quant_config._w2.scale = None
@staticmethod
def _supports_current_device() -> bool:
return current_platform.is_cuda_alike()
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return (
weight_key
in (
kInt4Static,
kInt4Static32,
kInt4StaticAsym,
kInt4Static32Asym,
)
and activation_key is None
)
@property
def quant_dtype(self) -> torch.dtype | str | None:
return None
@property
def block_shape(self) -> list[int] | None:
return None
@property
def expects_unquantized_inputs(self) -> bool:
return True
def apply(
self,
output: torch.Tensor,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
a2_scale: torch.Tensor | None,
workspace13: torch.Tensor,
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
):
if w1.element_size() < 2:
raise RuntimeError(
"Int4EmulationTritonExperts.apply() received packed int4 weights "
"(element_size < 2). Weights must be dequantized to BF16 before "
"the forward pass via convert_to_wna16_moe_kernel_format."
)
return super().apply(
output=output,
hidden_states=hidden_states,
w1=w1,
w2=w2,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=activation,
global_num_experts=global_num_experts,
expert_map=expert_map,
a1q_scale=None,
a2_scale=None,
workspace13=workspace13,
workspace2=workspace2,
expert_tokens_meta=expert_tokens_meta,
apply_router_weight_on_input=apply_router_weight_on_input,
)
@@ -51,6 +51,7 @@ class WNA16MoEBackend(Enum):
CPU = "CPU"
FLASHINFER_TRTLLM = "FLASHINFER_TRTLLM"
XPU = "XPU"
EMULATION = "EMULATION"
def backend_to_kernel_cls(
@@ -87,6 +88,12 @@ def backend_to_kernel_cls(
)
return [CPUExpertsInt4]
elif backend == WNA16MoEBackend.EMULATION:
from vllm.model_executor.layers.fused_moe.experts.int4_emulation_moe import (
Int4EmulationTritonExperts,
)
return [Int4EmulationTritonExperts]
else:
raise ValueError(f"Unknown WNA16 MoE backend: {backend.value}")
@@ -105,6 +112,7 @@ def _get_priority_backends() -> list[WNA16MoEBackend]:
WNA16MoEBackend.MARLIN,
WNA16MoEBackend.BATCHED_MARLIN,
WNA16MoEBackend.HUMMING,
WNA16MoEBackend.EMULATION,
]
return _AVAILABLE_BACKENDS
@@ -115,6 +123,7 @@ def map_wna16_backend(runner_backend: MoEBackend) -> WNA16MoEBackend:
"marlin": WNA16MoEBackend.MARLIN,
"humming": WNA16MoEBackend.HUMMING,
"flashinfer_trtllm": WNA16MoEBackend.FLASHINFER_TRTLLM,
"emulation": WNA16MoEBackend.EMULATION,
}
if backend := mapping.get(runner_backend):
return backend
@@ -215,6 +224,9 @@ def make_wna16_moe_quant_config(
w2_bias: torch.Tensor | None = None,
a1_gscale: torch.Tensor | None = None,
a2_gscale: torch.Tensor | None = None,
gemm1_clamp_limit: float | None = None,
gemm1_alpha: float | None = None,
gemm1_beta: float | None = None,
) -> FusedMoEQuantConfig:
"""Create the FusedMoEQuantConfig for 4 or 8-bit WNA16 MoE."""
if num_bits == 4:
@@ -228,6 +240,9 @@ def make_wna16_moe_quant_config(
block_shape=[0, group_size],
a1_gscale=a1_gscale,
a2_gscale=a2_gscale,
gemm1_clamp_limit=gemm1_clamp_limit,
gemm1_alpha=gemm1_alpha,
gemm1_beta=gemm1_beta,
)
else:
assert num_bits == 8
@@ -241,6 +256,9 @@ def make_wna16_moe_quant_config(
block_shape=[0, group_size],
a1_gscale=a1_gscale,
a2_gscale=a2_gscale,
gemm1_clamp_limit=gemm1_clamp_limit,
gemm1_alpha=gemm1_alpha,
gemm1_beta=gemm1_beta,
)
@@ -263,19 +281,23 @@ def make_wna16_moe_kernel(
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
CPUExpertsInt4,
)
from vllm.model_executor.layers.fused_moe.experts.int4_emulation_moe import (
Int4EmulationTritonExperts,
)
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import (
XPUExpertsWNA16,
)
# Currently, we only support TrtLlmMxint4ExpertsMonolithic, MarlinExperts,
# BatchedMarlinExperts, XPUExpertsWNA16, CPUExpertsInt4, and the Humming
# grouped/indexed experts.
# BatchedMarlinExperts, XPUExpertsWNA16, CPUExpertsInt4, the Humming
# grouped/indexed experts, and Int4EmulationTritonExperts
allowed_experts: tuple[type[mk.FusedMoEExperts], ...] = (
MarlinExperts,
BatchedMarlinExperts,
TrtLlmMxint4ExpertsMonolithic,
XPUExpertsWNA16,
CPUExpertsInt4,
Int4EmulationTritonExperts,
)
if backend == WNA16MoEBackend.HUMMING:
allowed_experts += tuple(backend_to_kernel_cls(WNA16MoEBackend.HUMMING))
@@ -1017,6 +1039,243 @@ def _humming_wna16_weight_schema(
)
def _unpack_and_dequant_int4_gptq(
w_int32: torch.Tensor,
scale: torch.Tensor,
qzeros: torch.Tensor | None,
transpose_output: bool,
output_dtype: torch.dtype = torch.bfloat16,
) -> torch.Tensor:
"""Unpack GPTQ-packed int4 weights and dequantize to output_dtype.
Args:
w_int32: packed weights, shape [E, K_packed, N] where K_packed = K//8
(8 nibbles per int32, LSB-first in the K dimension).
scale: per-group scales, shape [E, K//group_size, N], float16.
qzeros: optional asymmetric zero-points, shape [E, K//gs, N//8], int32.
None for symmetric (uint4b8 with implicit bias 8).
transpose_output: if True return [E, N, K]; if False return [E, K, N].
output_dtype: target floating-point dtype (bfloat16 or float16).
Returns:
Dequantized weight tensor in the requested layout.
"""
E, K_packed, N = w_int32.shape
K = K_packed * 8
# Unpack: [E, K_packed, N] -> [E, K_packed, N, 8] via bit-shifts.
# The nibble index (last dim) enumerates K rows within each packed column,
# so we must fuse K_packed and the nibble dim, not N and the nibble dim.
# Permute to [E, K_packed, 8, N] before reshaping to [E, K, N].
shifts = torch.arange(8, device=w_int32.device, dtype=torch.int32) * 4
nibbles = (w_int32.unsqueeze(-1) >> shifts) & 0xF # [E, K_packed, N, 8]
# Reshape to [E, K, N]: fuse K_packed and nibble index (dim 1 and 3)
w = nibbles.permute(0, 1, 3, 2).reshape(E, K, N).to(torch.int16)
if qzeros is None:
# Symmetric uint4b8: subtract bias so the range is [-8, 7]
w = w - 8
else:
# Asymmetric: unpack zero-points (same 8-nibble packing) and subtract
# qzeros shape: [E, K//gs, N//8] int32
gs = K // scale.shape[1]
n_gs = scale.shape[1]
zp_shifts = torch.arange(8, device=qzeros.device, dtype=torch.int32) * 4
zp_nibbles = (qzeros.unsqueeze(-1) >> zp_shifts) & 0xF # [E, n_gs, N//8, 8]
zp = zp_nibbles.reshape(E, n_gs, N).to(torch.int16) # [E, n_gs, N]
zp = zp.repeat_interleave(gs, dim=1) # [E, K, N]
w = w - zp
# Broadcast scale [E, K//gs, N] -> [E, K, N]
gs = K // scale.shape[1]
scale_broadcast = scale.repeat_interleave(gs, dim=1).to(output_dtype)
w_dequant = w.to(output_dtype) * scale_broadcast # [E, K, N]
if transpose_output:
return w_dequant.permute(0, 2, 1).contiguous() # [E, N, K]
return w_dequant.contiguous() # [E, K, N]
def _unpack_and_dequant_int4_awq(
w_int32: torch.Tensor,
scale: torch.Tensor,
qzeros: torch.Tensor | None,
transpose_output: bool,
output_dtype: torch.dtype = torch.bfloat16,
) -> torch.Tensor:
"""Unpack AWQ-packed int4 weights and dequantize to output_dtype.
AWQ packs along the N (column) dimension with an interleave permutation
[0,2,4,6,1,3,5,7] applied before packing, so unpacking must undo that.
Args:
w_int32: packed weights, shape [E, K, N_packed] where N_packed = N//8
(8 nibbles per int32, packed along N with AWQ interleaving).
scale: per-group scales, shape [E, K//group_size, N], float16.
qzeros: asymmetric zero-points, shape [E, K//gs, N_packed], int32.
None for symmetric (uint4b8 with implicit bias 8).
transpose_output: if True return [E, N, K]; if False return [E, K, N].
output_dtype: target floating-point dtype (bfloat16 or float16).
Returns:
Dequantized weight tensor in the requested layout.
"""
E, K, N_packed = w_int32.shape
N = N_packed * 8
# Unpack 8 nibbles per int32 along the N dimension (LSB-first)
shifts = torch.arange(8, device=w_int32.device, dtype=torch.int32) * 4
# [E, K, N_packed, 8] -> [E, K, N_packed*8] = [E, K, N_interleaved]
nibbles = (w_int32.unsqueeze(-1) >> shifts) & 0xF
w_interleaved = nibbles.reshape(E, K, N) # [E, K, N] but column-interleaved
# Undo AWQ interleave: packed order is [0,2,4,6,1,3,5,7] within each group
# of 8. Inverse: position i in packed -> original column interleave[i].
# To reverse: we need the inverse permutation so that
# w[:, :, inv_interleave] = w_interleaved gives the natural column order.
interleave = torch.tensor([0, 2, 4, 6, 1, 3, 5, 7], device=w_int32.device)
inv_interleave = torch.empty_like(interleave)
inv_interleave[interleave] = torch.arange(8, device=w_int32.device)
# Apply inverse interleave within each group of 8 columns
w_reshaped = w_interleaved.reshape(E, K, N // 8, 8) # [E, K, groups, 8]
w_reordered = w_reshaped[:, :, :, inv_interleave] # undo interleave
w = w_reordered.reshape(E, K, N).to(torch.int16) # [E, K, N]
if qzeros is None:
w = w - 8
else:
# qzeros: [E, K//gs, N_packed] int32, same AWQ column packing
gs = K // scale.shape[1]
n_gs = scale.shape[1]
zp_nibbles = (qzeros.unsqueeze(-1) >> shifts) & 0xF # [E, n_gs, N_packed, 8]
zp_interleaved = zp_nibbles.reshape(E, n_gs, N)
zp_reshaped = zp_interleaved.reshape(E, n_gs, N // 8, 8)
zp_reordered = zp_reshaped[:, :, :, inv_interleave]
zp = zp_reordered.reshape(E, n_gs, N).to(torch.int16) # [E, n_gs, N]
zp = zp.repeat_interleave(gs, dim=1) # [E, K, N]
w = w - zp
gs = K // scale.shape[1]
scale_broadcast = scale.repeat_interleave(gs, dim=1).to(output_dtype) # [E, K, N]
w_dequant = w.to(output_dtype) * scale_broadcast # [E, K, N]
if transpose_output:
return w_dequant.permute(0, 2, 1).contiguous() # [E, N, K]
return w_dequant.contiguous() # [E, K, N]
def _process_weights_emulation_gptq(
w13: torch.Tensor,
w2: torch.Tensor,
w13_scale: torch.Tensor,
w2_scale: torch.Tensor,
w13_qzeros: torch.Tensor | None,
w2_qzeros: torch.Tensor | None,
) -> tuple:
"""Dequantize int4 weights to BF16 for the emulation backend.
Inputs are in GPTQ packed format:
w13: [E, K//8, 2*N] int32 (gate+up proj stacked on dim 2)
w2: [E, N//8, K] int32
w13_scale: [E, K//gs, 2*N] float16
w2_scale: [E, N//gs, K] float16
Outputs (what TritonExperts expects):
w13_out: [E, 2*N, K] bfloat16
w2_out: [E, K, N] bfloat16
"""
# w13: packed along K (dim 1), output cols are 2*N (dim 2)
# transpose_output=True yields [E, 2*N, K]
w13_bf16 = _unpack_and_dequant_int4_gptq(
w13, w13_scale, w13_qzeros, transpose_output=True
)
# w2: packed along N (dim 1 is N//8), output cols are K (dim 2)
# After unpacking we get [E, N, K]; we want [E, K, N] for TritonExperts
# transpose_output=False gives [E, N, K], then we permute once more
w2_unpacked = _unpack_and_dequant_int4_gptq(
w2, w2_scale, w2_qzeros, transpose_output=False
) # [E, N, K]
w2_bf16 = w2_unpacked.permute(0, 2, 1).contiguous() # [E, K, N]
dummy = torch.ones(1, dtype=torch.float16, device=w13.device)
return (
w13_bf16, # w13_qweight (now bf16, not int32)
w2_bf16, # w2_qweight (now bf16, not int32)
dummy, # w13_scales (unused; nulled out in Int4EmulationTritonExperts)
dummy, # w2_scales (unused)
None, # w13_g_idx
None, # w2_g_idx
None, # w13_g_idx_sort_indices
None, # w2_g_idx_sort_indices
None, # w13_qzeros
None, # w2_qzeros
None, # w13_input_global_scale
None, # w2_input_global_scale
None, # w13_bias
None, # w2_bias
)
def _process_weights_emulation_awq(
w13: torch.Tensor,
w2: torch.Tensor,
w13_scale: torch.Tensor,
w2_scale: torch.Tensor,
w13_qzeros: torch.Tensor | None,
w2_qzeros: torch.Tensor | None,
) -> tuple:
"""Dequantize AWQ int4 weights to BF16 for the emulation backend.
AWQ inputs:
w13: [E, K, 2*N//8] int32 (packed along N, gate+up on dim 2)
w2: [E, N, K//8] int32 (packed along K)
w13_scale: [E, K//gs, 2*N] float16
w2_scale: [E, N//gs, K] float16
Outputs (what TritonExperts expects):
w13_out: [E, 2*N, K] bfloat16
w2_out: [E, K, N] bfloat16
"""
# w13: AWQ-packed along N (dim 2), K is unpacked in dim 1
# _unpack_and_dequant_int4_awq with transpose_output=True yields [E, 2*N, K]
w13_bf16 = _unpack_and_dequant_int4_awq(
w13, w13_scale, w13_qzeros, transpose_output=True
)
# w2: AWQ packs along K (dim 2 is K//8), N is unpacked in dim 1.
# AWQ w2 is [E, N, K//8] — same column-pack format applied to the K dim.
# _unpack_and_dequant_int4_awq expects [E, rows, N_packed] where the
# packed dim is columns. Treat dim 1 as rows and dim 2 as N_packed:
# unpacking gives [E, N, K]. Then permute to [E, K, N].
w2_unpacked = _unpack_and_dequant_int4_awq(
w2, w2_scale, w2_qzeros, transpose_output=False
) # [E, N, K]
w2_bf16 = w2_unpacked.permute(0, 2, 1).contiguous() # [E, K, N]
dummy = torch.ones(1, dtype=torch.float16, device=w13.device)
return (
w13_bf16,
w2_bf16,
dummy,
dummy,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
)
def convert_to_wna16_moe_kernel_format(
backend: WNA16MoEBackend,
layer: torch.nn.Module,
@@ -1203,5 +1462,25 @@ def convert_to_wna16_moe_kernel_format(
w13_bias_out,
w2_bias_out,
)
elif backend == WNA16MoEBackend.EMULATION:
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
if isinstance(quant_config, AutoAWQConfig):
return _process_weights_emulation_awq(
w13,
w2,
w13_scale,
w2_scale,
w13_qzeros,
w2_qzeros,
)
return _process_weights_emulation_gptq(
w13,
w2,
w13_scale,
w2_scale,
w13_qzeros,
w2_qzeros,
)
else:
raise ValueError(f"Unsupported wna16 MoE backend: {backend.value}")
@@ -774,6 +774,9 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
w2_bias=getattr(layer, "w2_bias", None),
a1_gscale=getattr(layer, "w13_input_global_scale", None),
a2_gscale=getattr(layer, "w2_input_global_scale", None),
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
)
def select_gemm_impl(
@@ -142,6 +142,23 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
return CompressedTensorsW4A16FlydslMoEMethod(
weight_quant, input_quant, layer.moe_config
)
elif moe_backend == "emulation":
# Although this is called 'Marlin', actually it selects
# emulation backend by calling select_wna16_moe_backend.
# TODO: we need to update CompressedTensorsWNA16MoeMethod
# to honor "--moe-backend" option
from .compressed_tensors_moe_wna16_marlin import (
CompressedTensorsWNA16MarlinMoEMethod,
)
logger.info_once(
"Using CompressedTensorsWNA16MarlinMoEMethod "
"(emulation backend requested)"
)
return CompressedTensorsWNA16MarlinMoEMethod(
weight_quant, input_quant, layer.moe_config, layer_name
)
from .compressed_tensors_moe_wna16 import (
CompressedTensorsWNA16MoEMethod,
)
@@ -108,7 +108,8 @@ class CompressedTensorsW4A16FlydslMoEMethod(CompressedTensorsMoEMethod):
# grouped actorder isn't supported by this kernel
assert weight_quant.actorder != "group"
assert weight_quant.symmetric, (
"Only symmetric quantization is supported for MoE"
"Only symmetric quantization is supported for MoE. "
"Try --moe-backend emulation."
)
def create_weights(
@@ -48,7 +48,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
# grouped actorder isn't supported by this kernel
assert weight_quant.actorder != "group"
assert weight_quant.symmetric, (
"Only symmetric quantization is supported for MoE"
"Only symmetric quantization is supported for MoE. "
"Try --moe-backend emulation."
)
def create_weights(
@@ -467,10 +467,16 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
# Marlin-specific parameters (not needed for Flashinfer)
if not is_flashinfer:
replace_parameter(layer, "w13_weight_g_idx", w13_g_idx_processed)
replace_parameter(layer, "w2_weight_g_idx", w2_g_idx_processed)
replace_parameter(layer, "w13_g_idx_sort_indices", w13_g_idx_sort_indices)
replace_parameter(layer, "w2_g_idx_sort_indices", w2_g_idx_sort_indices)
if w13_g_idx_processed is not None:
replace_parameter(layer, "w13_weight_g_idx", w13_g_idx_processed)
if w2_g_idx_processed is not None:
replace_parameter(layer, "w2_weight_g_idx", w2_g_idx_processed)
if w13_g_idx_sort_indices is not None:
replace_parameter(
layer, "w13_g_idx_sort_indices", w13_g_idx_sort_indices
)
if w2_g_idx_sort_indices is not None:
replace_parameter(layer, "w2_g_idx_sort_indices", w2_g_idx_sort_indices)
# Register input global scales if present
if w13_input_global_scale is not None:
@@ -484,8 +490,11 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
torch.nn.Parameter(w2_input_global_scale, requires_grad=False),
)
if self.experts_cls is not None and issubclass(
self.experts_cls, FusedMoEExpertsModular
# Marlin workspace — only needed for Marlin-family backends, not emulation.
if (
self.experts_cls is not None
and issubclass(self.experts_cls, FusedMoEExpertsModular)
and self.wna16_backend != WNA16MoEBackend.EMULATION
):
layer.workspace = marlin_make_workspace_new(
layer.w13_weight_g_idx.device, 4
@@ -528,6 +537,9 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
num_bits=self.num_bits,
w1_zp=getattr(layer, "w13_weight_zero_point", None),
w2_zp=getattr(layer, "w2_weight_zero_point", None),
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
)
def apply_monolithic(