[MoE Refactor] Migrate MoeWNA16Method quantization method over to using the new MK oracle scheme. (#44120)

Signed-off-by: Bill Nell <bnell@redhat.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
bnellnm
2026-07-21 19:20:30 -07:00
committed by GitHub
co-authored by Claude mgoin
parent 1750e443f2
commit ec59c1579f
11 changed files with 811 additions and 201 deletions
+79
View File
@@ -5,13 +5,17 @@
Run `pytest tests/quantization/test_auto_gptq.py -v -s`.
"""
from types import SimpleNamespace
import pytest
import torch
from tests.quantization.utils import is_quant_method_supported
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.quantization.auto_gptq import (
AutoGPTQConfig,
AutoGPTQLinearMethod,
AutoGPTQMoEMethod,
)
PROMPT = "On the surface of Mars, we found"
@@ -54,3 +58,78 @@ def test_auto_gptq_quantization_method(vllm_runner, model_id: str, monkeypatch):
def test_auto_gptq_config_get_name():
"""Test that AutoGPTQConfig.get_name() returns 'auto_gptq'."""
assert AutoGPTQConfig.get_name() == "auto_gptq"
def test_auto_gptq_moe_creates_zero_initialized_expert_biases():
method = object.__new__(AutoGPTQMoEMethod)
method.quant_config = AutoGPTQConfig(4, 128, False, True, False, {}, {})
method.input_dtype = None
method.experts_cls = None
layer = torch.nn.Module()
method.create_weights(
layer=layer,
num_experts=2,
hidden_size=8,
intermediate_size_per_partition=4,
params_dtype=torch.float16,
intermediate_size_full=4,
weight_loader=lambda *args, **kwargs: None,
)
assert layer.w13_bias.shape == (2, 8)
assert layer.w2_bias.shape == (2, 8)
assert torch.count_nonzero(layer.w13_bias) == 0
assert torch.count_nonzero(layer.w2_bias) == 0
def test_routed_experts_loads_per_expert_biases():
class Loader:
quant_config = None
quant_method = object()
moe_config = SimpleNamespace(
is_act_and_mul=True,
tp_rank=0,
moe_parallel_config=SimpleNamespace(tp_size=1),
)
_get_hidden_dim = staticmethod(RoutedExperts._get_hidden_dim)
_narrow_expert_data_for_padding = staticmethod(
RoutedExperts._narrow_expert_data_for_padding
)
_load_w13 = RoutedExperts._load_w13
_loaded_expert_biases = set()
@staticmethod
def _map_global_expert_id_to_local_expert_id(expert_id):
return expert_id
loader = Loader()
w13_bias = torch.nn.Parameter(torch.zeros(1, 8), requires_grad=False)
w2_bias = torch.nn.Parameter(torch.zeros(1, 4), requires_grad=False)
for shard_id, loaded in (
("w1", torch.tensor([1.0, 2.0, 3.0, 4.0])),
("w3", torch.tensor([5.0, 6.0, 7.0, 8.0])),
):
assert RoutedExperts.weight_loader(
loader,
w13_bias,
loaded,
weight_name="model.layers.0.mlp.experts.w13_bias",
shard_id=shard_id,
expert_id=0,
return_success=True,
)
assert RoutedExperts.weight_loader(
loader,
w2_bias,
torch.tensor([9.0, 10.0, 11.0, 12.0]),
weight_name="model.layers.0.mlp.experts.w2_bias",
shard_id="w2",
expert_id=0,
return_success=True,
)
assert torch.equal(w13_bias, torch.arange(1, 9, dtype=torch.float32).reshape(1, 8))
assert torch.equal(w2_bias, torch.arange(9, 13, dtype=torch.float32).reshape(1, 4))
assert loader._loaded_expert_biases == {"w13_bias", "w2_bias"}
+181 -32
View File
@@ -5,45 +5,194 @@ from types import SimpleNamespace
import pytest
import torch
from compressed_tensors.quantization import (
ActivationOrdering,
QuantizationArgs,
QuantizationStrategy,
QuantizationType,
)
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Method
from vllm.platforms import current_platform
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
WNA16MoEBackend,
_backend_incompatibility_reason,
_convert_moe_wna16_humming_tensors,
convert_to_wna16_moe_kernel_format,
map_wna16_backend,
)
from vllm.model_executor.layers.quantization import moe_wna16
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
from vllm.model_executor.layers.quantization.moe_wna16 import (
MoeWNA16Config,
MoeWNA16Method,
)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test on CUDA")
def test_moe_wna16_apply_passes_layer_activation(monkeypatch):
captured_kwargs = {}
def test_map_wna16_backend_supports_triton():
assert map_wna16_backend("triton") == WNA16MoEBackend.TRITON
def fake_fused_experts(*args, **kwargs):
captured_kwargs.update(kwargs)
return torch.empty(1, 2)
monkeypatch.setattr(
"vllm.model_executor.layers.fused_moe.fused_experts",
fake_fused_experts,
@pytest.mark.parametrize(
("backend", "quant_config", "may_have_zp", "may_have_bias", "expected"),
[
(
WNA16MoEBackend.TRITON,
AutoAWQConfig(4, 128, True, False),
True,
False,
"AutoAWQ weight layout",
),
(
WNA16MoEBackend.TRITON,
AutoGPTQConfig(4, 128, True, True, False, {}, {}),
False,
False,
"activation ordering",
),
(
WNA16MoEBackend.TRITON,
QuantizationArgs(
num_bits=4,
type=QuantizationType.INT,
strategy=QuantizationStrategy.GROUP,
symmetric=True,
dynamic=False,
group_size=128,
actorder=ActivationOrdering.GROUP,
),
False,
False,
"activation ordering",
),
(
WNA16MoEBackend.TRITON,
AutoGPTQConfig(4, 128, False, True, False, {}, {}),
False,
True,
"bias",
),
(
WNA16MoEBackend.MARLIN,
MoeWNA16Config(
linear_quant_method="gptq",
weight_bits=4,
group_size=128,
has_zp=False,
lm_head_quantized=False,
modules_to_not_convert=None,
full_config={},
),
False,
False,
"MoeWNA16 checkpoint layout",
),
],
)
def test_wna16_oracle_rejects_incompatible_quant_structures(
backend, quant_config, may_have_zp, may_have_bias, expected
):
reason = _backend_incompatibility_reason(
backend=backend,
quant_config=quant_config,
may_have_zp=may_have_zp,
may_have_bias=may_have_bias,
)
assert reason is not None
assert expected in reason
def test_compressed_tensors_weights_are_transposed_for_triton():
quant_config = QuantizationArgs(
num_bits=4,
type=QuantizationType.INT,
strategy=QuantizationStrategy.GROUP,
symmetric=True,
dynamic=False,
group_size=32,
)
w13 = torch.arange(16, dtype=torch.int32).reshape(1, 2, 8)
w2 = torch.arange(12, dtype=torch.int32).reshape(1, 2, 6)
w13_scale = torch.arange(32, dtype=torch.float16).reshape(1, 4, 8)
w2_scale = torch.arange(18, dtype=torch.float16).reshape(1, 3, 6)
converted = convert_to_wna16_moe_kernel_format(
backend=WNA16MoEBackend.TRITON,
layer=torch.nn.Module(),
quant_config=quant_config,
input_dtype=None,
w13=w13,
w2=w2,
w13_scale=w13_scale,
w2_scale=w2_scale,
)
assert converted is not None
assert torch.equal(converted[0], w13.transpose(1, 2).contiguous().view(torch.uint8))
assert torch.equal(converted[1], w2.transpose(1, 2).contiguous().view(torch.uint8))
assert torch.equal(converted[2], w13_scale.transpose(1, 2).contiguous())
assert torch.equal(converted[3], w2_scale.transpose(1, 2).contiguous())
def test_moe_wna16_setup_forwards_selected_backend(monkeypatch):
method = object.__new__(MoeWNA16Method)
method.experts_cls = object
method.wna16_backend = WNA16MoEBackend.HUMMING
method.moe = object()
quant_config = object()
method.get_fused_moe_quant_config = lambda layer: quant_config
layer = SimpleNamespace(_expert_routing_tables=lambda: (None, None, None))
captured = {}
kernel = object()
def fake_make_wna16_moe_kernel(**kwargs):
captured.update(kwargs)
return kernel
monkeypatch.setattr(moe_wna16, "make_wna16_moe_kernel", fake_make_wna16_moe_kernel)
method._setup_kernel(layer)
assert method.moe_kernel is kernel
assert captured["backend"] == WNA16MoEBackend.HUMMING
assert captured["layer"] is layer
def test_moe_wna16_humming_adapter_repacks_uint8_tensors():
qweight = torch.arange(32, dtype=torch.uint8).reshape(1, 4, 8)
scales = torch.arange(16, dtype=torch.float16).reshape(1, 4, 4)
qzeros = torch.arange(16, dtype=torch.uint8).reshape(1, 8, 2)
converted = _convert_moe_wna16_humming_tensors(
{"qweight": qweight, "scales": scales, "qzeros": qzeros},
has_zero_point=True,
)
assert torch.equal(converted["weight"], qweight.view(torch.int32))
assert converted["weight"].shape == (1, 4, 2)
assert torch.equal(converted["weight_scale"], scales)
expected_qzeros = (
qzeros.transpose(-1, -2)
.contiguous()
.view(torch.int32)
.transpose(-1, -2)
.contiguous()
)
assert torch.equal(converted["zero_point"], expected_qzeros)
assert converted["zero_point"].shape == (1, 2, 2)
def test_moe_wna16_uses_humming_quant_config(monkeypatch):
from vllm.model_executor.layers.quantization.utils import humming_utils
method = object.__new__(MoeWNA16Method)
method.moe = SimpleNamespace(disable_inplace=False)
method.moe_quant_config = object()
layer = SimpleNamespace(
w13_qweight=torch.empty(1, 2),
w2_qweight=torch.empty(1, 2),
activation=MoEActivation.GELU_TANH,
apply_router_weight_on_input=False,
global_num_experts=1,
expert_map=None,
method.wna16_backend = WNA16MoEBackend.HUMMING
layer = object()
quant_config = object()
monkeypatch.setattr(
humming_utils,
"get_humming_moe_quant_config",
lambda actual_layer: quant_config if actual_layer is layer else None,
)
output = method.apply(
layer,
x=torch.empty(1, 2),
topk_weights=torch.empty(1, 1),
topk_ids=torch.empty(1, 1, dtype=torch.int32),
shared_experts=None,
shared_experts_input=None,
)
assert output.shape == (1, 2)
assert captured_kwargs["activation"] is MoEActivation.GELU_TANH
assert method.get_fused_moe_quant_config(layer) is quant_config
@@ -43,7 +43,10 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Static128BlockSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
kInt4Static,
kInt4Static32,
kInt8DynamicTokenSym,
kInt8Static,
kInt8StaticChannelSym,
)
from vllm.platforms import current_platform
@@ -542,40 +545,45 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
class TritonWNA16Experts(TritonExperts):
@staticmethod
def _supports_current_device() -> bool:
raise NotImplementedError(
"TritonWNA16Experts is not yet used by an Oracle. "
"This method should not be called."
)
return current_platform.is_cuda_alike() or current_platform.is_xpu()
@staticmethod
def _supports_no_act_and_mul() -> bool:
raise NotImplementedError(
"TritonWNA16Experts is not yet used by an Oracle. "
"This method should not be called."
)
return True
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
raise NotImplementedError(
"TritonWNA16Experts is not yet used by an Oracle. "
"This method should not be called."
)
SUPPORTED_W = [
kInt4Static,
kInt8Static,
kInt4Static32,
# other group sizes?
]
return weight_key in SUPPORTED_W
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
raise NotImplementedError(
"TritonWNA16Experts is not yet used by an Oracle. "
"This method should not be called."
)
return activation in [
MoEActivation.SILU,
MoEActivation.GELU,
MoEActivation.GELU_TANH,
MoEActivation.SWIGLUOAI,
MoEActivation.SWIGLUSTEP,
MoEActivation.SILU_NO_MUL,
MoEActivation.GELU_NO_MUL,
MoEActivation.GELU_TANH_NO_MUL,
MoEActivation.RELU2_NO_MUL,
]
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
raise NotImplementedError(
"TritonWNA16Experts is not yet used by an Oracle. "
"This method should not be called."
# Why?
return not (
moe_parallel_config.use_fi_nvl_two_sided_kernels
or moe_parallel_config.use_fi_nvl_one_sided_kernels
)
def apply(
@@ -598,7 +606,9 @@ class TritonWNA16Experts(TritonExperts):
):
# Check constraints.
if self.quant_config.use_int4_w4a16:
assert hidden_states.size(-1) // 2 == w1.size(2), "Hidden size mismatch"
assert hidden_states.size(-1) // 2 == w1.size(2), (
f"Hidden size mismatch {hidden_states.size(-1) // 2} == {w1.size(2)}"
)
else:
assert hidden_states.size(-1) == w1.size(2), (
f"Hidden size mismatch {hidden_states.size(-1)} != {w1.size(2)}"
@@ -31,6 +31,7 @@ from vllm.model_executor.layers.fused_moe.utils import (
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import next_power_of_2
from vllm.utils.platform_utils import get_device_name_as_file_name
from vllm.utils.torch_utils import direct_register_custom_op
@@ -1298,7 +1299,11 @@ def get_default_config(
bit = 4 if dtype == "int4_w4a16" else 8
use_moe_wna16_cuda = should_moe_wna16_use_cuda(M * topk, block_shape[1], E, bit)
if use_moe_wna16_cuda:
config = {"BLOCK_SIZE_M": min(16, M), "SPLIT_K": 1}
config = {
"BLOCK_SIZE_M": min(16, next_power_of_2(M)),
"GROUP_SIZE_M": 1,
"SPLIT_K": 1,
}
elif M <= 20:
config = {"BLOCK_SIZE_M": 16, "GROUP_SIZE_M": 1, "SPLIT_K": 1}
elif M <= 40:
@@ -24,6 +24,9 @@ from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
MarlinExperts,
MarlinExpertsBase,
)
from vllm.model_executor.layers.fused_moe.experts.triton_moe import (
TritonWNA16Experts,
)
from vllm.model_executor.layers.fused_moe.experts.trtllm_mxint4_moe import (
TrtLlmMxint4ExpertsMonolithic,
)
@@ -50,6 +53,7 @@ class WNA16MoEBackend(Enum):
HUMMING = "HUMMING"
CPU = "CPU"
FLASHINFER_TRTLLM = "FLASHINFER_TRTLLM"
TRITON = "TRITON"
XPU = "XPU"
EMULATION = "EMULATION"
@@ -76,6 +80,8 @@ def backend_to_kernel_cls(
return [BatchedMarlinExperts]
elif backend == WNA16MoEBackend.FLASHINFER_TRTLLM:
return [TrtLlmMxint4ExpertsMonolithic]
elif backend == WNA16MoEBackend.TRITON:
return [TritonWNA16Experts]
elif backend == WNA16MoEBackend.XPU:
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import (
XPUExpertsWNA16,
@@ -107,19 +113,56 @@ def _get_priority_backends() -> list[WNA16MoEBackend]:
if current_platform.is_xpu():
return [WNA16MoEBackend.XPU]
_AVAILABLE_BACKENDS = [
return [
WNA16MoEBackend.FLASHINFER_TRTLLM,
WNA16MoEBackend.MARLIN,
WNA16MoEBackend.BATCHED_MARLIN,
WNA16MoEBackend.TRITON,
WNA16MoEBackend.HUMMING,
WNA16MoEBackend.EMULATION,
]
return _AVAILABLE_BACKENDS
def _backend_incompatibility_reason(
backend: WNA16MoEBackend,
quant_config: QuantizationConfig | QuantizationArgs,
may_have_zp: bool,
may_have_bias: bool,
) -> str | None:
if backend == WNA16MoEBackend.FLASHINFER_TRTLLM and (may_have_zp or may_have_bias):
return "zero points and bias are not supported"
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
if backend == WNA16MoEBackend.TRITON:
if may_have_bias:
return "expert bias is not supported"
if isinstance(quant_config, AutoAWQConfig):
return "the AutoAWQ weight layout is not supported"
if isinstance(quant_config, AutoGPTQConfig) and quant_config.desc_act:
return "GPTQ activation ordering is not supported"
if (
isinstance(quant_config, QuantizationArgs)
and quant_config.actorder == "group"
):
return "group activation ordering is not supported"
if isinstance(quant_config, MoeWNA16Config) and backend in (
WNA16MoEBackend.MARLIN,
WNA16MoEBackend.BATCHED_MARLIN,
WNA16MoEBackend.EMULATION,
):
return "the MoeWNA16 checkpoint layout is not supported"
return None
def map_wna16_backend(runner_backend: MoEBackend) -> WNA16MoEBackend:
"""Map user's MoEBackend to WNA16MoEBackend."""
mapping = {
"triton": WNA16MoEBackend.TRITON,
"marlin": WNA16MoEBackend.MARLIN,
"humming": WNA16MoEBackend.HUMMING,
"flashinfer_trtllm": WNA16MoEBackend.FLASHINFER_TRTLLM,
@@ -136,6 +179,9 @@ def map_wna16_backend(runner_backend: MoEBackend) -> WNA16MoEBackend:
def select_wna16_moe_backend(
config: FusedMoEConfig,
weight_key: QuantKey,
quant_config: QuantizationConfig | QuantizationArgs,
may_have_zp: bool,
may_have_bias: bool,
) -> tuple[WNA16MoEBackend, type[mk.FusedMoEExperts]]:
"""Select the WNA16 MoE backend.
@@ -143,6 +189,9 @@ def select_wna16_moe_backend(
config: the shared ``FusedMoEConfig`` for this layer.
weight_key: The QuantKey describing the weight quantization.
Must have int4 or int8 type.
quant_config: Quantization structure and checkpoint format description.
may_have_zp: Whether the integration can provide weight zero points.
may_have_bias: Whether the integration can provide expert bias.
Returns:
A tuple of (``WNA16MoEBackend``, experts class or ``None``).
@@ -189,6 +238,11 @@ def select_wna16_moe_backend(
runner_backend = config.moe_backend
if runner_backend != "auto":
requested_backend = map_wna16_backend(runner_backend)
reason = _backend_incompatibility_reason(
requested_backend, quant_config, may_have_zp, may_have_bias
)
if reason is not None:
raise ValueError(_make_log_unsupported(requested_backend, reason))
return _return_or_raise(
requested_backend, config, weight_key, None, activation_format
)
@@ -197,6 +251,12 @@ def select_wna16_moe_backend(
AVAILABLE_BACKENDS = _get_priority_backends()
for backend in AVAILABLE_BACKENDS:
reason = _backend_incompatibility_reason(
backend, quant_config, may_have_zp, may_have_bias
)
if reason is not None:
logger.debug_once(_make_log_unsupported(backend, reason), scope="local")
continue
activation_key = None # always BF16 activation for WNA16 MoE
for k_cls in backend_to_kernel_cls(backend):
supported, reason = k_cls.is_supported_config(
@@ -294,6 +354,7 @@ def make_wna16_moe_kernel(
allowed_experts: tuple[type[mk.FusedMoEExperts], ...] = (
MarlinExperts,
BatchedMarlinExperts,
TritonWNA16Experts,
TrtLlmMxint4ExpertsMonolithic,
XPUExpertsWNA16,
CPUExpertsInt4,
@@ -315,6 +376,7 @@ def make_wna16_moe_kernel(
assert prepare_finalize is not None
logger.info_once("Using %s", prepare_finalize.__class__.__name__, scope="local")
logger.info_once("Using %s", experts_cls.__name__, scope="local")
extra_args: dict[str, Any] = {}
if backend == WNA16MoEBackend.HUMMING:
@@ -329,35 +391,17 @@ def make_wna16_moe_kernel(
"is_k_full": is_k_full,
}
if experts_cls is XPUExpertsWNA16:
assert (
prepare_finalize.activation_format == mk.FusedMoEActivationFormat.Standard
), (
"XPUExpertsWNA16 only supports the Standard activation format; "
"xpu_fused_moe(is_int4=True) does not implement BatchedExperts."
)
experts: mk.FusedMoEExperts = XPUExpertsWNA16(
moe_config=moe_config,
quant_config=moe_quant_config,
)
elif (
prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts
):
if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
assert max_num_tokens is not None
experts = experts_cls(
max_num_tokens=max_num_tokens,
num_dispatchers=prepare_finalize.num_dispatchers(),
moe_config=moe_config,
quant_config=moe_quant_config,
**extra_args,
)
else:
experts = experts_cls(
moe_config=moe_config,
quant_config=moe_quant_config,
**extra_args,
)
extra_args["max_num_tokens"] = max_num_tokens
extra_args["num_dispatchers"] = prepare_finalize.num_dispatchers()
experts = experts_cls(
moe_config=moe_config,
quant_config=moe_quant_config,
**extra_args,
)
return mk.FusedMoEKernel(
prepare_finalize,
@@ -1034,11 +1078,68 @@ def _humming_wna16_weight_schema(
"sym": quant_config.is_sym,
}
raise TypeError(
"Humming WNA16 MoE requires AutoAWQConfig or AutoGPTQConfig, "
"Humming WNA16 checkpoint schema requires AutoAWQConfig or "
"AutoGPTQConfig, "
f"got {type(quant_config).__name__}."
)
def _convert_moe_wna16_humming_tensors(
tensors: dict[str, torch.Tensor], has_zero_point: bool
) -> dict[str, torch.Tensor]:
"""Convert MoeWNA16's N-first uint8 packing to Humming's int32 packing."""
if sys.byteorder != "little":
raise NotImplementedError(
"MoeWNA16 to Humming conversion requires a little-endian host."
)
output = {
"weight": tensors["qweight"].contiguous().view(torch.int32),
"weight_scale": tensors["scales"],
}
if has_zero_point:
qzeros = tensors["qzeros"]
output["zero_point"] = (
qzeros.transpose(-1, -2)
.contiguous()
.view(torch.int32)
.transpose(-1, -2)
.contiguous()
)
return output
class _MoeWNA16HummingWeightSchema:
"""Adapter from MoeWNA16's generic packed layout to Humming's layout."""
def __init__(self, bits: int, group_size: int, has_zero_point: bool) -> None:
self.bits = bits
self.group_size = group_size
self.has_zero_point = has_zero_point
def convert_humming(
self,
tensors: dict[str, torch.Tensor],
shape_n_stacks: list[int],
shape_k_stacks: list[int],
param_dtype: torch.dtype,
num_experts: int | None = None,
) -> tuple[Any, dict[str, torch.Tensor]]:
del shape_n_stacks, shape_k_stacks, num_experts
from vllm.utils.humming import HummingWeightSchema, dtypes
output = _convert_moe_wna16_humming_tensors(
tensors, has_zero_point=self.has_zero_point
)
output["weight_scale"] = output["weight_scale"].to(param_dtype)
schema = HummingWeightSchema(
b_dtype=dtypes.DataType.from_str(f"uint{self.bits}"),
weight_scale_group_size=self.group_size,
has_zero_point=self.has_zero_point,
)
return schema, output
def _unpack_and_dequant_int4_gptq(
w_int32: torch.Tensor,
scale: torch.Tensor,
@@ -1323,13 +1424,27 @@ def convert_to_wna16_moe_kernel_format(
input_dtype: optional activation dtype, usually should be 16 bit.
"""
if backend == WNA16MoEBackend.HUMMING:
from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
from vllm.model_executor.layers.quantization.utils.humming_utils import (
convert_to_humming_moe_kernel_format,
)
convert_to_humming_moe_kernel_format(
layer, quant_config=_humming_wna16_weight_schema(quant_config)
)
if isinstance(quant_config, MoeWNA16Config):
from vllm.utils.humming import HummingInputSchema
convert_to_humming_moe_kernel_format(
layer,
weight_schema=_MoeWNA16HummingWeightSchema(
bits=quant_config.weight_bits,
group_size=layer.group_size,
has_zero_point=quant_config.has_zp,
),
input_schema=HummingInputSchema(),
)
else:
convert_to_humming_moe_kernel_format(
layer, quant_config=_humming_wna16_weight_schema(quant_config)
)
return None
if backend in (
@@ -1344,28 +1459,9 @@ def convert_to_wna16_moe_kernel_format(
)
if isinstance(quant_config, AutoAWQConfig):
if w13_qzeros is None or w2_qzeros is None:
raise ValueError("AWQ Marlin MoE requires zero-point tensors.")
weight_bits = quant_config.weight_bits
num_bits = quant_config.weight_bits
pack_factor = quant_config.pack_factor
group_size = quant_config.group_size
return _process_awq_weights_marlin(
layer,
weight_bits,
pack_factor,
group_size,
input_dtype,
w13,
w2,
w13_scale,
w2_scale,
w13_qzeros,
w2_qzeros,
w13_bias,
w2_bias,
)
elif isinstance(quant_config, AutoGPTQConfig):
num_bits = quant_config.quant_type.size_bits
pack_factor = quant_config.pack_factor
@@ -1378,29 +1474,51 @@ def convert_to_wna16_moe_kernel_format(
actorder = quant_config.actorder
else:
raise TypeError(
"Marlin WNA16 MoE backend requires AutoAWQConfig, AutoGPTQConfig or "
"Marlin WNA16 MoE backend requires AutoGPTQConfig, AutoAWQConfig or "
f"QuantizationArgs, got {type(quant_config).__name__}."
)
if w13_g_idx is None or w2_g_idx is None:
raise ValueError("GPTQ Marlin MoE requires g_idx tensors.")
return _process_weights_marlin(
layer,
input_dtype,
num_bits,
pack_factor,
group_size,
actorder,
w13,
w2,
w13_scale,
w2_scale,
w13_g_idx,
w2_g_idx,
w13_qzeros,
w2_qzeros,
w13_bias,
w2_bias,
)
if isinstance(quant_config, AutoAWQConfig):
if w13_qzeros is None or w2_qzeros is None:
raise ValueError("AWQ Marlin MoE requires zero-point tensors.")
return _process_awq_weights_marlin(
layer,
num_bits,
pack_factor,
group_size,
input_dtype,
w13,
w2,
w13_scale,
w2_scale,
w13_qzeros,
w2_qzeros,
w13_bias,
w2_bias,
)
else:
if w13_g_idx is None or w2_g_idx is None:
raise ValueError("GPTQ Marlin MoE requires g_idx tensors.")
return _process_weights_marlin(
layer,
input_dtype,
num_bits,
pack_factor,
group_size,
actorder,
w13,
w2,
w13_scale,
w2_scale,
w13_g_idx,
w2_g_idx,
w13_qzeros,
w2_qzeros,
w13_bias,
w2_bias,
)
elif backend == WNA16MoEBackend.CPU:
return _process_weights_cpu(
quant_config,
@@ -1482,5 +1600,47 @@ def convert_to_wna16_moe_kernel_format(
w13_qzeros,
w2_qzeros,
)
elif backend == WNA16MoEBackend.TRITON:
# Two possible input layouts depending on the quantization source:
#
# MoeWNA16 (uint8): (E, N_out, K // bit8_pack) — N-first
# → just view as uint8 (no-op)
#
# AutoGPTQ/compressed-tensors (int32, K-first):
# (E, K // pack32, N_out)
# → transpose to N-first, then view as uint8 to get
# (E, N_out, K // bit8_pack) [int32 = 4 bytes → 4 uint8s]
# Scales: (E, K // gs, N_out) → transpose → (E, N_out, K // gs)
from vllm.model_executor.layers.quantization.auto_gptq import (
AutoGPTQConfig,
)
if isinstance(quant_config, (AutoGPTQConfig, QuantizationArgs)):
# These integrations build in K-first format even when the Triton
# backend is selected. Transpose to N-first first.
w13_uint8 = w13.transpose(1, 2).contiguous().view(torch.uint8)
w2_uint8 = w2.transpose(1, 2).contiguous().view(torch.uint8)
w13_scale = w13_scale.transpose(1, 2).contiguous()
w2_scale = w2_scale.transpose(1, 2).contiguous()
else:
# MoeWNA16 uses N-first uint8 weights and scales.
w13_uint8 = w13.view(torch.uint8)
w2_uint8 = w2.view(torch.uint8)
return (
w13_uint8,
w2_uint8,
w13_scale,
w2_scale,
None,
None,
None,
None,
w13_qzeros,
w2_qzeros,
None,
None,
w13_bias,
w2_bias,
)
else:
raise ValueError(f"Unsupported wna16 MoE backend: {backend.value}")
@@ -114,6 +114,7 @@ class RoutedExperts(PluggableLayer):
self.e_score_correction_bias = e_score_correction_bias
self.apply_router_weight_on_input = apply_router_weight_on_input
# End random parameters
self._loaded_expert_biases: set[str] = set()
self.quant_method = self._get_quant_method(
self.layer_name,
@@ -691,6 +692,25 @@ class RoutedExperts(PluggableLayer):
expert_data = param.data if full_load else param.data[expert_id]
if "bias" in weight_name:
self._loaded_expert_biases.add(weight_name.rsplit(".", 1)[-1])
if shard_id == "w2":
expert_data = self._narrow_expert_data_for_padding(
expert_data,
loaded_weight,
hidden_dim=0,
)
expert_data.copy_(loaded_weight)
else:
self._load_w13(
shard_id=shard_id,
shard_dim=0,
loaded_weight=loaded_weight,
expert_data=expert_data,
tp_rank=self.moe_config.tp_rank,
)
return True if return_success else None
# Case input scale: input_scale loading is only supported for fp8
if "input_scale" in weight_name:
# this is needed for compressed-tensors only
@@ -560,6 +560,9 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
self.wna16_moe_backend, self.experts_cls = select_wna16_moe_backend(
moe,
kInt4Static,
quant_config=self.quant_config,
may_have_zp=self.quant_config.zero_point,
may_have_bias=True,
)
def create_weights(
@@ -489,6 +489,9 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
self.wna16_moe_backend, self.experts_cls = select_wna16_moe_backend(
moe,
weight_key,
quant_config=self.quant_config,
may_have_zp=True,
may_have_bias=True,
)
def create_weights(
@@ -577,25 +580,25 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
set_weight_attrs(w2_scales, extra_weight_attrs)
# don't shard the w2 scales when running act order
set_weight_attrs(w2_scales, {"load_full_w2": self.quant_config.desc_act})
# up_proj scales
# up_proj zero points
w13_qzeros = torch.nn.Parameter(
torch.empty(
num_experts,
scales_size13,
2 * intermediate_size_per_partition // self.quant_config.pack_factor,
dtype=params_dtype,
dtype=torch.int32,
),
requires_grad=False,
)
layer.register_parameter("w13_qzeros", w13_qzeros)
set_weight_attrs(w13_qzeros, extra_weight_attrs)
# down_proj scales
# down_proj zero points
w2_qzeros = torch.nn.Parameter(
torch.empty(
num_experts,
scales_size2,
hidden_size // self.quant_config.pack_factor,
dtype=params_dtype,
dtype=torch.int32,
),
requires_grad=False,
)
@@ -644,6 +647,26 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
layer.register_parameter("w2_g_idx_sort_indices", w2_g_idx_sort_indices)
set_weight_attrs(w2_g_idx_sort_indices, extra_weight_attrs)
# Some GPTQ checkpoints contain expert biases even when the model
# architecture does not declare them. Zero initialization keeps
# checkpoints without biases equivalent to the bias-free path.
w13_bias = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition,
dtype=params_dtype,
),
requires_grad=False,
)
layer.register_parameter("w13_bias", w13_bias)
set_weight_attrs(w13_bias, extra_weight_attrs)
w2_bias = torch.nn.Parameter(
torch.zeros(num_experts, hidden_size, dtype=params_dtype),
requires_grad=False,
)
layer.register_parameter("w2_bias", w2_bias)
set_weight_attrs(w2_bias, extra_weight_attrs)
if self.experts_cls is not None and issubclass(
self.experts_cls, FusedMoEExpertsModular
):
@@ -651,12 +674,31 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
layer.workspace = marlin_make_workspace_new(device, 4)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
def replace_or_register(name: str, val: torch.Tensor | None):
if val is None:
return
if hasattr(layer, name):
replace_parameter(layer, name, val)
else:
layer.register_parameter(
name, torch.nn.Parameter(val, requires_grad=False)
)
is_a_8bit = self.input_dtype is not None and self.input_dtype.itemsize == 1
if is_a_8bit:
assert self.quant_config.quant_type.size_bits == 8, (
"W8A8-INT8 is not supported by marlin kernel."
)
assert not is_a_8bit or self.quant_config.quant_type.size_bits == 8, (
"W8A8-INT8 is not supported by marlin kernel."
)
w13_bias = getattr(layer, "w13_bias", None)
if "w13_bias" not in layer._loaded_expert_biases:
layer.register_parameter("w13_bias", None)
w13_bias = None
w2_bias = getattr(layer, "w2_bias", None)
if "w2_bias" not in layer._loaded_expert_biases:
layer.register_parameter("w2_bias", None)
w2_bias = None
converted = convert_to_wna16_moe_kernel_format(
backend=self.wna16_moe_backend,
@@ -669,8 +711,10 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
w2_scale=layer.w2_scales,
w13_g_idx=layer.w13_g_idx,
w2_g_idx=layer.w2_g_idx,
w13_bias=getattr(layer, "w13_bias", None),
w2_bias=getattr(layer, "w2_bias", None),
w13_bias=w13_bias,
w2_bias=w2_bias,
w13_qzeros=getattr(layer, "w13_qzeros", None),
w2_qzeros=getattr(layer, "w2_qzeros", None),
)
if converted is None:
@@ -703,42 +747,12 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
replace_parameter(layer, "w2_g_idx", w2_g_idx)
replace_parameter(layer, "w13_g_idx_sort_indices", w13_g_idx_sort_indices)
replace_parameter(layer, "w2_g_idx_sort_indices", w2_g_idx_sort_indices)
if w13_qzeros is not None:
replace_parameter(layer, "w13_qzeros", w13_qzeros)
if w2_qzeros is not None:
replace_parameter(layer, "w2_qzeros", w2_qzeros)
if w13_input_global_scale is not None:
if hasattr(layer, "w13_input_global_scale"):
replace_parameter(
layer, "w13_input_global_scale", w13_input_global_scale
)
else:
layer.register_parameter(
"w13_input_global_scale",
torch.nn.Parameter(w13_input_global_scale, requires_grad=False),
)
if w2_input_global_scale is not None:
if hasattr(layer, "w2_input_global_scale"):
replace_parameter(layer, "w2_input_global_scale", w2_input_global_scale)
else:
layer.register_parameter(
"w2_input_global_scale",
torch.nn.Parameter(w2_input_global_scale, requires_grad=False),
)
if w13_bias is not None:
if hasattr(layer, "w13_bias"):
replace_parameter(layer, "w13_bias", w13_bias)
else:
layer.register_parameter(
"w13_bias", torch.nn.Parameter(w13_bias, requires_grad=False)
)
if w2_bias is not None:
if hasattr(layer, "w2_bias"):
replace_parameter(layer, "w2_bias", w2_bias)
else:
layer.register_parameter(
"w2_bias", torch.nn.Parameter(w2_bias, requires_grad=False)
)
replace_or_register("w13_input_global_scale", w13_input_global_scale)
replace_or_register("w2_input_global_scale", w2_input_global_scale)
replace_or_register("w13_bias", w13_bias)
replace_or_register("w2_bias", w2_bias)
replace_or_register("w13_qzeros", w13_qzeros)
replace_or_register("w2_qzeros", w2_qzeros)
# The modular kernel reads w13_weight/w2_weight; marlin keeps *_qweight.
layer.w13_weight = layer.w13_qweight
@@ -92,7 +92,15 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
self.wna16_backend, self.experts_cls = select_wna16_moe_backend(
config=self.moe,
weight_key=weight_key,
quant_config=self.weight_quant,
may_have_zp=not self.symmetric,
may_have_bias=False,
)
self.is_marlin = self.wna16_backend in [
WNA16MoEBackend.MARLIN,
WNA16MoEBackend.BATCHED_MARLIN,
]
self.is_transposed = self.wna16_backend != WNA16MoEBackend.FLASHINFER_TRTLLM
def get_weight_shape(
self,
@@ -118,7 +126,6 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
"num_groups_w2 must be provided for weight scales/zero_points"
)
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
is_flashinfer = self.wna16_backend == WNA16MoEBackend.FLASHINFER_TRTLLM
shape_map = {
"w13_weight": {
"Flashinfer": (
@@ -177,7 +184,7 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
),
},
}
backend_key = "Flashinfer" if is_flashinfer else "Marlin"
backend_key = "Marlin" if self.is_transposed else "Flashinfer"
return shape_map[weight_name][backend_key]
@staticmethod
@@ -217,9 +224,8 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
# Will transpose the loaded weight along the
# intermediate and hidden dim sizes. Will
# shard for TP along the transposed dims
is_transposed = self.wna16_backend != WNA16MoEBackend.FLASHINFER_TRTLLM
extra_weight_attrs.update(
{"is_transposed": is_transposed, "quant_method": self.strategy}
{"is_transposed": self.is_transposed, "quant_method": self.strategy}
)
w13_weight = torch.nn.Parameter(
@@ -415,7 +421,6 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# Process weights using the shared oracle infrastructure
is_flashinfer = self.wna16_backend == WNA16MoEBackend.FLASHINFER_TRTLLM
converted = convert_to_wna16_moe_kernel_format(
backend=self.wna16_backend,
layer=layer,
@@ -430,6 +435,7 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
w13_qzeros=getattr(layer, "w13_weight_zero_point", None),
w2_qzeros=getattr(layer, "w2_weight_zero_point", None),
)
if converted is None:
# In-place backends (e.g. Humming) are not wired through this
# marlin-only method; fail clearly rather than unpacking None.
@@ -437,6 +443,7 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
f"{type(self).__name__} does not support the "
f"{self.wna16_backend.value} MoE backend."
)
(
w13_qweight,
w2_qweight,
@@ -466,7 +473,7 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
replace_parameter(layer, "w2_weight_zero_point", w2_qzeros)
# Marlin-specific parameters (not needed for Flashinfer)
if not is_flashinfer:
if self.is_marlin:
if w13_g_idx_processed is not None:
replace_parameter(layer, "w13_weight_g_idx", w13_g_idx_processed)
if w2_g_idx_processed is not None:
@@ -510,7 +517,7 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
# Add Marlin-specific arguments
marlin_args: dict[str, Any] = {}
if not is_flashinfer:
if self.is_marlin:
marlin_args = {
"w13_g_idx": layer.w13_weight_g_idx,
"w2_g_idx": layer.w2_weight_g_idx,
@@ -15,8 +15,13 @@ from vllm.model_executor.layers.fused_moe import (
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
int4_w4a16_moe_quant_config,
int8_w8a16_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
WNA16MoEBackend,
convert_to_wna16_moe_kernel_format,
make_wna16_moe_kernel,
make_wna16_moe_quant_config,
select_wna16_moe_backend,
)
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
UnquantizedFusedMoEMethod,
@@ -27,7 +32,15 @@ from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
QuantizeMethodBase,
)
from vllm.model_executor.utils import set_weight_attrs
from vllm.model_executor.layers.quantization.utils.quant_utils import (
INT4_DTYPE,
INT8_DTYPE,
QuantKey,
kInt4Static32GroupScale,
kInt4StaticGroupScale,
kInt8StaticGroupScale,
)
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
from vllm.platforms import current_platform
@@ -199,6 +212,34 @@ class MoeWNA16Method(FusedMoEMethodBase):
super().__init__(moe)
self.quant_config = quant_config
num_bits = self.quant_config.weight_bits
group_size = self.quant_config.group_size
if num_bits == 4:
quant_type = INT4_DTYPE
if group_size == 32:
scale = kInt4Static32GroupScale
else:
scale = kInt4StaticGroupScale
elif num_bits == 8:
assert group_size == -1
quant_type = INT8_DTYPE
scale = kInt8StaticGroupScale
else:
raise ValueError("MoeWNA16Method only supports int4 and int8 now.")
weight_key = QuantKey(quant_type, scale)
# Select WNA16 MoE backend via oracle.
# handle ZP?
self.wna16_backend, self.experts_cls = select_wna16_moe_backend(
config=self.moe,
weight_key=weight_key,
quant_config=self.quant_config,
may_have_zp=self.quant_config.has_zp,
may_have_bias=False,
)
def create_weights(
self,
layer: RoutedExperts,
@@ -322,23 +363,104 @@ class MoeWNA16Method(FusedMoEMethodBase):
def get_fused_moe_quant_config(
self, layer: RoutedExperts
) -> FusedMoEQuantConfig | None:
weight_bits = self.quant_config.weight_bits
has_zp = self.quant_config.has_zp
assert weight_bits == 4 or weight_bits == 8
config_builder = (
int4_w4a16_moe_quant_config
if weight_bits == 4
else int8_w8a16_moe_quant_config
)
if self.wna16_backend == WNA16MoEBackend.HUMMING:
from vllm.model_executor.layers.quantization.utils.humming_utils import (
get_humming_moe_quant_config,
)
return config_builder(
return get_humming_moe_quant_config(layer)
has_zp = self.quant_config.has_zp
return make_wna16_moe_quant_config(
w1_scale=layer.w13_scales,
w2_scale=layer.w2_scales,
w1_zp=layer.w13_qzeros if has_zp else None,
w2_zp=layer.w2_qzeros if has_zp else None,
block_shape=[0, layer.group_size],
group_size=layer.group_size,
num_bits=self.quant_config.weight_bits,
)
def _setup_kernel(self, layer: RoutedExperts):
assert self.experts_cls is not None
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
assert self.moe_quant_config is not None
self.moe_kernel = make_wna16_moe_kernel(
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
experts_cls=self.experts_cls,
backend=self.wna16_backend,
layer=layer,
routing_tables=layer._expert_routing_tables(),
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
has_zp = self.quant_config.has_zp
converted = convert_to_wna16_moe_kernel_format(
backend=self.wna16_backend,
layer=layer,
quant_config=self.quant_config,
input_dtype=None,
w13=layer.w13_qweight,
w2=layer.w2_qweight,
w13_scale=layer.w13_scales,
w2_scale=layer.w2_scales,
w13_g_idx=None,
w2_g_idx=None,
w13_qzeros=layer.w13_qzeros if has_zp else None,
w2_qzeros=layer.w2_qzeros if has_zp else None,
)
if converted is None:
# Backend rewrote the layer's params in place (e.g. Humming).
self._setup_kernel(layer)
return
(
w13_qweight,
w2_qweight,
w13_scales,
w2_scales,
_,
_,
_,
_,
w13_qzeros,
w2_qzeros,
w13_input_global_scale,
w2_input_global_scale,
_, # w13_bias
_, # w2_bias
) = converted
# Replace common parameters
replace_parameter(layer, "w13_qweight", w13_qweight)
replace_parameter(layer, "w2_qweight", w2_qweight)
replace_parameter(layer, "w13_scales", w13_scales)
replace_parameter(layer, "w2_scales", w2_scales)
layer.w13_weight = layer.w13_qweight
layer.w2_weight = layer.w2_qweight
if has_zp:
assert w13_qzeros is not None and w2_qzeros is not None
replace_parameter(layer, "w13_qzeros", w13_qzeros)
replace_parameter(layer, "w2_qzeros", w2_qzeros)
# Marlin-specific parameters (not needed for Flashinfer)
if self.wna16_backend != WNA16MoEBackend.FLASHINFER_TRTLLM:
# Register input global scales if present
if w13_input_global_scale is not None:
layer.register_parameter(
"w13_input_global_scale",
torch.nn.Parameter(w13_input_global_scale, requires_grad=False),
)
if w2_input_global_scale is not None:
layer.register_parameter(
"w2_input_global_scale",
torch.nn.Parameter(w2_input_global_scale, requires_grad=False),
)
self._setup_kernel(layer)
def apply(
self,
layer: RoutedExperts,
@@ -348,19 +470,44 @@ class MoeWNA16Method(FusedMoEMethodBase):
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
from vllm.model_executor.layers.fused_moe import fused_experts
return fused_experts(
assert not self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply(
x,
layer.w13_qweight,
layer.w2_qweight,
layer.w13_weight,
layer.w2_weight,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=layer.activation,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
quant_config=self.moe_quant_config,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
shared_experts=shared_experts,
shared_experts_input=shared_experts_input,
)
def apply_monolithic(
self,
layer: RoutedExperts,
x: torch.Tensor,
router_logits: torch.Tensor,
input_ids: torch.Tensor | None = None,
) -> torch.Tensor:
assert self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply_monolithic(
x,
layer.w13_weight,
layer.w2_weight,
router_logits,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
num_expert_group=layer.num_expert_group,
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
)
@staticmethod
@@ -3,7 +3,7 @@
from collections.abc import Mapping
from copy import deepcopy
from types import MappingProxyType
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
import regex as re
import torch
@@ -69,6 +69,20 @@ def get_dynamic_override(
return default_value
def flatten_list(lst: list[Any]) -> list[Any]:
output = []
def _flatten(lst: list[Any]):
for i in lst:
if isinstance(i, list):
_flatten(i)
else:
output.append(i)
_flatten(lst)
return output
def is_layer_gptq_quantized(
prefix: str,
quantized_layers: list[str],
@@ -83,6 +97,8 @@ def is_layer_gptq_quantized(
proj_name = prefix.split(".")[-1]
quantized_layers = flatten_list(quantized_layers)
# Fused layers like gate_up_proj or qkv_proj will not be fused
# in the safetensors checkpoint. So, we convert the name
# from the fused version to unfused + check to make sure that