[MoE Refactor] Migrate MoeWNA16Method quantization method over to using the new MK oracle scheme. (#44120)

Signed-off-by: Bill Nell <bnell@redhat.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
bnellnm
2026-07-21 19:20:30 -07:00
committed by GitHub
co-authored by Claude mgoin
parent 1750e443f2
commit ec59c1579f
11 changed files with 811 additions and 201 deletions
@@ -560,6 +560,9 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
self.wna16_moe_backend, self.experts_cls = select_wna16_moe_backend(
moe,
kInt4Static,
quant_config=self.quant_config,
may_have_zp=self.quant_config.zero_point,
may_have_bias=True,
)
def create_weights(
@@ -489,6 +489,9 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
self.wna16_moe_backend, self.experts_cls = select_wna16_moe_backend(
moe,
weight_key,
quant_config=self.quant_config,
may_have_zp=True,
may_have_bias=True,
)
def create_weights(
@@ -577,25 +580,25 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
set_weight_attrs(w2_scales, extra_weight_attrs)
# don't shard the w2 scales when running act order
set_weight_attrs(w2_scales, {"load_full_w2": self.quant_config.desc_act})
# up_proj scales
# up_proj zero points
w13_qzeros = torch.nn.Parameter(
torch.empty(
num_experts,
scales_size13,
2 * intermediate_size_per_partition // self.quant_config.pack_factor,
dtype=params_dtype,
dtype=torch.int32,
),
requires_grad=False,
)
layer.register_parameter("w13_qzeros", w13_qzeros)
set_weight_attrs(w13_qzeros, extra_weight_attrs)
# down_proj scales
# down_proj zero points
w2_qzeros = torch.nn.Parameter(
torch.empty(
num_experts,
scales_size2,
hidden_size // self.quant_config.pack_factor,
dtype=params_dtype,
dtype=torch.int32,
),
requires_grad=False,
)
@@ -644,6 +647,26 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
layer.register_parameter("w2_g_idx_sort_indices", w2_g_idx_sort_indices)
set_weight_attrs(w2_g_idx_sort_indices, extra_weight_attrs)
# Some GPTQ checkpoints contain expert biases even when the model
# architecture does not declare them. Zero initialization keeps
# checkpoints without biases equivalent to the bias-free path.
w13_bias = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition,
dtype=params_dtype,
),
requires_grad=False,
)
layer.register_parameter("w13_bias", w13_bias)
set_weight_attrs(w13_bias, extra_weight_attrs)
w2_bias = torch.nn.Parameter(
torch.zeros(num_experts, hidden_size, dtype=params_dtype),
requires_grad=False,
)
layer.register_parameter("w2_bias", w2_bias)
set_weight_attrs(w2_bias, extra_weight_attrs)
if self.experts_cls is not None and issubclass(
self.experts_cls, FusedMoEExpertsModular
):
@@ -651,12 +674,31 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
layer.workspace = marlin_make_workspace_new(device, 4)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
def replace_or_register(name: str, val: torch.Tensor | None):
if val is None:
return
if hasattr(layer, name):
replace_parameter(layer, name, val)
else:
layer.register_parameter(
name, torch.nn.Parameter(val, requires_grad=False)
)
is_a_8bit = self.input_dtype is not None and self.input_dtype.itemsize == 1
if is_a_8bit:
assert self.quant_config.quant_type.size_bits == 8, (
"W8A8-INT8 is not supported by marlin kernel."
)
assert not is_a_8bit or self.quant_config.quant_type.size_bits == 8, (
"W8A8-INT8 is not supported by marlin kernel."
)
w13_bias = getattr(layer, "w13_bias", None)
if "w13_bias" not in layer._loaded_expert_biases:
layer.register_parameter("w13_bias", None)
w13_bias = None
w2_bias = getattr(layer, "w2_bias", None)
if "w2_bias" not in layer._loaded_expert_biases:
layer.register_parameter("w2_bias", None)
w2_bias = None
converted = convert_to_wna16_moe_kernel_format(
backend=self.wna16_moe_backend,
@@ -669,8 +711,10 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
w2_scale=layer.w2_scales,
w13_g_idx=layer.w13_g_idx,
w2_g_idx=layer.w2_g_idx,
w13_bias=getattr(layer, "w13_bias", None),
w2_bias=getattr(layer, "w2_bias", None),
w13_bias=w13_bias,
w2_bias=w2_bias,
w13_qzeros=getattr(layer, "w13_qzeros", None),
w2_qzeros=getattr(layer, "w2_qzeros", None),
)
if converted is None:
@@ -703,42 +747,12 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
replace_parameter(layer, "w2_g_idx", w2_g_idx)
replace_parameter(layer, "w13_g_idx_sort_indices", w13_g_idx_sort_indices)
replace_parameter(layer, "w2_g_idx_sort_indices", w2_g_idx_sort_indices)
if w13_qzeros is not None:
replace_parameter(layer, "w13_qzeros", w13_qzeros)
if w2_qzeros is not None:
replace_parameter(layer, "w2_qzeros", w2_qzeros)
if w13_input_global_scale is not None:
if hasattr(layer, "w13_input_global_scale"):
replace_parameter(
layer, "w13_input_global_scale", w13_input_global_scale
)
else:
layer.register_parameter(
"w13_input_global_scale",
torch.nn.Parameter(w13_input_global_scale, requires_grad=False),
)
if w2_input_global_scale is not None:
if hasattr(layer, "w2_input_global_scale"):
replace_parameter(layer, "w2_input_global_scale", w2_input_global_scale)
else:
layer.register_parameter(
"w2_input_global_scale",
torch.nn.Parameter(w2_input_global_scale, requires_grad=False),
)
if w13_bias is not None:
if hasattr(layer, "w13_bias"):
replace_parameter(layer, "w13_bias", w13_bias)
else:
layer.register_parameter(
"w13_bias", torch.nn.Parameter(w13_bias, requires_grad=False)
)
if w2_bias is not None:
if hasattr(layer, "w2_bias"):
replace_parameter(layer, "w2_bias", w2_bias)
else:
layer.register_parameter(
"w2_bias", torch.nn.Parameter(w2_bias, requires_grad=False)
)
replace_or_register("w13_input_global_scale", w13_input_global_scale)
replace_or_register("w2_input_global_scale", w2_input_global_scale)
replace_or_register("w13_bias", w13_bias)
replace_or_register("w2_bias", w2_bias)
replace_or_register("w13_qzeros", w13_qzeros)
replace_or_register("w2_qzeros", w2_qzeros)
# The modular kernel reads w13_weight/w2_weight; marlin keeps *_qweight.
layer.w13_weight = layer.w13_qweight
@@ -92,7 +92,15 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
self.wna16_backend, self.experts_cls = select_wna16_moe_backend(
config=self.moe,
weight_key=weight_key,
quant_config=self.weight_quant,
may_have_zp=not self.symmetric,
may_have_bias=False,
)
self.is_marlin = self.wna16_backend in [
WNA16MoEBackend.MARLIN,
WNA16MoEBackend.BATCHED_MARLIN,
]
self.is_transposed = self.wna16_backend != WNA16MoEBackend.FLASHINFER_TRTLLM
def get_weight_shape(
self,
@@ -118,7 +126,6 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
"num_groups_w2 must be provided for weight scales/zero_points"
)
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
is_flashinfer = self.wna16_backend == WNA16MoEBackend.FLASHINFER_TRTLLM
shape_map = {
"w13_weight": {
"Flashinfer": (
@@ -177,7 +184,7 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
),
},
}
backend_key = "Flashinfer" if is_flashinfer else "Marlin"
backend_key = "Marlin" if self.is_transposed else "Flashinfer"
return shape_map[weight_name][backend_key]
@staticmethod
@@ -217,9 +224,8 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
# Will transpose the loaded weight along the
# intermediate and hidden dim sizes. Will
# shard for TP along the transposed dims
is_transposed = self.wna16_backend != WNA16MoEBackend.FLASHINFER_TRTLLM
extra_weight_attrs.update(
{"is_transposed": is_transposed, "quant_method": self.strategy}
{"is_transposed": self.is_transposed, "quant_method": self.strategy}
)
w13_weight = torch.nn.Parameter(
@@ -415,7 +421,6 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# Process weights using the shared oracle infrastructure
is_flashinfer = self.wna16_backend == WNA16MoEBackend.FLASHINFER_TRTLLM
converted = convert_to_wna16_moe_kernel_format(
backend=self.wna16_backend,
layer=layer,
@@ -430,6 +435,7 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
w13_qzeros=getattr(layer, "w13_weight_zero_point", None),
w2_qzeros=getattr(layer, "w2_weight_zero_point", None),
)
if converted is None:
# In-place backends (e.g. Humming) are not wired through this
# marlin-only method; fail clearly rather than unpacking None.
@@ -437,6 +443,7 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
f"{type(self).__name__} does not support the "
f"{self.wna16_backend.value} MoE backend."
)
(
w13_qweight,
w2_qweight,
@@ -466,7 +473,7 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
replace_parameter(layer, "w2_weight_zero_point", w2_qzeros)
# Marlin-specific parameters (not needed for Flashinfer)
if not is_flashinfer:
if self.is_marlin:
if w13_g_idx_processed is not None:
replace_parameter(layer, "w13_weight_g_idx", w13_g_idx_processed)
if w2_g_idx_processed is not None:
@@ -510,7 +517,7 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
# Add Marlin-specific arguments
marlin_args: dict[str, Any] = {}
if not is_flashinfer:
if self.is_marlin:
marlin_args = {
"w13_g_idx": layer.w13_weight_g_idx,
"w2_g_idx": layer.w2_weight_g_idx,
@@ -15,8 +15,13 @@ from vllm.model_executor.layers.fused_moe import (
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
int4_w4a16_moe_quant_config,
int8_w8a16_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
WNA16MoEBackend,
convert_to_wna16_moe_kernel_format,
make_wna16_moe_kernel,
make_wna16_moe_quant_config,
select_wna16_moe_backend,
)
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
UnquantizedFusedMoEMethod,
@@ -27,7 +32,15 @@ from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
QuantizeMethodBase,
)
from vllm.model_executor.utils import set_weight_attrs
from vllm.model_executor.layers.quantization.utils.quant_utils import (
INT4_DTYPE,
INT8_DTYPE,
QuantKey,
kInt4Static32GroupScale,
kInt4StaticGroupScale,
kInt8StaticGroupScale,
)
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
from vllm.platforms import current_platform
@@ -199,6 +212,34 @@ class MoeWNA16Method(FusedMoEMethodBase):
super().__init__(moe)
self.quant_config = quant_config
num_bits = self.quant_config.weight_bits
group_size = self.quant_config.group_size
if num_bits == 4:
quant_type = INT4_DTYPE
if group_size == 32:
scale = kInt4Static32GroupScale
else:
scale = kInt4StaticGroupScale
elif num_bits == 8:
assert group_size == -1
quant_type = INT8_DTYPE
scale = kInt8StaticGroupScale
else:
raise ValueError("MoeWNA16Method only supports int4 and int8 now.")
weight_key = QuantKey(quant_type, scale)
# Select WNA16 MoE backend via oracle.
# handle ZP?
self.wna16_backend, self.experts_cls = select_wna16_moe_backend(
config=self.moe,
weight_key=weight_key,
quant_config=self.quant_config,
may_have_zp=self.quant_config.has_zp,
may_have_bias=False,
)
def create_weights(
self,
layer: RoutedExperts,
@@ -322,23 +363,104 @@ class MoeWNA16Method(FusedMoEMethodBase):
def get_fused_moe_quant_config(
self, layer: RoutedExperts
) -> FusedMoEQuantConfig | None:
weight_bits = self.quant_config.weight_bits
has_zp = self.quant_config.has_zp
assert weight_bits == 4 or weight_bits == 8
config_builder = (
int4_w4a16_moe_quant_config
if weight_bits == 4
else int8_w8a16_moe_quant_config
)
if self.wna16_backend == WNA16MoEBackend.HUMMING:
from vllm.model_executor.layers.quantization.utils.humming_utils import (
get_humming_moe_quant_config,
)
return config_builder(
return get_humming_moe_quant_config(layer)
has_zp = self.quant_config.has_zp
return make_wna16_moe_quant_config(
w1_scale=layer.w13_scales,
w2_scale=layer.w2_scales,
w1_zp=layer.w13_qzeros if has_zp else None,
w2_zp=layer.w2_qzeros if has_zp else None,
block_shape=[0, layer.group_size],
group_size=layer.group_size,
num_bits=self.quant_config.weight_bits,
)
def _setup_kernel(self, layer: RoutedExperts):
assert self.experts_cls is not None
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
assert self.moe_quant_config is not None
self.moe_kernel = make_wna16_moe_kernel(
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
experts_cls=self.experts_cls,
backend=self.wna16_backend,
layer=layer,
routing_tables=layer._expert_routing_tables(),
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
has_zp = self.quant_config.has_zp
converted = convert_to_wna16_moe_kernel_format(
backend=self.wna16_backend,
layer=layer,
quant_config=self.quant_config,
input_dtype=None,
w13=layer.w13_qweight,
w2=layer.w2_qweight,
w13_scale=layer.w13_scales,
w2_scale=layer.w2_scales,
w13_g_idx=None,
w2_g_idx=None,
w13_qzeros=layer.w13_qzeros if has_zp else None,
w2_qzeros=layer.w2_qzeros if has_zp else None,
)
if converted is None:
# Backend rewrote the layer's params in place (e.g. Humming).
self._setup_kernel(layer)
return
(
w13_qweight,
w2_qweight,
w13_scales,
w2_scales,
_,
_,
_,
_,
w13_qzeros,
w2_qzeros,
w13_input_global_scale,
w2_input_global_scale,
_, # w13_bias
_, # w2_bias
) = converted
# Replace common parameters
replace_parameter(layer, "w13_qweight", w13_qweight)
replace_parameter(layer, "w2_qweight", w2_qweight)
replace_parameter(layer, "w13_scales", w13_scales)
replace_parameter(layer, "w2_scales", w2_scales)
layer.w13_weight = layer.w13_qweight
layer.w2_weight = layer.w2_qweight
if has_zp:
assert w13_qzeros is not None and w2_qzeros is not None
replace_parameter(layer, "w13_qzeros", w13_qzeros)
replace_parameter(layer, "w2_qzeros", w2_qzeros)
# Marlin-specific parameters (not needed for Flashinfer)
if self.wna16_backend != WNA16MoEBackend.FLASHINFER_TRTLLM:
# Register input global scales if present
if w13_input_global_scale is not None:
layer.register_parameter(
"w13_input_global_scale",
torch.nn.Parameter(w13_input_global_scale, requires_grad=False),
)
if w2_input_global_scale is not None:
layer.register_parameter(
"w2_input_global_scale",
torch.nn.Parameter(w2_input_global_scale, requires_grad=False),
)
self._setup_kernel(layer)
def apply(
self,
layer: RoutedExperts,
@@ -348,19 +470,44 @@ class MoeWNA16Method(FusedMoEMethodBase):
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
from vllm.model_executor.layers.fused_moe import fused_experts
return fused_experts(
assert not self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply(
x,
layer.w13_qweight,
layer.w2_qweight,
layer.w13_weight,
layer.w2_weight,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=layer.activation,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
quant_config=self.moe_quant_config,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
shared_experts=shared_experts,
shared_experts_input=shared_experts_input,
)
def apply_monolithic(
self,
layer: RoutedExperts,
x: torch.Tensor,
router_logits: torch.Tensor,
input_ids: torch.Tensor | None = None,
) -> torch.Tensor:
assert self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply_monolithic(
x,
layer.w13_weight,
layer.w2_weight,
router_logits,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
num_expert_group=layer.num_expert_group,
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
)
@staticmethod
@@ -3,7 +3,7 @@
from collections.abc import Mapping
from copy import deepcopy
from types import MappingProxyType
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
import regex as re
import torch
@@ -69,6 +69,20 @@ def get_dynamic_override(
return default_value
def flatten_list(lst: list[Any]) -> list[Any]:
output = []
def _flatten(lst: list[Any]):
for i in lst:
if isinstance(i, list):
_flatten(i)
else:
output.append(i)
_flatten(lst)
return output
def is_layer_gptq_quantized(
prefix: str,
quantized_layers: list[str],
@@ -83,6 +97,8 @@ def is_layer_gptq_quantized(
proj_name = prefix.split(".")[-1]
quantized_layers = flatten_list(quantized_layers)
# Fused layers like gate_up_proj or qkv_proj will not be fused
# in the safetensors checkpoint. So, we convert the name
# from the fused version to unfused + check to make sure that