[Quant][Kernel] Remove GPTQ Group/Dynamic Activation Ordering (#54809)

Signed-off-by: Roderick-Wu <roderickwu2003@gmail.com>
Signed-off-by: Roderick Wu <roderickwu2003@gmail.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Roderick Wu
2026-09-08 12:45:12 -07:00
committed by GitHub
co-authored by mgoin
parent 1a522b6949
commit 2c9d68f80b
99 changed files with 1182 additions and 3917 deletions
@@ -439,7 +439,6 @@ class AutoAWQMarlinLinearMethod(LinearMethodBase):
act_type=params_dtype if self.input_dtype is None else self.input_dtype,
group_size=self.quant_config.group_size,
zero_points=self.quant_config.zero_point,
has_g_idx=False,
)
kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config)
@@ -558,11 +557,6 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
}
)
intermediate_size_full = extra_weight_attrs.pop(
"intermediate_size_full", intermediate_size_per_partition
)
self.is_k_full = intermediate_size_per_partition == intermediate_size_full
w13_qweight = Parameter(
torch.empty(
num_experts,
@@ -672,10 +666,6 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
w2,
w13_scale,
w2_scale,
w13_g_idx,
w2_g_idx,
w13_g_idx_sort_indices,
w2_g_idx_sort_indices,
w13_qzeros,
w2_qzeros,
w13_input_global_scale,
@@ -696,14 +686,6 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
replace_parameter(layer, "w13_scales", w13_scale)
replace_parameter(layer, "w2_scales", w2_scale)
_replace_or_register_parameter(
layer, "w13_g_idx_sort_indices", w13_g_idx_sort_indices
)
_replace_or_register_parameter(
layer, "w2_g_idx_sort_indices", w2_g_idx_sort_indices
)
_replace_or_register_parameter(layer, "w13_g_idx", w13_g_idx)
_replace_or_register_parameter(layer, "w2_g_idx", w2_g_idx)
_replace_or_register_parameter(layer, "w13_qzeros", w13_qzeros)
_replace_or_register_parameter(layer, "w2_qzeros", w2_qzeros)
_replace_or_register_parameter(
@@ -726,11 +708,6 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
moe_config=self.moe,
experts_cls=self.experts_cls,
backend=self.wna16_moe_backend,
is_k_full=self.is_k_full,
w13_g_idx=getattr(layer, "w13_g_idx", None),
w2_g_idx=getattr(layer, "w2_g_idx", None),
w13_g_idx_sort_indices=getattr(layer, "w13_g_idx_sort_indices", None),
w2_g_idx_sort_indices=getattr(layer, "w2_g_idx_sort_indices", None),
routing_tables=layer._expert_routing_tables(),
)
@@ -40,6 +40,7 @@ from vllm.model_executor.layers.quantization.utils import replace_parameter
from vllm.model_executor.layers.quantization.utils.gptq_utils import (
get_dynamic_override,
get_linear_quant_method,
normalize_and_validate_gptq_desc_act,
override_config,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
@@ -59,7 +60,6 @@ from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
PackedColumnParameter,
PackedvLLMParameter,
RowvLLMParameter,
)
from vllm.scalar_type import scalar_types
from vllm.transformers_utils.config import get_safetensors_params_metadata
@@ -115,10 +115,7 @@ class AutoGPTQConfig(QuantizationConfig):
modules_in_block_to_quantize: list[str] | None = None,
) -> None:
super().__init__()
if desc_act and group_size == -1:
# In this case, act_order == True is the same as act_order == False
# (since we have only one group per output channel)
desc_act = False
desc_act = normalize_and_validate_gptq_desc_act(desc_act, group_size, dynamic)
# GPTQModel use `dynamic` config property to allow per module
# quantization config so each module can be individually optimized.
@@ -150,7 +147,7 @@ class AutoGPTQConfig(QuantizationConfig):
self.pack_factor = 32 // weight_bits # packed into int32
self.group_size = group_size
self.desc_act = desc_act
self.desc_act: bool = desc_act
self.lm_head_quantized = lm_head_quantized
self.full_config = full_config
@@ -244,7 +241,7 @@ class AutoGPTQConfig(QuantizationConfig):
from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
if not check_moe_marlin_supports_layer(
layer, self.group_size, allow_tile_padding=not self.desc_act
layer, self.group_size, allow_tile_padding=True
):
logger.warning_once(
f"Layer '{prefix}' is not supported by GPTQMoeMarlin. "
@@ -348,7 +345,6 @@ class AutoGPTQLinearMethod(LinearMethodBase):
act_type=params_dtype if input_dtype is None else input_dtype,
group_size=self.quant_config.group_size,
zero_points=False,
has_g_idx=self.quant_config.desc_act,
)
kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config)
@@ -365,7 +361,7 @@ class AutoGPTQLinearMethod(LinearMethodBase):
# Determine sharding
if marlin_repeat_scales_on_all_ranks(
self.quant_config.desc_act, self.quant_config.group_size, is_row_parallel
self.quant_config.group_size, is_row_parallel
):
# By setting scale_dim == None, weight_loader will
# repeat the scales on each GPU in TP>1 case.
@@ -391,16 +387,6 @@ class AutoGPTQLinearMethod(LinearMethodBase):
weight_loader=weight_loader,
)
# Activation order
g_idx = RowvLLMParameter(
data=torch.empty(
input_size_per_partition,
dtype=torch.int32,
),
input_dim=0,
weight_loader=weight_loader,
)
qzeros_args = {
"data": torch.empty(
scales_and_zp_size,
@@ -440,7 +426,6 @@ class AutoGPTQLinearMethod(LinearMethodBase):
)
layer.register_parameter("qweight", qweight)
layer.register_parameter("g_idx", g_idx)
layer.register_parameter("scales", scales)
layer.register_parameter("qzeros", qzeros)
@@ -449,7 +434,6 @@ class AutoGPTQLinearMethod(LinearMethodBase):
w_q_param_name="qweight",
w_s_param_name="scales",
w_zp_param_name="qzeros",
w_gidx_param_name="g_idx",
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
@@ -492,7 +476,7 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
quant_config=self.quant_config,
may_have_zp=not self.quant_config.is_sym,
may_have_bias=True,
allow_tile_padding=not self.quant_config.desc_act,
allow_tile_padding=True,
)
def create_weights(
@@ -512,19 +496,9 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
"W8A8-INT8 is not supported by marlin kernel."
)
intermediate_size_full = extra_weight_attrs.pop("intermediate_size_full")
self.is_k_full = (not self.quant_config.desc_act) or (
intermediate_size_per_partition == intermediate_size_full
)
if self.quant_config.group_size != -1:
scales_size13 = hidden_size // self.quant_config.group_size
w2_scales_size = (
intermediate_size_full
if self.quant_config.desc_act
else intermediate_size_per_partition
)
w2_scales_size = intermediate_size_per_partition
scales_size2 = w2_scales_size // self.quant_config.group_size
strategy = FusedMoeWeightScaleSupported.GROUP.value
else:
@@ -579,8 +553,6 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
)
layer.register_parameter("w2_scales", w2_scales)
set_weight_attrs(w2_scales, extra_weight_attrs)
# don't shard the w2 scales when running act order
set_weight_attrs(w2_scales, {"load_full_w2": self.quant_config.desc_act})
# up_proj zero points
w13_qzeros = torch.nn.Parameter(
torch.empty(
@@ -607,48 +579,6 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
)
layer.register_parameter("w2_qzeros", w2_qzeros)
set_weight_attrs(w2_qzeros, extra_weight_attrs)
# don't shard the w2 scales when running act order
set_weight_attrs(w2_qzeros, {"load_full_w2": self.quant_config.desc_act})
w13_g_idx = torch.nn.Parameter(
torch.empty(
num_experts,
hidden_size,
dtype=torch.int32,
),
requires_grad=False,
)
layer.register_parameter("w13_g_idx", w13_g_idx)
set_weight_attrs(w13_g_idx, extra_weight_attrs)
w2_g_idx = torch.nn.Parameter(
torch.empty(
num_experts,
intermediate_size_per_partition,
dtype=torch.int32,
),
requires_grad=False,
)
layer.register_parameter("w2_g_idx", w2_g_idx)
set_weight_attrs(w2_g_idx, extra_weight_attrs)
w13_g_idx_sort_indices = torch.nn.Parameter(
torch.empty(
num_experts,
hidden_size,
dtype=torch.int32,
),
requires_grad=False,
)
layer.register_parameter("w13_g_idx_sort_indices", w13_g_idx_sort_indices)
set_weight_attrs(w13_g_idx_sort_indices, extra_weight_attrs)
w2_g_idx_sort_indices = torch.nn.Parameter(
torch.empty(
num_experts,
intermediate_size_per_partition,
dtype=torch.int32,
),
requires_grad=False,
)
layer.register_parameter("w2_g_idx_sort_indices", w2_g_idx_sort_indices)
set_weight_attrs(w2_g_idx_sort_indices, extra_weight_attrs)
# Some GPTQ checkpoints contain expert biases even when the model
# architecture does not declare them. Zero initialization keeps
@@ -712,8 +642,6 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
w2=layer.w2_qweight,
w13_scale=layer.w13_scales,
w2_scale=layer.w2_scales,
w13_g_idx=layer.w13_g_idx,
w2_g_idx=layer.w2_g_idx,
w13_bias=w13_bias,
w2_bias=w2_bias,
w13_qzeros=getattr(layer, "w13_qzeros", None),
@@ -730,10 +658,6 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
w2,
w13_scale,
w2_scale,
w13_g_idx,
w2_g_idx,
w13_g_idx_sort_indices,
w2_g_idx_sort_indices,
w13_qzeros,
w2_qzeros,
w13_input_global_scale,
@@ -746,10 +670,6 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
replace_parameter(layer, "w2_qweight", w2)
replace_parameter(layer, "w13_scales", w13_scale)
replace_parameter(layer, "w2_scales", w2_scale)
replace_parameter(layer, "w13_g_idx", w13_g_idx)
replace_parameter(layer, "w2_g_idx", w2_g_idx)
replace_parameter(layer, "w13_g_idx_sort_indices", w13_g_idx_sort_indices)
replace_parameter(layer, "w2_g_idx_sort_indices", w2_g_idx_sort_indices)
replace_or_register("w13_input_global_scale", w13_input_global_scale)
replace_or_register("w2_input_global_scale", w2_input_global_scale)
replace_or_register("w13_bias", w13_bias)
@@ -772,11 +692,6 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
moe_config=self.moe,
experts_cls=self.experts_cls,
backend=self.wna16_moe_backend,
is_k_full=self.is_k_full,
w13_g_idx=getattr(layer, "w13_g_idx", None),
w2_g_idx=getattr(layer, "w2_g_idx", None),
w13_g_idx_sort_indices=getattr(layer, "w13_g_idx_sort_indices", None),
w2_g_idx_sort_indices=getattr(layer, "w2_g_idx_sort_indices", None),
routing_tables=layer._expert_routing_tables(),
)
@@ -238,6 +238,13 @@ class QuantizationConfig(ABC):
}
return WeightsMapper(orig_to_new_regex=orig_to_new_regex)
@staticmethod
def get_checkpoint_weight_mapper() -> "WeightsMapper":
"""Discard activation-order metadata unused by supported kernels."""
from vllm.model_executor.models.utils import WeightsMapper
return WeightsMapper(orig_to_new_suffix={".g_idx": None})
def apply_vllm_mapper( # noqa: B027
self, hf_to_vllm_mapper: "WeightsMapper"
):
@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any, Literal, cast
import torch
from compressed_tensors.config import CompressionFormat, SparsityCompressionConfig
from compressed_tensors.quantization import (
ActivationOrdering,
QuantizationArgs,
QuantizationStrategy,
QuantizationType,
@@ -325,9 +326,20 @@ class CompressedTensorsConfig(QuantizationConfig):
targets = quant_config.get("targets")
for target in targets:
target_scheme_map[target] = {}
target_scheme_map[target]["weights"] = QuantizationArgs.model_validate(
weight_quant = QuantizationArgs.model_validate(
quant_config.get("weights")
)
if weight_quant.actorder in (
ActivationOrdering.GROUP.value,
ActivationOrdering.DYNAMIC.value,
):
raise ValueError(
"Compressed-tensors group/dynamic activation ordering "
f"(actorder={weight_quant.actorder!r}) is no longer supported. "
"Use a checkpoint with static/weight activation ordering or "
"without activation ordering."
)
target_scheme_map[target]["weights"] = weight_quant
target_scheme_map[target]["input_activations"] = None
target_scheme_map[target]["format"] = quant_config.get("format")
@@ -754,7 +766,6 @@ class CompressedTensorsConfig(QuantizationConfig):
strategy=weight_quant.strategy,
symmetric=weight_quant.symmetric,
group_size=weight_quant.group_size,
actorder=weight_quant.actorder,
)
# Must come before the WNA16 check; standard 4/8-bit weight-only (no
@@ -803,7 +814,6 @@ class CompressedTensorsConfig(QuantizationConfig):
strategy=weight_quant.strategy,
symmetric=weight_quant.symmetric,
group_size=weight_quant.group_size,
actorder=weight_quant.actorder,
layer_name=layer_name,
)
@@ -104,8 +104,6 @@ class CompressedTensorsW4A16FlydslMoEMethod(CompressedTensorsMoEMethod):
assert weight_quant.strategy == "group"
assert weight_quant.group_size == 32
self.group_size = weight_quant.group_size
# grouped actorder isn't supported by this kernel
assert weight_quant.actorder != "group"
assert weight_quant.symmetric, (
"Only symmetric quantization is supported for MoE. "
"Try --moe-backend emulation."
@@ -180,7 +178,6 @@ class CompressedTensorsW4A16FlydslMoEMethod(CompressedTensorsMoEMethod):
)
layer.register_parameter("w2_weight_scale", w2_scale)
set_weight_attrs(w2_scale, extra_weight_attrs)
set_weight_attrs(w2_scale, {"load_full_w2": False})
w2_weight_shape = torch.nn.Parameter(
torch.empty(num_experts, 2), requires_grad=False
@@ -194,50 +191,6 @@ class CompressedTensorsW4A16FlydslMoEMethod(CompressedTensorsMoEMethod):
layer.register_parameter("w13_weight_shape", w13_weight_shape)
set_weight_attrs(w13_weight_shape, extra_weight_attrs)
w13_g_idx = torch.nn.Parameter(
torch.empty(
num_experts,
hidden_size,
dtype=torch.int32,
),
requires_grad=False,
)
layer.register_parameter("w13_weight_g_idx", w13_g_idx)
set_weight_attrs(w13_g_idx, extra_weight_attrs)
w2_g_idx = torch.nn.Parameter(
torch.empty(
num_experts,
intermediate_size_per_partition,
dtype=torch.int32,
),
requires_grad=False,
)
layer.register_parameter("w2_weight_g_idx", w2_g_idx)
set_weight_attrs(w2_g_idx, extra_weight_attrs)
w13_g_idx_sort_indices = torch.nn.Parameter(
torch.empty(
num_experts,
hidden_size,
dtype=torch.int32,
),
requires_grad=False,
)
layer.register_parameter("w13_g_idx_sort_indices", w13_g_idx_sort_indices)
set_weight_attrs(w13_g_idx_sort_indices, extra_weight_attrs)
w2_g_idx_sort_indices = torch.nn.Parameter(
torch.empty(
num_experts,
intermediate_size_per_partition,
dtype=torch.int32,
),
requires_grad=False,
)
layer.register_parameter("w2_g_idx_sort_indices", w2_g_idx_sort_indices)
set_weight_attrs(w2_g_idx_sort_indices, extra_weight_attrs)
layer.a13_scale = None
layer.a2_scale = None
@@ -47,7 +47,6 @@ class CompressedTensorsW4A8Fp8MoEMethod(CompressedTensorsMoEMethod):
assert self.weight_quant.symmetric, (
"Only symmetric quantization is supported for W4A8 MoE"
)
assert self.weight_quant.actorder != "group"
assert self.group_size == 128, "Only group size 128 supported for W4A8 MoE"
self.w4a8_backend, self.experts_cls = select_w4a8_moe_backend(
@@ -3,13 +3,10 @@
import math
from fractions import Fraction
from typing import Any
import torch
from compressed_tensors.quantization import (
ActivationOrdering,
QuantizationArgs,
QuantizationStrategy,
)
from vllm.logger import init_logger
@@ -75,7 +72,6 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
self.group_size = (
-1 if weight_quant.group_size is None else weight_quant.group_size
)
self.actorder = weight_quant.actorder
# Extract quant_type and create weight key for oracle selection
self.quant_type = (
@@ -104,11 +100,6 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
weight_key = QuantKey(self.quant_type, scale, symmetric=self.symmetric)
is_actorder = self.strategy == QuantizationStrategy.GROUP and self.actorder in (
ActivationOrdering.GROUP,
ActivationOrdering.DYNAMIC,
)
# Select WNA16 MoE backend via oracle.
self.wna16_backend, self.experts_cls = select_wna16_moe_backend(
config=self.moe,
@@ -116,7 +107,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
quant_config=self.weight_quant,
may_have_zp=not self.symmetric,
may_have_bias=False,
allow_tile_padding=not is_actorder,
allow_tile_padding=True,
)
self.is_marlin = self.wna16_backend in [
@@ -130,14 +121,12 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
if self.is_marlin:
assert check_moe_marlin_supports_config(
self.moe, self.group_size, allow_tile_padding=not is_actorder
self.moe, self.group_size, allow_tile_padding=True
)
self.input_dtype = get_marlin_input_dtype(layer_name)
else:
# channelwise is not supported by this kernel
assert weight_quant.strategy == "group"
# grouped actorder isn't supported by this kernel
assert weight_quant.actorder != "group"
# Non-Marlin WNA16 always uses bf16/fp16 inputs
self.input_dtype = torch.bfloat16
@@ -230,29 +219,6 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
assert shape[1] is not None and shape[2] is not None
return shape[0], shape[1], shape[2]
@staticmethod
def _w2_scale_sharding(
actorder,
group_size: int,
intermediate_size_per_partition: int,
intermediate_size_full: int,
) -> tuple[bool, int, bool]:
"""Decide how to shard w2 group scales across TP for WNA16 Marlin MoE.
Only ``actorder="group"`` permutes activations by ``g_idx`` at runtime
and therefore needs the full-K (unsharded) w2 scales plus ``is_k_full``.
``actorder="weight"``/``"static"`` (and ``None``) reorder weights at
quantization time, so scales shard normally per TP rank.
"""
load_full_w2 = (actorder == "group") and group_size != -1
w2_scales_size = (
intermediate_size_full if load_full_w2 else intermediate_size_per_partition
)
is_k_full = (actorder != "group") or (
intermediate_size_per_partition == intermediate_size_full
)
return load_full_w2, w2_scales_size, is_k_full
def create_weights(
self,
layer: torch.nn.Module,
@@ -262,8 +228,6 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
params_dtype: torch.dtype,
**extra_weight_attrs,
):
intermediate_size_full = extra_weight_attrs.pop("intermediate_size_full")
# Will transpose the loaded weight along the
# intermediate and hidden dim sizes. Will
# shard for TP along the transposed dims
@@ -301,13 +265,6 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
layer.register_parameter("w2_weight_packed", w2_weight)
set_weight_attrs(w2_weight, extra_weight_attrs)
load_full_w2, w2_scales_size, self.is_k_full = self._w2_scale_sharding(
self.actorder,
self.group_size,
intermediate_size_per_partition,
intermediate_size_full,
)
if self.strategy == "channel":
num_groups_w2 = num_groups_w13 = 1
self.group_size = -1
@@ -318,10 +275,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
f"({hidden_size}) to be divisible by group_size "
f"({self.group_size})."
)
if (
not load_full_w2
and intermediate_size_per_partition % self.group_size != 0
):
if intermediate_size_per_partition % self.group_size != 0:
raise ValueError(
"CompressedTensors WNA16 MoE with static group "
"scales requires the MoE intermediate size per "
@@ -331,7 +285,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
"otherwise cross TP shard boundaries; use a compatible TP "
"size or enable expert parallelism."
)
num_groups_w2 = w2_scales_size // self.group_size
num_groups_w2 = intermediate_size_per_partition // self.group_size
num_groups_w13 = hidden_size // self.group_size
if not self.symmetric:
@@ -389,7 +343,6 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
)
layer.register_parameter("w2_weight_scale", w2_scale)
set_weight_attrs(w2_scale, extra_weight_attrs)
set_weight_attrs(w2_scale, {"load_full_w2": load_full_w2})
if not self.symmetric:
w13_zp = torch.nn.Parameter(
@@ -436,50 +389,6 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
layer.register_parameter("w13_weight_shape", w13_weight_shape)
set_weight_attrs(w13_weight_shape, extra_weight_attrs)
w13_g_idx = torch.nn.Parameter(
torch.empty(
num_experts,
hidden_size,
dtype=torch.int32,
),
requires_grad=False,
)
layer.register_parameter("w13_weight_g_idx", w13_g_idx)
set_weight_attrs(w13_g_idx, extra_weight_attrs)
w2_g_idx = torch.nn.Parameter(
torch.empty(
num_experts,
intermediate_size_per_partition,
dtype=torch.int32,
),
requires_grad=False,
)
layer.register_parameter("w2_weight_g_idx", w2_g_idx)
set_weight_attrs(w2_g_idx, extra_weight_attrs)
w13_g_idx_sort_indices = torch.nn.Parameter(
torch.empty(
num_experts,
hidden_size,
dtype=torch.int32,
),
requires_grad=False,
)
layer.register_parameter("w13_g_idx_sort_indices", w13_g_idx_sort_indices)
set_weight_attrs(w13_g_idx_sort_indices, extra_weight_attrs)
w2_g_idx_sort_indices = torch.nn.Parameter(
torch.empty(
num_experts,
intermediate_size_per_partition,
dtype=torch.int32,
),
requires_grad=False,
)
layer.register_parameter("w2_g_idx_sort_indices", w2_g_idx_sort_indices)
set_weight_attrs(w2_g_idx_sort_indices, extra_weight_attrs)
layer.a13_scale = None
layer.a2_scale = None
@@ -488,24 +397,12 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
assert self.moe_quant_config is not None
# Add Marlin-specific arguments
marlin_args: dict[str, Any] = {}
if self.is_marlin:
marlin_args = {
"w13_g_idx": layer.w13_weight_g_idx,
"w2_g_idx": layer.w2_weight_g_idx,
"w13_g_idx_sort_indices": layer.w13_g_idx_sort_indices,
"w2_g_idx_sort_indices": layer.w2_g_idx_sort_indices,
"is_k_full": self.is_k_full,
}
self.moe_kernel = make_wna16_moe_kernel(
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
experts_cls=self.experts_cls,
backend=self.wna16_backend,
routing_tables=layer._expert_routing_tables(),
**marlin_args,
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
@@ -519,8 +416,6 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
w2=layer.w2_weight_packed,
w13_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
w13_g_idx=layer.w13_weight_g_idx,
w2_g_idx=layer.w2_weight_g_idx,
w13_qzeros=getattr(layer, "w13_weight_zero_point", None),
w2_qzeros=getattr(layer, "w2_weight_zero_point", None),
)
@@ -534,10 +429,6 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
w2_qweight,
w13_scales,
w2_scales,
w13_g_idx_processed,
w2_g_idx_processed,
w13_g_idx_sort_indices,
w2_g_idx_sort_indices,
w13_qzeros,
w2_qzeros,
w13_input_global_scale,
@@ -561,19 +452,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
replace_parameter(layer, "w13_weight_zero_point", w13_qzeros)
replace_parameter(layer, "w2_weight_zero_point", w2_qzeros)
# Marlin-specific parameters (not needed for Flashinfer)
if self.is_marlin:
if w13_g_idx_processed is not None:
replace_parameter(layer, "w13_weight_g_idx", w13_g_idx_processed)
if w2_g_idx_processed is not None:
replace_parameter(layer, "w2_weight_g_idx", w2_g_idx_processed)
if w13_g_idx_sort_indices is not None:
replace_parameter(
layer, "w13_g_idx_sort_indices", w13_g_idx_sort_indices
)
if w2_g_idx_sort_indices is not None:
replace_parameter(layer, "w2_g_idx_sort_indices", w2_g_idx_sort_indices)
# Register input global scales if present
if w13_input_global_scale is not None:
layer.register_parameter(
@@ -593,7 +472,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
and self.wna16_backend != WNA16MoEBackend.EMULATION
):
layer.workspace = marlin_make_workspace_new(
layer.w13_weight_g_idx.device,
layer.w13_weight_packed.device,
4,
existing=getattr(layer, "workspace", None),
)
@@ -65,15 +65,14 @@ class CompressedTensorsWNA16RDNA3MoEMethod(CompressedTensorsWNA16MoEMethod):
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
device = layer.w13_weight_packed.device
num_experts = layer.w13_weight_packed.shape[0]
empty_g_idx = torch.empty(0, dtype=torch.int32, device=device)
# Shuffle weights in-place per expert (exllama nibble interleave)
for e in range(num_experts):
w13_e = layer.w13_weight_packed.data[e].contiguous()
ops.gptq_shuffle(w13_e, empty_g_idx, 4)
ops.gptq_shuffle(w13_e, 4)
layer.w13_weight_packed.data[e] = w13_e
w2_e = layer.w2_weight_packed.data[e].contiguous()
ops.gptq_shuffle(w2_e, empty_g_idx, 4)
ops.gptq_shuffle(w2_e, 4)
layer.w2_weight_packed.data[e] = w2_e
# Keep scales as [E, groups, N] in activation dtype
@@ -4,7 +4,6 @@
from collections.abc import Callable
import torch
from compressed_tensors.quantization import ActivationOrdering
from vllm.distributed.utils import verify_group_size_divides_partition
from vllm.logger import init_logger
@@ -43,13 +42,11 @@ class CompressedTensorsW4A8Fp8(CompressedTensorsScheme):
num_bits: int,
group_size: int | None = None,
symmetric: bool | None = True,
actorder: ActivationOrdering | None = None,
):
self.pack_factor = 32 // num_bits
self.strategy = strategy
self.symmetric = symmetric
self.group_size = -1 if group_size is None else group_size
self.has_g_idx = actorder == ActivationOrdering.GROUP
if self.group_size != 128 or self.strategy != "group":
raise ValueError(
@@ -92,7 +89,6 @@ class CompressedTensorsW4A8Fp8(CompressedTensorsScheme):
act_type=torch.float8_e4m3fn, # always use fp8(e4m3)
group_size=self.group_size,
zero_points=not self.symmetric,
has_g_idx=self.has_g_idx,
out_type=params_dtype,
)
@@ -106,7 +102,7 @@ class CompressedTensorsW4A8Fp8(CompressedTensorsScheme):
group_size = self.group_size if self.group_size != -1 else input_size
row_parallel = input_size != input_size_per_partition
partition_scales = not marlin_repeat_scales_on_all_ranks(
self.has_g_idx, self.group_size, row_parallel
self.group_size, row_parallel
)
scales_and_zp_size = input_size // group_size
@@ -162,7 +158,6 @@ class CompressedTensorsW4A8Fp8(CompressedTensorsScheme):
w_q_param_name="weight_packed",
w_s_param_name="weight_scale",
w_zp_param_name="weight_zero_point",
w_gidx_param_name="weight_g_idx",
)
# Checkpoints are serialized in compressed-tensors format, which is
@@ -98,7 +98,6 @@ class CompressedTensorsW4A8Int(CompressedTensorsScheme):
act_type=params_dtype,
group_size=effective_group_size,
zero_points=False,
has_g_idx=False,
)
kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config)
@@ -140,7 +139,6 @@ class CompressedTensorsW4A8Int(CompressedTensorsScheme):
w_q_param_name="weight_packed",
w_s_param_name="weight_scale",
w_zp_param_name=None,
w_gidx_param_name=None,
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
@@ -6,7 +6,6 @@ from collections.abc import Callable
from fractions import Fraction
import torch
from compressed_tensors.quantization import ActivationOrdering
from vllm.distributed.utils import verify_group_size_divides_partition
from vllm.logger import init_logger
@@ -28,7 +27,6 @@ from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
PackedColumnParameter,
PackedvLLMParameter,
RowvLLMParameter,
)
from vllm.scalar_type import scalar_types
@@ -57,7 +55,6 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
num_bits: int,
group_size: int | None = None,
symmetric: bool | None = True,
actorder: ActivationOrdering | None = None,
layer_name: str | None = None,
):
self.num_bits = num_bits
@@ -65,7 +62,6 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
self.strategy = strategy
self.symmetric = symmetric
self.group_size = -1 if group_size is None else group_size
self.has_g_idx = actorder == ActivationOrdering.GROUP
self.layer_name = layer_name
if self.group_size == -1 and self.strategy != "channel":
@@ -128,7 +124,6 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
act_type=params_dtype,
group_size=self.group_size,
zero_points=not self.symmetric,
has_g_idx=self.has_g_idx,
)
kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config)
@@ -146,7 +141,7 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
group_size = self.group_size if self.group_size != -1 else input_size
row_parallel = input_size != input_size_per_partition
partition_scales = not marlin_repeat_scales_on_all_ranks(
self.has_g_idx, self.group_size, row_parallel
self.group_size, row_parallel
)
scales_and_zp_size = input_size // group_size
@@ -226,24 +221,11 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
if not self.symmetric:
layer.register_parameter("weight_zero_point", qzeros)
# group index (for activation reordering)
if self.has_g_idx:
weight_g_idx = RowvLLMParameter(
data=torch.empty(
input_size_per_partition,
dtype=torch.int32,
),
input_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("weight_g_idx", weight_g_idx)
self.kernel = kernel_type(
mp_linear_kernel_config,
w_q_param_name="weight_packed",
w_s_param_name="weight_scale",
w_zp_param_name="weight_zero_point",
w_gidx_param_name="weight_g_idx",
)
# Checkpoints are serialized in compressed-tensors format, which is
@@ -144,7 +144,6 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
act_type=params_dtype,
group_size=self.group_size,
zero_points=False,
has_g_idx=False,
)
kernel_type = choose_mp_linear_kernel(mp_config)
@@ -166,7 +165,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
group_size = self.group_size if self.group_size != -1 else input_size
row_parallel = input_size != input_size_per_partition
partition_scales = not marlin_repeat_scales_on_all_ranks(
False, self.group_size, row_parallel
self.group_size, row_parallel
)
scales_size = input_size // group_size
if partition_scales:
@@ -144,7 +144,6 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
act_type=params_dtype,
group_size=self.group_size,
zero_points=False,
has_g_idx=False,
)
kernel_type = choose_mp_linear_kernel(mp_config)
@@ -166,7 +165,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
group_size = self.group_size if self.group_size != -1 else input_size
row_parallel = input_size != input_size_per_partition
partition_scales = not marlin_repeat_scales_on_all_ranks(
False, self.group_size, row_parallel
self.group_size, row_parallel
)
scales_size = input_size // group_size
if partition_scales:
@@ -114,7 +114,6 @@ class CompressedTensorsWNA8O8Int(CompressedTensorsScheme):
act_type=params_dtype, # activation quant applied externally (SRQ)
group_size=self.group_size,
zero_points=False,
has_g_idx=False,
)
self.kernel = choose_mp_linear_kernel(mp_config)(
mp_config,
@@ -167,7 +166,7 @@ class CompressedTensorsWNA8O8Int(CompressedTensorsScheme):
# Scale: per-output-channel, or per group along the input dim under TP.
group_size = self.group_size if self.group_size != -1 else input_size
partitioned = not marlin_repeat_scales_on_all_ranks(
False, self.group_size, input_size != input_size_per_partition
self.group_size, input_size != input_size_per_partition
)
scales = (input_size_per_partition if partitioned else input_size) // group_size
scale_data = torch.empty(out, scales, dtype=params_dtype)
@@ -106,7 +106,7 @@ class INCXPUW4A8LinearMethod(INCXPULinearMethod):
layer.scales_fp16,
layer.qzeros,
self.group_size,
None,
None, # Retained by the external XPU op ABI.
bias,
)
return out.to(x.dtype).reshape(out_shape)
@@ -14,7 +14,6 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
PackedvLLMParameter,
RowvLLMParameter,
)
from vllm.scalar_type import scalar_types
@@ -254,16 +253,6 @@ class INCXPULinearBase(INCLinearScheme):
layer.register_parameter("scales", scales)
layer.register_parameter("qzeros", qzeros)
g_idx = RowvLLMParameter(
data=torch.tensor(
[i // self.group_size for i in range(input_size_per_partition)],
dtype=torch.int32,
),
input_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("g_idx", g_idx)
def _convert_awq_qweight_to_gptq(self, qw: torch.Tensor) -> torch.Tensor:
"""Convert AWQ qweight [K, N // pf] to GPTQ qweight [K // pf, N].
@@ -356,7 +345,7 @@ class INCXPULinearMethod(INCXPULinearBase):
layer.scales,
layer.qzeros,
self.group_size,
None,
None, # Retained by the external XPU op ABI.
)
return out.reshape(out_shape)
@@ -32,6 +32,9 @@ from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
QuantizeMethodBase,
)
from vllm.model_executor.layers.quantization.utils.gptq_utils import (
normalize_and_validate_gptq_desc_act,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
INT4_DTYPE,
INT8_DTYPE,
@@ -69,7 +72,11 @@ class MoeWNA16Config(QuantizationConfig):
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
if self.linear_quant_method == "gptq":
pass
normalize_and_validate_gptq_desc_act(
full_config.get("desc_act", False),
group_size,
full_config.get("dynamic") or {},
)
elif self.linear_quant_method in ("awq", "awq_marlin"):
capability_tuple = current_platform.get_device_capability()
device_capability = (
@@ -148,7 +155,16 @@ class MoeWNA16Config(QuantizationConfig):
# Extract data from quant config.
quant_method = quant_config.get("quant_method", "").lower()
num_bits = quant_config.get("bits")
desc_act = quant_config.get("desc_act")
desc_act = quant_config.get("desc_act", False)
try:
normalize_and_validate_gptq_desc_act(
desc_act,
quant_config.get("group_size"),
quant_config.get("dynamic") or {},
)
supported_act_order = True
except ValueError:
supported_act_order = False
capability_tuple = current_platform.get_device_capability()
device_capability = (
@@ -159,7 +175,9 @@ class MoeWNA16Config(QuantizationConfig):
awq_min_capability = AutoAWQConfig.get_min_capability()
gptq_compatible = quant_method == "gptq" and not desc_act and num_bits in [4, 8]
gptq_compatible = (
quant_method == "gptq" and supported_act_order and num_bits in [4, 8]
)
awq_compatible = (
quant_method == "awq"
and num_bits == 4
@@ -356,7 +374,7 @@ class MoeWNA16Method(FusedMoEMethodBase):
if self.quant_config.linear_quant_method == "gptq":
# some param are unused, but we need to init them in order to
# load weights
invalid_param_keys = ["w13_g_idx", "w2_g_idx"]
invalid_param_keys: list[str] = []
if not self.quant_config.has_zp:
invalid_param_keys += ["w13_qzeros", "w2_qzeros"]
for key in invalid_param_keys:
@@ -414,8 +432,6 @@ class MoeWNA16Method(FusedMoEMethodBase):
w2=layer.w2_qweight,
w13_scale=layer.w13_scales,
w2_scale=layer.w2_scales,
w13_g_idx=None,
w2_g_idx=None,
w13_qzeros=layer.w13_qzeros if has_zp else None,
w2_qzeros=layer.w2_qzeros if has_zp else None,
)
@@ -430,10 +446,6 @@ class MoeWNA16Method(FusedMoEMethodBase):
w2_qweight,
w13_scales,
w2_scales,
_,
_,
_,
_,
w13_qzeros,
w2_qzeros,
w13_input_global_scale,
@@ -576,8 +588,6 @@ class MoeWNA16Method(FusedMoEMethodBase):
expert_id: int,
return_success: bool = False,
):
if "g_idx" in weight_name:
return False if return_success else None
if not layer.quant_config.has_zp and "qzeros" in weight_name:
return False if return_success else None
@@ -20,6 +20,36 @@ else:
AutoGPTQConfig = object
def normalize_and_validate_gptq_desc_act(
desc_act: bool,
group_size: int | None,
dynamic: Mapping[str, Any],
) -> bool:
base_desc_act = desc_act
if desc_act and group_size == -1:
# With one group per output channel, activation ordering is a no-op.
desc_act = False
if desc_act:
raise ValueError(
"GPTQ group activation ordering (desc_act=True) is no longer "
"supported. Use a checkpoint with static activation ordering "
"or desc_act=False."
)
for pattern, overrides in dynamic.items():
if pattern.startswith("-:") or not isinstance(overrides, Mapping):
continue
dynamic_desc_act = overrides.get("desc_act", base_desc_act)
dynamic_group_size = overrides.get("group_size", group_size)
if dynamic_desc_act and dynamic_group_size != -1:
raise ValueError(
"GPTQ group activation ordering (desc_act=True) is no longer "
f"supported, but is enabled by dynamic rule {pattern!r}. Use "
"a checkpoint with static activation ordering or desc_act=False."
)
return desc_act
# Match dynamic rules with module name (prefix) and override quantize
# config if module (prefix) matches a rule
def override_config(config: AutoGPTQConfig, prefix: str):
@@ -33,6 +63,10 @@ def override_config(config: AutoGPTQConfig, prefix: str):
if isinstance(desc_act, bool):
config.desc_act = desc_act
if config.desc_act and config.group_size == -1:
# Activation ordering is a no-op for channelwise quantization.
config.desc_act = False
config.pack_factor = 32 // config.weight_bits # packed into int32
assert isinstance(config, AutoGPTQConfig)
is_sym = get_dynamic_override(config, prefix, "sym", config.is_sym)
@@ -359,11 +359,11 @@ def check_moe_marlin_supports_config(
) -> bool:
"""Whether the fused MoE Marlin kernel supports ``config``.
Callers without act-order may pass ``allow_tile_padding=True``: a
tile-misaligned intermediate size is then zero-padded to a valid thread
tile at weight prep (see marlin_moe_padded_intermediate), so only a group
straddling the padded boundary stays unsupported. hidden_size is the MoE
I/O extent and is never padded. Act-order keeps the strict shape.
With ``allow_tile_padding=True``, a tile-misaligned intermediate size is
zero-padded to a valid thread tile at weight prep (see
marlin_moe_padded_intermediate), so only a group straddling the padded
boundary stays unsupported. hidden_size is the MoE I/O extent and is never
padded.
"""
if current_platform.is_rocm():
return False
@@ -433,30 +433,17 @@ def marlin_make_workspace_new(
return torch.zeros(size, dtype=torch.int, device=device, requires_grad=False)
def marlin_is_k_full(act_order: bool, is_row_parallel: bool) -> bool:
return (not act_order) or (act_order and not is_row_parallel)
def marlin_repeat_scales_on_all_ranks(
act_order: bool, group_size: int, is_row_parallel: bool
) -> bool:
# Need to repeat scales on every rank if act_ordering or
# channelwise and RowParallelLinear
def marlin_repeat_scales_on_all_ranks(group_size: int, is_row_parallel: bool) -> bool:
is_channelwise = group_size == -1
return act_order or (is_channelwise and is_row_parallel)
return is_channelwise and is_row_parallel
def marlin_make_empty_g_idx(device: torch.device) -> torch.Tensor:
def marlin_make_empty(device: torch.device) -> torch.Tensor:
return torch.nn.Parameter(
torch.empty(0, dtype=torch.int, device=device), requires_grad=False
)
def marlin_sort_g_idx(g_idx: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
g_idx_sort_indices = torch.argsort(g_idx).to(torch.int)
return g_idx[g_idx_sort_indices], g_idx_sort_indices
def get_scale_perms():
scale_perm: list[int] = []
for i in range(8):
@@ -700,13 +687,10 @@ def apply_gptq_marlin_linear(
weight: torch.Tensor,
weight_scale: torch.Tensor,
weight_zp: torch.Tensor,
g_idx: torch.Tensor,
g_idx_sort_indices: torch.Tensor,
workspace: torch.Tensor,
wtype: ScalarType,
output_size_per_partition: int,
input_size_per_partition: int,
is_k_full: bool,
input_global_scale: torch.Tensor | None = None,
bias: torch.Tensor | None = None,
use_fp32_reduce: bool = USE_FP32_REDUCE_DEFAULT,
@@ -749,14 +733,11 @@ def apply_gptq_marlin_linear(
a_scales,
None,
weight_zp,
g_idx,
g_idx_sort_indices,
workspace,
wtype,
size_m=reshaped_x.shape[0],
size_n=padded_n,
size_k=padded_k,
is_k_full=is_k_full,
use_atomic_add=use_atomic_add,
use_fp32_reduce=use_fp32_reduce,
is_zp_float=False,
@@ -203,8 +203,6 @@ def apply_fp4_marlin_linear(
a_scales=a_scales,
global_scale=weight_global_scale,
b_zeros=None,
g_idx=None,
perm=None,
workspace=workspace,
b_q_type=scalar_types.float4_e2m1f,
size_m=reshaped_x.size(0),
@@ -246,14 +244,12 @@ def prepare_fp4_layer_for_marlin(
# WEIGHT
# Repack weights to marlin format
perm = torch.empty(0, dtype=torch.int, device=device)
qweight = layer.weight.view(torch.int32).T.contiguous()
qweight = marlin_pad_qweight(qweight, part_size_n, part_size_k, padded_n, padded_k)
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
marlin_qweight = ops.gptq_marlin_repack(
b_q_weight=qweight,
perm=perm,
size_k=padded_k,
size_n=padded_n,
num_bits=4,
@@ -312,7 +308,6 @@ def _repack_marlin_experts(
weight: torch.Tensor,
size_n: int,
size_k: int,
perm: torch.Tensor,
is_a_8bit: bool,
) -> torch.Tensor:
"""Repack each expert to marlin format into a preallocated output."""
@@ -322,7 +317,6 @@ def _repack_marlin_experts(
qweight = weight[i].view(torch.int32).T.contiguous()
marlin_qweight = ops.gptq_marlin_repack(
b_q_weight=qweight,
perm=perm,
size_k=size_k,
size_n=size_n,
num_bits=4,
@@ -401,7 +395,6 @@ def prepare_nvfp4_moe_layer_for_marlin(
layer.workspace = marlin_make_workspace_new(
device, 4, existing=getattr(layer, "workspace", None)
)
perm = torch.empty(0, dtype=torch.int, device=device)
# WEIGHT
# Repack weights to marlin format
@@ -417,7 +410,7 @@ def prepare_nvfp4_moe_layer_for_marlin(
weight = pad_w2(weight, packing=2)
size_k = padded_N
return _repack_marlin_experts(weight, size_n, size_k, perm, is_a_8bit)
return _repack_marlin_experts(weight, size_n, size_k, is_a_8bit)
w13 = repack_weight(w13, "w13")
w2 = repack_weight(w2, "w2")
@@ -492,7 +485,6 @@ def prepare_moe_fp4_layer_for_marlin(
layer.workspace = marlin_make_workspace_new(
device, 4, existing=getattr(layer, "workspace", None)
)
perm = torch.empty(0, dtype=torch.int, device=device)
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
# WEIGHT
@@ -506,7 +498,7 @@ def prepare_moe_fp4_layer_for_marlin(
assert weight.shape == (e, size_n, size_k // 2)
weight = _repack_marlin_experts(weight, size_n, size_k, perm, is_a_8bit)
weight = _repack_marlin_experts(weight, size_n, size_k, is_a_8bit)
weight = torch.nn.Parameter(weight, requires_grad=False)
setattr(layer, name, weight)
@@ -621,10 +613,8 @@ def prepare_moe_mxfp4_layer_for_marlin(
n = w13.shape[1] // 2 # intermediate_size_per_partition
k = w13.shape[2] * 2 # hidden_size
device = w13.device
param_dtype = layer.params_dtype
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
perm = torch.empty(0, dtype=torch.int, device=device)
# WEIGHT: Repack weights to marlin format
def repack_weight(weight: torch.Tensor, name: str) -> torch.Tensor:
@@ -635,7 +625,7 @@ def prepare_moe_mxfp4_layer_for_marlin(
assert weight.shape == (e, size_n, size_k // 2)
return _repack_marlin_experts(weight, size_n, size_k, perm, is_a_8bit)
return _repack_marlin_experts(weight, size_n, size_k, is_a_8bit)
w13 = repack_weight(w13, "w13")
w2 = repack_weight(w2, "w2")
@@ -691,8 +681,6 @@ def rand_marlin_weight_nvfp4_like(weight, group_size, input_dtype=None):
assert not is_a_8bit, "NVFP4 weight + INT8/FP8 activation is not supported."
assert group_size > 0
size_n, size_k = weight.shape
device = weight.device
scales = weight.view(size_n, -1, group_size).abs().max(-1)[0] / 6
global_scale = scales.max() / 448
scales = (scales / global_scale).to(torch.float8_e4m3fn)
@@ -720,7 +708,6 @@ def rand_marlin_weight_nvfp4_like(weight, group_size, input_dtype=None):
marlin_qweight = ops.gptq_marlin_repack(
b_q_weight=fp4_weight.view(torch.int32).T.contiguous(),
perm=torch.empty(0, dtype=torch.int, device=device),
size_k=size_k,
size_n=size_n,
num_bits=4,
@@ -751,8 +738,6 @@ def rand_marlin_weight_mxfp4_like(weight, group_size, input_dtype=None):
assert group_size > 0
size_n, size_k = weight.shape
device = weight.device
scales = torch.randint(
110,
120,
@@ -779,11 +764,9 @@ def rand_marlin_weight_mxfp4_like(weight, group_size, input_dtype=None):
).view(size_n, size_k)
weight_ref = weight_ref * scales.repeat_interleave(group_size, 1).to(weight.dtype)
perm = torch.empty(0, dtype=torch.int, device=device)
fp4_weight = fp4_weight.view(torch.int32).T.contiguous()
marlin_qweight = ops.gptq_marlin_repack(
b_q_weight=fp4_weight,
perm=perm,
size_k=size_k,
size_n=size_n,
num_bits=4,
@@ -89,8 +89,6 @@ def apply_fp8_marlin_linear(
a_scales=a_scales,
global_scale=None,
b_zeros=None,
g_idx=None,
perm=None,
workspace=workspace,
b_q_type=scalar_types.float8_e4m3fn,
size_m=reshaped_x.size(0),
@@ -138,7 +136,6 @@ def prepare_fp8_layer_for_marlin(
# WEIGHT
# Repack weights to marlin format
perm = torch.empty(0, dtype=torch.int, device=device)
qweight = pack_fp8_to_int32(layer.weight, size_k_first)
if not size_k_first:
qweight = qweight.T.contiguous()
@@ -146,7 +143,6 @@ def prepare_fp8_layer_for_marlin(
marlin_qweight = ops.gptq_marlin_repack(
b_q_weight=qweight,
perm=perm,
size_k=padded_k,
size_n=padded_n,
num_bits=8,
@@ -283,7 +279,6 @@ def prepare_fp8_moe_layer_for_marlin(
layer.workspace = marlin_make_workspace_new(
device, 4, existing=getattr(layer, "workspace", None)
)
perm = torch.empty(0, dtype=torch.int, device=device)
# WEIGHT
# Repack weights to marlin format
@@ -295,7 +290,7 @@ def prepare_fp8_moe_layer_for_marlin(
qweight = qweight.T.contiguous()
marlin_qweight = ops.gptq_marlin_repack(
b_q_weight=qweight, perm=perm, size_k=size_k, size_n=size_n, num_bits=8
b_q_weight=qweight, size_k=size_k, size_n=size_n, num_bits=8
)
tensor_list.append(marlin_qweight)
@@ -434,8 +429,6 @@ def apply_mxfp8_marlin_linear(
a_scales=None,
global_scale=None,
b_zeros=None,
g_idx=None,
perm=None,
workspace=workspace,
b_q_type=scalar_types.float8_e4m3fn,
size_m=reshaped_x.size(0),
@@ -470,14 +463,12 @@ def prepare_mxfp8_layer_for_marlin(layer: torch.nn.Module) -> None:
)
# WEIGHT - repack FP8 weights to Marlin format
perm = torch.empty(0, dtype=torch.int, device=device)
qweight = pack_fp8_to_int32(layer.weight, size_k_first=False)
qweight = qweight.T.contiguous()
qweight = marlin_pad_qweight(qweight, part_size_n, part_size_k, padded_n, padded_k)
marlin_qweight = ops.gptq_marlin_repack(
b_q_weight=qweight,
perm=perm,
size_k=padded_k,
size_n=padded_n,
num_bits=8,
@@ -551,7 +542,6 @@ def prepare_mxfp8_moe_layer_for_marlin(
device = w13.device
param_dtype = torch.get_default_dtype()
perm = torch.empty(0, dtype=torch.int, device=device)
layer.workspace = marlin_make_workspace_new(
device, 4, existing=getattr(layer, "workspace", None)
@@ -571,7 +561,6 @@ def prepare_mxfp8_moe_layer_for_marlin(
qweight = qweight.T.contiguous()
marlin_qweight = ops.gptq_marlin_repack(
b_q_weight=qweight,
perm=perm,
size_k=size_k,
size_n=size_n,
num_bits=8,
@@ -615,8 +604,6 @@ def marlin_quant_fp8_torch(weight, group_size, input_dtype=None):
assert input_dtype == torch.float8_e4m3fn
size_n, size_k = weight.shape
device = weight.device
if group_size != -1:
scales = weight.view(size_n, -1, group_size).abs().max(-1)[0] / 448
repeated_scales = scales.repeat_interleave(group_size, 1)
@@ -629,10 +616,8 @@ def marlin_quant_fp8_torch(weight, group_size, input_dtype=None):
weight_ref = fp8_weight.to(weight.dtype) * repeated_scales
packed_weight = pack_fp8_to_int32(fp8_weight, False).T.contiguous()
perm = torch.empty(0, dtype=torch.int, device=device)
marlin_qweight = ops.gptq_marlin_repack(
b_q_weight=packed_weight,
perm=perm,
size_k=size_k,
size_n=size_n,
num_bits=8,
@@ -13,7 +13,6 @@ from .quant_utils import (
get_pack_factor,
gptq_quantize_weights,
quantize_weights,
sort_weights,
)
@@ -129,8 +128,6 @@ def marlin_quantize(
w: torch.Tensor,
quant_type: ScalarType,
group_size: int,
act_order: bool,
test_perm: torch.Tensor | None = None,
input_dtype: torch.dtype | None = None,
):
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
@@ -143,16 +140,7 @@ def marlin_quantize(
group_size = size_k
assert group_size <= size_k
# Quantize (and apply act_order if provided)
w_ref, q_w, s, g_idx, rand_perm = gptq_quantize_weights(
w, quant_type, group_size, act_order, test_perm
)
# For act_order, sort the "weights" and "g_idx" so that group ids are
# increasing
sort_indices = torch.empty(0, dtype=torch.int, device=w.device)
if act_order:
q_w, g_idx, sort_indices = sort_weights(q_w, g_idx)
w_ref, q_w, s = gptq_quantize_weights(w, quant_type, group_size)
# Reformat to marlin
weight_perm = get_weight_perm(num_bits, is_a_8bit)
@@ -166,7 +154,7 @@ def marlin_quantize(
marlin_s = marlin_s * 512
# Create result
res_list = [w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, rand_perm]
res_list = [w_ref, marlin_q_w, marlin_s]
for i in range(len(res_list)):
res_list[i] = res_list[i].to(w.device)
@@ -691,36 +691,6 @@ def get_pack_factor(num_bits):
return 32 // num_bits
def permute_rows(
q_w: torch.Tensor,
w_ref: torch.Tensor,
group_size: int,
test_perm: torch.Tensor | None = None,
):
assert q_w.shape == w_ref.shape
orig_device = q_w.device
k_size, _ = q_w.shape
g_idx = torch.zeros((k_size,), dtype=torch.int32)
for i in range(k_size):
g_idx[i] = i // group_size
# Simulate act_order by doing a random permutation on K
rand_perm = test_perm if test_perm is not None else torch.randperm(k_size)
g_idx = g_idx[rand_perm].contiguous()
q_w = q_w[rand_perm, :].contiguous()
w_ref = w_ref[rand_perm, :].contiguous()
return (
w_ref.to(device=orig_device),
q_w.to(device=orig_device),
g_idx.to(device=orig_device),
rand_perm.to(device=orig_device),
)
def quantize_weights(
w: torch.Tensor,
quant_type: ScalarType,
@@ -824,8 +794,6 @@ def gptq_quantize_weights(
w: torch.Tensor,
quant_type: ScalarType,
group_size: int,
act_order: bool,
test_perm: torch.Tensor | None = None,
):
size_k, _ = w.shape
@@ -838,35 +806,7 @@ def gptq_quantize_weights(
)
w_ref, w_q, w_s, _ = quantize_weights(w, quant_type, group_size)
# Apply act_order
g_idx = torch.empty(0, dtype=torch.int, device=w.device)
rand_perm = torch.empty(0, dtype=torch.int, device=w.device)
if act_order:
assert group_size < size_k, (
"For act_order, groupsize = {} must be less than size_k = {}".format(
group_size, size_k
)
)
w_ref, w_q, g_idx, rand_perm = permute_rows(w_q, w_ref, group_size, test_perm)
return w_ref, w_q, w_s, g_idx, rand_perm
def sort_weights(q_w: torch.Tensor, g_idx: torch.Tensor):
orig_device = q_w.device
sort_indices = torch.argsort(g_idx).to(dtype=torch.int32) # Sort based on g_idx
g_idx = g_idx[sort_indices].contiguous()
q_w = q_w[sort_indices, :].contiguous()
return (
q_w.to(device=orig_device),
g_idx.to(device=orig_device),
sort_indices.to(device=orig_device),
)
return w_ref, w_q, w_s
def pack_rows(