[Quant][Kernel] Remove GPTQ Group/Dynamic Activation Ordering (#54809)
Signed-off-by: Roderick-Wu <roderickwu2003@gmail.com> Signed-off-by: Roderick Wu <roderickwu2003@gmail.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -81,10 +81,6 @@ def _fused_marlin_moe(
|
||||
input_global_scale2: torch.Tensor | None = None,
|
||||
global_scale1: torch.Tensor | None = None,
|
||||
global_scale2: torch.Tensor | None = None,
|
||||
g_idx1: torch.Tensor | None = None,
|
||||
g_idx2: torch.Tensor | None = None,
|
||||
sort_indices1: torch.Tensor | None = None,
|
||||
sort_indices2: torch.Tensor | None = None,
|
||||
w1_zeros: torch.Tensor | None = None,
|
||||
w2_zeros: torch.Tensor | None = None,
|
||||
workspace: torch.Tensor | None = None,
|
||||
@@ -92,7 +88,6 @@ def _fused_marlin_moe(
|
||||
intermediate_cache2: torch.Tensor | None = None,
|
||||
output: torch.Tensor | None = None,
|
||||
input_dtype: torch.dtype | None = None,
|
||||
is_k_full: bool = True,
|
||||
activation_config: ApplyMoEActivationConfig | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert hidden_states.ndim == 2
|
||||
@@ -142,8 +137,6 @@ def _fused_marlin_moe(
|
||||
a_scales1,
|
||||
global_scale1,
|
||||
w1_zeros,
|
||||
g_idx1,
|
||||
sort_indices1,
|
||||
workspace,
|
||||
sorted_token_ids,
|
||||
expert_ids,
|
||||
@@ -156,7 +149,6 @@ def _fused_marlin_moe(
|
||||
size_m=M,
|
||||
size_n=w13_num_shards * N,
|
||||
size_k=K,
|
||||
is_k_full=is_k_full,
|
||||
use_atomic_add=False,
|
||||
use_fp32_reduce=True,
|
||||
is_zp_float=False,
|
||||
@@ -209,8 +201,6 @@ def _fused_marlin_moe(
|
||||
a_scales2,
|
||||
global_scale2,
|
||||
w2_zeros,
|
||||
g_idx2,
|
||||
sort_indices2,
|
||||
workspace,
|
||||
sorted_token_ids,
|
||||
expert_ids,
|
||||
@@ -223,7 +213,6 @@ def _fused_marlin_moe(
|
||||
size_m=M * num_topk,
|
||||
size_n=K,
|
||||
size_k=N,
|
||||
is_k_full=is_k_full,
|
||||
use_atomic_add=False,
|
||||
use_fp32_reduce=True,
|
||||
is_zp_float=False,
|
||||
@@ -253,16 +242,11 @@ def fused_marlin_moe(
|
||||
input_global_scale2: torch.Tensor | None = None,
|
||||
global_scale1: torch.Tensor | None = None,
|
||||
global_scale2: torch.Tensor | None = None,
|
||||
g_idx1: torch.Tensor | None = None,
|
||||
g_idx2: torch.Tensor | None = None,
|
||||
sort_indices1: torch.Tensor | None = None,
|
||||
sort_indices2: torch.Tensor | None = None,
|
||||
w1_zeros: torch.Tensor | None = None,
|
||||
w2_zeros: torch.Tensor | None = None,
|
||||
workspace: torch.Tensor | None = None,
|
||||
intermediate_cache13: torch.Tensor | None = None,
|
||||
intermediate_cache2: torch.Tensor | None = None,
|
||||
is_k_full: bool = True,
|
||||
output: torch.Tensor | None = None,
|
||||
input_dtype: torch.dtype | None = None,
|
||||
activation_config: ApplyMoEActivationConfig | None = None,
|
||||
@@ -277,12 +261,6 @@ def fused_marlin_moe(
|
||||
- w2 (torch.Tensor): The second set of expert weights.
|
||||
- w1_scale (torch.Tensor): Scale to be used for w1.
|
||||
- w2_scale (torch.Tensor): Scale to be used for w2.
|
||||
- g_idx1 (torch.Tensor|None): The first set of act_order indices.
|
||||
- g_idx2 (torch.Tensor|None): The second set of act_order indices.
|
||||
- sort_indices1 (torch.Tensor|None): The first act_order input
|
||||
permutation.
|
||||
- sort_indices2 (torch.Tensor|None): The second act_order input
|
||||
permutation.
|
||||
- topk_weights (torch.Tensor): Top-k weights.
|
||||
- topk_ids (torch.Tensor): Indices of topk-k elements.
|
||||
- w1_zeros (torch.Tensor|None): Optional zero points to be used for w1.
|
||||
@@ -372,10 +350,6 @@ def fused_marlin_moe(
|
||||
input_global_scale2=input_global_scale2,
|
||||
global_scale1=global_scale1,
|
||||
global_scale2=global_scale2,
|
||||
g_idx1=g_idx1,
|
||||
g_idx2=g_idx2,
|
||||
sort_indices1=sort_indices1,
|
||||
sort_indices2=sort_indices2,
|
||||
w1_zeros=w1_zeros,
|
||||
w2_zeros=w2_zeros,
|
||||
workspace=workspace,
|
||||
@@ -383,7 +357,6 @@ def fused_marlin_moe(
|
||||
intermediate_cache2=intermediate_cache2,
|
||||
output=None,
|
||||
input_dtype=input_dtype,
|
||||
is_k_full=is_k_full,
|
||||
).view(-1, topk, K)
|
||||
|
||||
if output is None:
|
||||
@@ -416,16 +389,11 @@ def batched_fused_marlin_moe(
|
||||
input_global_scale2: torch.Tensor | None = None,
|
||||
global_scale1: torch.Tensor | None = None,
|
||||
global_scale2: torch.Tensor | None = None,
|
||||
g_idx1: torch.Tensor | None = None,
|
||||
g_idx2: torch.Tensor | None = None,
|
||||
sort_indices1: torch.Tensor | None = None,
|
||||
sort_indices2: torch.Tensor | None = None,
|
||||
w1_zeros: torch.Tensor | None = None,
|
||||
w2_zeros: torch.Tensor | None = None,
|
||||
workspace: torch.Tensor | None = None,
|
||||
intermediate_cache13: torch.Tensor | None = None,
|
||||
intermediate_cache2: torch.Tensor | None = None,
|
||||
is_k_full: bool = True,
|
||||
output: torch.Tensor | None = None,
|
||||
input_dtype: torch.dtype | None = None,
|
||||
activation_func: Callable[..., None] | None = None,
|
||||
@@ -548,10 +516,6 @@ def batched_fused_marlin_moe(
|
||||
input_global_scale2=input_global_scale2,
|
||||
global_scale1=global_scale1,
|
||||
global_scale2=global_scale2,
|
||||
g_idx1=g_idx1,
|
||||
g_idx2=g_idx2,
|
||||
sort_indices1=sort_indices1,
|
||||
sort_indices2=sort_indices2,
|
||||
w1_zeros=w1_zeros,
|
||||
w2_zeros=w2_zeros,
|
||||
workspace=workspace,
|
||||
@@ -559,7 +523,6 @@ def batched_fused_marlin_moe(
|
||||
intermediate_cache2=intermediate_cache2,
|
||||
output=output.view(-1, K) if output is not None else output,
|
||||
input_dtype=input_dtype,
|
||||
is_k_full=is_k_full,
|
||||
)
|
||||
|
||||
output = output.view(B, BATCH_TOKENS_MAX, K)
|
||||
@@ -574,11 +537,6 @@ class MarlinExpertsBase(mk.FusedMoEExpertsModular):
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
max_num_tokens: int | None = None,
|
||||
num_dispatchers: int | None = None,
|
||||
w13_g_idx: torch.Tensor | None = None,
|
||||
w2_g_idx: torch.Tensor | None = None,
|
||||
w13_g_idx_sort_indices: torch.Tensor | None = None,
|
||||
w2_g_idx_sort_indices: torch.Tensor | None = None,
|
||||
is_k_full: bool = True,
|
||||
):
|
||||
# TODO (varun) : Enable activation quantization
|
||||
assert (
|
||||
@@ -588,11 +546,6 @@ class MarlinExpertsBase(mk.FusedMoEExpertsModular):
|
||||
or quant_config.use_int8_w8a16
|
||||
or quant_config.use_fp8_w8a16
|
||||
), "Supports only {mxfp,nvfp,int}4_w4a16, int8_w8a16 or fp8_w8a16"
|
||||
self.w13_g_idx = w13_g_idx
|
||||
self.w2_g_idx = w2_g_idx
|
||||
self.w13_g_idx_sort_indices = w13_g_idx_sort_indices
|
||||
self.w2_g_idx_sort_indices = w2_g_idx_sort_indices
|
||||
self.is_k_full = is_k_full
|
||||
self.input_dtype = get_marlin_input_dtype()
|
||||
|
||||
super().__init__(
|
||||
@@ -786,11 +739,6 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
|
||||
# output buffer allocation. Please refer to workspace_shapes().
|
||||
intermediate_cache13=workspace2,
|
||||
intermediate_cache2=workspace13,
|
||||
g_idx1=self.w13_g_idx,
|
||||
g_idx2=self.w2_g_idx,
|
||||
sort_indices1=self.w13_g_idx_sort_indices,
|
||||
sort_indices2=self.w2_g_idx_sort_indices,
|
||||
is_k_full=self.is_k_full,
|
||||
input_dtype=self.input_dtype,
|
||||
)
|
||||
return
|
||||
@@ -901,11 +849,6 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
|
||||
output=output,
|
||||
intermediate_cache13=workspace2,
|
||||
intermediate_cache2=workspace13,
|
||||
g_idx1=self.w13_g_idx,
|
||||
g_idx2=self.w2_g_idx,
|
||||
sort_indices1=self.w13_g_idx_sort_indices,
|
||||
sort_indices2=self.w2_g_idx_sort_indices,
|
||||
is_k_full=self.is_k_full,
|
||||
input_dtype=self.input_dtype,
|
||||
)
|
||||
|
||||
@@ -931,22 +874,12 @@ class BatchedMarlinExperts(MarlinExpertsBase):
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
max_num_tokens: int,
|
||||
num_dispatchers: int,
|
||||
w13_g_idx: torch.Tensor | None = None,
|
||||
w2_g_idx: torch.Tensor | None = None,
|
||||
w13_g_idx_sort_indices: torch.Tensor | None = None,
|
||||
w2_g_idx_sort_indices: torch.Tensor | None = None,
|
||||
is_k_full: bool = True,
|
||||
):
|
||||
super().__init__(
|
||||
moe_config=moe_config,
|
||||
quant_config=quant_config,
|
||||
max_num_tokens=max_num_tokens,
|
||||
num_dispatchers=num_dispatchers,
|
||||
w13_g_idx=w13_g_idx,
|
||||
w2_g_idx=w2_g_idx,
|
||||
w13_g_idx_sort_indices=w13_g_idx_sort_indices,
|
||||
w2_g_idx_sort_indices=w2_g_idx_sort_indices,
|
||||
is_k_full=is_k_full,
|
||||
)
|
||||
|
||||
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
|
||||
@@ -1038,14 +971,9 @@ class BatchedMarlinExperts(MarlinExpertsBase):
|
||||
output=output,
|
||||
intermediate_cache13=workspace13,
|
||||
intermediate_cache2=workspace2,
|
||||
g_idx1=self.w13_g_idx,
|
||||
g_idx2=self.w2_g_idx,
|
||||
sort_indices1=self.w13_g_idx_sort_indices,
|
||||
sort_indices2=self.w2_g_idx_sort_indices,
|
||||
w1_zeros=self.w1_zp,
|
||||
w2_zeros=self.w2_zp,
|
||||
input_dtype=self.input_dtype,
|
||||
is_k_full=self.is_k_full,
|
||||
activation_func=activation_func,
|
||||
activation_config=self.activation_config,
|
||||
)
|
||||
|
||||
@@ -22,7 +22,6 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
|
||||
BatchedMarlinExperts,
|
||||
MarlinExperts,
|
||||
MarlinExpertsBase,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_moe import (
|
||||
TritonWNA16Experts,
|
||||
@@ -362,11 +361,6 @@ def make_wna16_moe_kernel(
|
||||
moe_config: FusedMoEConfig,
|
||||
experts_cls: type[mk.FusedMoEExperts],
|
||||
backend: WNA16MoEBackend = WNA16MoEBackend.MARLIN,
|
||||
is_k_full: bool = False,
|
||||
w13_g_idx: torch.Tensor | None = None,
|
||||
w2_g_idx: torch.Tensor | None = None,
|
||||
w13_g_idx_sort_indices: torch.Tensor | None = None,
|
||||
w2_g_idx_sort_indices: torch.Tensor | None = None,
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> mk.FusedMoEKernel:
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
@@ -413,15 +407,6 @@ def make_wna16_moe_kernel(
|
||||
logger.info_once("Using %s", experts_cls.__name__, scope="local")
|
||||
|
||||
extra_args: dict[str, Any] = {}
|
||||
if issubclass(experts_cls, MarlinExpertsBase):
|
||||
extra_args = {
|
||||
"w13_g_idx": w13_g_idx,
|
||||
"w2_g_idx": w2_g_idx,
|
||||
"w13_g_idx_sort_indices": w13_g_idx_sort_indices,
|
||||
"w2_g_idx_sort_indices": w2_g_idx_sort_indices,
|
||||
"is_k_full": is_k_full,
|
||||
}
|
||||
|
||||
if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
|
||||
max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
|
||||
assert max_num_tokens is not None
|
||||
@@ -450,8 +435,6 @@ def _process_weights_flashinfer(
|
||||
w2_qweight: torch.Tensor,
|
||||
w13_scales: torch.Tensor,
|
||||
w2_scales: torch.Tensor,
|
||||
w13_g_idx: torch.Tensor,
|
||||
w2_g_idx: torch.Tensor,
|
||||
w13_bias: torch.Tensor | None = None,
|
||||
w2_bias: torch.Tensor | None = None,
|
||||
) -> tuple[
|
||||
@@ -459,10 +442,6 @@ def _process_weights_flashinfer(
|
||||
torch.Tensor, # w2_qweight
|
||||
torch.Tensor, # w13_scales
|
||||
torch.Tensor, # w2_scales
|
||||
torch.Tensor, # w13_g_idx
|
||||
torch.Tensor, # w2_g_idx
|
||||
torch.Tensor | None, # w13_g_idx_sort_indices
|
||||
torch.Tensor | None, # w2_g_idx_sort_indices
|
||||
torch.Tensor | None, # w13_qzeros
|
||||
torch.Tensor | None, # w2_qzeros
|
||||
torch.Tensor | None, # w13_input_global_scale
|
||||
@@ -475,7 +454,7 @@ def _process_weights_flashinfer(
|
||||
Steps
|
||||
-----
|
||||
1. Transform weights/scales via ``prepare_static_weights_for_trtllm_mxint4_moe``.
|
||||
2. Return transformed tensors, passing through g_idx/bias unchanged.
|
||||
2. Return transformed tensors and biases.
|
||||
"""
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_mxint4_moe import (
|
||||
prepare_static_weights_for_trtllm_mxint4_moe,
|
||||
@@ -493,10 +472,6 @@ def _process_weights_flashinfer(
|
||||
dict_weights_mxint4["gemm2_weights"],
|
||||
dict_weights_mxint4["gemm1_scales"],
|
||||
dict_weights_mxint4["gemm2_scales"],
|
||||
w13_g_idx,
|
||||
w2_g_idx,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
@@ -540,13 +515,10 @@ def _process_weights_marlin(
|
||||
num_bits: int,
|
||||
pack_factor: int,
|
||||
group_size: int,
|
||||
actorder: str | None,
|
||||
w13_qweight: torch.Tensor,
|
||||
w2_qweight: torch.Tensor,
|
||||
w13_scales: torch.Tensor,
|
||||
w2_scales: torch.Tensor,
|
||||
w13_g_idx: torch.Tensor,
|
||||
w2_g_idx: torch.Tensor,
|
||||
w13_qzeros: torch.Tensor | None = None,
|
||||
w2_qzeros: torch.Tensor | None = None,
|
||||
w13_bias: torch.Tensor | None = None,
|
||||
@@ -556,10 +528,6 @@ def _process_weights_marlin(
|
||||
torch.Tensor, # w2_qweight
|
||||
torch.Tensor, # w13_scales
|
||||
torch.Tensor, # w2_scales
|
||||
torch.Tensor, # w13_g_idx
|
||||
torch.Tensor, # w2_g_idx
|
||||
torch.Tensor, # w13_g_idx_sort_indices
|
||||
torch.Tensor, # w2_g_idx_sort_indices
|
||||
torch.Tensor | None, # w13_qzeros
|
||||
torch.Tensor | None, # w2_qzeros
|
||||
torch.Tensor | None, # w13_input_global_scale
|
||||
@@ -573,10 +541,9 @@ def _process_weights_marlin(
|
||||
Steps
|
||||
-----
|
||||
1. Optional FP8 preprocessing of packed weights / scales.
|
||||
2. Sort / reset g_idx tensors for act-order handling.
|
||||
3. Repack weights via ``gptq_marlin_moe_repack``.
|
||||
4. Permute scales (and optionally extract INT8 global scales).
|
||||
5. Permute bias tensors.
|
||||
2. Repack weights via ``gptq_marlin_moe_repack``.
|
||||
3. Permute scales (and optionally extract INT8 global scales).
|
||||
4. Permute bias tensors.
|
||||
"""
|
||||
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
|
||||
|
||||
@@ -584,8 +551,6 @@ def _process_weights_marlin(
|
||||
marlin_w2_qweight: torch.Tensor
|
||||
marlin_w13_scales: torch.Tensor
|
||||
marlin_w2_scales: torch.Tensor
|
||||
w13_g_idx_sort_indices: torch.Tensor | None = None
|
||||
w2_g_idx_sort_indices: torch.Tensor | None = None
|
||||
w13_input_global_scale: torch.Tensor | None = None
|
||||
w2_input_global_scale: torch.Tensor | None = None
|
||||
w13_bias_out: torch.Tensor | None = None
|
||||
@@ -606,13 +571,9 @@ def _process_weights_marlin(
|
||||
|
||||
# --- Pad the intermediate size to a valid Marlin thread tile ---
|
||||
# GPTQ packs along K: w13's N is in the (shard) columns, w2's N in the rows.
|
||||
# Act-order keeps the strict shape and is never padded.
|
||||
N = layer.intermediate_size_per_partition
|
||||
padded_N = marlin_moe_padded_intermediate(N, group_size)
|
||||
if padded_N != N:
|
||||
assert actorder != "group", (
|
||||
"Marlin MoE thread-tile padding is unsupported with act-order"
|
||||
)
|
||||
marlin_w13_qweight = _pad_w13_shard_cols(marlin_w13_qweight, N, padded_N)
|
||||
marlin_w2_qweight = _pad_rows(marlin_w2_qweight, padded_N // pack_factor)
|
||||
marlin_w13_scales = _pad_w13_shard_cols(marlin_w13_scales, N, padded_N)
|
||||
@@ -627,44 +588,9 @@ def _process_weights_marlin(
|
||||
if w13_bias is not None:
|
||||
w13_bias = _pad_w13_bias(w13_bias, N, padded_N)
|
||||
|
||||
# --- Process act_order (g_idx) ---
|
||||
if actorder == "group":
|
||||
num_experts = w13_g_idx.shape[0]
|
||||
w13_g_idx_sort_indices = torch.empty_like(w13_g_idx)
|
||||
w2_g_idx_sort_indices = torch.empty_like(w2_g_idx)
|
||||
w13_sorted_g_idx = torch.empty_like(w13_g_idx)
|
||||
w2_sorted_g_idx = torch.empty_like(w2_g_idx)
|
||||
for e in range(num_experts):
|
||||
w13_g_idx_sort_indices[e] = torch.argsort(w13_g_idx[e]).to(torch.int32)
|
||||
w2_g_idx_sort_indices[e] = torch.argsort(w2_g_idx[e]).to(torch.int32)
|
||||
w13_sorted_g_idx[e] = w13_g_idx[e][w13_g_idx_sort_indices[e]]
|
||||
w2_sorted_g_idx[e] = w2_g_idx[e][w2_g_idx_sort_indices[e]]
|
||||
w13_g_idx = w13_sorted_g_idx
|
||||
w2_g_idx = w2_sorted_g_idx
|
||||
else:
|
||||
num_experts = w13_g_idx.shape[0]
|
||||
device = w13_g_idx.device
|
||||
w13_g_idx = torch.nn.Parameter(
|
||||
torch.empty((num_experts, 0), dtype=torch.int32, device=device),
|
||||
requires_grad=False,
|
||||
)
|
||||
w2_g_idx = torch.nn.Parameter(
|
||||
torch.empty((num_experts, 0), dtype=torch.int32, device=device),
|
||||
requires_grad=False,
|
||||
)
|
||||
w13_g_idx_sort_indices = torch.nn.Parameter(
|
||||
torch.empty((num_experts, 0), dtype=torch.int32, device=device),
|
||||
requires_grad=False,
|
||||
)
|
||||
w2_g_idx_sort_indices = torch.nn.Parameter(
|
||||
torch.empty((num_experts, 0), dtype=torch.int32, device=device),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
# --- Repack weights ---
|
||||
marlin_w13_qweight = ops.gptq_marlin_moe_repack(
|
||||
marlin_w13_qweight,
|
||||
w13_g_idx_sort_indices,
|
||||
marlin_w13_qweight.shape[1] * pack_factor,
|
||||
marlin_w13_qweight.shape[2],
|
||||
num_bits,
|
||||
@@ -672,7 +598,6 @@ def _process_weights_marlin(
|
||||
)
|
||||
marlin_w2_qweight = ops.gptq_marlin_moe_repack(
|
||||
marlin_w2_qweight,
|
||||
w2_g_idx_sort_indices,
|
||||
marlin_w2_qweight.shape[1] * pack_factor,
|
||||
marlin_w2_qweight.shape[2],
|
||||
num_bits,
|
||||
@@ -734,10 +659,6 @@ def _process_weights_marlin(
|
||||
marlin_w2_qweight,
|
||||
marlin_w13_scales,
|
||||
marlin_w2_scales,
|
||||
w13_g_idx,
|
||||
w2_g_idx,
|
||||
w13_g_idx_sort_indices,
|
||||
w2_g_idx_sort_indices,
|
||||
w13_qzeros,
|
||||
w2_qzeros,
|
||||
w13_input_global_scale,
|
||||
@@ -766,10 +687,6 @@ def _process_awq_weights_marlin(
|
||||
torch.Tensor, # w2_qweight
|
||||
torch.Tensor, # w13_scales
|
||||
torch.Tensor, # w2_scales
|
||||
torch.Tensor | None, # w13_g_idx
|
||||
torch.Tensor | None, # w2_g_idx
|
||||
torch.Tensor | None, # w13_g_idx_sort_indices
|
||||
torch.Tensor | None, # w2_g_idx_sort_indices
|
||||
torch.Tensor | None, # w13_qzeros
|
||||
torch.Tensor | None, # w2_qzeros
|
||||
torch.Tensor | None, # w13_input_global_scale
|
||||
@@ -782,8 +699,6 @@ def _process_awq_weights_marlin(
|
||||
AWQ checkpoints use a different packing order than GPTQ, so they need
|
||||
AWQ-specific weight repacking and zero-point conversion before Marlin runs.
|
||||
"""
|
||||
num_experts = w13_qweight.shape[0]
|
||||
device = w13_qweight.device
|
||||
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
|
||||
w13_input_global_scale: torch.Tensor | None = None
|
||||
w2_input_global_scale: torch.Tensor | None = None
|
||||
@@ -823,18 +738,8 @@ def _process_awq_weights_marlin(
|
||||
if w13_bias is not None:
|
||||
w13_bias = _pad_w13_bias(w13_bias, N, padded_N)
|
||||
|
||||
w13_g_idx_sort_indices = torch.nn.Parameter(
|
||||
torch.empty((num_experts, 0), dtype=torch.int32, device=device),
|
||||
requires_grad=False,
|
||||
)
|
||||
w2_g_idx_sort_indices = torch.nn.Parameter(
|
||||
torch.empty((num_experts, 0), dtype=torch.int32, device=device),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
marlin_w13_qweight = ops.awq_marlin_moe_repack(
|
||||
w13_qweight,
|
||||
w13_g_idx_sort_indices,
|
||||
size_k=w13_qweight.shape[1],
|
||||
size_n=w13_qweight.shape[2] * pack_factor,
|
||||
num_bits=weight_bits,
|
||||
@@ -842,7 +747,6 @@ def _process_awq_weights_marlin(
|
||||
)
|
||||
marlin_w2_qweight = ops.awq_marlin_moe_repack(
|
||||
w2_qweight,
|
||||
w2_g_idx_sort_indices,
|
||||
size_k=w2_qweight.shape[1],
|
||||
size_n=w2_qweight.shape[2] * pack_factor,
|
||||
num_bits=weight_bits,
|
||||
@@ -898,10 +802,6 @@ def _process_awq_weights_marlin(
|
||||
marlin_w2_qweight,
|
||||
marlin_w13_scales,
|
||||
marlin_w2_scales,
|
||||
None,
|
||||
None,
|
||||
w13_g_idx_sort_indices,
|
||||
w2_g_idx_sort_indices,
|
||||
marlin_w13_qzeros,
|
||||
marlin_w2_qzeros,
|
||||
w13_input_global_scale,
|
||||
@@ -917,8 +817,6 @@ def _process_weights_cpu(
|
||||
w2: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
w13_g_idx: torch.Tensor | None = None,
|
||||
w2_g_idx: torch.Tensor | None = None,
|
||||
w13_qzeros: torch.Tensor | None = None,
|
||||
w2_qzeros: torch.Tensor | None = None,
|
||||
w13_bias: torch.Tensor | None = None,
|
||||
@@ -928,10 +826,6 @@ def _process_weights_cpu(
|
||||
torch.Tensor, # w2_qweight
|
||||
torch.Tensor, # w13_scales
|
||||
torch.Tensor, # w2_scales
|
||||
torch.Tensor | None, # w13_g_idx
|
||||
torch.Tensor | None, # w2_g_idx
|
||||
torch.Tensor | None, # w13_g_idx_sort_indices
|
||||
torch.Tensor | None, # w2_g_idx_sort_indices
|
||||
torch.Tensor | None, # w13_qzeros
|
||||
torch.Tensor | None, # w2_qzeros
|
||||
torch.Tensor | None, # w13_input_global_scale
|
||||
@@ -962,7 +856,7 @@ def _process_weights_cpu(
|
||||
if isinstance(quant_config, AutoGPTQConfig) and quant_config.desc_act:
|
||||
raise NotImplementedError(
|
||||
"CPU WNA16 MoE backend does not support GPTQ with "
|
||||
"desc_act=True. The fused MoE kernel has no g_idx "
|
||||
"desc_act=True. The fused MoE kernel has no activation "
|
||||
"reordering support."
|
||||
)
|
||||
cpu_quant_algo = ops.CPUQuantAlgo.GPTQ
|
||||
@@ -1009,10 +903,6 @@ def _process_weights_cpu(
|
||||
blocked_w2,
|
||||
blocked_s13,
|
||||
blocked_s2,
|
||||
w13_g_idx,
|
||||
w2_g_idx,
|
||||
None, # w13_g_idx_sort_indices (unused on CPU)
|
||||
None, # w2_g_idx_sort_indices (unused on CPU)
|
||||
blocked_z13,
|
||||
blocked_z2,
|
||||
None, # w13_input_global_scale
|
||||
@@ -1352,10 +1242,6 @@ def _process_weights_emulation_gptq(
|
||||
w2_bf16, # w2_qweight (now bf16, not int32)
|
||||
dummy, # w13_scales (unused; nulled out in Int4EmulationTritonExperts)
|
||||
dummy, # w2_scales (unused)
|
||||
None, # w13_g_idx
|
||||
None, # w2_g_idx
|
||||
None, # w13_g_idx_sort_indices
|
||||
None, # w2_g_idx_sort_indices
|
||||
None, # w13_qzeros
|
||||
None, # w2_qzeros
|
||||
None, # w13_input_global_scale
|
||||
@@ -1413,10 +1299,6 @@ def _process_weights_emulation_awq(
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
@@ -1429,8 +1311,6 @@ def convert_to_wna16_moe_kernel_format(
|
||||
w2: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
w13_g_idx: torch.Tensor | None = None,
|
||||
w2_g_idx: torch.Tensor | None = None,
|
||||
w13_qzeros: torch.Tensor | None = None,
|
||||
w2_qzeros: torch.Tensor | None = None,
|
||||
w13_bias: torch.Tensor | None = None,
|
||||
@@ -1441,10 +1321,6 @@ def convert_to_wna16_moe_kernel_format(
|
||||
torch.Tensor, # w2_qweight
|
||||
torch.Tensor, # w13_scales
|
||||
torch.Tensor, # w2_scales
|
||||
torch.Tensor | None, # w13_g_idx
|
||||
torch.Tensor | None, # w2_g_idx
|
||||
torch.Tensor | None, # w13_g_idx_sort_indices
|
||||
torch.Tensor | None, # w2_g_idx_sort_indices
|
||||
torch.Tensor | None, # w13_qzeros
|
||||
torch.Tensor | None, # w2_qzeros
|
||||
torch.Tensor | None, # w13_input_global_scale
|
||||
@@ -1509,14 +1385,12 @@ def convert_to_wna16_moe_kernel_format(
|
||||
num_bits = quant_config.quant_type.size_bits
|
||||
pack_factor = quant_config.pack_factor
|
||||
group_size = quant_config.group_size
|
||||
actorder = "group" if quant_config.desc_act else None
|
||||
elif isinstance(quant_config, QuantizationArgs):
|
||||
num_bits = quant_config.num_bits
|
||||
pack_factor = 32 // quant_config.num_bits
|
||||
group_size = (
|
||||
-1 if quant_config.group_size is None else quant_config.group_size
|
||||
)
|
||||
actorder = quant_config.actorder
|
||||
else:
|
||||
raise TypeError(
|
||||
"Marlin WNA16 MoE backend requires AutoGPTQConfig, AutoAWQConfig or "
|
||||
@@ -1543,22 +1417,16 @@ def convert_to_wna16_moe_kernel_format(
|
||||
w2_bias,
|
||||
)
|
||||
else:
|
||||
if w13_g_idx is None or w2_g_idx is None:
|
||||
raise ValueError("GPTQ Marlin MoE requires g_idx tensors.")
|
||||
|
||||
return _process_weights_marlin(
|
||||
layer,
|
||||
input_dtype,
|
||||
num_bits,
|
||||
pack_factor,
|
||||
group_size,
|
||||
actorder,
|
||||
w13,
|
||||
w2,
|
||||
w13_scale,
|
||||
w2_scale,
|
||||
w13_g_idx,
|
||||
w2_g_idx,
|
||||
w13_qzeros,
|
||||
w2_qzeros,
|
||||
w13_bias,
|
||||
@@ -1571,8 +1439,6 @@ def convert_to_wna16_moe_kernel_format(
|
||||
w2,
|
||||
w13_scale,
|
||||
w2_scale,
|
||||
w13_g_idx,
|
||||
w2_g_idx,
|
||||
w13_qzeros,
|
||||
w2_qzeros,
|
||||
w13_bias,
|
||||
@@ -1584,8 +1450,6 @@ def convert_to_wna16_moe_kernel_format(
|
||||
w2,
|
||||
w13_scale,
|
||||
w2_scale,
|
||||
w13_g_idx,
|
||||
w2_g_idx,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
)
|
||||
@@ -1608,16 +1472,11 @@ def convert_to_wna16_moe_kernel_format(
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
)
|
||||
empty = torch.empty((0,), dtype=torch.int32, device=w13.device)
|
||||
return (
|
||||
w13_xpu,
|
||||
w2_xpu,
|
||||
w13_scale_xpu,
|
||||
w2_scale_xpu,
|
||||
empty, # w13_g_idx
|
||||
empty, # w2_g_idx
|
||||
empty, # w13_g_idx_sort_indices
|
||||
empty, # w2_g_idx_sort_indices
|
||||
None, # w13_qzeros — sym int4 on XPU has none; kernel does uint4b8→s4
|
||||
None, # w2_qzeros
|
||||
None, # w13_input_global_scale
|
||||
@@ -1712,10 +1571,6 @@ def convert_to_wna16_moe_kernel_format(
|
||||
w2_uint8,
|
||||
w13_scale,
|
||||
w2_scale,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
w13_qzeros,
|
||||
w2_qzeros,
|
||||
None,
|
||||
|
||||
@@ -168,12 +168,6 @@ class RoutedExperts(PluggableLayer):
|
||||
"global_num_experts": moe_config.num_experts,
|
||||
}
|
||||
|
||||
# need full intermediate size pre-sharding for WNA16 act order
|
||||
if self._needs_intermediate_size_param(self.quant_method):
|
||||
moe_quant_params["intermediate_size_full"] = (
|
||||
self.moe_config.intermediate_size
|
||||
)
|
||||
|
||||
self.quant_method.create_weights(layer=self, **moe_quant_params)
|
||||
|
||||
self.lora_base_layer_prefix = ""
|
||||
@@ -206,14 +200,6 @@ class RoutedExperts(PluggableLayer):
|
||||
assert isinstance(quant_method, FusedMoEMethodBase)
|
||||
return quant_method
|
||||
|
||||
# TODO(bnell): make this a method on quant_method
|
||||
def _needs_intermediate_size_param(self, quant_method: FusedMoEMethodBase) -> bool:
|
||||
return quant_method.__class__.__name__ in (
|
||||
"AutoGPTQMoEMethod",
|
||||
"CompressedTensorsWNA16MoEMethod",
|
||||
"CompressedTensorsW4A16FlydslMoEMethod",
|
||||
)
|
||||
|
||||
def _ensure_moe_quant_config_init(self):
|
||||
if self.quant_method.moe_quant_config is None:
|
||||
# Note: the moe_quant_config can't be constructed until after
|
||||
@@ -349,7 +335,6 @@ class RoutedExperts(PluggableLayer):
|
||||
shard_id: str,
|
||||
loaded_weight: torch.Tensor,
|
||||
tp_rank: int,
|
||||
load_full_w2: bool = False,
|
||||
):
|
||||
"""
|
||||
Load grouped weight scales for group quantization or model weights
|
||||
@@ -360,17 +345,13 @@ class RoutedExperts(PluggableLayer):
|
||||
shard_id: either w1, w2, or w3
|
||||
loaded_weight: checkpoint weight to load into the param
|
||||
tp_rank: tensor parallel rank
|
||||
load_full_w2: whether or not the w2 loaded should be sharded.
|
||||
"""
|
||||
if shard_id == "w2":
|
||||
# In the case where we have actorder/g_idx, we do not partition the
|
||||
# w2 scales, as indicated by `load_full` argument, for all tp cases
|
||||
self._load_w2(
|
||||
shard_dim=shard_dim,
|
||||
loaded_weight=loaded_weight,
|
||||
expert_data=expert_data,
|
||||
tp_rank=tp_rank,
|
||||
load_full=load_full_w2,
|
||||
)
|
||||
elif shard_id in ("w1", "w3"):
|
||||
self._load_w13(
|
||||
@@ -532,13 +513,11 @@ class RoutedExperts(PluggableLayer):
|
||||
shard_dim: int,
|
||||
loaded_weight: torch.Tensor,
|
||||
tp_rank: int,
|
||||
load_full: bool = False,
|
||||
):
|
||||
# Index the loaded weight for tp sharding.
|
||||
# down_proj: "RowParallel" so tp sharding on input_dim
|
||||
# Only narrow if the loaded_weight is not a scalar (0-dim tensor)
|
||||
# and we're not loading the full weight
|
||||
if not load_full and loaded_weight.ndim > 0:
|
||||
# Only narrow if the loaded_weight is not a scalar (0-dim tensor).
|
||||
if loaded_weight.ndim > 0:
|
||||
# Same padding fix as _load_w13: use unpadded per-rank size.
|
||||
tp_size = self.moe_config.moe_parallel_config.tp_size
|
||||
loaded_per_rank = loaded_weight.shape[shard_dim] // tp_size
|
||||
@@ -571,25 +550,6 @@ class RoutedExperts(PluggableLayer):
|
||||
# _to_scalar's reshape(()) would reject the size-2 weight_shape.
|
||||
param_data[expert_id] = loaded_weight
|
||||
|
||||
def _load_g_idx(
|
||||
self,
|
||||
shard_id: str,
|
||||
expert_data: torch.Tensor,
|
||||
shard_dim: int,
|
||||
loaded_weight: torch.Tensor,
|
||||
tp_rank: int,
|
||||
):
|
||||
if shard_id == "w2":
|
||||
self._load_w2(
|
||||
shard_dim=shard_dim,
|
||||
loaded_weight=loaded_weight,
|
||||
expert_data=expert_data,
|
||||
tp_rank=tp_rank,
|
||||
)
|
||||
else:
|
||||
assert shard_id in ("w1", "w3")
|
||||
expert_data.copy_(loaded_weight)
|
||||
|
||||
@overload
|
||||
def weight_loader(
|
||||
self,
|
||||
@@ -739,17 +699,6 @@ class RoutedExperts(PluggableLayer):
|
||||
)
|
||||
return True if return_success else None
|
||||
|
||||
# Case g_idx
|
||||
if "g_idx" in weight_name:
|
||||
self._load_g_idx(
|
||||
shard_dim=0,
|
||||
shard_id=shard_id,
|
||||
loaded_weight=loaded_weight,
|
||||
expert_data=expert_data,
|
||||
tp_rank=self.moe_config.tp_rank,
|
||||
)
|
||||
return True if return_success else None
|
||||
|
||||
# TODO @dsikka: ModelOpt should follow the proper MoE loading pattern
|
||||
if "ModelOpt" in quant_method_name:
|
||||
# Determine per-tensor weight scale patterns based on variant
|
||||
@@ -853,7 +802,6 @@ class RoutedExperts(PluggableLayer):
|
||||
loaded_weight=loaded_weight,
|
||||
expert_data=expert_data,
|
||||
tp_rank=self.moe_config.tp_rank,
|
||||
load_full_w2=getattr(param, "load_full_w2", False),
|
||||
)
|
||||
elif quant_method == FusedMoeWeightScaleSupported.TENSOR.value:
|
||||
self._load_per_tensor_weight_scale(
|
||||
|
||||
@@ -439,7 +439,6 @@ class AutoAWQMarlinLinearMethod(LinearMethodBase):
|
||||
act_type=params_dtype if self.input_dtype is None else self.input_dtype,
|
||||
group_size=self.quant_config.group_size,
|
||||
zero_points=self.quant_config.zero_point,
|
||||
has_g_idx=False,
|
||||
)
|
||||
|
||||
kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config)
|
||||
@@ -558,11 +557,6 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
|
||||
}
|
||||
)
|
||||
|
||||
intermediate_size_full = extra_weight_attrs.pop(
|
||||
"intermediate_size_full", intermediate_size_per_partition
|
||||
)
|
||||
self.is_k_full = intermediate_size_per_partition == intermediate_size_full
|
||||
|
||||
w13_qweight = Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
@@ -672,10 +666,6 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
|
||||
w2,
|
||||
w13_scale,
|
||||
w2_scale,
|
||||
w13_g_idx,
|
||||
w2_g_idx,
|
||||
w13_g_idx_sort_indices,
|
||||
w2_g_idx_sort_indices,
|
||||
w13_qzeros,
|
||||
w2_qzeros,
|
||||
w13_input_global_scale,
|
||||
@@ -696,14 +686,6 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
|
||||
|
||||
replace_parameter(layer, "w13_scales", w13_scale)
|
||||
replace_parameter(layer, "w2_scales", w2_scale)
|
||||
_replace_or_register_parameter(
|
||||
layer, "w13_g_idx_sort_indices", w13_g_idx_sort_indices
|
||||
)
|
||||
_replace_or_register_parameter(
|
||||
layer, "w2_g_idx_sort_indices", w2_g_idx_sort_indices
|
||||
)
|
||||
_replace_or_register_parameter(layer, "w13_g_idx", w13_g_idx)
|
||||
_replace_or_register_parameter(layer, "w2_g_idx", w2_g_idx)
|
||||
_replace_or_register_parameter(layer, "w13_qzeros", w13_qzeros)
|
||||
_replace_or_register_parameter(layer, "w2_qzeros", w2_qzeros)
|
||||
_replace_or_register_parameter(
|
||||
@@ -726,11 +708,6 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.wna16_moe_backend,
|
||||
is_k_full=self.is_k_full,
|
||||
w13_g_idx=getattr(layer, "w13_g_idx", None),
|
||||
w2_g_idx=getattr(layer, "w2_g_idx", None),
|
||||
w13_g_idx_sort_indices=getattr(layer, "w13_g_idx_sort_indices", None),
|
||||
w2_g_idx_sort_indices=getattr(layer, "w2_g_idx_sort_indices", None),
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
|
||||
|
||||
@@ -40,6 +40,7 @@ from vllm.model_executor.layers.quantization.utils import replace_parameter
|
||||
from vllm.model_executor.layers.quantization.utils.gptq_utils import (
|
||||
get_dynamic_override,
|
||||
get_linear_quant_method,
|
||||
normalize_and_validate_gptq_desc_act,
|
||||
override_config,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
@@ -59,7 +60,6 @@ from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
PackedColumnParameter,
|
||||
PackedvLLMParameter,
|
||||
RowvLLMParameter,
|
||||
)
|
||||
from vllm.scalar_type import scalar_types
|
||||
from vllm.transformers_utils.config import get_safetensors_params_metadata
|
||||
@@ -115,10 +115,7 @@ class AutoGPTQConfig(QuantizationConfig):
|
||||
modules_in_block_to_quantize: list[str] | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if desc_act and group_size == -1:
|
||||
# In this case, act_order == True is the same as act_order == False
|
||||
# (since we have only one group per output channel)
|
||||
desc_act = False
|
||||
desc_act = normalize_and_validate_gptq_desc_act(desc_act, group_size, dynamic)
|
||||
|
||||
# GPTQModel use `dynamic` config property to allow per module
|
||||
# quantization config so each module can be individually optimized.
|
||||
@@ -150,7 +147,7 @@ class AutoGPTQConfig(QuantizationConfig):
|
||||
|
||||
self.pack_factor = 32 // weight_bits # packed into int32
|
||||
self.group_size = group_size
|
||||
self.desc_act = desc_act
|
||||
self.desc_act: bool = desc_act
|
||||
self.lm_head_quantized = lm_head_quantized
|
||||
self.full_config = full_config
|
||||
|
||||
@@ -244,7 +241,7 @@ class AutoGPTQConfig(QuantizationConfig):
|
||||
from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
|
||||
|
||||
if not check_moe_marlin_supports_layer(
|
||||
layer, self.group_size, allow_tile_padding=not self.desc_act
|
||||
layer, self.group_size, allow_tile_padding=True
|
||||
):
|
||||
logger.warning_once(
|
||||
f"Layer '{prefix}' is not supported by GPTQMoeMarlin. "
|
||||
@@ -348,7 +345,6 @@ class AutoGPTQLinearMethod(LinearMethodBase):
|
||||
act_type=params_dtype if input_dtype is None else input_dtype,
|
||||
group_size=self.quant_config.group_size,
|
||||
zero_points=False,
|
||||
has_g_idx=self.quant_config.desc_act,
|
||||
)
|
||||
|
||||
kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config)
|
||||
@@ -365,7 +361,7 @@ class AutoGPTQLinearMethod(LinearMethodBase):
|
||||
|
||||
# Determine sharding
|
||||
if marlin_repeat_scales_on_all_ranks(
|
||||
self.quant_config.desc_act, self.quant_config.group_size, is_row_parallel
|
||||
self.quant_config.group_size, is_row_parallel
|
||||
):
|
||||
# By setting scale_dim == None, weight_loader will
|
||||
# repeat the scales on each GPU in TP>1 case.
|
||||
@@ -391,16 +387,6 @@ class AutoGPTQLinearMethod(LinearMethodBase):
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
|
||||
# Activation order
|
||||
g_idx = RowvLLMParameter(
|
||||
data=torch.empty(
|
||||
input_size_per_partition,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
input_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
|
||||
qzeros_args = {
|
||||
"data": torch.empty(
|
||||
scales_and_zp_size,
|
||||
@@ -440,7 +426,6 @@ class AutoGPTQLinearMethod(LinearMethodBase):
|
||||
)
|
||||
|
||||
layer.register_parameter("qweight", qweight)
|
||||
layer.register_parameter("g_idx", g_idx)
|
||||
layer.register_parameter("scales", scales)
|
||||
layer.register_parameter("qzeros", qzeros)
|
||||
|
||||
@@ -449,7 +434,6 @@ class AutoGPTQLinearMethod(LinearMethodBase):
|
||||
w_q_param_name="qweight",
|
||||
w_s_param_name="scales",
|
||||
w_zp_param_name="qzeros",
|
||||
w_gidx_param_name="g_idx",
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
@@ -492,7 +476,7 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
quant_config=self.quant_config,
|
||||
may_have_zp=not self.quant_config.is_sym,
|
||||
may_have_bias=True,
|
||||
allow_tile_padding=not self.quant_config.desc_act,
|
||||
allow_tile_padding=True,
|
||||
)
|
||||
|
||||
def create_weights(
|
||||
@@ -512,19 +496,9 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
"W8A8-INT8 is not supported by marlin kernel."
|
||||
)
|
||||
|
||||
intermediate_size_full = extra_weight_attrs.pop("intermediate_size_full")
|
||||
|
||||
self.is_k_full = (not self.quant_config.desc_act) or (
|
||||
intermediate_size_per_partition == intermediate_size_full
|
||||
)
|
||||
|
||||
if self.quant_config.group_size != -1:
|
||||
scales_size13 = hidden_size // self.quant_config.group_size
|
||||
w2_scales_size = (
|
||||
intermediate_size_full
|
||||
if self.quant_config.desc_act
|
||||
else intermediate_size_per_partition
|
||||
)
|
||||
w2_scales_size = intermediate_size_per_partition
|
||||
scales_size2 = w2_scales_size // self.quant_config.group_size
|
||||
strategy = FusedMoeWeightScaleSupported.GROUP.value
|
||||
else:
|
||||
@@ -579,8 +553,6 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
layer.register_parameter("w2_scales", w2_scales)
|
||||
set_weight_attrs(w2_scales, extra_weight_attrs)
|
||||
# don't shard the w2 scales when running act order
|
||||
set_weight_attrs(w2_scales, {"load_full_w2": self.quant_config.desc_act})
|
||||
# up_proj zero points
|
||||
w13_qzeros = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
@@ -607,48 +579,6 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
layer.register_parameter("w2_qzeros", w2_qzeros)
|
||||
set_weight_attrs(w2_qzeros, extra_weight_attrs)
|
||||
# don't shard the w2 scales when running act order
|
||||
set_weight_attrs(w2_qzeros, {"load_full_w2": self.quant_config.desc_act})
|
||||
w13_g_idx = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_g_idx", w13_g_idx)
|
||||
set_weight_attrs(w13_g_idx, extra_weight_attrs)
|
||||
w2_g_idx = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
intermediate_size_per_partition,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_g_idx", w2_g_idx)
|
||||
set_weight_attrs(w2_g_idx, extra_weight_attrs)
|
||||
w13_g_idx_sort_indices = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_g_idx_sort_indices", w13_g_idx_sort_indices)
|
||||
set_weight_attrs(w13_g_idx_sort_indices, extra_weight_attrs)
|
||||
w2_g_idx_sort_indices = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
intermediate_size_per_partition,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_g_idx_sort_indices", w2_g_idx_sort_indices)
|
||||
set_weight_attrs(w2_g_idx_sort_indices, extra_weight_attrs)
|
||||
|
||||
# Some GPTQ checkpoints contain expert biases even when the model
|
||||
# architecture does not declare them. Zero initialization keeps
|
||||
@@ -712,8 +642,6 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
w2=layer.w2_qweight,
|
||||
w13_scale=layer.w13_scales,
|
||||
w2_scale=layer.w2_scales,
|
||||
w13_g_idx=layer.w13_g_idx,
|
||||
w2_g_idx=layer.w2_g_idx,
|
||||
w13_bias=w13_bias,
|
||||
w2_bias=w2_bias,
|
||||
w13_qzeros=getattr(layer, "w13_qzeros", None),
|
||||
@@ -730,10 +658,6 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
w2,
|
||||
w13_scale,
|
||||
w2_scale,
|
||||
w13_g_idx,
|
||||
w2_g_idx,
|
||||
w13_g_idx_sort_indices,
|
||||
w2_g_idx_sort_indices,
|
||||
w13_qzeros,
|
||||
w2_qzeros,
|
||||
w13_input_global_scale,
|
||||
@@ -746,10 +670,6 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
replace_parameter(layer, "w2_qweight", w2)
|
||||
replace_parameter(layer, "w13_scales", w13_scale)
|
||||
replace_parameter(layer, "w2_scales", w2_scale)
|
||||
replace_parameter(layer, "w13_g_idx", w13_g_idx)
|
||||
replace_parameter(layer, "w2_g_idx", w2_g_idx)
|
||||
replace_parameter(layer, "w13_g_idx_sort_indices", w13_g_idx_sort_indices)
|
||||
replace_parameter(layer, "w2_g_idx_sort_indices", w2_g_idx_sort_indices)
|
||||
replace_or_register("w13_input_global_scale", w13_input_global_scale)
|
||||
replace_or_register("w2_input_global_scale", w2_input_global_scale)
|
||||
replace_or_register("w13_bias", w13_bias)
|
||||
@@ -772,11 +692,6 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.wna16_moe_backend,
|
||||
is_k_full=self.is_k_full,
|
||||
w13_g_idx=getattr(layer, "w13_g_idx", None),
|
||||
w2_g_idx=getattr(layer, "w2_g_idx", None),
|
||||
w13_g_idx_sort_indices=getattr(layer, "w13_g_idx_sort_indices", None),
|
||||
w2_g_idx_sort_indices=getattr(layer, "w2_g_idx_sort_indices", None),
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
|
||||
|
||||
@@ -238,6 +238,13 @@ class QuantizationConfig(ABC):
|
||||
}
|
||||
return WeightsMapper(orig_to_new_regex=orig_to_new_regex)
|
||||
|
||||
@staticmethod
|
||||
def get_checkpoint_weight_mapper() -> "WeightsMapper":
|
||||
"""Discard activation-order metadata unused by supported kernels."""
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
|
||||
return WeightsMapper(orig_to_new_suffix={".g_idx": None})
|
||||
|
||||
def apply_vllm_mapper( # noqa: B027
|
||||
self, hf_to_vllm_mapper: "WeightsMapper"
|
||||
):
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any, Literal, cast
|
||||
import torch
|
||||
from compressed_tensors.config import CompressionFormat, SparsityCompressionConfig
|
||||
from compressed_tensors.quantization import (
|
||||
ActivationOrdering,
|
||||
QuantizationArgs,
|
||||
QuantizationStrategy,
|
||||
QuantizationType,
|
||||
@@ -325,9 +326,20 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
targets = quant_config.get("targets")
|
||||
for target in targets:
|
||||
target_scheme_map[target] = {}
|
||||
target_scheme_map[target]["weights"] = QuantizationArgs.model_validate(
|
||||
weight_quant = QuantizationArgs.model_validate(
|
||||
quant_config.get("weights")
|
||||
)
|
||||
if weight_quant.actorder in (
|
||||
ActivationOrdering.GROUP.value,
|
||||
ActivationOrdering.DYNAMIC.value,
|
||||
):
|
||||
raise ValueError(
|
||||
"Compressed-tensors group/dynamic activation ordering "
|
||||
f"(actorder={weight_quant.actorder!r}) is no longer supported. "
|
||||
"Use a checkpoint with static/weight activation ordering or "
|
||||
"without activation ordering."
|
||||
)
|
||||
target_scheme_map[target]["weights"] = weight_quant
|
||||
|
||||
target_scheme_map[target]["input_activations"] = None
|
||||
target_scheme_map[target]["format"] = quant_config.get("format")
|
||||
@@ -754,7 +766,6 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
strategy=weight_quant.strategy,
|
||||
symmetric=weight_quant.symmetric,
|
||||
group_size=weight_quant.group_size,
|
||||
actorder=weight_quant.actorder,
|
||||
)
|
||||
|
||||
# Must come before the WNA16 check; standard 4/8-bit weight-only (no
|
||||
@@ -803,7 +814,6 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
strategy=weight_quant.strategy,
|
||||
symmetric=weight_quant.symmetric,
|
||||
group_size=weight_quant.group_size,
|
||||
actorder=weight_quant.actorder,
|
||||
layer_name=layer_name,
|
||||
)
|
||||
|
||||
|
||||
-47
@@ -104,8 +104,6 @@ class CompressedTensorsW4A16FlydslMoEMethod(CompressedTensorsMoEMethod):
|
||||
assert weight_quant.strategy == "group"
|
||||
assert weight_quant.group_size == 32
|
||||
self.group_size = weight_quant.group_size
|
||||
# grouped actorder isn't supported by this kernel
|
||||
assert weight_quant.actorder != "group"
|
||||
assert weight_quant.symmetric, (
|
||||
"Only symmetric quantization is supported for MoE. "
|
||||
"Try --moe-backend emulation."
|
||||
@@ -180,7 +178,6 @@ class CompressedTensorsW4A16FlydslMoEMethod(CompressedTensorsMoEMethod):
|
||||
)
|
||||
layer.register_parameter("w2_weight_scale", w2_scale)
|
||||
set_weight_attrs(w2_scale, extra_weight_attrs)
|
||||
set_weight_attrs(w2_scale, {"load_full_w2": False})
|
||||
|
||||
w2_weight_shape = torch.nn.Parameter(
|
||||
torch.empty(num_experts, 2), requires_grad=False
|
||||
@@ -194,50 +191,6 @@ class CompressedTensorsW4A16FlydslMoEMethod(CompressedTensorsMoEMethod):
|
||||
layer.register_parameter("w13_weight_shape", w13_weight_shape)
|
||||
set_weight_attrs(w13_weight_shape, extra_weight_attrs)
|
||||
|
||||
w13_g_idx = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight_g_idx", w13_g_idx)
|
||||
set_weight_attrs(w13_g_idx, extra_weight_attrs)
|
||||
|
||||
w2_g_idx = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
intermediate_size_per_partition,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight_g_idx", w2_g_idx)
|
||||
set_weight_attrs(w2_g_idx, extra_weight_attrs)
|
||||
|
||||
w13_g_idx_sort_indices = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_g_idx_sort_indices", w13_g_idx_sort_indices)
|
||||
set_weight_attrs(w13_g_idx_sort_indices, extra_weight_attrs)
|
||||
|
||||
w2_g_idx_sort_indices = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
intermediate_size_per_partition,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_g_idx_sort_indices", w2_g_idx_sort_indices)
|
||||
set_weight_attrs(w2_g_idx_sort_indices, extra_weight_attrs)
|
||||
|
||||
layer.a13_scale = None
|
||||
layer.a2_scale = None
|
||||
|
||||
|
||||
-1
@@ -47,7 +47,6 @@ class CompressedTensorsW4A8Fp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
assert self.weight_quant.symmetric, (
|
||||
"Only symmetric quantization is supported for W4A8 MoE"
|
||||
)
|
||||
assert self.weight_quant.actorder != "group"
|
||||
assert self.group_size == 128, "Only group size 128 supported for W4A8 MoE"
|
||||
|
||||
self.w4a8_backend, self.experts_cls = select_w4a8_moe_backend(
|
||||
|
||||
+5
-126
@@ -3,13 +3,10 @@
|
||||
|
||||
import math
|
||||
from fractions import Fraction
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from compressed_tensors.quantization import (
|
||||
ActivationOrdering,
|
||||
QuantizationArgs,
|
||||
QuantizationStrategy,
|
||||
)
|
||||
|
||||
from vllm.logger import init_logger
|
||||
@@ -75,7 +72,6 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
self.group_size = (
|
||||
-1 if weight_quant.group_size is None else weight_quant.group_size
|
||||
)
|
||||
self.actorder = weight_quant.actorder
|
||||
|
||||
# Extract quant_type and create weight key for oracle selection
|
||||
self.quant_type = (
|
||||
@@ -104,11 +100,6 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
|
||||
weight_key = QuantKey(self.quant_type, scale, symmetric=self.symmetric)
|
||||
|
||||
is_actorder = self.strategy == QuantizationStrategy.GROUP and self.actorder in (
|
||||
ActivationOrdering.GROUP,
|
||||
ActivationOrdering.DYNAMIC,
|
||||
)
|
||||
|
||||
# Select WNA16 MoE backend via oracle.
|
||||
self.wna16_backend, self.experts_cls = select_wna16_moe_backend(
|
||||
config=self.moe,
|
||||
@@ -116,7 +107,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
quant_config=self.weight_quant,
|
||||
may_have_zp=not self.symmetric,
|
||||
may_have_bias=False,
|
||||
allow_tile_padding=not is_actorder,
|
||||
allow_tile_padding=True,
|
||||
)
|
||||
|
||||
self.is_marlin = self.wna16_backend in [
|
||||
@@ -130,14 +121,12 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
|
||||
if self.is_marlin:
|
||||
assert check_moe_marlin_supports_config(
|
||||
self.moe, self.group_size, allow_tile_padding=not is_actorder
|
||||
self.moe, self.group_size, allow_tile_padding=True
|
||||
)
|
||||
self.input_dtype = get_marlin_input_dtype(layer_name)
|
||||
else:
|
||||
# channelwise is not supported by this kernel
|
||||
assert weight_quant.strategy == "group"
|
||||
# grouped actorder isn't supported by this kernel
|
||||
assert weight_quant.actorder != "group"
|
||||
|
||||
# Non-Marlin WNA16 always uses bf16/fp16 inputs
|
||||
self.input_dtype = torch.bfloat16
|
||||
@@ -230,29 +219,6 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
assert shape[1] is not None and shape[2] is not None
|
||||
return shape[0], shape[1], shape[2]
|
||||
|
||||
@staticmethod
|
||||
def _w2_scale_sharding(
|
||||
actorder,
|
||||
group_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
intermediate_size_full: int,
|
||||
) -> tuple[bool, int, bool]:
|
||||
"""Decide how to shard w2 group scales across TP for WNA16 Marlin MoE.
|
||||
|
||||
Only ``actorder="group"`` permutes activations by ``g_idx`` at runtime
|
||||
and therefore needs the full-K (unsharded) w2 scales plus ``is_k_full``.
|
||||
``actorder="weight"``/``"static"`` (and ``None``) reorder weights at
|
||||
quantization time, so scales shard normally per TP rank.
|
||||
"""
|
||||
load_full_w2 = (actorder == "group") and group_size != -1
|
||||
w2_scales_size = (
|
||||
intermediate_size_full if load_full_w2 else intermediate_size_per_partition
|
||||
)
|
||||
is_k_full = (actorder != "group") or (
|
||||
intermediate_size_per_partition == intermediate_size_full
|
||||
)
|
||||
return load_full_w2, w2_scales_size, is_k_full
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
@@ -262,8 +228,6 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
intermediate_size_full = extra_weight_attrs.pop("intermediate_size_full")
|
||||
|
||||
# Will transpose the loaded weight along the
|
||||
# intermediate and hidden dim sizes. Will
|
||||
# shard for TP along the transposed dims
|
||||
@@ -301,13 +265,6 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
layer.register_parameter("w2_weight_packed", w2_weight)
|
||||
set_weight_attrs(w2_weight, extra_weight_attrs)
|
||||
|
||||
load_full_w2, w2_scales_size, self.is_k_full = self._w2_scale_sharding(
|
||||
self.actorder,
|
||||
self.group_size,
|
||||
intermediate_size_per_partition,
|
||||
intermediate_size_full,
|
||||
)
|
||||
|
||||
if self.strategy == "channel":
|
||||
num_groups_w2 = num_groups_w13 = 1
|
||||
self.group_size = -1
|
||||
@@ -318,10 +275,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
f"({hidden_size}) to be divisible by group_size "
|
||||
f"({self.group_size})."
|
||||
)
|
||||
if (
|
||||
not load_full_w2
|
||||
and intermediate_size_per_partition % self.group_size != 0
|
||||
):
|
||||
if intermediate_size_per_partition % self.group_size != 0:
|
||||
raise ValueError(
|
||||
"CompressedTensors WNA16 MoE with static group "
|
||||
"scales requires the MoE intermediate size per "
|
||||
@@ -331,7 +285,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
"otherwise cross TP shard boundaries; use a compatible TP "
|
||||
"size or enable expert parallelism."
|
||||
)
|
||||
num_groups_w2 = w2_scales_size // self.group_size
|
||||
num_groups_w2 = intermediate_size_per_partition // self.group_size
|
||||
num_groups_w13 = hidden_size // self.group_size
|
||||
|
||||
if not self.symmetric:
|
||||
@@ -389,7 +343,6 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
)
|
||||
layer.register_parameter("w2_weight_scale", w2_scale)
|
||||
set_weight_attrs(w2_scale, extra_weight_attrs)
|
||||
set_weight_attrs(w2_scale, {"load_full_w2": load_full_w2})
|
||||
|
||||
if not self.symmetric:
|
||||
w13_zp = torch.nn.Parameter(
|
||||
@@ -436,50 +389,6 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
layer.register_parameter("w13_weight_shape", w13_weight_shape)
|
||||
set_weight_attrs(w13_weight_shape, extra_weight_attrs)
|
||||
|
||||
w13_g_idx = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight_g_idx", w13_g_idx)
|
||||
set_weight_attrs(w13_g_idx, extra_weight_attrs)
|
||||
|
||||
w2_g_idx = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
intermediate_size_per_partition,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight_g_idx", w2_g_idx)
|
||||
set_weight_attrs(w2_g_idx, extra_weight_attrs)
|
||||
|
||||
w13_g_idx_sort_indices = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_g_idx_sort_indices", w13_g_idx_sort_indices)
|
||||
set_weight_attrs(w13_g_idx_sort_indices, extra_weight_attrs)
|
||||
|
||||
w2_g_idx_sort_indices = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
intermediate_size_per_partition,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_g_idx_sort_indices", w2_g_idx_sort_indices)
|
||||
set_weight_attrs(w2_g_idx_sort_indices, extra_weight_attrs)
|
||||
|
||||
layer.a13_scale = None
|
||||
layer.a2_scale = None
|
||||
|
||||
@@ -488,24 +397,12 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
assert self.moe_quant_config is not None
|
||||
|
||||
# Add Marlin-specific arguments
|
||||
marlin_args: dict[str, Any] = {}
|
||||
if self.is_marlin:
|
||||
marlin_args = {
|
||||
"w13_g_idx": layer.w13_weight_g_idx,
|
||||
"w2_g_idx": layer.w2_weight_g_idx,
|
||||
"w13_g_idx_sort_indices": layer.w13_g_idx_sort_indices,
|
||||
"w2_g_idx_sort_indices": layer.w2_g_idx_sort_indices,
|
||||
"is_k_full": self.is_k_full,
|
||||
}
|
||||
|
||||
self.moe_kernel = make_wna16_moe_kernel(
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.wna16_backend,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
**marlin_args,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
@@ -519,8 +416,6 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
w2=layer.w2_weight_packed,
|
||||
w13_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
w13_g_idx=layer.w13_weight_g_idx,
|
||||
w2_g_idx=layer.w2_weight_g_idx,
|
||||
w13_qzeros=getattr(layer, "w13_weight_zero_point", None),
|
||||
w2_qzeros=getattr(layer, "w2_weight_zero_point", None),
|
||||
)
|
||||
@@ -534,10 +429,6 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
w2_qweight,
|
||||
w13_scales,
|
||||
w2_scales,
|
||||
w13_g_idx_processed,
|
||||
w2_g_idx_processed,
|
||||
w13_g_idx_sort_indices,
|
||||
w2_g_idx_sort_indices,
|
||||
w13_qzeros,
|
||||
w2_qzeros,
|
||||
w13_input_global_scale,
|
||||
@@ -561,19 +452,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
replace_parameter(layer, "w13_weight_zero_point", w13_qzeros)
|
||||
replace_parameter(layer, "w2_weight_zero_point", w2_qzeros)
|
||||
|
||||
# Marlin-specific parameters (not needed for Flashinfer)
|
||||
if self.is_marlin:
|
||||
if w13_g_idx_processed is not None:
|
||||
replace_parameter(layer, "w13_weight_g_idx", w13_g_idx_processed)
|
||||
if w2_g_idx_processed is not None:
|
||||
replace_parameter(layer, "w2_weight_g_idx", w2_g_idx_processed)
|
||||
if w13_g_idx_sort_indices is not None:
|
||||
replace_parameter(
|
||||
layer, "w13_g_idx_sort_indices", w13_g_idx_sort_indices
|
||||
)
|
||||
if w2_g_idx_sort_indices is not None:
|
||||
replace_parameter(layer, "w2_g_idx_sort_indices", w2_g_idx_sort_indices)
|
||||
|
||||
# Register input global scales if present
|
||||
if w13_input_global_scale is not None:
|
||||
layer.register_parameter(
|
||||
@@ -593,7 +472,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
and self.wna16_backend != WNA16MoEBackend.EMULATION
|
||||
):
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
layer.w13_weight_g_idx.device,
|
||||
layer.w13_weight_packed.device,
|
||||
4,
|
||||
existing=getattr(layer, "workspace", None),
|
||||
)
|
||||
|
||||
+2
-3
@@ -65,15 +65,14 @@ class CompressedTensorsWNA16RDNA3MoEMethod(CompressedTensorsWNA16MoEMethod):
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
device = layer.w13_weight_packed.device
|
||||
num_experts = layer.w13_weight_packed.shape[0]
|
||||
empty_g_idx = torch.empty(0, dtype=torch.int32, device=device)
|
||||
|
||||
# Shuffle weights in-place per expert (exllama nibble interleave)
|
||||
for e in range(num_experts):
|
||||
w13_e = layer.w13_weight_packed.data[e].contiguous()
|
||||
ops.gptq_shuffle(w13_e, empty_g_idx, 4)
|
||||
ops.gptq_shuffle(w13_e, 4)
|
||||
layer.w13_weight_packed.data[e] = w13_e
|
||||
w2_e = layer.w2_weight_packed.data[e].contiguous()
|
||||
ops.gptq_shuffle(w2_e, empty_g_idx, 4)
|
||||
ops.gptq_shuffle(w2_e, 4)
|
||||
layer.w2_weight_packed.data[e] = w2_e
|
||||
|
||||
# Keep scales as [E, groups, N] in activation dtype
|
||||
|
||||
+1
-6
@@ -4,7 +4,6 @@
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
from compressed_tensors.quantization import ActivationOrdering
|
||||
|
||||
from vllm.distributed.utils import verify_group_size_divides_partition
|
||||
from vllm.logger import init_logger
|
||||
@@ -43,13 +42,11 @@ class CompressedTensorsW4A8Fp8(CompressedTensorsScheme):
|
||||
num_bits: int,
|
||||
group_size: int | None = None,
|
||||
symmetric: bool | None = True,
|
||||
actorder: ActivationOrdering | None = None,
|
||||
):
|
||||
self.pack_factor = 32 // num_bits
|
||||
self.strategy = strategy
|
||||
self.symmetric = symmetric
|
||||
self.group_size = -1 if group_size is None else group_size
|
||||
self.has_g_idx = actorder == ActivationOrdering.GROUP
|
||||
|
||||
if self.group_size != 128 or self.strategy != "group":
|
||||
raise ValueError(
|
||||
@@ -92,7 +89,6 @@ class CompressedTensorsW4A8Fp8(CompressedTensorsScheme):
|
||||
act_type=torch.float8_e4m3fn, # always use fp8(e4m3)
|
||||
group_size=self.group_size,
|
||||
zero_points=not self.symmetric,
|
||||
has_g_idx=self.has_g_idx,
|
||||
out_type=params_dtype,
|
||||
)
|
||||
|
||||
@@ -106,7 +102,7 @@ class CompressedTensorsW4A8Fp8(CompressedTensorsScheme):
|
||||
group_size = self.group_size if self.group_size != -1 else input_size
|
||||
row_parallel = input_size != input_size_per_partition
|
||||
partition_scales = not marlin_repeat_scales_on_all_ranks(
|
||||
self.has_g_idx, self.group_size, row_parallel
|
||||
self.group_size, row_parallel
|
||||
)
|
||||
|
||||
scales_and_zp_size = input_size // group_size
|
||||
@@ -162,7 +158,6 @@ class CompressedTensorsW4A8Fp8(CompressedTensorsScheme):
|
||||
w_q_param_name="weight_packed",
|
||||
w_s_param_name="weight_scale",
|
||||
w_zp_param_name="weight_zero_point",
|
||||
w_gidx_param_name="weight_g_idx",
|
||||
)
|
||||
|
||||
# Checkpoints are serialized in compressed-tensors format, which is
|
||||
|
||||
-2
@@ -98,7 +98,6 @@ class CompressedTensorsW4A8Int(CompressedTensorsScheme):
|
||||
act_type=params_dtype,
|
||||
group_size=effective_group_size,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
|
||||
kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config)
|
||||
@@ -140,7 +139,6 @@ class CompressedTensorsW4A8Int(CompressedTensorsScheme):
|
||||
w_q_param_name="weight_packed",
|
||||
w_s_param_name="weight_scale",
|
||||
w_zp_param_name=None,
|
||||
w_gidx_param_name=None,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
|
||||
+1
-19
@@ -6,7 +6,6 @@ from collections.abc import Callable
|
||||
from fractions import Fraction
|
||||
|
||||
import torch
|
||||
from compressed_tensors.quantization import ActivationOrdering
|
||||
|
||||
from vllm.distributed.utils import verify_group_size_divides_partition
|
||||
from vllm.logger import init_logger
|
||||
@@ -28,7 +27,6 @@ from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
PackedColumnParameter,
|
||||
PackedvLLMParameter,
|
||||
RowvLLMParameter,
|
||||
)
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
@@ -57,7 +55,6 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
|
||||
num_bits: int,
|
||||
group_size: int | None = None,
|
||||
symmetric: bool | None = True,
|
||||
actorder: ActivationOrdering | None = None,
|
||||
layer_name: str | None = None,
|
||||
):
|
||||
self.num_bits = num_bits
|
||||
@@ -65,7 +62,6 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
|
||||
self.strategy = strategy
|
||||
self.symmetric = symmetric
|
||||
self.group_size = -1 if group_size is None else group_size
|
||||
self.has_g_idx = actorder == ActivationOrdering.GROUP
|
||||
self.layer_name = layer_name
|
||||
|
||||
if self.group_size == -1 and self.strategy != "channel":
|
||||
@@ -128,7 +124,6 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
|
||||
act_type=params_dtype,
|
||||
group_size=self.group_size,
|
||||
zero_points=not self.symmetric,
|
||||
has_g_idx=self.has_g_idx,
|
||||
)
|
||||
|
||||
kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config)
|
||||
@@ -146,7 +141,7 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
|
||||
group_size = self.group_size if self.group_size != -1 else input_size
|
||||
row_parallel = input_size != input_size_per_partition
|
||||
partition_scales = not marlin_repeat_scales_on_all_ranks(
|
||||
self.has_g_idx, self.group_size, row_parallel
|
||||
self.group_size, row_parallel
|
||||
)
|
||||
|
||||
scales_and_zp_size = input_size // group_size
|
||||
@@ -226,24 +221,11 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
|
||||
if not self.symmetric:
|
||||
layer.register_parameter("weight_zero_point", qzeros)
|
||||
|
||||
# group index (for activation reordering)
|
||||
if self.has_g_idx:
|
||||
weight_g_idx = RowvLLMParameter(
|
||||
data=torch.empty(
|
||||
input_size_per_partition,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
input_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight_g_idx", weight_g_idx)
|
||||
|
||||
self.kernel = kernel_type(
|
||||
mp_linear_kernel_config,
|
||||
w_q_param_name="weight_packed",
|
||||
w_s_param_name="weight_scale",
|
||||
w_zp_param_name="weight_zero_point",
|
||||
w_gidx_param_name="weight_g_idx",
|
||||
)
|
||||
|
||||
# Checkpoints are serialized in compressed-tensors format, which is
|
||||
|
||||
+1
-2
@@ -144,7 +144,6 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
|
||||
act_type=params_dtype,
|
||||
group_size=self.group_size,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
|
||||
kernel_type = choose_mp_linear_kernel(mp_config)
|
||||
@@ -166,7 +165,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
|
||||
group_size = self.group_size if self.group_size != -1 else input_size
|
||||
row_parallel = input_size != input_size_per_partition
|
||||
partition_scales = not marlin_repeat_scales_on_all_ranks(
|
||||
False, self.group_size, row_parallel
|
||||
self.group_size, row_parallel
|
||||
)
|
||||
scales_size = input_size // group_size
|
||||
if partition_scales:
|
||||
|
||||
+1
-2
@@ -144,7 +144,6 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
|
||||
act_type=params_dtype,
|
||||
group_size=self.group_size,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
|
||||
kernel_type = choose_mp_linear_kernel(mp_config)
|
||||
@@ -166,7 +165,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
|
||||
group_size = self.group_size if self.group_size != -1 else input_size
|
||||
row_parallel = input_size != input_size_per_partition
|
||||
partition_scales = not marlin_repeat_scales_on_all_ranks(
|
||||
False, self.group_size, row_parallel
|
||||
self.group_size, row_parallel
|
||||
)
|
||||
scales_size = input_size // group_size
|
||||
if partition_scales:
|
||||
|
||||
+1
-2
@@ -114,7 +114,6 @@ class CompressedTensorsWNA8O8Int(CompressedTensorsScheme):
|
||||
act_type=params_dtype, # activation quant applied externally (SRQ)
|
||||
group_size=self.group_size,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
self.kernel = choose_mp_linear_kernel(mp_config)(
|
||||
mp_config,
|
||||
@@ -167,7 +166,7 @@ class CompressedTensorsWNA8O8Int(CompressedTensorsScheme):
|
||||
# Scale: per-output-channel, or per group along the input dim under TP.
|
||||
group_size = self.group_size if self.group_size != -1 else input_size
|
||||
partitioned = not marlin_repeat_scales_on_all_ranks(
|
||||
False, self.group_size, input_size != input_size_per_partition
|
||||
self.group_size, input_size != input_size_per_partition
|
||||
)
|
||||
scales = (input_size_per_partition if partitioned else input_size) // group_size
|
||||
scale_data = torch.empty(out, scales, dtype=params_dtype)
|
||||
|
||||
@@ -106,7 +106,7 @@ class INCXPUW4A8LinearMethod(INCXPULinearMethod):
|
||||
layer.scales_fp16,
|
||||
layer.qzeros,
|
||||
self.group_size,
|
||||
None,
|
||||
None, # Retained by the external XPU op ABI.
|
||||
bias,
|
||||
)
|
||||
return out.to(x.dtype).reshape(out_shape)
|
||||
|
||||
@@ -14,7 +14,6 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
PackedvLLMParameter,
|
||||
RowvLLMParameter,
|
||||
)
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
@@ -254,16 +253,6 @@ class INCXPULinearBase(INCLinearScheme):
|
||||
layer.register_parameter("scales", scales)
|
||||
layer.register_parameter("qzeros", qzeros)
|
||||
|
||||
g_idx = RowvLLMParameter(
|
||||
data=torch.tensor(
|
||||
[i // self.group_size for i in range(input_size_per_partition)],
|
||||
dtype=torch.int32,
|
||||
),
|
||||
input_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("g_idx", g_idx)
|
||||
|
||||
def _convert_awq_qweight_to_gptq(self, qw: torch.Tensor) -> torch.Tensor:
|
||||
"""Convert AWQ qweight [K, N // pf] to GPTQ qweight [K // pf, N].
|
||||
|
||||
@@ -356,7 +345,7 @@ class INCXPULinearMethod(INCXPULinearBase):
|
||||
layer.scales,
|
||||
layer.qzeros,
|
||||
self.group_size,
|
||||
None,
|
||||
None, # Retained by the external XPU op ABI.
|
||||
)
|
||||
return out.reshape(out_shape)
|
||||
|
||||
|
||||
@@ -32,6 +32,9 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.gptq_utils import (
|
||||
normalize_and_validate_gptq_desc_act,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
INT4_DTYPE,
|
||||
INT8_DTYPE,
|
||||
@@ -69,7 +72,11 @@ class MoeWNA16Config(QuantizationConfig):
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
|
||||
if self.linear_quant_method == "gptq":
|
||||
pass
|
||||
normalize_and_validate_gptq_desc_act(
|
||||
full_config.get("desc_act", False),
|
||||
group_size,
|
||||
full_config.get("dynamic") or {},
|
||||
)
|
||||
elif self.linear_quant_method in ("awq", "awq_marlin"):
|
||||
capability_tuple = current_platform.get_device_capability()
|
||||
device_capability = (
|
||||
@@ -148,7 +155,16 @@ class MoeWNA16Config(QuantizationConfig):
|
||||
# Extract data from quant config.
|
||||
quant_method = quant_config.get("quant_method", "").lower()
|
||||
num_bits = quant_config.get("bits")
|
||||
desc_act = quant_config.get("desc_act")
|
||||
desc_act = quant_config.get("desc_act", False)
|
||||
try:
|
||||
normalize_and_validate_gptq_desc_act(
|
||||
desc_act,
|
||||
quant_config.get("group_size"),
|
||||
quant_config.get("dynamic") or {},
|
||||
)
|
||||
supported_act_order = True
|
||||
except ValueError:
|
||||
supported_act_order = False
|
||||
|
||||
capability_tuple = current_platform.get_device_capability()
|
||||
device_capability = (
|
||||
@@ -159,7 +175,9 @@ class MoeWNA16Config(QuantizationConfig):
|
||||
|
||||
awq_min_capability = AutoAWQConfig.get_min_capability()
|
||||
|
||||
gptq_compatible = quant_method == "gptq" and not desc_act and num_bits in [4, 8]
|
||||
gptq_compatible = (
|
||||
quant_method == "gptq" and supported_act_order and num_bits in [4, 8]
|
||||
)
|
||||
awq_compatible = (
|
||||
quant_method == "awq"
|
||||
and num_bits == 4
|
||||
@@ -356,7 +374,7 @@ class MoeWNA16Method(FusedMoEMethodBase):
|
||||
if self.quant_config.linear_quant_method == "gptq":
|
||||
# some param are unused, but we need to init them in order to
|
||||
# load weights
|
||||
invalid_param_keys = ["w13_g_idx", "w2_g_idx"]
|
||||
invalid_param_keys: list[str] = []
|
||||
if not self.quant_config.has_zp:
|
||||
invalid_param_keys += ["w13_qzeros", "w2_qzeros"]
|
||||
for key in invalid_param_keys:
|
||||
@@ -414,8 +432,6 @@ class MoeWNA16Method(FusedMoEMethodBase):
|
||||
w2=layer.w2_qweight,
|
||||
w13_scale=layer.w13_scales,
|
||||
w2_scale=layer.w2_scales,
|
||||
w13_g_idx=None,
|
||||
w2_g_idx=None,
|
||||
w13_qzeros=layer.w13_qzeros if has_zp else None,
|
||||
w2_qzeros=layer.w2_qzeros if has_zp else None,
|
||||
)
|
||||
@@ -430,10 +446,6 @@ class MoeWNA16Method(FusedMoEMethodBase):
|
||||
w2_qweight,
|
||||
w13_scales,
|
||||
w2_scales,
|
||||
_,
|
||||
_,
|
||||
_,
|
||||
_,
|
||||
w13_qzeros,
|
||||
w2_qzeros,
|
||||
w13_input_global_scale,
|
||||
@@ -576,8 +588,6 @@ class MoeWNA16Method(FusedMoEMethodBase):
|
||||
expert_id: int,
|
||||
return_success: bool = False,
|
||||
):
|
||||
if "g_idx" in weight_name:
|
||||
return False if return_success else None
|
||||
if not layer.quant_config.has_zp and "qzeros" in weight_name:
|
||||
return False if return_success else None
|
||||
|
||||
|
||||
@@ -20,6 +20,36 @@ else:
|
||||
AutoGPTQConfig = object
|
||||
|
||||
|
||||
def normalize_and_validate_gptq_desc_act(
|
||||
desc_act: bool,
|
||||
group_size: int | None,
|
||||
dynamic: Mapping[str, Any],
|
||||
) -> bool:
|
||||
base_desc_act = desc_act
|
||||
if desc_act and group_size == -1:
|
||||
# With one group per output channel, activation ordering is a no-op.
|
||||
desc_act = False
|
||||
if desc_act:
|
||||
raise ValueError(
|
||||
"GPTQ group activation ordering (desc_act=True) is no longer "
|
||||
"supported. Use a checkpoint with static activation ordering "
|
||||
"or desc_act=False."
|
||||
)
|
||||
|
||||
for pattern, overrides in dynamic.items():
|
||||
if pattern.startswith("-:") or not isinstance(overrides, Mapping):
|
||||
continue
|
||||
dynamic_desc_act = overrides.get("desc_act", base_desc_act)
|
||||
dynamic_group_size = overrides.get("group_size", group_size)
|
||||
if dynamic_desc_act and dynamic_group_size != -1:
|
||||
raise ValueError(
|
||||
"GPTQ group activation ordering (desc_act=True) is no longer "
|
||||
f"supported, but is enabled by dynamic rule {pattern!r}. Use "
|
||||
"a checkpoint with static activation ordering or desc_act=False."
|
||||
)
|
||||
return desc_act
|
||||
|
||||
|
||||
# Match dynamic rules with module name (prefix) and override quantize
|
||||
# config if module (prefix) matches a rule
|
||||
def override_config(config: AutoGPTQConfig, prefix: str):
|
||||
@@ -33,6 +63,10 @@ def override_config(config: AutoGPTQConfig, prefix: str):
|
||||
if isinstance(desc_act, bool):
|
||||
config.desc_act = desc_act
|
||||
|
||||
if config.desc_act and config.group_size == -1:
|
||||
# Activation ordering is a no-op for channelwise quantization.
|
||||
config.desc_act = False
|
||||
|
||||
config.pack_factor = 32 // config.weight_bits # packed into int32
|
||||
assert isinstance(config, AutoGPTQConfig)
|
||||
is_sym = get_dynamic_override(config, prefix, "sym", config.is_sym)
|
||||
|
||||
@@ -359,11 +359,11 @@ def check_moe_marlin_supports_config(
|
||||
) -> bool:
|
||||
"""Whether the fused MoE Marlin kernel supports ``config``.
|
||||
|
||||
Callers without act-order may pass ``allow_tile_padding=True``: a
|
||||
tile-misaligned intermediate size is then zero-padded to a valid thread
|
||||
tile at weight prep (see marlin_moe_padded_intermediate), so only a group
|
||||
straddling the padded boundary stays unsupported. hidden_size is the MoE
|
||||
I/O extent and is never padded. Act-order keeps the strict shape.
|
||||
With ``allow_tile_padding=True``, a tile-misaligned intermediate size is
|
||||
zero-padded to a valid thread tile at weight prep (see
|
||||
marlin_moe_padded_intermediate), so only a group straddling the padded
|
||||
boundary stays unsupported. hidden_size is the MoE I/O extent and is never
|
||||
padded.
|
||||
"""
|
||||
if current_platform.is_rocm():
|
||||
return False
|
||||
@@ -433,30 +433,17 @@ def marlin_make_workspace_new(
|
||||
return torch.zeros(size, dtype=torch.int, device=device, requires_grad=False)
|
||||
|
||||
|
||||
def marlin_is_k_full(act_order: bool, is_row_parallel: bool) -> bool:
|
||||
return (not act_order) or (act_order and not is_row_parallel)
|
||||
|
||||
|
||||
def marlin_repeat_scales_on_all_ranks(
|
||||
act_order: bool, group_size: int, is_row_parallel: bool
|
||||
) -> bool:
|
||||
# Need to repeat scales on every rank if act_ordering or
|
||||
# channelwise and RowParallelLinear
|
||||
def marlin_repeat_scales_on_all_ranks(group_size: int, is_row_parallel: bool) -> bool:
|
||||
is_channelwise = group_size == -1
|
||||
return act_order or (is_channelwise and is_row_parallel)
|
||||
return is_channelwise and is_row_parallel
|
||||
|
||||
|
||||
def marlin_make_empty_g_idx(device: torch.device) -> torch.Tensor:
|
||||
def marlin_make_empty(device: torch.device) -> torch.Tensor:
|
||||
return torch.nn.Parameter(
|
||||
torch.empty(0, dtype=torch.int, device=device), requires_grad=False
|
||||
)
|
||||
|
||||
|
||||
def marlin_sort_g_idx(g_idx: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
g_idx_sort_indices = torch.argsort(g_idx).to(torch.int)
|
||||
return g_idx[g_idx_sort_indices], g_idx_sort_indices
|
||||
|
||||
|
||||
def get_scale_perms():
|
||||
scale_perm: list[int] = []
|
||||
for i in range(8):
|
||||
@@ -700,13 +687,10 @@ def apply_gptq_marlin_linear(
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
weight_zp: torch.Tensor,
|
||||
g_idx: torch.Tensor,
|
||||
g_idx_sort_indices: torch.Tensor,
|
||||
workspace: torch.Tensor,
|
||||
wtype: ScalarType,
|
||||
output_size_per_partition: int,
|
||||
input_size_per_partition: int,
|
||||
is_k_full: bool,
|
||||
input_global_scale: torch.Tensor | None = None,
|
||||
bias: torch.Tensor | None = None,
|
||||
use_fp32_reduce: bool = USE_FP32_REDUCE_DEFAULT,
|
||||
@@ -749,14 +733,11 @@ def apply_gptq_marlin_linear(
|
||||
a_scales,
|
||||
None,
|
||||
weight_zp,
|
||||
g_idx,
|
||||
g_idx_sort_indices,
|
||||
workspace,
|
||||
wtype,
|
||||
size_m=reshaped_x.shape[0],
|
||||
size_n=padded_n,
|
||||
size_k=padded_k,
|
||||
is_k_full=is_k_full,
|
||||
use_atomic_add=use_atomic_add,
|
||||
use_fp32_reduce=use_fp32_reduce,
|
||||
is_zp_float=False,
|
||||
|
||||
@@ -203,8 +203,6 @@ def apply_fp4_marlin_linear(
|
||||
a_scales=a_scales,
|
||||
global_scale=weight_global_scale,
|
||||
b_zeros=None,
|
||||
g_idx=None,
|
||||
perm=None,
|
||||
workspace=workspace,
|
||||
b_q_type=scalar_types.float4_e2m1f,
|
||||
size_m=reshaped_x.size(0),
|
||||
@@ -246,14 +244,12 @@ def prepare_fp4_layer_for_marlin(
|
||||
|
||||
# WEIGHT
|
||||
# Repack weights to marlin format
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
qweight = layer.weight.view(torch.int32).T.contiguous()
|
||||
qweight = marlin_pad_qweight(qweight, part_size_n, part_size_k, padded_n, padded_k)
|
||||
|
||||
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=qweight,
|
||||
perm=perm,
|
||||
size_k=padded_k,
|
||||
size_n=padded_n,
|
||||
num_bits=4,
|
||||
@@ -312,7 +308,6 @@ def _repack_marlin_experts(
|
||||
weight: torch.Tensor,
|
||||
size_n: int,
|
||||
size_k: int,
|
||||
perm: torch.Tensor,
|
||||
is_a_8bit: bool,
|
||||
) -> torch.Tensor:
|
||||
"""Repack each expert to marlin format into a preallocated output."""
|
||||
@@ -322,7 +317,6 @@ def _repack_marlin_experts(
|
||||
qweight = weight[i].view(torch.int32).T.contiguous()
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=qweight,
|
||||
perm=perm,
|
||||
size_k=size_k,
|
||||
size_n=size_n,
|
||||
num_bits=4,
|
||||
@@ -401,7 +395,6 @@ def prepare_nvfp4_moe_layer_for_marlin(
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, 4, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
|
||||
# WEIGHT
|
||||
# Repack weights to marlin format
|
||||
@@ -417,7 +410,7 @@ def prepare_nvfp4_moe_layer_for_marlin(
|
||||
weight = pad_w2(weight, packing=2)
|
||||
size_k = padded_N
|
||||
|
||||
return _repack_marlin_experts(weight, size_n, size_k, perm, is_a_8bit)
|
||||
return _repack_marlin_experts(weight, size_n, size_k, is_a_8bit)
|
||||
|
||||
w13 = repack_weight(w13, "w13")
|
||||
w2 = repack_weight(w2, "w2")
|
||||
@@ -492,7 +485,6 @@ def prepare_moe_fp4_layer_for_marlin(
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, 4, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
|
||||
|
||||
# WEIGHT
|
||||
@@ -506,7 +498,7 @@ def prepare_moe_fp4_layer_for_marlin(
|
||||
|
||||
assert weight.shape == (e, size_n, size_k // 2)
|
||||
|
||||
weight = _repack_marlin_experts(weight, size_n, size_k, perm, is_a_8bit)
|
||||
weight = _repack_marlin_experts(weight, size_n, size_k, is_a_8bit)
|
||||
weight = torch.nn.Parameter(weight, requires_grad=False)
|
||||
|
||||
setattr(layer, name, weight)
|
||||
@@ -621,10 +613,8 @@ def prepare_moe_mxfp4_layer_for_marlin(
|
||||
n = w13.shape[1] // 2 # intermediate_size_per_partition
|
||||
k = w13.shape[2] * 2 # hidden_size
|
||||
|
||||
device = w13.device
|
||||
param_dtype = layer.params_dtype
|
||||
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
|
||||
# WEIGHT: Repack weights to marlin format
|
||||
def repack_weight(weight: torch.Tensor, name: str) -> torch.Tensor:
|
||||
@@ -635,7 +625,7 @@ def prepare_moe_mxfp4_layer_for_marlin(
|
||||
|
||||
assert weight.shape == (e, size_n, size_k // 2)
|
||||
|
||||
return _repack_marlin_experts(weight, size_n, size_k, perm, is_a_8bit)
|
||||
return _repack_marlin_experts(weight, size_n, size_k, is_a_8bit)
|
||||
|
||||
w13 = repack_weight(w13, "w13")
|
||||
w2 = repack_weight(w2, "w2")
|
||||
@@ -691,8 +681,6 @@ def rand_marlin_weight_nvfp4_like(weight, group_size, input_dtype=None):
|
||||
assert not is_a_8bit, "NVFP4 weight + INT8/FP8 activation is not supported."
|
||||
assert group_size > 0
|
||||
size_n, size_k = weight.shape
|
||||
device = weight.device
|
||||
|
||||
scales = weight.view(size_n, -1, group_size).abs().max(-1)[0] / 6
|
||||
global_scale = scales.max() / 448
|
||||
scales = (scales / global_scale).to(torch.float8_e4m3fn)
|
||||
@@ -720,7 +708,6 @@ def rand_marlin_weight_nvfp4_like(weight, group_size, input_dtype=None):
|
||||
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=fp4_weight.view(torch.int32).T.contiguous(),
|
||||
perm=torch.empty(0, dtype=torch.int, device=device),
|
||||
size_k=size_k,
|
||||
size_n=size_n,
|
||||
num_bits=4,
|
||||
@@ -751,8 +738,6 @@ def rand_marlin_weight_mxfp4_like(weight, group_size, input_dtype=None):
|
||||
|
||||
assert group_size > 0
|
||||
size_n, size_k = weight.shape
|
||||
device = weight.device
|
||||
|
||||
scales = torch.randint(
|
||||
110,
|
||||
120,
|
||||
@@ -779,11 +764,9 @@ def rand_marlin_weight_mxfp4_like(weight, group_size, input_dtype=None):
|
||||
).view(size_n, size_k)
|
||||
weight_ref = weight_ref * scales.repeat_interleave(group_size, 1).to(weight.dtype)
|
||||
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
fp4_weight = fp4_weight.view(torch.int32).T.contiguous()
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=fp4_weight,
|
||||
perm=perm,
|
||||
size_k=size_k,
|
||||
size_n=size_n,
|
||||
num_bits=4,
|
||||
|
||||
@@ -89,8 +89,6 @@ def apply_fp8_marlin_linear(
|
||||
a_scales=a_scales,
|
||||
global_scale=None,
|
||||
b_zeros=None,
|
||||
g_idx=None,
|
||||
perm=None,
|
||||
workspace=workspace,
|
||||
b_q_type=scalar_types.float8_e4m3fn,
|
||||
size_m=reshaped_x.size(0),
|
||||
@@ -138,7 +136,6 @@ def prepare_fp8_layer_for_marlin(
|
||||
|
||||
# WEIGHT
|
||||
# Repack weights to marlin format
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
qweight = pack_fp8_to_int32(layer.weight, size_k_first)
|
||||
if not size_k_first:
|
||||
qweight = qweight.T.contiguous()
|
||||
@@ -146,7 +143,6 @@ def prepare_fp8_layer_for_marlin(
|
||||
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=qweight,
|
||||
perm=perm,
|
||||
size_k=padded_k,
|
||||
size_n=padded_n,
|
||||
num_bits=8,
|
||||
@@ -283,7 +279,6 @@ def prepare_fp8_moe_layer_for_marlin(
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, 4, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
|
||||
# WEIGHT
|
||||
# Repack weights to marlin format
|
||||
@@ -295,7 +290,7 @@ def prepare_fp8_moe_layer_for_marlin(
|
||||
qweight = qweight.T.contiguous()
|
||||
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=qweight, perm=perm, size_k=size_k, size_n=size_n, num_bits=8
|
||||
b_q_weight=qweight, size_k=size_k, size_n=size_n, num_bits=8
|
||||
)
|
||||
tensor_list.append(marlin_qweight)
|
||||
|
||||
@@ -434,8 +429,6 @@ def apply_mxfp8_marlin_linear(
|
||||
a_scales=None,
|
||||
global_scale=None,
|
||||
b_zeros=None,
|
||||
g_idx=None,
|
||||
perm=None,
|
||||
workspace=workspace,
|
||||
b_q_type=scalar_types.float8_e4m3fn,
|
||||
size_m=reshaped_x.size(0),
|
||||
@@ -470,14 +463,12 @@ def prepare_mxfp8_layer_for_marlin(layer: torch.nn.Module) -> None:
|
||||
)
|
||||
|
||||
# WEIGHT - repack FP8 weights to Marlin format
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
qweight = pack_fp8_to_int32(layer.weight, size_k_first=False)
|
||||
qweight = qweight.T.contiguous()
|
||||
qweight = marlin_pad_qweight(qweight, part_size_n, part_size_k, padded_n, padded_k)
|
||||
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=qweight,
|
||||
perm=perm,
|
||||
size_k=padded_k,
|
||||
size_n=padded_n,
|
||||
num_bits=8,
|
||||
@@ -551,7 +542,6 @@ def prepare_mxfp8_moe_layer_for_marlin(
|
||||
|
||||
device = w13.device
|
||||
param_dtype = torch.get_default_dtype()
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, 4, existing=getattr(layer, "workspace", None)
|
||||
@@ -571,7 +561,6 @@ def prepare_mxfp8_moe_layer_for_marlin(
|
||||
qweight = qweight.T.contiguous()
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=qweight,
|
||||
perm=perm,
|
||||
size_k=size_k,
|
||||
size_n=size_n,
|
||||
num_bits=8,
|
||||
@@ -615,8 +604,6 @@ def marlin_quant_fp8_torch(weight, group_size, input_dtype=None):
|
||||
assert input_dtype == torch.float8_e4m3fn
|
||||
|
||||
size_n, size_k = weight.shape
|
||||
device = weight.device
|
||||
|
||||
if group_size != -1:
|
||||
scales = weight.view(size_n, -1, group_size).abs().max(-1)[0] / 448
|
||||
repeated_scales = scales.repeat_interleave(group_size, 1)
|
||||
@@ -629,10 +616,8 @@ def marlin_quant_fp8_torch(weight, group_size, input_dtype=None):
|
||||
weight_ref = fp8_weight.to(weight.dtype) * repeated_scales
|
||||
|
||||
packed_weight = pack_fp8_to_int32(fp8_weight, False).T.contiguous()
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=packed_weight,
|
||||
perm=perm,
|
||||
size_k=size_k,
|
||||
size_n=size_n,
|
||||
num_bits=8,
|
||||
|
||||
@@ -13,7 +13,6 @@ from .quant_utils import (
|
||||
get_pack_factor,
|
||||
gptq_quantize_weights,
|
||||
quantize_weights,
|
||||
sort_weights,
|
||||
)
|
||||
|
||||
|
||||
@@ -129,8 +128,6 @@ def marlin_quantize(
|
||||
w: torch.Tensor,
|
||||
quant_type: ScalarType,
|
||||
group_size: int,
|
||||
act_order: bool,
|
||||
test_perm: torch.Tensor | None = None,
|
||||
input_dtype: torch.dtype | None = None,
|
||||
):
|
||||
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
|
||||
@@ -143,16 +140,7 @@ def marlin_quantize(
|
||||
group_size = size_k
|
||||
assert group_size <= size_k
|
||||
|
||||
# Quantize (and apply act_order if provided)
|
||||
w_ref, q_w, s, g_idx, rand_perm = gptq_quantize_weights(
|
||||
w, quant_type, group_size, act_order, test_perm
|
||||
)
|
||||
|
||||
# For act_order, sort the "weights" and "g_idx" so that group ids are
|
||||
# increasing
|
||||
sort_indices = torch.empty(0, dtype=torch.int, device=w.device)
|
||||
if act_order:
|
||||
q_w, g_idx, sort_indices = sort_weights(q_w, g_idx)
|
||||
w_ref, q_w, s = gptq_quantize_weights(w, quant_type, group_size)
|
||||
|
||||
# Reformat to marlin
|
||||
weight_perm = get_weight_perm(num_bits, is_a_8bit)
|
||||
@@ -166,7 +154,7 @@ def marlin_quantize(
|
||||
marlin_s = marlin_s * 512
|
||||
|
||||
# Create result
|
||||
res_list = [w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, rand_perm]
|
||||
res_list = [w_ref, marlin_q_w, marlin_s]
|
||||
for i in range(len(res_list)):
|
||||
res_list[i] = res_list[i].to(w.device)
|
||||
|
||||
|
||||
@@ -691,36 +691,6 @@ def get_pack_factor(num_bits):
|
||||
return 32 // num_bits
|
||||
|
||||
|
||||
def permute_rows(
|
||||
q_w: torch.Tensor,
|
||||
w_ref: torch.Tensor,
|
||||
group_size: int,
|
||||
test_perm: torch.Tensor | None = None,
|
||||
):
|
||||
assert q_w.shape == w_ref.shape
|
||||
|
||||
orig_device = q_w.device
|
||||
k_size, _ = q_w.shape
|
||||
|
||||
g_idx = torch.zeros((k_size,), dtype=torch.int32)
|
||||
for i in range(k_size):
|
||||
g_idx[i] = i // group_size
|
||||
|
||||
# Simulate act_order by doing a random permutation on K
|
||||
rand_perm = test_perm if test_perm is not None else torch.randperm(k_size)
|
||||
|
||||
g_idx = g_idx[rand_perm].contiguous()
|
||||
q_w = q_w[rand_perm, :].contiguous()
|
||||
w_ref = w_ref[rand_perm, :].contiguous()
|
||||
|
||||
return (
|
||||
w_ref.to(device=orig_device),
|
||||
q_w.to(device=orig_device),
|
||||
g_idx.to(device=orig_device),
|
||||
rand_perm.to(device=orig_device),
|
||||
)
|
||||
|
||||
|
||||
def quantize_weights(
|
||||
w: torch.Tensor,
|
||||
quant_type: ScalarType,
|
||||
@@ -824,8 +794,6 @@ def gptq_quantize_weights(
|
||||
w: torch.Tensor,
|
||||
quant_type: ScalarType,
|
||||
group_size: int,
|
||||
act_order: bool,
|
||||
test_perm: torch.Tensor | None = None,
|
||||
):
|
||||
size_k, _ = w.shape
|
||||
|
||||
@@ -838,35 +806,7 @@ def gptq_quantize_weights(
|
||||
)
|
||||
|
||||
w_ref, w_q, w_s, _ = quantize_weights(w, quant_type, group_size)
|
||||
|
||||
# Apply act_order
|
||||
g_idx = torch.empty(0, dtype=torch.int, device=w.device)
|
||||
rand_perm = torch.empty(0, dtype=torch.int, device=w.device)
|
||||
if act_order:
|
||||
assert group_size < size_k, (
|
||||
"For act_order, groupsize = {} must be less than size_k = {}".format(
|
||||
group_size, size_k
|
||||
)
|
||||
)
|
||||
|
||||
w_ref, w_q, g_idx, rand_perm = permute_rows(w_q, w_ref, group_size, test_perm)
|
||||
|
||||
return w_ref, w_q, w_s, g_idx, rand_perm
|
||||
|
||||
|
||||
def sort_weights(q_w: torch.Tensor, g_idx: torch.Tensor):
|
||||
orig_device = q_w.device
|
||||
|
||||
sort_indices = torch.argsort(g_idx).to(dtype=torch.int32) # Sort based on g_idx
|
||||
|
||||
g_idx = g_idx[sort_indices].contiguous()
|
||||
q_w = q_w[sort_indices, :].contiguous()
|
||||
|
||||
return (
|
||||
q_w.to(device=orig_device),
|
||||
g_idx.to(device=orig_device),
|
||||
sort_indices.to(device=orig_device),
|
||||
)
|
||||
return w_ref, w_q, w_s
|
||||
|
||||
|
||||
def pack_rows(
|
||||
|
||||
@@ -463,8 +463,7 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
output_dim = getattr(param, "output_dim", None)
|
||||
packed_dim = getattr(param, "packed_dim", None)
|
||||
|
||||
# If parameter does not have output dim, then it should
|
||||
# be copied onto all gpus (e.g. g_idx for act_order gptq).
|
||||
# Parameters without an output dimension are copied onto all GPUs.
|
||||
if output_dim is None:
|
||||
if (
|
||||
loaded_weight.ndim == 0
|
||||
|
||||
Reference in New Issue
Block a user