[Quant][Kernel] Remove GPTQ Group/Dynamic Activation Ordering (#54809)

Signed-off-by: Roderick-Wu <roderickwu2003@gmail.com>
Signed-off-by: Roderick Wu <roderickwu2003@gmail.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Roderick Wu
2026-09-08 12:45:12 -07:00
committed by GitHub
co-authored by mgoin
parent 1a522b6949
commit 2c9d68f80b
99 changed files with 1182 additions and 3917 deletions
@@ -19,7 +19,6 @@ class MPLinearLayerConfig:
act_type: torch.dtype
group_size: int
zero_points: bool
has_g_idx: bool
out_type: torch.dtype | None = None
@@ -40,7 +39,6 @@ class MPLinearKernel(ABC):
w_q_param_name: str,
w_s_param_name: str,
w_zp_param_name: str | None = None,
w_gidx_param_name: str | None = None,
) -> None:
assert self.can_implement(c)
self.config = c
@@ -48,10 +46,7 @@ class MPLinearKernel(ABC):
self.w_s_name = w_s_param_name
if c.zero_points:
assert w_zp_param_name is not None
if c.has_g_idx:
assert w_gidx_param_name is not None
self.w_zp_name: str | None = w_zp_param_name
self.w_gidx_name: str | None = w_gidx_param_name
@abstractmethod
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
@@ -83,12 +78,10 @@ class MPLinearKernel(ABC):
) -> tuple[
torch.Tensor, # w_q
torch.Tensor, # w_s
torch.Tensor | None, # w_zp,
torch.Tensor | None, # w_gidx
torch.Tensor | None, # w_zp
]:
return (
getattr(layer, self.w_q_name),
getattr(layer, self.w_s_name),
getattr(layer, self.w_zp_name or "", None),
getattr(layer, self.w_gidx_name or "", None),
)
@@ -23,9 +23,6 @@ class AllSparkLinearKernel(MPLinearKernel):
@classmethod
def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]:
if c.has_g_idx:
return False, "Act reordering currently not supported by AllSpark"
if c.zero_points:
return False, "Zero points currently not supported by AllSpark"
@@ -91,7 +88,7 @@ class AllSparkLinearKernel(MPLinearKernel):
) -> torch.Tensor:
c = self.config
gemm_args = self.gemm_args
w_q, w_s, _, _ = self._get_weight_params(layer)
w_q, w_s, _ = self._get_weight_params(layer)
reshaped_x = x.reshape(-1, x.shape[-1])
out_shape = x.shape[:-1] + (c.partition_weight_shape[1],)
@@ -43,12 +43,6 @@ class ConchLinearKernel(MPLinearKernel):
)
return False, error_msg
if c.has_g_idx:
return (
False,
"Activation reordering (g_idx) is not supported by ConchLinearKernel",
)
if find_spec("conch") is None:
error_msg = (
"conch-triton-kernels is not installed, please "
@@ -130,7 +124,7 @@ class ConchLinearKernel(MPLinearKernel):
) -> torch.Tensor:
from conch.ops.quantization.gemm import mixed_precision_gemm
w_q, w_s, w_zp, _ = self._get_weight_params(layer)
w_q, w_s, w_zp = self._get_weight_params(layer)
# Map channelwise group_size=-1 to the actual input dimension K.
# The conch kernel computes stride_mul = block_k / group_size;
@@ -144,9 +144,6 @@ class CPUWNA16LinearKernel(MPLinearKernel):
if (not self.config.zero_points) and (self.w_zp_name is not None):
setattr(layer, self.w_zp_name, None)
if (not self.config.has_g_idx) and (self.w_gidx_name is not None):
setattr(layer, self.w_gidx_name, None)
weights = getattr(layer, self.w_q_name)
# Require GPTQ pack format
assert weights.input_dim == weights.packed_dim
@@ -172,7 +169,6 @@ class CPUWNA16LinearKernel(MPLinearKernel):
supports_riscv = current_platform.get_cpu_architecture() == CpuArchEnum.RISCV
layer.use_w4a8 = (
envs.VLLM_CPU_INT4_W4A8
and not self.config.has_g_idx
and self.config.act_type == torch.bfloat16
and (supports_amx or supports_riscv)
)
@@ -189,7 +185,7 @@ class CPUWNA16LinearKernel(MPLinearKernel):
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
w_q, w_s, w_zp, w_gidx = self._get_weight_params(layer)
w_q, w_s, w_zp = self._get_weight_params(layer)
if layer.use_w4a8:
x = ops.int4_scaled_mm_cpu(
x=x,
@@ -204,7 +200,6 @@ class CPUWNA16LinearKernel(MPLinearKernel):
q_weight=w_q,
scales=w_s,
zeros=w_zp,
g_idx=w_gidx,
bias=bias,
pack_factor=8, # 32 // 4
isa_hint=layer.isa_hint,
@@ -39,9 +39,6 @@ class CutlassW4A8LinearKernel(MPLinearKernel):
if c.act_type != torch.float8_e4m3fn:
return False, "CUTLASS W4A8 only supports FP8 (e4m3) activations"
if c.has_g_idx:
return False, "Act reordering not supported by CUTLASS W4A8"
if c.zero_points:
return False, "Zero points not supported by CUTLASS W4A8"
@@ -109,7 +106,7 @@ class CutlassW4A8LinearKernel(MPLinearKernel):
bias: torch.Tensor | None = None,
) -> torch.Tensor:
c = self.config
w_q, w_s, _, _ = self._get_weight_params(layer)
w_q, w_s, _ = self._get_weight_params(layer)
w_ch_s = layer.weight_chan_scale
x_2d = x.reshape(-1, x.shape[-1])
@@ -32,14 +32,6 @@ class ExllamaLinearKernel(MPLinearKernel):
"Exllama is only supported on CUDA and ROCm",
)
if c.has_g_idx and c.partition_weight_shape[0] != c.full_weight_shape[0]:
return (
False,
"Act reordering currently not supported by Exllama, "
"when the input features are partitioned across "
"devices",
)
if c.partition_weight_shape[1] % (32 // c.weight_type.size_bits) != 0:
return (
False,
@@ -111,29 +103,11 @@ class ExllamaLinearKernel(MPLinearKernel):
layer, self.w_zp_name, torch.nn.Parameter(zeros, requires_grad=False)
)
if c.has_g_idx:
def transform_w_g_idx(x):
# Exllama wants the permutation array instead of the group
# indices
return torch.argsort(x).to(torch.int)
self._transform_param(layer, self.w_gidx_name, transform_w_g_idx) # type: ignore
else:
self.w_gidx_name = "g_idx"
empty_g_idx = torch.nn.Parameter(
torch.empty((0,), dtype=torch.int, device=device), requires_grad=False
)
setattr(layer, self.w_gidx_name, empty_g_idx)
def transform_w_q(x):
assert isinstance(x, BasevLLMParameter)
assert self.w_gidx_name is not None
g_idx = getattr(layer, self.w_gidx_name)
permute_param_layout_(x, input_dim=0, output_dim=1, packed_dim=0)
x_cont = x.data.contiguous()
ops.gptq_shuffle(x_cont, g_idx, c.weight_type.size_bits)
ops.gptq_shuffle(x_cont, c.weight_type.size_bits)
return x_cont
def transform_w_s(x):
@@ -157,17 +131,15 @@ class ExllamaLinearKernel(MPLinearKernel):
x_2d = x.reshape(-1, x.shape[-1])
out_shape = x.shape[:-1] + (c.partition_weight_shape[1],)
w_q, w_s, w_zp, w_g_idx = self._get_weight_params(layer)
w_q, w_s, w_zp = self._get_weight_params(layer)
# gptq_gemm supports GPTQv2 format by passing use_v2_format=True.
# However, the MPLinearLayerConfig doesn't contain format info.
# So hardcode GPTQv1 format here, to keep its behavior unchanged.
use_v2_format = False
assert w_zp is not None, "Zero points are required by Exllama"
assert w_g_idx is not None, "Group index is required by Exllama"
output = ops.gptq_gemm(
x_2d, w_q, w_zp, w_s, w_g_idx, True, use_v2_format, c.weight_type.size_bits
x_2d, w_q, w_zp, w_s, True, use_v2_format, c.weight_type.size_bits
)
if bias is not None:
@@ -21,8 +21,6 @@ class HummingLinearKernel(MPLinearKernel):
return False, "Humming is only supported on CUDA"
if not has_humming():
return False, "Humming is not installed"
if c.has_g_idx:
return False, "Humming does not support act-order (g_idx)"
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import partial
import torch
@@ -12,7 +11,6 @@ from vllm.model_executor.layers.quantization.utils.machete_utils import (
query_machete_supported_quant_types,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
pack_quantized_values_into_int32,
unpack_quantized_values_into_int32,
)
from vllm.model_executor.parameter import BasevLLMParameter, permute_param_layout_
@@ -35,14 +33,6 @@ class MacheteLinearKernel(MPLinearKernel):
if not current_platform.is_device_capability(90):
return False, "Machete requires compute capability of 90 (Hopper)"
if c.has_g_idx and c.partition_weight_shape[0] != c.full_weight_shape[0]:
return (
False,
"Act reordering currently not supported by Machete, "
"when the input features are partitioned across "
"devices",
)
if c.weight_type not in query_machete_supported_quant_types(c.zero_points):
return (
False,
@@ -70,29 +60,9 @@ class MacheteLinearKernel(MPLinearKernel):
def process_weights_after_loading(self, layer: torch.nn.Module):
c = self.config
if c.has_g_idx:
assert self.w_gidx_name is not None
perm = torch.argsort(getattr(layer, self.w_gidx_name)).to(torch.int)
self.act_perm = lambda x: x[:, perm]
# use `ops.permute_cols` if possible
if (
c.act_type in [torch.float16, torch.bfloat16]
and c.partition_weight_shape[0] % 8 == 0
):
self.act_perm = partial(ops.permute_cols, perm=perm)
def transform_w_q(x):
assert isinstance(x, BasevLLMParameter)
permute_param_layout_(x, input_dim=0, output_dim=1, packed_dim=0)
if c.has_g_idx:
x_unpacked = unpack_quantized_values_into_int32(
x.data, c.weight_type, packed_dim=0
)
x_perm = x_unpacked[perm, :]
x.data = pack_quantized_values_into_int32(
x_perm, c.weight_type, packed_dim=0
)
x.data = ops.machete_prepack_B(
x.data.t().contiguous().t(),
a_type=c.act_type,
@@ -131,14 +101,11 @@ class MacheteLinearKernel(MPLinearKernel):
bias: torch.Tensor | None = None,
) -> torch.Tensor:
c = self.config
w_q, w_s, w_zp, _ = self._get_weight_params(layer)
w_q, w_s, w_zp = self._get_weight_params(layer)
x_2d = x.reshape(-1, x.shape[-1])
out_shape = x.shape[:-1] + (c.partition_weight_shape[1],)
if c.has_g_idx:
x_2d = self.act_perm(x_2d)
if c.zero_points:
assert w_zp is not None
else:
@@ -8,10 +8,8 @@ from vllm import _custom_ops as ops
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
MARLIN_SUPPORTED_GROUP_SIZES,
apply_gptq_marlin_linear,
check_marlin_supports_shape,
marlin_act_int8_process_scales,
marlin_is_k_full,
marlin_make_empty_g_idx,
marlin_make_empty,
marlin_make_workspace_new,
marlin_pad_dim,
marlin_pad_qweight,
@@ -19,13 +17,11 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
marlin_padded_nk,
marlin_permute_bias,
marlin_permute_scales,
marlin_sort_g_idx,
marlin_zero_points,
query_marlin_supported_quant_types,
unpack_cols,
)
from vllm.model_executor.parameter import BasevLLMParameter, permute_param_layout_
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
@@ -59,16 +55,6 @@ class MarlinLinearKernel(MPLinearKernel):
f"{MARLIN_SUPPORTED_GROUP_SIZES}",
)
if c.has_g_idx:
# Act-order couples K to the full-model group layout, so tile
# padding is not supported; keep the strict shape check.
return check_marlin_supports_shape(
c.partition_weight_shape[1], # out_features
c.partition_weight_shape[0], # in_features
c.full_weight_shape[0], # in_features
c.group_size,
)
# A group straddling TP ranks cannot be fixed by padding.
if (
c.group_size != -1
@@ -102,25 +88,16 @@ class MarlinLinearKernel(MPLinearKernel):
getattr(layer, self.w_s_name).data * 512
)
row_parallel = c.partition_weight_shape[0] != c.full_weight_shape[0]
self.is_k_full = marlin_is_k_full(c.has_g_idx, row_parallel)
size_k, size_n = c.partition_weight_shape
if c.has_g_idx:
# Act-order shapes were strictly validated in can_implement.
padded_n, padded_k = size_n, size_k
else:
padded_n, padded_k = marlin_padded_nk(size_n, size_k, c.group_size)
padded_n, padded_k = marlin_padded_nk(size_n, size_k, c.group_size)
# Allocate marlin workspace, reusing existing storage on reload.
self.workspace = marlin_make_workspace_new(
device, existing=getattr(self, "workspace", None)
)
# Default names since marlin requires empty parameters for these,
# Default name since marlin requires empty parameter for zp,
# TODO: remove this requirement from marlin (allow optional tensors)
if self.w_gidx_name is None:
self.w_gidx_name = "g_idx"
if self.w_zp_name is None:
self.w_zp_name = "w_zp"
@@ -131,7 +108,6 @@ class MarlinLinearKernel(MPLinearKernel):
marlin_pad_qweight(
x.data.contiguous(), size_n, size_k, padded_n, padded_k
),
perm=layer.g_idx_sort_indices,
size_k=padded_k,
size_n=padded_n,
num_bits=c.weight_type.size_bits,
@@ -172,18 +148,6 @@ class MarlinLinearKernel(MPLinearKernel):
layer.input_global_scale = None
return x
if c.has_g_idx:
g_idx, g_idx_sort_indices = marlin_sort_g_idx(
getattr(layer, self.w_gidx_name)
)
self._transform_param(layer, self.w_gidx_name, lambda _: g_idx)
replace_parameter(
layer, "g_idx_sort_indices", g_idx_sort_indices, prefer_copy=True
)
else:
setattr(layer, self.w_gidx_name, marlin_make_empty_g_idx(device))
layer.g_idx_sort_indices = marlin_make_empty_g_idx(device)
if c.zero_points:
grouped_k = size_k // c.group_size if c.group_size != -1 else 1
padded_grouped_k = padded_k // c.group_size if c.group_size != -1 else 1
@@ -211,7 +175,7 @@ class MarlinLinearKernel(MPLinearKernel):
),
)
else:
setattr(layer, self.w_zp_name, marlin_make_empty_g_idx(device))
setattr(layer, self.w_zp_name, marlin_make_empty(device))
self._transform_param(layer, self.w_q_name, transform_w_q)
self._transform_param(layer, self.w_s_name, transform_w_s)
@@ -227,23 +191,16 @@ class MarlinLinearKernel(MPLinearKernel):
bias: torch.Tensor | None = None,
) -> torch.Tensor:
c = self.config
w_q, w_s, w_zp, w_gidx = self._get_weight_params(layer)
# `process_weights_after_loading` will ensure w_zp and w_gidx are not
# None for marlin
w_q, w_s, w_zp = self._get_weight_params(layer)
return apply_gptq_marlin_linear(
input=x,
weight=w_q,
weight_scale=w_s,
weight_zp=w_zp, # type: ignore
g_idx=w_gidx, # type: ignore
g_idx_sort_indices=layer.g_idx_sort_indices,
workspace=self.workspace,
wtype=c.weight_type,
input_size_per_partition=c.partition_weight_shape[0],
output_size_per_partition=c.partition_weight_shape[1],
is_k_full=self.is_k_full,
input_global_scale=getattr(layer, "input_global_scale", None),
bias=bias,
input_dtype=c.act_type,
@@ -86,13 +86,6 @@ class RDNA3W4A16LinearKernel(MPLinearKernel):
"W4A16 kernel (qzeros packing)",
)
if c.has_g_idx and c.partition_weight_shape[0] != c.full_weight_shape[0]:
return (
False,
"Act-order with TP-partitioned input features is not "
"supported by the RDNA3 W4A16 kernel",
)
return True, None
# ----- Weight prep (identical layout/shuffle as ExllamaLinearKernel) -----
@@ -127,34 +120,14 @@ class RDNA3W4A16LinearKernel(MPLinearKernel):
layer, self.w_zp_name, torch.nn.Parameter(zeros, requires_grad=False)
)
# Act-order: convert g_idx to the inverse permutation array exllama
# expects (kernel reads a[perm[k]] instead of using groups indirected
# by g_idx[k]).
if c.has_g_idx:
def transform_w_g_idx(x):
return torch.argsort(x).to(torch.int)
self._transform_param(layer, self.w_gidx_name, transform_w_g_idx) # type: ignore
else:
self.w_gidx_name = "g_idx"
empty_g_idx = torch.nn.Parameter(
torch.empty((0,), dtype=torch.int, device=device),
requires_grad=False,
)
setattr(layer, self.w_gidx_name, empty_g_idx)
def transform_w_q(x):
assert isinstance(x, BasevLLMParameter)
assert self.w_gidx_name is not None
g_idx = getattr(layer, self.w_gidx_name)
permute_param_layout_(x, input_dim=0, output_dim=1, packed_dim=0)
x_cont = x.data.contiguous()
# Same 4-bit shuffle as exllama. The RDNA3 kernel reads weights in
# the same shuffled int32 layout and uses the (qa & 0x000F000F)
# bit-trick on top.
ops.gptq_shuffle(x_cont, g_idx, c.weight_type.size_bits)
ops.gptq_shuffle(x_cont, c.weight_type.size_bits)
return x_cont
def transform_w_s(x):
@@ -181,12 +154,9 @@ class RDNA3W4A16LinearKernel(MPLinearKernel):
x_2d = x.reshape(-1, x.shape[-1])
out_shape = x.shape[:-1] + (c.partition_weight_shape[1],)
w_q, w_s, w_zp, w_g_idx = self._get_weight_params(layer)
w_q, w_s, w_zp = self._get_weight_params(layer)
assert w_zp is not None, "Zero points are required by RDNA3 W4A16"
assert w_g_idx is not None, "g_idx tensor (possibly empty) required"
output = ops.gptq_gemm_rdna3(x_2d, w_q, w_zp, w_s, w_g_idx, False)
output = ops.gptq_gemm_rdna3(x_2d, w_q, w_zp, w_s, False)
if bias is not None:
output.add_(bias)
@@ -142,13 +142,13 @@ def _triton_w4a16_skinny_fmt_kernel(
b = tl.interleave(b, b)
b = (b >> shifts_full) & 0xF
g_idx = (k_start * BLOCK_K) // group_size
scale_ptrs = scales_ptr + offs_n * num_groups + g_idx
group_idx = (k_start * BLOCK_K) // group_size
scale_ptrs = scales_ptr + offs_n * num_groups + group_idx
scale_mask = offs_n < N
scales = tl.load(scale_ptrs, mask=scale_mask, other=1.0)
if HAS_ZP:
zp_ptrs = zp_ptr + offs_n * num_groups + g_idx
zp_ptrs = zp_ptr + offs_n * num_groups + group_idx
zp_raw = tl.load(zp_ptrs, mask=scale_mask, other=0.0)
b_fp = (b.to(scales.dtype) - zp_raw[:, None]) * scales[:, None]
else:
@@ -476,9 +476,6 @@ class RDNAHybridW4A16LinearKernel(MPLinearKernel):
if c.act_type not in (torch.float16, torch.bfloat16):
return False, "requires float16 or bfloat16 activations"
if c.has_g_idx:
return False, "does not support g_idx reordering"
gs = c.group_size
if gs not in SUPPORTED_GROUP_SIZES:
return (
@@ -541,7 +538,7 @@ class RDNAHybridW4A16LinearKernel(MPLinearKernel):
from vllm.utils.platform_utils import num_compute_units
c = self.config
w_q, w_s, w_zp, _ = self._get_weight_params(layer)
w_q, w_s, w_zp = self._get_weight_params(layer)
x_2d = x.reshape(-1, x.shape[-1])
N = w_q.shape[0]
@@ -126,10 +126,10 @@ def triton_w4a16_gemm_kernel(
b = (b >> shifts) & 0xF
# ---- Compute scale/zero group row index ----
g_idx = (k_start * BLOCK_K) // group_size
group_idx = (k_start * BLOCK_K) // group_size
# ---- Load scales: [BLOCK_N] → broadcast to [BLOCK_K, BLOCK_N] ----
scale_offset = g_idx * N + offs_sn
scale_offset = group_idx * N + offs_sn
scale_mask = offs_sn < N
scales = tl.load(scales_ptr + scale_offset, mask=scale_mask, other=1.0)
scales = tl.broadcast_to(scales[None, :], (BLOCK_K, BLOCK_N))
@@ -137,7 +137,7 @@ def triton_w4a16_gemm_kernel(
# ---- Load / compute zeros ----
if HAS_ZP:
# Load packed zeros row: [BLOCK_N//8] int32
zero_offset = g_idx * (N // 8) + offs_bn
zero_offset = group_idx * (N // 8) + offs_bn
zero_mask = offs_bn < N // 8
z_packed = tl.load(zeros_ptr + zero_offset, mask=zero_mask, other=0)
# Unpack to [BLOCK_N] using same interleave+shift pattern
@@ -237,7 +237,7 @@ def _triton_w4a16_gemm_impl(
BLOCK_M, BLOCK_N, BLOCK_K = 128, 128, 32
# The kernel loads scales/zeros for a single group per BLOCK_K tile
# (one g_idx per iteration). If BLOCK_K > group_size, rows at the tail
# (one group index per iteration). If BLOCK_K > group_size, rows at the tail
# of the tile dequantize with the wrong group's scales, silently
# corrupting the output. Clamp BLOCK_K to group_size to keep one
# scale group per tile.
@@ -340,13 +340,6 @@ class TritonW4A16LinearKernel(MPLinearKernel):
"(8 int4 values packed per int32)",
)
if c.has_g_idx:
return (
False,
"Activation reordering (g_idx) is not supported by "
"TritonW4A16LinearKernel",
)
gs = c.group_size
if (
gs not in TRITON_W4A16_SUPPORTED_GROUP_SIZES
@@ -471,7 +464,7 @@ class TritonW4A16LinearKernel(MPLinearKernel):
self, layer: torch.nn.Module, x: torch.Tensor, bias: torch.Tensor | None = None
) -> torch.Tensor:
c = self.config
w_q, w_s, w_zp, _ = self._get_weight_params(layer)
w_q, w_s, w_zp = self._get_weight_params(layer)
x_2d = x.reshape(-1, x.shape[-1]).contiguous()
out_shape = x.shape[:-1] + (c.partition_weight_shape[1],)
@@ -54,10 +54,8 @@ class XPUwNa16LinearKernel(MPLinearKernel):
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module):
# Default names since marlin requires empty parameters for these,
# Default name since marlin requires empty parameter for zp,
# TODO: remove this requirement from marlin (allow optional tensors)
if self.w_gidx_name is None:
self.w_gidx_name = "g_idx"
if self.w_zp_name is None:
self.w_zp_name = "w_zp"
@@ -90,17 +88,6 @@ class XPUwNa16LinearKernel(MPLinearKernel):
setattr(
layer, self.w_zp_name, Parameter(weight_zero_point, requires_grad=False)
)
if self.config.has_g_idx:
setattr(
layer,
self.w_gidx_name,
Parameter(
getattr(layer, self.w_gidx_name).data.t().contiguous(),
requires_grad=False,
),
)
else:
setattr(layer, self.w_gidx_name, None)
def apply_weights(
self,
@@ -109,7 +96,7 @@ class XPUwNa16LinearKernel(MPLinearKernel):
bias: torch.Tensor | None = None,
) -> torch.Tensor:
reshaped_x = x.reshape(-1, x.shape[-1])
w_q, w_s, w_zp, w_gidx = self._get_weight_params(layer)
w_q, w_s, w_zp = self._get_weight_params(layer)
out = torch.ops._xpu_C.int4_gemm_w4a16(
reshaped_x,
w_q.t(),
@@ -117,7 +104,7 @@ class XPUwNa16LinearKernel(MPLinearKernel):
w_s,
w_zp,
self.config.group_size,
w_gidx,
None, # Retained by the external XPU op ABI.
)
return out
@@ -224,7 +211,7 @@ class XPUW4A8IntLinearKernel(MPLinearKernel):
layer.weight_scale,
layer.weight_zero_point,
self.config.group_size,
None, # g_idx not currently supported
None, # Retained by the external XPU op ABI.
bias,
)
@@ -52,8 +52,6 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
"zentorch_woq_linear} are not registered.",
)
if c.has_g_idx:
return False, "ZentorchWNA16 does not support activation re-ordering."
return True, None
def _zentorch_woq_eligible(self, layer: torch.nn.Module) -> bool:
@@ -62,12 +60,6 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
Constraints (any failure -> ``cpu_gemm_wna16`` path via ``super()``
with ``layer`` untouched).
"""
if (
self.w_gidx_name is not None
and getattr(layer, self.w_gidx_name, None) is not None
) or (getattr(self.config, "has_g_idx", False)):
return False
weight_packed = getattr(layer, self.w_q_name, None)
weight_scale = getattr(layer, self.w_s_name, None)
if weight_packed is None or weight_scale is None:
@@ -112,7 +104,7 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
if not self._zentorch_woq_eligible(layer):
logger.info_once(
"[zen_cpu] ZentorchWNA16 fast path not eligible for this "
"layer (AWQ pack layout, g_idx, or non-int32 storage); "
"layer (AWQ pack layout or non-int32 storage); "
"falling back to CPUWNA16LinearKernel (cpu_gemm_wna16)."
)
super().process_weights_after_loading(layer)
@@ -121,9 +113,6 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
if (not self.config.zero_points) and (self.w_zp_name is not None):
setattr(layer, self.w_zp_name, None)
if (not self.config.has_g_idx) and (self.w_gidx_name is not None):
setattr(layer, self.w_gidx_name, None)
weight_q = getattr(layer, self.w_q_name)
weight_s = getattr(layer, self.w_s_name)
weight_packed = weight_q.data if hasattr(weight_q, "data") else weight_q