[Quant][Kernel] Remove GPTQ Group/Dynamic Activation Ordering (#54809)
Signed-off-by: Roderick-Wu <roderickwu2003@gmail.com> Signed-off-by: Roderick Wu <roderickwu2003@gmail.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -19,7 +19,6 @@ class MPLinearLayerConfig:
|
||||
act_type: torch.dtype
|
||||
group_size: int
|
||||
zero_points: bool
|
||||
has_g_idx: bool
|
||||
out_type: torch.dtype | None = None
|
||||
|
||||
|
||||
@@ -40,7 +39,6 @@ class MPLinearKernel(ABC):
|
||||
w_q_param_name: str,
|
||||
w_s_param_name: str,
|
||||
w_zp_param_name: str | None = None,
|
||||
w_gidx_param_name: str | None = None,
|
||||
) -> None:
|
||||
assert self.can_implement(c)
|
||||
self.config = c
|
||||
@@ -48,10 +46,7 @@ class MPLinearKernel(ABC):
|
||||
self.w_s_name = w_s_param_name
|
||||
if c.zero_points:
|
||||
assert w_zp_param_name is not None
|
||||
if c.has_g_idx:
|
||||
assert w_gidx_param_name is not None
|
||||
self.w_zp_name: str | None = w_zp_param_name
|
||||
self.w_gidx_name: str | None = w_gidx_param_name
|
||||
|
||||
@abstractmethod
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
@@ -83,12 +78,10 @@ class MPLinearKernel(ABC):
|
||||
) -> tuple[
|
||||
torch.Tensor, # w_q
|
||||
torch.Tensor, # w_s
|
||||
torch.Tensor | None, # w_zp,
|
||||
torch.Tensor | None, # w_gidx
|
||||
torch.Tensor | None, # w_zp
|
||||
]:
|
||||
return (
|
||||
getattr(layer, self.w_q_name),
|
||||
getattr(layer, self.w_s_name),
|
||||
getattr(layer, self.w_zp_name or "", None),
|
||||
getattr(layer, self.w_gidx_name or "", None),
|
||||
)
|
||||
|
||||
@@ -23,9 +23,6 @@ class AllSparkLinearKernel(MPLinearKernel):
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]:
|
||||
if c.has_g_idx:
|
||||
return False, "Act reordering currently not supported by AllSpark"
|
||||
|
||||
if c.zero_points:
|
||||
return False, "Zero points currently not supported by AllSpark"
|
||||
|
||||
@@ -91,7 +88,7 @@ class AllSparkLinearKernel(MPLinearKernel):
|
||||
) -> torch.Tensor:
|
||||
c = self.config
|
||||
gemm_args = self.gemm_args
|
||||
w_q, w_s, _, _ = self._get_weight_params(layer)
|
||||
w_q, w_s, _ = self._get_weight_params(layer)
|
||||
|
||||
reshaped_x = x.reshape(-1, x.shape[-1])
|
||||
out_shape = x.shape[:-1] + (c.partition_weight_shape[1],)
|
||||
|
||||
@@ -43,12 +43,6 @@ class ConchLinearKernel(MPLinearKernel):
|
||||
)
|
||||
return False, error_msg
|
||||
|
||||
if c.has_g_idx:
|
||||
return (
|
||||
False,
|
||||
"Activation reordering (g_idx) is not supported by ConchLinearKernel",
|
||||
)
|
||||
|
||||
if find_spec("conch") is None:
|
||||
error_msg = (
|
||||
"conch-triton-kernels is not installed, please "
|
||||
@@ -130,7 +124,7 @@ class ConchLinearKernel(MPLinearKernel):
|
||||
) -> torch.Tensor:
|
||||
from conch.ops.quantization.gemm import mixed_precision_gemm
|
||||
|
||||
w_q, w_s, w_zp, _ = self._get_weight_params(layer)
|
||||
w_q, w_s, w_zp = self._get_weight_params(layer)
|
||||
|
||||
# Map channelwise group_size=-1 to the actual input dimension K.
|
||||
# The conch kernel computes stride_mul = block_k / group_size;
|
||||
|
||||
@@ -144,9 +144,6 @@ class CPUWNA16LinearKernel(MPLinearKernel):
|
||||
if (not self.config.zero_points) and (self.w_zp_name is not None):
|
||||
setattr(layer, self.w_zp_name, None)
|
||||
|
||||
if (not self.config.has_g_idx) and (self.w_gidx_name is not None):
|
||||
setattr(layer, self.w_gidx_name, None)
|
||||
|
||||
weights = getattr(layer, self.w_q_name)
|
||||
# Require GPTQ pack format
|
||||
assert weights.input_dim == weights.packed_dim
|
||||
@@ -172,7 +169,6 @@ class CPUWNA16LinearKernel(MPLinearKernel):
|
||||
supports_riscv = current_platform.get_cpu_architecture() == CpuArchEnum.RISCV
|
||||
layer.use_w4a8 = (
|
||||
envs.VLLM_CPU_INT4_W4A8
|
||||
and not self.config.has_g_idx
|
||||
and self.config.act_type == torch.bfloat16
|
||||
and (supports_amx or supports_riscv)
|
||||
)
|
||||
@@ -189,7 +185,7 @@ class CPUWNA16LinearKernel(MPLinearKernel):
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
w_q, w_s, w_zp, w_gidx = self._get_weight_params(layer)
|
||||
w_q, w_s, w_zp = self._get_weight_params(layer)
|
||||
if layer.use_w4a8:
|
||||
x = ops.int4_scaled_mm_cpu(
|
||||
x=x,
|
||||
@@ -204,7 +200,6 @@ class CPUWNA16LinearKernel(MPLinearKernel):
|
||||
q_weight=w_q,
|
||||
scales=w_s,
|
||||
zeros=w_zp,
|
||||
g_idx=w_gidx,
|
||||
bias=bias,
|
||||
pack_factor=8, # 32 // 4
|
||||
isa_hint=layer.isa_hint,
|
||||
|
||||
@@ -39,9 +39,6 @@ class CutlassW4A8LinearKernel(MPLinearKernel):
|
||||
if c.act_type != torch.float8_e4m3fn:
|
||||
return False, "CUTLASS W4A8 only supports FP8 (e4m3) activations"
|
||||
|
||||
if c.has_g_idx:
|
||||
return False, "Act reordering not supported by CUTLASS W4A8"
|
||||
|
||||
if c.zero_points:
|
||||
return False, "Zero points not supported by CUTLASS W4A8"
|
||||
|
||||
@@ -109,7 +106,7 @@ class CutlassW4A8LinearKernel(MPLinearKernel):
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
c = self.config
|
||||
w_q, w_s, _, _ = self._get_weight_params(layer)
|
||||
w_q, w_s, _ = self._get_weight_params(layer)
|
||||
w_ch_s = layer.weight_chan_scale
|
||||
|
||||
x_2d = x.reshape(-1, x.shape[-1])
|
||||
|
||||
@@ -32,14 +32,6 @@ class ExllamaLinearKernel(MPLinearKernel):
|
||||
"Exllama is only supported on CUDA and ROCm",
|
||||
)
|
||||
|
||||
if c.has_g_idx and c.partition_weight_shape[0] != c.full_weight_shape[0]:
|
||||
return (
|
||||
False,
|
||||
"Act reordering currently not supported by Exllama, "
|
||||
"when the input features are partitioned across "
|
||||
"devices",
|
||||
)
|
||||
|
||||
if c.partition_weight_shape[1] % (32 // c.weight_type.size_bits) != 0:
|
||||
return (
|
||||
False,
|
||||
@@ -111,29 +103,11 @@ class ExllamaLinearKernel(MPLinearKernel):
|
||||
layer, self.w_zp_name, torch.nn.Parameter(zeros, requires_grad=False)
|
||||
)
|
||||
|
||||
if c.has_g_idx:
|
||||
|
||||
def transform_w_g_idx(x):
|
||||
# Exllama wants the permutation array instead of the group
|
||||
# indices
|
||||
return torch.argsort(x).to(torch.int)
|
||||
|
||||
self._transform_param(layer, self.w_gidx_name, transform_w_g_idx) # type: ignore
|
||||
else:
|
||||
self.w_gidx_name = "g_idx"
|
||||
empty_g_idx = torch.nn.Parameter(
|
||||
torch.empty((0,), dtype=torch.int, device=device), requires_grad=False
|
||||
)
|
||||
setattr(layer, self.w_gidx_name, empty_g_idx)
|
||||
|
||||
def transform_w_q(x):
|
||||
assert isinstance(x, BasevLLMParameter)
|
||||
assert self.w_gidx_name is not None
|
||||
g_idx = getattr(layer, self.w_gidx_name)
|
||||
|
||||
permute_param_layout_(x, input_dim=0, output_dim=1, packed_dim=0)
|
||||
x_cont = x.data.contiguous()
|
||||
ops.gptq_shuffle(x_cont, g_idx, c.weight_type.size_bits)
|
||||
ops.gptq_shuffle(x_cont, c.weight_type.size_bits)
|
||||
return x_cont
|
||||
|
||||
def transform_w_s(x):
|
||||
@@ -157,17 +131,15 @@ class ExllamaLinearKernel(MPLinearKernel):
|
||||
x_2d = x.reshape(-1, x.shape[-1])
|
||||
out_shape = x.shape[:-1] + (c.partition_weight_shape[1],)
|
||||
|
||||
w_q, w_s, w_zp, w_g_idx = self._get_weight_params(layer)
|
||||
|
||||
w_q, w_s, w_zp = self._get_weight_params(layer)
|
||||
# gptq_gemm supports GPTQv2 format by passing use_v2_format=True.
|
||||
# However, the MPLinearLayerConfig doesn't contain format info.
|
||||
# So hardcode GPTQv1 format here, to keep its behavior unchanged.
|
||||
use_v2_format = False
|
||||
|
||||
assert w_zp is not None, "Zero points are required by Exllama"
|
||||
assert w_g_idx is not None, "Group index is required by Exllama"
|
||||
output = ops.gptq_gemm(
|
||||
x_2d, w_q, w_zp, w_s, w_g_idx, True, use_v2_format, c.weight_type.size_bits
|
||||
x_2d, w_q, w_zp, w_s, True, use_v2_format, c.weight_type.size_bits
|
||||
)
|
||||
|
||||
if bias is not None:
|
||||
|
||||
@@ -21,8 +21,6 @@ class HummingLinearKernel(MPLinearKernel):
|
||||
return False, "Humming is only supported on CUDA"
|
||||
if not has_humming():
|
||||
return False, "Humming is not installed"
|
||||
if c.has_g_idx:
|
||||
return False, "Humming does not support act-order (g_idx)"
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from functools import partial
|
||||
|
||||
import torch
|
||||
|
||||
@@ -12,7 +11,6 @@ from vllm.model_executor.layers.quantization.utils.machete_utils import (
|
||||
query_machete_supported_quant_types,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
pack_quantized_values_into_int32,
|
||||
unpack_quantized_values_into_int32,
|
||||
)
|
||||
from vllm.model_executor.parameter import BasevLLMParameter, permute_param_layout_
|
||||
@@ -35,14 +33,6 @@ class MacheteLinearKernel(MPLinearKernel):
|
||||
if not current_platform.is_device_capability(90):
|
||||
return False, "Machete requires compute capability of 90 (Hopper)"
|
||||
|
||||
if c.has_g_idx and c.partition_weight_shape[0] != c.full_weight_shape[0]:
|
||||
return (
|
||||
False,
|
||||
"Act reordering currently not supported by Machete, "
|
||||
"when the input features are partitioned across "
|
||||
"devices",
|
||||
)
|
||||
|
||||
if c.weight_type not in query_machete_supported_quant_types(c.zero_points):
|
||||
return (
|
||||
False,
|
||||
@@ -70,29 +60,9 @@ class MacheteLinearKernel(MPLinearKernel):
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module):
|
||||
c = self.config
|
||||
|
||||
if c.has_g_idx:
|
||||
assert self.w_gidx_name is not None
|
||||
perm = torch.argsort(getattr(layer, self.w_gidx_name)).to(torch.int)
|
||||
|
||||
self.act_perm = lambda x: x[:, perm]
|
||||
# use `ops.permute_cols` if possible
|
||||
if (
|
||||
c.act_type in [torch.float16, torch.bfloat16]
|
||||
and c.partition_weight_shape[0] % 8 == 0
|
||||
):
|
||||
self.act_perm = partial(ops.permute_cols, perm=perm)
|
||||
|
||||
def transform_w_q(x):
|
||||
assert isinstance(x, BasevLLMParameter)
|
||||
permute_param_layout_(x, input_dim=0, output_dim=1, packed_dim=0)
|
||||
if c.has_g_idx:
|
||||
x_unpacked = unpack_quantized_values_into_int32(
|
||||
x.data, c.weight_type, packed_dim=0
|
||||
)
|
||||
x_perm = x_unpacked[perm, :]
|
||||
x.data = pack_quantized_values_into_int32(
|
||||
x_perm, c.weight_type, packed_dim=0
|
||||
)
|
||||
x.data = ops.machete_prepack_B(
|
||||
x.data.t().contiguous().t(),
|
||||
a_type=c.act_type,
|
||||
@@ -131,14 +101,11 @@ class MacheteLinearKernel(MPLinearKernel):
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
c = self.config
|
||||
w_q, w_s, w_zp, _ = self._get_weight_params(layer)
|
||||
w_q, w_s, w_zp = self._get_weight_params(layer)
|
||||
|
||||
x_2d = x.reshape(-1, x.shape[-1])
|
||||
out_shape = x.shape[:-1] + (c.partition_weight_shape[1],)
|
||||
|
||||
if c.has_g_idx:
|
||||
x_2d = self.act_perm(x_2d)
|
||||
|
||||
if c.zero_points:
|
||||
assert w_zp is not None
|
||||
else:
|
||||
|
||||
@@ -8,10 +8,8 @@ from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
MARLIN_SUPPORTED_GROUP_SIZES,
|
||||
apply_gptq_marlin_linear,
|
||||
check_marlin_supports_shape,
|
||||
marlin_act_int8_process_scales,
|
||||
marlin_is_k_full,
|
||||
marlin_make_empty_g_idx,
|
||||
marlin_make_empty,
|
||||
marlin_make_workspace_new,
|
||||
marlin_pad_dim,
|
||||
marlin_pad_qweight,
|
||||
@@ -19,13 +17,11 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
marlin_padded_nk,
|
||||
marlin_permute_bias,
|
||||
marlin_permute_scales,
|
||||
marlin_sort_g_idx,
|
||||
marlin_zero_points,
|
||||
query_marlin_supported_quant_types,
|
||||
unpack_cols,
|
||||
)
|
||||
from vllm.model_executor.parameter import BasevLLMParameter, permute_param_layout_
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
@@ -59,16 +55,6 @@ class MarlinLinearKernel(MPLinearKernel):
|
||||
f"{MARLIN_SUPPORTED_GROUP_SIZES}",
|
||||
)
|
||||
|
||||
if c.has_g_idx:
|
||||
# Act-order couples K to the full-model group layout, so tile
|
||||
# padding is not supported; keep the strict shape check.
|
||||
return check_marlin_supports_shape(
|
||||
c.partition_weight_shape[1], # out_features
|
||||
c.partition_weight_shape[0], # in_features
|
||||
c.full_weight_shape[0], # in_features
|
||||
c.group_size,
|
||||
)
|
||||
|
||||
# A group straddling TP ranks cannot be fixed by padding.
|
||||
if (
|
||||
c.group_size != -1
|
||||
@@ -102,25 +88,16 @@ class MarlinLinearKernel(MPLinearKernel):
|
||||
getattr(layer, self.w_s_name).data * 512
|
||||
)
|
||||
|
||||
row_parallel = c.partition_weight_shape[0] != c.full_weight_shape[0]
|
||||
self.is_k_full = marlin_is_k_full(c.has_g_idx, row_parallel)
|
||||
|
||||
size_k, size_n = c.partition_weight_shape
|
||||
if c.has_g_idx:
|
||||
# Act-order shapes were strictly validated in can_implement.
|
||||
padded_n, padded_k = size_n, size_k
|
||||
else:
|
||||
padded_n, padded_k = marlin_padded_nk(size_n, size_k, c.group_size)
|
||||
padded_n, padded_k = marlin_padded_nk(size_n, size_k, c.group_size)
|
||||
|
||||
# Allocate marlin workspace, reusing existing storage on reload.
|
||||
self.workspace = marlin_make_workspace_new(
|
||||
device, existing=getattr(self, "workspace", None)
|
||||
)
|
||||
|
||||
# Default names since marlin requires empty parameters for these,
|
||||
# Default name since marlin requires empty parameter for zp,
|
||||
# TODO: remove this requirement from marlin (allow optional tensors)
|
||||
if self.w_gidx_name is None:
|
||||
self.w_gidx_name = "g_idx"
|
||||
if self.w_zp_name is None:
|
||||
self.w_zp_name = "w_zp"
|
||||
|
||||
@@ -131,7 +108,6 @@ class MarlinLinearKernel(MPLinearKernel):
|
||||
marlin_pad_qweight(
|
||||
x.data.contiguous(), size_n, size_k, padded_n, padded_k
|
||||
),
|
||||
perm=layer.g_idx_sort_indices,
|
||||
size_k=padded_k,
|
||||
size_n=padded_n,
|
||||
num_bits=c.weight_type.size_bits,
|
||||
@@ -172,18 +148,6 @@ class MarlinLinearKernel(MPLinearKernel):
|
||||
layer.input_global_scale = None
|
||||
return x
|
||||
|
||||
if c.has_g_idx:
|
||||
g_idx, g_idx_sort_indices = marlin_sort_g_idx(
|
||||
getattr(layer, self.w_gidx_name)
|
||||
)
|
||||
self._transform_param(layer, self.w_gidx_name, lambda _: g_idx)
|
||||
replace_parameter(
|
||||
layer, "g_idx_sort_indices", g_idx_sort_indices, prefer_copy=True
|
||||
)
|
||||
else:
|
||||
setattr(layer, self.w_gidx_name, marlin_make_empty_g_idx(device))
|
||||
layer.g_idx_sort_indices = marlin_make_empty_g_idx(device)
|
||||
|
||||
if c.zero_points:
|
||||
grouped_k = size_k // c.group_size if c.group_size != -1 else 1
|
||||
padded_grouped_k = padded_k // c.group_size if c.group_size != -1 else 1
|
||||
@@ -211,7 +175,7 @@ class MarlinLinearKernel(MPLinearKernel):
|
||||
),
|
||||
)
|
||||
else:
|
||||
setattr(layer, self.w_zp_name, marlin_make_empty_g_idx(device))
|
||||
setattr(layer, self.w_zp_name, marlin_make_empty(device))
|
||||
self._transform_param(layer, self.w_q_name, transform_w_q)
|
||||
self._transform_param(layer, self.w_s_name, transform_w_s)
|
||||
|
||||
@@ -227,23 +191,16 @@ class MarlinLinearKernel(MPLinearKernel):
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
c = self.config
|
||||
w_q, w_s, w_zp, w_gidx = self._get_weight_params(layer)
|
||||
|
||||
# `process_weights_after_loading` will ensure w_zp and w_gidx are not
|
||||
# None for marlin
|
||||
|
||||
w_q, w_s, w_zp = self._get_weight_params(layer)
|
||||
return apply_gptq_marlin_linear(
|
||||
input=x,
|
||||
weight=w_q,
|
||||
weight_scale=w_s,
|
||||
weight_zp=w_zp, # type: ignore
|
||||
g_idx=w_gidx, # type: ignore
|
||||
g_idx_sort_indices=layer.g_idx_sort_indices,
|
||||
workspace=self.workspace,
|
||||
wtype=c.weight_type,
|
||||
input_size_per_partition=c.partition_weight_shape[0],
|
||||
output_size_per_partition=c.partition_weight_shape[1],
|
||||
is_k_full=self.is_k_full,
|
||||
input_global_scale=getattr(layer, "input_global_scale", None),
|
||||
bias=bias,
|
||||
input_dtype=c.act_type,
|
||||
|
||||
@@ -86,13 +86,6 @@ class RDNA3W4A16LinearKernel(MPLinearKernel):
|
||||
"W4A16 kernel (qzeros packing)",
|
||||
)
|
||||
|
||||
if c.has_g_idx and c.partition_weight_shape[0] != c.full_weight_shape[0]:
|
||||
return (
|
||||
False,
|
||||
"Act-order with TP-partitioned input features is not "
|
||||
"supported by the RDNA3 W4A16 kernel",
|
||||
)
|
||||
|
||||
return True, None
|
||||
|
||||
# ----- Weight prep (identical layout/shuffle as ExllamaLinearKernel) -----
|
||||
@@ -127,34 +120,14 @@ class RDNA3W4A16LinearKernel(MPLinearKernel):
|
||||
layer, self.w_zp_name, torch.nn.Parameter(zeros, requires_grad=False)
|
||||
)
|
||||
|
||||
# Act-order: convert g_idx to the inverse permutation array exllama
|
||||
# expects (kernel reads a[perm[k]] instead of using groups indirected
|
||||
# by g_idx[k]).
|
||||
if c.has_g_idx:
|
||||
|
||||
def transform_w_g_idx(x):
|
||||
return torch.argsort(x).to(torch.int)
|
||||
|
||||
self._transform_param(layer, self.w_gidx_name, transform_w_g_idx) # type: ignore
|
||||
else:
|
||||
self.w_gidx_name = "g_idx"
|
||||
empty_g_idx = torch.nn.Parameter(
|
||||
torch.empty((0,), dtype=torch.int, device=device),
|
||||
requires_grad=False,
|
||||
)
|
||||
setattr(layer, self.w_gidx_name, empty_g_idx)
|
||||
|
||||
def transform_w_q(x):
|
||||
assert isinstance(x, BasevLLMParameter)
|
||||
assert self.w_gidx_name is not None
|
||||
g_idx = getattr(layer, self.w_gidx_name)
|
||||
|
||||
permute_param_layout_(x, input_dim=0, output_dim=1, packed_dim=0)
|
||||
x_cont = x.data.contiguous()
|
||||
# Same 4-bit shuffle as exllama. The RDNA3 kernel reads weights in
|
||||
# the same shuffled int32 layout and uses the (qa & 0x000F000F)
|
||||
# bit-trick on top.
|
||||
ops.gptq_shuffle(x_cont, g_idx, c.weight_type.size_bits)
|
||||
ops.gptq_shuffle(x_cont, c.weight_type.size_bits)
|
||||
return x_cont
|
||||
|
||||
def transform_w_s(x):
|
||||
@@ -181,12 +154,9 @@ class RDNA3W4A16LinearKernel(MPLinearKernel):
|
||||
x_2d = x.reshape(-1, x.shape[-1])
|
||||
out_shape = x.shape[:-1] + (c.partition_weight_shape[1],)
|
||||
|
||||
w_q, w_s, w_zp, w_g_idx = self._get_weight_params(layer)
|
||||
|
||||
w_q, w_s, w_zp = self._get_weight_params(layer)
|
||||
assert w_zp is not None, "Zero points are required by RDNA3 W4A16"
|
||||
assert w_g_idx is not None, "g_idx tensor (possibly empty) required"
|
||||
|
||||
output = ops.gptq_gemm_rdna3(x_2d, w_q, w_zp, w_s, w_g_idx, False)
|
||||
output = ops.gptq_gemm_rdna3(x_2d, w_q, w_zp, w_s, False)
|
||||
|
||||
if bias is not None:
|
||||
output.add_(bias)
|
||||
|
||||
@@ -142,13 +142,13 @@ def _triton_w4a16_skinny_fmt_kernel(
|
||||
b = tl.interleave(b, b)
|
||||
b = (b >> shifts_full) & 0xF
|
||||
|
||||
g_idx = (k_start * BLOCK_K) // group_size
|
||||
scale_ptrs = scales_ptr + offs_n * num_groups + g_idx
|
||||
group_idx = (k_start * BLOCK_K) // group_size
|
||||
scale_ptrs = scales_ptr + offs_n * num_groups + group_idx
|
||||
scale_mask = offs_n < N
|
||||
scales = tl.load(scale_ptrs, mask=scale_mask, other=1.0)
|
||||
|
||||
if HAS_ZP:
|
||||
zp_ptrs = zp_ptr + offs_n * num_groups + g_idx
|
||||
zp_ptrs = zp_ptr + offs_n * num_groups + group_idx
|
||||
zp_raw = tl.load(zp_ptrs, mask=scale_mask, other=0.0)
|
||||
b_fp = (b.to(scales.dtype) - zp_raw[:, None]) * scales[:, None]
|
||||
else:
|
||||
@@ -476,9 +476,6 @@ class RDNAHybridW4A16LinearKernel(MPLinearKernel):
|
||||
if c.act_type not in (torch.float16, torch.bfloat16):
|
||||
return False, "requires float16 or bfloat16 activations"
|
||||
|
||||
if c.has_g_idx:
|
||||
return False, "does not support g_idx reordering"
|
||||
|
||||
gs = c.group_size
|
||||
if gs not in SUPPORTED_GROUP_SIZES:
|
||||
return (
|
||||
@@ -541,7 +538,7 @@ class RDNAHybridW4A16LinearKernel(MPLinearKernel):
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
|
||||
c = self.config
|
||||
w_q, w_s, w_zp, _ = self._get_weight_params(layer)
|
||||
w_q, w_s, w_zp = self._get_weight_params(layer)
|
||||
|
||||
x_2d = x.reshape(-1, x.shape[-1])
|
||||
N = w_q.shape[0]
|
||||
|
||||
@@ -126,10 +126,10 @@ def triton_w4a16_gemm_kernel(
|
||||
b = (b >> shifts) & 0xF
|
||||
|
||||
# ---- Compute scale/zero group row index ----
|
||||
g_idx = (k_start * BLOCK_K) // group_size
|
||||
group_idx = (k_start * BLOCK_K) // group_size
|
||||
|
||||
# ---- Load scales: [BLOCK_N] → broadcast to [BLOCK_K, BLOCK_N] ----
|
||||
scale_offset = g_idx * N + offs_sn
|
||||
scale_offset = group_idx * N + offs_sn
|
||||
scale_mask = offs_sn < N
|
||||
scales = tl.load(scales_ptr + scale_offset, mask=scale_mask, other=1.0)
|
||||
scales = tl.broadcast_to(scales[None, :], (BLOCK_K, BLOCK_N))
|
||||
@@ -137,7 +137,7 @@ def triton_w4a16_gemm_kernel(
|
||||
# ---- Load / compute zeros ----
|
||||
if HAS_ZP:
|
||||
# Load packed zeros row: [BLOCK_N//8] int32
|
||||
zero_offset = g_idx * (N // 8) + offs_bn
|
||||
zero_offset = group_idx * (N // 8) + offs_bn
|
||||
zero_mask = offs_bn < N // 8
|
||||
z_packed = tl.load(zeros_ptr + zero_offset, mask=zero_mask, other=0)
|
||||
# Unpack to [BLOCK_N] using same interleave+shift pattern
|
||||
@@ -237,7 +237,7 @@ def _triton_w4a16_gemm_impl(
|
||||
BLOCK_M, BLOCK_N, BLOCK_K = 128, 128, 32
|
||||
|
||||
# The kernel loads scales/zeros for a single group per BLOCK_K tile
|
||||
# (one g_idx per iteration). If BLOCK_K > group_size, rows at the tail
|
||||
# (one group index per iteration). If BLOCK_K > group_size, rows at the tail
|
||||
# of the tile dequantize with the wrong group's scales, silently
|
||||
# corrupting the output. Clamp BLOCK_K to group_size to keep one
|
||||
# scale group per tile.
|
||||
@@ -340,13 +340,6 @@ class TritonW4A16LinearKernel(MPLinearKernel):
|
||||
"(8 int4 values packed per int32)",
|
||||
)
|
||||
|
||||
if c.has_g_idx:
|
||||
return (
|
||||
False,
|
||||
"Activation reordering (g_idx) is not supported by "
|
||||
"TritonW4A16LinearKernel",
|
||||
)
|
||||
|
||||
gs = c.group_size
|
||||
if (
|
||||
gs not in TRITON_W4A16_SUPPORTED_GROUP_SIZES
|
||||
@@ -471,7 +464,7 @@ class TritonW4A16LinearKernel(MPLinearKernel):
|
||||
self, layer: torch.nn.Module, x: torch.Tensor, bias: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
c = self.config
|
||||
w_q, w_s, w_zp, _ = self._get_weight_params(layer)
|
||||
w_q, w_s, w_zp = self._get_weight_params(layer)
|
||||
|
||||
x_2d = x.reshape(-1, x.shape[-1]).contiguous()
|
||||
out_shape = x.shape[:-1] + (c.partition_weight_shape[1],)
|
||||
|
||||
@@ -54,10 +54,8 @@ class XPUwNa16LinearKernel(MPLinearKernel):
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module):
|
||||
# Default names since marlin requires empty parameters for these,
|
||||
# Default name since marlin requires empty parameter for zp,
|
||||
# TODO: remove this requirement from marlin (allow optional tensors)
|
||||
if self.w_gidx_name is None:
|
||||
self.w_gidx_name = "g_idx"
|
||||
if self.w_zp_name is None:
|
||||
self.w_zp_name = "w_zp"
|
||||
|
||||
@@ -90,17 +88,6 @@ class XPUwNa16LinearKernel(MPLinearKernel):
|
||||
setattr(
|
||||
layer, self.w_zp_name, Parameter(weight_zero_point, requires_grad=False)
|
||||
)
|
||||
if self.config.has_g_idx:
|
||||
setattr(
|
||||
layer,
|
||||
self.w_gidx_name,
|
||||
Parameter(
|
||||
getattr(layer, self.w_gidx_name).data.t().contiguous(),
|
||||
requires_grad=False,
|
||||
),
|
||||
)
|
||||
else:
|
||||
setattr(layer, self.w_gidx_name, None)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
@@ -109,7 +96,7 @@ class XPUwNa16LinearKernel(MPLinearKernel):
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
reshaped_x = x.reshape(-1, x.shape[-1])
|
||||
w_q, w_s, w_zp, w_gidx = self._get_weight_params(layer)
|
||||
w_q, w_s, w_zp = self._get_weight_params(layer)
|
||||
out = torch.ops._xpu_C.int4_gemm_w4a16(
|
||||
reshaped_x,
|
||||
w_q.t(),
|
||||
@@ -117,7 +104,7 @@ class XPUwNa16LinearKernel(MPLinearKernel):
|
||||
w_s,
|
||||
w_zp,
|
||||
self.config.group_size,
|
||||
w_gidx,
|
||||
None, # Retained by the external XPU op ABI.
|
||||
)
|
||||
return out
|
||||
|
||||
@@ -224,7 +211,7 @@ class XPUW4A8IntLinearKernel(MPLinearKernel):
|
||||
layer.weight_scale,
|
||||
layer.weight_zero_point,
|
||||
self.config.group_size,
|
||||
None, # g_idx not currently supported
|
||||
None, # Retained by the external XPU op ABI.
|
||||
bias,
|
||||
)
|
||||
|
||||
|
||||
@@ -52,8 +52,6 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
|
||||
"zentorch_woq_linear} are not registered.",
|
||||
)
|
||||
|
||||
if c.has_g_idx:
|
||||
return False, "ZentorchWNA16 does not support activation re-ordering."
|
||||
return True, None
|
||||
|
||||
def _zentorch_woq_eligible(self, layer: torch.nn.Module) -> bool:
|
||||
@@ -62,12 +60,6 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
|
||||
Constraints (any failure -> ``cpu_gemm_wna16`` path via ``super()``
|
||||
with ``layer`` untouched).
|
||||
"""
|
||||
if (
|
||||
self.w_gidx_name is not None
|
||||
and getattr(layer, self.w_gidx_name, None) is not None
|
||||
) or (getattr(self.config, "has_g_idx", False)):
|
||||
return False
|
||||
|
||||
weight_packed = getattr(layer, self.w_q_name, None)
|
||||
weight_scale = getattr(layer, self.w_s_name, None)
|
||||
if weight_packed is None or weight_scale is None:
|
||||
@@ -112,7 +104,7 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
|
||||
if not self._zentorch_woq_eligible(layer):
|
||||
logger.info_once(
|
||||
"[zen_cpu] ZentorchWNA16 fast path not eligible for this "
|
||||
"layer (AWQ pack layout, g_idx, or non-int32 storage); "
|
||||
"layer (AWQ pack layout or non-int32 storage); "
|
||||
"falling back to CPUWNA16LinearKernel (cpu_gemm_wna16)."
|
||||
)
|
||||
super().process_weights_after_loading(layer)
|
||||
@@ -121,9 +113,6 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
|
||||
if (not self.config.zero_points) and (self.w_zp_name is not None):
|
||||
setattr(layer, self.w_zp_name, None)
|
||||
|
||||
if (not self.config.has_g_idx) and (self.w_gidx_name is not None):
|
||||
setattr(layer, self.w_gidx_name, None)
|
||||
|
||||
weight_q = getattr(layer, self.w_q_name)
|
||||
weight_s = getattr(layer, self.w_s_name)
|
||||
weight_packed = weight_q.data if hasattr(weight_q, "data") else weight_q
|
||||
|
||||
Reference in New Issue
Block a user