[Kernel] Consolidate Marlin thread-tile padding across all dense Marlin paths (#45295)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Michael Goin
2026-06-12 13:46:21 -07:00
committed by GitHub
parent 78739c1946
commit 9eaacb23ec
7 changed files with 770 additions and 66 deletions
@@ -13,6 +13,10 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
marlin_is_k_full,
marlin_make_empty_g_idx,
marlin_make_workspace_new,
marlin_pad_dim,
marlin_pad_qweight,
marlin_pad_scales,
marlin_padded_nk,
marlin_permute_bias,
marlin_permute_scales,
marlin_sort_g_idx,
@@ -54,12 +58,29 @@ class MarlinLinearKernel(MPLinearKernel):
f"{MARLIN_SUPPORTED_GROUP_SIZES}",
)
return check_marlin_supports_shape(
c.partition_weight_shape[1], # out_features
c.partition_weight_shape[0], # in_features
c.full_weight_shape[0], # in_features
c.group_size,
)
if c.has_g_idx:
# Act-order couples K to the full-model group layout, so tile
# padding is not supported; keep the strict shape check.
return check_marlin_supports_shape(
c.partition_weight_shape[1], # out_features
c.partition_weight_shape[0], # in_features
c.full_weight_shape[0], # in_features
c.group_size,
)
# A group straddling TP ranks cannot be fixed by padding.
if (
c.group_size != -1
and c.group_size < c.full_weight_shape[0]
and c.partition_weight_shape[0] % c.group_size != 0
):
return False, (
f"in_features per partition {c.partition_weight_shape[0]} is "
f"not divisible by group_size = {c.group_size}."
)
# Tile misalignment is fixed by zero-padding at weight prep.
return True, None
# note assumes that
# `weight_packed` is: {input_dim = 0, output_dim = 1, packed_dim = 0}
@@ -83,6 +104,13 @@ class MarlinLinearKernel(MPLinearKernel):
row_parallel = c.partition_weight_shape[0] != c.full_weight_shape[0]
self.is_k_full = marlin_is_k_full(c.has_g_idx, row_parallel)
size_k, size_n = c.partition_weight_shape
if c.has_g_idx:
# Act-order shapes were strictly validated in can_implement.
padded_n, padded_k = size_n, size_k
else:
padded_n, padded_k = marlin_padded_nk(size_n, size_k, c.group_size)
# Allocate marlin workspace.
self.workspace = marlin_make_workspace_new(device)
@@ -97,10 +125,12 @@ class MarlinLinearKernel(MPLinearKernel):
assert isinstance(x, BasevLLMParameter)
permute_param_layout_(x, input_dim=0, output_dim=1, packed_dim=0)
x.data = ops.gptq_marlin_repack(
x.data.contiguous(),
marlin_pad_qweight(
x.data.contiguous(), size_n, size_k, padded_n, padded_k
),
perm=layer.g_idx_sort_indices,
size_k=c.partition_weight_shape[0],
size_n=c.partition_weight_shape[1],
size_k=padded_k,
size_n=padded_n,
num_bits=c.weight_type.size_bits,
is_a_8bit=is_a_8bit,
)
@@ -110,9 +140,16 @@ class MarlinLinearKernel(MPLinearKernel):
assert isinstance(x, BasevLLMParameter)
permute_param_layout_(x, input_dim=0, output_dim=1)
x.data = marlin_permute_scales(
x.data.contiguous(),
size_k=c.partition_weight_shape[0],
size_n=c.partition_weight_shape[1],
marlin_pad_scales(
x.data.contiguous(),
size_n,
size_k,
padded_n,
padded_k,
c.group_size,
),
size_k=padded_k,
size_n=padded_n,
group_size=c.group_size,
is_a_8bit=is_a_8bit,
)
@@ -143,21 +180,27 @@ class MarlinLinearKernel(MPLinearKernel):
layer.g_idx_sort_indices = marlin_make_empty_g_idx(device)
if c.zero_points:
grouped_k = (
c.partition_weight_shape[0] // c.group_size if c.group_size != -1 else 1
)
grouped_k = size_k // c.group_size if c.group_size != -1 else 1
padded_grouped_k = padded_k // c.group_size if c.group_size != -1 else 1
self._transform_param(
layer,
self.w_zp_name,
lambda x: marlin_zero_points(
unpack_cols(
x.t(),
c.weight_type.size_bits,
grouped_k,
c.partition_weight_shape[1],
marlin_pad_scales(
unpack_cols(
x.t(),
c.weight_type.size_bits,
grouped_k,
size_n,
),
size_n,
size_k,
padded_n,
padded_k,
c.group_size,
),
size_k=grouped_k,
size_n=c.partition_weight_shape[1],
size_k=padded_grouped_k,
size_n=padded_n,
num_bits=c.weight_type.size_bits,
is_a_8bit=is_a_8bit,
),
@@ -168,7 +211,9 @@ class MarlinLinearKernel(MPLinearKernel):
self._transform_param(layer, self.w_s_name, transform_w_s)
if hasattr(layer, "bias") and layer.bias is not None:
layer.bias.data = marlin_permute_bias(layer.bias)
layer.bias.data = marlin_permute_bias(
marlin_pad_dim(layer.bias, size_n, padded_n)
)
def apply_weights(
self,