[ROCm][Perf] W4A16: keep skinny GEMM zero-points packed 4-bit (#54965)

Signed-off-by: Matthias Gehre <matthias.gehre@amd.com>
This commit is contained in:
Matthias Gehre
2026-09-14 17:40:25 +08:00
committed by GitHub
parent 238cb2b191
commit dc2e8f1157
4 changed files with 117 additions and 45 deletions
@@ -77,7 +77,7 @@ def _triton_w4a16_skinny_fmt_kernel(
a_ptr, # [M, K] fp16/bf16 activations
b_ptr, # [N, K//8] int32 packed (ExLlama shuffle, K is packed dim)
scales_ptr, # [N, K//G] fp16/bf16 scales (skinny layout)
zp_ptr, # [N, K//G] fp16/bf16 raw zero-points (when HAS_ZP=True)
zp_ptr, # [N//8, K//G] int32 zero-points (when HAS_ZP=True)
c_ptr, # [M, N] fp16/bf16 output
# Dimensions
M,
@@ -103,8 +103,9 @@ def _triton_w4a16_skinny_fmt_kernel(
| (val[1]<<16) | (val[3]<<20) | (val[5]<<24) | (val[7]<<28)
Scales are [N, K//G] (skinny layout, NOT transposed).
When HAS_ZP=True, raw zero-points zp_raw are loaded from zp_ptr [N, K//G]
and subtracted directly: (nibble - zp_raw) * scale.
When HAS_ZP=True, zp_ptr holds [N//8, K//G] int32 with row n's raw
zero-point at word[n//8] bits 4*(n%8), and dequant is
(nibble - zp_raw) * scale.
When HAS_ZP=False, only the constant ZP_BIAS is subtracted (symmetric).
"""
pid_m = tl.program_id(0)
@@ -148,9 +149,10 @@ def _triton_w4a16_skinny_fmt_kernel(
scales = tl.load(scale_ptrs, mask=scale_mask, other=1.0)
if HAS_ZP:
zp_ptrs = zp_ptr + offs_n * num_groups + group_idx
zp_raw = tl.load(zp_ptrs, mask=scale_mask, other=0.0)
b_fp = (b.to(scales.dtype) - zp_raw[:, None]) * scales[:, None]
zp_ptrs = zp_ptr + (offs_n // 8) * num_groups + group_idx
zp_word = tl.load(zp_ptrs, mask=scale_mask, other=0)
zp_raw = (zp_word >> (4 * (offs_n % 8))) & 0xF
b_fp = (b - zp_raw[:, None]).to(scales.dtype) * scales[:, None]
else:
b_fp = (b - ZP_BIAS).to(scales.dtype) * scales[:, None]
@@ -192,7 +194,7 @@ def triton_w4a16_skinny_fmt_gemm(
scales: torch.Tensor, # [N, K//G] fp16/bf16
group_size: int,
zp_bias: int = 8,
zp: torch.Tensor | None = None, # [N, K//G] per-group zero-points
zp: torch.Tensor | None = None, # [N//8, K//G] int32 zero-points
) -> torch.Tensor:
"""
Fused W4A16 GEMM reading from skinny weight format [N, K//8].
@@ -203,8 +205,8 @@ def triton_w4a16_skinny_fmt_gemm(
scales: Per-group scales [N, K//G], same dtype as a.
group_size: Quantization group size (resolved from -1 to K by caller).
zp_bias: Constant zero bias (default 8 for unsigned int4).
zp: Raw per-group zero-points [N, K//G] (asymmetric),
stored as zp_raw in activation dtype. When provided,
zp: Raw per-group zero-points [N//8, K//G] int32, row n at
word[n//8] bits 4*(n%8) (asymmetric). When provided,
dequant is (nibble - zp_raw) * scale.
Returns:
@@ -225,8 +227,9 @@ def triton_w4a16_skinny_fmt_gemm(
)
if zp is not None:
assert zp.is_contiguous(), "Zero-points must be contiguous"
assert zp.shape == (N, num_groups), (
f"zp shape mismatch: {zp.shape} vs ({N}, {num_groups})"
assert N % 8 == 0, f"N must be divisible by 8 for packed zp, got {N}"
assert zp.shape == (N // 8, num_groups), (
f"zp shape mismatch: {zp.shape} vs ({N // 8}, {num_groups})"
)
has_zp = zp is not None
@@ -381,7 +384,10 @@ def _rdna_hybrid_w4a16_apply_impl(
cu_count: int,
group_size: int,
) -> torch.Tensor:
"""Dispatch between skinny GEMM and Triton based on batch size M."""
"""Dispatch between skinny GEMM and Triton based on batch size M.
``w_zp`` is [N//8, K//G] int32 for asymmetric layers, None for symmetric.
"""
import vllm._custom_ops as ops
M = x_2d.shape[0]
@@ -520,10 +526,7 @@ class RDNAHybridW4A16LinearKernel(MPLinearKernel):
assert self.w_zp_name is not None
w_zp_raw = getattr(layer, self.w_zp_name)
permute_param_layout_(w_zp_raw, input_dim=1, output_dim=0, packed_dim=0)
zp_unpacked = unpack_quantized_values_into_int32(
w_zp_raw.data, c.weight_type, packed_dim=0
)
w_zp = zp_unpacked.to(c.act_type).contiguous()
w_zp = w_zp_raw.data.contiguous()
self._transform_param(layer, self.w_zp_name, lambda x: w_zp)
self._transform_param(layer, self.w_q_name, lambda x: w_q_skinny)