From d468322dc1c89377f65488973d2f1927e9bbe5cb Mon Sep 17 00:00:00 2001 From: jatseng-ai Date: Fri, 10 Apr 2026 03:25:27 -0700 Subject: [PATCH] [Kernel][Hardware][AMD] Add TritonW4A16LinearKernel for ROCm (#37352) Signed-off-by: jatseng-ai Signed-off-by: jatseng-ai Signed-off-by: Matthias Gehre Co-authored-by: Claude Co-authored-by: Matthias Gehre --- .../test_rocm_compressed_tensors_w4a16.py | 35 ++ .../kernels/quantization/test_triton_w4a16.py | 304 +++++++++++++ .../test_w4a16_kernel_selection.py | 53 +++ .../model_executor/kernels/linear/__init__.py | 5 + .../linear/mixed_precision/__init__.py | 4 + .../linear/mixed_precision/triton_w4a16.py | 430 ++++++++++++++++++ 6 files changed, 831 insertions(+) create mode 100644 tests/kernels/quantization/test_rocm_compressed_tensors_w4a16.py create mode 100644 tests/kernels/quantization/test_triton_w4a16.py create mode 100644 tests/kernels/quantization/test_w4a16_kernel_selection.py create mode 100644 vllm/model_executor/kernels/linear/mixed_precision/triton_w4a16.py diff --git a/tests/kernels/quantization/test_rocm_compressed_tensors_w4a16.py b/tests/kernels/quantization/test_rocm_compressed_tensors_w4a16.py new file mode 100644 index 0000000000..a9b35a4ea6 --- /dev/null +++ b/tests/kernels/quantization/test_rocm_compressed_tensors_w4a16.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""End-to-end smoke test for CT W4A16 models on ROCm. + +This validates that a real compressed-tensors W4A16 model can run inference +end-to-end (which will exercise the Triton W4A16 kernel when selected). + +Run `pytest tests/kernels/quantization/test_rocm_compressed_tensors_w4a16.py`. +""" + +import pytest + +from vllm.platforms import current_platform + + +@pytest.mark.parametrize( + "model_path", + [ + # Listed in tests/weight_loading/models.txt + "nm-testing/tinyllama-oneshot-w4a16-group128-v2", + ], +) +@pytest.mark.parametrize("max_tokens", [32]) +@pytest.mark.skipif(not current_platform.is_rocm(), reason="Should only run on ROCm") +def test_rocm_compressed_tensors_w4a16_e2e( + vllm_runner, example_prompts, model_path, max_tokens +): + # Use fp16 activations for maximum compatibility. + # gpu_memory_utilization lowered to work on shared nodes. + with vllm_runner( + model_path, dtype="float16", gpu_memory_utilization=0.3 + ) as vllm_model: + # If the W4A16 kernel is broken, this will typically throw. + vllm_model.generate_greedy(example_prompts, max_tokens=max_tokens) diff --git a/tests/kernels/quantization/test_triton_w4a16.py b/tests/kernels/quantization/test_triton_w4a16.py new file mode 100644 index 0000000000..6502f52442 --- /dev/null +++ b/tests/kernels/quantization/test_triton_w4a16.py @@ -0,0 +1,304 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for the ROCm Triton W4A16 GEMM kernel. + +Run `pytest tests/kernels/quantization/test_triton_w4a16.py`. +""" + +import importlib + +import pytest +import torch + +from vllm.platforms import current_platform +from vllm.utils.torch_utils import set_random_seed + +# This test module is ROCm/Triton specific. Avoid import-time failures on +# non-ROCm or environments without Triton by skipping early. +if not current_platform.is_rocm(): + pytest.skip("ROCm only", allow_module_level=True) + +pytest.importorskip("triton") + +device = "cuda" + +triton_w4a16_module = importlib.import_module( + "vllm.model_executor.kernels.linear.mixed_precision.triton_w4a16" +) +triton_w4a16_gemm = triton_w4a16_module.triton_w4a16_gemm +TritonW4A16LinearKernel = triton_w4a16_module.TritonW4A16LinearKernel + + +def _pack_int4_along_n(w_int4_kn: torch.Tensor) -> torch.Tensor: + """Pack int4 values along N: [K, N] -> [K, N//8] int32.""" + assert w_int4_kn.dtype == torch.int32 + K, N = w_int4_kn.shape + assert N % 8 == 0 + shifts = torch.arange(8, device=w_int4_kn.device, dtype=torch.int32) * 4 + return torch.sum( + (w_int4_kn.view(K, N // 8, 8) & 0xF) << shifts, + dim=2, + dtype=torch.int32, + ).contiguous() + + +def _unpack_int4_along_n(w_packed_kn8: torch.Tensor) -> torch.Tensor: + """Unpack int4 values along N: [K, N//8] -> [K, N] int32.""" + assert w_packed_kn8.dtype == torch.int32 + K, N8 = w_packed_kn8.shape + shifts = torch.arange(8, device=w_packed_kn8.device, dtype=torch.int32) * 4 + nibbles = (w_packed_kn8.unsqueeze(-1) >> shifts) & 0xF + return nibbles.reshape(K, N8 * 8) + + +def _pack_int4_along_k_to_ckpt(w_int4_kn: torch.Tensor) -> torch.Tensor: + """Pack int4 values along K into CT checkpoint layout: [K,N] -> [N, K//8].""" + assert w_int4_kn.dtype == torch.int32 + K, N = w_int4_kn.shape + assert K % 8 == 0 + out = torch.zeros((N, K // 8), dtype=torch.int32, device=w_int4_kn.device) + for i in range(8): + out |= (w_int4_kn[i::8, :].t() & 0xF) << (i * 4) + return out.contiguous() + + +def _w4a16_reference( + a_mk: torch.Tensor, + b_packed_kn8: torch.Tensor, + scales_gn: torch.Tensor, + *, + group_size: int, + qzeros_gn8: torch.Tensor | None, + zp_bias: int, +) -> torch.Tensor: + """Reference implementation for W4A16. + + a_mk: [M,K] fp16/bf16 + b_packed_kn8: [K, N//8] int32, N-packed int4 weights + scales_gn: [K//G, N] fp16/bf16 + qzeros_gn8: [K//G, N//8] int32, N-packed int4 zeros, or None + """ + assert a_mk.dtype in (torch.float16, torch.bfloat16) + assert b_packed_kn8.dtype == torch.int32 + assert scales_gn.dtype == a_mk.dtype + + M, K = a_mk.shape + N = b_packed_kn8.shape[1] * 8 + assert b_packed_kn8.shape[0] == K + + assert group_size > 0 and K % group_size == 0 + G = group_size + num_groups = K // G + assert scales_gn.shape == (num_groups, N) + + w_int4 = _unpack_int4_along_n(b_packed_kn8) # [K,N] + if qzeros_gn8 is None: + z_full = torch.full((K, N), zp_bias, dtype=torch.int32, device=a_mk.device) + else: + assert qzeros_gn8.shape == (num_groups, N // 8) + z_gn = _unpack_int4_along_n(qzeros_gn8) # [G,N] in groups + z_full = z_gn.repeat_interleave(G, dim=0) # [K,N] + + s_full = scales_gn.repeat_interleave(G, dim=0).to(torch.float32) # [K,N] + w_fp = (w_int4 - z_full).to(torch.float32) * s_full # [K,N] + + out = a_mk.to(torch.float32) @ w_fp # [M,N] + return out.to(a_mk.dtype) + + +@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only") +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +@pytest.mark.parametrize( + "M,K,N,G,has_zp", + [ + (1, 256, 256, 32, False), + (17, 256, 512, 32, False), + (32, 512, 256, 64, False), + (33, 512, 512, 128, False), + (64, 1024, 256, 256, False), + (128, 256, 1024, 32, True), + (64, 512, 512, 64, True), + ], +) +def test_triton_w4a16_gemm_matches_reference(dtype, M, K, N, G, has_zp): + if not torch.cuda.is_available(): + pytest.skip("CUDA/HIP device not available") + if N % 8 != 0 or K % G != 0: + pytest.skip("Invalid test shape") + + set_random_seed(0) + + a = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype) + w_int4 = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32) + b_packed = _pack_int4_along_n(w_int4) + + scales = (0.05 * torch.rand((K // G, N), device=device, dtype=torch.float32)).to( + dtype + ) + + qzeros = None + if has_zp: + zeros_int4 = torch.randint(0, 16, (K // G, N), device=device, dtype=torch.int32) + qzeros = _pack_int4_along_n(zeros_int4) + + out = triton_w4a16_gemm( + a=a, + b_q=b_packed, + scales=scales, + qzeros=qzeros, + group_size=G, + zp_bias=8, + ) + ref = _w4a16_reference( + a, + b_packed, + scales, + group_size=G, + qzeros_gn8=qzeros, + zp_bias=8, + ) + + torch.testing.assert_close(out, ref, rtol=1e-2, atol=1e-2) + + +@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only") +def test_triton_w4a16_gemm_requires_contiguous_inputs(): + if not torch.cuda.is_available(): + pytest.skip("CUDA/HIP device not available") + + set_random_seed(0) + M, K, N, G = 32, 256, 256, 32 + a = torch.randn((K, M), device=device, dtype=torch.float16).t() # non-contiguous + w_int4 = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32) + b_packed = _pack_int4_along_n(w_int4) + scales = torch.rand((K // G, N), device=device, dtype=torch.float16) + + with pytest.raises(AssertionError): + triton_w4a16_gemm( + a=a, + b_q=b_packed, + scales=scales, + qzeros=None, + group_size=G, + zp_bias=8, + ) + + +@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only") +def test_triton_w4a16_process_weights_after_loading_repacks_layout(): + if not torch.cuda.is_available(): + pytest.skip("CUDA/HIP device not available") + + from vllm.config import VllmConfig, set_current_vllm_config + from vllm.distributed import ( + ensure_model_parallel_initialized, + init_distributed_environment, + ) + from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import ( + MPLinearLayerConfig, + ) + from vllm.model_executor.parameter import ( + GroupQuantScaleParameter, + PackedColumnParameter, + PackedvLLMParameter, + ) + from vllm.scalar_type import scalar_types + + with set_current_vllm_config(VllmConfig()): + init_distributed_environment( + world_size=1, + rank=0, + distributed_init_method="tcp://127.0.0.1:0", + local_rank=0, + ) + ensure_model_parallel_initialized(1, 1) + + set_random_seed(0) + + # Small-but-nontrivial shapes. + K, N = 256, 256 + G = 32 + assert K % 8 == 0 and N % 8 == 0 and K % G == 0 + + # Build a canonical int4 weight grid then pack into the CT checkpoint layout. + w_int4_kn = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32) + w_ckpt_nk8 = _pack_int4_along_k_to_ckpt(w_int4_kn) # [N, K//8] + + # Scales in CT checkpoint layout for WNA16: [N, K//G] + scales_ckpt_nkg = 0.05 * torch.rand((N, K // G), device=device, dtype=torch.float16) + + # Asymmetric case: zero points in CT checkpoint layout [N//8, K//G] (N-packed) + zeros_int4_gn = torch.randint(0, 16, (K // G, N), device=device, dtype=torch.int32) + zeros_packed_gn8 = _pack_int4_along_n(zeros_int4_gn) # [K//G, N//8] + zeros_ckpt_n8kg = zeros_packed_gn8.t().contiguous() # [N//8, K//G] + + config = MPLinearLayerConfig( + full_weight_shape=(K, N), + partition_weight_shape=(K, N), + weight_type=scalar_types.uint4, # asymmetric + act_type=torch.float16, + group_size=G, + zero_points=True, + has_g_idx=False, + ) + kernel = TritonW4A16LinearKernel( + config, + w_q_param_name="weight_packed", + w_s_param_name="weight_scale", + w_zp_param_name="weight_zero_point", + w_gidx_param_name=None, + ) + + # Build dummy layer with vLLM parameter wrappers. + weight_loader = lambda *args, **kwargs: None + + class DummyLayer(torch.nn.Module): + pass + + layer = DummyLayer() + layer.register_parameter( + "weight_packed", + PackedvLLMParameter( + data=w_ckpt_nk8, + weight_loader=weight_loader, + input_dim=1, + output_dim=0, + packed_factor=8, + packed_dim=1, + ), + ) + layer.register_parameter( + "weight_scale", + GroupQuantScaleParameter( + data=scales_ckpt_nkg, + weight_loader=weight_loader, + input_dim=1, + output_dim=0, + ), + ) + layer.register_parameter( + "weight_zero_point", + PackedColumnParameter( + data=zeros_ckpt_n8kg, + weight_loader=weight_loader, + output_dim=0, + packed_factor=8, + packed_dim=0, + ), + ) + + kernel.process_weights_after_loading(layer) + + # Expected transformed layouts. + expected_w_kn8 = _pack_int4_along_n(w_int4_kn) # [K, N//8] + expected_scales_gn = scales_ckpt_nkg.t().contiguous() # [K//G, N] + expected_zeros_gn8 = zeros_ckpt_n8kg.t().contiguous() # [K//G, N//8] + + assert tuple(layer.weight_packed.shape) == (K, N // 8) + assert tuple(layer.weight_scale.shape) == (K // G, N) + assert tuple(layer.weight_zero_point.shape) == (K // G, N // 8) + + torch.testing.assert_close(layer.weight_packed, expected_w_kn8) + torch.testing.assert_close(layer.weight_scale, expected_scales_gn) + torch.testing.assert_close(layer.weight_zero_point, expected_zeros_gn8) diff --git a/tests/kernels/quantization/test_w4a16_kernel_selection.py b/tests/kernels/quantization/test_w4a16_kernel_selection.py new file mode 100644 index 0000000000..f0696191d3 --- /dev/null +++ b/tests/kernels/quantization/test_w4a16_kernel_selection.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for W4A16 kernel selection logic (ROCm). + +Run `pytest tests/kernels/quantization/test_w4a16_kernel_selection.py`. +""" + +import pytest +import torch + +from vllm.model_executor.kernels.linear import ( + MPLinearLayerConfig, + choose_mp_linear_kernel, +) +from vllm.platforms import current_platform +from vllm.scalar_type import scalar_types + + +@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only") +def test_choose_mp_linear_kernel_picks_triton_w4a16_for_uint4b8(): + # int4 weights, 16-bit activations (CT W4A16 typical config). + K, N = 1024, 256 + config = MPLinearLayerConfig( + full_weight_shape=(K, N), + partition_weight_shape=(K, N), + weight_type=scalar_types.uint4b8, # symmetric int4 (bias=8) + act_type=torch.float16, + group_size=128, + zero_points=False, + has_g_idx=False, + ) + + kernel_type = choose_mp_linear_kernel(config) + assert kernel_type.__name__ == "TritonW4A16LinearKernel" + + +@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only") +def test_choose_mp_linear_kernel_picks_triton_w4a16_for_uint4_asymmetric(): + # Asymmetric int4 weights should also be supported (explicit zero points). + K, N = 512, 512 + config = MPLinearLayerConfig( + full_weight_shape=(K, N), + partition_weight_shape=(K, N), + weight_type=scalar_types.uint4, # asymmetric int4 (explicit zeros) + act_type=torch.bfloat16, + group_size=64, + zero_points=True, + has_g_idx=False, + ) + + kernel_type = choose_mp_linear_kernel(config) + assert kernel_type.__name__ == "TritonW4A16LinearKernel" diff --git a/vllm/model_executor/kernels/linear/__init__.py b/vllm/model_executor/kernels/linear/__init__.py index 6a026eef83..958bea0f96 100644 --- a/vllm/model_executor/kernels/linear/__init__.py +++ b/vllm/model_executor/kernels/linear/__init__.py @@ -51,6 +51,9 @@ from vllm.model_executor.kernels.linear.mixed_precision.machete import ( from vllm.model_executor.kernels.linear.mixed_precision.marlin import ( MarlinLinearKernel, ) +from vllm.model_executor.kernels.linear.mixed_precision.triton_w4a16 import ( + TritonW4A16LinearKernel, +) from vllm.model_executor.kernels.linear.mixed_precision.xpu import ( XPUW4A8IntLinearKernel, XPUwNa16LinearKernel, @@ -203,6 +206,7 @@ _POSSIBLE_KERNELS: dict[PlatformEnum, list[type[MPLinearKernel]]] = { ExllamaLinearKernel, ], PlatformEnum.ROCM: [ + TritonW4A16LinearKernel, ConchLinearKernel, ExllamaLinearKernel, ], @@ -667,6 +671,7 @@ __all__ = [ "ExllamaLinearKernel", "MacheteLinearKernel", "MarlinLinearKernel", + "TritonW4A16LinearKernel", "XPUW4A8IntLinearKernel", "XPUwNa16LinearKernel", "CutlassNvFp4LinearKernel", diff --git a/vllm/model_executor/kernels/linear/mixed_precision/__init__.py b/vllm/model_executor/kernels/linear/mixed_precision/__init__.py index 6c144a5ec8..4d659b3604 100644 --- a/vllm/model_executor/kernels/linear/mixed_precision/__init__.py +++ b/vllm/model_executor/kernels/linear/mixed_precision/__init__.py @@ -29,6 +29,9 @@ from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import ( MPLinearKernel, MPLinearLayerConfig, ) +from vllm.model_executor.kernels.linear.mixed_precision.triton_w4a16 import ( + TritonW4A16LinearKernel, +) from vllm.model_executor.kernels.linear.mixed_precision.xpu import ( XPUW4A8IntLinearKernel, XPUwNa16LinearKernel, @@ -45,6 +48,7 @@ __all__ = [ "ExllamaLinearKernel", "MacheteLinearKernel", "MarlinLinearKernel", + "TritonW4A16LinearKernel", "XPUW4A8IntLinearKernel", "XPUwNa16LinearKernel", ] diff --git a/vllm/model_executor/kernels/linear/mixed_precision/triton_w4a16.py b/vllm/model_executor/kernels/linear/mixed_precision/triton_w4a16.py new file mode 100644 index 0000000000..7bc8ca584c --- /dev/null +++ b/vllm/model_executor/kernels/linear/mixed_precision/triton_w4a16.py @@ -0,0 +1,430 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +Triton-based W4A16 GEMM kernel for ROCm MI300. + +Implements fused int4-weight dequantization + fp16 GEMM in a single kernel, +using GPTQ sequential packing (8 int4 values per int32, shifts [0,4,...,28]). +Plugs into the MPLinearKernel selection system and is preferred over +MarlinLinearKernel/ExllamaLinearKernel on ROCm. + +Weight layout expected by this kernel (post-process_weights_after_loading): + qweight: [K, N//8] int32 — rows=K (input), cols=N//8 (N is packed) + scales: [K//G, N] fp16/bf16 + qzeros: [K//G, N//8] int32 (optional; None for symmetric uint4b8) + +Checkpoint layout from compressed_tensors_wNa16 create_weights: + weight_packed: [N, K//8] int32 (output_dim=0, input_dim=1, packed_dim=1) + weight_scale: [N, K//G] fp16 (output_dim=0, input_dim=1) + weight_zero_point: [N//8, K//G] int32 (output_dim=0, packed_dim=0) +""" + +import torch + +from vllm.model_executor.layers.quantization.utils import replace_parameter +from vllm.model_executor.parameter import BasevLLMParameter, permute_param_layout_ +from vllm.platforms import current_platform +from vllm.scalar_type import scalar_types +from vllm.triton_utils import tl, triton + +from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig + +TRITON_W4A16_SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128, 256] +TRITON_W4A16_SUPPORTED_QUANT_TYPES = [ + scalar_types.uint4b8, # symmetric GPTQ (bias=8) + scalar_types.uint4, # asymmetric with explicit zeros +] + + +@triton.jit +def triton_w4a16_gemm_kernel( + # Pointers + a_ptr, # [M, K] fp16/bf16 activations + b_ptr, # [K, N//8] int32 packed 4-bit weights (N is the packed dim) + scales_ptr, # [K//G, N] fp16/bf16 scales + zeros_ptr, # [K//G, N//8] int32 packed zeros (unused when HAS_ZP=False) + c_ptr, # [M, N] fp16/bf16 output + # Dimensions + M, + N, + K, + # Strides + stride_am, + stride_ak, + stride_bk, + stride_bn, # stride in b along the packed N//8 dim + stride_cm, + stride_cn, + # Quantization parameters + group_size, + # Whether explicit zero points are provided + HAS_ZP: tl.constexpr, + # Zero bias used when HAS_ZP is False (e.g. 8 for uint4b8) + ZP_BIAS: tl.constexpr, + # Block sizes (tuned for MI300 wavefront=64) + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, +): + """ + Fused W4A16 GEMM: C[M,N] = A[M,K] @ dequant(B)[K,N] + + B is stored as [K, N//8] int32 using GPTQ sequential packing: + each int32 packs 8 consecutive N-values at bit offsets [0,4,8,12,16,20,24,28]. + + Dequant: w_fp = (w_int4 - zero) * scale + HAS_ZP=True: zero is loaded from zeros_ptr and unpacked + HAS_ZP=False: zero = ZP_BIAS constant (e.g. 8 for uint4b8 symmetric) + """ + pid_m = tl.program_id(0) + pid_n = tl.program_id(1) + + # Row/col offsets for this tile + offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M) + offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N) + + # b/zeros are stored with N packed: N//8 int32 columns per K row + offs_bn = pid_n * (BLOCK_N // 8) + tl.arange(0, BLOCK_N // 8) + + # GPTQ sequential shifts tiled across BLOCK_N: + # [0,4,8,...,28] repeating for every group of 8 N-values. + # Build 1D shifts_1d of length BLOCK_N: column j gets shift (j % 8) * 4. + shifts_row = tl.arange(0, 8) * 4 # [8] + shifts_1d_2d = tl.broadcast_to(shifts_row[None, :], (BLOCK_N // 8, 8)) + shifts_1d = tl.reshape(shifts_1d_2d, (BLOCK_N,)) # [BLOCK_N] + # Broadcast to [BLOCK_K, BLOCK_N] for weight unpacking + shifts = tl.broadcast_to(shifts_1d[None, :], (BLOCK_K, BLOCK_N)) + + # Scales column offsets: full N-width (one scale per output neuron) + offs_sn = pid_n * BLOCK_N + tl.arange(0, BLOCK_N) + + accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) + + for k_start in range(0, tl.cdiv(K, BLOCK_K)): + offs_k = k_start * BLOCK_K + tl.arange(0, BLOCK_K) + mask_k = offs_k < K + + # ---- Load activations A: [BLOCK_M, BLOCK_K] ---- + a_ptrs = a_ptr + offs_m[:, None] * stride_am + offs_k[None, :] * stride_ak + mask_a = (offs_m[:, None] < M) & mask_k[None, :] + a = tl.load(a_ptrs, mask=mask_a, other=0.0) + + # ---- Load packed weights B: [BLOCK_K, BLOCK_N//8] int32 ---- + b_ptrs = b_ptr + offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn + mask_b = mask_k[:, None] & (offs_bn[None, :] < N // 8) + b_packed = tl.load(b_ptrs, mask=mask_b, other=0) + + # ---- Unpack int4 weights → [BLOCK_K, BLOCK_N] ---- + # tl.interleave(x, x) doubles the last dim by interleaving. + # Starting from [BLOCK_K, BLOCK_N//8], three interleaves give + # [BLOCK_K, BLOCK_N], where each int32 is replicated 8 times. + b = tl.interleave(b_packed, b_packed) + b = tl.interleave(b, b) + b = tl.interleave(b, b) + # Extract the correct 4-bit nibble for each output column + b = (b >> shifts) & 0xF + + # ---- Compute scale/zero group row index ---- + g_idx = (k_start * BLOCK_K) // group_size + + # ---- Load scales: [BLOCK_N] → broadcast to [BLOCK_K, BLOCK_N] ---- + scale_offset = g_idx * N + offs_sn + scale_mask = offs_sn < N + scales = tl.load(scales_ptr + scale_offset, mask=scale_mask, other=1.0) + scales = tl.broadcast_to(scales[None, :], (BLOCK_K, BLOCK_N)) + + # ---- Load / compute zeros ---- + if HAS_ZP: + # Load packed zeros row: [BLOCK_N//8] int32 + zero_offset = g_idx * (N // 8) + offs_bn + zero_mask = offs_bn < N // 8 + z_packed = tl.load(zeros_ptr + zero_offset, mask=zero_mask, other=0) + # Unpack to [BLOCK_N] using same interleave+shift pattern + z = tl.interleave(z_packed, z_packed) + z = tl.interleave(z, z) + z = tl.interleave(z, z) + z = (z >> shifts_1d) & 0xF + z = tl.broadcast_to(z[None, :], (BLOCK_K, BLOCK_N)) + else: + z = tl.full((BLOCK_K, BLOCK_N), ZP_BIAS, dtype=tl.int32) + + # ---- Dequantize: (w - zero) * scale ---- + b_fp = (b - z).to(a.dtype) * scales + + # ---- Accumulate ---- + accumulator += tl.dot(a, b_fp, out_dtype=tl.float32) + + # ---- Store output C: [BLOCK_M, BLOCK_N] ---- + c = accumulator.to(c_ptr.type.element_ty) + c_ptrs = c_ptr + offs_m[:, None] * stride_cm + offs_n[None, :] * stride_cn + mask_c = (offs_m[:, None] < M) & (offs_n[None, :] < N) + tl.store(c_ptrs, c, mask=mask_c) + + +def triton_w4a16_gemm( + a: torch.Tensor, # [M, K] fp16/bf16 + b_q: torch.Tensor, # [K, N//8] int32 + scales: torch.Tensor, # [K//G, N] fp16/bf16 + qzeros: torch.Tensor | None, # [K//G, N//8] int32, or None + group_size: int, + zp_bias: int = 8, # bias for uint4b8 when qzeros is None +) -> torch.Tensor: + """ + Fused W4A16 GEMM using GPTQ-packed int4 weights. + + Args: + a: Activation matrix [M, K], float16 or bfloat16. + b_q: Packed weight matrix [K, N//8], int32 (GPTQ sequential). + scales: Per-group scales [K//G, N], same dtype as a. + qzeros: Per-group packed zero points [K//G, N//8] int32, or None + for symmetric quantization (uses zp_bias instead). + group_size: Quantization group size (resolved from -1 to K by caller). + zp_bias: Constant zero used when qzeros is None (default 8 for uint4b8). + + Returns: + Output matrix [M, N], same dtype as a. + """ + assert a.is_contiguous(), "Activation matrix must be contiguous" + assert b_q.is_contiguous(), "Weight matrix must be contiguous" + assert scales.is_contiguous(), "Scales must be contiguous" + + M, K = a.shape + N = b_q.shape[1] * 8 + + assert b_q.shape == (K, N // 8), ( + f"b_q shape mismatch: {b_q.shape} vs ({K}, {N // 8})" + ) + assert scales.shape == (K // group_size, N), ( + f"scales shape mismatch: {scales.shape} vs ({K // group_size}, {N})" + ) + if qzeros is not None: + assert qzeros.shape == (K // group_size, N // 8), ( + f"qzeros shape mismatch: {qzeros.shape}" + ) + + c = torch.empty((M, N), dtype=a.dtype, device=a.device) + + has_zp = qzeros is not None + # Provide a dummy pointer when HAS_ZP=False (Triton requires a valid ptr) + zeros_ptr = qzeros if has_zp else b_q + + if current_platform.is_rocm(): + from vllm.platforms.rocm import on_gfx1x + + if on_gfx1x(): + # Tuned for RDNA 3.5 (gfx1151, 40 CUs, 32-wide wavefronts). + if M <= 32: + BLOCK_M, BLOCK_N, BLOCK_K = 32, 32, 64 + elif M <= 64: + BLOCK_M, BLOCK_N, BLOCK_K = 64, 64, 32 + else: + BLOCK_M, BLOCK_N, BLOCK_K = 128, 32, 64 + else: + # Tuned for MI300 (gfx942, 304 CUs, 64-wide wavefronts). + if M <= 32: + BLOCK_M, BLOCK_N, BLOCK_K = 32, 64, 32 + elif M <= 64: + BLOCK_M, BLOCK_N, BLOCK_K = 64, 64, 32 + else: + BLOCK_M, BLOCK_N, BLOCK_K = 128, 128, 32 + else: + if M <= 32: + BLOCK_M, BLOCK_N, BLOCK_K = 32, 64, 32 + elif M <= 64: + BLOCK_M, BLOCK_N, BLOCK_K = 64, 64, 32 + else: + BLOCK_M, BLOCK_N, BLOCK_K = 128, 128, 32 + + grid = (triton.cdiv(M, BLOCK_M), triton.cdiv(N, BLOCK_N)) + + triton_w4a16_gemm_kernel[grid]( + a, + b_q, + scales, + zeros_ptr, + c, + M, + N, + K, + a.stride(0), + a.stride(1), + b_q.stride(0), + b_q.stride(1), + c.stride(0), + c.stride(1), + group_size=group_size, + HAS_ZP=has_zp, + ZP_BIAS=zp_bias, + BLOCK_M=BLOCK_M, + BLOCK_N=BLOCK_N, + BLOCK_K=BLOCK_K, + ) + return c + + +class TritonW4A16LinearKernel(MPLinearKernel): + """ + Triton-based W4A16 GEMM kernel for ROCm (MI300 and newer). + + Supports GPTQ-format int4 weights (uint4b8 symmetric, uint4 asymmetric) + with grouped quantization. Weight tensors are transposed from the + compressed-tensors checkpoint layout to the kernel's [K, N//8] layout. + """ + + SUPPORTED_QUANT_TYPES = TRITON_W4A16_SUPPORTED_QUANT_TYPES + + @classmethod + def get_min_capability(cls) -> int: + # Triton handles capability checks itself + return 0 + + @classmethod + def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]: + if not current_platform.is_rocm(): + return False, "TritonW4A16LinearKernel only targets ROCm" + + if c.weight_type not in cls.SUPPORTED_QUANT_TYPES: + return ( + False, + f"Quant type {c.weight_type} not supported; " + f"supported: {cls.SUPPORTED_QUANT_TYPES}", + ) + + if c.act_type not in (torch.float16, torch.bfloat16): + return False, "Only float16/bfloat16 activations are supported" + + N = c.partition_weight_shape[1] + if N % 8 != 0: + return ( + False, + f"Output features ({N}) must be divisible by 8 " + "(8 int4 values packed per int32)", + ) + + if c.has_g_idx: + return ( + False, + "Activation reordering (g_idx) is not supported by " + "TritonW4A16LinearKernel", + ) + + gs = c.group_size + if ( + gs not in TRITON_W4A16_SUPPORTED_GROUP_SIZES + and gs != c.full_weight_shape[0] + ): + return ( + False, + f"Group size {gs} not supported; " + f"supported: {TRITON_W4A16_SUPPORTED_GROUP_SIZES} " + f"or full K ({c.full_weight_shape[0]})", + ) + + K = c.partition_weight_shape[0] + eff_gs = gs if gs != -1 else K + if K % eff_gs != 0: + return (False, f"Input features {K} not divisible by group size {eff_gs}") + + return True, None + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + """ + Convert compressed-tensors checkpoint layout to kernel layout. + + Checkpoint (from compressed_tensors_wNa16.create_weights): + weight_packed: [N, K//8] int32 input_dim=1, output_dim=0, packed_dim=1 + weight_scale: [N, K//G] fp16 input_dim=1, output_dim=0 + weight_zero_point: [N//8, K//G] int32 output_dim=0, packed_dim=0 + + Kernel needs: + qweight: [K, N//8] int32 (transpose weight_packed) + scales: [K//G, N] fp16 (transpose weight_scale) + qzeros: [K//G, N//8] int32 (transpose weight_zero_point) + """ + + # ---- Transform qweight: [N, K//8] → [K//8, N] → back to [K, N//8] ---- + # permute_param_layout_(x, input_dim=0, output_dim=1) rearranges so that + # the input(K) dimension is at physical dim 0 and output(N) at dim 1. + # Checkpoint has input_dim=1, output_dim=0, packed_dim=1 (K is packed). + # After permute we get [K//8, N] (K packed at dim 0, N at dim 1). + # The kernel wants [K, N//8] (K at dim 0, N packed at dim 1), so we + # then transpose: [K//8, N].T = [N, K//8] — that's not right. + # + # Actually we need to change WHAT is packed: + # Original packing: K packed into K//8 (8 K-values per int32) + # Kernel packing: N packed into N//8 (8 N-values per int32) + # These require a full repack, not just a transpose. + # + # Simple approach: unpack → transpose the full [N, K] → repack as [K, N//8]. + # This is done CPU-side at load time (one-time cost). + def repack_w_q(x: BasevLLMParameter) -> BasevLLMParameter: + # x.data is [N, K//8] int32, K packed (GPTQ checkpoint format) + # Step 1: bring to [N, K//8] with output(N) at dim 0 + permute_param_layout_(x, input_dim=1, output_dim=0, packed_dim=1) + w = x.data # [N, K//8] int32 + + N_dim, K8 = w.shape + K_dim = K8 * 8 + # Step 2: unpack to [N, K] int32 (vectorized) + shifts = torch.arange(8, device=w.device, dtype=torch.int32) * 4 + w_unpacked = ((w.unsqueeze(-1) >> shifts) & 0xF).reshape(N_dim, K_dim) + # Step 3: transpose to [K, N] int32 + w_KN = w_unpacked.t().contiguous() + # Step 4: repack N into N//8 int32 values → [K, N//8] (vectorized) + N8 = N_dim // 8 + w_repacked = torch.sum( + (w_KN.view(K_dim, N8, 8) & 0xF) << shifts, + dim=2, + dtype=torch.int32, + ) + x.data = w_repacked.contiguous() + return x + + def repack_w_s(x: BasevLLMParameter) -> BasevLLMParameter: + # x.data is [N, K//G] fp16, bring to [K//G, N] + permute_param_layout_(x, input_dim=1, output_dim=0) + x.data = x.data.t().contiguous() + return x + + self._transform_param(layer, self.w_q_name, repack_w_q) + self._transform_param(layer, self.w_s_name, repack_w_s) + + if self.w_zp_name is not None: + zp = getattr(layer, self.w_zp_name, None) + if zp is not None: + # Checkpoint: [N//8, K//G] int32 (N packed at dim 0, K//G at dim 1) + # Kernel needs: [K//G, N//8] — just transpose + replace_parameter( + layer, + self.w_zp_name, + torch.nn.Parameter(zp.data.t().contiguous(), requires_grad=False), + ) + + def apply_weights( + self, layer: torch.nn.Module, x: torch.Tensor, bias: torch.Tensor | None = None + ) -> torch.Tensor: + c = self.config + w_q, w_s, w_zp, _ = self._get_weight_params(layer) + + x_2d = x.reshape(-1, x.shape[-1]).contiguous() + out_shape = x.shape[:-1] + (c.partition_weight_shape[1],) + + K = c.partition_weight_shape[0] + group_size = c.group_size if c.group_size != -1 else K + + # For symmetric types (uint4b8), use the scalar bias; no zeros tensor + zp_bias = c.weight_type.bias if c.weight_type.has_bias() else 0 + + output = triton_w4a16_gemm( + a=x_2d, + b_q=w_q, + scales=w_s, + qzeros=w_zp, + group_size=group_size, + zp_bias=zp_bias, + ) + + if bias is not None: + output.add_(bias) + + return output.reshape(out_shape)