[Kernel][Hardware][AMD] Add TritonW4A16LinearKernel for ROCm (#37352)
Signed-off-by: jatseng-ai <jatseng@amd.com> Signed-off-by: jatseng-ai <janet.tseng@amd.com> Signed-off-by: Matthias Gehre <matthias.gehre@amd.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Matthias Gehre <matthias.gehre@amd.com>
This commit is contained in:
co-authored by
Claude
Matthias Gehre
parent
967146e7bd
commit
d468322dc1
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""End-to-end smoke test for CT W4A16 models on ROCm.
|
||||
|
||||
This validates that a real compressed-tensors W4A16 model can run inference
|
||||
end-to-end (which will exercise the Triton W4A16 kernel when selected).
|
||||
|
||||
Run `pytest tests/kernels/quantization/test_rocm_compressed_tensors_w4a16.py`.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_path",
|
||||
[
|
||||
# Listed in tests/weight_loading/models.txt
|
||||
"nm-testing/tinyllama-oneshot-w4a16-group128-v2",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("max_tokens", [32])
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="Should only run on ROCm")
|
||||
def test_rocm_compressed_tensors_w4a16_e2e(
|
||||
vllm_runner, example_prompts, model_path, max_tokens
|
||||
):
|
||||
# Use fp16 activations for maximum compatibility.
|
||||
# gpu_memory_utilization lowered to work on shared nodes.
|
||||
with vllm_runner(
|
||||
model_path, dtype="float16", gpu_memory_utilization=0.3
|
||||
) as vllm_model:
|
||||
# If the W4A16 kernel is broken, this will typically throw.
|
||||
vllm_model.generate_greedy(example_prompts, max_tokens=max_tokens)
|
||||
@@ -0,0 +1,304 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the ROCm Triton W4A16 GEMM kernel.
|
||||
|
||||
Run `pytest tests/kernels/quantization/test_triton_w4a16.py`.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
# This test module is ROCm/Triton specific. Avoid import-time failures on
|
||||
# non-ROCm or environments without Triton by skipping early.
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("ROCm only", allow_module_level=True)
|
||||
|
||||
pytest.importorskip("triton")
|
||||
|
||||
device = "cuda"
|
||||
|
||||
triton_w4a16_module = importlib.import_module(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.triton_w4a16"
|
||||
)
|
||||
triton_w4a16_gemm = triton_w4a16_module.triton_w4a16_gemm
|
||||
TritonW4A16LinearKernel = triton_w4a16_module.TritonW4A16LinearKernel
|
||||
|
||||
|
||||
def _pack_int4_along_n(w_int4_kn: torch.Tensor) -> torch.Tensor:
|
||||
"""Pack int4 values along N: [K, N] -> [K, N//8] int32."""
|
||||
assert w_int4_kn.dtype == torch.int32
|
||||
K, N = w_int4_kn.shape
|
||||
assert N % 8 == 0
|
||||
shifts = torch.arange(8, device=w_int4_kn.device, dtype=torch.int32) * 4
|
||||
return torch.sum(
|
||||
(w_int4_kn.view(K, N // 8, 8) & 0xF) << shifts,
|
||||
dim=2,
|
||||
dtype=torch.int32,
|
||||
).contiguous()
|
||||
|
||||
|
||||
def _unpack_int4_along_n(w_packed_kn8: torch.Tensor) -> torch.Tensor:
|
||||
"""Unpack int4 values along N: [K, N//8] -> [K, N] int32."""
|
||||
assert w_packed_kn8.dtype == torch.int32
|
||||
K, N8 = w_packed_kn8.shape
|
||||
shifts = torch.arange(8, device=w_packed_kn8.device, dtype=torch.int32) * 4
|
||||
nibbles = (w_packed_kn8.unsqueeze(-1) >> shifts) & 0xF
|
||||
return nibbles.reshape(K, N8 * 8)
|
||||
|
||||
|
||||
def _pack_int4_along_k_to_ckpt(w_int4_kn: torch.Tensor) -> torch.Tensor:
|
||||
"""Pack int4 values along K into CT checkpoint layout: [K,N] -> [N, K//8]."""
|
||||
assert w_int4_kn.dtype == torch.int32
|
||||
K, N = w_int4_kn.shape
|
||||
assert K % 8 == 0
|
||||
out = torch.zeros((N, K // 8), dtype=torch.int32, device=w_int4_kn.device)
|
||||
for i in range(8):
|
||||
out |= (w_int4_kn[i::8, :].t() & 0xF) << (i * 4)
|
||||
return out.contiguous()
|
||||
|
||||
|
||||
def _w4a16_reference(
|
||||
a_mk: torch.Tensor,
|
||||
b_packed_kn8: torch.Tensor,
|
||||
scales_gn: torch.Tensor,
|
||||
*,
|
||||
group_size: int,
|
||||
qzeros_gn8: torch.Tensor | None,
|
||||
zp_bias: int,
|
||||
) -> torch.Tensor:
|
||||
"""Reference implementation for W4A16.
|
||||
|
||||
a_mk: [M,K] fp16/bf16
|
||||
b_packed_kn8: [K, N//8] int32, N-packed int4 weights
|
||||
scales_gn: [K//G, N] fp16/bf16
|
||||
qzeros_gn8: [K//G, N//8] int32, N-packed int4 zeros, or None
|
||||
"""
|
||||
assert a_mk.dtype in (torch.float16, torch.bfloat16)
|
||||
assert b_packed_kn8.dtype == torch.int32
|
||||
assert scales_gn.dtype == a_mk.dtype
|
||||
|
||||
M, K = a_mk.shape
|
||||
N = b_packed_kn8.shape[1] * 8
|
||||
assert b_packed_kn8.shape[0] == K
|
||||
|
||||
assert group_size > 0 and K % group_size == 0
|
||||
G = group_size
|
||||
num_groups = K // G
|
||||
assert scales_gn.shape == (num_groups, N)
|
||||
|
||||
w_int4 = _unpack_int4_along_n(b_packed_kn8) # [K,N]
|
||||
if qzeros_gn8 is None:
|
||||
z_full = torch.full((K, N), zp_bias, dtype=torch.int32, device=a_mk.device)
|
||||
else:
|
||||
assert qzeros_gn8.shape == (num_groups, N // 8)
|
||||
z_gn = _unpack_int4_along_n(qzeros_gn8) # [G,N] in groups
|
||||
z_full = z_gn.repeat_interleave(G, dim=0) # [K,N]
|
||||
|
||||
s_full = scales_gn.repeat_interleave(G, dim=0).to(torch.float32) # [K,N]
|
||||
w_fp = (w_int4 - z_full).to(torch.float32) * s_full # [K,N]
|
||||
|
||||
out = a_mk.to(torch.float32) @ w_fp # [M,N]
|
||||
return out.to(a_mk.dtype)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize(
|
||||
"M,K,N,G,has_zp",
|
||||
[
|
||||
(1, 256, 256, 32, False),
|
||||
(17, 256, 512, 32, False),
|
||||
(32, 512, 256, 64, False),
|
||||
(33, 512, 512, 128, False),
|
||||
(64, 1024, 256, 256, False),
|
||||
(128, 256, 1024, 32, True),
|
||||
(64, 512, 512, 64, True),
|
||||
],
|
||||
)
|
||||
def test_triton_w4a16_gemm_matches_reference(dtype, M, K, N, G, has_zp):
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA/HIP device not available")
|
||||
if N % 8 != 0 or K % G != 0:
|
||||
pytest.skip("Invalid test shape")
|
||||
|
||||
set_random_seed(0)
|
||||
|
||||
a = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
|
||||
w_int4 = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32)
|
||||
b_packed = _pack_int4_along_n(w_int4)
|
||||
|
||||
scales = (0.05 * torch.rand((K // G, N), device=device, dtype=torch.float32)).to(
|
||||
dtype
|
||||
)
|
||||
|
||||
qzeros = None
|
||||
if has_zp:
|
||||
zeros_int4 = torch.randint(0, 16, (K // G, N), device=device, dtype=torch.int32)
|
||||
qzeros = _pack_int4_along_n(zeros_int4)
|
||||
|
||||
out = triton_w4a16_gemm(
|
||||
a=a,
|
||||
b_q=b_packed,
|
||||
scales=scales,
|
||||
qzeros=qzeros,
|
||||
group_size=G,
|
||||
zp_bias=8,
|
||||
)
|
||||
ref = _w4a16_reference(
|
||||
a,
|
||||
b_packed,
|
||||
scales,
|
||||
group_size=G,
|
||||
qzeros_gn8=qzeros,
|
||||
zp_bias=8,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out, ref, rtol=1e-2, atol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
|
||||
def test_triton_w4a16_gemm_requires_contiguous_inputs():
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA/HIP device not available")
|
||||
|
||||
set_random_seed(0)
|
||||
M, K, N, G = 32, 256, 256, 32
|
||||
a = torch.randn((K, M), device=device, dtype=torch.float16).t() # non-contiguous
|
||||
w_int4 = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32)
|
||||
b_packed = _pack_int4_along_n(w_int4)
|
||||
scales = torch.rand((K // G, N), device=device, dtype=torch.float16)
|
||||
|
||||
with pytest.raises(AssertionError):
|
||||
triton_w4a16_gemm(
|
||||
a=a,
|
||||
b_q=b_packed,
|
||||
scales=scales,
|
||||
qzeros=None,
|
||||
group_size=G,
|
||||
zp_bias=8,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
|
||||
def test_triton_w4a16_process_weights_after_loading_repacks_layout():
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA/HIP device not available")
|
||||
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
ensure_model_parallel_initialized,
|
||||
init_distributed_environment,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import (
|
||||
MPLinearLayerConfig,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
PackedColumnParameter,
|
||||
PackedvLLMParameter,
|
||||
)
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
init_distributed_environment(
|
||||
world_size=1,
|
||||
rank=0,
|
||||
distributed_init_method="tcp://127.0.0.1:0",
|
||||
local_rank=0,
|
||||
)
|
||||
ensure_model_parallel_initialized(1, 1)
|
||||
|
||||
set_random_seed(0)
|
||||
|
||||
# Small-but-nontrivial shapes.
|
||||
K, N = 256, 256
|
||||
G = 32
|
||||
assert K % 8 == 0 and N % 8 == 0 and K % G == 0
|
||||
|
||||
# Build a canonical int4 weight grid then pack into the CT checkpoint layout.
|
||||
w_int4_kn = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32)
|
||||
w_ckpt_nk8 = _pack_int4_along_k_to_ckpt(w_int4_kn) # [N, K//8]
|
||||
|
||||
# Scales in CT checkpoint layout for WNA16: [N, K//G]
|
||||
scales_ckpt_nkg = 0.05 * torch.rand((N, K // G), device=device, dtype=torch.float16)
|
||||
|
||||
# Asymmetric case: zero points in CT checkpoint layout [N//8, K//G] (N-packed)
|
||||
zeros_int4_gn = torch.randint(0, 16, (K // G, N), device=device, dtype=torch.int32)
|
||||
zeros_packed_gn8 = _pack_int4_along_n(zeros_int4_gn) # [K//G, N//8]
|
||||
zeros_ckpt_n8kg = zeros_packed_gn8.t().contiguous() # [N//8, K//G]
|
||||
|
||||
config = MPLinearLayerConfig(
|
||||
full_weight_shape=(K, N),
|
||||
partition_weight_shape=(K, N),
|
||||
weight_type=scalar_types.uint4, # asymmetric
|
||||
act_type=torch.float16,
|
||||
group_size=G,
|
||||
zero_points=True,
|
||||
has_g_idx=False,
|
||||
)
|
||||
kernel = TritonW4A16LinearKernel(
|
||||
config,
|
||||
w_q_param_name="weight_packed",
|
||||
w_s_param_name="weight_scale",
|
||||
w_zp_param_name="weight_zero_point",
|
||||
w_gidx_param_name=None,
|
||||
)
|
||||
|
||||
# Build dummy layer with vLLM parameter wrappers.
|
||||
weight_loader = lambda *args, **kwargs: None
|
||||
|
||||
class DummyLayer(torch.nn.Module):
|
||||
pass
|
||||
|
||||
layer = DummyLayer()
|
||||
layer.register_parameter(
|
||||
"weight_packed",
|
||||
PackedvLLMParameter(
|
||||
data=w_ckpt_nk8,
|
||||
weight_loader=weight_loader,
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
packed_factor=8,
|
||||
packed_dim=1,
|
||||
),
|
||||
)
|
||||
layer.register_parameter(
|
||||
"weight_scale",
|
||||
GroupQuantScaleParameter(
|
||||
data=scales_ckpt_nkg,
|
||||
weight_loader=weight_loader,
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
),
|
||||
)
|
||||
layer.register_parameter(
|
||||
"weight_zero_point",
|
||||
PackedColumnParameter(
|
||||
data=zeros_ckpt_n8kg,
|
||||
weight_loader=weight_loader,
|
||||
output_dim=0,
|
||||
packed_factor=8,
|
||||
packed_dim=0,
|
||||
),
|
||||
)
|
||||
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
# Expected transformed layouts.
|
||||
expected_w_kn8 = _pack_int4_along_n(w_int4_kn) # [K, N//8]
|
||||
expected_scales_gn = scales_ckpt_nkg.t().contiguous() # [K//G, N]
|
||||
expected_zeros_gn8 = zeros_ckpt_n8kg.t().contiguous() # [K//G, N//8]
|
||||
|
||||
assert tuple(layer.weight_packed.shape) == (K, N // 8)
|
||||
assert tuple(layer.weight_scale.shape) == (K // G, N)
|
||||
assert tuple(layer.weight_zero_point.shape) == (K // G, N // 8)
|
||||
|
||||
torch.testing.assert_close(layer.weight_packed, expected_w_kn8)
|
||||
torch.testing.assert_close(layer.weight_scale, expected_scales_gn)
|
||||
torch.testing.assert_close(layer.weight_zero_point, expected_zeros_gn8)
|
||||
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for W4A16 kernel selection logic (ROCm).
|
||||
|
||||
Run `pytest tests/kernels/quantization/test_w4a16_kernel_selection.py`.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
MPLinearLayerConfig,
|
||||
choose_mp_linear_kernel,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
|
||||
def test_choose_mp_linear_kernel_picks_triton_w4a16_for_uint4b8():
|
||||
# int4 weights, 16-bit activations (CT W4A16 typical config).
|
||||
K, N = 1024, 256
|
||||
config = MPLinearLayerConfig(
|
||||
full_weight_shape=(K, N),
|
||||
partition_weight_shape=(K, N),
|
||||
weight_type=scalar_types.uint4b8, # symmetric int4 (bias=8)
|
||||
act_type=torch.float16,
|
||||
group_size=128,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
|
||||
kernel_type = choose_mp_linear_kernel(config)
|
||||
assert kernel_type.__name__ == "TritonW4A16LinearKernel"
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
|
||||
def test_choose_mp_linear_kernel_picks_triton_w4a16_for_uint4_asymmetric():
|
||||
# Asymmetric int4 weights should also be supported (explicit zero points).
|
||||
K, N = 512, 512
|
||||
config = MPLinearLayerConfig(
|
||||
full_weight_shape=(K, N),
|
||||
partition_weight_shape=(K, N),
|
||||
weight_type=scalar_types.uint4, # asymmetric int4 (explicit zeros)
|
||||
act_type=torch.bfloat16,
|
||||
group_size=64,
|
||||
zero_points=True,
|
||||
has_g_idx=False,
|
||||
)
|
||||
|
||||
kernel_type = choose_mp_linear_kernel(config)
|
||||
assert kernel_type.__name__ == "TritonW4A16LinearKernel"
|
||||
@@ -51,6 +51,9 @@ from vllm.model_executor.kernels.linear.mixed_precision.machete import (
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.marlin import (
|
||||
MarlinLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.triton_w4a16 import (
|
||||
TritonW4A16LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.xpu import (
|
||||
XPUW4A8IntLinearKernel,
|
||||
XPUwNa16LinearKernel,
|
||||
@@ -203,6 +206,7 @@ _POSSIBLE_KERNELS: dict[PlatformEnum, list[type[MPLinearKernel]]] = {
|
||||
ExllamaLinearKernel,
|
||||
],
|
||||
PlatformEnum.ROCM: [
|
||||
TritonW4A16LinearKernel,
|
||||
ConchLinearKernel,
|
||||
ExllamaLinearKernel,
|
||||
],
|
||||
@@ -667,6 +671,7 @@ __all__ = [
|
||||
"ExllamaLinearKernel",
|
||||
"MacheteLinearKernel",
|
||||
"MarlinLinearKernel",
|
||||
"TritonW4A16LinearKernel",
|
||||
"XPUW4A8IntLinearKernel",
|
||||
"XPUwNa16LinearKernel",
|
||||
"CutlassNvFp4LinearKernel",
|
||||
|
||||
@@ -29,6 +29,9 @@ from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import (
|
||||
MPLinearKernel,
|
||||
MPLinearLayerConfig,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.triton_w4a16 import (
|
||||
TritonW4A16LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.xpu import (
|
||||
XPUW4A8IntLinearKernel,
|
||||
XPUwNa16LinearKernel,
|
||||
@@ -45,6 +48,7 @@ __all__ = [
|
||||
"ExllamaLinearKernel",
|
||||
"MacheteLinearKernel",
|
||||
"MarlinLinearKernel",
|
||||
"TritonW4A16LinearKernel",
|
||||
"XPUW4A8IntLinearKernel",
|
||||
"XPUwNa16LinearKernel",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,430 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Triton-based W4A16 GEMM kernel for ROCm MI300.
|
||||
|
||||
Implements fused int4-weight dequantization + fp16 GEMM in a single kernel,
|
||||
using GPTQ sequential packing (8 int4 values per int32, shifts [0,4,...,28]).
|
||||
Plugs into the MPLinearKernel selection system and is preferred over
|
||||
MarlinLinearKernel/ExllamaLinearKernel on ROCm.
|
||||
|
||||
Weight layout expected by this kernel (post-process_weights_after_loading):
|
||||
qweight: [K, N//8] int32 — rows=K (input), cols=N//8 (N is packed)
|
||||
scales: [K//G, N] fp16/bf16
|
||||
qzeros: [K//G, N//8] int32 (optional; None for symmetric uint4b8)
|
||||
|
||||
Checkpoint layout from compressed_tensors_wNa16 create_weights:
|
||||
weight_packed: [N, K//8] int32 (output_dim=0, input_dim=1, packed_dim=1)
|
||||
weight_scale: [N, K//G] fp16 (output_dim=0, input_dim=1)
|
||||
weight_zero_point: [N//8, K//G] int32 (output_dim=0, packed_dim=0)
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils import replace_parameter
|
||||
from vllm.model_executor.parameter import BasevLLMParameter, permute_param_layout_
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
|
||||
|
||||
TRITON_W4A16_SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128, 256]
|
||||
TRITON_W4A16_SUPPORTED_QUANT_TYPES = [
|
||||
scalar_types.uint4b8, # symmetric GPTQ (bias=8)
|
||||
scalar_types.uint4, # asymmetric with explicit zeros
|
||||
]
|
||||
|
||||
|
||||
@triton.jit
|
||||
def triton_w4a16_gemm_kernel(
|
||||
# Pointers
|
||||
a_ptr, # [M, K] fp16/bf16 activations
|
||||
b_ptr, # [K, N//8] int32 packed 4-bit weights (N is the packed dim)
|
||||
scales_ptr, # [K//G, N] fp16/bf16 scales
|
||||
zeros_ptr, # [K//G, N//8] int32 packed zeros (unused when HAS_ZP=False)
|
||||
c_ptr, # [M, N] fp16/bf16 output
|
||||
# Dimensions
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
# Strides
|
||||
stride_am,
|
||||
stride_ak,
|
||||
stride_bk,
|
||||
stride_bn, # stride in b along the packed N//8 dim
|
||||
stride_cm,
|
||||
stride_cn,
|
||||
# Quantization parameters
|
||||
group_size,
|
||||
# Whether explicit zero points are provided
|
||||
HAS_ZP: tl.constexpr,
|
||||
# Zero bias used when HAS_ZP is False (e.g. 8 for uint4b8)
|
||||
ZP_BIAS: tl.constexpr,
|
||||
# Block sizes (tuned for MI300 wavefront=64)
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Fused W4A16 GEMM: C[M,N] = A[M,K] @ dequant(B)[K,N]
|
||||
|
||||
B is stored as [K, N//8] int32 using GPTQ sequential packing:
|
||||
each int32 packs 8 consecutive N-values at bit offsets [0,4,8,12,16,20,24,28].
|
||||
|
||||
Dequant: w_fp = (w_int4 - zero) * scale
|
||||
HAS_ZP=True: zero is loaded from zeros_ptr and unpacked
|
||||
HAS_ZP=False: zero = ZP_BIAS constant (e.g. 8 for uint4b8 symmetric)
|
||||
"""
|
||||
pid_m = tl.program_id(0)
|
||||
pid_n = tl.program_id(1)
|
||||
|
||||
# Row/col offsets for this tile
|
||||
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
|
||||
# b/zeros are stored with N packed: N//8 int32 columns per K row
|
||||
offs_bn = pid_n * (BLOCK_N // 8) + tl.arange(0, BLOCK_N // 8)
|
||||
|
||||
# GPTQ sequential shifts tiled across BLOCK_N:
|
||||
# [0,4,8,...,28] repeating for every group of 8 N-values.
|
||||
# Build 1D shifts_1d of length BLOCK_N: column j gets shift (j % 8) * 4.
|
||||
shifts_row = tl.arange(0, 8) * 4 # [8]
|
||||
shifts_1d_2d = tl.broadcast_to(shifts_row[None, :], (BLOCK_N // 8, 8))
|
||||
shifts_1d = tl.reshape(shifts_1d_2d, (BLOCK_N,)) # [BLOCK_N]
|
||||
# Broadcast to [BLOCK_K, BLOCK_N] for weight unpacking
|
||||
shifts = tl.broadcast_to(shifts_1d[None, :], (BLOCK_K, BLOCK_N))
|
||||
|
||||
# Scales column offsets: full N-width (one scale per output neuron)
|
||||
offs_sn = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
|
||||
accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
|
||||
|
||||
for k_start in range(0, tl.cdiv(K, BLOCK_K)):
|
||||
offs_k = k_start * BLOCK_K + tl.arange(0, BLOCK_K)
|
||||
mask_k = offs_k < K
|
||||
|
||||
# ---- Load activations A: [BLOCK_M, BLOCK_K] ----
|
||||
a_ptrs = a_ptr + offs_m[:, None] * stride_am + offs_k[None, :] * stride_ak
|
||||
mask_a = (offs_m[:, None] < M) & mask_k[None, :]
|
||||
a = tl.load(a_ptrs, mask=mask_a, other=0.0)
|
||||
|
||||
# ---- Load packed weights B: [BLOCK_K, BLOCK_N//8] int32 ----
|
||||
b_ptrs = b_ptr + offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn
|
||||
mask_b = mask_k[:, None] & (offs_bn[None, :] < N // 8)
|
||||
b_packed = tl.load(b_ptrs, mask=mask_b, other=0)
|
||||
|
||||
# ---- Unpack int4 weights → [BLOCK_K, BLOCK_N] ----
|
||||
# tl.interleave(x, x) doubles the last dim by interleaving.
|
||||
# Starting from [BLOCK_K, BLOCK_N//8], three interleaves give
|
||||
# [BLOCK_K, BLOCK_N], where each int32 is replicated 8 times.
|
||||
b = tl.interleave(b_packed, b_packed)
|
||||
b = tl.interleave(b, b)
|
||||
b = tl.interleave(b, b)
|
||||
# Extract the correct 4-bit nibble for each output column
|
||||
b = (b >> shifts) & 0xF
|
||||
|
||||
# ---- Compute scale/zero group row index ----
|
||||
g_idx = (k_start * BLOCK_K) // group_size
|
||||
|
||||
# ---- Load scales: [BLOCK_N] → broadcast to [BLOCK_K, BLOCK_N] ----
|
||||
scale_offset = g_idx * N + offs_sn
|
||||
scale_mask = offs_sn < N
|
||||
scales = tl.load(scales_ptr + scale_offset, mask=scale_mask, other=1.0)
|
||||
scales = tl.broadcast_to(scales[None, :], (BLOCK_K, BLOCK_N))
|
||||
|
||||
# ---- Load / compute zeros ----
|
||||
if HAS_ZP:
|
||||
# Load packed zeros row: [BLOCK_N//8] int32
|
||||
zero_offset = g_idx * (N // 8) + offs_bn
|
||||
zero_mask = offs_bn < N // 8
|
||||
z_packed = tl.load(zeros_ptr + zero_offset, mask=zero_mask, other=0)
|
||||
# Unpack to [BLOCK_N] using same interleave+shift pattern
|
||||
z = tl.interleave(z_packed, z_packed)
|
||||
z = tl.interleave(z, z)
|
||||
z = tl.interleave(z, z)
|
||||
z = (z >> shifts_1d) & 0xF
|
||||
z = tl.broadcast_to(z[None, :], (BLOCK_K, BLOCK_N))
|
||||
else:
|
||||
z = tl.full((BLOCK_K, BLOCK_N), ZP_BIAS, dtype=tl.int32)
|
||||
|
||||
# ---- Dequantize: (w - zero) * scale ----
|
||||
b_fp = (b - z).to(a.dtype) * scales
|
||||
|
||||
# ---- Accumulate ----
|
||||
accumulator += tl.dot(a, b_fp, out_dtype=tl.float32)
|
||||
|
||||
# ---- Store output C: [BLOCK_M, BLOCK_N] ----
|
||||
c = accumulator.to(c_ptr.type.element_ty)
|
||||
c_ptrs = c_ptr + offs_m[:, None] * stride_cm + offs_n[None, :] * stride_cn
|
||||
mask_c = (offs_m[:, None] < M) & (offs_n[None, :] < N)
|
||||
tl.store(c_ptrs, c, mask=mask_c)
|
||||
|
||||
|
||||
def triton_w4a16_gemm(
|
||||
a: torch.Tensor, # [M, K] fp16/bf16
|
||||
b_q: torch.Tensor, # [K, N//8] int32
|
||||
scales: torch.Tensor, # [K//G, N] fp16/bf16
|
||||
qzeros: torch.Tensor | None, # [K//G, N//8] int32, or None
|
||||
group_size: int,
|
||||
zp_bias: int = 8, # bias for uint4b8 when qzeros is None
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Fused W4A16 GEMM using GPTQ-packed int4 weights.
|
||||
|
||||
Args:
|
||||
a: Activation matrix [M, K], float16 or bfloat16.
|
||||
b_q: Packed weight matrix [K, N//8], int32 (GPTQ sequential).
|
||||
scales: Per-group scales [K//G, N], same dtype as a.
|
||||
qzeros: Per-group packed zero points [K//G, N//8] int32, or None
|
||||
for symmetric quantization (uses zp_bias instead).
|
||||
group_size: Quantization group size (resolved from -1 to K by caller).
|
||||
zp_bias: Constant zero used when qzeros is None (default 8 for uint4b8).
|
||||
|
||||
Returns:
|
||||
Output matrix [M, N], same dtype as a.
|
||||
"""
|
||||
assert a.is_contiguous(), "Activation matrix must be contiguous"
|
||||
assert b_q.is_contiguous(), "Weight matrix must be contiguous"
|
||||
assert scales.is_contiguous(), "Scales must be contiguous"
|
||||
|
||||
M, K = a.shape
|
||||
N = b_q.shape[1] * 8
|
||||
|
||||
assert b_q.shape == (K, N // 8), (
|
||||
f"b_q shape mismatch: {b_q.shape} vs ({K}, {N // 8})"
|
||||
)
|
||||
assert scales.shape == (K // group_size, N), (
|
||||
f"scales shape mismatch: {scales.shape} vs ({K // group_size}, {N})"
|
||||
)
|
||||
if qzeros is not None:
|
||||
assert qzeros.shape == (K // group_size, N // 8), (
|
||||
f"qzeros shape mismatch: {qzeros.shape}"
|
||||
)
|
||||
|
||||
c = torch.empty((M, N), dtype=a.dtype, device=a.device)
|
||||
|
||||
has_zp = qzeros is not None
|
||||
# Provide a dummy pointer when HAS_ZP=False (Triton requires a valid ptr)
|
||||
zeros_ptr = qzeros if has_zp else b_q
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_gfx1x
|
||||
|
||||
if on_gfx1x():
|
||||
# Tuned for RDNA 3.5 (gfx1151, 40 CUs, 32-wide wavefronts).
|
||||
if M <= 32:
|
||||
BLOCK_M, BLOCK_N, BLOCK_K = 32, 32, 64
|
||||
elif M <= 64:
|
||||
BLOCK_M, BLOCK_N, BLOCK_K = 64, 64, 32
|
||||
else:
|
||||
BLOCK_M, BLOCK_N, BLOCK_K = 128, 32, 64
|
||||
else:
|
||||
# Tuned for MI300 (gfx942, 304 CUs, 64-wide wavefronts).
|
||||
if M <= 32:
|
||||
BLOCK_M, BLOCK_N, BLOCK_K = 32, 64, 32
|
||||
elif M <= 64:
|
||||
BLOCK_M, BLOCK_N, BLOCK_K = 64, 64, 32
|
||||
else:
|
||||
BLOCK_M, BLOCK_N, BLOCK_K = 128, 128, 32
|
||||
else:
|
||||
if M <= 32:
|
||||
BLOCK_M, BLOCK_N, BLOCK_K = 32, 64, 32
|
||||
elif M <= 64:
|
||||
BLOCK_M, BLOCK_N, BLOCK_K = 64, 64, 32
|
||||
else:
|
||||
BLOCK_M, BLOCK_N, BLOCK_K = 128, 128, 32
|
||||
|
||||
grid = (triton.cdiv(M, BLOCK_M), triton.cdiv(N, BLOCK_N))
|
||||
|
||||
triton_w4a16_gemm_kernel[grid](
|
||||
a,
|
||||
b_q,
|
||||
scales,
|
||||
zeros_ptr,
|
||||
c,
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
a.stride(0),
|
||||
a.stride(1),
|
||||
b_q.stride(0),
|
||||
b_q.stride(1),
|
||||
c.stride(0),
|
||||
c.stride(1),
|
||||
group_size=group_size,
|
||||
HAS_ZP=has_zp,
|
||||
ZP_BIAS=zp_bias,
|
||||
BLOCK_M=BLOCK_M,
|
||||
BLOCK_N=BLOCK_N,
|
||||
BLOCK_K=BLOCK_K,
|
||||
)
|
||||
return c
|
||||
|
||||
|
||||
class TritonW4A16LinearKernel(MPLinearKernel):
|
||||
"""
|
||||
Triton-based W4A16 GEMM kernel for ROCm (MI300 and newer).
|
||||
|
||||
Supports GPTQ-format int4 weights (uint4b8 symmetric, uint4 asymmetric)
|
||||
with grouped quantization. Weight tensors are transposed from the
|
||||
compressed-tensors checkpoint layout to the kernel's [K, N//8] layout.
|
||||
"""
|
||||
|
||||
SUPPORTED_QUANT_TYPES = TRITON_W4A16_SUPPORTED_QUANT_TYPES
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
# Triton handles capability checks itself
|
||||
return 0
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_rocm():
|
||||
return False, "TritonW4A16LinearKernel only targets ROCm"
|
||||
|
||||
if c.weight_type not in cls.SUPPORTED_QUANT_TYPES:
|
||||
return (
|
||||
False,
|
||||
f"Quant type {c.weight_type} not supported; "
|
||||
f"supported: {cls.SUPPORTED_QUANT_TYPES}",
|
||||
)
|
||||
|
||||
if c.act_type not in (torch.float16, torch.bfloat16):
|
||||
return False, "Only float16/bfloat16 activations are supported"
|
||||
|
||||
N = c.partition_weight_shape[1]
|
||||
if N % 8 != 0:
|
||||
return (
|
||||
False,
|
||||
f"Output features ({N}) must be divisible by 8 "
|
||||
"(8 int4 values packed per int32)",
|
||||
)
|
||||
|
||||
if c.has_g_idx:
|
||||
return (
|
||||
False,
|
||||
"Activation reordering (g_idx) is not supported by "
|
||||
"TritonW4A16LinearKernel",
|
||||
)
|
||||
|
||||
gs = c.group_size
|
||||
if (
|
||||
gs not in TRITON_W4A16_SUPPORTED_GROUP_SIZES
|
||||
and gs != c.full_weight_shape[0]
|
||||
):
|
||||
return (
|
||||
False,
|
||||
f"Group size {gs} not supported; "
|
||||
f"supported: {TRITON_W4A16_SUPPORTED_GROUP_SIZES} "
|
||||
f"or full K ({c.full_weight_shape[0]})",
|
||||
)
|
||||
|
||||
K = c.partition_weight_shape[0]
|
||||
eff_gs = gs if gs != -1 else K
|
||||
if K % eff_gs != 0:
|
||||
return (False, f"Input features {K} not divisible by group size {eff_gs}")
|
||||
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
"""
|
||||
Convert compressed-tensors checkpoint layout to kernel layout.
|
||||
|
||||
Checkpoint (from compressed_tensors_wNa16.create_weights):
|
||||
weight_packed: [N, K//8] int32 input_dim=1, output_dim=0, packed_dim=1
|
||||
weight_scale: [N, K//G] fp16 input_dim=1, output_dim=0
|
||||
weight_zero_point: [N//8, K//G] int32 output_dim=0, packed_dim=0
|
||||
|
||||
Kernel needs:
|
||||
qweight: [K, N//8] int32 (transpose weight_packed)
|
||||
scales: [K//G, N] fp16 (transpose weight_scale)
|
||||
qzeros: [K//G, N//8] int32 (transpose weight_zero_point)
|
||||
"""
|
||||
|
||||
# ---- Transform qweight: [N, K//8] → [K//8, N] → back to [K, N//8] ----
|
||||
# permute_param_layout_(x, input_dim=0, output_dim=1) rearranges so that
|
||||
# the input(K) dimension is at physical dim 0 and output(N) at dim 1.
|
||||
# Checkpoint has input_dim=1, output_dim=0, packed_dim=1 (K is packed).
|
||||
# After permute we get [K//8, N] (K packed at dim 0, N at dim 1).
|
||||
# The kernel wants [K, N//8] (K at dim 0, N packed at dim 1), so we
|
||||
# then transpose: [K//8, N].T = [N, K//8] — that's not right.
|
||||
#
|
||||
# Actually we need to change WHAT is packed:
|
||||
# Original packing: K packed into K//8 (8 K-values per int32)
|
||||
# Kernel packing: N packed into N//8 (8 N-values per int32)
|
||||
# These require a full repack, not just a transpose.
|
||||
#
|
||||
# Simple approach: unpack → transpose the full [N, K] → repack as [K, N//8].
|
||||
# This is done CPU-side at load time (one-time cost).
|
||||
def repack_w_q(x: BasevLLMParameter) -> BasevLLMParameter:
|
||||
# x.data is [N, K//8] int32, K packed (GPTQ checkpoint format)
|
||||
# Step 1: bring to [N, K//8] with output(N) at dim 0
|
||||
permute_param_layout_(x, input_dim=1, output_dim=0, packed_dim=1)
|
||||
w = x.data # [N, K//8] int32
|
||||
|
||||
N_dim, K8 = w.shape
|
||||
K_dim = K8 * 8
|
||||
# Step 2: unpack to [N, K] int32 (vectorized)
|
||||
shifts = torch.arange(8, device=w.device, dtype=torch.int32) * 4
|
||||
w_unpacked = ((w.unsqueeze(-1) >> shifts) & 0xF).reshape(N_dim, K_dim)
|
||||
# Step 3: transpose to [K, N] int32
|
||||
w_KN = w_unpacked.t().contiguous()
|
||||
# Step 4: repack N into N//8 int32 values → [K, N//8] (vectorized)
|
||||
N8 = N_dim // 8
|
||||
w_repacked = torch.sum(
|
||||
(w_KN.view(K_dim, N8, 8) & 0xF) << shifts,
|
||||
dim=2,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
x.data = w_repacked.contiguous()
|
||||
return x
|
||||
|
||||
def repack_w_s(x: BasevLLMParameter) -> BasevLLMParameter:
|
||||
# x.data is [N, K//G] fp16, bring to [K//G, N]
|
||||
permute_param_layout_(x, input_dim=1, output_dim=0)
|
||||
x.data = x.data.t().contiguous()
|
||||
return x
|
||||
|
||||
self._transform_param(layer, self.w_q_name, repack_w_q)
|
||||
self._transform_param(layer, self.w_s_name, repack_w_s)
|
||||
|
||||
if self.w_zp_name is not None:
|
||||
zp = getattr(layer, self.w_zp_name, None)
|
||||
if zp is not None:
|
||||
# Checkpoint: [N//8, K//G] int32 (N packed at dim 0, K//G at dim 1)
|
||||
# Kernel needs: [K//G, N//8] — just transpose
|
||||
replace_parameter(
|
||||
layer,
|
||||
self.w_zp_name,
|
||||
torch.nn.Parameter(zp.data.t().contiguous(), requires_grad=False),
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
self, layer: torch.nn.Module, x: torch.Tensor, bias: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
c = self.config
|
||||
w_q, w_s, w_zp, _ = self._get_weight_params(layer)
|
||||
|
||||
x_2d = x.reshape(-1, x.shape[-1]).contiguous()
|
||||
out_shape = x.shape[:-1] + (c.partition_weight_shape[1],)
|
||||
|
||||
K = c.partition_weight_shape[0]
|
||||
group_size = c.group_size if c.group_size != -1 else K
|
||||
|
||||
# For symmetric types (uint4b8), use the scalar bias; no zeros tensor
|
||||
zp_bias = c.weight_type.bias if c.weight_type.has_bias() else 0
|
||||
|
||||
output = triton_w4a16_gemm(
|
||||
a=x_2d,
|
||||
b_q=w_q,
|
||||
scales=w_s,
|
||||
qzeros=w_zp,
|
||||
group_size=group_size,
|
||||
zp_bias=zp_bias,
|
||||
)
|
||||
|
||||
if bias is not None:
|
||||
output.add_(bias)
|
||||
|
||||
return output.reshape(out_shape)
|
||||
Reference in New Issue
Block a user