[ROCm][BugFix] Triton W4A16 handling for GPTQ/AutoGPTQ qzeros layout (#47770)
Signed-off-by: giuseppegrossi <ggrossi@amd.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
0885b51981
commit
1d99f0f421
@@ -302,3 +302,160 @@ def test_triton_w4a16_process_weights_after_loading_repacks_layout():
|
||||
torch.testing.assert_close(layer.weight_packed, expected_w_kn8)
|
||||
torch.testing.assert_close(layer.weight_scale, expected_scales_gn)
|
||||
torch.testing.assert_close(layer.weight_zero_point, expected_zeros_gn8)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
|
||||
def test_triton_w4a16_process_weights_after_loading_keeps_gptq_qzeros_layout():
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA/HIP device not available")
|
||||
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
ensure_model_parallel_initialized,
|
||||
init_distributed_environment,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import (
|
||||
MPLinearLayerConfig,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
PackedvLLMParameter,
|
||||
)
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
init_distributed_environment(
|
||||
world_size=1,
|
||||
rank=0,
|
||||
distributed_init_method="tcp://127.0.0.1:0",
|
||||
local_rank=0,
|
||||
)
|
||||
ensure_model_parallel_initialized(1, 1)
|
||||
|
||||
set_random_seed(0)
|
||||
|
||||
K, N = 256, 256
|
||||
G = 32
|
||||
|
||||
w_int4_kn = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32)
|
||||
w_gptq_k8n = _pack_int4_along_k_to_ckpt(w_int4_kn).t().contiguous()
|
||||
scales_gptq_gn = 0.05 * torch.rand((K // G, N), device=device, dtype=torch.float16)
|
||||
zeros_int4_gn = torch.randint(0, 16, (K // G, N), device=device, dtype=torch.int32)
|
||||
zeros_gptq_gn8 = _pack_int4_along_n(zeros_int4_gn)
|
||||
|
||||
config = MPLinearLayerConfig(
|
||||
full_weight_shape=(K, N),
|
||||
partition_weight_shape=(K, N),
|
||||
weight_type=scalar_types.uint4,
|
||||
act_type=torch.float16,
|
||||
group_size=G,
|
||||
zero_points=True,
|
||||
has_g_idx=False,
|
||||
)
|
||||
kernel = TritonW4A16LinearKernel(
|
||||
config,
|
||||
w_q_param_name="qweight",
|
||||
w_s_param_name="scales",
|
||||
w_zp_param_name="qzeros",
|
||||
w_gidx_param_name=None,
|
||||
)
|
||||
|
||||
weight_loader = lambda *args, **kwargs: None
|
||||
|
||||
class DummyLayer(torch.nn.Module):
|
||||
pass
|
||||
|
||||
layer = DummyLayer()
|
||||
layer.register_parameter(
|
||||
"qweight",
|
||||
PackedvLLMParameter(
|
||||
data=w_gptq_k8n,
|
||||
weight_loader=weight_loader,
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
packed_factor=8,
|
||||
packed_dim=0,
|
||||
),
|
||||
)
|
||||
layer.register_parameter(
|
||||
"scales",
|
||||
GroupQuantScaleParameter(
|
||||
data=scales_gptq_gn,
|
||||
weight_loader=weight_loader,
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
),
|
||||
)
|
||||
layer.register_parameter(
|
||||
"qzeros",
|
||||
PackedvLLMParameter(
|
||||
data=zeros_gptq_gn8,
|
||||
weight_loader=weight_loader,
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
packed_factor=8,
|
||||
packed_dim=1,
|
||||
),
|
||||
)
|
||||
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
expected_w_kn8 = _pack_int4_along_n(w_int4_kn)
|
||||
|
||||
assert tuple(layer.qweight.shape) == (K, N // 8)
|
||||
assert tuple(layer.scales.shape) == (K // G, N)
|
||||
assert tuple(layer.qzeros.shape) == (K // G, N // 8)
|
||||
|
||||
torch.testing.assert_close(layer.qweight, expected_w_kn8)
|
||||
torch.testing.assert_close(layer.scales, scales_gptq_gn)
|
||||
torch.testing.assert_close(layer.qzeros, zeros_gptq_gn8)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
|
||||
def test_triton_w4a16_symmetric_apply_ignores_qzeros(monkeypatch):
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import (
|
||||
MPLinearLayerConfig,
|
||||
)
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
K, N, G = 256, 256, 32
|
||||
config = MPLinearLayerConfig(
|
||||
full_weight_shape=(K, N),
|
||||
partition_weight_shape=(K, N),
|
||||
weight_type=scalar_types.uint4b8,
|
||||
act_type=torch.float16,
|
||||
group_size=G,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
kernel = TritonW4A16LinearKernel(
|
||||
config,
|
||||
w_q_param_name="qweight",
|
||||
w_s_param_name="scales",
|
||||
w_zp_param_name="qzeros",
|
||||
w_gidx_param_name=None,
|
||||
)
|
||||
|
||||
class DummyLayer(torch.nn.Module):
|
||||
pass
|
||||
|
||||
layer = DummyLayer()
|
||||
layer.qweight = torch.empty((K, N // 8), device=device, dtype=torch.int32)
|
||||
layer.scales = torch.empty((K // G, N), device=device, dtype=torch.float16)
|
||||
layer.qzeros = torch.empty((1, 1), device=device, dtype=torch.int32)
|
||||
|
||||
captured = {}
|
||||
|
||||
def fake_gemm(*, a, b_q, scales, qzeros, group_size, zp_bias):
|
||||
captured["qzeros"] = qzeros
|
||||
captured["zp_bias"] = zp_bias
|
||||
return torch.empty((a.shape[0], N), device=a.device, dtype=a.dtype)
|
||||
|
||||
monkeypatch.setattr(triton_w4a16_module, "triton_w4a16_gemm", fake_gemm)
|
||||
|
||||
x = torch.empty((2, K), device=device, dtype=torch.float16)
|
||||
output = kernel.apply_weights(layer, x)
|
||||
|
||||
assert output.shape == (2, N)
|
||||
assert captured["qzeros"] is None
|
||||
assert captured["zp_bias"] == scalar_types.uint4b8.bias
|
||||
|
||||
@@ -400,12 +400,29 @@ class TritonW4A16LinearKernel(MPLinearKernel):
|
||||
if self.w_zp_name is not None:
|
||||
zp = getattr(layer, self.w_zp_name, None)
|
||||
if zp is not None:
|
||||
# Checkpoint: [N//8, K//G] int32 (N packed at dim 0, K//G at dim 1)
|
||||
# Kernel needs: [K//G, N//8] — just transpose
|
||||
c = self.config
|
||||
K, N = c.partition_weight_shape
|
||||
group_size = c.group_size if c.group_size != -1 else K
|
||||
expected_shape = (K // group_size, N // 8)
|
||||
transposed_shape = (N // 8, K // group_size)
|
||||
|
||||
if tuple(zp.data.shape) == expected_shape:
|
||||
# GPTQ/AutoGPTQ already stores qzeros in the kernel layout.
|
||||
qzeros = zp.data.contiguous()
|
||||
elif tuple(zp.data.shape) == transposed_shape:
|
||||
# Compressed-tensors stores qzeros transposed from what the
|
||||
# kernel needs.
|
||||
qzeros = zp.data.t().contiguous()
|
||||
else:
|
||||
raise AssertionError(
|
||||
f"{self.w_zp_name} shape mismatch: {zp.data.shape}; "
|
||||
f"expected {expected_shape} or {transposed_shape}"
|
||||
)
|
||||
|
||||
replace_parameter(
|
||||
layer,
|
||||
self.w_zp_name,
|
||||
torch.nn.Parameter(zp.data.t().contiguous(), requires_grad=False),
|
||||
torch.nn.Parameter(qzeros, requires_grad=False),
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
@@ -420,14 +437,17 @@ class TritonW4A16LinearKernel(MPLinearKernel):
|
||||
K = c.partition_weight_shape[0]
|
||||
group_size = c.group_size if c.group_size != -1 else K
|
||||
|
||||
# For symmetric types (uint4b8), use the scalar bias; no zeros tensor
|
||||
# For symmetric types (uint4b8), use the scalar bias; no zeros tensor.
|
||||
# Some checkpoint loaders still register qzeros parameters for GPTQ
|
||||
# layers, but they are not part of the symmetric kernel contract.
|
||||
zp_bias = c.weight_type.bias if c.weight_type.has_bias() else 0
|
||||
qzeros = None if c.weight_type.has_bias() else w_zp
|
||||
|
||||
output = triton_w4a16_gemm(
|
||||
a=x_2d,
|
||||
b_q=w_q,
|
||||
scales=w_s,
|
||||
qzeros=w_zp,
|
||||
qzeros=qzeros,
|
||||
group_size=group_size,
|
||||
zp_bias=zp_bias,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user