[ROCm][Perf] W4A16: pad gfx11 weight and activation strides (#56301)

Signed-off-by: Matthias Gehre <matthias.gehre@amd.com>
This commit is contained in:
Matthias Gehre
2026-10-06 08:22:33 -05:00
committed by GitHub
parent 049507aa76
commit b267fe4881
4 changed files with 316 additions and 46 deletions
@@ -563,3 +563,182 @@ def test_wvsplitk_int4_g_rejects_unpacked_zero_points():
with pytest.raises(RuntimeError, match="Zero points must be int32 or uint32"):
ops.wvSplitK_int4_g(w, a, scales, num_compute_units(), G, zp_unpacked, None)
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
@pytest.mark.skipif(
not hasattr(torch.ops, "_rocm_C")
or not hasattr(torch.ops._rocm_C, "wvSplitK_int4_g"),
reason="wvSplitK_int4_g not built",
)
def test_wvsplitk_int4_g_rejects_undersized_weight_view():
"""A sufficient row stride does not prove the row holds K/2 packed bytes.
Row padding makes stride(0) > K/2 legitimate, but the logical size still
has to be checked: this view has a valid stride and half the required
bytes, and without the size check the kernel reads past it.
"""
import vllm._custom_ops as ops
from vllm.utils.platform_utils import num_compute_units
K, N, G, M = 512, 256, 128, 1
a = torch.randn((M, K), device=device, dtype=torch.float16)
# Full-size backing allocation, so a missing check misreads rather than
# running off the end of the buffer.
backing = torch.randint(0, 255, (N, K // 2), device=device, dtype=torch.uint8).view(
torch.int8
)
undersized = backing[:, : K // 4]
assert undersized.stride(0) == K // 2
scales = torch.rand((N, K // G), device=device, dtype=torch.float16)
with pytest.raises(RuntimeError, match=r"must contain M\*K/2 bytes"):
ops.wvSplitK_int4_g(undersized, a, scales, num_compute_units(), G, None, None)
# ---------------------------------------------------------------------------
# gfx11 weight row-stride padding
# ---------------------------------------------------------------------------
_weight_pad_bytes = hybrid_module._weight_pad_bytes
_act_pad_bytes = hybrid_module._act_pad_bytes
_WEIGHT_CLIFF_BYTES = hybrid_module._WEIGHT_CLIFF_BYTES
_ACT_CLIFF_BYTES = hybrid_module._ACT_CLIFF_BYTES
_STRIDE_PAD_BYTES = hybrid_module._STRIDE_PAD_BYTES
def test_weight_pad_bytes_only_moves_strides_on_the_cliff():
"""Pad 1024 B multiples, leave everything else dense."""
# On the cliff (K % 2048 == 0): 1024 B multiples.
for row_bytes in (1024, 2048, 4096, 5120, 6144, 7168, 8192):
assert _weight_pad_bytes(row_bytes) == _STRIDE_PAD_BYTES
# Off it, including strides that are 512 B multiples but not 1024 B ones:
# padding those measured as a real loss on gfx1151 (-14% at M=1 on a
# 4864 B row).
for row_bytes in (1280, 1536, 2560, 4864, 9472, 12800):
assert _weight_pad_bytes(row_bytes) == 0
# The padded stride is never back on the cliff.
for row_bytes in range(16, 16384, 16):
padded = row_bytes + _weight_pad_bytes(row_bytes)
assert padded % _WEIGHT_CLIFF_BYTES != 0 or row_bytes % _WEIGHT_CLIFF_BYTES
def test_act_pad_bytes_only_moves_strides_on_the_cliff():
"""Activations sit on a wider cliff than the packed weight: 2048 B."""
# On the cliff (K % 1024 == 0 for a 2-byte dtype): 2048 B multiples.
for row_bytes in (2048, 4096, 8192, 16384, 20480, 24576, 43008):
assert _act_pad_bytes(row_bytes) == _STRIDE_PAD_BYTES
# Off it, including 1024 B multiples that are not 2048 B ones -- the
# weight cliff must not be applied to activations.
for row_bytes in (1024, 3072, 5120, 10752, 19456, 37888):
assert _act_pad_bytes(row_bytes) == 0
# The padded stride is never back on the cliff.
for row_bytes in range(16, 65536, 16):
padded = row_bytes + _act_pad_bytes(row_bytes)
assert padded % _ACT_CLIFF_BYTES != 0 or row_bytes % _ACT_CLIFF_BYTES
def _row_padded_copy(t: torch.Tensor, pad_cols: int) -> torch.Tensor:
"""Copy of ``t`` whose stride(0) is ``t.shape[1] + pad_cols``."""
rows, cols = t.shape
buf = torch.empty((rows, cols + pad_cols), dtype=t.dtype, device=t.device)
buf[:, :cols].copy_(t)
return buf[:, :cols]
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("has_zp", [False, True])
@pytest.mark.parametrize("M", [1, MAX_SKINNY_BATCH_SIZE + 1])
def test_weight_row_padding_does_not_change_results(dtype, has_zp, M):
"""Padding the weight rows is a pure layout change.
Both the HIP skinny kernel (M small) and the Triton prefill kernel take the
weight row stride from the tensor, so a padded layout must produce exactly
the same numbers as the dense one.
"""
from vllm.utils.platform_utils import num_compute_units
set_random_seed(0)
K, N, G = 512, 256, 128
a = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
w_int4_nk = torch.randint(0, 16, (N, K), device=device, dtype=torch.int32)
w_q = pack_int4_exllama_shuffle(w_int4_nk).view(torch.int8)
scales = (0.05 * torch.rand((N, K // G), device=device, dtype=torch.float32)).to(
dtype
)
zp = (
_pack_zp_rows_for_kernel(
torch.randint(0, 16, (N, K // G), device=device, dtype=torch.int32)
)
if has_zp
else None
)
# 32 int32 columns = 128 B of pad, the production pad size.
w_q_pad = _row_padded_copy(w_q.view(torch.int32), 32).view(torch.int8)
assert w_q_pad.stride(0) == w_q.stride(0) + _STRIDE_PAD_BYTES
cu_count = num_compute_units()
dense = torch.ops.vllm.rdna_hybrid_w4a16_apply(
a, w_q, scales, zp, None, cu_count, G
)
padded = torch.ops.vllm.rdna_hybrid_w4a16_apply(
a, w_q_pad, scales, zp, None, cu_count, G
)
torch.testing.assert_close(padded, dense, rtol=0, atol=0)
@pytest.mark.skipif(
not hybrid_module._on_gfx1151(),
reason="row-stride padding is only enabled on gfx1151",
)
@pytest.mark.parametrize(
"K,expected_weight_stride",
[
(512, 256), # 256 B row: off the cliff, stored dense
(8192, 4096 + 128), # 4096 B row: on the cliff, padded
],
)
def test_process_weights_pads_cliff_rows(K, expected_weight_stride, dist_init):
"""The stored weight row stride follows the padding rule."""
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import (
MPLinearLayerConfig,
)
from vllm.scalar_type import scalar_types
set_random_seed(0)
N, G = 128, 128
w_int4_kn = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32)
layer = _build_dummy_layer(
_pack_int4_along_k_to_ckpt(w_int4_kn),
0.05 * torch.rand((N, K // G), device=device, dtype=torch.float16),
zeros_ckpt=None,
)
config = MPLinearLayerConfig(
full_weight_shape=(K, N),
partition_weight_shape=(K, N),
weight_type=scalar_types.uint4b8,
act_type=torch.float16,
group_size=G,
zero_points=False,
)
RDNAHybridW4A16LinearKernel(
config,
w_q_param_name="weight_packed",
w_s_param_name="weight_scale",
w_zp_param_name=None,
).process_weights_after_loading(layer)
# Weight rows: stride is in int8 elements, i.e. bytes.
assert layer.weight_packed.stride(0) == expected_weight_stride
# The int32 view the Triton path uses survives the padded stride.
assert tuple(layer.weight_packed.view(torch.int32).shape) == (N, K // 8)
# Metadata is untouched by this change.
assert layer.weight_scale.is_contiguous()