[ROCm][Perf] W4A16: speed up skinny GEMM decode, including speculative batches (#57199)

Signed-off-by: Robert Esclapez Garcia <robert.garcia@amd.com>
Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
Robert Esclapez
2026-10-13 17:21:30 +00:00
committed by GitHub
co-authored by Claude Opus 4.7 Douglas Lehr
parent 7fad7db330
commit e83222396f
4 changed files with 603 additions and 117 deletions
@@ -19,7 +19,7 @@ if not current_platform.is_rocm():
pytest.importorskip("triton")
from vllm.platforms.rocm import on_gfx1x # noqa: E402
from vllm.platforms.rocm import on_gfx1x, on_gfx115x # noqa: E402
device = "cuda"
@@ -446,22 +446,28 @@ def _hip_skinny_reference(
scales_nkg: torch.Tensor,
*,
group_size: int,
zp_bias: int,
zp_nkg: torch.Tensor | None,
) -> torch.Tensor:
"""Reference for symmetric HIP skinny: C = A @ (W - zp_bias) * S."""
"""Reference for HIP skinny: C = A @ (W - zp) * S.
``zp_nkg`` is [N, K//G] int32 raw zero points for the asymmetric case, or
None for symmetric uint4b8, where the kernel subtracts a constant 8.
"""
K = a_mk.shape[1]
N = w_int4_nk.shape[0]
num_groups = K // group_size
w_fp = (w_int4_nk.to(torch.float32) - zp_bias).view(N, num_groups, group_size)
w_g = w_int4_nk.to(torch.float32).view(N, num_groups, group_size)
zp = 8.0 if zp_nkg is None else zp_nkg.to(torch.float32).unsqueeze(-1)
s = scales_nkg.to(torch.float32).unsqueeze(-1)
w_dequant = (w_fp * s).view(N, K)
w_dequant = ((w_g - zp) * s).view(N, K)
return (a_mk.to(torch.float32) @ w_dequant.t()).to(a_mk.dtype)
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("has_zp", [False, True])
@pytest.mark.parametrize(
"M,K,N,G",
[
@@ -470,13 +476,50 @@ def _hip_skinny_reference(
(1, 512, 256, 128),
(2, 512, 256, 64),
(3, 256, 512, 64),
# M is the kernel's batch dimension; 4 and 5 reach dispatch tuples that
# 1..3 never take, and 5 is MAX_SKINNY_BATCH_SIZE.
(4, 512, 256, 32),
(4, 256, 512, 64),
(5, 512, 256, 128),
(5, 256, 256, 32),
# N == 8 is exactly one packed zero-point word.
(1, 256, 8, 32),
# K must exceed THRDS * A_CHUNK * UNRL for the K loop to run a whole
# unrolled block; every K above is below that bound, so without these
# the deep-K path goes untested. 2560 is not a multiple of the 1024 K
# step the batch 2 and 4 tuples take, so it also hits the ragged tail.
(1, 4096, 256, 128),
(2, 2560, 256, 32),
(3, 4096, 256, 64),
(4, 2560, 256, 128),
(5, 4096, 256, 32),
# On gfx115x, batch 1 with K == 4096, or symmetric with K a multiple of
# 8192, takes a tuple with A_CHUNK = 32 instead of 16. G = 32 is the
# tight case there: each lane's A_CHUNK spans exactly one scale group.
(1, 4096, 256, 32),
(1, 8192, 256, 128),
(1, 16384, 256, 64),
# K * batch beyond what LDS holds (32768 fp16 elements), so gfx115x
# walks K in chunks; elsewhere the op rejects these, as the layer
# routes them to Triton. The per-row window shrinks as the batch grows,
# so these cover a single reload and several, with and without a short
# final chunk. 20992 is not a multiple of the K step either, so it
# crosses a chunk boundary *and* has a ragged K tail.
(2, 20480, 512, 64),
(3, 16384, 512, 32),
(4, 16384, 512, 128),
(5, 21504, 512, 32),
(5, 20992, 512, 128),
],
)
def test_hip_skinny_wvSplitK_int4_g(dtype, M, K, N, G):
def test_hip_skinny_wvSplitK_int4_g(dtype, M, K, N, G, has_zp):
"""Test HIP wvSplitK_int4_g kernel directly via _custom_ops."""
import vllm._custom_ops as ops
from vllm.utils.platform_utils import num_compute_units
if K * M > LDS_CAPACITY_ELEMENTS and not on_gfx115x():
pytest.skip("deep-K HIP path is gfx115x only")
set_random_seed(0)
a = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
@@ -489,10 +532,64 @@ def test_hip_skinny_wvSplitK_int4_g(dtype, M, K, N, G):
dtype
)
cu_count = num_compute_units()
out = ops.wvSplitK_int4_g(b_packed_i8, a, scales, cu_count, G)
zp_nkg = None
zp_packed = None
if has_zp:
zp_nkg = torch.randint(0, 16, (N, K // G), device=device, dtype=torch.int32)
zp_packed = _pack_zp_rows_for_kernel(zp_nkg)
ref = _hip_skinny_reference(a, w_int4_nk, scales, group_size=G, zp_bias=8)
cu_count = num_compute_units()
out = ops.wvSplitK_int4_g(b_packed_i8, a, scales, cu_count, G, zp_packed)
ref = _hip_skinny_reference(a, w_int4_nk, scales, group_size=G, zp_nkg=zp_nkg)
torch.testing.assert_close(out, ref, rtol=1e-2, atol=5e-2)
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("G", [32, 128])
@pytest.mark.parametrize("has_zp", [False, True])
@pytest.mark.parametrize("M", [2, 64], ids=["M=2_decode", "M=64_prefill"])
def test_hip_skinny_padded_group_stride(dtype, G, has_zp, M):
"""Scale and zero-point rows may be padded past K // group_size.
Both paths index them by their own row stride, so a padded row is read
correctly rather than bleeding into the next row. M picks the path: the
HIP skinny kernel at 2, the Triton prefill kernel at 64.
"""
from vllm.utils.platform_utils import num_compute_units
set_random_seed(0)
K, N = 2048, 256
num_groups = K // G
pad = 7
a = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
w_int4_nk = torch.randint(0, 16, (N, K), device=device, dtype=torch.int32)
b_packed_i8 = pack_int4_exllama_shuffle(w_int4_nk).view(torch.int8)
scales = (0.05 * torch.rand((N, num_groups), device=device)).to(dtype)
scales_padded = torch.zeros((N, num_groups + pad), device=device, dtype=dtype)
scales_padded[:, :num_groups] = scales
scales_view = scales_padded[:, :num_groups]
assert scales_view.stride(0) == num_groups + pad
zp_nkg = zp_view = None
if has_zp:
zp_nkg = torch.randint(0, 16, (N, num_groups), device=device, dtype=torch.int32)
zp_packed = _pack_zp_rows_for_kernel(zp_nkg)
zp_padded = torch.zeros(
(N // 8, num_groups + pad), device=device, dtype=torch.int32
)
zp_padded[:, :num_groups] = zp_packed
zp_view = zp_padded[:, :num_groups]
cu_count = num_compute_units()
out = torch.ops.vllm.rdna_hybrid_w4a16_apply(
a, b_packed_i8, scales_view, zp_view, None, cu_count, G
)
ref = _hip_skinny_reference(a, w_int4_nk, scales, group_size=G, zp_nkg=zp_nkg)
torch.testing.assert_close(out, ref, rtol=1e-2, atol=5e-2)
@@ -510,6 +607,8 @@ def test_hip_skinny_wvSplitK_int4_g(dtype, M, K, N, G):
(1, 256, 256, 64),
(1, 512, 256, 32),
(1, 512, 256, 128),
# Past LDS: HIP on gfx115x, Triton elsewhere.
(2, 20480, 256, 64),
(32, 512, 256, 64),
(64, 1024, 256, 128),
],
@@ -535,7 +634,7 @@ def test_rdna_hybrid_w4a16_dispatch(dtype, M, K, N, G):
a, b_packed_i8, scales, None, None, cu_count, G
)
ref = _hip_skinny_reference(a, w_int4_nk, scales, group_size=G, zp_bias=8)
ref = _hip_skinny_reference(a, w_int4_nk, scales, group_size=G, zp_nkg=None)
torch.testing.assert_close(out, ref, rtol=1e-2, atol=5e-2)
@@ -748,17 +847,30 @@ def test_process_weights_pads_cliff_rows(K, expected_weight_stride, dist_init):
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
@pytest.mark.parametrize(
"M,K,expected_path",
"M,K,gfx115x,expected_path",
[
(4, 8192, "hip"),
(4, 9728, "hip"),
(4, 9856, "triton"),
(MAX_SKINNY_BATCH_SIZE + 1, 1024, "triton"),
(4, 8192, False, "hip"),
(4, 9728, False, "hip"),
(4, 9856, False, "triton"),
(4, 9856, True, "hip"),
(MAX_SKINNY_BATCH_SIZE + 1, 1024, True, "triton"),
],
ids=[
"regular_limit",
"medium_range",
"above_medium",
"above_medium_gfx115x",
"batch_too_large",
],
ids=["regular_limit", "medium_range", "above_medium", "batch_too_large"],
)
def test_rdna_hybrid_w4a16_dispatch_boundaries(M, K, expected_path, monkeypatch):
"""Route only supported regular and medium skinny shapes to HIP."""
def test_rdna_hybrid_w4a16_dispatch_boundaries(
M, K, gfx115x, expected_path, monkeypatch
):
"""Route only supported regular and medium skinny shapes to HIP.
gfx115x drops the K*M bound, so a shape past the medium limit still goes to
HIP there; the batch bound holds on every target.
"""
import vllm._custom_ops as ops
N, G = 16, 128
@@ -777,6 +889,7 @@ def test_rdna_hybrid_w4a16_dispatch_boundaries(M, K, expected_path, monkeypatch)
monkeypatch.setattr(ops, "wvSplitK_int4_g", fake_hip)
monkeypatch.setattr(hybrid_module, "triton_w4a16_skinny_fmt_gemm", fake_triton)
monkeypatch.setattr(hybrid_module, "_on_gfx115x", lambda: gfx115x)
output = hybrid_module._rdna_hybrid_w4a16_apply_impl(
x_mk, w_q, scales, None, None, 40, G