[ROCm][Perf] W4A16: speed up skinny GEMM decode, including speculative batches (#57199)
Signed-off-by: Robert Esclapez Garcia <robert.garcia@amd.com> Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
Douglas Lehr
parent
7fad7db330
commit
e83222396f
@@ -19,7 +19,7 @@ if not current_platform.is_rocm():
|
||||
|
||||
pytest.importorskip("triton")
|
||||
|
||||
from vllm.platforms.rocm import on_gfx1x # noqa: E402
|
||||
from vllm.platforms.rocm import on_gfx1x, on_gfx115x # noqa: E402
|
||||
|
||||
device = "cuda"
|
||||
|
||||
@@ -446,22 +446,28 @@ def _hip_skinny_reference(
|
||||
scales_nkg: torch.Tensor,
|
||||
*,
|
||||
group_size: int,
|
||||
zp_bias: int,
|
||||
zp_nkg: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
"""Reference for symmetric HIP skinny: C = A @ (W - zp_bias) * S."""
|
||||
"""Reference for HIP skinny: C = A @ (W - zp) * S.
|
||||
|
||||
``zp_nkg`` is [N, K//G] int32 raw zero points for the asymmetric case, or
|
||||
None for symmetric uint4b8, where the kernel subtracts a constant 8.
|
||||
"""
|
||||
K = a_mk.shape[1]
|
||||
N = w_int4_nk.shape[0]
|
||||
num_groups = K // group_size
|
||||
|
||||
w_fp = (w_int4_nk.to(torch.float32) - zp_bias).view(N, num_groups, group_size)
|
||||
w_g = w_int4_nk.to(torch.float32).view(N, num_groups, group_size)
|
||||
zp = 8.0 if zp_nkg is None else zp_nkg.to(torch.float32).unsqueeze(-1)
|
||||
s = scales_nkg.to(torch.float32).unsqueeze(-1)
|
||||
w_dequant = (w_fp * s).view(N, K)
|
||||
w_dequant = ((w_g - zp) * s).view(N, K)
|
||||
|
||||
return (a_mk.to(torch.float32) @ w_dequant.t()).to(a_mk.dtype)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("has_zp", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"M,K,N,G",
|
||||
[
|
||||
@@ -470,13 +476,50 @@ def _hip_skinny_reference(
|
||||
(1, 512, 256, 128),
|
||||
(2, 512, 256, 64),
|
||||
(3, 256, 512, 64),
|
||||
# M is the kernel's batch dimension; 4 and 5 reach dispatch tuples that
|
||||
# 1..3 never take, and 5 is MAX_SKINNY_BATCH_SIZE.
|
||||
(4, 512, 256, 32),
|
||||
(4, 256, 512, 64),
|
||||
(5, 512, 256, 128),
|
||||
(5, 256, 256, 32),
|
||||
# N == 8 is exactly one packed zero-point word.
|
||||
(1, 256, 8, 32),
|
||||
# K must exceed THRDS * A_CHUNK * UNRL for the K loop to run a whole
|
||||
# unrolled block; every K above is below that bound, so without these
|
||||
# the deep-K path goes untested. 2560 is not a multiple of the 1024 K
|
||||
# step the batch 2 and 4 tuples take, so it also hits the ragged tail.
|
||||
(1, 4096, 256, 128),
|
||||
(2, 2560, 256, 32),
|
||||
(3, 4096, 256, 64),
|
||||
(4, 2560, 256, 128),
|
||||
(5, 4096, 256, 32),
|
||||
# On gfx115x, batch 1 with K == 4096, or symmetric with K a multiple of
|
||||
# 8192, takes a tuple with A_CHUNK = 32 instead of 16. G = 32 is the
|
||||
# tight case there: each lane's A_CHUNK spans exactly one scale group.
|
||||
(1, 4096, 256, 32),
|
||||
(1, 8192, 256, 128),
|
||||
(1, 16384, 256, 64),
|
||||
# K * batch beyond what LDS holds (32768 fp16 elements), so gfx115x
|
||||
# walks K in chunks; elsewhere the op rejects these, as the layer
|
||||
# routes them to Triton. The per-row window shrinks as the batch grows,
|
||||
# so these cover a single reload and several, with and without a short
|
||||
# final chunk. 20992 is not a multiple of the K step either, so it
|
||||
# crosses a chunk boundary *and* has a ragged K tail.
|
||||
(2, 20480, 512, 64),
|
||||
(3, 16384, 512, 32),
|
||||
(4, 16384, 512, 128),
|
||||
(5, 21504, 512, 32),
|
||||
(5, 20992, 512, 128),
|
||||
],
|
||||
)
|
||||
def test_hip_skinny_wvSplitK_int4_g(dtype, M, K, N, G):
|
||||
def test_hip_skinny_wvSplitK_int4_g(dtype, M, K, N, G, has_zp):
|
||||
"""Test HIP wvSplitK_int4_g kernel directly via _custom_ops."""
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
|
||||
if K * M > LDS_CAPACITY_ELEMENTS and not on_gfx115x():
|
||||
pytest.skip("deep-K HIP path is gfx115x only")
|
||||
|
||||
set_random_seed(0)
|
||||
|
||||
a = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
|
||||
@@ -489,10 +532,64 @@ def test_hip_skinny_wvSplitK_int4_g(dtype, M, K, N, G):
|
||||
dtype
|
||||
)
|
||||
|
||||
cu_count = num_compute_units()
|
||||
out = ops.wvSplitK_int4_g(b_packed_i8, a, scales, cu_count, G)
|
||||
zp_nkg = None
|
||||
zp_packed = None
|
||||
if has_zp:
|
||||
zp_nkg = torch.randint(0, 16, (N, K // G), device=device, dtype=torch.int32)
|
||||
zp_packed = _pack_zp_rows_for_kernel(zp_nkg)
|
||||
|
||||
ref = _hip_skinny_reference(a, w_int4_nk, scales, group_size=G, zp_bias=8)
|
||||
cu_count = num_compute_units()
|
||||
out = ops.wvSplitK_int4_g(b_packed_i8, a, scales, cu_count, G, zp_packed)
|
||||
|
||||
ref = _hip_skinny_reference(a, w_int4_nk, scales, group_size=G, zp_nkg=zp_nkg)
|
||||
|
||||
torch.testing.assert_close(out, ref, rtol=1e-2, atol=5e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("G", [32, 128])
|
||||
@pytest.mark.parametrize("has_zp", [False, True])
|
||||
@pytest.mark.parametrize("M", [2, 64], ids=["M=2_decode", "M=64_prefill"])
|
||||
def test_hip_skinny_padded_group_stride(dtype, G, has_zp, M):
|
||||
"""Scale and zero-point rows may be padded past K // group_size.
|
||||
|
||||
Both paths index them by their own row stride, so a padded row is read
|
||||
correctly rather than bleeding into the next row. M picks the path: the
|
||||
HIP skinny kernel at 2, the Triton prefill kernel at 64.
|
||||
"""
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
|
||||
set_random_seed(0)
|
||||
K, N = 2048, 256
|
||||
num_groups = K // G
|
||||
pad = 7
|
||||
|
||||
a = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
|
||||
w_int4_nk = torch.randint(0, 16, (N, K), device=device, dtype=torch.int32)
|
||||
b_packed_i8 = pack_int4_exllama_shuffle(w_int4_nk).view(torch.int8)
|
||||
|
||||
scales = (0.05 * torch.rand((N, num_groups), device=device)).to(dtype)
|
||||
scales_padded = torch.zeros((N, num_groups + pad), device=device, dtype=dtype)
|
||||
scales_padded[:, :num_groups] = scales
|
||||
scales_view = scales_padded[:, :num_groups]
|
||||
assert scales_view.stride(0) == num_groups + pad
|
||||
|
||||
zp_nkg = zp_view = None
|
||||
if has_zp:
|
||||
zp_nkg = torch.randint(0, 16, (N, num_groups), device=device, dtype=torch.int32)
|
||||
zp_packed = _pack_zp_rows_for_kernel(zp_nkg)
|
||||
zp_padded = torch.zeros(
|
||||
(N // 8, num_groups + pad), device=device, dtype=torch.int32
|
||||
)
|
||||
zp_padded[:, :num_groups] = zp_packed
|
||||
zp_view = zp_padded[:, :num_groups]
|
||||
|
||||
cu_count = num_compute_units()
|
||||
out = torch.ops.vllm.rdna_hybrid_w4a16_apply(
|
||||
a, b_packed_i8, scales_view, zp_view, None, cu_count, G
|
||||
)
|
||||
ref = _hip_skinny_reference(a, w_int4_nk, scales, group_size=G, zp_nkg=zp_nkg)
|
||||
|
||||
torch.testing.assert_close(out, ref, rtol=1e-2, atol=5e-2)
|
||||
|
||||
@@ -510,6 +607,8 @@ def test_hip_skinny_wvSplitK_int4_g(dtype, M, K, N, G):
|
||||
(1, 256, 256, 64),
|
||||
(1, 512, 256, 32),
|
||||
(1, 512, 256, 128),
|
||||
# Past LDS: HIP on gfx115x, Triton elsewhere.
|
||||
(2, 20480, 256, 64),
|
||||
(32, 512, 256, 64),
|
||||
(64, 1024, 256, 128),
|
||||
],
|
||||
@@ -535,7 +634,7 @@ def test_rdna_hybrid_w4a16_dispatch(dtype, M, K, N, G):
|
||||
a, b_packed_i8, scales, None, None, cu_count, G
|
||||
)
|
||||
|
||||
ref = _hip_skinny_reference(a, w_int4_nk, scales, group_size=G, zp_bias=8)
|
||||
ref = _hip_skinny_reference(a, w_int4_nk, scales, group_size=G, zp_nkg=None)
|
||||
|
||||
torch.testing.assert_close(out, ref, rtol=1e-2, atol=5e-2)
|
||||
|
||||
@@ -748,17 +847,30 @@ def test_process_weights_pads_cliff_rows(K, expected_weight_stride, dist_init):
|
||||
|
||||
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
|
||||
@pytest.mark.parametrize(
|
||||
"M,K,expected_path",
|
||||
"M,K,gfx115x,expected_path",
|
||||
[
|
||||
(4, 8192, "hip"),
|
||||
(4, 9728, "hip"),
|
||||
(4, 9856, "triton"),
|
||||
(MAX_SKINNY_BATCH_SIZE + 1, 1024, "triton"),
|
||||
(4, 8192, False, "hip"),
|
||||
(4, 9728, False, "hip"),
|
||||
(4, 9856, False, "triton"),
|
||||
(4, 9856, True, "hip"),
|
||||
(MAX_SKINNY_BATCH_SIZE + 1, 1024, True, "triton"),
|
||||
],
|
||||
ids=[
|
||||
"regular_limit",
|
||||
"medium_range",
|
||||
"above_medium",
|
||||
"above_medium_gfx115x",
|
||||
"batch_too_large",
|
||||
],
|
||||
ids=["regular_limit", "medium_range", "above_medium", "batch_too_large"],
|
||||
)
|
||||
def test_rdna_hybrid_w4a16_dispatch_boundaries(M, K, expected_path, monkeypatch):
|
||||
"""Route only supported regular and medium skinny shapes to HIP."""
|
||||
def test_rdna_hybrid_w4a16_dispatch_boundaries(
|
||||
M, K, gfx115x, expected_path, monkeypatch
|
||||
):
|
||||
"""Route only supported regular and medium skinny shapes to HIP.
|
||||
|
||||
gfx115x drops the K*M bound, so a shape past the medium limit still goes to
|
||||
HIP there; the batch bound holds on every target.
|
||||
"""
|
||||
import vllm._custom_ops as ops
|
||||
|
||||
N, G = 16, 128
|
||||
@@ -777,6 +889,7 @@ def test_rdna_hybrid_w4a16_dispatch_boundaries(M, K, expected_path, monkeypatch)
|
||||
|
||||
monkeypatch.setattr(ops, "wvSplitK_int4_g", fake_hip)
|
||||
monkeypatch.setattr(hybrid_module, "triton_w4a16_skinny_fmt_gemm", fake_triton)
|
||||
monkeypatch.setattr(hybrid_module, "_on_gfx115x", lambda: gfx115x)
|
||||
|
||||
output = hybrid_module._rdna_hybrid_w4a16_apply_impl(
|
||||
x_mk, w_q, scales, None, None, 40, G
|
||||
|
||||
Reference in New Issue
Block a user