[ROCm][Bugfix][Perf] Tune multi-stream shared experts use; wvSplitKrc fixes (#56098)

Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Matt
2026-09-10 15:14:22 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent ae71862c51
commit 9163190dda
6 changed files with 368 additions and 76 deletions
@@ -53,6 +53,17 @@ OPTIONS_WVSPLITK_FP8 = [
DTYPES = [torch.bfloat16, torch.float16]
# kWvSlots in csrc/rocm/skinny_gemms.cu; beyond it streams take overflow
# workspaces.
WVSPLITKRC_SLOTS = 8
# The CU count is the grid size. Production passes num_compute_units(); a small
# value manufactures better concurrency for these tests.
WVSPLITKRC_TEST_CU = 16
# Gate spin, ~125 ms on gfx950 -- must outlast the host enqueue loop it gates.
_GATE_CYCLES = 300_000_000
# Specific (N, K, M) combinations for targeted testing
NKM_FACTORS_LLMM1 = [
# Small, medium, large cases
@@ -116,6 +127,15 @@ K_FACTORS_WVSPLITKRC = [2880, 2880 + 8, 3072, 3072 + 8]
# M tiles are 64 rows, +16 for a partial tile
M_FACTORS_WVSPLITKRC = [128, 128 + 16, 256, 256 + 16, 640, 640 + 16]
# (N, K, M) with more K-shards than the readback can stage in one LDS pass.
# Spans both CHUNKK values and both N-tile counts.
NKM_FACTORS_WVSPLITKRC_LARGE_K = [
(128, 6144, 128),
(96, 8192, 128),
(128, 12288, 128),
(32, 12288, 128),
]
NKM_FACTORS_WVSPLITK_FP8 = [
# FP8-specific cases with K % 16 == 0
(1, 16, 16),
@@ -222,6 +242,83 @@ def test_rocm_wvsplitkrc_kernel(n, k, m, dtype, padded_a, bias_mode, xnorm, seed
torch.testing.assert_close(out, ref_out, atol=1e-3, rtol=1e-2)
@pytest.mark.parametrize("n,k,m", NKM_FACTORS_WVSPLITKRC_LARGE_K)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="only test for rocm")
@pytest.mark.skipif(not on_gfx950(), reason="only meant for gfx950")
def test_rocm_wvsplitkrc_large_k(n, k, m, dtype, seed):
"""K large enough that the split-K readback must stage LDS in batches."""
torch.manual_seed(seed)
xavier = math.sqrt(2 / k)
A = (torch.rand(n, k, dtype=dtype, device="cuda") * 2 - 1) * xavier
B = (torch.rand(m, k, dtype=dtype, device="cuda") * 2 - 1) * xavier
ref_out = torch.nn.functional.linear(A, B, None)
out = ops.wvSplitKrc(A, B, num_compute_units(), None)
torch.testing.assert_close(out, ref_out, atol=1e-3, rtol=1e-2)
@pytest.mark.parametrize("n_streams", [WVSPLITKRC_SLOTS, WVSPLITKRC_SLOTS + 4])
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="only test for rocm")
@pytest.mark.skipif(not on_gfx950(), reason="only meant for gfx950")
def test_rocm_wvsplitkrc_multistream(n_streams, dtype, seed):
"""Concurrent streams must not share a split-K workspace.
The workspace is cleared by protocol rather than per invocation, so sharing
one corrupts the reduction and returns wrong numbers rather than failing.
n_streams > WVSPLITKRC_SLOTS additionally exercises the overflow path.
"""
torch.manual_seed(seed)
n, k, m = 16, 2048, 256
iters = 4
xavier = math.sqrt(2 / k)
B = torch.randn(m, k, dtype=dtype, device="cuda") * xavier
As = [
torch.randn(n, k, dtype=dtype, device="cuda") * xavier for _ in range(n_streams)
]
refs = [torch.nn.functional.linear(A, B, None) for A in As]
torch.accelerator.synchronize()
streams = [torch.Stream() for _ in range(n_streams)]
outs: list[list[torch.Tensor]] = [[] for _ in range(n_streams)]
_release_together(streams)
# Round-robin the enqueues so kernels from different streams are in flight
# at the same time.
for _ in range(iters):
for i, s in enumerate(streams):
with torch.cuda.stream(s):
outs[i].append(ops.wvSplitKrc(As[i], B, WVSPLITKRC_TEST_CU, None))
for s in streams:
torch.accelerator.current_stream().wait_stream(s)
torch.accelerator.synchronize()
for i in range(n_streams):
for out in outs[i]:
torch.testing.assert_close(out, refs[i], atol=1e-3, rtol=1e-2)
def _release_together(streams):
"""Release every stream at once, once everything below is enqueued.
An event recorded on an idle stream is already complete, so waiting on it
gates nothing -- the spin, not the event, is what holds the streams.
"""
blocker = torch.Stream()
with torch.cuda.stream(blocker):
torch.cuda._sleep(_GATE_CYCLES)
gate = torch.Event()
gate.record()
for s in streams:
s.wait_event(gate)
@pytest.mark.parametrize("n,k,m", NKM_FACTORS_LLMM1)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("rows_per_block", [2, 4, 8, 16])