[ROCm][Bugfix][Perf] Tune multi-stream shared experts use; wvSplitKrc fixes (#56098)

Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Matt
2026-09-10 15:14:22 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent ae71862c51
commit 9163190dda
6 changed files with 368 additions and 76 deletions
+50 -19
View File
@@ -249,6 +249,45 @@ def use_aiter_triton_gemm(n, m, k, dtype):
)
def wvsplitkrc_dispatch(n: int, k: int, m: int, cu_count: int) -> tuple[int, bool]:
"""Pick the K-shard split for wvSplitKrc and say whether the shape fits.
Mirrors wvSplitKrc() in csrc/rocm/skinny_gemms.cu, which is also where the
shard cap is explained. Both must pick the same chunkk or the workspace
check here bounds the wrong k_rnd.
Returns:
The CHUNKK the kernel will dispatch with, and whether the CU budget and
split-K workspace admit the shape at all.
"""
# Next ^2 of n
N_p2 = 1 << (n - 1).bit_length()
# How many of 4 waves in a group can work on same 16 Ms at same time?
# This reduces the Ms each group works on, i.e. increasing the CUs needed.
GrpsShrB = min(N_p2 // 16, 4)
# With 64 Ms per CU (each of 4 SIMDs working on a 16x16 tile), and each
# working on a 512-shard of K, how many CUs would we need?
CuNeeded = ((m + 64 - 1) // 64) * ((k + 512 - 1) // 512) * GrpsShrB
CHUNKK2_MAX_SHARDS = 11
shards_chunkk2 = (k + 256 - 1) // 256 # 256-wide shards
chunkk = (
2
if (
N_p2 != 16
and CuNeeded * 2 <= cu_count
and shards_chunkk2 <= CHUNKK2_MAX_SHARDS
)
else 1
)
# Deterministic reduction stores one fp32 partial per (M, N, k-shard); all
# of them must fit the split-K workspace.
k_rnd = (k + 512 // chunkk - 1) // (512 // chunkk)
fits = N_p2 * m * k_rnd <= 128 * 1024 * 12 and CuNeeded <= cu_count
return chunkk, fits
def rocm_unquantized_gemm_impl(
x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None = None
) -> torch.Tensor:
@@ -260,21 +299,7 @@ def rocm_unquantized_gemm_impl(
cu_count = num_compute_units()
# Next ^2 of n
N_p2 = 1 << (n - 1).bit_length()
# With 64 Ms per CU (each of 4 SIMDs working on a 16x16 tile),
# and each working on a 512-shard of K, how many CUs would we need?
rndup_cus = ((m + 64 - 1) // 64) * ((k + 512 - 1) // 512)
# How many of 4 waves in a group can work on same 16 Ms at same time?
# This reduces the Ms each group works on, i.e. increasing the number of CUs needed.
GrpsShrB = min(N_p2 // 16, 4)
# Given the above, how many CUs would we need?
CuNeeded = rndup_cus * GrpsShrB
# Deterministic reduction stores one float workspace value per K shard.
fits_wvsplitkrc = (
N_p2 * m * ((k + 512 - 1) // 512)
) <= 128 * 1024 * 12 # deterministic
fits_wvsplitkrc &= CuNeeded <= cu_count
_, fits_wvsplitkrc = wvsplitkrc_dispatch(n, k, m, cu_count)
skinny_operands_compatible = weight.is_contiguous() and (
bias is None or bias.is_contiguous()
@@ -362,12 +387,18 @@ direct_register_custom_op(
@functools.cache
def warmup_rocm_skinny_gemm_workspaces(device: torch.device) -> None:
"""Eagerly allocate wvSplitKrc's process-lifetime static workspaces.
"""Eagerly allocate wvSplitKrc's per-device split-K workspace pool.
They are otherwise created lazily on the first qualifying GEMM
wvSplitKrc partitions one per-device allocation into ``kWvSlots`` slots
(csrc/rocm/skinny_gemms.cu) and hands each stream one on first use, so that
two streams never share the split-K partials and counters.
The pool is otherwise created lazily on the first qualifying GEMM
(csrc/rocm/skinny_gemms.cu), which can be the first real request — after
the KV cache backing buffer exists. If one landed in that segment's
rounding tail, it would pin the entire segment at engine shutdown.
the KV cache backing buffer exists. If it landed in that segment's rounding
tail, it would pin the entire segment at engine shutdown; it could also land
inside a cudagraph capture, where it would be taken from the graph's private
pool and its zero-fill would become a replayed graph node.
"""
from vllm.platforms.rocm import on_gfx950