[ROCm][Bugfix][Perf] Tune multi-stream shared experts use; wvSplitKrc fixes (#56098)

Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Matt
2026-09-10 15:14:22 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent ae71862c51
commit 9163190dda
6 changed files with 368 additions and 76 deletions
@@ -283,24 +283,11 @@ class MoERunner(MoERunnerInterface):
self._shared_experts: SharedExperts | None = None
if shared_experts is not None:
can_overlap = lambda: self._quant_method.mk_can_overlap_shared_experts
# When unquantized, shared expert inputs alias the hidden states,
# which can lead to race condition in multi-stream mode. Quantized
# routed experts copy the input into a fresh buffer first, breaking
# the alias, so overlap is safe. Only observed on ROCm.
routed_input_is_quantized = lambda: (
self.routed_experts.quant_method.moe_quant_config is not None
and self.routed_experts.quant_method.moe_quant_config.quant_dtype
is not None
)
is_multistream_safe = lambda: (
not current_platform.is_rocm() or routed_input_is_quantized()
)
self._shared_experts = SharedExperts(
shared_experts,
moe_config=moe_config,
enable_dbo=enable_dbo,
mk_can_overlap_shared_experts=can_overlap,
is_multistream_safe=is_multistream_safe,
)
# Needed for string -> MoERunner layer lookup in custom ops.
@@ -43,7 +43,6 @@ class SharedExperts(torch.nn.Module):
moe_config: FusedMoEConfig,
enable_dbo: bool,
mk_can_overlap_shared_experts: Callable[[], bool],
is_multistream_safe: Callable[[], bool],
):
super().__init__()
@@ -58,10 +57,6 @@ class SharedExperts(torch.nn.Module):
self._mk_can_overlap_shared_experts = mk_can_overlap_shared_experts
# Might not be safe to run multi-stream mode if routed and shared experts
# alias the same inputs
self._is_multistream_safe = is_multistream_safe
# Allow disabling of the separate shared experts stream for
# debug purposes.
# TODO: Remove this after more extensive testings with TP/DP
@@ -112,19 +107,11 @@ class SharedExperts(torch.nn.Module):
if self._mk_can_overlap_shared_experts():
return SharedExpertsOrder.MK_INTERNAL_OVERLAPPED
# On ROCm, empirically only DP-only deployments benefit from the overlap.
overlap_is_beneficial = not current_platform.is_rocm() or (
self._moe_config.moe_parallel_config.dp_size > 1
and self._moe_config.moe_parallel_config.tp_size == 1
)
should_run_shared_in_aux_stream = (
current_platform.is_cuda_alike()
and self._stream is not None
and hidden_states.shape[0]
<= envs.VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD
and overlap_is_beneficial
and self._is_multistream_safe()
)
if should_run_shared_in_aux_stream:
+50 -19
View File
@@ -249,6 +249,45 @@ def use_aiter_triton_gemm(n, m, k, dtype):
)
def wvsplitkrc_dispatch(n: int, k: int, m: int, cu_count: int) -> tuple[int, bool]:
"""Pick the K-shard split for wvSplitKrc and say whether the shape fits.
Mirrors wvSplitKrc() in csrc/rocm/skinny_gemms.cu, which is also where the
shard cap is explained. Both must pick the same chunkk or the workspace
check here bounds the wrong k_rnd.
Returns:
The CHUNKK the kernel will dispatch with, and whether the CU budget and
split-K workspace admit the shape at all.
"""
# Next ^2 of n
N_p2 = 1 << (n - 1).bit_length()
# How many of 4 waves in a group can work on same 16 Ms at same time?
# This reduces the Ms each group works on, i.e. increasing the CUs needed.
GrpsShrB = min(N_p2 // 16, 4)
# With 64 Ms per CU (each of 4 SIMDs working on a 16x16 tile), and each
# working on a 512-shard of K, how many CUs would we need?
CuNeeded = ((m + 64 - 1) // 64) * ((k + 512 - 1) // 512) * GrpsShrB
CHUNKK2_MAX_SHARDS = 11
shards_chunkk2 = (k + 256 - 1) // 256 # 256-wide shards
chunkk = (
2
if (
N_p2 != 16
and CuNeeded * 2 <= cu_count
and shards_chunkk2 <= CHUNKK2_MAX_SHARDS
)
else 1
)
# Deterministic reduction stores one fp32 partial per (M, N, k-shard); all
# of them must fit the split-K workspace.
k_rnd = (k + 512 // chunkk - 1) // (512 // chunkk)
fits = N_p2 * m * k_rnd <= 128 * 1024 * 12 and CuNeeded <= cu_count
return chunkk, fits
def rocm_unquantized_gemm_impl(
x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None = None
) -> torch.Tensor:
@@ -260,21 +299,7 @@ def rocm_unquantized_gemm_impl(
cu_count = num_compute_units()
# Next ^2 of n
N_p2 = 1 << (n - 1).bit_length()
# With 64 Ms per CU (each of 4 SIMDs working on a 16x16 tile),
# and each working on a 512-shard of K, how many CUs would we need?
rndup_cus = ((m + 64 - 1) // 64) * ((k + 512 - 1) // 512)
# How many of 4 waves in a group can work on same 16 Ms at same time?
# This reduces the Ms each group works on, i.e. increasing the number of CUs needed.
GrpsShrB = min(N_p2 // 16, 4)
# Given the above, how many CUs would we need?
CuNeeded = rndup_cus * GrpsShrB
# Deterministic reduction stores one float workspace value per K shard.
fits_wvsplitkrc = (
N_p2 * m * ((k + 512 - 1) // 512)
) <= 128 * 1024 * 12 # deterministic
fits_wvsplitkrc &= CuNeeded <= cu_count
_, fits_wvsplitkrc = wvsplitkrc_dispatch(n, k, m, cu_count)
skinny_operands_compatible = weight.is_contiguous() and (
bias is None or bias.is_contiguous()
@@ -362,12 +387,18 @@ direct_register_custom_op(
@functools.cache
def warmup_rocm_skinny_gemm_workspaces(device: torch.device) -> None:
"""Eagerly allocate wvSplitKrc's process-lifetime static workspaces.
"""Eagerly allocate wvSplitKrc's per-device split-K workspace pool.
They are otherwise created lazily on the first qualifying GEMM
wvSplitKrc partitions one per-device allocation into ``kWvSlots`` slots
(csrc/rocm/skinny_gemms.cu) and hands each stream one on first use, so that
two streams never share the split-K partials and counters.
The pool is otherwise created lazily on the first qualifying GEMM
(csrc/rocm/skinny_gemms.cu), which can be the first real request — after
the KV cache backing buffer exists. If one landed in that segment's
rounding tail, it would pin the entire segment at engine shutdown.
the KV cache backing buffer exists. If it landed in that segment's rounding
tail, it would pin the entire segment at engine shutdown; it could also land
inside a cudagraph capture, where it would be taken from the graph's private
pool and its zero-fill would become a replayed graph node.
"""
from vllm.platforms.rocm import on_gfx950