[ROCm][Bugfix][Perf] Tune multi-stream shared experts use; wvSplitKrc fixes (#56098)
Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -283,24 +283,11 @@ class MoERunner(MoERunnerInterface):
|
||||
self._shared_experts: SharedExperts | None = None
|
||||
if shared_experts is not None:
|
||||
can_overlap = lambda: self._quant_method.mk_can_overlap_shared_experts
|
||||
# When unquantized, shared expert inputs alias the hidden states,
|
||||
# which can lead to race condition in multi-stream mode. Quantized
|
||||
# routed experts copy the input into a fresh buffer first, breaking
|
||||
# the alias, so overlap is safe. Only observed on ROCm.
|
||||
routed_input_is_quantized = lambda: (
|
||||
self.routed_experts.quant_method.moe_quant_config is not None
|
||||
and self.routed_experts.quant_method.moe_quant_config.quant_dtype
|
||||
is not None
|
||||
)
|
||||
is_multistream_safe = lambda: (
|
||||
not current_platform.is_rocm() or routed_input_is_quantized()
|
||||
)
|
||||
self._shared_experts = SharedExperts(
|
||||
shared_experts,
|
||||
moe_config=moe_config,
|
||||
enable_dbo=enable_dbo,
|
||||
mk_can_overlap_shared_experts=can_overlap,
|
||||
is_multistream_safe=is_multistream_safe,
|
||||
)
|
||||
|
||||
# Needed for string -> MoERunner layer lookup in custom ops.
|
||||
|
||||
@@ -43,7 +43,6 @@ class SharedExperts(torch.nn.Module):
|
||||
moe_config: FusedMoEConfig,
|
||||
enable_dbo: bool,
|
||||
mk_can_overlap_shared_experts: Callable[[], bool],
|
||||
is_multistream_safe: Callable[[], bool],
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
@@ -58,10 +57,6 @@ class SharedExperts(torch.nn.Module):
|
||||
|
||||
self._mk_can_overlap_shared_experts = mk_can_overlap_shared_experts
|
||||
|
||||
# Might not be safe to run multi-stream mode if routed and shared experts
|
||||
# alias the same inputs
|
||||
self._is_multistream_safe = is_multistream_safe
|
||||
|
||||
# Allow disabling of the separate shared experts stream for
|
||||
# debug purposes.
|
||||
# TODO: Remove this after more extensive testings with TP/DP
|
||||
@@ -112,19 +107,11 @@ class SharedExperts(torch.nn.Module):
|
||||
if self._mk_can_overlap_shared_experts():
|
||||
return SharedExpertsOrder.MK_INTERNAL_OVERLAPPED
|
||||
|
||||
# On ROCm, empirically only DP-only deployments benefit from the overlap.
|
||||
overlap_is_beneficial = not current_platform.is_rocm() or (
|
||||
self._moe_config.moe_parallel_config.dp_size > 1
|
||||
and self._moe_config.moe_parallel_config.tp_size == 1
|
||||
)
|
||||
|
||||
should_run_shared_in_aux_stream = (
|
||||
current_platform.is_cuda_alike()
|
||||
and self._stream is not None
|
||||
and hidden_states.shape[0]
|
||||
<= envs.VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD
|
||||
and overlap_is_beneficial
|
||||
and self._is_multistream_safe()
|
||||
)
|
||||
|
||||
if should_run_shared_in_aux_stream:
|
||||
|
||||
@@ -249,6 +249,45 @@ def use_aiter_triton_gemm(n, m, k, dtype):
|
||||
)
|
||||
|
||||
|
||||
def wvsplitkrc_dispatch(n: int, k: int, m: int, cu_count: int) -> tuple[int, bool]:
|
||||
"""Pick the K-shard split for wvSplitKrc and say whether the shape fits.
|
||||
|
||||
Mirrors wvSplitKrc() in csrc/rocm/skinny_gemms.cu, which is also where the
|
||||
shard cap is explained. Both must pick the same chunkk or the workspace
|
||||
check here bounds the wrong k_rnd.
|
||||
|
||||
Returns:
|
||||
The CHUNKK the kernel will dispatch with, and whether the CU budget and
|
||||
split-K workspace admit the shape at all.
|
||||
"""
|
||||
# Next ^2 of n
|
||||
N_p2 = 1 << (n - 1).bit_length()
|
||||
# How many of 4 waves in a group can work on same 16 Ms at same time?
|
||||
# This reduces the Ms each group works on, i.e. increasing the CUs needed.
|
||||
GrpsShrB = min(N_p2 // 16, 4)
|
||||
# With 64 Ms per CU (each of 4 SIMDs working on a 16x16 tile), and each
|
||||
# working on a 512-shard of K, how many CUs would we need?
|
||||
CuNeeded = ((m + 64 - 1) // 64) * ((k + 512 - 1) // 512) * GrpsShrB
|
||||
|
||||
CHUNKK2_MAX_SHARDS = 11
|
||||
shards_chunkk2 = (k + 256 - 1) // 256 # 256-wide shards
|
||||
chunkk = (
|
||||
2
|
||||
if (
|
||||
N_p2 != 16
|
||||
and CuNeeded * 2 <= cu_count
|
||||
and shards_chunkk2 <= CHUNKK2_MAX_SHARDS
|
||||
)
|
||||
else 1
|
||||
)
|
||||
|
||||
# Deterministic reduction stores one fp32 partial per (M, N, k-shard); all
|
||||
# of them must fit the split-K workspace.
|
||||
k_rnd = (k + 512 // chunkk - 1) // (512 // chunkk)
|
||||
fits = N_p2 * m * k_rnd <= 128 * 1024 * 12 and CuNeeded <= cu_count
|
||||
return chunkk, fits
|
||||
|
||||
|
||||
def rocm_unquantized_gemm_impl(
|
||||
x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
@@ -260,21 +299,7 @@ def rocm_unquantized_gemm_impl(
|
||||
|
||||
cu_count = num_compute_units()
|
||||
|
||||
# Next ^2 of n
|
||||
N_p2 = 1 << (n - 1).bit_length()
|
||||
# With 64 Ms per CU (each of 4 SIMDs working on a 16x16 tile),
|
||||
# and each working on a 512-shard of K, how many CUs would we need?
|
||||
rndup_cus = ((m + 64 - 1) // 64) * ((k + 512 - 1) // 512)
|
||||
# How many of 4 waves in a group can work on same 16 Ms at same time?
|
||||
# This reduces the Ms each group works on, i.e. increasing the number of CUs needed.
|
||||
GrpsShrB = min(N_p2 // 16, 4)
|
||||
# Given the above, how many CUs would we need?
|
||||
CuNeeded = rndup_cus * GrpsShrB
|
||||
# Deterministic reduction stores one float workspace value per K shard.
|
||||
fits_wvsplitkrc = (
|
||||
N_p2 * m * ((k + 512 - 1) // 512)
|
||||
) <= 128 * 1024 * 12 # deterministic
|
||||
fits_wvsplitkrc &= CuNeeded <= cu_count
|
||||
_, fits_wvsplitkrc = wvsplitkrc_dispatch(n, k, m, cu_count)
|
||||
|
||||
skinny_operands_compatible = weight.is_contiguous() and (
|
||||
bias is None or bias.is_contiguous()
|
||||
@@ -362,12 +387,18 @@ direct_register_custom_op(
|
||||
|
||||
@functools.cache
|
||||
def warmup_rocm_skinny_gemm_workspaces(device: torch.device) -> None:
|
||||
"""Eagerly allocate wvSplitKrc's process-lifetime static workspaces.
|
||||
"""Eagerly allocate wvSplitKrc's per-device split-K workspace pool.
|
||||
|
||||
They are otherwise created lazily on the first qualifying GEMM
|
||||
wvSplitKrc partitions one per-device allocation into ``kWvSlots`` slots
|
||||
(csrc/rocm/skinny_gemms.cu) and hands each stream one on first use, so that
|
||||
two streams never share the split-K partials and counters.
|
||||
|
||||
The pool is otherwise created lazily on the first qualifying GEMM
|
||||
(csrc/rocm/skinny_gemms.cu), which can be the first real request — after
|
||||
the KV cache backing buffer exists. If one landed in that segment's
|
||||
rounding tail, it would pin the entire segment at engine shutdown.
|
||||
the KV cache backing buffer exists. If it landed in that segment's rounding
|
||||
tail, it would pin the entire segment at engine shutdown; it could also land
|
||||
inside a cudagraph capture, where it would be taken from the graph's private
|
||||
pool and its zero-fill would become a replayed graph node.
|
||||
"""
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
|
||||
Reference in New Issue
Block a user