From 9163190dda009a310d8c175d63860ac7c671ca90 Mon Sep 17 00:00:00 2001 From: Matt <156021403+mawong-amd@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:14:22 -0700 Subject: [PATCH] [ROCm][Bugfix][Perf] Tune multi-stream shared experts use; wvSplitKrc fixes (#56098) Signed-off-by: Matthew Wong Co-authored-by: Claude Opus 5 (1M context) --- csrc/rocm/skinny_gemms.cu | 246 +++++++++++++++--- .../quantization/test_rocm_skinny_gemms.py | 97 +++++++ vllm/distributed/parallel_state.py | 6 + .../layers/fused_moe/runner/moe_runner.py | 13 - .../layers/fused_moe/runner/shared_experts.py | 13 - vllm/model_executor/layers/utils.py | 69 +++-- 6 files changed, 368 insertions(+), 76 deletions(-) diff --git a/csrc/rocm/skinny_gemms.cu b/csrc/rocm/skinny_gemms.cu index 615cdabed5..79af081455 100644 --- a/csrc/rocm/skinny_gemms.cu +++ b/csrc/rocm/skinny_gemms.cu @@ -8,12 +8,20 @@ #include #include +#include +#include +#include #include "../cuda_compat.h" #include "dispatch_utils.h" #include "quantization/w8a8/fp8/common.cuh" #include "core/batch_invariant.hpp" +// Number of streams the pre-allocated split-K pool covers. ~7.5 MiB per slot. +static constexpr int64_t kWvSlots = 8; +// Slots held back for capturing streams, which cannot allocate one themselves. +static constexpr int64_t kWvCaptureSlots = 8; + // TODO(rasmith): The kernels in this file are susceptible to integer overflow // issues, do not take strides, and are unable to handle PyTorch tensors that // return is_contiguous() as False (the tensors may actually be contiguous @@ -1364,9 +1372,18 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS) }; using big4 = __attribute__((__vector_size__(4 * sizeof(bigType)))) __bf16; - __shared__ scalar_t stg[WvPrGrp * WVLDS / GrpsShrB]; + // The main-loop staging areas are separated from the split-K readback by a + // __syncthreads(), so the readback may reuse all of LDS. + __shared__ union { + struct { + scalar_t stg[WvPrGrp * WVLDS / GrpsShrB]; + scalar_t s[max_lds_len - WvPrGrp * WVLDS / GrpsShrB]; + } mainloop; + scalar_t all[max_lds_len]; + } lds; + auto& stg = lds.mainloop.stg; + auto& s = lds.mainloop.s; unsigned int* myStg = (unsigned int*)(&stg[WVLDS * (threadIdx.y / GrpsShrB)]); - __shared__ scalar_t s[max_lds_len - WvPrGrp * WVLDS / GrpsShrB]; #ifndef WVSPLITKRC_1KPASS constexpr int TUC_ = (THRDS * UNRL * A_CHUNK); @@ -1682,33 +1699,69 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS) if (my_cntr + 1 == k_rnd) { cntr[adr_] = 0; // clear for next round if constexpr (DTRMNSTC) { + constexpr uint32_t NT = N / NTILE / GrpsShrB; + // The readback stages k-shard partials through LDS, which bounds how + // many shards may be in flight at once. + constexpr uint32_t LDS_F4 = + max_lds_len * sizeof(scalar_t) / sizeof(float4); + constexpr uint32_t KSBMAX = LDS_F4 / (THRDS * 4 * NT); + static_assert(KSBMAX >= 1, "LDS cannot stage even one k-shard"); + const uint32_t KSB = (k_rnd < KSBMAX) ? k_rnd : KSBMAX; + + // First batch is peeled; it is the only batch for most shapes. #pragma unroll - for (int ks = 0; ks < k_rnd; ks++) { - for (uint32_t nt = 0; nt < N / NTILE / GrpsShrB; nt++) { + for (uint32_t kb = 0; kb < KSB; kb++) { + for (uint32_t nt = 0; nt < NT; nt++) { int g_nindx = (nt * NTILE + (N / GrpsShrB) * (threadIdx.y % GrpsShrB)) / 4; int g_adr = g_mindx * 4 + 0 + M * g_nindx * 4; __builtin_amdgcn_global_load_lds( - (float4*)(&glbl[g_adr + M * N * ks]), - &(((float4*)s)[(threadIdx.y * THRDS) + ks * THRDS * 4 + - nt * THRDS * 4 * k_rnd]), + (float4*)(&glbl[g_adr + M * N * kb]), + &(((float4*)lds.all)[(threadIdx.y * THRDS) + kb * THRDS * 4 + + nt * THRDS * 4 * KSB]), 16, 0, 0); } } - if (BIAS) - for (uint32_t nt = 0; nt < N / NTILE / GrpsShrB; nt++) { + if (BIAS) // overlaps the DMAs above + for (uint32_t nt = 0; nt < NT; nt++) { for (uint32_t j = 0; j < 4; j++) { int nindx = (j + (threadIdx.x / 16) * 4) + nt * NTILE + (N / GrpsShrB) * (threadIdx.y % GrpsShrB); biases[nt][j] = BIAS[(mindx % Bx) + (nindx % By) * Bx]; } } - asm volatile("s_waitcnt 0"); - for (int ks = 0; ks < k_rnd; ks++) { - for (uint32_t nt = 0; nt < N / NTILE / GrpsShrB; nt++) { - float4 eval = ((float4*)s)[(threadIdx.x + threadIdx.y * THRDS) + - ks * THRDS * 4 + nt * THRDS * 4 * k_rnd]; - vals[nt].f4 += eval; + asm volatile("s_waitcnt vmcnt(0)" ::: "memory"); + for (uint32_t kb = 0; kb < KSB; kb++) { + for (uint32_t nt = 0; nt < NT; nt++) { + vals[nt].f4 += + ((float4*)lds.all)[(threadIdx.x + threadIdx.y * THRDS) + + kb * THRDS * 4 + nt * THRDS * 4 * KSB]; + } + } + + for (uint32_t ks0 = KSB; ks0 < k_rnd; ks0 += KSB) { + const uint32_t ksn = (k_rnd - ks0 < KSB) ? (k_rnd - ks0) : KSB; + // prior reads must retire before the staging is overwritten + asm volatile("s_waitcnt lgkmcnt(0)" ::: "memory"); + for (uint32_t kb = 0; kb < ksn; kb++) { + for (uint32_t nt = 0; nt < NT; nt++) { + int g_nindx = + (nt * NTILE + (N / GrpsShrB) * (threadIdx.y % GrpsShrB)) / 4; + int g_adr = g_mindx * 4 + 0 + M * g_nindx * 4; + __builtin_amdgcn_global_load_lds( + (float4*)(&glbl[g_adr + M * N * (ks0 + kb)]), + &(((float4*)lds.all)[(threadIdx.y * THRDS) + kb * THRDS * 4 + + nt * THRDS * 4 * KSB]), + 16, 0, 0); + } + } + asm volatile("s_waitcnt vmcnt(0)" ::: "memory"); + for (uint32_t kb = 0; kb < ksn; kb++) { + for (uint32_t nt = 0; nt < NT; nt++) { + vals[nt].f4 += + ((float4*)lds.all)[(threadIdx.x + threadIdx.y * THRDS) + + kb * THRDS * 4 + nt * THRDS * 4 * KSB]; + } } } } else { @@ -1771,7 +1824,7 @@ __global__ void wvSplitKrc_(const int actlN, const int K, const int Kap, torch::Tensor wvSplitKrc(const at::Tensor& in_a, const at::Tensor& in_b, const std::optional& in_bias, const int64_t CuCount) { - int _DTRMNSTC = 1; // vllm::vllm_is_batch_invariant(); + static constexpr bool _DTRMNSTC = true; auto M_in = in_b.size(0); auto N_in = in_a.size(0); @@ -1819,26 +1872,157 @@ torch::Tensor wvSplitKrc(const at::Tensor& in_a, const at::Tensor& in_b, if (CuNeeded > CuCount) throw std::runtime_error("Invalid wvSplitKrc size"); - // Can we increase SplitK by shrinking the K-shared to 256? - int chunkk = (CuNeeded * 2 <= CuCount) ? 2 : 1; + // Can we increase SplitK by shrinking the K-shard to 256? That doubles both + // the CUs needed and the k-shards to read back, and the readback is serial in + // the shard count, so it pays only up to a crossover. The crossover is not a + // constant -- it falls as M grows -- so one cap cannot sit on it for every + // shape; this value minimizes total time over the admitted shapes. Retune it + // if the readback cost changes, and note the directions are not symmetric: + // capping too high costs about twice what capping too low does. + // Mirrored by rocm_unquantized_gemm_impl() in + // vllm/model_executor/layers/utils.py; both must pick the same chunkk or the + // host-side fit check bounds the wrong k_rnd. + constexpr int64_t CHUNKK2_MAX_SHARDS = 11; + const int64_t shards_chunkk2 = (K_in + 256 - 1) / 256; // 256 == 512 / 2 + const int chunkk = (N_p2 != 16 && CuNeeded * 2 <= CuCount && + shards_chunkk2 <= CHUNKK2_MAX_SHARDS) + ? 2 + : 1; - static torch::Tensor axl_glbl = - torch::zeros( - 128 * 1024 * (_DTRMNSTC ? 12 : 1), - torch::TensorOptions().dtype(torch::kFloat32).device(in_a.device())) - .detach(); - static torch::Tensor axl_cntr = - torch::zeros( - 128 * 1024 * (_DTRMNSTC ? 12 : 1) / 4, - torch::TensorOptions().dtype(torch::kInt).device(in_a.device())) - .detach(); - auto glbl = axl_glbl.data_ptr(); - auto cntr = axl_cntr.data_ptr(); + // One fp32 partial per (M, N, k-shard) must fit the split-K workspace. + const int64_t k_rnd = (K_in + 512 / chunkk - 1) / (512 / chunkk); + constexpr int64_t glbl_cap = 128 * 1024 * (_DTRMNSTC ? 12 : 1); + const int64_t glbl_needed = + _DTRMNSTC ? (int64_t)N_p2 * M_in * k_rnd : (int64_t)N_p2 * M_in; + TORCH_CHECK(glbl_needed <= glbl_cap, + "wvSplitKrc: split-K workspace too small", " (need ", glbl_needed, + " > ", glbl_cap, " floats) for N=", N_in, " K=", K_in, + " M=", M_in); + + // Split-K workspaces: one pre-partitioned pool per device. + // These must be zeroed before first use, and the kernel re-zeros them after + // each use. + // To avoid unnecessary overhead, we do not allocate them zeroed + // per-invocation. However, a simple static allocation does not work with + // multi-streams, because concurrent streams can alias into the same + // workspace. Solution: pre-allocate kWvSlots = 8 slots per device, plus + // kWvCaptureSlots reserved for capturing streams, and zero the whole pool + // once at creation. On first use, a stream takes one of the slots if any are + // still available. On subsequent uses, it re-uses the slot. If no slots are + // available, fall back to allocating a zeroed one per call, which is safe but + // adds overhead. + // A capturing stream must neither allocate nor enqueue anything -- either + // would be captured into the graph -- so it is only handed a reserved slot. + struct WvSplitKrcPool { + torch::Tensor glbl, cntr; + int64_t glbl_stride = 0, cntr_stride = 0; + int next_slot = 0; + int next_capture_slot = 0; + std::map slot_of; + std::map capture_slot_of; + }; + // mutex not strictly necessary if caller is always single-threaded (Python) + // but this preserves Torch thread_local semantics for the C++ API + static std::mutex wv_pool_mu; + static std::map wv_pools; + + float* glbl; + int* cntr; + // Fallback space + torch::Tensor spill_glbl, spill_cntr; + { + std::lock_guard wv_lk(wv_pool_mu); + const int wv_dev = static_cast(in_a.device().index()); + WvSplitKrcPool& P = wv_pools[wv_dev]; + constexpr int64_t wv_gf = 128 * 1024 * (_DTRMNSTC ? 12 : 1); + constexpr int64_t wv_ci = wv_gf / 4; + constexpr int64_t wv_all_slots = kWvSlots + kWvCaptureSlots; + + // A stream whose capture was already invalidated reports an error; treat + // that as not capturing, since the capture is lost either way. + cudaStreamCaptureStatus capture_status = cudaStreamCaptureStatusNone; + unsigned long long capture_id = 0; + if (cudaStreamGetCaptureInfo(stream, &capture_status, &capture_id) != + cudaSuccess) { + capture_status = cudaStreamCaptureStatusNone; + } + const bool wv_capturing = capture_status != cudaStreamCaptureStatusNone; + + if (!P.glbl.defined()) { + TORCH_CHECK(!wv_capturing, + "wvSplitKrc: the split-K workspace pool cannot be allocated " + "during graph capture. Call " + "warmup_rocm_skinny_gemm_workspaces() " + "(vllm/model_executor/layers/utils.py) eagerly first."); + P.glbl_stride = wv_gf; + P.cntr_stride = wv_ci; + P.glbl = torch::zeros(wv_gf * wv_all_slots, torch::TensorOptions() + .dtype(torch::kFloat32) + .device(in_a.device())) + .detach(); + P.cntr = + torch::zeros( + wv_ci * wv_all_slots, + torch::TensorOptions().dtype(torch::kInt).device(in_a.device())) + .detach(); + // The mutex orders host access to the map, not device execution, so it + // alone would let another stream be handed a slot the fill above is + // still clearing. Synchronizing here, inside the lock, is what makes + // every slot safe to use without any further per-slot zeroing. + const cudaError_t wv_err = cudaStreamSynchronize(stream); + TORCH_CHECK(wv_err == cudaSuccess, + "wvSplitKrc: failed to zero the split-K workspace pool: ", + cudaGetErrorString(wv_err)); + } + + int wv_slot = -1; + if (wv_capturing) { + // One workspace per capturing stream, so graphs captured on the same + // stream share it -- safe because at most one of them replays at a time + // (vLLM captures one graph per padded batch size, and runs one per step). + auto cap_it = P.capture_slot_of.find(stream); + if (cap_it == P.capture_slot_of.end()) { + TORCH_CHECK(P.next_capture_slot < kWvCaptureSlots, + "wvSplitKrc: no free capture workspace slot; more than ", + kWvCaptureSlots, + " distinct streams have captured a graph containing " + "wvSplitKrc. Raise kWvCaptureSlots in " + "csrc/rocm/skinny_gemms.cu."); + cap_it = P.capture_slot_of + .emplace(stream, static_cast(kWvSlots) + + P.next_capture_slot++) + .first; + } + wv_slot = cap_it->second; + } else { + auto slot_it = P.slot_of.find(stream); + if (slot_it != P.slot_of.end()) { + wv_slot = slot_it->second; + } else if (P.next_slot < kWvSlots) { + wv_slot = P.next_slot++; + P.slot_of.emplace(stream, wv_slot); + } + } + + if (wv_slot >= 0) { + glbl = P.glbl.data_ptr() + wv_slot * P.glbl_stride; + cntr = P.cntr.data_ptr() + wv_slot * P.cntr_stride; + } else { + spill_glbl = torch::zeros( + wv_gf, + torch::TensorOptions().dtype(torch::kFloat32).device(in_a.device())); + spill_cntr = torch::zeros( + wv_ci, + torch::TensorOptions().dtype(torch::kInt).device(in_a.device())); + glbl = spill_glbl.data_ptr(); + cntr = spill_cntr.data_ptr(); + } + } #define WVSPLITKrc(_N, _GrpsShrB, _CHUNKK) \ { \ dim3 block(64, 4); \ - if (_DTRMNSTC) \ + if constexpr (_DTRMNSTC) \ wvSplitKrc_ \ <<>>(N_in, K_in, Kap_in, M_in, Bx_in, By_in, \ af4, bf4, biasf4, glbl, cntr, c, \ diff --git a/tests/kernels/quantization/test_rocm_skinny_gemms.py b/tests/kernels/quantization/test_rocm_skinny_gemms.py index f0de9c8345..7a7fe326bf 100644 --- a/tests/kernels/quantization/test_rocm_skinny_gemms.py +++ b/tests/kernels/quantization/test_rocm_skinny_gemms.py @@ -53,6 +53,17 @@ OPTIONS_WVSPLITK_FP8 = [ DTYPES = [torch.bfloat16, torch.float16] +# kWvSlots in csrc/rocm/skinny_gemms.cu; beyond it streams take overflow +# workspaces. +WVSPLITKRC_SLOTS = 8 + +# The CU count is the grid size. Production passes num_compute_units(); a small +# value manufactures better concurrency for these tests. +WVSPLITKRC_TEST_CU = 16 + +# Gate spin, ~125 ms on gfx950 -- must outlast the host enqueue loop it gates. +_GATE_CYCLES = 300_000_000 + # Specific (N, K, M) combinations for targeted testing NKM_FACTORS_LLMM1 = [ # Small, medium, large cases @@ -116,6 +127,15 @@ K_FACTORS_WVSPLITKRC = [2880, 2880 + 8, 3072, 3072 + 8] # M tiles are 64 rows, +16 for a partial tile M_FACTORS_WVSPLITKRC = [128, 128 + 16, 256, 256 + 16, 640, 640 + 16] +# (N, K, M) with more K-shards than the readback can stage in one LDS pass. +# Spans both CHUNKK values and both N-tile counts. +NKM_FACTORS_WVSPLITKRC_LARGE_K = [ + (128, 6144, 128), + (96, 8192, 128), + (128, 12288, 128), + (32, 12288, 128), +] + NKM_FACTORS_WVSPLITK_FP8 = [ # FP8-specific cases with K % 16 == 0 (1, 16, 16), @@ -222,6 +242,83 @@ def test_rocm_wvsplitkrc_kernel(n, k, m, dtype, padded_a, bias_mode, xnorm, seed torch.testing.assert_close(out, ref_out, atol=1e-3, rtol=1e-2) +@pytest.mark.parametrize("n,k,m", NKM_FACTORS_WVSPLITKRC_LARGE_K) +@pytest.mark.parametrize("dtype", DTYPES) +@pytest.mark.parametrize("seed", SEEDS) +@pytest.mark.skipif(not current_platform.is_rocm(), reason="only test for rocm") +@pytest.mark.skipif(not on_gfx950(), reason="only meant for gfx950") +def test_rocm_wvsplitkrc_large_k(n, k, m, dtype, seed): + """K large enough that the split-K readback must stage LDS in batches.""" + torch.manual_seed(seed) + + xavier = math.sqrt(2 / k) + A = (torch.rand(n, k, dtype=dtype, device="cuda") * 2 - 1) * xavier + B = (torch.rand(m, k, dtype=dtype, device="cuda") * 2 - 1) * xavier + + ref_out = torch.nn.functional.linear(A, B, None) + out = ops.wvSplitKrc(A, B, num_compute_units(), None) + + torch.testing.assert_close(out, ref_out, atol=1e-3, rtol=1e-2) + + +@pytest.mark.parametrize("n_streams", [WVSPLITKRC_SLOTS, WVSPLITKRC_SLOTS + 4]) +@pytest.mark.parametrize("dtype", DTYPES) +@pytest.mark.parametrize("seed", SEEDS) +@pytest.mark.skipif(not current_platform.is_rocm(), reason="only test for rocm") +@pytest.mark.skipif(not on_gfx950(), reason="only meant for gfx950") +def test_rocm_wvsplitkrc_multistream(n_streams, dtype, seed): + """Concurrent streams must not share a split-K workspace. + + The workspace is cleared by protocol rather than per invocation, so sharing + one corrupts the reduction and returns wrong numbers rather than failing. + n_streams > WVSPLITKRC_SLOTS additionally exercises the overflow path. + """ + torch.manual_seed(seed) + + n, k, m = 16, 2048, 256 + iters = 4 + xavier = math.sqrt(2 / k) + + B = torch.randn(m, k, dtype=dtype, device="cuda") * xavier + As = [ + torch.randn(n, k, dtype=dtype, device="cuda") * xavier for _ in range(n_streams) + ] + refs = [torch.nn.functional.linear(A, B, None) for A in As] + torch.accelerator.synchronize() + + streams = [torch.Stream() for _ in range(n_streams)] + outs: list[list[torch.Tensor]] = [[] for _ in range(n_streams)] + _release_together(streams) + # Round-robin the enqueues so kernels from different streams are in flight + # at the same time. + for _ in range(iters): + for i, s in enumerate(streams): + with torch.cuda.stream(s): + outs[i].append(ops.wvSplitKrc(As[i], B, WVSPLITKRC_TEST_CU, None)) + for s in streams: + torch.accelerator.current_stream().wait_stream(s) + torch.accelerator.synchronize() + + for i in range(n_streams): + for out in outs[i]: + torch.testing.assert_close(out, refs[i], atol=1e-3, rtol=1e-2) + + +def _release_together(streams): + """Release every stream at once, once everything below is enqueued. + + An event recorded on an idle stream is already complete, so waiting on it + gates nothing -- the spin, not the event, is what holds the streams. + """ + blocker = torch.Stream() + with torch.cuda.stream(blocker): + torch.cuda._sleep(_GATE_CYCLES) + gate = torch.Event() + gate.record() + for s in streams: + s.wait_event(gate) + + @pytest.mark.parametrize("n,k,m", NKM_FACTORS_LLMM1) @pytest.mark.parametrize("dtype", DTYPES) @pytest.mark.parametrize("rows_per_block", [2, 4, 8, 16]) diff --git a/vllm/distributed/parallel_state.py b/vllm/distributed/parallel_state.py index 92deff1183..fd068dff1c 100644 --- a/vllm/distributed/parallel_state.py +++ b/vllm/distributed/parallel_state.py @@ -2357,6 +2357,12 @@ def cleanup_dist_env_and_memory(shutdown_ray: bool = False): from vllm.platforms import current_platform if not current_platform.is_cpu(): + from vllm.triton_utils import HAS_TRITON + + if HAS_TRITON: + from vllm.v1.sample.ops.topk_topp_triton import reset_buffer_cache + + reset_buffer_cache() torch.accelerator.empty_cache() try: torch.accelerator.empty_host_cache() diff --git a/vllm/model_executor/layers/fused_moe/runner/moe_runner.py b/vllm/model_executor/layers/fused_moe/runner/moe_runner.py index daaff3b79b..2868690ebe 100644 --- a/vllm/model_executor/layers/fused_moe/runner/moe_runner.py +++ b/vllm/model_executor/layers/fused_moe/runner/moe_runner.py @@ -283,24 +283,11 @@ class MoERunner(MoERunnerInterface): self._shared_experts: SharedExperts | None = None if shared_experts is not None: can_overlap = lambda: self._quant_method.mk_can_overlap_shared_experts - # When unquantized, shared expert inputs alias the hidden states, - # which can lead to race condition in multi-stream mode. Quantized - # routed experts copy the input into a fresh buffer first, breaking - # the alias, so overlap is safe. Only observed on ROCm. - routed_input_is_quantized = lambda: ( - self.routed_experts.quant_method.moe_quant_config is not None - and self.routed_experts.quant_method.moe_quant_config.quant_dtype - is not None - ) - is_multistream_safe = lambda: ( - not current_platform.is_rocm() or routed_input_is_quantized() - ) self._shared_experts = SharedExperts( shared_experts, moe_config=moe_config, enable_dbo=enable_dbo, mk_can_overlap_shared_experts=can_overlap, - is_multistream_safe=is_multistream_safe, ) # Needed for string -> MoERunner layer lookup in custom ops. diff --git a/vllm/model_executor/layers/fused_moe/runner/shared_experts.py b/vllm/model_executor/layers/fused_moe/runner/shared_experts.py index 5b9587c853..4687cc5469 100644 --- a/vllm/model_executor/layers/fused_moe/runner/shared_experts.py +++ b/vllm/model_executor/layers/fused_moe/runner/shared_experts.py @@ -43,7 +43,6 @@ class SharedExperts(torch.nn.Module): moe_config: FusedMoEConfig, enable_dbo: bool, mk_can_overlap_shared_experts: Callable[[], bool], - is_multistream_safe: Callable[[], bool], ): super().__init__() @@ -58,10 +57,6 @@ class SharedExperts(torch.nn.Module): self._mk_can_overlap_shared_experts = mk_can_overlap_shared_experts - # Might not be safe to run multi-stream mode if routed and shared experts - # alias the same inputs - self._is_multistream_safe = is_multistream_safe - # Allow disabling of the separate shared experts stream for # debug purposes. # TODO: Remove this after more extensive testings with TP/DP @@ -112,19 +107,11 @@ class SharedExperts(torch.nn.Module): if self._mk_can_overlap_shared_experts(): return SharedExpertsOrder.MK_INTERNAL_OVERLAPPED - # On ROCm, empirically only DP-only deployments benefit from the overlap. - overlap_is_beneficial = not current_platform.is_rocm() or ( - self._moe_config.moe_parallel_config.dp_size > 1 - and self._moe_config.moe_parallel_config.tp_size == 1 - ) - should_run_shared_in_aux_stream = ( current_platform.is_cuda_alike() and self._stream is not None and hidden_states.shape[0] <= envs.VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD - and overlap_is_beneficial - and self._is_multistream_safe() ) if should_run_shared_in_aux_stream: diff --git a/vllm/model_executor/layers/utils.py b/vllm/model_executor/layers/utils.py index 3e19c2b389..f483a619b2 100644 --- a/vllm/model_executor/layers/utils.py +++ b/vllm/model_executor/layers/utils.py @@ -249,6 +249,45 @@ def use_aiter_triton_gemm(n, m, k, dtype): ) +def wvsplitkrc_dispatch(n: int, k: int, m: int, cu_count: int) -> tuple[int, bool]: + """Pick the K-shard split for wvSplitKrc and say whether the shape fits. + + Mirrors wvSplitKrc() in csrc/rocm/skinny_gemms.cu, which is also where the + shard cap is explained. Both must pick the same chunkk or the workspace + check here bounds the wrong k_rnd. + + Returns: + The CHUNKK the kernel will dispatch with, and whether the CU budget and + split-K workspace admit the shape at all. + """ + # Next ^2 of n + N_p2 = 1 << (n - 1).bit_length() + # How many of 4 waves in a group can work on same 16 Ms at same time? + # This reduces the Ms each group works on, i.e. increasing the CUs needed. + GrpsShrB = min(N_p2 // 16, 4) + # With 64 Ms per CU (each of 4 SIMDs working on a 16x16 tile), and each + # working on a 512-shard of K, how many CUs would we need? + CuNeeded = ((m + 64 - 1) // 64) * ((k + 512 - 1) // 512) * GrpsShrB + + CHUNKK2_MAX_SHARDS = 11 + shards_chunkk2 = (k + 256 - 1) // 256 # 256-wide shards + chunkk = ( + 2 + if ( + N_p2 != 16 + and CuNeeded * 2 <= cu_count + and shards_chunkk2 <= CHUNKK2_MAX_SHARDS + ) + else 1 + ) + + # Deterministic reduction stores one fp32 partial per (M, N, k-shard); all + # of them must fit the split-K workspace. + k_rnd = (k + 512 // chunkk - 1) // (512 // chunkk) + fits = N_p2 * m * k_rnd <= 128 * 1024 * 12 and CuNeeded <= cu_count + return chunkk, fits + + def rocm_unquantized_gemm_impl( x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None = None ) -> torch.Tensor: @@ -260,21 +299,7 @@ def rocm_unquantized_gemm_impl( cu_count = num_compute_units() - # Next ^2 of n - N_p2 = 1 << (n - 1).bit_length() - # With 64 Ms per CU (each of 4 SIMDs working on a 16x16 tile), - # and each working on a 512-shard of K, how many CUs would we need? - rndup_cus = ((m + 64 - 1) // 64) * ((k + 512 - 1) // 512) - # How many of 4 waves in a group can work on same 16 Ms at same time? - # This reduces the Ms each group works on, i.e. increasing the number of CUs needed. - GrpsShrB = min(N_p2 // 16, 4) - # Given the above, how many CUs would we need? - CuNeeded = rndup_cus * GrpsShrB - # Deterministic reduction stores one float workspace value per K shard. - fits_wvsplitkrc = ( - N_p2 * m * ((k + 512 - 1) // 512) - ) <= 128 * 1024 * 12 # deterministic - fits_wvsplitkrc &= CuNeeded <= cu_count + _, fits_wvsplitkrc = wvsplitkrc_dispatch(n, k, m, cu_count) skinny_operands_compatible = weight.is_contiguous() and ( bias is None or bias.is_contiguous() @@ -362,12 +387,18 @@ direct_register_custom_op( @functools.cache def warmup_rocm_skinny_gemm_workspaces(device: torch.device) -> None: - """Eagerly allocate wvSplitKrc's process-lifetime static workspaces. + """Eagerly allocate wvSplitKrc's per-device split-K workspace pool. - They are otherwise created lazily on the first qualifying GEMM + wvSplitKrc partitions one per-device allocation into ``kWvSlots`` slots + (csrc/rocm/skinny_gemms.cu) and hands each stream one on first use, so that + two streams never share the split-K partials and counters. + + The pool is otherwise created lazily on the first qualifying GEMM (csrc/rocm/skinny_gemms.cu), which can be the first real request — after - the KV cache backing buffer exists. If one landed in that segment's - rounding tail, it would pin the entire segment at engine shutdown. + the KV cache backing buffer exists. If it landed in that segment's rounding + tail, it would pin the entire segment at engine shutdown; it could also land + inside a cudagraph capture, where it would be taken from the graph's private + pool and its zero-fill would become a replayed graph node. """ from vllm.platforms.rocm import on_gfx950