[ROCm][Bugfix][Perf] Tune multi-stream shared experts use; wvSplitKrc fixes (#56098)

Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Matt
2026-09-10 15:14:22 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent ae71862c51
commit 9163190dda
6 changed files with 368 additions and 76 deletions
+215 -31
View File
@@ -8,12 +8,20 @@
#include <stdexcept>
#include <algorithm>
#include <map>
#include <mutex>
#include <utility>
#include "../cuda_compat.h"
#include "dispatch_utils.h"
#include "quantization/w8a8/fp8/common.cuh"
#include "core/batch_invariant.hpp"
// Number of streams the pre-allocated split-K pool covers. ~7.5 MiB per slot.
static constexpr int64_t kWvSlots = 8;
// Slots held back for capturing streams, which cannot allocate one themselves.
static constexpr int64_t kWvCaptureSlots = 8;
// TODO(rasmith): The kernels in this file are susceptible to integer overflow
// issues, do not take strides, and are unable to handle PyTorch tensors that
// return is_contiguous() as False (the tensors may actually be contiguous
@@ -1364,9 +1372,18 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
};
using big4 = __attribute__((__vector_size__(4 * sizeof(bigType)))) __bf16;
__shared__ scalar_t stg[WvPrGrp * WVLDS / GrpsShrB];
// The main-loop staging areas are separated from the split-K readback by a
// __syncthreads(), so the readback may reuse all of LDS.
__shared__ union {
struct {
scalar_t stg[WvPrGrp * WVLDS / GrpsShrB];
scalar_t s[max_lds_len - WvPrGrp * WVLDS / GrpsShrB];
} mainloop;
scalar_t all[max_lds_len];
} lds;
auto& stg = lds.mainloop.stg;
auto& s = lds.mainloop.s;
unsigned int* myStg = (unsigned int*)(&stg[WVLDS * (threadIdx.y / GrpsShrB)]);
__shared__ scalar_t s[max_lds_len - WvPrGrp * WVLDS / GrpsShrB];
#ifndef WVSPLITKRC_1KPASS
constexpr int TUC_ = (THRDS * UNRL * A_CHUNK);
@@ -1682,33 +1699,69 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
if (my_cntr + 1 == k_rnd) {
cntr[adr_] = 0; // clear for next round
if constexpr (DTRMNSTC) {
constexpr uint32_t NT = N / NTILE / GrpsShrB;
// The readback stages k-shard partials through LDS, which bounds how
// many shards may be in flight at once.
constexpr uint32_t LDS_F4 =
max_lds_len * sizeof(scalar_t) / sizeof(float4);
constexpr uint32_t KSBMAX = LDS_F4 / (THRDS * 4 * NT);
static_assert(KSBMAX >= 1, "LDS cannot stage even one k-shard");
const uint32_t KSB = (k_rnd < KSBMAX) ? k_rnd : KSBMAX;
// First batch is peeled; it is the only batch for most shapes.
#pragma unroll
for (int ks = 0; ks < k_rnd; ks++) {
for (uint32_t nt = 0; nt < N / NTILE / GrpsShrB; nt++) {
for (uint32_t kb = 0; kb < KSB; kb++) {
for (uint32_t nt = 0; nt < NT; nt++) {
int g_nindx =
(nt * NTILE + (N / GrpsShrB) * (threadIdx.y % GrpsShrB)) / 4;
int g_adr = g_mindx * 4 + 0 + M * g_nindx * 4;
__builtin_amdgcn_global_load_lds(
(float4*)(&glbl[g_adr + M * N * ks]),
&(((float4*)s)[(threadIdx.y * THRDS) + ks * THRDS * 4 +
nt * THRDS * 4 * k_rnd]),
(float4*)(&glbl[g_adr + M * N * kb]),
&(((float4*)lds.all)[(threadIdx.y * THRDS) + kb * THRDS * 4 +
nt * THRDS * 4 * KSB]),
16, 0, 0);
}
}
if (BIAS)
for (uint32_t nt = 0; nt < N / NTILE / GrpsShrB; nt++) {
if (BIAS) // overlaps the DMAs above
for (uint32_t nt = 0; nt < NT; nt++) {
for (uint32_t j = 0; j < 4; j++) {
int nindx = (j + (threadIdx.x / 16) * 4) + nt * NTILE +
(N / GrpsShrB) * (threadIdx.y % GrpsShrB);
biases[nt][j] = BIAS[(mindx % Bx) + (nindx % By) * Bx];
}
}
asm volatile("s_waitcnt 0");
for (int ks = 0; ks < k_rnd; ks++) {
for (uint32_t nt = 0; nt < N / NTILE / GrpsShrB; nt++) {
float4 eval = ((float4*)s)[(threadIdx.x + threadIdx.y * THRDS) +
ks * THRDS * 4 + nt * THRDS * 4 * k_rnd];
vals[nt].f4 += eval;
asm volatile("s_waitcnt vmcnt(0)" ::: "memory");
for (uint32_t kb = 0; kb < KSB; kb++) {
for (uint32_t nt = 0; nt < NT; nt++) {
vals[nt].f4 +=
((float4*)lds.all)[(threadIdx.x + threadIdx.y * THRDS) +
kb * THRDS * 4 + nt * THRDS * 4 * KSB];
}
}
for (uint32_t ks0 = KSB; ks0 < k_rnd; ks0 += KSB) {
const uint32_t ksn = (k_rnd - ks0 < KSB) ? (k_rnd - ks0) : KSB;
// prior reads must retire before the staging is overwritten
asm volatile("s_waitcnt lgkmcnt(0)" ::: "memory");
for (uint32_t kb = 0; kb < ksn; kb++) {
for (uint32_t nt = 0; nt < NT; nt++) {
int g_nindx =
(nt * NTILE + (N / GrpsShrB) * (threadIdx.y % GrpsShrB)) / 4;
int g_adr = g_mindx * 4 + 0 + M * g_nindx * 4;
__builtin_amdgcn_global_load_lds(
(float4*)(&glbl[g_adr + M * N * (ks0 + kb)]),
&(((float4*)lds.all)[(threadIdx.y * THRDS) + kb * THRDS * 4 +
nt * THRDS * 4 * KSB]),
16, 0, 0);
}
}
asm volatile("s_waitcnt vmcnt(0)" ::: "memory");
for (uint32_t kb = 0; kb < ksn; kb++) {
for (uint32_t nt = 0; nt < NT; nt++) {
vals[nt].f4 +=
((float4*)lds.all)[(threadIdx.x + threadIdx.y * THRDS) +
kb * THRDS * 4 + nt * THRDS * 4 * KSB];
}
}
}
} else {
@@ -1771,7 +1824,7 @@ __global__ void wvSplitKrc_(const int actlN, const int K, const int Kap,
torch::Tensor wvSplitKrc(const at::Tensor& in_a, const at::Tensor& in_b,
const std::optional<at::Tensor>& in_bias,
const int64_t CuCount) {
int _DTRMNSTC = 1; // vllm::vllm_is_batch_invariant();
static constexpr bool _DTRMNSTC = true;
auto M_in = in_b.size(0);
auto N_in = in_a.size(0);
@@ -1819,26 +1872,157 @@ torch::Tensor wvSplitKrc(const at::Tensor& in_a, const at::Tensor& in_b,
if (CuNeeded > CuCount) throw std::runtime_error("Invalid wvSplitKrc size");
// Can we increase SplitK by shrinking the K-shared to 256?
int chunkk = (CuNeeded * 2 <= CuCount) ? 2 : 1;
// Can we increase SplitK by shrinking the K-shard to 256? That doubles both
// the CUs needed and the k-shards to read back, and the readback is serial in
// the shard count, so it pays only up to a crossover. The crossover is not a
// constant -- it falls as M grows -- so one cap cannot sit on it for every
// shape; this value minimizes total time over the admitted shapes. Retune it
// if the readback cost changes, and note the directions are not symmetric:
// capping too high costs about twice what capping too low does.
// Mirrored by rocm_unquantized_gemm_impl() in
// vllm/model_executor/layers/utils.py; both must pick the same chunkk or the
// host-side fit check bounds the wrong k_rnd.
constexpr int64_t CHUNKK2_MAX_SHARDS = 11;
const int64_t shards_chunkk2 = (K_in + 256 - 1) / 256; // 256 == 512 / 2
const int chunkk = (N_p2 != 16 && CuNeeded * 2 <= CuCount &&
shards_chunkk2 <= CHUNKK2_MAX_SHARDS)
? 2
: 1;
static torch::Tensor axl_glbl =
torch::zeros(
128 * 1024 * (_DTRMNSTC ? 12 : 1),
torch::TensorOptions().dtype(torch::kFloat32).device(in_a.device()))
.detach();
static torch::Tensor axl_cntr =
torch::zeros(
128 * 1024 * (_DTRMNSTC ? 12 : 1) / 4,
torch::TensorOptions().dtype(torch::kInt).device(in_a.device()))
.detach();
auto glbl = axl_glbl.data_ptr<float>();
auto cntr = axl_cntr.data_ptr<int>();
// One fp32 partial per (M, N, k-shard) must fit the split-K workspace.
const int64_t k_rnd = (K_in + 512 / chunkk - 1) / (512 / chunkk);
constexpr int64_t glbl_cap = 128 * 1024 * (_DTRMNSTC ? 12 : 1);
const int64_t glbl_needed =
_DTRMNSTC ? (int64_t)N_p2 * M_in * k_rnd : (int64_t)N_p2 * M_in;
TORCH_CHECK(glbl_needed <= glbl_cap,
"wvSplitKrc: split-K workspace too small", " (need ", glbl_needed,
" > ", glbl_cap, " floats) for N=", N_in, " K=", K_in,
" M=", M_in);
// Split-K workspaces: one pre-partitioned pool per device.
// These must be zeroed before first use, and the kernel re-zeros them after
// each use.
// To avoid unnecessary overhead, we do not allocate them zeroed
// per-invocation. However, a simple static allocation does not work with
// multi-streams, because concurrent streams can alias into the same
// workspace. Solution: pre-allocate kWvSlots = 8 slots per device, plus
// kWvCaptureSlots reserved for capturing streams, and zero the whole pool
// once at creation. On first use, a stream takes one of the slots if any are
// still available. On subsequent uses, it re-uses the slot. If no slots are
// available, fall back to allocating a zeroed one per call, which is safe but
// adds overhead.
// A capturing stream must neither allocate nor enqueue anything -- either
// would be captured into the graph -- so it is only handed a reserved slot.
struct WvSplitKrcPool {
torch::Tensor glbl, cntr;
int64_t glbl_stride = 0, cntr_stride = 0;
int next_slot = 0;
int next_capture_slot = 0;
std::map<cudaStream_t, int> slot_of;
std::map<cudaStream_t, int> capture_slot_of;
};
// mutex not strictly necessary if caller is always single-threaded (Python)
// but this preserves Torch thread_local semantics for the C++ API
static std::mutex wv_pool_mu;
static std::map<int, WvSplitKrcPool> wv_pools;
float* glbl;
int* cntr;
// Fallback space
torch::Tensor spill_glbl, spill_cntr;
{
std::lock_guard<std::mutex> wv_lk(wv_pool_mu);
const int wv_dev = static_cast<int>(in_a.device().index());
WvSplitKrcPool& P = wv_pools[wv_dev];
constexpr int64_t wv_gf = 128 * 1024 * (_DTRMNSTC ? 12 : 1);
constexpr int64_t wv_ci = wv_gf / 4;
constexpr int64_t wv_all_slots = kWvSlots + kWvCaptureSlots;
// A stream whose capture was already invalidated reports an error; treat
// that as not capturing, since the capture is lost either way.
cudaStreamCaptureStatus capture_status = cudaStreamCaptureStatusNone;
unsigned long long capture_id = 0;
if (cudaStreamGetCaptureInfo(stream, &capture_status, &capture_id) !=
cudaSuccess) {
capture_status = cudaStreamCaptureStatusNone;
}
const bool wv_capturing = capture_status != cudaStreamCaptureStatusNone;
if (!P.glbl.defined()) {
TORCH_CHECK(!wv_capturing,
"wvSplitKrc: the split-K workspace pool cannot be allocated "
"during graph capture. Call "
"warmup_rocm_skinny_gemm_workspaces() "
"(vllm/model_executor/layers/utils.py) eagerly first.");
P.glbl_stride = wv_gf;
P.cntr_stride = wv_ci;
P.glbl = torch::zeros(wv_gf * wv_all_slots, torch::TensorOptions()
.dtype(torch::kFloat32)
.device(in_a.device()))
.detach();
P.cntr =
torch::zeros(
wv_ci * wv_all_slots,
torch::TensorOptions().dtype(torch::kInt).device(in_a.device()))
.detach();
// The mutex orders host access to the map, not device execution, so it
// alone would let another stream be handed a slot the fill above is
// still clearing. Synchronizing here, inside the lock, is what makes
// every slot safe to use without any further per-slot zeroing.
const cudaError_t wv_err = cudaStreamSynchronize(stream);
TORCH_CHECK(wv_err == cudaSuccess,
"wvSplitKrc: failed to zero the split-K workspace pool: ",
cudaGetErrorString(wv_err));
}
int wv_slot = -1;
if (wv_capturing) {
// One workspace per capturing stream, so graphs captured on the same
// stream share it -- safe because at most one of them replays at a time
// (vLLM captures one graph per padded batch size, and runs one per step).
auto cap_it = P.capture_slot_of.find(stream);
if (cap_it == P.capture_slot_of.end()) {
TORCH_CHECK(P.next_capture_slot < kWvCaptureSlots,
"wvSplitKrc: no free capture workspace slot; more than ",
kWvCaptureSlots,
" distinct streams have captured a graph containing "
"wvSplitKrc. Raise kWvCaptureSlots in "
"csrc/rocm/skinny_gemms.cu.");
cap_it = P.capture_slot_of
.emplace(stream, static_cast<int>(kWvSlots) +
P.next_capture_slot++)
.first;
}
wv_slot = cap_it->second;
} else {
auto slot_it = P.slot_of.find(stream);
if (slot_it != P.slot_of.end()) {
wv_slot = slot_it->second;
} else if (P.next_slot < kWvSlots) {
wv_slot = P.next_slot++;
P.slot_of.emplace(stream, wv_slot);
}
}
if (wv_slot >= 0) {
glbl = P.glbl.data_ptr<float>() + wv_slot * P.glbl_stride;
cntr = P.cntr.data_ptr<int>() + wv_slot * P.cntr_stride;
} else {
spill_glbl = torch::zeros(
wv_gf,
torch::TensorOptions().dtype(torch::kFloat32).device(in_a.device()));
spill_cntr = torch::zeros(
wv_ci,
torch::TensorOptions().dtype(torch::kInt).device(in_a.device()));
glbl = spill_glbl.data_ptr<float>();
cntr = spill_cntr.data_ptr<int>();
}
}
#define WVSPLITKrc(_N, _GrpsShrB, _CHUNKK) \
{ \
dim3 block(64, 4); \
if (_DTRMNSTC) \
if constexpr (_DTRMNSTC) \
wvSplitKrc_<fptype, 64, 16, 4, 8, 1, _N, _GrpsShrB, _CHUNKK, 1> \
<<<grid, block, 0, stream>>>(N_in, K_in, Kap_in, M_in, Bx_in, By_in, \
af4, bf4, biasf4, glbl, cntr, c, \
@@ -53,6 +53,17 @@ OPTIONS_WVSPLITK_FP8 = [
DTYPES = [torch.bfloat16, torch.float16]
# kWvSlots in csrc/rocm/skinny_gemms.cu; beyond it streams take overflow
# workspaces.
WVSPLITKRC_SLOTS = 8
# The CU count is the grid size. Production passes num_compute_units(); a small
# value manufactures better concurrency for these tests.
WVSPLITKRC_TEST_CU = 16
# Gate spin, ~125 ms on gfx950 -- must outlast the host enqueue loop it gates.
_GATE_CYCLES = 300_000_000
# Specific (N, K, M) combinations for targeted testing
NKM_FACTORS_LLMM1 = [
# Small, medium, large cases
@@ -116,6 +127,15 @@ K_FACTORS_WVSPLITKRC = [2880, 2880 + 8, 3072, 3072 + 8]
# M tiles are 64 rows, +16 for a partial tile
M_FACTORS_WVSPLITKRC = [128, 128 + 16, 256, 256 + 16, 640, 640 + 16]
# (N, K, M) with more K-shards than the readback can stage in one LDS pass.
# Spans both CHUNKK values and both N-tile counts.
NKM_FACTORS_WVSPLITKRC_LARGE_K = [
(128, 6144, 128),
(96, 8192, 128),
(128, 12288, 128),
(32, 12288, 128),
]
NKM_FACTORS_WVSPLITK_FP8 = [
# FP8-specific cases with K % 16 == 0
(1, 16, 16),
@@ -222,6 +242,83 @@ def test_rocm_wvsplitkrc_kernel(n, k, m, dtype, padded_a, bias_mode, xnorm, seed
torch.testing.assert_close(out, ref_out, atol=1e-3, rtol=1e-2)
@pytest.mark.parametrize("n,k,m", NKM_FACTORS_WVSPLITKRC_LARGE_K)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="only test for rocm")
@pytest.mark.skipif(not on_gfx950(), reason="only meant for gfx950")
def test_rocm_wvsplitkrc_large_k(n, k, m, dtype, seed):
"""K large enough that the split-K readback must stage LDS in batches."""
torch.manual_seed(seed)
xavier = math.sqrt(2 / k)
A = (torch.rand(n, k, dtype=dtype, device="cuda") * 2 - 1) * xavier
B = (torch.rand(m, k, dtype=dtype, device="cuda") * 2 - 1) * xavier
ref_out = torch.nn.functional.linear(A, B, None)
out = ops.wvSplitKrc(A, B, num_compute_units(), None)
torch.testing.assert_close(out, ref_out, atol=1e-3, rtol=1e-2)
@pytest.mark.parametrize("n_streams", [WVSPLITKRC_SLOTS, WVSPLITKRC_SLOTS + 4])
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="only test for rocm")
@pytest.mark.skipif(not on_gfx950(), reason="only meant for gfx950")
def test_rocm_wvsplitkrc_multistream(n_streams, dtype, seed):
"""Concurrent streams must not share a split-K workspace.
The workspace is cleared by protocol rather than per invocation, so sharing
one corrupts the reduction and returns wrong numbers rather than failing.
n_streams > WVSPLITKRC_SLOTS additionally exercises the overflow path.
"""
torch.manual_seed(seed)
n, k, m = 16, 2048, 256
iters = 4
xavier = math.sqrt(2 / k)
B = torch.randn(m, k, dtype=dtype, device="cuda") * xavier
As = [
torch.randn(n, k, dtype=dtype, device="cuda") * xavier for _ in range(n_streams)
]
refs = [torch.nn.functional.linear(A, B, None) for A in As]
torch.accelerator.synchronize()
streams = [torch.Stream() for _ in range(n_streams)]
outs: list[list[torch.Tensor]] = [[] for _ in range(n_streams)]
_release_together(streams)
# Round-robin the enqueues so kernels from different streams are in flight
# at the same time.
for _ in range(iters):
for i, s in enumerate(streams):
with torch.cuda.stream(s):
outs[i].append(ops.wvSplitKrc(As[i], B, WVSPLITKRC_TEST_CU, None))
for s in streams:
torch.accelerator.current_stream().wait_stream(s)
torch.accelerator.synchronize()
for i in range(n_streams):
for out in outs[i]:
torch.testing.assert_close(out, refs[i], atol=1e-3, rtol=1e-2)
def _release_together(streams):
"""Release every stream at once, once everything below is enqueued.
An event recorded on an idle stream is already complete, so waiting on it
gates nothing -- the spin, not the event, is what holds the streams.
"""
blocker = torch.Stream()
with torch.cuda.stream(blocker):
torch.cuda._sleep(_GATE_CYCLES)
gate = torch.Event()
gate.record()
for s in streams:
s.wait_event(gate)
@pytest.mark.parametrize("n,k,m", NKM_FACTORS_LLMM1)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("rows_per_block", [2, 4, 8, 16])
+6
View File
@@ -2357,6 +2357,12 @@ def cleanup_dist_env_and_memory(shutdown_ray: bool = False):
from vllm.platforms import current_platform
if not current_platform.is_cpu():
from vllm.triton_utils import HAS_TRITON
if HAS_TRITON:
from vllm.v1.sample.ops.topk_topp_triton import reset_buffer_cache
reset_buffer_cache()
torch.accelerator.empty_cache()
try:
torch.accelerator.empty_host_cache()
@@ -283,24 +283,11 @@ class MoERunner(MoERunnerInterface):
self._shared_experts: SharedExperts | None = None
if shared_experts is not None:
can_overlap = lambda: self._quant_method.mk_can_overlap_shared_experts
# When unquantized, shared expert inputs alias the hidden states,
# which can lead to race condition in multi-stream mode. Quantized
# routed experts copy the input into a fresh buffer first, breaking
# the alias, so overlap is safe. Only observed on ROCm.
routed_input_is_quantized = lambda: (
self.routed_experts.quant_method.moe_quant_config is not None
and self.routed_experts.quant_method.moe_quant_config.quant_dtype
is not None
)
is_multistream_safe = lambda: (
not current_platform.is_rocm() or routed_input_is_quantized()
)
self._shared_experts = SharedExperts(
shared_experts,
moe_config=moe_config,
enable_dbo=enable_dbo,
mk_can_overlap_shared_experts=can_overlap,
is_multistream_safe=is_multistream_safe,
)
# Needed for string -> MoERunner layer lookup in custom ops.
@@ -43,7 +43,6 @@ class SharedExperts(torch.nn.Module):
moe_config: FusedMoEConfig,
enable_dbo: bool,
mk_can_overlap_shared_experts: Callable[[], bool],
is_multistream_safe: Callable[[], bool],
):
super().__init__()
@@ -58,10 +57,6 @@ class SharedExperts(torch.nn.Module):
self._mk_can_overlap_shared_experts = mk_can_overlap_shared_experts
# Might not be safe to run multi-stream mode if routed and shared experts
# alias the same inputs
self._is_multistream_safe = is_multistream_safe
# Allow disabling of the separate shared experts stream for
# debug purposes.
# TODO: Remove this after more extensive testings with TP/DP
@@ -112,19 +107,11 @@ class SharedExperts(torch.nn.Module):
if self._mk_can_overlap_shared_experts():
return SharedExpertsOrder.MK_INTERNAL_OVERLAPPED
# On ROCm, empirically only DP-only deployments benefit from the overlap.
overlap_is_beneficial = not current_platform.is_rocm() or (
self._moe_config.moe_parallel_config.dp_size > 1
and self._moe_config.moe_parallel_config.tp_size == 1
)
should_run_shared_in_aux_stream = (
current_platform.is_cuda_alike()
and self._stream is not None
and hidden_states.shape[0]
<= envs.VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD
and overlap_is_beneficial
and self._is_multistream_safe()
)
if should_run_shared_in_aux_stream:
+50 -19
View File
@@ -249,6 +249,45 @@ def use_aiter_triton_gemm(n, m, k, dtype):
)
def wvsplitkrc_dispatch(n: int, k: int, m: int, cu_count: int) -> tuple[int, bool]:
"""Pick the K-shard split for wvSplitKrc and say whether the shape fits.
Mirrors wvSplitKrc() in csrc/rocm/skinny_gemms.cu, which is also where the
shard cap is explained. Both must pick the same chunkk or the workspace
check here bounds the wrong k_rnd.
Returns:
The CHUNKK the kernel will dispatch with, and whether the CU budget and
split-K workspace admit the shape at all.
"""
# Next ^2 of n
N_p2 = 1 << (n - 1).bit_length()
# How many of 4 waves in a group can work on same 16 Ms at same time?
# This reduces the Ms each group works on, i.e. increasing the CUs needed.
GrpsShrB = min(N_p2 // 16, 4)
# With 64 Ms per CU (each of 4 SIMDs working on a 16x16 tile), and each
# working on a 512-shard of K, how many CUs would we need?
CuNeeded = ((m + 64 - 1) // 64) * ((k + 512 - 1) // 512) * GrpsShrB
CHUNKK2_MAX_SHARDS = 11
shards_chunkk2 = (k + 256 - 1) // 256 # 256-wide shards
chunkk = (
2
if (
N_p2 != 16
and CuNeeded * 2 <= cu_count
and shards_chunkk2 <= CHUNKK2_MAX_SHARDS
)
else 1
)
# Deterministic reduction stores one fp32 partial per (M, N, k-shard); all
# of them must fit the split-K workspace.
k_rnd = (k + 512 // chunkk - 1) // (512 // chunkk)
fits = N_p2 * m * k_rnd <= 128 * 1024 * 12 and CuNeeded <= cu_count
return chunkk, fits
def rocm_unquantized_gemm_impl(
x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None = None
) -> torch.Tensor:
@@ -260,21 +299,7 @@ def rocm_unquantized_gemm_impl(
cu_count = num_compute_units()
# Next ^2 of n
N_p2 = 1 << (n - 1).bit_length()
# With 64 Ms per CU (each of 4 SIMDs working on a 16x16 tile),
# and each working on a 512-shard of K, how many CUs would we need?
rndup_cus = ((m + 64 - 1) // 64) * ((k + 512 - 1) // 512)
# How many of 4 waves in a group can work on same 16 Ms at same time?
# This reduces the Ms each group works on, i.e. increasing the number of CUs needed.
GrpsShrB = min(N_p2 // 16, 4)
# Given the above, how many CUs would we need?
CuNeeded = rndup_cus * GrpsShrB
# Deterministic reduction stores one float workspace value per K shard.
fits_wvsplitkrc = (
N_p2 * m * ((k + 512 - 1) // 512)
) <= 128 * 1024 * 12 # deterministic
fits_wvsplitkrc &= CuNeeded <= cu_count
_, fits_wvsplitkrc = wvsplitkrc_dispatch(n, k, m, cu_count)
skinny_operands_compatible = weight.is_contiguous() and (
bias is None or bias.is_contiguous()
@@ -362,12 +387,18 @@ direct_register_custom_op(
@functools.cache
def warmup_rocm_skinny_gemm_workspaces(device: torch.device) -> None:
"""Eagerly allocate wvSplitKrc's process-lifetime static workspaces.
"""Eagerly allocate wvSplitKrc's per-device split-K workspace pool.
They are otherwise created lazily on the first qualifying GEMM
wvSplitKrc partitions one per-device allocation into ``kWvSlots`` slots
(csrc/rocm/skinny_gemms.cu) and hands each stream one on first use, so that
two streams never share the split-K partials and counters.
The pool is otherwise created lazily on the first qualifying GEMM
(csrc/rocm/skinny_gemms.cu), which can be the first real request — after
the KV cache backing buffer exists. If one landed in that segment's
rounding tail, it would pin the entire segment at engine shutdown.
the KV cache backing buffer exists. If it landed in that segment's rounding
tail, it would pin the entire segment at engine shutdown; it could also land
inside a cudagraph capture, where it would be taken from the graph's private
pool and its zero-fill would become a replayed graph node.
"""
from vllm.platforms.rocm import on_gfx950