[ROCm][Bugfix][Perf] Tune multi-stream shared experts use; wvSplitKrc fixes (#56098)
Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
+215
-31
@@ -8,12 +8,20 @@
|
||||
|
||||
#include <stdexcept>
|
||||
#include <algorithm>
|
||||
#include <map>
|
||||
#include <mutex>
|
||||
#include <utility>
|
||||
|
||||
#include "../cuda_compat.h"
|
||||
#include "dispatch_utils.h"
|
||||
#include "quantization/w8a8/fp8/common.cuh"
|
||||
#include "core/batch_invariant.hpp"
|
||||
|
||||
// Number of streams the pre-allocated split-K pool covers. ~7.5 MiB per slot.
|
||||
static constexpr int64_t kWvSlots = 8;
|
||||
// Slots held back for capturing streams, which cannot allocate one themselves.
|
||||
static constexpr int64_t kWvCaptureSlots = 8;
|
||||
|
||||
// TODO(rasmith): The kernels in this file are susceptible to integer overflow
|
||||
// issues, do not take strides, and are unable to handle PyTorch tensors that
|
||||
// return is_contiguous() as False (the tensors may actually be contiguous
|
||||
@@ -1364,9 +1372,18 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
|
||||
};
|
||||
using big4 = __attribute__((__vector_size__(4 * sizeof(bigType)))) __bf16;
|
||||
|
||||
__shared__ scalar_t stg[WvPrGrp * WVLDS / GrpsShrB];
|
||||
// The main-loop staging areas are separated from the split-K readback by a
|
||||
// __syncthreads(), so the readback may reuse all of LDS.
|
||||
__shared__ union {
|
||||
struct {
|
||||
scalar_t stg[WvPrGrp * WVLDS / GrpsShrB];
|
||||
scalar_t s[max_lds_len - WvPrGrp * WVLDS / GrpsShrB];
|
||||
} mainloop;
|
||||
scalar_t all[max_lds_len];
|
||||
} lds;
|
||||
auto& stg = lds.mainloop.stg;
|
||||
auto& s = lds.mainloop.s;
|
||||
unsigned int* myStg = (unsigned int*)(&stg[WVLDS * (threadIdx.y / GrpsShrB)]);
|
||||
__shared__ scalar_t s[max_lds_len - WvPrGrp * WVLDS / GrpsShrB];
|
||||
|
||||
#ifndef WVSPLITKRC_1KPASS
|
||||
constexpr int TUC_ = (THRDS * UNRL * A_CHUNK);
|
||||
@@ -1682,33 +1699,69 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
|
||||
if (my_cntr + 1 == k_rnd) {
|
||||
cntr[adr_] = 0; // clear for next round
|
||||
if constexpr (DTRMNSTC) {
|
||||
constexpr uint32_t NT = N / NTILE / GrpsShrB;
|
||||
// The readback stages k-shard partials through LDS, which bounds how
|
||||
// many shards may be in flight at once.
|
||||
constexpr uint32_t LDS_F4 =
|
||||
max_lds_len * sizeof(scalar_t) / sizeof(float4);
|
||||
constexpr uint32_t KSBMAX = LDS_F4 / (THRDS * 4 * NT);
|
||||
static_assert(KSBMAX >= 1, "LDS cannot stage even one k-shard");
|
||||
const uint32_t KSB = (k_rnd < KSBMAX) ? k_rnd : KSBMAX;
|
||||
|
||||
// First batch is peeled; it is the only batch for most shapes.
|
||||
#pragma unroll
|
||||
for (int ks = 0; ks < k_rnd; ks++) {
|
||||
for (uint32_t nt = 0; nt < N / NTILE / GrpsShrB; nt++) {
|
||||
for (uint32_t kb = 0; kb < KSB; kb++) {
|
||||
for (uint32_t nt = 0; nt < NT; nt++) {
|
||||
int g_nindx =
|
||||
(nt * NTILE + (N / GrpsShrB) * (threadIdx.y % GrpsShrB)) / 4;
|
||||
int g_adr = g_mindx * 4 + 0 + M * g_nindx * 4;
|
||||
__builtin_amdgcn_global_load_lds(
|
||||
(float4*)(&glbl[g_adr + M * N * ks]),
|
||||
&(((float4*)s)[(threadIdx.y * THRDS) + ks * THRDS * 4 +
|
||||
nt * THRDS * 4 * k_rnd]),
|
||||
(float4*)(&glbl[g_adr + M * N * kb]),
|
||||
&(((float4*)lds.all)[(threadIdx.y * THRDS) + kb * THRDS * 4 +
|
||||
nt * THRDS * 4 * KSB]),
|
||||
16, 0, 0);
|
||||
}
|
||||
}
|
||||
if (BIAS)
|
||||
for (uint32_t nt = 0; nt < N / NTILE / GrpsShrB; nt++) {
|
||||
if (BIAS) // overlaps the DMAs above
|
||||
for (uint32_t nt = 0; nt < NT; nt++) {
|
||||
for (uint32_t j = 0; j < 4; j++) {
|
||||
int nindx = (j + (threadIdx.x / 16) * 4) + nt * NTILE +
|
||||
(N / GrpsShrB) * (threadIdx.y % GrpsShrB);
|
||||
biases[nt][j] = BIAS[(mindx % Bx) + (nindx % By) * Bx];
|
||||
}
|
||||
}
|
||||
asm volatile("s_waitcnt 0");
|
||||
for (int ks = 0; ks < k_rnd; ks++) {
|
||||
for (uint32_t nt = 0; nt < N / NTILE / GrpsShrB; nt++) {
|
||||
float4 eval = ((float4*)s)[(threadIdx.x + threadIdx.y * THRDS) +
|
||||
ks * THRDS * 4 + nt * THRDS * 4 * k_rnd];
|
||||
vals[nt].f4 += eval;
|
||||
asm volatile("s_waitcnt vmcnt(0)" ::: "memory");
|
||||
for (uint32_t kb = 0; kb < KSB; kb++) {
|
||||
for (uint32_t nt = 0; nt < NT; nt++) {
|
||||
vals[nt].f4 +=
|
||||
((float4*)lds.all)[(threadIdx.x + threadIdx.y * THRDS) +
|
||||
kb * THRDS * 4 + nt * THRDS * 4 * KSB];
|
||||
}
|
||||
}
|
||||
|
||||
for (uint32_t ks0 = KSB; ks0 < k_rnd; ks0 += KSB) {
|
||||
const uint32_t ksn = (k_rnd - ks0 < KSB) ? (k_rnd - ks0) : KSB;
|
||||
// prior reads must retire before the staging is overwritten
|
||||
asm volatile("s_waitcnt lgkmcnt(0)" ::: "memory");
|
||||
for (uint32_t kb = 0; kb < ksn; kb++) {
|
||||
for (uint32_t nt = 0; nt < NT; nt++) {
|
||||
int g_nindx =
|
||||
(nt * NTILE + (N / GrpsShrB) * (threadIdx.y % GrpsShrB)) / 4;
|
||||
int g_adr = g_mindx * 4 + 0 + M * g_nindx * 4;
|
||||
__builtin_amdgcn_global_load_lds(
|
||||
(float4*)(&glbl[g_adr + M * N * (ks0 + kb)]),
|
||||
&(((float4*)lds.all)[(threadIdx.y * THRDS) + kb * THRDS * 4 +
|
||||
nt * THRDS * 4 * KSB]),
|
||||
16, 0, 0);
|
||||
}
|
||||
}
|
||||
asm volatile("s_waitcnt vmcnt(0)" ::: "memory");
|
||||
for (uint32_t kb = 0; kb < ksn; kb++) {
|
||||
for (uint32_t nt = 0; nt < NT; nt++) {
|
||||
vals[nt].f4 +=
|
||||
((float4*)lds.all)[(threadIdx.x + threadIdx.y * THRDS) +
|
||||
kb * THRDS * 4 + nt * THRDS * 4 * KSB];
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -1771,7 +1824,7 @@ __global__ void wvSplitKrc_(const int actlN, const int K, const int Kap,
|
||||
torch::Tensor wvSplitKrc(const at::Tensor& in_a, const at::Tensor& in_b,
|
||||
const std::optional<at::Tensor>& in_bias,
|
||||
const int64_t CuCount) {
|
||||
int _DTRMNSTC = 1; // vllm::vllm_is_batch_invariant();
|
||||
static constexpr bool _DTRMNSTC = true;
|
||||
|
||||
auto M_in = in_b.size(0);
|
||||
auto N_in = in_a.size(0);
|
||||
@@ -1819,26 +1872,157 @@ torch::Tensor wvSplitKrc(const at::Tensor& in_a, const at::Tensor& in_b,
|
||||
|
||||
if (CuNeeded > CuCount) throw std::runtime_error("Invalid wvSplitKrc size");
|
||||
|
||||
// Can we increase SplitK by shrinking the K-shared to 256?
|
||||
int chunkk = (CuNeeded * 2 <= CuCount) ? 2 : 1;
|
||||
// Can we increase SplitK by shrinking the K-shard to 256? That doubles both
|
||||
// the CUs needed and the k-shards to read back, and the readback is serial in
|
||||
// the shard count, so it pays only up to a crossover. The crossover is not a
|
||||
// constant -- it falls as M grows -- so one cap cannot sit on it for every
|
||||
// shape; this value minimizes total time over the admitted shapes. Retune it
|
||||
// if the readback cost changes, and note the directions are not symmetric:
|
||||
// capping too high costs about twice what capping too low does.
|
||||
// Mirrored by rocm_unquantized_gemm_impl() in
|
||||
// vllm/model_executor/layers/utils.py; both must pick the same chunkk or the
|
||||
// host-side fit check bounds the wrong k_rnd.
|
||||
constexpr int64_t CHUNKK2_MAX_SHARDS = 11;
|
||||
const int64_t shards_chunkk2 = (K_in + 256 - 1) / 256; // 256 == 512 / 2
|
||||
const int chunkk = (N_p2 != 16 && CuNeeded * 2 <= CuCount &&
|
||||
shards_chunkk2 <= CHUNKK2_MAX_SHARDS)
|
||||
? 2
|
||||
: 1;
|
||||
|
||||
static torch::Tensor axl_glbl =
|
||||
torch::zeros(
|
||||
128 * 1024 * (_DTRMNSTC ? 12 : 1),
|
||||
torch::TensorOptions().dtype(torch::kFloat32).device(in_a.device()))
|
||||
.detach();
|
||||
static torch::Tensor axl_cntr =
|
||||
torch::zeros(
|
||||
128 * 1024 * (_DTRMNSTC ? 12 : 1) / 4,
|
||||
torch::TensorOptions().dtype(torch::kInt).device(in_a.device()))
|
||||
.detach();
|
||||
auto glbl = axl_glbl.data_ptr<float>();
|
||||
auto cntr = axl_cntr.data_ptr<int>();
|
||||
// One fp32 partial per (M, N, k-shard) must fit the split-K workspace.
|
||||
const int64_t k_rnd = (K_in + 512 / chunkk - 1) / (512 / chunkk);
|
||||
constexpr int64_t glbl_cap = 128 * 1024 * (_DTRMNSTC ? 12 : 1);
|
||||
const int64_t glbl_needed =
|
||||
_DTRMNSTC ? (int64_t)N_p2 * M_in * k_rnd : (int64_t)N_p2 * M_in;
|
||||
TORCH_CHECK(glbl_needed <= glbl_cap,
|
||||
"wvSplitKrc: split-K workspace too small", " (need ", glbl_needed,
|
||||
" > ", glbl_cap, " floats) for N=", N_in, " K=", K_in,
|
||||
" M=", M_in);
|
||||
|
||||
// Split-K workspaces: one pre-partitioned pool per device.
|
||||
// These must be zeroed before first use, and the kernel re-zeros them after
|
||||
// each use.
|
||||
// To avoid unnecessary overhead, we do not allocate them zeroed
|
||||
// per-invocation. However, a simple static allocation does not work with
|
||||
// multi-streams, because concurrent streams can alias into the same
|
||||
// workspace. Solution: pre-allocate kWvSlots = 8 slots per device, plus
|
||||
// kWvCaptureSlots reserved for capturing streams, and zero the whole pool
|
||||
// once at creation. On first use, a stream takes one of the slots if any are
|
||||
// still available. On subsequent uses, it re-uses the slot. If no slots are
|
||||
// available, fall back to allocating a zeroed one per call, which is safe but
|
||||
// adds overhead.
|
||||
// A capturing stream must neither allocate nor enqueue anything -- either
|
||||
// would be captured into the graph -- so it is only handed a reserved slot.
|
||||
struct WvSplitKrcPool {
|
||||
torch::Tensor glbl, cntr;
|
||||
int64_t glbl_stride = 0, cntr_stride = 0;
|
||||
int next_slot = 0;
|
||||
int next_capture_slot = 0;
|
||||
std::map<cudaStream_t, int> slot_of;
|
||||
std::map<cudaStream_t, int> capture_slot_of;
|
||||
};
|
||||
// mutex not strictly necessary if caller is always single-threaded (Python)
|
||||
// but this preserves Torch thread_local semantics for the C++ API
|
||||
static std::mutex wv_pool_mu;
|
||||
static std::map<int, WvSplitKrcPool> wv_pools;
|
||||
|
||||
float* glbl;
|
||||
int* cntr;
|
||||
// Fallback space
|
||||
torch::Tensor spill_glbl, spill_cntr;
|
||||
{
|
||||
std::lock_guard<std::mutex> wv_lk(wv_pool_mu);
|
||||
const int wv_dev = static_cast<int>(in_a.device().index());
|
||||
WvSplitKrcPool& P = wv_pools[wv_dev];
|
||||
constexpr int64_t wv_gf = 128 * 1024 * (_DTRMNSTC ? 12 : 1);
|
||||
constexpr int64_t wv_ci = wv_gf / 4;
|
||||
constexpr int64_t wv_all_slots = kWvSlots + kWvCaptureSlots;
|
||||
|
||||
// A stream whose capture was already invalidated reports an error; treat
|
||||
// that as not capturing, since the capture is lost either way.
|
||||
cudaStreamCaptureStatus capture_status = cudaStreamCaptureStatusNone;
|
||||
unsigned long long capture_id = 0;
|
||||
if (cudaStreamGetCaptureInfo(stream, &capture_status, &capture_id) !=
|
||||
cudaSuccess) {
|
||||
capture_status = cudaStreamCaptureStatusNone;
|
||||
}
|
||||
const bool wv_capturing = capture_status != cudaStreamCaptureStatusNone;
|
||||
|
||||
if (!P.glbl.defined()) {
|
||||
TORCH_CHECK(!wv_capturing,
|
||||
"wvSplitKrc: the split-K workspace pool cannot be allocated "
|
||||
"during graph capture. Call "
|
||||
"warmup_rocm_skinny_gemm_workspaces() "
|
||||
"(vllm/model_executor/layers/utils.py) eagerly first.");
|
||||
P.glbl_stride = wv_gf;
|
||||
P.cntr_stride = wv_ci;
|
||||
P.glbl = torch::zeros(wv_gf * wv_all_slots, torch::TensorOptions()
|
||||
.dtype(torch::kFloat32)
|
||||
.device(in_a.device()))
|
||||
.detach();
|
||||
P.cntr =
|
||||
torch::zeros(
|
||||
wv_ci * wv_all_slots,
|
||||
torch::TensorOptions().dtype(torch::kInt).device(in_a.device()))
|
||||
.detach();
|
||||
// The mutex orders host access to the map, not device execution, so it
|
||||
// alone would let another stream be handed a slot the fill above is
|
||||
// still clearing. Synchronizing here, inside the lock, is what makes
|
||||
// every slot safe to use without any further per-slot zeroing.
|
||||
const cudaError_t wv_err = cudaStreamSynchronize(stream);
|
||||
TORCH_CHECK(wv_err == cudaSuccess,
|
||||
"wvSplitKrc: failed to zero the split-K workspace pool: ",
|
||||
cudaGetErrorString(wv_err));
|
||||
}
|
||||
|
||||
int wv_slot = -1;
|
||||
if (wv_capturing) {
|
||||
// One workspace per capturing stream, so graphs captured on the same
|
||||
// stream share it -- safe because at most one of them replays at a time
|
||||
// (vLLM captures one graph per padded batch size, and runs one per step).
|
||||
auto cap_it = P.capture_slot_of.find(stream);
|
||||
if (cap_it == P.capture_slot_of.end()) {
|
||||
TORCH_CHECK(P.next_capture_slot < kWvCaptureSlots,
|
||||
"wvSplitKrc: no free capture workspace slot; more than ",
|
||||
kWvCaptureSlots,
|
||||
" distinct streams have captured a graph containing "
|
||||
"wvSplitKrc. Raise kWvCaptureSlots in "
|
||||
"csrc/rocm/skinny_gemms.cu.");
|
||||
cap_it = P.capture_slot_of
|
||||
.emplace(stream, static_cast<int>(kWvSlots) +
|
||||
P.next_capture_slot++)
|
||||
.first;
|
||||
}
|
||||
wv_slot = cap_it->second;
|
||||
} else {
|
||||
auto slot_it = P.slot_of.find(stream);
|
||||
if (slot_it != P.slot_of.end()) {
|
||||
wv_slot = slot_it->second;
|
||||
} else if (P.next_slot < kWvSlots) {
|
||||
wv_slot = P.next_slot++;
|
||||
P.slot_of.emplace(stream, wv_slot);
|
||||
}
|
||||
}
|
||||
|
||||
if (wv_slot >= 0) {
|
||||
glbl = P.glbl.data_ptr<float>() + wv_slot * P.glbl_stride;
|
||||
cntr = P.cntr.data_ptr<int>() + wv_slot * P.cntr_stride;
|
||||
} else {
|
||||
spill_glbl = torch::zeros(
|
||||
wv_gf,
|
||||
torch::TensorOptions().dtype(torch::kFloat32).device(in_a.device()));
|
||||
spill_cntr = torch::zeros(
|
||||
wv_ci,
|
||||
torch::TensorOptions().dtype(torch::kInt).device(in_a.device()));
|
||||
glbl = spill_glbl.data_ptr<float>();
|
||||
cntr = spill_cntr.data_ptr<int>();
|
||||
}
|
||||
}
|
||||
|
||||
#define WVSPLITKrc(_N, _GrpsShrB, _CHUNKK) \
|
||||
{ \
|
||||
dim3 block(64, 4); \
|
||||
if (_DTRMNSTC) \
|
||||
if constexpr (_DTRMNSTC) \
|
||||
wvSplitKrc_<fptype, 64, 16, 4, 8, 1, _N, _GrpsShrB, _CHUNKK, 1> \
|
||||
<<<grid, block, 0, stream>>>(N_in, K_in, Kap_in, M_in, Bx_in, By_in, \
|
||||
af4, bf4, biasf4, glbl, cntr, c, \
|
||||
|
||||
@@ -53,6 +53,17 @@ OPTIONS_WVSPLITK_FP8 = [
|
||||
|
||||
DTYPES = [torch.bfloat16, torch.float16]
|
||||
|
||||
# kWvSlots in csrc/rocm/skinny_gemms.cu; beyond it streams take overflow
|
||||
# workspaces.
|
||||
WVSPLITKRC_SLOTS = 8
|
||||
|
||||
# The CU count is the grid size. Production passes num_compute_units(); a small
|
||||
# value manufactures better concurrency for these tests.
|
||||
WVSPLITKRC_TEST_CU = 16
|
||||
|
||||
# Gate spin, ~125 ms on gfx950 -- must outlast the host enqueue loop it gates.
|
||||
_GATE_CYCLES = 300_000_000
|
||||
|
||||
# Specific (N, K, M) combinations for targeted testing
|
||||
NKM_FACTORS_LLMM1 = [
|
||||
# Small, medium, large cases
|
||||
@@ -116,6 +127,15 @@ K_FACTORS_WVSPLITKRC = [2880, 2880 + 8, 3072, 3072 + 8]
|
||||
# M tiles are 64 rows, +16 for a partial tile
|
||||
M_FACTORS_WVSPLITKRC = [128, 128 + 16, 256, 256 + 16, 640, 640 + 16]
|
||||
|
||||
# (N, K, M) with more K-shards than the readback can stage in one LDS pass.
|
||||
# Spans both CHUNKK values and both N-tile counts.
|
||||
NKM_FACTORS_WVSPLITKRC_LARGE_K = [
|
||||
(128, 6144, 128),
|
||||
(96, 8192, 128),
|
||||
(128, 12288, 128),
|
||||
(32, 12288, 128),
|
||||
]
|
||||
|
||||
NKM_FACTORS_WVSPLITK_FP8 = [
|
||||
# FP8-specific cases with K % 16 == 0
|
||||
(1, 16, 16),
|
||||
@@ -222,6 +242,83 @@ def test_rocm_wvsplitkrc_kernel(n, k, m, dtype, padded_a, bias_mode, xnorm, seed
|
||||
torch.testing.assert_close(out, ref_out, atol=1e-3, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n,k,m", NKM_FACTORS_WVSPLITKRC_LARGE_K)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="only test for rocm")
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="only meant for gfx950")
|
||||
def test_rocm_wvsplitkrc_large_k(n, k, m, dtype, seed):
|
||||
"""K large enough that the split-K readback must stage LDS in batches."""
|
||||
torch.manual_seed(seed)
|
||||
|
||||
xavier = math.sqrt(2 / k)
|
||||
A = (torch.rand(n, k, dtype=dtype, device="cuda") * 2 - 1) * xavier
|
||||
B = (torch.rand(m, k, dtype=dtype, device="cuda") * 2 - 1) * xavier
|
||||
|
||||
ref_out = torch.nn.functional.linear(A, B, None)
|
||||
out = ops.wvSplitKrc(A, B, num_compute_units(), None)
|
||||
|
||||
torch.testing.assert_close(out, ref_out, atol=1e-3, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n_streams", [WVSPLITKRC_SLOTS, WVSPLITKRC_SLOTS + 4])
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="only test for rocm")
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="only meant for gfx950")
|
||||
def test_rocm_wvsplitkrc_multistream(n_streams, dtype, seed):
|
||||
"""Concurrent streams must not share a split-K workspace.
|
||||
|
||||
The workspace is cleared by protocol rather than per invocation, so sharing
|
||||
one corrupts the reduction and returns wrong numbers rather than failing.
|
||||
n_streams > WVSPLITKRC_SLOTS additionally exercises the overflow path.
|
||||
"""
|
||||
torch.manual_seed(seed)
|
||||
|
||||
n, k, m = 16, 2048, 256
|
||||
iters = 4
|
||||
xavier = math.sqrt(2 / k)
|
||||
|
||||
B = torch.randn(m, k, dtype=dtype, device="cuda") * xavier
|
||||
As = [
|
||||
torch.randn(n, k, dtype=dtype, device="cuda") * xavier for _ in range(n_streams)
|
||||
]
|
||||
refs = [torch.nn.functional.linear(A, B, None) for A in As]
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
streams = [torch.Stream() for _ in range(n_streams)]
|
||||
outs: list[list[torch.Tensor]] = [[] for _ in range(n_streams)]
|
||||
_release_together(streams)
|
||||
# Round-robin the enqueues so kernels from different streams are in flight
|
||||
# at the same time.
|
||||
for _ in range(iters):
|
||||
for i, s in enumerate(streams):
|
||||
with torch.cuda.stream(s):
|
||||
outs[i].append(ops.wvSplitKrc(As[i], B, WVSPLITKRC_TEST_CU, None))
|
||||
for s in streams:
|
||||
torch.accelerator.current_stream().wait_stream(s)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
for i in range(n_streams):
|
||||
for out in outs[i]:
|
||||
torch.testing.assert_close(out, refs[i], atol=1e-3, rtol=1e-2)
|
||||
|
||||
|
||||
def _release_together(streams):
|
||||
"""Release every stream at once, once everything below is enqueued.
|
||||
|
||||
An event recorded on an idle stream is already complete, so waiting on it
|
||||
gates nothing -- the spin, not the event, is what holds the streams.
|
||||
"""
|
||||
blocker = torch.Stream()
|
||||
with torch.cuda.stream(blocker):
|
||||
torch.cuda._sleep(_GATE_CYCLES)
|
||||
gate = torch.Event()
|
||||
gate.record()
|
||||
for s in streams:
|
||||
s.wait_event(gate)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n,k,m", NKM_FACTORS_LLMM1)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("rows_per_block", [2, 4, 8, 16])
|
||||
|
||||
@@ -2357,6 +2357,12 @@ def cleanup_dist_env_and_memory(shutdown_ray: bool = False):
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_cpu():
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
if HAS_TRITON:
|
||||
from vllm.v1.sample.ops.topk_topp_triton import reset_buffer_cache
|
||||
|
||||
reset_buffer_cache()
|
||||
torch.accelerator.empty_cache()
|
||||
try:
|
||||
torch.accelerator.empty_host_cache()
|
||||
|
||||
@@ -283,24 +283,11 @@ class MoERunner(MoERunnerInterface):
|
||||
self._shared_experts: SharedExperts | None = None
|
||||
if shared_experts is not None:
|
||||
can_overlap = lambda: self._quant_method.mk_can_overlap_shared_experts
|
||||
# When unquantized, shared expert inputs alias the hidden states,
|
||||
# which can lead to race condition in multi-stream mode. Quantized
|
||||
# routed experts copy the input into a fresh buffer first, breaking
|
||||
# the alias, so overlap is safe. Only observed on ROCm.
|
||||
routed_input_is_quantized = lambda: (
|
||||
self.routed_experts.quant_method.moe_quant_config is not None
|
||||
and self.routed_experts.quant_method.moe_quant_config.quant_dtype
|
||||
is not None
|
||||
)
|
||||
is_multistream_safe = lambda: (
|
||||
not current_platform.is_rocm() or routed_input_is_quantized()
|
||||
)
|
||||
self._shared_experts = SharedExperts(
|
||||
shared_experts,
|
||||
moe_config=moe_config,
|
||||
enable_dbo=enable_dbo,
|
||||
mk_can_overlap_shared_experts=can_overlap,
|
||||
is_multistream_safe=is_multistream_safe,
|
||||
)
|
||||
|
||||
# Needed for string -> MoERunner layer lookup in custom ops.
|
||||
|
||||
@@ -43,7 +43,6 @@ class SharedExperts(torch.nn.Module):
|
||||
moe_config: FusedMoEConfig,
|
||||
enable_dbo: bool,
|
||||
mk_can_overlap_shared_experts: Callable[[], bool],
|
||||
is_multistream_safe: Callable[[], bool],
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
@@ -58,10 +57,6 @@ class SharedExperts(torch.nn.Module):
|
||||
|
||||
self._mk_can_overlap_shared_experts = mk_can_overlap_shared_experts
|
||||
|
||||
# Might not be safe to run multi-stream mode if routed and shared experts
|
||||
# alias the same inputs
|
||||
self._is_multistream_safe = is_multistream_safe
|
||||
|
||||
# Allow disabling of the separate shared experts stream for
|
||||
# debug purposes.
|
||||
# TODO: Remove this after more extensive testings with TP/DP
|
||||
@@ -112,19 +107,11 @@ class SharedExperts(torch.nn.Module):
|
||||
if self._mk_can_overlap_shared_experts():
|
||||
return SharedExpertsOrder.MK_INTERNAL_OVERLAPPED
|
||||
|
||||
# On ROCm, empirically only DP-only deployments benefit from the overlap.
|
||||
overlap_is_beneficial = not current_platform.is_rocm() or (
|
||||
self._moe_config.moe_parallel_config.dp_size > 1
|
||||
and self._moe_config.moe_parallel_config.tp_size == 1
|
||||
)
|
||||
|
||||
should_run_shared_in_aux_stream = (
|
||||
current_platform.is_cuda_alike()
|
||||
and self._stream is not None
|
||||
and hidden_states.shape[0]
|
||||
<= envs.VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD
|
||||
and overlap_is_beneficial
|
||||
and self._is_multistream_safe()
|
||||
)
|
||||
|
||||
if should_run_shared_in_aux_stream:
|
||||
|
||||
@@ -249,6 +249,45 @@ def use_aiter_triton_gemm(n, m, k, dtype):
|
||||
)
|
||||
|
||||
|
||||
def wvsplitkrc_dispatch(n: int, k: int, m: int, cu_count: int) -> tuple[int, bool]:
|
||||
"""Pick the K-shard split for wvSplitKrc and say whether the shape fits.
|
||||
|
||||
Mirrors wvSplitKrc() in csrc/rocm/skinny_gemms.cu, which is also where the
|
||||
shard cap is explained. Both must pick the same chunkk or the workspace
|
||||
check here bounds the wrong k_rnd.
|
||||
|
||||
Returns:
|
||||
The CHUNKK the kernel will dispatch with, and whether the CU budget and
|
||||
split-K workspace admit the shape at all.
|
||||
"""
|
||||
# Next ^2 of n
|
||||
N_p2 = 1 << (n - 1).bit_length()
|
||||
# How many of 4 waves in a group can work on same 16 Ms at same time?
|
||||
# This reduces the Ms each group works on, i.e. increasing the CUs needed.
|
||||
GrpsShrB = min(N_p2 // 16, 4)
|
||||
# With 64 Ms per CU (each of 4 SIMDs working on a 16x16 tile), and each
|
||||
# working on a 512-shard of K, how many CUs would we need?
|
||||
CuNeeded = ((m + 64 - 1) // 64) * ((k + 512 - 1) // 512) * GrpsShrB
|
||||
|
||||
CHUNKK2_MAX_SHARDS = 11
|
||||
shards_chunkk2 = (k + 256 - 1) // 256 # 256-wide shards
|
||||
chunkk = (
|
||||
2
|
||||
if (
|
||||
N_p2 != 16
|
||||
and CuNeeded * 2 <= cu_count
|
||||
and shards_chunkk2 <= CHUNKK2_MAX_SHARDS
|
||||
)
|
||||
else 1
|
||||
)
|
||||
|
||||
# Deterministic reduction stores one fp32 partial per (M, N, k-shard); all
|
||||
# of them must fit the split-K workspace.
|
||||
k_rnd = (k + 512 // chunkk - 1) // (512 // chunkk)
|
||||
fits = N_p2 * m * k_rnd <= 128 * 1024 * 12 and CuNeeded <= cu_count
|
||||
return chunkk, fits
|
||||
|
||||
|
||||
def rocm_unquantized_gemm_impl(
|
||||
x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
@@ -260,21 +299,7 @@ def rocm_unquantized_gemm_impl(
|
||||
|
||||
cu_count = num_compute_units()
|
||||
|
||||
# Next ^2 of n
|
||||
N_p2 = 1 << (n - 1).bit_length()
|
||||
# With 64 Ms per CU (each of 4 SIMDs working on a 16x16 tile),
|
||||
# and each working on a 512-shard of K, how many CUs would we need?
|
||||
rndup_cus = ((m + 64 - 1) // 64) * ((k + 512 - 1) // 512)
|
||||
# How many of 4 waves in a group can work on same 16 Ms at same time?
|
||||
# This reduces the Ms each group works on, i.e. increasing the number of CUs needed.
|
||||
GrpsShrB = min(N_p2 // 16, 4)
|
||||
# Given the above, how many CUs would we need?
|
||||
CuNeeded = rndup_cus * GrpsShrB
|
||||
# Deterministic reduction stores one float workspace value per K shard.
|
||||
fits_wvsplitkrc = (
|
||||
N_p2 * m * ((k + 512 - 1) // 512)
|
||||
) <= 128 * 1024 * 12 # deterministic
|
||||
fits_wvsplitkrc &= CuNeeded <= cu_count
|
||||
_, fits_wvsplitkrc = wvsplitkrc_dispatch(n, k, m, cu_count)
|
||||
|
||||
skinny_operands_compatible = weight.is_contiguous() and (
|
||||
bias is None or bias.is_contiguous()
|
||||
@@ -362,12 +387,18 @@ direct_register_custom_op(
|
||||
|
||||
@functools.cache
|
||||
def warmup_rocm_skinny_gemm_workspaces(device: torch.device) -> None:
|
||||
"""Eagerly allocate wvSplitKrc's process-lifetime static workspaces.
|
||||
"""Eagerly allocate wvSplitKrc's per-device split-K workspace pool.
|
||||
|
||||
They are otherwise created lazily on the first qualifying GEMM
|
||||
wvSplitKrc partitions one per-device allocation into ``kWvSlots`` slots
|
||||
(csrc/rocm/skinny_gemms.cu) and hands each stream one on first use, so that
|
||||
two streams never share the split-K partials and counters.
|
||||
|
||||
The pool is otherwise created lazily on the first qualifying GEMM
|
||||
(csrc/rocm/skinny_gemms.cu), which can be the first real request — after
|
||||
the KV cache backing buffer exists. If one landed in that segment's
|
||||
rounding tail, it would pin the entire segment at engine shutdown.
|
||||
the KV cache backing buffer exists. If it landed in that segment's rounding
|
||||
tail, it would pin the entire segment at engine shutdown; it could also land
|
||||
inside a cudagraph capture, where it would be taken from the graph's private
|
||||
pool and its zero-fill would become a replayed graph node.
|
||||
"""
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
|
||||
Reference in New Issue
Block a user