[Kernel] Vectorized fp32 moe_sum reduction and support any topk (#46643)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -11,6 +11,7 @@
|
||||
#include "../../cuda_compat.h"
|
||||
#include "libtorch_stable/core/math.hpp"
|
||||
#include "libtorch_stable/dispatch_utils.h"
|
||||
#include "libtorch_stable/quantization/vectorization.cuh"
|
||||
#include "libtorch_stable/torch_utils.h"
|
||||
|
||||
#define CEILDIV(x, y) (((x) + (y) - 1) / (y))
|
||||
@@ -349,19 +350,102 @@ __global__ void count_and_sort_expert_tokens_kernel(
|
||||
max_num_tokens_padded, nullptr, 0, topk_num, has_expert_map);
|
||||
}
|
||||
|
||||
// Reduce the topk expert outputs per token (summed in fp32). The output is
|
||||
// dense [num_tokens, d]; the input is addressed by its strides so non-
|
||||
// contiguous inputs work without a copy. A 16B-vectorized path is used when
|
||||
// the hidden dim is contiguous (innermost stride 1) and aligned; otherwise a
|
||||
// scalar kernel reads via arbitrary strides. topk is a compile-time constant
|
||||
// for common values and runtime otherwise.
|
||||
|
||||
// Elements per 16-byte vector (8 for bf16/fp16, 4 for fp32).
|
||||
template <typename scalar_t>
|
||||
constexpr int MOE_SUM_VEC = 16 / sizeof(scalar_t);
|
||||
|
||||
template <typename scalar_t, int TOPK>
|
||||
__global__ void moe_sum_kernel(
|
||||
scalar_t* __restrict__ out, // [..., d]
|
||||
const scalar_t* __restrict__ input, // [..., topk, d]
|
||||
const int d) {
|
||||
const int64_t token_idx = blockIdx.x;
|
||||
for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
|
||||
scalar_t x = 0.0;
|
||||
__global__ void moe_sum_vec_kernel(
|
||||
scalar_t* __restrict__ out, // [num_tokens, d], contiguous
|
||||
const scalar_t* __restrict__ input, // [num_tokens, topk, d], d contiguous
|
||||
const int64_t num_tokens, const int d, const int64_t stride_token,
|
||||
const int64_t stride_topk) {
|
||||
using vec_t = vllm::vec_n_t<scalar_t, MOE_SUM_VEC<scalar_t>>; // 16-byte pack
|
||||
constexpr int VEC = MOE_SUM_VEC<scalar_t>;
|
||||
const int64_t n_vec = d / VEC;
|
||||
const int64_t total = num_tokens * n_vec;
|
||||
for (int64_t i = blockIdx.x * blockDim.x + threadIdx.x; i < total;
|
||||
i += (int64_t)gridDim.x * blockDim.x) {
|
||||
const int64_t token = i / n_vec;
|
||||
const int64_t v = i % n_vec;
|
||||
const scalar_t* in_tok = input + token * stride_token + v * VEC;
|
||||
|
||||
float acc[VEC];
|
||||
#pragma unroll
|
||||
for (int j = 0; j < VEC; ++j) acc[j] = 0.f;
|
||||
|
||||
#pragma unroll
|
||||
for (int k = 0; k < TOPK; ++k) {
|
||||
x += VLLM_LDG(&input[token_idx * TOPK * d + k * d + idx]);
|
||||
vec_t packed = *reinterpret_cast<const vec_t*>(in_tok + k * stride_topk);
|
||||
#pragma unroll
|
||||
for (int j = 0; j < VEC; ++j) acc[j] += static_cast<float>(packed.val[j]);
|
||||
}
|
||||
out[token_idx * d + idx] = x;
|
||||
|
||||
vec_t outp;
|
||||
#pragma unroll
|
||||
for (int j = 0; j < VEC; ++j) outp.val[j] = static_cast<scalar_t>(acc[j]);
|
||||
*reinterpret_cast<vec_t*>(out + token * d + v * VEC) = outp;
|
||||
}
|
||||
}
|
||||
|
||||
// Runtime-topk variant of the above.
|
||||
template <typename scalar_t>
|
||||
__global__ void moe_sum_vec_dynamic_kernel(
|
||||
scalar_t* __restrict__ out, // [num_tokens, d], contiguous
|
||||
const scalar_t* __restrict__ input, // [num_tokens, topk, d], d contiguous
|
||||
const int64_t num_tokens, const int d, const int topk,
|
||||
const int64_t stride_token, const int64_t stride_topk) {
|
||||
using vec_t = vllm::vec_n_t<scalar_t, MOE_SUM_VEC<scalar_t>>;
|
||||
constexpr int VEC = MOE_SUM_VEC<scalar_t>;
|
||||
const int64_t n_vec = d / VEC;
|
||||
const int64_t total = num_tokens * n_vec;
|
||||
for (int64_t i = blockIdx.x * blockDim.x + threadIdx.x; i < total;
|
||||
i += (int64_t)gridDim.x * blockDim.x) {
|
||||
const int64_t token = i / n_vec;
|
||||
const int64_t v = i % n_vec;
|
||||
const scalar_t* in_tok = input + token * stride_token + v * VEC;
|
||||
|
||||
float acc[VEC];
|
||||
#pragma unroll
|
||||
for (int j = 0; j < VEC; ++j) acc[j] = 0.f;
|
||||
|
||||
for (int k = 0; k < topk; ++k) {
|
||||
vec_t packed = *reinterpret_cast<const vec_t*>(in_tok + k * stride_topk);
|
||||
#pragma unroll
|
||||
for (int j = 0; j < VEC; ++j) acc[j] += static_cast<float>(packed.val[j]);
|
||||
}
|
||||
|
||||
vec_t outp;
|
||||
#pragma unroll
|
||||
for (int j = 0; j < VEC; ++j) outp.val[j] = static_cast<scalar_t>(acc[j]);
|
||||
*reinterpret_cast<vec_t*>(out + token * d + v * VEC) = outp;
|
||||
}
|
||||
}
|
||||
|
||||
// Stride-aware scalar fallback: handles unaligned/non-vectorizable hidden dims
|
||||
// (including a non-contiguous hidden stride) via per-element strided reads.
|
||||
template <typename scalar_t>
|
||||
__global__ void moe_sum_scalar_kernel(
|
||||
scalar_t* __restrict__ out, // [num_tokens, d], contiguous
|
||||
const scalar_t* __restrict__ input, // [num_tokens, topk, d]
|
||||
const int d, const int topk, const int64_t stride_token,
|
||||
const int64_t stride_topk, const int64_t stride_hidden) {
|
||||
const int64_t token_idx = blockIdx.x;
|
||||
const scalar_t* in_tok = input + token_idx * stride_token;
|
||||
for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
|
||||
float x = 0.f;
|
||||
for (int k = 0; k < topk; ++k) {
|
||||
x += static_cast<float>(
|
||||
VLLM_LDG(&in_tok[k * stride_topk + idx * stride_hidden]));
|
||||
}
|
||||
out[token_idx * d + idx] = static_cast<scalar_t>(x);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -626,52 +710,82 @@ void batched_moe_align_block_size(int64_t max_tokens_per_batch,
|
||||
void moe_sum(torch::stable::Tensor& input, // [num_tokens, topk, hidden_size]
|
||||
torch::stable::Tensor& output) // [num_tokens, hidden_size]
|
||||
{
|
||||
// Output is dense and written in place, so it must be contiguous. The input
|
||||
// is read by its strides (no copy); only the hidden dim needs to be
|
||||
// contiguous to take the vectorized path.
|
||||
STD_TORCH_CHECK(output.is_contiguous(),
|
||||
"moe_sum expects a contiguous output");
|
||||
|
||||
const int hidden_size = input.size(-1);
|
||||
const auto num_tokens = output.numel() / hidden_size;
|
||||
const int64_t num_tokens = output.numel() / hidden_size;
|
||||
const int topk = input.size(1);
|
||||
const int64_t stride_token = input.stride(0);
|
||||
const int64_t stride_topk = input.stride(1);
|
||||
const int64_t stride_hidden = input.stride(2);
|
||||
|
||||
dim3 grid(num_tokens);
|
||||
dim3 block(std::min(hidden_size, 1024));
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
output.get_device_index());
|
||||
const cudaStream_t stream =
|
||||
get_current_cuda_stream(output.get_device_index());
|
||||
|
||||
switch (topk) {
|
||||
case 2:
|
||||
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
|
||||
input.scalar_type(), "moe_sum_kernel", [&] {
|
||||
vllm::moe::moe_sum_kernel<scalar_t, 2><<<grid, block, 0, stream>>>(
|
||||
reinterpret_cast<scalar_t*>(output.mutable_data_ptr()),
|
||||
reinterpret_cast<const scalar_t*>(input.const_data_ptr()),
|
||||
hidden_size);
|
||||
});
|
||||
break;
|
||||
#define LAUNCH_MOE_SUM_VEC(TOPK) \
|
||||
vllm::moe::moe_sum_vec_kernel<scalar_t, TOPK> \
|
||||
<<<grid, dim3(block), 0, stream>>>( \
|
||||
out_ptr, in_ptr, num_tokens, hidden_size, stride_token, stride_topk)
|
||||
|
||||
case 3:
|
||||
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
|
||||
input.scalar_type(), "moe_sum_kernel", [&] {
|
||||
vllm::moe::moe_sum_kernel<scalar_t, 3><<<grid, block, 0, stream>>>(
|
||||
reinterpret_cast<scalar_t*>(output.mutable_data_ptr()),
|
||||
reinterpret_cast<const scalar_t*>(input.const_data_ptr()),
|
||||
hidden_size);
|
||||
});
|
||||
break;
|
||||
VLLM_STABLE_DISPATCH_FLOATING_TYPES(input.scalar_type(), "moe_sum", [&] {
|
||||
constexpr int VEC = vllm::moe::MOE_SUM_VEC<scalar_t>;
|
||||
constexpr int WIDTH = VEC * sizeof(scalar_t); // 16 bytes
|
||||
auto* out_ptr = reinterpret_cast<scalar_t*>(output.mutable_data_ptr());
|
||||
auto* in_ptr = reinterpret_cast<const scalar_t*>(input.const_data_ptr());
|
||||
|
||||
case 4:
|
||||
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
|
||||
input.scalar_type(), "moe_sum_kernel", [&] {
|
||||
vllm::moe::moe_sum_kernel<scalar_t, 4><<<grid, block, 0, stream>>>(
|
||||
reinterpret_cast<scalar_t*>(output.mutable_data_ptr()),
|
||||
reinterpret_cast<const scalar_t*>(input.const_data_ptr()),
|
||||
hidden_size);
|
||||
});
|
||||
break;
|
||||
|
||||
default:
|
||||
torch::stable::sum_out(output, input, std::array<int64_t, 1>{1});
|
||||
break;
|
||||
}
|
||||
// Vectorize along hidden only when it is contiguous (innermost stride 1),
|
||||
// a whole number of vectors, and every row offset stays 16B-aligned.
|
||||
const bool can_vec = (stride_hidden == 1) && (hidden_size % VEC == 0) &&
|
||||
(stride_token % VEC == 0) &&
|
||||
(stride_topk % VEC == 0) &&
|
||||
(reinterpret_cast<uintptr_t>(in_ptr) % WIDTH == 0) &&
|
||||
(reinterpret_cast<uintptr_t>(out_ptr) % WIDTH == 0);
|
||||
if (can_vec) {
|
||||
const int64_t n_vec = hidden_size / VEC;
|
||||
const int64_t total = num_tokens * n_vec;
|
||||
const int block = 256;
|
||||
const dim3 grid(std::min<int64_t>((total + block - 1) / block, 65535));
|
||||
switch (topk) {
|
||||
case 1:
|
||||
LAUNCH_MOE_SUM_VEC(1);
|
||||
break;
|
||||
case 2:
|
||||
LAUNCH_MOE_SUM_VEC(2);
|
||||
break;
|
||||
case 4:
|
||||
LAUNCH_MOE_SUM_VEC(4);
|
||||
break;
|
||||
case 6:
|
||||
LAUNCH_MOE_SUM_VEC(6);
|
||||
break;
|
||||
case 8:
|
||||
LAUNCH_MOE_SUM_VEC(8);
|
||||
break;
|
||||
case 9:
|
||||
LAUNCH_MOE_SUM_VEC(9);
|
||||
break;
|
||||
default:
|
||||
vllm::moe::moe_sum_vec_dynamic_kernel<scalar_t>
|
||||
<<<grid, dim3(block), 0, stream>>>(out_ptr, in_ptr, num_tokens,
|
||||
hidden_size, topk,
|
||||
stride_token, stride_topk);
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
dim3 grid(num_tokens);
|
||||
dim3 block(std::min(hidden_size, 1024));
|
||||
vllm::moe::moe_sum_scalar_kernel<scalar_t><<<grid, block, 0, stream>>>(
|
||||
out_ptr, in_ptr, hidden_size, topk, stride_token, stride_topk,
|
||||
stride_hidden);
|
||||
}
|
||||
});
|
||||
#undef LAUNCH_MOE_SUM_VEC
|
||||
}
|
||||
|
||||
void moe_lora_align_block_size(
|
||||
|
||||
@@ -1243,15 +1243,27 @@ def test_batched_moe_align_block_size_opcheck():
|
||||
)
|
||||
|
||||
|
||||
# topk=8 covers topk > 4; k=511 covers the non-vectorized scalar path. The
|
||||
# layouts exercise contiguous input plus the two non-contiguous cases: a
|
||||
# transpose (strided hidden -> scalar gather) and a topk-slice (hidden still
|
||||
# contiguous -> vectorized).
|
||||
@pytest.mark.parametrize("m", [1, 33, 222])
|
||||
@pytest.mark.parametrize("topk", TOP_KS)
|
||||
@pytest.mark.parametrize("topk", [*TOP_KS, 8])
|
||||
@pytest.mark.parametrize("k", [128, 511, 1024])
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
|
||||
def test_moe_sum(m: int, topk: int, k: int, dtype: torch.dtype):
|
||||
input = torch.randn((m, topk, k), device="cuda", dtype=dtype)
|
||||
@pytest.mark.parametrize("layout", ["contig", "transpose", "slice"])
|
||||
def test_moe_sum(m: int, topk: int, k: int, dtype: torch.dtype, layout: str):
|
||||
if layout == "transpose":
|
||||
input = torch.randn((m, k, topk), device="cuda", dtype=dtype).transpose(1, 2)
|
||||
elif layout == "slice":
|
||||
input = torch.randn((m, 2 * topk, k), device="cuda", dtype=dtype)[:, ::2, :]
|
||||
else:
|
||||
input = torch.randn((m, topk, k), device="cuda", dtype=dtype)
|
||||
assert input.is_contiguous() == (layout == "contig")
|
||||
actual = torch.empty((m, k), device="cuda", dtype=dtype)
|
||||
|
||||
expected = input.sum(dim=1)
|
||||
# Reduction accumulates in fp32.
|
||||
expected = input.float().sum(dim=1).to(dtype)
|
||||
torch.ops._moe_C.moe_sum(input, actual)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=0)
|
||||
|
||||
Reference in New Issue
Block a user