[Kernel] Vectorized fp32 moe_sum reduction and support any topk (#46643)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Michael Goin
2026-06-25 14:02:28 -07:00
committed by GitHub
co-authored by Claude
parent 2a6f8f0c05
commit e8c24a7695
2 changed files with 175 additions and 49 deletions
+159 -45
View File
@@ -11,6 +11,7 @@
#include "../../cuda_compat.h"
#include "libtorch_stable/core/math.hpp"
#include "libtorch_stable/dispatch_utils.h"
#include "libtorch_stable/quantization/vectorization.cuh"
#include "libtorch_stable/torch_utils.h"
#define CEILDIV(x, y) (((x) + (y) - 1) / (y))
@@ -349,19 +350,102 @@ __global__ void count_and_sort_expert_tokens_kernel(
max_num_tokens_padded, nullptr, 0, topk_num, has_expert_map);
}
// Reduce the topk expert outputs per token (summed in fp32). The output is
// dense [num_tokens, d]; the input is addressed by its strides so non-
// contiguous inputs work without a copy. A 16B-vectorized path is used when
// the hidden dim is contiguous (innermost stride 1) and aligned; otherwise a
// scalar kernel reads via arbitrary strides. topk is a compile-time constant
// for common values and runtime otherwise.
// Elements per 16-byte vector (8 for bf16/fp16, 4 for fp32).
template <typename scalar_t>
constexpr int MOE_SUM_VEC = 16 / sizeof(scalar_t);
template <typename scalar_t, int TOPK>
__global__ void moe_sum_kernel(
scalar_t* __restrict__ out, // [..., d]
const scalar_t* __restrict__ input, // [..., topk, d]
const int d) {
const int64_t token_idx = blockIdx.x;
for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
scalar_t x = 0.0;
__global__ void moe_sum_vec_kernel(
scalar_t* __restrict__ out, // [num_tokens, d], contiguous
const scalar_t* __restrict__ input, // [num_tokens, topk, d], d contiguous
const int64_t num_tokens, const int d, const int64_t stride_token,
const int64_t stride_topk) {
using vec_t = vllm::vec_n_t<scalar_t, MOE_SUM_VEC<scalar_t>>; // 16-byte pack
constexpr int VEC = MOE_SUM_VEC<scalar_t>;
const int64_t n_vec = d / VEC;
const int64_t total = num_tokens * n_vec;
for (int64_t i = blockIdx.x * blockDim.x + threadIdx.x; i < total;
i += (int64_t)gridDim.x * blockDim.x) {
const int64_t token = i / n_vec;
const int64_t v = i % n_vec;
const scalar_t* in_tok = input + token * stride_token + v * VEC;
float acc[VEC];
#pragma unroll
for (int j = 0; j < VEC; ++j) acc[j] = 0.f;
#pragma unroll
for (int k = 0; k < TOPK; ++k) {
x += VLLM_LDG(&input[token_idx * TOPK * d + k * d + idx]);
vec_t packed = *reinterpret_cast<const vec_t*>(in_tok + k * stride_topk);
#pragma unroll
for (int j = 0; j < VEC; ++j) acc[j] += static_cast<float>(packed.val[j]);
}
out[token_idx * d + idx] = x;
vec_t outp;
#pragma unroll
for (int j = 0; j < VEC; ++j) outp.val[j] = static_cast<scalar_t>(acc[j]);
*reinterpret_cast<vec_t*>(out + token * d + v * VEC) = outp;
}
}
// Runtime-topk variant of the above.
template <typename scalar_t>
__global__ void moe_sum_vec_dynamic_kernel(
scalar_t* __restrict__ out, // [num_tokens, d], contiguous
const scalar_t* __restrict__ input, // [num_tokens, topk, d], d contiguous
const int64_t num_tokens, const int d, const int topk,
const int64_t stride_token, const int64_t stride_topk) {
using vec_t = vllm::vec_n_t<scalar_t, MOE_SUM_VEC<scalar_t>>;
constexpr int VEC = MOE_SUM_VEC<scalar_t>;
const int64_t n_vec = d / VEC;
const int64_t total = num_tokens * n_vec;
for (int64_t i = blockIdx.x * blockDim.x + threadIdx.x; i < total;
i += (int64_t)gridDim.x * blockDim.x) {
const int64_t token = i / n_vec;
const int64_t v = i % n_vec;
const scalar_t* in_tok = input + token * stride_token + v * VEC;
float acc[VEC];
#pragma unroll
for (int j = 0; j < VEC; ++j) acc[j] = 0.f;
for (int k = 0; k < topk; ++k) {
vec_t packed = *reinterpret_cast<const vec_t*>(in_tok + k * stride_topk);
#pragma unroll
for (int j = 0; j < VEC; ++j) acc[j] += static_cast<float>(packed.val[j]);
}
vec_t outp;
#pragma unroll
for (int j = 0; j < VEC; ++j) outp.val[j] = static_cast<scalar_t>(acc[j]);
*reinterpret_cast<vec_t*>(out + token * d + v * VEC) = outp;
}
}
// Stride-aware scalar fallback: handles unaligned/non-vectorizable hidden dims
// (including a non-contiguous hidden stride) via per-element strided reads.
template <typename scalar_t>
__global__ void moe_sum_scalar_kernel(
scalar_t* __restrict__ out, // [num_tokens, d], contiguous
const scalar_t* __restrict__ input, // [num_tokens, topk, d]
const int d, const int topk, const int64_t stride_token,
const int64_t stride_topk, const int64_t stride_hidden) {
const int64_t token_idx = blockIdx.x;
const scalar_t* in_tok = input + token_idx * stride_token;
for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
float x = 0.f;
for (int k = 0; k < topk; ++k) {
x += static_cast<float>(
VLLM_LDG(&in_tok[k * stride_topk + idx * stride_hidden]));
}
out[token_idx * d + idx] = static_cast<scalar_t>(x);
}
}
@@ -626,52 +710,82 @@ void batched_moe_align_block_size(int64_t max_tokens_per_batch,
void moe_sum(torch::stable::Tensor& input, // [num_tokens, topk, hidden_size]
torch::stable::Tensor& output) // [num_tokens, hidden_size]
{
// Output is dense and written in place, so it must be contiguous. The input
// is read by its strides (no copy); only the hidden dim needs to be
// contiguous to take the vectorized path.
STD_TORCH_CHECK(output.is_contiguous(),
"moe_sum expects a contiguous output");
const int hidden_size = input.size(-1);
const auto num_tokens = output.numel() / hidden_size;
const int64_t num_tokens = output.numel() / hidden_size;
const int topk = input.size(1);
const int64_t stride_token = input.stride(0);
const int64_t stride_topk = input.stride(1);
const int64_t stride_hidden = input.stride(2);
dim3 grid(num_tokens);
dim3 block(std::min(hidden_size, 1024));
const torch::stable::accelerator::DeviceGuard device_guard(
output.get_device_index());
const cudaStream_t stream =
get_current_cuda_stream(output.get_device_index());
switch (topk) {
case 2:
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
input.scalar_type(), "moe_sum_kernel", [&] {
vllm::moe::moe_sum_kernel<scalar_t, 2><<<grid, block, 0, stream>>>(
reinterpret_cast<scalar_t*>(output.mutable_data_ptr()),
reinterpret_cast<const scalar_t*>(input.const_data_ptr()),
hidden_size);
});
break;
#define LAUNCH_MOE_SUM_VEC(TOPK) \
vllm::moe::moe_sum_vec_kernel<scalar_t, TOPK> \
<<<grid, dim3(block), 0, stream>>>( \
out_ptr, in_ptr, num_tokens, hidden_size, stride_token, stride_topk)
case 3:
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
input.scalar_type(), "moe_sum_kernel", [&] {
vllm::moe::moe_sum_kernel<scalar_t, 3><<<grid, block, 0, stream>>>(
reinterpret_cast<scalar_t*>(output.mutable_data_ptr()),
reinterpret_cast<const scalar_t*>(input.const_data_ptr()),
hidden_size);
});
break;
VLLM_STABLE_DISPATCH_FLOATING_TYPES(input.scalar_type(), "moe_sum", [&] {
constexpr int VEC = vllm::moe::MOE_SUM_VEC<scalar_t>;
constexpr int WIDTH = VEC * sizeof(scalar_t); // 16 bytes
auto* out_ptr = reinterpret_cast<scalar_t*>(output.mutable_data_ptr());
auto* in_ptr = reinterpret_cast<const scalar_t*>(input.const_data_ptr());
case 4:
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
input.scalar_type(), "moe_sum_kernel", [&] {
vllm::moe::moe_sum_kernel<scalar_t, 4><<<grid, block, 0, stream>>>(
reinterpret_cast<scalar_t*>(output.mutable_data_ptr()),
reinterpret_cast<const scalar_t*>(input.const_data_ptr()),
hidden_size);
});
break;
default:
torch::stable::sum_out(output, input, std::array<int64_t, 1>{1});
break;
}
// Vectorize along hidden only when it is contiguous (innermost stride 1),
// a whole number of vectors, and every row offset stays 16B-aligned.
const bool can_vec = (stride_hidden == 1) && (hidden_size % VEC == 0) &&
(stride_token % VEC == 0) &&
(stride_topk % VEC == 0) &&
(reinterpret_cast<uintptr_t>(in_ptr) % WIDTH == 0) &&
(reinterpret_cast<uintptr_t>(out_ptr) % WIDTH == 0);
if (can_vec) {
const int64_t n_vec = hidden_size / VEC;
const int64_t total = num_tokens * n_vec;
const int block = 256;
const dim3 grid(std::min<int64_t>((total + block - 1) / block, 65535));
switch (topk) {
case 1:
LAUNCH_MOE_SUM_VEC(1);
break;
case 2:
LAUNCH_MOE_SUM_VEC(2);
break;
case 4:
LAUNCH_MOE_SUM_VEC(4);
break;
case 6:
LAUNCH_MOE_SUM_VEC(6);
break;
case 8:
LAUNCH_MOE_SUM_VEC(8);
break;
case 9:
LAUNCH_MOE_SUM_VEC(9);
break;
default:
vllm::moe::moe_sum_vec_dynamic_kernel<scalar_t>
<<<grid, dim3(block), 0, stream>>>(out_ptr, in_ptr, num_tokens,
hidden_size, topk,
stride_token, stride_topk);
break;
}
} else {
dim3 grid(num_tokens);
dim3 block(std::min(hidden_size, 1024));
vllm::moe::moe_sum_scalar_kernel<scalar_t><<<grid, block, 0, stream>>>(
out_ptr, in_ptr, hidden_size, topk, stride_token, stride_topk,
stride_hidden);
}
});
#undef LAUNCH_MOE_SUM_VEC
}
void moe_lora_align_block_size(
+16 -4
View File
@@ -1243,15 +1243,27 @@ def test_batched_moe_align_block_size_opcheck():
)
# topk=8 covers topk > 4; k=511 covers the non-vectorized scalar path. The
# layouts exercise contiguous input plus the two non-contiguous cases: a
# transpose (strided hidden -> scalar gather) and a topk-slice (hidden still
# contiguous -> vectorized).
@pytest.mark.parametrize("m", [1, 33, 222])
@pytest.mark.parametrize("topk", TOP_KS)
@pytest.mark.parametrize("topk", [*TOP_KS, 8])
@pytest.mark.parametrize("k", [128, 511, 1024])
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
def test_moe_sum(m: int, topk: int, k: int, dtype: torch.dtype):
input = torch.randn((m, topk, k), device="cuda", dtype=dtype)
@pytest.mark.parametrize("layout", ["contig", "transpose", "slice"])
def test_moe_sum(m: int, topk: int, k: int, dtype: torch.dtype, layout: str):
if layout == "transpose":
input = torch.randn((m, k, topk), device="cuda", dtype=dtype).transpose(1, 2)
elif layout == "slice":
input = torch.randn((m, 2 * topk, k), device="cuda", dtype=dtype)[:, ::2, :]
else:
input = torch.randn((m, topk, k), device="cuda", dtype=dtype)
assert input.is_contiguous() == (layout == "contig")
actual = torch.empty((m, k), device="cuda", dtype=dtype)
expected = input.sum(dim=1)
# Reduction accumulates in fp32.
expected = input.float().sum(dim=1).to(dtype)
torch.ops._moe_C.moe_sum(input, actual)
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=0)