From e8c24a769576fa318ca93fbd927128246a534325 Mon Sep 17 00:00:00 2001 From: Michael Goin Date: Thu, 25 Jun 2026 15:02:28 -0600 Subject: [PATCH] [Kernel] Vectorized fp32 `moe_sum` reduction and support any topk (#46643) Signed-off-by: mgoin Co-authored-by: Claude --- .../moe/moe_align_sum_kernels.cu | 204 ++++++++++++++---- tests/kernels/moe/test_moe.py | 20 +- 2 files changed, 175 insertions(+), 49 deletions(-) diff --git a/csrc/libtorch_stable/moe/moe_align_sum_kernels.cu b/csrc/libtorch_stable/moe/moe_align_sum_kernels.cu index 1e84238134..985c47b076 100644 --- a/csrc/libtorch_stable/moe/moe_align_sum_kernels.cu +++ b/csrc/libtorch_stable/moe/moe_align_sum_kernels.cu @@ -11,6 +11,7 @@ #include "../../cuda_compat.h" #include "libtorch_stable/core/math.hpp" #include "libtorch_stable/dispatch_utils.h" +#include "libtorch_stable/quantization/vectorization.cuh" #include "libtorch_stable/torch_utils.h" #define CEILDIV(x, y) (((x) + (y) - 1) / (y)) @@ -349,19 +350,102 @@ __global__ void count_and_sort_expert_tokens_kernel( max_num_tokens_padded, nullptr, 0, topk_num, has_expert_map); } +// Reduce the topk expert outputs per token (summed in fp32). The output is +// dense [num_tokens, d]; the input is addressed by its strides so non- +// contiguous inputs work without a copy. A 16B-vectorized path is used when +// the hidden dim is contiguous (innermost stride 1) and aligned; otherwise a +// scalar kernel reads via arbitrary strides. topk is a compile-time constant +// for common values and runtime otherwise. + +// Elements per 16-byte vector (8 for bf16/fp16, 4 for fp32). +template +constexpr int MOE_SUM_VEC = 16 / sizeof(scalar_t); + template -__global__ void moe_sum_kernel( - scalar_t* __restrict__ out, // [..., d] - const scalar_t* __restrict__ input, // [..., topk, d] - const int d) { - const int64_t token_idx = blockIdx.x; - for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) { - scalar_t x = 0.0; +__global__ void moe_sum_vec_kernel( + scalar_t* __restrict__ out, // [num_tokens, d], contiguous + const scalar_t* __restrict__ input, // [num_tokens, topk, d], d contiguous + const int64_t num_tokens, const int d, const int64_t stride_token, + const int64_t stride_topk) { + using vec_t = vllm::vec_n_t>; // 16-byte pack + constexpr int VEC = MOE_SUM_VEC; + const int64_t n_vec = d / VEC; + const int64_t total = num_tokens * n_vec; + for (int64_t i = blockIdx.x * blockDim.x + threadIdx.x; i < total; + i += (int64_t)gridDim.x * blockDim.x) { + const int64_t token = i / n_vec; + const int64_t v = i % n_vec; + const scalar_t* in_tok = input + token * stride_token + v * VEC; + + float acc[VEC]; +#pragma unroll + for (int j = 0; j < VEC; ++j) acc[j] = 0.f; + #pragma unroll for (int k = 0; k < TOPK; ++k) { - x += VLLM_LDG(&input[token_idx * TOPK * d + k * d + idx]); + vec_t packed = *reinterpret_cast(in_tok + k * stride_topk); +#pragma unroll + for (int j = 0; j < VEC; ++j) acc[j] += static_cast(packed.val[j]); } - out[token_idx * d + idx] = x; + + vec_t outp; +#pragma unroll + for (int j = 0; j < VEC; ++j) outp.val[j] = static_cast(acc[j]); + *reinterpret_cast(out + token * d + v * VEC) = outp; + } +} + +// Runtime-topk variant of the above. +template +__global__ void moe_sum_vec_dynamic_kernel( + scalar_t* __restrict__ out, // [num_tokens, d], contiguous + const scalar_t* __restrict__ input, // [num_tokens, topk, d], d contiguous + const int64_t num_tokens, const int d, const int topk, + const int64_t stride_token, const int64_t stride_topk) { + using vec_t = vllm::vec_n_t>; + constexpr int VEC = MOE_SUM_VEC; + const int64_t n_vec = d / VEC; + const int64_t total = num_tokens * n_vec; + for (int64_t i = blockIdx.x * blockDim.x + threadIdx.x; i < total; + i += (int64_t)gridDim.x * blockDim.x) { + const int64_t token = i / n_vec; + const int64_t v = i % n_vec; + const scalar_t* in_tok = input + token * stride_token + v * VEC; + + float acc[VEC]; +#pragma unroll + for (int j = 0; j < VEC; ++j) acc[j] = 0.f; + + for (int k = 0; k < topk; ++k) { + vec_t packed = *reinterpret_cast(in_tok + k * stride_topk); +#pragma unroll + for (int j = 0; j < VEC; ++j) acc[j] += static_cast(packed.val[j]); + } + + vec_t outp; +#pragma unroll + for (int j = 0; j < VEC; ++j) outp.val[j] = static_cast(acc[j]); + *reinterpret_cast(out + token * d + v * VEC) = outp; + } +} + +// Stride-aware scalar fallback: handles unaligned/non-vectorizable hidden dims +// (including a non-contiguous hidden stride) via per-element strided reads. +template +__global__ void moe_sum_scalar_kernel( + scalar_t* __restrict__ out, // [num_tokens, d], contiguous + const scalar_t* __restrict__ input, // [num_tokens, topk, d] + const int d, const int topk, const int64_t stride_token, + const int64_t stride_topk, const int64_t stride_hidden) { + const int64_t token_idx = blockIdx.x; + const scalar_t* in_tok = input + token_idx * stride_token; + for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) { + float x = 0.f; + for (int k = 0; k < topk; ++k) { + x += static_cast( + VLLM_LDG(&in_tok[k * stride_topk + idx * stride_hidden])); + } + out[token_idx * d + idx] = static_cast(x); } } @@ -626,52 +710,82 @@ void batched_moe_align_block_size(int64_t max_tokens_per_batch, void moe_sum(torch::stable::Tensor& input, // [num_tokens, topk, hidden_size] torch::stable::Tensor& output) // [num_tokens, hidden_size] { + // Output is dense and written in place, so it must be contiguous. The input + // is read by its strides (no copy); only the hidden dim needs to be + // contiguous to take the vectorized path. + STD_TORCH_CHECK(output.is_contiguous(), + "moe_sum expects a contiguous output"); + const int hidden_size = input.size(-1); - const auto num_tokens = output.numel() / hidden_size; + const int64_t num_tokens = output.numel() / hidden_size; const int topk = input.size(1); + const int64_t stride_token = input.stride(0); + const int64_t stride_topk = input.stride(1); + const int64_t stride_hidden = input.stride(2); - dim3 grid(num_tokens); - dim3 block(std::min(hidden_size, 1024)); const torch::stable::accelerator::DeviceGuard device_guard( output.get_device_index()); const cudaStream_t stream = get_current_cuda_stream(output.get_device_index()); - switch (topk) { - case 2: - VLLM_STABLE_DISPATCH_FLOATING_TYPES( - input.scalar_type(), "moe_sum_kernel", [&] { - vllm::moe::moe_sum_kernel<<>>( - reinterpret_cast(output.mutable_data_ptr()), - reinterpret_cast(input.const_data_ptr()), - hidden_size); - }); - break; +#define LAUNCH_MOE_SUM_VEC(TOPK) \ + vllm::moe::moe_sum_vec_kernel \ + <<>>( \ + out_ptr, in_ptr, num_tokens, hidden_size, stride_token, stride_topk) - case 3: - VLLM_STABLE_DISPATCH_FLOATING_TYPES( - input.scalar_type(), "moe_sum_kernel", [&] { - vllm::moe::moe_sum_kernel<<>>( - reinterpret_cast(output.mutable_data_ptr()), - reinterpret_cast(input.const_data_ptr()), - hidden_size); - }); - break; + VLLM_STABLE_DISPATCH_FLOATING_TYPES(input.scalar_type(), "moe_sum", [&] { + constexpr int VEC = vllm::moe::MOE_SUM_VEC; + constexpr int WIDTH = VEC * sizeof(scalar_t); // 16 bytes + auto* out_ptr = reinterpret_cast(output.mutable_data_ptr()); + auto* in_ptr = reinterpret_cast(input.const_data_ptr()); - case 4: - VLLM_STABLE_DISPATCH_FLOATING_TYPES( - input.scalar_type(), "moe_sum_kernel", [&] { - vllm::moe::moe_sum_kernel<<>>( - reinterpret_cast(output.mutable_data_ptr()), - reinterpret_cast(input.const_data_ptr()), - hidden_size); - }); - break; - - default: - torch::stable::sum_out(output, input, std::array{1}); - break; - } + // Vectorize along hidden only when it is contiguous (innermost stride 1), + // a whole number of vectors, and every row offset stays 16B-aligned. + const bool can_vec = (stride_hidden == 1) && (hidden_size % VEC == 0) && + (stride_token % VEC == 0) && + (stride_topk % VEC == 0) && + (reinterpret_cast(in_ptr) % WIDTH == 0) && + (reinterpret_cast(out_ptr) % WIDTH == 0); + if (can_vec) { + const int64_t n_vec = hidden_size / VEC; + const int64_t total = num_tokens * n_vec; + const int block = 256; + const dim3 grid(std::min((total + block - 1) / block, 65535)); + switch (topk) { + case 1: + LAUNCH_MOE_SUM_VEC(1); + break; + case 2: + LAUNCH_MOE_SUM_VEC(2); + break; + case 4: + LAUNCH_MOE_SUM_VEC(4); + break; + case 6: + LAUNCH_MOE_SUM_VEC(6); + break; + case 8: + LAUNCH_MOE_SUM_VEC(8); + break; + case 9: + LAUNCH_MOE_SUM_VEC(9); + break; + default: + vllm::moe::moe_sum_vec_dynamic_kernel + <<>>(out_ptr, in_ptr, num_tokens, + hidden_size, topk, + stride_token, stride_topk); + break; + } + } else { + dim3 grid(num_tokens); + dim3 block(std::min(hidden_size, 1024)); + vllm::moe::moe_sum_scalar_kernel<<>>( + out_ptr, in_ptr, hidden_size, topk, stride_token, stride_topk, + stride_hidden); + } + }); +#undef LAUNCH_MOE_SUM_VEC } void moe_lora_align_block_size( diff --git a/tests/kernels/moe/test_moe.py b/tests/kernels/moe/test_moe.py index 45cd17b3b1..f8b98c82a2 100644 --- a/tests/kernels/moe/test_moe.py +++ b/tests/kernels/moe/test_moe.py @@ -1243,15 +1243,27 @@ def test_batched_moe_align_block_size_opcheck(): ) +# topk=8 covers topk > 4; k=511 covers the non-vectorized scalar path. The +# layouts exercise contiguous input plus the two non-contiguous cases: a +# transpose (strided hidden -> scalar gather) and a topk-slice (hidden still +# contiguous -> vectorized). @pytest.mark.parametrize("m", [1, 33, 222]) -@pytest.mark.parametrize("topk", TOP_KS) +@pytest.mark.parametrize("topk", [*TOP_KS, 8]) @pytest.mark.parametrize("k", [128, 511, 1024]) @pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) -def test_moe_sum(m: int, topk: int, k: int, dtype: torch.dtype): - input = torch.randn((m, topk, k), device="cuda", dtype=dtype) +@pytest.mark.parametrize("layout", ["contig", "transpose", "slice"]) +def test_moe_sum(m: int, topk: int, k: int, dtype: torch.dtype, layout: str): + if layout == "transpose": + input = torch.randn((m, k, topk), device="cuda", dtype=dtype).transpose(1, 2) + elif layout == "slice": + input = torch.randn((m, 2 * topk, k), device="cuda", dtype=dtype)[:, ::2, :] + else: + input = torch.randn((m, topk, k), device="cuda", dtype=dtype) + assert input.is_contiguous() == (layout == "contig") actual = torch.empty((m, k), device="cuda", dtype=dtype) - expected = input.sum(dim=1) + # Reduction accumulates in fp32. + expected = input.float().sum(dim=1).to(dtype) torch.ops._moe_C.moe_sum(input, actual) torch.testing.assert_close(actual, expected, atol=2e-2, rtol=0)