diff --git a/benchmarks/kernels/benchmark_moe_align_block_size.py b/benchmarks/kernels/benchmark_moe_align_block_size.py index a340500379..7ddf5b3ecf 100644 --- a/benchmarks/kernels/benchmark_moe_align_block_size.py +++ b/benchmarks/kernels/benchmark_moe_align_block_size.py @@ -5,6 +5,7 @@ import itertools import torch +import vllm._custom_ops as ops from vllm.model_executor.layers.fused_moe.moe_align_block_size import ( moe_align_block_size, ) @@ -68,8 +69,87 @@ def benchmark(num_tokens, num_experts, topk, ep_size, provider): return 1000 * ms, 1000 * max_ms, 1000 * min_ms +batched_configs = list( + itertools.product( + [8, 32, 64, 256], + [16, 256, 512, 2048], + [8, 32, 48, 64], + ["uniform", "skewed", "mostly_empty"], + ) +) + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=[ + "num_batches", + "max_tokens_per_batch", + "block_size", + "distribution", + ], + x_vals=batched_configs, + line_arg="provider", + line_vals=["vllm"], + line_names=["vLLM"], + plot_name="batched-moe-align-block-size-performance", + args={}, + ) +) +def benchmark_batched( + num_batches, + max_tokens_per_batch, + block_size, + distribution, + provider, +): + set_random_seed(0) + if distribution == "uniform": + expert_num_tokens = torch.full( + (num_batches,), + max_tokens_per_batch, + dtype=torch.int32, + device="cuda", + ) + elif distribution == "skewed": + expert_num_tokens = torch.ones((num_batches,), dtype=torch.int32, device="cuda") + expert_num_tokens[0] = max_tokens_per_batch + else: + expert_num_tokens = torch.zeros( + (num_batches,), dtype=torch.int32, device="cuda" + ) + expert_num_tokens[::8] = max_tokens_per_batch + + max_num_tokens_padded = ( + num_batches + * ((max_tokens_per_batch + block_size - 1) // block_size) + * block_size + ) + sorted_ids = torch.empty((max_num_tokens_padded,), dtype=torch.int32, device="cuda") + expert_ids = torch.empty( + (max_num_tokens_padded // block_size,), dtype=torch.int32, device="cuda" + ) + num_tokens_post_pad = torch.empty((1,), dtype=torch.int32, device="cuda") + + quantiles = [0.5, 0.2, 0.8] + if provider == "vllm": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: ops.batched_moe_align_block_size( + max_tokens_per_batch, + block_size, + expert_num_tokens, + sorted_ids, + expert_ids, + num_tokens_post_pad, + ), + quantiles=quantiles, + ) + + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + if __name__ == "__main__": parser = argparse.ArgumentParser() + parser.add_argument("--batched", action="store_true") parser.add_argument( "--num_experts", type=int, @@ -85,4 +165,5 @@ if __name__ == "__main__": ) args = parser.parse_args() - benchmark.run(print_data=True, show_plots=True) + selected_benchmark = benchmark_batched if args.batched else benchmark + selected_benchmark.run(print_data=True, show_plots=True) diff --git a/csrc/libtorch_stable/moe/moe_align_sum_kernels.cu b/csrc/libtorch_stable/moe/moe_align_sum_kernels.cu index 98a04de783..d66b78fe83 100644 --- a/csrc/libtorch_stable/moe/moe_align_sum_kernels.cu +++ b/csrc/libtorch_stable/moe/moe_align_sum_kernels.cu @@ -23,20 +23,18 @@ namespace batched_moe_align_block_size { // Note num_threads needs to be 1024 for BlockScan Reduction in the kernel. static constexpr int32_t num_threads = 1024; static constexpr int32_t num_blocks = 1; +template __global__ void batched_moe_align_block_size_kernel( int32_t const num_batches, int32_t const max_tokens_per_batch, int32_t const block_size, int32_t const* __restrict__ batch_num_tokens, int32_t* __restrict__ sorted_ids, int32_t* __restrict__ block_ids, int32_t* __restrict__ num_tokens_post_pad) { - // TODO(varun): This is a naive implementation. Could be optimized. - - size_t const batch_id = threadIdx.x; size_t const stride = blockDim.x * gridDim.x; int32_t const num_blocks_per_batch = CEILDIV(max_tokens_per_batch, block_size); - int32_t const sorted_ids_size = - num_blocks_per_batch * num_batches * block_size; - int32_t const block_ids_size = sorted_ids_size / block_size; + size_t const sorted_ids_size = + static_cast(num_blocks_per_batch) * num_batches * block_size; + size_t const block_ids_size = sorted_ids_size / block_size; int32_t const SENTINEL = num_batches * max_tokens_per_batch; // To denote invalid entries. // Initialize sorted_ids @@ -49,8 +47,8 @@ __global__ void batched_moe_align_block_size_kernel( } int32_t b_num_tokens = 0; - if (batch_id < num_batches) { - b_num_tokens = batch_num_tokens[batch_id]; + if (threadIdx.x < num_batches) { + b_num_tokens = batch_num_tokens[threadIdx.x]; } int32_t const ceil_b_num_tokens = CEILDIV(b_num_tokens, block_size) * block_size; @@ -62,20 +60,56 @@ __global__ void batched_moe_align_block_size_kernel( BlockScan(temp_storage).ExclusiveSum(ceil_b_num_tokens, cumsum_val); __syncthreads(); - bool const is_last_batch = batch_id == (num_batches - 1); - if (is_last_batch) { + if (threadIdx.x == num_batches - 1) { *num_tokens_post_pad = cumsum_val + ceil_b_num_tokens; } - if (batch_id < num_batches) { - int32_t const batch_offset = batch_id * max_tokens_per_batch; + if constexpr (cooperative_writes) { + __shared__ int32_t batch_cumsum[num_threads]; + __shared__ int32_t valid_tokens[num_threads]; + __shared__ int32_t batch_num_blocks[num_threads]; + if (threadIdx.x < num_batches) { + batch_cumsum[threadIdx.x] = cumsum_val; + valid_tokens[threadIdx.x] = b_num_tokens; + batch_num_blocks[threadIdx.x] = ceil_b_num_tokens / block_size; + } + __syncthreads(); + + int32_t const max_num_groups = blockDim.x / WARP_SIZE; + int32_t num_groups = 1; + while (num_groups < num_batches && num_groups < max_num_groups) { + num_groups *= 2; + } + int32_t const threads_per_batch = blockDim.x / num_groups; + int32_t const group_id = threadIdx.x / threads_per_batch; + int32_t const group_offset = threadIdx.x % threads_per_batch; + + // Assign at least one warp to each batch when possible. + for (int32_t batch_id = group_id; batch_id < num_batches; + batch_id += num_groups) { + size_t const batch_offset = + static_cast(batch_id) * max_tokens_per_batch; + size_t const cumsum = batch_cumsum[batch_id]; + for (size_t i = group_offset; i < valid_tokens[batch_id]; + i += threads_per_batch) { + sorted_ids[cumsum + i] = static_cast(batch_offset + i); + } + + size_t const block_start = cumsum / block_size; + for (size_t i = group_offset; i < batch_num_blocks[batch_id]; + i += threads_per_batch) { + block_ids[block_start + i] = batch_id; + } + } + } else if (threadIdx.x < num_batches) { + size_t const batch_id = threadIdx.x; + size_t const batch_offset = batch_id * max_tokens_per_batch; for (size_t i = 0; i < b_num_tokens; ++i) { - sorted_ids[cumsum_val + i] = batch_offset + i; + sorted_ids[cumsum_val + i] = static_cast(batch_offset + i); } - int32_t const block_start = cumsum_val / block_size; - int32_t const num_blocks = ceil_b_num_tokens / block_size; - for (size_t i = 0; i < num_blocks; ++i) { + size_t const block_start = cumsum_val / block_size; + for (size_t i = 0; i < ceil_b_num_tokens / block_size; ++i) { block_ids[block_start + i] = batch_id; } } @@ -747,8 +781,16 @@ void batched_moe_align_block_size(int64_t max_tokens_per_batch, STD_TORCH_CHECK(num_tokens_post_pad.size(0) == 1); STD_TORCH_CHECK(B <= batched_kernel::num_threads); - batched_kernel::batched_moe_align_block_size_kernel<<< - batched_kernel::num_blocks, batched_kernel::num_threads, 0, stream>>>( + // Avoid coordination overhead for small capacities or many batches. + int64_t const cooperative_threshold = std::max(256, 8 * B); + bool const use_cooperative_writes = + max_tokens_per_batch >= cooperative_threshold; + auto kernel = + use_cooperative_writes + ? batched_kernel::batched_moe_align_block_size_kernel + : batched_kernel::batched_moe_align_block_size_kernel; + kernel<<>>( B, max_tokens_per_batch, block_size, reinterpret_cast(batch_num_tokens.const_data_ptr()), reinterpret_cast(sorted_ids.mutable_data_ptr()), diff --git a/tests/kernels/moe/test_moe_align_block_size.py b/tests/kernels/moe/test_moe_align_block_size.py index 7d07af4080..8c16690785 100644 --- a/tests/kernels/moe/test_moe_align_block_size.py +++ b/tests/kernels/moe/test_moe_align_block_size.py @@ -329,13 +329,15 @@ def test_moe_align_block_size_deterministic(): @pytest.mark.parametrize("max_tokens_per_batch", [13, 16, 512]) @pytest.mark.parametrize("num_experts", [8, 16, 32, 64]) -@pytest.mark.parametrize("block_size", [8, 16, 32, 64]) -@pytest.mark.parametrize("simulate_empty_batches", [False, True]) +@pytest.mark.parametrize("block_size", [8, 16, 32, 48, 64]) +@pytest.mark.parametrize( + "token_distribution", ["random", "alternating_empty", "skewed"] +) def test_batched_moe_align_block_size( max_tokens_per_batch: int, num_experts: int, block_size: int, - simulate_empty_batches: bool, + token_distribution: str, ): def ref_outputs( expert_num_tokens: torch.Tensor, @@ -384,15 +386,17 @@ def test_batched_moe_align_block_size( # Compute expert_num_tokens expert_num_tokens = torch.randint( low=0, - high=max_tokens_per_batch, + high=max_tokens_per_batch + 1, size=(num_experts,), device="cpu", dtype=torch.int32, ) - if simulate_empty_batches: - # mark half the batches to have 0 tokens - zero_batches = torch.randperm(num_experts)[: num_experts // 2] - expert_num_tokens[zero_batches] = 0 + if token_distribution == "alternating_empty": + expert_num_tokens[::2] = 0 + elif token_distribution == "skewed": + expert_num_tokens.zero_() + expert_num_tokens[0] = max_tokens_per_batch + expert_num_tokens[-1] = 1 # ref outputs ref_sorted_ids, ref_expert_ids, ref_num_tokens_post_pad = ref_outputs(