[Kernel][MoE] Optimize batched_moe_align_block_size with cooperative writes (#53280)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -5,6 +5,7 @@ import itertools
|
||||
|
||||
import torch
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
|
||||
moe_align_block_size,
|
||||
)
|
||||
@@ -68,8 +69,87 @@ def benchmark(num_tokens, num_experts, topk, ep_size, provider):
|
||||
return 1000 * ms, 1000 * max_ms, 1000 * min_ms
|
||||
|
||||
|
||||
batched_configs = list(
|
||||
itertools.product(
|
||||
[8, 32, 64, 256],
|
||||
[16, 256, 512, 2048],
|
||||
[8, 32, 48, 64],
|
||||
["uniform", "skewed", "mostly_empty"],
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@triton.testing.perf_report(
|
||||
triton.testing.Benchmark(
|
||||
x_names=[
|
||||
"num_batches",
|
||||
"max_tokens_per_batch",
|
||||
"block_size",
|
||||
"distribution",
|
||||
],
|
||||
x_vals=batched_configs,
|
||||
line_arg="provider",
|
||||
line_vals=["vllm"],
|
||||
line_names=["vLLM"],
|
||||
plot_name="batched-moe-align-block-size-performance",
|
||||
args={},
|
||||
)
|
||||
)
|
||||
def benchmark_batched(
|
||||
num_batches,
|
||||
max_tokens_per_batch,
|
||||
block_size,
|
||||
distribution,
|
||||
provider,
|
||||
):
|
||||
set_random_seed(0)
|
||||
if distribution == "uniform":
|
||||
expert_num_tokens = torch.full(
|
||||
(num_batches,),
|
||||
max_tokens_per_batch,
|
||||
dtype=torch.int32,
|
||||
device="cuda",
|
||||
)
|
||||
elif distribution == "skewed":
|
||||
expert_num_tokens = torch.ones((num_batches,), dtype=torch.int32, device="cuda")
|
||||
expert_num_tokens[0] = max_tokens_per_batch
|
||||
else:
|
||||
expert_num_tokens = torch.zeros(
|
||||
(num_batches,), dtype=torch.int32, device="cuda"
|
||||
)
|
||||
expert_num_tokens[::8] = max_tokens_per_batch
|
||||
|
||||
max_num_tokens_padded = (
|
||||
num_batches
|
||||
* ((max_tokens_per_batch + block_size - 1) // block_size)
|
||||
* block_size
|
||||
)
|
||||
sorted_ids = torch.empty((max_num_tokens_padded,), dtype=torch.int32, device="cuda")
|
||||
expert_ids = torch.empty(
|
||||
(max_num_tokens_padded // block_size,), dtype=torch.int32, device="cuda"
|
||||
)
|
||||
num_tokens_post_pad = torch.empty((1,), dtype=torch.int32, device="cuda")
|
||||
|
||||
quantiles = [0.5, 0.2, 0.8]
|
||||
if provider == "vllm":
|
||||
ms, min_ms, max_ms = triton.testing.do_bench(
|
||||
lambda: ops.batched_moe_align_block_size(
|
||||
max_tokens_per_batch,
|
||||
block_size,
|
||||
expert_num_tokens,
|
||||
sorted_ids,
|
||||
expert_ids,
|
||||
num_tokens_post_pad,
|
||||
),
|
||||
quantiles=quantiles,
|
||||
)
|
||||
|
||||
return 1000 * ms, 1000 * max_ms, 1000 * min_ms
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--batched", action="store_true")
|
||||
parser.add_argument(
|
||||
"--num_experts",
|
||||
type=int,
|
||||
@@ -85,4 +165,5 @@ if __name__ == "__main__":
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
benchmark.run(print_data=True, show_plots=True)
|
||||
selected_benchmark = benchmark_batched if args.batched else benchmark
|
||||
selected_benchmark.run(print_data=True, show_plots=True)
|
||||
|
||||
@@ -23,20 +23,18 @@ namespace batched_moe_align_block_size {
|
||||
// Note num_threads needs to be 1024 for BlockScan Reduction in the kernel.
|
||||
static constexpr int32_t num_threads = 1024;
|
||||
static constexpr int32_t num_blocks = 1;
|
||||
template <bool cooperative_writes>
|
||||
__global__ void batched_moe_align_block_size_kernel(
|
||||
int32_t const num_batches, int32_t const max_tokens_per_batch,
|
||||
int32_t const block_size, int32_t const* __restrict__ batch_num_tokens,
|
||||
int32_t* __restrict__ sorted_ids, int32_t* __restrict__ block_ids,
|
||||
int32_t* __restrict__ num_tokens_post_pad) {
|
||||
// TODO(varun): This is a naive implementation. Could be optimized.
|
||||
|
||||
size_t const batch_id = threadIdx.x;
|
||||
size_t const stride = blockDim.x * gridDim.x;
|
||||
int32_t const num_blocks_per_batch =
|
||||
CEILDIV(max_tokens_per_batch, block_size);
|
||||
int32_t const sorted_ids_size =
|
||||
num_blocks_per_batch * num_batches * block_size;
|
||||
int32_t const block_ids_size = sorted_ids_size / block_size;
|
||||
size_t const sorted_ids_size =
|
||||
static_cast<size_t>(num_blocks_per_batch) * num_batches * block_size;
|
||||
size_t const block_ids_size = sorted_ids_size / block_size;
|
||||
int32_t const SENTINEL =
|
||||
num_batches * max_tokens_per_batch; // To denote invalid entries.
|
||||
// Initialize sorted_ids
|
||||
@@ -49,8 +47,8 @@ __global__ void batched_moe_align_block_size_kernel(
|
||||
}
|
||||
|
||||
int32_t b_num_tokens = 0;
|
||||
if (batch_id < num_batches) {
|
||||
b_num_tokens = batch_num_tokens[batch_id];
|
||||
if (threadIdx.x < num_batches) {
|
||||
b_num_tokens = batch_num_tokens[threadIdx.x];
|
||||
}
|
||||
int32_t const ceil_b_num_tokens =
|
||||
CEILDIV(b_num_tokens, block_size) * block_size;
|
||||
@@ -62,20 +60,56 @@ __global__ void batched_moe_align_block_size_kernel(
|
||||
BlockScan(temp_storage).ExclusiveSum(ceil_b_num_tokens, cumsum_val);
|
||||
__syncthreads();
|
||||
|
||||
bool const is_last_batch = batch_id == (num_batches - 1);
|
||||
if (is_last_batch) {
|
||||
if (threadIdx.x == num_batches - 1) {
|
||||
*num_tokens_post_pad = cumsum_val + ceil_b_num_tokens;
|
||||
}
|
||||
|
||||
if (batch_id < num_batches) {
|
||||
int32_t const batch_offset = batch_id * max_tokens_per_batch;
|
||||
if constexpr (cooperative_writes) {
|
||||
__shared__ int32_t batch_cumsum[num_threads];
|
||||
__shared__ int32_t valid_tokens[num_threads];
|
||||
__shared__ int32_t batch_num_blocks[num_threads];
|
||||
if (threadIdx.x < num_batches) {
|
||||
batch_cumsum[threadIdx.x] = cumsum_val;
|
||||
valid_tokens[threadIdx.x] = b_num_tokens;
|
||||
batch_num_blocks[threadIdx.x] = ceil_b_num_tokens / block_size;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
int32_t const max_num_groups = blockDim.x / WARP_SIZE;
|
||||
int32_t num_groups = 1;
|
||||
while (num_groups < num_batches && num_groups < max_num_groups) {
|
||||
num_groups *= 2;
|
||||
}
|
||||
int32_t const threads_per_batch = blockDim.x / num_groups;
|
||||
int32_t const group_id = threadIdx.x / threads_per_batch;
|
||||
int32_t const group_offset = threadIdx.x % threads_per_batch;
|
||||
|
||||
// Assign at least one warp to each batch when possible.
|
||||
for (int32_t batch_id = group_id; batch_id < num_batches;
|
||||
batch_id += num_groups) {
|
||||
size_t const batch_offset =
|
||||
static_cast<size_t>(batch_id) * max_tokens_per_batch;
|
||||
size_t const cumsum = batch_cumsum[batch_id];
|
||||
for (size_t i = group_offset; i < valid_tokens[batch_id];
|
||||
i += threads_per_batch) {
|
||||
sorted_ids[cumsum + i] = static_cast<int32_t>(batch_offset + i);
|
||||
}
|
||||
|
||||
size_t const block_start = cumsum / block_size;
|
||||
for (size_t i = group_offset; i < batch_num_blocks[batch_id];
|
||||
i += threads_per_batch) {
|
||||
block_ids[block_start + i] = batch_id;
|
||||
}
|
||||
}
|
||||
} else if (threadIdx.x < num_batches) {
|
||||
size_t const batch_id = threadIdx.x;
|
||||
size_t const batch_offset = batch_id * max_tokens_per_batch;
|
||||
for (size_t i = 0; i < b_num_tokens; ++i) {
|
||||
sorted_ids[cumsum_val + i] = batch_offset + i;
|
||||
sorted_ids[cumsum_val + i] = static_cast<int32_t>(batch_offset + i);
|
||||
}
|
||||
|
||||
int32_t const block_start = cumsum_val / block_size;
|
||||
int32_t const num_blocks = ceil_b_num_tokens / block_size;
|
||||
for (size_t i = 0; i < num_blocks; ++i) {
|
||||
size_t const block_start = cumsum_val / block_size;
|
||||
for (size_t i = 0; i < ceil_b_num_tokens / block_size; ++i) {
|
||||
block_ids[block_start + i] = batch_id;
|
||||
}
|
||||
}
|
||||
@@ -747,8 +781,16 @@ void batched_moe_align_block_size(int64_t max_tokens_per_batch,
|
||||
STD_TORCH_CHECK(num_tokens_post_pad.size(0) == 1);
|
||||
STD_TORCH_CHECK(B <= batched_kernel::num_threads);
|
||||
|
||||
batched_kernel::batched_moe_align_block_size_kernel<<<
|
||||
batched_kernel::num_blocks, batched_kernel::num_threads, 0, stream>>>(
|
||||
// Avoid coordination overhead for small capacities or many batches.
|
||||
int64_t const cooperative_threshold = std::max<int64_t>(256, 8 * B);
|
||||
bool const use_cooperative_writes =
|
||||
max_tokens_per_batch >= cooperative_threshold;
|
||||
auto kernel =
|
||||
use_cooperative_writes
|
||||
? batched_kernel::batched_moe_align_block_size_kernel<true>
|
||||
: batched_kernel::batched_moe_align_block_size_kernel<false>;
|
||||
kernel<<<batched_kernel::num_blocks, batched_kernel::num_threads, 0,
|
||||
stream>>>(
|
||||
B, max_tokens_per_batch, block_size,
|
||||
reinterpret_cast<const int32_t*>(batch_num_tokens.const_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(sorted_ids.mutable_data_ptr()),
|
||||
|
||||
@@ -329,13 +329,15 @@ def test_moe_align_block_size_deterministic():
|
||||
|
||||
@pytest.mark.parametrize("max_tokens_per_batch", [13, 16, 512])
|
||||
@pytest.mark.parametrize("num_experts", [8, 16, 32, 64])
|
||||
@pytest.mark.parametrize("block_size", [8, 16, 32, 64])
|
||||
@pytest.mark.parametrize("simulate_empty_batches", [False, True])
|
||||
@pytest.mark.parametrize("block_size", [8, 16, 32, 48, 64])
|
||||
@pytest.mark.parametrize(
|
||||
"token_distribution", ["random", "alternating_empty", "skewed"]
|
||||
)
|
||||
def test_batched_moe_align_block_size(
|
||||
max_tokens_per_batch: int,
|
||||
num_experts: int,
|
||||
block_size: int,
|
||||
simulate_empty_batches: bool,
|
||||
token_distribution: str,
|
||||
):
|
||||
def ref_outputs(
|
||||
expert_num_tokens: torch.Tensor,
|
||||
@@ -384,15 +386,17 @@ def test_batched_moe_align_block_size(
|
||||
# Compute expert_num_tokens
|
||||
expert_num_tokens = torch.randint(
|
||||
low=0,
|
||||
high=max_tokens_per_batch,
|
||||
high=max_tokens_per_batch + 1,
|
||||
size=(num_experts,),
|
||||
device="cpu",
|
||||
dtype=torch.int32,
|
||||
)
|
||||
if simulate_empty_batches:
|
||||
# mark half the batches to have 0 tokens
|
||||
zero_batches = torch.randperm(num_experts)[: num_experts // 2]
|
||||
expert_num_tokens[zero_batches] = 0
|
||||
if token_distribution == "alternating_empty":
|
||||
expert_num_tokens[::2] = 0
|
||||
elif token_distribution == "skewed":
|
||||
expert_num_tokens.zero_()
|
||||
expert_num_tokens[0] = max_tokens_per_batch
|
||||
expert_num_tokens[-1] = 1
|
||||
|
||||
# ref outputs
|
||||
ref_sorted_ids, ref_expert_ids, ref_num_tokens_post_pad = ref_outputs(
|
||||
|
||||
Reference in New Issue
Block a user