From 82b7d49a6e30cf6d1d53838667ee20f09ac1f08b Mon Sep 17 00:00:00 2001 From: Misha Goin Date: Tue, 1 Sep 2026 11:34:13 -0400 Subject: [PATCH] [MoE] Generalize masked activation for padded layouts (#51217) Signed-off-by: mgoin Co-authored-by: OpenAI Codex --- csrc/libtorch_stable/activation_kernels.cu | 345 +++++++++++++++--- .../moe_permute_unpermute_kernel.cu | 15 +- csrc/libtorch_stable/ops.h | 15 +- csrc/libtorch_stable/torch_bindings.cpp | 10 +- tests/kernels/core/test_activation.py | 208 +++++++++++ tests/kernels/moe/test_batched_moe.py | 148 +++++++- tests/kernels/moe/test_moe.py | 261 ++++++++++++- .../kernels/moe/test_moe_permute_unpermute.py | 36 ++ tests/kernels/moe/test_situ_mul_fp8_quant.py | 14 +- tests/kernels/moe/utils.py | 4 +- .../layers/fused_moe/activation.py | 161 ++++++-- .../fused_moe/experts/fused_batched_moe.py | 60 ++- .../fused_moe/experts/fused_humming_moe.py | 103 +++--- .../layers/fused_moe/experts/marlin_moe.py | 29 +- .../layers/fused_moe/modular_kernel.py | 4 +- 15 files changed, 1217 insertions(+), 196 deletions(-) diff --git a/csrc/libtorch_stable/activation_kernels.cu b/csrc/libtorch_stable/activation_kernels.cu index 56d0058dfc..a7ee87b35f 100644 --- a/csrc/libtorch_stable/activation_kernels.cu +++ b/csrc/libtorch_stable/activation_kernels.cu @@ -500,10 +500,8 @@ template __global__ void situ_and_mul_kernel( scalar_t* __restrict__ out, // [..., d] const scalar_t* __restrict__ input, // [..., 2, d] - const int d, const float beta, const float linear_beta, - const int64_t* __restrict__ valid_rows_ptr) { + const int d, const float beta, const float linear_beta) { const int64_t row = blockIdx.x; - if (valid_rows_ptr != nullptr && row >= *valid_rows_ptr) return; const scalar_t* gate_ptr = input + row * 2 * d; const scalar_t* up_ptr = gate_ptr + d; scalar_t* out_ptr = out + row * d; @@ -558,15 +556,16 @@ template 0.0f; @@ -768,24 +768,61 @@ __global__ void situ_and_mul_quant_group_scalar_kernel( } } -template +constexpr int kMaxMaskedTokenBlocks = 32; + +template +__device__ __forceinline__ bool get_masked_row_range( + const int* __restrict__ valid_token_counts, const int max_num_tokens, + int64_t& first_row, int64_t& end_row, int64_t& row_stride) { + if constexpr (BATCHED_EXPERTS) { + // [E, T, *]: z lanes grid-stride over one expert's valid token prefix. + const int expert = blockIdx.y; + const int num_tokens = + max(0, min(valid_token_counts[expert], max_num_tokens)); + const int token_block = blockIdx.z; + if (token_block >= num_tokens) { + return false; + } + const int64_t expert_first_row = + static_cast(expert) * max_num_tokens; + first_row = expert_first_row + token_block; + end_row = expert_first_row + num_tokens; + row_stride = gridDim.z; + } else { + // [T, *]: y lanes grid-stride over the single valid token prefix. + const int num_tokens = max(0, min(valid_token_counts[0], max_num_tokens)); + const int token_block = blockIdx.y; + if (token_block >= num_tokens) { + return false; + } + first_row = token_block; + end_row = num_tokens; + row_stride = gridDim.y; + } + return first_row < end_row; +} + +__device__ __forceinline__ int get_masked_feature_index() { + return blockIdx.x * blockDim.x + threadIdx.x; +} + +template __global__ void masked_situ_and_mul_kernel( scalar_t* __restrict__ out, const scalar_t* __restrict__ input, - const int* __restrict__ expert_num_tokens, const int max_num_tokens, + const int* __restrict__ valid_token_counts, const int max_num_tokens, const int d, const float beta, const float linear_beta) { - const int expert = blockIdx.y; - const int num_tokens = expert_num_tokens[expert]; - const int idx = blockIdx.x * blockDim.x + threadIdx.x; - if (idx >= d || num_tokens == 0) { + int64_t first_row, end_row, row_stride; + const int idx = get_masked_feature_index(); + if (idx >= d || + !get_masked_row_range(valid_token_counts, max_num_tokens, + first_row, end_row, row_stride)) { return; } const bool clamp_up = linear_beta > 0.0f; const float inv_beta = 1.0f / beta; const float inv_linear_beta = clamp_up ? 1.0f / linear_beta : 0.0f; - const int64_t expert_row = static_cast(expert) * max_num_tokens; - for (int token = 0; token < num_tokens; ++token) { - const int64_t row = expert_row + token; + for (int64_t row = first_row; row < end_row; row += row_stride) { const scalar_t* gate_ptr = input + row * 2 * d; const scalar_t* up_ptr = gate_ptr + d; scalar_t* out_ptr = out + row * d; @@ -796,6 +833,119 @@ __global__ void masked_situ_and_mul_kernel( } } +template +__global__ void masked_act_and_mul_kernel( + scalar_t* __restrict__ out, const scalar_t* __restrict__ input, + const int* __restrict__ valid_token_counts, const int max_num_tokens, + const int d, const float limit, const float alpha, const float beta) { + int64_t first_row, end_row, row_stride; + const int idx = get_masked_feature_index(); + if (idx >= d || + !get_masked_row_range(valid_token_counts, max_num_tokens, + first_row, end_row, row_stride)) { + return; + } + + for (int64_t row = first_row; row < end_row; row += row_stride) { + const scalar_t* x_ptr = input + row * 2 * d; + const scalar_t* y_ptr = x_ptr + d; + scalar_t* out_ptr = out + row * d; + const scalar_t x = VLLM_LDG(&x_ptr[idx]); + const scalar_t y = VLLM_LDG(&y_ptr[idx]); + out_ptr[idx] = compute(x, y, limit, + alpha, beta); + } +} + +template +__global__ void masked_swigluoai_and_mul_kernel( + scalar_t* __restrict__ out, const scalar_t* __restrict__ input, + const int* __restrict__ valid_token_counts, const int max_num_tokens, + const int d, const float alpha, const float limit) { + int64_t first_row, end_row, row_stride; + const int idx = get_masked_feature_index(); + if (idx >= d || + !get_masked_row_range(valid_token_counts, max_num_tokens, + first_row, end_row, row_stride)) { + return; + } + + for (int64_t row = first_row; row < end_row; row += row_stride) { + const scalar_t* in_ptr = input + row * 2 * d; + scalar_t* out_ptr = out + row * d; + out_ptr[idx] = + swigluoai_and_mul(in_ptr[2 * idx], in_ptr[2 * idx + 1], alpha, limit); + } +} + +template +__global__ void masked_swiglustep_and_mul_kernel( + scalar_t* __restrict__ out, const scalar_t* __restrict__ input, + const int* __restrict__ valid_token_counts, const int max_num_tokens, + const int d, const float limit) { + int64_t first_row, end_row, row_stride; + const int idx = get_masked_feature_index(); + if (idx >= d || + !get_masked_row_range(valid_token_counts, max_num_tokens, + first_row, end_row, row_stride)) { + return; + } + + for (int64_t row = first_row; row < end_row; row += row_stride) { + const scalar_t* gate_ptr = input + row * 2 * d; + const scalar_t* up_ptr = gate_ptr + d; + scalar_t* out_ptr = out + row * d; + const float gate = (float)VLLM_LDG(&gate_ptr[idx]); + const float up = (float)VLLM_LDG(&up_ptr[idx]); + const float gate_silu = gate / (1.0f + expf(-gate)); + const float gate_clamped = fminf(gate_silu, limit); + const float up_clamped = fmaxf(fminf(up, limit), -limit); + out_ptr[idx] = (scalar_t)(gate_clamped * up_clamped); + } +} + +template +__device__ __forceinline__ T silu_no_mul_kernel(const T& x) { + return silu_kernel(x, 1.0f); +} + +template +__device__ __forceinline__ T gelu_no_mul_kernel(const T& x) { + return gelu_kernel(x, 1.0f); +} + +template +__device__ __forceinline__ T gelu_tanh_no_mul_kernel(const T& x) { + return gelu_tanh_kernel(x, 1.0f); +} + +template +__device__ __forceinline__ T relu2_no_mul_kernel(const T& x) { + const float value = fmaxf((float)x, 0.0f); + return (T)(value * value); +} + +template +__global__ void masked_activation_kernel( + scalar_t* __restrict__ out, const scalar_t* __restrict__ input, + const int* __restrict__ valid_token_counts, const int max_num_tokens, + const int d) { + int64_t first_row, end_row, row_stride; + const int idx = get_masked_feature_index(); + if (idx >= d || + !get_masked_row_range(valid_token_counts, max_num_tokens, + first_row, end_row, row_stride)) { + return; + } + + for (int64_t row = first_row; row < end_row; row += row_stride) { + const int64_t offset = row * d + idx; + out[offset] = ACT_FN(VLLM_LDG(&input[offset])); + } +} + } // namespace vllm #define LAUNCH_ACTIVATION_GATE_KERNEL_WITH_PARAM(KERNEL, PACKED_KERNEL, PARAM) \ @@ -890,8 +1040,7 @@ void swigluoai_and_mul(torch::stable::Tensor& out, // [..., d] // through), matching SituAndMul(linear_beta=None) on the Python side. void situ_and_mul(torch::stable::Tensor& out, // [..., d] torch::stable::Tensor& input, // [..., 2 * d] - double beta, double linear_beta, - std::optional valid_rows) { + double beta, double linear_beta) { int d = input.size(-1) / 2; int64_t num_tokens = input.numel() / input.size(-1); if (num_tokens == 0) { @@ -899,14 +1048,6 @@ void situ_and_mul(torch::stable::Tensor& out, // [..., d] } dim3 grid(num_tokens); dim3 block(std::min(d, 1024)); - const int64_t* valid_rows_ptr = nullptr; - if (valid_rows.has_value()) { - STD_TORCH_CHECK(valid_rows->is_cuda() && valid_rows->get_device_index() == - input.get_device_index(), - "situ_and_mul: valid_rows must be a CUDA tensor on the " - "same device as input"); - valid_rows_ptr = valid_rows->const_data_ptr(); - } const torch::stable::accelerator::DeviceGuard device_guard( input.get_device_index()); const cudaStream_t stream = get_current_cuda_stream(); @@ -914,7 +1055,7 @@ void situ_and_mul(torch::stable::Tensor& out, // [..., d] input.scalar_type(), "situ_and_mul_kernel", [&] { vllm::situ_and_mul_kernel<<>>( out.mutable_data_ptr(), input.const_data_ptr(), - d, (float)beta, (float)linear_beta, valid_rows_ptr); + d, (float)beta, (float)linear_beta); }); } @@ -925,7 +1066,7 @@ void masked_situ_and_mul(torch::stable::Tensor& out, // [E, T, d] int num_experts = input.size(0); int max_num_tokens = input.size(1); int d = input.size(2) / 2; - if (num_experts == 0 || max_num_tokens == 0) { + if (num_experts == 0 || max_num_tokens == 0 || d == 0) { return; } constexpr int block_size = 256; @@ -936,21 +1077,128 @@ void masked_situ_and_mul(torch::stable::Tensor& out, // [E, T, d] const cudaStream_t stream = get_current_cuda_stream(); VLLM_STABLE_DISPATCH_FLOATING_TYPES( input.scalar_type(), "masked_situ_and_mul_kernel", [&] { - vllm::masked_situ_and_mul_kernel<<>>( - out.mutable_data_ptr(), input.const_data_ptr(), - expert_num_tokens.const_data_ptr(), max_num_tokens, d, - (float)beta, (float)linear_beta); + vllm::masked_situ_and_mul_kernel + <<>>( + out.mutable_data_ptr(), + input.const_data_ptr(), + expert_num_tokens.const_data_ptr(), max_num_tokens, d, + (float)beta, (float)linear_beta); }); } +#define LAUNCH_MASKED_ACT_AND_MUL(KERNEL, ACT_FIRST, HAS_CLAMP, LIMIT, ALPHA, \ + BETA) \ + vllm::masked_act_and_mul_kernel, ACT_FIRST, \ + HAS_CLAMP, BATCHED_EXPERTS> \ + <<>>(out.mutable_data_ptr(), \ + input.const_data_ptr(), \ + valid_token_counts.const_data_ptr(), \ + max_num_tokens, d, LIMIT, ALPHA, BETA) + +#define LAUNCH_MASKED_ACTIVATION(KERNEL) \ + vllm::masked_activation_kernel, BATCHED_EXPERTS> \ + <<>>( \ + out.mutable_data_ptr(), input.const_data_ptr(), \ + valid_token_counts.const_data_ptr(), max_num_tokens, d) + +void masked_moe_activation( + torch::stable::Tensor& out, // [T, d] or [E, T, d] + torch::stable::Tensor& input, // [T, d] or [E, T, d], gated has 2 * d + const torch::stable::Tensor& valid_token_counts, + const std::string& activation, double clamp_limit, double alpha, + double beta, double situ_beta, double situ_linear_beta) { + const bool batched_experts = input.dim() == 3; + const int num_experts = batched_experts ? input.size(0) : 1; + const int max_num_tokens = batched_experts ? input.size(1) : input.size(0); + const int d = out.size(-1); + if (num_experts == 0 || max_num_tokens == 0 || d == 0) { + return; + } + + constexpr int block_size = 256; + const int feature_blocks = (d + block_size - 1) / block_size; + const int token_blocks = + std::min(max_num_tokens, vllm::kMaxMaskedTokenBlocks); + // Batched grid: (feature tile, expert, token lane); flat grid: (feature + // tile, token lane). Token lanes grid-stride each valid prefix. + dim3 grid = batched_experts ? dim3(feature_blocks, num_experts, token_blocks) + : dim3(feature_blocks, token_blocks); + dim3 block(block_size); + const torch::stable::accelerator::DeviceGuard device_guard( + input.get_device_index()); + const cudaStream_t stream = get_current_cuda_stream(); + auto launch = [&]() { + VLLM_STABLE_DISPATCH_FLOATING_TYPES( + input.scalar_type(), "masked_moe_activation", [&] { + if (activation == "silu") { + LAUNCH_MASKED_ACT_AND_MUL(vllm::silu_kernel, true, false, 0.0f, + 1.0f, 0.0f); + } else if (activation == "silu_with_clamp") { + LAUNCH_MASKED_ACT_AND_MUL(vllm::silu_kernel, true, true, + (float)clamp_limit, (float)alpha, + (float)beta); + } else if (activation == "gelu") { + LAUNCH_MASKED_ACT_AND_MUL(vllm::gelu_kernel, true, false, 0.0f, + 1.0f, 0.0f); + } else if (activation == "gelu_tanh") { + LAUNCH_MASKED_ACT_AND_MUL(vllm::gelu_tanh_kernel, true, false, 0.0f, + 1.0f, 0.0f); + } else if (activation == "situ") { + vllm::masked_situ_and_mul_kernel + <<>>( + out.mutable_data_ptr(), + input.const_data_ptr(), + valid_token_counts.const_data_ptr(), max_num_tokens, d, + (float)situ_beta, (float)situ_linear_beta); + } else if (activation == "swigluoai") { + vllm::masked_swigluoai_and_mul_kernel + <<>>( + out.mutable_data_ptr(), + input.const_data_ptr(), + valid_token_counts.const_data_ptr(), max_num_tokens, d, + (float)alpha, (float)clamp_limit); + } else if (activation == "swigluoai_uninterleave") { + LAUNCH_MASKED_ACT_AND_MUL(vllm::silu_kernel, true, true, + (float)clamp_limit, (float)alpha, + (float)beta); + } else if (activation == "swiglustep") { + vllm::masked_swiglustep_and_mul_kernel + <<>>( + out.mutable_data_ptr(), + input.const_data_ptr(), + valid_token_counts.const_data_ptr(), max_num_tokens, d, + (float)clamp_limit); + } else if (activation == "silu_no_mul") { + LAUNCH_MASKED_ACTIVATION(vllm::silu_no_mul_kernel); + } else if (activation == "gelu_no_mul") { + LAUNCH_MASKED_ACTIVATION(vllm::gelu_no_mul_kernel); + } else if (activation == "gelu_tanh_no_mul") { + LAUNCH_MASKED_ACTIVATION(vllm::gelu_tanh_no_mul_kernel); + } else if (activation == "relu2_no_mul") { + LAUNCH_MASKED_ACTIVATION(vllm::relu2_no_mul_kernel); + } else { + STD_TORCH_CHECK(false, + "Unsupported masked MoE activation: ", activation); + } + }); + }; + if (batched_experts) { + launch.template operator()(); + } else { + launch.template operator()(); + } +} + +#undef LAUNCH_MASKED_ACT_AND_MUL +#undef LAUNCH_MASKED_ACTIVATION // Fused SITU activation and block-FP8 quantization for Humming w2. -// `valid_rows` is the int32 DeepEP token count (psum[-1]); rows = it * `topk`, -// so padding is excluded on-device and skipped rows receive scale 1. +// `num_valid_tokens` is the int32 DeepEP token count (psum[-1]); rows = it * +// `topk`, so padding is excluded on-device and skipped rows receive scale 1. void situ_and_mul_quant(torch::stable::Tensor& out, // [..., d] (fp8) torch::stable::Tensor& scale, // [..., 1 or d/128] torch::stable::Tensor& input, // [..., 2 * d] double beta, double linear_beta, int64_t group_size, - std::optional valid_rows, + std::optional num_valid_tokens, int64_t topk) { STD_TORCH_CHECK( out.scalar_type() == torch::headeronly::ScalarType::Float8_e4m3fn || @@ -974,16 +1222,17 @@ void situ_and_mul_quant(torch::stable::Tensor& out, // [..., d] (fp8) } STD_TORCH_CHECK(out.size(-1) == d && out.numel() == num_tokens * (int64_t)d, "situ_and_mul_quant: out shape must be [num_tokens, d]"); - const int32_t* valid_rows_ptr = nullptr; - if (valid_rows.has_value()) { - STD_TORCH_CHECK(valid_rows->is_cuda() && valid_rows->get_device_index() == - input.get_device_index(), - "situ_and_mul_quant: valid_rows must be a CUDA tensor on " - "the same device as input"); + const int32_t* num_valid_tokens_ptr = nullptr; + if (num_valid_tokens.has_value()) { STD_TORCH_CHECK( - valid_rows->scalar_type() == torch::headeronly::ScalarType::Int, - "situ_and_mul_quant: valid_rows must be int32 (psum count)"); - valid_rows_ptr = valid_rows->const_data_ptr(); + num_valid_tokens->is_cuda() && + num_valid_tokens->get_device_index() == input.get_device_index(), + "situ_and_mul_quant: num_valid_tokens must be a CUDA tensor on the " + "same device as input"); + STD_TORCH_CHECK( + num_valid_tokens->scalar_type() == torch::headeronly::ScalarType::Int, + "situ_and_mul_quant: num_valid_tokens must be int32 (psum count)"); + num_valid_tokens_ptr = num_valid_tokens->const_data_ptr(); } const torch::stable::accelerator::DeviceGuard device_guard( input.get_device_index()); @@ -1031,7 +1280,7 @@ void situ_and_mul_quant(torch::stable::Tensor& out, // [..., d] (fp8) out.mutable_data_ptr(), scale.mutable_data_ptr(), input.const_data_ptr(), num_tokens, - valid_rows_ptr, topk); + num_valid_tokens_ptr, topk); return; } } @@ -1045,7 +1294,7 @@ void situ_and_mul_quant(torch::stable::Tensor& out, // [..., d] (fp8) scale.mutable_data_ptr(), input.const_data_ptr(), d, num_groups, (float)beta, (float)linear_beta, num_tokens, - valid_rows_ptr, topk); + num_valid_tokens_ptr, topk); }); }); } diff --git a/csrc/libtorch_stable/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.cu b/csrc/libtorch_stable/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.cu index f5ec32c390..ec7851da3d 100644 --- a/csrc/libtorch_stable/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.cu +++ b/csrc/libtorch_stable/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.cu @@ -146,16 +146,17 @@ __global__ void preprocessTopkIdKernel(int* topk_id_ptr, int size, } __syncthreads(); - // query global expert id in expert map. - // if global expert id = -1 in exert map, plus n_expert - // else set global expert id = exert map[global expert id] if (offset + tidx < bound) { auto topk_id = topk_id_ptr[offset + tidx]; - auto local_expert_idx = smem_expert_map[topk_id]; - if (local_expert_idx == -1) { - topk_id += num_experts; + if (topk_id < 0 || topk_id >= num_experts) { + topk_id = num_experts; } else { - topk_id = local_expert_idx; + auto local_expert_idx = smem_expert_map[topk_id]; + if (local_expert_idx == -1) { + topk_id += num_experts; + } else { + topk_id = local_expert_idx; + } } __syncwarp(); topk_id_ptr[offset + tidx] = topk_id; diff --git a/csrc/libtorch_stable/ops.h b/csrc/libtorch_stable/ops.h index 8ac50538db..1f36f7796c 100644 --- a/csrc/libtorch_stable/ops.h +++ b/csrc/libtorch_stable/ops.h @@ -530,20 +530,25 @@ void fatrelu_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input, double threshold); void swigluoai_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input, double alpha = 1.702, double limit = 7.0); -void situ_and_mul( - torch::stable::Tensor& out, torch::stable::Tensor& input, double beta = 1.0, - double linear_beta = -1.0, - std::optional valid_rows = std::nullopt); +void situ_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input, + double beta = 1.0, double linear_beta = -1.0); void situ_and_mul_quant( torch::stable::Tensor& out, torch::stable::Tensor& scale, torch::stable::Tensor& input, double beta = 1.0, double linear_beta = -1.0, int64_t group_size = 0, - std::optional valid_rows = std::nullopt, + std::optional num_valid_tokens = std::nullopt, int64_t topk = 1); void masked_situ_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input, const torch::stable::Tensor& expert_num_tokens, double beta = 1.0, double linear_beta = -1.0); +void masked_moe_activation(torch::stable::Tensor& out, + torch::stable::Tensor& input, + const torch::stable::Tensor& valid_token_counts, + const std::string& activation, + double clamp_limit = 0.0, double alpha = 1.0, + double beta = 0.0, double situ_beta = 1.0, + double situ_linear_beta = -1.0); void gelu_new(torch::stable::Tensor& out, torch::stable::Tensor& input); void gelu_fast(torch::stable::Tensor& out, torch::stable::Tensor& input); void gelu_quick(torch::stable::Tensor& out, torch::stable::Tensor& input); diff --git a/csrc/libtorch_stable/torch_bindings.cpp b/csrc/libtorch_stable/torch_bindings.cpp index 505ec6588a..e5fa535993 100644 --- a/csrc/libtorch_stable/torch_bindings.cpp +++ b/csrc/libtorch_stable/torch_bindings.cpp @@ -614,7 +614,7 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) { // SituGLU implementation used in Kimi models. ops.def( "situ_and_mul(Tensor! out, Tensor input, float beta=1.0, float " - "linear_beta=-1.0, Tensor? valid_rows=None) -> ()"); + "linear_beta=-1.0) -> ()"); // Fused SituGLU activation + dynamic FP8 quantization for the Humming w2 path // (writes the fp8 down input and its float32 scale). group_size=0 -> // per-token scale [.., 1]; group_size=128 -> k-major block-FP8 scale [.., @@ -622,10 +622,15 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) { ops.def( "situ_and_mul_quant(Tensor! out, Tensor! scale, Tensor input, " "float beta=1.0, float linear_beta=-1.0, int group_size=0, " - "Tensor? valid_rows=None, int topk=1) -> ()"); + "Tensor? num_valid_tokens=None, int topk=1) -> ()"); ops.def( "masked_situ_and_mul(Tensor! out, Tensor input, Tensor " "expert_num_tokens, float beta=1.0, float linear_beta=-1.0) -> ()"); + ops.def( + "masked_moe_activation(Tensor! out, Tensor input, Tensor " + "valid_token_counts, str activation, float clamp_limit=0.0, float " + "alpha=1.0, float beta=0.0, float situ_beta=1.0, float " + "situ_linear_beta=-1.0) -> ()"); // GELU implementation used in GPT-2. ops.def("gelu_new(Tensor! out, Tensor input) -> ()"); @@ -852,6 +857,7 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) { ops.impl("situ_and_mul", TORCH_BOX(&situ_and_mul)); ops.impl("situ_and_mul_quant", TORCH_BOX(&situ_and_mul_quant)); ops.impl("masked_situ_and_mul", TORCH_BOX(&masked_situ_and_mul)); + ops.impl("masked_moe_activation", TORCH_BOX(&masked_moe_activation)); ops.impl("gelu_new", TORCH_BOX(&gelu_new)); ops.impl("gelu_fast", TORCH_BOX(&gelu_fast)); ops.impl("gelu_quick", TORCH_BOX(&gelu_quick)); diff --git a/tests/kernels/core/test_activation.py b/tests/kernels/core/test_activation.py index f13deccafd..b1a52a5d55 100644 --- a/tests/kernels/core/test_activation.py +++ b/tests/kernels/core/test_activation.py @@ -22,7 +22,13 @@ from vllm.model_executor.layers.activation import ( SwigluStepAndMul, swiglustep_and_mul_triton, ) +from vllm.model_executor.layers.fused_moe.activation import ( + ApplyMoEActivationConfig, + MoEActivation, + apply_moe_activation, +) from vllm.model_executor.layers.fused_moe.utils import swiglu_limit_func +from vllm.platforms import current_platform from vllm.utils.torch_utils import set_random_seed DTYPES = [torch.half, torch.bfloat16, torch.float] @@ -34,6 +40,100 @@ CUDA_DEVICES = [ ] +def test_masked_moe_activation_rejects_unsupported_activation() -> None: + input = torch.empty(1, 1, 2) + output = torch.empty(1, 1, 1) + valid_token_counts = torch.ones(1, dtype=torch.int32) + + with pytest.raises(NotImplementedError, match="relu2"): + apply_moe_activation( + MoEActivation.RELU2, + output, + input, + valid_token_counts=valid_token_counts, + ) + + +def test_moe_silu_clamp_uses_native_xpu_fallback( + default_vllm_config, monkeypatch +) -> None: + monkeypatch.setattr(current_platform, "is_xpu", lambda: True) + clamp_limit = 3.0 + input = torch.tensor([[12.0, -12.0, 8.0, -8.0], [-2.0, 2.0, -4.0, 4.0]]) + output = torch.empty(2, 2) + + apply_moe_activation( + MoEActivation.SILU, + output, + input, + activation_config=ApplyMoEActivationConfig(clamp_limit=clamp_limit), + ) + + expected = SiluAndMulWithClamp(clamp_limit, compile_native=False).forward_native( + input + ) + torch.testing.assert_close(output, expected) + + +def _assert_masked_moe_activation( + activation: MoEActivation, + activation_config: ApplyMoEActivationConfig, + *, + dtype: torch.dtype, + mask_layout: str, + d: int, + max_num_tokens: int, +) -> None: + device = CUDA_DEVICES[0] + num_experts = 4 + input_dim = 2 * d if activation.is_gated else d + if mask_layout == "flat": + input = torch.randn(max_num_tokens, input_dim, dtype=dtype, device=device) + valid_token_counts = torch.tensor( + [max_num_tokens // 2], dtype=torch.int32, device=device + ) + output = torch.full((max_num_tokens, d), 42.0, dtype=dtype, device=device) + else: + input = torch.randn( + num_experts, max_num_tokens, input_dim, dtype=dtype, device=device + ) + valid_token_counts = torch.tensor( + [0, 1, max_num_tokens // 2, max_num_tokens], + dtype=torch.int32, + device=device, + ) + output = torch.full( + (num_experts, max_num_tokens, d), 42.0, dtype=dtype, device=device + ) + + apply_moe_activation( + activation, + output, + input, + activation_config=activation_config, + valid_token_counts=valid_token_counts, + ) + + batched_input = input.view(-1, max_num_tokens, input_dim) + batched_output = output.view(-1, max_num_tokens, d) + for expert, num_tokens in enumerate(valid_token_counts.cpu().tolist()): + if num_tokens: + expected = torch.empty((num_tokens, d), dtype=dtype, device=device) + apply_moe_activation( + activation, + expected, + batched_input[expert, :num_tokens].clone(), + activation_config=activation_config, + ) + torch.testing.assert_close( + batched_output[expert, :num_tokens], + expected, + atol=get_default_atol(output), + rtol=get_default_rtol(output), + ) + assert torch.all(batched_output[expert, num_tokens:] == 42.0) + + @pytest.mark.parametrize( "activation", [ @@ -252,6 +352,114 @@ def test_masked_situ_and_mul( ) +@pytest.mark.parametrize( + ("activation", "activation_config"), + [ + (MoEActivation.SILU, ApplyMoEActivationConfig()), + ( + MoEActivation.SILU, + ApplyMoEActivationConfig(clamp_limit=3.0), + ), + (MoEActivation.GELU, ApplyMoEActivationConfig()), + (MoEActivation.GELU_TANH, ApplyMoEActivationConfig()), + ( + MoEActivation.SITU, + ApplyMoEActivationConfig( + activation_situ_beta=1.5, + activation_situ_linear_beta=2.0, + ), + ), + (MoEActivation.SWIGLUOAI, ApplyMoEActivationConfig()), + ( + MoEActivation.SWIGLUOAI_UNINTERLEAVE, + ApplyMoEActivationConfig(clamp_limit=3.0, alpha=1.3, beta=0.5), + ), + (MoEActivation.SWIGLUSTEP, ApplyMoEActivationConfig()), + (MoEActivation.SILU_NO_MUL, ApplyMoEActivationConfig()), + (MoEActivation.GELU_NO_MUL, ApplyMoEActivationConfig()), + (MoEActivation.GELU_TANH_NO_MUL, ApplyMoEActivationConfig()), + (MoEActivation.RELU2_NO_MUL, ApplyMoEActivationConfig()), + ], + ids=[ + "silu", + "silu_clamp", + "gelu", + "gelu_tanh", + "situ", + "swigluoai", + "swigluoai_uninterleave", + "swiglustep", + "silu_no_mul", + "gelu_no_mul", + "gelu_tanh_no_mul", + "relu2_no_mul", + ], +) +@torch.inference_mode() +def test_masked_moe_activation_dispatch( + default_vllm_config, + activation: MoEActivation, + activation_config: ApplyMoEActivationConfig, +) -> None: + _assert_masked_moe_activation( + activation, + activation_config, + dtype=torch.bfloat16, + mask_layout="batched_experts", + d=513, + max_num_tokens=7, + ) + + +@pytest.mark.parametrize( + ("activation", "activation_config", "mask_layout"), + [ + pytest.param( + MoEActivation.SILU, + ApplyMoEActivationConfig(), + "flat", + id="flat", + ), + pytest.param( + MoEActivation.SITU, + ApplyMoEActivationConfig( + activation_situ_beta=1.5, + activation_situ_linear_beta=2.0, + ), + "batched_experts", + id="batched-experts", + ), + ], +) +@torch.inference_mode() +def test_masked_moe_activation_grid_stride( + default_vllm_config, + activation: MoEActivation, + activation_config: ApplyMoEActivationConfig, + mask_layout: str, +) -> None: + _assert_masked_moe_activation( + activation, + activation_config, + dtype=torch.half, + mask_layout=mask_layout, + d=513, + max_num_tokens=67, + ) + + +@torch.inference_mode() +def test_masked_moe_activation_opcheck(default_vllm_config) -> None: + device = CUDA_DEVICES[0] + input = torch.randn(2, 3, 64, dtype=torch.half, device=device) + output = torch.empty(2, 3, 32, dtype=torch.half, device=device) + valid_token_counts = torch.tensor([1, 3], dtype=torch.int32, device=device) + opcheck( + torch.ops._C.masked_moe_activation, + (output, input, valid_token_counts, "silu", 0.0, 1.0, 0.0, 1.0, -1.0), + ) + + @pytest.mark.parametrize( "activation", [ diff --git a/tests/kernels/moe/test_batched_moe.py b/tests/kernels/moe/test_batched_moe.py index 19375e6ce9..9be58e6337 100644 --- a/tests/kernels/moe/test_batched_moe.py +++ b/tests/kernels/moe/test_batched_moe.py @@ -8,6 +8,7 @@ import torch from tests.kernels.moe.utils import ( batched_moe, + make_dummy_moe_config, make_quantized_test_activations, make_test_weights, naive_batched_moe, @@ -16,7 +17,13 @@ from tests.kernels.quant_utils import native_batched_masked_quant_matmul from tests.kernels.utils import torch_experts from vllm.config import VllmConfig, set_current_vllm_config from vllm.model_executor.layers.fused_moe import fused_topk +from vllm.model_executor.layers.fused_moe.activation import ( + ApplyMoEActivationConfig, + MoEActivation, + apply_moe_activation, +) from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import ( + BatchedTritonExperts, invoke_moe_batched_triton_kernel, ) from vllm.platforms import current_platform @@ -496,11 +503,146 @@ def test_batched_mm_td_zero_expert_tokens(): # BatchedTritonExperts device enablement (XPU) def test_batched_triton_experts_supports_current_device(): - from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import ( - BatchedTritonExperts, + assert BatchedTritonExperts._supports_current_device() + + +@pytest.mark.parametrize( + "activation", + [ + MoEActivation.SITU, + MoEActivation.SWIGLUOAI_UNINTERLEAVE, + MoEActivation.SWIGLUSTEP, + ], +) +def test_batched_triton_supports_masked_activations( + monkeypatch, activation: MoEActivation +): + monkeypatch.setattr(current_platform, "is_xpu", lambda: False) + assert BatchedTritonExperts._supports_activation(activation) + + monkeypatch.setattr(current_platform, "is_xpu", lambda: True) + assert not BatchedTritonExperts._supports_activation(activation) + + +def _torch_experts_with_shared_activation( + a: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + activation: MoEActivation, + activation_config: ApplyMoEActivationConfig, +) -> torch.Tensor: + num_tokens, hidden_dim = a.shape + topk = topk_ids.shape[1] + routed_input = a[:, None, :].expand(-1, topk, -1).reshape(-1, hidden_dim) + routed_experts = topk_ids.reshape(-1) + routed_output = torch.zeros_like(routed_input) + + for expert in range(w1.shape[0]): + expert_mask = routed_experts == expert + if not expert_mask.any(): + continue + projected = routed_input[expert_mask] @ w1[expert].T + activated = torch.empty( + projected.shape[0], w2.shape[2], dtype=a.dtype, device=a.device + ) + apply_moe_activation( + activation, + activated, + projected, + activation_config=activation_config, + ) + routed_output[expert_mask] = activated @ w2[expert].T + + return ( + ( + routed_output.view(num_tokens, topk, hidden_dim).float() + * topk_weights[..., None] + ) + .sum(dim=1) + .to(a.dtype) ) - assert BatchedTritonExperts._supports_current_device() + +_MASKED_BATCHED_ACTIVATION_CASES = [ + pytest.param( + MoEActivation.SITU, + ApplyMoEActivationConfig( + activation_situ_beta=1.5, + activation_situ_linear_beta=2.0, + ), + id="situ", + ), + pytest.param( + MoEActivation.SWIGLUOAI_UNINTERLEAVE, + ApplyMoEActivationConfig(clamp_limit=3.0, alpha=1.3, beta=0.5), + id="swigluoai-uninterleave", + ), + pytest.param( + MoEActivation.SWIGLUSTEP, + ApplyMoEActivationConfig(), + id="swiglustep", + ), +] + + +@pytest.mark.parametrize( + ("activation", "activation_config"), _MASKED_BATCHED_ACTIVATION_CASES +) +@torch.inference_mode() +def test_batched_experts_masked_activation_end_to_end( + activation: MoEActivation, + activation_config: ApplyMoEActivationConfig, +): + if current_platform.is_xpu(): + pytest.skip("Masked activations are not enabled on XPU") + + from vllm.v1.worker.workspace import init_workspace_manager + + set_random_seed(7) + device = current_platform.device_type + init_workspace_manager(torch.device(f"{device}:0")) + m, n, k, e, topk = 16, 128, 128, 8, 2 + + a = torch.randn((m, k), device=device, dtype=torch.bfloat16) / 10 + score = torch.randn((m, e), device=device, dtype=torch.bfloat16) + w1 = torch.randn((e, 2 * n, k), device=device, dtype=torch.bfloat16) / 15 + w2 = torch.randn((e, k, n), device=device, dtype=torch.bfloat16) / 15 + moe_config = make_dummy_moe_config( + num_experts=e, + experts_per_token=topk, + hidden_dim=k, + intermediate_size=n, + in_dtype=a.dtype, + activation=activation, + ) + moe_config.swiglu_limit = activation_config.clamp_limit + moe_config.swiglu_alpha = activation_config.alpha + moe_config.swiglu_beta = activation_config.beta + moe_config.activation_situ_beta = activation_config.activation_situ_beta + moe_config.activation_situ_linear_beta = ( + activation_config.activation_situ_linear_beta + ) + + routing_weights = torch.softmax(score.float(), dim=-1) + topk_weights, topk_ids = torch.topk(routing_weights, topk, dim=-1) + topk_ids = topk_ids.to(torch.int32) + with set_current_vllm_config(vllm_config): + baseline_output = _torch_experts_with_shared_activation( + a, + w1, + w2, + topk_weights, + topk_ids, + activation, + activation_config, + ) + triton_output = batched_moe( + a, w1, w2, topk_weights, topk_ids, moe_config=moe_config + ) + + torch.testing.assert_close(triton_output, baseline_output, atol=3e-2, rtol=2e-2) @pytest.mark.parametrize("m,n,k,e,topk", [(32, 512, 512, 8, 2), (45, 1024, 128, 8, 1)]) diff --git a/tests/kernels/moe/test_moe.py b/tests/kernels/moe/test_moe.py index 26e0370a3e..a288e0557a 100644 --- a/tests/kernels/moe/test_moe.py +++ b/tests/kernels/moe/test_moe.py @@ -1377,6 +1377,102 @@ def test_humming_activation_metadata_tracks_shared_apply(activation: MoEActivati ) == apply_moe_activation_supported(activation) +@pytest.mark.parametrize( + ("env_override", "use_ep", "expected"), + [ + (None, False, "indexed"), + (None, True, "grouped_contiguous"), + ("auto", True, "grouped_contiguous"), + ("indexed", True, "indexed"), + ("grouped", False, "grouped_contiguous"), + ], +) +def test_humming_selects_gemm_from_parallelism_and_override( + monkeypatch: pytest.MonkeyPatch, + env_override: str | None, + use_ep: bool, + expected: str, +): + from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import ( + get_humming_moe_gemm_type, + ) + + if env_override is None: + monkeypatch.delenv("VLLM_HUMMING_MOE_GEMM_TYPE", raising=False) + else: + monkeypatch.setenv("VLLM_HUMMING_MOE_GEMM_TYPE", env_override) + moe_config = make_dummy_moe_config() + moe_config.moe_parallel_config.use_ep = use_ep + + assert get_humming_moe_gemm_type(moe_config) == expected + + +@pytest.mark.parametrize( + ("dp_token_counts", "topk_shape", "expected"), + [ + pytest.param(None, (7, 3), 21, id="runtime-layout"), + pytest.param([3, 5], (48, 1), 48, id="expanded-dp-layout"), + ], +) +def test_humming_global_valid_shape_m( + monkeypatch: pytest.MonkeyPatch, + dp_token_counts: list[int] | None, + topk_shape: tuple[int, int], + expected: int, +): + from types import SimpleNamespace + + import vllm.model_executor.layers.fused_moe.experts.fused_humming_moe as humming + + dp_metadata = ( + None + if dp_token_counts is None + else SimpleNamespace( + num_tokens_across_dp_cpu=torch.tensor(dp_token_counts, dtype=torch.int32) + ) + ) + monkeypatch.setattr( + humming, + "get_forward_context", + lambda: SimpleNamespace(dp_metadata=dp_metadata), + ) + experts = SimpleNamespace(moe_config=make_dummy_moe_config(experts_per_token=6)) + topk_ids = torch.empty(topk_shape, dtype=torch.int64) + + result = humming.HummingExpertsBase.get_global_valid_shape_m(experts, topk_ids) + + assert result == expected + + +def test_humming_permute_scratch_is_keyed_by_runtime_topk( + monkeypatch: pytest.MonkeyPatch, +): + from types import SimpleNamespace + from unittest.mock import Mock + + import vllm.model_executor.layers.fused_moe.experts.fused_humming_moe as humming + + scratch_topk6 = Mock() + scratch_topk1 = Mock() + scratch_type = Mock(side_effect=[scratch_topk6, scratch_topk1]) + monkeypatch.setattr(humming, "moe_permute_unpermute_supported", lambda: True) + monkeypatch.setattr(humming, "MoEPermuteScratch", scratch_type) + moe_config = make_dummy_moe_config(max_num_tokens=512, experts_per_token=6) + moe_config.moe_parallel_config.dp_size = 2 + experts = SimpleNamespace(_permute_scratch={}, moe_config=moe_config) + + assert humming.HummingExpertsBase._get_permute_scratch(experts, 6) is scratch_topk6 + assert humming.HummingExpertsBase._get_permute_scratch(experts, 1) is scratch_topk1 + assert humming.HummingExpertsBase._get_permute_scratch(experts, 6) is scratch_topk6 + + assert scratch_type.call_count == 2 + first_call, second_call = scratch_type.call_args_list + assert first_call.kwargs["max_num_tokens"] == 1024 + assert first_call.kwargs["topk"] == 6 + assert second_call.kwargs["max_num_tokens"] == 6144 + assert second_call.kwargs["topk"] == 1 + + def test_humming_delegates_to_instance_activation(): from types import SimpleNamespace from unittest.mock import Mock @@ -1408,10 +1504,173 @@ def test_humming_delegates_to_instance_activation(): activation=MoEActivation.SWIGLUOAI_UNINTERLEAVE, input=input, output=output, - valid_rows=None, ) +def test_humming_grouped_apply_forwards_valid_prefix( + monkeypatch: pytest.MonkeyPatch, +): + from types import MethodType, SimpleNamespace + from unittest.mock import Mock + + import vllm.model_executor.layers.fused_moe.experts.fused_humming_moe as humming + from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import ( + HummingExpertsBase, + HummingGroupedExperts, + ) + + activation_func = Mock() + buffers = { + "gate_up_output": torch.empty(6, 4), + "activation_output": torch.empty(6, 2), + "down_output": torch.empty(6, 4), + } + experts = SimpleNamespace( + activation=activation_func, + num_experts=2, + estimate_local_valid_shape_m=lambda _: 6, + prepare_buffers=lambda *_: buffers, + _get_permute_scratch=lambda _: None, + quantize_input=lambda _, *, inputs, input_scale=None, quanted_input: ( + inputs, + input_scale, + ), + humming_forward=Mock(), + compute_config_str="", + w13_tuning_config_str="", + w2_tuning_config_str="", + ) + experts.apply_activation = MethodType(HummingExpertsBase.apply_activation, experts) + + hidden_states = torch.empty(3, 4) + output = torch.empty_like(hidden_states) + topk_ids = torch.zeros((3, 2), dtype=torch.int64) + topk_weights = torch.ones_like(topk_ids, dtype=torch.float32) + expected_counts = torch.tensor([4], dtype=torch.int32) + expert_offsets = torch.tensor([0, 2, 4], dtype=torch.int64) + monkeypatch.setattr( + humming, + "moe_permute", + lambda **_: ( + torch.empty(6, 4), + None, + expert_offsets, + torch.arange(6), + None, + ), + ) + monkeypatch.setattr(humming, "moe_unpermute", lambda **_: None) + + HummingGroupedExperts.apply( + experts, + output=output, + hidden_states=hidden_states, + w1=torch.empty(0), + w2=torch.empty(0), + topk_weights=topk_weights, + topk_ids=topk_ids, + activation=MoEActivation.SITU, + global_num_experts=2, + expert_map=None, + a1q_scale=None, + a2_scale=None, + workspace13=torch.empty(0), + workspace2=torch.empty(0), + expert_tokens_meta=None, + apply_router_weight_on_input=False, + ) + + activation_func.assert_called_once() + call_kwargs = activation_func.call_args.kwargs + assert call_kwargs["activation"] == MoEActivation.SITU + assert call_kwargs["input"].shape == (6, 4) + assert call_kwargs["output"].shape == (6, 2) + torch.testing.assert_close(call_kwargs["valid_token_counts"], expected_counts) + + +def test_batched_marlin_activation_uses_expert_token_counts( + monkeypatch: pytest.MonkeyPatch, +): + from types import SimpleNamespace + from unittest.mock import Mock + + import vllm.model_executor.layers.fused_moe.experts.marlin_moe as marlin + from vllm.model_executor.layers.fused_moe.experts.marlin_moe import ( + BatchedMarlinExperts, + ) + from vllm.model_executor.layers.fused_moe.modular_kernel import ( + ExpertTokensMetadata, + ) + + activation_func = Mock() + experts = SimpleNamespace( + activation=activation_func, + w1_bias=None, + w2_bias=None, + w1_scale=None, + w2_scale=None, + quant_type_id=0, + a1_gscale=None, + a2_gscale=None, + w13_g_idx=None, + w2_g_idx=None, + w13_g_idx_sort_indices=None, + w2_g_idx_sort_indices=None, + w1_zp=None, + w2_zp=None, + input_dtype=None, + is_k_full=True, + activation_config=ApplyMoEActivationConfig( + clamp_limit=3.0, alpha=1.3, beta=0.5 + ), + ) + hidden_states = torch.empty(2, 3, 4) + expert_num_tokens = torch.tensor([1, 2], dtype=torch.int32) + topk_ids = torch.zeros((2, 3), dtype=torch.int64) + expert_map = torch.arange(2, dtype=torch.int32) + + def fake_batched_fused_marlin_moe(**kwargs): + kwargs["activation_func"]( + kwargs["activation"], + torch.empty(6, 4), + torch.empty(6, 8), + topk_ids=topk_ids, + expert_map=expert_map, + ) + return kwargs["output"] + + monkeypatch.setattr( + marlin, "batched_fused_marlin_moe", fake_batched_fused_marlin_moe + ) + output = torch.empty_like(hidden_states) + + BatchedMarlinExperts.apply( + experts, + output=output, + hidden_states=hidden_states, + w1=torch.empty(0), + w2=torch.empty(0), + topk_weights=torch.ones_like(topk_ids, dtype=torch.float32), + topk_ids=topk_ids, + activation=MoEActivation.SWIGLUOAI_UNINTERLEAVE, + global_num_experts=2, + expert_map=expert_map, + a1q_scale=None, + a2_scale=None, + workspace13=torch.empty(0), + workspace2=torch.empty(0), + expert_tokens_meta=ExpertTokensMetadata(expert_num_tokens, None), + apply_router_weight_on_input=False, + ) + + activation_func.assert_called_once() + call = activation_func.call_args + assert call.args[0] == MoEActivation.SWIGLUOAI_UNINTERLEAVE + assert call.args[1].shape == (2, 3, 4) + assert call.args[2].shape == (2, 3, 8) + torch.testing.assert_close(call.kwargs["valid_token_counts"], expert_num_tokens) + + @pytest.mark.parametrize( "activation", [ diff --git a/tests/kernels/moe/test_moe_permute_unpermute.py b/tests/kernels/moe/test_moe_permute_unpermute.py index 03db46851e..4089bbfe84 100644 --- a/tests/kernels/moe/test_moe_permute_unpermute.py +++ b/tests/kernels/moe/test_moe_permute_unpermute.py @@ -286,3 +286,39 @@ def test_moe_permute_reuses_scratch_buffers(dtype: torch.dtype): permuted_idx_1.untyped_storage().data_ptr() == scratch.permuted_idx.untyped_storage().data_ptr() ) + + +def test_moe_permute_ignores_invalid_expert_ids_with_scratch() -> None: + if not moe_permute_unpermute_supported(): + pytest.skip("moe_permute_unpermute is not supported on this platform.") + + hidden_states = torch.arange(5 * 16, dtype=torch.bfloat16, device="cuda").view( + 5, 16 + ) + topk_ids = torch.tensor([[0], [-1], [1], [4], [2]], device="cuda") + expert_map = torch.tensor([0, 1, -1, -1], dtype=torch.int32, device="cuda") + scratch = MoEPermuteScratch( + max_num_tokens=5, + topk=1, + num_experts=4, + num_local_experts=2, + device=hidden_states.device, + hidden_size=16, + hidden_dtype=hidden_states.dtype, + ) + + permuted, _, expert_offsets, _, _ = moe_permute( + hidden_states=hidden_states, + a1q_scale=None, + topk_ids=topk_ids, + n_expert=4, + n_local_expert=2, + expert_map=expert_map, + scratch=scratch, + ) + + torch.testing.assert_close( + expert_offsets, + torch.tensor([0, 1, 2], dtype=torch.int64, device="cuda"), + ) + torch.testing.assert_close(permuted[:2], hidden_states[[0, 2]]) diff --git a/tests/kernels/moe/test_situ_mul_fp8_quant.py b/tests/kernels/moe/test_situ_mul_fp8_quant.py index 3ad2833076..37a829d082 100644 --- a/tests/kernels/moe/test_situ_mul_fp8_quant.py +++ b/tests/kernels/moe/test_situ_mul_fp8_quant.py @@ -5,7 +5,7 @@ quant kernel (``situ_and_mul_quant``). Mirrors ``test_silu_mul_fp8_quant_deep_gemm.py`` in structure: build a reference activation + block-FP8 quant, drive random per-row scale magnitudes, exercise -the masked (``valid_rows``) contiguous layout, and compare per valid row. +the masked contiguous layout, and compare per valid row. The kernel computes SITU in fp32 and converts to fp8 with hardware rounding, which torch cannot reproduce bit-exactly. It is therefore validated against the @@ -40,7 +40,7 @@ SITU_LINEAR_BETA = 25.0 SITU_D = 3072 GROUP_SIZE = 128 -# (num_tokens, valid_rows). valid_rows=None => all rows valid. 2048/4096 rows +# (num_tokens, num_valid_tokens). None means all rows are valid. 2048/4096 rows # span multiple grid-stride waves of the persistent grid (GRID_DIM = 132*8). CASES = [ (1, None), @@ -111,7 +111,7 @@ def test_situ_and_mul_quant_pipelined( (num_tokens, d // GROUP_SIZE), -1.0, dtype=torch.float32, device=DEVICE ) out_sentinel = out.clone() - valid_rows = ( + num_valid_tokens = ( None if valid is None else torch.tensor(valid, dtype=torch.int32, device=DEVICE) ) @@ -122,7 +122,7 @@ def test_situ_and_mul_quant_pipelined( beta=SITU_BETA, linear_beta=SITU_LINEAR_BETA, group_size=GROUP_SIZE, - valid_rows=valid_rows, + num_valid_tokens=num_valid_tokens, ) # Padding contract: skipped rows keep their bytes; their scale becomes 1.0. @@ -157,7 +157,7 @@ def test_situ_and_mul_quant_pipelined( @pytest.mark.parametrize("tokens,topk", [(300, 8), (1, 4), (256, 6)]) @torch.inference_mode() def test_situ_and_mul_quant_topk_row_bound(tokens: int, topk: int) -> None: - """valid_rows is a token count; the kernel bounds rows at tokens*topk.""" + """The kernel expands num_valid_tokens by topk to bound rows.""" set_random_seed(7) d = SITU_D num_tokens = tokens * topk + 37 # padding tail past the valid rows @@ -168,7 +168,7 @@ def test_situ_and_mul_quant_topk_row_bound(tokens: int, topk: int) -> None: (num_tokens, d // GROUP_SIZE), -1.0, dtype=torch.float32, device=DEVICE ) out_sentinel = out.clone() - valid_rows = torch.tensor(tokens, dtype=torch.int32, device=DEVICE) + num_valid_tokens = torch.tensor(tokens, dtype=torch.int32, device=DEVICE) situ_and_mul_quant( out, @@ -177,7 +177,7 @@ def test_situ_and_mul_quant_topk_row_bound(tokens: int, topk: int) -> None: beta=SITU_BETA, linear_beta=SITU_LINEAR_BETA, group_size=GROUP_SIZE, - valid_rows=valid_rows, + num_valid_tokens=num_valid_tokens, topk=topk, ) diff --git a/tests/kernels/moe/utils.py b/tests/kernels/moe/utils.py index e82d315715..1b1b48d44d 100644 --- a/tests/kernels/moe/utils.py +++ b/tests/kernels/moe/utils.py @@ -128,6 +128,7 @@ def batched_moe( quant_dtype: torch.dtype | None = None, per_act_token_quant: bool = False, block_shape: list[int] | None = None, + moe_config: FusedMoEConfig | None = None, ) -> torch.Tensor: max_num_tokens = round_up(a.shape[0], 64) @@ -141,7 +142,8 @@ def batched_moe( a2_scale=a2_scale, ) - moe_config = make_dummy_moe_config() + if moe_config is None: + moe_config = make_dummy_moe_config() fused_experts = FusedMoEKernel( BatchedPrepareAndFinalize( diff --git a/vllm/model_executor/layers/fused_moe/activation.py b/vllm/model_executor/layers/fused_moe/activation.py index e9212cacf7..d83301bed1 100644 --- a/vllm/model_executor/layers/fused_moe/activation.py +++ b/vllm/model_executor/layers/fused_moe/activation.py @@ -9,6 +9,8 @@ from typing import TYPE_CHECKING import torch import torch.nn.functional as F +from vllm.platforms import current_platform + if TYPE_CHECKING: from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, @@ -134,11 +136,31 @@ _APPLY_MOE_ACTIVATIONS = frozenset( ) +_MASKED_MOE_ACTIVATION_NAMES: dict[MoEActivation, str] = { + MoEActivation.SILU: "silu", + MoEActivation.GELU: "gelu", + MoEActivation.GELU_TANH: "gelu_tanh", + MoEActivation.SITU: "situ", + MoEActivation.SWIGLUOAI: "swigluoai", + MoEActivation.SWIGLUOAI_UNINTERLEAVE: "swigluoai_uninterleave", + MoEActivation.SWIGLUSTEP: "swiglustep", + MoEActivation.SILU_NO_MUL: "silu_no_mul", + MoEActivation.GELU_NO_MUL: "gelu_no_mul", + MoEActivation.GELU_TANH_NO_MUL: "gelu_tanh_no_mul", + MoEActivation.RELU2_NO_MUL: "relu2_no_mul", +} + + def apply_moe_activation_supported(activation: MoEActivation) -> bool: """Whether ``apply_moe_activation`` supports an activation.""" return activation in _APPLY_MOE_ACTIVATIONS +def apply_moe_activation_masked_supported(activation: MoEActivation) -> bool: + """Whether the masked ``apply_moe_activation`` path supports an activation.""" + return activation in _MASKED_MOE_ACTIVATION_NAMES + + @dataclass(frozen=True) class ApplyMoEActivationConfig: """Configuration forwarded to ``apply_moe_activation``.""" @@ -177,6 +199,96 @@ class ApplyMoEActivationConfig: _DEFAULT_APPLY_MOE_ACTIVATION_CONFIG = ApplyMoEActivationConfig() +def _validate_moe_activation_shapes( + activation: MoEActivation, + output: torch.Tensor, + input: torch.Tensor, + expected_dim: int, +) -> None: + assert input.dim() == expected_dim, f"Input must be {expected_dim}D" + assert output.dim() == expected_dim, f"Output must be {expected_dim}D" + assert input.shape[:-1] == output.shape[:-1], ( + f"Input/output leading shapes must match: {input.shape} vs {output.shape}" + ) + if activation.is_gated: + assert output.size(-1) * 2 == input.size(-1), ( + f"{activation.value} expects 2x ratio: " + f"{output.size(-1) * 2} vs {input.size(-1)}" + ) + else: + assert output.size(-1) == input.size(-1), ( + f"{activation.value} expects equal sizes: " + f"{output.size(-1)} vs {input.size(-1)}" + ) + + +def _apply_moe_activation_masked( + activation: MoEActivation, + output: torch.Tensor, + input: torch.Tensor, + valid_token_counts: torch.Tensor, + config: ApplyMoEActivationConfig, +) -> torch.Tensor: + masked_activation = _MASKED_MOE_ACTIVATION_NAMES.get(activation) + if masked_activation is None: + raise NotImplementedError( + f"Masked MoE activation is not implemented for {activation.value}" + ) + + assert input.dim() in (2, 3), "Masked input must be 2D or 3D" + _validate_moe_activation_shapes(activation, output, input, expected_dim=input.dim()) + assert input.dtype == output.dtype, "Input and output dtypes must match" + assert input.device == output.device, "Input and output devices must match" + assert input.is_contiguous(), "Input must be contiguous" + assert output.is_contiguous(), "Output must be contiguous" + assert valid_token_counts.dtype == torch.int32, ( + "valid_token_counts must use torch.int32" + ) + assert valid_token_counts.dim() == 1, "valid_token_counts must be 1D" + assert valid_token_counts.device == input.device, ( + "valid_token_counts must be on the input device" + ) + assert valid_token_counts.is_contiguous(), "valid_token_counts must be contiguous" + expected_counts = input.size(0) if input.dim() == 3 else 1 + assert valid_token_counts.size(0) == expected_counts, ( + f"valid_token_counts must have {expected_counts} element(s) for " + f"{input.dim()}D input" + ) + + masked_clamp_limit = 0.0 if config.clamp_limit is None else config.clamp_limit + masked_alpha = config.alpha + if activation == MoEActivation.SILU and config.clamp_limit is not None: + masked_activation = "silu_with_clamp" + elif activation == MoEActivation.SITU: + assert config.activation_situ_beta is not None, ( + "SITU requires activation_situ_beta from FusedMoEConfig" + ) + elif activation == MoEActivation.SWIGLUOAI: + masked_clamp_limit = 7.0 + masked_alpha = 1.702 + elif activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE: + assert config.clamp_limit is not None, ( + "SWIGLUOAI_UNINTERLEAVE requires clamp_limit" + ) + elif activation == MoEActivation.SWIGLUSTEP: + masked_clamp_limit = 7.0 + + torch.ops._C.masked_moe_activation( + output, + input, + valid_token_counts, + masked_activation, + masked_clamp_limit, + masked_alpha, + config.beta, + 1.0 if config.activation_situ_beta is None else config.activation_situ_beta, + -1.0 + if config.activation_situ_linear_beta is None + else config.activation_situ_linear_beta, + ) + return output + + def silu_and_mul_with_clamp( output: torch.Tensor, input: torch.Tensor, @@ -184,7 +296,12 @@ def silu_and_mul_with_clamp( topk_ids: torch.Tensor | None = None, expert_map: torch.Tensor | None = None, ) -> None: - if topk_ids is not None and expert_map is not None: + if current_platform.is_xpu(): + d = input.shape[-1] // 2 + gate = torch.clamp(input[..., :d], max=clamp_limit) + up = torch.clamp(input[..., d:], min=-clamp_limit, max=clamp_limit) + output.copy_(gate * torch.sigmoid(gate) * up) + elif topk_ids is not None and expert_map is not None: from vllm.model_executor.layers.fused_moe.utils import swiglu_limit_func swiglu_limit_func(output, input, clamp_limit, topk_ids, expert_map) @@ -201,12 +318,15 @@ def apply_moe_activation( activation_config: ApplyMoEActivationConfig | None = None, topk_ids: torch.Tensor | None = None, expert_map: torch.Tensor | None = None, - valid_rows: torch.Tensor | None = None, + valid_token_counts: torch.Tensor | None = None, ) -> torch.Tensor: """Apply MoE activation function. The configuration drives specialized activation behavior. Routing tensors - remain per-call inputs because they depend on the current token assignment. + and valid token counts remain per-call inputs because they depend on the + current token assignment. A single token count masks a flat ``[T, D]`` + buffer; one count per expert masks each prefix in a padded ``[E, T, D]`` + buffer. """ config = ( _DEFAULT_APPLY_MOE_ACTIVATION_CONFIG @@ -214,19 +334,17 @@ def apply_moe_activation( else activation_config ) - assert input.dim() == 2, "Input must be 2D" - assert output.dim() == 2, "Output must be 2D" - if activation.is_gated: - assert output.size(-1) * 2 == input.size(-1), ( - f"{activation.value} expects 2x ratio: " - f"{output.size(-1) * 2} vs {input.size(-1)}" - ) - else: - assert output.size(-1) == input.size(-1), ( - f"{activation.value} expects equal sizes: " - f"{output.size(-1)} vs {input.size(-1)}" + if valid_token_counts is not None: + return _apply_moe_activation_masked( + activation, + output, + input, + valid_token_counts, + config, ) + _validate_moe_activation_shapes(activation, output, input, expected_dim=2) + # Activations with gated multiplication (gate × activation(up)) if activation == MoEActivation.SILU: if config.clamp_limit is not None: @@ -257,7 +375,6 @@ def apply_moe_activation( -1.0 if config.activation_situ_linear_beta is None else config.activation_situ_linear_beta, - valid_rows, ) elif activation == MoEActivation.SWIGLUOAI: torch.ops._C.swigluoai_and_mul(output, input) @@ -298,7 +415,7 @@ def situ_and_mul_quant( beta: float, linear_beta: float | None, group_size: int = 0, - valid_rows: torch.Tensor | None = None, + num_valid_tokens: torch.Tensor | None = None, topk: int = 1, ) -> None: """Fused Kimi SITU activation + dynamic FP8 quantization. @@ -314,11 +431,11 @@ def situ_and_mul_quant( [M, d // 128], matching humming ``quant_input(group_size=128, float32)``. ``linear_beta`` <= 0 (or None) means "unset" (up passed through), matching - ``SituAndMul(linear_beta=None)``. ``valid_rows`` (int32 scalar tensor) is the - DeepEP v2 contiguous-layout valid *token* count (e.g. ``psum[-1:]``); the - kernel multiplies it by ``topk`` to bound rows, so no host-side cast/multiply - is needed. Padding rows are skipped and receive a benign scale of 1.0. Kept - in this module so that every ``torch.ops._C.situ*`` call lives in one file. + ``SituAndMul(linear_beta=None)``. ``num_valid_tokens`` (int32 scalar tensor) + is the DeepEP v2 contiguous-layout token count (e.g. ``psum[-1:]``); the + kernel multiplies it by ``topk`` to bound rows. Padding rows are skipped and + receive a benign scale of 1.0. Kept in this module so that every + ``torch.ops._C.situ*`` call lives in one file. """ torch.ops._C.situ_and_mul_quant( output, @@ -327,6 +444,6 @@ def situ_and_mul_quant( beta, -1.0 if linear_beta is None else linear_beta, group_size, - valid_rows, + num_valid_tokens, topk, ) diff --git a/vllm/model_executor/layers/fused_moe/experts/fused_batched_moe.py b/vllm/model_executor/layers/fused_moe/experts/fused_batched_moe.py index bad632071a..f16b351d35 100644 --- a/vllm/model_executor/layers/fused_moe/experts/fused_batched_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/fused_batched_moe.py @@ -5,7 +5,10 @@ import torch import vllm.model_executor.layers.fused_moe.modular_kernel as mk -from vllm.model_executor.layers.fused_moe.activation import MoEActivation +from vllm.model_executor.layers.fused_moe.activation import ( + MoEActivation, + apply_moe_activation_masked_supported, +) from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, FusedMoEParallelConfig, @@ -19,7 +22,6 @@ from vllm.model_executor.layers.fused_moe.utils import ( _resize_cache, moe_kernel_quantize_input, normalize_batched_scales_shape, - swiglu_limit_func, ) from vllm.model_executor.layers.quantization.utils.quant_utils import ( QuantKey, @@ -839,16 +841,18 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular): @staticmethod def _supports_activation(activation: MoEActivation) -> bool: - return activation in [ - MoEActivation.SILU, - MoEActivation.GELU, - MoEActivation.GELU_TANH, - MoEActivation.SWIGLUOAI, - MoEActivation.SILU_NO_MUL, - MoEActivation.GELU_NO_MUL, - MoEActivation.GELU_TANH_NO_MUL, - MoEActivation.RELU2_NO_MUL, - ] + if current_platform.is_xpu(): + return activation in [ + MoEActivation.SILU, + MoEActivation.GELU, + MoEActivation.GELU_TANH, + MoEActivation.SWIGLUOAI, + MoEActivation.SILU_NO_MUL, + MoEActivation.GELU_NO_MUL, + MoEActivation.GELU_TANH_NO_MUL, + MoEActivation.RELU2_NO_MUL, + ] + return apply_moe_activation_masked_supported(activation) @staticmethod def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool: @@ -858,20 +862,6 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular): # Let PrepareAndFinalize::finalize() decide the impl. return TopKWeightAndReduceDelegate() - def activation( - self, - activation: MoEActivation, - output: torch.Tensor, - input: torch.Tensor, - **kwargs, - ) -> None: - gemm1_clamp_limit = self.quant_config.gemm1_clamp_limit - if activation == MoEActivation.SILU and gemm1_clamp_limit is not None: - swiglu_limit_func(output, input, float(gemm1_clamp_limit)) - return - - super().activation(activation, output, input) - def workspace_shapes( self, M: int, @@ -998,11 +988,19 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular): intermediate_cache2.fill_(0) # TODO (bnell): use triton utility from batched deep gemm. - self.activation( - activation, - intermediate_cache2.view(-1, activation_out_dim), - intermediate_cache1.view(-1, N), - ) + if current_platform.is_xpu(): + self.activation( + activation, + intermediate_cache2.view(-1, activation_out_dim), + intermediate_cache1.view(-1, N), + ) + else: + self.activation( + activation, + intermediate_cache2, + intermediate_cache1, + valid_token_counts=expert_num_tokens, + ) qintermediate_cache2, a2q_scale = batched_moe_kernel_quantize_input( intermediate_cache2, diff --git a/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py b/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py index 8a1f117e3f..86079605a3 100644 --- a/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py @@ -35,10 +35,7 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import ( TopKWeightAndReduceDelegate, TopKWeightAndReduceNoOP, ) -from vllm.model_executor.layers.fused_moe.utils import ( - _resize_cache, - swiglu_limit_func, -) +from vllm.model_executor.layers.fused_moe.utils import _resize_cache from vllm.model_executor.layers.quantization.utils.quant_utils import ( INT4_DTYPE, INT8_DTYPE, @@ -103,17 +100,18 @@ def _is_supported_wna16_weight_key(weight_key: QuantKey | None) -> bool: ) -def get_humming_moe_gemm_type() -> str: +def get_humming_moe_gemm_type(moe_config: FusedMoEConfig | None = None) -> str: env_gemm_type: str | None = envs.VLLM_HUMMING_MOE_GEMM_TYPE - gemm_type = "indexed" - if env_gemm_type is not None: + if env_gemm_type is not None and env_gemm_type.lower() != "auto": env_gemm_type = env_gemm_type.lower() - if env_gemm_type == "indexed": - gemm_type = env_gemm_type - elif env_gemm_type in ["grouped_contiguous", "grouped"]: + if env_gemm_type == "grouped": gemm_type = "grouped_contiguous" else: - gemm_type = "indexed" + gemm_type = env_gemm_type + elif moe_config is not None and moe_config.moe_parallel_config.use_ep: + gemm_type = "grouped_contiguous" + else: + gemm_type = "indexed" logger.info_once(f"Using {gemm_type} gemm for humming moe") # noqa return gemm_type @@ -178,7 +176,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): max_num_tokens=max_num_tokens, num_dispatchers=num_dispatchers, ) - self._permute_scratch: MoEPermuteScratch | None = None + self._permute_scratch: dict[int, MoEPermuteScratch] = {} def init_humming_moe(self): from vllm.utils.humming import get_heuristics_config @@ -257,26 +255,36 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): **kwargs, ) - def _get_permute_scratch(self) -> MoEPermuteScratch | None: - if self._permute_scratch is None and moe_permute_unpermute_supported(): - self._permute_scratch = MoEPermuteScratch( - max_num_tokens=self.moe_config.max_num_tokens, - topk=self.moe_config.experts_per_token, + def _get_permute_scratch(self, topk: int) -> MoEPermuteScratch | None: + if not moe_permute_unpermute_supported(): + return None + + scratch = self._permute_scratch.get(topk) + if scratch is None: + max_expanded_rows = ( + self.moe_config.max_num_tokens + * self.moe_config.dp_size + * self.moe_config.experts_per_token + ) + scratch = MoEPermuteScratch( + max_num_tokens=math.ceil(max_expanded_rows / topk), + topk=topk, num_experts=self.moe_config.num_experts, num_local_experts=self.moe_config.num_local_experts, device=torch.device(self.moe_config.device), hidden_size=self.moe_config.hidden_dim, hidden_dtype=self.moe_config.in_dtype, ) - return self._permute_scratch + self._permute_scratch[topk] = scratch + return scratch def get_global_valid_shape_m(self, topk_ids: torch.Tensor): - num_tokens = topk_ids.size(0) ctx = get_forward_context() if ctx.dp_metadata is not None: num_tokens = ctx.dp_metadata.num_tokens_across_dp_cpu.sum().item() + return num_tokens * self.moe_config.experts_per_token - return num_tokens * topk_ids.size(1) + return topk_ids.size(0) * topk_ids.size(1) def estimate_local_valid_shape_m(self, topk_ids: torch.Tensor): # estimate shape_m for kernel tuning @@ -645,7 +653,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): if supported: assert hasattr(cls, "humming_gemm_type") gemm_type = cls.humming_gemm_type().value.lower() - preferred_gemm_type = get_humming_moe_gemm_type() + preferred_gemm_type = get_humming_moe_gemm_type(moe_config) supported = preferred_gemm_type.lower() == gemm_type if not supported: reason = ( @@ -660,25 +668,16 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): activation: MoEActivation, output: torch.Tensor, input: torch.Tensor, - valid_rows: torch.Tensor | None = None, + valid_token_counts: torch.Tensor | None = None, ) -> None: - activation_config = self.activation_config - if ( - activation == MoEActivation.SILU - and activation_config.clamp_limit is not None - ): - swiglu_limit_func( - output=output, - input=input, - swiglu_limit=activation_config.clamp_limit, - ) - else: - self.activation( - activation=activation, - input=input, - output=output, - valid_rows=valid_rows, - ) + activation_kwargs: dict[str, Any] = dict( + activation=activation, + input=input, + output=output, + ) + if valid_token_counts is not None: + activation_kwargs["valid_token_counts"] = valid_token_counts + self.activation(**activation_kwargs) def fused_situ_quant_enabled(self, activation: MoEActivation) -> bool: """Whether the SITU activation + w2 quant can be fused into one kernel. @@ -718,7 +717,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): self, gate_up_output: torch.Tensor, quanted_down_input: torch.Tensor, - valid_rows: torch.Tensor | None, + num_valid_tokens: torch.Tensor | None, topk: int = 1, ) -> tuple[torch.Tensor, torch.Tensor]: """Run the fused SITU activation + FP8 quant for the w2 input. @@ -750,7 +749,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): beta=cfg.activation_situ_beta, linear_beta=cfg.activation_situ_linear_beta, group_size=group_size, - valid_rows=valid_rows, + num_valid_tokens=num_valid_tokens, topk=topk, ) return quanted_down_input, input_scale @@ -878,8 +877,8 @@ class HummingIndexedExperts(HummingExpertsBase): ) # psum[-1:] is the DeepEP valid *token* count as a zero-cost int32 view. - # situ (rows = tokens*topk) multiplies by topk on-device; mul_sum bounds - # on tokens, so both read this pointer directly -- no host cast/multiply. + # The fused path and mul_sum consume tokens directly. The unfused + # activation consumes the expanded row count (tokens * topk). valid_tokens = None topk = topk_ids.size(1) if ( @@ -894,21 +893,18 @@ class HummingIndexedExperts(HummingExpertsBase): inputs, input_scale = self.fused_situ_quant( gate_up_output=buffers["gate_up_output"], quanted_down_input=buffers["quanted_down_input"], - valid_rows=valid_tokens, + num_valid_tokens=valid_tokens, topk=topk, ) else: - # Fallback situ_and_mul takes int64 row counts (tokens*topk). - valid_rows = ( - valid_tokens.to(torch.int64) * topk - if valid_tokens is not None - else None + valid_token_counts = ( + valid_tokens * topk if valid_tokens is not None else None ) self.apply_activation( activation=activation, input=buffers["gate_up_output"], output=buffers["activation_output"], - valid_rows=valid_rows, + valid_token_counts=valid_token_counts, ) inputs, input_scale = self.quantize_input( @@ -1002,7 +998,7 @@ class HummingGroupedExperts(HummingExpertsBase): n_expert=global_num_experts, n_local_expert=self.num_experts, expert_map=expert_map, - scratch=self._get_permute_scratch(), + scratch=self._get_permute_scratch(topk_ids.size(1)), ) inputs, input_scale = self.quantize_input( @@ -1028,6 +1024,11 @@ class HummingGroupedExperts(HummingExpertsBase): activation=activation, input=buffers["gate_up_output"], output=buffers["activation_output"], + valid_token_counts=( + expert_first_token_offset[-1:].to(torch.int32) + if expert_tokens_meta is None + else None + ), ) inputs, input_scale = self.quantize_input( diff --git a/vllm/model_executor/layers/fused_moe/experts/marlin_moe.py b/vllm/model_executor/layers/fused_moe/experts/marlin_moe.py index c3eb5eefb3..7014e40763 100644 --- a/vllm/model_executor/layers/fused_moe/experts/marlin_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/marlin_moe.py @@ -13,6 +13,7 @@ from vllm.model_executor.layers.fused_moe.activation import ( ApplyMoEActivationConfig, MoEActivation, apply_moe_activation, + apply_moe_activation_masked_supported, apply_moe_activation_supported, ) from vllm.model_executor.layers.fused_moe.config import ( @@ -955,6 +956,10 @@ class BatchedMarlinExperts(MarlinExpertsBase): def activation_format() -> mk.FusedMoEActivationFormat: return mk.FusedMoEActivationFormat.BatchedExperts + @staticmethod + def _supports_activation(activation: MoEActivation) -> bool: + return apply_moe_activation_masked_supported(activation) + def workspace_shapes( self, M: int, @@ -1000,26 +1005,18 @@ class BatchedMarlinExperts(MarlinExpertsBase): act: MoEActivation, act_output: torch.Tensor, act_input: torch.Tensor, - **kwargs, + *, + topk_ids: torch.Tensor | None = None, + expert_map: torch.Tensor | None = None, ) -> None: - if act != MoEActivation.SITU: - self.activation( - act, - act_output, - act_input, - **kwargs, - ) - return - num_experts, max_num_tokens = hidden_states.shape[:2] - beta = self.activation_config.activation_situ_beta - linear_beta = self.activation_config.activation_situ_linear_beta - torch.ops._C.masked_situ_and_mul( + self.activation( + act, act_output.view(num_experts, max_num_tokens, -1), act_input.view(num_experts, max_num_tokens, -1), - expert_tokens_meta.expert_num_tokens, - 1.0 if beta is None else beta, - -1.0 if linear_beta is None else linear_beta, + topk_ids=topk_ids, + expert_map=expert_map, + valid_token_counts=expert_tokens_meta.expert_num_tokens, ) return batched_fused_marlin_moe( diff --git a/vllm/model_executor/layers/fused_moe/modular_kernel.py b/vllm/model_executor/layers/fused_moe/modular_kernel.py index 90736c09a1..7d4833a9df 100644 --- a/vllm/model_executor/layers/fused_moe/modular_kernel.py +++ b/vllm/model_executor/layers/fused_moe/modular_kernel.py @@ -902,7 +902,7 @@ class FusedMoEExpertsModular(FusedMoEExperts): *, topk_ids: torch.Tensor | None = None, expert_map: torch.Tensor | None = None, - valid_rows: torch.Tensor | None = None, + valid_token_counts: torch.Tensor | None = None, ) -> None: apply_moe_activation( activation, @@ -911,7 +911,7 @@ class FusedMoEExpertsModular(FusedMoEExperts): activation_config=self.activation_config, topk_ids=topk_ids, expert_map=expert_map, - valid_rows=valid_rows, + valid_token_counts=valid_token_counts, ) @abstractmethod