[MoE] Generalize masked activation for padded layouts (#51217)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Misha Goin
2026-09-01 11:34:13 -04:00
committed by GitHub
co-authored by OpenAI Codex
parent a232e29e9d
commit 82b7d49a6e
15 changed files with 1217 additions and 196 deletions
+297 -48
View File
@@ -500,10 +500,8 @@ template <typename scalar_t>
__global__ void situ_and_mul_kernel(
scalar_t* __restrict__ out, // [..., d]
const scalar_t* __restrict__ input, // [..., 2, d]
const int d, const float beta, const float linear_beta,
const int64_t* __restrict__ valid_rows_ptr) {
const int d, const float beta, const float linear_beta) {
const int64_t row = blockIdx.x;
if (valid_rows_ptr != nullptr && row >= *valid_rows_ptr) return;
const scalar_t* gate_ptr = input + row * 2 * d;
const scalar_t* up_ptr = gate_ptr + d;
scalar_t* out_ptr = out + row * d;
@@ -558,15 +556,16 @@ template <typename scalar_t, typename fp8_type, int THREADS, int NUM_STAGES,
__global__ void situ_and_mul_quant_group_pipelined_kernel(
fp8_type* __restrict__ out, float* __restrict__ scale_out,
const scalar_t* __restrict__ input, const int64_t num_rows,
const int32_t* __restrict__ valid_rows_ptr, const int64_t topk) {
const int32_t* __restrict__ num_valid_tokens_ptr, const int64_t topk) {
const int tid = threadIdx.x;
const int warp_id = tid / WARP_SIZE;
const int lane_id = tid % WARP_SIZE;
static constexpr int NUM_WARPS = THREADS / WARP_SIZE;
static constexpr int NUM_GROUPS = D / GROUP_SIZE;
const int64_t row_bound =
valid_rows_ptr != nullptr
? max((int64_t)0, min((int64_t)(*valid_rows_ptr) * topk, num_rows))
num_valid_tokens_ptr != nullptr
? max((int64_t)0,
min((int64_t)(*num_valid_tokens_ptr) * topk, num_rows))
: num_rows;
static_assert(NUM_GROUPS % 4 == 0,
@@ -710,15 +709,16 @@ __global__ void situ_and_mul_quant_group_scalar_kernel(
const scalar_t* __restrict__ input, // [num_tokens, 2, d]
const int d, const int num_groups, const float beta,
const float linear_beta, const int64_t num_rows,
const int32_t* __restrict__ valid_rows_ptr, const int64_t topk) {
const int32_t* __restrict__ num_valid_tokens_ptr, const int64_t topk) {
static constexpr int ELTS_PER_LANE = GROUP_SIZE / WARP_SIZE;
const int tid = threadIdx.x;
const int warp_id = tid / WARP_SIZE;
const int lane_id = tid % WARP_SIZE;
const int num_warps = blockDim.x / WARP_SIZE;
const int64_t row_bound =
valid_rows_ptr != nullptr
? max((int64_t)0, min((int64_t)(*valid_rows_ptr) * topk, num_rows))
num_valid_tokens_ptr != nullptr
? max((int64_t)0,
min((int64_t)(*num_valid_tokens_ptr) * topk, num_rows))
: num_rows;
const bool clamp_up = linear_beta > 0.0f;
@@ -768,24 +768,61 @@ __global__ void situ_and_mul_quant_group_scalar_kernel(
}
}
template <typename scalar_t>
constexpr int kMaxMaskedTokenBlocks = 32;
template <bool BATCHED_EXPERTS>
__device__ __forceinline__ bool get_masked_row_range(
const int* __restrict__ valid_token_counts, const int max_num_tokens,
int64_t& first_row, int64_t& end_row, int64_t& row_stride) {
if constexpr (BATCHED_EXPERTS) {
// [E, T, *]: z lanes grid-stride over one expert's valid token prefix.
const int expert = blockIdx.y;
const int num_tokens =
max(0, min(valid_token_counts[expert], max_num_tokens));
const int token_block = blockIdx.z;
if (token_block >= num_tokens) {
return false;
}
const int64_t expert_first_row =
static_cast<int64_t>(expert) * max_num_tokens;
first_row = expert_first_row + token_block;
end_row = expert_first_row + num_tokens;
row_stride = gridDim.z;
} else {
// [T, *]: y lanes grid-stride over the single valid token prefix.
const int num_tokens = max(0, min(valid_token_counts[0], max_num_tokens));
const int token_block = blockIdx.y;
if (token_block >= num_tokens) {
return false;
}
first_row = token_block;
end_row = num_tokens;
row_stride = gridDim.y;
}
return first_row < end_row;
}
__device__ __forceinline__ int get_masked_feature_index() {
return blockIdx.x * blockDim.x + threadIdx.x;
}
template <typename scalar_t, bool BATCHED_EXPERTS>
__global__ void masked_situ_and_mul_kernel(
scalar_t* __restrict__ out, const scalar_t* __restrict__ input,
const int* __restrict__ expert_num_tokens, const int max_num_tokens,
const int* __restrict__ valid_token_counts, const int max_num_tokens,
const int d, const float beta, const float linear_beta) {
const int expert = blockIdx.y;
const int num_tokens = expert_num_tokens[expert];
const int idx = blockIdx.x * blockDim.x + threadIdx.x;
if (idx >= d || num_tokens == 0) {
int64_t first_row, end_row, row_stride;
const int idx = get_masked_feature_index();
if (idx >= d ||
!get_masked_row_range<BATCHED_EXPERTS>(valid_token_counts, max_num_tokens,
first_row, end_row, row_stride)) {
return;
}
const bool clamp_up = linear_beta > 0.0f;
const float inv_beta = 1.0f / beta;
const float inv_linear_beta = clamp_up ? 1.0f / linear_beta : 0.0f;
const int64_t expert_row = static_cast<int64_t>(expert) * max_num_tokens;
for (int token = 0; token < num_tokens; ++token) {
const int64_t row = expert_row + token;
for (int64_t row = first_row; row < end_row; row += row_stride) {
const scalar_t* gate_ptr = input + row * 2 * d;
const scalar_t* up_ptr = gate_ptr + d;
scalar_t* out_ptr = out + row * d;
@@ -796,6 +833,119 @@ __global__ void masked_situ_and_mul_kernel(
}
}
template <typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&, const float),
bool act_first, bool HAS_CLAMP, bool BATCHED_EXPERTS>
__global__ void masked_act_and_mul_kernel(
scalar_t* __restrict__ out, const scalar_t* __restrict__ input,
const int* __restrict__ valid_token_counts, const int max_num_tokens,
const int d, const float limit, const float alpha, const float beta) {
int64_t first_row, end_row, row_stride;
const int idx = get_masked_feature_index();
if (idx >= d ||
!get_masked_row_range<BATCHED_EXPERTS>(valid_token_counts, max_num_tokens,
first_row, end_row, row_stride)) {
return;
}
for (int64_t row = first_row; row < end_row; row += row_stride) {
const scalar_t* x_ptr = input + row * 2 * d;
const scalar_t* y_ptr = x_ptr + d;
scalar_t* out_ptr = out + row * d;
const scalar_t x = VLLM_LDG(&x_ptr[idx]);
const scalar_t y = VLLM_LDG(&y_ptr[idx]);
out_ptr[idx] = compute<scalar_t, ACT_FN, act_first, HAS_CLAMP>(x, y, limit,
alpha, beta);
}
}
template <typename scalar_t, bool BATCHED_EXPERTS>
__global__ void masked_swigluoai_and_mul_kernel(
scalar_t* __restrict__ out, const scalar_t* __restrict__ input,
const int* __restrict__ valid_token_counts, const int max_num_tokens,
const int d, const float alpha, const float limit) {
int64_t first_row, end_row, row_stride;
const int idx = get_masked_feature_index();
if (idx >= d ||
!get_masked_row_range<BATCHED_EXPERTS>(valid_token_counts, max_num_tokens,
first_row, end_row, row_stride)) {
return;
}
for (int64_t row = first_row; row < end_row; row += row_stride) {
const scalar_t* in_ptr = input + row * 2 * d;
scalar_t* out_ptr = out + row * d;
out_ptr[idx] =
swigluoai_and_mul(in_ptr[2 * idx], in_ptr[2 * idx + 1], alpha, limit);
}
}
template <typename scalar_t, bool BATCHED_EXPERTS>
__global__ void masked_swiglustep_and_mul_kernel(
scalar_t* __restrict__ out, const scalar_t* __restrict__ input,
const int* __restrict__ valid_token_counts, const int max_num_tokens,
const int d, const float limit) {
int64_t first_row, end_row, row_stride;
const int idx = get_masked_feature_index();
if (idx >= d ||
!get_masked_row_range<BATCHED_EXPERTS>(valid_token_counts, max_num_tokens,
first_row, end_row, row_stride)) {
return;
}
for (int64_t row = first_row; row < end_row; row += row_stride) {
const scalar_t* gate_ptr = input + row * 2 * d;
const scalar_t* up_ptr = gate_ptr + d;
scalar_t* out_ptr = out + row * d;
const float gate = (float)VLLM_LDG(&gate_ptr[idx]);
const float up = (float)VLLM_LDG(&up_ptr[idx]);
const float gate_silu = gate / (1.0f + expf(-gate));
const float gate_clamped = fminf(gate_silu, limit);
const float up_clamped = fmaxf(fminf(up, limit), -limit);
out_ptr[idx] = (scalar_t)(gate_clamped * up_clamped);
}
}
template <typename T>
__device__ __forceinline__ T silu_no_mul_kernel(const T& x) {
return silu_kernel(x, 1.0f);
}
template <typename T>
__device__ __forceinline__ T gelu_no_mul_kernel(const T& x) {
return gelu_kernel(x, 1.0f);
}
template <typename T>
__device__ __forceinline__ T gelu_tanh_no_mul_kernel(const T& x) {
return gelu_tanh_kernel(x, 1.0f);
}
template <typename T>
__device__ __forceinline__ T relu2_no_mul_kernel(const T& x) {
const float value = fmaxf((float)x, 0.0f);
return (T)(value * value);
}
template <typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&),
bool BATCHED_EXPERTS>
__global__ void masked_activation_kernel(
scalar_t* __restrict__ out, const scalar_t* __restrict__ input,
const int* __restrict__ valid_token_counts, const int max_num_tokens,
const int d) {
int64_t first_row, end_row, row_stride;
const int idx = get_masked_feature_index();
if (idx >= d ||
!get_masked_row_range<BATCHED_EXPERTS>(valid_token_counts, max_num_tokens,
first_row, end_row, row_stride)) {
return;
}
for (int64_t row = first_row; row < end_row; row += row_stride) {
const int64_t offset = row * d + idx;
out[offset] = ACT_FN(VLLM_LDG(&input[offset]));
}
}
} // namespace vllm
#define LAUNCH_ACTIVATION_GATE_KERNEL_WITH_PARAM(KERNEL, PACKED_KERNEL, PARAM) \
@@ -890,8 +1040,7 @@ void swigluoai_and_mul(torch::stable::Tensor& out, // [..., d]
// through), matching SituAndMul(linear_beta=None) on the Python side.
void situ_and_mul(torch::stable::Tensor& out, // [..., d]
torch::stable::Tensor& input, // [..., 2 * d]
double beta, double linear_beta,
std::optional<torch::stable::Tensor> valid_rows) {
double beta, double linear_beta) {
int d = input.size(-1) / 2;
int64_t num_tokens = input.numel() / input.size(-1);
if (num_tokens == 0) {
@@ -899,14 +1048,6 @@ void situ_and_mul(torch::stable::Tensor& out, // [..., d]
}
dim3 grid(num_tokens);
dim3 block(std::min(d, 1024));
const int64_t* valid_rows_ptr = nullptr;
if (valid_rows.has_value()) {
STD_TORCH_CHECK(valid_rows->is_cuda() && valid_rows->get_device_index() ==
input.get_device_index(),
"situ_and_mul: valid_rows must be a CUDA tensor on the "
"same device as input");
valid_rows_ptr = valid_rows->const_data_ptr<int64_t>();
}
const torch::stable::accelerator::DeviceGuard device_guard(
input.get_device_index());
const cudaStream_t stream = get_current_cuda_stream();
@@ -914,7 +1055,7 @@ void situ_and_mul(torch::stable::Tensor& out, // [..., d]
input.scalar_type(), "situ_and_mul_kernel", [&] {
vllm::situ_and_mul_kernel<scalar_t><<<grid, block, 0, stream>>>(
out.mutable_data_ptr<scalar_t>(), input.const_data_ptr<scalar_t>(),
d, (float)beta, (float)linear_beta, valid_rows_ptr);
d, (float)beta, (float)linear_beta);
});
}
@@ -925,7 +1066,7 @@ void masked_situ_and_mul(torch::stable::Tensor& out, // [E, T, d]
int num_experts = input.size(0);
int max_num_tokens = input.size(1);
int d = input.size(2) / 2;
if (num_experts == 0 || max_num_tokens == 0) {
if (num_experts == 0 || max_num_tokens == 0 || d == 0) {
return;
}
constexpr int block_size = 256;
@@ -936,21 +1077,128 @@ void masked_situ_and_mul(torch::stable::Tensor& out, // [E, T, d]
const cudaStream_t stream = get_current_cuda_stream();
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
input.scalar_type(), "masked_situ_and_mul_kernel", [&] {
vllm::masked_situ_and_mul_kernel<scalar_t><<<grid, block, 0, stream>>>(
out.mutable_data_ptr<scalar_t>(), input.const_data_ptr<scalar_t>(),
expert_num_tokens.const_data_ptr<int>(), max_num_tokens, d,
(float)beta, (float)linear_beta);
vllm::masked_situ_and_mul_kernel<scalar_t, true>
<<<grid, block, 0, stream>>>(
out.mutable_data_ptr<scalar_t>(),
input.const_data_ptr<scalar_t>(),
expert_num_tokens.const_data_ptr<int>(), max_num_tokens, d,
(float)beta, (float)linear_beta);
});
}
#define LAUNCH_MASKED_ACT_AND_MUL(KERNEL, ACT_FIRST, HAS_CLAMP, LIMIT, ALPHA, \
BETA) \
vllm::masked_act_and_mul_kernel<scalar_t, KERNEL<scalar_t>, ACT_FIRST, \
HAS_CLAMP, BATCHED_EXPERTS> \
<<<grid, block, 0, stream>>>(out.mutable_data_ptr<scalar_t>(), \
input.const_data_ptr<scalar_t>(), \
valid_token_counts.const_data_ptr<int>(), \
max_num_tokens, d, LIMIT, ALPHA, BETA)
#define LAUNCH_MASKED_ACTIVATION(KERNEL) \
vllm::masked_activation_kernel<scalar_t, KERNEL<scalar_t>, BATCHED_EXPERTS> \
<<<grid, block, 0, stream>>>( \
out.mutable_data_ptr<scalar_t>(), input.const_data_ptr<scalar_t>(), \
valid_token_counts.const_data_ptr<int>(), max_num_tokens, d)
void masked_moe_activation(
torch::stable::Tensor& out, // [T, d] or [E, T, d]
torch::stable::Tensor& input, // [T, d] or [E, T, d], gated has 2 * d
const torch::stable::Tensor& valid_token_counts,
const std::string& activation, double clamp_limit, double alpha,
double beta, double situ_beta, double situ_linear_beta) {
const bool batched_experts = input.dim() == 3;
const int num_experts = batched_experts ? input.size(0) : 1;
const int max_num_tokens = batched_experts ? input.size(1) : input.size(0);
const int d = out.size(-1);
if (num_experts == 0 || max_num_tokens == 0 || d == 0) {
return;
}
constexpr int block_size = 256;
const int feature_blocks = (d + block_size - 1) / block_size;
const int token_blocks =
std::min(max_num_tokens, vllm::kMaxMaskedTokenBlocks);
// Batched grid: (feature tile, expert, token lane); flat grid: (feature
// tile, token lane). Token lanes grid-stride each valid prefix.
dim3 grid = batched_experts ? dim3(feature_blocks, num_experts, token_blocks)
: dim3(feature_blocks, token_blocks);
dim3 block(block_size);
const torch::stable::accelerator::DeviceGuard device_guard(
input.get_device_index());
const cudaStream_t stream = get_current_cuda_stream();
auto launch = [&]<bool BATCHED_EXPERTS>() {
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
input.scalar_type(), "masked_moe_activation", [&] {
if (activation == "silu") {
LAUNCH_MASKED_ACT_AND_MUL(vllm::silu_kernel, true, false, 0.0f,
1.0f, 0.0f);
} else if (activation == "silu_with_clamp") {
LAUNCH_MASKED_ACT_AND_MUL(vllm::silu_kernel, true, true,
(float)clamp_limit, (float)alpha,
(float)beta);
} else if (activation == "gelu") {
LAUNCH_MASKED_ACT_AND_MUL(vllm::gelu_kernel, true, false, 0.0f,
1.0f, 0.0f);
} else if (activation == "gelu_tanh") {
LAUNCH_MASKED_ACT_AND_MUL(vllm::gelu_tanh_kernel, true, false, 0.0f,
1.0f, 0.0f);
} else if (activation == "situ") {
vllm::masked_situ_and_mul_kernel<scalar_t, BATCHED_EXPERTS>
<<<grid, block, 0, stream>>>(
out.mutable_data_ptr<scalar_t>(),
input.const_data_ptr<scalar_t>(),
valid_token_counts.const_data_ptr<int>(), max_num_tokens, d,
(float)situ_beta, (float)situ_linear_beta);
} else if (activation == "swigluoai") {
vllm::masked_swigluoai_and_mul_kernel<scalar_t, BATCHED_EXPERTS>
<<<grid, block, 0, stream>>>(
out.mutable_data_ptr<scalar_t>(),
input.const_data_ptr<scalar_t>(),
valid_token_counts.const_data_ptr<int>(), max_num_tokens, d,
(float)alpha, (float)clamp_limit);
} else if (activation == "swigluoai_uninterleave") {
LAUNCH_MASKED_ACT_AND_MUL(vllm::silu_kernel, true, true,
(float)clamp_limit, (float)alpha,
(float)beta);
} else if (activation == "swiglustep") {
vllm::masked_swiglustep_and_mul_kernel<scalar_t, BATCHED_EXPERTS>
<<<grid, block, 0, stream>>>(
out.mutable_data_ptr<scalar_t>(),
input.const_data_ptr<scalar_t>(),
valid_token_counts.const_data_ptr<int>(), max_num_tokens, d,
(float)clamp_limit);
} else if (activation == "silu_no_mul") {
LAUNCH_MASKED_ACTIVATION(vllm::silu_no_mul_kernel);
} else if (activation == "gelu_no_mul") {
LAUNCH_MASKED_ACTIVATION(vllm::gelu_no_mul_kernel);
} else if (activation == "gelu_tanh_no_mul") {
LAUNCH_MASKED_ACTIVATION(vllm::gelu_tanh_no_mul_kernel);
} else if (activation == "relu2_no_mul") {
LAUNCH_MASKED_ACTIVATION(vllm::relu2_no_mul_kernel);
} else {
STD_TORCH_CHECK(false,
"Unsupported masked MoE activation: ", activation);
}
});
};
if (batched_experts) {
launch.template operator()<true>();
} else {
launch.template operator()<false>();
}
}
#undef LAUNCH_MASKED_ACT_AND_MUL
#undef LAUNCH_MASKED_ACTIVATION
// Fused SITU activation and block-FP8 quantization for Humming w2.
// `valid_rows` is the int32 DeepEP token count (psum[-1]); rows = it * `topk`,
// so padding is excluded on-device and skipped rows receive scale 1.
// `num_valid_tokens` is the int32 DeepEP token count (psum[-1]); rows = it *
// `topk`, so padding is excluded on-device and skipped rows receive scale 1.
void situ_and_mul_quant(torch::stable::Tensor& out, // [..., d] (fp8)
torch::stable::Tensor& scale, // [..., 1 or d/128]
torch::stable::Tensor& input, // [..., 2 * d]
double beta, double linear_beta, int64_t group_size,
std::optional<torch::stable::Tensor> valid_rows,
std::optional<torch::stable::Tensor> num_valid_tokens,
int64_t topk) {
STD_TORCH_CHECK(
out.scalar_type() == torch::headeronly::ScalarType::Float8_e4m3fn ||
@@ -974,16 +1222,17 @@ void situ_and_mul_quant(torch::stable::Tensor& out, // [..., d] (fp8)
}
STD_TORCH_CHECK(out.size(-1) == d && out.numel() == num_tokens * (int64_t)d,
"situ_and_mul_quant: out shape must be [num_tokens, d]");
const int32_t* valid_rows_ptr = nullptr;
if (valid_rows.has_value()) {
STD_TORCH_CHECK(valid_rows->is_cuda() && valid_rows->get_device_index() ==
input.get_device_index(),
"situ_and_mul_quant: valid_rows must be a CUDA tensor on "
"the same device as input");
const int32_t* num_valid_tokens_ptr = nullptr;
if (num_valid_tokens.has_value()) {
STD_TORCH_CHECK(
valid_rows->scalar_type() == torch::headeronly::ScalarType::Int,
"situ_and_mul_quant: valid_rows must be int32 (psum count)");
valid_rows_ptr = valid_rows->const_data_ptr<int32_t>();
num_valid_tokens->is_cuda() &&
num_valid_tokens->get_device_index() == input.get_device_index(),
"situ_and_mul_quant: num_valid_tokens must be a CUDA tensor on the "
"same device as input");
STD_TORCH_CHECK(
num_valid_tokens->scalar_type() == torch::headeronly::ScalarType::Int,
"situ_and_mul_quant: num_valid_tokens must be int32 (psum count)");
num_valid_tokens_ptr = num_valid_tokens->const_data_ptr<int32_t>();
}
const torch::stable::accelerator::DeviceGuard device_guard(
input.get_device_index());
@@ -1031,7 +1280,7 @@ void situ_and_mul_quant(torch::stable::Tensor& out, // [..., d] (fp8)
out.mutable_data_ptr<fp8_t>(),
scale.mutable_data_ptr<float>(),
input.const_data_ptr<scalar_t>(), num_tokens,
valid_rows_ptr, topk);
num_valid_tokens_ptr, topk);
return;
}
}
@@ -1045,7 +1294,7 @@ void situ_and_mul_quant(torch::stable::Tensor& out, // [..., d] (fp8)
scale.mutable_data_ptr<float>(),
input.const_data_ptr<scalar_t>(), d, num_groups,
(float)beta, (float)linear_beta, num_tokens,
valid_rows_ptr, topk);
num_valid_tokens_ptr, topk);
});
});
}
@@ -146,16 +146,17 @@ __global__ void preprocessTopkIdKernel(int* topk_id_ptr, int size,
}
__syncthreads();
// query global expert id in expert map.
// if global expert id = -1 in exert map, plus n_expert
// else set global expert id = exert map[global expert id]
if (offset + tidx < bound) {
auto topk_id = topk_id_ptr[offset + tidx];
auto local_expert_idx = smem_expert_map[topk_id];
if (local_expert_idx == -1) {
topk_id += num_experts;
if (topk_id < 0 || topk_id >= num_experts) {
topk_id = num_experts;
} else {
topk_id = local_expert_idx;
auto local_expert_idx = smem_expert_map[topk_id];
if (local_expert_idx == -1) {
topk_id += num_experts;
} else {
topk_id = local_expert_idx;
}
}
__syncwarp();
topk_id_ptr[offset + tidx] = topk_id;
+10 -5
View File
@@ -530,20 +530,25 @@ void fatrelu_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input,
double threshold);
void swigluoai_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input,
double alpha = 1.702, double limit = 7.0);
void situ_and_mul(
torch::stable::Tensor& out, torch::stable::Tensor& input, double beta = 1.0,
double linear_beta = -1.0,
std::optional<torch::stable::Tensor> valid_rows = std::nullopt);
void situ_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input,
double beta = 1.0, double linear_beta = -1.0);
void situ_and_mul_quant(
torch::stable::Tensor& out, torch::stable::Tensor& scale,
torch::stable::Tensor& input, double beta = 1.0, double linear_beta = -1.0,
int64_t group_size = 0,
std::optional<torch::stable::Tensor> valid_rows = std::nullopt,
std::optional<torch::stable::Tensor> num_valid_tokens = std::nullopt,
int64_t topk = 1);
void masked_situ_and_mul(torch::stable::Tensor& out,
torch::stable::Tensor& input,
const torch::stable::Tensor& expert_num_tokens,
double beta = 1.0, double linear_beta = -1.0);
void masked_moe_activation(torch::stable::Tensor& out,
torch::stable::Tensor& input,
const torch::stable::Tensor& valid_token_counts,
const std::string& activation,
double clamp_limit = 0.0, double alpha = 1.0,
double beta = 0.0, double situ_beta = 1.0,
double situ_linear_beta = -1.0);
void gelu_new(torch::stable::Tensor& out, torch::stable::Tensor& input);
void gelu_fast(torch::stable::Tensor& out, torch::stable::Tensor& input);
void gelu_quick(torch::stable::Tensor& out, torch::stable::Tensor& input);
+8 -2
View File
@@ -614,7 +614,7 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
// SituGLU implementation used in Kimi models.
ops.def(
"situ_and_mul(Tensor! out, Tensor input, float beta=1.0, float "
"linear_beta=-1.0, Tensor? valid_rows=None) -> ()");
"linear_beta=-1.0) -> ()");
// Fused SituGLU activation + dynamic FP8 quantization for the Humming w2 path
// (writes the fp8 down input and its float32 scale). group_size=0 ->
// per-token scale [.., 1]; group_size=128 -> k-major block-FP8 scale [..,
@@ -622,10 +622,15 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
ops.def(
"situ_and_mul_quant(Tensor! out, Tensor! scale, Tensor input, "
"float beta=1.0, float linear_beta=-1.0, int group_size=0, "
"Tensor? valid_rows=None, int topk=1) -> ()");
"Tensor? num_valid_tokens=None, int topk=1) -> ()");
ops.def(
"masked_situ_and_mul(Tensor! out, Tensor input, Tensor "
"expert_num_tokens, float beta=1.0, float linear_beta=-1.0) -> ()");
ops.def(
"masked_moe_activation(Tensor! out, Tensor input, Tensor "
"valid_token_counts, str activation, float clamp_limit=0.0, float "
"alpha=1.0, float beta=0.0, float situ_beta=1.0, float "
"situ_linear_beta=-1.0) -> ()");
// GELU implementation used in GPT-2.
ops.def("gelu_new(Tensor! out, Tensor input) -> ()");
@@ -852,6 +857,7 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
ops.impl("situ_and_mul", TORCH_BOX(&situ_and_mul));
ops.impl("situ_and_mul_quant", TORCH_BOX(&situ_and_mul_quant));
ops.impl("masked_situ_and_mul", TORCH_BOX(&masked_situ_and_mul));
ops.impl("masked_moe_activation", TORCH_BOX(&masked_moe_activation));
ops.impl("gelu_new", TORCH_BOX(&gelu_new));
ops.impl("gelu_fast", TORCH_BOX(&gelu_fast));
ops.impl("gelu_quick", TORCH_BOX(&gelu_quick));
+208
View File
@@ -22,7 +22,13 @@ from vllm.model_executor.layers.activation import (
SwigluStepAndMul,
swiglustep_and_mul_triton,
)
from vllm.model_executor.layers.fused_moe.activation import (
ApplyMoEActivationConfig,
MoEActivation,
apply_moe_activation,
)
from vllm.model_executor.layers.fused_moe.utils import swiglu_limit_func
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
DTYPES = [torch.half, torch.bfloat16, torch.float]
@@ -34,6 +40,100 @@ CUDA_DEVICES = [
]
def test_masked_moe_activation_rejects_unsupported_activation() -> None:
input = torch.empty(1, 1, 2)
output = torch.empty(1, 1, 1)
valid_token_counts = torch.ones(1, dtype=torch.int32)
with pytest.raises(NotImplementedError, match="relu2"):
apply_moe_activation(
MoEActivation.RELU2,
output,
input,
valid_token_counts=valid_token_counts,
)
def test_moe_silu_clamp_uses_native_xpu_fallback(
default_vllm_config, monkeypatch
) -> None:
monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
clamp_limit = 3.0
input = torch.tensor([[12.0, -12.0, 8.0, -8.0], [-2.0, 2.0, -4.0, 4.0]])
output = torch.empty(2, 2)
apply_moe_activation(
MoEActivation.SILU,
output,
input,
activation_config=ApplyMoEActivationConfig(clamp_limit=clamp_limit),
)
expected = SiluAndMulWithClamp(clamp_limit, compile_native=False).forward_native(
input
)
torch.testing.assert_close(output, expected)
def _assert_masked_moe_activation(
activation: MoEActivation,
activation_config: ApplyMoEActivationConfig,
*,
dtype: torch.dtype,
mask_layout: str,
d: int,
max_num_tokens: int,
) -> None:
device = CUDA_DEVICES[0]
num_experts = 4
input_dim = 2 * d if activation.is_gated else d
if mask_layout == "flat":
input = torch.randn(max_num_tokens, input_dim, dtype=dtype, device=device)
valid_token_counts = torch.tensor(
[max_num_tokens // 2], dtype=torch.int32, device=device
)
output = torch.full((max_num_tokens, d), 42.0, dtype=dtype, device=device)
else:
input = torch.randn(
num_experts, max_num_tokens, input_dim, dtype=dtype, device=device
)
valid_token_counts = torch.tensor(
[0, 1, max_num_tokens // 2, max_num_tokens],
dtype=torch.int32,
device=device,
)
output = torch.full(
(num_experts, max_num_tokens, d), 42.0, dtype=dtype, device=device
)
apply_moe_activation(
activation,
output,
input,
activation_config=activation_config,
valid_token_counts=valid_token_counts,
)
batched_input = input.view(-1, max_num_tokens, input_dim)
batched_output = output.view(-1, max_num_tokens, d)
for expert, num_tokens in enumerate(valid_token_counts.cpu().tolist()):
if num_tokens:
expected = torch.empty((num_tokens, d), dtype=dtype, device=device)
apply_moe_activation(
activation,
expected,
batched_input[expert, :num_tokens].clone(),
activation_config=activation_config,
)
torch.testing.assert_close(
batched_output[expert, :num_tokens],
expected,
atol=get_default_atol(output),
rtol=get_default_rtol(output),
)
assert torch.all(batched_output[expert, num_tokens:] == 42.0)
@pytest.mark.parametrize(
"activation",
[
@@ -252,6 +352,114 @@ def test_masked_situ_and_mul(
)
@pytest.mark.parametrize(
("activation", "activation_config"),
[
(MoEActivation.SILU, ApplyMoEActivationConfig()),
(
MoEActivation.SILU,
ApplyMoEActivationConfig(clamp_limit=3.0),
),
(MoEActivation.GELU, ApplyMoEActivationConfig()),
(MoEActivation.GELU_TANH, ApplyMoEActivationConfig()),
(
MoEActivation.SITU,
ApplyMoEActivationConfig(
activation_situ_beta=1.5,
activation_situ_linear_beta=2.0,
),
),
(MoEActivation.SWIGLUOAI, ApplyMoEActivationConfig()),
(
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
ApplyMoEActivationConfig(clamp_limit=3.0, alpha=1.3, beta=0.5),
),
(MoEActivation.SWIGLUSTEP, ApplyMoEActivationConfig()),
(MoEActivation.SILU_NO_MUL, ApplyMoEActivationConfig()),
(MoEActivation.GELU_NO_MUL, ApplyMoEActivationConfig()),
(MoEActivation.GELU_TANH_NO_MUL, ApplyMoEActivationConfig()),
(MoEActivation.RELU2_NO_MUL, ApplyMoEActivationConfig()),
],
ids=[
"silu",
"silu_clamp",
"gelu",
"gelu_tanh",
"situ",
"swigluoai",
"swigluoai_uninterleave",
"swiglustep",
"silu_no_mul",
"gelu_no_mul",
"gelu_tanh_no_mul",
"relu2_no_mul",
],
)
@torch.inference_mode()
def test_masked_moe_activation_dispatch(
default_vllm_config,
activation: MoEActivation,
activation_config: ApplyMoEActivationConfig,
) -> None:
_assert_masked_moe_activation(
activation,
activation_config,
dtype=torch.bfloat16,
mask_layout="batched_experts",
d=513,
max_num_tokens=7,
)
@pytest.mark.parametrize(
("activation", "activation_config", "mask_layout"),
[
pytest.param(
MoEActivation.SILU,
ApplyMoEActivationConfig(),
"flat",
id="flat",
),
pytest.param(
MoEActivation.SITU,
ApplyMoEActivationConfig(
activation_situ_beta=1.5,
activation_situ_linear_beta=2.0,
),
"batched_experts",
id="batched-experts",
),
],
)
@torch.inference_mode()
def test_masked_moe_activation_grid_stride(
default_vllm_config,
activation: MoEActivation,
activation_config: ApplyMoEActivationConfig,
mask_layout: str,
) -> None:
_assert_masked_moe_activation(
activation,
activation_config,
dtype=torch.half,
mask_layout=mask_layout,
d=513,
max_num_tokens=67,
)
@torch.inference_mode()
def test_masked_moe_activation_opcheck(default_vllm_config) -> None:
device = CUDA_DEVICES[0]
input = torch.randn(2, 3, 64, dtype=torch.half, device=device)
output = torch.empty(2, 3, 32, dtype=torch.half, device=device)
valid_token_counts = torch.tensor([1, 3], dtype=torch.int32, device=device)
opcheck(
torch.ops._C.masked_moe_activation,
(output, input, valid_token_counts, "silu", 0.0, 1.0, 0.0, 1.0, -1.0),
)
@pytest.mark.parametrize(
"activation",
[
+145 -3
View File
@@ -8,6 +8,7 @@ import torch
from tests.kernels.moe.utils import (
batched_moe,
make_dummy_moe_config,
make_quantized_test_activations,
make_test_weights,
naive_batched_moe,
@@ -16,7 +17,13 @@ from tests.kernels.quant_utils import native_batched_masked_quant_matmul
from tests.kernels.utils import torch_experts
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe.activation import (
ApplyMoEActivationConfig,
MoEActivation,
apply_moe_activation,
)
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
BatchedTritonExperts,
invoke_moe_batched_triton_kernel,
)
from vllm.platforms import current_platform
@@ -496,11 +503,146 @@ def test_batched_mm_td_zero_expert_tokens():
# BatchedTritonExperts device enablement (XPU)
def test_batched_triton_experts_supports_current_device():
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
BatchedTritonExperts,
assert BatchedTritonExperts._supports_current_device()
@pytest.mark.parametrize(
"activation",
[
MoEActivation.SITU,
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
MoEActivation.SWIGLUSTEP,
],
)
def test_batched_triton_supports_masked_activations(
monkeypatch, activation: MoEActivation
):
monkeypatch.setattr(current_platform, "is_xpu", lambda: False)
assert BatchedTritonExperts._supports_activation(activation)
monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
assert not BatchedTritonExperts._supports_activation(activation)
def _torch_experts_with_shared_activation(
a: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: MoEActivation,
activation_config: ApplyMoEActivationConfig,
) -> torch.Tensor:
num_tokens, hidden_dim = a.shape
topk = topk_ids.shape[1]
routed_input = a[:, None, :].expand(-1, topk, -1).reshape(-1, hidden_dim)
routed_experts = topk_ids.reshape(-1)
routed_output = torch.zeros_like(routed_input)
for expert in range(w1.shape[0]):
expert_mask = routed_experts == expert
if not expert_mask.any():
continue
projected = routed_input[expert_mask] @ w1[expert].T
activated = torch.empty(
projected.shape[0], w2.shape[2], dtype=a.dtype, device=a.device
)
apply_moe_activation(
activation,
activated,
projected,
activation_config=activation_config,
)
routed_output[expert_mask] = activated @ w2[expert].T
return (
(
routed_output.view(num_tokens, topk, hidden_dim).float()
* topk_weights[..., None]
)
.sum(dim=1)
.to(a.dtype)
)
assert BatchedTritonExperts._supports_current_device()
_MASKED_BATCHED_ACTIVATION_CASES = [
pytest.param(
MoEActivation.SITU,
ApplyMoEActivationConfig(
activation_situ_beta=1.5,
activation_situ_linear_beta=2.0,
),
id="situ",
),
pytest.param(
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
ApplyMoEActivationConfig(clamp_limit=3.0, alpha=1.3, beta=0.5),
id="swigluoai-uninterleave",
),
pytest.param(
MoEActivation.SWIGLUSTEP,
ApplyMoEActivationConfig(),
id="swiglustep",
),
]
@pytest.mark.parametrize(
("activation", "activation_config"), _MASKED_BATCHED_ACTIVATION_CASES
)
@torch.inference_mode()
def test_batched_experts_masked_activation_end_to_end(
activation: MoEActivation,
activation_config: ApplyMoEActivationConfig,
):
if current_platform.is_xpu():
pytest.skip("Masked activations are not enabled on XPU")
from vllm.v1.worker.workspace import init_workspace_manager
set_random_seed(7)
device = current_platform.device_type
init_workspace_manager(torch.device(f"{device}:0"))
m, n, k, e, topk = 16, 128, 128, 8, 2
a = torch.randn((m, k), device=device, dtype=torch.bfloat16) / 10
score = torch.randn((m, e), device=device, dtype=torch.bfloat16)
w1 = torch.randn((e, 2 * n, k), device=device, dtype=torch.bfloat16) / 15
w2 = torch.randn((e, k, n), device=device, dtype=torch.bfloat16) / 15
moe_config = make_dummy_moe_config(
num_experts=e,
experts_per_token=topk,
hidden_dim=k,
intermediate_size=n,
in_dtype=a.dtype,
activation=activation,
)
moe_config.swiglu_limit = activation_config.clamp_limit
moe_config.swiglu_alpha = activation_config.alpha
moe_config.swiglu_beta = activation_config.beta
moe_config.activation_situ_beta = activation_config.activation_situ_beta
moe_config.activation_situ_linear_beta = (
activation_config.activation_situ_linear_beta
)
routing_weights = torch.softmax(score.float(), dim=-1)
topk_weights, topk_ids = torch.topk(routing_weights, topk, dim=-1)
topk_ids = topk_ids.to(torch.int32)
with set_current_vllm_config(vllm_config):
baseline_output = _torch_experts_with_shared_activation(
a,
w1,
w2,
topk_weights,
topk_ids,
activation,
activation_config,
)
triton_output = batched_moe(
a, w1, w2, topk_weights, topk_ids, moe_config=moe_config
)
torch.testing.assert_close(triton_output, baseline_output, atol=3e-2, rtol=2e-2)
@pytest.mark.parametrize("m,n,k,e,topk", [(32, 512, 512, 8, 2), (45, 1024, 128, 8, 1)])
+260 -1
View File
@@ -1377,6 +1377,102 @@ def test_humming_activation_metadata_tracks_shared_apply(activation: MoEActivati
) == apply_moe_activation_supported(activation)
@pytest.mark.parametrize(
("env_override", "use_ep", "expected"),
[
(None, False, "indexed"),
(None, True, "grouped_contiguous"),
("auto", True, "grouped_contiguous"),
("indexed", True, "indexed"),
("grouped", False, "grouped_contiguous"),
],
)
def test_humming_selects_gemm_from_parallelism_and_override(
monkeypatch: pytest.MonkeyPatch,
env_override: str | None,
use_ep: bool,
expected: str,
):
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
get_humming_moe_gemm_type,
)
if env_override is None:
monkeypatch.delenv("VLLM_HUMMING_MOE_GEMM_TYPE", raising=False)
else:
monkeypatch.setenv("VLLM_HUMMING_MOE_GEMM_TYPE", env_override)
moe_config = make_dummy_moe_config()
moe_config.moe_parallel_config.use_ep = use_ep
assert get_humming_moe_gemm_type(moe_config) == expected
@pytest.mark.parametrize(
("dp_token_counts", "topk_shape", "expected"),
[
pytest.param(None, (7, 3), 21, id="runtime-layout"),
pytest.param([3, 5], (48, 1), 48, id="expanded-dp-layout"),
],
)
def test_humming_global_valid_shape_m(
monkeypatch: pytest.MonkeyPatch,
dp_token_counts: list[int] | None,
topk_shape: tuple[int, int],
expected: int,
):
from types import SimpleNamespace
import vllm.model_executor.layers.fused_moe.experts.fused_humming_moe as humming
dp_metadata = (
None
if dp_token_counts is None
else SimpleNamespace(
num_tokens_across_dp_cpu=torch.tensor(dp_token_counts, dtype=torch.int32)
)
)
monkeypatch.setattr(
humming,
"get_forward_context",
lambda: SimpleNamespace(dp_metadata=dp_metadata),
)
experts = SimpleNamespace(moe_config=make_dummy_moe_config(experts_per_token=6))
topk_ids = torch.empty(topk_shape, dtype=torch.int64)
result = humming.HummingExpertsBase.get_global_valid_shape_m(experts, topk_ids)
assert result == expected
def test_humming_permute_scratch_is_keyed_by_runtime_topk(
monkeypatch: pytest.MonkeyPatch,
):
from types import SimpleNamespace
from unittest.mock import Mock
import vllm.model_executor.layers.fused_moe.experts.fused_humming_moe as humming
scratch_topk6 = Mock()
scratch_topk1 = Mock()
scratch_type = Mock(side_effect=[scratch_topk6, scratch_topk1])
monkeypatch.setattr(humming, "moe_permute_unpermute_supported", lambda: True)
monkeypatch.setattr(humming, "MoEPermuteScratch", scratch_type)
moe_config = make_dummy_moe_config(max_num_tokens=512, experts_per_token=6)
moe_config.moe_parallel_config.dp_size = 2
experts = SimpleNamespace(_permute_scratch={}, moe_config=moe_config)
assert humming.HummingExpertsBase._get_permute_scratch(experts, 6) is scratch_topk6
assert humming.HummingExpertsBase._get_permute_scratch(experts, 1) is scratch_topk1
assert humming.HummingExpertsBase._get_permute_scratch(experts, 6) is scratch_topk6
assert scratch_type.call_count == 2
first_call, second_call = scratch_type.call_args_list
assert first_call.kwargs["max_num_tokens"] == 1024
assert first_call.kwargs["topk"] == 6
assert second_call.kwargs["max_num_tokens"] == 6144
assert second_call.kwargs["topk"] == 1
def test_humming_delegates_to_instance_activation():
from types import SimpleNamespace
from unittest.mock import Mock
@@ -1408,10 +1504,173 @@ def test_humming_delegates_to_instance_activation():
activation=MoEActivation.SWIGLUOAI_UNINTERLEAVE,
input=input,
output=output,
valid_rows=None,
)
def test_humming_grouped_apply_forwards_valid_prefix(
monkeypatch: pytest.MonkeyPatch,
):
from types import MethodType, SimpleNamespace
from unittest.mock import Mock
import vllm.model_executor.layers.fused_moe.experts.fused_humming_moe as humming
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
HummingExpertsBase,
HummingGroupedExperts,
)
activation_func = Mock()
buffers = {
"gate_up_output": torch.empty(6, 4),
"activation_output": torch.empty(6, 2),
"down_output": torch.empty(6, 4),
}
experts = SimpleNamespace(
activation=activation_func,
num_experts=2,
estimate_local_valid_shape_m=lambda _: 6,
prepare_buffers=lambda *_: buffers,
_get_permute_scratch=lambda _: None,
quantize_input=lambda _, *, inputs, input_scale=None, quanted_input: (
inputs,
input_scale,
),
humming_forward=Mock(),
compute_config_str="",
w13_tuning_config_str="",
w2_tuning_config_str="",
)
experts.apply_activation = MethodType(HummingExpertsBase.apply_activation, experts)
hidden_states = torch.empty(3, 4)
output = torch.empty_like(hidden_states)
topk_ids = torch.zeros((3, 2), dtype=torch.int64)
topk_weights = torch.ones_like(topk_ids, dtype=torch.float32)
expected_counts = torch.tensor([4], dtype=torch.int32)
expert_offsets = torch.tensor([0, 2, 4], dtype=torch.int64)
monkeypatch.setattr(
humming,
"moe_permute",
lambda **_: (
torch.empty(6, 4),
None,
expert_offsets,
torch.arange(6),
None,
),
)
monkeypatch.setattr(humming, "moe_unpermute", lambda **_: None)
HummingGroupedExperts.apply(
experts,
output=output,
hidden_states=hidden_states,
w1=torch.empty(0),
w2=torch.empty(0),
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=MoEActivation.SITU,
global_num_experts=2,
expert_map=None,
a1q_scale=None,
a2_scale=None,
workspace13=torch.empty(0),
workspace2=torch.empty(0),
expert_tokens_meta=None,
apply_router_weight_on_input=False,
)
activation_func.assert_called_once()
call_kwargs = activation_func.call_args.kwargs
assert call_kwargs["activation"] == MoEActivation.SITU
assert call_kwargs["input"].shape == (6, 4)
assert call_kwargs["output"].shape == (6, 2)
torch.testing.assert_close(call_kwargs["valid_token_counts"], expected_counts)
def test_batched_marlin_activation_uses_expert_token_counts(
monkeypatch: pytest.MonkeyPatch,
):
from types import SimpleNamespace
from unittest.mock import Mock
import vllm.model_executor.layers.fused_moe.experts.marlin_moe as marlin
from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
BatchedMarlinExperts,
)
from vllm.model_executor.layers.fused_moe.modular_kernel import (
ExpertTokensMetadata,
)
activation_func = Mock()
experts = SimpleNamespace(
activation=activation_func,
w1_bias=None,
w2_bias=None,
w1_scale=None,
w2_scale=None,
quant_type_id=0,
a1_gscale=None,
a2_gscale=None,
w13_g_idx=None,
w2_g_idx=None,
w13_g_idx_sort_indices=None,
w2_g_idx_sort_indices=None,
w1_zp=None,
w2_zp=None,
input_dtype=None,
is_k_full=True,
activation_config=ApplyMoEActivationConfig(
clamp_limit=3.0, alpha=1.3, beta=0.5
),
)
hidden_states = torch.empty(2, 3, 4)
expert_num_tokens = torch.tensor([1, 2], dtype=torch.int32)
topk_ids = torch.zeros((2, 3), dtype=torch.int64)
expert_map = torch.arange(2, dtype=torch.int32)
def fake_batched_fused_marlin_moe(**kwargs):
kwargs["activation_func"](
kwargs["activation"],
torch.empty(6, 4),
torch.empty(6, 8),
topk_ids=topk_ids,
expert_map=expert_map,
)
return kwargs["output"]
monkeypatch.setattr(
marlin, "batched_fused_marlin_moe", fake_batched_fused_marlin_moe
)
output = torch.empty_like(hidden_states)
BatchedMarlinExperts.apply(
experts,
output=output,
hidden_states=hidden_states,
w1=torch.empty(0),
w2=torch.empty(0),
topk_weights=torch.ones_like(topk_ids, dtype=torch.float32),
topk_ids=topk_ids,
activation=MoEActivation.SWIGLUOAI_UNINTERLEAVE,
global_num_experts=2,
expert_map=expert_map,
a1q_scale=None,
a2_scale=None,
workspace13=torch.empty(0),
workspace2=torch.empty(0),
expert_tokens_meta=ExpertTokensMetadata(expert_num_tokens, None),
apply_router_weight_on_input=False,
)
activation_func.assert_called_once()
call = activation_func.call_args
assert call.args[0] == MoEActivation.SWIGLUOAI_UNINTERLEAVE
assert call.args[1].shape == (2, 3, 4)
assert call.args[2].shape == (2, 3, 8)
torch.testing.assert_close(call.kwargs["valid_token_counts"], expert_num_tokens)
@pytest.mark.parametrize(
"activation",
[
@@ -286,3 +286,39 @@ def test_moe_permute_reuses_scratch_buffers(dtype: torch.dtype):
permuted_idx_1.untyped_storage().data_ptr()
== scratch.permuted_idx.untyped_storage().data_ptr()
)
def test_moe_permute_ignores_invalid_expert_ids_with_scratch() -> None:
if not moe_permute_unpermute_supported():
pytest.skip("moe_permute_unpermute is not supported on this platform.")
hidden_states = torch.arange(5 * 16, dtype=torch.bfloat16, device="cuda").view(
5, 16
)
topk_ids = torch.tensor([[0], [-1], [1], [4], [2]], device="cuda")
expert_map = torch.tensor([0, 1, -1, -1], dtype=torch.int32, device="cuda")
scratch = MoEPermuteScratch(
max_num_tokens=5,
topk=1,
num_experts=4,
num_local_experts=2,
device=hidden_states.device,
hidden_size=16,
hidden_dtype=hidden_states.dtype,
)
permuted, _, expert_offsets, _, _ = moe_permute(
hidden_states=hidden_states,
a1q_scale=None,
topk_ids=topk_ids,
n_expert=4,
n_local_expert=2,
expert_map=expert_map,
scratch=scratch,
)
torch.testing.assert_close(
expert_offsets,
torch.tensor([0, 1, 2], dtype=torch.int64, device="cuda"),
)
torch.testing.assert_close(permuted[:2], hidden_states[[0, 2]])
+7 -7
View File
@@ -5,7 +5,7 @@ quant kernel (``situ_and_mul_quant``).
Mirrors ``test_silu_mul_fp8_quant_deep_gemm.py`` in structure: build a reference
activation + block-FP8 quant, drive random per-row scale magnitudes, exercise
the masked (``valid_rows``) contiguous layout, and compare per valid row.
the masked contiguous layout, and compare per valid row.
The kernel computes SITU in fp32 and converts to fp8 with hardware rounding,
which torch cannot reproduce bit-exactly. It is therefore validated against the
@@ -40,7 +40,7 @@ SITU_LINEAR_BETA = 25.0
SITU_D = 3072
GROUP_SIZE = 128
# (num_tokens, valid_rows). valid_rows=None => all rows valid. 2048/4096 rows
# (num_tokens, num_valid_tokens). None means all rows are valid. 2048/4096 rows
# span multiple grid-stride waves of the persistent grid (GRID_DIM = 132*8).
CASES = [
(1, None),
@@ -111,7 +111,7 @@ def test_situ_and_mul_quant_pipelined(
(num_tokens, d // GROUP_SIZE), -1.0, dtype=torch.float32, device=DEVICE
)
out_sentinel = out.clone()
valid_rows = (
num_valid_tokens = (
None if valid is None else torch.tensor(valid, dtype=torch.int32, device=DEVICE)
)
@@ -122,7 +122,7 @@ def test_situ_and_mul_quant_pipelined(
beta=SITU_BETA,
linear_beta=SITU_LINEAR_BETA,
group_size=GROUP_SIZE,
valid_rows=valid_rows,
num_valid_tokens=num_valid_tokens,
)
# Padding contract: skipped rows keep their bytes; their scale becomes 1.0.
@@ -157,7 +157,7 @@ def test_situ_and_mul_quant_pipelined(
@pytest.mark.parametrize("tokens,topk", [(300, 8), (1, 4), (256, 6)])
@torch.inference_mode()
def test_situ_and_mul_quant_topk_row_bound(tokens: int, topk: int) -> None:
"""valid_rows is a token count; the kernel bounds rows at tokens*topk."""
"""The kernel expands num_valid_tokens by topk to bound rows."""
set_random_seed(7)
d = SITU_D
num_tokens = tokens * topk + 37 # padding tail past the valid rows
@@ -168,7 +168,7 @@ def test_situ_and_mul_quant_topk_row_bound(tokens: int, topk: int) -> None:
(num_tokens, d // GROUP_SIZE), -1.0, dtype=torch.float32, device=DEVICE
)
out_sentinel = out.clone()
valid_rows = torch.tensor(tokens, dtype=torch.int32, device=DEVICE)
num_valid_tokens = torch.tensor(tokens, dtype=torch.int32, device=DEVICE)
situ_and_mul_quant(
out,
@@ -177,7 +177,7 @@ def test_situ_and_mul_quant_topk_row_bound(tokens: int, topk: int) -> None:
beta=SITU_BETA,
linear_beta=SITU_LINEAR_BETA,
group_size=GROUP_SIZE,
valid_rows=valid_rows,
num_valid_tokens=num_valid_tokens,
topk=topk,
)
+3 -1
View File
@@ -128,6 +128,7 @@ def batched_moe(
quant_dtype: torch.dtype | None = None,
per_act_token_quant: bool = False,
block_shape: list[int] | None = None,
moe_config: FusedMoEConfig | None = None,
) -> torch.Tensor:
max_num_tokens = round_up(a.shape[0], 64)
@@ -141,7 +142,8 @@ def batched_moe(
a2_scale=a2_scale,
)
moe_config = make_dummy_moe_config()
if moe_config is None:
moe_config = make_dummy_moe_config()
fused_experts = FusedMoEKernel(
BatchedPrepareAndFinalize(
@@ -9,6 +9,8 @@ from typing import TYPE_CHECKING
import torch
import torch.nn.functional as F
from vllm.platforms import current_platform
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
@@ -134,11 +136,31 @@ _APPLY_MOE_ACTIVATIONS = frozenset(
)
_MASKED_MOE_ACTIVATION_NAMES: dict[MoEActivation, str] = {
MoEActivation.SILU: "silu",
MoEActivation.GELU: "gelu",
MoEActivation.GELU_TANH: "gelu_tanh",
MoEActivation.SITU: "situ",
MoEActivation.SWIGLUOAI: "swigluoai",
MoEActivation.SWIGLUOAI_UNINTERLEAVE: "swigluoai_uninterleave",
MoEActivation.SWIGLUSTEP: "swiglustep",
MoEActivation.SILU_NO_MUL: "silu_no_mul",
MoEActivation.GELU_NO_MUL: "gelu_no_mul",
MoEActivation.GELU_TANH_NO_MUL: "gelu_tanh_no_mul",
MoEActivation.RELU2_NO_MUL: "relu2_no_mul",
}
def apply_moe_activation_supported(activation: MoEActivation) -> bool:
"""Whether ``apply_moe_activation`` supports an activation."""
return activation in _APPLY_MOE_ACTIVATIONS
def apply_moe_activation_masked_supported(activation: MoEActivation) -> bool:
"""Whether the masked ``apply_moe_activation`` path supports an activation."""
return activation in _MASKED_MOE_ACTIVATION_NAMES
@dataclass(frozen=True)
class ApplyMoEActivationConfig:
"""Configuration forwarded to ``apply_moe_activation``."""
@@ -177,6 +199,96 @@ class ApplyMoEActivationConfig:
_DEFAULT_APPLY_MOE_ACTIVATION_CONFIG = ApplyMoEActivationConfig()
def _validate_moe_activation_shapes(
activation: MoEActivation,
output: torch.Tensor,
input: torch.Tensor,
expected_dim: int,
) -> None:
assert input.dim() == expected_dim, f"Input must be {expected_dim}D"
assert output.dim() == expected_dim, f"Output must be {expected_dim}D"
assert input.shape[:-1] == output.shape[:-1], (
f"Input/output leading shapes must match: {input.shape} vs {output.shape}"
)
if activation.is_gated:
assert output.size(-1) * 2 == input.size(-1), (
f"{activation.value} expects 2x ratio: "
f"{output.size(-1) * 2} vs {input.size(-1)}"
)
else:
assert output.size(-1) == input.size(-1), (
f"{activation.value} expects equal sizes: "
f"{output.size(-1)} vs {input.size(-1)}"
)
def _apply_moe_activation_masked(
activation: MoEActivation,
output: torch.Tensor,
input: torch.Tensor,
valid_token_counts: torch.Tensor,
config: ApplyMoEActivationConfig,
) -> torch.Tensor:
masked_activation = _MASKED_MOE_ACTIVATION_NAMES.get(activation)
if masked_activation is None:
raise NotImplementedError(
f"Masked MoE activation is not implemented for {activation.value}"
)
assert input.dim() in (2, 3), "Masked input must be 2D or 3D"
_validate_moe_activation_shapes(activation, output, input, expected_dim=input.dim())
assert input.dtype == output.dtype, "Input and output dtypes must match"
assert input.device == output.device, "Input and output devices must match"
assert input.is_contiguous(), "Input must be contiguous"
assert output.is_contiguous(), "Output must be contiguous"
assert valid_token_counts.dtype == torch.int32, (
"valid_token_counts must use torch.int32"
)
assert valid_token_counts.dim() == 1, "valid_token_counts must be 1D"
assert valid_token_counts.device == input.device, (
"valid_token_counts must be on the input device"
)
assert valid_token_counts.is_contiguous(), "valid_token_counts must be contiguous"
expected_counts = input.size(0) if input.dim() == 3 else 1
assert valid_token_counts.size(0) == expected_counts, (
f"valid_token_counts must have {expected_counts} element(s) for "
f"{input.dim()}D input"
)
masked_clamp_limit = 0.0 if config.clamp_limit is None else config.clamp_limit
masked_alpha = config.alpha
if activation == MoEActivation.SILU and config.clamp_limit is not None:
masked_activation = "silu_with_clamp"
elif activation == MoEActivation.SITU:
assert config.activation_situ_beta is not None, (
"SITU requires activation_situ_beta from FusedMoEConfig"
)
elif activation == MoEActivation.SWIGLUOAI:
masked_clamp_limit = 7.0
masked_alpha = 1.702
elif activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
assert config.clamp_limit is not None, (
"SWIGLUOAI_UNINTERLEAVE requires clamp_limit"
)
elif activation == MoEActivation.SWIGLUSTEP:
masked_clamp_limit = 7.0
torch.ops._C.masked_moe_activation(
output,
input,
valid_token_counts,
masked_activation,
masked_clamp_limit,
masked_alpha,
config.beta,
1.0 if config.activation_situ_beta is None else config.activation_situ_beta,
-1.0
if config.activation_situ_linear_beta is None
else config.activation_situ_linear_beta,
)
return output
def silu_and_mul_with_clamp(
output: torch.Tensor,
input: torch.Tensor,
@@ -184,7 +296,12 @@ def silu_and_mul_with_clamp(
topk_ids: torch.Tensor | None = None,
expert_map: torch.Tensor | None = None,
) -> None:
if topk_ids is not None and expert_map is not None:
if current_platform.is_xpu():
d = input.shape[-1] // 2
gate = torch.clamp(input[..., :d], max=clamp_limit)
up = torch.clamp(input[..., d:], min=-clamp_limit, max=clamp_limit)
output.copy_(gate * torch.sigmoid(gate) * up)
elif topk_ids is not None and expert_map is not None:
from vllm.model_executor.layers.fused_moe.utils import swiglu_limit_func
swiglu_limit_func(output, input, clamp_limit, topk_ids, expert_map)
@@ -201,12 +318,15 @@ def apply_moe_activation(
activation_config: ApplyMoEActivationConfig | None = None,
topk_ids: torch.Tensor | None = None,
expert_map: torch.Tensor | None = None,
valid_rows: torch.Tensor | None = None,
valid_token_counts: torch.Tensor | None = None,
) -> torch.Tensor:
"""Apply MoE activation function.
The configuration drives specialized activation behavior. Routing tensors
remain per-call inputs because they depend on the current token assignment.
and valid token counts remain per-call inputs because they depend on the
current token assignment. A single token count masks a flat ``[T, D]``
buffer; one count per expert masks each prefix in a padded ``[E, T, D]``
buffer.
"""
config = (
_DEFAULT_APPLY_MOE_ACTIVATION_CONFIG
@@ -214,19 +334,17 @@ def apply_moe_activation(
else activation_config
)
assert input.dim() == 2, "Input must be 2D"
assert output.dim() == 2, "Output must be 2D"
if activation.is_gated:
assert output.size(-1) * 2 == input.size(-1), (
f"{activation.value} expects 2x ratio: "
f"{output.size(-1) * 2} vs {input.size(-1)}"
)
else:
assert output.size(-1) == input.size(-1), (
f"{activation.value} expects equal sizes: "
f"{output.size(-1)} vs {input.size(-1)}"
if valid_token_counts is not None:
return _apply_moe_activation_masked(
activation,
output,
input,
valid_token_counts,
config,
)
_validate_moe_activation_shapes(activation, output, input, expected_dim=2)
# Activations with gated multiplication (gate × activation(up))
if activation == MoEActivation.SILU:
if config.clamp_limit is not None:
@@ -257,7 +375,6 @@ def apply_moe_activation(
-1.0
if config.activation_situ_linear_beta is None
else config.activation_situ_linear_beta,
valid_rows,
)
elif activation == MoEActivation.SWIGLUOAI:
torch.ops._C.swigluoai_and_mul(output, input)
@@ -298,7 +415,7 @@ def situ_and_mul_quant(
beta: float,
linear_beta: float | None,
group_size: int = 0,
valid_rows: torch.Tensor | None = None,
num_valid_tokens: torch.Tensor | None = None,
topk: int = 1,
) -> None:
"""Fused Kimi SITU activation + dynamic FP8 quantization.
@@ -314,11 +431,11 @@ def situ_and_mul_quant(
[M, d // 128], matching humming ``quant_input(group_size=128, float32)``.
``linear_beta`` <= 0 (or None) means "unset" (up passed through), matching
``SituAndMul(linear_beta=None)``. ``valid_rows`` (int32 scalar tensor) is the
DeepEP v2 contiguous-layout valid *token* count (e.g. ``psum[-1:]``); the
kernel multiplies it by ``topk`` to bound rows, so no host-side cast/multiply
is needed. Padding rows are skipped and receive a benign scale of 1.0. Kept
in this module so that every ``torch.ops._C.situ*`` call lives in one file.
``SituAndMul(linear_beta=None)``. ``num_valid_tokens`` (int32 scalar tensor)
is the DeepEP v2 contiguous-layout token count (e.g. ``psum[-1:]``); the
kernel multiplies it by ``topk`` to bound rows. Padding rows are skipped and
receive a benign scale of 1.0. Kept in this module so that every
``torch.ops._C.situ*`` call lives in one file.
"""
torch.ops._C.situ_and_mul_quant(
output,
@@ -327,6 +444,6 @@ def situ_and_mul_quant(
beta,
-1.0 if linear_beta is None else linear_beta,
group_size,
valid_rows,
num_valid_tokens,
topk,
)
@@ -5,7 +5,10 @@
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.activation import (
MoEActivation,
apply_moe_activation_masked_supported,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
@@ -19,7 +22,6 @@ from vllm.model_executor.layers.fused_moe.utils import (
_resize_cache,
moe_kernel_quantize_input,
normalize_batched_scales_shape,
swiglu_limit_func,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
@@ -839,16 +841,18 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular):
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation in [
MoEActivation.SILU,
MoEActivation.GELU,
MoEActivation.GELU_TANH,
MoEActivation.SWIGLUOAI,
MoEActivation.SILU_NO_MUL,
MoEActivation.GELU_NO_MUL,
MoEActivation.GELU_TANH_NO_MUL,
MoEActivation.RELU2_NO_MUL,
]
if current_platform.is_xpu():
return activation in [
MoEActivation.SILU,
MoEActivation.GELU,
MoEActivation.GELU_TANH,
MoEActivation.SWIGLUOAI,
MoEActivation.SILU_NO_MUL,
MoEActivation.GELU_NO_MUL,
MoEActivation.GELU_TANH_NO_MUL,
MoEActivation.RELU2_NO_MUL,
]
return apply_moe_activation_masked_supported(activation)
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
@@ -858,20 +862,6 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular):
# Let PrepareAndFinalize::finalize() decide the impl.
return TopKWeightAndReduceDelegate()
def activation(
self,
activation: MoEActivation,
output: torch.Tensor,
input: torch.Tensor,
**kwargs,
) -> None:
gemm1_clamp_limit = self.quant_config.gemm1_clamp_limit
if activation == MoEActivation.SILU and gemm1_clamp_limit is not None:
swiglu_limit_func(output, input, float(gemm1_clamp_limit))
return
super().activation(activation, output, input)
def workspace_shapes(
self,
M: int,
@@ -998,11 +988,19 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular):
intermediate_cache2.fill_(0)
# TODO (bnell): use triton utility from batched deep gemm.
self.activation(
activation,
intermediate_cache2.view(-1, activation_out_dim),
intermediate_cache1.view(-1, N),
)
if current_platform.is_xpu():
self.activation(
activation,
intermediate_cache2.view(-1, activation_out_dim),
intermediate_cache1.view(-1, N),
)
else:
self.activation(
activation,
intermediate_cache2,
intermediate_cache1,
valid_token_counts=expert_num_tokens,
)
qintermediate_cache2, a2q_scale = batched_moe_kernel_quantize_input(
intermediate_cache2,
@@ -35,10 +35,7 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceDelegate,
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.fused_moe.utils import (
_resize_cache,
swiglu_limit_func,
)
from vllm.model_executor.layers.fused_moe.utils import _resize_cache
from vllm.model_executor.layers.quantization.utils.quant_utils import (
INT4_DTYPE,
INT8_DTYPE,
@@ -103,17 +100,18 @@ def _is_supported_wna16_weight_key(weight_key: QuantKey | None) -> bool:
)
def get_humming_moe_gemm_type() -> str:
def get_humming_moe_gemm_type(moe_config: FusedMoEConfig | None = None) -> str:
env_gemm_type: str | None = envs.VLLM_HUMMING_MOE_GEMM_TYPE
gemm_type = "indexed"
if env_gemm_type is not None:
if env_gemm_type is not None and env_gemm_type.lower() != "auto":
env_gemm_type = env_gemm_type.lower()
if env_gemm_type == "indexed":
gemm_type = env_gemm_type
elif env_gemm_type in ["grouped_contiguous", "grouped"]:
if env_gemm_type == "grouped":
gemm_type = "grouped_contiguous"
else:
gemm_type = "indexed"
gemm_type = env_gemm_type
elif moe_config is not None and moe_config.moe_parallel_config.use_ep:
gemm_type = "grouped_contiguous"
else:
gemm_type = "indexed"
logger.info_once(f"Using {gemm_type} gemm for humming moe") # noqa
return gemm_type
@@ -178,7 +176,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
max_num_tokens=max_num_tokens,
num_dispatchers=num_dispatchers,
)
self._permute_scratch: MoEPermuteScratch | None = None
self._permute_scratch: dict[int, MoEPermuteScratch] = {}
def init_humming_moe(self):
from vllm.utils.humming import get_heuristics_config
@@ -257,26 +255,36 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
**kwargs,
)
def _get_permute_scratch(self) -> MoEPermuteScratch | None:
if self._permute_scratch is None and moe_permute_unpermute_supported():
self._permute_scratch = MoEPermuteScratch(
max_num_tokens=self.moe_config.max_num_tokens,
topk=self.moe_config.experts_per_token,
def _get_permute_scratch(self, topk: int) -> MoEPermuteScratch | None:
if not moe_permute_unpermute_supported():
return None
scratch = self._permute_scratch.get(topk)
if scratch is None:
max_expanded_rows = (
self.moe_config.max_num_tokens
* self.moe_config.dp_size
* self.moe_config.experts_per_token
)
scratch = MoEPermuteScratch(
max_num_tokens=math.ceil(max_expanded_rows / topk),
topk=topk,
num_experts=self.moe_config.num_experts,
num_local_experts=self.moe_config.num_local_experts,
device=torch.device(self.moe_config.device),
hidden_size=self.moe_config.hidden_dim,
hidden_dtype=self.moe_config.in_dtype,
)
return self._permute_scratch
self._permute_scratch[topk] = scratch
return scratch
def get_global_valid_shape_m(self, topk_ids: torch.Tensor):
num_tokens = topk_ids.size(0)
ctx = get_forward_context()
if ctx.dp_metadata is not None:
num_tokens = ctx.dp_metadata.num_tokens_across_dp_cpu.sum().item()
return num_tokens * self.moe_config.experts_per_token
return num_tokens * topk_ids.size(1)
return topk_ids.size(0) * topk_ids.size(1)
def estimate_local_valid_shape_m(self, topk_ids: torch.Tensor):
# estimate shape_m for kernel tuning
@@ -645,7 +653,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
if supported:
assert hasattr(cls, "humming_gemm_type")
gemm_type = cls.humming_gemm_type().value.lower()
preferred_gemm_type = get_humming_moe_gemm_type()
preferred_gemm_type = get_humming_moe_gemm_type(moe_config)
supported = preferred_gemm_type.lower() == gemm_type
if not supported:
reason = (
@@ -660,25 +668,16 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
activation: MoEActivation,
output: torch.Tensor,
input: torch.Tensor,
valid_rows: torch.Tensor | None = None,
valid_token_counts: torch.Tensor | None = None,
) -> None:
activation_config = self.activation_config
if (
activation == MoEActivation.SILU
and activation_config.clamp_limit is not None
):
swiglu_limit_func(
output=output,
input=input,
swiglu_limit=activation_config.clamp_limit,
)
else:
self.activation(
activation=activation,
input=input,
output=output,
valid_rows=valid_rows,
)
activation_kwargs: dict[str, Any] = dict(
activation=activation,
input=input,
output=output,
)
if valid_token_counts is not None:
activation_kwargs["valid_token_counts"] = valid_token_counts
self.activation(**activation_kwargs)
def fused_situ_quant_enabled(self, activation: MoEActivation) -> bool:
"""Whether the SITU activation + w2 quant can be fused into one kernel.
@@ -718,7 +717,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
self,
gate_up_output: torch.Tensor,
quanted_down_input: torch.Tensor,
valid_rows: torch.Tensor | None,
num_valid_tokens: torch.Tensor | None,
topk: int = 1,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Run the fused SITU activation + FP8 quant for the w2 input.
@@ -750,7 +749,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
beta=cfg.activation_situ_beta,
linear_beta=cfg.activation_situ_linear_beta,
group_size=group_size,
valid_rows=valid_rows,
num_valid_tokens=num_valid_tokens,
topk=topk,
)
return quanted_down_input, input_scale
@@ -878,8 +877,8 @@ class HummingIndexedExperts(HummingExpertsBase):
)
# psum[-1:] is the DeepEP valid *token* count as a zero-cost int32 view.
# situ (rows = tokens*topk) multiplies by topk on-device; mul_sum bounds
# on tokens, so both read this pointer directly -- no host cast/multiply.
# The fused path and mul_sum consume tokens directly. The unfused
# activation consumes the expanded row count (tokens * topk).
valid_tokens = None
topk = topk_ids.size(1)
if (
@@ -894,21 +893,18 @@ class HummingIndexedExperts(HummingExpertsBase):
inputs, input_scale = self.fused_situ_quant(
gate_up_output=buffers["gate_up_output"],
quanted_down_input=buffers["quanted_down_input"],
valid_rows=valid_tokens,
num_valid_tokens=valid_tokens,
topk=topk,
)
else:
# Fallback situ_and_mul takes int64 row counts (tokens*topk).
valid_rows = (
valid_tokens.to(torch.int64) * topk
if valid_tokens is not None
else None
valid_token_counts = (
valid_tokens * topk if valid_tokens is not None else None
)
self.apply_activation(
activation=activation,
input=buffers["gate_up_output"],
output=buffers["activation_output"],
valid_rows=valid_rows,
valid_token_counts=valid_token_counts,
)
inputs, input_scale = self.quantize_input(
@@ -1002,7 +998,7 @@ class HummingGroupedExperts(HummingExpertsBase):
n_expert=global_num_experts,
n_local_expert=self.num_experts,
expert_map=expert_map,
scratch=self._get_permute_scratch(),
scratch=self._get_permute_scratch(topk_ids.size(1)),
)
inputs, input_scale = self.quantize_input(
@@ -1028,6 +1024,11 @@ class HummingGroupedExperts(HummingExpertsBase):
activation=activation,
input=buffers["gate_up_output"],
output=buffers["activation_output"],
valid_token_counts=(
expert_first_token_offset[-1:].to(torch.int32)
if expert_tokens_meta is None
else None
),
)
inputs, input_scale = self.quantize_input(
@@ -13,6 +13,7 @@ from vllm.model_executor.layers.fused_moe.activation import (
ApplyMoEActivationConfig,
MoEActivation,
apply_moe_activation,
apply_moe_activation_masked_supported,
apply_moe_activation_supported,
)
from vllm.model_executor.layers.fused_moe.config import (
@@ -955,6 +956,10 @@ class BatchedMarlinExperts(MarlinExpertsBase):
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.BatchedExperts
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return apply_moe_activation_masked_supported(activation)
def workspace_shapes(
self,
M: int,
@@ -1000,26 +1005,18 @@ class BatchedMarlinExperts(MarlinExpertsBase):
act: MoEActivation,
act_output: torch.Tensor,
act_input: torch.Tensor,
**kwargs,
*,
topk_ids: torch.Tensor | None = None,
expert_map: torch.Tensor | None = None,
) -> None:
if act != MoEActivation.SITU:
self.activation(
act,
act_output,
act_input,
**kwargs,
)
return
num_experts, max_num_tokens = hidden_states.shape[:2]
beta = self.activation_config.activation_situ_beta
linear_beta = self.activation_config.activation_situ_linear_beta
torch.ops._C.masked_situ_and_mul(
self.activation(
act,
act_output.view(num_experts, max_num_tokens, -1),
act_input.view(num_experts, max_num_tokens, -1),
expert_tokens_meta.expert_num_tokens,
1.0 if beta is None else beta,
-1.0 if linear_beta is None else linear_beta,
topk_ids=topk_ids,
expert_map=expert_map,
valid_token_counts=expert_tokens_meta.expert_num_tokens,
)
return batched_fused_marlin_moe(
@@ -902,7 +902,7 @@ class FusedMoEExpertsModular(FusedMoEExperts):
*,
topk_ids: torch.Tensor | None = None,
expert_map: torch.Tensor | None = None,
valid_rows: torch.Tensor | None = None,
valid_token_counts: torch.Tensor | None = None,
) -> None:
apply_moe_activation(
activation,
@@ -911,7 +911,7 @@ class FusedMoEExpertsModular(FusedMoEExperts):
activation_config=self.activation_config,
topk_ids=topk_ids,
expert_map=expert_map,
valid_rows=valid_rows,
valid_token_counts=valid_token_counts,
)
@abstractmethod