[MoE] Generalize masked activation for padded layouts (#51217)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
a232e29e9d
commit
82b7d49a6e
@@ -500,10 +500,8 @@ template <typename scalar_t>
|
||||
__global__ void situ_and_mul_kernel(
|
||||
scalar_t* __restrict__ out, // [..., d]
|
||||
const scalar_t* __restrict__ input, // [..., 2, d]
|
||||
const int d, const float beta, const float linear_beta,
|
||||
const int64_t* __restrict__ valid_rows_ptr) {
|
||||
const int d, const float beta, const float linear_beta) {
|
||||
const int64_t row = blockIdx.x;
|
||||
if (valid_rows_ptr != nullptr && row >= *valid_rows_ptr) return;
|
||||
const scalar_t* gate_ptr = input + row * 2 * d;
|
||||
const scalar_t* up_ptr = gate_ptr + d;
|
||||
scalar_t* out_ptr = out + row * d;
|
||||
@@ -558,15 +556,16 @@ template <typename scalar_t, typename fp8_type, int THREADS, int NUM_STAGES,
|
||||
__global__ void situ_and_mul_quant_group_pipelined_kernel(
|
||||
fp8_type* __restrict__ out, float* __restrict__ scale_out,
|
||||
const scalar_t* __restrict__ input, const int64_t num_rows,
|
||||
const int32_t* __restrict__ valid_rows_ptr, const int64_t topk) {
|
||||
const int32_t* __restrict__ num_valid_tokens_ptr, const int64_t topk) {
|
||||
const int tid = threadIdx.x;
|
||||
const int warp_id = tid / WARP_SIZE;
|
||||
const int lane_id = tid % WARP_SIZE;
|
||||
static constexpr int NUM_WARPS = THREADS / WARP_SIZE;
|
||||
static constexpr int NUM_GROUPS = D / GROUP_SIZE;
|
||||
const int64_t row_bound =
|
||||
valid_rows_ptr != nullptr
|
||||
? max((int64_t)0, min((int64_t)(*valid_rows_ptr) * topk, num_rows))
|
||||
num_valid_tokens_ptr != nullptr
|
||||
? max((int64_t)0,
|
||||
min((int64_t)(*num_valid_tokens_ptr) * topk, num_rows))
|
||||
: num_rows;
|
||||
|
||||
static_assert(NUM_GROUPS % 4 == 0,
|
||||
@@ -710,15 +709,16 @@ __global__ void situ_and_mul_quant_group_scalar_kernel(
|
||||
const scalar_t* __restrict__ input, // [num_tokens, 2, d]
|
||||
const int d, const int num_groups, const float beta,
|
||||
const float linear_beta, const int64_t num_rows,
|
||||
const int32_t* __restrict__ valid_rows_ptr, const int64_t topk) {
|
||||
const int32_t* __restrict__ num_valid_tokens_ptr, const int64_t topk) {
|
||||
static constexpr int ELTS_PER_LANE = GROUP_SIZE / WARP_SIZE;
|
||||
const int tid = threadIdx.x;
|
||||
const int warp_id = tid / WARP_SIZE;
|
||||
const int lane_id = tid % WARP_SIZE;
|
||||
const int num_warps = blockDim.x / WARP_SIZE;
|
||||
const int64_t row_bound =
|
||||
valid_rows_ptr != nullptr
|
||||
? max((int64_t)0, min((int64_t)(*valid_rows_ptr) * topk, num_rows))
|
||||
num_valid_tokens_ptr != nullptr
|
||||
? max((int64_t)0,
|
||||
min((int64_t)(*num_valid_tokens_ptr) * topk, num_rows))
|
||||
: num_rows;
|
||||
|
||||
const bool clamp_up = linear_beta > 0.0f;
|
||||
@@ -768,24 +768,61 @@ __global__ void situ_and_mul_quant_group_scalar_kernel(
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
constexpr int kMaxMaskedTokenBlocks = 32;
|
||||
|
||||
template <bool BATCHED_EXPERTS>
|
||||
__device__ __forceinline__ bool get_masked_row_range(
|
||||
const int* __restrict__ valid_token_counts, const int max_num_tokens,
|
||||
int64_t& first_row, int64_t& end_row, int64_t& row_stride) {
|
||||
if constexpr (BATCHED_EXPERTS) {
|
||||
// [E, T, *]: z lanes grid-stride over one expert's valid token prefix.
|
||||
const int expert = blockIdx.y;
|
||||
const int num_tokens =
|
||||
max(0, min(valid_token_counts[expert], max_num_tokens));
|
||||
const int token_block = blockIdx.z;
|
||||
if (token_block >= num_tokens) {
|
||||
return false;
|
||||
}
|
||||
const int64_t expert_first_row =
|
||||
static_cast<int64_t>(expert) * max_num_tokens;
|
||||
first_row = expert_first_row + token_block;
|
||||
end_row = expert_first_row + num_tokens;
|
||||
row_stride = gridDim.z;
|
||||
} else {
|
||||
// [T, *]: y lanes grid-stride over the single valid token prefix.
|
||||
const int num_tokens = max(0, min(valid_token_counts[0], max_num_tokens));
|
||||
const int token_block = blockIdx.y;
|
||||
if (token_block >= num_tokens) {
|
||||
return false;
|
||||
}
|
||||
first_row = token_block;
|
||||
end_row = num_tokens;
|
||||
row_stride = gridDim.y;
|
||||
}
|
||||
return first_row < end_row;
|
||||
}
|
||||
|
||||
__device__ __forceinline__ int get_masked_feature_index() {
|
||||
return blockIdx.x * blockDim.x + threadIdx.x;
|
||||
}
|
||||
|
||||
template <typename scalar_t, bool BATCHED_EXPERTS>
|
||||
__global__ void masked_situ_and_mul_kernel(
|
||||
scalar_t* __restrict__ out, const scalar_t* __restrict__ input,
|
||||
const int* __restrict__ expert_num_tokens, const int max_num_tokens,
|
||||
const int* __restrict__ valid_token_counts, const int max_num_tokens,
|
||||
const int d, const float beta, const float linear_beta) {
|
||||
const int expert = blockIdx.y;
|
||||
const int num_tokens = expert_num_tokens[expert];
|
||||
const int idx = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
if (idx >= d || num_tokens == 0) {
|
||||
int64_t first_row, end_row, row_stride;
|
||||
const int idx = get_masked_feature_index();
|
||||
if (idx >= d ||
|
||||
!get_masked_row_range<BATCHED_EXPERTS>(valid_token_counts, max_num_tokens,
|
||||
first_row, end_row, row_stride)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const bool clamp_up = linear_beta > 0.0f;
|
||||
const float inv_beta = 1.0f / beta;
|
||||
const float inv_linear_beta = clamp_up ? 1.0f / linear_beta : 0.0f;
|
||||
const int64_t expert_row = static_cast<int64_t>(expert) * max_num_tokens;
|
||||
for (int token = 0; token < num_tokens; ++token) {
|
||||
const int64_t row = expert_row + token;
|
||||
for (int64_t row = first_row; row < end_row; row += row_stride) {
|
||||
const scalar_t* gate_ptr = input + row * 2 * d;
|
||||
const scalar_t* up_ptr = gate_ptr + d;
|
||||
scalar_t* out_ptr = out + row * d;
|
||||
@@ -796,6 +833,119 @@ __global__ void masked_situ_and_mul_kernel(
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&, const float),
|
||||
bool act_first, bool HAS_CLAMP, bool BATCHED_EXPERTS>
|
||||
__global__ void masked_act_and_mul_kernel(
|
||||
scalar_t* __restrict__ out, const scalar_t* __restrict__ input,
|
||||
const int* __restrict__ valid_token_counts, const int max_num_tokens,
|
||||
const int d, const float limit, const float alpha, const float beta) {
|
||||
int64_t first_row, end_row, row_stride;
|
||||
const int idx = get_masked_feature_index();
|
||||
if (idx >= d ||
|
||||
!get_masked_row_range<BATCHED_EXPERTS>(valid_token_counts, max_num_tokens,
|
||||
first_row, end_row, row_stride)) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (int64_t row = first_row; row < end_row; row += row_stride) {
|
||||
const scalar_t* x_ptr = input + row * 2 * d;
|
||||
const scalar_t* y_ptr = x_ptr + d;
|
||||
scalar_t* out_ptr = out + row * d;
|
||||
const scalar_t x = VLLM_LDG(&x_ptr[idx]);
|
||||
const scalar_t y = VLLM_LDG(&y_ptr[idx]);
|
||||
out_ptr[idx] = compute<scalar_t, ACT_FN, act_first, HAS_CLAMP>(x, y, limit,
|
||||
alpha, beta);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t, bool BATCHED_EXPERTS>
|
||||
__global__ void masked_swigluoai_and_mul_kernel(
|
||||
scalar_t* __restrict__ out, const scalar_t* __restrict__ input,
|
||||
const int* __restrict__ valid_token_counts, const int max_num_tokens,
|
||||
const int d, const float alpha, const float limit) {
|
||||
int64_t first_row, end_row, row_stride;
|
||||
const int idx = get_masked_feature_index();
|
||||
if (idx >= d ||
|
||||
!get_masked_row_range<BATCHED_EXPERTS>(valid_token_counts, max_num_tokens,
|
||||
first_row, end_row, row_stride)) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (int64_t row = first_row; row < end_row; row += row_stride) {
|
||||
const scalar_t* in_ptr = input + row * 2 * d;
|
||||
scalar_t* out_ptr = out + row * d;
|
||||
out_ptr[idx] =
|
||||
swigluoai_and_mul(in_ptr[2 * idx], in_ptr[2 * idx + 1], alpha, limit);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t, bool BATCHED_EXPERTS>
|
||||
__global__ void masked_swiglustep_and_mul_kernel(
|
||||
scalar_t* __restrict__ out, const scalar_t* __restrict__ input,
|
||||
const int* __restrict__ valid_token_counts, const int max_num_tokens,
|
||||
const int d, const float limit) {
|
||||
int64_t first_row, end_row, row_stride;
|
||||
const int idx = get_masked_feature_index();
|
||||
if (idx >= d ||
|
||||
!get_masked_row_range<BATCHED_EXPERTS>(valid_token_counts, max_num_tokens,
|
||||
first_row, end_row, row_stride)) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (int64_t row = first_row; row < end_row; row += row_stride) {
|
||||
const scalar_t* gate_ptr = input + row * 2 * d;
|
||||
const scalar_t* up_ptr = gate_ptr + d;
|
||||
scalar_t* out_ptr = out + row * d;
|
||||
const float gate = (float)VLLM_LDG(&gate_ptr[idx]);
|
||||
const float up = (float)VLLM_LDG(&up_ptr[idx]);
|
||||
const float gate_silu = gate / (1.0f + expf(-gate));
|
||||
const float gate_clamped = fminf(gate_silu, limit);
|
||||
const float up_clamped = fmaxf(fminf(up, limit), -limit);
|
||||
out_ptr[idx] = (scalar_t)(gate_clamped * up_clamped);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__device__ __forceinline__ T silu_no_mul_kernel(const T& x) {
|
||||
return silu_kernel(x, 1.0f);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__device__ __forceinline__ T gelu_no_mul_kernel(const T& x) {
|
||||
return gelu_kernel(x, 1.0f);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__device__ __forceinline__ T gelu_tanh_no_mul_kernel(const T& x) {
|
||||
return gelu_tanh_kernel(x, 1.0f);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__device__ __forceinline__ T relu2_no_mul_kernel(const T& x) {
|
||||
const float value = fmaxf((float)x, 0.0f);
|
||||
return (T)(value * value);
|
||||
}
|
||||
|
||||
template <typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&),
|
||||
bool BATCHED_EXPERTS>
|
||||
__global__ void masked_activation_kernel(
|
||||
scalar_t* __restrict__ out, const scalar_t* __restrict__ input,
|
||||
const int* __restrict__ valid_token_counts, const int max_num_tokens,
|
||||
const int d) {
|
||||
int64_t first_row, end_row, row_stride;
|
||||
const int idx = get_masked_feature_index();
|
||||
if (idx >= d ||
|
||||
!get_masked_row_range<BATCHED_EXPERTS>(valid_token_counts, max_num_tokens,
|
||||
first_row, end_row, row_stride)) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (int64_t row = first_row; row < end_row; row += row_stride) {
|
||||
const int64_t offset = row * d + idx;
|
||||
out[offset] = ACT_FN(VLLM_LDG(&input[offset]));
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace vllm
|
||||
|
||||
#define LAUNCH_ACTIVATION_GATE_KERNEL_WITH_PARAM(KERNEL, PACKED_KERNEL, PARAM) \
|
||||
@@ -890,8 +1040,7 @@ void swigluoai_and_mul(torch::stable::Tensor& out, // [..., d]
|
||||
// through), matching SituAndMul(linear_beta=None) on the Python side.
|
||||
void situ_and_mul(torch::stable::Tensor& out, // [..., d]
|
||||
torch::stable::Tensor& input, // [..., 2 * d]
|
||||
double beta, double linear_beta,
|
||||
std::optional<torch::stable::Tensor> valid_rows) {
|
||||
double beta, double linear_beta) {
|
||||
int d = input.size(-1) / 2;
|
||||
int64_t num_tokens = input.numel() / input.size(-1);
|
||||
if (num_tokens == 0) {
|
||||
@@ -899,14 +1048,6 @@ void situ_and_mul(torch::stable::Tensor& out, // [..., d]
|
||||
}
|
||||
dim3 grid(num_tokens);
|
||||
dim3 block(std::min(d, 1024));
|
||||
const int64_t* valid_rows_ptr = nullptr;
|
||||
if (valid_rows.has_value()) {
|
||||
STD_TORCH_CHECK(valid_rows->is_cuda() && valid_rows->get_device_index() ==
|
||||
input.get_device_index(),
|
||||
"situ_and_mul: valid_rows must be a CUDA tensor on the "
|
||||
"same device as input");
|
||||
valid_rows_ptr = valid_rows->const_data_ptr<int64_t>();
|
||||
}
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
input.get_device_index());
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
@@ -914,7 +1055,7 @@ void situ_and_mul(torch::stable::Tensor& out, // [..., d]
|
||||
input.scalar_type(), "situ_and_mul_kernel", [&] {
|
||||
vllm::situ_and_mul_kernel<scalar_t><<<grid, block, 0, stream>>>(
|
||||
out.mutable_data_ptr<scalar_t>(), input.const_data_ptr<scalar_t>(),
|
||||
d, (float)beta, (float)linear_beta, valid_rows_ptr);
|
||||
d, (float)beta, (float)linear_beta);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -925,7 +1066,7 @@ void masked_situ_and_mul(torch::stable::Tensor& out, // [E, T, d]
|
||||
int num_experts = input.size(0);
|
||||
int max_num_tokens = input.size(1);
|
||||
int d = input.size(2) / 2;
|
||||
if (num_experts == 0 || max_num_tokens == 0) {
|
||||
if (num_experts == 0 || max_num_tokens == 0 || d == 0) {
|
||||
return;
|
||||
}
|
||||
constexpr int block_size = 256;
|
||||
@@ -936,21 +1077,128 @@ void masked_situ_and_mul(torch::stable::Tensor& out, // [E, T, d]
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
|
||||
input.scalar_type(), "masked_situ_and_mul_kernel", [&] {
|
||||
vllm::masked_situ_and_mul_kernel<scalar_t><<<grid, block, 0, stream>>>(
|
||||
out.mutable_data_ptr<scalar_t>(), input.const_data_ptr<scalar_t>(),
|
||||
expert_num_tokens.const_data_ptr<int>(), max_num_tokens, d,
|
||||
(float)beta, (float)linear_beta);
|
||||
vllm::masked_situ_and_mul_kernel<scalar_t, true>
|
||||
<<<grid, block, 0, stream>>>(
|
||||
out.mutable_data_ptr<scalar_t>(),
|
||||
input.const_data_ptr<scalar_t>(),
|
||||
expert_num_tokens.const_data_ptr<int>(), max_num_tokens, d,
|
||||
(float)beta, (float)linear_beta);
|
||||
});
|
||||
}
|
||||
|
||||
#define LAUNCH_MASKED_ACT_AND_MUL(KERNEL, ACT_FIRST, HAS_CLAMP, LIMIT, ALPHA, \
|
||||
BETA) \
|
||||
vllm::masked_act_and_mul_kernel<scalar_t, KERNEL<scalar_t>, ACT_FIRST, \
|
||||
HAS_CLAMP, BATCHED_EXPERTS> \
|
||||
<<<grid, block, 0, stream>>>(out.mutable_data_ptr<scalar_t>(), \
|
||||
input.const_data_ptr<scalar_t>(), \
|
||||
valid_token_counts.const_data_ptr<int>(), \
|
||||
max_num_tokens, d, LIMIT, ALPHA, BETA)
|
||||
|
||||
#define LAUNCH_MASKED_ACTIVATION(KERNEL) \
|
||||
vllm::masked_activation_kernel<scalar_t, KERNEL<scalar_t>, BATCHED_EXPERTS> \
|
||||
<<<grid, block, 0, stream>>>( \
|
||||
out.mutable_data_ptr<scalar_t>(), input.const_data_ptr<scalar_t>(), \
|
||||
valid_token_counts.const_data_ptr<int>(), max_num_tokens, d)
|
||||
|
||||
void masked_moe_activation(
|
||||
torch::stable::Tensor& out, // [T, d] or [E, T, d]
|
||||
torch::stable::Tensor& input, // [T, d] or [E, T, d], gated has 2 * d
|
||||
const torch::stable::Tensor& valid_token_counts,
|
||||
const std::string& activation, double clamp_limit, double alpha,
|
||||
double beta, double situ_beta, double situ_linear_beta) {
|
||||
const bool batched_experts = input.dim() == 3;
|
||||
const int num_experts = batched_experts ? input.size(0) : 1;
|
||||
const int max_num_tokens = batched_experts ? input.size(1) : input.size(0);
|
||||
const int d = out.size(-1);
|
||||
if (num_experts == 0 || max_num_tokens == 0 || d == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
constexpr int block_size = 256;
|
||||
const int feature_blocks = (d + block_size - 1) / block_size;
|
||||
const int token_blocks =
|
||||
std::min(max_num_tokens, vllm::kMaxMaskedTokenBlocks);
|
||||
// Batched grid: (feature tile, expert, token lane); flat grid: (feature
|
||||
// tile, token lane). Token lanes grid-stride each valid prefix.
|
||||
dim3 grid = batched_experts ? dim3(feature_blocks, num_experts, token_blocks)
|
||||
: dim3(feature_blocks, token_blocks);
|
||||
dim3 block(block_size);
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
input.get_device_index());
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
auto launch = [&]<bool BATCHED_EXPERTS>() {
|
||||
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
|
||||
input.scalar_type(), "masked_moe_activation", [&] {
|
||||
if (activation == "silu") {
|
||||
LAUNCH_MASKED_ACT_AND_MUL(vllm::silu_kernel, true, false, 0.0f,
|
||||
1.0f, 0.0f);
|
||||
} else if (activation == "silu_with_clamp") {
|
||||
LAUNCH_MASKED_ACT_AND_MUL(vllm::silu_kernel, true, true,
|
||||
(float)clamp_limit, (float)alpha,
|
||||
(float)beta);
|
||||
} else if (activation == "gelu") {
|
||||
LAUNCH_MASKED_ACT_AND_MUL(vllm::gelu_kernel, true, false, 0.0f,
|
||||
1.0f, 0.0f);
|
||||
} else if (activation == "gelu_tanh") {
|
||||
LAUNCH_MASKED_ACT_AND_MUL(vllm::gelu_tanh_kernel, true, false, 0.0f,
|
||||
1.0f, 0.0f);
|
||||
} else if (activation == "situ") {
|
||||
vllm::masked_situ_and_mul_kernel<scalar_t, BATCHED_EXPERTS>
|
||||
<<<grid, block, 0, stream>>>(
|
||||
out.mutable_data_ptr<scalar_t>(),
|
||||
input.const_data_ptr<scalar_t>(),
|
||||
valid_token_counts.const_data_ptr<int>(), max_num_tokens, d,
|
||||
(float)situ_beta, (float)situ_linear_beta);
|
||||
} else if (activation == "swigluoai") {
|
||||
vllm::masked_swigluoai_and_mul_kernel<scalar_t, BATCHED_EXPERTS>
|
||||
<<<grid, block, 0, stream>>>(
|
||||
out.mutable_data_ptr<scalar_t>(),
|
||||
input.const_data_ptr<scalar_t>(),
|
||||
valid_token_counts.const_data_ptr<int>(), max_num_tokens, d,
|
||||
(float)alpha, (float)clamp_limit);
|
||||
} else if (activation == "swigluoai_uninterleave") {
|
||||
LAUNCH_MASKED_ACT_AND_MUL(vllm::silu_kernel, true, true,
|
||||
(float)clamp_limit, (float)alpha,
|
||||
(float)beta);
|
||||
} else if (activation == "swiglustep") {
|
||||
vllm::masked_swiglustep_and_mul_kernel<scalar_t, BATCHED_EXPERTS>
|
||||
<<<grid, block, 0, stream>>>(
|
||||
out.mutable_data_ptr<scalar_t>(),
|
||||
input.const_data_ptr<scalar_t>(),
|
||||
valid_token_counts.const_data_ptr<int>(), max_num_tokens, d,
|
||||
(float)clamp_limit);
|
||||
} else if (activation == "silu_no_mul") {
|
||||
LAUNCH_MASKED_ACTIVATION(vllm::silu_no_mul_kernel);
|
||||
} else if (activation == "gelu_no_mul") {
|
||||
LAUNCH_MASKED_ACTIVATION(vllm::gelu_no_mul_kernel);
|
||||
} else if (activation == "gelu_tanh_no_mul") {
|
||||
LAUNCH_MASKED_ACTIVATION(vllm::gelu_tanh_no_mul_kernel);
|
||||
} else if (activation == "relu2_no_mul") {
|
||||
LAUNCH_MASKED_ACTIVATION(vllm::relu2_no_mul_kernel);
|
||||
} else {
|
||||
STD_TORCH_CHECK(false,
|
||||
"Unsupported masked MoE activation: ", activation);
|
||||
}
|
||||
});
|
||||
};
|
||||
if (batched_experts) {
|
||||
launch.template operator()<true>();
|
||||
} else {
|
||||
launch.template operator()<false>();
|
||||
}
|
||||
}
|
||||
|
||||
#undef LAUNCH_MASKED_ACT_AND_MUL
|
||||
#undef LAUNCH_MASKED_ACTIVATION
|
||||
// Fused SITU activation and block-FP8 quantization for Humming w2.
|
||||
// `valid_rows` is the int32 DeepEP token count (psum[-1]); rows = it * `topk`,
|
||||
// so padding is excluded on-device and skipped rows receive scale 1.
|
||||
// `num_valid_tokens` is the int32 DeepEP token count (psum[-1]); rows = it *
|
||||
// `topk`, so padding is excluded on-device and skipped rows receive scale 1.
|
||||
void situ_and_mul_quant(torch::stable::Tensor& out, // [..., d] (fp8)
|
||||
torch::stable::Tensor& scale, // [..., 1 or d/128]
|
||||
torch::stable::Tensor& input, // [..., 2 * d]
|
||||
double beta, double linear_beta, int64_t group_size,
|
||||
std::optional<torch::stable::Tensor> valid_rows,
|
||||
std::optional<torch::stable::Tensor> num_valid_tokens,
|
||||
int64_t topk) {
|
||||
STD_TORCH_CHECK(
|
||||
out.scalar_type() == torch::headeronly::ScalarType::Float8_e4m3fn ||
|
||||
@@ -974,16 +1222,17 @@ void situ_and_mul_quant(torch::stable::Tensor& out, // [..., d] (fp8)
|
||||
}
|
||||
STD_TORCH_CHECK(out.size(-1) == d && out.numel() == num_tokens * (int64_t)d,
|
||||
"situ_and_mul_quant: out shape must be [num_tokens, d]");
|
||||
const int32_t* valid_rows_ptr = nullptr;
|
||||
if (valid_rows.has_value()) {
|
||||
STD_TORCH_CHECK(valid_rows->is_cuda() && valid_rows->get_device_index() ==
|
||||
input.get_device_index(),
|
||||
"situ_and_mul_quant: valid_rows must be a CUDA tensor on "
|
||||
"the same device as input");
|
||||
const int32_t* num_valid_tokens_ptr = nullptr;
|
||||
if (num_valid_tokens.has_value()) {
|
||||
STD_TORCH_CHECK(
|
||||
valid_rows->scalar_type() == torch::headeronly::ScalarType::Int,
|
||||
"situ_and_mul_quant: valid_rows must be int32 (psum count)");
|
||||
valid_rows_ptr = valid_rows->const_data_ptr<int32_t>();
|
||||
num_valid_tokens->is_cuda() &&
|
||||
num_valid_tokens->get_device_index() == input.get_device_index(),
|
||||
"situ_and_mul_quant: num_valid_tokens must be a CUDA tensor on the "
|
||||
"same device as input");
|
||||
STD_TORCH_CHECK(
|
||||
num_valid_tokens->scalar_type() == torch::headeronly::ScalarType::Int,
|
||||
"situ_and_mul_quant: num_valid_tokens must be int32 (psum count)");
|
||||
num_valid_tokens_ptr = num_valid_tokens->const_data_ptr<int32_t>();
|
||||
}
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
input.get_device_index());
|
||||
@@ -1031,7 +1280,7 @@ void situ_and_mul_quant(torch::stable::Tensor& out, // [..., d] (fp8)
|
||||
out.mutable_data_ptr<fp8_t>(),
|
||||
scale.mutable_data_ptr<float>(),
|
||||
input.const_data_ptr<scalar_t>(), num_tokens,
|
||||
valid_rows_ptr, topk);
|
||||
num_valid_tokens_ptr, topk);
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -1045,7 +1294,7 @@ void situ_and_mul_quant(torch::stable::Tensor& out, // [..., d] (fp8)
|
||||
scale.mutable_data_ptr<float>(),
|
||||
input.const_data_ptr<scalar_t>(), d, num_groups,
|
||||
(float)beta, (float)linear_beta, num_tokens,
|
||||
valid_rows_ptr, topk);
|
||||
num_valid_tokens_ptr, topk);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
@@ -146,16 +146,17 @@ __global__ void preprocessTopkIdKernel(int* topk_id_ptr, int size,
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
// query global expert id in expert map.
|
||||
// if global expert id = -1 in exert map, plus n_expert
|
||||
// else set global expert id = exert map[global expert id]
|
||||
if (offset + tidx < bound) {
|
||||
auto topk_id = topk_id_ptr[offset + tidx];
|
||||
auto local_expert_idx = smem_expert_map[topk_id];
|
||||
if (local_expert_idx == -1) {
|
||||
topk_id += num_experts;
|
||||
if (topk_id < 0 || topk_id >= num_experts) {
|
||||
topk_id = num_experts;
|
||||
} else {
|
||||
topk_id = local_expert_idx;
|
||||
auto local_expert_idx = smem_expert_map[topk_id];
|
||||
if (local_expert_idx == -1) {
|
||||
topk_id += num_experts;
|
||||
} else {
|
||||
topk_id = local_expert_idx;
|
||||
}
|
||||
}
|
||||
__syncwarp();
|
||||
topk_id_ptr[offset + tidx] = topk_id;
|
||||
|
||||
@@ -530,20 +530,25 @@ void fatrelu_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input,
|
||||
double threshold);
|
||||
void swigluoai_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input,
|
||||
double alpha = 1.702, double limit = 7.0);
|
||||
void situ_and_mul(
|
||||
torch::stable::Tensor& out, torch::stable::Tensor& input, double beta = 1.0,
|
||||
double linear_beta = -1.0,
|
||||
std::optional<torch::stable::Tensor> valid_rows = std::nullopt);
|
||||
void situ_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input,
|
||||
double beta = 1.0, double linear_beta = -1.0);
|
||||
void situ_and_mul_quant(
|
||||
torch::stable::Tensor& out, torch::stable::Tensor& scale,
|
||||
torch::stable::Tensor& input, double beta = 1.0, double linear_beta = -1.0,
|
||||
int64_t group_size = 0,
|
||||
std::optional<torch::stable::Tensor> valid_rows = std::nullopt,
|
||||
std::optional<torch::stable::Tensor> num_valid_tokens = std::nullopt,
|
||||
int64_t topk = 1);
|
||||
void masked_situ_and_mul(torch::stable::Tensor& out,
|
||||
torch::stable::Tensor& input,
|
||||
const torch::stable::Tensor& expert_num_tokens,
|
||||
double beta = 1.0, double linear_beta = -1.0);
|
||||
void masked_moe_activation(torch::stable::Tensor& out,
|
||||
torch::stable::Tensor& input,
|
||||
const torch::stable::Tensor& valid_token_counts,
|
||||
const std::string& activation,
|
||||
double clamp_limit = 0.0, double alpha = 1.0,
|
||||
double beta = 0.0, double situ_beta = 1.0,
|
||||
double situ_linear_beta = -1.0);
|
||||
void gelu_new(torch::stable::Tensor& out, torch::stable::Tensor& input);
|
||||
void gelu_fast(torch::stable::Tensor& out, torch::stable::Tensor& input);
|
||||
void gelu_quick(torch::stable::Tensor& out, torch::stable::Tensor& input);
|
||||
|
||||
@@ -614,7 +614,7 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
// SituGLU implementation used in Kimi models.
|
||||
ops.def(
|
||||
"situ_and_mul(Tensor! out, Tensor input, float beta=1.0, float "
|
||||
"linear_beta=-1.0, Tensor? valid_rows=None) -> ()");
|
||||
"linear_beta=-1.0) -> ()");
|
||||
// Fused SituGLU activation + dynamic FP8 quantization for the Humming w2 path
|
||||
// (writes the fp8 down input and its float32 scale). group_size=0 ->
|
||||
// per-token scale [.., 1]; group_size=128 -> k-major block-FP8 scale [..,
|
||||
@@ -622,10 +622,15 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
ops.def(
|
||||
"situ_and_mul_quant(Tensor! out, Tensor! scale, Tensor input, "
|
||||
"float beta=1.0, float linear_beta=-1.0, int group_size=0, "
|
||||
"Tensor? valid_rows=None, int topk=1) -> ()");
|
||||
"Tensor? num_valid_tokens=None, int topk=1) -> ()");
|
||||
ops.def(
|
||||
"masked_situ_and_mul(Tensor! out, Tensor input, Tensor "
|
||||
"expert_num_tokens, float beta=1.0, float linear_beta=-1.0) -> ()");
|
||||
ops.def(
|
||||
"masked_moe_activation(Tensor! out, Tensor input, Tensor "
|
||||
"valid_token_counts, str activation, float clamp_limit=0.0, float "
|
||||
"alpha=1.0, float beta=0.0, float situ_beta=1.0, float "
|
||||
"situ_linear_beta=-1.0) -> ()");
|
||||
|
||||
// GELU implementation used in GPT-2.
|
||||
ops.def("gelu_new(Tensor! out, Tensor input) -> ()");
|
||||
@@ -852,6 +857,7 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
|
||||
ops.impl("situ_and_mul", TORCH_BOX(&situ_and_mul));
|
||||
ops.impl("situ_and_mul_quant", TORCH_BOX(&situ_and_mul_quant));
|
||||
ops.impl("masked_situ_and_mul", TORCH_BOX(&masked_situ_and_mul));
|
||||
ops.impl("masked_moe_activation", TORCH_BOX(&masked_moe_activation));
|
||||
ops.impl("gelu_new", TORCH_BOX(&gelu_new));
|
||||
ops.impl("gelu_fast", TORCH_BOX(&gelu_fast));
|
||||
ops.impl("gelu_quick", TORCH_BOX(&gelu_quick));
|
||||
|
||||
@@ -22,7 +22,13 @@ from vllm.model_executor.layers.activation import (
|
||||
SwigluStepAndMul,
|
||||
swiglustep_and_mul_triton,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.activation import (
|
||||
ApplyMoEActivationConfig,
|
||||
MoEActivation,
|
||||
apply_moe_activation,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import swiglu_limit_func
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
DTYPES = [torch.half, torch.bfloat16, torch.float]
|
||||
@@ -34,6 +40,100 @@ CUDA_DEVICES = [
|
||||
]
|
||||
|
||||
|
||||
def test_masked_moe_activation_rejects_unsupported_activation() -> None:
|
||||
input = torch.empty(1, 1, 2)
|
||||
output = torch.empty(1, 1, 1)
|
||||
valid_token_counts = torch.ones(1, dtype=torch.int32)
|
||||
|
||||
with pytest.raises(NotImplementedError, match="relu2"):
|
||||
apply_moe_activation(
|
||||
MoEActivation.RELU2,
|
||||
output,
|
||||
input,
|
||||
valid_token_counts=valid_token_counts,
|
||||
)
|
||||
|
||||
|
||||
def test_moe_silu_clamp_uses_native_xpu_fallback(
|
||||
default_vllm_config, monkeypatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
|
||||
clamp_limit = 3.0
|
||||
input = torch.tensor([[12.0, -12.0, 8.0, -8.0], [-2.0, 2.0, -4.0, 4.0]])
|
||||
output = torch.empty(2, 2)
|
||||
|
||||
apply_moe_activation(
|
||||
MoEActivation.SILU,
|
||||
output,
|
||||
input,
|
||||
activation_config=ApplyMoEActivationConfig(clamp_limit=clamp_limit),
|
||||
)
|
||||
|
||||
expected = SiluAndMulWithClamp(clamp_limit, compile_native=False).forward_native(
|
||||
input
|
||||
)
|
||||
torch.testing.assert_close(output, expected)
|
||||
|
||||
|
||||
def _assert_masked_moe_activation(
|
||||
activation: MoEActivation,
|
||||
activation_config: ApplyMoEActivationConfig,
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
mask_layout: str,
|
||||
d: int,
|
||||
max_num_tokens: int,
|
||||
) -> None:
|
||||
device = CUDA_DEVICES[0]
|
||||
num_experts = 4
|
||||
input_dim = 2 * d if activation.is_gated else d
|
||||
if mask_layout == "flat":
|
||||
input = torch.randn(max_num_tokens, input_dim, dtype=dtype, device=device)
|
||||
valid_token_counts = torch.tensor(
|
||||
[max_num_tokens // 2], dtype=torch.int32, device=device
|
||||
)
|
||||
output = torch.full((max_num_tokens, d), 42.0, dtype=dtype, device=device)
|
||||
else:
|
||||
input = torch.randn(
|
||||
num_experts, max_num_tokens, input_dim, dtype=dtype, device=device
|
||||
)
|
||||
valid_token_counts = torch.tensor(
|
||||
[0, 1, max_num_tokens // 2, max_num_tokens],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
output = torch.full(
|
||||
(num_experts, max_num_tokens, d), 42.0, dtype=dtype, device=device
|
||||
)
|
||||
|
||||
apply_moe_activation(
|
||||
activation,
|
||||
output,
|
||||
input,
|
||||
activation_config=activation_config,
|
||||
valid_token_counts=valid_token_counts,
|
||||
)
|
||||
|
||||
batched_input = input.view(-1, max_num_tokens, input_dim)
|
||||
batched_output = output.view(-1, max_num_tokens, d)
|
||||
for expert, num_tokens in enumerate(valid_token_counts.cpu().tolist()):
|
||||
if num_tokens:
|
||||
expected = torch.empty((num_tokens, d), dtype=dtype, device=device)
|
||||
apply_moe_activation(
|
||||
activation,
|
||||
expected,
|
||||
batched_input[expert, :num_tokens].clone(),
|
||||
activation_config=activation_config,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
batched_output[expert, :num_tokens],
|
||||
expected,
|
||||
atol=get_default_atol(output),
|
||||
rtol=get_default_rtol(output),
|
||||
)
|
||||
assert torch.all(batched_output[expert, num_tokens:] == 42.0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"activation",
|
||||
[
|
||||
@@ -252,6 +352,114 @@ def test_masked_situ_and_mul(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("activation", "activation_config"),
|
||||
[
|
||||
(MoEActivation.SILU, ApplyMoEActivationConfig()),
|
||||
(
|
||||
MoEActivation.SILU,
|
||||
ApplyMoEActivationConfig(clamp_limit=3.0),
|
||||
),
|
||||
(MoEActivation.GELU, ApplyMoEActivationConfig()),
|
||||
(MoEActivation.GELU_TANH, ApplyMoEActivationConfig()),
|
||||
(
|
||||
MoEActivation.SITU,
|
||||
ApplyMoEActivationConfig(
|
||||
activation_situ_beta=1.5,
|
||||
activation_situ_linear_beta=2.0,
|
||||
),
|
||||
),
|
||||
(MoEActivation.SWIGLUOAI, ApplyMoEActivationConfig()),
|
||||
(
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
ApplyMoEActivationConfig(clamp_limit=3.0, alpha=1.3, beta=0.5),
|
||||
),
|
||||
(MoEActivation.SWIGLUSTEP, ApplyMoEActivationConfig()),
|
||||
(MoEActivation.SILU_NO_MUL, ApplyMoEActivationConfig()),
|
||||
(MoEActivation.GELU_NO_MUL, ApplyMoEActivationConfig()),
|
||||
(MoEActivation.GELU_TANH_NO_MUL, ApplyMoEActivationConfig()),
|
||||
(MoEActivation.RELU2_NO_MUL, ApplyMoEActivationConfig()),
|
||||
],
|
||||
ids=[
|
||||
"silu",
|
||||
"silu_clamp",
|
||||
"gelu",
|
||||
"gelu_tanh",
|
||||
"situ",
|
||||
"swigluoai",
|
||||
"swigluoai_uninterleave",
|
||||
"swiglustep",
|
||||
"silu_no_mul",
|
||||
"gelu_no_mul",
|
||||
"gelu_tanh_no_mul",
|
||||
"relu2_no_mul",
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_masked_moe_activation_dispatch(
|
||||
default_vllm_config,
|
||||
activation: MoEActivation,
|
||||
activation_config: ApplyMoEActivationConfig,
|
||||
) -> None:
|
||||
_assert_masked_moe_activation(
|
||||
activation,
|
||||
activation_config,
|
||||
dtype=torch.bfloat16,
|
||||
mask_layout="batched_experts",
|
||||
d=513,
|
||||
max_num_tokens=7,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("activation", "activation_config", "mask_layout"),
|
||||
[
|
||||
pytest.param(
|
||||
MoEActivation.SILU,
|
||||
ApplyMoEActivationConfig(),
|
||||
"flat",
|
||||
id="flat",
|
||||
),
|
||||
pytest.param(
|
||||
MoEActivation.SITU,
|
||||
ApplyMoEActivationConfig(
|
||||
activation_situ_beta=1.5,
|
||||
activation_situ_linear_beta=2.0,
|
||||
),
|
||||
"batched_experts",
|
||||
id="batched-experts",
|
||||
),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_masked_moe_activation_grid_stride(
|
||||
default_vllm_config,
|
||||
activation: MoEActivation,
|
||||
activation_config: ApplyMoEActivationConfig,
|
||||
mask_layout: str,
|
||||
) -> None:
|
||||
_assert_masked_moe_activation(
|
||||
activation,
|
||||
activation_config,
|
||||
dtype=torch.half,
|
||||
mask_layout=mask_layout,
|
||||
d=513,
|
||||
max_num_tokens=67,
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_masked_moe_activation_opcheck(default_vllm_config) -> None:
|
||||
device = CUDA_DEVICES[0]
|
||||
input = torch.randn(2, 3, 64, dtype=torch.half, device=device)
|
||||
output = torch.empty(2, 3, 32, dtype=torch.half, device=device)
|
||||
valid_token_counts = torch.tensor([1, 3], dtype=torch.int32, device=device)
|
||||
opcheck(
|
||||
torch.ops._C.masked_moe_activation,
|
||||
(output, input, valid_token_counts, "silu", 0.0, 1.0, 0.0, 1.0, -1.0),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"activation",
|
||||
[
|
||||
|
||||
@@ -8,6 +8,7 @@ import torch
|
||||
|
||||
from tests.kernels.moe.utils import (
|
||||
batched_moe,
|
||||
make_dummy_moe_config,
|
||||
make_quantized_test_activations,
|
||||
make_test_weights,
|
||||
naive_batched_moe,
|
||||
@@ -16,7 +17,13 @@ from tests.kernels.quant_utils import native_batched_masked_quant_matmul
|
||||
from tests.kernels.utils import torch_experts
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.activation import (
|
||||
ApplyMoEActivationConfig,
|
||||
MoEActivation,
|
||||
apply_moe_activation,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
|
||||
BatchedTritonExperts,
|
||||
invoke_moe_batched_triton_kernel,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
@@ -496,11 +503,146 @@ def test_batched_mm_td_zero_expert_tokens():
|
||||
|
||||
# BatchedTritonExperts device enablement (XPU)
|
||||
def test_batched_triton_experts_supports_current_device():
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
|
||||
BatchedTritonExperts,
|
||||
assert BatchedTritonExperts._supports_current_device()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"activation",
|
||||
[
|
||||
MoEActivation.SITU,
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
MoEActivation.SWIGLUSTEP,
|
||||
],
|
||||
)
|
||||
def test_batched_triton_supports_masked_activations(
|
||||
monkeypatch, activation: MoEActivation
|
||||
):
|
||||
monkeypatch.setattr(current_platform, "is_xpu", lambda: False)
|
||||
assert BatchedTritonExperts._supports_activation(activation)
|
||||
|
||||
monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
|
||||
assert not BatchedTritonExperts._supports_activation(activation)
|
||||
|
||||
|
||||
def _torch_experts_with_shared_activation(
|
||||
a: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
activation_config: ApplyMoEActivationConfig,
|
||||
) -> torch.Tensor:
|
||||
num_tokens, hidden_dim = a.shape
|
||||
topk = topk_ids.shape[1]
|
||||
routed_input = a[:, None, :].expand(-1, topk, -1).reshape(-1, hidden_dim)
|
||||
routed_experts = topk_ids.reshape(-1)
|
||||
routed_output = torch.zeros_like(routed_input)
|
||||
|
||||
for expert in range(w1.shape[0]):
|
||||
expert_mask = routed_experts == expert
|
||||
if not expert_mask.any():
|
||||
continue
|
||||
projected = routed_input[expert_mask] @ w1[expert].T
|
||||
activated = torch.empty(
|
||||
projected.shape[0], w2.shape[2], dtype=a.dtype, device=a.device
|
||||
)
|
||||
apply_moe_activation(
|
||||
activation,
|
||||
activated,
|
||||
projected,
|
||||
activation_config=activation_config,
|
||||
)
|
||||
routed_output[expert_mask] = activated @ w2[expert].T
|
||||
|
||||
return (
|
||||
(
|
||||
routed_output.view(num_tokens, topk, hidden_dim).float()
|
||||
* topk_weights[..., None]
|
||||
)
|
||||
.sum(dim=1)
|
||||
.to(a.dtype)
|
||||
)
|
||||
|
||||
assert BatchedTritonExperts._supports_current_device()
|
||||
|
||||
_MASKED_BATCHED_ACTIVATION_CASES = [
|
||||
pytest.param(
|
||||
MoEActivation.SITU,
|
||||
ApplyMoEActivationConfig(
|
||||
activation_situ_beta=1.5,
|
||||
activation_situ_linear_beta=2.0,
|
||||
),
|
||||
id="situ",
|
||||
),
|
||||
pytest.param(
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
ApplyMoEActivationConfig(clamp_limit=3.0, alpha=1.3, beta=0.5),
|
||||
id="swigluoai-uninterleave",
|
||||
),
|
||||
pytest.param(
|
||||
MoEActivation.SWIGLUSTEP,
|
||||
ApplyMoEActivationConfig(),
|
||||
id="swiglustep",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("activation", "activation_config"), _MASKED_BATCHED_ACTIVATION_CASES
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_batched_experts_masked_activation_end_to_end(
|
||||
activation: MoEActivation,
|
||||
activation_config: ApplyMoEActivationConfig,
|
||||
):
|
||||
if current_platform.is_xpu():
|
||||
pytest.skip("Masked activations are not enabled on XPU")
|
||||
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
set_random_seed(7)
|
||||
device = current_platform.device_type
|
||||
init_workspace_manager(torch.device(f"{device}:0"))
|
||||
m, n, k, e, topk = 16, 128, 128, 8, 2
|
||||
|
||||
a = torch.randn((m, k), device=device, dtype=torch.bfloat16) / 10
|
||||
score = torch.randn((m, e), device=device, dtype=torch.bfloat16)
|
||||
w1 = torch.randn((e, 2 * n, k), device=device, dtype=torch.bfloat16) / 15
|
||||
w2 = torch.randn((e, k, n), device=device, dtype=torch.bfloat16) / 15
|
||||
moe_config = make_dummy_moe_config(
|
||||
num_experts=e,
|
||||
experts_per_token=topk,
|
||||
hidden_dim=k,
|
||||
intermediate_size=n,
|
||||
in_dtype=a.dtype,
|
||||
activation=activation,
|
||||
)
|
||||
moe_config.swiglu_limit = activation_config.clamp_limit
|
||||
moe_config.swiglu_alpha = activation_config.alpha
|
||||
moe_config.swiglu_beta = activation_config.beta
|
||||
moe_config.activation_situ_beta = activation_config.activation_situ_beta
|
||||
moe_config.activation_situ_linear_beta = (
|
||||
activation_config.activation_situ_linear_beta
|
||||
)
|
||||
|
||||
routing_weights = torch.softmax(score.float(), dim=-1)
|
||||
topk_weights, topk_ids = torch.topk(routing_weights, topk, dim=-1)
|
||||
topk_ids = topk_ids.to(torch.int32)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
baseline_output = _torch_experts_with_shared_activation(
|
||||
a,
|
||||
w1,
|
||||
w2,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
activation,
|
||||
activation_config,
|
||||
)
|
||||
triton_output = batched_moe(
|
||||
a, w1, w2, topk_weights, topk_ids, moe_config=moe_config
|
||||
)
|
||||
|
||||
torch.testing.assert_close(triton_output, baseline_output, atol=3e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,n,k,e,topk", [(32, 512, 512, 8, 2), (45, 1024, 128, 8, 1)])
|
||||
|
||||
@@ -1377,6 +1377,102 @@ def test_humming_activation_metadata_tracks_shared_apply(activation: MoEActivati
|
||||
) == apply_moe_activation_supported(activation)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("env_override", "use_ep", "expected"),
|
||||
[
|
||||
(None, False, "indexed"),
|
||||
(None, True, "grouped_contiguous"),
|
||||
("auto", True, "grouped_contiguous"),
|
||||
("indexed", True, "indexed"),
|
||||
("grouped", False, "grouped_contiguous"),
|
||||
],
|
||||
)
|
||||
def test_humming_selects_gemm_from_parallelism_and_override(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
env_override: str | None,
|
||||
use_ep: bool,
|
||||
expected: str,
|
||||
):
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
get_humming_moe_gemm_type,
|
||||
)
|
||||
|
||||
if env_override is None:
|
||||
monkeypatch.delenv("VLLM_HUMMING_MOE_GEMM_TYPE", raising=False)
|
||||
else:
|
||||
monkeypatch.setenv("VLLM_HUMMING_MOE_GEMM_TYPE", env_override)
|
||||
moe_config = make_dummy_moe_config()
|
||||
moe_config.moe_parallel_config.use_ep = use_ep
|
||||
|
||||
assert get_humming_moe_gemm_type(moe_config) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("dp_token_counts", "topk_shape", "expected"),
|
||||
[
|
||||
pytest.param(None, (7, 3), 21, id="runtime-layout"),
|
||||
pytest.param([3, 5], (48, 1), 48, id="expanded-dp-layout"),
|
||||
],
|
||||
)
|
||||
def test_humming_global_valid_shape_m(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
dp_token_counts: list[int] | None,
|
||||
topk_shape: tuple[int, int],
|
||||
expected: int,
|
||||
):
|
||||
from types import SimpleNamespace
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.experts.fused_humming_moe as humming
|
||||
|
||||
dp_metadata = (
|
||||
None
|
||||
if dp_token_counts is None
|
||||
else SimpleNamespace(
|
||||
num_tokens_across_dp_cpu=torch.tensor(dp_token_counts, dtype=torch.int32)
|
||||
)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
humming,
|
||||
"get_forward_context",
|
||||
lambda: SimpleNamespace(dp_metadata=dp_metadata),
|
||||
)
|
||||
experts = SimpleNamespace(moe_config=make_dummy_moe_config(experts_per_token=6))
|
||||
topk_ids = torch.empty(topk_shape, dtype=torch.int64)
|
||||
|
||||
result = humming.HummingExpertsBase.get_global_valid_shape_m(experts, topk_ids)
|
||||
|
||||
assert result == expected
|
||||
|
||||
|
||||
def test_humming_permute_scratch_is_keyed_by_runtime_topk(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.experts.fused_humming_moe as humming
|
||||
|
||||
scratch_topk6 = Mock()
|
||||
scratch_topk1 = Mock()
|
||||
scratch_type = Mock(side_effect=[scratch_topk6, scratch_topk1])
|
||||
monkeypatch.setattr(humming, "moe_permute_unpermute_supported", lambda: True)
|
||||
monkeypatch.setattr(humming, "MoEPermuteScratch", scratch_type)
|
||||
moe_config = make_dummy_moe_config(max_num_tokens=512, experts_per_token=6)
|
||||
moe_config.moe_parallel_config.dp_size = 2
|
||||
experts = SimpleNamespace(_permute_scratch={}, moe_config=moe_config)
|
||||
|
||||
assert humming.HummingExpertsBase._get_permute_scratch(experts, 6) is scratch_topk6
|
||||
assert humming.HummingExpertsBase._get_permute_scratch(experts, 1) is scratch_topk1
|
||||
assert humming.HummingExpertsBase._get_permute_scratch(experts, 6) is scratch_topk6
|
||||
|
||||
assert scratch_type.call_count == 2
|
||||
first_call, second_call = scratch_type.call_args_list
|
||||
assert first_call.kwargs["max_num_tokens"] == 1024
|
||||
assert first_call.kwargs["topk"] == 6
|
||||
assert second_call.kwargs["max_num_tokens"] == 6144
|
||||
assert second_call.kwargs["topk"] == 1
|
||||
|
||||
|
||||
def test_humming_delegates_to_instance_activation():
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
@@ -1408,10 +1504,173 @@ def test_humming_delegates_to_instance_activation():
|
||||
activation=MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
input=input,
|
||||
output=output,
|
||||
valid_rows=None,
|
||||
)
|
||||
|
||||
|
||||
def test_humming_grouped_apply_forwards_valid_prefix(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
from types import MethodType, SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.experts.fused_humming_moe as humming
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
HummingExpertsBase,
|
||||
HummingGroupedExperts,
|
||||
)
|
||||
|
||||
activation_func = Mock()
|
||||
buffers = {
|
||||
"gate_up_output": torch.empty(6, 4),
|
||||
"activation_output": torch.empty(6, 2),
|
||||
"down_output": torch.empty(6, 4),
|
||||
}
|
||||
experts = SimpleNamespace(
|
||||
activation=activation_func,
|
||||
num_experts=2,
|
||||
estimate_local_valid_shape_m=lambda _: 6,
|
||||
prepare_buffers=lambda *_: buffers,
|
||||
_get_permute_scratch=lambda _: None,
|
||||
quantize_input=lambda _, *, inputs, input_scale=None, quanted_input: (
|
||||
inputs,
|
||||
input_scale,
|
||||
),
|
||||
humming_forward=Mock(),
|
||||
compute_config_str="",
|
||||
w13_tuning_config_str="",
|
||||
w2_tuning_config_str="",
|
||||
)
|
||||
experts.apply_activation = MethodType(HummingExpertsBase.apply_activation, experts)
|
||||
|
||||
hidden_states = torch.empty(3, 4)
|
||||
output = torch.empty_like(hidden_states)
|
||||
topk_ids = torch.zeros((3, 2), dtype=torch.int64)
|
||||
topk_weights = torch.ones_like(topk_ids, dtype=torch.float32)
|
||||
expected_counts = torch.tensor([4], dtype=torch.int32)
|
||||
expert_offsets = torch.tensor([0, 2, 4], dtype=torch.int64)
|
||||
monkeypatch.setattr(
|
||||
humming,
|
||||
"moe_permute",
|
||||
lambda **_: (
|
||||
torch.empty(6, 4),
|
||||
None,
|
||||
expert_offsets,
|
||||
torch.arange(6),
|
||||
None,
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(humming, "moe_unpermute", lambda **_: None)
|
||||
|
||||
HummingGroupedExperts.apply(
|
||||
experts,
|
||||
output=output,
|
||||
hidden_states=hidden_states,
|
||||
w1=torch.empty(0),
|
||||
w2=torch.empty(0),
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=MoEActivation.SITU,
|
||||
global_num_experts=2,
|
||||
expert_map=None,
|
||||
a1q_scale=None,
|
||||
a2_scale=None,
|
||||
workspace13=torch.empty(0),
|
||||
workspace2=torch.empty(0),
|
||||
expert_tokens_meta=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
activation_func.assert_called_once()
|
||||
call_kwargs = activation_func.call_args.kwargs
|
||||
assert call_kwargs["activation"] == MoEActivation.SITU
|
||||
assert call_kwargs["input"].shape == (6, 4)
|
||||
assert call_kwargs["output"].shape == (6, 2)
|
||||
torch.testing.assert_close(call_kwargs["valid_token_counts"], expected_counts)
|
||||
|
||||
|
||||
def test_batched_marlin_activation_uses_expert_token_counts(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.experts.marlin_moe as marlin
|
||||
from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
|
||||
BatchedMarlinExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import (
|
||||
ExpertTokensMetadata,
|
||||
)
|
||||
|
||||
activation_func = Mock()
|
||||
experts = SimpleNamespace(
|
||||
activation=activation_func,
|
||||
w1_bias=None,
|
||||
w2_bias=None,
|
||||
w1_scale=None,
|
||||
w2_scale=None,
|
||||
quant_type_id=0,
|
||||
a1_gscale=None,
|
||||
a2_gscale=None,
|
||||
w13_g_idx=None,
|
||||
w2_g_idx=None,
|
||||
w13_g_idx_sort_indices=None,
|
||||
w2_g_idx_sort_indices=None,
|
||||
w1_zp=None,
|
||||
w2_zp=None,
|
||||
input_dtype=None,
|
||||
is_k_full=True,
|
||||
activation_config=ApplyMoEActivationConfig(
|
||||
clamp_limit=3.0, alpha=1.3, beta=0.5
|
||||
),
|
||||
)
|
||||
hidden_states = torch.empty(2, 3, 4)
|
||||
expert_num_tokens = torch.tensor([1, 2], dtype=torch.int32)
|
||||
topk_ids = torch.zeros((2, 3), dtype=torch.int64)
|
||||
expert_map = torch.arange(2, dtype=torch.int32)
|
||||
|
||||
def fake_batched_fused_marlin_moe(**kwargs):
|
||||
kwargs["activation_func"](
|
||||
kwargs["activation"],
|
||||
torch.empty(6, 4),
|
||||
torch.empty(6, 8),
|
||||
topk_ids=topk_ids,
|
||||
expert_map=expert_map,
|
||||
)
|
||||
return kwargs["output"]
|
||||
|
||||
monkeypatch.setattr(
|
||||
marlin, "batched_fused_marlin_moe", fake_batched_fused_marlin_moe
|
||||
)
|
||||
output = torch.empty_like(hidden_states)
|
||||
|
||||
BatchedMarlinExperts.apply(
|
||||
experts,
|
||||
output=output,
|
||||
hidden_states=hidden_states,
|
||||
w1=torch.empty(0),
|
||||
w2=torch.empty(0),
|
||||
topk_weights=torch.ones_like(topk_ids, dtype=torch.float32),
|
||||
topk_ids=topk_ids,
|
||||
activation=MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
global_num_experts=2,
|
||||
expert_map=expert_map,
|
||||
a1q_scale=None,
|
||||
a2_scale=None,
|
||||
workspace13=torch.empty(0),
|
||||
workspace2=torch.empty(0),
|
||||
expert_tokens_meta=ExpertTokensMetadata(expert_num_tokens, None),
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
activation_func.assert_called_once()
|
||||
call = activation_func.call_args
|
||||
assert call.args[0] == MoEActivation.SWIGLUOAI_UNINTERLEAVE
|
||||
assert call.args[1].shape == (2, 3, 4)
|
||||
assert call.args[2].shape == (2, 3, 8)
|
||||
torch.testing.assert_close(call.kwargs["valid_token_counts"], expert_num_tokens)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"activation",
|
||||
[
|
||||
|
||||
@@ -286,3 +286,39 @@ def test_moe_permute_reuses_scratch_buffers(dtype: torch.dtype):
|
||||
permuted_idx_1.untyped_storage().data_ptr()
|
||||
== scratch.permuted_idx.untyped_storage().data_ptr()
|
||||
)
|
||||
|
||||
|
||||
def test_moe_permute_ignores_invalid_expert_ids_with_scratch() -> None:
|
||||
if not moe_permute_unpermute_supported():
|
||||
pytest.skip("moe_permute_unpermute is not supported on this platform.")
|
||||
|
||||
hidden_states = torch.arange(5 * 16, dtype=torch.bfloat16, device="cuda").view(
|
||||
5, 16
|
||||
)
|
||||
topk_ids = torch.tensor([[0], [-1], [1], [4], [2]], device="cuda")
|
||||
expert_map = torch.tensor([0, 1, -1, -1], dtype=torch.int32, device="cuda")
|
||||
scratch = MoEPermuteScratch(
|
||||
max_num_tokens=5,
|
||||
topk=1,
|
||||
num_experts=4,
|
||||
num_local_experts=2,
|
||||
device=hidden_states.device,
|
||||
hidden_size=16,
|
||||
hidden_dtype=hidden_states.dtype,
|
||||
)
|
||||
|
||||
permuted, _, expert_offsets, _, _ = moe_permute(
|
||||
hidden_states=hidden_states,
|
||||
a1q_scale=None,
|
||||
topk_ids=topk_ids,
|
||||
n_expert=4,
|
||||
n_local_expert=2,
|
||||
expert_map=expert_map,
|
||||
scratch=scratch,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
expert_offsets,
|
||||
torch.tensor([0, 1, 2], dtype=torch.int64, device="cuda"),
|
||||
)
|
||||
torch.testing.assert_close(permuted[:2], hidden_states[[0, 2]])
|
||||
|
||||
@@ -5,7 +5,7 @@ quant kernel (``situ_and_mul_quant``).
|
||||
|
||||
Mirrors ``test_silu_mul_fp8_quant_deep_gemm.py`` in structure: build a reference
|
||||
activation + block-FP8 quant, drive random per-row scale magnitudes, exercise
|
||||
the masked (``valid_rows``) contiguous layout, and compare per valid row.
|
||||
the masked contiguous layout, and compare per valid row.
|
||||
|
||||
The kernel computes SITU in fp32 and converts to fp8 with hardware rounding,
|
||||
which torch cannot reproduce bit-exactly. It is therefore validated against the
|
||||
@@ -40,7 +40,7 @@ SITU_LINEAR_BETA = 25.0
|
||||
SITU_D = 3072
|
||||
GROUP_SIZE = 128
|
||||
|
||||
# (num_tokens, valid_rows). valid_rows=None => all rows valid. 2048/4096 rows
|
||||
# (num_tokens, num_valid_tokens). None means all rows are valid. 2048/4096 rows
|
||||
# span multiple grid-stride waves of the persistent grid (GRID_DIM = 132*8).
|
||||
CASES = [
|
||||
(1, None),
|
||||
@@ -111,7 +111,7 @@ def test_situ_and_mul_quant_pipelined(
|
||||
(num_tokens, d // GROUP_SIZE), -1.0, dtype=torch.float32, device=DEVICE
|
||||
)
|
||||
out_sentinel = out.clone()
|
||||
valid_rows = (
|
||||
num_valid_tokens = (
|
||||
None if valid is None else torch.tensor(valid, dtype=torch.int32, device=DEVICE)
|
||||
)
|
||||
|
||||
@@ -122,7 +122,7 @@ def test_situ_and_mul_quant_pipelined(
|
||||
beta=SITU_BETA,
|
||||
linear_beta=SITU_LINEAR_BETA,
|
||||
group_size=GROUP_SIZE,
|
||||
valid_rows=valid_rows,
|
||||
num_valid_tokens=num_valid_tokens,
|
||||
)
|
||||
|
||||
# Padding contract: skipped rows keep their bytes; their scale becomes 1.0.
|
||||
@@ -157,7 +157,7 @@ def test_situ_and_mul_quant_pipelined(
|
||||
@pytest.mark.parametrize("tokens,topk", [(300, 8), (1, 4), (256, 6)])
|
||||
@torch.inference_mode()
|
||||
def test_situ_and_mul_quant_topk_row_bound(tokens: int, topk: int) -> None:
|
||||
"""valid_rows is a token count; the kernel bounds rows at tokens*topk."""
|
||||
"""The kernel expands num_valid_tokens by topk to bound rows."""
|
||||
set_random_seed(7)
|
||||
d = SITU_D
|
||||
num_tokens = tokens * topk + 37 # padding tail past the valid rows
|
||||
@@ -168,7 +168,7 @@ def test_situ_and_mul_quant_topk_row_bound(tokens: int, topk: int) -> None:
|
||||
(num_tokens, d // GROUP_SIZE), -1.0, dtype=torch.float32, device=DEVICE
|
||||
)
|
||||
out_sentinel = out.clone()
|
||||
valid_rows = torch.tensor(tokens, dtype=torch.int32, device=DEVICE)
|
||||
num_valid_tokens = torch.tensor(tokens, dtype=torch.int32, device=DEVICE)
|
||||
|
||||
situ_and_mul_quant(
|
||||
out,
|
||||
@@ -177,7 +177,7 @@ def test_situ_and_mul_quant_topk_row_bound(tokens: int, topk: int) -> None:
|
||||
beta=SITU_BETA,
|
||||
linear_beta=SITU_LINEAR_BETA,
|
||||
group_size=GROUP_SIZE,
|
||||
valid_rows=valid_rows,
|
||||
num_valid_tokens=num_valid_tokens,
|
||||
topk=topk,
|
||||
)
|
||||
|
||||
|
||||
@@ -128,6 +128,7 @@ def batched_moe(
|
||||
quant_dtype: torch.dtype | None = None,
|
||||
per_act_token_quant: bool = False,
|
||||
block_shape: list[int] | None = None,
|
||||
moe_config: FusedMoEConfig | None = None,
|
||||
) -> torch.Tensor:
|
||||
max_num_tokens = round_up(a.shape[0], 64)
|
||||
|
||||
@@ -141,7 +142,8 @@ def batched_moe(
|
||||
a2_scale=a2_scale,
|
||||
)
|
||||
|
||||
moe_config = make_dummy_moe_config()
|
||||
if moe_config is None:
|
||||
moe_config = make_dummy_moe_config()
|
||||
|
||||
fused_experts = FusedMoEKernel(
|
||||
BatchedPrepareAndFinalize(
|
||||
|
||||
@@ -9,6 +9,8 @@ from typing import TYPE_CHECKING
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
@@ -134,11 +136,31 @@ _APPLY_MOE_ACTIVATIONS = frozenset(
|
||||
)
|
||||
|
||||
|
||||
_MASKED_MOE_ACTIVATION_NAMES: dict[MoEActivation, str] = {
|
||||
MoEActivation.SILU: "silu",
|
||||
MoEActivation.GELU: "gelu",
|
||||
MoEActivation.GELU_TANH: "gelu_tanh",
|
||||
MoEActivation.SITU: "situ",
|
||||
MoEActivation.SWIGLUOAI: "swigluoai",
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE: "swigluoai_uninterleave",
|
||||
MoEActivation.SWIGLUSTEP: "swiglustep",
|
||||
MoEActivation.SILU_NO_MUL: "silu_no_mul",
|
||||
MoEActivation.GELU_NO_MUL: "gelu_no_mul",
|
||||
MoEActivation.GELU_TANH_NO_MUL: "gelu_tanh_no_mul",
|
||||
MoEActivation.RELU2_NO_MUL: "relu2_no_mul",
|
||||
}
|
||||
|
||||
|
||||
def apply_moe_activation_supported(activation: MoEActivation) -> bool:
|
||||
"""Whether ``apply_moe_activation`` supports an activation."""
|
||||
return activation in _APPLY_MOE_ACTIVATIONS
|
||||
|
||||
|
||||
def apply_moe_activation_masked_supported(activation: MoEActivation) -> bool:
|
||||
"""Whether the masked ``apply_moe_activation`` path supports an activation."""
|
||||
return activation in _MASKED_MOE_ACTIVATION_NAMES
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ApplyMoEActivationConfig:
|
||||
"""Configuration forwarded to ``apply_moe_activation``."""
|
||||
@@ -177,6 +199,96 @@ class ApplyMoEActivationConfig:
|
||||
_DEFAULT_APPLY_MOE_ACTIVATION_CONFIG = ApplyMoEActivationConfig()
|
||||
|
||||
|
||||
def _validate_moe_activation_shapes(
|
||||
activation: MoEActivation,
|
||||
output: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
expected_dim: int,
|
||||
) -> None:
|
||||
assert input.dim() == expected_dim, f"Input must be {expected_dim}D"
|
||||
assert output.dim() == expected_dim, f"Output must be {expected_dim}D"
|
||||
assert input.shape[:-1] == output.shape[:-1], (
|
||||
f"Input/output leading shapes must match: {input.shape} vs {output.shape}"
|
||||
)
|
||||
if activation.is_gated:
|
||||
assert output.size(-1) * 2 == input.size(-1), (
|
||||
f"{activation.value} expects 2x ratio: "
|
||||
f"{output.size(-1) * 2} vs {input.size(-1)}"
|
||||
)
|
||||
else:
|
||||
assert output.size(-1) == input.size(-1), (
|
||||
f"{activation.value} expects equal sizes: "
|
||||
f"{output.size(-1)} vs {input.size(-1)}"
|
||||
)
|
||||
|
||||
|
||||
def _apply_moe_activation_masked(
|
||||
activation: MoEActivation,
|
||||
output: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
valid_token_counts: torch.Tensor,
|
||||
config: ApplyMoEActivationConfig,
|
||||
) -> torch.Tensor:
|
||||
masked_activation = _MASKED_MOE_ACTIVATION_NAMES.get(activation)
|
||||
if masked_activation is None:
|
||||
raise NotImplementedError(
|
||||
f"Masked MoE activation is not implemented for {activation.value}"
|
||||
)
|
||||
|
||||
assert input.dim() in (2, 3), "Masked input must be 2D or 3D"
|
||||
_validate_moe_activation_shapes(activation, output, input, expected_dim=input.dim())
|
||||
assert input.dtype == output.dtype, "Input and output dtypes must match"
|
||||
assert input.device == output.device, "Input and output devices must match"
|
||||
assert input.is_contiguous(), "Input must be contiguous"
|
||||
assert output.is_contiguous(), "Output must be contiguous"
|
||||
assert valid_token_counts.dtype == torch.int32, (
|
||||
"valid_token_counts must use torch.int32"
|
||||
)
|
||||
assert valid_token_counts.dim() == 1, "valid_token_counts must be 1D"
|
||||
assert valid_token_counts.device == input.device, (
|
||||
"valid_token_counts must be on the input device"
|
||||
)
|
||||
assert valid_token_counts.is_contiguous(), "valid_token_counts must be contiguous"
|
||||
expected_counts = input.size(0) if input.dim() == 3 else 1
|
||||
assert valid_token_counts.size(0) == expected_counts, (
|
||||
f"valid_token_counts must have {expected_counts} element(s) for "
|
||||
f"{input.dim()}D input"
|
||||
)
|
||||
|
||||
masked_clamp_limit = 0.0 if config.clamp_limit is None else config.clamp_limit
|
||||
masked_alpha = config.alpha
|
||||
if activation == MoEActivation.SILU and config.clamp_limit is not None:
|
||||
masked_activation = "silu_with_clamp"
|
||||
elif activation == MoEActivation.SITU:
|
||||
assert config.activation_situ_beta is not None, (
|
||||
"SITU requires activation_situ_beta from FusedMoEConfig"
|
||||
)
|
||||
elif activation == MoEActivation.SWIGLUOAI:
|
||||
masked_clamp_limit = 7.0
|
||||
masked_alpha = 1.702
|
||||
elif activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
|
||||
assert config.clamp_limit is not None, (
|
||||
"SWIGLUOAI_UNINTERLEAVE requires clamp_limit"
|
||||
)
|
||||
elif activation == MoEActivation.SWIGLUSTEP:
|
||||
masked_clamp_limit = 7.0
|
||||
|
||||
torch.ops._C.masked_moe_activation(
|
||||
output,
|
||||
input,
|
||||
valid_token_counts,
|
||||
masked_activation,
|
||||
masked_clamp_limit,
|
||||
masked_alpha,
|
||||
config.beta,
|
||||
1.0 if config.activation_situ_beta is None else config.activation_situ_beta,
|
||||
-1.0
|
||||
if config.activation_situ_linear_beta is None
|
||||
else config.activation_situ_linear_beta,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
def silu_and_mul_with_clamp(
|
||||
output: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
@@ -184,7 +296,12 @@ def silu_and_mul_with_clamp(
|
||||
topk_ids: torch.Tensor | None = None,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
if topk_ids is not None and expert_map is not None:
|
||||
if current_platform.is_xpu():
|
||||
d = input.shape[-1] // 2
|
||||
gate = torch.clamp(input[..., :d], max=clamp_limit)
|
||||
up = torch.clamp(input[..., d:], min=-clamp_limit, max=clamp_limit)
|
||||
output.copy_(gate * torch.sigmoid(gate) * up)
|
||||
elif topk_ids is not None and expert_map is not None:
|
||||
from vllm.model_executor.layers.fused_moe.utils import swiglu_limit_func
|
||||
|
||||
swiglu_limit_func(output, input, clamp_limit, topk_ids, expert_map)
|
||||
@@ -201,12 +318,15 @@ def apply_moe_activation(
|
||||
activation_config: ApplyMoEActivationConfig | None = None,
|
||||
topk_ids: torch.Tensor | None = None,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
valid_rows: torch.Tensor | None = None,
|
||||
valid_token_counts: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Apply MoE activation function.
|
||||
|
||||
The configuration drives specialized activation behavior. Routing tensors
|
||||
remain per-call inputs because they depend on the current token assignment.
|
||||
and valid token counts remain per-call inputs because they depend on the
|
||||
current token assignment. A single token count masks a flat ``[T, D]``
|
||||
buffer; one count per expert masks each prefix in a padded ``[E, T, D]``
|
||||
buffer.
|
||||
"""
|
||||
config = (
|
||||
_DEFAULT_APPLY_MOE_ACTIVATION_CONFIG
|
||||
@@ -214,19 +334,17 @@ def apply_moe_activation(
|
||||
else activation_config
|
||||
)
|
||||
|
||||
assert input.dim() == 2, "Input must be 2D"
|
||||
assert output.dim() == 2, "Output must be 2D"
|
||||
if activation.is_gated:
|
||||
assert output.size(-1) * 2 == input.size(-1), (
|
||||
f"{activation.value} expects 2x ratio: "
|
||||
f"{output.size(-1) * 2} vs {input.size(-1)}"
|
||||
)
|
||||
else:
|
||||
assert output.size(-1) == input.size(-1), (
|
||||
f"{activation.value} expects equal sizes: "
|
||||
f"{output.size(-1)} vs {input.size(-1)}"
|
||||
if valid_token_counts is not None:
|
||||
return _apply_moe_activation_masked(
|
||||
activation,
|
||||
output,
|
||||
input,
|
||||
valid_token_counts,
|
||||
config,
|
||||
)
|
||||
|
||||
_validate_moe_activation_shapes(activation, output, input, expected_dim=2)
|
||||
|
||||
# Activations with gated multiplication (gate × activation(up))
|
||||
if activation == MoEActivation.SILU:
|
||||
if config.clamp_limit is not None:
|
||||
@@ -257,7 +375,6 @@ def apply_moe_activation(
|
||||
-1.0
|
||||
if config.activation_situ_linear_beta is None
|
||||
else config.activation_situ_linear_beta,
|
||||
valid_rows,
|
||||
)
|
||||
elif activation == MoEActivation.SWIGLUOAI:
|
||||
torch.ops._C.swigluoai_and_mul(output, input)
|
||||
@@ -298,7 +415,7 @@ def situ_and_mul_quant(
|
||||
beta: float,
|
||||
linear_beta: float | None,
|
||||
group_size: int = 0,
|
||||
valid_rows: torch.Tensor | None = None,
|
||||
num_valid_tokens: torch.Tensor | None = None,
|
||||
topk: int = 1,
|
||||
) -> None:
|
||||
"""Fused Kimi SITU activation + dynamic FP8 quantization.
|
||||
@@ -314,11 +431,11 @@ def situ_and_mul_quant(
|
||||
[M, d // 128], matching humming ``quant_input(group_size=128, float32)``.
|
||||
|
||||
``linear_beta`` <= 0 (or None) means "unset" (up passed through), matching
|
||||
``SituAndMul(linear_beta=None)``. ``valid_rows`` (int32 scalar tensor) is the
|
||||
DeepEP v2 contiguous-layout valid *token* count (e.g. ``psum[-1:]``); the
|
||||
kernel multiplies it by ``topk`` to bound rows, so no host-side cast/multiply
|
||||
is needed. Padding rows are skipped and receive a benign scale of 1.0. Kept
|
||||
in this module so that every ``torch.ops._C.situ*`` call lives in one file.
|
||||
``SituAndMul(linear_beta=None)``. ``num_valid_tokens`` (int32 scalar tensor)
|
||||
is the DeepEP v2 contiguous-layout token count (e.g. ``psum[-1:]``); the
|
||||
kernel multiplies it by ``topk`` to bound rows. Padding rows are skipped and
|
||||
receive a benign scale of 1.0. Kept in this module so that every
|
||||
``torch.ops._C.situ*`` call lives in one file.
|
||||
"""
|
||||
torch.ops._C.situ_and_mul_quant(
|
||||
output,
|
||||
@@ -327,6 +444,6 @@ def situ_and_mul_quant(
|
||||
beta,
|
||||
-1.0 if linear_beta is None else linear_beta,
|
||||
group_size,
|
||||
valid_rows,
|
||||
num_valid_tokens,
|
||||
topk,
|
||||
)
|
||||
|
||||
@@ -5,7 +5,10 @@
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.activation import (
|
||||
MoEActivation,
|
||||
apply_moe_activation_masked_supported,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
@@ -19,7 +22,6 @@ from vllm.model_executor.layers.fused_moe.utils import (
|
||||
_resize_cache,
|
||||
moe_kernel_quantize_input,
|
||||
normalize_batched_scales_shape,
|
||||
swiglu_limit_func,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
@@ -839,16 +841,18 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular):
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return activation in [
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.GELU,
|
||||
MoEActivation.GELU_TANH,
|
||||
MoEActivation.SWIGLUOAI,
|
||||
MoEActivation.SILU_NO_MUL,
|
||||
MoEActivation.GELU_NO_MUL,
|
||||
MoEActivation.GELU_TANH_NO_MUL,
|
||||
MoEActivation.RELU2_NO_MUL,
|
||||
]
|
||||
if current_platform.is_xpu():
|
||||
return activation in [
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.GELU,
|
||||
MoEActivation.GELU_TANH,
|
||||
MoEActivation.SWIGLUOAI,
|
||||
MoEActivation.SILU_NO_MUL,
|
||||
MoEActivation.GELU_NO_MUL,
|
||||
MoEActivation.GELU_TANH_NO_MUL,
|
||||
MoEActivation.RELU2_NO_MUL,
|
||||
]
|
||||
return apply_moe_activation_masked_supported(activation)
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
|
||||
@@ -858,20 +862,6 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular):
|
||||
# Let PrepareAndFinalize::finalize() decide the impl.
|
||||
return TopKWeightAndReduceDelegate()
|
||||
|
||||
def activation(
|
||||
self,
|
||||
activation: MoEActivation,
|
||||
output: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
gemm1_clamp_limit = self.quant_config.gemm1_clamp_limit
|
||||
if activation == MoEActivation.SILU and gemm1_clamp_limit is not None:
|
||||
swiglu_limit_func(output, input, float(gemm1_clamp_limit))
|
||||
return
|
||||
|
||||
super().activation(activation, output, input)
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
M: int,
|
||||
@@ -998,11 +988,19 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular):
|
||||
intermediate_cache2.fill_(0)
|
||||
|
||||
# TODO (bnell): use triton utility from batched deep gemm.
|
||||
self.activation(
|
||||
activation,
|
||||
intermediate_cache2.view(-1, activation_out_dim),
|
||||
intermediate_cache1.view(-1, N),
|
||||
)
|
||||
if current_platform.is_xpu():
|
||||
self.activation(
|
||||
activation,
|
||||
intermediate_cache2.view(-1, activation_out_dim),
|
||||
intermediate_cache1.view(-1, N),
|
||||
)
|
||||
else:
|
||||
self.activation(
|
||||
activation,
|
||||
intermediate_cache2,
|
||||
intermediate_cache1,
|
||||
valid_token_counts=expert_num_tokens,
|
||||
)
|
||||
|
||||
qintermediate_cache2, a2q_scale = batched_moe_kernel_quantize_input(
|
||||
intermediate_cache2,
|
||||
|
||||
@@ -35,10 +35,7 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceDelegate,
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
_resize_cache,
|
||||
swiglu_limit_func,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import _resize_cache
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
INT4_DTYPE,
|
||||
INT8_DTYPE,
|
||||
@@ -103,17 +100,18 @@ def _is_supported_wna16_weight_key(weight_key: QuantKey | None) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def get_humming_moe_gemm_type() -> str:
|
||||
def get_humming_moe_gemm_type(moe_config: FusedMoEConfig | None = None) -> str:
|
||||
env_gemm_type: str | None = envs.VLLM_HUMMING_MOE_GEMM_TYPE
|
||||
gemm_type = "indexed"
|
||||
if env_gemm_type is not None:
|
||||
if env_gemm_type is not None and env_gemm_type.lower() != "auto":
|
||||
env_gemm_type = env_gemm_type.lower()
|
||||
if env_gemm_type == "indexed":
|
||||
gemm_type = env_gemm_type
|
||||
elif env_gemm_type in ["grouped_contiguous", "grouped"]:
|
||||
if env_gemm_type == "grouped":
|
||||
gemm_type = "grouped_contiguous"
|
||||
else:
|
||||
gemm_type = "indexed"
|
||||
gemm_type = env_gemm_type
|
||||
elif moe_config is not None and moe_config.moe_parallel_config.use_ep:
|
||||
gemm_type = "grouped_contiguous"
|
||||
else:
|
||||
gemm_type = "indexed"
|
||||
|
||||
logger.info_once(f"Using {gemm_type} gemm for humming moe") # noqa
|
||||
return gemm_type
|
||||
@@ -178,7 +176,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
max_num_tokens=max_num_tokens,
|
||||
num_dispatchers=num_dispatchers,
|
||||
)
|
||||
self._permute_scratch: MoEPermuteScratch | None = None
|
||||
self._permute_scratch: dict[int, MoEPermuteScratch] = {}
|
||||
|
||||
def init_humming_moe(self):
|
||||
from vllm.utils.humming import get_heuristics_config
|
||||
@@ -257,26 +255,36 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def _get_permute_scratch(self) -> MoEPermuteScratch | None:
|
||||
if self._permute_scratch is None and moe_permute_unpermute_supported():
|
||||
self._permute_scratch = MoEPermuteScratch(
|
||||
max_num_tokens=self.moe_config.max_num_tokens,
|
||||
topk=self.moe_config.experts_per_token,
|
||||
def _get_permute_scratch(self, topk: int) -> MoEPermuteScratch | None:
|
||||
if not moe_permute_unpermute_supported():
|
||||
return None
|
||||
|
||||
scratch = self._permute_scratch.get(topk)
|
||||
if scratch is None:
|
||||
max_expanded_rows = (
|
||||
self.moe_config.max_num_tokens
|
||||
* self.moe_config.dp_size
|
||||
* self.moe_config.experts_per_token
|
||||
)
|
||||
scratch = MoEPermuteScratch(
|
||||
max_num_tokens=math.ceil(max_expanded_rows / topk),
|
||||
topk=topk,
|
||||
num_experts=self.moe_config.num_experts,
|
||||
num_local_experts=self.moe_config.num_local_experts,
|
||||
device=torch.device(self.moe_config.device),
|
||||
hidden_size=self.moe_config.hidden_dim,
|
||||
hidden_dtype=self.moe_config.in_dtype,
|
||||
)
|
||||
return self._permute_scratch
|
||||
self._permute_scratch[topk] = scratch
|
||||
return scratch
|
||||
|
||||
def get_global_valid_shape_m(self, topk_ids: torch.Tensor):
|
||||
num_tokens = topk_ids.size(0)
|
||||
ctx = get_forward_context()
|
||||
if ctx.dp_metadata is not None:
|
||||
num_tokens = ctx.dp_metadata.num_tokens_across_dp_cpu.sum().item()
|
||||
return num_tokens * self.moe_config.experts_per_token
|
||||
|
||||
return num_tokens * topk_ids.size(1)
|
||||
return topk_ids.size(0) * topk_ids.size(1)
|
||||
|
||||
def estimate_local_valid_shape_m(self, topk_ids: torch.Tensor):
|
||||
# estimate shape_m for kernel tuning
|
||||
@@ -645,7 +653,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
if supported:
|
||||
assert hasattr(cls, "humming_gemm_type")
|
||||
gemm_type = cls.humming_gemm_type().value.lower()
|
||||
preferred_gemm_type = get_humming_moe_gemm_type()
|
||||
preferred_gemm_type = get_humming_moe_gemm_type(moe_config)
|
||||
supported = preferred_gemm_type.lower() == gemm_type
|
||||
if not supported:
|
||||
reason = (
|
||||
@@ -660,25 +668,16 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
activation: MoEActivation,
|
||||
output: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
valid_rows: torch.Tensor | None = None,
|
||||
valid_token_counts: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
activation_config = self.activation_config
|
||||
if (
|
||||
activation == MoEActivation.SILU
|
||||
and activation_config.clamp_limit is not None
|
||||
):
|
||||
swiglu_limit_func(
|
||||
output=output,
|
||||
input=input,
|
||||
swiglu_limit=activation_config.clamp_limit,
|
||||
)
|
||||
else:
|
||||
self.activation(
|
||||
activation=activation,
|
||||
input=input,
|
||||
output=output,
|
||||
valid_rows=valid_rows,
|
||||
)
|
||||
activation_kwargs: dict[str, Any] = dict(
|
||||
activation=activation,
|
||||
input=input,
|
||||
output=output,
|
||||
)
|
||||
if valid_token_counts is not None:
|
||||
activation_kwargs["valid_token_counts"] = valid_token_counts
|
||||
self.activation(**activation_kwargs)
|
||||
|
||||
def fused_situ_quant_enabled(self, activation: MoEActivation) -> bool:
|
||||
"""Whether the SITU activation + w2 quant can be fused into one kernel.
|
||||
@@ -718,7 +717,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
self,
|
||||
gate_up_output: torch.Tensor,
|
||||
quanted_down_input: torch.Tensor,
|
||||
valid_rows: torch.Tensor | None,
|
||||
num_valid_tokens: torch.Tensor | None,
|
||||
topk: int = 1,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Run the fused SITU activation + FP8 quant for the w2 input.
|
||||
@@ -750,7 +749,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
beta=cfg.activation_situ_beta,
|
||||
linear_beta=cfg.activation_situ_linear_beta,
|
||||
group_size=group_size,
|
||||
valid_rows=valid_rows,
|
||||
num_valid_tokens=num_valid_tokens,
|
||||
topk=topk,
|
||||
)
|
||||
return quanted_down_input, input_scale
|
||||
@@ -878,8 +877,8 @@ class HummingIndexedExperts(HummingExpertsBase):
|
||||
)
|
||||
|
||||
# psum[-1:] is the DeepEP valid *token* count as a zero-cost int32 view.
|
||||
# situ (rows = tokens*topk) multiplies by topk on-device; mul_sum bounds
|
||||
# on tokens, so both read this pointer directly -- no host cast/multiply.
|
||||
# The fused path and mul_sum consume tokens directly. The unfused
|
||||
# activation consumes the expanded row count (tokens * topk).
|
||||
valid_tokens = None
|
||||
topk = topk_ids.size(1)
|
||||
if (
|
||||
@@ -894,21 +893,18 @@ class HummingIndexedExperts(HummingExpertsBase):
|
||||
inputs, input_scale = self.fused_situ_quant(
|
||||
gate_up_output=buffers["gate_up_output"],
|
||||
quanted_down_input=buffers["quanted_down_input"],
|
||||
valid_rows=valid_tokens,
|
||||
num_valid_tokens=valid_tokens,
|
||||
topk=topk,
|
||||
)
|
||||
else:
|
||||
# Fallback situ_and_mul takes int64 row counts (tokens*topk).
|
||||
valid_rows = (
|
||||
valid_tokens.to(torch.int64) * topk
|
||||
if valid_tokens is not None
|
||||
else None
|
||||
valid_token_counts = (
|
||||
valid_tokens * topk if valid_tokens is not None else None
|
||||
)
|
||||
self.apply_activation(
|
||||
activation=activation,
|
||||
input=buffers["gate_up_output"],
|
||||
output=buffers["activation_output"],
|
||||
valid_rows=valid_rows,
|
||||
valid_token_counts=valid_token_counts,
|
||||
)
|
||||
|
||||
inputs, input_scale = self.quantize_input(
|
||||
@@ -1002,7 +998,7 @@ class HummingGroupedExperts(HummingExpertsBase):
|
||||
n_expert=global_num_experts,
|
||||
n_local_expert=self.num_experts,
|
||||
expert_map=expert_map,
|
||||
scratch=self._get_permute_scratch(),
|
||||
scratch=self._get_permute_scratch(topk_ids.size(1)),
|
||||
)
|
||||
|
||||
inputs, input_scale = self.quantize_input(
|
||||
@@ -1028,6 +1024,11 @@ class HummingGroupedExperts(HummingExpertsBase):
|
||||
activation=activation,
|
||||
input=buffers["gate_up_output"],
|
||||
output=buffers["activation_output"],
|
||||
valid_token_counts=(
|
||||
expert_first_token_offset[-1:].to(torch.int32)
|
||||
if expert_tokens_meta is None
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
inputs, input_scale = self.quantize_input(
|
||||
|
||||
@@ -13,6 +13,7 @@ from vllm.model_executor.layers.fused_moe.activation import (
|
||||
ApplyMoEActivationConfig,
|
||||
MoEActivation,
|
||||
apply_moe_activation,
|
||||
apply_moe_activation_masked_supported,
|
||||
apply_moe_activation_supported,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
@@ -955,6 +956,10 @@ class BatchedMarlinExperts(MarlinExpertsBase):
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.BatchedExperts
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return apply_moe_activation_masked_supported(activation)
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
M: int,
|
||||
@@ -1000,26 +1005,18 @@ class BatchedMarlinExperts(MarlinExpertsBase):
|
||||
act: MoEActivation,
|
||||
act_output: torch.Tensor,
|
||||
act_input: torch.Tensor,
|
||||
**kwargs,
|
||||
*,
|
||||
topk_ids: torch.Tensor | None = None,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
if act != MoEActivation.SITU:
|
||||
self.activation(
|
||||
act,
|
||||
act_output,
|
||||
act_input,
|
||||
**kwargs,
|
||||
)
|
||||
return
|
||||
|
||||
num_experts, max_num_tokens = hidden_states.shape[:2]
|
||||
beta = self.activation_config.activation_situ_beta
|
||||
linear_beta = self.activation_config.activation_situ_linear_beta
|
||||
torch.ops._C.masked_situ_and_mul(
|
||||
self.activation(
|
||||
act,
|
||||
act_output.view(num_experts, max_num_tokens, -1),
|
||||
act_input.view(num_experts, max_num_tokens, -1),
|
||||
expert_tokens_meta.expert_num_tokens,
|
||||
1.0 if beta is None else beta,
|
||||
-1.0 if linear_beta is None else linear_beta,
|
||||
topk_ids=topk_ids,
|
||||
expert_map=expert_map,
|
||||
valid_token_counts=expert_tokens_meta.expert_num_tokens,
|
||||
)
|
||||
|
||||
return batched_fused_marlin_moe(
|
||||
|
||||
@@ -902,7 +902,7 @@ class FusedMoEExpertsModular(FusedMoEExperts):
|
||||
*,
|
||||
topk_ids: torch.Tensor | None = None,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
valid_rows: torch.Tensor | None = None,
|
||||
valid_token_counts: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
apply_moe_activation(
|
||||
activation,
|
||||
@@ -911,7 +911,7 @@ class FusedMoEExpertsModular(FusedMoEExperts):
|
||||
activation_config=self.activation_config,
|
||||
topk_ids=topk_ids,
|
||||
expert_map=expert_map,
|
||||
valid_rows=valid_rows,
|
||||
valid_token_counts=valid_token_counts,
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
|
||||
Reference in New Issue
Block a user