[MoE] Generalize masked activation for padded layouts (#51217)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
a232e29e9d
commit
82b7d49a6e
@@ -500,10 +500,8 @@ template <typename scalar_t>
|
||||
__global__ void situ_and_mul_kernel(
|
||||
scalar_t* __restrict__ out, // [..., d]
|
||||
const scalar_t* __restrict__ input, // [..., 2, d]
|
||||
const int d, const float beta, const float linear_beta,
|
||||
const int64_t* __restrict__ valid_rows_ptr) {
|
||||
const int d, const float beta, const float linear_beta) {
|
||||
const int64_t row = blockIdx.x;
|
||||
if (valid_rows_ptr != nullptr && row >= *valid_rows_ptr) return;
|
||||
const scalar_t* gate_ptr = input + row * 2 * d;
|
||||
const scalar_t* up_ptr = gate_ptr + d;
|
||||
scalar_t* out_ptr = out + row * d;
|
||||
@@ -558,15 +556,16 @@ template <typename scalar_t, typename fp8_type, int THREADS, int NUM_STAGES,
|
||||
__global__ void situ_and_mul_quant_group_pipelined_kernel(
|
||||
fp8_type* __restrict__ out, float* __restrict__ scale_out,
|
||||
const scalar_t* __restrict__ input, const int64_t num_rows,
|
||||
const int32_t* __restrict__ valid_rows_ptr, const int64_t topk) {
|
||||
const int32_t* __restrict__ num_valid_tokens_ptr, const int64_t topk) {
|
||||
const int tid = threadIdx.x;
|
||||
const int warp_id = tid / WARP_SIZE;
|
||||
const int lane_id = tid % WARP_SIZE;
|
||||
static constexpr int NUM_WARPS = THREADS / WARP_SIZE;
|
||||
static constexpr int NUM_GROUPS = D / GROUP_SIZE;
|
||||
const int64_t row_bound =
|
||||
valid_rows_ptr != nullptr
|
||||
? max((int64_t)0, min((int64_t)(*valid_rows_ptr) * topk, num_rows))
|
||||
num_valid_tokens_ptr != nullptr
|
||||
? max((int64_t)0,
|
||||
min((int64_t)(*num_valid_tokens_ptr) * topk, num_rows))
|
||||
: num_rows;
|
||||
|
||||
static_assert(NUM_GROUPS % 4 == 0,
|
||||
@@ -710,15 +709,16 @@ __global__ void situ_and_mul_quant_group_scalar_kernel(
|
||||
const scalar_t* __restrict__ input, // [num_tokens, 2, d]
|
||||
const int d, const int num_groups, const float beta,
|
||||
const float linear_beta, const int64_t num_rows,
|
||||
const int32_t* __restrict__ valid_rows_ptr, const int64_t topk) {
|
||||
const int32_t* __restrict__ num_valid_tokens_ptr, const int64_t topk) {
|
||||
static constexpr int ELTS_PER_LANE = GROUP_SIZE / WARP_SIZE;
|
||||
const int tid = threadIdx.x;
|
||||
const int warp_id = tid / WARP_SIZE;
|
||||
const int lane_id = tid % WARP_SIZE;
|
||||
const int num_warps = blockDim.x / WARP_SIZE;
|
||||
const int64_t row_bound =
|
||||
valid_rows_ptr != nullptr
|
||||
? max((int64_t)0, min((int64_t)(*valid_rows_ptr) * topk, num_rows))
|
||||
num_valid_tokens_ptr != nullptr
|
||||
? max((int64_t)0,
|
||||
min((int64_t)(*num_valid_tokens_ptr) * topk, num_rows))
|
||||
: num_rows;
|
||||
|
||||
const bool clamp_up = linear_beta > 0.0f;
|
||||
@@ -768,24 +768,61 @@ __global__ void situ_and_mul_quant_group_scalar_kernel(
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
constexpr int kMaxMaskedTokenBlocks = 32;
|
||||
|
||||
template <bool BATCHED_EXPERTS>
|
||||
__device__ __forceinline__ bool get_masked_row_range(
|
||||
const int* __restrict__ valid_token_counts, const int max_num_tokens,
|
||||
int64_t& first_row, int64_t& end_row, int64_t& row_stride) {
|
||||
if constexpr (BATCHED_EXPERTS) {
|
||||
// [E, T, *]: z lanes grid-stride over one expert's valid token prefix.
|
||||
const int expert = blockIdx.y;
|
||||
const int num_tokens =
|
||||
max(0, min(valid_token_counts[expert], max_num_tokens));
|
||||
const int token_block = blockIdx.z;
|
||||
if (token_block >= num_tokens) {
|
||||
return false;
|
||||
}
|
||||
const int64_t expert_first_row =
|
||||
static_cast<int64_t>(expert) * max_num_tokens;
|
||||
first_row = expert_first_row + token_block;
|
||||
end_row = expert_first_row + num_tokens;
|
||||
row_stride = gridDim.z;
|
||||
} else {
|
||||
// [T, *]: y lanes grid-stride over the single valid token prefix.
|
||||
const int num_tokens = max(0, min(valid_token_counts[0], max_num_tokens));
|
||||
const int token_block = blockIdx.y;
|
||||
if (token_block >= num_tokens) {
|
||||
return false;
|
||||
}
|
||||
first_row = token_block;
|
||||
end_row = num_tokens;
|
||||
row_stride = gridDim.y;
|
||||
}
|
||||
return first_row < end_row;
|
||||
}
|
||||
|
||||
__device__ __forceinline__ int get_masked_feature_index() {
|
||||
return blockIdx.x * blockDim.x + threadIdx.x;
|
||||
}
|
||||
|
||||
template <typename scalar_t, bool BATCHED_EXPERTS>
|
||||
__global__ void masked_situ_and_mul_kernel(
|
||||
scalar_t* __restrict__ out, const scalar_t* __restrict__ input,
|
||||
const int* __restrict__ expert_num_tokens, const int max_num_tokens,
|
||||
const int* __restrict__ valid_token_counts, const int max_num_tokens,
|
||||
const int d, const float beta, const float linear_beta) {
|
||||
const int expert = blockIdx.y;
|
||||
const int num_tokens = expert_num_tokens[expert];
|
||||
const int idx = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
if (idx >= d || num_tokens == 0) {
|
||||
int64_t first_row, end_row, row_stride;
|
||||
const int idx = get_masked_feature_index();
|
||||
if (idx >= d ||
|
||||
!get_masked_row_range<BATCHED_EXPERTS>(valid_token_counts, max_num_tokens,
|
||||
first_row, end_row, row_stride)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const bool clamp_up = linear_beta > 0.0f;
|
||||
const float inv_beta = 1.0f / beta;
|
||||
const float inv_linear_beta = clamp_up ? 1.0f / linear_beta : 0.0f;
|
||||
const int64_t expert_row = static_cast<int64_t>(expert) * max_num_tokens;
|
||||
for (int token = 0; token < num_tokens; ++token) {
|
||||
const int64_t row = expert_row + token;
|
||||
for (int64_t row = first_row; row < end_row; row += row_stride) {
|
||||
const scalar_t* gate_ptr = input + row * 2 * d;
|
||||
const scalar_t* up_ptr = gate_ptr + d;
|
||||
scalar_t* out_ptr = out + row * d;
|
||||
@@ -796,6 +833,119 @@ __global__ void masked_situ_and_mul_kernel(
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&, const float),
|
||||
bool act_first, bool HAS_CLAMP, bool BATCHED_EXPERTS>
|
||||
__global__ void masked_act_and_mul_kernel(
|
||||
scalar_t* __restrict__ out, const scalar_t* __restrict__ input,
|
||||
const int* __restrict__ valid_token_counts, const int max_num_tokens,
|
||||
const int d, const float limit, const float alpha, const float beta) {
|
||||
int64_t first_row, end_row, row_stride;
|
||||
const int idx = get_masked_feature_index();
|
||||
if (idx >= d ||
|
||||
!get_masked_row_range<BATCHED_EXPERTS>(valid_token_counts, max_num_tokens,
|
||||
first_row, end_row, row_stride)) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (int64_t row = first_row; row < end_row; row += row_stride) {
|
||||
const scalar_t* x_ptr = input + row * 2 * d;
|
||||
const scalar_t* y_ptr = x_ptr + d;
|
||||
scalar_t* out_ptr = out + row * d;
|
||||
const scalar_t x = VLLM_LDG(&x_ptr[idx]);
|
||||
const scalar_t y = VLLM_LDG(&y_ptr[idx]);
|
||||
out_ptr[idx] = compute<scalar_t, ACT_FN, act_first, HAS_CLAMP>(x, y, limit,
|
||||
alpha, beta);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t, bool BATCHED_EXPERTS>
|
||||
__global__ void masked_swigluoai_and_mul_kernel(
|
||||
scalar_t* __restrict__ out, const scalar_t* __restrict__ input,
|
||||
const int* __restrict__ valid_token_counts, const int max_num_tokens,
|
||||
const int d, const float alpha, const float limit) {
|
||||
int64_t first_row, end_row, row_stride;
|
||||
const int idx = get_masked_feature_index();
|
||||
if (idx >= d ||
|
||||
!get_masked_row_range<BATCHED_EXPERTS>(valid_token_counts, max_num_tokens,
|
||||
first_row, end_row, row_stride)) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (int64_t row = first_row; row < end_row; row += row_stride) {
|
||||
const scalar_t* in_ptr = input + row * 2 * d;
|
||||
scalar_t* out_ptr = out + row * d;
|
||||
out_ptr[idx] =
|
||||
swigluoai_and_mul(in_ptr[2 * idx], in_ptr[2 * idx + 1], alpha, limit);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t, bool BATCHED_EXPERTS>
|
||||
__global__ void masked_swiglustep_and_mul_kernel(
|
||||
scalar_t* __restrict__ out, const scalar_t* __restrict__ input,
|
||||
const int* __restrict__ valid_token_counts, const int max_num_tokens,
|
||||
const int d, const float limit) {
|
||||
int64_t first_row, end_row, row_stride;
|
||||
const int idx = get_masked_feature_index();
|
||||
if (idx >= d ||
|
||||
!get_masked_row_range<BATCHED_EXPERTS>(valid_token_counts, max_num_tokens,
|
||||
first_row, end_row, row_stride)) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (int64_t row = first_row; row < end_row; row += row_stride) {
|
||||
const scalar_t* gate_ptr = input + row * 2 * d;
|
||||
const scalar_t* up_ptr = gate_ptr + d;
|
||||
scalar_t* out_ptr = out + row * d;
|
||||
const float gate = (float)VLLM_LDG(&gate_ptr[idx]);
|
||||
const float up = (float)VLLM_LDG(&up_ptr[idx]);
|
||||
const float gate_silu = gate / (1.0f + expf(-gate));
|
||||
const float gate_clamped = fminf(gate_silu, limit);
|
||||
const float up_clamped = fmaxf(fminf(up, limit), -limit);
|
||||
out_ptr[idx] = (scalar_t)(gate_clamped * up_clamped);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__device__ __forceinline__ T silu_no_mul_kernel(const T& x) {
|
||||
return silu_kernel(x, 1.0f);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__device__ __forceinline__ T gelu_no_mul_kernel(const T& x) {
|
||||
return gelu_kernel(x, 1.0f);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__device__ __forceinline__ T gelu_tanh_no_mul_kernel(const T& x) {
|
||||
return gelu_tanh_kernel(x, 1.0f);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__device__ __forceinline__ T relu2_no_mul_kernel(const T& x) {
|
||||
const float value = fmaxf((float)x, 0.0f);
|
||||
return (T)(value * value);
|
||||
}
|
||||
|
||||
template <typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&),
|
||||
bool BATCHED_EXPERTS>
|
||||
__global__ void masked_activation_kernel(
|
||||
scalar_t* __restrict__ out, const scalar_t* __restrict__ input,
|
||||
const int* __restrict__ valid_token_counts, const int max_num_tokens,
|
||||
const int d) {
|
||||
int64_t first_row, end_row, row_stride;
|
||||
const int idx = get_masked_feature_index();
|
||||
if (idx >= d ||
|
||||
!get_masked_row_range<BATCHED_EXPERTS>(valid_token_counts, max_num_tokens,
|
||||
first_row, end_row, row_stride)) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (int64_t row = first_row; row < end_row; row += row_stride) {
|
||||
const int64_t offset = row * d + idx;
|
||||
out[offset] = ACT_FN(VLLM_LDG(&input[offset]));
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace vllm
|
||||
|
||||
#define LAUNCH_ACTIVATION_GATE_KERNEL_WITH_PARAM(KERNEL, PACKED_KERNEL, PARAM) \
|
||||
@@ -890,8 +1040,7 @@ void swigluoai_and_mul(torch::stable::Tensor& out, // [..., d]
|
||||
// through), matching SituAndMul(linear_beta=None) on the Python side.
|
||||
void situ_and_mul(torch::stable::Tensor& out, // [..., d]
|
||||
torch::stable::Tensor& input, // [..., 2 * d]
|
||||
double beta, double linear_beta,
|
||||
std::optional<torch::stable::Tensor> valid_rows) {
|
||||
double beta, double linear_beta) {
|
||||
int d = input.size(-1) / 2;
|
||||
int64_t num_tokens = input.numel() / input.size(-1);
|
||||
if (num_tokens == 0) {
|
||||
@@ -899,14 +1048,6 @@ void situ_and_mul(torch::stable::Tensor& out, // [..., d]
|
||||
}
|
||||
dim3 grid(num_tokens);
|
||||
dim3 block(std::min(d, 1024));
|
||||
const int64_t* valid_rows_ptr = nullptr;
|
||||
if (valid_rows.has_value()) {
|
||||
STD_TORCH_CHECK(valid_rows->is_cuda() && valid_rows->get_device_index() ==
|
||||
input.get_device_index(),
|
||||
"situ_and_mul: valid_rows must be a CUDA tensor on the "
|
||||
"same device as input");
|
||||
valid_rows_ptr = valid_rows->const_data_ptr<int64_t>();
|
||||
}
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
input.get_device_index());
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
@@ -914,7 +1055,7 @@ void situ_and_mul(torch::stable::Tensor& out, // [..., d]
|
||||
input.scalar_type(), "situ_and_mul_kernel", [&] {
|
||||
vllm::situ_and_mul_kernel<scalar_t><<<grid, block, 0, stream>>>(
|
||||
out.mutable_data_ptr<scalar_t>(), input.const_data_ptr<scalar_t>(),
|
||||
d, (float)beta, (float)linear_beta, valid_rows_ptr);
|
||||
d, (float)beta, (float)linear_beta);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -925,7 +1066,7 @@ void masked_situ_and_mul(torch::stable::Tensor& out, // [E, T, d]
|
||||
int num_experts = input.size(0);
|
||||
int max_num_tokens = input.size(1);
|
||||
int d = input.size(2) / 2;
|
||||
if (num_experts == 0 || max_num_tokens == 0) {
|
||||
if (num_experts == 0 || max_num_tokens == 0 || d == 0) {
|
||||
return;
|
||||
}
|
||||
constexpr int block_size = 256;
|
||||
@@ -936,21 +1077,128 @@ void masked_situ_and_mul(torch::stable::Tensor& out, // [E, T, d]
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
|
||||
input.scalar_type(), "masked_situ_and_mul_kernel", [&] {
|
||||
vllm::masked_situ_and_mul_kernel<scalar_t><<<grid, block, 0, stream>>>(
|
||||
out.mutable_data_ptr<scalar_t>(), input.const_data_ptr<scalar_t>(),
|
||||
expert_num_tokens.const_data_ptr<int>(), max_num_tokens, d,
|
||||
(float)beta, (float)linear_beta);
|
||||
vllm::masked_situ_and_mul_kernel<scalar_t, true>
|
||||
<<<grid, block, 0, stream>>>(
|
||||
out.mutable_data_ptr<scalar_t>(),
|
||||
input.const_data_ptr<scalar_t>(),
|
||||
expert_num_tokens.const_data_ptr<int>(), max_num_tokens, d,
|
||||
(float)beta, (float)linear_beta);
|
||||
});
|
||||
}
|
||||
|
||||
#define LAUNCH_MASKED_ACT_AND_MUL(KERNEL, ACT_FIRST, HAS_CLAMP, LIMIT, ALPHA, \
|
||||
BETA) \
|
||||
vllm::masked_act_and_mul_kernel<scalar_t, KERNEL<scalar_t>, ACT_FIRST, \
|
||||
HAS_CLAMP, BATCHED_EXPERTS> \
|
||||
<<<grid, block, 0, stream>>>(out.mutable_data_ptr<scalar_t>(), \
|
||||
input.const_data_ptr<scalar_t>(), \
|
||||
valid_token_counts.const_data_ptr<int>(), \
|
||||
max_num_tokens, d, LIMIT, ALPHA, BETA)
|
||||
|
||||
#define LAUNCH_MASKED_ACTIVATION(KERNEL) \
|
||||
vllm::masked_activation_kernel<scalar_t, KERNEL<scalar_t>, BATCHED_EXPERTS> \
|
||||
<<<grid, block, 0, stream>>>( \
|
||||
out.mutable_data_ptr<scalar_t>(), input.const_data_ptr<scalar_t>(), \
|
||||
valid_token_counts.const_data_ptr<int>(), max_num_tokens, d)
|
||||
|
||||
void masked_moe_activation(
|
||||
torch::stable::Tensor& out, // [T, d] or [E, T, d]
|
||||
torch::stable::Tensor& input, // [T, d] or [E, T, d], gated has 2 * d
|
||||
const torch::stable::Tensor& valid_token_counts,
|
||||
const std::string& activation, double clamp_limit, double alpha,
|
||||
double beta, double situ_beta, double situ_linear_beta) {
|
||||
const bool batched_experts = input.dim() == 3;
|
||||
const int num_experts = batched_experts ? input.size(0) : 1;
|
||||
const int max_num_tokens = batched_experts ? input.size(1) : input.size(0);
|
||||
const int d = out.size(-1);
|
||||
if (num_experts == 0 || max_num_tokens == 0 || d == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
constexpr int block_size = 256;
|
||||
const int feature_blocks = (d + block_size - 1) / block_size;
|
||||
const int token_blocks =
|
||||
std::min(max_num_tokens, vllm::kMaxMaskedTokenBlocks);
|
||||
// Batched grid: (feature tile, expert, token lane); flat grid: (feature
|
||||
// tile, token lane). Token lanes grid-stride each valid prefix.
|
||||
dim3 grid = batched_experts ? dim3(feature_blocks, num_experts, token_blocks)
|
||||
: dim3(feature_blocks, token_blocks);
|
||||
dim3 block(block_size);
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
input.get_device_index());
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
auto launch = [&]<bool BATCHED_EXPERTS>() {
|
||||
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
|
||||
input.scalar_type(), "masked_moe_activation", [&] {
|
||||
if (activation == "silu") {
|
||||
LAUNCH_MASKED_ACT_AND_MUL(vllm::silu_kernel, true, false, 0.0f,
|
||||
1.0f, 0.0f);
|
||||
} else if (activation == "silu_with_clamp") {
|
||||
LAUNCH_MASKED_ACT_AND_MUL(vllm::silu_kernel, true, true,
|
||||
(float)clamp_limit, (float)alpha,
|
||||
(float)beta);
|
||||
} else if (activation == "gelu") {
|
||||
LAUNCH_MASKED_ACT_AND_MUL(vllm::gelu_kernel, true, false, 0.0f,
|
||||
1.0f, 0.0f);
|
||||
} else if (activation == "gelu_tanh") {
|
||||
LAUNCH_MASKED_ACT_AND_MUL(vllm::gelu_tanh_kernel, true, false, 0.0f,
|
||||
1.0f, 0.0f);
|
||||
} else if (activation == "situ") {
|
||||
vllm::masked_situ_and_mul_kernel<scalar_t, BATCHED_EXPERTS>
|
||||
<<<grid, block, 0, stream>>>(
|
||||
out.mutable_data_ptr<scalar_t>(),
|
||||
input.const_data_ptr<scalar_t>(),
|
||||
valid_token_counts.const_data_ptr<int>(), max_num_tokens, d,
|
||||
(float)situ_beta, (float)situ_linear_beta);
|
||||
} else if (activation == "swigluoai") {
|
||||
vllm::masked_swigluoai_and_mul_kernel<scalar_t, BATCHED_EXPERTS>
|
||||
<<<grid, block, 0, stream>>>(
|
||||
out.mutable_data_ptr<scalar_t>(),
|
||||
input.const_data_ptr<scalar_t>(),
|
||||
valid_token_counts.const_data_ptr<int>(), max_num_tokens, d,
|
||||
(float)alpha, (float)clamp_limit);
|
||||
} else if (activation == "swigluoai_uninterleave") {
|
||||
LAUNCH_MASKED_ACT_AND_MUL(vllm::silu_kernel, true, true,
|
||||
(float)clamp_limit, (float)alpha,
|
||||
(float)beta);
|
||||
} else if (activation == "swiglustep") {
|
||||
vllm::masked_swiglustep_and_mul_kernel<scalar_t, BATCHED_EXPERTS>
|
||||
<<<grid, block, 0, stream>>>(
|
||||
out.mutable_data_ptr<scalar_t>(),
|
||||
input.const_data_ptr<scalar_t>(),
|
||||
valid_token_counts.const_data_ptr<int>(), max_num_tokens, d,
|
||||
(float)clamp_limit);
|
||||
} else if (activation == "silu_no_mul") {
|
||||
LAUNCH_MASKED_ACTIVATION(vllm::silu_no_mul_kernel);
|
||||
} else if (activation == "gelu_no_mul") {
|
||||
LAUNCH_MASKED_ACTIVATION(vllm::gelu_no_mul_kernel);
|
||||
} else if (activation == "gelu_tanh_no_mul") {
|
||||
LAUNCH_MASKED_ACTIVATION(vllm::gelu_tanh_no_mul_kernel);
|
||||
} else if (activation == "relu2_no_mul") {
|
||||
LAUNCH_MASKED_ACTIVATION(vllm::relu2_no_mul_kernel);
|
||||
} else {
|
||||
STD_TORCH_CHECK(false,
|
||||
"Unsupported masked MoE activation: ", activation);
|
||||
}
|
||||
});
|
||||
};
|
||||
if (batched_experts) {
|
||||
launch.template operator()<true>();
|
||||
} else {
|
||||
launch.template operator()<false>();
|
||||
}
|
||||
}
|
||||
|
||||
#undef LAUNCH_MASKED_ACT_AND_MUL
|
||||
#undef LAUNCH_MASKED_ACTIVATION
|
||||
// Fused SITU activation and block-FP8 quantization for Humming w2.
|
||||
// `valid_rows` is the int32 DeepEP token count (psum[-1]); rows = it * `topk`,
|
||||
// so padding is excluded on-device and skipped rows receive scale 1.
|
||||
// `num_valid_tokens` is the int32 DeepEP token count (psum[-1]); rows = it *
|
||||
// `topk`, so padding is excluded on-device and skipped rows receive scale 1.
|
||||
void situ_and_mul_quant(torch::stable::Tensor& out, // [..., d] (fp8)
|
||||
torch::stable::Tensor& scale, // [..., 1 or d/128]
|
||||
torch::stable::Tensor& input, // [..., 2 * d]
|
||||
double beta, double linear_beta, int64_t group_size,
|
||||
std::optional<torch::stable::Tensor> valid_rows,
|
||||
std::optional<torch::stable::Tensor> num_valid_tokens,
|
||||
int64_t topk) {
|
||||
STD_TORCH_CHECK(
|
||||
out.scalar_type() == torch::headeronly::ScalarType::Float8_e4m3fn ||
|
||||
@@ -974,16 +1222,17 @@ void situ_and_mul_quant(torch::stable::Tensor& out, // [..., d] (fp8)
|
||||
}
|
||||
STD_TORCH_CHECK(out.size(-1) == d && out.numel() == num_tokens * (int64_t)d,
|
||||
"situ_and_mul_quant: out shape must be [num_tokens, d]");
|
||||
const int32_t* valid_rows_ptr = nullptr;
|
||||
if (valid_rows.has_value()) {
|
||||
STD_TORCH_CHECK(valid_rows->is_cuda() && valid_rows->get_device_index() ==
|
||||
input.get_device_index(),
|
||||
"situ_and_mul_quant: valid_rows must be a CUDA tensor on "
|
||||
"the same device as input");
|
||||
const int32_t* num_valid_tokens_ptr = nullptr;
|
||||
if (num_valid_tokens.has_value()) {
|
||||
STD_TORCH_CHECK(
|
||||
valid_rows->scalar_type() == torch::headeronly::ScalarType::Int,
|
||||
"situ_and_mul_quant: valid_rows must be int32 (psum count)");
|
||||
valid_rows_ptr = valid_rows->const_data_ptr<int32_t>();
|
||||
num_valid_tokens->is_cuda() &&
|
||||
num_valid_tokens->get_device_index() == input.get_device_index(),
|
||||
"situ_and_mul_quant: num_valid_tokens must be a CUDA tensor on the "
|
||||
"same device as input");
|
||||
STD_TORCH_CHECK(
|
||||
num_valid_tokens->scalar_type() == torch::headeronly::ScalarType::Int,
|
||||
"situ_and_mul_quant: num_valid_tokens must be int32 (psum count)");
|
||||
num_valid_tokens_ptr = num_valid_tokens->const_data_ptr<int32_t>();
|
||||
}
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
input.get_device_index());
|
||||
@@ -1031,7 +1280,7 @@ void situ_and_mul_quant(torch::stable::Tensor& out, // [..., d] (fp8)
|
||||
out.mutable_data_ptr<fp8_t>(),
|
||||
scale.mutable_data_ptr<float>(),
|
||||
input.const_data_ptr<scalar_t>(), num_tokens,
|
||||
valid_rows_ptr, topk);
|
||||
num_valid_tokens_ptr, topk);
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -1045,7 +1294,7 @@ void situ_and_mul_quant(torch::stable::Tensor& out, // [..., d] (fp8)
|
||||
scale.mutable_data_ptr<float>(),
|
||||
input.const_data_ptr<scalar_t>(), d, num_groups,
|
||||
(float)beta, (float)linear_beta, num_tokens,
|
||||
valid_rows_ptr, topk);
|
||||
num_valid_tokens_ptr, topk);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
@@ -146,16 +146,17 @@ __global__ void preprocessTopkIdKernel(int* topk_id_ptr, int size,
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
// query global expert id in expert map.
|
||||
// if global expert id = -1 in exert map, plus n_expert
|
||||
// else set global expert id = exert map[global expert id]
|
||||
if (offset + tidx < bound) {
|
||||
auto topk_id = topk_id_ptr[offset + tidx];
|
||||
auto local_expert_idx = smem_expert_map[topk_id];
|
||||
if (local_expert_idx == -1) {
|
||||
topk_id += num_experts;
|
||||
if (topk_id < 0 || topk_id >= num_experts) {
|
||||
topk_id = num_experts;
|
||||
} else {
|
||||
topk_id = local_expert_idx;
|
||||
auto local_expert_idx = smem_expert_map[topk_id];
|
||||
if (local_expert_idx == -1) {
|
||||
topk_id += num_experts;
|
||||
} else {
|
||||
topk_id = local_expert_idx;
|
||||
}
|
||||
}
|
||||
__syncwarp();
|
||||
topk_id_ptr[offset + tidx] = topk_id;
|
||||
|
||||
@@ -530,20 +530,25 @@ void fatrelu_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input,
|
||||
double threshold);
|
||||
void swigluoai_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input,
|
||||
double alpha = 1.702, double limit = 7.0);
|
||||
void situ_and_mul(
|
||||
torch::stable::Tensor& out, torch::stable::Tensor& input, double beta = 1.0,
|
||||
double linear_beta = -1.0,
|
||||
std::optional<torch::stable::Tensor> valid_rows = std::nullopt);
|
||||
void situ_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input,
|
||||
double beta = 1.0, double linear_beta = -1.0);
|
||||
void situ_and_mul_quant(
|
||||
torch::stable::Tensor& out, torch::stable::Tensor& scale,
|
||||
torch::stable::Tensor& input, double beta = 1.0, double linear_beta = -1.0,
|
||||
int64_t group_size = 0,
|
||||
std::optional<torch::stable::Tensor> valid_rows = std::nullopt,
|
||||
std::optional<torch::stable::Tensor> num_valid_tokens = std::nullopt,
|
||||
int64_t topk = 1);
|
||||
void masked_situ_and_mul(torch::stable::Tensor& out,
|
||||
torch::stable::Tensor& input,
|
||||
const torch::stable::Tensor& expert_num_tokens,
|
||||
double beta = 1.0, double linear_beta = -1.0);
|
||||
void masked_moe_activation(torch::stable::Tensor& out,
|
||||
torch::stable::Tensor& input,
|
||||
const torch::stable::Tensor& valid_token_counts,
|
||||
const std::string& activation,
|
||||
double clamp_limit = 0.0, double alpha = 1.0,
|
||||
double beta = 0.0, double situ_beta = 1.0,
|
||||
double situ_linear_beta = -1.0);
|
||||
void gelu_new(torch::stable::Tensor& out, torch::stable::Tensor& input);
|
||||
void gelu_fast(torch::stable::Tensor& out, torch::stable::Tensor& input);
|
||||
void gelu_quick(torch::stable::Tensor& out, torch::stable::Tensor& input);
|
||||
|
||||
@@ -614,7 +614,7 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
// SituGLU implementation used in Kimi models.
|
||||
ops.def(
|
||||
"situ_and_mul(Tensor! out, Tensor input, float beta=1.0, float "
|
||||
"linear_beta=-1.0, Tensor? valid_rows=None) -> ()");
|
||||
"linear_beta=-1.0) -> ()");
|
||||
// Fused SituGLU activation + dynamic FP8 quantization for the Humming w2 path
|
||||
// (writes the fp8 down input and its float32 scale). group_size=0 ->
|
||||
// per-token scale [.., 1]; group_size=128 -> k-major block-FP8 scale [..,
|
||||
@@ -622,10 +622,15 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
ops.def(
|
||||
"situ_and_mul_quant(Tensor! out, Tensor! scale, Tensor input, "
|
||||
"float beta=1.0, float linear_beta=-1.0, int group_size=0, "
|
||||
"Tensor? valid_rows=None, int topk=1) -> ()");
|
||||
"Tensor? num_valid_tokens=None, int topk=1) -> ()");
|
||||
ops.def(
|
||||
"masked_situ_and_mul(Tensor! out, Tensor input, Tensor "
|
||||
"expert_num_tokens, float beta=1.0, float linear_beta=-1.0) -> ()");
|
||||
ops.def(
|
||||
"masked_moe_activation(Tensor! out, Tensor input, Tensor "
|
||||
"valid_token_counts, str activation, float clamp_limit=0.0, float "
|
||||
"alpha=1.0, float beta=0.0, float situ_beta=1.0, float "
|
||||
"situ_linear_beta=-1.0) -> ()");
|
||||
|
||||
// GELU implementation used in GPT-2.
|
||||
ops.def("gelu_new(Tensor! out, Tensor input) -> ()");
|
||||
@@ -852,6 +857,7 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
|
||||
ops.impl("situ_and_mul", TORCH_BOX(&situ_and_mul));
|
||||
ops.impl("situ_and_mul_quant", TORCH_BOX(&situ_and_mul_quant));
|
||||
ops.impl("masked_situ_and_mul", TORCH_BOX(&masked_situ_and_mul));
|
||||
ops.impl("masked_moe_activation", TORCH_BOX(&masked_moe_activation));
|
||||
ops.impl("gelu_new", TORCH_BOX(&gelu_new));
|
||||
ops.impl("gelu_fast", TORCH_BOX(&gelu_fast));
|
||||
ops.impl("gelu_quick", TORCH_BOX(&gelu_quick));
|
||||
|
||||
Reference in New Issue
Block a user