[Feature][Humming] Humming feature integration (#56685)
Signed-off-by: jinzhen.ljz <jinzhen.ljz@antgroup.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Codex
mgoin
parent
27757dde02
commit
7d99c2c4fd
@@ -236,7 +236,7 @@ __device__ void _moe_align_block_size_small_batch_expert(
|
||||
int32_t* __restrict__ expert_map, int32_t num_experts, int32_t block_size,
|
||||
size_t numel, int32_t max_num_tokens_padded, int32_t max_num_m_blocks,
|
||||
int32_t inactive_expert_id, int32_t model_offset, int32_t topk_num,
|
||||
int32_t* token_mask, bool has_expert_map) {
|
||||
int32_t* token_mask, bool has_expert_map, int32_t* scatter_idx = nullptr) {
|
||||
// Compute input buffer offsets. Typically these will all be 0, except when
|
||||
// using Multi LoRA.
|
||||
int sorted_token_ids_offset = max_num_tokens_padded * model_offset;
|
||||
@@ -318,9 +318,14 @@ __device__ void _moe_align_block_size_small_batch_expert(
|
||||
}
|
||||
|
||||
for (size_t i = tid; i < numel; i += stride) {
|
||||
if (int expert_id = get_local_expert_id(i, topk_ids, expert_map,
|
||||
num_experts, has_expert_map);
|
||||
expert_id != -1) {
|
||||
int expert_id = get_local_expert_id(i, topk_ids, expert_map, num_experts,
|
||||
has_expert_map);
|
||||
bool valid =
|
||||
expert_id != -1 && (token_mask == nullptr || token_mask[i / topk_num]);
|
||||
if (scatter_idx != nullptr) {
|
||||
scatter_idx[i] = valid ? static_cast<int32_t>(i) : -1;
|
||||
}
|
||||
if (expert_id != -1) {
|
||||
int32_t rank_post_pad =
|
||||
tokens_cnts[tid * num_experts + expert_id] + cumsum[expert_id];
|
||||
|
||||
@@ -338,14 +343,20 @@ __device__ void _count_and_sort_expert_tokens(
|
||||
int32_t* __restrict__ sorted_token_ids, int32_t* __restrict__ cumsum_buffer,
|
||||
int32_t* __restrict__ expert_map, size_t numel, int32_t num_experts,
|
||||
int32_t max_num_tokens_padded, int32_t* __restrict__ token_mask,
|
||||
int32_t model_offset, int32_t topk_num, bool has_expert_map) {
|
||||
int32_t model_offset, int32_t topk_num, bool has_expert_map,
|
||||
int32_t* scatter_idx = nullptr) {
|
||||
const size_t tid = blockIdx.y * blockDim.x + threadIdx.x;
|
||||
const size_t stride = blockDim.x * gridDim.y;
|
||||
|
||||
for (size_t i = tid; i < numel; i += stride) {
|
||||
if (int expert_id = get_local_expert_id(i, topk_ids, expert_map,
|
||||
num_experts, has_expert_map);
|
||||
expert_id != -1) {
|
||||
int expert_id = get_local_expert_id(i, topk_ids, expert_map, num_experts,
|
||||
has_expert_map);
|
||||
bool valid =
|
||||
expert_id != -1 && (token_mask == nullptr || token_mask[i / topk_num]);
|
||||
if (scatter_idx != nullptr) {
|
||||
scatter_idx[i] = valid ? static_cast<int32_t>(i) : -1;
|
||||
}
|
||||
if (expert_id != -1) {
|
||||
if (token_mask == nullptr || token_mask[i / topk_num]) {
|
||||
int32_t rank_post_pad = atomicAdd(
|
||||
&cumsum_buffer[(model_offset * (num_experts + 1)) + expert_id], 1);
|
||||
@@ -377,10 +388,11 @@ __global__ void count_and_sort_expert_tokens_kernel(
|
||||
const scalar_t* __restrict__ topk_ids,
|
||||
int32_t* __restrict__ sorted_token_ids, int32_t* __restrict__ cumsum_buffer,
|
||||
int32_t* __restrict__ expert_map, size_t numel, int32_t num_experts,
|
||||
int32_t max_num_tokens_padded, int32_t topk_num, bool has_expert_map) {
|
||||
int32_t max_num_tokens_padded, int32_t topk_num, bool has_expert_map,
|
||||
int32_t* scatter_idx) {
|
||||
_count_and_sort_expert_tokens(
|
||||
topk_ids, sorted_token_ids, cumsum_buffer, expert_map, numel, num_experts,
|
||||
max_num_tokens_padded, nullptr, 0, topk_num, has_expert_map);
|
||||
max_num_tokens_padded, nullptr, 0, topk_num, has_expert_map, scatter_idx);
|
||||
}
|
||||
|
||||
// Reduce the topk expert outputs per token (summed in fp32). The output is
|
||||
@@ -533,12 +545,12 @@ __global__ void moe_align_block_size_small_batch_expert_kernel(
|
||||
int32_t* __restrict__ total_tokens_post_pad,
|
||||
int32_t* __restrict__ expert_map, int32_t num_experts, int32_t block_size,
|
||||
size_t numel, int32_t max_num_tokens_padded, int32_t topk_num,
|
||||
bool has_expert_map) {
|
||||
bool has_expert_map, int32_t* scatter_idx) {
|
||||
_moe_align_block_size_small_batch_expert<scalar_t, fill_threads>(
|
||||
topk_ids, sorted_token_ids, expert_ids, total_tokens_post_pad, expert_map,
|
||||
num_experts, block_size, numel, max_num_tokens_padded,
|
||||
CEILDIV(max_num_tokens_padded, block_size), -1, 0, topk_num, nullptr,
|
||||
has_expert_map);
|
||||
has_expert_map, scatter_idx);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
@@ -658,16 +670,31 @@ __global__ void moe_lora_align_block_size_small_batch_expert_kernel(
|
||||
|
||||
// taken from
|
||||
// https://github.com/sgl-project/sglang/blob/8b5f83ed3b7d2a49ad5c5cd5aa61c5d502f47dbc
|
||||
void moe_align_block_size(
|
||||
torch::stable::Tensor topk_ids, int64_t num_experts, int64_t block_size,
|
||||
torch::stable::Tensor sorted_token_ids, torch::stable::Tensor experts_ids,
|
||||
torch::stable::Tensor num_tokens_post_pad,
|
||||
std::optional<torch::stable::Tensor> maybe_expert_map) {
|
||||
void moe_align_block_size(torch::stable::Tensor topk_ids, int64_t num_experts,
|
||||
int64_t block_size,
|
||||
torch::stable::Tensor sorted_token_ids,
|
||||
torch::stable::Tensor experts_ids,
|
||||
torch::stable::Tensor num_tokens_post_pad,
|
||||
std::optional<torch::stable::Tensor> maybe_expert_map,
|
||||
std::optional<torch::stable::Tensor> scatter_idx) {
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
topk_ids.get_device_index());
|
||||
const cudaStream_t stream =
|
||||
get_current_cuda_stream(topk_ids.get_device_index());
|
||||
|
||||
int32_t* scatter_ptr = nullptr;
|
||||
if (scatter_idx.has_value()) {
|
||||
STD_TORCH_CHECK(
|
||||
scatter_idx->scalar_type() == torch::headeronly::ScalarType::Int,
|
||||
"scatter_idx must be int32");
|
||||
STD_TORCH_CHECK(scatter_idx->device() == topk_ids.device() &&
|
||||
scatter_idx->is_contiguous() &&
|
||||
scatter_idx->numel() == topk_ids.numel(),
|
||||
"scatter_idx must be contiguous, on the same device, and "
|
||||
"match topk_ids.numel()");
|
||||
scatter_ptr = reinterpret_cast<int32_t*>(scatter_idx->mutable_data_ptr());
|
||||
}
|
||||
|
||||
int64_t padded_num_experts =
|
||||
((num_experts + WARP_SIZE - 1) / WARP_SIZE) * WARP_SIZE;
|
||||
int experts_per_warp = WARP_SIZE;
|
||||
@@ -713,7 +740,8 @@ void moe_align_block_size(
|
||||
num_tokens_post_pad.mutable_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(expert_map.mutable_data_ptr()),
|
||||
num_experts, block_size, topk_ids.numel(),
|
||||
sorted_token_ids.size(0), topk_ids.size(1), has_expert_map);
|
||||
sorted_token_ids.size(0), topk_ids.size(1), has_expert_map,
|
||||
scatter_ptr);
|
||||
} else {
|
||||
torch::stable::Tensor cumsum_buffer = torch::stable::new_empty(
|
||||
topk_ids, {num_experts + 1}, torch::headeronly::ScalarType::Int);
|
||||
@@ -753,7 +781,7 @@ void moe_align_block_size(
|
||||
reinterpret_cast<int32_t*>(cumsum_buffer.mutable_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(expert_map.mutable_data_ptr()),
|
||||
topk_ids.numel(), num_experts, sorted_token_ids.size(0),
|
||||
topk_ids.size(1), has_expert_map);
|
||||
topk_ids.size(1), has_expert_map, scatter_ptr);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -36,11 +36,13 @@ void moe_sum(torch::stable::Tensor& input, torch::stable::Tensor& output,
|
||||
std::optional<torch::stable::Tensor> topk_ids,
|
||||
std::optional<torch::stable::Tensor> expert_map);
|
||||
|
||||
void moe_align_block_size(
|
||||
torch::stable::Tensor topk_ids, int64_t num_experts, int64_t block_size,
|
||||
torch::stable::Tensor sorted_token_ids, torch::stable::Tensor experts_ids,
|
||||
torch::stable::Tensor num_tokens_post_pad,
|
||||
std::optional<torch::stable::Tensor> maybe_expert_map);
|
||||
void moe_align_block_size(torch::stable::Tensor topk_ids, int64_t num_experts,
|
||||
int64_t block_size,
|
||||
torch::stable::Tensor sorted_token_ids,
|
||||
torch::stable::Tensor experts_ids,
|
||||
torch::stable::Tensor num_tokens_post_pad,
|
||||
std::optional<torch::stable::Tensor> maybe_expert_map,
|
||||
std::optional<torch::stable::Tensor> scatter_idx);
|
||||
|
||||
void batched_moe_align_block_size(
|
||||
int64_t max_tokens_per_batch, int64_t block_size,
|
||||
|
||||
@@ -56,20 +56,18 @@ int64_t moe_permute_sort_workspace_size(int64_t num_expanded_rows,
|
||||
CubKeyValueSorter::getWorkspaceSize(num_expanded_rows, n_expert));
|
||||
}
|
||||
|
||||
void moe_permute_impl(
|
||||
const torch::stable::Tensor& input, // [n_token, hidden]
|
||||
const torch::stable::Tensor& topk_ids, // [n_token, topk]
|
||||
const torch::stable::Tensor& token_expert_indices, // [n_token, topk]
|
||||
torch::stable::Tensor moe_sort_routing(
|
||||
const torch::stable::Tensor& topk_ids, // [n_token, topk]
|
||||
const torch::stable::Tensor& token_expert_indices, // [n_token, topk]
|
||||
const std::optional<torch::stable::Tensor>& expert_map, // [n_expert]
|
||||
int64_t n_expert, int64_t n_local_expert, int64_t topk,
|
||||
torch::stable::Tensor& permuted_input, // [permuted_size, hidden]
|
||||
torch::stable::Tensor& expert_first_token_offset, // [n_local_expert + 1]
|
||||
torch::stable::Tensor& inv_permuted_idx, // [n_token, topk]
|
||||
torch::stable::Tensor& permuted_idx, // [permute_size]
|
||||
const std::optional<torch::stable::Tensor>& maybe_sort_workspace,
|
||||
const std::optional<torch::stable::Tensor>& maybe_permuted_experts_id,
|
||||
const std::optional<torch::stable::Tensor>& maybe_sorted_row_idx,
|
||||
const std::optional<torch::stable::Tensor>& maybe_topk_ids_for_sort) {
|
||||
const std::optional<torch::stable::Tensor>& maybe_topk_ids_for_sort,
|
||||
bool build_inverse) {
|
||||
STD_TORCH_CHECK(expert_first_token_offset.scalar_type() ==
|
||||
torch::headeronly::ScalarType::Long,
|
||||
"expert_first_token_offset must be int64");
|
||||
@@ -88,12 +86,11 @@ void moe_permute_impl(
|
||||
"token_expert_indices shape must be same as inv_permuted_idx");
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
input.get_device_index());
|
||||
auto device = input.device();
|
||||
auto n_token = input.sizes()[0];
|
||||
auto n_hidden = input.sizes()[1];
|
||||
topk_ids.get_device_index());
|
||||
auto device = topk_ids.device();
|
||||
auto n_token = topk_ids.sizes()[0];
|
||||
auto expanded_rows = n_token * topk;
|
||||
auto stream = get_current_cuda_stream(input.get_device_index());
|
||||
auto stream = get_current_cuda_stream(topk_ids.get_device_index());
|
||||
|
||||
auto sorter_size = moe_permute_sort_workspace_size(expanded_rows, n_expert);
|
||||
auto sort_workspace = maybe_allocate_tensor(
|
||||
@@ -107,13 +104,11 @@ void moe_permute_impl(
|
||||
torch::headeronly::ScalarType::Int, device, "sorted_row_idx");
|
||||
|
||||
CubKeyValueSorter sorter{};
|
||||
int64_t* valid_num_ptr = nullptr;
|
||||
torch::stable::Tensor topk_ids_for_sort = topk_ids;
|
||||
|
||||
if (expert_map.has_value()) {
|
||||
const int* expert_map_ptr = get_ptr<int>(expert_map.value());
|
||||
valid_num_ptr =
|
||||
get_ptr<int64_t>(expert_first_token_offset) + n_local_expert;
|
||||
if (expert_map.has_value() || build_inverse) {
|
||||
const int* expert_map_ptr =
|
||||
expert_map.has_value() ? get_ptr<int>(expert_map.value()) : nullptr;
|
||||
topk_ids_for_sort = maybe_allocate_tensor(
|
||||
maybe_topk_ids_for_sort, topk_ids.sizes(),
|
||||
torch::headeronly::ScalarType::Int, device, "topk_ids_for_sort");
|
||||
@@ -126,7 +121,39 @@ void moe_permute_impl(
|
||||
get_ptr<const int>(topk_ids_for_sort), get_ptr<int>(token_expert_indices),
|
||||
get_ptr<int>(permuted_experts_id), get_ptr<int>(sorted_row_idx),
|
||||
get_ptr<int64_t>(expert_first_token_offset), n_token, n_expert,
|
||||
n_local_expert, topk, sorter, get_ptr<int>(sort_workspace), stream);
|
||||
n_local_expert, topk, sorter, get_ptr<int>(sort_workspace), stream,
|
||||
build_inverse ? get_ptr<int>(inv_permuted_idx) : nullptr);
|
||||
return sorted_row_idx;
|
||||
}
|
||||
|
||||
void moe_permute_impl(
|
||||
const torch::stable::Tensor& input, // [n_token, hidden]
|
||||
const torch::stable::Tensor& topk_ids, // [n_token, topk]
|
||||
const torch::stable::Tensor& token_expert_indices, // [n_token, topk]
|
||||
const std::optional<torch::stable::Tensor>& expert_map, // [n_expert]
|
||||
int64_t n_expert, int64_t n_local_expert, int64_t topk,
|
||||
torch::stable::Tensor& permuted_input, // [permuted_size, hidden]
|
||||
torch::stable::Tensor& expert_first_token_offset, // [n_local_expert + 1]
|
||||
torch::stable::Tensor& inv_permuted_idx, // [n_token, topk]
|
||||
torch::stable::Tensor& permuted_idx, // [permute_size]
|
||||
const std::optional<torch::stable::Tensor>& maybe_sort_workspace,
|
||||
const std::optional<torch::stable::Tensor>& maybe_permuted_experts_id,
|
||||
const std::optional<torch::stable::Tensor>& maybe_sorted_row_idx,
|
||||
const std::optional<torch::stable::Tensor>& maybe_topk_ids_for_sort) {
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
input.get_device_index());
|
||||
auto sorted_row_idx = moe_sort_routing(
|
||||
topk_ids, token_expert_indices, expert_map, n_expert, n_local_expert,
|
||||
topk, expert_first_token_offset, inv_permuted_idx, maybe_sort_workspace,
|
||||
maybe_permuted_experts_id, maybe_sorted_row_idx, maybe_topk_ids_for_sort,
|
||||
false);
|
||||
auto stream = get_current_cuda_stream(input.get_device_index());
|
||||
auto n_token = input.size(0);
|
||||
auto n_hidden = input.size(1);
|
||||
auto valid_num_ptr =
|
||||
expert_map.has_value()
|
||||
? get_ptr<int64_t>(expert_first_token_offset) + n_local_expert
|
||||
: nullptr;
|
||||
|
||||
MOE_DISPATCH(input.scalar_type(), [&] {
|
||||
expandInputRowsKernelLauncher<scalar_t>(
|
||||
@@ -171,6 +198,22 @@ void moe_permute_with_scratch(
|
||||
topk_ids_for_sort);
|
||||
}
|
||||
|
||||
void moe_prepare_scatter(const torch::stable::Tensor& topk_ids,
|
||||
const torch::stable::Tensor& token_expert_indices,
|
||||
const std::optional<torch::stable::Tensor>& expert_map,
|
||||
int64_t n_expert, int64_t n_local_expert,
|
||||
torch::stable::Tensor& expert_first_token_offset,
|
||||
torch::stable::Tensor& scatter_idx,
|
||||
torch::stable::Tensor& sort_workspace,
|
||||
torch::stable::Tensor& sorted_experts,
|
||||
torch::stable::Tensor& sorted_rows,
|
||||
torch::stable::Tensor& topk_ids_for_sort) {
|
||||
moe_sort_routing(topk_ids, token_expert_indices, expert_map, n_expert,
|
||||
n_local_expert, topk_ids.size(1), expert_first_token_offset,
|
||||
scatter_idx, sort_workspace, sorted_experts, sorted_rows,
|
||||
topk_ids_for_sort, true);
|
||||
}
|
||||
|
||||
void moe_unpermute(
|
||||
const torch::stable::Tensor&
|
||||
permuted_hidden_states, // [n_token * topk, hidden]
|
||||
@@ -315,6 +358,19 @@ void moe_unpermute(
|
||||
STD_TORCH_CHECK(false, "moe_unpermute is not supported on CUDA < 12.0");
|
||||
}
|
||||
|
||||
void moe_prepare_scatter(const torch::stable::Tensor& topk_ids,
|
||||
const torch::stable::Tensor& token_expert_indices,
|
||||
const std::optional<torch::stable::Tensor>& expert_map,
|
||||
int64_t n_expert, int64_t n_local_expert,
|
||||
torch::stable::Tensor& expert_first_token_offset,
|
||||
torch::stable::Tensor& scatter_idx,
|
||||
torch::stable::Tensor& sort_workspace,
|
||||
torch::stable::Tensor& sorted_experts,
|
||||
torch::stable::Tensor& sorted_rows,
|
||||
torch::stable::Tensor& topk_ids_for_sort) {
|
||||
STD_TORCH_CHECK(false, "moe_prepare_scatter is not supported on CUDA < 12.0");
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
bool moe_permute_unpermute_supported() {
|
||||
@@ -328,5 +384,6 @@ bool moe_permute_unpermute_supported() {
|
||||
STABLE_TORCH_LIBRARY_IMPL(_moe_C, CUDA, m) {
|
||||
m.impl("moe_permute", TORCH_BOX(&moe_permute));
|
||||
m.impl("moe_permute_with_scratch", TORCH_BOX(&moe_permute_with_scratch));
|
||||
m.impl("moe_prepare_scatter", TORCH_BOX(&moe_prepare_scatter));
|
||||
m.impl("moe_unpermute", TORCH_BOX(&moe_unpermute));
|
||||
}
|
||||
|
||||
+30
-24
@@ -82,34 +82,40 @@ __device__ inline int64_t findTotalEltsLessThanTarget(T const* sorted_indices,
|
||||
return target_location + 1;
|
||||
}
|
||||
|
||||
// Calculates the start offset of the tokens for a given expert. The last
|
||||
// element is the total number of valid tokens
|
||||
__global__ void computeExpertFirstTokenOffsetKernel(
|
||||
// Computes expert offsets (ending with the valid row count) and optional
|
||||
// inverse indices.
|
||||
__global__ void computeExpertOffsetsAndInverseKernel(
|
||||
int const* sorted_experts, int64_t const sorted_experts_len,
|
||||
int const num_experts, int64_t* expert_first_token_offset) {
|
||||
// First, compute the global tid. We only need 1 thread per expert.
|
||||
int const expert = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
int const num_experts, int64_t* expert_first_token_offset,
|
||||
int const* sorted_rows, int* inverse) {
|
||||
int const index = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
|
||||
// Note that expert goes [0, num_experts] (inclusive) because we want a count
|
||||
// for the total number of active tokens at the end of the scan.
|
||||
if (expert >= num_experts + 1) {
|
||||
return;
|
||||
if (index < num_experts + 1) {
|
||||
expert_first_token_offset[index] =
|
||||
findTotalEltsLessThanTarget(sorted_experts, sorted_experts_len, index);
|
||||
}
|
||||
if (inverse != nullptr && index < sorted_experts_len) {
|
||||
inverse[sorted_rows[index]] = index;
|
||||
}
|
||||
expert_first_token_offset[expert] =
|
||||
findTotalEltsLessThanTarget(sorted_experts, sorted_experts_len, expert);
|
||||
}
|
||||
|
||||
void computeExpertFirstTokenOffset(int const* sorted_indices,
|
||||
int const total_indices,
|
||||
int const num_experts,
|
||||
int64_t* expert_first_token_offset,
|
||||
cudaStream_t stream) {
|
||||
int const num_entries = num_experts + 1;
|
||||
int const threads = std::min(1024, num_entries);
|
||||
void computeExpertOffsetsAndInverse(int const* sorted_indices,
|
||||
int const total_indices,
|
||||
int const num_experts,
|
||||
int64_t* expert_first_token_offset,
|
||||
int const* sorted_rows, int* inverse,
|
||||
cudaStream_t stream) {
|
||||
int const num_entries = inverse != nullptr
|
||||
? std::max(num_experts + 1, total_indices)
|
||||
: num_experts + 1;
|
||||
int const threads = inverse != nullptr ? 256 : std::min(1024, num_entries);
|
||||
int const blocks = (num_entries + threads - 1) / threads;
|
||||
|
||||
computeExpertFirstTokenOffsetKernel<<<blocks, threads, 0, stream>>>(
|
||||
sorted_indices, total_indices, num_experts, expert_first_token_offset);
|
||||
computeExpertOffsetsAndInverseKernel<<<blocks, threads, 0, stream>>>(
|
||||
sorted_indices, total_indices, num_experts, expert_first_token_offset,
|
||||
sorted_rows, inverse);
|
||||
}
|
||||
|
||||
void sortAndScanExpert(const int* expert_for_source_row, const int* source_rows,
|
||||
@@ -117,7 +123,7 @@ void sortAndScanExpert(const int* expert_for_source_row, const int* source_rows,
|
||||
int64_t* expert_first_token_offset, int num_rows,
|
||||
int num_experts, int num_experts_per_node, int k,
|
||||
CubKeyValueSorter& sorter, void* sorter_ws,
|
||||
cudaStream_t stream) {
|
||||
cudaStream_t stream, int* inverse) {
|
||||
int64_t const expanded_num_rows = static_cast<int64_t>(k) * num_rows;
|
||||
// We need to use the full num_experts because that is the sentinel value used
|
||||
// by topk for disabled experts
|
||||
@@ -127,9 +133,9 @@ void sortAndScanExpert(const int* expert_for_source_row, const int* source_rows,
|
||||
sorter.run((void*)sorter_ws, sorter_ws_size_bytes, expert_for_source_row,
|
||||
permuted_experts, source_rows, permuted_rows, expanded_num_rows,
|
||||
stream);
|
||||
computeExpertFirstTokenOffset(permuted_experts, expanded_num_rows,
|
||||
num_experts_per_node, expert_first_token_offset,
|
||||
stream);
|
||||
computeExpertOffsetsAndInverse(
|
||||
permuted_experts, expanded_num_rows, num_experts_per_node,
|
||||
expert_first_token_offset, permuted_rows, inverse, stream);
|
||||
}
|
||||
|
||||
__global__ void preprocessTopkIdKernel(int* topk_id_ptr, int size,
|
||||
@@ -142,7 +148,7 @@ __global__ void preprocessTopkIdKernel(int* topk_id_ptr, int size,
|
||||
extern __shared__ int smem_expert_map[];
|
||||
// store expert_map in smem
|
||||
for (int i = tidx; i < num_experts; i += blockDim.x) {
|
||||
smem_expert_map[i] = expert_map_ptr[i];
|
||||
smem_expert_map[i] = expert_map_ptr ? expert_map_ptr[i] : i;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
|
||||
@@ -43,18 +43,19 @@ class CubKeyValueSorter {
|
||||
int num_bits_;
|
||||
};
|
||||
|
||||
void computeExpertFirstTokenOffset(int const* sorted_indices,
|
||||
int const total_indices,
|
||||
int const num_experts,
|
||||
int64_t* expert_first_token_offset,
|
||||
cudaStream_t stream);
|
||||
void computeExpertOffsetsAndInverse(int const* sorted_indices,
|
||||
int const total_indices,
|
||||
int const num_experts,
|
||||
int64_t* expert_first_token_offset,
|
||||
int const* sorted_rows, int* inverse,
|
||||
cudaStream_t stream);
|
||||
|
||||
void sortAndScanExpert(const int* expert_for_source_row, const int* source_rows,
|
||||
int* permuted_experts, int* permuted_rows,
|
||||
int64_t* expert_first_token_offset, int num_rows,
|
||||
int num_experts, int num_experts_per_node, int k,
|
||||
CubKeyValueSorter& sorter, void* sorter_ws,
|
||||
cudaStream_t stream);
|
||||
cudaStream_t stream, int* inverse = nullptr);
|
||||
|
||||
template <typename T>
|
||||
void expandInputRowsKernelLauncher(
|
||||
|
||||
@@ -39,7 +39,8 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_moe_C, m) {
|
||||
" int block_size, Tensor! sorted_token_ids,"
|
||||
" Tensor! experts_ids,"
|
||||
" Tensor! num_tokens_post_pad,"
|
||||
" Tensor? maybe_expert_map) -> ()");
|
||||
" Tensor? maybe_expert_map,"
|
||||
" Tensor(a!)? scatter_idx=None) -> ()");
|
||||
|
||||
// Aligning the number of tokens to be processed by each expert such
|
||||
// that it is divisible by the block size, but for the batched case.
|
||||
@@ -105,6 +106,13 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_moe_C, m) {
|
||||
"permuted_idx, Tensor! sort_workspace, Tensor! permuted_experts_id, "
|
||||
"Tensor! sorted_row_idx, Tensor! topk_ids_for_sort)->()");
|
||||
|
||||
m.def(
|
||||
"moe_prepare_scatter(Tensor topk_ids, Tensor token_expert_indices, "
|
||||
"Tensor? expert_map, int n_expert, int n_local_expert, "
|
||||
"Tensor! expert_first_token_offset, Tensor! scatter_idx, "
|
||||
"Tensor! sort_workspace, Tensor! sorted_experts, Tensor! sorted_rows, "
|
||||
"Tensor! topk_ids_for_sort)->()");
|
||||
|
||||
m.def(
|
||||
"moe_unpermute(Tensor permuted_hidden_states, Tensor topk_weights,"
|
||||
"Tensor inv_permuted_idx, Tensor? expert_first_token_offset, "
|
||||
|
||||
Reference in New Issue
Block a user