[Feature][Humming] Humming feature integration (#56685)

Signed-off-by: jinzhen.ljz <jinzhen.ljz@antgroup.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Jinzhen Lin
2026-09-20 08:11:01 -07:00
committed by GitHub
co-authored by Codex mgoin
parent 27757dde02
commit 7d99c2c4fd
62 changed files with 2040 additions and 1086 deletions
@@ -236,7 +236,7 @@ __device__ void _moe_align_block_size_small_batch_expert(
int32_t* __restrict__ expert_map, int32_t num_experts, int32_t block_size,
size_t numel, int32_t max_num_tokens_padded, int32_t max_num_m_blocks,
int32_t inactive_expert_id, int32_t model_offset, int32_t topk_num,
int32_t* token_mask, bool has_expert_map) {
int32_t* token_mask, bool has_expert_map, int32_t* scatter_idx = nullptr) {
// Compute input buffer offsets. Typically these will all be 0, except when
// using Multi LoRA.
int sorted_token_ids_offset = max_num_tokens_padded * model_offset;
@@ -318,9 +318,14 @@ __device__ void _moe_align_block_size_small_batch_expert(
}
for (size_t i = tid; i < numel; i += stride) {
if (int expert_id = get_local_expert_id(i, topk_ids, expert_map,
num_experts, has_expert_map);
expert_id != -1) {
int expert_id = get_local_expert_id(i, topk_ids, expert_map, num_experts,
has_expert_map);
bool valid =
expert_id != -1 && (token_mask == nullptr || token_mask[i / topk_num]);
if (scatter_idx != nullptr) {
scatter_idx[i] = valid ? static_cast<int32_t>(i) : -1;
}
if (expert_id != -1) {
int32_t rank_post_pad =
tokens_cnts[tid * num_experts + expert_id] + cumsum[expert_id];
@@ -338,14 +343,20 @@ __device__ void _count_and_sort_expert_tokens(
int32_t* __restrict__ sorted_token_ids, int32_t* __restrict__ cumsum_buffer,
int32_t* __restrict__ expert_map, size_t numel, int32_t num_experts,
int32_t max_num_tokens_padded, int32_t* __restrict__ token_mask,
int32_t model_offset, int32_t topk_num, bool has_expert_map) {
int32_t model_offset, int32_t topk_num, bool has_expert_map,
int32_t* scatter_idx = nullptr) {
const size_t tid = blockIdx.y * blockDim.x + threadIdx.x;
const size_t stride = blockDim.x * gridDim.y;
for (size_t i = tid; i < numel; i += stride) {
if (int expert_id = get_local_expert_id(i, topk_ids, expert_map,
num_experts, has_expert_map);
expert_id != -1) {
int expert_id = get_local_expert_id(i, topk_ids, expert_map, num_experts,
has_expert_map);
bool valid =
expert_id != -1 && (token_mask == nullptr || token_mask[i / topk_num]);
if (scatter_idx != nullptr) {
scatter_idx[i] = valid ? static_cast<int32_t>(i) : -1;
}
if (expert_id != -1) {
if (token_mask == nullptr || token_mask[i / topk_num]) {
int32_t rank_post_pad = atomicAdd(
&cumsum_buffer[(model_offset * (num_experts + 1)) + expert_id], 1);
@@ -377,10 +388,11 @@ __global__ void count_and_sort_expert_tokens_kernel(
const scalar_t* __restrict__ topk_ids,
int32_t* __restrict__ sorted_token_ids, int32_t* __restrict__ cumsum_buffer,
int32_t* __restrict__ expert_map, size_t numel, int32_t num_experts,
int32_t max_num_tokens_padded, int32_t topk_num, bool has_expert_map) {
int32_t max_num_tokens_padded, int32_t topk_num, bool has_expert_map,
int32_t* scatter_idx) {
_count_and_sort_expert_tokens(
topk_ids, sorted_token_ids, cumsum_buffer, expert_map, numel, num_experts,
max_num_tokens_padded, nullptr, 0, topk_num, has_expert_map);
max_num_tokens_padded, nullptr, 0, topk_num, has_expert_map, scatter_idx);
}
// Reduce the topk expert outputs per token (summed in fp32). The output is
@@ -533,12 +545,12 @@ __global__ void moe_align_block_size_small_batch_expert_kernel(
int32_t* __restrict__ total_tokens_post_pad,
int32_t* __restrict__ expert_map, int32_t num_experts, int32_t block_size,
size_t numel, int32_t max_num_tokens_padded, int32_t topk_num,
bool has_expert_map) {
bool has_expert_map, int32_t* scatter_idx) {
_moe_align_block_size_small_batch_expert<scalar_t, fill_threads>(
topk_ids, sorted_token_ids, expert_ids, total_tokens_post_pad, expert_map,
num_experts, block_size, numel, max_num_tokens_padded,
CEILDIV(max_num_tokens_padded, block_size), -1, 0, topk_num, nullptr,
has_expert_map);
has_expert_map, scatter_idx);
}
template <typename scalar_t>
@@ -658,16 +670,31 @@ __global__ void moe_lora_align_block_size_small_batch_expert_kernel(
// taken from
// https://github.com/sgl-project/sglang/blob/8b5f83ed3b7d2a49ad5c5cd5aa61c5d502f47dbc
void moe_align_block_size(
torch::stable::Tensor topk_ids, int64_t num_experts, int64_t block_size,
torch::stable::Tensor sorted_token_ids, torch::stable::Tensor experts_ids,
torch::stable::Tensor num_tokens_post_pad,
std::optional<torch::stable::Tensor> maybe_expert_map) {
void moe_align_block_size(torch::stable::Tensor topk_ids, int64_t num_experts,
int64_t block_size,
torch::stable::Tensor sorted_token_ids,
torch::stable::Tensor experts_ids,
torch::stable::Tensor num_tokens_post_pad,
std::optional<torch::stable::Tensor> maybe_expert_map,
std::optional<torch::stable::Tensor> scatter_idx) {
const torch::stable::accelerator::DeviceGuard device_guard(
topk_ids.get_device_index());
const cudaStream_t stream =
get_current_cuda_stream(topk_ids.get_device_index());
int32_t* scatter_ptr = nullptr;
if (scatter_idx.has_value()) {
STD_TORCH_CHECK(
scatter_idx->scalar_type() == torch::headeronly::ScalarType::Int,
"scatter_idx must be int32");
STD_TORCH_CHECK(scatter_idx->device() == topk_ids.device() &&
scatter_idx->is_contiguous() &&
scatter_idx->numel() == topk_ids.numel(),
"scatter_idx must be contiguous, on the same device, and "
"match topk_ids.numel()");
scatter_ptr = reinterpret_cast<int32_t*>(scatter_idx->mutable_data_ptr());
}
int64_t padded_num_experts =
((num_experts + WARP_SIZE - 1) / WARP_SIZE) * WARP_SIZE;
int experts_per_warp = WARP_SIZE;
@@ -713,7 +740,8 @@ void moe_align_block_size(
num_tokens_post_pad.mutable_data_ptr()),
reinterpret_cast<int32_t*>(expert_map.mutable_data_ptr()),
num_experts, block_size, topk_ids.numel(),
sorted_token_ids.size(0), topk_ids.size(1), has_expert_map);
sorted_token_ids.size(0), topk_ids.size(1), has_expert_map,
scatter_ptr);
} else {
torch::stable::Tensor cumsum_buffer = torch::stable::new_empty(
topk_ids, {num_experts + 1}, torch::headeronly::ScalarType::Int);
@@ -753,7 +781,7 @@ void moe_align_block_size(
reinterpret_cast<int32_t*>(cumsum_buffer.mutable_data_ptr()),
reinterpret_cast<int32_t*>(expert_map.mutable_data_ptr()),
topk_ids.numel(), num_experts, sorted_token_ids.size(0),
topk_ids.size(1), has_expert_map);
topk_ids.size(1), has_expert_map, scatter_ptr);
}
});
}
+7 -5
View File
@@ -36,11 +36,13 @@ void moe_sum(torch::stable::Tensor& input, torch::stable::Tensor& output,
std::optional<torch::stable::Tensor> topk_ids,
std::optional<torch::stable::Tensor> expert_map);
void moe_align_block_size(
torch::stable::Tensor topk_ids, int64_t num_experts, int64_t block_size,
torch::stable::Tensor sorted_token_ids, torch::stable::Tensor experts_ids,
torch::stable::Tensor num_tokens_post_pad,
std::optional<torch::stable::Tensor> maybe_expert_map);
void moe_align_block_size(torch::stable::Tensor topk_ids, int64_t num_experts,
int64_t block_size,
torch::stable::Tensor sorted_token_ids,
torch::stable::Tensor experts_ids,
torch::stable::Tensor num_tokens_post_pad,
std::optional<torch::stable::Tensor> maybe_expert_map,
std::optional<torch::stable::Tensor> scatter_idx);
void batched_moe_align_block_size(
int64_t max_tokens_per_batch, int64_t block_size,
@@ -56,20 +56,18 @@ int64_t moe_permute_sort_workspace_size(int64_t num_expanded_rows,
CubKeyValueSorter::getWorkspaceSize(num_expanded_rows, n_expert));
}
void moe_permute_impl(
const torch::stable::Tensor& input, // [n_token, hidden]
const torch::stable::Tensor& topk_ids, // [n_token, topk]
const torch::stable::Tensor& token_expert_indices, // [n_token, topk]
torch::stable::Tensor moe_sort_routing(
const torch::stable::Tensor& topk_ids, // [n_token, topk]
const torch::stable::Tensor& token_expert_indices, // [n_token, topk]
const std::optional<torch::stable::Tensor>& expert_map, // [n_expert]
int64_t n_expert, int64_t n_local_expert, int64_t topk,
torch::stable::Tensor& permuted_input, // [permuted_size, hidden]
torch::stable::Tensor& expert_first_token_offset, // [n_local_expert + 1]
torch::stable::Tensor& inv_permuted_idx, // [n_token, topk]
torch::stable::Tensor& permuted_idx, // [permute_size]
const std::optional<torch::stable::Tensor>& maybe_sort_workspace,
const std::optional<torch::stable::Tensor>& maybe_permuted_experts_id,
const std::optional<torch::stable::Tensor>& maybe_sorted_row_idx,
const std::optional<torch::stable::Tensor>& maybe_topk_ids_for_sort) {
const std::optional<torch::stable::Tensor>& maybe_topk_ids_for_sort,
bool build_inverse) {
STD_TORCH_CHECK(expert_first_token_offset.scalar_type() ==
torch::headeronly::ScalarType::Long,
"expert_first_token_offset must be int64");
@@ -88,12 +86,11 @@ void moe_permute_impl(
"token_expert_indices shape must be same as inv_permuted_idx");
const torch::stable::accelerator::DeviceGuard device_guard(
input.get_device_index());
auto device = input.device();
auto n_token = input.sizes()[0];
auto n_hidden = input.sizes()[1];
topk_ids.get_device_index());
auto device = topk_ids.device();
auto n_token = topk_ids.sizes()[0];
auto expanded_rows = n_token * topk;
auto stream = get_current_cuda_stream(input.get_device_index());
auto stream = get_current_cuda_stream(topk_ids.get_device_index());
auto sorter_size = moe_permute_sort_workspace_size(expanded_rows, n_expert);
auto sort_workspace = maybe_allocate_tensor(
@@ -107,13 +104,11 @@ void moe_permute_impl(
torch::headeronly::ScalarType::Int, device, "sorted_row_idx");
CubKeyValueSorter sorter{};
int64_t* valid_num_ptr = nullptr;
torch::stable::Tensor topk_ids_for_sort = topk_ids;
if (expert_map.has_value()) {
const int* expert_map_ptr = get_ptr<int>(expert_map.value());
valid_num_ptr =
get_ptr<int64_t>(expert_first_token_offset) + n_local_expert;
if (expert_map.has_value() || build_inverse) {
const int* expert_map_ptr =
expert_map.has_value() ? get_ptr<int>(expert_map.value()) : nullptr;
topk_ids_for_sort = maybe_allocate_tensor(
maybe_topk_ids_for_sort, topk_ids.sizes(),
torch::headeronly::ScalarType::Int, device, "topk_ids_for_sort");
@@ -126,7 +121,39 @@ void moe_permute_impl(
get_ptr<const int>(topk_ids_for_sort), get_ptr<int>(token_expert_indices),
get_ptr<int>(permuted_experts_id), get_ptr<int>(sorted_row_idx),
get_ptr<int64_t>(expert_first_token_offset), n_token, n_expert,
n_local_expert, topk, sorter, get_ptr<int>(sort_workspace), stream);
n_local_expert, topk, sorter, get_ptr<int>(sort_workspace), stream,
build_inverse ? get_ptr<int>(inv_permuted_idx) : nullptr);
return sorted_row_idx;
}
void moe_permute_impl(
const torch::stable::Tensor& input, // [n_token, hidden]
const torch::stable::Tensor& topk_ids, // [n_token, topk]
const torch::stable::Tensor& token_expert_indices, // [n_token, topk]
const std::optional<torch::stable::Tensor>& expert_map, // [n_expert]
int64_t n_expert, int64_t n_local_expert, int64_t topk,
torch::stable::Tensor& permuted_input, // [permuted_size, hidden]
torch::stable::Tensor& expert_first_token_offset, // [n_local_expert + 1]
torch::stable::Tensor& inv_permuted_idx, // [n_token, topk]
torch::stable::Tensor& permuted_idx, // [permute_size]
const std::optional<torch::stable::Tensor>& maybe_sort_workspace,
const std::optional<torch::stable::Tensor>& maybe_permuted_experts_id,
const std::optional<torch::stable::Tensor>& maybe_sorted_row_idx,
const std::optional<torch::stable::Tensor>& maybe_topk_ids_for_sort) {
const torch::stable::accelerator::DeviceGuard device_guard(
input.get_device_index());
auto sorted_row_idx = moe_sort_routing(
topk_ids, token_expert_indices, expert_map, n_expert, n_local_expert,
topk, expert_first_token_offset, inv_permuted_idx, maybe_sort_workspace,
maybe_permuted_experts_id, maybe_sorted_row_idx, maybe_topk_ids_for_sort,
false);
auto stream = get_current_cuda_stream(input.get_device_index());
auto n_token = input.size(0);
auto n_hidden = input.size(1);
auto valid_num_ptr =
expert_map.has_value()
? get_ptr<int64_t>(expert_first_token_offset) + n_local_expert
: nullptr;
MOE_DISPATCH(input.scalar_type(), [&] {
expandInputRowsKernelLauncher<scalar_t>(
@@ -171,6 +198,22 @@ void moe_permute_with_scratch(
topk_ids_for_sort);
}
void moe_prepare_scatter(const torch::stable::Tensor& topk_ids,
const torch::stable::Tensor& token_expert_indices,
const std::optional<torch::stable::Tensor>& expert_map,
int64_t n_expert, int64_t n_local_expert,
torch::stable::Tensor& expert_first_token_offset,
torch::stable::Tensor& scatter_idx,
torch::stable::Tensor& sort_workspace,
torch::stable::Tensor& sorted_experts,
torch::stable::Tensor& sorted_rows,
torch::stable::Tensor& topk_ids_for_sort) {
moe_sort_routing(topk_ids, token_expert_indices, expert_map, n_expert,
n_local_expert, topk_ids.size(1), expert_first_token_offset,
scatter_idx, sort_workspace, sorted_experts, sorted_rows,
topk_ids_for_sort, true);
}
void moe_unpermute(
const torch::stable::Tensor&
permuted_hidden_states, // [n_token * topk, hidden]
@@ -315,6 +358,19 @@ void moe_unpermute(
STD_TORCH_CHECK(false, "moe_unpermute is not supported on CUDA < 12.0");
}
void moe_prepare_scatter(const torch::stable::Tensor& topk_ids,
const torch::stable::Tensor& token_expert_indices,
const std::optional<torch::stable::Tensor>& expert_map,
int64_t n_expert, int64_t n_local_expert,
torch::stable::Tensor& expert_first_token_offset,
torch::stable::Tensor& scatter_idx,
torch::stable::Tensor& sort_workspace,
torch::stable::Tensor& sorted_experts,
torch::stable::Tensor& sorted_rows,
torch::stable::Tensor& topk_ids_for_sort) {
STD_TORCH_CHECK(false, "moe_prepare_scatter is not supported on CUDA < 12.0");
}
#endif
bool moe_permute_unpermute_supported() {
@@ -328,5 +384,6 @@ bool moe_permute_unpermute_supported() {
STABLE_TORCH_LIBRARY_IMPL(_moe_C, CUDA, m) {
m.impl("moe_permute", TORCH_BOX(&moe_permute));
m.impl("moe_permute_with_scratch", TORCH_BOX(&moe_permute_with_scratch));
m.impl("moe_prepare_scatter", TORCH_BOX(&moe_prepare_scatter));
m.impl("moe_unpermute", TORCH_BOX(&moe_unpermute));
}
@@ -82,34 +82,40 @@ __device__ inline int64_t findTotalEltsLessThanTarget(T const* sorted_indices,
return target_location + 1;
}
// Calculates the start offset of the tokens for a given expert. The last
// element is the total number of valid tokens
__global__ void computeExpertFirstTokenOffsetKernel(
// Computes expert offsets (ending with the valid row count) and optional
// inverse indices.
__global__ void computeExpertOffsetsAndInverseKernel(
int const* sorted_experts, int64_t const sorted_experts_len,
int const num_experts, int64_t* expert_first_token_offset) {
// First, compute the global tid. We only need 1 thread per expert.
int const expert = blockIdx.x * blockDim.x + threadIdx.x;
int const num_experts, int64_t* expert_first_token_offset,
int const* sorted_rows, int* inverse) {
int const index = blockIdx.x * blockDim.x + threadIdx.x;
// Note that expert goes [0, num_experts] (inclusive) because we want a count
// for the total number of active tokens at the end of the scan.
if (expert >= num_experts + 1) {
return;
if (index < num_experts + 1) {
expert_first_token_offset[index] =
findTotalEltsLessThanTarget(sorted_experts, sorted_experts_len, index);
}
if (inverse != nullptr && index < sorted_experts_len) {
inverse[sorted_rows[index]] = index;
}
expert_first_token_offset[expert] =
findTotalEltsLessThanTarget(sorted_experts, sorted_experts_len, expert);
}
void computeExpertFirstTokenOffset(int const* sorted_indices,
int const total_indices,
int const num_experts,
int64_t* expert_first_token_offset,
cudaStream_t stream) {
int const num_entries = num_experts + 1;
int const threads = std::min(1024, num_entries);
void computeExpertOffsetsAndInverse(int const* sorted_indices,
int const total_indices,
int const num_experts,
int64_t* expert_first_token_offset,
int const* sorted_rows, int* inverse,
cudaStream_t stream) {
int const num_entries = inverse != nullptr
? std::max(num_experts + 1, total_indices)
: num_experts + 1;
int const threads = inverse != nullptr ? 256 : std::min(1024, num_entries);
int const blocks = (num_entries + threads - 1) / threads;
computeExpertFirstTokenOffsetKernel<<<blocks, threads, 0, stream>>>(
sorted_indices, total_indices, num_experts, expert_first_token_offset);
computeExpertOffsetsAndInverseKernel<<<blocks, threads, 0, stream>>>(
sorted_indices, total_indices, num_experts, expert_first_token_offset,
sorted_rows, inverse);
}
void sortAndScanExpert(const int* expert_for_source_row, const int* source_rows,
@@ -117,7 +123,7 @@ void sortAndScanExpert(const int* expert_for_source_row, const int* source_rows,
int64_t* expert_first_token_offset, int num_rows,
int num_experts, int num_experts_per_node, int k,
CubKeyValueSorter& sorter, void* sorter_ws,
cudaStream_t stream) {
cudaStream_t stream, int* inverse) {
int64_t const expanded_num_rows = static_cast<int64_t>(k) * num_rows;
// We need to use the full num_experts because that is the sentinel value used
// by topk for disabled experts
@@ -127,9 +133,9 @@ void sortAndScanExpert(const int* expert_for_source_row, const int* source_rows,
sorter.run((void*)sorter_ws, sorter_ws_size_bytes, expert_for_source_row,
permuted_experts, source_rows, permuted_rows, expanded_num_rows,
stream);
computeExpertFirstTokenOffset(permuted_experts, expanded_num_rows,
num_experts_per_node, expert_first_token_offset,
stream);
computeExpertOffsetsAndInverse(
permuted_experts, expanded_num_rows, num_experts_per_node,
expert_first_token_offset, permuted_rows, inverse, stream);
}
__global__ void preprocessTopkIdKernel(int* topk_id_ptr, int size,
@@ -142,7 +148,7 @@ __global__ void preprocessTopkIdKernel(int* topk_id_ptr, int size,
extern __shared__ int smem_expert_map[];
// store expert_map in smem
for (int i = tidx; i < num_experts; i += blockDim.x) {
smem_expert_map[i] = expert_map_ptr[i];
smem_expert_map[i] = expert_map_ptr ? expert_map_ptr[i] : i;
}
__syncthreads();
@@ -43,18 +43,19 @@ class CubKeyValueSorter {
int num_bits_;
};
void computeExpertFirstTokenOffset(int const* sorted_indices,
int const total_indices,
int const num_experts,
int64_t* expert_first_token_offset,
cudaStream_t stream);
void computeExpertOffsetsAndInverse(int const* sorted_indices,
int const total_indices,
int const num_experts,
int64_t* expert_first_token_offset,
int const* sorted_rows, int* inverse,
cudaStream_t stream);
void sortAndScanExpert(const int* expert_for_source_row, const int* source_rows,
int* permuted_experts, int* permuted_rows,
int64_t* expert_first_token_offset, int num_rows,
int num_experts, int num_experts_per_node, int k,
CubKeyValueSorter& sorter, void* sorter_ws,
cudaStream_t stream);
cudaStream_t stream, int* inverse = nullptr);
template <typename T>
void expandInputRowsKernelLauncher(
+9 -1
View File
@@ -39,7 +39,8 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_moe_C, m) {
" int block_size, Tensor! sorted_token_ids,"
" Tensor! experts_ids,"
" Tensor! num_tokens_post_pad,"
" Tensor? maybe_expert_map) -> ()");
" Tensor? maybe_expert_map,"
" Tensor(a!)? scatter_idx=None) -> ()");
// Aligning the number of tokens to be processed by each expert such
// that it is divisible by the block size, but for the batched case.
@@ -105,6 +106,13 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_moe_C, m) {
"permuted_idx, Tensor! sort_workspace, Tensor! permuted_experts_id, "
"Tensor! sorted_row_idx, Tensor! topk_ids_for_sort)->()");
m.def(
"moe_prepare_scatter(Tensor topk_ids, Tensor token_expert_indices, "
"Tensor? expert_map, int n_expert, int n_local_expert, "
"Tensor! expert_first_token_offset, Tensor! scatter_idx, "
"Tensor! sort_workspace, Tensor! sorted_experts, Tensor! sorted_rows, "
"Tensor! topk_ids_for_sort)->()");
m.def(
"moe_unpermute(Tensor permuted_hidden_states, Tensor topk_weights,"
"Tensor inv_permuted_idx, Tensor? expert_first_token_offset, "