[10c/n] Migrate MoE kernels to torch stable ABI (#44565)

Signed-off-by: Chris Leonard <chleonar@redhat.com>
Co-authored-by: Shengqi Chen <harry-chen@outlook.com>
This commit is contained in:
Chris Leonard
2026-06-10 23:02:26 -07:00
committed by GitHub
co-authored by Shengqi Chen
parent 2f2c5cf4f1
commit 6e64c1bab1
34 changed files with 1296 additions and 1098 deletions
+1 -1
View File
@@ -233,7 +233,7 @@ actionlint
shellcheck*/
# Ignore moe/marlin_moe gen code
csrc/moe/marlin_moe_wna16/kernel_*
csrc/libtorch_stable/moe/marlin_moe_wna16/kernel_*
# Ignore ep_kernels_workspace folder
ep_kernels_workspace/
+1 -1
View File
@@ -21,7 +21,7 @@ repos:
rev: v21.1.2
hooks:
- id: clang-format
exclude: 'csrc/(moe/topk_softmax_kernels.cu|libtorch_stable/quantization/gguf/(ggml-common.h|dequantize.cuh|vecdotq.cuh|mmq.cuh|mmvq.cuh))|vllm/third_party/.*'
exclude: 'csrc/(libtorch_stable/moe/topk_softmax_kernels.cu|libtorch_stable/quantization/gguf/(ggml-common.h|dequantize.cuh|vecdotq.cuh|mmq.cuh|mmvq.cuh))|vllm/third_party/.*'
types_or: [c++, cuda]
args: [--style=file, --verbose]
- repo: https://github.com/DavidAnson/markdownlint-cli2
+61 -20
View File
@@ -1115,25 +1115,25 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
endif()
#
# _moe_C extension
# _moe_C_stable_libtorch extension
#
set(VLLM_MOE_EXT_SRC
"csrc/moe/torch_bindings.cpp"
"csrc/moe/moe_align_sum_kernels.cu"
"csrc/moe/topk_softmax_kernels.cu"
"csrc/moe/topk_softplus_sqrt_kernels.cu")
"csrc/libtorch_stable/moe/torch_bindings.cpp"
"csrc/libtorch_stable/moe/moe_align_sum_kernels.cu"
"csrc/libtorch_stable/moe/topk_softmax_kernels.cu"
"csrc/libtorch_stable/moe/topk_softplus_sqrt_kernels.cu")
if(VLLM_GPU_LANG STREQUAL "CUDA")
list(APPEND VLLM_MOE_EXT_SRC
"csrc/moe/moe_wna16.cu"
"csrc/moe/grouped_topk_kernels.cu")
"csrc/libtorch_stable/moe/moe_wna16.cu"
"csrc/libtorch_stable/moe/grouped_topk_kernels.cu")
endif()
if(VLLM_GPU_LANG STREQUAL "CUDA")
set(MOE_PERMUTE_SRC
"csrc/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.cu"
"csrc/moe/moe_permute_unpermute_op.cu")
"csrc/libtorch_stable/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.cu"
"csrc/libtorch_stable/moe/moe_permute_unpermute_op.cu")
list(APPEND VLLM_MOE_EXT_SRC "${MOE_PERMUTE_SRC}")
endif()
@@ -1144,7 +1144,7 @@ set_gencode_flags_for_srcs(
if(VLLM_GPU_LANG STREQUAL "CUDA")
set(VLLM_MOE_WNA16_SRC
"csrc/moe/moe_wna16.cu")
"csrc/libtorch_stable/moe/moe_wna16.cu")
set_gencode_flags_for_srcs(
SRCS "${VLLM_MOE_WNA16_SRC}"
@@ -1175,7 +1175,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
# preselected input type pairs and schedules.
# Generate sources:
set(MOE_MARLIN_GEN_SCRIPT
${CMAKE_CURRENT_SOURCE_DIR}/csrc/moe/marlin_moe_wna16/generate_kernels.py)
${CMAKE_CURRENT_SOURCE_DIR}/csrc/libtorch_stable/moe/marlin_moe_wna16/generate_kernels.py)
file(MD5 ${MOE_MARLIN_GEN_SCRIPT} MOE_MARLIN_GEN_SCRIPT_HASH)
list(JOIN CUDA_ARCHS "," CUDA_ARCHS_STR)
set(MOE_MARLIN_GEN_SCRIPT_HASH_AND_ARCH "${MOE_MARLIN_GEN_SCRIPT_HASH}(ARCH:${CUDA_ARCHS_STR})")
@@ -1210,7 +1210,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
endif()
if (MARLIN_MOE_ARCHS)
file(GLOB MARLIN_MOE_SRC "csrc/moe/marlin_moe_wna16/sm80_kernel_*.cu")
file(GLOB MARLIN_MOE_SRC "csrc/libtorch_stable/moe/marlin_moe_wna16/sm80_kernel_*.cu")
set_gencode_flags_for_srcs(
SRCS "${MARLIN_MOE_SRC}"
CUDA_ARCHS "${MARLIN_MOE_ARCHS}")
@@ -1222,7 +1222,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
endif()
if (MARLIN_MOE_SM75_ARCHS)
file(GLOB MARLIN_MOE_SM75_SRC "csrc/moe/marlin_moe_wna16/sm75_kernel_*.cu")
file(GLOB MARLIN_MOE_SM75_SRC "csrc/libtorch_stable/moe/marlin_moe_wna16/sm75_kernel_*.cu")
set_gencode_flags_for_srcs(
SRCS "${MARLIN_MOE_SM75_SRC}"
CUDA_ARCHS "${MARLIN_MOE_SM75_ARCHS}")
@@ -1234,7 +1234,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
endif()
if (MARLIN_MOE_FP8_ARCHS)
file(GLOB MARLIN_MOE_FP8_SRC "csrc/moe/marlin_moe_wna16/sm89_kernel_*.cu")
file(GLOB MARLIN_MOE_FP8_SRC "csrc/libtorch_stable/moe/marlin_moe_wna16/sm89_kernel_*.cu")
set_gencode_flags_for_srcs(
SRCS "${MARLIN_MOE_FP8_SRC}"
CUDA_ARCHS "${MARLIN_MOE_FP8_ARCHS}")
@@ -1245,7 +1245,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
list(APPEND VLLM_MOE_EXT_SRC ${MARLIN_MOE_FP8_SRC})
endif()
set(MARLIN_MOE_OTHER_SRC "csrc/moe/marlin_moe_wna16/ops.cu")
set(MARLIN_MOE_OTHER_SRC "csrc/libtorch_stable/moe/marlin_moe_wna16/ops.cu")
set_gencode_flags_for_srcs(
SRCS "${MARLIN_MOE_OTHER_SRC}"
CUDA_ARCHS "${MARLIN_MOE_OTHER_ARCHS}")
@@ -1266,9 +1266,9 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
cuda_archs_sm90plus(SM90PLUS_ROUTER_GEMM_ARCHS "${CUDA_ARCHS}")
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.0 AND SM90PLUS_ROUTER_GEMM_ARCHS)
set(DSV3_ROUTER_GEMM_SRC
"csrc/moe/dsv3_router_gemm_entry.cu"
"csrc/moe/dsv3_router_gemm_float_out.cu"
"csrc/moe/dsv3_router_gemm_bf16_out.cu")
"csrc/libtorch_stable/moe/dsv3_router_gemm_entry.cu"
"csrc/libtorch_stable/moe/dsv3_router_gemm_float_out.cu"
"csrc/libtorch_stable/moe/dsv3_router_gemm_bf16_out.cu")
set_gencode_flags_for_srcs(
SRCS "${DSV3_ROUTER_GEMM_SRC}"
CUDA_ARCHS "${SM90PLUS_ROUTER_GEMM_ARCHS}")
@@ -1281,9 +1281,9 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
endif()
endif()
message(STATUS "Enabling moe extension.")
message(STATUS "Enabling MoE C_stable extension.")
define_extension_target(
_moe_C
_moe_C_stable_libtorch
DESTINATION vllm
LANGUAGE ${VLLM_GPU_LANG}
SOURCES ${VLLM_MOE_EXT_SRC}
@@ -1294,6 +1294,47 @@ define_extension_target(
USE_SABI 3
WITH_SOABI)
# Needed to use cuda/hip APIs from C-shim
if(VLLM_GPU_LANG STREQUAL "CUDA")
# Set TORCH_TARGET_VERSION for stable ABI compatibility.
# This ensures we only use C-shim APIs available in PyTorch 2.11.
# _moe_C_stable_libtorch is abi compatible with PyTorch >= TORCH_TARGET_VERSION
# which is currently set to 2.11.
target_compile_definitions(_moe_C_stable_libtorch PRIVATE
TORCH_TARGET_VERSION=0x020B000000000000ULL)
target_compile_definitions(_moe_C_stable_libtorch PRIVATE USE_CUDA)
# Needed by CUTLASS kernels
target_compile_definitions(_moe_C_stable_libtorch PRIVATE
CUTLASS_ENABLE_DIRECT_CUDA_DRIVER_CALL=1)
elseif(VLLM_GPU_LANG STREQUAL "HIP")
# Set TORCH_TARGET_VERSION for stable ABI compatibility.
# This ensures we only use C-shim APIs available in PyTorch 2.10.
# _moe_C_stable_libtorch is abi compatible with PyTorch >= TORCH_TARGET_VERSION
# which is currently set to 2.10.
target_compile_definitions(_moe_C_stable_libtorch PRIVATE
TORCH_TARGET_VERSION=0x020A000000000000ULL)
target_compile_definitions(_moe_C_stable_libtorch PRIVATE USE_ROCM)
endif()
# On ROCm, _moe_C_stable_libtorch calls raw HIP APIs (e.g. hipGetDevice in
# get_device_prop()) which must resolve to the same libamdhip64.so that
# PyTorch uses. When PyTorch bundles its own copy (pip/conda wheels),
# the raw HIP calls would otherwise resolve to the system ROCm copy,
# initializing a second HIP runtime that corrupts device state (wrong
# device on DeviceGuard, core dumps on multi-GPU tests).
#
# If PyTorch doesn't bundle libamdhip64 (built from source against system
# ROCm), there is only one copy in the process and no action is needed —
# the HIP compiler already links the system libamdhip64 automatically.
if(VLLM_GPU_LANG STREQUAL "HIP")
find_library(_MOE_STABLE_TORCH_AMDHIP64 amdhip64
PATHS "${TORCH_INSTALL_PREFIX}/lib" NO_DEFAULT_PATH)
if(_MOE_STABLE_TORCH_AMDHIP64)
message(STATUS "Found PyTorch-bundled libamdhip64 for _moe_C_stable_libtorch at ${_MOE_STABLE_TORCH_AMDHIP64}")
target_link_libraries(_moe_C_stable_libtorch PRIVATE ${_MOE_STABLE_TORCH_AMDHIP64})
endif()
endif()
if(VLLM_GPU_LANG STREQUAL "HIP")
#
# _rocm_C extension
+22
View File
@@ -30,6 +30,28 @@
THO_DISPATCH_SWITCH(TYPE, NAME, \
VLLM_STABLE_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
#define VLLM_STABLE_DISPATCH_CASE_INTEGRAL_TYPES(...) \
THO_DISPATCH_CASE(torch::headeronly::ScalarType::Byte, __VA_ARGS__) \
THO_DISPATCH_CASE(torch::headeronly::ScalarType::Char, __VA_ARGS__) \
THO_DISPATCH_CASE(torch::headeronly::ScalarType::Short, __VA_ARGS__) \
THO_DISPATCH_CASE(torch::headeronly::ScalarType::Int, __VA_ARGS__) \
THO_DISPATCH_CASE(torch::headeronly::ScalarType::Long, __VA_ARGS__)
#define VLLM_STABLE_DISPATCH_CASE_INTEGRAL_AND_UNSIGNED_TYPES(...) \
VLLM_STABLE_DISPATCH_CASE_INTEGRAL_TYPES(__VA_ARGS__) \
THO_DISPATCH_CASE(torch::headeronly::ScalarType::UInt16, __VA_ARGS__) \
THO_DISPATCH_CASE(torch::headeronly::ScalarType::UInt32, __VA_ARGS__) \
THO_DISPATCH_CASE(torch::headeronly::ScalarType::UInt64, __VA_ARGS__)
#define VLLM_STABLE_DISPATCH_INTEGRAL_TYPES(TYPE, NAME, ...) \
THO_DISPATCH_SWITCH(TYPE, NAME, \
VLLM_STABLE_DISPATCH_CASE_INTEGRAL_TYPES(__VA_ARGS__))
#define VLLM_STABLE_DISPATCH_INTEGRAL_AND_UNSIGNED_TYPES(TYPE, NAME, ...) \
THO_DISPATCH_SWITCH( \
TYPE, NAME, \
VLLM_STABLE_DISPATCH_CASE_INTEGRAL_AND_UNSIGNED_TYPES(__VA_ARGS__))
// FP8 type dispatch - ROCm uses FNUZ format, CUDA uses OCP format
#ifdef USE_ROCM
#define VLLM_STABLE_DISPATCH_CASE_FP8_TYPES(...) \
@@ -18,14 +18,11 @@
* limitations under the License.
*/
#include <ATen/ATen.h>
#include <ATen/cuda/CUDAContext.h>
#include <cstdint>
#include <cuda_bf16.h>
#include <cuda_runtime.h>
#include "dsv3_router_gemm_utils.h"
// Custom FMA implementation using PTX assembly instructions
__device__ __forceinline__ void fma(float2& d, float2 const& a, float2 const& b,
float2 const& c) {
@@ -18,15 +18,25 @@
* limitations under the License.
*/
#include <ATen/ATen.h>
#include <ATen/cuda/CUDAContext.h>
#include <torch/all.h>
#include <torch/csrc/stable/library.h>
#include <torch/csrc/stable/tensor.h>
#include <torch/headeronly/core/ScalarType.h>
#include "libtorch_stable/torch_utils.h"
#include <cuda_bf16.h>
#include <cuda_runtime.h>
#include "core/registration.h"
#include "dsv3_router_gemm_utils.h"
#include <stdexcept>
namespace {
inline int getSMVersion() {
auto* props = get_device_prop();
return props->major * 10 + props->minor;
}
} // namespace
static constexpr int DEFAULT_NUM_EXPERTS = 256;
static constexpr int KIMI_K2_NUM_EXPERTS = 384;
@@ -98,40 +108,48 @@ struct LoopUnroller<kEnd, kEnd, kNumExperts, kHiddenDim> {
}
};
void dsv3_router_gemm(at::Tensor& output, // [num_tokens, num_experts]
const at::Tensor& mat_a, // [num_tokens, hidden_dim]
const at::Tensor& mat_b // [num_experts, hidden_dim]
void dsv3_router_gemm(
torch::stable::Tensor& output, // [num_tokens, num_experts]
torch::stable::Tensor const& mat_a, // [num_tokens, hidden_dim]
torch::stable::Tensor const& mat_b // [num_experts, hidden_dim]
) {
TORCH_CHECK(output.dim() == 2 && mat_a.dim() == 2 && mat_b.dim() == 2);
STD_TORCH_CHECK(output.dim() == 2 && mat_a.dim() == 2 && mat_b.dim() == 2);
const int num_tokens = mat_a.size(0);
const int num_experts = mat_b.size(0);
const int hidden_dim = mat_a.size(1);
TORCH_CHECK(mat_a.size(1) == mat_b.size(1),
"mat_a and mat_b must have the same hidden_dim");
TORCH_CHECK(hidden_dim == DEFAULT_HIDDEN_DIM,
"Expected hidden_dim=", DEFAULT_HIDDEN_DIM,
", but got hidden_dim=", hidden_dim);
TORCH_CHECK(
STD_TORCH_CHECK(mat_a.size(1) == mat_b.size(1),
"mat_a and mat_b must have the same hidden_dim");
STD_TORCH_CHECK(hidden_dim == DEFAULT_HIDDEN_DIM,
"Expected hidden_dim=", DEFAULT_HIDDEN_DIM,
", but got hidden_dim=", hidden_dim);
STD_TORCH_CHECK(
num_experts == DEFAULT_NUM_EXPERTS || num_experts == KIMI_K2_NUM_EXPERTS,
"Expected num_experts=", DEFAULT_NUM_EXPERTS,
" or num_experts=", KIMI_K2_NUM_EXPERTS,
", but got num_experts=", num_experts);
TORCH_CHECK(num_tokens >= 1 && num_tokens <= 16,
"currently num_tokens must be less than or equal to 16 for "
"router_gemm");
TORCH_CHECK(mat_a.dtype() == at::kBFloat16, "mat_a must be bf16");
TORCH_CHECK(mat_b.dtype() == at::kBFloat16, "mat_b must be bf16");
TORCH_CHECK(output.dtype() == at::kFloat || output.dtype() == at::kBFloat16,
"output must be float32 or bf16");
STD_TORCH_CHECK(num_tokens >= 1 && num_tokens <= 16,
"currently num_tokens must be less than or equal to 16 for "
"router_gemm");
STD_TORCH_CHECK(
mat_a.scalar_type() == torch::headeronly::ScalarType::BFloat16,
"mat_a must be bf16");
STD_TORCH_CHECK(
mat_b.scalar_type() == torch::headeronly::ScalarType::BFloat16,
"mat_b must be bf16");
STD_TORCH_CHECK(
output.scalar_type() == torch::headeronly::ScalarType::Float ||
output.scalar_type() == torch::headeronly::ScalarType::BFloat16,
"output must be float32 or bf16");
auto const sm = getSMVersion();
TORCH_CHECK(sm >= 90 && sm <= 103, "required SM_103 >= CUDA ARCH >= SM_90");
const int sm = getSMVersion();
STD_TORCH_CHECK(sm >= 90 && sm <= 103,
"required SM_103 >= CUDA ARCH >= SM_90");
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
const cudaStream_t stream = get_current_cuda_stream(mat_a.get_device_index());
if (output.dtype() == at::kFloat) {
if (output.scalar_type() == torch::headeronly::ScalarType::Float) {
if (num_experts == DEFAULT_NUM_EXPERTS) {
LoopUnroller<1, 16, DEFAULT_NUM_EXPERTS, DEFAULT_HIDDEN_DIM>::
unroll_float_output(
@@ -145,7 +163,7 @@ void dsv3_router_gemm(at::Tensor& output, // [num_tokens, num_experts]
reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()),
reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), stream);
}
} else if (output.dtype() == at::kBFloat16) {
} else if (output.scalar_type() == torch::headeronly::ScalarType::BFloat16) {
if (num_experts == DEFAULT_NUM_EXPERTS) {
LoopUnroller<1, 16, DEFAULT_NUM_EXPERTS, DEFAULT_HIDDEN_DIM>::
unroll_bf16_output(
@@ -164,6 +182,6 @@ void dsv3_router_gemm(at::Tensor& output, // [num_tokens, num_experts]
}
}
TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) {
m.impl("dsv3_router_gemm", &dsv3_router_gemm);
STABLE_TORCH_LIBRARY_IMPL(_moe_C, CUDA, m) {
m.impl("dsv3_router_gemm", TORCH_BOX(&dsv3_router_gemm));
}
@@ -18,14 +18,11 @@
* limitations under the License.
*/
#include <ATen/ATen.h>
#include <ATen/cuda/CUDAContext.h>
#include <cstdint>
#include <cuda_bf16.h>
#include <cuda_runtime.h>
#include "dsv3_router_gemm_utils.h"
// Custom FMA implementation using PTX assembly instructions
__device__ __forceinline__ void fma(float2& d, float2 const& a, float2 const& b,
float2 const& c) {
@@ -18,9 +18,14 @@
* limitations under the License.
*/
#include "moeTopKFuncs.cuh"
#include <c10/cuda/CUDAStream.h>
#include <torch/all.h>
#include <torch/csrc/stable/tensor.h>
#include <torch/headeronly/core/ScalarType.h>
#include "libtorch_stable/torch_utils.h"
#include <cmath>
#include <tuple>
#include <cuda_fp16.h>
#include <cuda_bf16.h>
#include <cuda/std/limits>
@@ -1001,38 +1006,40 @@ INSTANTIATE_NOAUX_TC(__nv_bfloat16, __nv_bfloat16, int32_t, SCORING_NONE);
} // end namespace moe
} // namespace vllm
std::tuple<torch::Tensor, torch::Tensor> grouped_topk(
torch::Tensor const& scores, int64_t n_group, int64_t topk_group,
std::tuple<torch::stable::Tensor, torch::stable::Tensor> grouped_topk(
torch::stable::Tensor const& scores, int64_t n_group, int64_t topk_group,
int64_t topk, bool renormalize, double routed_scaling_factor,
torch::Tensor const& bias, int64_t scoring_func = 0) {
auto data_type = scores.scalar_type();
auto bias_type = bias.scalar_type();
auto input_size = scores.sizes();
int64_t num_tokens = input_size[0];
int64_t num_experts = input_size[1];
TORCH_CHECK(input_size.size() == 2, "scores must be a 2D Tensor");
TORCH_CHECK(n_group > 0, "n_group must be positive");
TORCH_CHECK(topk > 0, "topk must be positive");
TORCH_CHECK(topk_group > 0, "topk_group must be positive");
TORCH_CHECK(topk_group <= n_group, "topk_group must be <= n_group");
TORCH_CHECK(num_experts % n_group == 0,
"num_experts should be divisible by n_group");
TORCH_CHECK(n_group <= 32,
"n_group should be smaller than or equal to 32 for now");
TORCH_CHECK(topk <= 32, "topk should be smaller than or equal to 32 for now");
TORCH_CHECK(topk <= topk_group * (num_experts / n_group),
"topk must be <= topk_group * (num_experts / n_group)");
TORCH_CHECK(scoring_func == vllm::moe::SCORING_NONE ||
scoring_func == vllm::moe::SCORING_SIGMOID,
"scoring_func must be SCORING_NONE (0) or SCORING_SIGMOID (1)");
torch::stable::Tensor const& bias, int64_t scoring_func = 0) {
const auto data_type = scores.scalar_type();
const auto bias_type = bias.scalar_type();
STD_TORCH_CHECK(scores.dim() == 2, "scores must be a 2D Tensor");
const int64_t num_tokens = scores.size(0);
const int64_t num_experts = scores.size(1);
STD_TORCH_CHECK(n_group > 0, "n_group must be positive");
STD_TORCH_CHECK(topk > 0, "topk must be positive");
STD_TORCH_CHECK(topk_group > 0, "topk_group must be positive");
STD_TORCH_CHECK(topk_group <= n_group, "topk_group must be <= n_group");
STD_TORCH_CHECK(num_experts % n_group == 0,
"num_experts should be divisible by n_group");
STD_TORCH_CHECK(n_group <= 32,
"n_group should be smaller than or equal to 32 for now");
STD_TORCH_CHECK(topk <= 32,
"topk should be smaller than or equal to 32 for now");
STD_TORCH_CHECK(topk <= topk_group * (num_experts / n_group),
"topk must be <= topk_group * (num_experts / n_group)");
STD_TORCH_CHECK(
scoring_func == vllm::moe::SCORING_NONE ||
scoring_func == vllm::moe::SCORING_SIGMOID,
"scoring_func must be SCORING_NONE (0) or SCORING_SIGMOID (1)");
// Always output float32 for topk_values (eliminates Python-side conversion)
torch::Tensor topk_values = torch::empty(
{num_tokens, topk}, torch::dtype(torch::kFloat32).device(torch::kCUDA));
torch::Tensor topk_indices = torch::empty(
{num_tokens, topk}, torch::dtype(torch::kInt32).device(torch::kCUDA));
auto topk_values = torch::stable::new_empty(
scores, {num_tokens, topk}, torch::headeronly::ScalarType::Float);
auto topk_indices = torch::stable::new_empty(
scores, {num_tokens, topk}, torch::headeronly::ScalarType::Int);
auto stream = c10::cuda::getCurrentCUDAStream(scores.get_device());
const cudaStream_t stream =
get_current_cuda_stream(scores.get_device_index());
auto const sf = static_cast<vllm::moe::ScoringFunc>(scoring_func);
#define LAUNCH_KERNEL_SF(T, BiasT, IdxT) \
@@ -1057,7 +1064,7 @@ std::tuple<torch::Tensor, torch::Tensor> grouped_topk(
routed_scaling_factor, false, stream); \
break; \
default: \
throw std::invalid_argument("Unsupported scoring_func"); \
STD_TORCH_CHECK(false, "Unsupported scoring_func"); \
break; \
} \
} while (0)
@@ -1065,17 +1072,18 @@ std::tuple<torch::Tensor, torch::Tensor> grouped_topk(
#define LAUNCH_KERNEL(T, IdxT) \
do { \
switch (bias_type) { \
case torch::kFloat16: \
case torch::headeronly::ScalarType::Half: \
LAUNCH_KERNEL_SF(T, half, IdxT); \
break; \
case torch::kFloat32: \
case torch::headeronly::ScalarType::Float: \
LAUNCH_KERNEL_SF(T, float, IdxT); \
break; \
case torch::kBFloat16: \
case torch::headeronly::ScalarType::BFloat16: \
LAUNCH_KERNEL_SF(T, __nv_bfloat16, IdxT); \
break; \
default: \
throw std::invalid_argument( \
STD_TORCH_CHECK( \
false, \
"Invalid bias dtype, only supports float16, float32, and " \
"bfloat16"); \
break; \
@@ -1083,22 +1091,22 @@ std::tuple<torch::Tensor, torch::Tensor> grouped_topk(
} while (0)
switch (data_type) {
case torch::kFloat16:
case torch::headeronly::ScalarType::Half:
// Handle Float16
LAUNCH_KERNEL(half, int32_t);
break;
case torch::kFloat32:
case torch::headeronly::ScalarType::Float:
// Handle Float32
LAUNCH_KERNEL(float, int32_t);
break;
case torch::kBFloat16:
case torch::headeronly::ScalarType::BFloat16:
// Handle BFloat16
LAUNCH_KERNEL(__nv_bfloat16, int32_t);
break;
default:
// Handle other data types
throw std::invalid_argument(
"Invalid dtype, only supports float16, float32, and bfloat16");
STD_TORCH_CHECK(
false, "Invalid dtype, only supports float16, float32, and bfloat16");
break;
}
#undef LAUNCH_KERNEL
@@ -302,7 +302,7 @@ def generate_new_kernels():
if not SUPPORT_FP8 and kernel_selector_str != FILE_HEAD_COMMENT:
kernel_selector_str += (
"else if (a_type == vllm::kFE4M3fn)\n"
" TORCH_CHECK(false, "
" STD_TORCH_CHECK(false, "
'"marlin kernel with fp8 activation is not built.");'
)
@@ -24,7 +24,15 @@
#endif
#include "kernel.h"
#include "core/registration.h"
#include <torch/csrc/stable/accelerator.h>
#include <torch/csrc/stable/library.h>
#include <torch/csrc/stable/ops.h>
#include <torch/csrc/stable/tensor.h>
#include <torch/headeronly/core/ScalarType.h>
#include <torch/headeronly/util/Exception.h>
#include "libtorch_stable/torch_utils.h"
#define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \
static_assert(std::is_same<scalar_t, half>::value || \
@@ -350,18 +358,18 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
bool m_block_size_8 = moe_block_size == 8;
bool is_a_8bit = a_type.size_bits() == 8;
TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m,
", ", prob_n, ", ", prob_k, "]");
STD_TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [",
prob_m, ", ", prob_n, ", ", prob_k, "]");
int group_blocks = 0;
if (has_act_order) {
if (is_k_full) {
TORCH_CHECK(group_size != -1);
STD_TORCH_CHECK(group_size != -1);
group_blocks = group_size / 16;
TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k,
" is not divisible by group_blocks = ", group_blocks);
STD_TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k,
" is not divisible by group_blocks = ", group_blocks);
} else {
TORCH_CHECK(group_size == 0);
STD_TORCH_CHECK(group_size == 0);
group_blocks = 0;
}
} else {
@@ -369,8 +377,8 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
group_blocks = -1;
} else {
group_blocks = group_size / 16;
TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k,
" is not divisible by group_blocks = ", group_blocks);
STD_TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k,
" is not divisible by group_blocks = ", group_blocks);
}
}
@@ -407,7 +415,7 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
else if (moe_block_size == 64)
kernel = permute_cols_kernel<64>;
else
TORCH_CHECK(false, "unsupported moe_block_size ", moe_block_size);
STD_TORCH_CHECK(false, "unsupported moe_block_size ", moe_block_size);
// avoid ">>>" being formatted to "> > >"
// clang-format off
@@ -428,25 +436,25 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
int max_shared_mem = 0;
cudaDeviceGetAttribute(&max_shared_mem,
cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
TORCH_CHECK(max_shared_mem > 0);
STD_TORCH_CHECK(max_shared_mem > 0);
int major_capability, minor_capability;
cudaDeviceGetAttribute(&major_capability, cudaDevAttrComputeCapabilityMajor,
dev);
cudaDeviceGetAttribute(&minor_capability, cudaDevAttrComputeCapabilityMinor,
dev);
TORCH_CHECK(major_capability * 10 + minor_capability >= 75,
"marlin kernel only support Turing or newer GPUs.");
STD_TORCH_CHECK(major_capability * 10 + minor_capability >= 75,
"marlin kernel only support Turing or newer GPUs.");
int stages = 4;
if (major_capability == 7 && minor_capability == 5) {
stages = 2;
TORCH_CHECK(a_type == vllm::kFloat16 || a_type == vllm::kS8,
"Turing only support FP16 or INT8 activation.");
STD_TORCH_CHECK(a_type == vllm::kFloat16 || a_type == vllm::kS8,
"Turing only support FP16 or INT8 activation.");
}
if (a_type == vllm::kFE4M3fn) {
TORCH_CHECK(major_capability * 10 + minor_capability >= 89,
"FP8 only support Ada Lovelace or newer GPUs.");
TORCH_CHECK(
STD_TORCH_CHECK(major_capability * 10 + minor_capability >= 89,
"FP8 only support Ada Lovelace or newer GPUs.");
STD_TORCH_CHECK(
major_capability * 10 + minor_capability == 89 ||
major_capability == 12,
"Marlin W4A8-FP8 only support SM89 or SM12x device (It is slower than "
@@ -460,10 +468,10 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
thread_tfg = thread_config_t{thread_k, thread_n, thread_k * thread_n / 64};
if (blocks_per_sm == -1) blocks_per_sm = 1;
exec_cfg = exec_config_t{blocks_per_sm, thread_tfg};
TORCH_CHECK(prob_n % thread_n == 0, "prob_n = ", prob_n,
" is not divisible by thread_n = ", thread_n);
TORCH_CHECK(prob_k % thread_k == 0, "prob_k = ", prob_k,
" is not divisible by thread_k = ", thread_k);
STD_TORCH_CHECK(prob_n % thread_n == 0, "prob_n = ", prob_n,
" is not divisible by thread_n = ", thread_n);
STD_TORCH_CHECK(prob_k % thread_k == 0, "prob_k = ", prob_k,
" is not divisible by thread_k = ", thread_k);
} else {
// Auto config
exec_cfg = determine_exec_config(
@@ -484,19 +492,19 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
int thread_k_blocks = thread_k / 16;
int thread_n_blocks = thread_n / 16;
TORCH_CHECK(is_valid_config(thread_tfg, m_block_size_8, thread_m_blocks,
prob_m, prob_n, prob_k, num_bits, group_size,
has_act_order, is_k_full, has_zp, is_zp_float,
is_a_8bit, stages, max_shared_mem),
"Invalid thread config: thread_m_blocks = ", thread_m_blocks,
", thread_k = ", thread_tfg.thread_k,
", thread_n = ", thread_tfg.thread_n,
", num_threads = ", thread_tfg.num_threads, " for MKN = [",
prob_m, ", ", prob_k, ", ", prob_n, "] and num_bits = ", num_bits,
", group_size = ", group_size,
", has_act_order = ", has_act_order, ", is_k_full = ", is_k_full,
", has_zp = ", has_zp, ", is_zp_float = ", is_zp_float,
", max_shared_mem = ", max_shared_mem);
STD_TORCH_CHECK(
is_valid_config(thread_tfg, m_block_size_8, thread_m_blocks, prob_m,
prob_n, prob_k, num_bits, group_size, has_act_order,
is_k_full, has_zp, is_zp_float, is_a_8bit, stages,
max_shared_mem),
"Invalid thread config: thread_m_blocks = ", thread_m_blocks,
", thread_k = ", thread_tfg.thread_k,
", thread_n = ", thread_tfg.thread_n,
", num_threads = ", thread_tfg.num_threads, " for MKN = [", prob_m, ", ",
prob_k, ", ", prob_n, "] and num_bits = ", num_bits,
", group_size = ", group_size, ", has_act_order = ", has_act_order,
", is_k_full = ", is_k_full, ", has_zp = ", has_zp,
", is_zp_float = ", is_zp_float, ", max_shared_mem = ", max_shared_mem);
int sh_cache_size =
get_kernel_cache_size(thread_tfg, m_block_size_8, thread_m_blocks, prob_m,
@@ -509,13 +517,13 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
num_threads, is_zp_float, stages);
if (kernel == MarlinDefault) {
TORCH_CHECK(false, "Unsupported shapes: MNK = [", prob_m, ", ", prob_n,
", ", prob_k, "]", ", has_act_order = ", has_act_order,
", num_groups = ", num_groups, ", group_size = ", group_size,
", thread_m_blocks = ", thread_m_blocks,
", thread_n_blocks = ", thread_n_blocks,
", thread_k_blocks = ", thread_k_blocks,
", num_bits = ", num_bits);
STD_TORCH_CHECK(
false, "Unsupported shapes: MNK = [", prob_m, ", ", prob_n, ", ",
prob_k, "]", ", has_act_order = ", has_act_order,
", num_groups = ", num_groups, ", group_size = ", group_size,
", thread_m_blocks = ", thread_m_blocks,
", thread_n_blocks = ", thread_n_blocks,
", thread_k_blocks = ", thread_k_blocks, ", num_bits = ", num_bits);
}
cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize,
@@ -532,75 +540,81 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
} // namespace MARLIN_NAMESPACE_NAME
torch::Tensor moe_wna16_marlin_gemm(
torch::Tensor& a, std::optional<torch::Tensor> c_or_none,
torch::Tensor& b_q_weight,
std::optional<torch::Tensor> const& b_bias_or_none, torch::Tensor& b_scales,
std::optional<torch::Tensor> const& a_scales_or_none,
std::optional<torch::Tensor> const& global_scale_or_none,
std::optional<torch::Tensor> const& b_zeros_or_none,
std::optional<torch::Tensor> const& g_idx_or_none,
std::optional<torch::Tensor> const& perm_or_none, torch::Tensor& workspace,
torch::Tensor& sorted_token_ids, torch::Tensor& expert_ids,
torch::Tensor& num_tokens_past_padded, torch::Tensor& topk_weights,
int64_t moe_block_size, int64_t top_k, bool mul_topk_weights,
vllm::ScalarTypeId const& b_type_id, int64_t size_m, int64_t size_n,
int64_t size_k, bool is_k_full, bool use_atomic_add, bool use_fp32_reduce,
bool is_zp_float, int64_t thread_k, int64_t thread_n,
torch::stable::Tensor moe_wna16_marlin_gemm(
torch::stable::Tensor& a, std::optional<torch::stable::Tensor> c_or_none,
torch::stable::Tensor& b_q_weight,
std::optional<torch::stable::Tensor> const& b_bias_or_none,
torch::stable::Tensor& b_scales,
std::optional<torch::stable::Tensor> const& a_scales_or_none,
std::optional<torch::stable::Tensor> const& global_scale_or_none,
std::optional<torch::stable::Tensor> const& b_zeros_or_none,
std::optional<torch::stable::Tensor> const& g_idx_or_none,
std::optional<torch::stable::Tensor> const& perm_or_none,
torch::stable::Tensor& workspace, torch::stable::Tensor& sorted_token_ids,
torch::stable::Tensor& expert_ids,
torch::stable::Tensor& num_tokens_past_padded,
torch::stable::Tensor& topk_weights, int64_t moe_block_size, int64_t top_k,
bool mul_topk_weights, vllm::ScalarTypeId const& b_type_id, int64_t size_m,
int64_t size_n, int64_t size_k, bool is_k_full, bool use_atomic_add,
bool use_fp32_reduce, bool is_zp_float, int64_t thread_k, int64_t thread_n,
int64_t blocks_per_sm) {
vllm::ScalarTypeId a_type_id, c_type_id, s_type_id;
auto c_dtype = a.dtype();
if (a.scalar_type() == at::ScalarType::Half) {
auto c_dtype = a.scalar_type();
if (a.scalar_type() == torch::headeronly::ScalarType::Half) {
a_type_id = vllm::kFloat16.id();
c_type_id = vllm::kFloat16.id();
} else if (a.scalar_type() == at::ScalarType::BFloat16) {
} else if (a.scalar_type() == torch::headeronly::ScalarType::BFloat16) {
a_type_id = vllm::kBFloat16.id();
c_type_id = vllm::kBFloat16.id();
} else {
c_dtype = b_scales.dtype();
if (b_scales.scalar_type() == at::ScalarType::Half) {
c_dtype = b_scales.scalar_type();
if (b_scales.scalar_type() == torch::headeronly::ScalarType::Half) {
c_type_id = vllm::kFloat16.id();
} else if (b_scales.scalar_type() == at::ScalarType::BFloat16) {
} else if (b_scales.scalar_type() ==
torch::headeronly::ScalarType::BFloat16) {
c_type_id = vllm::kBFloat16.id();
} else {
c_type_id = vllm::kBFloat16.id();
TORCH_CHECK(c_or_none.has_value(), "c must be passed for W4A8-FP4");
torch::Tensor c = c_or_none.value();
c_dtype = c.dtype();
STD_TORCH_CHECK(c_or_none.has_value(), "c must be passed for W4A8-FP4");
torch::stable::Tensor c = c_or_none.value();
c_dtype = c.scalar_type();
if (c.scalar_type() == at::ScalarType::Half) {
if (c.scalar_type() == torch::headeronly::ScalarType::Half) {
c_type_id = vllm::kFloat16.id();
} else if (c.scalar_type() == at::ScalarType::BFloat16) {
} else if (c.scalar_type() == torch::headeronly::ScalarType::BFloat16) {
c_type_id = vllm::kBFloat16.id();
} else {
TORCH_CHECK(false, "unsupported c dtype");
STD_TORCH_CHECK(false, "unsupported c dtype");
}
}
if (a.scalar_type() == at::ScalarType::Float8_e4m3fn) {
if (a.scalar_type() == torch::headeronly::ScalarType::Float8_e4m3fn) {
a_type_id = vllm::kFE4M3fn.id();
} else if (a.scalar_type() == at::ScalarType::Char) {
} else if (a.scalar_type() == torch::headeronly::ScalarType::Char) {
a_type_id = vllm::kS8.id();
} else {
TORCH_CHECK(false, "unsupported `a` scalar_type");
STD_TORCH_CHECK(false, "unsupported `a` scalar_type");
}
}
s_type_id = c_type_id;
if (b_type_id == vllm::kFE2M1f.id()) {
if (b_scales.scalar_type() == at::ScalarType::Float8_e4m3fn) {
if (b_scales.scalar_type() ==
torch::headeronly::ScalarType::Float8_e4m3fn) {
s_type_id = vllm::kFE4M3fn.id();
} else if (b_scales.scalar_type() == at::ScalarType::Float8_e8m0fnu) {
} else if (b_scales.scalar_type() ==
torch::headeronly::ScalarType::Float8_e8m0fnu) {
s_type_id = vllm::kFE8M0fnu.id();
} else {
TORCH_CHECK(false,
"When b_type = float4_e2m1f, b_scale scalar type must be",
"float8_e4m3fn (for NVFP4) or float8_e8m0fnu (for MXFP4).");
STD_TORCH_CHECK(
false, "When b_type = float4_e2m1f, b_scale scalar type must be",
"float8_e4m3fn (for NVFP4) or float8_e8m0fnu (for MXFP4).");
}
} else if (b_type_id == vllm::kFE4M3fn.id() &&
b_scales.scalar_type() == at::ScalarType::Float8_e8m0fnu) {
b_scales.scalar_type() ==
torch::headeronly::ScalarType::Float8_e8m0fnu) {
s_type_id = vllm::kFE8M0fnu.id();
}
@@ -613,58 +627,60 @@ torch::Tensor moe_wna16_marlin_gemm(
int num_experts = b_q_weight.size(0);
if (moe_block_size != 8) {
TORCH_CHECK(moe_block_size % 16 == 0,
"unsupported moe_block_size=", moe_block_size);
TORCH_CHECK(moe_block_size >= 16 && moe_block_size <= 64,
"unsupported moe_block_size=", moe_block_size);
STD_TORCH_CHECK(moe_block_size % 16 == 0,
"unsupported moe_block_size=", moe_block_size);
STD_TORCH_CHECK(moe_block_size >= 16 && moe_block_size <= 64,
"unsupported moe_block_size=", moe_block_size);
}
// Verify A
TORCH_CHECK(a.size(0) == size_m, "Shape mismatch: a.size(0) = ", a.size(0),
", size_m = ", size_m);
TORCH_CHECK(a.size(1) == size_k, "Shape mismatch: a.size(1) = ", a.size(1),
", size_k = ", size_k);
STD_TORCH_CHECK(a.size(0) == size_m,
"Shape mismatch: a.size(0) = ", a.size(0),
", size_m = ", size_m);
STD_TORCH_CHECK(a.size(1) == size_k,
"Shape mismatch: a.size(1) = ", a.size(1),
", size_k = ", size_k);
// Verify B
TORCH_CHECK(
STD_TORCH_CHECK(
size_k % MARLIN_NAMESPACE_NAME::tile_size == 0, "size_k = ", size_k,
" is not divisible by tile_size = ", MARLIN_NAMESPACE_NAME::tile_size);
TORCH_CHECK((size_k / MARLIN_NAMESPACE_NAME::tile_size) == b_q_weight.size(1),
"Shape mismatch: b_q_weight.size(1) = ", b_q_weight.size(1),
", size_k = ", size_k,
", tile_size = ", MARLIN_NAMESPACE_NAME::tile_size);
TORCH_CHECK(
STD_TORCH_CHECK(
(size_k / MARLIN_NAMESPACE_NAME::tile_size) == b_q_weight.size(1),
"Shape mismatch: b_q_weight.size(1) = ", b_q_weight.size(1),
", size_k = ", size_k,
", tile_size = ", MARLIN_NAMESPACE_NAME::tile_size);
STD_TORCH_CHECK(
b_q_weight.size(2) % MARLIN_NAMESPACE_NAME::tile_size == 0,
"b_q_weight.size(2) = ", b_q_weight.size(2),
" is not divisible by tile_size = ", MARLIN_NAMESPACE_NAME::tile_size);
int actual_size_n =
(b_q_weight.size(2) / MARLIN_NAMESPACE_NAME::tile_size) * pack_factor;
TORCH_CHECK(size_n == actual_size_n, "size_n = ", size_n,
", actual_size_n = ", actual_size_n);
STD_TORCH_CHECK(size_n == actual_size_n, "size_n = ", size_n,
", actual_size_n = ", actual_size_n);
// Verify device and strides
TORCH_CHECK(a.device().is_cuda(), "A is not on GPU");
TORCH_CHECK(a.is_contiguous(), "A is not contiguous");
STD_TORCH_CHECK(a.device().is_cuda(), "A is not on GPU");
STD_TORCH_CHECK(a.is_contiguous(), "A is not contiguous");
TORCH_CHECK(b_q_weight.device().is_cuda(), "b_q_weight is not on GPU");
TORCH_CHECK(b_q_weight.is_contiguous(), "b_q_weight is not contiguous");
STD_TORCH_CHECK(b_q_weight.device().is_cuda(), "b_q_weight is not on GPU");
STD_TORCH_CHECK(b_q_weight.is_contiguous(), "b_q_weight is not contiguous");
TORCH_CHECK(b_scales.device().is_cuda(), "b_scales is not on GPU");
TORCH_CHECK(b_scales.is_contiguous(), "b_scales is not contiguous");
STD_TORCH_CHECK(b_scales.device().is_cuda(), "b_scales is not on GPU");
STD_TORCH_CHECK(b_scales.is_contiguous(), "b_scales is not contiguous");
torch::Tensor a_scales;
auto options = torch::TensorOptions().dtype(c_dtype).device(a.device());
auto options_fp32 =
torch::TensorOptions().dtype(at::kFloat).device(a.device());
torch::stable::Tensor a_scales;
constexpr auto kFloat = torch::headeronly::ScalarType::Float;
if (a_scales_or_none.has_value()) {
a_scales = a_scales_or_none.value();
TORCH_CHECK(a_type.size_bits() == 8,
"a_scales can only be used for 8bit activation.");
STD_TORCH_CHECK(a_type.size_bits() == 8,
"a_scales can only be used for 8bit activation.");
} else {
a_scales = torch::empty({0}, options_fp32);
TORCH_CHECK(a_type.size_bits() != 8,
"the a_scales parameter must be passed for 8bit activation.");
a_scales = torch::stable::new_empty(a, {0}, kFloat);
STD_TORCH_CHECK(
a_type.size_bits() != 8,
"the a_scales parameter must be passed for 8bit activation.");
}
// sms: number of SMs to use for the kernel
@@ -672,82 +688,84 @@ torch::Tensor moe_wna16_marlin_gemm(
cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, a.get_device());
// Alloc buffers
const at::cuda::OptionalCUDAGuard device_guard(device_of(a));
torch::Tensor c;
torch::stable::accelerator::DeviceGuard device_guard(a.get_device_index());
torch::stable::Tensor c;
if (c_or_none.has_value()) {
c = c_or_none.value();
TORCH_CHECK(c.device().is_cuda(), "c is not on GPU");
TORCH_CHECK(c.is_contiguous(), "c is not contiguous");
TORCH_CHECK(c.size(0) == size_m * top_k,
"Shape mismatch: c.size(0) = ", c.size(0),
", size_m * topk = ", size_m * top_k);
TORCH_CHECK(c.size(1) == size_n, "Shape mismatch: c.size(1) = ", c.size(1),
", size_n = ", size_n);
STD_TORCH_CHECK(c.device().is_cuda(), "c is not on GPU");
STD_TORCH_CHECK(c.is_contiguous(), "c is not contiguous");
STD_TORCH_CHECK(c.size(0) == size_m * top_k,
"Shape mismatch: c.size(0) = ", c.size(0),
", size_m * topk = ", size_m * top_k);
STD_TORCH_CHECK(c.size(1) == size_n,
"Shape mismatch: c.size(1) = ", c.size(1),
", size_n = ", size_n);
} else {
c = torch::empty({size_m * top_k, size_n}, options);
c = torch::stable::new_empty(a, {size_m * top_k, size_n}, c_dtype);
}
// Alloc C tmp buffer that is going to be used for the global reduce
torch::Tensor c_tmp;
torch::stable::Tensor c_tmp;
if (use_fp32_reduce && !use_atomic_add) {
// max num of threadblocks is sms * 4
long max_c_tmp_size = min(
(long)size_n * sorted_token_ids.size(0),
(long)sms * 4 * moe_block_size * MARLIN_NAMESPACE_NAME::max_thread_n);
if (moe_block_size == 8) max_c_tmp_size *= 2;
c_tmp = torch::empty({max_c_tmp_size}, options_fp32);
c_tmp = torch::stable::new_empty(a, {max_c_tmp_size}, kFloat);
} else {
c_tmp = torch::empty({0}, options_fp32);
c_tmp = torch::stable::new_empty(a, {0}, kFloat);
}
// Detect groupsize and act_order
int num_groups = -1;
int group_size = -1;
int rank = b_scales.sizes().size();
TORCH_CHECK(rank == 3, "b_scales rank = ", rank, " is not 3");
TORCH_CHECK(b_scales.size(2) == size_n, "b_scales dim 2 = ", b_scales.size(2),
" is not size_n = ", size_n);
int rank = b_scales.dim();
STD_TORCH_CHECK(rank == 3, "b_scales rank = ", rank, " is not 3");
STD_TORCH_CHECK(b_scales.size(2) == size_n,
"b_scales dim 2 = ", b_scales.size(2),
" is not size_n = ", size_n);
num_groups = b_scales.size(1);
torch::Tensor g_idx, perm, a_tmp;
torch::stable::Tensor g_idx, perm, a_tmp;
if (g_idx_or_none.has_value() && perm_or_none.has_value()) {
g_idx = g_idx_or_none.value();
perm = perm_or_none.value();
TORCH_CHECK(g_idx.device().is_cuda(), "g_idx is not on GPU");
TORCH_CHECK(g_idx.is_contiguous(), "g_idx is not contiguous");
TORCH_CHECK(perm.device().is_cuda(), "perm is not on GPU");
TORCH_CHECK(perm.is_contiguous(), "perm is not contiguous");
STD_TORCH_CHECK(g_idx.device().is_cuda(), "g_idx is not on GPU");
STD_TORCH_CHECK(g_idx.is_contiguous(), "g_idx is not contiguous");
STD_TORCH_CHECK(perm.device().is_cuda(), "perm is not on GPU");
STD_TORCH_CHECK(perm.is_contiguous(), "perm is not contiguous");
// Verify g_idx and perm
TORCH_CHECK((g_idx.size(-1) == 0 && perm.size(-1) == 0) ||
(g_idx.size(-1) == size_k && perm.size(-1) == size_k),
"Unexpected g_idx.size(-1) = ", g_idx.size(-1),
" and perm.size(-1) = ", perm.size(-1),
", where size_k = ", size_k);
STD_TORCH_CHECK((g_idx.size(-1) == 0 && perm.size(-1) == 0) ||
(g_idx.size(-1) == size_k && perm.size(-1) == size_k),
"Unexpected g_idx.size(-1) = ", g_idx.size(-1),
" and perm.size(-1) = ", perm.size(-1),
", where size_k = ", size_k);
} else {
g_idx = torch::empty({0}, options);
perm = torch::empty({0}, options);
a_tmp = torch::empty({0}, options);
g_idx = torch::stable::new_empty(a, {0}, c_dtype);
perm = torch::stable::new_empty(a, {0}, c_dtype);
a_tmp = torch::stable::new_empty(a, {0}, c_dtype);
}
bool has_act_order = g_idx.size(-1) > 0 && perm.size(-1) > 0;
if (has_act_order) {
a_tmp = torch::empty({size_m * top_k, size_k}, options);
a_tmp = torch::stable::new_empty(a, {size_m * top_k, size_k}, c_dtype);
if (is_k_full) {
TORCH_CHECK(num_groups > 1, "For act_order, num_groups must be > 1");
TORCH_CHECK(size_k % num_groups == 0, "size_k = ", size_k,
", is not divisible by num_groups = ", num_groups);
STD_TORCH_CHECK(num_groups > 1, "For act_order, num_groups must be > 1");
STD_TORCH_CHECK(size_k % num_groups == 0, "size_k = ", size_k,
", is not divisible by num_groups = ", num_groups);
group_size = size_k / num_groups;
} else {
group_size = 0;
}
} else {
a_tmp = torch::empty({0}, options);
a_tmp = torch::stable::new_empty(a, {0}, c_dtype);
if (num_groups > 1) {
TORCH_CHECK(
STD_TORCH_CHECK(
size_k % num_groups == 0, "size_k = ", size_k,
", is not divisible by b_scales.size(1) = ", b_scales.size(1));
group_size = size_k / num_groups;
@@ -756,119 +774,125 @@ torch::Tensor moe_wna16_marlin_gemm(
}
}
torch::Tensor global_scale;
torch::stable::Tensor global_scale;
if (global_scale_or_none.has_value()) {
global_scale = global_scale_or_none.value();
TORCH_CHECK(b_type == vllm::kFE2M1f && s_type == vllm::kFE4M3fn,
"global_scale can only be used for nvfp4 format.");
STD_TORCH_CHECK(b_type == vllm::kFE2M1f && s_type == vllm::kFE4M3fn,
"global_scale can only be used for nvfp4 format.");
} else {
global_scale = torch::empty({0}, options_fp32);
TORCH_CHECK(!(b_type == vllm::kFE2M1f && s_type == vllm::kFE4M3fn),
"the global_scale parameter must be passed for nvfp4 format.");
global_scale = torch::stable::new_empty(a, {0}, kFloat);
STD_TORCH_CHECK(
!(b_type == vllm::kFE2M1f && s_type == vllm::kFE4M3fn),
"the global_scale parameter must be passed for nvfp4 format.");
}
bool has_bias = b_bias_or_none.has_value();
torch::Tensor b_bias;
torch::stable::Tensor b_bias;
if (has_bias) {
b_bias = b_bias_or_none.value();
TORCH_CHECK(b_bias.device().is_cuda(), "b_bias is not on GPU");
TORCH_CHECK(b_bias.is_contiguous(), "b_bias is not contiguous");
TORCH_CHECK(b_bias.size(1) == size_n, "b_bias.size(1) != size_n");
TORCH_CHECK(b_bias.stride(1) == 1, "b_bias.stride(1) != 1");
STD_TORCH_CHECK(b_bias.device().is_cuda(), "b_bias is not on GPU");
STD_TORCH_CHECK(b_bias.is_contiguous(), "b_bias is not contiguous");
STD_TORCH_CHECK(b_bias.size(1) == size_n, "b_bias.size(1) != size_n");
STD_TORCH_CHECK(b_bias.stride(1) == 1, "b_bias.stride(1) != 1");
} else {
b_bias = torch::empty({0}, options);
b_bias = torch::stable::new_empty(a, {0}, c_dtype);
}
torch::Tensor b_zeros;
torch::stable::Tensor b_zeros;
if (b_zeros_or_none.has_value()) {
b_zeros = b_zeros_or_none.value();
TORCH_CHECK(b_zeros.device().is_cuda(), "b_zeros is not on GPU");
TORCH_CHECK(b_zeros.is_contiguous(), "b_zeros is not contiguous");
STD_TORCH_CHECK(b_zeros.device().is_cuda(), "b_zeros is not on GPU");
STD_TORCH_CHECK(b_zeros.is_contiguous(), "b_zeros is not contiguous");
} else {
b_zeros = torch::empty({0}, options);
b_zeros = torch::stable::new_empty(a, {0}, c_dtype);
}
bool has_zp = b_zeros.size(-1) > 0;
if (has_zp) {
TORCH_CHECK(
STD_TORCH_CHECK(
b_type == vllm::kU4 || b_type == vllm::kU8,
"b_type must be u4 or u8 when has_zp = True. Got = ", b_type.str());
} else {
TORCH_CHECK(b_type == vllm::kU4B8 || b_type == vllm::kU8B128 ||
b_type == vllm::kS4 || b_type == vllm::kS8 ||
b_type == vllm::kFE4M3fn || b_type == vllm::kFE2M1f,
"b_type must be uint4b8, uint8b128, int4, int8, "
"float8_e4m3fn or float4_e2m1f when has_zp = False. Got = ",
b_type.str());
STD_TORCH_CHECK(b_type == vllm::kU4B8 || b_type == vllm::kU8B128 ||
b_type == vllm::kS4 || b_type == vllm::kS8 ||
b_type == vllm::kFE4M3fn || b_type == vllm::kFE2M1f,
"b_type must be uint4b8, uint8b128, int4, int8, "
"float8_e4m3fn or float4_e2m1f when has_zp = False. Got = ",
b_type.str());
}
if (has_zp && is_zp_float) {
TORCH_CHECK(a.scalar_type() == at::ScalarType::Half,
"Computation type must be float16 (half) when using float zero "
"points.");
STD_TORCH_CHECK(
a.scalar_type() == torch::headeronly::ScalarType::Half,
"Computation type must be float16 (half) when using float zero "
"points.");
}
// Verify b_zeros
if (has_zp) {
int rank = b_zeros.sizes().size();
TORCH_CHECK(rank == 3, "b_zeros rank = ", rank, " is not 3");
int rank = b_zeros.dim();
STD_TORCH_CHECK(rank == 3, "b_zeros rank = ", rank, " is not 3");
if (is_zp_float) {
TORCH_CHECK(b_zeros.size(2) == size_n,
"b_zeros dim 2 = ", b_zeros.size(2),
" is not size_n = ", size_n);
TORCH_CHECK(num_groups == b_zeros.size(1),
"b_zeros dim 1 = ", b_zeros.size(1),
" is not num_groups = ", num_groups);
TORCH_CHECK(num_groups != -1, "num_groups must be != -1");
STD_TORCH_CHECK(b_zeros.size(2) == size_n,
"b_zeros dim 2 = ", b_zeros.size(2),
" is not size_n = ", size_n);
STD_TORCH_CHECK(num_groups == b_zeros.size(1),
"b_zeros dim 1 = ", b_zeros.size(1),
" is not num_groups = ", num_groups);
STD_TORCH_CHECK(num_groups != -1, "num_groups must be != -1");
} else {
TORCH_CHECK(b_zeros.size(1) == num_groups,
"b_zeros dim 1 = ", b_zeros.size(1),
" is not num_groups = ", num_groups);
TORCH_CHECK(b_zeros.size(2) == size_n / pack_factor,
"b_zeros dim 2 = ", b_zeros.size(2),
" is not size_n / pack_factor = ", size_n / pack_factor);
STD_TORCH_CHECK(b_zeros.size(1) == num_groups,
"b_zeros dim 1 = ", b_zeros.size(1),
" is not num_groups = ", num_groups);
STD_TORCH_CHECK(b_zeros.size(2) == size_n / pack_factor,
"b_zeros dim 2 = ", b_zeros.size(2),
" is not size_n / pack_factor = ", size_n / pack_factor);
}
}
// Verify workspace size
TORCH_CHECK(size_n % MARLIN_NAMESPACE_NAME::min_thread_n == 0,
"size_n = ", size_n, ", is not divisible by min_thread_n = ",
MARLIN_NAMESPACE_NAME::min_thread_n);
STD_TORCH_CHECK(size_n % MARLIN_NAMESPACE_NAME::min_thread_n == 0,
"size_n = ", size_n, ", is not divisible by min_thread_n = ",
MARLIN_NAMESPACE_NAME::min_thread_n);
int max_n_tiles = size_n / MARLIN_NAMESPACE_NAME::min_thread_n;
int min_workspace_size = min(
max_n_tiles * (int)(sorted_token_ids.size(0) / moe_block_size), sms * 4);
TORCH_CHECK(workspace.numel() >= min_workspace_size,
"workspace.numel = ", workspace.numel(),
" is below min_workspace_size = ", min_workspace_size);
STD_TORCH_CHECK(workspace.numel() >= min_workspace_size,
"workspace.numel = ", workspace.numel(),
" is below min_workspace_size = ", min_workspace_size);
int dev = a.get_device();
TORCH_CHECK(a_scales.scalar_type() == at::ScalarType::Float,
"scalar type of a_scales must be float");
TORCH_CHECK(global_scale.scalar_type() == at::ScalarType::Float,
"scalar type of global_scale must be float");
STD_TORCH_CHECK(
a_scales.scalar_type() == torch::headeronly::ScalarType::Float,
"scalar type of a_scales must be float");
STD_TORCH_CHECK(
global_scale.scalar_type() == torch::headeronly::ScalarType::Float,
"scalar type of global_scale must be float");
if (a_type.size_bits() == 16) {
TORCH_CHECK(
STD_TORCH_CHECK(
a.scalar_type() == c.scalar_type(),
"scalar type of a must be the same with c for 16 bit activation");
}
MARLIN_NAMESPACE_NAME::marlin_mm(
a.data_ptr(), b_q_weight.data_ptr(), c.data_ptr(), c_tmp.data_ptr(),
b_bias.data_ptr(), a_scales.data_ptr(), b_scales.data_ptr(),
global_scale.data_ptr(), b_zeros.data_ptr(), g_idx.data_ptr(),
perm.data_ptr(), a_tmp.data_ptr(), sorted_token_ids.data_ptr(),
expert_ids.data_ptr(), num_tokens_past_padded.data_ptr(),
topk_weights.data_ptr(), moe_block_size, num_experts, top_k,
mul_topk_weights, size_m, size_n, size_k, workspace.data_ptr(), a_type,
b_type, c_type, s_type, has_bias, has_act_order, is_k_full, has_zp,
num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
a.const_data_ptr(), b_q_weight.const_data_ptr(), c.mutable_data_ptr(),
c_tmp.mutable_data_ptr(), b_bias.mutable_data_ptr(),
a_scales.mutable_data_ptr(), b_scales.mutable_data_ptr(),
global_scale.mutable_data_ptr(), b_zeros.mutable_data_ptr(),
g_idx.mutable_data_ptr(), perm.mutable_data_ptr(),
a_tmp.mutable_data_ptr(), sorted_token_ids.mutable_data_ptr(),
expert_ids.mutable_data_ptr(), num_tokens_past_padded.mutable_data_ptr(),
topk_weights.mutable_data_ptr(), moe_block_size, num_experts, top_k,
mul_topk_weights, size_m, size_n, size_k, workspace.mutable_data_ptr(),
a_type, b_type, c_type, s_type, has_bias, has_act_order, is_k_full,
has_zp, num_groups, group_size, dev, get_current_cuda_stream(dev),
thread_k, thread_n, sms, blocks_per_sm, use_atomic_add, use_fp32_reduce,
is_zp_float);
return c;
}
TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) {
m.impl("moe_wna16_marlin_gemm", &moe_wna16_marlin_gemm);
STABLE_TORCH_LIBRARY_IMPL(_moe_C, CUDA, m) {
m.impl("moe_wna16_marlin_gemm", TORCH_BOX(&moe_wna16_marlin_gemm));
}
@@ -1,14 +1,17 @@
#include <torch/all.h>
#include <ATen/cuda/CUDAContext.h>
#include <c10/cuda/CUDAGuard.h>
#include <array>
#include <cub/cub.cuh>
#include <ATen/ATen.h>
#include <ATen/cuda/Atomic.cuh>
#include <cuda_runtime.h>
#include <torch/csrc/stable/macros.h>
#include <torch/csrc/stable/accelerator.h>
#include <torch/csrc/stable/ops.h>
#include <torch/csrc/stable/tensor.h>
#include <torch/headeronly/core/ScalarType.h>
#include "../cuda_compat.h"
#include "../dispatch_utils.h"
#include "../../cuda_compat.h"
#include "core/math.hpp"
#include "libtorch_stable/dispatch_utils.h"
#include "libtorch_stable/torch_utils.h"
#define CEILDIV(x, y) (((x) + (y) - 1) / (y))
@@ -492,12 +495,13 @@ __global__ void moe_lora_align_block_size_small_batch_expert_kernel(
// taken from
// https://github.com/sgl-project/sglang/blob/8b5f83ed3b7d2a49ad5c5cd5aa61c5d502f47dbc
void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
int64_t block_size, torch::Tensor sorted_token_ids,
torch::Tensor experts_ids,
torch::Tensor num_tokens_post_pad,
std::optional<torch::Tensor> maybe_expert_map) {
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
void moe_align_block_size(
torch::stable::Tensor topk_ids, int64_t num_experts, int64_t block_size,
torch::stable::Tensor sorted_token_ids, torch::stable::Tensor experts_ids,
torch::stable::Tensor num_tokens_post_pad,
std::optional<torch::stable::Tensor> maybe_expert_map) {
const cudaStream_t stream =
get_current_cuda_stream(topk_ids.get_device_index());
int64_t padded_num_experts =
((num_experts + WARP_SIZE - 1) / WARP_SIZE) * WARP_SIZE;
@@ -506,19 +510,18 @@ void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
threads = ((threads + WARP_SIZE - 1) / WARP_SIZE) * WARP_SIZE;
// BlockScan uses 1024 threads and assigns one thread per expert.
TORCH_CHECK(padded_num_experts < 1024,
"padded_num_experts must be less than 1024");
auto options_int =
torch::TensorOptions().dtype(torch::kInt).device(topk_ids.device());
STD_TORCH_CHECK(padded_num_experts < 1024,
"padded_num_experts must be less than 1024");
bool has_expert_map = maybe_expert_map.has_value();
torch::Tensor expert_map;
torch::stable::Tensor expert_map;
if (has_expert_map) {
expert_map = maybe_expert_map.value();
} else {
expert_map = torch::empty({0}, options_int);
expert_map = torch::stable::new_empty(topk_ids, {0},
torch::headeronly::ScalarType::Int);
}
VLLM_DISPATCH_INTEGRAL_AND_UNSIGNED_TYPES(
VLLM_STABLE_DISPATCH_INTEGRAL_AND_UNSIGNED_TYPES(
topk_ids.scalar_type(), "moe_align_block_size_kernel", [&] {
// calc needed amount of shared mem for `cumsum` tensors
bool small_batch_expert_mode =
@@ -538,16 +541,17 @@ void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
scalar_t, fill_threads>;
small_batch_expert_kernel<<<1, fill_threads + threads,
shared_mem_size, stream>>>(
topk_ids.data_ptr<scalar_t>(),
sorted_token_ids.data_ptr<int32_t>(),
experts_ids.data_ptr<int32_t>(),
num_tokens_post_pad.data_ptr<int32_t>(),
expert_map.data_ptr<int32_t>(), num_experts, block_size,
topk_ids.numel(), sorted_token_ids.size(0), topk_ids.size(1),
has_expert_map);
reinterpret_cast<const scalar_t*>(topk_ids.const_data_ptr()),
reinterpret_cast<int32_t*>(sorted_token_ids.mutable_data_ptr()),
reinterpret_cast<int32_t*>(experts_ids.mutable_data_ptr()),
reinterpret_cast<int32_t*>(
num_tokens_post_pad.mutable_data_ptr()),
reinterpret_cast<int32_t*>(expert_map.mutable_data_ptr()),
num_experts, block_size, topk_ids.numel(),
sorted_token_ids.size(0), topk_ids.size(1), has_expert_map);
} else {
torch::Tensor cumsum_buffer =
torch::empty({num_experts + 1}, options_int);
torch::stable::Tensor cumsum_buffer = torch::stable::new_empty(
topk_ids, {num_experts + 1}, torch::headeronly::ScalarType::Int);
auto align_kernel = vllm::moe::moe_align_block_size_kernel<scalar_t>;
size_t num_warps = CEILDIV(padded_num_experts, experts_per_warp);
@@ -558,14 +562,16 @@ void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
// blockIdx.x == 0: counting experts and aligning
// blockIdx.x == 1: filling sorted_token_ids
align_kernel<<<2, threads, shared_mem_size, stream>>>(
topk_ids.data_ptr<scalar_t>(),
sorted_token_ids.data_ptr<int32_t>(),
experts_ids.data_ptr<int32_t>(),
num_tokens_post_pad.data_ptr<int32_t>(),
expert_map.data_ptr<int32_t>(), num_experts, padded_num_experts,
experts_per_warp, block_size, topk_ids.numel(),
cumsum_buffer.data_ptr<int32_t>(), sorted_token_ids.size(0),
topk_ids.size(1), has_expert_map);
reinterpret_cast<const scalar_t*>(topk_ids.const_data_ptr()),
reinterpret_cast<int32_t*>(sorted_token_ids.mutable_data_ptr()),
reinterpret_cast<int32_t*>(experts_ids.mutable_data_ptr()),
reinterpret_cast<int32_t*>(
num_tokens_post_pad.mutable_data_ptr()),
reinterpret_cast<int32_t*>(expert_map.mutable_data_ptr()),
num_experts, padded_num_experts, experts_per_warp, block_size,
topk_ids.numel(),
reinterpret_cast<int32_t*>(cumsum_buffer.mutable_data_ptr()),
sorted_token_ids.size(0), topk_ids.size(1), has_expert_map);
const int block_threads = std::min(256, (int)threads);
const int num_blocks =
@@ -577,9 +583,10 @@ void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
auto sort_kernel =
vllm::moe::count_and_sort_expert_tokens_kernel<scalar_t>;
sort_kernel<<<gridDims, block_threads, 0, stream>>>(
topk_ids.data_ptr<scalar_t>(),
sorted_token_ids.data_ptr<int32_t>(),
cumsum_buffer.data_ptr<int32_t>(), expert_map.data_ptr<int32_t>(),
reinterpret_cast<const scalar_t*>(topk_ids.const_data_ptr()),
reinterpret_cast<int32_t*>(sorted_token_ids.mutable_data_ptr()),
reinterpret_cast<int32_t*>(cumsum_buffer.mutable_data_ptr()),
reinterpret_cast<int32_t*>(expert_map.mutable_data_ptr()),
topk_ids.numel(), num_experts, sorted_token_ids.size(0),
topk_ids.size(1), has_expert_map);
}
@@ -588,33 +595,36 @@ void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
void batched_moe_align_block_size(int64_t max_tokens_per_batch,
int64_t block_size,
torch::Tensor const& batch_num_tokens,
torch::Tensor sorted_ids,
torch::Tensor batch_ids,
torch::Tensor num_tokens_post_pad) {
const torch::stable::Tensor& batch_num_tokens,
torch::stable::Tensor sorted_ids,
torch::stable::Tensor batch_ids,
torch::stable::Tensor num_tokens_post_pad) {
namespace batched_kernel = vllm::moe::batched_moe_align_block_size;
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
const cudaStream_t stream =
get_current_cuda_stream(batch_num_tokens.get_device_index());
int32_t const B = batch_num_tokens.size(0);
int32_t const num_blocks_per_batch =
round_to_next_multiple_of(max_tokens_per_batch, block_size) / block_size;
int32_t const num_blocks = num_blocks_per_batch * B;
int64_t const sorted_ids_size = num_blocks * block_size;
TORCH_CHECK(sorted_ids.size(0) == sorted_ids_size);
TORCH_CHECK(batch_ids.size(0) == sorted_ids_size / block_size);
TORCH_CHECK(num_tokens_post_pad.size(0) == 1);
TORCH_CHECK(B <= batched_kernel::num_threads);
STD_TORCH_CHECK(sorted_ids.size(0) == sorted_ids_size);
STD_TORCH_CHECK(batch_ids.size(0) == sorted_ids_size / block_size);
STD_TORCH_CHECK(num_tokens_post_pad.size(0) == 1);
STD_TORCH_CHECK(B <= batched_kernel::num_threads);
batched_kernel::batched_moe_align_block_size_kernel<<<
batched_kernel::num_blocks, batched_kernel::num_threads, 0, stream>>>(
B, max_tokens_per_batch, block_size, batch_num_tokens.data_ptr<int32_t>(),
sorted_ids.data_ptr<int32_t>(), batch_ids.data_ptr<int32_t>(),
num_tokens_post_pad.data_ptr<int32_t>());
B, max_tokens_per_batch, block_size,
reinterpret_cast<const int32_t*>(batch_num_tokens.const_data_ptr()),
reinterpret_cast<int32_t*>(sorted_ids.mutable_data_ptr()),
reinterpret_cast<int32_t*>(batch_ids.mutable_data_ptr()),
reinterpret_cast<int32_t*>(num_tokens_post_pad.mutable_data_ptr()));
}
void moe_sum(torch::Tensor& input, // [num_tokens, topk, hidden_size]
torch::Tensor& output) // [num_tokens, hidden_size]
void moe_sum(torch::stable::Tensor& input, // [num_tokens, topk, hidden_size]
torch::stable::Tensor& output) // [num_tokens, hidden_size]
{
const int hidden_size = input.size(-1);
const auto num_tokens = output.numel() / hidden_size;
@@ -622,77 +632,86 @@ void moe_sum(torch::Tensor& input, // [num_tokens, topk, hidden_size]
dim3 grid(num_tokens);
dim3 block(std::min(hidden_size, 1024));
const at::cuda::OptionalCUDAGuard device_guard(device_of(output));
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
const torch::stable::accelerator::DeviceGuard device_guard(
output.get_device_index());
const cudaStream_t stream =
get_current_cuda_stream(output.get_device_index());
switch (topk) {
case 2:
VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "moe_sum_kernel", [&] {
vllm::moe::moe_sum_kernel<scalar_t, 2><<<grid, block, 0, stream>>>(
output.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(),
hidden_size);
});
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
input.scalar_type(), "moe_sum_kernel", [&] {
vllm::moe::moe_sum_kernel<scalar_t, 2><<<grid, block, 0, stream>>>(
reinterpret_cast<scalar_t*>(output.mutable_data_ptr()),
reinterpret_cast<const scalar_t*>(input.const_data_ptr()),
hidden_size);
});
break;
case 3:
VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "moe_sum_kernel", [&] {
vllm::moe::moe_sum_kernel<scalar_t, 3><<<grid, block, 0, stream>>>(
output.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(),
hidden_size);
});
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
input.scalar_type(), "moe_sum_kernel", [&] {
vllm::moe::moe_sum_kernel<scalar_t, 3><<<grid, block, 0, stream>>>(
reinterpret_cast<scalar_t*>(output.mutable_data_ptr()),
reinterpret_cast<const scalar_t*>(input.const_data_ptr()),
hidden_size);
});
break;
case 4:
VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "moe_sum_kernel", [&] {
vllm::moe::moe_sum_kernel<scalar_t, 4><<<grid, block, 0, stream>>>(
output.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(),
hidden_size);
});
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
input.scalar_type(), "moe_sum_kernel", [&] {
vllm::moe::moe_sum_kernel<scalar_t, 4><<<grid, block, 0, stream>>>(
reinterpret_cast<scalar_t*>(output.mutable_data_ptr()),
reinterpret_cast<const scalar_t*>(input.const_data_ptr()),
hidden_size);
});
break;
default:
at::sum_out(output, input, 1);
torch::stable::sum_out(output, input, std::array<int64_t, 1>{1});
break;
}
}
void moe_lora_align_block_size(
torch::Tensor topk_ids, torch::Tensor token_lora_mapping,
torch::stable::Tensor topk_ids, torch::stable::Tensor token_lora_mapping,
int64_t num_experts, int64_t block_size, int64_t max_loras,
int64_t max_num_tokens_padded, int64_t max_num_m_blocks,
torch::Tensor sorted_token_ids, torch::Tensor expert_ids,
torch::Tensor num_tokens_post_pad, torch::Tensor adapter_enabled,
torch::Tensor lora_ids, std::optional<torch::Tensor> maybe_expert_map) {
torch::stable::Tensor sorted_token_ids, torch::stable::Tensor expert_ids,
torch::stable::Tensor num_tokens_post_pad,
torch::stable::Tensor adapter_enabled, torch::stable::Tensor lora_ids,
std::optional<torch::stable::Tensor> maybe_expert_map) {
const int topk_num = topk_ids.size(1);
TORCH_CHECK(block_size > 0, "block_size should be greater than 0. ");
STD_TORCH_CHECK(block_size > 0, "block_size should be greater than 0. ");
int device_max_shared_mem;
auto dev = topk_ids.get_device();
int dev = topk_ids.get_device_index();
cudaDeviceGetAttribute(&device_max_shared_mem,
cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
const cudaStream_t stream = get_current_cuda_stream(dev);
int64_t padded_num_experts =
((num_experts + WARP_SIZE - 1) / WARP_SIZE) * WARP_SIZE;
// BlockScan uses 1024 threads and assigns one thread per expert.
TORCH_CHECK(padded_num_experts < 1024,
"padded_num_experts must be less than 1024");
STD_TORCH_CHECK(padded_num_experts < 1024,
"padded_num_experts must be less than 1024");
auto options_int =
torch::TensorOptions().dtype(torch::kInt).device(topk_ids.device());
torch::Tensor token_mask =
torch::empty({max_loras * topk_ids.size(0)}, options_int);
torch::stable::Tensor token_mask =
torch::stable::new_empty(topk_ids, {max_loras * topk_ids.size(0)},
torch::headeronly::ScalarType::Int);
bool has_expert_map = maybe_expert_map.has_value();
torch::Tensor expert_map;
torch::stable::Tensor expert_map;
if (has_expert_map) {
expert_map = maybe_expert_map.value();
} else {
expert_map = torch::empty({0}, options_int);
expert_map = torch::stable::new_empty(topk_ids, {0},
torch::headeronly::ScalarType::Int);
}
VLLM_DISPATCH_INTEGRAL_TYPES(
VLLM_STABLE_DISPATCH_INTEGRAL_TYPES(
topk_ids.scalar_type(), "moe_lora_align_sum_kernel", [&] {
bool small_batch_expert_mode =
(topk_ids.numel() < 1024) && (num_experts <= 64);
@@ -703,7 +722,7 @@ void moe_lora_align_block_size(
(num_thread + 1) * num_experts * sizeof(int32_t) +
(num_experts + 1) * sizeof(int32_t);
if (shared_mem > device_max_shared_mem) {
TORCH_CHECK(false, "Shared memory usage exceeds device limit.");
STD_TORCH_CHECK(false, "Shared memory usage exceeds device limit.");
}
// threadIdx.x >= fill_threads: counting experts and aligning
@@ -714,7 +733,7 @@ void moe_lora_align_block_size(
auto kernel =
vllm::moe::moe_lora_align_block_size_small_batch_expert_kernel<
scalar_t, fill_threads>;
AT_CUDA_CHECK(VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize(
STD_CUDA_CHECK(VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize(
(void*)kernel, shared_mem));
// Grid size is (max_loras + 1) because active_lora_ids has length
// max_loras + 1: sorted-unique values of token_lora_mapping, which
@@ -725,15 +744,21 @@ void moe_lora_align_block_size(
// MoE-LoRA kernels. This mirrors the fix made for the Triton
// _fused_moe_lora_kernel grid in vllm-project/vllm#32277.
kernel<<<max_loras + 1, blockDim, shared_mem, stream>>>(
topk_ids.data_ptr<scalar_t>(),
token_lora_mapping.data_ptr<int32_t>(), block_size,
expert_map.data_ptr<int32_t>(), num_experts, max_loras,
topk_ids.numel(), max_num_tokens_padded, max_num_m_blocks,
sorted_token_ids.data_ptr<int32_t>(),
expert_ids.data_ptr<int32_t>(), topk_num,
num_tokens_post_pad.data_ptr<int32_t>(),
adapter_enabled.data_ptr<int32_t>(), lora_ids.data_ptr<int32_t>(),
token_mask.data_ptr<int32_t>(), has_expert_map);
reinterpret_cast<scalar_t*>(topk_ids.mutable_data_ptr()),
reinterpret_cast<int32_t*>(token_lora_mapping.mutable_data_ptr()),
block_size,
reinterpret_cast<int32_t*>(expert_map.mutable_data_ptr()),
num_experts, max_loras, topk_ids.numel(), max_num_tokens_padded,
max_num_m_blocks,
reinterpret_cast<int32_t*>(sorted_token_ids.mutable_data_ptr()),
reinterpret_cast<int32_t*>(expert_ids.mutable_data_ptr()),
topk_num,
reinterpret_cast<int32_t*>(
num_tokens_post_pad.mutable_data_ptr()),
reinterpret_cast<int32_t*>(adapter_enabled.mutable_data_ptr()),
reinterpret_cast<int32_t*>(lora_ids.mutable_data_ptr()),
reinterpret_cast<int32_t*>(token_mask.mutable_data_ptr()),
has_expert_map);
} else {
int num_thread = 1024;
dim3 blockDim(num_thread);
@@ -742,8 +767,9 @@ void moe_lora_align_block_size(
size_t shared_mem_size = num_warps * WARP_SIZE * sizeof(int32_t);
// cumsum buffer
torch::Tensor cumsum =
torch::zeros({max_loras * (num_experts + 1)}, options_int);
torch::stable::Tensor cumsum = torch::stable::new_zeros(
topk_ids, {max_loras * (num_experts + 1)},
torch::headeronly::ScalarType::Int);
auto align_kernel =
vllm::moe::moe_lora_align_block_size_kernel<scalar_t>;
@@ -759,16 +785,23 @@ void moe_lora_align_block_size(
// blockIdx.x % 2 == 1: filling sorted_token_ids
align_kernel<<<(max_loras + 1) * 2, blockDim, shared_mem_size,
stream>>>(
topk_ids.data_ptr<scalar_t>(),
token_lora_mapping.data_ptr<int32_t>(), block_size,
expert_map.data_ptr<int32_t>(), num_experts, max_loras,
topk_ids.numel(), max_num_tokens_padded, max_num_m_blocks,
sorted_token_ids.data_ptr<int32_t>(),
expert_ids.data_ptr<int32_t>(), topk_num,
num_tokens_post_pad.data_ptr<int32_t>(),
adapter_enabled.data_ptr<int32_t>(), cumsum.data_ptr<int32_t>(),
WARP_SIZE, padded_num_experts, lora_ids.data_ptr<int32_t>(),
token_mask.data_ptr<int32_t>(), has_expert_map);
reinterpret_cast<scalar_t*>(topk_ids.mutable_data_ptr()),
reinterpret_cast<int32_t*>(token_lora_mapping.mutable_data_ptr()),
block_size,
reinterpret_cast<int32_t*>(expert_map.mutable_data_ptr()),
num_experts, max_loras, topk_ids.numel(), max_num_tokens_padded,
max_num_m_blocks,
reinterpret_cast<int32_t*>(sorted_token_ids.mutable_data_ptr()),
reinterpret_cast<int32_t*>(expert_ids.mutable_data_ptr()),
topk_num,
reinterpret_cast<int32_t*>(
num_tokens_post_pad.mutable_data_ptr()),
reinterpret_cast<int32_t*>(adapter_enabled.mutable_data_ptr()),
reinterpret_cast<int32_t*>(cumsum.mutable_data_ptr()), WARP_SIZE,
padded_num_experts,
reinterpret_cast<int32_t*>(lora_ids.mutable_data_ptr()),
reinterpret_cast<int32_t*>(token_mask.mutable_data_ptr()),
has_expert_map);
const int block_threads = std::min(256, (int)num_thread);
const int num_blocks =
@@ -785,12 +818,16 @@ void moe_lora_align_block_size(
vllm::moe::lora_count_and_sort_expert_tokens_kernel<scalar_t>;
sort_kernel<<<gridDims, block_threads, 0, stream>>>(
topk_ids.data_ptr<scalar_t>(),
sorted_token_ids.data_ptr<int32_t>(), cumsum.data_ptr<int32_t>(),
expert_map.data_ptr<int32_t>(), topk_ids.numel(), num_experts,
max_num_tokens_padded, topk_num, token_mask.data_ptr<int32_t>(),
max_loras, lora_ids.data_ptr<int32_t>(),
adapter_enabled.data_ptr<int32_t>(), has_expert_map);
reinterpret_cast<const scalar_t*>(topk_ids.const_data_ptr()),
reinterpret_cast<int32_t*>(sorted_token_ids.mutable_data_ptr()),
reinterpret_cast<int32_t*>(cumsum.mutable_data_ptr()),
reinterpret_cast<int32_t*>(expert_map.mutable_data_ptr()),
topk_ids.numel(), num_experts, max_num_tokens_padded, topk_num,
reinterpret_cast<int32_t*>(token_mask.mutable_data_ptr()),
max_loras,
reinterpret_cast<int32_t*>(lora_ids.mutable_data_ptr()),
reinterpret_cast<int32_t*>(adapter_enabled.mutable_data_ptr()),
has_expert_map);
}
});
}
+87
View File
@@ -0,0 +1,87 @@
#pragma once
#include <torch/csrc/stable/tensor.h>
#include <optional>
#include <tuple>
void topk_softmax(torch::stable::Tensor& topk_weights,
torch::stable::Tensor& topk_indices,
torch::stable::Tensor& token_expert_indices,
torch::stable::Tensor& gating_output, bool renormalize,
std::optional<torch::stable::Tensor> bias);
void topk_sigmoid(torch::stable::Tensor& topk_weights,
torch::stable::Tensor& topk_indices,
torch::stable::Tensor& token_expert_indices,
torch::stable::Tensor& gating_output, bool renormalize,
std::optional<torch::stable::Tensor> bias);
void topk_softplus_sqrt(
torch::stable::Tensor& topk_weights, torch::stable::Tensor& topk_indices,
torch::stable::Tensor& token_expert_indices,
torch::stable::Tensor& gating_output, bool renormalize,
double routed_scaling_factor,
const std::optional<torch::stable::Tensor>& correction_bias,
const std::optional<torch::stable::Tensor>& input_ids,
const std::optional<torch::stable::Tensor>& tid2eid);
void moe_sum(torch::stable::Tensor& input, torch::stable::Tensor& output);
void moe_align_block_size(
torch::stable::Tensor topk_ids, int64_t num_experts, int64_t block_size,
torch::stable::Tensor sorted_token_ids, torch::stable::Tensor experts_ids,
torch::stable::Tensor num_tokens_post_pad,
std::optional<torch::stable::Tensor> maybe_expert_map);
void batched_moe_align_block_size(
int64_t max_tokens_per_batch, int64_t block_size,
const torch::stable::Tensor& expert_num_tokens,
torch::stable::Tensor sorted_ids, torch::stable::Tensor expert_ids,
torch::stable::Tensor num_tokens_post_pad);
void moe_lora_align_block_size(
torch::stable::Tensor topk_ids, torch::stable::Tensor token_lora_mapping,
int64_t num_experts, int64_t block_size, int64_t max_loras,
int64_t max_num_tokens_padded, int64_t max_num_m_blocks,
torch::stable::Tensor sorted_token_ids, torch::stable::Tensor expert_ids,
torch::stable::Tensor num_tokens_post_pad,
torch::stable::Tensor adapter_enabled, torch::stable::Tensor lora_ids,
std::optional<torch::stable::Tensor> maybe_expert_map);
#ifndef USE_ROCM
torch::stable::Tensor moe_wna16_gemm(
torch::stable::Tensor input, torch::stable::Tensor output,
torch::stable::Tensor b_qweight, torch::stable::Tensor b_scales,
std::optional<torch::stable::Tensor> b_qzeros,
std::optional<torch::stable::Tensor> topk_weights,
torch::stable::Tensor sorted_token_ids, torch::stable::Tensor expert_ids,
torch::stable::Tensor num_tokens_post_pad, int64_t top_k,
int64_t BLOCK_SIZE_M, int64_t BLOCK_SIZE_N, int64_t BLOCK_SIZE_K,
int64_t bit);
std::tuple<torch::stable::Tensor, torch::stable::Tensor> grouped_topk(
const torch::stable::Tensor& scores, int64_t n_group, int64_t topk_group,
int64_t topk, bool renormalize, double routed_scaling_factor,
const torch::stable::Tensor& bias, int64_t scoring_func);
#endif
bool moe_permute_unpermute_supported();
int64_t moe_permute_sort_workspace_size(int64_t num_expanded_rows,
int64_t num_expert);
void shuffle_rows(const torch::stable::Tensor& input_tensor,
const torch::stable::Tensor& dst2src_map,
torch::stable::Tensor& output_tensor);
#ifndef USE_ROCM
// DeepSeek V3 optimized router GEMM kernel for SM90+
// Computes output = mat_a @ mat_b.T where:
// mat_a: [num_tokens, hidden_dim] in bf16
// mat_b: [num_experts, hidden_dim] in bf16
// output: [num_tokens, num_experts] in bf16 or fp32
// Supports num_tokens in [1, 16], num_experts in {256, 384}, hidden_dim = 7168
void dsv3_router_gemm(torch::stable::Tensor& output,
const torch::stable::Tensor& mat_a,
const torch::stable::Tensor& mat_b);
#endif
@@ -0,0 +1,319 @@
#include <cuda.h>
#include <cuda_runtime.h>
#include <torch/csrc/stable/accelerator.h>
#include <torch/csrc/stable/ops.h>
#include <torch/csrc/stable/tensor.h>
#include <torch/headeronly/core/ScalarType.h>
#include <torch/headeronly/util/Exception.h>
#include "core/registration.h"
#include "libtorch_stable/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.h"
#include "libtorch_stable/torch_utils.h"
#include <torch/csrc/stable/library.h>
// moe_permute kernels require at least CUDA 12.0
#if defined(CUDA_VERSION) && (CUDA_VERSION >= 12000)
namespace {
int64_t product_integers(torch::headeronly::IntHeaderOnlyArrayRef sizes) {
int64_t numel = 1;
for (int64_t s : sizes) {
numel *= s;
}
return numel;
}
torch::stable::Tensor maybe_allocate_tensor(
const std::optional<torch::stable::Tensor>& maybe_tensor,
torch::headeronly::IntHeaderOnlyArrayRef expected_sizes,
torch::headeronly::ScalarType dtype, torch::stable::Device device,
char const* name) {
auto expected_numel = product_integers(expected_sizes);
if (maybe_tensor.has_value()) {
auto tensor = maybe_tensor.value();
STD_TORCH_CHECK(tensor.device() == device, name,
" must be on the same device");
STD_TORCH_CHECK(tensor.scalar_type() == dtype, name,
" has incorrect dtype");
STD_TORCH_CHECK(tensor.is_contiguous(), name, " must be contiguous");
STD_TORCH_CHECK(tensor.numel() >= expected_numel, name,
" is too small for the requested shape");
auto flat_tensor = torch::stable::view(tensor, {tensor.numel()});
return torch::stable::view(
torch::stable::narrow(flat_tensor, 0, 0, expected_numel),
expected_sizes);
}
return torch::stable::empty(expected_sizes, dtype, std::nullopt, device);
}
} // namespace
int64_t moe_permute_sort_workspace_size(int64_t num_expanded_rows,
int64_t n_expert) {
return static_cast<int64_t>(
CubKeyValueSorter::getWorkspaceSize(num_expanded_rows, n_expert));
}
void moe_permute_impl(
const torch::stable::Tensor& input, // [n_token, hidden]
const torch::stable::Tensor& topk_ids, // [n_token, topk]
const torch::stable::Tensor& token_expert_indices, // [n_token, topk]
const std::optional<torch::stable::Tensor>& expert_map, // [n_expert]
int64_t n_expert, int64_t n_local_expert, int64_t topk,
torch::stable::Tensor& permuted_input, // [permuted_size, hidden]
torch::stable::Tensor& expert_first_token_offset, // [n_local_expert + 1]
torch::stable::Tensor& inv_permuted_idx, // [n_token, topk]
torch::stable::Tensor& permuted_idx, // [permute_size]
const std::optional<torch::stable::Tensor>& maybe_sort_workspace,
const std::optional<torch::stable::Tensor>& maybe_permuted_experts_id,
const std::optional<torch::stable::Tensor>& maybe_sorted_row_idx,
const std::optional<torch::stable::Tensor>& maybe_topk_ids_for_sort) {
STD_TORCH_CHECK(expert_first_token_offset.scalar_type() ==
torch::headeronly::ScalarType::Long,
"expert_first_token_offset must be int64");
STD_TORCH_CHECK(topk_ids.scalar_type() == torch::headeronly::ScalarType::Int,
"topk_ids must be int32");
STD_TORCH_CHECK(
token_expert_indices.scalar_type() == torch::headeronly::ScalarType::Int,
"token_expert_indices must be int32");
STD_TORCH_CHECK(
inv_permuted_idx.scalar_type() == torch::headeronly::ScalarType::Int,
"inv_permuted_idx must be int32");
STD_TORCH_CHECK(expert_first_token_offset.size(0) == n_local_expert + 1,
"expert_first_token_offset shape != n_local_expert+1");
STD_TORCH_CHECK(
inv_permuted_idx.sizes().equals(token_expert_indices.sizes()),
"token_expert_indices shape must be same as inv_permuted_idx");
auto device = input.device();
auto n_token = input.sizes()[0];
auto n_hidden = input.sizes()[1];
auto expanded_rows = n_token * topk;
auto stream = get_current_cuda_stream(input.get_device_index());
auto sorter_size = moe_permute_sort_workspace_size(expanded_rows, n_expert);
auto sort_workspace = maybe_allocate_tensor(
maybe_sort_workspace, {sorter_size}, torch::headeronly::ScalarType::Char,
device, "sort_workspace");
auto permuted_experts_id = maybe_allocate_tensor(
maybe_permuted_experts_id, topk_ids.sizes(),
torch::headeronly::ScalarType::Int, device, "permuted_experts_id");
auto sorted_row_idx = maybe_allocate_tensor(
maybe_sorted_row_idx, inv_permuted_idx.sizes(),
torch::headeronly::ScalarType::Int, device, "sorted_row_idx");
CubKeyValueSorter sorter{};
int64_t* valid_num_ptr = nullptr;
torch::stable::Tensor topk_ids_for_sort = topk_ids;
if (expert_map.has_value()) {
const int* expert_map_ptr = get_ptr<int>(expert_map.value());
valid_num_ptr =
get_ptr<int64_t>(expert_first_token_offset) + n_local_expert;
topk_ids_for_sort = maybe_allocate_tensor(
maybe_topk_ids_for_sort, topk_ids.sizes(),
torch::headeronly::ScalarType::Int, device, "topk_ids_for_sort");
torch::stable::copy_(topk_ids_for_sort, topk_ids);
preprocessTopkIdLauncher(get_ptr<int>(topk_ids_for_sort), n_token * topk,
expert_map_ptr, n_expert, stream);
}
sortAndScanExpert(
get_ptr<const int>(topk_ids_for_sort), get_ptr<int>(token_expert_indices),
get_ptr<int>(permuted_experts_id), get_ptr<int>(sorted_row_idx),
get_ptr<int64_t>(expert_first_token_offset), n_token, n_expert,
n_local_expert, topk, sorter, get_ptr<int>(sort_workspace), stream);
MOE_DISPATCH(input.scalar_type(), [&] {
expandInputRowsKernelLauncher<scalar_t>(
get_ptr<scalar_t>(input), get_ptr<scalar_t>(permuted_input),
get_ptr<int>(sorted_row_idx), get_ptr<int>(inv_permuted_idx),
get_ptr<int>(permuted_idx), get_ptr<int64_t>(expert_first_token_offset),
n_token, valid_num_ptr, n_hidden, topk, n_local_expert, stream);
});
}
void moe_permute(
const torch::stable::Tensor& input, // [n_token, hidden]
const torch::stable::Tensor& topk_ids, // [n_token, topk]
const torch::stable::Tensor& token_expert_indices, // [n_token, topk]
const std::optional<torch::stable::Tensor>& expert_map, // [n_expert]
int64_t n_expert, int64_t n_local_expert, int64_t topk,
torch::stable::Tensor& permuted_input, // [permuted_size, hidden]
torch::stable::Tensor& expert_first_token_offset, // [n_local_expert + 1]
torch::stable::Tensor& inv_permuted_idx, // [n_token, topk]
torch::stable::Tensor& permuted_idx) { // [permute_size]
moe_permute_impl(input, topk_ids, token_expert_indices, expert_map, n_expert,
n_local_expert, topk, permuted_input,
expert_first_token_offset, inv_permuted_idx, permuted_idx,
std::nullopt, std::nullopt, std::nullopt, std::nullopt);
}
void moe_permute_with_scratch(
const torch::stable::Tensor& input, const torch::stable::Tensor& topk_ids,
const torch::stable::Tensor& token_expert_indices,
const std::optional<torch::stable::Tensor>& expert_map, int64_t n_expert,
int64_t n_local_expert, int64_t topk, torch::stable::Tensor& permuted_input,
torch::stable::Tensor& expert_first_token_offset,
torch::stable::Tensor& inv_permuted_idx,
torch::stable::Tensor& permuted_idx, torch::stable::Tensor& sort_workspace,
torch::stable::Tensor& permuted_experts_id,
torch::stable::Tensor& sorted_row_idx,
torch::stable::Tensor& topk_ids_for_sort) {
moe_permute_impl(input, topk_ids, token_expert_indices, expert_map, n_expert,
n_local_expert, topk, permuted_input,
expert_first_token_offset, inv_permuted_idx, permuted_idx,
sort_workspace, permuted_experts_id, sorted_row_idx,
topk_ids_for_sort);
}
void moe_unpermute(
const torch::stable::Tensor&
permuted_hidden_states, // [n_token * topk, hidden]
const torch::stable::Tensor& topk_weights, // [n_token, topk]
const torch::stable::Tensor& inv_permuted_idx, // [n_token, topk]
const std::optional<torch::stable::Tensor>&
expert_first_token_offset, // [n_local_expert+1]
int64_t topk,
torch::stable::Tensor& hidden_states) { // [n_token, hidden]
STD_TORCH_CHECK(
permuted_hidden_states.scalar_type() == hidden_states.scalar_type(),
"permuted_hidden_states dtype must be same as hidden_states");
auto n_token = hidden_states.size(0);
auto n_hidden = hidden_states.size(1);
auto stream = get_current_cuda_stream(hidden_states.get_device_index());
int64_t const* valid_ptr = nullptr;
if (expert_first_token_offset.has_value()) {
int n_local_expert = expert_first_token_offset.value().size(0) - 1;
valid_ptr =
get_ptr<int64_t>(expert_first_token_offset.value()) + n_local_expert;
}
MOE_DISPATCH(hidden_states.scalar_type(), [&] {
finalizeMoeRoutingKernelLauncher<scalar_t, scalar_t>(
get_ptr<scalar_t>(permuted_hidden_states),
get_ptr<scalar_t>(hidden_states), get_ptr<float>(topk_weights),
get_ptr<int>(inv_permuted_idx), n_token, n_hidden, topk, valid_ptr,
stream);
});
}
template <typename T>
__global__ void shuffleInputRowsKernel(const T* input,
const int32_t* dst2src_map, T* output,
int64_t num_src_rows,
int64_t num_dst_rows, int64_t num_cols) {
int64_t dest_row_idx = blockIdx.x;
int64_t const source_row_idx = dst2src_map[dest_row_idx];
if (blockIdx.x < num_dst_rows) {
// Load 128-bits per thread
constexpr int64_t ELEM_PER_THREAD = 128 / sizeof(T) / 8;
using DataElem = cutlass::Array<T, ELEM_PER_THREAD>;
// Duplicate and permute rows
auto const* source_row_ptr =
reinterpret_cast<DataElem const*>(input + source_row_idx * num_cols);
auto* dest_row_ptr =
reinterpret_cast<DataElem*>(output + dest_row_idx * num_cols);
int64_t const start_offset = threadIdx.x;
int64_t const stride = blockDim.x;
int64_t const num_elems_in_col = num_cols / ELEM_PER_THREAD;
for (int elem_index = start_offset; elem_index < num_elems_in_col;
elem_index += stride) {
dest_row_ptr[elem_index] = source_row_ptr[elem_index];
}
}
}
void shuffle_rows(const torch::stable::Tensor& input_tensor,
const torch::stable::Tensor& dst2src_map,
torch::stable::Tensor& output_tensor) {
STD_TORCH_CHECK(input_tensor.scalar_type() == output_tensor.scalar_type(),
"Input and output tensors must have the same data type");
auto stream = get_current_cuda_stream(output_tensor.get_device_index());
const int64_t blocks = output_tensor.size(0);
const int64_t threads = 256;
const int64_t num_dest_rows = output_tensor.size(0);
const int64_t num_src_rows = input_tensor.size(0);
const int64_t num_cols = input_tensor.size(1);
STD_TORCH_CHECK(!(num_cols % (128 / input_tensor.element_size() / 8)),
"num_cols must be divisible by 128 / "
"input_tensor.element_size() / 8");
MOE_DISPATCH(input_tensor.scalar_type(), [&] {
shuffleInputRowsKernel<scalar_t><<<blocks, threads, 0, stream>>>(
reinterpret_cast<const scalar_t*>(input_tensor.const_data_ptr()),
reinterpret_cast<const int32_t*>(dst2src_map.const_data_ptr()),
reinterpret_cast<scalar_t*>(output_tensor.mutable_data_ptr()),
num_src_rows, num_dest_rows, num_cols);
});
}
#else
int64_t moe_permute_sort_workspace_size(int64_t num_expanded_rows,
int64_t n_expert) {
STD_TORCH_CHECK(
false, "moe_permute_sort_workspace_size is not supported on CUDA < 12.0");
}
void moe_permute(const torch::stable::Tensor& input,
const torch::stable::Tensor& topk_ids,
const torch::stable::Tensor& token_expert_indices,
const std::optional<torch::stable::Tensor>& expert_map,
int64_t n_expert, int64_t n_local_expert, int64_t topk,
torch::stable::Tensor& permuted_input,
torch::stable::Tensor& expert_first_token_offset,
torch::stable::Tensor& inv_permuted_idx,
torch::stable::Tensor& permuted_idx) {
STD_TORCH_CHECK(false, "moe_permute is not supported on CUDA < 12.0");
}
void moe_permute_with_scratch(
const torch::stable::Tensor& input, const torch::stable::Tensor& topk_ids,
const torch::stable::Tensor& token_expert_indices,
const std::optional<torch::stable::Tensor>& expert_map, int64_t n_expert,
int64_t n_local_expert, int64_t topk, torch::stable::Tensor& permuted_input,
torch::stable::Tensor& expert_first_token_offset,
torch::stable::Tensor& inv_permuted_idx,
torch::stable::Tensor& permuted_idx, torch::stable::Tensor& sort_workspace,
torch::stable::Tensor& permuted_experts_id,
torch::stable::Tensor& sorted_row_idx,
torch::stable::Tensor& topk_ids_for_sort) {
STD_TORCH_CHECK(false,
"moe_permute_with_scratch is not supported on CUDA < 12.0");
}
void moe_unpermute(
const torch::stable::Tensor& permuted_hidden_states,
const torch::stable::Tensor& topk_weights,
const torch::stable::Tensor& inv_permuted_idx,
const std::optional<torch::stable::Tensor>& expert_first_token_offset,
int64_t topk, torch::stable::Tensor& hidden_states) {
STD_TORCH_CHECK(false, "moe_unpermute is not supported on CUDA < 12.0");
}
#endif
bool moe_permute_unpermute_supported() {
#if defined(CUDA_VERSION) && (CUDA_VERSION >= 12000)
return true;
#else
return false;
#endif
}
STABLE_TORCH_LIBRARY_IMPL(_moe_C, CUDA, m) {
m.impl("moe_permute", TORCH_BOX(&moe_permute));
m.impl("moe_permute_with_scratch", TORCH_BOX(&moe_permute_with_scratch));
m.impl("moe_unpermute", TORCH_BOX(&moe_unpermute));
}
@@ -1,11 +1,14 @@
#include <algorithm>
#include <torch/all.h>
#include <c10/cuda/CUDAGuard.h>
#include <ATen/cuda/CUDAContext.h>
#include <cuda_runtime.h>
#include <torch/csrc/stable/accelerator.h>
#include <torch/csrc/stable/ops.h>
#include <torch/csrc/stable/tensor.h>
#include <torch/headeronly/core/ScalarType.h>
#include <cuda_fp16.h>
#include <cuda_bf16.h>
#include "libtorch_stable/torch_utils.h"
#include "moe_wna16_utils.h"
#define DIVIDE(x, size) (((x) + (size) - 1) / (size))
@@ -263,7 +266,7 @@ void run_moe_wna16_gemm(const scalar_t* input, scalar_t* output,
}
const int shared_mem_size = BLOCK_SIZE_M * BLOCK_SIZE_K * 2;
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
const cudaStream_t stream = get_current_cuda_stream();
kernel<<<gridDim, blockDim, shared_mem_size, stream>>>(
input, output, b_qweight, b_scales, b_qzeros, topk_weights,
sorted_token_ids, expert_ids, num_tokens_post_pad, num_experts,
@@ -271,17 +274,18 @@ void run_moe_wna16_gemm(const scalar_t* input, scalar_t* output,
BLOCK_SIZE_K, has_zp, mul_topk_weight);
}
torch::Tensor moe_wna16_gemm(torch::Tensor input, torch::Tensor output,
torch::Tensor b_qweight, torch::Tensor b_scales,
std::optional<torch::Tensor> b_qzeros,
std::optional<torch::Tensor> topk_weights,
torch::Tensor sorted_token_ids,
torch::Tensor expert_ids,
torch::Tensor num_tokens_post_pad, int64_t top_k,
int64_t BLOCK_SIZE_M, int64_t BLOCK_SIZE_N,
int64_t BLOCK_SIZE_K, int64_t bit) {
const at::cuda::OptionalCUDAGuard device_guard(device_of(input));
output.zero_();
torch::stable::Tensor moe_wna16_gemm(
torch::stable::Tensor input, torch::stable::Tensor output,
torch::stable::Tensor b_qweight, torch::stable::Tensor b_scales,
std::optional<torch::stable::Tensor> b_qzeros,
std::optional<torch::stable::Tensor> topk_weights,
torch::stable::Tensor sorted_token_ids, torch::stable::Tensor expert_ids,
torch::stable::Tensor num_tokens_post_pad, int64_t top_k,
int64_t BLOCK_SIZE_M, int64_t BLOCK_SIZE_N, int64_t BLOCK_SIZE_K,
int64_t bit) {
const torch::stable::accelerator::DeviceGuard device_guard(
input.get_device_index());
torch::stable::zero_(output);
const int num_experts = b_qweight.size(0);
const int size_m = input.size(0);
@@ -291,52 +295,56 @@ torch::Tensor moe_wna16_gemm(torch::Tensor input, torch::Tensor output,
int64_t EM = sorted_token_ids.size(0);
if (size_m <= BLOCK_SIZE_M) {
EM = min(EM, size_m * BLOCK_SIZE_M * top_k);
EM = std::min(EM, size_m * BLOCK_SIZE_M * top_k);
}
const int num_token_blocks = (EM + BLOCK_SIZE_M - 1) / BLOCK_SIZE_M;
const uint32_t* b_qzeros_ptr;
if (b_qzeros.has_value())
b_qzeros_ptr = (const uint32_t*)b_qzeros.value().data_ptr<uint8_t>();
b_qzeros_ptr = (const uint32_t*)b_qzeros.value().const_data_ptr<uint8_t>();
const float* topk_weights_ptr = nullptr;
if (topk_weights.has_value())
topk_weights_ptr = (const float*)topk_weights.value().data_ptr<float>();
topk_weights_ptr =
(const float*)topk_weights.value().const_data_ptr<float>();
int groups_per_block_row = BLOCK_SIZE_K / group_size;
TORCH_CHECK(bit == 4 || bit == 8, "bit must be 4 or 8");
TORCH_CHECK(size_k % BLOCK_SIZE_K == 0,
"size_k must divisible by BLOCK_SIZE_K");
TORCH_CHECK(BLOCK_SIZE_K % group_size == 0,
"BLOCK_SIZE_K must divisible by group_size");
TORCH_CHECK(BLOCK_SIZE_M <= 64, "BLOCK_SIZE_M must less or equal to 64");
TORCH_CHECK(groups_per_block_row == 1 || groups_per_block_row == 2 ||
groups_per_block_row == 4 || groups_per_block_row == 8,
"BLOCK_SIZE_K // group_size must be one of [1, 2, 4, 8]");
STD_TORCH_CHECK(bit == 4 || bit == 8, "bit must be 4 or 8");
STD_TORCH_CHECK(size_k % BLOCK_SIZE_K == 0,
"size_k must divisible by BLOCK_SIZE_K");
STD_TORCH_CHECK(BLOCK_SIZE_K % group_size == 0,
"BLOCK_SIZE_K must divisible by group_size");
STD_TORCH_CHECK(BLOCK_SIZE_M <= 64, "BLOCK_SIZE_M must less or equal to 64");
STD_TORCH_CHECK(groups_per_block_row == 1 || groups_per_block_row == 2 ||
groups_per_block_row == 4 || groups_per_block_row == 8,
"BLOCK_SIZE_K // group_size must be one of [1, 2, 4, 8]");
if (input.scalar_type() == at::ScalarType::Half) {
if (input.scalar_type() == torch::headeronly::ScalarType::Half) {
run_moe_wna16_gemm<half>(
(const half*)input.data_ptr<at::Half>(),
(half*)output.data_ptr<at::Half>(),
(const uint32_t*)b_qweight.data_ptr<uint8_t>(),
(const half*)b_scales.data_ptr<at::Half>(), b_qzeros_ptr,
topk_weights_ptr, sorted_token_ids.data_ptr<int32_t>(),
expert_ids.data_ptr<int32_t>(), num_tokens_post_pad.data_ptr<int32_t>(),
num_experts, group_size, num_token_blocks, top_k, size_m, size_n,
size_k, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, bit,
b_qzeros.has_value(), topk_weights.has_value());
} else if (input.scalar_type() == at::ScalarType::BFloat16) {
reinterpret_cast<const half*>(input.const_data_ptr()),
reinterpret_cast<half*>(output.mutable_data_ptr()),
(const uint32_t*)b_qweight.const_data_ptr<uint8_t>(),
reinterpret_cast<const half*>(b_scales.const_data_ptr()), b_qzeros_ptr,
topk_weights_ptr, sorted_token_ids.const_data_ptr<int32_t>(),
expert_ids.const_data_ptr<int32_t>(),
num_tokens_post_pad.const_data_ptr<int32_t>(), num_experts, group_size,
num_token_blocks, top_k, size_m, size_n, size_k, BLOCK_SIZE_M,
BLOCK_SIZE_N, BLOCK_SIZE_K, bit, b_qzeros.has_value(),
topk_weights.has_value());
} else if (input.scalar_type() == torch::headeronly::ScalarType::BFloat16) {
run_moe_wna16_gemm<nv_bfloat16>(
(const nv_bfloat16*)input.data_ptr<at::BFloat16>(),
(nv_bfloat16*)output.data_ptr<at::BFloat16>(),
(const uint32_t*)b_qweight.data_ptr<uint8_t>(),
(const nv_bfloat16*)b_scales.data_ptr<at::BFloat16>(), b_qzeros_ptr,
topk_weights_ptr, sorted_token_ids.data_ptr<int32_t>(),
expert_ids.data_ptr<int32_t>(), num_tokens_post_pad.data_ptr<int32_t>(),
num_experts, group_size, num_token_blocks, top_k, size_m, size_n,
size_k, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, bit,
b_qzeros.has_value(), topk_weights.has_value());
reinterpret_cast<const nv_bfloat16*>(input.const_data_ptr()),
reinterpret_cast<nv_bfloat16*>(output.mutable_data_ptr()),
(const uint32_t*)b_qweight.const_data_ptr<uint8_t>(),
reinterpret_cast<const nv_bfloat16*>(b_scales.const_data_ptr()),
b_qzeros_ptr, topk_weights_ptr,
sorted_token_ids.const_data_ptr<int32_t>(),
expert_ids.const_data_ptr<int32_t>(),
num_tokens_post_pad.const_data_ptr<int32_t>(), num_experts, group_size,
num_token_blocks, top_k, size_m, size_n, size_k, BLOCK_SIZE_M,
BLOCK_SIZE_N, BLOCK_SIZE_K, bit, b_qzeros.has_value(),
topk_weights.has_value());
} else {
TORCH_CHECK(false, "moe_wna16_gemm only supports bfloat16 and float16");
STD_TORCH_CHECK(false, "moe_wna16_gemm only supports bfloat16 and float16");
}
return output;
}
@@ -0,0 +1,60 @@
#pragma once
#include <cuda_fp8.h>
#include <torch/headeronly/core/ScalarType.h>
#include <torch/headeronly/util/Exception.h>
#define MOE_SWITCH(TYPE, ...) \
const auto _st = (TYPE); \
switch (_st) { \
__VA_ARGS__ \
default: \
STD_TORCH_CHECK(false, "[moe permute]data type dispatch fail!") \
}
#define MOE_DISPATCH_CASE(enum_type, ...) \
case enum_type: { \
using scalar_t = ScalarType2CudaType<enum_type>::type; \
__VA_ARGS__(); \
break; \
}
#define MOE_DISPATCH_FLOAT_CASE(...) \
MOE_DISPATCH_CASE(torch::headeronly::ScalarType::Float, __VA_ARGS__) \
MOE_DISPATCH_CASE(torch::headeronly::ScalarType::Half, __VA_ARGS__) \
MOE_DISPATCH_CASE(torch::headeronly::ScalarType::BFloat16, __VA_ARGS__) \
MOE_DISPATCH_CASE(torch::headeronly::ScalarType::Float8_e5m2, __VA_ARGS__) \
MOE_DISPATCH_CASE(torch::headeronly::ScalarType::Float8_e4m3fn, __VA_ARGS__) \
MOE_DISPATCH_CASE(torch::headeronly::ScalarType::Byte, __VA_ARGS__)
#define MOE_DISPATCH(TYPE, ...) \
MOE_SWITCH(TYPE, MOE_DISPATCH_FLOAT_CASE(__VA_ARGS__))
template <torch::headeronly::ScalarType type>
struct ScalarType2CudaType;
template <>
struct ScalarType2CudaType<torch::headeronly::ScalarType::Float> {
using type = float;
};
template <>
struct ScalarType2CudaType<torch::headeronly::ScalarType::Half> {
using type = half;
};
template <>
struct ScalarType2CudaType<torch::headeronly::ScalarType::BFloat16> {
using type = __nv_bfloat16;
};
// uint8 for packed fp4
template <>
struct ScalarType2CudaType<torch::headeronly::ScalarType::Byte> {
using type = uint8_t;
};
template <>
struct ScalarType2CudaType<torch::headeronly::ScalarType::Float8_e5m2> {
using type = __nv_fp8_e5m2;
};
template <>
struct ScalarType2CudaType<torch::headeronly::ScalarType::Float8_e4m3fn> {
using type = __nv_fp8_e4m3;
};
@@ -1,5 +1,7 @@
#include <cuda.h>
#include <torch/headeronly/util/Exception.h>
#include "moe_permute_unpermute_kernel.h"
#include "libtorch_stable/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.h"
// moe_permute kernels require at least CUDA 12.0
#if defined(CUDA_VERSION) && (CUDA_VERSION >= 12000)
@@ -48,9 +50,10 @@ void CubKeyValueSorter::run(void* workspace, size_t const workspace_size,
size_t expected_ws_size = getWorkspaceSize(num_key_value_pairs, num_experts_);
size_t actual_ws_size = workspace_size;
TORCH_CHECK(expected_ws_size <= workspace_size,
"[CubKeyValueSorter::run] The allocated workspace is too small "
"to run this problem.");
STD_TORCH_CHECK(
expected_ws_size <= workspace_size,
"[CubKeyValueSorter::run] The allocated workspace is too small "
"to run this problem.");
cub::DeviceRadixSort::SortPairs(workspace, actual_ws_size, keys_in, keys_out,
values_in, values_out, num_key_value_pairs, 0,
num_bits_, stream);
@@ -2,23 +2,24 @@
// reference from tensorrt_llm moe kernel implementation archive in
// https://github.com/BBuf/tensorrt-llm-moe/tree/master
#include <c10/core/ScalarType.h>
#include <torch/all.h>
#include "dispatch.h"
#include <torch/csrc/stable/tensor.h>
#include <cub/cub.cuh>
#include <cub/device/device_radix_sort.cuh>
#include <cub/util_type.cuh>
#include "cutlass/numeric_size.h"
#include "cutlass/array.h"
#include "cutlass/numeric_size.h"
#include "libtorch_stable/moe/permute_unpermute_kernels/dispatch.h"
template <typename T>
inline T* get_ptr(torch::Tensor& t) {
return reinterpret_cast<T*>(t.data_ptr());
inline T* get_ptr(torch::stable::Tensor& t) {
return reinterpret_cast<T*>(t.mutable_data_ptr());
}
template <typename T>
inline const T* get_ptr(const torch::Tensor& t) {
return reinterpret_cast<const T*>(t.data_ptr());
inline const T* get_ptr(const torch::stable::Tensor& t) {
return reinterpret_cast<const T*>(t.const_data_ptr());
}
class CubKeyValueSorter {
@@ -17,11 +17,16 @@
* limitations under the License.
*/
#include <type_traits>
#include <torch/all.h>
#include <ATen/cuda/CUDAContext.h>
#include <c10/cuda/CUDAGuard.h>
#include "../cuda_compat.h"
#include "../cub_helpers.h"
#include <cuda_runtime.h>
#include <torch/csrc/stable/accelerator.h>
#include <torch/csrc/stable/tensor.h>
#include <torch/headeronly/core/ScalarType.h>
#include <torch/headeronly/util/Exception.h>
#include "../../cuda_compat.h"
#include "../../cub_helpers.h"
#include "libtorch_stable/torch_utils.h"
#ifndef USE_ROCM
#include <cuda_bf16.h>
@@ -713,7 +718,7 @@ void topkGatingKernelLauncher(
break;
#endif
default: {
TORCH_CHECK(workspace != nullptr,
STD_TORCH_CHECK(workspace != nullptr,
"workspace must be provided for num_experts that are not a power of 2 or multiple of 64.");
static constexpr int TPB = 256;
if constexpr (SF == SCORING_SOFTMAX) {
@@ -723,7 +728,7 @@ void topkGatingKernelLauncher(
moeSigmoid<TPB, InputType><<<num_tokens, TPB, 0, stream>>>(
gating_output, nullptr, workspace, num_experts);
} else {
TORCH_CHECK(false, "Unsupported scoring func");
STD_TORCH_CHECK(false, "Unsupported scoring func");
}
moeTopK<TPB><<<num_tokens, TPB, 0, stream>>>(
workspace, nullptr, topk_weights, topk_indices, token_expert_indices,
@@ -738,63 +743,65 @@ void topkGatingKernelLauncher(
template<typename ComputeType, vllm::moe::ScoringFunc SF>
void dispatch_topk_launch(
torch::Tensor& gating_output,
torch::Tensor& topk_weights,
torch::Tensor& topk_indices,
torch::Tensor& token_expert_indices,
torch::Tensor& softmax_workspace,
torch::stable::Tensor& gating_output,
torch::stable::Tensor& topk_weights,
torch::stable::Tensor& topk_indices,
torch::stable::Tensor& token_expert_indices,
torch::stable::Tensor& softmax_workspace,
int num_tokens, int num_experts, int topk, bool renormalize,
std::optional<torch::Tensor> bias,
std::optional<torch::stable::Tensor> bias,
cudaStream_t stream)
{
const float* bias_ptr = nullptr;
if (bias.has_value()) {
const torch::Tensor& bias_tensor = bias.value();
TORCH_CHECK(bias_tensor.scalar_type() == at::ScalarType::Float, "bias tensor must be float32");
TORCH_CHECK(bias_tensor.dim() == 1, "bias tensor must be 1D");
TORCH_CHECK(bias_tensor.size(0) == num_experts, "bias size mismatch, expected: ", num_experts);
TORCH_CHECK(bias_tensor.is_contiguous(), "bias tensor must be contiguous");
bias_ptr = bias_tensor.data_ptr<float>();
const torch::stable::Tensor& bias_tensor = bias.value();
STD_TORCH_CHECK(bias_tensor.scalar_type() == torch::headeronly::ScalarType::Float,
"bias tensor must be float32");
STD_TORCH_CHECK(bias_tensor.dim() == 1, "bias tensor must be 1D");
STD_TORCH_CHECK(bias_tensor.size(0) == num_experts,
"bias size mismatch, expected: ", num_experts);
STD_TORCH_CHECK(bias_tensor.is_contiguous(), "bias tensor must be contiguous");
bias_ptr = bias_tensor.const_data_ptr<float>();
}
if (topk_indices.scalar_type() == at::ScalarType::Int) {
if (topk_indices.scalar_type() == torch::headeronly::ScalarType::Int) {
vllm::moe::topkGatingKernelLauncher<int, ComputeType, SF>(
reinterpret_cast<const ComputeType*>(gating_output.data_ptr()),
topk_weights.data_ptr<float>(),
topk_indices.data_ptr<int>(),
token_expert_indices.data_ptr<int>(),
softmax_workspace.data_ptr<float>(),
reinterpret_cast<const ComputeType*>(gating_output.const_data_ptr()),
topk_weights.mutable_data_ptr<float>(),
topk_indices.mutable_data_ptr<int>(),
token_expert_indices.mutable_data_ptr<int>(),
softmax_workspace.mutable_data_ptr<float>(),
num_tokens, num_experts, topk, renormalize,
bias_ptr, stream);
} else if (topk_indices.scalar_type() == at::ScalarType::UInt32) {
} else if (topk_indices.scalar_type() == torch::headeronly::ScalarType::UInt32) {
vllm::moe::topkGatingKernelLauncher<uint32_t, ComputeType, SF>(
reinterpret_cast<const ComputeType*>(gating_output.data_ptr()),
topk_weights.data_ptr<float>(),
topk_indices.data_ptr<uint32_t>(),
token_expert_indices.data_ptr<int>(),
softmax_workspace.data_ptr<float>(),
reinterpret_cast<const ComputeType*>(gating_output.const_data_ptr()),
topk_weights.mutable_data_ptr<float>(),
topk_indices.mutable_data_ptr<uint32_t>(),
token_expert_indices.mutable_data_ptr<int>(),
softmax_workspace.mutable_data_ptr<float>(),
num_tokens, num_experts, topk, renormalize,
bias_ptr, stream);
} else {
TORCH_CHECK(topk_indices.scalar_type() == at::ScalarType::Long);
STD_TORCH_CHECK(topk_indices.scalar_type() == torch::headeronly::ScalarType::Long);
vllm::moe::topkGatingKernelLauncher<int64_t, ComputeType, SF>(
reinterpret_cast<const ComputeType*>(gating_output.data_ptr()),
topk_weights.data_ptr<float>(),
topk_indices.data_ptr<int64_t>(),
token_expert_indices.data_ptr<int>(),
softmax_workspace.data_ptr<float>(),
reinterpret_cast<const ComputeType*>(gating_output.const_data_ptr()),
topk_weights.mutable_data_ptr<float>(),
topk_indices.mutable_data_ptr<int64_t>(),
token_expert_indices.mutable_data_ptr<int>(),
softmax_workspace.mutable_data_ptr<float>(),
num_tokens, num_experts, topk, renormalize,
bias_ptr, stream);
}
}
void topk_softmax(
torch::Tensor& topk_weights, // [num_tokens, topk]
torch::Tensor& topk_indices, // [num_tokens, topk]
torch::Tensor& token_expert_indices, // [num_tokens, topk]
torch::Tensor& gating_output, // [num_tokens, num_experts]
torch::stable::Tensor& topk_weights, // [num_tokens, topk]
torch::stable::Tensor& topk_indices, // [num_tokens, topk]
torch::stable::Tensor& token_expert_indices, // [num_tokens, topk]
torch::stable::Tensor& gating_output, // [num_tokens, num_experts]
bool renormalize,
std::optional<torch::Tensor> bias)
std::optional<torch::stable::Tensor> bias)
{
const int num_experts = gating_output.size(-1);
const auto num_tokens = gating_output.numel() / num_experts;
@@ -804,35 +811,36 @@ void topk_softmax(
const bool needs_workspace = !is_pow_2 || num_experts > 256;
const int64_t workspace_size = needs_workspace ? num_tokens * num_experts : 0;
const at::cuda::OptionalCUDAGuard device_guard(device_of(gating_output));
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
const auto workspace_options = gating_output.options().dtype(at::ScalarType::Float);
torch::Tensor softmax_workspace = torch::empty({workspace_size}, workspace_options);
torch::stable::accelerator::DeviceGuard guard(gating_output.get_device_index());
const cudaStream_t stream =
get_current_cuda_stream(gating_output.get_device_index());
auto softmax_workspace = torch::stable::new_empty(
gating_output, {workspace_size}, torch::headeronly::ScalarType::Float);
if (gating_output.scalar_type() == at::ScalarType::Float) {
if (gating_output.scalar_type() == torch::headeronly::ScalarType::Float) {
dispatch_topk_launch<float, vllm::moe::SCORING_SOFTMAX>(gating_output, topk_weights, topk_indices,
token_expert_indices, softmax_workspace, num_tokens, num_experts, topk, renormalize,
bias, stream);
} else if (gating_output.scalar_type() == at::ScalarType::Half) {
} else if (gating_output.scalar_type() == torch::headeronly::ScalarType::Half) {
dispatch_topk_launch<__half, vllm::moe::SCORING_SOFTMAX>(gating_output, topk_weights, topk_indices,
token_expert_indices, softmax_workspace, num_tokens, num_experts, topk, renormalize,
bias, stream);
} else if (gating_output.scalar_type() == at::ScalarType::BFloat16) {
} else if (gating_output.scalar_type() == torch::headeronly::ScalarType::BFloat16) {
dispatch_topk_launch<__nv_bfloat16, vllm::moe::SCORING_SOFTMAX>(gating_output, topk_weights, topk_indices,
token_expert_indices, softmax_workspace, num_tokens, num_experts, topk, renormalize,
bias, stream);
} else {
TORCH_CHECK(false, "Unsupported gating_output data type: ", gating_output.scalar_type());
STD_TORCH_CHECK(false, "Unsupported gating_output data type: ", gating_output.scalar_type());
}
}
void topk_sigmoid(
torch::Tensor& topk_weights, // [num_tokens, topk]
torch::Tensor& topk_indices, // [num_tokens, topk]
torch::Tensor& token_expert_indices, // [num_tokens, topk]
torch::Tensor& gating_output, // [num_tokens, num_experts]
torch::stable::Tensor& topk_weights, // [num_tokens, topk]
torch::stable::Tensor& topk_indices, // [num_tokens, topk]
torch::stable::Tensor& token_expert_indices, // [num_tokens, topk]
torch::stable::Tensor& gating_output, // [num_tokens, num_experts]
bool renormalize,
std::optional<torch::Tensor> bias)
std::optional<torch::stable::Tensor> bias)
{
const int num_experts = gating_output.size(-1);
const auto num_tokens = gating_output.numel() / num_experts;
@@ -842,24 +850,25 @@ void topk_sigmoid(
const bool needs_workspace = !is_pow_2 || num_experts > 256;
const int64_t workspace_size = needs_workspace ? num_tokens * num_experts : 0;
const at::cuda::OptionalCUDAGuard device_guard(device_of(gating_output));
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
const auto workspace_options = gating_output.options().dtype(at::ScalarType::Float);
torch::Tensor workspace = torch::empty({workspace_size}, workspace_options);
torch::stable::accelerator::DeviceGuard guard(gating_output.get_device_index());
const cudaStream_t stream =
get_current_cuda_stream(gating_output.get_device_index());
auto workspace = torch::stable::new_empty(
gating_output, {workspace_size}, torch::headeronly::ScalarType::Float);
if (gating_output.scalar_type() == at::ScalarType::Float) {
if (gating_output.scalar_type() == torch::headeronly::ScalarType::Float) {
dispatch_topk_launch<float, vllm::moe::SCORING_SIGMOID>(gating_output, topk_weights, topk_indices,
token_expert_indices, workspace, num_tokens, num_experts, topk, renormalize,
bias, stream);
} else if (gating_output.scalar_type() == at::ScalarType::Half) {
} else if (gating_output.scalar_type() == torch::headeronly::ScalarType::Half) {
dispatch_topk_launch<__half, vllm::moe::SCORING_SIGMOID>(gating_output, topk_weights, topk_indices,
token_expert_indices, workspace, num_tokens, num_experts, topk, renormalize,
bias, stream);
} else if (gating_output.scalar_type() == at::ScalarType::BFloat16) {
} else if (gating_output.scalar_type() == torch::headeronly::ScalarType::BFloat16) {
dispatch_topk_launch<__nv_bfloat16, vllm::moe::SCORING_SIGMOID>(gating_output, topk_weights, topk_indices,
token_expert_indices, workspace, num_tokens, num_experts, topk, renormalize,
bias, stream);
} else {
TORCH_CHECK(false, "Unsupported gating_output data type: ", gating_output.scalar_type());
STD_TORCH_CHECK(false, "Unsupported gating_output data type: ", gating_output.scalar_type());
}
}
@@ -18,11 +18,16 @@
* limitations under the License.
*/
#include <type_traits>
#include <torch/all.h>
#include <ATen/cuda/CUDAContext.h>
#include <c10/cuda/CUDAGuard.h>
#include "../cuda_compat.h"
#include "../cub_helpers.h"
#include <cuda_runtime.h>
#include <torch/csrc/stable/accelerator.h>
#include <torch/csrc/stable/tensor.h>
#include <torch/headeronly/core/ScalarType.h>
#include <torch/headeronly/util/Exception.h>
#include "../../cuda_compat.h"
#include "../../cub_helpers.h"
#include "libtorch_stable/torch_utils.h"
#ifndef USE_ROCM
#include <cuda_bf16.h>
#include <cuda_fp16.h>
@@ -618,7 +623,7 @@ void topkGatingSoftplusSqrtKernelLauncher(
LAUNCH_SOFTPLUS_SQRT(576, WARPS_PER_TB, BYTES_PER_LDG_MULTIPLE_64_NARROW);
break;
default: {
TORCH_CHECK(false, "Unsupported expert number: ", num_experts);
STD_TORCH_CHECK(false, "Unsupported expert number: ", num_experts);
}
}
}
@@ -628,100 +633,109 @@ void topkGatingSoftplusSqrtKernelLauncher(
template <typename ComputeType>
void dispatch_topk_softplus_sqrt_launch(
const ComputeType* gating_output, torch::Tensor& topk_weights,
torch::Tensor& topk_indices, torch::Tensor& token_expert_indices,
int num_tokens, int num_experts, int topk, bool renormalize,
double routed_scaling_factor,
const c10::optional<torch::Tensor>& correction_bias,
const c10::optional<torch::Tensor>& input_ids,
const c10::optional<torch::Tensor>& tid2eid, cudaStream_t stream) {
const ComputeType* gating_output, torch::stable::Tensor& topk_weights,
torch::stable::Tensor& topk_indices,
torch::stable::Tensor& token_expert_indices, int num_tokens,
int num_experts, int topk, bool renormalize, double routed_scaling_factor,
const std::optional<torch::stable::Tensor>& correction_bias,
const std::optional<torch::stable::Tensor>& input_ids,
const std::optional<torch::stable::Tensor>& tid2eid, cudaStream_t stream) {
const float* bias_ptr = nullptr;
if (correction_bias.has_value()) {
bias_ptr = correction_bias.value().data_ptr<float>();
bias_ptr = correction_bias.value().const_data_ptr<float>();
}
bool use_hash = false;
if (tid2eid.has_value()) {
TORCH_CHECK(input_ids.has_value(), "input_ids is required for hash MoE");
STD_TORCH_CHECK(input_ids.has_value(),
"input_ids is required for hash MoE");
use_hash = true;
}
if (topk_indices.scalar_type() == at::ScalarType::Int) {
if (topk_indices.scalar_type() == torch::headeronly::ScalarType::Int) {
const int* input_ids_ptr = nullptr;
const int* tid2eid_ptr = nullptr;
if (tid2eid.has_value()) {
input_ids_ptr = input_ids.value().data_ptr<int>();
tid2eid_ptr = tid2eid.value().data_ptr<int>();
input_ids_ptr = input_ids.value().const_data_ptr<int>();
tid2eid_ptr = tid2eid.value().const_data_ptr<int>();
}
vllm::moe::topkGatingSoftplusSqrtKernelLauncher<int, ComputeType>(
gating_output, topk_weights.data_ptr<float>(),
topk_indices.data_ptr<int>(), token_expert_indices.data_ptr<int>(),
num_tokens, num_experts, topk, renormalize, routed_scaling_factor,
bias_ptr, use_hash, input_ids_ptr, tid2eid_ptr, stream);
} else if (topk_indices.scalar_type() == at::ScalarType::UInt32) {
gating_output, topk_weights.mutable_data_ptr<float>(),
topk_indices.mutable_data_ptr<int>(),
token_expert_indices.mutable_data_ptr<int>(), num_tokens, num_experts,
topk, renormalize, routed_scaling_factor, bias_ptr, use_hash,
input_ids_ptr, tid2eid_ptr, stream);
} else if (topk_indices.scalar_type() ==
torch::headeronly::ScalarType::UInt32) {
const uint32_t* input_ids_ptr = nullptr;
const uint32_t* tid2eid_ptr = nullptr;
if (tid2eid.has_value()) {
input_ids_ptr = input_ids.value().data_ptr<uint32_t>();
tid2eid_ptr = tid2eid.value().data_ptr<uint32_t>();
input_ids_ptr = input_ids.value().const_data_ptr<uint32_t>();
tid2eid_ptr = tid2eid.value().const_data_ptr<uint32_t>();
}
vllm::moe::topkGatingSoftplusSqrtKernelLauncher<uint32_t, ComputeType>(
gating_output, topk_weights.data_ptr<float>(),
topk_indices.data_ptr<uint32_t>(), token_expert_indices.data_ptr<int>(),
num_tokens, num_experts, topk, renormalize, routed_scaling_factor,
bias_ptr, use_hash, input_ids_ptr, tid2eid_ptr, stream);
gating_output, topk_weights.mutable_data_ptr<float>(),
topk_indices.mutable_data_ptr<uint32_t>(),
token_expert_indices.mutable_data_ptr<int>(), num_tokens, num_experts,
topk, renormalize, routed_scaling_factor, bias_ptr, use_hash,
input_ids_ptr, tid2eid_ptr, stream);
} else {
TORCH_CHECK(topk_indices.scalar_type() == at::ScalarType::Long);
STD_TORCH_CHECK(topk_indices.scalar_type() ==
torch::headeronly::ScalarType::Long);
const int64_t* input_ids_ptr = nullptr;
const int64_t* tid2eid_ptr = nullptr;
if (tid2eid.has_value()) {
input_ids_ptr = input_ids.value().data_ptr<int64_t>();
tid2eid_ptr = tid2eid.value().data_ptr<int64_t>();
input_ids_ptr = input_ids.value().const_data_ptr<int64_t>();
tid2eid_ptr = tid2eid.value().const_data_ptr<int64_t>();
}
vllm::moe::topkGatingSoftplusSqrtKernelLauncher<int64_t, ComputeType>(
gating_output, topk_weights.data_ptr<float>(),
topk_indices.data_ptr<int64_t>(), token_expert_indices.data_ptr<int>(),
num_tokens, num_experts, topk, renormalize, routed_scaling_factor,
bias_ptr, use_hash, input_ids_ptr, tid2eid_ptr, stream);
gating_output, topk_weights.mutable_data_ptr<float>(),
topk_indices.mutable_data_ptr<int64_t>(),
token_expert_indices.mutable_data_ptr<int>(), num_tokens, num_experts,
topk, renormalize, routed_scaling_factor, bias_ptr, use_hash,
input_ids_ptr, tid2eid_ptr, stream);
}
}
void topk_softplus_sqrt(
torch::Tensor& topk_weights, // [num_tokens, topk]
torch::Tensor& topk_indices, // [num_tokens, topk]
torch::Tensor& token_expert_indices, // [num_tokens, topk]
torch::Tensor& gating_output, // [num_tokens, num_experts]
torch::stable::Tensor& topk_weights, // [num_tokens, topk]
torch::stable::Tensor& topk_indices, // [num_tokens, topk]
torch::stable::Tensor& token_expert_indices, // [num_tokens, topk]
torch::stable::Tensor& gating_output, // [num_tokens, num_experts]
bool renormalize, double routed_scaling_factor,
const c10::optional<torch::Tensor>& correction_bias,
const c10::optional<torch::Tensor>& input_ids,
const c10::optional<torch::Tensor>& tid2eid) {
const std::optional<torch::stable::Tensor>& correction_bias,
const std::optional<torch::stable::Tensor>& input_ids,
const std::optional<torch::stable::Tensor>& tid2eid) {
const int num_experts = gating_output.size(-1);
const auto num_tokens = gating_output.numel() / num_experts;
const int topk = topk_weights.size(-1);
const at::cuda::OptionalCUDAGuard device_guard(device_of(gating_output));
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
const torch::stable::accelerator::DeviceGuard guard(
gating_output.get_device_index());
const cudaStream_t stream =
get_current_cuda_stream(gating_output.get_device_index());
if (gating_output.scalar_type() == at::ScalarType::Float) {
if (gating_output.scalar_type() == torch::headeronly::ScalarType::Float) {
dispatch_topk_softplus_sqrt_launch<float>(
gating_output.data_ptr<float>(), topk_weights, topk_indices,
gating_output.const_data_ptr<float>(), topk_weights, topk_indices,
token_expert_indices, num_tokens, num_experts, topk, renormalize,
routed_scaling_factor, correction_bias, input_ids, tid2eid, stream);
} else if (gating_output.scalar_type() == at::ScalarType::Half) {
} else if (gating_output.scalar_type() ==
torch::headeronly::ScalarType::Half) {
dispatch_topk_softplus_sqrt_launch<__half>(
reinterpret_cast<const __half*>(gating_output.data_ptr<at::Half>()),
reinterpret_cast<const __half*>(gating_output.const_data_ptr()),
topk_weights, topk_indices, token_expert_indices, num_tokens,
num_experts, topk, renormalize, routed_scaling_factor, correction_bias,
input_ids, tid2eid, stream);
} else if (gating_output.scalar_type() == at::ScalarType::BFloat16) {
} else if (gating_output.scalar_type() ==
torch::headeronly::ScalarType::BFloat16) {
dispatch_topk_softplus_sqrt_launch<__nv_bfloat16>(
reinterpret_cast<const __nv_bfloat16*>(
gating_output.data_ptr<at::BFloat16>()),
reinterpret_cast<const __nv_bfloat16*>(gating_output.const_data_ptr()),
topk_weights, topk_indices, token_expert_indices, num_tokens,
num_experts, topk, renormalize, routed_scaling_factor, correction_bias,
input_ids, tid2eid, stream);
} else {
TORCH_CHECK(false, "Unsupported gating_output data type: ",
gating_output.scalar_type());
STD_TORCH_CHECK(false, "Unsupported gating_output data type: ",
gating_output.scalar_type());
}
}
@@ -1,32 +1,30 @@
#include "core/registration.h"
#include "moe_ops.h"
TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
#include <torch/csrc/stable/library.h>
STABLE_TORCH_LIBRARY_FRAGMENT(_moe_C, m) {
// Apply topk softmax to the gating outputs.
m.def(
"topk_softmax(Tensor! topk_weights, Tensor! topk_indices, Tensor! "
"token_expert_indices, Tensor gating_output, bool renormalize, Tensor? "
"bias) -> ()");
m.impl("topk_softmax", torch::kCUDA, &topk_softmax);
// Apply topk sigmoid to the gating outputs.
m.def(
"topk_sigmoid(Tensor! topk_weights, Tensor! topk_indices, Tensor! "
"token_expert_indices, Tensor gating_output, bool renormalize, Tensor? "
"bias) -> ()");
m.impl("topk_sigmoid", torch::kCUDA, &topk_sigmoid);
m.def(
"topk_softplus_sqrt(Tensor! topk_weights, Tensor! topk_indices, Tensor! "
"token_expert_indices, Tensor gating_output, bool renormalize, float "
"routed_scaling_factor, Tensor? "
"bias, Tensor? input_ids, Tensor? tid2eid) -> ()");
m.impl("topk_softplus_sqrt", torch::kCUDA, &topk_softplus_sqrt);
// Calculate the result of moe by summing up the partial results
// from all selected experts.
m.def("moe_sum(Tensor input, Tensor! output) -> ()");
m.impl("moe_sum", torch::kCUDA, &moe_sum);
// Aligning the number of tokens to be processed by each expert such
// that it is divisible by the block size.
@@ -36,7 +34,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
" Tensor! experts_ids,"
" Tensor! num_tokens_post_pad,"
" Tensor? maybe_expert_map) -> ()");
m.impl("moe_align_block_size", torch::kCUDA, &moe_align_block_size);
// Aligning the number of tokens to be processed by each expert such
// that it is divisible by the block size, but for the batched case.
@@ -46,8 +43,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
" Tensor! sorted_token_ids,"
" Tensor! experts_ids,"
" Tensor! num_tokens_post_pad) -> ()");
m.impl("batched_moe_align_block_size", torch::kCUDA,
&batched_moe_align_block_size);
// Aligning the number of tokens to be processed by each expert such
// that it is divisible by the block size.
@@ -64,8 +59,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
" Tensor !adapter_enabled,"
" Tensor !lora_ids,"
" Tensor? maybe_expert_map) -> () ");
m.impl("moe_lora_align_block_size", torch::kCUDA, &moe_lora_align_block_size);
#ifndef USE_ROCM
m.def(
"moe_wna16_gemm(Tensor input, Tensor! output, Tensor b_qweight, "
@@ -75,8 +68,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
"int top_k, int BLOCK_SIZE_M, int BLOCK_SIZE_N, int BLOCK_SIZE_K, "
"int bit) -> Tensor");
m.impl("moe_wna16_gemm", torch::kCUDA, &moe_wna16_gemm);
m.def(
"moe_wna16_marlin_gemm(Tensor! a, Tensor? c_or_none,"
"Tensor! b_q_weight, Tensor? b_bias_or_none,"
@@ -118,14 +109,11 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
m.def(
"moe_permute_sort_workspace_size(int num_expanded_rows, int n_expert) -> "
"int");
m.impl("moe_permute_unpermute_supported", &moe_permute_unpermute_supported);
m.impl("moe_permute_sort_workspace_size", &moe_permute_sort_workspace_size);
// Row shuffle for MoE
m.def(
"shuffle_rows(Tensor input_tensor, Tensor dst2src_map, Tensor! "
"output_tensor) -> ()");
m.impl("shuffle_rows", torch::kCUDA, &shuffle_rows);
// Apply grouped topk routing to select experts.
m.def(
@@ -133,7 +121,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
"topk_group, int topk, bool renormalize, float "
"routed_scaling_factor, Tensor bias, int scoring_func) -> (Tensor, "
"Tensor)");
m.impl("grouped_topk", torch::kCUDA, &grouped_topk);
// DeepSeek V3 optimized router GEMM for SM90+
m.def("dsv3_router_gemm(Tensor! output, Tensor mat_a, Tensor mat_b) -> ()");
@@ -141,4 +128,30 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
#endif
}
REGISTER_EXTENSION(TORCH_EXTENSION_NAME)
STABLE_TORCH_LIBRARY_IMPL(_moe_C, CUDA, m) {
m.impl("topk_softmax", TORCH_BOX(&topk_softmax));
m.impl("topk_sigmoid", TORCH_BOX(&topk_sigmoid));
m.impl("topk_softplus_sqrt", TORCH_BOX(&topk_softplus_sqrt));
m.impl("moe_sum", TORCH_BOX(&moe_sum));
m.impl("moe_align_block_size", TORCH_BOX(&moe_align_block_size));
m.impl("batched_moe_align_block_size",
TORCH_BOX(&batched_moe_align_block_size));
m.impl("moe_lora_align_block_size", TORCH_BOX(&moe_lora_align_block_size));
#ifndef USE_ROCM
m.impl("moe_wna16_gemm", TORCH_BOX(&moe_wna16_gemm));
m.impl("shuffle_rows", TORCH_BOX(&shuffle_rows));
m.impl("grouped_topk", TORCH_BOX(&grouped_topk));
#endif
}
#ifndef USE_ROCM
// Primitive-only ops have no tensor to dispatch on.
STABLE_TORCH_LIBRARY_IMPL(_moe_C, CompositeExplicitAutograd, m) {
m.impl("moe_permute_unpermute_supported",
TORCH_BOX(&moe_permute_unpermute_supported));
m.impl("moe_permute_sort_workspace_size",
TORCH_BOX(&moe_permute_sort_workspace_size));
}
#endif
REGISTER_EXTENSION(_moe_C_stable_libtorch)
+16
View File
@@ -246,6 +246,22 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
"awq_dequantize(Tensor _kernel, Tensor _scaling_factors, "
"Tensor _zeros, SymInt split_k_iters, int thx, int thy) -> Tensor");
// Expert-specialization mxfp8 blockscaled grouped quantization (SM100+).
ops.def(
"mxfp8_experts_quant("
" Tensor input, Tensor problem_sizes, Tensor expert_offsets,"
" Tensor blockscale_offsets, Tensor! quant_output, Tensor! scale_factor)"
" -> ()");
// conditionally compiled so impl registration is in source file
// Expert-specialization mxfp8 blockscaled grouped GEMM (SM100+).
ops.def(
"cutlass_mxfp8_grouped_mm("
" Tensor a, Tensor b, Tensor sfa, Tensor sfb, Tensor! out,"
" Tensor problem_sizes, Tensor expert_offsets, Tensor blockscale_offsets)"
" -> ()");
// conditionally compiled so impl registration is in source file
// DeepSeek V3 fused A GEMM (SM 9.0+, bf16 only, 1-16 tokens).
// conditionally compiled so impl registration is in source file
ops.def(
-31
View File
@@ -1,31 +0,0 @@
/*
* Adapted from SGLang's sgl-kernel implementation, which was adapted from
* https://github.com/NVIDIA/TensorRT-LLM/blob/main/cpp/tensorrt_llm/kernels/dsv3MinLatencyKernels/dsv3RouterGemm.cu
* https://github.com/NVIDIA/TensorRT-LLM/blob/main/cpp/tensorrt_llm/thop/dsv3RouterGemmOp.cpp
*
* Copyright (c) 2019-2023, NVIDIA CORPORATION. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <ATen/cuda/CUDAContext.h>
#include <cstdlib>
#include <mutex>
inline int getSMVersion() {
auto* props = at::cuda::getCurrentDeviceProperties();
return props->major * 10 + props->minor;
}
-81
View File
@@ -1,81 +0,0 @@
#pragma once
#include <torch/all.h>
void topk_softmax(torch::Tensor& topk_weights, torch::Tensor& topk_indices,
torch::Tensor& token_expert_indices,
torch::Tensor& gating_output, bool renormalize,
std::optional<torch::Tensor> bias);
void topk_sigmoid(torch::Tensor& topk_weights, torch::Tensor& topk_indices,
torch::Tensor& token_expert_indices,
torch::Tensor& gating_output, bool renormalize,
std::optional<torch::Tensor> bias);
void topk_softplus_sqrt(torch::Tensor& topk_weights,
torch::Tensor& topk_indices,
torch::Tensor& token_expert_indices,
torch::Tensor& gating_output, bool renormalize,
double routed_scaling_factor,
const c10::optional<torch::Tensor>& correction_bias,
const c10::optional<torch::Tensor>& input_ids,
const c10::optional<torch::Tensor>& tid2eid);
void moe_sum(torch::Tensor& input, torch::Tensor& output);
void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
int64_t block_size, torch::Tensor sorted_token_ids,
torch::Tensor experts_ids,
torch::Tensor num_tokens_post_pad,
std::optional<torch::Tensor> maybe_expert_map);
void batched_moe_align_block_size(int64_t max_tokens_per_batch,
int64_t block_size,
torch::Tensor const& expert_num_tokens,
torch::Tensor sorted_ids,
torch::Tensor expert_ids,
torch::Tensor num_tokens_post_pad);
void moe_lora_align_block_size(
torch::Tensor topk_ids, torch::Tensor token_lora_mapping,
int64_t num_experts, int64_t block_size, int64_t max_loras,
int64_t max_num_tokens_padded, int64_t max_num_m_blocks,
torch::Tensor sorted_token_ids, torch::Tensor expert_ids,
torch::Tensor num_tokens_post_pad, torch::Tensor adapter_enabled,
torch::Tensor lora_ids, std::optional<torch::Tensor> maybe_expert_map);
#ifndef USE_ROCM
torch::Tensor moe_wna16_gemm(torch::Tensor input, torch::Tensor output,
torch::Tensor b_qweight, torch::Tensor b_scales,
std::optional<torch::Tensor> b_qzeros,
std::optional<torch::Tensor> topk_weights,
torch::Tensor sorted_token_ids,
torch::Tensor expert_ids,
torch::Tensor num_tokens_post_pad, int64_t top_k,
int64_t BLOCK_SIZE_M, int64_t BLOCK_SIZE_N,
int64_t BLOCK_SIZE_K, int64_t bit);
std::tuple<torch::Tensor, torch::Tensor> grouped_topk(
torch::Tensor const& scores, int64_t n_group, int64_t topk_group,
int64_t topk, bool renormalize, double routed_scaling_factor,
torch::Tensor const& bias, int64_t scoring_func);
#endif
bool moe_permute_unpermute_supported();
int64_t moe_permute_sort_workspace_size(int64_t num_expanded_rows,
int64_t num_experts);
void shuffle_rows(const torch::Tensor& input_tensor,
const torch::Tensor& dst2src_map,
torch::Tensor& output_tensor);
#ifndef USE_ROCM
// DeepSeek V3 optimized router GEMM kernel for SM90+
// Computes output = mat_a @ mat_b.T where:
// mat_a: [num_tokens, hidden_dim] in bf16
// mat_b: [num_experts, hidden_dim] in bf16
// output: [num_tokens, num_experts] in bf16 or fp32
// Supports num_tokens in [1, 16], num_experts in {256, 384}, hidden_dim = 7168
void dsv3_router_gemm(torch::Tensor& output, const torch::Tensor& mat_a,
const torch::Tensor& mat_b);
#endif
-286
View File
@@ -1,286 +0,0 @@
#include <c10/core/ScalarType.h>
#include <torch/all.h>
#include <ATen/cuda/CUDAContext.h>
#include "permute_unpermute_kernels/moe_permute_unpermute_kernel.h"
#include "permute_unpermute_kernels/dispatch.h"
#include "core/registration.h"
// moe_permute kernels require at least CUDA 12.0
#if defined(CUDA_VERSION) && (CUDA_VERSION >= 12000)
namespace {
torch::Tensor maybe_allocate_tensor(
const std::optional<torch::Tensor>& maybe_tensor,
at::IntArrayRef expected_sizes, torch::ScalarType dtype, c10::Device device,
char const* name) {
auto expected_numel = c10::multiply_integers(expected_sizes);
if (maybe_tensor.has_value()) {
auto tensor = maybe_tensor.value();
TORCH_CHECK(tensor.device() == device, name, " must be on the same device");
TORCH_CHECK(tensor.scalar_type() == dtype, name, " has incorrect dtype");
TORCH_CHECK(tensor.is_contiguous(), name, " must be contiguous");
TORCH_CHECK(tensor.numel() >= expected_numel, name,
" is too small for the requested shape");
auto flat_tensor = tensor.view({tensor.numel()});
return flat_tensor.narrow(0, 0, expected_numel).view(expected_sizes);
}
return torch::empty(expected_sizes, torch::dtype(dtype).device(device));
}
} // namespace
int64_t moe_permute_sort_workspace_size(int64_t num_expanded_rows,
int64_t n_expert) {
return static_cast<int64_t>(
CubKeyValueSorter::getWorkspaceSize(num_expanded_rows, n_expert));
}
void moe_permute_impl(
const torch::Tensor& input, // [n_token, hidden]
const torch::Tensor& topk_ids, // [n_token, topk]
const torch::Tensor& token_expert_indices, // [n_token, topk]
const std::optional<torch::Tensor>& expert_map, // [n_expert]
int64_t n_expert, int64_t n_local_expert, int64_t topk,
torch::Tensor& permuted_input, // [permuted_size, hidden]
torch::Tensor& expert_first_token_offset, // [n_local_expert + 1]
torch::Tensor& inv_permuted_idx, // [n_token, topk]
torch::Tensor& permuted_idx, // [permute_size]
const std::optional<torch::Tensor>& maybe_sort_workspace,
const std::optional<torch::Tensor>& maybe_permuted_experts_id,
const std::optional<torch::Tensor>& maybe_sorted_row_idx,
const std::optional<torch::Tensor>& maybe_topk_ids_for_sort) {
TORCH_CHECK(expert_first_token_offset.scalar_type() == at::ScalarType::Long,
"expert_first_token_offset must be int64");
TORCH_CHECK(topk_ids.scalar_type() == at::ScalarType::Int,
"topk_ids must be int32");
TORCH_CHECK(token_expert_indices.scalar_type() == at::ScalarType::Int,
"token_expert_indices must be int32");
TORCH_CHECK(inv_permuted_idx.scalar_type() == at::ScalarType::Int,
"inv_permuted_idx must be int32");
TORCH_CHECK(expert_first_token_offset.size(0) == n_local_expert + 1,
"expert_first_token_offset shape != n_local_expert+1");
TORCH_CHECK(inv_permuted_idx.sizes() == token_expert_indices.sizes(),
"token_expert_indices shape must be same as inv_permuted_idx");
auto device = input.device();
auto n_token = input.sizes()[0];
auto n_hidden = input.sizes()[1];
auto expanded_rows = n_token * topk;
auto stream = at::cuda::getCurrentCUDAStream().stream();
auto sorter_size = moe_permute_sort_workspace_size(expanded_rows, n_expert);
auto sort_workspace =
maybe_allocate_tensor(maybe_sort_workspace, {sorter_size}, torch::kInt8,
device, "sort_workspace");
auto permuted_experts_id =
maybe_allocate_tensor(maybe_permuted_experts_id, topk_ids.sizes(),
at::ScalarType::Int, device, "permuted_experts_id");
auto sorted_row_idx =
maybe_allocate_tensor(maybe_sorted_row_idx, inv_permuted_idx.sizes(),
at::ScalarType::Int, device, "sorted_row_idx");
CubKeyValueSorter sorter{};
int64_t* valid_num_ptr = nullptr;
torch::Tensor topk_ids_for_sort = topk_ids;
if (expert_map.has_value()) {
const int* expert_map_ptr = get_ptr<int>(expert_map.value());
valid_num_ptr =
get_ptr<int64_t>(expert_first_token_offset) + n_local_expert;
topk_ids_for_sort =
maybe_allocate_tensor(maybe_topk_ids_for_sort, topk_ids.sizes(),
at::ScalarType::Int, device, "topk_ids_for_sort");
topk_ids_for_sort.copy_(topk_ids);
preprocessTopkIdLauncher(get_ptr<int>(topk_ids_for_sort), n_token * topk,
expert_map_ptr, n_expert, stream);
}
sortAndScanExpert(
get_ptr<const int>(topk_ids_for_sort), get_ptr<int>(token_expert_indices),
get_ptr<int>(permuted_experts_id), get_ptr<int>(sorted_row_idx),
get_ptr<int64_t>(expert_first_token_offset), n_token, n_expert,
n_local_expert, topk, sorter, get_ptr<int>(sort_workspace), stream);
MOE_DISPATCH(input.scalar_type(), [&] {
expandInputRowsKernelLauncher<scalar_t>(
get_ptr<scalar_t>(input), get_ptr<scalar_t>(permuted_input),
get_ptr<int>(sorted_row_idx), get_ptr<int>(inv_permuted_idx),
get_ptr<int>(permuted_idx), get_ptr<int64_t>(expert_first_token_offset),
n_token, valid_num_ptr, n_hidden, topk, n_local_expert, stream);
});
}
void moe_permute(
const torch::Tensor& input, // [n_token, hidden]
const torch::Tensor& topk_ids, // [n_token, topk]
const torch::Tensor& token_expert_indices, // [n_token, topk]
const std::optional<torch::Tensor>& expert_map, // [n_expert]
int64_t n_expert, int64_t n_local_expert, int64_t topk,
torch::Tensor& permuted_input, // [permuted_size, hidden]
torch::Tensor& expert_first_token_offset, // [n_local_expert + 1]
torch::Tensor& inv_permuted_idx, // [n_token, topk]
torch::Tensor& permuted_idx) { // [permute_size]
moe_permute_impl(input, topk_ids, token_expert_indices, expert_map, n_expert,
n_local_expert, topk, permuted_input,
expert_first_token_offset, inv_permuted_idx, permuted_idx,
std::nullopt, std::nullopt, std::nullopt, std::nullopt);
}
void moe_permute_with_scratch(
const torch::Tensor& input, const torch::Tensor& topk_ids,
const torch::Tensor& token_expert_indices,
const std::optional<torch::Tensor>& expert_map, int64_t n_expert,
int64_t n_local_expert, int64_t topk, torch::Tensor& permuted_input,
torch::Tensor& expert_first_token_offset, torch::Tensor& inv_permuted_idx,
torch::Tensor& permuted_idx, torch::Tensor& sort_workspace,
torch::Tensor& permuted_experts_id, torch::Tensor& sorted_row_idx,
torch::Tensor& topk_ids_for_sort) {
moe_permute_impl(input, topk_ids, token_expert_indices, expert_map, n_expert,
n_local_expert, topk, permuted_input,
expert_first_token_offset, inv_permuted_idx, permuted_idx,
sort_workspace, permuted_experts_id, sorted_row_idx,
topk_ids_for_sort);
}
void moe_unpermute(
const torch::Tensor& permuted_hidden_states, // [n_token * topk, hidden]
const torch::Tensor& topk_weights, // [n_token, topk]
const torch::Tensor& inv_permuted_idx, // [n_token, topk]
const std::optional<torch::Tensor>&
expert_first_token_offset, // [n_local_expert+1]
int64_t topk,
torch::Tensor& hidden_states // [n_token, hidden]
) {
TORCH_CHECK(
permuted_hidden_states.scalar_type() == hidden_states.scalar_type(),
"permuted_hidden_states dtype must be same as hidden_states");
auto n_token = hidden_states.size(0);
auto n_hidden = hidden_states.size(1);
auto stream = at::cuda::getCurrentCUDAStream().stream();
int64_t const* valid_ptr = nullptr;
if (expert_first_token_offset.has_value()) {
int n_local_expert = expert_first_token_offset.value().size(0) - 1;
valid_ptr =
get_ptr<int64_t>(expert_first_token_offset.value()) + n_local_expert;
}
MOE_DISPATCH(hidden_states.scalar_type(), [&] {
finalizeMoeRoutingKernelLauncher<scalar_t, scalar_t>(
get_ptr<scalar_t>(permuted_hidden_states),
get_ptr<scalar_t>(hidden_states), get_ptr<float>(topk_weights),
get_ptr<int>(inv_permuted_idx), n_token, n_hidden, topk, valid_ptr,
stream);
});
}
template <typename T>
__global__ void shuffleInputRowsKernel(const T* input,
const int32_t* dst2src_map, T* output,
int64_t num_src_rows,
int64_t num_dst_rows, int64_t num_cols) {
int64_t dest_row_idx = blockIdx.x;
int64_t const source_row_idx = dst2src_map[dest_row_idx];
if (blockIdx.x < num_dst_rows) {
// Load 128-bits per thread
constexpr int64_t ELEM_PER_THREAD = 128 / sizeof(T) / 8;
using DataElem = cutlass::Array<T, ELEM_PER_THREAD>;
// Duplicate and permute rows
auto const* source_row_ptr =
reinterpret_cast<DataElem const*>(input + source_row_idx * num_cols);
auto* dest_row_ptr =
reinterpret_cast<DataElem*>(output + dest_row_idx * num_cols);
int64_t const start_offset = threadIdx.x;
int64_t const stride = blockDim.x;
int64_t const num_elems_in_col = num_cols / ELEM_PER_THREAD;
for (int elem_index = start_offset; elem_index < num_elems_in_col;
elem_index += stride) {
dest_row_ptr[elem_index] = source_row_ptr[elem_index];
}
}
}
void shuffle_rows(const torch::Tensor& input_tensor,
const torch::Tensor& dst2src_map,
torch::Tensor& output_tensor) {
TORCH_CHECK(input_tensor.scalar_type() == output_tensor.scalar_type(),
"Input and output tensors must have the same data type");
auto stream = at::cuda::getCurrentCUDAStream().stream();
int64_t const blocks = output_tensor.size(0);
int64_t const threads = 256;
int64_t const num_dest_rows = output_tensor.size(0);
int64_t const num_src_rows = input_tensor.size(0);
int64_t const num_cols = input_tensor.size(1);
TORCH_CHECK(!(num_cols % (128 / sizeof(input_tensor.scalar_type()) / 8)),
"num_cols must be divisible by 128 / "
"sizeof(input_tensor.scalar_type()) / 8");
MOE_DISPATCH(input_tensor.scalar_type(), [&] {
shuffleInputRowsKernel<scalar_t><<<blocks, threads, 0, stream>>>(
reinterpret_cast<scalar_t*>(input_tensor.data_ptr()),
dst2src_map.data_ptr<int32_t>(),
reinterpret_cast<scalar_t*>(output_tensor.data_ptr()), num_src_rows,
num_dest_rows, num_cols);
});
}
#else
int64_t moe_permute_sort_workspace_size(int64_t num_expanded_rows,
int64_t n_expert) {
TORCH_CHECK(
false, "moe_permute_sort_workspace_size is not supported on CUDA < 12.0");
}
void moe_permute(const torch::Tensor& input, const torch::Tensor& topk_ids,
const torch::Tensor& token_expert_indices,
const std::optional<torch::Tensor>& expert_map,
int64_t n_expert, int64_t n_local_expert, int64_t topk,
torch::Tensor& permuted_input,
torch::Tensor& expert_first_token_offset,
torch::Tensor& inv_permuted_idx, torch::Tensor& permuted_idx) {
TORCH_CHECK(false, "moe_permute is not supported on CUDA < 12.0");
}
void moe_permute_with_scratch(
const torch::Tensor& input, const torch::Tensor& topk_ids,
const torch::Tensor& token_expert_indices,
const std::optional<torch::Tensor>& expert_map, int64_t n_expert,
int64_t n_local_expert, int64_t topk, torch::Tensor& permuted_input,
torch::Tensor& expert_first_token_offset, torch::Tensor& inv_permuted_idx,
torch::Tensor& permuted_idx, torch::Tensor& sort_workspace,
torch::Tensor& permuted_experts_id, torch::Tensor& sorted_row_idx,
torch::Tensor& topk_ids_for_sort) {
TORCH_CHECK(false,
"moe_permute_with_scratch is not supported on CUDA < 12.0");
}
void moe_unpermute(
const torch::Tensor& permuted_hidden_states,
const torch::Tensor& topk_weights, const torch::Tensor& inv_permuted_idx,
const std::optional<torch::Tensor>& expert_first_token_offset, int64_t topk,
torch::Tensor& hidden_states) {
TORCH_CHECK(false, "moe_unpermute is not supported on CUDA < 12.0");
}
#endif
bool moe_permute_unpermute_supported() {
#if defined(CUDA_VERSION) && (CUDA_VERSION >= 12000)
return true;
#else
return false;
#endif
}
TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) {
m.impl("moe_permute", &moe_permute);
m.impl("moe_permute_with_scratch", &moe_permute_with_scratch);
m.impl("moe_unpermute", &moe_unpermute);
}
@@ -1,59 +0,0 @@
#pragma once
#include <cuda_fp8.h>
#define MOE_SWITCH(TYPE, ...) \
at::ScalarType _st = ::detail::scalar_type(TYPE); \
switch (_st) { \
__VA_ARGS__ \
default: \
TORCH_CHECK(false, "[moe permute]data type dispatch fail!") \
}
#define MOE_DISPATCH_CASE(enum_type, ...) \
case enum_type: { \
using scalar_t = ScalarType2CudaType<enum_type>::type; \
__VA_ARGS__(); \
break; \
}
#define MOE_DISPATCH_FLOAT_CASE(...) \
MOE_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
MOE_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \
MOE_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) \
MOE_DISPATCH_CASE(at::ScalarType::Float8_e5m2, __VA_ARGS__) \
MOE_DISPATCH_CASE(at::ScalarType::Float8_e4m3fn, __VA_ARGS__) \
MOE_DISPATCH_CASE(at::ScalarType::Byte, __VA_ARGS__)
#define MOE_DISPATCH(TYPE, ...) \
MOE_SWITCH(TYPE, MOE_DISPATCH_FLOAT_CASE(__VA_ARGS__))
template <at::ScalarType type>
struct ScalarType2CudaType;
template <>
struct ScalarType2CudaType<at::ScalarType::Float> {
using type = float;
};
template <>
struct ScalarType2CudaType<at::ScalarType::Half> {
using type = half;
};
template <>
struct ScalarType2CudaType<at::ScalarType::BFloat16> {
using type = __nv_bfloat16;
};
// uint8 for packed fp4
template <>
struct ScalarType2CudaType<at::ScalarType::Byte> {
using type = uint8_t;
};
// #if __CUDA_ARCH__ >= 890
// fp8
template <>
struct ScalarType2CudaType<at::ScalarType::Float8_e5m2> {
using type = __nv_fp8_e5m2;
};
template <>
struct ScalarType2CudaType<at::ScalarType::Float8_e4m3fn> {
using type = __nv_fp8_e4m3;
};
// #endif
-19
View File
@@ -130,25 +130,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
"Tensor? qzeros_or_none, bool inplace) -> Tensor");
// conditionally compiled so impl registrations are in source file
#endif
#ifndef USE_ROCM
// Expert-specialization mxfp8 blockscaled grouped quantization (SM100+).
ops.def(
"mxfp8_experts_quant("
" Tensor input, Tensor problem_sizes, Tensor expert_offsets,"
" Tensor blockscale_offsets, Tensor! quant_output, Tensor! scale_factor)"
" -> ()");
// conditionally compiled so impl registration is in source file
// Expert-specialization mxfp8 blockscaled grouped GEMM (SM100+).
ops.def(
"cutlass_mxfp8_grouped_mm("
" Tensor a, Tensor b, Tensor sfa, Tensor sfb, Tensor! out,"
" Tensor problem_sizes, Tensor expert_offsets, Tensor blockscale_offsets)"
" -> ()");
// conditionally compiled so impl registration is in source file
#endif
}
+2 -2
View File
@@ -755,7 +755,7 @@ class precompiled_wheel_utils:
{
"vllm/_C.abi3.so",
"vllm/_C_stable_libtorch.abi3.so",
"vllm/_moe_C.abi3.so",
"vllm/_moe_C_stable_libtorch.abi3.so",
"vllm/_flashmla_C.abi3.so",
"vllm/_flashmla_extension_C.abi3.so",
"vllm/_sparse_flashmla_C.abi3.so",
@@ -1081,7 +1081,6 @@ def get_requirements() -> list[str]:
ext_modules = []
if _is_cuda() or _is_hip():
ext_modules.append(CMakeExtension(name="vllm._moe_C"))
ext_modules.append(CMakeExtension(name="vllm.cumem_allocator"))
# Optional since this doesn't get built (produce an .so file). This is just
# copying the relevant .py files from the source repository.
@@ -1135,6 +1134,7 @@ if _build_custom_ops():
ext_modules.append(CMakeExtension(name="vllm._C"))
if _is_cuda() or _is_hip():
ext_modules.append(CMakeExtension(name="vllm._C_stable_libtorch"))
ext_modules.append(CMakeExtension(name="vllm._moe_C_stable_libtorch"))
package_data = {
"vllm": [
+1 -1
View File
@@ -254,7 +254,7 @@ class Platform:
except ImportError as e:
logger.warning("Failed to import from vllm._C: %r", e)
with contextlib.suppress(ImportError):
import vllm._moe_C # noqa: F401
import vllm._moe_C_stable_libtorch # noqa: F401
@classmethod
def get_attn_backend_cls(