From 6e64c1bab1875a5c096860dff9b7250f7d484094 Mon Sep 17 00:00:00 2001 From: Chris Leonard Date: Thu, 11 Jun 2026 02:02:26 -0400 Subject: [PATCH] [10c/n] Migrate MoE kernels to torch stable ABI (#44565) Signed-off-by: Chris Leonard Co-authored-by: Shengqi Chen --- .gitignore | 2 +- .pre-commit-config.yaml | 2 +- CMakeLists.txt | 81 +++- csrc/libtorch_stable/dispatch_utils.h | 22 + .../moe/dsv3_router_gemm_bf16_out.cu | 5 +- .../moe/dsv3_router_gemm_entry.cu | 76 +-- .../moe/dsv3_router_gemm_float_out.cu | 5 +- .../moe/grouped_topk_kernels.cu | 88 ++-- .../moe/marlin_moe_wna16/.gitignore | 0 .../moe/marlin_moe_wna16/generate_kernels.py | 2 +- .../moe/marlin_moe_wna16/kernel.h | 0 .../moe/marlin_moe_wna16/marlin_template.h | 0 .../moe/marlin_moe_wna16/ops.cu | 440 +++++++++--------- .../moe/moeTopKFuncs.cuh | 0 .../moe/moe_align_sum_kernels.cu | 271 ++++++----- csrc/libtorch_stable/moe/moe_ops.h | 87 ++++ .../moe/moe_permute_unpermute_op.cu | 319 +++++++++++++ csrc/{ => libtorch_stable}/moe/moe_wna16.cu | 104 +++-- .../moe/moe_wna16_utils.h | 0 .../moe/permute_unpermute_kernels/dispatch.h | 60 +++ .../moe_permute_unpermute_kernel.cu | 11 +- .../moe_permute_unpermute_kernel.h | 17 +- .../moe_permute_unpermute_kernel.inl | 0 .../moe/topk_softmax_kernels.cu | 135 +++--- .../moe/topk_softplus_sqrt_kernels.cu | 122 ++--- .../moe/torch_bindings.cpp | 47 +- csrc/libtorch_stable/torch_bindings.cpp | 16 + csrc/moe/dsv3_router_gemm_utils.h | 31 -- csrc/moe/moe_ops.h | 81 ---- csrc/moe/moe_permute_unpermute_op.cu | 286 ------------ csrc/moe/permute_unpermute_kernels/dispatch.h | 59 --- csrc/torch_bindings.cpp | 19 - setup.py | 4 +- vllm/platforms/interface.py | 2 +- 34 files changed, 1296 insertions(+), 1098 deletions(-) rename csrc/{ => libtorch_stable}/moe/dsv3_router_gemm_bf16_out.cu (99%) rename csrc/{ => libtorch_stable}/moe/dsv3_router_gemm_entry.cu (74%) rename csrc/{ => libtorch_stable}/moe/dsv3_router_gemm_float_out.cu (99%) rename csrc/{ => libtorch_stable}/moe/grouped_topk_kernels.cu (94%) rename csrc/{ => libtorch_stable}/moe/marlin_moe_wna16/.gitignore (100%) rename csrc/{ => libtorch_stable}/moe/marlin_moe_wna16/generate_kernels.py (99%) rename csrc/{ => libtorch_stable}/moe/marlin_moe_wna16/kernel.h (100%) rename csrc/{ => libtorch_stable}/moe/marlin_moe_wna16/marlin_template.h (100%) rename csrc/{ => libtorch_stable}/moe/marlin_moe_wna16/ops.cu (62%) rename csrc/{ => libtorch_stable}/moe/moeTopKFuncs.cuh (100%) rename csrc/{ => libtorch_stable}/moe/moe_align_sum_kernels.cu (74%) create mode 100644 csrc/libtorch_stable/moe/moe_ops.h create mode 100644 csrc/libtorch_stable/moe/moe_permute_unpermute_op.cu rename csrc/{ => libtorch_stable}/moe/moe_wna16.cu (77%) rename csrc/{ => libtorch_stable}/moe/moe_wna16_utils.h (100%) create mode 100644 csrc/libtorch_stable/moe/permute_unpermute_kernels/dispatch.h rename csrc/{ => libtorch_stable}/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.cu (95%) rename csrc/{ => libtorch_stable}/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.h (89%) rename csrc/{ => libtorch_stable}/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.inl (100%) rename csrc/{ => libtorch_stable}/moe/topk_softmax_kernels.cu (88%) rename csrc/{ => libtorch_stable}/moe/topk_softplus_sqrt_kernels.cu (87%) rename csrc/{ => libtorch_stable}/moe/torch_bindings.cpp (82%) delete mode 100644 csrc/moe/dsv3_router_gemm_utils.h delete mode 100644 csrc/moe/moe_ops.h delete mode 100644 csrc/moe/moe_permute_unpermute_op.cu delete mode 100644 csrc/moe/permute_unpermute_kernels/dispatch.h diff --git a/.gitignore b/.gitignore index 2c4e135e58..8dde75e43e 100644 --- a/.gitignore +++ b/.gitignore @@ -233,7 +233,7 @@ actionlint shellcheck*/ # Ignore moe/marlin_moe gen code -csrc/moe/marlin_moe_wna16/kernel_* +csrc/libtorch_stable/moe/marlin_moe_wna16/kernel_* # Ignore ep_kernels_workspace folder ep_kernels_workspace/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dff099e369..d0c83833a6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,7 +21,7 @@ repos: rev: v21.1.2 hooks: - id: clang-format - exclude: 'csrc/(moe/topk_softmax_kernels.cu|libtorch_stable/quantization/gguf/(ggml-common.h|dequantize.cuh|vecdotq.cuh|mmq.cuh|mmvq.cuh))|vllm/third_party/.*' + exclude: 'csrc/(libtorch_stable/moe/topk_softmax_kernels.cu|libtorch_stable/quantization/gguf/(ggml-common.h|dequantize.cuh|vecdotq.cuh|mmq.cuh|mmvq.cuh))|vllm/third_party/.*' types_or: [c++, cuda] args: [--style=file, --verbose] - repo: https://github.com/DavidAnson/markdownlint-cli2 diff --git a/CMakeLists.txt b/CMakeLists.txt index d956e29e39..20a44be8f1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1115,25 +1115,25 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") endif() # -# _moe_C extension +# _moe_C_stable_libtorch extension # set(VLLM_MOE_EXT_SRC - "csrc/moe/torch_bindings.cpp" - "csrc/moe/moe_align_sum_kernels.cu" - "csrc/moe/topk_softmax_kernels.cu" - "csrc/moe/topk_softplus_sqrt_kernels.cu") + "csrc/libtorch_stable/moe/torch_bindings.cpp" + "csrc/libtorch_stable/moe/moe_align_sum_kernels.cu" + "csrc/libtorch_stable/moe/topk_softmax_kernels.cu" + "csrc/libtorch_stable/moe/topk_softplus_sqrt_kernels.cu") if(VLLM_GPU_LANG STREQUAL "CUDA") list(APPEND VLLM_MOE_EXT_SRC - "csrc/moe/moe_wna16.cu" - "csrc/moe/grouped_topk_kernels.cu") + "csrc/libtorch_stable/moe/moe_wna16.cu" + "csrc/libtorch_stable/moe/grouped_topk_kernels.cu") endif() if(VLLM_GPU_LANG STREQUAL "CUDA") set(MOE_PERMUTE_SRC - "csrc/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.cu" - "csrc/moe/moe_permute_unpermute_op.cu") + "csrc/libtorch_stable/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.cu" + "csrc/libtorch_stable/moe/moe_permute_unpermute_op.cu") list(APPEND VLLM_MOE_EXT_SRC "${MOE_PERMUTE_SRC}") endif() @@ -1144,7 +1144,7 @@ set_gencode_flags_for_srcs( if(VLLM_GPU_LANG STREQUAL "CUDA") set(VLLM_MOE_WNA16_SRC - "csrc/moe/moe_wna16.cu") + "csrc/libtorch_stable/moe/moe_wna16.cu") set_gencode_flags_for_srcs( SRCS "${VLLM_MOE_WNA16_SRC}" @@ -1175,7 +1175,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA") # preselected input type pairs and schedules. # Generate sources: set(MOE_MARLIN_GEN_SCRIPT - ${CMAKE_CURRENT_SOURCE_DIR}/csrc/moe/marlin_moe_wna16/generate_kernels.py) + ${CMAKE_CURRENT_SOURCE_DIR}/csrc/libtorch_stable/moe/marlin_moe_wna16/generate_kernels.py) file(MD5 ${MOE_MARLIN_GEN_SCRIPT} MOE_MARLIN_GEN_SCRIPT_HASH) list(JOIN CUDA_ARCHS "," CUDA_ARCHS_STR) set(MOE_MARLIN_GEN_SCRIPT_HASH_AND_ARCH "${MOE_MARLIN_GEN_SCRIPT_HASH}(ARCH:${CUDA_ARCHS_STR})") @@ -1210,7 +1210,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA") endif() if (MARLIN_MOE_ARCHS) - file(GLOB MARLIN_MOE_SRC "csrc/moe/marlin_moe_wna16/sm80_kernel_*.cu") + file(GLOB MARLIN_MOE_SRC "csrc/libtorch_stable/moe/marlin_moe_wna16/sm80_kernel_*.cu") set_gencode_flags_for_srcs( SRCS "${MARLIN_MOE_SRC}" CUDA_ARCHS "${MARLIN_MOE_ARCHS}") @@ -1222,7 +1222,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA") endif() if (MARLIN_MOE_SM75_ARCHS) - file(GLOB MARLIN_MOE_SM75_SRC "csrc/moe/marlin_moe_wna16/sm75_kernel_*.cu") + file(GLOB MARLIN_MOE_SM75_SRC "csrc/libtorch_stable/moe/marlin_moe_wna16/sm75_kernel_*.cu") set_gencode_flags_for_srcs( SRCS "${MARLIN_MOE_SM75_SRC}" CUDA_ARCHS "${MARLIN_MOE_SM75_ARCHS}") @@ -1234,7 +1234,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA") endif() if (MARLIN_MOE_FP8_ARCHS) - file(GLOB MARLIN_MOE_FP8_SRC "csrc/moe/marlin_moe_wna16/sm89_kernel_*.cu") + file(GLOB MARLIN_MOE_FP8_SRC "csrc/libtorch_stable/moe/marlin_moe_wna16/sm89_kernel_*.cu") set_gencode_flags_for_srcs( SRCS "${MARLIN_MOE_FP8_SRC}" CUDA_ARCHS "${MARLIN_MOE_FP8_ARCHS}") @@ -1245,7 +1245,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA") list(APPEND VLLM_MOE_EXT_SRC ${MARLIN_MOE_FP8_SRC}) endif() - set(MARLIN_MOE_OTHER_SRC "csrc/moe/marlin_moe_wna16/ops.cu") + set(MARLIN_MOE_OTHER_SRC "csrc/libtorch_stable/moe/marlin_moe_wna16/ops.cu") set_gencode_flags_for_srcs( SRCS "${MARLIN_MOE_OTHER_SRC}" CUDA_ARCHS "${MARLIN_MOE_OTHER_ARCHS}") @@ -1266,9 +1266,9 @@ if(VLLM_GPU_LANG STREQUAL "CUDA") cuda_archs_sm90plus(SM90PLUS_ROUTER_GEMM_ARCHS "${CUDA_ARCHS}") if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.0 AND SM90PLUS_ROUTER_GEMM_ARCHS) set(DSV3_ROUTER_GEMM_SRC - "csrc/moe/dsv3_router_gemm_entry.cu" - "csrc/moe/dsv3_router_gemm_float_out.cu" - "csrc/moe/dsv3_router_gemm_bf16_out.cu") + "csrc/libtorch_stable/moe/dsv3_router_gemm_entry.cu" + "csrc/libtorch_stable/moe/dsv3_router_gemm_float_out.cu" + "csrc/libtorch_stable/moe/dsv3_router_gemm_bf16_out.cu") set_gencode_flags_for_srcs( SRCS "${DSV3_ROUTER_GEMM_SRC}" CUDA_ARCHS "${SM90PLUS_ROUTER_GEMM_ARCHS}") @@ -1281,9 +1281,9 @@ if(VLLM_GPU_LANG STREQUAL "CUDA") endif() endif() -message(STATUS "Enabling moe extension.") +message(STATUS "Enabling MoE C_stable extension.") define_extension_target( - _moe_C + _moe_C_stable_libtorch DESTINATION vllm LANGUAGE ${VLLM_GPU_LANG} SOURCES ${VLLM_MOE_EXT_SRC} @@ -1294,6 +1294,47 @@ define_extension_target( USE_SABI 3 WITH_SOABI) +# Needed to use cuda/hip APIs from C-shim +if(VLLM_GPU_LANG STREQUAL "CUDA") + # Set TORCH_TARGET_VERSION for stable ABI compatibility. + # This ensures we only use C-shim APIs available in PyTorch 2.11. + # _moe_C_stable_libtorch is abi compatible with PyTorch >= TORCH_TARGET_VERSION + # which is currently set to 2.11. + target_compile_definitions(_moe_C_stable_libtorch PRIVATE + TORCH_TARGET_VERSION=0x020B000000000000ULL) + target_compile_definitions(_moe_C_stable_libtorch PRIVATE USE_CUDA) + # Needed by CUTLASS kernels + target_compile_definitions(_moe_C_stable_libtorch PRIVATE + CUTLASS_ENABLE_DIRECT_CUDA_DRIVER_CALL=1) +elseif(VLLM_GPU_LANG STREQUAL "HIP") + # Set TORCH_TARGET_VERSION for stable ABI compatibility. + # This ensures we only use C-shim APIs available in PyTorch 2.10. + # _moe_C_stable_libtorch is abi compatible with PyTorch >= TORCH_TARGET_VERSION + # which is currently set to 2.10. + target_compile_definitions(_moe_C_stable_libtorch PRIVATE + TORCH_TARGET_VERSION=0x020A000000000000ULL) + target_compile_definitions(_moe_C_stable_libtorch PRIVATE USE_ROCM) +endif() + +# On ROCm, _moe_C_stable_libtorch calls raw HIP APIs (e.g. hipGetDevice in +# get_device_prop()) which must resolve to the same libamdhip64.so that +# PyTorch uses. When PyTorch bundles its own copy (pip/conda wheels), +# the raw HIP calls would otherwise resolve to the system ROCm copy, +# initializing a second HIP runtime that corrupts device state (wrong +# device on DeviceGuard, core dumps on multi-GPU tests). +# +# If PyTorch doesn't bundle libamdhip64 (built from source against system +# ROCm), there is only one copy in the process and no action is needed — +# the HIP compiler already links the system libamdhip64 automatically. +if(VLLM_GPU_LANG STREQUAL "HIP") + find_library(_MOE_STABLE_TORCH_AMDHIP64 amdhip64 + PATHS "${TORCH_INSTALL_PREFIX}/lib" NO_DEFAULT_PATH) + if(_MOE_STABLE_TORCH_AMDHIP64) + message(STATUS "Found PyTorch-bundled libamdhip64 for _moe_C_stable_libtorch at ${_MOE_STABLE_TORCH_AMDHIP64}") + target_link_libraries(_moe_C_stable_libtorch PRIVATE ${_MOE_STABLE_TORCH_AMDHIP64}) + endif() +endif() + if(VLLM_GPU_LANG STREQUAL "HIP") # # _rocm_C extension diff --git a/csrc/libtorch_stable/dispatch_utils.h b/csrc/libtorch_stable/dispatch_utils.h index e9478236a0..cd67ac751c 100644 --- a/csrc/libtorch_stable/dispatch_utils.h +++ b/csrc/libtorch_stable/dispatch_utils.h @@ -30,6 +30,28 @@ THO_DISPATCH_SWITCH(TYPE, NAME, \ VLLM_STABLE_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__)) +#define VLLM_STABLE_DISPATCH_CASE_INTEGRAL_TYPES(...) \ + THO_DISPATCH_CASE(torch::headeronly::ScalarType::Byte, __VA_ARGS__) \ + THO_DISPATCH_CASE(torch::headeronly::ScalarType::Char, __VA_ARGS__) \ + THO_DISPATCH_CASE(torch::headeronly::ScalarType::Short, __VA_ARGS__) \ + THO_DISPATCH_CASE(torch::headeronly::ScalarType::Int, __VA_ARGS__) \ + THO_DISPATCH_CASE(torch::headeronly::ScalarType::Long, __VA_ARGS__) + +#define VLLM_STABLE_DISPATCH_CASE_INTEGRAL_AND_UNSIGNED_TYPES(...) \ + VLLM_STABLE_DISPATCH_CASE_INTEGRAL_TYPES(__VA_ARGS__) \ + THO_DISPATCH_CASE(torch::headeronly::ScalarType::UInt16, __VA_ARGS__) \ + THO_DISPATCH_CASE(torch::headeronly::ScalarType::UInt32, __VA_ARGS__) \ + THO_DISPATCH_CASE(torch::headeronly::ScalarType::UInt64, __VA_ARGS__) + +#define VLLM_STABLE_DISPATCH_INTEGRAL_TYPES(TYPE, NAME, ...) \ + THO_DISPATCH_SWITCH(TYPE, NAME, \ + VLLM_STABLE_DISPATCH_CASE_INTEGRAL_TYPES(__VA_ARGS__)) + +#define VLLM_STABLE_DISPATCH_INTEGRAL_AND_UNSIGNED_TYPES(TYPE, NAME, ...) \ + THO_DISPATCH_SWITCH( \ + TYPE, NAME, \ + VLLM_STABLE_DISPATCH_CASE_INTEGRAL_AND_UNSIGNED_TYPES(__VA_ARGS__)) + // FP8 type dispatch - ROCm uses FNUZ format, CUDA uses OCP format #ifdef USE_ROCM #define VLLM_STABLE_DISPATCH_CASE_FP8_TYPES(...) \ diff --git a/csrc/moe/dsv3_router_gemm_bf16_out.cu b/csrc/libtorch_stable/moe/dsv3_router_gemm_bf16_out.cu similarity index 99% rename from csrc/moe/dsv3_router_gemm_bf16_out.cu rename to csrc/libtorch_stable/moe/dsv3_router_gemm_bf16_out.cu index b11ba991b2..776c92678d 100644 --- a/csrc/moe/dsv3_router_gemm_bf16_out.cu +++ b/csrc/libtorch_stable/moe/dsv3_router_gemm_bf16_out.cu @@ -18,14 +18,11 @@ * limitations under the License. */ -#include -#include +#include #include #include -#include "dsv3_router_gemm_utils.h" - // Custom FMA implementation using PTX assembly instructions __device__ __forceinline__ void fma(float2& d, float2 const& a, float2 const& b, float2 const& c) { diff --git a/csrc/moe/dsv3_router_gemm_entry.cu b/csrc/libtorch_stable/moe/dsv3_router_gemm_entry.cu similarity index 74% rename from csrc/moe/dsv3_router_gemm_entry.cu rename to csrc/libtorch_stable/moe/dsv3_router_gemm_entry.cu index 38fb681c22..1de1a319e4 100644 --- a/csrc/moe/dsv3_router_gemm_entry.cu +++ b/csrc/libtorch_stable/moe/dsv3_router_gemm_entry.cu @@ -18,15 +18,25 @@ * limitations under the License. */ -#include -#include -#include +#include +#include +#include + +#include "libtorch_stable/torch_utils.h" #include #include -#include "core/registration.h" -#include "dsv3_router_gemm_utils.h" +#include + +namespace { + +inline int getSMVersion() { + auto* props = get_device_prop(); + return props->major * 10 + props->minor; +} + +} // namespace static constexpr int DEFAULT_NUM_EXPERTS = 256; static constexpr int KIMI_K2_NUM_EXPERTS = 384; @@ -98,40 +108,48 @@ struct LoopUnroller { } }; -void dsv3_router_gemm(at::Tensor& output, // [num_tokens, num_experts] - const at::Tensor& mat_a, // [num_tokens, hidden_dim] - const at::Tensor& mat_b // [num_experts, hidden_dim] +void dsv3_router_gemm( + torch::stable::Tensor& output, // [num_tokens, num_experts] + torch::stable::Tensor const& mat_a, // [num_tokens, hidden_dim] + torch::stable::Tensor const& mat_b // [num_experts, hidden_dim] ) { - TORCH_CHECK(output.dim() == 2 && mat_a.dim() == 2 && mat_b.dim() == 2); + STD_TORCH_CHECK(output.dim() == 2 && mat_a.dim() == 2 && mat_b.dim() == 2); const int num_tokens = mat_a.size(0); const int num_experts = mat_b.size(0); const int hidden_dim = mat_a.size(1); - TORCH_CHECK(mat_a.size(1) == mat_b.size(1), - "mat_a and mat_b must have the same hidden_dim"); - TORCH_CHECK(hidden_dim == DEFAULT_HIDDEN_DIM, - "Expected hidden_dim=", DEFAULT_HIDDEN_DIM, - ", but got hidden_dim=", hidden_dim); - TORCH_CHECK( + STD_TORCH_CHECK(mat_a.size(1) == mat_b.size(1), + "mat_a and mat_b must have the same hidden_dim"); + STD_TORCH_CHECK(hidden_dim == DEFAULT_HIDDEN_DIM, + "Expected hidden_dim=", DEFAULT_HIDDEN_DIM, + ", but got hidden_dim=", hidden_dim); + STD_TORCH_CHECK( num_experts == DEFAULT_NUM_EXPERTS || num_experts == KIMI_K2_NUM_EXPERTS, "Expected num_experts=", DEFAULT_NUM_EXPERTS, " or num_experts=", KIMI_K2_NUM_EXPERTS, ", but got num_experts=", num_experts); - TORCH_CHECK(num_tokens >= 1 && num_tokens <= 16, - "currently num_tokens must be less than or equal to 16 for " - "router_gemm"); - TORCH_CHECK(mat_a.dtype() == at::kBFloat16, "mat_a must be bf16"); - TORCH_CHECK(mat_b.dtype() == at::kBFloat16, "mat_b must be bf16"); - TORCH_CHECK(output.dtype() == at::kFloat || output.dtype() == at::kBFloat16, - "output must be float32 or bf16"); + STD_TORCH_CHECK(num_tokens >= 1 && num_tokens <= 16, + "currently num_tokens must be less than or equal to 16 for " + "router_gemm"); + STD_TORCH_CHECK( + mat_a.scalar_type() == torch::headeronly::ScalarType::BFloat16, + "mat_a must be bf16"); + STD_TORCH_CHECK( + mat_b.scalar_type() == torch::headeronly::ScalarType::BFloat16, + "mat_b must be bf16"); + STD_TORCH_CHECK( + output.scalar_type() == torch::headeronly::ScalarType::Float || + output.scalar_type() == torch::headeronly::ScalarType::BFloat16, + "output must be float32 or bf16"); - auto const sm = getSMVersion(); - TORCH_CHECK(sm >= 90 && sm <= 103, "required SM_103 >= CUDA ARCH >= SM_90"); + const int sm = getSMVersion(); + STD_TORCH_CHECK(sm >= 90 && sm <= 103, + "required SM_103 >= CUDA ARCH >= SM_90"); - const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + const cudaStream_t stream = get_current_cuda_stream(mat_a.get_device_index()); - if (output.dtype() == at::kFloat) { + if (output.scalar_type() == torch::headeronly::ScalarType::Float) { if (num_experts == DEFAULT_NUM_EXPERTS) { LoopUnroller<1, 16, DEFAULT_NUM_EXPERTS, DEFAULT_HIDDEN_DIM>:: unroll_float_output( @@ -145,7 +163,7 @@ void dsv3_router_gemm(at::Tensor& output, // [num_tokens, num_experts] reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()), reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), stream); } - } else if (output.dtype() == at::kBFloat16) { + } else if (output.scalar_type() == torch::headeronly::ScalarType::BFloat16) { if (num_experts == DEFAULT_NUM_EXPERTS) { LoopUnroller<1, 16, DEFAULT_NUM_EXPERTS, DEFAULT_HIDDEN_DIM>:: unroll_bf16_output( @@ -164,6 +182,6 @@ void dsv3_router_gemm(at::Tensor& output, // [num_tokens, num_experts] } } -TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) { - m.impl("dsv3_router_gemm", &dsv3_router_gemm); +STABLE_TORCH_LIBRARY_IMPL(_moe_C, CUDA, m) { + m.impl("dsv3_router_gemm", TORCH_BOX(&dsv3_router_gemm)); } diff --git a/csrc/moe/dsv3_router_gemm_float_out.cu b/csrc/libtorch_stable/moe/dsv3_router_gemm_float_out.cu similarity index 99% rename from csrc/moe/dsv3_router_gemm_float_out.cu rename to csrc/libtorch_stable/moe/dsv3_router_gemm_float_out.cu index 2756cba0b1..113ad27638 100644 --- a/csrc/moe/dsv3_router_gemm_float_out.cu +++ b/csrc/libtorch_stable/moe/dsv3_router_gemm_float_out.cu @@ -18,14 +18,11 @@ * limitations under the License. */ -#include -#include +#include #include #include -#include "dsv3_router_gemm_utils.h" - // Custom FMA implementation using PTX assembly instructions __device__ __forceinline__ void fma(float2& d, float2 const& a, float2 const& b, float2 const& c) { diff --git a/csrc/moe/grouped_topk_kernels.cu b/csrc/libtorch_stable/moe/grouped_topk_kernels.cu similarity index 94% rename from csrc/moe/grouped_topk_kernels.cu rename to csrc/libtorch_stable/moe/grouped_topk_kernels.cu index 6a4dad3be7..a28edf3a55 100644 --- a/csrc/moe/grouped_topk_kernels.cu +++ b/csrc/libtorch_stable/moe/grouped_topk_kernels.cu @@ -18,9 +18,14 @@ * limitations under the License. */ #include "moeTopKFuncs.cuh" -#include -#include + +#include +#include + +#include "libtorch_stable/torch_utils.h" + #include +#include #include #include #include @@ -1001,38 +1006,40 @@ INSTANTIATE_NOAUX_TC(__nv_bfloat16, __nv_bfloat16, int32_t, SCORING_NONE); } // end namespace moe } // namespace vllm -std::tuple grouped_topk( - torch::Tensor const& scores, int64_t n_group, int64_t topk_group, +std::tuple grouped_topk( + torch::stable::Tensor const& scores, int64_t n_group, int64_t topk_group, int64_t topk, bool renormalize, double routed_scaling_factor, - torch::Tensor const& bias, int64_t scoring_func = 0) { - auto data_type = scores.scalar_type(); - auto bias_type = bias.scalar_type(); - auto input_size = scores.sizes(); - int64_t num_tokens = input_size[0]; - int64_t num_experts = input_size[1]; - TORCH_CHECK(input_size.size() == 2, "scores must be a 2D Tensor"); - TORCH_CHECK(n_group > 0, "n_group must be positive"); - TORCH_CHECK(topk > 0, "topk must be positive"); - TORCH_CHECK(topk_group > 0, "topk_group must be positive"); - TORCH_CHECK(topk_group <= n_group, "topk_group must be <= n_group"); - TORCH_CHECK(num_experts % n_group == 0, - "num_experts should be divisible by n_group"); - TORCH_CHECK(n_group <= 32, - "n_group should be smaller than or equal to 32 for now"); - TORCH_CHECK(topk <= 32, "topk should be smaller than or equal to 32 for now"); - TORCH_CHECK(topk <= topk_group * (num_experts / n_group), - "topk must be <= topk_group * (num_experts / n_group)"); - TORCH_CHECK(scoring_func == vllm::moe::SCORING_NONE || - scoring_func == vllm::moe::SCORING_SIGMOID, - "scoring_func must be SCORING_NONE (0) or SCORING_SIGMOID (1)"); + torch::stable::Tensor const& bias, int64_t scoring_func = 0) { + const auto data_type = scores.scalar_type(); + const auto bias_type = bias.scalar_type(); + STD_TORCH_CHECK(scores.dim() == 2, "scores must be a 2D Tensor"); + const int64_t num_tokens = scores.size(0); + const int64_t num_experts = scores.size(1); + STD_TORCH_CHECK(n_group > 0, "n_group must be positive"); + STD_TORCH_CHECK(topk > 0, "topk must be positive"); + STD_TORCH_CHECK(topk_group > 0, "topk_group must be positive"); + STD_TORCH_CHECK(topk_group <= n_group, "topk_group must be <= n_group"); + STD_TORCH_CHECK(num_experts % n_group == 0, + "num_experts should be divisible by n_group"); + STD_TORCH_CHECK(n_group <= 32, + "n_group should be smaller than or equal to 32 for now"); + STD_TORCH_CHECK(topk <= 32, + "topk should be smaller than or equal to 32 for now"); + STD_TORCH_CHECK(topk <= topk_group * (num_experts / n_group), + "topk must be <= topk_group * (num_experts / n_group)"); + STD_TORCH_CHECK( + scoring_func == vllm::moe::SCORING_NONE || + scoring_func == vllm::moe::SCORING_SIGMOID, + "scoring_func must be SCORING_NONE (0) or SCORING_SIGMOID (1)"); // Always output float32 for topk_values (eliminates Python-side conversion) - torch::Tensor topk_values = torch::empty( - {num_tokens, topk}, torch::dtype(torch::kFloat32).device(torch::kCUDA)); - torch::Tensor topk_indices = torch::empty( - {num_tokens, topk}, torch::dtype(torch::kInt32).device(torch::kCUDA)); + auto topk_values = torch::stable::new_empty( + scores, {num_tokens, topk}, torch::headeronly::ScalarType::Float); + auto topk_indices = torch::stable::new_empty( + scores, {num_tokens, topk}, torch::headeronly::ScalarType::Int); - auto stream = c10::cuda::getCurrentCUDAStream(scores.get_device()); + const cudaStream_t stream = + get_current_cuda_stream(scores.get_device_index()); auto const sf = static_cast(scoring_func); #define LAUNCH_KERNEL_SF(T, BiasT, IdxT) \ @@ -1057,7 +1064,7 @@ std::tuple grouped_topk( routed_scaling_factor, false, stream); \ break; \ default: \ - throw std::invalid_argument("Unsupported scoring_func"); \ + STD_TORCH_CHECK(false, "Unsupported scoring_func"); \ break; \ } \ } while (0) @@ -1065,17 +1072,18 @@ std::tuple grouped_topk( #define LAUNCH_KERNEL(T, IdxT) \ do { \ switch (bias_type) { \ - case torch::kFloat16: \ + case torch::headeronly::ScalarType::Half: \ LAUNCH_KERNEL_SF(T, half, IdxT); \ break; \ - case torch::kFloat32: \ + case torch::headeronly::ScalarType::Float: \ LAUNCH_KERNEL_SF(T, float, IdxT); \ break; \ - case torch::kBFloat16: \ + case torch::headeronly::ScalarType::BFloat16: \ LAUNCH_KERNEL_SF(T, __nv_bfloat16, IdxT); \ break; \ default: \ - throw std::invalid_argument( \ + STD_TORCH_CHECK( \ + false, \ "Invalid bias dtype, only supports float16, float32, and " \ "bfloat16"); \ break; \ @@ -1083,22 +1091,22 @@ std::tuple grouped_topk( } while (0) switch (data_type) { - case torch::kFloat16: + case torch::headeronly::ScalarType::Half: // Handle Float16 LAUNCH_KERNEL(half, int32_t); break; - case torch::kFloat32: + case torch::headeronly::ScalarType::Float: // Handle Float32 LAUNCH_KERNEL(float, int32_t); break; - case torch::kBFloat16: + case torch::headeronly::ScalarType::BFloat16: // Handle BFloat16 LAUNCH_KERNEL(__nv_bfloat16, int32_t); break; default: // Handle other data types - throw std::invalid_argument( - "Invalid dtype, only supports float16, float32, and bfloat16"); + STD_TORCH_CHECK( + false, "Invalid dtype, only supports float16, float32, and bfloat16"); break; } #undef LAUNCH_KERNEL diff --git a/csrc/moe/marlin_moe_wna16/.gitignore b/csrc/libtorch_stable/moe/marlin_moe_wna16/.gitignore similarity index 100% rename from csrc/moe/marlin_moe_wna16/.gitignore rename to csrc/libtorch_stable/moe/marlin_moe_wna16/.gitignore diff --git a/csrc/moe/marlin_moe_wna16/generate_kernels.py b/csrc/libtorch_stable/moe/marlin_moe_wna16/generate_kernels.py similarity index 99% rename from csrc/moe/marlin_moe_wna16/generate_kernels.py rename to csrc/libtorch_stable/moe/marlin_moe_wna16/generate_kernels.py index 6ddda1d51d..64b47b607b 100644 --- a/csrc/moe/marlin_moe_wna16/generate_kernels.py +++ b/csrc/libtorch_stable/moe/marlin_moe_wna16/generate_kernels.py @@ -302,7 +302,7 @@ def generate_new_kernels(): if not SUPPORT_FP8 and kernel_selector_str != FILE_HEAD_COMMENT: kernel_selector_str += ( "else if (a_type == vllm::kFE4M3fn)\n" - " TORCH_CHECK(false, " + " STD_TORCH_CHECK(false, " '"marlin kernel with fp8 activation is not built.");' ) diff --git a/csrc/moe/marlin_moe_wna16/kernel.h b/csrc/libtorch_stable/moe/marlin_moe_wna16/kernel.h similarity index 100% rename from csrc/moe/marlin_moe_wna16/kernel.h rename to csrc/libtorch_stable/moe/marlin_moe_wna16/kernel.h diff --git a/csrc/moe/marlin_moe_wna16/marlin_template.h b/csrc/libtorch_stable/moe/marlin_moe_wna16/marlin_template.h similarity index 100% rename from csrc/moe/marlin_moe_wna16/marlin_template.h rename to csrc/libtorch_stable/moe/marlin_moe_wna16/marlin_template.h diff --git a/csrc/moe/marlin_moe_wna16/ops.cu b/csrc/libtorch_stable/moe/marlin_moe_wna16/ops.cu similarity index 62% rename from csrc/moe/marlin_moe_wna16/ops.cu rename to csrc/libtorch_stable/moe/marlin_moe_wna16/ops.cu index 82cba2978b..177eefa2c6 100644 --- a/csrc/moe/marlin_moe_wna16/ops.cu +++ b/csrc/libtorch_stable/moe/marlin_moe_wna16/ops.cu @@ -24,7 +24,15 @@ #endif #include "kernel.h" -#include "core/registration.h" + +#include +#include +#include +#include +#include +#include + +#include "libtorch_stable/torch_utils.h" #define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \ static_assert(std::is_same::value || \ @@ -350,18 +358,18 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias, bool m_block_size_8 = moe_block_size == 8; bool is_a_8bit = a_type.size_bits() == 8; - TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m, - ", ", prob_n, ", ", prob_k, "]"); + STD_TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", + prob_m, ", ", prob_n, ", ", prob_k, "]"); int group_blocks = 0; if (has_act_order) { if (is_k_full) { - TORCH_CHECK(group_size != -1); + STD_TORCH_CHECK(group_size != -1); group_blocks = group_size / 16; - TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k, - " is not divisible by group_blocks = ", group_blocks); + STD_TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k, + " is not divisible by group_blocks = ", group_blocks); } else { - TORCH_CHECK(group_size == 0); + STD_TORCH_CHECK(group_size == 0); group_blocks = 0; } } else { @@ -369,8 +377,8 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias, group_blocks = -1; } else { group_blocks = group_size / 16; - TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k, - " is not divisible by group_blocks = ", group_blocks); + STD_TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k, + " is not divisible by group_blocks = ", group_blocks); } } @@ -407,7 +415,7 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias, else if (moe_block_size == 64) kernel = permute_cols_kernel<64>; else - TORCH_CHECK(false, "unsupported moe_block_size ", moe_block_size); + STD_TORCH_CHECK(false, "unsupported moe_block_size ", moe_block_size); // avoid ">>>" being formatted to "> > >" // clang-format off @@ -428,25 +436,25 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias, int max_shared_mem = 0; cudaDeviceGetAttribute(&max_shared_mem, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev); - TORCH_CHECK(max_shared_mem > 0); + STD_TORCH_CHECK(max_shared_mem > 0); int major_capability, minor_capability; cudaDeviceGetAttribute(&major_capability, cudaDevAttrComputeCapabilityMajor, dev); cudaDeviceGetAttribute(&minor_capability, cudaDevAttrComputeCapabilityMinor, dev); - TORCH_CHECK(major_capability * 10 + minor_capability >= 75, - "marlin kernel only support Turing or newer GPUs."); + STD_TORCH_CHECK(major_capability * 10 + minor_capability >= 75, + "marlin kernel only support Turing or newer GPUs."); int stages = 4; if (major_capability == 7 && minor_capability == 5) { stages = 2; - TORCH_CHECK(a_type == vllm::kFloat16 || a_type == vllm::kS8, - "Turing only support FP16 or INT8 activation."); + STD_TORCH_CHECK(a_type == vllm::kFloat16 || a_type == vllm::kS8, + "Turing only support FP16 or INT8 activation."); } if (a_type == vllm::kFE4M3fn) { - TORCH_CHECK(major_capability * 10 + minor_capability >= 89, - "FP8 only support Ada Lovelace or newer GPUs."); - TORCH_CHECK( + STD_TORCH_CHECK(major_capability * 10 + minor_capability >= 89, + "FP8 only support Ada Lovelace or newer GPUs."); + STD_TORCH_CHECK( major_capability * 10 + minor_capability == 89 || major_capability == 12, "Marlin W4A8-FP8 only support SM89 or SM12x device (It is slower than " @@ -460,10 +468,10 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias, thread_tfg = thread_config_t{thread_k, thread_n, thread_k * thread_n / 64}; if (blocks_per_sm == -1) blocks_per_sm = 1; exec_cfg = exec_config_t{blocks_per_sm, thread_tfg}; - TORCH_CHECK(prob_n % thread_n == 0, "prob_n = ", prob_n, - " is not divisible by thread_n = ", thread_n); - TORCH_CHECK(prob_k % thread_k == 0, "prob_k = ", prob_k, - " is not divisible by thread_k = ", thread_k); + STD_TORCH_CHECK(prob_n % thread_n == 0, "prob_n = ", prob_n, + " is not divisible by thread_n = ", thread_n); + STD_TORCH_CHECK(prob_k % thread_k == 0, "prob_k = ", prob_k, + " is not divisible by thread_k = ", thread_k); } else { // Auto config exec_cfg = determine_exec_config( @@ -484,19 +492,19 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias, int thread_k_blocks = thread_k / 16; int thread_n_blocks = thread_n / 16; - TORCH_CHECK(is_valid_config(thread_tfg, m_block_size_8, thread_m_blocks, - prob_m, prob_n, prob_k, num_bits, group_size, - has_act_order, is_k_full, has_zp, is_zp_float, - is_a_8bit, stages, max_shared_mem), - "Invalid thread config: thread_m_blocks = ", thread_m_blocks, - ", thread_k = ", thread_tfg.thread_k, - ", thread_n = ", thread_tfg.thread_n, - ", num_threads = ", thread_tfg.num_threads, " for MKN = [", - prob_m, ", ", prob_k, ", ", prob_n, "] and num_bits = ", num_bits, - ", group_size = ", group_size, - ", has_act_order = ", has_act_order, ", is_k_full = ", is_k_full, - ", has_zp = ", has_zp, ", is_zp_float = ", is_zp_float, - ", max_shared_mem = ", max_shared_mem); + STD_TORCH_CHECK( + is_valid_config(thread_tfg, m_block_size_8, thread_m_blocks, prob_m, + prob_n, prob_k, num_bits, group_size, has_act_order, + is_k_full, has_zp, is_zp_float, is_a_8bit, stages, + max_shared_mem), + "Invalid thread config: thread_m_blocks = ", thread_m_blocks, + ", thread_k = ", thread_tfg.thread_k, + ", thread_n = ", thread_tfg.thread_n, + ", num_threads = ", thread_tfg.num_threads, " for MKN = [", prob_m, ", ", + prob_k, ", ", prob_n, "] and num_bits = ", num_bits, + ", group_size = ", group_size, ", has_act_order = ", has_act_order, + ", is_k_full = ", is_k_full, ", has_zp = ", has_zp, + ", is_zp_float = ", is_zp_float, ", max_shared_mem = ", max_shared_mem); int sh_cache_size = get_kernel_cache_size(thread_tfg, m_block_size_8, thread_m_blocks, prob_m, @@ -509,13 +517,13 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias, num_threads, is_zp_float, stages); if (kernel == MarlinDefault) { - TORCH_CHECK(false, "Unsupported shapes: MNK = [", prob_m, ", ", prob_n, - ", ", prob_k, "]", ", has_act_order = ", has_act_order, - ", num_groups = ", num_groups, ", group_size = ", group_size, - ", thread_m_blocks = ", thread_m_blocks, - ", thread_n_blocks = ", thread_n_blocks, - ", thread_k_blocks = ", thread_k_blocks, - ", num_bits = ", num_bits); + STD_TORCH_CHECK( + false, "Unsupported shapes: MNK = [", prob_m, ", ", prob_n, ", ", + prob_k, "]", ", has_act_order = ", has_act_order, + ", num_groups = ", num_groups, ", group_size = ", group_size, + ", thread_m_blocks = ", thread_m_blocks, + ", thread_n_blocks = ", thread_n_blocks, + ", thread_k_blocks = ", thread_k_blocks, ", num_bits = ", num_bits); } cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, @@ -532,75 +540,81 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias, } // namespace MARLIN_NAMESPACE_NAME -torch::Tensor moe_wna16_marlin_gemm( - torch::Tensor& a, std::optional c_or_none, - torch::Tensor& b_q_weight, - std::optional const& b_bias_or_none, torch::Tensor& b_scales, - std::optional const& a_scales_or_none, - std::optional const& global_scale_or_none, - std::optional const& b_zeros_or_none, - std::optional const& g_idx_or_none, - std::optional const& perm_or_none, torch::Tensor& workspace, - torch::Tensor& sorted_token_ids, torch::Tensor& expert_ids, - torch::Tensor& num_tokens_past_padded, torch::Tensor& topk_weights, - int64_t moe_block_size, int64_t top_k, bool mul_topk_weights, - vllm::ScalarTypeId const& b_type_id, int64_t size_m, int64_t size_n, - int64_t size_k, bool is_k_full, bool use_atomic_add, bool use_fp32_reduce, - bool is_zp_float, int64_t thread_k, int64_t thread_n, +torch::stable::Tensor moe_wna16_marlin_gemm( + torch::stable::Tensor& a, std::optional c_or_none, + torch::stable::Tensor& b_q_weight, + std::optional const& b_bias_or_none, + torch::stable::Tensor& b_scales, + std::optional const& a_scales_or_none, + std::optional const& global_scale_or_none, + std::optional const& b_zeros_or_none, + std::optional const& g_idx_or_none, + std::optional const& perm_or_none, + torch::stable::Tensor& workspace, torch::stable::Tensor& sorted_token_ids, + torch::stable::Tensor& expert_ids, + torch::stable::Tensor& num_tokens_past_padded, + torch::stable::Tensor& topk_weights, int64_t moe_block_size, int64_t top_k, + bool mul_topk_weights, vllm::ScalarTypeId const& b_type_id, int64_t size_m, + int64_t size_n, int64_t size_k, bool is_k_full, bool use_atomic_add, + bool use_fp32_reduce, bool is_zp_float, int64_t thread_k, int64_t thread_n, int64_t blocks_per_sm) { vllm::ScalarTypeId a_type_id, c_type_id, s_type_id; - auto c_dtype = a.dtype(); - if (a.scalar_type() == at::ScalarType::Half) { + auto c_dtype = a.scalar_type(); + if (a.scalar_type() == torch::headeronly::ScalarType::Half) { a_type_id = vllm::kFloat16.id(); c_type_id = vllm::kFloat16.id(); - } else if (a.scalar_type() == at::ScalarType::BFloat16) { + } else if (a.scalar_type() == torch::headeronly::ScalarType::BFloat16) { a_type_id = vllm::kBFloat16.id(); c_type_id = vllm::kBFloat16.id(); } else { - c_dtype = b_scales.dtype(); - if (b_scales.scalar_type() == at::ScalarType::Half) { + c_dtype = b_scales.scalar_type(); + if (b_scales.scalar_type() == torch::headeronly::ScalarType::Half) { c_type_id = vllm::kFloat16.id(); - } else if (b_scales.scalar_type() == at::ScalarType::BFloat16) { + } else if (b_scales.scalar_type() == + torch::headeronly::ScalarType::BFloat16) { c_type_id = vllm::kBFloat16.id(); } else { c_type_id = vllm::kBFloat16.id(); - TORCH_CHECK(c_or_none.has_value(), "c must be passed for W4A8-FP4"); - torch::Tensor c = c_or_none.value(); - c_dtype = c.dtype(); + STD_TORCH_CHECK(c_or_none.has_value(), "c must be passed for W4A8-FP4"); + torch::stable::Tensor c = c_or_none.value(); + c_dtype = c.scalar_type(); - if (c.scalar_type() == at::ScalarType::Half) { + if (c.scalar_type() == torch::headeronly::ScalarType::Half) { c_type_id = vllm::kFloat16.id(); - } else if (c.scalar_type() == at::ScalarType::BFloat16) { + } else if (c.scalar_type() == torch::headeronly::ScalarType::BFloat16) { c_type_id = vllm::kBFloat16.id(); } else { - TORCH_CHECK(false, "unsupported c dtype"); + STD_TORCH_CHECK(false, "unsupported c dtype"); } } - if (a.scalar_type() == at::ScalarType::Float8_e4m3fn) { + if (a.scalar_type() == torch::headeronly::ScalarType::Float8_e4m3fn) { a_type_id = vllm::kFE4M3fn.id(); - } else if (a.scalar_type() == at::ScalarType::Char) { + } else if (a.scalar_type() == torch::headeronly::ScalarType::Char) { a_type_id = vllm::kS8.id(); } else { - TORCH_CHECK(false, "unsupported `a` scalar_type"); + STD_TORCH_CHECK(false, "unsupported `a` scalar_type"); } } s_type_id = c_type_id; if (b_type_id == vllm::kFE2M1f.id()) { - if (b_scales.scalar_type() == at::ScalarType::Float8_e4m3fn) { + if (b_scales.scalar_type() == + torch::headeronly::ScalarType::Float8_e4m3fn) { s_type_id = vllm::kFE4M3fn.id(); - } else if (b_scales.scalar_type() == at::ScalarType::Float8_e8m0fnu) { + } else if (b_scales.scalar_type() == + torch::headeronly::ScalarType::Float8_e8m0fnu) { s_type_id = vllm::kFE8M0fnu.id(); } else { - TORCH_CHECK(false, - "When b_type = float4_e2m1f, b_scale scalar type must be", - "float8_e4m3fn (for NVFP4) or float8_e8m0fnu (for MXFP4)."); + STD_TORCH_CHECK( + false, "When b_type = float4_e2m1f, b_scale scalar type must be", + "float8_e4m3fn (for NVFP4) or float8_e8m0fnu (for MXFP4)."); } } else if (b_type_id == vllm::kFE4M3fn.id() && - b_scales.scalar_type() == at::ScalarType::Float8_e8m0fnu) { + b_scales.scalar_type() == + torch::headeronly::ScalarType::Float8_e8m0fnu) { s_type_id = vllm::kFE8M0fnu.id(); } @@ -613,58 +627,60 @@ torch::Tensor moe_wna16_marlin_gemm( int num_experts = b_q_weight.size(0); if (moe_block_size != 8) { - TORCH_CHECK(moe_block_size % 16 == 0, - "unsupported moe_block_size=", moe_block_size); - TORCH_CHECK(moe_block_size >= 16 && moe_block_size <= 64, - "unsupported moe_block_size=", moe_block_size); + STD_TORCH_CHECK(moe_block_size % 16 == 0, + "unsupported moe_block_size=", moe_block_size); + STD_TORCH_CHECK(moe_block_size >= 16 && moe_block_size <= 64, + "unsupported moe_block_size=", moe_block_size); } // Verify A - TORCH_CHECK(a.size(0) == size_m, "Shape mismatch: a.size(0) = ", a.size(0), - ", size_m = ", size_m); - TORCH_CHECK(a.size(1) == size_k, "Shape mismatch: a.size(1) = ", a.size(1), - ", size_k = ", size_k); + STD_TORCH_CHECK(a.size(0) == size_m, + "Shape mismatch: a.size(0) = ", a.size(0), + ", size_m = ", size_m); + STD_TORCH_CHECK(a.size(1) == size_k, + "Shape mismatch: a.size(1) = ", a.size(1), + ", size_k = ", size_k); // Verify B - TORCH_CHECK( + STD_TORCH_CHECK( size_k % MARLIN_NAMESPACE_NAME::tile_size == 0, "size_k = ", size_k, " is not divisible by tile_size = ", MARLIN_NAMESPACE_NAME::tile_size); - TORCH_CHECK((size_k / MARLIN_NAMESPACE_NAME::tile_size) == b_q_weight.size(1), - "Shape mismatch: b_q_weight.size(1) = ", b_q_weight.size(1), - ", size_k = ", size_k, - ", tile_size = ", MARLIN_NAMESPACE_NAME::tile_size); - TORCH_CHECK( + STD_TORCH_CHECK( + (size_k / MARLIN_NAMESPACE_NAME::tile_size) == b_q_weight.size(1), + "Shape mismatch: b_q_weight.size(1) = ", b_q_weight.size(1), + ", size_k = ", size_k, + ", tile_size = ", MARLIN_NAMESPACE_NAME::tile_size); + STD_TORCH_CHECK( b_q_weight.size(2) % MARLIN_NAMESPACE_NAME::tile_size == 0, "b_q_weight.size(2) = ", b_q_weight.size(2), " is not divisible by tile_size = ", MARLIN_NAMESPACE_NAME::tile_size); int actual_size_n = (b_q_weight.size(2) / MARLIN_NAMESPACE_NAME::tile_size) * pack_factor; - TORCH_CHECK(size_n == actual_size_n, "size_n = ", size_n, - ", actual_size_n = ", actual_size_n); + STD_TORCH_CHECK(size_n == actual_size_n, "size_n = ", size_n, + ", actual_size_n = ", actual_size_n); // Verify device and strides - TORCH_CHECK(a.device().is_cuda(), "A is not on GPU"); - TORCH_CHECK(a.is_contiguous(), "A is not contiguous"); + STD_TORCH_CHECK(a.device().is_cuda(), "A is not on GPU"); + STD_TORCH_CHECK(a.is_contiguous(), "A is not contiguous"); - TORCH_CHECK(b_q_weight.device().is_cuda(), "b_q_weight is not on GPU"); - TORCH_CHECK(b_q_weight.is_contiguous(), "b_q_weight is not contiguous"); + STD_TORCH_CHECK(b_q_weight.device().is_cuda(), "b_q_weight is not on GPU"); + STD_TORCH_CHECK(b_q_weight.is_contiguous(), "b_q_weight is not contiguous"); - TORCH_CHECK(b_scales.device().is_cuda(), "b_scales is not on GPU"); - TORCH_CHECK(b_scales.is_contiguous(), "b_scales is not contiguous"); + STD_TORCH_CHECK(b_scales.device().is_cuda(), "b_scales is not on GPU"); + STD_TORCH_CHECK(b_scales.is_contiguous(), "b_scales is not contiguous"); - torch::Tensor a_scales; - auto options = torch::TensorOptions().dtype(c_dtype).device(a.device()); - auto options_fp32 = - torch::TensorOptions().dtype(at::kFloat).device(a.device()); + torch::stable::Tensor a_scales; + constexpr auto kFloat = torch::headeronly::ScalarType::Float; if (a_scales_or_none.has_value()) { a_scales = a_scales_or_none.value(); - TORCH_CHECK(a_type.size_bits() == 8, - "a_scales can only be used for 8bit activation."); + STD_TORCH_CHECK(a_type.size_bits() == 8, + "a_scales can only be used for 8bit activation."); } else { - a_scales = torch::empty({0}, options_fp32); - TORCH_CHECK(a_type.size_bits() != 8, - "the a_scales parameter must be passed for 8bit activation."); + a_scales = torch::stable::new_empty(a, {0}, kFloat); + STD_TORCH_CHECK( + a_type.size_bits() != 8, + "the a_scales parameter must be passed for 8bit activation."); } // sms: number of SMs to use for the kernel @@ -672,82 +688,84 @@ torch::Tensor moe_wna16_marlin_gemm( cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, a.get_device()); // Alloc buffers - const at::cuda::OptionalCUDAGuard device_guard(device_of(a)); - torch::Tensor c; + torch::stable::accelerator::DeviceGuard device_guard(a.get_device_index()); + torch::stable::Tensor c; if (c_or_none.has_value()) { c = c_or_none.value(); - TORCH_CHECK(c.device().is_cuda(), "c is not on GPU"); - TORCH_CHECK(c.is_contiguous(), "c is not contiguous"); - TORCH_CHECK(c.size(0) == size_m * top_k, - "Shape mismatch: c.size(0) = ", c.size(0), - ", size_m * topk = ", size_m * top_k); - TORCH_CHECK(c.size(1) == size_n, "Shape mismatch: c.size(1) = ", c.size(1), - ", size_n = ", size_n); + STD_TORCH_CHECK(c.device().is_cuda(), "c is not on GPU"); + STD_TORCH_CHECK(c.is_contiguous(), "c is not contiguous"); + STD_TORCH_CHECK(c.size(0) == size_m * top_k, + "Shape mismatch: c.size(0) = ", c.size(0), + ", size_m * topk = ", size_m * top_k); + STD_TORCH_CHECK(c.size(1) == size_n, + "Shape mismatch: c.size(1) = ", c.size(1), + ", size_n = ", size_n); } else { - c = torch::empty({size_m * top_k, size_n}, options); + c = torch::stable::new_empty(a, {size_m * top_k, size_n}, c_dtype); } // Alloc C tmp buffer that is going to be used for the global reduce - torch::Tensor c_tmp; + torch::stable::Tensor c_tmp; if (use_fp32_reduce && !use_atomic_add) { // max num of threadblocks is sms * 4 long max_c_tmp_size = min( (long)size_n * sorted_token_ids.size(0), (long)sms * 4 * moe_block_size * MARLIN_NAMESPACE_NAME::max_thread_n); if (moe_block_size == 8) max_c_tmp_size *= 2; - c_tmp = torch::empty({max_c_tmp_size}, options_fp32); + c_tmp = torch::stable::new_empty(a, {max_c_tmp_size}, kFloat); } else { - c_tmp = torch::empty({0}, options_fp32); + c_tmp = torch::stable::new_empty(a, {0}, kFloat); } // Detect groupsize and act_order int num_groups = -1; int group_size = -1; - int rank = b_scales.sizes().size(); - TORCH_CHECK(rank == 3, "b_scales rank = ", rank, " is not 3"); - TORCH_CHECK(b_scales.size(2) == size_n, "b_scales dim 2 = ", b_scales.size(2), - " is not size_n = ", size_n); + int rank = b_scales.dim(); + STD_TORCH_CHECK(rank == 3, "b_scales rank = ", rank, " is not 3"); + STD_TORCH_CHECK(b_scales.size(2) == size_n, + "b_scales dim 2 = ", b_scales.size(2), + " is not size_n = ", size_n); num_groups = b_scales.size(1); - torch::Tensor g_idx, perm, a_tmp; + torch::stable::Tensor g_idx, perm, a_tmp; if (g_idx_or_none.has_value() && perm_or_none.has_value()) { g_idx = g_idx_or_none.value(); perm = perm_or_none.value(); - TORCH_CHECK(g_idx.device().is_cuda(), "g_idx is not on GPU"); - TORCH_CHECK(g_idx.is_contiguous(), "g_idx is not contiguous"); - TORCH_CHECK(perm.device().is_cuda(), "perm is not on GPU"); - TORCH_CHECK(perm.is_contiguous(), "perm is not contiguous"); + STD_TORCH_CHECK(g_idx.device().is_cuda(), "g_idx is not on GPU"); + STD_TORCH_CHECK(g_idx.is_contiguous(), "g_idx is not contiguous"); + STD_TORCH_CHECK(perm.device().is_cuda(), "perm is not on GPU"); + STD_TORCH_CHECK(perm.is_contiguous(), "perm is not contiguous"); // Verify g_idx and perm - TORCH_CHECK((g_idx.size(-1) == 0 && perm.size(-1) == 0) || - (g_idx.size(-1) == size_k && perm.size(-1) == size_k), - "Unexpected g_idx.size(-1) = ", g_idx.size(-1), - " and perm.size(-1) = ", perm.size(-1), - ", where size_k = ", size_k); + STD_TORCH_CHECK((g_idx.size(-1) == 0 && perm.size(-1) == 0) || + (g_idx.size(-1) == size_k && perm.size(-1) == size_k), + "Unexpected g_idx.size(-1) = ", g_idx.size(-1), + " and perm.size(-1) = ", perm.size(-1), + ", where size_k = ", size_k); } else { - g_idx = torch::empty({0}, options); - perm = torch::empty({0}, options); - a_tmp = torch::empty({0}, options); + g_idx = torch::stable::new_empty(a, {0}, c_dtype); + perm = torch::stable::new_empty(a, {0}, c_dtype); + a_tmp = torch::stable::new_empty(a, {0}, c_dtype); } bool has_act_order = g_idx.size(-1) > 0 && perm.size(-1) > 0; if (has_act_order) { - a_tmp = torch::empty({size_m * top_k, size_k}, options); + a_tmp = torch::stable::new_empty(a, {size_m * top_k, size_k}, c_dtype); if (is_k_full) { - TORCH_CHECK(num_groups > 1, "For act_order, num_groups must be > 1"); - TORCH_CHECK(size_k % num_groups == 0, "size_k = ", size_k, - ", is not divisible by num_groups = ", num_groups); + STD_TORCH_CHECK(num_groups > 1, "For act_order, num_groups must be > 1"); + STD_TORCH_CHECK(size_k % num_groups == 0, "size_k = ", size_k, + ", is not divisible by num_groups = ", num_groups); group_size = size_k / num_groups; } else { group_size = 0; } } else { - a_tmp = torch::empty({0}, options); + a_tmp = torch::stable::new_empty(a, {0}, c_dtype); if (num_groups > 1) { - TORCH_CHECK( + STD_TORCH_CHECK( size_k % num_groups == 0, "size_k = ", size_k, ", is not divisible by b_scales.size(1) = ", b_scales.size(1)); group_size = size_k / num_groups; @@ -756,119 +774,125 @@ torch::Tensor moe_wna16_marlin_gemm( } } - torch::Tensor global_scale; + torch::stable::Tensor global_scale; if (global_scale_or_none.has_value()) { global_scale = global_scale_or_none.value(); - TORCH_CHECK(b_type == vllm::kFE2M1f && s_type == vllm::kFE4M3fn, - "global_scale can only be used for nvfp4 format."); + STD_TORCH_CHECK(b_type == vllm::kFE2M1f && s_type == vllm::kFE4M3fn, + "global_scale can only be used for nvfp4 format."); } else { - global_scale = torch::empty({0}, options_fp32); - TORCH_CHECK(!(b_type == vllm::kFE2M1f && s_type == vllm::kFE4M3fn), - "the global_scale parameter must be passed for nvfp4 format."); + global_scale = torch::stable::new_empty(a, {0}, kFloat); + STD_TORCH_CHECK( + !(b_type == vllm::kFE2M1f && s_type == vllm::kFE4M3fn), + "the global_scale parameter must be passed for nvfp4 format."); } bool has_bias = b_bias_or_none.has_value(); - torch::Tensor b_bias; + torch::stable::Tensor b_bias; if (has_bias) { b_bias = b_bias_or_none.value(); - TORCH_CHECK(b_bias.device().is_cuda(), "b_bias is not on GPU"); - TORCH_CHECK(b_bias.is_contiguous(), "b_bias is not contiguous"); - TORCH_CHECK(b_bias.size(1) == size_n, "b_bias.size(1) != size_n"); - TORCH_CHECK(b_bias.stride(1) == 1, "b_bias.stride(1) != 1"); + STD_TORCH_CHECK(b_bias.device().is_cuda(), "b_bias is not on GPU"); + STD_TORCH_CHECK(b_bias.is_contiguous(), "b_bias is not contiguous"); + STD_TORCH_CHECK(b_bias.size(1) == size_n, "b_bias.size(1) != size_n"); + STD_TORCH_CHECK(b_bias.stride(1) == 1, "b_bias.stride(1) != 1"); } else { - b_bias = torch::empty({0}, options); + b_bias = torch::stable::new_empty(a, {0}, c_dtype); } - torch::Tensor b_zeros; + torch::stable::Tensor b_zeros; if (b_zeros_or_none.has_value()) { b_zeros = b_zeros_or_none.value(); - TORCH_CHECK(b_zeros.device().is_cuda(), "b_zeros is not on GPU"); - TORCH_CHECK(b_zeros.is_contiguous(), "b_zeros is not contiguous"); + STD_TORCH_CHECK(b_zeros.device().is_cuda(), "b_zeros is not on GPU"); + STD_TORCH_CHECK(b_zeros.is_contiguous(), "b_zeros is not contiguous"); } else { - b_zeros = torch::empty({0}, options); + b_zeros = torch::stable::new_empty(a, {0}, c_dtype); } bool has_zp = b_zeros.size(-1) > 0; if (has_zp) { - TORCH_CHECK( + STD_TORCH_CHECK( b_type == vllm::kU4 || b_type == vllm::kU8, "b_type must be u4 or u8 when has_zp = True. Got = ", b_type.str()); } else { - TORCH_CHECK(b_type == vllm::kU4B8 || b_type == vllm::kU8B128 || - b_type == vllm::kS4 || b_type == vllm::kS8 || - b_type == vllm::kFE4M3fn || b_type == vllm::kFE2M1f, - "b_type must be uint4b8, uint8b128, int4, int8, " - "float8_e4m3fn or float4_e2m1f when has_zp = False. Got = ", - b_type.str()); + STD_TORCH_CHECK(b_type == vllm::kU4B8 || b_type == vllm::kU8B128 || + b_type == vllm::kS4 || b_type == vllm::kS8 || + b_type == vllm::kFE4M3fn || b_type == vllm::kFE2M1f, + "b_type must be uint4b8, uint8b128, int4, int8, " + "float8_e4m3fn or float4_e2m1f when has_zp = False. Got = ", + b_type.str()); } if (has_zp && is_zp_float) { - TORCH_CHECK(a.scalar_type() == at::ScalarType::Half, - "Computation type must be float16 (half) when using float zero " - "points."); + STD_TORCH_CHECK( + a.scalar_type() == torch::headeronly::ScalarType::Half, + "Computation type must be float16 (half) when using float zero " + "points."); } // Verify b_zeros if (has_zp) { - int rank = b_zeros.sizes().size(); - TORCH_CHECK(rank == 3, "b_zeros rank = ", rank, " is not 3"); + int rank = b_zeros.dim(); + STD_TORCH_CHECK(rank == 3, "b_zeros rank = ", rank, " is not 3"); if (is_zp_float) { - TORCH_CHECK(b_zeros.size(2) == size_n, - "b_zeros dim 2 = ", b_zeros.size(2), - " is not size_n = ", size_n); - TORCH_CHECK(num_groups == b_zeros.size(1), - "b_zeros dim 1 = ", b_zeros.size(1), - " is not num_groups = ", num_groups); - TORCH_CHECK(num_groups != -1, "num_groups must be != -1"); + STD_TORCH_CHECK(b_zeros.size(2) == size_n, + "b_zeros dim 2 = ", b_zeros.size(2), + " is not size_n = ", size_n); + STD_TORCH_CHECK(num_groups == b_zeros.size(1), + "b_zeros dim 1 = ", b_zeros.size(1), + " is not num_groups = ", num_groups); + STD_TORCH_CHECK(num_groups != -1, "num_groups must be != -1"); } else { - TORCH_CHECK(b_zeros.size(1) == num_groups, - "b_zeros dim 1 = ", b_zeros.size(1), - " is not num_groups = ", num_groups); - TORCH_CHECK(b_zeros.size(2) == size_n / pack_factor, - "b_zeros dim 2 = ", b_zeros.size(2), - " is not size_n / pack_factor = ", size_n / pack_factor); + STD_TORCH_CHECK(b_zeros.size(1) == num_groups, + "b_zeros dim 1 = ", b_zeros.size(1), + " is not num_groups = ", num_groups); + STD_TORCH_CHECK(b_zeros.size(2) == size_n / pack_factor, + "b_zeros dim 2 = ", b_zeros.size(2), + " is not size_n / pack_factor = ", size_n / pack_factor); } } // Verify workspace size - TORCH_CHECK(size_n % MARLIN_NAMESPACE_NAME::min_thread_n == 0, - "size_n = ", size_n, ", is not divisible by min_thread_n = ", - MARLIN_NAMESPACE_NAME::min_thread_n); + STD_TORCH_CHECK(size_n % MARLIN_NAMESPACE_NAME::min_thread_n == 0, + "size_n = ", size_n, ", is not divisible by min_thread_n = ", + MARLIN_NAMESPACE_NAME::min_thread_n); int max_n_tiles = size_n / MARLIN_NAMESPACE_NAME::min_thread_n; int min_workspace_size = min( max_n_tiles * (int)(sorted_token_ids.size(0) / moe_block_size), sms * 4); - TORCH_CHECK(workspace.numel() >= min_workspace_size, - "workspace.numel = ", workspace.numel(), - " is below min_workspace_size = ", min_workspace_size); + STD_TORCH_CHECK(workspace.numel() >= min_workspace_size, + "workspace.numel = ", workspace.numel(), + " is below min_workspace_size = ", min_workspace_size); int dev = a.get_device(); - TORCH_CHECK(a_scales.scalar_type() == at::ScalarType::Float, - "scalar type of a_scales must be float"); - TORCH_CHECK(global_scale.scalar_type() == at::ScalarType::Float, - "scalar type of global_scale must be float"); + STD_TORCH_CHECK( + a_scales.scalar_type() == torch::headeronly::ScalarType::Float, + "scalar type of a_scales must be float"); + STD_TORCH_CHECK( + global_scale.scalar_type() == torch::headeronly::ScalarType::Float, + "scalar type of global_scale must be float"); if (a_type.size_bits() == 16) { - TORCH_CHECK( + STD_TORCH_CHECK( a.scalar_type() == c.scalar_type(), "scalar type of a must be the same with c for 16 bit activation"); } MARLIN_NAMESPACE_NAME::marlin_mm( - a.data_ptr(), b_q_weight.data_ptr(), c.data_ptr(), c_tmp.data_ptr(), - b_bias.data_ptr(), a_scales.data_ptr(), b_scales.data_ptr(), - global_scale.data_ptr(), b_zeros.data_ptr(), g_idx.data_ptr(), - perm.data_ptr(), a_tmp.data_ptr(), sorted_token_ids.data_ptr(), - expert_ids.data_ptr(), num_tokens_past_padded.data_ptr(), - topk_weights.data_ptr(), moe_block_size, num_experts, top_k, - mul_topk_weights, size_m, size_n, size_k, workspace.data_ptr(), a_type, - b_type, c_type, s_type, has_bias, has_act_order, is_k_full, has_zp, - num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev), + a.const_data_ptr(), b_q_weight.const_data_ptr(), c.mutable_data_ptr(), + c_tmp.mutable_data_ptr(), b_bias.mutable_data_ptr(), + a_scales.mutable_data_ptr(), b_scales.mutable_data_ptr(), + global_scale.mutable_data_ptr(), b_zeros.mutable_data_ptr(), + g_idx.mutable_data_ptr(), perm.mutable_data_ptr(), + a_tmp.mutable_data_ptr(), sorted_token_ids.mutable_data_ptr(), + expert_ids.mutable_data_ptr(), num_tokens_past_padded.mutable_data_ptr(), + topk_weights.mutable_data_ptr(), moe_block_size, num_experts, top_k, + mul_topk_weights, size_m, size_n, size_k, workspace.mutable_data_ptr(), + a_type, b_type, c_type, s_type, has_bias, has_act_order, is_k_full, + has_zp, num_groups, group_size, dev, get_current_cuda_stream(dev), thread_k, thread_n, sms, blocks_per_sm, use_atomic_add, use_fp32_reduce, is_zp_float); return c; } -TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) { - m.impl("moe_wna16_marlin_gemm", &moe_wna16_marlin_gemm); +STABLE_TORCH_LIBRARY_IMPL(_moe_C, CUDA, m) { + m.impl("moe_wna16_marlin_gemm", TORCH_BOX(&moe_wna16_marlin_gemm)); } diff --git a/csrc/moe/moeTopKFuncs.cuh b/csrc/libtorch_stable/moe/moeTopKFuncs.cuh similarity index 100% rename from csrc/moe/moeTopKFuncs.cuh rename to csrc/libtorch_stable/moe/moeTopKFuncs.cuh diff --git a/csrc/moe/moe_align_sum_kernels.cu b/csrc/libtorch_stable/moe/moe_align_sum_kernels.cu similarity index 74% rename from csrc/moe/moe_align_sum_kernels.cu rename to csrc/libtorch_stable/moe/moe_align_sum_kernels.cu index a8fa59b193..d7c68ff25a 100644 --- a/csrc/moe/moe_align_sum_kernels.cu +++ b/csrc/libtorch_stable/moe/moe_align_sum_kernels.cu @@ -1,14 +1,17 @@ -#include -#include -#include +#include #include -#include -#include +#include +#include +#include +#include +#include +#include -#include "../cuda_compat.h" -#include "../dispatch_utils.h" +#include "../../cuda_compat.h" #include "core/math.hpp" +#include "libtorch_stable/dispatch_utils.h" +#include "libtorch_stable/torch_utils.h" #define CEILDIV(x, y) (((x) + (y) - 1) / (y)) @@ -492,12 +495,13 @@ __global__ void moe_lora_align_block_size_small_batch_expert_kernel( // taken from // https://github.com/sgl-project/sglang/blob/8b5f83ed3b7d2a49ad5c5cd5aa61c5d502f47dbc -void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts, - int64_t block_size, torch::Tensor sorted_token_ids, - torch::Tensor experts_ids, - torch::Tensor num_tokens_post_pad, - std::optional maybe_expert_map) { - const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); +void moe_align_block_size( + torch::stable::Tensor topk_ids, int64_t num_experts, int64_t block_size, + torch::stable::Tensor sorted_token_ids, torch::stable::Tensor experts_ids, + torch::stable::Tensor num_tokens_post_pad, + std::optional maybe_expert_map) { + const cudaStream_t stream = + get_current_cuda_stream(topk_ids.get_device_index()); int64_t padded_num_experts = ((num_experts + WARP_SIZE - 1) / WARP_SIZE) * WARP_SIZE; @@ -506,19 +510,18 @@ void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts, threads = ((threads + WARP_SIZE - 1) / WARP_SIZE) * WARP_SIZE; // BlockScan uses 1024 threads and assigns one thread per expert. - TORCH_CHECK(padded_num_experts < 1024, - "padded_num_experts must be less than 1024"); - auto options_int = - torch::TensorOptions().dtype(torch::kInt).device(topk_ids.device()); + STD_TORCH_CHECK(padded_num_experts < 1024, + "padded_num_experts must be less than 1024"); bool has_expert_map = maybe_expert_map.has_value(); - torch::Tensor expert_map; + torch::stable::Tensor expert_map; if (has_expert_map) { expert_map = maybe_expert_map.value(); } else { - expert_map = torch::empty({0}, options_int); + expert_map = torch::stable::new_empty(topk_ids, {0}, + torch::headeronly::ScalarType::Int); } - VLLM_DISPATCH_INTEGRAL_AND_UNSIGNED_TYPES( + VLLM_STABLE_DISPATCH_INTEGRAL_AND_UNSIGNED_TYPES( topk_ids.scalar_type(), "moe_align_block_size_kernel", [&] { // calc needed amount of shared mem for `cumsum` tensors bool small_batch_expert_mode = @@ -538,16 +541,17 @@ void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts, scalar_t, fill_threads>; small_batch_expert_kernel<<<1, fill_threads + threads, shared_mem_size, stream>>>( - topk_ids.data_ptr(), - sorted_token_ids.data_ptr(), - experts_ids.data_ptr(), - num_tokens_post_pad.data_ptr(), - expert_map.data_ptr(), num_experts, block_size, - topk_ids.numel(), sorted_token_ids.size(0), topk_ids.size(1), - has_expert_map); + reinterpret_cast(topk_ids.const_data_ptr()), + reinterpret_cast(sorted_token_ids.mutable_data_ptr()), + reinterpret_cast(experts_ids.mutable_data_ptr()), + reinterpret_cast( + num_tokens_post_pad.mutable_data_ptr()), + reinterpret_cast(expert_map.mutable_data_ptr()), + num_experts, block_size, topk_ids.numel(), + sorted_token_ids.size(0), topk_ids.size(1), has_expert_map); } else { - torch::Tensor cumsum_buffer = - torch::empty({num_experts + 1}, options_int); + torch::stable::Tensor cumsum_buffer = torch::stable::new_empty( + topk_ids, {num_experts + 1}, torch::headeronly::ScalarType::Int); auto align_kernel = vllm::moe::moe_align_block_size_kernel; size_t num_warps = CEILDIV(padded_num_experts, experts_per_warp); @@ -558,14 +562,16 @@ void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts, // blockIdx.x == 0: counting experts and aligning // blockIdx.x == 1: filling sorted_token_ids align_kernel<<<2, threads, shared_mem_size, stream>>>( - topk_ids.data_ptr(), - sorted_token_ids.data_ptr(), - experts_ids.data_ptr(), - num_tokens_post_pad.data_ptr(), - expert_map.data_ptr(), num_experts, padded_num_experts, - experts_per_warp, block_size, topk_ids.numel(), - cumsum_buffer.data_ptr(), sorted_token_ids.size(0), - topk_ids.size(1), has_expert_map); + reinterpret_cast(topk_ids.const_data_ptr()), + reinterpret_cast(sorted_token_ids.mutable_data_ptr()), + reinterpret_cast(experts_ids.mutable_data_ptr()), + reinterpret_cast( + num_tokens_post_pad.mutable_data_ptr()), + reinterpret_cast(expert_map.mutable_data_ptr()), + num_experts, padded_num_experts, experts_per_warp, block_size, + topk_ids.numel(), + reinterpret_cast(cumsum_buffer.mutable_data_ptr()), + sorted_token_ids.size(0), topk_ids.size(1), has_expert_map); const int block_threads = std::min(256, (int)threads); const int num_blocks = @@ -577,9 +583,10 @@ void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts, auto sort_kernel = vllm::moe::count_and_sort_expert_tokens_kernel; sort_kernel<<>>( - topk_ids.data_ptr(), - sorted_token_ids.data_ptr(), - cumsum_buffer.data_ptr(), expert_map.data_ptr(), + reinterpret_cast(topk_ids.const_data_ptr()), + reinterpret_cast(sorted_token_ids.mutable_data_ptr()), + reinterpret_cast(cumsum_buffer.mutable_data_ptr()), + reinterpret_cast(expert_map.mutable_data_ptr()), topk_ids.numel(), num_experts, sorted_token_ids.size(0), topk_ids.size(1), has_expert_map); } @@ -588,33 +595,36 @@ void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts, void batched_moe_align_block_size(int64_t max_tokens_per_batch, int64_t block_size, - torch::Tensor const& batch_num_tokens, - torch::Tensor sorted_ids, - torch::Tensor batch_ids, - torch::Tensor num_tokens_post_pad) { + const torch::stable::Tensor& batch_num_tokens, + torch::stable::Tensor sorted_ids, + torch::stable::Tensor batch_ids, + torch::stable::Tensor num_tokens_post_pad) { namespace batched_kernel = vllm::moe::batched_moe_align_block_size; - const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + const cudaStream_t stream = + get_current_cuda_stream(batch_num_tokens.get_device_index()); int32_t const B = batch_num_tokens.size(0); int32_t const num_blocks_per_batch = round_to_next_multiple_of(max_tokens_per_batch, block_size) / block_size; int32_t const num_blocks = num_blocks_per_batch * B; int64_t const sorted_ids_size = num_blocks * block_size; - TORCH_CHECK(sorted_ids.size(0) == sorted_ids_size); - TORCH_CHECK(batch_ids.size(0) == sorted_ids_size / block_size); - TORCH_CHECK(num_tokens_post_pad.size(0) == 1); - TORCH_CHECK(B <= batched_kernel::num_threads); + STD_TORCH_CHECK(sorted_ids.size(0) == sorted_ids_size); + STD_TORCH_CHECK(batch_ids.size(0) == sorted_ids_size / block_size); + STD_TORCH_CHECK(num_tokens_post_pad.size(0) == 1); + STD_TORCH_CHECK(B <= batched_kernel::num_threads); batched_kernel::batched_moe_align_block_size_kernel<<< batched_kernel::num_blocks, batched_kernel::num_threads, 0, stream>>>( - B, max_tokens_per_batch, block_size, batch_num_tokens.data_ptr(), - sorted_ids.data_ptr(), batch_ids.data_ptr(), - num_tokens_post_pad.data_ptr()); + B, max_tokens_per_batch, block_size, + reinterpret_cast(batch_num_tokens.const_data_ptr()), + reinterpret_cast(sorted_ids.mutable_data_ptr()), + reinterpret_cast(batch_ids.mutable_data_ptr()), + reinterpret_cast(num_tokens_post_pad.mutable_data_ptr())); } -void moe_sum(torch::Tensor& input, // [num_tokens, topk, hidden_size] - torch::Tensor& output) // [num_tokens, hidden_size] +void moe_sum(torch::stable::Tensor& input, // [num_tokens, topk, hidden_size] + torch::stable::Tensor& output) // [num_tokens, hidden_size] { const int hidden_size = input.size(-1); const auto num_tokens = output.numel() / hidden_size; @@ -622,77 +632,86 @@ void moe_sum(torch::Tensor& input, // [num_tokens, topk, hidden_size] dim3 grid(num_tokens); dim3 block(std::min(hidden_size, 1024)); - const at::cuda::OptionalCUDAGuard device_guard(device_of(output)); - const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + const torch::stable::accelerator::DeviceGuard device_guard( + output.get_device_index()); + const cudaStream_t stream = + get_current_cuda_stream(output.get_device_index()); switch (topk) { case 2: - VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "moe_sum_kernel", [&] { - vllm::moe::moe_sum_kernel<<>>( - output.data_ptr(), input.data_ptr(), - hidden_size); - }); + VLLM_STABLE_DISPATCH_FLOATING_TYPES( + input.scalar_type(), "moe_sum_kernel", [&] { + vllm::moe::moe_sum_kernel<<>>( + reinterpret_cast(output.mutable_data_ptr()), + reinterpret_cast(input.const_data_ptr()), + hidden_size); + }); break; case 3: - VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "moe_sum_kernel", [&] { - vllm::moe::moe_sum_kernel<<>>( - output.data_ptr(), input.data_ptr(), - hidden_size); - }); + VLLM_STABLE_DISPATCH_FLOATING_TYPES( + input.scalar_type(), "moe_sum_kernel", [&] { + vllm::moe::moe_sum_kernel<<>>( + reinterpret_cast(output.mutable_data_ptr()), + reinterpret_cast(input.const_data_ptr()), + hidden_size); + }); break; case 4: - VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "moe_sum_kernel", [&] { - vllm::moe::moe_sum_kernel<<>>( - output.data_ptr(), input.data_ptr(), - hidden_size); - }); + VLLM_STABLE_DISPATCH_FLOATING_TYPES( + input.scalar_type(), "moe_sum_kernel", [&] { + vllm::moe::moe_sum_kernel<<>>( + reinterpret_cast(output.mutable_data_ptr()), + reinterpret_cast(input.const_data_ptr()), + hidden_size); + }); break; default: - at::sum_out(output, input, 1); + torch::stable::sum_out(output, input, std::array{1}); break; } } void moe_lora_align_block_size( - torch::Tensor topk_ids, torch::Tensor token_lora_mapping, + torch::stable::Tensor topk_ids, torch::stable::Tensor token_lora_mapping, int64_t num_experts, int64_t block_size, int64_t max_loras, int64_t max_num_tokens_padded, int64_t max_num_m_blocks, - torch::Tensor sorted_token_ids, torch::Tensor expert_ids, - torch::Tensor num_tokens_post_pad, torch::Tensor adapter_enabled, - torch::Tensor lora_ids, std::optional maybe_expert_map) { + torch::stable::Tensor sorted_token_ids, torch::stable::Tensor expert_ids, + torch::stable::Tensor num_tokens_post_pad, + torch::stable::Tensor adapter_enabled, torch::stable::Tensor lora_ids, + std::optional maybe_expert_map) { const int topk_num = topk_ids.size(1); - TORCH_CHECK(block_size > 0, "block_size should be greater than 0. "); + STD_TORCH_CHECK(block_size > 0, "block_size should be greater than 0. "); int device_max_shared_mem; - auto dev = topk_ids.get_device(); + int dev = topk_ids.get_device_index(); cudaDeviceGetAttribute(&device_max_shared_mem, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev); - const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + const cudaStream_t stream = get_current_cuda_stream(dev); int64_t padded_num_experts = ((num_experts + WARP_SIZE - 1) / WARP_SIZE) * WARP_SIZE; // BlockScan uses 1024 threads and assigns one thread per expert. - TORCH_CHECK(padded_num_experts < 1024, - "padded_num_experts must be less than 1024"); + STD_TORCH_CHECK(padded_num_experts < 1024, + "padded_num_experts must be less than 1024"); - auto options_int = - torch::TensorOptions().dtype(torch::kInt).device(topk_ids.device()); - torch::Tensor token_mask = - torch::empty({max_loras * topk_ids.size(0)}, options_int); + torch::stable::Tensor token_mask = + torch::stable::new_empty(topk_ids, {max_loras * topk_ids.size(0)}, + torch::headeronly::ScalarType::Int); bool has_expert_map = maybe_expert_map.has_value(); - torch::Tensor expert_map; + torch::stable::Tensor expert_map; if (has_expert_map) { expert_map = maybe_expert_map.value(); } else { - expert_map = torch::empty({0}, options_int); + expert_map = torch::stable::new_empty(topk_ids, {0}, + torch::headeronly::ScalarType::Int); } - VLLM_DISPATCH_INTEGRAL_TYPES( + VLLM_STABLE_DISPATCH_INTEGRAL_TYPES( topk_ids.scalar_type(), "moe_lora_align_sum_kernel", [&] { bool small_batch_expert_mode = (topk_ids.numel() < 1024) && (num_experts <= 64); @@ -703,7 +722,7 @@ void moe_lora_align_block_size( (num_thread + 1) * num_experts * sizeof(int32_t) + (num_experts + 1) * sizeof(int32_t); if (shared_mem > device_max_shared_mem) { - TORCH_CHECK(false, "Shared memory usage exceeds device limit."); + STD_TORCH_CHECK(false, "Shared memory usage exceeds device limit."); } // threadIdx.x >= fill_threads: counting experts and aligning @@ -714,7 +733,7 @@ void moe_lora_align_block_size( auto kernel = vllm::moe::moe_lora_align_block_size_small_batch_expert_kernel< scalar_t, fill_threads>; - AT_CUDA_CHECK(VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize( + STD_CUDA_CHECK(VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize( (void*)kernel, shared_mem)); // Grid size is (max_loras + 1) because active_lora_ids has length // max_loras + 1: sorted-unique values of token_lora_mapping, which @@ -725,15 +744,21 @@ void moe_lora_align_block_size( // MoE-LoRA kernels. This mirrors the fix made for the Triton // _fused_moe_lora_kernel grid in vllm-project/vllm#32277. kernel<<>>( - topk_ids.data_ptr(), - token_lora_mapping.data_ptr(), block_size, - expert_map.data_ptr(), num_experts, max_loras, - topk_ids.numel(), max_num_tokens_padded, max_num_m_blocks, - sorted_token_ids.data_ptr(), - expert_ids.data_ptr(), topk_num, - num_tokens_post_pad.data_ptr(), - adapter_enabled.data_ptr(), lora_ids.data_ptr(), - token_mask.data_ptr(), has_expert_map); + reinterpret_cast(topk_ids.mutable_data_ptr()), + reinterpret_cast(token_lora_mapping.mutable_data_ptr()), + block_size, + reinterpret_cast(expert_map.mutable_data_ptr()), + num_experts, max_loras, topk_ids.numel(), max_num_tokens_padded, + max_num_m_blocks, + reinterpret_cast(sorted_token_ids.mutable_data_ptr()), + reinterpret_cast(expert_ids.mutable_data_ptr()), + topk_num, + reinterpret_cast( + num_tokens_post_pad.mutable_data_ptr()), + reinterpret_cast(adapter_enabled.mutable_data_ptr()), + reinterpret_cast(lora_ids.mutable_data_ptr()), + reinterpret_cast(token_mask.mutable_data_ptr()), + has_expert_map); } else { int num_thread = 1024; dim3 blockDim(num_thread); @@ -742,8 +767,9 @@ void moe_lora_align_block_size( size_t shared_mem_size = num_warps * WARP_SIZE * sizeof(int32_t); // cumsum buffer - torch::Tensor cumsum = - torch::zeros({max_loras * (num_experts + 1)}, options_int); + torch::stable::Tensor cumsum = torch::stable::new_zeros( + topk_ids, {max_loras * (num_experts + 1)}, + torch::headeronly::ScalarType::Int); auto align_kernel = vllm::moe::moe_lora_align_block_size_kernel; @@ -759,16 +785,23 @@ void moe_lora_align_block_size( // blockIdx.x % 2 == 1: filling sorted_token_ids align_kernel<<<(max_loras + 1) * 2, blockDim, shared_mem_size, stream>>>( - topk_ids.data_ptr(), - token_lora_mapping.data_ptr(), block_size, - expert_map.data_ptr(), num_experts, max_loras, - topk_ids.numel(), max_num_tokens_padded, max_num_m_blocks, - sorted_token_ids.data_ptr(), - expert_ids.data_ptr(), topk_num, - num_tokens_post_pad.data_ptr(), - adapter_enabled.data_ptr(), cumsum.data_ptr(), - WARP_SIZE, padded_num_experts, lora_ids.data_ptr(), - token_mask.data_ptr(), has_expert_map); + reinterpret_cast(topk_ids.mutable_data_ptr()), + reinterpret_cast(token_lora_mapping.mutable_data_ptr()), + block_size, + reinterpret_cast(expert_map.mutable_data_ptr()), + num_experts, max_loras, topk_ids.numel(), max_num_tokens_padded, + max_num_m_blocks, + reinterpret_cast(sorted_token_ids.mutable_data_ptr()), + reinterpret_cast(expert_ids.mutable_data_ptr()), + topk_num, + reinterpret_cast( + num_tokens_post_pad.mutable_data_ptr()), + reinterpret_cast(adapter_enabled.mutable_data_ptr()), + reinterpret_cast(cumsum.mutable_data_ptr()), WARP_SIZE, + padded_num_experts, + reinterpret_cast(lora_ids.mutable_data_ptr()), + reinterpret_cast(token_mask.mutable_data_ptr()), + has_expert_map); const int block_threads = std::min(256, (int)num_thread); const int num_blocks = @@ -785,12 +818,16 @@ void moe_lora_align_block_size( vllm::moe::lora_count_and_sort_expert_tokens_kernel; sort_kernel<<>>( - topk_ids.data_ptr(), - sorted_token_ids.data_ptr(), cumsum.data_ptr(), - expert_map.data_ptr(), topk_ids.numel(), num_experts, - max_num_tokens_padded, topk_num, token_mask.data_ptr(), - max_loras, lora_ids.data_ptr(), - adapter_enabled.data_ptr(), has_expert_map); + reinterpret_cast(topk_ids.const_data_ptr()), + reinterpret_cast(sorted_token_ids.mutable_data_ptr()), + reinterpret_cast(cumsum.mutable_data_ptr()), + reinterpret_cast(expert_map.mutable_data_ptr()), + topk_ids.numel(), num_experts, max_num_tokens_padded, topk_num, + reinterpret_cast(token_mask.mutable_data_ptr()), + max_loras, + reinterpret_cast(lora_ids.mutable_data_ptr()), + reinterpret_cast(adapter_enabled.mutable_data_ptr()), + has_expert_map); } }); } \ No newline at end of file diff --git a/csrc/libtorch_stable/moe/moe_ops.h b/csrc/libtorch_stable/moe/moe_ops.h new file mode 100644 index 0000000000..43cbb7f86d --- /dev/null +++ b/csrc/libtorch_stable/moe/moe_ops.h @@ -0,0 +1,87 @@ +#pragma once + +#include + +#include +#include + +void topk_softmax(torch::stable::Tensor& topk_weights, + torch::stable::Tensor& topk_indices, + torch::stable::Tensor& token_expert_indices, + torch::stable::Tensor& gating_output, bool renormalize, + std::optional bias); + +void topk_sigmoid(torch::stable::Tensor& topk_weights, + torch::stable::Tensor& topk_indices, + torch::stable::Tensor& token_expert_indices, + torch::stable::Tensor& gating_output, bool renormalize, + std::optional bias); + +void topk_softplus_sqrt( + torch::stable::Tensor& topk_weights, torch::stable::Tensor& topk_indices, + torch::stable::Tensor& token_expert_indices, + torch::stable::Tensor& gating_output, bool renormalize, + double routed_scaling_factor, + const std::optional& correction_bias, + const std::optional& input_ids, + const std::optional& tid2eid); + +void moe_sum(torch::stable::Tensor& input, torch::stable::Tensor& output); + +void moe_align_block_size( + torch::stable::Tensor topk_ids, int64_t num_experts, int64_t block_size, + torch::stable::Tensor sorted_token_ids, torch::stable::Tensor experts_ids, + torch::stable::Tensor num_tokens_post_pad, + std::optional maybe_expert_map); + +void batched_moe_align_block_size( + int64_t max_tokens_per_batch, int64_t block_size, + const torch::stable::Tensor& expert_num_tokens, + torch::stable::Tensor sorted_ids, torch::stable::Tensor expert_ids, + torch::stable::Tensor num_tokens_post_pad); + +void moe_lora_align_block_size( + torch::stable::Tensor topk_ids, torch::stable::Tensor token_lora_mapping, + int64_t num_experts, int64_t block_size, int64_t max_loras, + int64_t max_num_tokens_padded, int64_t max_num_m_blocks, + torch::stable::Tensor sorted_token_ids, torch::stable::Tensor expert_ids, + torch::stable::Tensor num_tokens_post_pad, + torch::stable::Tensor adapter_enabled, torch::stable::Tensor lora_ids, + std::optional maybe_expert_map); +#ifndef USE_ROCM +torch::stable::Tensor moe_wna16_gemm( + torch::stable::Tensor input, torch::stable::Tensor output, + torch::stable::Tensor b_qweight, torch::stable::Tensor b_scales, + std::optional b_qzeros, + std::optional topk_weights, + torch::stable::Tensor sorted_token_ids, torch::stable::Tensor expert_ids, + torch::stable::Tensor num_tokens_post_pad, int64_t top_k, + int64_t BLOCK_SIZE_M, int64_t BLOCK_SIZE_N, int64_t BLOCK_SIZE_K, + int64_t bit); + +std::tuple grouped_topk( + const torch::stable::Tensor& scores, int64_t n_group, int64_t topk_group, + int64_t topk, bool renormalize, double routed_scaling_factor, + const torch::stable::Tensor& bias, int64_t scoring_func); +#endif + +bool moe_permute_unpermute_supported(); + +int64_t moe_permute_sort_workspace_size(int64_t num_expanded_rows, + int64_t num_expert); + +void shuffle_rows(const torch::stable::Tensor& input_tensor, + const torch::stable::Tensor& dst2src_map, + torch::stable::Tensor& output_tensor); + +#ifndef USE_ROCM +// DeepSeek V3 optimized router GEMM kernel for SM90+ +// Computes output = mat_a @ mat_b.T where: +// mat_a: [num_tokens, hidden_dim] in bf16 +// mat_b: [num_experts, hidden_dim] in bf16 +// output: [num_tokens, num_experts] in bf16 or fp32 +// Supports num_tokens in [1, 16], num_experts in {256, 384}, hidden_dim = 7168 +void dsv3_router_gemm(torch::stable::Tensor& output, + const torch::stable::Tensor& mat_a, + const torch::stable::Tensor& mat_b); +#endif diff --git a/csrc/libtorch_stable/moe/moe_permute_unpermute_op.cu b/csrc/libtorch_stable/moe/moe_permute_unpermute_op.cu new file mode 100644 index 0000000000..b688265eaa --- /dev/null +++ b/csrc/libtorch_stable/moe/moe_permute_unpermute_op.cu @@ -0,0 +1,319 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "core/registration.h" +#include "libtorch_stable/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.h" +#include "libtorch_stable/torch_utils.h" + +#include + +// moe_permute kernels require at least CUDA 12.0 +#if defined(CUDA_VERSION) && (CUDA_VERSION >= 12000) + +namespace { + +int64_t product_integers(torch::headeronly::IntHeaderOnlyArrayRef sizes) { + int64_t numel = 1; + for (int64_t s : sizes) { + numel *= s; + } + return numel; +} + +torch::stable::Tensor maybe_allocate_tensor( + const std::optional& maybe_tensor, + torch::headeronly::IntHeaderOnlyArrayRef expected_sizes, + torch::headeronly::ScalarType dtype, torch::stable::Device device, + char const* name) { + auto expected_numel = product_integers(expected_sizes); + if (maybe_tensor.has_value()) { + auto tensor = maybe_tensor.value(); + STD_TORCH_CHECK(tensor.device() == device, name, + " must be on the same device"); + STD_TORCH_CHECK(tensor.scalar_type() == dtype, name, + " has incorrect dtype"); + STD_TORCH_CHECK(tensor.is_contiguous(), name, " must be contiguous"); + STD_TORCH_CHECK(tensor.numel() >= expected_numel, name, + " is too small for the requested shape"); + auto flat_tensor = torch::stable::view(tensor, {tensor.numel()}); + return torch::stable::view( + torch::stable::narrow(flat_tensor, 0, 0, expected_numel), + expected_sizes); + } + return torch::stable::empty(expected_sizes, dtype, std::nullopt, device); +} + +} // namespace + +int64_t moe_permute_sort_workspace_size(int64_t num_expanded_rows, + int64_t n_expert) { + return static_cast( + CubKeyValueSorter::getWorkspaceSize(num_expanded_rows, n_expert)); +} + +void moe_permute_impl( + const torch::stable::Tensor& input, // [n_token, hidden] + const torch::stable::Tensor& topk_ids, // [n_token, topk] + const torch::stable::Tensor& token_expert_indices, // [n_token, topk] + const std::optional& expert_map, // [n_expert] + int64_t n_expert, int64_t n_local_expert, int64_t topk, + torch::stable::Tensor& permuted_input, // [permuted_size, hidden] + torch::stable::Tensor& expert_first_token_offset, // [n_local_expert + 1] + torch::stable::Tensor& inv_permuted_idx, // [n_token, topk] + torch::stable::Tensor& permuted_idx, // [permute_size] + const std::optional& maybe_sort_workspace, + const std::optional& maybe_permuted_experts_id, + const std::optional& maybe_sorted_row_idx, + const std::optional& maybe_topk_ids_for_sort) { + STD_TORCH_CHECK(expert_first_token_offset.scalar_type() == + torch::headeronly::ScalarType::Long, + "expert_first_token_offset must be int64"); + STD_TORCH_CHECK(topk_ids.scalar_type() == torch::headeronly::ScalarType::Int, + "topk_ids must be int32"); + STD_TORCH_CHECK( + token_expert_indices.scalar_type() == torch::headeronly::ScalarType::Int, + "token_expert_indices must be int32"); + STD_TORCH_CHECK( + inv_permuted_idx.scalar_type() == torch::headeronly::ScalarType::Int, + "inv_permuted_idx must be int32"); + STD_TORCH_CHECK(expert_first_token_offset.size(0) == n_local_expert + 1, + "expert_first_token_offset shape != n_local_expert+1"); + STD_TORCH_CHECK( + inv_permuted_idx.sizes().equals(token_expert_indices.sizes()), + "token_expert_indices shape must be same as inv_permuted_idx"); + + auto device = input.device(); + auto n_token = input.sizes()[0]; + auto n_hidden = input.sizes()[1]; + auto expanded_rows = n_token * topk; + auto stream = get_current_cuda_stream(input.get_device_index()); + + auto sorter_size = moe_permute_sort_workspace_size(expanded_rows, n_expert); + auto sort_workspace = maybe_allocate_tensor( + maybe_sort_workspace, {sorter_size}, torch::headeronly::ScalarType::Char, + device, "sort_workspace"); + auto permuted_experts_id = maybe_allocate_tensor( + maybe_permuted_experts_id, topk_ids.sizes(), + torch::headeronly::ScalarType::Int, device, "permuted_experts_id"); + auto sorted_row_idx = maybe_allocate_tensor( + maybe_sorted_row_idx, inv_permuted_idx.sizes(), + torch::headeronly::ScalarType::Int, device, "sorted_row_idx"); + + CubKeyValueSorter sorter{}; + int64_t* valid_num_ptr = nullptr; + torch::stable::Tensor topk_ids_for_sort = topk_ids; + + if (expert_map.has_value()) { + const int* expert_map_ptr = get_ptr(expert_map.value()); + valid_num_ptr = + get_ptr(expert_first_token_offset) + n_local_expert; + topk_ids_for_sort = maybe_allocate_tensor( + maybe_topk_ids_for_sort, topk_ids.sizes(), + torch::headeronly::ScalarType::Int, device, "topk_ids_for_sort"); + torch::stable::copy_(topk_ids_for_sort, topk_ids); + preprocessTopkIdLauncher(get_ptr(topk_ids_for_sort), n_token * topk, + expert_map_ptr, n_expert, stream); + } + + sortAndScanExpert( + get_ptr(topk_ids_for_sort), get_ptr(token_expert_indices), + get_ptr(permuted_experts_id), get_ptr(sorted_row_idx), + get_ptr(expert_first_token_offset), n_token, n_expert, + n_local_expert, topk, sorter, get_ptr(sort_workspace), stream); + + MOE_DISPATCH(input.scalar_type(), [&] { + expandInputRowsKernelLauncher( + get_ptr(input), get_ptr(permuted_input), + get_ptr(sorted_row_idx), get_ptr(inv_permuted_idx), + get_ptr(permuted_idx), get_ptr(expert_first_token_offset), + n_token, valid_num_ptr, n_hidden, topk, n_local_expert, stream); + }); +} + +void moe_permute( + const torch::stable::Tensor& input, // [n_token, hidden] + const torch::stable::Tensor& topk_ids, // [n_token, topk] + const torch::stable::Tensor& token_expert_indices, // [n_token, topk] + const std::optional& expert_map, // [n_expert] + int64_t n_expert, int64_t n_local_expert, int64_t topk, + torch::stable::Tensor& permuted_input, // [permuted_size, hidden] + torch::stable::Tensor& expert_first_token_offset, // [n_local_expert + 1] + torch::stable::Tensor& inv_permuted_idx, // [n_token, topk] + torch::stable::Tensor& permuted_idx) { // [permute_size] + moe_permute_impl(input, topk_ids, token_expert_indices, expert_map, n_expert, + n_local_expert, topk, permuted_input, + expert_first_token_offset, inv_permuted_idx, permuted_idx, + std::nullopt, std::nullopt, std::nullopt, std::nullopt); +} + +void moe_permute_with_scratch( + const torch::stable::Tensor& input, const torch::stable::Tensor& topk_ids, + const torch::stable::Tensor& token_expert_indices, + const std::optional& expert_map, int64_t n_expert, + int64_t n_local_expert, int64_t topk, torch::stable::Tensor& permuted_input, + torch::stable::Tensor& expert_first_token_offset, + torch::stable::Tensor& inv_permuted_idx, + torch::stable::Tensor& permuted_idx, torch::stable::Tensor& sort_workspace, + torch::stable::Tensor& permuted_experts_id, + torch::stable::Tensor& sorted_row_idx, + torch::stable::Tensor& topk_ids_for_sort) { + moe_permute_impl(input, topk_ids, token_expert_indices, expert_map, n_expert, + n_local_expert, topk, permuted_input, + expert_first_token_offset, inv_permuted_idx, permuted_idx, + sort_workspace, permuted_experts_id, sorted_row_idx, + topk_ids_for_sort); +} + +void moe_unpermute( + const torch::stable::Tensor& + permuted_hidden_states, // [n_token * topk, hidden] + const torch::stable::Tensor& topk_weights, // [n_token, topk] + const torch::stable::Tensor& inv_permuted_idx, // [n_token, topk] + const std::optional& + expert_first_token_offset, // [n_local_expert+1] + int64_t topk, + torch::stable::Tensor& hidden_states) { // [n_token, hidden] + STD_TORCH_CHECK( + permuted_hidden_states.scalar_type() == hidden_states.scalar_type(), + "permuted_hidden_states dtype must be same as hidden_states"); + + auto n_token = hidden_states.size(0); + auto n_hidden = hidden_states.size(1); + auto stream = get_current_cuda_stream(hidden_states.get_device_index()); + + int64_t const* valid_ptr = nullptr; + if (expert_first_token_offset.has_value()) { + int n_local_expert = expert_first_token_offset.value().size(0) - 1; + valid_ptr = + get_ptr(expert_first_token_offset.value()) + n_local_expert; + } + + MOE_DISPATCH(hidden_states.scalar_type(), [&] { + finalizeMoeRoutingKernelLauncher( + get_ptr(permuted_hidden_states), + get_ptr(hidden_states), get_ptr(topk_weights), + get_ptr(inv_permuted_idx), n_token, n_hidden, topk, valid_ptr, + stream); + }); +} + +template +__global__ void shuffleInputRowsKernel(const T* input, + const int32_t* dst2src_map, T* output, + int64_t num_src_rows, + int64_t num_dst_rows, int64_t num_cols) { + int64_t dest_row_idx = blockIdx.x; + int64_t const source_row_idx = dst2src_map[dest_row_idx]; + + if (blockIdx.x < num_dst_rows) { + // Load 128-bits per thread + constexpr int64_t ELEM_PER_THREAD = 128 / sizeof(T) / 8; + using DataElem = cutlass::Array; + + // Duplicate and permute rows + auto const* source_row_ptr = + reinterpret_cast(input + source_row_idx * num_cols); + auto* dest_row_ptr = + reinterpret_cast(output + dest_row_idx * num_cols); + + int64_t const start_offset = threadIdx.x; + int64_t const stride = blockDim.x; + int64_t const num_elems_in_col = num_cols / ELEM_PER_THREAD; + + for (int elem_index = start_offset; elem_index < num_elems_in_col; + elem_index += stride) { + dest_row_ptr[elem_index] = source_row_ptr[elem_index]; + } + } +} + +void shuffle_rows(const torch::stable::Tensor& input_tensor, + const torch::stable::Tensor& dst2src_map, + torch::stable::Tensor& output_tensor) { + STD_TORCH_CHECK(input_tensor.scalar_type() == output_tensor.scalar_type(), + "Input and output tensors must have the same data type"); + + auto stream = get_current_cuda_stream(output_tensor.get_device_index()); + const int64_t blocks = output_tensor.size(0); + const int64_t threads = 256; + const int64_t num_dest_rows = output_tensor.size(0); + const int64_t num_src_rows = input_tensor.size(0); + const int64_t num_cols = input_tensor.size(1); + + STD_TORCH_CHECK(!(num_cols % (128 / input_tensor.element_size() / 8)), + "num_cols must be divisible by 128 / " + "input_tensor.element_size() / 8"); + + MOE_DISPATCH(input_tensor.scalar_type(), [&] { + shuffleInputRowsKernel<<>>( + reinterpret_cast(input_tensor.const_data_ptr()), + reinterpret_cast(dst2src_map.const_data_ptr()), + reinterpret_cast(output_tensor.mutable_data_ptr()), + num_src_rows, num_dest_rows, num_cols); + }); +} + +#else + +int64_t moe_permute_sort_workspace_size(int64_t num_expanded_rows, + int64_t n_expert) { + STD_TORCH_CHECK( + false, "moe_permute_sort_workspace_size is not supported on CUDA < 12.0"); +} + +void moe_permute(const torch::stable::Tensor& input, + const torch::stable::Tensor& topk_ids, + const torch::stable::Tensor& token_expert_indices, + const std::optional& expert_map, + int64_t n_expert, int64_t n_local_expert, int64_t topk, + torch::stable::Tensor& permuted_input, + torch::stable::Tensor& expert_first_token_offset, + torch::stable::Tensor& inv_permuted_idx, + torch::stable::Tensor& permuted_idx) { + STD_TORCH_CHECK(false, "moe_permute is not supported on CUDA < 12.0"); +} + +void moe_permute_with_scratch( + const torch::stable::Tensor& input, const torch::stable::Tensor& topk_ids, + const torch::stable::Tensor& token_expert_indices, + const std::optional& expert_map, int64_t n_expert, + int64_t n_local_expert, int64_t topk, torch::stable::Tensor& permuted_input, + torch::stable::Tensor& expert_first_token_offset, + torch::stable::Tensor& inv_permuted_idx, + torch::stable::Tensor& permuted_idx, torch::stable::Tensor& sort_workspace, + torch::stable::Tensor& permuted_experts_id, + torch::stable::Tensor& sorted_row_idx, + torch::stable::Tensor& topk_ids_for_sort) { + STD_TORCH_CHECK(false, + "moe_permute_with_scratch is not supported on CUDA < 12.0"); +} + +void moe_unpermute( + const torch::stable::Tensor& permuted_hidden_states, + const torch::stable::Tensor& topk_weights, + const torch::stable::Tensor& inv_permuted_idx, + const std::optional& expert_first_token_offset, + int64_t topk, torch::stable::Tensor& hidden_states) { + STD_TORCH_CHECK(false, "moe_unpermute is not supported on CUDA < 12.0"); +} + +#endif + +bool moe_permute_unpermute_supported() { +#if defined(CUDA_VERSION) && (CUDA_VERSION >= 12000) + return true; +#else + return false; +#endif +} + +STABLE_TORCH_LIBRARY_IMPL(_moe_C, CUDA, m) { + m.impl("moe_permute", TORCH_BOX(&moe_permute)); + m.impl("moe_permute_with_scratch", TORCH_BOX(&moe_permute_with_scratch)); + m.impl("moe_unpermute", TORCH_BOX(&moe_unpermute)); +} \ No newline at end of file diff --git a/csrc/moe/moe_wna16.cu b/csrc/libtorch_stable/moe/moe_wna16.cu similarity index 77% rename from csrc/moe/moe_wna16.cu rename to csrc/libtorch_stable/moe/moe_wna16.cu index 7b6a111c00..9345a7c9f7 100644 --- a/csrc/moe/moe_wna16.cu +++ b/csrc/libtorch_stable/moe/moe_wna16.cu @@ -1,11 +1,14 @@ +#include -#include -#include -#include #include +#include +#include +#include +#include #include #include +#include "libtorch_stable/torch_utils.h" #include "moe_wna16_utils.h" #define DIVIDE(x, size) (((x) + (size) - 1) / (size)) @@ -263,7 +266,7 @@ void run_moe_wna16_gemm(const scalar_t* input, scalar_t* output, } const int shared_mem_size = BLOCK_SIZE_M * BLOCK_SIZE_K * 2; - const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + const cudaStream_t stream = get_current_cuda_stream(); kernel<<>>( input, output, b_qweight, b_scales, b_qzeros, topk_weights, sorted_token_ids, expert_ids, num_tokens_post_pad, num_experts, @@ -271,17 +274,18 @@ void run_moe_wna16_gemm(const scalar_t* input, scalar_t* output, BLOCK_SIZE_K, has_zp, mul_topk_weight); } -torch::Tensor moe_wna16_gemm(torch::Tensor input, torch::Tensor output, - torch::Tensor b_qweight, torch::Tensor b_scales, - std::optional b_qzeros, - std::optional topk_weights, - torch::Tensor sorted_token_ids, - torch::Tensor expert_ids, - torch::Tensor num_tokens_post_pad, int64_t top_k, - int64_t BLOCK_SIZE_M, int64_t BLOCK_SIZE_N, - int64_t BLOCK_SIZE_K, int64_t bit) { - const at::cuda::OptionalCUDAGuard device_guard(device_of(input)); - output.zero_(); +torch::stable::Tensor moe_wna16_gemm( + torch::stable::Tensor input, torch::stable::Tensor output, + torch::stable::Tensor b_qweight, torch::stable::Tensor b_scales, + std::optional b_qzeros, + std::optional topk_weights, + torch::stable::Tensor sorted_token_ids, torch::stable::Tensor expert_ids, + torch::stable::Tensor num_tokens_post_pad, int64_t top_k, + int64_t BLOCK_SIZE_M, int64_t BLOCK_SIZE_N, int64_t BLOCK_SIZE_K, + int64_t bit) { + const torch::stable::accelerator::DeviceGuard device_guard( + input.get_device_index()); + torch::stable::zero_(output); const int num_experts = b_qweight.size(0); const int size_m = input.size(0); @@ -291,52 +295,56 @@ torch::Tensor moe_wna16_gemm(torch::Tensor input, torch::Tensor output, int64_t EM = sorted_token_ids.size(0); if (size_m <= BLOCK_SIZE_M) { - EM = min(EM, size_m * BLOCK_SIZE_M * top_k); + EM = std::min(EM, size_m * BLOCK_SIZE_M * top_k); } const int num_token_blocks = (EM + BLOCK_SIZE_M - 1) / BLOCK_SIZE_M; const uint32_t* b_qzeros_ptr; if (b_qzeros.has_value()) - b_qzeros_ptr = (const uint32_t*)b_qzeros.value().data_ptr(); + b_qzeros_ptr = (const uint32_t*)b_qzeros.value().const_data_ptr(); const float* topk_weights_ptr = nullptr; if (topk_weights.has_value()) - topk_weights_ptr = (const float*)topk_weights.value().data_ptr(); + topk_weights_ptr = + (const float*)topk_weights.value().const_data_ptr(); int groups_per_block_row = BLOCK_SIZE_K / group_size; - TORCH_CHECK(bit == 4 || bit == 8, "bit must be 4 or 8"); - TORCH_CHECK(size_k % BLOCK_SIZE_K == 0, - "size_k must divisible by BLOCK_SIZE_K"); - TORCH_CHECK(BLOCK_SIZE_K % group_size == 0, - "BLOCK_SIZE_K must divisible by group_size"); - TORCH_CHECK(BLOCK_SIZE_M <= 64, "BLOCK_SIZE_M must less or equal to 64"); - TORCH_CHECK(groups_per_block_row == 1 || groups_per_block_row == 2 || - groups_per_block_row == 4 || groups_per_block_row == 8, - "BLOCK_SIZE_K // group_size must be one of [1, 2, 4, 8]"); + STD_TORCH_CHECK(bit == 4 || bit == 8, "bit must be 4 or 8"); + STD_TORCH_CHECK(size_k % BLOCK_SIZE_K == 0, + "size_k must divisible by BLOCK_SIZE_K"); + STD_TORCH_CHECK(BLOCK_SIZE_K % group_size == 0, + "BLOCK_SIZE_K must divisible by group_size"); + STD_TORCH_CHECK(BLOCK_SIZE_M <= 64, "BLOCK_SIZE_M must less or equal to 64"); + STD_TORCH_CHECK(groups_per_block_row == 1 || groups_per_block_row == 2 || + groups_per_block_row == 4 || groups_per_block_row == 8, + "BLOCK_SIZE_K // group_size must be one of [1, 2, 4, 8]"); - if (input.scalar_type() == at::ScalarType::Half) { + if (input.scalar_type() == torch::headeronly::ScalarType::Half) { run_moe_wna16_gemm( - (const half*)input.data_ptr(), - (half*)output.data_ptr(), - (const uint32_t*)b_qweight.data_ptr(), - (const half*)b_scales.data_ptr(), b_qzeros_ptr, - topk_weights_ptr, sorted_token_ids.data_ptr(), - expert_ids.data_ptr(), num_tokens_post_pad.data_ptr(), - num_experts, group_size, num_token_blocks, top_k, size_m, size_n, - size_k, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, bit, - b_qzeros.has_value(), topk_weights.has_value()); - } else if (input.scalar_type() == at::ScalarType::BFloat16) { + reinterpret_cast(input.const_data_ptr()), + reinterpret_cast(output.mutable_data_ptr()), + (const uint32_t*)b_qweight.const_data_ptr(), + reinterpret_cast(b_scales.const_data_ptr()), b_qzeros_ptr, + topk_weights_ptr, sorted_token_ids.const_data_ptr(), + expert_ids.const_data_ptr(), + num_tokens_post_pad.const_data_ptr(), num_experts, group_size, + num_token_blocks, top_k, size_m, size_n, size_k, BLOCK_SIZE_M, + BLOCK_SIZE_N, BLOCK_SIZE_K, bit, b_qzeros.has_value(), + topk_weights.has_value()); + } else if (input.scalar_type() == torch::headeronly::ScalarType::BFloat16) { run_moe_wna16_gemm( - (const nv_bfloat16*)input.data_ptr(), - (nv_bfloat16*)output.data_ptr(), - (const uint32_t*)b_qweight.data_ptr(), - (const nv_bfloat16*)b_scales.data_ptr(), b_qzeros_ptr, - topk_weights_ptr, sorted_token_ids.data_ptr(), - expert_ids.data_ptr(), num_tokens_post_pad.data_ptr(), - num_experts, group_size, num_token_blocks, top_k, size_m, size_n, - size_k, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, bit, - b_qzeros.has_value(), topk_weights.has_value()); + reinterpret_cast(input.const_data_ptr()), + reinterpret_cast(output.mutable_data_ptr()), + (const uint32_t*)b_qweight.const_data_ptr(), + reinterpret_cast(b_scales.const_data_ptr()), + b_qzeros_ptr, topk_weights_ptr, + sorted_token_ids.const_data_ptr(), + expert_ids.const_data_ptr(), + num_tokens_post_pad.const_data_ptr(), num_experts, group_size, + num_token_blocks, top_k, size_m, size_n, size_k, BLOCK_SIZE_M, + BLOCK_SIZE_N, BLOCK_SIZE_K, bit, b_qzeros.has_value(), + topk_weights.has_value()); } else { - TORCH_CHECK(false, "moe_wna16_gemm only supports bfloat16 and float16"); + STD_TORCH_CHECK(false, "moe_wna16_gemm only supports bfloat16 and float16"); } return output; } diff --git a/csrc/moe/moe_wna16_utils.h b/csrc/libtorch_stable/moe/moe_wna16_utils.h similarity index 100% rename from csrc/moe/moe_wna16_utils.h rename to csrc/libtorch_stable/moe/moe_wna16_utils.h diff --git a/csrc/libtorch_stable/moe/permute_unpermute_kernels/dispatch.h b/csrc/libtorch_stable/moe/permute_unpermute_kernels/dispatch.h new file mode 100644 index 0000000000..976233dd48 --- /dev/null +++ b/csrc/libtorch_stable/moe/permute_unpermute_kernels/dispatch.h @@ -0,0 +1,60 @@ +#pragma once + +#include +#include +#include + +#define MOE_SWITCH(TYPE, ...) \ + const auto _st = (TYPE); \ + switch (_st) { \ + __VA_ARGS__ \ + default: \ + STD_TORCH_CHECK(false, "[moe permute]data type dispatch fail!") \ + } + +#define MOE_DISPATCH_CASE(enum_type, ...) \ + case enum_type: { \ + using scalar_t = ScalarType2CudaType::type; \ + __VA_ARGS__(); \ + break; \ + } + +#define MOE_DISPATCH_FLOAT_CASE(...) \ + MOE_DISPATCH_CASE(torch::headeronly::ScalarType::Float, __VA_ARGS__) \ + MOE_DISPATCH_CASE(torch::headeronly::ScalarType::Half, __VA_ARGS__) \ + MOE_DISPATCH_CASE(torch::headeronly::ScalarType::BFloat16, __VA_ARGS__) \ + MOE_DISPATCH_CASE(torch::headeronly::ScalarType::Float8_e5m2, __VA_ARGS__) \ + MOE_DISPATCH_CASE(torch::headeronly::ScalarType::Float8_e4m3fn, __VA_ARGS__) \ + MOE_DISPATCH_CASE(torch::headeronly::ScalarType::Byte, __VA_ARGS__) + +#define MOE_DISPATCH(TYPE, ...) \ + MOE_SWITCH(TYPE, MOE_DISPATCH_FLOAT_CASE(__VA_ARGS__)) + +template +struct ScalarType2CudaType; + +template <> +struct ScalarType2CudaType { + using type = float; +}; +template <> +struct ScalarType2CudaType { + using type = half; +}; +template <> +struct ScalarType2CudaType { + using type = __nv_bfloat16; +}; +// uint8 for packed fp4 +template <> +struct ScalarType2CudaType { + using type = uint8_t; +}; +template <> +struct ScalarType2CudaType { + using type = __nv_fp8_e5m2; +}; +template <> +struct ScalarType2CudaType { + using type = __nv_fp8_e4m3; +}; \ No newline at end of file diff --git a/csrc/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.cu b/csrc/libtorch_stable/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.cu similarity index 95% rename from csrc/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.cu rename to csrc/libtorch_stable/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.cu index 2cc2003216..f5ec32c390 100644 --- a/csrc/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.cu +++ b/csrc/libtorch_stable/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.cu @@ -1,5 +1,7 @@ +#include +#include -#include "moe_permute_unpermute_kernel.h" +#include "libtorch_stable/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.h" // moe_permute kernels require at least CUDA 12.0 #if defined(CUDA_VERSION) && (CUDA_VERSION >= 12000) @@ -48,9 +50,10 @@ void CubKeyValueSorter::run(void* workspace, size_t const workspace_size, size_t expected_ws_size = getWorkspaceSize(num_key_value_pairs, num_experts_); size_t actual_ws_size = workspace_size; - TORCH_CHECK(expected_ws_size <= workspace_size, - "[CubKeyValueSorter::run] The allocated workspace is too small " - "to run this problem."); + STD_TORCH_CHECK( + expected_ws_size <= workspace_size, + "[CubKeyValueSorter::run] The allocated workspace is too small " + "to run this problem."); cub::DeviceRadixSort::SortPairs(workspace, actual_ws_size, keys_in, keys_out, values_in, values_out, num_key_value_pairs, 0, num_bits_, stream); diff --git a/csrc/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.h b/csrc/libtorch_stable/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.h similarity index 89% rename from csrc/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.h rename to csrc/libtorch_stable/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.h index fe44d30155..89c278a4ed 100644 --- a/csrc/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.h +++ b/csrc/libtorch_stable/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.h @@ -2,23 +2,24 @@ // reference from tensorrt_llm moe kernel implementation archive in // https://github.com/BBuf/tensorrt-llm-moe/tree/master -#include -#include -#include "dispatch.h" +#include + #include #include #include -#include "cutlass/numeric_size.h" + #include "cutlass/array.h" +#include "cutlass/numeric_size.h" +#include "libtorch_stable/moe/permute_unpermute_kernels/dispatch.h" template -inline T* get_ptr(torch::Tensor& t) { - return reinterpret_cast(t.data_ptr()); +inline T* get_ptr(torch::stable::Tensor& t) { + return reinterpret_cast(t.mutable_data_ptr()); } template -inline const T* get_ptr(const torch::Tensor& t) { - return reinterpret_cast(t.data_ptr()); +inline const T* get_ptr(const torch::stable::Tensor& t) { + return reinterpret_cast(t.const_data_ptr()); } class CubKeyValueSorter { diff --git a/csrc/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.inl b/csrc/libtorch_stable/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.inl similarity index 100% rename from csrc/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.inl rename to csrc/libtorch_stable/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.inl diff --git a/csrc/moe/topk_softmax_kernels.cu b/csrc/libtorch_stable/moe/topk_softmax_kernels.cu similarity index 88% rename from csrc/moe/topk_softmax_kernels.cu rename to csrc/libtorch_stable/moe/topk_softmax_kernels.cu index 57461a044f..e8453579ba 100644 --- a/csrc/moe/topk_softmax_kernels.cu +++ b/csrc/libtorch_stable/moe/topk_softmax_kernels.cu @@ -17,11 +17,16 @@ * limitations under the License. */ #include -#include -#include -#include -#include "../cuda_compat.h" -#include "../cub_helpers.h" + +#include +#include +#include +#include +#include + +#include "../../cuda_compat.h" +#include "../../cub_helpers.h" +#include "libtorch_stable/torch_utils.h" #ifndef USE_ROCM #include @@ -713,7 +718,7 @@ void topkGatingKernelLauncher( break; #endif default: { - TORCH_CHECK(workspace != nullptr, + STD_TORCH_CHECK(workspace != nullptr, "workspace must be provided for num_experts that are not a power of 2 or multiple of 64."); static constexpr int TPB = 256; if constexpr (SF == SCORING_SOFTMAX) { @@ -723,7 +728,7 @@ void topkGatingKernelLauncher( moeSigmoid<<>>( gating_output, nullptr, workspace, num_experts); } else { - TORCH_CHECK(false, "Unsupported scoring func"); + STD_TORCH_CHECK(false, "Unsupported scoring func"); } moeTopK<<>>( workspace, nullptr, topk_weights, topk_indices, token_expert_indices, @@ -738,63 +743,65 @@ void topkGatingKernelLauncher( template void dispatch_topk_launch( - torch::Tensor& gating_output, - torch::Tensor& topk_weights, - torch::Tensor& topk_indices, - torch::Tensor& token_expert_indices, - torch::Tensor& softmax_workspace, + torch::stable::Tensor& gating_output, + torch::stable::Tensor& topk_weights, + torch::stable::Tensor& topk_indices, + torch::stable::Tensor& token_expert_indices, + torch::stable::Tensor& softmax_workspace, int num_tokens, int num_experts, int topk, bool renormalize, - std::optional bias, + std::optional bias, cudaStream_t stream) { const float* bias_ptr = nullptr; if (bias.has_value()) { - const torch::Tensor& bias_tensor = bias.value(); - TORCH_CHECK(bias_tensor.scalar_type() == at::ScalarType::Float, "bias tensor must be float32"); - TORCH_CHECK(bias_tensor.dim() == 1, "bias tensor must be 1D"); - TORCH_CHECK(bias_tensor.size(0) == num_experts, "bias size mismatch, expected: ", num_experts); - TORCH_CHECK(bias_tensor.is_contiguous(), "bias tensor must be contiguous"); - bias_ptr = bias_tensor.data_ptr(); + const torch::stable::Tensor& bias_tensor = bias.value(); + STD_TORCH_CHECK(bias_tensor.scalar_type() == torch::headeronly::ScalarType::Float, + "bias tensor must be float32"); + STD_TORCH_CHECK(bias_tensor.dim() == 1, "bias tensor must be 1D"); + STD_TORCH_CHECK(bias_tensor.size(0) == num_experts, + "bias size mismatch, expected: ", num_experts); + STD_TORCH_CHECK(bias_tensor.is_contiguous(), "bias tensor must be contiguous"); + bias_ptr = bias_tensor.const_data_ptr(); } - if (topk_indices.scalar_type() == at::ScalarType::Int) { + if (topk_indices.scalar_type() == torch::headeronly::ScalarType::Int) { vllm::moe::topkGatingKernelLauncher( - reinterpret_cast(gating_output.data_ptr()), - topk_weights.data_ptr(), - topk_indices.data_ptr(), - token_expert_indices.data_ptr(), - softmax_workspace.data_ptr(), + reinterpret_cast(gating_output.const_data_ptr()), + topk_weights.mutable_data_ptr(), + topk_indices.mutable_data_ptr(), + token_expert_indices.mutable_data_ptr(), + softmax_workspace.mutable_data_ptr(), num_tokens, num_experts, topk, renormalize, bias_ptr, stream); - } else if (topk_indices.scalar_type() == at::ScalarType::UInt32) { + } else if (topk_indices.scalar_type() == torch::headeronly::ScalarType::UInt32) { vllm::moe::topkGatingKernelLauncher( - reinterpret_cast(gating_output.data_ptr()), - topk_weights.data_ptr(), - topk_indices.data_ptr(), - token_expert_indices.data_ptr(), - softmax_workspace.data_ptr(), + reinterpret_cast(gating_output.const_data_ptr()), + topk_weights.mutable_data_ptr(), + topk_indices.mutable_data_ptr(), + token_expert_indices.mutable_data_ptr(), + softmax_workspace.mutable_data_ptr(), num_tokens, num_experts, topk, renormalize, bias_ptr, stream); } else { - TORCH_CHECK(topk_indices.scalar_type() == at::ScalarType::Long); + STD_TORCH_CHECK(topk_indices.scalar_type() == torch::headeronly::ScalarType::Long); vllm::moe::topkGatingKernelLauncher( - reinterpret_cast(gating_output.data_ptr()), - topk_weights.data_ptr(), - topk_indices.data_ptr(), - token_expert_indices.data_ptr(), - softmax_workspace.data_ptr(), + reinterpret_cast(gating_output.const_data_ptr()), + topk_weights.mutable_data_ptr(), + topk_indices.mutable_data_ptr(), + token_expert_indices.mutable_data_ptr(), + softmax_workspace.mutable_data_ptr(), num_tokens, num_experts, topk, renormalize, bias_ptr, stream); } } void topk_softmax( - torch::Tensor& topk_weights, // [num_tokens, topk] - torch::Tensor& topk_indices, // [num_tokens, topk] - torch::Tensor& token_expert_indices, // [num_tokens, topk] - torch::Tensor& gating_output, // [num_tokens, num_experts] + torch::stable::Tensor& topk_weights, // [num_tokens, topk] + torch::stable::Tensor& topk_indices, // [num_tokens, topk] + torch::stable::Tensor& token_expert_indices, // [num_tokens, topk] + torch::stable::Tensor& gating_output, // [num_tokens, num_experts] bool renormalize, - std::optional bias) + std::optional bias) { const int num_experts = gating_output.size(-1); const auto num_tokens = gating_output.numel() / num_experts; @@ -804,35 +811,36 @@ void topk_softmax( const bool needs_workspace = !is_pow_2 || num_experts > 256; const int64_t workspace_size = needs_workspace ? num_tokens * num_experts : 0; - const at::cuda::OptionalCUDAGuard device_guard(device_of(gating_output)); - const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); - const auto workspace_options = gating_output.options().dtype(at::ScalarType::Float); - torch::Tensor softmax_workspace = torch::empty({workspace_size}, workspace_options); + torch::stable::accelerator::DeviceGuard guard(gating_output.get_device_index()); + const cudaStream_t stream = + get_current_cuda_stream(gating_output.get_device_index()); + auto softmax_workspace = torch::stable::new_empty( + gating_output, {workspace_size}, torch::headeronly::ScalarType::Float); - if (gating_output.scalar_type() == at::ScalarType::Float) { + if (gating_output.scalar_type() == torch::headeronly::ScalarType::Float) { dispatch_topk_launch(gating_output, topk_weights, topk_indices, token_expert_indices, softmax_workspace, num_tokens, num_experts, topk, renormalize, bias, stream); - } else if (gating_output.scalar_type() == at::ScalarType::Half) { + } else if (gating_output.scalar_type() == torch::headeronly::ScalarType::Half) { dispatch_topk_launch<__half, vllm::moe::SCORING_SOFTMAX>(gating_output, topk_weights, topk_indices, token_expert_indices, softmax_workspace, num_tokens, num_experts, topk, renormalize, bias, stream); - } else if (gating_output.scalar_type() == at::ScalarType::BFloat16) { + } else if (gating_output.scalar_type() == torch::headeronly::ScalarType::BFloat16) { dispatch_topk_launch<__nv_bfloat16, vllm::moe::SCORING_SOFTMAX>(gating_output, topk_weights, topk_indices, token_expert_indices, softmax_workspace, num_tokens, num_experts, topk, renormalize, bias, stream); } else { - TORCH_CHECK(false, "Unsupported gating_output data type: ", gating_output.scalar_type()); + STD_TORCH_CHECK(false, "Unsupported gating_output data type: ", gating_output.scalar_type()); } } void topk_sigmoid( - torch::Tensor& topk_weights, // [num_tokens, topk] - torch::Tensor& topk_indices, // [num_tokens, topk] - torch::Tensor& token_expert_indices, // [num_tokens, topk] - torch::Tensor& gating_output, // [num_tokens, num_experts] + torch::stable::Tensor& topk_weights, // [num_tokens, topk] + torch::stable::Tensor& topk_indices, // [num_tokens, topk] + torch::stable::Tensor& token_expert_indices, // [num_tokens, topk] + torch::stable::Tensor& gating_output, // [num_tokens, num_experts] bool renormalize, - std::optional bias) + std::optional bias) { const int num_experts = gating_output.size(-1); const auto num_tokens = gating_output.numel() / num_experts; @@ -842,24 +850,25 @@ void topk_sigmoid( const bool needs_workspace = !is_pow_2 || num_experts > 256; const int64_t workspace_size = needs_workspace ? num_tokens * num_experts : 0; - const at::cuda::OptionalCUDAGuard device_guard(device_of(gating_output)); - const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); - const auto workspace_options = gating_output.options().dtype(at::ScalarType::Float); - torch::Tensor workspace = torch::empty({workspace_size}, workspace_options); + torch::stable::accelerator::DeviceGuard guard(gating_output.get_device_index()); + const cudaStream_t stream = + get_current_cuda_stream(gating_output.get_device_index()); + auto workspace = torch::stable::new_empty( + gating_output, {workspace_size}, torch::headeronly::ScalarType::Float); - if (gating_output.scalar_type() == at::ScalarType::Float) { + if (gating_output.scalar_type() == torch::headeronly::ScalarType::Float) { dispatch_topk_launch(gating_output, topk_weights, topk_indices, token_expert_indices, workspace, num_tokens, num_experts, topk, renormalize, bias, stream); - } else if (gating_output.scalar_type() == at::ScalarType::Half) { + } else if (gating_output.scalar_type() == torch::headeronly::ScalarType::Half) { dispatch_topk_launch<__half, vllm::moe::SCORING_SIGMOID>(gating_output, topk_weights, topk_indices, token_expert_indices, workspace, num_tokens, num_experts, topk, renormalize, bias, stream); - } else if (gating_output.scalar_type() == at::ScalarType::BFloat16) { + } else if (gating_output.scalar_type() == torch::headeronly::ScalarType::BFloat16) { dispatch_topk_launch<__nv_bfloat16, vllm::moe::SCORING_SIGMOID>(gating_output, topk_weights, topk_indices, token_expert_indices, workspace, num_tokens, num_experts, topk, renormalize, bias, stream); } else { - TORCH_CHECK(false, "Unsupported gating_output data type: ", gating_output.scalar_type()); + STD_TORCH_CHECK(false, "Unsupported gating_output data type: ", gating_output.scalar_type()); } } diff --git a/csrc/moe/topk_softplus_sqrt_kernels.cu b/csrc/libtorch_stable/moe/topk_softplus_sqrt_kernels.cu similarity index 87% rename from csrc/moe/topk_softplus_sqrt_kernels.cu rename to csrc/libtorch_stable/moe/topk_softplus_sqrt_kernels.cu index d5bb8edadc..7efe13b4d9 100644 --- a/csrc/moe/topk_softplus_sqrt_kernels.cu +++ b/csrc/libtorch_stable/moe/topk_softplus_sqrt_kernels.cu @@ -18,11 +18,16 @@ * limitations under the License. */ #include -#include -#include -#include -#include "../cuda_compat.h" -#include "../cub_helpers.h" + +#include +#include +#include +#include +#include + +#include "../../cuda_compat.h" +#include "../../cub_helpers.h" +#include "libtorch_stable/torch_utils.h" #ifndef USE_ROCM #include #include @@ -618,7 +623,7 @@ void topkGatingSoftplusSqrtKernelLauncher( LAUNCH_SOFTPLUS_SQRT(576, WARPS_PER_TB, BYTES_PER_LDG_MULTIPLE_64_NARROW); break; default: { - TORCH_CHECK(false, "Unsupported expert number: ", num_experts); + STD_TORCH_CHECK(false, "Unsupported expert number: ", num_experts); } } } @@ -628,100 +633,109 @@ void topkGatingSoftplusSqrtKernelLauncher( template void dispatch_topk_softplus_sqrt_launch( - const ComputeType* gating_output, torch::Tensor& topk_weights, - torch::Tensor& topk_indices, torch::Tensor& token_expert_indices, - int num_tokens, int num_experts, int topk, bool renormalize, - double routed_scaling_factor, - const c10::optional& correction_bias, - const c10::optional& input_ids, - const c10::optional& tid2eid, cudaStream_t stream) { + const ComputeType* gating_output, torch::stable::Tensor& topk_weights, + torch::stable::Tensor& topk_indices, + torch::stable::Tensor& token_expert_indices, int num_tokens, + int num_experts, int topk, bool renormalize, double routed_scaling_factor, + const std::optional& correction_bias, + const std::optional& input_ids, + const std::optional& tid2eid, cudaStream_t stream) { const float* bias_ptr = nullptr; if (correction_bias.has_value()) { - bias_ptr = correction_bias.value().data_ptr(); + bias_ptr = correction_bias.value().const_data_ptr(); } bool use_hash = false; if (tid2eid.has_value()) { - TORCH_CHECK(input_ids.has_value(), "input_ids is required for hash MoE"); + STD_TORCH_CHECK(input_ids.has_value(), + "input_ids is required for hash MoE"); use_hash = true; } - if (topk_indices.scalar_type() == at::ScalarType::Int) { + if (topk_indices.scalar_type() == torch::headeronly::ScalarType::Int) { const int* input_ids_ptr = nullptr; const int* tid2eid_ptr = nullptr; if (tid2eid.has_value()) { - input_ids_ptr = input_ids.value().data_ptr(); - tid2eid_ptr = tid2eid.value().data_ptr(); + input_ids_ptr = input_ids.value().const_data_ptr(); + tid2eid_ptr = tid2eid.value().const_data_ptr(); } vllm::moe::topkGatingSoftplusSqrtKernelLauncher( - gating_output, topk_weights.data_ptr(), - topk_indices.data_ptr(), token_expert_indices.data_ptr(), - num_tokens, num_experts, topk, renormalize, routed_scaling_factor, - bias_ptr, use_hash, input_ids_ptr, tid2eid_ptr, stream); - } else if (topk_indices.scalar_type() == at::ScalarType::UInt32) { + gating_output, topk_weights.mutable_data_ptr(), + topk_indices.mutable_data_ptr(), + token_expert_indices.mutable_data_ptr(), num_tokens, num_experts, + topk, renormalize, routed_scaling_factor, bias_ptr, use_hash, + input_ids_ptr, tid2eid_ptr, stream); + } else if (topk_indices.scalar_type() == + torch::headeronly::ScalarType::UInt32) { const uint32_t* input_ids_ptr = nullptr; const uint32_t* tid2eid_ptr = nullptr; if (tid2eid.has_value()) { - input_ids_ptr = input_ids.value().data_ptr(); - tid2eid_ptr = tid2eid.value().data_ptr(); + input_ids_ptr = input_ids.value().const_data_ptr(); + tid2eid_ptr = tid2eid.value().const_data_ptr(); } vllm::moe::topkGatingSoftplusSqrtKernelLauncher( - gating_output, topk_weights.data_ptr(), - topk_indices.data_ptr(), token_expert_indices.data_ptr(), - num_tokens, num_experts, topk, renormalize, routed_scaling_factor, - bias_ptr, use_hash, input_ids_ptr, tid2eid_ptr, stream); + gating_output, topk_weights.mutable_data_ptr(), + topk_indices.mutable_data_ptr(), + token_expert_indices.mutable_data_ptr(), num_tokens, num_experts, + topk, renormalize, routed_scaling_factor, bias_ptr, use_hash, + input_ids_ptr, tid2eid_ptr, stream); } else { - TORCH_CHECK(topk_indices.scalar_type() == at::ScalarType::Long); + STD_TORCH_CHECK(topk_indices.scalar_type() == + torch::headeronly::ScalarType::Long); const int64_t* input_ids_ptr = nullptr; const int64_t* tid2eid_ptr = nullptr; if (tid2eid.has_value()) { - input_ids_ptr = input_ids.value().data_ptr(); - tid2eid_ptr = tid2eid.value().data_ptr(); + input_ids_ptr = input_ids.value().const_data_ptr(); + tid2eid_ptr = tid2eid.value().const_data_ptr(); } vllm::moe::topkGatingSoftplusSqrtKernelLauncher( - gating_output, topk_weights.data_ptr(), - topk_indices.data_ptr(), token_expert_indices.data_ptr(), - num_tokens, num_experts, topk, renormalize, routed_scaling_factor, - bias_ptr, use_hash, input_ids_ptr, tid2eid_ptr, stream); + gating_output, topk_weights.mutable_data_ptr(), + topk_indices.mutable_data_ptr(), + token_expert_indices.mutable_data_ptr(), num_tokens, num_experts, + topk, renormalize, routed_scaling_factor, bias_ptr, use_hash, + input_ids_ptr, tid2eid_ptr, stream); } } void topk_softplus_sqrt( - torch::Tensor& topk_weights, // [num_tokens, topk] - torch::Tensor& topk_indices, // [num_tokens, topk] - torch::Tensor& token_expert_indices, // [num_tokens, topk] - torch::Tensor& gating_output, // [num_tokens, num_experts] + torch::stable::Tensor& topk_weights, // [num_tokens, topk] + torch::stable::Tensor& topk_indices, // [num_tokens, topk] + torch::stable::Tensor& token_expert_indices, // [num_tokens, topk] + torch::stable::Tensor& gating_output, // [num_tokens, num_experts] bool renormalize, double routed_scaling_factor, - const c10::optional& correction_bias, - const c10::optional& input_ids, - const c10::optional& tid2eid) { + const std::optional& correction_bias, + const std::optional& input_ids, + const std::optional& tid2eid) { const int num_experts = gating_output.size(-1); const auto num_tokens = gating_output.numel() / num_experts; const int topk = topk_weights.size(-1); - const at::cuda::OptionalCUDAGuard device_guard(device_of(gating_output)); - const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + const torch::stable::accelerator::DeviceGuard guard( + gating_output.get_device_index()); + const cudaStream_t stream = + get_current_cuda_stream(gating_output.get_device_index()); - if (gating_output.scalar_type() == at::ScalarType::Float) { + if (gating_output.scalar_type() == torch::headeronly::ScalarType::Float) { dispatch_topk_softplus_sqrt_launch( - gating_output.data_ptr(), topk_weights, topk_indices, + gating_output.const_data_ptr(), topk_weights, topk_indices, token_expert_indices, num_tokens, num_experts, topk, renormalize, routed_scaling_factor, correction_bias, input_ids, tid2eid, stream); - } else if (gating_output.scalar_type() == at::ScalarType::Half) { + } else if (gating_output.scalar_type() == + torch::headeronly::ScalarType::Half) { dispatch_topk_softplus_sqrt_launch<__half>( - reinterpret_cast(gating_output.data_ptr()), + reinterpret_cast(gating_output.const_data_ptr()), topk_weights, topk_indices, token_expert_indices, num_tokens, num_experts, topk, renormalize, routed_scaling_factor, correction_bias, input_ids, tid2eid, stream); - } else if (gating_output.scalar_type() == at::ScalarType::BFloat16) { + } else if (gating_output.scalar_type() == + torch::headeronly::ScalarType::BFloat16) { dispatch_topk_softplus_sqrt_launch<__nv_bfloat16>( - reinterpret_cast( - gating_output.data_ptr()), + reinterpret_cast(gating_output.const_data_ptr()), topk_weights, topk_indices, token_expert_indices, num_tokens, num_experts, topk, renormalize, routed_scaling_factor, correction_bias, input_ids, tid2eid, stream); } else { - TORCH_CHECK(false, "Unsupported gating_output data type: ", - gating_output.scalar_type()); + STD_TORCH_CHECK(false, "Unsupported gating_output data type: ", + gating_output.scalar_type()); } } \ No newline at end of file diff --git a/csrc/moe/torch_bindings.cpp b/csrc/libtorch_stable/moe/torch_bindings.cpp similarity index 82% rename from csrc/moe/torch_bindings.cpp rename to csrc/libtorch_stable/moe/torch_bindings.cpp index 99230f03b4..bfcb0074e5 100644 --- a/csrc/moe/torch_bindings.cpp +++ b/csrc/libtorch_stable/moe/torch_bindings.cpp @@ -1,32 +1,30 @@ #include "core/registration.h" #include "moe_ops.h" -TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) { +#include + +STABLE_TORCH_LIBRARY_FRAGMENT(_moe_C, m) { // Apply topk softmax to the gating outputs. m.def( "topk_softmax(Tensor! topk_weights, Tensor! topk_indices, Tensor! " "token_expert_indices, Tensor gating_output, bool renormalize, Tensor? " "bias) -> ()"); - m.impl("topk_softmax", torch::kCUDA, &topk_softmax); // Apply topk sigmoid to the gating outputs. m.def( "topk_sigmoid(Tensor! topk_weights, Tensor! topk_indices, Tensor! " "token_expert_indices, Tensor gating_output, bool renormalize, Tensor? " "bias) -> ()"); - m.impl("topk_sigmoid", torch::kCUDA, &topk_sigmoid); m.def( "topk_softplus_sqrt(Tensor! topk_weights, Tensor! topk_indices, Tensor! " "token_expert_indices, Tensor gating_output, bool renormalize, float " "routed_scaling_factor, Tensor? " "bias, Tensor? input_ids, Tensor? tid2eid) -> ()"); - m.impl("topk_softplus_sqrt", torch::kCUDA, &topk_softplus_sqrt); // Calculate the result of moe by summing up the partial results // from all selected experts. m.def("moe_sum(Tensor input, Tensor! output) -> ()"); - m.impl("moe_sum", torch::kCUDA, &moe_sum); // Aligning the number of tokens to be processed by each expert such // that it is divisible by the block size. @@ -36,7 +34,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) { " Tensor! experts_ids," " Tensor! num_tokens_post_pad," " Tensor? maybe_expert_map) -> ()"); - m.impl("moe_align_block_size", torch::kCUDA, &moe_align_block_size); // Aligning the number of tokens to be processed by each expert such // that it is divisible by the block size, but for the batched case. @@ -46,8 +43,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) { " Tensor! sorted_token_ids," " Tensor! experts_ids," " Tensor! num_tokens_post_pad) -> ()"); - m.impl("batched_moe_align_block_size", torch::kCUDA, - &batched_moe_align_block_size); // Aligning the number of tokens to be processed by each expert such // that it is divisible by the block size. @@ -64,8 +59,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) { " Tensor !adapter_enabled," " Tensor !lora_ids," " Tensor? maybe_expert_map) -> () "); - m.impl("moe_lora_align_block_size", torch::kCUDA, &moe_lora_align_block_size); - #ifndef USE_ROCM m.def( "moe_wna16_gemm(Tensor input, Tensor! output, Tensor b_qweight, " @@ -75,8 +68,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) { "int top_k, int BLOCK_SIZE_M, int BLOCK_SIZE_N, int BLOCK_SIZE_K, " "int bit) -> Tensor"); - m.impl("moe_wna16_gemm", torch::kCUDA, &moe_wna16_gemm); - m.def( "moe_wna16_marlin_gemm(Tensor! a, Tensor? c_or_none," "Tensor! b_q_weight, Tensor? b_bias_or_none," @@ -118,14 +109,11 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) { m.def( "moe_permute_sort_workspace_size(int num_expanded_rows, int n_expert) -> " "int"); - m.impl("moe_permute_unpermute_supported", &moe_permute_unpermute_supported); - m.impl("moe_permute_sort_workspace_size", &moe_permute_sort_workspace_size); // Row shuffle for MoE m.def( "shuffle_rows(Tensor input_tensor, Tensor dst2src_map, Tensor! " "output_tensor) -> ()"); - m.impl("shuffle_rows", torch::kCUDA, &shuffle_rows); // Apply grouped topk routing to select experts. m.def( @@ -133,7 +121,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) { "topk_group, int topk, bool renormalize, float " "routed_scaling_factor, Tensor bias, int scoring_func) -> (Tensor, " "Tensor)"); - m.impl("grouped_topk", torch::kCUDA, &grouped_topk); // DeepSeek V3 optimized router GEMM for SM90+ m.def("dsv3_router_gemm(Tensor! output, Tensor mat_a, Tensor mat_b) -> ()"); @@ -141,4 +128,30 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) { #endif } -REGISTER_EXTENSION(TORCH_EXTENSION_NAME) +STABLE_TORCH_LIBRARY_IMPL(_moe_C, CUDA, m) { + m.impl("topk_softmax", TORCH_BOX(&topk_softmax)); + m.impl("topk_sigmoid", TORCH_BOX(&topk_sigmoid)); + m.impl("topk_softplus_sqrt", TORCH_BOX(&topk_softplus_sqrt)); + m.impl("moe_sum", TORCH_BOX(&moe_sum)); + m.impl("moe_align_block_size", TORCH_BOX(&moe_align_block_size)); + m.impl("batched_moe_align_block_size", + TORCH_BOX(&batched_moe_align_block_size)); + m.impl("moe_lora_align_block_size", TORCH_BOX(&moe_lora_align_block_size)); +#ifndef USE_ROCM + m.impl("moe_wna16_gemm", TORCH_BOX(&moe_wna16_gemm)); + m.impl("shuffle_rows", TORCH_BOX(&shuffle_rows)); + m.impl("grouped_topk", TORCH_BOX(&grouped_topk)); +#endif +} + +#ifndef USE_ROCM +// Primitive-only ops have no tensor to dispatch on. +STABLE_TORCH_LIBRARY_IMPL(_moe_C, CompositeExplicitAutograd, m) { + m.impl("moe_permute_unpermute_supported", + TORCH_BOX(&moe_permute_unpermute_supported)); + m.impl("moe_permute_sort_workspace_size", + TORCH_BOX(&moe_permute_sort_workspace_size)); +} +#endif + +REGISTER_EXTENSION(_moe_C_stable_libtorch) diff --git a/csrc/libtorch_stable/torch_bindings.cpp b/csrc/libtorch_stable/torch_bindings.cpp index e7d1b3669f..816f266504 100644 --- a/csrc/libtorch_stable/torch_bindings.cpp +++ b/csrc/libtorch_stable/torch_bindings.cpp @@ -246,6 +246,22 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) { "awq_dequantize(Tensor _kernel, Tensor _scaling_factors, " "Tensor _zeros, SymInt split_k_iters, int thx, int thy) -> Tensor"); + // Expert-specialization mxfp8 blockscaled grouped quantization (SM100+). + ops.def( + "mxfp8_experts_quant(" + " Tensor input, Tensor problem_sizes, Tensor expert_offsets," + " Tensor blockscale_offsets, Tensor! quant_output, Tensor! scale_factor)" + " -> ()"); + // conditionally compiled so impl registration is in source file + + // Expert-specialization mxfp8 blockscaled grouped GEMM (SM100+). + ops.def( + "cutlass_mxfp8_grouped_mm(" + " Tensor a, Tensor b, Tensor sfa, Tensor sfb, Tensor! out," + " Tensor problem_sizes, Tensor expert_offsets, Tensor blockscale_offsets)" + " -> ()"); + // conditionally compiled so impl registration is in source file + // DeepSeek V3 fused A GEMM (SM 9.0+, bf16 only, 1-16 tokens). // conditionally compiled so impl registration is in source file ops.def( diff --git a/csrc/moe/dsv3_router_gemm_utils.h b/csrc/moe/dsv3_router_gemm_utils.h deleted file mode 100644 index 9b533bcabf..0000000000 --- a/csrc/moe/dsv3_router_gemm_utils.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Adapted from SGLang's sgl-kernel implementation, which was adapted from - * https://github.com/NVIDIA/TensorRT-LLM/blob/main/cpp/tensorrt_llm/kernels/dsv3MinLatencyKernels/dsv3RouterGemm.cu - * https://github.com/NVIDIA/TensorRT-LLM/blob/main/cpp/tensorrt_llm/thop/dsv3RouterGemmOp.cpp - * - * Copyright (c) 2019-2023, NVIDIA CORPORATION. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -#include -#include - -inline int getSMVersion() { - auto* props = at::cuda::getCurrentDeviceProperties(); - return props->major * 10 + props->minor; -} diff --git a/csrc/moe/moe_ops.h b/csrc/moe/moe_ops.h deleted file mode 100644 index ca2776c6ed..0000000000 --- a/csrc/moe/moe_ops.h +++ /dev/null @@ -1,81 +0,0 @@ -#pragma once - -#include - -void topk_softmax(torch::Tensor& topk_weights, torch::Tensor& topk_indices, - torch::Tensor& token_expert_indices, - torch::Tensor& gating_output, bool renormalize, - std::optional bias); - -void topk_sigmoid(torch::Tensor& topk_weights, torch::Tensor& topk_indices, - torch::Tensor& token_expert_indices, - torch::Tensor& gating_output, bool renormalize, - std::optional bias); - -void topk_softplus_sqrt(torch::Tensor& topk_weights, - torch::Tensor& topk_indices, - torch::Tensor& token_expert_indices, - torch::Tensor& gating_output, bool renormalize, - double routed_scaling_factor, - const c10::optional& correction_bias, - const c10::optional& input_ids, - const c10::optional& tid2eid); - -void moe_sum(torch::Tensor& input, torch::Tensor& output); - -void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts, - int64_t block_size, torch::Tensor sorted_token_ids, - torch::Tensor experts_ids, - torch::Tensor num_tokens_post_pad, - std::optional maybe_expert_map); - -void batched_moe_align_block_size(int64_t max_tokens_per_batch, - int64_t block_size, - torch::Tensor const& expert_num_tokens, - torch::Tensor sorted_ids, - torch::Tensor expert_ids, - torch::Tensor num_tokens_post_pad); - -void moe_lora_align_block_size( - torch::Tensor topk_ids, torch::Tensor token_lora_mapping, - int64_t num_experts, int64_t block_size, int64_t max_loras, - int64_t max_num_tokens_padded, int64_t max_num_m_blocks, - torch::Tensor sorted_token_ids, torch::Tensor expert_ids, - torch::Tensor num_tokens_post_pad, torch::Tensor adapter_enabled, - torch::Tensor lora_ids, std::optional maybe_expert_map); -#ifndef USE_ROCM -torch::Tensor moe_wna16_gemm(torch::Tensor input, torch::Tensor output, - torch::Tensor b_qweight, torch::Tensor b_scales, - std::optional b_qzeros, - std::optional topk_weights, - torch::Tensor sorted_token_ids, - torch::Tensor expert_ids, - torch::Tensor num_tokens_post_pad, int64_t top_k, - int64_t BLOCK_SIZE_M, int64_t BLOCK_SIZE_N, - int64_t BLOCK_SIZE_K, int64_t bit); - -std::tuple grouped_topk( - torch::Tensor const& scores, int64_t n_group, int64_t topk_group, - int64_t topk, bool renormalize, double routed_scaling_factor, - torch::Tensor const& bias, int64_t scoring_func); -#endif - -bool moe_permute_unpermute_supported(); - -int64_t moe_permute_sort_workspace_size(int64_t num_expanded_rows, - int64_t num_experts); - -void shuffle_rows(const torch::Tensor& input_tensor, - const torch::Tensor& dst2src_map, - torch::Tensor& output_tensor); - -#ifndef USE_ROCM -// DeepSeek V3 optimized router GEMM kernel for SM90+ -// Computes output = mat_a @ mat_b.T where: -// mat_a: [num_tokens, hidden_dim] in bf16 -// mat_b: [num_experts, hidden_dim] in bf16 -// output: [num_tokens, num_experts] in bf16 or fp32 -// Supports num_tokens in [1, 16], num_experts in {256, 384}, hidden_dim = 7168 -void dsv3_router_gemm(torch::Tensor& output, const torch::Tensor& mat_a, - const torch::Tensor& mat_b); -#endif diff --git a/csrc/moe/moe_permute_unpermute_op.cu b/csrc/moe/moe_permute_unpermute_op.cu deleted file mode 100644 index 6fce009ae6..0000000000 --- a/csrc/moe/moe_permute_unpermute_op.cu +++ /dev/null @@ -1,286 +0,0 @@ -#include -#include -#include -#include "permute_unpermute_kernels/moe_permute_unpermute_kernel.h" -#include "permute_unpermute_kernels/dispatch.h" -#include "core/registration.h" - -// moe_permute kernels require at least CUDA 12.0 -#if defined(CUDA_VERSION) && (CUDA_VERSION >= 12000) - -namespace { - -torch::Tensor maybe_allocate_tensor( - const std::optional& maybe_tensor, - at::IntArrayRef expected_sizes, torch::ScalarType dtype, c10::Device device, - char const* name) { - auto expected_numel = c10::multiply_integers(expected_sizes); - if (maybe_tensor.has_value()) { - auto tensor = maybe_tensor.value(); - TORCH_CHECK(tensor.device() == device, name, " must be on the same device"); - TORCH_CHECK(tensor.scalar_type() == dtype, name, " has incorrect dtype"); - TORCH_CHECK(tensor.is_contiguous(), name, " must be contiguous"); - TORCH_CHECK(tensor.numel() >= expected_numel, name, - " is too small for the requested shape"); - auto flat_tensor = tensor.view({tensor.numel()}); - return flat_tensor.narrow(0, 0, expected_numel).view(expected_sizes); - } - return torch::empty(expected_sizes, torch::dtype(dtype).device(device)); -} - -} // namespace - -int64_t moe_permute_sort_workspace_size(int64_t num_expanded_rows, - int64_t n_expert) { - return static_cast( - CubKeyValueSorter::getWorkspaceSize(num_expanded_rows, n_expert)); -} - -void moe_permute_impl( - const torch::Tensor& input, // [n_token, hidden] - const torch::Tensor& topk_ids, // [n_token, topk] - const torch::Tensor& token_expert_indices, // [n_token, topk] - const std::optional& expert_map, // [n_expert] - int64_t n_expert, int64_t n_local_expert, int64_t topk, - torch::Tensor& permuted_input, // [permuted_size, hidden] - torch::Tensor& expert_first_token_offset, // [n_local_expert + 1] - torch::Tensor& inv_permuted_idx, // [n_token, topk] - torch::Tensor& permuted_idx, // [permute_size] - const std::optional& maybe_sort_workspace, - const std::optional& maybe_permuted_experts_id, - const std::optional& maybe_sorted_row_idx, - const std::optional& maybe_topk_ids_for_sort) { - TORCH_CHECK(expert_first_token_offset.scalar_type() == at::ScalarType::Long, - "expert_first_token_offset must be int64"); - TORCH_CHECK(topk_ids.scalar_type() == at::ScalarType::Int, - "topk_ids must be int32"); - TORCH_CHECK(token_expert_indices.scalar_type() == at::ScalarType::Int, - "token_expert_indices must be int32"); - TORCH_CHECK(inv_permuted_idx.scalar_type() == at::ScalarType::Int, - "inv_permuted_idx must be int32"); - TORCH_CHECK(expert_first_token_offset.size(0) == n_local_expert + 1, - "expert_first_token_offset shape != n_local_expert+1"); - TORCH_CHECK(inv_permuted_idx.sizes() == token_expert_indices.sizes(), - "token_expert_indices shape must be same as inv_permuted_idx"); - auto device = input.device(); - auto n_token = input.sizes()[0]; - auto n_hidden = input.sizes()[1]; - auto expanded_rows = n_token * topk; - auto stream = at::cuda::getCurrentCUDAStream().stream(); - - auto sorter_size = moe_permute_sort_workspace_size(expanded_rows, n_expert); - auto sort_workspace = - maybe_allocate_tensor(maybe_sort_workspace, {sorter_size}, torch::kInt8, - device, "sort_workspace"); - auto permuted_experts_id = - maybe_allocate_tensor(maybe_permuted_experts_id, topk_ids.sizes(), - at::ScalarType::Int, device, "permuted_experts_id"); - auto sorted_row_idx = - maybe_allocate_tensor(maybe_sorted_row_idx, inv_permuted_idx.sizes(), - at::ScalarType::Int, device, "sorted_row_idx"); - - CubKeyValueSorter sorter{}; - int64_t* valid_num_ptr = nullptr; - torch::Tensor topk_ids_for_sort = topk_ids; - - if (expert_map.has_value()) { - const int* expert_map_ptr = get_ptr(expert_map.value()); - valid_num_ptr = - get_ptr(expert_first_token_offset) + n_local_expert; - topk_ids_for_sort = - maybe_allocate_tensor(maybe_topk_ids_for_sort, topk_ids.sizes(), - at::ScalarType::Int, device, "topk_ids_for_sort"); - topk_ids_for_sort.copy_(topk_ids); - preprocessTopkIdLauncher(get_ptr(topk_ids_for_sort), n_token * topk, - expert_map_ptr, n_expert, stream); - } - - sortAndScanExpert( - get_ptr(topk_ids_for_sort), get_ptr(token_expert_indices), - get_ptr(permuted_experts_id), get_ptr(sorted_row_idx), - get_ptr(expert_first_token_offset), n_token, n_expert, - n_local_expert, topk, sorter, get_ptr(sort_workspace), stream); - - MOE_DISPATCH(input.scalar_type(), [&] { - expandInputRowsKernelLauncher( - get_ptr(input), get_ptr(permuted_input), - get_ptr(sorted_row_idx), get_ptr(inv_permuted_idx), - get_ptr(permuted_idx), get_ptr(expert_first_token_offset), - n_token, valid_num_ptr, n_hidden, topk, n_local_expert, stream); - }); -} - -void moe_permute( - const torch::Tensor& input, // [n_token, hidden] - const torch::Tensor& topk_ids, // [n_token, topk] - const torch::Tensor& token_expert_indices, // [n_token, topk] - const std::optional& expert_map, // [n_expert] - int64_t n_expert, int64_t n_local_expert, int64_t topk, - torch::Tensor& permuted_input, // [permuted_size, hidden] - torch::Tensor& expert_first_token_offset, // [n_local_expert + 1] - torch::Tensor& inv_permuted_idx, // [n_token, topk] - torch::Tensor& permuted_idx) { // [permute_size] - moe_permute_impl(input, topk_ids, token_expert_indices, expert_map, n_expert, - n_local_expert, topk, permuted_input, - expert_first_token_offset, inv_permuted_idx, permuted_idx, - std::nullopt, std::nullopt, std::nullopt, std::nullopt); -} - -void moe_permute_with_scratch( - const torch::Tensor& input, const torch::Tensor& topk_ids, - const torch::Tensor& token_expert_indices, - const std::optional& expert_map, int64_t n_expert, - int64_t n_local_expert, int64_t topk, torch::Tensor& permuted_input, - torch::Tensor& expert_first_token_offset, torch::Tensor& inv_permuted_idx, - torch::Tensor& permuted_idx, torch::Tensor& sort_workspace, - torch::Tensor& permuted_experts_id, torch::Tensor& sorted_row_idx, - torch::Tensor& topk_ids_for_sort) { - moe_permute_impl(input, topk_ids, token_expert_indices, expert_map, n_expert, - n_local_expert, topk, permuted_input, - expert_first_token_offset, inv_permuted_idx, permuted_idx, - sort_workspace, permuted_experts_id, sorted_row_idx, - topk_ids_for_sort); -} - -void moe_unpermute( - const torch::Tensor& permuted_hidden_states, // [n_token * topk, hidden] - const torch::Tensor& topk_weights, // [n_token, topk] - const torch::Tensor& inv_permuted_idx, // [n_token, topk] - const std::optional& - expert_first_token_offset, // [n_local_expert+1] - int64_t topk, - torch::Tensor& hidden_states // [n_token, hidden] -) { - TORCH_CHECK( - permuted_hidden_states.scalar_type() == hidden_states.scalar_type(), - "permuted_hidden_states dtype must be same as hidden_states"); - auto n_token = hidden_states.size(0); - auto n_hidden = hidden_states.size(1); - auto stream = at::cuda::getCurrentCUDAStream().stream(); - - int64_t const* valid_ptr = nullptr; - if (expert_first_token_offset.has_value()) { - int n_local_expert = expert_first_token_offset.value().size(0) - 1; - valid_ptr = - get_ptr(expert_first_token_offset.value()) + n_local_expert; - } - - MOE_DISPATCH(hidden_states.scalar_type(), [&] { - finalizeMoeRoutingKernelLauncher( - get_ptr(permuted_hidden_states), - get_ptr(hidden_states), get_ptr(topk_weights), - get_ptr(inv_permuted_idx), n_token, n_hidden, topk, valid_ptr, - stream); - }); -} - -template -__global__ void shuffleInputRowsKernel(const T* input, - const int32_t* dst2src_map, T* output, - int64_t num_src_rows, - int64_t num_dst_rows, int64_t num_cols) { - int64_t dest_row_idx = blockIdx.x; - int64_t const source_row_idx = dst2src_map[dest_row_idx]; - - if (blockIdx.x < num_dst_rows) { - // Load 128-bits per thread - constexpr int64_t ELEM_PER_THREAD = 128 / sizeof(T) / 8; - using DataElem = cutlass::Array; - - // Duplicate and permute rows - auto const* source_row_ptr = - reinterpret_cast(input + source_row_idx * num_cols); - auto* dest_row_ptr = - reinterpret_cast(output + dest_row_idx * num_cols); - - int64_t const start_offset = threadIdx.x; - int64_t const stride = blockDim.x; - int64_t const num_elems_in_col = num_cols / ELEM_PER_THREAD; - - for (int elem_index = start_offset; elem_index < num_elems_in_col; - elem_index += stride) { - dest_row_ptr[elem_index] = source_row_ptr[elem_index]; - } - } -} - -void shuffle_rows(const torch::Tensor& input_tensor, - const torch::Tensor& dst2src_map, - torch::Tensor& output_tensor) { - TORCH_CHECK(input_tensor.scalar_type() == output_tensor.scalar_type(), - "Input and output tensors must have the same data type"); - - auto stream = at::cuda::getCurrentCUDAStream().stream(); - int64_t const blocks = output_tensor.size(0); - int64_t const threads = 256; - int64_t const num_dest_rows = output_tensor.size(0); - int64_t const num_src_rows = input_tensor.size(0); - int64_t const num_cols = input_tensor.size(1); - - TORCH_CHECK(!(num_cols % (128 / sizeof(input_tensor.scalar_type()) / 8)), - "num_cols must be divisible by 128 / " - "sizeof(input_tensor.scalar_type()) / 8"); - - MOE_DISPATCH(input_tensor.scalar_type(), [&] { - shuffleInputRowsKernel<<>>( - reinterpret_cast(input_tensor.data_ptr()), - dst2src_map.data_ptr(), - reinterpret_cast(output_tensor.data_ptr()), num_src_rows, - num_dest_rows, num_cols); - }); -} - -#else - -int64_t moe_permute_sort_workspace_size(int64_t num_expanded_rows, - int64_t n_expert) { - TORCH_CHECK( - false, "moe_permute_sort_workspace_size is not supported on CUDA < 12.0"); -} - -void moe_permute(const torch::Tensor& input, const torch::Tensor& topk_ids, - const torch::Tensor& token_expert_indices, - const std::optional& expert_map, - int64_t n_expert, int64_t n_local_expert, int64_t topk, - torch::Tensor& permuted_input, - torch::Tensor& expert_first_token_offset, - torch::Tensor& inv_permuted_idx, torch::Tensor& permuted_idx) { - TORCH_CHECK(false, "moe_permute is not supported on CUDA < 12.0"); -} - -void moe_permute_with_scratch( - const torch::Tensor& input, const torch::Tensor& topk_ids, - const torch::Tensor& token_expert_indices, - const std::optional& expert_map, int64_t n_expert, - int64_t n_local_expert, int64_t topk, torch::Tensor& permuted_input, - torch::Tensor& expert_first_token_offset, torch::Tensor& inv_permuted_idx, - torch::Tensor& permuted_idx, torch::Tensor& sort_workspace, - torch::Tensor& permuted_experts_id, torch::Tensor& sorted_row_idx, - torch::Tensor& topk_ids_for_sort) { - TORCH_CHECK(false, - "moe_permute_with_scratch is not supported on CUDA < 12.0"); -} - -void moe_unpermute( - const torch::Tensor& permuted_hidden_states, - const torch::Tensor& topk_weights, const torch::Tensor& inv_permuted_idx, - const std::optional& expert_first_token_offset, int64_t topk, - torch::Tensor& hidden_states) { - TORCH_CHECK(false, "moe_unpermute is not supported on CUDA < 12.0"); -} - -#endif - -bool moe_permute_unpermute_supported() { -#if defined(CUDA_VERSION) && (CUDA_VERSION >= 12000) - return true; -#else - return false; -#endif -} - -TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) { - m.impl("moe_permute", &moe_permute); - m.impl("moe_permute_with_scratch", &moe_permute_with_scratch); - m.impl("moe_unpermute", &moe_unpermute); -} \ No newline at end of file diff --git a/csrc/moe/permute_unpermute_kernels/dispatch.h b/csrc/moe/permute_unpermute_kernels/dispatch.h deleted file mode 100644 index d0f1ea4ade..0000000000 --- a/csrc/moe/permute_unpermute_kernels/dispatch.h +++ /dev/null @@ -1,59 +0,0 @@ -#pragma once -#include -#define MOE_SWITCH(TYPE, ...) \ - at::ScalarType _st = ::detail::scalar_type(TYPE); \ - switch (_st) { \ - __VA_ARGS__ \ - default: \ - TORCH_CHECK(false, "[moe permute]data type dispatch fail!") \ - } - -#define MOE_DISPATCH_CASE(enum_type, ...) \ - case enum_type: { \ - using scalar_t = ScalarType2CudaType::type; \ - __VA_ARGS__(); \ - break; \ - } -#define MOE_DISPATCH_FLOAT_CASE(...) \ - MOE_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \ - MOE_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \ - MOE_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) \ - MOE_DISPATCH_CASE(at::ScalarType::Float8_e5m2, __VA_ARGS__) \ - MOE_DISPATCH_CASE(at::ScalarType::Float8_e4m3fn, __VA_ARGS__) \ - MOE_DISPATCH_CASE(at::ScalarType::Byte, __VA_ARGS__) - -#define MOE_DISPATCH(TYPE, ...) \ - MOE_SWITCH(TYPE, MOE_DISPATCH_FLOAT_CASE(__VA_ARGS__)) - -template -struct ScalarType2CudaType; - -template <> -struct ScalarType2CudaType { - using type = float; -}; -template <> -struct ScalarType2CudaType { - using type = half; -}; -template <> -struct ScalarType2CudaType { - using type = __nv_bfloat16; -}; -// uint8 for packed fp4 -template <> -struct ScalarType2CudaType { - using type = uint8_t; -}; - -// #if __CUDA_ARCH__ >= 890 -// fp8 -template <> -struct ScalarType2CudaType { - using type = __nv_fp8_e5m2; -}; -template <> -struct ScalarType2CudaType { - using type = __nv_fp8_e4m3; -}; -// #endif \ No newline at end of file diff --git a/csrc/torch_bindings.cpp b/csrc/torch_bindings.cpp index c63e59c3b0..58524c4c5d 100644 --- a/csrc/torch_bindings.cpp +++ b/csrc/torch_bindings.cpp @@ -130,25 +130,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) { "Tensor? qzeros_or_none, bool inplace) -> Tensor"); // conditionally compiled so impl registrations are in source file -#endif - -#ifndef USE_ROCM - // Expert-specialization mxfp8 blockscaled grouped quantization (SM100+). - ops.def( - "mxfp8_experts_quant(" - " Tensor input, Tensor problem_sizes, Tensor expert_offsets," - " Tensor blockscale_offsets, Tensor! quant_output, Tensor! scale_factor)" - " -> ()"); - // conditionally compiled so impl registration is in source file - - // Expert-specialization mxfp8 blockscaled grouped GEMM (SM100+). - ops.def( - "cutlass_mxfp8_grouped_mm(" - " Tensor a, Tensor b, Tensor sfa, Tensor sfb, Tensor! out," - " Tensor problem_sizes, Tensor expert_offsets, Tensor blockscale_offsets)" - " -> ()"); - // conditionally compiled so impl registration is in source file - #endif } diff --git a/setup.py b/setup.py index 1df47b4e7d..a5b919f383 100644 --- a/setup.py +++ b/setup.py @@ -755,7 +755,7 @@ class precompiled_wheel_utils: { "vllm/_C.abi3.so", "vllm/_C_stable_libtorch.abi3.so", - "vllm/_moe_C.abi3.so", + "vllm/_moe_C_stable_libtorch.abi3.so", "vllm/_flashmla_C.abi3.so", "vllm/_flashmla_extension_C.abi3.so", "vllm/_sparse_flashmla_C.abi3.so", @@ -1081,7 +1081,6 @@ def get_requirements() -> list[str]: ext_modules = [] if _is_cuda() or _is_hip(): - ext_modules.append(CMakeExtension(name="vllm._moe_C")) ext_modules.append(CMakeExtension(name="vllm.cumem_allocator")) # Optional since this doesn't get built (produce an .so file). This is just # copying the relevant .py files from the source repository. @@ -1135,6 +1134,7 @@ if _build_custom_ops(): ext_modules.append(CMakeExtension(name="vllm._C")) if _is_cuda() or _is_hip(): ext_modules.append(CMakeExtension(name="vllm._C_stable_libtorch")) + ext_modules.append(CMakeExtension(name="vllm._moe_C_stable_libtorch")) package_data = { "vllm": [ diff --git a/vllm/platforms/interface.py b/vllm/platforms/interface.py index 413234b402..a725b6f9d3 100644 --- a/vllm/platforms/interface.py +++ b/vllm/platforms/interface.py @@ -254,7 +254,7 @@ class Platform: except ImportError as e: logger.warning("Failed to import from vllm._C: %r", e) with contextlib.suppress(ImportError): - import vllm._moe_C # noqa: F401 + import vllm._moe_C_stable_libtorch # noqa: F401 @classmethod def get_attn_backend_cls(