[10c/n] Migrate MoE kernels to torch stable ABI (#44565)
Signed-off-by: Chris Leonard <chleonar@redhat.com> Co-authored-by: Shengqi Chen <harry-chen@outlook.com>
This commit is contained in:
co-authored by
Shengqi Chen
parent
2f2c5cf4f1
commit
6e64c1bab1
+1
-1
@@ -233,7 +233,7 @@ actionlint
|
||||
shellcheck*/
|
||||
|
||||
# Ignore moe/marlin_moe gen code
|
||||
csrc/moe/marlin_moe_wna16/kernel_*
|
||||
csrc/libtorch_stable/moe/marlin_moe_wna16/kernel_*
|
||||
|
||||
# Ignore ep_kernels_workspace folder
|
||||
ep_kernels_workspace/
|
||||
|
||||
@@ -21,7 +21,7 @@ repos:
|
||||
rev: v21.1.2
|
||||
hooks:
|
||||
- id: clang-format
|
||||
exclude: 'csrc/(moe/topk_softmax_kernels.cu|libtorch_stable/quantization/gguf/(ggml-common.h|dequantize.cuh|vecdotq.cuh|mmq.cuh|mmvq.cuh))|vllm/third_party/.*'
|
||||
exclude: 'csrc/(libtorch_stable/moe/topk_softmax_kernels.cu|libtorch_stable/quantization/gguf/(ggml-common.h|dequantize.cuh|vecdotq.cuh|mmq.cuh|mmvq.cuh))|vllm/third_party/.*'
|
||||
types_or: [c++, cuda]
|
||||
args: [--style=file, --verbose]
|
||||
- repo: https://github.com/DavidAnson/markdownlint-cli2
|
||||
|
||||
+61
-20
@@ -1115,25 +1115,25 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
|
||||
endif()
|
||||
|
||||
#
|
||||
# _moe_C extension
|
||||
# _moe_C_stable_libtorch extension
|
||||
#
|
||||
|
||||
set(VLLM_MOE_EXT_SRC
|
||||
"csrc/moe/torch_bindings.cpp"
|
||||
"csrc/moe/moe_align_sum_kernels.cu"
|
||||
"csrc/moe/topk_softmax_kernels.cu"
|
||||
"csrc/moe/topk_softplus_sqrt_kernels.cu")
|
||||
"csrc/libtorch_stable/moe/torch_bindings.cpp"
|
||||
"csrc/libtorch_stable/moe/moe_align_sum_kernels.cu"
|
||||
"csrc/libtorch_stable/moe/topk_softmax_kernels.cu"
|
||||
"csrc/libtorch_stable/moe/topk_softplus_sqrt_kernels.cu")
|
||||
|
||||
if(VLLM_GPU_LANG STREQUAL "CUDA")
|
||||
list(APPEND VLLM_MOE_EXT_SRC
|
||||
"csrc/moe/moe_wna16.cu"
|
||||
"csrc/moe/grouped_topk_kernels.cu")
|
||||
"csrc/libtorch_stable/moe/moe_wna16.cu"
|
||||
"csrc/libtorch_stable/moe/grouped_topk_kernels.cu")
|
||||
endif()
|
||||
|
||||
if(VLLM_GPU_LANG STREQUAL "CUDA")
|
||||
set(MOE_PERMUTE_SRC
|
||||
"csrc/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.cu"
|
||||
"csrc/moe/moe_permute_unpermute_op.cu")
|
||||
"csrc/libtorch_stable/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.cu"
|
||||
"csrc/libtorch_stable/moe/moe_permute_unpermute_op.cu")
|
||||
|
||||
list(APPEND VLLM_MOE_EXT_SRC "${MOE_PERMUTE_SRC}")
|
||||
endif()
|
||||
@@ -1144,7 +1144,7 @@ set_gencode_flags_for_srcs(
|
||||
|
||||
if(VLLM_GPU_LANG STREQUAL "CUDA")
|
||||
set(VLLM_MOE_WNA16_SRC
|
||||
"csrc/moe/moe_wna16.cu")
|
||||
"csrc/libtorch_stable/moe/moe_wna16.cu")
|
||||
|
||||
set_gencode_flags_for_srcs(
|
||||
SRCS "${VLLM_MOE_WNA16_SRC}"
|
||||
@@ -1175,7 +1175,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
|
||||
# preselected input type pairs and schedules.
|
||||
# Generate sources:
|
||||
set(MOE_MARLIN_GEN_SCRIPT
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/csrc/moe/marlin_moe_wna16/generate_kernels.py)
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/csrc/libtorch_stable/moe/marlin_moe_wna16/generate_kernels.py)
|
||||
file(MD5 ${MOE_MARLIN_GEN_SCRIPT} MOE_MARLIN_GEN_SCRIPT_HASH)
|
||||
list(JOIN CUDA_ARCHS "," CUDA_ARCHS_STR)
|
||||
set(MOE_MARLIN_GEN_SCRIPT_HASH_AND_ARCH "${MOE_MARLIN_GEN_SCRIPT_HASH}(ARCH:${CUDA_ARCHS_STR})")
|
||||
@@ -1210,7 +1210,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
|
||||
endif()
|
||||
|
||||
if (MARLIN_MOE_ARCHS)
|
||||
file(GLOB MARLIN_MOE_SRC "csrc/moe/marlin_moe_wna16/sm80_kernel_*.cu")
|
||||
file(GLOB MARLIN_MOE_SRC "csrc/libtorch_stable/moe/marlin_moe_wna16/sm80_kernel_*.cu")
|
||||
set_gencode_flags_for_srcs(
|
||||
SRCS "${MARLIN_MOE_SRC}"
|
||||
CUDA_ARCHS "${MARLIN_MOE_ARCHS}")
|
||||
@@ -1222,7 +1222,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
|
||||
endif()
|
||||
|
||||
if (MARLIN_MOE_SM75_ARCHS)
|
||||
file(GLOB MARLIN_MOE_SM75_SRC "csrc/moe/marlin_moe_wna16/sm75_kernel_*.cu")
|
||||
file(GLOB MARLIN_MOE_SM75_SRC "csrc/libtorch_stable/moe/marlin_moe_wna16/sm75_kernel_*.cu")
|
||||
set_gencode_flags_for_srcs(
|
||||
SRCS "${MARLIN_MOE_SM75_SRC}"
|
||||
CUDA_ARCHS "${MARLIN_MOE_SM75_ARCHS}")
|
||||
@@ -1234,7 +1234,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
|
||||
endif()
|
||||
|
||||
if (MARLIN_MOE_FP8_ARCHS)
|
||||
file(GLOB MARLIN_MOE_FP8_SRC "csrc/moe/marlin_moe_wna16/sm89_kernel_*.cu")
|
||||
file(GLOB MARLIN_MOE_FP8_SRC "csrc/libtorch_stable/moe/marlin_moe_wna16/sm89_kernel_*.cu")
|
||||
set_gencode_flags_for_srcs(
|
||||
SRCS "${MARLIN_MOE_FP8_SRC}"
|
||||
CUDA_ARCHS "${MARLIN_MOE_FP8_ARCHS}")
|
||||
@@ -1245,7 +1245,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
|
||||
list(APPEND VLLM_MOE_EXT_SRC ${MARLIN_MOE_FP8_SRC})
|
||||
endif()
|
||||
|
||||
set(MARLIN_MOE_OTHER_SRC "csrc/moe/marlin_moe_wna16/ops.cu")
|
||||
set(MARLIN_MOE_OTHER_SRC "csrc/libtorch_stable/moe/marlin_moe_wna16/ops.cu")
|
||||
set_gencode_flags_for_srcs(
|
||||
SRCS "${MARLIN_MOE_OTHER_SRC}"
|
||||
CUDA_ARCHS "${MARLIN_MOE_OTHER_ARCHS}")
|
||||
@@ -1266,9 +1266,9 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
|
||||
cuda_archs_sm90plus(SM90PLUS_ROUTER_GEMM_ARCHS "${CUDA_ARCHS}")
|
||||
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.0 AND SM90PLUS_ROUTER_GEMM_ARCHS)
|
||||
set(DSV3_ROUTER_GEMM_SRC
|
||||
"csrc/moe/dsv3_router_gemm_entry.cu"
|
||||
"csrc/moe/dsv3_router_gemm_float_out.cu"
|
||||
"csrc/moe/dsv3_router_gemm_bf16_out.cu")
|
||||
"csrc/libtorch_stable/moe/dsv3_router_gemm_entry.cu"
|
||||
"csrc/libtorch_stable/moe/dsv3_router_gemm_float_out.cu"
|
||||
"csrc/libtorch_stable/moe/dsv3_router_gemm_bf16_out.cu")
|
||||
set_gencode_flags_for_srcs(
|
||||
SRCS "${DSV3_ROUTER_GEMM_SRC}"
|
||||
CUDA_ARCHS "${SM90PLUS_ROUTER_GEMM_ARCHS}")
|
||||
@@ -1281,9 +1281,9 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
message(STATUS "Enabling moe extension.")
|
||||
message(STATUS "Enabling MoE C_stable extension.")
|
||||
define_extension_target(
|
||||
_moe_C
|
||||
_moe_C_stable_libtorch
|
||||
DESTINATION vllm
|
||||
LANGUAGE ${VLLM_GPU_LANG}
|
||||
SOURCES ${VLLM_MOE_EXT_SRC}
|
||||
@@ -1294,6 +1294,47 @@ define_extension_target(
|
||||
USE_SABI 3
|
||||
WITH_SOABI)
|
||||
|
||||
# Needed to use cuda/hip APIs from C-shim
|
||||
if(VLLM_GPU_LANG STREQUAL "CUDA")
|
||||
# Set TORCH_TARGET_VERSION for stable ABI compatibility.
|
||||
# This ensures we only use C-shim APIs available in PyTorch 2.11.
|
||||
# _moe_C_stable_libtorch is abi compatible with PyTorch >= TORCH_TARGET_VERSION
|
||||
# which is currently set to 2.11.
|
||||
target_compile_definitions(_moe_C_stable_libtorch PRIVATE
|
||||
TORCH_TARGET_VERSION=0x020B000000000000ULL)
|
||||
target_compile_definitions(_moe_C_stable_libtorch PRIVATE USE_CUDA)
|
||||
# Needed by CUTLASS kernels
|
||||
target_compile_definitions(_moe_C_stable_libtorch PRIVATE
|
||||
CUTLASS_ENABLE_DIRECT_CUDA_DRIVER_CALL=1)
|
||||
elseif(VLLM_GPU_LANG STREQUAL "HIP")
|
||||
# Set TORCH_TARGET_VERSION for stable ABI compatibility.
|
||||
# This ensures we only use C-shim APIs available in PyTorch 2.10.
|
||||
# _moe_C_stable_libtorch is abi compatible with PyTorch >= TORCH_TARGET_VERSION
|
||||
# which is currently set to 2.10.
|
||||
target_compile_definitions(_moe_C_stable_libtorch PRIVATE
|
||||
TORCH_TARGET_VERSION=0x020A000000000000ULL)
|
||||
target_compile_definitions(_moe_C_stable_libtorch PRIVATE USE_ROCM)
|
||||
endif()
|
||||
|
||||
# On ROCm, _moe_C_stable_libtorch calls raw HIP APIs (e.g. hipGetDevice in
|
||||
# get_device_prop()) which must resolve to the same libamdhip64.so that
|
||||
# PyTorch uses. When PyTorch bundles its own copy (pip/conda wheels),
|
||||
# the raw HIP calls would otherwise resolve to the system ROCm copy,
|
||||
# initializing a second HIP runtime that corrupts device state (wrong
|
||||
# device on DeviceGuard, core dumps on multi-GPU tests).
|
||||
#
|
||||
# If PyTorch doesn't bundle libamdhip64 (built from source against system
|
||||
# ROCm), there is only one copy in the process and no action is needed —
|
||||
# the HIP compiler already links the system libamdhip64 automatically.
|
||||
if(VLLM_GPU_LANG STREQUAL "HIP")
|
||||
find_library(_MOE_STABLE_TORCH_AMDHIP64 amdhip64
|
||||
PATHS "${TORCH_INSTALL_PREFIX}/lib" NO_DEFAULT_PATH)
|
||||
if(_MOE_STABLE_TORCH_AMDHIP64)
|
||||
message(STATUS "Found PyTorch-bundled libamdhip64 for _moe_C_stable_libtorch at ${_MOE_STABLE_TORCH_AMDHIP64}")
|
||||
target_link_libraries(_moe_C_stable_libtorch PRIVATE ${_MOE_STABLE_TORCH_AMDHIP64})
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(VLLM_GPU_LANG STREQUAL "HIP")
|
||||
#
|
||||
# _rocm_C extension
|
||||
|
||||
@@ -30,6 +30,28 @@
|
||||
THO_DISPATCH_SWITCH(TYPE, NAME, \
|
||||
VLLM_STABLE_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
|
||||
|
||||
#define VLLM_STABLE_DISPATCH_CASE_INTEGRAL_TYPES(...) \
|
||||
THO_DISPATCH_CASE(torch::headeronly::ScalarType::Byte, __VA_ARGS__) \
|
||||
THO_DISPATCH_CASE(torch::headeronly::ScalarType::Char, __VA_ARGS__) \
|
||||
THO_DISPATCH_CASE(torch::headeronly::ScalarType::Short, __VA_ARGS__) \
|
||||
THO_DISPATCH_CASE(torch::headeronly::ScalarType::Int, __VA_ARGS__) \
|
||||
THO_DISPATCH_CASE(torch::headeronly::ScalarType::Long, __VA_ARGS__)
|
||||
|
||||
#define VLLM_STABLE_DISPATCH_CASE_INTEGRAL_AND_UNSIGNED_TYPES(...) \
|
||||
VLLM_STABLE_DISPATCH_CASE_INTEGRAL_TYPES(__VA_ARGS__) \
|
||||
THO_DISPATCH_CASE(torch::headeronly::ScalarType::UInt16, __VA_ARGS__) \
|
||||
THO_DISPATCH_CASE(torch::headeronly::ScalarType::UInt32, __VA_ARGS__) \
|
||||
THO_DISPATCH_CASE(torch::headeronly::ScalarType::UInt64, __VA_ARGS__)
|
||||
|
||||
#define VLLM_STABLE_DISPATCH_INTEGRAL_TYPES(TYPE, NAME, ...) \
|
||||
THO_DISPATCH_SWITCH(TYPE, NAME, \
|
||||
VLLM_STABLE_DISPATCH_CASE_INTEGRAL_TYPES(__VA_ARGS__))
|
||||
|
||||
#define VLLM_STABLE_DISPATCH_INTEGRAL_AND_UNSIGNED_TYPES(TYPE, NAME, ...) \
|
||||
THO_DISPATCH_SWITCH( \
|
||||
TYPE, NAME, \
|
||||
VLLM_STABLE_DISPATCH_CASE_INTEGRAL_AND_UNSIGNED_TYPES(__VA_ARGS__))
|
||||
|
||||
// FP8 type dispatch - ROCm uses FNUZ format, CUDA uses OCP format
|
||||
#ifdef USE_ROCM
|
||||
#define VLLM_STABLE_DISPATCH_CASE_FP8_TYPES(...) \
|
||||
|
||||
+1
-4
@@ -18,14 +18,11 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include <ATen/ATen.h>
|
||||
#include <ATen/cuda/CUDAContext.h>
|
||||
#include <cstdint>
|
||||
|
||||
#include <cuda_bf16.h>
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
#include "dsv3_router_gemm_utils.h"
|
||||
|
||||
// Custom FMA implementation using PTX assembly instructions
|
||||
__device__ __forceinline__ void fma(float2& d, float2 const& a, float2 const& b,
|
||||
float2 const& c) {
|
||||
+47
-29
@@ -18,15 +18,25 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include <ATen/ATen.h>
|
||||
#include <ATen/cuda/CUDAContext.h>
|
||||
#include <torch/all.h>
|
||||
#include <torch/csrc/stable/library.h>
|
||||
#include <torch/csrc/stable/tensor.h>
|
||||
#include <torch/headeronly/core/ScalarType.h>
|
||||
|
||||
#include "libtorch_stable/torch_utils.h"
|
||||
|
||||
#include <cuda_bf16.h>
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
#include "core/registration.h"
|
||||
#include "dsv3_router_gemm_utils.h"
|
||||
#include <stdexcept>
|
||||
|
||||
namespace {
|
||||
|
||||
inline int getSMVersion() {
|
||||
auto* props = get_device_prop();
|
||||
return props->major * 10 + props->minor;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
static constexpr int DEFAULT_NUM_EXPERTS = 256;
|
||||
static constexpr int KIMI_K2_NUM_EXPERTS = 384;
|
||||
@@ -98,40 +108,48 @@ struct LoopUnroller<kEnd, kEnd, kNumExperts, kHiddenDim> {
|
||||
}
|
||||
};
|
||||
|
||||
void dsv3_router_gemm(at::Tensor& output, // [num_tokens, num_experts]
|
||||
const at::Tensor& mat_a, // [num_tokens, hidden_dim]
|
||||
const at::Tensor& mat_b // [num_experts, hidden_dim]
|
||||
void dsv3_router_gemm(
|
||||
torch::stable::Tensor& output, // [num_tokens, num_experts]
|
||||
torch::stable::Tensor const& mat_a, // [num_tokens, hidden_dim]
|
||||
torch::stable::Tensor const& mat_b // [num_experts, hidden_dim]
|
||||
) {
|
||||
TORCH_CHECK(output.dim() == 2 && mat_a.dim() == 2 && mat_b.dim() == 2);
|
||||
STD_TORCH_CHECK(output.dim() == 2 && mat_a.dim() == 2 && mat_b.dim() == 2);
|
||||
|
||||
const int num_tokens = mat_a.size(0);
|
||||
const int num_experts = mat_b.size(0);
|
||||
const int hidden_dim = mat_a.size(1);
|
||||
|
||||
TORCH_CHECK(mat_a.size(1) == mat_b.size(1),
|
||||
"mat_a and mat_b must have the same hidden_dim");
|
||||
TORCH_CHECK(hidden_dim == DEFAULT_HIDDEN_DIM,
|
||||
"Expected hidden_dim=", DEFAULT_HIDDEN_DIM,
|
||||
", but got hidden_dim=", hidden_dim);
|
||||
TORCH_CHECK(
|
||||
STD_TORCH_CHECK(mat_a.size(1) == mat_b.size(1),
|
||||
"mat_a and mat_b must have the same hidden_dim");
|
||||
STD_TORCH_CHECK(hidden_dim == DEFAULT_HIDDEN_DIM,
|
||||
"Expected hidden_dim=", DEFAULT_HIDDEN_DIM,
|
||||
", but got hidden_dim=", hidden_dim);
|
||||
STD_TORCH_CHECK(
|
||||
num_experts == DEFAULT_NUM_EXPERTS || num_experts == KIMI_K2_NUM_EXPERTS,
|
||||
"Expected num_experts=", DEFAULT_NUM_EXPERTS,
|
||||
" or num_experts=", KIMI_K2_NUM_EXPERTS,
|
||||
", but got num_experts=", num_experts);
|
||||
TORCH_CHECK(num_tokens >= 1 && num_tokens <= 16,
|
||||
"currently num_tokens must be less than or equal to 16 for "
|
||||
"router_gemm");
|
||||
TORCH_CHECK(mat_a.dtype() == at::kBFloat16, "mat_a must be bf16");
|
||||
TORCH_CHECK(mat_b.dtype() == at::kBFloat16, "mat_b must be bf16");
|
||||
TORCH_CHECK(output.dtype() == at::kFloat || output.dtype() == at::kBFloat16,
|
||||
"output must be float32 or bf16");
|
||||
STD_TORCH_CHECK(num_tokens >= 1 && num_tokens <= 16,
|
||||
"currently num_tokens must be less than or equal to 16 for "
|
||||
"router_gemm");
|
||||
STD_TORCH_CHECK(
|
||||
mat_a.scalar_type() == torch::headeronly::ScalarType::BFloat16,
|
||||
"mat_a must be bf16");
|
||||
STD_TORCH_CHECK(
|
||||
mat_b.scalar_type() == torch::headeronly::ScalarType::BFloat16,
|
||||
"mat_b must be bf16");
|
||||
STD_TORCH_CHECK(
|
||||
output.scalar_type() == torch::headeronly::ScalarType::Float ||
|
||||
output.scalar_type() == torch::headeronly::ScalarType::BFloat16,
|
||||
"output must be float32 or bf16");
|
||||
|
||||
auto const sm = getSMVersion();
|
||||
TORCH_CHECK(sm >= 90 && sm <= 103, "required SM_103 >= CUDA ARCH >= SM_90");
|
||||
const int sm = getSMVersion();
|
||||
STD_TORCH_CHECK(sm >= 90 && sm <= 103,
|
||||
"required SM_103 >= CUDA ARCH >= SM_90");
|
||||
|
||||
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
|
||||
const cudaStream_t stream = get_current_cuda_stream(mat_a.get_device_index());
|
||||
|
||||
if (output.dtype() == at::kFloat) {
|
||||
if (output.scalar_type() == torch::headeronly::ScalarType::Float) {
|
||||
if (num_experts == DEFAULT_NUM_EXPERTS) {
|
||||
LoopUnroller<1, 16, DEFAULT_NUM_EXPERTS, DEFAULT_HIDDEN_DIM>::
|
||||
unroll_float_output(
|
||||
@@ -145,7 +163,7 @@ void dsv3_router_gemm(at::Tensor& output, // [num_tokens, num_experts]
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()),
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), stream);
|
||||
}
|
||||
} else if (output.dtype() == at::kBFloat16) {
|
||||
} else if (output.scalar_type() == torch::headeronly::ScalarType::BFloat16) {
|
||||
if (num_experts == DEFAULT_NUM_EXPERTS) {
|
||||
LoopUnroller<1, 16, DEFAULT_NUM_EXPERTS, DEFAULT_HIDDEN_DIM>::
|
||||
unroll_bf16_output(
|
||||
@@ -164,6 +182,6 @@ void dsv3_router_gemm(at::Tensor& output, // [num_tokens, num_experts]
|
||||
}
|
||||
}
|
||||
|
||||
TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) {
|
||||
m.impl("dsv3_router_gemm", &dsv3_router_gemm);
|
||||
STABLE_TORCH_LIBRARY_IMPL(_moe_C, CUDA, m) {
|
||||
m.impl("dsv3_router_gemm", TORCH_BOX(&dsv3_router_gemm));
|
||||
}
|
||||
+1
-4
@@ -18,14 +18,11 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include <ATen/ATen.h>
|
||||
#include <ATen/cuda/CUDAContext.h>
|
||||
#include <cstdint>
|
||||
|
||||
#include <cuda_bf16.h>
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
#include "dsv3_router_gemm_utils.h"
|
||||
|
||||
// Custom FMA implementation using PTX assembly instructions
|
||||
__device__ __forceinline__ void fma(float2& d, float2 const& a, float2 const& b,
|
||||
float2 const& c) {
|
||||
+48
-40
@@ -18,9 +18,14 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
#include "moeTopKFuncs.cuh"
|
||||
#include <c10/cuda/CUDAStream.h>
|
||||
#include <torch/all.h>
|
||||
|
||||
#include <torch/csrc/stable/tensor.h>
|
||||
#include <torch/headeronly/core/ScalarType.h>
|
||||
|
||||
#include "libtorch_stable/torch_utils.h"
|
||||
|
||||
#include <cmath>
|
||||
#include <tuple>
|
||||
#include <cuda_fp16.h>
|
||||
#include <cuda_bf16.h>
|
||||
#include <cuda/std/limits>
|
||||
@@ -1001,38 +1006,40 @@ INSTANTIATE_NOAUX_TC(__nv_bfloat16, __nv_bfloat16, int32_t, SCORING_NONE);
|
||||
} // end namespace moe
|
||||
} // namespace vllm
|
||||
|
||||
std::tuple<torch::Tensor, torch::Tensor> grouped_topk(
|
||||
torch::Tensor const& scores, int64_t n_group, int64_t topk_group,
|
||||
std::tuple<torch::stable::Tensor, torch::stable::Tensor> grouped_topk(
|
||||
torch::stable::Tensor const& scores, int64_t n_group, int64_t topk_group,
|
||||
int64_t topk, bool renormalize, double routed_scaling_factor,
|
||||
torch::Tensor const& bias, int64_t scoring_func = 0) {
|
||||
auto data_type = scores.scalar_type();
|
||||
auto bias_type = bias.scalar_type();
|
||||
auto input_size = scores.sizes();
|
||||
int64_t num_tokens = input_size[0];
|
||||
int64_t num_experts = input_size[1];
|
||||
TORCH_CHECK(input_size.size() == 2, "scores must be a 2D Tensor");
|
||||
TORCH_CHECK(n_group > 0, "n_group must be positive");
|
||||
TORCH_CHECK(topk > 0, "topk must be positive");
|
||||
TORCH_CHECK(topk_group > 0, "topk_group must be positive");
|
||||
TORCH_CHECK(topk_group <= n_group, "topk_group must be <= n_group");
|
||||
TORCH_CHECK(num_experts % n_group == 0,
|
||||
"num_experts should be divisible by n_group");
|
||||
TORCH_CHECK(n_group <= 32,
|
||||
"n_group should be smaller than or equal to 32 for now");
|
||||
TORCH_CHECK(topk <= 32, "topk should be smaller than or equal to 32 for now");
|
||||
TORCH_CHECK(topk <= topk_group * (num_experts / n_group),
|
||||
"topk must be <= topk_group * (num_experts / n_group)");
|
||||
TORCH_CHECK(scoring_func == vllm::moe::SCORING_NONE ||
|
||||
scoring_func == vllm::moe::SCORING_SIGMOID,
|
||||
"scoring_func must be SCORING_NONE (0) or SCORING_SIGMOID (1)");
|
||||
torch::stable::Tensor const& bias, int64_t scoring_func = 0) {
|
||||
const auto data_type = scores.scalar_type();
|
||||
const auto bias_type = bias.scalar_type();
|
||||
STD_TORCH_CHECK(scores.dim() == 2, "scores must be a 2D Tensor");
|
||||
const int64_t num_tokens = scores.size(0);
|
||||
const int64_t num_experts = scores.size(1);
|
||||
STD_TORCH_CHECK(n_group > 0, "n_group must be positive");
|
||||
STD_TORCH_CHECK(topk > 0, "topk must be positive");
|
||||
STD_TORCH_CHECK(topk_group > 0, "topk_group must be positive");
|
||||
STD_TORCH_CHECK(topk_group <= n_group, "topk_group must be <= n_group");
|
||||
STD_TORCH_CHECK(num_experts % n_group == 0,
|
||||
"num_experts should be divisible by n_group");
|
||||
STD_TORCH_CHECK(n_group <= 32,
|
||||
"n_group should be smaller than or equal to 32 for now");
|
||||
STD_TORCH_CHECK(topk <= 32,
|
||||
"topk should be smaller than or equal to 32 for now");
|
||||
STD_TORCH_CHECK(topk <= topk_group * (num_experts / n_group),
|
||||
"topk must be <= topk_group * (num_experts / n_group)");
|
||||
STD_TORCH_CHECK(
|
||||
scoring_func == vllm::moe::SCORING_NONE ||
|
||||
scoring_func == vllm::moe::SCORING_SIGMOID,
|
||||
"scoring_func must be SCORING_NONE (0) or SCORING_SIGMOID (1)");
|
||||
|
||||
// Always output float32 for topk_values (eliminates Python-side conversion)
|
||||
torch::Tensor topk_values = torch::empty(
|
||||
{num_tokens, topk}, torch::dtype(torch::kFloat32).device(torch::kCUDA));
|
||||
torch::Tensor topk_indices = torch::empty(
|
||||
{num_tokens, topk}, torch::dtype(torch::kInt32).device(torch::kCUDA));
|
||||
auto topk_values = torch::stable::new_empty(
|
||||
scores, {num_tokens, topk}, torch::headeronly::ScalarType::Float);
|
||||
auto topk_indices = torch::stable::new_empty(
|
||||
scores, {num_tokens, topk}, torch::headeronly::ScalarType::Int);
|
||||
|
||||
auto stream = c10::cuda::getCurrentCUDAStream(scores.get_device());
|
||||
const cudaStream_t stream =
|
||||
get_current_cuda_stream(scores.get_device_index());
|
||||
auto const sf = static_cast<vllm::moe::ScoringFunc>(scoring_func);
|
||||
|
||||
#define LAUNCH_KERNEL_SF(T, BiasT, IdxT) \
|
||||
@@ -1057,7 +1064,7 @@ std::tuple<torch::Tensor, torch::Tensor> grouped_topk(
|
||||
routed_scaling_factor, false, stream); \
|
||||
break; \
|
||||
default: \
|
||||
throw std::invalid_argument("Unsupported scoring_func"); \
|
||||
STD_TORCH_CHECK(false, "Unsupported scoring_func"); \
|
||||
break; \
|
||||
} \
|
||||
} while (0)
|
||||
@@ -1065,17 +1072,18 @@ std::tuple<torch::Tensor, torch::Tensor> grouped_topk(
|
||||
#define LAUNCH_KERNEL(T, IdxT) \
|
||||
do { \
|
||||
switch (bias_type) { \
|
||||
case torch::kFloat16: \
|
||||
case torch::headeronly::ScalarType::Half: \
|
||||
LAUNCH_KERNEL_SF(T, half, IdxT); \
|
||||
break; \
|
||||
case torch::kFloat32: \
|
||||
case torch::headeronly::ScalarType::Float: \
|
||||
LAUNCH_KERNEL_SF(T, float, IdxT); \
|
||||
break; \
|
||||
case torch::kBFloat16: \
|
||||
case torch::headeronly::ScalarType::BFloat16: \
|
||||
LAUNCH_KERNEL_SF(T, __nv_bfloat16, IdxT); \
|
||||
break; \
|
||||
default: \
|
||||
throw std::invalid_argument( \
|
||||
STD_TORCH_CHECK( \
|
||||
false, \
|
||||
"Invalid bias dtype, only supports float16, float32, and " \
|
||||
"bfloat16"); \
|
||||
break; \
|
||||
@@ -1083,22 +1091,22 @@ std::tuple<torch::Tensor, torch::Tensor> grouped_topk(
|
||||
} while (0)
|
||||
|
||||
switch (data_type) {
|
||||
case torch::kFloat16:
|
||||
case torch::headeronly::ScalarType::Half:
|
||||
// Handle Float16
|
||||
LAUNCH_KERNEL(half, int32_t);
|
||||
break;
|
||||
case torch::kFloat32:
|
||||
case torch::headeronly::ScalarType::Float:
|
||||
// Handle Float32
|
||||
LAUNCH_KERNEL(float, int32_t);
|
||||
break;
|
||||
case torch::kBFloat16:
|
||||
case torch::headeronly::ScalarType::BFloat16:
|
||||
// Handle BFloat16
|
||||
LAUNCH_KERNEL(__nv_bfloat16, int32_t);
|
||||
break;
|
||||
default:
|
||||
// Handle other data types
|
||||
throw std::invalid_argument(
|
||||
"Invalid dtype, only supports float16, float32, and bfloat16");
|
||||
STD_TORCH_CHECK(
|
||||
false, "Invalid dtype, only supports float16, float32, and bfloat16");
|
||||
break;
|
||||
}
|
||||
#undef LAUNCH_KERNEL
|
||||
+1
-1
@@ -302,7 +302,7 @@ def generate_new_kernels():
|
||||
if not SUPPORT_FP8 and kernel_selector_str != FILE_HEAD_COMMENT:
|
||||
kernel_selector_str += (
|
||||
"else if (a_type == vllm::kFE4M3fn)\n"
|
||||
" TORCH_CHECK(false, "
|
||||
" STD_TORCH_CHECK(false, "
|
||||
'"marlin kernel with fp8 activation is not built.");'
|
||||
)
|
||||
|
||||
+232
-208
@@ -24,7 +24,15 @@
|
||||
#endif
|
||||
|
||||
#include "kernel.h"
|
||||
#include "core/registration.h"
|
||||
|
||||
#include <torch/csrc/stable/accelerator.h>
|
||||
#include <torch/csrc/stable/library.h>
|
||||
#include <torch/csrc/stable/ops.h>
|
||||
#include <torch/csrc/stable/tensor.h>
|
||||
#include <torch/headeronly/core/ScalarType.h>
|
||||
#include <torch/headeronly/util/Exception.h>
|
||||
|
||||
#include "libtorch_stable/torch_utils.h"
|
||||
|
||||
#define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \
|
||||
static_assert(std::is_same<scalar_t, half>::value || \
|
||||
@@ -350,18 +358,18 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
|
||||
bool m_block_size_8 = moe_block_size == 8;
|
||||
bool is_a_8bit = a_type.size_bits() == 8;
|
||||
|
||||
TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m,
|
||||
", ", prob_n, ", ", prob_k, "]");
|
||||
STD_TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [",
|
||||
prob_m, ", ", prob_n, ", ", prob_k, "]");
|
||||
|
||||
int group_blocks = 0;
|
||||
if (has_act_order) {
|
||||
if (is_k_full) {
|
||||
TORCH_CHECK(group_size != -1);
|
||||
STD_TORCH_CHECK(group_size != -1);
|
||||
group_blocks = group_size / 16;
|
||||
TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k,
|
||||
" is not divisible by group_blocks = ", group_blocks);
|
||||
STD_TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k,
|
||||
" is not divisible by group_blocks = ", group_blocks);
|
||||
} else {
|
||||
TORCH_CHECK(group_size == 0);
|
||||
STD_TORCH_CHECK(group_size == 0);
|
||||
group_blocks = 0;
|
||||
}
|
||||
} else {
|
||||
@@ -369,8 +377,8 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
|
||||
group_blocks = -1;
|
||||
} else {
|
||||
group_blocks = group_size / 16;
|
||||
TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k,
|
||||
" is not divisible by group_blocks = ", group_blocks);
|
||||
STD_TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k,
|
||||
" is not divisible by group_blocks = ", group_blocks);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -407,7 +415,7 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
|
||||
else if (moe_block_size == 64)
|
||||
kernel = permute_cols_kernel<64>;
|
||||
else
|
||||
TORCH_CHECK(false, "unsupported moe_block_size ", moe_block_size);
|
||||
STD_TORCH_CHECK(false, "unsupported moe_block_size ", moe_block_size);
|
||||
|
||||
// avoid ">>>" being formatted to "> > >"
|
||||
// clang-format off
|
||||
@@ -428,25 +436,25 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
|
||||
int max_shared_mem = 0;
|
||||
cudaDeviceGetAttribute(&max_shared_mem,
|
||||
cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
|
||||
TORCH_CHECK(max_shared_mem > 0);
|
||||
STD_TORCH_CHECK(max_shared_mem > 0);
|
||||
|
||||
int major_capability, minor_capability;
|
||||
cudaDeviceGetAttribute(&major_capability, cudaDevAttrComputeCapabilityMajor,
|
||||
dev);
|
||||
cudaDeviceGetAttribute(&minor_capability, cudaDevAttrComputeCapabilityMinor,
|
||||
dev);
|
||||
TORCH_CHECK(major_capability * 10 + minor_capability >= 75,
|
||||
"marlin kernel only support Turing or newer GPUs.");
|
||||
STD_TORCH_CHECK(major_capability * 10 + minor_capability >= 75,
|
||||
"marlin kernel only support Turing or newer GPUs.");
|
||||
int stages = 4;
|
||||
if (major_capability == 7 && minor_capability == 5) {
|
||||
stages = 2;
|
||||
TORCH_CHECK(a_type == vllm::kFloat16 || a_type == vllm::kS8,
|
||||
"Turing only support FP16 or INT8 activation.");
|
||||
STD_TORCH_CHECK(a_type == vllm::kFloat16 || a_type == vllm::kS8,
|
||||
"Turing only support FP16 or INT8 activation.");
|
||||
}
|
||||
if (a_type == vllm::kFE4M3fn) {
|
||||
TORCH_CHECK(major_capability * 10 + minor_capability >= 89,
|
||||
"FP8 only support Ada Lovelace or newer GPUs.");
|
||||
TORCH_CHECK(
|
||||
STD_TORCH_CHECK(major_capability * 10 + minor_capability >= 89,
|
||||
"FP8 only support Ada Lovelace or newer GPUs.");
|
||||
STD_TORCH_CHECK(
|
||||
major_capability * 10 + minor_capability == 89 ||
|
||||
major_capability == 12,
|
||||
"Marlin W4A8-FP8 only support SM89 or SM12x device (It is slower than "
|
||||
@@ -460,10 +468,10 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
|
||||
thread_tfg = thread_config_t{thread_k, thread_n, thread_k * thread_n / 64};
|
||||
if (blocks_per_sm == -1) blocks_per_sm = 1;
|
||||
exec_cfg = exec_config_t{blocks_per_sm, thread_tfg};
|
||||
TORCH_CHECK(prob_n % thread_n == 0, "prob_n = ", prob_n,
|
||||
" is not divisible by thread_n = ", thread_n);
|
||||
TORCH_CHECK(prob_k % thread_k == 0, "prob_k = ", prob_k,
|
||||
" is not divisible by thread_k = ", thread_k);
|
||||
STD_TORCH_CHECK(prob_n % thread_n == 0, "prob_n = ", prob_n,
|
||||
" is not divisible by thread_n = ", thread_n);
|
||||
STD_TORCH_CHECK(prob_k % thread_k == 0, "prob_k = ", prob_k,
|
||||
" is not divisible by thread_k = ", thread_k);
|
||||
} else {
|
||||
// Auto config
|
||||
exec_cfg = determine_exec_config(
|
||||
@@ -484,19 +492,19 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
|
||||
int thread_k_blocks = thread_k / 16;
|
||||
int thread_n_blocks = thread_n / 16;
|
||||
|
||||
TORCH_CHECK(is_valid_config(thread_tfg, m_block_size_8, thread_m_blocks,
|
||||
prob_m, prob_n, prob_k, num_bits, group_size,
|
||||
has_act_order, is_k_full, has_zp, is_zp_float,
|
||||
is_a_8bit, stages, max_shared_mem),
|
||||
"Invalid thread config: thread_m_blocks = ", thread_m_blocks,
|
||||
", thread_k = ", thread_tfg.thread_k,
|
||||
", thread_n = ", thread_tfg.thread_n,
|
||||
", num_threads = ", thread_tfg.num_threads, " for MKN = [",
|
||||
prob_m, ", ", prob_k, ", ", prob_n, "] and num_bits = ", num_bits,
|
||||
", group_size = ", group_size,
|
||||
", has_act_order = ", has_act_order, ", is_k_full = ", is_k_full,
|
||||
", has_zp = ", has_zp, ", is_zp_float = ", is_zp_float,
|
||||
", max_shared_mem = ", max_shared_mem);
|
||||
STD_TORCH_CHECK(
|
||||
is_valid_config(thread_tfg, m_block_size_8, thread_m_blocks, prob_m,
|
||||
prob_n, prob_k, num_bits, group_size, has_act_order,
|
||||
is_k_full, has_zp, is_zp_float, is_a_8bit, stages,
|
||||
max_shared_mem),
|
||||
"Invalid thread config: thread_m_blocks = ", thread_m_blocks,
|
||||
", thread_k = ", thread_tfg.thread_k,
|
||||
", thread_n = ", thread_tfg.thread_n,
|
||||
", num_threads = ", thread_tfg.num_threads, " for MKN = [", prob_m, ", ",
|
||||
prob_k, ", ", prob_n, "] and num_bits = ", num_bits,
|
||||
", group_size = ", group_size, ", has_act_order = ", has_act_order,
|
||||
", is_k_full = ", is_k_full, ", has_zp = ", has_zp,
|
||||
", is_zp_float = ", is_zp_float, ", max_shared_mem = ", max_shared_mem);
|
||||
|
||||
int sh_cache_size =
|
||||
get_kernel_cache_size(thread_tfg, m_block_size_8, thread_m_blocks, prob_m,
|
||||
@@ -509,13 +517,13 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
|
||||
num_threads, is_zp_float, stages);
|
||||
|
||||
if (kernel == MarlinDefault) {
|
||||
TORCH_CHECK(false, "Unsupported shapes: MNK = [", prob_m, ", ", prob_n,
|
||||
", ", prob_k, "]", ", has_act_order = ", has_act_order,
|
||||
", num_groups = ", num_groups, ", group_size = ", group_size,
|
||||
", thread_m_blocks = ", thread_m_blocks,
|
||||
", thread_n_blocks = ", thread_n_blocks,
|
||||
", thread_k_blocks = ", thread_k_blocks,
|
||||
", num_bits = ", num_bits);
|
||||
STD_TORCH_CHECK(
|
||||
false, "Unsupported shapes: MNK = [", prob_m, ", ", prob_n, ", ",
|
||||
prob_k, "]", ", has_act_order = ", has_act_order,
|
||||
", num_groups = ", num_groups, ", group_size = ", group_size,
|
||||
", thread_m_blocks = ", thread_m_blocks,
|
||||
", thread_n_blocks = ", thread_n_blocks,
|
||||
", thread_k_blocks = ", thread_k_blocks, ", num_bits = ", num_bits);
|
||||
}
|
||||
|
||||
cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize,
|
||||
@@ -532,75 +540,81 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
|
||||
|
||||
} // namespace MARLIN_NAMESPACE_NAME
|
||||
|
||||
torch::Tensor moe_wna16_marlin_gemm(
|
||||
torch::Tensor& a, std::optional<torch::Tensor> c_or_none,
|
||||
torch::Tensor& b_q_weight,
|
||||
std::optional<torch::Tensor> const& b_bias_or_none, torch::Tensor& b_scales,
|
||||
std::optional<torch::Tensor> const& a_scales_or_none,
|
||||
std::optional<torch::Tensor> const& global_scale_or_none,
|
||||
std::optional<torch::Tensor> const& b_zeros_or_none,
|
||||
std::optional<torch::Tensor> const& g_idx_or_none,
|
||||
std::optional<torch::Tensor> const& perm_or_none, torch::Tensor& workspace,
|
||||
torch::Tensor& sorted_token_ids, torch::Tensor& expert_ids,
|
||||
torch::Tensor& num_tokens_past_padded, torch::Tensor& topk_weights,
|
||||
int64_t moe_block_size, int64_t top_k, bool mul_topk_weights,
|
||||
vllm::ScalarTypeId const& b_type_id, int64_t size_m, int64_t size_n,
|
||||
int64_t size_k, bool is_k_full, bool use_atomic_add, bool use_fp32_reduce,
|
||||
bool is_zp_float, int64_t thread_k, int64_t thread_n,
|
||||
torch::stable::Tensor moe_wna16_marlin_gemm(
|
||||
torch::stable::Tensor& a, std::optional<torch::stable::Tensor> c_or_none,
|
||||
torch::stable::Tensor& b_q_weight,
|
||||
std::optional<torch::stable::Tensor> const& b_bias_or_none,
|
||||
torch::stable::Tensor& b_scales,
|
||||
std::optional<torch::stable::Tensor> const& a_scales_or_none,
|
||||
std::optional<torch::stable::Tensor> const& global_scale_or_none,
|
||||
std::optional<torch::stable::Tensor> const& b_zeros_or_none,
|
||||
std::optional<torch::stable::Tensor> const& g_idx_or_none,
|
||||
std::optional<torch::stable::Tensor> const& perm_or_none,
|
||||
torch::stable::Tensor& workspace, torch::stable::Tensor& sorted_token_ids,
|
||||
torch::stable::Tensor& expert_ids,
|
||||
torch::stable::Tensor& num_tokens_past_padded,
|
||||
torch::stable::Tensor& topk_weights, int64_t moe_block_size, int64_t top_k,
|
||||
bool mul_topk_weights, vllm::ScalarTypeId const& b_type_id, int64_t size_m,
|
||||
int64_t size_n, int64_t size_k, bool is_k_full, bool use_atomic_add,
|
||||
bool use_fp32_reduce, bool is_zp_float, int64_t thread_k, int64_t thread_n,
|
||||
int64_t blocks_per_sm) {
|
||||
vllm::ScalarTypeId a_type_id, c_type_id, s_type_id;
|
||||
|
||||
auto c_dtype = a.dtype();
|
||||
if (a.scalar_type() == at::ScalarType::Half) {
|
||||
auto c_dtype = a.scalar_type();
|
||||
if (a.scalar_type() == torch::headeronly::ScalarType::Half) {
|
||||
a_type_id = vllm::kFloat16.id();
|
||||
c_type_id = vllm::kFloat16.id();
|
||||
} else if (a.scalar_type() == at::ScalarType::BFloat16) {
|
||||
} else if (a.scalar_type() == torch::headeronly::ScalarType::BFloat16) {
|
||||
a_type_id = vllm::kBFloat16.id();
|
||||
c_type_id = vllm::kBFloat16.id();
|
||||
} else {
|
||||
c_dtype = b_scales.dtype();
|
||||
if (b_scales.scalar_type() == at::ScalarType::Half) {
|
||||
c_dtype = b_scales.scalar_type();
|
||||
if (b_scales.scalar_type() == torch::headeronly::ScalarType::Half) {
|
||||
c_type_id = vllm::kFloat16.id();
|
||||
} else if (b_scales.scalar_type() == at::ScalarType::BFloat16) {
|
||||
} else if (b_scales.scalar_type() ==
|
||||
torch::headeronly::ScalarType::BFloat16) {
|
||||
c_type_id = vllm::kBFloat16.id();
|
||||
} else {
|
||||
c_type_id = vllm::kBFloat16.id();
|
||||
|
||||
TORCH_CHECK(c_or_none.has_value(), "c must be passed for W4A8-FP4");
|
||||
torch::Tensor c = c_or_none.value();
|
||||
c_dtype = c.dtype();
|
||||
STD_TORCH_CHECK(c_or_none.has_value(), "c must be passed for W4A8-FP4");
|
||||
torch::stable::Tensor c = c_or_none.value();
|
||||
c_dtype = c.scalar_type();
|
||||
|
||||
if (c.scalar_type() == at::ScalarType::Half) {
|
||||
if (c.scalar_type() == torch::headeronly::ScalarType::Half) {
|
||||
c_type_id = vllm::kFloat16.id();
|
||||
} else if (c.scalar_type() == at::ScalarType::BFloat16) {
|
||||
} else if (c.scalar_type() == torch::headeronly::ScalarType::BFloat16) {
|
||||
c_type_id = vllm::kBFloat16.id();
|
||||
} else {
|
||||
TORCH_CHECK(false, "unsupported c dtype");
|
||||
STD_TORCH_CHECK(false, "unsupported c dtype");
|
||||
}
|
||||
}
|
||||
|
||||
if (a.scalar_type() == at::ScalarType::Float8_e4m3fn) {
|
||||
if (a.scalar_type() == torch::headeronly::ScalarType::Float8_e4m3fn) {
|
||||
a_type_id = vllm::kFE4M3fn.id();
|
||||
} else if (a.scalar_type() == at::ScalarType::Char) {
|
||||
} else if (a.scalar_type() == torch::headeronly::ScalarType::Char) {
|
||||
a_type_id = vllm::kS8.id();
|
||||
} else {
|
||||
TORCH_CHECK(false, "unsupported `a` scalar_type");
|
||||
STD_TORCH_CHECK(false, "unsupported `a` scalar_type");
|
||||
}
|
||||
}
|
||||
|
||||
s_type_id = c_type_id;
|
||||
if (b_type_id == vllm::kFE2M1f.id()) {
|
||||
if (b_scales.scalar_type() == at::ScalarType::Float8_e4m3fn) {
|
||||
if (b_scales.scalar_type() ==
|
||||
torch::headeronly::ScalarType::Float8_e4m3fn) {
|
||||
s_type_id = vllm::kFE4M3fn.id();
|
||||
} else if (b_scales.scalar_type() == at::ScalarType::Float8_e8m0fnu) {
|
||||
} else if (b_scales.scalar_type() ==
|
||||
torch::headeronly::ScalarType::Float8_e8m0fnu) {
|
||||
s_type_id = vllm::kFE8M0fnu.id();
|
||||
} else {
|
||||
TORCH_CHECK(false,
|
||||
"When b_type = float4_e2m1f, b_scale scalar type must be",
|
||||
"float8_e4m3fn (for NVFP4) or float8_e8m0fnu (for MXFP4).");
|
||||
STD_TORCH_CHECK(
|
||||
false, "When b_type = float4_e2m1f, b_scale scalar type must be",
|
||||
"float8_e4m3fn (for NVFP4) or float8_e8m0fnu (for MXFP4).");
|
||||
}
|
||||
} else if (b_type_id == vllm::kFE4M3fn.id() &&
|
||||
b_scales.scalar_type() == at::ScalarType::Float8_e8m0fnu) {
|
||||
b_scales.scalar_type() ==
|
||||
torch::headeronly::ScalarType::Float8_e8m0fnu) {
|
||||
s_type_id = vllm::kFE8M0fnu.id();
|
||||
}
|
||||
|
||||
@@ -613,58 +627,60 @@ torch::Tensor moe_wna16_marlin_gemm(
|
||||
int num_experts = b_q_weight.size(0);
|
||||
|
||||
if (moe_block_size != 8) {
|
||||
TORCH_CHECK(moe_block_size % 16 == 0,
|
||||
"unsupported moe_block_size=", moe_block_size);
|
||||
TORCH_CHECK(moe_block_size >= 16 && moe_block_size <= 64,
|
||||
"unsupported moe_block_size=", moe_block_size);
|
||||
STD_TORCH_CHECK(moe_block_size % 16 == 0,
|
||||
"unsupported moe_block_size=", moe_block_size);
|
||||
STD_TORCH_CHECK(moe_block_size >= 16 && moe_block_size <= 64,
|
||||
"unsupported moe_block_size=", moe_block_size);
|
||||
}
|
||||
|
||||
// Verify A
|
||||
TORCH_CHECK(a.size(0) == size_m, "Shape mismatch: a.size(0) = ", a.size(0),
|
||||
", size_m = ", size_m);
|
||||
TORCH_CHECK(a.size(1) == size_k, "Shape mismatch: a.size(1) = ", a.size(1),
|
||||
", size_k = ", size_k);
|
||||
STD_TORCH_CHECK(a.size(0) == size_m,
|
||||
"Shape mismatch: a.size(0) = ", a.size(0),
|
||||
", size_m = ", size_m);
|
||||
STD_TORCH_CHECK(a.size(1) == size_k,
|
||||
"Shape mismatch: a.size(1) = ", a.size(1),
|
||||
", size_k = ", size_k);
|
||||
|
||||
// Verify B
|
||||
TORCH_CHECK(
|
||||
STD_TORCH_CHECK(
|
||||
size_k % MARLIN_NAMESPACE_NAME::tile_size == 0, "size_k = ", size_k,
|
||||
" is not divisible by tile_size = ", MARLIN_NAMESPACE_NAME::tile_size);
|
||||
TORCH_CHECK((size_k / MARLIN_NAMESPACE_NAME::tile_size) == b_q_weight.size(1),
|
||||
"Shape mismatch: b_q_weight.size(1) = ", b_q_weight.size(1),
|
||||
", size_k = ", size_k,
|
||||
", tile_size = ", MARLIN_NAMESPACE_NAME::tile_size);
|
||||
TORCH_CHECK(
|
||||
STD_TORCH_CHECK(
|
||||
(size_k / MARLIN_NAMESPACE_NAME::tile_size) == b_q_weight.size(1),
|
||||
"Shape mismatch: b_q_weight.size(1) = ", b_q_weight.size(1),
|
||||
", size_k = ", size_k,
|
||||
", tile_size = ", MARLIN_NAMESPACE_NAME::tile_size);
|
||||
STD_TORCH_CHECK(
|
||||
b_q_weight.size(2) % MARLIN_NAMESPACE_NAME::tile_size == 0,
|
||||
"b_q_weight.size(2) = ", b_q_weight.size(2),
|
||||
" is not divisible by tile_size = ", MARLIN_NAMESPACE_NAME::tile_size);
|
||||
int actual_size_n =
|
||||
(b_q_weight.size(2) / MARLIN_NAMESPACE_NAME::tile_size) * pack_factor;
|
||||
TORCH_CHECK(size_n == actual_size_n, "size_n = ", size_n,
|
||||
", actual_size_n = ", actual_size_n);
|
||||
STD_TORCH_CHECK(size_n == actual_size_n, "size_n = ", size_n,
|
||||
", actual_size_n = ", actual_size_n);
|
||||
|
||||
// Verify device and strides
|
||||
TORCH_CHECK(a.device().is_cuda(), "A is not on GPU");
|
||||
TORCH_CHECK(a.is_contiguous(), "A is not contiguous");
|
||||
STD_TORCH_CHECK(a.device().is_cuda(), "A is not on GPU");
|
||||
STD_TORCH_CHECK(a.is_contiguous(), "A is not contiguous");
|
||||
|
||||
TORCH_CHECK(b_q_weight.device().is_cuda(), "b_q_weight is not on GPU");
|
||||
TORCH_CHECK(b_q_weight.is_contiguous(), "b_q_weight is not contiguous");
|
||||
STD_TORCH_CHECK(b_q_weight.device().is_cuda(), "b_q_weight is not on GPU");
|
||||
STD_TORCH_CHECK(b_q_weight.is_contiguous(), "b_q_weight is not contiguous");
|
||||
|
||||
TORCH_CHECK(b_scales.device().is_cuda(), "b_scales is not on GPU");
|
||||
TORCH_CHECK(b_scales.is_contiguous(), "b_scales is not contiguous");
|
||||
STD_TORCH_CHECK(b_scales.device().is_cuda(), "b_scales is not on GPU");
|
||||
STD_TORCH_CHECK(b_scales.is_contiguous(), "b_scales is not contiguous");
|
||||
|
||||
torch::Tensor a_scales;
|
||||
auto options = torch::TensorOptions().dtype(c_dtype).device(a.device());
|
||||
auto options_fp32 =
|
||||
torch::TensorOptions().dtype(at::kFloat).device(a.device());
|
||||
torch::stable::Tensor a_scales;
|
||||
constexpr auto kFloat = torch::headeronly::ScalarType::Float;
|
||||
|
||||
if (a_scales_or_none.has_value()) {
|
||||
a_scales = a_scales_or_none.value();
|
||||
TORCH_CHECK(a_type.size_bits() == 8,
|
||||
"a_scales can only be used for 8bit activation.");
|
||||
STD_TORCH_CHECK(a_type.size_bits() == 8,
|
||||
"a_scales can only be used for 8bit activation.");
|
||||
} else {
|
||||
a_scales = torch::empty({0}, options_fp32);
|
||||
TORCH_CHECK(a_type.size_bits() != 8,
|
||||
"the a_scales parameter must be passed for 8bit activation.");
|
||||
a_scales = torch::stable::new_empty(a, {0}, kFloat);
|
||||
STD_TORCH_CHECK(
|
||||
a_type.size_bits() != 8,
|
||||
"the a_scales parameter must be passed for 8bit activation.");
|
||||
}
|
||||
|
||||
// sms: number of SMs to use for the kernel
|
||||
@@ -672,82 +688,84 @@ torch::Tensor moe_wna16_marlin_gemm(
|
||||
cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, a.get_device());
|
||||
|
||||
// Alloc buffers
|
||||
const at::cuda::OptionalCUDAGuard device_guard(device_of(a));
|
||||
torch::Tensor c;
|
||||
torch::stable::accelerator::DeviceGuard device_guard(a.get_device_index());
|
||||
torch::stable::Tensor c;
|
||||
if (c_or_none.has_value()) {
|
||||
c = c_or_none.value();
|
||||
TORCH_CHECK(c.device().is_cuda(), "c is not on GPU");
|
||||
TORCH_CHECK(c.is_contiguous(), "c is not contiguous");
|
||||
TORCH_CHECK(c.size(0) == size_m * top_k,
|
||||
"Shape mismatch: c.size(0) = ", c.size(0),
|
||||
", size_m * topk = ", size_m * top_k);
|
||||
TORCH_CHECK(c.size(1) == size_n, "Shape mismatch: c.size(1) = ", c.size(1),
|
||||
", size_n = ", size_n);
|
||||
STD_TORCH_CHECK(c.device().is_cuda(), "c is not on GPU");
|
||||
STD_TORCH_CHECK(c.is_contiguous(), "c is not contiguous");
|
||||
STD_TORCH_CHECK(c.size(0) == size_m * top_k,
|
||||
"Shape mismatch: c.size(0) = ", c.size(0),
|
||||
", size_m * topk = ", size_m * top_k);
|
||||
STD_TORCH_CHECK(c.size(1) == size_n,
|
||||
"Shape mismatch: c.size(1) = ", c.size(1),
|
||||
", size_n = ", size_n);
|
||||
} else {
|
||||
c = torch::empty({size_m * top_k, size_n}, options);
|
||||
c = torch::stable::new_empty(a, {size_m * top_k, size_n}, c_dtype);
|
||||
}
|
||||
|
||||
// Alloc C tmp buffer that is going to be used for the global reduce
|
||||
torch::Tensor c_tmp;
|
||||
torch::stable::Tensor c_tmp;
|
||||
if (use_fp32_reduce && !use_atomic_add) {
|
||||
// max num of threadblocks is sms * 4
|
||||
long max_c_tmp_size = min(
|
||||
(long)size_n * sorted_token_ids.size(0),
|
||||
(long)sms * 4 * moe_block_size * MARLIN_NAMESPACE_NAME::max_thread_n);
|
||||
if (moe_block_size == 8) max_c_tmp_size *= 2;
|
||||
c_tmp = torch::empty({max_c_tmp_size}, options_fp32);
|
||||
c_tmp = torch::stable::new_empty(a, {max_c_tmp_size}, kFloat);
|
||||
} else {
|
||||
c_tmp = torch::empty({0}, options_fp32);
|
||||
c_tmp = torch::stable::new_empty(a, {0}, kFloat);
|
||||
}
|
||||
|
||||
// Detect groupsize and act_order
|
||||
int num_groups = -1;
|
||||
int group_size = -1;
|
||||
|
||||
int rank = b_scales.sizes().size();
|
||||
TORCH_CHECK(rank == 3, "b_scales rank = ", rank, " is not 3");
|
||||
TORCH_CHECK(b_scales.size(2) == size_n, "b_scales dim 2 = ", b_scales.size(2),
|
||||
" is not size_n = ", size_n);
|
||||
int rank = b_scales.dim();
|
||||
STD_TORCH_CHECK(rank == 3, "b_scales rank = ", rank, " is not 3");
|
||||
STD_TORCH_CHECK(b_scales.size(2) == size_n,
|
||||
"b_scales dim 2 = ", b_scales.size(2),
|
||||
" is not size_n = ", size_n);
|
||||
num_groups = b_scales.size(1);
|
||||
|
||||
torch::Tensor g_idx, perm, a_tmp;
|
||||
torch::stable::Tensor g_idx, perm, a_tmp;
|
||||
if (g_idx_or_none.has_value() && perm_or_none.has_value()) {
|
||||
g_idx = g_idx_or_none.value();
|
||||
perm = perm_or_none.value();
|
||||
|
||||
TORCH_CHECK(g_idx.device().is_cuda(), "g_idx is not on GPU");
|
||||
TORCH_CHECK(g_idx.is_contiguous(), "g_idx is not contiguous");
|
||||
TORCH_CHECK(perm.device().is_cuda(), "perm is not on GPU");
|
||||
TORCH_CHECK(perm.is_contiguous(), "perm is not contiguous");
|
||||
STD_TORCH_CHECK(g_idx.device().is_cuda(), "g_idx is not on GPU");
|
||||
STD_TORCH_CHECK(g_idx.is_contiguous(), "g_idx is not contiguous");
|
||||
STD_TORCH_CHECK(perm.device().is_cuda(), "perm is not on GPU");
|
||||
STD_TORCH_CHECK(perm.is_contiguous(), "perm is not contiguous");
|
||||
|
||||
// Verify g_idx and perm
|
||||
TORCH_CHECK((g_idx.size(-1) == 0 && perm.size(-1) == 0) ||
|
||||
(g_idx.size(-1) == size_k && perm.size(-1) == size_k),
|
||||
"Unexpected g_idx.size(-1) = ", g_idx.size(-1),
|
||||
" and perm.size(-1) = ", perm.size(-1),
|
||||
", where size_k = ", size_k);
|
||||
STD_TORCH_CHECK((g_idx.size(-1) == 0 && perm.size(-1) == 0) ||
|
||||
(g_idx.size(-1) == size_k && perm.size(-1) == size_k),
|
||||
"Unexpected g_idx.size(-1) = ", g_idx.size(-1),
|
||||
" and perm.size(-1) = ", perm.size(-1),
|
||||
", where size_k = ", size_k);
|
||||
} else {
|
||||
g_idx = torch::empty({0}, options);
|
||||
perm = torch::empty({0}, options);
|
||||
a_tmp = torch::empty({0}, options);
|
||||
g_idx = torch::stable::new_empty(a, {0}, c_dtype);
|
||||
perm = torch::stable::new_empty(a, {0}, c_dtype);
|
||||
a_tmp = torch::stable::new_empty(a, {0}, c_dtype);
|
||||
}
|
||||
bool has_act_order = g_idx.size(-1) > 0 && perm.size(-1) > 0;
|
||||
|
||||
if (has_act_order) {
|
||||
a_tmp = torch::empty({size_m * top_k, size_k}, options);
|
||||
a_tmp = torch::stable::new_empty(a, {size_m * top_k, size_k}, c_dtype);
|
||||
if (is_k_full) {
|
||||
TORCH_CHECK(num_groups > 1, "For act_order, num_groups must be > 1");
|
||||
TORCH_CHECK(size_k % num_groups == 0, "size_k = ", size_k,
|
||||
", is not divisible by num_groups = ", num_groups);
|
||||
STD_TORCH_CHECK(num_groups > 1, "For act_order, num_groups must be > 1");
|
||||
STD_TORCH_CHECK(size_k % num_groups == 0, "size_k = ", size_k,
|
||||
", is not divisible by num_groups = ", num_groups);
|
||||
group_size = size_k / num_groups;
|
||||
} else {
|
||||
group_size = 0;
|
||||
}
|
||||
|
||||
} else {
|
||||
a_tmp = torch::empty({0}, options);
|
||||
a_tmp = torch::stable::new_empty(a, {0}, c_dtype);
|
||||
if (num_groups > 1) {
|
||||
TORCH_CHECK(
|
||||
STD_TORCH_CHECK(
|
||||
size_k % num_groups == 0, "size_k = ", size_k,
|
||||
", is not divisible by b_scales.size(1) = ", b_scales.size(1));
|
||||
group_size = size_k / num_groups;
|
||||
@@ -756,119 +774,125 @@ torch::Tensor moe_wna16_marlin_gemm(
|
||||
}
|
||||
}
|
||||
|
||||
torch::Tensor global_scale;
|
||||
torch::stable::Tensor global_scale;
|
||||
if (global_scale_or_none.has_value()) {
|
||||
global_scale = global_scale_or_none.value();
|
||||
TORCH_CHECK(b_type == vllm::kFE2M1f && s_type == vllm::kFE4M3fn,
|
||||
"global_scale can only be used for nvfp4 format.");
|
||||
STD_TORCH_CHECK(b_type == vllm::kFE2M1f && s_type == vllm::kFE4M3fn,
|
||||
"global_scale can only be used for nvfp4 format.");
|
||||
} else {
|
||||
global_scale = torch::empty({0}, options_fp32);
|
||||
TORCH_CHECK(!(b_type == vllm::kFE2M1f && s_type == vllm::kFE4M3fn),
|
||||
"the global_scale parameter must be passed for nvfp4 format.");
|
||||
global_scale = torch::stable::new_empty(a, {0}, kFloat);
|
||||
STD_TORCH_CHECK(
|
||||
!(b_type == vllm::kFE2M1f && s_type == vllm::kFE4M3fn),
|
||||
"the global_scale parameter must be passed for nvfp4 format.");
|
||||
}
|
||||
|
||||
bool has_bias = b_bias_or_none.has_value();
|
||||
torch::Tensor b_bias;
|
||||
torch::stable::Tensor b_bias;
|
||||
if (has_bias) {
|
||||
b_bias = b_bias_or_none.value();
|
||||
TORCH_CHECK(b_bias.device().is_cuda(), "b_bias is not on GPU");
|
||||
TORCH_CHECK(b_bias.is_contiguous(), "b_bias is not contiguous");
|
||||
TORCH_CHECK(b_bias.size(1) == size_n, "b_bias.size(1) != size_n");
|
||||
TORCH_CHECK(b_bias.stride(1) == 1, "b_bias.stride(1) != 1");
|
||||
STD_TORCH_CHECK(b_bias.device().is_cuda(), "b_bias is not on GPU");
|
||||
STD_TORCH_CHECK(b_bias.is_contiguous(), "b_bias is not contiguous");
|
||||
STD_TORCH_CHECK(b_bias.size(1) == size_n, "b_bias.size(1) != size_n");
|
||||
STD_TORCH_CHECK(b_bias.stride(1) == 1, "b_bias.stride(1) != 1");
|
||||
} else {
|
||||
b_bias = torch::empty({0}, options);
|
||||
b_bias = torch::stable::new_empty(a, {0}, c_dtype);
|
||||
}
|
||||
|
||||
torch::Tensor b_zeros;
|
||||
torch::stable::Tensor b_zeros;
|
||||
if (b_zeros_or_none.has_value()) {
|
||||
b_zeros = b_zeros_or_none.value();
|
||||
TORCH_CHECK(b_zeros.device().is_cuda(), "b_zeros is not on GPU");
|
||||
TORCH_CHECK(b_zeros.is_contiguous(), "b_zeros is not contiguous");
|
||||
STD_TORCH_CHECK(b_zeros.device().is_cuda(), "b_zeros is not on GPU");
|
||||
STD_TORCH_CHECK(b_zeros.is_contiguous(), "b_zeros is not contiguous");
|
||||
} else {
|
||||
b_zeros = torch::empty({0}, options);
|
||||
b_zeros = torch::stable::new_empty(a, {0}, c_dtype);
|
||||
}
|
||||
bool has_zp = b_zeros.size(-1) > 0;
|
||||
if (has_zp) {
|
||||
TORCH_CHECK(
|
||||
STD_TORCH_CHECK(
|
||||
b_type == vllm::kU4 || b_type == vllm::kU8,
|
||||
"b_type must be u4 or u8 when has_zp = True. Got = ", b_type.str());
|
||||
} else {
|
||||
TORCH_CHECK(b_type == vllm::kU4B8 || b_type == vllm::kU8B128 ||
|
||||
b_type == vllm::kS4 || b_type == vllm::kS8 ||
|
||||
b_type == vllm::kFE4M3fn || b_type == vllm::kFE2M1f,
|
||||
"b_type must be uint4b8, uint8b128, int4, int8, "
|
||||
"float8_e4m3fn or float4_e2m1f when has_zp = False. Got = ",
|
||||
b_type.str());
|
||||
STD_TORCH_CHECK(b_type == vllm::kU4B8 || b_type == vllm::kU8B128 ||
|
||||
b_type == vllm::kS4 || b_type == vllm::kS8 ||
|
||||
b_type == vllm::kFE4M3fn || b_type == vllm::kFE2M1f,
|
||||
"b_type must be uint4b8, uint8b128, int4, int8, "
|
||||
"float8_e4m3fn or float4_e2m1f when has_zp = False. Got = ",
|
||||
b_type.str());
|
||||
}
|
||||
|
||||
if (has_zp && is_zp_float) {
|
||||
TORCH_CHECK(a.scalar_type() == at::ScalarType::Half,
|
||||
"Computation type must be float16 (half) when using float zero "
|
||||
"points.");
|
||||
STD_TORCH_CHECK(
|
||||
a.scalar_type() == torch::headeronly::ScalarType::Half,
|
||||
"Computation type must be float16 (half) when using float zero "
|
||||
"points.");
|
||||
}
|
||||
|
||||
// Verify b_zeros
|
||||
if (has_zp) {
|
||||
int rank = b_zeros.sizes().size();
|
||||
TORCH_CHECK(rank == 3, "b_zeros rank = ", rank, " is not 3");
|
||||
int rank = b_zeros.dim();
|
||||
STD_TORCH_CHECK(rank == 3, "b_zeros rank = ", rank, " is not 3");
|
||||
if (is_zp_float) {
|
||||
TORCH_CHECK(b_zeros.size(2) == size_n,
|
||||
"b_zeros dim 2 = ", b_zeros.size(2),
|
||||
" is not size_n = ", size_n);
|
||||
TORCH_CHECK(num_groups == b_zeros.size(1),
|
||||
"b_zeros dim 1 = ", b_zeros.size(1),
|
||||
" is not num_groups = ", num_groups);
|
||||
TORCH_CHECK(num_groups != -1, "num_groups must be != -1");
|
||||
STD_TORCH_CHECK(b_zeros.size(2) == size_n,
|
||||
"b_zeros dim 2 = ", b_zeros.size(2),
|
||||
" is not size_n = ", size_n);
|
||||
STD_TORCH_CHECK(num_groups == b_zeros.size(1),
|
||||
"b_zeros dim 1 = ", b_zeros.size(1),
|
||||
" is not num_groups = ", num_groups);
|
||||
STD_TORCH_CHECK(num_groups != -1, "num_groups must be != -1");
|
||||
} else {
|
||||
TORCH_CHECK(b_zeros.size(1) == num_groups,
|
||||
"b_zeros dim 1 = ", b_zeros.size(1),
|
||||
" is not num_groups = ", num_groups);
|
||||
TORCH_CHECK(b_zeros.size(2) == size_n / pack_factor,
|
||||
"b_zeros dim 2 = ", b_zeros.size(2),
|
||||
" is not size_n / pack_factor = ", size_n / pack_factor);
|
||||
STD_TORCH_CHECK(b_zeros.size(1) == num_groups,
|
||||
"b_zeros dim 1 = ", b_zeros.size(1),
|
||||
" is not num_groups = ", num_groups);
|
||||
STD_TORCH_CHECK(b_zeros.size(2) == size_n / pack_factor,
|
||||
"b_zeros dim 2 = ", b_zeros.size(2),
|
||||
" is not size_n / pack_factor = ", size_n / pack_factor);
|
||||
}
|
||||
}
|
||||
|
||||
// Verify workspace size
|
||||
TORCH_CHECK(size_n % MARLIN_NAMESPACE_NAME::min_thread_n == 0,
|
||||
"size_n = ", size_n, ", is not divisible by min_thread_n = ",
|
||||
MARLIN_NAMESPACE_NAME::min_thread_n);
|
||||
STD_TORCH_CHECK(size_n % MARLIN_NAMESPACE_NAME::min_thread_n == 0,
|
||||
"size_n = ", size_n, ", is not divisible by min_thread_n = ",
|
||||
MARLIN_NAMESPACE_NAME::min_thread_n);
|
||||
|
||||
int max_n_tiles = size_n / MARLIN_NAMESPACE_NAME::min_thread_n;
|
||||
int min_workspace_size = min(
|
||||
max_n_tiles * (int)(sorted_token_ids.size(0) / moe_block_size), sms * 4);
|
||||
TORCH_CHECK(workspace.numel() >= min_workspace_size,
|
||||
"workspace.numel = ", workspace.numel(),
|
||||
" is below min_workspace_size = ", min_workspace_size);
|
||||
STD_TORCH_CHECK(workspace.numel() >= min_workspace_size,
|
||||
"workspace.numel = ", workspace.numel(),
|
||||
" is below min_workspace_size = ", min_workspace_size);
|
||||
|
||||
int dev = a.get_device();
|
||||
|
||||
TORCH_CHECK(a_scales.scalar_type() == at::ScalarType::Float,
|
||||
"scalar type of a_scales must be float");
|
||||
TORCH_CHECK(global_scale.scalar_type() == at::ScalarType::Float,
|
||||
"scalar type of global_scale must be float");
|
||||
STD_TORCH_CHECK(
|
||||
a_scales.scalar_type() == torch::headeronly::ScalarType::Float,
|
||||
"scalar type of a_scales must be float");
|
||||
STD_TORCH_CHECK(
|
||||
global_scale.scalar_type() == torch::headeronly::ScalarType::Float,
|
||||
"scalar type of global_scale must be float");
|
||||
if (a_type.size_bits() == 16) {
|
||||
TORCH_CHECK(
|
||||
STD_TORCH_CHECK(
|
||||
a.scalar_type() == c.scalar_type(),
|
||||
"scalar type of a must be the same with c for 16 bit activation");
|
||||
}
|
||||
|
||||
MARLIN_NAMESPACE_NAME::marlin_mm(
|
||||
a.data_ptr(), b_q_weight.data_ptr(), c.data_ptr(), c_tmp.data_ptr(),
|
||||
b_bias.data_ptr(), a_scales.data_ptr(), b_scales.data_ptr(),
|
||||
global_scale.data_ptr(), b_zeros.data_ptr(), g_idx.data_ptr(),
|
||||
perm.data_ptr(), a_tmp.data_ptr(), sorted_token_ids.data_ptr(),
|
||||
expert_ids.data_ptr(), num_tokens_past_padded.data_ptr(),
|
||||
topk_weights.data_ptr(), moe_block_size, num_experts, top_k,
|
||||
mul_topk_weights, size_m, size_n, size_k, workspace.data_ptr(), a_type,
|
||||
b_type, c_type, s_type, has_bias, has_act_order, is_k_full, has_zp,
|
||||
num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
|
||||
a.const_data_ptr(), b_q_weight.const_data_ptr(), c.mutable_data_ptr(),
|
||||
c_tmp.mutable_data_ptr(), b_bias.mutable_data_ptr(),
|
||||
a_scales.mutable_data_ptr(), b_scales.mutable_data_ptr(),
|
||||
global_scale.mutable_data_ptr(), b_zeros.mutable_data_ptr(),
|
||||
g_idx.mutable_data_ptr(), perm.mutable_data_ptr(),
|
||||
a_tmp.mutable_data_ptr(), sorted_token_ids.mutable_data_ptr(),
|
||||
expert_ids.mutable_data_ptr(), num_tokens_past_padded.mutable_data_ptr(),
|
||||
topk_weights.mutable_data_ptr(), moe_block_size, num_experts, top_k,
|
||||
mul_topk_weights, size_m, size_n, size_k, workspace.mutable_data_ptr(),
|
||||
a_type, b_type, c_type, s_type, has_bias, has_act_order, is_k_full,
|
||||
has_zp, num_groups, group_size, dev, get_current_cuda_stream(dev),
|
||||
thread_k, thread_n, sms, blocks_per_sm, use_atomic_add, use_fp32_reduce,
|
||||
is_zp_float);
|
||||
|
||||
return c;
|
||||
}
|
||||
|
||||
TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) {
|
||||
m.impl("moe_wna16_marlin_gemm", &moe_wna16_marlin_gemm);
|
||||
STABLE_TORCH_LIBRARY_IMPL(_moe_C, CUDA, m) {
|
||||
m.impl("moe_wna16_marlin_gemm", TORCH_BOX(&moe_wna16_marlin_gemm));
|
||||
}
|
||||
+154
-117
@@ -1,14 +1,17 @@
|
||||
#include <torch/all.h>
|
||||
#include <ATen/cuda/CUDAContext.h>
|
||||
#include <c10/cuda/CUDAGuard.h>
|
||||
#include <array>
|
||||
#include <cub/cub.cuh>
|
||||
|
||||
#include <ATen/ATen.h>
|
||||
#include <ATen/cuda/Atomic.cuh>
|
||||
#include <cuda_runtime.h>
|
||||
#include <torch/csrc/stable/macros.h>
|
||||
#include <torch/csrc/stable/accelerator.h>
|
||||
#include <torch/csrc/stable/ops.h>
|
||||
#include <torch/csrc/stable/tensor.h>
|
||||
#include <torch/headeronly/core/ScalarType.h>
|
||||
|
||||
#include "../cuda_compat.h"
|
||||
#include "../dispatch_utils.h"
|
||||
#include "../../cuda_compat.h"
|
||||
#include "core/math.hpp"
|
||||
#include "libtorch_stable/dispatch_utils.h"
|
||||
#include "libtorch_stable/torch_utils.h"
|
||||
|
||||
#define CEILDIV(x, y) (((x) + (y) - 1) / (y))
|
||||
|
||||
@@ -492,12 +495,13 @@ __global__ void moe_lora_align_block_size_small_batch_expert_kernel(
|
||||
|
||||
// taken from
|
||||
// https://github.com/sgl-project/sglang/blob/8b5f83ed3b7d2a49ad5c5cd5aa61c5d502f47dbc
|
||||
void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
|
||||
int64_t block_size, torch::Tensor sorted_token_ids,
|
||||
torch::Tensor experts_ids,
|
||||
torch::Tensor num_tokens_post_pad,
|
||||
std::optional<torch::Tensor> maybe_expert_map) {
|
||||
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
|
||||
void moe_align_block_size(
|
||||
torch::stable::Tensor topk_ids, int64_t num_experts, int64_t block_size,
|
||||
torch::stable::Tensor sorted_token_ids, torch::stable::Tensor experts_ids,
|
||||
torch::stable::Tensor num_tokens_post_pad,
|
||||
std::optional<torch::stable::Tensor> maybe_expert_map) {
|
||||
const cudaStream_t stream =
|
||||
get_current_cuda_stream(topk_ids.get_device_index());
|
||||
|
||||
int64_t padded_num_experts =
|
||||
((num_experts + WARP_SIZE - 1) / WARP_SIZE) * WARP_SIZE;
|
||||
@@ -506,19 +510,18 @@ void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
|
||||
threads = ((threads + WARP_SIZE - 1) / WARP_SIZE) * WARP_SIZE;
|
||||
|
||||
// BlockScan uses 1024 threads and assigns one thread per expert.
|
||||
TORCH_CHECK(padded_num_experts < 1024,
|
||||
"padded_num_experts must be less than 1024");
|
||||
auto options_int =
|
||||
torch::TensorOptions().dtype(torch::kInt).device(topk_ids.device());
|
||||
STD_TORCH_CHECK(padded_num_experts < 1024,
|
||||
"padded_num_experts must be less than 1024");
|
||||
bool has_expert_map = maybe_expert_map.has_value();
|
||||
torch::Tensor expert_map;
|
||||
torch::stable::Tensor expert_map;
|
||||
if (has_expert_map) {
|
||||
expert_map = maybe_expert_map.value();
|
||||
} else {
|
||||
expert_map = torch::empty({0}, options_int);
|
||||
expert_map = torch::stable::new_empty(topk_ids, {0},
|
||||
torch::headeronly::ScalarType::Int);
|
||||
}
|
||||
|
||||
VLLM_DISPATCH_INTEGRAL_AND_UNSIGNED_TYPES(
|
||||
VLLM_STABLE_DISPATCH_INTEGRAL_AND_UNSIGNED_TYPES(
|
||||
topk_ids.scalar_type(), "moe_align_block_size_kernel", [&] {
|
||||
// calc needed amount of shared mem for `cumsum` tensors
|
||||
bool small_batch_expert_mode =
|
||||
@@ -538,16 +541,17 @@ void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
|
||||
scalar_t, fill_threads>;
|
||||
small_batch_expert_kernel<<<1, fill_threads + threads,
|
||||
shared_mem_size, stream>>>(
|
||||
topk_ids.data_ptr<scalar_t>(),
|
||||
sorted_token_ids.data_ptr<int32_t>(),
|
||||
experts_ids.data_ptr<int32_t>(),
|
||||
num_tokens_post_pad.data_ptr<int32_t>(),
|
||||
expert_map.data_ptr<int32_t>(), num_experts, block_size,
|
||||
topk_ids.numel(), sorted_token_ids.size(0), topk_ids.size(1),
|
||||
has_expert_map);
|
||||
reinterpret_cast<const scalar_t*>(topk_ids.const_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(sorted_token_ids.mutable_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(experts_ids.mutable_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(
|
||||
num_tokens_post_pad.mutable_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(expert_map.mutable_data_ptr()),
|
||||
num_experts, block_size, topk_ids.numel(),
|
||||
sorted_token_ids.size(0), topk_ids.size(1), has_expert_map);
|
||||
} else {
|
||||
torch::Tensor cumsum_buffer =
|
||||
torch::empty({num_experts + 1}, options_int);
|
||||
torch::stable::Tensor cumsum_buffer = torch::stable::new_empty(
|
||||
topk_ids, {num_experts + 1}, torch::headeronly::ScalarType::Int);
|
||||
auto align_kernel = vllm::moe::moe_align_block_size_kernel<scalar_t>;
|
||||
|
||||
size_t num_warps = CEILDIV(padded_num_experts, experts_per_warp);
|
||||
@@ -558,14 +562,16 @@ void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
|
||||
// blockIdx.x == 0: counting experts and aligning
|
||||
// blockIdx.x == 1: filling sorted_token_ids
|
||||
align_kernel<<<2, threads, shared_mem_size, stream>>>(
|
||||
topk_ids.data_ptr<scalar_t>(),
|
||||
sorted_token_ids.data_ptr<int32_t>(),
|
||||
experts_ids.data_ptr<int32_t>(),
|
||||
num_tokens_post_pad.data_ptr<int32_t>(),
|
||||
expert_map.data_ptr<int32_t>(), num_experts, padded_num_experts,
|
||||
experts_per_warp, block_size, topk_ids.numel(),
|
||||
cumsum_buffer.data_ptr<int32_t>(), sorted_token_ids.size(0),
|
||||
topk_ids.size(1), has_expert_map);
|
||||
reinterpret_cast<const scalar_t*>(topk_ids.const_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(sorted_token_ids.mutable_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(experts_ids.mutable_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(
|
||||
num_tokens_post_pad.mutable_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(expert_map.mutable_data_ptr()),
|
||||
num_experts, padded_num_experts, experts_per_warp, block_size,
|
||||
topk_ids.numel(),
|
||||
reinterpret_cast<int32_t*>(cumsum_buffer.mutable_data_ptr()),
|
||||
sorted_token_ids.size(0), topk_ids.size(1), has_expert_map);
|
||||
|
||||
const int block_threads = std::min(256, (int)threads);
|
||||
const int num_blocks =
|
||||
@@ -577,9 +583,10 @@ void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
|
||||
auto sort_kernel =
|
||||
vllm::moe::count_and_sort_expert_tokens_kernel<scalar_t>;
|
||||
sort_kernel<<<gridDims, block_threads, 0, stream>>>(
|
||||
topk_ids.data_ptr<scalar_t>(),
|
||||
sorted_token_ids.data_ptr<int32_t>(),
|
||||
cumsum_buffer.data_ptr<int32_t>(), expert_map.data_ptr<int32_t>(),
|
||||
reinterpret_cast<const scalar_t*>(topk_ids.const_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(sorted_token_ids.mutable_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(cumsum_buffer.mutable_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(expert_map.mutable_data_ptr()),
|
||||
topk_ids.numel(), num_experts, sorted_token_ids.size(0),
|
||||
topk_ids.size(1), has_expert_map);
|
||||
}
|
||||
@@ -588,33 +595,36 @@ void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
|
||||
|
||||
void batched_moe_align_block_size(int64_t max_tokens_per_batch,
|
||||
int64_t block_size,
|
||||
torch::Tensor const& batch_num_tokens,
|
||||
torch::Tensor sorted_ids,
|
||||
torch::Tensor batch_ids,
|
||||
torch::Tensor num_tokens_post_pad) {
|
||||
const torch::stable::Tensor& batch_num_tokens,
|
||||
torch::stable::Tensor sorted_ids,
|
||||
torch::stable::Tensor batch_ids,
|
||||
torch::stable::Tensor num_tokens_post_pad) {
|
||||
namespace batched_kernel = vllm::moe::batched_moe_align_block_size;
|
||||
|
||||
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
|
||||
const cudaStream_t stream =
|
||||
get_current_cuda_stream(batch_num_tokens.get_device_index());
|
||||
int32_t const B = batch_num_tokens.size(0);
|
||||
int32_t const num_blocks_per_batch =
|
||||
round_to_next_multiple_of(max_tokens_per_batch, block_size) / block_size;
|
||||
int32_t const num_blocks = num_blocks_per_batch * B;
|
||||
int64_t const sorted_ids_size = num_blocks * block_size;
|
||||
|
||||
TORCH_CHECK(sorted_ids.size(0) == sorted_ids_size);
|
||||
TORCH_CHECK(batch_ids.size(0) == sorted_ids_size / block_size);
|
||||
TORCH_CHECK(num_tokens_post_pad.size(0) == 1);
|
||||
TORCH_CHECK(B <= batched_kernel::num_threads);
|
||||
STD_TORCH_CHECK(sorted_ids.size(0) == sorted_ids_size);
|
||||
STD_TORCH_CHECK(batch_ids.size(0) == sorted_ids_size / block_size);
|
||||
STD_TORCH_CHECK(num_tokens_post_pad.size(0) == 1);
|
||||
STD_TORCH_CHECK(B <= batched_kernel::num_threads);
|
||||
|
||||
batched_kernel::batched_moe_align_block_size_kernel<<<
|
||||
batched_kernel::num_blocks, batched_kernel::num_threads, 0, stream>>>(
|
||||
B, max_tokens_per_batch, block_size, batch_num_tokens.data_ptr<int32_t>(),
|
||||
sorted_ids.data_ptr<int32_t>(), batch_ids.data_ptr<int32_t>(),
|
||||
num_tokens_post_pad.data_ptr<int32_t>());
|
||||
B, max_tokens_per_batch, block_size,
|
||||
reinterpret_cast<const int32_t*>(batch_num_tokens.const_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(sorted_ids.mutable_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(batch_ids.mutable_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(num_tokens_post_pad.mutable_data_ptr()));
|
||||
}
|
||||
|
||||
void moe_sum(torch::Tensor& input, // [num_tokens, topk, hidden_size]
|
||||
torch::Tensor& output) // [num_tokens, hidden_size]
|
||||
void moe_sum(torch::stable::Tensor& input, // [num_tokens, topk, hidden_size]
|
||||
torch::stable::Tensor& output) // [num_tokens, hidden_size]
|
||||
{
|
||||
const int hidden_size = input.size(-1);
|
||||
const auto num_tokens = output.numel() / hidden_size;
|
||||
@@ -622,77 +632,86 @@ void moe_sum(torch::Tensor& input, // [num_tokens, topk, hidden_size]
|
||||
|
||||
dim3 grid(num_tokens);
|
||||
dim3 block(std::min(hidden_size, 1024));
|
||||
const at::cuda::OptionalCUDAGuard device_guard(device_of(output));
|
||||
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
output.get_device_index());
|
||||
const cudaStream_t stream =
|
||||
get_current_cuda_stream(output.get_device_index());
|
||||
|
||||
switch (topk) {
|
||||
case 2:
|
||||
VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "moe_sum_kernel", [&] {
|
||||
vllm::moe::moe_sum_kernel<scalar_t, 2><<<grid, block, 0, stream>>>(
|
||||
output.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(),
|
||||
hidden_size);
|
||||
});
|
||||
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
|
||||
input.scalar_type(), "moe_sum_kernel", [&] {
|
||||
vllm::moe::moe_sum_kernel<scalar_t, 2><<<grid, block, 0, stream>>>(
|
||||
reinterpret_cast<scalar_t*>(output.mutable_data_ptr()),
|
||||
reinterpret_cast<const scalar_t*>(input.const_data_ptr()),
|
||||
hidden_size);
|
||||
});
|
||||
break;
|
||||
|
||||
case 3:
|
||||
VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "moe_sum_kernel", [&] {
|
||||
vllm::moe::moe_sum_kernel<scalar_t, 3><<<grid, block, 0, stream>>>(
|
||||
output.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(),
|
||||
hidden_size);
|
||||
});
|
||||
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
|
||||
input.scalar_type(), "moe_sum_kernel", [&] {
|
||||
vllm::moe::moe_sum_kernel<scalar_t, 3><<<grid, block, 0, stream>>>(
|
||||
reinterpret_cast<scalar_t*>(output.mutable_data_ptr()),
|
||||
reinterpret_cast<const scalar_t*>(input.const_data_ptr()),
|
||||
hidden_size);
|
||||
});
|
||||
break;
|
||||
|
||||
case 4:
|
||||
VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "moe_sum_kernel", [&] {
|
||||
vllm::moe::moe_sum_kernel<scalar_t, 4><<<grid, block, 0, stream>>>(
|
||||
output.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(),
|
||||
hidden_size);
|
||||
});
|
||||
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
|
||||
input.scalar_type(), "moe_sum_kernel", [&] {
|
||||
vllm::moe::moe_sum_kernel<scalar_t, 4><<<grid, block, 0, stream>>>(
|
||||
reinterpret_cast<scalar_t*>(output.mutable_data_ptr()),
|
||||
reinterpret_cast<const scalar_t*>(input.const_data_ptr()),
|
||||
hidden_size);
|
||||
});
|
||||
break;
|
||||
|
||||
default:
|
||||
at::sum_out(output, input, 1);
|
||||
torch::stable::sum_out(output, input, std::array<int64_t, 1>{1});
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
void moe_lora_align_block_size(
|
||||
torch::Tensor topk_ids, torch::Tensor token_lora_mapping,
|
||||
torch::stable::Tensor topk_ids, torch::stable::Tensor token_lora_mapping,
|
||||
int64_t num_experts, int64_t block_size, int64_t max_loras,
|
||||
int64_t max_num_tokens_padded, int64_t max_num_m_blocks,
|
||||
torch::Tensor sorted_token_ids, torch::Tensor expert_ids,
|
||||
torch::Tensor num_tokens_post_pad, torch::Tensor adapter_enabled,
|
||||
torch::Tensor lora_ids, std::optional<torch::Tensor> maybe_expert_map) {
|
||||
torch::stable::Tensor sorted_token_ids, torch::stable::Tensor expert_ids,
|
||||
torch::stable::Tensor num_tokens_post_pad,
|
||||
torch::stable::Tensor adapter_enabled, torch::stable::Tensor lora_ids,
|
||||
std::optional<torch::stable::Tensor> maybe_expert_map) {
|
||||
const int topk_num = topk_ids.size(1);
|
||||
|
||||
TORCH_CHECK(block_size > 0, "block_size should be greater than 0. ");
|
||||
STD_TORCH_CHECK(block_size > 0, "block_size should be greater than 0. ");
|
||||
|
||||
int device_max_shared_mem;
|
||||
auto dev = topk_ids.get_device();
|
||||
int dev = topk_ids.get_device_index();
|
||||
cudaDeviceGetAttribute(&device_max_shared_mem,
|
||||
cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
|
||||
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
|
||||
const cudaStream_t stream = get_current_cuda_stream(dev);
|
||||
|
||||
int64_t padded_num_experts =
|
||||
((num_experts + WARP_SIZE - 1) / WARP_SIZE) * WARP_SIZE;
|
||||
|
||||
// BlockScan uses 1024 threads and assigns one thread per expert.
|
||||
TORCH_CHECK(padded_num_experts < 1024,
|
||||
"padded_num_experts must be less than 1024");
|
||||
STD_TORCH_CHECK(padded_num_experts < 1024,
|
||||
"padded_num_experts must be less than 1024");
|
||||
|
||||
auto options_int =
|
||||
torch::TensorOptions().dtype(torch::kInt).device(topk_ids.device());
|
||||
torch::Tensor token_mask =
|
||||
torch::empty({max_loras * topk_ids.size(0)}, options_int);
|
||||
torch::stable::Tensor token_mask =
|
||||
torch::stable::new_empty(topk_ids, {max_loras * topk_ids.size(0)},
|
||||
torch::headeronly::ScalarType::Int);
|
||||
bool has_expert_map = maybe_expert_map.has_value();
|
||||
torch::Tensor expert_map;
|
||||
torch::stable::Tensor expert_map;
|
||||
if (has_expert_map) {
|
||||
expert_map = maybe_expert_map.value();
|
||||
} else {
|
||||
expert_map = torch::empty({0}, options_int);
|
||||
expert_map = torch::stable::new_empty(topk_ids, {0},
|
||||
torch::headeronly::ScalarType::Int);
|
||||
}
|
||||
|
||||
VLLM_DISPATCH_INTEGRAL_TYPES(
|
||||
VLLM_STABLE_DISPATCH_INTEGRAL_TYPES(
|
||||
topk_ids.scalar_type(), "moe_lora_align_sum_kernel", [&] {
|
||||
bool small_batch_expert_mode =
|
||||
(topk_ids.numel() < 1024) && (num_experts <= 64);
|
||||
@@ -703,7 +722,7 @@ void moe_lora_align_block_size(
|
||||
(num_thread + 1) * num_experts * sizeof(int32_t) +
|
||||
(num_experts + 1) * sizeof(int32_t);
|
||||
if (shared_mem > device_max_shared_mem) {
|
||||
TORCH_CHECK(false, "Shared memory usage exceeds device limit.");
|
||||
STD_TORCH_CHECK(false, "Shared memory usage exceeds device limit.");
|
||||
}
|
||||
|
||||
// threadIdx.x >= fill_threads: counting experts and aligning
|
||||
@@ -714,7 +733,7 @@ void moe_lora_align_block_size(
|
||||
auto kernel =
|
||||
vllm::moe::moe_lora_align_block_size_small_batch_expert_kernel<
|
||||
scalar_t, fill_threads>;
|
||||
AT_CUDA_CHECK(VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize(
|
||||
STD_CUDA_CHECK(VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize(
|
||||
(void*)kernel, shared_mem));
|
||||
// Grid size is (max_loras + 1) because active_lora_ids has length
|
||||
// max_loras + 1: sorted-unique values of token_lora_mapping, which
|
||||
@@ -725,15 +744,21 @@ void moe_lora_align_block_size(
|
||||
// MoE-LoRA kernels. This mirrors the fix made for the Triton
|
||||
// _fused_moe_lora_kernel grid in vllm-project/vllm#32277.
|
||||
kernel<<<max_loras + 1, blockDim, shared_mem, stream>>>(
|
||||
topk_ids.data_ptr<scalar_t>(),
|
||||
token_lora_mapping.data_ptr<int32_t>(), block_size,
|
||||
expert_map.data_ptr<int32_t>(), num_experts, max_loras,
|
||||
topk_ids.numel(), max_num_tokens_padded, max_num_m_blocks,
|
||||
sorted_token_ids.data_ptr<int32_t>(),
|
||||
expert_ids.data_ptr<int32_t>(), topk_num,
|
||||
num_tokens_post_pad.data_ptr<int32_t>(),
|
||||
adapter_enabled.data_ptr<int32_t>(), lora_ids.data_ptr<int32_t>(),
|
||||
token_mask.data_ptr<int32_t>(), has_expert_map);
|
||||
reinterpret_cast<scalar_t*>(topk_ids.mutable_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(token_lora_mapping.mutable_data_ptr()),
|
||||
block_size,
|
||||
reinterpret_cast<int32_t*>(expert_map.mutable_data_ptr()),
|
||||
num_experts, max_loras, topk_ids.numel(), max_num_tokens_padded,
|
||||
max_num_m_blocks,
|
||||
reinterpret_cast<int32_t*>(sorted_token_ids.mutable_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(expert_ids.mutable_data_ptr()),
|
||||
topk_num,
|
||||
reinterpret_cast<int32_t*>(
|
||||
num_tokens_post_pad.mutable_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(adapter_enabled.mutable_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(lora_ids.mutable_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(token_mask.mutable_data_ptr()),
|
||||
has_expert_map);
|
||||
} else {
|
||||
int num_thread = 1024;
|
||||
dim3 blockDim(num_thread);
|
||||
@@ -742,8 +767,9 @@ void moe_lora_align_block_size(
|
||||
size_t shared_mem_size = num_warps * WARP_SIZE * sizeof(int32_t);
|
||||
|
||||
// cumsum buffer
|
||||
torch::Tensor cumsum =
|
||||
torch::zeros({max_loras * (num_experts + 1)}, options_int);
|
||||
torch::stable::Tensor cumsum = torch::stable::new_zeros(
|
||||
topk_ids, {max_loras * (num_experts + 1)},
|
||||
torch::headeronly::ScalarType::Int);
|
||||
|
||||
auto align_kernel =
|
||||
vllm::moe::moe_lora_align_block_size_kernel<scalar_t>;
|
||||
@@ -759,16 +785,23 @@ void moe_lora_align_block_size(
|
||||
// blockIdx.x % 2 == 1: filling sorted_token_ids
|
||||
align_kernel<<<(max_loras + 1) * 2, blockDim, shared_mem_size,
|
||||
stream>>>(
|
||||
topk_ids.data_ptr<scalar_t>(),
|
||||
token_lora_mapping.data_ptr<int32_t>(), block_size,
|
||||
expert_map.data_ptr<int32_t>(), num_experts, max_loras,
|
||||
topk_ids.numel(), max_num_tokens_padded, max_num_m_blocks,
|
||||
sorted_token_ids.data_ptr<int32_t>(),
|
||||
expert_ids.data_ptr<int32_t>(), topk_num,
|
||||
num_tokens_post_pad.data_ptr<int32_t>(),
|
||||
adapter_enabled.data_ptr<int32_t>(), cumsum.data_ptr<int32_t>(),
|
||||
WARP_SIZE, padded_num_experts, lora_ids.data_ptr<int32_t>(),
|
||||
token_mask.data_ptr<int32_t>(), has_expert_map);
|
||||
reinterpret_cast<scalar_t*>(topk_ids.mutable_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(token_lora_mapping.mutable_data_ptr()),
|
||||
block_size,
|
||||
reinterpret_cast<int32_t*>(expert_map.mutable_data_ptr()),
|
||||
num_experts, max_loras, topk_ids.numel(), max_num_tokens_padded,
|
||||
max_num_m_blocks,
|
||||
reinterpret_cast<int32_t*>(sorted_token_ids.mutable_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(expert_ids.mutable_data_ptr()),
|
||||
topk_num,
|
||||
reinterpret_cast<int32_t*>(
|
||||
num_tokens_post_pad.mutable_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(adapter_enabled.mutable_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(cumsum.mutable_data_ptr()), WARP_SIZE,
|
||||
padded_num_experts,
|
||||
reinterpret_cast<int32_t*>(lora_ids.mutable_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(token_mask.mutable_data_ptr()),
|
||||
has_expert_map);
|
||||
|
||||
const int block_threads = std::min(256, (int)num_thread);
|
||||
const int num_blocks =
|
||||
@@ -785,12 +818,16 @@ void moe_lora_align_block_size(
|
||||
vllm::moe::lora_count_and_sort_expert_tokens_kernel<scalar_t>;
|
||||
|
||||
sort_kernel<<<gridDims, block_threads, 0, stream>>>(
|
||||
topk_ids.data_ptr<scalar_t>(),
|
||||
sorted_token_ids.data_ptr<int32_t>(), cumsum.data_ptr<int32_t>(),
|
||||
expert_map.data_ptr<int32_t>(), topk_ids.numel(), num_experts,
|
||||
max_num_tokens_padded, topk_num, token_mask.data_ptr<int32_t>(),
|
||||
max_loras, lora_ids.data_ptr<int32_t>(),
|
||||
adapter_enabled.data_ptr<int32_t>(), has_expert_map);
|
||||
reinterpret_cast<const scalar_t*>(topk_ids.const_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(sorted_token_ids.mutable_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(cumsum.mutable_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(expert_map.mutable_data_ptr()),
|
||||
topk_ids.numel(), num_experts, max_num_tokens_padded, topk_num,
|
||||
reinterpret_cast<int32_t*>(token_mask.mutable_data_ptr()),
|
||||
max_loras,
|
||||
reinterpret_cast<int32_t*>(lora_ids.mutable_data_ptr()),
|
||||
reinterpret_cast<int32_t*>(adapter_enabled.mutable_data_ptr()),
|
||||
has_expert_map);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
#pragma once
|
||||
|
||||
#include <torch/csrc/stable/tensor.h>
|
||||
|
||||
#include <optional>
|
||||
#include <tuple>
|
||||
|
||||
void topk_softmax(torch::stable::Tensor& topk_weights,
|
||||
torch::stable::Tensor& topk_indices,
|
||||
torch::stable::Tensor& token_expert_indices,
|
||||
torch::stable::Tensor& gating_output, bool renormalize,
|
||||
std::optional<torch::stable::Tensor> bias);
|
||||
|
||||
void topk_sigmoid(torch::stable::Tensor& topk_weights,
|
||||
torch::stable::Tensor& topk_indices,
|
||||
torch::stable::Tensor& token_expert_indices,
|
||||
torch::stable::Tensor& gating_output, bool renormalize,
|
||||
std::optional<torch::stable::Tensor> bias);
|
||||
|
||||
void topk_softplus_sqrt(
|
||||
torch::stable::Tensor& topk_weights, torch::stable::Tensor& topk_indices,
|
||||
torch::stable::Tensor& token_expert_indices,
|
||||
torch::stable::Tensor& gating_output, bool renormalize,
|
||||
double routed_scaling_factor,
|
||||
const std::optional<torch::stable::Tensor>& correction_bias,
|
||||
const std::optional<torch::stable::Tensor>& input_ids,
|
||||
const std::optional<torch::stable::Tensor>& tid2eid);
|
||||
|
||||
void moe_sum(torch::stable::Tensor& input, torch::stable::Tensor& output);
|
||||
|
||||
void moe_align_block_size(
|
||||
torch::stable::Tensor topk_ids, int64_t num_experts, int64_t block_size,
|
||||
torch::stable::Tensor sorted_token_ids, torch::stable::Tensor experts_ids,
|
||||
torch::stable::Tensor num_tokens_post_pad,
|
||||
std::optional<torch::stable::Tensor> maybe_expert_map);
|
||||
|
||||
void batched_moe_align_block_size(
|
||||
int64_t max_tokens_per_batch, int64_t block_size,
|
||||
const torch::stable::Tensor& expert_num_tokens,
|
||||
torch::stable::Tensor sorted_ids, torch::stable::Tensor expert_ids,
|
||||
torch::stable::Tensor num_tokens_post_pad);
|
||||
|
||||
void moe_lora_align_block_size(
|
||||
torch::stable::Tensor topk_ids, torch::stable::Tensor token_lora_mapping,
|
||||
int64_t num_experts, int64_t block_size, int64_t max_loras,
|
||||
int64_t max_num_tokens_padded, int64_t max_num_m_blocks,
|
||||
torch::stable::Tensor sorted_token_ids, torch::stable::Tensor expert_ids,
|
||||
torch::stable::Tensor num_tokens_post_pad,
|
||||
torch::stable::Tensor adapter_enabled, torch::stable::Tensor lora_ids,
|
||||
std::optional<torch::stable::Tensor> maybe_expert_map);
|
||||
#ifndef USE_ROCM
|
||||
torch::stable::Tensor moe_wna16_gemm(
|
||||
torch::stable::Tensor input, torch::stable::Tensor output,
|
||||
torch::stable::Tensor b_qweight, torch::stable::Tensor b_scales,
|
||||
std::optional<torch::stable::Tensor> b_qzeros,
|
||||
std::optional<torch::stable::Tensor> topk_weights,
|
||||
torch::stable::Tensor sorted_token_ids, torch::stable::Tensor expert_ids,
|
||||
torch::stable::Tensor num_tokens_post_pad, int64_t top_k,
|
||||
int64_t BLOCK_SIZE_M, int64_t BLOCK_SIZE_N, int64_t BLOCK_SIZE_K,
|
||||
int64_t bit);
|
||||
|
||||
std::tuple<torch::stable::Tensor, torch::stable::Tensor> grouped_topk(
|
||||
const torch::stable::Tensor& scores, int64_t n_group, int64_t topk_group,
|
||||
int64_t topk, bool renormalize, double routed_scaling_factor,
|
||||
const torch::stable::Tensor& bias, int64_t scoring_func);
|
||||
#endif
|
||||
|
||||
bool moe_permute_unpermute_supported();
|
||||
|
||||
int64_t moe_permute_sort_workspace_size(int64_t num_expanded_rows,
|
||||
int64_t num_expert);
|
||||
|
||||
void shuffle_rows(const torch::stable::Tensor& input_tensor,
|
||||
const torch::stable::Tensor& dst2src_map,
|
||||
torch::stable::Tensor& output_tensor);
|
||||
|
||||
#ifndef USE_ROCM
|
||||
// DeepSeek V3 optimized router GEMM kernel for SM90+
|
||||
// Computes output = mat_a @ mat_b.T where:
|
||||
// mat_a: [num_tokens, hidden_dim] in bf16
|
||||
// mat_b: [num_experts, hidden_dim] in bf16
|
||||
// output: [num_tokens, num_experts] in bf16 or fp32
|
||||
// Supports num_tokens in [1, 16], num_experts in {256, 384}, hidden_dim = 7168
|
||||
void dsv3_router_gemm(torch::stable::Tensor& output,
|
||||
const torch::stable::Tensor& mat_a,
|
||||
const torch::stable::Tensor& mat_b);
|
||||
#endif
|
||||
@@ -0,0 +1,319 @@
|
||||
#include <cuda.h>
|
||||
#include <cuda_runtime.h>
|
||||
#include <torch/csrc/stable/accelerator.h>
|
||||
#include <torch/csrc/stable/ops.h>
|
||||
#include <torch/csrc/stable/tensor.h>
|
||||
#include <torch/headeronly/core/ScalarType.h>
|
||||
#include <torch/headeronly/util/Exception.h>
|
||||
|
||||
#include "core/registration.h"
|
||||
#include "libtorch_stable/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.h"
|
||||
#include "libtorch_stable/torch_utils.h"
|
||||
|
||||
#include <torch/csrc/stable/library.h>
|
||||
|
||||
// moe_permute kernels require at least CUDA 12.0
|
||||
#if defined(CUDA_VERSION) && (CUDA_VERSION >= 12000)
|
||||
|
||||
namespace {
|
||||
|
||||
int64_t product_integers(torch::headeronly::IntHeaderOnlyArrayRef sizes) {
|
||||
int64_t numel = 1;
|
||||
for (int64_t s : sizes) {
|
||||
numel *= s;
|
||||
}
|
||||
return numel;
|
||||
}
|
||||
|
||||
torch::stable::Tensor maybe_allocate_tensor(
|
||||
const std::optional<torch::stable::Tensor>& maybe_tensor,
|
||||
torch::headeronly::IntHeaderOnlyArrayRef expected_sizes,
|
||||
torch::headeronly::ScalarType dtype, torch::stable::Device device,
|
||||
char const* name) {
|
||||
auto expected_numel = product_integers(expected_sizes);
|
||||
if (maybe_tensor.has_value()) {
|
||||
auto tensor = maybe_tensor.value();
|
||||
STD_TORCH_CHECK(tensor.device() == device, name,
|
||||
" must be on the same device");
|
||||
STD_TORCH_CHECK(tensor.scalar_type() == dtype, name,
|
||||
" has incorrect dtype");
|
||||
STD_TORCH_CHECK(tensor.is_contiguous(), name, " must be contiguous");
|
||||
STD_TORCH_CHECK(tensor.numel() >= expected_numel, name,
|
||||
" is too small for the requested shape");
|
||||
auto flat_tensor = torch::stable::view(tensor, {tensor.numel()});
|
||||
return torch::stable::view(
|
||||
torch::stable::narrow(flat_tensor, 0, 0, expected_numel),
|
||||
expected_sizes);
|
||||
}
|
||||
return torch::stable::empty(expected_sizes, dtype, std::nullopt, device);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int64_t moe_permute_sort_workspace_size(int64_t num_expanded_rows,
|
||||
int64_t n_expert) {
|
||||
return static_cast<int64_t>(
|
||||
CubKeyValueSorter::getWorkspaceSize(num_expanded_rows, n_expert));
|
||||
}
|
||||
|
||||
void moe_permute_impl(
|
||||
const torch::stable::Tensor& input, // [n_token, hidden]
|
||||
const torch::stable::Tensor& topk_ids, // [n_token, topk]
|
||||
const torch::stable::Tensor& token_expert_indices, // [n_token, topk]
|
||||
const std::optional<torch::stable::Tensor>& expert_map, // [n_expert]
|
||||
int64_t n_expert, int64_t n_local_expert, int64_t topk,
|
||||
torch::stable::Tensor& permuted_input, // [permuted_size, hidden]
|
||||
torch::stable::Tensor& expert_first_token_offset, // [n_local_expert + 1]
|
||||
torch::stable::Tensor& inv_permuted_idx, // [n_token, topk]
|
||||
torch::stable::Tensor& permuted_idx, // [permute_size]
|
||||
const std::optional<torch::stable::Tensor>& maybe_sort_workspace,
|
||||
const std::optional<torch::stable::Tensor>& maybe_permuted_experts_id,
|
||||
const std::optional<torch::stable::Tensor>& maybe_sorted_row_idx,
|
||||
const std::optional<torch::stable::Tensor>& maybe_topk_ids_for_sort) {
|
||||
STD_TORCH_CHECK(expert_first_token_offset.scalar_type() ==
|
||||
torch::headeronly::ScalarType::Long,
|
||||
"expert_first_token_offset must be int64");
|
||||
STD_TORCH_CHECK(topk_ids.scalar_type() == torch::headeronly::ScalarType::Int,
|
||||
"topk_ids must be int32");
|
||||
STD_TORCH_CHECK(
|
||||
token_expert_indices.scalar_type() == torch::headeronly::ScalarType::Int,
|
||||
"token_expert_indices must be int32");
|
||||
STD_TORCH_CHECK(
|
||||
inv_permuted_idx.scalar_type() == torch::headeronly::ScalarType::Int,
|
||||
"inv_permuted_idx must be int32");
|
||||
STD_TORCH_CHECK(expert_first_token_offset.size(0) == n_local_expert + 1,
|
||||
"expert_first_token_offset shape != n_local_expert+1");
|
||||
STD_TORCH_CHECK(
|
||||
inv_permuted_idx.sizes().equals(token_expert_indices.sizes()),
|
||||
"token_expert_indices shape must be same as inv_permuted_idx");
|
||||
|
||||
auto device = input.device();
|
||||
auto n_token = input.sizes()[0];
|
||||
auto n_hidden = input.sizes()[1];
|
||||
auto expanded_rows = n_token * topk;
|
||||
auto stream = get_current_cuda_stream(input.get_device_index());
|
||||
|
||||
auto sorter_size = moe_permute_sort_workspace_size(expanded_rows, n_expert);
|
||||
auto sort_workspace = maybe_allocate_tensor(
|
||||
maybe_sort_workspace, {sorter_size}, torch::headeronly::ScalarType::Char,
|
||||
device, "sort_workspace");
|
||||
auto permuted_experts_id = maybe_allocate_tensor(
|
||||
maybe_permuted_experts_id, topk_ids.sizes(),
|
||||
torch::headeronly::ScalarType::Int, device, "permuted_experts_id");
|
||||
auto sorted_row_idx = maybe_allocate_tensor(
|
||||
maybe_sorted_row_idx, inv_permuted_idx.sizes(),
|
||||
torch::headeronly::ScalarType::Int, device, "sorted_row_idx");
|
||||
|
||||
CubKeyValueSorter sorter{};
|
||||
int64_t* valid_num_ptr = nullptr;
|
||||
torch::stable::Tensor topk_ids_for_sort = topk_ids;
|
||||
|
||||
if (expert_map.has_value()) {
|
||||
const int* expert_map_ptr = get_ptr<int>(expert_map.value());
|
||||
valid_num_ptr =
|
||||
get_ptr<int64_t>(expert_first_token_offset) + n_local_expert;
|
||||
topk_ids_for_sort = maybe_allocate_tensor(
|
||||
maybe_topk_ids_for_sort, topk_ids.sizes(),
|
||||
torch::headeronly::ScalarType::Int, device, "topk_ids_for_sort");
|
||||
torch::stable::copy_(topk_ids_for_sort, topk_ids);
|
||||
preprocessTopkIdLauncher(get_ptr<int>(topk_ids_for_sort), n_token * topk,
|
||||
expert_map_ptr, n_expert, stream);
|
||||
}
|
||||
|
||||
sortAndScanExpert(
|
||||
get_ptr<const int>(topk_ids_for_sort), get_ptr<int>(token_expert_indices),
|
||||
get_ptr<int>(permuted_experts_id), get_ptr<int>(sorted_row_idx),
|
||||
get_ptr<int64_t>(expert_first_token_offset), n_token, n_expert,
|
||||
n_local_expert, topk, sorter, get_ptr<int>(sort_workspace), stream);
|
||||
|
||||
MOE_DISPATCH(input.scalar_type(), [&] {
|
||||
expandInputRowsKernelLauncher<scalar_t>(
|
||||
get_ptr<scalar_t>(input), get_ptr<scalar_t>(permuted_input),
|
||||
get_ptr<int>(sorted_row_idx), get_ptr<int>(inv_permuted_idx),
|
||||
get_ptr<int>(permuted_idx), get_ptr<int64_t>(expert_first_token_offset),
|
||||
n_token, valid_num_ptr, n_hidden, topk, n_local_expert, stream);
|
||||
});
|
||||
}
|
||||
|
||||
void moe_permute(
|
||||
const torch::stable::Tensor& input, // [n_token, hidden]
|
||||
const torch::stable::Tensor& topk_ids, // [n_token, topk]
|
||||
const torch::stable::Tensor& token_expert_indices, // [n_token, topk]
|
||||
const std::optional<torch::stable::Tensor>& expert_map, // [n_expert]
|
||||
int64_t n_expert, int64_t n_local_expert, int64_t topk,
|
||||
torch::stable::Tensor& permuted_input, // [permuted_size, hidden]
|
||||
torch::stable::Tensor& expert_first_token_offset, // [n_local_expert + 1]
|
||||
torch::stable::Tensor& inv_permuted_idx, // [n_token, topk]
|
||||
torch::stable::Tensor& permuted_idx) { // [permute_size]
|
||||
moe_permute_impl(input, topk_ids, token_expert_indices, expert_map, n_expert,
|
||||
n_local_expert, topk, permuted_input,
|
||||
expert_first_token_offset, inv_permuted_idx, permuted_idx,
|
||||
std::nullopt, std::nullopt, std::nullopt, std::nullopt);
|
||||
}
|
||||
|
||||
void moe_permute_with_scratch(
|
||||
const torch::stable::Tensor& input, const torch::stable::Tensor& topk_ids,
|
||||
const torch::stable::Tensor& token_expert_indices,
|
||||
const std::optional<torch::stable::Tensor>& expert_map, int64_t n_expert,
|
||||
int64_t n_local_expert, int64_t topk, torch::stable::Tensor& permuted_input,
|
||||
torch::stable::Tensor& expert_first_token_offset,
|
||||
torch::stable::Tensor& inv_permuted_idx,
|
||||
torch::stable::Tensor& permuted_idx, torch::stable::Tensor& sort_workspace,
|
||||
torch::stable::Tensor& permuted_experts_id,
|
||||
torch::stable::Tensor& sorted_row_idx,
|
||||
torch::stable::Tensor& topk_ids_for_sort) {
|
||||
moe_permute_impl(input, topk_ids, token_expert_indices, expert_map, n_expert,
|
||||
n_local_expert, topk, permuted_input,
|
||||
expert_first_token_offset, inv_permuted_idx, permuted_idx,
|
||||
sort_workspace, permuted_experts_id, sorted_row_idx,
|
||||
topk_ids_for_sort);
|
||||
}
|
||||
|
||||
void moe_unpermute(
|
||||
const torch::stable::Tensor&
|
||||
permuted_hidden_states, // [n_token * topk, hidden]
|
||||
const torch::stable::Tensor& topk_weights, // [n_token, topk]
|
||||
const torch::stable::Tensor& inv_permuted_idx, // [n_token, topk]
|
||||
const std::optional<torch::stable::Tensor>&
|
||||
expert_first_token_offset, // [n_local_expert+1]
|
||||
int64_t topk,
|
||||
torch::stable::Tensor& hidden_states) { // [n_token, hidden]
|
||||
STD_TORCH_CHECK(
|
||||
permuted_hidden_states.scalar_type() == hidden_states.scalar_type(),
|
||||
"permuted_hidden_states dtype must be same as hidden_states");
|
||||
|
||||
auto n_token = hidden_states.size(0);
|
||||
auto n_hidden = hidden_states.size(1);
|
||||
auto stream = get_current_cuda_stream(hidden_states.get_device_index());
|
||||
|
||||
int64_t const* valid_ptr = nullptr;
|
||||
if (expert_first_token_offset.has_value()) {
|
||||
int n_local_expert = expert_first_token_offset.value().size(0) - 1;
|
||||
valid_ptr =
|
||||
get_ptr<int64_t>(expert_first_token_offset.value()) + n_local_expert;
|
||||
}
|
||||
|
||||
MOE_DISPATCH(hidden_states.scalar_type(), [&] {
|
||||
finalizeMoeRoutingKernelLauncher<scalar_t, scalar_t>(
|
||||
get_ptr<scalar_t>(permuted_hidden_states),
|
||||
get_ptr<scalar_t>(hidden_states), get_ptr<float>(topk_weights),
|
||||
get_ptr<int>(inv_permuted_idx), n_token, n_hidden, topk, valid_ptr,
|
||||
stream);
|
||||
});
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__global__ void shuffleInputRowsKernel(const T* input,
|
||||
const int32_t* dst2src_map, T* output,
|
||||
int64_t num_src_rows,
|
||||
int64_t num_dst_rows, int64_t num_cols) {
|
||||
int64_t dest_row_idx = blockIdx.x;
|
||||
int64_t const source_row_idx = dst2src_map[dest_row_idx];
|
||||
|
||||
if (blockIdx.x < num_dst_rows) {
|
||||
// Load 128-bits per thread
|
||||
constexpr int64_t ELEM_PER_THREAD = 128 / sizeof(T) / 8;
|
||||
using DataElem = cutlass::Array<T, ELEM_PER_THREAD>;
|
||||
|
||||
// Duplicate and permute rows
|
||||
auto const* source_row_ptr =
|
||||
reinterpret_cast<DataElem const*>(input + source_row_idx * num_cols);
|
||||
auto* dest_row_ptr =
|
||||
reinterpret_cast<DataElem*>(output + dest_row_idx * num_cols);
|
||||
|
||||
int64_t const start_offset = threadIdx.x;
|
||||
int64_t const stride = blockDim.x;
|
||||
int64_t const num_elems_in_col = num_cols / ELEM_PER_THREAD;
|
||||
|
||||
for (int elem_index = start_offset; elem_index < num_elems_in_col;
|
||||
elem_index += stride) {
|
||||
dest_row_ptr[elem_index] = source_row_ptr[elem_index];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void shuffle_rows(const torch::stable::Tensor& input_tensor,
|
||||
const torch::stable::Tensor& dst2src_map,
|
||||
torch::stable::Tensor& output_tensor) {
|
||||
STD_TORCH_CHECK(input_tensor.scalar_type() == output_tensor.scalar_type(),
|
||||
"Input and output tensors must have the same data type");
|
||||
|
||||
auto stream = get_current_cuda_stream(output_tensor.get_device_index());
|
||||
const int64_t blocks = output_tensor.size(0);
|
||||
const int64_t threads = 256;
|
||||
const int64_t num_dest_rows = output_tensor.size(0);
|
||||
const int64_t num_src_rows = input_tensor.size(0);
|
||||
const int64_t num_cols = input_tensor.size(1);
|
||||
|
||||
STD_TORCH_CHECK(!(num_cols % (128 / input_tensor.element_size() / 8)),
|
||||
"num_cols must be divisible by 128 / "
|
||||
"input_tensor.element_size() / 8");
|
||||
|
||||
MOE_DISPATCH(input_tensor.scalar_type(), [&] {
|
||||
shuffleInputRowsKernel<scalar_t><<<blocks, threads, 0, stream>>>(
|
||||
reinterpret_cast<const scalar_t*>(input_tensor.const_data_ptr()),
|
||||
reinterpret_cast<const int32_t*>(dst2src_map.const_data_ptr()),
|
||||
reinterpret_cast<scalar_t*>(output_tensor.mutable_data_ptr()),
|
||||
num_src_rows, num_dest_rows, num_cols);
|
||||
});
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
int64_t moe_permute_sort_workspace_size(int64_t num_expanded_rows,
|
||||
int64_t n_expert) {
|
||||
STD_TORCH_CHECK(
|
||||
false, "moe_permute_sort_workspace_size is not supported on CUDA < 12.0");
|
||||
}
|
||||
|
||||
void moe_permute(const torch::stable::Tensor& input,
|
||||
const torch::stable::Tensor& topk_ids,
|
||||
const torch::stable::Tensor& token_expert_indices,
|
||||
const std::optional<torch::stable::Tensor>& expert_map,
|
||||
int64_t n_expert, int64_t n_local_expert, int64_t topk,
|
||||
torch::stable::Tensor& permuted_input,
|
||||
torch::stable::Tensor& expert_first_token_offset,
|
||||
torch::stable::Tensor& inv_permuted_idx,
|
||||
torch::stable::Tensor& permuted_idx) {
|
||||
STD_TORCH_CHECK(false, "moe_permute is not supported on CUDA < 12.0");
|
||||
}
|
||||
|
||||
void moe_permute_with_scratch(
|
||||
const torch::stable::Tensor& input, const torch::stable::Tensor& topk_ids,
|
||||
const torch::stable::Tensor& token_expert_indices,
|
||||
const std::optional<torch::stable::Tensor>& expert_map, int64_t n_expert,
|
||||
int64_t n_local_expert, int64_t topk, torch::stable::Tensor& permuted_input,
|
||||
torch::stable::Tensor& expert_first_token_offset,
|
||||
torch::stable::Tensor& inv_permuted_idx,
|
||||
torch::stable::Tensor& permuted_idx, torch::stable::Tensor& sort_workspace,
|
||||
torch::stable::Tensor& permuted_experts_id,
|
||||
torch::stable::Tensor& sorted_row_idx,
|
||||
torch::stable::Tensor& topk_ids_for_sort) {
|
||||
STD_TORCH_CHECK(false,
|
||||
"moe_permute_with_scratch is not supported on CUDA < 12.0");
|
||||
}
|
||||
|
||||
void moe_unpermute(
|
||||
const torch::stable::Tensor& permuted_hidden_states,
|
||||
const torch::stable::Tensor& topk_weights,
|
||||
const torch::stable::Tensor& inv_permuted_idx,
|
||||
const std::optional<torch::stable::Tensor>& expert_first_token_offset,
|
||||
int64_t topk, torch::stable::Tensor& hidden_states) {
|
||||
STD_TORCH_CHECK(false, "moe_unpermute is not supported on CUDA < 12.0");
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
bool moe_permute_unpermute_supported() {
|
||||
#if defined(CUDA_VERSION) && (CUDA_VERSION >= 12000)
|
||||
return true;
|
||||
#else
|
||||
return false;
|
||||
#endif
|
||||
}
|
||||
|
||||
STABLE_TORCH_LIBRARY_IMPL(_moe_C, CUDA, m) {
|
||||
m.impl("moe_permute", TORCH_BOX(&moe_permute));
|
||||
m.impl("moe_permute_with_scratch", TORCH_BOX(&moe_permute_with_scratch));
|
||||
m.impl("moe_unpermute", TORCH_BOX(&moe_unpermute));
|
||||
}
|
||||
@@ -1,11 +1,14 @@
|
||||
#include <algorithm>
|
||||
|
||||
#include <torch/all.h>
|
||||
#include <c10/cuda/CUDAGuard.h>
|
||||
#include <ATen/cuda/CUDAContext.h>
|
||||
#include <cuda_runtime.h>
|
||||
#include <torch/csrc/stable/accelerator.h>
|
||||
#include <torch/csrc/stable/ops.h>
|
||||
#include <torch/csrc/stable/tensor.h>
|
||||
#include <torch/headeronly/core/ScalarType.h>
|
||||
|
||||
#include <cuda_fp16.h>
|
||||
#include <cuda_bf16.h>
|
||||
#include "libtorch_stable/torch_utils.h"
|
||||
#include "moe_wna16_utils.h"
|
||||
|
||||
#define DIVIDE(x, size) (((x) + (size) - 1) / (size))
|
||||
@@ -263,7 +266,7 @@ void run_moe_wna16_gemm(const scalar_t* input, scalar_t* output,
|
||||
}
|
||||
|
||||
const int shared_mem_size = BLOCK_SIZE_M * BLOCK_SIZE_K * 2;
|
||||
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
kernel<<<gridDim, blockDim, shared_mem_size, stream>>>(
|
||||
input, output, b_qweight, b_scales, b_qzeros, topk_weights,
|
||||
sorted_token_ids, expert_ids, num_tokens_post_pad, num_experts,
|
||||
@@ -271,17 +274,18 @@ void run_moe_wna16_gemm(const scalar_t* input, scalar_t* output,
|
||||
BLOCK_SIZE_K, has_zp, mul_topk_weight);
|
||||
}
|
||||
|
||||
torch::Tensor moe_wna16_gemm(torch::Tensor input, torch::Tensor output,
|
||||
torch::Tensor b_qweight, torch::Tensor b_scales,
|
||||
std::optional<torch::Tensor> b_qzeros,
|
||||
std::optional<torch::Tensor> topk_weights,
|
||||
torch::Tensor sorted_token_ids,
|
||||
torch::Tensor expert_ids,
|
||||
torch::Tensor num_tokens_post_pad, int64_t top_k,
|
||||
int64_t BLOCK_SIZE_M, int64_t BLOCK_SIZE_N,
|
||||
int64_t BLOCK_SIZE_K, int64_t bit) {
|
||||
const at::cuda::OptionalCUDAGuard device_guard(device_of(input));
|
||||
output.zero_();
|
||||
torch::stable::Tensor moe_wna16_gemm(
|
||||
torch::stable::Tensor input, torch::stable::Tensor output,
|
||||
torch::stable::Tensor b_qweight, torch::stable::Tensor b_scales,
|
||||
std::optional<torch::stable::Tensor> b_qzeros,
|
||||
std::optional<torch::stable::Tensor> topk_weights,
|
||||
torch::stable::Tensor sorted_token_ids, torch::stable::Tensor expert_ids,
|
||||
torch::stable::Tensor num_tokens_post_pad, int64_t top_k,
|
||||
int64_t BLOCK_SIZE_M, int64_t BLOCK_SIZE_N, int64_t BLOCK_SIZE_K,
|
||||
int64_t bit) {
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
input.get_device_index());
|
||||
torch::stable::zero_(output);
|
||||
|
||||
const int num_experts = b_qweight.size(0);
|
||||
const int size_m = input.size(0);
|
||||
@@ -291,52 +295,56 @@ torch::Tensor moe_wna16_gemm(torch::Tensor input, torch::Tensor output,
|
||||
|
||||
int64_t EM = sorted_token_ids.size(0);
|
||||
if (size_m <= BLOCK_SIZE_M) {
|
||||
EM = min(EM, size_m * BLOCK_SIZE_M * top_k);
|
||||
EM = std::min(EM, size_m * BLOCK_SIZE_M * top_k);
|
||||
}
|
||||
const int num_token_blocks = (EM + BLOCK_SIZE_M - 1) / BLOCK_SIZE_M;
|
||||
|
||||
const uint32_t* b_qzeros_ptr;
|
||||
if (b_qzeros.has_value())
|
||||
b_qzeros_ptr = (const uint32_t*)b_qzeros.value().data_ptr<uint8_t>();
|
||||
b_qzeros_ptr = (const uint32_t*)b_qzeros.value().const_data_ptr<uint8_t>();
|
||||
const float* topk_weights_ptr = nullptr;
|
||||
if (topk_weights.has_value())
|
||||
topk_weights_ptr = (const float*)topk_weights.value().data_ptr<float>();
|
||||
topk_weights_ptr =
|
||||
(const float*)topk_weights.value().const_data_ptr<float>();
|
||||
|
||||
int groups_per_block_row = BLOCK_SIZE_K / group_size;
|
||||
TORCH_CHECK(bit == 4 || bit == 8, "bit must be 4 or 8");
|
||||
TORCH_CHECK(size_k % BLOCK_SIZE_K == 0,
|
||||
"size_k must divisible by BLOCK_SIZE_K");
|
||||
TORCH_CHECK(BLOCK_SIZE_K % group_size == 0,
|
||||
"BLOCK_SIZE_K must divisible by group_size");
|
||||
TORCH_CHECK(BLOCK_SIZE_M <= 64, "BLOCK_SIZE_M must less or equal to 64");
|
||||
TORCH_CHECK(groups_per_block_row == 1 || groups_per_block_row == 2 ||
|
||||
groups_per_block_row == 4 || groups_per_block_row == 8,
|
||||
"BLOCK_SIZE_K // group_size must be one of [1, 2, 4, 8]");
|
||||
STD_TORCH_CHECK(bit == 4 || bit == 8, "bit must be 4 or 8");
|
||||
STD_TORCH_CHECK(size_k % BLOCK_SIZE_K == 0,
|
||||
"size_k must divisible by BLOCK_SIZE_K");
|
||||
STD_TORCH_CHECK(BLOCK_SIZE_K % group_size == 0,
|
||||
"BLOCK_SIZE_K must divisible by group_size");
|
||||
STD_TORCH_CHECK(BLOCK_SIZE_M <= 64, "BLOCK_SIZE_M must less or equal to 64");
|
||||
STD_TORCH_CHECK(groups_per_block_row == 1 || groups_per_block_row == 2 ||
|
||||
groups_per_block_row == 4 || groups_per_block_row == 8,
|
||||
"BLOCK_SIZE_K // group_size must be one of [1, 2, 4, 8]");
|
||||
|
||||
if (input.scalar_type() == at::ScalarType::Half) {
|
||||
if (input.scalar_type() == torch::headeronly::ScalarType::Half) {
|
||||
run_moe_wna16_gemm<half>(
|
||||
(const half*)input.data_ptr<at::Half>(),
|
||||
(half*)output.data_ptr<at::Half>(),
|
||||
(const uint32_t*)b_qweight.data_ptr<uint8_t>(),
|
||||
(const half*)b_scales.data_ptr<at::Half>(), b_qzeros_ptr,
|
||||
topk_weights_ptr, sorted_token_ids.data_ptr<int32_t>(),
|
||||
expert_ids.data_ptr<int32_t>(), num_tokens_post_pad.data_ptr<int32_t>(),
|
||||
num_experts, group_size, num_token_blocks, top_k, size_m, size_n,
|
||||
size_k, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, bit,
|
||||
b_qzeros.has_value(), topk_weights.has_value());
|
||||
} else if (input.scalar_type() == at::ScalarType::BFloat16) {
|
||||
reinterpret_cast<const half*>(input.const_data_ptr()),
|
||||
reinterpret_cast<half*>(output.mutable_data_ptr()),
|
||||
(const uint32_t*)b_qweight.const_data_ptr<uint8_t>(),
|
||||
reinterpret_cast<const half*>(b_scales.const_data_ptr()), b_qzeros_ptr,
|
||||
topk_weights_ptr, sorted_token_ids.const_data_ptr<int32_t>(),
|
||||
expert_ids.const_data_ptr<int32_t>(),
|
||||
num_tokens_post_pad.const_data_ptr<int32_t>(), num_experts, group_size,
|
||||
num_token_blocks, top_k, size_m, size_n, size_k, BLOCK_SIZE_M,
|
||||
BLOCK_SIZE_N, BLOCK_SIZE_K, bit, b_qzeros.has_value(),
|
||||
topk_weights.has_value());
|
||||
} else if (input.scalar_type() == torch::headeronly::ScalarType::BFloat16) {
|
||||
run_moe_wna16_gemm<nv_bfloat16>(
|
||||
(const nv_bfloat16*)input.data_ptr<at::BFloat16>(),
|
||||
(nv_bfloat16*)output.data_ptr<at::BFloat16>(),
|
||||
(const uint32_t*)b_qweight.data_ptr<uint8_t>(),
|
||||
(const nv_bfloat16*)b_scales.data_ptr<at::BFloat16>(), b_qzeros_ptr,
|
||||
topk_weights_ptr, sorted_token_ids.data_ptr<int32_t>(),
|
||||
expert_ids.data_ptr<int32_t>(), num_tokens_post_pad.data_ptr<int32_t>(),
|
||||
num_experts, group_size, num_token_blocks, top_k, size_m, size_n,
|
||||
size_k, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, bit,
|
||||
b_qzeros.has_value(), topk_weights.has_value());
|
||||
reinterpret_cast<const nv_bfloat16*>(input.const_data_ptr()),
|
||||
reinterpret_cast<nv_bfloat16*>(output.mutable_data_ptr()),
|
||||
(const uint32_t*)b_qweight.const_data_ptr<uint8_t>(),
|
||||
reinterpret_cast<const nv_bfloat16*>(b_scales.const_data_ptr()),
|
||||
b_qzeros_ptr, topk_weights_ptr,
|
||||
sorted_token_ids.const_data_ptr<int32_t>(),
|
||||
expert_ids.const_data_ptr<int32_t>(),
|
||||
num_tokens_post_pad.const_data_ptr<int32_t>(), num_experts, group_size,
|
||||
num_token_blocks, top_k, size_m, size_n, size_k, BLOCK_SIZE_M,
|
||||
BLOCK_SIZE_N, BLOCK_SIZE_K, bit, b_qzeros.has_value(),
|
||||
topk_weights.has_value());
|
||||
} else {
|
||||
TORCH_CHECK(false, "moe_wna16_gemm only supports bfloat16 and float16");
|
||||
STD_TORCH_CHECK(false, "moe_wna16_gemm only supports bfloat16 and float16");
|
||||
}
|
||||
return output;
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
#pragma once
|
||||
|
||||
#include <cuda_fp8.h>
|
||||
#include <torch/headeronly/core/ScalarType.h>
|
||||
#include <torch/headeronly/util/Exception.h>
|
||||
|
||||
#define MOE_SWITCH(TYPE, ...) \
|
||||
const auto _st = (TYPE); \
|
||||
switch (_st) { \
|
||||
__VA_ARGS__ \
|
||||
default: \
|
||||
STD_TORCH_CHECK(false, "[moe permute]data type dispatch fail!") \
|
||||
}
|
||||
|
||||
#define MOE_DISPATCH_CASE(enum_type, ...) \
|
||||
case enum_type: { \
|
||||
using scalar_t = ScalarType2CudaType<enum_type>::type; \
|
||||
__VA_ARGS__(); \
|
||||
break; \
|
||||
}
|
||||
|
||||
#define MOE_DISPATCH_FLOAT_CASE(...) \
|
||||
MOE_DISPATCH_CASE(torch::headeronly::ScalarType::Float, __VA_ARGS__) \
|
||||
MOE_DISPATCH_CASE(torch::headeronly::ScalarType::Half, __VA_ARGS__) \
|
||||
MOE_DISPATCH_CASE(torch::headeronly::ScalarType::BFloat16, __VA_ARGS__) \
|
||||
MOE_DISPATCH_CASE(torch::headeronly::ScalarType::Float8_e5m2, __VA_ARGS__) \
|
||||
MOE_DISPATCH_CASE(torch::headeronly::ScalarType::Float8_e4m3fn, __VA_ARGS__) \
|
||||
MOE_DISPATCH_CASE(torch::headeronly::ScalarType::Byte, __VA_ARGS__)
|
||||
|
||||
#define MOE_DISPATCH(TYPE, ...) \
|
||||
MOE_SWITCH(TYPE, MOE_DISPATCH_FLOAT_CASE(__VA_ARGS__))
|
||||
|
||||
template <torch::headeronly::ScalarType type>
|
||||
struct ScalarType2CudaType;
|
||||
|
||||
template <>
|
||||
struct ScalarType2CudaType<torch::headeronly::ScalarType::Float> {
|
||||
using type = float;
|
||||
};
|
||||
template <>
|
||||
struct ScalarType2CudaType<torch::headeronly::ScalarType::Half> {
|
||||
using type = half;
|
||||
};
|
||||
template <>
|
||||
struct ScalarType2CudaType<torch::headeronly::ScalarType::BFloat16> {
|
||||
using type = __nv_bfloat16;
|
||||
};
|
||||
// uint8 for packed fp4
|
||||
template <>
|
||||
struct ScalarType2CudaType<torch::headeronly::ScalarType::Byte> {
|
||||
using type = uint8_t;
|
||||
};
|
||||
template <>
|
||||
struct ScalarType2CudaType<torch::headeronly::ScalarType::Float8_e5m2> {
|
||||
using type = __nv_fp8_e5m2;
|
||||
};
|
||||
template <>
|
||||
struct ScalarType2CudaType<torch::headeronly::ScalarType::Float8_e4m3fn> {
|
||||
using type = __nv_fp8_e4m3;
|
||||
};
|
||||
+7
-4
@@ -1,5 +1,7 @@
|
||||
#include <cuda.h>
|
||||
#include <torch/headeronly/util/Exception.h>
|
||||
|
||||
#include "moe_permute_unpermute_kernel.h"
|
||||
#include "libtorch_stable/moe/permute_unpermute_kernels/moe_permute_unpermute_kernel.h"
|
||||
|
||||
// moe_permute kernels require at least CUDA 12.0
|
||||
#if defined(CUDA_VERSION) && (CUDA_VERSION >= 12000)
|
||||
@@ -48,9 +50,10 @@ void CubKeyValueSorter::run(void* workspace, size_t const workspace_size,
|
||||
size_t expected_ws_size = getWorkspaceSize(num_key_value_pairs, num_experts_);
|
||||
size_t actual_ws_size = workspace_size;
|
||||
|
||||
TORCH_CHECK(expected_ws_size <= workspace_size,
|
||||
"[CubKeyValueSorter::run] The allocated workspace is too small "
|
||||
"to run this problem.");
|
||||
STD_TORCH_CHECK(
|
||||
expected_ws_size <= workspace_size,
|
||||
"[CubKeyValueSorter::run] The allocated workspace is too small "
|
||||
"to run this problem.");
|
||||
cub::DeviceRadixSort::SortPairs(workspace, actual_ws_size, keys_in, keys_out,
|
||||
values_in, values_out, num_key_value_pairs, 0,
|
||||
num_bits_, stream);
|
||||
+9
-8
@@ -2,23 +2,24 @@
|
||||
// reference from tensorrt_llm moe kernel implementation archive in
|
||||
// https://github.com/BBuf/tensorrt-llm-moe/tree/master
|
||||
|
||||
#include <c10/core/ScalarType.h>
|
||||
#include <torch/all.h>
|
||||
#include "dispatch.h"
|
||||
#include <torch/csrc/stable/tensor.h>
|
||||
|
||||
#include <cub/cub.cuh>
|
||||
#include <cub/device/device_radix_sort.cuh>
|
||||
#include <cub/util_type.cuh>
|
||||
#include "cutlass/numeric_size.h"
|
||||
|
||||
#include "cutlass/array.h"
|
||||
#include "cutlass/numeric_size.h"
|
||||
#include "libtorch_stable/moe/permute_unpermute_kernels/dispatch.h"
|
||||
|
||||
template <typename T>
|
||||
inline T* get_ptr(torch::Tensor& t) {
|
||||
return reinterpret_cast<T*>(t.data_ptr());
|
||||
inline T* get_ptr(torch::stable::Tensor& t) {
|
||||
return reinterpret_cast<T*>(t.mutable_data_ptr());
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline const T* get_ptr(const torch::Tensor& t) {
|
||||
return reinterpret_cast<const T*>(t.data_ptr());
|
||||
inline const T* get_ptr(const torch::stable::Tensor& t) {
|
||||
return reinterpret_cast<const T*>(t.const_data_ptr());
|
||||
}
|
||||
|
||||
class CubKeyValueSorter {
|
||||
+72
-63
@@ -17,11 +17,16 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
#include <type_traits>
|
||||
#include <torch/all.h>
|
||||
#include <ATen/cuda/CUDAContext.h>
|
||||
#include <c10/cuda/CUDAGuard.h>
|
||||
#include "../cuda_compat.h"
|
||||
#include "../cub_helpers.h"
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
#include <torch/csrc/stable/accelerator.h>
|
||||
#include <torch/csrc/stable/tensor.h>
|
||||
#include <torch/headeronly/core/ScalarType.h>
|
||||
#include <torch/headeronly/util/Exception.h>
|
||||
|
||||
#include "../../cuda_compat.h"
|
||||
#include "../../cub_helpers.h"
|
||||
#include "libtorch_stable/torch_utils.h"
|
||||
|
||||
#ifndef USE_ROCM
|
||||
#include <cuda_bf16.h>
|
||||
@@ -713,7 +718,7 @@ void topkGatingKernelLauncher(
|
||||
break;
|
||||
#endif
|
||||
default: {
|
||||
TORCH_CHECK(workspace != nullptr,
|
||||
STD_TORCH_CHECK(workspace != nullptr,
|
||||
"workspace must be provided for num_experts that are not a power of 2 or multiple of 64.");
|
||||
static constexpr int TPB = 256;
|
||||
if constexpr (SF == SCORING_SOFTMAX) {
|
||||
@@ -723,7 +728,7 @@ void topkGatingKernelLauncher(
|
||||
moeSigmoid<TPB, InputType><<<num_tokens, TPB, 0, stream>>>(
|
||||
gating_output, nullptr, workspace, num_experts);
|
||||
} else {
|
||||
TORCH_CHECK(false, "Unsupported scoring func");
|
||||
STD_TORCH_CHECK(false, "Unsupported scoring func");
|
||||
}
|
||||
moeTopK<TPB><<<num_tokens, TPB, 0, stream>>>(
|
||||
workspace, nullptr, topk_weights, topk_indices, token_expert_indices,
|
||||
@@ -738,63 +743,65 @@ void topkGatingKernelLauncher(
|
||||
|
||||
template<typename ComputeType, vllm::moe::ScoringFunc SF>
|
||||
void dispatch_topk_launch(
|
||||
torch::Tensor& gating_output,
|
||||
torch::Tensor& topk_weights,
|
||||
torch::Tensor& topk_indices,
|
||||
torch::Tensor& token_expert_indices,
|
||||
torch::Tensor& softmax_workspace,
|
||||
torch::stable::Tensor& gating_output,
|
||||
torch::stable::Tensor& topk_weights,
|
||||
torch::stable::Tensor& topk_indices,
|
||||
torch::stable::Tensor& token_expert_indices,
|
||||
torch::stable::Tensor& softmax_workspace,
|
||||
int num_tokens, int num_experts, int topk, bool renormalize,
|
||||
std::optional<torch::Tensor> bias,
|
||||
std::optional<torch::stable::Tensor> bias,
|
||||
cudaStream_t stream)
|
||||
{
|
||||
const float* bias_ptr = nullptr;
|
||||
if (bias.has_value()) {
|
||||
const torch::Tensor& bias_tensor = bias.value();
|
||||
TORCH_CHECK(bias_tensor.scalar_type() == at::ScalarType::Float, "bias tensor must be float32");
|
||||
TORCH_CHECK(bias_tensor.dim() == 1, "bias tensor must be 1D");
|
||||
TORCH_CHECK(bias_tensor.size(0) == num_experts, "bias size mismatch, expected: ", num_experts);
|
||||
TORCH_CHECK(bias_tensor.is_contiguous(), "bias tensor must be contiguous");
|
||||
bias_ptr = bias_tensor.data_ptr<float>();
|
||||
const torch::stable::Tensor& bias_tensor = bias.value();
|
||||
STD_TORCH_CHECK(bias_tensor.scalar_type() == torch::headeronly::ScalarType::Float,
|
||||
"bias tensor must be float32");
|
||||
STD_TORCH_CHECK(bias_tensor.dim() == 1, "bias tensor must be 1D");
|
||||
STD_TORCH_CHECK(bias_tensor.size(0) == num_experts,
|
||||
"bias size mismatch, expected: ", num_experts);
|
||||
STD_TORCH_CHECK(bias_tensor.is_contiguous(), "bias tensor must be contiguous");
|
||||
bias_ptr = bias_tensor.const_data_ptr<float>();
|
||||
}
|
||||
|
||||
if (topk_indices.scalar_type() == at::ScalarType::Int) {
|
||||
if (topk_indices.scalar_type() == torch::headeronly::ScalarType::Int) {
|
||||
vllm::moe::topkGatingKernelLauncher<int, ComputeType, SF>(
|
||||
reinterpret_cast<const ComputeType*>(gating_output.data_ptr()),
|
||||
topk_weights.data_ptr<float>(),
|
||||
topk_indices.data_ptr<int>(),
|
||||
token_expert_indices.data_ptr<int>(),
|
||||
softmax_workspace.data_ptr<float>(),
|
||||
reinterpret_cast<const ComputeType*>(gating_output.const_data_ptr()),
|
||||
topk_weights.mutable_data_ptr<float>(),
|
||||
topk_indices.mutable_data_ptr<int>(),
|
||||
token_expert_indices.mutable_data_ptr<int>(),
|
||||
softmax_workspace.mutable_data_ptr<float>(),
|
||||
num_tokens, num_experts, topk, renormalize,
|
||||
bias_ptr, stream);
|
||||
} else if (topk_indices.scalar_type() == at::ScalarType::UInt32) {
|
||||
} else if (topk_indices.scalar_type() == torch::headeronly::ScalarType::UInt32) {
|
||||
vllm::moe::topkGatingKernelLauncher<uint32_t, ComputeType, SF>(
|
||||
reinterpret_cast<const ComputeType*>(gating_output.data_ptr()),
|
||||
topk_weights.data_ptr<float>(),
|
||||
topk_indices.data_ptr<uint32_t>(),
|
||||
token_expert_indices.data_ptr<int>(),
|
||||
softmax_workspace.data_ptr<float>(),
|
||||
reinterpret_cast<const ComputeType*>(gating_output.const_data_ptr()),
|
||||
topk_weights.mutable_data_ptr<float>(),
|
||||
topk_indices.mutable_data_ptr<uint32_t>(),
|
||||
token_expert_indices.mutable_data_ptr<int>(),
|
||||
softmax_workspace.mutable_data_ptr<float>(),
|
||||
num_tokens, num_experts, topk, renormalize,
|
||||
bias_ptr, stream);
|
||||
} else {
|
||||
TORCH_CHECK(topk_indices.scalar_type() == at::ScalarType::Long);
|
||||
STD_TORCH_CHECK(topk_indices.scalar_type() == torch::headeronly::ScalarType::Long);
|
||||
vllm::moe::topkGatingKernelLauncher<int64_t, ComputeType, SF>(
|
||||
reinterpret_cast<const ComputeType*>(gating_output.data_ptr()),
|
||||
topk_weights.data_ptr<float>(),
|
||||
topk_indices.data_ptr<int64_t>(),
|
||||
token_expert_indices.data_ptr<int>(),
|
||||
softmax_workspace.data_ptr<float>(),
|
||||
reinterpret_cast<const ComputeType*>(gating_output.const_data_ptr()),
|
||||
topk_weights.mutable_data_ptr<float>(),
|
||||
topk_indices.mutable_data_ptr<int64_t>(),
|
||||
token_expert_indices.mutable_data_ptr<int>(),
|
||||
softmax_workspace.mutable_data_ptr<float>(),
|
||||
num_tokens, num_experts, topk, renormalize,
|
||||
bias_ptr, stream);
|
||||
}
|
||||
}
|
||||
|
||||
void topk_softmax(
|
||||
torch::Tensor& topk_weights, // [num_tokens, topk]
|
||||
torch::Tensor& topk_indices, // [num_tokens, topk]
|
||||
torch::Tensor& token_expert_indices, // [num_tokens, topk]
|
||||
torch::Tensor& gating_output, // [num_tokens, num_experts]
|
||||
torch::stable::Tensor& topk_weights, // [num_tokens, topk]
|
||||
torch::stable::Tensor& topk_indices, // [num_tokens, topk]
|
||||
torch::stable::Tensor& token_expert_indices, // [num_tokens, topk]
|
||||
torch::stable::Tensor& gating_output, // [num_tokens, num_experts]
|
||||
bool renormalize,
|
||||
std::optional<torch::Tensor> bias)
|
||||
std::optional<torch::stable::Tensor> bias)
|
||||
{
|
||||
const int num_experts = gating_output.size(-1);
|
||||
const auto num_tokens = gating_output.numel() / num_experts;
|
||||
@@ -804,35 +811,36 @@ void topk_softmax(
|
||||
const bool needs_workspace = !is_pow_2 || num_experts > 256;
|
||||
const int64_t workspace_size = needs_workspace ? num_tokens * num_experts : 0;
|
||||
|
||||
const at::cuda::OptionalCUDAGuard device_guard(device_of(gating_output));
|
||||
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
|
||||
const auto workspace_options = gating_output.options().dtype(at::ScalarType::Float);
|
||||
torch::Tensor softmax_workspace = torch::empty({workspace_size}, workspace_options);
|
||||
torch::stable::accelerator::DeviceGuard guard(gating_output.get_device_index());
|
||||
const cudaStream_t stream =
|
||||
get_current_cuda_stream(gating_output.get_device_index());
|
||||
auto softmax_workspace = torch::stable::new_empty(
|
||||
gating_output, {workspace_size}, torch::headeronly::ScalarType::Float);
|
||||
|
||||
if (gating_output.scalar_type() == at::ScalarType::Float) {
|
||||
if (gating_output.scalar_type() == torch::headeronly::ScalarType::Float) {
|
||||
dispatch_topk_launch<float, vllm::moe::SCORING_SOFTMAX>(gating_output, topk_weights, topk_indices,
|
||||
token_expert_indices, softmax_workspace, num_tokens, num_experts, topk, renormalize,
|
||||
bias, stream);
|
||||
} else if (gating_output.scalar_type() == at::ScalarType::Half) {
|
||||
} else if (gating_output.scalar_type() == torch::headeronly::ScalarType::Half) {
|
||||
dispatch_topk_launch<__half, vllm::moe::SCORING_SOFTMAX>(gating_output, topk_weights, topk_indices,
|
||||
token_expert_indices, softmax_workspace, num_tokens, num_experts, topk, renormalize,
|
||||
bias, stream);
|
||||
} else if (gating_output.scalar_type() == at::ScalarType::BFloat16) {
|
||||
} else if (gating_output.scalar_type() == torch::headeronly::ScalarType::BFloat16) {
|
||||
dispatch_topk_launch<__nv_bfloat16, vllm::moe::SCORING_SOFTMAX>(gating_output, topk_weights, topk_indices,
|
||||
token_expert_indices, softmax_workspace, num_tokens, num_experts, topk, renormalize,
|
||||
bias, stream);
|
||||
} else {
|
||||
TORCH_CHECK(false, "Unsupported gating_output data type: ", gating_output.scalar_type());
|
||||
STD_TORCH_CHECK(false, "Unsupported gating_output data type: ", gating_output.scalar_type());
|
||||
}
|
||||
}
|
||||
|
||||
void topk_sigmoid(
|
||||
torch::Tensor& topk_weights, // [num_tokens, topk]
|
||||
torch::Tensor& topk_indices, // [num_tokens, topk]
|
||||
torch::Tensor& token_expert_indices, // [num_tokens, topk]
|
||||
torch::Tensor& gating_output, // [num_tokens, num_experts]
|
||||
torch::stable::Tensor& topk_weights, // [num_tokens, topk]
|
||||
torch::stable::Tensor& topk_indices, // [num_tokens, topk]
|
||||
torch::stable::Tensor& token_expert_indices, // [num_tokens, topk]
|
||||
torch::stable::Tensor& gating_output, // [num_tokens, num_experts]
|
||||
bool renormalize,
|
||||
std::optional<torch::Tensor> bias)
|
||||
std::optional<torch::stable::Tensor> bias)
|
||||
{
|
||||
const int num_experts = gating_output.size(-1);
|
||||
const auto num_tokens = gating_output.numel() / num_experts;
|
||||
@@ -842,24 +850,25 @@ void topk_sigmoid(
|
||||
const bool needs_workspace = !is_pow_2 || num_experts > 256;
|
||||
const int64_t workspace_size = needs_workspace ? num_tokens * num_experts : 0;
|
||||
|
||||
const at::cuda::OptionalCUDAGuard device_guard(device_of(gating_output));
|
||||
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
|
||||
const auto workspace_options = gating_output.options().dtype(at::ScalarType::Float);
|
||||
torch::Tensor workspace = torch::empty({workspace_size}, workspace_options);
|
||||
torch::stable::accelerator::DeviceGuard guard(gating_output.get_device_index());
|
||||
const cudaStream_t stream =
|
||||
get_current_cuda_stream(gating_output.get_device_index());
|
||||
auto workspace = torch::stable::new_empty(
|
||||
gating_output, {workspace_size}, torch::headeronly::ScalarType::Float);
|
||||
|
||||
if (gating_output.scalar_type() == at::ScalarType::Float) {
|
||||
if (gating_output.scalar_type() == torch::headeronly::ScalarType::Float) {
|
||||
dispatch_topk_launch<float, vllm::moe::SCORING_SIGMOID>(gating_output, topk_weights, topk_indices,
|
||||
token_expert_indices, workspace, num_tokens, num_experts, topk, renormalize,
|
||||
bias, stream);
|
||||
} else if (gating_output.scalar_type() == at::ScalarType::Half) {
|
||||
} else if (gating_output.scalar_type() == torch::headeronly::ScalarType::Half) {
|
||||
dispatch_topk_launch<__half, vllm::moe::SCORING_SIGMOID>(gating_output, topk_weights, topk_indices,
|
||||
token_expert_indices, workspace, num_tokens, num_experts, topk, renormalize,
|
||||
bias, stream);
|
||||
} else if (gating_output.scalar_type() == at::ScalarType::BFloat16) {
|
||||
} else if (gating_output.scalar_type() == torch::headeronly::ScalarType::BFloat16) {
|
||||
dispatch_topk_launch<__nv_bfloat16, vllm::moe::SCORING_SIGMOID>(gating_output, topk_weights, topk_indices,
|
||||
token_expert_indices, workspace, num_tokens, num_experts, topk, renormalize,
|
||||
bias, stream);
|
||||
} else {
|
||||
TORCH_CHECK(false, "Unsupported gating_output data type: ", gating_output.scalar_type());
|
||||
STD_TORCH_CHECK(false, "Unsupported gating_output data type: ", gating_output.scalar_type());
|
||||
}
|
||||
}
|
||||
+68
-54
@@ -18,11 +18,16 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
#include <type_traits>
|
||||
#include <torch/all.h>
|
||||
#include <ATen/cuda/CUDAContext.h>
|
||||
#include <c10/cuda/CUDAGuard.h>
|
||||
#include "../cuda_compat.h"
|
||||
#include "../cub_helpers.h"
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
#include <torch/csrc/stable/accelerator.h>
|
||||
#include <torch/csrc/stable/tensor.h>
|
||||
#include <torch/headeronly/core/ScalarType.h>
|
||||
#include <torch/headeronly/util/Exception.h>
|
||||
|
||||
#include "../../cuda_compat.h"
|
||||
#include "../../cub_helpers.h"
|
||||
#include "libtorch_stable/torch_utils.h"
|
||||
#ifndef USE_ROCM
|
||||
#include <cuda_bf16.h>
|
||||
#include <cuda_fp16.h>
|
||||
@@ -618,7 +623,7 @@ void topkGatingSoftplusSqrtKernelLauncher(
|
||||
LAUNCH_SOFTPLUS_SQRT(576, WARPS_PER_TB, BYTES_PER_LDG_MULTIPLE_64_NARROW);
|
||||
break;
|
||||
default: {
|
||||
TORCH_CHECK(false, "Unsupported expert number: ", num_experts);
|
||||
STD_TORCH_CHECK(false, "Unsupported expert number: ", num_experts);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -628,100 +633,109 @@ void topkGatingSoftplusSqrtKernelLauncher(
|
||||
|
||||
template <typename ComputeType>
|
||||
void dispatch_topk_softplus_sqrt_launch(
|
||||
const ComputeType* gating_output, torch::Tensor& topk_weights,
|
||||
torch::Tensor& topk_indices, torch::Tensor& token_expert_indices,
|
||||
int num_tokens, int num_experts, int topk, bool renormalize,
|
||||
double routed_scaling_factor,
|
||||
const c10::optional<torch::Tensor>& correction_bias,
|
||||
const c10::optional<torch::Tensor>& input_ids,
|
||||
const c10::optional<torch::Tensor>& tid2eid, cudaStream_t stream) {
|
||||
const ComputeType* gating_output, torch::stable::Tensor& topk_weights,
|
||||
torch::stable::Tensor& topk_indices,
|
||||
torch::stable::Tensor& token_expert_indices, int num_tokens,
|
||||
int num_experts, int topk, bool renormalize, double routed_scaling_factor,
|
||||
const std::optional<torch::stable::Tensor>& correction_bias,
|
||||
const std::optional<torch::stable::Tensor>& input_ids,
|
||||
const std::optional<torch::stable::Tensor>& tid2eid, cudaStream_t stream) {
|
||||
const float* bias_ptr = nullptr;
|
||||
if (correction_bias.has_value()) {
|
||||
bias_ptr = correction_bias.value().data_ptr<float>();
|
||||
bias_ptr = correction_bias.value().const_data_ptr<float>();
|
||||
}
|
||||
bool use_hash = false;
|
||||
if (tid2eid.has_value()) {
|
||||
TORCH_CHECK(input_ids.has_value(), "input_ids is required for hash MoE");
|
||||
STD_TORCH_CHECK(input_ids.has_value(),
|
||||
"input_ids is required for hash MoE");
|
||||
use_hash = true;
|
||||
}
|
||||
if (topk_indices.scalar_type() == at::ScalarType::Int) {
|
||||
if (topk_indices.scalar_type() == torch::headeronly::ScalarType::Int) {
|
||||
const int* input_ids_ptr = nullptr;
|
||||
const int* tid2eid_ptr = nullptr;
|
||||
if (tid2eid.has_value()) {
|
||||
input_ids_ptr = input_ids.value().data_ptr<int>();
|
||||
tid2eid_ptr = tid2eid.value().data_ptr<int>();
|
||||
input_ids_ptr = input_ids.value().const_data_ptr<int>();
|
||||
tid2eid_ptr = tid2eid.value().const_data_ptr<int>();
|
||||
}
|
||||
|
||||
vllm::moe::topkGatingSoftplusSqrtKernelLauncher<int, ComputeType>(
|
||||
gating_output, topk_weights.data_ptr<float>(),
|
||||
topk_indices.data_ptr<int>(), token_expert_indices.data_ptr<int>(),
|
||||
num_tokens, num_experts, topk, renormalize, routed_scaling_factor,
|
||||
bias_ptr, use_hash, input_ids_ptr, tid2eid_ptr, stream);
|
||||
} else if (topk_indices.scalar_type() == at::ScalarType::UInt32) {
|
||||
gating_output, topk_weights.mutable_data_ptr<float>(),
|
||||
topk_indices.mutable_data_ptr<int>(),
|
||||
token_expert_indices.mutable_data_ptr<int>(), num_tokens, num_experts,
|
||||
topk, renormalize, routed_scaling_factor, bias_ptr, use_hash,
|
||||
input_ids_ptr, tid2eid_ptr, stream);
|
||||
} else if (topk_indices.scalar_type() ==
|
||||
torch::headeronly::ScalarType::UInt32) {
|
||||
const uint32_t* input_ids_ptr = nullptr;
|
||||
const uint32_t* tid2eid_ptr = nullptr;
|
||||
if (tid2eid.has_value()) {
|
||||
input_ids_ptr = input_ids.value().data_ptr<uint32_t>();
|
||||
tid2eid_ptr = tid2eid.value().data_ptr<uint32_t>();
|
||||
input_ids_ptr = input_ids.value().const_data_ptr<uint32_t>();
|
||||
tid2eid_ptr = tid2eid.value().const_data_ptr<uint32_t>();
|
||||
}
|
||||
vllm::moe::topkGatingSoftplusSqrtKernelLauncher<uint32_t, ComputeType>(
|
||||
gating_output, topk_weights.data_ptr<float>(),
|
||||
topk_indices.data_ptr<uint32_t>(), token_expert_indices.data_ptr<int>(),
|
||||
num_tokens, num_experts, topk, renormalize, routed_scaling_factor,
|
||||
bias_ptr, use_hash, input_ids_ptr, tid2eid_ptr, stream);
|
||||
gating_output, topk_weights.mutable_data_ptr<float>(),
|
||||
topk_indices.mutable_data_ptr<uint32_t>(),
|
||||
token_expert_indices.mutable_data_ptr<int>(), num_tokens, num_experts,
|
||||
topk, renormalize, routed_scaling_factor, bias_ptr, use_hash,
|
||||
input_ids_ptr, tid2eid_ptr, stream);
|
||||
} else {
|
||||
TORCH_CHECK(topk_indices.scalar_type() == at::ScalarType::Long);
|
||||
STD_TORCH_CHECK(topk_indices.scalar_type() ==
|
||||
torch::headeronly::ScalarType::Long);
|
||||
|
||||
const int64_t* input_ids_ptr = nullptr;
|
||||
const int64_t* tid2eid_ptr = nullptr;
|
||||
if (tid2eid.has_value()) {
|
||||
input_ids_ptr = input_ids.value().data_ptr<int64_t>();
|
||||
tid2eid_ptr = tid2eid.value().data_ptr<int64_t>();
|
||||
input_ids_ptr = input_ids.value().const_data_ptr<int64_t>();
|
||||
tid2eid_ptr = tid2eid.value().const_data_ptr<int64_t>();
|
||||
}
|
||||
|
||||
vllm::moe::topkGatingSoftplusSqrtKernelLauncher<int64_t, ComputeType>(
|
||||
gating_output, topk_weights.data_ptr<float>(),
|
||||
topk_indices.data_ptr<int64_t>(), token_expert_indices.data_ptr<int>(),
|
||||
num_tokens, num_experts, topk, renormalize, routed_scaling_factor,
|
||||
bias_ptr, use_hash, input_ids_ptr, tid2eid_ptr, stream);
|
||||
gating_output, topk_weights.mutable_data_ptr<float>(),
|
||||
topk_indices.mutable_data_ptr<int64_t>(),
|
||||
token_expert_indices.mutable_data_ptr<int>(), num_tokens, num_experts,
|
||||
topk, renormalize, routed_scaling_factor, bias_ptr, use_hash,
|
||||
input_ids_ptr, tid2eid_ptr, stream);
|
||||
}
|
||||
}
|
||||
|
||||
void topk_softplus_sqrt(
|
||||
torch::Tensor& topk_weights, // [num_tokens, topk]
|
||||
torch::Tensor& topk_indices, // [num_tokens, topk]
|
||||
torch::Tensor& token_expert_indices, // [num_tokens, topk]
|
||||
torch::Tensor& gating_output, // [num_tokens, num_experts]
|
||||
torch::stable::Tensor& topk_weights, // [num_tokens, topk]
|
||||
torch::stable::Tensor& topk_indices, // [num_tokens, topk]
|
||||
torch::stable::Tensor& token_expert_indices, // [num_tokens, topk]
|
||||
torch::stable::Tensor& gating_output, // [num_tokens, num_experts]
|
||||
bool renormalize, double routed_scaling_factor,
|
||||
const c10::optional<torch::Tensor>& correction_bias,
|
||||
const c10::optional<torch::Tensor>& input_ids,
|
||||
const c10::optional<torch::Tensor>& tid2eid) {
|
||||
const std::optional<torch::stable::Tensor>& correction_bias,
|
||||
const std::optional<torch::stable::Tensor>& input_ids,
|
||||
const std::optional<torch::stable::Tensor>& tid2eid) {
|
||||
const int num_experts = gating_output.size(-1);
|
||||
const auto num_tokens = gating_output.numel() / num_experts;
|
||||
const int topk = topk_weights.size(-1);
|
||||
const at::cuda::OptionalCUDAGuard device_guard(device_of(gating_output));
|
||||
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
|
||||
const torch::stable::accelerator::DeviceGuard guard(
|
||||
gating_output.get_device_index());
|
||||
const cudaStream_t stream =
|
||||
get_current_cuda_stream(gating_output.get_device_index());
|
||||
|
||||
if (gating_output.scalar_type() == at::ScalarType::Float) {
|
||||
if (gating_output.scalar_type() == torch::headeronly::ScalarType::Float) {
|
||||
dispatch_topk_softplus_sqrt_launch<float>(
|
||||
gating_output.data_ptr<float>(), topk_weights, topk_indices,
|
||||
gating_output.const_data_ptr<float>(), topk_weights, topk_indices,
|
||||
token_expert_indices, num_tokens, num_experts, topk, renormalize,
|
||||
routed_scaling_factor, correction_bias, input_ids, tid2eid, stream);
|
||||
} else if (gating_output.scalar_type() == at::ScalarType::Half) {
|
||||
} else if (gating_output.scalar_type() ==
|
||||
torch::headeronly::ScalarType::Half) {
|
||||
dispatch_topk_softplus_sqrt_launch<__half>(
|
||||
reinterpret_cast<const __half*>(gating_output.data_ptr<at::Half>()),
|
||||
reinterpret_cast<const __half*>(gating_output.const_data_ptr()),
|
||||
topk_weights, topk_indices, token_expert_indices, num_tokens,
|
||||
num_experts, topk, renormalize, routed_scaling_factor, correction_bias,
|
||||
input_ids, tid2eid, stream);
|
||||
} else if (gating_output.scalar_type() == at::ScalarType::BFloat16) {
|
||||
} else if (gating_output.scalar_type() ==
|
||||
torch::headeronly::ScalarType::BFloat16) {
|
||||
dispatch_topk_softplus_sqrt_launch<__nv_bfloat16>(
|
||||
reinterpret_cast<const __nv_bfloat16*>(
|
||||
gating_output.data_ptr<at::BFloat16>()),
|
||||
reinterpret_cast<const __nv_bfloat16*>(gating_output.const_data_ptr()),
|
||||
topk_weights, topk_indices, token_expert_indices, num_tokens,
|
||||
num_experts, topk, renormalize, routed_scaling_factor, correction_bias,
|
||||
input_ids, tid2eid, stream);
|
||||
} else {
|
||||
TORCH_CHECK(false, "Unsupported gating_output data type: ",
|
||||
gating_output.scalar_type());
|
||||
STD_TORCH_CHECK(false, "Unsupported gating_output data type: ",
|
||||
gating_output.scalar_type());
|
||||
}
|
||||
}
|
||||
@@ -1,32 +1,30 @@
|
||||
#include "core/registration.h"
|
||||
#include "moe_ops.h"
|
||||
|
||||
TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
|
||||
#include <torch/csrc/stable/library.h>
|
||||
|
||||
STABLE_TORCH_LIBRARY_FRAGMENT(_moe_C, m) {
|
||||
// Apply topk softmax to the gating outputs.
|
||||
m.def(
|
||||
"topk_softmax(Tensor! topk_weights, Tensor! topk_indices, Tensor! "
|
||||
"token_expert_indices, Tensor gating_output, bool renormalize, Tensor? "
|
||||
"bias) -> ()");
|
||||
m.impl("topk_softmax", torch::kCUDA, &topk_softmax);
|
||||
|
||||
// Apply topk sigmoid to the gating outputs.
|
||||
m.def(
|
||||
"topk_sigmoid(Tensor! topk_weights, Tensor! topk_indices, Tensor! "
|
||||
"token_expert_indices, Tensor gating_output, bool renormalize, Tensor? "
|
||||
"bias) -> ()");
|
||||
m.impl("topk_sigmoid", torch::kCUDA, &topk_sigmoid);
|
||||
|
||||
m.def(
|
||||
"topk_softplus_sqrt(Tensor! topk_weights, Tensor! topk_indices, Tensor! "
|
||||
"token_expert_indices, Tensor gating_output, bool renormalize, float "
|
||||
"routed_scaling_factor, Tensor? "
|
||||
"bias, Tensor? input_ids, Tensor? tid2eid) -> ()");
|
||||
m.impl("topk_softplus_sqrt", torch::kCUDA, &topk_softplus_sqrt);
|
||||
|
||||
// Calculate the result of moe by summing up the partial results
|
||||
// from all selected experts.
|
||||
m.def("moe_sum(Tensor input, Tensor! output) -> ()");
|
||||
m.impl("moe_sum", torch::kCUDA, &moe_sum);
|
||||
|
||||
// Aligning the number of tokens to be processed by each expert such
|
||||
// that it is divisible by the block size.
|
||||
@@ -36,7 +34,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
|
||||
" Tensor! experts_ids,"
|
||||
" Tensor! num_tokens_post_pad,"
|
||||
" Tensor? maybe_expert_map) -> ()");
|
||||
m.impl("moe_align_block_size", torch::kCUDA, &moe_align_block_size);
|
||||
|
||||
// Aligning the number of tokens to be processed by each expert such
|
||||
// that it is divisible by the block size, but for the batched case.
|
||||
@@ -46,8 +43,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
|
||||
" Tensor! sorted_token_ids,"
|
||||
" Tensor! experts_ids,"
|
||||
" Tensor! num_tokens_post_pad) -> ()");
|
||||
m.impl("batched_moe_align_block_size", torch::kCUDA,
|
||||
&batched_moe_align_block_size);
|
||||
|
||||
// Aligning the number of tokens to be processed by each expert such
|
||||
// that it is divisible by the block size.
|
||||
@@ -64,8 +59,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
|
||||
" Tensor !adapter_enabled,"
|
||||
" Tensor !lora_ids,"
|
||||
" Tensor? maybe_expert_map) -> () ");
|
||||
m.impl("moe_lora_align_block_size", torch::kCUDA, &moe_lora_align_block_size);
|
||||
|
||||
#ifndef USE_ROCM
|
||||
m.def(
|
||||
"moe_wna16_gemm(Tensor input, Tensor! output, Tensor b_qweight, "
|
||||
@@ -75,8 +68,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
|
||||
"int top_k, int BLOCK_SIZE_M, int BLOCK_SIZE_N, int BLOCK_SIZE_K, "
|
||||
"int bit) -> Tensor");
|
||||
|
||||
m.impl("moe_wna16_gemm", torch::kCUDA, &moe_wna16_gemm);
|
||||
|
||||
m.def(
|
||||
"moe_wna16_marlin_gemm(Tensor! a, Tensor? c_or_none,"
|
||||
"Tensor! b_q_weight, Tensor? b_bias_or_none,"
|
||||
@@ -118,14 +109,11 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
|
||||
m.def(
|
||||
"moe_permute_sort_workspace_size(int num_expanded_rows, int n_expert) -> "
|
||||
"int");
|
||||
m.impl("moe_permute_unpermute_supported", &moe_permute_unpermute_supported);
|
||||
m.impl("moe_permute_sort_workspace_size", &moe_permute_sort_workspace_size);
|
||||
|
||||
// Row shuffle for MoE
|
||||
m.def(
|
||||
"shuffle_rows(Tensor input_tensor, Tensor dst2src_map, Tensor! "
|
||||
"output_tensor) -> ()");
|
||||
m.impl("shuffle_rows", torch::kCUDA, &shuffle_rows);
|
||||
|
||||
// Apply grouped topk routing to select experts.
|
||||
m.def(
|
||||
@@ -133,7 +121,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
|
||||
"topk_group, int topk, bool renormalize, float "
|
||||
"routed_scaling_factor, Tensor bias, int scoring_func) -> (Tensor, "
|
||||
"Tensor)");
|
||||
m.impl("grouped_topk", torch::kCUDA, &grouped_topk);
|
||||
|
||||
// DeepSeek V3 optimized router GEMM for SM90+
|
||||
m.def("dsv3_router_gemm(Tensor! output, Tensor mat_a, Tensor mat_b) -> ()");
|
||||
@@ -141,4 +128,30 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
|
||||
#endif
|
||||
}
|
||||
|
||||
REGISTER_EXTENSION(TORCH_EXTENSION_NAME)
|
||||
STABLE_TORCH_LIBRARY_IMPL(_moe_C, CUDA, m) {
|
||||
m.impl("topk_softmax", TORCH_BOX(&topk_softmax));
|
||||
m.impl("topk_sigmoid", TORCH_BOX(&topk_sigmoid));
|
||||
m.impl("topk_softplus_sqrt", TORCH_BOX(&topk_softplus_sqrt));
|
||||
m.impl("moe_sum", TORCH_BOX(&moe_sum));
|
||||
m.impl("moe_align_block_size", TORCH_BOX(&moe_align_block_size));
|
||||
m.impl("batched_moe_align_block_size",
|
||||
TORCH_BOX(&batched_moe_align_block_size));
|
||||
m.impl("moe_lora_align_block_size", TORCH_BOX(&moe_lora_align_block_size));
|
||||
#ifndef USE_ROCM
|
||||
m.impl("moe_wna16_gemm", TORCH_BOX(&moe_wna16_gemm));
|
||||
m.impl("shuffle_rows", TORCH_BOX(&shuffle_rows));
|
||||
m.impl("grouped_topk", TORCH_BOX(&grouped_topk));
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifndef USE_ROCM
|
||||
// Primitive-only ops have no tensor to dispatch on.
|
||||
STABLE_TORCH_LIBRARY_IMPL(_moe_C, CompositeExplicitAutograd, m) {
|
||||
m.impl("moe_permute_unpermute_supported",
|
||||
TORCH_BOX(&moe_permute_unpermute_supported));
|
||||
m.impl("moe_permute_sort_workspace_size",
|
||||
TORCH_BOX(&moe_permute_sort_workspace_size));
|
||||
}
|
||||
#endif
|
||||
|
||||
REGISTER_EXTENSION(_moe_C_stable_libtorch)
|
||||
@@ -246,6 +246,22 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"awq_dequantize(Tensor _kernel, Tensor _scaling_factors, "
|
||||
"Tensor _zeros, SymInt split_k_iters, int thx, int thy) -> Tensor");
|
||||
|
||||
// Expert-specialization mxfp8 blockscaled grouped quantization (SM100+).
|
||||
ops.def(
|
||||
"mxfp8_experts_quant("
|
||||
" Tensor input, Tensor problem_sizes, Tensor expert_offsets,"
|
||||
" Tensor blockscale_offsets, Tensor! quant_output, Tensor! scale_factor)"
|
||||
" -> ()");
|
||||
// conditionally compiled so impl registration is in source file
|
||||
|
||||
// Expert-specialization mxfp8 blockscaled grouped GEMM (SM100+).
|
||||
ops.def(
|
||||
"cutlass_mxfp8_grouped_mm("
|
||||
" Tensor a, Tensor b, Tensor sfa, Tensor sfb, Tensor! out,"
|
||||
" Tensor problem_sizes, Tensor expert_offsets, Tensor blockscale_offsets)"
|
||||
" -> ()");
|
||||
// conditionally compiled so impl registration is in source file
|
||||
|
||||
// DeepSeek V3 fused A GEMM (SM 9.0+, bf16 only, 1-16 tokens).
|
||||
// conditionally compiled so impl registration is in source file
|
||||
ops.def(
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
/*
|
||||
* Adapted from SGLang's sgl-kernel implementation, which was adapted from
|
||||
* https://github.com/NVIDIA/TensorRT-LLM/blob/main/cpp/tensorrt_llm/kernels/dsv3MinLatencyKernels/dsv3RouterGemm.cu
|
||||
* https://github.com/NVIDIA/TensorRT-LLM/blob/main/cpp/tensorrt_llm/thop/dsv3RouterGemmOp.cpp
|
||||
*
|
||||
* Copyright (c) 2019-2023, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <ATen/cuda/CUDAContext.h>
|
||||
|
||||
#include <cstdlib>
|
||||
#include <mutex>
|
||||
|
||||
inline int getSMVersion() {
|
||||
auto* props = at::cuda::getCurrentDeviceProperties();
|
||||
return props->major * 10 + props->minor;
|
||||
}
|
||||
@@ -1,81 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <torch/all.h>
|
||||
|
||||
void topk_softmax(torch::Tensor& topk_weights, torch::Tensor& topk_indices,
|
||||
torch::Tensor& token_expert_indices,
|
||||
torch::Tensor& gating_output, bool renormalize,
|
||||
std::optional<torch::Tensor> bias);
|
||||
|
||||
void topk_sigmoid(torch::Tensor& topk_weights, torch::Tensor& topk_indices,
|
||||
torch::Tensor& token_expert_indices,
|
||||
torch::Tensor& gating_output, bool renormalize,
|
||||
std::optional<torch::Tensor> bias);
|
||||
|
||||
void topk_softplus_sqrt(torch::Tensor& topk_weights,
|
||||
torch::Tensor& topk_indices,
|
||||
torch::Tensor& token_expert_indices,
|
||||
torch::Tensor& gating_output, bool renormalize,
|
||||
double routed_scaling_factor,
|
||||
const c10::optional<torch::Tensor>& correction_bias,
|
||||
const c10::optional<torch::Tensor>& input_ids,
|
||||
const c10::optional<torch::Tensor>& tid2eid);
|
||||
|
||||
void moe_sum(torch::Tensor& input, torch::Tensor& output);
|
||||
|
||||
void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
|
||||
int64_t block_size, torch::Tensor sorted_token_ids,
|
||||
torch::Tensor experts_ids,
|
||||
torch::Tensor num_tokens_post_pad,
|
||||
std::optional<torch::Tensor> maybe_expert_map);
|
||||
|
||||
void batched_moe_align_block_size(int64_t max_tokens_per_batch,
|
||||
int64_t block_size,
|
||||
torch::Tensor const& expert_num_tokens,
|
||||
torch::Tensor sorted_ids,
|
||||
torch::Tensor expert_ids,
|
||||
torch::Tensor num_tokens_post_pad);
|
||||
|
||||
void moe_lora_align_block_size(
|
||||
torch::Tensor topk_ids, torch::Tensor token_lora_mapping,
|
||||
int64_t num_experts, int64_t block_size, int64_t max_loras,
|
||||
int64_t max_num_tokens_padded, int64_t max_num_m_blocks,
|
||||
torch::Tensor sorted_token_ids, torch::Tensor expert_ids,
|
||||
torch::Tensor num_tokens_post_pad, torch::Tensor adapter_enabled,
|
||||
torch::Tensor lora_ids, std::optional<torch::Tensor> maybe_expert_map);
|
||||
#ifndef USE_ROCM
|
||||
torch::Tensor moe_wna16_gemm(torch::Tensor input, torch::Tensor output,
|
||||
torch::Tensor b_qweight, torch::Tensor b_scales,
|
||||
std::optional<torch::Tensor> b_qzeros,
|
||||
std::optional<torch::Tensor> topk_weights,
|
||||
torch::Tensor sorted_token_ids,
|
||||
torch::Tensor expert_ids,
|
||||
torch::Tensor num_tokens_post_pad, int64_t top_k,
|
||||
int64_t BLOCK_SIZE_M, int64_t BLOCK_SIZE_N,
|
||||
int64_t BLOCK_SIZE_K, int64_t bit);
|
||||
|
||||
std::tuple<torch::Tensor, torch::Tensor> grouped_topk(
|
||||
torch::Tensor const& scores, int64_t n_group, int64_t topk_group,
|
||||
int64_t topk, bool renormalize, double routed_scaling_factor,
|
||||
torch::Tensor const& bias, int64_t scoring_func);
|
||||
#endif
|
||||
|
||||
bool moe_permute_unpermute_supported();
|
||||
|
||||
int64_t moe_permute_sort_workspace_size(int64_t num_expanded_rows,
|
||||
int64_t num_experts);
|
||||
|
||||
void shuffle_rows(const torch::Tensor& input_tensor,
|
||||
const torch::Tensor& dst2src_map,
|
||||
torch::Tensor& output_tensor);
|
||||
|
||||
#ifndef USE_ROCM
|
||||
// DeepSeek V3 optimized router GEMM kernel for SM90+
|
||||
// Computes output = mat_a @ mat_b.T where:
|
||||
// mat_a: [num_tokens, hidden_dim] in bf16
|
||||
// mat_b: [num_experts, hidden_dim] in bf16
|
||||
// output: [num_tokens, num_experts] in bf16 or fp32
|
||||
// Supports num_tokens in [1, 16], num_experts in {256, 384}, hidden_dim = 7168
|
||||
void dsv3_router_gemm(torch::Tensor& output, const torch::Tensor& mat_a,
|
||||
const torch::Tensor& mat_b);
|
||||
#endif
|
||||
@@ -1,286 +0,0 @@
|
||||
#include <c10/core/ScalarType.h>
|
||||
#include <torch/all.h>
|
||||
#include <ATen/cuda/CUDAContext.h>
|
||||
#include "permute_unpermute_kernels/moe_permute_unpermute_kernel.h"
|
||||
#include "permute_unpermute_kernels/dispatch.h"
|
||||
#include "core/registration.h"
|
||||
|
||||
// moe_permute kernels require at least CUDA 12.0
|
||||
#if defined(CUDA_VERSION) && (CUDA_VERSION >= 12000)
|
||||
|
||||
namespace {
|
||||
|
||||
torch::Tensor maybe_allocate_tensor(
|
||||
const std::optional<torch::Tensor>& maybe_tensor,
|
||||
at::IntArrayRef expected_sizes, torch::ScalarType dtype, c10::Device device,
|
||||
char const* name) {
|
||||
auto expected_numel = c10::multiply_integers(expected_sizes);
|
||||
if (maybe_tensor.has_value()) {
|
||||
auto tensor = maybe_tensor.value();
|
||||
TORCH_CHECK(tensor.device() == device, name, " must be on the same device");
|
||||
TORCH_CHECK(tensor.scalar_type() == dtype, name, " has incorrect dtype");
|
||||
TORCH_CHECK(tensor.is_contiguous(), name, " must be contiguous");
|
||||
TORCH_CHECK(tensor.numel() >= expected_numel, name,
|
||||
" is too small for the requested shape");
|
||||
auto flat_tensor = tensor.view({tensor.numel()});
|
||||
return flat_tensor.narrow(0, 0, expected_numel).view(expected_sizes);
|
||||
}
|
||||
return torch::empty(expected_sizes, torch::dtype(dtype).device(device));
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int64_t moe_permute_sort_workspace_size(int64_t num_expanded_rows,
|
||||
int64_t n_expert) {
|
||||
return static_cast<int64_t>(
|
||||
CubKeyValueSorter::getWorkspaceSize(num_expanded_rows, n_expert));
|
||||
}
|
||||
|
||||
void moe_permute_impl(
|
||||
const torch::Tensor& input, // [n_token, hidden]
|
||||
const torch::Tensor& topk_ids, // [n_token, topk]
|
||||
const torch::Tensor& token_expert_indices, // [n_token, topk]
|
||||
const std::optional<torch::Tensor>& expert_map, // [n_expert]
|
||||
int64_t n_expert, int64_t n_local_expert, int64_t topk,
|
||||
torch::Tensor& permuted_input, // [permuted_size, hidden]
|
||||
torch::Tensor& expert_first_token_offset, // [n_local_expert + 1]
|
||||
torch::Tensor& inv_permuted_idx, // [n_token, topk]
|
||||
torch::Tensor& permuted_idx, // [permute_size]
|
||||
const std::optional<torch::Tensor>& maybe_sort_workspace,
|
||||
const std::optional<torch::Tensor>& maybe_permuted_experts_id,
|
||||
const std::optional<torch::Tensor>& maybe_sorted_row_idx,
|
||||
const std::optional<torch::Tensor>& maybe_topk_ids_for_sort) {
|
||||
TORCH_CHECK(expert_first_token_offset.scalar_type() == at::ScalarType::Long,
|
||||
"expert_first_token_offset must be int64");
|
||||
TORCH_CHECK(topk_ids.scalar_type() == at::ScalarType::Int,
|
||||
"topk_ids must be int32");
|
||||
TORCH_CHECK(token_expert_indices.scalar_type() == at::ScalarType::Int,
|
||||
"token_expert_indices must be int32");
|
||||
TORCH_CHECK(inv_permuted_idx.scalar_type() == at::ScalarType::Int,
|
||||
"inv_permuted_idx must be int32");
|
||||
TORCH_CHECK(expert_first_token_offset.size(0) == n_local_expert + 1,
|
||||
"expert_first_token_offset shape != n_local_expert+1");
|
||||
TORCH_CHECK(inv_permuted_idx.sizes() == token_expert_indices.sizes(),
|
||||
"token_expert_indices shape must be same as inv_permuted_idx");
|
||||
auto device = input.device();
|
||||
auto n_token = input.sizes()[0];
|
||||
auto n_hidden = input.sizes()[1];
|
||||
auto expanded_rows = n_token * topk;
|
||||
auto stream = at::cuda::getCurrentCUDAStream().stream();
|
||||
|
||||
auto sorter_size = moe_permute_sort_workspace_size(expanded_rows, n_expert);
|
||||
auto sort_workspace =
|
||||
maybe_allocate_tensor(maybe_sort_workspace, {sorter_size}, torch::kInt8,
|
||||
device, "sort_workspace");
|
||||
auto permuted_experts_id =
|
||||
maybe_allocate_tensor(maybe_permuted_experts_id, topk_ids.sizes(),
|
||||
at::ScalarType::Int, device, "permuted_experts_id");
|
||||
auto sorted_row_idx =
|
||||
maybe_allocate_tensor(maybe_sorted_row_idx, inv_permuted_idx.sizes(),
|
||||
at::ScalarType::Int, device, "sorted_row_idx");
|
||||
|
||||
CubKeyValueSorter sorter{};
|
||||
int64_t* valid_num_ptr = nullptr;
|
||||
torch::Tensor topk_ids_for_sort = topk_ids;
|
||||
|
||||
if (expert_map.has_value()) {
|
||||
const int* expert_map_ptr = get_ptr<int>(expert_map.value());
|
||||
valid_num_ptr =
|
||||
get_ptr<int64_t>(expert_first_token_offset) + n_local_expert;
|
||||
topk_ids_for_sort =
|
||||
maybe_allocate_tensor(maybe_topk_ids_for_sort, topk_ids.sizes(),
|
||||
at::ScalarType::Int, device, "topk_ids_for_sort");
|
||||
topk_ids_for_sort.copy_(topk_ids);
|
||||
preprocessTopkIdLauncher(get_ptr<int>(topk_ids_for_sort), n_token * topk,
|
||||
expert_map_ptr, n_expert, stream);
|
||||
}
|
||||
|
||||
sortAndScanExpert(
|
||||
get_ptr<const int>(topk_ids_for_sort), get_ptr<int>(token_expert_indices),
|
||||
get_ptr<int>(permuted_experts_id), get_ptr<int>(sorted_row_idx),
|
||||
get_ptr<int64_t>(expert_first_token_offset), n_token, n_expert,
|
||||
n_local_expert, topk, sorter, get_ptr<int>(sort_workspace), stream);
|
||||
|
||||
MOE_DISPATCH(input.scalar_type(), [&] {
|
||||
expandInputRowsKernelLauncher<scalar_t>(
|
||||
get_ptr<scalar_t>(input), get_ptr<scalar_t>(permuted_input),
|
||||
get_ptr<int>(sorted_row_idx), get_ptr<int>(inv_permuted_idx),
|
||||
get_ptr<int>(permuted_idx), get_ptr<int64_t>(expert_first_token_offset),
|
||||
n_token, valid_num_ptr, n_hidden, topk, n_local_expert, stream);
|
||||
});
|
||||
}
|
||||
|
||||
void moe_permute(
|
||||
const torch::Tensor& input, // [n_token, hidden]
|
||||
const torch::Tensor& topk_ids, // [n_token, topk]
|
||||
const torch::Tensor& token_expert_indices, // [n_token, topk]
|
||||
const std::optional<torch::Tensor>& expert_map, // [n_expert]
|
||||
int64_t n_expert, int64_t n_local_expert, int64_t topk,
|
||||
torch::Tensor& permuted_input, // [permuted_size, hidden]
|
||||
torch::Tensor& expert_first_token_offset, // [n_local_expert + 1]
|
||||
torch::Tensor& inv_permuted_idx, // [n_token, topk]
|
||||
torch::Tensor& permuted_idx) { // [permute_size]
|
||||
moe_permute_impl(input, topk_ids, token_expert_indices, expert_map, n_expert,
|
||||
n_local_expert, topk, permuted_input,
|
||||
expert_first_token_offset, inv_permuted_idx, permuted_idx,
|
||||
std::nullopt, std::nullopt, std::nullopt, std::nullopt);
|
||||
}
|
||||
|
||||
void moe_permute_with_scratch(
|
||||
const torch::Tensor& input, const torch::Tensor& topk_ids,
|
||||
const torch::Tensor& token_expert_indices,
|
||||
const std::optional<torch::Tensor>& expert_map, int64_t n_expert,
|
||||
int64_t n_local_expert, int64_t topk, torch::Tensor& permuted_input,
|
||||
torch::Tensor& expert_first_token_offset, torch::Tensor& inv_permuted_idx,
|
||||
torch::Tensor& permuted_idx, torch::Tensor& sort_workspace,
|
||||
torch::Tensor& permuted_experts_id, torch::Tensor& sorted_row_idx,
|
||||
torch::Tensor& topk_ids_for_sort) {
|
||||
moe_permute_impl(input, topk_ids, token_expert_indices, expert_map, n_expert,
|
||||
n_local_expert, topk, permuted_input,
|
||||
expert_first_token_offset, inv_permuted_idx, permuted_idx,
|
||||
sort_workspace, permuted_experts_id, sorted_row_idx,
|
||||
topk_ids_for_sort);
|
||||
}
|
||||
|
||||
void moe_unpermute(
|
||||
const torch::Tensor& permuted_hidden_states, // [n_token * topk, hidden]
|
||||
const torch::Tensor& topk_weights, // [n_token, topk]
|
||||
const torch::Tensor& inv_permuted_idx, // [n_token, topk]
|
||||
const std::optional<torch::Tensor>&
|
||||
expert_first_token_offset, // [n_local_expert+1]
|
||||
int64_t topk,
|
||||
torch::Tensor& hidden_states // [n_token, hidden]
|
||||
) {
|
||||
TORCH_CHECK(
|
||||
permuted_hidden_states.scalar_type() == hidden_states.scalar_type(),
|
||||
"permuted_hidden_states dtype must be same as hidden_states");
|
||||
auto n_token = hidden_states.size(0);
|
||||
auto n_hidden = hidden_states.size(1);
|
||||
auto stream = at::cuda::getCurrentCUDAStream().stream();
|
||||
|
||||
int64_t const* valid_ptr = nullptr;
|
||||
if (expert_first_token_offset.has_value()) {
|
||||
int n_local_expert = expert_first_token_offset.value().size(0) - 1;
|
||||
valid_ptr =
|
||||
get_ptr<int64_t>(expert_first_token_offset.value()) + n_local_expert;
|
||||
}
|
||||
|
||||
MOE_DISPATCH(hidden_states.scalar_type(), [&] {
|
||||
finalizeMoeRoutingKernelLauncher<scalar_t, scalar_t>(
|
||||
get_ptr<scalar_t>(permuted_hidden_states),
|
||||
get_ptr<scalar_t>(hidden_states), get_ptr<float>(topk_weights),
|
||||
get_ptr<int>(inv_permuted_idx), n_token, n_hidden, topk, valid_ptr,
|
||||
stream);
|
||||
});
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__global__ void shuffleInputRowsKernel(const T* input,
|
||||
const int32_t* dst2src_map, T* output,
|
||||
int64_t num_src_rows,
|
||||
int64_t num_dst_rows, int64_t num_cols) {
|
||||
int64_t dest_row_idx = blockIdx.x;
|
||||
int64_t const source_row_idx = dst2src_map[dest_row_idx];
|
||||
|
||||
if (blockIdx.x < num_dst_rows) {
|
||||
// Load 128-bits per thread
|
||||
constexpr int64_t ELEM_PER_THREAD = 128 / sizeof(T) / 8;
|
||||
using DataElem = cutlass::Array<T, ELEM_PER_THREAD>;
|
||||
|
||||
// Duplicate and permute rows
|
||||
auto const* source_row_ptr =
|
||||
reinterpret_cast<DataElem const*>(input + source_row_idx * num_cols);
|
||||
auto* dest_row_ptr =
|
||||
reinterpret_cast<DataElem*>(output + dest_row_idx * num_cols);
|
||||
|
||||
int64_t const start_offset = threadIdx.x;
|
||||
int64_t const stride = blockDim.x;
|
||||
int64_t const num_elems_in_col = num_cols / ELEM_PER_THREAD;
|
||||
|
||||
for (int elem_index = start_offset; elem_index < num_elems_in_col;
|
||||
elem_index += stride) {
|
||||
dest_row_ptr[elem_index] = source_row_ptr[elem_index];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void shuffle_rows(const torch::Tensor& input_tensor,
|
||||
const torch::Tensor& dst2src_map,
|
||||
torch::Tensor& output_tensor) {
|
||||
TORCH_CHECK(input_tensor.scalar_type() == output_tensor.scalar_type(),
|
||||
"Input and output tensors must have the same data type");
|
||||
|
||||
auto stream = at::cuda::getCurrentCUDAStream().stream();
|
||||
int64_t const blocks = output_tensor.size(0);
|
||||
int64_t const threads = 256;
|
||||
int64_t const num_dest_rows = output_tensor.size(0);
|
||||
int64_t const num_src_rows = input_tensor.size(0);
|
||||
int64_t const num_cols = input_tensor.size(1);
|
||||
|
||||
TORCH_CHECK(!(num_cols % (128 / sizeof(input_tensor.scalar_type()) / 8)),
|
||||
"num_cols must be divisible by 128 / "
|
||||
"sizeof(input_tensor.scalar_type()) / 8");
|
||||
|
||||
MOE_DISPATCH(input_tensor.scalar_type(), [&] {
|
||||
shuffleInputRowsKernel<scalar_t><<<blocks, threads, 0, stream>>>(
|
||||
reinterpret_cast<scalar_t*>(input_tensor.data_ptr()),
|
||||
dst2src_map.data_ptr<int32_t>(),
|
||||
reinterpret_cast<scalar_t*>(output_tensor.data_ptr()), num_src_rows,
|
||||
num_dest_rows, num_cols);
|
||||
});
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
int64_t moe_permute_sort_workspace_size(int64_t num_expanded_rows,
|
||||
int64_t n_expert) {
|
||||
TORCH_CHECK(
|
||||
false, "moe_permute_sort_workspace_size is not supported on CUDA < 12.0");
|
||||
}
|
||||
|
||||
void moe_permute(const torch::Tensor& input, const torch::Tensor& topk_ids,
|
||||
const torch::Tensor& token_expert_indices,
|
||||
const std::optional<torch::Tensor>& expert_map,
|
||||
int64_t n_expert, int64_t n_local_expert, int64_t topk,
|
||||
torch::Tensor& permuted_input,
|
||||
torch::Tensor& expert_first_token_offset,
|
||||
torch::Tensor& inv_permuted_idx, torch::Tensor& permuted_idx) {
|
||||
TORCH_CHECK(false, "moe_permute is not supported on CUDA < 12.0");
|
||||
}
|
||||
|
||||
void moe_permute_with_scratch(
|
||||
const torch::Tensor& input, const torch::Tensor& topk_ids,
|
||||
const torch::Tensor& token_expert_indices,
|
||||
const std::optional<torch::Tensor>& expert_map, int64_t n_expert,
|
||||
int64_t n_local_expert, int64_t topk, torch::Tensor& permuted_input,
|
||||
torch::Tensor& expert_first_token_offset, torch::Tensor& inv_permuted_idx,
|
||||
torch::Tensor& permuted_idx, torch::Tensor& sort_workspace,
|
||||
torch::Tensor& permuted_experts_id, torch::Tensor& sorted_row_idx,
|
||||
torch::Tensor& topk_ids_for_sort) {
|
||||
TORCH_CHECK(false,
|
||||
"moe_permute_with_scratch is not supported on CUDA < 12.0");
|
||||
}
|
||||
|
||||
void moe_unpermute(
|
||||
const torch::Tensor& permuted_hidden_states,
|
||||
const torch::Tensor& topk_weights, const torch::Tensor& inv_permuted_idx,
|
||||
const std::optional<torch::Tensor>& expert_first_token_offset, int64_t topk,
|
||||
torch::Tensor& hidden_states) {
|
||||
TORCH_CHECK(false, "moe_unpermute is not supported on CUDA < 12.0");
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
bool moe_permute_unpermute_supported() {
|
||||
#if defined(CUDA_VERSION) && (CUDA_VERSION >= 12000)
|
||||
return true;
|
||||
#else
|
||||
return false;
|
||||
#endif
|
||||
}
|
||||
|
||||
TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) {
|
||||
m.impl("moe_permute", &moe_permute);
|
||||
m.impl("moe_permute_with_scratch", &moe_permute_with_scratch);
|
||||
m.impl("moe_unpermute", &moe_unpermute);
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
#pragma once
|
||||
#include <cuda_fp8.h>
|
||||
#define MOE_SWITCH(TYPE, ...) \
|
||||
at::ScalarType _st = ::detail::scalar_type(TYPE); \
|
||||
switch (_st) { \
|
||||
__VA_ARGS__ \
|
||||
default: \
|
||||
TORCH_CHECK(false, "[moe permute]data type dispatch fail!") \
|
||||
}
|
||||
|
||||
#define MOE_DISPATCH_CASE(enum_type, ...) \
|
||||
case enum_type: { \
|
||||
using scalar_t = ScalarType2CudaType<enum_type>::type; \
|
||||
__VA_ARGS__(); \
|
||||
break; \
|
||||
}
|
||||
#define MOE_DISPATCH_FLOAT_CASE(...) \
|
||||
MOE_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
|
||||
MOE_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \
|
||||
MOE_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) \
|
||||
MOE_DISPATCH_CASE(at::ScalarType::Float8_e5m2, __VA_ARGS__) \
|
||||
MOE_DISPATCH_CASE(at::ScalarType::Float8_e4m3fn, __VA_ARGS__) \
|
||||
MOE_DISPATCH_CASE(at::ScalarType::Byte, __VA_ARGS__)
|
||||
|
||||
#define MOE_DISPATCH(TYPE, ...) \
|
||||
MOE_SWITCH(TYPE, MOE_DISPATCH_FLOAT_CASE(__VA_ARGS__))
|
||||
|
||||
template <at::ScalarType type>
|
||||
struct ScalarType2CudaType;
|
||||
|
||||
template <>
|
||||
struct ScalarType2CudaType<at::ScalarType::Float> {
|
||||
using type = float;
|
||||
};
|
||||
template <>
|
||||
struct ScalarType2CudaType<at::ScalarType::Half> {
|
||||
using type = half;
|
||||
};
|
||||
template <>
|
||||
struct ScalarType2CudaType<at::ScalarType::BFloat16> {
|
||||
using type = __nv_bfloat16;
|
||||
};
|
||||
// uint8 for packed fp4
|
||||
template <>
|
||||
struct ScalarType2CudaType<at::ScalarType::Byte> {
|
||||
using type = uint8_t;
|
||||
};
|
||||
|
||||
// #if __CUDA_ARCH__ >= 890
|
||||
// fp8
|
||||
template <>
|
||||
struct ScalarType2CudaType<at::ScalarType::Float8_e5m2> {
|
||||
using type = __nv_fp8_e5m2;
|
||||
};
|
||||
template <>
|
||||
struct ScalarType2CudaType<at::ScalarType::Float8_e4m3fn> {
|
||||
using type = __nv_fp8_e4m3;
|
||||
};
|
||||
// #endif
|
||||
@@ -130,25 +130,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
|
||||
"Tensor? qzeros_or_none, bool inplace) -> Tensor");
|
||||
// conditionally compiled so impl registrations are in source file
|
||||
|
||||
#endif
|
||||
|
||||
#ifndef USE_ROCM
|
||||
// Expert-specialization mxfp8 blockscaled grouped quantization (SM100+).
|
||||
ops.def(
|
||||
"mxfp8_experts_quant("
|
||||
" Tensor input, Tensor problem_sizes, Tensor expert_offsets,"
|
||||
" Tensor blockscale_offsets, Tensor! quant_output, Tensor! scale_factor)"
|
||||
" -> ()");
|
||||
// conditionally compiled so impl registration is in source file
|
||||
|
||||
// Expert-specialization mxfp8 blockscaled grouped GEMM (SM100+).
|
||||
ops.def(
|
||||
"cutlass_mxfp8_grouped_mm("
|
||||
" Tensor a, Tensor b, Tensor sfa, Tensor sfb, Tensor! out,"
|
||||
" Tensor problem_sizes, Tensor expert_offsets, Tensor blockscale_offsets)"
|
||||
" -> ()");
|
||||
// conditionally compiled so impl registration is in source file
|
||||
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
@@ -755,7 +755,7 @@ class precompiled_wheel_utils:
|
||||
{
|
||||
"vllm/_C.abi3.so",
|
||||
"vllm/_C_stable_libtorch.abi3.so",
|
||||
"vllm/_moe_C.abi3.so",
|
||||
"vllm/_moe_C_stable_libtorch.abi3.so",
|
||||
"vllm/_flashmla_C.abi3.so",
|
||||
"vllm/_flashmla_extension_C.abi3.so",
|
||||
"vllm/_sparse_flashmla_C.abi3.so",
|
||||
@@ -1081,7 +1081,6 @@ def get_requirements() -> list[str]:
|
||||
ext_modules = []
|
||||
|
||||
if _is_cuda() or _is_hip():
|
||||
ext_modules.append(CMakeExtension(name="vllm._moe_C"))
|
||||
ext_modules.append(CMakeExtension(name="vllm.cumem_allocator"))
|
||||
# Optional since this doesn't get built (produce an .so file). This is just
|
||||
# copying the relevant .py files from the source repository.
|
||||
@@ -1135,6 +1134,7 @@ if _build_custom_ops():
|
||||
ext_modules.append(CMakeExtension(name="vllm._C"))
|
||||
if _is_cuda() or _is_hip():
|
||||
ext_modules.append(CMakeExtension(name="vllm._C_stable_libtorch"))
|
||||
ext_modules.append(CMakeExtension(name="vllm._moe_C_stable_libtorch"))
|
||||
|
||||
package_data = {
|
||||
"vllm": [
|
||||
|
||||
@@ -254,7 +254,7 @@ class Platform:
|
||||
except ImportError as e:
|
||||
logger.warning("Failed to import from vllm._C: %r", e)
|
||||
with contextlib.suppress(ImportError):
|
||||
import vllm._moe_C # noqa: F401
|
||||
import vllm._moe_C_stable_libtorch # noqa: F401
|
||||
|
||||
@classmethod
|
||||
def get_attn_backend_cls(
|
||||
|
||||
Reference in New Issue
Block a user