Merge branch 'main' into patches
This commit is contained in:
@@ -1059,33 +1059,6 @@ void situ_and_mul(torch::stable::Tensor& out, // [..., d]
|
||||
});
|
||||
}
|
||||
|
||||
void masked_situ_and_mul(torch::stable::Tensor& out, // [E, T, d]
|
||||
torch::stable::Tensor& input, // [E, T, 2 * d]
|
||||
const torch::stable::Tensor& expert_num_tokens,
|
||||
double beta, double linear_beta) {
|
||||
int num_experts = input.size(0);
|
||||
int max_num_tokens = input.size(1);
|
||||
int d = input.size(2) / 2;
|
||||
if (num_experts == 0 || max_num_tokens == 0 || d == 0) {
|
||||
return;
|
||||
}
|
||||
constexpr int block_size = 256;
|
||||
dim3 grid((d + block_size - 1) / block_size, num_experts);
|
||||
dim3 block(block_size);
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
input.get_device_index());
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
|
||||
input.scalar_type(), "masked_situ_and_mul_kernel", [&] {
|
||||
vllm::masked_situ_and_mul_kernel<scalar_t, true>
|
||||
<<<grid, block, 0, stream>>>(
|
||||
out.mutable_data_ptr<scalar_t>(),
|
||||
input.const_data_ptr<scalar_t>(),
|
||||
expert_num_tokens.const_data_ptr<int>(), max_num_tokens, d,
|
||||
(float)beta, (float)linear_beta);
|
||||
});
|
||||
}
|
||||
|
||||
#define LAUNCH_MASKED_ACT_AND_MUL(KERNEL, ACT_FIRST, HAS_CLAMP, LIMIT, ALPHA, \
|
||||
BETA) \
|
||||
vllm::masked_act_and_mul_kernel<scalar_t, KERNEL<scalar_t>, ACT_FIRST, \
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
//
|
||||
// Router GEMM: activation(T) x weight(fp32) -> fp32, M<=32, for the
|
||||
// Router GEMM: activation(T) x weight(fp32) -> fp32, M<=48, for the
|
||||
// supported (E, H) pairs listed at the bottom of this file.
|
||||
// Supports bf16 or fp32 activation; weight is always fp32.
|
||||
|
||||
@@ -214,8 +214,26 @@ void invokeFp32RouterGemm(float* output, InputT const* mat_a,
|
||||
// production-fidelity harness (CUDA-graph replay, per-layer cold weights).
|
||||
// Only enabled on the Blackwell family where it was validated; Hopper and
|
||||
// other shapes / fp32 activation keep the legacy geometry.
|
||||
if constexpr (std::is_same_v<InputT, __nv_bfloat16> && kNumExperts == 128 &&
|
||||
kHiddenDim == 6144) {
|
||||
if constexpr (std::is_same_v<InputT, __nv_bfloat16> && kNumExperts == 256 &&
|
||||
kHiddenDim == 5120) {
|
||||
if (!isBlackwellFamily()) {
|
||||
launchFp32RouterGemm<InputT, 128, 1, kNumTokens, kNumExperts, kHiddenDim>(
|
||||
output, mat_a, mat_b, stream);
|
||||
return;
|
||||
}
|
||||
if constexpr (kNumTokens > 16 && kNumTokens % 8 == 0) {
|
||||
constexpr int block_size = kNumTokens == 24 ? 160 : 128;
|
||||
launchFp32RouterGemm<InputT, block_size, 2, kNumTokens, kNumExperts,
|
||||
kHiddenDim, 4>(output, mat_a, mat_b, stream);
|
||||
} else {
|
||||
constexpr int block_size = kNumTokens == 1 ? 64 : 128;
|
||||
constexpr int token_groups = kNumTokens % 2 == 0 ? 2 : 1;
|
||||
launchFp32RouterGemm<InputT, block_size, 1, kNumTokens, kNumExperts,
|
||||
kHiddenDim, token_groups>(output, mat_a, mat_b,
|
||||
stream);
|
||||
}
|
||||
} else if constexpr (std::is_same_v<InputT, __nv_bfloat16> &&
|
||||
kNumExperts == 128 && kHiddenDim == 6144) {
|
||||
// MiniMax-M3. Legacy 128/1 only fills 128 blocks and pays the same
|
||||
// accumulator register cliffs; B300 sweep:
|
||||
// even M in [6, 10] : BS=384, EPB=1, 2 token groups (1.26-1.43x)
|
||||
@@ -261,6 +279,27 @@ void invokeFp32RouterGemm(float* output, InputT const* mat_a,
|
||||
launchFp32RouterGemm<InputT, 128, 1, kNumTokens, kNumExperts, kHiddenDim>(
|
||||
output, mat_a, mat_b, stream);
|
||||
}
|
||||
} else if constexpr (std::is_same_v<InputT, __nv_bfloat16> &&
|
||||
kNumExperts == 256 && kHiddenDim == 8192) {
|
||||
// MuseSpark1x: the 8MB weight makes this shape FFMA-bound past M~10, so
|
||||
// the wider block wins at every M and token groups clear the M >= 28
|
||||
// accumulator cliff. Thresholds are from a B300 sweep. BS=192/384 are not
|
||||
// instantiable: VPT=8 requires kBlockSize to divide kHiddenDim / 8 = 1024.
|
||||
if (!isBlackwellFamily()) {
|
||||
launchFp32RouterGemm<InputT, 128, 1, kNumTokens, kNumExperts, kHiddenDim>(
|
||||
output, mat_a, mat_b, stream);
|
||||
return;
|
||||
}
|
||||
if constexpr (kNumTokens >= 10 && kNumTokens % 2 == 0) {
|
||||
launchFp32RouterGemm<InputT, 256, 2, kNumTokens, kNumExperts, kHiddenDim,
|
||||
2>(output, mat_a, mat_b, stream);
|
||||
} else if constexpr (kNumTokens >= 13) {
|
||||
launchFp32RouterGemm<InputT, 256, 2, kNumTokens, kNumExperts, kHiddenDim>(
|
||||
output, mat_a, mat_b, stream);
|
||||
} else {
|
||||
launchFp32RouterGemm<InputT, 256, 1, kNumTokens, kNumExperts, kHiddenDim>(
|
||||
output, mat_a, mat_b, stream);
|
||||
}
|
||||
} else {
|
||||
launchFp32RouterGemm<InputT, 128, 1, kNumTokens, kNumExperts, kHiddenDim>(
|
||||
output, mat_a, mat_b, stream);
|
||||
@@ -268,8 +307,9 @@ void invokeFp32RouterGemm(float* output, InputT const* mat_a,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Explicit instantiations: M=1..32, for both input types, for the supported
|
||||
// (E, H) pairs: (256, 3072) [MiniMax-M2/M2.5] and (128, 6144) [MiniMax-M3].
|
||||
// Explicit instantiations for both input types.
|
||||
// Supported (E, H) pairs: (256, 3072), (128, 6144), (256, 5120),
|
||||
// (352, 4096) [Step-5], and (256, 8192) [Muse Spark].
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#define INSTANTIATE(T, M, E, H) \
|
||||
@@ -315,5 +355,36 @@ INSTANTIATE_ALL(__nv_bfloat16, 256, 3072)
|
||||
INSTANTIATE_ALL(float, 128, 6144)
|
||||
INSTANTIATE_ALL(__nv_bfloat16, 128, 6144)
|
||||
|
||||
INSTANTIATE_ALL(float, 256, 5120)
|
||||
INSTANTIATE_ALL(__nv_bfloat16, 256, 5120)
|
||||
|
||||
INSTANTIATE_ALL(float, 352, 4096)
|
||||
INSTANTIATE_ALL(__nv_bfloat16, 352, 4096)
|
||||
INSTANTIATE_ALL(float, 256, 8192)
|
||||
INSTANTIATE_ALL(__nv_bfloat16, 256, 8192)
|
||||
|
||||
#define INSTANTIATE_LARGE(T, E, H) \
|
||||
INSTANTIATE(T, 33, E, H) \
|
||||
INSTANTIATE(T, 34, E, H) \
|
||||
INSTANTIATE(T, 35, E, H) \
|
||||
INSTANTIATE(T, 36, E, H) \
|
||||
INSTANTIATE(T, 37, E, H) \
|
||||
INSTANTIATE(T, 38, E, H) \
|
||||
INSTANTIATE(T, 39, E, H) \
|
||||
INSTANTIATE(T, 40, E, H) \
|
||||
INSTANTIATE(T, 41, E, H) \
|
||||
INSTANTIATE(T, 42, E, H) \
|
||||
INSTANTIATE(T, 43, E, H) \
|
||||
INSTANTIATE(T, 44, E, H) \
|
||||
INSTANTIATE(T, 45, E, H) \
|
||||
INSTANTIATE(T, 46, E, H) \
|
||||
INSTANTIATE(T, 47, E, H) \
|
||||
INSTANTIATE(T, 48, E, H)
|
||||
|
||||
INSTANTIATE_LARGE(float, 256, 5120)
|
||||
INSTANTIATE_LARGE(__nv_bfloat16, 256, 5120)
|
||||
|
||||
#undef INSTANTIATE_LARGE
|
||||
|
||||
#undef INSTANTIATE_ALL
|
||||
#undef INSTANTIATE
|
||||
|
||||
@@ -25,10 +25,13 @@ inline int getSMVersion() {
|
||||
static constexpr int FP32_MAX_TOKENS = 32;
|
||||
|
||||
// Supported (hidden_dim, num_experts) pairs (must match the instantiations in
|
||||
// fp32_router_gemm.cu): (3072, 256) for MiniMax-M2/M2.5, (6144, 128) for M3.
|
||||
// fp32_router_gemm.cu).
|
||||
static inline bool fp32_router_gemm_supported(int hidden_dim, int num_experts) {
|
||||
return (hidden_dim == 3072 && num_experts == 256) ||
|
||||
(hidden_dim == 6144 && num_experts == 128);
|
||||
(hidden_dim == 6144 && num_experts == 128) ||
|
||||
(hidden_dim == 5120 && num_experts == 256) ||
|
||||
(hidden_dim == 4096 && num_experts == 352) ||
|
||||
(hidden_dim == 8192 && num_experts == 256);
|
||||
}
|
||||
|
||||
// Forward declarations — 4 template params must match fp32_router_gemm.cu
|
||||
@@ -60,8 +63,7 @@ struct Fp32LoopUnroller<InputT, kNumExperts, kHiddenDim, kEnd, kEnd> {
|
||||
invokeFp32RouterGemm<InputT, kEnd, kNumExperts, kHiddenDim>(
|
||||
output, mat_a, mat_b, stream);
|
||||
} else {
|
||||
throw std::invalid_argument(
|
||||
"fp32_router_gemm: num_tokens must be in [1, 32]");
|
||||
throw std::invalid_argument("fp32_router_gemm: unsupported num_tokens");
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -77,6 +79,15 @@ void dispatchFp32RouterGemm(int num_experts, int hidden_dim, int num_tokens,
|
||||
} else if (num_experts == 128 && hidden_dim == 6144) {
|
||||
Fp32LoopUnroller<InputT, 128, 6144, 1, FP32_MAX_TOKENS>::unroll(
|
||||
num_tokens, output, mat_a, mat_b, stream);
|
||||
} else if (num_experts == 256 && hidden_dim == 5120) {
|
||||
Fp32LoopUnroller<InputT, 256, 5120, 1, 48>::unroll(num_tokens, output,
|
||||
mat_a, mat_b, stream);
|
||||
} else if (num_experts == 352 && hidden_dim == 4096) {
|
||||
Fp32LoopUnroller<InputT, 352, 4096, 1, FP32_MAX_TOKENS>::unroll(
|
||||
num_tokens, output, mat_a, mat_b, stream);
|
||||
} else if (num_experts == 256 && hidden_dim == 8192) {
|
||||
Fp32LoopUnroller<InputT, 256, 8192, 1, FP32_MAX_TOKENS>::unroll(
|
||||
num_tokens, output, mat_a, mat_b, stream);
|
||||
} else {
|
||||
throw std::invalid_argument(
|
||||
"fp32_router_gemm: unsupported (hidden_dim, num_experts) pair");
|
||||
@@ -111,9 +122,11 @@ void fp32_router_gemm(
|
||||
STD_TORCH_CHECK(
|
||||
fp32_router_gemm_supported(hidden_dim, num_experts),
|
||||
"fp32_router_gemm: supported (hidden_dim, num_experts) pairs are "
|
||||
"(3072, 256) and (6144, 128)");
|
||||
STD_TORCH_CHECK(num_tokens <= FP32_MAX_TOKENS,
|
||||
"fp32_router_gemm: num_tokens must be in [0, 32]");
|
||||
"(3072, 256), (6144, 128), (5120, 256), (4096, 352), and (8192, 256)");
|
||||
const int max_tokens =
|
||||
hidden_dim == 5120 && num_experts == 256 ? 48 : FP32_MAX_TOKENS;
|
||||
STD_TORCH_CHECK(num_tokens <= max_tokens,
|
||||
"fp32_router_gemm: num_tokens exceeds supported range");
|
||||
STD_TORCH_CHECK(
|
||||
mat_a.scalar_type() == torch::headeronly::ScalarType::Float ||
|
||||
mat_a.scalar_type() == torch::headeronly::ScalarType::BFloat16,
|
||||
|
||||
@@ -579,10 +579,6 @@ void situ_and_mul_quant(
|
||||
int64_t group_size = 0,
|
||||
std::optional<torch::stable::Tensor> num_valid_tokens = std::nullopt,
|
||||
int64_t topk = 1);
|
||||
void masked_situ_and_mul(torch::stable::Tensor& out,
|
||||
torch::stable::Tensor& input,
|
||||
const torch::stable::Tensor& expert_num_tokens,
|
||||
double beta = 1.0, double linear_beta = -1.0);
|
||||
void masked_moe_activation(torch::stable::Tensor& out,
|
||||
torch::stable::Tensor& input,
|
||||
const torch::stable::Tensor& valid_token_counts,
|
||||
|
||||
@@ -135,6 +135,13 @@ hadamard_transform_kernel(b16* a, b16* out, int total_num_chunks) {
|
||||
|
||||
#pragma unroll
|
||||
for (int64_t k = 0; k < num_chunks; k++) {
|
||||
// The last warp may own fewer chunks than num_chunks: loading the
|
||||
// rest would read past the end of `a`. The cp.async.wait_group counts
|
||||
// below already assume only real_num_chunks groups were committed.
|
||||
if constexpr(enable_mask) {
|
||||
if (k >= real_num_chunks)
|
||||
break;
|
||||
}
|
||||
size_t shared_ptr = __cvta_generic_to_shared(b_frag_ptr);
|
||||
#if (__CUDA_ARCH__ >= 900) // SM90
|
||||
asm volatile(
|
||||
|
||||
@@ -640,9 +640,6 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"situ_and_mul_quant(Tensor! out, Tensor! scale, Tensor input, "
|
||||
"float beta=1.0, float linear_beta=-1.0, int group_size=0, "
|
||||
"Tensor? num_valid_tokens=None, int topk=1) -> ()");
|
||||
ops.def(
|
||||
"masked_situ_and_mul(Tensor! out, Tensor input, Tensor "
|
||||
"expert_num_tokens, float beta=1.0, float linear_beta=-1.0) -> ()");
|
||||
ops.def(
|
||||
"masked_moe_activation(Tensor! out, Tensor input, Tensor "
|
||||
"valid_token_counts, str activation, float clamp_limit=0.0, float "
|
||||
@@ -875,7 +872,6 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
|
||||
ops.impl("swigluoai_and_mul", TORCH_BOX(&swigluoai_and_mul));
|
||||
ops.impl("situ_and_mul", TORCH_BOX(&situ_and_mul));
|
||||
ops.impl("situ_and_mul_quant", TORCH_BOX(&situ_and_mul_quant));
|
||||
ops.impl("masked_situ_and_mul", TORCH_BOX(&masked_situ_and_mul));
|
||||
ops.impl("masked_moe_activation", TORCH_BOX(&masked_moe_activation));
|
||||
ops.impl("gelu_new", TORCH_BOX(&gelu_new));
|
||||
ops.impl("gelu_fast", TORCH_BOX(&gelu_fast));
|
||||
|
||||
Reference in New Issue
Block a user