Merge branch 'main' into patches

This commit is contained in:
2026-10-17 08:06:56 +00:00
222 changed files with 14823 additions and 3606 deletions
-27
View File
@@ -1059,33 +1059,6 @@ void situ_and_mul(torch::stable::Tensor& out, // [..., d]
});
}
void masked_situ_and_mul(torch::stable::Tensor& out, // [E, T, d]
torch::stable::Tensor& input, // [E, T, 2 * d]
const torch::stable::Tensor& expert_num_tokens,
double beta, double linear_beta) {
int num_experts = input.size(0);
int max_num_tokens = input.size(1);
int d = input.size(2) / 2;
if (num_experts == 0 || max_num_tokens == 0 || d == 0) {
return;
}
constexpr int block_size = 256;
dim3 grid((d + block_size - 1) / block_size, num_experts);
dim3 block(block_size);
const torch::stable::accelerator::DeviceGuard device_guard(
input.get_device_index());
const cudaStream_t stream = get_current_cuda_stream();
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
input.scalar_type(), "masked_situ_and_mul_kernel", [&] {
vllm::masked_situ_and_mul_kernel<scalar_t, true>
<<<grid, block, 0, stream>>>(
out.mutable_data_ptr<scalar_t>(),
input.const_data_ptr<scalar_t>(),
expert_num_tokens.const_data_ptr<int>(), max_num_tokens, d,
(float)beta, (float)linear_beta);
});
}
#define LAUNCH_MASKED_ACT_AND_MUL(KERNEL, ACT_FIRST, HAS_CLAMP, LIMIT, ALPHA, \
BETA) \
vllm::masked_act_and_mul_kernel<scalar_t, KERNEL<scalar_t>, ACT_FIRST, \
+76 -5
View File
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//
// Router GEMM: activation(T) x weight(fp32) -> fp32, M<=32, for the
// Router GEMM: activation(T) x weight(fp32) -> fp32, M<=48, for the
// supported (E, H) pairs listed at the bottom of this file.
// Supports bf16 or fp32 activation; weight is always fp32.
@@ -214,8 +214,26 @@ void invokeFp32RouterGemm(float* output, InputT const* mat_a,
// production-fidelity harness (CUDA-graph replay, per-layer cold weights).
// Only enabled on the Blackwell family where it was validated; Hopper and
// other shapes / fp32 activation keep the legacy geometry.
if constexpr (std::is_same_v<InputT, __nv_bfloat16> && kNumExperts == 128 &&
kHiddenDim == 6144) {
if constexpr (std::is_same_v<InputT, __nv_bfloat16> && kNumExperts == 256 &&
kHiddenDim == 5120) {
if (!isBlackwellFamily()) {
launchFp32RouterGemm<InputT, 128, 1, kNumTokens, kNumExperts, kHiddenDim>(
output, mat_a, mat_b, stream);
return;
}
if constexpr (kNumTokens > 16 && kNumTokens % 8 == 0) {
constexpr int block_size = kNumTokens == 24 ? 160 : 128;
launchFp32RouterGemm<InputT, block_size, 2, kNumTokens, kNumExperts,
kHiddenDim, 4>(output, mat_a, mat_b, stream);
} else {
constexpr int block_size = kNumTokens == 1 ? 64 : 128;
constexpr int token_groups = kNumTokens % 2 == 0 ? 2 : 1;
launchFp32RouterGemm<InputT, block_size, 1, kNumTokens, kNumExperts,
kHiddenDim, token_groups>(output, mat_a, mat_b,
stream);
}
} else if constexpr (std::is_same_v<InputT, __nv_bfloat16> &&
kNumExperts == 128 && kHiddenDim == 6144) {
// MiniMax-M3. Legacy 128/1 only fills 128 blocks and pays the same
// accumulator register cliffs; B300 sweep:
// even M in [6, 10] : BS=384, EPB=1, 2 token groups (1.26-1.43x)
@@ -261,6 +279,27 @@ void invokeFp32RouterGemm(float* output, InputT const* mat_a,
launchFp32RouterGemm<InputT, 128, 1, kNumTokens, kNumExperts, kHiddenDim>(
output, mat_a, mat_b, stream);
}
} else if constexpr (std::is_same_v<InputT, __nv_bfloat16> &&
kNumExperts == 256 && kHiddenDim == 8192) {
// MuseSpark1x: the 8MB weight makes this shape FFMA-bound past M~10, so
// the wider block wins at every M and token groups clear the M >= 28
// accumulator cliff. Thresholds are from a B300 sweep. BS=192/384 are not
// instantiable: VPT=8 requires kBlockSize to divide kHiddenDim / 8 = 1024.
if (!isBlackwellFamily()) {
launchFp32RouterGemm<InputT, 128, 1, kNumTokens, kNumExperts, kHiddenDim>(
output, mat_a, mat_b, stream);
return;
}
if constexpr (kNumTokens >= 10 && kNumTokens % 2 == 0) {
launchFp32RouterGemm<InputT, 256, 2, kNumTokens, kNumExperts, kHiddenDim,
2>(output, mat_a, mat_b, stream);
} else if constexpr (kNumTokens >= 13) {
launchFp32RouterGemm<InputT, 256, 2, kNumTokens, kNumExperts, kHiddenDim>(
output, mat_a, mat_b, stream);
} else {
launchFp32RouterGemm<InputT, 256, 1, kNumTokens, kNumExperts, kHiddenDim>(
output, mat_a, mat_b, stream);
}
} else {
launchFp32RouterGemm<InputT, 128, 1, kNumTokens, kNumExperts, kHiddenDim>(
output, mat_a, mat_b, stream);
@@ -268,8 +307,9 @@ void invokeFp32RouterGemm(float* output, InputT const* mat_a,
}
// ---------------------------------------------------------------------------
// Explicit instantiations: M=1..32, for both input types, for the supported
// (E, H) pairs: (256, 3072) [MiniMax-M2/M2.5] and (128, 6144) [MiniMax-M3].
// Explicit instantiations for both input types.
// Supported (E, H) pairs: (256, 3072), (128, 6144), (256, 5120),
// (352, 4096) [Step-5], and (256, 8192) [Muse Spark].
// ---------------------------------------------------------------------------
#define INSTANTIATE(T, M, E, H) \
@@ -315,5 +355,36 @@ INSTANTIATE_ALL(__nv_bfloat16, 256, 3072)
INSTANTIATE_ALL(float, 128, 6144)
INSTANTIATE_ALL(__nv_bfloat16, 128, 6144)
INSTANTIATE_ALL(float, 256, 5120)
INSTANTIATE_ALL(__nv_bfloat16, 256, 5120)
INSTANTIATE_ALL(float, 352, 4096)
INSTANTIATE_ALL(__nv_bfloat16, 352, 4096)
INSTANTIATE_ALL(float, 256, 8192)
INSTANTIATE_ALL(__nv_bfloat16, 256, 8192)
#define INSTANTIATE_LARGE(T, E, H) \
INSTANTIATE(T, 33, E, H) \
INSTANTIATE(T, 34, E, H) \
INSTANTIATE(T, 35, E, H) \
INSTANTIATE(T, 36, E, H) \
INSTANTIATE(T, 37, E, H) \
INSTANTIATE(T, 38, E, H) \
INSTANTIATE(T, 39, E, H) \
INSTANTIATE(T, 40, E, H) \
INSTANTIATE(T, 41, E, H) \
INSTANTIATE(T, 42, E, H) \
INSTANTIATE(T, 43, E, H) \
INSTANTIATE(T, 44, E, H) \
INSTANTIATE(T, 45, E, H) \
INSTANTIATE(T, 46, E, H) \
INSTANTIATE(T, 47, E, H) \
INSTANTIATE(T, 48, E, H)
INSTANTIATE_LARGE(float, 256, 5120)
INSTANTIATE_LARGE(__nv_bfloat16, 256, 5120)
#undef INSTANTIATE_LARGE
#undef INSTANTIATE_ALL
#undef INSTANTIATE
+20 -7
View File
@@ -25,10 +25,13 @@ inline int getSMVersion() {
static constexpr int FP32_MAX_TOKENS = 32;
// Supported (hidden_dim, num_experts) pairs (must match the instantiations in
// fp32_router_gemm.cu): (3072, 256) for MiniMax-M2/M2.5, (6144, 128) for M3.
// fp32_router_gemm.cu).
static inline bool fp32_router_gemm_supported(int hidden_dim, int num_experts) {
return (hidden_dim == 3072 && num_experts == 256) ||
(hidden_dim == 6144 && num_experts == 128);
(hidden_dim == 6144 && num_experts == 128) ||
(hidden_dim == 5120 && num_experts == 256) ||
(hidden_dim == 4096 && num_experts == 352) ||
(hidden_dim == 8192 && num_experts == 256);
}
// Forward declarations — 4 template params must match fp32_router_gemm.cu
@@ -60,8 +63,7 @@ struct Fp32LoopUnroller<InputT, kNumExperts, kHiddenDim, kEnd, kEnd> {
invokeFp32RouterGemm<InputT, kEnd, kNumExperts, kHiddenDim>(
output, mat_a, mat_b, stream);
} else {
throw std::invalid_argument(
"fp32_router_gemm: num_tokens must be in [1, 32]");
throw std::invalid_argument("fp32_router_gemm: unsupported num_tokens");
}
}
};
@@ -77,6 +79,15 @@ void dispatchFp32RouterGemm(int num_experts, int hidden_dim, int num_tokens,
} else if (num_experts == 128 && hidden_dim == 6144) {
Fp32LoopUnroller<InputT, 128, 6144, 1, FP32_MAX_TOKENS>::unroll(
num_tokens, output, mat_a, mat_b, stream);
} else if (num_experts == 256 && hidden_dim == 5120) {
Fp32LoopUnroller<InputT, 256, 5120, 1, 48>::unroll(num_tokens, output,
mat_a, mat_b, stream);
} else if (num_experts == 352 && hidden_dim == 4096) {
Fp32LoopUnroller<InputT, 352, 4096, 1, FP32_MAX_TOKENS>::unroll(
num_tokens, output, mat_a, mat_b, stream);
} else if (num_experts == 256 && hidden_dim == 8192) {
Fp32LoopUnroller<InputT, 256, 8192, 1, FP32_MAX_TOKENS>::unroll(
num_tokens, output, mat_a, mat_b, stream);
} else {
throw std::invalid_argument(
"fp32_router_gemm: unsupported (hidden_dim, num_experts) pair");
@@ -111,9 +122,11 @@ void fp32_router_gemm(
STD_TORCH_CHECK(
fp32_router_gemm_supported(hidden_dim, num_experts),
"fp32_router_gemm: supported (hidden_dim, num_experts) pairs are "
"(3072, 256) and (6144, 128)");
STD_TORCH_CHECK(num_tokens <= FP32_MAX_TOKENS,
"fp32_router_gemm: num_tokens must be in [0, 32]");
"(3072, 256), (6144, 128), (5120, 256), (4096, 352), and (8192, 256)");
const int max_tokens =
hidden_dim == 5120 && num_experts == 256 ? 48 : FP32_MAX_TOKENS;
STD_TORCH_CHECK(num_tokens <= max_tokens,
"fp32_router_gemm: num_tokens exceeds supported range");
STD_TORCH_CHECK(
mat_a.scalar_type() == torch::headeronly::ScalarType::Float ||
mat_a.scalar_type() == torch::headeronly::ScalarType::BFloat16,
-4
View File
@@ -579,10 +579,6 @@ void situ_and_mul_quant(
int64_t group_size = 0,
std::optional<torch::stable::Tensor> num_valid_tokens = std::nullopt,
int64_t topk = 1);
void masked_situ_and_mul(torch::stable::Tensor& out,
torch::stable::Tensor& input,
const torch::stable::Tensor& expert_num_tokens,
double beta = 1.0, double linear_beta = -1.0);
void masked_moe_activation(torch::stable::Tensor& out,
torch::stable::Tensor& input,
const torch::stable::Tensor& valid_token_counts,
@@ -135,6 +135,13 @@ hadamard_transform_kernel(b16* a, b16* out, int total_num_chunks) {
#pragma unroll
for (int64_t k = 0; k < num_chunks; k++) {
// The last warp may own fewer chunks than num_chunks: loading the
// rest would read past the end of `a`. The cp.async.wait_group counts
// below already assume only real_num_chunks groups were committed.
if constexpr(enable_mask) {
if (k >= real_num_chunks)
break;
}
size_t shared_ptr = __cvta_generic_to_shared(b_frag_ptr);
#if (__CUDA_ARCH__ >= 900) // SM90
asm volatile(
-4
View File
@@ -640,9 +640,6 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
"situ_and_mul_quant(Tensor! out, Tensor! scale, Tensor input, "
"float beta=1.0, float linear_beta=-1.0, int group_size=0, "
"Tensor? num_valid_tokens=None, int topk=1) -> ()");
ops.def(
"masked_situ_and_mul(Tensor! out, Tensor input, Tensor "
"expert_num_tokens, float beta=1.0, float linear_beta=-1.0) -> ()");
ops.def(
"masked_moe_activation(Tensor! out, Tensor input, Tensor "
"valid_token_counts, str activation, float clamp_limit=0.0, float "
@@ -875,7 +872,6 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
ops.impl("swigluoai_and_mul", TORCH_BOX(&swigluoai_and_mul));
ops.impl("situ_and_mul", TORCH_BOX(&situ_and_mul));
ops.impl("situ_and_mul_quant", TORCH_BOX(&situ_and_mul_quant));
ops.impl("masked_situ_and_mul", TORCH_BOX(&masked_situ_and_mul));
ops.impl("masked_moe_activation", TORCH_BOX(&masked_moe_activation));
ops.impl("gelu_new", TORCH_BOX(&gelu_new));
ops.impl("gelu_fast", TORCH_BOX(&gelu_fast));