[Model] Add MiniMax M3 support (#45381)

Signed-off-by: youkaichao <youkaichao@gmail.com>
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Signed-off-by: functionstackx <47992694+functionstackx@users.noreply.github.com>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: functionstackx <47992694+functionstackx@users.noreply.github.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
youkaichao
2026-06-16 01:01:25 +08:00
committed by GitHub
co-authored by OpenAI Codex Isotr0py Thien Tran Bugen Zhao Jee Jee Li Roger Wang functionstackx Yongye Zhu Jee Jee Li
parent a3195fab7b
commit 0a1c5034f5
108 changed files with 14734 additions and 311 deletions
+77 -41
View File
@@ -10,11 +10,20 @@
namespace vllm {
template <typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&),
// `alpha` and `beta` are applied to opposite operands:
// - alpha lives INSIDE the activation (the activated half): the gated
// activation computes act_half * sigmoid(alpha * act_half).
// - beta is added to the OTHER (non-activated) half before the multiply.
// So the result is always ACT(act_half, alpha) * (other_half + beta).
// Which half is which depends on `act_first` (see below). Defaults
// alpha=1.0, beta=0.0 reproduce the plain SwiGLU/GeGLU behavior.
template <typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&, const float),
bool act_first, bool HAS_CLAMP>
__device__ __forceinline__ scalar_t compute(const scalar_t& x,
const scalar_t& y,
const float limit) {
const float limit,
const float alpha,
const float beta) {
if constexpr (act_first) {
scalar_t gate = x;
scalar_t up = y;
@@ -22,7 +31,9 @@ __device__ __forceinline__ scalar_t compute(const scalar_t& x,
gate = (scalar_t)fminf((float)gate, limit);
up = (scalar_t)fmaxf(fminf((float)up, limit), -limit);
}
return ACT_FN(gate) * up;
// act_first: gate is the activated half -> alpha applies to gate;
// beta is added to up (the non-activated half).
return (scalar_t)(ACT_FN(gate, alpha) * ((float)up + beta));
} else {
scalar_t gate = x;
scalar_t up = y;
@@ -30,55 +41,68 @@ __device__ __forceinline__ scalar_t compute(const scalar_t& x,
gate = (scalar_t)fmaxf(fminf((float)gate, limit), -limit);
up = (scalar_t)fminf((float)up, limit);
}
return gate * ACT_FN(up);
// !act_first: up is the activated half -> alpha applies to up;
// beta is added to gate (the non-activated half).
return (scalar_t)(((float)gate + beta) * ACT_FN(up, alpha));
}
}
template <typename packed_t, packed_t (*PACKED_ACT_FN)(const packed_t&),
template <typename packed_t,
packed_t (*PACKED_ACT_FN)(const packed_t&, const float),
bool act_first, bool HAS_CLAMP>
__device__ __forceinline__ packed_t packed_compute(const packed_t& x,
const packed_t& y,
const float limit) {
const float limit,
const float alpha,
const float beta) {
if constexpr (act_first) {
packed_t gate = x;
packed_t up = y;
float2 u = cast_to_float2(up);
if constexpr (HAS_CLAMP) {
float2 g = cast_to_float2(gate);
float2 u = cast_to_float2(up);
g.x = fminf(g.x, limit);
g.y = fminf(g.y, limit);
u.x = fmaxf(fminf(u.x, limit), -limit);
u.y = fmaxf(fminf(u.y, limit), -limit);
gate = cast_to_packed<packed_t>(g);
up = cast_to_packed<packed_t>(u);
}
return packed_mul(PACKED_ACT_FN(gate), up);
// act_first: gate is the activated half -> alpha applies to gate;
// beta is added to up (the non-activated half).
float2 activated = cast_to_float2(PACKED_ACT_FN(gate, alpha));
activated.x *= u.x + beta;
activated.y *= u.y + beta;
return cast_to_packed<packed_t>(activated);
} else {
packed_t gate = x;
packed_t up = y;
float2 g = cast_to_float2(gate);
if constexpr (HAS_CLAMP) {
float2 g = cast_to_float2(gate);
float2 u = cast_to_float2(up);
g.x = fmaxf(fminf(g.x, limit), -limit);
g.y = fmaxf(fminf(g.y, limit), -limit);
u.x = fminf(u.x, limit);
u.y = fminf(u.y, limit);
gate = cast_to_packed<packed_t>(g);
up = cast_to_packed<packed_t>(u);
}
return packed_mul(gate, PACKED_ACT_FN(up));
// !act_first: up is the activated half -> alpha applies to up;
// beta is added to gate (the non-activated half).
float2 activated = cast_to_float2(PACKED_ACT_FN(up, alpha));
activated.x *= g.x + beta;
activated.y *= g.y + beta;
return cast_to_packed<packed_t>(activated);
}
}
// Activation and gating kernel template.
template <typename scalar_t, typename packed_t,
scalar_t (*ACT_FN)(const scalar_t&),
packed_t (*PACKED_ACT_FN)(const packed_t&), bool act_first,
bool use_vec, bool HAS_CLAMP, bool use_256b = false>
scalar_t (*ACT_FN)(const scalar_t&, const float),
packed_t (*PACKED_ACT_FN)(const packed_t&, const float),
bool act_first, bool use_vec, bool HAS_CLAMP, bool use_256b = false>
__global__ void act_and_mul_kernel(
scalar_t* __restrict__ out, // [..., d]
const scalar_t* __restrict__ input, // [..., 2, d]
const int d, const float limit) {
const int d, const float limit, const float alpha, const float beta) {
const scalar_t* x_ptr = input + blockIdx.x * 2 * d;
const scalar_t* y_ptr = x_ptr + d;
scalar_t* out_ptr = out + blockIdx.x * d;
@@ -105,7 +129,7 @@ __global__ void act_and_mul_kernel(
for (int j = 0; j < pvec_t::NUM_ELTS; j++) {
x.elts[j] =
packed_compute<packed_t, PACKED_ACT_FN, act_first, HAS_CLAMP>(
x.elts[j], y.elts[j], limit);
x.elts[j], y.elts[j], limit, alpha, beta);
}
if constexpr (use_256b) {
st256(x, &out_vec[i]);
@@ -118,29 +142,34 @@ __global__ void act_and_mul_kernel(
for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
const scalar_t x = VLLM_LDG(&x_ptr[idx]);
const scalar_t y = VLLM_LDG(&y_ptr[idx]);
out_ptr[idx] =
compute<scalar_t, ACT_FN, act_first, HAS_CLAMP>(x, y, limit);
out_ptr[idx] = compute<scalar_t, ACT_FN, act_first, HAS_CLAMP>(
x, y, limit, alpha, beta);
}
}
}
// Gated activations take an `alpha` argument that scales the sigmoid input
// (`x * sigmoid(alpha * x)`). alpha defaults to 1.0 at all call sites, which
// is exactly SiLU; only the clamp path (silu_and_mul_with_clamp) passes a
// non-default alpha. Activations that do not use alpha simply ignore it.
template <typename T>
__device__ __forceinline__ T silu_kernel(const T& x) {
// x * sigmoid(x)
return (T)(((float)x) / (1.0f + expf((float)-x)));
__device__ __forceinline__ T silu_kernel(const T& x, const float alpha) {
// x * sigmoid(alpha * x)
return (T)(((float)x) / (1.0f + expf((float)-x * alpha)));
}
template <typename packed_t>
__device__ __forceinline__ packed_t packed_silu_kernel(const packed_t& val) {
// x * sigmoid(x)
__device__ __forceinline__ packed_t packed_silu_kernel(const packed_t& val,
const float alpha) {
// x * sigmoid(alpha * x)
float2 fval = cast_to_float2(val);
fval.x = fval.x / (1.0f + expf(-fval.x));
fval.y = fval.y / (1.0f + expf(-fval.y));
fval.x = fval.x / (1.0f + expf(-fval.x * alpha));
fval.y = fval.y / (1.0f + expf(-fval.y * alpha));
return cast_to_packed<packed_t>(fval);
}
template <typename T>
__device__ __forceinline__ T gelu_kernel(const T& x) {
__device__ __forceinline__ T gelu_kernel(const T& x, const float /*alpha*/) {
// Equivalent to PyTorch GELU with 'none' approximation.
// Refer to:
// https://github.com/pytorch/pytorch/blob/8ac9b20d4b090c213799e81acf48a55ea8d437d6/aten/src/ATen/native/cuda/ActivationGeluKernel.cu#L36-L38
@@ -150,7 +179,8 @@ __device__ __forceinline__ T gelu_kernel(const T& x) {
}
template <typename packed_t>
__device__ __forceinline__ packed_t packed_gelu_kernel(const packed_t& val) {
__device__ __forceinline__ packed_t packed_gelu_kernel(const packed_t& val,
const float /*alpha*/) {
// Equivalent to PyTorch GELU with 'none' approximation.
// Refer to:
// https://github.com/pytorch/pytorch/blob/8ac9b20d4b090c213799e81acf48a55ea8d437d6/aten/src/ATen/native/cuda/ActivationGeluKernel.cu#L36-L38
@@ -162,7 +192,8 @@ __device__ __forceinline__ packed_t packed_gelu_kernel(const packed_t& val) {
}
template <typename T>
__device__ __forceinline__ T gelu_tanh_kernel(const T& x) {
__device__ __forceinline__ T gelu_tanh_kernel(const T& x,
const float /*alpha*/) {
// Equivalent to PyTorch GELU with 'tanh' approximation.
// Refer to:
// https://github.com/pytorch/pytorch/blob/8ac9b20d4b090c213799e81acf48a55ea8d437d6/aten/src/ATen/native/cuda/ActivationGeluKernel.cu#L25-L30
@@ -176,7 +207,7 @@ __device__ __forceinline__ T gelu_tanh_kernel(const T& x) {
template <typename packed_t>
__device__ __forceinline__ packed_t
packed_gelu_tanh_kernel(const packed_t& val) {
packed_gelu_tanh_kernel(const packed_t& val, const float /*alpha*/) {
// Equivalent to PyTorch GELU with 'tanh' approximation.
// Refer to:
// https://github.com/pytorch/pytorch/blob/8ac9b20d4b090c213799e81acf48a55ea8d437d6/aten/src/ATen/native/cuda/ActivationGeluKernel.cu#L25-L30
@@ -202,7 +233,7 @@ packed_gelu_tanh_kernel(const packed_t& val) {
// clamped (max only) and up input is clamped (both sides) before the
// activation function is applied.
#define LAUNCH_ACTIVATION_GATE_KERNEL(KERNEL, PACKED_KERNEL, ACT_FIRST, \
HAS_CLAMP, LIMIT) \
HAS_CLAMP, LIMIT, ALPHA, BETA) \
auto dtype = input.scalar_type(); \
int d = input.size(-1) / 2; \
int64_t num_tokens = input.numel() / input.size(-1); \
@@ -230,7 +261,7 @@ packed_gelu_tanh_kernel(const packed_t& val) {
PACKED_KERNEL<typename vllm::PackedTypeConverter<scalar_t>::Type>, \
ACT_FIRST, true, HAS_CLAMP, true><<<grid, block, 0, stream>>>( \
out.mutable_data_ptr<scalar_t>(), \
input.const_data_ptr<scalar_t>(), d, LIMIT); \
input.const_data_ptr<scalar_t>(), d, LIMIT, ALPHA, BETA); \
}); \
} else { \
VLLM_STABLE_DISPATCH_FLOATING_TYPES(dtype, "act_and_mul_kernel", [&] { \
@@ -240,7 +271,7 @@ packed_gelu_tanh_kernel(const packed_t& val) {
PACKED_KERNEL<typename vllm::PackedTypeConverter<scalar_t>::Type>, \
ACT_FIRST, true, HAS_CLAMP, false><<<grid, block, 0, stream>>>( \
out.mutable_data_ptr<scalar_t>(), \
input.const_data_ptr<scalar_t>(), d, LIMIT); \
input.const_data_ptr<scalar_t>(), d, LIMIT, ALPHA, BETA); \
}); \
} \
} else { \
@@ -252,7 +283,7 @@ packed_gelu_tanh_kernel(const packed_t& val) {
PACKED_KERNEL<typename vllm::PackedTypeConverter<scalar_t>::Type>, \
ACT_FIRST, false, HAS_CLAMP><<<grid, block, 0, stream>>>( \
out.mutable_data_ptr<scalar_t>(), input.const_data_ptr<scalar_t>(), \
d, LIMIT); \
d, LIMIT, ALPHA, BETA); \
}); \
}
@@ -260,14 +291,18 @@ void silu_and_mul(torch::stable::Tensor& out, // [..., d]
torch::stable::Tensor& input) // [..., 2 * d]
{
LAUNCH_ACTIVATION_GATE_KERNEL(vllm::silu_kernel, vllm::packed_silu_kernel,
true, false, 0.0f);
true, false, 0.0f, 1.0f, 0.0f);
}
void silu_and_mul_clamp(torch::stable::Tensor& out, // [..., d]
torch::stable::Tensor& input, // [..., 2 * d]
double limit) {
double limit, double alpha, double beta) {
// out = (gate.clamp(max=limit) * sigmoid(alpha * gate.clamp(max=limit)))
// * (up.clamp(+-limit) + beta)
// alpha=1.0, beta=0.0 reduce this to silu(gate) * up.
LAUNCH_ACTIVATION_GATE_KERNEL(vllm::silu_kernel, vllm::packed_silu_kernel,
true, true, (float)limit);
true, true, (float)limit, (float)alpha,
(float)beta);
}
void mul_and_silu(torch::stable::Tensor& out, // [..., d]
@@ -276,21 +311,22 @@ void mul_and_silu(torch::stable::Tensor& out, // [..., d]
// The difference between mul_and_silu and silu_and_mul is that mul_and_silu
// applies the silu to the latter half of the input.
LAUNCH_ACTIVATION_GATE_KERNEL(vllm::silu_kernel, vllm::packed_silu_kernel,
false, false, 0.0f);
false, false, 0.0f, 1.0f, 0.0f);
}
void gelu_and_mul(torch::stable::Tensor& out, // [..., d]
torch::stable::Tensor& input) // [..., 2 * d]
{
LAUNCH_ACTIVATION_GATE_KERNEL(vllm::gelu_kernel, vllm::packed_gelu_kernel,
true, false, 0.0f);
true, false, 0.0f, 1.0f, 0.0f);
}
void gelu_tanh_and_mul(torch::stable::Tensor& out, // [..., d]
torch::stable::Tensor& input) // [..., 2 * d]
{
LAUNCH_ACTIVATION_GATE_KERNEL(
vllm::gelu_tanh_kernel, vllm::packed_gelu_tanh_kernel, true, false, 0.0f);
LAUNCH_ACTIVATION_GATE_KERNEL(vllm::gelu_tanh_kernel,
vllm::packed_gelu_tanh_kernel, true, false,
0.0f, 1.0f, 0.0f);
}
namespace vllm {
+42 -39
View File
@@ -175,49 +175,52 @@ void invokeFp32RouterGemm(float* output, InputT const* mat_a,
}
// ---------------------------------------------------------------------------
// Explicit instantiations: M=1..32, E=256, H=3072, for both input types
// Explicit instantiations: M=1..32, for both input types, for the supported
// (E, H) pairs: (256, 3072) [MiniMax-M2/M2.5] and (128, 6144) [MiniMax-M3].
// ---------------------------------------------------------------------------
#define INSTANTIATE(T, M) \
template void invokeFp32RouterGemm<T, M, 256, 3072>( \
float*, T const*, float const*, cudaStream_t);
#define INSTANTIATE(T, M, E, H) \
template void invokeFp32RouterGemm<T, M, E, H>(float*, T const*, \
float const*, cudaStream_t);
#define INSTANTIATE_ALL(T) \
INSTANTIATE(T, 1) \
INSTANTIATE(T, 2) \
INSTANTIATE(T, 3) \
INSTANTIATE(T, 4) \
INSTANTIATE(T, 5) \
INSTANTIATE(T, 6) \
INSTANTIATE(T, 7) \
INSTANTIATE(T, 8) \
INSTANTIATE(T, 9) \
INSTANTIATE(T, 10) \
INSTANTIATE(T, 11) \
INSTANTIATE(T, 12) \
INSTANTIATE(T, 13) \
INSTANTIATE(T, 14) \
INSTANTIATE(T, 15) \
INSTANTIATE(T, 16) \
INSTANTIATE(T, 17) \
INSTANTIATE(T, 18) \
INSTANTIATE(T, 19) \
INSTANTIATE(T, 20) \
INSTANTIATE(T, 21) \
INSTANTIATE(T, 22) \
INSTANTIATE(T, 23) \
INSTANTIATE(T, 24) \
INSTANTIATE(T, 25) \
INSTANTIATE(T, 26) \
INSTANTIATE(T, 27) \
INSTANTIATE(T, 28) \
INSTANTIATE(T, 29) \
INSTANTIATE(T, 30) \
INSTANTIATE(T, 31) \
INSTANTIATE(T, 32)
#define INSTANTIATE_ALL(T, E, H) \
INSTANTIATE(T, 1, E, H) \
INSTANTIATE(T, 2, E, H) \
INSTANTIATE(T, 3, E, H) \
INSTANTIATE(T, 4, E, H) \
INSTANTIATE(T, 5, E, H) \
INSTANTIATE(T, 6, E, H) \
INSTANTIATE(T, 7, E, H) \
INSTANTIATE(T, 8, E, H) \
INSTANTIATE(T, 9, E, H) \
INSTANTIATE(T, 10, E, H) \
INSTANTIATE(T, 11, E, H) \
INSTANTIATE(T, 12, E, H) \
INSTANTIATE(T, 13, E, H) \
INSTANTIATE(T, 14, E, H) \
INSTANTIATE(T, 15, E, H) \
INSTANTIATE(T, 16, E, H) \
INSTANTIATE(T, 17, E, H) \
INSTANTIATE(T, 18, E, H) \
INSTANTIATE(T, 19, E, H) \
INSTANTIATE(T, 20, E, H) \
INSTANTIATE(T, 21, E, H) \
INSTANTIATE(T, 22, E, H) \
INSTANTIATE(T, 23, E, H) \
INSTANTIATE(T, 24, E, H) \
INSTANTIATE(T, 25, E, H) \
INSTANTIATE(T, 26, E, H) \
INSTANTIATE(T, 27, E, H) \
INSTANTIATE(T, 28, E, H) \
INSTANTIATE(T, 29, E, H) \
INSTANTIATE(T, 30, E, H) \
INSTANTIATE(T, 31, E, H) \
INSTANTIATE(T, 32, E, H)
INSTANTIATE_ALL(float)
INSTANTIATE_ALL(__nv_bfloat16)
INSTANTIATE_ALL(float, 256, 3072)
INSTANTIATE_ALL(__nv_bfloat16, 256, 3072)
INSTANTIATE_ALL(float, 128, 6144)
INSTANTIATE_ALL(__nv_bfloat16, 128, 6144)
#undef INSTANTIATE_ALL
#undef INSTANTIATE
+42 -18
View File
@@ -22,36 +22,42 @@ inline int getSMVersion() {
} // namespace
static constexpr int FP32_NUM_EXPERTS = 256;
static constexpr int FP32_HIDDEN_DIM = 3072;
static constexpr int FP32_MAX_TOKENS = 32;
// Supported (hidden_dim, num_experts) pairs (must match the instantiations in
// fp32_router_gemm.cu): (3072, 256) for MiniMax-M2/M2.5, (6144, 128) for M3.
static inline bool fp32_router_gemm_supported(int hidden_dim, int num_experts) {
return (hidden_dim == 3072 && num_experts == 256) ||
(hidden_dim == 6144 && num_experts == 128);
}
// Forward declarations — 4 template params must match fp32_router_gemm.cu
template <typename InputT, int kNumTokens, int kNumExperts, int kHiddenDim>
void invokeFp32RouterGemm(float* output, InputT const* mat_a,
float const* mat_b, cudaStream_t stream);
// LoopUnroller templated on InputT
template <typename InputT, int kBegin, int kEnd>
// LoopUnroller templated on InputT, kNumExperts and kHiddenDim
template <typename InputT, int kNumExperts, int kHiddenDim, int kBegin,
int kEnd>
struct Fp32LoopUnroller {
static void unroll(int num_tokens, float* output, InputT const* mat_a,
float const* mat_b, cudaStream_t stream) {
if (num_tokens == kBegin) {
invokeFp32RouterGemm<InputT, kBegin, FP32_NUM_EXPERTS, FP32_HIDDEN_DIM>(
invokeFp32RouterGemm<InputT, kBegin, kNumExperts, kHiddenDim>(
output, mat_a, mat_b, stream);
} else {
Fp32LoopUnroller<InputT, kBegin + 1, kEnd>::unroll(num_tokens, output,
mat_a, mat_b, stream);
Fp32LoopUnroller<InputT, kNumExperts, kHiddenDim, kBegin + 1,
kEnd>::unroll(num_tokens, output, mat_a, mat_b, stream);
}
}
};
template <typename InputT, int kEnd>
struct Fp32LoopUnroller<InputT, kEnd, kEnd> {
template <typename InputT, int kNumExperts, int kHiddenDim, int kEnd>
struct Fp32LoopUnroller<InputT, kNumExperts, kHiddenDim, kEnd, kEnd> {
static void unroll(int num_tokens, float* output, InputT const* mat_a,
float const* mat_b, cudaStream_t stream) {
if (num_tokens == kEnd) {
invokeFp32RouterGemm<InputT, kEnd, FP32_NUM_EXPERTS, FP32_HIDDEN_DIM>(
invokeFp32RouterGemm<InputT, kEnd, kNumExperts, kHiddenDim>(
output, mat_a, mat_b, stream);
} else {
throw std::invalid_argument(
@@ -60,6 +66,23 @@ struct Fp32LoopUnroller<InputT, kEnd, kEnd> {
}
};
// Dispatch over the supported (num_experts, hidden_dim) pairs.
template <typename InputT>
void dispatchFp32RouterGemm(int num_experts, int hidden_dim, int num_tokens,
float* output, InputT const* mat_a,
float const* mat_b, cudaStream_t stream) {
if (num_experts == 256 && hidden_dim == 3072) {
Fp32LoopUnroller<InputT, 256, 3072, 1, FP32_MAX_TOKENS>::unroll(
num_tokens, output, mat_a, mat_b, stream);
} else if (num_experts == 128 && hidden_dim == 6144) {
Fp32LoopUnroller<InputT, 128, 6144, 1, FP32_MAX_TOKENS>::unroll(
num_tokens, output, mat_a, mat_b, stream);
} else {
throw std::invalid_argument(
"fp32_router_gemm: unsupported (hidden_dim, num_experts) pair");
}
}
void fp32_router_gemm(
torch::stable::Tensor& output, // [num_tokens, num_experts]
torch::stable::Tensor const& mat_a, // [num_tokens, hidden_dim]
@@ -85,10 +108,10 @@ void fp32_router_gemm(
STD_TORCH_CHECK(
mat_a.size(1) == mat_b.size(1),
"fp32_router_gemm: mat_a and mat_b must have the same hidden_dim");
STD_TORCH_CHECK(hidden_dim == FP32_HIDDEN_DIM,
"fp32_router_gemm: expected hidden_dim=3072");
STD_TORCH_CHECK(num_experts == FP32_NUM_EXPERTS,
"fp32_router_gemm: expected num_experts=256");
STD_TORCH_CHECK(
fp32_router_gemm_supported(hidden_dim, num_experts),
"fp32_router_gemm: supported (hidden_dim, num_experts) pairs are "
"(3072, 256) and (6144, 128)");
STD_TORCH_CHECK(num_tokens <= FP32_MAX_TOKENS,
"fp32_router_gemm: num_tokens must be in [0, 32]");
STD_TORCH_CHECK(
@@ -113,12 +136,13 @@ void fp32_router_gemm(
if (mat_a.scalar_type() == torch::headeronly::ScalarType::BFloat16) {
auto const* mat_a_ptr =
reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr());
Fp32LoopUnroller<__nv_bfloat16, 1, FP32_MAX_TOKENS>::unroll(
num_tokens, out_ptr, mat_a_ptr, mat_b_ptr, stream);
dispatchFp32RouterGemm<__nv_bfloat16>(num_experts, hidden_dim, num_tokens,
out_ptr, mat_a_ptr, mat_b_ptr,
stream);
} else {
auto const* mat_a_ptr = reinterpret_cast<float const*>(mat_a.data_ptr());
Fp32LoopUnroller<float, 1, FP32_MAX_TOKENS>::unroll(
num_tokens, out_ptr, mat_a_ptr, mat_b_ptr, stream);
dispatchFp32RouterGemm<float>(num_experts, hidden_dim, num_tokens, out_ptr,
mat_a_ptr, mat_b_ptr, stream);
}
}
@@ -0,0 +1,635 @@
/*
* SPDX-License-Identifier: Apache-2.0
* SPDX-FileCopyrightText: Copyright contributors to the vLLM project
*
* Horizontally-fused MiniMax-M3 attention pre-processing kernel.
*
* Replaces the per-token Python sequence in
* ``MiniMaxM3SparseAttention.forward`` / ``MiniMaxM3Attention.forward``:
*
* q = q_norm(q); k = k_norm(k); q, k = rotary_emb(pos, q, k)
* index_q = index_q_norm(index_q); index_k = index_k_norm(index_k)
* index_q, index_k = rotary_emb(pos, index_q, index_k)
* _insert_kv(k, v, index_k)
*
* All branches share head_dim=128 and the *same* partial-NeoX RoPE table
* (``rotary_dim`` rotated, the trailing dims pass through). The four norms
* are Gemma-style RMSNorm (``x * rsqrt(mean(x^2)+eps) * (1 + weight)``) with
* independent weights.
*
* Everything lives in a single fused ``qkv`` tensor. The sparse layer's
* fused projection (MinimaxM3QKVParallelLinearWithIndexer) emits, per token::
*
* [ q | k | v | index_q | index_k ] (the "5 results")
*
* while the dense layer emits just ``[ q | k | v ]``. The kernel reads the
* index branch straight out of that packed row -- no separate index tensors.
*
* One kernel, one grid; each warp owns one (token, head-slot) pair. Slot
* enumeration per token:
* [0, nq) Q heads -> norm(q_w) + RoPE, write
* qkv [nq, nq+nkv) K heads -> norm(k_w) + RoPE, write
* qkv
* (+ insert into key cache)
* [nq+nkv, nq+2*nkv) V heads -> insert into value cache
* IQ heads (niq) -> norm(iq_w) + RoPE, write iq
* IK (1) -> norm(ik_w) + RoPE
* (+ insert into index cache)
*
* The IQ/IK warps address the index_q/index_k sub-blocks *inside* qkv at the
* fixed physical offsets (nq+2*nkv)*128 and (nq+2*nkv+niq)*128.
*
* Dense vs sparse is a compile-time choice via the ``kIsSparse``/``kInsertKV``
* template bools (3 instantiations: dense <false,false>, sparse-profiling
* <true,false>, sparse-serving <true,true>), so the index slots, the V slots
* and the cache inserts fold away entirely on paths that don't use them. The
* dense layer passes no caches/index: norm+RoPE happens in place and the
* generic ``Attention`` layer owns the cache write.
*
* Q/K and (sparse) index_q/index_k are all rewritten in place inside the fused
* ``qkv`` tensor. Caches (bf16) are scatter-written by slot.
*/
#include <cmath>
#include <cuda_runtime.h>
#include <type_traits>
#include "torch_utils.h"
#include "../cuda_compat.h"
#include "../type_convert.cuh"
#include "dispatch_utils.h"
#ifndef FINAL_MASK
#ifdef USE_ROCM
#define FINAL_MASK 0xffffffffffffffffULL
#else
#define FINAL_MASK 0xffffffffu
#endif
#endif
namespace vllm {
namespace minimax_m3_fused_ops {
namespace {
inline int getSMVersion() {
auto* props = get_device_prop();
return props->major * 10 + props->minor;
}
} // namespace
// ────────────────────────────────────────────────────────────────────────────
// Constants (hard-coded for MiniMax-M3-preview).
// ────────────────────────────────────────────────────────────────────────────
constexpr int kHeadDim = 128;
constexpr int kNumLanes = 32;
constexpr int kElemsPerLane = kHeadDim / kNumLanes; // 4
// ────────────────────────────────────────────────────────────────────────────
// Helpers
// ────────────────────────────────────────────────────────────────────────────
__device__ __forceinline__ float warpReduceSum(float val) {
#pragma unroll
for (int mask = 16; mask > 0; mask >>= 1) {
val += __shfl_xor_sync(FINAL_MASK, val, mask, 32);
}
return val;
}
// Gemma RMSNorm over the full head (no-op when ``weight == nullptr``), rounded
// back to scalar_t like the materialized unfused norm output, followed by
// partial NeoX RoPE on the leading ``rotary_dim`` dims. Each lane owns
// ``kElemsPerLane`` contiguous dims [laneId*4, laneId*4+4).
template <typename scalar_t>
__device__ __forceinline__ void normAndRope(
float (&elems)[kElemsPerLane], int const laneId, float const eps,
scalar_t const* __restrict__ weight, // [kHeadDim] or nullptr (no norm)
bool const do_rope, int const rotary_dim,
scalar_t const* __restrict__ cos_ptr, // cos_sin_cache + pos*rotary_dim
bool const apply_norm) {
// ── Gemma RMSNorm: x * rsqrt(mean(x^2)+eps) * (1 + w) ──────────────────
if (apply_norm) {
float sumsq = 0.0f;
#pragma unroll
for (int i = 0; i < kElemsPerLane; i++) sumsq += elems[i] * elems[i];
sumsq = warpReduceSum(sumsq);
float const rms_rcp = rsqrtf(sumsq / static_cast<float>(kHeadDim) + eps);
#pragma unroll
for (int i = 0; i < kElemsPerLane; i++) {
int const dim = laneId * kElemsPerLane + i;
float const w = 1.0f + static_cast<float>(weight[dim]);
elems[i] = elems[i] * rms_rcp * w;
}
}
// ── Partial NeoX RoPE on dims [0, rotary_dim) ──────────────────────────
// half = rotary_dim/2. Pair (i, i+half) for i in [0, half). Lane L owns
// dims [4L, 4L+4); since half is a multiple of 4, a lane lies wholly in the
// first half (own=x[i]) or second half (own=x[i+half]); its partner lives
// ``half/4`` lanes away (XOR with that distance).
if (do_rope) {
int const half = rotary_dim / 2;
int const dim0 = laneId * kElemsPerLane;
bool const in_rope = dim0 < rotary_dim;
int const lane_xor = half / kElemsPerLane; // partner-lane distance
float partner[kElemsPerLane];
#pragma unroll
for (int i = 0; i < kElemsPerLane; i++) {
partner[i] = __shfl_xor_sync(FINAL_MASK, elems[i], lane_xor, 32);
}
if (in_rope) {
bool const first_half = dim0 < half;
int const i_base = first_half ? dim0 : (dim0 - half); // cos/sin index
scalar_t const* sin_ptr = cos_ptr + half;
#pragma unroll
for (int i = 0; i < kElemsPerLane; i++) {
float const c = static_cast<float>(cos_ptr[i_base + i]);
float const s = static_cast<float>(sin_ptr[i_base + i]);
if (first_half) {
elems[i] = elems[i] * c - partner[i] * s;
} else {
elems[i] = elems[i] * c + partner[i] * s;
}
}
}
}
}
// Load 4 contiguous bf16 -> 4 fp32 registers.
template <typename scalar_t>
__device__ __forceinline__ void loadElems(scalar_t const* __restrict__ src,
float (&elems)[kElemsPerLane]) {
using Converter = vllm::_typeConvert<scalar_t>;
uint2 v = *reinterpret_cast<uint2 const*>(src);
auto const* p =
reinterpret_cast<typename Converter::packed_hip_type const*>(&v);
#pragma unroll
for (int i = 0; i < kElemsPerLane / 2; i++) {
float2 f2 = Converter::convert(p[i]);
elems[2 * i] = f2.x;
elems[2 * i + 1] = f2.y;
}
}
// Store 4 fp32 registers -> 4 contiguous bf16.
template <typename scalar_t>
__device__ __forceinline__ void storeElems(
scalar_t* __restrict__ dst, float const (&elems)[kElemsPerLane]) {
using Converter = vllm::_typeConvert<scalar_t>;
uint2 v;
auto* p = reinterpret_cast<typename Converter::packed_hip_type*>(&v);
#pragma unroll
for (int i = 0; i < kElemsPerLane / 2; i++) {
p[i] = Converter::convert(make_float2(elems[2 * i], elems[2 * i + 1]));
}
*reinterpret_cast<uint2*>(dst) = v;
}
// ────────────────────────────────────────────────────────────────────────────
// Kernel
// ────────────────────────────────────────────────────────────────────────────
// Grid: 1D, ceil(num_tokens * slots_per_token / warps_per_block).
// Each warp = one (token, slot).
//
// `kIsSparse` and `kInsertKV` are compile-time template bools, so all the
// branch decisions that distinguish the dense layer from the sparse layer
// (index slots, KV/index inserts, V slots) fold away per instantiation.
// Three instantiations are built: dense <false,false>, sparse-profiling
// <true,false> and sparse-serving <true,true>. Slots per token:
// Q : nq (always — norm+RoPE)
// K : nkv (always — norm+RoPE; +K-cache insert)
// V : nkv only if kInsertKV (V-cache insert; no warps in dense)
// IQ: niq only if kIsSparse (norm+RoPE)
// IK: 1 only if kIsSparse (norm+RoPE; +index-cache insert)
template <typename scalar_t, bool kIsSparse, bool kInsertKV>
__global__ void fusedMiniMaxM3QNormRopeKVInsertKernel(
scalar_t* __restrict__ qkv, // [N, qkv_row] in/out (packs index if sparse)
scalar_t* __restrict__ q_out, // [N, nq*128] contiguous, or nullptr
scalar_t* __restrict__ index_q_out, // [N, niq*128] contiguous, or nullptr
scalar_t const* __restrict__ q_norm_w,
scalar_t const* __restrict__ k_norm_w,
scalar_t const* __restrict__ iq_norm_w,
scalar_t const* __restrict__ ik_norm_w,
scalar_t const* __restrict__ cos_sin_cache, // [max_pos, rotary_dim]
int64_t const* __restrict__ positions, // [N] i64
int64_t const* __restrict__ slot_mapping, // main K/V slots or nullptr
int64_t const* __restrict__ index_slot_mapping, // index K slots/nullptr
scalar_t* __restrict__ kv_cache, // [nb,2,bs,nkv,128] or nullptr
scalar_t* __restrict__ index_cache, // [nb*bs, 128] or nullptr
float const eps, int const rotary_dim, int const num_tokens, int const nq,
int const nkv, int const niq, int const block_size,
// kv_cache strides (in elements) for logical shape [nb, 2, bs, nkv, 128].
// The head_dim (last) dim is always innermost-contiguous (stride 1), so the
// NHD/HND layout choice is fully captured by these four strides: NHD keeps
// s_token < s_head, HND swaps them. dim_base addresses head_dim directly.
int64_t const kv_s_block, int64_t const kv_s_kv, int64_t const kv_s_token,
int64_t const kv_s_head) {
#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)
// _typeConvert<BFloat16> is unavailable on pre-Ampere; the M3 kernel only
// runs with bf16/fp16 inputs in practice. Discard the bf16 body there.
if constexpr (std::is_same_v<scalar_t, c10::BFloat16>) {
return;
} else {
#endif
int const warpsPerBlock = blockDim.x / 32;
int const laneId = threadIdx.x % 32;
int const globalWarpIdx = blockIdx.x * warpsPerBlock + (threadIdx.x / 32);
// Slot layout (compile-time gated: dense has neither V nor index slots).
int const v_slots = kInsertKV ? nkv : 0;
int const idx_slots = kIsSparse ? niq + 1 : 0;
int const slots_per_token = nq + nkv + v_slots + idx_slots;
int const tokenIdx = globalWarpIdx / slots_per_token;
int const slot = globalWarpIdx % slots_per_token;
if (tokenIdx >= num_tokens) return;
// Slot boundaries.
int const k_begin = nq;
int const v_begin = nq + nkv; // valid only when kInsertKV
int const iq_begin = nq + nkv + v_slots; // index block start
int const ik_slot = iq_begin + niq; // valid only when kIsSparse
bool const isQ = slot < k_begin;
bool const isK = slot >= k_begin && slot < v_begin;
bool isV = false;
if constexpr (kInsertKV) isV = slot >= v_begin && slot < v_begin + nkv;
bool isIQ = false, isIK = false;
if constexpr (kIsSparse) {
isIQ = slot >= iq_begin && slot < ik_slot;
isIK = slot == ik_slot;
}
int const dim_base = laneId * kElemsPerLane;
// Physical row width of qkv: the dense layer packs [q|k|v]; the sparse
// layer additionally packs [index_q (niq heads) | index_k (1 head)].
int const qkv_row = (nq + 2 * nkv + (kIsSparse ? (niq + 1) : 0)) * kHeadDim;
// ── Resolve source pointer + per-branch parameters. ────────────────────
scalar_t* row_ptr = nullptr; // in-place output location
scalar_t const* norm_w = nullptr; // nullptr -> skip norm (V)
bool do_rope = true;
int head = 0; // kv head index for inserts
if (isQ) {
row_ptr =
qkv + static_cast<int64_t>(tokenIdx) * qkv_row + slot * kHeadDim;
norm_w = q_norm_w;
} else if (isK) {
head = slot - k_begin;
row_ptr =
qkv + static_cast<int64_t>(tokenIdx) * qkv_row + slot * kHeadDim;
norm_w = k_norm_w;
} else if (isV) {
// qkv V section starts at slot index (nq + nkv): slot * kHeadDim is the
// correct in-tensor offset.
head = slot - v_begin;
row_ptr =
qkv + static_cast<int64_t>(tokenIdx) * qkv_row + slot * kHeadDim;
norm_w = nullptr; // V: no norm, no rope
do_rope = false;
} else if (isIQ) {
// index_q sub-block lives at physical offset (nq+2*nkv)*128 in qkv.
int const ih = slot - iq_begin;
row_ptr = qkv + static_cast<int64_t>(tokenIdx) * qkv_row +
(nq + 2 * nkv + ih) * kHeadDim;
norm_w = iq_norm_w;
} else { // isIK -- single shared index key at (nq+2*nkv+niq)*128.
row_ptr = qkv + static_cast<int64_t>(tokenIdx) * qkv_row +
(nq + 2 * nkv + niq) * kHeadDim;
norm_w = ik_norm_w;
}
// Store destination. Q and index_q are gathered into dedicated contiguous
// output buffers (when provided) so the downstream SM100 sparse kernel's
// flat TMA descriptor can address them as [tokens*heads, head_dim]; this
// folds the de-interleaving into the store the kernel already does, instead
// of a separate q.contiguous() copy. Everything else stays in place.
scalar_t* store_ptr = row_ptr;
if (isQ && q_out != nullptr) {
store_ptr = q_out + static_cast<int64_t>(tokenIdx) * nq * kHeadDim +
slot * kHeadDim;
} else if (isIQ && index_q_out != nullptr) {
store_ptr = index_q_out +
static_cast<int64_t>(tokenIdx) * niq * kHeadDim +
(slot - iq_begin) * kHeadDim;
}
// PDL: wait for the predecessor kernel (the qkv-projection GEMM that
// produces ``qkv``) to finish before touching any global memory. No-op
// when PDL is not enabled on the launch. The CUDA runtime wrapper emits
// the griddepcontrol.wait PTX with the required memory clobber internally.
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
cudaGridDependencySynchronize();
#endif
// ── Load -> norm+rope (fp32) -> store back in place. ───────────────────
float elems[kElemsPerLane];
loadElems<scalar_t>(row_ptr + dim_base, elems);
if (!isV) {
int64_t const pos = positions[tokenIdx];
scalar_t const* cos_ptr = cos_sin_cache + pos * rotary_dim;
normAndRope<scalar_t>(elems, laneId, eps, norm_w, do_rope, rotary_dim,
cos_ptr, /*apply_norm=*/norm_w != nullptr);
storeElems<scalar_t>(store_ptr + dim_base, elems);
}
// ── Cache inserts (sparse serving only). ───────────────────────────────
if constexpr (kInsertKV) {
// Guard (not early-return) so every thread reaches the PDL trigger below.
int64_t const sm = (isK || isV)
? slot_mapping[tokenIdx]
: (isIK ? index_slot_mapping[tokenIdx] : -1);
if (sm >= 0) { // skip padded / unscheduled tokens
if (isIK) {
scalar_t* dst = index_cache + sm * kHeadDim + dim_base;
storeElems<scalar_t>(dst, elems);
} else if (isK || isV) {
// kv_cache logical shape [num_blocks, 2, block_size, nkv, head_dim].
// Paging is logical (block = sm/block_size, token = sm%block_size);
// the physical NHD/HND layout is honoured via the passed strides.
int64_t const b = sm / block_size;
int64_t const t = sm % block_size;
int const kv = isK ? 0 : 1;
int64_t const off =
b * kv_s_block + kv * kv_s_kv + t * kv_s_token + head * kv_s_head;
storeElems<scalar_t>(kv_cache + off + dim_base, elems);
}
}
}
// PDL: signal that this kernel is done so a dependent successor may launch
// early. No-op when PDL is not enabled on the launch.
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
cudaTriggerProgrammaticLaunchCompletion();
#endif
#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)
}
#endif
}
// ────────────────────────────────────────────────────────────────────────────
// Launch wrapper
// ────────────────────────────────────────────────────────────────────────────
template <typename scalar_t>
void launchFusedMiniMaxM3(scalar_t* qkv, scalar_t* q_out, scalar_t* index_q_out,
scalar_t const* q_norm_w, scalar_t const* k_norm_w,
scalar_t const* iq_norm_w, scalar_t const* ik_norm_w,
scalar_t const* cos_sin_cache,
int64_t const* positions, int64_t const* slot_mapping,
int64_t const* index_slot_mapping, scalar_t* kv_cache,
scalar_t* index_cache, float const eps,
int const rotary_dim, int const num_tokens,
int const nq, int const nkv, int const niq,
int const block_size, int64_t const kv_s_block,
int64_t const kv_s_kv, int64_t const kv_s_token,
int64_t const kv_s_head, bool const has_index,
bool const insert_kv, cudaStream_t stream) {
// Slot count must match the kernel's compile-time gating.
int const v_slots = insert_kv ? nkv : 0;
int const idx_slots = has_index ? niq + 1 : 0;
int const slots_per_token = nq + nkv + v_slots + idx_slots;
constexpr int kBlockSize = 256;
constexpr int kWarpsPerBlock = kBlockSize / 32;
int64_t const total_warps =
static_cast<int64_t>(num_tokens) * slots_per_token;
int const grid =
static_cast<int>((total_warps + kWarpsPerBlock - 1) / kWarpsPerBlock);
if (grid == 0) return;
#ifndef USE_ROCM
// PDL: enable programmatic stream serialization whenever the hardware
// supports it (SM90+). On pre-Hopper GPUs the attribute is unavailable, so
// leave numAttrs = 0 and launch as a regular kernel via cudaLaunchKernelEx.
static int const sm_version = getSMVersion();
cudaLaunchConfig_t config;
config.gridDim = dim3(grid);
config.blockDim = dim3(kBlockSize);
config.dynamicSmemBytes = 0;
config.stream = stream;
cudaLaunchAttribute attrs[1];
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
attrs[0].val.programmaticStreamSerializationAllowed = 1;
config.attrs = attrs;
config.numAttrs = (sm_version >= 90) ? 1 : 0;
#define LAUNCH(IS_SPARSE, INSERT) \
cudaLaunchKernelEx( \
&config, \
fusedMiniMaxM3QNormRopeKVInsertKernel<scalar_t, IS_SPARSE, INSERT>, \
qkv, q_out, index_q_out, q_norm_w, k_norm_w, iq_norm_w, ik_norm_w, \
cos_sin_cache, positions, slot_mapping, index_slot_mapping, kv_cache, \
index_cache, eps, rotary_dim, num_tokens, nq, nkv, niq, block_size, \
kv_s_block, kv_s_kv, kv_s_token, kv_s_head)
#else
// ROCm: standard kernel launch syntax (no PDL/stream serialization).
// clang-format off
#define LAUNCH(IS_SPARSE, INSERT) \
fusedMiniMaxM3QNormRopeKVInsertKernel<scalar_t, IS_SPARSE, INSERT> \
<<<grid, kBlockSize, 0, stream>>>( \
qkv, q_out, index_q_out, q_norm_w, k_norm_w, iq_norm_w, \
ik_norm_w, cos_sin_cache, positions, slot_mapping, \
index_slot_mapping, kv_cache, index_cache, eps, rotary_dim, \
num_tokens, nq, nkv, niq, block_size, kv_s_block, kv_s_kv, \
kv_s_token, kv_s_head)
// clang-format on
#endif
if (has_index) {
if (insert_kv) {
LAUNCH(true, true); // sparse serving
} else {
LAUNCH(true, false); // sparse profiling
}
} else {
// Dense layer: never has an index branch and never inserts here (the
// generic Attention layer owns the KV insert).
LAUNCH(false, false);
}
#undef LAUNCH
}
} // namespace minimax_m3_fused_ops
} // namespace vllm
// ────────────────────────────────────────────────────────────────────────────
// Torch op wrapper
// ────────────────────────────────────────────────────────────────────────────
void fused_minimax_m3_qknorm_rope_kv_insert(
torch::stable::Tensor& qkv, // [N, qkv_row] (packs index if sparse)
torch::stable::Tensor const& q_norm_weight, // [128]
torch::stable::Tensor const& k_norm_weight, // [128]
torch::stable::Tensor const& cos_sin_cache, // [max_pos, rotary_dim]
torch::stable::Tensor const& positions, // [N] i64
int64_t num_heads, int64_t num_kv_heads, int64_t rotary_dim, double eps,
std::optional<torch::stable::Tensor> index_q_norm_weight, // [128]
std::optional<torch::stable::Tensor> index_k_norm_weight, // [128]
int64_t num_index_heads, // niq; 0 => dense
std::optional<torch::stable::Tensor> slot_mapping, // [N] i64
std::optional<torch::stable::Tensor> index_slot_mapping, // [N] i64
std::optional<torch::stable::Tensor> kv_cache, // [nb,2,bs,nkv,128]
std::optional<torch::stable::Tensor> index_cache, // [nb,bs,128]
int64_t block_size,
std::optional<torch::stable::Tensor> q_out, // [N, nq*128] contiguous
std::optional<torch::stable::Tensor>
index_q_out) { // [N, niq*128] contiguous
STD_TORCH_CHECK(qkv.is_cuda() && qkv.is_contiguous(),
"qkv must be contiguous CUDA");
STD_TORCH_CHECK(
positions.is_cuda() &&
positions.scalar_type() == torch::headeronly::ScalarType::Long,
"positions must be int64 CUDA");
STD_TORCH_CHECK(cos_sin_cache.is_cuda() && cos_sin_cache.is_contiguous(),
"cos_sin_cache must be contiguous CUDA");
STD_TORCH_CHECK(cos_sin_cache.scalar_type() == qkv.scalar_type(),
"cos_sin_cache dtype must match qkv");
STD_TORCH_CHECK(
cos_sin_cache.dim() == 2 && cos_sin_cache.size(1) == rotary_dim,
"cos_sin_cache shape [max_pos, rotary_dim]");
STD_TORCH_CHECK(q_norm_weight.scalar_type() == qkv.scalar_type() &&
k_norm_weight.scalar_type() == qkv.scalar_type(),
"q/k norm weight dtype must match qkv");
STD_TORCH_CHECK(
q_norm_weight.numel() == vllm::minimax_m3_fused_ops::kHeadDim &&
k_norm_weight.numel() == vllm::minimax_m3_fused_ops::kHeadDim,
"q/k norm weight must have 128 elements");
STD_TORCH_CHECK(rotary_dim > 0 && rotary_dim % 8 == 0 &&
rotary_dim <= vllm::minimax_m3_fused_ops::kHeadDim,
"rotary_dim must be a positive multiple of 8 and <= 128");
int const num_tokens = static_cast<int>(qkv.size(0));
int const nq = static_cast<int>(num_heads);
int const nkv = static_cast<int>(num_kv_heads);
int const niq = static_cast<int>(num_index_heads);
// The sparse layer packs the index branch ([index_q (niq heads) | index_k
// (1 head)]) right after [q|k|v] in the same row; the dense layer does not.
bool const has_index = niq > 0;
bool const insert_kv = kv_cache.has_value();
int const kHeadDim = vllm::minimax_m3_fused_ops::kHeadDim;
int const expected_row =
(nq + 2 * nkv + (has_index ? niq + 1 : 0)) * kHeadDim;
STD_TORCH_CHECK(qkv.size(1) == expected_row,
"qkv last dim must be (num_heads + 2*num_kv_heads"
" + num_index_heads + 1) * 128 for sparse, "
"(num_heads + 2*num_kv_heads) * 128 for dense");
// Only the sparse layer inserts here (dense lets the generic Attention layer
// own the KV write); there is no dense+insert kernel instantiation.
STD_TORCH_CHECK(
!insert_kv || has_index,
"insert mode (kv_cache) requires the index branch (sparse layer)");
if (has_index) {
STD_TORCH_CHECK(
index_q_norm_weight.has_value() && index_k_norm_weight.has_value(),
"index branch requires both index norm weights");
STD_TORCH_CHECK(index_q_norm_weight->scalar_type() == qkv.scalar_type() &&
index_k_norm_weight->scalar_type() == qkv.scalar_type(),
"index norm weights dtype must match qkv");
STD_TORCH_CHECK(index_q_norm_weight->numel() == kHeadDim &&
index_k_norm_weight->numel() == kHeadDim,
"index norm weights must have 128 elements");
}
// kv_cache strides (logical shape [nb, 2, bs, nkv, head_dim]). Read straight
// off the tensor so the kernel honours whatever physical layout the attention
// backend allocated (NHD: stride order (0,1,2,3,4); HND: (0,1,3,2,4)). No new
// op argument is needed -- the strides ride along with the tensor itself.
int64_t kv_s_block = 0, kv_s_kv = 0, kv_s_token = 0, kv_s_head = 0;
torch::stable::Tensor const* effective_index_slot_mapping = nullptr;
if (insert_kv) {
STD_TORCH_CHECK(
slot_mapping.has_value() && slot_mapping->is_cuda() &&
slot_mapping->scalar_type() == torch::headeronly::ScalarType::Long,
"insert mode requires int64 CUDA slot_mapping");
STD_TORCH_CHECK(
!index_slot_mapping.has_value() ||
(index_slot_mapping->is_cuda() &&
index_slot_mapping->scalar_type() ==
torch::headeronly::ScalarType::Long &&
index_slot_mapping->numel() == slot_mapping->numel()),
"index_slot_mapping must be int64 CUDA with slot_mapping length");
STD_TORCH_CHECK(kv_cache->scalar_type() == qkv.scalar_type(),
"kv_cache dtype must match qkv (bf16 cache only)");
STD_TORCH_CHECK(index_cache.has_value() &&
index_cache->scalar_type() == qkv.scalar_type(),
"insert mode requires matching index_cache");
STD_TORCH_CHECK(kv_cache->dim() == 5 && kv_cache->stride(4) == 1,
"kv_cache must be [nb,2,bs,nkv,head_dim] with contiguous "
"head_dim (stride(4)==1)");
kv_s_block = kv_cache->stride(0);
kv_s_kv = kv_cache->stride(1);
kv_s_token = kv_cache->stride(2);
kv_s_head = kv_cache->stride(3);
effective_index_slot_mapping = index_slot_mapping.has_value()
? &index_slot_mapping.value()
: &slot_mapping.value();
}
// Optional contiguous gather targets: when given, the normed/roped q (and
// index_q) are written here instead of in place, so callers avoid a separate
// .contiguous() copy. index_q_out only makes sense on the sparse path.
if (q_out.has_value()) {
STD_TORCH_CHECK(
q_out->is_cuda() && q_out->is_contiguous() &&
q_out->scalar_type() == qkv.scalar_type(),
"q_out must be a contiguous CUDA tensor matching qkv dtype");
STD_TORCH_CHECK(
q_out->numel() == static_cast<int64_t>(num_tokens) * nq * kHeadDim,
"q_out must have num_tokens * num_heads * 128 elements");
}
if (index_q_out.has_value()) {
STD_TORCH_CHECK(
has_index,
"index_q_out requires the index branch (num_index_heads > 0)");
STD_TORCH_CHECK(
index_q_out->is_cuda() && index_q_out->is_contiguous() &&
index_q_out->scalar_type() == qkv.scalar_type(),
"index_q_out must be a contiguous CUDA tensor matching qkv dtype");
STD_TORCH_CHECK(index_q_out->numel() ==
static_cast<int64_t>(num_tokens) * niq * kHeadDim,
"index_q_out must have num_tokens * num_index_heads * 128 "
"elements");
}
const torch::stable::accelerator::DeviceGuard device_guard(
qkv.get_device_index());
auto stream = get_current_cuda_stream(qkv.get_device_index());
VLLM_STABLE_DISPATCH_HALF_TYPES(
qkv.scalar_type(), "fused_minimax_m3_qknorm_rope_kv_insert", [&] {
using st = scalar_t;
vllm::minimax_m3_fused_ops::launchFusedMiniMaxM3<st>(
reinterpret_cast<st*>(qkv.data_ptr()),
q_out.has_value() ? reinterpret_cast<st*>(q_out->data_ptr())
: nullptr,
index_q_out.has_value()
? reinterpret_cast<st*>(index_q_out->data_ptr())
: nullptr,
reinterpret_cast<st const*>(q_norm_weight.data_ptr()),
reinterpret_cast<st const*>(k_norm_weight.data_ptr()),
has_index
? reinterpret_cast<st const*>(index_q_norm_weight->data_ptr())
: nullptr,
has_index
? reinterpret_cast<st const*>(index_k_norm_weight->data_ptr())
: nullptr,
reinterpret_cast<st const*>(cos_sin_cache.data_ptr()),
reinterpret_cast<int64_t const*>(positions.data_ptr()),
insert_kv
? reinterpret_cast<int64_t const*>(slot_mapping->data_ptr())
: nullptr,
insert_kv ? reinterpret_cast<int64_t const*>(
effective_index_slot_mapping->data_ptr())
: nullptr,
insert_kv ? reinterpret_cast<st*>(kv_cache->data_ptr()) : nullptr,
(insert_kv && has_index)
? reinterpret_cast<st*>(index_cache->data_ptr())
: nullptr,
static_cast<float>(eps), static_cast<int>(rotary_dim), num_tokens,
nq, nkv, niq, static_cast<int>(block_size), kv_s_block, kv_s_kv,
kv_s_token, kv_s_head, has_index, insert_kv, stream);
});
}
+20 -1
View File
@@ -281,6 +281,24 @@ minimax_allreduce_rms_qk(torch::stable::Tensor qkv,
int64_t const nranks, double const eps);
#endif
// Horizontally-fused MiniMax-M3 QK-norm + partial NeoX RoPE (+ optional KV /
// index-cache insert). Dense layer: norm+RoPE only; sparse layer: also packs
// the index branch and scatters k/v/index_k into their paged caches.
void fused_minimax_m3_qknorm_rope_kv_insert(
torch::stable::Tensor& qkv, torch::stable::Tensor const& q_norm_weight,
torch::stable::Tensor const& k_norm_weight,
torch::stable::Tensor const& cos_sin_cache,
torch::stable::Tensor const& positions, int64_t num_heads,
int64_t num_kv_heads, int64_t rotary_dim, double eps,
std::optional<torch::stable::Tensor> index_q_norm_weight,
std::optional<torch::stable::Tensor> index_k_norm_weight,
int64_t num_index_heads, std::optional<torch::stable::Tensor> slot_mapping,
std::optional<torch::stable::Tensor> index_slot_mapping,
std::optional<torch::stable::Tensor> kv_cache,
std::optional<torch::stable::Tensor> index_cache, int64_t block_size,
std::optional<torch::stable::Tensor> q_out,
std::optional<torch::stable::Tensor> index_q_out);
// Sampler kernels (shared CUDA/ROCm)
void apply_repetition_penalties_(
torch::stable::Tensor& logits, const torch::stable::Tensor& prompt_mask,
@@ -346,7 +364,8 @@ void free_shared_buffer(int64_t buffer);
// Activation kernels (shared CUDA/ROCm)
void silu_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input);
void silu_and_mul_clamp(torch::stable::Tensor& out,
torch::stable::Tensor& input, double limit);
torch::stable::Tensor& input, double limit,
double alpha = 1.0, double beta = 0.0);
void mul_and_silu(torch::stable::Tensor& out, torch::stable::Tensor& input);
void gelu_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input);
void gelu_tanh_and_mul(torch::stable::Tensor& out,
+18 -2
View File
@@ -461,6 +461,18 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
"float eps) -> (Tensor, Tensor)");
#endif
// Horizontally-fused MiniMax-M3 QK-norm + partial NeoX RoPE + KV-insert.
ops.def(
"fused_minimax_m3_qknorm_rope_kv_insert("
"Tensor! qkv, Tensor q_norm_weight, Tensor k_norm_weight, "
"Tensor cos_sin_cache, Tensor positions, int num_heads, "
"int num_kv_heads, int rotary_dim, float eps, "
"Tensor? index_q_norm_weight, Tensor? index_k_norm_weight, "
"int num_index_heads, "
"Tensor? slot_mapping, Tensor? index_slot_mapping, "
"Tensor!? kv_cache, Tensor!? index_cache, "
"int block_size, Tensor!? q_out, Tensor!? index_q_out) -> ()");
// Apply repetition penalties to logits in-place.
ops.def(
"apply_repetition_penalties_(Tensor! logits, Tensor prompt_mask, "
@@ -488,9 +500,11 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
ops.def("mul_and_silu(Tensor! out, Tensor input) -> ()");
// SwiGLU activation with input clamping.
// alpha scales the sigmoid (gate * sigmoid(alpha * gate)); beta is added to
// the up half (up + beta). Defaults alpha=1.0, beta=0.0 give silu(gate)*up.
ops.def(
"silu_and_mul_with_clamp(Tensor! result, Tensor input, float limit) "
"-> ()");
"silu_and_mul_with_clamp(Tensor! result, Tensor input, float limit, "
"float alpha=1.0, float beta=0.0) -> ()");
// Activation function used in GeGLU with `none` approximation.
ops.def("gelu_and_mul(Tensor! out, Tensor input) -> ()");
@@ -679,6 +693,8 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
ops.impl("minimax_allreduce_rms", TORCH_BOX(&minimax_allreduce_rms));
ops.impl("minimax_allreduce_rms_qk", TORCH_BOX(&minimax_allreduce_rms_qk));
#endif
ops.impl("fused_minimax_m3_qknorm_rope_kv_insert",
TORCH_BOX(&fused_minimax_m3_qknorm_rope_kv_insert));
// Sampler kernels (shared CUDA/ROCm)
ops.impl("apply_repetition_penalties_",
+2 -1
View File
@@ -50,7 +50,8 @@ void rotary_embedding(torch::Tensor& positions, torch::Tensor& query,
void silu_and_mul(torch::Tensor& out, torch::Tensor& input);
void silu_and_mul_clamp(torch::Tensor& out, torch::Tensor& input, double limit);
void silu_and_mul_clamp(torch::Tensor& out, torch::Tensor& input, double limit,
double alpha = 1.0, double beta = 0.0);
void silu_and_mul_quant(torch::Tensor& out, torch::Tensor& input,
torch::Tensor& scale);