[Model] Add MiniMax M3 support (#45381)
Signed-off-by: youkaichao <youkaichao@gmail.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: functionstackx <47992694+functionstackx@users.noreply.github.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: functionstackx <47992694+functionstackx@users.noreply.github.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
co-authored by
OpenAI Codex
Isotr0py
Thien Tran
Bugen Zhao
Jee Jee Li
Roger Wang
functionstackx
Yongye Zhu
Jee Jee Li
parent
a3195fab7b
commit
0a1c5034f5
@@ -10,11 +10,20 @@
|
||||
|
||||
namespace vllm {
|
||||
|
||||
template <typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&),
|
||||
// `alpha` and `beta` are applied to opposite operands:
|
||||
// - alpha lives INSIDE the activation (the activated half): the gated
|
||||
// activation computes act_half * sigmoid(alpha * act_half).
|
||||
// - beta is added to the OTHER (non-activated) half before the multiply.
|
||||
// So the result is always ACT(act_half, alpha) * (other_half + beta).
|
||||
// Which half is which depends on `act_first` (see below). Defaults
|
||||
// alpha=1.0, beta=0.0 reproduce the plain SwiGLU/GeGLU behavior.
|
||||
template <typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&, const float),
|
||||
bool act_first, bool HAS_CLAMP>
|
||||
__device__ __forceinline__ scalar_t compute(const scalar_t& x,
|
||||
const scalar_t& y,
|
||||
const float limit) {
|
||||
const float limit,
|
||||
const float alpha,
|
||||
const float beta) {
|
||||
if constexpr (act_first) {
|
||||
scalar_t gate = x;
|
||||
scalar_t up = y;
|
||||
@@ -22,7 +31,9 @@ __device__ __forceinline__ scalar_t compute(const scalar_t& x,
|
||||
gate = (scalar_t)fminf((float)gate, limit);
|
||||
up = (scalar_t)fmaxf(fminf((float)up, limit), -limit);
|
||||
}
|
||||
return ACT_FN(gate) * up;
|
||||
// act_first: gate is the activated half -> alpha applies to gate;
|
||||
// beta is added to up (the non-activated half).
|
||||
return (scalar_t)(ACT_FN(gate, alpha) * ((float)up + beta));
|
||||
} else {
|
||||
scalar_t gate = x;
|
||||
scalar_t up = y;
|
||||
@@ -30,55 +41,68 @@ __device__ __forceinline__ scalar_t compute(const scalar_t& x,
|
||||
gate = (scalar_t)fmaxf(fminf((float)gate, limit), -limit);
|
||||
up = (scalar_t)fminf((float)up, limit);
|
||||
}
|
||||
return gate * ACT_FN(up);
|
||||
// !act_first: up is the activated half -> alpha applies to up;
|
||||
// beta is added to gate (the non-activated half).
|
||||
return (scalar_t)(((float)gate + beta) * ACT_FN(up, alpha));
|
||||
}
|
||||
}
|
||||
|
||||
template <typename packed_t, packed_t (*PACKED_ACT_FN)(const packed_t&),
|
||||
template <typename packed_t,
|
||||
packed_t (*PACKED_ACT_FN)(const packed_t&, const float),
|
||||
bool act_first, bool HAS_CLAMP>
|
||||
__device__ __forceinline__ packed_t packed_compute(const packed_t& x,
|
||||
const packed_t& y,
|
||||
const float limit) {
|
||||
const float limit,
|
||||
const float alpha,
|
||||
const float beta) {
|
||||
if constexpr (act_first) {
|
||||
packed_t gate = x;
|
||||
packed_t up = y;
|
||||
float2 u = cast_to_float2(up);
|
||||
if constexpr (HAS_CLAMP) {
|
||||
float2 g = cast_to_float2(gate);
|
||||
float2 u = cast_to_float2(up);
|
||||
g.x = fminf(g.x, limit);
|
||||
g.y = fminf(g.y, limit);
|
||||
u.x = fmaxf(fminf(u.x, limit), -limit);
|
||||
u.y = fmaxf(fminf(u.y, limit), -limit);
|
||||
gate = cast_to_packed<packed_t>(g);
|
||||
up = cast_to_packed<packed_t>(u);
|
||||
}
|
||||
return packed_mul(PACKED_ACT_FN(gate), up);
|
||||
// act_first: gate is the activated half -> alpha applies to gate;
|
||||
// beta is added to up (the non-activated half).
|
||||
float2 activated = cast_to_float2(PACKED_ACT_FN(gate, alpha));
|
||||
activated.x *= u.x + beta;
|
||||
activated.y *= u.y + beta;
|
||||
return cast_to_packed<packed_t>(activated);
|
||||
} else {
|
||||
packed_t gate = x;
|
||||
packed_t up = y;
|
||||
float2 g = cast_to_float2(gate);
|
||||
if constexpr (HAS_CLAMP) {
|
||||
float2 g = cast_to_float2(gate);
|
||||
float2 u = cast_to_float2(up);
|
||||
g.x = fmaxf(fminf(g.x, limit), -limit);
|
||||
g.y = fmaxf(fminf(g.y, limit), -limit);
|
||||
u.x = fminf(u.x, limit);
|
||||
u.y = fminf(u.y, limit);
|
||||
gate = cast_to_packed<packed_t>(g);
|
||||
up = cast_to_packed<packed_t>(u);
|
||||
}
|
||||
return packed_mul(gate, PACKED_ACT_FN(up));
|
||||
// !act_first: up is the activated half -> alpha applies to up;
|
||||
// beta is added to gate (the non-activated half).
|
||||
float2 activated = cast_to_float2(PACKED_ACT_FN(up, alpha));
|
||||
activated.x *= g.x + beta;
|
||||
activated.y *= g.y + beta;
|
||||
return cast_to_packed<packed_t>(activated);
|
||||
}
|
||||
}
|
||||
|
||||
// Activation and gating kernel template.
|
||||
template <typename scalar_t, typename packed_t,
|
||||
scalar_t (*ACT_FN)(const scalar_t&),
|
||||
packed_t (*PACKED_ACT_FN)(const packed_t&), bool act_first,
|
||||
bool use_vec, bool HAS_CLAMP, bool use_256b = false>
|
||||
scalar_t (*ACT_FN)(const scalar_t&, const float),
|
||||
packed_t (*PACKED_ACT_FN)(const packed_t&, const float),
|
||||
bool act_first, bool use_vec, bool HAS_CLAMP, bool use_256b = false>
|
||||
__global__ void act_and_mul_kernel(
|
||||
scalar_t* __restrict__ out, // [..., d]
|
||||
const scalar_t* __restrict__ input, // [..., 2, d]
|
||||
const int d, const float limit) {
|
||||
const int d, const float limit, const float alpha, const float beta) {
|
||||
const scalar_t* x_ptr = input + blockIdx.x * 2 * d;
|
||||
const scalar_t* y_ptr = x_ptr + d;
|
||||
scalar_t* out_ptr = out + blockIdx.x * d;
|
||||
@@ -105,7 +129,7 @@ __global__ void act_and_mul_kernel(
|
||||
for (int j = 0; j < pvec_t::NUM_ELTS; j++) {
|
||||
x.elts[j] =
|
||||
packed_compute<packed_t, PACKED_ACT_FN, act_first, HAS_CLAMP>(
|
||||
x.elts[j], y.elts[j], limit);
|
||||
x.elts[j], y.elts[j], limit, alpha, beta);
|
||||
}
|
||||
if constexpr (use_256b) {
|
||||
st256(x, &out_vec[i]);
|
||||
@@ -118,29 +142,34 @@ __global__ void act_and_mul_kernel(
|
||||
for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
|
||||
const scalar_t x = VLLM_LDG(&x_ptr[idx]);
|
||||
const scalar_t y = VLLM_LDG(&y_ptr[idx]);
|
||||
out_ptr[idx] =
|
||||
compute<scalar_t, ACT_FN, act_first, HAS_CLAMP>(x, y, limit);
|
||||
out_ptr[idx] = compute<scalar_t, ACT_FN, act_first, HAS_CLAMP>(
|
||||
x, y, limit, alpha, beta);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Gated activations take an `alpha` argument that scales the sigmoid input
|
||||
// (`x * sigmoid(alpha * x)`). alpha defaults to 1.0 at all call sites, which
|
||||
// is exactly SiLU; only the clamp path (silu_and_mul_with_clamp) passes a
|
||||
// non-default alpha. Activations that do not use alpha simply ignore it.
|
||||
template <typename T>
|
||||
__device__ __forceinline__ T silu_kernel(const T& x) {
|
||||
// x * sigmoid(x)
|
||||
return (T)(((float)x) / (1.0f + expf((float)-x)));
|
||||
__device__ __forceinline__ T silu_kernel(const T& x, const float alpha) {
|
||||
// x * sigmoid(alpha * x)
|
||||
return (T)(((float)x) / (1.0f + expf((float)-x * alpha)));
|
||||
}
|
||||
|
||||
template <typename packed_t>
|
||||
__device__ __forceinline__ packed_t packed_silu_kernel(const packed_t& val) {
|
||||
// x * sigmoid(x)
|
||||
__device__ __forceinline__ packed_t packed_silu_kernel(const packed_t& val,
|
||||
const float alpha) {
|
||||
// x * sigmoid(alpha * x)
|
||||
float2 fval = cast_to_float2(val);
|
||||
fval.x = fval.x / (1.0f + expf(-fval.x));
|
||||
fval.y = fval.y / (1.0f + expf(-fval.y));
|
||||
fval.x = fval.x / (1.0f + expf(-fval.x * alpha));
|
||||
fval.y = fval.y / (1.0f + expf(-fval.y * alpha));
|
||||
return cast_to_packed<packed_t>(fval);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__device__ __forceinline__ T gelu_kernel(const T& x) {
|
||||
__device__ __forceinline__ T gelu_kernel(const T& x, const float /*alpha*/) {
|
||||
// Equivalent to PyTorch GELU with 'none' approximation.
|
||||
// Refer to:
|
||||
// https://github.com/pytorch/pytorch/blob/8ac9b20d4b090c213799e81acf48a55ea8d437d6/aten/src/ATen/native/cuda/ActivationGeluKernel.cu#L36-L38
|
||||
@@ -150,7 +179,8 @@ __device__ __forceinline__ T gelu_kernel(const T& x) {
|
||||
}
|
||||
|
||||
template <typename packed_t>
|
||||
__device__ __forceinline__ packed_t packed_gelu_kernel(const packed_t& val) {
|
||||
__device__ __forceinline__ packed_t packed_gelu_kernel(const packed_t& val,
|
||||
const float /*alpha*/) {
|
||||
// Equivalent to PyTorch GELU with 'none' approximation.
|
||||
// Refer to:
|
||||
// https://github.com/pytorch/pytorch/blob/8ac9b20d4b090c213799e81acf48a55ea8d437d6/aten/src/ATen/native/cuda/ActivationGeluKernel.cu#L36-L38
|
||||
@@ -162,7 +192,8 @@ __device__ __forceinline__ packed_t packed_gelu_kernel(const packed_t& val) {
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__device__ __forceinline__ T gelu_tanh_kernel(const T& x) {
|
||||
__device__ __forceinline__ T gelu_tanh_kernel(const T& x,
|
||||
const float /*alpha*/) {
|
||||
// Equivalent to PyTorch GELU with 'tanh' approximation.
|
||||
// Refer to:
|
||||
// https://github.com/pytorch/pytorch/blob/8ac9b20d4b090c213799e81acf48a55ea8d437d6/aten/src/ATen/native/cuda/ActivationGeluKernel.cu#L25-L30
|
||||
@@ -176,7 +207,7 @@ __device__ __forceinline__ T gelu_tanh_kernel(const T& x) {
|
||||
|
||||
template <typename packed_t>
|
||||
__device__ __forceinline__ packed_t
|
||||
packed_gelu_tanh_kernel(const packed_t& val) {
|
||||
packed_gelu_tanh_kernel(const packed_t& val, const float /*alpha*/) {
|
||||
// Equivalent to PyTorch GELU with 'tanh' approximation.
|
||||
// Refer to:
|
||||
// https://github.com/pytorch/pytorch/blob/8ac9b20d4b090c213799e81acf48a55ea8d437d6/aten/src/ATen/native/cuda/ActivationGeluKernel.cu#L25-L30
|
||||
@@ -202,7 +233,7 @@ packed_gelu_tanh_kernel(const packed_t& val) {
|
||||
// clamped (max only) and up input is clamped (both sides) before the
|
||||
// activation function is applied.
|
||||
#define LAUNCH_ACTIVATION_GATE_KERNEL(KERNEL, PACKED_KERNEL, ACT_FIRST, \
|
||||
HAS_CLAMP, LIMIT) \
|
||||
HAS_CLAMP, LIMIT, ALPHA, BETA) \
|
||||
auto dtype = input.scalar_type(); \
|
||||
int d = input.size(-1) / 2; \
|
||||
int64_t num_tokens = input.numel() / input.size(-1); \
|
||||
@@ -230,7 +261,7 @@ packed_gelu_tanh_kernel(const packed_t& val) {
|
||||
PACKED_KERNEL<typename vllm::PackedTypeConverter<scalar_t>::Type>, \
|
||||
ACT_FIRST, true, HAS_CLAMP, true><<<grid, block, 0, stream>>>( \
|
||||
out.mutable_data_ptr<scalar_t>(), \
|
||||
input.const_data_ptr<scalar_t>(), d, LIMIT); \
|
||||
input.const_data_ptr<scalar_t>(), d, LIMIT, ALPHA, BETA); \
|
||||
}); \
|
||||
} else { \
|
||||
VLLM_STABLE_DISPATCH_FLOATING_TYPES(dtype, "act_and_mul_kernel", [&] { \
|
||||
@@ -240,7 +271,7 @@ packed_gelu_tanh_kernel(const packed_t& val) {
|
||||
PACKED_KERNEL<typename vllm::PackedTypeConverter<scalar_t>::Type>, \
|
||||
ACT_FIRST, true, HAS_CLAMP, false><<<grid, block, 0, stream>>>( \
|
||||
out.mutable_data_ptr<scalar_t>(), \
|
||||
input.const_data_ptr<scalar_t>(), d, LIMIT); \
|
||||
input.const_data_ptr<scalar_t>(), d, LIMIT, ALPHA, BETA); \
|
||||
}); \
|
||||
} \
|
||||
} else { \
|
||||
@@ -252,7 +283,7 @@ packed_gelu_tanh_kernel(const packed_t& val) {
|
||||
PACKED_KERNEL<typename vllm::PackedTypeConverter<scalar_t>::Type>, \
|
||||
ACT_FIRST, false, HAS_CLAMP><<<grid, block, 0, stream>>>( \
|
||||
out.mutable_data_ptr<scalar_t>(), input.const_data_ptr<scalar_t>(), \
|
||||
d, LIMIT); \
|
||||
d, LIMIT, ALPHA, BETA); \
|
||||
}); \
|
||||
}
|
||||
|
||||
@@ -260,14 +291,18 @@ void silu_and_mul(torch::stable::Tensor& out, // [..., d]
|
||||
torch::stable::Tensor& input) // [..., 2 * d]
|
||||
{
|
||||
LAUNCH_ACTIVATION_GATE_KERNEL(vllm::silu_kernel, vllm::packed_silu_kernel,
|
||||
true, false, 0.0f);
|
||||
true, false, 0.0f, 1.0f, 0.0f);
|
||||
}
|
||||
|
||||
void silu_and_mul_clamp(torch::stable::Tensor& out, // [..., d]
|
||||
torch::stable::Tensor& input, // [..., 2 * d]
|
||||
double limit) {
|
||||
double limit, double alpha, double beta) {
|
||||
// out = (gate.clamp(max=limit) * sigmoid(alpha * gate.clamp(max=limit)))
|
||||
// * (up.clamp(+-limit) + beta)
|
||||
// alpha=1.0, beta=0.0 reduce this to silu(gate) * up.
|
||||
LAUNCH_ACTIVATION_GATE_KERNEL(vllm::silu_kernel, vllm::packed_silu_kernel,
|
||||
true, true, (float)limit);
|
||||
true, true, (float)limit, (float)alpha,
|
||||
(float)beta);
|
||||
}
|
||||
|
||||
void mul_and_silu(torch::stable::Tensor& out, // [..., d]
|
||||
@@ -276,21 +311,22 @@ void mul_and_silu(torch::stable::Tensor& out, // [..., d]
|
||||
// The difference between mul_and_silu and silu_and_mul is that mul_and_silu
|
||||
// applies the silu to the latter half of the input.
|
||||
LAUNCH_ACTIVATION_GATE_KERNEL(vllm::silu_kernel, vllm::packed_silu_kernel,
|
||||
false, false, 0.0f);
|
||||
false, false, 0.0f, 1.0f, 0.0f);
|
||||
}
|
||||
|
||||
void gelu_and_mul(torch::stable::Tensor& out, // [..., d]
|
||||
torch::stable::Tensor& input) // [..., 2 * d]
|
||||
{
|
||||
LAUNCH_ACTIVATION_GATE_KERNEL(vllm::gelu_kernel, vllm::packed_gelu_kernel,
|
||||
true, false, 0.0f);
|
||||
true, false, 0.0f, 1.0f, 0.0f);
|
||||
}
|
||||
|
||||
void gelu_tanh_and_mul(torch::stable::Tensor& out, // [..., d]
|
||||
torch::stable::Tensor& input) // [..., 2 * d]
|
||||
{
|
||||
LAUNCH_ACTIVATION_GATE_KERNEL(
|
||||
vllm::gelu_tanh_kernel, vllm::packed_gelu_tanh_kernel, true, false, 0.0f);
|
||||
LAUNCH_ACTIVATION_GATE_KERNEL(vllm::gelu_tanh_kernel,
|
||||
vllm::packed_gelu_tanh_kernel, true, false,
|
||||
0.0f, 1.0f, 0.0f);
|
||||
}
|
||||
|
||||
namespace vllm {
|
||||
|
||||
@@ -175,49 +175,52 @@ void invokeFp32RouterGemm(float* output, InputT const* mat_a,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Explicit instantiations: M=1..32, E=256, H=3072, for both input types
|
||||
// Explicit instantiations: M=1..32, for both input types, for the supported
|
||||
// (E, H) pairs: (256, 3072) [MiniMax-M2/M2.5] and (128, 6144) [MiniMax-M3].
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#define INSTANTIATE(T, M) \
|
||||
template void invokeFp32RouterGemm<T, M, 256, 3072>( \
|
||||
float*, T const*, float const*, cudaStream_t);
|
||||
#define INSTANTIATE(T, M, E, H) \
|
||||
template void invokeFp32RouterGemm<T, M, E, H>(float*, T const*, \
|
||||
float const*, cudaStream_t);
|
||||
|
||||
#define INSTANTIATE_ALL(T) \
|
||||
INSTANTIATE(T, 1) \
|
||||
INSTANTIATE(T, 2) \
|
||||
INSTANTIATE(T, 3) \
|
||||
INSTANTIATE(T, 4) \
|
||||
INSTANTIATE(T, 5) \
|
||||
INSTANTIATE(T, 6) \
|
||||
INSTANTIATE(T, 7) \
|
||||
INSTANTIATE(T, 8) \
|
||||
INSTANTIATE(T, 9) \
|
||||
INSTANTIATE(T, 10) \
|
||||
INSTANTIATE(T, 11) \
|
||||
INSTANTIATE(T, 12) \
|
||||
INSTANTIATE(T, 13) \
|
||||
INSTANTIATE(T, 14) \
|
||||
INSTANTIATE(T, 15) \
|
||||
INSTANTIATE(T, 16) \
|
||||
INSTANTIATE(T, 17) \
|
||||
INSTANTIATE(T, 18) \
|
||||
INSTANTIATE(T, 19) \
|
||||
INSTANTIATE(T, 20) \
|
||||
INSTANTIATE(T, 21) \
|
||||
INSTANTIATE(T, 22) \
|
||||
INSTANTIATE(T, 23) \
|
||||
INSTANTIATE(T, 24) \
|
||||
INSTANTIATE(T, 25) \
|
||||
INSTANTIATE(T, 26) \
|
||||
INSTANTIATE(T, 27) \
|
||||
INSTANTIATE(T, 28) \
|
||||
INSTANTIATE(T, 29) \
|
||||
INSTANTIATE(T, 30) \
|
||||
INSTANTIATE(T, 31) \
|
||||
INSTANTIATE(T, 32)
|
||||
#define INSTANTIATE_ALL(T, E, H) \
|
||||
INSTANTIATE(T, 1, E, H) \
|
||||
INSTANTIATE(T, 2, E, H) \
|
||||
INSTANTIATE(T, 3, E, H) \
|
||||
INSTANTIATE(T, 4, E, H) \
|
||||
INSTANTIATE(T, 5, E, H) \
|
||||
INSTANTIATE(T, 6, E, H) \
|
||||
INSTANTIATE(T, 7, E, H) \
|
||||
INSTANTIATE(T, 8, E, H) \
|
||||
INSTANTIATE(T, 9, E, H) \
|
||||
INSTANTIATE(T, 10, E, H) \
|
||||
INSTANTIATE(T, 11, E, H) \
|
||||
INSTANTIATE(T, 12, E, H) \
|
||||
INSTANTIATE(T, 13, E, H) \
|
||||
INSTANTIATE(T, 14, E, H) \
|
||||
INSTANTIATE(T, 15, E, H) \
|
||||
INSTANTIATE(T, 16, E, H) \
|
||||
INSTANTIATE(T, 17, E, H) \
|
||||
INSTANTIATE(T, 18, E, H) \
|
||||
INSTANTIATE(T, 19, E, H) \
|
||||
INSTANTIATE(T, 20, E, H) \
|
||||
INSTANTIATE(T, 21, E, H) \
|
||||
INSTANTIATE(T, 22, E, H) \
|
||||
INSTANTIATE(T, 23, E, H) \
|
||||
INSTANTIATE(T, 24, E, H) \
|
||||
INSTANTIATE(T, 25, E, H) \
|
||||
INSTANTIATE(T, 26, E, H) \
|
||||
INSTANTIATE(T, 27, E, H) \
|
||||
INSTANTIATE(T, 28, E, H) \
|
||||
INSTANTIATE(T, 29, E, H) \
|
||||
INSTANTIATE(T, 30, E, H) \
|
||||
INSTANTIATE(T, 31, E, H) \
|
||||
INSTANTIATE(T, 32, E, H)
|
||||
|
||||
INSTANTIATE_ALL(float)
|
||||
INSTANTIATE_ALL(__nv_bfloat16)
|
||||
INSTANTIATE_ALL(float, 256, 3072)
|
||||
INSTANTIATE_ALL(__nv_bfloat16, 256, 3072)
|
||||
INSTANTIATE_ALL(float, 128, 6144)
|
||||
INSTANTIATE_ALL(__nv_bfloat16, 128, 6144)
|
||||
|
||||
#undef INSTANTIATE_ALL
|
||||
#undef INSTANTIATE
|
||||
|
||||
@@ -22,36 +22,42 @@ inline int getSMVersion() {
|
||||
|
||||
} // namespace
|
||||
|
||||
static constexpr int FP32_NUM_EXPERTS = 256;
|
||||
static constexpr int FP32_HIDDEN_DIM = 3072;
|
||||
static constexpr int FP32_MAX_TOKENS = 32;
|
||||
|
||||
// Supported (hidden_dim, num_experts) pairs (must match the instantiations in
|
||||
// fp32_router_gemm.cu): (3072, 256) for MiniMax-M2/M2.5, (6144, 128) for M3.
|
||||
static inline bool fp32_router_gemm_supported(int hidden_dim, int num_experts) {
|
||||
return (hidden_dim == 3072 && num_experts == 256) ||
|
||||
(hidden_dim == 6144 && num_experts == 128);
|
||||
}
|
||||
|
||||
// Forward declarations — 4 template params must match fp32_router_gemm.cu
|
||||
template <typename InputT, int kNumTokens, int kNumExperts, int kHiddenDim>
|
||||
void invokeFp32RouterGemm(float* output, InputT const* mat_a,
|
||||
float const* mat_b, cudaStream_t stream);
|
||||
|
||||
// LoopUnroller templated on InputT
|
||||
template <typename InputT, int kBegin, int kEnd>
|
||||
// LoopUnroller templated on InputT, kNumExperts and kHiddenDim
|
||||
template <typename InputT, int kNumExperts, int kHiddenDim, int kBegin,
|
||||
int kEnd>
|
||||
struct Fp32LoopUnroller {
|
||||
static void unroll(int num_tokens, float* output, InputT const* mat_a,
|
||||
float const* mat_b, cudaStream_t stream) {
|
||||
if (num_tokens == kBegin) {
|
||||
invokeFp32RouterGemm<InputT, kBegin, FP32_NUM_EXPERTS, FP32_HIDDEN_DIM>(
|
||||
invokeFp32RouterGemm<InputT, kBegin, kNumExperts, kHiddenDim>(
|
||||
output, mat_a, mat_b, stream);
|
||||
} else {
|
||||
Fp32LoopUnroller<InputT, kBegin + 1, kEnd>::unroll(num_tokens, output,
|
||||
mat_a, mat_b, stream);
|
||||
Fp32LoopUnroller<InputT, kNumExperts, kHiddenDim, kBegin + 1,
|
||||
kEnd>::unroll(num_tokens, output, mat_a, mat_b, stream);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <typename InputT, int kEnd>
|
||||
struct Fp32LoopUnroller<InputT, kEnd, kEnd> {
|
||||
template <typename InputT, int kNumExperts, int kHiddenDim, int kEnd>
|
||||
struct Fp32LoopUnroller<InputT, kNumExperts, kHiddenDim, kEnd, kEnd> {
|
||||
static void unroll(int num_tokens, float* output, InputT const* mat_a,
|
||||
float const* mat_b, cudaStream_t stream) {
|
||||
if (num_tokens == kEnd) {
|
||||
invokeFp32RouterGemm<InputT, kEnd, FP32_NUM_EXPERTS, FP32_HIDDEN_DIM>(
|
||||
invokeFp32RouterGemm<InputT, kEnd, kNumExperts, kHiddenDim>(
|
||||
output, mat_a, mat_b, stream);
|
||||
} else {
|
||||
throw std::invalid_argument(
|
||||
@@ -60,6 +66,23 @@ struct Fp32LoopUnroller<InputT, kEnd, kEnd> {
|
||||
}
|
||||
};
|
||||
|
||||
// Dispatch over the supported (num_experts, hidden_dim) pairs.
|
||||
template <typename InputT>
|
||||
void dispatchFp32RouterGemm(int num_experts, int hidden_dim, int num_tokens,
|
||||
float* output, InputT const* mat_a,
|
||||
float const* mat_b, cudaStream_t stream) {
|
||||
if (num_experts == 256 && hidden_dim == 3072) {
|
||||
Fp32LoopUnroller<InputT, 256, 3072, 1, FP32_MAX_TOKENS>::unroll(
|
||||
num_tokens, output, mat_a, mat_b, stream);
|
||||
} else if (num_experts == 128 && hidden_dim == 6144) {
|
||||
Fp32LoopUnroller<InputT, 128, 6144, 1, FP32_MAX_TOKENS>::unroll(
|
||||
num_tokens, output, mat_a, mat_b, stream);
|
||||
} else {
|
||||
throw std::invalid_argument(
|
||||
"fp32_router_gemm: unsupported (hidden_dim, num_experts) pair");
|
||||
}
|
||||
}
|
||||
|
||||
void fp32_router_gemm(
|
||||
torch::stable::Tensor& output, // [num_tokens, num_experts]
|
||||
torch::stable::Tensor const& mat_a, // [num_tokens, hidden_dim]
|
||||
@@ -85,10 +108,10 @@ void fp32_router_gemm(
|
||||
STD_TORCH_CHECK(
|
||||
mat_a.size(1) == mat_b.size(1),
|
||||
"fp32_router_gemm: mat_a and mat_b must have the same hidden_dim");
|
||||
STD_TORCH_CHECK(hidden_dim == FP32_HIDDEN_DIM,
|
||||
"fp32_router_gemm: expected hidden_dim=3072");
|
||||
STD_TORCH_CHECK(num_experts == FP32_NUM_EXPERTS,
|
||||
"fp32_router_gemm: expected num_experts=256");
|
||||
STD_TORCH_CHECK(
|
||||
fp32_router_gemm_supported(hidden_dim, num_experts),
|
||||
"fp32_router_gemm: supported (hidden_dim, num_experts) pairs are "
|
||||
"(3072, 256) and (6144, 128)");
|
||||
STD_TORCH_CHECK(num_tokens <= FP32_MAX_TOKENS,
|
||||
"fp32_router_gemm: num_tokens must be in [0, 32]");
|
||||
STD_TORCH_CHECK(
|
||||
@@ -113,12 +136,13 @@ void fp32_router_gemm(
|
||||
if (mat_a.scalar_type() == torch::headeronly::ScalarType::BFloat16) {
|
||||
auto const* mat_a_ptr =
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr());
|
||||
Fp32LoopUnroller<__nv_bfloat16, 1, FP32_MAX_TOKENS>::unroll(
|
||||
num_tokens, out_ptr, mat_a_ptr, mat_b_ptr, stream);
|
||||
dispatchFp32RouterGemm<__nv_bfloat16>(num_experts, hidden_dim, num_tokens,
|
||||
out_ptr, mat_a_ptr, mat_b_ptr,
|
||||
stream);
|
||||
} else {
|
||||
auto const* mat_a_ptr = reinterpret_cast<float const*>(mat_a.data_ptr());
|
||||
Fp32LoopUnroller<float, 1, FP32_MAX_TOKENS>::unroll(
|
||||
num_tokens, out_ptr, mat_a_ptr, mat_b_ptr, stream);
|
||||
dispatchFp32RouterGemm<float>(num_experts, hidden_dim, num_tokens, out_ptr,
|
||||
mat_a_ptr, mat_b_ptr, stream);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,635 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
* SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
*
|
||||
* Horizontally-fused MiniMax-M3 attention pre-processing kernel.
|
||||
*
|
||||
* Replaces the per-token Python sequence in
|
||||
* ``MiniMaxM3SparseAttention.forward`` / ``MiniMaxM3Attention.forward``:
|
||||
*
|
||||
* q = q_norm(q); k = k_norm(k); q, k = rotary_emb(pos, q, k)
|
||||
* index_q = index_q_norm(index_q); index_k = index_k_norm(index_k)
|
||||
* index_q, index_k = rotary_emb(pos, index_q, index_k)
|
||||
* _insert_kv(k, v, index_k)
|
||||
*
|
||||
* All branches share head_dim=128 and the *same* partial-NeoX RoPE table
|
||||
* (``rotary_dim`` rotated, the trailing dims pass through). The four norms
|
||||
* are Gemma-style RMSNorm (``x * rsqrt(mean(x^2)+eps) * (1 + weight)``) with
|
||||
* independent weights.
|
||||
*
|
||||
* Everything lives in a single fused ``qkv`` tensor. The sparse layer's
|
||||
* fused projection (MinimaxM3QKVParallelLinearWithIndexer) emits, per token::
|
||||
*
|
||||
* [ q | k | v | index_q | index_k ] (the "5 results")
|
||||
*
|
||||
* while the dense layer emits just ``[ q | k | v ]``. The kernel reads the
|
||||
* index branch straight out of that packed row -- no separate index tensors.
|
||||
*
|
||||
* One kernel, one grid; each warp owns one (token, head-slot) pair. Slot
|
||||
* enumeration per token:
|
||||
* [0, nq) Q heads -> norm(q_w) + RoPE, write
|
||||
* qkv [nq, nq+nkv) K heads -> norm(k_w) + RoPE, write
|
||||
* qkv
|
||||
* (+ insert into key cache)
|
||||
* [nq+nkv, nq+2*nkv) V heads -> insert into value cache
|
||||
* IQ heads (niq) -> norm(iq_w) + RoPE, write iq
|
||||
* IK (1) -> norm(ik_w) + RoPE
|
||||
* (+ insert into index cache)
|
||||
*
|
||||
* The IQ/IK warps address the index_q/index_k sub-blocks *inside* qkv at the
|
||||
* fixed physical offsets (nq+2*nkv)*128 and (nq+2*nkv+niq)*128.
|
||||
*
|
||||
* Dense vs sparse is a compile-time choice via the ``kIsSparse``/``kInsertKV``
|
||||
* template bools (3 instantiations: dense <false,false>, sparse-profiling
|
||||
* <true,false>, sparse-serving <true,true>), so the index slots, the V slots
|
||||
* and the cache inserts fold away entirely on paths that don't use them. The
|
||||
* dense layer passes no caches/index: norm+RoPE happens in place and the
|
||||
* generic ``Attention`` layer owns the cache write.
|
||||
*
|
||||
* Q/K and (sparse) index_q/index_k are all rewritten in place inside the fused
|
||||
* ``qkv`` tensor. Caches (bf16) are scatter-written by slot.
|
||||
*/
|
||||
|
||||
#include <cmath>
|
||||
#include <cuda_runtime.h>
|
||||
#include <type_traits>
|
||||
|
||||
#include "torch_utils.h"
|
||||
|
||||
#include "../cuda_compat.h"
|
||||
#include "../type_convert.cuh"
|
||||
#include "dispatch_utils.h"
|
||||
|
||||
#ifndef FINAL_MASK
|
||||
#ifdef USE_ROCM
|
||||
#define FINAL_MASK 0xffffffffffffffffULL
|
||||
#else
|
||||
#define FINAL_MASK 0xffffffffu
|
||||
#endif
|
||||
#endif
|
||||
|
||||
namespace vllm {
|
||||
namespace minimax_m3_fused_ops {
|
||||
|
||||
namespace {
|
||||
inline int getSMVersion() {
|
||||
auto* props = get_device_prop();
|
||||
return props->major * 10 + props->minor;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
// Constants (hard-coded for MiniMax-M3-preview).
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
constexpr int kHeadDim = 128;
|
||||
constexpr int kNumLanes = 32;
|
||||
constexpr int kElemsPerLane = kHeadDim / kNumLanes; // 4
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
// Helpers
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
__device__ __forceinline__ float warpReduceSum(float val) {
|
||||
#pragma unroll
|
||||
for (int mask = 16; mask > 0; mask >>= 1) {
|
||||
val += __shfl_xor_sync(FINAL_MASK, val, mask, 32);
|
||||
}
|
||||
return val;
|
||||
}
|
||||
|
||||
// Gemma RMSNorm over the full head (no-op when ``weight == nullptr``), rounded
|
||||
// back to scalar_t like the materialized unfused norm output, followed by
|
||||
// partial NeoX RoPE on the leading ``rotary_dim`` dims. Each lane owns
|
||||
// ``kElemsPerLane`` contiguous dims [laneId*4, laneId*4+4).
|
||||
template <typename scalar_t>
|
||||
__device__ __forceinline__ void normAndRope(
|
||||
float (&elems)[kElemsPerLane], int const laneId, float const eps,
|
||||
scalar_t const* __restrict__ weight, // [kHeadDim] or nullptr (no norm)
|
||||
bool const do_rope, int const rotary_dim,
|
||||
scalar_t const* __restrict__ cos_ptr, // cos_sin_cache + pos*rotary_dim
|
||||
bool const apply_norm) {
|
||||
// ── Gemma RMSNorm: x * rsqrt(mean(x^2)+eps) * (1 + w) ──────────────────
|
||||
if (apply_norm) {
|
||||
float sumsq = 0.0f;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kElemsPerLane; i++) sumsq += elems[i] * elems[i];
|
||||
sumsq = warpReduceSum(sumsq);
|
||||
float const rms_rcp = rsqrtf(sumsq / static_cast<float>(kHeadDim) + eps);
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kElemsPerLane; i++) {
|
||||
int const dim = laneId * kElemsPerLane + i;
|
||||
float const w = 1.0f + static_cast<float>(weight[dim]);
|
||||
elems[i] = elems[i] * rms_rcp * w;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Partial NeoX RoPE on dims [0, rotary_dim) ──────────────────────────
|
||||
// half = rotary_dim/2. Pair (i, i+half) for i in [0, half). Lane L owns
|
||||
// dims [4L, 4L+4); since half is a multiple of 4, a lane lies wholly in the
|
||||
// first half (own=x[i]) or second half (own=x[i+half]); its partner lives
|
||||
// ``half/4`` lanes away (XOR with that distance).
|
||||
if (do_rope) {
|
||||
int const half = rotary_dim / 2;
|
||||
int const dim0 = laneId * kElemsPerLane;
|
||||
bool const in_rope = dim0 < rotary_dim;
|
||||
int const lane_xor = half / kElemsPerLane; // partner-lane distance
|
||||
|
||||
float partner[kElemsPerLane];
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kElemsPerLane; i++) {
|
||||
partner[i] = __shfl_xor_sync(FINAL_MASK, elems[i], lane_xor, 32);
|
||||
}
|
||||
if (in_rope) {
|
||||
bool const first_half = dim0 < half;
|
||||
int const i_base = first_half ? dim0 : (dim0 - half); // cos/sin index
|
||||
scalar_t const* sin_ptr = cos_ptr + half;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kElemsPerLane; i++) {
|
||||
float const c = static_cast<float>(cos_ptr[i_base + i]);
|
||||
float const s = static_cast<float>(sin_ptr[i_base + i]);
|
||||
if (first_half) {
|
||||
elems[i] = elems[i] * c - partner[i] * s;
|
||||
} else {
|
||||
elems[i] = elems[i] * c + partner[i] * s;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Load 4 contiguous bf16 -> 4 fp32 registers.
|
||||
template <typename scalar_t>
|
||||
__device__ __forceinline__ void loadElems(scalar_t const* __restrict__ src,
|
||||
float (&elems)[kElemsPerLane]) {
|
||||
using Converter = vllm::_typeConvert<scalar_t>;
|
||||
uint2 v = *reinterpret_cast<uint2 const*>(src);
|
||||
auto const* p =
|
||||
reinterpret_cast<typename Converter::packed_hip_type const*>(&v);
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kElemsPerLane / 2; i++) {
|
||||
float2 f2 = Converter::convert(p[i]);
|
||||
elems[2 * i] = f2.x;
|
||||
elems[2 * i + 1] = f2.y;
|
||||
}
|
||||
}
|
||||
|
||||
// Store 4 fp32 registers -> 4 contiguous bf16.
|
||||
template <typename scalar_t>
|
||||
__device__ __forceinline__ void storeElems(
|
||||
scalar_t* __restrict__ dst, float const (&elems)[kElemsPerLane]) {
|
||||
using Converter = vllm::_typeConvert<scalar_t>;
|
||||
uint2 v;
|
||||
auto* p = reinterpret_cast<typename Converter::packed_hip_type*>(&v);
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kElemsPerLane / 2; i++) {
|
||||
p[i] = Converter::convert(make_float2(elems[2 * i], elems[2 * i + 1]));
|
||||
}
|
||||
*reinterpret_cast<uint2*>(dst) = v;
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
// Kernel
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
// Grid: 1D, ceil(num_tokens * slots_per_token / warps_per_block).
|
||||
// Each warp = one (token, slot).
|
||||
//
|
||||
// `kIsSparse` and `kInsertKV` are compile-time template bools, so all the
|
||||
// branch decisions that distinguish the dense layer from the sparse layer
|
||||
// (index slots, KV/index inserts, V slots) fold away per instantiation.
|
||||
// Three instantiations are built: dense <false,false>, sparse-profiling
|
||||
// <true,false> and sparse-serving <true,true>. Slots per token:
|
||||
// Q : nq (always — norm+RoPE)
|
||||
// K : nkv (always — norm+RoPE; +K-cache insert)
|
||||
// V : nkv only if kInsertKV (V-cache insert; no warps in dense)
|
||||
// IQ: niq only if kIsSparse (norm+RoPE)
|
||||
// IK: 1 only if kIsSparse (norm+RoPE; +index-cache insert)
|
||||
template <typename scalar_t, bool kIsSparse, bool kInsertKV>
|
||||
__global__ void fusedMiniMaxM3QNormRopeKVInsertKernel(
|
||||
scalar_t* __restrict__ qkv, // [N, qkv_row] in/out (packs index if sparse)
|
||||
scalar_t* __restrict__ q_out, // [N, nq*128] contiguous, or nullptr
|
||||
scalar_t* __restrict__ index_q_out, // [N, niq*128] contiguous, or nullptr
|
||||
scalar_t const* __restrict__ q_norm_w,
|
||||
scalar_t const* __restrict__ k_norm_w,
|
||||
scalar_t const* __restrict__ iq_norm_w,
|
||||
scalar_t const* __restrict__ ik_norm_w,
|
||||
scalar_t const* __restrict__ cos_sin_cache, // [max_pos, rotary_dim]
|
||||
int64_t const* __restrict__ positions, // [N] i64
|
||||
int64_t const* __restrict__ slot_mapping, // main K/V slots or nullptr
|
||||
int64_t const* __restrict__ index_slot_mapping, // index K slots/nullptr
|
||||
scalar_t* __restrict__ kv_cache, // [nb,2,bs,nkv,128] or nullptr
|
||||
scalar_t* __restrict__ index_cache, // [nb*bs, 128] or nullptr
|
||||
float const eps, int const rotary_dim, int const num_tokens, int const nq,
|
||||
int const nkv, int const niq, int const block_size,
|
||||
// kv_cache strides (in elements) for logical shape [nb, 2, bs, nkv, 128].
|
||||
// The head_dim (last) dim is always innermost-contiguous (stride 1), so the
|
||||
// NHD/HND layout choice is fully captured by these four strides: NHD keeps
|
||||
// s_token < s_head, HND swaps them. dim_base addresses head_dim directly.
|
||||
int64_t const kv_s_block, int64_t const kv_s_kv, int64_t const kv_s_token,
|
||||
int64_t const kv_s_head) {
|
||||
#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)
|
||||
// _typeConvert<BFloat16> is unavailable on pre-Ampere; the M3 kernel only
|
||||
// runs with bf16/fp16 inputs in practice. Discard the bf16 body there.
|
||||
if constexpr (std::is_same_v<scalar_t, c10::BFloat16>) {
|
||||
return;
|
||||
} else {
|
||||
#endif
|
||||
int const warpsPerBlock = blockDim.x / 32;
|
||||
int const laneId = threadIdx.x % 32;
|
||||
int const globalWarpIdx = blockIdx.x * warpsPerBlock + (threadIdx.x / 32);
|
||||
|
||||
// Slot layout (compile-time gated: dense has neither V nor index slots).
|
||||
int const v_slots = kInsertKV ? nkv : 0;
|
||||
int const idx_slots = kIsSparse ? niq + 1 : 0;
|
||||
int const slots_per_token = nq + nkv + v_slots + idx_slots;
|
||||
|
||||
int const tokenIdx = globalWarpIdx / slots_per_token;
|
||||
int const slot = globalWarpIdx % slots_per_token;
|
||||
if (tokenIdx >= num_tokens) return;
|
||||
|
||||
// Slot boundaries.
|
||||
int const k_begin = nq;
|
||||
int const v_begin = nq + nkv; // valid only when kInsertKV
|
||||
int const iq_begin = nq + nkv + v_slots; // index block start
|
||||
int const ik_slot = iq_begin + niq; // valid only when kIsSparse
|
||||
|
||||
bool const isQ = slot < k_begin;
|
||||
bool const isK = slot >= k_begin && slot < v_begin;
|
||||
bool isV = false;
|
||||
if constexpr (kInsertKV) isV = slot >= v_begin && slot < v_begin + nkv;
|
||||
bool isIQ = false, isIK = false;
|
||||
if constexpr (kIsSparse) {
|
||||
isIQ = slot >= iq_begin && slot < ik_slot;
|
||||
isIK = slot == ik_slot;
|
||||
}
|
||||
|
||||
int const dim_base = laneId * kElemsPerLane;
|
||||
// Physical row width of qkv: the dense layer packs [q|k|v]; the sparse
|
||||
// layer additionally packs [index_q (niq heads) | index_k (1 head)].
|
||||
int const qkv_row = (nq + 2 * nkv + (kIsSparse ? (niq + 1) : 0)) * kHeadDim;
|
||||
|
||||
// ── Resolve source pointer + per-branch parameters. ────────────────────
|
||||
scalar_t* row_ptr = nullptr; // in-place output location
|
||||
scalar_t const* norm_w = nullptr; // nullptr -> skip norm (V)
|
||||
bool do_rope = true;
|
||||
int head = 0; // kv head index for inserts
|
||||
|
||||
if (isQ) {
|
||||
row_ptr =
|
||||
qkv + static_cast<int64_t>(tokenIdx) * qkv_row + slot * kHeadDim;
|
||||
norm_w = q_norm_w;
|
||||
} else if (isK) {
|
||||
head = slot - k_begin;
|
||||
row_ptr =
|
||||
qkv + static_cast<int64_t>(tokenIdx) * qkv_row + slot * kHeadDim;
|
||||
norm_w = k_norm_w;
|
||||
} else if (isV) {
|
||||
// qkv V section starts at slot index (nq + nkv): slot * kHeadDim is the
|
||||
// correct in-tensor offset.
|
||||
head = slot - v_begin;
|
||||
row_ptr =
|
||||
qkv + static_cast<int64_t>(tokenIdx) * qkv_row + slot * kHeadDim;
|
||||
norm_w = nullptr; // V: no norm, no rope
|
||||
do_rope = false;
|
||||
} else if (isIQ) {
|
||||
// index_q sub-block lives at physical offset (nq+2*nkv)*128 in qkv.
|
||||
int const ih = slot - iq_begin;
|
||||
row_ptr = qkv + static_cast<int64_t>(tokenIdx) * qkv_row +
|
||||
(nq + 2 * nkv + ih) * kHeadDim;
|
||||
norm_w = iq_norm_w;
|
||||
} else { // isIK -- single shared index key at (nq+2*nkv+niq)*128.
|
||||
row_ptr = qkv + static_cast<int64_t>(tokenIdx) * qkv_row +
|
||||
(nq + 2 * nkv + niq) * kHeadDim;
|
||||
norm_w = ik_norm_w;
|
||||
}
|
||||
|
||||
// Store destination. Q and index_q are gathered into dedicated contiguous
|
||||
// output buffers (when provided) so the downstream SM100 sparse kernel's
|
||||
// flat TMA descriptor can address them as [tokens*heads, head_dim]; this
|
||||
// folds the de-interleaving into the store the kernel already does, instead
|
||||
// of a separate q.contiguous() copy. Everything else stays in place.
|
||||
scalar_t* store_ptr = row_ptr;
|
||||
if (isQ && q_out != nullptr) {
|
||||
store_ptr = q_out + static_cast<int64_t>(tokenIdx) * nq * kHeadDim +
|
||||
slot * kHeadDim;
|
||||
} else if (isIQ && index_q_out != nullptr) {
|
||||
store_ptr = index_q_out +
|
||||
static_cast<int64_t>(tokenIdx) * niq * kHeadDim +
|
||||
(slot - iq_begin) * kHeadDim;
|
||||
}
|
||||
|
||||
// PDL: wait for the predecessor kernel (the qkv-projection GEMM that
|
||||
// produces ``qkv``) to finish before touching any global memory. No-op
|
||||
// when PDL is not enabled on the launch. The CUDA runtime wrapper emits
|
||||
// the griddepcontrol.wait PTX with the required memory clobber internally.
|
||||
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
|
||||
cudaGridDependencySynchronize();
|
||||
#endif
|
||||
|
||||
// ── Load -> norm+rope (fp32) -> store back in place. ───────────────────
|
||||
float elems[kElemsPerLane];
|
||||
loadElems<scalar_t>(row_ptr + dim_base, elems);
|
||||
|
||||
if (!isV) {
|
||||
int64_t const pos = positions[tokenIdx];
|
||||
scalar_t const* cos_ptr = cos_sin_cache + pos * rotary_dim;
|
||||
normAndRope<scalar_t>(elems, laneId, eps, norm_w, do_rope, rotary_dim,
|
||||
cos_ptr, /*apply_norm=*/norm_w != nullptr);
|
||||
storeElems<scalar_t>(store_ptr + dim_base, elems);
|
||||
}
|
||||
|
||||
// ── Cache inserts (sparse serving only). ───────────────────────────────
|
||||
if constexpr (kInsertKV) {
|
||||
// Guard (not early-return) so every thread reaches the PDL trigger below.
|
||||
int64_t const sm = (isK || isV)
|
||||
? slot_mapping[tokenIdx]
|
||||
: (isIK ? index_slot_mapping[tokenIdx] : -1);
|
||||
if (sm >= 0) { // skip padded / unscheduled tokens
|
||||
if (isIK) {
|
||||
scalar_t* dst = index_cache + sm * kHeadDim + dim_base;
|
||||
storeElems<scalar_t>(dst, elems);
|
||||
} else if (isK || isV) {
|
||||
// kv_cache logical shape [num_blocks, 2, block_size, nkv, head_dim].
|
||||
// Paging is logical (block = sm/block_size, token = sm%block_size);
|
||||
// the physical NHD/HND layout is honoured via the passed strides.
|
||||
int64_t const b = sm / block_size;
|
||||
int64_t const t = sm % block_size;
|
||||
int const kv = isK ? 0 : 1;
|
||||
int64_t const off =
|
||||
b * kv_s_block + kv * kv_s_kv + t * kv_s_token + head * kv_s_head;
|
||||
storeElems<scalar_t>(kv_cache + off + dim_base, elems);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// PDL: signal that this kernel is done so a dependent successor may launch
|
||||
// early. No-op when PDL is not enabled on the launch.
|
||||
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
#endif
|
||||
#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
// Launch wrapper
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
template <typename scalar_t>
|
||||
void launchFusedMiniMaxM3(scalar_t* qkv, scalar_t* q_out, scalar_t* index_q_out,
|
||||
scalar_t const* q_norm_w, scalar_t const* k_norm_w,
|
||||
scalar_t const* iq_norm_w, scalar_t const* ik_norm_w,
|
||||
scalar_t const* cos_sin_cache,
|
||||
int64_t const* positions, int64_t const* slot_mapping,
|
||||
int64_t const* index_slot_mapping, scalar_t* kv_cache,
|
||||
scalar_t* index_cache, float const eps,
|
||||
int const rotary_dim, int const num_tokens,
|
||||
int const nq, int const nkv, int const niq,
|
||||
int const block_size, int64_t const kv_s_block,
|
||||
int64_t const kv_s_kv, int64_t const kv_s_token,
|
||||
int64_t const kv_s_head, bool const has_index,
|
||||
bool const insert_kv, cudaStream_t stream) {
|
||||
// Slot count must match the kernel's compile-time gating.
|
||||
int const v_slots = insert_kv ? nkv : 0;
|
||||
int const idx_slots = has_index ? niq + 1 : 0;
|
||||
int const slots_per_token = nq + nkv + v_slots + idx_slots;
|
||||
|
||||
constexpr int kBlockSize = 256;
|
||||
constexpr int kWarpsPerBlock = kBlockSize / 32;
|
||||
int64_t const total_warps =
|
||||
static_cast<int64_t>(num_tokens) * slots_per_token;
|
||||
int const grid =
|
||||
static_cast<int>((total_warps + kWarpsPerBlock - 1) / kWarpsPerBlock);
|
||||
if (grid == 0) return;
|
||||
|
||||
#ifndef USE_ROCM
|
||||
// PDL: enable programmatic stream serialization whenever the hardware
|
||||
// supports it (SM90+). On pre-Hopper GPUs the attribute is unavailable, so
|
||||
// leave numAttrs = 0 and launch as a regular kernel via cudaLaunchKernelEx.
|
||||
static int const sm_version = getSMVersion();
|
||||
cudaLaunchConfig_t config;
|
||||
config.gridDim = dim3(grid);
|
||||
config.blockDim = dim3(kBlockSize);
|
||||
config.dynamicSmemBytes = 0;
|
||||
config.stream = stream;
|
||||
cudaLaunchAttribute attrs[1];
|
||||
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
|
||||
attrs[0].val.programmaticStreamSerializationAllowed = 1;
|
||||
config.attrs = attrs;
|
||||
config.numAttrs = (sm_version >= 90) ? 1 : 0;
|
||||
|
||||
#define LAUNCH(IS_SPARSE, INSERT) \
|
||||
cudaLaunchKernelEx( \
|
||||
&config, \
|
||||
fusedMiniMaxM3QNormRopeKVInsertKernel<scalar_t, IS_SPARSE, INSERT>, \
|
||||
qkv, q_out, index_q_out, q_norm_w, k_norm_w, iq_norm_w, ik_norm_w, \
|
||||
cos_sin_cache, positions, slot_mapping, index_slot_mapping, kv_cache, \
|
||||
index_cache, eps, rotary_dim, num_tokens, nq, nkv, niq, block_size, \
|
||||
kv_s_block, kv_s_kv, kv_s_token, kv_s_head)
|
||||
#else
|
||||
// ROCm: standard kernel launch syntax (no PDL/stream serialization).
|
||||
// clang-format off
|
||||
#define LAUNCH(IS_SPARSE, INSERT) \
|
||||
fusedMiniMaxM3QNormRopeKVInsertKernel<scalar_t, IS_SPARSE, INSERT> \
|
||||
<<<grid, kBlockSize, 0, stream>>>( \
|
||||
qkv, q_out, index_q_out, q_norm_w, k_norm_w, iq_norm_w, \
|
||||
ik_norm_w, cos_sin_cache, positions, slot_mapping, \
|
||||
index_slot_mapping, kv_cache, index_cache, eps, rotary_dim, \
|
||||
num_tokens, nq, nkv, niq, block_size, kv_s_block, kv_s_kv, \
|
||||
kv_s_token, kv_s_head)
|
||||
// clang-format on
|
||||
#endif
|
||||
|
||||
if (has_index) {
|
||||
if (insert_kv) {
|
||||
LAUNCH(true, true); // sparse serving
|
||||
} else {
|
||||
LAUNCH(true, false); // sparse profiling
|
||||
}
|
||||
} else {
|
||||
// Dense layer: never has an index branch and never inserts here (the
|
||||
// generic Attention layer owns the KV insert).
|
||||
LAUNCH(false, false);
|
||||
}
|
||||
#undef LAUNCH
|
||||
}
|
||||
|
||||
} // namespace minimax_m3_fused_ops
|
||||
} // namespace vllm
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
// Torch op wrapper
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
void fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
torch::stable::Tensor& qkv, // [N, qkv_row] (packs index if sparse)
|
||||
torch::stable::Tensor const& q_norm_weight, // [128]
|
||||
torch::stable::Tensor const& k_norm_weight, // [128]
|
||||
torch::stable::Tensor const& cos_sin_cache, // [max_pos, rotary_dim]
|
||||
torch::stable::Tensor const& positions, // [N] i64
|
||||
int64_t num_heads, int64_t num_kv_heads, int64_t rotary_dim, double eps,
|
||||
std::optional<torch::stable::Tensor> index_q_norm_weight, // [128]
|
||||
std::optional<torch::stable::Tensor> index_k_norm_weight, // [128]
|
||||
int64_t num_index_heads, // niq; 0 => dense
|
||||
std::optional<torch::stable::Tensor> slot_mapping, // [N] i64
|
||||
std::optional<torch::stable::Tensor> index_slot_mapping, // [N] i64
|
||||
std::optional<torch::stable::Tensor> kv_cache, // [nb,2,bs,nkv,128]
|
||||
std::optional<torch::stable::Tensor> index_cache, // [nb,bs,128]
|
||||
int64_t block_size,
|
||||
std::optional<torch::stable::Tensor> q_out, // [N, nq*128] contiguous
|
||||
std::optional<torch::stable::Tensor>
|
||||
index_q_out) { // [N, niq*128] contiguous
|
||||
STD_TORCH_CHECK(qkv.is_cuda() && qkv.is_contiguous(),
|
||||
"qkv must be contiguous CUDA");
|
||||
STD_TORCH_CHECK(
|
||||
positions.is_cuda() &&
|
||||
positions.scalar_type() == torch::headeronly::ScalarType::Long,
|
||||
"positions must be int64 CUDA");
|
||||
STD_TORCH_CHECK(cos_sin_cache.is_cuda() && cos_sin_cache.is_contiguous(),
|
||||
"cos_sin_cache must be contiguous CUDA");
|
||||
STD_TORCH_CHECK(cos_sin_cache.scalar_type() == qkv.scalar_type(),
|
||||
"cos_sin_cache dtype must match qkv");
|
||||
STD_TORCH_CHECK(
|
||||
cos_sin_cache.dim() == 2 && cos_sin_cache.size(1) == rotary_dim,
|
||||
"cos_sin_cache shape [max_pos, rotary_dim]");
|
||||
|
||||
STD_TORCH_CHECK(q_norm_weight.scalar_type() == qkv.scalar_type() &&
|
||||
k_norm_weight.scalar_type() == qkv.scalar_type(),
|
||||
"q/k norm weight dtype must match qkv");
|
||||
STD_TORCH_CHECK(
|
||||
q_norm_weight.numel() == vllm::minimax_m3_fused_ops::kHeadDim &&
|
||||
k_norm_weight.numel() == vllm::minimax_m3_fused_ops::kHeadDim,
|
||||
"q/k norm weight must have 128 elements");
|
||||
STD_TORCH_CHECK(rotary_dim > 0 && rotary_dim % 8 == 0 &&
|
||||
rotary_dim <= vllm::minimax_m3_fused_ops::kHeadDim,
|
||||
"rotary_dim must be a positive multiple of 8 and <= 128");
|
||||
|
||||
int const num_tokens = static_cast<int>(qkv.size(0));
|
||||
int const nq = static_cast<int>(num_heads);
|
||||
int const nkv = static_cast<int>(num_kv_heads);
|
||||
int const niq = static_cast<int>(num_index_heads);
|
||||
|
||||
// The sparse layer packs the index branch ([index_q (niq heads) | index_k
|
||||
// (1 head)]) right after [q|k|v] in the same row; the dense layer does not.
|
||||
bool const has_index = niq > 0;
|
||||
bool const insert_kv = kv_cache.has_value();
|
||||
int const kHeadDim = vllm::minimax_m3_fused_ops::kHeadDim;
|
||||
int const expected_row =
|
||||
(nq + 2 * nkv + (has_index ? niq + 1 : 0)) * kHeadDim;
|
||||
STD_TORCH_CHECK(qkv.size(1) == expected_row,
|
||||
"qkv last dim must be (num_heads + 2*num_kv_heads"
|
||||
" + num_index_heads + 1) * 128 for sparse, "
|
||||
"(num_heads + 2*num_kv_heads) * 128 for dense");
|
||||
|
||||
// Only the sparse layer inserts here (dense lets the generic Attention layer
|
||||
// own the KV write); there is no dense+insert kernel instantiation.
|
||||
STD_TORCH_CHECK(
|
||||
!insert_kv || has_index,
|
||||
"insert mode (kv_cache) requires the index branch (sparse layer)");
|
||||
if (has_index) {
|
||||
STD_TORCH_CHECK(
|
||||
index_q_norm_weight.has_value() && index_k_norm_weight.has_value(),
|
||||
"index branch requires both index norm weights");
|
||||
STD_TORCH_CHECK(index_q_norm_weight->scalar_type() == qkv.scalar_type() &&
|
||||
index_k_norm_weight->scalar_type() == qkv.scalar_type(),
|
||||
"index norm weights dtype must match qkv");
|
||||
STD_TORCH_CHECK(index_q_norm_weight->numel() == kHeadDim &&
|
||||
index_k_norm_weight->numel() == kHeadDim,
|
||||
"index norm weights must have 128 elements");
|
||||
}
|
||||
// kv_cache strides (logical shape [nb, 2, bs, nkv, head_dim]). Read straight
|
||||
// off the tensor so the kernel honours whatever physical layout the attention
|
||||
// backend allocated (NHD: stride order (0,1,2,3,4); HND: (0,1,3,2,4)). No new
|
||||
// op argument is needed -- the strides ride along with the tensor itself.
|
||||
int64_t kv_s_block = 0, kv_s_kv = 0, kv_s_token = 0, kv_s_head = 0;
|
||||
torch::stable::Tensor const* effective_index_slot_mapping = nullptr;
|
||||
if (insert_kv) {
|
||||
STD_TORCH_CHECK(
|
||||
slot_mapping.has_value() && slot_mapping->is_cuda() &&
|
||||
slot_mapping->scalar_type() == torch::headeronly::ScalarType::Long,
|
||||
"insert mode requires int64 CUDA slot_mapping");
|
||||
STD_TORCH_CHECK(
|
||||
!index_slot_mapping.has_value() ||
|
||||
(index_slot_mapping->is_cuda() &&
|
||||
index_slot_mapping->scalar_type() ==
|
||||
torch::headeronly::ScalarType::Long &&
|
||||
index_slot_mapping->numel() == slot_mapping->numel()),
|
||||
"index_slot_mapping must be int64 CUDA with slot_mapping length");
|
||||
STD_TORCH_CHECK(kv_cache->scalar_type() == qkv.scalar_type(),
|
||||
"kv_cache dtype must match qkv (bf16 cache only)");
|
||||
STD_TORCH_CHECK(index_cache.has_value() &&
|
||||
index_cache->scalar_type() == qkv.scalar_type(),
|
||||
"insert mode requires matching index_cache");
|
||||
STD_TORCH_CHECK(kv_cache->dim() == 5 && kv_cache->stride(4) == 1,
|
||||
"kv_cache must be [nb,2,bs,nkv,head_dim] with contiguous "
|
||||
"head_dim (stride(4)==1)");
|
||||
kv_s_block = kv_cache->stride(0);
|
||||
kv_s_kv = kv_cache->stride(1);
|
||||
kv_s_token = kv_cache->stride(2);
|
||||
kv_s_head = kv_cache->stride(3);
|
||||
effective_index_slot_mapping = index_slot_mapping.has_value()
|
||||
? &index_slot_mapping.value()
|
||||
: &slot_mapping.value();
|
||||
}
|
||||
// Optional contiguous gather targets: when given, the normed/roped q (and
|
||||
// index_q) are written here instead of in place, so callers avoid a separate
|
||||
// .contiguous() copy. index_q_out only makes sense on the sparse path.
|
||||
if (q_out.has_value()) {
|
||||
STD_TORCH_CHECK(
|
||||
q_out->is_cuda() && q_out->is_contiguous() &&
|
||||
q_out->scalar_type() == qkv.scalar_type(),
|
||||
"q_out must be a contiguous CUDA tensor matching qkv dtype");
|
||||
STD_TORCH_CHECK(
|
||||
q_out->numel() == static_cast<int64_t>(num_tokens) * nq * kHeadDim,
|
||||
"q_out must have num_tokens * num_heads * 128 elements");
|
||||
}
|
||||
if (index_q_out.has_value()) {
|
||||
STD_TORCH_CHECK(
|
||||
has_index,
|
||||
"index_q_out requires the index branch (num_index_heads > 0)");
|
||||
STD_TORCH_CHECK(
|
||||
index_q_out->is_cuda() && index_q_out->is_contiguous() &&
|
||||
index_q_out->scalar_type() == qkv.scalar_type(),
|
||||
"index_q_out must be a contiguous CUDA tensor matching qkv dtype");
|
||||
STD_TORCH_CHECK(index_q_out->numel() ==
|
||||
static_cast<int64_t>(num_tokens) * niq * kHeadDim,
|
||||
"index_q_out must have num_tokens * num_index_heads * 128 "
|
||||
"elements");
|
||||
}
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
qkv.get_device_index());
|
||||
auto stream = get_current_cuda_stream(qkv.get_device_index());
|
||||
|
||||
VLLM_STABLE_DISPATCH_HALF_TYPES(
|
||||
qkv.scalar_type(), "fused_minimax_m3_qknorm_rope_kv_insert", [&] {
|
||||
using st = scalar_t;
|
||||
vllm::minimax_m3_fused_ops::launchFusedMiniMaxM3<st>(
|
||||
reinterpret_cast<st*>(qkv.data_ptr()),
|
||||
q_out.has_value() ? reinterpret_cast<st*>(q_out->data_ptr())
|
||||
: nullptr,
|
||||
index_q_out.has_value()
|
||||
? reinterpret_cast<st*>(index_q_out->data_ptr())
|
||||
: nullptr,
|
||||
reinterpret_cast<st const*>(q_norm_weight.data_ptr()),
|
||||
reinterpret_cast<st const*>(k_norm_weight.data_ptr()),
|
||||
has_index
|
||||
? reinterpret_cast<st const*>(index_q_norm_weight->data_ptr())
|
||||
: nullptr,
|
||||
has_index
|
||||
? reinterpret_cast<st const*>(index_k_norm_weight->data_ptr())
|
||||
: nullptr,
|
||||
reinterpret_cast<st const*>(cos_sin_cache.data_ptr()),
|
||||
reinterpret_cast<int64_t const*>(positions.data_ptr()),
|
||||
insert_kv
|
||||
? reinterpret_cast<int64_t const*>(slot_mapping->data_ptr())
|
||||
: nullptr,
|
||||
insert_kv ? reinterpret_cast<int64_t const*>(
|
||||
effective_index_slot_mapping->data_ptr())
|
||||
: nullptr,
|
||||
insert_kv ? reinterpret_cast<st*>(kv_cache->data_ptr()) : nullptr,
|
||||
(insert_kv && has_index)
|
||||
? reinterpret_cast<st*>(index_cache->data_ptr())
|
||||
: nullptr,
|
||||
static_cast<float>(eps), static_cast<int>(rotary_dim), num_tokens,
|
||||
nq, nkv, niq, static_cast<int>(block_size), kv_s_block, kv_s_kv,
|
||||
kv_s_token, kv_s_head, has_index, insert_kv, stream);
|
||||
});
|
||||
}
|
||||
@@ -281,6 +281,24 @@ minimax_allreduce_rms_qk(torch::stable::Tensor qkv,
|
||||
int64_t const nranks, double const eps);
|
||||
#endif
|
||||
|
||||
// Horizontally-fused MiniMax-M3 QK-norm + partial NeoX RoPE (+ optional KV /
|
||||
// index-cache insert). Dense layer: norm+RoPE only; sparse layer: also packs
|
||||
// the index branch and scatters k/v/index_k into their paged caches.
|
||||
void fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
torch::stable::Tensor& qkv, torch::stable::Tensor const& q_norm_weight,
|
||||
torch::stable::Tensor const& k_norm_weight,
|
||||
torch::stable::Tensor const& cos_sin_cache,
|
||||
torch::stable::Tensor const& positions, int64_t num_heads,
|
||||
int64_t num_kv_heads, int64_t rotary_dim, double eps,
|
||||
std::optional<torch::stable::Tensor> index_q_norm_weight,
|
||||
std::optional<torch::stable::Tensor> index_k_norm_weight,
|
||||
int64_t num_index_heads, std::optional<torch::stable::Tensor> slot_mapping,
|
||||
std::optional<torch::stable::Tensor> index_slot_mapping,
|
||||
std::optional<torch::stable::Tensor> kv_cache,
|
||||
std::optional<torch::stable::Tensor> index_cache, int64_t block_size,
|
||||
std::optional<torch::stable::Tensor> q_out,
|
||||
std::optional<torch::stable::Tensor> index_q_out);
|
||||
|
||||
// Sampler kernels (shared CUDA/ROCm)
|
||||
void apply_repetition_penalties_(
|
||||
torch::stable::Tensor& logits, const torch::stable::Tensor& prompt_mask,
|
||||
@@ -346,7 +364,8 @@ void free_shared_buffer(int64_t buffer);
|
||||
// Activation kernels (shared CUDA/ROCm)
|
||||
void silu_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input);
|
||||
void silu_and_mul_clamp(torch::stable::Tensor& out,
|
||||
torch::stable::Tensor& input, double limit);
|
||||
torch::stable::Tensor& input, double limit,
|
||||
double alpha = 1.0, double beta = 0.0);
|
||||
void mul_and_silu(torch::stable::Tensor& out, torch::stable::Tensor& input);
|
||||
void gelu_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input);
|
||||
void gelu_tanh_and_mul(torch::stable::Tensor& out,
|
||||
|
||||
@@ -461,6 +461,18 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"float eps) -> (Tensor, Tensor)");
|
||||
#endif
|
||||
|
||||
// Horizontally-fused MiniMax-M3 QK-norm + partial NeoX RoPE + KV-insert.
|
||||
ops.def(
|
||||
"fused_minimax_m3_qknorm_rope_kv_insert("
|
||||
"Tensor! qkv, Tensor q_norm_weight, Tensor k_norm_weight, "
|
||||
"Tensor cos_sin_cache, Tensor positions, int num_heads, "
|
||||
"int num_kv_heads, int rotary_dim, float eps, "
|
||||
"Tensor? index_q_norm_weight, Tensor? index_k_norm_weight, "
|
||||
"int num_index_heads, "
|
||||
"Tensor? slot_mapping, Tensor? index_slot_mapping, "
|
||||
"Tensor!? kv_cache, Tensor!? index_cache, "
|
||||
"int block_size, Tensor!? q_out, Tensor!? index_q_out) -> ()");
|
||||
|
||||
// Apply repetition penalties to logits in-place.
|
||||
ops.def(
|
||||
"apply_repetition_penalties_(Tensor! logits, Tensor prompt_mask, "
|
||||
@@ -488,9 +500,11 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
ops.def("mul_and_silu(Tensor! out, Tensor input) -> ()");
|
||||
|
||||
// SwiGLU activation with input clamping.
|
||||
// alpha scales the sigmoid (gate * sigmoid(alpha * gate)); beta is added to
|
||||
// the up half (up + beta). Defaults alpha=1.0, beta=0.0 give silu(gate)*up.
|
||||
ops.def(
|
||||
"silu_and_mul_with_clamp(Tensor! result, Tensor input, float limit) "
|
||||
"-> ()");
|
||||
"silu_and_mul_with_clamp(Tensor! result, Tensor input, float limit, "
|
||||
"float alpha=1.0, float beta=0.0) -> ()");
|
||||
|
||||
// Activation function used in GeGLU with `none` approximation.
|
||||
ops.def("gelu_and_mul(Tensor! out, Tensor input) -> ()");
|
||||
@@ -679,6 +693,8 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
|
||||
ops.impl("minimax_allreduce_rms", TORCH_BOX(&minimax_allreduce_rms));
|
||||
ops.impl("minimax_allreduce_rms_qk", TORCH_BOX(&minimax_allreduce_rms_qk));
|
||||
#endif
|
||||
ops.impl("fused_minimax_m3_qknorm_rope_kv_insert",
|
||||
TORCH_BOX(&fused_minimax_m3_qknorm_rope_kv_insert));
|
||||
|
||||
// Sampler kernels (shared CUDA/ROCm)
|
||||
ops.impl("apply_repetition_penalties_",
|
||||
|
||||
+2
-1
@@ -50,7 +50,8 @@ void rotary_embedding(torch::Tensor& positions, torch::Tensor& query,
|
||||
|
||||
void silu_and_mul(torch::Tensor& out, torch::Tensor& input);
|
||||
|
||||
void silu_and_mul_clamp(torch::Tensor& out, torch::Tensor& input, double limit);
|
||||
void silu_and_mul_clamp(torch::Tensor& out, torch::Tensor& input, double limit,
|
||||
double alpha = 1.0, double beta = 0.0);
|
||||
|
||||
void silu_and_mul_quant(torch::Tensor& out, torch::Tensor& input,
|
||||
torch::Tensor& scale);
|
||||
|
||||
Reference in New Issue
Block a user