[Quantization][Humming] Support MXFP4 weight + block-FP8 activation for MoE (#51332)

Signed-off-by: Elvir Crncevic <elvircrn@gmail.com>
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Signed-off-by: Elvir Crnčević <elvircrn@gmail.com>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Elvir Crnčević
2026-08-25 10:21:00 -04:00
committed by GitHub
co-authored by Claude Opus 4.8 Tyler Michael Smith mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 9c8e90eb26
commit ca29cd48ba
17 changed files with 1594 additions and 171 deletions
+401 -14
View File
@@ -1,9 +1,17 @@
#include <cuda.h>
#include <cuda_runtime.h>
#include <torch/csrc/stable/tensor.h>
#include <torch/headeronly/util/Float8_e4m3fn.h>
#ifndef USE_ROCM
#include <cuda_fp8.h>
#endif
#include <cmath>
#include <type_traits>
#include "../cuda_compat.h"
#include "async_util.cuh"
#include "cuda_vec_utils.cuh"
#include "dispatch_utils.h"
#include "torch_utils.h"
@@ -471,15 +479,31 @@ __global__ void swigluoai_and_mul_kernel(
// gate_out = beta * tanh(gate / beta) * sigmoid(gate)
// up_out = (linear_beta > 0) ? linear_beta * tanh(up / linear_beta) : up
// out = gate_out * up_out
// Compute is done in fp32 and written straight to `out` -- no intermediate
// tensors and no full-tensor fp32 upcast (the pure-torch forward_native
// allocated ~8 fp32 temporaries per call, which blows up MoE profiling).
__device__ __forceinline__ float situ_tanh(float x) { return tanhf(x); }
// Kimi-K3 SITU params; baked into the fused LDG kernel to fold at compile time.
static constexpr float SITU_BETA = 4.0f;
static constexpr float SITU_LINEAR_BETA = 25.0f;
__device__ __forceinline__ float situ_activation(float g, float u, float beta,
float linear_beta,
bool clamp_up, float inv_beta,
float inv_linear_beta) {
// sigmoid(g) == (1 + tanh(g/2)) / 2.
const float gate_out =
(0.5f * beta) * situ_tanh(g * inv_beta) * (1.0f + situ_tanh(g * 0.5f));
const float up_out =
clamp_up ? linear_beta * situ_tanh(u * inv_linear_beta) : u;
return gate_out * up_out;
}
template <typename scalar_t>
__global__ void situ_and_mul_kernel(
scalar_t* __restrict__ out, // [..., d]
const scalar_t* __restrict__ input, // [..., 2, d]
const int d, const float beta, const float linear_beta) {
const int d, const float beta, const float linear_beta,
const int64_t* __restrict__ valid_rows_ptr) {
const int64_t row = blockIdx.x;
if (valid_rows_ptr != nullptr && row >= *valid_rows_ptr) return;
const scalar_t* gate_ptr = input + row * 2 * d;
const scalar_t* up_ptr = gate_ptr + d;
scalar_t* out_ptr = out + row * d;
@@ -489,10 +513,258 @@ __global__ void situ_and_mul_kernel(
for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
const float g = (float)VLLM_LDG(&gate_ptr[idx]);
const float u = (float)VLLM_LDG(&up_ptr[idx]);
const float gate_out = beta * tanhf(g * inv_beta) / (1.0f + expf(-g));
const float up_out =
clamp_up ? linear_beta * tanhf(u * inv_linear_beta) : u;
out_ptr[idx] = (scalar_t)(gate_out * up_out);
out_ptr[idx] = (scalar_t)situ_activation(g, u, beta, linear_beta, clamp_up,
inv_beta, inv_linear_beta);
}
}
// Match Humming's hardware FP8 conversion. c10::Float8_e4m3fn's software cast
// can round ties differently from `cvt.rn.satfinite.e4m3.f32`.
__device__ __forceinline__ c10::Float8_e4m3fn cvt_fp8_hw(float x) {
#if !defined(USE_ROCM) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
return c10::Float8_e4m3fn(__nv_cvt_float_to_fp8(x, __NV_SATFINITE, __NV_E4M3),
c10::Float8_e4m3fn::from_bits());
#else
return static_cast<c10::Float8_e4m3fn>(x);
#endif
}
template <typename fp8_type>
__device__ __forceinline__ fp8_type quant_to_fp8(float val, float inv_scale,
float fp8_max) {
float x = val * inv_scale;
x = fmaxf(-fp8_max, fminf(x, fp8_max));
if constexpr (std::is_same_v<fp8_type, c10::Float8_e4m3fn>) {
return cvt_fp8_hw(x);
} else {
return static_cast<fp8_type>(x);
}
}
__device__ __forceinline__ float warp_reduce_max(float v) {
#pragma unroll
for (int offset = WARP_SIZE / 2; offset > 0; offset >>= 1) {
v = fmaxf(v, VLLM_SHFL_XOR_SYNC(v, offset));
}
return v;
}
__device__ __forceinline__ float warp_reduce_absmax(float v) {
return warp_reduce_max(v);
}
template <typename scalar_t, typename fp8_type, int THREADS, int NUM_STAGES,
int GROUP_SIZE, int GRID_DIM, int D>
__global__ void situ_and_mul_quant_group_pipelined_kernel(
fp8_type* __restrict__ out, float* __restrict__ scale_out,
const scalar_t* __restrict__ input, const int64_t num_rows,
const int32_t* __restrict__ valid_rows_ptr, const int64_t topk) {
const int tid = threadIdx.x;
const int warp_id = tid / WARP_SIZE;
const int lane_id = tid % WARP_SIZE;
static constexpr int NUM_WARPS = THREADS / WARP_SIZE;
static constexpr int NUM_GROUPS = D / GROUP_SIZE;
const int64_t row_bound =
valid_rows_ptr != nullptr
? max((int64_t)0, min((int64_t)(*valid_rows_ptr) * topk, num_rows))
: num_rows;
static_assert(NUM_GROUPS % 4 == 0,
"float4 scale fill needs NUM_GROUPS % 4 == 0");
static constexpr int NG4 = NUM_GROUPS / 4;
const int64_t pad_start4 = row_bound * NG4;
const int64_t pad_end4 = num_rows * NG4;
float4* scale4 = reinterpret_cast<float4*>(scale_out);
constexpr float4 ones{1.0f, 1.0f, 1.0f, 1.0f};
for (int64_t i = pad_start4 + (int64_t)blockIdx.x * THREADS + tid;
i < pad_end4; i += (int64_t)GRID_DIM * THREADS) {
scale4[i] = ones;
}
if constexpr (sizeof(scalar_t) == 2) {
static constexpr int LD_ELTS = 16 / sizeof(scalar_t);
static constexpr int ELTS_PER_LANE = GROUP_SIZE / WARP_SIZE;
static constexpr int STAGE_ELTS = 2 * GROUP_SIZE;
extern __shared__ __align__(16) unsigned char smem_raw[];
scalar_t* warp_smem = reinterpret_cast<scalar_t*>(smem_raw) +
(size_t)warp_id * NUM_STAGES * STAGE_ELTS;
static constexpr float beta = SITU_BETA;
static constexpr float linear_beta = SITU_LINEAR_BETA;
static constexpr bool clamp_up = linear_beta > 0.0f;
static constexpr float inv_beta = 1.0f / beta;
static constexpr float inv_linear_beta =
clamp_up ? 1.0f / linear_beta : 0.0f;
static constexpr float FP8_MAX =
std::is_same_v<fp8_type, c10::Float8_e4m3fn> ? 448.0f : 224.0f;
static constexpr float inv_fp8_max = 1.0f / FP8_MAX;
static_assert(
NUM_GROUPS % NUM_WARPS == 0,
"constexpr num_iters requires groups evenly split across warps");
static constexpr int num_iters = NUM_GROUPS / NUM_WARPS;
const bool up_half = lane_id >= WARP_SIZE / 2;
const int lane_l = up_half ? lane_id - WARP_SIZE / 2 : lane_id;
const int lane_src_off = (up_half ? D : 0) + lane_l * LD_ELTS;
const int lane_dst_off = (up_half ? GROUP_SIZE : 0) + lane_l * LD_ELTS;
static constexpr int warp_stride = NUM_WARPS * GROUP_SIZE;
const scalar_t* src_ptr = input + warp_id * GROUP_SIZE + lane_src_off;
static constexpr int src_row_stride = GRID_DIM * 2 * D;
const scalar_t* row_src = src_ptr + blockIdx.x * 2 * D;
uint32_t* out_ptr =
reinterpret_cast<uint32_t*>(out + warp_id * GROUP_SIZE) + lane_id;
static constexpr int out_row_stride = GRID_DIM * D / 4;
uint32_t* row_out = out_ptr + blockIdx.x * (D / 4);
float* scale_ptr = scale_out + warp_id;
static constexpr int scale_row_stride = GRID_DIM * NUM_GROUPS;
float* row_scale = scale_ptr + blockIdx.x * NUM_GROUPS;
#pragma unroll 1
for (int64_t row = blockIdx.x; row < row_bound; row += GRID_DIM,
row_src += src_row_stride, row_out += out_row_stride,
row_scale += scale_row_stride) {
const scalar_t* src = row_src;
auto issue_load = [&](int it, int slot) {
if (it < num_iters) {
cuda_async::cp_async_shared_global_16_cg(
warp_smem + (size_t)slot * STAGE_ELTS + lane_dst_off, src);
src += warp_stride;
}
cuda_async::cp_async_commit_group();
};
int load_slot = 0;
auto bump = [](int s) { return s + 1 == NUM_STAGES ? 0 : s + 1; };
#pragma unroll
for (int s = 0; s < NUM_STAGES - 1; s++) {
issue_load(s, load_slot);
load_slot = bump(load_slot);
}
uint32_t* out_st = row_out;
float* scale_st = row_scale;
int comp_slot = 0;
for (int it = 0; it < num_iters; it++) {
issue_load(it + NUM_STAGES - 1, load_slot);
load_slot = bump(load_slot);
cuda_async::cp_async_wait_group<NUM_STAGES - 1>();
const scalar_t* stage = warp_smem + (size_t)comp_slot * STAGE_ELTS;
comp_slot = bump(comp_slot);
static_assert(ELTS_PER_LANE == 4,
"expects GROUP_SIZE == 4 * WARP_SIZE");
union V {
float2 f2;
scalar_t s[ELTS_PER_LANE];
};
const float2* gate2 = reinterpret_cast<const float2*>(stage);
const float2* up2 = reinterpret_cast<const float2*>(stage + GROUP_SIZE);
const V gv{gate2[lane_id]};
const V uv{up2[lane_id]};
const scalar_t* gs = gv.s;
const scalar_t* us = uv.s;
float acts[ELTS_PER_LANE];
float thread_max = 0.0f;
#pragma unroll
for (int e = 0; e < ELTS_PER_LANE; e++) {
acts[e] = (float)(scalar_t)situ_activation(
(float)gs[e], (float)us[e], beta, linear_beta, clamp_up, inv_beta,
inv_linear_beta);
thread_max = fmaxf(thread_max, fabsf(acts[e]));
}
const float absmax = fmaxf(warp_reduce_absmax(thread_max), 1e-30f);
const float scale = absmax * inv_fp8_max;
if (lane_id == 0) *scale_st = scale;
scale_st += NUM_WARPS;
const float inv_scale = __fdividef(1.0f, scale);
union O {
uint32_t u;
fp8_type f[ELTS_PER_LANE];
};
O o;
#pragma unroll
for (int e = 0; e < ELTS_PER_LANE; e++) {
o.f[e] = quant_to_fp8<fp8_type>(acts[e], inv_scale, FP8_MAX);
}
out_st[0] = o.u;
out_st += NUM_WARPS * GROUP_SIZE / 4;
}
cuda_async::cp_async_wait_group<0>();
}
}
}
// Scalar fallback for the group path (odd d or the fp32 dispatch branch).
template <typename scalar_t, typename fp8_type, int GROUP_SIZE>
__global__ void situ_and_mul_quant_group_scalar_kernel(
fp8_type* __restrict__ out, // [num_tokens, d]
float* __restrict__ scale_out, // [num_tokens, num_groups]
const scalar_t* __restrict__ input, // [num_tokens, 2, d]
const int d, const int num_groups, const float beta,
const float linear_beta, const int64_t num_rows,
const int32_t* __restrict__ valid_rows_ptr, const int64_t topk) {
static constexpr int ELTS_PER_LANE = GROUP_SIZE / WARP_SIZE;
const int tid = threadIdx.x;
const int warp_id = tid / WARP_SIZE;
const int lane_id = tid % WARP_SIZE;
const int num_warps = blockDim.x / WARP_SIZE;
const int64_t row_bound =
valid_rows_ptr != nullptr
? max((int64_t)0, min((int64_t)(*valid_rows_ptr) * topk, num_rows))
: num_rows;
const bool clamp_up = linear_beta > 0.0f;
const float inv_beta = __fdividef(1.0f, beta);
const float inv_linear_beta = clamp_up ? __fdividef(1.0f, linear_beta) : 0.0f;
static constexpr float FP8_MAX =
std::is_same_v<fp8_type, c10::Float8_e4m3fn> ? 448.0f : 224.0f;
static constexpr float inv_fp8_max = 1.0f / FP8_MAX;
for (int64_t row = blockIdx.x; row < row_bound; row += gridDim.x) {
const scalar_t* gate_ptr = input + row * 2 * (int64_t)d;
const scalar_t* up_ptr = gate_ptr + d;
fp8_type* out_ptr = out + row * (int64_t)d;
float* scale_row = scale_out + row * (int64_t)num_groups;
for (int g = warp_id; g < num_groups; g += num_warps) {
const int base = g * GROUP_SIZE;
float acts[ELTS_PER_LANE];
float thread_max = 0.0f;
#pragma unroll
for (int e = 0; e < ELTS_PER_LANE; e++) {
const int idx = base + e * WARP_SIZE + lane_id;
const float gv = (float)VLLM_LDG(&gate_ptr[idx]);
const float uv = (float)VLLM_LDG(&up_ptr[idx]);
acts[e] = (float)(scalar_t)situ_activation(
gv, uv, beta, linear_beta, clamp_up, inv_beta, inv_linear_beta);
thread_max = fmaxf(thread_max, fabsf(acts[e]));
}
const float absmax = fmaxf(warp_reduce_max(thread_max), 1e-30f);
const float scale = absmax * inv_fp8_max;
if (lane_id == 0) scale_row[g] = scale;
const float inv_scale = __fdividef(1.0f, scale);
#pragma unroll
for (int e = 0; e < ELTS_PER_LANE; e++) {
const int idx = base + e * WARP_SIZE + lane_id;
out_ptr[idx] = quant_to_fp8<fp8_type>(acts[e], inv_scale, FP8_MAX);
}
}
}
// Fill skipped padding-row scales with 1 so they don't feed NaN into w2.
const int64_t pad_start = row_bound * (int64_t)num_groups;
const int64_t pad_end = num_rows * (int64_t)num_groups;
for (int64_t i = pad_start + (int64_t)blockIdx.x * blockDim.x + tid;
i < pad_end; i += (int64_t)gridDim.x * blockDim.x) {
scale_out[i] = 1.0f;
}
}
@@ -519,10 +791,8 @@ __global__ void masked_situ_and_mul_kernel(
scalar_t* out_ptr = out + row * d;
const float g = (float)VLLM_LDG(&gate_ptr[idx]);
const float u = (float)VLLM_LDG(&up_ptr[idx]);
const float gate_out = beta * tanhf(g * inv_beta) / (1.0f + expf(-g));
const float up_out =
clamp_up ? linear_beta * tanhf(u * inv_linear_beta) : u;
out_ptr[idx] = (scalar_t)(gate_out * up_out);
out_ptr[idx] = (scalar_t)situ_activation(g, u, beta, linear_beta, clamp_up,
inv_beta, inv_linear_beta);
}
}
@@ -620,7 +890,8 @@ void swigluoai_and_mul(torch::stable::Tensor& out, // [..., d]
// through), matching SituAndMul(linear_beta=None) on the Python side.
void situ_and_mul(torch::stable::Tensor& out, // [..., d]
torch::stable::Tensor& input, // [..., 2 * d]
double beta, double linear_beta) {
double beta, double linear_beta,
std::optional<torch::stable::Tensor> valid_rows) {
int d = input.size(-1) / 2;
int64_t num_tokens = input.numel() / input.size(-1);
if (num_tokens == 0) {
@@ -628,6 +899,14 @@ void situ_and_mul(torch::stable::Tensor& out, // [..., d]
}
dim3 grid(num_tokens);
dim3 block(std::min(d, 1024));
const int64_t* valid_rows_ptr = nullptr;
if (valid_rows.has_value()) {
STD_TORCH_CHECK(valid_rows->is_cuda() && valid_rows->get_device_index() ==
input.get_device_index(),
"situ_and_mul: valid_rows must be a CUDA tensor on the "
"same device as input");
valid_rows_ptr = valid_rows->const_data_ptr<int64_t>();
}
const torch::stable::accelerator::DeviceGuard device_guard(
input.get_device_index());
const cudaStream_t stream = get_current_cuda_stream();
@@ -635,7 +914,7 @@ void situ_and_mul(torch::stable::Tensor& out, // [..., d]
input.scalar_type(), "situ_and_mul_kernel", [&] {
vllm::situ_and_mul_kernel<scalar_t><<<grid, block, 0, stream>>>(
out.mutable_data_ptr<scalar_t>(), input.const_data_ptr<scalar_t>(),
d, (float)beta, (float)linear_beta);
d, (float)beta, (float)linear_beta, valid_rows_ptr);
});
}
@@ -663,6 +942,114 @@ void masked_situ_and_mul(torch::stable::Tensor& out, // [E, T, d]
(float)beta, (float)linear_beta);
});
}
// Fused SITU activation and block-FP8 quantization for Humming w2.
// `valid_rows` is the int32 DeepEP token count (psum[-1]); rows = it * `topk`,
// so padding is excluded on-device and skipped rows receive scale 1.
void situ_and_mul_quant(torch::stable::Tensor& out, // [..., d] (fp8)
torch::stable::Tensor& scale, // [..., 1 or d/128]
torch::stable::Tensor& input, // [..., 2 * d]
double beta, double linear_beta, int64_t group_size,
std::optional<torch::stable::Tensor> valid_rows,
int64_t topk) {
STD_TORCH_CHECK(
out.scalar_type() == torch::headeronly::ScalarType::Float8_e4m3fn ||
out.scalar_type() == torch::headeronly::ScalarType::Float8_e4m3fnuz,
"situ_and_mul_quant output must be FP8 e4m3");
STD_TORCH_CHECK(
input.scalar_type() == torch::headeronly::ScalarType::Half ||
input.scalar_type() == torch::headeronly::ScalarType::BFloat16,
"situ_and_mul_quant input must be FP16 or BF16");
STD_TORCH_CHECK(scale.scalar_type() == torch::headeronly::ScalarType::Float,
"situ_and_mul_quant scale must be float32");
STD_TORCH_CHECK(input.is_cuda() && out.is_cuda() && scale.is_cuda() &&
out.get_device_index() == input.get_device_index() &&
scale.get_device_index() == input.get_device_index(),
"situ_and_mul_quant: input, out and scale must be CUDA "
"tensors on the same device");
int d = input.size(-1) / 2;
int64_t num_tokens = input.numel() / input.size(-1);
if (num_tokens == 0) {
return;
}
STD_TORCH_CHECK(out.size(-1) == d && out.numel() == num_tokens * (int64_t)d,
"situ_and_mul_quant: out shape must be [num_tokens, d]");
const int32_t* valid_rows_ptr = nullptr;
if (valid_rows.has_value()) {
STD_TORCH_CHECK(valid_rows->is_cuda() && valid_rows->get_device_index() ==
input.get_device_index(),
"situ_and_mul_quant: valid_rows must be a CUDA tensor on "
"the same device as input");
STD_TORCH_CHECK(
valid_rows->scalar_type() == torch::headeronly::ScalarType::Int,
"situ_and_mul_quant: valid_rows must be int32 (psum count)");
valid_rows_ptr = valid_rows->const_data_ptr<int32_t>();
}
const torch::stable::accelerator::DeviceGuard device_guard(
input.get_device_index());
const cudaStream_t stream = get_current_cuda_stream();
static constexpr int BLOCKS_PER_SM = 8; // matches kernel __launch_bounds__
static constexpr int SM_COUNT = 132; // H200 (GH100, 132 SMs)
static constexpr int GRID_DIM = SM_COUNT * BLOCKS_PER_SM;
STD_TORCH_CHECK(group_size == 128,
"situ_and_mul_quant: only group_size 128 (block-FP8) "
"supported, got ",
group_size);
STD_TORCH_CHECK(d % group_size == 0, "situ_and_mul_quant: d (", d,
") must be divisible by group_size ", group_size);
{
const int num_groups = d / (int)group_size;
STD_TORCH_CHECK(scale.size(-1) == num_groups &&
scale.numel() == num_tokens * (int64_t)num_groups,
"situ_and_mul_quant: scale shape must be "
"[num_tokens, d/group_size]");
dim3 grid(GRID_DIM);
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
input.scalar_type(), "situ_and_mul_quant_group_kernel", [&] {
VLLM_STABLE_DISPATCH_FP8_TYPES(
out.scalar_type(), "situ_and_mul_quant_group_kernel_fp8", [&] {
#ifndef USE_ROCM
// The pipelined kernel's float2-per-lane geometry assumes a
// 32-lane warp (GROUP_SIZE == 4 * WARP_SIZE); on HIP (64-lane)
// fall back to the WARP_SIZE-generic scalar kernel.
constexpr int THREADS = 256;
constexpr int SITU_D = 3072; // Kimi-K3 fused w2 input dim
if constexpr (sizeof(scalar_t) == 2) {
if (d == SITU_D && (float)beta == vllm::SITU_BETA &&
(float)linear_beta == vllm::SITU_LINEAR_BETA) {
constexpr int D = SITU_D;
constexpr int GROUP_STAGES = 4;
constexpr int NUM_WARPS = THREADS / 32;
constexpr int STAGE_ELTS = 2 * 128;
dim3 block(THREADS);
size_t smem_bytes = (size_t)NUM_WARPS * GROUP_STAGES *
STAGE_ELTS * sizeof(scalar_t);
vllm::situ_and_mul_quant_group_pipelined_kernel<
scalar_t, fp8_t, THREADS, GROUP_STAGES, 128, GRID_DIM,
D><<<grid, block, smem_bytes, stream>>>(
out.mutable_data_ptr<fp8_t>(),
scale.mutable_data_ptr<float>(),
input.const_data_ptr<scalar_t>(), num_tokens,
valid_rows_ptr, topk);
return;
}
}
#endif
const int num_warps = std::min(num_groups, 1024 / WARP_SIZE);
dim3 block(num_warps * WARP_SIZE);
vllm::situ_and_mul_quant_group_scalar_kernel<scalar_t, fp8_t,
128>
<<<grid, block, 0, stream>>>(
out.mutable_data_ptr<fp8_t>(),
scale.mutable_data_ptr<float>(),
input.const_data_ptr<scalar_t>(), d, num_groups,
(float)beta, (float)linear_beta, num_tokens,
valid_rows_ptr, topk);
});
});
}
}
namespace vllm {
// Element-wise activation kernel template.
+10 -2
View File
@@ -526,8 +526,16 @@ void fatrelu_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input,
double threshold);
void swigluoai_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input,
double alpha = 1.702, double limit = 7.0);
void situ_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input,
double beta = 1.0, double linear_beta = -1.0);
void situ_and_mul(
torch::stable::Tensor& out, torch::stable::Tensor& input, double beta = 1.0,
double linear_beta = -1.0,
std::optional<torch::stable::Tensor> valid_rows = std::nullopt);
void situ_and_mul_quant(
torch::stable::Tensor& out, torch::stable::Tensor& scale,
torch::stable::Tensor& input, double beta = 1.0, double linear_beta = -1.0,
int64_t group_size = 0,
std::optional<torch::stable::Tensor> valid_rows = std::nullopt,
int64_t topk = 1);
void masked_situ_and_mul(torch::stable::Tensor& out,
torch::stable::Tensor& input,
const torch::stable::Tensor& expert_num_tokens,
+10 -1
View File
@@ -611,7 +611,15 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
// SituGLU implementation used in Kimi models.
ops.def(
"situ_and_mul(Tensor! out, Tensor input, float beta=1.0, float "
"linear_beta=-1.0) -> ()");
"linear_beta=-1.0, Tensor? valid_rows=None) -> ()");
// Fused SituGLU activation + dynamic FP8 quantization for the Humming w2 path
// (writes the fp8 down input and its float32 scale). group_size=0 ->
// per-token scale [.., 1]; group_size=128 -> k-major block-FP8 scale [..,
// d/128].
ops.def(
"situ_and_mul_quant(Tensor! out, Tensor! scale, Tensor input, "
"float beta=1.0, float linear_beta=-1.0, int group_size=0, "
"Tensor? valid_rows=None, int topk=1) -> ()");
ops.def(
"masked_situ_and_mul(Tensor! out, Tensor input, Tensor "
"expert_num_tokens, float beta=1.0, float linear_beta=-1.0) -> ()");
@@ -830,6 +838,7 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
ops.impl("fatrelu_and_mul", TORCH_BOX(&fatrelu_and_mul));
ops.impl("swigluoai_and_mul", TORCH_BOX(&swigluoai_and_mul));
ops.impl("situ_and_mul", TORCH_BOX(&situ_and_mul));
ops.impl("situ_and_mul_quant", TORCH_BOX(&situ_and_mul_quant));
ops.impl("masked_situ_and_mul", TORCH_BOX(&masked_situ_and_mul));
ops.impl("gelu_new", TORCH_BOX(&gelu_new));
ops.impl("gelu_fast", TORCH_BOX(&gelu_fast));