[Perf] RMSNorm uncontiguous support, 1.2~3.1x kernel performance improvement (#49750)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
Wentao Ye
2026-07-29 13:45:38 -07:00
committed by GitHub
parent 93477454f9
commit 82642d7d6c
9 changed files with 31 additions and 29 deletions
+15 -11
View File
@@ -110,7 +110,8 @@ fused_add_rms_norm_kernel(
const int64_t input_stride,
scalar_t* __restrict__ residual, // [..., hidden_size]
const scalar_t* __restrict__ weight, // [hidden_size], null if !HasWeight
const float epsilon, const int num_tokens, const int hidden_size) {
const float epsilon, const int num_tokens, const int hidden_size,
const int64_t residual_stride) {
// Sanity checks on our vector struct and type-punned pointer arithmetic
static_assert(std::is_pod_v<_f16Vec<scalar_t, width>>);
static_assert(sizeof(_f16Vec<scalar_t, width>) == sizeof(scalar_t) * width);
@@ -130,7 +131,7 @@ fused_add_rms_norm_kernel(
reinterpret_cast<const _f16Vec<scalar_t, width>*>(weight);
for (int idx = threadIdx.x; idx < vec_hidden_size; idx += blockDim.x) {
int id = blockIdx.x * vec_hidden_size + idx;
int64_t id = blockIdx.x * residual_stride / width + idx;
int64_t strided_id = blockIdx.x * vec_input_stride + idx;
_f16Vec<scalar_t, width> temp = input_v[strided_id];
temp += residual_v[id];
@@ -148,7 +149,7 @@ fused_add_rms_norm_kernel(
__syncthreads();
for (int idx = threadIdx.x; idx < vec_hidden_size; idx += blockDim.x) {
int id = blockIdx.x * vec_hidden_size + idx;
int64_t id = blockIdx.x * residual_stride / width + idx;
int64_t strided_id = blockIdx.x * vec_input_stride + idx;
_f16Vec<scalar_t, width> res = residual_v[id];
_f16Vec<scalar_t, width> out;
@@ -182,16 +183,17 @@ fused_add_rms_norm_kernel(
const int64_t input_stride,
scalar_t* __restrict__ residual, // [..., hidden_size]
const scalar_t* __restrict__ weight, // [hidden_size], null if !HasWeight
const float epsilon, const int num_tokens, const int hidden_size) {
const float epsilon, const int num_tokens, const int hidden_size,
const int64_t residual_stride) {
__shared__ float s_variance;
float variance = 0.0f;
for (int idx = threadIdx.x; idx < hidden_size; idx += blockDim.x) {
scalar_t z = input[blockIdx.x * input_stride + idx];
z += residual[blockIdx.x * hidden_size + idx];
z += residual[blockIdx.x * residual_stride + idx];
float x = (float)z;
variance += x * x;
residual[blockIdx.x * hidden_size + idx] = z;
residual[blockIdx.x * residual_stride + idx] = z;
}
using BlockReduce = cub::BlockReduce<float, 1024>;
@@ -204,7 +206,7 @@ fused_add_rms_norm_kernel(
__syncthreads();
for (int idx = threadIdx.x; idx < hidden_size; idx += blockDim.x) {
float x = (float)residual[blockIdx.x * hidden_size + idx];
float x = (float)residual[blockIdx.x * residual_stride + idx];
if constexpr (HasWeight) {
float w = (float)weight[idx];
input[blockIdx.x * input_stride + idx] = (scalar_t)(x * s_variance * w);
@@ -299,13 +301,13 @@ void rms_norm(torch::stable::Tensor& out, // [..., hidden_size]
input.mutable_data_ptr<scalar_t>(), input_stride, \
residual.mutable_data_ptr<scalar_t>(), \
weight->const_data_ptr<scalar_t>(), epsilon, num_tokens, \
hidden_size); \
hidden_size, residual_stride); \
} else { \
vllm::fused_add_rms_norm_kernel<scalar_t, width, false> \
<<<grid, block, 0, stream>>>( \
input.mutable_data_ptr<scalar_t>(), input_stride, \
residual.mutable_data_ptr<scalar_t>(), nullptr, epsilon, \
num_tokens, hidden_size); \
num_tokens, hidden_size, residual_stride); \
} \
});
@@ -314,13 +316,14 @@ void fused_add_rms_norm(torch::stable::Tensor& input, // [..., hidden_size]
std::optional<torch::stable::Tensor> weight,
double epsilon) {
STD_TORCH_CHECK(input.scalar_type() == residual.scalar_type());
STD_TORCH_CHECK(residual.is_contiguous());
STD_TORCH_CHECK(residual.stride(-1) == 1);
if (weight.has_value()) {
STD_TORCH_CHECK(weight->scalar_type() == input.scalar_type());
STD_TORCH_CHECK(weight->is_contiguous());
}
int hidden_size = input.size(-1);
int64_t input_stride = input.stride(-2);
int64_t residual_stride = residual.stride(-2);
int num_tokens = input.numel() / hidden_size;
dim3 grid(num_tokens);
@@ -343,7 +346,8 @@ void fused_add_rms_norm(torch::stable::Tensor& input, // [..., hidden_size]
auto inp_ptr = reinterpret_cast<std::uintptr_t>(input.data_ptr());
auto res_ptr = reinterpret_cast<std::uintptr_t>(residual.data_ptr());
bool offsets_are_multiple_of_vector_width =
hidden_size % vector_width == 0 && input_stride % vector_width == 0;
hidden_size % vector_width == 0 && input_stride % vector_width == 0 &&
residual_stride % vector_width == 0;
const bool has_weight = weight.has_value();
if (has_weight) {
auto wt_ptr = reinterpret_cast<std::uintptr_t>(weight->data_ptr());