[Kernel] Implement CUDA kernel for ReLUSquaredActivation (relu^2) (#39058)

Signed-off-by: Tanish Malekar <tanishmalekar32@gmail.com>
Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Tanish Malekar
2026-07-12 19:18:03 -07:00
committed by GitHub
co-authored by Claude Sonnet 4.6
parent 4c81772e8b
commit e26264f3ef
7 changed files with 140 additions and 2 deletions
@@ -669,6 +669,14 @@ __device__ __forceinline__ T gelu_quick_kernel(const T& x) {
return (T)(((float)x) / (1.0f + expf(-1.702f * (float)x)));
}
template <typename T>
__device__ __forceinline__ T relu_squared_kernel(const T& x) {
// relu(x)^2 — introduced in https://arxiv.org/abs/2109.08668v2
const float f = (float)x;
const float val = f > 0.0f ? f : 0.0f;
return (T)(val * val);
}
} // namespace vllm
void gelu_new(torch::stable::Tensor& out, // [..., d]
@@ -688,3 +696,9 @@ void gelu_quick(torch::stable::Tensor& out, // [..., d]
{
LAUNCH_ACTIVATION_KERNEL(vllm::gelu_quick_kernel);
}
void relu_squared(torch::stable::Tensor& out, // [..., d]
torch::stable::Tensor& input) // [..., d]
{
LAUNCH_ACTIVATION_KERNEL(vllm::relu_squared_kernel);
}
+2
View File
@@ -414,6 +414,8 @@ void gelu_new(torch::stable::Tensor& out, torch::stable::Tensor& input);
void gelu_fast(torch::stable::Tensor& out, torch::stable::Tensor& input);
void gelu_quick(torch::stable::Tensor& out, torch::stable::Tensor& input);
void relu_squared(torch::stable::Tensor& out, torch::stable::Tensor& input);
// INT8 quantization kernels (shared CUDA/ROCm)
void static_scaled_int8_quant(torch::stable::Tensor& out,
torch::stable::Tensor const& input,
+4
View File
@@ -539,6 +539,9 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
// Quick GELU implementation.
ops.def("gelu_quick(Tensor! out, Tensor input) -> ()");
// relu(x)^2 activation from https://arxiv.org/abs/2109.08668v2
ops.def("relu_squared(Tensor! out, Tensor input) -> ()");
// Compute int8 quantized tensor for given scaling factor.
ops.def(
"static_scaled_int8_quant(Tensor! result, Tensor input, Tensor scale,"
@@ -715,6 +718,7 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
ops.impl("gelu_new", TORCH_BOX(&gelu_new));
ops.impl("gelu_fast", TORCH_BOX(&gelu_fast));
ops.impl("gelu_quick", TORCH_BOX(&gelu_quick));
ops.impl("relu_squared", TORCH_BOX(&relu_squared));
ops.impl("silu_and_mul_with_clamp", TORCH_BOX(&silu_and_mul_clamp));
// INT8 quantization kernels
+2
View File
@@ -43,6 +43,8 @@ void gelu_fast(torch::Tensor& out, torch::Tensor& input);
void gelu_quick(torch::Tensor& out, torch::Tensor& input);
void relu_squared(torch::Tensor& out, torch::Tensor& input);
void static_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
torch::Tensor const& scale,
std::optional<torch::Tensor> const& azp);