[Kernel] Implement CUDA kernel for ReLUSquaredActivation (relu^2) (#39058)
Signed-off-by: Tanish Malekar <tanishmalekar32@gmail.com> Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
parent
4c81772e8b
commit
e26264f3ef
@@ -669,6 +669,14 @@ __device__ __forceinline__ T gelu_quick_kernel(const T& x) {
|
||||
return (T)(((float)x) / (1.0f + expf(-1.702f * (float)x)));
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__device__ __forceinline__ T relu_squared_kernel(const T& x) {
|
||||
// relu(x)^2 — introduced in https://arxiv.org/abs/2109.08668v2
|
||||
const float f = (float)x;
|
||||
const float val = f > 0.0f ? f : 0.0f;
|
||||
return (T)(val * val);
|
||||
}
|
||||
|
||||
} // namespace vllm
|
||||
|
||||
void gelu_new(torch::stable::Tensor& out, // [..., d]
|
||||
@@ -688,3 +696,9 @@ void gelu_quick(torch::stable::Tensor& out, // [..., d]
|
||||
{
|
||||
LAUNCH_ACTIVATION_KERNEL(vllm::gelu_quick_kernel);
|
||||
}
|
||||
|
||||
void relu_squared(torch::stable::Tensor& out, // [..., d]
|
||||
torch::stable::Tensor& input) // [..., d]
|
||||
{
|
||||
LAUNCH_ACTIVATION_KERNEL(vllm::relu_squared_kernel);
|
||||
}
|
||||
|
||||
@@ -414,6 +414,8 @@ void gelu_new(torch::stable::Tensor& out, torch::stable::Tensor& input);
|
||||
void gelu_fast(torch::stable::Tensor& out, torch::stable::Tensor& input);
|
||||
void gelu_quick(torch::stable::Tensor& out, torch::stable::Tensor& input);
|
||||
|
||||
void relu_squared(torch::stable::Tensor& out, torch::stable::Tensor& input);
|
||||
|
||||
// INT8 quantization kernels (shared CUDA/ROCm)
|
||||
void static_scaled_int8_quant(torch::stable::Tensor& out,
|
||||
torch::stable::Tensor const& input,
|
||||
|
||||
@@ -539,6 +539,9 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
// Quick GELU implementation.
|
||||
ops.def("gelu_quick(Tensor! out, Tensor input) -> ()");
|
||||
|
||||
// relu(x)^2 activation from https://arxiv.org/abs/2109.08668v2
|
||||
ops.def("relu_squared(Tensor! out, Tensor input) -> ()");
|
||||
|
||||
// Compute int8 quantized tensor for given scaling factor.
|
||||
ops.def(
|
||||
"static_scaled_int8_quant(Tensor! result, Tensor input, Tensor scale,"
|
||||
@@ -715,6 +718,7 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
|
||||
ops.impl("gelu_new", TORCH_BOX(&gelu_new));
|
||||
ops.impl("gelu_fast", TORCH_BOX(&gelu_fast));
|
||||
ops.impl("gelu_quick", TORCH_BOX(&gelu_quick));
|
||||
ops.impl("relu_squared", TORCH_BOX(&relu_squared));
|
||||
ops.impl("silu_and_mul_with_clamp", TORCH_BOX(&silu_and_mul_clamp));
|
||||
|
||||
// INT8 quantization kernels
|
||||
|
||||
@@ -43,6 +43,8 @@ void gelu_fast(torch::Tensor& out, torch::Tensor& input);
|
||||
|
||||
void gelu_quick(torch::Tensor& out, torch::Tensor& input);
|
||||
|
||||
void relu_squared(torch::Tensor& out, torch::Tensor& input);
|
||||
|
||||
void static_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
|
||||
torch::Tensor const& scale,
|
||||
std::optional<torch::Tensor> const& azp);
|
||||
|
||||
Reference in New Issue
Block a user