[Refactor][Kernel] Add global helper to deduplicate vectorized memory ops (#35105)
Signed-off-by: LopezCastroRoberto <rocastro@redhat.com> Signed-off-by: LopezCastroRoberto <roberto.lopez.castro@udc.es> Signed-off-by: Roberto L. Castro <38211239+LopezCastroRoberto@users.noreply.github.com>
This commit is contained in:
+90
-205
@@ -5,117 +5,11 @@
|
||||
#include <cmath>
|
||||
|
||||
#include "cuda_compat.h"
|
||||
#include "cuda_vec_utils.cuh"
|
||||
#include "dispatch_utils.h"
|
||||
|
||||
namespace vllm {
|
||||
|
||||
struct alignas(32) u32x8_t {
|
||||
uint32_t u0, u1, u2, u3, u4, u5, u6, u7;
|
||||
};
|
||||
|
||||
__device__ __forceinline__ void ld256(u32x8_t& val, const u32x8_t* ptr) {
|
||||
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 1000 && \
|
||||
defined(CUDA_VERSION) && CUDA_VERSION >= 12090
|
||||
asm volatile("ld.global.nc.v8.u32 {%0,%1,%2,%3,%4,%5,%6,%7}, [%8];\n"
|
||||
: "=r"(val.u0), "=r"(val.u1), "=r"(val.u2), "=r"(val.u3),
|
||||
"=r"(val.u4), "=r"(val.u5), "=r"(val.u6), "=r"(val.u7)
|
||||
: "l"(ptr));
|
||||
#else
|
||||
const uint4* uint_ptr = reinterpret_cast<const uint4*>(ptr);
|
||||
uint4 top_half = __ldg(&uint_ptr[0]);
|
||||
uint4 bottom_half = __ldg(&uint_ptr[1]);
|
||||
val.u0 = top_half.x;
|
||||
val.u1 = top_half.y;
|
||||
val.u2 = top_half.z;
|
||||
val.u3 = top_half.w;
|
||||
val.u4 = bottom_half.x;
|
||||
val.u5 = bottom_half.y;
|
||||
val.u6 = bottom_half.z;
|
||||
val.u7 = bottom_half.w;
|
||||
#endif
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void st256(u32x8_t& val, u32x8_t* ptr) {
|
||||
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 1000 && \
|
||||
defined(CUDA_VERSION) && CUDA_VERSION >= 12090
|
||||
asm volatile("st.global.v8.u32 [%0], {%1,%2,%3,%4,%5,%6,%7,%8};\n"
|
||||
:
|
||||
: "l"(ptr), "r"(val.u0), "r"(val.u1), "r"(val.u2), "r"(val.u3),
|
||||
"r"(val.u4), "r"(val.u5), "r"(val.u6), "r"(val.u7)
|
||||
: "memory");
|
||||
#else
|
||||
uint4* uint_ptr = reinterpret_cast<uint4*>(ptr);
|
||||
uint_ptr[0] = make_uint4(val.u0, val.u1, val.u2, val.u3);
|
||||
uint_ptr[1] = make_uint4(val.u4, val.u5, val.u6, val.u7);
|
||||
#endif
|
||||
}
|
||||
|
||||
template <bool support_256>
|
||||
struct VecTraits;
|
||||
|
||||
template <>
|
||||
struct VecTraits<true> {
|
||||
static constexpr int ARCH_MAX_VEC_SIZE = 32;
|
||||
using vec_t = u32x8_t;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct VecTraits<false> {
|
||||
static constexpr int ARCH_MAX_VEC_SIZE = 16;
|
||||
using vec_t = int4;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct PackedTraits;
|
||||
|
||||
template <>
|
||||
struct PackedTraits<c10::BFloat16> {
|
||||
using packed_t = __nv_bfloat162;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct PackedTraits<c10::Half> {
|
||||
using packed_t = __half2;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct PackedTraits<float> {
|
||||
using packed_t = float2;
|
||||
};
|
||||
|
||||
template <typename packed_t>
|
||||
__device__ __forceinline__ float2 cast_to_float2(const packed_t& val) {
|
||||
if constexpr (std::is_same_v<packed_t, __nv_bfloat162>) {
|
||||
return __bfloat1622float2(val);
|
||||
} else if constexpr (std::is_same_v<packed_t, __half2>) {
|
||||
return __half22float2(val);
|
||||
} else if constexpr (std::is_same_v<packed_t, float2>) {
|
||||
return float2(val);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename packed_t>
|
||||
__device__ __forceinline__ packed_t cast_to_packed(const float2& val) {
|
||||
if constexpr (std::is_same_v<packed_t, __nv_bfloat162>) {
|
||||
return __float22bfloat162_rn(val);
|
||||
} else if constexpr (std::is_same_v<packed_t, __half2>) {
|
||||
return __float22half2_rn(val);
|
||||
} else if constexpr (std::is_same_v<packed_t, float2>) {
|
||||
return float2(val);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename packed_t>
|
||||
__device__ __forceinline__ packed_t packed_mul(const packed_t& x,
|
||||
const packed_t& y) {
|
||||
if constexpr (std::is_same_v<packed_t, __nv_bfloat162> ||
|
||||
std::is_same_v<packed_t, __half2>) {
|
||||
return __hmul2(x, y);
|
||||
} else if constexpr (std::is_same_v<packed_t, float2>) {
|
||||
return make_float2(x.x * y.x, x.y * y.y);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&),
|
||||
bool act_first>
|
||||
__device__ __forceinline__ scalar_t compute(const scalar_t& x,
|
||||
@@ -131,16 +25,6 @@ __device__ __forceinline__ packed_t packed_compute(const packed_t& x,
|
||||
: packed_mul(x, PACKED_ACT_FN(y));
|
||||
}
|
||||
|
||||
// Check if all pointers are 16-byte aligned for int4 vectorized access
|
||||
__host__ __device__ __forceinline__ bool is_16byte_aligned(const void* ptr) {
|
||||
return (reinterpret_cast<uintptr_t>(ptr) & 15) == 0;
|
||||
}
|
||||
|
||||
// Check if all pointers are 16-byte aligned for longlong4_32a vectorized access
|
||||
__host__ __device__ __forceinline__ bool is_32byte_aligned(const void* ptr) {
|
||||
return (reinterpret_cast<uintptr_t>(ptr) & 31) == 0;
|
||||
}
|
||||
|
||||
// Activation and gating kernel template.
|
||||
template <typename scalar_t, typename packed_t,
|
||||
scalar_t (*ACT_FN)(const scalar_t&),
|
||||
@@ -155,36 +39,32 @@ __global__ void act_and_mul_kernel(
|
||||
scalar_t* out_ptr = out + blockIdx.x * d;
|
||||
|
||||
if constexpr (use_vec) {
|
||||
// Fast path: 128-bit/256-bit vectorized loop
|
||||
using vec_t = typename VecTraits<use_256b>::vec_t;
|
||||
constexpr int ARCH_MAX_VEC_SIZE = VecTraits<use_256b>::ARCH_MAX_VEC_SIZE;
|
||||
constexpr int VEC_SIZE = ARCH_MAX_VEC_SIZE / sizeof(packed_t);
|
||||
using cuda_t = typename CUDATypeConverter<scalar_t>::Type;
|
||||
using pvec_t = PackedVec<cuda_t, use_256b>;
|
||||
|
||||
const vec_t* x_vec = reinterpret_cast<const vec_t*>(x_ptr);
|
||||
const vec_t* y_vec = reinterpret_cast<const vec_t*>(y_ptr);
|
||||
vec_t* out_vec = reinterpret_cast<vec_t*>(out_ptr);
|
||||
const int num_vecs = d / 2 / VEC_SIZE;
|
||||
const pvec_t* x_vec = reinterpret_cast<const pvec_t*>(x_ptr);
|
||||
const pvec_t* y_vec = reinterpret_cast<const pvec_t*>(y_ptr);
|
||||
pvec_t* out_vec = reinterpret_cast<pvec_t*>(out_ptr);
|
||||
const int num_vecs = d / 2 / pvec_t::NUM_ELTS;
|
||||
|
||||
for (int i = threadIdx.x; i < num_vecs; i += blockDim.x) {
|
||||
vec_t x, y;
|
||||
pvec_t x, y;
|
||||
if constexpr (use_256b) {
|
||||
ld256(x, &x_vec[i]);
|
||||
ld256(y, &y_vec[i]);
|
||||
} else {
|
||||
x = VLLM_LDG(&x_vec[i]);
|
||||
y = VLLM_LDG(&y_vec[i]);
|
||||
ld128(x, &x_vec[i]);
|
||||
ld128(y, &y_vec[i]);
|
||||
}
|
||||
auto* xp = reinterpret_cast<packed_t*>(&x);
|
||||
auto* yp = reinterpret_cast<packed_t*>(&y);
|
||||
#pragma unroll
|
||||
for (int j = 0; j < VEC_SIZE; j++) {
|
||||
xp[j] =
|
||||
packed_compute<packed_t, PACKED_ACT_FN, act_first>(xp[j], yp[j]);
|
||||
for (int j = 0; j < pvec_t::NUM_ELTS; j++) {
|
||||
x.elts[j] = packed_compute<packed_t, PACKED_ACT_FN, act_first>(
|
||||
x.elts[j], y.elts[j]);
|
||||
}
|
||||
if constexpr (use_256b) {
|
||||
st256(x, &out_vec[i]);
|
||||
} else {
|
||||
out_vec[i] = x;
|
||||
st128(x, &out_vec[i]);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -272,51 +152,54 @@ packed_gelu_tanh_kernel(const packed_t& val) {
|
||||
// Launch activation and gating kernel.
|
||||
// Use ACT_FIRST (bool) indicating whether to apply the activation function
|
||||
// first.
|
||||
#define LAUNCH_ACTIVATION_GATE_KERNEL(KERNEL, PACKED_KERNEL, ACT_FIRST) \
|
||||
auto dtype = input.scalar_type(); \
|
||||
int d = input.size(-1) / 2; \
|
||||
int64_t num_tokens = input.numel() / input.size(-1); \
|
||||
if (num_tokens == 0) { \
|
||||
return; \
|
||||
} \
|
||||
dim3 grid(num_tokens); \
|
||||
int cc_major = at::cuda::getCurrentDeviceProperties()->major; \
|
||||
int support_vec = (cc_major >= 10 && num_tokens > 128) ? 32 : 16; \
|
||||
int vec_size = support_vec / at::elementSize(dtype); \
|
||||
const bool use_vec = (d % vec_size == 0); \
|
||||
const at::cuda::OptionalCUDAGuard device_guard(device_of(input)); \
|
||||
const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); \
|
||||
if (use_vec) { \
|
||||
dim3 block(std::min(d / vec_size, 1024)); \
|
||||
if (cc_major >= 10 && num_tokens > 128) { \
|
||||
VLLM_DISPATCH_FLOATING_TYPES(dtype, "act_and_mul_kernel", [&] { \
|
||||
vllm::act_and_mul_kernel< \
|
||||
scalar_t, typename vllm::PackedTraits<scalar_t>::packed_t, \
|
||||
KERNEL<scalar_t>, \
|
||||
PACKED_KERNEL<typename vllm::PackedTraits<scalar_t>::packed_t>, \
|
||||
ACT_FIRST, true, true><<<grid, block, 0, stream>>>( \
|
||||
out.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(), d); \
|
||||
}); \
|
||||
} else { \
|
||||
VLLM_DISPATCH_FLOATING_TYPES(dtype, "act_and_mul_kernel", [&] { \
|
||||
vllm::act_and_mul_kernel< \
|
||||
scalar_t, typename vllm::PackedTraits<scalar_t>::packed_t, \
|
||||
KERNEL<scalar_t>, \
|
||||
PACKED_KERNEL<typename vllm::PackedTraits<scalar_t>::packed_t>, \
|
||||
ACT_FIRST, true, false><<<grid, block, 0, stream>>>( \
|
||||
out.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(), d); \
|
||||
}); \
|
||||
} \
|
||||
} else { \
|
||||
dim3 block(std::min(d, 1024)); \
|
||||
VLLM_DISPATCH_FLOATING_TYPES(dtype, "act_and_mul_kernel", [&] { \
|
||||
vllm::act_and_mul_kernel< \
|
||||
scalar_t, typename vllm::PackedTraits<scalar_t>::packed_t, \
|
||||
KERNEL<scalar_t>, \
|
||||
PACKED_KERNEL<typename vllm::PackedTraits<scalar_t>::packed_t>, \
|
||||
ACT_FIRST, false><<<grid, block, 0, stream>>>( \
|
||||
out.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(), d); \
|
||||
}); \
|
||||
#define LAUNCH_ACTIVATION_GATE_KERNEL(KERNEL, PACKED_KERNEL, ACT_FIRST) \
|
||||
auto dtype = input.scalar_type(); \
|
||||
int d = input.size(-1) / 2; \
|
||||
int64_t num_tokens = input.numel() / input.size(-1); \
|
||||
if (num_tokens == 0) { \
|
||||
return; \
|
||||
} \
|
||||
dim3 grid(num_tokens); \
|
||||
int cc_major = at::cuda::getCurrentDeviceProperties()->major; \
|
||||
int support_vec = \
|
||||
(CUDA_VERSION >= 12090 && cc_major >= 10 && num_tokens > 128) \
|
||||
? vllm::VecTraits<true>::ARCH_MAX_VEC_SIZE \
|
||||
: vllm::VecTraits<false>::ARCH_MAX_VEC_SIZE; \
|
||||
int vec_size = support_vec / at::elementSize(dtype); \
|
||||
const bool use_vec = (d % vec_size == 0); \
|
||||
const at::cuda::OptionalCUDAGuard device_guard(device_of(input)); \
|
||||
const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); \
|
||||
if (use_vec) { \
|
||||
dim3 block(std::min(d / vec_size, 1024)); \
|
||||
if (CUDA_VERSION >= 12090 && cc_major >= 10 && num_tokens > 128) { \
|
||||
VLLM_DISPATCH_FLOATING_TYPES(dtype, "act_and_mul_kernel", [&] { \
|
||||
vllm::act_and_mul_kernel< \
|
||||
scalar_t, typename vllm::PackedTypeConverter<scalar_t>::Type, \
|
||||
KERNEL<scalar_t>, \
|
||||
PACKED_KERNEL<typename vllm::PackedTypeConverter<scalar_t>::Type>, \
|
||||
ACT_FIRST, true, true><<<grid, block, 0, stream>>>( \
|
||||
out.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(), d); \
|
||||
}); \
|
||||
} else { \
|
||||
VLLM_DISPATCH_FLOATING_TYPES(dtype, "act_and_mul_kernel", [&] { \
|
||||
vllm::act_and_mul_kernel< \
|
||||
scalar_t, typename vllm::PackedTypeConverter<scalar_t>::Type, \
|
||||
KERNEL<scalar_t>, \
|
||||
PACKED_KERNEL<typename vllm::PackedTypeConverter<scalar_t>::Type>, \
|
||||
ACT_FIRST, true, false><<<grid, block, 0, stream>>>( \
|
||||
out.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(), d); \
|
||||
}); \
|
||||
} \
|
||||
} else { \
|
||||
dim3 block(std::min(d, 1024)); \
|
||||
VLLM_DISPATCH_FLOATING_TYPES(dtype, "act_and_mul_kernel", [&] { \
|
||||
vllm::act_and_mul_kernel< \
|
||||
scalar_t, typename vllm::PackedTypeConverter<scalar_t>::Type, \
|
||||
KERNEL<scalar_t>, \
|
||||
PACKED_KERNEL<typename vllm::PackedTypeConverter<scalar_t>::Type>, \
|
||||
ACT_FIRST, false><<<grid, block, 0, stream>>>( \
|
||||
out.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(), d); \
|
||||
}); \
|
||||
}
|
||||
|
||||
void silu_and_mul(torch::Tensor& out, // [..., d]
|
||||
@@ -378,35 +261,31 @@ __global__ void act_and_mul_kernel_with_param(
|
||||
scalar_t* out_ptr = out + blockIdx.x * d;
|
||||
|
||||
if constexpr (use_vec) {
|
||||
// Fast path: 128-bit/256-bit vectorized loop
|
||||
using vec_t = typename VecTraits<use_256b>::vec_t;
|
||||
constexpr int ARCH_MAX_VEC_SIZE = VecTraits<use_256b>::ARCH_MAX_VEC_SIZE;
|
||||
constexpr int VEC_SIZE = ARCH_MAX_VEC_SIZE / sizeof(packed_t);
|
||||
using cuda_t = typename CUDATypeConverter<scalar_t>::Type;
|
||||
using pvec_t = PackedVec<cuda_t, use_256b>;
|
||||
|
||||
const vec_t* x_vec = reinterpret_cast<const vec_t*>(x_ptr);
|
||||
const vec_t* y_vec = reinterpret_cast<const vec_t*>(y_ptr);
|
||||
vec_t* out_vec = reinterpret_cast<vec_t*>(out_ptr);
|
||||
const int num_vecs = d / 2 / VEC_SIZE;
|
||||
const pvec_t* x_vec = reinterpret_cast<const pvec_t*>(x_ptr);
|
||||
const pvec_t* y_vec = reinterpret_cast<const pvec_t*>(y_ptr);
|
||||
pvec_t* out_vec = reinterpret_cast<pvec_t*>(out_ptr);
|
||||
const int num_vecs = d / 2 / pvec_t::NUM_ELTS;
|
||||
|
||||
for (int i = threadIdx.x; i < num_vecs; i += blockDim.x) {
|
||||
vec_t x, y;
|
||||
pvec_t x, y;
|
||||
if constexpr (use_256b) {
|
||||
ld256(x, &x_vec[i]);
|
||||
ld256(y, &y_vec[i]);
|
||||
} else {
|
||||
x = VLLM_LDG(&x_vec[i]);
|
||||
y = VLLM_LDG(&y_vec[i]);
|
||||
ld128(x, &x_vec[i]);
|
||||
ld128(y, &y_vec[i]);
|
||||
}
|
||||
auto* xp = reinterpret_cast<packed_t*>(&x);
|
||||
auto* yp = reinterpret_cast<packed_t*>(&y);
|
||||
#pragma unroll
|
||||
for (int j = 0; j < VEC_SIZE; j++) {
|
||||
xp[j] = packed_mul(PACKED_ACT_FN(xp[j], param), yp[j]);
|
||||
for (int j = 0; j < pvec_t::NUM_ELTS; j++) {
|
||||
x.elts[j] = packed_mul(PACKED_ACT_FN(x.elts[j], param), y.elts[j]);
|
||||
}
|
||||
if constexpr (use_256b) {
|
||||
st256(x, &out_vec[i]);
|
||||
} else {
|
||||
out_vec[i] = x;
|
||||
st128(x, &out_vec[i]);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -499,21 +378,24 @@ __global__ void swigluoai_and_mul_kernel(
|
||||
} \
|
||||
dim3 grid(num_tokens); \
|
||||
int cc_major = at::cuda::getCurrentDeviceProperties()->major; \
|
||||
int support_vec = (cc_major >= 10 && num_tokens > 128) ? 32 : 16; \
|
||||
int support_vec = \
|
||||
(CUDA_VERSION >= 12090 && cc_major >= 10 && num_tokens > 128) \
|
||||
? vllm::VecTraits<true>::ARCH_MAX_VEC_SIZE \
|
||||
: vllm::VecTraits<false>::ARCH_MAX_VEC_SIZE; \
|
||||
int vec_size = support_vec / at::elementSize(dtype); \
|
||||
const bool use_vec = (d % vec_size == 0); \
|
||||
const at::cuda::OptionalCUDAGuard device_guard(device_of(input)); \
|
||||
const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); \
|
||||
if (use_vec) { \
|
||||
dim3 block(std::min(d / vec_size, 1024)); \
|
||||
if (cc_major >= 10 && num_tokens > 128) { \
|
||||
if (CUDA_VERSION >= 12090 && cc_major >= 10 && num_tokens > 128) { \
|
||||
VLLM_DISPATCH_FLOATING_TYPES( \
|
||||
dtype, "act_and_mul_kernel_with_param", [&] { \
|
||||
vllm::act_and_mul_kernel_with_param< \
|
||||
scalar_t, typename vllm::PackedTraits<scalar_t>::packed_t, \
|
||||
scalar_t, typename vllm::PackedTypeConverter<scalar_t>::Type, \
|
||||
KERNEL<scalar_t>, \
|
||||
PACKED_KERNEL< \
|
||||
typename vllm::PackedTraits<scalar_t>::packed_t>, \
|
||||
typename vllm::PackedTypeConverter<scalar_t>::Type>, \
|
||||
true, true><<<grid, block, 0, stream>>>( \
|
||||
out.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(), d, \
|
||||
PARAM); \
|
||||
@@ -522,10 +404,10 @@ __global__ void swigluoai_and_mul_kernel(
|
||||
VLLM_DISPATCH_FLOATING_TYPES( \
|
||||
dtype, "act_and_mul_kernel_with_param", [&] { \
|
||||
vllm::act_and_mul_kernel_with_param< \
|
||||
scalar_t, typename vllm::PackedTraits<scalar_t>::packed_t, \
|
||||
scalar_t, typename vllm::PackedTypeConverter<scalar_t>::Type, \
|
||||
KERNEL<scalar_t>, \
|
||||
PACKED_KERNEL< \
|
||||
typename vllm::PackedTraits<scalar_t>::packed_t>, \
|
||||
typename vllm::PackedTypeConverter<scalar_t>::Type>, \
|
||||
true, false><<<grid, block, 0, stream>>>( \
|
||||
out.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(), d, \
|
||||
PARAM); \
|
||||
@@ -535,9 +417,9 @@ __global__ void swigluoai_and_mul_kernel(
|
||||
dim3 block(std::min(d, 1024)); \
|
||||
VLLM_DISPATCH_FLOATING_TYPES(dtype, "act_and_mul_kernel_with_param", [&] { \
|
||||
vllm::act_and_mul_kernel_with_param< \
|
||||
scalar_t, typename vllm::PackedTraits<scalar_t>::packed_t, \
|
||||
scalar_t, typename vllm::PackedTypeConverter<scalar_t>::Type, \
|
||||
KERNEL<scalar_t>, \
|
||||
PACKED_KERNEL<typename vllm::PackedTraits<scalar_t>::packed_t>, \
|
||||
PACKED_KERNEL<typename vllm::PackedTypeConverter<scalar_t>::Type>, \
|
||||
false><<<grid, block, 0, stream>>>( \
|
||||
out.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(), d, PARAM); \
|
||||
}); \
|
||||
@@ -629,14 +511,17 @@ __global__ void activation_kernel(
|
||||
} \
|
||||
dim3 grid(num_tokens); \
|
||||
int cc_major = at::cuda::getCurrentDeviceProperties()->major; \
|
||||
int support_vec = (cc_major >= 10 && num_tokens > 128) ? 32 : 16; \
|
||||
int support_vec = \
|
||||
(CUDA_VERSION >= 12090 && cc_major >= 10 && num_tokens > 128) \
|
||||
? vllm::VecTraits<true>::ARCH_MAX_VEC_SIZE \
|
||||
: vllm::VecTraits<false>::ARCH_MAX_VEC_SIZE; \
|
||||
int vec_size = support_vec / at::elementSize(dtype); \
|
||||
const bool use_vec = (d % vec_size == 0); \
|
||||
const at::cuda::OptionalCUDAGuard device_guard(device_of(input)); \
|
||||
const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); \
|
||||
if (use_vec) { \
|
||||
dim3 block(std::min(d / vec_size, 1024)); \
|
||||
if (cc_major >= 10 && num_tokens > 128) { \
|
||||
if (CUDA_VERSION >= 12090 && cc_major >= 10 && num_tokens > 128) { \
|
||||
VLLM_DISPATCH_FLOATING_TYPES(dtype, "activation_kernel", [&] { \
|
||||
vllm::activation_kernel<scalar_t, KERNEL<scalar_t>, true, true> \
|
||||
<<<grid, block, 0, stream>>>(out.data_ptr<scalar_t>(), \
|
||||
|
||||
Reference in New Issue
Block a user