[Kernel] PDL enablement for fusedQKNormRopeKernel (#55755)
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
@@ -189,6 +189,12 @@ __global__ void fusedQKNormRopeKernel(
|
||||
}
|
||||
int offsetThread = offsetWarp + laneId * numElemsPerThread;
|
||||
|
||||
// PDL: wait for the predecessor kernel (the QKV projection that produces
|
||||
// `qkv`) to complete before touching any global memory.
|
||||
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
|
||||
cudaGridDependencySynchronize();
|
||||
#endif
|
||||
|
||||
// Sum of squares for RMSNorm
|
||||
float sumOfSquares = 0.0f;
|
||||
|
||||
@@ -300,6 +306,11 @@ __global__ void fusedQKNormRopeKernel(
|
||||
*reinterpret_cast<vec_T*>(&qkv[offsetThread]) = vec;
|
||||
}
|
||||
|
||||
// PDL: signal completion so a dependent successor may launch early.
|
||||
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
#endif
|
||||
|
||||
#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)
|
||||
}
|
||||
#endif
|
||||
@@ -385,6 +396,13 @@ __global__ void fusedQKNormRopeKernelNTokenHeads(
|
||||
smem_storage + cos_sin_bytes +
|
||||
warpId * (HEADS_PER_WARP * qkv_tile_bytes);
|
||||
|
||||
// PDL: wait for the predecessor kernel (the QKV projection that produces
|
||||
// `qkv`) to complete before touching any global memory -- cp.async reads
|
||||
// global memory, so this must precede the first commit group.
|
||||
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
|
||||
cudaGridDependencySynchronize();
|
||||
#endif
|
||||
|
||||
// === Group 0: async load all heads' QKV into smem (issued first). ===
|
||||
for (int k = 0; k < num_heads_this_warp; ++k) {
|
||||
int const localHeadIdx = first_head + k;
|
||||
@@ -531,6 +549,11 @@ __global__ void fusedQKNormRopeKernelNTokenHeads(
|
||||
}
|
||||
}
|
||||
|
||||
// PDL: signal completion so a dependent successor may launch early.
|
||||
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
#endif
|
||||
|
||||
#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)
|
||||
}
|
||||
#endif
|
||||
@@ -547,6 +570,42 @@ __global__ void fusedQKNormRopeKernelNTokenHeads(
|
||||
__VA_ARGS__ \
|
||||
}
|
||||
|
||||
namespace {
|
||||
// The empirical value for small batch: overlapping with the predecessor QKV
|
||||
// projection only pays off while this kernel is launch-bound.
|
||||
constexpr int kPDLEnableTokens = 16;
|
||||
|
||||
// Launch helper for PDL (programmatic dependent launch), which lets this
|
||||
// kernel start while the predecessor QKV projection is still draining. The
|
||||
// launch attribute is the only switch: with it off, the kernels'
|
||||
// cudaGridDependencySynchronize() / cudaTriggerProgrammaticLaunchCompletion()
|
||||
// are no-ops.
|
||||
template <typename KernelFn, typename... Args>
|
||||
void launchWithPDL(KernelFn kernel, dim3 gridDim, dim3 blockDim,
|
||||
size_t smem_bytes, cudaStream_t stream, bool enable_pdl,
|
||||
Args... args) {
|
||||
#ifndef USE_ROCM
|
||||
cudaLaunchConfig_t config;
|
||||
config.gridDim = gridDim;
|
||||
config.blockDim = blockDim;
|
||||
config.dynamicSmemBytes = smem_bytes;
|
||||
config.stream = stream;
|
||||
cudaLaunchAttribute attrs[1];
|
||||
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
|
||||
attrs[0].val.programmaticStreamSerializationAllowed = enable_pdl;
|
||||
config.numAttrs = 1;
|
||||
config.attrs = attrs;
|
||||
cudaLaunchKernelEx(&config, kernel, args...);
|
||||
#else
|
||||
// ROCm: standard kernel launch syntax (no PDL/stream serialization).
|
||||
(void)enable_pdl;
|
||||
// clang-format off
|
||||
kernel<<<gridDim, blockDim, smem_bytes, stream>>>(args...);
|
||||
// clang-format on
|
||||
#endif
|
||||
}
|
||||
} // namespace
|
||||
|
||||
template <typename scalar_t_in, typename scalar_t_cache>
|
||||
void launchFusedQKNormRope(void* qkv, int const num_tokens,
|
||||
int const num_heads_q, int const num_heads_k,
|
||||
@@ -562,29 +621,33 @@ void launchFusedQKNormRope(void* qkv, int const num_tokens,
|
||||
int const gridSize = common::divUp(totalWarps, warpsPerBlock);
|
||||
dim3 gridDim(gridSize);
|
||||
dim3 blockDim(blockSize);
|
||||
bool const enable_pdl = num_tokens <= kPDLEnableTokens;
|
||||
switch (head_dim) {
|
||||
case 64:
|
||||
DISPATCH_INTERLEAVE(interleave, INTERLEAVE, {
|
||||
fusedQKNormRopeKernel<scalar_t_in, scalar_t_cache, 64, INTERLEAVE>
|
||||
<<<gridDim, blockDim, 0, stream>>>(
|
||||
qkv, num_heads_q, num_heads_k, num_heads_v, eps, q_weight,
|
||||
k_weight, cos_sin_cache, position_ids, num_tokens, rotary_dim);
|
||||
launchWithPDL(
|
||||
fusedQKNormRopeKernel<scalar_t_in, scalar_t_cache, 64, INTERLEAVE>,
|
||||
gridDim, blockDim, 0, stream, enable_pdl, qkv, num_heads_q,
|
||||
num_heads_k, num_heads_v, eps, q_weight, k_weight, cos_sin_cache,
|
||||
position_ids, num_tokens, rotary_dim);
|
||||
});
|
||||
break;
|
||||
case 128:
|
||||
DISPATCH_INTERLEAVE(interleave, INTERLEAVE, {
|
||||
fusedQKNormRopeKernel<scalar_t_in, scalar_t_cache, 128, INTERLEAVE>
|
||||
<<<gridDim, blockDim, 0, stream>>>(
|
||||
qkv, num_heads_q, num_heads_k, num_heads_v, eps, q_weight,
|
||||
k_weight, cos_sin_cache, position_ids, num_tokens, rotary_dim);
|
||||
launchWithPDL(
|
||||
fusedQKNormRopeKernel<scalar_t_in, scalar_t_cache, 128, INTERLEAVE>,
|
||||
gridDim, blockDim, 0, stream, enable_pdl, qkv, num_heads_q,
|
||||
num_heads_k, num_heads_v, eps, q_weight, k_weight, cos_sin_cache,
|
||||
position_ids, num_tokens, rotary_dim);
|
||||
});
|
||||
break;
|
||||
case 256:
|
||||
DISPATCH_INTERLEAVE(interleave, INTERLEAVE, {
|
||||
fusedQKNormRopeKernel<scalar_t_in, scalar_t_cache, 256, INTERLEAVE>
|
||||
<<<gridDim, blockDim, 0, stream>>>(
|
||||
qkv, num_heads_q, num_heads_k, num_heads_v, eps, q_weight,
|
||||
k_weight, cos_sin_cache, position_ids, num_tokens, rotary_dim);
|
||||
launchWithPDL(
|
||||
fusedQKNormRopeKernel<scalar_t_in, scalar_t_cache, 256, INTERLEAVE>,
|
||||
gridDim, blockDim, 0, stream, enable_pdl, qkv, num_heads_q,
|
||||
num_heads_k, num_heads_v, eps, q_weight, k_weight, cos_sin_cache,
|
||||
position_ids, num_tokens, rotary_dim);
|
||||
});
|
||||
break;
|
||||
default:
|
||||
@@ -644,6 +707,7 @@ void launchFusedQKNormRopeNTokenHeads(
|
||||
int const gridSize = common::divUp(total_warps, warpsPerBlock);
|
||||
dim3 gridDim(gridSize);
|
||||
dim3 blockDim(blockSize);
|
||||
bool const enable_pdl = num_tokens <= kPDLEnableTokens;
|
||||
// Cache element size: float=4, bfloat16=2 (host-safe; kernel uses same
|
||||
// layout).
|
||||
size_t const cache_elem_size =
|
||||
@@ -656,42 +720,42 @@ void launchFusedQKNormRopeNTokenHeads(
|
||||
warpsPerBlock * static_cast<size_t>(rotary_dim) * cache_elem_size +
|
||||
warpsPerBlock * qkv_smem_per_warp;
|
||||
|
||||
#define LAUNCH_N_TOKEN_HEADS(N) \
|
||||
do { \
|
||||
switch (head_dim) { \
|
||||
case 64: \
|
||||
DISPATCH_INTERLEAVE(interleave, INTERLEAVE, { \
|
||||
fusedQKNormRopeKernelNTokenHeads<scalar_t_in, scalar_t_cache, 64, \
|
||||
INTERLEAVE, (N)> \
|
||||
<<<gridDim, blockDim, smem_bytes, stream>>>( \
|
||||
qkv, num_heads_q, num_heads_k, num_heads_v, eps, q_weight, \
|
||||
k_weight, cos_sin_cache, position_ids, num_tokens, \
|
||||
rotary_dim); \
|
||||
}); \
|
||||
break; \
|
||||
case 128: \
|
||||
DISPATCH_INTERLEAVE(interleave, INTERLEAVE, { \
|
||||
fusedQKNormRopeKernelNTokenHeads<scalar_t_in, scalar_t_cache, 128, \
|
||||
INTERLEAVE, (N)> \
|
||||
<<<gridDim, blockDim, smem_bytes, stream>>>( \
|
||||
qkv, num_heads_q, num_heads_k, num_heads_v, eps, q_weight, \
|
||||
k_weight, cos_sin_cache, position_ids, num_tokens, \
|
||||
rotary_dim); \
|
||||
}); \
|
||||
break; \
|
||||
case 256: \
|
||||
DISPATCH_INTERLEAVE(interleave, INTERLEAVE, { \
|
||||
fusedQKNormRopeKernelNTokenHeads<scalar_t_in, scalar_t_cache, 256, \
|
||||
INTERLEAVE, (N)> \
|
||||
<<<gridDim, blockDim, smem_bytes, stream>>>( \
|
||||
qkv, num_heads_q, num_heads_k, num_heads_v, eps, q_weight, \
|
||||
k_weight, cos_sin_cache, position_ids, num_tokens, \
|
||||
rotary_dim); \
|
||||
}); \
|
||||
break; \
|
||||
default: \
|
||||
STD_TORCH_CHECK(false, "Unsupported head dimension: ", head_dim); \
|
||||
} \
|
||||
#define LAUNCH_N_TOKEN_HEADS(N) \
|
||||
do { \
|
||||
switch (head_dim) { \
|
||||
case 64: \
|
||||
DISPATCH_INTERLEAVE(interleave, INTERLEAVE, { \
|
||||
launchWithPDL( \
|
||||
fusedQKNormRopeKernelNTokenHeads<scalar_t_in, scalar_t_cache, \
|
||||
64, INTERLEAVE, (N)>, \
|
||||
gridDim, blockDim, smem_bytes, stream, enable_pdl, qkv, \
|
||||
num_heads_q, num_heads_k, num_heads_v, eps, q_weight, k_weight, \
|
||||
cos_sin_cache, position_ids, num_tokens, rotary_dim); \
|
||||
}); \
|
||||
break; \
|
||||
case 128: \
|
||||
DISPATCH_INTERLEAVE(interleave, INTERLEAVE, { \
|
||||
launchWithPDL( \
|
||||
fusedQKNormRopeKernelNTokenHeads<scalar_t_in, scalar_t_cache, \
|
||||
128, INTERLEAVE, (N)>, \
|
||||
gridDim, blockDim, smem_bytes, stream, enable_pdl, qkv, \
|
||||
num_heads_q, num_heads_k, num_heads_v, eps, q_weight, k_weight, \
|
||||
cos_sin_cache, position_ids, num_tokens, rotary_dim); \
|
||||
}); \
|
||||
break; \
|
||||
case 256: \
|
||||
DISPATCH_INTERLEAVE(interleave, INTERLEAVE, { \
|
||||
launchWithPDL( \
|
||||
fusedQKNormRopeKernelNTokenHeads<scalar_t_in, scalar_t_cache, \
|
||||
256, INTERLEAVE, (N)>, \
|
||||
gridDim, blockDim, smem_bytes, stream, enable_pdl, qkv, \
|
||||
num_heads_q, num_heads_k, num_heads_v, eps, q_weight, k_weight, \
|
||||
cos_sin_cache, position_ids, num_tokens, rotary_dim); \
|
||||
}); \
|
||||
break; \
|
||||
default: \
|
||||
STD_TORCH_CHECK(false, "Unsupported head dimension: ", head_dim); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
if (token_heads_per_warp == 2) {
|
||||
|
||||
Reference in New Issue
Block a user