[Kernel] PDL enablement for fusedQKNormRopeKernel (#55755)

Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
Jee Jee Li
2026-09-07 20:49:03 -07:00
committed by GitHub
co-authored by Yongye Zhu
parent 607a6e48c5
commit 9c297e3b8b
+112 -48
View File
@@ -189,6 +189,12 @@ __global__ void fusedQKNormRopeKernel(
}
int offsetThread = offsetWarp + laneId * numElemsPerThread;
// PDL: wait for the predecessor kernel (the QKV projection that produces
// `qkv`) to complete before touching any global memory.
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
cudaGridDependencySynchronize();
#endif
// Sum of squares for RMSNorm
float sumOfSquares = 0.0f;
@@ -300,6 +306,11 @@ __global__ void fusedQKNormRopeKernel(
*reinterpret_cast<vec_T*>(&qkv[offsetThread]) = vec;
}
// PDL: signal completion so a dependent successor may launch early.
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
cudaTriggerProgrammaticLaunchCompletion();
#endif
#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)
}
#endif
@@ -385,6 +396,13 @@ __global__ void fusedQKNormRopeKernelNTokenHeads(
smem_storage + cos_sin_bytes +
warpId * (HEADS_PER_WARP * qkv_tile_bytes);
// PDL: wait for the predecessor kernel (the QKV projection that produces
// `qkv`) to complete before touching any global memory -- cp.async reads
// global memory, so this must precede the first commit group.
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
cudaGridDependencySynchronize();
#endif
// === Group 0: async load all heads' QKV into smem (issued first). ===
for (int k = 0; k < num_heads_this_warp; ++k) {
int const localHeadIdx = first_head + k;
@@ -531,6 +549,11 @@ __global__ void fusedQKNormRopeKernelNTokenHeads(
}
}
// PDL: signal completion so a dependent successor may launch early.
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
cudaTriggerProgrammaticLaunchCompletion();
#endif
#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)
}
#endif
@@ -547,6 +570,42 @@ __global__ void fusedQKNormRopeKernelNTokenHeads(
__VA_ARGS__ \
}
namespace {
// The empirical value for small batch: overlapping with the predecessor QKV
// projection only pays off while this kernel is launch-bound.
constexpr int kPDLEnableTokens = 16;
// Launch helper for PDL (programmatic dependent launch), which lets this
// kernel start while the predecessor QKV projection is still draining. The
// launch attribute is the only switch: with it off, the kernels'
// cudaGridDependencySynchronize() / cudaTriggerProgrammaticLaunchCompletion()
// are no-ops.
template <typename KernelFn, typename... Args>
void launchWithPDL(KernelFn kernel, dim3 gridDim, dim3 blockDim,
size_t smem_bytes, cudaStream_t stream, bool enable_pdl,
Args... args) {
#ifndef USE_ROCM
cudaLaunchConfig_t config;
config.gridDim = gridDim;
config.blockDim = blockDim;
config.dynamicSmemBytes = smem_bytes;
config.stream = stream;
cudaLaunchAttribute attrs[1];
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
attrs[0].val.programmaticStreamSerializationAllowed = enable_pdl;
config.numAttrs = 1;
config.attrs = attrs;
cudaLaunchKernelEx(&config, kernel, args...);
#else
// ROCm: standard kernel launch syntax (no PDL/stream serialization).
(void)enable_pdl;
// clang-format off
kernel<<<gridDim, blockDim, smem_bytes, stream>>>(args...);
// clang-format on
#endif
}
} // namespace
template <typename scalar_t_in, typename scalar_t_cache>
void launchFusedQKNormRope(void* qkv, int const num_tokens,
int const num_heads_q, int const num_heads_k,
@@ -562,29 +621,33 @@ void launchFusedQKNormRope(void* qkv, int const num_tokens,
int const gridSize = common::divUp(totalWarps, warpsPerBlock);
dim3 gridDim(gridSize);
dim3 blockDim(blockSize);
bool const enable_pdl = num_tokens <= kPDLEnableTokens;
switch (head_dim) {
case 64:
DISPATCH_INTERLEAVE(interleave, INTERLEAVE, {
fusedQKNormRopeKernel<scalar_t_in, scalar_t_cache, 64, INTERLEAVE>
<<<gridDim, blockDim, 0, stream>>>(
qkv, num_heads_q, num_heads_k, num_heads_v, eps, q_weight,
k_weight, cos_sin_cache, position_ids, num_tokens, rotary_dim);
launchWithPDL(
fusedQKNormRopeKernel<scalar_t_in, scalar_t_cache, 64, INTERLEAVE>,
gridDim, blockDim, 0, stream, enable_pdl, qkv, num_heads_q,
num_heads_k, num_heads_v, eps, q_weight, k_weight, cos_sin_cache,
position_ids, num_tokens, rotary_dim);
});
break;
case 128:
DISPATCH_INTERLEAVE(interleave, INTERLEAVE, {
fusedQKNormRopeKernel<scalar_t_in, scalar_t_cache, 128, INTERLEAVE>
<<<gridDim, blockDim, 0, stream>>>(
qkv, num_heads_q, num_heads_k, num_heads_v, eps, q_weight,
k_weight, cos_sin_cache, position_ids, num_tokens, rotary_dim);
launchWithPDL(
fusedQKNormRopeKernel<scalar_t_in, scalar_t_cache, 128, INTERLEAVE>,
gridDim, blockDim, 0, stream, enable_pdl, qkv, num_heads_q,
num_heads_k, num_heads_v, eps, q_weight, k_weight, cos_sin_cache,
position_ids, num_tokens, rotary_dim);
});
break;
case 256:
DISPATCH_INTERLEAVE(interleave, INTERLEAVE, {
fusedQKNormRopeKernel<scalar_t_in, scalar_t_cache, 256, INTERLEAVE>
<<<gridDim, blockDim, 0, stream>>>(
qkv, num_heads_q, num_heads_k, num_heads_v, eps, q_weight,
k_weight, cos_sin_cache, position_ids, num_tokens, rotary_dim);
launchWithPDL(
fusedQKNormRopeKernel<scalar_t_in, scalar_t_cache, 256, INTERLEAVE>,
gridDim, blockDim, 0, stream, enable_pdl, qkv, num_heads_q,
num_heads_k, num_heads_v, eps, q_weight, k_weight, cos_sin_cache,
position_ids, num_tokens, rotary_dim);
});
break;
default:
@@ -644,6 +707,7 @@ void launchFusedQKNormRopeNTokenHeads(
int const gridSize = common::divUp(total_warps, warpsPerBlock);
dim3 gridDim(gridSize);
dim3 blockDim(blockSize);
bool const enable_pdl = num_tokens <= kPDLEnableTokens;
// Cache element size: float=4, bfloat16=2 (host-safe; kernel uses same
// layout).
size_t const cache_elem_size =
@@ -656,42 +720,42 @@ void launchFusedQKNormRopeNTokenHeads(
warpsPerBlock * static_cast<size_t>(rotary_dim) * cache_elem_size +
warpsPerBlock * qkv_smem_per_warp;
#define LAUNCH_N_TOKEN_HEADS(N) \
do { \
switch (head_dim) { \
case 64: \
DISPATCH_INTERLEAVE(interleave, INTERLEAVE, { \
fusedQKNormRopeKernelNTokenHeads<scalar_t_in, scalar_t_cache, 64, \
INTERLEAVE, (N)> \
<<<gridDim, blockDim, smem_bytes, stream>>>( \
qkv, num_heads_q, num_heads_k, num_heads_v, eps, q_weight, \
k_weight, cos_sin_cache, position_ids, num_tokens, \
rotary_dim); \
}); \
break; \
case 128: \
DISPATCH_INTERLEAVE(interleave, INTERLEAVE, { \
fusedQKNormRopeKernelNTokenHeads<scalar_t_in, scalar_t_cache, 128, \
INTERLEAVE, (N)> \
<<<gridDim, blockDim, smem_bytes, stream>>>( \
qkv, num_heads_q, num_heads_k, num_heads_v, eps, q_weight, \
k_weight, cos_sin_cache, position_ids, num_tokens, \
rotary_dim); \
}); \
break; \
case 256: \
DISPATCH_INTERLEAVE(interleave, INTERLEAVE, { \
fusedQKNormRopeKernelNTokenHeads<scalar_t_in, scalar_t_cache, 256, \
INTERLEAVE, (N)> \
<<<gridDim, blockDim, smem_bytes, stream>>>( \
qkv, num_heads_q, num_heads_k, num_heads_v, eps, q_weight, \
k_weight, cos_sin_cache, position_ids, num_tokens, \
rotary_dim); \
}); \
break; \
default: \
STD_TORCH_CHECK(false, "Unsupported head dimension: ", head_dim); \
} \
#define LAUNCH_N_TOKEN_HEADS(N) \
do { \
switch (head_dim) { \
case 64: \
DISPATCH_INTERLEAVE(interleave, INTERLEAVE, { \
launchWithPDL( \
fusedQKNormRopeKernelNTokenHeads<scalar_t_in, scalar_t_cache, \
64, INTERLEAVE, (N)>, \
gridDim, blockDim, smem_bytes, stream, enable_pdl, qkv, \
num_heads_q, num_heads_k, num_heads_v, eps, q_weight, k_weight, \
cos_sin_cache, position_ids, num_tokens, rotary_dim); \
}); \
break; \
case 128: \
DISPATCH_INTERLEAVE(interleave, INTERLEAVE, { \
launchWithPDL( \
fusedQKNormRopeKernelNTokenHeads<scalar_t_in, scalar_t_cache, \
128, INTERLEAVE, (N)>, \
gridDim, blockDim, smem_bytes, stream, enable_pdl, qkv, \
num_heads_q, num_heads_k, num_heads_v, eps, q_weight, k_weight, \
cos_sin_cache, position_ids, num_tokens, rotary_dim); \
}); \
break; \
case 256: \
DISPATCH_INTERLEAVE(interleave, INTERLEAVE, { \
launchWithPDL( \
fusedQKNormRopeKernelNTokenHeads<scalar_t_in, scalar_t_cache, \
256, INTERLEAVE, (N)>, \
gridDim, blockDim, smem_bytes, stream, enable_pdl, qkv, \
num_heads_q, num_heads_k, num_heads_v, eps, q_weight, k_weight, \
cos_sin_cache, position_ids, num_tokens, rotary_dim); \
}); \
break; \
default: \
STD_TORCH_CHECK(false, "Unsupported head dimension: ", head_dim); \
} \
} while (0)
if (token_heads_per_warp == 2) {