diff --git a/csrc/libtorch_stable/fused_qknorm_rope_kernel.cu b/csrc/libtorch_stable/fused_qknorm_rope_kernel.cu index a8e6d32a1b..01bad79cd8 100644 --- a/csrc/libtorch_stable/fused_qknorm_rope_kernel.cu +++ b/csrc/libtorch_stable/fused_qknorm_rope_kernel.cu @@ -189,6 +189,12 @@ __global__ void fusedQKNormRopeKernel( } int offsetThread = offsetWarp + laneId * numElemsPerThread; + // PDL: wait for the predecessor kernel (the QKV projection that produces + // `qkv`) to complete before touching any global memory. +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) + cudaGridDependencySynchronize(); +#endif + // Sum of squares for RMSNorm float sumOfSquares = 0.0f; @@ -300,6 +306,11 @@ __global__ void fusedQKNormRopeKernel( *reinterpret_cast(&qkv[offsetThread]) = vec; } + // PDL: signal completion so a dependent successor may launch early. +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) + cudaTriggerProgrammaticLaunchCompletion(); +#endif + #if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM) } #endif @@ -385,6 +396,13 @@ __global__ void fusedQKNormRopeKernelNTokenHeads( smem_storage + cos_sin_bytes + warpId * (HEADS_PER_WARP * qkv_tile_bytes); + // PDL: wait for the predecessor kernel (the QKV projection that produces + // `qkv`) to complete before touching any global memory -- cp.async reads + // global memory, so this must precede the first commit group. +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) + cudaGridDependencySynchronize(); +#endif + // === Group 0: async load all heads' QKV into smem (issued first). === for (int k = 0; k < num_heads_this_warp; ++k) { int const localHeadIdx = first_head + k; @@ -531,6 +549,11 @@ __global__ void fusedQKNormRopeKernelNTokenHeads( } } + // PDL: signal completion so a dependent successor may launch early. +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) + cudaTriggerProgrammaticLaunchCompletion(); +#endif + #if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM) } #endif @@ -547,6 +570,42 @@ __global__ void fusedQKNormRopeKernelNTokenHeads( __VA_ARGS__ \ } +namespace { +// The empirical value for small batch: overlapping with the predecessor QKV +// projection only pays off while this kernel is launch-bound. +constexpr int kPDLEnableTokens = 16; + +// Launch helper for PDL (programmatic dependent launch), which lets this +// kernel start while the predecessor QKV projection is still draining. The +// launch attribute is the only switch: with it off, the kernels' +// cudaGridDependencySynchronize() / cudaTriggerProgrammaticLaunchCompletion() +// are no-ops. +template +void launchWithPDL(KernelFn kernel, dim3 gridDim, dim3 blockDim, + size_t smem_bytes, cudaStream_t stream, bool enable_pdl, + Args... args) { +#ifndef USE_ROCM + cudaLaunchConfig_t config; + config.gridDim = gridDim; + config.blockDim = blockDim; + config.dynamicSmemBytes = smem_bytes; + config.stream = stream; + cudaLaunchAttribute attrs[1]; + attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; + attrs[0].val.programmaticStreamSerializationAllowed = enable_pdl; + config.numAttrs = 1; + config.attrs = attrs; + cudaLaunchKernelEx(&config, kernel, args...); +#else + // ROCm: standard kernel launch syntax (no PDL/stream serialization). + (void)enable_pdl; + // clang-format off + kernel<<>>(args...); + // clang-format on +#endif +} +} // namespace + template void launchFusedQKNormRope(void* qkv, int const num_tokens, int const num_heads_q, int const num_heads_k, @@ -562,29 +621,33 @@ void launchFusedQKNormRope(void* qkv, int const num_tokens, int const gridSize = common::divUp(totalWarps, warpsPerBlock); dim3 gridDim(gridSize); dim3 blockDim(blockSize); + bool const enable_pdl = num_tokens <= kPDLEnableTokens; switch (head_dim) { case 64: DISPATCH_INTERLEAVE(interleave, INTERLEAVE, { - fusedQKNormRopeKernel - <<>>( - qkv, num_heads_q, num_heads_k, num_heads_v, eps, q_weight, - k_weight, cos_sin_cache, position_ids, num_tokens, rotary_dim); + launchWithPDL( + fusedQKNormRopeKernel, + gridDim, blockDim, 0, stream, enable_pdl, qkv, num_heads_q, + num_heads_k, num_heads_v, eps, q_weight, k_weight, cos_sin_cache, + position_ids, num_tokens, rotary_dim); }); break; case 128: DISPATCH_INTERLEAVE(interleave, INTERLEAVE, { - fusedQKNormRopeKernel - <<>>( - qkv, num_heads_q, num_heads_k, num_heads_v, eps, q_weight, - k_weight, cos_sin_cache, position_ids, num_tokens, rotary_dim); + launchWithPDL( + fusedQKNormRopeKernel, + gridDim, blockDim, 0, stream, enable_pdl, qkv, num_heads_q, + num_heads_k, num_heads_v, eps, q_weight, k_weight, cos_sin_cache, + position_ids, num_tokens, rotary_dim); }); break; case 256: DISPATCH_INTERLEAVE(interleave, INTERLEAVE, { - fusedQKNormRopeKernel - <<>>( - qkv, num_heads_q, num_heads_k, num_heads_v, eps, q_weight, - k_weight, cos_sin_cache, position_ids, num_tokens, rotary_dim); + launchWithPDL( + fusedQKNormRopeKernel, + gridDim, blockDim, 0, stream, enable_pdl, qkv, num_heads_q, + num_heads_k, num_heads_v, eps, q_weight, k_weight, cos_sin_cache, + position_ids, num_tokens, rotary_dim); }); break; default: @@ -644,6 +707,7 @@ void launchFusedQKNormRopeNTokenHeads( int const gridSize = common::divUp(total_warps, warpsPerBlock); dim3 gridDim(gridSize); dim3 blockDim(blockSize); + bool const enable_pdl = num_tokens <= kPDLEnableTokens; // Cache element size: float=4, bfloat16=2 (host-safe; kernel uses same // layout). size_t const cache_elem_size = @@ -656,42 +720,42 @@ void launchFusedQKNormRopeNTokenHeads( warpsPerBlock * static_cast(rotary_dim) * cache_elem_size + warpsPerBlock * qkv_smem_per_warp; -#define LAUNCH_N_TOKEN_HEADS(N) \ - do { \ - switch (head_dim) { \ - case 64: \ - DISPATCH_INTERLEAVE(interleave, INTERLEAVE, { \ - fusedQKNormRopeKernelNTokenHeads \ - <<>>( \ - qkv, num_heads_q, num_heads_k, num_heads_v, eps, q_weight, \ - k_weight, cos_sin_cache, position_ids, num_tokens, \ - rotary_dim); \ - }); \ - break; \ - case 128: \ - DISPATCH_INTERLEAVE(interleave, INTERLEAVE, { \ - fusedQKNormRopeKernelNTokenHeads \ - <<>>( \ - qkv, num_heads_q, num_heads_k, num_heads_v, eps, q_weight, \ - k_weight, cos_sin_cache, position_ids, num_tokens, \ - rotary_dim); \ - }); \ - break; \ - case 256: \ - DISPATCH_INTERLEAVE(interleave, INTERLEAVE, { \ - fusedQKNormRopeKernelNTokenHeads \ - <<>>( \ - qkv, num_heads_q, num_heads_k, num_heads_v, eps, q_weight, \ - k_weight, cos_sin_cache, position_ids, num_tokens, \ - rotary_dim); \ - }); \ - break; \ - default: \ - STD_TORCH_CHECK(false, "Unsupported head dimension: ", head_dim); \ - } \ +#define LAUNCH_N_TOKEN_HEADS(N) \ + do { \ + switch (head_dim) { \ + case 64: \ + DISPATCH_INTERLEAVE(interleave, INTERLEAVE, { \ + launchWithPDL( \ + fusedQKNormRopeKernelNTokenHeads, \ + gridDim, blockDim, smem_bytes, stream, enable_pdl, qkv, \ + num_heads_q, num_heads_k, num_heads_v, eps, q_weight, k_weight, \ + cos_sin_cache, position_ids, num_tokens, rotary_dim); \ + }); \ + break; \ + case 128: \ + DISPATCH_INTERLEAVE(interleave, INTERLEAVE, { \ + launchWithPDL( \ + fusedQKNormRopeKernelNTokenHeads, \ + gridDim, blockDim, smem_bytes, stream, enable_pdl, qkv, \ + num_heads_q, num_heads_k, num_heads_v, eps, q_weight, k_weight, \ + cos_sin_cache, position_ids, num_tokens, rotary_dim); \ + }); \ + break; \ + case 256: \ + DISPATCH_INTERLEAVE(interleave, INTERLEAVE, { \ + launchWithPDL( \ + fusedQKNormRopeKernelNTokenHeads, \ + gridDim, blockDim, smem_bytes, stream, enable_pdl, qkv, \ + num_heads_q, num_heads_k, num_heads_v, eps, q_weight, k_weight, \ + cos_sin_cache, position_ids, num_tokens, rotary_dim); \ + }); \ + break; \ + default: \ + STD_TORCH_CHECK(false, "Unsupported head dimension: ", head_dim); \ + } \ } while (0) if (token_heads_per_warp == 2) {