diff --git a/csrc/libtorch_stable/dsv3_fused_a_gemm.cu b/csrc/libtorch_stable/dsv3_fused_a_gemm.cu index bdf749ddfc..cbf486fd54 100644 --- a/csrc/libtorch_stable/dsv3_fused_a_gemm.cu +++ b/csrc/libtorch_stable/dsv3_fused_a_gemm.cu @@ -328,7 +328,7 @@ struct GmemLoaderB { __device__ void issue_mainloop() { #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 - asm volatile("griddepcontrol.wait;"); + cudaGridDependencySynchronize(); #pragma unroll 1 for (int loop_idx = 0; loop_idx < k_iter_cnt; loop_idx++) { if (need_wait) { @@ -643,7 +643,7 @@ __global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel( mma_computer.issue_mainloop(); mma_computer.epi(); } - asm volatile("griddepcontrol.launch_dependents;"); + cudaTriggerProgrammaticLaunchCompletion(); #endif } diff --git a/csrc/libtorch_stable/fp32_router_gemm.cu b/csrc/libtorch_stable/fp32_router_gemm.cu index 80374d66a0..64393fad61 100644 --- a/csrc/libtorch_stable/fp32_router_gemm.cu +++ b/csrc/libtorch_stable/fp32_router_gemm.cu @@ -100,7 +100,7 @@ __global__ __launch_bounds__(128, 1) void fp32_router_gemm_kernel( } #if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) - asm volatile("griddepcontrol.wait;"); + cudaGridDependencySynchronize(); #endif for (int ki = 0; ki < k_iterations; ki++) { @@ -146,7 +146,7 @@ __global__ __launch_bounds__(128, 1) void fp32_router_gemm_kernel( } #if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) - asm volatile("griddepcontrol.launch_dependents;"); + cudaTriggerProgrammaticLaunchCompletion(); #endif } diff --git a/csrc/libtorch_stable/minimax_reduce_rms_kernel.cu b/csrc/libtorch_stable/minimax_reduce_rms_kernel.cu index da7fee5670..58d61b353d 100644 --- a/csrc/libtorch_stable/minimax_reduce_rms_kernel.cu +++ b/csrc/libtorch_stable/minimax_reduce_rms_kernel.cu @@ -249,7 +249,7 @@ __global__ void __launch_bounds__(1024) LamportComm comm(params.workspace, params.rank); int clear_access = comm.clear_size / kElemsPerAccess; #if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) - asm volatile("griddepcontrol.wait;"); + cudaGridDependencySynchronize(); #endif for (int idx = access_id; idx < tot_access; idx += access_stride, token_id += token_stride) { @@ -313,7 +313,7 @@ __global__ void __launch_bounds__(1024) } comm.update(params.size_q * NRanks); #if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) - asm volatile("griddepcontrol.launch_dependents;"); + cudaTriggerProgrammaticLaunchCompletion(); #endif } @@ -384,7 +384,7 @@ __global__ void __launch_bounds__(1024) DType norm_weight[kElemsPerAccess]{}; #if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) - asm volatile("griddepcontrol.wait;"); + cudaGridDependencySynchronize(); #endif if (is_q) { if (is_valid_q) { @@ -596,7 +596,7 @@ __global__ void __launch_bounds__(1024) } } // end group loop #if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) - asm volatile("griddepcontrol.launch_dependents;"); + cudaTriggerProgrammaticLaunchCompletion(); #endif int clear_access = static_cast(comm.clear_size / kElemsPerAccess); diff --git a/csrc/libtorch_stable/moe/dsv3_router_gemm_bf16_out.cu b/csrc/libtorch_stable/moe/dsv3_router_gemm_bf16_out.cu index bee4e00a8d..8695d1e808 100644 --- a/csrc/libtorch_stable/moe/dsv3_router_gemm_bf16_out.cu +++ b/csrc/libtorch_stable/moe/dsv3_router_gemm_bf16_out.cu @@ -78,7 +78,7 @@ __global__ __launch_bounds__(128, 1) void router_gemm_kernel_bf16_output( } #if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) - asm volatile("griddepcontrol.wait;"); + cudaGridDependencySynchronize(); #endif // Process the GEMM in chunks @@ -163,7 +163,7 @@ __global__ __launch_bounds__(128, 1) void router_gemm_kernel_bf16_output( } } #if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) - asm volatile("griddepcontrol.launch_dependents;"); + cudaTriggerProgrammaticLaunchCompletion(); #endif } diff --git a/csrc/libtorch_stable/moe/dsv3_router_gemm_float_out.cu b/csrc/libtorch_stable/moe/dsv3_router_gemm_float_out.cu index fe940d5433..58a2b44ae2 100644 --- a/csrc/libtorch_stable/moe/dsv3_router_gemm_float_out.cu +++ b/csrc/libtorch_stable/moe/dsv3_router_gemm_float_out.cu @@ -78,7 +78,7 @@ __global__ __launch_bounds__(128, 1) void router_gemm_kernel_float_output( } #if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) - asm volatile("griddepcontrol.wait;"); + cudaGridDependencySynchronize(); #endif // Process the GEMM in chunks @@ -163,7 +163,7 @@ __global__ __launch_bounds__(128, 1) void router_gemm_kernel_float_output( } } #if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) - asm volatile("griddepcontrol.launch_dependents;"); + cudaTriggerProgrammaticLaunchCompletion(); #endif } diff --git a/csrc/libtorch_stable/moe/grouped_topk_kernels.cu b/csrc/libtorch_stable/moe/grouped_topk_kernels.cu index a28edf3a55..3b5f98e6ee 100644 --- a/csrc/libtorch_stable/moe/grouped_topk_kernels.cu +++ b/csrc/libtorch_stable/moe/grouped_topk_kernels.cu @@ -48,7 +48,8 @@ static constexpr int NumTopGroupScores = 2; static constexpr int DefaultMaxNumTopExperts = 8; static constexpr int MaxSupportedTopExperts = 22; static constexpr int MaxNumTopGroups = 4; - +// The empirical value for small batch +static constexpr int PDLEnableTokens = 16; namespace warp_topk { template @@ -564,8 +565,8 @@ __global__ void grouped_topk_fused_kernel( T* s_group_scores = reinterpret_cast(ptr_u); #if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) - asm volatile("griddepcontrol.wait;"); // I think all prolog can be put before - // acqbulk because it's ptr arithmetic + cudaGridDependencySynchronize(); // I think all prolog can be put before + // acqbulk because it's ptr arithmetic #endif // phase 1: per-group scan @@ -609,7 +610,7 @@ __global__ void grouped_topk_fused_kernel( topk_values[i] = 1.0f / static_cast(topk_i32); } #if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) - asm volatile("griddepcontrol.launch_dependents;"); + cudaTriggerProgrammaticLaunchCompletion(); #endif return; } @@ -670,7 +671,7 @@ __global__ void grouped_topk_fused_kernel( } #if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) - asm volatile("griddepcontrol.launch_dependents;"); + cudaTriggerProgrammaticLaunchCompletion(); #endif } @@ -895,7 +896,8 @@ void invokeNoAuxTc(T* scores, float* topk_values, IdxT* topk_indices, int64_t const num_experts, int64_t const n_group, int64_t const topk_group, int64_t const topk, bool const renormalize, double const routed_scaling_factor, - bool enable_pdl = false, cudaStream_t const stream = 0) { + const bool enable_pdl = false, + cudaStream_t const stream = 0) { cudaLaunchConfig_t config; config.stream = stream; cudaLaunchAttribute attrs[1]; @@ -983,7 +985,7 @@ void invokeNoAuxTc(T* scores, float* topk_values, IdxT* topk_indices, int64_t const num_tokens, int64_t const num_experts, \ int64_t const n_group, int64_t const topk_group, int64_t const topk, \ bool const renormalize, double const routed_scaling_factor, \ - bool enable_pdl, cudaStream_t const stream); + const bool enable_pdl, cudaStream_t const stream); INSTANTIATE_NOAUX_TC(float, float, int32_t, SCORING_SIGMOID); INSTANTIATE_NOAUX_TC(float, half, int32_t, SCORING_SIGMOID); @@ -1037,7 +1039,7 @@ std::tuple grouped_topk( scores, {num_tokens, topk}, torch::headeronly::ScalarType::Float); auto topk_indices = torch::stable::new_empty( scores, {num_tokens, topk}, torch::headeronly::ScalarType::Int); - + const bool pdl_flag = num_tokens <= vllm::moe::PDLEnableTokens; const cudaStream_t stream = get_current_cuda_stream(scores.get_device_index()); auto const sf = static_cast(scoring_func); @@ -1052,7 +1054,7 @@ std::tuple grouped_topk( reinterpret_cast(topk_indices.mutable_data_ptr()), \ reinterpret_cast(bias.data_ptr()), num_tokens, \ num_experts, n_group, topk_group, topk, renormalize, \ - routed_scaling_factor, false, stream); \ + routed_scaling_factor, pdl_flag, stream); \ break; \ case vllm::moe::SCORING_SIGMOID: \ vllm::moe::invokeNoAuxTc( \ @@ -1061,7 +1063,7 @@ std::tuple grouped_topk( reinterpret_cast(topk_indices.mutable_data_ptr()), \ reinterpret_cast(bias.data_ptr()), num_tokens, \ num_experts, n_group, topk_group, topk, renormalize, \ - routed_scaling_factor, false, stream); \ + routed_scaling_factor, pdl_flag, stream); \ break; \ default: \ STD_TORCH_CHECK(false, "Unsupported scoring_func"); \ diff --git a/csrc/libtorch_stable/moe/topk_softplus_sqrt_kernels.cu b/csrc/libtorch_stable/moe/topk_softplus_sqrt_kernels.cu index 7efe13b4d9..0f85f9b4d3 100644 --- a/csrc/libtorch_stable/moe/topk_softplus_sqrt_kernels.cu +++ b/csrc/libtorch_stable/moe/topk_softplus_sqrt_kernels.cu @@ -173,7 +173,7 @@ __launch_bounds__(WARPS_PER_CTA* WARP_SIZE_PARAM) __global__ float row_chunk[VPT]; #if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) - asm volatile("griddepcontrol.wait;"); + cudaGridDependencySynchronize(); #endif // NOTE(zhuhaoran): dispatch different input types loading, BF16/FP16 convert @@ -300,7 +300,7 @@ __launch_bounds__(WARPS_PER_CTA* WARP_SIZE_PARAM) __global__ } } #if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) - asm volatile("griddepcontrol.launch_dependents;"); + cudaTriggerProgrammaticLaunchCompletion(); #endif return; } else { @@ -425,7 +425,7 @@ __launch_bounds__(WARPS_PER_CTA* WARP_SIZE_PARAM) __global__ } } #if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) - asm volatile("griddepcontrol.launch_dependents;"); + cudaTriggerProgrammaticLaunchCompletion(); #endif } }