From 2dfb8ba59098eb489197e1b4c643addffd51592e Mon Sep 17 00:00:00 2001 From: xiaozhoupy Date: Thu, 6 Aug 2026 15:53:17 +0800 Subject: [PATCH] [Perf][CUDA] Programmatic dependent launch for the DSA decode kernels (#50230) Signed-off-by: Peiyuan Zhou Co-authored-by: Claude Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../quantization/fp4/nvfp4_quant_kernels.cu | 50 +++++++++++++++---- vllm/models/deepseek_v32/common/kernels.py | 14 ++++++ 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/csrc/libtorch_stable/quantization/fp4/nvfp4_quant_kernels.cu b/csrc/libtorch_stable/quantization/fp4/nvfp4_quant_kernels.cu index f7c965dbc1..97d04f96bd 100644 --- a/csrc/libtorch_stable/quantization/fp4/nvfp4_quant_kernels.cu +++ b/csrc/libtorch_stable/quantization/fp4/nvfp4_quant_kernels.cu @@ -49,6 +49,11 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512)) static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD, "Vec size is not matched."); +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + cudaGridDependencySynchronize(); + cudaTriggerProgrammaticLaunchCompletion(); +#endif + // Precompute SF layout parameter (constant for entire kernel). int32_t const numKTiles = (outputCols + 63) / 64; @@ -123,6 +128,11 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512)) static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD, "Vec size is not matched."); +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + cudaGridDependencySynchronize(); + cudaTriggerProgrammaticLaunchCompletion(); +#endif + int32_t const colIdx = blockDim.x * blockIdx.y + threadIdx.x; int elem_idx = colIdx * CVT_FP4_ELTS_PER_THREAD; @@ -224,10 +234,21 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output, input.scalar_type(), "nvfp4_quant_kernel", [&] { using cuda_type = vllm::CUDATypeConverter::Type; auto input_ptr = static_cast(input.data_ptr()); - vllm::cvt_fp16_to_fp4<<>>( - m, n, output_n, num_padded_cols, input_ptr, input_sf_ptr, - reinterpret_cast(output_ptr), - reinterpret_cast(sf_out)); + cudaLaunchConfig_t config = {}; + config.gridDim = grid; + config.blockDim = block; + config.dynamicSmemBytes = 0; + config.stream = stream; + cudaLaunchAttribute attrs[1]; + attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; + attrs[0].val.programmaticStreamSerializationAllowed = 1; + config.numAttrs = 1; + config.attrs = attrs; + cudaLaunchKernelEx(&config, vllm::cvt_fp16_to_fp4, + m, n, output_n, num_padded_cols, input_ptr, + input_sf_ptr, + reinterpret_cast(output_ptr), + reinterpret_cast(sf_out)); }); } else { int num_packed_cols = output_n / CVT_FP4_ELTS_PER_THREAD; @@ -240,12 +261,21 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output, input.scalar_type(), "nvfp4_quant_kernel", [&] { using cuda_type = vllm::CUDATypeConverter::Type; auto input_ptr = static_cast(input.data_ptr()); - vllm::cvt_fp16_to_fp4_sf_major - <<>>( - m, n, output_n, output_sf_n_unpadded, num_packed_cols, - input_ptr, input_sf_ptr, - reinterpret_cast(output_ptr), - reinterpret_cast(sf_out)); + cudaLaunchConfig_t config = {}; + config.gridDim = grid; + config.blockDim = block; + config.dynamicSmemBytes = 0; + config.stream = stream; + cudaLaunchAttribute attrs[1]; + attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; + attrs[0].val.programmaticStreamSerializationAllowed = 1; + config.numAttrs = 1; + config.attrs = attrs; + cudaLaunchKernelEx( + &config, vllm::cvt_fp16_to_fp4_sf_major, m, n, + output_n, output_sf_n_unpadded, num_packed_cols, input_ptr, + input_sf_ptr, reinterpret_cast(output_ptr), + reinterpret_cast(sf_out)); }); } } diff --git a/vllm/models/deepseek_v32/common/kernels.py b/vllm/models/deepseek_v32/common/kernels.py index 0434d5bfb3..2c6e895fdb 100644 --- a/vllm/models/deepseek_v32/common/kernels.py +++ b/vllm/models/deepseek_v32/common/kernels.py @@ -150,9 +150,13 @@ def _fused_norm_rope_kernel( TOPK_BLOCK_SIZE: tl.constexpr, HAS_INDEXER: tl.constexpr, INDEX_ROPE_INTERLEAVE: tl.constexpr, + USE_PDL: tl.constexpr, ): pid = tl.program_id(0) tok_idx = tl.program_id(1) + if USE_PDL: + tl.extra.cuda.gdc_wait() + tl.extra.cuda.gdc_launch_dependents() if pid == 3: if not HAS_INDEXER: # Shared layer: reuse the previous indexer layer's top-k; do not @@ -474,6 +478,7 @@ def fused_norm_rope( if q_c_out is None: q_c_out = torch.empty_like(q_c) + use_pdl = current_platform.is_arch_support_pdl() _fused_norm_rope_kernel[(4, num_tokens)]( positions, # Q RMS norm @@ -532,6 +537,8 @@ def fused_norm_rope( TOPK_BLOCK_SIZE=1024, HAS_INDEXER=has_indexer, INDEX_ROPE_INTERLEAVE=index_rope_interleave, + USE_PDL=use_pdl, + launch_pdl=use_pdl, ) return q_c_out @@ -585,10 +592,14 @@ def _fused_q_kernel( HAS_INDEXER: tl.constexpr, INDEX_ROPE_INTERLEAVE: tl.constexpr, QUANTIZE_MQA: tl.constexpr, + USE_PDL: tl.constexpr, ): pid = tl.program_id(0) tok_idx = tl.program_id(1) head_idx = tl.program_id(2) + if USE_PDL: + tl.extra.cuda.gdc_wait() + tl.extra.cuda.gdc_launch_dependents() if pid == 2: # ql_nope quantize + pack into the front of mqa_q_fp8. On the bf16 @@ -857,6 +868,7 @@ def fused_q( ) return index_q_fp8, index_weights_out, mqa_q + use_pdl = current_platform.is_arch_support_pdl() _fused_q_kernel[(3, num_tokens, grid_heads)]( positions, q_pe, @@ -898,6 +910,8 @@ def fused_q( HAS_INDEXER=has_indexer, INDEX_ROPE_INTERLEAVE=index_rope_interleave, QUANTIZE_MQA=quantize_mqa, + USE_PDL=use_pdl, + launch_pdl=use_pdl, # num_warps=1 is optimal here: each program is a single 128-element # rope+quant, so the kernel is program-count/occupancy bound, not # per-program compute bound (swept 1/2/4/8 — 1 wins or ties everywhere).