[Perf][CUDA] Programmatic dependent launch for the DSA decode kernels (#50230)

Signed-off-by: Peiyuan Zhou <peiyuanzhou1994@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
xiaozhoupy
2026-08-06 00:53:17 -07:00
committed by GitHub
co-authored by Claude mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent febea17f6a
commit 2dfb8ba590
2 changed files with 54 additions and 10 deletions
@@ -49,6 +49,11 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD, static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD,
"Vec size is not matched."); "Vec size is not matched.");
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
cudaGridDependencySynchronize();
cudaTriggerProgrammaticLaunchCompletion();
#endif
// Precompute SF layout parameter (constant for entire kernel). // Precompute SF layout parameter (constant for entire kernel).
int32_t const numKTiles = (outputCols + 63) / 64; int32_t const numKTiles = (outputCols + 63) / 64;
@@ -123,6 +128,11 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD, static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD,
"Vec size is not matched."); "Vec size is not matched.");
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
cudaGridDependencySynchronize();
cudaTriggerProgrammaticLaunchCompletion();
#endif
int32_t const colIdx = blockDim.x * blockIdx.y + threadIdx.x; int32_t const colIdx = blockDim.x * blockIdx.y + threadIdx.x;
int elem_idx = colIdx * CVT_FP4_ELTS_PER_THREAD; int elem_idx = colIdx * CVT_FP4_ELTS_PER_THREAD;
@@ -224,10 +234,21 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output,
input.scalar_type(), "nvfp4_quant_kernel", [&] { input.scalar_type(), "nvfp4_quant_kernel", [&] {
using cuda_type = vllm::CUDATypeConverter<scalar_t>::Type; using cuda_type = vllm::CUDATypeConverter<scalar_t>::Type;
auto input_ptr = static_cast<cuda_type const*>(input.data_ptr()); auto input_ptr = static_cast<cuda_type const*>(input.data_ptr());
vllm::cvt_fp16_to_fp4<cuda_type, false><<<grid, block, 0, stream>>>( cudaLaunchConfig_t config = {};
m, n, output_n, num_padded_cols, input_ptr, input_sf_ptr, config.gridDim = grid;
reinterpret_cast<uint32_t*>(output_ptr), config.blockDim = block;
reinterpret_cast<uint32_t*>(sf_out)); config.dynamicSmemBytes = 0;
config.stream = stream;
cudaLaunchAttribute attrs[1];
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
attrs[0].val.programmaticStreamSerializationAllowed = 1;
config.numAttrs = 1;
config.attrs = attrs;
cudaLaunchKernelEx(&config, vllm::cvt_fp16_to_fp4<cuda_type, false>,
m, n, output_n, num_padded_cols, input_ptr,
input_sf_ptr,
reinterpret_cast<uint32_t*>(output_ptr),
reinterpret_cast<uint32_t*>(sf_out));
}); });
} else { } else {
int num_packed_cols = output_n / CVT_FP4_ELTS_PER_THREAD; int num_packed_cols = output_n / CVT_FP4_ELTS_PER_THREAD;
@@ -240,12 +261,21 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output,
input.scalar_type(), "nvfp4_quant_kernel", [&] { input.scalar_type(), "nvfp4_quant_kernel", [&] {
using cuda_type = vllm::CUDATypeConverter<scalar_t>::Type; using cuda_type = vllm::CUDATypeConverter<scalar_t>::Type;
auto input_ptr = static_cast<cuda_type const*>(input.data_ptr()); auto input_ptr = static_cast<cuda_type const*>(input.data_ptr());
vllm::cvt_fp16_to_fp4_sf_major<cuda_type, false> cudaLaunchConfig_t config = {};
<<<grid, block, 0, stream>>>( config.gridDim = grid;
m, n, output_n, output_sf_n_unpadded, num_packed_cols, config.blockDim = block;
input_ptr, input_sf_ptr, config.dynamicSmemBytes = 0;
reinterpret_cast<uint32_t*>(output_ptr), config.stream = stream;
reinterpret_cast<uint32_t*>(sf_out)); cudaLaunchAttribute attrs[1];
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
attrs[0].val.programmaticStreamSerializationAllowed = 1;
config.numAttrs = 1;
config.attrs = attrs;
cudaLaunchKernelEx(
&config, vllm::cvt_fp16_to_fp4_sf_major<cuda_type, false>, m, n,
output_n, output_sf_n_unpadded, num_packed_cols, input_ptr,
input_sf_ptr, reinterpret_cast<uint32_t*>(output_ptr),
reinterpret_cast<uint32_t*>(sf_out));
}); });
} }
} }
@@ -150,9 +150,13 @@ def _fused_norm_rope_kernel(
TOPK_BLOCK_SIZE: tl.constexpr, TOPK_BLOCK_SIZE: tl.constexpr,
HAS_INDEXER: tl.constexpr, HAS_INDEXER: tl.constexpr,
INDEX_ROPE_INTERLEAVE: tl.constexpr, INDEX_ROPE_INTERLEAVE: tl.constexpr,
USE_PDL: tl.constexpr,
): ):
pid = tl.program_id(0) pid = tl.program_id(0)
tok_idx = tl.program_id(1) tok_idx = tl.program_id(1)
if USE_PDL:
tl.extra.cuda.gdc_wait()
tl.extra.cuda.gdc_launch_dependents()
if pid == 3: if pid == 3:
if not HAS_INDEXER: if not HAS_INDEXER:
# Shared layer: reuse the previous indexer layer's top-k; do not # Shared layer: reuse the previous indexer layer's top-k; do not
@@ -474,6 +478,7 @@ def fused_norm_rope(
if q_c_out is None: if q_c_out is None:
q_c_out = torch.empty_like(q_c) q_c_out = torch.empty_like(q_c)
use_pdl = current_platform.is_arch_support_pdl()
_fused_norm_rope_kernel[(4, num_tokens)]( _fused_norm_rope_kernel[(4, num_tokens)](
positions, positions,
# Q RMS norm # Q RMS norm
@@ -532,6 +537,8 @@ def fused_norm_rope(
TOPK_BLOCK_SIZE=1024, TOPK_BLOCK_SIZE=1024,
HAS_INDEXER=has_indexer, HAS_INDEXER=has_indexer,
INDEX_ROPE_INTERLEAVE=index_rope_interleave, INDEX_ROPE_INTERLEAVE=index_rope_interleave,
USE_PDL=use_pdl,
launch_pdl=use_pdl,
) )
return q_c_out return q_c_out
@@ -585,10 +592,14 @@ def _fused_q_kernel(
HAS_INDEXER: tl.constexpr, HAS_INDEXER: tl.constexpr,
INDEX_ROPE_INTERLEAVE: tl.constexpr, INDEX_ROPE_INTERLEAVE: tl.constexpr,
QUANTIZE_MQA: tl.constexpr, QUANTIZE_MQA: tl.constexpr,
USE_PDL: tl.constexpr,
): ):
pid = tl.program_id(0) pid = tl.program_id(0)
tok_idx = tl.program_id(1) tok_idx = tl.program_id(1)
head_idx = tl.program_id(2) head_idx = tl.program_id(2)
if USE_PDL:
tl.extra.cuda.gdc_wait()
tl.extra.cuda.gdc_launch_dependents()
if pid == 2: if pid == 2:
# ql_nope quantize + pack into the front of mqa_q_fp8. On the bf16 # ql_nope quantize + pack into the front of mqa_q_fp8. On the bf16
@@ -857,6 +868,7 @@ def fused_q(
) )
return index_q_fp8, index_weights_out, mqa_q return index_q_fp8, index_weights_out, mqa_q
use_pdl = current_platform.is_arch_support_pdl()
_fused_q_kernel[(3, num_tokens, grid_heads)]( _fused_q_kernel[(3, num_tokens, grid_heads)](
positions, positions,
q_pe, q_pe,
@@ -898,6 +910,8 @@ def fused_q(
HAS_INDEXER=has_indexer, HAS_INDEXER=has_indexer,
INDEX_ROPE_INTERLEAVE=index_rope_interleave, INDEX_ROPE_INTERLEAVE=index_rope_interleave,
QUANTIZE_MQA=quantize_mqa, QUANTIZE_MQA=quantize_mqa,
USE_PDL=use_pdl,
launch_pdl=use_pdl,
# num_warps=1 is optimal here: each program is a single 128-element # num_warps=1 is optimal here: each program is a single 128-element
# rope+quant, so the kernel is program-count/occupancy bound, not # rope+quant, so the kernel is program-count/occupancy bound, not
# per-program compute bound (swept 1/2/4/8 — 1 wins or ties everywhere). # per-program compute bound (swept 1/2/4/8 — 1 wins or ties everywhere).