[Perf][CUDA] Programmatic dependent launch for the DSA decode kernels (#50230)

Signed-off-by: Peiyuan Zhou <peiyuanzhou1994@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
xiaozhoupy
2026-08-06 00:53:17 -07:00
committed by GitHub
co-authored by Claude mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent febea17f6a
commit 2dfb8ba590
2 changed files with 54 additions and 10 deletions
@@ -150,9 +150,13 @@ def _fused_norm_rope_kernel(
TOPK_BLOCK_SIZE: tl.constexpr,
HAS_INDEXER: tl.constexpr,
INDEX_ROPE_INTERLEAVE: tl.constexpr,
USE_PDL: tl.constexpr,
):
pid = tl.program_id(0)
tok_idx = tl.program_id(1)
if USE_PDL:
tl.extra.cuda.gdc_wait()
tl.extra.cuda.gdc_launch_dependents()
if pid == 3:
if not HAS_INDEXER:
# Shared layer: reuse the previous indexer layer's top-k; do not
@@ -474,6 +478,7 @@ def fused_norm_rope(
if q_c_out is None:
q_c_out = torch.empty_like(q_c)
use_pdl = current_platform.is_arch_support_pdl()
_fused_norm_rope_kernel[(4, num_tokens)](
positions,
# Q RMS norm
@@ -532,6 +537,8 @@ def fused_norm_rope(
TOPK_BLOCK_SIZE=1024,
HAS_INDEXER=has_indexer,
INDEX_ROPE_INTERLEAVE=index_rope_interleave,
USE_PDL=use_pdl,
launch_pdl=use_pdl,
)
return q_c_out
@@ -585,10 +592,14 @@ def _fused_q_kernel(
HAS_INDEXER: tl.constexpr,
INDEX_ROPE_INTERLEAVE: tl.constexpr,
QUANTIZE_MQA: tl.constexpr,
USE_PDL: tl.constexpr,
):
pid = tl.program_id(0)
tok_idx = tl.program_id(1)
head_idx = tl.program_id(2)
if USE_PDL:
tl.extra.cuda.gdc_wait()
tl.extra.cuda.gdc_launch_dependents()
if pid == 2:
# ql_nope quantize + pack into the front of mqa_q_fp8. On the bf16
@@ -857,6 +868,7 @@ def fused_q(
)
return index_q_fp8, index_weights_out, mqa_q
use_pdl = current_platform.is_arch_support_pdl()
_fused_q_kernel[(3, num_tokens, grid_heads)](
positions,
q_pe,
@@ -898,6 +910,8 @@ def fused_q(
HAS_INDEXER=has_indexer,
INDEX_ROPE_INTERLEAVE=index_rope_interleave,
QUANTIZE_MQA=quantize_mqa,
USE_PDL=use_pdl,
launch_pdl=use_pdl,
# num_warps=1 is optimal here: each program is a single 128-element
# rope+quant, so the kernel is program-count/occupancy bound, not
# per-program compute bound (swept 1/2/4/8 — 1 wins or ties everywhere).