[Perf][CUDA] Programmatic dependent launch for the DSA decode kernels (#50230)
Signed-off-by: Peiyuan Zhou <peiyuanzhou1994@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
febea17f6a
commit
2dfb8ba590
@@ -49,6 +49,11 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
|
|||||||
static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD,
|
static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD,
|
||||||
"Vec size is not matched.");
|
"Vec size is not matched.");
|
||||||
|
|
||||||
|
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
|
||||||
|
cudaGridDependencySynchronize();
|
||||||
|
cudaTriggerProgrammaticLaunchCompletion();
|
||||||
|
#endif
|
||||||
|
|
||||||
// Precompute SF layout parameter (constant for entire kernel).
|
// Precompute SF layout parameter (constant for entire kernel).
|
||||||
int32_t const numKTiles = (outputCols + 63) / 64;
|
int32_t const numKTiles = (outputCols + 63) / 64;
|
||||||
|
|
||||||
@@ -123,6 +128,11 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
|
|||||||
static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD,
|
static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD,
|
||||||
"Vec size is not matched.");
|
"Vec size is not matched.");
|
||||||
|
|
||||||
|
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
|
||||||
|
cudaGridDependencySynchronize();
|
||||||
|
cudaTriggerProgrammaticLaunchCompletion();
|
||||||
|
#endif
|
||||||
|
|
||||||
int32_t const colIdx = blockDim.x * blockIdx.y + threadIdx.x;
|
int32_t const colIdx = blockDim.x * blockIdx.y + threadIdx.x;
|
||||||
int elem_idx = colIdx * CVT_FP4_ELTS_PER_THREAD;
|
int elem_idx = colIdx * CVT_FP4_ELTS_PER_THREAD;
|
||||||
|
|
||||||
@@ -224,10 +234,21 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output,
|
|||||||
input.scalar_type(), "nvfp4_quant_kernel", [&] {
|
input.scalar_type(), "nvfp4_quant_kernel", [&] {
|
||||||
using cuda_type = vllm::CUDATypeConverter<scalar_t>::Type;
|
using cuda_type = vllm::CUDATypeConverter<scalar_t>::Type;
|
||||||
auto input_ptr = static_cast<cuda_type const*>(input.data_ptr());
|
auto input_ptr = static_cast<cuda_type const*>(input.data_ptr());
|
||||||
vllm::cvt_fp16_to_fp4<cuda_type, false><<<grid, block, 0, stream>>>(
|
cudaLaunchConfig_t config = {};
|
||||||
m, n, output_n, num_padded_cols, input_ptr, input_sf_ptr,
|
config.gridDim = grid;
|
||||||
reinterpret_cast<uint32_t*>(output_ptr),
|
config.blockDim = block;
|
||||||
reinterpret_cast<uint32_t*>(sf_out));
|
config.dynamicSmemBytes = 0;
|
||||||
|
config.stream = stream;
|
||||||
|
cudaLaunchAttribute attrs[1];
|
||||||
|
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
|
||||||
|
attrs[0].val.programmaticStreamSerializationAllowed = 1;
|
||||||
|
config.numAttrs = 1;
|
||||||
|
config.attrs = attrs;
|
||||||
|
cudaLaunchKernelEx(&config, vllm::cvt_fp16_to_fp4<cuda_type, false>,
|
||||||
|
m, n, output_n, num_padded_cols, input_ptr,
|
||||||
|
input_sf_ptr,
|
||||||
|
reinterpret_cast<uint32_t*>(output_ptr),
|
||||||
|
reinterpret_cast<uint32_t*>(sf_out));
|
||||||
});
|
});
|
||||||
} else {
|
} else {
|
||||||
int num_packed_cols = output_n / CVT_FP4_ELTS_PER_THREAD;
|
int num_packed_cols = output_n / CVT_FP4_ELTS_PER_THREAD;
|
||||||
@@ -240,12 +261,21 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output,
|
|||||||
input.scalar_type(), "nvfp4_quant_kernel", [&] {
|
input.scalar_type(), "nvfp4_quant_kernel", [&] {
|
||||||
using cuda_type = vllm::CUDATypeConverter<scalar_t>::Type;
|
using cuda_type = vllm::CUDATypeConverter<scalar_t>::Type;
|
||||||
auto input_ptr = static_cast<cuda_type const*>(input.data_ptr());
|
auto input_ptr = static_cast<cuda_type const*>(input.data_ptr());
|
||||||
vllm::cvt_fp16_to_fp4_sf_major<cuda_type, false>
|
cudaLaunchConfig_t config = {};
|
||||||
<<<grid, block, 0, stream>>>(
|
config.gridDim = grid;
|
||||||
m, n, output_n, output_sf_n_unpadded, num_packed_cols,
|
config.blockDim = block;
|
||||||
input_ptr, input_sf_ptr,
|
config.dynamicSmemBytes = 0;
|
||||||
reinterpret_cast<uint32_t*>(output_ptr),
|
config.stream = stream;
|
||||||
reinterpret_cast<uint32_t*>(sf_out));
|
cudaLaunchAttribute attrs[1];
|
||||||
|
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
|
||||||
|
attrs[0].val.programmaticStreamSerializationAllowed = 1;
|
||||||
|
config.numAttrs = 1;
|
||||||
|
config.attrs = attrs;
|
||||||
|
cudaLaunchKernelEx(
|
||||||
|
&config, vllm::cvt_fp16_to_fp4_sf_major<cuda_type, false>, m, n,
|
||||||
|
output_n, output_sf_n_unpadded, num_packed_cols, input_ptr,
|
||||||
|
input_sf_ptr, reinterpret_cast<uint32_t*>(output_ptr),
|
||||||
|
reinterpret_cast<uint32_t*>(sf_out));
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -150,9 +150,13 @@ def _fused_norm_rope_kernel(
|
|||||||
TOPK_BLOCK_SIZE: tl.constexpr,
|
TOPK_BLOCK_SIZE: tl.constexpr,
|
||||||
HAS_INDEXER: tl.constexpr,
|
HAS_INDEXER: tl.constexpr,
|
||||||
INDEX_ROPE_INTERLEAVE: tl.constexpr,
|
INDEX_ROPE_INTERLEAVE: tl.constexpr,
|
||||||
|
USE_PDL: tl.constexpr,
|
||||||
):
|
):
|
||||||
pid = tl.program_id(0)
|
pid = tl.program_id(0)
|
||||||
tok_idx = tl.program_id(1)
|
tok_idx = tl.program_id(1)
|
||||||
|
if USE_PDL:
|
||||||
|
tl.extra.cuda.gdc_wait()
|
||||||
|
tl.extra.cuda.gdc_launch_dependents()
|
||||||
if pid == 3:
|
if pid == 3:
|
||||||
if not HAS_INDEXER:
|
if not HAS_INDEXER:
|
||||||
# Shared layer: reuse the previous indexer layer's top-k; do not
|
# Shared layer: reuse the previous indexer layer's top-k; do not
|
||||||
@@ -474,6 +478,7 @@ def fused_norm_rope(
|
|||||||
|
|
||||||
if q_c_out is None:
|
if q_c_out is None:
|
||||||
q_c_out = torch.empty_like(q_c)
|
q_c_out = torch.empty_like(q_c)
|
||||||
|
use_pdl = current_platform.is_arch_support_pdl()
|
||||||
_fused_norm_rope_kernel[(4, num_tokens)](
|
_fused_norm_rope_kernel[(4, num_tokens)](
|
||||||
positions,
|
positions,
|
||||||
# Q RMS norm
|
# Q RMS norm
|
||||||
@@ -532,6 +537,8 @@ def fused_norm_rope(
|
|||||||
TOPK_BLOCK_SIZE=1024,
|
TOPK_BLOCK_SIZE=1024,
|
||||||
HAS_INDEXER=has_indexer,
|
HAS_INDEXER=has_indexer,
|
||||||
INDEX_ROPE_INTERLEAVE=index_rope_interleave,
|
INDEX_ROPE_INTERLEAVE=index_rope_interleave,
|
||||||
|
USE_PDL=use_pdl,
|
||||||
|
launch_pdl=use_pdl,
|
||||||
)
|
)
|
||||||
return q_c_out
|
return q_c_out
|
||||||
|
|
||||||
@@ -585,10 +592,14 @@ def _fused_q_kernel(
|
|||||||
HAS_INDEXER: tl.constexpr,
|
HAS_INDEXER: tl.constexpr,
|
||||||
INDEX_ROPE_INTERLEAVE: tl.constexpr,
|
INDEX_ROPE_INTERLEAVE: tl.constexpr,
|
||||||
QUANTIZE_MQA: tl.constexpr,
|
QUANTIZE_MQA: tl.constexpr,
|
||||||
|
USE_PDL: tl.constexpr,
|
||||||
):
|
):
|
||||||
pid = tl.program_id(0)
|
pid = tl.program_id(0)
|
||||||
tok_idx = tl.program_id(1)
|
tok_idx = tl.program_id(1)
|
||||||
head_idx = tl.program_id(2)
|
head_idx = tl.program_id(2)
|
||||||
|
if USE_PDL:
|
||||||
|
tl.extra.cuda.gdc_wait()
|
||||||
|
tl.extra.cuda.gdc_launch_dependents()
|
||||||
|
|
||||||
if pid == 2:
|
if pid == 2:
|
||||||
# ql_nope quantize + pack into the front of mqa_q_fp8. On the bf16
|
# ql_nope quantize + pack into the front of mqa_q_fp8. On the bf16
|
||||||
@@ -857,6 +868,7 @@ def fused_q(
|
|||||||
)
|
)
|
||||||
return index_q_fp8, index_weights_out, mqa_q
|
return index_q_fp8, index_weights_out, mqa_q
|
||||||
|
|
||||||
|
use_pdl = current_platform.is_arch_support_pdl()
|
||||||
_fused_q_kernel[(3, num_tokens, grid_heads)](
|
_fused_q_kernel[(3, num_tokens, grid_heads)](
|
||||||
positions,
|
positions,
|
||||||
q_pe,
|
q_pe,
|
||||||
@@ -898,6 +910,8 @@ def fused_q(
|
|||||||
HAS_INDEXER=has_indexer,
|
HAS_INDEXER=has_indexer,
|
||||||
INDEX_ROPE_INTERLEAVE=index_rope_interleave,
|
INDEX_ROPE_INTERLEAVE=index_rope_interleave,
|
||||||
QUANTIZE_MQA=quantize_mqa,
|
QUANTIZE_MQA=quantize_mqa,
|
||||||
|
USE_PDL=use_pdl,
|
||||||
|
launch_pdl=use_pdl,
|
||||||
# num_warps=1 is optimal here: each program is a single 128-element
|
# num_warps=1 is optimal here: each program is a single 128-element
|
||||||
# rope+quant, so the kernel is program-count/occupancy bound, not
|
# rope+quant, so the kernel is program-count/occupancy bound, not
|
||||||
# per-program compute bound (swept 1/2/4/8 — 1 wins or ties everywhere).
|
# per-program compute bound (swept 1/2/4/8 — 1 wins or ties everywhere).
|
||||||
|
|||||||
Reference in New Issue
Block a user