[Kernel] Add PDL support for DeepGEMM kernel (#42996)
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
@@ -37,6 +37,8 @@ def _fused_inv_rope_fp8_quant_per_head(
|
||||
ROPE_START: tl.constexpr,
|
||||
HALF_ROPE: tl.constexpr,
|
||||
TMA_ALIGNED_SCALES: tl.constexpr,
|
||||
USE_GDC: tl.constexpr,
|
||||
launch_pdl: tl.constexpr, # triton metadata
|
||||
):
|
||||
# int64: stride multiply overflows int32 past num_tokens=32768 (IMA).
|
||||
pid_token = tl.program_id(0).to(tl.int64)
|
||||
@@ -46,7 +48,9 @@ def _fused_inv_rope_fp8_quant_per_head(
|
||||
head_in_group = pid_gh % heads_per_group
|
||||
global_head = pid_gh
|
||||
qb_start = head_in_group * CHUNKS_PER_HEAD
|
||||
|
||||
if USE_GDC:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
tl.extra.cuda.gdc_wait()
|
||||
# Padding rows in the TMA-aligned scale buffer: fill with zero and skip quant.
|
||||
if pid_token >= num_tokens:
|
||||
if TMA_ALIGNED_SCALES:
|
||||
@@ -243,11 +247,8 @@ def _fused_inv_rope_fp8_quant_kernel_impl(
|
||||
(scale_inner * tma_aligned_T, 1, tma_aligned_T),
|
||||
)
|
||||
grid = (tma_aligned_T, n_groups * heads_per_group)
|
||||
pdl_kwargs = (
|
||||
{}
|
||||
if current_platform.is_rocm() or current_platform.is_xpu()
|
||||
else {"launch_pdl": False}
|
||||
)
|
||||
use_gdc = current_platform.is_arch_support_pdl()
|
||||
pdl_kwargs = {"launch_pdl": True} if use_gdc else {}
|
||||
_fused_inv_rope_fp8_quant_per_head[grid](
|
||||
o,
|
||||
positions,
|
||||
@@ -270,6 +271,7 @@ def _fused_inv_rope_fp8_quant_kernel_impl(
|
||||
ROPE_START=rope_start,
|
||||
HALF_ROPE=half_rope,
|
||||
TMA_ALIGNED_SCALES=tma_aligned_scales,
|
||||
USE_GDC=use_gdc,
|
||||
num_stages=1,
|
||||
**pdl_kwargs,
|
||||
num_warps=1,
|
||||
|
||||
Reference in New Issue
Block a user