[Perf][ROCm] Fix GDN KKT warmup regression on RDNA by avoiding fp32 tl.dot (#45000)
Signed-off-by: Saeid Rostami <srostami@amd.com> Signed-off-by: nemanjaudovic <nudovic@amd.com>
This commit is contained in:
@@ -10,12 +10,23 @@
|
|||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
|
from vllm.platforms import current_platform
|
||||||
from vllm.triton_utils import tl, triton
|
from vllm.triton_utils import tl, triton
|
||||||
|
|
||||||
from .index import prepare_chunk_indices
|
from .index import prepare_chunk_indices
|
||||||
from .op import exp
|
from .op import exp
|
||||||
from .utils import FLA_CHUNK_SIZE
|
from .utils import FLA_CHUNK_SIZE
|
||||||
|
|
||||||
|
# On RDNA (gfx11xx/gfx12xx) WMMA only
|
||||||
|
# accepts 16-bit/int inputs, so a widened (e.g. fp32) tl.dot is lowered to a
|
||||||
|
# software matmul (~190x amdgcn-stage blowup). There we cast both operands down
|
||||||
|
# to k's native storage dtype (bf16/fp16) so fast WMMA is used instead.
|
||||||
|
_CAST_DOT_TO_K_DTYPE = False
|
||||||
|
if current_platform.is_rocm():
|
||||||
|
from vllm.platforms.rocm import on_gfx1x
|
||||||
|
|
||||||
|
_CAST_DOT_TO_K_DTYPE = on_gfx1x()
|
||||||
|
|
||||||
|
|
||||||
@triton.heuristics(
|
@triton.heuristics(
|
||||||
{
|
{
|
||||||
@@ -48,6 +59,7 @@ def chunk_scaled_dot_kkt_fwd_kernel(
|
|||||||
BK: tl.constexpr,
|
BK: tl.constexpr,
|
||||||
IS_VARLEN: tl.constexpr,
|
IS_VARLEN: tl.constexpr,
|
||||||
USE_G: tl.constexpr,
|
USE_G: tl.constexpr,
|
||||||
|
CAST_DOT_TO_K_DTYPE: tl.constexpr,
|
||||||
):
|
):
|
||||||
i_t, i_bh = tl.program_id(0), tl.program_id(1)
|
i_t, i_bh = tl.program_id(0), tl.program_id(1)
|
||||||
i_b, i_h = i_bh // H, i_bh % H
|
i_b, i_h = i_bh // H, i_bh % H
|
||||||
@@ -83,7 +95,12 @@ def chunk_scaled_dot_kkt_fwd_kernel(
|
|||||||
)
|
)
|
||||||
b_k = tl.load(p_k, boundary_check=(0, 1))
|
b_k = tl.load(p_k, boundary_check=(0, 1))
|
||||||
b_kb = b_k * b_beta[:, None]
|
b_kb = b_k * b_beta[:, None]
|
||||||
b_A += tl.dot(b_kb, tl.trans(b_k).to(b_kb.dtype))
|
if CAST_DOT_TO_K_DTYPE:
|
||||||
|
# RDNA: force operands to k's native dtype so WMMA is used.
|
||||||
|
b_A += tl.dot(b_kb.to(b_k.dtype), tl.trans(b_k))
|
||||||
|
else:
|
||||||
|
# Keep the promoted precision of the beta-scaled operand (WGMMA/MFMA).
|
||||||
|
b_A += tl.dot(b_kb, tl.trans(b_k).to(b_kb.dtype))
|
||||||
|
|
||||||
if USE_G:
|
if USE_G:
|
||||||
p_g = tl.make_block_ptr(g + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,))
|
p_g = tl.make_block_ptr(g + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,))
|
||||||
@@ -154,5 +171,6 @@ def chunk_scaled_dot_kkt_fwd(
|
|||||||
Hg=Hg,
|
Hg=Hg,
|
||||||
K=K,
|
K=K,
|
||||||
BT=BT,
|
BT=BT,
|
||||||
|
CAST_DOT_TO_K_DTYPE=_CAST_DOT_TO_K_DTYPE,
|
||||||
)
|
)
|
||||||
return A
|
return A
|
||||||
|
|||||||
Reference in New Issue
Block a user