[ROCm][Bugfix]: dynamically align BLOCK_DMODEL with Lv in MLA decode kernel (#41119)
Signed-off-by: vllmellm <vllm.ellm@embeddedllm.com>
This commit is contained in:
@@ -459,25 +459,29 @@ def _decode_grouped_att_m_fwd(
|
||||
):
|
||||
# with is_mla there is only a single c_kv in smem.
|
||||
# could increase BLOCK or num_stages.
|
||||
BLOCK = 32
|
||||
Lk = k_buffer.shape[-1]
|
||||
Lv = v_buffer.shape[-1]
|
||||
|
||||
# [TODO] work around shmem limit on MI3xx
|
||||
if is_hip_ and Lk >= 576:
|
||||
BLOCK = 16
|
||||
|
||||
if Lk == 576:
|
||||
BLOCK_DMODEL = 512
|
||||
BLOCK_DPE = 64
|
||||
elif Lk == 288:
|
||||
BLOCK_DMODEL = 256
|
||||
BLOCK_DPE = 32
|
||||
# Align tile dimensions with latent rank for MLA to avoid shape mismatch.
|
||||
if is_mla:
|
||||
if not is_hip_ and Lk == 576:
|
||||
BLOCK_DMODEL = 512
|
||||
BLOCK_DPE = 64
|
||||
elif not is_hip_ and Lk == 288:
|
||||
BLOCK_DMODEL = 256
|
||||
BLOCK_DPE = 32
|
||||
else:
|
||||
BLOCK_DMODEL = triton.next_power_of_2(Lv)
|
||||
BLOCK_DPE = triton.next_power_of_2(Lk - Lv) if Lk > Lv else 0
|
||||
else:
|
||||
BLOCK_DMODEL = triton.next_power_of_2(Lk)
|
||||
BLOCK_DPE = 0
|
||||
BLOCK_DV = triton.next_power_of_2(Lv)
|
||||
|
||||
BLOCK = 32
|
||||
if is_hip_:
|
||||
BLOCK = 16
|
||||
|
||||
batch, head_num = q.shape[0], q.shape[1]
|
||||
kv_group_num = q.shape[1] // k_buffer.shape[-2]
|
||||
|
||||
@@ -496,6 +500,11 @@ def _decode_grouped_att_m_fwd(
|
||||
# https://github.com/triton-lang/triton/blob/main/third_party/amd/backend/compiler.py
|
||||
extra_kargs = {"waves_per_eu": 1, "matrix_instr_nonkdim": 16, "kpack": 2}
|
||||
num_stages = 1
|
||||
elif not is_hip_ and BLOCK_DMODEL >= 1024:
|
||||
# Avoid shared memory overflow on NVIDIA when BLOCK_DMODEL is large
|
||||
# like non-MLA D_QK=576, BLOCK_DMODEL=1024, BLOCK_H=16
|
||||
# exceeds 101376 bytes limit
|
||||
num_stages = 1
|
||||
|
||||
_fwd_grouped_kernel_stage1[grid](
|
||||
q,
|
||||
|
||||
Reference in New Issue
Block a user