[ROCm][Bugfix]: dynamically align BLOCK_DMODEL with Lv in MLA decode kernel (#41119)

Signed-off-by: vllmellm <vllm.ellm@embeddedllm.com>
This commit is contained in:
vllmellm
2026-05-11 11:14:19 +08:00
committed by GitHub
parent b1687527b8
commit 7f95e66a11
@@ -459,25 +459,29 @@ def _decode_grouped_att_m_fwd(
):
# with is_mla there is only a single c_kv in smem.
# could increase BLOCK or num_stages.
BLOCK = 32
Lk = k_buffer.shape[-1]
Lv = v_buffer.shape[-1]
# [TODO] work around shmem limit on MI3xx
if is_hip_ and Lk >= 576:
BLOCK = 16
if Lk == 576:
BLOCK_DMODEL = 512
BLOCK_DPE = 64
elif Lk == 288:
BLOCK_DMODEL = 256
BLOCK_DPE = 32
# Align tile dimensions with latent rank for MLA to avoid shape mismatch.
if is_mla:
if not is_hip_ and Lk == 576:
BLOCK_DMODEL = 512
BLOCK_DPE = 64
elif not is_hip_ and Lk == 288:
BLOCK_DMODEL = 256
BLOCK_DPE = 32
else:
BLOCK_DMODEL = triton.next_power_of_2(Lv)
BLOCK_DPE = triton.next_power_of_2(Lk - Lv) if Lk > Lv else 0
else:
BLOCK_DMODEL = triton.next_power_of_2(Lk)
BLOCK_DPE = 0
BLOCK_DV = triton.next_power_of_2(Lv)
BLOCK = 32
if is_hip_:
BLOCK = 16
batch, head_num = q.shape[0], q.shape[1]
kv_group_num = q.shape[1] // k_buffer.shape[-2]
@@ -496,6 +500,11 @@ def _decode_grouped_att_m_fwd(
# https://github.com/triton-lang/triton/blob/main/third_party/amd/backend/compiler.py
extra_kargs = {"waves_per_eu": 1, "matrix_instr_nonkdim": 16, "kpack": 2}
num_stages = 1
elif not is_hip_ and BLOCK_DMODEL >= 1024:
# Avoid shared memory overflow on NVIDIA when BLOCK_DMODEL is large
# like non-MLA D_QK=576, BLOCK_DMODEL=1024, BLOCK_H=16
# exceeds 101376 bytes limit
num_stages = 1
_fwd_grouped_kernel_stage1[grid](
q,