diff --git a/vllm/models/kimi_k3/amd/ops/third_party/kda/chunk.py b/vllm/models/kimi_k3/amd/ops/third_party/kda/chunk.py index e8b83d23ae..2e975e9f77 100644 --- a/vllm/models/kimi_k3/amd/ops/third_party/kda/chunk.py +++ b/vllm/models/kimi_k3/amd/ops/third_party/kda/chunk.py @@ -17,7 +17,11 @@ from vllm.third_party.flash_linear_attention.ops.chunk_delta_h import ( from vllm.third_party.flash_linear_attention.ops.cumsum import chunk_local_cumsum from vllm.third_party.flash_linear_attention.ops.index import prepare_chunk_indices from vllm.third_party.flash_linear_attention.ops.l2norm import l2norm_fwd -from vllm.third_party.flash_linear_attention.ops.op import exp2, log +from vllm.third_party.flash_linear_attention.ops.op import ( + exp2, + log, + make_tensor_descriptor, +) from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE, is_amd from vllm.triton_utils import tl, triton from vllm.utils.math_utils import RCP_LN2, cdiv, next_power_of_2 @@ -90,86 +94,35 @@ def recompute_w_u_fwd_kernel( T = eos - bos else: bos, eos = i_b * T, i_b * T + T - p_b = tl.make_block_ptr(beta + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,)) - b_b = tl.load(p_b, boundary_check=(0,)).to(tl.float32) + o_b = i_t * BT + tl.arange(0, BT) + b_b = tl.load(beta + (bos + o_b) * H + i_h, mask=o_b < T, other=0).to(tl.float32) - p_A = tl.make_block_ptr( - A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0) - ) - b_A = tl.load(p_A, boundary_check=(0, 1)) + desc_A = make_tensor_descriptor(A + (bos * H + i_h) * BT, [T, BT], [H * BT, 1], [BT, BT]) + b_A = desc_A.load([i_t * BT, 0]) + desc_v = make_tensor_descriptor(v + (bos * H + i_h) * V, [T, V], [H * V, 1], [BT, BV]) + desc_u = make_tensor_descriptor(u + (bos * H + i_h) * V, [T, V], [H * V, 1], [BT, BV]) for i_v in range(tl.cdiv(V, BV)): - p_v = tl.make_block_ptr( - v + (bos * H + i_h) * V, - (T, V), - (H * V, 1), - (i_t * BT, i_v * BV), - (BT, BV), - (1, 0), - ) - p_u = tl.make_block_ptr( - u + (bos * H + i_h) * V, - (T, V), - (H * V, 1), - (i_t * BT, i_v * BV), - (BT, BV), - (1, 0), - ) - b_v = tl.load(p_v, boundary_check=(0, 1)) + b_v = desc_v.load([i_t * BT, i_v * BV]) b_vb = (b_v * b_b[:, None]).to(b_v.dtype) b_u = tl.dot(b_A, b_vb, input_precision=DOT_PRECISION) - tl.store(p_u, b_u.to(p_u.dtype.element_ty), boundary_check=(0, 1)) + desc_u.store([i_t * BT, i_v * BV], b_u.to(desc_u.dtype)) + desc_w = make_tensor_descriptor(w + (bos * H + i_h) * K, [T, K], [H * K, 1], [BT, BK]) + desc_k = make_tensor_descriptor(k + (bos * H + i_h) * K, [T, K], [H * K, 1], [BT, BK]) + desc_gk = make_tensor_descriptor(gk + (bos * H + i_h) * K, [T, K], [H * K, 1], [BT, BK]) for i_k in range(tl.cdiv(K, BK)): - p_w = tl.make_block_ptr( - w + (bos * H + i_h) * K, - (T, K), - (H * K, 1), - (i_t * BT, i_k * BK), - (BT, BK), - (1, 0), - ) - p_k = tl.make_block_ptr( - k + (bos * H + i_h) * K, - (T, K), - (H * K, 1), - (i_t * BT, i_k * BK), - (BT, BK), - (1, 0), - ) - b_k = tl.load(p_k, boundary_check=(0, 1)) + b_k = desc_k.load([i_t * BT, i_k * BK]) b_kb = b_k * b_b[:, None] - p_gk = tl.make_block_ptr( - gk + (bos * H + i_h) * K, - (T, K), - (H * K, 1), - (i_t * BT, i_k * BK), - (BT, BK), - (1, 0), - ) - b_gk = tl.load(p_gk, boundary_check=(0, 1)) + b_gk = desc_gk.load([i_t * BT, i_k * BK]) b_kb *= exp2(b_gk) if STORE_QG: - p_q = tl.make_block_ptr( - q + (bos * H + i_h) * K, - (T, K), - (H * K, 1), - (i_t * BT, i_k * BK), - (BT, BK), - (1, 0), - ) - p_qg = tl.make_block_ptr( - qg + (bos * H + i_h) * K, - (T, K), - (H * K, 1), - (i_t * BT, i_k * BK), - (BT, BK), - (1, 0), - ) - b_q = tl.load(p_q, boundary_check=(0, 1)) + desc_q = make_tensor_descriptor(q + (bos * H + i_h) * K, [T, K], [H * K, 1], [BT, BK]) + desc_qg = make_tensor_descriptor(qg + (bos * H + i_h) * K, [T, K], [H * K, 1], [BT, BK]) + b_q = desc_q.load([i_t * BT, i_k * BK]) b_qg = b_q * exp2(b_gk) - tl.store(p_qg, b_qg.to(p_qg.dtype.element_ty), boundary_check=(0, 1)) + desc_qg.store([i_t * BT, i_k * BK], b_qg.to(desc_qg.dtype)) if STORE_KG: last_idx = min(i_t * BT + BT, T) - 1 @@ -180,18 +133,11 @@ def recompute_w_u_fwd_kernel( ) b_kg = b_k * exp2(b_gn - b_gk) - p_kg = tl.make_block_ptr( - kg + (bos * H + i_h) * K, - (T, K), - (H * K, 1), - (i_t * BT, i_k * BK), - (BT, BK), - (1, 0), - ) - tl.store(p_kg, b_kg.to(p_kg.dtype.element_ty), boundary_check=(0, 1)) + desc_kg = make_tensor_descriptor(kg + (bos * H + i_h) * K, [T, K], [H * K, 1], [BT, BK]) + desc_kg.store([i_t * BT, i_k * BK], b_kg.to(desc_kg.dtype)) b_w = tl.dot(b_A, b_kb.to(b_k.dtype)) - tl.store(p_w, b_w.to(p_w.dtype.element_ty), boundary_check=(0, 1)) + desc_w.store([i_t * BT, i_k * BK], b_w.to(desc_w.dtype)) def recompute_w_u_fwd( @@ -294,70 +240,32 @@ def chunk_gla_fwd_kernel_o( m_s = tl.arange(0, BT)[:, None] >= tl.arange(0, BT)[None, :] b_o = tl.zeros([BT, BV], dtype=tl.float32) + desc_q = make_tensor_descriptor(q + (bos * H + i_h) * K, [T, K], [H * K, 1], [BT, BK]) + desc_g = make_tensor_descriptor(g + (bos * H + i_h) * K, [T, K], [H * K, 1], [BT, BK]) + desc_h = make_tensor_descriptor(h + (i_tg * H + i_h) * K * V, [V, K], [K, 1], [BV, BK]) for i_k in range(tl.cdiv(K, BK)): - p_q = tl.make_block_ptr( - q + (bos * H + i_h) * K, - (T, K), - (H * K, 1), - (i_t * BT, i_k * BK), - (BT, BK), - (1, 0), - ) - p_g = tl.make_block_ptr( - g + (bos * H + i_h) * K, - (T, K), - (H * K, 1), - (i_t * BT, i_k * BK), - (BT, BK), - (1, 0), - ) - p_h = tl.make_block_ptr( - h + (i_tg * H + i_h) * K * V, - (V, K), - (K, 1), - (i_v * BV, i_k * BK), - (BV, BK), - (1, 0), - ) - # [BT, BK] - b_q = tl.load(p_q, boundary_check=(0, 1)) + b_q = desc_q.load([i_t * BT, i_k * BK]) b_q = (b_q * scale).to(b_q.dtype) # [BT, BK] - b_g = tl.load(p_g, boundary_check=(0, 1)) + b_g = desc_g.load([i_t * BT, i_k * BK]) # [BT, BK] b_qg = (b_q * exp2(b_g)).to(b_q.dtype) # [BV, BK] - b_h = tl.load(p_h, boundary_check=(0, 1)) + b_h = desc_h.load([i_v * BV, i_k * BK]) # [BT, BV] if i_k >= 0: b_o += tl.dot(b_qg, tl.trans(b_h).to(b_qg.dtype)) - p_v = tl.make_block_ptr( - v + (bos * H + i_h) * V, - (T, V), - (H * V, 1), - (i_t * BT, i_v * BV), - (BT, BV), - (1, 0), - ) - p_o = tl.make_block_ptr( - o + (bos * H + i_h) * V, - (T, V), - (H * V, 1), - (i_t * BT, i_v * BV), - (BT, BV), - (1, 0), - ) - p_A = tl.make_block_ptr( - A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0) - ) + desc_v = make_tensor_descriptor(v + (bos * H + i_h) * V, [T, V], [H * V, 1], [BT, BV]) + desc_o = make_tensor_descriptor(o + (bos * H + i_h) * V, [T, V], [H * V, 1], [BT, BV]) + desc_A = make_tensor_descriptor(A + (bos * H + i_h) * BT, [T, BT], [H * BT, 1], [BT, BT]) # [BT, BV] - b_v = tl.load(p_v, boundary_check=(0, 1)) + b_v = desc_v.load([i_t * BT, i_v * BV]) # [BT, BT] - b_A = tl.load(p_A, boundary_check=(0, 1)) + b_A = desc_A.load([i_t * BT, 0]) b_A = tl.where(m_s, b_A, 0.0).to(b_v.dtype) b_o += tl.dot(b_A, b_v, allow_tf32=False) - tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0, 1)) + desc_o.store([i_t * BT, i_v * BV], b_o.to(desc_o.dtype)) def chunk_gla_fwd_o_gk( @@ -479,34 +387,13 @@ def kda_gate_chunk_cumsum_vector_kernel( i_s -= 1 - p_s = tl.make_block_ptr( - s + (bos * H + i_h) * S, - (T, S), - (H * S, 1), - (i_t * BT, i_s * BS), - (BT, BS), - (1, 0), - ) - p_o = tl.make_block_ptr( - o + (bos * H + i_h) * S, - (T, S), - (H * S, 1), - (i_t * BT, i_s * BS), - (BT, BS), - (1, 0), - ) + desc_s = make_tensor_descriptor(s + (bos * H + i_h) * S, [T, S], [H * S, 1], [BT, BS]) + desc_o = make_tensor_descriptor(o + (bos * H + i_h) * S, [T, S], [H * S, 1], [BT, BS]) - b_s = tl.load(p_s, boundary_check=(0, 1)).to(tl.float32) + b_s = desc_s.load([i_t * BT, i_s * BS]).to(tl.float32) if HAS_BIAS: - p_bias = tl.make_block_ptr( - g_bias + i_h * S, - (S,), - (1,), - (i_s * BS,), - (BS,), - (0,), - ) - b_bias = tl.load(p_bias, boundary_check=(0,)).to(tl.float32) + desc_bias = make_tensor_descriptor(g_bias + i_h * S, [S], [1], [BS]) + b_bias = desc_bias.load([i_s * BS]).to(tl.float32) b_s += b_bias[None, :] b_a = tl.exp(tl.load(A_log + i_h).to(tl.float32)) @@ -524,7 +411,7 @@ def kda_gate_chunk_cumsum_vector_kernel( # Boundary loads return zero, but bias and gate activation can make padded # rows nonzero. Padding trails valid rows, so it only affects masked stores. b_o = tl.cumsum(b_gate, axis=0) * cumsum_scale - tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0, 1)) + desc_o.store([i_t * BT, i_s * BS], b_o.to(desc_o.dtype)) def fused_kda_gate_chunk_cumsum( @@ -862,25 +749,10 @@ def kda_gate_fwd_kernel( stride_row = H * D stride_col = 1 - g_ptr = tl.make_block_ptr( - base=g + i_h * D, - shape=(T, D), - strides=(stride_row, stride_col), - offsets=(n_t, 0), - block_shape=(BT, BD), - order=(1, 0), - ) + desc_g = make_tensor_descriptor(g + i_h * D, [T, D], [stride_row, stride_col], [BT, BD]) + desc_y = make_tensor_descriptor(y + i_h * D, [T, D], [stride_row, stride_col], [BT, BD]) - y_ptr = tl.make_block_ptr( - base=y + i_h * D, - shape=(T, D), - strides=(stride_row, stride_col), - offsets=(n_t, 0), - block_shape=(BT, BD), - order=(1, 0), - ) - - b_g = tl.load(g_ptr, boundary_check=(0, 1)).to(tl.float32) + b_g = desc_g.load([n_t, 0]).to(tl.float32) if HAS_BIAS: n_d = tl.arange(0, BD) @@ -898,7 +770,7 @@ def kda_gate_fwd_kernel( sp = tl.where(use_linear, b_g, (1.0 / beta) * log(1.0 + tl.exp(g_scaled))) b_y = -b_a * sp - tl.store(y_ptr, b_y.to(y.dtype.element_ty), boundary_check=(0, 1)) + desc_y.store([n_t, 0], b_y.to(desc_y.dtype)) def fused_kda_gate( diff --git a/vllm/models/kimi_k3/amd/ops/third_party/kda/chunk_intra.py b/vllm/models/kimi_k3/amd/ops/third_party/kda/chunk_intra.py index 8b9aa86f9e..0c600f3ba3 100644 --- a/vllm/models/kimi_k3/amd/ops/third_party/kda/chunk_intra.py +++ b/vllm/models/kimi_k3/amd/ops/third_party/kda/chunk_intra.py @@ -12,7 +12,11 @@ import torch from vllm.platforms import current_platform from vllm.third_party.flash_linear_attention.ops.index import prepare_chunk_indices -from vllm.third_party.flash_linear_attention.ops.op import exp2, gather +from vllm.third_party.flash_linear_attention.ops.op import ( + exp2, + gather, + make_tensor_descriptor, +) from vllm.third_party.flash_linear_attention.ops.utils import is_gather_supported from vllm.triton_utils import tl, triton @@ -125,33 +129,21 @@ def chunk_kda_fwd_kernel_inter_solve_fused( ################################################################################ # off-diagonal blocks ################################################################################ + desc_qk = make_tensor_descriptor(q, [T, K], [H * K, 1], [BC, BK]) + desc_kk = make_tensor_descriptor(k, [T, K], [H * K, 1], [BC, BK]) + desc_gk = make_tensor_descriptor(g, [T, K], [HV * K, 1], [BC, BK]) for i_k in range(tl.cdiv(K, BK)): o_k = i_k * BK + tl.arange(0, BK) m_k = o_k < K - p_k0 = tl.make_block_ptr( - k, (T, K), (H * K, 1), (i_tc0, i_k * BK), (BC, BK), (1, 0) - ) - p_g0 = tl.make_block_ptr( - g, (T, K), (HV * K, 1), (i_tc0, i_k * BK), (BC, BK), (1, 0) - ) - b_k0 = tl.load(p_k0, boundary_check=(0, 1)).to(tl.float32) - b_g0 = tl.load(p_g0, boundary_check=(0, 1)).to(tl.float32) + b_k0 = desc_kk.load([i_tc0, i_k * BK]).to(tl.float32) + b_g0 = desc_gk.load([i_tc0, i_k * BK]).to(tl.float32) if i_tc1 < T: - p_q1 = tl.make_block_ptr( - q, (T, K), (H * K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0) - ) - p_k1 = tl.make_block_ptr( - k, (T, K), (H * K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0) - ) - p_g1 = tl.make_block_ptr( - g, (T, K), (HV * K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0) - ) # [BC, BK] - b_q1 = tl.load(p_q1, boundary_check=(0, 1)).to(tl.float32) - b_k1 = tl.load(p_k1, boundary_check=(0, 1)).to(tl.float32) - b_g1 = tl.load(p_g1, boundary_check=(0, 1)).to(tl.float32) + b_q1 = desc_qk.load([i_tc1, i_k * BK]).to(tl.float32) + b_k1 = desc_kk.load([i_tc1, i_k * BK]).to(tl.float32) + b_g1 = desc_gk.load([i_tc1, i_k * BK]).to(tl.float32) # [BK] b_gn1 = tl.load(g + i_tc1 * HV * K + o_k, mask=m_k, other=0).to(tl.float32) # [BC, BK] @@ -163,19 +155,10 @@ def chunk_kda_fwd_kernel_inter_solve_fused( b_Akk10 += tl.dot(b_k1 * b_gqn, b_kgt) if i_tc2 < T: - p_q2 = tl.make_block_ptr( - q, (T, K), (H * K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0) - ) - p_k2 = tl.make_block_ptr( - k, (T, K), (H * K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0) - ) - p_g2 = tl.make_block_ptr( - g, (T, K), (HV * K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0) - ) # [BC, BK] - b_q2 = tl.load(p_q2, boundary_check=(0, 1)).to(tl.float32) - b_k2 = tl.load(p_k2, boundary_check=(0, 1)).to(tl.float32) - b_g2 = tl.load(p_g2, boundary_check=(0, 1)).to(tl.float32) + b_q2 = desc_qk.load([i_tc2, i_k * BK]).to(tl.float32) + b_k2 = desc_kk.load([i_tc2, i_k * BK]).to(tl.float32) + b_g2 = desc_gk.load([i_tc2, i_k * BK]).to(tl.float32) # [BK] b_gn2 = tl.load(g + i_tc2 * HV * K + o_k, mask=m_k, other=0).to( tl.float32 @@ -195,19 +178,10 @@ def chunk_kda_fwd_kernel_inter_solve_fused( b_Akk21 += tl.dot(b_kg2, b_kgt) if i_tc3 < T: - p_q3 = tl.make_block_ptr( - q, (T, K), (H * K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0) - ) - p_k3 = tl.make_block_ptr( - k, (T, K), (H * K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0) - ) - p_g3 = tl.make_block_ptr( - g, (T, K), (HV * K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0) - ) # [BC, BK] - b_q3 = tl.load(p_q3, boundary_check=(0, 1)).to(tl.float32) - b_k3 = tl.load(p_k3, boundary_check=(0, 1)).to(tl.float32) - b_g3 = tl.load(p_g3, boundary_check=(0, 1)).to(tl.float32) + b_q3 = desc_qk.load([i_tc3, i_k * BK]).to(tl.float32) + b_k3 = desc_kk.load([i_tc3, i_k * BK]).to(tl.float32) + b_g3 = desc_gk.load([i_tc3, i_k * BK]).to(tl.float32) # [BK] b_gn3 = tl.load(g + i_tc3 * HV * K + o_k, mask=m_k, other=0).to( tl.float32 @@ -235,83 +209,36 @@ def chunk_kda_fwd_kernel_inter_solve_fused( ################################################################################ # save off-diagonal Aqk blocks and prepare Akk ################################################################################ + desc_Aqk = make_tensor_descriptor(Aqk, [T, BT], [HV * BT, 1], [BC, BC]) + p_beta = beta + bos * HV + i_hv + o_bc = tl.arange(0, BC) if i_tc1 < T: - p_Aqk10 = tl.make_block_ptr( - Aqk, (T, BT), (HV * BT, 1), (i_tc1, 0), (BC, BC), (1, 0) - ) - tl.store( - p_Aqk10, (b_Aqk10 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1) - ) + desc_Aqk.store([i_tc1, 0], (b_Aqk10 * scale).to(desc_Aqk.dtype)) - p_b1 = tl.make_block_ptr( - beta + bos * HV + i_hv, (T,), (HV,), (i_tc1,), (BC,), (0,) - ) - b_b1 = tl.load(p_b1, boundary_check=(0,)).to(tl.float32) + b_b1 = tl.load(p_beta + (i_tc1 + o_bc) * HV, mask=i_tc1 + o_bc < T, other=0).to(tl.float32) b_Akk10 = b_Akk10 * b_b1[:, None] if i_tc2 < T: - p_Aqk20 = tl.make_block_ptr( - Aqk, (T, BT), (HV * BT, 1), (i_tc2, 0), (BC, BC), (1, 0) - ) - p_Aqk21 = tl.make_block_ptr( - Aqk, (T, BT), (HV * BT, 1), (i_tc2, BC), (BC, BC), (1, 0) - ) - tl.store( - p_Aqk20, (b_Aqk20 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1) - ) - tl.store( - p_Aqk21, (b_Aqk21 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1) - ) + desc_Aqk.store([i_tc2, 0], (b_Aqk20 * scale).to(desc_Aqk.dtype)) + desc_Aqk.store([i_tc2, BC], (b_Aqk21 * scale).to(desc_Aqk.dtype)) - p_b2 = tl.make_block_ptr( - beta + bos * HV + i_hv, (T,), (HV,), (i_tc2,), (BC,), (0,) - ) - b_b2 = tl.load(p_b2, boundary_check=(0,)).to(tl.float32) + b_b2 = tl.load(p_beta + (i_tc2 + o_bc) * HV, mask=i_tc2 + o_bc < T, other=0).to(tl.float32) b_Akk20 = b_Akk20 * b_b2[:, None] b_Akk21 = b_Akk21 * b_b2[:, None] if i_tc3 < T: - p_Aqk30 = tl.make_block_ptr( - Aqk, (T, BT), (HV * BT, 1), (i_tc3, 0), (BC, BC), (1, 0) - ) - p_Aqk31 = tl.make_block_ptr( - Aqk, (T, BT), (HV * BT, 1), (i_tc3, BC), (BC, BC), (1, 0) - ) - p_Aqk32 = tl.make_block_ptr( - Aqk, (T, BT), (HV * BT, 1), (i_tc3, 2 * BC), (BC, BC), (1, 0) - ) - tl.store( - p_Aqk30, (b_Aqk30 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1) - ) - tl.store( - p_Aqk31, (b_Aqk31 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1) - ) - tl.store( - p_Aqk32, (b_Aqk32 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1) - ) + desc_Aqk.store([i_tc3, 0], (b_Aqk30 * scale).to(desc_Aqk.dtype)) + desc_Aqk.store([i_tc3, BC], (b_Aqk31 * scale).to(desc_Aqk.dtype)) + desc_Aqk.store([i_tc3, 2 * BC], (b_Aqk32 * scale).to(desc_Aqk.dtype)) - p_b3 = tl.make_block_ptr( - beta + bos * HV + i_hv, (T,), (HV,), (i_tc3,), (BC,), (0,) - ) - b_b3 = tl.load(p_b3, boundary_check=(0,)).to(tl.float32) + b_b3 = tl.load(p_beta + (i_tc3 + o_bc) * HV, mask=i_tc3 + o_bc < T, other=0).to(tl.float32) b_Akk30 = b_Akk30 * b_b3[:, None] b_Akk31 = b_Akk31 * b_b3[:, None] b_Akk32 = b_Akk32 * b_b3[:, None] - p_Akk00 = tl.make_block_ptr( - Akkd, (T, BC), (HV * BC, 1), (i_tc0, 0), (BC, BC), (1, 0) - ) - p_Akk11 = tl.make_block_ptr( - Akkd, (T, BC), (HV * BC, 1), (i_tc1, 0), (BC, BC), (1, 0) - ) - p_Akk22 = tl.make_block_ptr( - Akkd, (T, BC), (HV * BC, 1), (i_tc2, 0), (BC, BC), (1, 0) - ) - p_Akk33 = tl.make_block_ptr( - Akkd, (T, BC), (HV * BC, 1), (i_tc3, 0), (BC, BC), (1, 0) - ) - b_Ai00 = tl.load(p_Akk00, boundary_check=(0, 1)).to(tl.float32) - b_Ai11 = tl.load(p_Akk11, boundary_check=(0, 1)).to(tl.float32) - b_Ai22 = tl.load(p_Akk22, boundary_check=(0, 1)).to(tl.float32) - b_Ai33 = tl.load(p_Akk33, boundary_check=(0, 1)).to(tl.float32) + desc_Akkd = make_tensor_descriptor(Akkd, [T, BC], [HV * BC, 1], [BC, BC]) + b_Ai00 = desc_Akkd.load([i_tc0, 0]).to(tl.float32) + b_Ai11 = desc_Akkd.load([i_tc1, 0]).to(tl.float32) + b_Ai22 = desc_Akkd.load([i_tc2, 0]).to(tl.float32) + b_Ai33 = desc_Akkd.load([i_tc3, 0]).to(tl.float32) ################################################################################ # forward substitution on diagonals @@ -397,47 +324,18 @@ def chunk_kda_fwd_kernel_inter_solve_fused( # store full Akk_inv to Akk ################################################################################ - p_Akk00 = tl.make_block_ptr( - Akk, (T, BT), (HV * BT, 1), (i_tc0, 0), (BC, BC), (1, 0) - ) - p_Akk10 = tl.make_block_ptr( - Akk, (T, BT), (HV * BT, 1), (i_tc1, 0), (BC, BC), (1, 0) - ) - p_Akk11 = tl.make_block_ptr( - Akk, (T, BT), (HV * BT, 1), (i_tc1, BC), (BC, BC), (1, 0) - ) - p_Akk20 = tl.make_block_ptr( - Akk, (T, BT), (HV * BT, 1), (i_tc2, 0), (BC, BC), (1, 0) - ) - p_Akk21 = tl.make_block_ptr( - Akk, (T, BT), (HV * BT, 1), (i_tc2, BC), (BC, BC), (1, 0) - ) - p_Akk22 = tl.make_block_ptr( - Akk, (T, BT), (HV * BT, 1), (i_tc2, 2 * BC), (BC, BC), (1, 0) - ) - p_Akk30 = tl.make_block_ptr( - Akk, (T, BT), (HV * BT, 1), (i_tc3, 0), (BC, BC), (1, 0) - ) - p_Akk31 = tl.make_block_ptr( - Akk, (T, BT), (HV * BT, 1), (i_tc3, BC), (BC, BC), (1, 0) - ) - p_Akk32 = tl.make_block_ptr( - Akk, (T, BT), (HV * BT, 1), (i_tc3, 2 * BC), (BC, BC), (1, 0) - ) - p_Akk33 = tl.make_block_ptr( - Akk, (T, BT), (HV * BT, 1), (i_tc3, 3 * BC), (BC, BC), (1, 0) - ) + desc_Akk = make_tensor_descriptor(Akk, [T, BT], [HV * BT, 1], [BC, BC]) - tl.store(p_Akk00, b_Ai00.to(Akk.dtype.element_ty), boundary_check=(0, 1)) - tl.store(p_Akk10, b_Ai10.to(Akk.dtype.element_ty), boundary_check=(0, 1)) - tl.store(p_Akk11, b_Ai11.to(Akk.dtype.element_ty), boundary_check=(0, 1)) - tl.store(p_Akk20, b_Ai20.to(Akk.dtype.element_ty), boundary_check=(0, 1)) - tl.store(p_Akk21, b_Ai21.to(Akk.dtype.element_ty), boundary_check=(0, 1)) - tl.store(p_Akk22, b_Ai22.to(Akk.dtype.element_ty), boundary_check=(0, 1)) - tl.store(p_Akk30, b_Ai30.to(Akk.dtype.element_ty), boundary_check=(0, 1)) - tl.store(p_Akk31, b_Ai31.to(Akk.dtype.element_ty), boundary_check=(0, 1)) - tl.store(p_Akk32, b_Ai32.to(Akk.dtype.element_ty), boundary_check=(0, 1)) - tl.store(p_Akk33, b_Ai33.to(Akk.dtype.element_ty), boundary_check=(0, 1)) + desc_Akk.store([i_tc0, 0], b_Ai00.to(desc_Akk.dtype)) + desc_Akk.store([i_tc1, 0], b_Ai10.to(desc_Akk.dtype)) + desc_Akk.store([i_tc1, BC], b_Ai11.to(desc_Akk.dtype)) + desc_Akk.store([i_tc2, 0], b_Ai20.to(desc_Akk.dtype)) + desc_Akk.store([i_tc2, BC], b_Ai21.to(desc_Akk.dtype)) + desc_Akk.store([i_tc2, 2 * BC], b_Ai22.to(desc_Akk.dtype)) + desc_Akk.store([i_tc3, 0], b_Ai30.to(desc_Akk.dtype)) + desc_Akk.store([i_tc3, BC], b_Ai31.to(desc_Akk.dtype)) + desc_Akk.store([i_tc3, 2 * BC], b_Ai32.to(desc_Akk.dtype)) + desc_Akk.store([i_tc3, 3 * BC], b_Ai33.to(desc_Akk.dtype)) @triton.heuristics( @@ -505,16 +403,14 @@ def chunk_kda_fwd_kernel_intra_sub_chunk( Aqk = Aqk + (bos * HV + i_hv) * BT Akk = Akk + (bos * HV + i_hv) * BC - p_q = tl.make_block_ptr(q, (T, K), (H * K, 1), (i_ti, 0), (BC, BK), (1, 0)) - p_k = tl.make_block_ptr(k, (T, K), (H * K, 1), (i_ti, 0), (BC, BK), (1, 0)) - p_g = tl.make_block_ptr(g, (T, K), (HV * K, 1), (i_ti, 0), (BC, BK), (1, 0)) + desc_q = make_tensor_descriptor(q, [T, K], [H * K, 1], [BC, BK]) + desc_k = make_tensor_descriptor(k, [T, K], [H * K, 1], [BC, BK]) + desc_g = make_tensor_descriptor(g, [T, K], [HV * K, 1], [BC, BK]) - p_beta = tl.make_block_ptr(beta, (T,), (HV,), (i_ti,), (BC,), (0,)) - - b_q = tl.load(p_q, boundary_check=(0, 1)) - b_k = tl.load(p_k, boundary_check=(0, 1)) - b_g = tl.load(p_g, boundary_check=(0, 1)) - b_beta = tl.load(p_beta, boundary_check=(0,)).to(tl.float32) + b_q = desc_q.load([i_ti, 0]) + b_k = desc_k.load([i_ti, 0]) + b_g = desc_g.load([i_ti, 0]) + b_beta = tl.load(beta + o_c * HV, mask=m_c, other=0).to(tl.float32) if USE_GATHER: b_gn = gather( @@ -546,12 +442,10 @@ def chunk_kda_fwd_kernel_intra_sub_chunk( b_Aqk = tl.where(m_Aqk, b_Aqk, 0.0) b_Akk = tl.where(m_Akk, b_Akk, 0.0) - p_Aqk = tl.make_block_ptr( - Aqk, (T, BT), (HV * BT, 1), (i_ti, i_i * BC), (BC, BC), (1, 0) - ) - p_Akk = tl.make_block_ptr(Akk, (T, BC), (HV * BC, 1), (i_ti, 0), (BC, BC), (1, 0)) - tl.store(p_Aqk, b_Aqk.to(Aqk.dtype.element_ty), boundary_check=(0, 1)) - tl.store(p_Akk, b_Akk.to(Akk.dtype.element_ty), boundary_check=(0, 1)) + desc_Aqk = make_tensor_descriptor(Aqk, [T, BT], [HV * BT, 1], [BC, BC]) + desc_Akk = make_tensor_descriptor(Akk, [T, BC], [HV * BC, 1], [BC, BC]) + desc_Aqk.store([i_ti, i_i * BC], b_Aqk.to(desc_Aqk.dtype)) + desc_Akk.store([i_ti, 0], b_Akk.to(desc_Akk.dtype)) tl.debug_barrier() @@ -566,7 +460,7 @@ def chunk_kda_fwd_kernel_intra_sub_chunk( b_a += tl.sum(b_a[:, None] * b_Ai, 0) b_Ai = tl.where((o_i == i)[:, None], b_a, b_Ai) b_Ai += m_I - tl.store(p_Akk, b_Ai.to(Akk.dtype.element_ty), boundary_check=(0, 1)) + desc_Akk.store([i_ti, 0], b_Ai.to(desc_Akk.dtype)) def chunk_kda_fwd_intra( diff --git a/vllm/models/kimi_k3/amd/ops/third_party/kda/chunk_intra_token_parallel.py b/vllm/models/kimi_k3/amd/ops/third_party/kda/chunk_intra_token_parallel.py index 86654f2644..7740e64d5d 100644 --- a/vllm/models/kimi_k3/amd/ops/third_party/kda/chunk_intra_token_parallel.py +++ b/vllm/models/kimi_k3/amd/ops/third_party/kda/chunk_intra_token_parallel.py @@ -12,7 +12,7 @@ import torch -from vllm.third_party.flash_linear_attention.ops.op import exp2 +from vllm.third_party.flash_linear_attention.ops.op import exp2, make_tensor_descriptor from vllm.triton_utils import tl, triton @@ -108,20 +108,15 @@ def chunk_kda_fwd_kernel_intra_token_parallel( b_k = tl.load(k + i_t * H * K + p_qk, mask=m_hk, other=0).to(tl.float32) # g: [B, T, HV, K], beta: [B, T, HV] - p_g = tl.make_block_ptr( - g + i_t * HV * K, (HV, K), (K, 1), (i_hg * BH, 0), (BH, BK), (1, 0) - ) - p_beta = tl.make_block_ptr(beta + i_t * HV, (HV,), (1,), (i_hg * BH,), (BH,), (0,)) - b_g = tl.load(p_g, boundary_check=(0, 1)).to(tl.float32) - b_beta = tl.load(p_beta, boundary_check=(0,)).to(tl.float32) + desc_g = make_tensor_descriptor(g + i_t * HV * K, [HV, K], [K, 1], [BH, BK]) + b_g = desc_g.load([i_hg * BH, 0]).to(tl.float32) + b_beta = tl.load(beta + i_t * HV + o_hv, mask=m_hv, other=0).to(tl.float32) b_k *= b_beta[:, None] for j in range(i_ts, min(i_t + 1, min(T, i_ts + BC))): b_kj = tl.load(k + j * H * K + p_qk, mask=m_hk, other=0).to(tl.float32) - p_gj = tl.make_block_ptr( - g + j * HV * K, (HV, K), (K, 1), (i_hg * BH, 0), (BH, BK), (1, 0) - ) - b_gj = tl.load(p_gj, boundary_check=(0, 1)).to(tl.float32) + desc_gj = make_tensor_descriptor(g + j * HV * K, [HV, K], [K, 1], [BH, BK]) + b_gj = desc_gj.load([i_hg * BH, 0]).to(tl.float32) b_kgj = tl.where(m_k[None, :], b_kj * exp2(b_g - b_gj), 0.0) b_Aqk = tl.sum(b_q * b_kgj, axis=1) * scale