diff --git a/vllm/model_executor/layers/fla/ops/chunk_delta_h.py b/vllm/model_executor/layers/fla/ops/chunk_delta_h.py index 83bf33079f..89a8c4d909 100644 --- a/vllm/model_executor/layers/fla/ops/chunk_delta_h.py +++ b/vllm/model_executor/layers/fla/ops/chunk_delta_h.py @@ -13,7 +13,7 @@ import torch from vllm.triton_utils import tl, triton from .index import prepare_chunk_indices, prepare_chunk_offsets -from .op import exp +from .op import exp, exp2 from .utils import FLA_CHUNK_SIZE, use_cuda_graph NUM_WARPS = [2, 4, 8, 16] @@ -65,6 +65,7 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64( STORE_FINAL_STATE: tl.constexpr, SAVE_NEW_VALUE: tl.constexpr, IS_VARLEN: tl.constexpr, + USE_EXP2: tl.constexpr, ): i_v, i_nh = tl.program_id(0), tl.program_id(1) i_n, i_h = i_nh // H, i_nh % H @@ -210,8 +211,12 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64( g + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,) ) b_g = tl.load(p_g, boundary_check=(0,)) - b_v = b_v * tl.where(m_t, exp(b_g_last - b_g), 0)[:, None] - b_g_last = exp(b_g_last) + if USE_EXP2: + b_v = b_v * tl.where(m_t, exp2(b_g_last - b_g), 0)[:, None] + b_g_last = exp2(b_g_last) + else: + b_v = b_v * tl.where(m_t, exp(b_g_last - b_g), 0)[:, None] + b_g_last = exp(b_g_last) b_h1 *= b_g_last if K > 64: b_h2 *= b_g_last @@ -227,7 +232,10 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64( mask=(o_k1 < K), other=0.0, ) - b_h1 *= exp(b_gk_last1)[None, :] + if USE_EXP2: + b_h1 *= exp2(b_gk_last1)[None, :] + else: + b_h1 *= exp(b_gk_last1)[None, :] if K > 64: o_k2 = 64 + o_k1 b_gk_last2 = tl.load( @@ -235,7 +243,10 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64( mask=(o_k2 < K), other=0.0, ) - b_h2 *= exp(b_gk_last2)[None, :] + if USE_EXP2: + b_h2 *= exp2(b_gk_last2)[None, :] + else: + b_h2 *= exp(b_gk_last2)[None, :] if K > 128: o_k3 = 128 + o_k1 b_gk_last3 = tl.load( @@ -243,7 +254,10 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64( mask=(o_k3 < K), other=0.0, ) - b_h3 *= exp(b_gk_last3)[None, :] + if USE_EXP2: + b_h3 *= exp2(b_gk_last3)[None, :] + else: + b_h3 *= exp(b_gk_last3)[None, :] if K > 192: o_k4 = 192 + o_k1 b_gk_last4 = tl.load( @@ -251,7 +265,10 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64( mask=(o_k4 < K), other=0.0, ) - b_h4 *= exp(b_gk_last4)[None, :] + if USE_EXP2: + b_h4 *= exp2(b_gk_last4)[None, :] + else: + b_h4 *= exp(b_gk_last4)[None, :] b_v = b_v.to(k.dtype.element_ty) p_k = tl.make_block_ptr( @@ -311,6 +328,7 @@ def chunk_gated_delta_rule_fwd_h( cu_seqlens: torch.Tensor | None = None, chunk_indices: torch.Tensor | None = None, chunk_offsets: torch.Tensor | None = None, + use_exp2: bool = False, ) -> tuple[torch.Tensor, torch.Tensor]: # This kernel is slightly different from fla to support Q/K with different head numbers. # In fla, Q/K always have the same head number, so Hg is always equal to H. @@ -357,5 +375,6 @@ def chunk_gated_delta_rule_fwd_h( K=K, V=V, BT=BT, + USE_EXP2=use_exp2, ) return h, v_new, final_state diff --git a/vllm/model_executor/layers/fla/ops/kda.py b/vllm/model_executor/layers/fla/ops/kda.py index 1a6f7cc0f5..5f1418fd24 100644 --- a/vllm/model_executor/layers/fla/ops/kda.py +++ b/vllm/model_executor/layers/fla/ops/kda.py @@ -14,14 +14,14 @@ import torch.nn as nn from vllm.model_executor.custom_op import CustomOp from vllm.triton_utils import tl, triton -from vllm.utils.math_utils import cdiv, next_power_of_2 +from vllm.utils.math_utils import RCP_LN2, cdiv, next_power_of_2 from .chunk_delta_h import chunk_gated_delta_rule_fwd_h from .cumsum import chunk_local_cumsum from .fused_recurrent import fused_recurrent_gated_delta_rule_fwd_kernel from .index import prepare_chunk_indices from .l2norm import l2norm_fwd -from .op import exp, log +from .op import exp2, log from .solve_tril import solve_tril from .utils import FLA_CHUNK_SIZE, is_amd @@ -594,16 +594,16 @@ def chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_inter( b_gn = tl.load(g + (i_t * BT + i_i * BC) * H * K + o_k, mask=m_k, other=0) # [BC, BK] b_g = tl.load(p_g, boundary_check=(0, 1)) - b_k = tl.load(p_k, boundary_check=(0, 1)) * exp(b_g - b_gn[None, :]) + b_k = tl.load(p_k, boundary_check=(0, 1)) * exp2(b_g - b_gn[None, :]) # [BK, BC] b_gk = tl.load(p_gk, boundary_check=(0, 1)) b_kt = tl.load(b_kt, boundary_check=(0, 1)) # [BC, BC] - b_ktg = b_kt * exp(b_gn[:, None] - b_gk) + b_ktg = b_kt * exp2(b_gn[:, None] - b_gk) b_A += tl.dot(b_k, b_ktg) b_q = tl.load(p_q, boundary_check=(0, 1)) - b_qg = b_q * exp(b_g - b_gn[None, :]) * scale + b_qg = b_q * exp2(b_g - b_gn[None, :]) * scale b_Aqk += tl.dot(b_qg, b_ktg) b_A *= b_b[:, None] @@ -703,7 +703,7 @@ def chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_intra( for j in range(0, min(BC, T - i_t * BT - i_i * BC)): b_kt = tl.load(p_kt, mask=m_k, other=0).to(tl.float32) b_gk = tl.load(p_gk, mask=m_k, other=0).to(tl.float32) - b_ktg = b_kt[None, :] * exp(b_g - b_gk[None, :]) + b_ktg = b_kt[None, :] * exp2(b_g - b_gk[None, :]) b_A = tl.sum(b_k * b_ktg, 1) b_A = tl.where(o_i > j, b_A, 0.0) b_Aqk = tl.sum(b_q * b_ktg, 1) @@ -912,7 +912,7 @@ def recompute_w_u_fwd_kernel( (1, 0), ) b_gk = tl.load(p_gk, boundary_check=(0, 1)) - b_kb *= exp(b_gk) + b_kb *= exp2(b_gk) if STORE_QG: p_q = tl.make_block_ptr( q + (bos * H + i_h) * K, @@ -931,7 +931,7 @@ def recompute_w_u_fwd_kernel( (1, 0), ) b_q = tl.load(p_q, boundary_check=(0, 1)) - b_qg = b_q * exp(b_gk) + b_qg = b_q * exp2(b_gk) tl.store(p_qg, b_qg.to(p_qg.dtype.element_ty), boundary_check=(0, 1)) if STORE_KG: last_idx = min(i_t * BT + BT, T) - 1 @@ -941,7 +941,7 @@ def recompute_w_u_fwd_kernel( b_gn = tl.load( gk + ((bos + last_idx) * H + i_h) * K + o_k, mask=m_k, other=0.0 ) - b_kg = b_k * exp(b_gn - b_gk) + b_kg = b_k * exp2(b_gn - b_gk) p_kg = tl.make_block_ptr( kg + (bos * H + i_h) * K, @@ -1089,7 +1089,7 @@ def chunk_gla_fwd_kernel_o( # [BT, BK] b_g = tl.load(p_g, boundary_check=(0, 1)) # [BT, BK] - b_qg = (b_q * exp(b_g)).to(b_q.dtype) + b_qg = (b_q * exp2(b_g)).to(b_q.dtype) # [BV, BK] b_h = tl.load(p_h, boundary_check=(0, 1)) # [BT, BV] @@ -1179,6 +1179,9 @@ def chunk_kda_fwd( ): chunk_size = FLA_CHUNK_SIZE g = chunk_local_cumsum(g, chunk_size=chunk_size, cu_seqlens=cu_seqlens) + # KDA evaluates cumulative gate decays with exp2. Convert from natural-log + # space so exp(x) is preserved as exp2(x / ln(2)). + g = g * RCP_LN2 # the intra Aqk is kept in fp32 # the computation has very marginal effect on the entire throughput A, Aqk = chunk_kda_scaled_dot_kkt_fwd( @@ -1209,6 +1212,7 @@ def chunk_kda_fwd( initial_state=initial_state, output_final_state=output_final_state, cu_seqlens=cu_seqlens, + use_exp2=True, ) del w, u, kg o = chunk_gla_fwd_o_gk( diff --git a/vllm/model_executor/layers/fla/ops/op.py b/vllm/model_executor/layers/fla/ops/op.py index a91975c8e5..777ca585f3 100644 --- a/vllm/model_executor/layers/fla/ops/op.py +++ b/vllm/model_executor/layers/fla/ops/op.py @@ -15,10 +15,12 @@ from .utils import is_gather_supported if os.environ.get("FLA_USE_FAST_OPS", "0") == "1": exp = tldevice.fast_expf + exp2 = tl.exp2 log = tldevice.fast_logf log2 = tldevice.fast_log2f else: exp = tl.exp + exp2 = tl.exp2 log = tl.log log2 = tl.log2