Update KDA chunk prefill decay to use exp2 semantics (#43195)
Signed-off-by: zexplorerhj <19794632+zexplorerhj@users.noreply.github.com> Co-authored-by: zexplorerhj <19794632+zexplorerhj@users.noreply.github.com>
This commit is contained in:
co-authored by
zexplorerhj
parent
0a54df2847
commit
b719b1635b
@@ -13,7 +13,7 @@ import torch
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
from .index import prepare_chunk_indices, prepare_chunk_offsets
|
||||
from .op import exp
|
||||
from .op import exp, exp2
|
||||
from .utils import FLA_CHUNK_SIZE, use_cuda_graph
|
||||
|
||||
NUM_WARPS = [2, 4, 8, 16]
|
||||
@@ -65,6 +65,7 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64(
|
||||
STORE_FINAL_STATE: tl.constexpr,
|
||||
SAVE_NEW_VALUE: tl.constexpr,
|
||||
IS_VARLEN: tl.constexpr,
|
||||
USE_EXP2: tl.constexpr,
|
||||
):
|
||||
i_v, i_nh = tl.program_id(0), tl.program_id(1)
|
||||
i_n, i_h = i_nh // H, i_nh % H
|
||||
@@ -210,8 +211,12 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64(
|
||||
g + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,)
|
||||
)
|
||||
b_g = tl.load(p_g, boundary_check=(0,))
|
||||
b_v = b_v * tl.where(m_t, exp(b_g_last - b_g), 0)[:, None]
|
||||
b_g_last = exp(b_g_last)
|
||||
if USE_EXP2:
|
||||
b_v = b_v * tl.where(m_t, exp2(b_g_last - b_g), 0)[:, None]
|
||||
b_g_last = exp2(b_g_last)
|
||||
else:
|
||||
b_v = b_v * tl.where(m_t, exp(b_g_last - b_g), 0)[:, None]
|
||||
b_g_last = exp(b_g_last)
|
||||
b_h1 *= b_g_last
|
||||
if K > 64:
|
||||
b_h2 *= b_g_last
|
||||
@@ -227,7 +232,10 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64(
|
||||
mask=(o_k1 < K),
|
||||
other=0.0,
|
||||
)
|
||||
b_h1 *= exp(b_gk_last1)[None, :]
|
||||
if USE_EXP2:
|
||||
b_h1 *= exp2(b_gk_last1)[None, :]
|
||||
else:
|
||||
b_h1 *= exp(b_gk_last1)[None, :]
|
||||
if K > 64:
|
||||
o_k2 = 64 + o_k1
|
||||
b_gk_last2 = tl.load(
|
||||
@@ -235,7 +243,10 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64(
|
||||
mask=(o_k2 < K),
|
||||
other=0.0,
|
||||
)
|
||||
b_h2 *= exp(b_gk_last2)[None, :]
|
||||
if USE_EXP2:
|
||||
b_h2 *= exp2(b_gk_last2)[None, :]
|
||||
else:
|
||||
b_h2 *= exp(b_gk_last2)[None, :]
|
||||
if K > 128:
|
||||
o_k3 = 128 + o_k1
|
||||
b_gk_last3 = tl.load(
|
||||
@@ -243,7 +254,10 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64(
|
||||
mask=(o_k3 < K),
|
||||
other=0.0,
|
||||
)
|
||||
b_h3 *= exp(b_gk_last3)[None, :]
|
||||
if USE_EXP2:
|
||||
b_h3 *= exp2(b_gk_last3)[None, :]
|
||||
else:
|
||||
b_h3 *= exp(b_gk_last3)[None, :]
|
||||
if K > 192:
|
||||
o_k4 = 192 + o_k1
|
||||
b_gk_last4 = tl.load(
|
||||
@@ -251,7 +265,10 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64(
|
||||
mask=(o_k4 < K),
|
||||
other=0.0,
|
||||
)
|
||||
b_h4 *= exp(b_gk_last4)[None, :]
|
||||
if USE_EXP2:
|
||||
b_h4 *= exp2(b_gk_last4)[None, :]
|
||||
else:
|
||||
b_h4 *= exp(b_gk_last4)[None, :]
|
||||
b_v = b_v.to(k.dtype.element_ty)
|
||||
|
||||
p_k = tl.make_block_ptr(
|
||||
@@ -311,6 +328,7 @@ def chunk_gated_delta_rule_fwd_h(
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
use_exp2: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# This kernel is slightly different from fla to support Q/K with different head numbers.
|
||||
# In fla, Q/K always have the same head number, so Hg is always equal to H.
|
||||
@@ -357,5 +375,6 @@ def chunk_gated_delta_rule_fwd_h(
|
||||
K=K,
|
||||
V=V,
|
||||
BT=BT,
|
||||
USE_EXP2=use_exp2,
|
||||
)
|
||||
return h, v_new, final_state
|
||||
|
||||
@@ -14,14 +14,14 @@ import torch.nn as nn
|
||||
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import cdiv, next_power_of_2
|
||||
from vllm.utils.math_utils import RCP_LN2, cdiv, next_power_of_2
|
||||
|
||||
from .chunk_delta_h import chunk_gated_delta_rule_fwd_h
|
||||
from .cumsum import chunk_local_cumsum
|
||||
from .fused_recurrent import fused_recurrent_gated_delta_rule_fwd_kernel
|
||||
from .index import prepare_chunk_indices
|
||||
from .l2norm import l2norm_fwd
|
||||
from .op import exp, log
|
||||
from .op import exp2, log
|
||||
from .solve_tril import solve_tril
|
||||
from .utils import FLA_CHUNK_SIZE, is_amd
|
||||
|
||||
@@ -594,16 +594,16 @@ def chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_inter(
|
||||
b_gn = tl.load(g + (i_t * BT + i_i * BC) * H * K + o_k, mask=m_k, other=0)
|
||||
# [BC, BK]
|
||||
b_g = tl.load(p_g, boundary_check=(0, 1))
|
||||
b_k = tl.load(p_k, boundary_check=(0, 1)) * exp(b_g - b_gn[None, :])
|
||||
b_k = tl.load(p_k, boundary_check=(0, 1)) * exp2(b_g - b_gn[None, :])
|
||||
# [BK, BC]
|
||||
b_gk = tl.load(p_gk, boundary_check=(0, 1))
|
||||
b_kt = tl.load(b_kt, boundary_check=(0, 1))
|
||||
# [BC, BC]
|
||||
b_ktg = b_kt * exp(b_gn[:, None] - b_gk)
|
||||
b_ktg = b_kt * exp2(b_gn[:, None] - b_gk)
|
||||
b_A += tl.dot(b_k, b_ktg)
|
||||
|
||||
b_q = tl.load(p_q, boundary_check=(0, 1))
|
||||
b_qg = b_q * exp(b_g - b_gn[None, :]) * scale
|
||||
b_qg = b_q * exp2(b_g - b_gn[None, :]) * scale
|
||||
b_Aqk += tl.dot(b_qg, b_ktg)
|
||||
|
||||
b_A *= b_b[:, None]
|
||||
@@ -703,7 +703,7 @@ def chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_intra(
|
||||
for j in range(0, min(BC, T - i_t * BT - i_i * BC)):
|
||||
b_kt = tl.load(p_kt, mask=m_k, other=0).to(tl.float32)
|
||||
b_gk = tl.load(p_gk, mask=m_k, other=0).to(tl.float32)
|
||||
b_ktg = b_kt[None, :] * exp(b_g - b_gk[None, :])
|
||||
b_ktg = b_kt[None, :] * exp2(b_g - b_gk[None, :])
|
||||
b_A = tl.sum(b_k * b_ktg, 1)
|
||||
b_A = tl.where(o_i > j, b_A, 0.0)
|
||||
b_Aqk = tl.sum(b_q * b_ktg, 1)
|
||||
@@ -912,7 +912,7 @@ def recompute_w_u_fwd_kernel(
|
||||
(1, 0),
|
||||
)
|
||||
b_gk = tl.load(p_gk, boundary_check=(0, 1))
|
||||
b_kb *= exp(b_gk)
|
||||
b_kb *= exp2(b_gk)
|
||||
if STORE_QG:
|
||||
p_q = tl.make_block_ptr(
|
||||
q + (bos * H + i_h) * K,
|
||||
@@ -931,7 +931,7 @@ def recompute_w_u_fwd_kernel(
|
||||
(1, 0),
|
||||
)
|
||||
b_q = tl.load(p_q, boundary_check=(0, 1))
|
||||
b_qg = b_q * exp(b_gk)
|
||||
b_qg = b_q * exp2(b_gk)
|
||||
tl.store(p_qg, b_qg.to(p_qg.dtype.element_ty), boundary_check=(0, 1))
|
||||
if STORE_KG:
|
||||
last_idx = min(i_t * BT + BT, T) - 1
|
||||
@@ -941,7 +941,7 @@ def recompute_w_u_fwd_kernel(
|
||||
b_gn = tl.load(
|
||||
gk + ((bos + last_idx) * H + i_h) * K + o_k, mask=m_k, other=0.0
|
||||
)
|
||||
b_kg = b_k * exp(b_gn - b_gk)
|
||||
b_kg = b_k * exp2(b_gn - b_gk)
|
||||
|
||||
p_kg = tl.make_block_ptr(
|
||||
kg + (bos * H + i_h) * K,
|
||||
@@ -1089,7 +1089,7 @@ def chunk_gla_fwd_kernel_o(
|
||||
# [BT, BK]
|
||||
b_g = tl.load(p_g, boundary_check=(0, 1))
|
||||
# [BT, BK]
|
||||
b_qg = (b_q * exp(b_g)).to(b_q.dtype)
|
||||
b_qg = (b_q * exp2(b_g)).to(b_q.dtype)
|
||||
# [BV, BK]
|
||||
b_h = tl.load(p_h, boundary_check=(0, 1))
|
||||
# [BT, BV]
|
||||
@@ -1179,6 +1179,9 @@ def chunk_kda_fwd(
|
||||
):
|
||||
chunk_size = FLA_CHUNK_SIZE
|
||||
g = chunk_local_cumsum(g, chunk_size=chunk_size, cu_seqlens=cu_seqlens)
|
||||
# KDA evaluates cumulative gate decays with exp2. Convert from natural-log
|
||||
# space so exp(x) is preserved as exp2(x / ln(2)).
|
||||
g = g * RCP_LN2
|
||||
# the intra Aqk is kept in fp32
|
||||
# the computation has very marginal effect on the entire throughput
|
||||
A, Aqk = chunk_kda_scaled_dot_kkt_fwd(
|
||||
@@ -1209,6 +1212,7 @@ def chunk_kda_fwd(
|
||||
initial_state=initial_state,
|
||||
output_final_state=output_final_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
use_exp2=True,
|
||||
)
|
||||
del w, u, kg
|
||||
o = chunk_gla_fwd_o_gk(
|
||||
|
||||
@@ -15,10 +15,12 @@ from .utils import is_gather_supported
|
||||
|
||||
if os.environ.get("FLA_USE_FAST_OPS", "0") == "1":
|
||||
exp = tldevice.fast_expf
|
||||
exp2 = tl.exp2
|
||||
log = tldevice.fast_logf
|
||||
log2 = tldevice.fast_log2f
|
||||
else:
|
||||
exp = tl.exp
|
||||
exp2 = tl.exp2
|
||||
log = tl.log
|
||||
log2 = tl.log2
|
||||
|
||||
|
||||
Reference in New Issue
Block a user