Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
+1 -3
View File
@@ -61,9 +61,7 @@ class ROCmLatentMoERunner(MoERunner):
shared_output: torch.Tensor,
trunc_size: int | None,
) -> torch.Tensor:
"""
Tier 2: column-parallel up-projection folded into the final reduce.
"""
"""Tier 2: column-parallel up-projection folded into the final reduce."""
if not self._logged_sharded_tail:
self._logged_sharded_tail = True
logger.info_once(
+1 -3
View File
@@ -317,9 +317,7 @@ class KimiMoE(nn.Module):
class KimiMLAAttention(nn.Module):
"""
Main reference: DeepseekV2 vllm Implementation
"""
"""Main reference: DeepseekV2 vllm Implementation."""
def __init__(
self,
+9
View File
@@ -213,9 +213,17 @@ def fused_kda_chunk(
Args:
qg: ``q * exp2(gk_cumsum)``, ``[1, T, H, 128]``.
w: chunk-local WY representation of the gated keys.
u: chunk-local WY representation of the values.
kg_t: chunk-major transposed gated keys, ``[chunks, H, 128, 64]``.
aqk: intra-chunk query-key attention, ``[chunks, H, 64, 64]``.
decay: ``exp2`` of each chunk's last gate row, ``[chunks, H, 128]``.
out: output buffer, ``[1, T, H, 128]``; may alias ``u``'s source.
scale: scale applied to the query-key products.
cu_seqlens: int32 cumulative sequence lengths.
initial_state: fp32 per-sequence initial recurrent state, or ``None``.
output_final_state: whether to return the final recurrent state.
chunk_offsets: int32 per-sequence first chunk index.
checkpoint_state: destination for the mid-prefill state snapshots,
fp32 ``[rows, H, 128, 128]``. Without
``checkpoint_state_indices`` it is a staging buffer indexed by
@@ -239,6 +247,7 @@ def fused_kda_chunk(
has_initial_state: bool ``[N]``; a false entry starts that sequence
from a zero state and its cache row is read only, never before the
walk writes it.
"""
if state_cache is not None:
if initial_state is not None or output_final_state:
@@ -52,6 +52,22 @@ def chunk_kda_prefill(
"""Run chunk KDA from raw gate and beta projections.
Args:
q: query tensor, ``[1, T, H, K]``.
k: key tensor, ``[1, T, H, K]``.
v: value tensor, ``[1, T, H, V]``.
raw_g: raw gate projection, before the activation.
raw_beta: raw beta projection, before the activation.
A_log: log of the per-head gate decay.
g_bias: optional per-head gate bias.
scale: scale applied to the query-key products. Defaults to
``k.shape[-1] ** -0.5``.
initial_state: fp32 per-sequence initial recurrent state, or ``None``.
output_final_state: whether to return the final recurrent state.
lower_bound: optional floor applied to the gate.
use_qk_l2norm_in_kernel: fold the q/k L2 norm into the kernel.
cu_seqlens: int32 cumulative sequence lengths.
chunk_indices: precomputed chunk indices for ``cu_seqlens``.
chunk_offsets: int32 per-sequence first chunk index.
use_fused_chunk: request the two-kernel ROCm path. It is used only when
every one of its preconditions holds; otherwise the Triton path
runs unchanged.
@@ -71,6 +87,7 @@ def chunk_kda_prefill(
Returns:
The output and, when requested, the final recurrent state.
"""
if scale is None:
scale = k.shape[-1] ** -0.5
+6 -7
View File
@@ -910,13 +910,12 @@ def fused_kda_gate(
threshold: float = 20.0,
lower_bound: float | None = None,
) -> torch.Tensor:
"""
Forward pass for KDA gate:
input g: [..., H*D]
param A: [H] or [1, 1, H, 1]
beta: softplus beta parameter
threshold: softplus threshold parameter
return : [..., H, D]
"""Forward pass for KDA gate:
input g: [..., H*D]
param A: [H] or [1, 1, H, 1]
beta: softplus beta parameter
threshold: softplus threshold parameter
return : [..., H, D]
"""
orig_shape = g.shape[:-1]
+1 -2
View File
@@ -59,8 +59,7 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
USE_SAFE_GATE: tl.constexpr,
SOLVE_TRIL_DOT_PRECISION: tl.constexpr,
):
"""
Fused kernel: compute inter-subchunk Akk + solve_tril in one pass.
"""Fused kernel: compute inter-subchunk Akk + solve_tril in one pass.
Prerequisite: token_parallel has already computed diagonal Akk blocks in Akkd.
This kernel:
@@ -151,8 +151,7 @@ def chunk_kda_fwd_intra_token_parallel(
chunk_size: int = 64,
sub_chunk_size: int = 16,
) -> None:
"""
Token-parallel implementation: each token gets its own thread block.
"""Token-parallel implementation: each token gets its own thread block.
Supports both fixed-length and variable-length sequences.
Reduces wasted computation on padding.
@@ -164,10 +163,12 @@ def chunk_kda_fwd_intra_token_parallel(
gk: [B, T, HV, K] cumsum of gates (HV >= H for GVA)
beta: [B, T, HV]
Aqk: [B, T, HV, BT] output tensor to write to
cu_seqlens: cumulative sequence lengths for variable-length input
Akk: [B, T, HV, BC] output tensor for diagonal blocks (fp32)
scale: attention scale
chunk_size: BT (default 64)
sub_chunk_size: BC (default 16)
"""
B, T, H, K, HV = *q.shape, gk.shape[2]
N = len(cu_seqlens) - 1 if cu_seqlens is not None else B
@@ -275,9 +275,7 @@ class LatentMoERunner(MoERunner):
shared_output: torch.Tensor,
trunc_size: int | None,
) -> torch.Tensor:
"""
Tier 2: column-parallel up-projection folded into the final reduce.
"""
"""Tier 2: column-parallel up-projection folded into the final reduce."""
transform = self.routed_output_transform
assert transform is not None
@@ -751,6 +751,7 @@ def run_kda_projection_overlap(
Returns:
The Q/K/V/G projection, F_B output, and beta projection.
"""
num_tokens = hidden_states.shape[0]
qkvg_weight = packed_weight[:_KDA_QKVG_SIZE]
+2 -1
View File
@@ -745,7 +745,8 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
slot_mapping: torch.Tensor,
) -> torch.Tensor:
"""Fused decode query-concat + latent cache insert, dispatched by cache
dtype (same policy as prefill: fp8 cache -> fp8 query)."""
dtype (same policy as prefill: fp8 cache -> fp8 query).
"""
if self.kv_cache_dtype == "fp8_ds_mla":
cache = self.kv_cache
if cache.dtype != torch.uint8:
+2
View File
@@ -339,6 +339,7 @@ class KimiRoutedOutputTransform(nn.Module):
residual: Optional tensor of the up-projection's output shape to
accumulate into. It is consumed in the GEMM's beta-add
epilogue, so adding it costs no extra kernel.
"""
if self.norm is not None:
hidden_states = self.norm(hidden_states)
@@ -796,6 +797,7 @@ class KimiMoE(nn.Module):
``router_output`` holds the grouped top-k weights and ``topk_ids``
the selected experts; otherwise ``router_output`` holds the raw gate
logits and ``topk_ids`` is ``None``.
"""
def _router(
@@ -59,7 +59,6 @@ def _mapping(
def validate_shape(*, tp_size: int, latent_dim: int, hidden_dim: int) -> None:
"""Validate constraints imposed by the fused collective mapping."""
if tp_size <= 0 or latent_dim <= 0 or hidden_dim <= 0:
raise ValueError("collective dimensions must be positive")
if hidden_dim % tp_size:
@@ -86,7 +85,6 @@ def _select_routed_schedule(
max_m: int,
) -> tuple[int, int]:
"""Match upstream MNNVL's one-cluster-per-token occupancy policy."""
_, cluster_ctas, threads, _ = _mapping(tp_size, latent_dim, hidden_dim)
sm_count = torch.cuda.get_device_properties(
torch.accelerator.current_device_index()
@@ -886,7 +884,6 @@ def compile_kernel(
top_k: int = 0,
) -> None:
"""Compile the rank/M specialization without retaining caller tensors."""
key = _compile_key(
rank,
tp_size,
@@ -79,7 +79,6 @@ def validate_configuration(
b_prime_stages: int,
) -> None:
"""Validate constraints imposed by this BF16 SM100 GEMM."""
if latent_dim <= 0 or shard_dim <= 0:
raise ValueError("GEMM K and N must be positive")
if latent_dim % 8 or shard_dim % 8:
@@ -52,7 +52,6 @@ def to_cute_dynamic_m(
mode is symbolic, so changing M within an Op's capacity reuses the same
compiled kernel.
"""
return to_cute(tensor, assumed_align).mark_compact_shape_dynamic(
mode=mode,
stride_order=tensor.dim_order(),
@@ -98,7 +97,6 @@ def load_global_u32x4(
side-effecting prevents loop-invariant motion and common-subexpression
elimination across polling iterations.
"""
address = pointer.toint(loc=loc, ip=ip)
opcode = "ld.volatile.global.v4.u32" if volatile else "ld.global.v4.u32"
out = llvm.inline_asm(
@@ -194,7 +192,6 @@ def finalize_top16_bf16(
ip=None,
):
"""Finalize one Kimi K3 top-16 vector with wide metadata loads."""
addresses = [
pointer.toint(loc=loc, ip=ip).ir_value(loc=loc, ip=ip)
for pointer in (gemm2_vector, route_indices, route_weights)
@@ -229,7 +226,6 @@ def store_global_u32x4(
ip=None,
) -> None:
"""Store four packed words to an ordinary or NVLS multicast global VA."""
words = [packed[i].ir_value(loc=loc, ip=ip) for i in range(4)]
opcode = "st.volatile.global.v4.u32" if volatile else "st.global.v4.u32"
llvm.inline_asm(
@@ -294,7 +290,6 @@ def store_global_u32(
@dsl_user_op
def stmc_bf16x8(address: Int64, packed, *, loc=None, ip=None) -> None:
"""Publish eight BF16 values through an NVLS multicast mapping."""
words = [packed[i].ir_value(loc=loc, ip=ip) for i in range(4)]
llvm.inline_asm(
None,
@@ -312,7 +307,6 @@ def stmc_bf16x8(address: Int64, packed, *, loc=None, ip=None) -> None:
@dsl_user_op
def store_lamport_sentinel_128(pointer: cute.Pointer, *, loc=None, ip=None) -> None:
"""Reset one Lamport fragment to four FP32 negative-zero bit patterns."""
address = pointer.toint(loc=loc, ip=ip)
value = Uint32(NEG_ZERO_F32_BITS).ir_value(loc=loc, ip=ip)
llvm.inline_asm(
@@ -333,7 +327,6 @@ def red_async_release_gpu_add_u32(
pointer: cute.Pointer, value: Uint32, *, loc=None, ip=None
) -> None:
"""The exact SM100 arrival primitive used by upstream LamportFlags."""
address = pointer.toint(loc=loc, ip=ip)
llvm.inline_asm(
None,
@@ -378,7 +371,6 @@ def map_shared_to_peer(
ip=None,
) -> Int32:
"""Map a local shared-memory slot to the same slot in a peer CTA."""
smem_address = smem_ptr.toint(loc=loc, ip=ip).ir_value()
return Int32(
llvm.inline_asm(
@@ -422,7 +414,6 @@ def store_shared_cluster_f32(
@dsl_user_op
def load_shared_f32x2(pointer: cute.Pointer, *, loc=None, ip=None):
"""Load two aligned FP32 DSM partials from local shared memory."""
address = Int32(pointer.toint(loc=loc, ip=ip))
out = llvm.inline_asm(
llvm.StructType.get_literal([T.f32()] * 2),
@@ -443,7 +434,6 @@ def load_shared_f32x2(pointer: cute.Pointer, *, loc=None, ip=None):
@dsl_user_op
def load_shared_f32x4(pointer: cute.Pointer, *, loc=None, ip=None):
"""Load four aligned FP32 DSM partials from local shared memory."""
address = Int32(pointer.toint(loc=loc, ip=ip))
out = llvm.inline_asm(
llvm.StructType.get_literal([T.f32()] * 4),
@@ -530,7 +520,6 @@ def sanitize_negative_zero_u32x2(packed):
@cute.jit
def sanitize_negative_zero(packed):
"""Turn real BF16 -0 into +0 so it cannot equal the empty sentinel."""
result = cute.make_rmem_tensor(cute.make_layout((4,)), Uint32)
for i in cutlass.range_constexpr(4):
word = packed[i]
@@ -547,7 +536,6 @@ def sanitize_negative_zero(packed):
@cute.jit
def fragment_is_dirty(packed):
"""Bit-exact upstream sentinel check: one comparison per 32-bit word."""
dirty = packed[0] == Uint32(NEG_ZERO_F32_BITS)
for i in cutlass.range_constexpr(1, 4):
dirty = dirty | (packed[i] == Uint32(NEG_ZERO_F32_BITS))
@@ -564,7 +552,6 @@ def warp_sum_specialized(
last_warp_mask: cutlass.Constexpr[int],
) -> Float32:
"""Warp sum supporting a compile-time partial final warp."""
if warp_idx == Int32(warps - 1) and cutlass.const_expr(last_warp_lanes < 32):
for offset in cutlass.range_constexpr(16, 0, -1):
# range_constexpr does not provide powers-of-two stepping.
+7 -10
View File
@@ -539,9 +539,7 @@ def fused_kda_gate_chunk_cumsum(
)
B, T, H, D = raw_g.shape
if raw_beta.shape != (B, T, H):
raise ValueError(
f"Expected raw_beta shape {(B, T, H)}, got {raw_beta.shape}"
)
raise ValueError(f"Expected raw_beta shape {(B, T, H)}, got {raw_beta.shape}")
if chunk_indices is None and cu_seqlens is not None:
chunk_indices = prepare_chunk_indices(cu_seqlens, chunk_size)
NT = cdiv(T, chunk_size) if cu_seqlens is None else len(chunk_indices)
@@ -901,13 +899,12 @@ def fused_kda_gate(
threshold: float = 20.0,
lower_bound: float | None = None,
) -> torch.Tensor:
"""
Forward pass for KDA gate:
input g: [..., H*D]
param A: [H] or [1, 1, H, 1]
beta: softplus beta parameter
threshold: softplus threshold parameter
return : [..., H, D]
"""Forward pass for KDA gate:
input g: [..., H*D]
param A: [H] or [1, 1, H, 1]
beta: softplus beta parameter
threshold: softplus threshold parameter
return : [..., H, D]
"""
orig_shape = g.shape[:-1]
+200 -98
View File
@@ -23,18 +23,20 @@ from .chunk_intra_token_parallel import chunk_kda_fwd_intra_token_parallel
################################################################################
@triton.heuristics({
'IS_VARLEN': lambda args: args['cu_seqlens'] is not None,
})
@triton.heuristics(
{
"IS_VARLEN": lambda args: args["cu_seqlens"] is not None,
}
)
@triton.autotune(
configs=[
triton.Config({'BK': BK}, num_warps=num_warps)
triton.Config({"BK": BK}, num_warps=num_warps)
for BK in [32, 64]
for num_warps in [1, 2, 4]
],
key=["H", "HV", "K", "BC"],
)
@triton.jit(do_not_specialize=['T'])
@triton.jit(do_not_specialize=["T"])
def chunk_kda_fwd_kernel_inter_solve_fused(
q,
k,
@@ -57,8 +59,7 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
USE_SAFE_GATE: tl.constexpr,
SOLVE_TRIL_DOT_PRECISION: tl.constexpr,
):
"""
Fused kernel: compute inter-subchunk Akk + solve_tril in one pass.
"""Fused kernel: compute inter-subchunk Akk + solve_tril in one pass.
Prerequisite: token_parallel has already computed diagonal Akk blocks in Akkd.
This kernel:
@@ -74,8 +75,14 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
i_h = i_hv // (HV // H)
if IS_VARLEN:
i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32)
bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(cu_seqlens + i_n + 1).to(tl.int32)
i_n, i_t = (
tl.load(chunk_indices + i_t * 2).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
)
bos, eos = (
tl.load(cu_seqlens + i_n).to(tl.int32),
tl.load(cu_seqlens + i_n + 1).to(tl.int32),
)
T = eos - bos
else:
bos, eos = i_b * T, i_b * T + T
@@ -122,21 +129,31 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
o_k = i_k * BK + tl.arange(0, BK)
m_k = o_k < K
p_k0 = tl.make_block_ptr(k, (T, K), (H*K, 1), (i_tc0, i_k * BK), (BC, BK), (1, 0))
p_g0 = tl.make_block_ptr(g, (T, K), (HV*K, 1), (i_tc0, i_k * BK), (BC, BK), (1, 0))
p_k0 = tl.make_block_ptr(
k, (T, K), (H * K, 1), (i_tc0, i_k * BK), (BC, BK), (1, 0)
)
p_g0 = tl.make_block_ptr(
g, (T, K), (HV * K, 1), (i_tc0, i_k * BK), (BC, BK), (1, 0)
)
b_k0 = tl.load(p_k0, boundary_check=(0, 1)).to(tl.float32)
b_g0 = tl.load(p_g0, boundary_check=(0, 1)).to(tl.float32)
if i_tc1 < T:
p_q1 = tl.make_block_ptr(q, (T, K), (H*K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0))
p_k1 = tl.make_block_ptr(k, (T, K), (H*K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0))
p_g1 = tl.make_block_ptr(g, (T, K), (HV*K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0))
p_q1 = tl.make_block_ptr(
q, (T, K), (H * K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0)
)
p_k1 = tl.make_block_ptr(
k, (T, K), (H * K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0)
)
p_g1 = tl.make_block_ptr(
g, (T, K), (HV * K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0)
)
# [BC, BK]
b_q1 = tl.load(p_q1, boundary_check=(0, 1)).to(tl.float32)
b_k1 = tl.load(p_k1, boundary_check=(0, 1)).to(tl.float32)
b_g1 = tl.load(p_g1, boundary_check=(0, 1)).to(tl.float32)
# [BK]
b_gn1 = tl.load(g + i_tc1 * HV*K + o_k, mask=m_k, other=0).to(tl.float32)
b_gn1 = tl.load(g + i_tc1 * HV * K + o_k, mask=m_k, other=0).to(tl.float32)
# [BC, BK]
b_gqn = tl.where(m_tc1[:, None], exp2(b_g1 - b_gn1[None, :]), 0)
# [BK, BC]
@@ -146,15 +163,23 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
b_Akk10 += tl.dot(b_k1 * b_gqn, b_kgt)
if i_tc2 < T:
p_q2 = tl.make_block_ptr(q, (T, K), (H*K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0))
p_k2 = tl.make_block_ptr(k, (T, K), (H*K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0))
p_g2 = tl.make_block_ptr(g, (T, K), (HV*K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0))
p_q2 = tl.make_block_ptr(
q, (T, K), (H * K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0)
)
p_k2 = tl.make_block_ptr(
k, (T, K), (H * K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0)
)
p_g2 = tl.make_block_ptr(
g, (T, K), (HV * K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0)
)
# [BC, BK]
b_q2 = tl.load(p_q2, boundary_check=(0, 1)).to(tl.float32)
b_k2 = tl.load(p_k2, boundary_check=(0, 1)).to(tl.float32)
b_g2 = tl.load(p_g2, boundary_check=(0, 1)).to(tl.float32)
# [BK]
b_gn2 = tl.load(g + i_tc2 * HV*K + o_k, mask=m_k, other=0).to(tl.float32)
b_gn2 = tl.load(g + i_tc2 * HV * K + o_k, mask=m_k, other=0).to(
tl.float32
)
# [BC, BK]
b_gqn2 = tl.where(m_tc2[:, None], exp2(b_g2 - b_gn2[None, :]), 0)
b_qg2 = b_q2 * b_gqn2
@@ -170,15 +195,23 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
b_Akk21 += tl.dot(b_kg2, b_kgt)
if i_tc3 < T:
p_q3 = tl.make_block_ptr(q, (T, K), (H*K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0))
p_k3 = tl.make_block_ptr(k, (T, K), (H*K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0))
p_g3 = tl.make_block_ptr(g, (T, K), (HV*K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0))
p_q3 = tl.make_block_ptr(
q, (T, K), (H * K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0)
)
p_k3 = tl.make_block_ptr(
k, (T, K), (H * K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0)
)
p_g3 = tl.make_block_ptr(
g, (T, K), (HV * K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0)
)
# [BC, BK]
b_q3 = tl.load(p_q3, boundary_check=(0, 1)).to(tl.float32)
b_k3 = tl.load(p_k3, boundary_check=(0, 1)).to(tl.float32)
b_g3 = tl.load(p_g3, boundary_check=(0, 1)).to(tl.float32)
# [BK]
b_gn3 = tl.load(g + i_tc3 * HV*K + o_k, mask=m_k, other=0).to(tl.float32)
b_gn3 = tl.load(g + i_tc3 * HV * K + o_k, mask=m_k, other=0).to(
tl.float32
)
# [BC, BK]
b_gqn3 = tl.where(m_tc3[:, None], exp2(b_g3 - b_gn3[None, :]), 0)
b_qg3 = b_q3 * b_gqn3
@@ -203,40 +236,78 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
# save off-diagonal Aqk blocks and prepare Akk
################################################################################
if i_tc1 < T:
p_Aqk10 = tl.make_block_ptr(Aqk, (T, BT), (HV*BT, 1), (i_tc1, 0), (BC, BC), (1, 0))
tl.store(p_Aqk10, (b_Aqk10 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1))
p_Aqk10 = tl.make_block_ptr(
Aqk, (T, BT), (HV * BT, 1), (i_tc1, 0), (BC, BC), (1, 0)
)
tl.store(
p_Aqk10, (b_Aqk10 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
)
p_b1 = tl.make_block_ptr(beta + bos * HV + i_hv, (T,), (HV,), (i_tc1,), (BC,), (0,))
p_b1 = tl.make_block_ptr(
beta + bos * HV + i_hv, (T,), (HV,), (i_tc1,), (BC,), (0,)
)
b_b1 = tl.load(p_b1, boundary_check=(0,)).to(tl.float32)
b_Akk10 = b_Akk10 * b_b1[:, None]
if i_tc2 < T:
p_Aqk20 = tl.make_block_ptr(Aqk, (T, BT), (HV*BT, 1), (i_tc2, 0), (BC, BC), (1, 0))
p_Aqk21 = tl.make_block_ptr(Aqk, (T, BT), (HV*BT, 1), (i_tc2, BC), (BC, BC), (1, 0))
tl.store(p_Aqk20, (b_Aqk20 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_Aqk21, (b_Aqk21 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1))
p_Aqk20 = tl.make_block_ptr(
Aqk, (T, BT), (HV * BT, 1), (i_tc2, 0), (BC, BC), (1, 0)
)
p_Aqk21 = tl.make_block_ptr(
Aqk, (T, BT), (HV * BT, 1), (i_tc2, BC), (BC, BC), (1, 0)
)
tl.store(
p_Aqk20, (b_Aqk20 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
)
tl.store(
p_Aqk21, (b_Aqk21 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
)
p_b2 = tl.make_block_ptr(beta + bos * HV + i_hv, (T,), (HV,), (i_tc2,), (BC,), (0,))
p_b2 = tl.make_block_ptr(
beta + bos * HV + i_hv, (T,), (HV,), (i_tc2,), (BC,), (0,)
)
b_b2 = tl.load(p_b2, boundary_check=(0,)).to(tl.float32)
b_Akk20 = b_Akk20 * b_b2[:, None]
b_Akk21 = b_Akk21 * b_b2[:, None]
if i_tc3 < T:
p_Aqk30 = tl.make_block_ptr(Aqk, (T, BT), (HV*BT, 1), (i_tc3, 0), (BC, BC), (1, 0))
p_Aqk31 = tl.make_block_ptr(Aqk, (T, BT), (HV*BT, 1), (i_tc3, BC), (BC, BC), (1, 0))
p_Aqk32 = tl.make_block_ptr(Aqk, (T, BT), (HV*BT, 1), (i_tc3, 2*BC), (BC, BC), (1, 0))
tl.store(p_Aqk30, (b_Aqk30 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_Aqk31, (b_Aqk31 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_Aqk32, (b_Aqk32 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1))
p_Aqk30 = tl.make_block_ptr(
Aqk, (T, BT), (HV * BT, 1), (i_tc3, 0), (BC, BC), (1, 0)
)
p_Aqk31 = tl.make_block_ptr(
Aqk, (T, BT), (HV * BT, 1), (i_tc3, BC), (BC, BC), (1, 0)
)
p_Aqk32 = tl.make_block_ptr(
Aqk, (T, BT), (HV * BT, 1), (i_tc3, 2 * BC), (BC, BC), (1, 0)
)
tl.store(
p_Aqk30, (b_Aqk30 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
)
tl.store(
p_Aqk31, (b_Aqk31 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
)
tl.store(
p_Aqk32, (b_Aqk32 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1)
)
p_b3 = tl.make_block_ptr(beta + bos * HV + i_hv, (T,), (HV,), (i_tc3,), (BC,), (0,))
p_b3 = tl.make_block_ptr(
beta + bos * HV + i_hv, (T,), (HV,), (i_tc3,), (BC,), (0,)
)
b_b3 = tl.load(p_b3, boundary_check=(0,)).to(tl.float32)
b_Akk30 = b_Akk30 * b_b3[:, None]
b_Akk31 = b_Akk31 * b_b3[:, None]
b_Akk32 = b_Akk32 * b_b3[:, None]
p_Akk00 = tl.make_block_ptr(Akkd, (T, BC), (HV*BC, 1), (i_tc0, 0), (BC, BC), (1, 0))
p_Akk11 = tl.make_block_ptr(Akkd, (T, BC), (HV*BC, 1), (i_tc1, 0), (BC, BC), (1, 0))
p_Akk22 = tl.make_block_ptr(Akkd, (T, BC), (HV*BC, 1), (i_tc2, 0), (BC, BC), (1, 0))
p_Akk33 = tl.make_block_ptr(Akkd, (T, BC), (HV*BC, 1), (i_tc3, 0), (BC, BC), (1, 0))
p_Akk00 = tl.make_block_ptr(
Akkd, (T, BC), (HV * BC, 1), (i_tc0, 0), (BC, BC), (1, 0)
)
p_Akk11 = tl.make_block_ptr(
Akkd, (T, BC), (HV * BC, 1), (i_tc1, 0), (BC, BC), (1, 0)
)
p_Akk22 = tl.make_block_ptr(
Akkd, (T, BC), (HV * BC, 1), (i_tc2, 0), (BC, BC), (1, 0)
)
p_Akk33 = tl.make_block_ptr(
Akkd, (T, BC), (HV * BC, 1), (i_tc3, 0), (BC, BC), (1, 0)
)
b_Ai00 = tl.load(p_Akk00, boundary_check=(0, 1)).to(tl.float32)
b_Ai11 = tl.load(p_Akk11, boundary_check=(0, 1)).to(tl.float32)
b_Ai22 = tl.load(p_Akk22, boundary_check=(0, 1)).to(tl.float32)
@@ -256,25 +327,25 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
b_Ai33 = -tl.where(m_A, b_Ai33, 0)
for i in range(2, min(BC, T - i_tc0)):
b_a00 = -tl.load(Akkd + (i_tc0 + i) * HV*BC + o_i)
b_a00 = tl.where(o_i < i, b_a00, 0.)
b_a00 = -tl.load(Akkd + (i_tc0 + i) * HV * BC + o_i)
b_a00 = tl.where(o_i < i, b_a00, 0.0)
b_a00 += tl.sum(b_a00[:, None] * b_Ai00, 0)
b_Ai00 = tl.where((o_i == i)[:, None], b_a00, b_Ai00)
for i in range(BC + 2, min(2*BC, T - i_tc0)):
b_a11 = -tl.load(Akkd + (i_tc0 + i) * HV*BC + o_i)
b_a11 = tl.where(o_i < i - BC, b_a11, 0.)
for i in range(BC + 2, min(2 * BC, T - i_tc0)):
b_a11 = -tl.load(Akkd + (i_tc0 + i) * HV * BC + o_i)
b_a11 = tl.where(o_i < i - BC, b_a11, 0.0)
b_a11 += tl.sum(b_a11[:, None] * b_Ai11, 0)
b_Ai11 = tl.where((o_i == i - BC)[:, None], b_a11, b_Ai11)
for i in range(2*BC + 2, min(3*BC, T - i_tc0)):
b_a22 = -tl.load(Akkd + (i_tc0 + i) * HV*BC + o_i)
b_a22 = tl.where(o_i < i - 2*BC, b_a22, 0.)
for i in range(2 * BC + 2, min(3 * BC, T - i_tc0)):
b_a22 = -tl.load(Akkd + (i_tc0 + i) * HV * BC + o_i)
b_a22 = tl.where(o_i < i - 2 * BC, b_a22, 0.0)
b_a22 += tl.sum(b_a22[:, None] * b_Ai22, 0)
b_Ai22 = tl.where((o_i == i - 2*BC)[:, None], b_a22, b_Ai22)
for i in range(3*BC + 2, min(4*BC, T - i_tc0)):
b_a33 = -tl.load(Akkd + (i_tc0 + i) * HV*BC + o_i)
b_a33 = tl.where(o_i < i - 3*BC, b_a33, 0.)
b_Ai22 = tl.where((o_i == i - 2 * BC)[:, None], b_a22, b_Ai22)
for i in range(3 * BC + 2, min(4 * BC, T - i_tc0)):
b_a33 = -tl.load(Akkd + (i_tc0 + i) * HV * BC + o_i)
b_a33 = tl.where(o_i < i - 3 * BC, b_a33, 0.0)
b_a33 += tl.sum(b_a33[:, None] * b_Ai33, 0)
b_Ai33 = tl.where((o_i == i - 3*BC)[:, None], b_a33, b_Ai33)
b_Ai33 = tl.where((o_i == i - 3 * BC)[:, None], b_a33, b_Ai33)
b_Ai00 += m_I
b_Ai11 += m_I
@@ -289,53 +360,73 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
b_Ai10 = -tl.dot(
tl.dot(b_Ai11, b_Akk10, input_precision=SOLVE_TRIL_DOT_PRECISION),
b_Ai00,
input_precision=SOLVE_TRIL_DOT_PRECISION
input_precision=SOLVE_TRIL_DOT_PRECISION,
)
b_Ai21 = -tl.dot(
tl.dot(b_Ai22, b_Akk21, input_precision=SOLVE_TRIL_DOT_PRECISION),
b_Ai11,
input_precision=SOLVE_TRIL_DOT_PRECISION
input_precision=SOLVE_TRIL_DOT_PRECISION,
)
b_Ai32 = -tl.dot(
tl.dot(b_Ai33, b_Akk32, input_precision=SOLVE_TRIL_DOT_PRECISION),
b_Ai22,
input_precision=SOLVE_TRIL_DOT_PRECISION
input_precision=SOLVE_TRIL_DOT_PRECISION,
)
b_Ai20 = -tl.dot(
b_Ai22,
tl.dot(b_Akk20, b_Ai00, input_precision=SOLVE_TRIL_DOT_PRECISION) +
tl.dot(b_Akk21, b_Ai10, input_precision=SOLVE_TRIL_DOT_PRECISION),
input_precision=SOLVE_TRIL_DOT_PRECISION
tl.dot(b_Akk20, b_Ai00, input_precision=SOLVE_TRIL_DOT_PRECISION)
+ tl.dot(b_Akk21, b_Ai10, input_precision=SOLVE_TRIL_DOT_PRECISION),
input_precision=SOLVE_TRIL_DOT_PRECISION,
)
b_Ai31 = -tl.dot(
b_Ai33,
tl.dot(b_Akk31, b_Ai11, input_precision=SOLVE_TRIL_DOT_PRECISION) +
tl.dot(b_Akk32, b_Ai21, input_precision=SOLVE_TRIL_DOT_PRECISION),
input_precision=SOLVE_TRIL_DOT_PRECISION
tl.dot(b_Akk31, b_Ai11, input_precision=SOLVE_TRIL_DOT_PRECISION)
+ tl.dot(b_Akk32, b_Ai21, input_precision=SOLVE_TRIL_DOT_PRECISION),
input_precision=SOLVE_TRIL_DOT_PRECISION,
)
b_Ai30 = -tl.dot(
b_Ai33,
tl.dot(b_Akk30, b_Ai00, input_precision=SOLVE_TRIL_DOT_PRECISION) +
tl.dot(b_Akk31, b_Ai10, input_precision=SOLVE_TRIL_DOT_PRECISION) +
tl.dot(b_Akk32, b_Ai20, input_precision=SOLVE_TRIL_DOT_PRECISION),
input_precision=SOLVE_TRIL_DOT_PRECISION
tl.dot(b_Akk30, b_Ai00, input_precision=SOLVE_TRIL_DOT_PRECISION)
+ tl.dot(b_Akk31, b_Ai10, input_precision=SOLVE_TRIL_DOT_PRECISION)
+ tl.dot(b_Akk32, b_Ai20, input_precision=SOLVE_TRIL_DOT_PRECISION),
input_precision=SOLVE_TRIL_DOT_PRECISION,
)
################################################################################
# store full Akk_inv to Akk
################################################################################
p_Akk00 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc0, 0), (BC, BC), (1, 0))
p_Akk10 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc1, 0), (BC, BC), (1, 0))
p_Akk11 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc1, BC), (BC, BC), (1, 0))
p_Akk20 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc2, 0), (BC, BC), (1, 0))
p_Akk21 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc2, BC), (BC, BC), (1, 0))
p_Akk22 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc2, 2*BC), (BC, BC), (1, 0))
p_Akk30 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc3, 0), (BC, BC), (1, 0))
p_Akk31 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc3, BC), (BC, BC), (1, 0))
p_Akk32 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc3, 2*BC), (BC, BC), (1, 0))
p_Akk33 = tl.make_block_ptr(Akk, (T, BT), (HV*BT, 1), (i_tc3, 3*BC), (BC, BC), (1, 0))
p_Akk00 = tl.make_block_ptr(
Akk, (T, BT), (HV * BT, 1), (i_tc0, 0), (BC, BC), (1, 0)
)
p_Akk10 = tl.make_block_ptr(
Akk, (T, BT), (HV * BT, 1), (i_tc1, 0), (BC, BC), (1, 0)
)
p_Akk11 = tl.make_block_ptr(
Akk, (T, BT), (HV * BT, 1), (i_tc1, BC), (BC, BC), (1, 0)
)
p_Akk20 = tl.make_block_ptr(
Akk, (T, BT), (HV * BT, 1), (i_tc2, 0), (BC, BC), (1, 0)
)
p_Akk21 = tl.make_block_ptr(
Akk, (T, BT), (HV * BT, 1), (i_tc2, BC), (BC, BC), (1, 0)
)
p_Akk22 = tl.make_block_ptr(
Akk, (T, BT), (HV * BT, 1), (i_tc2, 2 * BC), (BC, BC), (1, 0)
)
p_Akk30 = tl.make_block_ptr(
Akk, (T, BT), (HV * BT, 1), (i_tc3, 0), (BC, BC), (1, 0)
)
p_Akk31 = tl.make_block_ptr(
Akk, (T, BT), (HV * BT, 1), (i_tc3, BC), (BC, BC), (1, 0)
)
p_Akk32 = tl.make_block_ptr(
Akk, (T, BT), (HV * BT, 1), (i_tc3, 2 * BC), (BC, BC), (1, 0)
)
p_Akk33 = tl.make_block_ptr(
Akk, (T, BT), (HV * BT, 1), (i_tc3, 3 * BC), (BC, BC), (1, 0)
)
tl.store(p_Akk00, b_Ai00.to(Akk.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_Akk10, b_Ai10.to(Akk.dtype.element_ty), boundary_check=(0, 1))
@@ -349,18 +440,20 @@ def chunk_kda_fwd_kernel_inter_solve_fused(
tl.store(p_Akk33, b_Ai33.to(Akk.dtype.element_ty), boundary_check=(0, 1))
@triton.heuristics({
'IS_VARLEN': lambda args: args['cu_seqlens'] is not None,
})
@triton.heuristics(
{
"IS_VARLEN": lambda args: args["cu_seqlens"] is not None,
}
)
@triton.autotune(
configs=[
triton.Config({}, num_warps=num_warps, num_stages=num_stages)
for num_warps in [1, 2, 4, 8]
for num_stages in [2, 3, 4]
],
key=['BK', 'NC', 'BT', 'HV'],
key=["BK", "NC", "BT", "HV"],
)
@triton.jit(do_not_specialize=['B', 'T'])
@triton.jit(do_not_specialize=["B", "T"])
def chunk_kda_fwd_kernel_intra_sub_chunk(
q,
k,
@@ -386,8 +479,14 @@ def chunk_kda_fwd_kernel_intra_sub_chunk(
i_h = i_hv // (HV // H)
if IS_VARLEN:
i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32)
bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(cu_seqlens + i_n + 1).to(tl.int32)
i_n, i_t = (
tl.load(chunk_indices + i_t * 2).to(tl.int32),
tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
)
bos, eos = (
tl.load(cu_seqlens + i_n).to(tl.int32),
tl.load(cu_seqlens + i_n + 1).to(tl.int32),
)
T = eos - bos
else:
bos, eos = i_b * T, i_b * T + T
@@ -406,9 +505,9 @@ def chunk_kda_fwd_kernel_intra_sub_chunk(
Aqk = Aqk + (bos * HV + i_hv) * BT
Akk = Akk + (bos * HV + i_hv) * BC
p_q = tl.make_block_ptr(q, (T, K), (H*K, 1), (i_ti, 0), (BC, BK), (1, 0))
p_k = tl.make_block_ptr(k, (T, K), (H*K, 1), (i_ti, 0), (BC, BK), (1, 0))
p_g = tl.make_block_ptr(g, (T, K), (HV*K, 1), (i_ti, 0), (BC, BK), (1, 0))
p_q = tl.make_block_ptr(q, (T, K), (H * K, 1), (i_ti, 0), (BC, BK), (1, 0))
p_k = tl.make_block_ptr(k, (T, K), (H * K, 1), (i_ti, 0), (BC, BK), (1, 0))
p_g = tl.make_block_ptr(g, (T, K), (HV * K, 1), (i_ti, 0), (BC, BK), (1, 0))
p_beta = tl.make_block_ptr(beta, (T,), (HV,), (i_ti,), (BC,), (0,))
@@ -418,10 +517,12 @@ def chunk_kda_fwd_kernel_intra_sub_chunk(
b_beta = tl.load(p_beta, boundary_check=(0,)).to(tl.float32)
if USE_GATHER:
b_gn = gather(b_g, tl.full([1, BK], min(BC//2, T - i_ti - 1), dtype=tl.int16), axis=0)
b_gn = gather(
b_g, tl.full([1, BK], min(BC // 2, T - i_ti - 1), dtype=tl.int16), axis=0
)
else:
# caculate offset
p_gn = g + (i_ti + min(BC // 2, T - i_ti - 1)) * HV*K + tl.arange(0, BK)
p_gn = g + (i_ti + min(BC // 2, T - i_ti - 1)) * HV * K + tl.arange(0, BK)
b_gn = tl.load(p_gn, mask=tl.arange(0, BK) < K, other=0.0)
b_gn = b_gn[None, :]
@@ -429,8 +530,8 @@ def chunk_kda_fwd_kernel_intra_sub_chunk(
# less than 85 to avoid overflow in exp2
b_gm = (b_g - b_gn).to(tl.float32)
b_gq = tl.where(m_c[:, None], exp2(b_gm), 0.)
b_gk = tl.where(m_c[:, None], exp2(-b_gm), 0.)
b_gq = tl.where(m_c[:, None], exp2(b_gm), 0.0)
b_gk = tl.where(m_c[:, None], exp2(-b_gm), 0.0)
b_kgt = tl.trans(b_k * b_gk)
@@ -445,8 +546,10 @@ def chunk_kda_fwd_kernel_intra_sub_chunk(
b_Aqk = tl.where(m_Aqk, b_Aqk, 0.0)
b_Akk = tl.where(m_Akk, b_Akk, 0.0)
p_Aqk = tl.make_block_ptr(Aqk, (T, BT), (HV*BT, 1), (i_ti, i_i * BC), (BC, BC), (1, 0))
p_Akk = tl.make_block_ptr(Akk, (T, BC), (HV*BC, 1), (i_ti, 0), (BC, BC), (1, 0))
p_Aqk = tl.make_block_ptr(
Aqk, (T, BT), (HV * BT, 1), (i_ti, i_i * BC), (BC, BC), (1, 0)
)
p_Akk = tl.make_block_ptr(Akk, (T, BC), (HV * BC, 1), (i_ti, 0), (BC, BC), (1, 0))
tl.store(p_Aqk, b_Aqk.to(Aqk.dtype.element_ty), boundary_check=(0, 1))
tl.store(p_Akk, b_Akk.to(Akk.dtype.element_ty), boundary_check=(0, 1))
@@ -458,8 +561,8 @@ def chunk_kda_fwd_kernel_intra_sub_chunk(
b_Ai = -b_Akk
for i in range(2, min(BC, T - i_ti)):
b_a = -tl.load(Akk + (i_ti + i) * HV*BC + o_i)
b_a = tl.where(o_i < i, b_a, 0.)
b_a = -tl.load(Akk + (i_ti + i) * HV * BC + o_i)
b_a = tl.where(o_i < i, b_a, 0.0)
b_a += tl.sum(b_a[:, None] * b_Ai, 0)
b_Ai = tl.where((o_i == i)[:, None], b_a, b_Ai)
b_Ai += m_I
@@ -531,8 +634,7 @@ def chunk_kda_fwd_intra(
# Step 2: Fused inter + solve_tril (works for both fixed-len and varlen)
solve_tril_dot_precision = (
"tf32"
if current_platform.is_cuda()
and current_platform.has_device_capability(80)
if current_platform.is_cuda() and current_platform.has_device_capability(80)
else "ieee"
)
grid = (NT, B * HV)
@@ -16,18 +16,20 @@ from vllm.third_party.flash_linear_attention.ops.op import exp2
from vllm.triton_utils import tl, triton
@triton.heuristics({
'IS_VARLEN': lambda args: args['cu_seqlens'] is not None,
})
@triton.heuristics(
{
"IS_VARLEN": lambda args: args["cu_seqlens"] is not None,
}
)
@triton.autotune(
configs=[
triton.Config({'BH': BH}, num_warps=num_warps)
triton.Config({"BH": BH}, num_warps=num_warps)
for BH in [1, 2, 4, 8]
for num_warps in [1, 2, 4, 8]
],
key=["K", "H", "HV"],
)
@triton.jit(do_not_specialize=['T', 'N'])
@triton.jit(do_not_specialize=["T", "N"])
def chunk_kda_fwd_kernel_intra_token_parallel(
q,
k,
@@ -65,7 +67,10 @@ def chunk_kda_fwd_kernel_intra_token_parallel(
left = mid + 1
i_n = left
bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(cu_seqlens + i_n + 1).to(tl.int32)
bos, eos = (
tl.load(cu_seqlens + i_n).to(tl.int32),
tl.load(cu_seqlens + i_n + 1).to(tl.int32),
)
T = eos - bos
i_t = i_tg - bos
else:
@@ -82,11 +87,11 @@ def chunk_kda_fwd_kernel_intra_token_parallel(
G: tl.constexpr = HV // H
q += bos * H*K
k += bos * H*K
g += bos * HV*K
Aqk += bos * HV*BT
Akk += bos * HV*BC
q += bos * H * K
k += bos * H * K
g += bos * HV * K
Aqk += bos * HV * BT
Akk += bos * HV * BC
beta += bos * HV
BK: tl.constexpr = triton.next_power_of_2(K)
@@ -103,7 +108,9 @@ def chunk_kda_fwd_kernel_intra_token_parallel(
b_k = tl.load(k + i_t * H * K + p_qk, mask=m_hk, other=0).to(tl.float32)
# g: [B, T, HV, K], beta: [B, T, HV]
p_g = tl.make_block_ptr(g + i_t * HV * K, (HV, K), (K, 1), (i_hg * BH, 0), (BH, BK), (1, 0))
p_g = tl.make_block_ptr(
g + i_t * HV * K, (HV, K), (K, 1), (i_hg * BH, 0), (BH, BK), (1, 0)
)
p_beta = tl.make_block_ptr(beta + i_t * HV, (HV,), (1,), (i_hg * BH,), (BH,), (0,))
b_g = tl.load(p_g, boundary_check=(0, 1)).to(tl.float32)
b_beta = tl.load(p_beta, boundary_check=(0,)).to(tl.float32)
@@ -111,15 +118,25 @@ def chunk_kda_fwd_kernel_intra_token_parallel(
for j in range(i_ts, min(i_t + 1, min(T, i_ts + BC))):
b_kj = tl.load(k + j * H * K + p_qk, mask=m_hk, other=0).to(tl.float32)
p_gj = tl.make_block_ptr(g + j * HV * K, (HV, K), (K, 1), (i_hg * BH, 0), (BH, BK), (1, 0))
p_gj = tl.make_block_ptr(
g + j * HV * K, (HV, K), (K, 1), (i_hg * BH, 0), (BH, BK), (1, 0)
)
b_gj = tl.load(p_gj, boundary_check=(0, 1)).to(tl.float32)
b_kgj = tl.where(m_k[None, :], b_kj * exp2(b_g - b_gj), 0.0)
b_Aqk = tl.sum(b_q * b_kgj, axis=1) * scale
b_Akk = tl.sum(b_k * b_kgj, axis=1) * tl.where(j < i_t, 1.0, 0.0)
tl.store(Aqk + i_t * HV * BT + o_hv * BT + j % BT, b_Aqk.to(Aqk.dtype.element_ty), mask=m_hv)
tl.store(Akk + i_t * HV * BC + o_hv * BC + j - i_ts, b_Akk.to(Akk.dtype.element_ty), mask=m_hv)
tl.store(
Aqk + i_t * HV * BT + o_hv * BT + j % BT,
b_Aqk.to(Aqk.dtype.element_ty),
mask=m_hv,
)
tl.store(
Akk + i_t * HV * BC + o_hv * BC + j - i_ts,
b_Akk.to(Akk.dtype.element_ty),
mask=m_hv,
)
def chunk_kda_fwd_intra_token_parallel(
@@ -134,8 +151,7 @@ def chunk_kda_fwd_intra_token_parallel(
chunk_size: int = 64,
sub_chunk_size: int = 16,
) -> None:
"""
Token-parallel implementation: each token gets its own thread block.
"""Token-parallel implementation: each token gets its own thread block.
Supports both fixed-length and variable-length sequences.
Reduces wasted computation on padding.
@@ -147,17 +163,21 @@ def chunk_kda_fwd_intra_token_parallel(
gk: [B, T, HV, K] cumsum of gates (HV >= H for GVA)
beta: [B, T, HV]
Aqk: [B, T, HV, BT] output tensor to write to
cu_seqlens: cumulative sequence lengths for variable-length input
Akk: [B, T, HV, BC] output tensor for diagonal blocks (fp32)
scale: attention scale
chunk_size: BT (default 64)
sub_chunk_size: BC (default 16)
"""
B, T, H, K, HV = *q.shape, gk.shape[2]
N = len(cu_seqlens) - 1 if cu_seqlens is not None else B
BT = chunk_size
BC = sub_chunk_size
def grid(meta): return (B * T, triton.cdiv(HV, meta['BH']))
def grid(meta):
return (B * T, triton.cdiv(HV, meta["BH"]))
chunk_kda_fwd_kernel_intra_token_parallel[grid](
q=q,
k=k,
@@ -483,9 +483,7 @@ def fused_recurrent_kda(
)
@triton.jit(
do_not_specialize=["stride_beta_token", "stride_state_indices"]
)
@triton.jit(do_not_specialize=["stride_beta_token", "stride_state_indices"])
def fused_recurrent_kda_packed_decode_kernel(
mixed_qkv,
raw_g,