[ROCm] attn res + sigmoid_mul + conv fusions (#52968)
Signed-off-by: Omar Muhammad <omar.muhammad@amd.com> Signed-off-by: Nicholas Susanto <nicholas.susanto@amd.com> Co-authored-by: Nicholas Susanto <nicholas.susanto@amd.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Nicholas Susanto
Douglas Lehr
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
783380af39
commit
f019e3e105
@@ -134,6 +134,14 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
self.conv_size = kda_config["short_conv_kernel_size"]
|
||||
self.use_full_rank_gate = True
|
||||
|
||||
self._fused_qkv_conv_fn = None
|
||||
if rocm_aiter_ops.is_enabled():
|
||||
from aiter.ops.triton.gated_delta_net import (
|
||||
causal_conv1d_split_qkv_triton_tile_fn,
|
||||
)
|
||||
|
||||
self._fused_qkv_conv_fn = causal_conv1d_split_qkv_triton_tile_fn
|
||||
|
||||
# Keep f_a before the narrow beta shard, then pad each TP-local row to
|
||||
# select the aligned BF16 GEMM path. The padding also avoids an Inductor
|
||||
# correctness issue seen with the row-strided G view.
|
||||
@@ -580,35 +588,48 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
if mixed_qkv_ns is not None:
|
||||
assert g1_ns is not None and beta_ns is not None
|
||||
if m.num_prefills > 0:
|
||||
q_ns, k_ns, v_ns = mixed_qkv_ns.split(
|
||||
self.local_projection_size, dim=-1
|
||||
)
|
||||
|
||||
# Packed prefill conv would require copying V solely to make
|
||||
# it dense for KDA. Separate calls accept the strided inputs
|
||||
# and produce dense Q/K/V without that extra traffic.
|
||||
# TODO: Use packed conv once every KDA prefill backend accepts
|
||||
# row-strided Q/K/V directly.
|
||||
def _prefill_conv(
|
||||
x: torch.Tensor,
|
||||
state: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return causal_conv1d_fn(
|
||||
x.transpose(0, 1),
|
||||
weight,
|
||||
if self._fused_qkv_conv_fn is not None:
|
||||
# One launch (fused qkv) instead of 3
|
||||
q_ns, k_ns, v_ns = self._fused_qkv_conv_fn(
|
||||
mixed_qkv_ns.transpose(0, 1),
|
||||
conv_weights,
|
||||
None,
|
||||
activation="silu",
|
||||
conv_states=state,
|
||||
has_initial_state=has_initial_state,
|
||||
conv_state,
|
||||
non_spec_query_start_loc,
|
||||
self.local_projection_size,
|
||||
self.local_projection_size,
|
||||
cache_indices=non_spec_state_indices_tensor,
|
||||
query_start_loc=non_spec_query_start_loc,
|
||||
has_initial_state=has_initial_state,
|
||||
activation="silu",
|
||||
block_m=8,
|
||||
metadata=m,
|
||||
).transpose(0, 1)
|
||||
preserve_input_dtype=True,
|
||||
)
|
||||
else:
|
||||
q_ns, k_ns, v_ns = mixed_qkv_ns.split(
|
||||
self.local_projection_size, dim=-1
|
||||
)
|
||||
|
||||
q_ns = _prefill_conv(q_ns, q_conv_state, q_conv_weight)
|
||||
k_ns = _prefill_conv(k_ns, k_conv_state, k_conv_weight)
|
||||
v_ns = _prefill_conv(v_ns, v_conv_state, v_conv_weight)
|
||||
def _prefill_conv(
|
||||
x: torch.Tensor,
|
||||
state: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return causal_conv1d_fn(
|
||||
x.transpose(0, 1),
|
||||
weight,
|
||||
None,
|
||||
activation="silu",
|
||||
conv_states=state,
|
||||
has_initial_state=has_initial_state,
|
||||
cache_indices=non_spec_state_indices_tensor,
|
||||
query_start_loc=non_spec_query_start_loc,
|
||||
metadata=m,
|
||||
).transpose(0, 1)
|
||||
|
||||
q_ns = _prefill_conv(q_ns, q_conv_state, q_conv_weight)
|
||||
k_ns = _prefill_conv(k_ns, k_conv_state, k_conv_weight)
|
||||
v_ns = _prefill_conv(v_ns, v_conv_state, v_conv_weight)
|
||||
q_ns, k_ns, v_ns = (
|
||||
rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim)
|
||||
for x in (q_ns, k_ns, v_ns)
|
||||
@@ -752,15 +773,9 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
|
||||
# ---------- merge spec and non-spec outputs ----------
|
||||
if core_attn_out_spec is not None and core_attn_out_non_spec is not None:
|
||||
# Mixed batches require indexed placement in the original order.
|
||||
merged = torch.empty(
|
||||
(1, num_actual_tokens, *core_attn_out_spec.shape[2:]),
|
||||
dtype=core_attn_out_spec.dtype,
|
||||
device=core_attn_out_spec.device,
|
||||
)
|
||||
merged = core_attn_out[:, :num_actual_tokens]
|
||||
merged.index_copy_(1, spec_token_indx, core_attn_out_spec)
|
||||
merged.index_copy_(1, non_spec_token_indx, core_attn_out_non_spec)
|
||||
core_attn_out[0, :num_actual_tokens] = merged[0, :num_actual_tokens]
|
||||
elif core_attn_out_non_spec is not None:
|
||||
if core_attn_out_non_spec.data_ptr() != core_attn_out.data_ptr():
|
||||
core_attn_out[0, :num_actual_tokens] = core_attn_out_non_spec[
|
||||
|
||||
@@ -23,8 +23,8 @@ _OPT_MIN_SIZE = 2048
|
||||
|
||||
|
||||
class KimiK3MultiHeadLatentAttentionWrapper(MultiHeadLatentAttentionWrapper):
|
||||
"""Kimi-K3 MLA wrapper with eager AITER q/kv RMSNorm fusion and a fused
|
||||
decode Q-prep path."""
|
||||
"""Kimi-K3 MLA wrapper with eager AITER q/kv RMSNorm and output gate
|
||||
fusions and a fused decode Q-prep path."""
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
@@ -303,6 +303,12 @@ class KimiK3MultiHeadLatentAttentionWrapper(MultiHeadLatentAttentionWrapper):
|
||||
)
|
||||
|
||||
if self.g_proj is not None:
|
||||
attn_out = attn_out * self.g_proj(hidden_states)[0].sigmoid()
|
||||
gate = self.g_proj(hidden_states)[0]
|
||||
if rocm_aiter_ops.is_enabled():
|
||||
from aiter.ops.triton.fusions.fused_sigmoid_mul import fused_sigmoid_mul
|
||||
|
||||
attn_out = fused_sigmoid_mul(attn_out, gate)
|
||||
else:
|
||||
attn_out = attn_out * gate.sigmoid()
|
||||
|
||||
return self.o_proj(attn_out)[0]
|
||||
|
||||
Reference in New Issue
Block a user