[Perf] fuse kernels in gdn (#37813)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
@@ -24,6 +24,7 @@ from vllm.model_executor.layers.fla.ops import (
|
||||
chunk_gated_delta_rule as fla_chunk_gated_delta_rule,
|
||||
)
|
||||
from vllm.model_executor.layers.fla.ops import (
|
||||
fused_post_conv_prep,
|
||||
fused_recurrent_gated_delta_rule_packed_decode,
|
||||
fused_sigmoid_gating_delta_rule_update,
|
||||
)
|
||||
@@ -774,19 +775,44 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase):
|
||||
mixed_qkv_non_spec = None
|
||||
|
||||
query_spec, key_spec, value_spec = self.rearrange_mixed_qkv(mixed_qkv_spec)
|
||||
query_non_spec, key_non_spec, value_non_spec = self.rearrange_mixed_qkv(
|
||||
mixed_qkv_non_spec
|
||||
)
|
||||
|
||||
if attn_metadata.num_prefills > 0:
|
||||
g, beta = fused_gdn_gating(self.A_log, a, b, self.dt_bias)
|
||||
assert mixed_qkv_non_spec is not None, (
|
||||
"mixed_qkv_non_spec must be provided for prefill path"
|
||||
)
|
||||
if spec_sequence_masks is not None:
|
||||
g_non_spec = g.index_select(1, non_spec_token_indx)
|
||||
beta_non_spec = beta.index_select(1, non_spec_token_indx)
|
||||
a_non_spec = a.index_select(0, non_spec_token_indx)
|
||||
b_non_spec = b.index_select(0, non_spec_token_indx)
|
||||
else:
|
||||
g_non_spec = g
|
||||
beta_non_spec = beta
|
||||
a_non_spec = a
|
||||
b_non_spec = b
|
||||
|
||||
(
|
||||
query_non_spec,
|
||||
key_non_spec,
|
||||
value_non_spec,
|
||||
g_non_spec,
|
||||
beta_non_spec,
|
||||
) = fused_post_conv_prep(
|
||||
conv_output=mixed_qkv_non_spec,
|
||||
a=a_non_spec,
|
||||
b=b_non_spec,
|
||||
A_log=self.A_log,
|
||||
dt_bias=self.dt_bias,
|
||||
num_k_heads=self.num_k_heads // self.tp_size,
|
||||
head_k_dim=self.head_k_dim,
|
||||
head_v_dim=self.head_v_dim,
|
||||
apply_l2norm=True,
|
||||
output_g_exp=False,
|
||||
)
|
||||
query_non_spec = query_non_spec.unsqueeze(0)
|
||||
key_non_spec = key_non_spec.unsqueeze(0)
|
||||
value_non_spec = value_non_spec.unsqueeze(0)
|
||||
g_non_spec = g_non_spec.unsqueeze(0)
|
||||
beta_non_spec = beta_non_spec.unsqueeze(0)
|
||||
else:
|
||||
query_non_spec, key_non_spec, value_non_spec = self.rearrange_mixed_qkv(
|
||||
mixed_qkv_non_spec
|
||||
)
|
||||
g_non_spec = None
|
||||
beta_non_spec = None
|
||||
|
||||
@@ -832,7 +858,7 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase):
|
||||
initial_state=initial_state,
|
||||
output_final_state=True,
|
||||
cu_seqlens=non_spec_query_start_loc,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
)
|
||||
# Init cache
|
||||
ssm_state[non_spec_state_indices_tensor] = last_recurrent_state.to(
|
||||
|
||||
Reference in New Issue
Block a user