[MM][OOT] Support CPU seq_lens for OOT MMEncoderAttention kernels (#36605)
Signed-off-by: shen-shanshan <467638484@qq.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
co-authored by
Isotr0py
parent
57431d8231
commit
f0d3658c0f
@@ -297,11 +297,10 @@ def test_mha_attn_varlen_forward_flashinfer(
|
||||
hidden_size = num_heads * head_size
|
||||
tp_size = 1
|
||||
|
||||
sequence_lengths_np = MMEncoderAttention.maybe_compute_sequence_lengths(
|
||||
AttentionBackendEnum.FLASHINFER, cu_seqlens_np
|
||||
)
|
||||
sequence_lengths = torch.from_numpy(sequence_lengths_np).to(
|
||||
device, dtype=torch.int32, non_blocking=True
|
||||
sequence_lengths = MMEncoderAttention.maybe_compute_seq_lens(
|
||||
AttentionBackendEnum.FLASHINFER,
|
||||
cu_seqlens_np,
|
||||
device,
|
||||
)
|
||||
|
||||
max_seqlen_val = MMEncoderAttention.compute_max_seqlen(
|
||||
@@ -309,14 +308,12 @@ def test_mha_attn_varlen_forward_flashinfer(
|
||||
)
|
||||
max_seqlen = torch.tensor(max_seqlen_val, device=device, dtype=torch.int32)
|
||||
|
||||
cu_seqlens_np = MMEncoderAttention.maybe_recompute_cu_seqlens(
|
||||
cu_seqlens = MMEncoderAttention.maybe_recompute_cu_seqlens(
|
||||
AttentionBackendEnum.FLASHINFER,
|
||||
cu_seqlens_np,
|
||||
hidden_size,
|
||||
tp_size,
|
||||
)
|
||||
cu_seqlens = torch.from_numpy(cu_seqlens_np).to(
|
||||
device, dtype=torch.int32, non_blocking=True
|
||||
device,
|
||||
)
|
||||
|
||||
scale = 1.0 / head_size**0.5
|
||||
|
||||
Reference in New Issue
Block a user