[Encoder Decoder] Add flash_attn kernel support for encoder-decoder models (#9559)
This commit is contained in:
@@ -16,13 +16,13 @@ from tests.kernels.utils import *
|
||||
from vllm.attention import (Attention, AttentionBackend, AttentionMetadata,
|
||||
AttentionType)
|
||||
from vllm.attention.backends.utils import STR_NOT_IMPL_ENC_DEC_ROCM_HIP
|
||||
from vllm.attention.selector import (_Backend,
|
||||
from vllm.attention.selector import (_Backend, get_attn_backend,
|
||||
global_force_attn_backend_context_manager)
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# List of support backends for encoder/decoder models
|
||||
LIST_ENC_DEC_SUPPORTED_BACKENDS = [_Backend.XFORMERS]
|
||||
|
||||
LIST_ENC_DEC_SUPPORTED_BACKENDS = [_Backend.XFORMERS, _Backend.FLASH_ATTN]
|
||||
HEAD_SIZES = [64, 256]
|
||||
|
||||
NUM_HEADS = [1, 16]
|
||||
@@ -145,7 +145,8 @@ def _make_test_resources(test_pt: TestPoint, ) -> TestResources:
|
||||
test_pt.num_heads,
|
||||
test_pt.head_size,
|
||||
test_pt.block_size,
|
||||
device=CUDA_DEVICE)
|
||||
device=CUDA_DEVICE,
|
||||
backend=test_pt.backend_name)
|
||||
return TestResources(scale, attn_backend, attn, kv_cache)
|
||||
|
||||
|
||||
@@ -592,6 +593,7 @@ def _run_encoder_attention_test(
|
||||
attn: Attention,
|
||||
encoder_test_params: PhaseTestParameters,
|
||||
attn_metadata: AttentionMetadata,
|
||||
test_pt: TestPoint,
|
||||
) -> torch.Tensor:
|
||||
'''
|
||||
Run encoder attention.
|
||||
@@ -610,6 +612,8 @@ def _run_encoder_attention_test(
|
||||
(number_of_tokens x num_heads x head_size)
|
||||
query/key/value fields
|
||||
* attn_metadata: attention metadata for encoder/decoder-self attention
|
||||
* test_pt: The TestPoint object containing test details like number of
|
||||
model heads, head size, name of the backend being used etc.
|
||||
|
||||
Returns:
|
||||
* Attention.forward() applied to packed {query,key,value} and
|
||||
@@ -619,20 +623,31 @@ def _run_encoder_attention_test(
|
||||
attn_type = AttentionType.ENCODER
|
||||
packed_qkv = encoder_test_params.packed_qkvo.packed_qkv
|
||||
assert packed_qkv is not None
|
||||
return attn.forward(packed_qkv.query,
|
||||
packed_qkv.key,
|
||||
packed_qkv.value,
|
||||
torch.tensor([],
|
||||
dtype=torch.float32,
|
||||
device=packed_qkv.query.device),
|
||||
attn_metadata,
|
||||
attn_type=attn_type)
|
||||
with set_forward_context(attn_metadata):
|
||||
# In the test setup the shape of the query is
|
||||
# [batch_size, seq_len, num_heads, head_size]. However
|
||||
# the attention backend expect the shape to be
|
||||
# [num_tokens, hidden_size]. Hence reshape the query before
|
||||
# invoking the forward method.
|
||||
# TODO - Update the way we construct the query so that it
|
||||
# is shaped as [num_tokens, hidden_size] and we can skip the reshape.
|
||||
reshaped_query = packed_qkv.query.view(
|
||||
-1, test_pt.num_heads * test_pt.head_size)
|
||||
return attn.forward(reshaped_query,
|
||||
packed_qkv.key,
|
||||
packed_qkv.value,
|
||||
torch.tensor([],
|
||||
dtype=torch.float32,
|
||||
device=packed_qkv.query.device),
|
||||
attn_metadata,
|
||||
attn_type=attn_type)
|
||||
|
||||
|
||||
def _run_decoder_self_attention_test(
|
||||
test_rsrcs: TestResources,
|
||||
decoder_test_params: PhaseTestParameters,
|
||||
attn_metadata: AttentionMetadata,
|
||||
test_pt: TestPoint,
|
||||
) -> torch.Tensor:
|
||||
'''
|
||||
Run decoder self-attention test.
|
||||
@@ -650,6 +665,8 @@ def _run_decoder_self_attention_test(
|
||||
query/key/value fields
|
||||
* attn_metadata: attention metadata for decoder-self attention
|
||||
(contains KV cache memory-mapping)
|
||||
* test_pt: The TestPoint object containing test details like number of
|
||||
model heads, head size, name of the backend being used etc.
|
||||
|
||||
Returns:
|
||||
* Attention.forward() applied to packed_{query,key,value}, kv_cache
|
||||
@@ -660,12 +677,22 @@ def _run_decoder_self_attention_test(
|
||||
kv_cache = test_rsrcs.kv_cache
|
||||
packed_qkv = decoder_test_params.packed_qkvo.packed_qkv
|
||||
assert packed_qkv is not None
|
||||
return attn.forward(packed_qkv.query,
|
||||
packed_qkv.key,
|
||||
packed_qkv.value,
|
||||
kv_cache,
|
||||
attn_metadata,
|
||||
attn_type=attn_type)
|
||||
with set_forward_context(attn_metadata):
|
||||
# In the test setup the shape of the query is
|
||||
# [batch_size, seq_len, num_heads, head_size]. However
|
||||
# the attention backend expect the shape to be
|
||||
# [num_tokens, hidden_size]. Hence reshape the query before
|
||||
# invoking the forward method.
|
||||
# TODO - Update the way we construct the query so that it
|
||||
# is shaped as [num_tokens, hidden_size] and we can skip the reshape.
|
||||
reshaped_query = packed_qkv.query.view(
|
||||
-1, test_pt.num_heads * test_pt.head_size)
|
||||
return attn.forward(reshaped_query,
|
||||
packed_qkv.key,
|
||||
packed_qkv.value,
|
||||
kv_cache,
|
||||
attn_metadata,
|
||||
attn_type=attn_type)
|
||||
|
||||
|
||||
def _run_encoder_decoder_cross_attention_test(
|
||||
@@ -673,6 +700,7 @@ def _run_encoder_decoder_cross_attention_test(
|
||||
decoder_test_params: PhaseTestParameters,
|
||||
cross_test_params: Optional[PhaseTestParameters],
|
||||
attn_metadata: AttentionMetadata,
|
||||
test_pt: TestPoint,
|
||||
) -> torch.Tensor:
|
||||
'''
|
||||
Run encoder/decoder cross-attention test.
|
||||
@@ -701,6 +729,8 @@ def _run_encoder_decoder_cross_attention_test(
|
||||
(number_of_tokens x num_heads x head_size)
|
||||
key/value fields
|
||||
* attn_metadata: attention metadata for encoder/decoder-self attention
|
||||
* test_pt: The TestPoint object containing test details like number of
|
||||
model heads, head size, name of the backend being used etc.
|
||||
|
||||
Returns:
|
||||
* Attention.forward() applied to packed_{query,key,value}, kv_cache
|
||||
@@ -718,12 +748,37 @@ def _run_encoder_decoder_cross_attention_test(
|
||||
cross_pckd_qkv = cross_test_params.packed_qkvo.packed_qkv
|
||||
key = (None if cross_pckd_qkv is None else cross_pckd_qkv.key)
|
||||
value = (None if cross_pckd_qkv is None else cross_pckd_qkv.value)
|
||||
return attn.forward(decoder_test_params.packed_qkvo.packed_qkv.query,
|
||||
key,
|
||||
value,
|
||||
kv_cache,
|
||||
attn_metadata,
|
||||
attn_type=attn_type)
|
||||
with set_forward_context(attn_metadata):
|
||||
# In the test setup the shape of the query is
|
||||
# [batch_size, seq_len, num_heads, head_size]. However
|
||||
# the attention backend expect the shape to be
|
||||
# [num_tokens, hidden_size]. Hence reshape the query before
|
||||
# invoking the forward method.
|
||||
# TODO - Update the way we construct the query so that it
|
||||
# is shaped as [num_tokens, hidden_size] and we can skip the reshape.
|
||||
reshaped_query = decoder_test_params.packed_qkvo.packed_qkv.query.view(
|
||||
-1, test_pt.num_heads * test_pt.head_size)
|
||||
return attn.forward(reshaped_query,
|
||||
key,
|
||||
value,
|
||||
kv_cache,
|
||||
attn_metadata,
|
||||
attn_type=attn_type)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def set_reset_environment(attn_backend):
|
||||
# Set the default torch datatype to bfloat16 to enable
|
||||
# testing of the Flash Attention backend. Also clear the
|
||||
# cached value of the backend.
|
||||
default_dtype = torch.get_default_dtype()
|
||||
if attn_backend.name == 'FLASH_ATTN':
|
||||
torch.set_default_dtype(torch.bfloat16)
|
||||
get_attn_backend.cache_clear()
|
||||
yield
|
||||
# Reset the torch datatype to what it was before the test
|
||||
# so as not to impact the remaining tests.
|
||||
torch.set_default_dtype(default_dtype)
|
||||
|
||||
|
||||
@pytest.mark.skipif(current_platform.is_rocm(),
|
||||
@@ -773,10 +828,8 @@ def test_encoder_only(
|
||||
* max_dec_seq_len: max length of decoder input sequences
|
||||
* max_enc_seq_len: max length of encoder input sequences
|
||||
'''
|
||||
|
||||
# Force Attention wrapper backend
|
||||
with global_force_attn_backend_context_manager(attn_backend):
|
||||
|
||||
# Note: KV cache size of 4096 is arbitrary & chosen intentionally
|
||||
# to be more than necessary, since exceeding the kv cache size
|
||||
# is not part of this test
|
||||
@@ -807,10 +860,14 @@ def test_encoder_only(
|
||||
# PREFILL: encoder attention
|
||||
|
||||
enc_pckd_act_out: torch.Tensor = (_run_encoder_attention_test(
|
||||
test_rsrcs.attn, enc_test_params, prephase_attn_metadata))
|
||||
test_rsrcs.attn,
|
||||
enc_test_params,
|
||||
prephase_attn_metadata,
|
||||
test_pt=test_pt))
|
||||
|
||||
# - Is encoder attention result correct?
|
||||
assert_actual_matches_ideal(enc_test_params, enc_pckd_act_out)
|
||||
assert_actual_matches_ideal(enc_test_params, enc_pckd_act_out,
|
||||
attn_backend.name)
|
||||
|
||||
|
||||
@pytest.mark.skipif(current_platform.is_rocm(),
|
||||
@@ -892,10 +949,8 @@ def test_e2e_enc_dec_attn(
|
||||
* max_dec_seq_len: max length of decoder input sequences
|
||||
* max_enc_seq_len: max length of encoder input sequences
|
||||
'''
|
||||
|
||||
# Force Attention wrapper backend
|
||||
with global_force_attn_backend_context_manager(attn_backend):
|
||||
|
||||
# Note: KV cache size of 4096 is arbitrary & chosen intentionally
|
||||
# to be more than necessary, since exceeding the kv cache size
|
||||
# is not part of this test
|
||||
@@ -955,29 +1010,39 @@ def test_e2e_enc_dec_attn(
|
||||
|
||||
enc_pckd_act_out = _run_encoder_attention_test(test_rsrcs.attn,
|
||||
enc_test_params,
|
||||
prephase_attn_metadata)
|
||||
prephase_attn_metadata,
|
||||
test_pt=test_pt)
|
||||
|
||||
# - Is encoder attention result correct?
|
||||
assert_actual_matches_ideal(enc_test_params, enc_pckd_act_out)
|
||||
assert_actual_matches_ideal(enc_test_params, enc_pckd_act_out,
|
||||
attn_backend.name)
|
||||
|
||||
# PREFILL: decoder self-attention test
|
||||
|
||||
prephase_dec_pckd_act_out = _run_decoder_self_attention_test(
|
||||
test_rsrcs, prephase_dec_test_params, prephase_attn_metadata)
|
||||
test_rsrcs,
|
||||
prephase_dec_test_params,
|
||||
prephase_attn_metadata,
|
||||
test_pt=test_pt)
|
||||
|
||||
# - Is prefill decoder self-attention correct?
|
||||
assert_actual_matches_ideal(prephase_dec_test_params,
|
||||
prephase_dec_pckd_act_out)
|
||||
prephase_dec_pckd_act_out,
|
||||
attn_backend.name)
|
||||
|
||||
# PREFILL: encoder/decoder cross-attention test
|
||||
|
||||
prephase_cross_pckd_act_out = _run_encoder_decoder_cross_attention_test(
|
||||
test_rsrcs, prephase_dec_test_params, prephase_cross_test_params,
|
||||
prephase_attn_metadata)
|
||||
test_rsrcs,
|
||||
prephase_dec_test_params,
|
||||
prephase_cross_test_params,
|
||||
prephase_attn_metadata,
|
||||
test_pt=test_pt)
|
||||
|
||||
# - Is prefill encoder/decoder cross-attention correct?
|
||||
assert_actual_matches_ideal(prephase_cross_test_params,
|
||||
prephase_cross_pckd_act_out)
|
||||
prephase_cross_pckd_act_out,
|
||||
attn_backend.name)
|
||||
|
||||
# DECODE: build decode-phase attention metadata
|
||||
|
||||
@@ -993,17 +1058,26 @@ def test_e2e_enc_dec_attn(
|
||||
# DECODE: decoder self-attention test
|
||||
|
||||
decphase_dec_pckd_act_out = _run_decoder_self_attention_test(
|
||||
test_rsrcs, decphase_dec_test_params, decphase_attn_metadata)
|
||||
test_rsrcs,
|
||||
decphase_dec_test_params,
|
||||
decphase_attn_metadata,
|
||||
test_pt=test_pt)
|
||||
|
||||
# - Is decode-phase decoder self-attention correct?
|
||||
assert_actual_matches_ideal(decphase_dec_test_params,
|
||||
decphase_dec_pckd_act_out)
|
||||
decphase_dec_pckd_act_out,
|
||||
attn_backend.name)
|
||||
|
||||
# DECODE: encoder/decoder cross-attention test
|
||||
|
||||
decphase_cross_pckd_act_out = _run_encoder_decoder_cross_attention_test(
|
||||
test_rsrcs, decphase_dec_test_params, None, decphase_attn_metadata)
|
||||
test_rsrcs,
|
||||
decphase_dec_test_params,
|
||||
None,
|
||||
decphase_attn_metadata,
|
||||
test_pt=test_pt)
|
||||
|
||||
# - Is decode-phase encoder/decoder cross-attention correct?
|
||||
assert_actual_matches_ideal(decphase_cross_test_params,
|
||||
decphase_cross_pckd_act_out)
|
||||
decphase_cross_pckd_act_out,
|
||||
attn_backend.name)
|
||||
|
||||
+74
-16
@@ -13,8 +13,8 @@ from torch._prims_common import TensorLikeType
|
||||
|
||||
from vllm.attention import AttentionBackend, AttentionMetadata, AttentionType
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.utils import (STR_BACKEND_ENV_VAR, STR_XFORMERS_ATTN_VAL,
|
||||
make_tensor_with_pad)
|
||||
from vllm.utils import (STR_BACKEND_ENV_VAR, STR_FLASH_ATTN_VAL,
|
||||
STR_XFORMERS_ATTN_VAL, make_tensor_with_pad)
|
||||
|
||||
# For now, disable "test_aot_dispatch_dynamic" since there are some
|
||||
# bugs related to this test in PyTorch 2.4.
|
||||
@@ -525,17 +525,22 @@ def make_backend(backend_name: str) -> AttentionBackend:
|
||||
if backend_name == STR_XFORMERS_ATTN_VAL:
|
||||
# NOTE: xFormers backend cannot be imported for CPU and AMD GPUs.
|
||||
from vllm.attention.backends.xformers import XFormersBackend
|
||||
|
||||
return XFormersBackend()
|
||||
elif backend_name == STR_FLASH_ATTN_VAL:
|
||||
from vllm.attention.backends.flash_attn import FlashAttentionBackend
|
||||
return FlashAttentionBackend()
|
||||
|
||||
raise AssertionError(
|
||||
f"Unrecognized backend_name {backend_name} for unit test")
|
||||
|
||||
|
||||
def _make_metadata_tensors(
|
||||
seq_lens: Optional[List[int]], context_lens: Optional[List[int]],
|
||||
encoder_seq_lens: Optional[List[int]], device: Union[torch.device, str]
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, Any, Any, Optional[List[int]],
|
||||
torch.Tensor, Optional[int]]:
|
||||
seq_lens: Optional[List[int]],
|
||||
context_lens: Optional[List[int]],
|
||||
encoder_seq_lens: Optional[List[int]],
|
||||
device: Union[torch.device, str],
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, Any, Any, Optional[torch.Tensor],
|
||||
torch.Tensor, torch.Tensor, Optional[int]]:
|
||||
'''
|
||||
Build scalar & tensor values required to build attention metadata structure.
|
||||
|
||||
@@ -553,6 +558,8 @@ def _make_metadata_tensors(
|
||||
* max_context_len: max(context_lens)
|
||||
* max_seq_len: max(seq_lens)
|
||||
* seq_start_loc: start idx of each sequence
|
||||
* encoder_seq_lens_tensor: encoder seq_lens list, as tensor
|
||||
* encoder_seq_start_loc: start idx of each encoder sequence
|
||||
* max_encoder_seq_len: encoder seq_lens list, as tensor
|
||||
'''
|
||||
seq_lens_tensor = maybe_make_int_tensor(seq_lens, device)
|
||||
@@ -566,8 +573,26 @@ def _make_metadata_tensors(
|
||||
|
||||
seq_start_loc = None
|
||||
|
||||
if seq_lens_tensor is not None:
|
||||
seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1,
|
||||
dtype=torch.int32,
|
||||
device=seq_lens_tensor.device)
|
||||
torch.cumsum(seq_lens_tensor,
|
||||
dim=0,
|
||||
dtype=seq_start_loc.dtype,
|
||||
out=seq_start_loc[1:])
|
||||
|
||||
encoder_seq_start_loc = torch.zeros(encoder_seq_lens_tensor.shape[0] + 1,
|
||||
dtype=torch.int32,
|
||||
device=encoder_seq_lens_tensor.device)
|
||||
torch.cumsum(encoder_seq_lens_tensor,
|
||||
dim=0,
|
||||
dtype=encoder_seq_start_loc.dtype,
|
||||
out=encoder_seq_start_loc[1:])
|
||||
|
||||
return (seq_lens_tensor, context_lens_tensor, max_context_len, max_seq_len,
|
||||
seq_start_loc, encoder_seq_lens_tensor, max_encoder_seq_len)
|
||||
seq_start_loc, encoder_seq_lens_tensor, encoder_seq_start_loc,
|
||||
max_encoder_seq_len)
|
||||
|
||||
|
||||
def make_kv_cache(num_blocks: int,
|
||||
@@ -575,6 +600,7 @@ def make_kv_cache(num_blocks: int,
|
||||
head_size: int,
|
||||
block_size: int,
|
||||
device: Union[torch.device, str],
|
||||
backend: str,
|
||||
default_val: float = 0.0) -> torch.Tensor:
|
||||
'''
|
||||
Create a fake KV cache.
|
||||
@@ -591,10 +617,20 @@ def make_kv_cache(num_blocks: int,
|
||||
Returns:
|
||||
|
||||
* kv_cache: 2 x num_blocks x (block_size * num_heads * head_size)
|
||||
* for backend 'XFORMERS'
|
||||
* kv_cache: 2 x num_blocks x block_size x num_heads x head_size
|
||||
* for backend 'FLASH_ATTN'
|
||||
'''
|
||||
|
||||
kv_cache = torch.rand(
|
||||
(2, num_blocks, block_size * num_heads * head_size)).to(device)
|
||||
if backend == 'XFORMERS':
|
||||
kv_cache = torch.rand(
|
||||
(2, num_blocks, block_size * num_heads * head_size)).to(device)
|
||||
elif backend == 'FLASH_ATTN':
|
||||
kv_cache = torch.rand(
|
||||
(2, num_blocks, block_size, num_heads, head_size)).to(device)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unknown backend value: '{backend}'. Expected 'XFORMERS' or "
|
||||
f"'FLASH_ATTN'.")
|
||||
if default_val is not None:
|
||||
kv_cache[:, :, :] = default_val
|
||||
return kv_cache
|
||||
@@ -858,8 +894,9 @@ def make_test_metadata(
|
||||
context_lens_tensor,
|
||||
_,
|
||||
_,
|
||||
_,
|
||||
seq_start_loc,
|
||||
encoder_seq_lens_tensor,
|
||||
encoder_seq_start_loc,
|
||||
max_encoder_seq_len,
|
||||
) = _make_metadata_tensors(seq_lens,
|
||||
context_lens,
|
||||
@@ -874,6 +911,7 @@ def make_test_metadata(
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_tensor=seq_lens_tensor,
|
||||
seq_start_loc=seq_start_loc,
|
||||
max_prefill_seq_len=None if seq_lens is None else max(seq_lens),
|
||||
max_decode_seq_len=0,
|
||||
context_lens_tensor=context_lens_tensor,
|
||||
@@ -882,6 +920,7 @@ def make_test_metadata(
|
||||
num_encoder_tokens=num_encoder_tokens,
|
||||
encoder_seq_lens=encoder_seq_lens,
|
||||
encoder_seq_lens_tensor=encoder_seq_lens_tensor,
|
||||
encoder_seq_start_loc=encoder_seq_start_loc,
|
||||
max_encoder_seq_len=max_encoder_seq_len,
|
||||
cross_slot_mapping=(None if cross_kv_mmap is None else
|
||||
cross_kv_mmap.slot_mapping),
|
||||
@@ -904,8 +943,9 @@ def make_test_metadata(
|
||||
context_lens_tensor,
|
||||
_,
|
||||
_,
|
||||
_,
|
||||
seq_start_loc,
|
||||
encoder_seq_lens_tensor,
|
||||
encoder_seq_start_loc,
|
||||
max_encoder_seq_len,
|
||||
) = _make_metadata_tensors(seq_lens,
|
||||
context_lens,
|
||||
@@ -920,14 +960,17 @@ def make_test_metadata(
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_tensor=seq_lens_tensor,
|
||||
seq_start_loc=seq_start_loc,
|
||||
max_prefill_seq_len=0,
|
||||
max_decode_seq_len=max(seq_lens),
|
||||
max_decode_query_len=1,
|
||||
context_lens_tensor=context_lens_tensor,
|
||||
block_tables=kv_mmap.block_tables,
|
||||
use_cuda_graph=False,
|
||||
num_encoder_tokens=num_encoder_tokens,
|
||||
encoder_seq_lens=encoder_seq_lens,
|
||||
encoder_seq_lens_tensor=encoder_seq_lens_tensor,
|
||||
encoder_seq_start_loc=encoder_seq_start_loc,
|
||||
max_encoder_seq_len=max_encoder_seq_len,
|
||||
cross_slot_mapping=(None if cross_kv_mmap is None else
|
||||
cross_kv_mmap.slot_mapping),
|
||||
@@ -936,7 +979,8 @@ def make_test_metadata(
|
||||
|
||||
|
||||
def assert_actual_matches_ideal(test_params: PhaseTestParameters,
|
||||
output_under_test: torch.Tensor) -> None:
|
||||
output_under_test: torch.Tensor,
|
||||
backend: str) -> None:
|
||||
'''
|
||||
Assert that observed output matches the ideal output
|
||||
contained in the test parameters data structure.
|
||||
@@ -947,8 +991,22 @@ def assert_actual_matches_ideal(test_params: PhaseTestParameters,
|
||||
* output_under_test: actually observed output value
|
||||
'''
|
||||
ideal_output = test_params.packed_qkvo.ideal_output
|
||||
torch.testing.assert_close(ideal_output,
|
||||
output_under_test.view_as(ideal_output))
|
||||
if backend == 'XFORMERS':
|
||||
torch.testing.assert_close(ideal_output,
|
||||
output_under_test.view_as(ideal_output))
|
||||
|
||||
elif backend == 'FLASH_ATTN':
|
||||
# For FlashAttention override the accuracy thresholds to non default
|
||||
# values since we notice a higher difference between the ideal and
|
||||
# actual output.
|
||||
torch.testing.assert_close(ideal_output,
|
||||
output_under_test.view_as(ideal_output),
|
||||
atol=0.01,
|
||||
rtol=0.016)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unknown backend value: '{backend}'. Expected 'XFORMERS' or "
|
||||
f"'FLASH_ATTN'.")
|
||||
|
||||
|
||||
# Copied/modified from torch._refs.__init__.py
|
||||
|
||||
Reference in New Issue
Block a user