uncomment flex backend for batch invariant mode (#40842)
Signed-off-by: Angel Li <liangel@meta.com>
This commit is contained in:
@@ -26,6 +26,7 @@ TEST_MODEL = os.getenv("VLLM_TEST_MODEL", DEFAULT_MODEL)
|
||||
BACKENDS: list[str] = [
|
||||
"FLASH_ATTN",
|
||||
"TRITON_ATTN",
|
||||
"FLEX_ATTENTION",
|
||||
]
|
||||
|
||||
# FlashInfer temporarily disabled due to invariant CTA sizes.
|
||||
|
||||
@@ -99,6 +99,10 @@ class FlexAttentionBackend(AttentionBackend):
|
||||
"""FlexAttention supports both decoder and encoder-only attention."""
|
||||
return attn_type in (AttentionType.DECODER, AttentionType.ENCODER_ONLY)
|
||||
|
||||
@classmethod
|
||||
def supports_batch_invariance(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def supports_mm_prefix(cls) -> bool:
|
||||
"""FlexAttention supports full attention for image tokens."""
|
||||
@@ -326,15 +330,9 @@ class BlockSparsityHint(NamedTuple):
|
||||
|
||||
|
||||
def copy_to_persistent(dst, src):
|
||||
try:
|
||||
dst = dst.as_strided(src.shape, src.stride())
|
||||
except RuntimeError as e:
|
||||
raise RuntimeError(
|
||||
f"Fail to re-stride a persistent tensor of shape {dst.shape} "
|
||||
f"for a tensor of shape {src.shape}"
|
||||
) from e
|
||||
dst.copy_(src)
|
||||
return dst
|
||||
sliced = dst[tuple(slice(0, s) for s in src.shape)]
|
||||
sliced.copy_(src)
|
||||
return sliced
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
Reference in New Issue
Block a user