[FlexAttention] Avoid encoder block-mask compile explosion (#50339)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -20,7 +20,9 @@ from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _generate_random_text(word_count: int) -> str:
|
||||
"""Generate random text with approximately the specified word count."""
|
||||
"""Generate deterministic text with approximately the specified word count."""
|
||||
rng = random.Random(word_count)
|
||||
|
||||
# Common English words with focus on verbs and nouns for realistic text
|
||||
common_words = [
|
||||
# Essential articles and pronouns (minimal)
|
||||
@@ -178,7 +180,7 @@ def _generate_random_text(word_count: int) -> str:
|
||||
|
||||
words = []
|
||||
for _ in range(word_count):
|
||||
words.append(random.choice(common_words))
|
||||
words.append(rng.choice(common_words))
|
||||
|
||||
# Add some punctuation for more realistic text
|
||||
text = " ".join(words)
|
||||
@@ -187,7 +189,7 @@ def _generate_random_text(word_count: int) -> str:
|
||||
result = []
|
||||
for i, word in enumerate(words_list):
|
||||
result.append(word)
|
||||
if (i + 1) % random.randint(10, 20) == 0 and i < len(words_list) - 1:
|
||||
if (i + 1) % rng.randint(10, 20) == 0 and i < len(words_list) - 1:
|
||||
result[-1] += "."
|
||||
|
||||
return " ".join(result)
|
||||
|
||||
@@ -13,6 +13,7 @@ from tests.v1.attention.utils import (
|
||||
create_standard_kv_cache_spec,
|
||||
create_vllm_config,
|
||||
)
|
||||
from vllm.config import AttentionConfig
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.v1.attention.backends.flex_attention import (
|
||||
BlockSparsityHint,
|
||||
@@ -27,6 +28,42 @@ MINIMUM_TORCH_VERSION = version.parse("2.7.0")
|
||||
DIRECT_BUILD_VERSION = version.parse("2.9.dev0")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("supports_small_blocks", "uses_paged_kv", "expected"),
|
||||
[
|
||||
(True, True, (16, 16)),
|
||||
(True, False, (128, 128)),
|
||||
(False, True, (128, 128)),
|
||||
(False, False, (128, 128)),
|
||||
],
|
||||
)
|
||||
def test_flex_attention_default_block_sizes(
|
||||
supports_small_blocks: bool,
|
||||
uses_paged_kv: bool,
|
||||
expected: tuple[int, int],
|
||||
):
|
||||
block_sizes = FlexAttentionMetadataBuilder._get_block_sizes(
|
||||
AttentionConfig(),
|
||||
supports_small_blocks=supports_small_blocks,
|
||||
cache_block_size=16,
|
||||
uses_paged_kv=uses_paged_kv,
|
||||
)
|
||||
assert block_sizes == expected
|
||||
|
||||
|
||||
def test_flex_attention_explicit_block_sizes_override_encoder_defaults():
|
||||
block_sizes = FlexAttentionMetadataBuilder._get_block_sizes(
|
||||
AttentionConfig(
|
||||
flex_attn_q_block_size=64,
|
||||
flex_attn_kv_block_size=32,
|
||||
),
|
||||
supports_small_blocks=True,
|
||||
cache_block_size=16,
|
||||
uses_paged_kv=False,
|
||||
)
|
||||
assert block_sizes == (64, 32)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available() or TORCH_VERSION < MINIMUM_TORCH_VERSION,
|
||||
reason="CUDA not available or PyTorch version < 2.7",
|
||||
@@ -213,10 +250,12 @@ def test_encoder_flex_attention_vs_default_backend(vllm_runner):
|
||||
the default backend for encoder models.
|
||||
"""
|
||||
model_name = "BAAI/bge-base-en-v1.5"
|
||||
# Exercise packed sequence boundaries inside 128-token FlexAttention
|
||||
# blocks, including sequences that span more than one block.
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"hello " * 120,
|
||||
"world " * 130,
|
||||
"attention " * 254,
|
||||
]
|
||||
|
||||
# Run with flex attention
|
||||
@@ -225,7 +264,7 @@ def test_encoder_flex_attention_vs_default_backend(vllm_runner):
|
||||
runner="pooling",
|
||||
dtype=torch.bfloat16,
|
||||
tensor_parallel_size=1,
|
||||
max_model_len=100,
|
||||
max_model_len=384,
|
||||
enforce_eager=True,
|
||||
attention_config={"backend": "FLEX_ATTENTION"},
|
||||
) as llm_flex:
|
||||
@@ -237,7 +276,7 @@ def test_encoder_flex_attention_vs_default_backend(vllm_runner):
|
||||
runner="pooling",
|
||||
dtype=torch.bfloat16,
|
||||
tensor_parallel_size=1,
|
||||
max_model_len=100,
|
||||
max_model_len=384,
|
||||
enforce_eager=True,
|
||||
) as llm_default:
|
||||
default_outputs = llm_default.embed(prompts)
|
||||
|
||||
@@ -89,13 +89,15 @@ class AttentionConfig:
|
||||
flex_attn_q_block_size: int | None = None
|
||||
"""Logical Q block size for the flex attention block mask.
|
||||
Must be a power of 2 and divisible by flex_attn_block_m.
|
||||
If None, uses the default (16 on PyTorch >= 2.9, 128 otherwise)."""
|
||||
If None, uses 16 for paged KV attention on PyTorch >= 2.9, and 128
|
||||
for encoder-only attention or older PyTorch versions."""
|
||||
|
||||
flex_attn_kv_block_size: int | None = None
|
||||
"""Logical KV block size for the flex attention block mask.
|
||||
Must be a power of 2 and divisible by flex_attn_block_n.
|
||||
If None, uses the default (kv_cache_block_size on PyTorch >= 2.9,
|
||||
128 otherwise)."""
|
||||
If None, uses the KV cache block size for paged KV attention on
|
||||
PyTorch >= 2.9, and 128 for encoder-only attention or older PyTorch
|
||||
versions."""
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
|
||||
@@ -877,12 +877,14 @@ class FlexAttentionMetadataBuilder(AttentionMetadataBuilder[FlexAttentionMetadat
|
||||
self.block_size = kv_cache_spec.block_size
|
||||
self.kv_cache_spec = kv_cache_spec
|
||||
supports_small_blocks = is_torch_equal_or_newer("2.9.0.dev0")
|
||||
uses_paged_kv = not isinstance(kv_cache_spec, EncoderOnlyAttentionSpec)
|
||||
self.direct_build: bool = supports_small_blocks
|
||||
|
||||
self.q_block_size, self.kv_block_size = self._get_block_sizes(
|
||||
vllm_config.attention_config,
|
||||
supports_small_blocks,
|
||||
self.block_size,
|
||||
uses_paged_kv,
|
||||
)
|
||||
|
||||
if self.direct_build and self.kv_block_size != self.block_size:
|
||||
@@ -945,9 +947,17 @@ class FlexAttentionMetadataBuilder(AttentionMetadataBuilder[FlexAttentionMetadat
|
||||
attn_cfg,
|
||||
supports_small_blocks: bool,
|
||||
cache_block_size: int,
|
||||
uses_paged_kv: bool,
|
||||
) -> tuple[int, int]:
|
||||
q_block_size = 16 if supports_small_blocks else 128
|
||||
kv_block_size = cache_block_size if supports_small_blocks else 128
|
||||
# Small blocks are efficient with the direct mask builder, which maps
|
||||
# logical blocks to paged KV cache blocks without materializing and
|
||||
# sorting a generic block mask. Encoder-only attention has no paged KV
|
||||
# cache and therefore cannot use that path. Keeping its generic path at
|
||||
# 16-token blocks creates very large mask-conversion graphs and makes
|
||||
# Inductor compilation dominate the first request.
|
||||
use_small_blocks = supports_small_blocks and uses_paged_kv
|
||||
q_block_size = 16 if use_small_blocks else 128
|
||||
kv_block_size = cache_block_size if use_small_blocks else 128
|
||||
|
||||
q_block_size = attn_cfg.flex_attn_q_block_size or q_block_size
|
||||
if (q_block_size & (q_block_size - 1)) != 0 or (
|
||||
|
||||
Reference in New Issue
Block a user