[FlexAttention] Avoid encoder block-mask compile explosion (#50339)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-07-30 11:24:13 -04:00
committed by GitHub
parent 30b4e7f479
commit 5b95890742
4 changed files with 66 additions and 13 deletions
@@ -20,7 +20,9 @@ from vllm.platforms import current_platform
def _generate_random_text(word_count: int) -> str:
"""Generate random text with approximately the specified word count."""
"""Generate deterministic text with approximately the specified word count."""
rng = random.Random(word_count)
# Common English words with focus on verbs and nouns for realistic text
common_words = [
# Essential articles and pronouns (minimal)
@@ -178,7 +180,7 @@ def _generate_random_text(word_count: int) -> str:
words = []
for _ in range(word_count):
words.append(random.choice(common_words))
words.append(rng.choice(common_words))
# Add some punctuation for more realistic text
text = " ".join(words)
@@ -187,7 +189,7 @@ def _generate_random_text(word_count: int) -> str:
result = []
for i, word in enumerate(words_list):
result.append(word)
if (i + 1) % random.randint(10, 20) == 0 and i < len(words_list) - 1:
if (i + 1) % rng.randint(10, 20) == 0 and i < len(words_list) - 1:
result[-1] += "."
return " ".join(result)
+44 -5
View File
@@ -13,6 +13,7 @@ from tests.v1.attention.utils import (
create_standard_kv_cache_spec,
create_vllm_config,
)
from vllm.config import AttentionConfig
from vllm.model_executor.layers.attention import Attention
from vllm.v1.attention.backends.flex_attention import (
BlockSparsityHint,
@@ -27,6 +28,42 @@ MINIMUM_TORCH_VERSION = version.parse("2.7.0")
DIRECT_BUILD_VERSION = version.parse("2.9.dev0")
@pytest.mark.parametrize(
("supports_small_blocks", "uses_paged_kv", "expected"),
[
(True, True, (16, 16)),
(True, False, (128, 128)),
(False, True, (128, 128)),
(False, False, (128, 128)),
],
)
def test_flex_attention_default_block_sizes(
supports_small_blocks: bool,
uses_paged_kv: bool,
expected: tuple[int, int],
):
block_sizes = FlexAttentionMetadataBuilder._get_block_sizes(
AttentionConfig(),
supports_small_blocks=supports_small_blocks,
cache_block_size=16,
uses_paged_kv=uses_paged_kv,
)
assert block_sizes == expected
def test_flex_attention_explicit_block_sizes_override_encoder_defaults():
block_sizes = FlexAttentionMetadataBuilder._get_block_sizes(
AttentionConfig(
flex_attn_q_block_size=64,
flex_attn_kv_block_size=32,
),
supports_small_blocks=True,
cache_block_size=16,
uses_paged_kv=False,
)
assert block_sizes == (64, 32)
@pytest.mark.skipif(
not torch.cuda.is_available() or TORCH_VERSION < MINIMUM_TORCH_VERSION,
reason="CUDA not available or PyTorch version < 2.7",
@@ -213,10 +250,12 @@ def test_encoder_flex_attention_vs_default_backend(vllm_runner):
the default backend for encoder models.
"""
model_name = "BAAI/bge-base-en-v1.5"
# Exercise packed sequence boundaries inside 128-token FlexAttention
# blocks, including sequences that span more than one block.
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"hello " * 120,
"world " * 130,
"attention " * 254,
]
# Run with flex attention
@@ -225,7 +264,7 @@ def test_encoder_flex_attention_vs_default_backend(vllm_runner):
runner="pooling",
dtype=torch.bfloat16,
tensor_parallel_size=1,
max_model_len=100,
max_model_len=384,
enforce_eager=True,
attention_config={"backend": "FLEX_ATTENTION"},
) as llm_flex:
@@ -237,7 +276,7 @@ def test_encoder_flex_attention_vs_default_backend(vllm_runner):
runner="pooling",
dtype=torch.bfloat16,
tensor_parallel_size=1,
max_model_len=100,
max_model_len=384,
enforce_eager=True,
) as llm_default:
default_outputs = llm_default.embed(prompts)