[FlexAttention] Avoid encoder block-mask compile explosion (#50339)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-07-30 11:24:13 -04:00
committed by GitHub
parent 30b4e7f479
commit 5b95890742
4 changed files with 66 additions and 13 deletions
@@ -20,7 +20,9 @@ from vllm.platforms import current_platform
def _generate_random_text(word_count: int) -> str:
"""Generate random text with approximately the specified word count."""
"""Generate deterministic text with approximately the specified word count."""
rng = random.Random(word_count)
# Common English words with focus on verbs and nouns for realistic text
common_words = [
# Essential articles and pronouns (minimal)
@@ -178,7 +180,7 @@ def _generate_random_text(word_count: int) -> str:
words = []
for _ in range(word_count):
words.append(random.choice(common_words))
words.append(rng.choice(common_words))
# Add some punctuation for more realistic text
text = " ".join(words)
@@ -187,7 +189,7 @@ def _generate_random_text(word_count: int) -> str:
result = []
for i, word in enumerate(words_list):
result.append(word)
if (i + 1) % random.randint(10, 20) == 0 and i < len(words_list) - 1:
if (i + 1) % rng.randint(10, 20) == 0 and i < len(words_list) - 1:
result[-1] += "."
return " ".join(result)
+44 -5
View File
@@ -13,6 +13,7 @@ from tests.v1.attention.utils import (
create_standard_kv_cache_spec,
create_vllm_config,
)
from vllm.config import AttentionConfig
from vllm.model_executor.layers.attention import Attention
from vllm.v1.attention.backends.flex_attention import (
BlockSparsityHint,
@@ -27,6 +28,42 @@ MINIMUM_TORCH_VERSION = version.parse("2.7.0")
DIRECT_BUILD_VERSION = version.parse("2.9.dev0")
@pytest.mark.parametrize(
("supports_small_blocks", "uses_paged_kv", "expected"),
[
(True, True, (16, 16)),
(True, False, (128, 128)),
(False, True, (128, 128)),
(False, False, (128, 128)),
],
)
def test_flex_attention_default_block_sizes(
supports_small_blocks: bool,
uses_paged_kv: bool,
expected: tuple[int, int],
):
block_sizes = FlexAttentionMetadataBuilder._get_block_sizes(
AttentionConfig(),
supports_small_blocks=supports_small_blocks,
cache_block_size=16,
uses_paged_kv=uses_paged_kv,
)
assert block_sizes == expected
def test_flex_attention_explicit_block_sizes_override_encoder_defaults():
block_sizes = FlexAttentionMetadataBuilder._get_block_sizes(
AttentionConfig(
flex_attn_q_block_size=64,
flex_attn_kv_block_size=32,
),
supports_small_blocks=True,
cache_block_size=16,
uses_paged_kv=False,
)
assert block_sizes == (64, 32)
@pytest.mark.skipif(
not torch.cuda.is_available() or TORCH_VERSION < MINIMUM_TORCH_VERSION,
reason="CUDA not available or PyTorch version < 2.7",
@@ -213,10 +250,12 @@ def test_encoder_flex_attention_vs_default_backend(vllm_runner):
the default backend for encoder models.
"""
model_name = "BAAI/bge-base-en-v1.5"
# Exercise packed sequence boundaries inside 128-token FlexAttention
# blocks, including sequences that span more than one block.
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"hello " * 120,
"world " * 130,
"attention " * 254,
]
# Run with flex attention
@@ -225,7 +264,7 @@ def test_encoder_flex_attention_vs_default_backend(vllm_runner):
runner="pooling",
dtype=torch.bfloat16,
tensor_parallel_size=1,
max_model_len=100,
max_model_len=384,
enforce_eager=True,
attention_config={"backend": "FLEX_ATTENTION"},
) as llm_flex:
@@ -237,7 +276,7 @@ def test_encoder_flex_attention_vs_default_backend(vllm_runner):
runner="pooling",
dtype=torch.bfloat16,
tensor_parallel_size=1,
max_model_len=100,
max_model_len=384,
enforce_eager=True,
) as llm_default:
default_outputs = llm_default.embed(prompts)
+5 -3
View File
@@ -89,13 +89,15 @@ class AttentionConfig:
flex_attn_q_block_size: int | None = None
"""Logical Q block size for the flex attention block mask.
Must be a power of 2 and divisible by flex_attn_block_m.
If None, uses the default (16 on PyTorch >= 2.9, 128 otherwise)."""
If None, uses 16 for paged KV attention on PyTorch >= 2.9, and 128
for encoder-only attention or older PyTorch versions."""
flex_attn_kv_block_size: int | None = None
"""Logical KV block size for the flex attention block mask.
Must be a power of 2 and divisible by flex_attn_block_n.
If None, uses the default (kv_cache_block_size on PyTorch >= 2.9,
128 otherwise)."""
If None, uses the KV cache block size for paged KV attention on
PyTorch >= 2.9, and 128 for encoder-only attention or older PyTorch
versions."""
def compute_hash(self) -> str:
"""
+12 -2
View File
@@ -877,12 +877,14 @@ class FlexAttentionMetadataBuilder(AttentionMetadataBuilder[FlexAttentionMetadat
self.block_size = kv_cache_spec.block_size
self.kv_cache_spec = kv_cache_spec
supports_small_blocks = is_torch_equal_or_newer("2.9.0.dev0")
uses_paged_kv = not isinstance(kv_cache_spec, EncoderOnlyAttentionSpec)
self.direct_build: bool = supports_small_blocks
self.q_block_size, self.kv_block_size = self._get_block_sizes(
vllm_config.attention_config,
supports_small_blocks,
self.block_size,
uses_paged_kv,
)
if self.direct_build and self.kv_block_size != self.block_size:
@@ -945,9 +947,17 @@ class FlexAttentionMetadataBuilder(AttentionMetadataBuilder[FlexAttentionMetadat
attn_cfg,
supports_small_blocks: bool,
cache_block_size: int,
uses_paged_kv: bool,
) -> tuple[int, int]:
q_block_size = 16 if supports_small_blocks else 128
kv_block_size = cache_block_size if supports_small_blocks else 128
# Small blocks are efficient with the direct mask builder, which maps
# logical blocks to paged KV cache blocks without materializing and
# sorting a generic block mask. Encoder-only attention has no paged KV
# cache and therefore cannot use that path. Keeping its generic path at
# 16-token blocks creates very large mask-conversion graphs and makes
# Inductor compilation dominate the first request.
use_small_blocks = supports_small_blocks and uses_paged_kv
q_block_size = 16 if use_small_blocks else 128
kv_block_size = cache_block_size if use_small_blocks else 128
q_block_size = attn_cfg.flex_attn_q_block_size or q_block_size
if (q_block_size & (q_block_size - 1)) != 0 or (