diff --git a/tests/entrypoints/pooling/embed/test_online_long_text.py b/tests/entrypoints/pooling/embed/test_online_long_text.py index eaefbc0238..c1dc5bf1ed 100644 --- a/tests/entrypoints/pooling/embed/test_online_long_text.py +++ b/tests/entrypoints/pooling/embed/test_online_long_text.py @@ -20,7 +20,9 @@ from vllm.platforms import current_platform def _generate_random_text(word_count: int) -> str: - """Generate random text with approximately the specified word count.""" + """Generate deterministic text with approximately the specified word count.""" + rng = random.Random(word_count) + # Common English words with focus on verbs and nouns for realistic text common_words = [ # Essential articles and pronouns (minimal) @@ -178,7 +180,7 @@ def _generate_random_text(word_count: int) -> str: words = [] for _ in range(word_count): - words.append(random.choice(common_words)) + words.append(rng.choice(common_words)) # Add some punctuation for more realistic text text = " ".join(words) @@ -187,7 +189,7 @@ def _generate_random_text(word_count: int) -> str: result = [] for i, word in enumerate(words_list): result.append(word) - if (i + 1) % random.randint(10, 20) == 0 and i < len(words_list) - 1: + if (i + 1) % rng.randint(10, 20) == 0 and i < len(words_list) - 1: result[-1] += "." return " ".join(result) diff --git a/tests/kernels/test_flex_attention.py b/tests/kernels/test_flex_attention.py index f01dca373a..a80467ecc3 100644 --- a/tests/kernels/test_flex_attention.py +++ b/tests/kernels/test_flex_attention.py @@ -13,6 +13,7 @@ from tests.v1.attention.utils import ( create_standard_kv_cache_spec, create_vllm_config, ) +from vllm.config import AttentionConfig from vllm.model_executor.layers.attention import Attention from vllm.v1.attention.backends.flex_attention import ( BlockSparsityHint, @@ -27,6 +28,42 @@ MINIMUM_TORCH_VERSION = version.parse("2.7.0") DIRECT_BUILD_VERSION = version.parse("2.9.dev0") +@pytest.mark.parametrize( + ("supports_small_blocks", "uses_paged_kv", "expected"), + [ + (True, True, (16, 16)), + (True, False, (128, 128)), + (False, True, (128, 128)), + (False, False, (128, 128)), + ], +) +def test_flex_attention_default_block_sizes( + supports_small_blocks: bool, + uses_paged_kv: bool, + expected: tuple[int, int], +): + block_sizes = FlexAttentionMetadataBuilder._get_block_sizes( + AttentionConfig(), + supports_small_blocks=supports_small_blocks, + cache_block_size=16, + uses_paged_kv=uses_paged_kv, + ) + assert block_sizes == expected + + +def test_flex_attention_explicit_block_sizes_override_encoder_defaults(): + block_sizes = FlexAttentionMetadataBuilder._get_block_sizes( + AttentionConfig( + flex_attn_q_block_size=64, + flex_attn_kv_block_size=32, + ), + supports_small_blocks=True, + cache_block_size=16, + uses_paged_kv=False, + ) + assert block_sizes == (64, 32) + + @pytest.mark.skipif( not torch.cuda.is_available() or TORCH_VERSION < MINIMUM_TORCH_VERSION, reason="CUDA not available or PyTorch version < 2.7", @@ -213,10 +250,12 @@ def test_encoder_flex_attention_vs_default_backend(vllm_runner): the default backend for encoder models. """ model_name = "BAAI/bge-base-en-v1.5" + # Exercise packed sequence boundaries inside 128-token FlexAttention + # blocks, including sequences that span more than one block. prompts = [ - "Hello, my name is", - "The president of the United States is", - "The capital of France is", + "hello " * 120, + "world " * 130, + "attention " * 254, ] # Run with flex attention @@ -225,7 +264,7 @@ def test_encoder_flex_attention_vs_default_backend(vllm_runner): runner="pooling", dtype=torch.bfloat16, tensor_parallel_size=1, - max_model_len=100, + max_model_len=384, enforce_eager=True, attention_config={"backend": "FLEX_ATTENTION"}, ) as llm_flex: @@ -237,7 +276,7 @@ def test_encoder_flex_attention_vs_default_backend(vllm_runner): runner="pooling", dtype=torch.bfloat16, tensor_parallel_size=1, - max_model_len=100, + max_model_len=384, enforce_eager=True, ) as llm_default: default_outputs = llm_default.embed(prompts) diff --git a/vllm/config/attention.py b/vllm/config/attention.py index 78bc1f9537..089f13b705 100644 --- a/vllm/config/attention.py +++ b/vllm/config/attention.py @@ -89,13 +89,15 @@ class AttentionConfig: flex_attn_q_block_size: int | None = None """Logical Q block size for the flex attention block mask. Must be a power of 2 and divisible by flex_attn_block_m. - If None, uses the default (16 on PyTorch >= 2.9, 128 otherwise).""" + If None, uses 16 for paged KV attention on PyTorch >= 2.9, and 128 + for encoder-only attention or older PyTorch versions.""" flex_attn_kv_block_size: int | None = None """Logical KV block size for the flex attention block mask. Must be a power of 2 and divisible by flex_attn_block_n. - If None, uses the default (kv_cache_block_size on PyTorch >= 2.9, - 128 otherwise).""" + If None, uses the KV cache block size for paged KV attention on + PyTorch >= 2.9, and 128 for encoder-only attention or older PyTorch + versions.""" def compute_hash(self) -> str: """ diff --git a/vllm/v1/attention/backends/flex_attention.py b/vllm/v1/attention/backends/flex_attention.py index 83144751ae..b7bcac54bb 100644 --- a/vllm/v1/attention/backends/flex_attention.py +++ b/vllm/v1/attention/backends/flex_attention.py @@ -877,12 +877,14 @@ class FlexAttentionMetadataBuilder(AttentionMetadataBuilder[FlexAttentionMetadat self.block_size = kv_cache_spec.block_size self.kv_cache_spec = kv_cache_spec supports_small_blocks = is_torch_equal_or_newer("2.9.0.dev0") + uses_paged_kv = not isinstance(kv_cache_spec, EncoderOnlyAttentionSpec) self.direct_build: bool = supports_small_blocks self.q_block_size, self.kv_block_size = self._get_block_sizes( vllm_config.attention_config, supports_small_blocks, self.block_size, + uses_paged_kv, ) if self.direct_build and self.kv_block_size != self.block_size: @@ -945,9 +947,17 @@ class FlexAttentionMetadataBuilder(AttentionMetadataBuilder[FlexAttentionMetadat attn_cfg, supports_small_blocks: bool, cache_block_size: int, + uses_paged_kv: bool, ) -> tuple[int, int]: - q_block_size = 16 if supports_small_blocks else 128 - kv_block_size = cache_block_size if supports_small_blocks else 128 + # Small blocks are efficient with the direct mask builder, which maps + # logical blocks to paged KV cache blocks without materializing and + # sorting a generic block mask. Encoder-only attention has no paged KV + # cache and therefore cannot use that path. Keeping its generic path at + # 16-token blocks creates very large mask-conversion graphs and makes + # Inductor compilation dominate the first request. + use_small_blocks = supports_small_blocks and uses_paged_kv + q_block_size = 16 if use_small_blocks else 128 + kv_block_size = cache_block_size if use_small_blocks else 128 q_block_size = attn_cfg.flex_attn_q_block_size or q_block_size if (q_block_size & (q_block_size - 1)) != 0 or (