expose flex block size for batch invariant mode (#41252)

Signed-off-by: Angel Li <liangel@meta.com>
This commit is contained in:
liangel-02
2026-05-13 14:11:57 -07:00
committed by GitHub
parent 8efd508204
commit 6b5c389ee3
4 changed files with 114 additions and 8 deletions
@@ -341,6 +341,29 @@ class Attention(nn.Module, AttentionLayerBase):
f"but got {self.attn_backend.get_name()}."
)
if self.attn_backend.get_name() == "FLEX_ATTENTION":
block_m = vllm_config.attention_config.flex_attn_block_m
block_n = vllm_config.attention_config.flex_attn_block_n
if envs.VLLM_BATCH_INVARIANT and cache_config is not None:
if block_m is not None and block_m > cache_config.block_size:
raise ValueError(
f"flex_attn_block_m ({block_m}) must be "
f"<= cache block size ({cache_config.block_size}) for "
f"batch invariance"
)
if block_n is not None and block_n > cache_config.block_size:
raise ValueError(
f"flex_attn_block_n ({block_n}) must be "
f"<= cache block size ({cache_config.block_size}) for "
f"batch invariance"
)
if block_m is not None:
extra_impl_args.setdefault("block_m", block_m)
if block_n is not None:
extra_impl_args.setdefault("block_n", block_n)
impl_cls = self.attn_backend.get_impl_cls()
self.impl = impl_cls( # type: ignore[assignment] # impl_cls always returns an AttentionImpl subclass
num_heads,