expose flex block size for batch invariant mode (#41252)
Signed-off-by: Angel Li <liangel@meta.com>
This commit is contained in:
@@ -341,6 +341,29 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
f"but got {self.attn_backend.get_name()}."
|
||||
)
|
||||
|
||||
if self.attn_backend.get_name() == "FLEX_ATTENTION":
|
||||
block_m = vllm_config.attention_config.flex_attn_block_m
|
||||
block_n = vllm_config.attention_config.flex_attn_block_n
|
||||
|
||||
if envs.VLLM_BATCH_INVARIANT and cache_config is not None:
|
||||
if block_m is not None and block_m > cache_config.block_size:
|
||||
raise ValueError(
|
||||
f"flex_attn_block_m ({block_m}) must be "
|
||||
f"<= cache block size ({cache_config.block_size}) for "
|
||||
f"batch invariance"
|
||||
)
|
||||
if block_n is not None and block_n > cache_config.block_size:
|
||||
raise ValueError(
|
||||
f"flex_attn_block_n ({block_n}) must be "
|
||||
f"<= cache block size ({cache_config.block_size}) for "
|
||||
f"batch invariance"
|
||||
)
|
||||
|
||||
if block_m is not None:
|
||||
extra_impl_args.setdefault("block_m", block_m)
|
||||
if block_n is not None:
|
||||
extra_impl_args.setdefault("block_n", block_n)
|
||||
|
||||
impl_cls = self.attn_backend.get_impl_cls()
|
||||
self.impl = impl_cls( # type: ignore[assignment] # impl_cls always returns an AttentionImpl subclass
|
||||
num_heads,
|
||||
|
||||
Reference in New Issue
Block a user