FlexAttention non-causal support (#40394)

Signed-off-by: Fynn Schmitt-Ulms <fschmitt@redhat.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Fynn Schmitt-Ulms
2026-04-22 13:22:57 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent eb6661d522
commit 29f64c5f5e
3 changed files with 122 additions and 17 deletions
+29 -10
View File
@@ -36,7 +36,7 @@ from vllm.v1.attention.backend import (
AttentionType,
CommonAttentionMetadata,
)
from vllm.v1.kv_cache_interface import AttentionSpec
from vllm.v1.kv_cache_interface import AttentionSpec, EncoderOnlyAttentionSpec
logger = init_logger(__name__)
@@ -90,6 +90,10 @@ class FlexAttentionBackend(AttentionBackend):
def get_name() -> str:
return "FLEX_ATTENTION"
@classmethod
def supports_non_causal(cls) -> bool:
return True
@classmethod
def supports_attn_type(cls, attn_type: str) -> bool:
"""FlexAttention supports both decoder and encoder-only attention."""
@@ -294,6 +298,12 @@ def causal_mask_mod(
return q_idx >= kv_idx
def bidirectional_mask_mod(
b: torch.Tensor, h: torch.Tensor, q_idx: torch.Tensor, kv_idx: torch.Tensor
):
return q_idx >= 0
# Type alias for the block sparsity hint callable signature.
_block_sparsity_hint_signature = Callable[
[torch.Tensor, torch.Tensor, int], torch.Tensor
@@ -364,6 +374,7 @@ class FlexAttentionMetadata:
block_mask: BlockMask | None = None
score_mod: _score_mod_signature | None = None
logical_mask_mod: _mask_mod_signature = causal_mask_mod
uses_paged_kv: bool = True
doc_ids: torch.Tensor | None = None
direct_build: bool = True
q_block_size: int = 16
@@ -497,7 +508,7 @@ class FlexAttentionMetadata:
False,
)
return final_mask_mod if self.causal else sliding_window_mask_mod
return final_mask_mod if self.uses_paged_kv else sliding_window_mask_mod
def get_prefix_lm_mask_mod(self) -> _mask_mod_signature:
"""Creates the prefix LM mask_mod function for FlexAttention."""
@@ -541,8 +552,7 @@ class FlexAttentionMetadata:
def get_mask_mod(self):
# Stage-1: initialize the base mask_mod
# (causal mask for decoder or bidirectional mask for encoder)
has_custom_mask = self.logical_mask_mod is not causal_mask_mod
if self.causal or has_custom_mask:
if self.uses_paged_kv:
mask_mod = self.get_paged_mask_mod()
else:
mask_mod = self.get_bidirectional_mask_mod()
@@ -595,7 +605,7 @@ class FlexAttentionMetadata:
return transformed_score_mod
def _build_block_mask_direct(self) -> BlockMask:
"""Direct block mask construction for standard causal attention.
"""Direct block mask construction for paged KV cache attention.
This method constructs the block mask directly using
BlockMask.from_kv_blocks which is much more efficient than the
@@ -693,7 +703,9 @@ class FlexAttentionMetadata:
def build_block_mask(self) -> BlockMask:
mask_mod = self.get_mask_mod()
kv_len = self.total_cache_tokens if self.causal else self.num_actual_tokens
kv_len = (
self.total_cache_tokens if self.uses_paged_kv else self.num_actual_tokens
)
return create_block_mask_compiled(
mask_mod,
None,
@@ -842,8 +854,16 @@ class FlexAttentionMetadataBuilder(AttentionMetadataBuilder[FlexAttentionMetadat
offset_tensor = common_attn_metadata.compute_num_computed_tokens()
offset_tensor = copy_to_persistent(self.persistent_offset_tensor, offset_tensor)
uses_paged_kv = not isinstance(self.kv_cache_spec, EncoderOnlyAttentionSpec)
logical_mask_mod = (
bidirectional_mask_mod
if uses_paged_kv and not common_attn_metadata.causal
else causal_mask_mod
)
out = FlexAttentionMetadata(
causal=common_attn_metadata.causal,
logical_mask_mod=logical_mask_mod,
num_actual_tokens=num_actual_tokens,
max_query_len=max_query_len,
query_start_loc=query_start_loc,
@@ -863,10 +883,11 @@ class FlexAttentionMetadataBuilder(AttentionMetadataBuilder[FlexAttentionMetadat
total_cache_tokens=total_cache_tokens,
decode_offset=offset_tensor,
num_blocks_per_seq=num_blocks_per_seq,
uses_paged_kv=uses_paged_kv,
# FIXME(Isotr0py): direct build has issue to build bidirectional
# attention block mask for encoder-only models, disable it temporarily.
# see: https://github.com/vllm-project/vllm/pull/27329#issuecomment-3431484053
direct_build=(self.direct_build and common_attn_metadata.causal),
direct_build=self.direct_build and uses_paged_kv,
q_block_size=self.q_block_size,
kv_block_size=self.kv_block_size,
persistent_kv_indices=self.persistent_kv_indices,
@@ -1055,9 +1076,7 @@ class FlexAttentionImpl(AttentionImpl):
else:
attn_metadata.block_mask = attn_metadata.build_block_mask()
if not attn_metadata.causal:
assert self.attn_type == AttentionType.ENCODER_ONLY
if self.attn_type == AttentionType.ENCODER_ONLY:
query, key_tensor, value_tensor = map(
lambda x: self.view_as_4d(x).permute(0, 2, 1, 3),
(query, key, value),