[Attention] Make sliding-window support an explicit backend capability (#48011)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Andreas Karatzas
parent
75fe92a316
commit
bea70c7cfc
@@ -355,6 +355,7 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
use_mm_prefix=self.use_mm_prefix,
|
||||
use_per_head_quant_scales=use_per_head_quant_scales,
|
||||
attn_type=attn_type,
|
||||
has_sliding_window=sliding_window is not None,
|
||||
)
|
||||
else:
|
||||
self.attn_backend = attn_backend
|
||||
|
||||
@@ -258,6 +258,10 @@ class AttentionBackend(ABC):
|
||||
def supports_per_head_quant_scales(cls) -> bool:
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def supports_sliding_window(cls) -> bool:
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def supports_non_causal(cls) -> bool:
|
||||
"""Check if backend supports non-causal (bidirectional) attention
|
||||
@@ -319,6 +323,7 @@ class AttentionBackend(ABC):
|
||||
use_per_head_quant_scales: bool,
|
||||
device_capability: "DeviceCapability",
|
||||
attn_type: str,
|
||||
has_sliding_window: bool = False,
|
||||
use_non_causal: bool = False,
|
||||
use_batch_invariant: bool = False,
|
||||
use_kv_connector: bool = False,
|
||||
@@ -354,6 +359,8 @@ class AttentionBackend(ABC):
|
||||
invalid_reasons.append("compute capability not supported")
|
||||
if not cls.supports_attn_type(attn_type):
|
||||
invalid_reasons.append(f"attention type {attn_type} not supported")
|
||||
if has_sliding_window and not cls.supports_sliding_window():
|
||||
invalid_reasons.append("sliding window not supported")
|
||||
if use_non_causal and not cls.supports_non_causal():
|
||||
invalid_reasons.append("non-causal attention not supported")
|
||||
if use_batch_invariant and not cls.supports_batch_invariance():
|
||||
|
||||
@@ -67,6 +67,10 @@ class CPUAttentionBackend(AttentionBackend):
|
||||
def supports_non_causal(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def supports_sliding_window(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def supports_attn_type(cls, attn_type: str) -> bool:
|
||||
"""CPU attention supports decoder,
|
||||
|
||||
@@ -92,6 +92,10 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
def get_name() -> str:
|
||||
return "FLASH_ATTN"
|
||||
|
||||
@classmethod
|
||||
def supports_sliding_window(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def supports_batch_invariance(cls) -> bool:
|
||||
return True
|
||||
|
||||
@@ -380,6 +380,10 @@ class FlashInferBackend(AttentionBackend):
|
||||
def supports_non_causal(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def supports_sliding_window(cls) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls() -> type["FlashInferImpl"]:
|
||||
return FlashInferImpl
|
||||
|
||||
@@ -100,6 +100,10 @@ class FlexAttentionBackend(AttentionBackend):
|
||||
def get_name() -> str:
|
||||
return "FLEX_ATTENTION"
|
||||
|
||||
@classmethod
|
||||
def supports_sliding_window(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def supports_non_causal(cls) -> bool:
|
||||
return True
|
||||
|
||||
@@ -750,6 +750,10 @@ class AiterFlashAttentionBackend(AttentionBackend):
|
||||
def get_name() -> str:
|
||||
return "FLASH_ATTN"
|
||||
|
||||
@classmethod
|
||||
def supports_sliding_window(cls) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls() -> type["AiterFlashAttentionImpl"]:
|
||||
return AiterFlashAttentionImpl
|
||||
|
||||
@@ -220,6 +220,10 @@ class RocmAttentionBackend(AttentionBackend):
|
||||
def get_name() -> str:
|
||||
return "ROCM_ATTN"
|
||||
|
||||
@classmethod
|
||||
def supports_sliding_window(cls) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls() -> type["RocmAttentionImpl"]:
|
||||
return RocmAttentionImpl
|
||||
|
||||
@@ -306,6 +306,10 @@ class TritonAttentionBackend(AttentionBackend):
|
||||
def get_name() -> str:
|
||||
return "TRITON_ATTN"
|
||||
|
||||
@classmethod
|
||||
def supports_sliding_window(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def supports_batch_invariance(cls) -> bool:
|
||||
return True
|
||||
|
||||
@@ -29,6 +29,7 @@ class AttentionSelectorConfig(NamedTuple):
|
||||
use_mm_prefix: bool = False
|
||||
use_per_head_quant_scales: bool = False
|
||||
attn_type: str = AttentionType.DECODER
|
||||
has_sliding_window: bool = False
|
||||
use_non_causal: bool = False
|
||||
use_batch_invariant: bool = False
|
||||
use_kv_connector: bool = False
|
||||
@@ -45,6 +46,7 @@ class AttentionSelectorConfig(NamedTuple):
|
||||
f"use_mm_prefix={self.use_mm_prefix}, "
|
||||
f"use_per_head_quant_scales={self.use_per_head_quant_scales}, "
|
||||
f"attn_type={self.attn_type}, "
|
||||
f"has_sliding_window={self.has_sliding_window}, "
|
||||
f"use_non_causal={self.use_non_causal}, "
|
||||
f"use_batch_invariant={self.use_batch_invariant}, "
|
||||
f"use_kv_connector={self.use_kv_connector})"
|
||||
@@ -62,6 +64,7 @@ def get_attn_backend(
|
||||
use_per_head_quant_scales: bool = False,
|
||||
attn_type: str | None = None,
|
||||
num_heads: int | None = None,
|
||||
has_sliding_window: bool = False,
|
||||
) -> type[AttentionBackend]:
|
||||
"""Selects which attention backend to use and lazily imports it."""
|
||||
|
||||
@@ -77,6 +80,7 @@ def get_attn_backend(
|
||||
vllm_config = get_current_vllm_config()
|
||||
|
||||
cache_config = vllm_config.cache_config
|
||||
block_size: int | None
|
||||
if cache_config is not None and cache_config.user_specified_block_size:
|
||||
block_size = cache_config.block_size
|
||||
else:
|
||||
@@ -98,6 +102,7 @@ def get_attn_backend(
|
||||
use_mm_prefix=use_mm_prefix,
|
||||
use_per_head_quant_scales=use_per_head_quant_scales,
|
||||
attn_type=attn_type or AttentionType.DECODER,
|
||||
has_sliding_window=has_sliding_window,
|
||||
use_non_causal=vllm_config.attention_config.use_non_causal,
|
||||
use_batch_invariant=envs.VLLM_BATCH_INVARIANT,
|
||||
use_kv_connector=use_kv_connector,
|
||||
|
||||
Reference in New Issue
Block a user