diff --git a/vllm/model_executor/layers/attention/attention.py b/vllm/model_executor/layers/attention/attention.py index c2adf3499a..db3dffa854 100644 --- a/vllm/model_executor/layers/attention/attention.py +++ b/vllm/model_executor/layers/attention/attention.py @@ -355,6 +355,7 @@ class Attention(nn.Module, AttentionLayerBase): use_mm_prefix=self.use_mm_prefix, use_per_head_quant_scales=use_per_head_quant_scales, attn_type=attn_type, + has_sliding_window=sliding_window is not None, ) else: self.attn_backend = attn_backend diff --git a/vllm/v1/attention/backend.py b/vllm/v1/attention/backend.py index 4ecf0b9ad0..9559b05e91 100644 --- a/vllm/v1/attention/backend.py +++ b/vllm/v1/attention/backend.py @@ -258,6 +258,10 @@ class AttentionBackend(ABC): def supports_per_head_quant_scales(cls) -> bool: return False + @classmethod + def supports_sliding_window(cls) -> bool: + return False + @classmethod def supports_non_causal(cls) -> bool: """Check if backend supports non-causal (bidirectional) attention @@ -319,6 +323,7 @@ class AttentionBackend(ABC): use_per_head_quant_scales: bool, device_capability: "DeviceCapability", attn_type: str, + has_sliding_window: bool = False, use_non_causal: bool = False, use_batch_invariant: bool = False, use_kv_connector: bool = False, @@ -354,6 +359,8 @@ class AttentionBackend(ABC): invalid_reasons.append("compute capability not supported") if not cls.supports_attn_type(attn_type): invalid_reasons.append(f"attention type {attn_type} not supported") + if has_sliding_window and not cls.supports_sliding_window(): + invalid_reasons.append("sliding window not supported") if use_non_causal and not cls.supports_non_causal(): invalid_reasons.append("non-causal attention not supported") if use_batch_invariant and not cls.supports_batch_invariance(): diff --git a/vllm/v1/attention/backends/cpu_attn.py b/vllm/v1/attention/backends/cpu_attn.py index a5735bf313..e9c31f42f2 100644 --- a/vllm/v1/attention/backends/cpu_attn.py +++ b/vllm/v1/attention/backends/cpu_attn.py @@ -67,6 +67,10 @@ class CPUAttentionBackend(AttentionBackend): def supports_non_causal(cls) -> bool: return True + @classmethod + def supports_sliding_window(cls) -> bool: + return True + @classmethod def supports_attn_type(cls, attn_type: str) -> bool: """CPU attention supports decoder, diff --git a/vllm/v1/attention/backends/flash_attn.py b/vllm/v1/attention/backends/flash_attn.py index 844385326f..86198dda61 100755 --- a/vllm/v1/attention/backends/flash_attn.py +++ b/vllm/v1/attention/backends/flash_attn.py @@ -92,6 +92,10 @@ class FlashAttentionBackend(AttentionBackend): def get_name() -> str: return "FLASH_ATTN" + @classmethod + def supports_sliding_window(cls) -> bool: + return True + @classmethod def supports_batch_invariance(cls) -> bool: return True diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py index 699073939e..08ea5efdeb 100755 --- a/vllm/v1/attention/backends/flashinfer.py +++ b/vllm/v1/attention/backends/flashinfer.py @@ -380,6 +380,10 @@ class FlashInferBackend(AttentionBackend): def supports_non_causal(cls) -> bool: return True + @classmethod + def supports_sliding_window(cls) -> bool: + return True + @staticmethod def get_impl_cls() -> type["FlashInferImpl"]: return FlashInferImpl diff --git a/vllm/v1/attention/backends/flex_attention.py b/vllm/v1/attention/backends/flex_attention.py index b918f2adef..e7fe585219 100644 --- a/vllm/v1/attention/backends/flex_attention.py +++ b/vllm/v1/attention/backends/flex_attention.py @@ -100,6 +100,10 @@ class FlexAttentionBackend(AttentionBackend): def get_name() -> str: return "FLEX_ATTENTION" + @classmethod + def supports_sliding_window(cls) -> bool: + return True + @classmethod def supports_non_causal(cls) -> bool: return True diff --git a/vllm/v1/attention/backends/rocm_aiter_fa.py b/vllm/v1/attention/backends/rocm_aiter_fa.py index 4290c22ced..b2edc2d051 100644 --- a/vllm/v1/attention/backends/rocm_aiter_fa.py +++ b/vllm/v1/attention/backends/rocm_aiter_fa.py @@ -750,6 +750,10 @@ class AiterFlashAttentionBackend(AttentionBackend): def get_name() -> str: return "FLASH_ATTN" + @classmethod + def supports_sliding_window(cls) -> bool: + return True + @staticmethod def get_impl_cls() -> type["AiterFlashAttentionImpl"]: return AiterFlashAttentionImpl diff --git a/vllm/v1/attention/backends/rocm_attn.py b/vllm/v1/attention/backends/rocm_attn.py index 24f65ab31a..331b5c3fb3 100644 --- a/vllm/v1/attention/backends/rocm_attn.py +++ b/vllm/v1/attention/backends/rocm_attn.py @@ -220,6 +220,10 @@ class RocmAttentionBackend(AttentionBackend): def get_name() -> str: return "ROCM_ATTN" + @classmethod + def supports_sliding_window(cls) -> bool: + return True + @staticmethod def get_impl_cls() -> type["RocmAttentionImpl"]: return RocmAttentionImpl diff --git a/vllm/v1/attention/backends/triton_attn.py b/vllm/v1/attention/backends/triton_attn.py index fa2fe8f1e6..d348fede66 100644 --- a/vllm/v1/attention/backends/triton_attn.py +++ b/vllm/v1/attention/backends/triton_attn.py @@ -306,6 +306,10 @@ class TritonAttentionBackend(AttentionBackend): def get_name() -> str: return "TRITON_ATTN" + @classmethod + def supports_sliding_window(cls) -> bool: + return True + @classmethod def supports_batch_invariance(cls) -> bool: return True diff --git a/vllm/v1/attention/selector.py b/vllm/v1/attention/selector.py index 5d63553a7b..917f2f6f70 100644 --- a/vllm/v1/attention/selector.py +++ b/vllm/v1/attention/selector.py @@ -29,6 +29,7 @@ class AttentionSelectorConfig(NamedTuple): use_mm_prefix: bool = False use_per_head_quant_scales: bool = False attn_type: str = AttentionType.DECODER + has_sliding_window: bool = False use_non_causal: bool = False use_batch_invariant: bool = False use_kv_connector: bool = False @@ -45,6 +46,7 @@ class AttentionSelectorConfig(NamedTuple): f"use_mm_prefix={self.use_mm_prefix}, " f"use_per_head_quant_scales={self.use_per_head_quant_scales}, " f"attn_type={self.attn_type}, " + f"has_sliding_window={self.has_sliding_window}, " f"use_non_causal={self.use_non_causal}, " f"use_batch_invariant={self.use_batch_invariant}, " f"use_kv_connector={self.use_kv_connector})" @@ -62,6 +64,7 @@ def get_attn_backend( use_per_head_quant_scales: bool = False, attn_type: str | None = None, num_heads: int | None = None, + has_sliding_window: bool = False, ) -> type[AttentionBackend]: """Selects which attention backend to use and lazily imports it.""" @@ -77,6 +80,7 @@ def get_attn_backend( vllm_config = get_current_vllm_config() cache_config = vllm_config.cache_config + block_size: int | None if cache_config is not None and cache_config.user_specified_block_size: block_size = cache_config.block_size else: @@ -98,6 +102,7 @@ def get_attn_backend( use_mm_prefix=use_mm_prefix, use_per_head_quant_scales=use_per_head_quant_scales, attn_type=attn_type or AttentionType.DECODER, + has_sliding_window=has_sliding_window, use_non_causal=vllm_config.attention_config.use_non_causal, use_batch_invariant=envs.VLLM_BATCH_INVARIANT, use_kv_connector=use_kv_connector,