diff --git a/docs/design/attention_backends.md b/docs/design/attention_backends.md index 3fdc23ff84..d175add40b 100644 --- a/docs/design/attention_backends.md +++ b/docs/design/attention_backends.md @@ -56,6 +56,55 @@ llm = LLM( ## Backend Selection Behavior +### Triton/FlashAttention Composite + +On Hopper, `TRITON_FLASH_ATTN` is preferred for compatible multimodal-prefix +configurations. It uses Triton when the current queries need bidirectional +image attention and FlashAttention for causal text prefills and decode, with a +shared KV cache. The causal child must resolve to FA4. This happens +automatically for FA4-only shapes such as head size 512 and models whose +version policy promotes all layers to FA4, including Gemma 4. A standalone +head-size-256 configuration whose version policy selects FA3 falls back to +Triton for the whole layer. + +No attention override is needed for supported multimodal models: + +```bash +vllm serve google/gemma-4-31B-it +``` + +### Triton/FlashInfer Composite + +On Blackwell, `TRITON_FLASHINFER` is preferred for compatible +multimodal-prefix configurations, including Gemma 4 with BF16 or FP8 KV cache. It uses +Triton when a batch's current queries require bidirectional image attention, +and FlashInfer for causal text prefills and decode. Historical image tokens +alone do not select Triton. + +This instantiates `create_composite_attention_backend` from +`vllm/v1/attention/backends/composite.py` with Triton, +FlashInfer, and `MMPrefixAttentionRouting`. The reusable factory owns child +implementations, metadata dispatch, compatible cache requirements, and workspace +sharing. The routing policy selects the child and defines graph-capture safety. +Other combinations can reuse the same machinery. + +The backend supports head dimensions 256/512, FP16/BF16 and FP8 KV cache, +and 64-token kernel pages with a head-major cache layout. TRTLLM handles causal +attention at both head dimensions; its hdim512 kernels do not support 128-token +pages. +For Gemma 4, full CUDA graphs cover single-token batches; multi-token batches +use the non-full-graph execution path. Models whose image masks extend beyond +the sliding window cannot use full attention graphs with this composite. +Context parallelism, R-SWA, attention sinks, +and adaptive verification are not supported by this composite. + +It can also be selected explicitly: + +```bash +vllm serve google/gemma-4-31B-it \ + --attention-backend TRITON_FLASHINFER +``` + ### Manual Selection When you explicitly set a backend via `--attention-backend` or `AttentionConfig`: diff --git a/tests/kernels/attention/test_attention_selector.py b/tests/kernels/attention/test_attention_selector.py index d74a119e65..53867a5edf 100644 --- a/tests/kernels/attention/test_attention_selector.py +++ b/tests/kernels/attention/test_attention_selector.py @@ -808,3 +808,142 @@ def test_fa4_hd256_impl_selection(attn_type, sliding_window, expected): ) assert impl.vllm_flash_attn_version == expected assert impl.fa4_hd256 == (expected == 4) + + +@pytest.fixture +def blackwell_selection(): + with ( + patch.object( + type(current_platform), + "get_device_capability", + return_value=DeviceCapability(10, 0), + ), + patch( + "vllm.v1.attention.backends.fa_utils.is_fa_version_supported", + return_value=True, + ), + ): + yield + + +@pytest.fixture +def hopper_selection(): + with ( + patch.object( + type(current_platform), + "get_device_capability", + return_value=DeviceCapability(9, 0), + ), + patch( + "vllm.v1.attention.backends.fa_utils.is_fa_version_supported", + return_value=True, + ), + ): + yield + + +@blackwell_only +@pytest.mark.parametrize("use_mm_prefix", [False, True]) +@pytest.mark.parametrize("flash_attn_version", [None, 3, 4]) +def test_hopper_mm_prefix_selects_triton_flash_attn( + use_mm_prefix, flash_attn_version, hopper_selection +): + """Hopper selects the composite only when its causal route resolves FA4.""" + from vllm.engine.arg_utils import EngineArgs + + config = EngineArgs( + model="google/gemma-4-31B-it", + dtype="bfloat16", + attention_config=AttentionConfig(flash_attn_version=flash_attn_version), + ).create_engine_config() + with set_current_vllm_config(config): + backend = get_attn_backend( + 256, torch.bfloat16, None, use_mm_prefix=use_mm_prefix + ) + fa4_resolved = config.attention_config.flash_attn_version == 4 + if use_mm_prefix: + expected = "TRITON_FLASH_ATTN" if fa4_resolved else "TRITON_ATTN" + else: + expected = "FLASH_ATTN" + assert backend.get_name() == expected + + +@blackwell_only +@pytest.mark.parametrize("use_mm_prefix", [False, True]) +@pytest.mark.parametrize("kv_cache_dtype", [None, "fp8_e4m3"]) +def test_mm_prefix_selects_composite_without_changing_causal_default( + use_mm_prefix, kv_cache_dtype, blackwell_selection +): + """The image-mask requirement must reach CUDA's automatic backend priority.""" + from vllm.engine.arg_utils import EngineArgs + + config = EngineArgs( + model="google/gemma-4-31B-it", dtype="bfloat16" + ).create_engine_config() + with set_current_vllm_config(config): + backend = get_attn_backend( + 256, torch.bfloat16, kv_cache_dtype, use_mm_prefix=use_mm_prefix + ) + expected = "TRITON_FLASHINFER" if use_mm_prefix else "FLASHINFER" + assert backend.get_name() == expected + + +@blackwell_only +@pytest.mark.parametrize( + "feature_kwargs,reason", + [ + ({"use_rswa": True}, "R-SWA not supported"), + ({"kv_cache_dtype": "int8_per_token_head"}, "kv_cache_dtype not supported"), + ({"block_size": 16}, "block_size not supported"), + ({"use_dcp": True}, "DCP not supported"), + ({"use_pcp": True}, "PCP not supported"), + ({"use_adaptive_verification": True}, "device-cpu query lens mismatch"), + ], +) +def test_composite_rejects_features_not_shared_by_both_routes(feature_kwargs, reason): + """Automatic selection must not admit features only one child can execute.""" + from vllm.engine.arg_utils import EngineArgs + from vllm.v1.attention.backends.triton_flashinfer import TritonFlashInferBackend + + config = EngineArgs( + model="google/gemma-4-31B-it", dtype="bfloat16" + ).create_engine_config() + args = dict( + head_size=256, + dtype=torch.bfloat16, + kv_cache_dtype="auto", + block_size=128, + use_mla=False, + has_sink=False, + use_sparse=False, + use_mm_prefix=True, + use_per_head_quant_scales=False, + device_capability=DeviceCapability(10, 0), + attn_type=AttentionType.DECODER, + ) + args.update(feature_kwargs) + with set_current_vllm_config(config): + reasons = TritonFlashInferBackend.validate_configuration(**args) + assert any(reason in message for message in reasons), reasons + + +@blackwell_only +def test_rswa_selection_does_not_reuse_causal_result(blackwell_selection): + """An R-SWA requirement must invalidate a previously cached causal selection.""" + from vllm.engine.arg_utils import EngineArgs + + config = EngineArgs( + model="google/gemma-4-31B-it", + dtype="bfloat16", + attention_config={"backend": "TRITON_FLASHINFER"}, + ).create_engine_config() + with set_current_vllm_config(config): + assert ( + get_attn_backend(256, torch.bfloat16, None).get_name() + == "TRITON_FLASHINFER" + ) + config.model_config.model_arch_config.rswa_window = 128 + with pytest.raises(ValueError, match="R-SWA"): + get_attn_backend(256, torch.bfloat16, None) + config.attention_config.backend = AttentionBackendEnum.TRITON_ATTN + assert get_attn_backend(256, torch.bfloat16, None).get_name() == "TRITON_ATTN" diff --git a/tests/v1/attention/test_mm_prefix.py b/tests/v1/attention/test_mm_prefix.py index 6b8717389f..f878ede486 100644 --- a/tests/v1/attention/test_mm_prefix.py +++ b/tests/v1/attention/test_mm_prefix.py @@ -638,3 +638,249 @@ def test_mm_prefix_kv_cache_path(head_size: int): assert not torch.allclose(expected, causal_only, atol=2e-2, rtol=2e-2), ( "test batch does not actually exercise the mm_prefix branch" ) + + +@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA backend imports") +@pytest.mark.parametrize( + "query_len,seq_len,spans,expected,unclamped_expected", + [ + (16, 32, [(20, 28)], True, True), + (16, 32, [(0, 16)], False, True), + (16, 32, [(0, 17)], True, True), + (16, 32, [(32, 40)], False, False), + (16, 32, [(31, 40)], False, True), + (1, 24, [(20, 28)], False, True), + (0, 24, [(20, 28)], False, False), + (16, 32, [(20, 20)], False, False), + (16, 64, [(20, 28)], False, False), + ], +) +def test_composite_routes_queries_that_need_image_masking( + query_len, seq_len, spans, expected, unclamped_expected +): + """Historical images and single-query steps must use the causal graph path.""" + from types import SimpleNamespace + + from vllm.v1.attention.backends.composite import requires_mm_prefix + + common = SimpleNamespace( + max_query_len=query_len, + mm_req_doc_ranges={0: spans}, + query_start_loc_cpu=torch.tensor([0, query_len]), + seq_lens_cpu_upper_bound=torch.tensor([seq_len]), + ) + assert requires_mm_prefix(common) is expected + assert requires_mm_prefix(common, unclamped_window=True) is unclamped_expected + + +@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA attention kernels") +@pytest.mark.parametrize("head_size", [256, 512]) +@pytest.mark.parametrize( + "reverse_children,cache_dtype", + [(False, "auto"), (True, "auto"), (False, "fp8_e4m3")], +) +@pytest.mark.parametrize("causal_backend_name", ["FLASHINFER", "FLASH_ATTN"]) +def test_composite_shared_cache_across_image_and_causal_steps( + head_size, reverse_children, cache_dtype, causal_backend_name +): + """Changing routes must preserve KV writes, image masking and graph replay.""" + if causal_backend_name == "FLASHINFER": + if not current_platform.is_device_capability_family(100): + pytest.skip("The Triton/FlashInfer composite supports Blackwell") + else: + if not current_platform.is_device_capability(90): + pytest.skip("The Triton/FlashAttention composite supports Hopper") + if reverse_children or cache_dtype != "auto": + pytest.skip("Reverse-child and FP8 coverage is backend-independent") + + from vllm.config import set_current_vllm_config + from vllm.engine.arg_utils import EngineArgs + from vllm.model_executor.layers.attention import Attention + from vllm.utils.torch_utils import set_default_torch_dtype + from vllm.v1.attention.backend import AttentionCGSupport, CommonAttentionMetadata + from vllm.v1.attention.backends.composite import ( + MMPrefixAttentionRouting, + create_composite_attention_backend, + ) + from vllm.v1.attention.backends.flash_attn import ( + FlashAttentionMetadata, + ) + from vllm.v1.attention.backends.flashinfer import ( + FlashInferBackend, + FlashInferMetadata, + FlashInferTrtllmAPIDecode, + TRTLLMPrefill, + ) + from vllm.v1.attention.backends.triton_attn import ( + TritonAttentionBackend, + TritonAttentionMetadata, + ) + from vllm.v1.attention.backends.triton_flash_attn import ( + TritonFlashAttentionBackend, + ) + from vllm.v1.attention.backends.triton_flashinfer import ( + TritonFlashInferBackend, + ) + from vllm.v1.kv_cache_interface import FullAttentionSpec, SlidingWindowSpec + + if causal_backend_name == "FLASHINFER": + backend = TritonFlashInferBackend + causal_metadata_type = FlashInferMetadata + block_size = backend.get_supported_kernel_block_sizes()[0] + else: + backend = TritonFlashAttentionBackend + causal_metadata_type = FlashAttentionMetadata + block_size = backend.get_preferred_block_size(16) + num_blocks = (209 + block_size - 1) // block_size + torch.manual_seed(42) + attention_config = {"backend": backend.get_name()} + if causal_backend_name == "FLASH_ATTN": + attention_config["flash_attn_version"] = 4 + cfg = EngineArgs( + model="google/gemma-4-31B-it", + dtype="bfloat16", + max_model_len=512, + max_num_seqs=4, + max_num_batched_tokens=512, + block_size=block_size, + kv_cache_dtype=cache_dtype, + enforce_eager=True, + attention_config=attention_config, + ).create_engine_config() + cfg.cache_config.kv_cache_layout = "LBHNC" + window = 128 if head_size == 256 else None + storage_dtype = torch.uint8 if cache_dtype == "fp8_e4m3" else DTYPE + spec_kwargs = dict( + block_size=block_size, + num_kv_heads=16, + head_size=head_size, + dtype=storage_dtype, + kv_quant_mode=get_kv_quant_mode(cache_dtype), + ) + spec = ( + SlidingWindowSpec(**spec_kwargs, sliding_window=window) + if window + else FullAttentionSpec(**spec_kwargs) + ) + if reverse_children: + # Swapping the children and policy must preserve outputs and KV writes; + # dispatch cannot depend on a particular child's class or position. + class ReversedRouting(MMPrefixAttentionRouting): + capture_variant = 0 + + def select(self, metadata): + return 1 - super().select(metadata) + + @staticmethod + def variant_uses_mm_prefix(variant): + return variant == 1 + + backend = create_composite_attention_backend( + FlashInferBackend, + TritonAttentionBackend, + name="ReversedCompositeBackend", + backend_name="CUSTOM", + module=__name__, + routing_policy=ReversedRouting, + ) + with set_current_vllm_config(cfg), set_default_torch_dtype(DTYPE): + layer = Attention( + 32, + head_size, + head_size**-0.5, + num_kv_heads=16, + cache_config=cfg.cache_config, + prefix="composite_test", + per_layer_sliding_window=window, + attn_backend=backend, + ).to(DEVICE) + layer.mm_prefix_clamp_sliding_window = window is not None + assert backend.get_builder_cls().get_cudagraph_support(cfg, spec) == ( + AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE + ) + if window: + layer.mm_prefix_clamp_sliding_window = False + assert backend.get_builder_cls().get_cudagraph_support(cfg, spec) == ( + AttentionCGSupport.NEVER + ) + layer.mm_prefix_clamp_sliding_window = True + builder = backend.get_builder_cls()(spec, ["composite_test"], cfg, DEVICE) + builder.set_kernel_block_size(block_size) + # Match production, where unwritten page tails are zero-initialized. + cache = torch.zeros( + num_blocks, + 16, + block_size, + 2 * head_size, + dtype=storage_dtype, + device=DEVICE, + ) + keys = torch.randn(209, 16, head_size, dtype=DTYPE, device=DEVICE) + values = torch.randn_like(keys) + if cache_dtype == "fp8_e4m3": + keys = keys.to(torch.float8_e4m3fn).to(DTYPE) + values = values.to(torch.float8_e4m3fn).to(DTYPE) + table = torch.arange(num_blocks, dtype=torch.int32, device=DEVICE).unsqueeze(0) + previous = 0 + for end, spans, metadata_type in [ + (128, [(32, 111)], TritonAttentionMetadata), + (129, [(32, 111)], causal_metadata_type), + (145, [(32, 111)], causal_metadata_type), + (209, [(32, 111), (160, 207)], TritonAttentionMetadata), + ]: + qlen = end - previous + starts = torch.tensor([0, qlen], dtype=torch.int32) + lengths = torch.tensor([end], dtype=torch.int32) + common = CommonAttentionMetadata( + query_start_loc=starts.to(DEVICE), + query_start_loc_cpu=starts, + seq_lens=lengths.to(DEVICE), + seq_lens_cpu_upper_bound=lengths, + num_reqs=1, + num_actual_tokens=qlen, + max_query_len=qlen, + max_seq_len=end, + block_table_tensor=table, + slot_mapping=torch.arange(previous, end, device=DEVICE), + mm_req_doc_ranges={0: spans}, + ) + metadata = builder.build(0, common) + assert isinstance(metadata, metadata_type) + if isinstance(metadata, FlashInferMetadata): + if qlen == 1: + assert isinstance(metadata.decode, FlashInferTrtllmAPIDecode) + else: + assert isinstance(metadata.prefill, TRTLLMPrefill) + query = torch.randn(qlen, 32, head_size, dtype=DTYPE, device=DEVICE) + if cache_dtype == "fp8_e4m3": + query = query.to(torch.float8_e4m3fn).to(DTYPE) + key, value = keys[previous:end], values[previous:end] + output = torch.empty_like(query) + layer.impl.do_kv_cache_update(layer, key, value, cache, common.slot_mapping) + layer.impl.forward(layer, query, key, value, cache, metadata, output) + expected = _dense_reference( + query, + [keys[:end]], + [values[:end]], + [qlen], + [end], + {0: spans}, + window, + head_size**-0.5, + mm_clamp_sw=window or 0, + ) + torch.testing.assert_close(output.float(), expected, atol=2e-2, rtol=2e-2) + if qlen == 1: + metadata = builder.build_for_cudagraph_capture(common) + assert isinstance(metadata, metadata_type) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + layer.impl.forward( + layer, query, key, value, cache, metadata, output + ) + output.zero_() + graph.replay() + torch.testing.assert_close( + output.float(), expected, atol=2e-2, rtol=2e-2 + ) + previous = end diff --git a/tests/v1/spec_decode/test_dflash2.py b/tests/v1/spec_decode/test_dflash2.py index 0bff2d537e..e7ff2b6d5d 100644 --- a/tests/v1/spec_decode/test_dflash2.py +++ b/tests/v1/spec_decode/test_dflash2.py @@ -159,6 +159,7 @@ def test_dflash2_model_decoder_layer_cls(monkeypatch): model_config=SimpleNamespace( dtype=torch.float32, is_mm_prefix_lm=False, + rswa_window=None, ), kernel_config=SimpleNamespace( linear_backend="auto", diff --git a/vllm/platforms/cuda.py b/vllm/platforms/cuda.py index 1e5a5a3ad0..6d9444567a 100644 --- a/vllm/platforms/cuda.py +++ b/vllm/platforms/cuda.py @@ -87,6 +87,7 @@ def _get_backend_priorities( kv_cache_dtype: CacheDType | None = None, use_non_causal: bool = False, head_size: int | None = None, + use_mm_prefix: bool = False, ) -> list[AttentionBackendEnum]: """Get backend priorities with lazy import to avoid circular dependency.""" from vllm.utils.torch_utils import is_quantized_kv_cache @@ -156,6 +157,7 @@ def _get_backend_priorities( # So prefer FlashAttention when non-causal on SM100f. if device_capability.major == 10 and not use_non_causal: return [ + *([AttentionBackendEnum.TRITON_FLASHINFER] if use_mm_prefix else []), AttentionBackendEnum.FLASHINFER, AttentionBackendEnum.FLASH_ATTN, AttentionBackendEnum.TRITON_ATTN, @@ -164,6 +166,11 @@ def _get_backend_priorities( ] else: return [ + *( + [AttentionBackendEnum.TRITON_FLASH_ATTN] + if device_capability.major == 9 and use_mm_prefix + else [] + ), AttentionBackendEnum.FLASH_ATTN, AttentionBackendEnum.FLASHINFER, AttentionBackendEnum.TRITON_ATTN, @@ -394,6 +401,7 @@ class CudaPlatformBase(Platform): kv_cache_dtype=attn_selector_config.kv_cache_dtype, use_non_causal=attn_selector_config.use_non_causal, head_size=attn_selector_config.head_size, + use_mm_prefix=attn_selector_config.use_mm_prefix, ) for priority, backend in enumerate(backend_priorities): try: diff --git a/vllm/v1/attention/backend.py b/vllm/v1/attention/backend.py index c427d3a1de..01599e9e9c 100644 --- a/vllm/v1/attention/backend.py +++ b/vllm/v1/attention/backend.py @@ -185,6 +185,10 @@ class AttentionBackend(ABC): def supports_sliding_window(cls) -> bool: return False + @classmethod + def supports_rswa(cls) -> bool: + return False + @classmethod def supports_non_causal(cls) -> bool: """Check if backend supports non-causal (bidirectional) attention @@ -287,6 +291,7 @@ class AttentionBackend(ABC): use_pcp: bool = False, use_adaptive_verification: bool = False, use_dcp: bool = False, + use_rswa: bool = False, ) -> list[str]: invalid_reasons = [] if not cls.supports_head_size(head_size): @@ -321,6 +326,8 @@ class AttentionBackend(ABC): invalid_reasons.append(f"attention type {attn_type} not supported") if has_sliding_window and not cls.supports_sliding_window(): invalid_reasons.append("sliding window not supported") + if use_rswa and not cls.supports_rswa(): + invalid_reasons.append("R-SWA not supported") if use_non_causal and not cls.supports_non_causal(): invalid_reasons.append("non-causal attention not supported") if use_mla and use_non_causal and use_dcp and not cls.supports_non_causal_dcp(): @@ -854,6 +861,9 @@ class AttentionImplBase(ABC, Generic[T]): def process_weights_after_loading(self, act_dtype: torch.dtype): pass + def get_impl_variants(self) -> tuple["AttentionImplBase", ...]: + return (self,) + def prepare_for_batch(self, attn_metadata: T | None) -> None: """Prepare implementation-specific state for the current batch.""" diff --git a/vllm/v1/attention/backends/composite.py b/vllm/v1/attention/backends/composite.py new file mode 100644 index 0000000000..d091a29d87 --- /dev/null +++ b/vllm/v1/attention/backends/composite.py @@ -0,0 +1,509 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Reusable attention backend composition and routing policies.""" + +import functools +import math +from typing import ClassVar, Protocol, cast + +import torch + +from vllm.config import VllmConfig +from vllm.v1.attention.backend import ( + AttentionBackend, + AttentionCGSupport, + AttentionImpl, + AttentionMetadataBuilder, + CommonAttentionMetadata, +) +from vllm.v1.attention.backends.utils import get_supported_kv_cache_layouts +from vllm.v1.kv_cache_interface import KVCacheSpec + + +def requires_mm_prefix( + metadata: CommonAttentionMetadata, *, unclamped_window: bool = False +) -> bool: + ranges = metadata.mm_req_doc_ranges + # With one query per request there are no future keys to unmask. This + # also keeps single-token prefills on the captured FlashInfer path. + if not ranges or (metadata.max_query_len <= 1 and not unclamped_window): + return False + seq_lens = metadata.seq_lens_cpu_upper_bound + assert seq_lens is not None + starts = metadata.query_start_loc_cpu + for req_idx, spans in ranges.items(): + query_len = int(starts[req_idx + 1] - starts[req_idx]) + if query_len == 0 or (query_len == 1 and not unclamped_window): + continue + end = int(seq_lens[req_idx]) + begin = end - query_len + # An unclamped image can also expose keys behind the sliding window. + if unclamped_window and any( + start < stop and start < end and stop >= begin for start, stop in spans + ): + return True + # Prefill lengths are exact. Historical image ranges must not divert + # later text/decode queries to the image backend. + if any( + start < stop and start < end - 1 and stop > begin for start, stop in spans + ): + return True + return False + + +def _has_unclamped_window(layers) -> bool: + return any( + getattr(layer, "sliding_window", None) is not None + and not getattr(layer, "mm_prefix_clamp_sliding_window", False) + for layer in layers + ) + + +class CompositeAttentionRouting(Protocol): + """Select a child backend and declare routing constraints for composition.""" + + capture_variant: ClassVar[int] + """Child used during graph capture; replay must select the same child.""" + + def __init__(self, layer_names: list[str], vllm_config: VllmConfig) -> None: ... + + def select(self, metadata: CommonAttentionMetadata) -> int: + """Return 0 for the first child or 1 for the second child.""" + ... + + @staticmethod + def get_cudagraph_support( + vllm_config: VllmConfig, kv_cache_spec: KVCacheSpec + ) -> AttentionCGSupport: + """Limit graph support to batches whose routing stays invariant.""" + ... + + @staticmethod + def variant_uses_mm_prefix(variant: int) -> bool: + """Whether this child must support the model's image-mask requirement.""" + ... + + +class MMPrefixAttentionRouting(CompositeAttentionRouting): + """Route image masks to variant zero and causal queries to variant one.""" + + def __init__(self, layer_names: list[str], vllm_config: VllmConfig) -> None: + layers = vllm_config.compilation_config.static_forward_context + self.unclamped_window = _has_unclamped_window( + layers[name] for name in layer_names + ) + + def select(self, metadata: CommonAttentionMetadata) -> int: + return int( + not requires_mm_prefix(metadata, unclamped_window=self.unclamped_window) + ) + + capture_variant: ClassVar[int] = 1 + + @staticmethod + def get_cudagraph_support( + vllm_config: VllmConfig, kv_cache_spec: KVCacheSpec + ) -> AttentionCGSupport: + if _has_unclamped_window( + vllm_config.compilation_config.static_forward_context.values() + ): + return AttentionCGSupport.NEVER + return AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE + + @staticmethod + def variant_uses_mm_prefix(variant: int) -> bool: + return variant == 0 + + +def _intersect_kernel_block_sizes( + first: type[AttentionBackend], second: type[AttentionBackend] +): + from vllm.v1.attention.backend import MultipleOf + + result: list[int | MultipleOf] = [] + seen: set[tuple[type, int]] = set() + for lhs in first.get_supported_kernel_block_sizes(): + for rhs in second.get_supported_kernel_block_sizes(): + candidate: int | MultipleOf | None = None + if isinstance(lhs, MultipleOf) and isinstance(rhs, MultipleOf): + candidate = MultipleOf(math.lcm(lhs.base, rhs.base)) + elif isinstance(lhs, MultipleOf) and isinstance(rhs, int): + candidate = rhs if rhs % lhs.base == 0 else None + elif isinstance(lhs, int) and isinstance(rhs, MultipleOf): + candidate = lhs if lhs % rhs.base == 0 else None + elif lhs == rhs: + candidate = lhs + if candidate is None: + continue + value = candidate.base if isinstance(candidate, MultipleOf) else candidate + key = (type(candidate), value) + if key not in seen: + result.append(candidate) + seen.add(key) + return result + + +@functools.cache +def create_composite_attention_backend( + general_backend: type[AttentionBackend], + causal_backend: type[AttentionBackend], + *, + name: str, + backend_name: str, + module: str, + routing_policy: type[CompositeAttentionRouting], + head_sizes: tuple[int, ...] = (), + kernel_block_sizes: tuple[int, ...] = (), + device_major: int | None = None, +) -> type[AttentionBackend]: + """Compose two backends with shared storage and an explicit routing policy.""" + if general_backend.full_cls_name() == causal_backend.full_cls_name(): + return general_backend + general_impl_cls = general_backend.get_impl_cls() + causal_impl_cls = causal_backend.get_impl_cls() + general_builder_cls = general_backend.get_builder_cls() + causal_builder_cls = causal_backend.get_builder_cls() + if ( + general_backend.forward_includes_kv_cache_update + or causal_backend.forward_includes_kv_cache_update + ): + raise ValueError("Composite backends require separate KV cache updates") + + class CompositeAttentionImpl(AttentionImpl): + def __init__(self, *args, **kwargs) -> None: + self.general_impl = cast(AttentionImpl, general_impl_cls(*args, **kwargs)) + try: + self.causal_impl = cast(AttentionImpl, causal_impl_cls(*args, **kwargs)) + except TypeError as e: + raise ValueError( + f"Causal attention backend {causal_backend.get_name()} " + f"does not accept the attention arguments required by " + f"the general backend {general_backend.get_name()}: {e}" + ) from e + for name in ( + "num_heads", + "num_kv_heads", + "head_size", + "scale", + "kv_cache_dtype", + ): + if hasattr(self.general_impl, name): + setattr(self, name, getattr(self.general_impl, name)) + self.supports_quant_query_input = ( + self.general_impl.supports_quant_query_input + and self.causal_impl.supports_quant_query_input + ) + self.supports_pcp = self.general_impl.supports_pcp + self.supports_dcp = self.causal_impl.supports_dcp + self.can_return_lse_for_decode = self.causal_impl.can_return_lse_for_decode + self.lse_base_on_e = self.causal_impl.lse_base_on_e + self.need_to_return_lse_for_decode = ( + self.causal_impl.need_to_return_lse_for_decode + ) + self.supports_mtp_with_cp_non_trivial_interleave_size = ( + self.causal_impl.supports_mtp_with_cp_non_trivial_interleave_size + ) + + def get_impl_for_metadata(self, attn_metadata): + return ( + self.causal_impl + if getattr(attn_metadata, "_attention_backend_variant", 0) + else self.general_impl + ) + + def get_impl_variants(self): + return self.general_impl, self.causal_impl + + def forward( + self, + layer, + query, + key, + value, + kv_cache, + attn_metadata, + output, + output_scale=None, + output_block_scale=None, + ): + impl = self.get_impl_for_metadata(attn_metadata) + return impl.forward( + layer, + query, + key, + value, + kv_cache, + attn_metadata, + output, + output_scale, + output_block_scale, + ) + + def process_weights_after_loading(self, act_dtype: torch.dtype): + for impl in self.get_impl_variants(): + impl.process_weights_after_loading(act_dtype) + + def fused_output_quant_supported(self, quant_key): + return all( + impl.fused_output_quant_supported(quant_key) + for impl in self.get_impl_variants() + ) + + def fused_qk_norm_rope_kvcache_supported(self): + return self.general_impl.fused_qk_norm_rope_kvcache_supported() + + def do_qk_norm_rope_kvcache_update(self, *args, **kwargs): + method = self.general_impl.do_qk_norm_rope_kvcache_update + return method(*args, **kwargs) + + def fused_rope_kvcache_supported(self): + return self.general_impl.fused_rope_kvcache_supported() + + def do_rope_and_kv_cache_update(self, *args, **kwargs): + method = self.general_impl.do_rope_and_kv_cache_update + return method(*args, **kwargs) + + def do_kv_cache_update(self, *args, **kwargs): + method = self.general_impl.do_kv_cache_update # type: ignore[attr-defined] + return method(*args, **kwargs) + + class CompositeAttentionMetadataBuilder(AttentionMetadataBuilder): + requires_block_table_width = ( + general_builder_cls.requires_block_table_width + or causal_builder_cls.requires_block_table_width + ) + + def __init__(self, kv_cache_spec, layer_names, vllm_config, device, **kwargs): + super().__init__(kv_cache_spec, layer_names, vllm_config, device) + args = (kv_cache_spec, layer_names, vllm_config, device) + # Only the variant that asked for the extra ctor kwargs receives them. + self.general_builder = general_builder_cls( + *args, + **(kwargs if general_builder_cls.requires_block_table_width else {}), + ) + self.causal_builder = causal_builder_cls( + *args, + **(kwargs if causal_builder_cls.requires_block_table_width else {}), + ) + self.reorder_batch_threshold = min( + ( + builder.reorder_batch_threshold + for builder in self._builders + if builder.reorder_batch_threshold is not None + ), + default=None, + ) + self.routing = routing_policy(layer_names, vllm_config) + + def set_kernel_block_size(self, kernel_block_size): + super().set_kernel_block_size(kernel_block_size) + for builder in self._builders: + builder.set_kernel_block_size(kernel_block_size) + + @classmethod + def get_cudagraph_support(cls, vllm_config, kv_cache_spec): + return min( + general_builder_cls.get_cudagraph_support(vllm_config, kv_cache_spec), + causal_builder_cls.get_cudagraph_support(vllm_config, kv_cache_spec), + routing_policy.get_cudagraph_support(vllm_config, kv_cache_spec), + key=lambda support: support.value, + ) + + def _builder(self, common_attn_metadata): + backend_variant = self.routing.select(common_attn_metadata) + builder = self.causal_builder if backend_variant else self.general_builder + return builder, backend_variant + + def _build(self, method, common_attn_metadata, **kwargs): + builder, backend_variant = self._builder(common_attn_metadata) + metadata = getattr(builder, method)( + common_attn_metadata=common_attn_metadata, **kwargs + ) + metadata._attention_backend_variant = backend_variant + return metadata + + def build(self, common_prefix_len, common_attn_metadata, fast_build=False): + return self._build( + "build", + common_attn_metadata, + common_prefix_len=common_prefix_len, + fast_build=fast_build, + ) + + def build_for_cudagraph_capture(self, common_attn_metadata): + variant = routing_policy.capture_variant + metadata = self._builders[variant].build_for_cudagraph_capture( + common_attn_metadata + ) + metadata._attention_backend_variant = variant + return metadata + + @property + def _builders(self): + return self.general_builder, self.causal_builder + + # Forward the optional workspace protocol so that a wrapped builder + # which allocates one (FlashInfer) still joins the runner's cross-group + # sharing instead of allocating a second buffer behind the composite. + def _get_workspace_buffer(self): + for builder in self._builders: + if hasattr(builder, "_get_workspace_buffer"): + workspace = builder._get_workspace_buffer() + # The runner only calls the getter for the group that + # provides the buffer, so hand it to the sibling here. + self.set_workspace_buffer(workspace) + return workspace + return None + + def set_workspace_buffer(self, workspace_buffer): + for builder in self._builders: + if hasattr(builder, "set_workspace_buffer"): + builder.set_workspace_buffer(workspace_buffer) + + class CompositeAttentionBackend(AttentionBackend): + forward_includes_kv_cache_update = False + general_backend_cls = general_backend + causal_backend_cls = causal_backend + + @staticmethod + def get_name(): + return backend_name + + @staticmethod + def get_impl_cls(): + return CompositeAttentionImpl + + @staticmethod + def get_builder_cls(): + return CompositeAttentionMetadataBuilder + + @staticmethod + def get_supported_kernel_block_sizes(): + if kernel_block_sizes: + return [ + size + for size in kernel_block_sizes + if all( + backend.supports_block_size(size) + for backend in (general_backend, causal_backend) + ) + ] + return _intersect_kernel_block_sizes(general_backend, causal_backend) + + @classmethod + def supports_block_size(cls, block_size): + return bool( + cls.get_supported_kernel_block_sizes() + ) and super().supports_block_size(block_size) + + @classmethod + def get_supported_head_sizes(cls): + return list(head_sizes) + + @classmethod + def supports_head_size(cls, head_size): + return (not head_sizes or head_size in head_sizes) and all( + backend.supports_head_size(head_size) + for backend in (general_backend, causal_backend) + ) + + @classmethod + def supports_compute_capability(cls, capability): + return (device_major is None or capability.major == device_major) and all( + backend.supports_compute_capability(capability) + for backend in (general_backend, causal_backend) + ) + + @classmethod + def supports_dtype(cls, dtype): + return all( + backend.supports_dtype(dtype) + for backend in (general_backend, causal_backend) + ) + + @classmethod + def supports_kv_cache_dtype(cls, dtype): + return all( + backend.supports_kv_cache_dtype(dtype) + for backend in (general_backend, causal_backend) + ) + + @classmethod + def supports_mm_prefix(cls): + return all( + not routing_policy.variant_uses_mm_prefix(variant) + or backend.supports_mm_prefix() + for variant, backend in enumerate((general_backend, causal_backend)) + ) + + @classmethod + def supports_sliding_window(cls): + return all( + backend.supports_sliding_window() + for backend in (general_backend, causal_backend) + ) + + @classmethod + def supports_rswa(cls): + return all( + backend.supports_rswa() for backend in (general_backend, causal_backend) + ) + + @classmethod + def supports_pcp(cls): + return False + + @classmethod + def supports_dcp(cls): + return False + + @classmethod + def supports_device_cpu_query_lens_mismatch(cls): + return False + + @classmethod + def supports_combination( + cls, + head_size, + dtype, + kv_cache_dtype, + block_size, + use_mla, + has_sink, + use_sparse, + use_mm_prefix, + device_capability, + ): + for variant, backend in enumerate((general_backend, causal_backend)): + reason = backend.supports_combination( + head_size, + dtype, + kv_cache_dtype, + block_size, + use_mla, + has_sink, + use_sparse, + use_mm_prefix and routing_policy.variant_uses_mm_prefix(variant), + device_capability, + ) + if reason: + return f"{backend.get_name()}: {reason}" + return None + + @classmethod + def customize_spec(cls, spec): + general_spec = general_backend.customize_spec(spec) + if general_spec != causal_backend.customize_spec(spec): + raise ValueError("Composite backends require the same KV cache spec") + return general_spec + + @classmethod + def supported_kv_cache_layouts(cls): + return tuple( + get_supported_kv_cache_layouts((general_backend, causal_backend)) + ) + + CompositeAttentionBackend.__name__ = name + CompositeAttentionBackend.__qualname__ = name + CompositeAttentionBackend.__module__ = module + return CompositeAttentionBackend diff --git a/vllm/v1/attention/backends/flash_attn.py b/vllm/v1/attention/backends/flash_attn.py index 9ff543024f..9109a9c6d7 100755 --- a/vllm/v1/attention/backends/flash_attn.py +++ b/vllm/v1/attention/backends/flash_attn.py @@ -129,6 +129,10 @@ class FlashAttentionBackend(AttentionBackend): def get_name() -> str: return "FLASH_ATTN" + @classmethod + def supports_rswa(cls) -> bool: + return True + @classmethod def supports_sliding_window(cls) -> bool: return True diff --git a/vllm/v1/attention/backends/flex_attention.py b/vllm/v1/attention/backends/flex_attention.py index 7529d8d20f..70eded1d09 100644 --- a/vllm/v1/attention/backends/flex_attention.py +++ b/vllm/v1/attention/backends/flex_attention.py @@ -105,6 +105,10 @@ class FlexAttentionBackend(AttentionBackend): def get_name() -> str: return "FLEX_ATTENTION" + @classmethod + def supports_rswa(cls) -> bool: + return True + @classmethod def supports_sliding_window(cls) -> bool: return True diff --git a/vllm/v1/attention/backends/registry.py b/vllm/v1/attention/backends/registry.py index c1c6330576..085f2358fa 100644 --- a/vllm/v1/attention/backends/registry.py +++ b/vllm/v1/attention/backends/registry.py @@ -49,6 +49,9 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta): TRITON_ATTN_DIFFKV = ( "vllm.v1.attention.backends.triton_attn_diffkv.TritonAttentionDiffKVBackend" ) + TRITON_FLASH_ATTN = ( + "vllm.v1.attention.backends.triton_flash_attn.TritonFlashAttentionBackend" + ) ROCM_ATTN = "vllm.v1.attention.backends.rocm_attn.RocmAttentionBackend" ROCM_AITER_MLA = "vllm.v1.attention.backends.mla.rocm_aiter_mla.AiterMLABackend" ROCM_AITER_TRITON_MLA = ( @@ -63,6 +66,9 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta): XPU_MLA_SPARSE = "vllm.v1.attention.backends.mla.xpu_mla_sparse.XPUMLASparseBackend" TORCH_SDPA = "" # this tag is only used for ViT FLASHINFER = "vllm.v1.attention.backends.flashinfer.FlashInferBackend" + TRITON_FLASHINFER = ( + "vllm.v1.attention.backends.triton_flashinfer.TritonFlashInferBackend" + ) FLASHINFER_MLA = ( "vllm.v1.attention.backends.mla.flashinfer_mla.FlashInferMLABackend" ) diff --git a/vllm/v1/attention/backends/triton_attn.py b/vllm/v1/attention/backends/triton_attn.py index 48dd30d666..b7e2a41d00 100644 --- a/vllm/v1/attention/backends/triton_attn.py +++ b/vllm/v1/attention/backends/triton_attn.py @@ -323,6 +323,10 @@ class TritonAttentionBackend(AttentionBackend): def get_name() -> str: return "TRITON_ATTN" + @classmethod + def supports_rswa(cls) -> bool: + return True + @classmethod def supports_sliding_window(cls) -> bool: return True diff --git a/vllm/v1/attention/backends/triton_flash_attn.py b/vllm/v1/attention/backends/triton_flash_attn.py new file mode 100644 index 0000000000..0853b291cd --- /dev/null +++ b/vllm/v1/attention/backends/triton_flash_attn.py @@ -0,0 +1,67 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Triton image-mask and FlashAttention causal attention composite.""" + +import torch + +from vllm.config.cache import CacheDType +from vllm.platforms.interface import DeviceCapability +from vllm.v1.attention.backends.composite import ( + MMPrefixAttentionRouting, + create_composite_attention_backend, +) +from vllm.v1.attention.backends.fa_utils import get_flash_attn_version +from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend +from vllm.v1.attention.backends.triton_attn import TritonAttentionBackend + + +class _FA4Backend(FlashAttentionBackend): + @classmethod + def supports_combination( + cls, + head_size: int, + dtype: torch.dtype, + kv_cache_dtype: CacheDType | None, + block_size: int | None, + use_mla: bool, + has_sink: bool, + use_sparse: bool, + use_mm_prefix: bool, + device_capability: DeviceCapability, + ) -> str | None: + reason = super().supports_combination( + head_size, + dtype, + kv_cache_dtype, + block_size, + use_mla, + has_sink, + use_sparse, + use_mm_prefix, + device_capability, + ) + if reason is not None: + return reason + if ( + get_flash_attn_version( + head_size=head_size, + has_sinks=has_sink, + kv_cache_block_size=block_size, + supports_fa4_hd256=True, + ) + != 4 + ): + return "causal route requires FlashAttention v4" + return None + + +TritonFlashAttentionBackend = create_composite_attention_backend( + TritonAttentionBackend, + _FA4Backend, + name="TritonFlashAttentionBackend", + backend_name="TRITON_FLASH_ATTN", + module=__name__, + routing_policy=MMPrefixAttentionRouting, + head_sizes=(256, 512), + device_major=9, +) diff --git a/vllm/v1/attention/backends/triton_flashinfer.py b/vllm/v1/attention/backends/triton_flashinfer.py new file mode 100644 index 0000000000..0791227591 --- /dev/null +++ b/vllm/v1/attention/backends/triton_flashinfer.py @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Triton image-mask and FlashInfer causal attention composite.""" + +from vllm.v1.attention.backends.composite import ( + MMPrefixAttentionRouting, + create_composite_attention_backend, +) +from vllm.v1.attention.backends.flashinfer import FlashInferBackend +from vllm.v1.attention.backends.triton_attn import TritonAttentionBackend + +TritonFlashInferBackend = create_composite_attention_backend( + TritonAttentionBackend, + FlashInferBackend, + name="TritonFlashInferBackend", + backend_name="TRITON_FLASHINFER", + module=__name__, + routing_policy=MMPrefixAttentionRouting, + head_sizes=(256, 512), + kernel_block_sizes=(64,), + device_major=10, +) diff --git a/vllm/v1/attention/backends/utils.py b/vllm/v1/attention/backends/utils.py index bc127ccb8b..a3adf70d25 100644 --- a/vllm/v1/attention/backends/utils.py +++ b/vllm/v1/attention/backends/utils.py @@ -8,6 +8,7 @@ from typing import ( TYPE_CHECKING, Any, Protocol, + TypeVar, cast, ) @@ -45,6 +46,19 @@ logger = init_logger(__name__) PAD_SLOT_ID = -1 NULL_BLOCK_ID = 0 + +ImplT = TypeVar("ImplT", bound=AttentionImpl) + + +def find_attention_impl_variant( + impl: AttentionImpl, impl_cls: type[ImplT] +) -> ImplT | None: + for variant in impl.get_impl_variants(): + if isinstance(variant, impl_cls): + return cast(ImplT, variant) + return None + + _LN_2 = math.log(2.0) @@ -342,8 +356,8 @@ def get_per_layer_parameters( per_layer_params: dict[str, PerLayerParameters] = {} for key, layer in layers.items(): - impl = layer.impl - assert isinstance(impl, cls_) + impl = find_attention_impl_variant(layer.impl, cls_) + assert impl is not None # Infer hyperparameters from the attention layer window_size = getattr(impl, "sliding_window", None) @@ -1013,6 +1027,9 @@ def create_fast_prefill_custom_backend( common_attn_metadata.logits_indices_padded ) self.num_logits_indices = common_attn_metadata.num_logits_indices + self._attention_backend_variant = getattr( + metadata, "_attention_backend_variant", 0 + ) return KVSharingFastPrefillAttentionMetadata(metadata, common_attn_metadata) diff --git a/vllm/v1/attention/selector.py b/vllm/v1/attention/selector.py index c9631fd457..286fdcb8df 100644 --- a/vllm/v1/attention/selector.py +++ b/vllm/v1/attention/selector.py @@ -36,6 +36,7 @@ class AttentionSelectorConfig(NamedTuple): use_pcp: bool = False use_adaptive_verification: bool = False use_dcp: bool = False + use_rswa: bool = False def __repr__(self): return ( @@ -55,7 +56,8 @@ class AttentionSelectorConfig(NamedTuple): f"use_kv_connector={self.use_kv_connector}, " f"use_adaptive_verification={self.use_adaptive_verification}, " f"use_pcp={self.use_pcp}, " - f"use_dcp={self.use_dcp})" + f"use_dcp={self.use_dcp}, " + f"use_rswa={self.use_rswa})" ) @@ -168,6 +170,10 @@ def get_attn_backend( use_pcp=vllm_config.parallel_config.prefill_context_parallel_size > 1, use_adaptive_verification=use_adaptive_verification, use_dcp=vllm_config.parallel_config.decode_context_parallel_size > 1, + use_rswa=( + vllm_config.model_config is not None + and vllm_config.model_config.rswa_window is not None + ), ) # A per-KV-group override (keyed by KVCacheSpecKind) takes precedence over