[Attention] Add Triton/FlashInfer composite for multimodal prefix attention (#56305)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Matthew Bonanni
2026-09-13 14:29:41 +00:00
committed by GitHub
co-authored by Codex Lucas Wilkinson Lucas Wilkinson OpenAI Codex
parent 8cf9de9080
commit 82a85dc1d2
15 changed files with 1095 additions and 3 deletions
+49
View File
@@ -56,6 +56,55 @@ llm = LLM(
## Backend Selection Behavior
### Triton/FlashAttention Composite
On Hopper, `TRITON_FLASH_ATTN` is preferred for compatible multimodal-prefix
configurations. It uses Triton when the current queries need bidirectional
image attention and FlashAttention for causal text prefills and decode, with a
shared KV cache. The causal child must resolve to FA4. This happens
automatically for FA4-only shapes such as head size 512 and models whose
version policy promotes all layers to FA4, including Gemma 4. A standalone
head-size-256 configuration whose version policy selects FA3 falls back to
Triton for the whole layer.
No attention override is needed for supported multimodal models:
```bash
vllm serve google/gemma-4-31B-it
```
### Triton/FlashInfer Composite
On Blackwell, `TRITON_FLASHINFER` is preferred for compatible
multimodal-prefix configurations, including Gemma 4 with BF16 or FP8 KV cache. It uses
Triton when a batch's current queries require bidirectional image attention,
and FlashInfer for causal text prefills and decode. Historical image tokens
alone do not select Triton.
This instantiates `create_composite_attention_backend` from
`vllm/v1/attention/backends/composite.py` with Triton,
FlashInfer, and `MMPrefixAttentionRouting`. The reusable factory owns child
implementations, metadata dispatch, compatible cache requirements, and workspace
sharing. The routing policy selects the child and defines graph-capture safety.
Other combinations can reuse the same machinery.
The backend supports head dimensions 256/512, FP16/BF16 and FP8 KV cache,
and 64-token kernel pages with a head-major cache layout. TRTLLM handles causal
attention at both head dimensions; its hdim512 kernels do not support 128-token
pages.
For Gemma 4, full CUDA graphs cover single-token batches; multi-token batches
use the non-full-graph execution path. Models whose image masks extend beyond
the sliding window cannot use full attention graphs with this composite.
Context parallelism, R-SWA, attention sinks,
and adaptive verification are not supported by this composite.
It can also be selected explicitly:
```bash
vllm serve google/gemma-4-31B-it \
--attention-backend TRITON_FLASHINFER
```
### Manual Selection
When you explicitly set a backend via `--attention-backend` or `AttentionConfig`:
@@ -808,3 +808,142 @@ def test_fa4_hd256_impl_selection(attn_type, sliding_window, expected):
)
assert impl.vllm_flash_attn_version == expected
assert impl.fa4_hd256 == (expected == 4)
@pytest.fixture
def blackwell_selection():
with (
patch.object(
type(current_platform),
"get_device_capability",
return_value=DeviceCapability(10, 0),
),
patch(
"vllm.v1.attention.backends.fa_utils.is_fa_version_supported",
return_value=True,
),
):
yield
@pytest.fixture
def hopper_selection():
with (
patch.object(
type(current_platform),
"get_device_capability",
return_value=DeviceCapability(9, 0),
),
patch(
"vllm.v1.attention.backends.fa_utils.is_fa_version_supported",
return_value=True,
),
):
yield
@blackwell_only
@pytest.mark.parametrize("use_mm_prefix", [False, True])
@pytest.mark.parametrize("flash_attn_version", [None, 3, 4])
def test_hopper_mm_prefix_selects_triton_flash_attn(
use_mm_prefix, flash_attn_version, hopper_selection
):
"""Hopper selects the composite only when its causal route resolves FA4."""
from vllm.engine.arg_utils import EngineArgs
config = EngineArgs(
model="google/gemma-4-31B-it",
dtype="bfloat16",
attention_config=AttentionConfig(flash_attn_version=flash_attn_version),
).create_engine_config()
with set_current_vllm_config(config):
backend = get_attn_backend(
256, torch.bfloat16, None, use_mm_prefix=use_mm_prefix
)
fa4_resolved = config.attention_config.flash_attn_version == 4
if use_mm_prefix:
expected = "TRITON_FLASH_ATTN" if fa4_resolved else "TRITON_ATTN"
else:
expected = "FLASH_ATTN"
assert backend.get_name() == expected
@blackwell_only
@pytest.mark.parametrize("use_mm_prefix", [False, True])
@pytest.mark.parametrize("kv_cache_dtype", [None, "fp8_e4m3"])
def test_mm_prefix_selects_composite_without_changing_causal_default(
use_mm_prefix, kv_cache_dtype, blackwell_selection
):
"""The image-mask requirement must reach CUDA's automatic backend priority."""
from vllm.engine.arg_utils import EngineArgs
config = EngineArgs(
model="google/gemma-4-31B-it", dtype="bfloat16"
).create_engine_config()
with set_current_vllm_config(config):
backend = get_attn_backend(
256, torch.bfloat16, kv_cache_dtype, use_mm_prefix=use_mm_prefix
)
expected = "TRITON_FLASHINFER" if use_mm_prefix else "FLASHINFER"
assert backend.get_name() == expected
@blackwell_only
@pytest.mark.parametrize(
"feature_kwargs,reason",
[
({"use_rswa": True}, "R-SWA not supported"),
({"kv_cache_dtype": "int8_per_token_head"}, "kv_cache_dtype not supported"),
({"block_size": 16}, "block_size not supported"),
({"use_dcp": True}, "DCP not supported"),
({"use_pcp": True}, "PCP not supported"),
({"use_adaptive_verification": True}, "device-cpu query lens mismatch"),
],
)
def test_composite_rejects_features_not_shared_by_both_routes(feature_kwargs, reason):
"""Automatic selection must not admit features only one child can execute."""
from vllm.engine.arg_utils import EngineArgs
from vllm.v1.attention.backends.triton_flashinfer import TritonFlashInferBackend
config = EngineArgs(
model="google/gemma-4-31B-it", dtype="bfloat16"
).create_engine_config()
args = dict(
head_size=256,
dtype=torch.bfloat16,
kv_cache_dtype="auto",
block_size=128,
use_mla=False,
has_sink=False,
use_sparse=False,
use_mm_prefix=True,
use_per_head_quant_scales=False,
device_capability=DeviceCapability(10, 0),
attn_type=AttentionType.DECODER,
)
args.update(feature_kwargs)
with set_current_vllm_config(config):
reasons = TritonFlashInferBackend.validate_configuration(**args)
assert any(reason in message for message in reasons), reasons
@blackwell_only
def test_rswa_selection_does_not_reuse_causal_result(blackwell_selection):
"""An R-SWA requirement must invalidate a previously cached causal selection."""
from vllm.engine.arg_utils import EngineArgs
config = EngineArgs(
model="google/gemma-4-31B-it",
dtype="bfloat16",
attention_config={"backend": "TRITON_FLASHINFER"},
).create_engine_config()
with set_current_vllm_config(config):
assert (
get_attn_backend(256, torch.bfloat16, None).get_name()
== "TRITON_FLASHINFER"
)
config.model_config.model_arch_config.rswa_window = 128
with pytest.raises(ValueError, match="R-SWA"):
get_attn_backend(256, torch.bfloat16, None)
config.attention_config.backend = AttentionBackendEnum.TRITON_ATTN
assert get_attn_backend(256, torch.bfloat16, None).get_name() == "TRITON_ATTN"
+246
View File
@@ -638,3 +638,249 @@ def test_mm_prefix_kv_cache_path(head_size: int):
assert not torch.allclose(expected, causal_only, atol=2e-2, rtol=2e-2), (
"test batch does not actually exercise the mm_prefix branch"
)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA backend imports")
@pytest.mark.parametrize(
"query_len,seq_len,spans,expected,unclamped_expected",
[
(16, 32, [(20, 28)], True, True),
(16, 32, [(0, 16)], False, True),
(16, 32, [(0, 17)], True, True),
(16, 32, [(32, 40)], False, False),
(16, 32, [(31, 40)], False, True),
(1, 24, [(20, 28)], False, True),
(0, 24, [(20, 28)], False, False),
(16, 32, [(20, 20)], False, False),
(16, 64, [(20, 28)], False, False),
],
)
def test_composite_routes_queries_that_need_image_masking(
query_len, seq_len, spans, expected, unclamped_expected
):
"""Historical images and single-query steps must use the causal graph path."""
from types import SimpleNamespace
from vllm.v1.attention.backends.composite import requires_mm_prefix
common = SimpleNamespace(
max_query_len=query_len,
mm_req_doc_ranges={0: spans},
query_start_loc_cpu=torch.tensor([0, query_len]),
seq_lens_cpu_upper_bound=torch.tensor([seq_len]),
)
assert requires_mm_prefix(common) is expected
assert requires_mm_prefix(common, unclamped_window=True) is unclamped_expected
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA attention kernels")
@pytest.mark.parametrize("head_size", [256, 512])
@pytest.mark.parametrize(
"reverse_children,cache_dtype",
[(False, "auto"), (True, "auto"), (False, "fp8_e4m3")],
)
@pytest.mark.parametrize("causal_backend_name", ["FLASHINFER", "FLASH_ATTN"])
def test_composite_shared_cache_across_image_and_causal_steps(
head_size, reverse_children, cache_dtype, causal_backend_name
):
"""Changing routes must preserve KV writes, image masking and graph replay."""
if causal_backend_name == "FLASHINFER":
if not current_platform.is_device_capability_family(100):
pytest.skip("The Triton/FlashInfer composite supports Blackwell")
else:
if not current_platform.is_device_capability(90):
pytest.skip("The Triton/FlashAttention composite supports Hopper")
if reverse_children or cache_dtype != "auto":
pytest.skip("Reverse-child and FP8 coverage is backend-independent")
from vllm.config import set_current_vllm_config
from vllm.engine.arg_utils import EngineArgs
from vllm.model_executor.layers.attention import Attention
from vllm.utils.torch_utils import set_default_torch_dtype
from vllm.v1.attention.backend import AttentionCGSupport, CommonAttentionMetadata
from vllm.v1.attention.backends.composite import (
MMPrefixAttentionRouting,
create_composite_attention_backend,
)
from vllm.v1.attention.backends.flash_attn import (
FlashAttentionMetadata,
)
from vllm.v1.attention.backends.flashinfer import (
FlashInferBackend,
FlashInferMetadata,
FlashInferTrtllmAPIDecode,
TRTLLMPrefill,
)
from vllm.v1.attention.backends.triton_attn import (
TritonAttentionBackend,
TritonAttentionMetadata,
)
from vllm.v1.attention.backends.triton_flash_attn import (
TritonFlashAttentionBackend,
)
from vllm.v1.attention.backends.triton_flashinfer import (
TritonFlashInferBackend,
)
from vllm.v1.kv_cache_interface import FullAttentionSpec, SlidingWindowSpec
if causal_backend_name == "FLASHINFER":
backend = TritonFlashInferBackend
causal_metadata_type = FlashInferMetadata
block_size = backend.get_supported_kernel_block_sizes()[0]
else:
backend = TritonFlashAttentionBackend
causal_metadata_type = FlashAttentionMetadata
block_size = backend.get_preferred_block_size(16)
num_blocks = (209 + block_size - 1) // block_size
torch.manual_seed(42)
attention_config = {"backend": backend.get_name()}
if causal_backend_name == "FLASH_ATTN":
attention_config["flash_attn_version"] = 4
cfg = EngineArgs(
model="google/gemma-4-31B-it",
dtype="bfloat16",
max_model_len=512,
max_num_seqs=4,
max_num_batched_tokens=512,
block_size=block_size,
kv_cache_dtype=cache_dtype,
enforce_eager=True,
attention_config=attention_config,
).create_engine_config()
cfg.cache_config.kv_cache_layout = "LBHNC"
window = 128 if head_size == 256 else None
storage_dtype = torch.uint8 if cache_dtype == "fp8_e4m3" else DTYPE
spec_kwargs = dict(
block_size=block_size,
num_kv_heads=16,
head_size=head_size,
dtype=storage_dtype,
kv_quant_mode=get_kv_quant_mode(cache_dtype),
)
spec = (
SlidingWindowSpec(**spec_kwargs, sliding_window=window)
if window
else FullAttentionSpec(**spec_kwargs)
)
if reverse_children:
# Swapping the children and policy must preserve outputs and KV writes;
# dispatch cannot depend on a particular child's class or position.
class ReversedRouting(MMPrefixAttentionRouting):
capture_variant = 0
def select(self, metadata):
return 1 - super().select(metadata)
@staticmethod
def variant_uses_mm_prefix(variant):
return variant == 1
backend = create_composite_attention_backend(
FlashInferBackend,
TritonAttentionBackend,
name="ReversedCompositeBackend",
backend_name="CUSTOM",
module=__name__,
routing_policy=ReversedRouting,
)
with set_current_vllm_config(cfg), set_default_torch_dtype(DTYPE):
layer = Attention(
32,
head_size,
head_size**-0.5,
num_kv_heads=16,
cache_config=cfg.cache_config,
prefix="composite_test",
per_layer_sliding_window=window,
attn_backend=backend,
).to(DEVICE)
layer.mm_prefix_clamp_sliding_window = window is not None
assert backend.get_builder_cls().get_cudagraph_support(cfg, spec) == (
AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
)
if window:
layer.mm_prefix_clamp_sliding_window = False
assert backend.get_builder_cls().get_cudagraph_support(cfg, spec) == (
AttentionCGSupport.NEVER
)
layer.mm_prefix_clamp_sliding_window = True
builder = backend.get_builder_cls()(spec, ["composite_test"], cfg, DEVICE)
builder.set_kernel_block_size(block_size)
# Match production, where unwritten page tails are zero-initialized.
cache = torch.zeros(
num_blocks,
16,
block_size,
2 * head_size,
dtype=storage_dtype,
device=DEVICE,
)
keys = torch.randn(209, 16, head_size, dtype=DTYPE, device=DEVICE)
values = torch.randn_like(keys)
if cache_dtype == "fp8_e4m3":
keys = keys.to(torch.float8_e4m3fn).to(DTYPE)
values = values.to(torch.float8_e4m3fn).to(DTYPE)
table = torch.arange(num_blocks, dtype=torch.int32, device=DEVICE).unsqueeze(0)
previous = 0
for end, spans, metadata_type in [
(128, [(32, 111)], TritonAttentionMetadata),
(129, [(32, 111)], causal_metadata_type),
(145, [(32, 111)], causal_metadata_type),
(209, [(32, 111), (160, 207)], TritonAttentionMetadata),
]:
qlen = end - previous
starts = torch.tensor([0, qlen], dtype=torch.int32)
lengths = torch.tensor([end], dtype=torch.int32)
common = CommonAttentionMetadata(
query_start_loc=starts.to(DEVICE),
query_start_loc_cpu=starts,
seq_lens=lengths.to(DEVICE),
seq_lens_cpu_upper_bound=lengths,
num_reqs=1,
num_actual_tokens=qlen,
max_query_len=qlen,
max_seq_len=end,
block_table_tensor=table,
slot_mapping=torch.arange(previous, end, device=DEVICE),
mm_req_doc_ranges={0: spans},
)
metadata = builder.build(0, common)
assert isinstance(metadata, metadata_type)
if isinstance(metadata, FlashInferMetadata):
if qlen == 1:
assert isinstance(metadata.decode, FlashInferTrtllmAPIDecode)
else:
assert isinstance(metadata.prefill, TRTLLMPrefill)
query = torch.randn(qlen, 32, head_size, dtype=DTYPE, device=DEVICE)
if cache_dtype == "fp8_e4m3":
query = query.to(torch.float8_e4m3fn).to(DTYPE)
key, value = keys[previous:end], values[previous:end]
output = torch.empty_like(query)
layer.impl.do_kv_cache_update(layer, key, value, cache, common.slot_mapping)
layer.impl.forward(layer, query, key, value, cache, metadata, output)
expected = _dense_reference(
query,
[keys[:end]],
[values[:end]],
[qlen],
[end],
{0: spans},
window,
head_size**-0.5,
mm_clamp_sw=window or 0,
)
torch.testing.assert_close(output.float(), expected, atol=2e-2, rtol=2e-2)
if qlen == 1:
metadata = builder.build_for_cudagraph_capture(common)
assert isinstance(metadata, metadata_type)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
layer.impl.forward(
layer, query, key, value, cache, metadata, output
)
output.zero_()
graph.replay()
torch.testing.assert_close(
output.float(), expected, atol=2e-2, rtol=2e-2
)
previous = end
+1
View File
@@ -159,6 +159,7 @@ def test_dflash2_model_decoder_layer_cls(monkeypatch):
model_config=SimpleNamespace(
dtype=torch.float32,
is_mm_prefix_lm=False,
rswa_window=None,
),
kernel_config=SimpleNamespace(
linear_backend="auto",
+8
View File
@@ -87,6 +87,7 @@ def _get_backend_priorities(
kv_cache_dtype: CacheDType | None = None,
use_non_causal: bool = False,
head_size: int | None = None,
use_mm_prefix: bool = False,
) -> list[AttentionBackendEnum]:
"""Get backend priorities with lazy import to avoid circular dependency."""
from vllm.utils.torch_utils import is_quantized_kv_cache
@@ -156,6 +157,7 @@ def _get_backend_priorities(
# So prefer FlashAttention when non-causal on SM100f.
if device_capability.major == 10 and not use_non_causal:
return [
*([AttentionBackendEnum.TRITON_FLASHINFER] if use_mm_prefix else []),
AttentionBackendEnum.FLASHINFER,
AttentionBackendEnum.FLASH_ATTN,
AttentionBackendEnum.TRITON_ATTN,
@@ -164,6 +166,11 @@ def _get_backend_priorities(
]
else:
return [
*(
[AttentionBackendEnum.TRITON_FLASH_ATTN]
if device_capability.major == 9 and use_mm_prefix
else []
),
AttentionBackendEnum.FLASH_ATTN,
AttentionBackendEnum.FLASHINFER,
AttentionBackendEnum.TRITON_ATTN,
@@ -394,6 +401,7 @@ class CudaPlatformBase(Platform):
kv_cache_dtype=attn_selector_config.kv_cache_dtype,
use_non_causal=attn_selector_config.use_non_causal,
head_size=attn_selector_config.head_size,
use_mm_prefix=attn_selector_config.use_mm_prefix,
)
for priority, backend in enumerate(backend_priorities):
try:
+10
View File
@@ -185,6 +185,10 @@ class AttentionBackend(ABC):
def supports_sliding_window(cls) -> bool:
return False
@classmethod
def supports_rswa(cls) -> bool:
return False
@classmethod
def supports_non_causal(cls) -> bool:
"""Check if backend supports non-causal (bidirectional) attention
@@ -287,6 +291,7 @@ class AttentionBackend(ABC):
use_pcp: bool = False,
use_adaptive_verification: bool = False,
use_dcp: bool = False,
use_rswa: bool = False,
) -> list[str]:
invalid_reasons = []
if not cls.supports_head_size(head_size):
@@ -321,6 +326,8 @@ class AttentionBackend(ABC):
invalid_reasons.append(f"attention type {attn_type} not supported")
if has_sliding_window and not cls.supports_sliding_window():
invalid_reasons.append("sliding window not supported")
if use_rswa and not cls.supports_rswa():
invalid_reasons.append("R-SWA not supported")
if use_non_causal and not cls.supports_non_causal():
invalid_reasons.append("non-causal attention not supported")
if use_mla and use_non_causal and use_dcp and not cls.supports_non_causal_dcp():
@@ -854,6 +861,9 @@ class AttentionImplBase(ABC, Generic[T]):
def process_weights_after_loading(self, act_dtype: torch.dtype):
pass
def get_impl_variants(self) -> tuple["AttentionImplBase", ...]:
return (self,)
def prepare_for_batch(self, attn_metadata: T | None) -> None:
"""Prepare implementation-specific state for the current batch."""
+509
View File
@@ -0,0 +1,509 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Reusable attention backend composition and routing policies."""
import functools
import math
from typing import ClassVar, Protocol, cast
import torch
from vllm.config import VllmConfig
from vllm.v1.attention.backend import (
AttentionBackend,
AttentionCGSupport,
AttentionImpl,
AttentionMetadataBuilder,
CommonAttentionMetadata,
)
from vllm.v1.attention.backends.utils import get_supported_kv_cache_layouts
from vllm.v1.kv_cache_interface import KVCacheSpec
def requires_mm_prefix(
metadata: CommonAttentionMetadata, *, unclamped_window: bool = False
) -> bool:
ranges = metadata.mm_req_doc_ranges
# With one query per request there are no future keys to unmask. This
# also keeps single-token prefills on the captured FlashInfer path.
if not ranges or (metadata.max_query_len <= 1 and not unclamped_window):
return False
seq_lens = metadata.seq_lens_cpu_upper_bound
assert seq_lens is not None
starts = metadata.query_start_loc_cpu
for req_idx, spans in ranges.items():
query_len = int(starts[req_idx + 1] - starts[req_idx])
if query_len == 0 or (query_len == 1 and not unclamped_window):
continue
end = int(seq_lens[req_idx])
begin = end - query_len
# An unclamped image can also expose keys behind the sliding window.
if unclamped_window and any(
start < stop and start < end and stop >= begin for start, stop in spans
):
return True
# Prefill lengths are exact. Historical image ranges must not divert
# later text/decode queries to the image backend.
if any(
start < stop and start < end - 1 and stop > begin for start, stop in spans
):
return True
return False
def _has_unclamped_window(layers) -> bool:
return any(
getattr(layer, "sliding_window", None) is not None
and not getattr(layer, "mm_prefix_clamp_sliding_window", False)
for layer in layers
)
class CompositeAttentionRouting(Protocol):
"""Select a child backend and declare routing constraints for composition."""
capture_variant: ClassVar[int]
"""Child used during graph capture; replay must select the same child."""
def __init__(self, layer_names: list[str], vllm_config: VllmConfig) -> None: ...
def select(self, metadata: CommonAttentionMetadata) -> int:
"""Return 0 for the first child or 1 for the second child."""
...
@staticmethod
def get_cudagraph_support(
vllm_config: VllmConfig, kv_cache_spec: KVCacheSpec
) -> AttentionCGSupport:
"""Limit graph support to batches whose routing stays invariant."""
...
@staticmethod
def variant_uses_mm_prefix(variant: int) -> bool:
"""Whether this child must support the model's image-mask requirement."""
...
class MMPrefixAttentionRouting(CompositeAttentionRouting):
"""Route image masks to variant zero and causal queries to variant one."""
def __init__(self, layer_names: list[str], vllm_config: VllmConfig) -> None:
layers = vllm_config.compilation_config.static_forward_context
self.unclamped_window = _has_unclamped_window(
layers[name] for name in layer_names
)
def select(self, metadata: CommonAttentionMetadata) -> int:
return int(
not requires_mm_prefix(metadata, unclamped_window=self.unclamped_window)
)
capture_variant: ClassVar[int] = 1
@staticmethod
def get_cudagraph_support(
vllm_config: VllmConfig, kv_cache_spec: KVCacheSpec
) -> AttentionCGSupport:
if _has_unclamped_window(
vllm_config.compilation_config.static_forward_context.values()
):
return AttentionCGSupport.NEVER
return AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
@staticmethod
def variant_uses_mm_prefix(variant: int) -> bool:
return variant == 0
def _intersect_kernel_block_sizes(
first: type[AttentionBackend], second: type[AttentionBackend]
):
from vllm.v1.attention.backend import MultipleOf
result: list[int | MultipleOf] = []
seen: set[tuple[type, int]] = set()
for lhs in first.get_supported_kernel_block_sizes():
for rhs in second.get_supported_kernel_block_sizes():
candidate: int | MultipleOf | None = None
if isinstance(lhs, MultipleOf) and isinstance(rhs, MultipleOf):
candidate = MultipleOf(math.lcm(lhs.base, rhs.base))
elif isinstance(lhs, MultipleOf) and isinstance(rhs, int):
candidate = rhs if rhs % lhs.base == 0 else None
elif isinstance(lhs, int) and isinstance(rhs, MultipleOf):
candidate = lhs if lhs % rhs.base == 0 else None
elif lhs == rhs:
candidate = lhs
if candidate is None:
continue
value = candidate.base if isinstance(candidate, MultipleOf) else candidate
key = (type(candidate), value)
if key not in seen:
result.append(candidate)
seen.add(key)
return result
@functools.cache
def create_composite_attention_backend(
general_backend: type[AttentionBackend],
causal_backend: type[AttentionBackend],
*,
name: str,
backend_name: str,
module: str,
routing_policy: type[CompositeAttentionRouting],
head_sizes: tuple[int, ...] = (),
kernel_block_sizes: tuple[int, ...] = (),
device_major: int | None = None,
) -> type[AttentionBackend]:
"""Compose two backends with shared storage and an explicit routing policy."""
if general_backend.full_cls_name() == causal_backend.full_cls_name():
return general_backend
general_impl_cls = general_backend.get_impl_cls()
causal_impl_cls = causal_backend.get_impl_cls()
general_builder_cls = general_backend.get_builder_cls()
causal_builder_cls = causal_backend.get_builder_cls()
if (
general_backend.forward_includes_kv_cache_update
or causal_backend.forward_includes_kv_cache_update
):
raise ValueError("Composite backends require separate KV cache updates")
class CompositeAttentionImpl(AttentionImpl):
def __init__(self, *args, **kwargs) -> None:
self.general_impl = cast(AttentionImpl, general_impl_cls(*args, **kwargs))
try:
self.causal_impl = cast(AttentionImpl, causal_impl_cls(*args, **kwargs))
except TypeError as e:
raise ValueError(
f"Causal attention backend {causal_backend.get_name()} "
f"does not accept the attention arguments required by "
f"the general backend {general_backend.get_name()}: {e}"
) from e
for name in (
"num_heads",
"num_kv_heads",
"head_size",
"scale",
"kv_cache_dtype",
):
if hasattr(self.general_impl, name):
setattr(self, name, getattr(self.general_impl, name))
self.supports_quant_query_input = (
self.general_impl.supports_quant_query_input
and self.causal_impl.supports_quant_query_input
)
self.supports_pcp = self.general_impl.supports_pcp
self.supports_dcp = self.causal_impl.supports_dcp
self.can_return_lse_for_decode = self.causal_impl.can_return_lse_for_decode
self.lse_base_on_e = self.causal_impl.lse_base_on_e
self.need_to_return_lse_for_decode = (
self.causal_impl.need_to_return_lse_for_decode
)
self.supports_mtp_with_cp_non_trivial_interleave_size = (
self.causal_impl.supports_mtp_with_cp_non_trivial_interleave_size
)
def get_impl_for_metadata(self, attn_metadata):
return (
self.causal_impl
if getattr(attn_metadata, "_attention_backend_variant", 0)
else self.general_impl
)
def get_impl_variants(self):
return self.general_impl, self.causal_impl
def forward(
self,
layer,
query,
key,
value,
kv_cache,
attn_metadata,
output,
output_scale=None,
output_block_scale=None,
):
impl = self.get_impl_for_metadata(attn_metadata)
return impl.forward(
layer,
query,
key,
value,
kv_cache,
attn_metadata,
output,
output_scale,
output_block_scale,
)
def process_weights_after_loading(self, act_dtype: torch.dtype):
for impl in self.get_impl_variants():
impl.process_weights_after_loading(act_dtype)
def fused_output_quant_supported(self, quant_key):
return all(
impl.fused_output_quant_supported(quant_key)
for impl in self.get_impl_variants()
)
def fused_qk_norm_rope_kvcache_supported(self):
return self.general_impl.fused_qk_norm_rope_kvcache_supported()
def do_qk_norm_rope_kvcache_update(self, *args, **kwargs):
method = self.general_impl.do_qk_norm_rope_kvcache_update
return method(*args, **kwargs)
def fused_rope_kvcache_supported(self):
return self.general_impl.fused_rope_kvcache_supported()
def do_rope_and_kv_cache_update(self, *args, **kwargs):
method = self.general_impl.do_rope_and_kv_cache_update
return method(*args, **kwargs)
def do_kv_cache_update(self, *args, **kwargs):
method = self.general_impl.do_kv_cache_update # type: ignore[attr-defined]
return method(*args, **kwargs)
class CompositeAttentionMetadataBuilder(AttentionMetadataBuilder):
requires_block_table_width = (
general_builder_cls.requires_block_table_width
or causal_builder_cls.requires_block_table_width
)
def __init__(self, kv_cache_spec, layer_names, vllm_config, device, **kwargs):
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
args = (kv_cache_spec, layer_names, vllm_config, device)
# Only the variant that asked for the extra ctor kwargs receives them.
self.general_builder = general_builder_cls(
*args,
**(kwargs if general_builder_cls.requires_block_table_width else {}),
)
self.causal_builder = causal_builder_cls(
*args,
**(kwargs if causal_builder_cls.requires_block_table_width else {}),
)
self.reorder_batch_threshold = min(
(
builder.reorder_batch_threshold
for builder in self._builders
if builder.reorder_batch_threshold is not None
),
default=None,
)
self.routing = routing_policy(layer_names, vllm_config)
def set_kernel_block_size(self, kernel_block_size):
super().set_kernel_block_size(kernel_block_size)
for builder in self._builders:
builder.set_kernel_block_size(kernel_block_size)
@classmethod
def get_cudagraph_support(cls, vllm_config, kv_cache_spec):
return min(
general_builder_cls.get_cudagraph_support(vllm_config, kv_cache_spec),
causal_builder_cls.get_cudagraph_support(vllm_config, kv_cache_spec),
routing_policy.get_cudagraph_support(vllm_config, kv_cache_spec),
key=lambda support: support.value,
)
def _builder(self, common_attn_metadata):
backend_variant = self.routing.select(common_attn_metadata)
builder = self.causal_builder if backend_variant else self.general_builder
return builder, backend_variant
def _build(self, method, common_attn_metadata, **kwargs):
builder, backend_variant = self._builder(common_attn_metadata)
metadata = getattr(builder, method)(
common_attn_metadata=common_attn_metadata, **kwargs
)
metadata._attention_backend_variant = backend_variant
return metadata
def build(self, common_prefix_len, common_attn_metadata, fast_build=False):
return self._build(
"build",
common_attn_metadata,
common_prefix_len=common_prefix_len,
fast_build=fast_build,
)
def build_for_cudagraph_capture(self, common_attn_metadata):
variant = routing_policy.capture_variant
metadata = self._builders[variant].build_for_cudagraph_capture(
common_attn_metadata
)
metadata._attention_backend_variant = variant
return metadata
@property
def _builders(self):
return self.general_builder, self.causal_builder
# Forward the optional workspace protocol so that a wrapped builder
# which allocates one (FlashInfer) still joins the runner's cross-group
# sharing instead of allocating a second buffer behind the composite.
def _get_workspace_buffer(self):
for builder in self._builders:
if hasattr(builder, "_get_workspace_buffer"):
workspace = builder._get_workspace_buffer()
# The runner only calls the getter for the group that
# provides the buffer, so hand it to the sibling here.
self.set_workspace_buffer(workspace)
return workspace
return None
def set_workspace_buffer(self, workspace_buffer):
for builder in self._builders:
if hasattr(builder, "set_workspace_buffer"):
builder.set_workspace_buffer(workspace_buffer)
class CompositeAttentionBackend(AttentionBackend):
forward_includes_kv_cache_update = False
general_backend_cls = general_backend
causal_backend_cls = causal_backend
@staticmethod
def get_name():
return backend_name
@staticmethod
def get_impl_cls():
return CompositeAttentionImpl
@staticmethod
def get_builder_cls():
return CompositeAttentionMetadataBuilder
@staticmethod
def get_supported_kernel_block_sizes():
if kernel_block_sizes:
return [
size
for size in kernel_block_sizes
if all(
backend.supports_block_size(size)
for backend in (general_backend, causal_backend)
)
]
return _intersect_kernel_block_sizes(general_backend, causal_backend)
@classmethod
def supports_block_size(cls, block_size):
return bool(
cls.get_supported_kernel_block_sizes()
) and super().supports_block_size(block_size)
@classmethod
def get_supported_head_sizes(cls):
return list(head_sizes)
@classmethod
def supports_head_size(cls, head_size):
return (not head_sizes or head_size in head_sizes) and all(
backend.supports_head_size(head_size)
for backend in (general_backend, causal_backend)
)
@classmethod
def supports_compute_capability(cls, capability):
return (device_major is None or capability.major == device_major) and all(
backend.supports_compute_capability(capability)
for backend in (general_backend, causal_backend)
)
@classmethod
def supports_dtype(cls, dtype):
return all(
backend.supports_dtype(dtype)
for backend in (general_backend, causal_backend)
)
@classmethod
def supports_kv_cache_dtype(cls, dtype):
return all(
backend.supports_kv_cache_dtype(dtype)
for backend in (general_backend, causal_backend)
)
@classmethod
def supports_mm_prefix(cls):
return all(
not routing_policy.variant_uses_mm_prefix(variant)
or backend.supports_mm_prefix()
for variant, backend in enumerate((general_backend, causal_backend))
)
@classmethod
def supports_sliding_window(cls):
return all(
backend.supports_sliding_window()
for backend in (general_backend, causal_backend)
)
@classmethod
def supports_rswa(cls):
return all(
backend.supports_rswa() for backend in (general_backend, causal_backend)
)
@classmethod
def supports_pcp(cls):
return False
@classmethod
def supports_dcp(cls):
return False
@classmethod
def supports_device_cpu_query_lens_mismatch(cls):
return False
@classmethod
def supports_combination(
cls,
head_size,
dtype,
kv_cache_dtype,
block_size,
use_mla,
has_sink,
use_sparse,
use_mm_prefix,
device_capability,
):
for variant, backend in enumerate((general_backend, causal_backend)):
reason = backend.supports_combination(
head_size,
dtype,
kv_cache_dtype,
block_size,
use_mla,
has_sink,
use_sparse,
use_mm_prefix and routing_policy.variant_uses_mm_prefix(variant),
device_capability,
)
if reason:
return f"{backend.get_name()}: {reason}"
return None
@classmethod
def customize_spec(cls, spec):
general_spec = general_backend.customize_spec(spec)
if general_spec != causal_backend.customize_spec(spec):
raise ValueError("Composite backends require the same KV cache spec")
return general_spec
@classmethod
def supported_kv_cache_layouts(cls):
return tuple(
get_supported_kv_cache_layouts((general_backend, causal_backend))
)
CompositeAttentionBackend.__name__ = name
CompositeAttentionBackend.__qualname__ = name
CompositeAttentionBackend.__module__ = module
return CompositeAttentionBackend
+4
View File
@@ -129,6 +129,10 @@ class FlashAttentionBackend(AttentionBackend):
def get_name() -> str:
return "FLASH_ATTN"
@classmethod
def supports_rswa(cls) -> bool:
return True
@classmethod
def supports_sliding_window(cls) -> bool:
return True
@@ -105,6 +105,10 @@ class FlexAttentionBackend(AttentionBackend):
def get_name() -> str:
return "FLEX_ATTENTION"
@classmethod
def supports_rswa(cls) -> bool:
return True
@classmethod
def supports_sliding_window(cls) -> bool:
return True
+6
View File
@@ -49,6 +49,9 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta):
TRITON_ATTN_DIFFKV = (
"vllm.v1.attention.backends.triton_attn_diffkv.TritonAttentionDiffKVBackend"
)
TRITON_FLASH_ATTN = (
"vllm.v1.attention.backends.triton_flash_attn.TritonFlashAttentionBackend"
)
ROCM_ATTN = "vllm.v1.attention.backends.rocm_attn.RocmAttentionBackend"
ROCM_AITER_MLA = "vllm.v1.attention.backends.mla.rocm_aiter_mla.AiterMLABackend"
ROCM_AITER_TRITON_MLA = (
@@ -63,6 +66,9 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta):
XPU_MLA_SPARSE = "vllm.v1.attention.backends.mla.xpu_mla_sparse.XPUMLASparseBackend"
TORCH_SDPA = "" # this tag is only used for ViT
FLASHINFER = "vllm.v1.attention.backends.flashinfer.FlashInferBackend"
TRITON_FLASHINFER = (
"vllm.v1.attention.backends.triton_flashinfer.TritonFlashInferBackend"
)
FLASHINFER_MLA = (
"vllm.v1.attention.backends.mla.flashinfer_mla.FlashInferMLABackend"
)
@@ -323,6 +323,10 @@ class TritonAttentionBackend(AttentionBackend):
def get_name() -> str:
return "TRITON_ATTN"
@classmethod
def supports_rswa(cls) -> bool:
return True
@classmethod
def supports_sliding_window(cls) -> bool:
return True
@@ -0,0 +1,67 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Triton image-mask and FlashAttention causal attention composite."""
import torch
from vllm.config.cache import CacheDType
from vllm.platforms.interface import DeviceCapability
from vllm.v1.attention.backends.composite import (
MMPrefixAttentionRouting,
create_composite_attention_backend,
)
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
from vllm.v1.attention.backends.triton_attn import TritonAttentionBackend
class _FA4Backend(FlashAttentionBackend):
@classmethod
def supports_combination(
cls,
head_size: int,
dtype: torch.dtype,
kv_cache_dtype: CacheDType | None,
block_size: int | None,
use_mla: bool,
has_sink: bool,
use_sparse: bool,
use_mm_prefix: bool,
device_capability: DeviceCapability,
) -> str | None:
reason = super().supports_combination(
head_size,
dtype,
kv_cache_dtype,
block_size,
use_mla,
has_sink,
use_sparse,
use_mm_prefix,
device_capability,
)
if reason is not None:
return reason
if (
get_flash_attn_version(
head_size=head_size,
has_sinks=has_sink,
kv_cache_block_size=block_size,
supports_fa4_hd256=True,
)
!= 4
):
return "causal route requires FlashAttention v4"
return None
TritonFlashAttentionBackend = create_composite_attention_backend(
TritonAttentionBackend,
_FA4Backend,
name="TritonFlashAttentionBackend",
backend_name="TRITON_FLASH_ATTN",
module=__name__,
routing_policy=MMPrefixAttentionRouting,
head_sizes=(256, 512),
device_major=9,
)
@@ -0,0 +1,22 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Triton image-mask and FlashInfer causal attention composite."""
from vllm.v1.attention.backends.composite import (
MMPrefixAttentionRouting,
create_composite_attention_backend,
)
from vllm.v1.attention.backends.flashinfer import FlashInferBackend
from vllm.v1.attention.backends.triton_attn import TritonAttentionBackend
TritonFlashInferBackend = create_composite_attention_backend(
TritonAttentionBackend,
FlashInferBackend,
name="TritonFlashInferBackend",
backend_name="TRITON_FLASHINFER",
module=__name__,
routing_policy=MMPrefixAttentionRouting,
head_sizes=(256, 512),
kernel_block_sizes=(64,),
device_major=10,
)
+19 -2
View File
@@ -8,6 +8,7 @@ from typing import (
TYPE_CHECKING,
Any,
Protocol,
TypeVar,
cast,
)
@@ -45,6 +46,19 @@ logger = init_logger(__name__)
PAD_SLOT_ID = -1
NULL_BLOCK_ID = 0
ImplT = TypeVar("ImplT", bound=AttentionImpl)
def find_attention_impl_variant(
impl: AttentionImpl, impl_cls: type[ImplT]
) -> ImplT | None:
for variant in impl.get_impl_variants():
if isinstance(variant, impl_cls):
return cast(ImplT, variant)
return None
_LN_2 = math.log(2.0)
@@ -342,8 +356,8 @@ def get_per_layer_parameters(
per_layer_params: dict[str, PerLayerParameters] = {}
for key, layer in layers.items():
impl = layer.impl
assert isinstance(impl, cls_)
impl = find_attention_impl_variant(layer.impl, cls_)
assert impl is not None
# Infer hyperparameters from the attention layer
window_size = getattr(impl, "sliding_window", None)
@@ -1013,6 +1027,9 @@ def create_fast_prefill_custom_backend(
common_attn_metadata.logits_indices_padded
)
self.num_logits_indices = common_attn_metadata.num_logits_indices
self._attention_backend_variant = getattr(
metadata, "_attention_backend_variant", 0
)
return KVSharingFastPrefillAttentionMetadata(metadata, common_attn_metadata)
+7 -1
View File
@@ -36,6 +36,7 @@ class AttentionSelectorConfig(NamedTuple):
use_pcp: bool = False
use_adaptive_verification: bool = False
use_dcp: bool = False
use_rswa: bool = False
def __repr__(self):
return (
@@ -55,7 +56,8 @@ class AttentionSelectorConfig(NamedTuple):
f"use_kv_connector={self.use_kv_connector}, "
f"use_adaptive_verification={self.use_adaptive_verification}, "
f"use_pcp={self.use_pcp}, "
f"use_dcp={self.use_dcp})"
f"use_dcp={self.use_dcp}, "
f"use_rswa={self.use_rswa})"
)
@@ -168,6 +170,10 @@ def get_attn_backend(
use_pcp=vllm_config.parallel_config.prefill_context_parallel_size > 1,
use_adaptive_verification=use_adaptive_verification,
use_dcp=vllm_config.parallel_config.decode_context_parallel_size > 1,
use_rswa=(
vllm_config.model_config is not None
and vllm_config.model_config.rswa_window is not None
),
)
# A per-KV-group override (keyed by KVCacheSpecKind) takes precedence over