[Attention] Add Triton/FlashInfer composite for multimodal prefix attention (#56305)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Matthew Bonanni
2026-09-13 14:29:41 +00:00
committed by GitHub
co-authored by Codex Lucas Wilkinson Lucas Wilkinson OpenAI Codex
parent 8cf9de9080
commit 82a85dc1d2
15 changed files with 1095 additions and 3 deletions
@@ -808,3 +808,142 @@ def test_fa4_hd256_impl_selection(attn_type, sliding_window, expected):
)
assert impl.vllm_flash_attn_version == expected
assert impl.fa4_hd256 == (expected == 4)
@pytest.fixture
def blackwell_selection():
with (
patch.object(
type(current_platform),
"get_device_capability",
return_value=DeviceCapability(10, 0),
),
patch(
"vllm.v1.attention.backends.fa_utils.is_fa_version_supported",
return_value=True,
),
):
yield
@pytest.fixture
def hopper_selection():
with (
patch.object(
type(current_platform),
"get_device_capability",
return_value=DeviceCapability(9, 0),
),
patch(
"vllm.v1.attention.backends.fa_utils.is_fa_version_supported",
return_value=True,
),
):
yield
@blackwell_only
@pytest.mark.parametrize("use_mm_prefix", [False, True])
@pytest.mark.parametrize("flash_attn_version", [None, 3, 4])
def test_hopper_mm_prefix_selects_triton_flash_attn(
use_mm_prefix, flash_attn_version, hopper_selection
):
"""Hopper selects the composite only when its causal route resolves FA4."""
from vllm.engine.arg_utils import EngineArgs
config = EngineArgs(
model="google/gemma-4-31B-it",
dtype="bfloat16",
attention_config=AttentionConfig(flash_attn_version=flash_attn_version),
).create_engine_config()
with set_current_vllm_config(config):
backend = get_attn_backend(
256, torch.bfloat16, None, use_mm_prefix=use_mm_prefix
)
fa4_resolved = config.attention_config.flash_attn_version == 4
if use_mm_prefix:
expected = "TRITON_FLASH_ATTN" if fa4_resolved else "TRITON_ATTN"
else:
expected = "FLASH_ATTN"
assert backend.get_name() == expected
@blackwell_only
@pytest.mark.parametrize("use_mm_prefix", [False, True])
@pytest.mark.parametrize("kv_cache_dtype", [None, "fp8_e4m3"])
def test_mm_prefix_selects_composite_without_changing_causal_default(
use_mm_prefix, kv_cache_dtype, blackwell_selection
):
"""The image-mask requirement must reach CUDA's automatic backend priority."""
from vllm.engine.arg_utils import EngineArgs
config = EngineArgs(
model="google/gemma-4-31B-it", dtype="bfloat16"
).create_engine_config()
with set_current_vllm_config(config):
backend = get_attn_backend(
256, torch.bfloat16, kv_cache_dtype, use_mm_prefix=use_mm_prefix
)
expected = "TRITON_FLASHINFER" if use_mm_prefix else "FLASHINFER"
assert backend.get_name() == expected
@blackwell_only
@pytest.mark.parametrize(
"feature_kwargs,reason",
[
({"use_rswa": True}, "R-SWA not supported"),
({"kv_cache_dtype": "int8_per_token_head"}, "kv_cache_dtype not supported"),
({"block_size": 16}, "block_size not supported"),
({"use_dcp": True}, "DCP not supported"),
({"use_pcp": True}, "PCP not supported"),
({"use_adaptive_verification": True}, "device-cpu query lens mismatch"),
],
)
def test_composite_rejects_features_not_shared_by_both_routes(feature_kwargs, reason):
"""Automatic selection must not admit features only one child can execute."""
from vllm.engine.arg_utils import EngineArgs
from vllm.v1.attention.backends.triton_flashinfer import TritonFlashInferBackend
config = EngineArgs(
model="google/gemma-4-31B-it", dtype="bfloat16"
).create_engine_config()
args = dict(
head_size=256,
dtype=torch.bfloat16,
kv_cache_dtype="auto",
block_size=128,
use_mla=False,
has_sink=False,
use_sparse=False,
use_mm_prefix=True,
use_per_head_quant_scales=False,
device_capability=DeviceCapability(10, 0),
attn_type=AttentionType.DECODER,
)
args.update(feature_kwargs)
with set_current_vllm_config(config):
reasons = TritonFlashInferBackend.validate_configuration(**args)
assert any(reason in message for message in reasons), reasons
@blackwell_only
def test_rswa_selection_does_not_reuse_causal_result(blackwell_selection):
"""An R-SWA requirement must invalidate a previously cached causal selection."""
from vllm.engine.arg_utils import EngineArgs
config = EngineArgs(
model="google/gemma-4-31B-it",
dtype="bfloat16",
attention_config={"backend": "TRITON_FLASHINFER"},
).create_engine_config()
with set_current_vllm_config(config):
assert (
get_attn_backend(256, torch.bfloat16, None).get_name()
== "TRITON_FLASHINFER"
)
config.model_config.model_arch_config.rswa_window = 128
with pytest.raises(ValueError, match="R-SWA"):
get_attn_backend(256, torch.bfloat16, None)
config.attention_config.backend = AttentionBackendEnum.TRITON_ATTN
assert get_attn_backend(256, torch.bfloat16, None).get_name() == "TRITON_ATTN"
+246
View File
@@ -638,3 +638,249 @@ def test_mm_prefix_kv_cache_path(head_size: int):
assert not torch.allclose(expected, causal_only, atol=2e-2, rtol=2e-2), (
"test batch does not actually exercise the mm_prefix branch"
)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA backend imports")
@pytest.mark.parametrize(
"query_len,seq_len,spans,expected,unclamped_expected",
[
(16, 32, [(20, 28)], True, True),
(16, 32, [(0, 16)], False, True),
(16, 32, [(0, 17)], True, True),
(16, 32, [(32, 40)], False, False),
(16, 32, [(31, 40)], False, True),
(1, 24, [(20, 28)], False, True),
(0, 24, [(20, 28)], False, False),
(16, 32, [(20, 20)], False, False),
(16, 64, [(20, 28)], False, False),
],
)
def test_composite_routes_queries_that_need_image_masking(
query_len, seq_len, spans, expected, unclamped_expected
):
"""Historical images and single-query steps must use the causal graph path."""
from types import SimpleNamespace
from vllm.v1.attention.backends.composite import requires_mm_prefix
common = SimpleNamespace(
max_query_len=query_len,
mm_req_doc_ranges={0: spans},
query_start_loc_cpu=torch.tensor([0, query_len]),
seq_lens_cpu_upper_bound=torch.tensor([seq_len]),
)
assert requires_mm_prefix(common) is expected
assert requires_mm_prefix(common, unclamped_window=True) is unclamped_expected
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA attention kernels")
@pytest.mark.parametrize("head_size", [256, 512])
@pytest.mark.parametrize(
"reverse_children,cache_dtype",
[(False, "auto"), (True, "auto"), (False, "fp8_e4m3")],
)
@pytest.mark.parametrize("causal_backend_name", ["FLASHINFER", "FLASH_ATTN"])
def test_composite_shared_cache_across_image_and_causal_steps(
head_size, reverse_children, cache_dtype, causal_backend_name
):
"""Changing routes must preserve KV writes, image masking and graph replay."""
if causal_backend_name == "FLASHINFER":
if not current_platform.is_device_capability_family(100):
pytest.skip("The Triton/FlashInfer composite supports Blackwell")
else:
if not current_platform.is_device_capability(90):
pytest.skip("The Triton/FlashAttention composite supports Hopper")
if reverse_children or cache_dtype != "auto":
pytest.skip("Reverse-child and FP8 coverage is backend-independent")
from vllm.config import set_current_vllm_config
from vllm.engine.arg_utils import EngineArgs
from vllm.model_executor.layers.attention import Attention
from vllm.utils.torch_utils import set_default_torch_dtype
from vllm.v1.attention.backend import AttentionCGSupport, CommonAttentionMetadata
from vllm.v1.attention.backends.composite import (
MMPrefixAttentionRouting,
create_composite_attention_backend,
)
from vllm.v1.attention.backends.flash_attn import (
FlashAttentionMetadata,
)
from vllm.v1.attention.backends.flashinfer import (
FlashInferBackend,
FlashInferMetadata,
FlashInferTrtllmAPIDecode,
TRTLLMPrefill,
)
from vllm.v1.attention.backends.triton_attn import (
TritonAttentionBackend,
TritonAttentionMetadata,
)
from vllm.v1.attention.backends.triton_flash_attn import (
TritonFlashAttentionBackend,
)
from vllm.v1.attention.backends.triton_flashinfer import (
TritonFlashInferBackend,
)
from vllm.v1.kv_cache_interface import FullAttentionSpec, SlidingWindowSpec
if causal_backend_name == "FLASHINFER":
backend = TritonFlashInferBackend
causal_metadata_type = FlashInferMetadata
block_size = backend.get_supported_kernel_block_sizes()[0]
else:
backend = TritonFlashAttentionBackend
causal_metadata_type = FlashAttentionMetadata
block_size = backend.get_preferred_block_size(16)
num_blocks = (209 + block_size - 1) // block_size
torch.manual_seed(42)
attention_config = {"backend": backend.get_name()}
if causal_backend_name == "FLASH_ATTN":
attention_config["flash_attn_version"] = 4
cfg = EngineArgs(
model="google/gemma-4-31B-it",
dtype="bfloat16",
max_model_len=512,
max_num_seqs=4,
max_num_batched_tokens=512,
block_size=block_size,
kv_cache_dtype=cache_dtype,
enforce_eager=True,
attention_config=attention_config,
).create_engine_config()
cfg.cache_config.kv_cache_layout = "LBHNC"
window = 128 if head_size == 256 else None
storage_dtype = torch.uint8 if cache_dtype == "fp8_e4m3" else DTYPE
spec_kwargs = dict(
block_size=block_size,
num_kv_heads=16,
head_size=head_size,
dtype=storage_dtype,
kv_quant_mode=get_kv_quant_mode(cache_dtype),
)
spec = (
SlidingWindowSpec(**spec_kwargs, sliding_window=window)
if window
else FullAttentionSpec(**spec_kwargs)
)
if reverse_children:
# Swapping the children and policy must preserve outputs and KV writes;
# dispatch cannot depend on a particular child's class or position.
class ReversedRouting(MMPrefixAttentionRouting):
capture_variant = 0
def select(self, metadata):
return 1 - super().select(metadata)
@staticmethod
def variant_uses_mm_prefix(variant):
return variant == 1
backend = create_composite_attention_backend(
FlashInferBackend,
TritonAttentionBackend,
name="ReversedCompositeBackend",
backend_name="CUSTOM",
module=__name__,
routing_policy=ReversedRouting,
)
with set_current_vllm_config(cfg), set_default_torch_dtype(DTYPE):
layer = Attention(
32,
head_size,
head_size**-0.5,
num_kv_heads=16,
cache_config=cfg.cache_config,
prefix="composite_test",
per_layer_sliding_window=window,
attn_backend=backend,
).to(DEVICE)
layer.mm_prefix_clamp_sliding_window = window is not None
assert backend.get_builder_cls().get_cudagraph_support(cfg, spec) == (
AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
)
if window:
layer.mm_prefix_clamp_sliding_window = False
assert backend.get_builder_cls().get_cudagraph_support(cfg, spec) == (
AttentionCGSupport.NEVER
)
layer.mm_prefix_clamp_sliding_window = True
builder = backend.get_builder_cls()(spec, ["composite_test"], cfg, DEVICE)
builder.set_kernel_block_size(block_size)
# Match production, where unwritten page tails are zero-initialized.
cache = torch.zeros(
num_blocks,
16,
block_size,
2 * head_size,
dtype=storage_dtype,
device=DEVICE,
)
keys = torch.randn(209, 16, head_size, dtype=DTYPE, device=DEVICE)
values = torch.randn_like(keys)
if cache_dtype == "fp8_e4m3":
keys = keys.to(torch.float8_e4m3fn).to(DTYPE)
values = values.to(torch.float8_e4m3fn).to(DTYPE)
table = torch.arange(num_blocks, dtype=torch.int32, device=DEVICE).unsqueeze(0)
previous = 0
for end, spans, metadata_type in [
(128, [(32, 111)], TritonAttentionMetadata),
(129, [(32, 111)], causal_metadata_type),
(145, [(32, 111)], causal_metadata_type),
(209, [(32, 111), (160, 207)], TritonAttentionMetadata),
]:
qlen = end - previous
starts = torch.tensor([0, qlen], dtype=torch.int32)
lengths = torch.tensor([end], dtype=torch.int32)
common = CommonAttentionMetadata(
query_start_loc=starts.to(DEVICE),
query_start_loc_cpu=starts,
seq_lens=lengths.to(DEVICE),
seq_lens_cpu_upper_bound=lengths,
num_reqs=1,
num_actual_tokens=qlen,
max_query_len=qlen,
max_seq_len=end,
block_table_tensor=table,
slot_mapping=torch.arange(previous, end, device=DEVICE),
mm_req_doc_ranges={0: spans},
)
metadata = builder.build(0, common)
assert isinstance(metadata, metadata_type)
if isinstance(metadata, FlashInferMetadata):
if qlen == 1:
assert isinstance(metadata.decode, FlashInferTrtllmAPIDecode)
else:
assert isinstance(metadata.prefill, TRTLLMPrefill)
query = torch.randn(qlen, 32, head_size, dtype=DTYPE, device=DEVICE)
if cache_dtype == "fp8_e4m3":
query = query.to(torch.float8_e4m3fn).to(DTYPE)
key, value = keys[previous:end], values[previous:end]
output = torch.empty_like(query)
layer.impl.do_kv_cache_update(layer, key, value, cache, common.slot_mapping)
layer.impl.forward(layer, query, key, value, cache, metadata, output)
expected = _dense_reference(
query,
[keys[:end]],
[values[:end]],
[qlen],
[end],
{0: spans},
window,
head_size**-0.5,
mm_clamp_sw=window or 0,
)
torch.testing.assert_close(output.float(), expected, atol=2e-2, rtol=2e-2)
if qlen == 1:
metadata = builder.build_for_cudagraph_capture(common)
assert isinstance(metadata, metadata_type)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
layer.impl.forward(
layer, query, key, value, cache, metadata, output
)
output.zero_()
graph.replay()
torch.testing.assert_close(
output.float(), expected, atol=2e-2, rtol=2e-2
)
previous = end
+1
View File
@@ -159,6 +159,7 @@ def test_dflash2_model_decoder_layer_cls(monkeypatch):
model_config=SimpleNamespace(
dtype=torch.float32,
is_mm_prefix_lm=False,
rswa_window=None,
),
kernel_config=SimpleNamespace(
linear_backend="auto",