[Attention] Add Triton/FlashInfer composite for multimodal prefix attention (#56305)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Codex
Lucas Wilkinson
Lucas Wilkinson
OpenAI Codex
parent
8cf9de9080
commit
82a85dc1d2
@@ -808,3 +808,142 @@ def test_fa4_hd256_impl_selection(attn_type, sliding_window, expected):
|
||||
)
|
||||
assert impl.vllm_flash_attn_version == expected
|
||||
assert impl.fa4_hd256 == (expected == 4)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def blackwell_selection():
|
||||
with (
|
||||
patch.object(
|
||||
type(current_platform),
|
||||
"get_device_capability",
|
||||
return_value=DeviceCapability(10, 0),
|
||||
),
|
||||
patch(
|
||||
"vllm.v1.attention.backends.fa_utils.is_fa_version_supported",
|
||||
return_value=True,
|
||||
),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hopper_selection():
|
||||
with (
|
||||
patch.object(
|
||||
type(current_platform),
|
||||
"get_device_capability",
|
||||
return_value=DeviceCapability(9, 0),
|
||||
),
|
||||
patch(
|
||||
"vllm.v1.attention.backends.fa_utils.is_fa_version_supported",
|
||||
return_value=True,
|
||||
),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
@blackwell_only
|
||||
@pytest.mark.parametrize("use_mm_prefix", [False, True])
|
||||
@pytest.mark.parametrize("flash_attn_version", [None, 3, 4])
|
||||
def test_hopper_mm_prefix_selects_triton_flash_attn(
|
||||
use_mm_prefix, flash_attn_version, hopper_selection
|
||||
):
|
||||
"""Hopper selects the composite only when its causal route resolves FA4."""
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
|
||||
config = EngineArgs(
|
||||
model="google/gemma-4-31B-it",
|
||||
dtype="bfloat16",
|
||||
attention_config=AttentionConfig(flash_attn_version=flash_attn_version),
|
||||
).create_engine_config()
|
||||
with set_current_vllm_config(config):
|
||||
backend = get_attn_backend(
|
||||
256, torch.bfloat16, None, use_mm_prefix=use_mm_prefix
|
||||
)
|
||||
fa4_resolved = config.attention_config.flash_attn_version == 4
|
||||
if use_mm_prefix:
|
||||
expected = "TRITON_FLASH_ATTN" if fa4_resolved else "TRITON_ATTN"
|
||||
else:
|
||||
expected = "FLASH_ATTN"
|
||||
assert backend.get_name() == expected
|
||||
|
||||
|
||||
@blackwell_only
|
||||
@pytest.mark.parametrize("use_mm_prefix", [False, True])
|
||||
@pytest.mark.parametrize("kv_cache_dtype", [None, "fp8_e4m3"])
|
||||
def test_mm_prefix_selects_composite_without_changing_causal_default(
|
||||
use_mm_prefix, kv_cache_dtype, blackwell_selection
|
||||
):
|
||||
"""The image-mask requirement must reach CUDA's automatic backend priority."""
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
|
||||
config = EngineArgs(
|
||||
model="google/gemma-4-31B-it", dtype="bfloat16"
|
||||
).create_engine_config()
|
||||
with set_current_vllm_config(config):
|
||||
backend = get_attn_backend(
|
||||
256, torch.bfloat16, kv_cache_dtype, use_mm_prefix=use_mm_prefix
|
||||
)
|
||||
expected = "TRITON_FLASHINFER" if use_mm_prefix else "FLASHINFER"
|
||||
assert backend.get_name() == expected
|
||||
|
||||
|
||||
@blackwell_only
|
||||
@pytest.mark.parametrize(
|
||||
"feature_kwargs,reason",
|
||||
[
|
||||
({"use_rswa": True}, "R-SWA not supported"),
|
||||
({"kv_cache_dtype": "int8_per_token_head"}, "kv_cache_dtype not supported"),
|
||||
({"block_size": 16}, "block_size not supported"),
|
||||
({"use_dcp": True}, "DCP not supported"),
|
||||
({"use_pcp": True}, "PCP not supported"),
|
||||
({"use_adaptive_verification": True}, "device-cpu query lens mismatch"),
|
||||
],
|
||||
)
|
||||
def test_composite_rejects_features_not_shared_by_both_routes(feature_kwargs, reason):
|
||||
"""Automatic selection must not admit features only one child can execute."""
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.v1.attention.backends.triton_flashinfer import TritonFlashInferBackend
|
||||
|
||||
config = EngineArgs(
|
||||
model="google/gemma-4-31B-it", dtype="bfloat16"
|
||||
).create_engine_config()
|
||||
args = dict(
|
||||
head_size=256,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="auto",
|
||||
block_size=128,
|
||||
use_mla=False,
|
||||
has_sink=False,
|
||||
use_sparse=False,
|
||||
use_mm_prefix=True,
|
||||
use_per_head_quant_scales=False,
|
||||
device_capability=DeviceCapability(10, 0),
|
||||
attn_type=AttentionType.DECODER,
|
||||
)
|
||||
args.update(feature_kwargs)
|
||||
with set_current_vllm_config(config):
|
||||
reasons = TritonFlashInferBackend.validate_configuration(**args)
|
||||
assert any(reason in message for message in reasons), reasons
|
||||
|
||||
|
||||
@blackwell_only
|
||||
def test_rswa_selection_does_not_reuse_causal_result(blackwell_selection):
|
||||
"""An R-SWA requirement must invalidate a previously cached causal selection."""
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
|
||||
config = EngineArgs(
|
||||
model="google/gemma-4-31B-it",
|
||||
dtype="bfloat16",
|
||||
attention_config={"backend": "TRITON_FLASHINFER"},
|
||||
).create_engine_config()
|
||||
with set_current_vllm_config(config):
|
||||
assert (
|
||||
get_attn_backend(256, torch.bfloat16, None).get_name()
|
||||
== "TRITON_FLASHINFER"
|
||||
)
|
||||
config.model_config.model_arch_config.rswa_window = 128
|
||||
with pytest.raises(ValueError, match="R-SWA"):
|
||||
get_attn_backend(256, torch.bfloat16, None)
|
||||
config.attention_config.backend = AttentionBackendEnum.TRITON_ATTN
|
||||
assert get_attn_backend(256, torch.bfloat16, None).get_name() == "TRITON_ATTN"
|
||||
|
||||
@@ -638,3 +638,249 @@ def test_mm_prefix_kv_cache_path(head_size: int):
|
||||
assert not torch.allclose(expected, causal_only, atol=2e-2, rtol=2e-2), (
|
||||
"test batch does not actually exercise the mm_prefix branch"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA backend imports")
|
||||
@pytest.mark.parametrize(
|
||||
"query_len,seq_len,spans,expected,unclamped_expected",
|
||||
[
|
||||
(16, 32, [(20, 28)], True, True),
|
||||
(16, 32, [(0, 16)], False, True),
|
||||
(16, 32, [(0, 17)], True, True),
|
||||
(16, 32, [(32, 40)], False, False),
|
||||
(16, 32, [(31, 40)], False, True),
|
||||
(1, 24, [(20, 28)], False, True),
|
||||
(0, 24, [(20, 28)], False, False),
|
||||
(16, 32, [(20, 20)], False, False),
|
||||
(16, 64, [(20, 28)], False, False),
|
||||
],
|
||||
)
|
||||
def test_composite_routes_queries_that_need_image_masking(
|
||||
query_len, seq_len, spans, expected, unclamped_expected
|
||||
):
|
||||
"""Historical images and single-query steps must use the causal graph path."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.v1.attention.backends.composite import requires_mm_prefix
|
||||
|
||||
common = SimpleNamespace(
|
||||
max_query_len=query_len,
|
||||
mm_req_doc_ranges={0: spans},
|
||||
query_start_loc_cpu=torch.tensor([0, query_len]),
|
||||
seq_lens_cpu_upper_bound=torch.tensor([seq_len]),
|
||||
)
|
||||
assert requires_mm_prefix(common) is expected
|
||||
assert requires_mm_prefix(common, unclamped_window=True) is unclamped_expected
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA attention kernels")
|
||||
@pytest.mark.parametrize("head_size", [256, 512])
|
||||
@pytest.mark.parametrize(
|
||||
"reverse_children,cache_dtype",
|
||||
[(False, "auto"), (True, "auto"), (False, "fp8_e4m3")],
|
||||
)
|
||||
@pytest.mark.parametrize("causal_backend_name", ["FLASHINFER", "FLASH_ATTN"])
|
||||
def test_composite_shared_cache_across_image_and_causal_steps(
|
||||
head_size, reverse_children, cache_dtype, causal_backend_name
|
||||
):
|
||||
"""Changing routes must preserve KV writes, image masking and graph replay."""
|
||||
if causal_backend_name == "FLASHINFER":
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("The Triton/FlashInfer composite supports Blackwell")
|
||||
else:
|
||||
if not current_platform.is_device_capability(90):
|
||||
pytest.skip("The Triton/FlashAttention composite supports Hopper")
|
||||
if reverse_children or cache_dtype != "auto":
|
||||
pytest.skip("Reverse-child and FP8 coverage is backend-independent")
|
||||
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
from vllm.v1.attention.backend import AttentionCGSupport, CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.composite import (
|
||||
MMPrefixAttentionRouting,
|
||||
create_composite_attention_backend,
|
||||
)
|
||||
from vllm.v1.attention.backends.flash_attn import (
|
||||
FlashAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.flashinfer import (
|
||||
FlashInferBackend,
|
||||
FlashInferMetadata,
|
||||
FlashInferTrtllmAPIDecode,
|
||||
TRTLLMPrefill,
|
||||
)
|
||||
from vllm.v1.attention.backends.triton_attn import (
|
||||
TritonAttentionBackend,
|
||||
TritonAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.triton_flash_attn import (
|
||||
TritonFlashAttentionBackend,
|
||||
)
|
||||
from vllm.v1.attention.backends.triton_flashinfer import (
|
||||
TritonFlashInferBackend,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, SlidingWindowSpec
|
||||
|
||||
if causal_backend_name == "FLASHINFER":
|
||||
backend = TritonFlashInferBackend
|
||||
causal_metadata_type = FlashInferMetadata
|
||||
block_size = backend.get_supported_kernel_block_sizes()[0]
|
||||
else:
|
||||
backend = TritonFlashAttentionBackend
|
||||
causal_metadata_type = FlashAttentionMetadata
|
||||
block_size = backend.get_preferred_block_size(16)
|
||||
num_blocks = (209 + block_size - 1) // block_size
|
||||
torch.manual_seed(42)
|
||||
attention_config = {"backend": backend.get_name()}
|
||||
if causal_backend_name == "FLASH_ATTN":
|
||||
attention_config["flash_attn_version"] = 4
|
||||
cfg = EngineArgs(
|
||||
model="google/gemma-4-31B-it",
|
||||
dtype="bfloat16",
|
||||
max_model_len=512,
|
||||
max_num_seqs=4,
|
||||
max_num_batched_tokens=512,
|
||||
block_size=block_size,
|
||||
kv_cache_dtype=cache_dtype,
|
||||
enforce_eager=True,
|
||||
attention_config=attention_config,
|
||||
).create_engine_config()
|
||||
cfg.cache_config.kv_cache_layout = "LBHNC"
|
||||
window = 128 if head_size == 256 else None
|
||||
storage_dtype = torch.uint8 if cache_dtype == "fp8_e4m3" else DTYPE
|
||||
spec_kwargs = dict(
|
||||
block_size=block_size,
|
||||
num_kv_heads=16,
|
||||
head_size=head_size,
|
||||
dtype=storage_dtype,
|
||||
kv_quant_mode=get_kv_quant_mode(cache_dtype),
|
||||
)
|
||||
spec = (
|
||||
SlidingWindowSpec(**spec_kwargs, sliding_window=window)
|
||||
if window
|
||||
else FullAttentionSpec(**spec_kwargs)
|
||||
)
|
||||
if reverse_children:
|
||||
# Swapping the children and policy must preserve outputs and KV writes;
|
||||
# dispatch cannot depend on a particular child's class or position.
|
||||
class ReversedRouting(MMPrefixAttentionRouting):
|
||||
capture_variant = 0
|
||||
|
||||
def select(self, metadata):
|
||||
return 1 - super().select(metadata)
|
||||
|
||||
@staticmethod
|
||||
def variant_uses_mm_prefix(variant):
|
||||
return variant == 1
|
||||
|
||||
backend = create_composite_attention_backend(
|
||||
FlashInferBackend,
|
||||
TritonAttentionBackend,
|
||||
name="ReversedCompositeBackend",
|
||||
backend_name="CUSTOM",
|
||||
module=__name__,
|
||||
routing_policy=ReversedRouting,
|
||||
)
|
||||
with set_current_vllm_config(cfg), set_default_torch_dtype(DTYPE):
|
||||
layer = Attention(
|
||||
32,
|
||||
head_size,
|
||||
head_size**-0.5,
|
||||
num_kv_heads=16,
|
||||
cache_config=cfg.cache_config,
|
||||
prefix="composite_test",
|
||||
per_layer_sliding_window=window,
|
||||
attn_backend=backend,
|
||||
).to(DEVICE)
|
||||
layer.mm_prefix_clamp_sliding_window = window is not None
|
||||
assert backend.get_builder_cls().get_cudagraph_support(cfg, spec) == (
|
||||
AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
||||
)
|
||||
if window:
|
||||
layer.mm_prefix_clamp_sliding_window = False
|
||||
assert backend.get_builder_cls().get_cudagraph_support(cfg, spec) == (
|
||||
AttentionCGSupport.NEVER
|
||||
)
|
||||
layer.mm_prefix_clamp_sliding_window = True
|
||||
builder = backend.get_builder_cls()(spec, ["composite_test"], cfg, DEVICE)
|
||||
builder.set_kernel_block_size(block_size)
|
||||
# Match production, where unwritten page tails are zero-initialized.
|
||||
cache = torch.zeros(
|
||||
num_blocks,
|
||||
16,
|
||||
block_size,
|
||||
2 * head_size,
|
||||
dtype=storage_dtype,
|
||||
device=DEVICE,
|
||||
)
|
||||
keys = torch.randn(209, 16, head_size, dtype=DTYPE, device=DEVICE)
|
||||
values = torch.randn_like(keys)
|
||||
if cache_dtype == "fp8_e4m3":
|
||||
keys = keys.to(torch.float8_e4m3fn).to(DTYPE)
|
||||
values = values.to(torch.float8_e4m3fn).to(DTYPE)
|
||||
table = torch.arange(num_blocks, dtype=torch.int32, device=DEVICE).unsqueeze(0)
|
||||
previous = 0
|
||||
for end, spans, metadata_type in [
|
||||
(128, [(32, 111)], TritonAttentionMetadata),
|
||||
(129, [(32, 111)], causal_metadata_type),
|
||||
(145, [(32, 111)], causal_metadata_type),
|
||||
(209, [(32, 111), (160, 207)], TritonAttentionMetadata),
|
||||
]:
|
||||
qlen = end - previous
|
||||
starts = torch.tensor([0, qlen], dtype=torch.int32)
|
||||
lengths = torch.tensor([end], dtype=torch.int32)
|
||||
common = CommonAttentionMetadata(
|
||||
query_start_loc=starts.to(DEVICE),
|
||||
query_start_loc_cpu=starts,
|
||||
seq_lens=lengths.to(DEVICE),
|
||||
seq_lens_cpu_upper_bound=lengths,
|
||||
num_reqs=1,
|
||||
num_actual_tokens=qlen,
|
||||
max_query_len=qlen,
|
||||
max_seq_len=end,
|
||||
block_table_tensor=table,
|
||||
slot_mapping=torch.arange(previous, end, device=DEVICE),
|
||||
mm_req_doc_ranges={0: spans},
|
||||
)
|
||||
metadata = builder.build(0, common)
|
||||
assert isinstance(metadata, metadata_type)
|
||||
if isinstance(metadata, FlashInferMetadata):
|
||||
if qlen == 1:
|
||||
assert isinstance(metadata.decode, FlashInferTrtllmAPIDecode)
|
||||
else:
|
||||
assert isinstance(metadata.prefill, TRTLLMPrefill)
|
||||
query = torch.randn(qlen, 32, head_size, dtype=DTYPE, device=DEVICE)
|
||||
if cache_dtype == "fp8_e4m3":
|
||||
query = query.to(torch.float8_e4m3fn).to(DTYPE)
|
||||
key, value = keys[previous:end], values[previous:end]
|
||||
output = torch.empty_like(query)
|
||||
layer.impl.do_kv_cache_update(layer, key, value, cache, common.slot_mapping)
|
||||
layer.impl.forward(layer, query, key, value, cache, metadata, output)
|
||||
expected = _dense_reference(
|
||||
query,
|
||||
[keys[:end]],
|
||||
[values[:end]],
|
||||
[qlen],
|
||||
[end],
|
||||
{0: spans},
|
||||
window,
|
||||
head_size**-0.5,
|
||||
mm_clamp_sw=window or 0,
|
||||
)
|
||||
torch.testing.assert_close(output.float(), expected, atol=2e-2, rtol=2e-2)
|
||||
if qlen == 1:
|
||||
metadata = builder.build_for_cudagraph_capture(common)
|
||||
assert isinstance(metadata, metadata_type)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
layer.impl.forward(
|
||||
layer, query, key, value, cache, metadata, output
|
||||
)
|
||||
output.zero_()
|
||||
graph.replay()
|
||||
torch.testing.assert_close(
|
||||
output.float(), expected, atol=2e-2, rtol=2e-2
|
||||
)
|
||||
previous = end
|
||||
|
||||
@@ -159,6 +159,7 @@ def test_dflash2_model_decoder_layer_cls(monkeypatch):
|
||||
model_config=SimpleNamespace(
|
||||
dtype=torch.float32,
|
||||
is_mm_prefix_lm=False,
|
||||
rswa_window=None,
|
||||
),
|
||||
kernel_config=SimpleNamespace(
|
||||
linear_backend="auto",
|
||||
|
||||
Reference in New Issue
Block a user