[Attention] Add Triton/FlashInfer composite for multimodal prefix attention (#56305)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Codex
Lucas Wilkinson
Lucas Wilkinson
OpenAI Codex
parent
8cf9de9080
commit
82a85dc1d2
@@ -808,3 +808,142 @@ def test_fa4_hd256_impl_selection(attn_type, sliding_window, expected):
|
||||
)
|
||||
assert impl.vllm_flash_attn_version == expected
|
||||
assert impl.fa4_hd256 == (expected == 4)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def blackwell_selection():
|
||||
with (
|
||||
patch.object(
|
||||
type(current_platform),
|
||||
"get_device_capability",
|
||||
return_value=DeviceCapability(10, 0),
|
||||
),
|
||||
patch(
|
||||
"vllm.v1.attention.backends.fa_utils.is_fa_version_supported",
|
||||
return_value=True,
|
||||
),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hopper_selection():
|
||||
with (
|
||||
patch.object(
|
||||
type(current_platform),
|
||||
"get_device_capability",
|
||||
return_value=DeviceCapability(9, 0),
|
||||
),
|
||||
patch(
|
||||
"vllm.v1.attention.backends.fa_utils.is_fa_version_supported",
|
||||
return_value=True,
|
||||
),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
@blackwell_only
|
||||
@pytest.mark.parametrize("use_mm_prefix", [False, True])
|
||||
@pytest.mark.parametrize("flash_attn_version", [None, 3, 4])
|
||||
def test_hopper_mm_prefix_selects_triton_flash_attn(
|
||||
use_mm_prefix, flash_attn_version, hopper_selection
|
||||
):
|
||||
"""Hopper selects the composite only when its causal route resolves FA4."""
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
|
||||
config = EngineArgs(
|
||||
model="google/gemma-4-31B-it",
|
||||
dtype="bfloat16",
|
||||
attention_config=AttentionConfig(flash_attn_version=flash_attn_version),
|
||||
).create_engine_config()
|
||||
with set_current_vllm_config(config):
|
||||
backend = get_attn_backend(
|
||||
256, torch.bfloat16, None, use_mm_prefix=use_mm_prefix
|
||||
)
|
||||
fa4_resolved = config.attention_config.flash_attn_version == 4
|
||||
if use_mm_prefix:
|
||||
expected = "TRITON_FLASH_ATTN" if fa4_resolved else "TRITON_ATTN"
|
||||
else:
|
||||
expected = "FLASH_ATTN"
|
||||
assert backend.get_name() == expected
|
||||
|
||||
|
||||
@blackwell_only
|
||||
@pytest.mark.parametrize("use_mm_prefix", [False, True])
|
||||
@pytest.mark.parametrize("kv_cache_dtype", [None, "fp8_e4m3"])
|
||||
def test_mm_prefix_selects_composite_without_changing_causal_default(
|
||||
use_mm_prefix, kv_cache_dtype, blackwell_selection
|
||||
):
|
||||
"""The image-mask requirement must reach CUDA's automatic backend priority."""
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
|
||||
config = EngineArgs(
|
||||
model="google/gemma-4-31B-it", dtype="bfloat16"
|
||||
).create_engine_config()
|
||||
with set_current_vllm_config(config):
|
||||
backend = get_attn_backend(
|
||||
256, torch.bfloat16, kv_cache_dtype, use_mm_prefix=use_mm_prefix
|
||||
)
|
||||
expected = "TRITON_FLASHINFER" if use_mm_prefix else "FLASHINFER"
|
||||
assert backend.get_name() == expected
|
||||
|
||||
|
||||
@blackwell_only
|
||||
@pytest.mark.parametrize(
|
||||
"feature_kwargs,reason",
|
||||
[
|
||||
({"use_rswa": True}, "R-SWA not supported"),
|
||||
({"kv_cache_dtype": "int8_per_token_head"}, "kv_cache_dtype not supported"),
|
||||
({"block_size": 16}, "block_size not supported"),
|
||||
({"use_dcp": True}, "DCP not supported"),
|
||||
({"use_pcp": True}, "PCP not supported"),
|
||||
({"use_adaptive_verification": True}, "device-cpu query lens mismatch"),
|
||||
],
|
||||
)
|
||||
def test_composite_rejects_features_not_shared_by_both_routes(feature_kwargs, reason):
|
||||
"""Automatic selection must not admit features only one child can execute."""
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.v1.attention.backends.triton_flashinfer import TritonFlashInferBackend
|
||||
|
||||
config = EngineArgs(
|
||||
model="google/gemma-4-31B-it", dtype="bfloat16"
|
||||
).create_engine_config()
|
||||
args = dict(
|
||||
head_size=256,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="auto",
|
||||
block_size=128,
|
||||
use_mla=False,
|
||||
has_sink=False,
|
||||
use_sparse=False,
|
||||
use_mm_prefix=True,
|
||||
use_per_head_quant_scales=False,
|
||||
device_capability=DeviceCapability(10, 0),
|
||||
attn_type=AttentionType.DECODER,
|
||||
)
|
||||
args.update(feature_kwargs)
|
||||
with set_current_vllm_config(config):
|
||||
reasons = TritonFlashInferBackend.validate_configuration(**args)
|
||||
assert any(reason in message for message in reasons), reasons
|
||||
|
||||
|
||||
@blackwell_only
|
||||
def test_rswa_selection_does_not_reuse_causal_result(blackwell_selection):
|
||||
"""An R-SWA requirement must invalidate a previously cached causal selection."""
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
|
||||
config = EngineArgs(
|
||||
model="google/gemma-4-31B-it",
|
||||
dtype="bfloat16",
|
||||
attention_config={"backend": "TRITON_FLASHINFER"},
|
||||
).create_engine_config()
|
||||
with set_current_vllm_config(config):
|
||||
assert (
|
||||
get_attn_backend(256, torch.bfloat16, None).get_name()
|
||||
== "TRITON_FLASHINFER"
|
||||
)
|
||||
config.model_config.model_arch_config.rswa_window = 128
|
||||
with pytest.raises(ValueError, match="R-SWA"):
|
||||
get_attn_backend(256, torch.bfloat16, None)
|
||||
config.attention_config.backend = AttentionBackendEnum.TRITON_ATTN
|
||||
assert get_attn_backend(256, torch.bfloat16, None).get_name() == "TRITON_ATTN"
|
||||
|
||||
Reference in New Issue
Block a user