[Attention] Add Triton/FlashInfer composite for multimodal prefix attention (#56305)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Codex
Lucas Wilkinson
Lucas Wilkinson
OpenAI Codex
parent
8cf9de9080
commit
82a85dc1d2
@@ -56,6 +56,55 @@ llm = LLM(
|
||||
|
||||
## Backend Selection Behavior
|
||||
|
||||
### Triton/FlashAttention Composite
|
||||
|
||||
On Hopper, `TRITON_FLASH_ATTN` is preferred for compatible multimodal-prefix
|
||||
configurations. It uses Triton when the current queries need bidirectional
|
||||
image attention and FlashAttention for causal text prefills and decode, with a
|
||||
shared KV cache. The causal child must resolve to FA4. This happens
|
||||
automatically for FA4-only shapes such as head size 512 and models whose
|
||||
version policy promotes all layers to FA4, including Gemma 4. A standalone
|
||||
head-size-256 configuration whose version policy selects FA3 falls back to
|
||||
Triton for the whole layer.
|
||||
|
||||
No attention override is needed for supported multimodal models:
|
||||
|
||||
```bash
|
||||
vllm serve google/gemma-4-31B-it
|
||||
```
|
||||
|
||||
### Triton/FlashInfer Composite
|
||||
|
||||
On Blackwell, `TRITON_FLASHINFER` is preferred for compatible
|
||||
multimodal-prefix configurations, including Gemma 4 with BF16 or FP8 KV cache. It uses
|
||||
Triton when a batch's current queries require bidirectional image attention,
|
||||
and FlashInfer for causal text prefills and decode. Historical image tokens
|
||||
alone do not select Triton.
|
||||
|
||||
This instantiates `create_composite_attention_backend` from
|
||||
`vllm/v1/attention/backends/composite.py` with Triton,
|
||||
FlashInfer, and `MMPrefixAttentionRouting`. The reusable factory owns child
|
||||
implementations, metadata dispatch, compatible cache requirements, and workspace
|
||||
sharing. The routing policy selects the child and defines graph-capture safety.
|
||||
Other combinations can reuse the same machinery.
|
||||
|
||||
The backend supports head dimensions 256/512, FP16/BF16 and FP8 KV cache,
|
||||
and 64-token kernel pages with a head-major cache layout. TRTLLM handles causal
|
||||
attention at both head dimensions; its hdim512 kernels do not support 128-token
|
||||
pages.
|
||||
For Gemma 4, full CUDA graphs cover single-token batches; multi-token batches
|
||||
use the non-full-graph execution path. Models whose image masks extend beyond
|
||||
the sliding window cannot use full attention graphs with this composite.
|
||||
Context parallelism, R-SWA, attention sinks,
|
||||
and adaptive verification are not supported by this composite.
|
||||
|
||||
It can also be selected explicitly:
|
||||
|
||||
```bash
|
||||
vllm serve google/gemma-4-31B-it \
|
||||
--attention-backend TRITON_FLASHINFER
|
||||
```
|
||||
|
||||
### Manual Selection
|
||||
|
||||
When you explicitly set a backend via `--attention-backend` or `AttentionConfig`:
|
||||
|
||||
@@ -808,3 +808,142 @@ def test_fa4_hd256_impl_selection(attn_type, sliding_window, expected):
|
||||
)
|
||||
assert impl.vllm_flash_attn_version == expected
|
||||
assert impl.fa4_hd256 == (expected == 4)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def blackwell_selection():
|
||||
with (
|
||||
patch.object(
|
||||
type(current_platform),
|
||||
"get_device_capability",
|
||||
return_value=DeviceCapability(10, 0),
|
||||
),
|
||||
patch(
|
||||
"vllm.v1.attention.backends.fa_utils.is_fa_version_supported",
|
||||
return_value=True,
|
||||
),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hopper_selection():
|
||||
with (
|
||||
patch.object(
|
||||
type(current_platform),
|
||||
"get_device_capability",
|
||||
return_value=DeviceCapability(9, 0),
|
||||
),
|
||||
patch(
|
||||
"vllm.v1.attention.backends.fa_utils.is_fa_version_supported",
|
||||
return_value=True,
|
||||
),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
@blackwell_only
|
||||
@pytest.mark.parametrize("use_mm_prefix", [False, True])
|
||||
@pytest.mark.parametrize("flash_attn_version", [None, 3, 4])
|
||||
def test_hopper_mm_prefix_selects_triton_flash_attn(
|
||||
use_mm_prefix, flash_attn_version, hopper_selection
|
||||
):
|
||||
"""Hopper selects the composite only when its causal route resolves FA4."""
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
|
||||
config = EngineArgs(
|
||||
model="google/gemma-4-31B-it",
|
||||
dtype="bfloat16",
|
||||
attention_config=AttentionConfig(flash_attn_version=flash_attn_version),
|
||||
).create_engine_config()
|
||||
with set_current_vllm_config(config):
|
||||
backend = get_attn_backend(
|
||||
256, torch.bfloat16, None, use_mm_prefix=use_mm_prefix
|
||||
)
|
||||
fa4_resolved = config.attention_config.flash_attn_version == 4
|
||||
if use_mm_prefix:
|
||||
expected = "TRITON_FLASH_ATTN" if fa4_resolved else "TRITON_ATTN"
|
||||
else:
|
||||
expected = "FLASH_ATTN"
|
||||
assert backend.get_name() == expected
|
||||
|
||||
|
||||
@blackwell_only
|
||||
@pytest.mark.parametrize("use_mm_prefix", [False, True])
|
||||
@pytest.mark.parametrize("kv_cache_dtype", [None, "fp8_e4m3"])
|
||||
def test_mm_prefix_selects_composite_without_changing_causal_default(
|
||||
use_mm_prefix, kv_cache_dtype, blackwell_selection
|
||||
):
|
||||
"""The image-mask requirement must reach CUDA's automatic backend priority."""
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
|
||||
config = EngineArgs(
|
||||
model="google/gemma-4-31B-it", dtype="bfloat16"
|
||||
).create_engine_config()
|
||||
with set_current_vllm_config(config):
|
||||
backend = get_attn_backend(
|
||||
256, torch.bfloat16, kv_cache_dtype, use_mm_prefix=use_mm_prefix
|
||||
)
|
||||
expected = "TRITON_FLASHINFER" if use_mm_prefix else "FLASHINFER"
|
||||
assert backend.get_name() == expected
|
||||
|
||||
|
||||
@blackwell_only
|
||||
@pytest.mark.parametrize(
|
||||
"feature_kwargs,reason",
|
||||
[
|
||||
({"use_rswa": True}, "R-SWA not supported"),
|
||||
({"kv_cache_dtype": "int8_per_token_head"}, "kv_cache_dtype not supported"),
|
||||
({"block_size": 16}, "block_size not supported"),
|
||||
({"use_dcp": True}, "DCP not supported"),
|
||||
({"use_pcp": True}, "PCP not supported"),
|
||||
({"use_adaptive_verification": True}, "device-cpu query lens mismatch"),
|
||||
],
|
||||
)
|
||||
def test_composite_rejects_features_not_shared_by_both_routes(feature_kwargs, reason):
|
||||
"""Automatic selection must not admit features only one child can execute."""
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.v1.attention.backends.triton_flashinfer import TritonFlashInferBackend
|
||||
|
||||
config = EngineArgs(
|
||||
model="google/gemma-4-31B-it", dtype="bfloat16"
|
||||
).create_engine_config()
|
||||
args = dict(
|
||||
head_size=256,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="auto",
|
||||
block_size=128,
|
||||
use_mla=False,
|
||||
has_sink=False,
|
||||
use_sparse=False,
|
||||
use_mm_prefix=True,
|
||||
use_per_head_quant_scales=False,
|
||||
device_capability=DeviceCapability(10, 0),
|
||||
attn_type=AttentionType.DECODER,
|
||||
)
|
||||
args.update(feature_kwargs)
|
||||
with set_current_vllm_config(config):
|
||||
reasons = TritonFlashInferBackend.validate_configuration(**args)
|
||||
assert any(reason in message for message in reasons), reasons
|
||||
|
||||
|
||||
@blackwell_only
|
||||
def test_rswa_selection_does_not_reuse_causal_result(blackwell_selection):
|
||||
"""An R-SWA requirement must invalidate a previously cached causal selection."""
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
|
||||
config = EngineArgs(
|
||||
model="google/gemma-4-31B-it",
|
||||
dtype="bfloat16",
|
||||
attention_config={"backend": "TRITON_FLASHINFER"},
|
||||
).create_engine_config()
|
||||
with set_current_vllm_config(config):
|
||||
assert (
|
||||
get_attn_backend(256, torch.bfloat16, None).get_name()
|
||||
== "TRITON_FLASHINFER"
|
||||
)
|
||||
config.model_config.model_arch_config.rswa_window = 128
|
||||
with pytest.raises(ValueError, match="R-SWA"):
|
||||
get_attn_backend(256, torch.bfloat16, None)
|
||||
config.attention_config.backend = AttentionBackendEnum.TRITON_ATTN
|
||||
assert get_attn_backend(256, torch.bfloat16, None).get_name() == "TRITON_ATTN"
|
||||
|
||||
@@ -638,3 +638,249 @@ def test_mm_prefix_kv_cache_path(head_size: int):
|
||||
assert not torch.allclose(expected, causal_only, atol=2e-2, rtol=2e-2), (
|
||||
"test batch does not actually exercise the mm_prefix branch"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA backend imports")
|
||||
@pytest.mark.parametrize(
|
||||
"query_len,seq_len,spans,expected,unclamped_expected",
|
||||
[
|
||||
(16, 32, [(20, 28)], True, True),
|
||||
(16, 32, [(0, 16)], False, True),
|
||||
(16, 32, [(0, 17)], True, True),
|
||||
(16, 32, [(32, 40)], False, False),
|
||||
(16, 32, [(31, 40)], False, True),
|
||||
(1, 24, [(20, 28)], False, True),
|
||||
(0, 24, [(20, 28)], False, False),
|
||||
(16, 32, [(20, 20)], False, False),
|
||||
(16, 64, [(20, 28)], False, False),
|
||||
],
|
||||
)
|
||||
def test_composite_routes_queries_that_need_image_masking(
|
||||
query_len, seq_len, spans, expected, unclamped_expected
|
||||
):
|
||||
"""Historical images and single-query steps must use the causal graph path."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.v1.attention.backends.composite import requires_mm_prefix
|
||||
|
||||
common = SimpleNamespace(
|
||||
max_query_len=query_len,
|
||||
mm_req_doc_ranges={0: spans},
|
||||
query_start_loc_cpu=torch.tensor([0, query_len]),
|
||||
seq_lens_cpu_upper_bound=torch.tensor([seq_len]),
|
||||
)
|
||||
assert requires_mm_prefix(common) is expected
|
||||
assert requires_mm_prefix(common, unclamped_window=True) is unclamped_expected
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA attention kernels")
|
||||
@pytest.mark.parametrize("head_size", [256, 512])
|
||||
@pytest.mark.parametrize(
|
||||
"reverse_children,cache_dtype",
|
||||
[(False, "auto"), (True, "auto"), (False, "fp8_e4m3")],
|
||||
)
|
||||
@pytest.mark.parametrize("causal_backend_name", ["FLASHINFER", "FLASH_ATTN"])
|
||||
def test_composite_shared_cache_across_image_and_causal_steps(
|
||||
head_size, reverse_children, cache_dtype, causal_backend_name
|
||||
):
|
||||
"""Changing routes must preserve KV writes, image masking and graph replay."""
|
||||
if causal_backend_name == "FLASHINFER":
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("The Triton/FlashInfer composite supports Blackwell")
|
||||
else:
|
||||
if not current_platform.is_device_capability(90):
|
||||
pytest.skip("The Triton/FlashAttention composite supports Hopper")
|
||||
if reverse_children or cache_dtype != "auto":
|
||||
pytest.skip("Reverse-child and FP8 coverage is backend-independent")
|
||||
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
from vllm.v1.attention.backend import AttentionCGSupport, CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.composite import (
|
||||
MMPrefixAttentionRouting,
|
||||
create_composite_attention_backend,
|
||||
)
|
||||
from vllm.v1.attention.backends.flash_attn import (
|
||||
FlashAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.flashinfer import (
|
||||
FlashInferBackend,
|
||||
FlashInferMetadata,
|
||||
FlashInferTrtllmAPIDecode,
|
||||
TRTLLMPrefill,
|
||||
)
|
||||
from vllm.v1.attention.backends.triton_attn import (
|
||||
TritonAttentionBackend,
|
||||
TritonAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.triton_flash_attn import (
|
||||
TritonFlashAttentionBackend,
|
||||
)
|
||||
from vllm.v1.attention.backends.triton_flashinfer import (
|
||||
TritonFlashInferBackend,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, SlidingWindowSpec
|
||||
|
||||
if causal_backend_name == "FLASHINFER":
|
||||
backend = TritonFlashInferBackend
|
||||
causal_metadata_type = FlashInferMetadata
|
||||
block_size = backend.get_supported_kernel_block_sizes()[0]
|
||||
else:
|
||||
backend = TritonFlashAttentionBackend
|
||||
causal_metadata_type = FlashAttentionMetadata
|
||||
block_size = backend.get_preferred_block_size(16)
|
||||
num_blocks = (209 + block_size - 1) // block_size
|
||||
torch.manual_seed(42)
|
||||
attention_config = {"backend": backend.get_name()}
|
||||
if causal_backend_name == "FLASH_ATTN":
|
||||
attention_config["flash_attn_version"] = 4
|
||||
cfg = EngineArgs(
|
||||
model="google/gemma-4-31B-it",
|
||||
dtype="bfloat16",
|
||||
max_model_len=512,
|
||||
max_num_seqs=4,
|
||||
max_num_batched_tokens=512,
|
||||
block_size=block_size,
|
||||
kv_cache_dtype=cache_dtype,
|
||||
enforce_eager=True,
|
||||
attention_config=attention_config,
|
||||
).create_engine_config()
|
||||
cfg.cache_config.kv_cache_layout = "LBHNC"
|
||||
window = 128 if head_size == 256 else None
|
||||
storage_dtype = torch.uint8 if cache_dtype == "fp8_e4m3" else DTYPE
|
||||
spec_kwargs = dict(
|
||||
block_size=block_size,
|
||||
num_kv_heads=16,
|
||||
head_size=head_size,
|
||||
dtype=storage_dtype,
|
||||
kv_quant_mode=get_kv_quant_mode(cache_dtype),
|
||||
)
|
||||
spec = (
|
||||
SlidingWindowSpec(**spec_kwargs, sliding_window=window)
|
||||
if window
|
||||
else FullAttentionSpec(**spec_kwargs)
|
||||
)
|
||||
if reverse_children:
|
||||
# Swapping the children and policy must preserve outputs and KV writes;
|
||||
# dispatch cannot depend on a particular child's class or position.
|
||||
class ReversedRouting(MMPrefixAttentionRouting):
|
||||
capture_variant = 0
|
||||
|
||||
def select(self, metadata):
|
||||
return 1 - super().select(metadata)
|
||||
|
||||
@staticmethod
|
||||
def variant_uses_mm_prefix(variant):
|
||||
return variant == 1
|
||||
|
||||
backend = create_composite_attention_backend(
|
||||
FlashInferBackend,
|
||||
TritonAttentionBackend,
|
||||
name="ReversedCompositeBackend",
|
||||
backend_name="CUSTOM",
|
||||
module=__name__,
|
||||
routing_policy=ReversedRouting,
|
||||
)
|
||||
with set_current_vllm_config(cfg), set_default_torch_dtype(DTYPE):
|
||||
layer = Attention(
|
||||
32,
|
||||
head_size,
|
||||
head_size**-0.5,
|
||||
num_kv_heads=16,
|
||||
cache_config=cfg.cache_config,
|
||||
prefix="composite_test",
|
||||
per_layer_sliding_window=window,
|
||||
attn_backend=backend,
|
||||
).to(DEVICE)
|
||||
layer.mm_prefix_clamp_sliding_window = window is not None
|
||||
assert backend.get_builder_cls().get_cudagraph_support(cfg, spec) == (
|
||||
AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
||||
)
|
||||
if window:
|
||||
layer.mm_prefix_clamp_sliding_window = False
|
||||
assert backend.get_builder_cls().get_cudagraph_support(cfg, spec) == (
|
||||
AttentionCGSupport.NEVER
|
||||
)
|
||||
layer.mm_prefix_clamp_sliding_window = True
|
||||
builder = backend.get_builder_cls()(spec, ["composite_test"], cfg, DEVICE)
|
||||
builder.set_kernel_block_size(block_size)
|
||||
# Match production, where unwritten page tails are zero-initialized.
|
||||
cache = torch.zeros(
|
||||
num_blocks,
|
||||
16,
|
||||
block_size,
|
||||
2 * head_size,
|
||||
dtype=storage_dtype,
|
||||
device=DEVICE,
|
||||
)
|
||||
keys = torch.randn(209, 16, head_size, dtype=DTYPE, device=DEVICE)
|
||||
values = torch.randn_like(keys)
|
||||
if cache_dtype == "fp8_e4m3":
|
||||
keys = keys.to(torch.float8_e4m3fn).to(DTYPE)
|
||||
values = values.to(torch.float8_e4m3fn).to(DTYPE)
|
||||
table = torch.arange(num_blocks, dtype=torch.int32, device=DEVICE).unsqueeze(0)
|
||||
previous = 0
|
||||
for end, spans, metadata_type in [
|
||||
(128, [(32, 111)], TritonAttentionMetadata),
|
||||
(129, [(32, 111)], causal_metadata_type),
|
||||
(145, [(32, 111)], causal_metadata_type),
|
||||
(209, [(32, 111), (160, 207)], TritonAttentionMetadata),
|
||||
]:
|
||||
qlen = end - previous
|
||||
starts = torch.tensor([0, qlen], dtype=torch.int32)
|
||||
lengths = torch.tensor([end], dtype=torch.int32)
|
||||
common = CommonAttentionMetadata(
|
||||
query_start_loc=starts.to(DEVICE),
|
||||
query_start_loc_cpu=starts,
|
||||
seq_lens=lengths.to(DEVICE),
|
||||
seq_lens_cpu_upper_bound=lengths,
|
||||
num_reqs=1,
|
||||
num_actual_tokens=qlen,
|
||||
max_query_len=qlen,
|
||||
max_seq_len=end,
|
||||
block_table_tensor=table,
|
||||
slot_mapping=torch.arange(previous, end, device=DEVICE),
|
||||
mm_req_doc_ranges={0: spans},
|
||||
)
|
||||
metadata = builder.build(0, common)
|
||||
assert isinstance(metadata, metadata_type)
|
||||
if isinstance(metadata, FlashInferMetadata):
|
||||
if qlen == 1:
|
||||
assert isinstance(metadata.decode, FlashInferTrtllmAPIDecode)
|
||||
else:
|
||||
assert isinstance(metadata.prefill, TRTLLMPrefill)
|
||||
query = torch.randn(qlen, 32, head_size, dtype=DTYPE, device=DEVICE)
|
||||
if cache_dtype == "fp8_e4m3":
|
||||
query = query.to(torch.float8_e4m3fn).to(DTYPE)
|
||||
key, value = keys[previous:end], values[previous:end]
|
||||
output = torch.empty_like(query)
|
||||
layer.impl.do_kv_cache_update(layer, key, value, cache, common.slot_mapping)
|
||||
layer.impl.forward(layer, query, key, value, cache, metadata, output)
|
||||
expected = _dense_reference(
|
||||
query,
|
||||
[keys[:end]],
|
||||
[values[:end]],
|
||||
[qlen],
|
||||
[end],
|
||||
{0: spans},
|
||||
window,
|
||||
head_size**-0.5,
|
||||
mm_clamp_sw=window or 0,
|
||||
)
|
||||
torch.testing.assert_close(output.float(), expected, atol=2e-2, rtol=2e-2)
|
||||
if qlen == 1:
|
||||
metadata = builder.build_for_cudagraph_capture(common)
|
||||
assert isinstance(metadata, metadata_type)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
layer.impl.forward(
|
||||
layer, query, key, value, cache, metadata, output
|
||||
)
|
||||
output.zero_()
|
||||
graph.replay()
|
||||
torch.testing.assert_close(
|
||||
output.float(), expected, atol=2e-2, rtol=2e-2
|
||||
)
|
||||
previous = end
|
||||
|
||||
@@ -159,6 +159,7 @@ def test_dflash2_model_decoder_layer_cls(monkeypatch):
|
||||
model_config=SimpleNamespace(
|
||||
dtype=torch.float32,
|
||||
is_mm_prefix_lm=False,
|
||||
rswa_window=None,
|
||||
),
|
||||
kernel_config=SimpleNamespace(
|
||||
linear_backend="auto",
|
||||
|
||||
@@ -87,6 +87,7 @@ def _get_backend_priorities(
|
||||
kv_cache_dtype: CacheDType | None = None,
|
||||
use_non_causal: bool = False,
|
||||
head_size: int | None = None,
|
||||
use_mm_prefix: bool = False,
|
||||
) -> list[AttentionBackendEnum]:
|
||||
"""Get backend priorities with lazy import to avoid circular dependency."""
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
@@ -156,6 +157,7 @@ def _get_backend_priorities(
|
||||
# So prefer FlashAttention when non-causal on SM100f.
|
||||
if device_capability.major == 10 and not use_non_causal:
|
||||
return [
|
||||
*([AttentionBackendEnum.TRITON_FLASHINFER] if use_mm_prefix else []),
|
||||
AttentionBackendEnum.FLASHINFER,
|
||||
AttentionBackendEnum.FLASH_ATTN,
|
||||
AttentionBackendEnum.TRITON_ATTN,
|
||||
@@ -164,6 +166,11 @@ def _get_backend_priorities(
|
||||
]
|
||||
else:
|
||||
return [
|
||||
*(
|
||||
[AttentionBackendEnum.TRITON_FLASH_ATTN]
|
||||
if device_capability.major == 9 and use_mm_prefix
|
||||
else []
|
||||
),
|
||||
AttentionBackendEnum.FLASH_ATTN,
|
||||
AttentionBackendEnum.FLASHINFER,
|
||||
AttentionBackendEnum.TRITON_ATTN,
|
||||
@@ -394,6 +401,7 @@ class CudaPlatformBase(Platform):
|
||||
kv_cache_dtype=attn_selector_config.kv_cache_dtype,
|
||||
use_non_causal=attn_selector_config.use_non_causal,
|
||||
head_size=attn_selector_config.head_size,
|
||||
use_mm_prefix=attn_selector_config.use_mm_prefix,
|
||||
)
|
||||
for priority, backend in enumerate(backend_priorities):
|
||||
try:
|
||||
|
||||
@@ -185,6 +185,10 @@ class AttentionBackend(ABC):
|
||||
def supports_sliding_window(cls) -> bool:
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def supports_rswa(cls) -> bool:
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def supports_non_causal(cls) -> bool:
|
||||
"""Check if backend supports non-causal (bidirectional) attention
|
||||
@@ -287,6 +291,7 @@ class AttentionBackend(ABC):
|
||||
use_pcp: bool = False,
|
||||
use_adaptive_verification: bool = False,
|
||||
use_dcp: bool = False,
|
||||
use_rswa: bool = False,
|
||||
) -> list[str]:
|
||||
invalid_reasons = []
|
||||
if not cls.supports_head_size(head_size):
|
||||
@@ -321,6 +326,8 @@ class AttentionBackend(ABC):
|
||||
invalid_reasons.append(f"attention type {attn_type} not supported")
|
||||
if has_sliding_window and not cls.supports_sliding_window():
|
||||
invalid_reasons.append("sliding window not supported")
|
||||
if use_rswa and not cls.supports_rswa():
|
||||
invalid_reasons.append("R-SWA not supported")
|
||||
if use_non_causal and not cls.supports_non_causal():
|
||||
invalid_reasons.append("non-causal attention not supported")
|
||||
if use_mla and use_non_causal and use_dcp and not cls.supports_non_causal_dcp():
|
||||
@@ -854,6 +861,9 @@ class AttentionImplBase(ABC, Generic[T]):
|
||||
def process_weights_after_loading(self, act_dtype: torch.dtype):
|
||||
pass
|
||||
|
||||
def get_impl_variants(self) -> tuple["AttentionImplBase", ...]:
|
||||
return (self,)
|
||||
|
||||
def prepare_for_batch(self, attn_metadata: T | None) -> None:
|
||||
"""Prepare implementation-specific state for the current batch."""
|
||||
|
||||
|
||||
@@ -0,0 +1,509 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Reusable attention backend composition and routing policies."""
|
||||
|
||||
import functools
|
||||
import math
|
||||
from typing import ClassVar, Protocol, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionCGSupport,
|
||||
AttentionImpl,
|
||||
AttentionMetadataBuilder,
|
||||
CommonAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import get_supported_kv_cache_layouts
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpec
|
||||
|
||||
|
||||
def requires_mm_prefix(
|
||||
metadata: CommonAttentionMetadata, *, unclamped_window: bool = False
|
||||
) -> bool:
|
||||
ranges = metadata.mm_req_doc_ranges
|
||||
# With one query per request there are no future keys to unmask. This
|
||||
# also keeps single-token prefills on the captured FlashInfer path.
|
||||
if not ranges or (metadata.max_query_len <= 1 and not unclamped_window):
|
||||
return False
|
||||
seq_lens = metadata.seq_lens_cpu_upper_bound
|
||||
assert seq_lens is not None
|
||||
starts = metadata.query_start_loc_cpu
|
||||
for req_idx, spans in ranges.items():
|
||||
query_len = int(starts[req_idx + 1] - starts[req_idx])
|
||||
if query_len == 0 or (query_len == 1 and not unclamped_window):
|
||||
continue
|
||||
end = int(seq_lens[req_idx])
|
||||
begin = end - query_len
|
||||
# An unclamped image can also expose keys behind the sliding window.
|
||||
if unclamped_window and any(
|
||||
start < stop and start < end and stop >= begin for start, stop in spans
|
||||
):
|
||||
return True
|
||||
# Prefill lengths are exact. Historical image ranges must not divert
|
||||
# later text/decode queries to the image backend.
|
||||
if any(
|
||||
start < stop and start < end - 1 and stop > begin for start, stop in spans
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _has_unclamped_window(layers) -> bool:
|
||||
return any(
|
||||
getattr(layer, "sliding_window", None) is not None
|
||||
and not getattr(layer, "mm_prefix_clamp_sliding_window", False)
|
||||
for layer in layers
|
||||
)
|
||||
|
||||
|
||||
class CompositeAttentionRouting(Protocol):
|
||||
"""Select a child backend and declare routing constraints for composition."""
|
||||
|
||||
capture_variant: ClassVar[int]
|
||||
"""Child used during graph capture; replay must select the same child."""
|
||||
|
||||
def __init__(self, layer_names: list[str], vllm_config: VllmConfig) -> None: ...
|
||||
|
||||
def select(self, metadata: CommonAttentionMetadata) -> int:
|
||||
"""Return 0 for the first child or 1 for the second child."""
|
||||
...
|
||||
|
||||
@staticmethod
|
||||
def get_cudagraph_support(
|
||||
vllm_config: VllmConfig, kv_cache_spec: KVCacheSpec
|
||||
) -> AttentionCGSupport:
|
||||
"""Limit graph support to batches whose routing stays invariant."""
|
||||
...
|
||||
|
||||
@staticmethod
|
||||
def variant_uses_mm_prefix(variant: int) -> bool:
|
||||
"""Whether this child must support the model's image-mask requirement."""
|
||||
...
|
||||
|
||||
|
||||
class MMPrefixAttentionRouting(CompositeAttentionRouting):
|
||||
"""Route image masks to variant zero and causal queries to variant one."""
|
||||
|
||||
def __init__(self, layer_names: list[str], vllm_config: VllmConfig) -> None:
|
||||
layers = vllm_config.compilation_config.static_forward_context
|
||||
self.unclamped_window = _has_unclamped_window(
|
||||
layers[name] for name in layer_names
|
||||
)
|
||||
|
||||
def select(self, metadata: CommonAttentionMetadata) -> int:
|
||||
return int(
|
||||
not requires_mm_prefix(metadata, unclamped_window=self.unclamped_window)
|
||||
)
|
||||
|
||||
capture_variant: ClassVar[int] = 1
|
||||
|
||||
@staticmethod
|
||||
def get_cudagraph_support(
|
||||
vllm_config: VllmConfig, kv_cache_spec: KVCacheSpec
|
||||
) -> AttentionCGSupport:
|
||||
if _has_unclamped_window(
|
||||
vllm_config.compilation_config.static_forward_context.values()
|
||||
):
|
||||
return AttentionCGSupport.NEVER
|
||||
return AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
||||
|
||||
@staticmethod
|
||||
def variant_uses_mm_prefix(variant: int) -> bool:
|
||||
return variant == 0
|
||||
|
||||
|
||||
def _intersect_kernel_block_sizes(
|
||||
first: type[AttentionBackend], second: type[AttentionBackend]
|
||||
):
|
||||
from vllm.v1.attention.backend import MultipleOf
|
||||
|
||||
result: list[int | MultipleOf] = []
|
||||
seen: set[tuple[type, int]] = set()
|
||||
for lhs in first.get_supported_kernel_block_sizes():
|
||||
for rhs in second.get_supported_kernel_block_sizes():
|
||||
candidate: int | MultipleOf | None = None
|
||||
if isinstance(lhs, MultipleOf) and isinstance(rhs, MultipleOf):
|
||||
candidate = MultipleOf(math.lcm(lhs.base, rhs.base))
|
||||
elif isinstance(lhs, MultipleOf) and isinstance(rhs, int):
|
||||
candidate = rhs if rhs % lhs.base == 0 else None
|
||||
elif isinstance(lhs, int) and isinstance(rhs, MultipleOf):
|
||||
candidate = lhs if lhs % rhs.base == 0 else None
|
||||
elif lhs == rhs:
|
||||
candidate = lhs
|
||||
if candidate is None:
|
||||
continue
|
||||
value = candidate.base if isinstance(candidate, MultipleOf) else candidate
|
||||
key = (type(candidate), value)
|
||||
if key not in seen:
|
||||
result.append(candidate)
|
||||
seen.add(key)
|
||||
return result
|
||||
|
||||
|
||||
@functools.cache
|
||||
def create_composite_attention_backend(
|
||||
general_backend: type[AttentionBackend],
|
||||
causal_backend: type[AttentionBackend],
|
||||
*,
|
||||
name: str,
|
||||
backend_name: str,
|
||||
module: str,
|
||||
routing_policy: type[CompositeAttentionRouting],
|
||||
head_sizes: tuple[int, ...] = (),
|
||||
kernel_block_sizes: tuple[int, ...] = (),
|
||||
device_major: int | None = None,
|
||||
) -> type[AttentionBackend]:
|
||||
"""Compose two backends with shared storage and an explicit routing policy."""
|
||||
if general_backend.full_cls_name() == causal_backend.full_cls_name():
|
||||
return general_backend
|
||||
general_impl_cls = general_backend.get_impl_cls()
|
||||
causal_impl_cls = causal_backend.get_impl_cls()
|
||||
general_builder_cls = general_backend.get_builder_cls()
|
||||
causal_builder_cls = causal_backend.get_builder_cls()
|
||||
if (
|
||||
general_backend.forward_includes_kv_cache_update
|
||||
or causal_backend.forward_includes_kv_cache_update
|
||||
):
|
||||
raise ValueError("Composite backends require separate KV cache updates")
|
||||
|
||||
class CompositeAttentionImpl(AttentionImpl):
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
self.general_impl = cast(AttentionImpl, general_impl_cls(*args, **kwargs))
|
||||
try:
|
||||
self.causal_impl = cast(AttentionImpl, causal_impl_cls(*args, **kwargs))
|
||||
except TypeError as e:
|
||||
raise ValueError(
|
||||
f"Causal attention backend {causal_backend.get_name()} "
|
||||
f"does not accept the attention arguments required by "
|
||||
f"the general backend {general_backend.get_name()}: {e}"
|
||||
) from e
|
||||
for name in (
|
||||
"num_heads",
|
||||
"num_kv_heads",
|
||||
"head_size",
|
||||
"scale",
|
||||
"kv_cache_dtype",
|
||||
):
|
||||
if hasattr(self.general_impl, name):
|
||||
setattr(self, name, getattr(self.general_impl, name))
|
||||
self.supports_quant_query_input = (
|
||||
self.general_impl.supports_quant_query_input
|
||||
and self.causal_impl.supports_quant_query_input
|
||||
)
|
||||
self.supports_pcp = self.general_impl.supports_pcp
|
||||
self.supports_dcp = self.causal_impl.supports_dcp
|
||||
self.can_return_lse_for_decode = self.causal_impl.can_return_lse_for_decode
|
||||
self.lse_base_on_e = self.causal_impl.lse_base_on_e
|
||||
self.need_to_return_lse_for_decode = (
|
||||
self.causal_impl.need_to_return_lse_for_decode
|
||||
)
|
||||
self.supports_mtp_with_cp_non_trivial_interleave_size = (
|
||||
self.causal_impl.supports_mtp_with_cp_non_trivial_interleave_size
|
||||
)
|
||||
|
||||
def get_impl_for_metadata(self, attn_metadata):
|
||||
return (
|
||||
self.causal_impl
|
||||
if getattr(attn_metadata, "_attention_backend_variant", 0)
|
||||
else self.general_impl
|
||||
)
|
||||
|
||||
def get_impl_variants(self):
|
||||
return self.general_impl, self.causal_impl
|
||||
|
||||
def forward(
|
||||
self,
|
||||
layer,
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
kv_cache,
|
||||
attn_metadata,
|
||||
output,
|
||||
output_scale=None,
|
||||
output_block_scale=None,
|
||||
):
|
||||
impl = self.get_impl_for_metadata(attn_metadata)
|
||||
return impl.forward(
|
||||
layer,
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
kv_cache,
|
||||
attn_metadata,
|
||||
output,
|
||||
output_scale,
|
||||
output_block_scale,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, act_dtype: torch.dtype):
|
||||
for impl in self.get_impl_variants():
|
||||
impl.process_weights_after_loading(act_dtype)
|
||||
|
||||
def fused_output_quant_supported(self, quant_key):
|
||||
return all(
|
||||
impl.fused_output_quant_supported(quant_key)
|
||||
for impl in self.get_impl_variants()
|
||||
)
|
||||
|
||||
def fused_qk_norm_rope_kvcache_supported(self):
|
||||
return self.general_impl.fused_qk_norm_rope_kvcache_supported()
|
||||
|
||||
def do_qk_norm_rope_kvcache_update(self, *args, **kwargs):
|
||||
method = self.general_impl.do_qk_norm_rope_kvcache_update
|
||||
return method(*args, **kwargs)
|
||||
|
||||
def fused_rope_kvcache_supported(self):
|
||||
return self.general_impl.fused_rope_kvcache_supported()
|
||||
|
||||
def do_rope_and_kv_cache_update(self, *args, **kwargs):
|
||||
method = self.general_impl.do_rope_and_kv_cache_update
|
||||
return method(*args, **kwargs)
|
||||
|
||||
def do_kv_cache_update(self, *args, **kwargs):
|
||||
method = self.general_impl.do_kv_cache_update # type: ignore[attr-defined]
|
||||
return method(*args, **kwargs)
|
||||
|
||||
class CompositeAttentionMetadataBuilder(AttentionMetadataBuilder):
|
||||
requires_block_table_width = (
|
||||
general_builder_cls.requires_block_table_width
|
||||
or causal_builder_cls.requires_block_table_width
|
||||
)
|
||||
|
||||
def __init__(self, kv_cache_spec, layer_names, vllm_config, device, **kwargs):
|
||||
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||
args = (kv_cache_spec, layer_names, vllm_config, device)
|
||||
# Only the variant that asked for the extra ctor kwargs receives them.
|
||||
self.general_builder = general_builder_cls(
|
||||
*args,
|
||||
**(kwargs if general_builder_cls.requires_block_table_width else {}),
|
||||
)
|
||||
self.causal_builder = causal_builder_cls(
|
||||
*args,
|
||||
**(kwargs if causal_builder_cls.requires_block_table_width else {}),
|
||||
)
|
||||
self.reorder_batch_threshold = min(
|
||||
(
|
||||
builder.reorder_batch_threshold
|
||||
for builder in self._builders
|
||||
if builder.reorder_batch_threshold is not None
|
||||
),
|
||||
default=None,
|
||||
)
|
||||
self.routing = routing_policy(layer_names, vllm_config)
|
||||
|
||||
def set_kernel_block_size(self, kernel_block_size):
|
||||
super().set_kernel_block_size(kernel_block_size)
|
||||
for builder in self._builders:
|
||||
builder.set_kernel_block_size(kernel_block_size)
|
||||
|
||||
@classmethod
|
||||
def get_cudagraph_support(cls, vllm_config, kv_cache_spec):
|
||||
return min(
|
||||
general_builder_cls.get_cudagraph_support(vllm_config, kv_cache_spec),
|
||||
causal_builder_cls.get_cudagraph_support(vllm_config, kv_cache_spec),
|
||||
routing_policy.get_cudagraph_support(vllm_config, kv_cache_spec),
|
||||
key=lambda support: support.value,
|
||||
)
|
||||
|
||||
def _builder(self, common_attn_metadata):
|
||||
backend_variant = self.routing.select(common_attn_metadata)
|
||||
builder = self.causal_builder if backend_variant else self.general_builder
|
||||
return builder, backend_variant
|
||||
|
||||
def _build(self, method, common_attn_metadata, **kwargs):
|
||||
builder, backend_variant = self._builder(common_attn_metadata)
|
||||
metadata = getattr(builder, method)(
|
||||
common_attn_metadata=common_attn_metadata, **kwargs
|
||||
)
|
||||
metadata._attention_backend_variant = backend_variant
|
||||
return metadata
|
||||
|
||||
def build(self, common_prefix_len, common_attn_metadata, fast_build=False):
|
||||
return self._build(
|
||||
"build",
|
||||
common_attn_metadata,
|
||||
common_prefix_len=common_prefix_len,
|
||||
fast_build=fast_build,
|
||||
)
|
||||
|
||||
def build_for_cudagraph_capture(self, common_attn_metadata):
|
||||
variant = routing_policy.capture_variant
|
||||
metadata = self._builders[variant].build_for_cudagraph_capture(
|
||||
common_attn_metadata
|
||||
)
|
||||
metadata._attention_backend_variant = variant
|
||||
return metadata
|
||||
|
||||
@property
|
||||
def _builders(self):
|
||||
return self.general_builder, self.causal_builder
|
||||
|
||||
# Forward the optional workspace protocol so that a wrapped builder
|
||||
# which allocates one (FlashInfer) still joins the runner's cross-group
|
||||
# sharing instead of allocating a second buffer behind the composite.
|
||||
def _get_workspace_buffer(self):
|
||||
for builder in self._builders:
|
||||
if hasattr(builder, "_get_workspace_buffer"):
|
||||
workspace = builder._get_workspace_buffer()
|
||||
# The runner only calls the getter for the group that
|
||||
# provides the buffer, so hand it to the sibling here.
|
||||
self.set_workspace_buffer(workspace)
|
||||
return workspace
|
||||
return None
|
||||
|
||||
def set_workspace_buffer(self, workspace_buffer):
|
||||
for builder in self._builders:
|
||||
if hasattr(builder, "set_workspace_buffer"):
|
||||
builder.set_workspace_buffer(workspace_buffer)
|
||||
|
||||
class CompositeAttentionBackend(AttentionBackend):
|
||||
forward_includes_kv_cache_update = False
|
||||
general_backend_cls = general_backend
|
||||
causal_backend_cls = causal_backend
|
||||
|
||||
@staticmethod
|
||||
def get_name():
|
||||
return backend_name
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls():
|
||||
return CompositeAttentionImpl
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls():
|
||||
return CompositeAttentionMetadataBuilder
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes():
|
||||
if kernel_block_sizes:
|
||||
return [
|
||||
size
|
||||
for size in kernel_block_sizes
|
||||
if all(
|
||||
backend.supports_block_size(size)
|
||||
for backend in (general_backend, causal_backend)
|
||||
)
|
||||
]
|
||||
return _intersect_kernel_block_sizes(general_backend, causal_backend)
|
||||
|
||||
@classmethod
|
||||
def supports_block_size(cls, block_size):
|
||||
return bool(
|
||||
cls.get_supported_kernel_block_sizes()
|
||||
) and super().supports_block_size(block_size)
|
||||
|
||||
@classmethod
|
||||
def get_supported_head_sizes(cls):
|
||||
return list(head_sizes)
|
||||
|
||||
@classmethod
|
||||
def supports_head_size(cls, head_size):
|
||||
return (not head_sizes or head_size in head_sizes) and all(
|
||||
backend.supports_head_size(head_size)
|
||||
for backend in (general_backend, causal_backend)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def supports_compute_capability(cls, capability):
|
||||
return (device_major is None or capability.major == device_major) and all(
|
||||
backend.supports_compute_capability(capability)
|
||||
for backend in (general_backend, causal_backend)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def supports_dtype(cls, dtype):
|
||||
return all(
|
||||
backend.supports_dtype(dtype)
|
||||
for backend in (general_backend, causal_backend)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def supports_kv_cache_dtype(cls, dtype):
|
||||
return all(
|
||||
backend.supports_kv_cache_dtype(dtype)
|
||||
for backend in (general_backend, causal_backend)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def supports_mm_prefix(cls):
|
||||
return all(
|
||||
not routing_policy.variant_uses_mm_prefix(variant)
|
||||
or backend.supports_mm_prefix()
|
||||
for variant, backend in enumerate((general_backend, causal_backend))
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def supports_sliding_window(cls):
|
||||
return all(
|
||||
backend.supports_sliding_window()
|
||||
for backend in (general_backend, causal_backend)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def supports_rswa(cls):
|
||||
return all(
|
||||
backend.supports_rswa() for backend in (general_backend, causal_backend)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def supports_pcp(cls):
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def supports_dcp(cls):
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def supports_device_cpu_query_lens_mismatch(cls):
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def supports_combination(
|
||||
cls,
|
||||
head_size,
|
||||
dtype,
|
||||
kv_cache_dtype,
|
||||
block_size,
|
||||
use_mla,
|
||||
has_sink,
|
||||
use_sparse,
|
||||
use_mm_prefix,
|
||||
device_capability,
|
||||
):
|
||||
for variant, backend in enumerate((general_backend, causal_backend)):
|
||||
reason = backend.supports_combination(
|
||||
head_size,
|
||||
dtype,
|
||||
kv_cache_dtype,
|
||||
block_size,
|
||||
use_mla,
|
||||
has_sink,
|
||||
use_sparse,
|
||||
use_mm_prefix and routing_policy.variant_uses_mm_prefix(variant),
|
||||
device_capability,
|
||||
)
|
||||
if reason:
|
||||
return f"{backend.get_name()}: {reason}"
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def customize_spec(cls, spec):
|
||||
general_spec = general_backend.customize_spec(spec)
|
||||
if general_spec != causal_backend.customize_spec(spec):
|
||||
raise ValueError("Composite backends require the same KV cache spec")
|
||||
return general_spec
|
||||
|
||||
@classmethod
|
||||
def supported_kv_cache_layouts(cls):
|
||||
return tuple(
|
||||
get_supported_kv_cache_layouts((general_backend, causal_backend))
|
||||
)
|
||||
|
||||
CompositeAttentionBackend.__name__ = name
|
||||
CompositeAttentionBackend.__qualname__ = name
|
||||
CompositeAttentionBackend.__module__ = module
|
||||
return CompositeAttentionBackend
|
||||
@@ -129,6 +129,10 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
def get_name() -> str:
|
||||
return "FLASH_ATTN"
|
||||
|
||||
@classmethod
|
||||
def supports_rswa(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def supports_sliding_window(cls) -> bool:
|
||||
return True
|
||||
|
||||
@@ -105,6 +105,10 @@ class FlexAttentionBackend(AttentionBackend):
|
||||
def get_name() -> str:
|
||||
return "FLEX_ATTENTION"
|
||||
|
||||
@classmethod
|
||||
def supports_rswa(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def supports_sliding_window(cls) -> bool:
|
||||
return True
|
||||
|
||||
@@ -49,6 +49,9 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta):
|
||||
TRITON_ATTN_DIFFKV = (
|
||||
"vllm.v1.attention.backends.triton_attn_diffkv.TritonAttentionDiffKVBackend"
|
||||
)
|
||||
TRITON_FLASH_ATTN = (
|
||||
"vllm.v1.attention.backends.triton_flash_attn.TritonFlashAttentionBackend"
|
||||
)
|
||||
ROCM_ATTN = "vllm.v1.attention.backends.rocm_attn.RocmAttentionBackend"
|
||||
ROCM_AITER_MLA = "vllm.v1.attention.backends.mla.rocm_aiter_mla.AiterMLABackend"
|
||||
ROCM_AITER_TRITON_MLA = (
|
||||
@@ -63,6 +66,9 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta):
|
||||
XPU_MLA_SPARSE = "vllm.v1.attention.backends.mla.xpu_mla_sparse.XPUMLASparseBackend"
|
||||
TORCH_SDPA = "" # this tag is only used for ViT
|
||||
FLASHINFER = "vllm.v1.attention.backends.flashinfer.FlashInferBackend"
|
||||
TRITON_FLASHINFER = (
|
||||
"vllm.v1.attention.backends.triton_flashinfer.TritonFlashInferBackend"
|
||||
)
|
||||
FLASHINFER_MLA = (
|
||||
"vllm.v1.attention.backends.mla.flashinfer_mla.FlashInferMLABackend"
|
||||
)
|
||||
|
||||
@@ -323,6 +323,10 @@ class TritonAttentionBackend(AttentionBackend):
|
||||
def get_name() -> str:
|
||||
return "TRITON_ATTN"
|
||||
|
||||
@classmethod
|
||||
def supports_rswa(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def supports_sliding_window(cls) -> bool:
|
||||
return True
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Triton image-mask and FlashAttention causal attention composite."""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.v1.attention.backends.composite import (
|
||||
MMPrefixAttentionRouting,
|
||||
create_composite_attention_backend,
|
||||
)
|
||||
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
from vllm.v1.attention.backends.triton_attn import TritonAttentionBackend
|
||||
|
||||
|
||||
class _FA4Backend(FlashAttentionBackend):
|
||||
@classmethod
|
||||
def supports_combination(
|
||||
cls,
|
||||
head_size: int,
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: CacheDType | None,
|
||||
block_size: int | None,
|
||||
use_mla: bool,
|
||||
has_sink: bool,
|
||||
use_sparse: bool,
|
||||
use_mm_prefix: bool,
|
||||
device_capability: DeviceCapability,
|
||||
) -> str | None:
|
||||
reason = super().supports_combination(
|
||||
head_size,
|
||||
dtype,
|
||||
kv_cache_dtype,
|
||||
block_size,
|
||||
use_mla,
|
||||
has_sink,
|
||||
use_sparse,
|
||||
use_mm_prefix,
|
||||
device_capability,
|
||||
)
|
||||
if reason is not None:
|
||||
return reason
|
||||
if (
|
||||
get_flash_attn_version(
|
||||
head_size=head_size,
|
||||
has_sinks=has_sink,
|
||||
kv_cache_block_size=block_size,
|
||||
supports_fa4_hd256=True,
|
||||
)
|
||||
!= 4
|
||||
):
|
||||
return "causal route requires FlashAttention v4"
|
||||
return None
|
||||
|
||||
|
||||
TritonFlashAttentionBackend = create_composite_attention_backend(
|
||||
TritonAttentionBackend,
|
||||
_FA4Backend,
|
||||
name="TritonFlashAttentionBackend",
|
||||
backend_name="TRITON_FLASH_ATTN",
|
||||
module=__name__,
|
||||
routing_policy=MMPrefixAttentionRouting,
|
||||
head_sizes=(256, 512),
|
||||
device_major=9,
|
||||
)
|
||||
@@ -0,0 +1,22 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Triton image-mask and FlashInfer causal attention composite."""
|
||||
|
||||
from vllm.v1.attention.backends.composite import (
|
||||
MMPrefixAttentionRouting,
|
||||
create_composite_attention_backend,
|
||||
)
|
||||
from vllm.v1.attention.backends.flashinfer import FlashInferBackend
|
||||
from vllm.v1.attention.backends.triton_attn import TritonAttentionBackend
|
||||
|
||||
TritonFlashInferBackend = create_composite_attention_backend(
|
||||
TritonAttentionBackend,
|
||||
FlashInferBackend,
|
||||
name="TritonFlashInferBackend",
|
||||
backend_name="TRITON_FLASHINFER",
|
||||
module=__name__,
|
||||
routing_policy=MMPrefixAttentionRouting,
|
||||
head_sizes=(256, 512),
|
||||
kernel_block_sizes=(64,),
|
||||
device_major=10,
|
||||
)
|
||||
@@ -8,6 +8,7 @@ from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Protocol,
|
||||
TypeVar,
|
||||
cast,
|
||||
)
|
||||
|
||||
@@ -45,6 +46,19 @@ logger = init_logger(__name__)
|
||||
PAD_SLOT_ID = -1
|
||||
NULL_BLOCK_ID = 0
|
||||
|
||||
|
||||
ImplT = TypeVar("ImplT", bound=AttentionImpl)
|
||||
|
||||
|
||||
def find_attention_impl_variant(
|
||||
impl: AttentionImpl, impl_cls: type[ImplT]
|
||||
) -> ImplT | None:
|
||||
for variant in impl.get_impl_variants():
|
||||
if isinstance(variant, impl_cls):
|
||||
return cast(ImplT, variant)
|
||||
return None
|
||||
|
||||
|
||||
_LN_2 = math.log(2.0)
|
||||
|
||||
|
||||
@@ -342,8 +356,8 @@ def get_per_layer_parameters(
|
||||
per_layer_params: dict[str, PerLayerParameters] = {}
|
||||
|
||||
for key, layer in layers.items():
|
||||
impl = layer.impl
|
||||
assert isinstance(impl, cls_)
|
||||
impl = find_attention_impl_variant(layer.impl, cls_)
|
||||
assert impl is not None
|
||||
|
||||
# Infer hyperparameters from the attention layer
|
||||
window_size = getattr(impl, "sliding_window", None)
|
||||
@@ -1013,6 +1027,9 @@ def create_fast_prefill_custom_backend(
|
||||
common_attn_metadata.logits_indices_padded
|
||||
)
|
||||
self.num_logits_indices = common_attn_metadata.num_logits_indices
|
||||
self._attention_backend_variant = getattr(
|
||||
metadata, "_attention_backend_variant", 0
|
||||
)
|
||||
|
||||
return KVSharingFastPrefillAttentionMetadata(metadata, common_attn_metadata)
|
||||
|
||||
|
||||
@@ -36,6 +36,7 @@ class AttentionSelectorConfig(NamedTuple):
|
||||
use_pcp: bool = False
|
||||
use_adaptive_verification: bool = False
|
||||
use_dcp: bool = False
|
||||
use_rswa: bool = False
|
||||
|
||||
def __repr__(self):
|
||||
return (
|
||||
@@ -55,7 +56,8 @@ class AttentionSelectorConfig(NamedTuple):
|
||||
f"use_kv_connector={self.use_kv_connector}, "
|
||||
f"use_adaptive_verification={self.use_adaptive_verification}, "
|
||||
f"use_pcp={self.use_pcp}, "
|
||||
f"use_dcp={self.use_dcp})"
|
||||
f"use_dcp={self.use_dcp}, "
|
||||
f"use_rswa={self.use_rswa})"
|
||||
)
|
||||
|
||||
|
||||
@@ -168,6 +170,10 @@ def get_attn_backend(
|
||||
use_pcp=vllm_config.parallel_config.prefill_context_parallel_size > 1,
|
||||
use_adaptive_verification=use_adaptive_verification,
|
||||
use_dcp=vllm_config.parallel_config.decode_context_parallel_size > 1,
|
||||
use_rswa=(
|
||||
vllm_config.model_config is not None
|
||||
and vllm_config.model_config.rswa_window is not None
|
||||
),
|
||||
)
|
||||
|
||||
# A per-KV-group override (keyed by KVCacheSpecKind) takes precedence over
|
||||
|
||||
Reference in New Issue
Block a user