[Attention] Add Triton/FlashInfer composite for multimodal prefix attention (#56305)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Codex
Lucas Wilkinson
Lucas Wilkinson
OpenAI Codex
parent
8cf9de9080
commit
82a85dc1d2
@@ -87,6 +87,7 @@ def _get_backend_priorities(
|
||||
kv_cache_dtype: CacheDType | None = None,
|
||||
use_non_causal: bool = False,
|
||||
head_size: int | None = None,
|
||||
use_mm_prefix: bool = False,
|
||||
) -> list[AttentionBackendEnum]:
|
||||
"""Get backend priorities with lazy import to avoid circular dependency."""
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
@@ -156,6 +157,7 @@ def _get_backend_priorities(
|
||||
# So prefer FlashAttention when non-causal on SM100f.
|
||||
if device_capability.major == 10 and not use_non_causal:
|
||||
return [
|
||||
*([AttentionBackendEnum.TRITON_FLASHINFER] if use_mm_prefix else []),
|
||||
AttentionBackendEnum.FLASHINFER,
|
||||
AttentionBackendEnum.FLASH_ATTN,
|
||||
AttentionBackendEnum.TRITON_ATTN,
|
||||
@@ -164,6 +166,11 @@ def _get_backend_priorities(
|
||||
]
|
||||
else:
|
||||
return [
|
||||
*(
|
||||
[AttentionBackendEnum.TRITON_FLASH_ATTN]
|
||||
if device_capability.major == 9 and use_mm_prefix
|
||||
else []
|
||||
),
|
||||
AttentionBackendEnum.FLASH_ATTN,
|
||||
AttentionBackendEnum.FLASHINFER,
|
||||
AttentionBackendEnum.TRITON_ATTN,
|
||||
@@ -394,6 +401,7 @@ class CudaPlatformBase(Platform):
|
||||
kv_cache_dtype=attn_selector_config.kv_cache_dtype,
|
||||
use_non_causal=attn_selector_config.use_non_causal,
|
||||
head_size=attn_selector_config.head_size,
|
||||
use_mm_prefix=attn_selector_config.use_mm_prefix,
|
||||
)
|
||||
for priority, backend in enumerate(backend_priorities):
|
||||
try:
|
||||
|
||||
@@ -185,6 +185,10 @@ class AttentionBackend(ABC):
|
||||
def supports_sliding_window(cls) -> bool:
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def supports_rswa(cls) -> bool:
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def supports_non_causal(cls) -> bool:
|
||||
"""Check if backend supports non-causal (bidirectional) attention
|
||||
@@ -287,6 +291,7 @@ class AttentionBackend(ABC):
|
||||
use_pcp: bool = False,
|
||||
use_adaptive_verification: bool = False,
|
||||
use_dcp: bool = False,
|
||||
use_rswa: bool = False,
|
||||
) -> list[str]:
|
||||
invalid_reasons = []
|
||||
if not cls.supports_head_size(head_size):
|
||||
@@ -321,6 +326,8 @@ class AttentionBackend(ABC):
|
||||
invalid_reasons.append(f"attention type {attn_type} not supported")
|
||||
if has_sliding_window and not cls.supports_sliding_window():
|
||||
invalid_reasons.append("sliding window not supported")
|
||||
if use_rswa and not cls.supports_rswa():
|
||||
invalid_reasons.append("R-SWA not supported")
|
||||
if use_non_causal and not cls.supports_non_causal():
|
||||
invalid_reasons.append("non-causal attention not supported")
|
||||
if use_mla and use_non_causal and use_dcp and not cls.supports_non_causal_dcp():
|
||||
@@ -854,6 +861,9 @@ class AttentionImplBase(ABC, Generic[T]):
|
||||
def process_weights_after_loading(self, act_dtype: torch.dtype):
|
||||
pass
|
||||
|
||||
def get_impl_variants(self) -> tuple["AttentionImplBase", ...]:
|
||||
return (self,)
|
||||
|
||||
def prepare_for_batch(self, attn_metadata: T | None) -> None:
|
||||
"""Prepare implementation-specific state for the current batch."""
|
||||
|
||||
|
||||
@@ -0,0 +1,509 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Reusable attention backend composition and routing policies."""
|
||||
|
||||
import functools
|
||||
import math
|
||||
from typing import ClassVar, Protocol, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionCGSupport,
|
||||
AttentionImpl,
|
||||
AttentionMetadataBuilder,
|
||||
CommonAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import get_supported_kv_cache_layouts
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpec
|
||||
|
||||
|
||||
def requires_mm_prefix(
|
||||
metadata: CommonAttentionMetadata, *, unclamped_window: bool = False
|
||||
) -> bool:
|
||||
ranges = metadata.mm_req_doc_ranges
|
||||
# With one query per request there are no future keys to unmask. This
|
||||
# also keeps single-token prefills on the captured FlashInfer path.
|
||||
if not ranges or (metadata.max_query_len <= 1 and not unclamped_window):
|
||||
return False
|
||||
seq_lens = metadata.seq_lens_cpu_upper_bound
|
||||
assert seq_lens is not None
|
||||
starts = metadata.query_start_loc_cpu
|
||||
for req_idx, spans in ranges.items():
|
||||
query_len = int(starts[req_idx + 1] - starts[req_idx])
|
||||
if query_len == 0 or (query_len == 1 and not unclamped_window):
|
||||
continue
|
||||
end = int(seq_lens[req_idx])
|
||||
begin = end - query_len
|
||||
# An unclamped image can also expose keys behind the sliding window.
|
||||
if unclamped_window and any(
|
||||
start < stop and start < end and stop >= begin for start, stop in spans
|
||||
):
|
||||
return True
|
||||
# Prefill lengths are exact. Historical image ranges must not divert
|
||||
# later text/decode queries to the image backend.
|
||||
if any(
|
||||
start < stop and start < end - 1 and stop > begin for start, stop in spans
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _has_unclamped_window(layers) -> bool:
|
||||
return any(
|
||||
getattr(layer, "sliding_window", None) is not None
|
||||
and not getattr(layer, "mm_prefix_clamp_sliding_window", False)
|
||||
for layer in layers
|
||||
)
|
||||
|
||||
|
||||
class CompositeAttentionRouting(Protocol):
|
||||
"""Select a child backend and declare routing constraints for composition."""
|
||||
|
||||
capture_variant: ClassVar[int]
|
||||
"""Child used during graph capture; replay must select the same child."""
|
||||
|
||||
def __init__(self, layer_names: list[str], vllm_config: VllmConfig) -> None: ...
|
||||
|
||||
def select(self, metadata: CommonAttentionMetadata) -> int:
|
||||
"""Return 0 for the first child or 1 for the second child."""
|
||||
...
|
||||
|
||||
@staticmethod
|
||||
def get_cudagraph_support(
|
||||
vllm_config: VllmConfig, kv_cache_spec: KVCacheSpec
|
||||
) -> AttentionCGSupport:
|
||||
"""Limit graph support to batches whose routing stays invariant."""
|
||||
...
|
||||
|
||||
@staticmethod
|
||||
def variant_uses_mm_prefix(variant: int) -> bool:
|
||||
"""Whether this child must support the model's image-mask requirement."""
|
||||
...
|
||||
|
||||
|
||||
class MMPrefixAttentionRouting(CompositeAttentionRouting):
|
||||
"""Route image masks to variant zero and causal queries to variant one."""
|
||||
|
||||
def __init__(self, layer_names: list[str], vllm_config: VllmConfig) -> None:
|
||||
layers = vllm_config.compilation_config.static_forward_context
|
||||
self.unclamped_window = _has_unclamped_window(
|
||||
layers[name] for name in layer_names
|
||||
)
|
||||
|
||||
def select(self, metadata: CommonAttentionMetadata) -> int:
|
||||
return int(
|
||||
not requires_mm_prefix(metadata, unclamped_window=self.unclamped_window)
|
||||
)
|
||||
|
||||
capture_variant: ClassVar[int] = 1
|
||||
|
||||
@staticmethod
|
||||
def get_cudagraph_support(
|
||||
vllm_config: VllmConfig, kv_cache_spec: KVCacheSpec
|
||||
) -> AttentionCGSupport:
|
||||
if _has_unclamped_window(
|
||||
vllm_config.compilation_config.static_forward_context.values()
|
||||
):
|
||||
return AttentionCGSupport.NEVER
|
||||
return AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
||||
|
||||
@staticmethod
|
||||
def variant_uses_mm_prefix(variant: int) -> bool:
|
||||
return variant == 0
|
||||
|
||||
|
||||
def _intersect_kernel_block_sizes(
|
||||
first: type[AttentionBackend], second: type[AttentionBackend]
|
||||
):
|
||||
from vllm.v1.attention.backend import MultipleOf
|
||||
|
||||
result: list[int | MultipleOf] = []
|
||||
seen: set[tuple[type, int]] = set()
|
||||
for lhs in first.get_supported_kernel_block_sizes():
|
||||
for rhs in second.get_supported_kernel_block_sizes():
|
||||
candidate: int | MultipleOf | None = None
|
||||
if isinstance(lhs, MultipleOf) and isinstance(rhs, MultipleOf):
|
||||
candidate = MultipleOf(math.lcm(lhs.base, rhs.base))
|
||||
elif isinstance(lhs, MultipleOf) and isinstance(rhs, int):
|
||||
candidate = rhs if rhs % lhs.base == 0 else None
|
||||
elif isinstance(lhs, int) and isinstance(rhs, MultipleOf):
|
||||
candidate = lhs if lhs % rhs.base == 0 else None
|
||||
elif lhs == rhs:
|
||||
candidate = lhs
|
||||
if candidate is None:
|
||||
continue
|
||||
value = candidate.base if isinstance(candidate, MultipleOf) else candidate
|
||||
key = (type(candidate), value)
|
||||
if key not in seen:
|
||||
result.append(candidate)
|
||||
seen.add(key)
|
||||
return result
|
||||
|
||||
|
||||
@functools.cache
|
||||
def create_composite_attention_backend(
|
||||
general_backend: type[AttentionBackend],
|
||||
causal_backend: type[AttentionBackend],
|
||||
*,
|
||||
name: str,
|
||||
backend_name: str,
|
||||
module: str,
|
||||
routing_policy: type[CompositeAttentionRouting],
|
||||
head_sizes: tuple[int, ...] = (),
|
||||
kernel_block_sizes: tuple[int, ...] = (),
|
||||
device_major: int | None = None,
|
||||
) -> type[AttentionBackend]:
|
||||
"""Compose two backends with shared storage and an explicit routing policy."""
|
||||
if general_backend.full_cls_name() == causal_backend.full_cls_name():
|
||||
return general_backend
|
||||
general_impl_cls = general_backend.get_impl_cls()
|
||||
causal_impl_cls = causal_backend.get_impl_cls()
|
||||
general_builder_cls = general_backend.get_builder_cls()
|
||||
causal_builder_cls = causal_backend.get_builder_cls()
|
||||
if (
|
||||
general_backend.forward_includes_kv_cache_update
|
||||
or causal_backend.forward_includes_kv_cache_update
|
||||
):
|
||||
raise ValueError("Composite backends require separate KV cache updates")
|
||||
|
||||
class CompositeAttentionImpl(AttentionImpl):
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
self.general_impl = cast(AttentionImpl, general_impl_cls(*args, **kwargs))
|
||||
try:
|
||||
self.causal_impl = cast(AttentionImpl, causal_impl_cls(*args, **kwargs))
|
||||
except TypeError as e:
|
||||
raise ValueError(
|
||||
f"Causal attention backend {causal_backend.get_name()} "
|
||||
f"does not accept the attention arguments required by "
|
||||
f"the general backend {general_backend.get_name()}: {e}"
|
||||
) from e
|
||||
for name in (
|
||||
"num_heads",
|
||||
"num_kv_heads",
|
||||
"head_size",
|
||||
"scale",
|
||||
"kv_cache_dtype",
|
||||
):
|
||||
if hasattr(self.general_impl, name):
|
||||
setattr(self, name, getattr(self.general_impl, name))
|
||||
self.supports_quant_query_input = (
|
||||
self.general_impl.supports_quant_query_input
|
||||
and self.causal_impl.supports_quant_query_input
|
||||
)
|
||||
self.supports_pcp = self.general_impl.supports_pcp
|
||||
self.supports_dcp = self.causal_impl.supports_dcp
|
||||
self.can_return_lse_for_decode = self.causal_impl.can_return_lse_for_decode
|
||||
self.lse_base_on_e = self.causal_impl.lse_base_on_e
|
||||
self.need_to_return_lse_for_decode = (
|
||||
self.causal_impl.need_to_return_lse_for_decode
|
||||
)
|
||||
self.supports_mtp_with_cp_non_trivial_interleave_size = (
|
||||
self.causal_impl.supports_mtp_with_cp_non_trivial_interleave_size
|
||||
)
|
||||
|
||||
def get_impl_for_metadata(self, attn_metadata):
|
||||
return (
|
||||
self.causal_impl
|
||||
if getattr(attn_metadata, "_attention_backend_variant", 0)
|
||||
else self.general_impl
|
||||
)
|
||||
|
||||
def get_impl_variants(self):
|
||||
return self.general_impl, self.causal_impl
|
||||
|
||||
def forward(
|
||||
self,
|
||||
layer,
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
kv_cache,
|
||||
attn_metadata,
|
||||
output,
|
||||
output_scale=None,
|
||||
output_block_scale=None,
|
||||
):
|
||||
impl = self.get_impl_for_metadata(attn_metadata)
|
||||
return impl.forward(
|
||||
layer,
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
kv_cache,
|
||||
attn_metadata,
|
||||
output,
|
||||
output_scale,
|
||||
output_block_scale,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, act_dtype: torch.dtype):
|
||||
for impl in self.get_impl_variants():
|
||||
impl.process_weights_after_loading(act_dtype)
|
||||
|
||||
def fused_output_quant_supported(self, quant_key):
|
||||
return all(
|
||||
impl.fused_output_quant_supported(quant_key)
|
||||
for impl in self.get_impl_variants()
|
||||
)
|
||||
|
||||
def fused_qk_norm_rope_kvcache_supported(self):
|
||||
return self.general_impl.fused_qk_norm_rope_kvcache_supported()
|
||||
|
||||
def do_qk_norm_rope_kvcache_update(self, *args, **kwargs):
|
||||
method = self.general_impl.do_qk_norm_rope_kvcache_update
|
||||
return method(*args, **kwargs)
|
||||
|
||||
def fused_rope_kvcache_supported(self):
|
||||
return self.general_impl.fused_rope_kvcache_supported()
|
||||
|
||||
def do_rope_and_kv_cache_update(self, *args, **kwargs):
|
||||
method = self.general_impl.do_rope_and_kv_cache_update
|
||||
return method(*args, **kwargs)
|
||||
|
||||
def do_kv_cache_update(self, *args, **kwargs):
|
||||
method = self.general_impl.do_kv_cache_update # type: ignore[attr-defined]
|
||||
return method(*args, **kwargs)
|
||||
|
||||
class CompositeAttentionMetadataBuilder(AttentionMetadataBuilder):
|
||||
requires_block_table_width = (
|
||||
general_builder_cls.requires_block_table_width
|
||||
or causal_builder_cls.requires_block_table_width
|
||||
)
|
||||
|
||||
def __init__(self, kv_cache_spec, layer_names, vllm_config, device, **kwargs):
|
||||
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||
args = (kv_cache_spec, layer_names, vllm_config, device)
|
||||
# Only the variant that asked for the extra ctor kwargs receives them.
|
||||
self.general_builder = general_builder_cls(
|
||||
*args,
|
||||
**(kwargs if general_builder_cls.requires_block_table_width else {}),
|
||||
)
|
||||
self.causal_builder = causal_builder_cls(
|
||||
*args,
|
||||
**(kwargs if causal_builder_cls.requires_block_table_width else {}),
|
||||
)
|
||||
self.reorder_batch_threshold = min(
|
||||
(
|
||||
builder.reorder_batch_threshold
|
||||
for builder in self._builders
|
||||
if builder.reorder_batch_threshold is not None
|
||||
),
|
||||
default=None,
|
||||
)
|
||||
self.routing = routing_policy(layer_names, vllm_config)
|
||||
|
||||
def set_kernel_block_size(self, kernel_block_size):
|
||||
super().set_kernel_block_size(kernel_block_size)
|
||||
for builder in self._builders:
|
||||
builder.set_kernel_block_size(kernel_block_size)
|
||||
|
||||
@classmethod
|
||||
def get_cudagraph_support(cls, vllm_config, kv_cache_spec):
|
||||
return min(
|
||||
general_builder_cls.get_cudagraph_support(vllm_config, kv_cache_spec),
|
||||
causal_builder_cls.get_cudagraph_support(vllm_config, kv_cache_spec),
|
||||
routing_policy.get_cudagraph_support(vllm_config, kv_cache_spec),
|
||||
key=lambda support: support.value,
|
||||
)
|
||||
|
||||
def _builder(self, common_attn_metadata):
|
||||
backend_variant = self.routing.select(common_attn_metadata)
|
||||
builder = self.causal_builder if backend_variant else self.general_builder
|
||||
return builder, backend_variant
|
||||
|
||||
def _build(self, method, common_attn_metadata, **kwargs):
|
||||
builder, backend_variant = self._builder(common_attn_metadata)
|
||||
metadata = getattr(builder, method)(
|
||||
common_attn_metadata=common_attn_metadata, **kwargs
|
||||
)
|
||||
metadata._attention_backend_variant = backend_variant
|
||||
return metadata
|
||||
|
||||
def build(self, common_prefix_len, common_attn_metadata, fast_build=False):
|
||||
return self._build(
|
||||
"build",
|
||||
common_attn_metadata,
|
||||
common_prefix_len=common_prefix_len,
|
||||
fast_build=fast_build,
|
||||
)
|
||||
|
||||
def build_for_cudagraph_capture(self, common_attn_metadata):
|
||||
variant = routing_policy.capture_variant
|
||||
metadata = self._builders[variant].build_for_cudagraph_capture(
|
||||
common_attn_metadata
|
||||
)
|
||||
metadata._attention_backend_variant = variant
|
||||
return metadata
|
||||
|
||||
@property
|
||||
def _builders(self):
|
||||
return self.general_builder, self.causal_builder
|
||||
|
||||
# Forward the optional workspace protocol so that a wrapped builder
|
||||
# which allocates one (FlashInfer) still joins the runner's cross-group
|
||||
# sharing instead of allocating a second buffer behind the composite.
|
||||
def _get_workspace_buffer(self):
|
||||
for builder in self._builders:
|
||||
if hasattr(builder, "_get_workspace_buffer"):
|
||||
workspace = builder._get_workspace_buffer()
|
||||
# The runner only calls the getter for the group that
|
||||
# provides the buffer, so hand it to the sibling here.
|
||||
self.set_workspace_buffer(workspace)
|
||||
return workspace
|
||||
return None
|
||||
|
||||
def set_workspace_buffer(self, workspace_buffer):
|
||||
for builder in self._builders:
|
||||
if hasattr(builder, "set_workspace_buffer"):
|
||||
builder.set_workspace_buffer(workspace_buffer)
|
||||
|
||||
class CompositeAttentionBackend(AttentionBackend):
|
||||
forward_includes_kv_cache_update = False
|
||||
general_backend_cls = general_backend
|
||||
causal_backend_cls = causal_backend
|
||||
|
||||
@staticmethod
|
||||
def get_name():
|
||||
return backend_name
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls():
|
||||
return CompositeAttentionImpl
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls():
|
||||
return CompositeAttentionMetadataBuilder
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes():
|
||||
if kernel_block_sizes:
|
||||
return [
|
||||
size
|
||||
for size in kernel_block_sizes
|
||||
if all(
|
||||
backend.supports_block_size(size)
|
||||
for backend in (general_backend, causal_backend)
|
||||
)
|
||||
]
|
||||
return _intersect_kernel_block_sizes(general_backend, causal_backend)
|
||||
|
||||
@classmethod
|
||||
def supports_block_size(cls, block_size):
|
||||
return bool(
|
||||
cls.get_supported_kernel_block_sizes()
|
||||
) and super().supports_block_size(block_size)
|
||||
|
||||
@classmethod
|
||||
def get_supported_head_sizes(cls):
|
||||
return list(head_sizes)
|
||||
|
||||
@classmethod
|
||||
def supports_head_size(cls, head_size):
|
||||
return (not head_sizes or head_size in head_sizes) and all(
|
||||
backend.supports_head_size(head_size)
|
||||
for backend in (general_backend, causal_backend)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def supports_compute_capability(cls, capability):
|
||||
return (device_major is None or capability.major == device_major) and all(
|
||||
backend.supports_compute_capability(capability)
|
||||
for backend in (general_backend, causal_backend)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def supports_dtype(cls, dtype):
|
||||
return all(
|
||||
backend.supports_dtype(dtype)
|
||||
for backend in (general_backend, causal_backend)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def supports_kv_cache_dtype(cls, dtype):
|
||||
return all(
|
||||
backend.supports_kv_cache_dtype(dtype)
|
||||
for backend in (general_backend, causal_backend)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def supports_mm_prefix(cls):
|
||||
return all(
|
||||
not routing_policy.variant_uses_mm_prefix(variant)
|
||||
or backend.supports_mm_prefix()
|
||||
for variant, backend in enumerate((general_backend, causal_backend))
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def supports_sliding_window(cls):
|
||||
return all(
|
||||
backend.supports_sliding_window()
|
||||
for backend in (general_backend, causal_backend)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def supports_rswa(cls):
|
||||
return all(
|
||||
backend.supports_rswa() for backend in (general_backend, causal_backend)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def supports_pcp(cls):
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def supports_dcp(cls):
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def supports_device_cpu_query_lens_mismatch(cls):
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def supports_combination(
|
||||
cls,
|
||||
head_size,
|
||||
dtype,
|
||||
kv_cache_dtype,
|
||||
block_size,
|
||||
use_mla,
|
||||
has_sink,
|
||||
use_sparse,
|
||||
use_mm_prefix,
|
||||
device_capability,
|
||||
):
|
||||
for variant, backend in enumerate((general_backend, causal_backend)):
|
||||
reason = backend.supports_combination(
|
||||
head_size,
|
||||
dtype,
|
||||
kv_cache_dtype,
|
||||
block_size,
|
||||
use_mla,
|
||||
has_sink,
|
||||
use_sparse,
|
||||
use_mm_prefix and routing_policy.variant_uses_mm_prefix(variant),
|
||||
device_capability,
|
||||
)
|
||||
if reason:
|
||||
return f"{backend.get_name()}: {reason}"
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def customize_spec(cls, spec):
|
||||
general_spec = general_backend.customize_spec(spec)
|
||||
if general_spec != causal_backend.customize_spec(spec):
|
||||
raise ValueError("Composite backends require the same KV cache spec")
|
||||
return general_spec
|
||||
|
||||
@classmethod
|
||||
def supported_kv_cache_layouts(cls):
|
||||
return tuple(
|
||||
get_supported_kv_cache_layouts((general_backend, causal_backend))
|
||||
)
|
||||
|
||||
CompositeAttentionBackend.__name__ = name
|
||||
CompositeAttentionBackend.__qualname__ = name
|
||||
CompositeAttentionBackend.__module__ = module
|
||||
return CompositeAttentionBackend
|
||||
@@ -129,6 +129,10 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
def get_name() -> str:
|
||||
return "FLASH_ATTN"
|
||||
|
||||
@classmethod
|
||||
def supports_rswa(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def supports_sliding_window(cls) -> bool:
|
||||
return True
|
||||
|
||||
@@ -105,6 +105,10 @@ class FlexAttentionBackend(AttentionBackend):
|
||||
def get_name() -> str:
|
||||
return "FLEX_ATTENTION"
|
||||
|
||||
@classmethod
|
||||
def supports_rswa(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def supports_sliding_window(cls) -> bool:
|
||||
return True
|
||||
|
||||
@@ -49,6 +49,9 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta):
|
||||
TRITON_ATTN_DIFFKV = (
|
||||
"vllm.v1.attention.backends.triton_attn_diffkv.TritonAttentionDiffKVBackend"
|
||||
)
|
||||
TRITON_FLASH_ATTN = (
|
||||
"vllm.v1.attention.backends.triton_flash_attn.TritonFlashAttentionBackend"
|
||||
)
|
||||
ROCM_ATTN = "vllm.v1.attention.backends.rocm_attn.RocmAttentionBackend"
|
||||
ROCM_AITER_MLA = "vllm.v1.attention.backends.mla.rocm_aiter_mla.AiterMLABackend"
|
||||
ROCM_AITER_TRITON_MLA = (
|
||||
@@ -63,6 +66,9 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta):
|
||||
XPU_MLA_SPARSE = "vllm.v1.attention.backends.mla.xpu_mla_sparse.XPUMLASparseBackend"
|
||||
TORCH_SDPA = "" # this tag is only used for ViT
|
||||
FLASHINFER = "vllm.v1.attention.backends.flashinfer.FlashInferBackend"
|
||||
TRITON_FLASHINFER = (
|
||||
"vllm.v1.attention.backends.triton_flashinfer.TritonFlashInferBackend"
|
||||
)
|
||||
FLASHINFER_MLA = (
|
||||
"vllm.v1.attention.backends.mla.flashinfer_mla.FlashInferMLABackend"
|
||||
)
|
||||
|
||||
@@ -323,6 +323,10 @@ class TritonAttentionBackend(AttentionBackend):
|
||||
def get_name() -> str:
|
||||
return "TRITON_ATTN"
|
||||
|
||||
@classmethod
|
||||
def supports_rswa(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def supports_sliding_window(cls) -> bool:
|
||||
return True
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Triton image-mask and FlashAttention causal attention composite."""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.v1.attention.backends.composite import (
|
||||
MMPrefixAttentionRouting,
|
||||
create_composite_attention_backend,
|
||||
)
|
||||
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
from vllm.v1.attention.backends.triton_attn import TritonAttentionBackend
|
||||
|
||||
|
||||
class _FA4Backend(FlashAttentionBackend):
|
||||
@classmethod
|
||||
def supports_combination(
|
||||
cls,
|
||||
head_size: int,
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: CacheDType | None,
|
||||
block_size: int | None,
|
||||
use_mla: bool,
|
||||
has_sink: bool,
|
||||
use_sparse: bool,
|
||||
use_mm_prefix: bool,
|
||||
device_capability: DeviceCapability,
|
||||
) -> str | None:
|
||||
reason = super().supports_combination(
|
||||
head_size,
|
||||
dtype,
|
||||
kv_cache_dtype,
|
||||
block_size,
|
||||
use_mla,
|
||||
has_sink,
|
||||
use_sparse,
|
||||
use_mm_prefix,
|
||||
device_capability,
|
||||
)
|
||||
if reason is not None:
|
||||
return reason
|
||||
if (
|
||||
get_flash_attn_version(
|
||||
head_size=head_size,
|
||||
has_sinks=has_sink,
|
||||
kv_cache_block_size=block_size,
|
||||
supports_fa4_hd256=True,
|
||||
)
|
||||
!= 4
|
||||
):
|
||||
return "causal route requires FlashAttention v4"
|
||||
return None
|
||||
|
||||
|
||||
TritonFlashAttentionBackend = create_composite_attention_backend(
|
||||
TritonAttentionBackend,
|
||||
_FA4Backend,
|
||||
name="TritonFlashAttentionBackend",
|
||||
backend_name="TRITON_FLASH_ATTN",
|
||||
module=__name__,
|
||||
routing_policy=MMPrefixAttentionRouting,
|
||||
head_sizes=(256, 512),
|
||||
device_major=9,
|
||||
)
|
||||
@@ -0,0 +1,22 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Triton image-mask and FlashInfer causal attention composite."""
|
||||
|
||||
from vllm.v1.attention.backends.composite import (
|
||||
MMPrefixAttentionRouting,
|
||||
create_composite_attention_backend,
|
||||
)
|
||||
from vllm.v1.attention.backends.flashinfer import FlashInferBackend
|
||||
from vllm.v1.attention.backends.triton_attn import TritonAttentionBackend
|
||||
|
||||
TritonFlashInferBackend = create_composite_attention_backend(
|
||||
TritonAttentionBackend,
|
||||
FlashInferBackend,
|
||||
name="TritonFlashInferBackend",
|
||||
backend_name="TRITON_FLASHINFER",
|
||||
module=__name__,
|
||||
routing_policy=MMPrefixAttentionRouting,
|
||||
head_sizes=(256, 512),
|
||||
kernel_block_sizes=(64,),
|
||||
device_major=10,
|
||||
)
|
||||
@@ -8,6 +8,7 @@ from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Protocol,
|
||||
TypeVar,
|
||||
cast,
|
||||
)
|
||||
|
||||
@@ -45,6 +46,19 @@ logger = init_logger(__name__)
|
||||
PAD_SLOT_ID = -1
|
||||
NULL_BLOCK_ID = 0
|
||||
|
||||
|
||||
ImplT = TypeVar("ImplT", bound=AttentionImpl)
|
||||
|
||||
|
||||
def find_attention_impl_variant(
|
||||
impl: AttentionImpl, impl_cls: type[ImplT]
|
||||
) -> ImplT | None:
|
||||
for variant in impl.get_impl_variants():
|
||||
if isinstance(variant, impl_cls):
|
||||
return cast(ImplT, variant)
|
||||
return None
|
||||
|
||||
|
||||
_LN_2 = math.log(2.0)
|
||||
|
||||
|
||||
@@ -342,8 +356,8 @@ def get_per_layer_parameters(
|
||||
per_layer_params: dict[str, PerLayerParameters] = {}
|
||||
|
||||
for key, layer in layers.items():
|
||||
impl = layer.impl
|
||||
assert isinstance(impl, cls_)
|
||||
impl = find_attention_impl_variant(layer.impl, cls_)
|
||||
assert impl is not None
|
||||
|
||||
# Infer hyperparameters from the attention layer
|
||||
window_size = getattr(impl, "sliding_window", None)
|
||||
@@ -1013,6 +1027,9 @@ def create_fast_prefill_custom_backend(
|
||||
common_attn_metadata.logits_indices_padded
|
||||
)
|
||||
self.num_logits_indices = common_attn_metadata.num_logits_indices
|
||||
self._attention_backend_variant = getattr(
|
||||
metadata, "_attention_backend_variant", 0
|
||||
)
|
||||
|
||||
return KVSharingFastPrefillAttentionMetadata(metadata, common_attn_metadata)
|
||||
|
||||
|
||||
@@ -36,6 +36,7 @@ class AttentionSelectorConfig(NamedTuple):
|
||||
use_pcp: bool = False
|
||||
use_adaptive_verification: bool = False
|
||||
use_dcp: bool = False
|
||||
use_rswa: bool = False
|
||||
|
||||
def __repr__(self):
|
||||
return (
|
||||
@@ -55,7 +56,8 @@ class AttentionSelectorConfig(NamedTuple):
|
||||
f"use_kv_connector={self.use_kv_connector}, "
|
||||
f"use_adaptive_verification={self.use_adaptive_verification}, "
|
||||
f"use_pcp={self.use_pcp}, "
|
||||
f"use_dcp={self.use_dcp})"
|
||||
f"use_dcp={self.use_dcp}, "
|
||||
f"use_rswa={self.use_rswa})"
|
||||
)
|
||||
|
||||
|
||||
@@ -168,6 +170,10 @@ def get_attn_backend(
|
||||
use_pcp=vllm_config.parallel_config.prefill_context_parallel_size > 1,
|
||||
use_adaptive_verification=use_adaptive_verification,
|
||||
use_dcp=vllm_config.parallel_config.decode_context_parallel_size > 1,
|
||||
use_rswa=(
|
||||
vllm_config.model_config is not None
|
||||
and vllm_config.model_config.rswa_window is not None
|
||||
),
|
||||
)
|
||||
|
||||
# A per-KV-group override (keyed by KVCacheSpecKind) takes precedence over
|
||||
|
||||
Reference in New Issue
Block a user