Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
371 lines
12 KiB
Python
371 lines
12 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""ROCm attention backend for token-major segmented Triton attention."""
|
|
|
|
import os
|
|
from pathlib import Path
|
|
from typing import TYPE_CHECKING, ClassVar
|
|
|
|
import torch
|
|
|
|
import vllm.envs as envs
|
|
from vllm._aiter_ops import rocm_aiter_ops
|
|
from vllm.config import VllmConfig, get_current_vllm_config
|
|
from vllm.config.cache import CacheDType
|
|
from vllm.logger import init_logger
|
|
from vllm.platforms import current_platform
|
|
from vllm.utils.torch_utils import is_quantized_kv_cache
|
|
from vllm.v1.attention.backend import AttentionLayer, AttentionType
|
|
from vllm.v1.attention.backends.rocm_attn import (
|
|
RocmAttentionBackend,
|
|
RocmAttentionImpl,
|
|
RocmAttentionMetadata,
|
|
RocmAttentionMetadataBuilder,
|
|
)
|
|
from vllm.v1.attention.backends.utils import get_num_attention_heads_from_layers
|
|
from vllm.v1.attention.ops.segmented_attention import (
|
|
get_segmented_attention_workspace,
|
|
segmented_attention,
|
|
segmented_attention_workspace_size,
|
|
)
|
|
from vllm.v1.attention.ops.triton_reshape_and_cache_flash import (
|
|
triton_reshape_and_cache_flash,
|
|
)
|
|
from vllm.v1.kv_cache_interface import AttentionSpec
|
|
|
|
if TYPE_CHECKING:
|
|
from vllm.platforms.interface import DeviceCapability
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
def is_rdna() -> bool:
|
|
if not current_platform.is_rocm():
|
|
return False
|
|
from vllm.platforms.rocm import on_gfx1x
|
|
|
|
return on_gfx1x()
|
|
|
|
|
|
class RocmSegmentedAttentionMetadataBuilder(RocmAttentionMetadataBuilder):
|
|
def __init__(
|
|
self,
|
|
kv_cache_spec: AttentionSpec,
|
|
layer_names: list[str],
|
|
vllm_config: VllmConfig,
|
|
device: torch.device,
|
|
):
|
|
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
|
model_config = vllm_config.model_config
|
|
self.num_heads_q = get_num_attention_heads_from_layers(
|
|
vllm_config, layer_names
|
|
) or model_config.get_num_attention_heads(vllm_config.parallel_config)
|
|
self.num_heads_kv = kv_cache_spec.num_kv_heads
|
|
self.headdim = kv_cache_spec.head_size
|
|
|
|
|
|
class RocmSegmentedAttentionBackend(RocmAttentionBackend):
|
|
"""Explicit token-major ROCm backend optimized for segmented prefill."""
|
|
|
|
supported_dtypes: ClassVar[list[torch.dtype]] = [
|
|
torch.float16,
|
|
torch.bfloat16,
|
|
]
|
|
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
|
"auto",
|
|
"float16",
|
|
"bfloat16",
|
|
"fp8",
|
|
"fp8_e4m3",
|
|
]
|
|
|
|
@staticmethod
|
|
def get_name() -> str:
|
|
return "ROCM_SEGMENTED_ATTN"
|
|
|
|
@staticmethod
|
|
def get_impl_cls() -> type["RocmSegmentedAttentionImpl"]:
|
|
return RocmSegmentedAttentionImpl
|
|
|
|
@staticmethod
|
|
def get_builder_cls() -> type["RocmSegmentedAttentionMetadataBuilder"]:
|
|
return RocmSegmentedAttentionMetadataBuilder
|
|
|
|
@classmethod
|
|
def get_supported_head_sizes(cls) -> list[int]:
|
|
return [64, 128, 256]
|
|
|
|
@classmethod
|
|
def supports_attn_type(cls, attn_type: str) -> bool:
|
|
return attn_type == AttentionType.DECODER
|
|
|
|
@classmethod
|
|
def supports_non_causal(cls) -> bool:
|
|
return True
|
|
|
|
@classmethod
|
|
def supports_sliding_window(cls) -> bool:
|
|
return True
|
|
|
|
@classmethod
|
|
def supports_sink(cls) -> bool:
|
|
return True
|
|
|
|
@classmethod
|
|
def supports_mm_prefix(cls) -> bool:
|
|
return False
|
|
|
|
@classmethod
|
|
def supports_combination(
|
|
cls,
|
|
head_size: int,
|
|
dtype: torch.dtype,
|
|
kv_cache_dtype: CacheDType | None,
|
|
block_size: int | None,
|
|
use_mla: bool,
|
|
has_sink: bool,
|
|
use_sparse: bool,
|
|
use_mm_prefix: bool,
|
|
device_capability: "DeviceCapability",
|
|
) -> str | None:
|
|
del (
|
|
head_size,
|
|
dtype,
|
|
block_size,
|
|
use_mla,
|
|
has_sink,
|
|
use_sparse,
|
|
use_mm_prefix,
|
|
device_capability,
|
|
)
|
|
if not is_rdna():
|
|
return "ROCM_SEGMENTED_ATTN requires AMD RDNA GPUs on ROCm"
|
|
|
|
from vllm.platforms.rocm import on_gfx12x
|
|
|
|
if kv_cache_dtype in ("fp8", "fp8_e4m3") and not on_gfx12x():
|
|
return "FP8 segmented attention requires gfx12"
|
|
return None
|
|
|
|
|
|
class RocmSegmentedAttentionImpl(RocmAttentionImpl):
|
|
def __init__(
|
|
self,
|
|
num_heads: int,
|
|
head_size: int,
|
|
scale: float,
|
|
num_kv_heads: int,
|
|
alibi_slopes: list[float] | None,
|
|
sliding_window: int | None,
|
|
kv_cache_dtype: str,
|
|
logits_soft_cap: float | None = None,
|
|
attn_type: AttentionType = AttentionType.DECODER,
|
|
kv_sharing_target_layer_name: int | None = None,
|
|
sinks: torch.Tensor | None = None,
|
|
) -> None:
|
|
if not is_rdna():
|
|
raise ValueError("ROCM_SEGMENTED_ATTN requires AMD RDNA GPUs on ROCm")
|
|
|
|
super().__init__(
|
|
num_heads,
|
|
head_size,
|
|
scale,
|
|
num_kv_heads,
|
|
alibi_slopes,
|
|
sliding_window,
|
|
kv_cache_dtype,
|
|
logits_soft_cap,
|
|
attn_type,
|
|
kv_sharing_target_layer_name,
|
|
sinks,
|
|
)
|
|
from vllm.platforms.rocm import on_gfx1x, on_gfx12x
|
|
|
|
fp8 = kv_cache_dtype in ("fp8", "fp8_e4m3")
|
|
arch_supported = on_gfx12x() if fp8 else on_gfx1x()
|
|
gqa_ratio_supported = (
|
|
num_kv_heads > 0
|
|
and num_heads % num_kv_heads == 0
|
|
and 1 <= self.num_queries_per_kv <= 16
|
|
)
|
|
if not (
|
|
arch_supported
|
|
and attn_type == AttentionType.DECODER
|
|
and head_size in (64, 128, 256)
|
|
and gqa_ratio_supported
|
|
and kv_cache_dtype in ("auto", "float16", "bfloat16", "fp8", "fp8_e4m3")
|
|
and alibi_slopes is None
|
|
):
|
|
raise ValueError(
|
|
"ROCM_SEGMENTED_ATTN requires decoder attention on gfx1x "
|
|
"(gfx12 for FP8), head size 64, 128 or 256, GQA ratio 1-16, and "
|
|
"does not support ALiBi."
|
|
)
|
|
# Startup tuning and target/draft compilation must share a Triton cache.
|
|
cache_dir = os.environ.setdefault(
|
|
"TRITON_CACHE_DIR",
|
|
str(Path(envs.VLLM_CACHE_ROOT) / "rocm_segmented_attention" / "triton"),
|
|
)
|
|
Path(cache_dir).mkdir(parents=True, exist_ok=True)
|
|
logger.info_once("Using token-major ROCm segmented Triton attention")
|
|
self._segmented_attention_warmed_up = False
|
|
self._segmented_attention_config: VllmConfig | None = None
|
|
|
|
def process_weights_after_loading(self, act_dtype: torch.dtype):
|
|
super().process_weights_after_loading(act_dtype)
|
|
config = get_current_vllm_config()
|
|
self._segmented_attention_config = config
|
|
|
|
def _autotune_enabled(self) -> bool:
|
|
config = self._segmented_attention_config
|
|
return (
|
|
config is not None
|
|
and config.kernel_config.enable_rocm_segmented_attn_autotune
|
|
)
|
|
|
|
def _get_workspace(self, device):
|
|
config = self._segmented_attention_config
|
|
assert config is not None
|
|
sizes = segmented_attention_workspace_size(
|
|
config.scheduler_config.max_num_seqs,
|
|
self.num_heads,
|
|
self.num_kv_heads,
|
|
self.head_size,
|
|
config.model_config.max_model_len,
|
|
max_tokens=config.scheduler_config.max_num_batched_tokens,
|
|
fp8=self.kv_cache_dtype in ("fp8", "fp8_e4m3"),
|
|
autotune=self._autotune_enabled(),
|
|
)
|
|
return get_segmented_attention_workspace(device, sizes)
|
|
|
|
def _split_kv_cache(
|
|
self,
|
|
kv_cache: torch.Tensor,
|
|
num_kv_heads: int | None = None,
|
|
head_size: int | None = None,
|
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
|
num_kv_heads = num_kv_heads or self.num_kv_heads
|
|
head_size = head_size or self.head_size
|
|
return (
|
|
kv_cache[:, 0].unflatten(-1, (num_kv_heads, head_size)),
|
|
kv_cache[:, 1].unflatten(-1, (num_kv_heads, head_size)),
|
|
)
|
|
|
|
def forward(
|
|
self,
|
|
layer: torch.nn.Module,
|
|
query: torch.Tensor,
|
|
key: torch.Tensor,
|
|
value: torch.Tensor,
|
|
kv_cache: torch.Tensor,
|
|
attn_metadata: RocmAttentionMetadata,
|
|
output: torch.Tensor,
|
|
output_scale: torch.Tensor | None = None,
|
|
output_block_scale: torch.Tensor | None = None,
|
|
) -> torch.Tensor:
|
|
if output_block_scale is not None:
|
|
raise NotImplementedError(
|
|
"fused block_scale output quantization is not supported"
|
|
)
|
|
# Profile runs allocate the same fixed scratch used during graph replay.
|
|
workspace = self._get_workspace(query.device)
|
|
if attn_metadata is None:
|
|
return output.fill_(0)
|
|
|
|
num_actual_tokens = attn_metadata.num_actual_tokens
|
|
key_cache, value_cache = self._split_kv_cache(kv_cache)
|
|
if is_quantized_kv_cache(self.kv_cache_dtype):
|
|
key_cache = key_cache.view(self.fp8_dtype)
|
|
value_cache = value_cache.view(self.fp8_dtype)
|
|
if query.dtype == self.fp8_dtype and layer._q_scale_float != 1.0:
|
|
raise NotImplementedError(
|
|
"A non-1.0 q_scale with an FP8 query is not supported"
|
|
)
|
|
|
|
segmented_attention(
|
|
query=query[:num_actual_tokens],
|
|
key=key[:num_actual_tokens],
|
|
value=value[:num_actual_tokens],
|
|
output=output[:num_actual_tokens],
|
|
kv_cache_dtype=self.kv_cache_dtype,
|
|
key_cache=key_cache,
|
|
value_cache=value_cache,
|
|
block_table=attn_metadata.block_table,
|
|
query_start_loc=attn_metadata.query_start_loc,
|
|
seq_lens=attn_metadata.seq_lens,
|
|
max_seq_len=attn_metadata.max_seq_len,
|
|
max_query_len=attn_metadata.max_query_len,
|
|
k_scale=layer._k_scale,
|
|
v_scale=layer._v_scale,
|
|
sm_scale=self.scale,
|
|
sliding_window=self.sliding_window[0],
|
|
output_scale=output_scale,
|
|
sinks=self.sinks,
|
|
causal=attn_metadata.causal,
|
|
softcap=self.logits_soft_cap,
|
|
workspace=workspace,
|
|
use_tuned_config=self._autotune_enabled(),
|
|
)
|
|
return output
|
|
|
|
def do_kv_cache_update(
|
|
self,
|
|
layer: AttentionLayer,
|
|
key: torch.Tensor,
|
|
value: torch.Tensor,
|
|
kv_cache: torch.Tensor,
|
|
slot_mapping: torch.Tensor,
|
|
):
|
|
key_cache, value_cache = self._split_kv_cache(kv_cache)
|
|
if is_quantized_kv_cache(self.kv_cache_dtype):
|
|
key_cache = key_cache.view(self.fp8_dtype)
|
|
value_cache = value_cache.view(self.fp8_dtype)
|
|
triton_reshape_and_cache_flash(
|
|
key,
|
|
value,
|
|
key_cache,
|
|
value_cache,
|
|
slot_mapping,
|
|
self.kv_cache_dtype,
|
|
layer._k_scale,
|
|
layer._v_scale,
|
|
)
|
|
|
|
def do_rope_and_kv_cache_update(
|
|
self,
|
|
layer: AttentionLayer,
|
|
query: torch.Tensor,
|
|
key: torch.Tensor,
|
|
value: torch.Tensor,
|
|
positions: torch.Tensor,
|
|
cos_sin_cache: torch.Tensor,
|
|
is_neox: bool,
|
|
kv_cache: torch.Tensor,
|
|
layer_slot_mapping: torch.Tensor,
|
|
):
|
|
key_cache, value_cache = self._split_kv_cache(
|
|
kv_cache,
|
|
layer.num_kv_heads, # type: ignore[attr-defined]
|
|
layer.head_size, # type: ignore[attr-defined]
|
|
)
|
|
is_fp8_kv_cache = is_quantized_kv_cache(self.kv_cache_dtype)
|
|
if is_fp8_kv_cache:
|
|
key_cache = key_cache.view(self.fp8_dtype)
|
|
value_cache = value_cache.view(self.fp8_dtype)
|
|
|
|
rocm_aiter_ops.triton_rope_and_cache(
|
|
query,
|
|
key,
|
|
value,
|
|
positions,
|
|
cos_sin_cache,
|
|
is_neox,
|
|
key_cache,
|
|
value_cache,
|
|
layer_slot_mapping,
|
|
layer._k_scale,
|
|
layer._v_scale,
|
|
True,
|
|
is_fp8_kv_cache,
|
|
)
|