[ROCM][Perf] Add ROCM_SEGMENTED_ATTN attention-backend works across RDNA3/3.5/4 (#59132)

Signed-off-by: root <jeffaw99@hotmail.com>
Signed-off-by: big-yellow-duck <jeffaw99@hotmail.com>
Co-authored-by: Tan Pin Siang <1716735+tanpinsiang@users.noreply.github.com>
Co-authored-by: vllmellm <190700713+vllmellm@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
big_yellow_duck
2026-10-06 07:53:47 +00:00
committed by GitHub
co-authored by Tan Pin Siang vllmellm Codex
parent 15c1cc12ac
commit edfccc9469
9 changed files with 4431 additions and 0 deletions
File diff suppressed because it is too large Load Diff
@@ -61,6 +61,216 @@ def test_aiter_unified_attention_uses_dedicated_metadata_builder():
)
def test_segmented_attention_uses_dedicated_backend_components():
from vllm.v1.attention.backends.rocm_segmented_attn import (
RocmSegmentedAttentionBackend,
RocmSegmentedAttentionImpl,
RocmSegmentedAttentionMetadataBuilder,
)
assert (
RocmSegmentedAttentionBackend.get_builder_cls()
is RocmSegmentedAttentionMetadataBuilder
)
assert RocmSegmentedAttentionBackend.get_impl_cls() is RocmSegmentedAttentionImpl
assert RocmSegmentedAttentionBackend.get_name() == "ROCM_SEGMENTED_ATTN"
assert RocmSegmentedAttentionBackend.supports_non_causal()
assert RocmSegmentedAttentionBackend.supports_sliding_window()
assert RocmSegmentedAttentionBackend.supports_sink()
assert not RocmSegmentedAttentionBackend.supports_mm_prefix()
@pytest.mark.parametrize(
"sliding_window,logits_soft_cap,has_sinks",
[(4096, None, False), (None, 50.0, False), (None, None, True)],
)
def test_segmented_attention_admits_unified_fallback_features(
monkeypatch, sliding_window, logits_soft_cap, has_sinks
):
from vllm.platforms import rocm
from vllm.v1.attention.backend import AttentionType
from vllm.v1.attention.backends.rocm_segmented_attn import (
RocmSegmentedAttentionImpl,
)
monkeypatch.setattr(rocm, "on_gfx1x", lambda: True)
monkeypatch.setattr(rocm, "on_gfx12x", lambda: True)
sinks = torch.zeros(8) if has_sinks else None
impl = RocmSegmentedAttentionImpl(
num_heads=8,
head_size=128,
scale=128**-0.5,
num_kv_heads=2,
alibi_slopes=None,
sliding_window=sliding_window,
kv_cache_dtype="auto",
logits_soft_cap=logits_soft_cap,
attn_type=AttentionType.DECODER,
sinks=sinks,
)
assert isinstance(impl, RocmSegmentedAttentionImpl)
@pytest.mark.parametrize(
"platform,rdna",
[("cuda", False), ("cpu", False), ("xpu", False), ("rocm", False), ("rocm", True)],
)
def test_segmented_attention_requires_rocm_rdna(monkeypatch, platform, rdna):
"""Reject unsupported platforms before probing ROCm or constructing kernels."""
from vllm.platforms import rocm
from vllm.platforms.interface import DeviceCapability
from vllm.v1.attention.backends import rocm_segmented_attn as segmented
monkeypatch.setattr(
segmented,
"current_platform",
SimpleNamespace(is_rocm=lambda: platform == "rocm"),
)
def on_rdna():
assert platform == "rocm", "Must not probe ROCm on another platform"
return rdna
monkeypatch.setattr(rocm, "on_gfx1x", on_rdna)
reason = segmented.RocmSegmentedAttentionBackend.supports_combination(
128,
torch.bfloat16,
"auto",
32,
False,
False,
False,
False,
DeviceCapability(12, 0),
)
if rdna:
assert reason is None
segmented.RocmSegmentedAttentionImpl(8, 128, 128**-0.5, 2, None, None, "auto")
else:
assert reason == "ROCM_SEGMENTED_ATTN requires AMD RDNA GPUs on ROCm"
with pytest.raises(ValueError, match="requires AMD RDNA GPUs on ROCm"):
segmented.RocmSegmentedAttentionImpl(
8, 128, 128**-0.5, 2, None, None, "auto"
)
@pytest.mark.parametrize("explicit_cache", [False, True])
def test_segmented_triton_cache_survives_standalone_compilation(
monkeypatch, tmp_path, explicit_cache
):
"""Backend initialization preserves one cache across model compilations."""
import os
from vllm.compilation.compiler_interface import InductorStandaloneAdaptor
from vllm.platforms import rocm
from vllm.v1.attention.backends.rocm_segmented_attn import (
RocmSegmentedAttentionImpl,
)
monkeypatch.setattr(rocm, "on_gfx1x", lambda: True)
monkeypatch.setenv("VLLM_CACHE_ROOT", str(tmp_path))
monkeypatch.delenv("TRITON_CACHE_DIR", raising=False)
monkeypatch.delenv("TORCHINDUCTOR_CACHE_DIR", raising=False)
expected = tmp_path / "rocm_segmented_attention" / "triton"
if explicit_cache:
expected = tmp_path / "custom"
monkeypatch.setenv("TRITON_CACHE_DIR", str(expected))
for model in ("target", "draft", "restart_target"):
RocmSegmentedAttentionImpl(8, 128, 128**-0.5, 2, None, None, "auto")
assert os.environ["TRITON_CACHE_DIR"] == str(expected)
assert expected.is_dir()
InductorStandaloneAdaptor("binary").initialize_cache(str(tmp_path / model))
assert os.environ["TRITON_CACHE_DIR"] == str(expected)
def test_segmented_attention_is_opt_in(monkeypatch):
from vllm.platforms import rocm
from vllm.platforms.rocm import RocmPlatform, _get_backend_priorities
assert AttentionBackendEnum.ROCM_SEGMENTED_ATTN not in _get_backend_priorities(
use_mla=False, use_sparse=False
)
monkeypatch.setattr(rocm, "on_gfx1x", lambda: True)
monkeypatch.setattr(rocm, "on_gfx12x", lambda: True)
config = AttentionSelectorConfig(
head_size=128,
dtype=torch.bfloat16,
kv_cache_dtype="fp8",
block_size=32,
)
path = RocmPlatform.get_attn_backend_cls(
selected_backend=AttentionBackendEnum.ROCM_SEGMENTED_ATTN,
attn_selector_config=config,
)
assert path == AttentionBackendEnum.ROCM_SEGMENTED_ATTN.get_path()
def test_segmented_attention_rejects_mm_prefix_at_selection(monkeypatch):
from vllm.platforms import rocm
from vllm.platforms.rocm import RocmPlatform
monkeypatch.setattr(rocm, "on_gfx1x", lambda: True)
config = AttentionSelectorConfig(
head_size=128,
dtype=torch.bfloat16,
kv_cache_dtype="auto",
block_size=32,
use_mm_prefix=True,
)
with pytest.raises(ValueError, match="multimodal token full attention"):
RocmPlatform.get_attn_backend_cls(
selected_backend=AttentionBackendEnum.ROCM_SEGMENTED_ATTN,
attn_selector_config=config,
)
def test_segmented_attention_forward_uses_dedicated_dispatch(monkeypatch):
from vllm.config import KernelConfig
from vllm.platforms import rocm
from vllm.v1.attention.backends.rocm_segmented_attn import (
RocmSegmentedAttentionImpl,
)
monkeypatch.setattr(rocm, "on_gfx1x", lambda: True)
impl = RocmSegmentedAttentionImpl(8, 128, 128**-0.5, 2, None, None, "auto")
impl._segmented_attention_config = SimpleNamespace(
kernel_config=KernelConfig(),
scheduler_config=SimpleNamespace(max_num_seqs=1, max_num_batched_tokens=3),
model_config=SimpleNamespace(max_model_len=16),
)
query = torch.empty(3, 8, 128, dtype=torch.bfloat16)
key = torch.empty(3, 2, 128, dtype=torch.bfloat16)
value = torch.empty_like(key)
output = torch.empty_like(query)
kv_cache = torch.empty(2, 2, 16, 256, dtype=torch.bfloat16)
metadata = SimpleNamespace(
use_cascade=False,
num_actual_tokens=3,
block_table=torch.zeros(1, 1, dtype=torch.int32),
query_start_loc=torch.tensor([0, 3], dtype=torch.int32),
seq_lens=torch.tensor([3], dtype=torch.int32),
max_seq_len=3,
max_query_len=3,
causal=True,
)
layer = SimpleNamespace(_k_scale=torch.ones(()), _v_scale=torch.ones(()))
with patch(
"vllm.v1.attention.backends.rocm_segmented_attn.segmented_attention"
) as dispatch:
result = impl.forward(layer, query, key, value, kv_cache, metadata, output)
assert result is output
dispatch.assert_called_once()
assert dispatch.call_args.kwargs["key_cache"].shape == (2, 16, 2, 128)
def test_aiter_unified_attention_capture_preserves_query_start_locations():
from vllm.v1.attention.backends.rocm_aiter_unified_attn import (
RocmAiterUnifiedAttentionMetadataBuilder,