[ROCM][Perf] Add ROCM_SEGMENTED_ATTN attention-backend works across RDNA3/3.5/4 (#59132)
Signed-off-by: root <jeffaw99@hotmail.com> Signed-off-by: big-yellow-duck <jeffaw99@hotmail.com> Co-authored-by: Tan Pin Siang <1716735+tanpinsiang@users.noreply.github.com> Co-authored-by: vllmellm <190700713+vllmellm@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Tan Pin Siang
vllmellm
Codex
parent
15c1cc12ac
commit
edfccc9469
File diff suppressed because it is too large
Load Diff
@@ -61,6 +61,216 @@ def test_aiter_unified_attention_uses_dedicated_metadata_builder():
|
||||
)
|
||||
|
||||
|
||||
def test_segmented_attention_uses_dedicated_backend_components():
|
||||
from vllm.v1.attention.backends.rocm_segmented_attn import (
|
||||
RocmSegmentedAttentionBackend,
|
||||
RocmSegmentedAttentionImpl,
|
||||
RocmSegmentedAttentionMetadataBuilder,
|
||||
)
|
||||
|
||||
assert (
|
||||
RocmSegmentedAttentionBackend.get_builder_cls()
|
||||
is RocmSegmentedAttentionMetadataBuilder
|
||||
)
|
||||
assert RocmSegmentedAttentionBackend.get_impl_cls() is RocmSegmentedAttentionImpl
|
||||
assert RocmSegmentedAttentionBackend.get_name() == "ROCM_SEGMENTED_ATTN"
|
||||
assert RocmSegmentedAttentionBackend.supports_non_causal()
|
||||
assert RocmSegmentedAttentionBackend.supports_sliding_window()
|
||||
assert RocmSegmentedAttentionBackend.supports_sink()
|
||||
assert not RocmSegmentedAttentionBackend.supports_mm_prefix()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"sliding_window,logits_soft_cap,has_sinks",
|
||||
[(4096, None, False), (None, 50.0, False), (None, None, True)],
|
||||
)
|
||||
def test_segmented_attention_admits_unified_fallback_features(
|
||||
monkeypatch, sliding_window, logits_soft_cap, has_sinks
|
||||
):
|
||||
from vllm.platforms import rocm
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
from vllm.v1.attention.backends.rocm_segmented_attn import (
|
||||
RocmSegmentedAttentionImpl,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(rocm, "on_gfx1x", lambda: True)
|
||||
monkeypatch.setattr(rocm, "on_gfx12x", lambda: True)
|
||||
sinks = torch.zeros(8) if has_sinks else None
|
||||
|
||||
impl = RocmSegmentedAttentionImpl(
|
||||
num_heads=8,
|
||||
head_size=128,
|
||||
scale=128**-0.5,
|
||||
num_kv_heads=2,
|
||||
alibi_slopes=None,
|
||||
sliding_window=sliding_window,
|
||||
kv_cache_dtype="auto",
|
||||
logits_soft_cap=logits_soft_cap,
|
||||
attn_type=AttentionType.DECODER,
|
||||
sinks=sinks,
|
||||
)
|
||||
|
||||
assert isinstance(impl, RocmSegmentedAttentionImpl)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"platform,rdna",
|
||||
[("cuda", False), ("cpu", False), ("xpu", False), ("rocm", False), ("rocm", True)],
|
||||
)
|
||||
def test_segmented_attention_requires_rocm_rdna(monkeypatch, platform, rdna):
|
||||
"""Reject unsupported platforms before probing ROCm or constructing kernels."""
|
||||
from vllm.platforms import rocm
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.v1.attention.backends import rocm_segmented_attn as segmented
|
||||
|
||||
monkeypatch.setattr(
|
||||
segmented,
|
||||
"current_platform",
|
||||
SimpleNamespace(is_rocm=lambda: platform == "rocm"),
|
||||
)
|
||||
|
||||
def on_rdna():
|
||||
assert platform == "rocm", "Must not probe ROCm on another platform"
|
||||
return rdna
|
||||
|
||||
monkeypatch.setattr(rocm, "on_gfx1x", on_rdna)
|
||||
reason = segmented.RocmSegmentedAttentionBackend.supports_combination(
|
||||
128,
|
||||
torch.bfloat16,
|
||||
"auto",
|
||||
32,
|
||||
False,
|
||||
False,
|
||||
False,
|
||||
False,
|
||||
DeviceCapability(12, 0),
|
||||
)
|
||||
if rdna:
|
||||
assert reason is None
|
||||
segmented.RocmSegmentedAttentionImpl(8, 128, 128**-0.5, 2, None, None, "auto")
|
||||
else:
|
||||
assert reason == "ROCM_SEGMENTED_ATTN requires AMD RDNA GPUs on ROCm"
|
||||
with pytest.raises(ValueError, match="requires AMD RDNA GPUs on ROCm"):
|
||||
segmented.RocmSegmentedAttentionImpl(
|
||||
8, 128, 128**-0.5, 2, None, None, "auto"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("explicit_cache", [False, True])
|
||||
def test_segmented_triton_cache_survives_standalone_compilation(
|
||||
monkeypatch, tmp_path, explicit_cache
|
||||
):
|
||||
"""Backend initialization preserves one cache across model compilations."""
|
||||
import os
|
||||
|
||||
from vllm.compilation.compiler_interface import InductorStandaloneAdaptor
|
||||
from vllm.platforms import rocm
|
||||
from vllm.v1.attention.backends.rocm_segmented_attn import (
|
||||
RocmSegmentedAttentionImpl,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(rocm, "on_gfx1x", lambda: True)
|
||||
monkeypatch.setenv("VLLM_CACHE_ROOT", str(tmp_path))
|
||||
monkeypatch.delenv("TRITON_CACHE_DIR", raising=False)
|
||||
monkeypatch.delenv("TORCHINDUCTOR_CACHE_DIR", raising=False)
|
||||
expected = tmp_path / "rocm_segmented_attention" / "triton"
|
||||
if explicit_cache:
|
||||
expected = tmp_path / "custom"
|
||||
monkeypatch.setenv("TRITON_CACHE_DIR", str(expected))
|
||||
for model in ("target", "draft", "restart_target"):
|
||||
RocmSegmentedAttentionImpl(8, 128, 128**-0.5, 2, None, None, "auto")
|
||||
assert os.environ["TRITON_CACHE_DIR"] == str(expected)
|
||||
assert expected.is_dir()
|
||||
InductorStandaloneAdaptor("binary").initialize_cache(str(tmp_path / model))
|
||||
assert os.environ["TRITON_CACHE_DIR"] == str(expected)
|
||||
|
||||
|
||||
def test_segmented_attention_is_opt_in(monkeypatch):
|
||||
from vllm.platforms import rocm
|
||||
from vllm.platforms.rocm import RocmPlatform, _get_backend_priorities
|
||||
|
||||
assert AttentionBackendEnum.ROCM_SEGMENTED_ATTN not in _get_backend_priorities(
|
||||
use_mla=False, use_sparse=False
|
||||
)
|
||||
|
||||
monkeypatch.setattr(rocm, "on_gfx1x", lambda: True)
|
||||
monkeypatch.setattr(rocm, "on_gfx12x", lambda: True)
|
||||
config = AttentionSelectorConfig(
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="fp8",
|
||||
block_size=32,
|
||||
)
|
||||
|
||||
path = RocmPlatform.get_attn_backend_cls(
|
||||
selected_backend=AttentionBackendEnum.ROCM_SEGMENTED_ATTN,
|
||||
attn_selector_config=config,
|
||||
)
|
||||
|
||||
assert path == AttentionBackendEnum.ROCM_SEGMENTED_ATTN.get_path()
|
||||
|
||||
|
||||
def test_segmented_attention_rejects_mm_prefix_at_selection(monkeypatch):
|
||||
from vllm.platforms import rocm
|
||||
from vllm.platforms.rocm import RocmPlatform
|
||||
|
||||
monkeypatch.setattr(rocm, "on_gfx1x", lambda: True)
|
||||
config = AttentionSelectorConfig(
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="auto",
|
||||
block_size=32,
|
||||
use_mm_prefix=True,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="multimodal token full attention"):
|
||||
RocmPlatform.get_attn_backend_cls(
|
||||
selected_backend=AttentionBackendEnum.ROCM_SEGMENTED_ATTN,
|
||||
attn_selector_config=config,
|
||||
)
|
||||
|
||||
|
||||
def test_segmented_attention_forward_uses_dedicated_dispatch(monkeypatch):
|
||||
from vllm.config import KernelConfig
|
||||
from vllm.platforms import rocm
|
||||
from vllm.v1.attention.backends.rocm_segmented_attn import (
|
||||
RocmSegmentedAttentionImpl,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(rocm, "on_gfx1x", lambda: True)
|
||||
impl = RocmSegmentedAttentionImpl(8, 128, 128**-0.5, 2, None, None, "auto")
|
||||
impl._segmented_attention_config = SimpleNamespace(
|
||||
kernel_config=KernelConfig(),
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=1, max_num_batched_tokens=3),
|
||||
model_config=SimpleNamespace(max_model_len=16),
|
||||
)
|
||||
query = torch.empty(3, 8, 128, dtype=torch.bfloat16)
|
||||
key = torch.empty(3, 2, 128, dtype=torch.bfloat16)
|
||||
value = torch.empty_like(key)
|
||||
output = torch.empty_like(query)
|
||||
kv_cache = torch.empty(2, 2, 16, 256, dtype=torch.bfloat16)
|
||||
metadata = SimpleNamespace(
|
||||
use_cascade=False,
|
||||
num_actual_tokens=3,
|
||||
block_table=torch.zeros(1, 1, dtype=torch.int32),
|
||||
query_start_loc=torch.tensor([0, 3], dtype=torch.int32),
|
||||
seq_lens=torch.tensor([3], dtype=torch.int32),
|
||||
max_seq_len=3,
|
||||
max_query_len=3,
|
||||
causal=True,
|
||||
)
|
||||
layer = SimpleNamespace(_k_scale=torch.ones(()), _v_scale=torch.ones(()))
|
||||
|
||||
with patch(
|
||||
"vllm.v1.attention.backends.rocm_segmented_attn.segmented_attention"
|
||||
) as dispatch:
|
||||
result = impl.forward(layer, query, key, value, kv_cache, metadata, output)
|
||||
|
||||
assert result is output
|
||||
dispatch.assert_called_once()
|
||||
assert dispatch.call_args.kwargs["key_cache"].shape == (2, 16, 2, 128)
|
||||
|
||||
|
||||
def test_aiter_unified_attention_capture_preserves_query_start_locations():
|
||||
from vllm.v1.attention.backends.rocm_aiter_unified_attn import (
|
||||
RocmAiterUnifiedAttentionMetadataBuilder,
|
||||
|
||||
Reference in New Issue
Block a user