Enable gfx1250 ROCm architecture (#46516)

Signed-off-by: Douglas Lehr <Doug.Lehr@amd.com>
Signed-off-by: jpvillam <juan.villamizar@amd.com>
Signed-off-by: Daniel <danichan@amd.com>
Signed-off-by: Jaden Mathias <jaden.mathias@amd.com>
Signed-off-by: jpvillam <jpvillam@amd.com>
Signed-off-by: Juan Villamizar <100237675+jpvillam-amd@users.noreply.github.com>
Signed-off-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
Signed-off-by: Jaden Mathias <Jaden.Mathias@amd.com>
Co-authored-by: Douglas Lehr <Doug.Lehr@amd.com>
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
Co-authored-by: Daniel <danichan@amd.com>
Co-authored-by: Jaden Mathias <jaden.mathias@amd.com>
Co-authored-by: jpvillam <jpvillam@amd.com>
Co-authored-by: root <root@ctheliosr-1b114-f01-2.mnb.dcgpu>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
Co-authored-by: dllehr <douglehr@amd.com>
Co-authored-by: Daniel Chan <Daniel.Chan@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Juan Villamizar
2026-07-31 01:41:12 +00:00
committed by GitHub
co-authored by Douglas Lehr Claude Opus 4.7 Daniel Jaden Mathias jpvillam root Douglas Lehr dllehr Daniel Chan Andreas Karatzas
parent 553fcb82d5
commit 4f1da84eb5
33 changed files with 1850 additions and 104 deletions
@@ -31,7 +31,7 @@ def _get_attention_backend_params() -> list[str | None]:
falls back to ROCM_AITER_UNIFIED_ATTN or TRITON_ATTN for
cross-attention since ROCM_ATTN doesn't support ENCODER_DECODER)
- TRITON_ATTN: always available on ROCm
- ROCM_AITER_UNIFIED_ATTN: only on gfx942/gfx950
- ROCM_AITER_UNIFIED_ATTN: only on gfx942/gfx950/gfx1250
On non-ROCm platforms, we just run with the default backend.
"""
@@ -40,9 +40,9 @@ def _get_attention_backend_params() -> list[str | None]:
if current_platform.is_rocm():
backends: list[str | None] = [None, "TRITON_ATTN"]
from vllm.platforms.rocm import _ON_MI3XX
from vllm.platforms.rocm import get_cdna_version
if _ON_MI3XX:
if get_cdna_version() > 2:
backends.append("ROCM_AITER_UNIFIED_ATTN")
return backends
except Exception:
@@ -37,13 +37,13 @@ def _get_rocm_attention_config(model_name):
if "whisper" in model_name.lower():
try:
from vllm.platforms.rocm import _ON_MI3XX
from vllm.platforms.rocm import get_cdna_version
if _ON_MI3XX:
if get_cdna_version() > 2:
return {"backend": "ROCM_AITER_UNIFIED_ATTN"}
except ImportError:
logger.warning(
"Could not import _ON_MI3XX from rocm platform, "
"Could not check cdna version from rocm platform, "
"falling back to TRITON_ATTN for Whisper."
)
return {"backend": "TRITON_ATTN"}
@@ -15,15 +15,15 @@ from tests.kernels.attention.test_triton_unified_attention import ref_paged_attn
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
_SKIP_NON_MI3XX = True
_SKIP_NON_CDNA_2_PLUS = True
if current_platform.is_rocm():
from vllm.platforms.rocm import on_mi3xx
from vllm.platforms.rocm import get_cdna_version
_SKIP_NON_MI3XX = not on_mi3xx()
_SKIP_NON_CDNA_2_PLUS = get_cdna_version() < 2
pytestmark = [
pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-specific tests"),
pytest.mark.skipif(_SKIP_NON_MI3XX, reason="MI300/MI350 ROCm only"),
pytest.mark.skipif(_SKIP_NON_CDNA_2_PLUS, reason="CDNA 2+ ROCm only"),
]
NUM_Q_HEADS = 8
@@ -4,6 +4,8 @@
Test modular OAI Triton MoE
"""
from __future__ import annotations
import pytest
import torch
@@ -48,6 +50,35 @@ MNK = [
]
def deepseek_v4_flash_moe_topology():
"""MoE sizes for the DeepSeek-V4-Flash`.
Default weights from: ``deepseek-ai/DeepSeek-V4-Flash`.
``moe_intermediate_size``, ``n_routed_experts``, and ``num_experts_per_tok``.
"""
defaults = {
"hidden_size": 4096,
"moe_intermediate_size": 2048,
"n_routed_experts": 256,
"num_experts_per_tok": 6,
}
return defaults
def scaled_deepseek_v4_flash_problem(
*,
dim_scale: int = 8,
expert_scale: int = 8,
):
"""Smaller K/N/E for kernel tests; keeps production top_k and K:N ratio (~2:1)."""
t = deepseek_v4_flash_moe_topology()
k = max(128, t["hidden_size"] // dim_scale)
n = max(64, t["moe_intermediate_size"] // dim_scale)
num_experts = max(t["num_experts_per_tok"], t["n_routed_experts"] // expert_scale)
topk = t["num_experts_per_tok"]
return k, n, num_experts, topk
def unshuffle_weight(w: torch.Tensor):
first = w[..., ::2]
second = w[..., 1::2]
@@ -160,8 +191,8 @@ def oai_triton_moe_impl(
x: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
w1_scale: "PrecisionConfig",
w2_scale: "PrecisionConfig",
w1_scale: PrecisionConfig,
w2_scale: PrecisionConfig,
w1_bias: torch.Tensor | None,
w2_bias: torch.Tensor | None,
num_experts: int,
@@ -261,3 +292,93 @@ def test_oai_triton_moe(
)
assert_close(ref=out_ref, tri=out, maxtol=0.025, rmstol=0.005)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
)
def test_unfused_oai_triton_experts_apply_direct_deepseek_v4_topology(workspace_init):
"""Exercise ``UnfusedOAITritonExperts.apply`` with explicit workspaces.
Same MoE topology as ``launch_dsv4.sh`` / DeepSeek-V4-Flash ``config.json``,
with linear dimensions and expert count scaled down for test GPU memory.
"""
wait_for_gpu_memory_to_clear(devices=[0], threshold_ratio=0.1)
set_random_seed(0)
k, n, num_experts, topk = scaled_deepseek_v4_flash_problem()
m = 7
dtype = torch.bfloat16
(
w1,
w2,
w1_bias,
w2_bias,
w1_tri,
w2_tri,
w1_bias_tri,
w2_bias_tri,
w1_precision_config,
w2_precision_config,
) = make_weights(dtype, k, n, num_experts)
x = torch.randn((m, k), dtype=dtype, device="cuda")
router_logits = torch.randn(m, num_experts, device="cuda", dtype=dtype)
topk_weights, topk_ids = torch.topk(router_logits, k=topk, dim=-1, sorted=True)
topk_weights = torch.nn.functional.softmax(topk_weights, dim=-1)
quant_config = mxfp4_w4a16_moe_quant_config(
w1_bias=w1_bias_tri,
w2_bias=w2_bias_tri,
w1_scale=w1_precision_config,
w2_scale=w2_precision_config,
)
moe_config = make_dummy_moe_config(
num_experts=num_experts,
experts_per_token=topk,
hidden_dim=k,
intermediate_size=n,
)
experts = UnfusedOAITritonExperts(moe_config, quant_config)
if not UnfusedOAITritonExperts._supports_current_device():
pytest.skip("UnfusedOAITritonExperts does not support this device")
_, _, N, K, top_k = experts.moe_problem_size(x, w1_tri, w2_tri, topk_ids)
assert top_k == topk
ws13_shape, ws2_shape, out_shape = experts.workspace_shapes(
m,
N,
K,
topk,
num_experts,
num_experts,
None,
MoEActivation.SWIGLUOAI,
)
workspace13 = torch.empty(ws13_shape, dtype=dtype, device="cuda")
workspace2 = torch.empty(ws2_shape, dtype=dtype, device="cuda")
output = torch.empty(out_shape, dtype=dtype, device="cuda")
with set_current_vllm_config(VllmConfig()):
out_ref = torch_moe_impl(x, w1, w2, w1_bias, w2_bias, topk_weights, topk_ids)
experts.apply(
output=output,
hidden_states=x,
w1=w1_tri,
w2=w2_tri,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=MoEActivation.SWIGLUOAI,
global_num_experts=num_experts,
expert_map=None,
a1q_scale=None,
a2_scale=None,
workspace13=workspace13,
workspace2=workspace2,
expert_tokens_meta=None,
apply_router_weight_on_input=False,
)
assert_close(ref=out_ref, tri=output, maxtol=0.025, rmstol=0.005)
+3 -3
View File
@@ -29,13 +29,13 @@ from vllm.platforms import current_platform
def _is_aiter_capable() -> bool:
"""Check if the platform supports AITER (gfx942/gfx950)."""
"""Check if the platform supports AITER (gfx942/gfx950/gfx1250)."""
if not current_platform.is_rocm():
return False
try:
from vllm.platforms.rocm import _ON_MI3XX
from vllm.platforms.rocm import get_cdna_version
return _ON_MI3XX
return get_cdna_version() > 2
except ImportError:
return False
@@ -26,6 +26,7 @@ def test_rocm_unquantized_gemm_gfx1x_wvsplitk_path(monkeypatch):
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
@@ -52,6 +53,7 @@ def test_rocm_unquantized_gemm_gfx1x_n_gt_5_falls_back(monkeypatch):
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
@@ -76,6 +78,7 @@ def test_rocm_unquantized_gemm_gfx950_wvsplitkrc_path(monkeypatch):
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: True)
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
wvsplitkrc_mock = MagicMock(side_effect=lambda x_view, w, _, __: x_view @ w.t())
@@ -45,9 +45,10 @@ models: dict[str, str | None] = {
def granite_speech_attention_config():
"""Return attention config for Granite Speech tests on ROCm."""
if current_platform.is_rocm():
from vllm.platforms.rocm import on_mi3xx
from vllm.platforms.rocm import get_cdna_version
if on_mi3xx():
# -1 (unknown arch) is truthy; gate on CDNA3+ like other call sites.
if get_cdna_version() > 2:
return {"backend": "ROCM_AITER_FA"}
return {"backend": "TRITON_ATTN"}
return None
@@ -21,11 +21,11 @@ from ...utils import compare_two_settings, multi_gpu_test
from ..utils import check_embeddings_close, check_logprobs_close
if current_platform.is_rocm():
from vllm.platforms.rocm import on_gfx9
from vllm.platforms.rocm import on_cdna
pytestmark = pytest.mark.skipif(
on_gfx9(),
reason="bitsandbytes not supported on gfx9 (warp size 64 limitation)",
on_cdna(),
reason="bitsandbytes not supported on CDNA (warp size 64 limitation)",
)
models_4bit_to_test = [
@@ -28,16 +28,9 @@ def mock_vllm_config():
@pytest.fixture
def mock_on_gfx9():
"""Mock gfx9 arch detection to return True."""
with patch("vllm.platforms.rocm.on_gfx9", return_value=True):
yield
@pytest.fixture
def mock_on_mi3xx():
"""Mock mi3xx arch detection to return True."""
with patch("vllm.platforms.rocm.on_mi3xx", return_value=True):
def mock_get_cdna_version():
"""Mock cdna version arch detection to return True."""
with patch("vllm.platforms.rocm.get_cdna_version", return_value=3):
yield
@@ -111,8 +104,7 @@ def test_standard_attention_backend_selection(
selected_backend,
expected_backend_path,
mock_vllm_config,
mock_on_gfx9,
mock_on_mi3xx,
mock_get_cdna_version,
monkeypatch,
):
"""Test standard attention backend selection with various configurations."""
@@ -305,12 +297,12 @@ def test_mla_backend_selection(
def test_aiter_fa_requires_mi3xx(mock_vllm_config):
"""Test that ROCM_AITER_FA requires mi3xx architecture."""
"""Test that ROCM_AITER_FA requires CDNA3+ architecture."""
from vllm.platforms.rocm import RocmPlatform
# Mock on_mi3xx to return False (used by supports_compute_capability)
# Mock cdna version to return 1 (used by supports_compute_capability)
with (
patch("vllm.platforms.rocm.on_mi3xx", return_value=False),
patch("vllm.platforms.rocm.get_cdna_version", return_value=1),
pytest.raises(
ValueError,
match="compute capability not supported",