Enable gfx1250 ROCm architecture (#46516)
Signed-off-by: Douglas Lehr <Doug.Lehr@amd.com> Signed-off-by: jpvillam <juan.villamizar@amd.com> Signed-off-by: Daniel <danichan@amd.com> Signed-off-by: Jaden Mathias <jaden.mathias@amd.com> Signed-off-by: jpvillam <jpvillam@amd.com> Signed-off-by: Juan Villamizar <100237675+jpvillam-amd@users.noreply.github.com> Signed-off-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com> Signed-off-by: Jaden Mathias <Jaden.Mathias@amd.com> Co-authored-by: Douglas Lehr <Doug.Lehr@amd.com> Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com> Co-authored-by: Daniel <danichan@amd.com> Co-authored-by: Jaden Mathias <jaden.mathias@amd.com> Co-authored-by: jpvillam <jpvillam@amd.com> Co-authored-by: root <root@ctheliosr-1b114-f01-2.mnb.dcgpu> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com> Co-authored-by: dllehr <douglehr@amd.com> Co-authored-by: Daniel Chan <Daniel.Chan@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Douglas Lehr
Claude Opus 4.7
Daniel
Jaden Mathias
jpvillam
root
Douglas Lehr
dllehr
Daniel Chan
Andreas Karatzas
parent
553fcb82d5
commit
4f1da84eb5
+3
-3
@@ -31,7 +31,7 @@ def _get_attention_backend_params() -> list[str | None]:
|
||||
falls back to ROCM_AITER_UNIFIED_ATTN or TRITON_ATTN for
|
||||
cross-attention since ROCM_ATTN doesn't support ENCODER_DECODER)
|
||||
- TRITON_ATTN: always available on ROCm
|
||||
- ROCM_AITER_UNIFIED_ATTN: only on gfx942/gfx950
|
||||
- ROCM_AITER_UNIFIED_ATTN: only on gfx942/gfx950/gfx1250
|
||||
|
||||
On non-ROCm platforms, we just run with the default backend.
|
||||
"""
|
||||
@@ -40,9 +40,9 @@ def _get_attention_backend_params() -> list[str | None]:
|
||||
|
||||
if current_platform.is_rocm():
|
||||
backends: list[str | None] = [None, "TRITON_ATTN"]
|
||||
from vllm.platforms.rocm import _ON_MI3XX
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
|
||||
if _ON_MI3XX:
|
||||
if get_cdna_version() > 2:
|
||||
backends.append("ROCM_AITER_UNIFIED_ATTN")
|
||||
return backends
|
||||
except Exception:
|
||||
|
||||
@@ -37,13 +37,13 @@ def _get_rocm_attention_config(model_name):
|
||||
|
||||
if "whisper" in model_name.lower():
|
||||
try:
|
||||
from vllm.platforms.rocm import _ON_MI3XX
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
|
||||
if _ON_MI3XX:
|
||||
if get_cdna_version() > 2:
|
||||
return {"backend": "ROCM_AITER_UNIFIED_ATTN"}
|
||||
except ImportError:
|
||||
logger.warning(
|
||||
"Could not import _ON_MI3XX from rocm platform, "
|
||||
"Could not check cdna version from rocm platform, "
|
||||
"falling back to TRITON_ATTN for Whisper."
|
||||
)
|
||||
return {"backend": "TRITON_ATTN"}
|
||||
|
||||
@@ -15,15 +15,15 @@ from tests.kernels.attention.test_triton_unified_attention import ref_paged_attn
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
_SKIP_NON_MI3XX = True
|
||||
_SKIP_NON_CDNA_2_PLUS = True
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_mi3xx
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
|
||||
_SKIP_NON_MI3XX = not on_mi3xx()
|
||||
_SKIP_NON_CDNA_2_PLUS = get_cdna_version() < 2
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-specific tests"),
|
||||
pytest.mark.skipif(_SKIP_NON_MI3XX, reason="MI300/MI350 ROCm only"),
|
||||
pytest.mark.skipif(_SKIP_NON_CDNA_2_PLUS, reason="CDNA 2+ ROCm only"),
|
||||
]
|
||||
|
||||
NUM_Q_HEADS = 8
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
Test modular OAI Triton MoE
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -48,6 +50,35 @@ MNK = [
|
||||
]
|
||||
|
||||
|
||||
def deepseek_v4_flash_moe_topology():
|
||||
"""MoE sizes for the DeepSeek-V4-Flash`.
|
||||
Default weights from: ``deepseek-ai/DeepSeek-V4-Flash`.
|
||||
``moe_intermediate_size``, ``n_routed_experts``, and ``num_experts_per_tok``.
|
||||
"""
|
||||
defaults = {
|
||||
"hidden_size": 4096,
|
||||
"moe_intermediate_size": 2048,
|
||||
"n_routed_experts": 256,
|
||||
"num_experts_per_tok": 6,
|
||||
}
|
||||
|
||||
return defaults
|
||||
|
||||
|
||||
def scaled_deepseek_v4_flash_problem(
|
||||
*,
|
||||
dim_scale: int = 8,
|
||||
expert_scale: int = 8,
|
||||
):
|
||||
"""Smaller K/N/E for kernel tests; keeps production top_k and K:N ratio (~2:1)."""
|
||||
t = deepseek_v4_flash_moe_topology()
|
||||
k = max(128, t["hidden_size"] // dim_scale)
|
||||
n = max(64, t["moe_intermediate_size"] // dim_scale)
|
||||
num_experts = max(t["num_experts_per_tok"], t["n_routed_experts"] // expert_scale)
|
||||
topk = t["num_experts_per_tok"]
|
||||
return k, n, num_experts, topk
|
||||
|
||||
|
||||
def unshuffle_weight(w: torch.Tensor):
|
||||
first = w[..., ::2]
|
||||
second = w[..., 1::2]
|
||||
@@ -160,8 +191,8 @@ def oai_triton_moe_impl(
|
||||
x: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w1_scale: "PrecisionConfig",
|
||||
w2_scale: "PrecisionConfig",
|
||||
w1_scale: PrecisionConfig,
|
||||
w2_scale: PrecisionConfig,
|
||||
w1_bias: torch.Tensor | None,
|
||||
w2_bias: torch.Tensor | None,
|
||||
num_experts: int,
|
||||
@@ -261,3 +292,93 @@ def test_oai_triton_moe(
|
||||
)
|
||||
|
||||
assert_close(ref=out_ref, tri=out, maxtol=0.025, rmstol=0.005)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
def test_unfused_oai_triton_experts_apply_direct_deepseek_v4_topology(workspace_init):
|
||||
"""Exercise ``UnfusedOAITritonExperts.apply`` with explicit workspaces.
|
||||
|
||||
Same MoE topology as ``launch_dsv4.sh`` / DeepSeek-V4-Flash ``config.json``,
|
||||
with linear dimensions and expert count scaled down for test GPU memory.
|
||||
"""
|
||||
wait_for_gpu_memory_to_clear(devices=[0], threshold_ratio=0.1)
|
||||
set_random_seed(0)
|
||||
|
||||
k, n, num_experts, topk = scaled_deepseek_v4_flash_problem()
|
||||
m = 7
|
||||
dtype = torch.bfloat16
|
||||
|
||||
(
|
||||
w1,
|
||||
w2,
|
||||
w1_bias,
|
||||
w2_bias,
|
||||
w1_tri,
|
||||
w2_tri,
|
||||
w1_bias_tri,
|
||||
w2_bias_tri,
|
||||
w1_precision_config,
|
||||
w2_precision_config,
|
||||
) = make_weights(dtype, k, n, num_experts)
|
||||
|
||||
x = torch.randn((m, k), dtype=dtype, device="cuda")
|
||||
router_logits = torch.randn(m, num_experts, device="cuda", dtype=dtype)
|
||||
topk_weights, topk_ids = torch.topk(router_logits, k=topk, dim=-1, sorted=True)
|
||||
topk_weights = torch.nn.functional.softmax(topk_weights, dim=-1)
|
||||
|
||||
quant_config = mxfp4_w4a16_moe_quant_config(
|
||||
w1_bias=w1_bias_tri,
|
||||
w2_bias=w2_bias_tri,
|
||||
w1_scale=w1_precision_config,
|
||||
w2_scale=w2_precision_config,
|
||||
)
|
||||
moe_config = make_dummy_moe_config(
|
||||
num_experts=num_experts,
|
||||
experts_per_token=topk,
|
||||
hidden_dim=k,
|
||||
intermediate_size=n,
|
||||
)
|
||||
experts = UnfusedOAITritonExperts(moe_config, quant_config)
|
||||
|
||||
if not UnfusedOAITritonExperts._supports_current_device():
|
||||
pytest.skip("UnfusedOAITritonExperts does not support this device")
|
||||
|
||||
_, _, N, K, top_k = experts.moe_problem_size(x, w1_tri, w2_tri, topk_ids)
|
||||
assert top_k == topk
|
||||
ws13_shape, ws2_shape, out_shape = experts.workspace_shapes(
|
||||
m,
|
||||
N,
|
||||
K,
|
||||
topk,
|
||||
num_experts,
|
||||
num_experts,
|
||||
None,
|
||||
MoEActivation.SWIGLUOAI,
|
||||
)
|
||||
workspace13 = torch.empty(ws13_shape, dtype=dtype, device="cuda")
|
||||
workspace2 = torch.empty(ws2_shape, dtype=dtype, device="cuda")
|
||||
output = torch.empty(out_shape, dtype=dtype, device="cuda")
|
||||
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
out_ref = torch_moe_impl(x, w1, w2, w1_bias, w2_bias, topk_weights, topk_ids)
|
||||
experts.apply(
|
||||
output=output,
|
||||
hidden_states=x,
|
||||
w1=w1_tri,
|
||||
w2=w2_tri,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=MoEActivation.SWIGLUOAI,
|
||||
global_num_experts=num_experts,
|
||||
expert_map=None,
|
||||
a1q_scale=None,
|
||||
a2_scale=None,
|
||||
workspace13=workspace13,
|
||||
workspace2=workspace2,
|
||||
expert_tokens_meta=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
assert_close(ref=out_ref, tri=output, maxtol=0.025, rmstol=0.005)
|
||||
|
||||
@@ -29,13 +29,13 @@ from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _is_aiter_capable() -> bool:
|
||||
"""Check if the platform supports AITER (gfx942/gfx950)."""
|
||||
"""Check if the platform supports AITER (gfx942/gfx950/gfx1250)."""
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
try:
|
||||
from vllm.platforms.rocm import _ON_MI3XX
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
|
||||
return _ON_MI3XX
|
||||
return get_cdna_version() > 2
|
||||
except ImportError:
|
||||
return False
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ def test_rocm_unquantized_gemm_gfx1x_wvsplitk_path(monkeypatch):
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)
|
||||
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
|
||||
|
||||
wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
|
||||
@@ -52,6 +53,7 @@ def test_rocm_unquantized_gemm_gfx1x_n_gt_5_falls_back(monkeypatch):
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)
|
||||
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
|
||||
|
||||
wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
|
||||
@@ -76,6 +78,7 @@ def test_rocm_unquantized_gemm_gfx950_wvsplitkrc_path(monkeypatch):
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: True)
|
||||
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
|
||||
|
||||
wvsplitkrc_mock = MagicMock(side_effect=lambda x_view, w, _, __: x_view @ w.t())
|
||||
|
||||
@@ -45,9 +45,10 @@ models: dict[str, str | None] = {
|
||||
def granite_speech_attention_config():
|
||||
"""Return attention config for Granite Speech tests on ROCm."""
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_mi3xx
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
|
||||
if on_mi3xx():
|
||||
# -1 (unknown arch) is truthy; gate on CDNA3+ like other call sites.
|
||||
if get_cdna_version() > 2:
|
||||
return {"backend": "ROCM_AITER_FA"}
|
||||
return {"backend": "TRITON_ATTN"}
|
||||
return None
|
||||
|
||||
@@ -21,11 +21,11 @@ from ...utils import compare_two_settings, multi_gpu_test
|
||||
from ..utils import check_embeddings_close, check_logprobs_close
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_gfx9
|
||||
from vllm.platforms.rocm import on_cdna
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
on_gfx9(),
|
||||
reason="bitsandbytes not supported on gfx9 (warp size 64 limitation)",
|
||||
on_cdna(),
|
||||
reason="bitsandbytes not supported on CDNA (warp size 64 limitation)",
|
||||
)
|
||||
|
||||
models_4bit_to_test = [
|
||||
|
||||
@@ -28,16 +28,9 @@ def mock_vllm_config():
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_on_gfx9():
|
||||
"""Mock gfx9 arch detection to return True."""
|
||||
with patch("vllm.platforms.rocm.on_gfx9", return_value=True):
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_on_mi3xx():
|
||||
"""Mock mi3xx arch detection to return True."""
|
||||
with patch("vllm.platforms.rocm.on_mi3xx", return_value=True):
|
||||
def mock_get_cdna_version():
|
||||
"""Mock cdna version arch detection to return True."""
|
||||
with patch("vllm.platforms.rocm.get_cdna_version", return_value=3):
|
||||
yield
|
||||
|
||||
|
||||
@@ -111,8 +104,7 @@ def test_standard_attention_backend_selection(
|
||||
selected_backend,
|
||||
expected_backend_path,
|
||||
mock_vllm_config,
|
||||
mock_on_gfx9,
|
||||
mock_on_mi3xx,
|
||||
mock_get_cdna_version,
|
||||
monkeypatch,
|
||||
):
|
||||
"""Test standard attention backend selection with various configurations."""
|
||||
@@ -305,12 +297,12 @@ def test_mla_backend_selection(
|
||||
|
||||
|
||||
def test_aiter_fa_requires_mi3xx(mock_vllm_config):
|
||||
"""Test that ROCM_AITER_FA requires mi3xx architecture."""
|
||||
"""Test that ROCM_AITER_FA requires CDNA3+ architecture."""
|
||||
from vllm.platforms.rocm import RocmPlatform
|
||||
|
||||
# Mock on_mi3xx to return False (used by supports_compute_capability)
|
||||
# Mock cdna version to return 1 (used by supports_compute_capability)
|
||||
with (
|
||||
patch("vllm.platforms.rocm.on_mi3xx", return_value=False),
|
||||
patch("vllm.platforms.rocm.get_cdna_version", return_value=1),
|
||||
pytest.raises(
|
||||
ValueError,
|
||||
match="compute capability not supported",
|
||||
|
||||
Reference in New Issue
Block a user