[ROCm][Perf] Use a stacked hipBLASLt GEMM for the bf16x3 router (#52668)

Signed-off-by: Aakif Nawaz <aakif.nawaz@amd.com>
This commit is contained in:
akii96
2026-10-05 17:28:07 +00:00
committed by GitHub
parent 54d93af9fb
commit 182247257f
5 changed files with 594 additions and 4 deletions
@@ -11,6 +11,8 @@ The ROCm branch is guarded on ``not bias`` because ``torch.mm`` has no bias
term; a biased gate must fall back so the bias is not silently dropped.
"""
import inspect
import pytest
import torch
@@ -168,3 +170,91 @@ def test_rocm_fp32_router_gemm_rejects_unsupported_configs(
on_gfx950=on_gfx950,
)
assert not gate.allow_fp32_router_gemm
# ---------------------------------------------------------------------------
# ROCm BF16x3 tier (fp32 router weights)
# ---------------------------------------------------------------------------
def _make_bf16x3_gate(monkeypatch, *, supported=True, **kwargs):
"""Build a gate with the ROCm bf16x3 platform check mocked."""
from vllm.model_executor.layers.fused_moe.router import (
bf16x3_router_gemm_rocm as rocm_bf16x3,
)
monkeypatch.setattr(rocm_bf16x3, "platform_supported", lambda: supported)
kwargs.setdefault("params_dtype", torch.float32)
return _make_gate(monkeypatch, is_rocm=True, **kwargs)
def test_rocm_bf16x3_enabled_for_fp32_weight(monkeypatch):
gate = _make_bf16x3_gate(monkeypatch)
assert gate.allow_rocm_bf16x3_router_gemm
# Built by process_weights_after_loading, not lazily in forward().
# Registered up front but still None, so named_buffers() skips it.
assert gate._bf16x3_weight is None
assert "_bf16x3_weight" in gate._buffers
def test_rocm_bf16x3_disabled_by_bias(monkeypatch):
# torch.mm has no bias term.
gate = _make_bf16x3_gate(monkeypatch, bias=True)
assert not gate.allow_rocm_bf16x3_router_gemm
def test_rocm_bf16x3_disabled_for_bf16_weight(monkeypatch):
# A bf16 weight belongs to the cuBLAS tier; there is nothing to split.
gate = _make_bf16x3_gate(monkeypatch, params_dtype=torch.bfloat16)
assert not gate.allow_rocm_bf16x3_router_gemm
def test_rocm_bf16x3_disabled_for_non_fp32_out_dtype(monkeypatch):
gate = _make_bf16x3_gate(monkeypatch, out_dtype=torch.bfloat16)
assert not gate.allow_rocm_bf16x3_router_gemm
def test_rocm_bf16x3_disabled_on_unsupported_arch(monkeypatch):
# Only gfx950 is benchmarked; gfx942 has a different fp32:bf16 MFMA ratio.
gate = _make_bf16x3_gate(monkeypatch, supported=False)
assert not gate.allow_rocm_bf16x3_router_gemm
assert not gate._rocm_bf16x3_weight_eligible
def test_rocm_bf16x3_no_split_built_when_tier_disabled(monkeypatch):
"""An ineligible out_dtype must not allocate a split that is never read;
it is 1.5x the router weight on every layer."""
gate = _make_bf16x3_gate(monkeypatch, out_dtype=torch.bfloat16)
gate.quant_method.process_weights_after_loading(gate)
assert gate._bf16x3_weight is None
def test_rocm_bf16x3_set_out_dtype_enables_tier(monkeypatch):
gate = _make_bf16x3_gate(monkeypatch, out_dtype=None)
assert not gate.allow_rocm_bf16x3_router_gemm
gate.set_out_dtype(torch.float32)
assert gate.allow_rocm_bf16x3_router_gemm
def test_rocm_bf16x3_dispatch_is_a_custom_op():
"""The num_tokens branch must stay opaque to Dynamo; see the op's impl."""
assert hasattr(torch.ops.vllm, "rocm_bf16x3_router_gemm_dispatch")
x = torch.zeros(8192, 6144, dtype=torch.bfloat16, device="meta")
w = torch.zeros(128, 6144, dtype=torch.float32, device="meta")
split = torch.zeros(3, 128, 6144, dtype=torch.bfloat16, device="meta")
out = torch.ops.vllm.rocm_bf16x3_router_gemm_dispatch(x, w, split)
assert out.shape == (8192, 128)
assert out.dtype == torch.float32
def test_rocm_bf16x3_is_reachable_for_low_m_shapes(monkeypatch):
"""The gfx950 low-M fp32 tier claims these shapes too and returns
unconditionally, so bf16x3 is only reachable if checked first."""
gate = _make_bf16x3_gate(
monkeypatch, input_size=6144, output_size=128, on_gfx950=True
)
assert gate.allow_fp32_router_gemm
assert gate.allow_rocm_bf16x3_router_gemm
source = inspect.getsource(type(gate).forward)
assert source.index("allow_rocm_bf16x3_router_gemm") < source.index(
"allow_fp32_router_gemm"
)